Select Page

Overwatch Docker Swarm containers

by | Aug 21, 2020 | Docker, Infrastructure, Jq, S3 | 0 comments

  • Objective: Monitoring Docker Swarm containers
  • Tools: Bash, Jq, AWS S3, HTML, CSS, jQuery
  • Result: Magic!

Docker Swarm Services

Let’s assume we’re having a 3 nodes Docker Swarm configuration: 1 x Manager Node and 2 x Worker Nodes:

docker node ls

This Swarm has several services running:

docker service ls

Useful stats

We’ll use Jq to get stats in a more useful format:

/usr/bin/docker stats --no-stream --format "{"container":"{{ .Container }}","name":"{{ .Name }}","memory":"{{ .MemPerc }}","cpu":"{{ .CPUPerc }}"}" | /usr/bin/jq -s . | /usr/bin/jq 'sort_by(.name)' >> stats.json

Send them to the cloud

We’ll create a script to do this docker-stats.sh:

#!/usr/bin/env bash

/usr/bin/docker stats --no-stream --format "{"container":"{{ .Container }}","name":"{{ .Name }}","memory":"{{ .MemPerc }}","cpu":"{{ .CPUPerc }}"}" | /usr/bin/jq -s . | /usr/bin/jq 'sort_by(.name)' >> stats.json

/usr/local/bin/aws s3 cp stats.json s3://awesome-bucket/stats/production/manager.json --acl public-read

rm stats.json

And we’ll have this script running every minute by adding a simple line to our Crontab file:

* * * * * /home/ubuntu/docker-stats.sh

We’ll create scripts for Workers too.

Let there be light

We’ll create 3 files: index.html, styles.css and scripts.js to display these stats in useful manner:

<!doctype html>

<html lang="en">
<head>
  <meta charset="utf-8">

  <title>Overwatch</title>
  <meta name="description" content="Fepra - Overwatch">
  <meta name="author" content="Catalin Ilinca - catalin@techwizard.ro">

  <link href="https://fonts.googleapis.com/css2?family=PT+Mono&display=swap" rel="stylesheet">
  <link rel="stylesheet" href="styles.css">
  <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
</head>

<body>
  <h1>🕵️‍♂️ Overwatch<span id="refresh"> - refreshing...</span></h1>
  <h2>Production</h2>
  <div class="node-wrapper vertical">
    <h3>Manager <span class="node-ip">XXX.XXX.XXX.XXX</span></h3>
    <div id="production-manager" class="node"></div>
  </div>
  <div class="node-wrapper vertical">
    <h3>Worker #1 <span class="node-ip">XXX.XXX.XXX.XXX</span></h3>
    <div id="production-worker-1" class="node"></div>
  </div>
  <div class="node-wrapper vertical">
    <h3>Worker #2 <span class="node-ip">XXX.XXX.XXX.XXX</span></h3>
    <div id="production-worker-2" class="node"></div>
  </div>
  <script src="scripts.js"></script>
</body>
</html>
body {
  font-family: 'PT Mono', monospace;
}

h1 {
  font-size: 16px;
}

h2 {
  font-size: 14px;
}

h3 {
  position: absolute;
  margin-top: -20px;
  margin-left: -5px;
  padding: 2px 5px;
  font-size: 12px;
  background: #FFFFFF;
}

#refresh {
  display: none;
}

.node-wrapper {
  display: inline-flex;
  border: 1px solid #CCCCCC;
  margin: 10px 0;
  padding: 10px;
  width: 1392px;
  border-radius: 5px;
}

.node-wrapper.vertical {
  width: 230px;
}

.node-wrapper:hover {
  border: 1px solid #333333;
}

.node {
  display: flex;
  flex-wrap: wrap;
  font-size: 10px;
}

.node-ip {
  font-size: 12px;
  padding: 2px 5px;
  display: inline-flex;
  background: #999999;
  color: #FFFFFF;
  border-radius: 5px;
}

.container {
  width: 200px;
  height: 42px;
  margin: 10px 5px;
  border: 1px dashed #CCCCCC;
  padding: 10px;
  border-radius: 5px;
  cursor: pointer;
}

.container:hover {
  background: #EEEEEE;
}

.container-id, .container-name, .container-cpu, .container-memory {
  text-align: center;
  margin: 5px;
  padding: 5px;
}

.container-id {
  position: absolute;
  display: inline-flex;
  margin-top: -15px;
  padding: 2px 5px;
  background: #999999;
  color: #FFFFFF;
  border-radius: 5px;
}

.container-name {
  overflow: hidden;
  white-space: nowrap;
  text-overflow: ellipsis;
}

.container-cpu, .container-memory {
  display: inline-flex;
  width: 90px;
  align-items: center;
  justify-content: center;
  margin: 0;
}

.alert {
  color: #FFFFFF;
  background: #FC0107;
}

.warning {
  color: #FFFFFF;
  background: #FD8008;
}

.normal {
  color: #FFFFFF;
  background: #009F00;
}
$.strPad = function(i,l,s) {
  var o = i.toString();
  if (!s) { s = '0'; }
  while (o.length < l) {
    o = s + o;
  }
  return o;
};

function getStats(){
  $('#refresh').show();

  // Production Manager

  $.getJSON( "/stats/production/manager.json", function( data ) {
    var items = [];
    $.each( data, function(key, val) {
      cpuClass = 'normal';
      if (parseFloat(val.cpu, 10) > 65.00 && parseFloat(val.cpu, 10) < 90.00) {
        cpuClass = 'warning';
      } else if (parseFloat(val.cpu, 10) >= 90.00) {
        cpuClass = 'alert';
      }
      memoryClass = 'normal';
      if (parseFloat(val.memory, 10) > 65.00 && parseFloat(val.memory, 10) < 90.00) {
        memoryClass = 'warning';
      } else if (parseFloat(val.memory, 10) >= 90.00) {
        memoryClass = 'alert';
      }
      item = "<div class="container">";
      item += "<div class="container-id">" + val.container + "</div>";
      item += "<div class="container-name">" + val.name + "</div>";
      item += "<div class="container-cpu " + cpuClass + "">CPU: " + val.cpu + "</div>";
      item += "<div class="container-memory " + memoryClass + "">RAM: " + val.memory + "</div>";
      item += "</div>";

      items.push(item);
    });

    $('#production-manager').html(items.join(''))
  });

  // Production Worker #1

  $.getJSON( "/stats/production/worker-1.json", function( data ) {
    var items = [];
    $.each( data, function(key, val) {
      cpuClass = 'normal';
      if (parseFloat(val.cpu, 10) > 65.00 && parseFloat(val.cpu, 10) < 90.00) {
        cpuClass = 'warning';
      } else if (parseFloat(val.cpu, 10) >= 90.00) {
        cpuClass = 'alert';
      }
      memoryClass = 'normal';
      if (parseFloat(val.memory, 10) > 65.00 && parseFloat(val.memory, 10) < 90.00) {
        memoryClass = 'warning';
      } else if (parseFloat(val.memory, 10) >= 90.00) {
        memoryClass = 'alert';
      }
      item = "<div class="container">";
      item += "<div class="container-id">" + val.container + "</div>";
      item += "<div class="container-name">" + val.name + "</div>";
      item += "<div class="container-cpu " + cpuClass + "">CPU: " + val.cpu + "</div>";
      item += "<div class="container-memory " + memoryClass + "">RAM: " + val.memory + "</div>";
      item += "</div>";

      items.push(item);
    });

    $('#production-worker-1').html(items.join(''))
  });

  // Production Worker #2

  $.getJSON( "/stats/production/worker-2.json", function( data ) {
    var items = [];
    $.each( data, function(key, val) {
      cpuClass = 'normal';
      if (parseFloat(val.cpu, 10) > 65.00 && parseFloat(val.cpu, 10) < 90.00) {
        cpuClass = 'warning';
      } else if (parseFloat(val.cpu, 10) >= 90.00) {
        cpuClass = 'alert';
      }
      memoryClass = 'normal';
      if (parseFloat(val.memory, 10) > 65.00 && parseFloat(val.memory, 10) < 90.00) {
        memoryClass = 'warning';
      } else if (parseFloat(val.memory, 10) >= 90.00) {
        memoryClass = 'alert';
      }
      item = "<div class="container">";
      item += "<div class="container-id">" + val.container + "</div>";
      item += "<div class="container-name">" + val.name + "</div>";
      item += "<div class="container-cpu " + cpuClass + "">CPU: " + val.cpu + "</div>";
      item += "<div class="container-memory " + memoryClass + "">RAM: " + val.memory + "</div>";
      item += "</div>";

      items.push(item);
    });

    $('#production-worker-2').html(items.join(''))
  });

  $('#refresh').hide();

  setTimeout(getStats, 60000);
}

getStats();

Result

Leave a Reply