tusd/docker/load-tests/2_plot_resource_usage.py

#!/usr/bin/env python3

import json
import re
import matplotlib.pyplot as plt

snapshots = []

with open("./resource-usage-log.txt") as file:
  current_snapshot = None
  for line in file:
    # The lines might contain the reset characters before the actual JSON.
    # This means that the entire resources for the current time have been
    # written out, so we add the latest snapshot to our list and continue
    # reading the next entries.
    first_backet = line.find("{")
    if first_backet == -1:
      continue

    if first_backet != 0:
      if current_snapshot is not None:
        snapshots.append(current_snapshot)

      current_snapshot = []
      line = line[first_backet:]

    current_snapshot.append(json.loads(line))

def parse_percentage(string):
  return float(string.strip('%'))

units = {"B": 1, "kB": 10**3, "MB": 10**6, "GB": 10**9, "TB": 10**12,
         "KiB": 2**10, "MiB": 2**20, "GiB": 2**30, "TiB": 2**40}

def parse_byte_size(size):
  number, unit = re.findall(r'([0-9\.]+)([A-Za-z]+)', size)[0]
  return int(float(number)*units[unit])

def parse_two_bytes(string):
  str1, str2 = string.split("/")
  return parse_byte_size(str1), parse_byte_size(str2)

s3_cpu = []
s3_mem = []
tusd_cpu = []
tusd_mem = []
tusd_net = []
uploader_cpu = []
uploader_mem = []
uploader_net = []
timestamp = []

for (i, snapshot) in enumerate(snapshots):
  a_s3_cpu = None
  a_s3_mem = None
  a_tusd_cpu = None
  a_tusd_mem = None
  a_tusd_net = None
  a_uploader_cpu = None
  a_uploader_mem = None
  a_uploader_net = None

  for entry in snapshot:
    if entry["Name"] == "load-tests-tusd-1":
      a_tusd_cpu = parse_percentage(entry["CPUPerc"])
      a_tusd_mem = parse_two_bytes(entry["MemUsage"])[0]
      a_tusd_net = parse_two_bytes(entry["NetIO"])[0]
    elif entry["Name"] == "load-tests-s3-1":
      a_s3_cpu = parse_percentage(entry["CPUPerc"])
      a_s3_mem = parse_two_bytes(entry["MemUsage"])[0]
    elif entry["Name"] == "load-tests-uploader-1":
      a_uploader_cpu = parse_percentage(entry["CPUPerc"])
      a_uploader_mem = parse_two_bytes(entry["MemUsage"])[0]
      a_uploader_net = parse_two_bytes(entry["NetIO"])[1]

  s3_cpu.append(a_s3_cpu)
  s3_mem.append(a_s3_mem)
  tusd_cpu.append(a_tusd_cpu)
  tusd_mem.append(a_tusd_mem)
  tusd_net.append(a_tusd_net)
  uploader_cpu.append(a_uploader_cpu)
  uploader_mem.append(a_uploader_mem)
  uploader_net.append(a_uploader_net)

  # The docker stats command is hard coded to output stats every 500ms:
  # https://github.com/docker/cli/blob/81c68913e4c2cb058b5a9fd5972e2989d9915b2c/cli/command/container/stats.go#L223
  timestamp.append(0.5 * i)

fig, axs = plt.subplots(3, 3, sharex=True, sharey='row')
axs[0, 0].plot(timestamp, tusd_cpu)
axs[0, 0].set_title('tusd CPU percentage')
axs[0, 0].set(ylabel='CPU perc', xlabel='time')
axs[0, 1].plot(timestamp, s3_cpu)
axs[0, 1].set_title('s3 CPU percentage')
axs[0, 1].set(ylabel='CPU perc', xlabel='time')
axs[0, 2].plot(timestamp, uploader_cpu)
axs[0, 2].set_title('uploader CPU percentage')
axs[0, 2].set(ylabel='CPU perc', xlabel='time')

axs[1, 0].plot(timestamp, tusd_mem)
axs[1, 0].set_title('tusd memory usage')
axs[1, 0].set(ylabel='mem perc', xlabel='time')
axs[1, 1].plot(timestamp, s3_mem)
axs[1, 1].set_title('s3 memory usage')
axs[1, 1].set(ylabel='mem perc', xlabel='time')
axs[1, 2].plot(timestamp, uploader_mem)
axs[1, 2].set_title('uploader memory usage')
axs[1, 2].set(ylabel='mem perc', xlabel='time')

axs[2, 0].plot(timestamp, tusd_net)
axs[2, 0].set_title('tusd network input')
axs[2, 0].set(ylabel='total volume', xlabel='time')
axs[2, 1].axis('off')
axs[2, 2].plot(timestamp, uploader_net)
axs[2, 2].set_title('uploader network output')
axs[2, 2].set(ylabel='total volume', xlabel='time')


# Hide x labels and tick labels for top plots and y ticks for right plots.
for ax in axs.flat:
    ax.label_outer()

plt.show()