Add script for plotting statistics from webrtc integration test logs.

webrtc/modules/video_coding/codecs/test/plot_webrtc_test_logs.py

+#  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+#
+#  Use of this source code is governed by a BSD-style license
+#  that can be found in the LICENSE file in the root of the source
+#  tree. An additional intellectual property rights grant can be found
+#  in the file PATENTS.  All contributing project authors may
+#  be found in the AUTHORS file in the root of the source tree.
+
+"""Plots statistics from WebRTC integration test logs.
+
+Usage: $ python plot_webrtc_test_logs.py filename.txt
+"""
+
+import numpy
+import sys
+import re
+
+import matplotlib.pyplot as plt
+
+# Log events.
+EVENT_START = 'RUN      ] TestWithBitrate/PlotVideoProcessorIntegrationTest.'
+EVENT_END = 'OK ] TestWithBitrate/PlotVideoProcessorIntegrationTest.'
+
+# Metrics to plot, tuple: (name to parse in file, label to use when plotting).
+BITRATE = ('Target Bitrate', 'bitrate (kbps)')
+WIDTH = ('Width', 'width')
+HEIGHT = ('Height', 'height')
+FILENAME = ('Filename', 'clip')
+CODEC_TYPE = ('Codec type', 'Codec')
+ENCODER_IMPLEMENTATION_NAME = ('Encoder implementation name', 'enc name')
+DECODER_IMPLEMENTATION_NAME = ('Decoder implementation name', 'dec name')
+NUM_FRAMES = ('Total # of frames', 'num frames')
+CORES = ('#CPU cores used', 'cores')
+DENOISING = ('Denoising', 'denoising')
+RESILIENCE = ('Resilience', 'resilience')
+ERROR_CONCEALMENT = ('Error concealment', 'error concealment')
+PSNR = ('PSNR avg', 'PSNR (dB)')
+SSIM = ('SSIM avg', 'SSIM')
+ENC_BITRATE = ('Encoding bitrate', 'encoded bitrate (kbps)')
+FRAMERATE = ('Frame rate', 'fps')
+NUM_DROPPED_FRAMES = ('Number of dropped frames', 'num dropped frames')
+NUM_FRAMES_TO_TARGET = ('Number of frames to approach target rate',
+                        'frames to reach target rate')
+ENCODE_TIME = ('Encoding time', 'encode time (us)')
+ENCODE_TIME_AVG = ('Encoding time', 'encode time (us) avg')
+DECODE_TIME = ('Decoding time', 'decode time (us)')
+DECODE_TIME_AVG = ('Decoding time', 'decode time (us) avg')
+FRAME_SIZE = ('Frame sizes', 'frame size (bytes)')
+FRAME_SIZE_AVG = ('Frame sizes', 'frame size (bytes) avg')
+AVG_KEY_FRAME_SIZE = ('Average key frame size', 'avg key frame size (bytes)')
+AVG_NON_KEY_FRAME_SIZE = ('Average non-key frame size',
+                          'avg non-key frame size (bytes)')
+
+# Settings.
+SETTINGS = [
+  WIDTH,
+  HEIGHT,
+  FILENAME,
+  CODEC_TYPE,
+  NUM_FRAMES,
+  ENCODER_IMPLEMENTATION_NAME,
+  DECODER_IMPLEMENTATION_NAME,
+  ENCODE_TIME,
+  DECODE_TIME,
+  FRAME_SIZE,
+]
+
+# Settings, options for x-axis.
+X_SETTINGS = [
+  CORES,
+  FRAMERATE,
+  DENOISING,
+  RESILIENCE,
+  ERROR_CONCEALMENT,
+  BITRATE,  # TODO(asapersson): Needs to be last.
+]
+
+# Results.
+RESULTS = [
+  PSNR,
+  SSIM,
+  ENC_BITRATE,
+  NUM_DROPPED_FRAMES,
+  NUM_FRAMES_TO_TARGET,
+  ENCODE_TIME_AVG,
+  DECODE_TIME_AVG,
+  AVG_KEY_FRAME_SIZE,
+  AVG_NON_KEY_FRAME_SIZE,
+]
+
+METRICS_TO_PARSE = []

[brandtr, 2017/02/09 13:32:42]

Could this be replaced by list concatenation?

METRICS_TO_PARSE = SETTINGS + X_SETTINGS + RESULTS

[åsapersson, 2017/02/09 16:39:37]

Done.

+for k in SETTINGS:
+  METRICS_TO_PARSE.append(k)
+for x_setting in X_SETTINGS:
+  METRICS_TO_PARSE.append(x_setting)
+for result in RESULTS:
+  METRICS_TO_PARSE.append(result)
+
+Y_METRICS = []

[brandtr, 2017/02/09 13:32:42]

And here a list comprehension?

Y_METRICS = [ res[1] for res in RESULTS ]

[åsapersson, 2017/02/09 16:39:37]

Done.

+for res in RESULTS:
+  Y_METRICS.append(res[1])
+
+
+# Parameters for plotting.
+FIG_SIZE_SCALE_FACTOR_Y = 2.8
+FIG_SIZE_SCALE_FACTOR_X = 2

[brandtr, 2017/02/09 13:32:42]

Place FIG_SIZE_SCALE_FACTOR_X before the corresponding Y? For consistency with
definitions above.

[åsapersson, 2017/02/09 16:39:37]

Done.

+GRID_COLOR = [0.45, 0.45, 0.45]
+
+
+def ToString(input_list):

[brandtr, 2017/02/09 13:32:42]

Move ToString and ToStringWithoutMetric closer to where they are being used,
i.e., main.

[åsapersson, 2017/02/09 16:39:37]

Done.

+  return ToStringWithoutMetric(input_list, ('', ''))
+
+
+def ToStringWithoutMetric(input_list, metric):
+  i = 1
+  output_str = ""
+  for m in input_list:
+    if m != metric:
+      output_str = output_str + ("%s. %s\n" % (i, m[1]))
+      i += 1
+  return output_str
+
+
+def ParseSetting(filename, setting):
+  """Parses setting from file.
+
+  Args:
+    filename: The name of the file.
+    setting: Name of setting to parse (e.g. width).
+
+  Returns:
+    A list holding parsed settings, e.g. ['width: 128.0', 'width: 160.0'] """
+
+  settings = []
+
+  f = open(filename)
+  while True:
+    line = f.readline()
+    if not line:
+      break
+    if re.search(r'%s' % EVENT_START, line):
+      # Parse event.
+      parsed = {}
+      while True:
+        line = f.readline()
+        if not line:
+          break
+        if re.search(r'%s' % EVENT_END, line):
+          # Add parsed setting to list.
+          if setting in parsed:
+            s = setting + ': ' + str(parsed[setting])
+            if s not in settings:
+              settings.append(s)
+          break
+
+        TryFindMetric(parsed, line, f)
+
+  f.close()
+  return settings
+
+
+def ParseMetrics(filename, setting1, setting2):
+  """Parses metrics from file.
+
+  Args:
+    filename: The name of the file.
+    setting1: First setting for sorting metrics (e.g. width).
+    setting2: Second setting for sorting metrics (e.g. cores).
+
+  Returns:
+    A dictionary holding parsed metrics.
+
+  For example:
+    metrics[key1][key2][measurement]
+
+  metrics = {
+  "width: 352": {
+    "cores: 1.0": {
+      "encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642],
+      "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
+      "bitrate (kbps)": [50, 100, 300, 500, 1000]
+    },
+    "cores: 2.0": {
+      "encode time (us)": [0.718005, 0.806925, 0.909726, 0.931835, 0.953642],
+      "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
+      "bitrate (kbps)": [50, 100, 300, 500, 1000]
+    },
+  },
+  "width: 176": {
+    "cores: 1.0": {
+      "encode time (us)": [0.857897, 0.91608, 0.959173, 0.971116, 0.980961],
+      "PSNR (dB)": [30.243646, 33.375592, 37.574387, 39.42184, 41.437897],
+      "bitrate (kbps)": [50, 100, 300, 500, 1000]
+    },
+  }
+  } """
+
+  metrics = {}
+
+  # Parse events.
+  f = open(filename)
+  while True:
+    line = f.readline()
+    if not line:
+      break
+    if re.search(r'%s' % EVENT_START, line):
+      # Parse event.
+      parsed = {}
+      while True:
+        line = f.readline()
+        if not line:
+          break
+        if re.search(r'%s' % EVENT_END, line):
+          # Add parsed values to metrics.
+          key1 = setting1 + ': ' + str(parsed[setting1])
+          key2 = setting2 + ': ' + str(parsed[setting2])
+          if key1 not in metrics:
+            metrics[key1] = {}
+          if key2 not in metrics[key1]:
+            metrics[key1][key2] = {}
+
+          for label in parsed:
+            if label not in metrics[key1][key2]:
+              metrics[key1][key2][label] = []
+            metrics[key1][key2][label].append(parsed[label])
+
+          break
+
+        TryFindMetric(parsed, line, f)
+
+  f.close()
+  return metrics
+
+
+def TryFindMetric(parsed, line, f):
+  for metric in METRICS_TO_PARSE:
+    name = metric[0]
+    label = metric[1]
+    if re.search(r'%s' % name, line):
+      found, value = GetMetric(name, line)
+      if not found:
+        # TODO(asapersson): Change format.
+        # Try find min, max, average stats.
+        found, minimum = GetMetric("Min", f.readline())
+        if not found:
+          return
+        found, maximum = GetMetric("Max", f.readline())
+        if not found:
+          return
+        found, average = GetMetric("Average", f.readline())
+        if not found:
+          return
+
+        parsed[label + ' min'] = minimum
+        parsed[label + ' max'] = maximum
+        parsed[label + ' avg'] = average
+
+      parsed[label] = value
+      return
+
+
+def GetMetric(name, string):
+  # Float.
+  pattern = r'%s\s*[:=]\s*([+-]?\d+\.*\d*)' % name

[brandtr, 2017/02/09 13:32:42]

I don't fully understand this regex. Maybe a short example could be added, that
shows a typical line matching this regex?

[åsapersson, 2017/02/09 16:39:37]

Done.

+  m = re.search(r'%s' % pattern, string)
+  if m is not None:
+    return StringToFloat(m.group(1))
+
+  # Alphanumeric characters.
+  pattern = r'%s\s*[:=]\s*(\w+)' % name

[brandtr, 2017/02/09 13:32:42]

Same thing here.

[åsapersson, 2017/02/09 16:39:37]

Done.

+  m = re.search(r'%s' % pattern, string)
+  if m is not None:
+    return True, m.group(1)
+
+  return False, -1
+
+
+def StringToFloat(value):
+  try:
+    value = float(value)
+  except ValueError:
+    print "Not a float, skipped %s" % value
+    return False, -1
+
+  return True, value
+
+
+def Plot(y_metric, x_metric, metrics):
+  """Plots y_metric vs x_metric per key in metrics.
+
+  For example:
+    y_metric = 'PSNR (dB)'
+    x_metric = 'bitrate (kbps)'
+    metrics = {
+      "cores: 1.0": {
+        "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
+        "bitrate (kbps)": [50, 100, 300, 500, 1000]
+      },
+      "cores: 2.0": {
+        "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551],
+        "bitrate (kbps)": [50, 100, 300, 500, 1000]
+      },
+    }
+    """
+  for key in metrics:
+    data = metrics[key]
+    if y_metric not in data:
+      print "Failed to find metric: %s" % y_metric
+      continue
+
+    y = numpy.array(data[y_metric])
+    x = numpy.array(data[x_metric])
+    if len(y) != len(x):
+      print "Length mismatch for %s, %s" % (y, x)
+      continue
+
+    label = y_metric + ' - ' + str(key)
+
+    plt.plot(x, y, label=label, linewidth=1.5, marker='o', markersize=5,
+             markeredgewidth=0.0)
+
+
+def PlotFigure(settings, y_metrics, x_metric, metrics, title):
+  """Plots metrics in y_metrics list. One figure is plotted and each entry
+  in the list is plotted in a subplot (and sorted per settings).
+
+  For example:
+    settings = ['width: 128.0', 'width: 160.0']. Sort subplot per setting.
+    y_metrics = ['PSNR (dB)', 'PSNR (dB)']. Metric to plot per subplot.
+    x_metric = 'bitrate (kbps)'
+
+  """
+
+  plt.figure()
+  plt.suptitle(title, fontsize='small', fontweight='bold')
+  rows = len(settings)
+  cols = 1
+  pos = 1
+  while pos <= rows:
+    plt.rc('grid', color=GRID_COLOR)
+    ax = plt.subplot(rows, cols, pos)
+    plt.grid()
+    plt.setp(ax.get_xticklabels(), visible=(pos == rows), fontsize='small')
+    plt.setp(ax.get_yticklabels(), fontsize='small')
+    setting = settings[pos - 1]
+    Plot(y_metrics[pos - 1], x_metric, metrics[setting])
+    plt.title(setting, fontsize='x-small')
+    plt.legend(fontsize='xx-small')
+    pos += 1
+
+  plt.xlabel(x_metric, fontsize='small')
+  plt.subplots_adjust(left=0.04, right=0.98, bottom=0.04, top=0.96, hspace=0.1)
+
+
+def GetTitle(filename):
+  title = ''
+  codec_types = ParseSetting(filename, CODEC_TYPE[1])
+  for i in range(0, len(codec_types)):
+    title += codec_types[i] + ', '
+
+  framerate = ParseSetting(filename, FRAMERATE[1])
+  for i in range(0, len(framerate)):
+    title += framerate[i].split('.')[0] + ', '
+
+  enc_names = ParseSetting(filename, ENCODER_IMPLEMENTATION_NAME[1])
+  for i in range(0, len(enc_names)):
+    title += enc_names[i] + ', '
+
+  dec_names = ParseSetting(filename, DECODER_IMPLEMENTATION_NAME[1])
+  for i in range(0, len(dec_names)):
+    title += dec_names[i] + ', '
+
+  filenames = ParseSetting(filename, FILENAME[1])
+  title += filenames[0].split('_')[0]
+
+  num_frames = ParseSetting(filename, NUM_FRAMES[1])
+  for i in range(0, len(num_frames)):
+    title += ' (' + num_frames[i].split('.')[0] + ')'
+
+  return title
+
+
+def GetIdx(text_list):
+  return int(raw_input(text_list)) - 1
+
+
+def main():
+  filename = sys.argv[1]
+
+  # Setup.
+  idx_metric = GetIdx("Choose metric:\n0. All\n%s" % ToString(RESULTS))
+  if idx_metric == -1:
+    # Plot all metrics. One subplot for each metric.
+    # Per subplot: metric vs bitrate (per resolution).
+    cores = ParseSetting(filename, CORES[1])
+    setting1 = CORES[1]
+    setting2 = WIDTH[1]
+    sub_keys = [cores[0]] * len(Y_METRICS)
+    y_metrics = Y_METRICS
+    x_metric = BITRATE[1]
+  else:
+    resolutions = ParseSetting(filename, WIDTH[1])
+    idx = GetIdx("Select metric for x-axis:\n%s" % ToString(X_SETTINGS))
+    if X_SETTINGS[idx] == BITRATE:
+      idx = GetIdx("Plot per:\n%s" % ToStringWithoutMetric(X_SETTINGS, BITRATE))
+      idx_setting = METRICS_TO_PARSE.index(X_SETTINGS[idx])
+      # Plot one metric. One subplot for each resolution.
+      # Per subplot: metric vs bitrate (per setting).
+      setting1 = WIDTH[1]
+      setting2 = METRICS_TO_PARSE[idx_setting][1]
+      sub_keys = resolutions
+      y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys)
+      x_metric = BITRATE[1]
+    else:
+      # Plot one metric. One subplot for each resolution.
+      # Per subplot: metric vs setting (per bitrate).
+      setting1 = WIDTH[1]
+      setting2 = BITRATE[1]
+      sub_keys = resolutions
+      y_metrics = [RESULTS[idx_metric][1]] * len(sub_keys)
+      x_metric = X_SETTINGS[idx][1]
+
+  metrics = ParseMetrics(filename, setting1, setting2)
+
+  # Stretch fig size.
+  figsize = plt.rcParams["figure.figsize"]
+  figsize[0] *= FIG_SIZE_SCALE_FACTOR_X
+  figsize[1] *= FIG_SIZE_SCALE_FACTOR_Y
+  plt.rcParams["figure.figsize"] = figsize
+
+  PlotFigure(sub_keys, y_metrics, x_metric, metrics, GetTitle(filename))
+
+  plt.show()
+
+
+if __name__ == '__main__':
+  main()