OLD | NEW |
1 # Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. | 1 # Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. |
2 # | 2 # |
3 # Use of this source code is governed by a BSD-style license | 3 # Use of this source code is governed by a BSD-style license |
4 # that can be found in the LICENSE file in the root of the source | 4 # that can be found in the LICENSE file in the root of the source |
5 # tree. An additional intellectual property rights grant can be found | 5 # tree. An additional intellectual property rights grant can be found |
6 # in the file PATENTS. All contributing project authors may | 6 # in the file PATENTS. All contributing project authors may |
7 # be found in the AUTHORS file in the root of the source tree. | 7 # be found in the AUTHORS file in the root of the source tree. |
8 | 8 |
9 """Plots statistics from WebRTC integration test logs. | 9 """Plots statistics from WebRTC integration test logs. |
10 | 10 |
11 Usage: $ python plot_webrtc_test_logs.py filename.txt | 11 Usage: $ python plot_webrtc_test_logs.py filename.txt |
12 """ | 12 """ |
13 | 13 |
14 import numpy | 14 import numpy |
15 import sys | 15 import sys |
16 import re | 16 import re |
17 | 17 |
18 import matplotlib.pyplot as plt | 18 import matplotlib.pyplot as plt |
19 | 19 |
20 # Log events. | 20 # Log events. |
21 EVENT_START = 'RUN ] CodecSettings/PlotVideoProcessorIntegrationTest.' | 21 EVENT_START = 'RUN ] CodecSettings/PlotVideoProcessorIntegrationTest.' |
22 EVENT_END = 'OK ] CodecSettings/PlotVideoProcessorIntegrationTest.' | 22 EVENT_END = 'OK ] CodecSettings/PlotVideoProcessorIntegrationTest.' |
23 | 23 |
24 # Metrics to plot, tuple: (name to parse in file, label to use when plotting). | 24 # Metrics to plot, tuple: (name to parse in file, label to use when plotting). |
25 BITRATE = ('Target Bitrate', 'bitrate (kbps)') | 25 BITRATE = ('Target Bitrate', 'target bitrate (kbps)') |
26 WIDTH = ('Width', 'width') | 26 WIDTH = ('Width', 'width') |
27 HEIGHT = ('Height', 'height') | 27 HEIGHT = ('Height', 'height') |
28 FILENAME = ('Filename', 'clip') | 28 FILENAME = ('Filename', 'clip') |
29 CODEC_TYPE = ('Codec type', 'Codec') | 29 CODEC_TYPE = ('Codec type', 'Codec') |
30 ENCODER_IMPLEMENTATION_NAME = ('Encoder implementation name', 'enc name') | 30 ENCODER_IMPLEMENTATION_NAME = ('Encoder implementation name', 'enc name') |
31 DECODER_IMPLEMENTATION_NAME = ('Decoder implementation name', 'dec name') | 31 DECODER_IMPLEMENTATION_NAME = ('Decoder implementation name', 'dec name') |
32 NUM_FRAMES = ('Total # of frames', 'num frames') | 32 CODEC_IMPLEMENTATION_NAME = ('Codec implementation name', 'codec name') |
33 CORES = ('#CPU cores used', 'CPU cores used') | 33 CORES = ('#CPU cores used', 'CPU cores used') |
34 DENOISING = ('Denoising', 'denoising') | 34 DENOISING = ('Denoising', 'denoising') |
35 RESILIENCE = ('Resilience', 'resilience') | 35 RESILIENCE = ('Resilience', 'resilience') |
36 ERROR_CONCEALMENT = ('Error concealment', 'error concealment') | 36 ERROR_CONCEALMENT = ('Error concealment', 'error concealment') |
37 QP = ('Average QP', 'avg QP') | 37 QP = ('Average QP', 'avg QP') |
38 PSNR = ('PSNR avg', 'PSNR (dB)') | 38 PSNR = ('PSNR avg', 'PSNR (dB)') |
39 SSIM = ('SSIM avg', 'SSIM') | 39 SSIM = ('SSIM avg', 'SSIM') |
40 ENC_BITRATE = ('Encoding bitrate', 'encoded bitrate (kbps)') | 40 ENC_BITRATE = ('Encoding bitrate', 'encoded bitrate (kbps)') |
41 FRAMERATE = ('Frame rate', 'fps') | 41 FRAMERATE = ('Frame rate', 'fps') |
| 42 NUM_FRAMES = ('Number of processed frames', 'num frames') |
42 NUM_DROPPED_FRAMES = ('Number of dropped frames', 'num dropped frames') | 43 NUM_DROPPED_FRAMES = ('Number of dropped frames', 'num dropped frames') |
43 NUM_FRAMES_TO_TARGET = ('Number of frames to approach target rate', | 44 NUM_FRAMES_TO_TARGET = ('Number of frames to approach target rate', |
44 'frames to reach target rate') | 45 'frames to reach target rate') |
45 ENCODE_TIME = ('Encoding time', 'encode time (us)') | 46 ENCODE_TIME = ('Encoding time', 'encode time (us)') |
46 ENCODE_TIME_AVG = ('Encoding time', 'encode time (us) avg') | 47 ENCODE_TIME_AVG = ('Encoding time', 'encode time (us) avg') |
47 DECODE_TIME = ('Decoding time', 'decode time (us)') | 48 DECODE_TIME = ('Decoding time', 'decode time (us)') |
48 DECODE_TIME_AVG = ('Decoding time', 'decode time (us) avg') | 49 DECODE_TIME_AVG = ('Decoding time', 'decode time (us) avg') |
49 FRAME_SIZE = ('Frame sizes', 'frame size (bytes)') | 50 FRAME_SIZE = ('Frame sizes', 'frame size (bytes)') |
50 FRAME_SIZE_AVG = ('Frame sizes', 'frame size (bytes) avg') | 51 FRAME_SIZE_AVG = ('Frame sizes', 'frame size (bytes) avg') |
51 AVG_KEY_FRAME_SIZE = ('Average key frame size', 'avg key frame size (bytes)') | 52 AVG_KEY_FRAME_SIZE = ('Average key frame size', 'avg key frame size (bytes)') |
(...skipping 19 matching lines...) Expand all Loading... |
71 RESILIENCE, | 72 RESILIENCE, |
72 ERROR_CONCEALMENT, | 73 ERROR_CONCEALMENT, |
73 BITRATE, # TODO(asapersson): Needs to be last. | 74 BITRATE, # TODO(asapersson): Needs to be last. |
74 ] | 75 ] |
75 | 76 |
76 # Settings, options for subplots. | 77 # Settings, options for subplots. |
77 SUBPLOT_SETTINGS = [ | 78 SUBPLOT_SETTINGS = [ |
78 CODEC_TYPE, | 79 CODEC_TYPE, |
79 ENCODER_IMPLEMENTATION_NAME, | 80 ENCODER_IMPLEMENTATION_NAME, |
80 DECODER_IMPLEMENTATION_NAME, | 81 DECODER_IMPLEMENTATION_NAME, |
| 82 CODEC_IMPLEMENTATION_NAME, |
81 ] + X_SETTINGS | 83 ] + X_SETTINGS |
82 | 84 |
83 # Results. | 85 # Results. |
84 RESULTS = [ | 86 RESULTS = [ |
85 PSNR, | 87 PSNR, |
86 SSIM, | 88 SSIM, |
87 ENC_BITRATE, | 89 ENC_BITRATE, |
88 NUM_DROPPED_FRAMES, | 90 NUM_DROPPED_FRAMES, |
89 NUM_FRAMES_TO_TARGET, | 91 NUM_FRAMES_TO_TARGET, |
90 ENCODE_TIME_AVG, | 92 ENCODE_TIME_AVG, |
91 DECODE_TIME_AVG, | 93 DECODE_TIME_AVG, |
92 QP, | 94 QP, |
93 AVG_KEY_FRAME_SIZE, | 95 AVG_KEY_FRAME_SIZE, |
94 AVG_NON_KEY_FRAME_SIZE, | 96 AVG_NON_KEY_FRAME_SIZE, |
95 ] | 97 ] |
96 | 98 |
97 METRICS_TO_PARSE = SETTINGS + SUBPLOT_SETTINGS + RESULTS | 99 METRICS_TO_PARSE = SETTINGS + SUBPLOT_SETTINGS + RESULTS |
98 | 100 |
99 Y_METRICS = [res[1] for res in RESULTS] | 101 Y_METRICS = [res[1] for res in RESULTS] |
100 | 102 |
101 # Parameters for plotting. | 103 # Parameters for plotting. |
102 FIG_SIZE_SCALE_FACTOR_X = 2 | 104 FIG_SIZE_SCALE_FACTOR_X = 1.6 |
103 FIG_SIZE_SCALE_FACTOR_Y = 2.8 | 105 FIG_SIZE_SCALE_FACTOR_Y = 1.8 |
104 GRID_COLOR = [0.45, 0.45, 0.45] | 106 GRID_COLOR = [0.45, 0.45, 0.45] |
105 | 107 |
106 | 108 |
107 def ParseSetting(filename, setting): | 109 def ParseSetting(filename, setting): |
108 """Parses setting from file. | 110 """Parses setting from file. |
109 | 111 |
110 Args: | 112 Args: |
111 filename: The name of the file. | 113 filename: The name of the file. |
112 setting: Name of setting to parse (e.g. width). | 114 setting: Name of setting to parse (e.g. width). |
113 | 115 |
(...skipping 164 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
278 "CPU cores used: 1.0": { | 280 "CPU cores used: 1.0": { |
279 "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], | 281 "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], |
280 "bitrate (kbps)": [50, 100, 300, 500, 1000] | 282 "bitrate (kbps)": [50, 100, 300, 500, 1000] |
281 }, | 283 }, |
282 "CPU cores used: 2.0": { | 284 "CPU cores used: 2.0": { |
283 "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], | 285 "PSNR (dB)": [25.546029, 29.465518, 34.723535, 36.428493, 38.686551], |
284 "bitrate (kbps)": [50, 100, 300, 500, 1000] | 286 "bitrate (kbps)": [50, 100, 300, 500, 1000] |
285 }, | 287 }, |
286 } | 288 } |
287 """ | 289 """ |
288 for key in metrics: | 290 for key in sorted(metrics): |
289 data = metrics[key] | 291 data = metrics[key] |
290 if y_metric not in data: | 292 if y_metric not in data: |
291 print "Failed to find metric: %s" % y_metric | 293 print "Failed to find metric: %s" % y_metric |
292 continue | 294 continue |
293 | 295 |
294 y = numpy.array(data[y_metric]) | 296 y = numpy.array(data[y_metric]) |
295 x = numpy.array(data[x_metric]) | 297 x = numpy.array(data[x_metric]) |
296 if len(y) != len(x): | 298 if len(y) != len(x): |
297 print "Length mismatch for %s, %s" % (y, x) | 299 print "Length mismatch for %s, %s" % (y, x) |
298 continue | 300 continue |
299 | 301 |
300 label = y_metric + ' - ' + str(key) | 302 label = y_metric + ' - ' + str(key) |
301 | 303 |
302 plt.plot(x, y, label=label, linewidth=1.5, marker='o', markersize=5, | 304 plt.plot(x, y, label=label, linewidth=1.5, marker='o', markersize=5, |
303 markeredgewidth=0.0) | 305 markeredgewidth=0.0) |
304 | 306 |
305 | 307 |
306 def PlotFigure(settings, y_metrics, x_metric, metrics, title): | 308 def PlotFigure(settings, y_metrics, x_metric, metrics, title): |
307 """Plots metrics in y_metrics list. One figure is plotted and each entry | 309 """Plots metrics in y_metrics list. One figure is plotted and each entry |
308 in the list is plotted in a subplot (and sorted per settings). | 310 in the list is plotted in a subplot (and sorted per settings). |
309 | 311 |
310 For example: | 312 For example: |
311 settings = ['width: 128.0', 'width: 160.0']. Sort subplot per setting. | 313 settings = ['width: 128.0', 'width: 160.0']. Sort subplot per setting. |
312 y_metrics = ['PSNR (dB)', 'PSNR (dB)']. Metric to plot per subplot. | 314 y_metrics = ['PSNR (dB)', 'PSNR (dB)']. Metric to plot per subplot. |
313 x_metric = 'bitrate (kbps)' | 315 x_metric = 'bitrate (kbps)' |
314 | 316 |
315 """ | 317 """ |
316 | 318 |
317 plt.figure() | 319 plt.figure() |
318 plt.suptitle(title, fontsize='small', fontweight='bold') | 320 plt.suptitle(title, fontsize='large', fontweight='bold') |
| 321 settings.sort() |
319 rows = len(settings) | 322 rows = len(settings) |
320 cols = 1 | 323 cols = 1 |
321 pos = 1 | 324 pos = 1 |
322 while pos <= rows: | 325 while pos <= rows: |
323 plt.rc('grid', color=GRID_COLOR) | 326 plt.rc('grid', color=GRID_COLOR) |
324 ax = plt.subplot(rows, cols, pos) | 327 ax = plt.subplot(rows, cols, pos) |
325 plt.grid() | 328 plt.grid() |
326 plt.setp(ax.get_xticklabels(), visible=(pos == rows), fontsize='small') | 329 plt.setp(ax.get_xticklabels(), visible=(pos == rows), fontsize='large') |
327 plt.setp(ax.get_yticklabels(), fontsize='small') | 330 plt.setp(ax.get_yticklabels(), fontsize='large') |
328 setting = settings[pos - 1] | 331 setting = settings[pos - 1] |
329 Plot(y_metrics[pos - 1], x_metric, metrics[setting]) | 332 Plot(y_metrics[pos - 1], x_metric, metrics[setting]) |
330 plt.title(setting, fontsize='x-small') | 333 if setting.startswith(WIDTH[1]): |
331 plt.legend(fontsize='xx-small') | 334 plt.title(setting, fontsize='medium') |
| 335 plt.legend(fontsize='large', loc='best') |
332 pos += 1 | 336 pos += 1 |
333 | 337 |
334 plt.xlabel(x_metric, fontsize='small') | 338 plt.xlabel(x_metric, fontsize='large') |
335 plt.subplots_adjust(left=0.04, right=0.98, bottom=0.04, top=0.96, hspace=0.1) | 339 plt.subplots_adjust(left=0.06, right=0.98, bottom=0.05, top=0.94, hspace=0.08) |
336 | 340 |
337 | 341 |
338 def GetTitle(filename): | 342 def GetTitle(filename, setting): |
339 title = '' | 343 title = '' |
340 codec_types = ParseSetting(filename, CODEC_TYPE[1]) | 344 if setting != CODEC_IMPLEMENTATION_NAME[1] and setting != CODEC_TYPE[1]: |
341 for i in range(0, len(codec_types)): | 345 codec_types = ParseSetting(filename, CODEC_TYPE[1]) |
342 title += codec_types[i] + ', ' | 346 for i in range(0, len(codec_types)): |
| 347 title += codec_types[i] + ', ' |
343 | 348 |
344 cores = ParseSetting(filename, CORES[1]) | 349 if setting != CORES[1]: |
345 for i in range(0, len(cores)): | 350 cores = ParseSetting(filename, CORES[1]) |
346 title += cores[i].split('.')[0] + ', ' | 351 for i in range(0, len(cores)): |
| 352 title += cores[i].split('.')[0] + ', ' |
347 | 353 |
348 framerate = ParseSetting(filename, FRAMERATE[1]) | 354 if setting != FRAMERATE[1]: |
349 for i in range(0, len(framerate)): | 355 framerate = ParseSetting(filename, FRAMERATE[1]) |
350 title += framerate[i].split('.')[0] + ', ' | 356 for i in range(0, len(framerate)): |
| 357 title += framerate[i].split('.')[0] + ', ' |
351 | 358 |
352 enc_names = ParseSetting(filename, ENCODER_IMPLEMENTATION_NAME[1]) | 359 if (setting != CODEC_IMPLEMENTATION_NAME[1] and |
353 for i in range(0, len(enc_names)): | 360 setting != ENCODER_IMPLEMENTATION_NAME[1]): |
354 title += enc_names[i] + ', ' | 361 enc_names = ParseSetting(filename, ENCODER_IMPLEMENTATION_NAME[1]) |
| 362 for i in range(0, len(enc_names)): |
| 363 title += enc_names[i] + ', ' |
355 | 364 |
356 dec_names = ParseSetting(filename, DECODER_IMPLEMENTATION_NAME[1]) | 365 if (setting != CODEC_IMPLEMENTATION_NAME[1] and |
357 for i in range(0, len(dec_names)): | 366 setting != DECODER_IMPLEMENTATION_NAME[1]): |
358 title += dec_names[i] + ', ' | 367 dec_names = ParseSetting(filename, DECODER_IMPLEMENTATION_NAME[1]) |
| 368 for i in range(0, len(dec_names)): |
| 369 title += dec_names[i] + ', ' |
359 | 370 |
360 filenames = ParseSetting(filename, FILENAME[1]) | 371 filenames = ParseSetting(filename, FILENAME[1]) |
361 title += filenames[0].split('_')[0] | 372 title += filenames[0].split('_')[0] |
362 | 373 |
363 num_frames = ParseSetting(filename, NUM_FRAMES[1]) | 374 num_frames = ParseSetting(filename, NUM_FRAMES[1]) |
364 for i in range(0, len(num_frames)): | 375 for i in range(0, len(num_frames)): |
365 title += ' (' + num_frames[i].split('.')[0] + ')' | 376 title += ' (' + num_frames[i].split('.')[0] + ')' |
366 | 377 |
367 return title | 378 return title |
368 | 379 |
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
423 x_metric = X_SETTINGS[idx][1] | 434 x_metric = X_SETTINGS[idx][1] |
424 | 435 |
425 metrics = ParseMetrics(filename, setting1, setting2) | 436 metrics = ParseMetrics(filename, setting1, setting2) |
426 | 437 |
427 # Stretch fig size. | 438 # Stretch fig size. |
428 figsize = plt.rcParams["figure.figsize"] | 439 figsize = plt.rcParams["figure.figsize"] |
429 figsize[0] *= FIG_SIZE_SCALE_FACTOR_X | 440 figsize[0] *= FIG_SIZE_SCALE_FACTOR_X |
430 figsize[1] *= FIG_SIZE_SCALE_FACTOR_Y | 441 figsize[1] *= FIG_SIZE_SCALE_FACTOR_Y |
431 plt.rcParams["figure.figsize"] = figsize | 442 plt.rcParams["figure.figsize"] = figsize |
432 | 443 |
433 PlotFigure(sub_keys, y_metrics, x_metric, metrics, GetTitle(filename)) | 444 PlotFigure(sub_keys, y_metrics, x_metric, metrics, |
| 445 GetTitle(filename, setting2)) |
434 | 446 |
435 plt.show() | 447 plt.show() |
436 | 448 |
437 | 449 |
438 if __name__ == '__main__': | 450 if __name__ == '__main__': |
439 main() | 451 main() |
OLD | NEW |