Add POLQA to low bandwidth audio test

webrtc/audio/test/low_bandwidth_audio_test.py

 #!/usr/bin/env python
 # Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
 #
 # Use of this source code is governed by a BSD-style license
 # that can be found in the LICENSE file in the root of the source
 # tree. An additional intellectual property rights grant can be found
 # in the file PATENTS.  All contributing project authors may
 # be found in the AUTHORS file in the root of the source tree.
 
 """
 This script is the wrapper that runs the low-bandwidth audio test.
 
 After running the test, post-process steps for calculating audio quality of the
 output files will be performed.
 """
 
 import argparse
+import collections
 import logging
 import os
 import re
 import shutil
 import subprocess
 import sys
 
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 SRC_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir,
@@ skipping 24 matching lines @@
   elif sys.platform == 'darwin':
     return 'mac'
   elif sys.platform.startswith('linux'):
     return 'linux'
 
 
 def _DownloadTools():
   tools_dir = os.path.join(SRC_DIR, 'tools-webrtc')
   toolchain_dir = os.path.join(tools_dir, 'audio_quality')
 
-  # Download pesq.
+  # Download PESQ and POLQA.
   download_script = os.path.join(tools_dir, 'download_tools.py')
   command = [sys.executable, download_script, toolchain_dir]
   subprocess.check_call(_LogCommand(command))
 
   pesq_path = os.path.join(toolchain_dir, _GetPlatform(), 'pesq')
-  return pesq_path
+  polqa_path = os.path.join(toolchain_dir, _GetPlatform(), 'PolqaOem64')
+  return pesq_path, polqa_path
 
 
 def ExtractTestRuns(lines, echo=False):
   """Extracts information about tests from the output of a test runner.
 
   Produces tuples (android_device, test_name, reference_file, degraded_file).
   """
   for line in lines:
     if echo:
       sys.stdout.write(line)
@@ skipping 22 matching lines @@
       subprocess.check_call(_LogCommand(adb_command))
   elif os.path.abspath(file_path) != os.path.abspath(out_file_path):
     if move:
       shutil.move(file_path, out_file_path)
     else:
       shutil.copy(file_path, out_file_path)
 
   return out_file_path
 
 
+def _RunPesq(executable_path, reference_file, degraded_file,
+             sampling_frequency_hz=16000):
+  directory = os.path.dirname(reference_file)
+  assert os.path.dirname(degraded_file) == directory
+
+  # Analyze audio.
+  command = [executable_path, '+%d' % sampling_frequency_hz,
+             os.path.basename(reference_file),
+             os.path.basename(degraded_file)]
+  # Need to provide paths in the current directory due to a bug in PESQ:
+  # On Mac, for some 'path/to/file.wav', if 'file.wav' is longer than
+  # 'path/to', PESQ crashes.
+  out = subprocess.check_output(_LogCommand(command),
+                                cwd=directory, stderr=subprocess.STDOUT)
+
+  # Find the scores in stdout of PESQ.
+  match = re.search(
+      r'Prediction \(Raw MOS, MOS-LQO\):\s+=\s+([\d.]+)\s+([\d.]+)', out)
+  if match:
+    raw_mos, _ = match.groups()
+
+    return {'pesq_mos': (raw_mos, 'score')}
+  else:
+    logging.error('PESQ: %s', out.splitlines()[-1])
+    return {}
+
+
+def _RunPolqa(executable_path, reference_file, degraded_file):
+  # Analyze audio.
+  command = [executable_path, '-q', '-LC', 'NB',
+             '-Ref', reference_file, '-Test', degraded_file]
+  try:
+    process = subprocess.Popen(_LogCommand(command),
+                               stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  except OSError as e:
+    if e.errno == os.errno.ENOENT:
+      logging.warning('POLQA executable missing, skipping test.')
+      return {}
+    else:
+      raise
+  out, err = process.communicate()
+
+  # Find the scores in stdout of POLQA.
+  match = re.search(r'\bMOS-LQO:\s+([\d.]+)', out)
+
+  if process.returncode != 0 or not match:
+    if process.returncode == 2:
+      logging.warning('%s (2)', err.strip())
+      logging.warning('POLQA license error, skipping test.')
+    else:
+      logging.error('%s (%d)', err.strip(), process.returncode)
+    return {}
+
+  mos_lqo, = match.groups()
+  return {'polqa_mos_lqo': (mos_lqo, 'score')}
+
+
+Analyzer = collections.namedtuple('Analyzer', ['func', 'executable',
+                                               'sampling_frequency_hz'])
+
+
 def main():
   # pylint: disable=W0101
   logging.basicConfig(level=logging.INFO)
 
   args = _ParseArgs()
 
-  pesq_path = _DownloadTools()
+  pesq_path, polqa_path = _DownloadTools()
 
   out_dir = os.path.join(args.build_dir, '..')
   if args.android:
     test_command = [os.path.join(args.build_dir, 'bin',
                                  'run_low_bandwidth_audio_test'), '-v']
   else:
     test_command = [os.path.join(args.build_dir, 'low_bandwidth_audio_test')]
 
-  # Start the test executable that produces audio files.
-  test_process = subprocess.Popen(_LogCommand(test_command),
-                                  stdout=subprocess.PIPE)
+  analyzers = [Analyzer(_RunPesq, pesq_path, 16000)]
+  # Check if POLQA can run at all, or skip the 48 kHz tests entirely.
+  example_path = os.path.join(SRC_DIR, 'resources',
+                              'voice_engine', 'audio_tiny48.wav')
+  if _RunPolqa(polqa_path, example_path, example_path):
+    analyzers.append(Analyzer(_RunPolqa, polqa_path, 48000))
 
-  try:
-    lines = iter(test_process.stdout.readline, '')
-    for result in ExtractTestRuns(lines, echo=True):
-      (android_device, test_name, reference_file, degraded_file) = result
+  for analyzer in analyzers:
+    # Start the test executable that produces audio files.
+    test_process = subprocess.Popen(

[kjellander_webrtc, 2017/04/10 12:30:02]

What happens in the case where we get a really bad run and the tool fails (like
the problem you were facing earlier) ?

I think we should at least catch + re-raise the error and print what command was
run, since the Exception being thrown usually don't reveal that kind of
information and is most likely harder to understand.

An alternative is to add a little more logging, like the logging.info in
_RunCommand (and/or verbose logging triggered by a --verbose flag).

[oprypin_webrtc, 2017/04/10 12:36:51]

I don't quite understand.

The executed command is logged with `_LogCommand`. The status code is propagated
with `return test_process.wait()` into `sys.exit()`. The stdout is echoed
(though without a logging prefix). By the way, this CL doesn't make any change
to the described process. What exactly can I improve?

[kjellander_webrtc, 2017/04/10 12:49:33]

You're right - this is fine. I didn't pay enough attention to that the .wait()
call is happening at line 228. I guess I assumed it to be inside the try+finally
block or something.

+        _LogCommand(test_command + ['--sampling_frequency=%d' %
+                                    analyzer.sampling_frequency_hz]),
+        stdout=subprocess.PIPE)
+    try:
+      lines = iter(test_process.stdout.readline, '')
+      for result in ExtractTestRuns(lines, echo=True):
+        (android_device, test_name, reference_file, degraded_file) = result
 
-      adb_prefix = (args.adb_path,)
-      if android_device:
-        adb_prefix += ('-s', android_device)
+        adb_prefix = (args.adb_path,)
+        if android_device:
+          adb_prefix += ('-s', android_device)
 
-      reference_file = _GetFile(reference_file, out_dir,
-                                android=args.android, adb_prefix=adb_prefix)
-      degraded_file = _GetFile(degraded_file, out_dir, move=True,
-                               android=args.android, adb_prefix=adb_prefix)
+        reference_file = _GetFile(reference_file, out_dir,
+                                  android=args.android, adb_prefix=adb_prefix)
+        degraded_file = _GetFile(degraded_file, out_dir, move=True,
+                                 android=args.android, adb_prefix=adb_prefix)
 
-      # Analyze audio.
-      pesq_command = [pesq_path, '+16000',
-                      os.path.basename(reference_file),
-                      os.path.basename(degraded_file)]
-      # Need to provide paths in the current directory due to a bug in PESQ:
-      # On Mac, for some 'path/to/file.wav', if 'file.wav' is longer than
-      # 'path/to', PESQ crashes.
-      pesq_output = subprocess.check_output(_LogCommand(pesq_command),
-                                            cwd=out_dir)
+        analyzer_results = analyzer.func(analyzer.executable,
+                                         reference_file, degraded_file)
+        for metric, (value, units) in analyzer_results.items():
+          # Output a result for the perf dashboard.
+          print 'RESULT %s: %s= %s %s' % (metric, test_name, value, units)
 
-      # Find the scores in stdout of pesq.
-      match = re.search(
-          r'Prediction \(Raw MOS, MOS-LQO\):\s+=\s+([\d.]+)\s+([\d.]+)',
-          pesq_output)
-      if match:
-        raw_mos, _ = match.groups()
-
-        # Output a result for the perf dashboard.
-        print 'RESULT pesq_mos: %s= %s score' % (test_name, raw_mos)
-      else:
-        logging.error('PESQ: %s', pesq_output.splitlines()[-1])
-
-      if args.remove:
-        os.remove(reference_file)
-        os.remove(degraded_file)
-  finally:
-    test_process.terminate()
+        if args.remove:
+          os.remove(reference_file)
+          os.remove(degraded_file)
+    finally:
+      test_process.terminate()
 
   return test_process.wait()
 
 
 if __name__ == '__main__':
   sys.exit(main())