APM QA refactoring: render stream support, echo path simulation, new export engine

webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py

Index: webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py
diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py
index a3bf9cceee3da85520865c3da9625101e69dadb4..672284a0721f254226987fd798d60f744dc3ce99 100644
--- a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py
+++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py
@@ -13,6 +13,7 @@ import logging
 import os
 
 from . import data_access
+from . import echo_path_simulation
 from . import eval_scores
 from . import eval_scores_factory
 from . import test_data_generation
@@ -20,7 +21,7 @@ from . import test_data_generation_factory
 
 
 class ApmModuleSimulator(object):
-  """APM module simulator class.
+  """Audio processing module (APM) simulator class.
   """
 
   _TEST_DATA_GENERATOR_CLASSES = (
@@ -46,21 +47,35 @@ class ApmModuleSimulator(object):
     self._test_data_generators = None
     self._evaluation_score_workers = None
     self._config_filepaths = None
-    self._input_filepaths = None
+    self._capture_input_filepaths = None
+    self._render_input_filepaths = None
 
-  def Run(self, config_filepaths, input_filepaths, test_data_generator_names,
-          eval_score_names, output_dir):
+  def Run(self, config_filepaths, capture_input_filepaths,
+          test_data_generator_names, eval_score_names, output_dir,
+          render_input_filepaths=None, echo_path_simulator_name=None):
     """Runs the APM simulation.
 
     Initializes paths and required instances, then runs all the simulations.
+    The render input can be optionally added. If added, the number of capture
+    input audio tracks and the number of render input audio tracks have to
+    equal. The two lists are used to form pairs of capture and render input.

[peah-webrtc, 2017/05/03 08:10:08]

input->inputs

[peah-webrtc, 2017/05/03 08:10:08]

be equal

 
     Args:
       config_filepaths: set of APM configuration files to test.
-      input_filepaths: set of input audio track files to test.
+      capture_input_filepaths: set of capture input audio track files to test.
       test_data_generator_names: set of test data generator names to test.
       eval_score_names: set of evaluation score names to test.
       output_dir: base path to the output directory for wav files and outcomes.
+      render_input_filepaths: set of render input audio track files to test.
+      echo_path_simulator_name: name of the echo path simulator to use when
+                                render input is provided.
     """
+    assert render_input_filepaths is None or (
+        len(capture_input_filepaths) == len(render_input_filepaths)), (
+            'render input set size not matching input set size')
+    assert render_input_filepaths is None or echo_path_simulator_name in (
+        echo_path_simulation.EchoPathSimulator.REGISTERED_CLASSES), (
+            'invalid echo path simulator')
     self._base_output_path = os.path.abspath(output_dir)
 
     # Instance test data generators.
@@ -79,7 +94,16 @@ class ApmModuleSimulator(object):
     self._config_filepaths = self._CreatePathsCollection(config_filepaths)
 
     # Set probing signal file paths.
-    self._input_filepaths = self._CreatePathsCollection(input_filepaths)
+    if render_input_filepaths is None:
+      # Capture input only.
+      self._capture_input_filepaths = self._CreatePathsCollection(
+          capture_input_filepaths)
+      self._render_input_filepaths = None
+    else:
+      # Set both capture and render input signals.
+      self._SetTestInputSignalFilePaths(
+          capture_input_filepaths, render_input_filepaths)
+      # TODO(alessiob): Instanciate EchoPathSimulator through factory.

[peah-webrtc, 2017/05/03 08:10:08]

Instanciate -> Instantiate

 
     self._SimulateAll()
 
@@ -89,42 +113,54 @@ class ApmModuleSimulator(object):
     Iterates over the combinations of APM configurations, probing signals, and
     test data generators.
     """
+    without_render_input = self._render_input_filepaths is None

[peah-webrtc, 2017/05/03 08:10:08]

suggestion: without_render_input -> no_render_input

That seems to work better in the statements below:
None if without_render_input -> None if no_render_input
if without_render_input -> if no_render_input 

+
     # Try different APM config files.
     for config_name in self._config_filepaths:
       config_filepath = self._config_filepaths[config_name]
 
       # Try different probing signal files.
-      for input_name in self._input_filepaths:
-        input_filepath = self._input_filepaths[input_name]
+      for capture_input_name in self._capture_input_filepaths:
+        capture_input_filepath = self._capture_input_filepaths[
+            capture_input_name]
+        render_input_filepath = None if without_render_input else (
+            self._render_input_filepaths[capture_input_name])
 
         # Try different test data generators.
         for test_data_generators in self._test_data_generators:
           logging.info('config: <%s>, input: <%s>, noise: <%s>',
-                       config_name, input_name, test_data_generators.NAME)
+                       config_name, capture_input_name,
+                       test_data_generators.NAME)
 
           # Output path for the input-noise pairs. It is used to cache the noisy
           # copies of the probing signals (shared across some simulations).
           input_noise_cache_path = os.path.join(
               self._base_output_path,
               '_cache',
-              'input_{}-noise_{}'.format(input_name, test_data_generators.NAME))
+              'input_{}-noise_{}'.format(capture_input_name,
+                                         test_data_generators.NAME))
           data_access.MakeDirectory(input_noise_cache_path)
           logging.debug('input-noise cache path: <%s>', input_noise_cache_path)
 
           # Full output path.
+          input_dir_suffix = capture_input_name if without_render_input else (
+              '{}-{}'.format(capture_input_name, self._ExtractFileName(
+                  render_input_filepath)))
           output_path = os.path.join(
               self._base_output_path,
               'cfg-{}'.format(config_name),
-              'input-{}'.format(input_name),
+              'input-{}'.format(input_dir_suffix),
               'gen-{}'.format(test_data_generators.NAME))
           data_access.MakeDirectory(output_path)
           logging.debug('output path: <%s>', output_path)
 
-          self._Simulate(test_data_generators, input_filepath,
-                         input_noise_cache_path, output_path, config_filepath)
+          self._Simulate(test_data_generators, capture_input_filepath,
+                         render_input_filepath, input_noise_cache_path,
+                         output_path, config_filepath,)
 
-  def _Simulate(self, test_data_generators, input_filepath,
-                input_noise_cache_path, output_path, config_filepath):
+  def _Simulate(self, test_data_generators, capture_input_filepath,
+                render_input_filepath, input_noise_cache_path, output_path,
+                config_filepath):
     """Runs a single set of simulation.
 
     Simulates a given combination of APM configuration, probing signal, and
@@ -133,14 +169,15 @@ class ApmModuleSimulator(object):
 
     Args:
       test_data_generators: TestDataGenerator instance.
-      input_filepath: input audio track file to test.
+      capture_input_filepath: clean capture input audio track file to test.
+      render_input_filepath: render input audio track file to test.
       input_noise_cache_path: path for the noisy audio track files.
       output_path: base output path for the test data generator.
       config_filepath: APM configuration file to test.
     """
     # Generate pairs of noisy input and reference signal files.
     test_data_generators.Generate(
-        input_signal_filepath=input_filepath,
+        input_signal_filepath=capture_input_filepath,
         input_noise_cache_path=input_noise_cache_path,
         base_output_path=output_path)
 
@@ -148,23 +185,31 @@ class ApmModuleSimulator(object):
     for config_name in test_data_generators.config_names:
       logging.info(' - test data generator config: <%s>', config_name)
 
-      # APM input and output signal paths.
-      noisy_signal_filepath = test_data_generators.noisy_signal_filepaths[
-          config_name]
+      # Paths to the test data generator output.
+      # Note that the reference signal does not depend on the render input
+      # which is optional.
+      noisy_capture_input_filepath = (
+          test_data_generators.noisy_signal_filepaths[config_name])
+      reference_signal_filepath = (
+          test_data_generators.reference_signal_filepaths[config_name])
+
+      # Paths to the APM input signals.
+      if render_input_filepath is not None:
+        logging.info(' - render input filepath: <%s>', render_input_filepath)
+        # TODO(alessiob): Simulate echo path using the EchoPathSimulator
+        # instance with which |noisy_capture_input_filepath| is updated.
+
+      # Path for the APM output file.
       evaluation_output_path = test_data_generators.apm_output_paths[
           config_name]
 
-      # Simulate a call using the audio processing module.
+      # Simulate a call using APM.
       self._audioproc_wrapper.Run(
           config_filepath=config_filepath,
-          input_filepath=noisy_signal_filepath,
+          capture_input_filepath=noisy_capture_input_filepath,
+          render_input_filepath=render_input_filepath,
           output_path=evaluation_output_path)
 
-      # Reference signal path for the evaluation step.
-      reference_signal_filepath = (
-          test_data_generators.reference_signal_filepaths[
-              config_name])
-
       # Evaluate.
       self._evaluator.Run(
           evaluation_score_workers=self._evaluation_score_workers,
@@ -172,6 +217,28 @@ class ApmModuleSimulator(object):
           reference_input_filepath=reference_signal_filepath,
           output_path=evaluation_output_path)
 
+  def _SetTestInputSignalFilePaths(self, capture_input_filepaths,
+                                   render_input_filepaths):
+    """Sets input and render input file paths collections.
+
+    Pairs the input and render input files by storing the file paths into two
+    collections. The key is the file name of the input file.
+
+    Args:
+      capture_input_filepaths: list of file paths.
+      render_input_filepaths: list of file paths.
+    """
+    self._capture_input_filepaths = {}
+    self._render_input_filepaths = {}
+    assert len(capture_input_filepaths) == len(render_input_filepaths)
+    for capture_input_filepath, render_input_filepath in zip(
+        capture_input_filepaths, render_input_filepaths):
+      name = self._ExtractFileName(capture_input_filepath)
+      self._capture_input_filepaths[name] = os.path.abspath(
+          capture_input_filepath)
+      self._render_input_filepaths[name] = os.path.abspath(
+          render_input_filepath)
+
   @classmethod
   def _CreatePathsCollection(cls, filepaths):
     """Creates a collection of file paths.
@@ -188,6 +255,10 @@ class ApmModuleSimulator(object):
     """
     filepaths_collection = {}
     for filepath in filepaths:
-      name = os.path.splitext(os.path.split(filepath)[1])[0]
+      name = cls._ExtractFileName(filepath)
       filepaths_collection[name] = os.path.abspath(filepath)
     return filepaths_collection
+
+  @classmethod
+  def _ExtractFileName(cls, filepath):
+    return os.path.splitext(os.path.split(filepath)[1])[0]