APM QA refactoring: render stream support, echo path simulation, new export engine

webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py

Index: webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py
diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py
index 42d76de0e85cb4cf7c337e5f25f1b956d3675fb4..42ae0483d077694da74bfdc409a929075d692cb5 100644
--- a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py
+++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py
@@ -13,6 +13,7 @@ import logging
 import os
 
 from . import data_access
+from . import echo_path_simulation
 from . import eval_scores
 from . import eval_scores_factory
 from . import test_data_generation
@@ -20,7 +21,7 @@ from . import test_data_generation_factory
 
 
 class ApmModuleSimulator(object):
-  """APM module simulator class.
+  """Audio processing module (APM) simulator class.
   """
 
   _TEST_DATA_GENERATOR_CLASSES = (
@@ -47,12 +48,17 @@ class ApmModuleSimulator(object):
     self._evaluation_score_workers = None
     self._config_filepaths = None
     self._input_filepaths = None
+    self._reverse_input_filepaths = None
 
   def Run(self, config_filepaths, input_filepaths, test_data_generator_names,
-          eval_score_names, output_dir):
+          eval_score_names, output_dir, reverse_input_filepaths=None,
+          echo_path_simulator_name=None):
     """Runs the APM simulation.
 
     Initializes paths and required instances, then runs all the simulations.
+    Reverse input can be optionally added. If added, the number of input audio
+    tracks and reverse input audio tracks has to equal. The two lists are used

[peah-webrtc, 2017/04/21 05:28:45]

have?

[AleBzk, 2017/04/24 14:30:42]

Done.

+    to form pairs of input and reverse input.
 
     Args:
       config_filepaths: set of APM configuration files to test.
@@ -60,7 +66,16 @@ class ApmModuleSimulator(object):
       test_data_generator_names: set of test data generator names to test.
       eval_score_names: set of evaluation score names to test.
       output_dir: base path to the output directory for wav files and outcomes.
+      reverse_input_filepaths: set of revert input audio track files to test.
+      echo_path_simulator_name: name of the echo path simulator to use when
+                                reverse input is provided.
     """
+    assert reverse_input_filepaths is None or len(input_filepaths) == len(
+        reverse_input_filepaths), ('reverse input set size not matching input '
+                                   'set size')
+    assert reverse_input_filepaths is None or echo_path_simulator_name in (
+        echo_path_simulation.EchoPathSimulator.REGISTERED_CLASSES), (
+            'reverse input ')
     self._base_output_path = os.path.abspath(output_dir)
 
     # Instance test data generators.
@@ -79,7 +94,15 @@ class ApmModuleSimulator(object):
     self._config_filepaths = self._CreatePathsCollection(config_filepaths)
 
     # Set probing signal file paths.
-    self._input_filepaths = self._CreatePathsCollection(input_filepaths)
+    if reverse_input_filepaths is None:
+      # Forward input only.
+      self._input_filepaths = self._CreatePathsCollection(input_filepaths)
+      self._reverse_input_filepaths = None
+    else:
+      # Set both forward and reverse input signals.
+      self._SetProbingSignalFilePaths(input_filepaths, reverse_input_filepaths)
+
+    # TODO(alessiob): Instance EchoPathSimulator through factory.

[peah-webrtc, 2017/04/21 05:28:45]

Instantiate ?

[AleBzk, 2017/04/24 14:30:41]

Done.

 
     self._SimulateAll()
 
@@ -89,6 +112,8 @@ class ApmModuleSimulator(object):
     Iterates over the combinations of APM configurations, probing signals, and
     test data generators.
     """
+    without_reverse_input = self._reverse_input_filepaths is None
+
     # Try different APM config files.
     for config_name in self._config_filepaths:
       config_filepath = self._config_filepaths[config_name]
@@ -96,6 +121,8 @@ class ApmModuleSimulator(object):
       # Try different probing signal files.
       for input_name in self._input_filepaths:
         input_filepath = self._input_filepaths[input_name]
+        reverse_input_filepath = None if without_reverse_input else (
+            self._reverse_input_filepaths[input_name])
 
         # Try different test data generators.
         for test_data_generators in self._test_data_generators:
@@ -112,19 +139,24 @@ class ApmModuleSimulator(object):
           logging.debug('input-noise cache path: <%s>', input_noise_cache_path)
 
           # Full output path.
+          input_dir_suffix = input_name if without_reverse_input else (
+              '{}-{}'.format(input_name, self._ExtractFileName(
+                  reverse_input_filepath)))
           output_path = os.path.join(
               self._base_output_path,
               'cfg-{}'.format(config_name),
-              'input-{}'.format(input_name),
+              'input-{}'.format(input_dir_suffix),
               'gen-{}'.format(test_data_generators.NAME))
           data_access.MakeDirectory(output_path)
           logging.debug('output path: <%s>', output_path)
 
           self._Simulate(test_data_generators, input_filepath,
-                         input_noise_cache_path, output_path, config_filepath)
+                         reverse_input_filepath, input_noise_cache_path,
+                         output_path, config_filepath,)
 
   def _Simulate(self, test_data_generators, input_filepath,
-                input_noise_cache_path, output_path, config_filepath):
+                reverse_input_filepath, input_noise_cache_path, output_path,
+                config_filepath):
     """Runs a single set of simulation.
 
     Simulates a given combination of APM configuration, probing signal, and
@@ -134,6 +166,7 @@ class ApmModuleSimulator(object):
     Args:
       test_data_generators: TestDataGenerator instance.
       input_filepath: input audio track file to test.
+      reverse_input_filepath: reverse input audio track file to test.
       input_noise_cache_path: path for the noisy audio track files.
       output_path: base output path for the test data generator.
       config_filepath: APM configuration file to test.
@@ -149,23 +182,33 @@ class ApmModuleSimulator(object):
       logging.info(' - test data generator config: <%s>',
                    test_data_generators_config_name)
 
-      # APM input and output signal paths.
+      # Paths to the test data generator output.
+      # Note that the reference signal does not depend on the reverse input
+      # which is optional.
       noisy_signal_filepath = test_data_generators.noisy_signal_filepaths[
           test_data_generators_config_name]
+      reference_signal_filepath = (
+          test_data_generators.reference_signal_filepaths[
+              test_data_generators_config_name])
+
+      # Path to the APM input signal.
+      apm_input_filepath = noisy_signal_filepath
+      if reverse_input_filepath is not None:
+        logging.info(' - reverse input filepath: <%s>', reverse_input_filepath)
+        # TODO(alessiob): Simulate echo path using the EchoPathSimulator
+        # instance with which |apm_input_filepath| is updated.
+
+      # Path for the APM output file.
       evaluation_output_path = test_data_generators.apm_output_paths[
           test_data_generators_config_name]
 
-      # Simulate a call using the audio processing module.
+      # Simulate a call using APM.
       self._audioproc_wrapper.Run(
           config_filepath=config_filepath,
-          input_filepath=noisy_signal_filepath,
+          input_filepath=apm_input_filepath,
+          reverse_input_filepath=reverse_input_filepath,
           output_path=evaluation_output_path)
 
-      # Reference signal path for the evaluation step.
-      reference_signal_filepath = (
-          test_data_generators.reference_signal_filepaths[
-              test_data_generators_config_name])
-
       # Evaluate.
       self._evaluator.Run(
           evaluation_score_workers=self._evaluation_score_workers,
@@ -173,6 +216,27 @@ class ApmModuleSimulator(object):
           reference_input_filepath=reference_signal_filepath,
           output_path=evaluation_output_path)
 
+  def _SetProbingSignalFilePaths(self, input_filepaths,

[peah-webrtc, 2017/04/21 05:28:45]

The naming of this method does not seem intuitive or descriptive to me. What is
ProbingSignals?

[AleBzk, 2017/04/24 14:30:42]

Renamed to _SetTestInputSignalFilePaths().

+                                 reverse_input_filepaths):
+    """Sets input and revers input file paths collections.
+
+    Pairs the input and rever input files by storing the file paths into two

[peah-webrtc, 2017/04/21 05:28:45]

reverse ?

[AleBzk, 2017/04/24 14:30:42]

Done.

+    collections. The key is the file name of the input file.
+
+    Args:
+      input_filepaths: list of file paths.
+      reverse_input_filepaths: list of file paths.
+    """
+    self._input_filepaths = {}
+    self._reverse_input_filepaths = {}
+    assert len(input_filepaths) == len(reverse_input_filepaths)
+    for input_filepath, reverse_input_filepath in zip(
+        input_filepaths, reverse_input_filepaths):
+      name = self._ExtractFileName(input_filepath)
+      self._input_filepaths[name] = os.path.abspath(input_filepath)
+      self._reverse_input_filepaths[name] = os.path.abspath(
+          reverse_input_filepath)
+
   @classmethod
   def _CreatePathsCollection(cls, filepaths):
     """Creates a collection of file paths.
@@ -189,6 +253,10 @@ class ApmModuleSimulator(object):
     """
     filepaths_collection = {}
     for filepath in filepaths:
-      name = os.path.splitext(os.path.split(filepath)[1])[0]
+      name = cls._ExtractFileName(filepath)
       filepaths_collection[name] = os.path.abspath(filepath)
     return filepaths_collection
+
+  @classmethod
+  def _ExtractFileName(cls, filepath):
+    return os.path.splitext(os.path.split(filepath)[1])[0]