APM QA refactoring: render stream support, echo path simulation, new export engine

webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py

 # Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
 #
 # Use of this source code is governed by a BSD-style license
 # that can be found in the LICENSE file in the root of the source
 # tree. An additional intellectual property rights grant can be found
 # in the file PATENTS.  All contributing project authors may
 # be found in the AUTHORS file in the root of the source tree.
 
 """APM module simulator.
 """
 
 import logging
 import os
 
 from . import data_access
+from . import echo_path_simulation
 from . import eval_scores
 from . import eval_scores_factory
 from . import test_data_generation
 from . import test_data_generation_factory
 
 
 class ApmModuleSimulator(object):
-  """APM module simulator class.
+  """Audio processing module (APM) simulator class.
   """
 
   _TEST_DATA_GENERATOR_CLASSES = (
       test_data_generation.TestDataGenerator.REGISTERED_CLASSES)
   _EVAL_SCORE_WORKER_CLASSES = eval_scores.EvaluationScore.REGISTERED_CLASSES
 
   def __init__(self, aechen_ir_database_path, polqa_tool_bin_path,
                ap_wrapper, evaluator):
     # Init.
     self._audioproc_wrapper = ap_wrapper
     self._evaluator = evaluator
 
     # Instance factory objects.
     self._test_data_generator_factory = (
         test_data_generation_factory.TestDataGeneratorFactory(
             aechen_ir_database_path=aechen_ir_database_path))
     self._evaluation_score_factory = (
         eval_scores_factory.EvaluationScoreWorkerFactory(
             polqa_tool_bin_path=polqa_tool_bin_path))
 
     # Properties for each run.
     self._base_output_path = None
     self._test_data_generators = None
     self._evaluation_score_workers = None
     self._config_filepaths = None
-    self._input_filepaths = None
+    self._capture_input_filepaths = None
+    self._render_input_filepaths = None
 
-  def Run(self, config_filepaths, input_filepaths, test_data_generator_names,
-          eval_score_names, output_dir):
+  def Run(self, config_filepaths, capture_input_filepaths,
+          test_data_generator_names, eval_score_names, output_dir,
+          render_input_filepaths=None, echo_path_simulator_name=None):
     """Runs the APM simulation.
 
     Initializes paths and required instances, then runs all the simulations.
+    The render input can be optionally added. If added, the number of capture
+    input audio tracks and the number of render input audio tracks have to
+    equal. The two lists are used to form pairs of capture and render input.

[peah-webrtc, 2017/05/03 08:10:08]

input->inputs

[peah-webrtc, 2017/05/03 08:10:08]

be equal

 
     Args:
       config_filepaths: set of APM configuration files to test.
-      input_filepaths: set of input audio track files to test.
+      capture_input_filepaths: set of capture input audio track files to test.
       test_data_generator_names: set of test data generator names to test.
       eval_score_names: set of evaluation score names to test.
       output_dir: base path to the output directory for wav files and outcomes.
+      render_input_filepaths: set of render input audio track files to test.
+      echo_path_simulator_name: name of the echo path simulator to use when
+                                render input is provided.
     """
+    assert render_input_filepaths is None or (
+        len(capture_input_filepaths) == len(render_input_filepaths)), (
+            'render input set size not matching input set size')
+    assert render_input_filepaths is None or echo_path_simulator_name in (
+        echo_path_simulation.EchoPathSimulator.REGISTERED_CLASSES), (
+            'invalid echo path simulator')
     self._base_output_path = os.path.abspath(output_dir)
 
     # Instance test data generators.
     self._test_data_generators = [self._test_data_generator_factory.GetInstance(
         test_data_generators_class=(
             self._TEST_DATA_GENERATOR_CLASSES[name])) for name in (
                 test_data_generator_names)]
 
     # Instance evaluation score workers.
     self._evaluation_score_workers = [
         self._evaluation_score_factory.GetInstance(
             evaluation_score_class=self._EVAL_SCORE_WORKER_CLASSES[name]) for (
                 name) in eval_score_names]
 
     # Set APM configuration file paths.
     self._config_filepaths = self._CreatePathsCollection(config_filepaths)
 
     # Set probing signal file paths.
-    self._input_filepaths = self._CreatePathsCollection(input_filepaths)
+    if render_input_filepaths is None:
+      # Capture input only.
+      self._capture_input_filepaths = self._CreatePathsCollection(
+          capture_input_filepaths)
+      self._render_input_filepaths = None
+    else:
+      # Set both capture and render input signals.
+      self._SetTestInputSignalFilePaths(
+          capture_input_filepaths, render_input_filepaths)
+      # TODO(alessiob): Instanciate EchoPathSimulator through factory.

[peah-webrtc, 2017/05/03 08:10:08]

Instanciate -> Instantiate

 
     self._SimulateAll()
 
   def _SimulateAll(self):
     """Runs all the simulations.
 
     Iterates over the combinations of APM configurations, probing signals, and
     test data generators.
     """
+    without_render_input = self._render_input_filepaths is None

[peah-webrtc, 2017/05/03 08:10:08]

suggestion: without_render_input -> no_render_input

That seems to work better in the statements below:
None if without_render_input -> None if no_render_input
if without_render_input -> if no_render_input 

+
     # Try different APM config files.
     for config_name in self._config_filepaths:
       config_filepath = self._config_filepaths[config_name]
 
       # Try different probing signal files.
-      for input_name in self._input_filepaths:
-        input_filepath = self._input_filepaths[input_name]
+      for capture_input_name in self._capture_input_filepaths:
+        capture_input_filepath = self._capture_input_filepaths[
+            capture_input_name]
+        render_input_filepath = None if without_render_input else (
+            self._render_input_filepaths[capture_input_name])
 
         # Try different test data generators.
         for test_data_generators in self._test_data_generators:
           logging.info('config: <%s>, input: <%s>, noise: <%s>',
-                       config_name, input_name, test_data_generators.NAME)
+                       config_name, capture_input_name,
+                       test_data_generators.NAME)
 
           # Output path for the input-noise pairs. It is used to cache the noisy
           # copies of the probing signals (shared across some simulations).
           input_noise_cache_path = os.path.join(
               self._base_output_path,
               '_cache',
-              'input_{}-noise_{}'.format(input_name, test_data_generators.NAME))
+              'input_{}-noise_{}'.format(capture_input_name,
+                                         test_data_generators.NAME))
           data_access.MakeDirectory(input_noise_cache_path)
           logging.debug('input-noise cache path: <%s>', input_noise_cache_path)
 
           # Full output path.
+          input_dir_suffix = capture_input_name if without_render_input else (
+              '{}-{}'.format(capture_input_name, self._ExtractFileName(
+                  render_input_filepath)))
           output_path = os.path.join(
               self._base_output_path,
               'cfg-{}'.format(config_name),
-              'input-{}'.format(input_name),
+              'input-{}'.format(input_dir_suffix),
               'gen-{}'.format(test_data_generators.NAME))
           data_access.MakeDirectory(output_path)
           logging.debug('output path: <%s>', output_path)
 
-          self._Simulate(test_data_generators, input_filepath,
-                         input_noise_cache_path, output_path, config_filepath)
+          self._Simulate(test_data_generators, capture_input_filepath,
+                         render_input_filepath, input_noise_cache_path,
+                         output_path, config_filepath,)
 
-  def _Simulate(self, test_data_generators, input_filepath,
-                input_noise_cache_path, output_path, config_filepath):
+  def _Simulate(self, test_data_generators, capture_input_filepath,
+                render_input_filepath, input_noise_cache_path, output_path,
+                config_filepath):
     """Runs a single set of simulation.
 
     Simulates a given combination of APM configuration, probing signal, and
     test data generator. It iterates over the test data generator
     internal configurations.
 
     Args:
       test_data_generators: TestDataGenerator instance.
-      input_filepath: input audio track file to test.
+      capture_input_filepath: clean capture input audio track file to test.
+      render_input_filepath: render input audio track file to test.
       input_noise_cache_path: path for the noisy audio track files.
       output_path: base output path for the test data generator.
       config_filepath: APM configuration file to test.
     """
     # Generate pairs of noisy input and reference signal files.
     test_data_generators.Generate(
-        input_signal_filepath=input_filepath,
+        input_signal_filepath=capture_input_filepath,
         input_noise_cache_path=input_noise_cache_path,
         base_output_path=output_path)
 
     # For each test data pair, simulate a call and evaluate.
     for config_name in test_data_generators.config_names:
       logging.info(' - test data generator config: <%s>', config_name)
 
-      # APM input and output signal paths.
-      noisy_signal_filepath = test_data_generators.noisy_signal_filepaths[
-          config_name]
+      # Paths to the test data generator output.
+      # Note that the reference signal does not depend on the render input
+      # which is optional.
+      noisy_capture_input_filepath = (
+          test_data_generators.noisy_signal_filepaths[config_name])
+      reference_signal_filepath = (
+          test_data_generators.reference_signal_filepaths[config_name])
+
+      # Paths to the APM input signals.
+      if render_input_filepath is not None:
+        logging.info(' - render input filepath: <%s>', render_input_filepath)
+        # TODO(alessiob): Simulate echo path using the EchoPathSimulator
+        # instance with which |noisy_capture_input_filepath| is updated.
+
+      # Path for the APM output file.
       evaluation_output_path = test_data_generators.apm_output_paths[
           config_name]
 
-      # Simulate a call using the audio processing module.
+      # Simulate a call using APM.
       self._audioproc_wrapper.Run(
           config_filepath=config_filepath,
-          input_filepath=noisy_signal_filepath,
+          capture_input_filepath=noisy_capture_input_filepath,
+          render_input_filepath=render_input_filepath,
           output_path=evaluation_output_path)
 
-      # Reference signal path for the evaluation step.
-      reference_signal_filepath = (
-          test_data_generators.reference_signal_filepaths[
-              config_name])
-
       # Evaluate.
       self._evaluator.Run(
           evaluation_score_workers=self._evaluation_score_workers,
           apm_output_filepath=self._audioproc_wrapper.output_filepath,
           reference_input_filepath=reference_signal_filepath,
           output_path=evaluation_output_path)
 
+  def _SetTestInputSignalFilePaths(self, capture_input_filepaths,
+                                   render_input_filepaths):
+    """Sets input and render input file paths collections.
+
+    Pairs the input and render input files by storing the file paths into two
+    collections. The key is the file name of the input file.
+
+    Args:
+      capture_input_filepaths: list of file paths.
+      render_input_filepaths: list of file paths.
+    """
+    self._capture_input_filepaths = {}
+    self._render_input_filepaths = {}
+    assert len(capture_input_filepaths) == len(render_input_filepaths)
+    for capture_input_filepath, render_input_filepath in zip(
+        capture_input_filepaths, render_input_filepaths):
+      name = self._ExtractFileName(capture_input_filepath)
+      self._capture_input_filepaths[name] = os.path.abspath(
+          capture_input_filepath)
+      self._render_input_filepaths[name] = os.path.abspath(
+          render_input_filepath)
+
   @classmethod
   def _CreatePathsCollection(cls, filepaths):
     """Creates a collection of file paths.
 
     Given a list of file paths, makes a collection with one item for each file
     path. The value is absolute path, the key is the file name without
     extenstion.
 
     Args:
       filepaths: list of file paths.
 
     Returns:
       A dict.
     """
     filepaths_collection = {}
     for filepath in filepaths:
-      name = os.path.splitext(os.path.split(filepath)[1])[0]
+      name = cls._ExtractFileName(filepath)
       filepaths_collection[name] = os.path.abspath(filepath)
     return filepaths_collection
+
+  @classmethod
+  def _ExtractFileName(cls, filepath):
+    return os.path.splitext(os.path.split(filepath)[1])[0]