APM QA refactoring: render stream support, echo path simulation, new export engine

webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py

 # Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
 #
 # Use of this source code is governed by a BSD-style license
 # that can be found in the LICENSE file in the root of the source
 # tree. An additional intellectual property rights grant can be found
 # in the file PATENTS.  All contributing project authors may
 # be found in the AUTHORS file in the root of the source tree.
 
 """APM module simulator.
 """
 
 import logging
 import os
 
 from . import data_access
+from . import echo_path_simulation
+from . import echo_path_simulation_factory
 from . import eval_scores
 from . import eval_scores_factory
+from . import input_mixer
 from . import test_data_generation
 from . import test_data_generation_factory
 
 
 class ApmModuleSimulator(object):
-  """APM module simulator class.
+  """Audio processing module (APM) simulator class.
   """
 
   _TEST_DATA_GENERATOR_CLASSES = (
       test_data_generation.TestDataGenerator.REGISTERED_CLASSES)
   _EVAL_SCORE_WORKER_CLASSES = eval_scores.EvaluationScore.REGISTERED_CLASSES
 
+  _PREFIX_APM_CONFIG = 'apmcfg-'
+  _PREFIX_CAPTURE = 'capture-'
+  _PREFIX_RENDER = 'render-'
+  _PREFIX_ECHO_SIMULATOR = 'echosim-'
+  _PREFIX_TEST_DATA_GEN = 'datagen-'
+  _PREFIX_TEST_DATA_GEN_PARAMS = 'datagen_params-'
+  _PREFIX_SCORE = 'score-'
+
   def __init__(self, aechen_ir_database_path, polqa_tool_bin_path,
                ap_wrapper, evaluator):
     # Init.
     self._audioproc_wrapper = ap_wrapper
     self._evaluator = evaluator
 
     # Instance factory objects.
     self._test_data_generator_factory = (
         test_data_generation_factory.TestDataGeneratorFactory(
+            output_directory_prefix=self._PREFIX_TEST_DATA_GEN_PARAMS,
             aechen_ir_database_path=aechen_ir_database_path))
     self._evaluation_score_factory = (
         eval_scores_factory.EvaluationScoreWorkerFactory(
+            score_filename_prefix=self._PREFIX_SCORE,
             polqa_tool_bin_path=polqa_tool_bin_path))
 
     # Properties for each run.
     self._base_output_path = None
     self._test_data_generators = None
     self._evaluation_score_workers = None
     self._config_filepaths = None
-    self._input_filepaths = None
+    self._capture_input_filepaths = None
+    self._render_input_filepaths = None
+    self._echo_path_simulator_class = None
 
-  def Run(self, config_filepaths, input_filepaths, test_data_generator_names,
-          eval_score_names, output_dir):
+  @classmethod
+  def GetPrefixApmConfig(cls):
+    return cls._PREFIX_APM_CONFIG

[ivoc, 2017/05/15 14:56:55]

How about making these constants public instead? (So without the _)

[AleBzk, 2017/05/19 15:45:35]

It'd be much less boilerplate, but nothing prevents the caller to change the
value.
If you think that formatting the var name as const is safe enough, then I can
simplify that.
The only automatic complain would come from the linter. And a reviewer can
quickly see that a constant is redefined.

WDYT?

+
+  @classmethod
+  def GetPrefixCapture(cls):
+    return cls._PREFIX_CAPTURE
+
+  @classmethod
+  def GetPrefixRender(cls):
+    return cls._PREFIX_RENDER
+
+  @classmethod
+  def GetPrefixEchoSimulator(cls):
+    return cls._PREFIX_ECHO_SIMULATOR
+
+  @classmethod
+  def GetPrefixTestDataGenerator(cls):
+    return cls._PREFIX_TEST_DATA_GEN
+
+  @classmethod
+  def GetPrefixTestDataGeneratorParameters(cls):
+    return cls._PREFIX_TEST_DATA_GEN_PARAMS
+
+  @classmethod
+  def GetPrefixScore(cls):
+    return cls._PREFIX_SCORE
+
+  def Run(self, config_filepaths, capture_input_filepaths,
+          test_data_generator_names, eval_score_names, output_dir,
+          render_input_filepaths=None, echo_path_simulator_name=(
+              echo_path_simulation.NoEchoPathSimulator.NAME)):
     """Runs the APM simulation.
 
     Initializes paths and required instances, then runs all the simulations.
+    The render input can be optionally added. If added, the number of capture
+    input audio tracks and the number of render input audio tracks have to

[ivoc, 2017/05/15 14:56:55]

input audio tracks have to be equal ("be" is missing)

[AleBzk, 2017/05/19 15:45:35]

Done.

+    equal. The two lists are used to form pairs of capture and render input.
 
     Args:
       config_filepaths: set of APM configuration files to test.
-      input_filepaths: set of input audio track files to test.
+      capture_input_filepaths: set of capture input audio track files to test.
       test_data_generator_names: set of test data generator names to test.
       eval_score_names: set of evaluation score names to test.
       output_dir: base path to the output directory for wav files and outcomes.
+      render_input_filepaths: set of render input audio track files to test.
+      echo_path_simulator_name: name of the echo path simulator to use when
+                                render input is provided.
     """
+    assert render_input_filepaths is None or (
+        len(capture_input_filepaths) == len(render_input_filepaths)), (
+            'render input set size not matching input set size')
+    assert render_input_filepaths is None or echo_path_simulator_name in (
+        echo_path_simulation.EchoPathSimulator.REGISTERED_CLASSES), (
+            'invalid echo path simulator')
     self._base_output_path = os.path.abspath(output_dir)
 
     # Instance test data generators.
     self._test_data_generators = [self._test_data_generator_factory.GetInstance(
         test_data_generators_class=(
             self._TEST_DATA_GENERATOR_CLASSES[name])) for name in (
                 test_data_generator_names)]
 
     # Instance evaluation score workers.
     self._evaluation_score_workers = [
         self._evaluation_score_factory.GetInstance(
             evaluation_score_class=self._EVAL_SCORE_WORKER_CLASSES[name]) for (
                 name) in eval_score_names]
 
     # Set APM configuration file paths.
     self._config_filepaths = self._CreatePathsCollection(config_filepaths)
 
     # Set probing signal file paths.
-    self._input_filepaths = self._CreatePathsCollection(input_filepaths)
+    if render_input_filepaths is None:
+      # Capture input only.
+      self._capture_input_filepaths = self._CreatePathsCollection(
+          capture_input_filepaths)
+      self._render_input_filepaths = None
+    else:
+      # Set both capture and render input signals.
+      self._SetTestInputSignalFilePaths(
+          capture_input_filepaths, render_input_filepaths)
+
+    # Set the echo path simulator class.
+    self._echo_path_simulator_class = (
+        echo_path_simulation.EchoPathSimulator.REGISTERED_CLASSES[
+            echo_path_simulator_name])
 
     self._SimulateAll()
 
   def _SimulateAll(self):
     """Runs all the simulations.
 
     Iterates over the combinations of APM configurations, probing signals, and
-    test data generators.
+    test data generators. This method is mainly responsible for the creation of
+    the cache and output directories required in order to call _Simulate().
     """
+    without_render_input = self._render_input_filepaths is None
+
     # Try different APM config files.
     for config_name in self._config_filepaths:
       config_filepath = self._config_filepaths[config_name]
 
-      # Try different probing signal files.
-      for input_name in self._input_filepaths:
-        input_filepath = self._input_filepaths[input_name]
+      # Try different capture-render pairs.
+      for capture_input_name in self._capture_input_filepaths:
+        capture_input_filepath = self._capture_input_filepaths[
+            capture_input_name]
+        render_input_filepath = None if without_render_input else (
+            self._render_input_filepaths[capture_input_name])
+        render_input_name = '(none)' if without_render_input else (
+            self._ExtractFileName(render_input_filepath))
+
+        # Instance echo path simulator (if needed).
+        echo_path_simulator = (
+            echo_path_simulation_factory.EchoPathSimulatorFactory.GetInstance(
+                self._echo_path_simulator_class, render_input_filepath))
 
         # Try different test data generators.
         for test_data_generators in self._test_data_generators:
-          logging.info('config: <%s>, input: <%s>, noise: <%s>',
-                       config_name, input_name, test_data_generators.NAME)
+          logging.info('APM config preset: <%s>, capture: <%s>, render: <%s>,'
+                       'test data generator: <%s>,  echo simulator: <%s>',
+                       config_name, capture_input_name, render_input_name,
+                       test_data_generators.NAME, echo_path_simulator.NAME)
 
-          # Output path for the input-noise pairs. It is used to cache the noisy
-          # copies of the probing signals (shared across some simulations).
-          input_noise_cache_path = os.path.join(
-              self._base_output_path,
-              '_cache',
-              'input_{}-noise_{}'.format(input_name, test_data_generators.NAME))
-          data_access.MakeDirectory(input_noise_cache_path)
-          logging.debug('input-noise cache path: <%s>', input_noise_cache_path)
+          # Output path for the generated test data.
+          # The path is used to cache the signals shared across simulations.
+          test_data_cache_path = os.path.join(
+              self._base_output_path, '_cache',
+              self._PREFIX_CAPTURE + capture_input_name,
+              self._PREFIX_TEST_DATA_GEN + test_data_generators.NAME)
+          data_access.MakeDirectory(test_data_cache_path)
+          logging.debug('test data cache path: <%s>', test_data_cache_path)
+
+          # Output path for the echo simulator and APM input mixer output.
+          echo_test_data_cache_path = os.path.join(
+              test_data_cache_path, 'echosim-{}'.format(
+                  echo_path_simulator.NAME))
+          data_access.MakeDirectory(echo_test_data_cache_path)
+          logging.debug('echo test data cache path: <%s>',
+                        echo_test_data_cache_path)
 
           # Full output path.
           output_path = os.path.join(
               self._base_output_path,
-              'cfg-{}'.format(config_name),
-              'input-{}'.format(input_name),
-              'gen-{}'.format(test_data_generators.NAME))
+              self._PREFIX_APM_CONFIG + config_name,
+              self._PREFIX_CAPTURE + capture_input_name,
+              self._PREFIX_RENDER + render_input_name,
+              self._PREFIX_ECHO_SIMULATOR + echo_path_simulator.NAME,
+              self._PREFIX_TEST_DATA_GEN + test_data_generators.NAME)
           data_access.MakeDirectory(output_path)
           logging.debug('output path: <%s>', output_path)
 
-          self._Simulate(test_data_generators, input_filepath,
-                         input_noise_cache_path, output_path, config_filepath)
+          self._Simulate(test_data_generators, capture_input_filepath,
+                         render_input_filepath, test_data_cache_path,
+                         echo_test_data_cache_path, output_path,
+                         config_filepath, echo_path_simulator)
 
-  def _Simulate(self, test_data_generators, input_filepath,
-                input_noise_cache_path, output_path, config_filepath):
+  def _Simulate(self, test_data_generators, clean_capture_input_filepath,
+                render_input_filepath, test_data_cache_path,
+                echo_test_data_cache_path, output_path, config_filepath,
+                echo_path_simulator):
     """Runs a single set of simulation.
 
     Simulates a given combination of APM configuration, probing signal, and
     test data generator. It iterates over the test data generator
     internal configurations.
 
     Args:
       test_data_generators: TestDataGenerator instance.
-      input_filepath: input audio track file to test.
-      input_noise_cache_path: path for the noisy audio track files.
+      clean_capture_input_filepath: clean capture input audio track file to

[ivoc, 2017/05/15 14:56:55]

It would be good to explain briefly what "clean" means in this context.

[AleBzk, 2017/05/19 15:45:35]

Done.

+                                    test.
+      render_input_filepath: render input audio track file to test.
+      test_data_cache_path: path for the generated test audio track files.
+      echo_test_data_cache_path: path for the echo simulator.
       output_path: base output path for the test data generator.
       config_filepath: APM configuration file to test.
+      echo_path_simulator: EchoPathSimulator instance.
     """
     # Generate pairs of noisy input and reference signal files.
     test_data_generators.Generate(
-        input_signal_filepath=input_filepath,
-        input_noise_cache_path=input_noise_cache_path,
+        input_signal_filepath=clean_capture_input_filepath,
+        test_data_cache_path=test_data_cache_path,
         base_output_path=output_path)
 
     # For each test data pair, simulate a call and evaluate.
     for config_name in test_data_generators.config_names:
       logging.info(' - test data generator config: <%s>', config_name)
 
-      # APM input and output signal paths.
-      noisy_signal_filepath = test_data_generators.noisy_signal_filepaths[
-          config_name]
+      # Paths to the test data generator output.
+      # Note that the reference signal does not depend on the render input
+      # which is optional.
+      noisy_capture_input_filepath = (
+          test_data_generators.noisy_signal_filepaths[config_name])
+      reference_signal_filepath = (
+          test_data_generators.reference_signal_filepaths[config_name])
+
+      # Output path for the evaluation (e.g., APM output file).
       evaluation_output_path = test_data_generators.apm_output_paths[
           config_name]
 
-      # Simulate a call using the audio processing module.
+      # Paths to the APM input signals.
+      echo_path_filpath = echo_path_simulator.Simulate(

[ivoc, 2017/05/15 14:56:55]

Should be "echo_path_filepath" I presume?

[AleBzk, 2017/05/19 15:45:35]

Done.

+          echo_test_data_cache_path)
+      apm_input_filepath = input_mixer.ApmInputMixer.Mix(
+          echo_test_data_cache_path, noisy_capture_input_filepath,
+          echo_path_filpath)
+
+      # Simulate a call using APM.
       self._audioproc_wrapper.Run(
           config_filepath=config_filepath,
-          input_filepath=noisy_signal_filepath,
+          capture_input_filepath=apm_input_filepath,
+          render_input_filepath=render_input_filepath,
           output_path=evaluation_output_path)
 
-      # Reference signal path for the evaluation step.
-      reference_signal_filepath = (
-          test_data_generators.reference_signal_filepaths[
-              config_name])
-
       # Evaluate.
       self._evaluator.Run(
           evaluation_score_workers=self._evaluation_score_workers,
           apm_output_filepath=self._audioproc_wrapper.output_filepath,
           reference_input_filepath=reference_signal_filepath,
           output_path=evaluation_output_path)
 
+      # Save simulation metadata.
+      data_access.Metadata.SaveAudioTestDataPaths(
+          output_path=evaluation_output_path,
+          clean_capture_input_filepath=clean_capture_input_filepath,
+          echo_free_capture_filepath=noisy_capture_input_filepath,
+          echo_filepath=echo_path_filpath,
+          render_filepath=render_input_filepath,
+          capture_filepath=apm_input_filepath,
+          apm_output_filepath=self._audioproc_wrapper.output_filepath,
+          apm_reference_filepath=reference_signal_filepath)
+
+  def _SetTestInputSignalFilePaths(self, capture_input_filepaths,
+                                   render_input_filepaths):
+    """Sets input and render input file paths collections.
+
+    Pairs the input and render input files by storing the file paths into two
+    collections. The key is the file name of the input file.
+
+    Args:
+      capture_input_filepaths: list of file paths.
+      render_input_filepaths: list of file paths.
+    """
+    self._capture_input_filepaths = {}
+    self._render_input_filepaths = {}
+    assert len(capture_input_filepaths) == len(render_input_filepaths)
+    for capture_input_filepath, render_input_filepath in zip(
+        capture_input_filepaths, render_input_filepaths):
+      name = self._ExtractFileName(capture_input_filepath)
+      self._capture_input_filepaths[name] = os.path.abspath(
+          capture_input_filepath)
+      self._render_input_filepaths[name] = os.path.abspath(
+          render_input_filepath)
+
   @classmethod
   def _CreatePathsCollection(cls, filepaths):
     """Creates a collection of file paths.
 
     Given a list of file paths, makes a collection with one item for each file
     path. The value is absolute path, the key is the file name without
     extenstion.
 
     Args:
       filepaths: list of file paths.
 
     Returns:
       A dict.
     """
     filepaths_collection = {}
     for filepath in filepaths:
-      name = os.path.splitext(os.path.split(filepath)[1])[0]
+      name = cls._ExtractFileName(filepath)
       filepaths_collection[name] = os.path.abspath(filepath)
     return filepaths_collection
+
+  @classmethod
+  def _ExtractFileName(cls, filepath):
+    return os.path.splitext(os.path.split(filepath)[1])[0]

[ivoc, 2017/05/15 14:56:55]

Why does this take [1] after splitting? Is that the final part with the filename
in it? In that case wouldn't -1 be more robust?

[AleBzk, 2017/05/19 15:45:35]

Done.