Added basic framework for AEC3 in the audio processing module

webrtc/modules/audio_processing/audio_processing_impl.cc

Index: webrtc/modules/audio_processing/audio_processing_impl.cc
diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc
index 5ca9275f0e215ddab3a648e04ce346850ab3423d..2cf3cef50037393257d627e601e9473ae1f4e39d 100644
--- a/webrtc/modules/audio_processing/audio_processing_impl.cc
+++ b/webrtc/modules/audio_processing/audio_processing_impl.cc
@@ -20,6 +20,7 @@
 #include "webrtc/common_audio/include/audio_util.h"
 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
 #include "webrtc/modules/audio_processing/aec/aec_core.h"
+#include "webrtc/modules/audio_processing/aec3/echo_canceller3.h"
 #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h"
 #include "webrtc/modules/audio_processing/audio_buffer.h"
 #include "webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h"
@@ -167,6 +168,7 @@ bool AudioProcessingImpl::ApmSubmoduleStates::Update(
     bool beamformer_enabled,
     bool adaptive_gain_controller_enabled,
     bool level_controller_enabled,
+    bool echo_canceller3_enabled,
     bool voice_activity_detector_enabled,
     bool level_estimator_enabled,
     bool transient_suppressor_enabled) {
@@ -184,6 +186,7 @@ bool AudioProcessingImpl::ApmSubmoduleStates::Update(
   changed |=
       (adaptive_gain_controller_enabled != adaptive_gain_controller_enabled_);
   changed |= (level_controller_enabled != level_controller_enabled_);
+  changed |= (echo_canceller3_enabled != echo_canceller3_enabled_);
   changed |= (level_estimator_enabled != level_estimator_enabled_);
   changed |=
       (voice_activity_detector_enabled != voice_activity_detector_enabled_);
@@ -198,6 +201,7 @@ bool AudioProcessingImpl::ApmSubmoduleStates::Update(
     beamformer_enabled_ = beamformer_enabled;
     adaptive_gain_controller_enabled_ = adaptive_gain_controller_enabled;
     level_controller_enabled_ = level_controller_enabled;
+    echo_canceller3_enabled_ = echo_canceller3_enabled;
     level_estimator_enabled_ = level_estimator_enabled;
     voice_activity_detector_enabled_ = voice_activity_detector_enabled;
     transient_suppressor_enabled_ = transient_suppressor_enabled;
@@ -224,14 +228,15 @@ bool AudioProcessingImpl::ApmSubmoduleStates::CaptureMultiBandProcessingActive()
     const {
   return low_cut_filter_enabled_ || echo_canceller_enabled_ ||
          mobile_echo_controller_enabled_ || noise_suppressor_enabled_ ||
-         beamformer_enabled_ || adaptive_gain_controller_enabled_;
+         beamformer_enabled_ || adaptive_gain_controller_enabled_ ||
+         echo_canceller3_enabled_;
 }
 
 bool AudioProcessingImpl::ApmSubmoduleStates::RenderMultiBandSubModulesActive()
     const {
   return RenderMultiBandProcessingActive() || echo_canceller_enabled_ ||
          mobile_echo_controller_enabled_ || adaptive_gain_controller_enabled_ ||
-         residual_echo_detector_enabled_;
+         residual_echo_detector_enabled_ || echo_canceller3_enabled_;
 }
 
 bool AudioProcessingImpl::ApmSubmoduleStates::RenderMultiBandProcessingActive()
@@ -271,6 +276,7 @@ struct AudioProcessingImpl::ApmPrivateSubmodules {
   std::unique_ptr<LowCutFilter> low_cut_filter;
   std::unique_ptr<LevelController> level_controller;
   std::unique_ptr<ResidualEchoDetector> residual_echo_detector;
+  std::unique_ptr<EchoCanceller3> echo_canceller3;
 };
 
 AudioProcessing* AudioProcessing::Create() {
@@ -462,12 +468,20 @@ int AudioProcessingImpl::InitializeLocked() {
     render_.render_audio.reset(nullptr);
     render_.render_converter.reset(nullptr);
   }
-  capture_.capture_audio.reset(
-      new AudioBuffer(formats_.api_format.input_stream().num_frames(),
-                      formats_.api_format.input_stream().num_channels(),
-                      capture_nonlocked_.capture_processing_format.num_frames(),
-                      capture_audiobuffer_num_channels,
-                      formats_.api_format.output_stream().num_frames()));
+  if (private_submodules_->echo_canceller3) {
+    capture_.capture_audio.reset(new AudioBuffer(
+        formats_.api_format.input_stream().num_frames(),
+        formats_.api_format.input_stream().num_channels(),
+        capture_nonlocked_.capture_processing_format.num_frames(), 1,
+        formats_.api_format.output_stream().num_frames()));
+  } else {
+    capture_.capture_audio.reset(new AudioBuffer(
+        formats_.api_format.input_stream().num_frames(),
+        formats_.api_format.input_stream().num_channels(),
+        capture_nonlocked_.capture_processing_format.num_frames(),
+        capture_audiobuffer_num_channels,
+        formats_.api_format.output_stream().num_frames()));
+  }
 
   public_submodules_->echo_cancellation->Initialize(
       proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(),
@@ -508,6 +522,7 @@ int AudioProcessingImpl::InitializeLocked() {
   public_submodules_->level_estimator->Initialize();
   InitializeLevelController();
   InitializeResidualEchoDetector();
+  InitializeEchoCanceller3();
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_dump_.debug_file->is_open()) {
@@ -561,7 +576,9 @@ int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) {
           submodule_states_.RenderMultiBandSubModulesActive());
   // TODO(aluebs): Remove this restriction once we figure out why the 3-band
   // splitting filter degrades the AEC performance.
-  if (render_processing_rate > kSampleRate32kHz) {
+  // TODO(peah): Verify that the band splitting is needed for the AEC3.
+  if (render_processing_rate > kSampleRate32kHz &&
+      !capture_nonlocked_.echo_canceller3_enabled) {
     render_processing_rate = submodule_states_.RenderMultiBandProcessingActive()
                                  ? kSampleRate32kHz
                                  : kSampleRate16kHz;
@@ -629,6 +646,24 @@ void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) {
 
   LOG(LS_INFO) << "Highpass filter activated: "
                << config_.high_pass_filter.enabled;
+
+  config_ok = EchoCanceller3::Validate(config.echo_canceller3);
+  if (!config_ok) {
+    LOG(LS_ERROR) << "AudioProcessing module config error" << std::endl
+                  << "echo canceller 3: "
+                  << EchoCanceller3::ToString(config.echo_canceller3)
+                  << std::endl
+                  << "Reverting to default parameter set";
+    AudioProcessing::Config::EchoCanceller3 default_config;

[aleloi, 2016/12/09 14:11:06]

Should default_config be assigned to config_.echo_canceller3 here?

[peah-webrtc, 2016/12/12 19:46:46]

Thanks! Good find! This made me find some other errors as well that I corrected
in the code above.

Done.

+  }
+
+  if (config.echo_canceller3.enabled !=
+      capture_nonlocked_.echo_canceller3_enabled) {
+    capture_nonlocked_.echo_canceller3_enabled = config.echo_canceller3.enabled;
+    InitializeEchoCanceller3();
+    LOG(LS_INFO) << "Echo canceller 3 activated: "
+                 << capture_nonlocked_.echo_canceller3_enabled;
+  }
 }
 
 void AudioProcessingImpl::SetExtraOptions(const webrtc::Config& config) {
@@ -1108,6 +1143,10 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
                                 levels.peak, 1, RmsLevel::kMinLevelDb, 64);
   }
 
+  if (private_submodules_->echo_canceller3) {
+    private_submodules_->echo_canceller3->AnalyzeCapture(capture_buffer);
+  }
+
   if (constants_.use_experimental_agc &&
       public_submodules_->gain_control->is_enabled()) {
     private_submodules_->agc_manager->AnalyzePreProcess(
@@ -1128,7 +1167,9 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
     capture_buffer->set_num_channels(1);
   }
 
-  if (private_submodules_->low_cut_filter) {
+  // TODO(peah): Move the AEC3 low-cut filter to this place.
+  if (private_submodules_->low_cut_filter &&
+      (!private_submodules_->echo_canceller3)) {

[aleloi, 2016/12/09 14:11:06]

Superfluous parentheses.

[ivoc, 2016/12/09 14:11:09]

The braces are not needed here, it's okay to use "if (a && !b)"

[peah-webrtc, 2016/12/12 19:46:46]

Done.

[peah-webrtc, 2016/12/12 19:46:46]

Done.

     private_submodules_->low_cut_filter->Process(capture_buffer);
   }
   RETURN_ON_ERR(
@@ -1142,6 +1183,10 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
     return AudioProcessing::kStreamParameterNotSetError;
   }
 
+  if (private_submodules_->echo_canceller3) {
+    private_submodules_->echo_canceller3->ProcessCapture(capture_buffer, false);
+  }
+
   RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio(
       capture_buffer, stream_delay_ms()));
 
@@ -1381,6 +1426,12 @@ int AudioProcessingImpl::ProcessRenderStreamLocked() {
 #endif
 
   QueueRenderAudio(render_buffer);
+  // TODO(peah): Perform the queueing ínside QueueRenderAudiuo().
+  if (private_submodules_->echo_canceller3) {
+    if (!private_submodules_->echo_canceller3->AnalyzeRender(render_buffer)) {
+      // TODO(peah): Lock and empty render queue, and try again.
+    }
+  }
 
   if (submodule_states_.RenderMultiBandProcessingActive() &&
       SampleRateSupportsMultiBand(
@@ -1604,6 +1655,7 @@ bool AudioProcessingImpl::UpdateActiveSubmoduleStates() {
       capture_nonlocked_.beamformer_enabled,
       public_submodules_->gain_control->is_enabled(),
       capture_nonlocked_.level_controller_enabled,
+      capture_nonlocked_.echo_canceller3_enabled,
       public_submodules_->voice_detection->is_enabled(),
       public_submodules_->level_estimator->is_enabled(),
       capture_.transient_suppressor_enabled);
@@ -1652,6 +1704,14 @@ void AudioProcessingImpl::InitializeLowCutFilter() {
     private_submodules_->low_cut_filter.reset();
   }
 }
+void AudioProcessingImpl::InitializeEchoCanceller3() {
+  if (capture_nonlocked_.echo_canceller3_enabled) {
+    private_submodules_->echo_canceller3.reset(
+        new EchoCanceller3(proc_sample_rate_hz(), true));
+  } else {
+    private_submodules_->echo_canceller3.reset();
+  }
+}
 
 void AudioProcessingImpl::InitializeLevelController() {
   private_submodules_->level_controller->Initialize(proc_sample_rate_hz());
@@ -1857,6 +1917,9 @@ int AudioProcessingImpl::WriteConfigMessage(bool forced) {
   if (constants_.agc_clipped_level_min != kClippedLevelMin) {
     experiments_description += "AgcClippingLevelExperiment;";
   }
+  if (capture_nonlocked_.echo_canceller3_enabled) {
+    experiments_description += "EchoCanceller3;";
+  }
   config.set_experiments_description(experiments_description);
 
   std::string serialized_config = config.SerializeAsString();