Major AEC3 render pipeline changes

webrtc/modules/audio_processing/aec3/echo_remover.cc

 /*
  *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include "webrtc/modules/audio_processing/aec3/echo_remover.h"
 
 #include <algorithm>
 #include <memory>
 #include <numeric>
 #include <string>
 
 #include "webrtc/base/array_view.h"
 #include "webrtc/base/atomicops.h"
 #include "webrtc/base/constructormagic.h"
 #include "webrtc/modules/audio_processing/aec3/aec3_common.h"
 #include "webrtc/modules/audio_processing/aec3/aec_state.h"
 #include "webrtc/modules/audio_processing/aec3/comfort_noise_generator.h"
 #include "webrtc/modules/audio_processing/aec3/echo_path_variability.h"
 #include "webrtc/modules/audio_processing/aec3/echo_remover_metrics.h"
-#include "webrtc/modules/audio_processing/aec3/fft_buffer.h"
 #include "webrtc/modules/audio_processing/aec3/fft_data.h"
 #include "webrtc/modules/audio_processing/aec3/output_selector.h"
 #include "webrtc/modules/audio_processing/aec3/power_echo_model.h"
+#include "webrtc/modules/audio_processing/aec3/render_buffer.h"
 #include "webrtc/modules/audio_processing/aec3/render_delay_buffer.h"
 #include "webrtc/modules/audio_processing/aec3/residual_echo_estimator.h"
 #include "webrtc/modules/audio_processing/aec3/subtractor.h"
 #include "webrtc/modules/audio_processing/aec3/suppression_filter.h"
 #include "webrtc/modules/audio_processing/aec3/suppression_gain.h"
 #include "webrtc/modules/audio_processing/logging/apm_data_dumper.h"
 
 namespace webrtc {
 
 namespace {
@@ skipping 14 matching lines @@
 
 // Class for removing the echo from the capture signal.
 class EchoRemoverImpl final : public EchoRemover {
  public:
   explicit EchoRemoverImpl(int sample_rate_hz);
   ~EchoRemoverImpl() override;
 
   // Removes the echo from a block of samples from the capture signal. The
   // supplied render signal is assumed to be pre-aligned with the capture
   // signal.
-  void ProcessBlock(
+  void ProcessCapture(
       const rtc::Optional<size_t>& external_echo_path_delay_estimate,
       const EchoPathVariability& echo_path_variability,
       bool capture_signal_saturation,
-      const std::vector<std::vector<float>>& render,
+      const RenderBuffer& render_buffer,
       std::vector<std::vector<float>>* capture) override;
 
   // Updates the status on whether echo leakage is detected in the output of the
   // echo remover.
   void UpdateEchoLeakageStatus(bool leakage_detected) override {
     echo_leakage_detected_ = leakage_detected;
   }
 
  private:
   static int instance_count_;
   const Aec3Fft fft_;
   std::unique_ptr<ApmDataDumper> data_dumper_;
   const Aec3Optimization optimization_;
   const int sample_rate_hz_;
   Subtractor subtractor_;
   SuppressionGain suppression_gain_;
   ComfortNoiseGenerator cng_;
   SuppressionFilter suppression_filter_;
   PowerEchoModel power_echo_model_;
-  FftBuffer X_buffer_;
+  RenderBuffer X_buffer_;
   RenderSignalAnalyzer render_signal_analyzer_;
   OutputSelector output_selector_;
   ResidualEchoEstimator residual_echo_estimator_;
   bool echo_leakage_detected_ = false;
-  std::array<float, kBlockSize> x_old_;
   AecState aec_state_;
   EchoRemoverMetrics metrics_;
 
   RTC_DISALLOW_COPY_AND_ASSIGN(EchoRemoverImpl);
 };
 
 int EchoRemoverImpl::instance_count_ = 0;
 
 EchoRemoverImpl::EchoRemoverImpl(int sample_rate_hz)
     : fft_(),
       data_dumper_(
           new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
       optimization_(DetectOptimization()),
       sample_rate_hz_(sample_rate_hz),
       subtractor_(data_dumper_.get(), optimization_),
       suppression_gain_(optimization_),
       cng_(optimization_),
       suppression_filter_(sample_rate_hz_),
       X_buffer_(optimization_,
+                NumBandsForRate(sample_rate_hz_),
                 std::max(subtractor_.MinFarendBufferLength(),
                          power_echo_model_.MinFarendBufferLength()),
                 subtractor_.NumBlocksInRenderSums()) {
   RTC_DCHECK(ValidFullBandRate(sample_rate_hz));
-  x_old_.fill(0.f);
 }
 
 EchoRemoverImpl::~EchoRemoverImpl() = default;
 
-void EchoRemoverImpl::ProcessBlock(
+void EchoRemoverImpl::ProcessCapture(
     const rtc::Optional<size_t>& echo_path_delay_samples,
     const EchoPathVariability& echo_path_variability,
     bool capture_signal_saturation,
-    const std::vector<std::vector<float>>& render,
+    const RenderBuffer& render_buffer,
     std::vector<std::vector<float>>* capture) {
-  const std::vector<std::vector<float>>& x = render;
+  const std::vector<std::vector<float>>& x = render_buffer.MostRecentBlock();
   std::vector<std::vector<float>>* y = capture;
 
   RTC_DCHECK(y);
   RTC_DCHECK_EQ(x.size(), NumBandsForRate(sample_rate_hz_));
   RTC_DCHECK_EQ(y->size(), NumBandsForRate(sample_rate_hz_));
   RTC_DCHECK_EQ(x[0].size(), kBlockSize);
   RTC_DCHECK_EQ((*y)[0].size(), kBlockSize);
   const std::vector<float>& x0 = x[0];
   std::vector<float>& y0 = (*y)[0];
 
   data_dumper_->DumpWav("aec3_processblock_capture_input", kBlockSize, &y0[0],
                         LowestBandRate(sample_rate_hz_), 1);
   data_dumper_->DumpWav("aec3_processblock_render_input", kBlockSize, &x0[0],
                         LowestBandRate(sample_rate_hz_), 1);
 
   aec_state_.UpdateCaptureSaturation(capture_signal_saturation);
 
   if (echo_path_variability.AudioPathChanged()) {
     subtractor_.HandleEchoPathChange(echo_path_variability);
-    power_echo_model_.HandleEchoPathChange(echo_path_variability);

[peah-webrtc, 2017/03/30 05:36:56]

I removed this since it caused a build error, and will anyway be removed in the
next CL.

     residual_echo_estimator_.HandleEchoPathChange(echo_path_variability);
   }
 
   std::array<float, kFftLengthBy2Plus1> Y2;
   std::array<float, kFftLengthBy2Plus1> S2_power;
   std::array<float, kFftLengthBy2Plus1> R2;
   std::array<float, kFftLengthBy2Plus1> S2_linear;
   std::array<float, kFftLengthBy2Plus1> G;
-  FftData X;
   FftData Y;
   FftData comfort_noise;
   FftData high_band_comfort_noise;
   SubtractorOutput subtractor_output;
   FftData& E_main = subtractor_output.E_main;
   auto& E2_main = subtractor_output.E2_main;
   auto& E2_shadow = subtractor_output.E2_shadow;
   auto& e_main = subtractor_output.e_main;
   auto& e_shadow = subtractor_output.e_shadow;
 
-  // Update the render signal buffer.
-  fft_.PaddedFft(x0, x_old_, &X);
-  X_buffer_.Insert(X);
-
   // Analyze the render signal.
-  render_signal_analyzer_.Update(X_buffer_, aec_state_.FilterDelay());
+  render_signal_analyzer_.Update(render_buffer, aec_state_.FilterDelay());
 
   // Perform linear echo cancellation.
-  subtractor_.Process(X_buffer_, y0, render_signal_analyzer_,
+  subtractor_.Process(render_buffer, y0, render_signal_analyzer_,
                       aec_state_.SaturatedCapture(), &subtractor_output);
 
   // Compute spectra.
   fft_.ZeroPaddedFft(y0, &Y);
   LinearEchoPower(E_main, Y, &S2_linear);
   Y.Spectrum(optimization_, &Y2);
 
   // Update the AEC state information.
   aec_state_.Update(subtractor_.FilterFrequencyResponse(),
-                    echo_path_delay_samples, X_buffer_, E2_main, E2_shadow, Y2,
-                    x0, echo_path_variability, echo_leakage_detected_);
+                    echo_path_delay_samples, render_buffer, E2_main, E2_shadow,
+                    Y2, x0, echo_path_variability, echo_leakage_detected_);
 
   // Use the power model to estimate the echo.
-  power_echo_model_.EstimateEcho(X_buffer_, Y2, aec_state_, &S2_power);
+  // TODO(peah): Remove in upcoming CL.
+  // power_echo_model_.EstimateEcho(render_buffer, Y2, aec_state_, &S2_power);

[peah-webrtc, 2017/03/30 05:36:56]

This will be removed in the next CL.

 
   // Choose the linear output.
   output_selector_.FormLinearOutput(e_main, y0);
   data_dumper_->DumpWav("aec3_output_linear", kBlockSize, &y0[0],
                         LowestBandRate(sample_rate_hz_), 1);
   const auto& E2 = output_selector_.UseSubtractorOutput() ? E2_main : Y2;
 
   // Estimate the residual echo power.
   residual_echo_estimator_.Estimate(
-      output_selector_.UseSubtractorOutput(), aec_state_, X_buffer_,
+      output_selector_.UseSubtractorOutput(), aec_state_, render_buffer,
       subtractor_.FilterFrequencyResponse(), E2_main, E2_shadow, S2_linear,
       S2_power, Y2, &R2);
 
   // Estimate the comfort noise.
   cng_.Compute(aec_state_, Y2, &comfort_noise, &high_band_comfort_noise);
 
   // Detect basic doubletalk.
   const bool doubletalk = BlockPower(e_shadow) < BlockPower(e_main);
 
   // A choose and apply echo suppression gain.
@@ skipping 39 matching lines @@
                         aec_state_.SaturatedCapture() ? 1 : 0);
 }
 
 }  // namespace
 
 EchoRemover* EchoRemover::Create(int sample_rate_hz) {
   return new EchoRemoverImpl(sample_rate_hz);
 }
 
 }  // namespace webrtc