Moving the residual echo detector outside of band-scheme in APM

webrtc/modules/audio_processing/audio_processing_impl.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 104 matching lines @@
       return uppermost_native_rate;
     }
     if (rate >= minimum_rate) {
       return rate;
     }
   }
   RTC_NOTREACHED();
   return uppermost_native_rate;
 }
 
-// Maximum length that a frame of samples can have.
-static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160;
+// Maximum lengths that frame of samples being passed from the render side to
+// the capture side can have (does not apply to AEC3).
+static const size_t kMaxAllowedValuesOfSamplesPerBand = 160;
+static const size_t kMaxAllowedValuesOfSamplesPerFrame = 480;
+
 // Maximum number of frames to buffer in the render queue.
 // TODO(peah): Decrease this once we properly handle hugely unbalanced
 // reverse and forward call numbers.
 static const size_t kMaxNumFramesToBuffer = 100;
 
 class HighPassFilterImpl : public HighPassFilter {
  public:
   explicit HighPassFilterImpl(AudioProcessingImpl* apm) : apm_(apm) {}
   ~HighPassFilterImpl() override = default;
 
@@ skipping 701 matching lines @@
       msg->add_output_channel(dest[i], channel_size);
     RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
                                           &debug_dump_.num_bytes_left_for_log_,
                                           &crit_debug_, &debug_dump_.capture));
   }
 #endif
 
   return kNoError;
 }
 
-void AudioProcessingImpl::QueueRenderAudio(AudioBuffer* audio) {
+void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) {
   EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(),
                                               num_reverse_channels(),
                                               &aec_render_queue_buffer_);
 
   RTC_DCHECK_GE(160, audio->num_frames_per_band());
 
   // Insert the samples into the queue.
   if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) {
     // The data queue is full and needs to be emptied.
     EmptyQueuedRenderAudio();
@@ skipping 22 matching lines @@
     // Insert the samples into the queue.
     if (!agc_render_signal_queue_->Insert(&agc_render_queue_buffer_)) {
       // The data queue is full and needs to be emptied.
       EmptyQueuedRenderAudio();
 
       // Retry the insert (should always work).
       bool result = agc_render_signal_queue_->Insert(&agc_render_queue_buffer_);
       RTC_DCHECK(result);
     }
   }
+}
 
+void AudioProcessingImpl::QueueNonbandedRenderAudio(AudioBuffer* audio) {
   ResidualEchoDetector::PackRenderAudioBuffer(audio, &red_render_queue_buffer_);
 
   // Insert the samples into the queue.
   if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) {
     // The data queue is full and needs to be emptied.
     EmptyQueuedRenderAudio();
 
     // Retry the insert (should always work).
     bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_);
     RTC_DCHECK(result);
   }
 }
 
 void AudioProcessingImpl::AllocateRenderQueue() {
   const size_t new_aec_render_queue_element_max_size =
       std::max(static_cast<size_t>(1),
-               kMaxAllowedValuesOfSamplesPerFrame *
+               kMaxAllowedValuesOfSamplesPerBand *
                    EchoCancellationImpl::NumCancellersRequired(
                        num_output_channels(), num_reverse_channels()));
 
   const size_t new_aecm_render_queue_element_max_size =
       std::max(static_cast<size_t>(1),
-               kMaxAllowedValuesOfSamplesPerFrame *
+               kMaxAllowedValuesOfSamplesPerBand *
                    EchoControlMobileImpl::NumCancellersRequired(
                        num_output_channels(), num_reverse_channels()));
 
   const size_t new_agc_render_queue_element_max_size =
-      std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame);
+      std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerBand);
 
   const size_t new_red_render_queue_element_max_size =
       std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame);
 
   // Reallocate the queues if the queue item sizes are too small to fit the
   // data to put in the queues.
   if (aec_render_queue_element_max_size_ <
       new_aec_render_queue_element_max_size) {
     aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size;
 
@@ skipping 298 matching lines @@
   // Ensure that the stream delay was set before the call to the
   // AECM ProcessCaptureAudio function.
   if (public_submodules_->echo_control_mobile->is_enabled() &&
       !was_stream_delay_set()) {
     return AudioProcessing::kStreamParameterNotSetError;
   }
 
   RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessCaptureAudio(
       capture_buffer, stream_delay_ms()));
 
-  if (config_.residual_echo_detector.enabled) {
-    private_submodules_->residual_echo_detector->AnalyzeCaptureAudio(
-        rtc::ArrayView<const float>(
-            capture_buffer->split_bands_const_f(0)[kBand0To8kHz],
-            capture_buffer->num_frames_per_band()));
-  }
 
   if (capture_nonlocked_.beamformer_enabled) {
     private_submodules_->beamformer->PostFilter(capture_buffer->split_data_f());
   }
 
   public_submodules_->voice_detection->ProcessCaptureAudio(capture_buffer);
 
   if (constants_.use_experimental_agc &&
       public_submodules_->gain_control->is_enabled() &&
       (!capture_nonlocked_.beamformer_enabled ||
        private_submodules_->beamformer->is_target_present())) {
     private_submodules_->agc_manager->Process(
         capture_buffer->split_bands_const(0)[kBand0To8kHz],
         capture_buffer->num_frames_per_band(), capture_nonlocked_.split_rate);
   }
   RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio(
       capture_buffer, echo_cancellation()->stream_has_echo()));
 
   if (submodule_states_.CaptureMultiBandProcessingActive() &&
       SampleRateSupportsMultiBand(
           capture_nonlocked_.capture_processing_format.sample_rate_hz())) {
     capture_buffer->MergeFrequencyBands();
   }
 
+  if (config_.residual_echo_detector.enabled) {
+    private_submodules_->residual_echo_detector->AnalyzeCaptureAudio(
+        rtc::ArrayView<const float>(capture_buffer->channels_f()[0],
+                                    capture_buffer->num_frames()));
+  }
+
   // TODO(aluebs): Investigate if the transient suppression placement should be
   // before or after the AGC.
   if (capture_.transient_suppressor_enabled) {
     float voice_probability =
         private_submodules_->agc_manager.get()
             ? private_submodules_->agc_manager->voice_probability()
             : 1.f;
 
     public_submodules_->transient_suppressor->Suppress(
         capture_buffer->channels_f()[0], capture_buffer->num_frames(),
@@ skipping 153 matching lines @@
 #endif
   render_.render_audio->DeinterleaveFrom(frame);
   RETURN_ON_ERR(ProcessRenderStreamLocked());
   render_.render_audio->InterleaveTo(
       frame, submodule_states_.RenderMultiBandProcessingActive());
   return kNoError;
 }
 
 int AudioProcessingImpl::ProcessRenderStreamLocked() {
   AudioBuffer* render_buffer = render_.render_audio.get();  // For brevity.
+
+  QueueNonbandedRenderAudio(render_buffer);
+
   if (submodule_states_.RenderMultiBandSubModulesActive() &&
       SampleRateSupportsMultiBand(
           formats_.render_processing_format.sample_rate_hz())) {
     render_buffer->SplitIntoFrequencyBands();
   }
 
 #if WEBRTC_INTELLIGIBILITY_ENHANCER
   if (capture_nonlocked_.intelligibility_enabled) {
     public_submodules_->intelligibility_enhancer->ProcessRenderAudio(
         render_buffer);
   }
 #endif
 
-  QueueRenderAudio(render_buffer);
+  QueueBandedRenderAudio(render_buffer);
   // TODO(peah): Perform the queueing ínside QueueRenderAudiuo().
   if (private_submodules_->echo_canceller3) {
     private_submodules_->echo_canceller3->AnalyzeRender(render_buffer);
   }
 
   if (submodule_states_.RenderMultiBandProcessingActive() &&
       SampleRateSupportsMultiBand(
           formats_.render_processing_format.sample_rate_hz())) {
     render_buffer->MergeFrequencyBands();
   }
@@ skipping 538 matching lines @@
       previous_agc_level(0),
       echo_path_gain_change(false) {}
 
 AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default;
 
 AudioProcessingImpl::ApmRenderState::ApmRenderState() = default;
 
 AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default;
 
 }  // namespace webrtc