Dont always downsample to 16kHz in the reverse stream in APM

webrtc/modules/audio_processing/audio_processing_impl.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 41 matching lines @@
 
 #define RETURN_ON_ERR(expr) \
   do {                      \
     int err = (expr);       \
     if (err != kNoError) {  \
       return err;           \
     }                       \
   } while (0)
 
 namespace webrtc {
+
+const int AudioProcessing::kNativeSampleRatesHz[] = {
+    AudioProcessing::kSampleRate8kHz,
+    AudioProcessing::kSampleRate16kHz,
+    AudioProcessing::kSampleRate32kHz,
+    AudioProcessing::kSampleRate48kHz};
+const size_t AudioProcessing::kNumNativeSampleRates =
+    arraysize(AudioProcessing::kNativeSampleRatesHz);
+const int AudioProcessing::kMaxNativeSampleRateHz = AudioProcessing::
+    kNativeSampleRatesHz[AudioProcessing::kNumNativeSampleRates - 1];

[peah-webrtc, 2016/03/10 06:59:24]

Afaics, this means that the tri-band 48 kHz splitting filter can be used on any
platform, right? What about the complexity for that on ARM, or is that addressed
in another CL?

[aluebs-webrtc, 2016/03/10 15:34:32]

That is adressed in this CL that already landed:
https://codereview.webrtc.org/1766103002/
This CL is going to need a rebase and conflict fixing before landing :)

+
 namespace {
 
 static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) {
   switch (layout) {
     case AudioProcessing::kMono:
     case AudioProcessing::kStereo:
       return false;
     case AudioProcessing::kMonoAndKeyboard:
     case AudioProcessing::kStereoAndKeyboard:
       return true;
   }
 
   assert(false);
   return false;
 }
+
+bool MultiBandSignal(int sample_rate_hz) {
+  return sample_rate_hz == AudioProcessing::kSampleRate32kHz ||
+         sample_rate_hz == AudioProcessing::kSampleRate48kHz;
+}
+
+int ClosestNativeRate(int min_proc_rate) {
+  for (int rate : AudioProcessing::kNativeSampleRatesHz) {
+    if (rate >= min_proc_rate) {
+      return rate;
+    }
+  }
+  return AudioProcessing::kMaxNativeSampleRateHz;
+}
+
 }  // namespace
 
 // Throughout webrtc, it's assumed that success is represented by zero.
 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero");
 
 struct AudioProcessingImpl::ApmPublicSubmodules {
   ApmPublicSubmodules()
       : echo_cancellation(nullptr),
         echo_control_mobile(nullptr),
         gain_control(nullptr) {}
@@ skipping 15 matching lines @@
 
 struct AudioProcessingImpl::ApmPrivateSubmodules {
   explicit ApmPrivateSubmodules(Beamformer<float>* beamformer)
       : beamformer(beamformer) {}
   // Accessed internally from capture or during initialization
   std::list<ProcessingComponent*> component_list;
   std::unique_ptr<Beamformer<float>> beamformer;
   std::unique_ptr<AgcManagerDirect> agc_manager;
 };
 
-const int AudioProcessing::kNativeSampleRatesHz[] = {
-    AudioProcessing::kSampleRate8kHz,
-    AudioProcessing::kSampleRate16kHz,
-    AudioProcessing::kSampleRate32kHz,
-    AudioProcessing::kSampleRate48kHz};
-const size_t AudioProcessing::kNumNativeSampleRates =
-    arraysize(AudioProcessing::kNativeSampleRatesHz);
-const int AudioProcessing::kMaxNativeSampleRateHz = AudioProcessing::
-    kNativeSampleRatesHz[AudioProcessing::kNumNativeSampleRates - 1];
-
 AudioProcessing* AudioProcessing::Create() {
   Config config;
   return Create(config, nullptr);
 }
 
 AudioProcessing* AudioProcessing::Create(const Config& config) {
   return Create(config, nullptr);
 }
 
 AudioProcessing* AudioProcessing::Create(const Config& config,
@@ skipping 225 matching lines @@
     return kBadNumberChannelsError;
   }
 
   if (capture_nonlocked_.beamformer_enabled &&
       num_in_channels != capture_.array_geometry.size()) {
     return kBadNumberChannelsError;
   }
 
   formats_.api_format = config;
 
-  // We process at the closest native rate >= min(input rate, output rate).
-  const int min_proc_rate =
-      std::min(formats_.api_format.input_stream().sample_rate_hz(),
-               formats_.api_format.output_stream().sample_rate_hz());
-  int fwd_proc_rate;
-  for (size_t i = 0; i < kNumNativeSampleRates; ++i) {
-    fwd_proc_rate = kNativeSampleRatesHz[i];
-    if (fwd_proc_rate >= min_proc_rate) {
-      break;
-    }
-  }
+  capture_nonlocked_.fwd_proc_format = StreamConfig(ClosestNativeRate(std::min(
+      formats_.api_format.input_stream().sample_rate_hz(),
+      formats_.api_format.output_stream().sample_rate_hz())));
 
-  capture_nonlocked_.fwd_proc_format = StreamConfig(fwd_proc_rate);
-
-  // We normally process the reverse stream at 16 kHz. Unless...
-  int rev_proc_rate = kSampleRate16kHz;
+  int rev_proc_rate = ClosestNativeRate(std::min(
+      formats_.api_format.reverse_input_stream().sample_rate_hz(),
+      formats_.api_format.reverse_output_stream().sample_rate_hz()));

[peah-webrtc, 2016/03/10 06:59:24]

This change basically means that the tri-band 48 kHz splitting filter is
activated for the reverse stream, right?

I should know this, but as the CL description says "Dont always downsample to
16kHz in the reverse stream in APM" I'm a bit confused. The code did not always
downsample to 16 lHz before the change either, right? For 32 kHz the 2-band
splitting filter was used, right?

[aluebs-webrtc, 2016/03/10 15:34:32]

That is right. But in that case 32kHz was more of an exception, because the
2-band splitting filter was better than the downsampler.

+  // If the forward sample rate is 8 kHz, the reverse stream is also processed
+  // at this rate.
   if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate8kHz) {
-    // ...the forward stream is at 8 kHz.
     rev_proc_rate = kSampleRate8kHz;
-  } else {
-    if (formats_.api_format.reverse_input_stream().sample_rate_hz() ==
-        kSampleRate32kHz) {
-      // ...or the input is at 32 kHz, in which case we use the splitting
-      // filter rather than the resampler.
-      rev_proc_rate = kSampleRate32kHz;
-    }
   }
 
   // Always downmix the reverse stream to mono for analysis. This has been
   // demonstrated to work well for AEC in most practical scenarios.
   formats_.rev_proc_format = StreamConfig(rev_proc_rate, 1);
 
   if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate32kHz ||
       capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate48kHz) {
     capture_nonlocked_.split_rate = kSampleRate16kHz;
   } else {
@@ skipping 238 matching lines @@
     debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM);
     audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
     const size_t data_size =
         sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_;
     msg->set_input_data(frame->data_, data_size);
   }
 #endif
 
   capture_.capture_audio->DeinterleaveFrom(frame);
   RETURN_ON_ERR(ProcessStreamLocked());
-  capture_.capture_audio->InterleaveTo(frame,
-                                       output_copy_needed(is_data_processed()));
+  capture_.capture_audio->InterleaveTo(frame, output_copy_needed());
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_dump_.debug_file->Open()) {
     audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
     const size_t data_size =
         sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_;
     msg->set_output_data(frame->data_, data_size);
     RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
                                           &debug_dump_.num_bytes_left_for_log_,
                                           &crit_debug_, &debug_dump_.capture));
@@ skipping 19 matching lines @@
 
   AudioBuffer* ca = capture_.capture_audio.get();  // For brevity.
 
   if (constants_.use_experimental_agc &&
       public_submodules_->gain_control->is_enabled()) {
     private_submodules_->agc_manager->AnalyzePreProcess(
         ca->channels()[0], ca->num_channels(),
         capture_nonlocked_.fwd_proc_format.num_frames());
   }
 
-  bool data_processed = is_data_processed();
-  if (analysis_needed(data_processed)) {
+  if (fwd_analysis_needed()) {
     ca->SplitIntoFrequencyBands();
   }
 
   if (capture_nonlocked_.beamformer_enabled) {
     private_submodules_->beamformer->ProcessChunk(*ca->split_data_f(),
                                                   ca->split_data_f());
     ca->set_num_channels(1);
   }
 
   public_submodules_->high_pass_filter->ProcessCaptureAudio(ca);
@@ skipping 18 matching lines @@
   if (constants_.use_experimental_agc &&
       public_submodules_->gain_control->is_enabled() &&
       (!capture_nonlocked_.beamformer_enabled ||
        private_submodules_->beamformer->is_target_present())) {
     private_submodules_->agc_manager->Process(
         ca->split_bands_const(0)[kBand0To8kHz], ca->num_frames_per_band(),
         capture_nonlocked_.split_rate);
   }
   RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio(ca));
 
-  if (synthesis_needed(data_processed)) {
+  if (fwd_synthesis_needed()) {
     ca->MergeFrequencyBands();
   }
 
   // TODO(aluebs): Investigate if the transient suppression placement should be
   // before or after the AGC.
   if (capture_.transient_suppressor_enabled) {
     float voice_probability =
         private_submodules_->agc_manager.get()
             ? private_submodules_->agc_manager->voice_probability()
             : 1.f;
@@ skipping 154 matching lines @@
                                           &debug_dump_.num_bytes_left_for_log_,
                                           &crit_debug_, &debug_dump_.render));
   }
 #endif
   render_.render_audio->DeinterleaveFrom(frame);
   return ProcessReverseStreamLocked();
 }
 
 int AudioProcessingImpl::ProcessReverseStreamLocked() {
   AudioBuffer* ra = render_.render_audio.get();  // For brevity.
-  if (formats_.rev_proc_format.sample_rate_hz() == kSampleRate32kHz) {
+  if (rev_analysis_needed()) {
     ra->SplitIntoFrequencyBands();
   }
 
   if (constants_.intelligibility_enabled) {
     // Currently run in single-threaded mode when the intelligibility
     // enhancer is activated.
     // TODO(peah): Fix to be properly multi-threaded.
     rtc::CritScope cs(&crit_capture_);
     public_submodules_->intelligibility_enhancer->ProcessRenderAudio(
         ra->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate,
         ra->num_channels());
   }
 
   RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessRenderAudio(ra));
   RETURN_ON_ERR(
       public_submodules_->echo_control_mobile->ProcessRenderAudio(ra));
   if (!constants_.use_experimental_agc) {
     RETURN_ON_ERR(public_submodules_->gain_control->ProcessRenderAudio(ra));
   }
 
-  if (formats_.rev_proc_format.sample_rate_hz() == kSampleRate32kHz &&
-      is_rev_processed()) {
+  if (rev_synthesis_needed()) {
     ra->MergeFrequencyBands();
   }
 
   return kNoError;
 }
 
 int AudioProcessingImpl::set_stream_delay_ms(int delay) {
   rtc::CritScope cs(&crit_capture_);
   Error retval = kNoError;
   capture_.was_stream_delay_set = true;
@@ skipping 191 matching lines @@
   for (auto item : private_submodules_->component_list) {
     if (item->is_component_enabled()) {
       return true;
     }
   }
 
   // The capture data is otherwise unchanged.
   return false;
 }
 
-bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const {
+bool AudioProcessingImpl::output_copy_needed() const {
   // Check if we've upmixed or downmixed the audio.
   return ((formats_.api_format.output_stream().num_channels() !=
            formats_.api_format.input_stream().num_channels()) ||
-          is_data_processed || capture_.transient_suppressor_enabled);
+          is_data_processed() || capture_.transient_suppressor_enabled);
 }
 
-bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const {
-  return (is_data_processed &&
-          (capture_nonlocked_.fwd_proc_format.sample_rate_hz() ==
-               kSampleRate32kHz ||
-           capture_nonlocked_.fwd_proc_format.sample_rate_hz() ==
-               kSampleRate48kHz));
+bool AudioProcessingImpl::fwd_synthesis_needed() const {
+  return (is_data_processed() &&
+          MultiBandSignal(capture_nonlocked_.fwd_proc_format.sample_rate_hz()));
 }
 
-bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const {
-  if (!is_data_processed &&
+bool AudioProcessingImpl::fwd_analysis_needed() const {
+  if (!is_data_processed() &&
       !public_submodules_->voice_detection->is_enabled() &&
       !capture_.transient_suppressor_enabled) {
     // Only public_submodules_->level_estimator is enabled.
     return false;
-  } else if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() ==
-                 kSampleRate32kHz ||
-             capture_nonlocked_.fwd_proc_format.sample_rate_hz() ==
-                 kSampleRate48kHz) {
+  } else if (MultiBandSignal(
+                 capture_nonlocked_.fwd_proc_format.sample_rate_hz())) {
     // Something besides public_submodules_->level_estimator is enabled, and we
     // have super-wb.
     return true;
   }
   return false;
 }
 
 bool AudioProcessingImpl::is_rev_processed() const {
   return constants_.intelligibility_enabled;
 }
 
+bool AudioProcessingImpl::rev_synthesis_needed() const {
+  return (is_rev_processed() &&
+          MultiBandSignal(formats_.rev_proc_format.sample_rate_hz()));
+}
+
+bool AudioProcessingImpl::rev_analysis_needed() const {
+  return MultiBandSignal(formats_.rev_proc_format.sample_rate_hz());
+}
+
 bool AudioProcessingImpl::render_check_rev_conversion_needed() const {
   return rev_conversion_needed();
 }
 
 bool AudioProcessingImpl::rev_conversion_needed() const {
   return (formats_.api_format.reverse_input_stream() !=
           formats_.api_format.reverse_output_stream());
 }
 
 void AudioProcessingImpl::InitializeExperimentalAgc() {
@@ skipping 257 matching lines @@
   debug_dump_.capture.event_msg->mutable_config()->CopyFrom(config);
 
   RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
                                         &debug_dump_.num_bytes_left_for_log_,
                                         &crit_debug_, &debug_dump_.capture));
   return kNoError;
 }
 #endif  // WEBRTC_AUDIOPROC_DEBUG_DUMP
 
 }  // namespace webrtc