New module for the adaptive level controlling functionality in the audio processing module

webrtc/modules/audio_processing/audio_processing_impl.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 13 matching lines @@
 #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h"
 #include "webrtc/modules/audio_processing/audio_buffer.h"
 #include "webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h"
 #include "webrtc/modules/audio_processing/common.h"
 #include "webrtc/modules/audio_processing/echo_cancellation_impl.h"
 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h"
 #include "webrtc/modules/audio_processing/gain_control_for_experimental_agc.h"
 #include "webrtc/modules/audio_processing/gain_control_impl.h"
 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h"
 #include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h"
+#include "webrtc/modules/audio_processing/level_controller/level_controller.h"
 #include "webrtc/modules/audio_processing/level_estimator_impl.h"
 #include "webrtc/modules/audio_processing/noise_suppression_impl.h"
 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h"
 #include "webrtc/modules/audio_processing/voice_detection_impl.h"
 #include "webrtc/modules/include/module_common_types.h"
 #include "webrtc/system_wrappers/include/file_wrapper.h"
 #include "webrtc/system_wrappers/include/logging.h"
 #include "webrtc/system_wrappers/include/metrics.h"
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
@@ skipping 81 matching lines @@
   std::unique_ptr<TransientSuppressor> transient_suppressor;
   std::unique_ptr<IntelligibilityEnhancer> intelligibility_enhancer;
 };
 
 struct AudioProcessingImpl::ApmPrivateSubmodules {
   explicit ApmPrivateSubmodules(Beamformer<float>* beamformer)
       : beamformer(beamformer) {}
   // Accessed internally from capture or during initialization
   std::unique_ptr<Beamformer<float>> beamformer;
   std::unique_ptr<AgcManagerDirect> agc_manager;
+  std::unique_ptr<LevelController> level_controller;
 };
 
 AudioProcessing* AudioProcessing::Create() {
   Config config;
   return Create(config, nullptr);
 }
 
 AudioProcessing* AudioProcessing::Create(const Config& config) {
   return Create(config, nullptr);
 }
@@ skipping 23 matching lines @@
                  config.Get<ExperimentalAgc>().enabled),
 #endif
 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
       capture_(false,
 #else
       capture_(config.Get<ExperimentalNs>().enabled,
 #endif
                config.Get<Beamforming>().array_geometry,
                config.Get<Beamforming>().target_direction),
       capture_nonlocked_(config.Get<Beamforming>().enabled,
-                         config.Get<Intelligibility>().enabled)
-{
+                         config.Get<Intelligibility>().enabled,
+                         config.Get<LevelControl>().enabled) {
   {
     rtc::CritScope cs_render(&crit_render_);
     rtc::CritScope cs_capture(&crit_capture_);
 
     public_submodules_->echo_cancellation.reset(
         new EchoCancellationImpl(&crit_render_, &crit_capture_));
     public_submodules_->echo_control_mobile.reset(
         new EchoControlMobileImpl(&crit_render_, &crit_capture_));
     public_submodules_->gain_control.reset(
         new GainControlImpl(&crit_capture_, &crit_capture_));
     public_submodules_->high_pass_filter.reset(
         new HighPassFilterImpl(&crit_capture_));
     public_submodules_->level_estimator.reset(
         new LevelEstimatorImpl(&crit_capture_));
     public_submodules_->noise_suppression.reset(
         new NoiseSuppressionImpl(&crit_capture_));
     public_submodules_->voice_detection.reset(
         new VoiceDetectionImpl(&crit_capture_));
     public_submodules_->gain_control_for_experimental_agc.reset(
         new GainControlForExperimentalAgc(
             public_submodules_->gain_control.get(), &crit_capture_));
+
+    private_submodules_->level_controller.reset(new LevelController());
   }
 
   SetExtraOptions(config);
 }
 
 AudioProcessingImpl::~AudioProcessingImpl() {
   // Depends on gain_control_ and
   // public_submodules_->gain_control_for_experimental_agc.
   private_submodules_->agc_manager.reset();
   // Depends on gain_control_.
@@ skipping 104 matching lines @@
   InitializeEchoCanceller();
   InitializeEchoControlMobile();
   InitializeExperimentalAgc();
   InitializeTransient();
   InitializeBeamformer();
   InitializeIntelligibility();
   InitializeHighPassFilter();
   InitializeNoiseSuppression();
   InitializeLevelEstimator();
   InitializeVoiceDetection();
+  InitializeLevelController();
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_dump_.debug_file->is_open()) {
     int err = WriteInitMessage();
     if (err != kNoError) {
       return err;
     }
   }
 #endif
 
@@ skipping 66 matching lines @@
 
   public_submodules_->echo_cancellation->SetExtraOptions(config);
 
   if (capture_.transient_suppressor_enabled !=
       config.Get<ExperimentalNs>().enabled) {
     capture_.transient_suppressor_enabled =
         config.Get<ExperimentalNs>().enabled;
     InitializeTransient();
   }
 
+  if (capture_nonlocked_.level_controller_enabled !=
+      config.Get<LevelControl>().enabled) {
+    capture_nonlocked_.level_controller_enabled =
+        config.Get<LevelControl>().enabled;
+    InitializeLevelController();
+  }
+
   if(capture_nonlocked_.intelligibility_enabled !=
      config.Get<Intelligibility>().enabled) {
     capture_nonlocked_.intelligibility_enabled =
         config.Get<Intelligibility>().enabled;
     InitializeIntelligibility();
   }
 
 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD
   if (capture_nonlocked_.beamformer_enabled !=
           config.Get<Beamforming>().enabled) {
@@ skipping 331 matching lines @@
             ? private_submodules_->agc_manager->voice_probability()
             : 1.f;
 
     public_submodules_->transient_suppressor->Suppress(
         ca->channels_f()[0], ca->num_frames(), ca->num_channels(),
         ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(),
         ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability,
         capture_.key_pressed);
   }
 
+  if (capture_nonlocked_.level_controller_enabled) {
+    private_submodules_->level_controller->Process(ca);

[aluebs-webrtc, 2016/06/28 22:45:02]

I am not sure how this component works (it is not in the CL description or
documented in the header file), but if it applies an additional gain to the
signal, we probably want to compensate for that in the gain_db parameter when
calling intelligibility_enhancer->SetCaptureNoiseEstimate() in line 728.

[peah-webrtc, 2016/06/29 06:22:09]

The component indeed applies an additional gain. 

I'll add an API call for obtaining the latest applied gain, but I'll let you use
that gain in the intelligibility enhancer in another CL since
1) I don't really understand how that gain is to be used in the intelligibility
enhancer.
2) It is nice to make this CL only concern the level control and not also affect
the intelligibility enhancer.
3) The intelligibility enhancer is currently not active by default so the
addition of the level control will not cause any downstream breakages.

Lets discuss it further offline if you want.

[aluebs-webrtc, 2016/06/29 16:40:28]

2) This CL concerns the IE, because it applies a gain without compensating for
it.
3) It can be manually enabled and some of us are using it already.
But I can do a separate CL if you want.

[peah-webrtc, 2016/06/29 20:39:59]

I definitely see that it affects the IE, but I think it is better if you do that
change as you know how that is designed. Furthermore, I'd expect that that
change needs to be verified carefully, as the level controller may change the
power of the signal by as much as 20 dB.

An alternative is to ensure that the IE and the level control are never active
at the same time.

+  }
+
   // The level estimator operates on the recombined data.
   public_submodules_->level_estimator->ProcessStream(ca);
 
   capture_.was_stream_delay_set = false;
   return kNoError;
 }
 
 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data,
                                               size_t samples_per_channel,
                                               int rev_sample_rate_hz,
@@ skipping 339 matching lines @@
   }
 
   // The capture data is otherwise unchanged.
   return false;
 }
 
 bool AudioProcessingImpl::output_copy_needed() const {
   // Check if we've upmixed or downmixed the audio.
   return ((formats_.api_format.output_stream().num_channels() !=
            formats_.api_format.input_stream().num_channels()) ||
-          is_fwd_processed() || capture_.transient_suppressor_enabled);
+          is_fwd_processed() || capture_.transient_suppressor_enabled ||
+          capture_nonlocked_.level_controller_enabled);
 }
 
 bool AudioProcessingImpl::fwd_synthesis_needed() const {
   return (is_fwd_processed() &&
           is_multi_band(capture_nonlocked_.fwd_proc_format.sample_rate_hz()));
 }
 
 bool AudioProcessingImpl::fwd_analysis_needed() const {
   if (!is_fwd_processed() &&
       !public_submodules_->voice_detection->is_enabled() &&
@@ skipping 108 matching lines @@
   public_submodules_->echo_control_mobile->Initialize(
       proc_split_sample_rate_hz(),
       num_reverse_channels(),
       num_output_channels());
 }
 
 void AudioProcessingImpl::InitializeLevelEstimator() {
   public_submodules_->level_estimator->Initialize();
 }
 
+void AudioProcessingImpl::InitializeLevelController() {
+  private_submodules_->level_controller->Initialize(proc_sample_rate_hz());
+}
+
 void AudioProcessingImpl::InitializeVoiceDetection() {
   public_submodules_->voice_detection->Initialize(proc_split_sample_rate_hz());
 }
 
 void AudioProcessingImpl::MaybeUpdateHistograms() {
   static const int kMinDiffDelayMs = 60;
 
   if (echo_cancellation()->is_enabled()) {
     // Activate delay_jumps_ counters if we know echo_cancellation is runnning.
     // If a stream has echo we know that the echo_cancellation is in process.
@@ skipping 195 matching lines @@
   debug_dump_.capture.event_msg->mutable_config()->CopyFrom(config);
 
   RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
                                         &debug_dump_.num_bytes_left_for_log_,
                                         &crit_debug_, &debug_dump_.capture));
   return kNoError;
 }
 #endif  // WEBRTC_AUDIOPROC_DEBUG_DUMP
 
 }  // namespace webrtc