Surface the IntelligibilityEnhancer on MediaConstraints

webrtc/modules/audio_processing/audio_processing_impl.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 145 matching lines @@
 
 AudioProcessingImpl::AudioProcessingImpl(const Config& config)
     : AudioProcessingImpl(config, nullptr) {}
 
 AudioProcessingImpl::AudioProcessingImpl(const Config& config,
                                          Beamformer<float>* beamformer)
     : public_submodules_(new ApmPublicSubmodules()),
       private_submodules_(new ApmPrivateSubmodules(beamformer)),
       constants_(config.Get<ExperimentalAgc>().startup_min_volume,
 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
-                 false,
+                 false),
 #else
-                 config.Get<ExperimentalAgc>().enabled,
+                 config.Get<ExperimentalAgc>().enabled),
 #endif
-                 config.Get<Intelligibility>().enabled),
-
 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
       capture_(false,
 #else
       capture_(config.Get<ExperimentalNs>().enabled,
 #endif
                config.Get<Beamforming>().array_geometry,
                config.Get<Beamforming>().target_direction),
-      capture_nonlocked_(config.Get<Beamforming>().enabled)
+      capture_nonlocked_(config.Get<Beamforming>().enabled,
+                         config.Get<Intelligibility>().enabled)
 {
   {
     rtc::CritScope cs_render(&crit_render_);
     rtc::CritScope cs_capture(&crit_capture_);
 
     public_submodules_->echo_cancellation.reset(
         new EchoCancellationImpl(&crit_render_, &crit_capture_));
     public_submodules_->echo_control_mobile.reset(
         new EchoControlMobileImpl(&crit_render_, &crit_capture_));
     public_submodules_->gain_control.reset(
@@ skipping 214 matching lines @@
 
   public_submodules_->echo_cancellation->SetExtraOptions(config);
 
   if (capture_.transient_suppressor_enabled !=
       config.Get<ExperimentalNs>().enabled) {
     capture_.transient_suppressor_enabled =
         config.Get<ExperimentalNs>().enabled;
     InitializeTransient();
   }
 
+  if(capture_nonlocked_.intelligibility_enabled !=

[peah-webrtc, 2016/05/05 18:49:41]

Could you please explain more why it is desired to move the activation of IE
from the constructor to SetExtraOptions? I was not part of the discussions
mentioned so I don't know the background.

In general I think SetExtraOptions is something we should deprecate ASAP and
instead do all the configuration at creation-time so I'd definitely prefer to
have the activation of IE at the APM creation rather than using SetExtraOptions.

[aluebs-webrtc, 2016/05/05 21:53:05]

It was not moved from the constructor. Just added the possibility to enable it
through SetExtraOptions. And I agree we should eventually deprecate this method
and do all configuration at creation-time, but unfortunately this is not how the
VoiceEngine works today. Probably solenberg can comment on this. The behavior of
the IE would be similar to the possibility to ask for a pointer to certain
component (AEC for example) and call the Enable method on it dynamically.

[peah-webrtc, 2016/05/05 22:13:18]

Yes, you are right, it is still activated in the constructor since the
constructor calls SetExtraOptions. The ability to activate this outside of the
constructor was my main concern though. I know that it is allowed to activate
other components this way, via the enable methods and the SetExtraOptions
method. But that I think is something we'll move away from as soon as we are
able to and therefore I think we should try hard not to include the activation
of anything else this way.

Regarding that, could you please explain why it is not possible to activate the
IE at construction time. Is the reason VoE?

[the sun, 2016/05/06 09:00:16]

Different processing can be enabled/disabled depending on options
selected/negotiated after the VoE has been constructed (the APM is constructed
at same time by VoE). The VoE could potentially recreate the APM instead when
options change and we wouldn't need this.

[aluebs-webrtc, 2016/05/06 16:27:10]

It would be awesome to move to recreating the APM when the options change, but I
don't think this CL is the right place to do it. And I think that having the
possibility of enabling IE in SetExtraOptions (as other components already are)
doesn't make it harder to move away from it.

[peah-webrtc, 2016/05/12 08:05:12]

What I meant was that we should be able to activate the IE at creation-time,
similarly to how the beamformer is activated, and similarly to how IE was
created before.

Does this CL really require the activation of IE to be done during runtime
instead of at creation?

To clarify, with this I don't mean that the APM should be recreated every time
an option change (that is for sure something for another CL). What I mean is
that I'm concerned that this CL changes the way that IE is activated/created
from being done at creation-time to being done at runtime. This is something
that we should go at lengths not do do.

[aluebs-webrtc, 2016/05/13 02:04:37]

The Beamformer can be activated only at creation time, because it is only used
in Chrome, which creates their own APM (doesn't use the VoiceEngine's one) and
sets all configs at creation time. The IE could only be activated at creation
time, because it was not activated anywhere. But because the main use case for
the IE is mobile we require it to be activated from the VoiceEngine, which today
activates components (calling Enable methods of other components for example)
during run time. Once the VoiceEngine migrates to re-creating the APM with new
options I totally agree we should drop SetExtraOptions and the possibility of
accessing the components directly for that matter.

[peah-webrtc, 2016/05/13 12:39:49]

Thanks, then I see the problem. 

+     config.Get<Intelligibility>().enabled) {
+    capture_nonlocked_.intelligibility_enabled =
+        config.Get<Intelligibility>().enabled;
+    InitializeIntelligibility();
+  }
+
 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD
   if (capture_nonlocked_.beamformer_enabled !=
           config.Get<Beamforming>().enabled) {
     capture_nonlocked_.beamformer_enabled = config.Get<Beamforming>().enabled;
     if (config.Get<Beamforming>().array_geometry.size() > 1) {
       capture_.array_geometry = config.Get<Beamforming>().array_geometry;
     }
     capture_.target_direction = config.Get<Beamforming>().target_direction;
     InitializeBeamformer();
   }
@@ skipping 273 matching lines @@
   }
 
   RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio(
       ca, stream_delay_ms()));
 
   if (public_submodules_->echo_control_mobile->is_enabled() &&
       public_submodules_->noise_suppression->is_enabled()) {
     ca->CopyLowPassToReference();
   }
   public_submodules_->noise_suppression->ProcessCaptureAudio(ca);
-  if (constants_.intelligibility_enabled) {
+  if (capture_nonlocked_.intelligibility_enabled) {
     RTC_DCHECK(public_submodules_->noise_suppression->is_enabled());
     RTC_DCHECK(public_submodules_->gain_control->is_enabled());
     public_submodules_->intelligibility_enhancer->SetCaptureNoiseEstimate(
         public_submodules_->noise_suppression->NoiseEstimate(),
         public_submodules_->gain_control->compression_gain_db());
   }
 
   // Ensure that the stream delay was set before the call to the
   // AECM ProcessCaptureAudio function.
   if (public_submodules_->echo_control_mobile->is_enabled() &&
@@ skipping 177 matching lines @@
   }
   return kNoError;
 }
 
 int AudioProcessingImpl::ProcessReverseStreamLocked() {
   AudioBuffer* ra = render_.render_audio.get();  // For brevity.
   if (rev_analysis_needed()) {
     ra->SplitIntoFrequencyBands();
   }
 
-  if (constants_.intelligibility_enabled) {
+  if (capture_nonlocked_.intelligibility_enabled) {
     public_submodules_->intelligibility_enhancer->ProcessRenderAudio(
         ra->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate,
         ra->num_channels());
   }
 
   RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessRenderAudio(ra));
   RETURN_ON_ERR(
       public_submodules_->echo_control_mobile->ProcessRenderAudio(ra));
   if (!constants_.use_experimental_agc) {
     RETURN_ON_ERR(public_submodules_->gain_control->ProcessRenderAudio(ra));
@@ skipping 227 matching lines @@
   } else if (is_multi_band(
                  capture_nonlocked_.fwd_proc_format.sample_rate_hz())) {
     // Something besides public_submodules_->level_estimator is enabled, and we
     // have super-wb.
     return true;
   }
   return false;
 }
 
 bool AudioProcessingImpl::is_rev_processed() const {
-  return constants_.intelligibility_enabled;
+  return capture_nonlocked_.intelligibility_enabled;
 }
 
 bool AudioProcessingImpl::rev_synthesis_needed() const {
   return (is_rev_processed() &&
           is_multi_band(formats_.rev_proc_format.sample_rate_hz()));
 }
 
 bool AudioProcessingImpl::rev_analysis_needed() const {
   return is_multi_band(formats_.rev_proc_format.sample_rate_hz()) &&
          (is_rev_processed() ||
@@ skipping 44 matching lines @@
     if (!private_submodules_->beamformer) {
       private_submodules_->beamformer.reset(new NonlinearBeamformer(
           capture_.array_geometry, capture_.target_direction));
     }
     private_submodules_->beamformer->Initialize(kChunkSizeMs,
                                                 capture_nonlocked_.split_rate);
   }
 }
 
 void AudioProcessingImpl::InitializeIntelligibility() {
-  if (constants_.intelligibility_enabled) {
+  if (capture_nonlocked_.intelligibility_enabled) {
     public_submodules_->intelligibility_enhancer.reset(
         new IntelligibilityEnhancer(capture_nonlocked_.split_rate,
                                     render_.render_audio->num_channels(),
                                     NoiseSuppressionImpl::num_noise_bins()));
   }
 }
 
 void AudioProcessingImpl::InitializeHighPassFilter() {
   public_submodules_->high_pass_filter->Initialize(num_proc_channels(),
                                                    proc_sample_rate_hz());
@@ skipping 232 matching lines @@
   debug_dump_.capture.event_msg->mutable_config()->CopyFrom(config);
 
   RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
                                         &debug_dump_.num_bytes_left_for_log_,
                                         &crit_debug_, &debug_dump_.capture));
   return kNoError;
 }
 #endif  // WEBRTC_AUDIOPROC_DEBUG_DUMP
 
 }  // namespace webrtc