Surface the IntelligibilityEnhancer on MediaConstraints

webrtc/media/engine/webrtcvoiceengine.cc

 /*
  *  Copyright (c) 2004 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 559 matching lines @@
     options.highpass_filter = rtc::Optional<bool>(true);
     options.stereo_swapping = rtc::Optional<bool>(false);
     options.audio_jitter_buffer_max_packets = rtc::Optional<int>(50);
     options.audio_jitter_buffer_fast_accelerate = rtc::Optional<bool>(false);
     options.typing_detection = rtc::Optional<bool>(true);
     options.adjust_agc_delta = rtc::Optional<int>(0);
     options.experimental_agc = rtc::Optional<bool>(false);
     options.extended_filter_aec = rtc::Optional<bool>(false);
     options.delay_agnostic_aec = rtc::Optional<bool>(false);
     options.experimental_ns = rtc::Optional<bool>(false);
+    options.intelligibility_enhancer = rtc::Optional<bool>(false);
     bool error = ApplyOptions(options);
     RTC_DCHECK(error);
   }
 
   SetDefaultDevices();
 }
 
 WebRtcVoiceEngine::~WebRtcVoiceEngine() {
   RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
   LOG(LS_INFO) << "WebRtcVoiceEngine::~WebRtcVoiceEngine";
@@ skipping 149 matching lines @@
         options.tx_agc_limiter.value_or(default_agc_config_.limiterEnable);
     if (voe_wrapper_->processing()->SetAgcConfig(default_agc_config_) == -1) {
       LOG_RTCERR3(SetAgcConfig,
                   default_agc_config_.targetLeveldBOv,
                   default_agc_config_.digitalCompressionGaindB,
                   default_agc_config_.limiterEnable);
       return false;
     }
   }
 
+  if (options.intelligibility_enhancer) {
+    intelligibility_enhancer_ = options.intelligibility_enhancer;
+  }
+  if (intelligibility_enhancer_ && *intelligibility_enhancer_) {
+    LOG(LS_INFO) << "Enabling NS when Intelligibility Enhancer is active.";
+    options.noise_suppression = intelligibility_enhancer_;
+  }
+
   if (options.noise_suppression) {
-    const bool built_in_ns = adm()->BuiltInNSIsAvailable();
-    if (built_in_ns) {
-      if (adm()->EnableBuiltInNS(*options.noise_suppression) == 0 &&
-          *options.noise_suppression) {
+    if (adm()->BuiltInNSIsAvailable()) {

[peah-webrtc, 2016/05/13 12:39:49]

I'm concerned about the AEC functionality as the IE has not been tested yet with
the software AEC.

How is this envisioned to be used at the moment? Is it possible to restrict the
usage of the IE to platforms which have HW AEC support?

I think that there could be a point in turning off the IE functionality if there
is no HW AEC available, similarly to this CL turns off the HW NS if IE is
active.

But it depends fully on how this intended to be used. If it is used in a way
without explicitly checking whether the HW AEC is available on the device then
there must be code for turning off the IE functionality if a HW AEC is not
available (at least until the IE has been properly tested with the AEC).

[aluebs-webrtc, 2016/05/13 16:24:28]

This CL doesn't enable IE, it just surfaces it to the MediaConstraints. Later we
will use this API to test it and have an A/B experiment limited to the team
before we decide to move forward with it (or not). I would argue that the only
way we can gather real-life data about the interaction between the IE and SW AEC
is by having that case be part of the testing and experiment. I am not saying we
shouldn't also test it offline on more recordings, I just think that those
recordings will never cover all possible scenarios and at the end of the day the
only way of being sure how it affects users is to test it on real usage.

[peah-webrtc, 2016/05/16 07:43:38]

Thanks for the clarification! I definitely agree on the value of real-life data
on the interaction between IE and SW AEC but I also definitely think it should
be preceded by a proper analysis of that interaction. To this date none such has
been done.

My main concern here is though that during the testing it should be known when
the IE is tested on a platform without HW AEC support. But from what you write
the current planned usage of the MediaConstraints will be such that it can only
be used within a limited control group which is great.

[aluebs-webrtc, 2016/05/16 20:21:16]

I completely agree we should do a proper analysis of the IE-AEC interaction and
I will do it myself. And yes, the plan is to start testing only within the team.

+      bool builtin_ns =
+          *options.noise_suppression &&
+          !(intelligibility_enhancer_ && *intelligibility_enhancer_);
+      if (adm()->EnableBuiltInNS(builtin_ns) == 0 && builtin_ns) {
         // Disable internal software NS if built-in NS is enabled,
         // i.e., replace the software NS with the built-in NS.
         options.noise_suppression = rtc::Optional<bool>(false);
         LOG(LS_INFO) << "Disabling NS since built-in NS will be used instead";
       }
     }
     if (voep->SetNsStatus(*options.noise_suppression, ns_mode) == -1) {
       LOG_RTCERR2(SetNsStatus, *options.noise_suppression, ns_mode);
       return false;
     } else {
@@ skipping 72 matching lines @@
 
   if (options.experimental_ns) {
     experimental_ns_ = options.experimental_ns;
   }
   if (experimental_ns_) {
     LOG(LS_INFO) << "Experimental ns is enabled? " << *experimental_ns_;
     config.Set<webrtc::ExperimentalNs>(
         new webrtc::ExperimentalNs(*experimental_ns_));
   }
 
+  if (intelligibility_enhancer_) {
+    LOG(LS_INFO) << "Intelligibility Enhancer is enabled? "
+                 << *intelligibility_enhancer_;
+    config.Set<webrtc::Intelligibility>(
+        new webrtc::Intelligibility(*intelligibility_enhancer_));
+  }
+
   // We check audioproc for the benefit of tests, since FakeWebRtcVoiceEngine
   // returns NULL on audio_processing().
   webrtc::AudioProcessing* audioproc = voe_wrapper_->base()->audio_processing();
   if (audioproc) {
     audioproc->SetExtraOptions(config);
   }
 
   if (options.recording_sample_rate) {
     LOG(LS_INFO) << "Recording sample rate is "
                  << *options.recording_sample_rate;
@@ skipping 1700 matching lines @@
     }
   } else {
     LOG(LS_INFO) << "Stopping playout for channel #" << channel;
     engine()->voe()->base()->StopPlayout(channel);
   }
   return true;
 }
 }  // namespace cricket
 
 #endif  // HAVE_WEBRTC_VOICE