Integrate Intelligibility with APM

webrtc/modules/audio_processing/audio_processing_impl.cc

Index: webrtc/modules/audio_processing/audio_processing_impl.cc
diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc
index 87b82a6a3509131adae9ed698cc0f896fd01d4c0..079ee952cf47bdd5aa40ac61185cc7fd05c12b11 100644
--- a/webrtc/modules/audio_processing/audio_processing_impl.cc
+++ b/webrtc/modules/audio_processing/audio_processing_impl.cc
@@ -28,6 +28,7 @@ extern "C" {
 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h"
 #include "webrtc/modules/audio_processing/gain_control_impl.h"
 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h"
+#include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h"
 #include "webrtc/modules/audio_processing/level_estimator_impl.h"
 #include "webrtc/modules/audio_processing/noise_suppression_impl.h"
 #include "webrtc/modules/audio_processing/processing_component.h"
@@ -195,7 +196,8 @@ AudioProcessingImpl::AudioProcessingImpl(const Config& config,
       beamformer_enabled_(config.Get<Beamforming>().enabled),
       beamformer_(beamformer),
       array_geometry_(config.Get<Beamforming>().array_geometry),
-      supports_48kHz_(config.Get<AudioProcessing48kHzSupport>().enabled) {
+      supports_48kHz_(config.Get<AudioProcessing48kHzSupport>().enabled),
+      intelligibility_enabled_(config.Get<Intelligibility>().enabled) {
   echo_cancellation_ = new EchoCancellationImpl(this, crit_);
   component_list_.push_back(echo_cancellation_);
 
@@ -305,6 +307,8 @@ int AudioProcessingImpl::InitializeLocked() {
 
   InitializeBeamformer();
 
+  InitializeIntelligibility();
+
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_file_->Open()) {
     int err = WriteInitMessage();
@@ -427,6 +431,11 @@ void AudioProcessingImpl::SetExtraOptions(const Config& config) {
     transient_suppressor_enabled_ = config.Get<ExperimentalNs>().enabled;
     InitializeTransient();
   }
+
+  if (intelligibility_enabled_ != config.Get<Intelligibility>().enabled) {
+    intelligibility_enabled_ = config.Get<Intelligibility>().enabled;
+    InitializeIntelligibility();
+  }
 }
 
 int AudioProcessingImpl::input_sample_rate_hz() const {
@@ -599,6 +608,14 @@ int AudioProcessingImpl::ProcessStreamLocked() {
   MaybeUpdateHistograms();
 
   AudioBuffer* ca = capture_audio_.get();  // For brevity.
+
+  if (intelligibility_enabled_) {
+    float voice_probability =
+        agc_manager_.get() ? agc_manager_->voice_probability() : 0.f;

[turaj, 2015/07/14 18:28:51]

Shouldn't we turn activity detector part of AGC on when intelligibility is
enabled? Otherwise we will process all frames as inactive.

[aluebs-webrtc, 2015/07/15 01:02:04]

Also, to avoid being out of sync, this should probably run after agc_manager,
right?

[ekm, 2015/07/17 19:59:38]

To avoid these problems, and simplify things, switched to enhancer using its own
capture VAD. Maybe optimize later.

+    intelligibility_enhancer_->ProcessCaptureAudio(
+        ca->split_channels_f(kBand0To8kHz), voice_probability);
+  }
+
   if (use_new_agc_ && gain_control_->is_enabled()) {
     agc_manager_->AnalyzePreProcess(ca->channels()[0],
                                     ca->num_channels(),
@@ -664,7 +681,7 @@ int AudioProcessingImpl::ProcessStreamLocked() {
   return kNoError;
 }
 
-int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data,
+int AudioProcessingImpl::AnalyzeReverseStream(float* const* data,
                                               int samples_per_channel,
                                               int sample_rate_hz,
                                               ChannelLayout layout) {
@@ -697,7 +714,12 @@ int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data,
 #endif
 
   render_audio_->CopyFrom(data, samples_per_channel, layout);
-  return AnalyzeReverseStreamLocked();
+  RETURN_ON_ERR(AnalyzeReverseStreamLocked());
+  if (intelligibility_enabled_) {
+    render_audio_->CopyTo(samples_per_channel, layout, data);
+  }
+
+  return kNoError;
 }
 
 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) {
@@ -740,7 +762,12 @@ int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) {
 #endif
 
   render_audio_->DeinterleaveFrom(frame);
-  return AnalyzeReverseStreamLocked();
+  RETURN_ON_ERR(AnalyzeReverseStreamLocked());
+  if (intelligibility_enabled_) {

[aluebs-webrtc, 2015/07/15 01:02:04]

You don't need this if statement, because InterleaveTo gets
intelligibility_enabled_ and only copies when it is true.

[ekm, 2015/07/17 19:59:38]

Done. As a result, I updated audio_buffer.InterleaveTo to allow for case of
single source channel and >1 dest channels. This should make InterleaveTo make
more sense, and keeps it possible to test intelligibility in voe_cmd_test. 

[aluebs-webrtc, 2015/07/20 19:33:42]

Ack

+    render_audio_->InterleaveTo(frame, intelligibility_enabled_);
+  }
+
+  return kNoError;
 }
 
 int AudioProcessingImpl::AnalyzeReverseStreamLocked() {
@@ -755,6 +782,11 @@ int AudioProcessingImpl::AnalyzeReverseStreamLocked() {
     RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra));
   }
 
+  if (intelligibility_enabled_) {

[turaj, 2015/07/14 18:28:51]

I suppose we want to do all the modifications to playout signal before echo
cancellation/echo control, otherwise they have to compensate for the changes
that enhancer makes to playout signal. 

[ekm, 2015/07/17 19:59:37]

Done.

+    intelligibility_enhancer_->ProcessRenderAudio(
+        ra->split_channels_f(kBand0To8kHz));

[aluebs-webrtc, 2015/07/15 01:02:04]

Maybe not for this CL, but at some point this needs to do something with the
higher bands, probably something simpler in the time domain.

[ekm, 2015/07/17 19:59:38]

Acknowledged. I think we'll save this for a later CL. Since the enhancer will
only be on in cases of very high noise and it's targeting speech, 0-8kHz should
be ok, but you're right we might get some strange behavior if we do nothing to
the higher bands. Do you have an idea of what might go wrong here? 

[aluebs-webrtc, 2015/07/20 19:33:42]

Agreed on leaving for another CL. I don't think anything will go explicitly
wrong, but if you are not processing the higher bands, you lose the ability to
shape the spectrum above 8kHz, which probably improves your enhancement.

[ekm, 2015/07/21 19:22:13]

Acknowledged.

+  }
+
   return kNoError;
 }
 
@@ -1001,6 +1033,17 @@ void AudioProcessingImpl::InitializeBeamformer() {
   }
 }
 
+void AudioProcessingImpl::InitializeIntelligibility() {
+  if (intelligibility_enabled_) {
+    if (!intelligibility_enhancer_) {

[aluebs-webrtc, 2015/07/15 01:02:04]

We probably want to reset the intelligibility_enhancer_ every time, since the
split_rate_ or fwd_proc_format_.num_channels() could have changed, right?

[ekm, 2015/07/17 19:59:38]

Done.

+      IntelligibilityEnhancer::Config config;
+      config.sample_rate_hz = split_rate_;
+      config.channels = fwd_in_format_.num_channels();

[aluebs-webrtc, 2015/07/15 01:02:04]

Shouldn't this be fwd_proc_format_?

[ekm, 2015/07/17 19:59:38]

Just set it to single channel for now, since that's the primary use case, and we
haven't yet decided on how to handle multiple channels.

[aluebs-webrtc, 2015/07/20 19:33:42]

Then I think it is better to set the correct value here and handle the multiple
channel cases in the enhancer, even if it is a CHECK for now. WDYT?

[ekm, 2015/07/21 19:22:13]

I agree. The problem was that the enhancer only had one num_channels member, so
didn't support case where render and capture have different number. I've now
added those members, so further changes to multi-channel case can be done in
enhancer. The enhancer actually can handle these cases, it's just not
particularly smart about it yet.

[aluebs-webrtc, 2015/07/21 21:30:22]

This is better.

+      intelligibility_enhancer_.reset(new IntelligibilityEnhancer(config));
+    }
+  }
+}
+
 void AudioProcessingImpl::MaybeUpdateHistograms() {
   static const int kMinDiffDelayMs = 60;