Integrate Intelligibility with APM

webrtc/modules/audio_processing/audio_processing_impl.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 10 matching lines @@
 #include "webrtc/modules/audio_processing/aec/aec_core.h"
 }
 #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h"
 #include "webrtc/modules/audio_processing/audio_buffer.h"
 #include "webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h"
 #include "webrtc/modules/audio_processing/common.h"
 #include "webrtc/modules/audio_processing/echo_cancellation_impl.h"
 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h"
 #include "webrtc/modules/audio_processing/gain_control_impl.h"
 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h"
+#include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h"
 #include "webrtc/modules/audio_processing/level_estimator_impl.h"
 #include "webrtc/modules/audio_processing/noise_suppression_impl.h"
 #include "webrtc/modules/audio_processing/processing_component.h"
 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h"
 #include "webrtc/modules/audio_processing/voice_detection_impl.h"
 #include "webrtc/modules/interface/module_common_types.h"
 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
 #include "webrtc/system_wrappers/interface/file_wrapper.h"
 #include "webrtc/system_wrappers/interface/logging.h"
 #include "webrtc/system_wrappers/interface/metrics.h"
@@ skipping 147 matching lines @@
 #endif
       agc_startup_min_volume_(config.Get<ExperimentalAgc>().startup_min_volume),
 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
       transient_suppressor_enabled_(false),
 #else
       transient_suppressor_enabled_(config.Get<ExperimentalNs>().enabled),
 #endif
       beamformer_enabled_(config.Get<Beamforming>().enabled),
       beamformer_(beamformer),
       array_geometry_(config.Get<Beamforming>().array_geometry),
-      supports_48kHz_(config.Get<AudioProcessing48kHzSupport>().enabled) {
+      supports_48kHz_(config.Get<AudioProcessing48kHzSupport>().enabled),
+      intelligibility_enabled_(config.Get<Intelligibility>().enabled) {
   echo_cancellation_ = new EchoCancellationImpl(this, crit_);
   component_list_.push_back(echo_cancellation_);
 
   echo_control_mobile_ = new EchoControlMobileImpl(this, crit_);
   component_list_.push_back(echo_control_mobile_);
 
   gain_control_ = new GainControlImpl(this, crit_);
   component_list_.push_back(gain_control_);
 
   high_pass_filter_ = new HighPassFilterImpl(this, crit_);
@@ skipping 89 matching lines @@
       return err;
     }
   }
 
   InitializeExperimentalAgc();
 
   InitializeTransient();
 
   InitializeBeamformer();
 
+  InitializeIntelligibility();
+
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_file_->Open()) {
     int err = WriteInitMessage();
     if (err != kNoError) {
       return err;
     }
   }
 #endif
 
   return kNoError;
@@ skipping 102 matching lines @@
 void AudioProcessingImpl::SetExtraOptions(const Config& config) {
   CriticalSectionScoped crit_scoped(crit_);
   for (auto item : component_list_) {
     item->SetExtraOptions(config);
   }
 
   if (transient_suppressor_enabled_ != config.Get<ExperimentalNs>().enabled) {
     transient_suppressor_enabled_ = config.Get<ExperimentalNs>().enabled;
     InitializeTransient();
   }
+
+  if (intelligibility_enabled_ != config.Get<Intelligibility>().enabled) {
+    intelligibility_enabled_ = config.Get<Intelligibility>().enabled;
+    InitializeIntelligibility();
+  }
 }
 
 int AudioProcessingImpl::input_sample_rate_hz() const {
   CriticalSectionScoped crit_scoped(crit_);
   return fwd_in_format_.rate();
 }
 
 int AudioProcessingImpl::sample_rate_hz() const {
   CriticalSectionScoped crit_scoped(crit_);
   return fwd_in_format_.rate();
@@ skipping 152 matching lines @@
     msg->set_delay(stream_delay_ms_);
     msg->set_drift(echo_cancellation_->stream_drift_samples());
     msg->set_level(gain_control()->stream_analog_level());
     msg->set_keypress(key_pressed_);
   }
 #endif
 
   MaybeUpdateHistograms();
 
   AudioBuffer* ca = capture_audio_.get();  // For brevity.
+
+  if (intelligibility_enabled_) {
+    float voice_probability =
+        agc_manager_.get() ? agc_manager_->voice_probability() : 0.f;

[turaj, 2015/07/14 18:28:51]

Shouldn't we turn activity detector part of AGC on when intelligibility is
enabled? Otherwise we will process all frames as inactive.

[aluebs-webrtc, 2015/07/15 01:02:04]

Also, to avoid being out of sync, this should probably run after agc_manager,
right?

[ekm, 2015/07/17 19:59:38]

To avoid these problems, and simplify things, switched to enhancer using its own
capture VAD. Maybe optimize later.

+    intelligibility_enhancer_->ProcessCaptureAudio(
+        ca->split_channels_f(kBand0To8kHz), voice_probability);
+  }
+
   if (use_new_agc_ && gain_control_->is_enabled()) {
     agc_manager_->AnalyzePreProcess(ca->channels()[0],
                                     ca->num_channels(),
                                     fwd_proc_format_.samples_per_channel());
   }
 
   bool data_processed = is_data_processed();
   if (analysis_needed(data_processed)) {
     ca->SplitIntoFrequencyBands();
   }
@@ skipping 45 matching lines @@
                                     key_pressed_);
   }
 
   // The level estimator operates on the recombined data.
   RETURN_ON_ERR(level_estimator_->ProcessStream(ca));
 
   was_stream_delay_set_ = false;
   return kNoError;
 }
 
-int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data,
+int AudioProcessingImpl::AnalyzeReverseStream(float* const* data,
                                               int samples_per_channel,
                                               int sample_rate_hz,
                                               ChannelLayout layout) {
   CriticalSectionScoped crit_scoped(crit_);
   if (data == NULL) {
     return kNullPointerError;
   }
 
   const int num_channels = ChannelsFromLayout(layout);
   RETURN_ON_ERR(MaybeInitializeLocked(fwd_in_format_.rate(),
@@ skipping 12 matching lines @@
     audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream();
     const size_t channel_size =
         sizeof(float) * rev_in_format_.samples_per_channel();
     for (int i = 0; i < num_channels; ++i)
       msg->add_channel(data[i], channel_size);
     RETURN_ON_ERR(WriteMessageToDebugFile());
   }
 #endif
 
   render_audio_->CopyFrom(data, samples_per_channel, layout);
-  return AnalyzeReverseStreamLocked();
+  RETURN_ON_ERR(AnalyzeReverseStreamLocked());
+  if (intelligibility_enabled_) {
+    render_audio_->CopyTo(samples_per_channel, layout, data);
+  }
+
+  return kNoError;
 }
 
 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) {
   CriticalSectionScoped crit_scoped(crit_);
   if (frame == NULL) {
     return kNullPointerError;
   }
   // Must be a native rate.
   if (frame->sample_rate_hz_ != kSampleRate8kHz &&
       frame->sample_rate_hz_ != kSampleRate16kHz &&
@@ skipping 22 matching lines @@
     audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream();
     const size_t data_size = sizeof(int16_t) *
                              frame->samples_per_channel_ *
                              frame->num_channels_;
     msg->set_data(frame->data_, data_size);
     RETURN_ON_ERR(WriteMessageToDebugFile());
   }
 #endif
 
   render_audio_->DeinterleaveFrom(frame);
-  return AnalyzeReverseStreamLocked();
+  RETURN_ON_ERR(AnalyzeReverseStreamLocked());
+  if (intelligibility_enabled_) {

[aluebs-webrtc, 2015/07/15 01:02:04]

You don't need this if statement, because InterleaveTo gets
intelligibility_enabled_ and only copies when it is true.

[ekm, 2015/07/17 19:59:38]

Done. As a result, I updated audio_buffer.InterleaveTo to allow for case of
single source channel and >1 dest channels. This should make InterleaveTo make
more sense, and keeps it possible to test intelligibility in voe_cmd_test. 

[aluebs-webrtc, 2015/07/20 19:33:42]

Ack

+    render_audio_->InterleaveTo(frame, intelligibility_enabled_);
+  }
+
+  return kNoError;
 }
 
 int AudioProcessingImpl::AnalyzeReverseStreamLocked() {
   AudioBuffer* ra = render_audio_.get();  // For brevity.
   if (rev_proc_format_.rate() == kSampleRate32kHz) {
     ra->SplitIntoFrequencyBands();
   }
 
   RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra));
   RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra));
   if (!use_new_agc_) {
     RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra));
   }
 
+  if (intelligibility_enabled_) {

[turaj, 2015/07/14 18:28:51]

I suppose we want to do all the modifications to playout signal before echo
cancellation/echo control, otherwise they have to compensate for the changes
that enhancer makes to playout signal. 

[ekm, 2015/07/17 19:59:37]

Done.

+    intelligibility_enhancer_->ProcessRenderAudio(
+        ra->split_channels_f(kBand0To8kHz));

[aluebs-webrtc, 2015/07/15 01:02:04]

Maybe not for this CL, but at some point this needs to do something with the
higher bands, probably something simpler in the time domain.

[ekm, 2015/07/17 19:59:38]

Acknowledged. I think we'll save this for a later CL. Since the enhancer will
only be on in cases of very high noise and it's targeting speech, 0-8kHz should
be ok, but you're right we might get some strange behavior if we do nothing to
the higher bands. Do you have an idea of what might go wrong here? 

[aluebs-webrtc, 2015/07/20 19:33:42]

Agreed on leaving for another CL. I don't think anything will go explicitly
wrong, but if you are not processing the higher bands, you lose the ability to
shape the spectrum above 8kHz, which probably improves your enhancement.

[ekm, 2015/07/21 19:22:13]

Acknowledged.

+  }
+
   return kNoError;
 }
 
 int AudioProcessingImpl::set_stream_delay_ms(int delay) {
   Error retval = kNoError;
   was_stream_delay_set_ = true;
   delay += delay_offset_ms_;
 
   if (delay < 0) {
     delay = 0;
@@ skipping 226 matching lines @@
 
 void AudioProcessingImpl::InitializeBeamformer() {
   if (beamformer_enabled_) {
     if (!beamformer_) {
       beamformer_.reset(new NonlinearBeamformer(array_geometry_));
     }
     beamformer_->Initialize(kChunkSizeMs, split_rate_);
   }
 }
 
+void AudioProcessingImpl::InitializeIntelligibility() {
+  if (intelligibility_enabled_) {
+    if (!intelligibility_enhancer_) {

[aluebs-webrtc, 2015/07/15 01:02:04]

We probably want to reset the intelligibility_enhancer_ every time, since the
split_rate_ or fwd_proc_format_.num_channels() could have changed, right?

[ekm, 2015/07/17 19:59:38]

Done.

+      IntelligibilityEnhancer::Config config;
+      config.sample_rate_hz = split_rate_;
+      config.channels = fwd_in_format_.num_channels();

[aluebs-webrtc, 2015/07/15 01:02:04]

Shouldn't this be fwd_proc_format_?

[ekm, 2015/07/17 19:59:38]

Just set it to single channel for now, since that's the primary use case, and we
haven't yet decided on how to handle multiple channels.

[aluebs-webrtc, 2015/07/20 19:33:42]

Then I think it is better to set the correct value here and handle the multiple
channel cases in the enhancer, even if it is a CHECK for now. WDYT?

[ekm, 2015/07/21 19:22:13]

I agree. The problem was that the enhancer only had one num_channels member, so
didn't support case where render and capture have different number. I've now
added those members, so further changes to multi-channel case can be done in
enhancer. The enhancer actually can handle these cases, it's just not
particularly smart about it yet.

[aluebs-webrtc, 2015/07/21 21:30:22]

This is better.

+      intelligibility_enhancer_.reset(new IntelligibilityEnhancer(config));
+    }
+  }
+}
+
 void AudioProcessingImpl::MaybeUpdateHistograms() {
   static const int kMinDiffDelayMs = 60;
 
   if (echo_cancellation()->is_enabled()) {
     // Activate delay_jumps_ counters if we know echo_cancellation is runnning.
     // If a stream has echo we know that the echo_cancellation is in process.
     if (stream_delay_jumps_ == -1 && echo_cancellation()->stream_has_echo()) {
       stream_delay_jumps_ = 0;
     }
     if (aec_system_delay_jumps_ == -1 &&
@@ skipping 92 matching lines @@
   int err = WriteMessageToDebugFile();
   if (err != kNoError) {
     return err;
   }
 
   return kNoError;
 }
 #endif  // WEBRTC_AUDIOPROC_DEBUG_DUMP
 
 }  // namespace webrtc