Integrate Intelligibility with APM

webrtc/modules/audio_processing/audio_processing_impl.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 10 matching lines @@
 #include "webrtc/modules/audio_processing/aec/aec_core.h"
 }
 #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h"
 #include "webrtc/modules/audio_processing/audio_buffer.h"
 #include "webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h"
 #include "webrtc/modules/audio_processing/common.h"
 #include "webrtc/modules/audio_processing/echo_cancellation_impl.h"
 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h"
 #include "webrtc/modules/audio_processing/gain_control_impl.h"
 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h"
+#include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h"
 #include "webrtc/modules/audio_processing/level_estimator_impl.h"
 #include "webrtc/modules/audio_processing/noise_suppression_impl.h"
 #include "webrtc/modules/audio_processing/processing_component.h"
 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h"
 #include "webrtc/modules/audio_processing/voice_detection_impl.h"
 #include "webrtc/modules/interface/module_common_types.h"
 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
 #include "webrtc/system_wrappers/interface/file_wrapper.h"
 #include "webrtc/system_wrappers/interface/logging.h"
 #include "webrtc/system_wrappers/interface/metrics.h"
@@ skipping 147 matching lines @@
 #endif
       agc_startup_min_volume_(config.Get<ExperimentalAgc>().startup_min_volume),
 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
       transient_suppressor_enabled_(false),
 #else
       transient_suppressor_enabled_(config.Get<ExperimentalNs>().enabled),
 #endif
       beamformer_enabled_(config.Get<Beamforming>().enabled),
       beamformer_(beamformer),
       array_geometry_(config.Get<Beamforming>().array_geometry),
-      supports_48kHz_(config.Get<AudioProcessing48kHzSupport>().enabled) {
+      supports_48kHz_(config.Get<AudioProcessing48kHzSupport>().enabled),
+      intelligibility_enabled_(config.Get<Intelligibility>().enabled) {
   echo_cancellation_ = new EchoCancellationImpl(this, crit_);
   component_list_.push_back(echo_cancellation_);
 
   echo_control_mobile_ = new EchoControlMobileImpl(this, crit_);
   component_list_.push_back(echo_control_mobile_);
 
   gain_control_ = new GainControlImpl(this, crit_);
   component_list_.push_back(gain_control_);
 
   high_pass_filter_ = new HighPassFilterImpl(this, crit_);
@@ skipping 89 matching lines @@
       return err;
     }
   }
 
   InitializeExperimentalAgc();
 
   InitializeTransient();
 
   InitializeBeamformer();
 
+  InitializeIntelligibility();
+
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_file_->Open()) {
     int err = WriteInitMessage();
     if (err != kNoError) {
       return err;
     }
   }
 #endif
 
   return kNoError;
@@ skipping 274 matching lines @@
     msg->set_delay(stream_delay_ms_);
     msg->set_drift(echo_cancellation_->stream_drift_samples());
     msg->set_level(gain_control()->stream_analog_level());
     msg->set_keypress(key_pressed_);
   }
 #endif
 
   MaybeUpdateHistograms();
 
   AudioBuffer* ca = capture_audio_.get();  // For brevity.
+
   if (use_new_agc_ && gain_control_->is_enabled()) {
     agc_manager_->AnalyzePreProcess(ca->channels()[0],
                                     ca->num_channels(),
                                     fwd_proc_format_.samples_per_channel());
   }
 
   bool data_processed = is_data_processed();
   if (analysis_needed(data_processed)) {
     ca->SplitIntoFrequencyBands();
   }
 
+  if (intelligibility_enabled_) {
+    intelligibility_enhancer_->AnalyzeCaptureAudio(
+        ca->split_channels_f(kBand0To8kHz), split_rate_, ca->num_channels());
+  }
+
   if (beamformer_enabled_) {
     beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f());
     ca->set_num_channels(1);
   }
 
   RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca));
   RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca));
   RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca));
   RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca));
 
@@ skipping 34 matching lines @@
                                     key_pressed_);
   }
 
   // The level estimator operates on the recombined data.
   RETURN_ON_ERR(level_estimator_->ProcessStream(ca));
 
   was_stream_delay_set_ = false;
   return kNoError;
 }
 
+int AudioProcessingImpl::ProcessReverseStream(float* const* data,
+                                              int samples_per_channel,
+                                              int rev_sample_rate_hz,
+                                              ChannelLayout layout) {
+  RETURN_ON_ERR(AnalyzeReverseStream(data, samples_per_channel,
+                                     rev_sample_rate_hz, layout));
+  if (is_rev_processed()) {
+    render_audio_->CopyTo(samples_per_channel, layout, data);
+  }
+
+  return kNoError;
+}
+
 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data,
                                               int samples_per_channel,
-                                              int sample_rate_hz,
+                                              int rev_sample_rate_hz,
                                               ChannelLayout layout) {
   CriticalSectionScoped crit_scoped(crit_);
   if (data == NULL) {
     return kNullPointerError;
   }
 
   const int num_channels = ChannelsFromLayout(layout);
-  RETURN_ON_ERR(MaybeInitializeLocked(fwd_in_format_.rate(),
-                                      fwd_out_format_.rate(),
-                                      sample_rate_hz,
-                                      fwd_in_format_.num_channels(),
-                                      fwd_out_format_.num_channels(),
-                                      num_channels));
+  RETURN_ON_ERR(
+      MaybeInitializeLocked(fwd_in_format_.rate(), fwd_out_format_.rate(),
+                            rev_sample_rate_hz, fwd_in_format_.num_channels(),
+                            fwd_out_format_.num_channels(), num_channels));
   if (samples_per_channel != rev_in_format_.samples_per_channel()) {
     return kBadDataLengthError;
   }
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_file_->Open()) {
     event_msg_->set_type(audioproc::Event::REVERSE_STREAM);
     audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream();
     const size_t channel_size =
         sizeof(float) * rev_in_format_.samples_per_channel();
     for (int i = 0; i < num_channels; ++i)
       msg->add_channel(data[i], channel_size);
     RETURN_ON_ERR(WriteMessageToDebugFile());
   }
 #endif
 
   render_audio_->CopyFrom(data, samples_per_channel, layout);
-  return AnalyzeReverseStreamLocked();
+  return ProcessReverseStreamLocked();
 }
 
 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) {
   CriticalSectionScoped crit_scoped(crit_);
   if (frame == NULL) {
     return kNullPointerError;
   }
   // Must be a native rate.
   if (frame->sample_rate_hz_ != kSampleRate8kHz &&
       frame->sample_rate_hz_ != kSampleRate16kHz &&
       frame->sample_rate_hz_ != kSampleRate32kHz &&
       frame->sample_rate_hz_ != kSampleRate48kHz) {
     return kBadSampleRateError;
   }
   // This interface does not tolerate different forward and reverse rates.
   if (frame->sample_rate_hz_ != fwd_in_format_.rate()) {
     return kBadSampleRateError;
   }
-
   RETURN_ON_ERR(MaybeInitializeLocked(fwd_in_format_.rate(),
                                       fwd_out_format_.rate(),
                                       frame->sample_rate_hz_,
                                       fwd_in_format_.num_channels(),
                                       fwd_in_format_.num_channels(),
                                       frame->num_channels_));
   if (frame->samples_per_channel_ != rev_in_format_.samples_per_channel()) {
     return kBadDataLengthError;
   }
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_file_->Open()) {
     event_msg_->set_type(audioproc::Event::REVERSE_STREAM);
     audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream();
     const size_t data_size = sizeof(int16_t) *
                              frame->samples_per_channel_ *
                              frame->num_channels_;
     msg->set_data(frame->data_, data_size);
     RETURN_ON_ERR(WriteMessageToDebugFile());
   }
 #endif
+  render_audio_->DeinterleaveFrom(frame);
+  RETURN_ON_ERR(ProcessReverseStreamLocked());
+  if (is_rev_processed()) {
+    render_audio_->InterleaveTo(frame, true);
+  }
 
-  render_audio_->DeinterleaveFrom(frame);
-  return AnalyzeReverseStreamLocked();
+  return kNoError;
 }
 
-int AudioProcessingImpl::AnalyzeReverseStreamLocked() {
+int AudioProcessingImpl::ProcessReverseStreamLocked() {
   AudioBuffer* ra = render_audio_.get();  // For brevity.
   if (rev_proc_format_.rate() == kSampleRate32kHz) {
     ra->SplitIntoFrequencyBands();
   }
 
+  if (intelligibility_enabled_) {
+    intelligibility_enhancer_->ProcessRenderAudio(
+        ra->split_channels_f(kBand0To8kHz), split_rate_, ra->num_channels());
+  }
+
   RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra));
   RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra));
   if (!use_new_agc_) {
     RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra));
   }
 
+  if (rev_proc_format_.rate() == kSampleRate32kHz && is_rev_processed()) {

[aluebs-webrtc, 2015/07/29 22:17:10]

Maybe this is a good time to make 32kHz and 48kHz consistent and use splitting
filter in both cases? Today 48Khz down-samples and 32kHz splits since the 2-band
splitting filter performance is better than the down-sampler, but the 3-band
splitting filter isn't. But soon the ineligibility enhancer will require the
higher bands anyway. WDYT?

[ekm, 2015/07/29 23:35:06]

I'd prefer to save this for a later cl, since this cl is pretty monstrous as is.
Also, with respect to intelligibility, this falls more under 'tuning' as we're
not sure yet how necessary this is for intelligibility; we might be able to get
away with just the 16k.

[aluebs-webrtc, 2015/07/30 15:28:07]

Leaving it for another CL sounds reasonable. But I think sooner or later the IE
will need this.

[ekm, 2015/07/30 21:23:50]

Yes. I'm interested to hear what the difference sounds like.

+    ra->MergeFrequencyBands();
+  }
+
   return kNoError;
 }
 
 int AudioProcessingImpl::set_stream_delay_ms(int delay) {
   Error retval = kNoError;
   was_stream_delay_set_ = true;
   delay += delay_offset_ms_;
 
   if (delay < 0) {
     delay = 0;
@@ skipping 194 matching lines @@
     // Only level_estimator_ is enabled.
     return false;
   } else if (fwd_proc_format_.rate() == kSampleRate32kHz ||
              fwd_proc_format_.rate() == kSampleRate48kHz) {
     // Something besides level_estimator_ is enabled, and we have super-wb.
     return true;
   }
   return false;
 }
 
+bool AudioProcessingImpl::is_rev_processed() const {
+  return intelligibility_enabled_ && intelligibility_enhancer_->active();

[Andrew MacDonald, 2015/07/29 03:52:27]

Hmm, checking active() is nice to save some complexity, but can this flip often?
You'll get a probably audible discontinuity when it flips if you don't smooth
this out.

One solution would be to have IE flip this on before it starts processing the
signal (that is, it returns the unprocessed signal for at least one chunk) and
flip it off after it stops (it returns at least one unprocessed chunk). It can
then internally smooth between these states.

If you don't have that feature today, I'd remove that chunk and leave a TODO
here.

Ah, reading further I see active() always returns true for now. So leave this,
but make sure you add something like the feature I described when you make it
non-const.

[ekm, 2015/07/29 23:35:06]

Yep, we have this smoothing feature implemented, and will add it in a later cl. 

+}
+
 void AudioProcessingImpl::InitializeExperimentalAgc() {
   if (use_new_agc_) {
     if (!agc_manager_.get()) {
       agc_manager_.reset(new AgcManagerDirect(gain_control_,
                                               gain_control_for_new_agc_.get(),
                                               agc_startup_min_volume_));
     }
     agc_manager_->Initialize();
     agc_manager_->SetCaptureMuted(output_will_be_muted_);
   }
@@ skipping 12 matching lines @@
 
 void AudioProcessingImpl::InitializeBeamformer() {
   if (beamformer_enabled_) {
     if (!beamformer_) {
       beamformer_.reset(new NonlinearBeamformer(array_geometry_));
     }
     beamformer_->Initialize(kChunkSizeMs, split_rate_);
   }
 }
 
+void AudioProcessingImpl::InitializeIntelligibility() {
+  if (intelligibility_enabled_) {
+    IntelligibilityEnhancer::Config config;
+    config.sample_rate_hz = split_rate_;
+    config.num_capture_channels = capture_audio_->num_channels();
+    config.num_render_channels = render_audio_->num_channels();
+    intelligibility_enhancer_.reset(new IntelligibilityEnhancer(config));
+  }
+}
+
 void AudioProcessingImpl::MaybeUpdateHistograms() {
   static const int kMinDiffDelayMs = 60;
 
   if (echo_cancellation()->is_enabled()) {
     // Activate delay_jumps_ counters if we know echo_cancellation is runnning.
     // If a stream has echo we know that the echo_cancellation is in process.
     if (stream_delay_jumps_ == -1 && echo_cancellation()->stream_has_echo()) {
       stream_delay_jumps_ = 0;
     }
     if (aec_system_delay_jumps_ == -1 &&
@@ skipping 92 matching lines @@
   int err = WriteMessageToDebugFile();
   if (err != kNoError) {
     return err;
   }
 
   return kNoError;
 }
 #endif  // WEBRTC_AUDIOPROC_DEBUG_DUMP
 
 }  // namespace webrtc