Integrate Intelligibility with APM

webrtc/modules/audio_processing/audio_processing_impl.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 10 matching lines @@
 #include "webrtc/modules/audio_processing/aec/aec_core.h"
 }
 #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h"
 #include "webrtc/modules/audio_processing/audio_buffer.h"
 #include "webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h"
 #include "webrtc/modules/audio_processing/common.h"
 #include "webrtc/modules/audio_processing/echo_cancellation_impl.h"
 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h"
 #include "webrtc/modules/audio_processing/gain_control_impl.h"
 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h"
+#include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h"
 #include "webrtc/modules/audio_processing/level_estimator_impl.h"
 #include "webrtc/modules/audio_processing/noise_suppression_impl.h"
 #include "webrtc/modules/audio_processing/processing_component.h"
 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h"
 #include "webrtc/modules/audio_processing/voice_detection_impl.h"
 #include "webrtc/modules/interface/module_common_types.h"
 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
 #include "webrtc/system_wrappers/interface/file_wrapper.h"
 #include "webrtc/system_wrappers/interface/logging.h"
 #include "webrtc/system_wrappers/interface/metrics.h"
@@ skipping 147 matching lines @@
 #endif
       agc_startup_min_volume_(config.Get<ExperimentalAgc>().startup_min_volume),
 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
       transient_suppressor_enabled_(false),
 #else
       transient_suppressor_enabled_(config.Get<ExperimentalNs>().enabled),
 #endif
       beamformer_enabled_(config.Get<Beamforming>().enabled),
       beamformer_(beamformer),
       array_geometry_(config.Get<Beamforming>().array_geometry),
-      supports_48kHz_(config.Get<AudioProcessing48kHzSupport>().enabled) {
+      supports_48kHz_(config.Get<AudioProcessing48kHzSupport>().enabled),
+      intelligibility_enabled_(config.Get<Intelligibility>().enabled) {
   echo_cancellation_ = new EchoCancellationImpl(this, crit_);
   component_list_.push_back(echo_cancellation_);
 
   echo_control_mobile_ = new EchoControlMobileImpl(this, crit_);
   component_list_.push_back(echo_control_mobile_);
 
   gain_control_ = new GainControlImpl(this, crit_);
   component_list_.push_back(gain_control_);
 
   high_pass_filter_ = new HighPassFilterImpl(this, crit_);
@@ skipping 89 matching lines @@
       return err;
     }
   }
 
   InitializeExperimentalAgc();
 
   InitializeTransient();
 
   InitializeBeamformer();
 
+  InitializeIntelligibility();
+
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_file_->Open()) {
     int err = WriteInitMessage();
     if (err != kNoError) {
       return err;
     }
   }
 #endif
 
   return kNoError;
@@ skipping 274 matching lines @@
     msg->set_delay(stream_delay_ms_);
     msg->set_drift(echo_cancellation_->stream_drift_samples());
     msg->set_level(gain_control()->stream_analog_level());
     msg->set_keypress(key_pressed_);
   }
 #endif
 
   MaybeUpdateHistograms();
 
   AudioBuffer* ca = capture_audio_.get();  // For brevity.
+
   if (use_new_agc_ && gain_control_->is_enabled()) {
     agc_manager_->AnalyzePreProcess(ca->channels()[0],
                                     ca->num_channels(),
                                     fwd_proc_format_.samples_per_channel());
   }
 
   bool data_processed = is_data_processed();
   if (analysis_needed(data_processed)) {
     ca->SplitIntoFrequencyBands();
   }
 
+  if (intelligibility_enabled_) {
+    intelligibility_enhancer_->AnalyzeCaptureAudio(
+        ca->split_channels_f(kBand0To8kHz), split_rate_, ca->num_channels());
+  }
+
   if (beamformer_enabled_) {
     beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f());
     ca->set_num_channels(1);
   }
 
   RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca));
   RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca));
   RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca));
   RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca));
 
@@ skipping 34 matching lines @@
                                     key_pressed_);
   }
 
   // The level estimator operates on the recombined data.
   RETURN_ON_ERR(level_estimator_->ProcessStream(ca));
 
   was_stream_delay_set_ = false;
   return kNoError;
 }
 
+int AudioProcessingImpl::ProcessReverseStream(float* const* data,
+                                              int samples_per_channel,
+                                              int rev_sample_rate_hz,
+                                              ChannelLayout layout) {
+  RETURN_ON_ERR(AnalyzeReverseStream(data, samples_per_channel,
+                                     rev_sample_rate_hz, layout));
+  if (intelligibility_enabled_) {
+    render_audio_->CopyTo(samples_per_channel, layout, data);
+  }
+
+  return kNoError;
+}
+
 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data,
                                               int samples_per_channel,
-                                              int sample_rate_hz,
+                                              int rev_sample_rate_hz,
                                               ChannelLayout layout) {
   CriticalSectionScoped crit_scoped(crit_);
   if (data == NULL) {
     return kNullPointerError;
   }
 
   const int num_channels = ChannelsFromLayout(layout);
-  RETURN_ON_ERR(MaybeInitializeLocked(fwd_in_format_.rate(),
-                                      fwd_out_format_.rate(),
-                                      sample_rate_hz,
-                                      fwd_in_format_.num_channels(),
-                                      fwd_out_format_.num_channels(),
-                                      num_channels));
+  RETURN_ON_ERR(
+      MaybeInitializeLocked(fwd_in_format_.rate(), fwd_out_format_.rate(),
+                            rev_sample_rate_hz, fwd_in_format_.num_channels(),
+                            fwd_out_format_.num_channels(), num_channels));
   if (samples_per_channel != rev_in_format_.samples_per_channel()) {
     return kBadDataLengthError;
   }
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_file_->Open()) {
     event_msg_->set_type(audioproc::Event::REVERSE_STREAM);
     audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream();
     const size_t channel_size =
         sizeof(float) * rev_in_format_.samples_per_channel();
     for (int i = 0; i < num_channels; ++i)
       msg->add_channel(data[i], channel_size);
     RETURN_ON_ERR(WriteMessageToDebugFile());
   }
 #endif
 
   render_audio_->CopyFrom(data, samples_per_channel, layout);
-  return AnalyzeReverseStreamLocked();
+  return ProcessReverseStreamLocked();
 }
 
 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) {

[Andrew MacDonald, 2015/07/24 23:50:39]

The AudioFrame* and float* AnalyzeReverseStreams have different behavior wrt IE.
Normally I'd ask you to add a ProcessReverseStream for AudioFrame as well, but
we really want to get rid of this AudioFrame junk, so I suppose this is OK...

[ekm, 2015/07/29 00:37:19]

Acknowledged.

   CriticalSectionScoped crit_scoped(crit_);
   if (frame == NULL) {
     return kNullPointerError;
   }
   // Must be a native rate.
   if (frame->sample_rate_hz_ != kSampleRate8kHz &&
       frame->sample_rate_hz_ != kSampleRate16kHz &&
       frame->sample_rate_hz_ != kSampleRate32kHz &&
       frame->sample_rate_hz_ != kSampleRate48kHz) {
     return kBadSampleRateError;
   }
   // This interface does not tolerate different forward and reverse rates.
   if (frame->sample_rate_hz_ != fwd_in_format_.rate()) {
     return kBadSampleRateError;
   }
-
   RETURN_ON_ERR(MaybeInitializeLocked(fwd_in_format_.rate(),
                                       fwd_out_format_.rate(),
                                       frame->sample_rate_hz_,
                                       fwd_in_format_.num_channels(),
                                       fwd_in_format_.num_channels(),
                                       frame->num_channels_));
   if (frame->samples_per_channel_ != rev_in_format_.samples_per_channel()) {
     return kBadDataLengthError;
   }
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_file_->Open()) {
     event_msg_->set_type(audioproc::Event::REVERSE_STREAM);
     audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream();
     const size_t data_size = sizeof(int16_t) *
                              frame->samples_per_channel_ *
                              frame->num_channels_;
     msg->set_data(frame->data_, data_size);
     RETURN_ON_ERR(WriteMessageToDebugFile());
   }
 #endif
+  render_audio_->DeinterleaveFrom(frame);
+  RETURN_ON_ERR(ProcessReverseStreamLocked());
+  render_audio_->InterleaveTo(frame, intelligibility_enabled_);

[Andrew MacDonald, 2015/07/24 23:50:39]

I think more clear if you add the check here:
if (intelligbility_enabled_) {
  InterleaveTo(...);
}

The bool parameter is just so the AudioFrame's VAD state can get updated, which
you don't care about on the reverse stream. 

[ekm, 2015/07/29 00:37:19]

Done. Added is_rev_processed() to clean up these checks.

[Andrew MacDonald, 2015/07/29 03:52:27]

Nice.

 
-  render_audio_->DeinterleaveFrom(frame);
-  return AnalyzeReverseStreamLocked();
+  return kNoError;
 }
 
-int AudioProcessingImpl::AnalyzeReverseStreamLocked() {
+int AudioProcessingImpl::ProcessReverseStreamLocked() {
   AudioBuffer* ra = render_audio_.get();  // For brevity.
   if (rev_proc_format_.rate() == kSampleRate32kHz) {
     ra->SplitIntoFrequencyBands();
   }
 
+  if (intelligibility_enabled_) {
+    intelligibility_enhancer_->ProcessRenderAudio(
+        ra->split_channels_f(kBand0To8kHz), split_rate_, ra->num_channels());
+  }
+
   RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra));
   RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra));
   if (!use_new_agc_) {
     RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra));
   }
 
+  if (rev_proc_format_.rate() == kSampleRate32kHz) {

[Andrew MacDonald, 2015/07/24 23:50:39]

&& intelligbility_enabled_

[ekm, 2015/07/29 00:37:19]

Done.

+    ra->MergeFrequencyBands();
+  }
+
   return kNoError;
 }
 
 int AudioProcessingImpl::set_stream_delay_ms(int delay) {
   Error retval = kNoError;
   was_stream_delay_set_ = true;
   delay += delay_offset_ms_;
 
   if (delay < 0) {
     delay = 0;
@@ skipping 226 matching lines @@
 
 void AudioProcessingImpl::InitializeBeamformer() {
   if (beamformer_enabled_) {
     if (!beamformer_) {
       beamformer_.reset(new NonlinearBeamformer(array_geometry_));
     }
     beamformer_->Initialize(kChunkSizeMs, split_rate_);
   }
 }
 
+void AudioProcessingImpl::InitializeIntelligibility() {
+  if (intelligibility_enabled_) {
+    IntelligibilityEnhancer::Config config;
+    config.sample_rate_hz = split_rate_;
+    config.num_capture_channels = capture_audio_->num_channels();
+    config.num_render_channels = render_audio_->num_channels();
+    intelligibility_enhancer_.reset(new IntelligibilityEnhancer(config));
+  }
+}
+
 void AudioProcessingImpl::MaybeUpdateHistograms() {
   static const int kMinDiffDelayMs = 60;
 
   if (echo_cancellation()->is_enabled()) {
     // Activate delay_jumps_ counters if we know echo_cancellation is runnning.
     // If a stream has echo we know that the echo_cancellation is in process.
     if (stream_delay_jumps_ == -1 && echo_cancellation()->stream_has_echo()) {
       stream_delay_jumps_ = 0;
     }
     if (aec_system_delay_jumps_ == -1 &&
@@ skipping 92 matching lines @@
   int err = WriteMessageToDebugFile();
   if (err != kNoError) {
     return err;
   }
 
   return kNoError;
 }
 #endif  // WEBRTC_AUDIOPROC_DEBUG_DUMP
 
 }  // namespace webrtc