Integrate Intelligibility with APM

webrtc/modules/audio_processing/audio_processing_impl.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 

[aluebs-webrtc, 2015/08/03 16:30:49]

Re-add blank line.

[ekm, 2015/08/05 05:24:08]

Done.

 #include "webrtc/modules/audio_processing/audio_processing_impl.h"
 
 #include <assert.h>
 #include <algorithm>
 
 #include "webrtc/base/checks.h"
 #include "webrtc/base/platform_file.h"
 #include "webrtc/common_audio/include/audio_util.h"
 #include "webrtc/common_audio/channel_buffer.h"
 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
 extern "C" {
 #include "webrtc/modules/audio_processing/aec/aec_core.h"
 }
 #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h"
 #include "webrtc/modules/audio_processing/audio_buffer.h"
 #include "webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h"
 #include "webrtc/modules/audio_processing/common.h"
 #include "webrtc/modules/audio_processing/echo_cancellation_impl.h"
 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h"
 #include "webrtc/modules/audio_processing/gain_control_impl.h"
 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h"
+#include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h"
 #include "webrtc/modules/audio_processing/level_estimator_impl.h"
 #include "webrtc/modules/audio_processing/noise_suppression_impl.h"
 #include "webrtc/modules/audio_processing/processing_component.h"
 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h"
 #include "webrtc/modules/audio_processing/voice_detection_impl.h"
 #include "webrtc/modules/interface/module_common_types.h"
 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
 #include "webrtc/system_wrappers/interface/file_wrapper.h"
 #include "webrtc/system_wrappers/interface/logging.h"
 #include "webrtc/system_wrappers/interface/metrics.h"
@@ skipping 135 matching lines @@
       level_estimator_(NULL),
       noise_suppression_(NULL),
       voice_detection_(NULL),
       crit_(CriticalSectionWrapper::CreateCriticalSection()),
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
       debug_file_(FileWrapper::Create()),
       event_msg_(new audioproc::Event()),
 #endif
       api_format_({{{kSampleRate16kHz, 1, false},
                     {kSampleRate16kHz, 1, false},
+                    {kSampleRate16kHz, 1, false},
                     {kSampleRate16kHz, 1, false}}}),
       fwd_proc_format_(kSampleRate16kHz),
       rev_proc_format_(kSampleRate16kHz, 1),
       split_rate_(kSampleRate16kHz),
       stream_delay_ms_(0),
       delay_offset_ms_(0),
       was_stream_delay_set_(false),
       last_stream_delay_ms_(0),
       last_aec_system_delay_ms_(0),
       stream_delay_jumps_(-1),
       aec_system_delay_jumps_(-1),
       output_will_be_muted_(false),
       key_pressed_(false),
 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
       use_new_agc_(false),
 #else
       use_new_agc_(config.Get<ExperimentalAgc>().enabled),
 #endif
       agc_startup_min_volume_(config.Get<ExperimentalAgc>().startup_min_volume),
 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
       transient_suppressor_enabled_(false),
 #else
       transient_suppressor_enabled_(config.Get<ExperimentalNs>().enabled),
 #endif
       beamformer_enabled_(config.Get<Beamforming>().enabled),
       beamformer_(beamformer),
-      array_geometry_(config.Get<Beamforming>().array_geometry) {
+      array_geometry_(config.Get<Beamforming>().array_geometry),
+      intelligibility_enabled_(config.Get<Intelligibility>().enabled) {
   echo_cancellation_ = new EchoCancellationImpl(this, crit_);
   component_list_.push_back(echo_cancellation_);
 
   echo_control_mobile_ = new EchoControlMobileImpl(this, crit_);
   component_list_.push_back(echo_control_mobile_);
 
   gain_control_ = new GainControlImpl(this, crit_);
   component_list_.push_back(gain_control_);
 
   high_pass_filter_ = new HighPassFilterImpl(this, crit_);
@@ skipping 51 matching lines @@
   return InitializeLocked(processing_config);
 }
 
 int AudioProcessingImpl::Initialize(int input_sample_rate_hz,
                                     int output_sample_rate_hz,
                                     int reverse_sample_rate_hz,
                                     ChannelLayout input_layout,
                                     ChannelLayout output_layout,
                                     ChannelLayout reverse_layout) {
   const ProcessingConfig processing_config = {
-      {{input_sample_rate_hz, ChannelsFromLayout(input_layout),
+      {{input_sample_rate_hz,
+        ChannelsFromLayout(input_layout),
         LayoutHasKeyboard(input_layout)},
-       {output_sample_rate_hz, ChannelsFromLayout(output_layout),
+       {output_sample_rate_hz,
+        ChannelsFromLayout(output_layout),
         LayoutHasKeyboard(output_layout)},
-       {reverse_sample_rate_hz, ChannelsFromLayout(reverse_layout),
+       {reverse_sample_rate_hz,
+        ChannelsFromLayout(reverse_layout),
+        LayoutHasKeyboard(reverse_layout)},
+       {reverse_sample_rate_hz,
+        ChannelsFromLayout(reverse_layout),
         LayoutHasKeyboard(reverse_layout)}}};
 
   return Initialize(processing_config);
 }
 
 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) {
   CriticalSectionScoped crit_scoped(crit_);
   return InitializeLocked(processing_config);
 }
 
 int AudioProcessingImpl::InitializeLocked() {
   const int fwd_audio_buffer_channels =
       beamformer_enabled_ ? api_format_.input_stream().num_channels()
                           : api_format_.output_stream().num_channels();
-  if (api_format_.reverse_stream().num_channels() > 0) {
+  const int rev_audio_buffer_out_num_frames =
+      api_format_.reverse_output_stream().num_frames() == 0
+          ? rev_proc_format_.num_frames()
+          : api_format_.reverse_output_stream().num_frames();
+  if (api_format_.reverse_input_stream().num_channels() > 0) {
     render_audio_.reset(new AudioBuffer(
-        api_format_.reverse_stream().num_frames(),
-        api_format_.reverse_stream().num_channels(),
+        api_format_.reverse_input_stream().num_frames(),
+        api_format_.reverse_input_stream().num_channels(),
         rev_proc_format_.num_frames(), rev_proc_format_.num_channels(),
-        rev_proc_format_.num_frames()));
+        rev_audio_buffer_out_num_frames));
   } else {
     render_audio_.reset(nullptr);
   }
   capture_audio_.reset(new AudioBuffer(
       api_format_.input_stream().num_frames(),
       api_format_.input_stream().num_channels(), fwd_proc_format_.num_frames(),
       fwd_audio_buffer_channels, api_format_.output_stream().num_frames()));
 
   // Initialize all components.
   for (auto item : component_list_) {
     int err = item->Initialize();
     if (err != kNoError) {
       return err;
     }
   }
 
   InitializeExperimentalAgc();
 
   InitializeTransient();
 
   InitializeBeamformer();
 
+  InitializeIntelligibility();
+
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_file_->Open()) {
     int err = WriteInitMessage();
     if (err != kNoError) {
       return err;
     }
   }
 #endif
 
   return kNoError;
@@ skipping 47 matching lines @@
   }
 
   fwd_proc_format_ = StreamConfig(fwd_proc_rate);
 
   // We normally process the reverse stream at 16 kHz. Unless...
   int rev_proc_rate = kSampleRate16kHz;
   if (fwd_proc_format_.sample_rate_hz() == kSampleRate8kHz) {
     // ...the forward stream is at 8 kHz.
     rev_proc_rate = kSampleRate8kHz;
   } else {
-    if (api_format_.reverse_stream().sample_rate_hz() == kSampleRate32kHz) {
+    if (api_format_.reverse_input_stream().sample_rate_hz() ==
+        kSampleRate32kHz) {
       // ...or the input is at 32 kHz, in which case we use the splitting
       // filter rather than the resampler.
       rev_proc_rate = kSampleRate32kHz;
     }
   }
 
   // Always downmix the reverse stream to mono for analysis. This has been
   // demonstrated to work well for AEC in most practical scenarios.
   rev_proc_format_ = StreamConfig(rev_proc_rate, 1);
 
@@ skipping 206 matching lines @@
     msg->set_delay(stream_delay_ms_);
     msg->set_drift(echo_cancellation_->stream_drift_samples());
     msg->set_level(gain_control()->stream_analog_level());
     msg->set_keypress(key_pressed_);
   }
 #endif
 
   MaybeUpdateHistograms();
 
   AudioBuffer* ca = capture_audio_.get();  // For brevity.
+
   if (use_new_agc_ && gain_control_->is_enabled()) {
     agc_manager_->AnalyzePreProcess(ca->channels()[0], ca->num_channels(),
                                     fwd_proc_format_.num_frames());
   }
 
   bool data_processed = is_data_processed();
   if (analysis_needed(data_processed)) {
     ca->SplitIntoFrequencyBands();
   }
 
+  if (intelligibility_enabled_) {
+    intelligibility_enhancer_->AnalyzeCaptureAudio(
+        ca->split_channels_f(kBand0To8kHz), split_rate_, ca->num_channels());
+  }
+
   if (beamformer_enabled_) {
     beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f());
     ca->set_num_channels(1);
   }
 
   RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca));
   RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca));
   RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca));
   RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca));
 
@@ skipping 30 matching lines @@
 
   // The level estimator operates on the recombined data.
   RETURN_ON_ERR(level_estimator_->ProcessStream(ca));
 
   was_stream_delay_set_ = false;
   return kNoError;
 }
 
 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data,
                                               int samples_per_channel,
-                                              int sample_rate_hz,
+                                              int rev_sample_rate_hz,
                                               ChannelLayout layout) {
   const StreamConfig reverse_config = {
-      sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout),
+      rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout),
   };
   if (samples_per_channel != reverse_config.num_frames()) {
     return kBadDataLengthError;
   }
-  return AnalyzeReverseStream(data, reverse_config);
+  return AnalyzeReverseStream(data, reverse_config, reverse_config, data);
+}
+
+int AudioProcessingImpl::ProcessReverseStream(
+    const float* const* src,
+    const StreamConfig& reverse_input_config,
+    const StreamConfig& reverse_output_config,
+    float* const* dest) {
+  RETURN_ON_ERR(AnalyzeReverseStream(src, reverse_input_config,
+                                     reverse_output_config, dest));
+  if (is_rev_processed()) {

[Andrew MacDonald, 2015/07/31 01:22:42]

So, we need to change this part. This is what I meant about being consistent: if
the user requests a different output config, you should provide that regardless
of processing.

However, if we have no processing and the user requests a different output
config, we need to convert from src not render_audio_. Otherwise, say the user
passes in 48 kHz and they request 44.1 kHz, they'll actually get 16 kHz audio
upsampled to 44.1.

Perhaps my olde AudioConverter can finally see the light of day?

if (is_rev_processed() {
  ...
} else {
  render_converter_->Convert(src, ..., dest);
}

where you only create render_converter_ in InitializeLocked() when appropriate.

Alex, what do you think? We could back out the output_config addition, but then
we'd still then need at least a copy from src to dest here, so it doesn't help a
lot.

[aluebs-webrtc, 2015/07/31 16:51:03]

I like your proposal. I don't think that dropping the out config helps, so I
would keep it, but I don't have a strong opinion.

[ekm, 2015/08/01 04:21:37]

I was intrigued and added the AudioConverter method, but kept interface the same
for now.

+    render_audio_->CopyTo(api_format_.reverse_output_stream(), dest);
+  }
+
+  return kNoError;
 }
 
 int AudioProcessingImpl::AnalyzeReverseStream(
-    const float* const* data,
-    const StreamConfig& reverse_config) {
+    const float* const* src,
+    const StreamConfig& reverse_input_config,
+    const StreamConfig& reverse_output_config,
+    const float* const* dest) {
   CriticalSectionScoped crit_scoped(crit_);
-  if (data == NULL) {
+  if (src == NULL) {
     return kNullPointerError;
   }
 
-  if (reverse_config.num_channels() <= 0) {
+  if (reverse_input_config.num_channels() <= 0) {
     return kBadNumberChannelsError;
   }
 
   ProcessingConfig processing_config = api_format_;
-  processing_config.reverse_stream() = reverse_config;
+  processing_config.reverse_input_stream() = reverse_input_config;
+  processing_config.reverse_output_stream() = reverse_output_config;
 
   RETURN_ON_ERR(MaybeInitializeLocked(processing_config));
-  assert(reverse_config.num_frames() ==
-         api_format_.reverse_stream().num_frames());
+  assert(reverse_input_config.num_frames() ==
+         api_format_.reverse_input_stream().num_frames());
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_file_->Open()) {
     event_msg_->set_type(audioproc::Event::REVERSE_STREAM);
     audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream();
     const size_t channel_size =
-        sizeof(float) * api_format_.reverse_stream().num_frames();
-    for (int i = 0; i < api_format_.reverse_stream().num_channels(); ++i)
-      msg->add_channel(data[i], channel_size);
+        sizeof(float) * api_format_.reverse_input_stream().num_frames();
+    for (int i = 0; i < api_format_.reverse_input_stream().num_channels(); ++i)
+      msg->add_channel(src[i], channel_size);
     RETURN_ON_ERR(WriteMessageToDebugFile());
   }
 #endif
 
-  render_audio_->CopyFrom(data, api_format_.reverse_stream());
-  return AnalyzeReverseStreamLocked();
+  render_audio_->CopyFrom(src, api_format_.reverse_input_stream());
+  return ProcessReverseStreamLocked();
+}
+
+int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) {
+  RETURN_ON_ERR(AnalyzeReverseStream(frame));
+  if (is_rev_processed()) {

[Andrew MacDonald, 2015/07/31 01:22:42]

You can leave this one, because the user has no way to specify an output config.

[ekm, 2015/08/01 04:21:37]

Acknowledged.

+    render_audio_->InterleaveTo(frame, true);
+  }
+
+  return kNoError;
 }
 
 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) {
   CriticalSectionScoped crit_scoped(crit_);
   if (frame == NULL) {
     return kNullPointerError;
   }
   // Must be a native rate.
   if (frame->sample_rate_hz_ != kSampleRate8kHz &&
       frame->sample_rate_hz_ != kSampleRate16kHz &&
       frame->sample_rate_hz_ != kSampleRate32kHz &&
       frame->sample_rate_hz_ != kSampleRate48kHz) {
     return kBadSampleRateError;
   }
   // This interface does not tolerate different forward and reverse rates.
   if (frame->sample_rate_hz_ != api_format_.input_stream().sample_rate_hz()) {
     return kBadSampleRateError;
   }
 
   if (frame->num_channels_ <= 0) {
     return kBadNumberChannelsError;
   }
 
   ProcessingConfig processing_config = api_format_;
-  processing_config.reverse_stream().set_sample_rate_hz(frame->sample_rate_hz_);
-  processing_config.reverse_stream().set_num_channels(frame->num_channels_);
+  processing_config.reverse_input_stream().set_sample_rate_hz(
+      frame->sample_rate_hz_);
+  processing_config.reverse_input_stream().set_num_channels(
+      frame->num_channels_);
+  processing_config.reverse_output_stream().set_sample_rate_hz(
+      frame->sample_rate_hz_);
+  processing_config.reverse_output_stream().set_num_channels(
+      frame->num_channels_);
 
   RETURN_ON_ERR(MaybeInitializeLocked(processing_config));
   if (frame->samples_per_channel_ !=
-      api_format_.reverse_stream().num_frames()) {
+      api_format_.reverse_input_stream().num_frames()) {
     return kBadDataLengthError;
   }
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_file_->Open()) {
     event_msg_->set_type(audioproc::Event::REVERSE_STREAM);
     audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream();
     const size_t data_size =
         sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_;
     msg->set_data(frame->data_, data_size);
     RETURN_ON_ERR(WriteMessageToDebugFile());
   }
 #endif
-
   render_audio_->DeinterleaveFrom(frame);
-  return AnalyzeReverseStreamLocked();
+  return ProcessReverseStreamLocked();
 }
 
-int AudioProcessingImpl::AnalyzeReverseStreamLocked() {
+int AudioProcessingImpl::ProcessReverseStreamLocked() {
   AudioBuffer* ra = render_audio_.get();  // For brevity.
   if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz) {
     ra->SplitIntoFrequencyBands();
   }
 
+  if (intelligibility_enabled_) {
+    intelligibility_enhancer_->ProcessRenderAudio(
+        ra->split_channels_f(kBand0To8kHz), split_rate_, ra->num_channels());
+  }
+
   RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra));
   RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra));
   if (!use_new_agc_) {
     RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra));
   }
 
+  if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz &&
+      is_rev_processed()) {
+    ra->MergeFrequencyBands();
+  }
+
   return kNoError;
 }
 
 int AudioProcessingImpl::set_stream_delay_ms(int delay) {
   Error retval = kNoError;
   was_stream_delay_set_ = true;
   delay += delay_offset_ms_;
 
   if (delay < 0) {
     delay = 0;
@@ skipping 196 matching lines @@
     // Only level_estimator_ is enabled.
     return false;
   } else if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz ||
              fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) {
     // Something besides level_estimator_ is enabled, and we have super-wb.
     return true;
   }
   return false;
 }
 
+bool AudioProcessingImpl::is_rev_processed() const {
+  return intelligibility_enabled_ && intelligibility_enhancer_->active();
+}
+
 void AudioProcessingImpl::InitializeExperimentalAgc() {
   if (use_new_agc_) {
     if (!agc_manager_.get()) {
       agc_manager_.reset(new AgcManagerDirect(gain_control_,
                                               gain_control_for_new_agc_.get(),
                                               agc_startup_min_volume_));
     }
     agc_manager_->Initialize();
     agc_manager_->SetCaptureMuted(output_will_be_muted_);
   }
@@ skipping 12 matching lines @@
 
 void AudioProcessingImpl::InitializeBeamformer() {
   if (beamformer_enabled_) {
     if (!beamformer_) {
       beamformer_.reset(new NonlinearBeamformer(array_geometry_));
     }
     beamformer_->Initialize(kChunkSizeMs, split_rate_);
   }
 }
 
+void AudioProcessingImpl::InitializeIntelligibility() {
+  if (intelligibility_enabled_) {
+    IntelligibilityEnhancer::Config config;
+    config.sample_rate_hz = split_rate_;
+    config.num_capture_channels = capture_audio_->num_channels();
+    config.num_render_channels = render_audio_->num_channels();
+    intelligibility_enhancer_.reset(new IntelligibilityEnhancer(config));
+  }
+}
+
 void AudioProcessingImpl::MaybeUpdateHistograms() {
   static const int kMinDiffDelayMs = 60;
 
   if (echo_cancellation()->is_enabled()) {
     // Activate delay_jumps_ counters if we know echo_cancellation is runnning.
     // If a stream has echo we know that the echo_cancellation is in process.
     if (stream_delay_jumps_ == -1 && echo_cancellation()->stream_has_echo()) {
       stream_delay_jumps_ = 0;
     }
     if (aec_system_delay_jumps_ == -1 &&
@@ skipping 78 matching lines @@
 
   return kNoError;
 }
 
 int AudioProcessingImpl::WriteInitMessage() {
   event_msg_->set_type(audioproc::Event::INIT);
   audioproc::Init* msg = event_msg_->mutable_init();
   msg->set_sample_rate(api_format_.input_stream().sample_rate_hz());
   msg->set_num_input_channels(api_format_.input_stream().num_channels());
   msg->set_num_output_channels(api_format_.output_stream().num_channels());
-  msg->set_num_reverse_channels(api_format_.reverse_stream().num_channels());
-  msg->set_reverse_sample_rate(api_format_.reverse_stream().sample_rate_hz());
+  msg->set_num_reverse_channels(
+      api_format_.reverse_input_stream().num_channels());
+  msg->set_reverse_sample_rate(
+      api_format_.reverse_input_stream().sample_rate_hz());
   msg->set_output_sample_rate(api_format_.output_stream().sample_rate_hz());
+  // TODO(ekmeyerson): Add reverse output fields to event_msg_.
 
   int err = WriteMessageToDebugFile();
   if (err != kNoError) {
     return err;
   }
 
   return kNoError;
 }
 #endif  // WEBRTC_AUDIOPROC_DEBUG_DUMP
 
 }  // namespace webrtc