Integrate Intelligibility with APM

webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 //
 //  Implements core class for intelligibility enhancer.
 //
 //  Details of the model and algorithm can be found in the original paper:
 //  http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=6882788
 //
 
 #include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h"
 
 #include <math.h>
 #include <stdlib.h>
-
 #include <algorithm>
 #include <numeric>
 
 #include "webrtc/base/checks.h"
-#include "webrtc/common_audio/vad/include/webrtc_vad.h"
 #include "webrtc/common_audio/window_generator.h"
+#include "webrtc/common_audio/include/audio_util.h"
 
 namespace webrtc {
 
 namespace {
 
 const int kErbResolution = 2;
 const int kWindowSizeMs = 2;
 const int kChunkSizeMs = 10;  // Size provided by APM.
 const float kClipFreq = 200.0f;
 const float kConfigRho = 0.02f;  // Default production and interpretation SNR.
 const float kKbdAlpha = 1.5f;
 const float kLambdaBot = -1.0f;      // Extreme values in bisection
 const float kLambdaTop = -10e-18f;  // search for lamda.
+const float kMinNoise = 10e-18f;
 
 }  // namespace
 
 using std::complex;
 using std::max;
 using std::min;
 using VarianceType = intelligibility::VarianceArray::StepType;
 
 IntelligibilityEnhancer::TransformCallback::TransformCallback(
     IntelligibilityEnhancer* parent,
     IntelligibilityEnhancer::AudioSource source)
     : parent_(parent), source_(source) {
 }
 
 void IntelligibilityEnhancer::TransformCallback::ProcessAudioBlock(
     const complex<float>* const* in_block,
     int in_channels,
     int frames,
     int /* out_channels */,
     complex<float>* const* out_block) {
   DCHECK_EQ(parent_->freqs_, frames);
   for (int i = 0; i < in_channels; ++i) {
     parent_->DispatchAudio(source_, in_block[i], out_block[i]);
   }
 }
 
-IntelligibilityEnhancer::IntelligibilityEnhancer(int erb_resolution,
-                                                 int sample_rate_hz,
-                                                 int channels,
-                                                 int cv_type,
-                                                 float cv_alpha,
-                                                 int cv_win,
-                                                 int analysis_rate,
-                                                 int variance_rate,
-                                                 float gain_limit)
+IntelligibilityEnhancer::IntelligibilityEnhancer()
+    : IntelligibilityEnhancer(IntelligibilityEnhancer::Config()) {
+}
+
+IntelligibilityEnhancer::IntelligibilityEnhancer(const Config& config)
     : freqs_(RealFourier::ComplexLength(
-          RealFourier::FftOrder(sample_rate_hz * kWindowSizeMs / 1000))),
+          RealFourier::FftOrder(config.sample_rate_hz * kWindowSizeMs / 1000))),
       window_size_(1 << RealFourier::FftOrder(freqs_)),
-      chunk_length_(sample_rate_hz * kChunkSizeMs / 1000),
-      bank_size_(GetBankSize(sample_rate_hz, erb_resolution)),
-      sample_rate_hz_(sample_rate_hz),
-      erb_resolution_(erb_resolution),
-      channels_(channels),
-      analysis_rate_(analysis_rate),
-      variance_rate_(variance_rate),
+      chunk_length_(config.sample_rate_hz * kChunkSizeMs / 1000),
+      bank_size_(GetBankSize(config.sample_rate_hz, kErbResolution)),
+      sample_rate_hz_(config.sample_rate_hz),
+      erb_resolution_(kErbResolution),
+      channels_(config.channels),
+      analysis_rate_(config.analysis_rate),
+      capture_vad_thresh_(config.capture_vad_thresh),
+      render_vad_thresh_(config.render_vad_thresh),
+      activate_snr_thresh_(config.activate_snr_thresh),
+      deactivate_snr_thresh_(config.deactivate_snr_thresh),
+      active_(false),
+      deactivating_(false),
       clear_variance_(freqs_,
-                      static_cast<VarianceType>(cv_type),
-                      cv_win,
-                      cv_alpha),
-      noise_variance_(freqs_, VarianceType::kStepInfinite, 475, 0.01f),
+                      config.var_type,
+                      config.var_window_size,
+                      config.var_decay_rate),
+      noise_variance_(freqs_,
+                      config.var_type,
+                      config.var_window_size,
+                      config.var_decay_rate),
       filtered_clear_var_(new float[bank_size_]),
       filtered_noise_var_(new float[bank_size_]),
       filter_bank_(bank_size_),
       center_freqs_(new float[bank_size_]),
       rho_(new float[bank_size_]),
       gains_eq_(new float[bank_size_]),
-      gain_applier_(freqs_, gain_limit),
+      gain_applier_(freqs_, config.gain_change_limit),
       temp_out_buffer_(nullptr),
-      input_audio_(new float* [channels]),
       kbd_window_(new float[window_size_]),
       render_callback_(this, AudioSource::kRenderStream),
       capture_callback_(this, AudioSource::kCaptureStream),
       block_count_(0),
       analysis_step_(0),
-      vad_high_(WebRtcVad_Create()),
-      vad_low_(WebRtcVad_Create()),
+      using_capture_vad_(true),
+      using_render_vad_(true),
       vad_tmp_buffer_(new int16_t[chunk_length_]) {
-  DCHECK_LE(kConfigRho, 1.0f);
+  DCHECK_LE(config.rho, 1.0f);
 
   CreateErbBank();
 
-  WebRtcVad_Init(vad_high_);
-  WebRtcVad_set_mode(vad_high_, 0);  // High likelihood of speech.
-  WebRtcVad_Init(vad_low_);
-  WebRtcVad_set_mode(vad_low_, 3);  // Low likelihood of speech.
-
   temp_out_buffer_ = static_cast<float**>(
       malloc(sizeof(*temp_out_buffer_) * channels_ +
              sizeof(**temp_out_buffer_) * chunk_length_ * channels_));
   for (int i = 0; i < channels_; ++i) {
     temp_out_buffer_[i] =
         reinterpret_cast<float*>(temp_out_buffer_ + channels_) +
         chunk_length_ * i;
   }
 
   // Assumes all rho equal.
   for (int i = 0; i < bank_size_; ++i) {
-    rho_[i] = kConfigRho * kConfigRho;
+    rho_[i] = config.rho * config.rho;
   }
 
   float freqs_khz = kClipFreq / 1000.0f;
   int erb_index = static_cast<int>(ceilf(
       11.17f * logf((freqs_khz + 0.312f) / (freqs_khz + 14.6575f)) + 43.0f));
-  start_freq_ = max(1, erb_index * kErbResolution);
+  start_freq_ = max(1, erb_index * erb_resolution_);
 
   WindowGenerator::KaiserBesselDerived(kKbdAlpha, window_size_,
                                        kbd_window_.get());
   render_mangler_.reset(new LappedTransform(
       channels_, channels_, chunk_length_, kbd_window_.get(), window_size_,
       window_size_ / 2, &render_callback_));
   capture_mangler_.reset(new LappedTransform(
       channels_, channels_, chunk_length_, kbd_window_.get(), window_size_,
       window_size_ / 2, &capture_callback_));
 }
 
 IntelligibilityEnhancer::~IntelligibilityEnhancer() {
-  WebRtcVad_Free(vad_low_);
-  WebRtcVad_Free(vad_high_);
   free(temp_out_buffer_);
 }
 
-void IntelligibilityEnhancer::ProcessRenderAudio(float* const* audio) {
-  for (int i = 0; i < chunk_length_; ++i) {
-    vad_tmp_buffer_[i] = (int16_t)audio[0][i];
+void IntelligibilityEnhancer::ProcessRenderAudio(float* const* audio,
+                                                 int sample_rate_hz,
+                                                 int num_channels,
+                                                 float voice_probability) {
+  render_voice_probability_ = voice_probability;
+  using_render_vad_ = false;
+  ProcessRenderAudio(audio, sample_rate_hz, num_channels);
+}
+
+void IntelligibilityEnhancer::ProcessRenderAudio(float* const* audio,
+                                                 int sample_rate_hz,
+                                                 int num_channels) {
+  CHECK_EQ(sample_rate_hz, sample_rate_hz_);
+  CHECK_EQ(num_channels, channels_);

[aluebs-webrtc, 2015/07/20 19:33:43]

Use num_channels_ for the member variable as well? It is clearer.

[ekm, 2015/07/21 01:02:44]

Done.

+
+  if (using_render_vad_) {
+    FloatToS16(audio[0], chunk_length_, vad_tmp_buffer_.get());
+    render_vad_.ProcessChunk(vad_tmp_buffer_.get(), chunk_length_,
+                             sample_rate_hz_);
+    render_voice_probability_ = render_vad_.last_voice_probability();
   }
-  has_voice_low_ = WebRtcVad_Process(vad_low_, sample_rate_hz_,
-                                     vad_tmp_buffer_.get(), chunk_length_) == 1;
 
-  // Process and enhance chunk of |audio|
-  render_mangler_->ProcessChunk(audio, temp_out_buffer_);
+  if (render_voice_probability_ >= render_vad_thresh_ || active_) {

[aluebs-webrtc, 2015/07/20 19:33:43]

I am not sure why the voice probability affects if the chunk gets processed.
Maybe I am just missing something.

[ekm, 2015/07/21 01:02:44]

If we're pretty sure the far-end chunk contains speech, we'd like to use it in
our estimation of the speech signal.

[aluebs-webrtc, 2015/07/21 01:50:55]

Oh! ProcessChunk doesn't really processes the chunk, it just uses it for
estimation, right? But then why are all chunks used for estimation when active?
I feel I am still missing something.

[ekm, 2015/07/21 19:22:13]

ProcessChunk transforms audio to frequency domain, which we need for both
estimation and processing. We overload the logic a bit here so we only have to
do the transform once. In this case, render_mangler_->ProcessChunk does the
transform then goes to ProcessClearBlock, where it decides what to do with the
freq data, either estimate, process or both.

[aluebs-webrtc, 2015/07/21 21:30:22]

Oh, I see. Thanks for clarifying.

+    render_mangler_->ProcessChunk(audio, temp_out_buffer_);
+  }
 
   for (int i = 0; i < channels_; ++i) {
     memcpy(audio[i], temp_out_buffer_[i],
            chunk_length_ * sizeof(**temp_out_buffer_));
   }
 }
 
-void IntelligibilityEnhancer::ProcessCaptureAudio(float* const* audio) {
-  for (int i = 0; i < chunk_length_; ++i) {
-    vad_tmp_buffer_[i] = (int16_t)audio[0][i];
+void IntelligibilityEnhancer::AnalyzeCaptureAudio(float* const* audio,
+                                                  int sample_rate_hz,
+                                                  int num_channels,
+                                                  float voice_probability) {
+  capture_voice_probability_ = voice_probability;
+  using_capture_vad_ = false;
+  AnalyzeCaptureAudio(audio, sample_rate_hz, num_channels);
+}
+
+void IntelligibilityEnhancer::AnalyzeCaptureAudio(float* const* audio,
+                                                  int sample_rate_hz,
+                                                  int num_channels) {
+  CHECK_EQ(sample_rate_hz, sample_rate_hz_);
+  CHECK_EQ(num_channels, channels_);
+
+  if (using_capture_vad_) {
+    FloatToS16(audio[0], chunk_length_, vad_tmp_buffer_.get());
+    capture_vad_.ProcessChunk(vad_tmp_buffer_.get(), chunk_length_,
+                              sample_rate_hz_);
+    capture_voice_probability_ = capture_vad_.last_voice_probability();
   }
-  // TODO(bercic): The VAD was always detecting voice in the noise stream,
-  // no matter what the aggressiveness, so it was temporarily disabled here.
 
-  #if 0
-    if (WebRtcVad_Process(vad_high_, sample_rate_hz_, vad_tmp_buffer_.get(),
-      chunk_length_) == 1) {
-      printf("capture HAS speech\n");
-      return;
-    }
-    printf("capture NO speech\n");
-  #endif
-
-  capture_mangler_->ProcessChunk(audio, temp_out_buffer_);
+  if (capture_voice_probability_ <= capture_vad_thresh_) {

[aluebs-webrtc, 2015/07/20 19:33:43]

I am not sure why the voice probability affects if the chunk gets processed.
Maybe I am just missing something.

[ekm, 2015/07/21 01:02:44]

If we're pretty sure the near-end chunk contains noise, we'd like to use it in
our estimation of the noise signal.

[aluebs-webrtc, 2015/07/21 01:50:55]

That makes sense. I was not aware that ProcessChunk was only estimating.

[ekm, 2015/07/21 19:22:13]

Yeah, in the capture case, the freq data is only used for estimation.

+    capture_mangler_->ProcessChunk(audio, temp_out_buffer_);
+  }
 }
 
 void IntelligibilityEnhancer::DispatchAudio(
     IntelligibilityEnhancer::AudioSource source,
     const complex<float>* in_block,
     complex<float>* out_block) {
   switch (source) {
     case kRenderStream:
       ProcessClearBlock(in_block, out_block);
       break;
     case kCaptureStream:
       ProcessNoiseBlock(in_block, out_block);
       break;
   }
 }
 
 void IntelligibilityEnhancer::ProcessClearBlock(const complex<float>* in_block,
                                                 complex<float>* out_block) {
   if (block_count_ < 2) {
     memset(out_block, 0, freqs_ * sizeof(*out_block));
     ++block_count_;
     return;
   }
 
-  // For now, always assumes enhancement is necessary.
-  // TODO(ekmeyerson): Change to only enhance if necessary,
-  // based on experiments with different cutoffs.
-  if (has_voice_low_ || true) {
+  if (render_voice_probability_ >= render_vad_thresh_) {

[aluebs-webrtc, 2015/07/20 19:33:43]

This changed from has_voice_low to checking if the render_voice_probability is
above the threshold. Isn't that just the opposite?

[ekm, 2015/07/21 01:02:44]

The has_voice_low_ was referring to detection of voice with 'low' aggressiveness
with the common_audio vad.

[aluebs-webrtc, 2015/07/21 01:50:55]

Oh, I see. Then it makes sense.

     clear_variance_.Step(in_block, false);
-    const float power_target = std::accumulate(
-        clear_variance_.variance(), clear_variance_.variance() + freqs_, 0.0f);
-
-    if (block_count_ % analysis_rate_ == analysis_rate_ - 1) {
+    if (active_ && !deactivating_ &&
+        block_count_ % analysis_rate_ == analysis_rate_ - 1) {
+      const float power_target = std::accumulate(
+          clear_variance_.variance(), clear_variance_.variance() + freqs_, 0.f);
       AnalyzeClearBlock(power_target);
       ++analysis_step_;
-      if (analysis_step_ == variance_rate_) {
-        analysis_step_ = 0;
-        clear_variance_.Clear();
-        noise_variance_.Clear();
-      }
     }
     ++block_count_;
   }
 
-  /* efidata(n,:) = sqrt(b(n)) * fidata(n,:) */
-  gain_applier_.Apply(in_block, out_block);
+  UpdateActivity();
+  if (active_) {
+    // efidata(n,:) = sqrt(b(n)) * fidata(n,:)
+    gain_applier_.Apply(in_block, out_block);
+  }
 }
 
 void IntelligibilityEnhancer::AnalyzeClearBlock(float power_target) {
   FilterVariance(clear_variance_.variance(), filtered_clear_var_.get());
   FilterVariance(noise_variance_.variance(), filtered_noise_var_.get());
 
   SolveForGainsGivenLambda(kLambdaTop, start_freq_, gains_eq_.get());
   const float power_top =
       DotProduct(gains_eq_.get(), filtered_clear_var_.get(), bank_size_);
   SolveForGainsGivenLambda(kLambdaBot, start_freq_, gains_eq_.get());
@@ skipping 146 matching lines @@
     sols[n] = fmax(0, sols[n]);
   }
 }
 
 void IntelligibilityEnhancer::FilterVariance(const float* var, float* result) {
   for (int i = 0; i < bank_size_; ++i) {
     result[i] = DotProduct(filter_bank_[i].data(), var, freqs_);
   }
 }
 
+float IntelligibilityEnhancer::SNR() {
+  float total_clear_var = std::accumulate(
+      clear_variance_.variance(), clear_variance_.variance() + freqs_, 0.f);
+  float total_noise_var =
+      std::accumulate(noise_variance_.variance(),
+                      noise_variance_.variance() + freqs_, kMinNoise);
+  return total_clear_var / total_noise_var;
+}
+
+void IntelligibilityEnhancer::UpdateActivity() {
+  const float snr = SNR();
+  if (snr <= activate_snr_thresh_) {
+    active_ = true;
+    deactivating_ = false;
+  } else if (active_ && !deactivating_ && snr >= deactivate_snr_thresh_) {
+    gain_applier_.Clear();
+    deactivating_ = true;
+  } else if (deactivating_ && gain_applier_.IsIdentity()) {
+    active_ = false;
+    deactivating_ = false;
+  }
+}
+
 float IntelligibilityEnhancer::DotProduct(const float* a,
                                           const float* b,
                                           int length) {
   float ret = 0.0f;
 
   for (int i = 0; i < length; ++i) {
     ret = fmaf(a[i], b[i], ret);
   }
   return ret;
 }
 
 }  // namespace webrtc