Integrate Intelligibility with APM

webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 //
 //  Implements core class for intelligibility enhancer.
 //
 //  Details of the model and algorithm can be found in the original paper:
 //  http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=6882788
 //
 
 #include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h"
 
 #include <math.h>
 #include <stdlib.h>
-
 #include <algorithm>
 #include <numeric>
 
 #include "webrtc/base/checks.h"
 #include "webrtc/common_audio/vad/include/webrtc_vad.h"
 #include "webrtc/common_audio/window_generator.h"
 
 namespace webrtc {
 
 namespace {
 
 const int kErbResolution = 2;
 const int kWindowSizeMs = 2;
 const int kChunkSizeMs = 10;  // Size provided by APM.
 const float kClipFreq = 200.0f;
 const float kConfigRho = 0.02f;  // Default production and interpretation SNR.
 const float kKbdAlpha = 1.5f;
 const float kLambdaBot = -1.0f;      // Extreme values in bisection
 const float kLambdaTop = -10e-18f;  // search for lamda.
+const float kVoiceDetected = 1.f;
+const float kNoiseDetected = 0.f;
 
 }  // namespace
 
 using std::complex;
 using std::max;
 using std::min;
 using VarianceType = intelligibility::VarianceArray::StepType;
 
 IntelligibilityEnhancer::TransformCallback::TransformCallback(
     IntelligibilityEnhancer* parent,
     IntelligibilityEnhancer::AudioSource source)
     : parent_(parent), source_(source) {
 }
 
 void IntelligibilityEnhancer::TransformCallback::ProcessAudioBlock(
     const complex<float>* const* in_block,
     int in_channels,
     int frames,
     int /* out_channels */,
     complex<float>* const* out_block) {
   DCHECK_EQ(parent_->freqs_, frames);
   for (int i = 0; i < in_channels; ++i) {
     parent_->DispatchAudio(source_, in_block[i], out_block[i]);
   }
 }
 
-IntelligibilityEnhancer::IntelligibilityEnhancer(int erb_resolution,
-                                                 int sample_rate_hz,
-                                                 int channels,
-                                                 int cv_type,
-                                                 float cv_alpha,
-                                                 int cv_win,
-                                                 int analysis_rate,
-                                                 int variance_rate,
-                                                 float gain_limit)
+IntelligibilityEnhancer::IntelligibilityEnhancer()
+    : IntelligibilityEnhancer(IntelligibilityEnhancer::Config()) {
+}
+
+IntelligibilityEnhancer::IntelligibilityEnhancer(const Config& config)
     : freqs_(RealFourier::ComplexLength(
-          RealFourier::FftOrder(sample_rate_hz * kWindowSizeMs / 1000))),
+          RealFourier::FftOrder(config.sample_rate_hz * kWindowSizeMs / 1000))),
       window_size_(1 << RealFourier::FftOrder(freqs_)),
-      chunk_length_(sample_rate_hz * kChunkSizeMs / 1000),
-      bank_size_(GetBankSize(sample_rate_hz, erb_resolution)),
-      sample_rate_hz_(sample_rate_hz),
-      erb_resolution_(erb_resolution),
-      channels_(channels),
-      analysis_rate_(analysis_rate),
-      variance_rate_(variance_rate),
+      chunk_length_(config.sample_rate_hz * kChunkSizeMs / 1000),
+      bank_size_(GetBankSize(config.sample_rate_hz, kErbResolution)),
+      sample_rate_hz_(config.sample_rate_hz),
+      erb_resolution_(kErbResolution),
+      channels_(config.channels),
+      analysis_rate_(config.analysis_rate),
+      capture_vad_thresh_(config.capture_vad_thresh),
+      render_vad_thresh_(config.render_vad_thresh),
       clear_variance_(freqs_,
-                      static_cast<VarianceType>(cv_type),
-                      cv_win,
-                      cv_alpha),
-      noise_variance_(freqs_, VarianceType::kStepInfinite, 475, 0.01f),
+                      config.var_type,
+                      config.var_window_size,
+                      config.var_decay_rate),
+      noise_variance_(freqs_,
+                      config.var_type,
+                      config.var_window_size,
+                      config.var_decay_rate),
       filtered_clear_var_(new float[bank_size_]),
       filtered_noise_var_(new float[bank_size_]),
       filter_bank_(bank_size_),
       center_freqs_(new float[bank_size_]),
       rho_(new float[bank_size_]),
       gains_eq_(new float[bank_size_]),
-      gain_applier_(freqs_, gain_limit),
+      gain_applier_(freqs_, config.gain_change_limit),
       temp_out_buffer_(nullptr),
-      input_audio_(new float* [channels]),
       kbd_window_(new float[window_size_]),
       render_callback_(this, AudioSource::kRenderStream),
       capture_callback_(this, AudioSource::kCaptureStream),
       block_count_(0),
       analysis_step_(0),
       vad_high_(WebRtcVad_Create()),
       vad_low_(WebRtcVad_Create()),
       vad_tmp_buffer_(new int16_t[chunk_length_]) {
-  DCHECK_LE(kConfigRho, 1.0f);
+  DCHECK_LE(config.rho, 1.0f);
 
   CreateErbBank();
 
   WebRtcVad_Init(vad_high_);
   WebRtcVad_set_mode(vad_high_, 0);  // High likelihood of speech.
   WebRtcVad_Init(vad_low_);
   WebRtcVad_set_mode(vad_low_, 3);  // Low likelihood of speech.
 
   temp_out_buffer_ = static_cast<float**>(
       malloc(sizeof(*temp_out_buffer_) * channels_ +
              sizeof(**temp_out_buffer_) * chunk_length_ * channels_));
   for (int i = 0; i < channels_; ++i) {
     temp_out_buffer_[i] =
         reinterpret_cast<float*>(temp_out_buffer_ + channels_) +
         chunk_length_ * i;
   }
 
   // Assumes all rho equal.
   for (int i = 0; i < bank_size_; ++i) {
-    rho_[i] = kConfigRho * kConfigRho;
+    rho_[i] = config.rho * config.rho;
   }
 
   float freqs_khz = kClipFreq / 1000.0f;
   int erb_index = static_cast<int>(ceilf(
       11.17f * logf((freqs_khz + 0.312f) / (freqs_khz + 14.6575f)) + 43.0f));
-  start_freq_ = max(1, erb_index * kErbResolution);
+  start_freq_ = max(1, erb_index * erb_resolution_);
 
   WindowGenerator::KaiserBesselDerived(kKbdAlpha, window_size_,
                                        kbd_window_.get());
   render_mangler_.reset(new LappedTransform(
       channels_, channels_, chunk_length_, kbd_window_.get(), window_size_,
       window_size_ / 2, &render_callback_));
   capture_mangler_.reset(new LappedTransform(
       channels_, channels_, chunk_length_, kbd_window_.get(), window_size_,
       window_size_ / 2, &capture_callback_));
 }
 
 IntelligibilityEnhancer::~IntelligibilityEnhancer() {
   WebRtcVad_Free(vad_low_);
   WebRtcVad_Free(vad_high_);
   free(temp_out_buffer_);
 }
 
 void IntelligibilityEnhancer::ProcessRenderAudio(float* const* audio) {
+  ProcessRenderAudio(audio, kVoiceDetected);
+}
+
+void IntelligibilityEnhancer::ProcessRenderAudio(float* const* audio,

[turaj, 2015/07/14 18:28:51]

I did not comprehend the logic here.

|voice_probability| specifies if the current chunk is active (see below for more
explanation on voice probability). Therefore, it has to be used as an indicator
whether the current chunk should be considered in variance computation or not.
If we have |voice_probability| we should not need WebRTC VAD decision. 

Furthermore, why a call to |render_mangler_| is based on |voice_probability|? I
don't think we should stop applying enhancement gains as soon as a chunk has low
activity probability. I think we should keep on applying enhancer, but low
activity chunks are discarded from variance computation.

We need yet another threshold which specifies if enhancement should be applied
at all. So I think the flow of the program is;
  1) Update variance of to-be-rendered signal on active chunks.
  2) Update variance of captured signal on in-active chunks.
  3) Make a decision based on the current state of enhancer (on or off) and the
     current signal-to-noise ratio whether enhancer should be on or off.

Obviously steps 1) and 2) are done through different API calls.

I like the idea of having two ProcessRenderAudio(), but I thought the caller is
supposed to call ProcessRenderAudio(float* const* audio) if they do not have any
information regarding the activity the given audio chunk, then we call WebRTC
VAD to get activity flag. On the other hand, if the caller has info regarding
the activity, they call ProcessRenderAudio(float* const* audio, float
voice_probability), then the given voice_probability is compared with a
threshold to decide if the given frame should be included in variance
computation.

Same thing for ProcessCaptureAudio()

[ekm, 2015/07/17 19:59:38]

Done. You're right, the logic was off and not fully implemented. I've updated
everything to meet this spec and use the new VAD. It also uses smoothing when
deactivating. There may be a more intuitive logic than what I have now. See
UpdateActivity().

+                                                 float voice_probability) {
   for (int i = 0; i < chunk_length_; ++i) {
     vad_tmp_buffer_[i] = (int16_t)audio[0][i];

[turaj, 2015/07/14 18:28:51]

You better check with APM guys, but I suppose you get audio in range -1 to 1,
therefore, a cast to int16_t is not a good idea. 

[aluebs-webrtc, 2015/07/15 01:02:04]

No, you get audio in the int16_t range. But for this there are beautiful tools
under audio_util. Also, if you need the audio in int16_t and float, you can
receive a IFChannelBuffer and let it take care of this? Not sure if it makes it
easier or not.

[ekm, 2015/07/17 19:59:38]

Done. audio_util is great!

   }
   has_voice_low_ = WebRtcVad_Process(vad_low_, sample_rate_hz_,
                                      vad_tmp_buffer_.get(), chunk_length_) == 1;
 
   // Process and enhance chunk of |audio|
-  render_mangler_->ProcessChunk(audio, temp_out_buffer_);
+  if (voice_probability >= render_vad_thresh_) {
+    render_mangler_->ProcessChunk(audio, temp_out_buffer_);
+  }
 
   for (int i = 0; i < channels_; ++i) {
     memcpy(audio[i], temp_out_buffer_[i],
            chunk_length_ * sizeof(**temp_out_buffer_));
   }
 }
 
 void IntelligibilityEnhancer::ProcessCaptureAudio(float* const* audio) {
+  ProcessCaptureAudio(audio, kNoiseDetected);
+}
+
+void IntelligibilityEnhancer::ProcessCaptureAudio(float* const* audio,
+                                                  float voice_probability) {
   for (int i = 0; i < chunk_length_; ++i) {
     vad_tmp_buffer_[i] = (int16_t)audio[0][i];
   }
   // TODO(bercic): The VAD was always detecting voice in the noise stream,
   // no matter what the aggressiveness, so it was temporarily disabled here.
 
   #if 0
     if (WebRtcVad_Process(vad_high_, sample_rate_hz_, vad_tmp_buffer_.get(),
       chunk_length_) == 1) {
       printf("capture HAS speech\n");
       return;
     }
     printf("capture NO speech\n");
   #endif
 
-  capture_mangler_->ProcessChunk(audio, temp_out_buffer_);
+  if (voice_probability <= capture_vad_thresh_) {
+    capture_mangler_->ProcessChunk(audio, temp_out_buffer_);
+  }
 }
 
 void IntelligibilityEnhancer::DispatchAudio(
     IntelligibilityEnhancer::AudioSource source,
     const complex<float>* in_block,
     complex<float>* out_block) {
   switch (source) {
     case kRenderStream:
       ProcessClearBlock(in_block, out_block);
       break;
@@ skipping 15 matching lines @@
   // TODO(ekmeyerson): Change to only enhance if necessary,
   // based on experiments with different cutoffs.
   if (has_voice_low_ || true) {
     clear_variance_.Step(in_block, false);
     const float power_target = std::accumulate(
         clear_variance_.variance(), clear_variance_.variance() + freqs_, 0.0f);
 
     if (block_count_ % analysis_rate_ == analysis_rate_ - 1) {
       AnalyzeClearBlock(power_target);
       ++analysis_step_;
-      if (analysis_step_ == variance_rate_) {
-        analysis_step_ = 0;
-        clear_variance_.Clear();
-        noise_variance_.Clear();
-      }
     }
     ++block_count_;
   }
 
   /* efidata(n,:) = sqrt(b(n)) * fidata(n,:) */
   gain_applier_.Apply(in_block, out_block);
 }
 
 void IntelligibilityEnhancer::AnalyzeClearBlock(float power_target) {
   FilterVariance(clear_variance_.variance(), filtered_clear_var_.get());
@@ skipping 164 matching lines @@
                                           int length) {
   float ret = 0.0f;
 
   for (int i = 0; i < length; ++i) {
     ret = fmaf(a[i], b[i], ret);
   }
   return ret;
 }
 
 }  // namespace webrtc