Compensate for the IntelligibilityEnhancer processing delay in high bands

webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 50 matching lines @@
     RTC_DCHECK_GT(filter_bank[i].size(), 0u);
     result[i] = kPowerNormalizationFactor *
                 DotProduct(filter_bank[i].data(), pow, filter_bank[i].size());
   }
 }
 
 }  // namespace
 
 IntelligibilityEnhancer::IntelligibilityEnhancer(int sample_rate_hz,
                                                  size_t num_render_channels,
+                                                 size_t num_bands,
                                                  size_t num_noise_bins)
     : freqs_(RealFourier::ComplexLength(
           RealFourier::FftOrder(sample_rate_hz * kWindowSizeMs / 1000))),
       num_noise_bins_(num_noise_bins),
       chunk_length_(static_cast<size_t>(sample_rate_hz * kChunkSizeMs / 1000)),
       bank_size_(GetBankSize(sample_rate_hz, kErbResolution)),
       sample_rate_hz_(sample_rate_hz),
       num_render_channels_(num_render_channels),
       clear_power_estimator_(freqs_, kDecayRate),
       noise_power_estimator_(num_noise_bins, kDecayRate),
@@ skipping 22 matching lines @@
             43.f));
   start_freq_ = std::max(static_cast<size_t>(1), erb_index * kErbResolution);
 
   size_t window_size = static_cast<size_t>(1) << RealFourier::FftOrder(freqs_);
   std::vector<float> kbd_window(window_size);
   WindowGenerator::KaiserBesselDerived(kKbdAlpha, window_size,
                                        kbd_window.data());
   render_mangler_.reset(new LappedTransform(
       num_render_channels_, num_render_channels_, chunk_length_,
       kbd_window.data(), window_size, window_size / 2, this));
+
+  const size_t initial_delay = render_mangler_->initial_delay();

[peah-webrtc, 2016/09/13 13:29:59]

Have you verified that this is indeed the delay introduced by the IE processing?

[aluebs-webrtc, 2016/09/14 00:35:54]

Yes.

+  for (size_t i = 0u; i < num_bands - 1; ++i) {
+    high_bands_buffers_.push_back(
+        std::unique_ptr<AudioRingBuffer>(new AudioRingBuffer(
+            num_render_channels_, chunk_length_ + initial_delay)));
+    high_bands_buffers_[i]->MoveReadPositionBackward(initial_delay);
+  }
 }
 
 IntelligibilityEnhancer::~IntelligibilityEnhancer() {
-  // Don't rely on this log, since the destructor isn't called when the app/tab
-  // is killed.
-  LOG(LS_INFO) << "Intelligibility Enhancer was active for "
-               << static_cast<float>(num_active_chunks_) / num_chunks_
-               << "% of the call.";
+  if (num_chunks_ > 0) {
+    // Don't rely on this log, since the destructor isn't called when the
+    // app/tab is killed.
+    LOG(LS_INFO) << "Intelligibility Enhancer was active for "

[peah-webrtc, 2016/09/13 13:29:59]

I think it would be good to have a log line for the case when num_chunks_ == 0
as well. When analyzing logs to find errors, missing log-lines is a source of
confusion.
WDYT?

[aluebs-webrtc, 2016/09/14 00:35:54]

Done.

+                 << 100.f * static_cast<float>(num_active_chunks_) / num_chunks_
+                 << "% of the call.";
+  }
 }
 
 void IntelligibilityEnhancer::SetCaptureNoiseEstimate(
     std::vector<float> noise, float gain) {
   RTC_DCHECK_EQ(noise.size(), num_noise_bins_);
   for (auto& bin : noise) {
     bin *= gain;
   }
   // Disregarding return value since buffer overflow is acceptable, because it
   // is not critical to get each noise estimate.
   if (noise_estimation_queue_.Insert(&noise)) {
   };
 }
 
-void IntelligibilityEnhancer::ProcessRenderAudio(float* const* audio,
-                                                 int sample_rate_hz,
-                                                 size_t num_channels) {
-  RTC_CHECK_EQ(sample_rate_hz_, sample_rate_hz);
-  RTC_CHECK_EQ(num_render_channels_, num_channels);
+void IntelligibilityEnhancer::ProcessRenderAudio(AudioBuffer* audio) {
+  RTC_DCHECK_EQ(num_render_channels_, audio->num_channels());
   while (noise_estimation_queue_.Remove(&noise_estimation_buffer_)) {
     noise_power_estimator_.Step(noise_estimation_buffer_.data());
   }
-  is_speech_ = IsSpeech(audio[0]);
-  render_mangler_->ProcessChunk(audio, audio);
+  float* const* low_band = audio->split_channels_f(kBand0To8kHz);
+  is_speech_ = IsSpeech(low_band[0]);
+  render_mangler_->ProcessChunk(low_band, low_band);
+  DelayHighBands(audio);
 }
 
 void IntelligibilityEnhancer::ProcessAudioBlock(
     const std::complex<float>* const* in_block,
     size_t in_channels,
     size_t frames,
     size_t /* out_channels */,
     std::complex<float>* const* out_block) {
   RTC_DCHECK_EQ(freqs_, frames);
   if (is_speech_) {
     clear_power_estimator_.Step(in_block[0]);
   }
-  SnrBasedEffectActivation();
-  ++num_chunks_;
   if (is_active_) {
     ++num_active_chunks_;
     if (num_chunks_ % kGainUpdatePeriod == 0) {
       MapToErbBands(clear_power_estimator_.power().data(), render_filter_bank_,
                     filtered_clear_pow_.data());
       MapToErbBands(noise_power_estimator_.power().data(), capture_filter_bank_,
                     filtered_noise_pow_.data());
       SolveForGainsGivenLambda(kLambdaTop, start_freq_, gains_eq_.data());
       const float power_target = std::accumulate(
           filtered_clear_pow_.data(),
           filtered_clear_pow_.data() + bank_size_,
           0.f);
       const float power_top =
           DotProduct(gains_eq_.data(), filtered_clear_pow_.data(), bank_size_);
       SolveForGainsGivenLambda(kLambdaBot, start_freq_, gains_eq_.data());
       const float power_bot =
           DotProduct(gains_eq_.data(), filtered_clear_pow_.data(), bank_size_);
       if (power_target >= power_bot && power_target <= power_top) {
         SolveForLambda(power_target);
         UpdateErbGains();
       }  // Else experiencing power underflow, so do nothing.
     }
   }
+  SnrBasedEffectActivation();

[peah-webrtc, 2016/09/13 13:29:59]

I think it makes sense. But please comment in the CL description on why you
moved this, as it impacts the processing and is not at all mentioned in the
description of the CL.

[aluebs-webrtc, 2016/09/14 00:35:54]

Moved it back. This was left from when is_active_ was needed in
ProcessRenderAudio to decide what pointer to pass in.

+  ++num_chunks_;
   for (size_t i = 0; i < in_channels; ++i) {
     gain_applier_.Apply(in_block[i], out_block[i]);
   }
 }
 
 void IntelligibilityEnhancer::SnrBasedEffectActivation() {
   const float* clear_psd = clear_power_estimator_.power().data();
   const float* noise_psd = noise_power_estimator_.power().data();
   const float clear_power =
       std::accumulate(clear_psd, clear_psd + freqs_, 0.f);
@@ skipping 170 matching lines @@
   FloatToS16(audio, chunk_length_, audio_s16_.data());
   vad_.ProcessChunk(audio_s16_.data(), chunk_length_, sample_rate_hz_);
   if (vad_.last_voice_probability() > kVoiceProbabilityThreshold) {
     chunks_since_voice_ = 0;
   } else if (chunks_since_voice_ < kSpeechOffsetDelay) {
     ++chunks_since_voice_;
   }
   return chunks_since_voice_ < kSpeechOffsetDelay;
 }
 
+void IntelligibilityEnhancer::DelayHighBands(AudioBuffer* audio) {
+  RTC_DCHECK_EQ(audio->num_bands(), high_bands_buffers_.size() + 1u);
+  for (size_t i = 0u; i < high_bands_buffers_.size(); ++i) {
+    Band band = static_cast<Band>(i + 1);
+    high_bands_buffers_[i]->Write(audio->split_channels_const_f(band),
+                                  num_render_channels_, chunk_length_);
+    high_bands_buffers_[i]->Read(audio->split_channels_f(band),
+                                 num_render_channels_, chunk_length_);
+  }
+}
+
 }  // namespace webrtc