Compensate for the IntelligibilityEnhancer processing delay in high bands

webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 50 matching lines @@
     RTC_DCHECK_GT(filter_bank[i].size(), 0u);
     result[i] = kPowerNormalizationFactor *
                 DotProduct(filter_bank[i].data(), pow, filter_bank[i].size());
   }
 }
 
 }  // namespace
 
 IntelligibilityEnhancer::IntelligibilityEnhancer(int sample_rate_hz,
                                                  size_t num_render_channels,
+                                                 size_t num_bands,
                                                  size_t num_noise_bins)
     : freqs_(RealFourier::ComplexLength(
           RealFourier::FftOrder(sample_rate_hz * kWindowSizeMs / 1000))),
       num_noise_bins_(num_noise_bins),
       chunk_length_(static_cast<size_t>(sample_rate_hz * kChunkSizeMs / 1000)),
       bank_size_(GetBankSize(sample_rate_hz, kErbResolution)),
       sample_rate_hz_(sample_rate_hz),
       num_render_channels_(num_render_channels),
       clear_power_estimator_(freqs_, kDecayRate),
       noise_power_estimator_(num_noise_bins, kDecayRate),
@@ skipping 22 matching lines @@
             43.f));
   start_freq_ = std::max(static_cast<size_t>(1), erb_index * kErbResolution);
 
   size_t window_size = static_cast<size_t>(1) << RealFourier::FftOrder(freqs_);
   std::vector<float> kbd_window(window_size);
   WindowGenerator::KaiserBesselDerived(kKbdAlpha, window_size,
                                        kbd_window.data());
   render_mangler_.reset(new LappedTransform(
       num_render_channels_, num_render_channels_, chunk_length_,
       kbd_window.data(), window_size, window_size / 2, this));
+
+  const size_t initial_delay = render_mangler_->initial_delay();
+  for (size_t i = 0u; i < num_bands - 1; ++i) {
+    high_bands_buffers_.push_back(std::unique_ptr<intelligibility::DelayBuffer>(
+        new intelligibility::DelayBuffer(initial_delay, num_render_channels_)));
+  }
 }
 
 IntelligibilityEnhancer::~IntelligibilityEnhancer() {
-  // Don't rely on this log, since the destructor isn't called when the app/tab
-  // is killed.
-  LOG(LS_INFO) << "Intelligibility Enhancer was active for "
-               << static_cast<float>(num_active_chunks_) / num_chunks_
-               << "% of the call.";
+  // Don't rely on this log, since the destructor isn't called when the
+  // app/tab is killed.
+  if (num_chunks_ > 0) {
+    LOG(LS_INFO) << "Intelligibility Enhancer was active for "
+                 << 100.f * static_cast<float>(num_active_chunks_) / num_chunks_
+                 << "% of the call.";
+  } else {
+    LOG(LS_INFO) << "Intelligibility Enhancer processed no chunk.";
+  }
 }
 
 void IntelligibilityEnhancer::SetCaptureNoiseEstimate(
     std::vector<float> noise, float gain) {
   RTC_DCHECK_EQ(noise.size(), num_noise_bins_);
   for (auto& bin : noise) {
     bin *= gain;
   }
   // Disregarding return value since buffer overflow is acceptable, because it
   // is not critical to get each noise estimate.
   if (noise_estimation_queue_.Insert(&noise)) {
   };
 }
 
-void IntelligibilityEnhancer::ProcessRenderAudio(float* const* audio,
-                                                 int sample_rate_hz,
-                                                 size_t num_channels) {
-  RTC_CHECK_EQ(sample_rate_hz_, sample_rate_hz);
-  RTC_CHECK_EQ(num_render_channels_, num_channels);
+void IntelligibilityEnhancer::ProcessRenderAudio(AudioBuffer* audio) {
+  RTC_DCHECK_EQ(num_render_channels_, audio->num_channels());
   while (noise_estimation_queue_.Remove(&noise_estimation_buffer_)) {
     noise_power_estimator_.Step(noise_estimation_buffer_.data());
   }
-  is_speech_ = IsSpeech(audio[0]);
-  render_mangler_->ProcessChunk(audio, audio);
+  float* const* low_band = audio->split_channels_f(kBand0To8kHz);
+  is_speech_ = IsSpeech(low_band[0]);
+  render_mangler_->ProcessChunk(low_band, low_band);
+  DelayHighBands(audio);
 }
 
 void IntelligibilityEnhancer::ProcessAudioBlock(
     const std::complex<float>* const* in_block,
     size_t in_channels,
     size_t frames,
     size_t /* out_channels */,
     std::complex<float>* const* out_block) {
   RTC_DCHECK_EQ(freqs_, frames);
   if (is_speech_) {
@@ skipping 207 matching lines @@
   FloatToS16(audio, chunk_length_, audio_s16_.data());
   vad_.ProcessChunk(audio_s16_.data(), chunk_length_, sample_rate_hz_);
   if (vad_.last_voice_probability() > kVoiceProbabilityThreshold) {
     chunks_since_voice_ = 0;
   } else if (chunks_since_voice_ < kSpeechOffsetDelay) {
     ++chunks_since_voice_;
   }
   return chunks_since_voice_ < kSpeechOffsetDelay;
 }
 
+void IntelligibilityEnhancer::DelayHighBands(AudioBuffer* audio) {
+  RTC_DCHECK_EQ(audio->num_bands(), high_bands_buffers_.size() + 1u);
+  for (size_t i = 0u; i < high_bands_buffers_.size(); ++i) {
+    Band band = static_cast<Band>(i + 1);
+    high_bands_buffers_[i]->Delay(audio->split_channels_f(band), chunk_length_);
+  }
+}
+
 }  // namespace webrtc