Convert IntelligibilityEnhancer to multi-threaded mode

webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_INTELLIGIBILITY_INTELLIGIBILITY_ENHANCER_H_
 #define WEBRTC_MODULES_AUDIO_PROCESSING_INTELLIGIBILITY_INTELLIGIBILITY_ENHANCER_H_
 
 #include <complex>
 #include <memory>
 #include <vector>
 
 #include "webrtc/common_audio/lapped_transform.h"
 #include "webrtc/common_audio/channel_buffer.h"
+#include "webrtc/common_audio/swap_queue.h"
 #include "webrtc/modules/audio_processing/intelligibility/intelligibility_utils.h"
+#include "webrtc/modules/audio_processing/processing_component.h"
 #include "webrtc/modules/audio_processing/vad/voice_activity_detector.h"
 
 namespace webrtc {
 
 // Speech intelligibility enhancement module. Reads render and capture
 // audio streams and modifies the render stream with a set of gains per
 // frequency bin to enhance speech against the noise background.
 // Details of the model and algorithm can be found in the original paper:
 // http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=6882788
 class IntelligibilityEnhancer : public LappedTransform::Callback {
  public:
-  IntelligibilityEnhancer(int sample_rate_hz, size_t num_render_channels);
+  IntelligibilityEnhancer(int sample_rate_hz,
+                          size_t num_render_channels,
+                          size_t num_noise_bins);
 
   // Sets the capture noise magnitude spectrum estimate.
   void SetCaptureNoiseEstimate(std::vector<float> noise);
 
   // Reads chunk of speech in time domain and updates with modified signal.
   void ProcessRenderAudio(float* const* audio,
                           int sample_rate_hz,
                           size_t num_channels);
   bool active() const;
 
@@ skipping 22 matching lines @@
   // Initializes ERB filterbank.
   std::vector<std::vector<float>> CreateErbBank(size_t num_freqs);
 
   // Analytically solves quadratic for optimal gains given |lambda|.
   // Negative gains are set to 0. Stores the results in |sols|.
   void SolveForGainsGivenLambda(float lambda, size_t start_freq, float* sols);
 
   // Returns true if the audio is speech.
   bool IsSpeech(const float* audio);
 
+  // TODO(aluebs): Decrease this once we properly handle hugely unbalanced
+  // reverse and forward call numbers.
+  static const size_t kMaxNumFramesToBuffer = 100;
+

[peah-webrtc, 2016/03/08 07:02:38]

The size of this buffer seems a bit high. The reason for the large sizes in the
buffers for the AECs is that data must not be dropped there.
But in this case that requirement is not that superimportant, right? (which is
reflected on the fact that the code that inserts data into the buffer
(correctly) does not take buffer overflows into account).

I would have expected a 10-50 ms of data to be sufficient for the buffer as what
is passed is estimates of the stationary part of the capture noise spectrum
which should not change much (as it is stationary) which means that dropped data
due to thread stalls should not at all be critical.

Wdyt?

[peah-webrtc, 2016/03/08 07:02:38]

Please rename as it is not frames that are buffered, but rather noise spectra,
right?

[aluebs-webrtc, 2016/03/08 10:53:00]

Done.

[aluebs-webrtc, 2016/03/08 10:53:00]

That makes a lot of sense. I just copied and pasted without much thought on the
actual value.

   const size_t freqs_;         // Num frequencies in frequency domain.
+  const size_t num_noise_bins_;
   const size_t chunk_length_;  // Chunk size in samples.
   const size_t bank_size_;     // Num ERB filters.
   const int sample_rate_hz_;
   const size_t num_render_channels_;
 
   intelligibility::PowerEstimator<std::complex<float>> clear_power_estimator_;
-  std::unique_ptr<intelligibility::PowerEstimator<float>>
-      noise_power_estimator_;
+  intelligibility::PowerEstimator<float> noise_power_estimator_;
   std::vector<float> filtered_clear_pow_;
   std::vector<float> filtered_noise_pow_;
   std::vector<float> center_freqs_;
   std::vector<std::vector<float>> capture_filter_bank_;
   std::vector<std::vector<float>> render_filter_bank_;
   size_t start_freq_;
 
   std::vector<float> gains_eq_;  // Pre-filter modified gains.
   intelligibility::GainApplier gain_applier_;
 
   std::unique_ptr<LappedTransform> render_mangler_;
 
   VoiceActivityDetector vad_;
   std::vector<int16_t> audio_s16_;
   size_t chunks_since_voice_;
   bool is_speech_;
+
+  std::vector<float> noise_estimation_buffer_;
+  rtc::scoped_ptr<SwapQueue<std::vector<float>, RenderQueueItemVerifier<float>>>
+      noise_estimation_queue_;
 };
 
 }  // namespace webrtc
 
 #endif  // WEBRTC_MODULES_AUDIO_PROCESSING_INTELLIGIBILITY_INTELLIGIBILITY_ENHANCER_H_