Integrate Intelligibility with APM

webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 15 matching lines @@
 typedef struct WebRtcVadInst VadInst;
 
 namespace webrtc {
 
 // Speech intelligibility enhancement module. Reads render and capture
 // audio streams and modifies the render stream with a set of gains per
 // frequency bin to enhance speech against the noise background.
 // Note: assumes speech and noise streams are already separated.
 class IntelligibilityEnhancer {
  public:
-  // Construct a new instance with the given filter bank resolution,
-  // sampling rate, number of channels and analysis rates.
-  // |analysis_rate| sets the number of input blocks (containing speech!)
-  // to elapse before a new gain computation is made. |variance_rate| specifies
-  // the number of gain recomputations after which the variances are reset.
-  // |cv_*| are parameters for the VarianceArray constructor for the
-  // clear speech stream.
-  // TODO(bercic): the |cv_*|, |*_rate| and |gain_limit| parameters should
-  // probably go away once fine tuning is done. They override the internal
-  // constants in the class (kGainChangeLimit, kAnalyzeRate, kVarianceRate).
-  IntelligibilityEnhancer(int erb_resolution,
-                          int sample_rate_hz,
-                          int channels,
-                          int cv_type,
-                          float cv_alpha,
-                          int cv_win,
-                          int analysis_rate,
-                          int variance_rate,
-                          float gain_limit);
+  struct Config {
+    // |var_*| are parameters for the VarianceArray constructor for the
+    // clear speech stream.
+    // TODO(bercic): the |var_*|, |*_rate| and |gain_limit| parameters should
+    // probably go away once fine tuning is done. They override the internal
+    // constants in the class (kGainChangeLimit, kAnalyzeRate, kVarianceRate).
+    Config()
+        : sample_rate_hz(16000),
+          channels(1),
+          var_type(intelligibility::VarianceArray::kStepDecaying),
+          var_decay_rate(0.9f),
+          var_window_size(10),
+          analysis_rate(800),
+          gain_change_limit(0.1f),
+          rho(0.02f),
+          capture_vad_thresh(1.f),
+          render_vad_thresh(0.f) {}
+    int sample_rate_hz;
+    int channels;
+    intelligibility::VarianceArray::StepType var_type;
+    float var_decay_rate;
+    int var_window_size;
+    int analysis_rate;
+    float gain_change_limit;
+    float rho;
+    float capture_vad_thresh;
+    float render_vad_thresh;
+  };
+
+  explicit IntelligibilityEnhancer(const Config& config);
+  IntelligibilityEnhancer();  // Initialize with default config.
+
   ~IntelligibilityEnhancer();
 
-  // Reads and processes chunk of noise stream in time domain.
-  void ProcessCaptureAudio(float* const* audio);
+  // Reads and processes chunk of noise stream in time domain. Only updates
+  // noise estimate when |voice_probability| below a threshold.
+  void ProcessCaptureAudio(float* const* audio, const float voice_probability);

[aluebs-webrtc, 2015/07/15 01:02:04]

Does it actually processes the capture audio or does it only analyze it? Maybe
we need a naming change.

[aluebs-webrtc, 2015/07/15 01:02:05]

In all of these methods you assume the sample rate and number of channels did
not change since the constructor. Maybe you want to have them as parameters and
assert on that?

[ekm, 2015/07/17 19:59:38]

Agreed. Similarly, in APM reverted AnalyzeReverseStream to const and added
ProcessReverseStream.

[ekm, 2015/07/17 19:59:38]

Done.

+  void ProcessCaptureAudio(float* const* audio);  // Assumes noise.

[aluebs-webrtc, 2015/07/15 01:02:04]

Do we want to surface both interfaces to the user?

[ekm, 2015/07/17 19:59:38]

I think it's nice to give the user the option of using their own voice
probabilities if they want (especially if they have really good ones). May be
useful if there is some very expensive VAD in the future that we only want to
run once for all APM. 

[aluebs-webrtc, 2015/07/20 19:33:42]

Agreed.

 
   // Reads chunk of speech in time domain and updates with modified signal.
-  void ProcessRenderAudio(float* const* audio);
+  // Only updates speech estimate when |voice_probability| above a threshold.
+  void ProcessRenderAudio(float* const* audio, const float voice_probability);
+  void ProcessRenderAudio(float* const* audio);  // Assumes speech.

[aluebs-webrtc, 2015/07/15 01:02:04]

Do we want to surface both interfaces to the user?

[ekm, 2015/07/17 19:59:38]

See above.

 
  private:
   enum AudioSource {
     kRenderStream = 0,  // Clear speech stream.
     kCaptureStream,  // Noise stream.
   };
 
   // Provides access point to the frequency domain.
   class TransformCallback : public LappedTransform::Callback {
    public:
@@ skipping 56 matching lines @@
   static float DotProduct(const float* a, const float* b, int length);
 
   const int freqs_;         // Num frequencies in frequency domain.
   const int window_size_;   // Window size in samples; also the block size.
   const int chunk_length_;  // Chunk size in samples.
   const int bank_size_;     // Num ERB filters.
   const int sample_rate_hz_;
   const int erb_resolution_;
   const int channels_;       // Num channels.
   const int analysis_rate_;  // Num blocks before gains recalculated.
-  const int variance_rate_;  // Num recalculations before history is cleared.
+  const float capture_vad_thresh_;  // Threshold for updating noise estimate.
+  const float render_vad_thresh_;   // Threshold for updating speech estimate.
 
   intelligibility::VarianceArray clear_variance_;
   intelligibility::VarianceArray noise_variance_;
   rtc::scoped_ptr<float[]> filtered_clear_var_;
   rtc::scoped_ptr<float[]> filtered_noise_var_;
   std::vector<std::vector<float>> filter_bank_;
   rtc::scoped_ptr<float[]> center_freqs_;
   int start_freq_;
   rtc::scoped_ptr<float[]> rho_;  // Production and interpretation SNR.
                                   // for each ERB band.
   rtc::scoped_ptr<float[]> gains_eq_;  // Pre-filter modified gains.
   intelligibility::GainApplier gain_applier_;
 
   // Destination buffer used to reassemble blocked chunks before overwriting
   // the original input array with modifications.
   // TODO(ekmeyerson): Switch to using ChannelBuffer.
   float** temp_out_buffer_;
 
-  rtc::scoped_ptr<float* []> input_audio_;
   rtc::scoped_ptr<float[]> kbd_window_;
   TransformCallback render_callback_;
   TransformCallback capture_callback_;
   rtc::scoped_ptr<LappedTransform> render_mangler_;
   rtc::scoped_ptr<LappedTransform> capture_mangler_;
   int block_count_;
   int analysis_step_;
 
   // TODO(bercic): Quick stopgap measure for voice detection in the clear
   // and noise streams.
   // Note: VAD currently does not affect anything in IntelligibilityEnhancer.
   VadInst* vad_high_;
   VadInst* vad_low_;
   rtc::scoped_ptr<int16_t[]> vad_tmp_buffer_;
   bool has_voice_low_;  // Whether voice detected in speech stream.
 };
 
 }  // namespace webrtc
 
 #endif  // WEBRTC_MODULES_AUDIO_PROCESSING_INTELLIGIBILITY_INTELLIGIBILITY_ENHANCER_H_