Integrate Intelligibility with APM

webrtc/modules/audio_processing/intelligibility/intelligibility_enhancer.h

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 //
 //  Specifies core class for intelligbility enhancement.
 //
 
 #ifndef WEBRTC_MODULES_AUDIO_PROCESSING_INTELLIGIBILITY_INTELLIGIBILITY_ENHANCER_H_
 #define WEBRTC_MODULES_AUDIO_PROCESSING_INTELLIGIBILITY_INTELLIGIBILITY_ENHANCER_H_
 
 #include <complex>
 #include <vector>
 
 #include "webrtc/base/scoped_ptr.h"
 #include "webrtc/common_audio/lapped_transform.h"
+#include "webrtc/modules/audio_processing/vad/voice_activity_detector.h"
 #include "webrtc/modules/audio_processing/intelligibility/intelligibility_utils.h"
 
-struct WebRtcVadInst;
-typedef struct WebRtcVadInst VadInst;
-
 namespace webrtc {
 
 // Speech intelligibility enhancement module. Reads render and capture
 // audio streams and modifies the render stream with a set of gains per
 // frequency bin to enhance speech against the noise background.
 // Note: assumes speech and noise streams are already separated.
 class IntelligibilityEnhancer {
  public:
-  // Construct a new instance with the given filter bank resolution,
-  // sampling rate, number of channels and analysis rates.
-  // |analysis_rate| sets the number of input blocks (containing speech!)
-  // to elapse before a new gain computation is made. |variance_rate| specifies
-  // the number of gain recomputations after which the variances are reset.
-  // |cv_*| are parameters for the VarianceArray constructor for the
-  // clear speech stream.
-  // TODO(bercic): the |cv_*|, |*_rate| and |gain_limit| parameters should
-  // probably go away once fine tuning is done. They override the internal
-  // constants in the class (kGainChangeLimit, kAnalyzeRate, kVarianceRate).
-  IntelligibilityEnhancer(int erb_resolution,
-                          int sample_rate_hz,
-                          int channels,
-                          int cv_type,
-                          float cv_alpha,
-                          int cv_win,
-                          int analysis_rate,
-                          int variance_rate,
-                          float gain_limit);
+  struct Config {
+    // |var_*| are parameters for the VarianceArray constructor for the
+    // clear speech stream.
+    // TODO(bercic): the |var_*|, |*_rate| and |gain_limit| parameters should
+    // probably go away once fine tuning is done. They override the internal
+    // constants in the class (kGainChangeLimit, kAnalyzeRate, kVarianceRate).
+    Config()
+        : sample_rate_hz(16000),
+          channels(1),
+          var_type(intelligibility::VarianceArray::kStepDecaying),
+          var_decay_rate(0.9f),
+          var_window_size(10),
+          analysis_rate(800),
+          gain_change_limit(0.1f),
+          rho(0.02f),
+          capture_vad_thresh(1.f),
+          render_vad_thresh(0.f),
+          activate_snr_thresh(0.f),
+          deactivate_snr_thresh(100000.f) {}
+    int sample_rate_hz;
+    int channels;
+    intelligibility::VarianceArray::StepType var_type;
+    float var_decay_rate;
+    int var_window_size;
+    int analysis_rate;
+    float gain_change_limit;
+    float rho;
+    float capture_vad_thresh;
+    float render_vad_thresh;
+    float activate_snr_thresh;
+    float deactivate_snr_thresh;
+  };
+
+  explicit IntelligibilityEnhancer(const Config& config);
+  IntelligibilityEnhancer();  // Initialize with default config.
+
   ~IntelligibilityEnhancer();
 
-  // Reads and processes chunk of noise stream in time domain.
-  void ProcessCaptureAudio(float* const* audio);
+  // Reads and processes chunk of noise stream in time domain. Only updates
+  // noise estimate when |voice_probability| below a threshold. Uses internal
+  // VAD when |voice_probability| not provided.
+  void AnalyzeCaptureAudio(float* const* audio,
+                           int sample_rate_hz,
+                           int num_channels,
+                           float voice_probability);
+  void AnalyzeCaptureAudio(float* const* audio,
+                           int sample_rate_hz,
+                           int num_channels);
 
   // Reads chunk of speech in time domain and updates with modified signal.
-  void ProcessRenderAudio(float* const* audio);
+  // Only updates speech estimate when |voice_probability| above a threshold.
+  // Uses internal VAD when |voice_probability| not provided.
+  void ProcessRenderAudio(float* const* audio,
+                          int sample_rate_hz,
+                          int num_channels,
+                          float voice_probability);
+  void ProcessRenderAudio(float* const* audio,
+                          int sample_rate_hz,
+                          int num_channels);
 
  private:
   enum AudioSource {
     kRenderStream = 0,  // Clear speech stream.
     kCaptureStream,  // Noise stream.
   };
 
   // Provides access point to the frequency domain.
   class TransformCallback : public LappedTransform::Callback {
    public:
@@ skipping 45 matching lines @@
   void CreateErbBank();
 
   // Analytically solves quadratic for optimal gains given |lambda|.
   // Negative gains are set to 0. Stores the results in |sols|.
   void SolveForGainsGivenLambda(float lambda, int start_freq, float* sols);
 
   // Computes variance across ERB filters from freq variance |var|.
   // Stores in |result|.
   void FilterVariance(const float* var, float* result);
 
+  // Returns ratio of total variance of clear to noise.
+  float SNR();
+
+  // Updates |active_| based on SNR.
+  void UpdateActivity();
+
   // Returns dot product of vectors specified by size |length| arrays |a|,|b|.
   static float DotProduct(const float* a, const float* b, int length);
 
   const int freqs_;         // Num frequencies in frequency domain.
   const int window_size_;   // Window size in samples; also the block size.
   const int chunk_length_;  // Chunk size in samples.
   const int bank_size_;     // Num ERB filters.
   const int sample_rate_hz_;
   const int erb_resolution_;
   const int channels_;       // Num channels.
   const int analysis_rate_;  // Num blocks before gains recalculated.
-  const int variance_rate_;  // Num recalculations before history is cleared.
+  const float capture_vad_thresh_;   // Threshold for updating noise estimate.
+  const float render_vad_thresh_;    // Threshold for updating speech estimate.
+  const float activate_snr_thresh_;  // Threshold for activating gain updates.
+  const float deactivate_snr_thresh_;  // Threshold for deactivating.
+
+  bool active_;        // Whether render gains are being updated.
+  bool deactivating_;  // True when we are smoothing enhancer off.
 
   intelligibility::VarianceArray clear_variance_;
   intelligibility::VarianceArray noise_variance_;
   rtc::scoped_ptr<float[]> filtered_clear_var_;
   rtc::scoped_ptr<float[]> filtered_noise_var_;
   std::vector<std::vector<float>> filter_bank_;
   rtc::scoped_ptr<float[]> center_freqs_;
   int start_freq_;
   rtc::scoped_ptr<float[]> rho_;  // Production and interpretation SNR.
                                   // for each ERB band.
   rtc::scoped_ptr<float[]> gains_eq_;  // Pre-filter modified gains.
   intelligibility::GainApplier gain_applier_;
 
   // Destination buffer used to reassemble blocked chunks before overwriting
   // the original input array with modifications.
   // TODO(ekmeyerson): Switch to using ChannelBuffer.
   float** temp_out_buffer_;
 
-  rtc::scoped_ptr<float* []> input_audio_;
   rtc::scoped_ptr<float[]> kbd_window_;
   TransformCallback render_callback_;
   TransformCallback capture_callback_;
   rtc::scoped_ptr<LappedTransform> render_mangler_;
   rtc::scoped_ptr<LappedTransform> capture_mangler_;
   int block_count_;
   int analysis_step_;
 
-  // TODO(bercic): Quick stopgap measure for voice detection in the clear
-  // and noise streams.
-  // Note: VAD currently does not affect anything in IntelligibilityEnhancer.
-  VadInst* vad_high_;
-  VadInst* vad_low_;
+  VoiceActivityDetector capture_vad_;
+  VoiceActivityDetector render_vad_;
+  float capture_voice_probability_;
+  float render_voice_probability_;
+  bool using_capture_vad_;
+  bool using_render_vad_;
   rtc::scoped_ptr<int16_t[]> vad_tmp_buffer_;
-  bool has_voice_low_;  // Whether voice detected in speech stream.
 };
 
 }  // namespace webrtc
 
 #endif  // WEBRTC_MODULES_AUDIO_PROCESSING_INTELLIGIBILITY_INTELLIGIBILITY_ENHANCER_H_