Integrate Intelligibility with APM

webrtc/modules/audio_processing/include/audio_processing.h

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 98 matching lines @@
   Beamforming()
       : enabled(false),
         array_geometry() {}
   Beamforming(bool enabled, const std::vector<Point>& array_geometry)
       : enabled(enabled),
         array_geometry(array_geometry) {}
   const bool enabled;
   const std::vector<Point> array_geometry;
 };
 
+// Use to enable intelligibility enhancer in audio processing. Must be provided
+// though the constructor. It will have no impact if used with
+// AudioProcessing::SetExtraOptions().
+//
+// Note: If enabled and the reverse stream has more than one output channel,
+// the reverse stream will become an upmixed mono signal.
+struct Intelligibility {
+  Intelligibility() : enabled(false) {}
+  explicit Intelligibility(bool enabled) : enabled(enabled) {}
+  bool enabled;
+};
+
 static const int kAudioProcMaxNativeSampleRateHz = 32000;
 
 // The Audio Processing Module (APM) provides a collection of voice processing
 // components designed for real-time communications software.
 //
 // APM operates on two audio streams on a frame-by-frame basis. Frames of the
 // primary stream, on which all processing is applied, are passed to
 // |ProcessStream()|. Frames of the reverse direction stream, which are used for
 // analysis by some components, are passed to |AnalyzeReverseStream()|. On the
 // client-side, this will typically be the near-end (capture) and far-end
@@ skipping 197 matching lines @@
   // reverse stream forms the echo reference signal. It is recommended, but not
   // necessary, to provide if gain control is enabled. On the server-side this
   // typically will not be used. If you're not sure what to pass in here,
   // chances are you don't need to use it.
   //
   // The |sample_rate_hz_|, |num_channels_|, and |samples_per_channel_|
   // members of |frame| must be valid. |sample_rate_hz_| must correspond to
   // |input_sample_rate_hz()|
   //
   // TODO(ajm): add const to input; requires an implementation fix.
+  // DEPRECATED: Use |ProcessReverseStream| instead.
+  // TODO(ekm): Remove once all users have updated to |ProcessReverseStream|.
   virtual int AnalyzeReverseStream(AudioFrame* frame) = 0;
 
+  // Same as |AnalyzeReverseStream|, but may modify |frame| if intelligibility
+  // is enabled.
+  virtual int ProcessReverseStream(AudioFrame* frame) = 0;
+
   // Accepts deinterleaved float audio with the range [-1, 1]. Each element
   // of |data| points to a channel buffer, arranged according to |layout|.
-  //
   // TODO(mgraczyk): Remove once clients are updated to use the new interface.
   virtual int AnalyzeReverseStream(const float* const* data,
                                    int samples_per_channel,
-                                   int sample_rate_hz,
+                                   int rev_sample_rate_hz,
                                    ChannelLayout layout) = 0;
 
   // Accepts deinterleaved float audio with the range [-1, 1]. Each element of
   // |data| points to a channel buffer, arranged according to |reverse_config|.
-  virtual int AnalyzeReverseStream(const float* const* data,
-                                   const StreamConfig& reverse_config) = 0;
+  virtual int ProcessReverseStream(const float* const* src,
+                                   const StreamConfig& reverse_input_config,
+                                   const StreamConfig& reverse_output_config,
+                                   float* const* dest) = 0;
 
   // This must be called if and only if echo processing is enabled.
   //
   // Sets the |delay| in ms between AnalyzeReverseStream() receiving a far-end
   // frame and ProcessStream() receiving a near-end frame containing the
   // corresponding echo. On the client-side this can be expressed as
   //   delay = (t_render - t_analyze) + (t_process - t_capture)
   // where,
   //   - t_analyze is the time a frame is passed to AnalyzeReverseStream() and
   //     t_render is the time the first sample of the same frame is rendered by
@@ skipping 124 matching lines @@
   void set_has_keyboard(bool value) { has_keyboard_ = value; }
 
   int sample_rate_hz() const { return sample_rate_hz_; }
 
   // The number of channels in the stream, not including the keyboard channel if
   // present.
   int num_channels() const { return num_channels_; }
 
   bool has_keyboard() const { return has_keyboard_; }
   int num_frames() const { return num_frames_; }
+  int num_samples() const { return num_channels_ * num_frames_; }
 
   bool operator==(const StreamConfig& other) const {
     return sample_rate_hz_ == other.sample_rate_hz_ &&
            num_channels_ == other.num_channels_ &&
            has_keyboard_ == other.has_keyboard_;
   }
 
   bool operator!=(const StreamConfig& other) const { return !(*this == other); }
 
  private:
   static int calculate_frames(int sample_rate_hz) {
     return AudioProcessing::kChunkSizeMs * sample_rate_hz / 1000;
   }
 
   int sample_rate_hz_;
   int num_channels_;
   bool has_keyboard_;
   int num_frames_;
 };
 
 class ProcessingConfig {
  public:
   enum StreamName {
     kInputStream,
     kOutputStream,
-    kReverseStream,
+    kReverseInputStream,
+    kReverseOutputStream,
     kNumStreamNames,
   };
 
   const StreamConfig& input_stream() const {
     return streams[StreamName::kInputStream];
   }
   const StreamConfig& output_stream() const {
     return streams[StreamName::kOutputStream];
   }
-  const StreamConfig& reverse_stream() const {
-    return streams[StreamName::kReverseStream];
+  const StreamConfig& reverse_input_stream() const {
+    return streams[StreamName::kReverseInputStream];
+  }
+  const StreamConfig& reverse_output_stream() const {
+    return streams[StreamName::kReverseOutputStream];
   }
 
   StreamConfig& input_stream() { return streams[StreamName::kInputStream]; }
   StreamConfig& output_stream() { return streams[StreamName::kOutputStream]; }
-  StreamConfig& reverse_stream() { return streams[StreamName::kReverseStream]; }
+  StreamConfig& reverse_input_stream() {
+    return streams[StreamName::kReverseInputStream];
+  }
+  StreamConfig& reverse_output_stream() {
+    return streams[StreamName::kReverseOutputStream];
+  }
 
   bool operator==(const ProcessingConfig& other) const {
     for (int i = 0; i < StreamName::kNumStreamNames; ++i) {
       if (this->streams[i] != other.streams[i]) {
         return false;
       }
     }
     return true;
   }
 
@@ skipping 359 matching lines @@
   // This does not impact the size of frames passed to |ProcessStream()|.
   virtual int set_frame_size_ms(int size) = 0;
   virtual int frame_size_ms() const = 0;
 
  protected:
   virtual ~VoiceDetection() {}
 };
 }  // namespace webrtc
 
 #endif  // WEBRTC_MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_