Allow more than 2 input channels in AudioProcessing.

webrtc/modules/audio_processing/audio_buffer.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/modules/audio_processing/audio_buffer.h"
 
 #include "webrtc/common_audio/include/audio_util.h"
 #include "webrtc/common_audio/resampler/push_sinc_resampler.h"
 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
 #include "webrtc/common_audio/channel_buffer.h"
 #include "webrtc/modules/audio_processing/common.h"
 
 namespace webrtc {
 namespace {
 
 const int kSamplesPer16kHzChannel = 160;
 const int kSamplesPer32kHzChannel = 320;
 const int kSamplesPer48kHzChannel = 480;
 
-bool HasKeyboardChannel(AudioProcessing::ChannelLayout layout) {
-  switch (layout) {
-    case AudioProcessing::kMono:
-    case AudioProcessing::kStereo:
-      return false;
-    case AudioProcessing::kMonoAndKeyboard:
-    case AudioProcessing::kStereoAndKeyboard:
-      return true;
+int KeyboardChannelIndex(const StreamConfig& stream_config) {
+  if (!stream_config.has_keyboard()) {
+    assert(false);
+    return -1;
   }
-  assert(false);
-  return false;
-}
 
-int KeyboardChannelIndex(AudioProcessing::ChannelLayout layout) {
-  switch (layout) {
-    case AudioProcessing::kMono:
-    case AudioProcessing::kStereo:
-      assert(false);
-      return -1;
-    case AudioProcessing::kMonoAndKeyboard:
+  switch (stream_config.num_channels()) {

[Andrew MacDonald, 2015/07/22 22:47:21]

I see that this is identical to the previous behavior, but why not make it
generic?

return stream_config.num_channels();

Also, does the num_channels count really not include the keyboard channel? That
seems counter-intuitive to me, but in any case should be documented in
StreamConfig.

[mgraczyk, 2015/07/23 00:16:54]

Changed to be generic.

I agree it makes sense for "num_channels" to match the number of channels passed
by the caller. However there I it is simpler if num_channels does not include
the keyboard channel for a few reasons.

First, everywhere we currently use num_channels() assumes that the channel count
does not include the keyboard.  I think that indicates that the more natural
usage is to have audio channel count and keyboard be distinct.

Second, clients will have to specify "has_keyboard" anyway when there is a
keyboard channel.  It seems strange for them to need to indicate information
about the keyboard channel in two places.

Third, am I right in assuming that the output and reverse streams will never
have a keyboard?  If only the input stream has one, maybe it's best to make that
the special case.

Lastly, what should AudioProcessing::num_input_channels,
AudioProcessing::num_output_channels, etc return? The number of channels
including the keyboard or excluding?  I believe that before this CL they
returned the number of channels excluding the keyboard, so it seems inconsistent
to ask clients to pass in one number yet return another from a method with the
same name.

What do you think?

[Andrew MacDonald, 2015/07/23 00:49:06]

That's right.
I agree with you. The last two arguments are particularly persuasive.

+    case 1:
       return 1;
-    case AudioProcessing::kStereoAndKeyboard:
+    case 2:
       return 2;
   }
   assert(false);
   return -1;
 }
 
-template <typename T>
-void StereoToMono(const T* left, const T* right, T* out,
-                  int num_frames) {
-  for (int i = 0; i < num_frames; ++i)
-    out[i] = (left[i] + right[i]) / 2;
-}
-
 int NumBandsFromSamplesPerChannel(int num_frames) {
   int num_bands = 1;
   if (num_frames == kSamplesPer32kHzChannel ||
       num_frames == kSamplesPer48kHzChannel) {
     num_bands = rtc::CheckedDivExact(num_frames,
                                      static_cast<int>(kSamplesPer16kHzChannel));
   }
   return num_bands;
 }
 
@@ skipping 13 matching lines @@
     num_bands_(NumBandsFromSamplesPerChannel(proc_num_frames_)),
     num_split_frames_(rtc::CheckedDivExact(proc_num_frames_, num_bands_)),
     mixed_low_pass_valid_(false),
     reference_copied_(false),
     activity_(AudioFrame::kVadUnknown),
     keyboard_data_(NULL),
     data_(new IFChannelBuffer(proc_num_frames_, num_proc_channels_)) {
   assert(input_num_frames_ > 0);
   assert(proc_num_frames_ > 0);
   assert(output_num_frames_ > 0);
-  assert(num_input_channels_ > 0 && num_input_channels_ <= 2);
+  assert(num_input_channels_ > 0);
   assert(num_proc_channels_ > 0 && num_proc_channels_ <= num_input_channels_);
 
   if (input_num_frames_ != proc_num_frames_ ||
       output_num_frames_ != proc_num_frames_) {
     // Create an intermediate buffer for resampling.
     process_buffer_.reset(new ChannelBuffer<float>(proc_num_frames_,
                                                    num_proc_channels_));
 
     if (input_num_frames_ != proc_num_frames_) {
       for (int i = 0; i < num_proc_channels_; ++i) {
@@ skipping 18 matching lines @@
                                           num_bands_));
     splitting_filter_.reset(new SplittingFilter(num_proc_channels_,
                                                 num_bands_,
                                                 proc_num_frames_));
   }
 }
 
 AudioBuffer::~AudioBuffer() {}
 
 void AudioBuffer::CopyFrom(const float* const* data,
-                           int num_frames,
-                           AudioProcessing::ChannelLayout layout) {
-  assert(num_frames == input_num_frames_);
-  assert(ChannelsFromLayout(layout) == num_input_channels_);
+                           const StreamConfig& stream_config) {
+  assert(stream_config.num_frames() == input_num_frames_);
+  assert(stream_config.num_channels() == num_input_channels_);
   InitForNewData();
   // Initialized lazily because there's a different condition in
   // DeinterleaveFrom.
-  if ((num_input_channels_ == 2 && num_proc_channels_ == 1) && !input_buffer_) {
+  const bool need_to_downmix =
+      num_input_channels_ > 1 && num_proc_channels_ == 1;
+  if (need_to_downmix && !input_buffer_) {
     input_buffer_.reset(
         new IFChannelBuffer(input_num_frames_, num_proc_channels_));
   }
 
-  if (HasKeyboardChannel(layout)) {
-    keyboard_data_ = data[KeyboardChannelIndex(layout)];
+  if (stream_config.has_keyboard()) {
+    keyboard_data_ = data[KeyboardChannelIndex(stream_config)];
   }
 
   // Downmix.
   const float* const* data_ptr = data;
-  if (num_input_channels_ == 2 && num_proc_channels_ == 1) {
-    StereoToMono(data[0],
-                 data[1],
-                 input_buffer_->fbuf()->channels()[0],
-                 input_num_frames_);
+  if (need_to_downmix) {
+    DownmixToMono<float, float>(data, input_num_frames_, num_input_channels_,
+                                input_buffer_->fbuf()->channels()[0]);
     data_ptr = input_buffer_->fbuf_const()->channels();
   }
 
   // Resample.
   if (input_num_frames_ != proc_num_frames_) {
     for (int i = 0; i < num_proc_channels_; ++i) {
       input_resamplers_[i]->Resample(data_ptr[i],
                                      input_num_frames_,
                                      process_buffer_->channels()[i],
                                      proc_num_frames_);
     }
     data_ptr = process_buffer_->channels();
   }
 
   // Convert to the S16 range.
   for (int i = 0; i < num_proc_channels_; ++i) {
     FloatToFloatS16(data_ptr[i],
                     proc_num_frames_,
                     data_->fbuf()->channels()[i]);
   }
 }
 
-void AudioBuffer::CopyTo(int num_frames,
-                         AudioProcessing::ChannelLayout layout,
+void AudioBuffer::CopyTo(const StreamConfig& stream_config,
                          float* const* data) {
-  assert(num_frames == output_num_frames_);
-  assert(ChannelsFromLayout(layout) == num_channels_);
+  assert(stream_config.num_frames() == output_num_frames_);
+  assert(stream_config.num_channels() == num_channels_);
 
   // Convert to the float range.
   float* const* data_ptr = data;
   if (output_num_frames_ != proc_num_frames_) {
     // Convert to an intermediate buffer for subsequent resampling.
     data_ptr = process_buffer_->channels();
   }
   for (int i = 0; i < num_channels_; ++i) {
     FloatS16ToFloat(data_->fbuf()->channels()[i],
                     proc_num_frames_,
@@ skipping 127 matching lines @@
 ChannelBuffer<float>* AudioBuffer::split_data_f() {
   mixed_low_pass_valid_ = false;
   return split_data_.get() ? split_data_->fbuf() : data_->fbuf();
 }
 
 const ChannelBuffer<float>* AudioBuffer::split_data_f() const {
   return split_data_.get() ? split_data_->fbuf_const() : data_->fbuf_const();
 }
 
 const int16_t* AudioBuffer::mixed_low_pass_data() {
-  // Currently only mixing stereo to mono is supported.
-  assert(num_proc_channels_ == 1 || num_proc_channels_ == 2);
-
   if (num_proc_channels_ == 1) {
     return split_bands_const(0)[kBand0To8kHz];
   }
 
   if (!mixed_low_pass_valid_) {
     if (!mixed_low_pass_channels_.get()) {
       mixed_low_pass_channels_.reset(
           new ChannelBuffer<int16_t>(num_split_frames_, 1));
     }
-    StereoToMono(split_bands_const(0)[kBand0To8kHz],
-                 split_bands_const(1)[kBand0To8kHz],
-                 mixed_low_pass_channels_->channels()[0],
-                 num_split_frames_);
+
+    DownmixToMono<int16_t, int32_t>(split_channels_const(kBand0To8kHz),
+                                    num_split_frames_, num_channels_,
+                                    mixed_low_pass_channels_->channels()[0]);
     mixed_low_pass_valid_ = true;
   }
   return mixed_low_pass_channels_->channels()[0];
 }
 
 const int16_t* AudioBuffer::low_pass_reference(int channel) const {
   if (!reference_copied_) {
     return NULL;
   }
 
@@ skipping 48 matching lines @@
         new IFChannelBuffer(input_num_frames_, num_proc_channels_));
   }
   activity_ = frame->vad_activity_;
 
   int16_t* const* deinterleaved;
   if (input_num_frames_ == proc_num_frames_) {
     deinterleaved = data_->ibuf()->channels();
   } else {
     deinterleaved = input_buffer_->ibuf()->channels();
   }
-  if (num_input_channels_ == 2 && num_proc_channels_ == 1) {
-    // Downmix directly; no explicit deinterleaving needed.
-    for (int i = 0; i < input_num_frames_; ++i) {
-      deinterleaved[0][i] = (frame->data_[i * 2] + frame->data_[i * 2 + 1]) / 2;
-    }
+  if (num_proc_channels_ == 1) {
+    // Downmix and deinterleave simultaneously.
+    DownmixInterleavedToMono(frame->data_, input_num_frames_,
+                             num_input_channels_, deinterleaved[0]);
   } else {
     assert(num_proc_channels_ == num_input_channels_);
     Deinterleave(frame->data_,
                  input_num_frames_,
                  num_proc_channels_,
                  deinterleaved);
   }
 
   // Resample.
   if (input_num_frames_ != proc_num_frames_) {
@@ skipping 41 matching lines @@
 
 void AudioBuffer::SplitIntoFrequencyBands() {
   splitting_filter_->Analysis(data_.get(), split_data_.get());
 }
 
 void AudioBuffer::MergeFrequencyBands() {
   splitting_filter_->Synthesis(split_data_.get(), data_.get());
 }
 
 }  // namespace webrtc