Allow more than 2 input channels in AudioProcessing.

webrtc/modules/audio_processing/audio_buffer.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/modules/audio_processing/audio_buffer.h"
 
 #include "webrtc/common_audio/include/audio_util.h"
 #include "webrtc/common_audio/resampler/push_sinc_resampler.h"
 #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
 #include "webrtc/common_audio/channel_buffer.h"
 #include "webrtc/modules/audio_processing/common.h"
 
 namespace webrtc {
 namespace {
 
 const int kSamplesPer16kHzChannel = 160;
 const int kSamplesPer32kHzChannel = 320;
 const int kSamplesPer48kHzChannel = 480;
 
-bool HasKeyboardChannel(AudioProcessing::ChannelLayout layout) {
-  switch (layout) {
-    case AudioProcessing::kMono:
-    case AudioProcessing::kStereo:
-      return false;
-    case AudioProcessing::kMonoAndKeyboard:
-    case AudioProcessing::kStereoAndKeyboard:
-      return true;
-  }
-  assert(false);
-  return false;
-}
-
-int KeyboardChannelIndex(AudioProcessing::ChannelLayout layout) {
-  switch (layout) {
-    case AudioProcessing::kMono:
-    case AudioProcessing::kStereo:
-      assert(false);
-      return -1;
-    case AudioProcessing::kMonoAndKeyboard:
+int KeyboardChannelIndex(const StreamConfig& stream_config) {

[aluebs-webrtc, 2015/07/14 23:12:42]

You could check inside here if stream_config.has_keayboard() and else return -1.

[mgraczyk, 2015/07/15 01:12:45]

Done, although the only callsite is conditional on has_keyboard being true.

+  switch (stream_config.num_channels()) {
+    case 1:
       return 1;
-    case AudioProcessing::kStereoAndKeyboard:
+    case 2:
       return 2;
   }
   assert(false);
   return -1;
 }
 
 template <typename T>
-void StereoToMono(const T* left, const T* right, T* out,
-                  int num_frames) {
-  for (int i = 0; i < num_frames; ++i)
-    out[i] = (left[i] + right[i]) / 2;
+void DownmixInterleavedToMono(const T* interleaved,

[aluebs-webrtc, 2015/07/14 23:12:42]

Is this specialization used anywhere?

[mgraczyk, 2015/07/15 01:12:46]

This isn't the specialization, it's a normal template definition.  I could
static_assert false here instead of providing default behavior, but this is the
behavior we'd want for floats.  What should I do? 

[aluebs-webrtc, 2015/07/15 18:04:05]

I meant, I don't think we have any interleaved float cases, right?

[mgraczyk, 2015/07/15 20:03:19]

That's true.  I removed the definition so now the function gets linked in for
int16_t input.

+                              T* deinterleaved,
+                              int num_multichannel_frames,
+                              int num_channels) {

[aluebs-webrtc, 2015/07/14 23:12:42]

Inputs before outputs?

[mgraczyk, 2015/07/15 01:12:45]

Done.

+  return DownmixInterleavedToMonoImpl<T, T>(

[aluebs-webrtc, 2015/07/14 23:12:42]

This return isn't needed, right?

[mgraczyk, 2015/07/15 01:12:45]

Done.

+      interleaved, deinterleaved, num_multichannel_frames, num_channels);
+}
+
+template <>

[aluebs-webrtc, 2015/07/14 23:12:42]

Why is this template needed? Just curious.

[mgraczyk, 2015/07/15 01:12:46]

I could write a function called "DownmixInterleavedInt16ToMono" instead.  I like
this because it allows us to worry about the buffer data types here instead of
at all the possible call sites. If we want to use int32 or int8 instead, we just
add another overload here.

[aluebs-webrtc, 2015/07/15 18:04:05]

Thanks for the explanation! Makes sense :)

+void DownmixInterleavedToMono<int16_t>(const int16_t* interleaved,
+                                       int16_t* deinterleaved,
+                                       int num_multichannel_frames,
+                                       int num_channels) {
+  return DownmixInterleavedToMonoImpl<int16_t, int32_t>(

[aluebs-webrtc, 2015/07/14 23:12:42]

This return isn't needed, right?

[mgraczyk, 2015/07/15 01:12:45]

Done.

+      interleaved, deinterleaved, num_multichannel_frames, num_channels);
 }
 
 int NumBandsFromSamplesPerChannel(int num_frames) {
   int num_bands = 1;
   if (num_frames == kSamplesPer32kHzChannel ||
       num_frames == kSamplesPer48kHzChannel) {
     num_bands = rtc::CheckedDivExact(num_frames,
                                      static_cast<int>(kSamplesPer16kHzChannel));
   }
   return num_bands;
@@ skipping 15 matching lines @@
     num_bands_(NumBandsFromSamplesPerChannel(proc_num_frames_)),
     num_split_frames_(rtc::CheckedDivExact(proc_num_frames_, num_bands_)),
     mixed_low_pass_valid_(false),
     reference_copied_(false),
     activity_(AudioFrame::kVadUnknown),
     keyboard_data_(NULL),
     data_(new IFChannelBuffer(proc_num_frames_, num_proc_channels_)) {
   assert(input_num_frames_ > 0);
   assert(proc_num_frames_ > 0);
   assert(output_num_frames_ > 0);
-  assert(num_input_channels_ > 0 && num_input_channels_ <= 2);
+  assert(num_input_channels_ > 0);
   assert(num_proc_channels_ > 0 && num_proc_channels_ <= num_input_channels_);
 
   if (input_num_frames_ != proc_num_frames_ ||
       output_num_frames_ != proc_num_frames_) {
     // Create an intermediate buffer for resampling.
     process_buffer_.reset(new ChannelBuffer<float>(proc_num_frames_,
                                                    num_proc_channels_));
 
     if (input_num_frames_ != proc_num_frames_) {
       for (int i = 0; i < num_proc_channels_; ++i) {
@@ skipping 18 matching lines @@
                                           num_bands_));
     splitting_filter_.reset(new SplittingFilter(num_proc_channels_,
                                                 num_bands_,
                                                 proc_num_frames_));
   }
 }
 
 AudioBuffer::~AudioBuffer() {}
 
 void AudioBuffer::CopyFrom(const float* const* data,
-                           int num_frames,
-                           AudioProcessing::ChannelLayout layout) {
-  assert(num_frames == input_num_frames_);
-  assert(ChannelsFromLayout(layout) == num_input_channels_);
+                           const StreamConfig& stream_config) {
+  assert(stream_config.samples_per_channel() == input_num_frames_);
+  assert(stream_config.num_channels() == num_input_channels_);
   InitForNewData();
   // Initialized lazily because there's a different condition in
   // DeinterleaveFrom.
-  if ((num_input_channels_ == 2 && num_proc_channels_ == 1) && !input_buffer_) {
+  const bool need_to_downmix =
+      num_input_channels_ > 1 && num_proc_channels_ == 1;
+  if (need_to_downmix && !input_buffer_) {
     input_buffer_.reset(
         new IFChannelBuffer(input_num_frames_, num_proc_channels_));
   }
 
-  if (HasKeyboardChannel(layout)) {
-    keyboard_data_ = data[KeyboardChannelIndex(layout)];
+  if (stream_config.has_keyboard()) {
+    keyboard_data_ = data[KeyboardChannelIndex(stream_config)];
   }
 
   // Downmix.
   const float* const* data_ptr = data;
-  if (num_input_channels_ == 2 && num_proc_channels_ == 1) {
-    StereoToMono(data[0],
-                 data[1],
-                 input_buffer_->fbuf()->channels()[0],
-                 input_num_frames_);
+  if (need_to_downmix) {
+    DownmixToMono<float, float>(input_num_frames_,
+                                input_buffer_->fbuf()->channels()[0], data,
+                                num_input_channels_);
     data_ptr = input_buffer_->fbuf_const()->channels();
   }
 
   // Resample.
   if (input_num_frames_ != proc_num_frames_) {
     for (int i = 0; i < num_proc_channels_; ++i) {
       input_resamplers_[i]->Resample(data_ptr[i],
                                      input_num_frames_,
                                      process_buffer_->channels()[i],
                                      proc_num_frames_);
     }
     data_ptr = process_buffer_->channels();
   }
 
   // Convert to the S16 range.
   for (int i = 0; i < num_proc_channels_; ++i) {
     FloatToFloatS16(data_ptr[i],
                     proc_num_frames_,
                     data_->fbuf()->channels()[i]);
   }
 }
 
-void AudioBuffer::CopyTo(int num_frames,
-                         AudioProcessing::ChannelLayout layout,
+void AudioBuffer::CopyTo(const StreamConfig& stream_config,
                          float* const* data) {
-  assert(num_frames == output_num_frames_);
-  assert(ChannelsFromLayout(layout) == num_channels_);
+  assert(stream_config.samples_per_channel() == output_num_frames_);
+  assert(stream_config.num_channels() == num_channels_);
 
   // Convert to the float range.
   float* const* data_ptr = data;
   if (output_num_frames_ != proc_num_frames_) {
     // Convert to an intermediate buffer for subsequent resampling.
     data_ptr = process_buffer_->channels();
   }
   for (int i = 0; i < num_channels_; ++i) {
     FloatS16ToFloat(data_->fbuf()->channels()[i],
                     proc_num_frames_,
@@ skipping 139 matching lines @@
 
   if (num_proc_channels_ == 1) {
     return split_bands_const(0)[kBand0To8kHz];
   }
 
   if (!mixed_low_pass_valid_) {
     if (!mixed_low_pass_channels_.get()) {
       mixed_low_pass_channels_.reset(
           new ChannelBuffer<int16_t>(num_split_frames_, 1));
     }
-    StereoToMono(split_bands_const(0)[kBand0To8kHz],
-                 split_bands_const(1)[kBand0To8kHz],
-                 mixed_low_pass_channels_->channels()[0],
-                 num_split_frames_);
+    DownmixStereoToMono<int16_t, int32_t>(

[aluebs-webrtc, 2015/07/14 23:12:42]

Shouldn't this support multiple channels?

[mgraczyk, 2015/07/15 01:12:46]

Done.  Does this work?

[aluebs-webrtc, 2015/07/15 18:04:05]

You don't need to do this manually, audio_buffer/channel_buffer does it for you.
Just use split_channels_const(kBand0To8kHz) instead of split_bands_const().

[mgraczyk, 2015/07/15 20:03:19]

Nice, Done.

+        num_split_frames_, mixed_low_pass_channels_->channels()[0],
+        split_bands_const(0)[kBand0To8kHz], split_bands_const(1)[kBand0To8kHz]);
     mixed_low_pass_valid_ = true;
   }
   return mixed_low_pass_channels_->channels()[0];
 }
 
 const int16_t* AudioBuffer::low_pass_reference(int channel) const {
   if (!reference_copied_) {
     return NULL;
   }
 
@@ skipping 48 matching lines @@
         new IFChannelBuffer(input_num_frames_, num_proc_channels_));
   }
   activity_ = frame->vad_activity_;
 
   int16_t* const* deinterleaved;
   if (input_num_frames_ == proc_num_frames_) {
     deinterleaved = data_->ibuf()->channels();
   } else {
     deinterleaved = input_buffer_->ibuf()->channels();
   }
-  if (num_input_channels_ == 2 && num_proc_channels_ == 1) {
-    // Downmix directly; no explicit deinterleaving needed.
-    for (int i = 0; i < input_num_frames_; ++i) {
-      deinterleaved[0][i] = (frame->data_[i * 2] + frame->data_[i * 2 + 1]) / 2;
-    }
+  if (num_proc_channels_ == 1) {

[mgraczyk, 2015/07/15 01:12:46]

This function works for any number of input channels.

+    // Downmix and deinterleave simultaneously.
+    DownmixInterleavedToMono(frame->data_, deinterleaved[0], input_num_frames_,
+                             num_input_channels_);
   } else {
     assert(num_proc_channels_ == num_input_channels_);
     Deinterleave(frame->data_,
                  input_num_frames_,
                  num_proc_channels_,
                  deinterleaved);
   }
 
   // Resample.
   if (input_num_frames_ != proc_num_frames_) {
@@ skipping 41 matching lines @@
 
 void AudioBuffer::SplitIntoFrequencyBands() {
   splitting_filter_->Analysis(data_.get(), split_data_.get());
 }
 
 void AudioBuffer::MergeFrequencyBands() {
   splitting_filter_->Synthesis(split_data_.get(), data_.get());
 }
 
 }  // namespace webrtc