Integrate Intelligibility with APM

webrtc/modules/audio_processing/audio_buffer.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 418 matching lines @@
     for (int i = 0; i < num_proc_channels_; ++i) {
       input_resamplers_[i]->Resample(input_buffer_->fbuf_const()->channels()[i],
                                      input_num_frames_,
                                      data_->fbuf()->channels()[i],
                                      proc_num_frames_);
     }
   }
 }
 
 void AudioBuffer::InterleaveTo(AudioFrame* frame, bool data_changed) const {
-  assert(proc_num_frames_ == output_num_frames_);
-  assert(num_channels_ == num_input_channels_);
-  assert(frame->num_channels_ == num_channels_);
-  assert(frame->samples_per_channel_ == proc_num_frames_);
   frame->vad_activity_ = activity_;
-
   if (!data_changed) {
     return;
   }
 
-  Interleave(data_->ibuf()->channels(),
-             proc_num_frames_,
-             num_channels_,
-             frame->data_);
+  assert(proc_num_frames_ == output_num_frames_);
+  assert(frame->num_channels_ == num_channels_ || num_channels_ == 1);
+  assert(frame->samples_per_channel_ == proc_num_frames_);
+
+  if (frame->num_channels_ == num_channels_) {
+    Interleave(data_->ibuf()->channels(), proc_num_frames_, num_channels_,
+               frame->data_);
+  } else {
+    // Copy single AudioBuffer channel into all AudioFrame channels

[Andrew MacDonald, 2015/07/24 23:50:39]

Period at the end, and can you note that this is specifically for
intelligibility enhancement?

This is a problem (amongst other things) with introducing this render-side
processing. APM wasn't designed to handle it and so you're kind of shoe-horning
it in. I don't have a better suggestion without large refactoring though.

Ideally you would do your processing on however many channels you received
before downmixing to mono. But that would add a bunch of complexity to a system
that's already straining under the load. I think this upmixing is OK, but you
should note in the interface that if intelligibility enhancement is enabled the
reverse stream will become an upmixed mono signal.

[ekm, 2015/07/29 00:37:19]

Done.

+    rtc::scoped_ptr<int16_t*> channel_ptr_copies(
+        new int16_t*[frame->num_channels_]);

[Andrew MacDonald, 2015/07/24 23:50:39]

Arg, why did you switch to using dynamic allocation?

But anyway, perhaps better to add a UpmixMonoToInterleave? See the new
DownmixInterleavedToMono Michael added here:
https://chromium.googlesource.com/external/webrtc/+/master/webrtc/common_audi...

[ekm, 2015/07/29 00:37:19]

Done.

+    for (int i = 0; i < frame->num_channels_; ++i) {
+      channel_ptr_copies.get()[i] = data_->ibuf()->channels()[0];
+    }
+    Interleave(channel_ptr_copies.get(), proc_num_frames_, num_channels_,
+               frame->data_);
+  }
 }
 
 void AudioBuffer::CopyLowPassToReference() {
   reference_copied_ = true;
   if (!low_pass_reference_channels_.get() ||
       low_pass_reference_channels_->num_channels() != num_channels_) {
     low_pass_reference_channels_.reset(
         new ChannelBuffer<int16_t>(num_split_frames_,
                                    num_proc_channels_));
   }
   for (int i = 0; i < num_proc_channels_; i++) {
     memcpy(low_pass_reference_channels_->channels()[i],
            split_bands_const(i)[kBand0To8kHz],
            low_pass_reference_channels_->num_frames_per_band() *
                sizeof(split_bands_const(i)[kBand0To8kHz][0]));
   }
 }
 
 void AudioBuffer::SplitIntoFrequencyBands() {
   splitting_filter_->Analysis(data_.get(), split_data_.get());
 }
 
 void AudioBuffer::MergeFrequencyBands() {
   splitting_filter_->Synthesis(split_data_.get(), data_.get());
 }
 
 }  // namespace webrtc