Changed mixing api and removed resampler

webrtc/modules/audio_mixer/source/new_audio_conference_mixer_impl.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/modules/audio_mixer/source/new_audio_conference_mixer_impl.h"
 
 #include <algorithm>
 #include <functional>
 
 #include "webrtc/modules/audio_conference_mixer/source/audio_frame_manipulator.h"
 #include "webrtc/modules/audio_mixer/include/audio_mixer_defines.h"
 #include "webrtc/modules/audio_processing/include/audio_processing.h"
 #include "webrtc/modules/utility/include/audio_frame_operations.h"
 #include "webrtc/system_wrappers/include/critical_section_wrapper.h"
 #include "webrtc/system_wrappers/include/trace.h"
+#include "webrtc/voice_engine/utility.h"
 
 namespace webrtc {
 namespace {
 
 class SourceFrame {
  public:
   SourceFrame(MixerAudioSource* p, AudioFrame* a, bool m, bool was_mixed_before)
       : audio_source_(p),
         audio_frame_(a),
         muted_(m),
@@ skipping 19 matching lines @@
     return energy_ > other.energy_;
   }
 
   MixerAudioSource* audio_source_;
   AudioFrame* audio_frame_;
   bool muted_;
   uint32_t energy_;
   bool was_mixed_before_;
 };
 
+// Remixes a frame between stereo and mono.

[aleloi2, 2016/08/06 10:13:50]

I couldn't find any function that does this in WebRTC. There was
common_audio/audio_converter, but that only did floats. voe::RemixAndResample
was not suitable either.

+void RemixFrame(AudioFrame* frame, size_t number_of_channels) {
+  RTC_DCHECK(number_of_channels == 1 || number_of_channels == 2);
+  if (frame->num_channels_ == 1 && number_of_channels == 2) {
+    AudioFrameOperations::MonoToStereo(frame);
+  } else if (frame->num_channels_ == 2 && number_of_channels == 1) {
+    AudioFrameOperations::StereoToMono(frame);
+  }
+}
+
 // Mix |frame| into |mixed_frame|, with saturation protection and upmixing.
 // These effects are applied to |frame| itself prior to mixing. Assumes that
 // |mixed_frame| always has at least as many channels as |frame|. Supports
 // stereo at most.
 //
 // TODO(andrew): consider not modifying |frame| here.
 void MixFrames(AudioFrame* mixed_frame, AudioFrame* frame, bool use_limiter) {
   RTC_DCHECK_GE(mixed_frame->num_channels_, frame->num_channels_);
   if (use_limiter) {
     // Divide by two to avoid saturation in the mixing.
     // This is only meaningful if the limiter will be used.
     *frame >>= 1;
   }
-  if (mixed_frame->num_channels_ > frame->num_channels_) {
-    // We only support mono-to-stereo.
-    RTC_DCHECK_EQ(mixed_frame->num_channels_, static_cast<size_t>(2));
-    RTC_DCHECK_EQ(frame->num_channels_, static_cast<size_t>(1));
-    AudioFrameOperations::MonoToStereo(frame);
-  }
-
+  RTC_DCHECK_EQ(frame->num_channels_, mixed_frame->num_channels_);
   *mixed_frame += *frame;
 }
 
-// Return the max number of channels from a |list| composed of AudioFrames.
-size_t MaxNumChannels(const AudioFrameList* list) {
-  size_t max_num_channels = 1;
-  for (AudioFrameList::const_iterator iter = list->begin(); iter != list->end();
-       ++iter) {
-    max_num_channels = std::max(max_num_channels, (*iter).frame->num_channels_);
-  }
-  return max_num_channels;
-}
-
 }  // namespace
 
 MixerAudioSource::MixerAudioSource() : _mixHistory(new NewMixHistory()) {}
 
 MixerAudioSource::~MixerAudioSource() {
   delete _mixHistory;
 }
 
 bool MixerAudioSource::IsMixed() const {
   return _mixHistory->IsMixed();
@@ skipping 78 matching lines @@
 
   if (_limiter->gain_control()->enable_limiter(true) != _limiter->kNoError)
     return false;
 
   if (_limiter->gain_control()->Enable(true) != _limiter->kNoError)
     return false;
 
   return true;
 }
 
-void NewAudioConferenceMixerImpl::Mix(AudioFrame* audio_frame_for_mixing) {
+void NewAudioConferenceMixerImpl::Mix(int sample_rate,
+                                      size_t number_of_channels,
+                                      AudioFrame* audio_frame_for_mixing) {
+  RTC_DCHECK(number_of_channels == 1 || number_of_channels == 2);
   size_t remainingAudioSourcesAllowedToMix = kMaximumAmountOfMixedAudioSources;
   RTC_DCHECK(thread_checker_.CalledOnValidThread());
   AudioFrameList mixList;
   AudioFrameList additionalFramesList;
   std::map<int, MixerAudioSource*> mixedAudioSourcesMap;
   {
     CriticalSectionScoped cs(_cbCrit.get());
+    Frequency mixing_frequency;
 
-    int32_t lowFreq = GetLowestMixingFrequency();
-    // SILK can run in 12 kHz and 24 kHz. These frequencies are not
-    // supported so use the closest higher frequency to not lose any
-    // information.
-    // TODO(aleloi): this is probably more appropriate to do in
-    //                GetLowestMixingFrequency().
-    if (lowFreq == 12000) {
-      lowFreq = 16000;
-    } else if (lowFreq == 24000) {
-      lowFreq = 32000;
+    switch (sample_rate) {
+      case 8000:
+        mixing_frequency = kNbInHz;
+        break;
+      case 16000:
+        mixing_frequency = kWbInHz;
+        break;
+      case 32000:
+        mixing_frequency = kSwbInHz;
+        break;
+      case 48000:
+        mixing_frequency = kFbInHz;
+        break;
+      default:
+        RTC_NOTREACHED();
     }
-    if (lowFreq <= 0) {
-      return;
-    } else {
-      switch (lowFreq) {
-        case 8000:
-          if (OutputFrequency() != kNbInHz) {
-            SetOutputFrequency(kNbInHz);
-          }
-          break;
-        case 16000:
-          if (OutputFrequency() != kWbInHz) {
-            SetOutputFrequency(kWbInHz);
-          }
-          break;
-        case 32000:
-          if (OutputFrequency() != kSwbInHz) {
-            SetOutputFrequency(kSwbInHz);
-          }
-          break;
-        case 48000:
-          if (OutputFrequency() != kFbInHz) {
-            SetOutputFrequency(kFbInHz);
-          }
-          break;
-        default:
-          RTC_NOTREACHED();
-          return;
-      }
+
+    if (OutputFrequency() != mixing_frequency) {
+      SetOutputFrequency(mixing_frequency);
     }
 
     mixList = UpdateToMix(remainingAudioSourcesAllowedToMix);
     remainingAudioSourcesAllowedToMix -= mixList.size();
     GetAdditionalAudio(&additionalFramesList);
   }
 
-  // TODO(aleloi): it might be better to decide the number of channels
-  //                with an API instead of dynamically.
-
-  // Find the max channels over all mixing lists.
-  const size_t num_mixed_channels =
-      std::max(MaxNumChannels(&mixList), MaxNumChannels(&additionalFramesList));
+  for (FrameAndMuteInfo& frame_and_mute : mixList) {

[aleloi2, 2016/08/06 10:13:50]

Previously, all frames were up-mixed if a single one was stereo, and then
remixed to what ADM wanted (probably stereo?) before going in to audio device
module. Now we just remix them to what ADM asks for before doing anything else.

+    RemixFrame(frame_and_mute.frame, number_of_channels);
+  }
+  for (FrameAndMuteInfo& frame_and_mute : additionalFramesList) {
+    RemixFrame(frame_and_mute.frame, number_of_channels);
+  }
 
   audio_frame_for_mixing->UpdateFrame(
       -1, _timeStamp, NULL, 0, _outputFrequency, AudioFrame::kNormalSpeech,
-      AudioFrame::kVadPassive, num_mixed_channels);
+      AudioFrame::kVadPassive, number_of_channels);
 
   _timeStamp += static_cast<uint32_t>(_sampleSize);
 
   use_limiter_ = num_mixed_audio_sources_ > 1 &&
                  _outputFrequency <= AudioProcessing::kMaxNativeSampleRateHz;
 
   // We only use the limiter if it supports the output sample rate and
   // we're actually mixing multiple streams.
   MixFromList(audio_frame_for_mixing, mixList, _id, use_limiter_);
 
@@ skipping 137 matching lines @@
     _minimumMixingFreq = freq;
     return 0;
   } else {
     WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
                  "SetMinimumMixingFrequency incorrect frequency: %i", freq);
     RTC_NOTREACHED();
     return -1;
   }
 }
 
-// Check all AudioFrames that are to be mixed. The highest sampling frequency
-// found is the lowest that can be used without losing information.
-int32_t NewAudioConferenceMixerImpl::GetLowestMixingFrequency() const {
-  const int audioSourceListFrequency =
-      GetLowestMixingFrequencyFromList(audio_source_list_);
-  const int anonymousListFrequency =
-      GetLowestMixingFrequencyFromList(additional_audio_source_list_);
-  const int highestFreq = (audioSourceListFrequency > anonymousListFrequency)
-                              ? audioSourceListFrequency
-                              : anonymousListFrequency;
-  // Check if the user specified a lowest mixing frequency.
-  if (_minimumMixingFreq != kLowestPossible) {
-    if (_minimumMixingFreq > highestFreq) {
-      return _minimumMixingFreq;
-    }
-  }
-  return highestFreq;
-}
-
-int32_t NewAudioConferenceMixerImpl::GetLowestMixingFrequencyFromList(
-    const MixerAudioSourceList& mixList) const {
-  int32_t highestFreq = 8000;
-  for (MixerAudioSourceList::const_iterator iter = mixList.begin();
-       iter != mixList.end(); ++iter) {
-    const int32_t neededFrequency = (*iter)->NeededFrequency(_id);
-    if (neededFrequency > highestFreq) {
-      highestFreq = neededFrequency;
-    }
-  }
-  return highestFreq;
-}
-
 AudioFrameList NewAudioConferenceMixerImpl::UpdateToMix(
     size_t maxAudioFrameCounter) const {
   AudioFrameList result;
   std::vector<SourceFrame> audioSourceMixingDataList;
 
   // Get audio source audio and put it in the struct vector.
   for (MixerAudioSource* audio_source : audio_source_list_) {
     auto audio_frame_with_info = audio_source->GetAudioFrameWithMuted(
         _id, static_cast<int>(_outputFrequency));
 
@@ skipping 207 matching lines @@
 
   if (error != _limiter->kNoError) {
     WEBRTC_TRACE(kTraceError, kTraceAudioMixerServer, _id,
                  "Error from AudioProcessing: %d", error);
     RTC_NOTREACHED();
     return false;
   }
   return true;
 }
 }  // namespace webrtc