OLD | NEW |
(Empty) | |
| 1 /* |
| 2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. |
| 3 * |
| 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ |
| 10 |
| 11 #include "webrtc/modules/audio_mixer/frame_combiner.h" |
| 12 |
| 13 #include <algorithm> |
| 14 #include <array> |
| 15 #include <functional> |
| 16 #include <memory> |
| 17 |
| 18 #include "webrtc/audio/utility/audio_frame_operations.h" |
| 19 #include "webrtc/base/logging.h" |
| 20 #include "webrtc/modules/audio_mixer/audio_frame_manipulator.h" |
| 21 #include "webrtc/modules/audio_mixer/audio_mixer_impl.h" |
| 22 |
| 23 namespace webrtc { |
| 24 namespace { |
| 25 |
| 26 // Stereo, 48 kHz, 10 ms. |
| 27 constexpr int kMaximalFrameSize = 2 * 48 * 10; |
| 28 |
| 29 void CombineZeroFrames(AudioFrame* audio_frame_for_mixing) { |
| 30 audio_frame_for_mixing->elapsed_time_ms_ = -1; |
| 31 AudioFrameOperations::Mute(audio_frame_for_mixing); |
| 32 } |
| 33 |
| 34 void CombineOneFrame(const AudioFrame* input_frame, |
| 35 AudioFrame* audio_frame_for_mixing) { |
| 36 audio_frame_for_mixing->timestamp_ = input_frame->timestamp_; |
| 37 audio_frame_for_mixing->elapsed_time_ms_ = input_frame->elapsed_time_ms_; |
| 38 std::copy(input_frame->data_, |
| 39 input_frame->data_ + |
| 40 input_frame->num_channels_ * input_frame->samples_per_channel_, |
| 41 audio_frame_for_mixing->data_); |
| 42 } |
| 43 |
| 44 std::unique_ptr<AudioProcessing> CreateLimiter() { |
| 45 Config config; |
| 46 config.Set<ExperimentalAgc>(new ExperimentalAgc(false)); |
| 47 std::unique_ptr<AudioProcessing> limiter(AudioProcessing::Create(config)); |
| 48 RTC_DCHECK(limiter); |
| 49 |
| 50 const auto check_no_error = [](int x) { |
| 51 RTC_DCHECK_EQ(x, AudioProcessing::kNoError); |
| 52 }; |
| 53 auto* const gain_control = limiter->gain_control(); |
| 54 check_no_error(gain_control->set_mode(GainControl::kFixedDigital)); |
| 55 |
| 56 // We smoothly limit the mixed frame to -7 dbFS. -6 would correspond to the |
| 57 // divide-by-2 but -7 is used instead to give a bit of headroom since the |
| 58 // AGC is not a hard limiter. |
| 59 check_no_error(gain_control->set_target_level_dbfs(7)); |
| 60 |
| 61 check_no_error(gain_control->set_compression_gain_db(0)); |
| 62 check_no_error(gain_control->enable_limiter(true)); |
| 63 check_no_error(gain_control->Enable(true)); |
| 64 return limiter; |
| 65 } |
| 66 } // namespace |
| 67 |
| 68 FrameCombiner::FrameCombiner(bool use_apm_limiter) |
| 69 : use_apm_limiter_(use_apm_limiter), |
| 70 limiter_(use_apm_limiter ? CreateLimiter() : nullptr) {} |
| 71 |
| 72 FrameCombiner::~FrameCombiner() = default; |
| 73 |
| 74 void FrameCombiner::Combine(const std::vector<AudioFrame*>& mix_list, |
| 75 size_t number_of_channels, |
| 76 int sample_rate, |
| 77 AudioFrame* audio_frame_for_mixing) const { |
| 78 RTC_DCHECK(audio_frame_for_mixing); |
| 79 const size_t samples_per_channel = static_cast<size_t>( |
| 80 (sample_rate * webrtc::AudioMixerImpl::kFrameDurationInMs) / 1000); |
| 81 |
| 82 for (const auto* frame : mix_list) { |
| 83 RTC_DCHECK_EQ(samples_per_channel, frame->samples_per_channel_); |
| 84 RTC_DCHECK_EQ(sample_rate, frame->sample_rate_hz_); |
| 85 } |
| 86 |
| 87 // Frames could be both stereo and mono. |
| 88 for (auto* frame : mix_list) { |
| 89 RemixFrame(number_of_channels, frame); |
| 90 } |
| 91 |
| 92 // TODO(aleloi): Issue bugs.webrtc.org/3390. |
| 93 // Audio frame timestamp. The 'timestamp_' field is set to dummy |
| 94 // value '0', because it is only supported in the one channel case and |
| 95 // is then updated in the helper functions. |
| 96 audio_frame_for_mixing->UpdateFrame( |
| 97 -1, 0, nullptr, samples_per_channel, sample_rate, AudioFrame::kUndefined, |
| 98 AudioFrame::kVadUnknown, number_of_channels); |
| 99 |
| 100 if (mix_list.size() == 0) { |
| 101 CombineZeroFrames(audio_frame_for_mixing); |
| 102 } else if (mix_list.size() == 1) { |
| 103 CombineOneFrame(mix_list.front(), audio_frame_for_mixing); |
| 104 } else { |
| 105 std::vector<rtc::ArrayView<const int16_t>> input_frames; |
| 106 for (size_t i = 0; i < mix_list.size(); ++i) { |
| 107 input_frames.push_back(rtc::ArrayView<const int16_t>( |
| 108 mix_list[i]->data_, samples_per_channel * number_of_channels)); |
| 109 } |
| 110 CombineMultipleFrames(input_frames, audio_frame_for_mixing); |
| 111 } |
| 112 } |
| 113 |
| 114 void FrameCombiner::CombineMultipleFrames( |
| 115 const std::vector<rtc::ArrayView<const int16_t>>& input_frames, |
| 116 AudioFrame* audio_frame_for_mixing) const { |
| 117 RTC_DCHECK(!input_frames.empty()); |
| 118 RTC_DCHECK(audio_frame_for_mixing); |
| 119 |
| 120 const size_t frame_length = input_frames.front().size(); |
| 121 for (const auto& frame : input_frames) { |
| 122 RTC_DCHECK_EQ(frame_length, frame.size()); |
| 123 } |
| 124 |
| 125 // Algorithm: int16 frames are added to a sufficiently large |
| 126 // statically allocated int32 buffer. For > 2 participants this is |
| 127 // more efficient than addition in place in the int16 audio |
| 128 // frame. The audio quality loss due to halving the samples is |
| 129 // smaller than 16-bit addition in place. |
| 130 RTC_DCHECK_GE(kMaximalFrameSize, frame_length); |
| 131 std::array<int32_t, kMaximalFrameSize> add_buffer; |
| 132 |
| 133 add_buffer.fill(0); |
| 134 |
| 135 for (const auto& frame : input_frames) { |
| 136 std::transform(frame.begin(), frame.end(), add_buffer.begin(), |
| 137 add_buffer.begin(), std::plus<int32_t>()); |
| 138 } |
| 139 |
| 140 if (use_apm_limiter_) { |
| 141 // Halve all samples to avoid saturation before limiting. |
| 142 std::transform(add_buffer.begin(), add_buffer.begin() + frame_length, |
| 143 audio_frame_for_mixing->data_, [](int32_t a) { |
| 144 return rtc::saturated_cast<int16_t>(a / 2); |
| 145 }); |
| 146 |
| 147 // Smoothly limit the audio. |
| 148 RTC_DCHECK(limiter_); |
| 149 const int error = limiter_->ProcessStream(audio_frame_for_mixing); |
| 150 if (error != limiter_->kNoError) { |
| 151 LOG_F(LS_ERROR) << "Error from AudioProcessing: " << error; |
| 152 RTC_NOTREACHED(); |
| 153 } |
| 154 |
| 155 // And now we can safely restore the level. This procedure results in |
| 156 // some loss of resolution, deemed acceptable. |
| 157 // |
| 158 // It's possible to apply the gain in the AGC (with a target level of 0 dbFS |
| 159 // and compression gain of 6 dB). However, in the transition frame when this |
| 160 // is enabled (moving from one to two audio sources) it has the potential to |
| 161 // create discontinuities in the mixed frame. |
| 162 // |
| 163 // Instead we double the frame (with addition since left-shifting a |
| 164 // negative value is undefined). |
| 165 AudioFrameOperations::Add(*audio_frame_for_mixing, audio_frame_for_mixing); |
| 166 } else { |
| 167 std::transform(add_buffer.begin(), add_buffer.begin() + frame_length, |
| 168 audio_frame_for_mixing->data_, |
| 169 [](int32_t a) { return rtc::saturated_cast<int16_t>(a); }); |
| 170 } |
| 171 } |
| 172 } // namespace webrtc |
OLD | NEW |