OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 #include "webrtc/modules/audio_processing/aec3/echo_canceller3.h" | 10 #include "webrtc/modules/audio_processing/aec3/echo_canceller3.h" |
11 | 11 |
| 12 #include <sstream> |
| 13 |
12 #include "webrtc/base/atomicops.h" | 14 #include "webrtc/base/atomicops.h" |
13 #include "webrtc/system_wrappers/include/logging.h" | 15 #include "webrtc/modules/audio_processing/logging/apm_data_dumper.h" |
14 | 16 |
15 namespace webrtc { | 17 namespace webrtc { |
16 | 18 |
| 19 namespace { |
| 20 |
| 21 bool DetectSaturation(rtc::ArrayView<const float> y) { |
| 22 for (auto y_k : y) { |
| 23 if (y_k >= 32767.0f || y_k <= -32768.0f) { |
| 24 return true; |
| 25 } |
| 26 } |
| 27 return false; |
| 28 } |
| 29 |
| 30 void FillSubFrameView(AudioBuffer* frame, |
| 31 size_t sub_frame_index, |
| 32 std::vector<rtc::ArrayView<float>>* sub_frame_view) { |
| 33 RTC_DCHECK_GE(1, sub_frame_index); |
| 34 RTC_DCHECK_LE(0, sub_frame_index); |
| 35 RTC_DCHECK_EQ(frame->num_bands(), sub_frame_view->size()); |
| 36 for (size_t k = 0; k < sub_frame_view->size(); ++k) { |
| 37 (*sub_frame_view)[k] = rtc::ArrayView<float>( |
| 38 &frame->split_bands_f(0)[k][sub_frame_index * kSubFrameLength], |
| 39 kSubFrameLength); |
| 40 } |
| 41 } |
| 42 |
| 43 void FillSubFrameView(std::vector<std::vector<float>>* frame, |
| 44 size_t sub_frame_index, |
| 45 std::vector<rtc::ArrayView<float>>* sub_frame_view) { |
| 46 RTC_DCHECK_GE(1, sub_frame_index); |
| 47 RTC_DCHECK_EQ(frame->size(), sub_frame_view->size()); |
| 48 for (size_t k = 0; k < frame->size(); ++k) { |
| 49 (*sub_frame_view)[k] = rtc::ArrayView<float>( |
| 50 &(*frame)[k][sub_frame_index * kSubFrameLength], kSubFrameLength); |
| 51 } |
| 52 } |
| 53 |
| 54 void ProcessCaptureFrameContent( |
| 55 AudioBuffer* capture, |
| 56 bool known_echo_path_change, |
| 57 bool saturated_microphone_signal, |
| 58 size_t sub_frame_index, |
| 59 FrameBlocker* capture_blocker, |
| 60 BlockFramer* output_framer, |
| 61 BlockProcessor* block_processor, |
| 62 std::vector<std::vector<float>>* block, |
| 63 std::vector<rtc::ArrayView<float>>* sub_frame_view) { |
| 64 FillSubFrameView(capture, sub_frame_index, sub_frame_view); |
| 65 capture_blocker->InsertSubFrameAndExtractBlock(*sub_frame_view, block); |
| 66 block_processor->ProcessCapture(known_echo_path_change, |
| 67 saturated_microphone_signal, block); |
| 68 output_framer->InsertBlockAndExtractSubFrame(*block, sub_frame_view); |
| 69 } |
| 70 |
| 71 void ProcessRemainingCaptureFrameContent( |
| 72 bool known_echo_path_change, |
| 73 bool saturated_microphone_signal, |
| 74 FrameBlocker* capture_blocker, |
| 75 BlockFramer* output_framer, |
| 76 BlockProcessor* block_processor, |
| 77 std::vector<std::vector<float>>* block) { |
| 78 if (!capture_blocker->IsBlockAvailable()) { |
| 79 return; |
| 80 } |
| 81 |
| 82 capture_blocker->ExtractBlock(block); |
| 83 block_processor->ProcessCapture(known_echo_path_change, |
| 84 saturated_microphone_signal, block); |
| 85 output_framer->InsertBlock(*block); |
| 86 } |
| 87 |
| 88 bool BufferRenderFrameContent( |
| 89 std::vector<std::vector<float>>* render_frame, |
| 90 size_t sub_frame_index, |
| 91 FrameBlocker* render_blocker, |
| 92 BlockProcessor* block_processor, |
| 93 std::vector<std::vector<float>>* block, |
| 94 std::vector<rtc::ArrayView<float>>* sub_frame_view) { |
| 95 FillSubFrameView(render_frame, sub_frame_index, sub_frame_view); |
| 96 render_blocker->InsertSubFrameAndExtractBlock(*sub_frame_view, block); |
| 97 return block_processor->BufferRender(block); |
| 98 } |
| 99 |
| 100 bool BufferRemainingRenderFrameContent(FrameBlocker* render_blocker, |
| 101 BlockProcessor* block_processor, |
| 102 std::vector<std::vector<float>>* block) { |
| 103 if (!render_blocker->IsBlockAvailable()) { |
| 104 return false; |
| 105 } |
| 106 render_blocker->ExtractBlock(block); |
| 107 return block_processor->BufferRender(block); |
| 108 } |
| 109 |
| 110 void CopyAudioBufferIntoFrame(AudioBuffer* buffer, |
| 111 size_t num_bands, |
| 112 size_t frame_length, |
| 113 std::vector<std::vector<float>>* frame) { |
| 114 RTC_DCHECK_EQ(num_bands, frame->size()); |
| 115 for (size_t i = 0; i < num_bands; ++i) { |
| 116 rtc::ArrayView<float> buffer_view(&buffer->split_bands_f(0)[i][0], |
| 117 frame_length); |
| 118 std::copy(buffer_view.begin(), buffer_view.end(), (*frame)[i].begin()); |
| 119 } |
| 120 } |
| 121 |
| 122 // [B,A] = butter(2,100/4000,'high') |
| 123 const CascadedBiQuadFilter::BiQuadCoefficients |
| 124 kHighPassFilterCoefficients_8kHz = {{0.94598f, -1.89195f, 0.94598f}, |
| 125 {-1.88903f, 0.89487f}}; |
| 126 const int kNumberOfHighPassBiQuads_8kHz = 1; |
| 127 |
| 128 // [B,A] = butter(2,100/8000,'high') |
| 129 const CascadedBiQuadFilter::BiQuadCoefficients |
| 130 kHighPassFilterCoefficients_16kHz = {{0.97261f, -1.94523f, 0.97261f}, |
| 131 {-1.94448f, 0.94598f}}; |
| 132 const int kNumberOfHighPassBiQuads_16kHz = 1; |
| 133 |
| 134 static constexpr size_t kRenderTransferQueueSize = 30; |
| 135 |
| 136 } // namespace |
| 137 |
| 138 class EchoCanceller3::RenderWriter { |
| 139 public: |
| 140 RenderWriter(ApmDataDumper* data_dumper, |
| 141 SwapQueue<std::vector<std::vector<float>>, |
| 142 Aec3RenderQueueItemVerifier>* render_transfer_queue, |
| 143 std::unique_ptr<CascadedBiQuadFilter> render_highpass_filter, |
| 144 int sample_rate_hz, |
| 145 int frame_length, |
| 146 int num_bands); |
| 147 ~RenderWriter(); |
| 148 bool Insert(AudioBuffer* render); |
| 149 |
| 150 private: |
| 151 ApmDataDumper* data_dumper_; |
| 152 const int sample_rate_hz_; |
| 153 const size_t frame_length_; |
| 154 const int num_bands_; |
| 155 std::unique_ptr<CascadedBiQuadFilter> render_highpass_filter_; |
| 156 std::vector<std::vector<float>> render_queue_input_frame_; |
| 157 SwapQueue<std::vector<std::vector<float>>, Aec3RenderQueueItemVerifier>* |
| 158 render_transfer_queue_; |
| 159 RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderWriter); |
| 160 }; |
| 161 |
| 162 EchoCanceller3::RenderWriter::RenderWriter( |
| 163 ApmDataDumper* data_dumper, |
| 164 SwapQueue<std::vector<std::vector<float>>, Aec3RenderQueueItemVerifier>* |
| 165 render_transfer_queue, |
| 166 std::unique_ptr<CascadedBiQuadFilter> render_highpass_filter, |
| 167 int sample_rate_hz, |
| 168 int frame_length, |
| 169 int num_bands) |
| 170 : data_dumper_(data_dumper), |
| 171 sample_rate_hz_(sample_rate_hz), |
| 172 frame_length_(frame_length), |
| 173 num_bands_(num_bands), |
| 174 render_highpass_filter_(std::move(render_highpass_filter)), |
| 175 render_queue_input_frame_(num_bands_, |
| 176 std::vector<float>(frame_length_, 0.f)), |
| 177 render_transfer_queue_(render_transfer_queue) { |
| 178 RTC_DCHECK(data_dumper); |
| 179 } |
| 180 |
| 181 EchoCanceller3::RenderWriter::~RenderWriter() = default; |
| 182 |
| 183 bool EchoCanceller3::RenderWriter::Insert(AudioBuffer* input) { |
| 184 RTC_DCHECK_EQ(1, input->num_channels()); |
| 185 RTC_DCHECK_EQ(num_bands_, input->num_bands()); |
| 186 RTC_DCHECK_EQ(frame_length_, input->num_frames_per_band()); |
| 187 data_dumper_->DumpWav("aec3_render_input", frame_length_, |
| 188 &input->split_bands_f(0)[0][0], |
| 189 LowestBandRate(sample_rate_hz_), 1); |
| 190 |
| 191 CopyAudioBufferIntoFrame(input, num_bands_, frame_length_, |
| 192 &render_queue_input_frame_); |
| 193 |
| 194 if (render_highpass_filter_) { |
| 195 render_highpass_filter_->Process(render_queue_input_frame_[0]); |
| 196 } |
| 197 |
| 198 return render_transfer_queue_->Insert(&render_queue_input_frame_); |
| 199 } |
| 200 |
17 int EchoCanceller3::instance_count_ = 0; | 201 int EchoCanceller3::instance_count_ = 0; |
18 | 202 |
19 EchoCanceller3::EchoCanceller3(int sample_rate_hz, bool use_anti_hum_filter) { | 203 EchoCanceller3::EchoCanceller3(int sample_rate_hz, bool use_highpass_filter) |
20 int band_sample_rate_hz = (sample_rate_hz == 8000 ? sample_rate_hz : 16000); | 204 : EchoCanceller3(sample_rate_hz, |
21 frame_length_ = rtc::CheckedDivExact(band_sample_rate_hz, 100); | 205 use_highpass_filter, |
22 | 206 std::unique_ptr<BlockProcessor>( |
23 LOG(LS_INFO) << "AEC3 created : " | 207 BlockProcessor::Create(sample_rate_hz))) {} |
24 << "{ instance_count: " << instance_count_ << "}"; | 208 EchoCanceller3::EchoCanceller3(int sample_rate_hz, |
25 instance_count_ = rtc::AtomicOps::Increment(&instance_count_); | 209 bool use_highpass_filter, |
| 210 std::unique_ptr<BlockProcessor> block_processor) |
| 211 : data_dumper_( |
| 212 new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), |
| 213 sample_rate_hz_(sample_rate_hz), |
| 214 num_bands_(NumBandsForRate(sample_rate_hz_)), |
| 215 frame_length_(rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100)), |
| 216 output_framer_(num_bands_), |
| 217 capture_blocker_(num_bands_), |
| 218 render_blocker_(num_bands_), |
| 219 render_transfer_queue_( |
| 220 kRenderTransferQueueSize, |
| 221 std::vector<std::vector<float>>( |
| 222 num_bands_, |
| 223 std::vector<float>(frame_length_, 0.f)), |
| 224 Aec3RenderQueueItemVerifier(num_bands_, frame_length_)), |
| 225 block_processor_(std::move(block_processor)), |
| 226 render_queue_output_frame_(num_bands_, |
| 227 std::vector<float>(frame_length_, 0.f)), |
| 228 block_(num_bands_, std::vector<float>(kBlockSize, 0.f)), |
| 229 sub_frame_view_(num_bands_) { |
| 230 std::unique_ptr<CascadedBiQuadFilter> render_highpass_filter; |
| 231 if (use_highpass_filter) { |
| 232 render_highpass_filter.reset(new CascadedBiQuadFilter( |
| 233 sample_rate_hz_ == 8000 ? kHighPassFilterCoefficients_8kHz |
| 234 : kHighPassFilterCoefficients_16kHz, |
| 235 sample_rate_hz_ == 8000 ? kNumberOfHighPassBiQuads_8kHz |
| 236 : kNumberOfHighPassBiQuads_16kHz)); |
| 237 capture_highpass_filter_.reset(new CascadedBiQuadFilter( |
| 238 sample_rate_hz_ == 8000 ? kHighPassFilterCoefficients_8kHz |
| 239 : kHighPassFilterCoefficients_16kHz, |
| 240 sample_rate_hz_ == 8000 ? kNumberOfHighPassBiQuads_8kHz |
| 241 : kNumberOfHighPassBiQuads_16kHz)); |
| 242 } |
| 243 |
| 244 render_writer_.reset( |
| 245 new RenderWriter(data_dumper_.get(), &render_transfer_queue_, |
| 246 std::move(render_highpass_filter), sample_rate_hz_, |
| 247 frame_length_, num_bands_)); |
| 248 |
| 249 RTC_DCHECK_EQ(num_bands_, std::max(sample_rate_hz_, 16000) / 16000); |
| 250 RTC_DCHECK_GE(kMaxNumBands, num_bands_); |
26 } | 251 } |
27 | 252 |
28 EchoCanceller3::~EchoCanceller3() = default; | 253 EchoCanceller3::~EchoCanceller3() = default; |
29 | 254 |
30 bool EchoCanceller3::AnalyzeRender(AudioBuffer* render) { | 255 bool EchoCanceller3::AnalyzeRender(AudioBuffer* render) { |
31 RTC_DCHECK_EQ(1u, render->num_channels()); | 256 RTC_DCHECK_RUNS_SERIALIZED(&render_race_checker_); |
32 RTC_DCHECK_EQ(frame_length_, render->num_frames_per_band()); | 257 RTC_DCHECK(render); |
33 return true; | 258 return render_writer_->Insert(render); |
34 } | 259 } |
35 | 260 |
36 void EchoCanceller3::AnalyzeCapture(AudioBuffer* capture) {} | 261 void EchoCanceller3::AnalyzeCapture(AudioBuffer* capture) { |
| 262 RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); |
| 263 RTC_DCHECK(capture); |
| 264 data_dumper_->DumpWav("aec3_capture_analyze_input", frame_length_, |
| 265 capture->channels_f()[0], sample_rate_hz_, 1); |
| 266 |
| 267 saturated_microphone_signal_ = false; |
| 268 for (size_t k = 0; k < capture->num_channels(); ++k) { |
| 269 saturated_microphone_signal_ |= |
| 270 DetectSaturation(rtc::ArrayView<const float>(capture->channels_f()[k], |
| 271 capture->num_frames())); |
| 272 if (saturated_microphone_signal_) { |
| 273 break; |
| 274 } |
| 275 } |
| 276 } |
37 | 277 |
38 void EchoCanceller3::ProcessCapture(AudioBuffer* capture, | 278 void EchoCanceller3::ProcessCapture(AudioBuffer* capture, |
39 bool known_echo_path_change) { | 279 bool known_echo_path_change) { |
| 280 RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); |
| 281 RTC_DCHECK(capture); |
40 RTC_DCHECK_EQ(1u, capture->num_channels()); | 282 RTC_DCHECK_EQ(1u, capture->num_channels()); |
| 283 RTC_DCHECK_EQ(num_bands_, capture->num_bands()); |
41 RTC_DCHECK_EQ(frame_length_, capture->num_frames_per_band()); | 284 RTC_DCHECK_EQ(frame_length_, capture->num_frames_per_band()); |
| 285 |
| 286 rtc::ArrayView<float> capture_lower_band = |
| 287 rtc::ArrayView<float>(&capture->split_bands_f(0)[0][0], frame_length_); |
| 288 |
| 289 data_dumper_->DumpWav("aec3_capture_input", capture_lower_band, |
| 290 LowestBandRate(sample_rate_hz_), 1); |
| 291 |
| 292 const bool render_buffer_overrun = EmptyRenderQueue(); |
| 293 RTC_DCHECK(!render_buffer_overrun); |
| 294 |
| 295 if (capture_highpass_filter_) { |
| 296 capture_highpass_filter_->Process(capture_lower_band); |
| 297 } |
| 298 |
| 299 ProcessCaptureFrameContent(capture, known_echo_path_change, |
| 300 saturated_microphone_signal_, 0, &capture_blocker_, |
| 301 &output_framer_, block_processor_.get(), &block_, |
| 302 &sub_frame_view_); |
| 303 |
| 304 if (sample_rate_hz_ != 8000) { |
| 305 ProcessCaptureFrameContent( |
| 306 capture, known_echo_path_change, saturated_microphone_signal_, 1, |
| 307 &capture_blocker_, &output_framer_, block_processor_.get(), &block_, |
| 308 &sub_frame_view_); |
| 309 } |
| 310 |
| 311 ProcessRemainingCaptureFrameContent( |
| 312 known_echo_path_change, saturated_microphone_signal_, &capture_blocker_, |
| 313 &output_framer_, block_processor_.get(), &block_); |
| 314 |
| 315 data_dumper_->DumpWav("aec3_capture_output", frame_length_, |
| 316 &capture->split_bands_f(0)[0][0], |
| 317 LowestBandRate(sample_rate_hz_), 1); |
42 } | 318 } |
43 | 319 |
44 std::string EchoCanceller3::ToString( | 320 std::string EchoCanceller3::ToString( |
45 const AudioProcessing::Config::EchoCanceller3& config) { | 321 const AudioProcessing::Config::EchoCanceller3& config) { |
46 std::stringstream ss; | 322 std::stringstream ss; |
47 ss << "{" | 323 ss << "{" |
48 << "enabled: " << (config.enabled ? "true" : "false") << "}"; | 324 << "enabled: " << (config.enabled ? "true" : "false") << "}"; |
49 return ss.str(); | 325 return ss.str(); |
50 } | 326 } |
51 | 327 |
52 bool EchoCanceller3::Validate( | 328 bool EchoCanceller3::Validate( |
53 const AudioProcessing::Config::EchoCanceller3& config) { | 329 const AudioProcessing::Config::EchoCanceller3& config) { |
54 return true; | 330 return true; |
55 } | 331 } |
56 | 332 |
| 333 bool EchoCanceller3::EmptyRenderQueue() { |
| 334 RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_); |
| 335 bool render_buffer_overrun = false; |
| 336 bool frame_to_buffer = |
| 337 render_transfer_queue_.Remove(&render_queue_output_frame_); |
| 338 while (frame_to_buffer) { |
| 339 render_buffer_overrun |= BufferRenderFrameContent( |
| 340 &render_queue_output_frame_, 0, &render_blocker_, |
| 341 block_processor_.get(), &block_, &sub_frame_view_); |
| 342 |
| 343 if (sample_rate_hz_ != 8000) { |
| 344 render_buffer_overrun |= BufferRenderFrameContent( |
| 345 &render_queue_output_frame_, 1, &render_blocker_, |
| 346 block_processor_.get(), &block_, &sub_frame_view_); |
| 347 } |
| 348 |
| 349 render_buffer_overrun |= BufferRemainingRenderFrameContent( |
| 350 &render_blocker_, block_processor_.get(), &block_); |
| 351 |
| 352 frame_to_buffer = |
| 353 render_transfer_queue_.Remove(&render_queue_output_frame_); |
| 354 } |
| 355 return render_buffer_overrun; |
| 356 } |
| 357 |
57 } // namespace webrtc | 358 } // namespace webrtc |
OLD | NEW |