OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include "webrtc/modules/audio_processing/audio_processing_impl.h" | 11 #include "webrtc/modules/audio_processing/audio_processing_impl.h" |
12 | 12 |
13 #include <assert.h> | 13 #include <assert.h> |
14 #include <algorithm> | |
15 | 14 |
16 #include "webrtc/base/checks.h" | 15 #include "webrtc/base/checks.h" |
17 #include "webrtc/base/platform_file.h" | 16 #include "webrtc/base/platform_file.h" |
18 #include "webrtc/common_audio/include/audio_util.h" | 17 #include "webrtc/common_audio/include/audio_util.h" |
19 #include "webrtc/common_audio/channel_buffer.h" | 18 #include "webrtc/common_audio/channel_buffer.h" |
20 #include "webrtc/common_audio/signal_processing/include/signal_processing_librar
y.h" | 19 #include "webrtc/common_audio/signal_processing/include/signal_processing_librar
y.h" |
21 extern "C" { | 20 extern "C" { |
22 #include "webrtc/modules/audio_processing/aec/aec_core.h" | 21 #include "webrtc/modules/audio_processing/aec/aec_core.h" |
23 } | 22 } |
24 #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h" | 23 #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h" |
(...skipping 17 matching lines...) Expand all Loading... |
42 | 41 |
43 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 42 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
44 // Files generated at build-time by the protobuf compiler. | 43 // Files generated at build-time by the protobuf compiler. |
45 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD | 44 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD |
46 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" | 45 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" |
47 #else | 46 #else |
48 #include "webrtc/audio_processing/debug.pb.h" | 47 #include "webrtc/audio_processing/debug.pb.h" |
49 #endif | 48 #endif |
50 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 49 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
51 | 50 |
52 #define RETURN_ON_ERR(expr) \ | 51 #define RETURN_ON_ERR(expr) \ |
53 do { \ | 52 do { \ |
54 int err = (expr); \ | 53 int err = (expr); \ |
55 if (err != kNoError) { \ | 54 if (err != kNoError) { \ |
56 return err; \ | 55 return err; \ |
57 } \ | 56 } \ |
58 } while (0) | 57 } while (0) |
59 | 58 |
60 namespace webrtc { | 59 namespace webrtc { |
61 namespace { | |
62 | |
63 static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) { | |
64 switch (layout) { | |
65 case AudioProcessing::kMono: | |
66 case AudioProcessing::kStereo: | |
67 return false; | |
68 case AudioProcessing::kMonoAndKeyboard: | |
69 case AudioProcessing::kStereoAndKeyboard: | |
70 return true; | |
71 } | |
72 | |
73 assert(false); | |
74 return false; | |
75 } | |
76 | |
77 } // namespace | |
78 | 60 |
79 // Throughout webrtc, it's assumed that success is represented by zero. | 61 // Throughout webrtc, it's assumed that success is represented by zero. |
80 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); | 62 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); |
81 | 63 |
82 // This class has two main functionalities: | 64 // This class has two main functionalities: |
83 // | 65 // |
84 // 1) It is returned instead of the real GainControl after the new AGC has been | 66 // 1) It is returned instead of the real GainControl after the new AGC has been |
85 // enabled in order to prevent an outside user from overriding compression | 67 // enabled in order to prevent an outside user from overriding compression |
86 // settings. It doesn't do anything in its implementation, except for | 68 // settings. It doesn't do anything in its implementation, except for |
87 // delegating the const methods and Enable calls to the real GainControl, so | 69 // delegating the const methods and Enable calls to the real GainControl, so |
88 // AGC can still be disabled. | 70 // AGC can still be disabled. |
89 // | 71 // |
90 // 2) It is injected into AgcManagerDirect and implements volume callbacks for | 72 // 2) It is injected into AgcManagerDirect and implements volume callbacks for |
91 // getting and setting the volume level. It just caches this value to be used | 73 // getting and setting the volume level. It just caches this value to be used |
92 // in VoiceEngine later. | 74 // in VoiceEngine later. |
93 class GainControlForNewAgc : public GainControl, public VolumeCallbacks { | 75 class GainControlForNewAgc : public GainControl, public VolumeCallbacks { |
94 public: | 76 public: |
95 explicit GainControlForNewAgc(GainControlImpl* gain_control) | 77 explicit GainControlForNewAgc(GainControlImpl* gain_control) |
96 : real_gain_control_(gain_control), volume_(0) {} | 78 : real_gain_control_(gain_control), |
| 79 volume_(0) { |
| 80 } |
97 | 81 |
98 // GainControl implementation. | 82 // GainControl implementation. |
99 int Enable(bool enable) override { | 83 int Enable(bool enable) override { |
100 return real_gain_control_->Enable(enable); | 84 return real_gain_control_->Enable(enable); |
101 } | 85 } |
102 bool is_enabled() const override { return real_gain_control_->is_enabled(); } | 86 bool is_enabled() const override { return real_gain_control_->is_enabled(); } |
103 int set_stream_analog_level(int level) override { | 87 int set_stream_analog_level(int level) override { |
104 volume_ = level; | 88 volume_ = level; |
105 return AudioProcessing::kNoError; | 89 return AudioProcessing::kNoError; |
106 } | 90 } |
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
175 gain_control_(NULL), | 159 gain_control_(NULL), |
176 high_pass_filter_(NULL), | 160 high_pass_filter_(NULL), |
177 level_estimator_(NULL), | 161 level_estimator_(NULL), |
178 noise_suppression_(NULL), | 162 noise_suppression_(NULL), |
179 voice_detection_(NULL), | 163 voice_detection_(NULL), |
180 crit_(CriticalSectionWrapper::CreateCriticalSection()), | 164 crit_(CriticalSectionWrapper::CreateCriticalSection()), |
181 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 165 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
182 debug_file_(FileWrapper::Create()), | 166 debug_file_(FileWrapper::Create()), |
183 event_msg_(new audioproc::Event()), | 167 event_msg_(new audioproc::Event()), |
184 #endif | 168 #endif |
185 api_format_({{{kSampleRate16kHz, 1, false}, | 169 fwd_in_format_(kSampleRate16kHz, 1), |
186 {kSampleRate16kHz, 1, false}, | |
187 {kSampleRate16kHz, 1, false}}}), | |
188 fwd_proc_format_(kSampleRate16kHz), | 170 fwd_proc_format_(kSampleRate16kHz), |
| 171 fwd_out_format_(kSampleRate16kHz, 1), |
| 172 rev_in_format_(kSampleRate16kHz, 1), |
189 rev_proc_format_(kSampleRate16kHz, 1), | 173 rev_proc_format_(kSampleRate16kHz, 1), |
190 split_rate_(kSampleRate16kHz), | 174 split_rate_(kSampleRate16kHz), |
191 stream_delay_ms_(0), | 175 stream_delay_ms_(0), |
192 delay_offset_ms_(0), | 176 delay_offset_ms_(0), |
193 was_stream_delay_set_(false), | 177 was_stream_delay_set_(false), |
194 last_stream_delay_ms_(0), | 178 last_stream_delay_ms_(0), |
195 last_aec_system_delay_ms_(0), | 179 last_aec_system_delay_ms_(0), |
196 stream_delay_jumps_(-1), | 180 stream_delay_jumps_(-1), |
197 aec_system_delay_jumps_(-1), | 181 aec_system_delay_jumps_(-1), |
198 output_will_be_muted_(false), | 182 output_will_be_muted_(false), |
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
262 crit_ = NULL; | 246 crit_ = NULL; |
263 } | 247 } |
264 | 248 |
265 int AudioProcessingImpl::Initialize() { | 249 int AudioProcessingImpl::Initialize() { |
266 CriticalSectionScoped crit_scoped(crit_); | 250 CriticalSectionScoped crit_scoped(crit_); |
267 return InitializeLocked(); | 251 return InitializeLocked(); |
268 } | 252 } |
269 | 253 |
270 int AudioProcessingImpl::set_sample_rate_hz(int rate) { | 254 int AudioProcessingImpl::set_sample_rate_hz(int rate) { |
271 CriticalSectionScoped crit_scoped(crit_); | 255 CriticalSectionScoped crit_scoped(crit_); |
272 | 256 return InitializeLocked(rate, |
273 ProcessingConfig processing_config = api_format_; | 257 rate, |
274 processing_config.input_stream().set_sample_rate_hz(rate); | 258 rev_in_format_.rate(), |
275 processing_config.output_stream().set_sample_rate_hz(rate); | 259 fwd_in_format_.num_channels(), |
276 return InitializeLocked(processing_config); | 260 fwd_out_format_.num_channels(), |
| 261 rev_in_format_.num_channels()); |
277 } | 262 } |
278 | 263 |
279 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, | 264 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, |
280 int output_sample_rate_hz, | 265 int output_sample_rate_hz, |
281 int reverse_sample_rate_hz, | 266 int reverse_sample_rate_hz, |
282 ChannelLayout input_layout, | 267 ChannelLayout input_layout, |
283 ChannelLayout output_layout, | 268 ChannelLayout output_layout, |
284 ChannelLayout reverse_layout) { | 269 ChannelLayout reverse_layout) { |
285 const ProcessingConfig processing_config = { | |
286 {{input_sample_rate_hz, ChannelsFromLayout(input_layout), | |
287 LayoutHasKeyboard(input_layout)}, | |
288 {output_sample_rate_hz, ChannelsFromLayout(output_layout), | |
289 LayoutHasKeyboard(output_layout)}, | |
290 {reverse_sample_rate_hz, ChannelsFromLayout(reverse_layout), | |
291 LayoutHasKeyboard(reverse_layout)}}}; | |
292 | |
293 return Initialize(processing_config); | |
294 } | |
295 | |
296 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { | |
297 CriticalSectionScoped crit_scoped(crit_); | 270 CriticalSectionScoped crit_scoped(crit_); |
298 return InitializeLocked(processing_config); | 271 return InitializeLocked(input_sample_rate_hz, |
| 272 output_sample_rate_hz, |
| 273 reverse_sample_rate_hz, |
| 274 ChannelsFromLayout(input_layout), |
| 275 ChannelsFromLayout(output_layout), |
| 276 ChannelsFromLayout(reverse_layout)); |
299 } | 277 } |
300 | 278 |
301 int AudioProcessingImpl::InitializeLocked() { | 279 int AudioProcessingImpl::InitializeLocked() { |
302 const int fwd_audio_buffer_channels = | 280 const int fwd_audio_buffer_channels = beamformer_enabled_ ? |
303 beamformer_enabled_ ? api_format_.input_stream().num_channels() | 281 fwd_in_format_.num_channels() : |
304 : api_format_.output_stream().num_channels(); | 282 fwd_out_format_.num_channels(); |
305 if (api_format_.reverse_stream().num_channels() > 0) { | 283 render_audio_.reset(new AudioBuffer(rev_in_format_.samples_per_channel(), |
306 render_audio_.reset(new AudioBuffer( | 284 rev_in_format_.num_channels(), |
307 api_format_.reverse_stream().num_frames(), | 285 rev_proc_format_.samples_per_channel(), |
308 api_format_.reverse_stream().num_channels(), | 286 rev_proc_format_.num_channels(), |
309 rev_proc_format_.num_frames(), rev_proc_format_.num_channels(), | 287 rev_proc_format_.samples_per_channel())); |
310 rev_proc_format_.num_frames())); | 288 capture_audio_.reset(new AudioBuffer(fwd_in_format_.samples_per_channel(), |
311 } else { | 289 fwd_in_format_.num_channels(), |
312 render_audio_.reset(nullptr); | 290 fwd_proc_format_.samples_per_channel(), |
313 } | 291 fwd_audio_buffer_channels, |
314 capture_audio_.reset(new AudioBuffer( | 292 fwd_out_format_.samples_per_channel())); |
315 api_format_.input_stream().num_frames(), | |
316 api_format_.input_stream().num_channels(), fwd_proc_format_.num_frames(), | |
317 fwd_audio_buffer_channels, api_format_.output_stream().num_frames())); | |
318 | 293 |
319 // Initialize all components. | 294 // Initialize all components. |
320 for (auto item : component_list_) { | 295 for (auto item : component_list_) { |
321 int err = item->Initialize(); | 296 int err = item->Initialize(); |
322 if (err != kNoError) { | 297 if (err != kNoError) { |
323 return err; | 298 return err; |
324 } | 299 } |
325 } | 300 } |
326 | 301 |
327 InitializeExperimentalAgc(); | 302 InitializeExperimentalAgc(); |
328 | 303 |
329 InitializeTransient(); | 304 InitializeTransient(); |
330 | 305 |
331 InitializeBeamformer(); | 306 InitializeBeamformer(); |
332 | 307 |
333 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 308 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
334 if (debug_file_->Open()) { | 309 if (debug_file_->Open()) { |
335 int err = WriteInitMessage(); | 310 int err = WriteInitMessage(); |
336 if (err != kNoError) { | 311 if (err != kNoError) { |
337 return err; | 312 return err; |
338 } | 313 } |
339 } | 314 } |
340 #endif | 315 #endif |
341 | 316 |
342 return kNoError; | 317 return kNoError; |
343 } | 318 } |
344 | 319 |
345 int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { | 320 int AudioProcessingImpl::InitializeLocked(int input_sample_rate_hz, |
346 for (const auto& stream : config.streams) { | 321 int output_sample_rate_hz, |
347 if (stream.num_channels() < 0) { | 322 int reverse_sample_rate_hz, |
348 return kBadNumberChannelsError; | 323 int num_input_channels, |
349 } | 324 int num_output_channels, |
350 if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) { | 325 int num_reverse_channels) { |
351 return kBadSampleRateError; | 326 if (input_sample_rate_hz <= 0 || |
352 } | 327 output_sample_rate_hz <= 0 || |
| 328 reverse_sample_rate_hz <= 0) { |
| 329 return kBadSampleRateError; |
353 } | 330 } |
354 | 331 if (num_output_channels > num_input_channels) { |
355 const int num_in_channels = config.input_stream().num_channels(); | 332 return kBadNumberChannelsError; |
356 const int num_out_channels = config.output_stream().num_channels(); | 333 } |
357 | 334 // Only mono and stereo supported currently. |
358 // Need at least one input channel. | 335 if (num_input_channels > 2 || num_input_channels < 1 || |
359 // Need either one output channel or as many outputs as there are inputs. | 336 num_output_channels > 2 || num_output_channels < 1 || |
360 if (num_in_channels == 0 || | 337 num_reverse_channels > 2 || num_reverse_channels < 1) { |
361 !(num_out_channels == 1 || num_out_channels == num_in_channels)) { | 338 return kBadNumberChannelsError; |
| 339 } |
| 340 if (beamformer_enabled_ && |
| 341 (static_cast<size_t>(num_input_channels) != array_geometry_.size() || |
| 342 num_output_channels > 1)) { |
362 return kBadNumberChannelsError; | 343 return kBadNumberChannelsError; |
363 } | 344 } |
364 | 345 |
365 if (beamformer_enabled_ && | 346 fwd_in_format_.set(input_sample_rate_hz, num_input_channels); |
366 (static_cast<size_t>(num_in_channels) != array_geometry_.size() || | 347 fwd_out_format_.set(output_sample_rate_hz, num_output_channels); |
367 num_out_channels > 1)) { | 348 rev_in_format_.set(reverse_sample_rate_hz, num_reverse_channels); |
368 return kBadNumberChannelsError; | |
369 } | |
370 | |
371 api_format_ = config; | |
372 | 349 |
373 // We process at the closest native rate >= min(input rate, output rate)... | 350 // We process at the closest native rate >= min(input rate, output rate)... |
374 const int min_proc_rate = | 351 int min_proc_rate = std::min(fwd_in_format_.rate(), fwd_out_format_.rate()); |
375 std::min(api_format_.input_stream().sample_rate_hz(), | |
376 api_format_.output_stream().sample_rate_hz()); | |
377 int fwd_proc_rate; | 352 int fwd_proc_rate; |
378 if (supports_48kHz_ && min_proc_rate > kSampleRate32kHz) { | 353 if (supports_48kHz_ && min_proc_rate > kSampleRate32kHz) { |
379 fwd_proc_rate = kSampleRate48kHz; | 354 fwd_proc_rate = kSampleRate48kHz; |
380 } else if (min_proc_rate > kSampleRate16kHz) { | 355 } else if (min_proc_rate > kSampleRate16kHz) { |
381 fwd_proc_rate = kSampleRate32kHz; | 356 fwd_proc_rate = kSampleRate32kHz; |
382 } else if (min_proc_rate > kSampleRate8kHz) { | 357 } else if (min_proc_rate > kSampleRate8kHz) { |
383 fwd_proc_rate = kSampleRate16kHz; | 358 fwd_proc_rate = kSampleRate16kHz; |
384 } else { | 359 } else { |
385 fwd_proc_rate = kSampleRate8kHz; | 360 fwd_proc_rate = kSampleRate8kHz; |
386 } | 361 } |
387 // ...with one exception. | 362 // ...with one exception. |
388 if (echo_control_mobile_->is_enabled() && min_proc_rate > kSampleRate16kHz) { | 363 if (echo_control_mobile_->is_enabled() && min_proc_rate > kSampleRate16kHz) { |
389 fwd_proc_rate = kSampleRate16kHz; | 364 fwd_proc_rate = kSampleRate16kHz; |
390 } | 365 } |
391 | 366 |
392 fwd_proc_format_ = StreamConfig(fwd_proc_rate); | 367 fwd_proc_format_.set(fwd_proc_rate); |
393 | 368 |
394 // We normally process the reverse stream at 16 kHz. Unless... | 369 // We normally process the reverse stream at 16 kHz. Unless... |
395 int rev_proc_rate = kSampleRate16kHz; | 370 int rev_proc_rate = kSampleRate16kHz; |
396 if (fwd_proc_format_.sample_rate_hz() == kSampleRate8kHz) { | 371 if (fwd_proc_format_.rate() == kSampleRate8kHz) { |
397 // ...the forward stream is at 8 kHz. | 372 // ...the forward stream is at 8 kHz. |
398 rev_proc_rate = kSampleRate8kHz; | 373 rev_proc_rate = kSampleRate8kHz; |
399 } else { | 374 } else { |
400 if (api_format_.reverse_stream().sample_rate_hz() == kSampleRate32kHz) { | 375 if (rev_in_format_.rate() == kSampleRate32kHz) { |
401 // ...or the input is at 32 kHz, in which case we use the splitting | 376 // ...or the input is at 32 kHz, in which case we use the splitting |
402 // filter rather than the resampler. | 377 // filter rather than the resampler. |
403 rev_proc_rate = kSampleRate32kHz; | 378 rev_proc_rate = kSampleRate32kHz; |
404 } | 379 } |
405 } | 380 } |
406 | 381 |
407 // Always downmix the reverse stream to mono for analysis. This has been | 382 // Always downmix the reverse stream to mono for analysis. This has been |
408 // demonstrated to work well for AEC in most practical scenarios. | 383 // demonstrated to work well for AEC in most practical scenarios. |
409 rev_proc_format_ = StreamConfig(rev_proc_rate, 1); | 384 rev_proc_format_.set(rev_proc_rate, 1); |
410 | 385 |
411 if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || | 386 if (fwd_proc_format_.rate() == kSampleRate32kHz || |
412 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { | 387 fwd_proc_format_.rate() == kSampleRate48kHz) { |
413 split_rate_ = kSampleRate16kHz; | 388 split_rate_ = kSampleRate16kHz; |
414 } else { | 389 } else { |
415 split_rate_ = fwd_proc_format_.sample_rate_hz(); | 390 split_rate_ = fwd_proc_format_.rate(); |
416 } | 391 } |
417 | 392 |
418 return InitializeLocked(); | 393 return InitializeLocked(); |
419 } | 394 } |
420 | 395 |
421 // Calls InitializeLocked() if any of the audio parameters have changed from | 396 // Calls InitializeLocked() if any of the audio parameters have changed from |
422 // their current values. | 397 // their current values. |
423 int AudioProcessingImpl::MaybeInitializeLocked( | 398 int AudioProcessingImpl::MaybeInitializeLocked(int input_sample_rate_hz, |
424 const ProcessingConfig& processing_config) { | 399 int output_sample_rate_hz, |
425 if (processing_config == api_format_) { | 400 int reverse_sample_rate_hz, |
| 401 int num_input_channels, |
| 402 int num_output_channels, |
| 403 int num_reverse_channels) { |
| 404 if (input_sample_rate_hz == fwd_in_format_.rate() && |
| 405 output_sample_rate_hz == fwd_out_format_.rate() && |
| 406 reverse_sample_rate_hz == rev_in_format_.rate() && |
| 407 num_input_channels == fwd_in_format_.num_channels() && |
| 408 num_output_channels == fwd_out_format_.num_channels() && |
| 409 num_reverse_channels == rev_in_format_.num_channels()) { |
426 return kNoError; | 410 return kNoError; |
427 } | 411 } |
428 return InitializeLocked(processing_config); | 412 return InitializeLocked(input_sample_rate_hz, |
| 413 output_sample_rate_hz, |
| 414 reverse_sample_rate_hz, |
| 415 num_input_channels, |
| 416 num_output_channels, |
| 417 num_reverse_channels); |
429 } | 418 } |
430 | 419 |
431 void AudioProcessingImpl::SetExtraOptions(const Config& config) { | 420 void AudioProcessingImpl::SetExtraOptions(const Config& config) { |
432 CriticalSectionScoped crit_scoped(crit_); | 421 CriticalSectionScoped crit_scoped(crit_); |
433 for (auto item : component_list_) { | 422 for (auto item : component_list_) { |
434 item->SetExtraOptions(config); | 423 item->SetExtraOptions(config); |
435 } | 424 } |
436 | 425 |
437 if (transient_suppressor_enabled_ != config.Get<ExperimentalNs>().enabled) { | 426 if (transient_suppressor_enabled_ != config.Get<ExperimentalNs>().enabled) { |
438 transient_suppressor_enabled_ = config.Get<ExperimentalNs>().enabled; | 427 transient_suppressor_enabled_ = config.Get<ExperimentalNs>().enabled; |
439 InitializeTransient(); | 428 InitializeTransient(); |
440 } | 429 } |
441 } | 430 } |
442 | 431 |
443 int AudioProcessingImpl::input_sample_rate_hz() const { | 432 int AudioProcessingImpl::input_sample_rate_hz() const { |
444 CriticalSectionScoped crit_scoped(crit_); | 433 CriticalSectionScoped crit_scoped(crit_); |
445 return api_format_.input_stream().sample_rate_hz(); | 434 return fwd_in_format_.rate(); |
446 } | 435 } |
447 | 436 |
448 int AudioProcessingImpl::sample_rate_hz() const { | 437 int AudioProcessingImpl::sample_rate_hz() const { |
449 CriticalSectionScoped crit_scoped(crit_); | 438 CriticalSectionScoped crit_scoped(crit_); |
450 return api_format_.input_stream().sample_rate_hz(); | 439 return fwd_in_format_.rate(); |
451 } | 440 } |
452 | 441 |
453 int AudioProcessingImpl::proc_sample_rate_hz() const { | 442 int AudioProcessingImpl::proc_sample_rate_hz() const { |
454 return fwd_proc_format_.sample_rate_hz(); | 443 return fwd_proc_format_.rate(); |
455 } | 444 } |
456 | 445 |
457 int AudioProcessingImpl::proc_split_sample_rate_hz() const { | 446 int AudioProcessingImpl::proc_split_sample_rate_hz() const { |
458 return split_rate_; | 447 return split_rate_; |
459 } | 448 } |
460 | 449 |
461 int AudioProcessingImpl::num_reverse_channels() const { | 450 int AudioProcessingImpl::num_reverse_channels() const { |
462 return rev_proc_format_.num_channels(); | 451 return rev_proc_format_.num_channels(); |
463 } | 452 } |
464 | 453 |
465 int AudioProcessingImpl::num_input_channels() const { | 454 int AudioProcessingImpl::num_input_channels() const { |
466 return api_format_.input_stream().num_channels(); | 455 return fwd_in_format_.num_channels(); |
467 } | 456 } |
468 | 457 |
469 int AudioProcessingImpl::num_output_channels() const { | 458 int AudioProcessingImpl::num_output_channels() const { |
470 return api_format_.output_stream().num_channels(); | 459 return fwd_out_format_.num_channels(); |
471 } | 460 } |
472 | 461 |
473 void AudioProcessingImpl::set_output_will_be_muted(bool muted) { | 462 void AudioProcessingImpl::set_output_will_be_muted(bool muted) { |
474 CriticalSectionScoped lock(crit_); | 463 CriticalSectionScoped lock(crit_); |
475 output_will_be_muted_ = muted; | 464 output_will_be_muted_ = muted; |
476 if (agc_manager_.get()) { | 465 if (agc_manager_.get()) { |
477 agc_manager_->SetCaptureMuted(output_will_be_muted_); | 466 agc_manager_->SetCaptureMuted(output_will_be_muted_); |
478 } | 467 } |
479 } | 468 } |
480 | 469 |
481 bool AudioProcessingImpl::output_will_be_muted() const { | 470 bool AudioProcessingImpl::output_will_be_muted() const { |
482 CriticalSectionScoped lock(crit_); | 471 CriticalSectionScoped lock(crit_); |
483 return output_will_be_muted_; | 472 return output_will_be_muted_; |
484 } | 473 } |
485 | 474 |
486 int AudioProcessingImpl::ProcessStream(const float* const* src, | 475 int AudioProcessingImpl::ProcessStream(const float* const* src, |
487 int samples_per_channel, | 476 int samples_per_channel, |
488 int input_sample_rate_hz, | 477 int input_sample_rate_hz, |
489 ChannelLayout input_layout, | 478 ChannelLayout input_layout, |
490 int output_sample_rate_hz, | 479 int output_sample_rate_hz, |
491 ChannelLayout output_layout, | 480 ChannelLayout output_layout, |
492 float* const* dest) { | 481 float* const* dest) { |
493 StreamConfig input_stream = api_format_.input_stream(); | |
494 input_stream.set_sample_rate_hz(input_sample_rate_hz); | |
495 input_stream.set_num_channels(ChannelsFromLayout(input_layout)); | |
496 input_stream.set_has_keyboard(LayoutHasKeyboard(input_layout)); | |
497 | |
498 StreamConfig output_stream = api_format_.output_stream(); | |
499 output_stream.set_sample_rate_hz(output_sample_rate_hz); | |
500 output_stream.set_num_channels(ChannelsFromLayout(output_layout)); | |
501 output_stream.set_has_keyboard(LayoutHasKeyboard(output_layout)); | |
502 | |
503 if (samples_per_channel != input_stream.num_frames()) { | |
504 return kBadDataLengthError; | |
505 } | |
506 return ProcessStream(src, input_stream, output_stream, dest); | |
507 } | |
508 | |
509 int AudioProcessingImpl::ProcessStream(const float* const* src, | |
510 const StreamConfig& input_config, | |
511 const StreamConfig& output_config, | |
512 float* const* dest) { | |
513 CriticalSectionScoped crit_scoped(crit_); | 482 CriticalSectionScoped crit_scoped(crit_); |
514 if (!src || !dest) { | 483 if (!src || !dest) { |
515 return kNullPointerError; | 484 return kNullPointerError; |
516 } | 485 } |
517 | 486 |
518 ProcessingConfig processing_config = api_format_; | 487 RETURN_ON_ERR(MaybeInitializeLocked(input_sample_rate_hz, |
519 processing_config.input_stream() = input_config; | 488 output_sample_rate_hz, |
520 processing_config.output_stream() = output_config; | 489 rev_in_format_.rate(), |
521 | 490 ChannelsFromLayout(input_layout), |
522 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | 491 ChannelsFromLayout(output_layout), |
523 assert(processing_config.input_stream().num_frames() == | 492 rev_in_format_.num_channels())); |
524 api_format_.input_stream().num_frames()); | 493 if (samples_per_channel != fwd_in_format_.samples_per_channel()) { |
| 494 return kBadDataLengthError; |
| 495 } |
525 | 496 |
526 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 497 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
527 if (debug_file_->Open()) { | 498 if (debug_file_->Open()) { |
528 event_msg_->set_type(audioproc::Event::STREAM); | 499 event_msg_->set_type(audioproc::Event::STREAM); |
529 audioproc::Stream* msg = event_msg_->mutable_stream(); | 500 audioproc::Stream* msg = event_msg_->mutable_stream(); |
530 const size_t channel_size = | 501 const size_t channel_size = |
531 sizeof(float) * api_format_.input_stream().num_frames(); | 502 sizeof(float) * fwd_in_format_.samples_per_channel(); |
532 for (int i = 0; i < api_format_.input_stream().num_channels(); ++i) | 503 for (int i = 0; i < fwd_in_format_.num_channels(); ++i) |
533 msg->add_input_channel(src[i], channel_size); | 504 msg->add_input_channel(src[i], channel_size); |
534 } | 505 } |
535 #endif | 506 #endif |
536 | 507 |
537 capture_audio_->CopyFrom(src, api_format_.input_stream()); | 508 capture_audio_->CopyFrom(src, samples_per_channel, input_layout); |
538 RETURN_ON_ERR(ProcessStreamLocked()); | 509 RETURN_ON_ERR(ProcessStreamLocked()); |
539 capture_audio_->CopyTo(api_format_.output_stream(), dest); | 510 capture_audio_->CopyTo(fwd_out_format_.samples_per_channel(), |
| 511 output_layout, |
| 512 dest); |
540 | 513 |
541 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 514 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
542 if (debug_file_->Open()) { | 515 if (debug_file_->Open()) { |
543 audioproc::Stream* msg = event_msg_->mutable_stream(); | 516 audioproc::Stream* msg = event_msg_->mutable_stream(); |
544 const size_t channel_size = | 517 const size_t channel_size = |
545 sizeof(float) * api_format_.input_stream().num_frames(); | 518 sizeof(float) * fwd_out_format_.samples_per_channel(); |
546 for (int i = 0; i < api_format_.input_stream().num_channels(); ++i) | 519 for (int i = 0; i < fwd_out_format_.num_channels(); ++i) |
547 msg->add_output_channel(dest[i], channel_size); | 520 msg->add_output_channel(dest[i], channel_size); |
548 RETURN_ON_ERR(WriteMessageToDebugFile()); | 521 RETURN_ON_ERR(WriteMessageToDebugFile()); |
549 } | 522 } |
550 #endif | 523 #endif |
551 | 524 |
552 return kNoError; | 525 return kNoError; |
553 } | 526 } |
554 | 527 |
555 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { | 528 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { |
556 CriticalSectionScoped crit_scoped(crit_); | 529 CriticalSectionScoped crit_scoped(crit_); |
557 if (!frame) { | 530 if (!frame) { |
558 return kNullPointerError; | 531 return kNullPointerError; |
559 } | 532 } |
560 // Must be a native rate. | 533 // Must be a native rate. |
561 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 534 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
562 frame->sample_rate_hz_ != kSampleRate16kHz && | 535 frame->sample_rate_hz_ != kSampleRate16kHz && |
563 frame->sample_rate_hz_ != kSampleRate32kHz && | 536 frame->sample_rate_hz_ != kSampleRate32kHz && |
564 frame->sample_rate_hz_ != kSampleRate48kHz) { | 537 frame->sample_rate_hz_ != kSampleRate48kHz) { |
565 return kBadSampleRateError; | 538 return kBadSampleRateError; |
566 } | 539 } |
567 if (echo_control_mobile_->is_enabled() && | 540 if (echo_control_mobile_->is_enabled() && |
568 frame->sample_rate_hz_ > kSampleRate16kHz) { | 541 frame->sample_rate_hz_ > kSampleRate16kHz) { |
569 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates"; | 542 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates"; |
570 return kUnsupportedComponentError; | 543 return kUnsupportedComponentError; |
571 } | 544 } |
572 | 545 |
573 // TODO(ajm): The input and output rates and channels are currently | 546 // TODO(ajm): The input and output rates and channels are currently |
574 // constrained to be identical in the int16 interface. | 547 // constrained to be identical in the int16 interface. |
575 ProcessingConfig processing_config = api_format_; | 548 RETURN_ON_ERR(MaybeInitializeLocked(frame->sample_rate_hz_, |
576 processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_); | 549 frame->sample_rate_hz_, |
577 processing_config.input_stream().set_num_channels(frame->num_channels_); | 550 rev_in_format_.rate(), |
578 processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_); | 551 frame->num_channels_, |
579 processing_config.output_stream().set_num_channels(frame->num_channels_); | 552 frame->num_channels_, |
580 | 553 rev_in_format_.num_channels())); |
581 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | 554 if (frame->samples_per_channel_ != fwd_in_format_.samples_per_channel()) { |
582 if (frame->samples_per_channel_ != api_format_.input_stream().num_frames()) { | |
583 return kBadDataLengthError; | 555 return kBadDataLengthError; |
584 } | 556 } |
585 | 557 |
586 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 558 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
587 if (debug_file_->Open()) { | 559 if (debug_file_->Open()) { |
588 event_msg_->set_type(audioproc::Event::STREAM); | 560 event_msg_->set_type(audioproc::Event::STREAM); |
589 audioproc::Stream* msg = event_msg_->mutable_stream(); | 561 audioproc::Stream* msg = event_msg_->mutable_stream(); |
590 const size_t data_size = | 562 const size_t data_size = sizeof(int16_t) * |
591 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 563 frame->samples_per_channel_ * |
| 564 frame->num_channels_; |
592 msg->set_input_data(frame->data_, data_size); | 565 msg->set_input_data(frame->data_, data_size); |
593 } | 566 } |
594 #endif | 567 #endif |
595 | 568 |
596 capture_audio_->DeinterleaveFrom(frame); | 569 capture_audio_->DeinterleaveFrom(frame); |
597 RETURN_ON_ERR(ProcessStreamLocked()); | 570 RETURN_ON_ERR(ProcessStreamLocked()); |
598 capture_audio_->InterleaveTo(frame, output_copy_needed(is_data_processed())); | 571 capture_audio_->InterleaveTo(frame, output_copy_needed(is_data_processed())); |
599 | 572 |
600 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 573 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
601 if (debug_file_->Open()) { | 574 if (debug_file_->Open()) { |
602 audioproc::Stream* msg = event_msg_->mutable_stream(); | 575 audioproc::Stream* msg = event_msg_->mutable_stream(); |
603 const size_t data_size = | 576 const size_t data_size = sizeof(int16_t) * |
604 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 577 frame->samples_per_channel_ * |
| 578 frame->num_channels_; |
605 msg->set_output_data(frame->data_, data_size); | 579 msg->set_output_data(frame->data_, data_size); |
606 RETURN_ON_ERR(WriteMessageToDebugFile()); | 580 RETURN_ON_ERR(WriteMessageToDebugFile()); |
607 } | 581 } |
608 #endif | 582 #endif |
609 | 583 |
610 return kNoError; | 584 return kNoError; |
611 } | 585 } |
612 | 586 |
| 587 |
613 int AudioProcessingImpl::ProcessStreamLocked() { | 588 int AudioProcessingImpl::ProcessStreamLocked() { |
614 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 589 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
615 if (debug_file_->Open()) { | 590 if (debug_file_->Open()) { |
616 audioproc::Stream* msg = event_msg_->mutable_stream(); | 591 audioproc::Stream* msg = event_msg_->mutable_stream(); |
617 msg->set_delay(stream_delay_ms_); | 592 msg->set_delay(stream_delay_ms_); |
618 msg->set_drift(echo_cancellation_->stream_drift_samples()); | 593 msg->set_drift(echo_cancellation_->stream_drift_samples()); |
619 msg->set_level(gain_control()->stream_analog_level()); | 594 msg->set_level(gain_control()->stream_analog_level()); |
620 msg->set_keypress(key_pressed_); | 595 msg->set_keypress(key_pressed_); |
621 } | 596 } |
622 #endif | 597 #endif |
623 | 598 |
624 MaybeUpdateHistograms(); | 599 MaybeUpdateHistograms(); |
625 | 600 |
626 AudioBuffer* ca = capture_audio_.get(); // For brevity. | 601 AudioBuffer* ca = capture_audio_.get(); // For brevity. |
627 if (use_new_agc_ && gain_control_->is_enabled()) { | 602 if (use_new_agc_ && gain_control_->is_enabled()) { |
628 agc_manager_->AnalyzePreProcess(ca->channels()[0], ca->num_channels(), | 603 agc_manager_->AnalyzePreProcess(ca->channels()[0], |
629 fwd_proc_format_.num_frames()); | 604 ca->num_channels(), |
| 605 fwd_proc_format_.samples_per_channel()); |
630 } | 606 } |
631 | 607 |
632 bool data_processed = is_data_processed(); | 608 bool data_processed = is_data_processed(); |
633 if (analysis_needed(data_processed)) { | 609 if (analysis_needed(data_processed)) { |
634 ca->SplitIntoFrequencyBands(); | 610 ca->SplitIntoFrequencyBands(); |
635 } | 611 } |
636 | 612 |
637 if (beamformer_enabled_) { | 613 if (beamformer_enabled_) { |
638 beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f()); | 614 beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f()); |
639 ca->set_num_channels(1); | 615 ca->set_num_channels(1); |
640 } | 616 } |
641 | 617 |
642 RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca)); | 618 RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca)); |
643 RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca)); | 619 RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca)); |
644 RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca)); | 620 RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca)); |
645 RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca)); | 621 RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca)); |
646 | 622 |
647 if (echo_control_mobile_->is_enabled() && noise_suppression_->is_enabled()) { | 623 if (echo_control_mobile_->is_enabled() && noise_suppression_->is_enabled()) { |
648 ca->CopyLowPassToReference(); | 624 ca->CopyLowPassToReference(); |
649 } | 625 } |
650 RETURN_ON_ERR(noise_suppression_->ProcessCaptureAudio(ca)); | 626 RETURN_ON_ERR(noise_suppression_->ProcessCaptureAudio(ca)); |
651 RETURN_ON_ERR(echo_control_mobile_->ProcessCaptureAudio(ca)); | 627 RETURN_ON_ERR(echo_control_mobile_->ProcessCaptureAudio(ca)); |
652 RETURN_ON_ERR(voice_detection_->ProcessCaptureAudio(ca)); | 628 RETURN_ON_ERR(voice_detection_->ProcessCaptureAudio(ca)); |
653 | 629 |
654 if (use_new_agc_ && gain_control_->is_enabled() && | 630 if (use_new_agc_ && |
| 631 gain_control_->is_enabled() && |
655 (!beamformer_enabled_ || beamformer_->is_target_present())) { | 632 (!beamformer_enabled_ || beamformer_->is_target_present())) { |
656 agc_manager_->Process(ca->split_bands_const(0)[kBand0To8kHz], | 633 agc_manager_->Process(ca->split_bands_const(0)[kBand0To8kHz], |
657 ca->num_frames_per_band(), split_rate_); | 634 ca->num_frames_per_band(), |
| 635 split_rate_); |
658 } | 636 } |
659 RETURN_ON_ERR(gain_control_->ProcessCaptureAudio(ca)); | 637 RETURN_ON_ERR(gain_control_->ProcessCaptureAudio(ca)); |
660 | 638 |
661 if (synthesis_needed(data_processed)) { | 639 if (synthesis_needed(data_processed)) { |
662 ca->MergeFrequencyBands(); | 640 ca->MergeFrequencyBands(); |
663 } | 641 } |
664 | 642 |
665 // TODO(aluebs): Investigate if the transient suppression placement should be | 643 // TODO(aluebs): Investigate if the transient suppression placement should be |
666 // before or after the AGC. | 644 // before or after the AGC. |
667 if (transient_suppressor_enabled_) { | 645 if (transient_suppressor_enabled_) { |
668 float voice_probability = | 646 float voice_probability = |
669 agc_manager_.get() ? agc_manager_->voice_probability() : 1.f; | 647 agc_manager_.get() ? agc_manager_->voice_probability() : 1.f; |
670 | 648 |
671 transient_suppressor_->Suppress( | 649 transient_suppressor_->Suppress(ca->channels_f()[0], |
672 ca->channels_f()[0], ca->num_frames(), ca->num_channels(), | 650 ca->num_frames(), |
673 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(), | 651 ca->num_channels(), |
674 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability, | 652 ca->split_bands_const_f(0)[kBand0To8kHz], |
675 key_pressed_); | 653 ca->num_frames_per_band(), |
| 654 ca->keyboard_data(), |
| 655 ca->num_keyboard_frames(), |
| 656 voice_probability, |
| 657 key_pressed_); |
676 } | 658 } |
677 | 659 |
678 // The level estimator operates on the recombined data. | 660 // The level estimator operates on the recombined data. |
679 RETURN_ON_ERR(level_estimator_->ProcessStream(ca)); | 661 RETURN_ON_ERR(level_estimator_->ProcessStream(ca)); |
680 | 662 |
681 was_stream_delay_set_ = false; | 663 was_stream_delay_set_ = false; |
682 return kNoError; | 664 return kNoError; |
683 } | 665 } |
684 | 666 |
685 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, | 667 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, |
686 int samples_per_channel, | 668 int samples_per_channel, |
687 int sample_rate_hz, | 669 int sample_rate_hz, |
688 ChannelLayout layout) { | 670 ChannelLayout layout) { |
689 const StreamConfig reverse_config = { | |
690 sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), | |
691 }; | |
692 if (samples_per_channel != reverse_config.num_frames()) { | |
693 return kBadDataLengthError; | |
694 } | |
695 return AnalyzeReverseStream(data, reverse_config); | |
696 } | |
697 | |
698 int AudioProcessingImpl::AnalyzeReverseStream( | |
699 const float* const* data, | |
700 const StreamConfig& reverse_config) { | |
701 CriticalSectionScoped crit_scoped(crit_); | 671 CriticalSectionScoped crit_scoped(crit_); |
702 if (data == NULL) { | 672 if (data == NULL) { |
703 return kNullPointerError; | 673 return kNullPointerError; |
704 } | 674 } |
705 | 675 |
706 if (reverse_config.num_channels() <= 0) { | 676 const int num_channels = ChannelsFromLayout(layout); |
707 return kBadNumberChannelsError; | 677 RETURN_ON_ERR(MaybeInitializeLocked(fwd_in_format_.rate(), |
| 678 fwd_out_format_.rate(), |
| 679 sample_rate_hz, |
| 680 fwd_in_format_.num_channels(), |
| 681 fwd_out_format_.num_channels(), |
| 682 num_channels)); |
| 683 if (samples_per_channel != rev_in_format_.samples_per_channel()) { |
| 684 return kBadDataLengthError; |
708 } | 685 } |
709 | 686 |
710 ProcessingConfig processing_config = api_format_; | |
711 processing_config.reverse_stream() = reverse_config; | |
712 | |
713 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | |
714 assert(reverse_config.num_frames() == | |
715 api_format_.reverse_stream().num_frames()); | |
716 | |
717 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 687 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
718 if (debug_file_->Open()) { | 688 if (debug_file_->Open()) { |
719 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); | 689 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); |
720 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); | 690 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); |
721 const size_t channel_size = | 691 const size_t channel_size = |
722 sizeof(float) * api_format_.reverse_stream().num_frames(); | 692 sizeof(float) * rev_in_format_.samples_per_channel(); |
723 for (int i = 0; i < api_format_.reverse_stream().num_channels(); ++i) | 693 for (int i = 0; i < num_channels; ++i) |
724 msg->add_channel(data[i], channel_size); | 694 msg->add_channel(data[i], channel_size); |
725 RETURN_ON_ERR(WriteMessageToDebugFile()); | 695 RETURN_ON_ERR(WriteMessageToDebugFile()); |
726 } | 696 } |
727 #endif | 697 #endif |
728 | 698 |
729 render_audio_->CopyFrom(data, api_format_.reverse_stream()); | 699 render_audio_->CopyFrom(data, samples_per_channel, layout); |
730 return AnalyzeReverseStreamLocked(); | 700 return AnalyzeReverseStreamLocked(); |
731 } | 701 } |
732 | 702 |
733 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { | 703 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { |
734 CriticalSectionScoped crit_scoped(crit_); | 704 CriticalSectionScoped crit_scoped(crit_); |
735 if (frame == NULL) { | 705 if (frame == NULL) { |
736 return kNullPointerError; | 706 return kNullPointerError; |
737 } | 707 } |
738 // Must be a native rate. | 708 // Must be a native rate. |
739 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 709 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
740 frame->sample_rate_hz_ != kSampleRate16kHz && | 710 frame->sample_rate_hz_ != kSampleRate16kHz && |
741 frame->sample_rate_hz_ != kSampleRate32kHz && | 711 frame->sample_rate_hz_ != kSampleRate32kHz && |
742 frame->sample_rate_hz_ != kSampleRate48kHz) { | 712 frame->sample_rate_hz_ != kSampleRate48kHz) { |
743 return kBadSampleRateError; | 713 return kBadSampleRateError; |
744 } | 714 } |
745 // This interface does not tolerate different forward and reverse rates. | 715 // This interface does not tolerate different forward and reverse rates. |
746 if (frame->sample_rate_hz_ != api_format_.input_stream().sample_rate_hz()) { | 716 if (frame->sample_rate_hz_ != fwd_in_format_.rate()) { |
747 return kBadSampleRateError; | 717 return kBadSampleRateError; |
748 } | 718 } |
749 | 719 |
750 if (frame->num_channels_ <= 0) { | 720 RETURN_ON_ERR(MaybeInitializeLocked(fwd_in_format_.rate(), |
751 return kBadNumberChannelsError; | 721 fwd_out_format_.rate(), |
752 } | 722 frame->sample_rate_hz_, |
753 | 723 fwd_in_format_.num_channels(), |
754 ProcessingConfig processing_config = api_format_; | 724 fwd_in_format_.num_channels(), |
755 processing_config.reverse_stream().set_sample_rate_hz(frame->sample_rate_hz_); | 725 frame->num_channels_)); |
756 processing_config.reverse_stream().set_num_channels(frame->num_channels_); | 726 if (frame->samples_per_channel_ != rev_in_format_.samples_per_channel()) { |
757 | |
758 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | |
759 if (frame->samples_per_channel_ != | |
760 api_format_.reverse_stream().num_frames()) { | |
761 return kBadDataLengthError; | 727 return kBadDataLengthError; |
762 } | 728 } |
763 | 729 |
764 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 730 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
765 if (debug_file_->Open()) { | 731 if (debug_file_->Open()) { |
766 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); | 732 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); |
767 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); | 733 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); |
768 const size_t data_size = | 734 const size_t data_size = sizeof(int16_t) * |
769 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 735 frame->samples_per_channel_ * |
| 736 frame->num_channels_; |
770 msg->set_data(frame->data_, data_size); | 737 msg->set_data(frame->data_, data_size); |
771 RETURN_ON_ERR(WriteMessageToDebugFile()); | 738 RETURN_ON_ERR(WriteMessageToDebugFile()); |
772 } | 739 } |
773 #endif | 740 #endif |
774 | 741 |
775 render_audio_->DeinterleaveFrom(frame); | 742 render_audio_->DeinterleaveFrom(frame); |
776 return AnalyzeReverseStreamLocked(); | 743 return AnalyzeReverseStreamLocked(); |
777 } | 744 } |
778 | 745 |
779 int AudioProcessingImpl::AnalyzeReverseStreamLocked() { | 746 int AudioProcessingImpl::AnalyzeReverseStreamLocked() { |
780 AudioBuffer* ra = render_audio_.get(); // For brevity. | 747 AudioBuffer* ra = render_audio_.get(); // For brevity. |
781 if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz) { | 748 if (rev_proc_format_.rate() == kSampleRate32kHz) { |
782 ra->SplitIntoFrequencyBands(); | 749 ra->SplitIntoFrequencyBands(); |
783 } | 750 } |
784 | 751 |
785 RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra)); | 752 RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra)); |
786 RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra)); | 753 RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra)); |
787 if (!use_new_agc_) { | 754 if (!use_new_agc_) { |
788 RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra)); | 755 RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra)); |
789 } | 756 } |
790 | 757 |
791 return kNoError; | 758 return kNoError; |
(...skipping 181 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
973 } else if (enabled_count == 2) { | 940 } else if (enabled_count == 2) { |
974 if (level_estimator_->is_enabled() && voice_detection_->is_enabled()) { | 941 if (level_estimator_->is_enabled() && voice_detection_->is_enabled()) { |
975 return false; | 942 return false; |
976 } | 943 } |
977 } | 944 } |
978 return true; | 945 return true; |
979 } | 946 } |
980 | 947 |
981 bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const { | 948 bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const { |
982 // Check if we've upmixed or downmixed the audio. | 949 // Check if we've upmixed or downmixed the audio. |
983 return ((api_format_.output_stream().num_channels() != | 950 return ((fwd_out_format_.num_channels() != fwd_in_format_.num_channels()) || |
984 api_format_.input_stream().num_channels()) || | |
985 is_data_processed || transient_suppressor_enabled_); | 951 is_data_processed || transient_suppressor_enabled_); |
986 } | 952 } |
987 | 953 |
988 bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const { | 954 bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const { |
989 return (is_data_processed && | 955 return (is_data_processed && (fwd_proc_format_.rate() == kSampleRate32kHz || |
990 (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || | 956 fwd_proc_format_.rate() == kSampleRate48kHz)); |
991 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz)); | |
992 } | 957 } |
993 | 958 |
994 bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const { | 959 bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const { |
995 if (!is_data_processed && !voice_detection_->is_enabled() && | 960 if (!is_data_processed && !voice_detection_->is_enabled() && |
996 !transient_suppressor_enabled_) { | 961 !transient_suppressor_enabled_) { |
997 // Only level_estimator_ is enabled. | 962 // Only level_estimator_ is enabled. |
998 return false; | 963 return false; |
999 } else if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || | 964 } else if (fwd_proc_format_.rate() == kSampleRate32kHz || |
1000 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { | 965 fwd_proc_format_.rate() == kSampleRate48kHz) { |
1001 // Something besides level_estimator_ is enabled, and we have super-wb. | 966 // Something besides level_estimator_ is enabled, and we have super-wb. |
1002 return true; | 967 return true; |
1003 } | 968 } |
1004 return false; | 969 return false; |
1005 } | 970 } |
1006 | 971 |
1007 void AudioProcessingImpl::InitializeExperimentalAgc() { | 972 void AudioProcessingImpl::InitializeExperimentalAgc() { |
1008 if (use_new_agc_) { | 973 if (use_new_agc_) { |
1009 if (!agc_manager_.get()) { | 974 if (!agc_manager_.get()) { |
1010 agc_manager_.reset(new AgcManagerDirect(gain_control_, | 975 agc_manager_.reset(new AgcManagerDirect(gain_control_, |
1011 gain_control_for_new_agc_.get(), | 976 gain_control_for_new_agc_.get(), |
1012 agc_startup_min_volume_)); | 977 agc_startup_min_volume_)); |
1013 } | 978 } |
1014 agc_manager_->Initialize(); | 979 agc_manager_->Initialize(); |
1015 agc_manager_->SetCaptureMuted(output_will_be_muted_); | 980 agc_manager_->SetCaptureMuted(output_will_be_muted_); |
1016 } | 981 } |
1017 } | 982 } |
1018 | 983 |
1019 void AudioProcessingImpl::InitializeTransient() { | 984 void AudioProcessingImpl::InitializeTransient() { |
1020 if (transient_suppressor_enabled_) { | 985 if (transient_suppressor_enabled_) { |
1021 if (!transient_suppressor_.get()) { | 986 if (!transient_suppressor_.get()) { |
1022 transient_suppressor_.reset(new TransientSuppressor()); | 987 transient_suppressor_.reset(new TransientSuppressor()); |
1023 } | 988 } |
1024 transient_suppressor_->Initialize( | 989 transient_suppressor_->Initialize(fwd_proc_format_.rate(), |
1025 fwd_proc_format_.sample_rate_hz(), split_rate_, | 990 split_rate_, |
1026 api_format_.output_stream().num_channels()); | 991 fwd_out_format_.num_channels()); |
1027 } | 992 } |
1028 } | 993 } |
1029 | 994 |
1030 void AudioProcessingImpl::InitializeBeamformer() { | 995 void AudioProcessingImpl::InitializeBeamformer() { |
1031 if (beamformer_enabled_) { | 996 if (beamformer_enabled_) { |
1032 if (!beamformer_) { | 997 if (!beamformer_) { |
1033 beamformer_.reset(new NonlinearBeamformer(array_geometry_)); | 998 beamformer_.reset(new NonlinearBeamformer(array_geometry_)); |
1034 } | 999 } |
1035 beamformer_->Initialize(kChunkSizeMs, split_rate_); | 1000 beamformer_->Initialize(kChunkSizeMs, split_rate_); |
1036 } | 1001 } |
(...skipping 22 matching lines...) Expand all Loading... |
1059 stream_delay_jumps_ = 0; // Activate counter if needed. | 1024 stream_delay_jumps_ = 0; // Activate counter if needed. |
1060 } | 1025 } |
1061 stream_delay_jumps_++; | 1026 stream_delay_jumps_++; |
1062 } | 1027 } |
1063 last_stream_delay_ms_ = stream_delay_ms_; | 1028 last_stream_delay_ms_ = stream_delay_ms_; |
1064 | 1029 |
1065 // Detect a jump in AEC system delay and log the difference. | 1030 // Detect a jump in AEC system delay and log the difference. |
1066 const int frames_per_ms = rtc::CheckedDivExact(split_rate_, 1000); | 1031 const int frames_per_ms = rtc::CheckedDivExact(split_rate_, 1000); |
1067 const int aec_system_delay_ms = | 1032 const int aec_system_delay_ms = |
1068 WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms; | 1033 WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms; |
1069 const int diff_aec_system_delay_ms = | 1034 const int diff_aec_system_delay_ms = aec_system_delay_ms - |
1070 aec_system_delay_ms - last_aec_system_delay_ms_; | 1035 last_aec_system_delay_ms_; |
1071 if (diff_aec_system_delay_ms > kMinDiffDelayMs && | 1036 if (diff_aec_system_delay_ms > kMinDiffDelayMs && |
1072 last_aec_system_delay_ms_ != 0) { | 1037 last_aec_system_delay_ms_ != 0) { |
1073 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", | 1038 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", |
1074 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, | 1039 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, |
1075 100); | 1040 100); |
1076 if (aec_system_delay_jumps_ == -1) { | 1041 if (aec_system_delay_jumps_ == -1) { |
1077 aec_system_delay_jumps_ = 0; // Activate counter if needed. | 1042 aec_system_delay_jumps_ = 0; // Activate counter if needed. |
1078 } | 1043 } |
1079 aec_system_delay_jumps_++; | 1044 aec_system_delay_jumps_++; |
1080 } | 1045 } |
(...skipping 19 matching lines...) Expand all Loading... |
1100 last_aec_system_delay_ms_ = 0; | 1065 last_aec_system_delay_ms_ = 0; |
1101 } | 1066 } |
1102 | 1067 |
1103 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1068 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
1104 int AudioProcessingImpl::WriteMessageToDebugFile() { | 1069 int AudioProcessingImpl::WriteMessageToDebugFile() { |
1105 int32_t size = event_msg_->ByteSize(); | 1070 int32_t size = event_msg_->ByteSize(); |
1106 if (size <= 0) { | 1071 if (size <= 0) { |
1107 return kUnspecifiedError; | 1072 return kUnspecifiedError; |
1108 } | 1073 } |
1109 #if defined(WEBRTC_ARCH_BIG_ENDIAN) | 1074 #if defined(WEBRTC_ARCH_BIG_ENDIAN) |
1110 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be | 1075 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be |
1111 // pretty safe in assuming little-endian. | 1076 // pretty safe in assuming little-endian. |
1112 #endif | 1077 #endif |
1113 | 1078 |
1114 if (!event_msg_->SerializeToString(&event_str_)) { | 1079 if (!event_msg_->SerializeToString(&event_str_)) { |
1115 return kUnspecifiedError; | 1080 return kUnspecifiedError; |
1116 } | 1081 } |
1117 | 1082 |
1118 // Write message preceded by its size. | 1083 // Write message preceded by its size. |
1119 if (!debug_file_->Write(&size, sizeof(int32_t))) { | 1084 if (!debug_file_->Write(&size, sizeof(int32_t))) { |
1120 return kFileError; | 1085 return kFileError; |
1121 } | 1086 } |
1122 if (!debug_file_->Write(event_str_.data(), event_str_.length())) { | 1087 if (!debug_file_->Write(event_str_.data(), event_str_.length())) { |
1123 return kFileError; | 1088 return kFileError; |
1124 } | 1089 } |
1125 | 1090 |
1126 event_msg_->Clear(); | 1091 event_msg_->Clear(); |
1127 | 1092 |
1128 return kNoError; | 1093 return kNoError; |
1129 } | 1094 } |
1130 | 1095 |
1131 int AudioProcessingImpl::WriteInitMessage() { | 1096 int AudioProcessingImpl::WriteInitMessage() { |
1132 event_msg_->set_type(audioproc::Event::INIT); | 1097 event_msg_->set_type(audioproc::Event::INIT); |
1133 audioproc::Init* msg = event_msg_->mutable_init(); | 1098 audioproc::Init* msg = event_msg_->mutable_init(); |
1134 msg->set_sample_rate(api_format_.input_stream().sample_rate_hz()); | 1099 msg->set_sample_rate(fwd_in_format_.rate()); |
1135 msg->set_num_input_channels(api_format_.input_stream().num_channels()); | 1100 msg->set_num_input_channels(fwd_in_format_.num_channels()); |
1136 msg->set_num_output_channels(api_format_.output_stream().num_channels()); | 1101 msg->set_num_output_channels(fwd_out_format_.num_channels()); |
1137 msg->set_num_reverse_channels(api_format_.reverse_stream().num_channels()); | 1102 msg->set_num_reverse_channels(rev_in_format_.num_channels()); |
1138 msg->set_reverse_sample_rate(api_format_.reverse_stream().sample_rate_hz()); | 1103 msg->set_reverse_sample_rate(rev_in_format_.rate()); |
1139 msg->set_output_sample_rate(api_format_.output_stream().sample_rate_hz()); | 1104 msg->set_output_sample_rate(fwd_out_format_.rate()); |
1140 | 1105 |
1141 int err = WriteMessageToDebugFile(); | 1106 int err = WriteMessageToDebugFile(); |
1142 if (err != kNoError) { | 1107 if (err != kNoError) { |
1143 return err; | 1108 return err; |
1144 } | 1109 } |
1145 | 1110 |
1146 return kNoError; | 1111 return kNoError; |
1147 } | 1112 } |
1148 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1113 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
1149 | 1114 |
1150 } // namespace webrtc | 1115 } // namespace webrtc |
OLD | NEW |