OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include "webrtc/modules/audio_processing/audio_processing_impl.h" | 11 #include "webrtc/modules/audio_processing/audio_processing_impl.h" |
12 | 12 |
13 #include <assert.h> | 13 #include <assert.h> |
14 #include <algorithm> | 14 #include <algorithm> |
15 | 15 |
16 #include "webrtc/base/checks.h" | 16 #include "webrtc/base/checks.h" |
17 #include "webrtc/base/platform_file.h" | 17 #include "webrtc/base/platform_file.h" |
| 18 #include "webrtc/common_audio/audio_converter.h" |
| 19 #include "webrtc/common_audio/channel_buffer.h" |
18 #include "webrtc/common_audio/include/audio_util.h" | 20 #include "webrtc/common_audio/include/audio_util.h" |
19 #include "webrtc/common_audio/channel_buffer.h" | |
20 #include "webrtc/common_audio/signal_processing/include/signal_processing_librar
y.h" | 21 #include "webrtc/common_audio/signal_processing/include/signal_processing_librar
y.h" |
21 extern "C" { | 22 extern "C" { |
22 #include "webrtc/modules/audio_processing/aec/aec_core.h" | 23 #include "webrtc/modules/audio_processing/aec/aec_core.h" |
23 } | 24 } |
24 #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h" | 25 #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h" |
25 #include "webrtc/modules/audio_processing/audio_buffer.h" | 26 #include "webrtc/modules/audio_processing/audio_buffer.h" |
26 #include "webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h" | 27 #include "webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h" |
27 #include "webrtc/modules/audio_processing/common.h" | 28 #include "webrtc/modules/audio_processing/common.h" |
28 #include "webrtc/modules/audio_processing/echo_cancellation_impl.h" | 29 #include "webrtc/modules/audio_processing/echo_cancellation_impl.h" |
29 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h" | 30 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h" |
30 #include "webrtc/modules/audio_processing/gain_control_impl.h" | 31 #include "webrtc/modules/audio_processing/gain_control_impl.h" |
31 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h" | 32 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h" |
| 33 #include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhanc
er.h" |
32 #include "webrtc/modules/audio_processing/level_estimator_impl.h" | 34 #include "webrtc/modules/audio_processing/level_estimator_impl.h" |
33 #include "webrtc/modules/audio_processing/noise_suppression_impl.h" | 35 #include "webrtc/modules/audio_processing/noise_suppression_impl.h" |
34 #include "webrtc/modules/audio_processing/processing_component.h" | 36 #include "webrtc/modules/audio_processing/processing_component.h" |
35 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h" | 37 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h" |
36 #include "webrtc/modules/audio_processing/voice_detection_impl.h" | 38 #include "webrtc/modules/audio_processing/voice_detection_impl.h" |
37 #include "webrtc/modules/interface/module_common_types.h" | 39 #include "webrtc/modules/interface/module_common_types.h" |
38 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" | 40 #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" |
39 #include "webrtc/system_wrappers/interface/file_wrapper.h" | 41 #include "webrtc/system_wrappers/interface/file_wrapper.h" |
40 #include "webrtc/system_wrappers/interface/logging.h" | 42 #include "webrtc/system_wrappers/interface/logging.h" |
41 #include "webrtc/system_wrappers/interface/metrics.h" | 43 #include "webrtc/system_wrappers/interface/metrics.h" |
(...skipping 135 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
177 level_estimator_(NULL), | 179 level_estimator_(NULL), |
178 noise_suppression_(NULL), | 180 noise_suppression_(NULL), |
179 voice_detection_(NULL), | 181 voice_detection_(NULL), |
180 crit_(CriticalSectionWrapper::CreateCriticalSection()), | 182 crit_(CriticalSectionWrapper::CreateCriticalSection()), |
181 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 183 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
182 debug_file_(FileWrapper::Create()), | 184 debug_file_(FileWrapper::Create()), |
183 event_msg_(new audioproc::Event()), | 185 event_msg_(new audioproc::Event()), |
184 #endif | 186 #endif |
185 api_format_({{{kSampleRate16kHz, 1, false}, | 187 api_format_({{{kSampleRate16kHz, 1, false}, |
186 {kSampleRate16kHz, 1, false}, | 188 {kSampleRate16kHz, 1, false}, |
| 189 {kSampleRate16kHz, 1, false}, |
187 {kSampleRate16kHz, 1, false}}}), | 190 {kSampleRate16kHz, 1, false}}}), |
188 fwd_proc_format_(kSampleRate16kHz), | 191 fwd_proc_format_(kSampleRate16kHz), |
189 rev_proc_format_(kSampleRate16kHz, 1), | 192 rev_proc_format_(kSampleRate16kHz, 1), |
190 split_rate_(kSampleRate16kHz), | 193 split_rate_(kSampleRate16kHz), |
191 stream_delay_ms_(0), | 194 stream_delay_ms_(0), |
192 delay_offset_ms_(0), | 195 delay_offset_ms_(0), |
193 was_stream_delay_set_(false), | 196 was_stream_delay_set_(false), |
194 last_stream_delay_ms_(0), | 197 last_stream_delay_ms_(0), |
195 last_aec_system_delay_ms_(0), | 198 last_aec_system_delay_ms_(0), |
196 stream_delay_jumps_(-1), | 199 stream_delay_jumps_(-1), |
197 aec_system_delay_jumps_(-1), | 200 aec_system_delay_jumps_(-1), |
198 output_will_be_muted_(false), | 201 output_will_be_muted_(false), |
199 key_pressed_(false), | 202 key_pressed_(false), |
200 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) | 203 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) |
201 use_new_agc_(false), | 204 use_new_agc_(false), |
202 #else | 205 #else |
203 use_new_agc_(config.Get<ExperimentalAgc>().enabled), | 206 use_new_agc_(config.Get<ExperimentalAgc>().enabled), |
204 #endif | 207 #endif |
205 agc_startup_min_volume_(config.Get<ExperimentalAgc>().startup_min_volume), | 208 agc_startup_min_volume_(config.Get<ExperimentalAgc>().startup_min_volume), |
206 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) | 209 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) |
207 transient_suppressor_enabled_(false), | 210 transient_suppressor_enabled_(false), |
208 #else | 211 #else |
209 transient_suppressor_enabled_(config.Get<ExperimentalNs>().enabled), | 212 transient_suppressor_enabled_(config.Get<ExperimentalNs>().enabled), |
210 #endif | 213 #endif |
211 beamformer_enabled_(config.Get<Beamforming>().enabled), | 214 beamformer_enabled_(config.Get<Beamforming>().enabled), |
212 beamformer_(beamformer), | 215 beamformer_(beamformer), |
213 array_geometry_(config.Get<Beamforming>().array_geometry) { | 216 array_geometry_(config.Get<Beamforming>().array_geometry), |
| 217 intelligibility_enabled_(config.Get<Intelligibility>().enabled) { |
214 echo_cancellation_ = new EchoCancellationImpl(this, crit_); | 218 echo_cancellation_ = new EchoCancellationImpl(this, crit_); |
215 component_list_.push_back(echo_cancellation_); | 219 component_list_.push_back(echo_cancellation_); |
216 | 220 |
217 echo_control_mobile_ = new EchoControlMobileImpl(this, crit_); | 221 echo_control_mobile_ = new EchoControlMobileImpl(this, crit_); |
218 component_list_.push_back(echo_control_mobile_); | 222 component_list_.push_back(echo_control_mobile_); |
219 | 223 |
220 gain_control_ = new GainControlImpl(this, crit_); | 224 gain_control_ = new GainControlImpl(this, crit_); |
221 component_list_.push_back(gain_control_); | 225 component_list_.push_back(gain_control_); |
222 | 226 |
223 high_pass_filter_ = new HighPassFilterImpl(this, crit_); | 227 high_pass_filter_ = new HighPassFilterImpl(this, crit_); |
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
275 return InitializeLocked(processing_config); | 279 return InitializeLocked(processing_config); |
276 } | 280 } |
277 | 281 |
278 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, | 282 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, |
279 int output_sample_rate_hz, | 283 int output_sample_rate_hz, |
280 int reverse_sample_rate_hz, | 284 int reverse_sample_rate_hz, |
281 ChannelLayout input_layout, | 285 ChannelLayout input_layout, |
282 ChannelLayout output_layout, | 286 ChannelLayout output_layout, |
283 ChannelLayout reverse_layout) { | 287 ChannelLayout reverse_layout) { |
284 const ProcessingConfig processing_config = { | 288 const ProcessingConfig processing_config = { |
285 {{input_sample_rate_hz, ChannelsFromLayout(input_layout), | 289 {{input_sample_rate_hz, |
| 290 ChannelsFromLayout(input_layout), |
286 LayoutHasKeyboard(input_layout)}, | 291 LayoutHasKeyboard(input_layout)}, |
287 {output_sample_rate_hz, ChannelsFromLayout(output_layout), | 292 {output_sample_rate_hz, |
| 293 ChannelsFromLayout(output_layout), |
288 LayoutHasKeyboard(output_layout)}, | 294 LayoutHasKeyboard(output_layout)}, |
289 {reverse_sample_rate_hz, ChannelsFromLayout(reverse_layout), | 295 {reverse_sample_rate_hz, |
| 296 ChannelsFromLayout(reverse_layout), |
| 297 LayoutHasKeyboard(reverse_layout)}, |
| 298 {reverse_sample_rate_hz, |
| 299 ChannelsFromLayout(reverse_layout), |
290 LayoutHasKeyboard(reverse_layout)}}}; | 300 LayoutHasKeyboard(reverse_layout)}}}; |
291 | 301 |
292 return Initialize(processing_config); | 302 return Initialize(processing_config); |
293 } | 303 } |
294 | 304 |
295 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { | 305 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { |
296 CriticalSectionScoped crit_scoped(crit_); | 306 CriticalSectionScoped crit_scoped(crit_); |
297 return InitializeLocked(processing_config); | 307 return InitializeLocked(processing_config); |
298 } | 308 } |
299 | 309 |
300 int AudioProcessingImpl::InitializeLocked() { | 310 int AudioProcessingImpl::InitializeLocked() { |
301 const int fwd_audio_buffer_channels = | 311 const int fwd_audio_buffer_channels = |
302 beamformer_enabled_ ? api_format_.input_stream().num_channels() | 312 beamformer_enabled_ ? api_format_.input_stream().num_channels() |
303 : api_format_.output_stream().num_channels(); | 313 : api_format_.output_stream().num_channels(); |
304 if (api_format_.reverse_stream().num_channels() > 0) { | 314 const int rev_audio_buffer_out_num_frames = |
| 315 api_format_.reverse_output_stream().num_frames() == 0 |
| 316 ? rev_proc_format_.num_frames() |
| 317 : api_format_.reverse_output_stream().num_frames(); |
| 318 if (api_format_.reverse_input_stream().num_channels() > 0) { |
305 render_audio_.reset(new AudioBuffer( | 319 render_audio_.reset(new AudioBuffer( |
306 api_format_.reverse_stream().num_frames(), | 320 api_format_.reverse_input_stream().num_frames(), |
307 api_format_.reverse_stream().num_channels(), | 321 api_format_.reverse_input_stream().num_channels(), |
308 rev_proc_format_.num_frames(), rev_proc_format_.num_channels(), | 322 rev_proc_format_.num_frames(), rev_proc_format_.num_channels(), |
309 rev_proc_format_.num_frames())); | 323 rev_audio_buffer_out_num_frames)); |
| 324 if (rev_conversion_needed()) { |
| 325 render_converter_ = AudioConverter::Create( |
| 326 api_format_.reverse_input_stream().num_channels(), |
| 327 api_format_.reverse_input_stream().num_frames(), |
| 328 api_format_.reverse_output_stream().num_channels(), |
| 329 api_format_.reverse_output_stream().num_frames()); |
| 330 } else { |
| 331 render_converter_.reset(nullptr); |
| 332 } |
310 } else { | 333 } else { |
311 render_audio_.reset(nullptr); | 334 render_audio_.reset(nullptr); |
| 335 render_converter_.reset(nullptr); |
312 } | 336 } |
313 capture_audio_.reset(new AudioBuffer( | 337 capture_audio_.reset(new AudioBuffer( |
314 api_format_.input_stream().num_frames(), | 338 api_format_.input_stream().num_frames(), |
315 api_format_.input_stream().num_channels(), fwd_proc_format_.num_frames(), | 339 api_format_.input_stream().num_channels(), fwd_proc_format_.num_frames(), |
316 fwd_audio_buffer_channels, api_format_.output_stream().num_frames())); | 340 fwd_audio_buffer_channels, api_format_.output_stream().num_frames())); |
317 | 341 |
318 // Initialize all components. | 342 // Initialize all components. |
319 for (auto item : component_list_) { | 343 for (auto item : component_list_) { |
320 int err = item->Initialize(); | 344 int err = item->Initialize(); |
321 if (err != kNoError) { | 345 if (err != kNoError) { |
322 return err; | 346 return err; |
323 } | 347 } |
324 } | 348 } |
325 | 349 |
326 InitializeExperimentalAgc(); | 350 InitializeExperimentalAgc(); |
327 | 351 |
328 InitializeTransient(); | 352 InitializeTransient(); |
329 | 353 |
330 InitializeBeamformer(); | 354 InitializeBeamformer(); |
331 | 355 |
| 356 InitializeIntelligibility(); |
| 357 |
332 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 358 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
333 if (debug_file_->Open()) { | 359 if (debug_file_->Open()) { |
334 int err = WriteInitMessage(); | 360 int err = WriteInitMessage(); |
335 if (err != kNoError) { | 361 if (err != kNoError) { |
336 return err; | 362 return err; |
337 } | 363 } |
338 } | 364 } |
339 #endif | 365 #endif |
340 | 366 |
341 return kNoError; | 367 return kNoError; |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
389 } | 415 } |
390 | 416 |
391 fwd_proc_format_ = StreamConfig(fwd_proc_rate); | 417 fwd_proc_format_ = StreamConfig(fwd_proc_rate); |
392 | 418 |
393 // We normally process the reverse stream at 16 kHz. Unless... | 419 // We normally process the reverse stream at 16 kHz. Unless... |
394 int rev_proc_rate = kSampleRate16kHz; | 420 int rev_proc_rate = kSampleRate16kHz; |
395 if (fwd_proc_format_.sample_rate_hz() == kSampleRate8kHz) { | 421 if (fwd_proc_format_.sample_rate_hz() == kSampleRate8kHz) { |
396 // ...the forward stream is at 8 kHz. | 422 // ...the forward stream is at 8 kHz. |
397 rev_proc_rate = kSampleRate8kHz; | 423 rev_proc_rate = kSampleRate8kHz; |
398 } else { | 424 } else { |
399 if (api_format_.reverse_stream().sample_rate_hz() == kSampleRate32kHz) { | 425 if (api_format_.reverse_input_stream().sample_rate_hz() == |
| 426 kSampleRate32kHz) { |
400 // ...or the input is at 32 kHz, in which case we use the splitting | 427 // ...or the input is at 32 kHz, in which case we use the splitting |
401 // filter rather than the resampler. | 428 // filter rather than the resampler. |
402 rev_proc_rate = kSampleRate32kHz; | 429 rev_proc_rate = kSampleRate32kHz; |
403 } | 430 } |
404 } | 431 } |
405 | 432 |
406 // Always downmix the reverse stream to mono for analysis. This has been | 433 // Always downmix the reverse stream to mono for analysis. This has been |
407 // demonstrated to work well for AEC in most practical scenarios. | 434 // demonstrated to work well for AEC in most practical scenarios. |
408 rev_proc_format_ = StreamConfig(rev_proc_rate, 1); | 435 rev_proc_format_ = StreamConfig(rev_proc_rate, 1); |
409 | 436 |
(...skipping 207 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
617 msg->set_delay(stream_delay_ms_); | 644 msg->set_delay(stream_delay_ms_); |
618 msg->set_drift(echo_cancellation_->stream_drift_samples()); | 645 msg->set_drift(echo_cancellation_->stream_drift_samples()); |
619 msg->set_level(gain_control()->stream_analog_level()); | 646 msg->set_level(gain_control()->stream_analog_level()); |
620 msg->set_keypress(key_pressed_); | 647 msg->set_keypress(key_pressed_); |
621 } | 648 } |
622 #endif | 649 #endif |
623 | 650 |
624 MaybeUpdateHistograms(); | 651 MaybeUpdateHistograms(); |
625 | 652 |
626 AudioBuffer* ca = capture_audio_.get(); // For brevity. | 653 AudioBuffer* ca = capture_audio_.get(); // For brevity. |
| 654 |
627 if (use_new_agc_ && gain_control_->is_enabled()) { | 655 if (use_new_agc_ && gain_control_->is_enabled()) { |
628 agc_manager_->AnalyzePreProcess(ca->channels()[0], ca->num_channels(), | 656 agc_manager_->AnalyzePreProcess(ca->channels()[0], ca->num_channels(), |
629 fwd_proc_format_.num_frames()); | 657 fwd_proc_format_.num_frames()); |
630 } | 658 } |
631 | 659 |
632 bool data_processed = is_data_processed(); | 660 bool data_processed = is_data_processed(); |
633 if (analysis_needed(data_processed)) { | 661 if (analysis_needed(data_processed)) { |
634 ca->SplitIntoFrequencyBands(); | 662 ca->SplitIntoFrequencyBands(); |
635 } | 663 } |
636 | 664 |
| 665 if (intelligibility_enabled_) { |
| 666 intelligibility_enhancer_->AnalyzeCaptureAudio( |
| 667 ca->split_channels_f(kBand0To8kHz), split_rate_, ca->num_channels()); |
| 668 } |
| 669 |
637 if (beamformer_enabled_) { | 670 if (beamformer_enabled_) { |
638 beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f()); | 671 beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f()); |
639 ca->set_num_channels(1); | 672 ca->set_num_channels(1); |
640 } | 673 } |
641 | 674 |
642 RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca)); | 675 RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca)); |
643 RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca)); | 676 RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca)); |
644 RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca)); | 677 RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca)); |
645 RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca)); | 678 RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca)); |
646 | 679 |
(...skipping 30 matching lines...) Expand all Loading... |
677 | 710 |
678 // The level estimator operates on the recombined data. | 711 // The level estimator operates on the recombined data. |
679 RETURN_ON_ERR(level_estimator_->ProcessStream(ca)); | 712 RETURN_ON_ERR(level_estimator_->ProcessStream(ca)); |
680 | 713 |
681 was_stream_delay_set_ = false; | 714 was_stream_delay_set_ = false; |
682 return kNoError; | 715 return kNoError; |
683 } | 716 } |
684 | 717 |
685 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, | 718 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, |
686 int samples_per_channel, | 719 int samples_per_channel, |
687 int sample_rate_hz, | 720 int rev_sample_rate_hz, |
688 ChannelLayout layout) { | 721 ChannelLayout layout) { |
689 const StreamConfig reverse_config = { | 722 const StreamConfig reverse_config = { |
690 sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), | 723 rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), |
691 }; | 724 }; |
692 if (samples_per_channel != reverse_config.num_frames()) { | 725 if (samples_per_channel != reverse_config.num_frames()) { |
693 return kBadDataLengthError; | 726 return kBadDataLengthError; |
694 } | 727 } |
695 return AnalyzeReverseStream(data, reverse_config); | 728 return AnalyzeReverseStream(data, reverse_config, reverse_config); |
| 729 } |
| 730 |
| 731 int AudioProcessingImpl::ProcessReverseStream( |
| 732 const float* const* src, |
| 733 const StreamConfig& reverse_input_config, |
| 734 const StreamConfig& reverse_output_config, |
| 735 float* const* dest) { |
| 736 RETURN_ON_ERR( |
| 737 AnalyzeReverseStream(src, reverse_input_config, reverse_output_config)); |
| 738 if (is_rev_processed()) { |
| 739 render_audio_->CopyTo(api_format_.reverse_output_stream(), dest); |
| 740 } else if (rev_conversion_needed()) { |
| 741 render_converter_->Convert(src, reverse_input_config.num_samples(), dest, |
| 742 reverse_output_config.num_samples()); |
| 743 } else { |
| 744 CopyAudioIfNeeded(src, reverse_input_config.num_frames(), |
| 745 reverse_input_config.num_channels(), dest); |
| 746 } |
| 747 |
| 748 return kNoError; |
696 } | 749 } |
697 | 750 |
698 int AudioProcessingImpl::AnalyzeReverseStream( | 751 int AudioProcessingImpl::AnalyzeReverseStream( |
699 const float* const* data, | 752 const float* const* src, |
700 const StreamConfig& reverse_config) { | 753 const StreamConfig& reverse_input_config, |
| 754 const StreamConfig& reverse_output_config) { |
701 CriticalSectionScoped crit_scoped(crit_); | 755 CriticalSectionScoped crit_scoped(crit_); |
702 if (data == NULL) { | 756 if (src == NULL) { |
703 return kNullPointerError; | 757 return kNullPointerError; |
704 } | 758 } |
705 | 759 |
706 if (reverse_config.num_channels() <= 0) { | 760 if (reverse_input_config.num_channels() <= 0) { |
707 return kBadNumberChannelsError; | 761 return kBadNumberChannelsError; |
708 } | 762 } |
709 | 763 |
710 ProcessingConfig processing_config = api_format_; | 764 ProcessingConfig processing_config = api_format_; |
711 processing_config.reverse_stream() = reverse_config; | 765 processing_config.reverse_input_stream() = reverse_input_config; |
| 766 processing_config.reverse_output_stream() = reverse_output_config; |
712 | 767 |
713 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | 768 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); |
714 assert(reverse_config.num_frames() == | 769 assert(reverse_input_config.num_frames() == |
715 api_format_.reverse_stream().num_frames()); | 770 api_format_.reverse_input_stream().num_frames()); |
716 | 771 |
717 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 772 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
718 if (debug_file_->Open()) { | 773 if (debug_file_->Open()) { |
719 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); | 774 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); |
720 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); | 775 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); |
721 const size_t channel_size = | 776 const size_t channel_size = |
722 sizeof(float) * api_format_.reverse_stream().num_frames(); | 777 sizeof(float) * api_format_.reverse_input_stream().num_frames(); |
723 for (int i = 0; i < api_format_.reverse_stream().num_channels(); ++i) | 778 for (int i = 0; i < api_format_.reverse_input_stream().num_channels(); ++i) |
724 msg->add_channel(data[i], channel_size); | 779 msg->add_channel(src[i], channel_size); |
725 RETURN_ON_ERR(WriteMessageToDebugFile()); | 780 RETURN_ON_ERR(WriteMessageToDebugFile()); |
726 } | 781 } |
727 #endif | 782 #endif |
728 | 783 |
729 render_audio_->CopyFrom(data, api_format_.reverse_stream()); | 784 render_audio_->CopyFrom(src, api_format_.reverse_input_stream()); |
730 return AnalyzeReverseStreamLocked(); | 785 return ProcessReverseStreamLocked(); |
| 786 } |
| 787 |
| 788 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { |
| 789 RETURN_ON_ERR(AnalyzeReverseStream(frame)); |
| 790 if (is_rev_processed()) { |
| 791 render_audio_->InterleaveTo(frame, true); |
| 792 } |
| 793 |
| 794 return kNoError; |
731 } | 795 } |
732 | 796 |
733 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { | 797 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { |
734 CriticalSectionScoped crit_scoped(crit_); | 798 CriticalSectionScoped crit_scoped(crit_); |
735 if (frame == NULL) { | 799 if (frame == NULL) { |
736 return kNullPointerError; | 800 return kNullPointerError; |
737 } | 801 } |
738 // Must be a native rate. | 802 // Must be a native rate. |
739 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 803 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
740 frame->sample_rate_hz_ != kSampleRate16kHz && | 804 frame->sample_rate_hz_ != kSampleRate16kHz && |
741 frame->sample_rate_hz_ != kSampleRate32kHz && | 805 frame->sample_rate_hz_ != kSampleRate32kHz && |
742 frame->sample_rate_hz_ != kSampleRate48kHz) { | 806 frame->sample_rate_hz_ != kSampleRate48kHz) { |
743 return kBadSampleRateError; | 807 return kBadSampleRateError; |
744 } | 808 } |
745 // This interface does not tolerate different forward and reverse rates. | 809 // This interface does not tolerate different forward and reverse rates. |
746 if (frame->sample_rate_hz_ != api_format_.input_stream().sample_rate_hz()) { | 810 if (frame->sample_rate_hz_ != api_format_.input_stream().sample_rate_hz()) { |
747 return kBadSampleRateError; | 811 return kBadSampleRateError; |
748 } | 812 } |
749 | 813 |
750 if (frame->num_channels_ <= 0) { | 814 if (frame->num_channels_ <= 0) { |
751 return kBadNumberChannelsError; | 815 return kBadNumberChannelsError; |
752 } | 816 } |
753 | 817 |
754 ProcessingConfig processing_config = api_format_; | 818 ProcessingConfig processing_config = api_format_; |
755 processing_config.reverse_stream().set_sample_rate_hz(frame->sample_rate_hz_); | 819 processing_config.reverse_input_stream().set_sample_rate_hz( |
756 processing_config.reverse_stream().set_num_channels(frame->num_channels_); | 820 frame->sample_rate_hz_); |
| 821 processing_config.reverse_input_stream().set_num_channels( |
| 822 frame->num_channels_); |
| 823 processing_config.reverse_output_stream().set_sample_rate_hz( |
| 824 frame->sample_rate_hz_); |
| 825 processing_config.reverse_output_stream().set_num_channels( |
| 826 frame->num_channels_); |
757 | 827 |
758 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | 828 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); |
759 if (frame->samples_per_channel_ != | 829 if (frame->samples_per_channel_ != |
760 api_format_.reverse_stream().num_frames()) { | 830 api_format_.reverse_input_stream().num_frames()) { |
761 return kBadDataLengthError; | 831 return kBadDataLengthError; |
762 } | 832 } |
763 | 833 |
764 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 834 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
765 if (debug_file_->Open()) { | 835 if (debug_file_->Open()) { |
766 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); | 836 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); |
767 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); | 837 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); |
768 const size_t data_size = | 838 const size_t data_size = |
769 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 839 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
770 msg->set_data(frame->data_, data_size); | 840 msg->set_data(frame->data_, data_size); |
771 RETURN_ON_ERR(WriteMessageToDebugFile()); | 841 RETURN_ON_ERR(WriteMessageToDebugFile()); |
772 } | 842 } |
773 #endif | 843 #endif |
774 | |
775 render_audio_->DeinterleaveFrom(frame); | 844 render_audio_->DeinterleaveFrom(frame); |
776 return AnalyzeReverseStreamLocked(); | 845 return ProcessReverseStreamLocked(); |
777 } | 846 } |
778 | 847 |
779 int AudioProcessingImpl::AnalyzeReverseStreamLocked() { | 848 int AudioProcessingImpl::ProcessReverseStreamLocked() { |
780 AudioBuffer* ra = render_audio_.get(); // For brevity. | 849 AudioBuffer* ra = render_audio_.get(); // For brevity. |
781 if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz) { | 850 if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz) { |
782 ra->SplitIntoFrequencyBands(); | 851 ra->SplitIntoFrequencyBands(); |
783 } | 852 } |
784 | 853 |
| 854 if (intelligibility_enabled_) { |
| 855 intelligibility_enhancer_->ProcessRenderAudio( |
| 856 ra->split_channels_f(kBand0To8kHz), split_rate_, ra->num_channels()); |
| 857 } |
| 858 |
785 RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra)); | 859 RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra)); |
786 RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra)); | 860 RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra)); |
787 if (!use_new_agc_) { | 861 if (!use_new_agc_) { |
788 RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra)); | 862 RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra)); |
789 } | 863 } |
790 | 864 |
| 865 if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz && |
| 866 is_rev_processed()) { |
| 867 ra->MergeFrequencyBands(); |
| 868 } |
| 869 |
791 return kNoError; | 870 return kNoError; |
792 } | 871 } |
793 | 872 |
794 int AudioProcessingImpl::set_stream_delay_ms(int delay) { | 873 int AudioProcessingImpl::set_stream_delay_ms(int delay) { |
795 Error retval = kNoError; | 874 Error retval = kNoError; |
796 was_stream_delay_set_ = true; | 875 was_stream_delay_set_ = true; |
797 delay += delay_offset_ms_; | 876 delay += delay_offset_ms_; |
798 | 877 |
799 if (delay < 0) { | 878 if (delay < 0) { |
800 delay = 0; | 879 delay = 0; |
(...skipping 196 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
997 // Only level_estimator_ is enabled. | 1076 // Only level_estimator_ is enabled. |
998 return false; | 1077 return false; |
999 } else if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || | 1078 } else if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || |
1000 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { | 1079 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { |
1001 // Something besides level_estimator_ is enabled, and we have super-wb. | 1080 // Something besides level_estimator_ is enabled, and we have super-wb. |
1002 return true; | 1081 return true; |
1003 } | 1082 } |
1004 return false; | 1083 return false; |
1005 } | 1084 } |
1006 | 1085 |
| 1086 bool AudioProcessingImpl::is_rev_processed() const { |
| 1087 return intelligibility_enabled_ && intelligibility_enhancer_->active(); |
| 1088 } |
| 1089 |
| 1090 bool AudioProcessingImpl::rev_conversion_needed() const { |
| 1091 return (api_format_.reverse_input_stream() != |
| 1092 api_format_.reverse_output_stream()); |
| 1093 } |
| 1094 |
1007 void AudioProcessingImpl::InitializeExperimentalAgc() { | 1095 void AudioProcessingImpl::InitializeExperimentalAgc() { |
1008 if (use_new_agc_) { | 1096 if (use_new_agc_) { |
1009 if (!agc_manager_.get()) { | 1097 if (!agc_manager_.get()) { |
1010 agc_manager_.reset(new AgcManagerDirect(gain_control_, | 1098 agc_manager_.reset(new AgcManagerDirect(gain_control_, |
1011 gain_control_for_new_agc_.get(), | 1099 gain_control_for_new_agc_.get(), |
1012 agc_startup_min_volume_)); | 1100 agc_startup_min_volume_)); |
1013 } | 1101 } |
1014 agc_manager_->Initialize(); | 1102 agc_manager_->Initialize(); |
1015 agc_manager_->SetCaptureMuted(output_will_be_muted_); | 1103 agc_manager_->SetCaptureMuted(output_will_be_muted_); |
1016 } | 1104 } |
(...skipping 12 matching lines...) Expand all Loading... |
1029 | 1117 |
1030 void AudioProcessingImpl::InitializeBeamformer() { | 1118 void AudioProcessingImpl::InitializeBeamformer() { |
1031 if (beamformer_enabled_) { | 1119 if (beamformer_enabled_) { |
1032 if (!beamformer_) { | 1120 if (!beamformer_) { |
1033 beamformer_.reset(new NonlinearBeamformer(array_geometry_)); | 1121 beamformer_.reset(new NonlinearBeamformer(array_geometry_)); |
1034 } | 1122 } |
1035 beamformer_->Initialize(kChunkSizeMs, split_rate_); | 1123 beamformer_->Initialize(kChunkSizeMs, split_rate_); |
1036 } | 1124 } |
1037 } | 1125 } |
1038 | 1126 |
| 1127 void AudioProcessingImpl::InitializeIntelligibility() { |
| 1128 if (intelligibility_enabled_) { |
| 1129 IntelligibilityEnhancer::Config config; |
| 1130 config.sample_rate_hz = split_rate_; |
| 1131 config.num_capture_channels = capture_audio_->num_channels(); |
| 1132 config.num_render_channels = render_audio_->num_channels(); |
| 1133 intelligibility_enhancer_.reset(new IntelligibilityEnhancer(config)); |
| 1134 } |
| 1135 } |
| 1136 |
1039 void AudioProcessingImpl::MaybeUpdateHistograms() { | 1137 void AudioProcessingImpl::MaybeUpdateHistograms() { |
1040 static const int kMinDiffDelayMs = 60; | 1138 static const int kMinDiffDelayMs = 60; |
1041 | 1139 |
1042 if (echo_cancellation()->is_enabled()) { | 1140 if (echo_cancellation()->is_enabled()) { |
1043 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. | 1141 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. |
1044 // If a stream has echo we know that the echo_cancellation is in process. | 1142 // If a stream has echo we know that the echo_cancellation is in process. |
1045 if (stream_delay_jumps_ == -1 && echo_cancellation()->stream_has_echo()) { | 1143 if (stream_delay_jumps_ == -1 && echo_cancellation()->stream_has_echo()) { |
1046 stream_delay_jumps_ = 0; | 1144 stream_delay_jumps_ = 0; |
1047 } | 1145 } |
1048 if (aec_system_delay_jumps_ == -1 && | 1146 if (aec_system_delay_jumps_ == -1 && |
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1127 | 1225 |
1128 return kNoError; | 1226 return kNoError; |
1129 } | 1227 } |
1130 | 1228 |
1131 int AudioProcessingImpl::WriteInitMessage() { | 1229 int AudioProcessingImpl::WriteInitMessage() { |
1132 event_msg_->set_type(audioproc::Event::INIT); | 1230 event_msg_->set_type(audioproc::Event::INIT); |
1133 audioproc::Init* msg = event_msg_->mutable_init(); | 1231 audioproc::Init* msg = event_msg_->mutable_init(); |
1134 msg->set_sample_rate(api_format_.input_stream().sample_rate_hz()); | 1232 msg->set_sample_rate(api_format_.input_stream().sample_rate_hz()); |
1135 msg->set_num_input_channels(api_format_.input_stream().num_channels()); | 1233 msg->set_num_input_channels(api_format_.input_stream().num_channels()); |
1136 msg->set_num_output_channels(api_format_.output_stream().num_channels()); | 1234 msg->set_num_output_channels(api_format_.output_stream().num_channels()); |
1137 msg->set_num_reverse_channels(api_format_.reverse_stream().num_channels()); | 1235 msg->set_num_reverse_channels( |
1138 msg->set_reverse_sample_rate(api_format_.reverse_stream().sample_rate_hz()); | 1236 api_format_.reverse_input_stream().num_channels()); |
| 1237 msg->set_reverse_sample_rate( |
| 1238 api_format_.reverse_input_stream().sample_rate_hz()); |
1139 msg->set_output_sample_rate(api_format_.output_stream().sample_rate_hz()); | 1239 msg->set_output_sample_rate(api_format_.output_stream().sample_rate_hz()); |
| 1240 // TODO(ekmeyerson): Add reverse output fields to event_msg_. |
1140 | 1241 |
1141 int err = WriteMessageToDebugFile(); | 1242 int err = WriteMessageToDebugFile(); |
1142 if (err != kNoError) { | 1243 if (err != kNoError) { |
1143 return err; | 1244 return err; |
1144 } | 1245 } |
1145 | 1246 |
1146 return kNoError; | 1247 return kNoError; |
1147 } | 1248 } |
1148 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1249 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
1149 | 1250 |
1150 } // namespace webrtc | 1251 } // namespace webrtc |
OLD | NEW |