OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
(...skipping 305 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
316 #endif | 316 #endif |
317 } | 317 } |
318 | 318 |
319 int AudioProcessingImpl::Initialize() { | 319 int AudioProcessingImpl::Initialize() { |
320 // Run in a single-threaded manner during initialization. | 320 // Run in a single-threaded manner during initialization. |
321 rtc::CritScope cs_render(&crit_render_); | 321 rtc::CritScope cs_render(&crit_render_); |
322 rtc::CritScope cs_capture(&crit_capture_); | 322 rtc::CritScope cs_capture(&crit_capture_); |
323 return InitializeLocked(); | 323 return InitializeLocked(); |
324 } | 324 } |
325 | 325 |
326 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, | 326 int AudioProcessingImpl::Initialize(int capture_input_sample_rate_hz, |
327 int output_sample_rate_hz, | 327 int capture_output_sample_rate_hz, |
328 int reverse_sample_rate_hz, | 328 int render_input_sample_rate_hz, |
329 ChannelLayout input_layout, | 329 ChannelLayout capture_input_layout, |
330 ChannelLayout output_layout, | 330 ChannelLayout capture_output_layout, |
331 ChannelLayout reverse_layout) { | 331 ChannelLayout render_input_layout) { |
332 const ProcessingConfig processing_config = { | 332 const ProcessingConfig processing_config = { |
333 {{input_sample_rate_hz, | 333 {{capture_input_sample_rate_hz, ChannelsFromLayout(capture_input_layout), |
334 ChannelsFromLayout(input_layout), | 334 LayoutHasKeyboard(capture_input_layout)}, |
335 LayoutHasKeyboard(input_layout)}, | 335 {capture_output_sample_rate_hz, |
336 {output_sample_rate_hz, | 336 ChannelsFromLayout(capture_output_layout), |
337 ChannelsFromLayout(output_layout), | 337 LayoutHasKeyboard(capture_output_layout)}, |
338 LayoutHasKeyboard(output_layout)}, | 338 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout), |
339 {reverse_sample_rate_hz, | 339 LayoutHasKeyboard(render_input_layout)}, |
340 ChannelsFromLayout(reverse_layout), | 340 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout), |
341 LayoutHasKeyboard(reverse_layout)}, | 341 LayoutHasKeyboard(render_input_layout)}}}; |
342 {reverse_sample_rate_hz, | |
343 ChannelsFromLayout(reverse_layout), | |
344 LayoutHasKeyboard(reverse_layout)}}}; | |
345 | 342 |
346 return Initialize(processing_config); | 343 return Initialize(processing_config); |
347 } | 344 } |
348 | 345 |
349 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { | 346 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { |
350 // Run in a single-threaded manner during initialization. | 347 // Run in a single-threaded manner during initialization. |
351 rtc::CritScope cs_render(&crit_render_); | 348 rtc::CritScope cs_render(&crit_render_); |
352 rtc::CritScope cs_capture(&crit_capture_); | 349 rtc::CritScope cs_capture(&crit_capture_); |
353 return InitializeLocked(processing_config); | 350 return InitializeLocked(processing_config); |
354 } | 351 } |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
386 // Called from both threads. Thread check is therefore not possible. | 383 // Called from both threads. Thread check is therefore not possible. |
387 if (processing_config == formats_.api_format && !force_initialization) { | 384 if (processing_config == formats_.api_format && !force_initialization) { |
388 return kNoError; | 385 return kNoError; |
389 } | 386 } |
390 | 387 |
391 rtc::CritScope cs_capture(&crit_capture_); | 388 rtc::CritScope cs_capture(&crit_capture_); |
392 return InitializeLocked(processing_config); | 389 return InitializeLocked(processing_config); |
393 } | 390 } |
394 | 391 |
395 int AudioProcessingImpl::InitializeLocked() { | 392 int AudioProcessingImpl::InitializeLocked() { |
396 const int fwd_audio_buffer_channels = | 393 const int capture_audiobuffer_num_channels = |
397 capture_nonlocked_.beamformer_enabled | 394 capture_nonlocked_.beamformer_enabled |
398 ? formats_.api_format.input_stream().num_channels() | 395 ? formats_.api_format.input_stream().num_channels() |
399 : formats_.api_format.output_stream().num_channels(); | 396 : formats_.api_format.output_stream().num_channels(); |
400 const int rev_audio_buffer_out_num_frames = | 397 const int render_audiobuffer_num_output_frames = |
401 formats_.api_format.reverse_output_stream().num_frames() == 0 | 398 formats_.api_format.reverse_output_stream().num_frames() == 0 |
402 ? formats_.rev_proc_format.num_frames() | 399 ? formats_.render_processing_format.num_frames() |
403 : formats_.api_format.reverse_output_stream().num_frames(); | 400 : formats_.api_format.reverse_output_stream().num_frames(); |
404 if (formats_.api_format.reverse_input_stream().num_channels() > 0) { | 401 if (formats_.api_format.reverse_input_stream().num_channels() > 0) { |
405 render_.render_audio.reset(new AudioBuffer( | 402 render_.render_audio.reset(new AudioBuffer( |
406 formats_.api_format.reverse_input_stream().num_frames(), | 403 formats_.api_format.reverse_input_stream().num_frames(), |
407 formats_.api_format.reverse_input_stream().num_channels(), | 404 formats_.api_format.reverse_input_stream().num_channels(), |
408 formats_.rev_proc_format.num_frames(), | 405 formats_.render_processing_format.num_frames(), |
409 formats_.rev_proc_format.num_channels(), | 406 formats_.render_processing_format.num_channels(), |
410 rev_audio_buffer_out_num_frames)); | 407 render_audiobuffer_num_output_frames)); |
411 if (formats_.api_format.reverse_input_stream() != | 408 if (formats_.api_format.reverse_input_stream() != |
412 formats_.api_format.reverse_output_stream()) { | 409 formats_.api_format.reverse_output_stream()) { |
413 render_.render_converter = AudioConverter::Create( | 410 render_.render_converter = AudioConverter::Create( |
414 formats_.api_format.reverse_input_stream().num_channels(), | 411 formats_.api_format.reverse_input_stream().num_channels(), |
415 formats_.api_format.reverse_input_stream().num_frames(), | 412 formats_.api_format.reverse_input_stream().num_frames(), |
416 formats_.api_format.reverse_output_stream().num_channels(), | 413 formats_.api_format.reverse_output_stream().num_channels(), |
417 formats_.api_format.reverse_output_stream().num_frames()); | 414 formats_.api_format.reverse_output_stream().num_frames()); |
418 } else { | 415 } else { |
419 render_.render_converter.reset(nullptr); | 416 render_.render_converter.reset(nullptr); |
420 } | 417 } |
421 } else { | 418 } else { |
422 render_.render_audio.reset(nullptr); | 419 render_.render_audio.reset(nullptr); |
423 render_.render_converter.reset(nullptr); | 420 render_.render_converter.reset(nullptr); |
424 } | 421 } |
425 capture_.capture_audio.reset( | 422 capture_.capture_audio.reset( |
426 new AudioBuffer(formats_.api_format.input_stream().num_frames(), | 423 new AudioBuffer(formats_.api_format.input_stream().num_frames(), |
427 formats_.api_format.input_stream().num_channels(), | 424 formats_.api_format.input_stream().num_channels(), |
428 capture_nonlocked_.fwd_proc_format.num_frames(), | 425 capture_nonlocked_.capture_processing_format.num_frames(), |
429 fwd_audio_buffer_channels, | 426 capture_audiobuffer_num_channels, |
430 formats_.api_format.output_stream().num_frames())); | 427 formats_.api_format.output_stream().num_frames())); |
431 | 428 |
432 InitializeGainController(); | 429 public_submodules_->gain_control->Initialize(num_proc_channels(), |
433 InitializeEchoCanceller(); | 430 proc_sample_rate_hz()); |
434 InitializeEchoControlMobile(); | 431 public_submodules_->echo_cancellation->Initialize( |
435 InitializeExperimentalAgc(); | 432 proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(), |
| 433 num_proc_channels()); |
| 434 public_submodules_->echo_control_mobile->Initialize( |
| 435 proc_split_sample_rate_hz(), num_reverse_channels(), |
| 436 num_output_channels()); |
| 437 if (constants_.use_experimental_agc) { |
| 438 if (!private_submodules_->agc_manager.get()) { |
| 439 private_submodules_->agc_manager.reset(new AgcManagerDirect( |
| 440 public_submodules_->gain_control.get(), |
| 441 public_submodules_->gain_control_for_experimental_agc.get(), |
| 442 constants_.agc_startup_min_volume)); |
| 443 } |
| 444 private_submodules_->agc_manager->Initialize(); |
| 445 private_submodules_->agc_manager->SetCaptureMuted( |
| 446 capture_.output_will_be_muted); |
| 447 } |
436 InitializeTransient(); | 448 InitializeTransient(); |
437 InitializeBeamformer(); | 449 InitializeBeamformer(); |
438 #if WEBRTC_INTELLIGIBILITY_ENHANCER | 450 #if WEBRTC_INTELLIGIBILITY_ENHANCER |
439 InitializeIntelligibility(); | 451 InitializeIntelligibility(); |
440 #endif | 452 #endif |
441 InitializeHighPassFilter(); | 453 public_submodules_->high_pass_filter->Initialize(num_proc_channels(), |
442 InitializeNoiseSuppression(); | 454 proc_sample_rate_hz()); |
443 InitializeLevelEstimator(); | 455 public_submodules_->noise_suppression->Initialize(num_proc_channels(), |
444 InitializeVoiceDetection(); | 456 proc_sample_rate_hz()); |
| 457 public_submodules_->voice_detection->Initialize(proc_split_sample_rate_hz()); |
| 458 public_submodules_->level_estimator->Initialize(); |
445 InitializeLevelController(); | 459 InitializeLevelController(); |
446 | 460 |
447 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 461 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
448 if (debug_dump_.debug_file->is_open()) { | 462 if (debug_dump_.debug_file->is_open()) { |
449 int err = WriteInitMessage(); | 463 int err = WriteInitMessage(); |
450 if (err != kNoError) { | 464 if (err != kNoError) { |
451 return err; | 465 return err; |
452 } | 466 } |
453 } | 467 } |
454 #endif | 468 #endif |
(...skipping 18 matching lines...) Expand all Loading... |
473 return kBadNumberChannelsError; | 487 return kBadNumberChannelsError; |
474 } | 488 } |
475 | 489 |
476 if (capture_nonlocked_.beamformer_enabled && | 490 if (capture_nonlocked_.beamformer_enabled && |
477 num_in_channels != capture_.array_geometry.size()) { | 491 num_in_channels != capture_.array_geometry.size()) { |
478 return kBadNumberChannelsError; | 492 return kBadNumberChannelsError; |
479 } | 493 } |
480 | 494 |
481 formats_.api_format = config; | 495 formats_.api_format = config; |
482 | 496 |
483 int fwd_proc_rate = FindNativeProcessRateToUse( | 497 int capture_processing_rate = FindNativeProcessRateToUse( |
484 std::min(formats_.api_format.input_stream().sample_rate_hz(), | 498 std::min(formats_.api_format.input_stream().sample_rate_hz(), |
485 formats_.api_format.output_stream().sample_rate_hz()), | 499 formats_.api_format.output_stream().sample_rate_hz()), |
486 submodule_states_.CaptureMultiBandSubModulesActive() || | 500 submodule_states_.CaptureMultiBandSubModulesActive() || |
487 submodule_states_.RenderMultiBandSubModulesActive()); | 501 submodule_states_.RenderMultiBandSubModulesActive()); |
488 | 502 |
489 capture_nonlocked_.fwd_proc_format = StreamConfig(fwd_proc_rate); | 503 capture_nonlocked_.capture_processing_format = |
| 504 StreamConfig(capture_processing_rate); |
490 | 505 |
491 int rev_proc_rate = FindNativeProcessRateToUse( | 506 int render_processing_rate = FindNativeProcessRateToUse( |
492 std::min(formats_.api_format.reverse_input_stream().sample_rate_hz(), | 507 std::min(formats_.api_format.reverse_input_stream().sample_rate_hz(), |
493 formats_.api_format.reverse_output_stream().sample_rate_hz()), | 508 formats_.api_format.reverse_output_stream().sample_rate_hz()), |
494 submodule_states_.CaptureMultiBandSubModulesActive() || | 509 submodule_states_.CaptureMultiBandSubModulesActive() || |
495 submodule_states_.RenderMultiBandSubModulesActive()); | 510 submodule_states_.RenderMultiBandSubModulesActive()); |
496 // TODO(aluebs): Remove this restriction once we figure out why the 3-band | 511 // TODO(aluebs): Remove this restriction once we figure out why the 3-band |
497 // splitting filter degrades the AEC performance. | 512 // splitting filter degrades the AEC performance. |
498 if (rev_proc_rate > kSampleRate32kHz) { | 513 if (render_processing_rate > kSampleRate32kHz) { |
499 rev_proc_rate = submodule_states_.RenderMultiBandProcessingActive() | 514 render_processing_rate = submodule_states_.RenderMultiBandProcessingActive() |
500 ? kSampleRate32kHz | 515 ? kSampleRate32kHz |
501 : kSampleRate16kHz; | 516 : kSampleRate16kHz; |
502 } | 517 } |
503 // If the forward sample rate is 8 kHz, the reverse stream is also processed | 518 // If the forward sample rate is 8 kHz, the render stream is also processed |
504 // at this rate. | 519 // at this rate. |
505 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate8kHz) { | 520 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() == |
506 rev_proc_rate = kSampleRate8kHz; | 521 kSampleRate8kHz) { |
| 522 render_processing_rate = kSampleRate8kHz; |
507 } else { | 523 } else { |
508 rev_proc_rate = std::max(rev_proc_rate, static_cast<int>(kSampleRate16kHz)); | 524 render_processing_rate = |
| 525 std::max(render_processing_rate, static_cast<int>(kSampleRate16kHz)); |
509 } | 526 } |
510 | 527 |
511 // Always downmix the reverse stream to mono for analysis. This has been | 528 // Always downmix the render stream to mono for analysis. This has been |
512 // demonstrated to work well for AEC in most practical scenarios. | 529 // demonstrated to work well for AEC in most practical scenarios. |
513 formats_.rev_proc_format = StreamConfig(rev_proc_rate, 1); | 530 formats_.render_processing_format = StreamConfig(render_processing_rate, 1); |
514 | 531 |
515 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate32kHz || | 532 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() == |
516 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate48kHz) { | 533 kSampleRate32kHz || |
| 534 capture_nonlocked_.capture_processing_format.sample_rate_hz() == |
| 535 kSampleRate48kHz) { |
517 capture_nonlocked_.split_rate = kSampleRate16kHz; | 536 capture_nonlocked_.split_rate = kSampleRate16kHz; |
518 } else { | 537 } else { |
519 capture_nonlocked_.split_rate = | 538 capture_nonlocked_.split_rate = |
520 capture_nonlocked_.fwd_proc_format.sample_rate_hz(); | 539 capture_nonlocked_.capture_processing_format.sample_rate_hz(); |
521 } | 540 } |
522 | 541 |
523 return InitializeLocked(); | 542 return InitializeLocked(); |
524 } | 543 } |
525 | 544 |
526 void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) { | 545 void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) { |
527 AudioProcessing::Config config_to_use = config; | 546 AudioProcessing::Config config_to_use = config; |
528 | 547 |
529 bool config_ok = LevelController::Validate(config_to_use.level_controller); | 548 bool config_ok = LevelController::Validate(config_to_use.level_controller); |
530 if (!config_ok) { | 549 if (!config_ok) { |
(...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
581 capture_.array_geometry = config.Get<Beamforming>().array_geometry; | 600 capture_.array_geometry = config.Get<Beamforming>().array_geometry; |
582 } | 601 } |
583 capture_.target_direction = config.Get<Beamforming>().target_direction; | 602 capture_.target_direction = config.Get<Beamforming>().target_direction; |
584 InitializeBeamformer(); | 603 InitializeBeamformer(); |
585 } | 604 } |
586 #endif // WEBRTC_ANDROID_PLATFORM_BUILD | 605 #endif // WEBRTC_ANDROID_PLATFORM_BUILD |
587 } | 606 } |
588 | 607 |
589 int AudioProcessingImpl::proc_sample_rate_hz() const { | 608 int AudioProcessingImpl::proc_sample_rate_hz() const { |
590 // Used as callback from submodules, hence locking is not allowed. | 609 // Used as callback from submodules, hence locking is not allowed. |
591 return capture_nonlocked_.fwd_proc_format.sample_rate_hz(); | 610 return capture_nonlocked_.capture_processing_format.sample_rate_hz(); |
592 } | 611 } |
593 | 612 |
594 int AudioProcessingImpl::proc_split_sample_rate_hz() const { | 613 int AudioProcessingImpl::proc_split_sample_rate_hz() const { |
595 // Used as callback from submodules, hence locking is not allowed. | 614 // Used as callback from submodules, hence locking is not allowed. |
596 return capture_nonlocked_.split_rate; | 615 return capture_nonlocked_.split_rate; |
597 } | 616 } |
598 | 617 |
599 size_t AudioProcessingImpl::num_reverse_channels() const { | 618 size_t AudioProcessingImpl::num_reverse_channels() const { |
600 // Used as callback from submodules, hence locking is not allowed. | 619 // Used as callback from submodules, hence locking is not allowed. |
601 return formats_.rev_proc_format.num_channels(); | 620 return formats_.render_processing_format.num_channels(); |
602 } | 621 } |
603 | 622 |
604 size_t AudioProcessingImpl::num_input_channels() const { | 623 size_t AudioProcessingImpl::num_input_channels() const { |
605 // Used as callback from submodules, hence locking is not allowed. | 624 // Used as callback from submodules, hence locking is not allowed. |
606 return formats_.api_format.input_stream().num_channels(); | 625 return formats_.api_format.input_stream().num_channels(); |
607 } | 626 } |
608 | 627 |
609 size_t AudioProcessingImpl::num_proc_channels() const { | 628 size_t AudioProcessingImpl::num_proc_channels() const { |
610 // Used as callback from submodules, hence locking is not allowed. | 629 // Used as callback from submodules, hence locking is not allowed. |
611 return capture_nonlocked_.beamformer_enabled ? 1 : num_output_channels(); | 630 return capture_nonlocked_.beamformer_enabled ? 1 : num_output_channels(); |
(...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
703 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); | 722 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
704 const size_t channel_size = | 723 const size_t channel_size = |
705 sizeof(float) * formats_.api_format.input_stream().num_frames(); | 724 sizeof(float) * formats_.api_format.input_stream().num_frames(); |
706 for (size_t i = 0; i < formats_.api_format.input_stream().num_channels(); | 725 for (size_t i = 0; i < formats_.api_format.input_stream().num_channels(); |
707 ++i) | 726 ++i) |
708 msg->add_input_channel(src[i], channel_size); | 727 msg->add_input_channel(src[i], channel_size); |
709 } | 728 } |
710 #endif | 729 #endif |
711 | 730 |
712 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream()); | 731 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream()); |
713 RETURN_ON_ERR(ProcessStreamLocked()); | 732 RETURN_ON_ERR(ProcessCaptureStreamLocked()); |
714 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest); | 733 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest); |
715 | 734 |
716 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 735 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
717 if (debug_dump_.debug_file->is_open()) { | 736 if (debug_dump_.debug_file->is_open()) { |
718 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); | 737 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
719 const size_t channel_size = | 738 const size_t channel_size = |
720 sizeof(float) * formats_.api_format.output_stream().num_frames(); | 739 sizeof(float) * formats_.api_format.output_stream().num_frames(); |
721 for (size_t i = 0; i < formats_.api_format.output_stream().num_channels(); | 740 for (size_t i = 0; i < formats_.api_format.output_stream().num_channels(); |
722 ++i) | 741 ++i) |
723 msg->add_output_channel(dest[i], channel_size); | 742 msg->add_output_channel(dest[i], channel_size); |
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
792 | 811 |
793 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); | 812 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); |
794 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); | 813 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
795 const size_t data_size = | 814 const size_t data_size = |
796 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 815 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
797 msg->set_input_data(frame->data_, data_size); | 816 msg->set_input_data(frame->data_, data_size); |
798 } | 817 } |
799 #endif | 818 #endif |
800 | 819 |
801 capture_.capture_audio->DeinterleaveFrom(frame); | 820 capture_.capture_audio->DeinterleaveFrom(frame); |
802 RETURN_ON_ERR(ProcessStreamLocked()); | 821 RETURN_ON_ERR(ProcessCaptureStreamLocked()); |
803 capture_.capture_audio->InterleaveTo( | 822 capture_.capture_audio->InterleaveTo( |
804 frame, submodule_states_.CaptureMultiBandProcessingActive()); | 823 frame, submodule_states_.CaptureMultiBandProcessingActive()); |
805 | 824 |
806 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 825 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
807 if (debug_dump_.debug_file->is_open()) { | 826 if (debug_dump_.debug_file->is_open()) { |
808 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); | 827 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
809 const size_t data_size = | 828 const size_t data_size = |
810 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 829 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
811 msg->set_output_data(frame->data_, data_size); | 830 msg->set_output_data(frame->data_, data_size); |
812 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), | 831 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
813 &debug_dump_.num_bytes_left_for_log_, | 832 &debug_dump_.num_bytes_left_for_log_, |
814 &crit_debug_, &debug_dump_.capture)); | 833 &crit_debug_, &debug_dump_.capture)); |
815 } | 834 } |
816 #endif | 835 #endif |
817 | 836 |
818 return kNoError; | 837 return kNoError; |
819 } | 838 } |
820 | 839 |
821 int AudioProcessingImpl::ProcessStreamLocked() { | 840 int AudioProcessingImpl::ProcessCaptureStreamLocked() { |
822 // Ensure that not both the AEC and AECM are active at the same time. | 841 // Ensure that not both the AEC and AECM are active at the same time. |
823 // TODO(peah): Simplify once the public API Enable functions for these | 842 // TODO(peah): Simplify once the public API Enable functions for these |
824 // are moved to APM. | 843 // are moved to APM. |
825 RTC_DCHECK(!(public_submodules_->echo_cancellation->is_enabled() && | 844 RTC_DCHECK(!(public_submodules_->echo_cancellation->is_enabled() && |
826 public_submodules_->echo_control_mobile->is_enabled())); | 845 public_submodules_->echo_control_mobile->is_enabled())); |
827 | 846 |
828 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 847 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
829 if (debug_dump_.debug_file->is_open()) { | 848 if (debug_dump_.debug_file->is_open()) { |
830 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); | 849 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
831 msg->set_delay(capture_nonlocked_.stream_delay_ms); | 850 msg->set_delay(capture_nonlocked_.stream_delay_ms); |
832 msg->set_drift( | 851 msg->set_drift( |
833 public_submodules_->echo_cancellation->stream_drift_samples()); | 852 public_submodules_->echo_cancellation->stream_drift_samples()); |
834 msg->set_level(gain_control()->stream_analog_level()); | 853 msg->set_level(gain_control()->stream_analog_level()); |
835 msg->set_keypress(capture_.key_pressed); | 854 msg->set_keypress(capture_.key_pressed); |
836 } | 855 } |
837 #endif | 856 #endif |
838 | 857 |
839 MaybeUpdateHistograms(); | 858 MaybeUpdateHistograms(); |
840 | 859 |
841 AudioBuffer* ca = capture_.capture_audio.get(); // For brevity. | 860 AudioBuffer* capture_buffer = capture_.capture_audio.get(); // For brevity. |
842 | 861 |
843 if (constants_.use_experimental_agc && | 862 if (constants_.use_experimental_agc && |
844 public_submodules_->gain_control->is_enabled()) { | 863 public_submodules_->gain_control->is_enabled()) { |
845 private_submodules_->agc_manager->AnalyzePreProcess( | 864 private_submodules_->agc_manager->AnalyzePreProcess( |
846 ca->channels()[0], ca->num_channels(), | 865 capture_buffer->channels()[0], capture_buffer->num_channels(), |
847 capture_nonlocked_.fwd_proc_format.num_frames()); | 866 capture_nonlocked_.capture_processing_format.num_frames()); |
848 } | 867 } |
849 | 868 |
850 if (submodule_states_.CaptureMultiBandSubModulesActive() && | 869 if (submodule_states_.CaptureMultiBandSubModulesActive() && |
851 SampleRateSupportsMultiBand( | 870 SampleRateSupportsMultiBand( |
852 capture_nonlocked_.fwd_proc_format.sample_rate_hz())) { | 871 capture_nonlocked_.capture_processing_format.sample_rate_hz())) { |
853 ca->SplitIntoFrequencyBands(); | 872 capture_buffer->SplitIntoFrequencyBands(); |
854 } | 873 } |
855 | 874 |
856 if (capture_nonlocked_.beamformer_enabled) { | 875 if (capture_nonlocked_.beamformer_enabled) { |
857 private_submodules_->beamformer->AnalyzeChunk(*ca->split_data_f()); | 876 private_submodules_->beamformer->AnalyzeChunk( |
| 877 *capture_buffer->split_data_f()); |
858 // Discards all channels by the leftmost one. | 878 // Discards all channels by the leftmost one. |
859 ca->set_num_channels(1); | 879 capture_buffer->set_num_channels(1); |
860 } | 880 } |
861 | 881 |
862 public_submodules_->high_pass_filter->ProcessCaptureAudio(ca); | 882 public_submodules_->high_pass_filter->ProcessCaptureAudio(capture_buffer); |
863 RETURN_ON_ERR(public_submodules_->gain_control->AnalyzeCaptureAudio(ca)); | 883 RETURN_ON_ERR( |
864 public_submodules_->noise_suppression->AnalyzeCaptureAudio(ca); | 884 public_submodules_->gain_control->AnalyzeCaptureAudio(capture_buffer)); |
| 885 public_submodules_->noise_suppression->AnalyzeCaptureAudio(capture_buffer); |
865 | 886 |
866 // Ensure that the stream delay was set before the call to the | 887 // Ensure that the stream delay was set before the call to the |
867 // AEC ProcessCaptureAudio function. | 888 // AEC ProcessCaptureAudio function. |
868 if (public_submodules_->echo_cancellation->is_enabled() && | 889 if (public_submodules_->echo_cancellation->is_enabled() && |
869 !was_stream_delay_set()) { | 890 !was_stream_delay_set()) { |
870 return AudioProcessing::kStreamParameterNotSetError; | 891 return AudioProcessing::kStreamParameterNotSetError; |
871 } | 892 } |
872 | 893 |
873 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio( | 894 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio( |
874 ca, stream_delay_ms())); | 895 capture_buffer, stream_delay_ms())); |
875 | 896 |
876 if (public_submodules_->echo_control_mobile->is_enabled() && | 897 if (public_submodules_->echo_control_mobile->is_enabled() && |
877 public_submodules_->noise_suppression->is_enabled()) { | 898 public_submodules_->noise_suppression->is_enabled()) { |
878 ca->CopyLowPassToReference(); | 899 capture_buffer->CopyLowPassToReference(); |
879 } | 900 } |
880 public_submodules_->noise_suppression->ProcessCaptureAudio(ca); | 901 public_submodules_->noise_suppression->ProcessCaptureAudio(capture_buffer); |
881 #if WEBRTC_INTELLIGIBILITY_ENHANCER | 902 #if WEBRTC_INTELLIGIBILITY_ENHANCER |
882 if (capture_nonlocked_.intelligibility_enabled) { | 903 if (capture_nonlocked_.intelligibility_enabled) { |
883 RTC_DCHECK(public_submodules_->noise_suppression->is_enabled()); | 904 RTC_DCHECK(public_submodules_->noise_suppression->is_enabled()); |
884 int gain_db = public_submodules_->gain_control->is_enabled() ? | 905 int gain_db = public_submodules_->gain_control->is_enabled() ? |
885 public_submodules_->gain_control->compression_gain_db() : | 906 public_submodules_->gain_control->compression_gain_db() : |
886 0; | 907 0; |
887 float gain = std::pow(10.f, gain_db / 20.f); | 908 float gain = std::pow(10.f, gain_db / 20.f); |
888 gain *= capture_nonlocked_.level_controller_enabled ? | 909 gain *= capture_nonlocked_.level_controller_enabled ? |
889 private_submodules_->level_controller->GetLastGain() : | 910 private_submodules_->level_controller->GetLastGain() : |
890 1.f; | 911 1.f; |
891 public_submodules_->intelligibility_enhancer->SetCaptureNoiseEstimate( | 912 public_submodules_->intelligibility_enhancer->SetCaptureNoiseEstimate( |
892 public_submodules_->noise_suppression->NoiseEstimate(), gain); | 913 public_submodules_->noise_suppression->NoiseEstimate(), gain); |
893 } | 914 } |
894 #endif | 915 #endif |
895 | 916 |
896 // Ensure that the stream delay was set before the call to the | 917 // Ensure that the stream delay was set before the call to the |
897 // AECM ProcessCaptureAudio function. | 918 // AECM ProcessCaptureAudio function. |
898 if (public_submodules_->echo_control_mobile->is_enabled() && | 919 if (public_submodules_->echo_control_mobile->is_enabled() && |
899 !was_stream_delay_set()) { | 920 !was_stream_delay_set()) { |
900 return AudioProcessing::kStreamParameterNotSetError; | 921 return AudioProcessing::kStreamParameterNotSetError; |
901 } | 922 } |
902 | 923 |
903 RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessCaptureAudio( | 924 RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessCaptureAudio( |
904 ca, stream_delay_ms())); | 925 capture_buffer, stream_delay_ms())); |
905 | 926 |
906 if (capture_nonlocked_.beamformer_enabled) { | 927 if (capture_nonlocked_.beamformer_enabled) { |
907 private_submodules_->beamformer->PostFilter(ca->split_data_f()); | 928 private_submodules_->beamformer->PostFilter(capture_buffer->split_data_f()); |
908 } | 929 } |
909 | 930 |
910 public_submodules_->voice_detection->ProcessCaptureAudio(ca); | 931 public_submodules_->voice_detection->ProcessCaptureAudio(capture_buffer); |
911 | 932 |
912 if (constants_.use_experimental_agc && | 933 if (constants_.use_experimental_agc && |
913 public_submodules_->gain_control->is_enabled() && | 934 public_submodules_->gain_control->is_enabled() && |
914 (!capture_nonlocked_.beamformer_enabled || | 935 (!capture_nonlocked_.beamformer_enabled || |
915 private_submodules_->beamformer->is_target_present())) { | 936 private_submodules_->beamformer->is_target_present())) { |
916 private_submodules_->agc_manager->Process( | 937 private_submodules_->agc_manager->Process( |
917 ca->split_bands_const(0)[kBand0To8kHz], ca->num_frames_per_band(), | 938 capture_buffer->split_bands_const(0)[kBand0To8kHz], |
918 capture_nonlocked_.split_rate); | 939 capture_buffer->num_frames_per_band(), capture_nonlocked_.split_rate); |
919 } | 940 } |
920 RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio( | 941 RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio( |
921 ca, echo_cancellation()->stream_has_echo())); | 942 capture_buffer, echo_cancellation()->stream_has_echo())); |
922 | 943 |
923 if (submodule_states_.CaptureMultiBandProcessingActive() && | 944 if (submodule_states_.CaptureMultiBandProcessingActive() && |
924 SampleRateSupportsMultiBand( | 945 SampleRateSupportsMultiBand( |
925 capture_nonlocked_.fwd_proc_format.sample_rate_hz())) { | 946 capture_nonlocked_.capture_processing_format.sample_rate_hz())) { |
926 ca->MergeFrequencyBands(); | 947 capture_buffer->MergeFrequencyBands(); |
927 } | 948 } |
928 | 949 |
929 // TODO(aluebs): Investigate if the transient suppression placement should be | 950 // TODO(aluebs): Investigate if the transient suppression placement should be |
930 // before or after the AGC. | 951 // before or after the AGC. |
931 if (capture_.transient_suppressor_enabled) { | 952 if (capture_.transient_suppressor_enabled) { |
932 float voice_probability = | 953 float voice_probability = |
933 private_submodules_->agc_manager.get() | 954 private_submodules_->agc_manager.get() |
934 ? private_submodules_->agc_manager->voice_probability() | 955 ? private_submodules_->agc_manager->voice_probability() |
935 : 1.f; | 956 : 1.f; |
936 | 957 |
937 public_submodules_->transient_suppressor->Suppress( | 958 public_submodules_->transient_suppressor->Suppress( |
938 ca->channels_f()[0], ca->num_frames(), ca->num_channels(), | 959 capture_buffer->channels_f()[0], capture_buffer->num_frames(), |
939 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(), | 960 capture_buffer->num_channels(), |
940 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability, | 961 capture_buffer->split_bands_const_f(0)[kBand0To8kHz], |
| 962 capture_buffer->num_frames_per_band(), capture_buffer->keyboard_data(), |
| 963 capture_buffer->num_keyboard_frames(), voice_probability, |
941 capture_.key_pressed); | 964 capture_.key_pressed); |
942 } | 965 } |
943 | 966 |
944 if (capture_nonlocked_.level_controller_enabled) { | 967 if (capture_nonlocked_.level_controller_enabled) { |
945 private_submodules_->level_controller->Process(ca); | 968 private_submodules_->level_controller->Process(capture_buffer); |
946 } | 969 } |
947 | 970 |
948 // The level estimator operates on the recombined data. | 971 // The level estimator operates on the recombined data. |
949 public_submodules_->level_estimator->ProcessStream(ca); | 972 public_submodules_->level_estimator->ProcessStream(capture_buffer); |
950 | 973 |
951 capture_.was_stream_delay_set = false; | 974 capture_.was_stream_delay_set = false; |
952 return kNoError; | 975 return kNoError; |
953 } | 976 } |
954 | 977 |
955 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, | 978 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, |
956 size_t samples_per_channel, | 979 size_t samples_per_channel, |
957 int rev_sample_rate_hz, | 980 int sample_rate_hz, |
958 ChannelLayout layout) { | 981 ChannelLayout layout) { |
959 TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_ChannelLayout"); | 982 TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_ChannelLayout"); |
960 rtc::CritScope cs(&crit_render_); | 983 rtc::CritScope cs(&crit_render_); |
961 const StreamConfig reverse_config = { | 984 const StreamConfig reverse_config = { |
962 rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), | 985 sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), |
963 }; | 986 }; |
964 if (samples_per_channel != reverse_config.num_frames()) { | 987 if (samples_per_channel != reverse_config.num_frames()) { |
965 return kBadDataLengthError; | 988 return kBadDataLengthError; |
966 } | 989 } |
967 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config); | 990 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config); |
968 } | 991 } |
969 | 992 |
970 int AudioProcessingImpl::ProcessReverseStream( | 993 int AudioProcessingImpl::ProcessReverseStream(const float* const* src, |
971 const float* const* src, | 994 const StreamConfig& input_config, |
972 const StreamConfig& reverse_input_config, | 995 const StreamConfig& output_config, |
973 const StreamConfig& reverse_output_config, | 996 float* const* dest) { |
974 float* const* dest) { | |
975 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig"); | 997 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig"); |
976 rtc::CritScope cs(&crit_render_); | 998 rtc::CritScope cs(&crit_render_); |
977 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, reverse_input_config, | 999 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, input_config, output_config)); |
978 reverse_output_config)); | |
979 if (submodule_states_.RenderMultiBandProcessingActive()) { | 1000 if (submodule_states_.RenderMultiBandProcessingActive()) { |
980 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(), | 1001 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(), |
981 dest); | 1002 dest); |
982 } else if (formats_.api_format.reverse_input_stream() != | 1003 } else if (formats_.api_format.reverse_input_stream() != |
983 formats_.api_format.reverse_output_stream()) { | 1004 formats_.api_format.reverse_output_stream()) { |
984 render_.render_converter->Convert(src, reverse_input_config.num_samples(), | 1005 render_.render_converter->Convert(src, input_config.num_samples(), dest, |
985 dest, | 1006 output_config.num_samples()); |
986 reverse_output_config.num_samples()); | |
987 } else { | 1007 } else { |
988 CopyAudioIfNeeded(src, reverse_input_config.num_frames(), | 1008 CopyAudioIfNeeded(src, input_config.num_frames(), |
989 reverse_input_config.num_channels(), dest); | 1009 input_config.num_channels(), dest); |
990 } | 1010 } |
991 | 1011 |
992 return kNoError; | 1012 return kNoError; |
993 } | 1013 } |
994 | 1014 |
995 int AudioProcessingImpl::AnalyzeReverseStreamLocked( | 1015 int AudioProcessingImpl::AnalyzeReverseStreamLocked( |
996 const float* const* src, | 1016 const float* const* src, |
997 const StreamConfig& reverse_input_config, | 1017 const StreamConfig& input_config, |
998 const StreamConfig& reverse_output_config) { | 1018 const StreamConfig& output_config) { |
999 if (src == nullptr) { | 1019 if (src == nullptr) { |
1000 return kNullPointerError; | 1020 return kNullPointerError; |
1001 } | 1021 } |
1002 | 1022 |
1003 if (reverse_input_config.num_channels() == 0) { | 1023 if (input_config.num_channels() == 0) { |
1004 return kBadNumberChannelsError; | 1024 return kBadNumberChannelsError; |
1005 } | 1025 } |
1006 | 1026 |
1007 ProcessingConfig processing_config = formats_.api_format; | 1027 ProcessingConfig processing_config = formats_.api_format; |
1008 processing_config.reverse_input_stream() = reverse_input_config; | 1028 processing_config.reverse_input_stream() = input_config; |
1009 processing_config.reverse_output_stream() = reverse_output_config; | 1029 processing_config.reverse_output_stream() = output_config; |
1010 | 1030 |
1011 RETURN_ON_ERR(MaybeInitializeRender(processing_config)); | 1031 RETURN_ON_ERR(MaybeInitializeRender(processing_config)); |
1012 RTC_DCHECK_EQ(reverse_input_config.num_frames(), | 1032 assert(input_config.num_frames() == |
1013 formats_.api_format.reverse_input_stream().num_frames()); | 1033 formats_.api_format.reverse_input_stream().num_frames()); |
1014 | 1034 |
1015 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1035 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
1016 if (debug_dump_.debug_file->is_open()) { | 1036 if (debug_dump_.debug_file->is_open()) { |
1017 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); | 1037 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); |
1018 audioproc::ReverseStream* msg = | 1038 audioproc::ReverseStream* msg = |
1019 debug_dump_.render.event_msg->mutable_reverse_stream(); | 1039 debug_dump_.render.event_msg->mutable_reverse_stream(); |
1020 const size_t channel_size = | 1040 const size_t channel_size = |
1021 sizeof(float) * formats_.api_format.reverse_input_stream().num_frames(); | 1041 sizeof(float) * formats_.api_format.reverse_input_stream().num_frames(); |
1022 for (size_t i = 0; | 1042 for (size_t i = 0; |
1023 i < formats_.api_format.reverse_input_stream().num_channels(); ++i) | 1043 i < formats_.api_format.reverse_input_stream().num_channels(); ++i) |
1024 msg->add_channel(src[i], channel_size); | 1044 msg->add_channel(src[i], channel_size); |
1025 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), | 1045 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
1026 &debug_dump_.num_bytes_left_for_log_, | 1046 &debug_dump_.num_bytes_left_for_log_, |
1027 &crit_debug_, &debug_dump_.render)); | 1047 &crit_debug_, &debug_dump_.render)); |
1028 } | 1048 } |
1029 #endif | 1049 #endif |
1030 | 1050 |
1031 render_.render_audio->CopyFrom(src, | 1051 render_.render_audio->CopyFrom(src, |
1032 formats_.api_format.reverse_input_stream()); | 1052 formats_.api_format.reverse_input_stream()); |
1033 return ProcessReverseStreamLocked(); | 1053 return ProcessRenderStreamLocked(); |
1034 } | 1054 } |
1035 | 1055 |
1036 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { | 1056 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { |
1037 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame"); | 1057 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame"); |
1038 rtc::CritScope cs(&crit_render_); | 1058 rtc::CritScope cs(&crit_render_); |
1039 if (frame == nullptr) { | 1059 if (frame == nullptr) { |
1040 return kNullPointerError; | 1060 return kNullPointerError; |
1041 } | 1061 } |
1042 // Must be a native rate. | 1062 // Must be a native rate. |
1043 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 1063 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
(...skipping 30 matching lines...) Expand all Loading... |
1074 debug_dump_.render.event_msg->mutable_reverse_stream(); | 1094 debug_dump_.render.event_msg->mutable_reverse_stream(); |
1075 const size_t data_size = | 1095 const size_t data_size = |
1076 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 1096 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
1077 msg->set_data(frame->data_, data_size); | 1097 msg->set_data(frame->data_, data_size); |
1078 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), | 1098 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
1079 &debug_dump_.num_bytes_left_for_log_, | 1099 &debug_dump_.num_bytes_left_for_log_, |
1080 &crit_debug_, &debug_dump_.render)); | 1100 &crit_debug_, &debug_dump_.render)); |
1081 } | 1101 } |
1082 #endif | 1102 #endif |
1083 render_.render_audio->DeinterleaveFrom(frame); | 1103 render_.render_audio->DeinterleaveFrom(frame); |
1084 RETURN_ON_ERR(ProcessReverseStreamLocked()); | 1104 RETURN_ON_ERR(ProcessRenderStreamLocked()); |
1085 render_.render_audio->InterleaveTo( | 1105 render_.render_audio->InterleaveTo( |
1086 frame, submodule_states_.RenderMultiBandProcessingActive()); | 1106 frame, submodule_states_.RenderMultiBandProcessingActive()); |
1087 return kNoError; | 1107 return kNoError; |
1088 } | 1108 } |
1089 | 1109 |
1090 int AudioProcessingImpl::ProcessReverseStreamLocked() { | 1110 int AudioProcessingImpl::ProcessRenderStreamLocked() { |
1091 AudioBuffer* ra = render_.render_audio.get(); // For brevity. | 1111 AudioBuffer* render_buffer = render_.render_audio.get(); // For brevity. |
1092 if (submodule_states_.RenderMultiBandSubModulesActive() && | 1112 if (submodule_states_.RenderMultiBandSubModulesActive() && |
1093 SampleRateSupportsMultiBand(formats_.rev_proc_format.sample_rate_hz())) { | 1113 SampleRateSupportsMultiBand( |
1094 ra->SplitIntoFrequencyBands(); | 1114 formats_.render_processing_format.sample_rate_hz())) { |
| 1115 render_buffer->SplitIntoFrequencyBands(); |
1095 } | 1116 } |
1096 | 1117 |
1097 #if WEBRTC_INTELLIGIBILITY_ENHANCER | 1118 #if WEBRTC_INTELLIGIBILITY_ENHANCER |
1098 if (capture_nonlocked_.intelligibility_enabled) { | 1119 if (capture_nonlocked_.intelligibility_enabled) { |
1099 public_submodules_->intelligibility_enhancer->ProcessRenderAudio( | 1120 public_submodules_->intelligibility_enhancer->ProcessRenderAudio( |
1100 ra->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate, | 1121 render_buffer->split_channels_f(kBand0To8kHz), |
1101 ra->num_channels()); | 1122 capture_nonlocked_.split_rate, render_buffer->num_channels()); |
1102 } | 1123 } |
1103 #endif | 1124 #endif |
1104 | 1125 |
1105 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessRenderAudio(ra)); | |
1106 RETURN_ON_ERR( | 1126 RETURN_ON_ERR( |
1107 public_submodules_->echo_control_mobile->ProcessRenderAudio(ra)); | 1127 public_submodules_->echo_cancellation->ProcessRenderAudio(render_buffer)); |
| 1128 RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessRenderAudio( |
| 1129 render_buffer)); |
1108 if (!constants_.use_experimental_agc) { | 1130 if (!constants_.use_experimental_agc) { |
1109 RETURN_ON_ERR(public_submodules_->gain_control->ProcessRenderAudio(ra)); | 1131 RETURN_ON_ERR( |
| 1132 public_submodules_->gain_control->ProcessRenderAudio(render_buffer)); |
1110 } | 1133 } |
1111 | 1134 |
1112 if (submodule_states_.RenderMultiBandProcessingActive() && | 1135 if (submodule_states_.RenderMultiBandProcessingActive() && |
1113 SampleRateSupportsMultiBand(formats_.rev_proc_format.sample_rate_hz())) { | 1136 SampleRateSupportsMultiBand( |
1114 ra->MergeFrequencyBands(); | 1137 formats_.render_processing_format.sample_rate_hz())) { |
| 1138 render_buffer->MergeFrequencyBands(); |
1115 } | 1139 } |
1116 | 1140 |
1117 return kNoError; | 1141 return kNoError; |
1118 } | 1142 } |
1119 | 1143 |
1120 int AudioProcessingImpl::set_stream_delay_ms(int delay) { | 1144 int AudioProcessingImpl::set_stream_delay_ms(int delay) { |
1121 rtc::CritScope cs(&crit_capture_); | 1145 rtc::CritScope cs(&crit_capture_); |
1122 Error retval = kNoError; | 1146 Error retval = kNoError; |
1123 capture_.was_stream_delay_set = true; | 1147 capture_.was_stream_delay_set = true; |
1124 delay += capture_.delay_offset_ms; | 1148 delay += capture_.delay_offset_ms; |
(...skipping 157 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1282 public_submodules_->noise_suppression->is_enabled(), | 1306 public_submodules_->noise_suppression->is_enabled(), |
1283 capture_nonlocked_.intelligibility_enabled, | 1307 capture_nonlocked_.intelligibility_enabled, |
1284 capture_nonlocked_.beamformer_enabled, | 1308 capture_nonlocked_.beamformer_enabled, |
1285 public_submodules_->gain_control->is_enabled(), | 1309 public_submodules_->gain_control->is_enabled(), |
1286 capture_nonlocked_.level_controller_enabled, | 1310 capture_nonlocked_.level_controller_enabled, |
1287 public_submodules_->voice_detection->is_enabled(), | 1311 public_submodules_->voice_detection->is_enabled(), |
1288 public_submodules_->level_estimator->is_enabled(), | 1312 public_submodules_->level_estimator->is_enabled(), |
1289 capture_.transient_suppressor_enabled); | 1313 capture_.transient_suppressor_enabled); |
1290 } | 1314 } |
1291 | 1315 |
1292 void AudioProcessingImpl::InitializeExperimentalAgc() { | |
1293 if (constants_.use_experimental_agc) { | |
1294 if (!private_submodules_->agc_manager.get()) { | |
1295 private_submodules_->agc_manager.reset(new AgcManagerDirect( | |
1296 public_submodules_->gain_control.get(), | |
1297 public_submodules_->gain_control_for_experimental_agc.get(), | |
1298 constants_.agc_startup_min_volume)); | |
1299 } | |
1300 private_submodules_->agc_manager->Initialize(); | |
1301 private_submodules_->agc_manager->SetCaptureMuted( | |
1302 capture_.output_will_be_muted); | |
1303 } | |
1304 } | |
1305 | 1316 |
1306 void AudioProcessingImpl::InitializeTransient() { | 1317 void AudioProcessingImpl::InitializeTransient() { |
1307 if (capture_.transient_suppressor_enabled) { | 1318 if (capture_.transient_suppressor_enabled) { |
1308 if (!public_submodules_->transient_suppressor.get()) { | 1319 if (!public_submodules_->transient_suppressor.get()) { |
1309 public_submodules_->transient_suppressor.reset(new TransientSuppressor()); | 1320 public_submodules_->transient_suppressor.reset(new TransientSuppressor()); |
1310 } | 1321 } |
1311 public_submodules_->transient_suppressor->Initialize( | 1322 public_submodules_->transient_suppressor->Initialize( |
1312 capture_nonlocked_.fwd_proc_format.sample_rate_hz(), | 1323 capture_nonlocked_.capture_processing_format.sample_rate_hz(), |
1313 capture_nonlocked_.split_rate, | 1324 capture_nonlocked_.split_rate, num_proc_channels()); |
1314 num_proc_channels()); | |
1315 } | 1325 } |
1316 } | 1326 } |
1317 | 1327 |
1318 void AudioProcessingImpl::InitializeBeamformer() { | 1328 void AudioProcessingImpl::InitializeBeamformer() { |
1319 if (capture_nonlocked_.beamformer_enabled) { | 1329 if (capture_nonlocked_.beamformer_enabled) { |
1320 if (!private_submodules_->beamformer) { | 1330 if (!private_submodules_->beamformer) { |
1321 private_submodules_->beamformer.reset(new NonlinearBeamformer( | 1331 private_submodules_->beamformer.reset(new NonlinearBeamformer( |
1322 capture_.array_geometry, 1u, capture_.target_direction)); | 1332 capture_.array_geometry, 1u, capture_.target_direction)); |
1323 } | 1333 } |
1324 private_submodules_->beamformer->Initialize(kChunkSizeMs, | 1334 private_submodules_->beamformer->Initialize(kChunkSizeMs, |
1325 capture_nonlocked_.split_rate); | 1335 capture_nonlocked_.split_rate); |
1326 } | 1336 } |
1327 } | 1337 } |
1328 | 1338 |
1329 void AudioProcessingImpl::InitializeIntelligibility() { | 1339 void AudioProcessingImpl::InitializeIntelligibility() { |
1330 #if WEBRTC_INTELLIGIBILITY_ENHANCER | 1340 #if WEBRTC_INTELLIGIBILITY_ENHANCER |
1331 if (capture_nonlocked_.intelligibility_enabled) { | 1341 if (capture_nonlocked_.intelligibility_enabled) { |
1332 public_submodules_->intelligibility_enhancer.reset( | 1342 public_submodules_->intelligibility_enhancer.reset( |
1333 new IntelligibilityEnhancer(capture_nonlocked_.split_rate, | 1343 new IntelligibilityEnhancer(capture_nonlocked_.split_rate, |
1334 render_.render_audio->num_channels(), | 1344 render_.render_audio->num_channels(), |
1335 NoiseSuppressionImpl::num_noise_bins())); | 1345 NoiseSuppressionImpl::num_noise_bins())); |
1336 } | 1346 } |
1337 #endif | 1347 #endif |
1338 } | 1348 } |
1339 | 1349 |
1340 void AudioProcessingImpl::InitializeHighPassFilter() { | |
1341 public_submodules_->high_pass_filter->Initialize(num_proc_channels(), | |
1342 proc_sample_rate_hz()); | |
1343 } | |
1344 | |
1345 void AudioProcessingImpl::InitializeNoiseSuppression() { | |
1346 public_submodules_->noise_suppression->Initialize(num_proc_channels(), | |
1347 proc_sample_rate_hz()); | |
1348 } | |
1349 | |
1350 void AudioProcessingImpl::InitializeEchoCanceller() { | |
1351 public_submodules_->echo_cancellation->Initialize( | |
1352 proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(), | |
1353 num_proc_channels()); | |
1354 } | |
1355 | |
1356 void AudioProcessingImpl::InitializeGainController() { | |
1357 public_submodules_->gain_control->Initialize(num_proc_channels(), | |
1358 proc_sample_rate_hz()); | |
1359 } | |
1360 | |
1361 void AudioProcessingImpl::InitializeEchoControlMobile() { | |
1362 public_submodules_->echo_control_mobile->Initialize( | |
1363 proc_split_sample_rate_hz(), | |
1364 num_reverse_channels(), | |
1365 num_output_channels()); | |
1366 } | |
1367 | |
1368 void AudioProcessingImpl::InitializeLevelEstimator() { | |
1369 public_submodules_->level_estimator->Initialize(); | |
1370 } | |
1371 | |
1372 void AudioProcessingImpl::InitializeLevelController() { | 1350 void AudioProcessingImpl::InitializeLevelController() { |
1373 private_submodules_->level_controller->Initialize(proc_sample_rate_hz()); | 1351 private_submodules_->level_controller->Initialize(proc_sample_rate_hz()); |
1374 } | 1352 } |
1375 | 1353 |
1376 void AudioProcessingImpl::InitializeVoiceDetection() { | |
1377 public_submodules_->voice_detection->Initialize(proc_split_sample_rate_hz()); | |
1378 } | |
1379 | |
1380 void AudioProcessingImpl::MaybeUpdateHistograms() { | 1354 void AudioProcessingImpl::MaybeUpdateHistograms() { |
1381 static const int kMinDiffDelayMs = 60; | 1355 static const int kMinDiffDelayMs = 60; |
1382 | 1356 |
1383 if (echo_cancellation()->is_enabled()) { | 1357 if (echo_cancellation()->is_enabled()) { |
1384 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. | 1358 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. |
1385 // If a stream has echo we know that the echo_cancellation is in process. | 1359 // If a stream has echo we know that the echo_cancellation is in process. |
1386 if (capture_.stream_delay_jumps == -1 && | 1360 if (capture_.stream_delay_jumps == -1 && |
1387 echo_cancellation()->stream_has_echo()) { | 1361 echo_cancellation()->stream_has_echo()) { |
1388 capture_.stream_delay_jumps = 0; | 1362 capture_.stream_delay_jumps = 0; |
1389 } | 1363 } |
(...skipping 208 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1598 delay_offset_ms(0), | 1572 delay_offset_ms(0), |
1599 was_stream_delay_set(false), | 1573 was_stream_delay_set(false), |
1600 last_stream_delay_ms(0), | 1574 last_stream_delay_ms(0), |
1601 last_aec_system_delay_ms(0), | 1575 last_aec_system_delay_ms(0), |
1602 stream_delay_jumps(-1), | 1576 stream_delay_jumps(-1), |
1603 output_will_be_muted(false), | 1577 output_will_be_muted(false), |
1604 key_pressed(false), | 1578 key_pressed(false), |
1605 transient_suppressor_enabled(transient_suppressor_enabled), | 1579 transient_suppressor_enabled(transient_suppressor_enabled), |
1606 array_geometry(array_geometry), | 1580 array_geometry(array_geometry), |
1607 target_direction(target_direction), | 1581 target_direction(target_direction), |
1608 fwd_proc_format(kSampleRate16kHz), | 1582 capture_processing_format(kSampleRate16kHz), |
1609 split_rate(kSampleRate16kHz) {} | 1583 split_rate(kSampleRate16kHz) {} |
1610 | 1584 |
1611 AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default; | 1585 AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default; |
1612 | 1586 |
1613 AudioProcessingImpl::ApmRenderState::ApmRenderState() = default; | 1587 AudioProcessingImpl::ApmRenderState::ApmRenderState() = default; |
1614 | 1588 |
1615 AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default; | 1589 AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default; |
1616 | 1590 |
1617 } // namespace webrtc | 1591 } // namespace webrtc |
OLD | NEW |