| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 * | 3 * |
| 4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ | 9 */ |
| 10 | 10 |
| (...skipping 305 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 316 #endif | 316 #endif |
| 317 } | 317 } |
| 318 | 318 |
| 319 int AudioProcessingImpl::Initialize() { | 319 int AudioProcessingImpl::Initialize() { |
| 320 // Run in a single-threaded manner during initialization. | 320 // Run in a single-threaded manner during initialization. |
| 321 rtc::CritScope cs_render(&crit_render_); | 321 rtc::CritScope cs_render(&crit_render_); |
| 322 rtc::CritScope cs_capture(&crit_capture_); | 322 rtc::CritScope cs_capture(&crit_capture_); |
| 323 return InitializeLocked(); | 323 return InitializeLocked(); |
| 324 } | 324 } |
| 325 | 325 |
| 326 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, | 326 int AudioProcessingImpl::Initialize(int capture_input_sample_rate_hz, |
| 327 int output_sample_rate_hz, | 327 int capture_output_sample_rate_hz, |
| 328 int reverse_sample_rate_hz, | 328 int render_input_sample_rate_hz, |
| 329 ChannelLayout input_layout, | 329 ChannelLayout capture_input_layout, |
| 330 ChannelLayout output_layout, | 330 ChannelLayout capture_output_layout, |
| 331 ChannelLayout reverse_layout) { | 331 ChannelLayout render_input_layout) { |
| 332 const ProcessingConfig processing_config = { | 332 const ProcessingConfig processing_config = { |
| 333 {{input_sample_rate_hz, | 333 {{capture_input_sample_rate_hz, ChannelsFromLayout(capture_input_layout), |
| 334 ChannelsFromLayout(input_layout), | 334 LayoutHasKeyboard(capture_input_layout)}, |
| 335 LayoutHasKeyboard(input_layout)}, | 335 {capture_output_sample_rate_hz, |
| 336 {output_sample_rate_hz, | 336 ChannelsFromLayout(capture_output_layout), |
| 337 ChannelsFromLayout(output_layout), | 337 LayoutHasKeyboard(capture_output_layout)}, |
| 338 LayoutHasKeyboard(output_layout)}, | 338 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout), |
| 339 {reverse_sample_rate_hz, | 339 LayoutHasKeyboard(render_input_layout)}, |
| 340 ChannelsFromLayout(reverse_layout), | 340 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout), |
| 341 LayoutHasKeyboard(reverse_layout)}, | 341 LayoutHasKeyboard(render_input_layout)}}}; |
| 342 {reverse_sample_rate_hz, | |
| 343 ChannelsFromLayout(reverse_layout), | |
| 344 LayoutHasKeyboard(reverse_layout)}}}; | |
| 345 | 342 |
| 346 return Initialize(processing_config); | 343 return Initialize(processing_config); |
| 347 } | 344 } |
| 348 | 345 |
| 349 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { | 346 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { |
| 350 // Run in a single-threaded manner during initialization. | 347 // Run in a single-threaded manner during initialization. |
| 351 rtc::CritScope cs_render(&crit_render_); | 348 rtc::CritScope cs_render(&crit_render_); |
| 352 rtc::CritScope cs_capture(&crit_capture_); | 349 rtc::CritScope cs_capture(&crit_capture_); |
| 353 return InitializeLocked(processing_config); | 350 return InitializeLocked(processing_config); |
| 354 } | 351 } |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 386 // Called from both threads. Thread check is therefore not possible. | 383 // Called from both threads. Thread check is therefore not possible. |
| 387 if (processing_config == formats_.api_format && !force_initialization) { | 384 if (processing_config == formats_.api_format && !force_initialization) { |
| 388 return kNoError; | 385 return kNoError; |
| 389 } | 386 } |
| 390 | 387 |
| 391 rtc::CritScope cs_capture(&crit_capture_); | 388 rtc::CritScope cs_capture(&crit_capture_); |
| 392 return InitializeLocked(processing_config); | 389 return InitializeLocked(processing_config); |
| 393 } | 390 } |
| 394 | 391 |
| 395 int AudioProcessingImpl::InitializeLocked() { | 392 int AudioProcessingImpl::InitializeLocked() { |
| 396 const int fwd_audio_buffer_channels = | 393 const int capture_audiobuffer_num_channels = |
| 397 capture_nonlocked_.beamformer_enabled | 394 capture_nonlocked_.beamformer_enabled |
| 398 ? formats_.api_format.input_stream().num_channels() | 395 ? formats_.api_format.input_stream().num_channels() |
| 399 : formats_.api_format.output_stream().num_channels(); | 396 : formats_.api_format.output_stream().num_channels(); |
| 400 const int rev_audio_buffer_out_num_frames = | 397 const int render_audiobuffer_num_output_frames = |
| 401 formats_.api_format.reverse_output_stream().num_frames() == 0 | 398 formats_.api_format.reverse_output_stream().num_frames() == 0 |
| 402 ? formats_.rev_proc_format.num_frames() | 399 ? formats_.render_processing_format.num_frames() |
| 403 : formats_.api_format.reverse_output_stream().num_frames(); | 400 : formats_.api_format.reverse_output_stream().num_frames(); |
| 404 if (formats_.api_format.reverse_input_stream().num_channels() > 0) { | 401 if (formats_.api_format.reverse_input_stream().num_channels() > 0) { |
| 405 render_.render_audio.reset(new AudioBuffer( | 402 render_.render_audio.reset(new AudioBuffer( |
| 406 formats_.api_format.reverse_input_stream().num_frames(), | 403 formats_.api_format.reverse_input_stream().num_frames(), |
| 407 formats_.api_format.reverse_input_stream().num_channels(), | 404 formats_.api_format.reverse_input_stream().num_channels(), |
| 408 formats_.rev_proc_format.num_frames(), | 405 formats_.render_processing_format.num_frames(), |
| 409 formats_.rev_proc_format.num_channels(), | 406 formats_.render_processing_format.num_channels(), |
| 410 rev_audio_buffer_out_num_frames)); | 407 render_audiobuffer_num_output_frames)); |
| 411 if (formats_.api_format.reverse_input_stream() != | 408 if (formats_.api_format.reverse_input_stream() != |
| 412 formats_.api_format.reverse_output_stream()) { | 409 formats_.api_format.reverse_output_stream()) { |
| 413 render_.render_converter = AudioConverter::Create( | 410 render_.render_converter = AudioConverter::Create( |
| 414 formats_.api_format.reverse_input_stream().num_channels(), | 411 formats_.api_format.reverse_input_stream().num_channels(), |
| 415 formats_.api_format.reverse_input_stream().num_frames(), | 412 formats_.api_format.reverse_input_stream().num_frames(), |
| 416 formats_.api_format.reverse_output_stream().num_channels(), | 413 formats_.api_format.reverse_output_stream().num_channels(), |
| 417 formats_.api_format.reverse_output_stream().num_frames()); | 414 formats_.api_format.reverse_output_stream().num_frames()); |
| 418 } else { | 415 } else { |
| 419 render_.render_converter.reset(nullptr); | 416 render_.render_converter.reset(nullptr); |
| 420 } | 417 } |
| 421 } else { | 418 } else { |
| 422 render_.render_audio.reset(nullptr); | 419 render_.render_audio.reset(nullptr); |
| 423 render_.render_converter.reset(nullptr); | 420 render_.render_converter.reset(nullptr); |
| 424 } | 421 } |
| 425 capture_.capture_audio.reset( | 422 capture_.capture_audio.reset( |
| 426 new AudioBuffer(formats_.api_format.input_stream().num_frames(), | 423 new AudioBuffer(formats_.api_format.input_stream().num_frames(), |
| 427 formats_.api_format.input_stream().num_channels(), | 424 formats_.api_format.input_stream().num_channels(), |
| 428 capture_nonlocked_.fwd_proc_format.num_frames(), | 425 capture_nonlocked_.capture_processing_format.num_frames(), |
| 429 fwd_audio_buffer_channels, | 426 capture_audiobuffer_num_channels, |
| 430 formats_.api_format.output_stream().num_frames())); | 427 formats_.api_format.output_stream().num_frames())); |
| 431 | 428 |
| 432 InitializeGainController(); | 429 public_submodules_->gain_control->Initialize(num_proc_channels(), |
| 433 InitializeEchoCanceller(); | 430 proc_sample_rate_hz()); |
| 434 InitializeEchoControlMobile(); | 431 public_submodules_->echo_cancellation->Initialize( |
| 435 InitializeExperimentalAgc(); | 432 proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(), |
| 433 num_proc_channels()); |
| 434 public_submodules_->echo_control_mobile->Initialize( |
| 435 proc_split_sample_rate_hz(), num_reverse_channels(), |
| 436 num_output_channels()); |
| 437 if (constants_.use_experimental_agc) { |
| 438 if (!private_submodules_->agc_manager.get()) { |
| 439 private_submodules_->agc_manager.reset(new AgcManagerDirect( |
| 440 public_submodules_->gain_control.get(), |
| 441 public_submodules_->gain_control_for_experimental_agc.get(), |
| 442 constants_.agc_startup_min_volume)); |
| 443 } |
| 444 private_submodules_->agc_manager->Initialize(); |
| 445 private_submodules_->agc_manager->SetCaptureMuted( |
| 446 capture_.output_will_be_muted); |
| 447 } |
| 436 InitializeTransient(); | 448 InitializeTransient(); |
| 437 InitializeBeamformer(); | 449 InitializeBeamformer(); |
| 438 #if WEBRTC_INTELLIGIBILITY_ENHANCER | 450 #if WEBRTC_INTELLIGIBILITY_ENHANCER |
| 439 InitializeIntelligibility(); | 451 InitializeIntelligibility(); |
| 440 #endif | 452 #endif |
| 441 InitializeHighPassFilter(); | 453 public_submodules_->high_pass_filter->Initialize(num_proc_channels(), |
| 442 InitializeNoiseSuppression(); | 454 proc_sample_rate_hz()); |
| 443 InitializeLevelEstimator(); | 455 public_submodules_->noise_suppression->Initialize(num_proc_channels(), |
| 444 InitializeVoiceDetection(); | 456 proc_sample_rate_hz()); |
| 457 public_submodules_->voice_detection->Initialize(proc_split_sample_rate_hz()); |
| 458 public_submodules_->level_estimator->Initialize(); |
| 445 InitializeLevelController(); | 459 InitializeLevelController(); |
| 446 | 460 |
| 447 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 461 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 448 if (debug_dump_.debug_file->is_open()) { | 462 if (debug_dump_.debug_file->is_open()) { |
| 449 int err = WriteInitMessage(); | 463 int err = WriteInitMessage(); |
| 450 if (err != kNoError) { | 464 if (err != kNoError) { |
| 451 return err; | 465 return err; |
| 452 } | 466 } |
| 453 } | 467 } |
| 454 #endif | 468 #endif |
| (...skipping 18 matching lines...) Expand all Loading... |
| 473 return kBadNumberChannelsError; | 487 return kBadNumberChannelsError; |
| 474 } | 488 } |
| 475 | 489 |
| 476 if (capture_nonlocked_.beamformer_enabled && | 490 if (capture_nonlocked_.beamformer_enabled && |
| 477 num_in_channels != capture_.array_geometry.size()) { | 491 num_in_channels != capture_.array_geometry.size()) { |
| 478 return kBadNumberChannelsError; | 492 return kBadNumberChannelsError; |
| 479 } | 493 } |
| 480 | 494 |
| 481 formats_.api_format = config; | 495 formats_.api_format = config; |
| 482 | 496 |
| 483 int fwd_proc_rate = FindNativeProcessRateToUse( | 497 int capture_processing_rate = FindNativeProcessRateToUse( |
| 484 std::min(formats_.api_format.input_stream().sample_rate_hz(), | 498 std::min(formats_.api_format.input_stream().sample_rate_hz(), |
| 485 formats_.api_format.output_stream().sample_rate_hz()), | 499 formats_.api_format.output_stream().sample_rate_hz()), |
| 486 submodule_states_.CaptureMultiBandSubModulesActive() || | 500 submodule_states_.CaptureMultiBandSubModulesActive() || |
| 487 submodule_states_.RenderMultiBandSubModulesActive()); | 501 submodule_states_.RenderMultiBandSubModulesActive()); |
| 488 | 502 |
| 489 capture_nonlocked_.fwd_proc_format = StreamConfig(fwd_proc_rate); | 503 capture_nonlocked_.capture_processing_format = |
| 504 StreamConfig(capture_processing_rate); |
| 490 | 505 |
| 491 int rev_proc_rate = FindNativeProcessRateToUse( | 506 int render_processing_rate = FindNativeProcessRateToUse( |
| 492 std::min(formats_.api_format.reverse_input_stream().sample_rate_hz(), | 507 std::min(formats_.api_format.reverse_input_stream().sample_rate_hz(), |
| 493 formats_.api_format.reverse_output_stream().sample_rate_hz()), | 508 formats_.api_format.reverse_output_stream().sample_rate_hz()), |
| 494 submodule_states_.CaptureMultiBandSubModulesActive() || | 509 submodule_states_.CaptureMultiBandSubModulesActive() || |
| 495 submodule_states_.RenderMultiBandSubModulesActive()); | 510 submodule_states_.RenderMultiBandSubModulesActive()); |
| 496 // TODO(aluebs): Remove this restriction once we figure out why the 3-band | 511 // TODO(aluebs): Remove this restriction once we figure out why the 3-band |
| 497 // splitting filter degrades the AEC performance. | 512 // splitting filter degrades the AEC performance. |
| 498 if (rev_proc_rate > kSampleRate32kHz) { | 513 if (render_processing_rate > kSampleRate32kHz) { |
| 499 rev_proc_rate = submodule_states_.RenderMultiBandProcessingActive() | 514 render_processing_rate = submodule_states_.RenderMultiBandProcessingActive() |
| 500 ? kSampleRate32kHz | 515 ? kSampleRate32kHz |
| 501 : kSampleRate16kHz; | 516 : kSampleRate16kHz; |
| 502 } | 517 } |
| 503 // If the forward sample rate is 8 kHz, the reverse stream is also processed | 518 // If the forward sample rate is 8 kHz, the render stream is also processed |
| 504 // at this rate. | 519 // at this rate. |
| 505 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate8kHz) { | 520 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() == |
| 506 rev_proc_rate = kSampleRate8kHz; | 521 kSampleRate8kHz) { |
| 522 render_processing_rate = kSampleRate8kHz; |
| 507 } else { | 523 } else { |
| 508 rev_proc_rate = std::max(rev_proc_rate, static_cast<int>(kSampleRate16kHz)); | 524 render_processing_rate = |
| 525 std::max(render_processing_rate, static_cast<int>(kSampleRate16kHz)); |
| 509 } | 526 } |
| 510 | 527 |
| 511 // Always downmix the reverse stream to mono for analysis. This has been | 528 // Always downmix the render stream to mono for analysis. This has been |
| 512 // demonstrated to work well for AEC in most practical scenarios. | 529 // demonstrated to work well for AEC in most practical scenarios. |
| 513 formats_.rev_proc_format = StreamConfig(rev_proc_rate, 1); | 530 formats_.render_processing_format = StreamConfig(render_processing_rate, 1); |
| 514 | 531 |
| 515 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate32kHz || | 532 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() == |
| 516 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate48kHz) { | 533 kSampleRate32kHz || |
| 534 capture_nonlocked_.capture_processing_format.sample_rate_hz() == |
| 535 kSampleRate48kHz) { |
| 517 capture_nonlocked_.split_rate = kSampleRate16kHz; | 536 capture_nonlocked_.split_rate = kSampleRate16kHz; |
| 518 } else { | 537 } else { |
| 519 capture_nonlocked_.split_rate = | 538 capture_nonlocked_.split_rate = |
| 520 capture_nonlocked_.fwd_proc_format.sample_rate_hz(); | 539 capture_nonlocked_.capture_processing_format.sample_rate_hz(); |
| 521 } | 540 } |
| 522 | 541 |
| 523 return InitializeLocked(); | 542 return InitializeLocked(); |
| 524 } | 543 } |
| 525 | 544 |
| 526 void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) { | 545 void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) { |
| 527 AudioProcessing::Config config_to_use = config; | 546 AudioProcessing::Config config_to_use = config; |
| 528 | 547 |
| 529 bool config_ok = LevelController::Validate(config_to_use.level_controller); | 548 bool config_ok = LevelController::Validate(config_to_use.level_controller); |
| 530 if (!config_ok) { | 549 if (!config_ok) { |
| (...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 581 capture_.array_geometry = config.Get<Beamforming>().array_geometry; | 600 capture_.array_geometry = config.Get<Beamforming>().array_geometry; |
| 582 } | 601 } |
| 583 capture_.target_direction = config.Get<Beamforming>().target_direction; | 602 capture_.target_direction = config.Get<Beamforming>().target_direction; |
| 584 InitializeBeamformer(); | 603 InitializeBeamformer(); |
| 585 } | 604 } |
| 586 #endif // WEBRTC_ANDROID_PLATFORM_BUILD | 605 #endif // WEBRTC_ANDROID_PLATFORM_BUILD |
| 587 } | 606 } |
| 588 | 607 |
| 589 int AudioProcessingImpl::proc_sample_rate_hz() const { | 608 int AudioProcessingImpl::proc_sample_rate_hz() const { |
| 590 // Used as callback from submodules, hence locking is not allowed. | 609 // Used as callback from submodules, hence locking is not allowed. |
| 591 return capture_nonlocked_.fwd_proc_format.sample_rate_hz(); | 610 return capture_nonlocked_.capture_processing_format.sample_rate_hz(); |
| 592 } | 611 } |
| 593 | 612 |
| 594 int AudioProcessingImpl::proc_split_sample_rate_hz() const { | 613 int AudioProcessingImpl::proc_split_sample_rate_hz() const { |
| 595 // Used as callback from submodules, hence locking is not allowed. | 614 // Used as callback from submodules, hence locking is not allowed. |
| 596 return capture_nonlocked_.split_rate; | 615 return capture_nonlocked_.split_rate; |
| 597 } | 616 } |
| 598 | 617 |
| 599 size_t AudioProcessingImpl::num_reverse_channels() const { | 618 size_t AudioProcessingImpl::num_reverse_channels() const { |
| 600 // Used as callback from submodules, hence locking is not allowed. | 619 // Used as callback from submodules, hence locking is not allowed. |
| 601 return formats_.rev_proc_format.num_channels(); | 620 return formats_.render_processing_format.num_channels(); |
| 602 } | 621 } |
| 603 | 622 |
| 604 size_t AudioProcessingImpl::num_input_channels() const { | 623 size_t AudioProcessingImpl::num_input_channels() const { |
| 605 // Used as callback from submodules, hence locking is not allowed. | 624 // Used as callback from submodules, hence locking is not allowed. |
| 606 return formats_.api_format.input_stream().num_channels(); | 625 return formats_.api_format.input_stream().num_channels(); |
| 607 } | 626 } |
| 608 | 627 |
| 609 size_t AudioProcessingImpl::num_proc_channels() const { | 628 size_t AudioProcessingImpl::num_proc_channels() const { |
| 610 // Used as callback from submodules, hence locking is not allowed. | 629 // Used as callback from submodules, hence locking is not allowed. |
| 611 return capture_nonlocked_.beamformer_enabled ? 1 : num_output_channels(); | 630 return capture_nonlocked_.beamformer_enabled ? 1 : num_output_channels(); |
| (...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 703 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); | 722 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
| 704 const size_t channel_size = | 723 const size_t channel_size = |
| 705 sizeof(float) * formats_.api_format.input_stream().num_frames(); | 724 sizeof(float) * formats_.api_format.input_stream().num_frames(); |
| 706 for (size_t i = 0; i < formats_.api_format.input_stream().num_channels(); | 725 for (size_t i = 0; i < formats_.api_format.input_stream().num_channels(); |
| 707 ++i) | 726 ++i) |
| 708 msg->add_input_channel(src[i], channel_size); | 727 msg->add_input_channel(src[i], channel_size); |
| 709 } | 728 } |
| 710 #endif | 729 #endif |
| 711 | 730 |
| 712 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream()); | 731 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream()); |
| 713 RETURN_ON_ERR(ProcessStreamLocked()); | 732 RETURN_ON_ERR(ProcessCaptureStreamLocked()); |
| 714 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest); | 733 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest); |
| 715 | 734 |
| 716 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 735 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 717 if (debug_dump_.debug_file->is_open()) { | 736 if (debug_dump_.debug_file->is_open()) { |
| 718 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); | 737 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
| 719 const size_t channel_size = | 738 const size_t channel_size = |
| 720 sizeof(float) * formats_.api_format.output_stream().num_frames(); | 739 sizeof(float) * formats_.api_format.output_stream().num_frames(); |
| 721 for (size_t i = 0; i < formats_.api_format.output_stream().num_channels(); | 740 for (size_t i = 0; i < formats_.api_format.output_stream().num_channels(); |
| 722 ++i) | 741 ++i) |
| 723 msg->add_output_channel(dest[i], channel_size); | 742 msg->add_output_channel(dest[i], channel_size); |
| (...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 792 | 811 |
| 793 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); | 812 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); |
| 794 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); | 813 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
| 795 const size_t data_size = | 814 const size_t data_size = |
| 796 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 815 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
| 797 msg->set_input_data(frame->data_, data_size); | 816 msg->set_input_data(frame->data_, data_size); |
| 798 } | 817 } |
| 799 #endif | 818 #endif |
| 800 | 819 |
| 801 capture_.capture_audio->DeinterleaveFrom(frame); | 820 capture_.capture_audio->DeinterleaveFrom(frame); |
| 802 RETURN_ON_ERR(ProcessStreamLocked()); | 821 RETURN_ON_ERR(ProcessCaptureStreamLocked()); |
| 803 capture_.capture_audio->InterleaveTo( | 822 capture_.capture_audio->InterleaveTo( |
| 804 frame, submodule_states_.CaptureMultiBandProcessingActive()); | 823 frame, submodule_states_.CaptureMultiBandProcessingActive()); |
| 805 | 824 |
| 806 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 825 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 807 if (debug_dump_.debug_file->is_open()) { | 826 if (debug_dump_.debug_file->is_open()) { |
| 808 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); | 827 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
| 809 const size_t data_size = | 828 const size_t data_size = |
| 810 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 829 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
| 811 msg->set_output_data(frame->data_, data_size); | 830 msg->set_output_data(frame->data_, data_size); |
| 812 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), | 831 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 813 &debug_dump_.num_bytes_left_for_log_, | 832 &debug_dump_.num_bytes_left_for_log_, |
| 814 &crit_debug_, &debug_dump_.capture)); | 833 &crit_debug_, &debug_dump_.capture)); |
| 815 } | 834 } |
| 816 #endif | 835 #endif |
| 817 | 836 |
| 818 return kNoError; | 837 return kNoError; |
| 819 } | 838 } |
| 820 | 839 |
| 821 int AudioProcessingImpl::ProcessStreamLocked() { | 840 int AudioProcessingImpl::ProcessCaptureStreamLocked() { |
| 822 // Ensure that not both the AEC and AECM are active at the same time. | 841 // Ensure that not both the AEC and AECM are active at the same time. |
| 823 // TODO(peah): Simplify once the public API Enable functions for these | 842 // TODO(peah): Simplify once the public API Enable functions for these |
| 824 // are moved to APM. | 843 // are moved to APM. |
| 825 RTC_DCHECK(!(public_submodules_->echo_cancellation->is_enabled() && | 844 RTC_DCHECK(!(public_submodules_->echo_cancellation->is_enabled() && |
| 826 public_submodules_->echo_control_mobile->is_enabled())); | 845 public_submodules_->echo_control_mobile->is_enabled())); |
| 827 | 846 |
| 828 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 847 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 829 if (debug_dump_.debug_file->is_open()) { | 848 if (debug_dump_.debug_file->is_open()) { |
| 830 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); | 849 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
| 831 msg->set_delay(capture_nonlocked_.stream_delay_ms); | 850 msg->set_delay(capture_nonlocked_.stream_delay_ms); |
| 832 msg->set_drift( | 851 msg->set_drift( |
| 833 public_submodules_->echo_cancellation->stream_drift_samples()); | 852 public_submodules_->echo_cancellation->stream_drift_samples()); |
| 834 msg->set_level(gain_control()->stream_analog_level()); | 853 msg->set_level(gain_control()->stream_analog_level()); |
| 835 msg->set_keypress(capture_.key_pressed); | 854 msg->set_keypress(capture_.key_pressed); |
| 836 } | 855 } |
| 837 #endif | 856 #endif |
| 838 | 857 |
| 839 MaybeUpdateHistograms(); | 858 MaybeUpdateHistograms(); |
| 840 | 859 |
| 841 AudioBuffer* ca = capture_.capture_audio.get(); // For brevity. | 860 AudioBuffer* capture_buffer = capture_.capture_audio.get(); // For brevity. |
| 842 | 861 |
| 843 if (constants_.use_experimental_agc && | 862 if (constants_.use_experimental_agc && |
| 844 public_submodules_->gain_control->is_enabled()) { | 863 public_submodules_->gain_control->is_enabled()) { |
| 845 private_submodules_->agc_manager->AnalyzePreProcess( | 864 private_submodules_->agc_manager->AnalyzePreProcess( |
| 846 ca->channels()[0], ca->num_channels(), | 865 capture_buffer->channels()[0], capture_buffer->num_channels(), |
| 847 capture_nonlocked_.fwd_proc_format.num_frames()); | 866 capture_nonlocked_.capture_processing_format.num_frames()); |
| 848 } | 867 } |
| 849 | 868 |
| 850 if (submodule_states_.CaptureMultiBandSubModulesActive() && | 869 if (submodule_states_.CaptureMultiBandSubModulesActive() && |
| 851 SampleRateSupportsMultiBand( | 870 SampleRateSupportsMultiBand( |
| 852 capture_nonlocked_.fwd_proc_format.sample_rate_hz())) { | 871 capture_nonlocked_.capture_processing_format.sample_rate_hz())) { |
| 853 ca->SplitIntoFrequencyBands(); | 872 capture_buffer->SplitIntoFrequencyBands(); |
| 854 } | 873 } |
| 855 | 874 |
| 856 if (capture_nonlocked_.beamformer_enabled) { | 875 if (capture_nonlocked_.beamformer_enabled) { |
| 857 private_submodules_->beamformer->AnalyzeChunk(*ca->split_data_f()); | 876 private_submodules_->beamformer->AnalyzeChunk( |
| 877 *capture_buffer->split_data_f()); |
| 858 // Discards all channels by the leftmost one. | 878 // Discards all channels by the leftmost one. |
| 859 ca->set_num_channels(1); | 879 capture_buffer->set_num_channels(1); |
| 860 } | 880 } |
| 861 | 881 |
| 862 public_submodules_->high_pass_filter->ProcessCaptureAudio(ca); | 882 public_submodules_->high_pass_filter->ProcessCaptureAudio(capture_buffer); |
| 863 RETURN_ON_ERR(public_submodules_->gain_control->AnalyzeCaptureAudio(ca)); | 883 RETURN_ON_ERR( |
| 864 public_submodules_->noise_suppression->AnalyzeCaptureAudio(ca); | 884 public_submodules_->gain_control->AnalyzeCaptureAudio(capture_buffer)); |
| 885 public_submodules_->noise_suppression->AnalyzeCaptureAudio(capture_buffer); |
| 865 | 886 |
| 866 // Ensure that the stream delay was set before the call to the | 887 // Ensure that the stream delay was set before the call to the |
| 867 // AEC ProcessCaptureAudio function. | 888 // AEC ProcessCaptureAudio function. |
| 868 if (public_submodules_->echo_cancellation->is_enabled() && | 889 if (public_submodules_->echo_cancellation->is_enabled() && |
| 869 !was_stream_delay_set()) { | 890 !was_stream_delay_set()) { |
| 870 return AudioProcessing::kStreamParameterNotSetError; | 891 return AudioProcessing::kStreamParameterNotSetError; |
| 871 } | 892 } |
| 872 | 893 |
| 873 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio( | 894 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio( |
| 874 ca, stream_delay_ms())); | 895 capture_buffer, stream_delay_ms())); |
| 875 | 896 |
| 876 if (public_submodules_->echo_control_mobile->is_enabled() && | 897 if (public_submodules_->echo_control_mobile->is_enabled() && |
| 877 public_submodules_->noise_suppression->is_enabled()) { | 898 public_submodules_->noise_suppression->is_enabled()) { |
| 878 ca->CopyLowPassToReference(); | 899 capture_buffer->CopyLowPassToReference(); |
| 879 } | 900 } |
| 880 public_submodules_->noise_suppression->ProcessCaptureAudio(ca); | 901 public_submodules_->noise_suppression->ProcessCaptureAudio(capture_buffer); |
| 881 #if WEBRTC_INTELLIGIBILITY_ENHANCER | 902 #if WEBRTC_INTELLIGIBILITY_ENHANCER |
| 882 if (capture_nonlocked_.intelligibility_enabled) { | 903 if (capture_nonlocked_.intelligibility_enabled) { |
| 883 RTC_DCHECK(public_submodules_->noise_suppression->is_enabled()); | 904 RTC_DCHECK(public_submodules_->noise_suppression->is_enabled()); |
| 884 int gain_db = public_submodules_->gain_control->is_enabled() ? | 905 int gain_db = public_submodules_->gain_control->is_enabled() ? |
| 885 public_submodules_->gain_control->compression_gain_db() : | 906 public_submodules_->gain_control->compression_gain_db() : |
| 886 0; | 907 0; |
| 887 float gain = std::pow(10.f, gain_db / 20.f); | 908 float gain = std::pow(10.f, gain_db / 20.f); |
| 888 gain *= capture_nonlocked_.level_controller_enabled ? | 909 gain *= capture_nonlocked_.level_controller_enabled ? |
| 889 private_submodules_->level_controller->GetLastGain() : | 910 private_submodules_->level_controller->GetLastGain() : |
| 890 1.f; | 911 1.f; |
| 891 public_submodules_->intelligibility_enhancer->SetCaptureNoiseEstimate( | 912 public_submodules_->intelligibility_enhancer->SetCaptureNoiseEstimate( |
| 892 public_submodules_->noise_suppression->NoiseEstimate(), gain); | 913 public_submodules_->noise_suppression->NoiseEstimate(), gain); |
| 893 } | 914 } |
| 894 #endif | 915 #endif |
| 895 | 916 |
| 896 // Ensure that the stream delay was set before the call to the | 917 // Ensure that the stream delay was set before the call to the |
| 897 // AECM ProcessCaptureAudio function. | 918 // AECM ProcessCaptureAudio function. |
| 898 if (public_submodules_->echo_control_mobile->is_enabled() && | 919 if (public_submodules_->echo_control_mobile->is_enabled() && |
| 899 !was_stream_delay_set()) { | 920 !was_stream_delay_set()) { |
| 900 return AudioProcessing::kStreamParameterNotSetError; | 921 return AudioProcessing::kStreamParameterNotSetError; |
| 901 } | 922 } |
| 902 | 923 |
| 903 RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessCaptureAudio( | 924 RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessCaptureAudio( |
| 904 ca, stream_delay_ms())); | 925 capture_buffer, stream_delay_ms())); |
| 905 | 926 |
| 906 if (capture_nonlocked_.beamformer_enabled) { | 927 if (capture_nonlocked_.beamformer_enabled) { |
| 907 private_submodules_->beamformer->PostFilter(ca->split_data_f()); | 928 private_submodules_->beamformer->PostFilter(capture_buffer->split_data_f()); |
| 908 } | 929 } |
| 909 | 930 |
| 910 public_submodules_->voice_detection->ProcessCaptureAudio(ca); | 931 public_submodules_->voice_detection->ProcessCaptureAudio(capture_buffer); |
| 911 | 932 |
| 912 if (constants_.use_experimental_agc && | 933 if (constants_.use_experimental_agc && |
| 913 public_submodules_->gain_control->is_enabled() && | 934 public_submodules_->gain_control->is_enabled() && |
| 914 (!capture_nonlocked_.beamformer_enabled || | 935 (!capture_nonlocked_.beamformer_enabled || |
| 915 private_submodules_->beamformer->is_target_present())) { | 936 private_submodules_->beamformer->is_target_present())) { |
| 916 private_submodules_->agc_manager->Process( | 937 private_submodules_->agc_manager->Process( |
| 917 ca->split_bands_const(0)[kBand0To8kHz], ca->num_frames_per_band(), | 938 capture_buffer->split_bands_const(0)[kBand0To8kHz], |
| 918 capture_nonlocked_.split_rate); | 939 capture_buffer->num_frames_per_band(), capture_nonlocked_.split_rate); |
| 919 } | 940 } |
| 920 RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio( | 941 RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio( |
| 921 ca, echo_cancellation()->stream_has_echo())); | 942 capture_buffer, echo_cancellation()->stream_has_echo())); |
| 922 | 943 |
| 923 if (submodule_states_.CaptureMultiBandProcessingActive() && | 944 if (submodule_states_.CaptureMultiBandProcessingActive() && |
| 924 SampleRateSupportsMultiBand( | 945 SampleRateSupportsMultiBand( |
| 925 capture_nonlocked_.fwd_proc_format.sample_rate_hz())) { | 946 capture_nonlocked_.capture_processing_format.sample_rate_hz())) { |
| 926 ca->MergeFrequencyBands(); | 947 capture_buffer->MergeFrequencyBands(); |
| 927 } | 948 } |
| 928 | 949 |
| 929 // TODO(aluebs): Investigate if the transient suppression placement should be | 950 // TODO(aluebs): Investigate if the transient suppression placement should be |
| 930 // before or after the AGC. | 951 // before or after the AGC. |
| 931 if (capture_.transient_suppressor_enabled) { | 952 if (capture_.transient_suppressor_enabled) { |
| 932 float voice_probability = | 953 float voice_probability = |
| 933 private_submodules_->agc_manager.get() | 954 private_submodules_->agc_manager.get() |
| 934 ? private_submodules_->agc_manager->voice_probability() | 955 ? private_submodules_->agc_manager->voice_probability() |
| 935 : 1.f; | 956 : 1.f; |
| 936 | 957 |
| 937 public_submodules_->transient_suppressor->Suppress( | 958 public_submodules_->transient_suppressor->Suppress( |
| 938 ca->channels_f()[0], ca->num_frames(), ca->num_channels(), | 959 capture_buffer->channels_f()[0], capture_buffer->num_frames(), |
| 939 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(), | 960 capture_buffer->num_channels(), |
| 940 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability, | 961 capture_buffer->split_bands_const_f(0)[kBand0To8kHz], |
| 962 capture_buffer->num_frames_per_band(), capture_buffer->keyboard_data(), |
| 963 capture_buffer->num_keyboard_frames(), voice_probability, |
| 941 capture_.key_pressed); | 964 capture_.key_pressed); |
| 942 } | 965 } |
| 943 | 966 |
| 944 if (capture_nonlocked_.level_controller_enabled) { | 967 if (capture_nonlocked_.level_controller_enabled) { |
| 945 private_submodules_->level_controller->Process(ca); | 968 private_submodules_->level_controller->Process(capture_buffer); |
| 946 } | 969 } |
| 947 | 970 |
| 948 // The level estimator operates on the recombined data. | 971 // The level estimator operates on the recombined data. |
| 949 public_submodules_->level_estimator->ProcessStream(ca); | 972 public_submodules_->level_estimator->ProcessStream(capture_buffer); |
| 950 | 973 |
| 951 capture_.was_stream_delay_set = false; | 974 capture_.was_stream_delay_set = false; |
| 952 return kNoError; | 975 return kNoError; |
| 953 } | 976 } |
| 954 | 977 |
| 955 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, | 978 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, |
| 956 size_t samples_per_channel, | 979 size_t samples_per_channel, |
| 957 int rev_sample_rate_hz, | 980 int sample_rate_hz, |
| 958 ChannelLayout layout) { | 981 ChannelLayout layout) { |
| 959 TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_ChannelLayout"); | 982 TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_ChannelLayout"); |
| 960 rtc::CritScope cs(&crit_render_); | 983 rtc::CritScope cs(&crit_render_); |
| 961 const StreamConfig reverse_config = { | 984 const StreamConfig reverse_config = { |
| 962 rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), | 985 sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), |
| 963 }; | 986 }; |
| 964 if (samples_per_channel != reverse_config.num_frames()) { | 987 if (samples_per_channel != reverse_config.num_frames()) { |
| 965 return kBadDataLengthError; | 988 return kBadDataLengthError; |
| 966 } | 989 } |
| 967 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config); | 990 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config); |
| 968 } | 991 } |
| 969 | 992 |
| 970 int AudioProcessingImpl::ProcessReverseStream( | 993 int AudioProcessingImpl::ProcessReverseStream(const float* const* src, |
| 971 const float* const* src, | 994 const StreamConfig& input_config, |
| 972 const StreamConfig& reverse_input_config, | 995 const StreamConfig& output_config, |
| 973 const StreamConfig& reverse_output_config, | 996 float* const* dest) { |
| 974 float* const* dest) { | |
| 975 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig"); | 997 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig"); |
| 976 rtc::CritScope cs(&crit_render_); | 998 rtc::CritScope cs(&crit_render_); |
| 977 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, reverse_input_config, | 999 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, input_config, output_config)); |
| 978 reverse_output_config)); | |
| 979 if (submodule_states_.RenderMultiBandProcessingActive()) { | 1000 if (submodule_states_.RenderMultiBandProcessingActive()) { |
| 980 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(), | 1001 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(), |
| 981 dest); | 1002 dest); |
| 982 } else if (formats_.api_format.reverse_input_stream() != | 1003 } else if (formats_.api_format.reverse_input_stream() != |
| 983 formats_.api_format.reverse_output_stream()) { | 1004 formats_.api_format.reverse_output_stream()) { |
| 984 render_.render_converter->Convert(src, reverse_input_config.num_samples(), | 1005 render_.render_converter->Convert(src, input_config.num_samples(), dest, |
| 985 dest, | 1006 output_config.num_samples()); |
| 986 reverse_output_config.num_samples()); | |
| 987 } else { | 1007 } else { |
| 988 CopyAudioIfNeeded(src, reverse_input_config.num_frames(), | 1008 CopyAudioIfNeeded(src, input_config.num_frames(), |
| 989 reverse_input_config.num_channels(), dest); | 1009 input_config.num_channels(), dest); |
| 990 } | 1010 } |
| 991 | 1011 |
| 992 return kNoError; | 1012 return kNoError; |
| 993 } | 1013 } |
| 994 | 1014 |
| 995 int AudioProcessingImpl::AnalyzeReverseStreamLocked( | 1015 int AudioProcessingImpl::AnalyzeReverseStreamLocked( |
| 996 const float* const* src, | 1016 const float* const* src, |
| 997 const StreamConfig& reverse_input_config, | 1017 const StreamConfig& input_config, |
| 998 const StreamConfig& reverse_output_config) { | 1018 const StreamConfig& output_config) { |
| 999 if (src == nullptr) { | 1019 if (src == nullptr) { |
| 1000 return kNullPointerError; | 1020 return kNullPointerError; |
| 1001 } | 1021 } |
| 1002 | 1022 |
| 1003 if (reverse_input_config.num_channels() == 0) { | 1023 if (input_config.num_channels() == 0) { |
| 1004 return kBadNumberChannelsError; | 1024 return kBadNumberChannelsError; |
| 1005 } | 1025 } |
| 1006 | 1026 |
| 1007 ProcessingConfig processing_config = formats_.api_format; | 1027 ProcessingConfig processing_config = formats_.api_format; |
| 1008 processing_config.reverse_input_stream() = reverse_input_config; | 1028 processing_config.reverse_input_stream() = input_config; |
| 1009 processing_config.reverse_output_stream() = reverse_output_config; | 1029 processing_config.reverse_output_stream() = output_config; |
| 1010 | 1030 |
| 1011 RETURN_ON_ERR(MaybeInitializeRender(processing_config)); | 1031 RETURN_ON_ERR(MaybeInitializeRender(processing_config)); |
| 1012 RTC_DCHECK_EQ(reverse_input_config.num_frames(), | 1032 assert(input_config.num_frames() == |
| 1013 formats_.api_format.reverse_input_stream().num_frames()); | 1033 formats_.api_format.reverse_input_stream().num_frames()); |
| 1014 | 1034 |
| 1015 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1035 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 1016 if (debug_dump_.debug_file->is_open()) { | 1036 if (debug_dump_.debug_file->is_open()) { |
| 1017 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); | 1037 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); |
| 1018 audioproc::ReverseStream* msg = | 1038 audioproc::ReverseStream* msg = |
| 1019 debug_dump_.render.event_msg->mutable_reverse_stream(); | 1039 debug_dump_.render.event_msg->mutable_reverse_stream(); |
| 1020 const size_t channel_size = | 1040 const size_t channel_size = |
| 1021 sizeof(float) * formats_.api_format.reverse_input_stream().num_frames(); | 1041 sizeof(float) * formats_.api_format.reverse_input_stream().num_frames(); |
| 1022 for (size_t i = 0; | 1042 for (size_t i = 0; |
| 1023 i < formats_.api_format.reverse_input_stream().num_channels(); ++i) | 1043 i < formats_.api_format.reverse_input_stream().num_channels(); ++i) |
| 1024 msg->add_channel(src[i], channel_size); | 1044 msg->add_channel(src[i], channel_size); |
| 1025 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), | 1045 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 1026 &debug_dump_.num_bytes_left_for_log_, | 1046 &debug_dump_.num_bytes_left_for_log_, |
| 1027 &crit_debug_, &debug_dump_.render)); | 1047 &crit_debug_, &debug_dump_.render)); |
| 1028 } | 1048 } |
| 1029 #endif | 1049 #endif |
| 1030 | 1050 |
| 1031 render_.render_audio->CopyFrom(src, | 1051 render_.render_audio->CopyFrom(src, |
| 1032 formats_.api_format.reverse_input_stream()); | 1052 formats_.api_format.reverse_input_stream()); |
| 1033 return ProcessReverseStreamLocked(); | 1053 return ProcessRenderStreamLocked(); |
| 1034 } | 1054 } |
| 1035 | 1055 |
| 1036 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { | 1056 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { |
| 1037 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame"); | 1057 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame"); |
| 1038 rtc::CritScope cs(&crit_render_); | 1058 rtc::CritScope cs(&crit_render_); |
| 1039 if (frame == nullptr) { | 1059 if (frame == nullptr) { |
| 1040 return kNullPointerError; | 1060 return kNullPointerError; |
| 1041 } | 1061 } |
| 1042 // Must be a native rate. | 1062 // Must be a native rate. |
| 1043 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 1063 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
| (...skipping 30 matching lines...) Expand all Loading... |
| 1074 debug_dump_.render.event_msg->mutable_reverse_stream(); | 1094 debug_dump_.render.event_msg->mutable_reverse_stream(); |
| 1075 const size_t data_size = | 1095 const size_t data_size = |
| 1076 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 1096 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
| 1077 msg->set_data(frame->data_, data_size); | 1097 msg->set_data(frame->data_, data_size); |
| 1078 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), | 1098 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 1079 &debug_dump_.num_bytes_left_for_log_, | 1099 &debug_dump_.num_bytes_left_for_log_, |
| 1080 &crit_debug_, &debug_dump_.render)); | 1100 &crit_debug_, &debug_dump_.render)); |
| 1081 } | 1101 } |
| 1082 #endif | 1102 #endif |
| 1083 render_.render_audio->DeinterleaveFrom(frame); | 1103 render_.render_audio->DeinterleaveFrom(frame); |
| 1084 RETURN_ON_ERR(ProcessReverseStreamLocked()); | 1104 RETURN_ON_ERR(ProcessRenderStreamLocked()); |
| 1085 render_.render_audio->InterleaveTo( | 1105 render_.render_audio->InterleaveTo( |
| 1086 frame, submodule_states_.RenderMultiBandProcessingActive()); | 1106 frame, submodule_states_.RenderMultiBandProcessingActive()); |
| 1087 return kNoError; | 1107 return kNoError; |
| 1088 } | 1108 } |
| 1089 | 1109 |
| 1090 int AudioProcessingImpl::ProcessReverseStreamLocked() { | 1110 int AudioProcessingImpl::ProcessRenderStreamLocked() { |
| 1091 AudioBuffer* ra = render_.render_audio.get(); // For brevity. | 1111 AudioBuffer* render_buffer = render_.render_audio.get(); // For brevity. |
| 1092 if (submodule_states_.RenderMultiBandSubModulesActive() && | 1112 if (submodule_states_.RenderMultiBandSubModulesActive() && |
| 1093 SampleRateSupportsMultiBand(formats_.rev_proc_format.sample_rate_hz())) { | 1113 SampleRateSupportsMultiBand( |
| 1094 ra->SplitIntoFrequencyBands(); | 1114 formats_.render_processing_format.sample_rate_hz())) { |
| 1115 render_buffer->SplitIntoFrequencyBands(); |
| 1095 } | 1116 } |
| 1096 | 1117 |
| 1097 #if WEBRTC_INTELLIGIBILITY_ENHANCER | 1118 #if WEBRTC_INTELLIGIBILITY_ENHANCER |
| 1098 if (capture_nonlocked_.intelligibility_enabled) { | 1119 if (capture_nonlocked_.intelligibility_enabled) { |
| 1099 public_submodules_->intelligibility_enhancer->ProcessRenderAudio( | 1120 public_submodules_->intelligibility_enhancer->ProcessRenderAudio( |
| 1100 ra->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate, | 1121 render_buffer->split_channels_f(kBand0To8kHz), |
| 1101 ra->num_channels()); | 1122 capture_nonlocked_.split_rate, render_buffer->num_channels()); |
| 1102 } | 1123 } |
| 1103 #endif | 1124 #endif |
| 1104 | 1125 |
| 1105 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessRenderAudio(ra)); | |
| 1106 RETURN_ON_ERR( | 1126 RETURN_ON_ERR( |
| 1107 public_submodules_->echo_control_mobile->ProcessRenderAudio(ra)); | 1127 public_submodules_->echo_cancellation->ProcessRenderAudio(render_buffer)); |
| 1128 RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessRenderAudio( |
| 1129 render_buffer)); |
| 1108 if (!constants_.use_experimental_agc) { | 1130 if (!constants_.use_experimental_agc) { |
| 1109 RETURN_ON_ERR(public_submodules_->gain_control->ProcessRenderAudio(ra)); | 1131 RETURN_ON_ERR( |
| 1132 public_submodules_->gain_control->ProcessRenderAudio(render_buffer)); |
| 1110 } | 1133 } |
| 1111 | 1134 |
| 1112 if (submodule_states_.RenderMultiBandProcessingActive() && | 1135 if (submodule_states_.RenderMultiBandProcessingActive() && |
| 1113 SampleRateSupportsMultiBand(formats_.rev_proc_format.sample_rate_hz())) { | 1136 SampleRateSupportsMultiBand( |
| 1114 ra->MergeFrequencyBands(); | 1137 formats_.render_processing_format.sample_rate_hz())) { |
| 1138 render_buffer->MergeFrequencyBands(); |
| 1115 } | 1139 } |
| 1116 | 1140 |
| 1117 return kNoError; | 1141 return kNoError; |
| 1118 } | 1142 } |
| 1119 | 1143 |
| 1120 int AudioProcessingImpl::set_stream_delay_ms(int delay) { | 1144 int AudioProcessingImpl::set_stream_delay_ms(int delay) { |
| 1121 rtc::CritScope cs(&crit_capture_); | 1145 rtc::CritScope cs(&crit_capture_); |
| 1122 Error retval = kNoError; | 1146 Error retval = kNoError; |
| 1123 capture_.was_stream_delay_set = true; | 1147 capture_.was_stream_delay_set = true; |
| 1124 delay += capture_.delay_offset_ms; | 1148 delay += capture_.delay_offset_ms; |
| (...skipping 157 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1282 public_submodules_->noise_suppression->is_enabled(), | 1306 public_submodules_->noise_suppression->is_enabled(), |
| 1283 capture_nonlocked_.intelligibility_enabled, | 1307 capture_nonlocked_.intelligibility_enabled, |
| 1284 capture_nonlocked_.beamformer_enabled, | 1308 capture_nonlocked_.beamformer_enabled, |
| 1285 public_submodules_->gain_control->is_enabled(), | 1309 public_submodules_->gain_control->is_enabled(), |
| 1286 capture_nonlocked_.level_controller_enabled, | 1310 capture_nonlocked_.level_controller_enabled, |
| 1287 public_submodules_->voice_detection->is_enabled(), | 1311 public_submodules_->voice_detection->is_enabled(), |
| 1288 public_submodules_->level_estimator->is_enabled(), | 1312 public_submodules_->level_estimator->is_enabled(), |
| 1289 capture_.transient_suppressor_enabled); | 1313 capture_.transient_suppressor_enabled); |
| 1290 } | 1314 } |
| 1291 | 1315 |
| 1292 void AudioProcessingImpl::InitializeExperimentalAgc() { | |
| 1293 if (constants_.use_experimental_agc) { | |
| 1294 if (!private_submodules_->agc_manager.get()) { | |
| 1295 private_submodules_->agc_manager.reset(new AgcManagerDirect( | |
| 1296 public_submodules_->gain_control.get(), | |
| 1297 public_submodules_->gain_control_for_experimental_agc.get(), | |
| 1298 constants_.agc_startup_min_volume)); | |
| 1299 } | |
| 1300 private_submodules_->agc_manager->Initialize(); | |
| 1301 private_submodules_->agc_manager->SetCaptureMuted( | |
| 1302 capture_.output_will_be_muted); | |
| 1303 } | |
| 1304 } | |
| 1305 | 1316 |
| 1306 void AudioProcessingImpl::InitializeTransient() { | 1317 void AudioProcessingImpl::InitializeTransient() { |
| 1307 if (capture_.transient_suppressor_enabled) { | 1318 if (capture_.transient_suppressor_enabled) { |
| 1308 if (!public_submodules_->transient_suppressor.get()) { | 1319 if (!public_submodules_->transient_suppressor.get()) { |
| 1309 public_submodules_->transient_suppressor.reset(new TransientSuppressor()); | 1320 public_submodules_->transient_suppressor.reset(new TransientSuppressor()); |
| 1310 } | 1321 } |
| 1311 public_submodules_->transient_suppressor->Initialize( | 1322 public_submodules_->transient_suppressor->Initialize( |
| 1312 capture_nonlocked_.fwd_proc_format.sample_rate_hz(), | 1323 capture_nonlocked_.capture_processing_format.sample_rate_hz(), |
| 1313 capture_nonlocked_.split_rate, | 1324 capture_nonlocked_.split_rate, num_proc_channels()); |
| 1314 num_proc_channels()); | |
| 1315 } | 1325 } |
| 1316 } | 1326 } |
| 1317 | 1327 |
| 1318 void AudioProcessingImpl::InitializeBeamformer() { | 1328 void AudioProcessingImpl::InitializeBeamformer() { |
| 1319 if (capture_nonlocked_.beamformer_enabled) { | 1329 if (capture_nonlocked_.beamformer_enabled) { |
| 1320 if (!private_submodules_->beamformer) { | 1330 if (!private_submodules_->beamformer) { |
| 1321 private_submodules_->beamformer.reset(new NonlinearBeamformer( | 1331 private_submodules_->beamformer.reset(new NonlinearBeamformer( |
| 1322 capture_.array_geometry, 1u, capture_.target_direction)); | 1332 capture_.array_geometry, 1u, capture_.target_direction)); |
| 1323 } | 1333 } |
| 1324 private_submodules_->beamformer->Initialize(kChunkSizeMs, | 1334 private_submodules_->beamformer->Initialize(kChunkSizeMs, |
| 1325 capture_nonlocked_.split_rate); | 1335 capture_nonlocked_.split_rate); |
| 1326 } | 1336 } |
| 1327 } | 1337 } |
| 1328 | 1338 |
| 1329 void AudioProcessingImpl::InitializeIntelligibility() { | 1339 void AudioProcessingImpl::InitializeIntelligibility() { |
| 1330 #if WEBRTC_INTELLIGIBILITY_ENHANCER | 1340 #if WEBRTC_INTELLIGIBILITY_ENHANCER |
| 1331 if (capture_nonlocked_.intelligibility_enabled) { | 1341 if (capture_nonlocked_.intelligibility_enabled) { |
| 1332 public_submodules_->intelligibility_enhancer.reset( | 1342 public_submodules_->intelligibility_enhancer.reset( |
| 1333 new IntelligibilityEnhancer(capture_nonlocked_.split_rate, | 1343 new IntelligibilityEnhancer(capture_nonlocked_.split_rate, |
| 1334 render_.render_audio->num_channels(), | 1344 render_.render_audio->num_channels(), |
| 1335 NoiseSuppressionImpl::num_noise_bins())); | 1345 NoiseSuppressionImpl::num_noise_bins())); |
| 1336 } | 1346 } |
| 1337 #endif | 1347 #endif |
| 1338 } | 1348 } |
| 1339 | 1349 |
| 1340 void AudioProcessingImpl::InitializeHighPassFilter() { | |
| 1341 public_submodules_->high_pass_filter->Initialize(num_proc_channels(), | |
| 1342 proc_sample_rate_hz()); | |
| 1343 } | |
| 1344 | |
| 1345 void AudioProcessingImpl::InitializeNoiseSuppression() { | |
| 1346 public_submodules_->noise_suppression->Initialize(num_proc_channels(), | |
| 1347 proc_sample_rate_hz()); | |
| 1348 } | |
| 1349 | |
| 1350 void AudioProcessingImpl::InitializeEchoCanceller() { | |
| 1351 public_submodules_->echo_cancellation->Initialize( | |
| 1352 proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(), | |
| 1353 num_proc_channels()); | |
| 1354 } | |
| 1355 | |
| 1356 void AudioProcessingImpl::InitializeGainController() { | |
| 1357 public_submodules_->gain_control->Initialize(num_proc_channels(), | |
| 1358 proc_sample_rate_hz()); | |
| 1359 } | |
| 1360 | |
| 1361 void AudioProcessingImpl::InitializeEchoControlMobile() { | |
| 1362 public_submodules_->echo_control_mobile->Initialize( | |
| 1363 proc_split_sample_rate_hz(), | |
| 1364 num_reverse_channels(), | |
| 1365 num_output_channels()); | |
| 1366 } | |
| 1367 | |
| 1368 void AudioProcessingImpl::InitializeLevelEstimator() { | |
| 1369 public_submodules_->level_estimator->Initialize(); | |
| 1370 } | |
| 1371 | |
| 1372 void AudioProcessingImpl::InitializeLevelController() { | 1350 void AudioProcessingImpl::InitializeLevelController() { |
| 1373 private_submodules_->level_controller->Initialize(proc_sample_rate_hz()); | 1351 private_submodules_->level_controller->Initialize(proc_sample_rate_hz()); |
| 1374 } | 1352 } |
| 1375 | 1353 |
| 1376 void AudioProcessingImpl::InitializeVoiceDetection() { | |
| 1377 public_submodules_->voice_detection->Initialize(proc_split_sample_rate_hz()); | |
| 1378 } | |
| 1379 | |
| 1380 void AudioProcessingImpl::MaybeUpdateHistograms() { | 1354 void AudioProcessingImpl::MaybeUpdateHistograms() { |
| 1381 static const int kMinDiffDelayMs = 60; | 1355 static const int kMinDiffDelayMs = 60; |
| 1382 | 1356 |
| 1383 if (echo_cancellation()->is_enabled()) { | 1357 if (echo_cancellation()->is_enabled()) { |
| 1384 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. | 1358 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. |
| 1385 // If a stream has echo we know that the echo_cancellation is in process. | 1359 // If a stream has echo we know that the echo_cancellation is in process. |
| 1386 if (capture_.stream_delay_jumps == -1 && | 1360 if (capture_.stream_delay_jumps == -1 && |
| 1387 echo_cancellation()->stream_has_echo()) { | 1361 echo_cancellation()->stream_has_echo()) { |
| 1388 capture_.stream_delay_jumps = 0; | 1362 capture_.stream_delay_jumps = 0; |
| 1389 } | 1363 } |
| (...skipping 208 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1598 delay_offset_ms(0), | 1572 delay_offset_ms(0), |
| 1599 was_stream_delay_set(false), | 1573 was_stream_delay_set(false), |
| 1600 last_stream_delay_ms(0), | 1574 last_stream_delay_ms(0), |
| 1601 last_aec_system_delay_ms(0), | 1575 last_aec_system_delay_ms(0), |
| 1602 stream_delay_jumps(-1), | 1576 stream_delay_jumps(-1), |
| 1603 output_will_be_muted(false), | 1577 output_will_be_muted(false), |
| 1604 key_pressed(false), | 1578 key_pressed(false), |
| 1605 transient_suppressor_enabled(transient_suppressor_enabled), | 1579 transient_suppressor_enabled(transient_suppressor_enabled), |
| 1606 array_geometry(array_geometry), | 1580 array_geometry(array_geometry), |
| 1607 target_direction(target_direction), | 1581 target_direction(target_direction), |
| 1608 fwd_proc_format(kSampleRate16kHz), | 1582 capture_processing_format(kSampleRate16kHz), |
| 1609 split_rate(kSampleRate16kHz) {} | 1583 split_rate(kSampleRate16kHz) {} |
| 1610 | 1584 |
| 1611 AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default; | 1585 AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default; |
| 1612 | 1586 |
| 1613 AudioProcessingImpl::ApmRenderState::ApmRenderState() = default; | 1587 AudioProcessingImpl::ApmRenderState::ApmRenderState() = default; |
| 1614 | 1588 |
| 1615 AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default; | 1589 AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default; |
| 1616 | 1590 |
| 1617 } // namespace webrtc | 1591 } // namespace webrtc |
| OLD | NEW |