Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 /* | 1 /* |
| 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 * | 3 * |
| 4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ | 9 */ |
| 10 | 10 |
| (...skipping 312 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 323 #endif | 323 #endif |
| 324 } | 324 } |
| 325 | 325 |
| 326 int AudioProcessingImpl::Initialize() { | 326 int AudioProcessingImpl::Initialize() { |
| 327 // Run in a single-threaded manner during initialization. | 327 // Run in a single-threaded manner during initialization. |
| 328 rtc::CritScope cs_render(&crit_render_); | 328 rtc::CritScope cs_render(&crit_render_); |
| 329 rtc::CritScope cs_capture(&crit_capture_); | 329 rtc::CritScope cs_capture(&crit_capture_); |
| 330 return InitializeLocked(); | 330 return InitializeLocked(); |
| 331 } | 331 } |
| 332 | 332 |
| 333 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, | 333 int AudioProcessingImpl::Initialize(int capture_input_sample_rate_hz, |
| 334 int output_sample_rate_hz, | 334 int capture_output_sample_rate_hz, |
| 335 int reverse_sample_rate_hz, | 335 int render_input_sample_rate_hz, |
| 336 ChannelLayout input_layout, | 336 ChannelLayout capture_input_layout, |
| 337 ChannelLayout output_layout, | 337 ChannelLayout capture_output_layout, |
| 338 ChannelLayout reverse_layout) { | 338 ChannelLayout render_input_layout) { |
| 339 const ProcessingConfig processing_config = { | 339 const ProcessingConfig processing_config = { |
| 340 {{input_sample_rate_hz, | 340 {{capture_input_sample_rate_hz, ChannelsFromLayout(capture_input_layout), |
| 341 ChannelsFromLayout(input_layout), | 341 LayoutHasKeyboard(capture_input_layout)}, |
| 342 LayoutHasKeyboard(input_layout)}, | 342 {capture_output_sample_rate_hz, |
| 343 {output_sample_rate_hz, | 343 ChannelsFromLayout(capture_output_layout), |
| 344 ChannelsFromLayout(output_layout), | 344 LayoutHasKeyboard(capture_output_layout)}, |
| 345 LayoutHasKeyboard(output_layout)}, | 345 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout), |
| 346 {reverse_sample_rate_hz, | 346 LayoutHasKeyboard(render_input_layout)}, |
| 347 ChannelsFromLayout(reverse_layout), | 347 {render_input_sample_rate_hz, ChannelsFromLayout(render_input_layout), |
| 348 LayoutHasKeyboard(reverse_layout)}, | 348 LayoutHasKeyboard(render_input_layout)}}}; |
| 349 {reverse_sample_rate_hz, | |
| 350 ChannelsFromLayout(reverse_layout), | |
| 351 LayoutHasKeyboard(reverse_layout)}}}; | |
| 352 | 349 |
| 353 return Initialize(processing_config); | 350 return Initialize(processing_config); |
| 354 } | 351 } |
| 355 | 352 |
| 356 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { | 353 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { |
| 357 // Run in a single-threaded manner during initialization. | 354 // Run in a single-threaded manner during initialization. |
| 358 rtc::CritScope cs_render(&crit_render_); | 355 rtc::CritScope cs_render(&crit_render_); |
| 359 rtc::CritScope cs_capture(&crit_capture_); | 356 rtc::CritScope cs_capture(&crit_capture_); |
| 360 return InitializeLocked(processing_config); | 357 return InitializeLocked(processing_config); |
| 361 } | 358 } |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 393 // Called from both threads. Thread check is therefore not possible. | 390 // Called from both threads. Thread check is therefore not possible. |
| 394 if (processing_config == formats_.api_format && !force_initialization) { | 391 if (processing_config == formats_.api_format && !force_initialization) { |
| 395 return kNoError; | 392 return kNoError; |
| 396 } | 393 } |
| 397 | 394 |
| 398 rtc::CritScope cs_capture(&crit_capture_); | 395 rtc::CritScope cs_capture(&crit_capture_); |
| 399 return InitializeLocked(processing_config); | 396 return InitializeLocked(processing_config); |
| 400 } | 397 } |
| 401 | 398 |
| 402 int AudioProcessingImpl::InitializeLocked() { | 399 int AudioProcessingImpl::InitializeLocked() { |
| 403 const int fwd_audio_buffer_channels = | 400 const int capture_audiobuffer_num_channels = |
| 404 capture_nonlocked_.beamformer_enabled | 401 capture_nonlocked_.beamformer_enabled |
| 405 ? formats_.api_format.input_stream().num_channels() | 402 ? formats_.api_format.input_stream().num_channels() |
| 406 : formats_.api_format.output_stream().num_channels(); | 403 : formats_.api_format.output_stream().num_channels(); |
| 407 const int rev_audio_buffer_out_num_frames = | 404 const int render_audiobuffer_num_output_frames = |
| 408 formats_.api_format.reverse_output_stream().num_frames() == 0 | 405 formats_.api_format.reverse_output_stream().num_frames() == 0 |
| 409 ? formats_.rev_proc_format.num_frames() | 406 ? formats_.render_processing_format.num_frames() |
| 410 : formats_.api_format.reverse_output_stream().num_frames(); | 407 : formats_.api_format.reverse_output_stream().num_frames(); |
| 411 if (formats_.api_format.reverse_input_stream().num_channels() > 0) { | 408 if (formats_.api_format.reverse_input_stream().num_channels() > 0) { |
| 412 render_.render_audio.reset(new AudioBuffer( | 409 render_.render_audio.reset(new AudioBuffer( |
| 413 formats_.api_format.reverse_input_stream().num_frames(), | 410 formats_.api_format.reverse_input_stream().num_frames(), |
| 414 formats_.api_format.reverse_input_stream().num_channels(), | 411 formats_.api_format.reverse_input_stream().num_channels(), |
| 415 formats_.rev_proc_format.num_frames(), | 412 formats_.render_processing_format.num_frames(), |
| 416 formats_.rev_proc_format.num_channels(), | 413 formats_.render_processing_format.num_channels(), |
| 417 rev_audio_buffer_out_num_frames)); | 414 render_audiobuffer_num_output_frames)); |
| 418 if (formats_.api_format.reverse_input_stream() != | 415 if (formats_.api_format.reverse_input_stream() != |
| 419 formats_.api_format.reverse_output_stream()) { | 416 formats_.api_format.reverse_output_stream()) { |
| 420 render_.render_converter = AudioConverter::Create( | 417 render_.render_converter = AudioConverter::Create( |
| 421 formats_.api_format.reverse_input_stream().num_channels(), | 418 formats_.api_format.reverse_input_stream().num_channels(), |
| 422 formats_.api_format.reverse_input_stream().num_frames(), | 419 formats_.api_format.reverse_input_stream().num_frames(), |
| 423 formats_.api_format.reverse_output_stream().num_channels(), | 420 formats_.api_format.reverse_output_stream().num_channels(), |
| 424 formats_.api_format.reverse_output_stream().num_frames()); | 421 formats_.api_format.reverse_output_stream().num_frames()); |
| 425 } else { | 422 } else { |
| 426 render_.render_converter.reset(nullptr); | 423 render_.render_converter.reset(nullptr); |
| 427 } | 424 } |
| 428 } else { | 425 } else { |
| 429 render_.render_audio.reset(nullptr); | 426 render_.render_audio.reset(nullptr); |
| 430 render_.render_converter.reset(nullptr); | 427 render_.render_converter.reset(nullptr); |
| 431 } | 428 } |
| 432 capture_.capture_audio.reset( | 429 capture_.capture_audio.reset( |
| 433 new AudioBuffer(formats_.api_format.input_stream().num_frames(), | 430 new AudioBuffer(formats_.api_format.input_stream().num_frames(), |
| 434 formats_.api_format.input_stream().num_channels(), | 431 formats_.api_format.input_stream().num_channels(), |
| 435 capture_nonlocked_.fwd_proc_format.num_frames(), | 432 capture_nonlocked_.capture_processing_format.num_frames(), |
| 436 fwd_audio_buffer_channels, | 433 capture_audiobuffer_num_channels, |
| 437 formats_.api_format.output_stream().num_frames())); | 434 formats_.api_format.output_stream().num_frames())); |
| 438 | 435 |
| 439 InitializeGainController(); | 436 public_submodules_->gain_control->Initialize(num_proc_channels(), |
| 440 InitializeEchoCanceller(); | 437 proc_sample_rate_hz()); |
| 441 InitializeEchoControlMobile(); | 438 public_submodules_->echo_cancellation->Initialize( |
| 442 InitializeExperimentalAgc(); | 439 proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(), |
| 440 num_proc_channels()); | |
| 441 public_submodules_->echo_control_mobile->Initialize( | |
| 442 proc_split_sample_rate_hz(), num_reverse_channels(), | |
| 443 num_output_channels()); | |
| 444 if (constants_.use_experimental_agc) { | |
| 445 if (!private_submodules_->agc_manager.get()) { | |
| 446 private_submodules_->agc_manager.reset(new AgcManagerDirect( | |
| 447 public_submodules_->gain_control.get(), | |
| 448 public_submodules_->gain_control_for_experimental_agc.get(), | |
| 449 constants_.agc_startup_min_volume)); | |
| 450 } | |
| 451 private_submodules_->agc_manager->Initialize(); | |
| 452 private_submodules_->agc_manager->SetCaptureMuted( | |
| 453 capture_.output_will_be_muted); | |
| 454 } | |
| 443 InitializeTransient(); | 455 InitializeTransient(); |
| 444 InitializeBeamformer(); | 456 InitializeBeamformer(); |
| 445 #if WEBRTC_INTELLIGIBILITY_ENHANCER | 457 #if WEBRTC_INTELLIGIBILITY_ENHANCER |
| 446 InitializeIntelligibility(); | 458 InitializeIntelligibility(); |
| 447 #endif | 459 #endif |
| 448 InitializeHighPassFilter(); | 460 public_submodules_->high_pass_filter->Initialize(num_proc_channels(), |
| 449 InitializeNoiseSuppression(); | 461 proc_sample_rate_hz()); |
| 450 InitializeLevelEstimator(); | 462 public_submodules_->noise_suppression->Initialize(num_proc_channels(), |
| 451 InitializeVoiceDetection(); | 463 proc_sample_rate_hz()); |
| 464 public_submodules_->voice_detection->Initialize(proc_split_sample_rate_hz()); | |
| 452 InitializeLevelController(); | 465 InitializeLevelController(); |
| 453 | 466 |
| 454 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 467 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 455 if (debug_dump_.debug_file->is_open()) { | 468 if (debug_dump_.debug_file->is_open()) { |
| 456 int err = WriteInitMessage(); | 469 int err = WriteInitMessage(); |
| 457 if (err != kNoError) { | 470 if (err != kNoError) { |
| 458 return err; | 471 return err; |
| 459 } | 472 } |
| 460 } | 473 } |
| 461 #endif | 474 #endif |
| (...skipping 18 matching lines...) Expand all Loading... | |
| 480 return kBadNumberChannelsError; | 493 return kBadNumberChannelsError; |
| 481 } | 494 } |
| 482 | 495 |
| 483 if (capture_nonlocked_.beamformer_enabled && | 496 if (capture_nonlocked_.beamformer_enabled && |
| 484 num_in_channels != capture_.array_geometry.size()) { | 497 num_in_channels != capture_.array_geometry.size()) { |
| 485 return kBadNumberChannelsError; | 498 return kBadNumberChannelsError; |
| 486 } | 499 } |
| 487 | 500 |
| 488 formats_.api_format = config; | 501 formats_.api_format = config; |
| 489 | 502 |
| 490 int fwd_proc_rate = FindNativeProcessRateToUse( | 503 int capture_processing_rate = FindNativeProcessRateToUse( |
| 491 std::min(formats_.api_format.input_stream().sample_rate_hz(), | 504 std::min(formats_.api_format.input_stream().sample_rate_hz(), |
| 492 formats_.api_format.output_stream().sample_rate_hz()), | 505 formats_.api_format.output_stream().sample_rate_hz()), |
| 493 submodule_states_.CaptureMultiBandSubModulesActive() || | 506 submodule_states_.CaptureMultiBandSubModulesActive() || |
| 494 submodule_states_.RenderMultiBandSubModulesActive()); | 507 submodule_states_.RenderMultiBandSubModulesActive()); |
| 495 | 508 |
| 496 capture_nonlocked_.fwd_proc_format = StreamConfig(fwd_proc_rate); | 509 capture_nonlocked_.capture_processing_format = |
| 510 StreamConfig(capture_processing_rate); | |
| 497 | 511 |
| 498 int rev_proc_rate = FindNativeProcessRateToUse( | 512 int render_processing_rate = FindNativeProcessRateToUse( |
| 499 std::min(formats_.api_format.reverse_input_stream().sample_rate_hz(), | 513 std::min(formats_.api_format.reverse_input_stream().sample_rate_hz(), |
| 500 formats_.api_format.reverse_output_stream().sample_rate_hz()), | 514 formats_.api_format.reverse_output_stream().sample_rate_hz()), |
| 501 submodule_states_.CaptureMultiBandSubModulesActive() || | 515 submodule_states_.CaptureMultiBandSubModulesActive() || |
| 502 submodule_states_.RenderMultiBandSubModulesActive()); | 516 submodule_states_.RenderMultiBandSubModulesActive()); |
| 503 // TODO(aluebs): Remove this restriction once we figure out why the 3-band | 517 // TODO(aluebs): Remove this restriction once we figure out why the 3-band |
| 504 // splitting filter degrades the AEC performance. | 518 // splitting filter degrades the AEC performance. |
| 505 if (rev_proc_rate > kSampleRate32kHz) { | 519 if (render_processing_rate > kSampleRate32kHz) { |
| 506 rev_proc_rate = submodule_states_.RenderMultiBandProcessingActive() | 520 render_processing_rate = submodule_states_.RenderMultiBandProcessingActive() |
| 507 ? kSampleRate32kHz | 521 ? kSampleRate32kHz |
| 508 : kSampleRate16kHz; | 522 : kSampleRate16kHz; |
| 509 } | 523 } |
| 510 // If the forward sample rate is 8 kHz, the reverse stream is also processed | 524 // If the forward sample rate is 8 kHz, the render stream is also processed |
| 511 // at this rate. | 525 // at this rate. |
| 512 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate8kHz) { | 526 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() == |
| 513 rev_proc_rate = kSampleRate8kHz; | 527 kSampleRate8kHz) { |
| 528 render_processing_rate = kSampleRate8kHz; | |
| 514 } else { | 529 } else { |
| 515 rev_proc_rate = std::max(rev_proc_rate, static_cast<int>(kSampleRate16kHz)); | 530 render_processing_rate = |
| 531 std::max(render_processing_rate, static_cast<int>(kSampleRate16kHz)); | |
| 516 } | 532 } |
| 517 | 533 |
| 518 // Always downmix the reverse stream to mono for analysis. This has been | 534 // Always downmix the render stream to mono for analysis. This has been |
| 519 // demonstrated to work well for AEC in most practical scenarios. | 535 // demonstrated to work well for AEC in most practical scenarios. |
| 520 formats_.rev_proc_format = StreamConfig(rev_proc_rate, 1); | 536 formats_.render_processing_format = StreamConfig(render_processing_rate, 1); |
| 521 | 537 |
| 522 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate32kHz || | 538 if (capture_nonlocked_.capture_processing_format.sample_rate_hz() == |
| 523 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate48kHz) { | 539 kSampleRate32kHz || |
| 540 capture_nonlocked_.capture_processing_format.sample_rate_hz() == | |
| 541 kSampleRate48kHz) { | |
| 524 capture_nonlocked_.split_rate = kSampleRate16kHz; | 542 capture_nonlocked_.split_rate = kSampleRate16kHz; |
| 525 } else { | 543 } else { |
| 526 capture_nonlocked_.split_rate = | 544 capture_nonlocked_.split_rate = |
| 527 capture_nonlocked_.fwd_proc_format.sample_rate_hz(); | 545 capture_nonlocked_.capture_processing_format.sample_rate_hz(); |
| 528 } | 546 } |
| 529 | 547 |
| 530 return InitializeLocked(); | 548 return InitializeLocked(); |
| 531 } | 549 } |
| 532 | 550 |
| 533 void AudioProcessingImpl::SetExtraOptions(const Config& config) { | 551 void AudioProcessingImpl::SetExtraOptions(const Config& config) { |
| 534 // Run in a single-threaded manner when setting the extra options. | 552 // Run in a single-threaded manner when setting the extra options. |
| 535 rtc::CritScope cs_render(&crit_render_); | 553 rtc::CritScope cs_render(&crit_render_); |
| 536 rtc::CritScope cs_capture(&crit_capture_); | 554 rtc::CritScope cs_capture(&crit_capture_); |
| 537 | 555 |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 571 capture_.array_geometry = config.Get<Beamforming>().array_geometry; | 589 capture_.array_geometry = config.Get<Beamforming>().array_geometry; |
| 572 } | 590 } |
| 573 capture_.target_direction = config.Get<Beamforming>().target_direction; | 591 capture_.target_direction = config.Get<Beamforming>().target_direction; |
| 574 InitializeBeamformer(); | 592 InitializeBeamformer(); |
| 575 } | 593 } |
| 576 #endif // WEBRTC_ANDROID_PLATFORM_BUILD | 594 #endif // WEBRTC_ANDROID_PLATFORM_BUILD |
| 577 } | 595 } |
| 578 | 596 |
| 579 int AudioProcessingImpl::proc_sample_rate_hz() const { | 597 int AudioProcessingImpl::proc_sample_rate_hz() const { |
| 580 // Used as callback from submodules, hence locking is not allowed. | 598 // Used as callback from submodules, hence locking is not allowed. |
| 581 return capture_nonlocked_.fwd_proc_format.sample_rate_hz(); | 599 return capture_nonlocked_.capture_processing_format.sample_rate_hz(); |
| 582 } | 600 } |
| 583 | 601 |
| 584 int AudioProcessingImpl::proc_split_sample_rate_hz() const { | 602 int AudioProcessingImpl::proc_split_sample_rate_hz() const { |
| 585 // Used as callback from submodules, hence locking is not allowed. | 603 // Used as callback from submodules, hence locking is not allowed. |
| 586 return capture_nonlocked_.split_rate; | 604 return capture_nonlocked_.split_rate; |
| 587 } | 605 } |
| 588 | 606 |
| 589 size_t AudioProcessingImpl::num_reverse_channels() const { | 607 size_t AudioProcessingImpl::num_reverse_channels() const { |
| 590 // Used as callback from submodules, hence locking is not allowed. | 608 // Used as callback from submodules, hence locking is not allowed. |
| 591 return formats_.rev_proc_format.num_channels(); | 609 return formats_.render_processing_format.num_channels(); |
| 592 } | 610 } |
| 593 | 611 |
| 594 size_t AudioProcessingImpl::num_input_channels() const { | 612 size_t AudioProcessingImpl::num_input_channels() const { |
| 595 // Used as callback from submodules, hence locking is not allowed. | 613 // Used as callback from submodules, hence locking is not allowed. |
| 596 return formats_.api_format.input_stream().num_channels(); | 614 return formats_.api_format.input_stream().num_channels(); |
| 597 } | 615 } |
| 598 | 616 |
| 599 size_t AudioProcessingImpl::num_proc_channels() const { | 617 size_t AudioProcessingImpl::num_proc_channels() const { |
| 600 // Used as callback from submodules, hence locking is not allowed. | 618 // Used as callback from submodules, hence locking is not allowed. |
| 601 return capture_nonlocked_.beamformer_enabled ? 1 : num_output_channels(); | 619 return capture_nonlocked_.beamformer_enabled ? 1 : num_output_channels(); |
| (...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 693 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); | 711 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
| 694 const size_t channel_size = | 712 const size_t channel_size = |
| 695 sizeof(float) * formats_.api_format.input_stream().num_frames(); | 713 sizeof(float) * formats_.api_format.input_stream().num_frames(); |
| 696 for (size_t i = 0; i < formats_.api_format.input_stream().num_channels(); | 714 for (size_t i = 0; i < formats_.api_format.input_stream().num_channels(); |
| 697 ++i) | 715 ++i) |
| 698 msg->add_input_channel(src[i], channel_size); | 716 msg->add_input_channel(src[i], channel_size); |
| 699 } | 717 } |
| 700 #endif | 718 #endif |
| 701 | 719 |
| 702 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream()); | 720 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream()); |
| 703 RETURN_ON_ERR(ProcessStreamLocked()); | 721 RETURN_ON_ERR(ProcessCaptureStreamLocked()); |
| 704 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest); | 722 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest); |
| 705 | 723 |
| 706 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 724 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 707 if (debug_dump_.debug_file->is_open()) { | 725 if (debug_dump_.debug_file->is_open()) { |
| 708 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); | 726 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
| 709 const size_t channel_size = | 727 const size_t channel_size = |
| 710 sizeof(float) * formats_.api_format.output_stream().num_frames(); | 728 sizeof(float) * formats_.api_format.output_stream().num_frames(); |
| 711 for (size_t i = 0; i < formats_.api_format.output_stream().num_channels(); | 729 for (size_t i = 0; i < formats_.api_format.output_stream().num_channels(); |
| 712 ++i) | 730 ++i) |
| 713 msg->add_output_channel(dest[i], channel_size); | 731 msg->add_output_channel(dest[i], channel_size); |
| (...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 782 | 800 |
| 783 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); | 801 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); |
| 784 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); | 802 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
| 785 const size_t data_size = | 803 const size_t data_size = |
| 786 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 804 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
| 787 msg->set_input_data(frame->data_, data_size); | 805 msg->set_input_data(frame->data_, data_size); |
| 788 } | 806 } |
| 789 #endif | 807 #endif |
| 790 | 808 |
| 791 capture_.capture_audio->DeinterleaveFrom(frame); | 809 capture_.capture_audio->DeinterleaveFrom(frame); |
| 792 RETURN_ON_ERR(ProcessStreamLocked()); | 810 RETURN_ON_ERR(ProcessCaptureStreamLocked()); |
| 793 capture_.capture_audio->InterleaveTo( | 811 capture_.capture_audio->InterleaveTo( |
| 794 frame, submodule_states_.CaptureMultiBandProcessingActive()); | 812 frame, submodule_states_.CaptureMultiBandProcessingActive()); |
| 795 | 813 |
| 796 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 814 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 797 if (debug_dump_.debug_file->is_open()) { | 815 if (debug_dump_.debug_file->is_open()) { |
| 798 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); | 816 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
| 799 const size_t data_size = | 817 const size_t data_size = |
| 800 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 818 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
| 801 msg->set_output_data(frame->data_, data_size); | 819 msg->set_output_data(frame->data_, data_size); |
| 802 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), | 820 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 803 &debug_dump_.num_bytes_left_for_log_, | 821 &debug_dump_.num_bytes_left_for_log_, |
| 804 &crit_debug_, &debug_dump_.capture)); | 822 &crit_debug_, &debug_dump_.capture)); |
| 805 } | 823 } |
| 806 #endif | 824 #endif |
| 807 | 825 |
| 808 return kNoError; | 826 return kNoError; |
| 809 } | 827 } |
| 810 | 828 |
| 811 int AudioProcessingImpl::ProcessStreamLocked() { | 829 int AudioProcessingImpl::ProcessCaptureStreamLocked() { |
| 812 // Ensure that not both the AEC and AECM are active at the same time. | 830 // Ensure that not both the AEC and AECM are active at the same time. |
| 813 // TODO(peah): Simplify once the public API Enable functions for these | 831 // TODO(peah): Simplify once the public API Enable functions for these |
| 814 // are moved to APM. | 832 // are moved to APM. |
| 815 RTC_DCHECK(!(public_submodules_->echo_cancellation->is_enabled() && | 833 RTC_DCHECK(!(public_submodules_->echo_cancellation->is_enabled() && |
| 816 public_submodules_->echo_control_mobile->is_enabled())); | 834 public_submodules_->echo_control_mobile->is_enabled())); |
| 817 | 835 |
| 818 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 836 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 819 if (debug_dump_.debug_file->is_open()) { | 837 if (debug_dump_.debug_file->is_open()) { |
| 820 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); | 838 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
| 821 msg->set_delay(capture_nonlocked_.stream_delay_ms); | 839 msg->set_delay(capture_nonlocked_.stream_delay_ms); |
| 822 msg->set_drift( | 840 msg->set_drift( |
| 823 public_submodules_->echo_cancellation->stream_drift_samples()); | 841 public_submodules_->echo_cancellation->stream_drift_samples()); |
| 824 msg->set_level(gain_control()->stream_analog_level()); | 842 msg->set_level(gain_control()->stream_analog_level()); |
| 825 msg->set_keypress(capture_.key_pressed); | 843 msg->set_keypress(capture_.key_pressed); |
| 826 } | 844 } |
| 827 #endif | 845 #endif |
| 828 | 846 |
| 829 MaybeUpdateHistograms(); | 847 MaybeUpdateHistograms(); |
| 830 | 848 |
| 831 AudioBuffer* ca = capture_.capture_audio.get(); // For brevity. | 849 AudioBuffer* capture_buffer = capture_.capture_audio.get(); // For brevity. |
| 832 | 850 |
| 833 if (constants_.use_experimental_agc && | 851 if (constants_.use_experimental_agc && |
| 834 public_submodules_->gain_control->is_enabled()) { | 852 public_submodules_->gain_control->is_enabled()) { |
| 835 private_submodules_->agc_manager->AnalyzePreProcess( | 853 private_submodules_->agc_manager->AnalyzePreProcess( |
| 836 ca->channels()[0], ca->num_channels(), | 854 capture_buffer->channels()[0], capture_buffer->num_channels(), |
| 837 capture_nonlocked_.fwd_proc_format.num_frames()); | 855 capture_nonlocked_.capture_processing_format.num_frames()); |
| 838 } | 856 } |
| 839 | 857 |
| 840 if (submodule_states_.CaptureMultiBandSubModulesActive() && | 858 if (submodule_states_.CaptureMultiBandSubModulesActive() && |
| 841 SampleRateSupportsMultiBand( | 859 SampleRateSupportsMultiBand( |
| 842 capture_nonlocked_.fwd_proc_format.sample_rate_hz())) { | 860 capture_nonlocked_.capture_processing_format.sample_rate_hz())) { |
| 843 ca->SplitIntoFrequencyBands(); | 861 capture_buffer->SplitIntoFrequencyBands(); |
| 844 } | 862 } |
| 845 | 863 |
| 846 if (capture_nonlocked_.beamformer_enabled) { | 864 if (capture_nonlocked_.beamformer_enabled) { |
| 847 private_submodules_->beamformer->AnalyzeChunk(*ca->split_data_f()); | 865 private_submodules_->beamformer->AnalyzeChunk( |
| 866 *capture_buffer->split_data_f()); | |
| 848 // Discards all channels by the leftmost one. | 867 // Discards all channels by the leftmost one. |
| 849 ca->set_num_channels(1); | 868 capture_buffer->set_num_channels(1); |
| 850 } | 869 } |
| 851 | 870 |
| 852 public_submodules_->high_pass_filter->ProcessCaptureAudio(ca); | 871 public_submodules_->high_pass_filter->ProcessCaptureAudio(capture_buffer); |
| 853 RETURN_ON_ERR(public_submodules_->gain_control->AnalyzeCaptureAudio(ca)); | 872 RETURN_ON_ERR( |
| 854 public_submodules_->noise_suppression->AnalyzeCaptureAudio(ca); | 873 public_submodules_->gain_control->AnalyzeCaptureAudio(capture_buffer)); |
| 874 public_submodules_->noise_suppression->AnalyzeCaptureAudio(capture_buffer); | |
| 855 | 875 |
| 856 // Ensure that the stream delay was set before the call to the | 876 // Ensure that the stream delay was set before the call to the |
| 857 // AEC ProcessCaptureAudio function. | 877 // AEC ProcessCaptureAudio function. |
| 858 if (public_submodules_->echo_cancellation->is_enabled() && | 878 if (public_submodules_->echo_cancellation->is_enabled() && |
| 859 !was_stream_delay_set()) { | 879 !was_stream_delay_set()) { |
| 860 return AudioProcessing::kStreamParameterNotSetError; | 880 return AudioProcessing::kStreamParameterNotSetError; |
| 861 } | 881 } |
| 862 | 882 |
| 863 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio( | 883 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio( |
| 864 ca, stream_delay_ms())); | 884 capture_buffer, stream_delay_ms())); |
| 865 | 885 |
| 866 if (public_submodules_->echo_control_mobile->is_enabled() && | 886 if (public_submodules_->echo_control_mobile->is_enabled() && |
| 867 public_submodules_->noise_suppression->is_enabled()) { | 887 public_submodules_->noise_suppression->is_enabled()) { |
| 868 ca->CopyLowPassToReference(); | 888 capture_buffer->CopyLowPassToReference(); |
| 869 } | 889 } |
| 870 public_submodules_->noise_suppression->ProcessCaptureAudio(ca); | 890 public_submodules_->noise_suppression->ProcessCaptureAudio(capture_buffer); |
| 871 #if WEBRTC_INTELLIGIBILITY_ENHANCER | 891 #if WEBRTC_INTELLIGIBILITY_ENHANCER |
| 872 if (capture_nonlocked_.intelligibility_enabled) { | 892 if (capture_nonlocked_.intelligibility_enabled) { |
| 873 RTC_DCHECK(public_submodules_->noise_suppression->is_enabled()); | 893 RTC_DCHECK(public_submodules_->noise_suppression->is_enabled()); |
| 874 int gain_db = public_submodules_->gain_control->is_enabled() ? | 894 int gain_db = public_submodules_->gain_control->is_enabled() ? |
| 875 public_submodules_->gain_control->compression_gain_db() : | 895 public_submodules_->gain_control->compression_gain_db() : |
| 876 0; | 896 0; |
| 877 float gain = std::pow(10.f, gain_db / 20.f); | 897 float gain = std::pow(10.f, gain_db / 20.f); |
| 878 gain *= capture_nonlocked_.level_controller_enabled ? | 898 gain *= capture_nonlocked_.level_controller_enabled ? |
| 879 private_submodules_->level_controller->GetLastGain() : | 899 private_submodules_->level_controller->GetLastGain() : |
| 880 1.f; | 900 1.f; |
| 881 public_submodules_->intelligibility_enhancer->SetCaptureNoiseEstimate( | 901 public_submodules_->intelligibility_enhancer->SetCaptureNoiseEstimate( |
| 882 public_submodules_->noise_suppression->NoiseEstimate(), gain); | 902 public_submodules_->noise_suppression->NoiseEstimate(), gain); |
| 883 } | 903 } |
| 884 #endif | 904 #endif |
| 885 | 905 |
| 886 // Ensure that the stream delay was set before the call to the | 906 // Ensure that the stream delay was set before the call to the |
| 887 // AECM ProcessCaptureAudio function. | 907 // AECM ProcessCaptureAudio function. |
| 888 if (public_submodules_->echo_control_mobile->is_enabled() && | 908 if (public_submodules_->echo_control_mobile->is_enabled() && |
| 889 !was_stream_delay_set()) { | 909 !was_stream_delay_set()) { |
| 890 return AudioProcessing::kStreamParameterNotSetError; | 910 return AudioProcessing::kStreamParameterNotSetError; |
| 891 } | 911 } |
| 892 | 912 |
| 893 RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessCaptureAudio( | 913 RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessCaptureAudio( |
| 894 ca, stream_delay_ms())); | 914 capture_buffer, stream_delay_ms())); |
| 895 | 915 |
| 896 if (capture_nonlocked_.beamformer_enabled) { | 916 if (capture_nonlocked_.beamformer_enabled) { |
| 897 private_submodules_->beamformer->PostFilter(ca->split_data_f()); | 917 private_submodules_->beamformer->PostFilter(capture_buffer->split_data_f()); |
| 898 } | 918 } |
| 899 | 919 |
| 900 public_submodules_->voice_detection->ProcessCaptureAudio(ca); | 920 public_submodules_->voice_detection->ProcessCaptureAudio(capture_buffer); |
| 901 | 921 |
| 902 if (constants_.use_experimental_agc && | 922 if (constants_.use_experimental_agc && |
| 903 public_submodules_->gain_control->is_enabled() && | 923 public_submodules_->gain_control->is_enabled() && |
| 904 (!capture_nonlocked_.beamformer_enabled || | 924 (!capture_nonlocked_.beamformer_enabled || |
| 905 private_submodules_->beamformer->is_target_present())) { | 925 private_submodules_->beamformer->is_target_present())) { |
| 906 private_submodules_->agc_manager->Process( | 926 private_submodules_->agc_manager->Process( |
| 907 ca->split_bands_const(0)[kBand0To8kHz], ca->num_frames_per_band(), | 927 capture_buffer->split_bands_const(0)[kBand0To8kHz], |
| 908 capture_nonlocked_.split_rate); | 928 capture_buffer->num_frames_per_band(), capture_nonlocked_.split_rate); |
| 909 } | 929 } |
| 910 RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio( | 930 RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio( |
| 911 ca, echo_cancellation()->stream_has_echo())); | 931 capture_buffer, echo_cancellation()->stream_has_echo())); |
| 912 | 932 |
| 913 if (submodule_states_.CaptureMultiBandProcessingActive() && | 933 if (submodule_states_.CaptureMultiBandProcessingActive() && |
| 914 SampleRateSupportsMultiBand( | 934 SampleRateSupportsMultiBand( |
| 915 capture_nonlocked_.fwd_proc_format.sample_rate_hz())) { | 935 capture_nonlocked_.capture_processing_format.sample_rate_hz())) { |
| 916 ca->MergeFrequencyBands(); | 936 capture_buffer->MergeFrequencyBands(); |
| 917 } | 937 } |
| 918 | 938 |
| 919 // TODO(aluebs): Investigate if the transient suppression placement should be | 939 // TODO(aluebs): Investigate if the transient suppression placement should be |
| 920 // before or after the AGC. | 940 // before or after the AGC. |
| 921 if (capture_.transient_suppressor_enabled) { | 941 if (capture_.transient_suppressor_enabled) { |
| 922 float voice_probability = | 942 float voice_probability = |
| 923 private_submodules_->agc_manager.get() | 943 private_submodules_->agc_manager.get() |
| 924 ? private_submodules_->agc_manager->voice_probability() | 944 ? private_submodules_->agc_manager->voice_probability() |
| 925 : 1.f; | 945 : 1.f; |
| 926 | 946 |
| 927 public_submodules_->transient_suppressor->Suppress( | 947 public_submodules_->transient_suppressor->Suppress( |
| 928 ca->channels_f()[0], ca->num_frames(), ca->num_channels(), | 948 capture_buffer->channels_f()[0], capture_buffer->num_frames(), |
| 929 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(), | 949 capture_buffer->num_channels(), |
| 930 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability, | 950 capture_buffer->split_bands_const_f(0)[kBand0To8kHz], |
| 951 capture_buffer->num_frames_per_band(), capture_buffer->keyboard_data(), | |
| 952 capture_buffer->num_keyboard_frames(), voice_probability, | |
| 931 capture_.key_pressed); | 953 capture_.key_pressed); |
| 932 } | 954 } |
| 933 | 955 |
| 934 if (capture_nonlocked_.level_controller_enabled) { | 956 if (capture_nonlocked_.level_controller_enabled) { |
| 935 private_submodules_->level_controller->Process(ca); | 957 private_submodules_->level_controller->Process(capture_buffer); |
| 936 } | 958 } |
| 937 | 959 |
| 938 // The level estimator operates on the recombined data. | 960 // The level estimator operates on the recombined data. |
| 939 public_submodules_->level_estimator->ProcessStream(ca); | 961 public_submodules_->level_estimator->ProcessStream(capture_buffer); |
| 940 | 962 |
| 941 capture_.was_stream_delay_set = false; | 963 capture_.was_stream_delay_set = false; |
| 942 return kNoError; | 964 return kNoError; |
| 943 } | 965 } |
| 944 | 966 |
| 945 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, | 967 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, |
| 946 size_t samples_per_channel, | 968 size_t samples_per_channel, |
| 947 int rev_sample_rate_hz, | 969 int sample_rate_hz, |
| 948 ChannelLayout layout) { | 970 ChannelLayout layout) { |
| 949 TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_ChannelLayout"); | 971 TRACE_EVENT0("webrtc", "AudioProcessing::AnalyzeReverseStream_ChannelLayout"); |
| 950 rtc::CritScope cs(&crit_render_); | 972 rtc::CritScope cs(&crit_render_); |
| 951 const StreamConfig reverse_config = { | 973 const StreamConfig reverse_config = { |
| 952 rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), | 974 sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), |
| 953 }; | 975 }; |
| 954 if (samples_per_channel != reverse_config.num_frames()) { | 976 if (samples_per_channel != reverse_config.num_frames()) { |
| 955 return kBadDataLengthError; | 977 return kBadDataLengthError; |
| 956 } | 978 } |
| 957 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config); | 979 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config); |
| 958 } | 980 } |
| 959 | 981 |
| 960 int AudioProcessingImpl::ProcessReverseStream( | 982 int AudioProcessingImpl::ProcessReverseStream(const float* const* src, |
| 961 const float* const* src, | 983 const StreamConfig& input_config, |
| 962 const StreamConfig& reverse_input_config, | 984 const StreamConfig& output_config, |
| 963 const StreamConfig& reverse_output_config, | 985 float* const* dest) { |
| 964 float* const* dest) { | |
| 965 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig"); | 986 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_StreamConfig"); |
| 966 rtc::CritScope cs(&crit_render_); | 987 rtc::CritScope cs(&crit_render_); |
| 967 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, reverse_input_config, | 988 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, input_config, output_config)); |
| 968 reverse_output_config)); | |
| 969 if (submodule_states_.RenderMultiBandProcessingActive()) { | 989 if (submodule_states_.RenderMultiBandProcessingActive()) { |
| 970 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(), | 990 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(), |
| 971 dest); | 991 dest); |
| 972 } else if (formats_.api_format.reverse_input_stream() != | 992 } else if (formats_.api_format.reverse_input_stream() != |
| 973 formats_.api_format.reverse_output_stream()) { | 993 formats_.api_format.reverse_output_stream()) { |
| 974 render_.render_converter->Convert(src, reverse_input_config.num_samples(), | 994 render_.render_converter->Convert(src, input_config.num_samples(), dest, |
| 975 dest, | 995 output_config.num_samples()); |
| 976 reverse_output_config.num_samples()); | |
| 977 } else { | 996 } else { |
| 978 CopyAudioIfNeeded(src, reverse_input_config.num_frames(), | 997 CopyAudioIfNeeded(src, input_config.num_frames(), |
| 979 reverse_input_config.num_channels(), dest); | 998 input_config.num_channels(), dest); |
| 980 } | 999 } |
| 981 | 1000 |
| 982 return kNoError; | 1001 return kNoError; |
| 983 } | 1002 } |
| 984 | 1003 |
| 985 int AudioProcessingImpl::AnalyzeReverseStreamLocked( | 1004 int AudioProcessingImpl::AnalyzeReverseStreamLocked( |
| 986 const float* const* src, | 1005 const float* const* src, |
| 987 const StreamConfig& reverse_input_config, | 1006 const StreamConfig& input_config, |
| 988 const StreamConfig& reverse_output_config) { | 1007 const StreamConfig& output_config) { |
| 989 if (src == nullptr) { | 1008 if (src == nullptr) { |
| 990 return kNullPointerError; | 1009 return kNullPointerError; |
| 991 } | 1010 } |
| 992 | 1011 |
| 993 if (reverse_input_config.num_channels() == 0) { | 1012 if (input_config.num_channels() == 0) { |
| 994 return kBadNumberChannelsError; | 1013 return kBadNumberChannelsError; |
| 995 } | 1014 } |
| 996 | 1015 |
| 997 ProcessingConfig processing_config = formats_.api_format; | 1016 ProcessingConfig processing_config = formats_.api_format; |
| 998 processing_config.reverse_input_stream() = reverse_input_config; | 1017 processing_config.reverse_input_stream() = input_config; |
| 999 processing_config.reverse_output_stream() = reverse_output_config; | 1018 processing_config.reverse_output_stream() = output_config; |
| 1000 | 1019 |
| 1001 RETURN_ON_ERR(MaybeInitializeRender(processing_config)); | 1020 RETURN_ON_ERR(MaybeInitializeRender(processing_config)); |
| 1002 assert(reverse_input_config.num_frames() == | 1021 assert(input_config.num_frames() == |
| 1003 formats_.api_format.reverse_input_stream().num_frames()); | 1022 formats_.api_format.reverse_input_stream().num_frames()); |
| 1004 | 1023 |
| 1005 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1024 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 1006 if (debug_dump_.debug_file->is_open()) { | 1025 if (debug_dump_.debug_file->is_open()) { |
| 1007 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); | 1026 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); |
| 1008 audioproc::ReverseStream* msg = | 1027 audioproc::ReverseStream* msg = |
| 1009 debug_dump_.render.event_msg->mutable_reverse_stream(); | 1028 debug_dump_.render.event_msg->mutable_reverse_stream(); |
| 1010 const size_t channel_size = | 1029 const size_t channel_size = |
| 1011 sizeof(float) * formats_.api_format.reverse_input_stream().num_frames(); | 1030 sizeof(float) * formats_.api_format.reverse_input_stream().num_frames(); |
| 1012 for (size_t i = 0; | 1031 for (size_t i = 0; |
| 1013 i < formats_.api_format.reverse_input_stream().num_channels(); ++i) | 1032 i < formats_.api_format.reverse_input_stream().num_channels(); ++i) |
| 1014 msg->add_channel(src[i], channel_size); | 1033 msg->add_channel(src[i], channel_size); |
| 1015 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), | 1034 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 1016 &debug_dump_.num_bytes_left_for_log_, | 1035 &debug_dump_.num_bytes_left_for_log_, |
| 1017 &crit_debug_, &debug_dump_.render)); | 1036 &crit_debug_, &debug_dump_.render)); |
| 1018 } | 1037 } |
| 1019 #endif | 1038 #endif |
| 1020 | 1039 |
| 1021 render_.render_audio->CopyFrom(src, | 1040 render_.render_audio->CopyFrom(src, |
| 1022 formats_.api_format.reverse_input_stream()); | 1041 formats_.api_format.reverse_input_stream()); |
| 1023 return ProcessReverseStreamLocked(); | 1042 return ProcessRenderStreamLocked(); |
| 1024 } | 1043 } |
| 1025 | 1044 |
| 1026 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { | 1045 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { |
| 1027 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame"); | 1046 TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame"); |
| 1028 rtc::CritScope cs(&crit_render_); | 1047 rtc::CritScope cs(&crit_render_); |
| 1029 if (frame == nullptr) { | 1048 if (frame == nullptr) { |
| 1030 return kNullPointerError; | 1049 return kNullPointerError; |
| 1031 } | 1050 } |
| 1032 // Must be a native rate. | 1051 // Must be a native rate. |
| 1033 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 1052 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
| (...skipping 30 matching lines...) Expand all Loading... | |
| 1064 debug_dump_.render.event_msg->mutable_reverse_stream(); | 1083 debug_dump_.render.event_msg->mutable_reverse_stream(); |
| 1065 const size_t data_size = | 1084 const size_t data_size = |
| 1066 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 1085 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
| 1067 msg->set_data(frame->data_, data_size); | 1086 msg->set_data(frame->data_, data_size); |
| 1068 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), | 1087 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 1069 &debug_dump_.num_bytes_left_for_log_, | 1088 &debug_dump_.num_bytes_left_for_log_, |
| 1070 &crit_debug_, &debug_dump_.render)); | 1089 &crit_debug_, &debug_dump_.render)); |
| 1071 } | 1090 } |
| 1072 #endif | 1091 #endif |
| 1073 render_.render_audio->DeinterleaveFrom(frame); | 1092 render_.render_audio->DeinterleaveFrom(frame); |
| 1074 RETURN_ON_ERR(ProcessReverseStreamLocked()); | 1093 RETURN_ON_ERR(ProcessRenderStreamLocked()); |
| 1075 render_.render_audio->InterleaveTo( | 1094 render_.render_audio->InterleaveTo( |
| 1076 frame, submodule_states_.RenderMultiBandProcessingActive()); | 1095 frame, submodule_states_.RenderMultiBandProcessingActive()); |
| 1077 return kNoError; | 1096 return kNoError; |
| 1078 } | 1097 } |
| 1079 | 1098 |
| 1080 int AudioProcessingImpl::ProcessReverseStreamLocked() { | 1099 int AudioProcessingImpl::ProcessRenderStreamLocked() { |
| 1081 AudioBuffer* ra = render_.render_audio.get(); // For brevity. | 1100 AudioBuffer* render_buffer = render_.render_audio.get(); // For brevity. |
| 1082 if (submodule_states_.RenderMultiBandSubModulesActive() && | 1101 if (submodule_states_.RenderMultiBandSubModulesActive() && |
| 1083 SampleRateSupportsMultiBand(formats_.rev_proc_format.sample_rate_hz())) { | 1102 SampleRateSupportsMultiBand( |
| 1084 ra->SplitIntoFrequencyBands(); | 1103 formats_.render_processing_format.sample_rate_hz())) { |
| 1104 render_buffer->SplitIntoFrequencyBands(); | |
| 1085 } | 1105 } |
| 1086 | 1106 |
| 1087 #if WEBRTC_INTELLIGIBILITY_ENHANCER | 1107 #if WEBRTC_INTELLIGIBILITY_ENHANCER |
| 1088 if (capture_nonlocked_.intelligibility_enabled) { | 1108 if (capture_nonlocked_.intelligibility_enabled) { |
| 1089 public_submodules_->intelligibility_enhancer->ProcessRenderAudio( | 1109 public_submodules_->intelligibility_enhancer->ProcessRenderAudio( |
| 1090 ra->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate, | 1110 render_buffer->split_channels_f(kBand0To8kHz), |
| 1091 ra->num_channels()); | 1111 capture_nonlocked_.split_rate, render_buffer->num_channels()); |
| 1092 } | 1112 } |
| 1093 #endif | 1113 #endif |
| 1094 | 1114 |
| 1095 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessRenderAudio(ra)); | |
| 1096 RETURN_ON_ERR( | 1115 RETURN_ON_ERR( |
| 1097 public_submodules_->echo_control_mobile->ProcessRenderAudio(ra)); | 1116 public_submodules_->echo_cancellation->ProcessRenderAudio(render_buffer)); |
| 1117 RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessRenderAudio( | |
| 1118 render_buffer)); | |
| 1098 if (!constants_.use_experimental_agc) { | 1119 if (!constants_.use_experimental_agc) { |
| 1099 RETURN_ON_ERR(public_submodules_->gain_control->ProcessRenderAudio(ra)); | 1120 RETURN_ON_ERR( |
| 1121 public_submodules_->gain_control->ProcessRenderAudio(render_buffer)); | |
| 1100 } | 1122 } |
| 1101 | 1123 |
| 1102 if (submodule_states_.RenderMultiBandProcessingActive() && | 1124 if (submodule_states_.RenderMultiBandProcessingActive() && |
| 1103 SampleRateSupportsMultiBand(formats_.rev_proc_format.sample_rate_hz())) { | 1125 SampleRateSupportsMultiBand( |
| 1104 ra->MergeFrequencyBands(); | 1126 formats_.render_processing_format.sample_rate_hz())) { |
| 1127 render_buffer->MergeFrequencyBands(); | |
| 1105 } | 1128 } |
| 1106 | 1129 |
| 1107 return kNoError; | 1130 return kNoError; |
| 1108 } | 1131 } |
| 1109 | 1132 |
| 1110 int AudioProcessingImpl::set_stream_delay_ms(int delay) { | 1133 int AudioProcessingImpl::set_stream_delay_ms(int delay) { |
| 1111 rtc::CritScope cs(&crit_capture_); | 1134 rtc::CritScope cs(&crit_capture_); |
| 1112 Error retval = kNoError; | 1135 Error retval = kNoError; |
| 1113 capture_.was_stream_delay_set = true; | 1136 capture_.was_stream_delay_set = true; |
| 1114 delay += capture_.delay_offset_ms; | 1137 delay += capture_.delay_offset_ms; |
| (...skipping 157 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1272 public_submodules_->noise_suppression->is_enabled(), | 1295 public_submodules_->noise_suppression->is_enabled(), |
| 1273 capture_nonlocked_.intelligibility_enabled, | 1296 capture_nonlocked_.intelligibility_enabled, |
| 1274 capture_nonlocked_.beamformer_enabled, | 1297 capture_nonlocked_.beamformer_enabled, |
| 1275 public_submodules_->gain_control->is_enabled(), | 1298 public_submodules_->gain_control->is_enabled(), |
| 1276 capture_nonlocked_.level_controller_enabled, | 1299 capture_nonlocked_.level_controller_enabled, |
| 1277 public_submodules_->voice_detection->is_enabled(), | 1300 public_submodules_->voice_detection->is_enabled(), |
| 1278 public_submodules_->level_estimator->is_enabled(), | 1301 public_submodules_->level_estimator->is_enabled(), |
| 1279 capture_.transient_suppressor_enabled); | 1302 capture_.transient_suppressor_enabled); |
| 1280 } | 1303 } |
| 1281 | 1304 |
| 1282 void AudioProcessingImpl::InitializeExperimentalAgc() { | |
| 1283 if (constants_.use_experimental_agc) { | |
| 1284 if (!private_submodules_->agc_manager.get()) { | |
| 1285 private_submodules_->agc_manager.reset(new AgcManagerDirect( | |
| 1286 public_submodules_->gain_control.get(), | |
| 1287 public_submodules_->gain_control_for_experimental_agc.get(), | |
| 1288 constants_.agc_startup_min_volume)); | |
| 1289 } | |
| 1290 private_submodules_->agc_manager->Initialize(); | |
| 1291 private_submodules_->agc_manager->SetCaptureMuted( | |
| 1292 capture_.output_will_be_muted); | |
| 1293 } | |
| 1294 } | |
| 1295 | 1305 |
| 1296 void AudioProcessingImpl::InitializeTransient() { | 1306 void AudioProcessingImpl::InitializeTransient() { |
| 1297 if (capture_.transient_suppressor_enabled) { | 1307 if (capture_.transient_suppressor_enabled) { |
| 1298 if (!public_submodules_->transient_suppressor.get()) { | 1308 if (!public_submodules_->transient_suppressor.get()) { |
| 1299 public_submodules_->transient_suppressor.reset(new TransientSuppressor()); | 1309 public_submodules_->transient_suppressor.reset(new TransientSuppressor()); |
| 1300 } | 1310 } |
| 1301 public_submodules_->transient_suppressor->Initialize( | 1311 public_submodules_->transient_suppressor->Initialize( |
| 1302 capture_nonlocked_.fwd_proc_format.sample_rate_hz(), | 1312 capture_nonlocked_.capture_processing_format.sample_rate_hz(), |
| 1303 capture_nonlocked_.split_rate, | 1313 capture_nonlocked_.split_rate, num_proc_channels()); |
| 1304 num_proc_channels()); | |
| 1305 } | 1314 } |
| 1306 } | 1315 } |
| 1307 | 1316 |
| 1308 void AudioProcessingImpl::InitializeBeamformer() { | 1317 void AudioProcessingImpl::InitializeBeamformer() { |
| 1309 if (capture_nonlocked_.beamformer_enabled) { | 1318 if (capture_nonlocked_.beamformer_enabled) { |
| 1310 if (!private_submodules_->beamformer) { | 1319 if (!private_submodules_->beamformer) { |
| 1311 private_submodules_->beamformer.reset(new NonlinearBeamformer( | 1320 private_submodules_->beamformer.reset(new NonlinearBeamformer( |
| 1312 capture_.array_geometry, 1u, capture_.target_direction)); | 1321 capture_.array_geometry, 1u, capture_.target_direction)); |
| 1313 } | 1322 } |
| 1314 private_submodules_->beamformer->Initialize(kChunkSizeMs, | 1323 private_submodules_->beamformer->Initialize(kChunkSizeMs, |
| 1315 capture_nonlocked_.split_rate); | 1324 capture_nonlocked_.split_rate); |
| 1316 } | 1325 } |
| 1317 } | 1326 } |
| 1318 | 1327 |
| 1319 void AudioProcessingImpl::InitializeIntelligibility() { | 1328 void AudioProcessingImpl::InitializeIntelligibility() { |
| 1320 #if WEBRTC_INTELLIGIBILITY_ENHANCER | 1329 #if WEBRTC_INTELLIGIBILITY_ENHANCER |
| 1321 if (capture_nonlocked_.intelligibility_enabled) { | 1330 if (capture_nonlocked_.intelligibility_enabled) { |
| 1322 public_submodules_->intelligibility_enhancer.reset( | 1331 public_submodules_->intelligibility_enhancer.reset( |
| 1323 new IntelligibilityEnhancer(capture_nonlocked_.split_rate, | 1332 new IntelligibilityEnhancer(capture_nonlocked_.split_rate, |
| 1324 render_.render_audio->num_channels(), | 1333 render_.render_audio->num_channels(), |
| 1325 NoiseSuppressionImpl::num_noise_bins())); | 1334 NoiseSuppressionImpl::num_noise_bins())); |
| 1326 } | 1335 } |
| 1327 #endif | 1336 #endif |
| 1328 } | 1337 } |
| 1329 | 1338 |
| 1330 void AudioProcessingImpl::InitializeHighPassFilter() { | |
| 1331 public_submodules_->high_pass_filter->Initialize(num_proc_channels(), | |
| 1332 proc_sample_rate_hz()); | |
| 1333 } | |
| 1334 | |
| 1335 void AudioProcessingImpl::InitializeNoiseSuppression() { | |
| 1336 public_submodules_->noise_suppression->Initialize(num_proc_channels(), | |
| 1337 proc_sample_rate_hz()); | |
| 1338 } | |
| 1339 | |
| 1340 void AudioProcessingImpl::InitializeEchoCanceller() { | |
| 1341 public_submodules_->echo_cancellation->Initialize( | |
| 1342 proc_sample_rate_hz(), num_reverse_channels(), num_output_channels(), | |
| 1343 num_proc_channels()); | |
| 1344 } | |
| 1345 | |
| 1346 void AudioProcessingImpl::InitializeGainController() { | |
| 1347 public_submodules_->gain_control->Initialize(num_proc_channels(), | |
| 1348 proc_sample_rate_hz()); | |
| 1349 } | |
| 1350 | |
| 1351 void AudioProcessingImpl::InitializeEchoControlMobile() { | |
| 1352 public_submodules_->echo_control_mobile->Initialize( | |
| 1353 proc_split_sample_rate_hz(), | |
| 1354 num_reverse_channels(), | |
| 1355 num_output_channels()); | |
| 1356 } | |
| 1357 | |
| 1358 void AudioProcessingImpl::InitializeLevelEstimator() { | |
| 1359 public_submodules_->level_estimator->Initialize(); | |
| 1360 } | |
| 1361 | |
| 1362 void AudioProcessingImpl::InitializeLevelController() { | 1339 void AudioProcessingImpl::InitializeLevelController() { |
| 1363 private_submodules_->level_controller->Initialize(proc_sample_rate_hz()); | 1340 private_submodules_->level_controller->Initialize(proc_sample_rate_hz()); |
| 1364 } | 1341 } |
| 1365 | 1342 |
| 1366 void AudioProcessingImpl::InitializeVoiceDetection() { | 1343 |
|
the sun
2016/09/16 08:34:38
too much space
peah-webrtc
2016/09/16 11:43:14
Oups! Thanks!
Done.
| |
| 1367 public_submodules_->voice_detection->Initialize(proc_split_sample_rate_hz()); | 1344 |
| 1368 } | 1345 |
| 1369 | 1346 |
| 1370 void AudioProcessingImpl::MaybeUpdateHistograms() { | 1347 void AudioProcessingImpl::MaybeUpdateHistograms() { |
| 1371 static const int kMinDiffDelayMs = 60; | 1348 static const int kMinDiffDelayMs = 60; |
| 1372 | 1349 |
| 1373 if (echo_cancellation()->is_enabled()) { | 1350 if (echo_cancellation()->is_enabled()) { |
| 1374 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. | 1351 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. |
| 1375 // If a stream has echo we know that the echo_cancellation is in process. | 1352 // If a stream has echo we know that the echo_cancellation is in process. |
| 1376 if (capture_.stream_delay_jumps == -1 && | 1353 if (capture_.stream_delay_jumps == -1 && |
| 1377 echo_cancellation()->stream_has_echo()) { | 1354 echo_cancellation()->stream_has_echo()) { |
| 1378 capture_.stream_delay_jumps = 0; | 1355 capture_.stream_delay_jumps = 0; |
| (...skipping 209 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1588 delay_offset_ms(0), | 1565 delay_offset_ms(0), |
| 1589 was_stream_delay_set(false), | 1566 was_stream_delay_set(false), |
| 1590 last_stream_delay_ms(0), | 1567 last_stream_delay_ms(0), |
| 1591 last_aec_system_delay_ms(0), | 1568 last_aec_system_delay_ms(0), |
| 1592 stream_delay_jumps(-1), | 1569 stream_delay_jumps(-1), |
| 1593 output_will_be_muted(false), | 1570 output_will_be_muted(false), |
| 1594 key_pressed(false), | 1571 key_pressed(false), |
| 1595 transient_suppressor_enabled(transient_suppressor_enabled), | 1572 transient_suppressor_enabled(transient_suppressor_enabled), |
| 1596 array_geometry(array_geometry), | 1573 array_geometry(array_geometry), |
| 1597 target_direction(target_direction), | 1574 target_direction(target_direction), |
| 1598 fwd_proc_format(kSampleRate16kHz), | 1575 capture_processing_format(kSampleRate16kHz), |
| 1599 split_rate(kSampleRate16kHz) {} | 1576 split_rate(kSampleRate16kHz) {} |
| 1600 | 1577 |
| 1601 AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default; | 1578 AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default; |
| 1602 | 1579 |
| 1603 AudioProcessingImpl::ApmRenderState::ApmRenderState() = default; | 1580 AudioProcessingImpl::ApmRenderState::ApmRenderState() = default; |
| 1604 | 1581 |
| 1605 AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default; | 1582 AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default; |
| 1606 | 1583 |
| 1607 } // namespace webrtc | 1584 } // namespace webrtc |
| OLD | NEW |