OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
(...skipping 104 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
115 return uppermost_native_rate; | 115 return uppermost_native_rate; |
116 } | 116 } |
117 if (rate >= minimum_rate) { | 117 if (rate >= minimum_rate) { |
118 return rate; | 118 return rate; |
119 } | 119 } |
120 } | 120 } |
121 RTC_NOTREACHED(); | 121 RTC_NOTREACHED(); |
122 return uppermost_native_rate; | 122 return uppermost_native_rate; |
123 } | 123 } |
124 | 124 |
125 // Maximum length that a frame of samples can have. | 125 // Maximum lengths that frame of samples being passed from the render side to |
126 static const size_t kMaxAllowedValuesOfSamplesPerFrame = 160; | 126 // the capture side can have (does not apply to AEC3). |
| 127 static const size_t kMaxAllowedValuesOfSamplesPerBand = 160; |
| 128 static const size_t kMaxAllowedValuesOfSamplesPerFrame = 480; |
| 129 |
127 // Maximum number of frames to buffer in the render queue. | 130 // Maximum number of frames to buffer in the render queue. |
128 // TODO(peah): Decrease this once we properly handle hugely unbalanced | 131 // TODO(peah): Decrease this once we properly handle hugely unbalanced |
129 // reverse and forward call numbers. | 132 // reverse and forward call numbers. |
130 static const size_t kMaxNumFramesToBuffer = 100; | 133 static const size_t kMaxNumFramesToBuffer = 100; |
131 | 134 |
132 class HighPassFilterImpl : public HighPassFilter { | 135 class HighPassFilterImpl : public HighPassFilter { |
133 public: | 136 public: |
134 explicit HighPassFilterImpl(AudioProcessingImpl* apm) : apm_(apm) {} | 137 explicit HighPassFilterImpl(AudioProcessingImpl* apm) : apm_(apm) {} |
135 ~HighPassFilterImpl() override = default; | 138 ~HighPassFilterImpl() override = default; |
136 | 139 |
(...skipping 701 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
838 msg->add_output_channel(dest[i], channel_size); | 841 msg->add_output_channel(dest[i], channel_size); |
839 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), | 842 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
840 &debug_dump_.num_bytes_left_for_log_, | 843 &debug_dump_.num_bytes_left_for_log_, |
841 &crit_debug_, &debug_dump_.capture)); | 844 &crit_debug_, &debug_dump_.capture)); |
842 } | 845 } |
843 #endif | 846 #endif |
844 | 847 |
845 return kNoError; | 848 return kNoError; |
846 } | 849 } |
847 | 850 |
848 void AudioProcessingImpl::QueueRenderAudio(AudioBuffer* audio) { | 851 void AudioProcessingImpl::QueueBandedRenderAudio(AudioBuffer* audio) { |
849 EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(), | 852 EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(), |
850 num_reverse_channels(), | 853 num_reverse_channels(), |
851 &aec_render_queue_buffer_); | 854 &aec_render_queue_buffer_); |
852 | 855 |
853 RTC_DCHECK_GE(160, audio->num_frames_per_band()); | 856 RTC_DCHECK_GE(160, audio->num_frames_per_band()); |
854 | 857 |
855 // Insert the samples into the queue. | 858 // Insert the samples into the queue. |
856 if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) { | 859 if (!aec_render_signal_queue_->Insert(&aec_render_queue_buffer_)) { |
857 // The data queue is full and needs to be emptied. | 860 // The data queue is full and needs to be emptied. |
858 EmptyQueuedRenderAudio(); | 861 EmptyQueuedRenderAudio(); |
(...skipping 22 matching lines...) Expand all Loading... |
881 // Insert the samples into the queue. | 884 // Insert the samples into the queue. |
882 if (!agc_render_signal_queue_->Insert(&agc_render_queue_buffer_)) { | 885 if (!agc_render_signal_queue_->Insert(&agc_render_queue_buffer_)) { |
883 // The data queue is full and needs to be emptied. | 886 // The data queue is full and needs to be emptied. |
884 EmptyQueuedRenderAudio(); | 887 EmptyQueuedRenderAudio(); |
885 | 888 |
886 // Retry the insert (should always work). | 889 // Retry the insert (should always work). |
887 bool result = agc_render_signal_queue_->Insert(&agc_render_queue_buffer_); | 890 bool result = agc_render_signal_queue_->Insert(&agc_render_queue_buffer_); |
888 RTC_DCHECK(result); | 891 RTC_DCHECK(result); |
889 } | 892 } |
890 } | 893 } |
| 894 } |
891 | 895 |
| 896 void AudioProcessingImpl::QueueNonbandedRenderAudio(AudioBuffer* audio) { |
892 ResidualEchoDetector::PackRenderAudioBuffer(audio, &red_render_queue_buffer_); | 897 ResidualEchoDetector::PackRenderAudioBuffer(audio, &red_render_queue_buffer_); |
893 | 898 |
894 // Insert the samples into the queue. | 899 // Insert the samples into the queue. |
895 if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) { | 900 if (!red_render_signal_queue_->Insert(&red_render_queue_buffer_)) { |
896 // The data queue is full and needs to be emptied. | 901 // The data queue is full and needs to be emptied. |
897 EmptyQueuedRenderAudio(); | 902 EmptyQueuedRenderAudio(); |
898 | 903 |
899 // Retry the insert (should always work). | 904 // Retry the insert (should always work). |
900 bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_); | 905 bool result = red_render_signal_queue_->Insert(&red_render_queue_buffer_); |
901 RTC_DCHECK(result); | 906 RTC_DCHECK(result); |
902 } | 907 } |
903 } | 908 } |
904 | 909 |
905 void AudioProcessingImpl::AllocateRenderQueue() { | 910 void AudioProcessingImpl::AllocateRenderQueue() { |
906 const size_t new_aec_render_queue_element_max_size = | 911 const size_t new_aec_render_queue_element_max_size = |
907 std::max(static_cast<size_t>(1), | 912 std::max(static_cast<size_t>(1), |
908 kMaxAllowedValuesOfSamplesPerFrame * | 913 kMaxAllowedValuesOfSamplesPerBand * |
909 EchoCancellationImpl::NumCancellersRequired( | 914 EchoCancellationImpl::NumCancellersRequired( |
910 num_output_channels(), num_reverse_channels())); | 915 num_output_channels(), num_reverse_channels())); |
911 | 916 |
912 const size_t new_aecm_render_queue_element_max_size = | 917 const size_t new_aecm_render_queue_element_max_size = |
913 std::max(static_cast<size_t>(1), | 918 std::max(static_cast<size_t>(1), |
914 kMaxAllowedValuesOfSamplesPerFrame * | 919 kMaxAllowedValuesOfSamplesPerBand * |
915 EchoControlMobileImpl::NumCancellersRequired( | 920 EchoControlMobileImpl::NumCancellersRequired( |
916 num_output_channels(), num_reverse_channels())); | 921 num_output_channels(), num_reverse_channels())); |
917 | 922 |
918 const size_t new_agc_render_queue_element_max_size = | 923 const size_t new_agc_render_queue_element_max_size = |
919 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame); | 924 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerBand); |
920 | 925 |
921 const size_t new_red_render_queue_element_max_size = | 926 const size_t new_red_render_queue_element_max_size = |
922 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame); | 927 std::max(static_cast<size_t>(1), kMaxAllowedValuesOfSamplesPerFrame); |
923 | 928 |
924 // Reallocate the queues if the queue item sizes are too small to fit the | 929 // Reallocate the queues if the queue item sizes are too small to fit the |
925 // data to put in the queues. | 930 // data to put in the queues. |
926 if (aec_render_queue_element_max_size_ < | 931 if (aec_render_queue_element_max_size_ < |
927 new_aec_render_queue_element_max_size) { | 932 new_aec_render_queue_element_max_size) { |
928 aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size; | 933 aec_render_queue_element_max_size_ = new_aec_render_queue_element_max_size; |
929 | 934 |
(...skipping 298 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1228 // Ensure that the stream delay was set before the call to the | 1233 // Ensure that the stream delay was set before the call to the |
1229 // AECM ProcessCaptureAudio function. | 1234 // AECM ProcessCaptureAudio function. |
1230 if (public_submodules_->echo_control_mobile->is_enabled() && | 1235 if (public_submodules_->echo_control_mobile->is_enabled() && |
1231 !was_stream_delay_set()) { | 1236 !was_stream_delay_set()) { |
1232 return AudioProcessing::kStreamParameterNotSetError; | 1237 return AudioProcessing::kStreamParameterNotSetError; |
1233 } | 1238 } |
1234 | 1239 |
1235 RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessCaptureAudio( | 1240 RETURN_ON_ERR(public_submodules_->echo_control_mobile->ProcessCaptureAudio( |
1236 capture_buffer, stream_delay_ms())); | 1241 capture_buffer, stream_delay_ms())); |
1237 | 1242 |
1238 if (config_.residual_echo_detector.enabled) { | |
1239 private_submodules_->residual_echo_detector->AnalyzeCaptureAudio( | |
1240 rtc::ArrayView<const float>( | |
1241 capture_buffer->split_bands_const_f(0)[kBand0To8kHz], | |
1242 capture_buffer->num_frames_per_band())); | |
1243 } | |
1244 | 1243 |
1245 if (capture_nonlocked_.beamformer_enabled) { | 1244 if (capture_nonlocked_.beamformer_enabled) { |
1246 private_submodules_->beamformer->PostFilter(capture_buffer->split_data_f()); | 1245 private_submodules_->beamformer->PostFilter(capture_buffer->split_data_f()); |
1247 } | 1246 } |
1248 | 1247 |
1249 public_submodules_->voice_detection->ProcessCaptureAudio(capture_buffer); | 1248 public_submodules_->voice_detection->ProcessCaptureAudio(capture_buffer); |
1250 | 1249 |
1251 if (constants_.use_experimental_agc && | 1250 if (constants_.use_experimental_agc && |
1252 public_submodules_->gain_control->is_enabled() && | 1251 public_submodules_->gain_control->is_enabled() && |
1253 (!capture_nonlocked_.beamformer_enabled || | 1252 (!capture_nonlocked_.beamformer_enabled || |
1254 private_submodules_->beamformer->is_target_present())) { | 1253 private_submodules_->beamformer->is_target_present())) { |
1255 private_submodules_->agc_manager->Process( | 1254 private_submodules_->agc_manager->Process( |
1256 capture_buffer->split_bands_const(0)[kBand0To8kHz], | 1255 capture_buffer->split_bands_const(0)[kBand0To8kHz], |
1257 capture_buffer->num_frames_per_band(), capture_nonlocked_.split_rate); | 1256 capture_buffer->num_frames_per_band(), capture_nonlocked_.split_rate); |
1258 } | 1257 } |
1259 RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio( | 1258 RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio( |
1260 capture_buffer, echo_cancellation()->stream_has_echo())); | 1259 capture_buffer, echo_cancellation()->stream_has_echo())); |
1261 | 1260 |
1262 if (submodule_states_.CaptureMultiBandProcessingActive() && | 1261 if (submodule_states_.CaptureMultiBandProcessingActive() && |
1263 SampleRateSupportsMultiBand( | 1262 SampleRateSupportsMultiBand( |
1264 capture_nonlocked_.capture_processing_format.sample_rate_hz())) { | 1263 capture_nonlocked_.capture_processing_format.sample_rate_hz())) { |
1265 capture_buffer->MergeFrequencyBands(); | 1264 capture_buffer->MergeFrequencyBands(); |
1266 } | 1265 } |
1267 | 1266 |
| 1267 if (config_.residual_echo_detector.enabled) { |
| 1268 private_submodules_->residual_echo_detector->AnalyzeCaptureAudio( |
| 1269 rtc::ArrayView<const float>(capture_buffer->channels_f()[0], |
| 1270 capture_buffer->num_frames())); |
| 1271 } |
| 1272 |
1268 // TODO(aluebs): Investigate if the transient suppression placement should be | 1273 // TODO(aluebs): Investigate if the transient suppression placement should be |
1269 // before or after the AGC. | 1274 // before or after the AGC. |
1270 if (capture_.transient_suppressor_enabled) { | 1275 if (capture_.transient_suppressor_enabled) { |
1271 float voice_probability = | 1276 float voice_probability = |
1272 private_submodules_->agc_manager.get() | 1277 private_submodules_->agc_manager.get() |
1273 ? private_submodules_->agc_manager->voice_probability() | 1278 ? private_submodules_->agc_manager->voice_probability() |
1274 : 1.f; | 1279 : 1.f; |
1275 | 1280 |
1276 public_submodules_->transient_suppressor->Suppress( | 1281 public_submodules_->transient_suppressor->Suppress( |
1277 capture_buffer->channels_f()[0], capture_buffer->num_frames(), | 1282 capture_buffer->channels_f()[0], capture_buffer->num_frames(), |
(...skipping 153 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1431 #endif | 1436 #endif |
1432 render_.render_audio->DeinterleaveFrom(frame); | 1437 render_.render_audio->DeinterleaveFrom(frame); |
1433 RETURN_ON_ERR(ProcessRenderStreamLocked()); | 1438 RETURN_ON_ERR(ProcessRenderStreamLocked()); |
1434 render_.render_audio->InterleaveTo( | 1439 render_.render_audio->InterleaveTo( |
1435 frame, submodule_states_.RenderMultiBandProcessingActive()); | 1440 frame, submodule_states_.RenderMultiBandProcessingActive()); |
1436 return kNoError; | 1441 return kNoError; |
1437 } | 1442 } |
1438 | 1443 |
1439 int AudioProcessingImpl::ProcessRenderStreamLocked() { | 1444 int AudioProcessingImpl::ProcessRenderStreamLocked() { |
1440 AudioBuffer* render_buffer = render_.render_audio.get(); // For brevity. | 1445 AudioBuffer* render_buffer = render_.render_audio.get(); // For brevity. |
| 1446 |
| 1447 QueueNonbandedRenderAudio(render_buffer); |
| 1448 |
1441 if (submodule_states_.RenderMultiBandSubModulesActive() && | 1449 if (submodule_states_.RenderMultiBandSubModulesActive() && |
1442 SampleRateSupportsMultiBand( | 1450 SampleRateSupportsMultiBand( |
1443 formats_.render_processing_format.sample_rate_hz())) { | 1451 formats_.render_processing_format.sample_rate_hz())) { |
1444 render_buffer->SplitIntoFrequencyBands(); | 1452 render_buffer->SplitIntoFrequencyBands(); |
1445 } | 1453 } |
1446 | 1454 |
1447 #if WEBRTC_INTELLIGIBILITY_ENHANCER | 1455 #if WEBRTC_INTELLIGIBILITY_ENHANCER |
1448 if (capture_nonlocked_.intelligibility_enabled) { | 1456 if (capture_nonlocked_.intelligibility_enabled) { |
1449 public_submodules_->intelligibility_enhancer->ProcessRenderAudio( | 1457 public_submodules_->intelligibility_enhancer->ProcessRenderAudio( |
1450 render_buffer); | 1458 render_buffer); |
1451 } | 1459 } |
1452 #endif | 1460 #endif |
1453 | 1461 |
1454 QueueRenderAudio(render_buffer); | 1462 QueueBandedRenderAudio(render_buffer); |
1455 // TODO(peah): Perform the queueing ínside QueueRenderAudiuo(). | 1463 // TODO(peah): Perform the queueing ínside QueueRenderAudiuo(). |
1456 if (private_submodules_->echo_canceller3) { | 1464 if (private_submodules_->echo_canceller3) { |
1457 private_submodules_->echo_canceller3->AnalyzeRender(render_buffer); | 1465 private_submodules_->echo_canceller3->AnalyzeRender(render_buffer); |
1458 } | 1466 } |
1459 | 1467 |
1460 if (submodule_states_.RenderMultiBandProcessingActive() && | 1468 if (submodule_states_.RenderMultiBandProcessingActive() && |
1461 SampleRateSupportsMultiBand( | 1469 SampleRateSupportsMultiBand( |
1462 formats_.render_processing_format.sample_rate_hz())) { | 1470 formats_.render_processing_format.sample_rate_hz())) { |
1463 render_buffer->MergeFrequencyBands(); | 1471 render_buffer->MergeFrequencyBands(); |
1464 } | 1472 } |
(...skipping 538 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2003 previous_agc_level(0), | 2011 previous_agc_level(0), |
2004 echo_path_gain_change(false) {} | 2012 echo_path_gain_change(false) {} |
2005 | 2013 |
2006 AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default; | 2014 AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default; |
2007 | 2015 |
2008 AudioProcessingImpl::ApmRenderState::ApmRenderState() = default; | 2016 AudioProcessingImpl::ApmRenderState::ApmRenderState() = default; |
2009 | 2017 |
2010 AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default; | 2018 AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default; |
2011 | 2019 |
2012 } // namespace webrtc | 2020 } // namespace webrtc |
OLD | NEW |