Chromium Code Reviews| Index: webrtc/modules/audio_processing/audio_processing_impl.cc |
| diff --git a/webrtc/modules/audio_processing/audio_processing_impl.cc b/webrtc/modules/audio_processing/audio_processing_impl.cc |
| index 222f749fb7b876fc55e53588a6d0914a192f90f1..64e051af0f34a58477333ea5087ee6d92025ae19 100644 |
| --- a/webrtc/modules/audio_processing/audio_processing_impl.cc |
| +++ b/webrtc/modules/audio_processing/audio_processing_impl.cc |
| @@ -82,15 +82,6 @@ const int AudioProcessing::kMaxNativeSampleRateHz = AudioProcessing:: |
| namespace { |
| -const int kInternalNativeRates[] = {AudioProcessing::kSampleRate8kHz, |
| - AudioProcessing::kSampleRate16kHz, |
| -#ifdef WEBRTC_ARCH_ARM_FAMILY |
| - AudioProcessing::kSampleRate32kHz}; |
| -#else |
| - AudioProcessing::kSampleRate32kHz, |
| - AudioProcessing::kSampleRate48kHz}; |
| -#endif // WEBRTC_ARCH_ARM_FAMILY |
| - |
| static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) { |
| switch (layout) { |
| case AudioProcessing::kMono: |
| @@ -105,18 +96,29 @@ static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) { |
| return false; |
| } |
| -bool is_multi_band(int sample_rate_hz) { |
| +bool SampleRateSupportsMultiBand(int sample_rate_hz) { |
| return sample_rate_hz == AudioProcessing::kSampleRate32kHz || |
| sample_rate_hz == AudioProcessing::kSampleRate48kHz; |
| } |
| -int ClosestHigherNativeRate(int min_proc_rate) { |
| - for (int rate : kInternalNativeRates) { |
| - if (rate >= min_proc_rate) { |
| +int NativeProcessRateToUse(int minimum_rate, bool band_splitting_required) { |
| +#ifdef WEBRTC_ARCH_ARM_FAMILY |
| + const size_t kMaxSplittingNativeProcessRateIndex = 2; |
| +#else |
| + const size_t kMaxSplittingNativeProcessRateIndex = 3; |
| +#endif |
| + RTC_DCHECK_LT(kMaxSplittingNativeProcessRateIndex, |
|
hlundin-webrtc
2016/09/06 10:41:32
You should be able to use a static_assert instead.
peah-webrtc
2016/09/07 06:28:11
Great suggestion but I ran into some problems. See
hlundin-webrtc
2016/09/07 21:36:05
Weird. I thought it'd work if you used constexpr i
peah-webrtc
2016/09/08 08:44:03
Acknowledged.
|
| + AudioProcessing::kNumNativeSampleRates); |
| + size_t uppermost_native_rate_index = |
| + (band_splitting_required ? kMaxSplittingNativeProcessRateIndex : 3); |
| + |
| + for (size_t k = 0; k <= uppermost_native_rate_index; ++k) { |
|
hlundin-webrtc
2016/09/06 10:41:32
Consider this alternative implementation of the fu
peah-webrtc
2016/09/07 06:28:11
This is a much nicer variant! Thanks!!!
I could n
kwiberg-webrtc
2016/09/08 08:17:27
What problems did you encounter? It feels like thi
peah-webrtc
2016/09/08 08:44:03
There are not that many constexpr's to add. I trie
|
| + int rate = AudioProcessing::kNativeSampleRatesHz[k]; |
| + if (rate >= minimum_rate) { |
| return rate; |
| } |
| } |
| - return kInternalNativeRates[arraysize(kInternalNativeRates) - 1]; |
| + return AudioProcessing::kNativeSampleRatesHz[uppermost_native_rate_index]; |
| } |
| } // namespace |
| @@ -124,6 +126,83 @@ int ClosestHigherNativeRate(int min_proc_rate) { |
| // Throughout webrtc, it's assumed that success is represented by zero. |
| static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); |
| +class AudioProcessingImpl::ApmSubmoduleStates { |
| + public: |
| + ApmSubmoduleStates(){}; |
| + // Updates the submodule state and returns true if it has changed. |
| + bool Update(bool hpf_enabled, |
| + bool aec_enabled, |
| + bool aecm_enabled, |
| + bool ns_enabled, |
| + bool ie_enabled, |
| + bool bf_enabled, |
| + bool agc_enabled, |
| + bool lc_enabled, |
| + bool vad_enabled, |
| + bool le_enabled, |
| + bool ts_enabled) { |
|
the sun
2016/09/02 20:11:34
What's "ts"? (I could look it up, but I pretend to
peah-webrtc
2016/09/07 06:28:11
ts is the transient suppressor.
The abbreviation
|
| + bool changed = false; |
| + changed |= (hpf_enabled != hpf_enabled_); |
| + changed |= (aec_enabled != aec_enabled_); |
| + changed |= (aecm_enabled != aecm_enabled_); |
| + changed |= (ns_enabled != ns_enabled_); |
| + changed |= (ie_enabled != ie_enabled_); |
| + changed |= (bf_enabled != bf_enabled_); |
| + changed |= (agc_enabled != agc_enabled_); |
| + changed |= (lc_enabled != lc_enabled_); |
| + changed |= (le_enabled != le_enabled_); |
| + changed |= (vad_enabled != vad_enabled_); |
| + changed |= (ts_enabled != ts_enabled_); |
| + if (changed) { |
| + hpf_enabled_ = hpf_enabled; |
| + aec_enabled_ = aec_enabled; |
| + aecm_enabled_ = aecm_enabled; |
| + ns_enabled_ = ns_enabled; |
| + ie_enabled_ = ie_enabled; |
| + bf_enabled_ = bf_enabled; |
| + agc_enabled_ = agc_enabled; |
| + lc_enabled_ = lc_enabled; |
| + le_enabled_ = le_enabled; |
| + vad_enabled_ = vad_enabled; |
| + ts_enabled_ = ts_enabled; |
| + } |
| + |
| + changed |= first_update_; |
| + first_update_ = false; |
| + return changed; |
| + } |
| + |
| + bool CaptureMultiBandModulesActive() const { |
| + return CaptureMultiBandEffectsActive() || ie_enabled_ || vad_enabled_; |
| + } |
| + |
| + bool CaptureMultiBandEffectsActive() const { |
| + return (hpf_enabled_ || aec_enabled_ || aecm_enabled_ || ns_enabled_ || |
|
hlundin-webrtc
2016/09/06 10:41:32
You can drop the () here too.
peah-webrtc
2016/09/07 06:28:11
Done.
|
| + bf_enabled_ || agc_enabled_); |
| + } |
| + |
| + bool RenderMultiBandModulesActive() const { |
| + return RenderMultiBandEffectsActive() || aec_enabled_ || aecm_enabled_ || |
| + agc_enabled_; |
| + } |
| + |
| + bool RenderMultiBandEffectsActive() const { return ie_enabled_; } |
| + |
| + private: |
| + bool hpf_enabled_ = false; |
| + bool aec_enabled_ = false; |
| + bool aecm_enabled_ = false; |
| + bool ns_enabled_ = false; |
| + bool ie_enabled_ = false; |
| + bool bf_enabled_ = false; |
| + bool agc_enabled_ = false; |
| + bool lc_enabled_ = false; |
| + bool le_enabled_ = false; |
| + bool vad_enabled_ = false; |
| + bool ts_enabled_ = false; |
| + bool first_update_ = true; |
| +}; |
| + |
| struct AudioProcessingImpl::ApmPublicSubmodules { |
| ApmPublicSubmodules() {} |
| // Accessed externally of APM without any lock acquired. |
| @@ -178,7 +257,8 @@ AudioProcessingImpl::AudioProcessingImpl(const Config& config) |
| AudioProcessingImpl::AudioProcessingImpl(const Config& config, |
| NonlinearBeamformer* beamformer) |
| - : public_submodules_(new ApmPublicSubmodules()), |
| + : submodule_states_(new ApmSubmoduleStates()), |
| + public_submodules_(new ApmPublicSubmodules()), |
| private_submodules_(new ApmPrivateSubmodules(beamformer)), |
| constants_(config.Get<ExperimentalAgc>().startup_min_volume, |
| #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) |
| @@ -275,12 +355,13 @@ int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { |
| int AudioProcessingImpl::MaybeInitializeRender( |
| const ProcessingConfig& processing_config) { |
| - return MaybeInitialize(processing_config); |
| + return MaybeInitialize(processing_config, false); |
| } |
| int AudioProcessingImpl::MaybeInitializeCapture( |
| - const ProcessingConfig& processing_config) { |
| - return MaybeInitialize(processing_config); |
| + const ProcessingConfig& processing_config, |
| + bool force_initialization) { |
| + return MaybeInitialize(processing_config, force_initialization); |
| } |
| #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| @@ -300,9 +381,10 @@ AudioProcessingImpl::ApmDebugDumpState::~ApmDebugDumpState() {} |
| // Calls InitializeLocked() if any of the audio parameters have changed from |
| // their current values (needs to be called while holding the crit_render_lock). |
| int AudioProcessingImpl::MaybeInitialize( |
| - const ProcessingConfig& processing_config) { |
| + const ProcessingConfig& processing_config, |
| + bool force_initialization) { |
| // Called from both threads. Thread check is therefore not possible. |
| - if (processing_config == formats_.api_format) { |
| + if (processing_config == formats_.api_format && !force_initialization) { |
| return kNoError; |
| } |
| @@ -326,7 +408,8 @@ int AudioProcessingImpl::InitializeLocked() { |
| formats_.rev_proc_format.num_frames(), |
| formats_.rev_proc_format.num_channels(), |
| rev_audio_buffer_out_num_frames)); |
| - if (rev_conversion_needed()) { |
| + if (formats_.api_format.reverse_input_stream() != |
| + formats_.api_format.reverse_output_stream()) { |
| render_.render_converter = AudioConverter::Create( |
| formats_.api_format.reverse_input_stream().num_channels(), |
| formats_.api_format.reverse_input_stream().num_frames(), |
| @@ -397,17 +480,25 @@ int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { |
| formats_.api_format = config; |
| - capture_nonlocked_.fwd_proc_format = StreamConfig(ClosestHigherNativeRate( |
| + int fwd_proc_rate = NativeProcessRateToUse( |
| std::min(formats_.api_format.input_stream().sample_rate_hz(), |
| - formats_.api_format.output_stream().sample_rate_hz()))); |
| + formats_.api_format.output_stream().sample_rate_hz()), |
| + submodule_states_->CaptureMultiBandModulesActive() || |
| + submodule_states_->RenderMultiBandModulesActive()); |
| + |
| + capture_nonlocked_.fwd_proc_format = StreamConfig(fwd_proc_rate); |
| - int rev_proc_rate = ClosestHigherNativeRate(std::min( |
| - formats_.api_format.reverse_input_stream().sample_rate_hz(), |
| - formats_.api_format.reverse_output_stream().sample_rate_hz())); |
| + int rev_proc_rate = NativeProcessRateToUse( |
| + std::min(formats_.api_format.reverse_input_stream().sample_rate_hz(), |
| + formats_.api_format.reverse_output_stream().sample_rate_hz()), |
| + submodule_states_->CaptureMultiBandModulesActive() || |
| + submodule_states_->RenderMultiBandModulesActive()); |
| // TODO(aluebs): Remove this restriction once we figure out why the 3-band |
| // splitting filter degrades the AEC performance. |
| if (rev_proc_rate > kSampleRate32kHz) { |
| - rev_proc_rate = is_rev_processed() ? kSampleRate32kHz : kSampleRate16kHz; |
| + rev_proc_rate = submodule_states_->RenderMultiBandEffectsActive() |
| + ? kSampleRate32kHz |
| + : kSampleRate16kHz; |
| } |
| // If the forward sample rate is 8 kHz, the reverse stream is also processed |
| // at this rate. |
| @@ -556,6 +647,7 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, |
| float* const* dest) { |
| TRACE_EVENT0("webrtc", "AudioProcessing::ProcessStream_StreamConfig"); |
| ProcessingConfig processing_config; |
| + bool reeinitialization_required; |
|
hlundin-webrtc
2016/09/06 10:41:32
Typo: reeinit... -> reinit...
peah-webrtc
2016/09/07 06:28:11
Done.
|
| { |
| // Acquire the capture lock in order to safely call the function |
| // that retrieves the render side data. This function accesses apm |
| @@ -570,6 +662,7 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, |
| } |
| processing_config = formats_.api_format; |
| + reeinitialization_required = UpdateActiveSubmoduleStates(); |
| } |
| processing_config.input_stream() = input_config; |
| @@ -578,7 +671,8 @@ int AudioProcessingImpl::ProcessStream(const float* const* src, |
| { |
| // Do conditional reinitialization. |
| rtc::CritScope cs_render(&crit_render_); |
| - RETURN_ON_ERR(MaybeInitializeCapture(processing_config)); |
| + RETURN_ON_ERR( |
| + MaybeInitializeCapture(processing_config, reeinitialization_required)); |
| } |
| rtc::CritScope cs_capture(&crit_capture_); |
| assert(processing_config.input_stream().num_frames() == |
| @@ -646,6 +740,7 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { |
| } |
| ProcessingConfig processing_config; |
| + bool reeinitialization_required; |
|
hlundin-webrtc
2016/09/06 10:41:32
Same typo.
peah-webrtc
2016/09/07 06:28:11
Done.
|
| { |
| // Aquire lock for the access of api_format. |
| // The lock is released immediately due to the conditional |
| @@ -654,6 +749,8 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { |
| // TODO(ajm): The input and output rates and channels are currently |
| // constrained to be identical in the int16 interface. |
| processing_config = formats_.api_format; |
| + |
| + reeinitialization_required = UpdateActiveSubmoduleStates(); |
| } |
| processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_); |
| processing_config.input_stream().set_num_channels(frame->num_channels_); |
| @@ -663,7 +760,8 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { |
| { |
| // Do conditional reinitialization. |
| rtc::CritScope cs_render(&crit_render_); |
| - RETURN_ON_ERR(MaybeInitializeCapture(processing_config)); |
| + RETURN_ON_ERR( |
| + MaybeInitializeCapture(processing_config, reeinitialization_required)); |
| } |
| rtc::CritScope cs_capture(&crit_capture_); |
| if (frame->samples_per_channel_ != |
| @@ -685,7 +783,7 @@ int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { |
| capture_.capture_audio->DeinterleaveFrom(frame); |
| RETURN_ON_ERR(ProcessStreamLocked()); |
| - capture_.capture_audio->InterleaveTo(frame, output_copy_needed()); |
| + capture_.capture_audio->InterleaveTo(frame, true); |
| #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| if (debug_dump_.debug_file->is_open()) { |
| @@ -731,7 +829,9 @@ int AudioProcessingImpl::ProcessStreamLocked() { |
| capture_nonlocked_.fwd_proc_format.num_frames()); |
| } |
| - if (fwd_analysis_needed()) { |
| + if (submodule_states_->CaptureMultiBandModulesActive() && |
| + SampleRateSupportsMultiBand( |
| + capture_nonlocked_.fwd_proc_format.sample_rate_hz())) { |
| ca->SplitIntoFrequencyBands(); |
| } |
| @@ -802,7 +902,9 @@ int AudioProcessingImpl::ProcessStreamLocked() { |
| RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio( |
| ca, echo_cancellation()->stream_has_echo())); |
| - if (fwd_synthesis_needed()) { |
| + if (submodule_states_->CaptureMultiBandEffectsActive() && |
| + SampleRateSupportsMultiBand( |
| + capture_nonlocked_.fwd_proc_format.sample_rate_hz())) { |
| ca->MergeFrequencyBands(); |
| } |
| @@ -856,10 +958,11 @@ int AudioProcessingImpl::ProcessReverseStream( |
| rtc::CritScope cs(&crit_render_); |
| RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, reverse_input_config, |
| reverse_output_config)); |
| - if (is_rev_processed()) { |
| + if (submodule_states_->RenderMultiBandEffectsActive()) { |
| render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(), |
| dest); |
| - } else if (render_check_rev_conversion_needed()) { |
| + } else if (formats_.api_format.reverse_input_stream() != |
| + formats_.api_format.reverse_output_stream()) { |
| render_.render_converter->Convert(src, reverse_input_config.num_samples(), |
| dest, |
| reverse_output_config.num_samples()); |
| @@ -961,15 +1064,14 @@ int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { |
| #endif |
| render_.render_audio->DeinterleaveFrom(frame); |
| RETURN_ON_ERR(ProcessReverseStreamLocked()); |
| - if (is_rev_processed()) { |
| - render_.render_audio->InterleaveTo(frame, true); |
| - } |
| + render_.render_audio->InterleaveTo(frame, true); |
| return kNoError; |
| } |
| int AudioProcessingImpl::ProcessReverseStreamLocked() { |
| AudioBuffer* ra = render_.render_audio.get(); // For brevity. |
| - if (rev_analysis_needed()) { |
| + if (submodule_states_->RenderMultiBandModulesActive() && |
| + SampleRateSupportsMultiBand(formats_.rev_proc_format.sample_rate_hz())) { |
| ra->SplitIntoFrequencyBands(); |
| } |
| @@ -988,7 +1090,8 @@ int AudioProcessingImpl::ProcessReverseStreamLocked() { |
| RETURN_ON_ERR(public_submodules_->gain_control->ProcessRenderAudio(ra)); |
| } |
| - if (rev_synthesis_needed()) { |
| + if (submodule_states_->RenderMultiBandEffectsActive() && |
| + SampleRateSupportsMultiBand(formats_.rev_proc_format.sample_rate_hz())) { |
| ra->MergeFrequencyBands(); |
| } |
| @@ -1122,20 +1225,14 @@ int AudioProcessingImpl::StopDebugRecording() { |
| } |
| EchoCancellation* AudioProcessingImpl::echo_cancellation() const { |
| - // Adding a lock here has no effect as it allows any access to the submodule |
| - // from the returned pointer. |
| return public_submodules_->echo_cancellation.get(); |
| } |
| EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const { |
| - // Adding a lock here has no effect as it allows any access to the submodule |
| - // from the returned pointer. |
| return public_submodules_->echo_control_mobile.get(); |
| } |
| GainControl* AudioProcessingImpl::gain_control() const { |
| - // Adding a lock here has no effect as it allows any access to the submodule |
| - // from the returned pointer. |
| if (constants_.use_experimental_agc) { |
| return public_submodules_->gain_control_for_experimental_agc.get(); |
| } |
| @@ -1143,103 +1240,34 @@ GainControl* AudioProcessingImpl::gain_control() const { |
| } |
| HighPassFilter* AudioProcessingImpl::high_pass_filter() const { |
| - // Adding a lock here has no effect as it allows any access to the submodule |
| - // from the returned pointer. |
| return public_submodules_->high_pass_filter.get(); |
| } |
| LevelEstimator* AudioProcessingImpl::level_estimator() const { |
| - // Adding a lock here has no effect as it allows any access to the submodule |
| - // from the returned pointer. |
| return public_submodules_->level_estimator.get(); |
| } |
| NoiseSuppression* AudioProcessingImpl::noise_suppression() const { |
| - // Adding a lock here has no effect as it allows any access to the submodule |
| - // from the returned pointer. |
| return public_submodules_->noise_suppression.get(); |
| } |
| VoiceDetection* AudioProcessingImpl::voice_detection() const { |
| - // Adding a lock here has no effect as it allows any access to the submodule |
| - // from the returned pointer. |
| return public_submodules_->voice_detection.get(); |
| } |
| -bool AudioProcessingImpl::is_fwd_processed() const { |
| - // The beamformer, noise suppressor and highpass filter |
| - // modify the data. |
| - if (capture_nonlocked_.beamformer_enabled || |
| - public_submodules_->high_pass_filter->is_enabled() || |
| - public_submodules_->noise_suppression->is_enabled() || |
| - public_submodules_->echo_cancellation->is_enabled() || |
| - public_submodules_->echo_control_mobile->is_enabled() || |
| - public_submodules_->gain_control->is_enabled()) { |
| - return true; |
| - } |
| - |
| - // The capture data is otherwise unchanged. |
| - return false; |
| -} |
| - |
| -bool AudioProcessingImpl::output_copy_needed() const { |
| - // Check if we've upmixed or downmixed the audio. |
| - return ((formats_.api_format.output_stream().num_channels() != |
| - formats_.api_format.input_stream().num_channels()) || |
| - is_fwd_processed() || capture_.transient_suppressor_enabled || |
| - capture_nonlocked_.level_controller_enabled); |
| -} |
| - |
| -bool AudioProcessingImpl::fwd_synthesis_needed() const { |
| - return (is_fwd_processed() && |
| - is_multi_band(capture_nonlocked_.fwd_proc_format.sample_rate_hz())); |
| -} |
| - |
| -bool AudioProcessingImpl::fwd_analysis_needed() const { |
| - if (!is_fwd_processed() && |
| - !public_submodules_->voice_detection->is_enabled() && |
| - !capture_.transient_suppressor_enabled) { |
| - // Only public_submodules_->level_estimator is enabled. |
| - return false; |
| - } else if (is_multi_band( |
| - capture_nonlocked_.fwd_proc_format.sample_rate_hz())) { |
| - // Something besides public_submodules_->level_estimator is enabled, and we |
| - // have super-wb. |
| - return true; |
| - } |
| - return false; |
| -} |
| - |
| -bool AudioProcessingImpl::is_rev_processed() const { |
| -#if WEBRTC_INTELLIGIBILITY_ENHANCER |
| - return capture_nonlocked_.intelligibility_enabled; |
| -#else |
| - return false; |
| -#endif |
| -} |
| - |
| -bool AudioProcessingImpl::rev_synthesis_needed() const { |
| - return (is_rev_processed() && |
| - is_multi_band(formats_.rev_proc_format.sample_rate_hz())); |
| -} |
| - |
| -bool AudioProcessingImpl::rev_analysis_needed() const { |
| - return is_multi_band(formats_.rev_proc_format.sample_rate_hz()) && |
| - (is_rev_processed() || |
| - public_submodules_->echo_cancellation |
| - ->is_enabled_render_side_query() || |
| - public_submodules_->echo_control_mobile |
| - ->is_enabled_render_side_query() || |
| - public_submodules_->gain_control->is_enabled_render_side_query()); |
| -} |
| - |
| -bool AudioProcessingImpl::render_check_rev_conversion_needed() const { |
| - return rev_conversion_needed(); |
| -} |
| - |
| -bool AudioProcessingImpl::rev_conversion_needed() const { |
| - return (formats_.api_format.reverse_input_stream() != |
| - formats_.api_format.reverse_output_stream()); |
| +bool AudioProcessingImpl::UpdateActiveSubmoduleStates() { |
| + return submodule_states_->Update( |
| + public_submodules_->high_pass_filter->is_enabled(), |
| + public_submodules_->echo_cancellation->is_enabled(), |
| + public_submodules_->echo_control_mobile->is_enabled(), |
| + public_submodules_->noise_suppression->is_enabled(), |
| + capture_nonlocked_.intelligibility_enabled, |
| + capture_nonlocked_.beamformer_enabled, |
| + public_submodules_->gain_control->is_enabled(), |
| + capture_nonlocked_.level_controller_enabled, |
| + public_submodules_->voice_detection->is_enabled(), |
| + public_submodules_->level_estimator->is_enabled(), |
| + capture_.transient_suppressor_enabled); |
| } |
| void AudioProcessingImpl::InitializeExperimentalAgc() { |