Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 /* | 1 /* |
| 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
| 3 * | 3 * |
| 4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ | 9 */ |
| 10 | 10 |
| (...skipping 18 matching lines...) Expand all Loading... | |
| 29 #include "webrtc/modules/audio_processing/echo_cancellation_impl.h" | 29 #include "webrtc/modules/audio_processing/echo_cancellation_impl.h" |
| 30 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h" | 30 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h" |
| 31 #include "webrtc/modules/audio_processing/gain_control_impl.h" | 31 #include "webrtc/modules/audio_processing/gain_control_impl.h" |
| 32 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h" | 32 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h" |
| 33 #include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhanc er.h" | 33 #include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhanc er.h" |
| 34 #include "webrtc/modules/audio_processing/level_estimator_impl.h" | 34 #include "webrtc/modules/audio_processing/level_estimator_impl.h" |
| 35 #include "webrtc/modules/audio_processing/noise_suppression_impl.h" | 35 #include "webrtc/modules/audio_processing/noise_suppression_impl.h" |
| 36 #include "webrtc/modules/audio_processing/processing_component.h" | 36 #include "webrtc/modules/audio_processing/processing_component.h" |
| 37 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h" | 37 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h" |
| 38 #include "webrtc/modules/audio_processing/voice_detection_impl.h" | 38 #include "webrtc/modules/audio_processing/voice_detection_impl.h" |
| 39 #include "webrtc/modules/include/module_common_types.h" | 39 #include "webrtc/modules/interface/module_common_types.h" |
| 40 #include "webrtc/system_wrappers/include/critical_section_wrapper.h" | |
| 41 #include "webrtc/system_wrappers/include/file_wrapper.h" | 40 #include "webrtc/system_wrappers/include/file_wrapper.h" |
| 42 #include "webrtc/system_wrappers/include/logging.h" | 41 #include "webrtc/system_wrappers/include/logging.h" |
| 43 #include "webrtc/system_wrappers/include/metrics.h" | 42 #include "webrtc/system_wrappers/include/metrics.h" |
| 44 | 43 |
| 45 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 44 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 46 // Files generated at build-time by the protobuf compiler. | 45 // Files generated at build-time by the protobuf compiler. |
| 47 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD | 46 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD |
| 48 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" | 47 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" |
| 49 #else | 48 #else |
| 50 #include "webrtc/audio_processing/debug.pb.h" | 49 #include "webrtc/audio_processing/debug.pb.h" |
| (...skipping 17 matching lines...) Expand all Loading... | |
| 68 case AudioProcessing::kStereo: | 67 case AudioProcessing::kStereo: |
| 69 return false; | 68 return false; |
| 70 case AudioProcessing::kMonoAndKeyboard: | 69 case AudioProcessing::kMonoAndKeyboard: |
| 71 case AudioProcessing::kStereoAndKeyboard: | 70 case AudioProcessing::kStereoAndKeyboard: |
| 72 return true; | 71 return true; |
| 73 } | 72 } |
| 74 | 73 |
| 75 assert(false); | 74 assert(false); |
| 76 return false; | 75 return false; |
| 77 } | 76 } |
| 77 } // namespace | |
| 78 | 78 |
| 79 } // namespace | 79 struct ApmExternallyAccessibleSubModules { |
| 80 ApmExternallyAccessibleSubModules() | |
| 81 : echo_cancellation(NULL), | |
| 82 echo_control_mobile(NULL), | |
| 83 gain_control(NULL), | |
| 84 high_pass_filter(NULL), | |
| 85 level_estimator(NULL), | |
| 86 noise_suppression(NULL), | |
| 87 voice_detection(NULL) {} | |
| 88 // Accessed externally of APM without any lock acquired. | |
| 89 EchoCancellationImpl* echo_cancellation; | |
| 90 EchoControlMobileImpl* echo_control_mobile; | |
| 91 GainControlImpl* gain_control; | |
| 92 HighPassFilterImpl* high_pass_filter; | |
| 93 LevelEstimatorImpl* level_estimator; | |
| 94 NoiseSuppressionImpl* noise_suppression; | |
| 95 VoiceDetectionImpl* voice_detection; | |
| 96 rtc::scoped_ptr<GainControlForNewAgc> gain_control_for_new_agc; | |
| 97 | |
| 98 // Accessed internally from both render and capture. | |
| 99 rtc::scoped_ptr<TransientSuppressor> transient_suppressor; | |
| 100 rtc::scoped_ptr<IntelligibilityEnhancer> intelligibility_enhancer; | |
| 101 }; | |
| 102 | |
| 103 struct ApmInternalSubModules { | |
| 104 explicit ApmInternalSubModules(Beamformer<float>* beamformer) | |
| 105 : beamformer(beamformer) {} | |
| 106 // Accessed internally from capture or during initialization | |
| 107 std::list<ProcessingComponent*> component_list; | |
| 108 rtc::scoped_ptr<Beamformer<float>> beamformer; | |
| 109 rtc::scoped_ptr<AgcManagerDirect> agc_manager; | |
| 110 }; | |
| 80 | 111 |
| 81 // Throughout webrtc, it's assumed that success is represented by zero. | 112 // Throughout webrtc, it's assumed that success is represented by zero. |
| 82 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); | 113 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); |
| 83 | 114 |
| 84 // This class has two main functionalities: | 115 // This class has two main functionalities: |
| 85 // | 116 // |
| 86 // 1) It is returned instead of the real GainControl after the new AGC has been | 117 // 1) It is returned instead of the real GainControl after the new AGC has been |
| 87 // enabled in order to prevent an outside user from overriding compression | 118 // enabled in order to prevent an outside user from overriding compression |
| 88 // settings. It doesn't do anything in its implementation, except for | 119 // settings. It doesn't do anything in its implementation, except for |
| 89 // delegating the const methods and Enable calls to the real GainControl, so | 120 // delegating the const methods and Enable calls to the real GainControl, so |
| (...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 176 } | 207 } |
| 177 | 208 |
| 178 return apm; | 209 return apm; |
| 179 } | 210 } |
| 180 | 211 |
| 181 AudioProcessingImpl::AudioProcessingImpl(const Config& config) | 212 AudioProcessingImpl::AudioProcessingImpl(const Config& config) |
| 182 : AudioProcessingImpl(config, nullptr) {} | 213 : AudioProcessingImpl(config, nullptr) {} |
| 183 | 214 |
| 184 AudioProcessingImpl::AudioProcessingImpl(const Config& config, | 215 AudioProcessingImpl::AudioProcessingImpl(const Config& config, |
| 185 Beamformer<float>* beamformer) | 216 Beamformer<float>* beamformer) |
| 186 : echo_cancellation_(NULL), | 217 : public_submodules_(new ApmExternallyAccessibleSubModules()), |
| 187 echo_control_mobile_(NULL), | 218 private_submodules_(new ApmInternalSubModules(beamformer)), |
| 188 gain_control_(NULL), | 219 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) |
| 189 high_pass_filter_(NULL), | 220 constants_(config.Get<ExperimentalAgc>().startup_min_volume, |
| 190 level_estimator_(NULL), | 221 config.Get<Beamforming>().array_geometry, |
| 191 noise_suppression_(NULL), | 222 false, |
| 192 voice_detection_(NULL), | 223 config.Get<Intelligibility>().enabled), |
| 193 crit_(CriticalSectionWrapper::CreateCriticalSection()), | 224 #else |
| 194 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 225 constants_(config.Get<ExperimentalAgc>().startup_min_volume, |
| 195 debug_file_(FileWrapper::Create()), | 226 config.Get<Beamforming>().array_geometry, |
| 196 event_msg_(new audioproc::Event()), | 227 config.Get<ExperimentalAgc>().enabled, |
| 228 config.Get<Intelligibility>().enabled), | |
| 197 #endif | 229 #endif |
| 198 fwd_proc_format_(kSampleRate16kHz), | 230 |
| 199 rev_proc_format_(kSampleRate16kHz, 1), | |
| 200 split_rate_(kSampleRate16kHz), | |
| 201 stream_delay_ms_(0), | |
| 202 delay_offset_ms_(0), | |
| 203 was_stream_delay_set_(false), | |
| 204 last_stream_delay_ms_(0), | |
| 205 last_aec_system_delay_ms_(0), | |
| 206 stream_delay_jumps_(-1), | |
| 207 aec_system_delay_jumps_(-1), | |
| 208 output_will_be_muted_(false), | |
| 209 key_pressed_(false), | |
| 210 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) | 231 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) |
| 211 use_new_agc_(false), | 232 capture_(false, config.Get<Beamforming>().enabled) |
| 212 #else | 233 #else |
| 213 use_new_agc_(config.Get<ExperimentalAgc>().enabled), | 234 capture_(config.Get<ExperimentalNs>().enabled, |
| 235 config.Get<Beamforming>().enabled) | |
| 214 #endif | 236 #endif |
| 215 agc_startup_min_volume_(config.Get<ExperimentalAgc>().startup_min_volume), | 237 { |
| 216 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) | |
| 217 transient_suppressor_enabled_(false), | |
| 218 #else | |
| 219 transient_suppressor_enabled_(config.Get<ExperimentalNs>().enabled), | |
| 220 #endif | |
| 221 beamformer_enabled_(config.Get<Beamforming>().enabled), | |
| 222 beamformer_(beamformer), | |
| 223 array_geometry_(config.Get<Beamforming>().array_geometry), | |
| 224 target_direction_(config.Get<Beamforming>().target_direction), | |
| 225 intelligibility_enabled_(config.Get<Intelligibility>().enabled) { | |
| 226 render_thread_checker_.DetachFromThread(); | 238 render_thread_checker_.DetachFromThread(); |
| 227 capture_thread_checker_.DetachFromThread(); | 239 capture_thread_checker_.DetachFromThread(); |
| 228 | 240 |
| 229 echo_cancellation_ = | 241 { |
| 230 new EchoCancellationImpl(this, crit_, &render_thread_checker_); | 242 rtc::CritScope cs_render(&crit_render_); |
|
hlundin-webrtc
2015/11/05 16:11:21
Can you avoid taking the locks in the ctor?
peah-webrtc
2015/11/06 09:54:31
Seems to work. I thought that the threadannotation
hlundin-webrtc
2015/11/20 10:36:30
I think the it intentionally turns a blind eye to
| |
| 231 component_list_.push_back(echo_cancellation_); | 243 rtc::CritScope cs_capture(&crit_capture_); |
| 232 | 244 |
| 233 echo_control_mobile_ = | 245 public_submodules_->echo_cancellation = new EchoCancellationImpl( |
| 234 new EchoControlMobileImpl(this, crit_, &render_thread_checker_); | 246 this, &crit_render_, &crit_capture_, &render_thread_checker_); |
| 235 component_list_.push_back(echo_control_mobile_); | 247 public_submodules_->echo_control_mobile = new EchoControlMobileImpl( |
| 248 this, &crit_render_, &crit_capture_, &render_thread_checker_); | |
| 249 public_submodules_->gain_control = | |
| 250 new GainControlImpl(this, &crit_capture_, &crit_capture_, | |
| 251 &render_thread_checker_, &capture_thread_checker_); | |
| 252 public_submodules_->high_pass_filter = | |
| 253 new HighPassFilterImpl(this, &crit_capture_); | |
| 254 public_submodules_->level_estimator = new LevelEstimatorImpl(this); | |
| 255 public_submodules_->noise_suppression = | |
| 256 new NoiseSuppressionImpl(this, &crit_capture_); | |
| 257 public_submodules_->voice_detection = | |
| 258 new VoiceDetectionImpl(this, &crit_capture_); | |
| 259 public_submodules_->gain_control_for_new_agc.reset( | |
| 260 new GainControlForNewAgc(public_submodules_->gain_control)); | |
| 236 | 261 |
| 237 gain_control_ = new GainControlImpl(this, crit_, &render_thread_checker_, | 262 private_submodules_->component_list.push_back( |
| 238 &capture_thread_checker_); | 263 public_submodules_->echo_cancellation); |
| 239 component_list_.push_back(gain_control_); | 264 private_submodules_->component_list.push_back( |
| 240 | 265 public_submodules_->echo_control_mobile); |
| 241 high_pass_filter_ = new HighPassFilterImpl(this, crit_); | 266 private_submodules_->component_list.push_back( |
| 242 component_list_.push_back(high_pass_filter_); | 267 public_submodules_->gain_control); |
| 243 | 268 private_submodules_->component_list.push_back( |
| 244 level_estimator_ = new LevelEstimatorImpl(this, crit_); | 269 public_submodules_->high_pass_filter); |
| 245 component_list_.push_back(level_estimator_); | 270 private_submodules_->component_list.push_back( |
| 246 | 271 public_submodules_->level_estimator); |
| 247 noise_suppression_ = new NoiseSuppressionImpl(this, crit_); | 272 private_submodules_->component_list.push_back( |
| 248 component_list_.push_back(noise_suppression_); | 273 public_submodules_->noise_suppression); |
| 249 | 274 private_submodules_->component_list.push_back( |
| 250 voice_detection_ = new VoiceDetectionImpl(this, crit_); | 275 public_submodules_->voice_detection); |
| 251 component_list_.push_back(voice_detection_); | 276 } |
| 252 | |
| 253 gain_control_for_new_agc_.reset(new GainControlForNewAgc(gain_control_)); | |
| 254 | 277 |
| 255 SetExtraOptions(config); | 278 SetExtraOptions(config); |
| 256 } | 279 } |
| 257 | 280 |
| 258 AudioProcessingImpl::~AudioProcessingImpl() { | 281 AudioProcessingImpl::~AudioProcessingImpl() { |
| 259 { | 282 { |
| 260 CriticalSectionScoped crit_scoped(crit_); | 283 rtc::CritScope cs_render(&crit_render_); |
|
hlundin-webrtc
2015/11/05 16:11:21
There is no point in acquiring locks in the dtor.
peah-webrtc
2015/11/06 09:54:31
Done.
| |
| 261 // Depends on gain_control_ and gain_control_for_new_agc_. | 284 rtc::CritScope cs_capture(&crit_capture_); |
| 262 agc_manager_.reset(); | 285 |
| 286 // Set submodule pointers to NULL to ensure proper behavior if the accessor | |
|
hlundin-webrtc
2015/11/05 16:11:21
This also makes no sense.
peah-webrtc
2015/11/06 09:54:31
Done.
| |
| 287 // methods | |
| 288 // are called during APM destruction. Note that the submodule objects are | |
| 289 // deleted | |
| 290 // below using the component vector. | |
| 291 public_submodules_->echo_cancellation = nullptr; | |
| 292 public_submodules_->echo_control_mobile = nullptr; | |
| 293 public_submodules_->gain_control = nullptr; | |
| 294 public_submodules_->high_pass_filter = nullptr; | |
| 295 public_submodules_->level_estimator = nullptr; | |
| 296 public_submodules_->noise_suppression = nullptr; | |
| 297 public_submodules_->voice_detection = nullptr; | |
| 298 | |
| 299 // Depends on gain_control_ and | |
| 300 // public_submodules_->gain_control_for_new_agc. | |
| 301 private_submodules_->agc_manager.reset(); | |
| 263 // Depends on gain_control_. | 302 // Depends on gain_control_. |
| 264 gain_control_for_new_agc_.reset(); | 303 public_submodules_->gain_control_for_new_agc.reset(); |
| 265 while (!component_list_.empty()) { | 304 while (!private_submodules_->component_list.empty()) { |
| 266 ProcessingComponent* component = component_list_.front(); | 305 ProcessingComponent* component = |
| 306 private_submodules_->component_list.front(); | |
| 267 component->Destroy(); | 307 component->Destroy(); |
| 268 delete component; | 308 delete component; |
| 269 component_list_.pop_front(); | 309 private_submodules_->component_list.pop_front(); |
| 270 } | 310 } |
| 271 | 311 |
| 272 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 312 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 273 if (debug_file_->Open()) { | 313 if (debug_dump_.debug_file->Open()) { |
| 274 debug_file_->CloseFile(); | 314 debug_dump_.debug_file->CloseFile(); |
| 275 } | 315 } |
| 276 #endif | 316 #endif |
| 277 } | 317 } |
| 278 delete crit_; | |
| 279 crit_ = NULL; | |
| 280 } | 318 } |
| 281 | 319 |
| 282 int AudioProcessingImpl::Initialize() { | 320 int AudioProcessingImpl::Initialize() { |
| 283 CriticalSectionScoped crit_scoped(crit_); | 321 // Run in a single-threaded manner during initialization. |
| 322 rtc::CritScope cs_render(&crit_render_); | |
| 323 rtc::CritScope cs_capture(&crit_capture_); | |
| 284 return InitializeLocked(); | 324 return InitializeLocked(); |
| 285 } | 325 } |
| 286 | 326 |
| 287 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, | 327 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, |
| 288 int output_sample_rate_hz, | 328 int output_sample_rate_hz, |
| 289 int reverse_sample_rate_hz, | 329 int reverse_sample_rate_hz, |
| 290 ChannelLayout input_layout, | 330 ChannelLayout input_layout, |
| 291 ChannelLayout output_layout, | 331 ChannelLayout output_layout, |
| 292 ChannelLayout reverse_layout) { | 332 ChannelLayout reverse_layout) { |
| 333 // Run in a single-threaded manner during initialization. | |
| 334 rtc::CritScope cs_render(&crit_render_); | |
| 335 rtc::CritScope cs_capture(&crit_capture_); | |
| 336 | |
| 293 const ProcessingConfig processing_config = { | 337 const ProcessingConfig processing_config = { |
| 294 {{input_sample_rate_hz, | 338 {{input_sample_rate_hz, |
| 295 ChannelsFromLayout(input_layout), | 339 ChannelsFromLayout(input_layout), |
| 296 LayoutHasKeyboard(input_layout)}, | 340 LayoutHasKeyboard(input_layout)}, |
| 297 {output_sample_rate_hz, | 341 {output_sample_rate_hz, |
| 298 ChannelsFromLayout(output_layout), | 342 ChannelsFromLayout(output_layout), |
| 299 LayoutHasKeyboard(output_layout)}, | 343 LayoutHasKeyboard(output_layout)}, |
| 300 {reverse_sample_rate_hz, | 344 {reverse_sample_rate_hz, |
| 301 ChannelsFromLayout(reverse_layout), | 345 ChannelsFromLayout(reverse_layout), |
| 302 LayoutHasKeyboard(reverse_layout)}, | 346 LayoutHasKeyboard(reverse_layout)}, |
| 303 {reverse_sample_rate_hz, | 347 {reverse_sample_rate_hz, |
| 304 ChannelsFromLayout(reverse_layout), | 348 ChannelsFromLayout(reverse_layout), |
| 305 LayoutHasKeyboard(reverse_layout)}}}; | 349 LayoutHasKeyboard(reverse_layout)}}}; |
| 306 | 350 |
| 307 return Initialize(processing_config); | 351 return Initialize(processing_config); |
| 308 } | 352 } |
| 309 | 353 |
| 310 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { | 354 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { |
| 311 CriticalSectionScoped crit_scoped(crit_); | 355 // Run in a single-threaded manner during initialization. |
| 356 rtc::CritScope cs_render(&crit_render_); | |
| 357 rtc::CritScope cs_capture(&crit_capture_); | |
| 312 return InitializeLocked(processing_config); | 358 return InitializeLocked(processing_config); |
| 313 } | 359 } |
| 314 | 360 |
| 315 // Calls InitializeLocked() if any of the audio parameters have changed from | 361 // Calls InitializeLocked() if any of the audio parameters have changed from |
| 316 // their current values. | 362 // their current values (needs to be called while holding the crit_render_ |
| 317 int AudioProcessingImpl::MaybeInitializeLocked( | 363 // lock). |
| 364 int AudioProcessingImpl::MaybeInitialize( | |
| 318 const ProcessingConfig& processing_config) { | 365 const ProcessingConfig& processing_config) { |
| 319 if (processing_config == shared_state_.api_format_) { | 366 if (processing_config == formats_.api_format) { |
| 320 return kNoError; | 367 return kNoError; |
| 321 } | 368 } |
| 369 | |
| 370 rtc::CritScope cs_capture(&crit_capture_); | |
| 322 return InitializeLocked(processing_config); | 371 return InitializeLocked(processing_config); |
| 323 } | 372 } |
| 324 | 373 |
| 325 int AudioProcessingImpl::InitializeLocked() { | 374 int AudioProcessingImpl::InitializeLocked() { |
| 326 const int fwd_audio_buffer_channels = | 375 const int fwd_audio_buffer_channels = |
| 327 beamformer_enabled_ | 376 capture_.beamformer_enabled |
| 328 ? shared_state_.api_format_.input_stream().num_channels() | 377 ? formats_.api_format.input_stream().num_channels() |
| 329 : shared_state_.api_format_.output_stream().num_channels(); | 378 : formats_.api_format.output_stream().num_channels(); |
| 330 const int rev_audio_buffer_out_num_frames = | 379 const int rev_audio_buffer_out_num_frames = |
| 331 shared_state_.api_format_.reverse_output_stream().num_frames() == 0 | 380 formats_.api_format.reverse_output_stream().num_frames() == 0 |
| 332 ? rev_proc_format_.num_frames() | 381 ? formats_.rev_proc_format.num_frames() |
| 333 : shared_state_.api_format_.reverse_output_stream().num_frames(); | 382 : formats_.api_format.reverse_output_stream().num_frames(); |
| 334 if (shared_state_.api_format_.reverse_input_stream().num_channels() > 0) { | 383 if (formats_.api_format.reverse_input_stream().num_channels() > 0) { |
| 335 render_audio_.reset(new AudioBuffer( | 384 render_.render_audio.reset(new AudioBuffer( |
| 336 shared_state_.api_format_.reverse_input_stream().num_frames(), | 385 formats_.api_format.reverse_input_stream().num_frames(), |
| 337 shared_state_.api_format_.reverse_input_stream().num_channels(), | 386 formats_.api_format.reverse_input_stream().num_channels(), |
| 338 rev_proc_format_.num_frames(), rev_proc_format_.num_channels(), | 387 formats_.rev_proc_format.num_frames(), |
| 388 formats_.rev_proc_format.num_channels(), | |
| 339 rev_audio_buffer_out_num_frames)); | 389 rev_audio_buffer_out_num_frames)); |
| 340 if (rev_conversion_needed()) { | 390 if (rev_conversion_needed()) { |
| 341 render_converter_ = AudioConverter::Create( | 391 render_.render_converter = AudioConverter::Create( |
| 342 shared_state_.api_format_.reverse_input_stream().num_channels(), | 392 formats_.api_format.reverse_input_stream().num_channels(), |
| 343 shared_state_.api_format_.reverse_input_stream().num_frames(), | 393 formats_.api_format.reverse_input_stream().num_frames(), |
| 344 shared_state_.api_format_.reverse_output_stream().num_channels(), | 394 formats_.api_format.reverse_output_stream().num_channels(), |
| 345 shared_state_.api_format_.reverse_output_stream().num_frames()); | 395 formats_.api_format.reverse_output_stream().num_frames()); |
| 346 } else { | 396 } else { |
| 347 render_converter_.reset(nullptr); | 397 render_.render_converter.reset(nullptr); |
| 348 } | 398 } |
| 349 } else { | 399 } else { |
| 350 render_audio_.reset(nullptr); | 400 render_.render_audio.reset(nullptr); |
| 351 render_converter_.reset(nullptr); | 401 render_.render_converter.reset(nullptr); |
| 352 } | 402 } |
| 353 capture_audio_.reset( | 403 capture_.capture_audio.reset( |
| 354 new AudioBuffer(shared_state_.api_format_.input_stream().num_frames(), | 404 new AudioBuffer(formats_.api_format.input_stream().num_frames(), |
| 355 shared_state_.api_format_.input_stream().num_channels(), | 405 formats_.api_format.input_stream().num_channels(), |
| 356 fwd_proc_format_.num_frames(), fwd_audio_buffer_channels, | 406 capture_nonlocked_.fwd_proc_format.num_frames(), |
| 357 shared_state_.api_format_.output_stream().num_frames())); | 407 fwd_audio_buffer_channels, |
| 408 formats_.api_format.output_stream().num_frames())); | |
| 358 | 409 |
| 359 // Initialize all components. | 410 // Initialize all components. |
| 360 for (auto item : component_list_) { | 411 for (auto item : private_submodules_->component_list) { |
| 361 int err = item->Initialize(); | 412 int err = item->Initialize(); |
| 362 if (err != kNoError) { | 413 if (err != kNoError) { |
| 363 return err; | 414 return err; |
| 364 } | 415 } |
| 365 } | 416 } |
| 366 | 417 |
| 367 InitializeExperimentalAgc(); | 418 InitializeExperimentalAgc(); |
| 368 | 419 |
| 369 InitializeTransient(); | 420 InitializeTransient(); |
| 370 | 421 |
| 371 InitializeBeamformer(); | 422 InitializeBeamformer(); |
| 372 | 423 |
| 373 InitializeIntelligibility(); | 424 InitializeIntelligibility(); |
| 374 | 425 |
| 375 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 426 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 376 if (debug_file_->Open()) { | 427 if (debug_dump_.debug_file->Open()) { |
| 377 int err = WriteInitMessage(); | 428 int err = WriteInitMessage(); |
| 378 if (err != kNoError) { | 429 if (err != kNoError) { |
| 379 return err; | 430 return err; |
| 380 } | 431 } |
| 381 } | 432 } |
| 382 #endif | 433 #endif |
| 383 | 434 |
| 384 return kNoError; | 435 return kNoError; |
| 385 } | 436 } |
| 386 | 437 |
| (...skipping 10 matching lines...) Expand all Loading... | |
| 397 const int num_in_channels = config.input_stream().num_channels(); | 448 const int num_in_channels = config.input_stream().num_channels(); |
| 398 const int num_out_channels = config.output_stream().num_channels(); | 449 const int num_out_channels = config.output_stream().num_channels(); |
| 399 | 450 |
| 400 // Need at least one input channel. | 451 // Need at least one input channel. |
| 401 // Need either one output channel or as many outputs as there are inputs. | 452 // Need either one output channel or as many outputs as there are inputs. |
| 402 if (num_in_channels == 0 || | 453 if (num_in_channels == 0 || |
| 403 !(num_out_channels == 1 || num_out_channels == num_in_channels)) { | 454 !(num_out_channels == 1 || num_out_channels == num_in_channels)) { |
| 404 return kBadNumberChannelsError; | 455 return kBadNumberChannelsError; |
| 405 } | 456 } |
| 406 | 457 |
| 407 if (beamformer_enabled_ && | 458 if (capture_.beamformer_enabled && (static_cast<size_t>(num_in_channels) != |
| 408 (static_cast<size_t>(num_in_channels) != array_geometry_.size() || | 459 constants_.array_geometry.size() || |
| 409 num_out_channels > 1)) { | 460 num_out_channels > 1)) { |
| 410 return kBadNumberChannelsError; | 461 return kBadNumberChannelsError; |
| 411 } | 462 } |
| 412 | 463 |
| 413 shared_state_.api_format_ = config; | 464 formats_.api_format = config; |
| 414 | 465 |
| 415 // We process at the closest native rate >= min(input rate, output rate)... | 466 // We process at the closest native rate >= min(input rate, output rate)... |
| 416 const int min_proc_rate = | 467 const int min_proc_rate = |
| 417 std::min(shared_state_.api_format_.input_stream().sample_rate_hz(), | 468 std::min(formats_.api_format.input_stream().sample_rate_hz(), |
| 418 shared_state_.api_format_.output_stream().sample_rate_hz()); | 469 formats_.api_format.output_stream().sample_rate_hz()); |
| 419 int fwd_proc_rate; | 470 int fwd_proc_rate; |
| 420 for (size_t i = 0; i < kNumNativeSampleRates; ++i) { | 471 for (size_t i = 0; i < kNumNativeSampleRates; ++i) { |
| 421 fwd_proc_rate = kNativeSampleRatesHz[i]; | 472 fwd_proc_rate = kNativeSampleRatesHz[i]; |
| 422 if (fwd_proc_rate >= min_proc_rate) { | 473 if (fwd_proc_rate >= min_proc_rate) { |
| 423 break; | 474 break; |
| 424 } | 475 } |
| 425 } | 476 } |
| 426 // ...with one exception. | 477 // ...with one exception. |
| 427 if (echo_control_mobile_->is_enabled() && | 478 if (public_submodules_->echo_control_mobile->is_enabled() && |
| 428 min_proc_rate > kMaxAECMSampleRateHz) { | 479 min_proc_rate > kMaxAECMSampleRateHz) { |
| 429 fwd_proc_rate = kMaxAECMSampleRateHz; | 480 fwd_proc_rate = kMaxAECMSampleRateHz; |
| 430 } | 481 } |
| 431 | 482 |
| 432 fwd_proc_format_ = StreamConfig(fwd_proc_rate); | 483 capture_nonlocked_.fwd_proc_format = StreamConfig(fwd_proc_rate); |
| 433 | 484 |
| 434 // We normally process the reverse stream at 16 kHz. Unless... | 485 // We normally process the reverse stream at 16 kHz. Unless... |
| 435 int rev_proc_rate = kSampleRate16kHz; | 486 int rev_proc_rate = kSampleRate16kHz; |
| 436 if (fwd_proc_format_.sample_rate_hz() == kSampleRate8kHz) { | 487 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate8kHz) { |
| 437 // ...the forward stream is at 8 kHz. | 488 // ...the forward stream is at 8 kHz. |
| 438 rev_proc_rate = kSampleRate8kHz; | 489 rev_proc_rate = kSampleRate8kHz; |
| 439 } else { | 490 } else { |
| 440 if (shared_state_.api_format_.reverse_input_stream().sample_rate_hz() == | 491 if (formats_.api_format.reverse_input_stream().sample_rate_hz() == |
| 441 kSampleRate32kHz) { | 492 kSampleRate32kHz) { |
| 442 // ...or the input is at 32 kHz, in which case we use the splitting | 493 // ...or the input is at 32 kHz, in which case we use the splitting |
| 443 // filter rather than the resampler. | 494 // filter rather than the resampler. |
| 444 rev_proc_rate = kSampleRate32kHz; | 495 rev_proc_rate = kSampleRate32kHz; |
| 445 } | 496 } |
| 446 } | 497 } |
| 447 | 498 |
| 448 // Always downmix the reverse stream to mono for analysis. This has been | 499 // Always downmix the reverse stream to mono for analysis. This has been |
| 449 // demonstrated to work well for AEC in most practical scenarios. | 500 // demonstrated to work well for AEC in most practical scenarios. |
| 450 rev_proc_format_ = StreamConfig(rev_proc_rate, 1); | 501 formats_.rev_proc_format = StreamConfig(rev_proc_rate, 1); |
| 451 | 502 |
| 452 if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || | 503 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate32kHz || |
| 453 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { | 504 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate48kHz) { |
| 454 split_rate_ = kSampleRate16kHz; | 505 capture_nonlocked_.split_rate = kSampleRate16kHz; |
| 455 } else { | 506 } else { |
| 456 split_rate_ = fwd_proc_format_.sample_rate_hz(); | 507 capture_nonlocked_.split_rate = |
| 508 capture_nonlocked_.fwd_proc_format.sample_rate_hz(); | |
| 457 } | 509 } |
| 458 | 510 |
| 459 return InitializeLocked(); | 511 return InitializeLocked(); |
| 460 } | 512 } |
| 461 | 513 |
| 462 void AudioProcessingImpl::SetExtraOptions(const Config& config) { | 514 void AudioProcessingImpl::SetExtraOptions(const Config& config) { |
| 463 CriticalSectionScoped crit_scoped(crit_); | 515 // Run in a single-threaded manner when setting the extra options. |
| 464 for (auto item : component_list_) { | 516 rtc::CritScope cs_render(&crit_render_); |
| 517 rtc::CritScope cs_capture(&crit_capture_); | |
| 518 for (auto item : private_submodules_->component_list) { | |
| 465 item->SetExtraOptions(config); | 519 item->SetExtraOptions(config); |
| 466 } | 520 } |
| 467 | 521 |
| 468 if (transient_suppressor_enabled_ != config.Get<ExperimentalNs>().enabled) { | 522 if (capture_.transient_suppressor_enabled != |
| 469 transient_suppressor_enabled_ = config.Get<ExperimentalNs>().enabled; | 523 config.Get<ExperimentalNs>().enabled) { |
| 524 capture_.transient_suppressor_enabled = | |
| 525 config.Get<ExperimentalNs>().enabled; | |
| 470 InitializeTransient(); | 526 InitializeTransient(); |
| 471 } | 527 } |
| 472 } | 528 } |
| 473 | 529 |
| 474 | 530 |
| 475 int AudioProcessingImpl::proc_sample_rate_hz() const { | 531 int AudioProcessingImpl::proc_sample_rate_hz() const { |
| 476 return fwd_proc_format_.sample_rate_hz(); | 532 // Only called from submodules beneath APM, hence locking is not needed. |
| 533 return capture_nonlocked_.fwd_proc_format.sample_rate_hz(); | |
| 477 } | 534 } |
| 478 | 535 |
| 479 int AudioProcessingImpl::proc_split_sample_rate_hz() const { | 536 int AudioProcessingImpl::proc_split_sample_rate_hz() const { |
| 480 return split_rate_; | 537 // Only called from submodules/tests beneath APM, hence locking is not needed. |
| 538 return capture_nonlocked_.split_rate; | |
| 481 } | 539 } |
| 482 | 540 |
| 483 int AudioProcessingImpl::num_reverse_channels() const { | 541 int AudioProcessingImpl::num_reverse_channels() const { |
| 484 return rev_proc_format_.num_channels(); | 542 // Only called from submodules/tests beneath APM, hence locking is not needed. |
| 543 return formats_.rev_proc_format.num_channels(); | |
| 485 } | 544 } |
| 486 | 545 |
| 487 int AudioProcessingImpl::num_input_channels() const { | 546 int AudioProcessingImpl::num_input_channels() const { |
| 488 return shared_state_.api_format_.input_stream().num_channels(); | 547 // Only called from submodules/tests beneath APM, hence locking is not needed. |
| 548 return formats_.api_format.input_stream().num_channels(); | |
| 489 } | 549 } |
| 490 | 550 |
| 491 int AudioProcessingImpl::num_output_channels() const { | 551 int AudioProcessingImpl::num_output_channels() const { |
| 492 return shared_state_.api_format_.output_stream().num_channels(); | 552 // Only called from submodules/tests beneath APM, hence locking is not needed. |
| 553 return formats_.api_format.output_stream().num_channels(); | |
| 493 } | 554 } |
| 494 | 555 |
| 495 void AudioProcessingImpl::set_output_will_be_muted(bool muted) { | 556 void AudioProcessingImpl::set_output_will_be_muted(bool muted) { |
| 496 CriticalSectionScoped lock(crit_); | 557 rtc::CritScope cs(&crit_capture_); |
| 497 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); | 558 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); |
| 498 output_will_be_muted_ = muted; | 559 capture_.output_will_be_muted = muted; |
| 499 if (agc_manager_.get()) { | 560 if (private_submodules_->agc_manager.get()) { |
| 500 agc_manager_->SetCaptureMuted(output_will_be_muted_); | 561 private_submodules_->agc_manager->SetCaptureMuted( |
| 562 capture_.output_will_be_muted); | |
| 501 } | 563 } |
| 502 } | 564 } |
| 503 | 565 |
| 504 | 566 |
| 505 int AudioProcessingImpl::ProcessStream(const float* const* src, | 567 int AudioProcessingImpl::ProcessStream(const float* const* src, |
| 506 size_t samples_per_channel, | 568 size_t samples_per_channel, |
| 507 int input_sample_rate_hz, | 569 int input_sample_rate_hz, |
| 508 ChannelLayout input_layout, | 570 ChannelLayout input_layout, |
| 509 int output_sample_rate_hz, | 571 int output_sample_rate_hz, |
| 510 ChannelLayout output_layout, | 572 ChannelLayout output_layout, |
| 511 float* const* dest) { | 573 float* const* dest) { |
| 512 CriticalSectionScoped crit_scoped(crit_); | |
| 513 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); | 574 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); |
| 514 StreamConfig input_stream = shared_state_.api_format_.input_stream(); | 575 StreamConfig input_stream; |
| 576 StreamConfig output_stream; | |
| 577 { | |
| 578 // Access the formats_.api_format.input_stream beneath the capture | |
| 579 // lock. | |
|
hlundin-webrtc
2015/11/05 16:11:21
Widow.
peah-webrtc
2015/11/06 09:54:31
Done.
| |
| 580 // The lock must be released as it is later required in the call | |
| 581 // to ProcessStream(,,,); | |
| 582 rtc::CritScope cs(&crit_capture_); | |
| 583 input_stream = formats_.api_format.input_stream(); | |
| 584 output_stream = formats_.api_format.output_stream(); | |
| 585 } | |
| 586 | |
| 515 input_stream.set_sample_rate_hz(input_sample_rate_hz); | 587 input_stream.set_sample_rate_hz(input_sample_rate_hz); |
| 516 input_stream.set_num_channels(ChannelsFromLayout(input_layout)); | 588 input_stream.set_num_channels(ChannelsFromLayout(input_layout)); |
| 517 input_stream.set_has_keyboard(LayoutHasKeyboard(input_layout)); | 589 input_stream.set_has_keyboard(LayoutHasKeyboard(input_layout)); |
| 518 | |
| 519 StreamConfig output_stream = shared_state_.api_format_.output_stream(); | |
| 520 output_stream.set_sample_rate_hz(output_sample_rate_hz); | 590 output_stream.set_sample_rate_hz(output_sample_rate_hz); |
| 521 output_stream.set_num_channels(ChannelsFromLayout(output_layout)); | 591 output_stream.set_num_channels(ChannelsFromLayout(output_layout)); |
| 522 output_stream.set_has_keyboard(LayoutHasKeyboard(output_layout)); | 592 output_stream.set_has_keyboard(LayoutHasKeyboard(output_layout)); |
| 523 | 593 |
| 524 if (samples_per_channel != input_stream.num_frames()) { | 594 if (samples_per_channel != input_stream.num_frames()) { |
| 525 return kBadDataLengthError; | 595 return kBadDataLengthError; |
| 526 } | 596 } |
| 527 return ProcessStream(src, input_stream, output_stream, dest); | 597 return ProcessStream(src, input_stream, output_stream, dest); |
| 528 } | 598 } |
| 529 | 599 |
| 530 int AudioProcessingImpl::ProcessStream(const float* const* src, | 600 int AudioProcessingImpl::ProcessStream(const float* const* src, |
| 531 const StreamConfig& input_config, | 601 const StreamConfig& input_config, |
| 532 const StreamConfig& output_config, | 602 const StreamConfig& output_config, |
| 533 float* const* dest) { | 603 float* const* dest) { |
| 534 CriticalSectionScoped crit_scoped(crit_); | |
| 535 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); | 604 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); |
| 605 { | |
| 606 // Acquire the capture lock in order to safely call the function | |
| 607 // that retrieves the render side data. This function accesses apm | |
| 608 // getters that need the capture lock held when being called. | |
| 609 rtc::CritScope cs_capture(&crit_capture_); | |
| 610 public_submodules_->echo_cancellation->ReadQueuedRenderData(); | |
| 611 public_submodules_->echo_control_mobile->ReadQueuedRenderData(); | |
| 612 public_submodules_->gain_control->ReadQueuedRenderData(); | |
| 613 } | |
| 536 if (!src || !dest) { | 614 if (!src || !dest) { |
| 537 return kNullPointerError; | 615 return kNullPointerError; |
| 538 } | 616 } |
| 539 | 617 |
| 540 echo_cancellation_->ReadQueuedRenderData(); | 618 ProcessingConfig processing_config = formats_.api_format; |
| 541 echo_control_mobile_->ReadQueuedRenderData(); | |
| 542 gain_control_->ReadQueuedRenderData(); | |
| 543 | |
| 544 ProcessingConfig processing_config = shared_state_.api_format_; | |
| 545 processing_config.input_stream() = input_config; | 619 processing_config.input_stream() = input_config; |
| 546 processing_config.output_stream() = output_config; | 620 processing_config.output_stream() = output_config; |
| 547 | 621 |
| 548 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | 622 { |
| 623 // Do conditional reinitialization. | |
| 624 rtc::CritScope cs_render(&crit_render_); | |
| 625 RETURN_ON_ERR(MaybeInitialize(processing_config)); | |
| 626 } | |
| 627 rtc::CritScope cs_capture(&crit_capture_); | |
| 628 | |
| 549 assert(processing_config.input_stream().num_frames() == | 629 assert(processing_config.input_stream().num_frames() == |
| 550 shared_state_.api_format_.input_stream().num_frames()); | 630 formats_.api_format.input_stream().num_frames()); |
| 551 | 631 |
| 552 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 632 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 553 if (debug_file_->Open()) { | 633 if (debug_dump_.debug_file->Open()) { |
| 554 RETURN_ON_ERR(WriteConfigMessage(false)); | 634 RETURN_ON_ERR(WriteConfigMessage(false)); |
| 555 | 635 |
| 556 event_msg_->set_type(audioproc::Event::STREAM); | 636 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); |
| 557 audioproc::Stream* msg = event_msg_->mutable_stream(); | 637 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
| 558 const size_t channel_size = | 638 const size_t channel_size = |
| 559 sizeof(float) * shared_state_.api_format_.input_stream().num_frames(); | 639 sizeof(float) * formats_.api_format.input_stream().num_frames(); |
| 560 for (int i = 0; i < shared_state_.api_format_.input_stream().num_channels(); | 640 for (int i = 0; i < formats_.api_format.input_stream().num_channels(); ++i) |
| 561 ++i) | |
| 562 msg->add_input_channel(src[i], channel_size); | 641 msg->add_input_channel(src[i], channel_size); |
| 563 } | 642 } |
| 564 #endif | 643 #endif |
| 565 | 644 |
| 566 capture_audio_->CopyFrom(src, shared_state_.api_format_.input_stream()); | 645 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream()); |
| 567 RETURN_ON_ERR(ProcessStreamLocked()); | 646 RETURN_ON_ERR(ProcessStreamLocked()); |
| 568 capture_audio_->CopyTo(shared_state_.api_format_.output_stream(), dest); | 647 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest); |
| 569 | 648 |
| 570 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 649 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 571 if (debug_file_->Open()) { | 650 if (debug_dump_.debug_file->Open()) { |
| 572 audioproc::Stream* msg = event_msg_->mutable_stream(); | 651 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
| 573 const size_t channel_size = | 652 const size_t channel_size = |
| 574 sizeof(float) * shared_state_.api_format_.output_stream().num_frames(); | 653 sizeof(float) * formats_.api_format.output_stream().num_frames(); |
| 575 for (int i = 0; | 654 for (int i = 0; i < formats_.api_format.output_stream().num_channels(); ++i) |
| 576 i < shared_state_.api_format_.output_stream().num_channels(); ++i) | |
| 577 msg->add_output_channel(dest[i], channel_size); | 655 msg->add_output_channel(dest[i], channel_size); |
| 578 RETURN_ON_ERR(WriteMessageToDebugFile()); | 656 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 657 &crit_debug_, &debug_dump_.capture)); | |
| 579 } | 658 } |
| 580 #endif | 659 #endif |
| 581 | 660 |
| 582 return kNoError; | 661 return kNoError; |
| 583 } | 662 } |
| 584 | 663 |
| 585 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { | 664 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { |
| 586 CriticalSectionScoped crit_scoped(crit_); | 665 // TODO(peah): Investigate why it is not possible to use a |
| 587 echo_cancellation_->ReadQueuedRenderData(); | 666 // thread checker for this method. |
| 588 echo_control_mobile_->ReadQueuedRenderData(); | 667 { |
| 589 gain_control_->ReadQueuedRenderData(); | 668 // Acquire the capture lock in order to safely call the function |
| 669 // that retrieves the render side data. This function accesses apm | |
| 670 // getters that need the capture lock held when being called. | |
| 671 // The lock needs to be released as | |
| 672 // public_submodules_->echo_control_mobile->is_enabled() aquires this lock | |
| 673 // as well. | |
| 674 rtc::CritScope cs_capture(&crit_capture_); | |
| 675 public_submodules_->echo_cancellation->ReadQueuedRenderData(); | |
| 676 public_submodules_->echo_control_mobile->ReadQueuedRenderData(); | |
| 677 public_submodules_->gain_control->ReadQueuedRenderData(); | |
| 678 } | |
| 590 | 679 |
| 591 if (!frame) { | 680 if (!frame) { |
| 592 return kNullPointerError; | 681 return kNullPointerError; |
| 593 } | 682 } |
| 594 // Must be a native rate. | 683 // Must be a native rate. |
| 595 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 684 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
| 596 frame->sample_rate_hz_ != kSampleRate16kHz && | 685 frame->sample_rate_hz_ != kSampleRate16kHz && |
| 597 frame->sample_rate_hz_ != kSampleRate32kHz && | 686 frame->sample_rate_hz_ != kSampleRate32kHz && |
| 598 frame->sample_rate_hz_ != kSampleRate48kHz) { | 687 frame->sample_rate_hz_ != kSampleRate48kHz) { |
| 599 return kBadSampleRateError; | 688 return kBadSampleRateError; |
| 600 } | 689 } |
| 601 | 690 |
| 602 if (echo_control_mobile_->is_enabled() && | 691 if (public_submodules_->echo_control_mobile->is_enabled() && |
| 603 frame->sample_rate_hz_ > kMaxAECMSampleRateHz) { | 692 frame->sample_rate_hz_ > kMaxAECMSampleRateHz) { |
| 604 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates"; | 693 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates"; |
| 605 return kUnsupportedComponentError; | 694 return kUnsupportedComponentError; |
| 606 } | 695 } |
| 607 | 696 |
| 608 // TODO(ajm): The input and output rates and channels are currently | 697 ProcessingConfig processing_config; |
| 609 // constrained to be identical in the int16 interface. | 698 { |
| 610 ProcessingConfig processing_config = shared_state_.api_format_; | 699 // Aquire lock for the access of api_format. |
| 700 // The lock is released immediately due to the conditional | |
| 701 // reinitialization. | |
| 702 rtc::CritScope cs_capture(&crit_capture_); | |
| 703 // TODO(ajm): The input and output rates and channels are currently | |
| 704 // constrained to be identical in the int16 interface. | |
| 705 processing_config = formats_.api_format; | |
| 706 } | |
| 611 processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_); | 707 processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_); |
| 612 processing_config.input_stream().set_num_channels(frame->num_channels_); | 708 processing_config.input_stream().set_num_channels(frame->num_channels_); |
| 613 processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_); | 709 processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_); |
| 614 processing_config.output_stream().set_num_channels(frame->num_channels_); | 710 processing_config.output_stream().set_num_channels(frame->num_channels_); |
| 615 | 711 |
| 616 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | 712 { |
| 713 // Do conditional reinitialization. | |
| 714 rtc::CritScope cs_render(&crit_render_); | |
| 715 RETURN_ON_ERR(MaybeInitialize(processing_config)); | |
| 716 } | |
| 717 rtc::CritScope cs_capture(&crit_capture_); | |
| 617 if (frame->samples_per_channel_ != | 718 if (frame->samples_per_channel_ != |
| 618 shared_state_.api_format_.input_stream().num_frames()) { | 719 formats_.api_format.input_stream().num_frames()) { |
| 619 return kBadDataLengthError; | 720 return kBadDataLengthError; |
| 620 } | 721 } |
| 621 | 722 |
| 622 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 723 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 623 if (debug_file_->Open()) { | 724 if (debug_dump_.debug_file->Open()) { |
| 624 event_msg_->set_type(audioproc::Event::STREAM); | 725 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); |
| 625 audioproc::Stream* msg = event_msg_->mutable_stream(); | 726 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
| 626 const size_t data_size = | 727 const size_t data_size = |
| 627 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 728 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
| 628 msg->set_input_data(frame->data_, data_size); | 729 msg->set_input_data(frame->data_, data_size); |
| 629 } | 730 } |
| 630 #endif | 731 #endif |
| 631 | 732 |
| 632 capture_audio_->DeinterleaveFrom(frame); | 733 capture_.capture_audio->DeinterleaveFrom(frame); |
| 633 RETURN_ON_ERR(ProcessStreamLocked()); | 734 RETURN_ON_ERR(ProcessStreamLocked()); |
| 634 capture_audio_->InterleaveTo(frame, output_copy_needed(is_data_processed())); | 735 capture_.capture_audio->InterleaveTo(frame, |
| 736 output_copy_needed(is_data_processed())); | |
| 635 | 737 |
| 636 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 738 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 637 if (debug_file_->Open()) { | 739 if (debug_dump_.debug_file->Open()) { |
| 638 audioproc::Stream* msg = event_msg_->mutable_stream(); | 740 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
| 639 const size_t data_size = | 741 const size_t data_size = |
| 640 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 742 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
| 641 msg->set_output_data(frame->data_, data_size); | 743 msg->set_output_data(frame->data_, data_size); |
| 642 RETURN_ON_ERR(WriteMessageToDebugFile()); | 744 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 745 &crit_debug_, &debug_dump_.capture)); | |
| 643 } | 746 } |
| 644 #endif | 747 #endif |
| 645 | 748 |
| 646 return kNoError; | 749 return kNoError; |
| 647 } | 750 } |
| 648 | 751 |
| 649 int AudioProcessingImpl::ProcessStreamLocked() { | 752 int AudioProcessingImpl::ProcessStreamLocked() { |
| 650 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 753 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 651 if (debug_file_->Open()) { | 754 if (debug_dump_.debug_file->Open()) { |
| 652 audioproc::Stream* msg = event_msg_->mutable_stream(); | 755 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
| 653 msg->set_delay(stream_delay_ms_); | 756 msg->set_delay(capture_nonlocked_.stream_delay_ms); |
| 654 msg->set_drift(echo_cancellation_->stream_drift_samples()); | 757 msg->set_drift( |
| 758 public_submodules_->echo_cancellation->stream_drift_samples()); | |
| 655 msg->set_level(gain_control()->stream_analog_level()); | 759 msg->set_level(gain_control()->stream_analog_level()); |
| 656 msg->set_keypress(key_pressed_); | 760 msg->set_keypress(capture_.key_pressed); |
| 657 } | 761 } |
| 658 #endif | 762 #endif |
| 659 | 763 |
| 660 MaybeUpdateHistograms(); | 764 MaybeUpdateHistograms(); |
| 661 | 765 |
| 662 AudioBuffer* ca = capture_audio_.get(); // For brevity. | 766 AudioBuffer* ca = capture_.capture_audio.get(); // For brevity. |
| 663 | 767 |
| 664 if (use_new_agc_ && gain_control_->is_enabled()) { | 768 if (constants_.use_new_agc && |
| 665 agc_manager_->AnalyzePreProcess(ca->channels()[0], ca->num_channels(), | 769 public_submodules_->gain_control->is_enabled()) { |
| 666 fwd_proc_format_.num_frames()); | 770 private_submodules_->agc_manager->AnalyzePreProcess( |
| 771 ca->channels()[0], ca->num_channels(), | |
| 772 capture_nonlocked_.fwd_proc_format.num_frames()); | |
| 667 } | 773 } |
| 668 | 774 |
| 669 bool data_processed = is_data_processed(); | 775 bool data_processed = is_data_processed(); |
| 670 if (analysis_needed(data_processed)) { | 776 if (analysis_needed(data_processed)) { |
| 671 ca->SplitIntoFrequencyBands(); | 777 ca->SplitIntoFrequencyBands(); |
| 672 } | 778 } |
| 673 | 779 |
| 674 if (intelligibility_enabled_) { | 780 if (constants_.intelligibility_enabled) { |
| 675 intelligibility_enhancer_->AnalyzeCaptureAudio( | 781 public_submodules_->intelligibility_enhancer->AnalyzeCaptureAudio( |
| 676 ca->split_channels_f(kBand0To8kHz), split_rate_, ca->num_channels()); | 782 ca->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate, |
| 783 ca->num_channels()); | |
| 677 } | 784 } |
| 678 | 785 |
| 679 if (beamformer_enabled_) { | 786 if (capture_.beamformer_enabled) { |
| 680 beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f()); | 787 private_submodules_->beamformer->ProcessChunk(*ca->split_data_f(), |
| 788 ca->split_data_f()); | |
| 681 ca->set_num_channels(1); | 789 ca->set_num_channels(1); |
| 682 } | 790 } |
| 683 | 791 |
| 684 RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca)); | 792 RETURN_ON_ERR(public_submodules_->high_pass_filter->ProcessCaptureAudio(ca)); |
| 685 RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca)); | 793 RETURN_ON_ERR(public_submodules_->gain_control->AnalyzeCaptureAudio(ca)); |
| 686 RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca)); | 794 RETURN_ON_ERR(public_submodules_->noise_suppression->AnalyzeCaptureAudio(ca)); |
| 687 RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca)); | 795 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio(ca)); |
| 688 | 796 |
| 689 if (echo_control_mobile_->is_enabled() && noise_suppression_->is_enabled()) { | 797 if (public_submodules_->echo_control_mobile->is_enabled() && |
| 798 public_submodules_->noise_suppression->is_enabled()) { | |
| 690 ca->CopyLowPassToReference(); | 799 ca->CopyLowPassToReference(); |
| 691 } | 800 } |
| 692 RETURN_ON_ERR(noise_suppression_->ProcessCaptureAudio(ca)); | 801 RETURN_ON_ERR(public_submodules_->noise_suppression->ProcessCaptureAudio(ca)); |
| 693 RETURN_ON_ERR(echo_control_mobile_->ProcessCaptureAudio(ca)); | 802 RETURN_ON_ERR( |
| 694 RETURN_ON_ERR(voice_detection_->ProcessCaptureAudio(ca)); | 803 public_submodules_->echo_control_mobile->ProcessCaptureAudio(ca)); |
| 804 RETURN_ON_ERR(public_submodules_->voice_detection->ProcessCaptureAudio(ca)); | |
| 695 | 805 |
| 696 if (use_new_agc_ && gain_control_->is_enabled() && | 806 if (constants_.use_new_agc && |
| 697 (!beamformer_enabled_ || beamformer_->is_target_present())) { | 807 public_submodules_->gain_control->is_enabled() && |
| 698 agc_manager_->Process(ca->split_bands_const(0)[kBand0To8kHz], | 808 (!capture_.beamformer_enabled || |
| 699 ca->num_frames_per_band(), split_rate_); | 809 private_submodules_->beamformer->is_target_present())) { |
| 810 private_submodules_->agc_manager->Process( | |
| 811 ca->split_bands_const(0)[kBand0To8kHz], ca->num_frames_per_band(), | |
| 812 capture_nonlocked_.split_rate); | |
| 700 } | 813 } |
| 701 RETURN_ON_ERR(gain_control_->ProcessCaptureAudio(ca)); | 814 RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio(ca)); |
| 702 | 815 |
| 703 if (synthesis_needed(data_processed)) { | 816 if (synthesis_needed(data_processed)) { |
| 704 ca->MergeFrequencyBands(); | 817 ca->MergeFrequencyBands(); |
| 705 } | 818 } |
| 706 | 819 |
| 707 // TODO(aluebs): Investigate if the transient suppression placement should be | 820 // TODO(aluebs): Investigate if the transient suppression placement should be |
| 708 // before or after the AGC. | 821 // before or after the AGC. |
| 709 if (transient_suppressor_enabled_) { | 822 if (capture_.transient_suppressor_enabled) { |
| 710 float voice_probability = | 823 float voice_probability = |
| 711 agc_manager_.get() ? agc_manager_->voice_probability() : 1.f; | 824 private_submodules_->agc_manager.get() |
| 825 ? private_submodules_->agc_manager->voice_probability() | |
| 826 : 1.f; | |
| 712 | 827 |
| 713 transient_suppressor_->Suppress( | 828 public_submodules_->transient_suppressor->Suppress( |
| 714 ca->channels_f()[0], ca->num_frames(), ca->num_channels(), | 829 ca->channels_f()[0], ca->num_frames(), ca->num_channels(), |
| 715 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(), | 830 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(), |
| 716 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability, | 831 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability, |
| 717 key_pressed_); | 832 capture_.key_pressed); |
| 718 } | 833 } |
| 719 | 834 |
| 720 // The level estimator operates on the recombined data. | 835 // The level estimator operates on the recombined data. |
| 721 RETURN_ON_ERR(level_estimator_->ProcessStream(ca)); | 836 RETURN_ON_ERR(public_submodules_->level_estimator->ProcessStream(ca)); |
| 722 | 837 |
| 723 was_stream_delay_set_ = false; | 838 capture_.was_stream_delay_set = false; |
| 724 return kNoError; | 839 return kNoError; |
| 725 } | 840 } |
| 726 | 841 |
| 727 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, | 842 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, |
| 728 size_t samples_per_channel, | 843 size_t samples_per_channel, |
| 729 int rev_sample_rate_hz, | 844 int rev_sample_rate_hz, |
| 730 ChannelLayout layout) { | 845 ChannelLayout layout) { |
| 731 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); | 846 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); |
| 847 rtc::CritScope cs(&crit_render_); | |
| 732 const StreamConfig reverse_config = { | 848 const StreamConfig reverse_config = { |
| 733 rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), | 849 rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), |
| 734 }; | 850 }; |
| 735 if (samples_per_channel != reverse_config.num_frames()) { | 851 if (samples_per_channel != reverse_config.num_frames()) { |
| 736 return kBadDataLengthError; | 852 return kBadDataLengthError; |
| 737 } | 853 } |
| 738 return AnalyzeReverseStream(data, reverse_config, reverse_config); | 854 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config); |
| 739 } | 855 } |
| 740 | 856 |
| 741 int AudioProcessingImpl::ProcessReverseStream( | 857 int AudioProcessingImpl::ProcessReverseStream( |
| 742 const float* const* src, | 858 const float* const* src, |
| 743 const StreamConfig& reverse_input_config, | 859 const StreamConfig& reverse_input_config, |
| 744 const StreamConfig& reverse_output_config, | 860 const StreamConfig& reverse_output_config, |
| 745 float* const* dest) { | 861 float* const* dest) { |
| 746 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); | 862 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); |
| 747 RETURN_ON_ERR( | 863 rtc::CritScope cs(&crit_render_); |
| 748 AnalyzeReverseStream(src, reverse_input_config, reverse_output_config)); | 864 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, reverse_input_config, |
| 865 reverse_output_config)); | |
| 749 if (is_rev_processed()) { | 866 if (is_rev_processed()) { |
| 750 render_audio_->CopyTo(shared_state_.api_format_.reverse_output_stream(), | 867 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(), |
| 751 dest); | 868 dest); |
| 752 } else if (rev_conversion_needed()) { | 869 } else if (rev_conversion_needed()) { |
| 753 render_converter_->Convert(src, reverse_input_config.num_samples(), dest, | 870 render_.render_converter->Convert(src, reverse_input_config.num_samples(), |
| 754 reverse_output_config.num_samples()); | 871 dest, |
| 872 reverse_output_config.num_samples()); | |
| 755 } else { | 873 } else { |
| 756 CopyAudioIfNeeded(src, reverse_input_config.num_frames(), | 874 CopyAudioIfNeeded(src, reverse_input_config.num_frames(), |
| 757 reverse_input_config.num_channels(), dest); | 875 reverse_input_config.num_channels(), dest); |
| 758 } | 876 } |
| 759 | 877 |
| 760 return kNoError; | 878 return kNoError; |
| 761 } | 879 } |
| 762 | 880 |
| 763 int AudioProcessingImpl::AnalyzeReverseStream( | 881 int AudioProcessingImpl::AnalyzeReverseStreamLocked( |
| 764 const float* const* src, | 882 const float* const* src, |
| 765 const StreamConfig& reverse_input_config, | 883 const StreamConfig& reverse_input_config, |
| 766 const StreamConfig& reverse_output_config) { | 884 const StreamConfig& reverse_output_config) { |
| 767 CriticalSectionScoped crit_scoped(crit_); | |
| 768 if (src == NULL) { | 885 if (src == NULL) { |
| 769 return kNullPointerError; | 886 return kNullPointerError; |
| 770 } | 887 } |
| 771 | 888 |
| 772 if (reverse_input_config.num_channels() <= 0) { | 889 if (reverse_input_config.num_channels() <= 0) { |
| 773 return kBadNumberChannelsError; | 890 return kBadNumberChannelsError; |
| 774 } | 891 } |
| 775 | 892 |
| 776 ProcessingConfig processing_config = shared_state_.api_format_; | 893 ProcessingConfig processing_config = formats_.api_format; |
| 777 processing_config.reverse_input_stream() = reverse_input_config; | 894 processing_config.reverse_input_stream() = reverse_input_config; |
| 778 processing_config.reverse_output_stream() = reverse_output_config; | 895 processing_config.reverse_output_stream() = reverse_output_config; |
| 779 | 896 |
| 780 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | 897 RETURN_ON_ERR(MaybeInitialize(processing_config)); |
| 781 assert(reverse_input_config.num_frames() == | 898 assert(reverse_input_config.num_frames() == |
| 782 shared_state_.api_format_.reverse_input_stream().num_frames()); | 899 formats_.api_format.reverse_input_stream().num_frames()); |
| 783 | 900 |
| 784 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 901 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 785 if (debug_file_->Open()) { | 902 if (debug_dump_.debug_file->Open()) { |
| 786 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); | 903 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); |
| 787 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); | 904 audioproc::ReverseStream* msg = |
| 905 debug_dump_.render.event_msg->mutable_reverse_stream(); | |
| 788 const size_t channel_size = | 906 const size_t channel_size = |
| 789 sizeof(float) * | 907 sizeof(float) * formats_.api_format.reverse_input_stream().num_frames(); |
| 790 shared_state_.api_format_.reverse_input_stream().num_frames(); | |
| 791 for (int i = 0; | 908 for (int i = 0; |
| 792 i < shared_state_.api_format_.reverse_input_stream().num_channels(); | 909 i < formats_.api_format.reverse_input_stream().num_channels(); ++i) |
| 793 ++i) | |
| 794 msg->add_channel(src[i], channel_size); | 910 msg->add_channel(src[i], channel_size); |
| 795 RETURN_ON_ERR(WriteMessageToDebugFile()); | 911 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 912 &crit_debug_, &debug_dump_.render)); | |
| 796 } | 913 } |
| 797 #endif | 914 #endif |
| 798 | 915 |
| 799 render_audio_->CopyFrom(src, | 916 render_.render_audio->CopyFrom(src, |
| 800 shared_state_.api_format_.reverse_input_stream()); | 917 formats_.api_format.reverse_input_stream()); |
| 801 return ProcessReverseStreamLocked(); | 918 return ProcessReverseStreamLocked(); |
| 802 } | 919 } |
| 803 | 920 |
| 804 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { | 921 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { |
| 805 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); | 922 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); |
| 806 RETURN_ON_ERR(AnalyzeReverseStream(frame)); | 923 RETURN_ON_ERR(AnalyzeReverseStream(frame)); |
| 924 rtc::CritScope cs(&crit_render_); | |
| 807 if (is_rev_processed()) { | 925 if (is_rev_processed()) { |
| 808 render_audio_->InterleaveTo(frame, true); | 926 render_.render_audio->InterleaveTo(frame, true); |
| 809 } | 927 } |
| 810 | 928 |
| 811 return kNoError; | 929 return kNoError; |
| 812 } | 930 } |
| 813 | 931 |
| 814 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { | 932 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { |
| 815 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); | 933 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); |
| 816 CriticalSectionScoped crit_scoped(crit_); | 934 rtc::CritScope cs(&crit_render_); |
| 817 if (frame == NULL) { | 935 if (frame == NULL) { |
| 818 return kNullPointerError; | 936 return kNullPointerError; |
| 819 } | 937 } |
| 820 // Must be a native rate. | 938 // Must be a native rate. |
| 821 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 939 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
| 822 frame->sample_rate_hz_ != kSampleRate16kHz && | 940 frame->sample_rate_hz_ != kSampleRate16kHz && |
| 823 frame->sample_rate_hz_ != kSampleRate32kHz && | 941 frame->sample_rate_hz_ != kSampleRate32kHz && |
| 824 frame->sample_rate_hz_ != kSampleRate48kHz) { | 942 frame->sample_rate_hz_ != kSampleRate48kHz) { |
| 825 return kBadSampleRateError; | 943 return kBadSampleRateError; |
| 826 } | 944 } |
| 827 // This interface does not tolerate different forward and reverse rates. | 945 // This interface does not tolerate different forward and reverse rates. |
| 828 if (frame->sample_rate_hz_ != | 946 if (frame->sample_rate_hz_ != |
| 829 shared_state_.api_format_.input_stream().sample_rate_hz()) { | 947 formats_.api_format.input_stream().sample_rate_hz()) { |
| 830 return kBadSampleRateError; | 948 return kBadSampleRateError; |
| 831 } | 949 } |
| 832 | 950 |
| 833 if (frame->num_channels_ <= 0) { | 951 if (frame->num_channels_ <= 0) { |
| 834 return kBadNumberChannelsError; | 952 return kBadNumberChannelsError; |
| 835 } | 953 } |
| 836 | 954 |
| 837 ProcessingConfig processing_config = shared_state_.api_format_; | 955 ProcessingConfig processing_config = formats_.api_format; |
| 838 processing_config.reverse_input_stream().set_sample_rate_hz( | 956 processing_config.reverse_input_stream().set_sample_rate_hz( |
| 839 frame->sample_rate_hz_); | 957 frame->sample_rate_hz_); |
| 840 processing_config.reverse_input_stream().set_num_channels( | 958 processing_config.reverse_input_stream().set_num_channels( |
| 841 frame->num_channels_); | 959 frame->num_channels_); |
| 842 processing_config.reverse_output_stream().set_sample_rate_hz( | 960 processing_config.reverse_output_stream().set_sample_rate_hz( |
| 843 frame->sample_rate_hz_); | 961 frame->sample_rate_hz_); |
| 844 processing_config.reverse_output_stream().set_num_channels( | 962 processing_config.reverse_output_stream().set_num_channels( |
| 845 frame->num_channels_); | 963 frame->num_channels_); |
| 846 | 964 |
| 847 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | 965 RETURN_ON_ERR(MaybeInitialize(processing_config)); |
| 848 if (frame->samples_per_channel_ != | 966 if (frame->samples_per_channel_ != |
| 849 shared_state_.api_format_.reverse_input_stream().num_frames()) { | 967 formats_.api_format.reverse_input_stream().num_frames()) { |
| 850 return kBadDataLengthError; | 968 return kBadDataLengthError; |
| 851 } | 969 } |
| 852 | 970 |
| 853 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 971 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 854 if (debug_file_->Open()) { | 972 if (debug_dump_.debug_file->Open()) { |
| 855 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); | 973 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); |
| 856 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); | 974 audioproc::ReverseStream* msg = |
| 975 debug_dump_.render.event_msg->mutable_reverse_stream(); | |
| 857 const size_t data_size = | 976 const size_t data_size = |
| 858 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 977 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
| 859 msg->set_data(frame->data_, data_size); | 978 msg->set_data(frame->data_, data_size); |
| 860 RETURN_ON_ERR(WriteMessageToDebugFile()); | 979 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 980 &crit_debug_, &debug_dump_.render)); | |
| 861 } | 981 } |
| 862 #endif | 982 #endif |
| 863 render_audio_->DeinterleaveFrom(frame); | 983 render_.render_audio->DeinterleaveFrom(frame); |
| 864 return ProcessReverseStreamLocked(); | 984 return ProcessReverseStreamLocked(); |
| 865 } | 985 } |
| 866 | 986 |
| 867 int AudioProcessingImpl::ProcessReverseStreamLocked() { | 987 int AudioProcessingImpl::ProcessReverseStreamLocked() { |
| 868 AudioBuffer* ra = render_audio_.get(); // For brevity. | 988 AudioBuffer* ra = render_.render_audio.get(); // For brevity. |
| 869 if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz) { | 989 if (formats_.rev_proc_format.sample_rate_hz() == kSampleRate32kHz) { |
| 870 ra->SplitIntoFrequencyBands(); | 990 ra->SplitIntoFrequencyBands(); |
| 871 } | 991 } |
| 872 | 992 |
| 873 if (intelligibility_enabled_) { | 993 if (constants_.intelligibility_enabled) { |
| 874 intelligibility_enhancer_->ProcessRenderAudio( | 994 // Currently run in single-threaded mode when the intelligibility |
| 875 ra->split_channels_f(kBand0To8kHz), split_rate_, ra->num_channels()); | 995 // enhancer is activated. |
| 996 // TODO(peah): Fix to be properly multi-threaded. | |
| 997 rtc::CritScope cs(&crit_capture_); | |
| 998 public_submodules_->intelligibility_enhancer->ProcessRenderAudio( | |
| 999 ra->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate, | |
| 1000 ra->num_channels()); | |
| 876 } | 1001 } |
| 877 | 1002 |
| 878 RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra)); | 1003 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessRenderAudio(ra)); |
| 879 RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra)); | 1004 RETURN_ON_ERR( |
| 880 if (!use_new_agc_) { | 1005 public_submodules_->echo_control_mobile->ProcessRenderAudio(ra)); |
| 881 RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra)); | 1006 if (!constants_.use_new_agc) { |
| 1007 RETURN_ON_ERR(public_submodules_->gain_control->ProcessRenderAudio(ra)); | |
| 882 } | 1008 } |
| 883 | 1009 |
| 884 if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz && | 1010 if (formats_.rev_proc_format.sample_rate_hz() == kSampleRate32kHz && |
| 885 is_rev_processed()) { | 1011 is_rev_processed()) { |
| 886 ra->MergeFrequencyBands(); | 1012 ra->MergeFrequencyBands(); |
| 887 } | 1013 } |
| 888 | 1014 |
| 889 return kNoError; | 1015 return kNoError; |
| 890 } | 1016 } |
| 891 | 1017 |
| 892 int AudioProcessingImpl::set_stream_delay_ms(int delay) { | 1018 int AudioProcessingImpl::set_stream_delay_ms(int delay) { |
| 1019 // TODO(peah): Refactor so that it is possible to add a threadchecker | |
| 1020 // (currently not solely called from the capture thread in all places). | |
| 1021 rtc::CritScope cs(&crit_capture_); | |
| 893 Error retval = kNoError; | 1022 Error retval = kNoError; |
| 894 was_stream_delay_set_ = true; | 1023 capture_.was_stream_delay_set = true; |
| 895 delay += delay_offset_ms_; | 1024 delay += capture_.delay_offset_ms; |
| 896 | 1025 |
| 897 if (delay < 0) { | 1026 if (delay < 0) { |
| 898 delay = 0; | 1027 delay = 0; |
| 899 retval = kBadStreamParameterWarning; | 1028 retval = kBadStreamParameterWarning; |
| 900 } | 1029 } |
| 901 | 1030 |
| 902 // TODO(ajm): the max is rather arbitrarily chosen; investigate. | 1031 // TODO(ajm): the max is rather arbitrarily chosen; investigate. |
| 903 if (delay > 500) { | 1032 if (delay > 500) { |
| 904 delay = 500; | 1033 delay = 500; |
| 905 retval = kBadStreamParameterWarning; | 1034 retval = kBadStreamParameterWarning; |
| 906 } | 1035 } |
| 907 | 1036 |
| 908 stream_delay_ms_ = delay; | 1037 capture_nonlocked_.stream_delay_ms = delay; |
| 909 return retval; | 1038 return retval; |
| 910 } | 1039 } |
| 911 | 1040 |
| 912 int AudioProcessingImpl::stream_delay_ms() const { | 1041 int AudioProcessingImpl::stream_delay_ms() const { |
| 913 return stream_delay_ms_; | 1042 return capture_nonlocked_.stream_delay_ms; |
| 914 } | 1043 } |
| 915 | 1044 |
| 916 bool AudioProcessingImpl::was_stream_delay_set() const { | 1045 bool AudioProcessingImpl::was_stream_delay_set() const { |
| 917 return was_stream_delay_set_; | 1046 // Only called from submodules/tests beneath APM, hence locking is not needed. |
| 1047 return capture_.was_stream_delay_set; | |
| 918 } | 1048 } |
| 919 | 1049 |
| 920 void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) { | 1050 void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) { |
| 921 key_pressed_ = key_pressed; | 1051 // Thread check not possible due to being called from different threads. |
| 1052 rtc::CritScope cs(&crit_capture_); | |
| 1053 capture_.key_pressed = key_pressed; | |
| 922 } | 1054 } |
| 923 | 1055 |
| 924 void AudioProcessingImpl::set_delay_offset_ms(int offset) { | 1056 void AudioProcessingImpl::set_delay_offset_ms(int offset) { |
| 925 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); | 1057 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); |
| 926 CriticalSectionScoped crit_scoped(crit_); | 1058 rtc::CritScope cs(&crit_capture_); |
| 927 delay_offset_ms_ = offset; | 1059 capture_.delay_offset_ms = offset; |
| 928 } | 1060 } |
| 929 | 1061 |
| 930 int AudioProcessingImpl::delay_offset_ms() const { | 1062 int AudioProcessingImpl::delay_offset_ms() const { |
| 931 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); | 1063 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); |
| 932 return delay_offset_ms_; | 1064 rtc::CritScope cs(&crit_capture_); |
| 1065 return capture_.delay_offset_ms; | |
| 933 } | 1066 } |
| 934 | 1067 |
| 935 int AudioProcessingImpl::StartDebugRecording( | 1068 int AudioProcessingImpl::StartDebugRecording( |
| 936 const char filename[AudioProcessing::kMaxFilenameSize]) { | 1069 const char filename[AudioProcessing::kMaxFilenameSize]) { |
| 937 CriticalSectionScoped crit_scoped(crit_); | 1070 // Run in a single-threaded manner. |
| 1071 rtc::CritScope cs_render(&crit_render_); | |
| 1072 rtc::CritScope cs_capture(&crit_capture_); | |
| 1073 | |
| 938 static_assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize, ""); | 1074 static_assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize, ""); |
| 939 | 1075 |
| 940 if (filename == NULL) { | 1076 if (filename == NULL) { |
| 941 return kNullPointerError; | 1077 return kNullPointerError; |
| 942 } | 1078 } |
| 943 | 1079 |
| 944 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1080 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 945 // Stop any ongoing recording. | 1081 // Stop any ongoing recording. |
| 946 if (debug_file_->Open()) { | 1082 if (debug_dump_.debug_file->Open()) { |
| 947 if (debug_file_->CloseFile() == -1) { | 1083 if (debug_dump_.debug_file->CloseFile() == -1) { |
| 948 return kFileError; | 1084 return kFileError; |
| 949 } | 1085 } |
| 950 } | 1086 } |
| 951 | 1087 |
| 952 if (debug_file_->OpenFile(filename, false) == -1) { | 1088 if (debug_dump_.debug_file->OpenFile(filename, false) == -1) { |
| 953 debug_file_->CloseFile(); | 1089 debug_dump_.debug_file->CloseFile(); |
| 954 return kFileError; | 1090 return kFileError; |
| 955 } | 1091 } |
| 956 | 1092 |
| 957 RETURN_ON_ERR(WriteConfigMessage(true)); | 1093 RETURN_ON_ERR(WriteConfigMessage(true)); |
| 958 RETURN_ON_ERR(WriteInitMessage()); | 1094 RETURN_ON_ERR(WriteInitMessage()); |
| 959 return kNoError; | 1095 return kNoError; |
| 960 #else | 1096 #else |
| 961 return kUnsupportedFunctionError; | 1097 return kUnsupportedFunctionError; |
| 962 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1098 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 963 } | 1099 } |
| 964 | 1100 |
| 965 int AudioProcessingImpl::StartDebugRecording(FILE* handle) { | 1101 int AudioProcessingImpl::StartDebugRecording(FILE* handle) { |
| 966 CriticalSectionScoped crit_scoped(crit_); | 1102 // Run in a single-threaded manner. |
| 1103 rtc::CritScope cs_render(&crit_render_); | |
| 1104 rtc::CritScope cs_capture(&crit_capture_); | |
| 967 | 1105 |
| 968 if (handle == NULL) { | 1106 if (handle == NULL) { |
| 969 return kNullPointerError; | 1107 return kNullPointerError; |
| 970 } | 1108 } |
| 971 | 1109 |
| 972 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1110 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 973 // Stop any ongoing recording. | 1111 // Stop any ongoing recording. |
| 974 if (debug_file_->Open()) { | 1112 if (debug_dump_.debug_file->Open()) { |
| 975 if (debug_file_->CloseFile() == -1) { | 1113 if (debug_dump_.debug_file->CloseFile() == -1) { |
| 976 return kFileError; | 1114 return kFileError; |
| 977 } | 1115 } |
| 978 } | 1116 } |
| 979 | 1117 |
| 980 if (debug_file_->OpenFromFileHandle(handle, true, false) == -1) { | 1118 if (debug_dump_.debug_file->OpenFromFileHandle(handle, true, false) == -1) { |
| 981 return kFileError; | 1119 return kFileError; |
| 982 } | 1120 } |
| 983 | 1121 |
| 984 RETURN_ON_ERR(WriteConfigMessage(true)); | 1122 RETURN_ON_ERR(WriteConfigMessage(true)); |
| 985 RETURN_ON_ERR(WriteInitMessage()); | 1123 RETURN_ON_ERR(WriteInitMessage()); |
| 986 return kNoError; | 1124 return kNoError; |
| 987 #else | 1125 #else |
| 988 return kUnsupportedFunctionError; | 1126 return kUnsupportedFunctionError; |
| 989 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1127 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 990 } | 1128 } |
| 991 | 1129 |
| 992 int AudioProcessingImpl::StartDebugRecordingForPlatformFile( | 1130 int AudioProcessingImpl::StartDebugRecordingForPlatformFile( |
| 993 rtc::PlatformFile handle) { | 1131 rtc::PlatformFile handle) { |
| 1132 // Run in a single-threaded manner. | |
| 1133 rtc::CritScope cs_render(&crit_render_); | |
| 1134 rtc::CritScope cs_capture(&crit_capture_); | |
| 994 FILE* stream = rtc::FdopenPlatformFileForWriting(handle); | 1135 FILE* stream = rtc::FdopenPlatformFileForWriting(handle); |
| 995 return StartDebugRecording(stream); | 1136 return StartDebugRecording(stream); |
| 996 } | 1137 } |
| 997 | 1138 |
| 998 int AudioProcessingImpl::StopDebugRecording() { | 1139 int AudioProcessingImpl::StopDebugRecording() { |
| 999 CriticalSectionScoped crit_scoped(crit_); | 1140 // Run in a single-threaded manner. |
| 1141 rtc::CritScope cs_render(&crit_render_); | |
| 1142 rtc::CritScope cs_capture(&crit_capture_); | |
| 1000 | 1143 |
| 1001 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1144 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 1002 // We just return if recording hasn't started. | 1145 // We just return if recording hasn't started. |
| 1003 if (debug_file_->Open()) { | 1146 if (debug_dump_.debug_file->Open()) { |
| 1004 if (debug_file_->CloseFile() == -1) { | 1147 if (debug_dump_.debug_file->CloseFile() == -1) { |
| 1005 return kFileError; | 1148 return kFileError; |
| 1006 } | 1149 } |
| 1007 } | 1150 } |
| 1008 return kNoError; | 1151 return kNoError; |
| 1009 #else | 1152 #else |
| 1010 return kUnsupportedFunctionError; | 1153 return kUnsupportedFunctionError; |
| 1011 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1154 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 1012 } | 1155 } |
| 1013 | 1156 |
| 1014 EchoCancellation* AudioProcessingImpl::echo_cancellation() const { | 1157 EchoCancellation* AudioProcessingImpl::echo_cancellation() const { |
| 1015 return echo_cancellation_; | 1158 // Adding a lock here has no effect as it allows any access to the submodule |
| 1159 // from the returned pointer. | |
| 1160 return public_submodules_->echo_cancellation; | |
| 1016 } | 1161 } |
| 1017 | 1162 |
| 1018 EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const { | 1163 EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const { |
| 1019 return echo_control_mobile_; | 1164 // Adding a lock here has no effect as it allows any access to the submodule |
| 1165 // from the returned pointer. | |
| 1166 return public_submodules_->echo_control_mobile; | |
| 1020 } | 1167 } |
| 1021 | 1168 |
| 1022 GainControl* AudioProcessingImpl::gain_control() const { | 1169 GainControl* AudioProcessingImpl::gain_control() const { |
| 1023 if (use_new_agc_) { | 1170 // Adding a lock here has no effect as it allows any access to the submodule |
| 1024 return gain_control_for_new_agc_.get(); | 1171 // from the returned pointer. |
| 1172 if (constants_.use_new_agc) { | |
| 1173 return public_submodules_->gain_control_for_new_agc.get(); | |
| 1025 } | 1174 } |
| 1026 return gain_control_; | 1175 return public_submodules_->gain_control; |
| 1027 } | 1176 } |
| 1028 | 1177 |
| 1029 HighPassFilter* AudioProcessingImpl::high_pass_filter() const { | 1178 HighPassFilter* AudioProcessingImpl::high_pass_filter() const { |
| 1030 return high_pass_filter_; | 1179 // Adding a lock here has no effect as it allows any access to the submodule |
| 1180 // from the returned pointer. | |
| 1181 return public_submodules_->high_pass_filter; | |
| 1031 } | 1182 } |
| 1032 | 1183 |
| 1033 LevelEstimator* AudioProcessingImpl::level_estimator() const { | 1184 LevelEstimator* AudioProcessingImpl::level_estimator() const { |
| 1034 return level_estimator_; | 1185 // Adding a lock here has no effect as it allows any access to the submodule |
| 1186 // from the returned pointer. | |
| 1187 return public_submodules_->level_estimator; | |
| 1035 } | 1188 } |
| 1036 | 1189 |
| 1037 NoiseSuppression* AudioProcessingImpl::noise_suppression() const { | 1190 NoiseSuppression* AudioProcessingImpl::noise_suppression() const { |
| 1038 return noise_suppression_; | 1191 // Adding a lock here has no effect as it allows any access to the submodule |
| 1192 // from the returned pointer. | |
| 1193 return public_submodules_->noise_suppression; | |
| 1039 } | 1194 } |
| 1040 | 1195 |
| 1041 VoiceDetection* AudioProcessingImpl::voice_detection() const { | 1196 VoiceDetection* AudioProcessingImpl::voice_detection() const { |
| 1042 return voice_detection_; | 1197 // Adding a lock here has no effect as it allows any access to the submodule |
| 1198 // from the returned pointer. | |
| 1199 return public_submodules_->voice_detection; | |
| 1043 } | 1200 } |
| 1044 | 1201 |
| 1045 bool AudioProcessingImpl::is_data_processed() const { | 1202 bool AudioProcessingImpl::is_data_processed() const { |
| 1046 if (beamformer_enabled_) { | 1203 if (capture_.beamformer_enabled) { |
| 1047 return true; | 1204 return true; |
| 1048 } | 1205 } |
| 1049 | 1206 |
| 1050 int enabled_count = 0; | 1207 int enabled_count = 0; |
| 1051 for (auto item : component_list_) { | 1208 for (auto item : private_submodules_->component_list) { |
| 1052 if (item->is_component_enabled()) { | 1209 if (item->is_component_enabled()) { |
| 1053 enabled_count++; | 1210 enabled_count++; |
| 1054 } | 1211 } |
| 1055 } | 1212 } |
| 1056 | 1213 |
| 1057 // Data is unchanged if no components are enabled, or if only level_estimator_ | 1214 // Data is unchanged if no components are enabled, or if only |
| 1058 // or voice_detection_ is enabled. | 1215 // public_submodules_->level_estimator |
| 1216 // or public_submodules_->voice_detection is enabled. | |
| 1059 if (enabled_count == 0) { | 1217 if (enabled_count == 0) { |
| 1060 return false; | 1218 return false; |
| 1061 } else if (enabled_count == 1) { | 1219 } else if (enabled_count == 1) { |
| 1062 if (level_estimator_->is_enabled() || voice_detection_->is_enabled()) { | 1220 if (public_submodules_->level_estimator->is_enabled() || |
| 1221 public_submodules_->voice_detection->is_enabled()) { | |
| 1063 return false; | 1222 return false; |
| 1064 } | 1223 } |
| 1065 } else if (enabled_count == 2) { | 1224 } else if (enabled_count == 2) { |
| 1066 if (level_estimator_->is_enabled() && voice_detection_->is_enabled()) { | 1225 if (public_submodules_->level_estimator->is_enabled() && |
| 1226 public_submodules_->voice_detection->is_enabled()) { | |
| 1067 return false; | 1227 return false; |
| 1068 } | 1228 } |
| 1069 } | 1229 } |
| 1070 return true; | 1230 return true; |
| 1071 } | 1231 } |
| 1072 | 1232 |
| 1073 bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const { | 1233 bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const { |
| 1074 // Check if we've upmixed or downmixed the audio. | 1234 // Check if we've upmixed or downmixed the audio. |
| 1075 return ((shared_state_.api_format_.output_stream().num_channels() != | 1235 return ((formats_.api_format.output_stream().num_channels() != |
| 1076 shared_state_.api_format_.input_stream().num_channels()) || | 1236 formats_.api_format.input_stream().num_channels()) || |
| 1077 is_data_processed || transient_suppressor_enabled_); | 1237 is_data_processed || capture_.transient_suppressor_enabled); |
| 1078 } | 1238 } |
| 1079 | 1239 |
| 1080 bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const { | 1240 bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const { |
| 1081 return (is_data_processed && | 1241 return (is_data_processed && |
| 1082 (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || | 1242 (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == |
| 1083 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz)); | 1243 kSampleRate32kHz || |
| 1244 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == | |
| 1245 kSampleRate48kHz)); | |
| 1084 } | 1246 } |
| 1085 | 1247 |
| 1086 bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const { | 1248 bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const { |
| 1087 if (!is_data_processed && !voice_detection_->is_enabled() && | 1249 if (!is_data_processed && |
| 1088 !transient_suppressor_enabled_) { | 1250 !public_submodules_->voice_detection->is_enabled() && |
| 1089 // Only level_estimator_ is enabled. | 1251 !capture_.transient_suppressor_enabled) { |
| 1252 // Only public_submodules_->level_estimator is enabled. | |
| 1090 return false; | 1253 return false; |
| 1091 } else if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || | 1254 } else if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == |
| 1092 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { | 1255 kSampleRate32kHz || |
| 1093 // Something besides level_estimator_ is enabled, and we have super-wb. | 1256 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == |
| 1257 kSampleRate48kHz) { | |
| 1258 // Something besides public_submodules_->level_estimator is enabled, and we | |
| 1259 // have super-wb. | |
| 1094 return true; | 1260 return true; |
| 1095 } | 1261 } |
| 1096 return false; | 1262 return false; |
| 1097 } | 1263 } |
| 1098 | 1264 |
| 1099 bool AudioProcessingImpl::is_rev_processed() const { | 1265 bool AudioProcessingImpl::is_rev_processed() const { |
| 1100 return intelligibility_enabled_ && intelligibility_enhancer_->active(); | 1266 return constants_.intelligibility_enabled && |
| 1267 public_submodules_->intelligibility_enhancer->active(); | |
| 1101 } | 1268 } |
| 1102 | 1269 |
| 1103 bool AudioProcessingImpl::rev_conversion_needed() const { | 1270 bool AudioProcessingImpl::rev_conversion_needed() const { |
| 1104 return (shared_state_.api_format_.reverse_input_stream() != | 1271 return (formats_.api_format.reverse_input_stream() != |
| 1105 shared_state_.api_format_.reverse_output_stream()); | 1272 formats_.api_format.reverse_output_stream()); |
| 1106 } | 1273 } |
| 1107 | 1274 |
| 1108 void AudioProcessingImpl::InitializeExperimentalAgc() { | 1275 void AudioProcessingImpl::InitializeExperimentalAgc() { |
| 1109 if (use_new_agc_) { | 1276 if (constants_.use_new_agc) { |
| 1110 if (!agc_manager_.get()) { | 1277 if (!private_submodules_->agc_manager.get()) { |
| 1111 agc_manager_.reset(new AgcManagerDirect(gain_control_, | 1278 private_submodules_->agc_manager.reset(new AgcManagerDirect( |
| 1112 gain_control_for_new_agc_.get(), | 1279 public_submodules_->gain_control, |
| 1113 agc_startup_min_volume_)); | 1280 public_submodules_->gain_control_for_new_agc.get(), |
| 1281 constants_.agc_startup_min_volume)); | |
| 1114 } | 1282 } |
| 1115 agc_manager_->Initialize(); | 1283 private_submodules_->agc_manager->Initialize(); |
| 1116 agc_manager_->SetCaptureMuted(output_will_be_muted_); | 1284 private_submodules_->agc_manager->SetCaptureMuted( |
| 1285 capture_.output_will_be_muted); | |
| 1117 } | 1286 } |
| 1118 } | 1287 } |
| 1119 | 1288 |
| 1120 void AudioProcessingImpl::InitializeTransient() { | 1289 void AudioProcessingImpl::InitializeTransient() { |
| 1121 if (transient_suppressor_enabled_) { | 1290 if (capture_.transient_suppressor_enabled) { |
| 1122 if (!transient_suppressor_.get()) { | 1291 if (!public_submodules_->transient_suppressor.get()) { |
| 1123 transient_suppressor_.reset(new TransientSuppressor()); | 1292 public_submodules_->transient_suppressor.reset(new TransientSuppressor()); |
| 1124 } | 1293 } |
| 1125 transient_suppressor_->Initialize( | 1294 public_submodules_->transient_suppressor->Initialize( |
| 1126 fwd_proc_format_.sample_rate_hz(), split_rate_, | 1295 capture_nonlocked_.fwd_proc_format.sample_rate_hz(), |
| 1127 shared_state_.api_format_.output_stream().num_channels()); | 1296 capture_nonlocked_.split_rate, |
| 1297 formats_.api_format.output_stream().num_channels()); | |
| 1128 } | 1298 } |
| 1129 } | 1299 } |
| 1130 | 1300 |
| 1131 void AudioProcessingImpl::InitializeBeamformer() { | 1301 void AudioProcessingImpl::InitializeBeamformer() { |
| 1132 if (beamformer_enabled_) { | 1302 if (capture_.beamformer_enabled) { |
| 1133 if (!beamformer_) { | 1303 if (!private_submodules_->beamformer) { |
| 1134 beamformer_.reset( | 1304 private_submodules_->beamformer.reset( |
| 1135 new NonlinearBeamformer(array_geometry_, target_direction_)); | 1305 new NonlinearBeamformer(constants_.array_geometry)); |
| 1136 } | 1306 } |
| 1137 beamformer_->Initialize(kChunkSizeMs, split_rate_); | 1307 private_submodules_->beamformer->Initialize(kChunkSizeMs, |
| 1308 capture_nonlocked_.split_rate); | |
| 1138 } | 1309 } |
| 1139 } | 1310 } |
| 1140 | 1311 |
| 1141 void AudioProcessingImpl::InitializeIntelligibility() { | 1312 void AudioProcessingImpl::InitializeIntelligibility() { |
| 1142 if (intelligibility_enabled_) { | 1313 if (constants_.intelligibility_enabled) { |
| 1143 IntelligibilityEnhancer::Config config; | 1314 IntelligibilityEnhancer::Config config; |
| 1144 config.sample_rate_hz = split_rate_; | 1315 config.sample_rate_hz = capture_nonlocked_.split_rate; |
| 1145 config.num_capture_channels = capture_audio_->num_channels(); | 1316 config.num_capture_channels = capture_.capture_audio->num_channels(); |
| 1146 config.num_render_channels = render_audio_->num_channels(); | 1317 config.num_render_channels = render_.render_audio->num_channels(); |
| 1147 intelligibility_enhancer_.reset(new IntelligibilityEnhancer(config)); | 1318 public_submodules_->intelligibility_enhancer.reset( |
| 1319 new IntelligibilityEnhancer(config)); | |
| 1148 } | 1320 } |
| 1149 } | 1321 } |
| 1150 | 1322 |
| 1151 void AudioProcessingImpl::MaybeUpdateHistograms() { | 1323 void AudioProcessingImpl::MaybeUpdateHistograms() { |
| 1152 static const int kMinDiffDelayMs = 60; | 1324 static const int kMinDiffDelayMs = 60; |
| 1153 | 1325 |
| 1154 if (echo_cancellation()->is_enabled()) { | 1326 if (echo_cancellation()->is_enabled()) { |
| 1155 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. | 1327 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. |
| 1156 // If a stream has echo we know that the echo_cancellation is in process. | 1328 // If a stream has echo we know that the echo_cancellation is in process. |
| 1157 if (stream_delay_jumps_ == -1 && echo_cancellation()->stream_has_echo()) { | 1329 if (capture_.stream_delay_jumps == -1 && |
| 1158 stream_delay_jumps_ = 0; | 1330 echo_cancellation()->stream_has_echo()) { |
| 1331 capture_.stream_delay_jumps = 0; | |
| 1159 } | 1332 } |
| 1160 if (aec_system_delay_jumps_ == -1 && | 1333 if (capture_.aec_system_delay_jumps == -1 && |
| 1161 echo_cancellation()->stream_has_echo()) { | 1334 echo_cancellation()->stream_has_echo()) { |
| 1162 aec_system_delay_jumps_ = 0; | 1335 capture_.aec_system_delay_jumps = 0; |
| 1163 } | 1336 } |
| 1164 | 1337 |
| 1165 // Detect a jump in platform reported system delay and log the difference. | 1338 // Detect a jump in platform reported system delay and log the difference. |
| 1166 const int diff_stream_delay_ms = stream_delay_ms_ - last_stream_delay_ms_; | 1339 const int diff_stream_delay_ms = |
| 1167 if (diff_stream_delay_ms > kMinDiffDelayMs && last_stream_delay_ms_ != 0) { | 1340 capture_nonlocked_.stream_delay_ms - capture_.last_stream_delay_ms; |
| 1341 if (diff_stream_delay_ms > kMinDiffDelayMs && | |
| 1342 capture_.last_stream_delay_ms != 0) { | |
| 1168 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.PlatformReportedStreamDelayJump", | 1343 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.PlatformReportedStreamDelayJump", |
| 1169 diff_stream_delay_ms, kMinDiffDelayMs, 1000, 100); | 1344 diff_stream_delay_ms, kMinDiffDelayMs, 1000, 100); |
| 1170 if (stream_delay_jumps_ == -1) { | 1345 if (capture_.stream_delay_jumps == -1) { |
| 1171 stream_delay_jumps_ = 0; // Activate counter if needed. | 1346 capture_.stream_delay_jumps = 0; // Activate counter if needed. |
| 1172 } | 1347 } |
| 1173 stream_delay_jumps_++; | 1348 capture_.stream_delay_jumps++; |
| 1174 } | 1349 } |
| 1175 last_stream_delay_ms_ = stream_delay_ms_; | 1350 capture_.last_stream_delay_ms = capture_nonlocked_.stream_delay_ms; |
| 1176 | 1351 |
| 1177 // Detect a jump in AEC system delay and log the difference. | 1352 // Detect a jump in AEC system delay and log the difference. |
| 1178 const int frames_per_ms = rtc::CheckedDivExact(split_rate_, 1000); | 1353 const int frames_per_ms = |
| 1354 rtc::CheckedDivExact(capture_nonlocked_.split_rate, 1000); | |
| 1179 const int aec_system_delay_ms = | 1355 const int aec_system_delay_ms = |
| 1180 WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms; | 1356 WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms; |
| 1181 const int diff_aec_system_delay_ms = | 1357 const int diff_aec_system_delay_ms = |
| 1182 aec_system_delay_ms - last_aec_system_delay_ms_; | 1358 aec_system_delay_ms - capture_.last_aec_system_delay_ms; |
| 1183 if (diff_aec_system_delay_ms > kMinDiffDelayMs && | 1359 if (diff_aec_system_delay_ms > kMinDiffDelayMs && |
| 1184 last_aec_system_delay_ms_ != 0) { | 1360 capture_.last_aec_system_delay_ms != 0) { |
| 1185 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", | 1361 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", |
| 1186 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, | 1362 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, |
| 1187 100); | 1363 100); |
| 1188 if (aec_system_delay_jumps_ == -1) { | 1364 if (capture_.aec_system_delay_jumps == -1) { |
| 1189 aec_system_delay_jumps_ = 0; // Activate counter if needed. | 1365 capture_.aec_system_delay_jumps = 0; // Activate counter if needed. |
| 1190 } | 1366 } |
| 1191 aec_system_delay_jumps_++; | 1367 capture_.aec_system_delay_jumps++; |
| 1192 } | 1368 } |
| 1193 last_aec_system_delay_ms_ = aec_system_delay_ms; | 1369 capture_.last_aec_system_delay_ms = aec_system_delay_ms; |
| 1194 } | 1370 } |
| 1195 } | 1371 } |
| 1196 | 1372 |
| 1197 void AudioProcessingImpl::UpdateHistogramsOnCallEnd() { | 1373 void AudioProcessingImpl::UpdateHistogramsOnCallEnd() { |
| 1198 CriticalSectionScoped crit_scoped(crit_); | 1374 // Run in a single-threaded manner. |
| 1199 if (stream_delay_jumps_ > -1) { | 1375 rtc::CritScope cs_render(&crit_render_); |
| 1376 rtc::CritScope cs_capture(&crit_capture_); | |
| 1377 | |
| 1378 if (capture_.stream_delay_jumps > -1) { | |
| 1200 RTC_HISTOGRAM_ENUMERATION( | 1379 RTC_HISTOGRAM_ENUMERATION( |
| 1201 "WebRTC.Audio.NumOfPlatformReportedStreamDelayJumps", | 1380 "WebRTC.Audio.NumOfPlatformReportedStreamDelayJumps", |
| 1202 stream_delay_jumps_, 51); | 1381 capture_.stream_delay_jumps, 51); |
| 1203 } | 1382 } |
| 1204 stream_delay_jumps_ = -1; | 1383 capture_.stream_delay_jumps = -1; |
| 1205 last_stream_delay_ms_ = 0; | 1384 capture_.last_stream_delay_ms = 0; |
| 1206 | 1385 |
| 1207 if (aec_system_delay_jumps_ > -1) { | 1386 if (capture_.aec_system_delay_jumps > -1) { |
| 1208 RTC_HISTOGRAM_ENUMERATION("WebRTC.Audio.NumOfAecSystemDelayJumps", | 1387 RTC_HISTOGRAM_ENUMERATION("WebRTC.Audio.NumOfAecSystemDelayJumps", |
| 1209 aec_system_delay_jumps_, 51); | 1388 capture_.aec_system_delay_jumps, 51); |
| 1210 } | 1389 } |
| 1211 aec_system_delay_jumps_ = -1; | 1390 capture_.aec_system_delay_jumps = -1; |
| 1212 last_aec_system_delay_ms_ = 0; | 1391 capture_.last_aec_system_delay_ms = 0; |
| 1213 } | 1392 } |
| 1214 | 1393 |
| 1215 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1394 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 1216 int AudioProcessingImpl::WriteMessageToDebugFile() { | 1395 int AudioProcessingImpl::WriteMessageToDebugFile( |
| 1217 int32_t size = event_msg_->ByteSize(); | 1396 FileWrapper* debug_file, |
| 1397 rtc::CriticalSection* crit_debug, | |
| 1398 ApmDebugDumpThreadState* debug_state) { | |
| 1399 int32_t size = debug_state->event_msg->ByteSize(); | |
| 1218 if (size <= 0) { | 1400 if (size <= 0) { |
| 1219 return kUnspecifiedError; | 1401 return kUnspecifiedError; |
| 1220 } | 1402 } |
| 1221 #if defined(WEBRTC_ARCH_BIG_ENDIAN) | 1403 #if defined(WEBRTC_ARCH_BIG_ENDIAN) |
| 1222 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be | 1404 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be |
| 1223 // pretty safe in assuming little-endian. | 1405 // pretty safe in assuming little-endian. |
| 1224 #endif | 1406 #endif |
| 1225 | 1407 |
| 1226 if (!event_msg_->SerializeToString(&event_str_)) { | 1408 if (!debug_state->event_msg->SerializeToString(&debug_state->event_str)) { |
| 1227 return kUnspecifiedError; | 1409 return kUnspecifiedError; |
| 1228 } | 1410 } |
| 1229 | 1411 |
| 1230 // Write message preceded by its size. | 1412 { |
| 1231 if (!debug_file_->Write(&size, sizeof(int32_t))) { | 1413 // Ensure atomic writes of the message. |
| 1232 return kFileError; | 1414 rtc::CritScope cs_capture(crit_debug); |
| 1233 } | 1415 // Write message preceded by its size. |
| 1234 if (!debug_file_->Write(event_str_.data(), event_str_.length())) { | 1416 if (!debug_file->Write(&size, sizeof(int32_t))) { |
| 1235 return kFileError; | 1417 return kFileError; |
| 1418 } | |
| 1419 if (!debug_file->Write(debug_state->event_str.data(), | |
| 1420 debug_state->event_str.length())) { | |
| 1421 return kFileError; | |
| 1422 } | |
| 1236 } | 1423 } |
| 1237 | 1424 |
| 1238 event_msg_->Clear(); | 1425 debug_state->event_msg->Clear(); |
| 1239 | 1426 |
| 1240 return kNoError; | 1427 return kNoError; |
| 1241 } | 1428 } |
| 1242 | 1429 |
| 1243 int AudioProcessingImpl::WriteInitMessage() { | 1430 int AudioProcessingImpl::WriteInitMessage() { |
| 1244 event_msg_->set_type(audioproc::Event::INIT); | 1431 debug_dump_.capture.event_msg->set_type(audioproc::Event::INIT); |
| 1245 audioproc::Init* msg = event_msg_->mutable_init(); | 1432 audioproc::Init* msg = debug_dump_.capture.event_msg->mutable_init(); |
| 1246 msg->set_sample_rate( | 1433 msg->set_sample_rate(formats_.api_format.input_stream().sample_rate_hz()); |
| 1247 shared_state_.api_format_.input_stream().sample_rate_hz()); | |
| 1248 msg->set_num_input_channels( | 1434 msg->set_num_input_channels( |
| 1249 shared_state_.api_format_.input_stream().num_channels()); | 1435 formats_.api_format.input_stream().num_channels()); |
| 1250 msg->set_num_output_channels( | 1436 msg->set_num_output_channels( |
| 1251 shared_state_.api_format_.output_stream().num_channels()); | 1437 formats_.api_format.output_stream().num_channels()); |
| 1252 msg->set_num_reverse_channels( | 1438 msg->set_num_reverse_channels( |
| 1253 shared_state_.api_format_.reverse_input_stream().num_channels()); | 1439 formats_.api_format.reverse_input_stream().num_channels()); |
| 1254 msg->set_reverse_sample_rate( | 1440 msg->set_reverse_sample_rate( |
| 1255 shared_state_.api_format_.reverse_input_stream().sample_rate_hz()); | 1441 formats_.api_format.reverse_input_stream().sample_rate_hz()); |
| 1256 msg->set_output_sample_rate( | 1442 msg->set_output_sample_rate( |
| 1257 shared_state_.api_format_.output_stream().sample_rate_hz()); | 1443 formats_.api_format.output_stream().sample_rate_hz()); |
| 1258 // TODO(ekmeyerson): Add reverse output fields to event_msg_. | 1444 // TODO(ekmeyerson): Add reverse output fields to |
| 1445 // debug_dump_.capture.event_msg. | |
| 1259 | 1446 |
| 1260 RETURN_ON_ERR(WriteMessageToDebugFile()); | 1447 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 1448 &crit_debug_, &debug_dump_.capture)); | |
| 1261 return kNoError; | 1449 return kNoError; |
| 1262 } | 1450 } |
| 1263 | 1451 |
| 1264 int AudioProcessingImpl::WriteConfigMessage(bool forced) { | 1452 int AudioProcessingImpl::WriteConfigMessage(bool forced) { |
| 1265 audioproc::Config config; | 1453 audioproc::Config config; |
| 1266 | 1454 |
| 1267 config.set_aec_enabled(echo_cancellation_->is_enabled()); | 1455 config.set_aec_enabled(public_submodules_->echo_cancellation->is_enabled()); |
| 1268 config.set_aec_delay_agnostic_enabled( | 1456 config.set_aec_delay_agnostic_enabled( |
| 1269 echo_cancellation_->is_delay_agnostic_enabled()); | 1457 public_submodules_->echo_cancellation->is_delay_agnostic_enabled()); |
| 1270 config.set_aec_drift_compensation_enabled( | 1458 config.set_aec_drift_compensation_enabled( |
| 1271 echo_cancellation_->is_drift_compensation_enabled()); | 1459 public_submodules_->echo_cancellation->is_drift_compensation_enabled()); |
| 1272 config.set_aec_extended_filter_enabled( | 1460 config.set_aec_extended_filter_enabled( |
| 1273 echo_cancellation_->is_extended_filter_enabled()); | 1461 public_submodules_->echo_cancellation->is_extended_filter_enabled()); |
| 1274 config.set_aec_suppression_level( | 1462 config.set_aec_suppression_level(static_cast<int>( |
| 1275 static_cast<int>(echo_cancellation_->suppression_level())); | 1463 public_submodules_->echo_cancellation->suppression_level())); |
| 1276 | 1464 |
| 1277 config.set_aecm_enabled(echo_control_mobile_->is_enabled()); | 1465 config.set_aecm_enabled( |
| 1466 public_submodules_->echo_control_mobile->is_enabled()); | |
| 1278 config.set_aecm_comfort_noise_enabled( | 1467 config.set_aecm_comfort_noise_enabled( |
| 1279 echo_control_mobile_->is_comfort_noise_enabled()); | 1468 public_submodules_->echo_control_mobile->is_comfort_noise_enabled()); |
| 1280 config.set_aecm_routing_mode( | 1469 config.set_aecm_routing_mode(static_cast<int>( |
| 1281 static_cast<int>(echo_control_mobile_->routing_mode())); | 1470 public_submodules_->echo_control_mobile->routing_mode())); |
| 1282 | 1471 |
| 1283 config.set_agc_enabled(gain_control_->is_enabled()); | 1472 config.set_agc_enabled(public_submodules_->gain_control->is_enabled()); |
| 1284 config.set_agc_mode(static_cast<int>(gain_control_->mode())); | 1473 config.set_agc_mode( |
| 1285 config.set_agc_limiter_enabled(gain_control_->is_limiter_enabled()); | 1474 static_cast<int>(public_submodules_->gain_control->mode())); |
| 1286 config.set_noise_robust_agc_enabled(use_new_agc_); | 1475 config.set_agc_limiter_enabled( |
| 1476 public_submodules_->gain_control->is_limiter_enabled()); | |
| 1477 config.set_noise_robust_agc_enabled(constants_.use_new_agc); | |
| 1287 | 1478 |
| 1288 config.set_hpf_enabled(high_pass_filter_->is_enabled()); | 1479 config.set_hpf_enabled(public_submodules_->high_pass_filter->is_enabled()); |
| 1289 | 1480 |
| 1290 config.set_ns_enabled(noise_suppression_->is_enabled()); | 1481 config.set_ns_enabled(public_submodules_->noise_suppression->is_enabled()); |
| 1291 config.set_ns_level(static_cast<int>(noise_suppression_->level())); | 1482 config.set_ns_level( |
| 1483 static_cast<int>(public_submodules_->noise_suppression->level())); | |
| 1292 | 1484 |
| 1293 config.set_transient_suppression_enabled(transient_suppressor_enabled_); | 1485 config.set_transient_suppression_enabled( |
| 1486 capture_.transient_suppressor_enabled); | |
| 1294 | 1487 |
| 1295 std::string serialized_config = config.SerializeAsString(); | 1488 std::string serialized_config = config.SerializeAsString(); |
| 1296 if (!forced && last_serialized_config_ == serialized_config) { | 1489 if (!forced && |
| 1490 debug_dump_.capture.last_serialized_config == serialized_config) { | |
| 1297 return kNoError; | 1491 return kNoError; |
| 1298 } | 1492 } |
| 1299 | 1493 |
| 1300 last_serialized_config_ = serialized_config; | 1494 debug_dump_.capture.last_serialized_config = serialized_config; |
| 1301 | 1495 |
| 1302 event_msg_->set_type(audioproc::Event::CONFIG); | 1496 debug_dump_.capture.event_msg->set_type(audioproc::Event::CONFIG); |
| 1303 event_msg_->mutable_config()->CopyFrom(config); | 1497 debug_dump_.capture.event_msg->mutable_config()->CopyFrom(config); |
| 1304 | 1498 |
| 1305 RETURN_ON_ERR(WriteMessageToDebugFile()); | 1499 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 1500 &crit_debug_, &debug_dump_.capture)); | |
| 1306 return kNoError; | 1501 return kNoError; |
| 1307 } | 1502 } |
| 1308 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1503 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
| 1309 | 1504 |
| 1310 } // namespace webrtc | 1505 } // namespace webrtc |
| OLD | NEW |