OLD | NEW |
---|---|
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
(...skipping 18 matching lines...) Expand all Loading... | |
29 #include "webrtc/modules/audio_processing/echo_cancellation_impl.h" | 29 #include "webrtc/modules/audio_processing/echo_cancellation_impl.h" |
30 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h" | 30 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h" |
31 #include "webrtc/modules/audio_processing/gain_control_impl.h" | 31 #include "webrtc/modules/audio_processing/gain_control_impl.h" |
32 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h" | 32 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h" |
33 #include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhanc er.h" | 33 #include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhanc er.h" |
34 #include "webrtc/modules/audio_processing/level_estimator_impl.h" | 34 #include "webrtc/modules/audio_processing/level_estimator_impl.h" |
35 #include "webrtc/modules/audio_processing/noise_suppression_impl.h" | 35 #include "webrtc/modules/audio_processing/noise_suppression_impl.h" |
36 #include "webrtc/modules/audio_processing/processing_component.h" | 36 #include "webrtc/modules/audio_processing/processing_component.h" |
37 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h" | 37 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h" |
38 #include "webrtc/modules/audio_processing/voice_detection_impl.h" | 38 #include "webrtc/modules/audio_processing/voice_detection_impl.h" |
39 #include "webrtc/modules/include/module_common_types.h" | 39 #include "webrtc/modules/interface/module_common_types.h" |
40 #include "webrtc/system_wrappers/include/critical_section_wrapper.h" | |
41 #include "webrtc/system_wrappers/include/file_wrapper.h" | 40 #include "webrtc/system_wrappers/include/file_wrapper.h" |
42 #include "webrtc/system_wrappers/include/logging.h" | 41 #include "webrtc/system_wrappers/include/logging.h" |
43 #include "webrtc/system_wrappers/include/metrics.h" | 42 #include "webrtc/system_wrappers/include/metrics.h" |
44 | 43 |
45 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 44 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
46 // Files generated at build-time by the protobuf compiler. | 45 // Files generated at build-time by the protobuf compiler. |
47 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD | 46 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD |
48 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" | 47 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" |
49 #else | 48 #else |
50 #include "webrtc/audio_processing/debug.pb.h" | 49 #include "webrtc/audio_processing/debug.pb.h" |
(...skipping 17 matching lines...) Expand all Loading... | |
68 case AudioProcessing::kStereo: | 67 case AudioProcessing::kStereo: |
69 return false; | 68 return false; |
70 case AudioProcessing::kMonoAndKeyboard: | 69 case AudioProcessing::kMonoAndKeyboard: |
71 case AudioProcessing::kStereoAndKeyboard: | 70 case AudioProcessing::kStereoAndKeyboard: |
72 return true; | 71 return true; |
73 } | 72 } |
74 | 73 |
75 assert(false); | 74 assert(false); |
76 return false; | 75 return false; |
77 } | 76 } |
77 } // namespace | |
78 | 78 |
79 } // namespace | 79 struct ApmExternallyAccessibleSubModules { |
80 ApmExternallyAccessibleSubModules() | |
81 : echo_cancellation(NULL), | |
82 echo_control_mobile(NULL), | |
83 gain_control(NULL), | |
84 high_pass_filter(NULL), | |
85 level_estimator(NULL), | |
86 noise_suppression(NULL), | |
87 voice_detection(NULL) {} | |
88 // Accessed externally of APM without any lock acquired. | |
89 EchoCancellationImpl* echo_cancellation; | |
90 EchoControlMobileImpl* echo_control_mobile; | |
91 GainControlImpl* gain_control; | |
92 HighPassFilterImpl* high_pass_filter; | |
93 LevelEstimatorImpl* level_estimator; | |
94 NoiseSuppressionImpl* noise_suppression; | |
95 VoiceDetectionImpl* voice_detection; | |
96 rtc::scoped_ptr<GainControlForNewAgc> gain_control_for_new_agc; | |
97 | |
98 // Accessed internally from both render and capture. | |
99 rtc::scoped_ptr<TransientSuppressor> transient_suppressor; | |
100 rtc::scoped_ptr<IntelligibilityEnhancer> intelligibility_enhancer; | |
101 }; | |
102 | |
103 struct ApmInternalSubModules { | |
104 explicit ApmInternalSubModules(Beamformer<float>* beamformer) | |
105 : beamformer(beamformer) {} | |
106 // Accessed internally from capture or during initialization | |
107 std::list<ProcessingComponent*> component_list; | |
108 rtc::scoped_ptr<Beamformer<float>> beamformer; | |
109 rtc::scoped_ptr<AgcManagerDirect> agc_manager; | |
110 }; | |
80 | 111 |
81 // Throughout webrtc, it's assumed that success is represented by zero. | 112 // Throughout webrtc, it's assumed that success is represented by zero. |
82 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); | 113 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); |
83 | 114 |
84 // This class has two main functionalities: | 115 // This class has two main functionalities: |
85 // | 116 // |
86 // 1) It is returned instead of the real GainControl after the new AGC has been | 117 // 1) It is returned instead of the real GainControl after the new AGC has been |
87 // enabled in order to prevent an outside user from overriding compression | 118 // enabled in order to prevent an outside user from overriding compression |
88 // settings. It doesn't do anything in its implementation, except for | 119 // settings. It doesn't do anything in its implementation, except for |
89 // delegating the const methods and Enable calls to the real GainControl, so | 120 // delegating the const methods and Enable calls to the real GainControl, so |
(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
176 } | 207 } |
177 | 208 |
178 return apm; | 209 return apm; |
179 } | 210 } |
180 | 211 |
181 AudioProcessingImpl::AudioProcessingImpl(const Config& config) | 212 AudioProcessingImpl::AudioProcessingImpl(const Config& config) |
182 : AudioProcessingImpl(config, nullptr) {} | 213 : AudioProcessingImpl(config, nullptr) {} |
183 | 214 |
184 AudioProcessingImpl::AudioProcessingImpl(const Config& config, | 215 AudioProcessingImpl::AudioProcessingImpl(const Config& config, |
185 Beamformer<float>* beamformer) | 216 Beamformer<float>* beamformer) |
186 : echo_cancellation_(NULL), | 217 : public_submodules_(new ApmExternallyAccessibleSubModules()), |
187 echo_control_mobile_(NULL), | 218 private_submodules_(new ApmInternalSubModules(beamformer)), |
188 gain_control_(NULL), | 219 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) |
189 high_pass_filter_(NULL), | 220 constants_(config.Get<ExperimentalAgc>().startup_min_volume, |
190 level_estimator_(NULL), | 221 config.Get<Beamforming>().array_geometry, |
191 noise_suppression_(NULL), | 222 false, |
192 voice_detection_(NULL), | 223 config.Get<Intelligibility>().enabled), |
193 crit_(CriticalSectionWrapper::CreateCriticalSection()), | 224 #else |
194 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 225 constants_(config.Get<ExperimentalAgc>().startup_min_volume, |
195 debug_file_(FileWrapper::Create()), | 226 config.Get<Beamforming>().array_geometry, |
196 event_msg_(new audioproc::Event()), | 227 config.Get<ExperimentalAgc>().enabled, |
228 config.Get<Intelligibility>().enabled), | |
197 #endif | 229 #endif |
198 fwd_proc_format_(kSampleRate16kHz), | 230 |
199 rev_proc_format_(kSampleRate16kHz, 1), | |
200 split_rate_(kSampleRate16kHz), | |
201 stream_delay_ms_(0), | |
202 delay_offset_ms_(0), | |
203 was_stream_delay_set_(false), | |
204 last_stream_delay_ms_(0), | |
205 last_aec_system_delay_ms_(0), | |
206 stream_delay_jumps_(-1), | |
207 aec_system_delay_jumps_(-1), | |
208 output_will_be_muted_(false), | |
209 key_pressed_(false), | |
210 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) | 231 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) |
211 use_new_agc_(false), | 232 capture_(false, config.Get<Beamforming>().enabled) |
212 #else | 233 #else |
213 use_new_agc_(config.Get<ExperimentalAgc>().enabled), | 234 capture_(config.Get<ExperimentalNs>().enabled, |
235 config.Get<Beamforming>().enabled) | |
214 #endif | 236 #endif |
215 agc_startup_min_volume_(config.Get<ExperimentalAgc>().startup_min_volume), | 237 { |
216 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) | |
217 transient_suppressor_enabled_(false), | |
218 #else | |
219 transient_suppressor_enabled_(config.Get<ExperimentalNs>().enabled), | |
220 #endif | |
221 beamformer_enabled_(config.Get<Beamforming>().enabled), | |
222 beamformer_(beamformer), | |
223 array_geometry_(config.Get<Beamforming>().array_geometry), | |
224 target_direction_(config.Get<Beamforming>().target_direction), | |
225 intelligibility_enabled_(config.Get<Intelligibility>().enabled) { | |
226 render_thread_checker_.DetachFromThread(); | 238 render_thread_checker_.DetachFromThread(); |
227 capture_thread_checker_.DetachFromThread(); | 239 capture_thread_checker_.DetachFromThread(); |
228 | 240 |
229 echo_cancellation_ = | 241 { |
230 new EchoCancellationImpl(this, crit_, &render_thread_checker_); | 242 rtc::CritScope cs_render(&crit_render_); |
hlundin-webrtc
2015/11/05 16:11:21
Can you avoid taking the locks in the ctor?
peah-webrtc
2015/11/06 09:54:31
Seems to work. I thought that the threadannotation
hlundin-webrtc
2015/11/20 10:36:30
I think the it intentionally turns a blind eye to
| |
231 component_list_.push_back(echo_cancellation_); | 243 rtc::CritScope cs_capture(&crit_capture_); |
232 | 244 |
233 echo_control_mobile_ = | 245 public_submodules_->echo_cancellation = new EchoCancellationImpl( |
234 new EchoControlMobileImpl(this, crit_, &render_thread_checker_); | 246 this, &crit_render_, &crit_capture_, &render_thread_checker_); |
235 component_list_.push_back(echo_control_mobile_); | 247 public_submodules_->echo_control_mobile = new EchoControlMobileImpl( |
248 this, &crit_render_, &crit_capture_, &render_thread_checker_); | |
249 public_submodules_->gain_control = | |
250 new GainControlImpl(this, &crit_capture_, &crit_capture_, | |
251 &render_thread_checker_, &capture_thread_checker_); | |
252 public_submodules_->high_pass_filter = | |
253 new HighPassFilterImpl(this, &crit_capture_); | |
254 public_submodules_->level_estimator = new LevelEstimatorImpl(this); | |
255 public_submodules_->noise_suppression = | |
256 new NoiseSuppressionImpl(this, &crit_capture_); | |
257 public_submodules_->voice_detection = | |
258 new VoiceDetectionImpl(this, &crit_capture_); | |
259 public_submodules_->gain_control_for_new_agc.reset( | |
260 new GainControlForNewAgc(public_submodules_->gain_control)); | |
236 | 261 |
237 gain_control_ = new GainControlImpl(this, crit_, &render_thread_checker_, | 262 private_submodules_->component_list.push_back( |
238 &capture_thread_checker_); | 263 public_submodules_->echo_cancellation); |
239 component_list_.push_back(gain_control_); | 264 private_submodules_->component_list.push_back( |
240 | 265 public_submodules_->echo_control_mobile); |
241 high_pass_filter_ = new HighPassFilterImpl(this, crit_); | 266 private_submodules_->component_list.push_back( |
242 component_list_.push_back(high_pass_filter_); | 267 public_submodules_->gain_control); |
243 | 268 private_submodules_->component_list.push_back( |
244 level_estimator_ = new LevelEstimatorImpl(this, crit_); | 269 public_submodules_->high_pass_filter); |
245 component_list_.push_back(level_estimator_); | 270 private_submodules_->component_list.push_back( |
246 | 271 public_submodules_->level_estimator); |
247 noise_suppression_ = new NoiseSuppressionImpl(this, crit_); | 272 private_submodules_->component_list.push_back( |
248 component_list_.push_back(noise_suppression_); | 273 public_submodules_->noise_suppression); |
249 | 274 private_submodules_->component_list.push_back( |
250 voice_detection_ = new VoiceDetectionImpl(this, crit_); | 275 public_submodules_->voice_detection); |
251 component_list_.push_back(voice_detection_); | 276 } |
252 | |
253 gain_control_for_new_agc_.reset(new GainControlForNewAgc(gain_control_)); | |
254 | 277 |
255 SetExtraOptions(config); | 278 SetExtraOptions(config); |
256 } | 279 } |
257 | 280 |
258 AudioProcessingImpl::~AudioProcessingImpl() { | 281 AudioProcessingImpl::~AudioProcessingImpl() { |
259 { | 282 { |
260 CriticalSectionScoped crit_scoped(crit_); | 283 rtc::CritScope cs_render(&crit_render_); |
hlundin-webrtc
2015/11/05 16:11:21
There is no point in acquiring locks in the dtor.
peah-webrtc
2015/11/06 09:54:31
Done.
| |
261 // Depends on gain_control_ and gain_control_for_new_agc_. | 284 rtc::CritScope cs_capture(&crit_capture_); |
262 agc_manager_.reset(); | 285 |
286 // Set submodule pointers to NULL to ensure proper behavior if the accessor | |
hlundin-webrtc
2015/11/05 16:11:21
This also makes no sense.
peah-webrtc
2015/11/06 09:54:31
Done.
| |
287 // methods | |
288 // are called during APM destruction. Note that the submodule objects are | |
289 // deleted | |
290 // below using the component vector. | |
291 public_submodules_->echo_cancellation = nullptr; | |
292 public_submodules_->echo_control_mobile = nullptr; | |
293 public_submodules_->gain_control = nullptr; | |
294 public_submodules_->high_pass_filter = nullptr; | |
295 public_submodules_->level_estimator = nullptr; | |
296 public_submodules_->noise_suppression = nullptr; | |
297 public_submodules_->voice_detection = nullptr; | |
298 | |
299 // Depends on gain_control_ and | |
300 // public_submodules_->gain_control_for_new_agc. | |
301 private_submodules_->agc_manager.reset(); | |
263 // Depends on gain_control_. | 302 // Depends on gain_control_. |
264 gain_control_for_new_agc_.reset(); | 303 public_submodules_->gain_control_for_new_agc.reset(); |
265 while (!component_list_.empty()) { | 304 while (!private_submodules_->component_list.empty()) { |
266 ProcessingComponent* component = component_list_.front(); | 305 ProcessingComponent* component = |
306 private_submodules_->component_list.front(); | |
267 component->Destroy(); | 307 component->Destroy(); |
268 delete component; | 308 delete component; |
269 component_list_.pop_front(); | 309 private_submodules_->component_list.pop_front(); |
270 } | 310 } |
271 | 311 |
272 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 312 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
273 if (debug_file_->Open()) { | 313 if (debug_dump_.debug_file->Open()) { |
274 debug_file_->CloseFile(); | 314 debug_dump_.debug_file->CloseFile(); |
275 } | 315 } |
276 #endif | 316 #endif |
277 } | 317 } |
278 delete crit_; | |
279 crit_ = NULL; | |
280 } | 318 } |
281 | 319 |
282 int AudioProcessingImpl::Initialize() { | 320 int AudioProcessingImpl::Initialize() { |
283 CriticalSectionScoped crit_scoped(crit_); | 321 // Run in a single-threaded manner during initialization. |
322 rtc::CritScope cs_render(&crit_render_); | |
323 rtc::CritScope cs_capture(&crit_capture_); | |
284 return InitializeLocked(); | 324 return InitializeLocked(); |
285 } | 325 } |
286 | 326 |
287 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, | 327 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, |
288 int output_sample_rate_hz, | 328 int output_sample_rate_hz, |
289 int reverse_sample_rate_hz, | 329 int reverse_sample_rate_hz, |
290 ChannelLayout input_layout, | 330 ChannelLayout input_layout, |
291 ChannelLayout output_layout, | 331 ChannelLayout output_layout, |
292 ChannelLayout reverse_layout) { | 332 ChannelLayout reverse_layout) { |
333 // Run in a single-threaded manner during initialization. | |
334 rtc::CritScope cs_render(&crit_render_); | |
335 rtc::CritScope cs_capture(&crit_capture_); | |
336 | |
293 const ProcessingConfig processing_config = { | 337 const ProcessingConfig processing_config = { |
294 {{input_sample_rate_hz, | 338 {{input_sample_rate_hz, |
295 ChannelsFromLayout(input_layout), | 339 ChannelsFromLayout(input_layout), |
296 LayoutHasKeyboard(input_layout)}, | 340 LayoutHasKeyboard(input_layout)}, |
297 {output_sample_rate_hz, | 341 {output_sample_rate_hz, |
298 ChannelsFromLayout(output_layout), | 342 ChannelsFromLayout(output_layout), |
299 LayoutHasKeyboard(output_layout)}, | 343 LayoutHasKeyboard(output_layout)}, |
300 {reverse_sample_rate_hz, | 344 {reverse_sample_rate_hz, |
301 ChannelsFromLayout(reverse_layout), | 345 ChannelsFromLayout(reverse_layout), |
302 LayoutHasKeyboard(reverse_layout)}, | 346 LayoutHasKeyboard(reverse_layout)}, |
303 {reverse_sample_rate_hz, | 347 {reverse_sample_rate_hz, |
304 ChannelsFromLayout(reverse_layout), | 348 ChannelsFromLayout(reverse_layout), |
305 LayoutHasKeyboard(reverse_layout)}}}; | 349 LayoutHasKeyboard(reverse_layout)}}}; |
306 | 350 |
307 return Initialize(processing_config); | 351 return Initialize(processing_config); |
308 } | 352 } |
309 | 353 |
310 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { | 354 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { |
311 CriticalSectionScoped crit_scoped(crit_); | 355 // Run in a single-threaded manner during initialization. |
356 rtc::CritScope cs_render(&crit_render_); | |
357 rtc::CritScope cs_capture(&crit_capture_); | |
312 return InitializeLocked(processing_config); | 358 return InitializeLocked(processing_config); |
313 } | 359 } |
314 | 360 |
315 // Calls InitializeLocked() if any of the audio parameters have changed from | 361 // Calls InitializeLocked() if any of the audio parameters have changed from |
316 // their current values. | 362 // their current values (needs to be called while holding the crit_render_ |
317 int AudioProcessingImpl::MaybeInitializeLocked( | 363 // lock). |
364 int AudioProcessingImpl::MaybeInitialize( | |
318 const ProcessingConfig& processing_config) { | 365 const ProcessingConfig& processing_config) { |
319 if (processing_config == shared_state_.api_format_) { | 366 if (processing_config == formats_.api_format) { |
320 return kNoError; | 367 return kNoError; |
321 } | 368 } |
369 | |
370 rtc::CritScope cs_capture(&crit_capture_); | |
322 return InitializeLocked(processing_config); | 371 return InitializeLocked(processing_config); |
323 } | 372 } |
324 | 373 |
325 int AudioProcessingImpl::InitializeLocked() { | 374 int AudioProcessingImpl::InitializeLocked() { |
326 const int fwd_audio_buffer_channels = | 375 const int fwd_audio_buffer_channels = |
327 beamformer_enabled_ | 376 capture_.beamformer_enabled |
328 ? shared_state_.api_format_.input_stream().num_channels() | 377 ? formats_.api_format.input_stream().num_channels() |
329 : shared_state_.api_format_.output_stream().num_channels(); | 378 : formats_.api_format.output_stream().num_channels(); |
330 const int rev_audio_buffer_out_num_frames = | 379 const int rev_audio_buffer_out_num_frames = |
331 shared_state_.api_format_.reverse_output_stream().num_frames() == 0 | 380 formats_.api_format.reverse_output_stream().num_frames() == 0 |
332 ? rev_proc_format_.num_frames() | 381 ? formats_.rev_proc_format.num_frames() |
333 : shared_state_.api_format_.reverse_output_stream().num_frames(); | 382 : formats_.api_format.reverse_output_stream().num_frames(); |
334 if (shared_state_.api_format_.reverse_input_stream().num_channels() > 0) { | 383 if (formats_.api_format.reverse_input_stream().num_channels() > 0) { |
335 render_audio_.reset(new AudioBuffer( | 384 render_.render_audio.reset(new AudioBuffer( |
336 shared_state_.api_format_.reverse_input_stream().num_frames(), | 385 formats_.api_format.reverse_input_stream().num_frames(), |
337 shared_state_.api_format_.reverse_input_stream().num_channels(), | 386 formats_.api_format.reverse_input_stream().num_channels(), |
338 rev_proc_format_.num_frames(), rev_proc_format_.num_channels(), | 387 formats_.rev_proc_format.num_frames(), |
388 formats_.rev_proc_format.num_channels(), | |
339 rev_audio_buffer_out_num_frames)); | 389 rev_audio_buffer_out_num_frames)); |
340 if (rev_conversion_needed()) { | 390 if (rev_conversion_needed()) { |
341 render_converter_ = AudioConverter::Create( | 391 render_.render_converter = AudioConverter::Create( |
342 shared_state_.api_format_.reverse_input_stream().num_channels(), | 392 formats_.api_format.reverse_input_stream().num_channels(), |
343 shared_state_.api_format_.reverse_input_stream().num_frames(), | 393 formats_.api_format.reverse_input_stream().num_frames(), |
344 shared_state_.api_format_.reverse_output_stream().num_channels(), | 394 formats_.api_format.reverse_output_stream().num_channels(), |
345 shared_state_.api_format_.reverse_output_stream().num_frames()); | 395 formats_.api_format.reverse_output_stream().num_frames()); |
346 } else { | 396 } else { |
347 render_converter_.reset(nullptr); | 397 render_.render_converter.reset(nullptr); |
348 } | 398 } |
349 } else { | 399 } else { |
350 render_audio_.reset(nullptr); | 400 render_.render_audio.reset(nullptr); |
351 render_converter_.reset(nullptr); | 401 render_.render_converter.reset(nullptr); |
352 } | 402 } |
353 capture_audio_.reset( | 403 capture_.capture_audio.reset( |
354 new AudioBuffer(shared_state_.api_format_.input_stream().num_frames(), | 404 new AudioBuffer(formats_.api_format.input_stream().num_frames(), |
355 shared_state_.api_format_.input_stream().num_channels(), | 405 formats_.api_format.input_stream().num_channels(), |
356 fwd_proc_format_.num_frames(), fwd_audio_buffer_channels, | 406 capture_nonlocked_.fwd_proc_format.num_frames(), |
357 shared_state_.api_format_.output_stream().num_frames())); | 407 fwd_audio_buffer_channels, |
408 formats_.api_format.output_stream().num_frames())); | |
358 | 409 |
359 // Initialize all components. | 410 // Initialize all components. |
360 for (auto item : component_list_) { | 411 for (auto item : private_submodules_->component_list) { |
361 int err = item->Initialize(); | 412 int err = item->Initialize(); |
362 if (err != kNoError) { | 413 if (err != kNoError) { |
363 return err; | 414 return err; |
364 } | 415 } |
365 } | 416 } |
366 | 417 |
367 InitializeExperimentalAgc(); | 418 InitializeExperimentalAgc(); |
368 | 419 |
369 InitializeTransient(); | 420 InitializeTransient(); |
370 | 421 |
371 InitializeBeamformer(); | 422 InitializeBeamformer(); |
372 | 423 |
373 InitializeIntelligibility(); | 424 InitializeIntelligibility(); |
374 | 425 |
375 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 426 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
376 if (debug_file_->Open()) { | 427 if (debug_dump_.debug_file->Open()) { |
377 int err = WriteInitMessage(); | 428 int err = WriteInitMessage(); |
378 if (err != kNoError) { | 429 if (err != kNoError) { |
379 return err; | 430 return err; |
380 } | 431 } |
381 } | 432 } |
382 #endif | 433 #endif |
383 | 434 |
384 return kNoError; | 435 return kNoError; |
385 } | 436 } |
386 | 437 |
(...skipping 10 matching lines...) Expand all Loading... | |
397 const int num_in_channels = config.input_stream().num_channels(); | 448 const int num_in_channels = config.input_stream().num_channels(); |
398 const int num_out_channels = config.output_stream().num_channels(); | 449 const int num_out_channels = config.output_stream().num_channels(); |
399 | 450 |
400 // Need at least one input channel. | 451 // Need at least one input channel. |
401 // Need either one output channel or as many outputs as there are inputs. | 452 // Need either one output channel or as many outputs as there are inputs. |
402 if (num_in_channels == 0 || | 453 if (num_in_channels == 0 || |
403 !(num_out_channels == 1 || num_out_channels == num_in_channels)) { | 454 !(num_out_channels == 1 || num_out_channels == num_in_channels)) { |
404 return kBadNumberChannelsError; | 455 return kBadNumberChannelsError; |
405 } | 456 } |
406 | 457 |
407 if (beamformer_enabled_ && | 458 if (capture_.beamformer_enabled && (static_cast<size_t>(num_in_channels) != |
408 (static_cast<size_t>(num_in_channels) != array_geometry_.size() || | 459 constants_.array_geometry.size() || |
409 num_out_channels > 1)) { | 460 num_out_channels > 1)) { |
410 return kBadNumberChannelsError; | 461 return kBadNumberChannelsError; |
411 } | 462 } |
412 | 463 |
413 shared_state_.api_format_ = config; | 464 formats_.api_format = config; |
414 | 465 |
415 // We process at the closest native rate >= min(input rate, output rate)... | 466 // We process at the closest native rate >= min(input rate, output rate)... |
416 const int min_proc_rate = | 467 const int min_proc_rate = |
417 std::min(shared_state_.api_format_.input_stream().sample_rate_hz(), | 468 std::min(formats_.api_format.input_stream().sample_rate_hz(), |
418 shared_state_.api_format_.output_stream().sample_rate_hz()); | 469 formats_.api_format.output_stream().sample_rate_hz()); |
419 int fwd_proc_rate; | 470 int fwd_proc_rate; |
420 for (size_t i = 0; i < kNumNativeSampleRates; ++i) { | 471 for (size_t i = 0; i < kNumNativeSampleRates; ++i) { |
421 fwd_proc_rate = kNativeSampleRatesHz[i]; | 472 fwd_proc_rate = kNativeSampleRatesHz[i]; |
422 if (fwd_proc_rate >= min_proc_rate) { | 473 if (fwd_proc_rate >= min_proc_rate) { |
423 break; | 474 break; |
424 } | 475 } |
425 } | 476 } |
426 // ...with one exception. | 477 // ...with one exception. |
427 if (echo_control_mobile_->is_enabled() && | 478 if (public_submodules_->echo_control_mobile->is_enabled() && |
428 min_proc_rate > kMaxAECMSampleRateHz) { | 479 min_proc_rate > kMaxAECMSampleRateHz) { |
429 fwd_proc_rate = kMaxAECMSampleRateHz; | 480 fwd_proc_rate = kMaxAECMSampleRateHz; |
430 } | 481 } |
431 | 482 |
432 fwd_proc_format_ = StreamConfig(fwd_proc_rate); | 483 capture_nonlocked_.fwd_proc_format = StreamConfig(fwd_proc_rate); |
433 | 484 |
434 // We normally process the reverse stream at 16 kHz. Unless... | 485 // We normally process the reverse stream at 16 kHz. Unless... |
435 int rev_proc_rate = kSampleRate16kHz; | 486 int rev_proc_rate = kSampleRate16kHz; |
436 if (fwd_proc_format_.sample_rate_hz() == kSampleRate8kHz) { | 487 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate8kHz) { |
437 // ...the forward stream is at 8 kHz. | 488 // ...the forward stream is at 8 kHz. |
438 rev_proc_rate = kSampleRate8kHz; | 489 rev_proc_rate = kSampleRate8kHz; |
439 } else { | 490 } else { |
440 if (shared_state_.api_format_.reverse_input_stream().sample_rate_hz() == | 491 if (formats_.api_format.reverse_input_stream().sample_rate_hz() == |
441 kSampleRate32kHz) { | 492 kSampleRate32kHz) { |
442 // ...or the input is at 32 kHz, in which case we use the splitting | 493 // ...or the input is at 32 kHz, in which case we use the splitting |
443 // filter rather than the resampler. | 494 // filter rather than the resampler. |
444 rev_proc_rate = kSampleRate32kHz; | 495 rev_proc_rate = kSampleRate32kHz; |
445 } | 496 } |
446 } | 497 } |
447 | 498 |
448 // Always downmix the reverse stream to mono for analysis. This has been | 499 // Always downmix the reverse stream to mono for analysis. This has been |
449 // demonstrated to work well for AEC in most practical scenarios. | 500 // demonstrated to work well for AEC in most practical scenarios. |
450 rev_proc_format_ = StreamConfig(rev_proc_rate, 1); | 501 formats_.rev_proc_format = StreamConfig(rev_proc_rate, 1); |
451 | 502 |
452 if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || | 503 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate32kHz || |
453 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { | 504 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate48kHz) { |
454 split_rate_ = kSampleRate16kHz; | 505 capture_nonlocked_.split_rate = kSampleRate16kHz; |
455 } else { | 506 } else { |
456 split_rate_ = fwd_proc_format_.sample_rate_hz(); | 507 capture_nonlocked_.split_rate = |
508 capture_nonlocked_.fwd_proc_format.sample_rate_hz(); | |
457 } | 509 } |
458 | 510 |
459 return InitializeLocked(); | 511 return InitializeLocked(); |
460 } | 512 } |
461 | 513 |
462 void AudioProcessingImpl::SetExtraOptions(const Config& config) { | 514 void AudioProcessingImpl::SetExtraOptions(const Config& config) { |
463 CriticalSectionScoped crit_scoped(crit_); | 515 // Run in a single-threaded manner when setting the extra options. |
464 for (auto item : component_list_) { | 516 rtc::CritScope cs_render(&crit_render_); |
517 rtc::CritScope cs_capture(&crit_capture_); | |
518 for (auto item : private_submodules_->component_list) { | |
465 item->SetExtraOptions(config); | 519 item->SetExtraOptions(config); |
466 } | 520 } |
467 | 521 |
468 if (transient_suppressor_enabled_ != config.Get<ExperimentalNs>().enabled) { | 522 if (capture_.transient_suppressor_enabled != |
469 transient_suppressor_enabled_ = config.Get<ExperimentalNs>().enabled; | 523 config.Get<ExperimentalNs>().enabled) { |
524 capture_.transient_suppressor_enabled = | |
525 config.Get<ExperimentalNs>().enabled; | |
470 InitializeTransient(); | 526 InitializeTransient(); |
471 } | 527 } |
472 } | 528 } |
473 | 529 |
474 | 530 |
475 int AudioProcessingImpl::proc_sample_rate_hz() const { | 531 int AudioProcessingImpl::proc_sample_rate_hz() const { |
476 return fwd_proc_format_.sample_rate_hz(); | 532 // Only called from submodules beneath APM, hence locking is not needed. |
533 return capture_nonlocked_.fwd_proc_format.sample_rate_hz(); | |
477 } | 534 } |
478 | 535 |
479 int AudioProcessingImpl::proc_split_sample_rate_hz() const { | 536 int AudioProcessingImpl::proc_split_sample_rate_hz() const { |
480 return split_rate_; | 537 // Only called from submodules/tests beneath APM, hence locking is not needed. |
538 return capture_nonlocked_.split_rate; | |
481 } | 539 } |
482 | 540 |
483 int AudioProcessingImpl::num_reverse_channels() const { | 541 int AudioProcessingImpl::num_reverse_channels() const { |
484 return rev_proc_format_.num_channels(); | 542 // Only called from submodules/tests beneath APM, hence locking is not needed. |
543 return formats_.rev_proc_format.num_channels(); | |
485 } | 544 } |
486 | 545 |
487 int AudioProcessingImpl::num_input_channels() const { | 546 int AudioProcessingImpl::num_input_channels() const { |
488 return shared_state_.api_format_.input_stream().num_channels(); | 547 // Only called from submodules/tests beneath APM, hence locking is not needed. |
548 return formats_.api_format.input_stream().num_channels(); | |
489 } | 549 } |
490 | 550 |
491 int AudioProcessingImpl::num_output_channels() const { | 551 int AudioProcessingImpl::num_output_channels() const { |
492 return shared_state_.api_format_.output_stream().num_channels(); | 552 // Only called from submodules/tests beneath APM, hence locking is not needed. |
553 return formats_.api_format.output_stream().num_channels(); | |
493 } | 554 } |
494 | 555 |
495 void AudioProcessingImpl::set_output_will_be_muted(bool muted) { | 556 void AudioProcessingImpl::set_output_will_be_muted(bool muted) { |
496 CriticalSectionScoped lock(crit_); | 557 rtc::CritScope cs(&crit_capture_); |
497 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); | 558 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); |
498 output_will_be_muted_ = muted; | 559 capture_.output_will_be_muted = muted; |
499 if (agc_manager_.get()) { | 560 if (private_submodules_->agc_manager.get()) { |
500 agc_manager_->SetCaptureMuted(output_will_be_muted_); | 561 private_submodules_->agc_manager->SetCaptureMuted( |
562 capture_.output_will_be_muted); | |
501 } | 563 } |
502 } | 564 } |
503 | 565 |
504 | 566 |
505 int AudioProcessingImpl::ProcessStream(const float* const* src, | 567 int AudioProcessingImpl::ProcessStream(const float* const* src, |
506 size_t samples_per_channel, | 568 size_t samples_per_channel, |
507 int input_sample_rate_hz, | 569 int input_sample_rate_hz, |
508 ChannelLayout input_layout, | 570 ChannelLayout input_layout, |
509 int output_sample_rate_hz, | 571 int output_sample_rate_hz, |
510 ChannelLayout output_layout, | 572 ChannelLayout output_layout, |
511 float* const* dest) { | 573 float* const* dest) { |
512 CriticalSectionScoped crit_scoped(crit_); | |
513 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); | 574 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); |
514 StreamConfig input_stream = shared_state_.api_format_.input_stream(); | 575 StreamConfig input_stream; |
576 StreamConfig output_stream; | |
577 { | |
578 // Access the formats_.api_format.input_stream beneath the capture | |
579 // lock. | |
hlundin-webrtc
2015/11/05 16:11:21
Widow.
peah-webrtc
2015/11/06 09:54:31
Done.
| |
580 // The lock must be released as it is later required in the call | |
581 // to ProcessStream(,,,); | |
582 rtc::CritScope cs(&crit_capture_); | |
583 input_stream = formats_.api_format.input_stream(); | |
584 output_stream = formats_.api_format.output_stream(); | |
585 } | |
586 | |
515 input_stream.set_sample_rate_hz(input_sample_rate_hz); | 587 input_stream.set_sample_rate_hz(input_sample_rate_hz); |
516 input_stream.set_num_channels(ChannelsFromLayout(input_layout)); | 588 input_stream.set_num_channels(ChannelsFromLayout(input_layout)); |
517 input_stream.set_has_keyboard(LayoutHasKeyboard(input_layout)); | 589 input_stream.set_has_keyboard(LayoutHasKeyboard(input_layout)); |
518 | |
519 StreamConfig output_stream = shared_state_.api_format_.output_stream(); | |
520 output_stream.set_sample_rate_hz(output_sample_rate_hz); | 590 output_stream.set_sample_rate_hz(output_sample_rate_hz); |
521 output_stream.set_num_channels(ChannelsFromLayout(output_layout)); | 591 output_stream.set_num_channels(ChannelsFromLayout(output_layout)); |
522 output_stream.set_has_keyboard(LayoutHasKeyboard(output_layout)); | 592 output_stream.set_has_keyboard(LayoutHasKeyboard(output_layout)); |
523 | 593 |
524 if (samples_per_channel != input_stream.num_frames()) { | 594 if (samples_per_channel != input_stream.num_frames()) { |
525 return kBadDataLengthError; | 595 return kBadDataLengthError; |
526 } | 596 } |
527 return ProcessStream(src, input_stream, output_stream, dest); | 597 return ProcessStream(src, input_stream, output_stream, dest); |
528 } | 598 } |
529 | 599 |
530 int AudioProcessingImpl::ProcessStream(const float* const* src, | 600 int AudioProcessingImpl::ProcessStream(const float* const* src, |
531 const StreamConfig& input_config, | 601 const StreamConfig& input_config, |
532 const StreamConfig& output_config, | 602 const StreamConfig& output_config, |
533 float* const* dest) { | 603 float* const* dest) { |
534 CriticalSectionScoped crit_scoped(crit_); | |
535 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); | 604 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); |
605 { | |
606 // Acquire the capture lock in order to safely call the function | |
607 // that retrieves the render side data. This function accesses apm | |
608 // getters that need the capture lock held when being called. | |
609 rtc::CritScope cs_capture(&crit_capture_); | |
610 public_submodules_->echo_cancellation->ReadQueuedRenderData(); | |
611 public_submodules_->echo_control_mobile->ReadQueuedRenderData(); | |
612 public_submodules_->gain_control->ReadQueuedRenderData(); | |
613 } | |
536 if (!src || !dest) { | 614 if (!src || !dest) { |
537 return kNullPointerError; | 615 return kNullPointerError; |
538 } | 616 } |
539 | 617 |
540 echo_cancellation_->ReadQueuedRenderData(); | 618 ProcessingConfig processing_config = formats_.api_format; |
541 echo_control_mobile_->ReadQueuedRenderData(); | |
542 gain_control_->ReadQueuedRenderData(); | |
543 | |
544 ProcessingConfig processing_config = shared_state_.api_format_; | |
545 processing_config.input_stream() = input_config; | 619 processing_config.input_stream() = input_config; |
546 processing_config.output_stream() = output_config; | 620 processing_config.output_stream() = output_config; |
547 | 621 |
548 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | 622 { |
623 // Do conditional reinitialization. | |
624 rtc::CritScope cs_render(&crit_render_); | |
625 RETURN_ON_ERR(MaybeInitialize(processing_config)); | |
626 } | |
627 rtc::CritScope cs_capture(&crit_capture_); | |
628 | |
549 assert(processing_config.input_stream().num_frames() == | 629 assert(processing_config.input_stream().num_frames() == |
550 shared_state_.api_format_.input_stream().num_frames()); | 630 formats_.api_format.input_stream().num_frames()); |
551 | 631 |
552 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 632 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
553 if (debug_file_->Open()) { | 633 if (debug_dump_.debug_file->Open()) { |
554 RETURN_ON_ERR(WriteConfigMessage(false)); | 634 RETURN_ON_ERR(WriteConfigMessage(false)); |
555 | 635 |
556 event_msg_->set_type(audioproc::Event::STREAM); | 636 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); |
557 audioproc::Stream* msg = event_msg_->mutable_stream(); | 637 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
558 const size_t channel_size = | 638 const size_t channel_size = |
559 sizeof(float) * shared_state_.api_format_.input_stream().num_frames(); | 639 sizeof(float) * formats_.api_format.input_stream().num_frames(); |
560 for (int i = 0; i < shared_state_.api_format_.input_stream().num_channels(); | 640 for (int i = 0; i < formats_.api_format.input_stream().num_channels(); ++i) |
561 ++i) | |
562 msg->add_input_channel(src[i], channel_size); | 641 msg->add_input_channel(src[i], channel_size); |
563 } | 642 } |
564 #endif | 643 #endif |
565 | 644 |
566 capture_audio_->CopyFrom(src, shared_state_.api_format_.input_stream()); | 645 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream()); |
567 RETURN_ON_ERR(ProcessStreamLocked()); | 646 RETURN_ON_ERR(ProcessStreamLocked()); |
568 capture_audio_->CopyTo(shared_state_.api_format_.output_stream(), dest); | 647 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest); |
569 | 648 |
570 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 649 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
571 if (debug_file_->Open()) { | 650 if (debug_dump_.debug_file->Open()) { |
572 audioproc::Stream* msg = event_msg_->mutable_stream(); | 651 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
573 const size_t channel_size = | 652 const size_t channel_size = |
574 sizeof(float) * shared_state_.api_format_.output_stream().num_frames(); | 653 sizeof(float) * formats_.api_format.output_stream().num_frames(); |
575 for (int i = 0; | 654 for (int i = 0; i < formats_.api_format.output_stream().num_channels(); ++i) |
576 i < shared_state_.api_format_.output_stream().num_channels(); ++i) | |
577 msg->add_output_channel(dest[i], channel_size); | 655 msg->add_output_channel(dest[i], channel_size); |
578 RETURN_ON_ERR(WriteMessageToDebugFile()); | 656 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
657 &crit_debug_, &debug_dump_.capture)); | |
579 } | 658 } |
580 #endif | 659 #endif |
581 | 660 |
582 return kNoError; | 661 return kNoError; |
583 } | 662 } |
584 | 663 |
585 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { | 664 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { |
586 CriticalSectionScoped crit_scoped(crit_); | 665 // TODO(peah): Investigate why it is not possible to use a |
587 echo_cancellation_->ReadQueuedRenderData(); | 666 // thread checker for this method. |
588 echo_control_mobile_->ReadQueuedRenderData(); | 667 { |
589 gain_control_->ReadQueuedRenderData(); | 668 // Acquire the capture lock in order to safely call the function |
669 // that retrieves the render side data. This function accesses apm | |
670 // getters that need the capture lock held when being called. | |
671 // The lock needs to be released as | |
672 // public_submodules_->echo_control_mobile->is_enabled() aquires this lock | |
673 // as well. | |
674 rtc::CritScope cs_capture(&crit_capture_); | |
675 public_submodules_->echo_cancellation->ReadQueuedRenderData(); | |
676 public_submodules_->echo_control_mobile->ReadQueuedRenderData(); | |
677 public_submodules_->gain_control->ReadQueuedRenderData(); | |
678 } | |
590 | 679 |
591 if (!frame) { | 680 if (!frame) { |
592 return kNullPointerError; | 681 return kNullPointerError; |
593 } | 682 } |
594 // Must be a native rate. | 683 // Must be a native rate. |
595 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 684 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
596 frame->sample_rate_hz_ != kSampleRate16kHz && | 685 frame->sample_rate_hz_ != kSampleRate16kHz && |
597 frame->sample_rate_hz_ != kSampleRate32kHz && | 686 frame->sample_rate_hz_ != kSampleRate32kHz && |
598 frame->sample_rate_hz_ != kSampleRate48kHz) { | 687 frame->sample_rate_hz_ != kSampleRate48kHz) { |
599 return kBadSampleRateError; | 688 return kBadSampleRateError; |
600 } | 689 } |
601 | 690 |
602 if (echo_control_mobile_->is_enabled() && | 691 if (public_submodules_->echo_control_mobile->is_enabled() && |
603 frame->sample_rate_hz_ > kMaxAECMSampleRateHz) { | 692 frame->sample_rate_hz_ > kMaxAECMSampleRateHz) { |
604 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates"; | 693 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates"; |
605 return kUnsupportedComponentError; | 694 return kUnsupportedComponentError; |
606 } | 695 } |
607 | 696 |
608 // TODO(ajm): The input and output rates and channels are currently | 697 ProcessingConfig processing_config; |
609 // constrained to be identical in the int16 interface. | 698 { |
610 ProcessingConfig processing_config = shared_state_.api_format_; | 699 // Aquire lock for the access of api_format. |
700 // The lock is released immediately due to the conditional | |
701 // reinitialization. | |
702 rtc::CritScope cs_capture(&crit_capture_); | |
703 // TODO(ajm): The input and output rates and channels are currently | |
704 // constrained to be identical in the int16 interface. | |
705 processing_config = formats_.api_format; | |
706 } | |
611 processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_); | 707 processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_); |
612 processing_config.input_stream().set_num_channels(frame->num_channels_); | 708 processing_config.input_stream().set_num_channels(frame->num_channels_); |
613 processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_); | 709 processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_); |
614 processing_config.output_stream().set_num_channels(frame->num_channels_); | 710 processing_config.output_stream().set_num_channels(frame->num_channels_); |
615 | 711 |
616 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | 712 { |
713 // Do conditional reinitialization. | |
714 rtc::CritScope cs_render(&crit_render_); | |
715 RETURN_ON_ERR(MaybeInitialize(processing_config)); | |
716 } | |
717 rtc::CritScope cs_capture(&crit_capture_); | |
617 if (frame->samples_per_channel_ != | 718 if (frame->samples_per_channel_ != |
618 shared_state_.api_format_.input_stream().num_frames()) { | 719 formats_.api_format.input_stream().num_frames()) { |
619 return kBadDataLengthError; | 720 return kBadDataLengthError; |
620 } | 721 } |
621 | 722 |
622 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 723 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
623 if (debug_file_->Open()) { | 724 if (debug_dump_.debug_file->Open()) { |
624 event_msg_->set_type(audioproc::Event::STREAM); | 725 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); |
625 audioproc::Stream* msg = event_msg_->mutable_stream(); | 726 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
626 const size_t data_size = | 727 const size_t data_size = |
627 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 728 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
628 msg->set_input_data(frame->data_, data_size); | 729 msg->set_input_data(frame->data_, data_size); |
629 } | 730 } |
630 #endif | 731 #endif |
631 | 732 |
632 capture_audio_->DeinterleaveFrom(frame); | 733 capture_.capture_audio->DeinterleaveFrom(frame); |
633 RETURN_ON_ERR(ProcessStreamLocked()); | 734 RETURN_ON_ERR(ProcessStreamLocked()); |
634 capture_audio_->InterleaveTo(frame, output_copy_needed(is_data_processed())); | 735 capture_.capture_audio->InterleaveTo(frame, |
736 output_copy_needed(is_data_processed())); | |
635 | 737 |
636 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 738 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
637 if (debug_file_->Open()) { | 739 if (debug_dump_.debug_file->Open()) { |
638 audioproc::Stream* msg = event_msg_->mutable_stream(); | 740 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
639 const size_t data_size = | 741 const size_t data_size = |
640 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 742 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
641 msg->set_output_data(frame->data_, data_size); | 743 msg->set_output_data(frame->data_, data_size); |
642 RETURN_ON_ERR(WriteMessageToDebugFile()); | 744 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
745 &crit_debug_, &debug_dump_.capture)); | |
643 } | 746 } |
644 #endif | 747 #endif |
645 | 748 |
646 return kNoError; | 749 return kNoError; |
647 } | 750 } |
648 | 751 |
649 int AudioProcessingImpl::ProcessStreamLocked() { | 752 int AudioProcessingImpl::ProcessStreamLocked() { |
650 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 753 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
651 if (debug_file_->Open()) { | 754 if (debug_dump_.debug_file->Open()) { |
652 audioproc::Stream* msg = event_msg_->mutable_stream(); | 755 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
653 msg->set_delay(stream_delay_ms_); | 756 msg->set_delay(capture_nonlocked_.stream_delay_ms); |
654 msg->set_drift(echo_cancellation_->stream_drift_samples()); | 757 msg->set_drift( |
758 public_submodules_->echo_cancellation->stream_drift_samples()); | |
655 msg->set_level(gain_control()->stream_analog_level()); | 759 msg->set_level(gain_control()->stream_analog_level()); |
656 msg->set_keypress(key_pressed_); | 760 msg->set_keypress(capture_.key_pressed); |
657 } | 761 } |
658 #endif | 762 #endif |
659 | 763 |
660 MaybeUpdateHistograms(); | 764 MaybeUpdateHistograms(); |
661 | 765 |
662 AudioBuffer* ca = capture_audio_.get(); // For brevity. | 766 AudioBuffer* ca = capture_.capture_audio.get(); // For brevity. |
663 | 767 |
664 if (use_new_agc_ && gain_control_->is_enabled()) { | 768 if (constants_.use_new_agc && |
665 agc_manager_->AnalyzePreProcess(ca->channels()[0], ca->num_channels(), | 769 public_submodules_->gain_control->is_enabled()) { |
666 fwd_proc_format_.num_frames()); | 770 private_submodules_->agc_manager->AnalyzePreProcess( |
771 ca->channels()[0], ca->num_channels(), | |
772 capture_nonlocked_.fwd_proc_format.num_frames()); | |
667 } | 773 } |
668 | 774 |
669 bool data_processed = is_data_processed(); | 775 bool data_processed = is_data_processed(); |
670 if (analysis_needed(data_processed)) { | 776 if (analysis_needed(data_processed)) { |
671 ca->SplitIntoFrequencyBands(); | 777 ca->SplitIntoFrequencyBands(); |
672 } | 778 } |
673 | 779 |
674 if (intelligibility_enabled_) { | 780 if (constants_.intelligibility_enabled) { |
675 intelligibility_enhancer_->AnalyzeCaptureAudio( | 781 public_submodules_->intelligibility_enhancer->AnalyzeCaptureAudio( |
676 ca->split_channels_f(kBand0To8kHz), split_rate_, ca->num_channels()); | 782 ca->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate, |
783 ca->num_channels()); | |
677 } | 784 } |
678 | 785 |
679 if (beamformer_enabled_) { | 786 if (capture_.beamformer_enabled) { |
680 beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f()); | 787 private_submodules_->beamformer->ProcessChunk(*ca->split_data_f(), |
788 ca->split_data_f()); | |
681 ca->set_num_channels(1); | 789 ca->set_num_channels(1); |
682 } | 790 } |
683 | 791 |
684 RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca)); | 792 RETURN_ON_ERR(public_submodules_->high_pass_filter->ProcessCaptureAudio(ca)); |
685 RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca)); | 793 RETURN_ON_ERR(public_submodules_->gain_control->AnalyzeCaptureAudio(ca)); |
686 RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca)); | 794 RETURN_ON_ERR(public_submodules_->noise_suppression->AnalyzeCaptureAudio(ca)); |
687 RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca)); | 795 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio(ca)); |
688 | 796 |
689 if (echo_control_mobile_->is_enabled() && noise_suppression_->is_enabled()) { | 797 if (public_submodules_->echo_control_mobile->is_enabled() && |
798 public_submodules_->noise_suppression->is_enabled()) { | |
690 ca->CopyLowPassToReference(); | 799 ca->CopyLowPassToReference(); |
691 } | 800 } |
692 RETURN_ON_ERR(noise_suppression_->ProcessCaptureAudio(ca)); | 801 RETURN_ON_ERR(public_submodules_->noise_suppression->ProcessCaptureAudio(ca)); |
693 RETURN_ON_ERR(echo_control_mobile_->ProcessCaptureAudio(ca)); | 802 RETURN_ON_ERR( |
694 RETURN_ON_ERR(voice_detection_->ProcessCaptureAudio(ca)); | 803 public_submodules_->echo_control_mobile->ProcessCaptureAudio(ca)); |
804 RETURN_ON_ERR(public_submodules_->voice_detection->ProcessCaptureAudio(ca)); | |
695 | 805 |
696 if (use_new_agc_ && gain_control_->is_enabled() && | 806 if (constants_.use_new_agc && |
697 (!beamformer_enabled_ || beamformer_->is_target_present())) { | 807 public_submodules_->gain_control->is_enabled() && |
698 agc_manager_->Process(ca->split_bands_const(0)[kBand0To8kHz], | 808 (!capture_.beamformer_enabled || |
699 ca->num_frames_per_band(), split_rate_); | 809 private_submodules_->beamformer->is_target_present())) { |
810 private_submodules_->agc_manager->Process( | |
811 ca->split_bands_const(0)[kBand0To8kHz], ca->num_frames_per_band(), | |
812 capture_nonlocked_.split_rate); | |
700 } | 813 } |
701 RETURN_ON_ERR(gain_control_->ProcessCaptureAudio(ca)); | 814 RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio(ca)); |
702 | 815 |
703 if (synthesis_needed(data_processed)) { | 816 if (synthesis_needed(data_processed)) { |
704 ca->MergeFrequencyBands(); | 817 ca->MergeFrequencyBands(); |
705 } | 818 } |
706 | 819 |
707 // TODO(aluebs): Investigate if the transient suppression placement should be | 820 // TODO(aluebs): Investigate if the transient suppression placement should be |
708 // before or after the AGC. | 821 // before or after the AGC. |
709 if (transient_suppressor_enabled_) { | 822 if (capture_.transient_suppressor_enabled) { |
710 float voice_probability = | 823 float voice_probability = |
711 agc_manager_.get() ? agc_manager_->voice_probability() : 1.f; | 824 private_submodules_->agc_manager.get() |
825 ? private_submodules_->agc_manager->voice_probability() | |
826 : 1.f; | |
712 | 827 |
713 transient_suppressor_->Suppress( | 828 public_submodules_->transient_suppressor->Suppress( |
714 ca->channels_f()[0], ca->num_frames(), ca->num_channels(), | 829 ca->channels_f()[0], ca->num_frames(), ca->num_channels(), |
715 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(), | 830 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(), |
716 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability, | 831 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability, |
717 key_pressed_); | 832 capture_.key_pressed); |
718 } | 833 } |
719 | 834 |
720 // The level estimator operates on the recombined data. | 835 // The level estimator operates on the recombined data. |
721 RETURN_ON_ERR(level_estimator_->ProcessStream(ca)); | 836 RETURN_ON_ERR(public_submodules_->level_estimator->ProcessStream(ca)); |
722 | 837 |
723 was_stream_delay_set_ = false; | 838 capture_.was_stream_delay_set = false; |
724 return kNoError; | 839 return kNoError; |
725 } | 840 } |
726 | 841 |
727 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, | 842 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, |
728 size_t samples_per_channel, | 843 size_t samples_per_channel, |
729 int rev_sample_rate_hz, | 844 int rev_sample_rate_hz, |
730 ChannelLayout layout) { | 845 ChannelLayout layout) { |
731 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); | 846 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); |
847 rtc::CritScope cs(&crit_render_); | |
732 const StreamConfig reverse_config = { | 848 const StreamConfig reverse_config = { |
733 rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), | 849 rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), |
734 }; | 850 }; |
735 if (samples_per_channel != reverse_config.num_frames()) { | 851 if (samples_per_channel != reverse_config.num_frames()) { |
736 return kBadDataLengthError; | 852 return kBadDataLengthError; |
737 } | 853 } |
738 return AnalyzeReverseStream(data, reverse_config, reverse_config); | 854 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config); |
739 } | 855 } |
740 | 856 |
741 int AudioProcessingImpl::ProcessReverseStream( | 857 int AudioProcessingImpl::ProcessReverseStream( |
742 const float* const* src, | 858 const float* const* src, |
743 const StreamConfig& reverse_input_config, | 859 const StreamConfig& reverse_input_config, |
744 const StreamConfig& reverse_output_config, | 860 const StreamConfig& reverse_output_config, |
745 float* const* dest) { | 861 float* const* dest) { |
746 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); | 862 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); |
747 RETURN_ON_ERR( | 863 rtc::CritScope cs(&crit_render_); |
748 AnalyzeReverseStream(src, reverse_input_config, reverse_output_config)); | 864 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, reverse_input_config, |
865 reverse_output_config)); | |
749 if (is_rev_processed()) { | 866 if (is_rev_processed()) { |
750 render_audio_->CopyTo(shared_state_.api_format_.reverse_output_stream(), | 867 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(), |
751 dest); | 868 dest); |
752 } else if (rev_conversion_needed()) { | 869 } else if (rev_conversion_needed()) { |
753 render_converter_->Convert(src, reverse_input_config.num_samples(), dest, | 870 render_.render_converter->Convert(src, reverse_input_config.num_samples(), |
754 reverse_output_config.num_samples()); | 871 dest, |
872 reverse_output_config.num_samples()); | |
755 } else { | 873 } else { |
756 CopyAudioIfNeeded(src, reverse_input_config.num_frames(), | 874 CopyAudioIfNeeded(src, reverse_input_config.num_frames(), |
757 reverse_input_config.num_channels(), dest); | 875 reverse_input_config.num_channels(), dest); |
758 } | 876 } |
759 | 877 |
760 return kNoError; | 878 return kNoError; |
761 } | 879 } |
762 | 880 |
763 int AudioProcessingImpl::AnalyzeReverseStream( | 881 int AudioProcessingImpl::AnalyzeReverseStreamLocked( |
764 const float* const* src, | 882 const float* const* src, |
765 const StreamConfig& reverse_input_config, | 883 const StreamConfig& reverse_input_config, |
766 const StreamConfig& reverse_output_config) { | 884 const StreamConfig& reverse_output_config) { |
767 CriticalSectionScoped crit_scoped(crit_); | |
768 if (src == NULL) { | 885 if (src == NULL) { |
769 return kNullPointerError; | 886 return kNullPointerError; |
770 } | 887 } |
771 | 888 |
772 if (reverse_input_config.num_channels() <= 0) { | 889 if (reverse_input_config.num_channels() <= 0) { |
773 return kBadNumberChannelsError; | 890 return kBadNumberChannelsError; |
774 } | 891 } |
775 | 892 |
776 ProcessingConfig processing_config = shared_state_.api_format_; | 893 ProcessingConfig processing_config = formats_.api_format; |
777 processing_config.reverse_input_stream() = reverse_input_config; | 894 processing_config.reverse_input_stream() = reverse_input_config; |
778 processing_config.reverse_output_stream() = reverse_output_config; | 895 processing_config.reverse_output_stream() = reverse_output_config; |
779 | 896 |
780 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | 897 RETURN_ON_ERR(MaybeInitialize(processing_config)); |
781 assert(reverse_input_config.num_frames() == | 898 assert(reverse_input_config.num_frames() == |
782 shared_state_.api_format_.reverse_input_stream().num_frames()); | 899 formats_.api_format.reverse_input_stream().num_frames()); |
783 | 900 |
784 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 901 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
785 if (debug_file_->Open()) { | 902 if (debug_dump_.debug_file->Open()) { |
786 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); | 903 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); |
787 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); | 904 audioproc::ReverseStream* msg = |
905 debug_dump_.render.event_msg->mutable_reverse_stream(); | |
788 const size_t channel_size = | 906 const size_t channel_size = |
789 sizeof(float) * | 907 sizeof(float) * formats_.api_format.reverse_input_stream().num_frames(); |
790 shared_state_.api_format_.reverse_input_stream().num_frames(); | |
791 for (int i = 0; | 908 for (int i = 0; |
792 i < shared_state_.api_format_.reverse_input_stream().num_channels(); | 909 i < formats_.api_format.reverse_input_stream().num_channels(); ++i) |
793 ++i) | |
794 msg->add_channel(src[i], channel_size); | 910 msg->add_channel(src[i], channel_size); |
795 RETURN_ON_ERR(WriteMessageToDebugFile()); | 911 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
912 &crit_debug_, &debug_dump_.render)); | |
796 } | 913 } |
797 #endif | 914 #endif |
798 | 915 |
799 render_audio_->CopyFrom(src, | 916 render_.render_audio->CopyFrom(src, |
800 shared_state_.api_format_.reverse_input_stream()); | 917 formats_.api_format.reverse_input_stream()); |
801 return ProcessReverseStreamLocked(); | 918 return ProcessReverseStreamLocked(); |
802 } | 919 } |
803 | 920 |
804 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { | 921 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { |
805 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); | 922 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); |
806 RETURN_ON_ERR(AnalyzeReverseStream(frame)); | 923 RETURN_ON_ERR(AnalyzeReverseStream(frame)); |
924 rtc::CritScope cs(&crit_render_); | |
807 if (is_rev_processed()) { | 925 if (is_rev_processed()) { |
808 render_audio_->InterleaveTo(frame, true); | 926 render_.render_audio->InterleaveTo(frame, true); |
809 } | 927 } |
810 | 928 |
811 return kNoError; | 929 return kNoError; |
812 } | 930 } |
813 | 931 |
814 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { | 932 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { |
815 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); | 933 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); |
816 CriticalSectionScoped crit_scoped(crit_); | 934 rtc::CritScope cs(&crit_render_); |
817 if (frame == NULL) { | 935 if (frame == NULL) { |
818 return kNullPointerError; | 936 return kNullPointerError; |
819 } | 937 } |
820 // Must be a native rate. | 938 // Must be a native rate. |
821 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 939 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
822 frame->sample_rate_hz_ != kSampleRate16kHz && | 940 frame->sample_rate_hz_ != kSampleRate16kHz && |
823 frame->sample_rate_hz_ != kSampleRate32kHz && | 941 frame->sample_rate_hz_ != kSampleRate32kHz && |
824 frame->sample_rate_hz_ != kSampleRate48kHz) { | 942 frame->sample_rate_hz_ != kSampleRate48kHz) { |
825 return kBadSampleRateError; | 943 return kBadSampleRateError; |
826 } | 944 } |
827 // This interface does not tolerate different forward and reverse rates. | 945 // This interface does not tolerate different forward and reverse rates. |
828 if (frame->sample_rate_hz_ != | 946 if (frame->sample_rate_hz_ != |
829 shared_state_.api_format_.input_stream().sample_rate_hz()) { | 947 formats_.api_format.input_stream().sample_rate_hz()) { |
830 return kBadSampleRateError; | 948 return kBadSampleRateError; |
831 } | 949 } |
832 | 950 |
833 if (frame->num_channels_ <= 0) { | 951 if (frame->num_channels_ <= 0) { |
834 return kBadNumberChannelsError; | 952 return kBadNumberChannelsError; |
835 } | 953 } |
836 | 954 |
837 ProcessingConfig processing_config = shared_state_.api_format_; | 955 ProcessingConfig processing_config = formats_.api_format; |
838 processing_config.reverse_input_stream().set_sample_rate_hz( | 956 processing_config.reverse_input_stream().set_sample_rate_hz( |
839 frame->sample_rate_hz_); | 957 frame->sample_rate_hz_); |
840 processing_config.reverse_input_stream().set_num_channels( | 958 processing_config.reverse_input_stream().set_num_channels( |
841 frame->num_channels_); | 959 frame->num_channels_); |
842 processing_config.reverse_output_stream().set_sample_rate_hz( | 960 processing_config.reverse_output_stream().set_sample_rate_hz( |
843 frame->sample_rate_hz_); | 961 frame->sample_rate_hz_); |
844 processing_config.reverse_output_stream().set_num_channels( | 962 processing_config.reverse_output_stream().set_num_channels( |
845 frame->num_channels_); | 963 frame->num_channels_); |
846 | 964 |
847 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | 965 RETURN_ON_ERR(MaybeInitialize(processing_config)); |
848 if (frame->samples_per_channel_ != | 966 if (frame->samples_per_channel_ != |
849 shared_state_.api_format_.reverse_input_stream().num_frames()) { | 967 formats_.api_format.reverse_input_stream().num_frames()) { |
850 return kBadDataLengthError; | 968 return kBadDataLengthError; |
851 } | 969 } |
852 | 970 |
853 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 971 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
854 if (debug_file_->Open()) { | 972 if (debug_dump_.debug_file->Open()) { |
855 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); | 973 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); |
856 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); | 974 audioproc::ReverseStream* msg = |
975 debug_dump_.render.event_msg->mutable_reverse_stream(); | |
857 const size_t data_size = | 976 const size_t data_size = |
858 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 977 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
859 msg->set_data(frame->data_, data_size); | 978 msg->set_data(frame->data_, data_size); |
860 RETURN_ON_ERR(WriteMessageToDebugFile()); | 979 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
980 &crit_debug_, &debug_dump_.render)); | |
861 } | 981 } |
862 #endif | 982 #endif |
863 render_audio_->DeinterleaveFrom(frame); | 983 render_.render_audio->DeinterleaveFrom(frame); |
864 return ProcessReverseStreamLocked(); | 984 return ProcessReverseStreamLocked(); |
865 } | 985 } |
866 | 986 |
867 int AudioProcessingImpl::ProcessReverseStreamLocked() { | 987 int AudioProcessingImpl::ProcessReverseStreamLocked() { |
868 AudioBuffer* ra = render_audio_.get(); // For brevity. | 988 AudioBuffer* ra = render_.render_audio.get(); // For brevity. |
869 if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz) { | 989 if (formats_.rev_proc_format.sample_rate_hz() == kSampleRate32kHz) { |
870 ra->SplitIntoFrequencyBands(); | 990 ra->SplitIntoFrequencyBands(); |
871 } | 991 } |
872 | 992 |
873 if (intelligibility_enabled_) { | 993 if (constants_.intelligibility_enabled) { |
874 intelligibility_enhancer_->ProcessRenderAudio( | 994 // Currently run in single-threaded mode when the intelligibility |
875 ra->split_channels_f(kBand0To8kHz), split_rate_, ra->num_channels()); | 995 // enhancer is activated. |
996 // TODO(peah): Fix to be properly multi-threaded. | |
997 rtc::CritScope cs(&crit_capture_); | |
998 public_submodules_->intelligibility_enhancer->ProcessRenderAudio( | |
999 ra->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate, | |
1000 ra->num_channels()); | |
876 } | 1001 } |
877 | 1002 |
878 RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra)); | 1003 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessRenderAudio(ra)); |
879 RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra)); | 1004 RETURN_ON_ERR( |
880 if (!use_new_agc_) { | 1005 public_submodules_->echo_control_mobile->ProcessRenderAudio(ra)); |
881 RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra)); | 1006 if (!constants_.use_new_agc) { |
1007 RETURN_ON_ERR(public_submodules_->gain_control->ProcessRenderAudio(ra)); | |
882 } | 1008 } |
883 | 1009 |
884 if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz && | 1010 if (formats_.rev_proc_format.sample_rate_hz() == kSampleRate32kHz && |
885 is_rev_processed()) { | 1011 is_rev_processed()) { |
886 ra->MergeFrequencyBands(); | 1012 ra->MergeFrequencyBands(); |
887 } | 1013 } |
888 | 1014 |
889 return kNoError; | 1015 return kNoError; |
890 } | 1016 } |
891 | 1017 |
892 int AudioProcessingImpl::set_stream_delay_ms(int delay) { | 1018 int AudioProcessingImpl::set_stream_delay_ms(int delay) { |
1019 // TODO(peah): Refactor so that it is possible to add a threadchecker | |
1020 // (currently not solely called from the capture thread in all places). | |
1021 rtc::CritScope cs(&crit_capture_); | |
893 Error retval = kNoError; | 1022 Error retval = kNoError; |
894 was_stream_delay_set_ = true; | 1023 capture_.was_stream_delay_set = true; |
895 delay += delay_offset_ms_; | 1024 delay += capture_.delay_offset_ms; |
896 | 1025 |
897 if (delay < 0) { | 1026 if (delay < 0) { |
898 delay = 0; | 1027 delay = 0; |
899 retval = kBadStreamParameterWarning; | 1028 retval = kBadStreamParameterWarning; |
900 } | 1029 } |
901 | 1030 |
902 // TODO(ajm): the max is rather arbitrarily chosen; investigate. | 1031 // TODO(ajm): the max is rather arbitrarily chosen; investigate. |
903 if (delay > 500) { | 1032 if (delay > 500) { |
904 delay = 500; | 1033 delay = 500; |
905 retval = kBadStreamParameterWarning; | 1034 retval = kBadStreamParameterWarning; |
906 } | 1035 } |
907 | 1036 |
908 stream_delay_ms_ = delay; | 1037 capture_nonlocked_.stream_delay_ms = delay; |
909 return retval; | 1038 return retval; |
910 } | 1039 } |
911 | 1040 |
912 int AudioProcessingImpl::stream_delay_ms() const { | 1041 int AudioProcessingImpl::stream_delay_ms() const { |
913 return stream_delay_ms_; | 1042 return capture_nonlocked_.stream_delay_ms; |
914 } | 1043 } |
915 | 1044 |
916 bool AudioProcessingImpl::was_stream_delay_set() const { | 1045 bool AudioProcessingImpl::was_stream_delay_set() const { |
917 return was_stream_delay_set_; | 1046 // Only called from submodules/tests beneath APM, hence locking is not needed. |
1047 return capture_.was_stream_delay_set; | |
918 } | 1048 } |
919 | 1049 |
920 void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) { | 1050 void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) { |
921 key_pressed_ = key_pressed; | 1051 // Thread check not possible due to being called from different threads. |
1052 rtc::CritScope cs(&crit_capture_); | |
1053 capture_.key_pressed = key_pressed; | |
922 } | 1054 } |
923 | 1055 |
924 void AudioProcessingImpl::set_delay_offset_ms(int offset) { | 1056 void AudioProcessingImpl::set_delay_offset_ms(int offset) { |
925 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); | 1057 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); |
926 CriticalSectionScoped crit_scoped(crit_); | 1058 rtc::CritScope cs(&crit_capture_); |
927 delay_offset_ms_ = offset; | 1059 capture_.delay_offset_ms = offset; |
928 } | 1060 } |
929 | 1061 |
930 int AudioProcessingImpl::delay_offset_ms() const { | 1062 int AudioProcessingImpl::delay_offset_ms() const { |
931 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); | 1063 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); |
932 return delay_offset_ms_; | 1064 rtc::CritScope cs(&crit_capture_); |
1065 return capture_.delay_offset_ms; | |
933 } | 1066 } |
934 | 1067 |
935 int AudioProcessingImpl::StartDebugRecording( | 1068 int AudioProcessingImpl::StartDebugRecording( |
936 const char filename[AudioProcessing::kMaxFilenameSize]) { | 1069 const char filename[AudioProcessing::kMaxFilenameSize]) { |
937 CriticalSectionScoped crit_scoped(crit_); | 1070 // Run in a single-threaded manner. |
1071 rtc::CritScope cs_render(&crit_render_); | |
1072 rtc::CritScope cs_capture(&crit_capture_); | |
1073 | |
938 static_assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize, ""); | 1074 static_assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize, ""); |
939 | 1075 |
940 if (filename == NULL) { | 1076 if (filename == NULL) { |
941 return kNullPointerError; | 1077 return kNullPointerError; |
942 } | 1078 } |
943 | 1079 |
944 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1080 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
945 // Stop any ongoing recording. | 1081 // Stop any ongoing recording. |
946 if (debug_file_->Open()) { | 1082 if (debug_dump_.debug_file->Open()) { |
947 if (debug_file_->CloseFile() == -1) { | 1083 if (debug_dump_.debug_file->CloseFile() == -1) { |
948 return kFileError; | 1084 return kFileError; |
949 } | 1085 } |
950 } | 1086 } |
951 | 1087 |
952 if (debug_file_->OpenFile(filename, false) == -1) { | 1088 if (debug_dump_.debug_file->OpenFile(filename, false) == -1) { |
953 debug_file_->CloseFile(); | 1089 debug_dump_.debug_file->CloseFile(); |
954 return kFileError; | 1090 return kFileError; |
955 } | 1091 } |
956 | 1092 |
957 RETURN_ON_ERR(WriteConfigMessage(true)); | 1093 RETURN_ON_ERR(WriteConfigMessage(true)); |
958 RETURN_ON_ERR(WriteInitMessage()); | 1094 RETURN_ON_ERR(WriteInitMessage()); |
959 return kNoError; | 1095 return kNoError; |
960 #else | 1096 #else |
961 return kUnsupportedFunctionError; | 1097 return kUnsupportedFunctionError; |
962 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1098 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
963 } | 1099 } |
964 | 1100 |
965 int AudioProcessingImpl::StartDebugRecording(FILE* handle) { | 1101 int AudioProcessingImpl::StartDebugRecording(FILE* handle) { |
966 CriticalSectionScoped crit_scoped(crit_); | 1102 // Run in a single-threaded manner. |
1103 rtc::CritScope cs_render(&crit_render_); | |
1104 rtc::CritScope cs_capture(&crit_capture_); | |
967 | 1105 |
968 if (handle == NULL) { | 1106 if (handle == NULL) { |
969 return kNullPointerError; | 1107 return kNullPointerError; |
970 } | 1108 } |
971 | 1109 |
972 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1110 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
973 // Stop any ongoing recording. | 1111 // Stop any ongoing recording. |
974 if (debug_file_->Open()) { | 1112 if (debug_dump_.debug_file->Open()) { |
975 if (debug_file_->CloseFile() == -1) { | 1113 if (debug_dump_.debug_file->CloseFile() == -1) { |
976 return kFileError; | 1114 return kFileError; |
977 } | 1115 } |
978 } | 1116 } |
979 | 1117 |
980 if (debug_file_->OpenFromFileHandle(handle, true, false) == -1) { | 1118 if (debug_dump_.debug_file->OpenFromFileHandle(handle, true, false) == -1) { |
981 return kFileError; | 1119 return kFileError; |
982 } | 1120 } |
983 | 1121 |
984 RETURN_ON_ERR(WriteConfigMessage(true)); | 1122 RETURN_ON_ERR(WriteConfigMessage(true)); |
985 RETURN_ON_ERR(WriteInitMessage()); | 1123 RETURN_ON_ERR(WriteInitMessage()); |
986 return kNoError; | 1124 return kNoError; |
987 #else | 1125 #else |
988 return kUnsupportedFunctionError; | 1126 return kUnsupportedFunctionError; |
989 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1127 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
990 } | 1128 } |
991 | 1129 |
992 int AudioProcessingImpl::StartDebugRecordingForPlatformFile( | 1130 int AudioProcessingImpl::StartDebugRecordingForPlatformFile( |
993 rtc::PlatformFile handle) { | 1131 rtc::PlatformFile handle) { |
1132 // Run in a single-threaded manner. | |
1133 rtc::CritScope cs_render(&crit_render_); | |
1134 rtc::CritScope cs_capture(&crit_capture_); | |
994 FILE* stream = rtc::FdopenPlatformFileForWriting(handle); | 1135 FILE* stream = rtc::FdopenPlatformFileForWriting(handle); |
995 return StartDebugRecording(stream); | 1136 return StartDebugRecording(stream); |
996 } | 1137 } |
997 | 1138 |
998 int AudioProcessingImpl::StopDebugRecording() { | 1139 int AudioProcessingImpl::StopDebugRecording() { |
999 CriticalSectionScoped crit_scoped(crit_); | 1140 // Run in a single-threaded manner. |
1141 rtc::CritScope cs_render(&crit_render_); | |
1142 rtc::CritScope cs_capture(&crit_capture_); | |
1000 | 1143 |
1001 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1144 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
1002 // We just return if recording hasn't started. | 1145 // We just return if recording hasn't started. |
1003 if (debug_file_->Open()) { | 1146 if (debug_dump_.debug_file->Open()) { |
1004 if (debug_file_->CloseFile() == -1) { | 1147 if (debug_dump_.debug_file->CloseFile() == -1) { |
1005 return kFileError; | 1148 return kFileError; |
1006 } | 1149 } |
1007 } | 1150 } |
1008 return kNoError; | 1151 return kNoError; |
1009 #else | 1152 #else |
1010 return kUnsupportedFunctionError; | 1153 return kUnsupportedFunctionError; |
1011 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1154 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
1012 } | 1155 } |
1013 | 1156 |
1014 EchoCancellation* AudioProcessingImpl::echo_cancellation() const { | 1157 EchoCancellation* AudioProcessingImpl::echo_cancellation() const { |
1015 return echo_cancellation_; | 1158 // Adding a lock here has no effect as it allows any access to the submodule |
1159 // from the returned pointer. | |
1160 return public_submodules_->echo_cancellation; | |
1016 } | 1161 } |
1017 | 1162 |
1018 EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const { | 1163 EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const { |
1019 return echo_control_mobile_; | 1164 // Adding a lock here has no effect as it allows any access to the submodule |
1165 // from the returned pointer. | |
1166 return public_submodules_->echo_control_mobile; | |
1020 } | 1167 } |
1021 | 1168 |
1022 GainControl* AudioProcessingImpl::gain_control() const { | 1169 GainControl* AudioProcessingImpl::gain_control() const { |
1023 if (use_new_agc_) { | 1170 // Adding a lock here has no effect as it allows any access to the submodule |
1024 return gain_control_for_new_agc_.get(); | 1171 // from the returned pointer. |
1172 if (constants_.use_new_agc) { | |
1173 return public_submodules_->gain_control_for_new_agc.get(); | |
1025 } | 1174 } |
1026 return gain_control_; | 1175 return public_submodules_->gain_control; |
1027 } | 1176 } |
1028 | 1177 |
1029 HighPassFilter* AudioProcessingImpl::high_pass_filter() const { | 1178 HighPassFilter* AudioProcessingImpl::high_pass_filter() const { |
1030 return high_pass_filter_; | 1179 // Adding a lock here has no effect as it allows any access to the submodule |
1180 // from the returned pointer. | |
1181 return public_submodules_->high_pass_filter; | |
1031 } | 1182 } |
1032 | 1183 |
1033 LevelEstimator* AudioProcessingImpl::level_estimator() const { | 1184 LevelEstimator* AudioProcessingImpl::level_estimator() const { |
1034 return level_estimator_; | 1185 // Adding a lock here has no effect as it allows any access to the submodule |
1186 // from the returned pointer. | |
1187 return public_submodules_->level_estimator; | |
1035 } | 1188 } |
1036 | 1189 |
1037 NoiseSuppression* AudioProcessingImpl::noise_suppression() const { | 1190 NoiseSuppression* AudioProcessingImpl::noise_suppression() const { |
1038 return noise_suppression_; | 1191 // Adding a lock here has no effect as it allows any access to the submodule |
1192 // from the returned pointer. | |
1193 return public_submodules_->noise_suppression; | |
1039 } | 1194 } |
1040 | 1195 |
1041 VoiceDetection* AudioProcessingImpl::voice_detection() const { | 1196 VoiceDetection* AudioProcessingImpl::voice_detection() const { |
1042 return voice_detection_; | 1197 // Adding a lock here has no effect as it allows any access to the submodule |
1198 // from the returned pointer. | |
1199 return public_submodules_->voice_detection; | |
1043 } | 1200 } |
1044 | 1201 |
1045 bool AudioProcessingImpl::is_data_processed() const { | 1202 bool AudioProcessingImpl::is_data_processed() const { |
1046 if (beamformer_enabled_) { | 1203 if (capture_.beamformer_enabled) { |
1047 return true; | 1204 return true; |
1048 } | 1205 } |
1049 | 1206 |
1050 int enabled_count = 0; | 1207 int enabled_count = 0; |
1051 for (auto item : component_list_) { | 1208 for (auto item : private_submodules_->component_list) { |
1052 if (item->is_component_enabled()) { | 1209 if (item->is_component_enabled()) { |
1053 enabled_count++; | 1210 enabled_count++; |
1054 } | 1211 } |
1055 } | 1212 } |
1056 | 1213 |
1057 // Data is unchanged if no components are enabled, or if only level_estimator_ | 1214 // Data is unchanged if no components are enabled, or if only |
1058 // or voice_detection_ is enabled. | 1215 // public_submodules_->level_estimator |
1216 // or public_submodules_->voice_detection is enabled. | |
1059 if (enabled_count == 0) { | 1217 if (enabled_count == 0) { |
1060 return false; | 1218 return false; |
1061 } else if (enabled_count == 1) { | 1219 } else if (enabled_count == 1) { |
1062 if (level_estimator_->is_enabled() || voice_detection_->is_enabled()) { | 1220 if (public_submodules_->level_estimator->is_enabled() || |
1221 public_submodules_->voice_detection->is_enabled()) { | |
1063 return false; | 1222 return false; |
1064 } | 1223 } |
1065 } else if (enabled_count == 2) { | 1224 } else if (enabled_count == 2) { |
1066 if (level_estimator_->is_enabled() && voice_detection_->is_enabled()) { | 1225 if (public_submodules_->level_estimator->is_enabled() && |
1226 public_submodules_->voice_detection->is_enabled()) { | |
1067 return false; | 1227 return false; |
1068 } | 1228 } |
1069 } | 1229 } |
1070 return true; | 1230 return true; |
1071 } | 1231 } |
1072 | 1232 |
1073 bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const { | 1233 bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const { |
1074 // Check if we've upmixed or downmixed the audio. | 1234 // Check if we've upmixed or downmixed the audio. |
1075 return ((shared_state_.api_format_.output_stream().num_channels() != | 1235 return ((formats_.api_format.output_stream().num_channels() != |
1076 shared_state_.api_format_.input_stream().num_channels()) || | 1236 formats_.api_format.input_stream().num_channels()) || |
1077 is_data_processed || transient_suppressor_enabled_); | 1237 is_data_processed || capture_.transient_suppressor_enabled); |
1078 } | 1238 } |
1079 | 1239 |
1080 bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const { | 1240 bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const { |
1081 return (is_data_processed && | 1241 return (is_data_processed && |
1082 (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || | 1242 (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == |
1083 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz)); | 1243 kSampleRate32kHz || |
1244 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == | |
1245 kSampleRate48kHz)); | |
1084 } | 1246 } |
1085 | 1247 |
1086 bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const { | 1248 bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const { |
1087 if (!is_data_processed && !voice_detection_->is_enabled() && | 1249 if (!is_data_processed && |
1088 !transient_suppressor_enabled_) { | 1250 !public_submodules_->voice_detection->is_enabled() && |
1089 // Only level_estimator_ is enabled. | 1251 !capture_.transient_suppressor_enabled) { |
1252 // Only public_submodules_->level_estimator is enabled. | |
1090 return false; | 1253 return false; |
1091 } else if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || | 1254 } else if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == |
1092 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { | 1255 kSampleRate32kHz || |
1093 // Something besides level_estimator_ is enabled, and we have super-wb. | 1256 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == |
1257 kSampleRate48kHz) { | |
1258 // Something besides public_submodules_->level_estimator is enabled, and we | |
1259 // have super-wb. | |
1094 return true; | 1260 return true; |
1095 } | 1261 } |
1096 return false; | 1262 return false; |
1097 } | 1263 } |
1098 | 1264 |
1099 bool AudioProcessingImpl::is_rev_processed() const { | 1265 bool AudioProcessingImpl::is_rev_processed() const { |
1100 return intelligibility_enabled_ && intelligibility_enhancer_->active(); | 1266 return constants_.intelligibility_enabled && |
1267 public_submodules_->intelligibility_enhancer->active(); | |
1101 } | 1268 } |
1102 | 1269 |
1103 bool AudioProcessingImpl::rev_conversion_needed() const { | 1270 bool AudioProcessingImpl::rev_conversion_needed() const { |
1104 return (shared_state_.api_format_.reverse_input_stream() != | 1271 return (formats_.api_format.reverse_input_stream() != |
1105 shared_state_.api_format_.reverse_output_stream()); | 1272 formats_.api_format.reverse_output_stream()); |
1106 } | 1273 } |
1107 | 1274 |
1108 void AudioProcessingImpl::InitializeExperimentalAgc() { | 1275 void AudioProcessingImpl::InitializeExperimentalAgc() { |
1109 if (use_new_agc_) { | 1276 if (constants_.use_new_agc) { |
1110 if (!agc_manager_.get()) { | 1277 if (!private_submodules_->agc_manager.get()) { |
1111 agc_manager_.reset(new AgcManagerDirect(gain_control_, | 1278 private_submodules_->agc_manager.reset(new AgcManagerDirect( |
1112 gain_control_for_new_agc_.get(), | 1279 public_submodules_->gain_control, |
1113 agc_startup_min_volume_)); | 1280 public_submodules_->gain_control_for_new_agc.get(), |
1281 constants_.agc_startup_min_volume)); | |
1114 } | 1282 } |
1115 agc_manager_->Initialize(); | 1283 private_submodules_->agc_manager->Initialize(); |
1116 agc_manager_->SetCaptureMuted(output_will_be_muted_); | 1284 private_submodules_->agc_manager->SetCaptureMuted( |
1285 capture_.output_will_be_muted); | |
1117 } | 1286 } |
1118 } | 1287 } |
1119 | 1288 |
1120 void AudioProcessingImpl::InitializeTransient() { | 1289 void AudioProcessingImpl::InitializeTransient() { |
1121 if (transient_suppressor_enabled_) { | 1290 if (capture_.transient_suppressor_enabled) { |
1122 if (!transient_suppressor_.get()) { | 1291 if (!public_submodules_->transient_suppressor.get()) { |
1123 transient_suppressor_.reset(new TransientSuppressor()); | 1292 public_submodules_->transient_suppressor.reset(new TransientSuppressor()); |
1124 } | 1293 } |
1125 transient_suppressor_->Initialize( | 1294 public_submodules_->transient_suppressor->Initialize( |
1126 fwd_proc_format_.sample_rate_hz(), split_rate_, | 1295 capture_nonlocked_.fwd_proc_format.sample_rate_hz(), |
1127 shared_state_.api_format_.output_stream().num_channels()); | 1296 capture_nonlocked_.split_rate, |
1297 formats_.api_format.output_stream().num_channels()); | |
1128 } | 1298 } |
1129 } | 1299 } |
1130 | 1300 |
1131 void AudioProcessingImpl::InitializeBeamformer() { | 1301 void AudioProcessingImpl::InitializeBeamformer() { |
1132 if (beamformer_enabled_) { | 1302 if (capture_.beamformer_enabled) { |
1133 if (!beamformer_) { | 1303 if (!private_submodules_->beamformer) { |
1134 beamformer_.reset( | 1304 private_submodules_->beamformer.reset( |
1135 new NonlinearBeamformer(array_geometry_, target_direction_)); | 1305 new NonlinearBeamformer(constants_.array_geometry)); |
1136 } | 1306 } |
1137 beamformer_->Initialize(kChunkSizeMs, split_rate_); | 1307 private_submodules_->beamformer->Initialize(kChunkSizeMs, |
1308 capture_nonlocked_.split_rate); | |
1138 } | 1309 } |
1139 } | 1310 } |
1140 | 1311 |
1141 void AudioProcessingImpl::InitializeIntelligibility() { | 1312 void AudioProcessingImpl::InitializeIntelligibility() { |
1142 if (intelligibility_enabled_) { | 1313 if (constants_.intelligibility_enabled) { |
1143 IntelligibilityEnhancer::Config config; | 1314 IntelligibilityEnhancer::Config config; |
1144 config.sample_rate_hz = split_rate_; | 1315 config.sample_rate_hz = capture_nonlocked_.split_rate; |
1145 config.num_capture_channels = capture_audio_->num_channels(); | 1316 config.num_capture_channels = capture_.capture_audio->num_channels(); |
1146 config.num_render_channels = render_audio_->num_channels(); | 1317 config.num_render_channels = render_.render_audio->num_channels(); |
1147 intelligibility_enhancer_.reset(new IntelligibilityEnhancer(config)); | 1318 public_submodules_->intelligibility_enhancer.reset( |
1319 new IntelligibilityEnhancer(config)); | |
1148 } | 1320 } |
1149 } | 1321 } |
1150 | 1322 |
1151 void AudioProcessingImpl::MaybeUpdateHistograms() { | 1323 void AudioProcessingImpl::MaybeUpdateHistograms() { |
1152 static const int kMinDiffDelayMs = 60; | 1324 static const int kMinDiffDelayMs = 60; |
1153 | 1325 |
1154 if (echo_cancellation()->is_enabled()) { | 1326 if (echo_cancellation()->is_enabled()) { |
1155 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. | 1327 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. |
1156 // If a stream has echo we know that the echo_cancellation is in process. | 1328 // If a stream has echo we know that the echo_cancellation is in process. |
1157 if (stream_delay_jumps_ == -1 && echo_cancellation()->stream_has_echo()) { | 1329 if (capture_.stream_delay_jumps == -1 && |
1158 stream_delay_jumps_ = 0; | 1330 echo_cancellation()->stream_has_echo()) { |
1331 capture_.stream_delay_jumps = 0; | |
1159 } | 1332 } |
1160 if (aec_system_delay_jumps_ == -1 && | 1333 if (capture_.aec_system_delay_jumps == -1 && |
1161 echo_cancellation()->stream_has_echo()) { | 1334 echo_cancellation()->stream_has_echo()) { |
1162 aec_system_delay_jumps_ = 0; | 1335 capture_.aec_system_delay_jumps = 0; |
1163 } | 1336 } |
1164 | 1337 |
1165 // Detect a jump in platform reported system delay and log the difference. | 1338 // Detect a jump in platform reported system delay and log the difference. |
1166 const int diff_stream_delay_ms = stream_delay_ms_ - last_stream_delay_ms_; | 1339 const int diff_stream_delay_ms = |
1167 if (diff_stream_delay_ms > kMinDiffDelayMs && last_stream_delay_ms_ != 0) { | 1340 capture_nonlocked_.stream_delay_ms - capture_.last_stream_delay_ms; |
1341 if (diff_stream_delay_ms > kMinDiffDelayMs && | |
1342 capture_.last_stream_delay_ms != 0) { | |
1168 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.PlatformReportedStreamDelayJump", | 1343 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.PlatformReportedStreamDelayJump", |
1169 diff_stream_delay_ms, kMinDiffDelayMs, 1000, 100); | 1344 diff_stream_delay_ms, kMinDiffDelayMs, 1000, 100); |
1170 if (stream_delay_jumps_ == -1) { | 1345 if (capture_.stream_delay_jumps == -1) { |
1171 stream_delay_jumps_ = 0; // Activate counter if needed. | 1346 capture_.stream_delay_jumps = 0; // Activate counter if needed. |
1172 } | 1347 } |
1173 stream_delay_jumps_++; | 1348 capture_.stream_delay_jumps++; |
1174 } | 1349 } |
1175 last_stream_delay_ms_ = stream_delay_ms_; | 1350 capture_.last_stream_delay_ms = capture_nonlocked_.stream_delay_ms; |
1176 | 1351 |
1177 // Detect a jump in AEC system delay and log the difference. | 1352 // Detect a jump in AEC system delay and log the difference. |
1178 const int frames_per_ms = rtc::CheckedDivExact(split_rate_, 1000); | 1353 const int frames_per_ms = |
1354 rtc::CheckedDivExact(capture_nonlocked_.split_rate, 1000); | |
1179 const int aec_system_delay_ms = | 1355 const int aec_system_delay_ms = |
1180 WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms; | 1356 WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms; |
1181 const int diff_aec_system_delay_ms = | 1357 const int diff_aec_system_delay_ms = |
1182 aec_system_delay_ms - last_aec_system_delay_ms_; | 1358 aec_system_delay_ms - capture_.last_aec_system_delay_ms; |
1183 if (diff_aec_system_delay_ms > kMinDiffDelayMs && | 1359 if (diff_aec_system_delay_ms > kMinDiffDelayMs && |
1184 last_aec_system_delay_ms_ != 0) { | 1360 capture_.last_aec_system_delay_ms != 0) { |
1185 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", | 1361 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", |
1186 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, | 1362 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, |
1187 100); | 1363 100); |
1188 if (aec_system_delay_jumps_ == -1) { | 1364 if (capture_.aec_system_delay_jumps == -1) { |
1189 aec_system_delay_jumps_ = 0; // Activate counter if needed. | 1365 capture_.aec_system_delay_jumps = 0; // Activate counter if needed. |
1190 } | 1366 } |
1191 aec_system_delay_jumps_++; | 1367 capture_.aec_system_delay_jumps++; |
1192 } | 1368 } |
1193 last_aec_system_delay_ms_ = aec_system_delay_ms; | 1369 capture_.last_aec_system_delay_ms = aec_system_delay_ms; |
1194 } | 1370 } |
1195 } | 1371 } |
1196 | 1372 |
1197 void AudioProcessingImpl::UpdateHistogramsOnCallEnd() { | 1373 void AudioProcessingImpl::UpdateHistogramsOnCallEnd() { |
1198 CriticalSectionScoped crit_scoped(crit_); | 1374 // Run in a single-threaded manner. |
1199 if (stream_delay_jumps_ > -1) { | 1375 rtc::CritScope cs_render(&crit_render_); |
1376 rtc::CritScope cs_capture(&crit_capture_); | |
1377 | |
1378 if (capture_.stream_delay_jumps > -1) { | |
1200 RTC_HISTOGRAM_ENUMERATION( | 1379 RTC_HISTOGRAM_ENUMERATION( |
1201 "WebRTC.Audio.NumOfPlatformReportedStreamDelayJumps", | 1380 "WebRTC.Audio.NumOfPlatformReportedStreamDelayJumps", |
1202 stream_delay_jumps_, 51); | 1381 capture_.stream_delay_jumps, 51); |
1203 } | 1382 } |
1204 stream_delay_jumps_ = -1; | 1383 capture_.stream_delay_jumps = -1; |
1205 last_stream_delay_ms_ = 0; | 1384 capture_.last_stream_delay_ms = 0; |
1206 | 1385 |
1207 if (aec_system_delay_jumps_ > -1) { | 1386 if (capture_.aec_system_delay_jumps > -1) { |
1208 RTC_HISTOGRAM_ENUMERATION("WebRTC.Audio.NumOfAecSystemDelayJumps", | 1387 RTC_HISTOGRAM_ENUMERATION("WebRTC.Audio.NumOfAecSystemDelayJumps", |
1209 aec_system_delay_jumps_, 51); | 1388 capture_.aec_system_delay_jumps, 51); |
1210 } | 1389 } |
1211 aec_system_delay_jumps_ = -1; | 1390 capture_.aec_system_delay_jumps = -1; |
1212 last_aec_system_delay_ms_ = 0; | 1391 capture_.last_aec_system_delay_ms = 0; |
1213 } | 1392 } |
1214 | 1393 |
1215 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1394 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
1216 int AudioProcessingImpl::WriteMessageToDebugFile() { | 1395 int AudioProcessingImpl::WriteMessageToDebugFile( |
1217 int32_t size = event_msg_->ByteSize(); | 1396 FileWrapper* debug_file, |
1397 rtc::CriticalSection* crit_debug, | |
1398 ApmDebugDumpThreadState* debug_state) { | |
1399 int32_t size = debug_state->event_msg->ByteSize(); | |
1218 if (size <= 0) { | 1400 if (size <= 0) { |
1219 return kUnspecifiedError; | 1401 return kUnspecifiedError; |
1220 } | 1402 } |
1221 #if defined(WEBRTC_ARCH_BIG_ENDIAN) | 1403 #if defined(WEBRTC_ARCH_BIG_ENDIAN) |
1222 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be | 1404 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be |
1223 // pretty safe in assuming little-endian. | 1405 // pretty safe in assuming little-endian. |
1224 #endif | 1406 #endif |
1225 | 1407 |
1226 if (!event_msg_->SerializeToString(&event_str_)) { | 1408 if (!debug_state->event_msg->SerializeToString(&debug_state->event_str)) { |
1227 return kUnspecifiedError; | 1409 return kUnspecifiedError; |
1228 } | 1410 } |
1229 | 1411 |
1230 // Write message preceded by its size. | 1412 { |
1231 if (!debug_file_->Write(&size, sizeof(int32_t))) { | 1413 // Ensure atomic writes of the message. |
1232 return kFileError; | 1414 rtc::CritScope cs_capture(crit_debug); |
1233 } | 1415 // Write message preceded by its size. |
1234 if (!debug_file_->Write(event_str_.data(), event_str_.length())) { | 1416 if (!debug_file->Write(&size, sizeof(int32_t))) { |
1235 return kFileError; | 1417 return kFileError; |
1418 } | |
1419 if (!debug_file->Write(debug_state->event_str.data(), | |
1420 debug_state->event_str.length())) { | |
1421 return kFileError; | |
1422 } | |
1236 } | 1423 } |
1237 | 1424 |
1238 event_msg_->Clear(); | 1425 debug_state->event_msg->Clear(); |
1239 | 1426 |
1240 return kNoError; | 1427 return kNoError; |
1241 } | 1428 } |
1242 | 1429 |
1243 int AudioProcessingImpl::WriteInitMessage() { | 1430 int AudioProcessingImpl::WriteInitMessage() { |
1244 event_msg_->set_type(audioproc::Event::INIT); | 1431 debug_dump_.capture.event_msg->set_type(audioproc::Event::INIT); |
1245 audioproc::Init* msg = event_msg_->mutable_init(); | 1432 audioproc::Init* msg = debug_dump_.capture.event_msg->mutable_init(); |
1246 msg->set_sample_rate( | 1433 msg->set_sample_rate(formats_.api_format.input_stream().sample_rate_hz()); |
1247 shared_state_.api_format_.input_stream().sample_rate_hz()); | |
1248 msg->set_num_input_channels( | 1434 msg->set_num_input_channels( |
1249 shared_state_.api_format_.input_stream().num_channels()); | 1435 formats_.api_format.input_stream().num_channels()); |
1250 msg->set_num_output_channels( | 1436 msg->set_num_output_channels( |
1251 shared_state_.api_format_.output_stream().num_channels()); | 1437 formats_.api_format.output_stream().num_channels()); |
1252 msg->set_num_reverse_channels( | 1438 msg->set_num_reverse_channels( |
1253 shared_state_.api_format_.reverse_input_stream().num_channels()); | 1439 formats_.api_format.reverse_input_stream().num_channels()); |
1254 msg->set_reverse_sample_rate( | 1440 msg->set_reverse_sample_rate( |
1255 shared_state_.api_format_.reverse_input_stream().sample_rate_hz()); | 1441 formats_.api_format.reverse_input_stream().sample_rate_hz()); |
1256 msg->set_output_sample_rate( | 1442 msg->set_output_sample_rate( |
1257 shared_state_.api_format_.output_stream().sample_rate_hz()); | 1443 formats_.api_format.output_stream().sample_rate_hz()); |
1258 // TODO(ekmeyerson): Add reverse output fields to event_msg_. | 1444 // TODO(ekmeyerson): Add reverse output fields to |
1445 // debug_dump_.capture.event_msg. | |
1259 | 1446 |
1260 RETURN_ON_ERR(WriteMessageToDebugFile()); | 1447 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
1448 &crit_debug_, &debug_dump_.capture)); | |
1261 return kNoError; | 1449 return kNoError; |
1262 } | 1450 } |
1263 | 1451 |
1264 int AudioProcessingImpl::WriteConfigMessage(bool forced) { | 1452 int AudioProcessingImpl::WriteConfigMessage(bool forced) { |
1265 audioproc::Config config; | 1453 audioproc::Config config; |
1266 | 1454 |
1267 config.set_aec_enabled(echo_cancellation_->is_enabled()); | 1455 config.set_aec_enabled(public_submodules_->echo_cancellation->is_enabled()); |
1268 config.set_aec_delay_agnostic_enabled( | 1456 config.set_aec_delay_agnostic_enabled( |
1269 echo_cancellation_->is_delay_agnostic_enabled()); | 1457 public_submodules_->echo_cancellation->is_delay_agnostic_enabled()); |
1270 config.set_aec_drift_compensation_enabled( | 1458 config.set_aec_drift_compensation_enabled( |
1271 echo_cancellation_->is_drift_compensation_enabled()); | 1459 public_submodules_->echo_cancellation->is_drift_compensation_enabled()); |
1272 config.set_aec_extended_filter_enabled( | 1460 config.set_aec_extended_filter_enabled( |
1273 echo_cancellation_->is_extended_filter_enabled()); | 1461 public_submodules_->echo_cancellation->is_extended_filter_enabled()); |
1274 config.set_aec_suppression_level( | 1462 config.set_aec_suppression_level(static_cast<int>( |
1275 static_cast<int>(echo_cancellation_->suppression_level())); | 1463 public_submodules_->echo_cancellation->suppression_level())); |
1276 | 1464 |
1277 config.set_aecm_enabled(echo_control_mobile_->is_enabled()); | 1465 config.set_aecm_enabled( |
1466 public_submodules_->echo_control_mobile->is_enabled()); | |
1278 config.set_aecm_comfort_noise_enabled( | 1467 config.set_aecm_comfort_noise_enabled( |
1279 echo_control_mobile_->is_comfort_noise_enabled()); | 1468 public_submodules_->echo_control_mobile->is_comfort_noise_enabled()); |
1280 config.set_aecm_routing_mode( | 1469 config.set_aecm_routing_mode(static_cast<int>( |
1281 static_cast<int>(echo_control_mobile_->routing_mode())); | 1470 public_submodules_->echo_control_mobile->routing_mode())); |
1282 | 1471 |
1283 config.set_agc_enabled(gain_control_->is_enabled()); | 1472 config.set_agc_enabled(public_submodules_->gain_control->is_enabled()); |
1284 config.set_agc_mode(static_cast<int>(gain_control_->mode())); | 1473 config.set_agc_mode( |
1285 config.set_agc_limiter_enabled(gain_control_->is_limiter_enabled()); | 1474 static_cast<int>(public_submodules_->gain_control->mode())); |
1286 config.set_noise_robust_agc_enabled(use_new_agc_); | 1475 config.set_agc_limiter_enabled( |
1476 public_submodules_->gain_control->is_limiter_enabled()); | |
1477 config.set_noise_robust_agc_enabled(constants_.use_new_agc); | |
1287 | 1478 |
1288 config.set_hpf_enabled(high_pass_filter_->is_enabled()); | 1479 config.set_hpf_enabled(public_submodules_->high_pass_filter->is_enabled()); |
1289 | 1480 |
1290 config.set_ns_enabled(noise_suppression_->is_enabled()); | 1481 config.set_ns_enabled(public_submodules_->noise_suppression->is_enabled()); |
1291 config.set_ns_level(static_cast<int>(noise_suppression_->level())); | 1482 config.set_ns_level( |
1483 static_cast<int>(public_submodules_->noise_suppression->level())); | |
1292 | 1484 |
1293 config.set_transient_suppression_enabled(transient_suppressor_enabled_); | 1485 config.set_transient_suppression_enabled( |
1486 capture_.transient_suppressor_enabled); | |
1294 | 1487 |
1295 std::string serialized_config = config.SerializeAsString(); | 1488 std::string serialized_config = config.SerializeAsString(); |
1296 if (!forced && last_serialized_config_ == serialized_config) { | 1489 if (!forced && |
1490 debug_dump_.capture.last_serialized_config == serialized_config) { | |
1297 return kNoError; | 1491 return kNoError; |
1298 } | 1492 } |
1299 | 1493 |
1300 last_serialized_config_ = serialized_config; | 1494 debug_dump_.capture.last_serialized_config = serialized_config; |
1301 | 1495 |
1302 event_msg_->set_type(audioproc::Event::CONFIG); | 1496 debug_dump_.capture.event_msg->set_type(audioproc::Event::CONFIG); |
1303 event_msg_->mutable_config()->CopyFrom(config); | 1497 debug_dump_.capture.event_msg->mutable_config()->CopyFrom(config); |
1304 | 1498 |
1305 RETURN_ON_ERR(WriteMessageToDebugFile()); | 1499 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
1500 &crit_debug_, &debug_dump_.capture)); | |
1306 return kNoError; | 1501 return kNoError; |
1307 } | 1502 } |
1308 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1503 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
1309 | 1504 |
1310 } // namespace webrtc | 1505 } // namespace webrtc |
OLD | NEW |