OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
(...skipping 19 matching lines...) Expand all Loading... |
30 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h" | 30 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h" |
31 #include "webrtc/modules/audio_processing/gain_control_impl.h" | 31 #include "webrtc/modules/audio_processing/gain_control_impl.h" |
32 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h" | 32 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h" |
33 #include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhanc
er.h" | 33 #include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhanc
er.h" |
34 #include "webrtc/modules/audio_processing/level_estimator_impl.h" | 34 #include "webrtc/modules/audio_processing/level_estimator_impl.h" |
35 #include "webrtc/modules/audio_processing/noise_suppression_impl.h" | 35 #include "webrtc/modules/audio_processing/noise_suppression_impl.h" |
36 #include "webrtc/modules/audio_processing/processing_component.h" | 36 #include "webrtc/modules/audio_processing/processing_component.h" |
37 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h" | 37 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h" |
38 #include "webrtc/modules/audio_processing/voice_detection_impl.h" | 38 #include "webrtc/modules/audio_processing/voice_detection_impl.h" |
39 #include "webrtc/modules/include/module_common_types.h" | 39 #include "webrtc/modules/include/module_common_types.h" |
40 #include "webrtc/system_wrappers/include/critical_section_wrapper.h" | |
41 #include "webrtc/system_wrappers/include/file_wrapper.h" | 40 #include "webrtc/system_wrappers/include/file_wrapper.h" |
42 #include "webrtc/system_wrappers/include/logging.h" | 41 #include "webrtc/system_wrappers/include/logging.h" |
43 #include "webrtc/system_wrappers/include/metrics.h" | 42 #include "webrtc/system_wrappers/include/metrics.h" |
44 | 43 |
45 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 44 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
46 // Files generated at build-time by the protobuf compiler. | 45 // Files generated at build-time by the protobuf compiler. |
47 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD | 46 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD |
48 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" | 47 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" |
49 #else | 48 #else |
50 #include "webrtc/audio_processing/debug.pb.h" | 49 #include "webrtc/audio_processing/debug.pb.h" |
(...skipping 17 matching lines...) Expand all Loading... |
68 case AudioProcessing::kStereo: | 67 case AudioProcessing::kStereo: |
69 return false; | 68 return false; |
70 case AudioProcessing::kMonoAndKeyboard: | 69 case AudioProcessing::kMonoAndKeyboard: |
71 case AudioProcessing::kStereoAndKeyboard: | 70 case AudioProcessing::kStereoAndKeyboard: |
72 return true; | 71 return true; |
73 } | 72 } |
74 | 73 |
75 assert(false); | 74 assert(false); |
76 return false; | 75 return false; |
77 } | 76 } |
| 77 } // namespace |
78 | 78 |
79 } // namespace | 79 struct AudioProcessingImpl::ApmPublicSubmodules { |
| 80 ApmPublicSubmodules() |
| 81 : echo_cancellation(nullptr), |
| 82 echo_control_mobile(nullptr), |
| 83 gain_control(nullptr), |
| 84 high_pass_filter(nullptr), |
| 85 level_estimator(nullptr), |
| 86 noise_suppression(nullptr), |
| 87 voice_detection(nullptr) {} |
| 88 // Accessed externally of APM without any lock acquired. |
| 89 EchoCancellationImpl* echo_cancellation; |
| 90 EchoControlMobileImpl* echo_control_mobile; |
| 91 GainControlImpl* gain_control; |
| 92 HighPassFilterImpl* high_pass_filter; |
| 93 LevelEstimatorImpl* level_estimator; |
| 94 NoiseSuppressionImpl* noise_suppression; |
| 95 VoiceDetectionImpl* voice_detection; |
| 96 rtc::scoped_ptr<GainControlForNewAgc> gain_control_for_new_agc; |
| 97 |
| 98 // Accessed internally from both render and capture. |
| 99 rtc::scoped_ptr<TransientSuppressor> transient_suppressor; |
| 100 rtc::scoped_ptr<IntelligibilityEnhancer> intelligibility_enhancer; |
| 101 }; |
| 102 |
| 103 struct AudioProcessingImpl::ApmPrivateSubmodules { |
| 104 explicit ApmPrivateSubmodules(Beamformer<float>* beamformer) |
| 105 : beamformer(beamformer) {} |
| 106 // Accessed internally from capture or during initialization |
| 107 std::list<ProcessingComponent*> component_list; |
| 108 rtc::scoped_ptr<Beamformer<float>> beamformer; |
| 109 rtc::scoped_ptr<AgcManagerDirect> agc_manager; |
| 110 }; |
80 | 111 |
81 // Throughout webrtc, it's assumed that success is represented by zero. | 112 // Throughout webrtc, it's assumed that success is represented by zero. |
82 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); | 113 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); |
83 | 114 |
84 // This class has two main functionalities: | 115 // This class has two main functionalities: |
85 // | 116 // |
86 // 1) It is returned instead of the real GainControl after the new AGC has been | 117 // 1) It is returned instead of the real GainControl after the new AGC has been |
87 // enabled in order to prevent an outside user from overriding compression | 118 // enabled in order to prevent an outside user from overriding compression |
88 // settings. It doesn't do anything in its implementation, except for | 119 // settings. It doesn't do anything in its implementation, except for |
89 // delegating the const methods and Enable calls to the real GainControl, so | 120 // delegating the const methods and Enable calls to the real GainControl, so |
(...skipping 75 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
165 | 196 |
166 AudioProcessing* AudioProcessing::Create(const Config& config) { | 197 AudioProcessing* AudioProcessing::Create(const Config& config) { |
167 return Create(config, nullptr); | 198 return Create(config, nullptr); |
168 } | 199 } |
169 | 200 |
170 AudioProcessing* AudioProcessing::Create(const Config& config, | 201 AudioProcessing* AudioProcessing::Create(const Config& config, |
171 Beamformer<float>* beamformer) { | 202 Beamformer<float>* beamformer) { |
172 AudioProcessingImpl* apm = new AudioProcessingImpl(config, beamformer); | 203 AudioProcessingImpl* apm = new AudioProcessingImpl(config, beamformer); |
173 if (apm->Initialize() != kNoError) { | 204 if (apm->Initialize() != kNoError) { |
174 delete apm; | 205 delete apm; |
175 apm = NULL; | 206 apm = nullptr; |
176 } | 207 } |
177 | 208 |
178 return apm; | 209 return apm; |
179 } | 210 } |
180 | 211 |
181 AudioProcessingImpl::AudioProcessingImpl(const Config& config) | 212 AudioProcessingImpl::AudioProcessingImpl(const Config& config) |
182 : AudioProcessingImpl(config, nullptr) {} | 213 : AudioProcessingImpl(config, nullptr) {} |
183 | 214 |
184 AudioProcessingImpl::AudioProcessingImpl(const Config& config, | 215 AudioProcessingImpl::AudioProcessingImpl(const Config& config, |
185 Beamformer<float>* beamformer) | 216 Beamformer<float>* beamformer) |
186 : echo_cancellation_(NULL), | 217 : public_submodules_(new ApmPublicSubmodules()), |
187 echo_control_mobile_(NULL), | 218 private_submodules_(new ApmPrivateSubmodules(beamformer)), |
188 gain_control_(NULL), | 219 constants_(config.Get<ExperimentalAgc>().startup_min_volume, |
189 high_pass_filter_(NULL), | 220 config.Get<Beamforming>().array_geometry, |
190 level_estimator_(NULL), | 221 config.Get<Beamforming>().target_direction, |
191 noise_suppression_(NULL), | 222 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) |
192 voice_detection_(NULL), | 223 false, |
193 crit_(CriticalSectionWrapper::CreateCriticalSection()), | 224 #else |
194 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 225 config.Get<ExperimentalAgc>().enabled, |
195 debug_file_(FileWrapper::Create()), | |
196 event_msg_(new audioproc::Event()), | |
197 #endif | 226 #endif |
198 fwd_proc_format_(kSampleRate16kHz), | 227 config.Get<Intelligibility>().enabled, |
199 rev_proc_format_(kSampleRate16kHz, 1), | 228 config.Get<Beamforming>().enabled), |
200 split_rate_(kSampleRate16kHz), | 229 |
201 stream_delay_ms_(0), | |
202 delay_offset_ms_(0), | |
203 was_stream_delay_set_(false), | |
204 last_stream_delay_ms_(0), | |
205 last_aec_system_delay_ms_(0), | |
206 stream_delay_jumps_(-1), | |
207 aec_system_delay_jumps_(-1), | |
208 output_will_be_muted_(false), | |
209 key_pressed_(false), | |
210 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) | 230 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) |
211 use_new_agc_(false), | 231 capture_(false) |
212 #else | 232 #else |
213 use_new_agc_(config.Get<ExperimentalAgc>().enabled), | 233 capture_(config.Get<ExperimentalNs>().enabled) |
214 #endif | 234 #endif |
215 agc_startup_min_volume_(config.Get<ExperimentalAgc>().startup_min_volume), | 235 { |
216 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) | 236 { |
217 transient_suppressor_enabled_(false), | 237 rtc::CritScope cs_render(&crit_render_); |
218 #else | 238 rtc::CritScope cs_capture(&crit_capture_); |
219 transient_suppressor_enabled_(config.Get<ExperimentalNs>().enabled), | |
220 #endif | |
221 beamformer_enabled_(config.Get<Beamforming>().enabled), | |
222 beamformer_(beamformer), | |
223 array_geometry_(config.Get<Beamforming>().array_geometry), | |
224 target_direction_(config.Get<Beamforming>().target_direction), | |
225 intelligibility_enabled_(config.Get<Intelligibility>().enabled) { | |
226 echo_cancellation_ = new EchoCancellationImpl(this, crit_); | |
227 component_list_.push_back(echo_cancellation_); | |
228 | 239 |
229 echo_control_mobile_ = new EchoControlMobileImpl(this, crit_); | 240 public_submodules_->echo_cancellation = |
230 component_list_.push_back(echo_control_mobile_); | 241 new EchoCancellationImpl(this, &crit_render_, &crit_capture_); |
| 242 public_submodules_->echo_control_mobile = |
| 243 new EchoControlMobileImpl(this, &crit_render_, &crit_capture_); |
| 244 public_submodules_->gain_control = |
| 245 new GainControlImpl(this, &crit_capture_, &crit_capture_); |
| 246 public_submodules_->high_pass_filter = |
| 247 new HighPassFilterImpl(this, &crit_capture_); |
| 248 public_submodules_->level_estimator = |
| 249 new LevelEstimatorImpl(this, &crit_capture_); |
| 250 public_submodules_->noise_suppression = |
| 251 new NoiseSuppressionImpl(this, &crit_capture_); |
| 252 public_submodules_->voice_detection = |
| 253 new VoiceDetectionImpl(this, &crit_capture_); |
| 254 public_submodules_->gain_control_for_new_agc.reset( |
| 255 new GainControlForNewAgc(public_submodules_->gain_control)); |
231 | 256 |
232 gain_control_ = new GainControlImpl(this, crit_); | 257 private_submodules_->component_list.push_back( |
233 component_list_.push_back(gain_control_); | 258 public_submodules_->echo_cancellation); |
234 | 259 private_submodules_->component_list.push_back( |
235 high_pass_filter_ = new HighPassFilterImpl(this, crit_); | 260 public_submodules_->echo_control_mobile); |
236 component_list_.push_back(high_pass_filter_); | 261 private_submodules_->component_list.push_back( |
237 | 262 public_submodules_->gain_control); |
238 level_estimator_ = new LevelEstimatorImpl(this, crit_); | 263 private_submodules_->component_list.push_back( |
239 component_list_.push_back(level_estimator_); | 264 public_submodules_->high_pass_filter); |
240 | 265 private_submodules_->component_list.push_back( |
241 noise_suppression_ = new NoiseSuppressionImpl(this, crit_); | 266 public_submodules_->level_estimator); |
242 component_list_.push_back(noise_suppression_); | 267 private_submodules_->component_list.push_back( |
243 | 268 public_submodules_->noise_suppression); |
244 voice_detection_ = new VoiceDetectionImpl(this, crit_); | 269 private_submodules_->component_list.push_back( |
245 component_list_.push_back(voice_detection_); | 270 public_submodules_->voice_detection); |
246 | 271 } |
247 gain_control_for_new_agc_.reset(new GainControlForNewAgc(gain_control_)); | |
248 | 272 |
249 SetExtraOptions(config); | 273 SetExtraOptions(config); |
250 } | 274 } |
251 | 275 |
252 AudioProcessingImpl::~AudioProcessingImpl() { | 276 AudioProcessingImpl::~AudioProcessingImpl() { |
253 { | 277 // Depends on gain_control_ and |
254 CriticalSectionScoped crit_scoped(crit_); | 278 // public_submodules_->gain_control_for_new_agc. |
255 // Depends on gain_control_ and gain_control_for_new_agc_. | 279 private_submodules_->agc_manager.reset(); |
256 agc_manager_.reset(); | 280 // Depends on gain_control_. |
257 // Depends on gain_control_. | 281 public_submodules_->gain_control_for_new_agc.reset(); |
258 gain_control_for_new_agc_.reset(); | 282 while (!private_submodules_->component_list.empty()) { |
259 while (!component_list_.empty()) { | 283 ProcessingComponent* component = |
260 ProcessingComponent* component = component_list_.front(); | 284 private_submodules_->component_list.front(); |
261 component->Destroy(); | 285 component->Destroy(); |
262 delete component; | 286 delete component; |
263 component_list_.pop_front(); | 287 private_submodules_->component_list.pop_front(); |
264 } | 288 } |
265 | 289 |
266 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 290 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
267 if (debug_file_->Open()) { | 291 if (debug_dump_.debug_file->Open()) { |
268 debug_file_->CloseFile(); | 292 debug_dump_.debug_file->CloseFile(); |
269 } | 293 } |
270 #endif | 294 #endif |
271 } | |
272 delete crit_; | |
273 crit_ = NULL; | |
274 } | 295 } |
275 | 296 |
276 int AudioProcessingImpl::Initialize() { | 297 int AudioProcessingImpl::Initialize() { |
277 CriticalSectionScoped crit_scoped(crit_); | 298 // Run in a single-threaded manner during initialization. |
| 299 rtc::CritScope cs_render(&crit_render_); |
| 300 rtc::CritScope cs_capture(&crit_capture_); |
278 return InitializeLocked(); | 301 return InitializeLocked(); |
279 } | 302 } |
280 | 303 |
281 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, | 304 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, |
282 int output_sample_rate_hz, | 305 int output_sample_rate_hz, |
283 int reverse_sample_rate_hz, | 306 int reverse_sample_rate_hz, |
284 ChannelLayout input_layout, | 307 ChannelLayout input_layout, |
285 ChannelLayout output_layout, | 308 ChannelLayout output_layout, |
286 ChannelLayout reverse_layout) { | 309 ChannelLayout reverse_layout) { |
287 const ProcessingConfig processing_config = { | 310 const ProcessingConfig processing_config = { |
288 {{input_sample_rate_hz, | 311 {{input_sample_rate_hz, |
289 ChannelsFromLayout(input_layout), | 312 ChannelsFromLayout(input_layout), |
290 LayoutHasKeyboard(input_layout)}, | 313 LayoutHasKeyboard(input_layout)}, |
291 {output_sample_rate_hz, | 314 {output_sample_rate_hz, |
292 ChannelsFromLayout(output_layout), | 315 ChannelsFromLayout(output_layout), |
293 LayoutHasKeyboard(output_layout)}, | 316 LayoutHasKeyboard(output_layout)}, |
294 {reverse_sample_rate_hz, | 317 {reverse_sample_rate_hz, |
295 ChannelsFromLayout(reverse_layout), | 318 ChannelsFromLayout(reverse_layout), |
296 LayoutHasKeyboard(reverse_layout)}, | 319 LayoutHasKeyboard(reverse_layout)}, |
297 {reverse_sample_rate_hz, | 320 {reverse_sample_rate_hz, |
298 ChannelsFromLayout(reverse_layout), | 321 ChannelsFromLayout(reverse_layout), |
299 LayoutHasKeyboard(reverse_layout)}}}; | 322 LayoutHasKeyboard(reverse_layout)}}}; |
300 | 323 |
301 return Initialize(processing_config); | 324 return Initialize(processing_config); |
302 } | 325 } |
303 | 326 |
304 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { | 327 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { |
305 CriticalSectionScoped crit_scoped(crit_); | 328 // Run in a single-threaded manner during initialization. |
| 329 rtc::CritScope cs_render(&crit_render_); |
| 330 rtc::CritScope cs_capture(&crit_capture_); |
306 return InitializeLocked(processing_config); | 331 return InitializeLocked(processing_config); |
307 } | 332 } |
308 | 333 |
309 int AudioProcessingImpl::MaybeInitializeLockedRender( | 334 int AudioProcessingImpl::MaybeInitializeRender( |
310 const ProcessingConfig& processing_config) { | 335 const ProcessingConfig& processing_config) { |
311 return MaybeInitializeLocked(processing_config); | 336 return MaybeInitialize(processing_config); |
312 } | 337 } |
313 | 338 |
314 int AudioProcessingImpl::MaybeInitializeLockedCapture( | 339 int AudioProcessingImpl::MaybeInitializeCapture( |
315 const ProcessingConfig& processing_config) { | 340 const ProcessingConfig& processing_config) { |
316 return MaybeInitializeLocked(processing_config); | 341 return MaybeInitialize(processing_config); |
317 } | 342 } |
318 | 343 |
319 // Calls InitializeLocked() if any of the audio parameters have changed from | 344 // Calls InitializeLocked() if any of the audio parameters have changed from |
320 // their current values. | 345 // their current values (needs to be called while holding the crit_render_lock). |
321 int AudioProcessingImpl::MaybeInitializeLocked( | 346 int AudioProcessingImpl::MaybeInitialize( |
322 const ProcessingConfig& processing_config) { | 347 const ProcessingConfig& processing_config) { |
323 if (processing_config == shared_state_.api_format_) { | 348 // Called from both threads. Thread check is therefore not possible. |
| 349 if (processing_config == formats_.api_format) { |
324 return kNoError; | 350 return kNoError; |
325 } | 351 } |
| 352 |
| 353 rtc::CritScope cs_capture(&crit_capture_); |
326 return InitializeLocked(processing_config); | 354 return InitializeLocked(processing_config); |
327 } | 355 } |
328 | 356 |
329 int AudioProcessingImpl::InitializeLocked() { | 357 int AudioProcessingImpl::InitializeLocked() { |
330 const int fwd_audio_buffer_channels = | 358 const int fwd_audio_buffer_channels = |
331 beamformer_enabled_ | 359 constants_.beamformer_enabled |
332 ? shared_state_.api_format_.input_stream().num_channels() | 360 ? formats_.api_format.input_stream().num_channels() |
333 : shared_state_.api_format_.output_stream().num_channels(); | 361 : formats_.api_format.output_stream().num_channels(); |
334 const int rev_audio_buffer_out_num_frames = | 362 const int rev_audio_buffer_out_num_frames = |
335 shared_state_.api_format_.reverse_output_stream().num_frames() == 0 | 363 formats_.api_format.reverse_output_stream().num_frames() == 0 |
336 ? rev_proc_format_.num_frames() | 364 ? formats_.rev_proc_format.num_frames() |
337 : shared_state_.api_format_.reverse_output_stream().num_frames(); | 365 : formats_.api_format.reverse_output_stream().num_frames(); |
338 if (shared_state_.api_format_.reverse_input_stream().num_channels() > 0) { | 366 if (formats_.api_format.reverse_input_stream().num_channels() > 0) { |
339 render_audio_.reset(new AudioBuffer( | 367 render_.render_audio.reset(new AudioBuffer( |
340 shared_state_.api_format_.reverse_input_stream().num_frames(), | 368 formats_.api_format.reverse_input_stream().num_frames(), |
341 shared_state_.api_format_.reverse_input_stream().num_channels(), | 369 formats_.api_format.reverse_input_stream().num_channels(), |
342 rev_proc_format_.num_frames(), rev_proc_format_.num_channels(), | 370 formats_.rev_proc_format.num_frames(), |
| 371 formats_.rev_proc_format.num_channels(), |
343 rev_audio_buffer_out_num_frames)); | 372 rev_audio_buffer_out_num_frames)); |
344 if (rev_conversion_needed()) { | 373 if (rev_conversion_needed()) { |
345 render_converter_ = AudioConverter::Create( | 374 render_.render_converter = AudioConverter::Create( |
346 shared_state_.api_format_.reverse_input_stream().num_channels(), | 375 formats_.api_format.reverse_input_stream().num_channels(), |
347 shared_state_.api_format_.reverse_input_stream().num_frames(), | 376 formats_.api_format.reverse_input_stream().num_frames(), |
348 shared_state_.api_format_.reverse_output_stream().num_channels(), | 377 formats_.api_format.reverse_output_stream().num_channels(), |
349 shared_state_.api_format_.reverse_output_stream().num_frames()); | 378 formats_.api_format.reverse_output_stream().num_frames()); |
350 } else { | 379 } else { |
351 render_converter_.reset(nullptr); | 380 render_.render_converter.reset(nullptr); |
352 } | 381 } |
353 } else { | 382 } else { |
354 render_audio_.reset(nullptr); | 383 render_.render_audio.reset(nullptr); |
355 render_converter_.reset(nullptr); | 384 render_.render_converter.reset(nullptr); |
356 } | 385 } |
357 capture_audio_.reset( | 386 capture_.capture_audio.reset( |
358 new AudioBuffer(shared_state_.api_format_.input_stream().num_frames(), | 387 new AudioBuffer(formats_.api_format.input_stream().num_frames(), |
359 shared_state_.api_format_.input_stream().num_channels(), | 388 formats_.api_format.input_stream().num_channels(), |
360 fwd_proc_format_.num_frames(), fwd_audio_buffer_channels, | 389 capture_nonlocked_.fwd_proc_format.num_frames(), |
361 shared_state_.api_format_.output_stream().num_frames())); | 390 fwd_audio_buffer_channels, |
| 391 formats_.api_format.output_stream().num_frames())); |
362 | 392 |
363 // Initialize all components. | 393 // Initialize all components. |
364 for (auto item : component_list_) { | 394 for (auto item : private_submodules_->component_list) { |
365 int err = item->Initialize(); | 395 int err = item->Initialize(); |
366 if (err != kNoError) { | 396 if (err != kNoError) { |
367 return err; | 397 return err; |
368 } | 398 } |
369 } | 399 } |
370 | 400 |
371 InitializeExperimentalAgc(); | 401 InitializeExperimentalAgc(); |
372 | 402 |
373 InitializeTransient(); | 403 InitializeTransient(); |
374 | 404 |
375 InitializeBeamformer(); | 405 InitializeBeamformer(); |
376 | 406 |
377 InitializeIntelligibility(); | 407 InitializeIntelligibility(); |
378 | 408 |
379 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 409 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
380 if (debug_file_->Open()) { | 410 if (debug_dump_.debug_file->Open()) { |
381 int err = WriteInitMessage(); | 411 int err = WriteInitMessage(); |
382 if (err != kNoError) { | 412 if (err != kNoError) { |
383 return err; | 413 return err; |
384 } | 414 } |
385 } | 415 } |
386 #endif | 416 #endif |
387 | 417 |
388 return kNoError; | 418 return kNoError; |
389 } | 419 } |
390 | 420 |
391 int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { | 421 int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { |
392 // TODO(peah): Refactor to be allowed to verify using thread annotations. | |
393 for (const auto& stream : config.streams) { | 422 for (const auto& stream : config.streams) { |
394 if (stream.num_channels() < 0) { | 423 if (stream.num_channels() < 0) { |
395 return kBadNumberChannelsError; | 424 return kBadNumberChannelsError; |
396 } | 425 } |
397 if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) { | 426 if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) { |
398 return kBadSampleRateError; | 427 return kBadSampleRateError; |
399 } | 428 } |
400 } | 429 } |
401 | 430 |
402 const int num_in_channels = config.input_stream().num_channels(); | 431 const int num_in_channels = config.input_stream().num_channels(); |
403 const int num_out_channels = config.output_stream().num_channels(); | 432 const int num_out_channels = config.output_stream().num_channels(); |
404 | 433 |
405 // Need at least one input channel. | 434 // Need at least one input channel. |
406 // Need either one output channel or as many outputs as there are inputs. | 435 // Need either one output channel or as many outputs as there are inputs. |
407 if (num_in_channels == 0 || | 436 if (num_in_channels == 0 || |
408 !(num_out_channels == 1 || num_out_channels == num_in_channels)) { | 437 !(num_out_channels == 1 || num_out_channels == num_in_channels)) { |
409 return kBadNumberChannelsError; | 438 return kBadNumberChannelsError; |
410 } | 439 } |
411 | 440 |
412 if (beamformer_enabled_ && | 441 if (constants_.beamformer_enabled && (static_cast<size_t>(num_in_channels) != |
413 (static_cast<size_t>(num_in_channels) != array_geometry_.size() || | 442 constants_.array_geometry.size() || |
414 num_out_channels > 1)) { | 443 num_out_channels > 1)) { |
415 return kBadNumberChannelsError; | 444 return kBadNumberChannelsError; |
416 } | 445 } |
417 | 446 |
418 shared_state_.api_format_ = config; | 447 formats_.api_format = config; |
419 | 448 |
420 // We process at the closest native rate >= min(input rate, output rate)... | 449 // We process at the closest native rate >= min(input rate, output rate)... |
421 const int min_proc_rate = | 450 const int min_proc_rate = |
422 std::min(shared_state_.api_format_.input_stream().sample_rate_hz(), | 451 std::min(formats_.api_format.input_stream().sample_rate_hz(), |
423 shared_state_.api_format_.output_stream().sample_rate_hz()); | 452 formats_.api_format.output_stream().sample_rate_hz()); |
424 int fwd_proc_rate; | 453 int fwd_proc_rate; |
425 for (size_t i = 0; i < kNumNativeSampleRates; ++i) { | 454 for (size_t i = 0; i < kNumNativeSampleRates; ++i) { |
426 fwd_proc_rate = kNativeSampleRatesHz[i]; | 455 fwd_proc_rate = kNativeSampleRatesHz[i]; |
427 if (fwd_proc_rate >= min_proc_rate) { | 456 if (fwd_proc_rate >= min_proc_rate) { |
428 break; | 457 break; |
429 } | 458 } |
430 } | 459 } |
431 // ...with one exception. | 460 // ...with one exception. |
432 if (echo_control_mobile_->is_enabled() && | 461 if (public_submodules_->echo_control_mobile->is_enabled() && |
433 min_proc_rate > kMaxAECMSampleRateHz) { | 462 min_proc_rate > kMaxAECMSampleRateHz) { |
434 fwd_proc_rate = kMaxAECMSampleRateHz; | 463 fwd_proc_rate = kMaxAECMSampleRateHz; |
435 } | 464 } |
436 | 465 |
437 fwd_proc_format_ = StreamConfig(fwd_proc_rate); | 466 capture_nonlocked_.fwd_proc_format = StreamConfig(fwd_proc_rate); |
438 | 467 |
439 // We normally process the reverse stream at 16 kHz. Unless... | 468 // We normally process the reverse stream at 16 kHz. Unless... |
440 int rev_proc_rate = kSampleRate16kHz; | 469 int rev_proc_rate = kSampleRate16kHz; |
441 if (fwd_proc_format_.sample_rate_hz() == kSampleRate8kHz) { | 470 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate8kHz) { |
442 // ...the forward stream is at 8 kHz. | 471 // ...the forward stream is at 8 kHz. |
443 rev_proc_rate = kSampleRate8kHz; | 472 rev_proc_rate = kSampleRate8kHz; |
444 } else { | 473 } else { |
445 if (shared_state_.api_format_.reverse_input_stream().sample_rate_hz() == | 474 if (formats_.api_format.reverse_input_stream().sample_rate_hz() == |
446 kSampleRate32kHz) { | 475 kSampleRate32kHz) { |
447 // ...or the input is at 32 kHz, in which case we use the splitting | 476 // ...or the input is at 32 kHz, in which case we use the splitting |
448 // filter rather than the resampler. | 477 // filter rather than the resampler. |
449 rev_proc_rate = kSampleRate32kHz; | 478 rev_proc_rate = kSampleRate32kHz; |
450 } | 479 } |
451 } | 480 } |
452 | 481 |
453 // Always downmix the reverse stream to mono for analysis. This has been | 482 // Always downmix the reverse stream to mono for analysis. This has been |
454 // demonstrated to work well for AEC in most practical scenarios. | 483 // demonstrated to work well for AEC in most practical scenarios. |
455 rev_proc_format_ = StreamConfig(rev_proc_rate, 1); | 484 formats_.rev_proc_format = StreamConfig(rev_proc_rate, 1); |
456 | 485 |
457 if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || | 486 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate32kHz || |
458 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { | 487 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate48kHz) { |
459 split_rate_ = kSampleRate16kHz; | 488 capture_nonlocked_.split_rate = kSampleRate16kHz; |
460 } else { | 489 } else { |
461 split_rate_ = fwd_proc_format_.sample_rate_hz(); | 490 capture_nonlocked_.split_rate = |
| 491 capture_nonlocked_.fwd_proc_format.sample_rate_hz(); |
462 } | 492 } |
463 | 493 |
464 return InitializeLocked(); | 494 return InitializeLocked(); |
465 } | 495 } |
466 | 496 |
467 void AudioProcessingImpl::SetExtraOptions(const Config& config) { | 497 void AudioProcessingImpl::SetExtraOptions(const Config& config) { |
468 CriticalSectionScoped crit_scoped(crit_); | 498 // Run in a single-threaded manner when setting the extra options. |
469 for (auto item : component_list_) { | 499 rtc::CritScope cs_render(&crit_render_); |
| 500 rtc::CritScope cs_capture(&crit_capture_); |
| 501 for (auto item : private_submodules_->component_list) { |
470 item->SetExtraOptions(config); | 502 item->SetExtraOptions(config); |
471 } | 503 } |
472 | 504 |
473 if (transient_suppressor_enabled_ != config.Get<ExperimentalNs>().enabled) { | 505 if (capture_.transient_suppressor_enabled != |
474 transient_suppressor_enabled_ = config.Get<ExperimentalNs>().enabled; | 506 config.Get<ExperimentalNs>().enabled) { |
| 507 capture_.transient_suppressor_enabled = |
| 508 config.Get<ExperimentalNs>().enabled; |
475 InitializeTransient(); | 509 InitializeTransient(); |
476 } | 510 } |
477 } | 511 } |
478 | 512 |
479 | |
480 int AudioProcessingImpl::proc_sample_rate_hz() const { | 513 int AudioProcessingImpl::proc_sample_rate_hz() const { |
481 // TODO(peah): Refactor to be allowed to verify using thread annotations. | 514 // Used as callback from submodules, hence locking is not allowed. |
482 return fwd_proc_format_.sample_rate_hz(); | 515 return capture_nonlocked_.fwd_proc_format.sample_rate_hz(); |
483 } | 516 } |
484 | 517 |
485 int AudioProcessingImpl::proc_split_sample_rate_hz() const { | 518 int AudioProcessingImpl::proc_split_sample_rate_hz() const { |
486 // TODO(peah): Refactor to be allowed to verify using thread annotations. | 519 // Used as callback from submodules, hence locking is not allowed. |
487 | 520 return capture_nonlocked_.split_rate; |
488 return split_rate_; | |
489 } | 521 } |
490 | 522 |
491 int AudioProcessingImpl::num_reverse_channels() const { | 523 int AudioProcessingImpl::num_reverse_channels() const { |
492 // TODO(peah): Refactor to be allowed to verify using thread annotations. | 524 // Used as callback from submodules, hence locking is not allowed. |
493 return rev_proc_format_.num_channels(); | 525 return formats_.rev_proc_format.num_channels(); |
494 } | 526 } |
495 | 527 |
496 int AudioProcessingImpl::num_input_channels() const { | 528 int AudioProcessingImpl::num_input_channels() const { |
497 return shared_state_.api_format_.input_stream().num_channels(); | 529 // Used as callback from submodules, hence locking is not allowed. |
| 530 return formats_.api_format.input_stream().num_channels(); |
498 } | 531 } |
499 | 532 |
500 int AudioProcessingImpl::num_output_channels() const { | 533 int AudioProcessingImpl::num_output_channels() const { |
501 // TODO(peah): Refactor to be allowed to verify using thread annotations. | 534 // Used as callback from submodules, hence locking is not allowed. |
502 return shared_state_.api_format_.output_stream().num_channels(); | 535 return formats_.api_format.output_stream().num_channels(); |
503 } | 536 } |
504 | 537 |
505 void AudioProcessingImpl::set_output_will_be_muted(bool muted) { | 538 void AudioProcessingImpl::set_output_will_be_muted(bool muted) { |
506 CriticalSectionScoped lock(crit_); | 539 rtc::CritScope cs(&crit_capture_); |
507 output_will_be_muted_ = muted; | 540 capture_.output_will_be_muted = muted; |
508 if (agc_manager_.get()) { | 541 if (private_submodules_->agc_manager.get()) { |
509 agc_manager_->SetCaptureMuted(output_will_be_muted_); | 542 private_submodules_->agc_manager->SetCaptureMuted( |
| 543 capture_.output_will_be_muted); |
510 } | 544 } |
511 } | 545 } |
512 | 546 |
513 | 547 |
514 int AudioProcessingImpl::ProcessStream(const float* const* src, | 548 int AudioProcessingImpl::ProcessStream(const float* const* src, |
515 size_t samples_per_channel, | 549 size_t samples_per_channel, |
516 int input_sample_rate_hz, | 550 int input_sample_rate_hz, |
517 ChannelLayout input_layout, | 551 ChannelLayout input_layout, |
518 int output_sample_rate_hz, | 552 int output_sample_rate_hz, |
519 ChannelLayout output_layout, | 553 ChannelLayout output_layout, |
520 float* const* dest) { | 554 float* const* dest) { |
521 CriticalSectionScoped crit_scoped(crit_); | 555 StreamConfig input_stream; |
522 StreamConfig input_stream = shared_state_.api_format_.input_stream(); | 556 StreamConfig output_stream; |
| 557 { |
| 558 // Access the formats_.api_format.input_stream beneath the capture lock. |
| 559 // The lock must be released as it is later required in the call |
| 560 // to ProcessStream(,,,); |
| 561 rtc::CritScope cs(&crit_capture_); |
| 562 input_stream = formats_.api_format.input_stream(); |
| 563 output_stream = formats_.api_format.output_stream(); |
| 564 } |
| 565 |
523 input_stream.set_sample_rate_hz(input_sample_rate_hz); | 566 input_stream.set_sample_rate_hz(input_sample_rate_hz); |
524 input_stream.set_num_channels(ChannelsFromLayout(input_layout)); | 567 input_stream.set_num_channels(ChannelsFromLayout(input_layout)); |
525 input_stream.set_has_keyboard(LayoutHasKeyboard(input_layout)); | 568 input_stream.set_has_keyboard(LayoutHasKeyboard(input_layout)); |
526 | |
527 StreamConfig output_stream = shared_state_.api_format_.output_stream(); | |
528 output_stream.set_sample_rate_hz(output_sample_rate_hz); | 569 output_stream.set_sample_rate_hz(output_sample_rate_hz); |
529 output_stream.set_num_channels(ChannelsFromLayout(output_layout)); | 570 output_stream.set_num_channels(ChannelsFromLayout(output_layout)); |
530 output_stream.set_has_keyboard(LayoutHasKeyboard(output_layout)); | 571 output_stream.set_has_keyboard(LayoutHasKeyboard(output_layout)); |
531 | 572 |
532 if (samples_per_channel != input_stream.num_frames()) { | 573 if (samples_per_channel != input_stream.num_frames()) { |
533 return kBadDataLengthError; | 574 return kBadDataLengthError; |
534 } | 575 } |
535 return ProcessStream(src, input_stream, output_stream, dest); | 576 return ProcessStream(src, input_stream, output_stream, dest); |
536 } | 577 } |
537 | 578 |
538 int AudioProcessingImpl::ProcessStream(const float* const* src, | 579 int AudioProcessingImpl::ProcessStream(const float* const* src, |
539 const StreamConfig& input_config, | 580 const StreamConfig& input_config, |
540 const StreamConfig& output_config, | 581 const StreamConfig& output_config, |
541 float* const* dest) { | 582 float* const* dest) { |
542 CriticalSectionScoped crit_scoped(crit_); | 583 ProcessingConfig processing_config; |
543 if (!src || !dest) { | 584 { |
544 return kNullPointerError; | 585 // Acquire the capture lock in order to safely call the function |
| 586 // that retrieves the render side data. This function accesses apm |
| 587 // getters that need the capture lock held when being called. |
| 588 rtc::CritScope cs_capture(&crit_capture_); |
| 589 public_submodules_->echo_cancellation->ReadQueuedRenderData(); |
| 590 public_submodules_->echo_control_mobile->ReadQueuedRenderData(); |
| 591 public_submodules_->gain_control->ReadQueuedRenderData(); |
| 592 |
| 593 if (!src || !dest) { |
| 594 return kNullPointerError; |
| 595 } |
| 596 |
| 597 processing_config = formats_.api_format; |
545 } | 598 } |
546 | 599 |
547 echo_cancellation_->ReadQueuedRenderData(); | |
548 echo_control_mobile_->ReadQueuedRenderData(); | |
549 gain_control_->ReadQueuedRenderData(); | |
550 | |
551 ProcessingConfig processing_config = shared_state_.api_format_; | |
552 processing_config.input_stream() = input_config; | 600 processing_config.input_stream() = input_config; |
553 processing_config.output_stream() = output_config; | 601 processing_config.output_stream() = output_config; |
554 | 602 |
555 RETURN_ON_ERR(MaybeInitializeLockedCapture(processing_config)); | 603 { |
| 604 // Do conditional reinitialization. |
| 605 rtc::CritScope cs_render(&crit_render_); |
| 606 RETURN_ON_ERR(MaybeInitializeCapture(processing_config)); |
| 607 } |
| 608 rtc::CritScope cs_capture(&crit_capture_); |
556 assert(processing_config.input_stream().num_frames() == | 609 assert(processing_config.input_stream().num_frames() == |
557 shared_state_.api_format_.input_stream().num_frames()); | 610 formats_.api_format.input_stream().num_frames()); |
558 | 611 |
559 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 612 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
560 if (debug_file_->Open()) { | 613 if (debug_dump_.debug_file->Open()) { |
561 RETURN_ON_ERR(WriteConfigMessage(false)); | 614 RETURN_ON_ERR(WriteConfigMessage(false)); |
562 | 615 |
563 event_msg_->set_type(audioproc::Event::STREAM); | 616 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); |
564 audioproc::Stream* msg = event_msg_->mutable_stream(); | 617 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
565 const size_t channel_size = | 618 const size_t channel_size = |
566 sizeof(float) * shared_state_.api_format_.input_stream().num_frames(); | 619 sizeof(float) * formats_.api_format.input_stream().num_frames(); |
567 for (int i = 0; i < shared_state_.api_format_.input_stream().num_channels(); | 620 for (int i = 0; i < formats_.api_format.input_stream().num_channels(); ++i) |
568 ++i) | |
569 msg->add_input_channel(src[i], channel_size); | 621 msg->add_input_channel(src[i], channel_size); |
570 } | 622 } |
571 #endif | 623 #endif |
572 | 624 |
573 capture_audio_->CopyFrom(src, shared_state_.api_format_.input_stream()); | 625 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream()); |
574 RETURN_ON_ERR(ProcessStreamLocked()); | 626 RETURN_ON_ERR(ProcessStreamLocked()); |
575 capture_audio_->CopyTo(shared_state_.api_format_.output_stream(), dest); | 627 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest); |
576 | 628 |
577 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 629 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
578 if (debug_file_->Open()) { | 630 if (debug_dump_.debug_file->Open()) { |
579 audioproc::Stream* msg = event_msg_->mutable_stream(); | 631 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
580 const size_t channel_size = | 632 const size_t channel_size = |
581 sizeof(float) * shared_state_.api_format_.output_stream().num_frames(); | 633 sizeof(float) * formats_.api_format.output_stream().num_frames(); |
582 for (int i = 0; | 634 for (int i = 0; i < formats_.api_format.output_stream().num_channels(); ++i) |
583 i < shared_state_.api_format_.output_stream().num_channels(); ++i) | |
584 msg->add_output_channel(dest[i], channel_size); | 635 msg->add_output_channel(dest[i], channel_size); |
585 RETURN_ON_ERR(WriteMessageToDebugFile()); | 636 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 637 &crit_debug_, &debug_dump_.capture)); |
586 } | 638 } |
587 #endif | 639 #endif |
588 | 640 |
589 return kNoError; | 641 return kNoError; |
590 } | 642 } |
591 | 643 |
592 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { | 644 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { |
593 CriticalSectionScoped crit_scoped(crit_); | 645 { |
594 echo_cancellation_->ReadQueuedRenderData(); | 646 // Acquire the capture lock in order to safely call the function |
595 echo_control_mobile_->ReadQueuedRenderData(); | 647 // that retrieves the render side data. This function accesses apm |
596 gain_control_->ReadQueuedRenderData(); | 648 // getters that need the capture lock held when being called. |
| 649 // The lock needs to be released as |
| 650 // public_submodules_->echo_control_mobile->is_enabled() aquires this lock |
| 651 // as well. |
| 652 rtc::CritScope cs_capture(&crit_capture_); |
| 653 public_submodules_->echo_cancellation->ReadQueuedRenderData(); |
| 654 public_submodules_->echo_control_mobile->ReadQueuedRenderData(); |
| 655 public_submodules_->gain_control->ReadQueuedRenderData(); |
| 656 } |
597 | 657 |
598 if (!frame) { | 658 if (!frame) { |
599 return kNullPointerError; | 659 return kNullPointerError; |
600 } | 660 } |
601 // Must be a native rate. | 661 // Must be a native rate. |
602 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 662 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
603 frame->sample_rate_hz_ != kSampleRate16kHz && | 663 frame->sample_rate_hz_ != kSampleRate16kHz && |
604 frame->sample_rate_hz_ != kSampleRate32kHz && | 664 frame->sample_rate_hz_ != kSampleRate32kHz && |
605 frame->sample_rate_hz_ != kSampleRate48kHz) { | 665 frame->sample_rate_hz_ != kSampleRate48kHz) { |
606 return kBadSampleRateError; | 666 return kBadSampleRateError; |
607 } | 667 } |
608 | 668 |
609 if (echo_control_mobile_->is_enabled() && | 669 if (public_submodules_->echo_control_mobile->is_enabled() && |
610 frame->sample_rate_hz_ > kMaxAECMSampleRateHz) { | 670 frame->sample_rate_hz_ > kMaxAECMSampleRateHz) { |
611 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates"; | 671 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates"; |
612 return kUnsupportedComponentError; | 672 return kUnsupportedComponentError; |
613 } | 673 } |
614 | 674 |
615 // TODO(ajm): The input and output rates and channels are currently | 675 ProcessingConfig processing_config; |
616 // constrained to be identical in the int16 interface. | 676 { |
617 ProcessingConfig processing_config = shared_state_.api_format_; | 677 // Aquire lock for the access of api_format. |
| 678 // The lock is released immediately due to the conditional |
| 679 // reinitialization. |
| 680 rtc::CritScope cs_capture(&crit_capture_); |
| 681 // TODO(ajm): The input and output rates and channels are currently |
| 682 // constrained to be identical in the int16 interface. |
| 683 processing_config = formats_.api_format; |
| 684 } |
618 processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_); | 685 processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_); |
619 processing_config.input_stream().set_num_channels(frame->num_channels_); | 686 processing_config.input_stream().set_num_channels(frame->num_channels_); |
620 processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_); | 687 processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_); |
621 processing_config.output_stream().set_num_channels(frame->num_channels_); | 688 processing_config.output_stream().set_num_channels(frame->num_channels_); |
622 | 689 |
623 RETURN_ON_ERR(MaybeInitializeLockedCapture(processing_config)); | 690 { |
| 691 // Do conditional reinitialization. |
| 692 rtc::CritScope cs_render(&crit_render_); |
| 693 RETURN_ON_ERR(MaybeInitializeCapture(processing_config)); |
| 694 } |
| 695 rtc::CritScope cs_capture(&crit_capture_); |
624 if (frame->samples_per_channel_ != | 696 if (frame->samples_per_channel_ != |
625 shared_state_.api_format_.input_stream().num_frames()) { | 697 formats_.api_format.input_stream().num_frames()) { |
626 return kBadDataLengthError; | 698 return kBadDataLengthError; |
627 } | 699 } |
628 | 700 |
629 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 701 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
630 if (debug_file_->Open()) { | 702 if (debug_dump_.debug_file->Open()) { |
631 event_msg_->set_type(audioproc::Event::STREAM); | 703 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM); |
632 audioproc::Stream* msg = event_msg_->mutable_stream(); | 704 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
633 const size_t data_size = | 705 const size_t data_size = |
634 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 706 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
635 msg->set_input_data(frame->data_, data_size); | 707 msg->set_input_data(frame->data_, data_size); |
636 } | 708 } |
637 #endif | 709 #endif |
638 | 710 |
639 capture_audio_->DeinterleaveFrom(frame); | 711 capture_.capture_audio->DeinterleaveFrom(frame); |
640 RETURN_ON_ERR(ProcessStreamLocked()); | 712 RETURN_ON_ERR(ProcessStreamLocked()); |
641 capture_audio_->InterleaveTo(frame, output_copy_needed(is_data_processed())); | 713 capture_.capture_audio->InterleaveTo(frame, |
| 714 output_copy_needed(is_data_processed())); |
642 | 715 |
643 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 716 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
644 if (debug_file_->Open()) { | 717 if (debug_dump_.debug_file->Open()) { |
645 audioproc::Stream* msg = event_msg_->mutable_stream(); | 718 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
646 const size_t data_size = | 719 const size_t data_size = |
647 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 720 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
648 msg->set_output_data(frame->data_, data_size); | 721 msg->set_output_data(frame->data_, data_size); |
649 RETURN_ON_ERR(WriteMessageToDebugFile()); | 722 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 723 &crit_debug_, &debug_dump_.capture)); |
650 } | 724 } |
651 #endif | 725 #endif |
652 | 726 |
653 return kNoError; | 727 return kNoError; |
654 } | 728 } |
655 | 729 |
656 int AudioProcessingImpl::ProcessStreamLocked() { | 730 int AudioProcessingImpl::ProcessStreamLocked() { |
657 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 731 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
658 if (debug_file_->Open()) { | 732 if (debug_dump_.debug_file->Open()) { |
659 audioproc::Stream* msg = event_msg_->mutable_stream(); | 733 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream(); |
660 msg->set_delay(stream_delay_ms_); | 734 msg->set_delay(capture_nonlocked_.stream_delay_ms); |
661 msg->set_drift(echo_cancellation_->stream_drift_samples()); | 735 msg->set_drift( |
| 736 public_submodules_->echo_cancellation->stream_drift_samples()); |
662 msg->set_level(gain_control()->stream_analog_level()); | 737 msg->set_level(gain_control()->stream_analog_level()); |
663 msg->set_keypress(key_pressed_); | 738 msg->set_keypress(capture_.key_pressed); |
664 } | 739 } |
665 #endif | 740 #endif |
666 | 741 |
667 MaybeUpdateHistograms(); | 742 MaybeUpdateHistograms(); |
668 | 743 |
669 AudioBuffer* ca = capture_audio_.get(); // For brevity. | 744 AudioBuffer* ca = capture_.capture_audio.get(); // For brevity. |
670 | 745 |
671 if (use_new_agc_ && gain_control_->is_enabled()) { | 746 if (constants_.use_new_agc && |
672 agc_manager_->AnalyzePreProcess(ca->channels()[0], ca->num_channels(), | 747 public_submodules_->gain_control->is_enabled()) { |
673 fwd_proc_format_.num_frames()); | 748 private_submodules_->agc_manager->AnalyzePreProcess( |
| 749 ca->channels()[0], ca->num_channels(), |
| 750 capture_nonlocked_.fwd_proc_format.num_frames()); |
674 } | 751 } |
675 | 752 |
676 bool data_processed = is_data_processed(); | 753 bool data_processed = is_data_processed(); |
677 if (analysis_needed(data_processed)) { | 754 if (analysis_needed(data_processed)) { |
678 ca->SplitIntoFrequencyBands(); | 755 ca->SplitIntoFrequencyBands(); |
679 } | 756 } |
680 | 757 |
681 if (intelligibility_enabled_) { | 758 if (constants_.intelligibility_enabled) { |
682 intelligibility_enhancer_->AnalyzeCaptureAudio( | 759 public_submodules_->intelligibility_enhancer->AnalyzeCaptureAudio( |
683 ca->split_channels_f(kBand0To8kHz), split_rate_, ca->num_channels()); | 760 ca->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate, |
| 761 ca->num_channels()); |
684 } | 762 } |
685 | 763 |
686 if (beamformer_enabled_) { | 764 if (constants_.beamformer_enabled) { |
687 beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f()); | 765 private_submodules_->beamformer->ProcessChunk(*ca->split_data_f(), |
| 766 ca->split_data_f()); |
688 ca->set_num_channels(1); | 767 ca->set_num_channels(1); |
689 } | 768 } |
690 | 769 |
691 RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca)); | 770 RETURN_ON_ERR(public_submodules_->high_pass_filter->ProcessCaptureAudio(ca)); |
692 RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca)); | 771 RETURN_ON_ERR(public_submodules_->gain_control->AnalyzeCaptureAudio(ca)); |
693 RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca)); | 772 RETURN_ON_ERR(public_submodules_->noise_suppression->AnalyzeCaptureAudio(ca)); |
694 RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca)); | 773 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio(ca)); |
695 | 774 |
696 if (echo_control_mobile_->is_enabled() && noise_suppression_->is_enabled()) { | 775 if (public_submodules_->echo_control_mobile->is_enabled() && |
| 776 public_submodules_->noise_suppression->is_enabled()) { |
697 ca->CopyLowPassToReference(); | 777 ca->CopyLowPassToReference(); |
698 } | 778 } |
699 RETURN_ON_ERR(noise_suppression_->ProcessCaptureAudio(ca)); | 779 RETURN_ON_ERR(public_submodules_->noise_suppression->ProcessCaptureAudio(ca)); |
700 RETURN_ON_ERR(echo_control_mobile_->ProcessCaptureAudio(ca)); | 780 RETURN_ON_ERR( |
701 RETURN_ON_ERR(voice_detection_->ProcessCaptureAudio(ca)); | 781 public_submodules_->echo_control_mobile->ProcessCaptureAudio(ca)); |
| 782 RETURN_ON_ERR(public_submodules_->voice_detection->ProcessCaptureAudio(ca)); |
702 | 783 |
703 if (use_new_agc_ && gain_control_->is_enabled() && | 784 if (constants_.use_new_agc && |
704 (!beamformer_enabled_ || beamformer_->is_target_present())) { | 785 public_submodules_->gain_control->is_enabled() && |
705 agc_manager_->Process(ca->split_bands_const(0)[kBand0To8kHz], | 786 (!constants_.beamformer_enabled || |
706 ca->num_frames_per_band(), split_rate_); | 787 private_submodules_->beamformer->is_target_present())) { |
| 788 private_submodules_->agc_manager->Process( |
| 789 ca->split_bands_const(0)[kBand0To8kHz], ca->num_frames_per_band(), |
| 790 capture_nonlocked_.split_rate); |
707 } | 791 } |
708 RETURN_ON_ERR(gain_control_->ProcessCaptureAudio(ca)); | 792 RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio(ca)); |
709 | 793 |
710 if (synthesis_needed(data_processed)) { | 794 if (synthesis_needed(data_processed)) { |
711 ca->MergeFrequencyBands(); | 795 ca->MergeFrequencyBands(); |
712 } | 796 } |
713 | 797 |
714 // TODO(aluebs): Investigate if the transient suppression placement should be | 798 // TODO(aluebs): Investigate if the transient suppression placement should be |
715 // before or after the AGC. | 799 // before or after the AGC. |
716 if (transient_suppressor_enabled_) { | 800 if (capture_.transient_suppressor_enabled) { |
717 float voice_probability = | 801 float voice_probability = |
718 agc_manager_.get() ? agc_manager_->voice_probability() : 1.f; | 802 private_submodules_->agc_manager.get() |
| 803 ? private_submodules_->agc_manager->voice_probability() |
| 804 : 1.f; |
719 | 805 |
720 transient_suppressor_->Suppress( | 806 public_submodules_->transient_suppressor->Suppress( |
721 ca->channels_f()[0], ca->num_frames(), ca->num_channels(), | 807 ca->channels_f()[0], ca->num_frames(), ca->num_channels(), |
722 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(), | 808 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(), |
723 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability, | 809 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability, |
724 key_pressed_); | 810 capture_.key_pressed); |
725 } | 811 } |
726 | 812 |
727 // The level estimator operates on the recombined data. | 813 // The level estimator operates on the recombined data. |
728 RETURN_ON_ERR(level_estimator_->ProcessStream(ca)); | 814 RETURN_ON_ERR(public_submodules_->level_estimator->ProcessStream(ca)); |
729 | 815 |
730 was_stream_delay_set_ = false; | 816 capture_.was_stream_delay_set = false; |
731 return kNoError; | 817 return kNoError; |
732 } | 818 } |
733 | 819 |
734 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, | 820 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, |
735 size_t samples_per_channel, | 821 size_t samples_per_channel, |
736 int rev_sample_rate_hz, | 822 int rev_sample_rate_hz, |
737 ChannelLayout layout) { | 823 ChannelLayout layout) { |
| 824 rtc::CritScope cs(&crit_render_); |
738 const StreamConfig reverse_config = { | 825 const StreamConfig reverse_config = { |
739 rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), | 826 rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), |
740 }; | 827 }; |
741 if (samples_per_channel != reverse_config.num_frames()) { | 828 if (samples_per_channel != reverse_config.num_frames()) { |
742 return kBadDataLengthError; | 829 return kBadDataLengthError; |
743 } | 830 } |
744 return AnalyzeReverseStream(data, reverse_config, reverse_config); | 831 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config); |
745 } | 832 } |
746 | 833 |
747 int AudioProcessingImpl::ProcessReverseStream( | 834 int AudioProcessingImpl::ProcessReverseStream( |
748 const float* const* src, | 835 const float* const* src, |
749 const StreamConfig& reverse_input_config, | 836 const StreamConfig& reverse_input_config, |
750 const StreamConfig& reverse_output_config, | 837 const StreamConfig& reverse_output_config, |
751 float* const* dest) { | 838 float* const* dest) { |
752 RETURN_ON_ERR( | 839 rtc::CritScope cs(&crit_render_); |
753 AnalyzeReverseStream(src, reverse_input_config, reverse_output_config)); | 840 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, reverse_input_config, |
| 841 reverse_output_config)); |
754 if (is_rev_processed()) { | 842 if (is_rev_processed()) { |
755 render_audio_->CopyTo(shared_state_.api_format_.reverse_output_stream(), | 843 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(), |
756 dest); | 844 dest); |
757 } else if (render_check_rev_conversion_needed()) { | 845 } else if (render_check_rev_conversion_needed()) { |
758 render_converter_->Convert(src, reverse_input_config.num_samples(), dest, | 846 render_.render_converter->Convert(src, reverse_input_config.num_samples(), |
759 reverse_output_config.num_samples()); | 847 dest, |
| 848 reverse_output_config.num_samples()); |
760 } else { | 849 } else { |
761 CopyAudioIfNeeded(src, reverse_input_config.num_frames(), | 850 CopyAudioIfNeeded(src, reverse_input_config.num_frames(), |
762 reverse_input_config.num_channels(), dest); | 851 reverse_input_config.num_channels(), dest); |
763 } | 852 } |
764 | 853 |
765 return kNoError; | 854 return kNoError; |
766 } | 855 } |
767 | 856 |
768 int AudioProcessingImpl::AnalyzeReverseStream( | 857 int AudioProcessingImpl::AnalyzeReverseStreamLocked( |
769 const float* const* src, | 858 const float* const* src, |
770 const StreamConfig& reverse_input_config, | 859 const StreamConfig& reverse_input_config, |
771 const StreamConfig& reverse_output_config) { | 860 const StreamConfig& reverse_output_config) { |
772 CriticalSectionScoped crit_scoped(crit_); | 861 if (src == nullptr) { |
773 if (src == NULL) { | |
774 return kNullPointerError; | 862 return kNullPointerError; |
775 } | 863 } |
776 | 864 |
777 if (reverse_input_config.num_channels() <= 0) { | 865 if (reverse_input_config.num_channels() <= 0) { |
778 return kBadNumberChannelsError; | 866 return kBadNumberChannelsError; |
779 } | 867 } |
780 | 868 |
781 ProcessingConfig processing_config = shared_state_.api_format_; | 869 ProcessingConfig processing_config = formats_.api_format; |
782 processing_config.reverse_input_stream() = reverse_input_config; | 870 processing_config.reverse_input_stream() = reverse_input_config; |
783 processing_config.reverse_output_stream() = reverse_output_config; | 871 processing_config.reverse_output_stream() = reverse_output_config; |
784 | 872 |
785 RETURN_ON_ERR(MaybeInitializeLockedRender(processing_config)); | 873 RETURN_ON_ERR(MaybeInitializeRender(processing_config)); |
786 assert(reverse_input_config.num_frames() == | 874 assert(reverse_input_config.num_frames() == |
787 shared_state_.api_format_.reverse_input_stream().num_frames()); | 875 formats_.api_format.reverse_input_stream().num_frames()); |
788 | 876 |
789 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 877 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
790 if (debug_file_->Open()) { | 878 if (debug_dump_.debug_file->Open()) { |
791 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); | 879 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); |
792 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); | 880 audioproc::ReverseStream* msg = |
| 881 debug_dump_.render.event_msg->mutable_reverse_stream(); |
793 const size_t channel_size = | 882 const size_t channel_size = |
794 sizeof(float) * | 883 sizeof(float) * formats_.api_format.reverse_input_stream().num_frames(); |
795 shared_state_.api_format_.reverse_input_stream().num_frames(); | |
796 for (int i = 0; | 884 for (int i = 0; |
797 i < shared_state_.api_format_.reverse_input_stream().num_channels(); | 885 i < formats_.api_format.reverse_input_stream().num_channels(); ++i) |
798 ++i) | |
799 msg->add_channel(src[i], channel_size); | 886 msg->add_channel(src[i], channel_size); |
800 RETURN_ON_ERR(WriteMessageToDebugFile()); | 887 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 888 &crit_debug_, &debug_dump_.render)); |
801 } | 889 } |
802 #endif | 890 #endif |
803 | 891 |
804 render_audio_->CopyFrom(src, | 892 render_.render_audio->CopyFrom(src, |
805 shared_state_.api_format_.reverse_input_stream()); | 893 formats_.api_format.reverse_input_stream()); |
806 return ProcessReverseStreamLocked(); | 894 return ProcessReverseStreamLocked(); |
807 } | 895 } |
808 | 896 |
809 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { | 897 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { |
810 RETURN_ON_ERR(AnalyzeReverseStream(frame)); | 898 RETURN_ON_ERR(AnalyzeReverseStream(frame)); |
| 899 rtc::CritScope cs(&crit_render_); |
811 if (is_rev_processed()) { | 900 if (is_rev_processed()) { |
812 render_audio_->InterleaveTo(frame, true); | 901 render_.render_audio->InterleaveTo(frame, true); |
813 } | 902 } |
814 | 903 |
815 return kNoError; | 904 return kNoError; |
816 } | 905 } |
817 | 906 |
818 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { | 907 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { |
819 CriticalSectionScoped crit_scoped(crit_); | 908 rtc::CritScope cs(&crit_render_); |
820 if (frame == NULL) { | 909 if (frame == nullptr) { |
821 return kNullPointerError; | 910 return kNullPointerError; |
822 } | 911 } |
823 // Must be a native rate. | 912 // Must be a native rate. |
824 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 913 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
825 frame->sample_rate_hz_ != kSampleRate16kHz && | 914 frame->sample_rate_hz_ != kSampleRate16kHz && |
826 frame->sample_rate_hz_ != kSampleRate32kHz && | 915 frame->sample_rate_hz_ != kSampleRate32kHz && |
827 frame->sample_rate_hz_ != kSampleRate48kHz) { | 916 frame->sample_rate_hz_ != kSampleRate48kHz) { |
828 return kBadSampleRateError; | 917 return kBadSampleRateError; |
829 } | 918 } |
830 // This interface does not tolerate different forward and reverse rates. | 919 // This interface does not tolerate different forward and reverse rates. |
831 if (frame->sample_rate_hz_ != | 920 if (frame->sample_rate_hz_ != |
832 shared_state_.api_format_.input_stream().sample_rate_hz()) { | 921 formats_.api_format.input_stream().sample_rate_hz()) { |
833 return kBadSampleRateError; | 922 return kBadSampleRateError; |
834 } | 923 } |
835 | 924 |
836 if (frame->num_channels_ <= 0) { | 925 if (frame->num_channels_ <= 0) { |
837 return kBadNumberChannelsError; | 926 return kBadNumberChannelsError; |
838 } | 927 } |
839 | 928 |
840 ProcessingConfig processing_config = shared_state_.api_format_; | 929 ProcessingConfig processing_config = formats_.api_format; |
841 processing_config.reverse_input_stream().set_sample_rate_hz( | 930 processing_config.reverse_input_stream().set_sample_rate_hz( |
842 frame->sample_rate_hz_); | 931 frame->sample_rate_hz_); |
843 processing_config.reverse_input_stream().set_num_channels( | 932 processing_config.reverse_input_stream().set_num_channels( |
844 frame->num_channels_); | 933 frame->num_channels_); |
845 processing_config.reverse_output_stream().set_sample_rate_hz( | 934 processing_config.reverse_output_stream().set_sample_rate_hz( |
846 frame->sample_rate_hz_); | 935 frame->sample_rate_hz_); |
847 processing_config.reverse_output_stream().set_num_channels( | 936 processing_config.reverse_output_stream().set_num_channels( |
848 frame->num_channels_); | 937 frame->num_channels_); |
849 | 938 |
850 RETURN_ON_ERR(MaybeInitializeLockedRender(processing_config)); | 939 RETURN_ON_ERR(MaybeInitializeRender(processing_config)); |
851 if (frame->samples_per_channel_ != | 940 if (frame->samples_per_channel_ != |
852 shared_state_.api_format_.reverse_input_stream().num_frames()) { | 941 formats_.api_format.reverse_input_stream().num_frames()) { |
853 return kBadDataLengthError; | 942 return kBadDataLengthError; |
854 } | 943 } |
855 | 944 |
856 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 945 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
857 if (debug_file_->Open()) { | 946 if (debug_dump_.debug_file->Open()) { |
858 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); | 947 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM); |
859 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); | 948 audioproc::ReverseStream* msg = |
| 949 debug_dump_.render.event_msg->mutable_reverse_stream(); |
860 const size_t data_size = | 950 const size_t data_size = |
861 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; | 951 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
862 msg->set_data(frame->data_, data_size); | 952 msg->set_data(frame->data_, data_size); |
863 RETURN_ON_ERR(WriteMessageToDebugFile()); | 953 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 954 &crit_debug_, &debug_dump_.render)); |
864 } | 955 } |
865 #endif | 956 #endif |
866 render_audio_->DeinterleaveFrom(frame); | 957 render_.render_audio->DeinterleaveFrom(frame); |
867 return ProcessReverseStreamLocked(); | 958 return ProcessReverseStreamLocked(); |
868 } | 959 } |
869 | 960 |
870 int AudioProcessingImpl::ProcessReverseStreamLocked() { | 961 int AudioProcessingImpl::ProcessReverseStreamLocked() { |
871 AudioBuffer* ra = render_audio_.get(); // For brevity. | 962 AudioBuffer* ra = render_.render_audio.get(); // For brevity. |
872 if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz) { | 963 if (formats_.rev_proc_format.sample_rate_hz() == kSampleRate32kHz) { |
873 ra->SplitIntoFrequencyBands(); | 964 ra->SplitIntoFrequencyBands(); |
874 } | 965 } |
875 | 966 |
876 if (intelligibility_enabled_) { | 967 if (constants_.intelligibility_enabled) { |
877 intelligibility_enhancer_->ProcessRenderAudio( | 968 // Currently run in single-threaded mode when the intelligibility |
878 ra->split_channels_f(kBand0To8kHz), split_rate_, ra->num_channels()); | 969 // enhancer is activated. |
| 970 // TODO(peah): Fix to be properly multi-threaded. |
| 971 rtc::CritScope cs(&crit_capture_); |
| 972 public_submodules_->intelligibility_enhancer->ProcessRenderAudio( |
| 973 ra->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate, |
| 974 ra->num_channels()); |
879 } | 975 } |
880 | 976 |
881 RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra)); | 977 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessRenderAudio(ra)); |
882 RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra)); | 978 RETURN_ON_ERR( |
883 if (!use_new_agc_) { | 979 public_submodules_->echo_control_mobile->ProcessRenderAudio(ra)); |
884 RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra)); | 980 if (!constants_.use_new_agc) { |
| 981 RETURN_ON_ERR(public_submodules_->gain_control->ProcessRenderAudio(ra)); |
885 } | 982 } |
886 | 983 |
887 if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz && | 984 if (formats_.rev_proc_format.sample_rate_hz() == kSampleRate32kHz && |
888 is_rev_processed()) { | 985 is_rev_processed()) { |
889 ra->MergeFrequencyBands(); | 986 ra->MergeFrequencyBands(); |
890 } | 987 } |
891 | 988 |
892 return kNoError; | 989 return kNoError; |
893 } | 990 } |
894 | 991 |
895 int AudioProcessingImpl::set_stream_delay_ms(int delay) { | 992 int AudioProcessingImpl::set_stream_delay_ms(int delay) { |
| 993 rtc::CritScope cs(&crit_capture_); |
896 Error retval = kNoError; | 994 Error retval = kNoError; |
897 was_stream_delay_set_ = true; | 995 capture_.was_stream_delay_set = true; |
898 delay += delay_offset_ms_; | 996 delay += capture_.delay_offset_ms; |
899 | 997 |
900 if (delay < 0) { | 998 if (delay < 0) { |
901 delay = 0; | 999 delay = 0; |
902 retval = kBadStreamParameterWarning; | 1000 retval = kBadStreamParameterWarning; |
903 } | 1001 } |
904 | 1002 |
905 // TODO(ajm): the max is rather arbitrarily chosen; investigate. | 1003 // TODO(ajm): the max is rather arbitrarily chosen; investigate. |
906 if (delay > 500) { | 1004 if (delay > 500) { |
907 delay = 500; | 1005 delay = 500; |
908 retval = kBadStreamParameterWarning; | 1006 retval = kBadStreamParameterWarning; |
909 } | 1007 } |
910 | 1008 |
911 stream_delay_ms_ = delay; | 1009 capture_nonlocked_.stream_delay_ms = delay; |
912 return retval; | 1010 return retval; |
913 } | 1011 } |
914 | 1012 |
915 int AudioProcessingImpl::stream_delay_ms() const { | 1013 int AudioProcessingImpl::stream_delay_ms() const { |
916 return stream_delay_ms_; | 1014 // Used as callback from submodules, hence locking is not allowed. |
| 1015 return capture_nonlocked_.stream_delay_ms; |
917 } | 1016 } |
918 | 1017 |
919 bool AudioProcessingImpl::was_stream_delay_set() const { | 1018 bool AudioProcessingImpl::was_stream_delay_set() const { |
920 return was_stream_delay_set_; | 1019 // Used as callback from submodules, hence locking is not allowed. |
| 1020 return capture_.was_stream_delay_set; |
921 } | 1021 } |
922 | 1022 |
923 void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) { | 1023 void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) { |
924 key_pressed_ = key_pressed; | 1024 rtc::CritScope cs(&crit_capture_); |
| 1025 capture_.key_pressed = key_pressed; |
925 } | 1026 } |
926 | 1027 |
927 void AudioProcessingImpl::set_delay_offset_ms(int offset) { | 1028 void AudioProcessingImpl::set_delay_offset_ms(int offset) { |
928 CriticalSectionScoped crit_scoped(crit_); | 1029 rtc::CritScope cs(&crit_capture_); |
929 delay_offset_ms_ = offset; | 1030 capture_.delay_offset_ms = offset; |
930 } | 1031 } |
931 | 1032 |
932 int AudioProcessingImpl::delay_offset_ms() const { | 1033 int AudioProcessingImpl::delay_offset_ms() const { |
933 return delay_offset_ms_; | 1034 rtc::CritScope cs(&crit_capture_); |
| 1035 return capture_.delay_offset_ms; |
934 } | 1036 } |
935 | 1037 |
936 int AudioProcessingImpl::StartDebugRecording( | 1038 int AudioProcessingImpl::StartDebugRecording( |
937 const char filename[AudioProcessing::kMaxFilenameSize]) { | 1039 const char filename[AudioProcessing::kMaxFilenameSize]) { |
938 CriticalSectionScoped crit_scoped(crit_); | 1040 // Run in a single-threaded manner. |
| 1041 rtc::CritScope cs_render(&crit_render_); |
| 1042 rtc::CritScope cs_capture(&crit_capture_); |
939 static_assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize, ""); | 1043 static_assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize, ""); |
940 | 1044 |
941 if (filename == NULL) { | 1045 if (filename == nullptr) { |
942 return kNullPointerError; | 1046 return kNullPointerError; |
943 } | 1047 } |
944 | 1048 |
945 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1049 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
946 // Stop any ongoing recording. | 1050 // Stop any ongoing recording. |
947 if (debug_file_->Open()) { | 1051 if (debug_dump_.debug_file->Open()) { |
948 if (debug_file_->CloseFile() == -1) { | 1052 if (debug_dump_.debug_file->CloseFile() == -1) { |
949 return kFileError; | 1053 return kFileError; |
950 } | 1054 } |
951 } | 1055 } |
952 | 1056 |
953 if (debug_file_->OpenFile(filename, false) == -1) { | 1057 if (debug_dump_.debug_file->OpenFile(filename, false) == -1) { |
954 debug_file_->CloseFile(); | 1058 debug_dump_.debug_file->CloseFile(); |
955 return kFileError; | 1059 return kFileError; |
956 } | 1060 } |
957 | 1061 |
958 RETURN_ON_ERR(WriteConfigMessage(true)); | 1062 RETURN_ON_ERR(WriteConfigMessage(true)); |
959 RETURN_ON_ERR(WriteInitMessage()); | 1063 RETURN_ON_ERR(WriteInitMessage()); |
960 return kNoError; | 1064 return kNoError; |
961 #else | 1065 #else |
962 return kUnsupportedFunctionError; | 1066 return kUnsupportedFunctionError; |
963 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1067 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
964 } | 1068 } |
965 | 1069 |
966 int AudioProcessingImpl::StartDebugRecording(FILE* handle) { | 1070 int AudioProcessingImpl::StartDebugRecording(FILE* handle) { |
967 CriticalSectionScoped crit_scoped(crit_); | 1071 // Run in a single-threaded manner. |
| 1072 rtc::CritScope cs_render(&crit_render_); |
| 1073 rtc::CritScope cs_capture(&crit_capture_); |
968 | 1074 |
969 if (handle == NULL) { | 1075 if (handle == nullptr) { |
970 return kNullPointerError; | 1076 return kNullPointerError; |
971 } | 1077 } |
972 | 1078 |
973 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1079 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
974 // Stop any ongoing recording. | 1080 // Stop any ongoing recording. |
975 if (debug_file_->Open()) { | 1081 if (debug_dump_.debug_file->Open()) { |
976 if (debug_file_->CloseFile() == -1) { | 1082 if (debug_dump_.debug_file->CloseFile() == -1) { |
977 return kFileError; | 1083 return kFileError; |
978 } | 1084 } |
979 } | 1085 } |
980 | 1086 |
981 if (debug_file_->OpenFromFileHandle(handle, true, false) == -1) { | 1087 if (debug_dump_.debug_file->OpenFromFileHandle(handle, true, false) == -1) { |
982 return kFileError; | 1088 return kFileError; |
983 } | 1089 } |
984 | 1090 |
985 RETURN_ON_ERR(WriteConfigMessage(true)); | 1091 RETURN_ON_ERR(WriteConfigMessage(true)); |
986 RETURN_ON_ERR(WriteInitMessage()); | 1092 RETURN_ON_ERR(WriteInitMessage()); |
987 return kNoError; | 1093 return kNoError; |
988 #else | 1094 #else |
989 return kUnsupportedFunctionError; | 1095 return kUnsupportedFunctionError; |
990 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1096 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
991 } | 1097 } |
992 | 1098 |
993 int AudioProcessingImpl::StartDebugRecordingForPlatformFile( | 1099 int AudioProcessingImpl::StartDebugRecordingForPlatformFile( |
994 rtc::PlatformFile handle) { | 1100 rtc::PlatformFile handle) { |
| 1101 // Run in a single-threaded manner. |
| 1102 rtc::CritScope cs_render(&crit_render_); |
| 1103 rtc::CritScope cs_capture(&crit_capture_); |
995 FILE* stream = rtc::FdopenPlatformFileForWriting(handle); | 1104 FILE* stream = rtc::FdopenPlatformFileForWriting(handle); |
996 return StartDebugRecording(stream); | 1105 return StartDebugRecording(stream); |
997 } | 1106 } |
998 | 1107 |
999 int AudioProcessingImpl::StopDebugRecording() { | 1108 int AudioProcessingImpl::StopDebugRecording() { |
1000 CriticalSectionScoped crit_scoped(crit_); | 1109 // Run in a single-threaded manner. |
| 1110 rtc::CritScope cs_render(&crit_render_); |
| 1111 rtc::CritScope cs_capture(&crit_capture_); |
1001 | 1112 |
1002 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1113 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
1003 // We just return if recording hasn't started. | 1114 // We just return if recording hasn't started. |
1004 if (debug_file_->Open()) { | 1115 if (debug_dump_.debug_file->Open()) { |
1005 if (debug_file_->CloseFile() == -1) { | 1116 if (debug_dump_.debug_file->CloseFile() == -1) { |
1006 return kFileError; | 1117 return kFileError; |
1007 } | 1118 } |
1008 } | 1119 } |
1009 return kNoError; | 1120 return kNoError; |
1010 #else | 1121 #else |
1011 return kUnsupportedFunctionError; | 1122 return kUnsupportedFunctionError; |
1012 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1123 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
1013 } | 1124 } |
1014 | 1125 |
1015 EchoCancellation* AudioProcessingImpl::echo_cancellation() const { | 1126 EchoCancellation* AudioProcessingImpl::echo_cancellation() const { |
1016 return echo_cancellation_; | 1127 // Adding a lock here has no effect as it allows any access to the submodule |
| 1128 // from the returned pointer. |
| 1129 return public_submodules_->echo_cancellation; |
1017 } | 1130 } |
1018 | 1131 |
1019 EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const { | 1132 EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const { |
1020 return echo_control_mobile_; | 1133 // Adding a lock here has no effect as it allows any access to the submodule |
| 1134 // from the returned pointer. |
| 1135 return public_submodules_->echo_control_mobile; |
1021 } | 1136 } |
1022 | 1137 |
1023 GainControl* AudioProcessingImpl::gain_control() const { | 1138 GainControl* AudioProcessingImpl::gain_control() const { |
1024 if (use_new_agc_) { | 1139 // Adding a lock here has no effect as it allows any access to the submodule |
1025 return gain_control_for_new_agc_.get(); | 1140 // from the returned pointer. |
| 1141 if (constants_.use_new_agc) { |
| 1142 return public_submodules_->gain_control_for_new_agc.get(); |
1026 } | 1143 } |
1027 return gain_control_; | 1144 return public_submodules_->gain_control; |
1028 } | 1145 } |
1029 | 1146 |
1030 HighPassFilter* AudioProcessingImpl::high_pass_filter() const { | 1147 HighPassFilter* AudioProcessingImpl::high_pass_filter() const { |
1031 return high_pass_filter_; | 1148 // Adding a lock here has no effect as it allows any access to the submodule |
| 1149 // from the returned pointer. |
| 1150 return public_submodules_->high_pass_filter; |
1032 } | 1151 } |
1033 | 1152 |
1034 LevelEstimator* AudioProcessingImpl::level_estimator() const { | 1153 LevelEstimator* AudioProcessingImpl::level_estimator() const { |
1035 return level_estimator_; | 1154 // Adding a lock here has no effect as it allows any access to the submodule |
| 1155 // from the returned pointer. |
| 1156 return public_submodules_->level_estimator; |
1036 } | 1157 } |
1037 | 1158 |
1038 NoiseSuppression* AudioProcessingImpl::noise_suppression() const { | 1159 NoiseSuppression* AudioProcessingImpl::noise_suppression() const { |
1039 return noise_suppression_; | 1160 // Adding a lock here has no effect as it allows any access to the submodule |
| 1161 // from the returned pointer. |
| 1162 return public_submodules_->noise_suppression; |
1040 } | 1163 } |
1041 | 1164 |
1042 VoiceDetection* AudioProcessingImpl::voice_detection() const { | 1165 VoiceDetection* AudioProcessingImpl::voice_detection() const { |
1043 return voice_detection_; | 1166 // Adding a lock here has no effect as it allows any access to the submodule |
| 1167 // from the returned pointer. |
| 1168 return public_submodules_->voice_detection; |
1044 } | 1169 } |
1045 | 1170 |
1046 bool AudioProcessingImpl::is_data_processed() const { | 1171 bool AudioProcessingImpl::is_data_processed() const { |
1047 if (beamformer_enabled_) { | 1172 if (constants_.beamformer_enabled) { |
1048 return true; | 1173 return true; |
1049 } | 1174 } |
1050 | 1175 |
1051 int enabled_count = 0; | 1176 int enabled_count = 0; |
1052 for (auto item : component_list_) { | 1177 for (auto item : private_submodules_->component_list) { |
1053 if (item->is_component_enabled()) { | 1178 if (item->is_component_enabled()) { |
1054 enabled_count++; | 1179 enabled_count++; |
1055 } | 1180 } |
1056 } | 1181 } |
1057 | 1182 |
1058 // Data is unchanged if no components are enabled, or if only level_estimator_ | 1183 // Data is unchanged if no components are enabled, or if only |
1059 // or voice_detection_ is enabled. | 1184 // public_submodules_->level_estimator |
| 1185 // or public_submodules_->voice_detection is enabled. |
1060 if (enabled_count == 0) { | 1186 if (enabled_count == 0) { |
1061 return false; | 1187 return false; |
1062 } else if (enabled_count == 1) { | 1188 } else if (enabled_count == 1) { |
1063 if (level_estimator_->is_enabled() || voice_detection_->is_enabled()) { | 1189 if (public_submodules_->level_estimator->is_enabled() || |
| 1190 public_submodules_->voice_detection->is_enabled()) { |
1064 return false; | 1191 return false; |
1065 } | 1192 } |
1066 } else if (enabled_count == 2) { | 1193 } else if (enabled_count == 2) { |
1067 if (level_estimator_->is_enabled() && voice_detection_->is_enabled()) { | 1194 if (public_submodules_->level_estimator->is_enabled() && |
| 1195 public_submodules_->voice_detection->is_enabled()) { |
1068 return false; | 1196 return false; |
1069 } | 1197 } |
1070 } | 1198 } |
1071 return true; | 1199 return true; |
1072 } | 1200 } |
1073 | 1201 |
1074 bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const { | 1202 bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const { |
1075 // Check if we've upmixed or downmixed the audio. | 1203 // Check if we've upmixed or downmixed the audio. |
1076 return ((shared_state_.api_format_.output_stream().num_channels() != | 1204 return ((formats_.api_format.output_stream().num_channels() != |
1077 shared_state_.api_format_.input_stream().num_channels()) || | 1205 formats_.api_format.input_stream().num_channels()) || |
1078 is_data_processed || transient_suppressor_enabled_); | 1206 is_data_processed || capture_.transient_suppressor_enabled); |
1079 } | 1207 } |
1080 | 1208 |
1081 bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const { | 1209 bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const { |
1082 return (is_data_processed && | 1210 return (is_data_processed && |
1083 (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || | 1211 (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == |
1084 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz)); | 1212 kSampleRate32kHz || |
| 1213 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == |
| 1214 kSampleRate48kHz)); |
1085 } | 1215 } |
1086 | 1216 |
1087 bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const { | 1217 bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const { |
1088 if (!is_data_processed && !voice_detection_->is_enabled() && | 1218 if (!is_data_processed && |
1089 !transient_suppressor_enabled_) { | 1219 !public_submodules_->voice_detection->is_enabled() && |
1090 // Only level_estimator_ is enabled. | 1220 !capture_.transient_suppressor_enabled) { |
| 1221 // Only public_submodules_->level_estimator is enabled. |
1091 return false; | 1222 return false; |
1092 } else if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || | 1223 } else if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == |
1093 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { | 1224 kSampleRate32kHz || |
1094 // Something besides level_estimator_ is enabled, and we have super-wb. | 1225 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == |
| 1226 kSampleRate48kHz) { |
| 1227 // Something besides public_submodules_->level_estimator is enabled, and we |
| 1228 // have super-wb. |
1095 return true; | 1229 return true; |
1096 } | 1230 } |
1097 return false; | 1231 return false; |
1098 } | 1232 } |
1099 | 1233 |
1100 bool AudioProcessingImpl::is_rev_processed() const { | 1234 bool AudioProcessingImpl::is_rev_processed() const { |
1101 return intelligibility_enabled_ && intelligibility_enhancer_->active(); | 1235 return constants_.intelligibility_enabled && |
| 1236 public_submodules_->intelligibility_enhancer->active(); |
1102 } | 1237 } |
1103 | 1238 |
1104 bool AudioProcessingImpl::render_check_rev_conversion_needed() const { | 1239 bool AudioProcessingImpl::render_check_rev_conversion_needed() const { |
1105 return rev_conversion_needed(); | 1240 return rev_conversion_needed(); |
1106 } | 1241 } |
1107 | 1242 |
1108 bool AudioProcessingImpl::rev_conversion_needed() const { | 1243 bool AudioProcessingImpl::rev_conversion_needed() const { |
1109 // TODO(peah): Refactor to be allowed to verify using thread annotations. | 1244 return (formats_.api_format.reverse_input_stream() != |
1110 return (shared_state_.api_format_.reverse_input_stream() != | 1245 formats_.api_format.reverse_output_stream()); |
1111 shared_state_.api_format_.reverse_output_stream()); | |
1112 } | 1246 } |
1113 | 1247 |
1114 void AudioProcessingImpl::InitializeExperimentalAgc() { | 1248 void AudioProcessingImpl::InitializeExperimentalAgc() { |
1115 // TODO(peah): Refactor to be allowed to verify using thread annotations. | 1249 if (constants_.use_new_agc) { |
1116 if (use_new_agc_) { | 1250 if (!private_submodules_->agc_manager.get()) { |
1117 if (!agc_manager_.get()) { | 1251 private_submodules_->agc_manager.reset(new AgcManagerDirect( |
1118 agc_manager_.reset(new AgcManagerDirect(gain_control_, | 1252 public_submodules_->gain_control, |
1119 gain_control_for_new_agc_.get(), | 1253 public_submodules_->gain_control_for_new_agc.get(), |
1120 agc_startup_min_volume_)); | 1254 constants_.agc_startup_min_volume)); |
1121 } | 1255 } |
1122 agc_manager_->Initialize(); | 1256 private_submodules_->agc_manager->Initialize(); |
1123 agc_manager_->SetCaptureMuted(output_will_be_muted_); | 1257 private_submodules_->agc_manager->SetCaptureMuted( |
| 1258 capture_.output_will_be_muted); |
1124 } | 1259 } |
1125 } | 1260 } |
1126 | 1261 |
1127 void AudioProcessingImpl::InitializeTransient() { | 1262 void AudioProcessingImpl::InitializeTransient() { |
1128 // TODO(peah): Refactor to be allowed to verify using thread annotations. | 1263 if (capture_.transient_suppressor_enabled) { |
1129 if (transient_suppressor_enabled_) { | 1264 if (!public_submodules_->transient_suppressor.get()) { |
1130 if (!transient_suppressor_.get()) { | 1265 public_submodules_->transient_suppressor.reset(new TransientSuppressor()); |
1131 transient_suppressor_.reset(new TransientSuppressor()); | |
1132 } | 1266 } |
1133 transient_suppressor_->Initialize( | 1267 public_submodules_->transient_suppressor->Initialize( |
1134 fwd_proc_format_.sample_rate_hz(), split_rate_, | 1268 capture_nonlocked_.fwd_proc_format.sample_rate_hz(), |
1135 shared_state_.api_format_.output_stream().num_channels()); | 1269 capture_nonlocked_.split_rate, |
| 1270 formats_.api_format.output_stream().num_channels()); |
1136 } | 1271 } |
1137 } | 1272 } |
1138 | 1273 |
1139 void AudioProcessingImpl::InitializeBeamformer() { | 1274 void AudioProcessingImpl::InitializeBeamformer() { |
1140 // TODO(peah): Refactor to be allowed to verify using thread annotations. | 1275 if (constants_.beamformer_enabled) { |
1141 if (beamformer_enabled_) { | 1276 if (!private_submodules_->beamformer) { |
1142 if (!beamformer_) { | 1277 private_submodules_->beamformer.reset(new NonlinearBeamformer( |
1143 beamformer_.reset( | 1278 constants_.array_geometry, constants_.target_direction)); |
1144 new NonlinearBeamformer(array_geometry_, target_direction_)); | |
1145 } | 1279 } |
1146 beamformer_->Initialize(kChunkSizeMs, split_rate_); | 1280 private_submodules_->beamformer->Initialize(kChunkSizeMs, |
| 1281 capture_nonlocked_.split_rate); |
1147 } | 1282 } |
1148 } | 1283 } |
1149 | 1284 |
1150 void AudioProcessingImpl::InitializeIntelligibility() { | 1285 void AudioProcessingImpl::InitializeIntelligibility() { |
1151 // TODO(peah): Refactor to be allowed to verify using thread annotations. | 1286 if (constants_.intelligibility_enabled) { |
1152 if (intelligibility_enabled_) { | |
1153 IntelligibilityEnhancer::Config config; | 1287 IntelligibilityEnhancer::Config config; |
1154 config.sample_rate_hz = split_rate_; | 1288 config.sample_rate_hz = capture_nonlocked_.split_rate; |
1155 config.num_capture_channels = capture_audio_->num_channels(); | 1289 config.num_capture_channels = capture_.capture_audio->num_channels(); |
1156 config.num_render_channels = render_audio_->num_channels(); | 1290 config.num_render_channels = render_.render_audio->num_channels(); |
1157 intelligibility_enhancer_.reset(new IntelligibilityEnhancer(config)); | 1291 public_submodules_->intelligibility_enhancer.reset( |
| 1292 new IntelligibilityEnhancer(config)); |
1158 } | 1293 } |
1159 } | 1294 } |
1160 | 1295 |
1161 void AudioProcessingImpl::MaybeUpdateHistograms() { | 1296 void AudioProcessingImpl::MaybeUpdateHistograms() { |
1162 static const int kMinDiffDelayMs = 60; | 1297 static const int kMinDiffDelayMs = 60; |
1163 | 1298 |
1164 if (echo_cancellation()->is_enabled()) { | 1299 if (echo_cancellation()->is_enabled()) { |
1165 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. | 1300 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. |
1166 // If a stream has echo we know that the echo_cancellation is in process. | 1301 // If a stream has echo we know that the echo_cancellation is in process. |
1167 if (stream_delay_jumps_ == -1 && echo_cancellation()->stream_has_echo()) { | 1302 if (capture_.stream_delay_jumps == -1 && |
1168 stream_delay_jumps_ = 0; | 1303 echo_cancellation()->stream_has_echo()) { |
| 1304 capture_.stream_delay_jumps = 0; |
1169 } | 1305 } |
1170 if (aec_system_delay_jumps_ == -1 && | 1306 if (capture_.aec_system_delay_jumps == -1 && |
1171 echo_cancellation()->stream_has_echo()) { | 1307 echo_cancellation()->stream_has_echo()) { |
1172 aec_system_delay_jumps_ = 0; | 1308 capture_.aec_system_delay_jumps = 0; |
1173 } | 1309 } |
1174 | 1310 |
1175 // Detect a jump in platform reported system delay and log the difference. | 1311 // Detect a jump in platform reported system delay and log the difference. |
1176 const int diff_stream_delay_ms = stream_delay_ms_ - last_stream_delay_ms_; | 1312 const int diff_stream_delay_ms = |
1177 if (diff_stream_delay_ms > kMinDiffDelayMs && last_stream_delay_ms_ != 0) { | 1313 capture_nonlocked_.stream_delay_ms - capture_.last_stream_delay_ms; |
| 1314 if (diff_stream_delay_ms > kMinDiffDelayMs && |
| 1315 capture_.last_stream_delay_ms != 0) { |
1178 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.PlatformReportedStreamDelayJump", | 1316 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.PlatformReportedStreamDelayJump", |
1179 diff_stream_delay_ms, kMinDiffDelayMs, 1000, 100); | 1317 diff_stream_delay_ms, kMinDiffDelayMs, 1000, 100); |
1180 if (stream_delay_jumps_ == -1) { | 1318 if (capture_.stream_delay_jumps == -1) { |
1181 stream_delay_jumps_ = 0; // Activate counter if needed. | 1319 capture_.stream_delay_jumps = 0; // Activate counter if needed. |
1182 } | 1320 } |
1183 stream_delay_jumps_++; | 1321 capture_.stream_delay_jumps++; |
1184 } | 1322 } |
1185 last_stream_delay_ms_ = stream_delay_ms_; | 1323 capture_.last_stream_delay_ms = capture_nonlocked_.stream_delay_ms; |
1186 | 1324 |
1187 // Detect a jump in AEC system delay and log the difference. | 1325 // Detect a jump in AEC system delay and log the difference. |
1188 const int frames_per_ms = rtc::CheckedDivExact(split_rate_, 1000); | 1326 const int frames_per_ms = |
| 1327 rtc::CheckedDivExact(capture_nonlocked_.split_rate, 1000); |
1189 const int aec_system_delay_ms = | 1328 const int aec_system_delay_ms = |
1190 WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms; | 1329 WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms; |
1191 const int diff_aec_system_delay_ms = | 1330 const int diff_aec_system_delay_ms = |
1192 aec_system_delay_ms - last_aec_system_delay_ms_; | 1331 aec_system_delay_ms - capture_.last_aec_system_delay_ms; |
1193 if (diff_aec_system_delay_ms > kMinDiffDelayMs && | 1332 if (diff_aec_system_delay_ms > kMinDiffDelayMs && |
1194 last_aec_system_delay_ms_ != 0) { | 1333 capture_.last_aec_system_delay_ms != 0) { |
1195 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", | 1334 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", |
1196 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, | 1335 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, |
1197 100); | 1336 100); |
1198 if (aec_system_delay_jumps_ == -1) { | 1337 if (capture_.aec_system_delay_jumps == -1) { |
1199 aec_system_delay_jumps_ = 0; // Activate counter if needed. | 1338 capture_.aec_system_delay_jumps = 0; // Activate counter if needed. |
1200 } | 1339 } |
1201 aec_system_delay_jumps_++; | 1340 capture_.aec_system_delay_jumps++; |
1202 } | 1341 } |
1203 last_aec_system_delay_ms_ = aec_system_delay_ms; | 1342 capture_.last_aec_system_delay_ms = aec_system_delay_ms; |
1204 } | 1343 } |
1205 } | 1344 } |
1206 | 1345 |
1207 void AudioProcessingImpl::UpdateHistogramsOnCallEnd() { | 1346 void AudioProcessingImpl::UpdateHistogramsOnCallEnd() { |
1208 CriticalSectionScoped crit_scoped(crit_); | 1347 // Run in a single-threaded manner. |
1209 if (stream_delay_jumps_ > -1) { | 1348 rtc::CritScope cs_render(&crit_render_); |
| 1349 rtc::CritScope cs_capture(&crit_capture_); |
| 1350 |
| 1351 if (capture_.stream_delay_jumps > -1) { |
1210 RTC_HISTOGRAM_ENUMERATION( | 1352 RTC_HISTOGRAM_ENUMERATION( |
1211 "WebRTC.Audio.NumOfPlatformReportedStreamDelayJumps", | 1353 "WebRTC.Audio.NumOfPlatformReportedStreamDelayJumps", |
1212 stream_delay_jumps_, 51); | 1354 capture_.stream_delay_jumps, 51); |
1213 } | 1355 } |
1214 stream_delay_jumps_ = -1; | 1356 capture_.stream_delay_jumps = -1; |
1215 last_stream_delay_ms_ = 0; | 1357 capture_.last_stream_delay_ms = 0; |
1216 | 1358 |
1217 if (aec_system_delay_jumps_ > -1) { | 1359 if (capture_.aec_system_delay_jumps > -1) { |
1218 RTC_HISTOGRAM_ENUMERATION("WebRTC.Audio.NumOfAecSystemDelayJumps", | 1360 RTC_HISTOGRAM_ENUMERATION("WebRTC.Audio.NumOfAecSystemDelayJumps", |
1219 aec_system_delay_jumps_, 51); | 1361 capture_.aec_system_delay_jumps, 51); |
1220 } | 1362 } |
1221 aec_system_delay_jumps_ = -1; | 1363 capture_.aec_system_delay_jumps = -1; |
1222 last_aec_system_delay_ms_ = 0; | 1364 capture_.last_aec_system_delay_ms = 0; |
1223 } | 1365 } |
1224 | 1366 |
1225 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1367 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
1226 int AudioProcessingImpl::WriteMessageToDebugFile() { | 1368 int AudioProcessingImpl::WriteMessageToDebugFile( |
1227 int32_t size = event_msg_->ByteSize(); | 1369 FileWrapper* debug_file, |
| 1370 rtc::CriticalSection* crit_debug, |
| 1371 ApmDebugDumpThreadState* debug_state) { |
| 1372 int32_t size = debug_state->event_msg->ByteSize(); |
1228 if (size <= 0) { | 1373 if (size <= 0) { |
1229 return kUnspecifiedError; | 1374 return kUnspecifiedError; |
1230 } | 1375 } |
1231 #if defined(WEBRTC_ARCH_BIG_ENDIAN) | 1376 #if defined(WEBRTC_ARCH_BIG_ENDIAN) |
1232 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be | 1377 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be |
1233 // pretty safe in assuming little-endian. | 1378 // pretty safe in assuming little-endian. |
1234 #endif | 1379 #endif |
1235 | 1380 |
1236 if (!event_msg_->SerializeToString(&event_str_)) { | 1381 if (!debug_state->event_msg->SerializeToString(&debug_state->event_str)) { |
1237 return kUnspecifiedError; | 1382 return kUnspecifiedError; |
1238 } | 1383 } |
1239 | 1384 |
1240 // Write message preceded by its size. | 1385 { |
1241 if (!debug_file_->Write(&size, sizeof(int32_t))) { | 1386 // Ensure atomic writes of the message. |
1242 return kFileError; | 1387 rtc::CritScope cs_capture(crit_debug); |
1243 } | 1388 // Write message preceded by its size. |
1244 if (!debug_file_->Write(event_str_.data(), event_str_.length())) { | 1389 if (!debug_file->Write(&size, sizeof(int32_t))) { |
1245 return kFileError; | 1390 return kFileError; |
| 1391 } |
| 1392 if (!debug_file->Write(debug_state->event_str.data(), |
| 1393 debug_state->event_str.length())) { |
| 1394 return kFileError; |
| 1395 } |
1246 } | 1396 } |
1247 | 1397 |
1248 event_msg_->Clear(); | 1398 debug_state->event_msg->Clear(); |
1249 | 1399 |
1250 return kNoError; | 1400 return kNoError; |
1251 } | 1401 } |
1252 | 1402 |
1253 int AudioProcessingImpl::WriteInitMessage() { | 1403 int AudioProcessingImpl::WriteInitMessage() { |
1254 // TODO(peah): Refactor to be allowed to verify using thread annotations. | 1404 debug_dump_.capture.event_msg->set_type(audioproc::Event::INIT); |
1255 event_msg_->set_type(audioproc::Event::INIT); | 1405 audioproc::Init* msg = debug_dump_.capture.event_msg->mutable_init(); |
1256 audioproc::Init* msg = event_msg_->mutable_init(); | 1406 msg->set_sample_rate(formats_.api_format.input_stream().sample_rate_hz()); |
1257 msg->set_sample_rate( | 1407 |
1258 shared_state_.api_format_.input_stream().sample_rate_hz()); | |
1259 msg->set_num_input_channels( | 1408 msg->set_num_input_channels( |
1260 shared_state_.api_format_.input_stream().num_channels()); | 1409 formats_.api_format.input_stream().num_channels()); |
1261 msg->set_num_output_channels( | 1410 msg->set_num_output_channels( |
1262 shared_state_.api_format_.output_stream().num_channels()); | 1411 formats_.api_format.output_stream().num_channels()); |
1263 msg->set_num_reverse_channels( | 1412 msg->set_num_reverse_channels( |
1264 shared_state_.api_format_.reverse_input_stream().num_channels()); | 1413 formats_.api_format.reverse_input_stream().num_channels()); |
1265 msg->set_reverse_sample_rate( | 1414 msg->set_reverse_sample_rate( |
1266 shared_state_.api_format_.reverse_input_stream().sample_rate_hz()); | 1415 formats_.api_format.reverse_input_stream().sample_rate_hz()); |
1267 msg->set_output_sample_rate( | 1416 msg->set_output_sample_rate( |
1268 shared_state_.api_format_.output_stream().sample_rate_hz()); | 1417 formats_.api_format.output_stream().sample_rate_hz()); |
1269 // TODO(ekmeyerson): Add reverse output fields to event_msg_. | 1418 // TODO(ekmeyerson): Add reverse output fields to |
| 1419 // debug_dump_.capture.event_msg. |
1270 | 1420 |
1271 RETURN_ON_ERR(WriteMessageToDebugFile()); | 1421 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 1422 &crit_debug_, &debug_dump_.capture)); |
1272 return kNoError; | 1423 return kNoError; |
1273 } | 1424 } |
1274 | 1425 |
1275 int AudioProcessingImpl::WriteConfigMessage(bool forced) { | 1426 int AudioProcessingImpl::WriteConfigMessage(bool forced) { |
1276 audioproc::Config config; | 1427 audioproc::Config config; |
1277 | 1428 |
1278 config.set_aec_enabled(echo_cancellation_->is_enabled()); | 1429 config.set_aec_enabled(public_submodules_->echo_cancellation->is_enabled()); |
1279 config.set_aec_delay_agnostic_enabled( | 1430 config.set_aec_delay_agnostic_enabled( |
1280 echo_cancellation_->is_delay_agnostic_enabled()); | 1431 public_submodules_->echo_cancellation->is_delay_agnostic_enabled()); |
1281 config.set_aec_drift_compensation_enabled( | 1432 config.set_aec_drift_compensation_enabled( |
1282 echo_cancellation_->is_drift_compensation_enabled()); | 1433 public_submodules_->echo_cancellation->is_drift_compensation_enabled()); |
1283 config.set_aec_extended_filter_enabled( | 1434 config.set_aec_extended_filter_enabled( |
1284 echo_cancellation_->is_extended_filter_enabled()); | 1435 public_submodules_->echo_cancellation->is_extended_filter_enabled()); |
1285 config.set_aec_suppression_level( | 1436 config.set_aec_suppression_level(static_cast<int>( |
1286 static_cast<int>(echo_cancellation_->suppression_level())); | 1437 public_submodules_->echo_cancellation->suppression_level())); |
1287 | 1438 |
1288 config.set_aecm_enabled(echo_control_mobile_->is_enabled()); | 1439 config.set_aecm_enabled( |
| 1440 public_submodules_->echo_control_mobile->is_enabled()); |
1289 config.set_aecm_comfort_noise_enabled( | 1441 config.set_aecm_comfort_noise_enabled( |
1290 echo_control_mobile_->is_comfort_noise_enabled()); | 1442 public_submodules_->echo_control_mobile->is_comfort_noise_enabled()); |
1291 config.set_aecm_routing_mode( | 1443 config.set_aecm_routing_mode(static_cast<int>( |
1292 static_cast<int>(echo_control_mobile_->routing_mode())); | 1444 public_submodules_->echo_control_mobile->routing_mode())); |
1293 | 1445 |
1294 config.set_agc_enabled(gain_control_->is_enabled()); | 1446 config.set_agc_enabled(public_submodules_->gain_control->is_enabled()); |
1295 config.set_agc_mode(static_cast<int>(gain_control_->mode())); | 1447 config.set_agc_mode( |
1296 config.set_agc_limiter_enabled(gain_control_->is_limiter_enabled()); | 1448 static_cast<int>(public_submodules_->gain_control->mode())); |
1297 config.set_noise_robust_agc_enabled(use_new_agc_); | 1449 config.set_agc_limiter_enabled( |
| 1450 public_submodules_->gain_control->is_limiter_enabled()); |
| 1451 config.set_noise_robust_agc_enabled(constants_.use_new_agc); |
1298 | 1452 |
1299 config.set_hpf_enabled(high_pass_filter_->is_enabled()); | 1453 config.set_hpf_enabled(public_submodules_->high_pass_filter->is_enabled()); |
1300 | 1454 |
1301 config.set_ns_enabled(noise_suppression_->is_enabled()); | 1455 config.set_ns_enabled(public_submodules_->noise_suppression->is_enabled()); |
1302 config.set_ns_level(static_cast<int>(noise_suppression_->level())); | 1456 config.set_ns_level( |
| 1457 static_cast<int>(public_submodules_->noise_suppression->level())); |
1303 | 1458 |
1304 config.set_transient_suppression_enabled(transient_suppressor_enabled_); | 1459 config.set_transient_suppression_enabled( |
| 1460 capture_.transient_suppressor_enabled); |
1305 | 1461 |
1306 std::string serialized_config = config.SerializeAsString(); | 1462 std::string serialized_config = config.SerializeAsString(); |
1307 if (!forced && last_serialized_config_ == serialized_config) { | 1463 if (!forced && |
| 1464 debug_dump_.capture.last_serialized_config == serialized_config) { |
1308 return kNoError; | 1465 return kNoError; |
1309 } | 1466 } |
1310 | 1467 |
1311 last_serialized_config_ = serialized_config; | 1468 debug_dump_.capture.last_serialized_config = serialized_config; |
1312 | 1469 |
1313 event_msg_->set_type(audioproc::Event::CONFIG); | 1470 debug_dump_.capture.event_msg->set_type(audioproc::Event::CONFIG); |
1314 event_msg_->mutable_config()->CopyFrom(config); | 1471 debug_dump_.capture.event_msg->mutable_config()->CopyFrom(config); |
1315 | 1472 |
1316 RETURN_ON_ERR(WriteMessageToDebugFile()); | 1473 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(), |
| 1474 &crit_debug_, &debug_dump_.capture)); |
1317 return kNoError; | 1475 return kNoError; |
1318 } | 1476 } |
1319 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1477 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
1320 | 1478 |
1321 } // namespace webrtc | 1479 } // namespace webrtc |
OLD | NEW |