Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(494)

Side by Side Diff: webrtc/modules/audio_processing/audio_processing_impl.cc

Issue 1424663003: Lock scheme #8: Introduced the new locking scheme (Closed) Base URL: https://chromium.googlesource.com/external/webrtc.git@add_threadcheckers_CL
Patch Set: Changes in response to reviewer comments Created 5 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 /* 1 /*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 * 3 *
4 * Use of this source code is governed by a BSD-style license 4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source 5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found 6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may 7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree. 8 * be found in the AUTHORS file in the root of the source tree.
9 */ 9 */
10 10
(...skipping 19 matching lines...) Expand all
30 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h" 30 #include "webrtc/modules/audio_processing/echo_control_mobile_impl.h"
31 #include "webrtc/modules/audio_processing/gain_control_impl.h" 31 #include "webrtc/modules/audio_processing/gain_control_impl.h"
32 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h" 32 #include "webrtc/modules/audio_processing/high_pass_filter_impl.h"
33 #include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhanc er.h" 33 #include "webrtc/modules/audio_processing/intelligibility/intelligibility_enhanc er.h"
34 #include "webrtc/modules/audio_processing/level_estimator_impl.h" 34 #include "webrtc/modules/audio_processing/level_estimator_impl.h"
35 #include "webrtc/modules/audio_processing/noise_suppression_impl.h" 35 #include "webrtc/modules/audio_processing/noise_suppression_impl.h"
36 #include "webrtc/modules/audio_processing/processing_component.h" 36 #include "webrtc/modules/audio_processing/processing_component.h"
37 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h" 37 #include "webrtc/modules/audio_processing/transient/transient_suppressor.h"
38 #include "webrtc/modules/audio_processing/voice_detection_impl.h" 38 #include "webrtc/modules/audio_processing/voice_detection_impl.h"
39 #include "webrtc/modules/include/module_common_types.h" 39 #include "webrtc/modules/include/module_common_types.h"
40 #include "webrtc/system_wrappers/include/critical_section_wrapper.h"
41 #include "webrtc/system_wrappers/include/file_wrapper.h" 40 #include "webrtc/system_wrappers/include/file_wrapper.h"
42 #include "webrtc/system_wrappers/include/logging.h" 41 #include "webrtc/system_wrappers/include/logging.h"
43 #include "webrtc/system_wrappers/include/metrics.h" 42 #include "webrtc/system_wrappers/include/metrics.h"
44 43
45 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 44 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
46 // Files generated at build-time by the protobuf compiler. 45 // Files generated at build-time by the protobuf compiler.
47 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD 46 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD
48 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" 47 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h"
49 #else 48 #else
50 #include "webrtc/audio_processing/debug.pb.h" 49 #include "webrtc/audio_processing/debug.pb.h"
(...skipping 17 matching lines...) Expand all
68 case AudioProcessing::kStereo: 67 case AudioProcessing::kStereo:
69 return false; 68 return false;
70 case AudioProcessing::kMonoAndKeyboard: 69 case AudioProcessing::kMonoAndKeyboard:
71 case AudioProcessing::kStereoAndKeyboard: 70 case AudioProcessing::kStereoAndKeyboard:
72 return true; 71 return true;
73 } 72 }
74 73
75 assert(false); 74 assert(false);
76 return false; 75 return false;
77 } 76 }
77 } // namespace
78 78
79 } // namespace 79 struct ApmPublicSubmodules {
80 ApmPublicSubmodules()
81 : echo_cancellation(nullptr),
82 echo_control_mobile(nullptr),
83 gain_control(nullptr),
84 high_pass_filter(nullptr),
85 level_estimator(nullptr),
86 noise_suppression(nullptr),
87 voice_detection(nullptr) {}
88 // Accessed externally of APM without any lock acquired.
89 EchoCancellationImpl* echo_cancellation;
90 EchoControlMobileImpl* echo_control_mobile;
91 GainControlImpl* gain_control;
92 HighPassFilterImpl* high_pass_filter;
93 LevelEstimatorImpl* level_estimator;
94 NoiseSuppressionImpl* noise_suppression;
95 VoiceDetectionImpl* voice_detection;
96 rtc::scoped_ptr<GainControlForNewAgc> gain_control_for_new_agc;
97
98 // Accessed internally from both render and capture.
99 rtc::scoped_ptr<TransientSuppressor> transient_suppressor;
100 rtc::scoped_ptr<IntelligibilityEnhancer> intelligibility_enhancer;
101 };
102
103 struct ApmPrivateSubmodules {
104 explicit ApmPrivateSubmodules(Beamformer<float>* beamformer)
105 : beamformer(beamformer) {}
106 // Accessed internally from capture or during initialization
107 std::list<ProcessingComponent*> component_list;
108 rtc::scoped_ptr<Beamformer<float>> beamformer;
109 rtc::scoped_ptr<AgcManagerDirect> agc_manager;
110 };
80 111
81 // Throughout webrtc, it's assumed that success is represented by zero. 112 // Throughout webrtc, it's assumed that success is represented by zero.
82 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); 113 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero");
83 114
84 // This class has two main functionalities: 115 // This class has two main functionalities:
85 // 116 //
86 // 1) It is returned instead of the real GainControl after the new AGC has been 117 // 1) It is returned instead of the real GainControl after the new AGC has been
87 // enabled in order to prevent an outside user from overriding compression 118 // enabled in order to prevent an outside user from overriding compression
88 // settings. It doesn't do anything in its implementation, except for 119 // settings. It doesn't do anything in its implementation, except for
89 // delegating the const methods and Enable calls to the real GainControl, so 120 // delegating the const methods and Enable calls to the real GainControl, so
(...skipping 75 matching lines...) Expand 10 before | Expand all | Expand 10 after
165 196
166 AudioProcessing* AudioProcessing::Create(const Config& config) { 197 AudioProcessing* AudioProcessing::Create(const Config& config) {
167 return Create(config, nullptr); 198 return Create(config, nullptr);
168 } 199 }
169 200
170 AudioProcessing* AudioProcessing::Create(const Config& config, 201 AudioProcessing* AudioProcessing::Create(const Config& config,
171 Beamformer<float>* beamformer) { 202 Beamformer<float>* beamformer) {
172 AudioProcessingImpl* apm = new AudioProcessingImpl(config, beamformer); 203 AudioProcessingImpl* apm = new AudioProcessingImpl(config, beamformer);
173 if (apm->Initialize() != kNoError) { 204 if (apm->Initialize() != kNoError) {
174 delete apm; 205 delete apm;
175 apm = NULL; 206 apm = nullptr;
176 } 207 }
177 208
178 return apm; 209 return apm;
179 } 210 }
180 211
181 AudioProcessingImpl::AudioProcessingImpl(const Config& config) 212 AudioProcessingImpl::AudioProcessingImpl(const Config& config)
182 : AudioProcessingImpl(config, nullptr) {} 213 : AudioProcessingImpl(config, nullptr) {}
183 214
184 AudioProcessingImpl::AudioProcessingImpl(const Config& config, 215 AudioProcessingImpl::AudioProcessingImpl(const Config& config,
185 Beamformer<float>* beamformer) 216 Beamformer<float>* beamformer)
186 : echo_cancellation_(NULL), 217 : public_submodules_(new ApmPublicSubmodules()),
187 echo_control_mobile_(NULL), 218 private_submodules_(new ApmPrivateSubmodules(beamformer)),
188 gain_control_(NULL), 219 constants_(config.Get<ExperimentalAgc>().startup_min_volume,
189 high_pass_filter_(NULL), 220 config.Get<Beamforming>().array_geometry,
190 level_estimator_(NULL), 221 config.Get<Beamforming>().target_direction,
191 noise_suppression_(NULL), 222 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
192 voice_detection_(NULL), 223 false,
193 crit_(CriticalSectionWrapper::CreateCriticalSection()), 224 #else
194 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 225 config.Get<ExperimentalAgc>().enabled,
195 debug_file_(FileWrapper::Create()),
196 event_msg_(new audioproc::Event()),
197 #endif 226 #endif
198 fwd_proc_format_(kSampleRate16kHz), 227 config.Get<Intelligibility>().enabled,
199 rev_proc_format_(kSampleRate16kHz, 1), 228 config.Get<Beamforming>().enabled),
200 split_rate_(kSampleRate16kHz), 229
201 stream_delay_ms_(0),
202 delay_offset_ms_(0),
203 was_stream_delay_set_(false),
204 last_stream_delay_ms_(0),
205 last_aec_system_delay_ms_(0),
206 stream_delay_jumps_(-1),
207 aec_system_delay_jumps_(-1),
208 output_will_be_muted_(false),
209 key_pressed_(false),
210 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) 230 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
211 use_new_agc_(false), 231 capture_(false)
212 #else 232 #else
213 use_new_agc_(config.Get<ExperimentalAgc>().enabled), 233 capture_(config.Get<ExperimentalNs>().enabled)
214 #endif 234 #endif
215 agc_startup_min_volume_(config.Get<ExperimentalAgc>().startup_min_volume), 235 {
216 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
217 transient_suppressor_enabled_(false),
218 #else
219 transient_suppressor_enabled_(config.Get<ExperimentalNs>().enabled),
220 #endif
221 beamformer_enabled_(config.Get<Beamforming>().enabled),
222 beamformer_(beamformer),
223 array_geometry_(config.Get<Beamforming>().array_geometry),
224 target_direction_(config.Get<Beamforming>().target_direction),
225 intelligibility_enabled_(config.Get<Intelligibility>().enabled) {
226 render_thread_checker_.DetachFromThread(); 236 render_thread_checker_.DetachFromThread();
227 capture_thread_checker_.DetachFromThread(); 237 capture_thread_checker_.DetachFromThread();
228 238
229 echo_cancellation_ = 239 {
230 new EchoCancellationImpl(this, crit_, &render_thread_checker_); 240 rtc::CritScope cs_render(&crit_render_);
231 component_list_.push_back(echo_cancellation_); 241 rtc::CritScope cs_capture(&crit_capture_);
232 242
233 echo_control_mobile_ = 243 public_submodules_->echo_cancellation = new EchoCancellationImpl(
234 new EchoControlMobileImpl(this, crit_, &render_thread_checker_); 244 this, &crit_render_, &crit_capture_, &render_thread_checker_);
235 component_list_.push_back(echo_control_mobile_); 245 public_submodules_->echo_control_mobile = new EchoControlMobileImpl(
246 this, &crit_render_, &crit_capture_, &render_thread_checker_);
247 public_submodules_->gain_control =
248 new GainControlImpl(this, &crit_capture_, &crit_capture_,
249 &render_thread_checker_, &capture_thread_checker_);
250 public_submodules_->high_pass_filter =
251 new HighPassFilterImpl(this, &crit_capture_);
252 public_submodules_->level_estimator = new LevelEstimatorImpl(this);
253 public_submodules_->noise_suppression =
254 new NoiseSuppressionImpl(this, &crit_capture_);
255 public_submodules_->voice_detection =
256 new VoiceDetectionImpl(this, &crit_capture_);
257 public_submodules_->gain_control_for_new_agc.reset(
258 new GainControlForNewAgc(public_submodules_->gain_control));
236 259
237 gain_control_ = new GainControlImpl(this, crit_, &render_thread_checker_, 260 private_submodules_->component_list.push_back(
238 &capture_thread_checker_); 261 public_submodules_->echo_cancellation);
239 component_list_.push_back(gain_control_); 262 private_submodules_->component_list.push_back(
240 263 public_submodules_->echo_control_mobile);
241 high_pass_filter_ = new HighPassFilterImpl(this, crit_); 264 private_submodules_->component_list.push_back(
242 component_list_.push_back(high_pass_filter_); 265 public_submodules_->gain_control);
243 266 private_submodules_->component_list.push_back(
244 level_estimator_ = new LevelEstimatorImpl(this, crit_); 267 public_submodules_->high_pass_filter);
245 component_list_.push_back(level_estimator_); 268 private_submodules_->component_list.push_back(
246 269 public_submodules_->level_estimator);
247 noise_suppression_ = new NoiseSuppressionImpl(this, crit_); 270 private_submodules_->component_list.push_back(
248 component_list_.push_back(noise_suppression_); 271 public_submodules_->noise_suppression);
249 272 private_submodules_->component_list.push_back(
250 voice_detection_ = new VoiceDetectionImpl(this, crit_); 273 public_submodules_->voice_detection);
251 component_list_.push_back(voice_detection_); 274 }
252
253 gain_control_for_new_agc_.reset(new GainControlForNewAgc(gain_control_));
254 275
255 SetExtraOptions(config); 276 SetExtraOptions(config);
256 } 277 }
257 278
258 AudioProcessingImpl::~AudioProcessingImpl() { 279 AudioProcessingImpl::~AudioProcessingImpl() {
259 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread()); 280 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread());
260 { 281
261 CriticalSectionScoped crit_scoped(crit_); 282 // Depends on gain_control_ and
262 // Depends on gain_control_ and gain_control_for_new_agc_. 283 // public_submodules_->gain_control_for_new_agc.
263 agc_manager_.reset(); 284 private_submodules_->agc_manager.reset();
264 // Depends on gain_control_. 285 // Depends on gain_control_.
265 gain_control_for_new_agc_.reset(); 286 public_submodules_->gain_control_for_new_agc.reset();
266 while (!component_list_.empty()) { 287 while (!private_submodules_->component_list.empty()) {
267 ProcessingComponent* component = component_list_.front(); 288 ProcessingComponent* component =
268 component->Destroy(); 289 private_submodules_->component_list.front();
269 delete component; 290 component->Destroy();
270 component_list_.pop_front(); 291 delete component;
271 } 292 private_submodules_->component_list.pop_front();
293 }
272 294
273 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 295 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
274 if (debug_file_->Open()) { 296 if (debug_dump_.debug_file->Open()) {
275 debug_file_->CloseFile(); 297 debug_dump_.debug_file->CloseFile();
276 } 298 }
277 #endif 299 #endif
278 }
279 delete crit_;
280 crit_ = NULL;
281 } 300 }
282 301
283 int AudioProcessingImpl::Initialize() { 302 int AudioProcessingImpl::Initialize() {
303 // Run in a single-threaded manner during initialization.
304 rtc::CritScope cs_render(&crit_render_);
305 rtc::CritScope cs_capture(&crit_capture_);
284 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread()); 306 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread());
285 CriticalSectionScoped crit_scoped(crit_);
286 return InitializeLocked(); 307 return InitializeLocked();
287 } 308 }
288 309
289 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, 310 int AudioProcessingImpl::Initialize(int input_sample_rate_hz,
290 int output_sample_rate_hz, 311 int output_sample_rate_hz,
291 int reverse_sample_rate_hz, 312 int reverse_sample_rate_hz,
292 ChannelLayout input_layout, 313 ChannelLayout input_layout,
293 ChannelLayout output_layout, 314 ChannelLayout output_layout,
294 ChannelLayout reverse_layout) { 315 ChannelLayout reverse_layout) {
295 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread()); 316 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread());
296 const ProcessingConfig processing_config = { 317 const ProcessingConfig processing_config = {
297 {{input_sample_rate_hz, 318 {{input_sample_rate_hz,
298 ChannelsFromLayout(input_layout), 319 ChannelsFromLayout(input_layout),
299 LayoutHasKeyboard(input_layout)}, 320 LayoutHasKeyboard(input_layout)},
300 {output_sample_rate_hz, 321 {output_sample_rate_hz,
301 ChannelsFromLayout(output_layout), 322 ChannelsFromLayout(output_layout),
302 LayoutHasKeyboard(output_layout)}, 323 LayoutHasKeyboard(output_layout)},
303 {reverse_sample_rate_hz, 324 {reverse_sample_rate_hz,
304 ChannelsFromLayout(reverse_layout), 325 ChannelsFromLayout(reverse_layout),
305 LayoutHasKeyboard(reverse_layout)}, 326 LayoutHasKeyboard(reverse_layout)},
306 {reverse_sample_rate_hz, 327 {reverse_sample_rate_hz,
307 ChannelsFromLayout(reverse_layout), 328 ChannelsFromLayout(reverse_layout),
308 LayoutHasKeyboard(reverse_layout)}}}; 329 LayoutHasKeyboard(reverse_layout)}}};
309 330
310 return Initialize(processing_config); 331 return Initialize(processing_config);
311 } 332 }
312 333
313 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { 334 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) {
335 // Run in a single-threaded manner during initialization.
336 rtc::CritScope cs_render(&crit_render_);
337 rtc::CritScope cs_capture(&crit_capture_);
314 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread()); 338 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread());
315 CriticalSectionScoped crit_scoped(crit_);
316 return InitializeLocked(processing_config); 339 return InitializeLocked(processing_config);
317 } 340 }
318 341
319 // Calls InitializeLocked() if any of the audio parameters have changed from 342 // Calls InitializeLocked() if any of the audio parameters have changed from
320 // their current values. 343 // their current values (needs to be called while holding the crit_render_lock).
321 int AudioProcessingImpl::MaybeInitializeLocked( 344 int AudioProcessingImpl::MaybeInitialize(
322 const ProcessingConfig& processing_config) { 345 const ProcessingConfig& processing_config) {
323 RTC_DCHECK(render_thread_checker_.CalledOnValidThread() || 346 RTC_DCHECK(render_thread_checker_.CalledOnValidThread() ||
324 capture_thread_checker_.CalledOnValidThread()); 347 capture_thread_checker_.CalledOnValidThread());
325 if (processing_config == shared_state_.api_format_) { 348 // Called from both threads. Thread check is therefore not possible.
349 if (processing_config == formats_.api_format) {
326 return kNoError; 350 return kNoError;
327 } 351 }
352
353 rtc::CritScope cs_capture(&crit_capture_);
328 return InitializeLocked(processing_config); 354 return InitializeLocked(processing_config);
329 } 355 }
330 356
331 int AudioProcessingImpl::InitializeLocked() { 357 int AudioProcessingImpl::InitializeLocked() {
332 const int fwd_audio_buffer_channels = 358 const int fwd_audio_buffer_channels =
333 beamformer_enabled_ 359 constants_.beamformer_enabled
334 ? shared_state_.api_format_.input_stream().num_channels() 360 ? formats_.api_format.input_stream().num_channels()
335 : shared_state_.api_format_.output_stream().num_channels(); 361 : formats_.api_format.output_stream().num_channels();
336 const int rev_audio_buffer_out_num_frames = 362 const int rev_audio_buffer_out_num_frames =
337 shared_state_.api_format_.reverse_output_stream().num_frames() == 0 363 formats_.api_format.reverse_output_stream().num_frames() == 0
338 ? rev_proc_format_.num_frames() 364 ? formats_.rev_proc_format.num_frames()
339 : shared_state_.api_format_.reverse_output_stream().num_frames(); 365 : formats_.api_format.reverse_output_stream().num_frames();
340 if (shared_state_.api_format_.reverse_input_stream().num_channels() > 0) { 366 if (formats_.api_format.reverse_input_stream().num_channels() > 0) {
341 render_audio_.reset(new AudioBuffer( 367 render_.render_audio.reset(new AudioBuffer(
342 shared_state_.api_format_.reverse_input_stream().num_frames(), 368 formats_.api_format.reverse_input_stream().num_frames(),
343 shared_state_.api_format_.reverse_input_stream().num_channels(), 369 formats_.api_format.reverse_input_stream().num_channels(),
344 rev_proc_format_.num_frames(), rev_proc_format_.num_channels(), 370 formats_.rev_proc_format.num_frames(),
371 formats_.rev_proc_format.num_channels(),
345 rev_audio_buffer_out_num_frames)); 372 rev_audio_buffer_out_num_frames));
346 if (rev_conversion_needed()) { 373 if (rev_conversion_needed()) {
347 render_converter_ = AudioConverter::Create( 374 render_.render_converter = AudioConverter::Create(
348 shared_state_.api_format_.reverse_input_stream().num_channels(), 375 formats_.api_format.reverse_input_stream().num_channels(),
349 shared_state_.api_format_.reverse_input_stream().num_frames(), 376 formats_.api_format.reverse_input_stream().num_frames(),
350 shared_state_.api_format_.reverse_output_stream().num_channels(), 377 formats_.api_format.reverse_output_stream().num_channels(),
351 shared_state_.api_format_.reverse_output_stream().num_frames()); 378 formats_.api_format.reverse_output_stream().num_frames());
352 } else { 379 } else {
353 render_converter_.reset(nullptr); 380 render_.render_converter.reset(nullptr);
354 } 381 }
355 } else { 382 } else {
356 render_audio_.reset(nullptr); 383 render_.render_audio.reset(nullptr);
357 render_converter_.reset(nullptr); 384 render_.render_converter.reset(nullptr);
358 } 385 }
359 capture_audio_.reset( 386 capture_.capture_audio.reset(
360 new AudioBuffer(shared_state_.api_format_.input_stream().num_frames(), 387 new AudioBuffer(formats_.api_format.input_stream().num_frames(),
361 shared_state_.api_format_.input_stream().num_channels(), 388 formats_.api_format.input_stream().num_channels(),
362 fwd_proc_format_.num_frames(), fwd_audio_buffer_channels, 389 capture_nonlocked_.fwd_proc_format.num_frames(),
363 shared_state_.api_format_.output_stream().num_frames())); 390 fwd_audio_buffer_channels,
391 formats_.api_format.output_stream().num_frames()));
364 392
365 // Initialize all components. 393 // Initialize all components.
366 for (auto item : component_list_) { 394 for (auto item : private_submodules_->component_list) {
367 int err = item->Initialize(); 395 int err = item->Initialize();
368 if (err != kNoError) { 396 if (err != kNoError) {
369 return err; 397 return err;
370 } 398 }
371 } 399 }
372 400
373 InitializeExperimentalAgc(); 401 InitializeExperimentalAgc();
374 402
375 InitializeTransient(); 403 InitializeTransient();
376 404
377 InitializeBeamformer(); 405 InitializeBeamformer();
378 406
379 InitializeIntelligibility(); 407 InitializeIntelligibility();
380 408
381 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 409 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
382 if (debug_file_->Open()) { 410 if (debug_dump_.debug_file->Open()) {
383 int err = WriteInitMessage(); 411 int err = WriteInitMessage();
384 if (err != kNoError) { 412 if (err != kNoError) {
385 return err; 413 return err;
386 } 414 }
387 } 415 }
388 #endif 416 #endif
389 417
390 return kNoError; 418 return kNoError;
391 } 419 }
392 420
393 int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { 421 int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) {
394 RTC_DCHECK(render_thread_checker_.CalledOnValidThread() ||
395 capture_thread_checker_.CalledOnValidThread());
396 for (const auto& stream : config.streams) { 422 for (const auto& stream : config.streams) {
397 if (stream.num_channels() < 0) { 423 if (stream.num_channels() < 0) {
398 return kBadNumberChannelsError; 424 return kBadNumberChannelsError;
399 } 425 }
400 if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) { 426 if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) {
401 return kBadSampleRateError; 427 return kBadSampleRateError;
402 } 428 }
403 } 429 }
404 430
405 const int num_in_channels = config.input_stream().num_channels(); 431 const int num_in_channels = config.input_stream().num_channels();
406 const int num_out_channels = config.output_stream().num_channels(); 432 const int num_out_channels = config.output_stream().num_channels();
407 433
408 // Need at least one input channel. 434 // Need at least one input channel.
409 // Need either one output channel or as many outputs as there are inputs. 435 // Need either one output channel or as many outputs as there are inputs.
410 if (num_in_channels == 0 || 436 if (num_in_channels == 0 ||
411 !(num_out_channels == 1 || num_out_channels == num_in_channels)) { 437 !(num_out_channels == 1 || num_out_channels == num_in_channels)) {
412 return kBadNumberChannelsError; 438 return kBadNumberChannelsError;
413 } 439 }
414 440
415 if (beamformer_enabled_ && 441 if (constants_.beamformer_enabled && (static_cast<size_t>(num_in_channels) !=
416 (static_cast<size_t>(num_in_channels) != array_geometry_.size() || 442 constants_.array_geometry.size() ||
417 num_out_channels > 1)) { 443 num_out_channels > 1)) {
418 return kBadNumberChannelsError; 444 return kBadNumberChannelsError;
419 } 445 }
420 446
421 shared_state_.api_format_ = config; 447 formats_.api_format = config;
422 448
423 // We process at the closest native rate >= min(input rate, output rate)... 449 // We process at the closest native rate >= min(input rate, output rate)...
424 const int min_proc_rate = 450 const int min_proc_rate =
425 std::min(shared_state_.api_format_.input_stream().sample_rate_hz(), 451 std::min(formats_.api_format.input_stream().sample_rate_hz(),
426 shared_state_.api_format_.output_stream().sample_rate_hz()); 452 formats_.api_format.output_stream().sample_rate_hz());
427 int fwd_proc_rate; 453 int fwd_proc_rate;
428 for (size_t i = 0; i < kNumNativeSampleRates; ++i) { 454 for (size_t i = 0; i < kNumNativeSampleRates; ++i) {
429 fwd_proc_rate = kNativeSampleRatesHz[i]; 455 fwd_proc_rate = kNativeSampleRatesHz[i];
430 if (fwd_proc_rate >= min_proc_rate) { 456 if (fwd_proc_rate >= min_proc_rate) {
431 break; 457 break;
432 } 458 }
433 } 459 }
434 // ...with one exception. 460 // ...with one exception.
435 if (echo_control_mobile_->is_enabled() && 461 if (public_submodules_->echo_control_mobile->is_enabled() &&
436 min_proc_rate > kMaxAECMSampleRateHz) { 462 min_proc_rate > kMaxAECMSampleRateHz) {
437 fwd_proc_rate = kMaxAECMSampleRateHz; 463 fwd_proc_rate = kMaxAECMSampleRateHz;
438 } 464 }
439 465
440 fwd_proc_format_ = StreamConfig(fwd_proc_rate); 466 capture_nonlocked_.fwd_proc_format = StreamConfig(fwd_proc_rate);
441 467
442 // We normally process the reverse stream at 16 kHz. Unless... 468 // We normally process the reverse stream at 16 kHz. Unless...
443 int rev_proc_rate = kSampleRate16kHz; 469 int rev_proc_rate = kSampleRate16kHz;
444 if (fwd_proc_format_.sample_rate_hz() == kSampleRate8kHz) { 470 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate8kHz) {
445 // ...the forward stream is at 8 kHz. 471 // ...the forward stream is at 8 kHz.
446 rev_proc_rate = kSampleRate8kHz; 472 rev_proc_rate = kSampleRate8kHz;
447 } else { 473 } else {
448 if (shared_state_.api_format_.reverse_input_stream().sample_rate_hz() == 474 if (formats_.api_format.reverse_input_stream().sample_rate_hz() ==
449 kSampleRate32kHz) { 475 kSampleRate32kHz) {
450 // ...or the input is at 32 kHz, in which case we use the splitting 476 // ...or the input is at 32 kHz, in which case we use the splitting
451 // filter rather than the resampler. 477 // filter rather than the resampler.
452 rev_proc_rate = kSampleRate32kHz; 478 rev_proc_rate = kSampleRate32kHz;
453 } 479 }
454 } 480 }
455 481
456 // Always downmix the reverse stream to mono for analysis. This has been 482 // Always downmix the reverse stream to mono for analysis. This has been
457 // demonstrated to work well for AEC in most practical scenarios. 483 // demonstrated to work well for AEC in most practical scenarios.
458 rev_proc_format_ = StreamConfig(rev_proc_rate, 1); 484 formats_.rev_proc_format = StreamConfig(rev_proc_rate, 1);
459 485
460 if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || 486 if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate32kHz ||
461 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { 487 capture_nonlocked_.fwd_proc_format.sample_rate_hz() == kSampleRate48kHz) {
462 split_rate_ = kSampleRate16kHz; 488 capture_nonlocked_.split_rate = kSampleRate16kHz;
463 } else { 489 } else {
464 split_rate_ = fwd_proc_format_.sample_rate_hz(); 490 capture_nonlocked_.split_rate =
491 capture_nonlocked_.fwd_proc_format.sample_rate_hz();
465 } 492 }
466 493
467 return InitializeLocked(); 494 return InitializeLocked();
468 } 495 }
469 496
470 void AudioProcessingImpl::SetExtraOptions(const Config& config) { 497 void AudioProcessingImpl::SetExtraOptions(const Config& config) {
471 CriticalSectionScoped crit_scoped(crit_); 498 // Run in a single-threaded manner when setting the extra options.
499 rtc::CritScope cs_render(&crit_render_);
500 rtc::CritScope cs_capture(&crit_capture_);
472 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread()); 501 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread());
473 for (auto item : component_list_) { 502 for (auto item : private_submodules_->component_list) {
474 item->SetExtraOptions(config); 503 item->SetExtraOptions(config);
475 } 504 }
476 505
477 if (transient_suppressor_enabled_ != config.Get<ExperimentalNs>().enabled) { 506 if (capture_.transient_suppressor_enabled !=
478 transient_suppressor_enabled_ = config.Get<ExperimentalNs>().enabled; 507 config.Get<ExperimentalNs>().enabled) {
508 capture_.transient_suppressor_enabled =
509 config.Get<ExperimentalNs>().enabled;
479 InitializeTransient(); 510 InitializeTransient();
480 } 511 }
481 } 512 }
482 513
483 514
484 int AudioProcessingImpl::proc_sample_rate_hz() const { 515 int AudioProcessingImpl::proc_sample_rate_hz() const {
485 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread() || 516 // Only called from submodules beneath APM, hence locking is not needed.
486 render_thread_checker_.CalledOnValidThread() || 517 return capture_nonlocked_.fwd_proc_format.sample_rate_hz();
487 capture_thread_checker_.CalledOnValidThread());
488 return fwd_proc_format_.sample_rate_hz();
489 } 518 }
490 519
491 int AudioProcessingImpl::proc_split_sample_rate_hz() const { 520 int AudioProcessingImpl::proc_split_sample_rate_hz() const {
492 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread() || 521 // Only called from submodules/tests beneath APM, hence locking is not needed.
493 render_thread_checker_.CalledOnValidThread() || 522 return capture_nonlocked_.split_rate;
494 capture_thread_checker_.CalledOnValidThread());
495 return split_rate_;
496 } 523 }
497 524
498 int AudioProcessingImpl::num_reverse_channels() const { 525 int AudioProcessingImpl::num_reverse_channels() const {
499 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread() || 526 // Only called from submodules/tests beneath APM, hence locking is not needed.
500 render_thread_checker_.CalledOnValidThread() || 527 return formats_.rev_proc_format.num_channels();
501 capture_thread_checker_.CalledOnValidThread());
502 return rev_proc_format_.num_channels();
503 } 528 }
504 529
505 int AudioProcessingImpl::num_input_channels() const { 530 int AudioProcessingImpl::num_input_channels() const {
531 // Only called from submodules/tests beneath APM, hence locking is not needed.
506 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 532 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
507 return shared_state_.api_format_.input_stream().num_channels(); 533 return formats_.api_format.input_stream().num_channels();
508 } 534 }
509 535
510 int AudioProcessingImpl::num_output_channels() const { 536 int AudioProcessingImpl::num_output_channels() const {
511 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread() || 537 // Only called from submodules/tests beneath APM, hence locking is not needed.
512 render_thread_checker_.CalledOnValidThread() || 538 return formats_.api_format.output_stream().num_channels();
513 capture_thread_checker_.CalledOnValidThread());
514 return shared_state_.api_format_.output_stream().num_channels();
515 } 539 }
516 540
517 void AudioProcessingImpl::set_output_will_be_muted(bool muted) { 541 void AudioProcessingImpl::set_output_will_be_muted(bool muted) {
518 CriticalSectionScoped lock(crit_); 542 rtc::CritScope cs(&crit_capture_);
519 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread()); 543 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread());
520 output_will_be_muted_ = muted; 544 capture_.output_will_be_muted = muted;
521 if (agc_manager_.get()) { 545 if (private_submodules_->agc_manager.get()) {
522 agc_manager_->SetCaptureMuted(output_will_be_muted_); 546 private_submodules_->agc_manager->SetCaptureMuted(
547 capture_.output_will_be_muted);
523 } 548 }
524 } 549 }
525 550
526 551
527 int AudioProcessingImpl::ProcessStream(const float* const* src, 552 int AudioProcessingImpl::ProcessStream(const float* const* src,
528 size_t samples_per_channel, 553 size_t samples_per_channel,
529 int input_sample_rate_hz, 554 int input_sample_rate_hz,
530 ChannelLayout input_layout, 555 ChannelLayout input_layout,
531 int output_sample_rate_hz, 556 int output_sample_rate_hz,
532 ChannelLayout output_layout, 557 ChannelLayout output_layout,
533 float* const* dest) { 558 float* const* dest) {
534 CriticalSectionScoped crit_scoped(crit_);
535 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 559 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
536 StreamConfig input_stream = shared_state_.api_format_.input_stream(); 560 StreamConfig input_stream;
561 StreamConfig output_stream;
562 {
563 // Access the formats_.api_format.input_stream beneath the capture lock.
564 // The lock must be released as it is later required in the call
565 // to ProcessStream(,,,);
566 rtc::CritScope cs(&crit_capture_);
567 input_stream = formats_.api_format.input_stream();
568 output_stream = formats_.api_format.output_stream();
569 }
570
537 input_stream.set_sample_rate_hz(input_sample_rate_hz); 571 input_stream.set_sample_rate_hz(input_sample_rate_hz);
538 input_stream.set_num_channels(ChannelsFromLayout(input_layout)); 572 input_stream.set_num_channels(ChannelsFromLayout(input_layout));
539 input_stream.set_has_keyboard(LayoutHasKeyboard(input_layout)); 573 input_stream.set_has_keyboard(LayoutHasKeyboard(input_layout));
540
541 StreamConfig output_stream = shared_state_.api_format_.output_stream();
542 output_stream.set_sample_rate_hz(output_sample_rate_hz); 574 output_stream.set_sample_rate_hz(output_sample_rate_hz);
543 output_stream.set_num_channels(ChannelsFromLayout(output_layout)); 575 output_stream.set_num_channels(ChannelsFromLayout(output_layout));
544 output_stream.set_has_keyboard(LayoutHasKeyboard(output_layout)); 576 output_stream.set_has_keyboard(LayoutHasKeyboard(output_layout));
545 577
546 if (samples_per_channel != input_stream.num_frames()) { 578 if (samples_per_channel != input_stream.num_frames()) {
547 return kBadDataLengthError; 579 return kBadDataLengthError;
548 } 580 }
549 return ProcessStream(src, input_stream, output_stream, dest); 581 return ProcessStream(src, input_stream, output_stream, dest);
550 } 582 }
551 583
552 int AudioProcessingImpl::ProcessStream(const float* const* src, 584 int AudioProcessingImpl::ProcessStream(const float* const* src,
553 const StreamConfig& input_config, 585 const StreamConfig& input_config,
554 const StreamConfig& output_config, 586 const StreamConfig& output_config,
555 float* const* dest) { 587 float* const* dest) {
556 CriticalSectionScoped crit_scoped(crit_);
557 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 588 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
589 {
590 // Acquire the capture lock in order to safely call the function
591 // that retrieves the render side data. This function accesses apm
592 // getters that need the capture lock held when being called.
593 rtc::CritScope cs_capture(&crit_capture_);
594 public_submodules_->echo_cancellation->ReadQueuedRenderData();
595 public_submodules_->echo_control_mobile->ReadQueuedRenderData();
596 public_submodules_->gain_control->ReadQueuedRenderData();
597 }
558 if (!src || !dest) { 598 if (!src || !dest) {
559 return kNullPointerError; 599 return kNullPointerError;
560 } 600 }
561 601
562 echo_cancellation_->ReadQueuedRenderData(); 602 ProcessingConfig processing_config = formats_.api_format;
563 echo_control_mobile_->ReadQueuedRenderData();
564 gain_control_->ReadQueuedRenderData();
565
566 ProcessingConfig processing_config = shared_state_.api_format_;
567 processing_config.input_stream() = input_config; 603 processing_config.input_stream() = input_config;
568 processing_config.output_stream() = output_config; 604 processing_config.output_stream() = output_config;
569 605
570 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); 606 {
607 // Do conditional reinitialization.
608 rtc::CritScope cs_render(&crit_render_);
609 RETURN_ON_ERR(MaybeInitialize(processing_config));
610 }
611 rtc::CritScope cs_capture(&crit_capture_);
612
571 assert(processing_config.input_stream().num_frames() == 613 assert(processing_config.input_stream().num_frames() ==
572 shared_state_.api_format_.input_stream().num_frames()); 614 formats_.api_format.input_stream().num_frames());
573 615
574 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 616 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
575 if (debug_file_->Open()) { 617 if (debug_dump_.debug_file->Open()) {
576 RETURN_ON_ERR(WriteConfigMessage(false)); 618 RETURN_ON_ERR(WriteConfigMessage(false));
577 619
578 event_msg_->set_type(audioproc::Event::STREAM); 620 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM);
579 audioproc::Stream* msg = event_msg_->mutable_stream(); 621 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
580 const size_t channel_size = 622 const size_t channel_size =
581 sizeof(float) * shared_state_.api_format_.input_stream().num_frames(); 623 sizeof(float) * formats_.api_format.input_stream().num_frames();
582 for (int i = 0; i < shared_state_.api_format_.input_stream().num_channels(); 624 for (int i = 0; i < formats_.api_format.input_stream().num_channels(); ++i)
583 ++i)
584 msg->add_input_channel(src[i], channel_size); 625 msg->add_input_channel(src[i], channel_size);
585 } 626 }
586 #endif 627 #endif
587 628
588 capture_audio_->CopyFrom(src, shared_state_.api_format_.input_stream()); 629 capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream());
589 RETURN_ON_ERR(ProcessStreamLocked()); 630 RETURN_ON_ERR(ProcessStreamLocked());
590 capture_audio_->CopyTo(shared_state_.api_format_.output_stream(), dest); 631 capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest);
591 632
592 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 633 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
593 if (debug_file_->Open()) { 634 if (debug_dump_.debug_file->Open()) {
594 audioproc::Stream* msg = event_msg_->mutable_stream(); 635 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
595 const size_t channel_size = 636 const size_t channel_size =
596 sizeof(float) * shared_state_.api_format_.output_stream().num_frames(); 637 sizeof(float) * formats_.api_format.output_stream().num_frames();
597 for (int i = 0; 638 for (int i = 0; i < formats_.api_format.output_stream().num_channels(); ++i)
598 i < shared_state_.api_format_.output_stream().num_channels(); ++i)
599 msg->add_output_channel(dest[i], channel_size); 639 msg->add_output_channel(dest[i], channel_size);
600 RETURN_ON_ERR(WriteMessageToDebugFile()); 640 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
641 &crit_debug_, &debug_dump_.capture));
601 } 642 }
602 #endif 643 #endif
603 644
604 return kNoError; 645 return kNoError;
605 } 646 }
606 647
607 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { 648 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) {
608 CriticalSectionScoped crit_scoped(crit_);
609 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 649 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
610 echo_cancellation_->ReadQueuedRenderData(); 650 {
611 echo_control_mobile_->ReadQueuedRenderData(); 651 // Acquire the capture lock in order to safely call the function
612 gain_control_->ReadQueuedRenderData(); 652 // that retrieves the render side data. This function accesses apm
653 // getters that need the capture lock held when being called.
654 // The lock needs to be released as
655 // public_submodules_->echo_control_mobile->is_enabled() aquires this lock
656 // as well.
657 rtc::CritScope cs_capture(&crit_capture_);
658 public_submodules_->echo_cancellation->ReadQueuedRenderData();
659 public_submodules_->echo_control_mobile->ReadQueuedRenderData();
660 public_submodules_->gain_control->ReadQueuedRenderData();
661 }
613 662
614 if (!frame) { 663 if (!frame) {
615 return kNullPointerError; 664 return kNullPointerError;
616 } 665 }
617 // Must be a native rate. 666 // Must be a native rate.
618 if (frame->sample_rate_hz_ != kSampleRate8kHz && 667 if (frame->sample_rate_hz_ != kSampleRate8kHz &&
619 frame->sample_rate_hz_ != kSampleRate16kHz && 668 frame->sample_rate_hz_ != kSampleRate16kHz &&
620 frame->sample_rate_hz_ != kSampleRate32kHz && 669 frame->sample_rate_hz_ != kSampleRate32kHz &&
621 frame->sample_rate_hz_ != kSampleRate48kHz) { 670 frame->sample_rate_hz_ != kSampleRate48kHz) {
622 return kBadSampleRateError; 671 return kBadSampleRateError;
623 } 672 }
624 673
625 if (echo_control_mobile_->is_enabled() && 674 if (public_submodules_->echo_control_mobile->is_enabled() &&
626 frame->sample_rate_hz_ > kMaxAECMSampleRateHz) { 675 frame->sample_rate_hz_ > kMaxAECMSampleRateHz) {
627 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates"; 676 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates";
628 return kUnsupportedComponentError; 677 return kUnsupportedComponentError;
629 } 678 }
630 679
631 // TODO(ajm): The input and output rates and channels are currently 680 ProcessingConfig processing_config;
632 // constrained to be identical in the int16 interface. 681 {
633 ProcessingConfig processing_config = shared_state_.api_format_; 682 // Aquire lock for the access of api_format.
683 // The lock is released immediately due to the conditional
684 // reinitialization.
685 rtc::CritScope cs_capture(&crit_capture_);
686 // TODO(ajm): The input and output rates and channels are currently
687 // constrained to be identical in the int16 interface.
688 processing_config = formats_.api_format;
689 }
634 processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_); 690 processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_);
635 processing_config.input_stream().set_num_channels(frame->num_channels_); 691 processing_config.input_stream().set_num_channels(frame->num_channels_);
636 processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_); 692 processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_);
637 processing_config.output_stream().set_num_channels(frame->num_channels_); 693 processing_config.output_stream().set_num_channels(frame->num_channels_);
638 694
639 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); 695 {
696 // Do conditional reinitialization.
697 rtc::CritScope cs_render(&crit_render_);
698 RETURN_ON_ERR(MaybeInitialize(processing_config));
699 }
700 rtc::CritScope cs_capture(&crit_capture_);
640 if (frame->samples_per_channel_ != 701 if (frame->samples_per_channel_ !=
641 shared_state_.api_format_.input_stream().num_frames()) { 702 formats_.api_format.input_stream().num_frames()) {
642 return kBadDataLengthError; 703 return kBadDataLengthError;
643 } 704 }
644 705
645 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 706 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
646 if (debug_file_->Open()) { 707 if (debug_dump_.debug_file->Open()) {
647 event_msg_->set_type(audioproc::Event::STREAM); 708 debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM);
648 audioproc::Stream* msg = event_msg_->mutable_stream(); 709 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
649 const size_t data_size = 710 const size_t data_size =
650 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; 711 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_;
651 msg->set_input_data(frame->data_, data_size); 712 msg->set_input_data(frame->data_, data_size);
652 } 713 }
653 #endif 714 #endif
654 715
655 capture_audio_->DeinterleaveFrom(frame); 716 capture_.capture_audio->DeinterleaveFrom(frame);
656 RETURN_ON_ERR(ProcessStreamLocked()); 717 RETURN_ON_ERR(ProcessStreamLocked());
657 capture_audio_->InterleaveTo(frame, output_copy_needed(is_data_processed())); 718 capture_.capture_audio->InterleaveTo(frame,
719 output_copy_needed(is_data_processed()));
658 720
659 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 721 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
660 if (debug_file_->Open()) { 722 if (debug_dump_.debug_file->Open()) {
661 audioproc::Stream* msg = event_msg_->mutable_stream(); 723 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
662 const size_t data_size = 724 const size_t data_size =
663 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; 725 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_;
664 msg->set_output_data(frame->data_, data_size); 726 msg->set_output_data(frame->data_, data_size);
665 RETURN_ON_ERR(WriteMessageToDebugFile()); 727 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
728 &crit_debug_, &debug_dump_.capture));
666 } 729 }
667 #endif 730 #endif
668 731
669 return kNoError; 732 return kNoError;
670 } 733 }
671 734
672 int AudioProcessingImpl::ProcessStreamLocked() { 735 int AudioProcessingImpl::ProcessStreamLocked() {
673 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 736 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
674 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 737 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
675 if (debug_file_->Open()) { 738 if (debug_dump_.debug_file->Open()) {
676 audioproc::Stream* msg = event_msg_->mutable_stream(); 739 audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
677 msg->set_delay(stream_delay_ms_); 740 msg->set_delay(capture_nonlocked_.stream_delay_ms);
678 msg->set_drift(echo_cancellation_->stream_drift_samples()); 741 msg->set_drift(
742 public_submodules_->echo_cancellation->stream_drift_samples());
679 msg->set_level(gain_control()->stream_analog_level()); 743 msg->set_level(gain_control()->stream_analog_level());
680 msg->set_keypress(key_pressed_); 744 msg->set_keypress(capture_.key_pressed);
681 } 745 }
682 #endif 746 #endif
683 747
684 MaybeUpdateHistograms(); 748 MaybeUpdateHistograms();
685 749
686 AudioBuffer* ca = capture_audio_.get(); // For brevity. 750 AudioBuffer* ca = capture_.capture_audio.get(); // For brevity.
687 751
688 if (use_new_agc_ && gain_control_->is_enabled()) { 752 if (constants_.use_new_agc &&
689 agc_manager_->AnalyzePreProcess(ca->channels()[0], ca->num_channels(), 753 public_submodules_->gain_control->is_enabled()) {
690 fwd_proc_format_.num_frames()); 754 private_submodules_->agc_manager->AnalyzePreProcess(
755 ca->channels()[0], ca->num_channels(),
756 capture_nonlocked_.fwd_proc_format.num_frames());
691 } 757 }
692 758
693 bool data_processed = is_data_processed(); 759 bool data_processed = is_data_processed();
694 if (analysis_needed(data_processed)) { 760 if (analysis_needed(data_processed)) {
695 ca->SplitIntoFrequencyBands(); 761 ca->SplitIntoFrequencyBands();
696 } 762 }
697 763
698 if (intelligibility_enabled_) { 764 if (constants_.intelligibility_enabled) {
699 intelligibility_enhancer_->AnalyzeCaptureAudio( 765 public_submodules_->intelligibility_enhancer->AnalyzeCaptureAudio(
700 ca->split_channels_f(kBand0To8kHz), split_rate_, ca->num_channels()); 766 ca->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate,
767 ca->num_channels());
701 } 768 }
702 769
703 if (beamformer_enabled_) { 770 if (constants_.beamformer_enabled) {
704 beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f()); 771 private_submodules_->beamformer->ProcessChunk(*ca->split_data_f(),
772 ca->split_data_f());
705 ca->set_num_channels(1); 773 ca->set_num_channels(1);
706 } 774 }
707 775
708 RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca)); 776 RETURN_ON_ERR(public_submodules_->high_pass_filter->ProcessCaptureAudio(ca));
709 RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca)); 777 RETURN_ON_ERR(public_submodules_->gain_control->AnalyzeCaptureAudio(ca));
710 RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca)); 778 RETURN_ON_ERR(public_submodules_->noise_suppression->AnalyzeCaptureAudio(ca));
711 RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca)); 779 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessCaptureAudio(ca));
712 780
713 if (echo_control_mobile_->is_enabled() && noise_suppression_->is_enabled()) { 781 if (public_submodules_->echo_control_mobile->is_enabled() &&
782 public_submodules_->noise_suppression->is_enabled()) {
714 ca->CopyLowPassToReference(); 783 ca->CopyLowPassToReference();
715 } 784 }
716 RETURN_ON_ERR(noise_suppression_->ProcessCaptureAudio(ca)); 785 RETURN_ON_ERR(public_submodules_->noise_suppression->ProcessCaptureAudio(ca));
717 RETURN_ON_ERR(echo_control_mobile_->ProcessCaptureAudio(ca)); 786 RETURN_ON_ERR(
718 RETURN_ON_ERR(voice_detection_->ProcessCaptureAudio(ca)); 787 public_submodules_->echo_control_mobile->ProcessCaptureAudio(ca));
788 RETURN_ON_ERR(public_submodules_->voice_detection->ProcessCaptureAudio(ca));
719 789
720 if (use_new_agc_ && gain_control_->is_enabled() && 790 if (constants_.use_new_agc &&
721 (!beamformer_enabled_ || beamformer_->is_target_present())) { 791 public_submodules_->gain_control->is_enabled() &&
722 agc_manager_->Process(ca->split_bands_const(0)[kBand0To8kHz], 792 (!constants_.beamformer_enabled ||
723 ca->num_frames_per_band(), split_rate_); 793 private_submodules_->beamformer->is_target_present())) {
794 private_submodules_->agc_manager->Process(
795 ca->split_bands_const(0)[kBand0To8kHz], ca->num_frames_per_band(),
796 capture_nonlocked_.split_rate);
724 } 797 }
725 RETURN_ON_ERR(gain_control_->ProcessCaptureAudio(ca)); 798 RETURN_ON_ERR(public_submodules_->gain_control->ProcessCaptureAudio(ca));
726 799
727 if (synthesis_needed(data_processed)) { 800 if (synthesis_needed(data_processed)) {
728 ca->MergeFrequencyBands(); 801 ca->MergeFrequencyBands();
729 } 802 }
730 803
731 // TODO(aluebs): Investigate if the transient suppression placement should be 804 // TODO(aluebs): Investigate if the transient suppression placement should be
732 // before or after the AGC. 805 // before or after the AGC.
733 if (transient_suppressor_enabled_) { 806 if (capture_.transient_suppressor_enabled) {
734 float voice_probability = 807 float voice_probability =
735 agc_manager_.get() ? agc_manager_->voice_probability() : 1.f; 808 private_submodules_->agc_manager.get()
809 ? private_submodules_->agc_manager->voice_probability()
810 : 1.f;
736 811
737 transient_suppressor_->Suppress( 812 public_submodules_->transient_suppressor->Suppress(
738 ca->channels_f()[0], ca->num_frames(), ca->num_channels(), 813 ca->channels_f()[0], ca->num_frames(), ca->num_channels(),
739 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(), 814 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(),
740 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability, 815 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability,
741 key_pressed_); 816 capture_.key_pressed);
742 } 817 }
743 818
744 // The level estimator operates on the recombined data. 819 // The level estimator operates on the recombined data.
745 RETURN_ON_ERR(level_estimator_->ProcessStream(ca)); 820 RETURN_ON_ERR(public_submodules_->level_estimator->ProcessStream(ca));
746 821
747 was_stream_delay_set_ = false; 822 capture_.was_stream_delay_set = false;
748 return kNoError; 823 return kNoError;
749 } 824 }
750 825
751 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, 826 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data,
752 size_t samples_per_channel, 827 size_t samples_per_channel,
753 int rev_sample_rate_hz, 828 int rev_sample_rate_hz,
754 ChannelLayout layout) { 829 ChannelLayout layout) {
755 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); 830 RTC_DCHECK(render_thread_checker_.CalledOnValidThread());
831 rtc::CritScope cs(&crit_render_);
756 const StreamConfig reverse_config = { 832 const StreamConfig reverse_config = {
757 rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), 833 rev_sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout),
758 }; 834 };
759 if (samples_per_channel != reverse_config.num_frames()) { 835 if (samples_per_channel != reverse_config.num_frames()) {
760 return kBadDataLengthError; 836 return kBadDataLengthError;
761 } 837 }
762 return AnalyzeReverseStream(data, reverse_config, reverse_config); 838 return AnalyzeReverseStreamLocked(data, reverse_config, reverse_config);
763 } 839 }
764 840
765 int AudioProcessingImpl::ProcessReverseStream( 841 int AudioProcessingImpl::ProcessReverseStream(
766 const float* const* src, 842 const float* const* src,
767 const StreamConfig& reverse_input_config, 843 const StreamConfig& reverse_input_config,
768 const StreamConfig& reverse_output_config, 844 const StreamConfig& reverse_output_config,
769 float* const* dest) { 845 float* const* dest) {
770 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); 846 RTC_DCHECK(render_thread_checker_.CalledOnValidThread());
771 RETURN_ON_ERR( 847 rtc::CritScope cs(&crit_render_);
772 AnalyzeReverseStream(src, reverse_input_config, reverse_output_config)); 848 RETURN_ON_ERR(AnalyzeReverseStreamLocked(src, reverse_input_config,
849 reverse_output_config));
773 if (is_rev_processed()) { 850 if (is_rev_processed()) {
774 render_audio_->CopyTo(shared_state_.api_format_.reverse_output_stream(), 851 render_.render_audio->CopyTo(formats_.api_format.reverse_output_stream(),
775 dest); 852 dest);
776 } else if (rev_conversion_needed()) { 853 } else if (rev_conversion_needed()) {
777 render_converter_->Convert(src, reverse_input_config.num_samples(), dest, 854 render_.render_converter->Convert(src, reverse_input_config.num_samples(),
778 reverse_output_config.num_samples()); 855 dest,
856 reverse_output_config.num_samples());
779 } else { 857 } else {
780 CopyAudioIfNeeded(src, reverse_input_config.num_frames(), 858 CopyAudioIfNeeded(src, reverse_input_config.num_frames(),
781 reverse_input_config.num_channels(), dest); 859 reverse_input_config.num_channels(), dest);
782 } 860 }
783 861
784 return kNoError; 862 return kNoError;
785 } 863 }
786 864
787 int AudioProcessingImpl::AnalyzeReverseStream( 865 int AudioProcessingImpl::AnalyzeReverseStreamLocked(
788 const float* const* src, 866 const float* const* src,
789 const StreamConfig& reverse_input_config, 867 const StreamConfig& reverse_input_config,
790 const StreamConfig& reverse_output_config) { 868 const StreamConfig& reverse_output_config) {
791 CriticalSectionScoped crit_scoped(crit_);
792 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); 869 RTC_DCHECK(render_thread_checker_.CalledOnValidThread());
793 if (src == NULL) { 870 if (src == nullptr) {
794 return kNullPointerError; 871 return kNullPointerError;
795 } 872 }
796 873
797 if (reverse_input_config.num_channels() <= 0) { 874 if (reverse_input_config.num_channels() <= 0) {
798 return kBadNumberChannelsError; 875 return kBadNumberChannelsError;
799 } 876 }
800 877
801 ProcessingConfig processing_config = shared_state_.api_format_; 878 ProcessingConfig processing_config = formats_.api_format;
802 processing_config.reverse_input_stream() = reverse_input_config; 879 processing_config.reverse_input_stream() = reverse_input_config;
803 processing_config.reverse_output_stream() = reverse_output_config; 880 processing_config.reverse_output_stream() = reverse_output_config;
804 881
805 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); 882 RETURN_ON_ERR(MaybeInitialize(processing_config));
806 assert(reverse_input_config.num_frames() == 883 assert(reverse_input_config.num_frames() ==
807 shared_state_.api_format_.reverse_input_stream().num_frames()); 884 formats_.api_format.reverse_input_stream().num_frames());
808 885
809 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 886 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
810 if (debug_file_->Open()) { 887 if (debug_dump_.debug_file->Open()) {
811 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); 888 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM);
812 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); 889 audioproc::ReverseStream* msg =
890 debug_dump_.render.event_msg->mutable_reverse_stream();
813 const size_t channel_size = 891 const size_t channel_size =
814 sizeof(float) * 892 sizeof(float) * formats_.api_format.reverse_input_stream().num_frames();
815 shared_state_.api_format_.reverse_input_stream().num_frames();
816 for (int i = 0; 893 for (int i = 0;
817 i < shared_state_.api_format_.reverse_input_stream().num_channels(); 894 i < formats_.api_format.reverse_input_stream().num_channels(); ++i)
818 ++i)
819 msg->add_channel(src[i], channel_size); 895 msg->add_channel(src[i], channel_size);
820 RETURN_ON_ERR(WriteMessageToDebugFile()); 896 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
897 &crit_debug_, &debug_dump_.render));
821 } 898 }
822 #endif 899 #endif
823 900
824 render_audio_->CopyFrom(src, 901 render_.render_audio->CopyFrom(src,
825 shared_state_.api_format_.reverse_input_stream()); 902 formats_.api_format.reverse_input_stream());
826 return ProcessReverseStreamLocked(); 903 return ProcessReverseStreamLocked();
827 } 904 }
828 905
829 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) { 906 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) {
830 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); 907 RTC_DCHECK(render_thread_checker_.CalledOnValidThread());
831 RETURN_ON_ERR(AnalyzeReverseStream(frame)); 908 RETURN_ON_ERR(AnalyzeReverseStream(frame));
909 rtc::CritScope cs(&crit_render_);
832 if (is_rev_processed()) { 910 if (is_rev_processed()) {
833 render_audio_->InterleaveTo(frame, true); 911 render_.render_audio->InterleaveTo(frame, true);
834 } 912 }
835 913
836 return kNoError; 914 return kNoError;
837 } 915 }
838 916
839 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { 917 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) {
840 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); 918 RTC_DCHECK(render_thread_checker_.CalledOnValidThread());
841 CriticalSectionScoped crit_scoped(crit_); 919 rtc::CritScope cs(&crit_render_);
842 if (frame == NULL) { 920 if (frame == nullptr) {
843 return kNullPointerError; 921 return kNullPointerError;
844 } 922 }
845 // Must be a native rate. 923 // Must be a native rate.
846 if (frame->sample_rate_hz_ != kSampleRate8kHz && 924 if (frame->sample_rate_hz_ != kSampleRate8kHz &&
847 frame->sample_rate_hz_ != kSampleRate16kHz && 925 frame->sample_rate_hz_ != kSampleRate16kHz &&
848 frame->sample_rate_hz_ != kSampleRate32kHz && 926 frame->sample_rate_hz_ != kSampleRate32kHz &&
849 frame->sample_rate_hz_ != kSampleRate48kHz) { 927 frame->sample_rate_hz_ != kSampleRate48kHz) {
850 return kBadSampleRateError; 928 return kBadSampleRateError;
851 } 929 }
852 // This interface does not tolerate different forward and reverse rates. 930 // This interface does not tolerate different forward and reverse rates.
853 if (frame->sample_rate_hz_ != 931 if (frame->sample_rate_hz_ !=
854 shared_state_.api_format_.input_stream().sample_rate_hz()) { 932 formats_.api_format.input_stream().sample_rate_hz()) {
855 return kBadSampleRateError; 933 return kBadSampleRateError;
856 } 934 }
857 935
858 if (frame->num_channels_ <= 0) { 936 if (frame->num_channels_ <= 0) {
859 return kBadNumberChannelsError; 937 return kBadNumberChannelsError;
860 } 938 }
861 939
862 ProcessingConfig processing_config = shared_state_.api_format_; 940 ProcessingConfig processing_config = formats_.api_format;
863 processing_config.reverse_input_stream().set_sample_rate_hz( 941 processing_config.reverse_input_stream().set_sample_rate_hz(
864 frame->sample_rate_hz_); 942 frame->sample_rate_hz_);
865 processing_config.reverse_input_stream().set_num_channels( 943 processing_config.reverse_input_stream().set_num_channels(
866 frame->num_channels_); 944 frame->num_channels_);
867 processing_config.reverse_output_stream().set_sample_rate_hz( 945 processing_config.reverse_output_stream().set_sample_rate_hz(
868 frame->sample_rate_hz_); 946 frame->sample_rate_hz_);
869 processing_config.reverse_output_stream().set_num_channels( 947 processing_config.reverse_output_stream().set_num_channels(
870 frame->num_channels_); 948 frame->num_channels_);
871 949
872 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); 950 RETURN_ON_ERR(MaybeInitialize(processing_config));
873 if (frame->samples_per_channel_ != 951 if (frame->samples_per_channel_ !=
874 shared_state_.api_format_.reverse_input_stream().num_frames()) { 952 formats_.api_format.reverse_input_stream().num_frames()) {
875 return kBadDataLengthError; 953 return kBadDataLengthError;
876 } 954 }
877 955
878 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 956 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
879 if (debug_file_->Open()) { 957 if (debug_dump_.debug_file->Open()) {
880 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); 958 debug_dump_.render.event_msg->set_type(audioproc::Event::REVERSE_STREAM);
881 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); 959 audioproc::ReverseStream* msg =
960 debug_dump_.render.event_msg->mutable_reverse_stream();
882 const size_t data_size = 961 const size_t data_size =
883 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; 962 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_;
884 msg->set_data(frame->data_, data_size); 963 msg->set_data(frame->data_, data_size);
885 RETURN_ON_ERR(WriteMessageToDebugFile()); 964 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
965 &crit_debug_, &debug_dump_.render));
886 } 966 }
887 #endif 967 #endif
888 render_audio_->DeinterleaveFrom(frame); 968 render_.render_audio->DeinterleaveFrom(frame);
889 return ProcessReverseStreamLocked(); 969 return ProcessReverseStreamLocked();
890 } 970 }
891 971
892 int AudioProcessingImpl::ProcessReverseStreamLocked() { 972 int AudioProcessingImpl::ProcessReverseStreamLocked() {
893 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); 973 RTC_DCHECK(render_thread_checker_.CalledOnValidThread());
894 AudioBuffer* ra = render_audio_.get(); // For brevity. 974 AudioBuffer* ra = render_.render_audio.get(); // For brevity.
895 if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz) { 975 if (formats_.rev_proc_format.sample_rate_hz() == kSampleRate32kHz) {
896 ra->SplitIntoFrequencyBands(); 976 ra->SplitIntoFrequencyBands();
897 } 977 }
898 978
899 if (intelligibility_enabled_) { 979 if (constants_.intelligibility_enabled) {
900 intelligibility_enhancer_->ProcessRenderAudio( 980 // Currently run in single-threaded mode when the intelligibility
901 ra->split_channels_f(kBand0To8kHz), split_rate_, ra->num_channels()); 981 // enhancer is activated.
982 // TODO(peah): Fix to be properly multi-threaded.
983 rtc::CritScope cs(&crit_capture_);
984 public_submodules_->intelligibility_enhancer->ProcessRenderAudio(
985 ra->split_channels_f(kBand0To8kHz), capture_nonlocked_.split_rate,
986 ra->num_channels());
902 } 987 }
903 988
904 RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra)); 989 RETURN_ON_ERR(public_submodules_->echo_cancellation->ProcessRenderAudio(ra));
905 RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra)); 990 RETURN_ON_ERR(
906 if (!use_new_agc_) { 991 public_submodules_->echo_control_mobile->ProcessRenderAudio(ra));
907 RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra)); 992 if (!constants_.use_new_agc) {
993 RETURN_ON_ERR(public_submodules_->gain_control->ProcessRenderAudio(ra));
908 } 994 }
909 995
910 if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz && 996 if (formats_.rev_proc_format.sample_rate_hz() == kSampleRate32kHz &&
911 is_rev_processed()) { 997 is_rev_processed()) {
912 ra->MergeFrequencyBands(); 998 ra->MergeFrequencyBands();
913 } 999 }
914 1000
915 return kNoError; 1001 return kNoError;
916 } 1002 }
917 1003
918 int AudioProcessingImpl::set_stream_delay_ms(int delay) { 1004 int AudioProcessingImpl::set_stream_delay_ms(int delay) {
919 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1005 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
1006 rtc::CritScope cs(&crit_capture_);
920 Error retval = kNoError; 1007 Error retval = kNoError;
921 was_stream_delay_set_ = true; 1008 capture_.was_stream_delay_set = true;
922 delay += delay_offset_ms_; 1009 delay += capture_.delay_offset_ms;
923 1010
924 if (delay < 0) { 1011 if (delay < 0) {
925 delay = 0; 1012 delay = 0;
926 retval = kBadStreamParameterWarning; 1013 retval = kBadStreamParameterWarning;
927 } 1014 }
928 1015
929 // TODO(ajm): the max is rather arbitrarily chosen; investigate. 1016 // TODO(ajm): the max is rather arbitrarily chosen; investigate.
930 if (delay > 500) { 1017 if (delay > 500) {
931 delay = 500; 1018 delay = 500;
932 retval = kBadStreamParameterWarning; 1019 retval = kBadStreamParameterWarning;
933 } 1020 }
934 1021
935 stream_delay_ms_ = delay; 1022 capture_nonlocked_.stream_delay_ms = delay;
936 return retval; 1023 return retval;
937 } 1024 }
938 1025
939 int AudioProcessingImpl::stream_delay_ms() const { 1026 int AudioProcessingImpl::stream_delay_ms() const {
940 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1027 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
941 return stream_delay_ms_; 1028 return capture_nonlocked_.stream_delay_ms;
942 } 1029 }
943 1030
944 bool AudioProcessingImpl::was_stream_delay_set() const { 1031 bool AudioProcessingImpl::was_stream_delay_set() const {
945 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1032 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
946 return was_stream_delay_set_; 1033 return capture_.was_stream_delay_set;
947 } 1034 }
948 1035
949 void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) { 1036 void AudioProcessingImpl::set_stream_key_pressed(bool key_pressed) {
1037 rtc::CritScope cs(&crit_capture_);
950 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1038 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
951 key_pressed_ = key_pressed; 1039 capture_.key_pressed = key_pressed;
952 } 1040 }
953 1041
954 void AudioProcessingImpl::set_delay_offset_ms(int offset) { 1042 void AudioProcessingImpl::set_delay_offset_ms(int offset) {
955 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1043 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
956 CriticalSectionScoped crit_scoped(crit_); 1044 rtc::CritScope cs(&crit_capture_);
957 delay_offset_ms_ = offset; 1045 capture_.delay_offset_ms = offset;
958 } 1046 }
959 1047
960 int AudioProcessingImpl::delay_offset_ms() const { 1048 int AudioProcessingImpl::delay_offset_ms() const {
961 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1049 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
962 return delay_offset_ms_; 1050 rtc::CritScope cs(&crit_capture_);
1051 return capture_.delay_offset_ms;
963 } 1052 }
964 1053
965 int AudioProcessingImpl::StartDebugRecording( 1054 int AudioProcessingImpl::StartDebugRecording(
966 const char filename[AudioProcessing::kMaxFilenameSize]) { 1055 const char filename[AudioProcessing::kMaxFilenameSize]) {
967 CriticalSectionScoped crit_scoped(crit_); 1056 // Run in a single-threaded manner.
1057 rtc::CritScope cs_render(&crit_render_);
1058 rtc::CritScope cs_capture(&crit_capture_);
968 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1059 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
969 static_assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize, ""); 1060 static_assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize, "");
970 1061
971 if (filename == NULL) { 1062 if (filename == nullptr) {
972 return kNullPointerError; 1063 return kNullPointerError;
973 } 1064 }
974 1065
975 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 1066 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
976 // Stop any ongoing recording. 1067 // Stop any ongoing recording.
977 if (debug_file_->Open()) { 1068 if (debug_dump_.debug_file->Open()) {
978 if (debug_file_->CloseFile() == -1) { 1069 if (debug_dump_.debug_file->CloseFile() == -1) {
979 return kFileError; 1070 return kFileError;
980 } 1071 }
981 } 1072 }
982 1073
983 if (debug_file_->OpenFile(filename, false) == -1) { 1074 if (debug_dump_.debug_file->OpenFile(filename, false) == -1) {
984 debug_file_->CloseFile(); 1075 debug_dump_.debug_file->CloseFile();
985 return kFileError; 1076 return kFileError;
986 } 1077 }
987 1078
988 RETURN_ON_ERR(WriteConfigMessage(true)); 1079 RETURN_ON_ERR(WriteConfigMessage(true));
989 RETURN_ON_ERR(WriteInitMessage()); 1080 RETURN_ON_ERR(WriteInitMessage());
990 return kNoError; 1081 return kNoError;
991 #else 1082 #else
992 return kUnsupportedFunctionError; 1083 return kUnsupportedFunctionError;
993 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP 1084 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
994 } 1085 }
995 1086
996 int AudioProcessingImpl::StartDebugRecording(FILE* handle) { 1087 int AudioProcessingImpl::StartDebugRecording(FILE* handle) {
997 CriticalSectionScoped crit_scoped(crit_); 1088 // Run in a single-threaded manner.
1089 rtc::CritScope cs_render(&crit_render_);
1090 rtc::CritScope cs_capture(&crit_capture_);
998 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1091 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
999 1092
1000 if (handle == NULL) { 1093 if (handle == nullptr) {
1001 return kNullPointerError; 1094 return kNullPointerError;
1002 } 1095 }
1003 1096
1004 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 1097 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
1005 // Stop any ongoing recording. 1098 // Stop any ongoing recording.
1006 if (debug_file_->Open()) { 1099 if (debug_dump_.debug_file->Open()) {
1007 if (debug_file_->CloseFile() == -1) { 1100 if (debug_dump_.debug_file->CloseFile() == -1) {
1008 return kFileError; 1101 return kFileError;
1009 } 1102 }
1010 } 1103 }
1011 1104
1012 if (debug_file_->OpenFromFileHandle(handle, true, false) == -1) { 1105 if (debug_dump_.debug_file->OpenFromFileHandle(handle, true, false) == -1) {
1013 return kFileError; 1106 return kFileError;
1014 } 1107 }
1015 1108
1016 RETURN_ON_ERR(WriteConfigMessage(true)); 1109 RETURN_ON_ERR(WriteConfigMessage(true));
1017 RETURN_ON_ERR(WriteInitMessage()); 1110 RETURN_ON_ERR(WriteInitMessage());
1018 return kNoError; 1111 return kNoError;
1019 #else 1112 #else
1020 return kUnsupportedFunctionError; 1113 return kUnsupportedFunctionError;
1021 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP 1114 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
1022 } 1115 }
1023 1116
1024 int AudioProcessingImpl::StartDebugRecordingForPlatformFile( 1117 int AudioProcessingImpl::StartDebugRecordingForPlatformFile(
1025 rtc::PlatformFile handle) { 1118 rtc::PlatformFile handle) {
1119 // Run in a single-threaded manner.
1120 rtc::CritScope cs_render(&crit_render_);
1121 rtc::CritScope cs_capture(&crit_capture_);
1026 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1122 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
1027 FILE* stream = rtc::FdopenPlatformFileForWriting(handle); 1123 FILE* stream = rtc::FdopenPlatformFileForWriting(handle);
1028 return StartDebugRecording(stream); 1124 return StartDebugRecording(stream);
1029 } 1125 }
1030 1126
1031 int AudioProcessingImpl::StopDebugRecording() { 1127 int AudioProcessingImpl::StopDebugRecording() {
1032 CriticalSectionScoped crit_scoped(crit_); 1128 // Run in a single-threaded manner.
1129 rtc::CritScope cs_render(&crit_render_);
1130 rtc::CritScope cs_capture(&crit_capture_);
1033 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1131 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
1034 1132
1035 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 1133 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
1036 // We just return if recording hasn't started. 1134 // We just return if recording hasn't started.
1037 if (debug_file_->Open()) { 1135 if (debug_dump_.debug_file->Open()) {
1038 if (debug_file_->CloseFile() == -1) { 1136 if (debug_dump_.debug_file->CloseFile() == -1) {
1039 return kFileError; 1137 return kFileError;
1040 } 1138 }
1041 } 1139 }
1042 return kNoError; 1140 return kNoError;
1043 #else 1141 #else
1044 return kUnsupportedFunctionError; 1142 return kUnsupportedFunctionError;
1045 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP 1143 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
1046 } 1144 }
1047 1145
1048 EchoCancellation* AudioProcessingImpl::echo_cancellation() const { 1146 EchoCancellation* AudioProcessingImpl::echo_cancellation() const {
1049 return echo_cancellation_; 1147 // Adding a lock here has no effect as it allows any access to the submodule
1148 // from the returned pointer.
1149 return public_submodules_->echo_cancellation;
1050 } 1150 }
1051 1151
1052 EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const { 1152 EchoControlMobile* AudioProcessingImpl::echo_control_mobile() const {
1053 return echo_control_mobile_; 1153 // Adding a lock here has no effect as it allows any access to the submodule
1154 // from the returned pointer.
1155 return public_submodules_->echo_control_mobile;
1054 } 1156 }
1055 1157
1056 GainControl* AudioProcessingImpl::gain_control() const { 1158 GainControl* AudioProcessingImpl::gain_control() const {
1057 if (use_new_agc_) { 1159 // Adding a lock here has no effect as it allows any access to the submodule
1058 return gain_control_for_new_agc_.get(); 1160 // from the returned pointer.
1161 if (constants_.use_new_agc) {
1162 return public_submodules_->gain_control_for_new_agc.get();
1059 } 1163 }
1060 return gain_control_; 1164 return public_submodules_->gain_control;
1061 } 1165 }
1062 1166
1063 HighPassFilter* AudioProcessingImpl::high_pass_filter() const { 1167 HighPassFilter* AudioProcessingImpl::high_pass_filter() const {
1064 return high_pass_filter_; 1168 // Adding a lock here has no effect as it allows any access to the submodule
1169 // from the returned pointer.
1170 return public_submodules_->high_pass_filter;
1065 } 1171 }
1066 1172
1067 LevelEstimator* AudioProcessingImpl::level_estimator() const { 1173 LevelEstimator* AudioProcessingImpl::level_estimator() const {
1068 return level_estimator_; 1174 // Adding a lock here has no effect as it allows any access to the submodule
1175 // from the returned pointer.
1176 return public_submodules_->level_estimator;
1069 } 1177 }
1070 1178
1071 NoiseSuppression* AudioProcessingImpl::noise_suppression() const { 1179 NoiseSuppression* AudioProcessingImpl::noise_suppression() const {
1072 return noise_suppression_; 1180 // Adding a lock here has no effect as it allows any access to the submodule
1181 // from the returned pointer.
1182 return public_submodules_->noise_suppression;
1073 } 1183 }
1074 1184
1075 VoiceDetection* AudioProcessingImpl::voice_detection() const { 1185 VoiceDetection* AudioProcessingImpl::voice_detection() const {
1076 return voice_detection_; 1186 // Adding a lock here has no effect as it allows any access to the submodule
1187 // from the returned pointer.
1188 return public_submodules_->voice_detection;
1077 } 1189 }
1078 1190
1079 bool AudioProcessingImpl::is_data_processed() const { 1191 bool AudioProcessingImpl::is_data_processed() const {
1080 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1192 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
1081 if (beamformer_enabled_) { 1193 if (constants_.beamformer_enabled) {
1082 return true; 1194 return true;
1083 } 1195 }
1084 1196
1085 int enabled_count = 0; 1197 int enabled_count = 0;
1086 for (auto item : component_list_) { 1198 for (auto item : private_submodules_->component_list) {
1087 if (item->is_component_enabled()) { 1199 if (item->is_component_enabled()) {
1088 enabled_count++; 1200 enabled_count++;
1089 } 1201 }
1090 } 1202 }
1091 1203
1092 // Data is unchanged if no components are enabled, or if only level_estimator_ 1204 // Data is unchanged if no components are enabled, or if only
1093 // or voice_detection_ is enabled. 1205 // public_submodules_->level_estimator
1206 // or public_submodules_->voice_detection is enabled.
1094 if (enabled_count == 0) { 1207 if (enabled_count == 0) {
1095 return false; 1208 return false;
1096 } else if (enabled_count == 1) { 1209 } else if (enabled_count == 1) {
1097 if (level_estimator_->is_enabled() || voice_detection_->is_enabled()) { 1210 if (public_submodules_->level_estimator->is_enabled() ||
1211 public_submodules_->voice_detection->is_enabled()) {
1098 return false; 1212 return false;
1099 } 1213 }
1100 } else if (enabled_count == 2) { 1214 } else if (enabled_count == 2) {
1101 if (level_estimator_->is_enabled() && voice_detection_->is_enabled()) { 1215 if (public_submodules_->level_estimator->is_enabled() &&
1216 public_submodules_->voice_detection->is_enabled()) {
1102 return false; 1217 return false;
1103 } 1218 }
1104 } 1219 }
1105 return true; 1220 return true;
1106 } 1221 }
1107 1222
1108 bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const { 1223 bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const {
1109 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1224 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
1110 // Check if we've upmixed or downmixed the audio. 1225 // Check if we've upmixed or downmixed the audio.
1111 return ((shared_state_.api_format_.output_stream().num_channels() != 1226 return ((formats_.api_format.output_stream().num_channels() !=
1112 shared_state_.api_format_.input_stream().num_channels()) || 1227 formats_.api_format.input_stream().num_channels()) ||
1113 is_data_processed || transient_suppressor_enabled_); 1228 is_data_processed || capture_.transient_suppressor_enabled);
1114 } 1229 }
1115 1230
1116 bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const { 1231 bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const {
1117 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1232 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
1118 return (is_data_processed && 1233 return (is_data_processed &&
1119 (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || 1234 (capture_nonlocked_.fwd_proc_format.sample_rate_hz() ==
1120 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz)); 1235 kSampleRate32kHz ||
1236 capture_nonlocked_.fwd_proc_format.sample_rate_hz() ==
1237 kSampleRate48kHz));
1121 } 1238 }
1122 1239
1123 bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const { 1240 bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const {
1124 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1241 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
1125 if (!is_data_processed && !voice_detection_->is_enabled() && 1242 if (!is_data_processed &&
1126 !transient_suppressor_enabled_) { 1243 !public_submodules_->voice_detection->is_enabled() &&
1127 // Only level_estimator_ is enabled. 1244 !capture_.transient_suppressor_enabled) {
1245 // Only public_submodules_->level_estimator is enabled.
1128 return false; 1246 return false;
1129 } else if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || 1247 } else if (capture_nonlocked_.fwd_proc_format.sample_rate_hz() ==
1130 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { 1248 kSampleRate32kHz ||
1131 // Something besides level_estimator_ is enabled, and we have super-wb. 1249 capture_nonlocked_.fwd_proc_format.sample_rate_hz() ==
1250 kSampleRate48kHz) {
1251 // Something besides public_submodules_->level_estimator is enabled, and we
1252 // have super-wb.
1132 return true; 1253 return true;
1133 } 1254 }
1134 return false; 1255 return false;
1135 } 1256 }
1136 1257
1137 bool AudioProcessingImpl::is_rev_processed() const { 1258 bool AudioProcessingImpl::is_rev_processed() const {
1138 RTC_DCHECK(render_thread_checker_.CalledOnValidThread()); 1259 RTC_DCHECK(render_thread_checker_.CalledOnValidThread());
1139 return intelligibility_enabled_ && intelligibility_enhancer_->active(); 1260 return constants_.intelligibility_enabled &&
1261 public_submodules_->intelligibility_enhancer->active();
1140 } 1262 }
1141 1263
1142 bool AudioProcessingImpl::rev_conversion_needed() const { 1264 bool AudioProcessingImpl::rev_conversion_needed() const {
1143 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread() || 1265 return (formats_.api_format.reverse_input_stream() !=
1144 render_thread_checker_.CalledOnValidThread() || 1266 formats_.api_format.reverse_output_stream());
1145 capture_thread_checker_.CalledOnValidThread());
1146 return (shared_state_.api_format_.reverse_input_stream() !=
1147 shared_state_.api_format_.reverse_output_stream());
1148 } 1267 }
1149 1268
1150 void AudioProcessingImpl::InitializeExperimentalAgc() { 1269 void AudioProcessingImpl::InitializeExperimentalAgc() {
1151 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread() || 1270 if (constants_.use_new_agc) {
1152 render_thread_checker_.CalledOnValidThread() || 1271 if (!private_submodules_->agc_manager.get()) {
1153 capture_thread_checker_.CalledOnValidThread()); 1272 private_submodules_->agc_manager.reset(new AgcManagerDirect(
1154 if (use_new_agc_) { 1273 public_submodules_->gain_control,
1155 if (!agc_manager_.get()) { 1274 public_submodules_->gain_control_for_new_agc.get(),
1156 agc_manager_.reset(new AgcManagerDirect(gain_control_, 1275 constants_.agc_startup_min_volume));
1157 gain_control_for_new_agc_.get(),
1158 agc_startup_min_volume_));
1159 } 1276 }
1160 agc_manager_->Initialize(); 1277 private_submodules_->agc_manager->Initialize();
1161 agc_manager_->SetCaptureMuted(output_will_be_muted_); 1278 private_submodules_->agc_manager->SetCaptureMuted(
1279 capture_.output_will_be_muted);
1162 } 1280 }
1163 } 1281 }
1164 1282
1165 void AudioProcessingImpl::InitializeTransient() { 1283 void AudioProcessingImpl::InitializeTransient() {
1166 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread() || 1284 if (capture_.transient_suppressor_enabled) {
1167 render_thread_checker_.CalledOnValidThread() || 1285 if (!public_submodules_->transient_suppressor.get()) {
1168 capture_thread_checker_.CalledOnValidThread()); 1286 public_submodules_->transient_suppressor.reset(new TransientSuppressor());
1169 if (transient_suppressor_enabled_) {
1170 if (!transient_suppressor_.get()) {
1171 transient_suppressor_.reset(new TransientSuppressor());
1172 } 1287 }
1173 transient_suppressor_->Initialize( 1288 public_submodules_->transient_suppressor->Initialize(
1174 fwd_proc_format_.sample_rate_hz(), split_rate_, 1289 capture_nonlocked_.fwd_proc_format.sample_rate_hz(),
1175 shared_state_.api_format_.output_stream().num_channels()); 1290 capture_nonlocked_.split_rate,
1291 formats_.api_format.output_stream().num_channels());
1176 } 1292 }
1177 } 1293 }
1178 1294
1179 void AudioProcessingImpl::InitializeBeamformer() { 1295 void AudioProcessingImpl::InitializeBeamformer() {
1180 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread() || 1296 if (constants_.beamformer_enabled) {
1181 render_thread_checker_.CalledOnValidThread() || 1297 if (!private_submodules_->beamformer) {
1182 capture_thread_checker_.CalledOnValidThread()); 1298 private_submodules_->beamformer.reset(new NonlinearBeamformer(
1183 if (beamformer_enabled_) { 1299 constants_.array_geometry, constants_.target_direction));
1184 if (!beamformer_) {
1185 beamformer_.reset(
1186 new NonlinearBeamformer(array_geometry_, target_direction_));
1187 } 1300 }
1188 beamformer_->Initialize(kChunkSizeMs, split_rate_); 1301 private_submodules_->beamformer->Initialize(kChunkSizeMs,
1302 capture_nonlocked_.split_rate);
1189 } 1303 }
1190 } 1304 }
1191 1305
1192 void AudioProcessingImpl::InitializeIntelligibility() { 1306 void AudioProcessingImpl::InitializeIntelligibility() {
1193 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread() || 1307 if (constants_.intelligibility_enabled) {
1194 render_thread_checker_.CalledOnValidThread() ||
1195 capture_thread_checker_.CalledOnValidThread());
1196 if (intelligibility_enabled_) {
1197 IntelligibilityEnhancer::Config config; 1308 IntelligibilityEnhancer::Config config;
1198 config.sample_rate_hz = split_rate_; 1309 config.sample_rate_hz = capture_nonlocked_.split_rate;
1199 config.num_capture_channels = capture_audio_->num_channels(); 1310 config.num_capture_channels = capture_.capture_audio->num_channels();
1200 config.num_render_channels = render_audio_->num_channels(); 1311 config.num_render_channels = render_.render_audio->num_channels();
1201 intelligibility_enhancer_.reset(new IntelligibilityEnhancer(config)); 1312 public_submodules_->intelligibility_enhancer.reset(
1313 new IntelligibilityEnhancer(config));
1202 } 1314 }
1203 } 1315 }
1204 1316
1205 void AudioProcessingImpl::MaybeUpdateHistograms() { 1317 void AudioProcessingImpl::MaybeUpdateHistograms() {
1206 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1318 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
1207 static const int kMinDiffDelayMs = 60; 1319 static const int kMinDiffDelayMs = 60;
1208 1320
1209 if (echo_cancellation()->is_enabled()) { 1321 if (echo_cancellation()->is_enabled()) {
1210 // Activate delay_jumps_ counters if we know echo_cancellation is runnning. 1322 // Activate delay_jumps_ counters if we know echo_cancellation is runnning.
1211 // If a stream has echo we know that the echo_cancellation is in process. 1323 // If a stream has echo we know that the echo_cancellation is in process.
1212 if (stream_delay_jumps_ == -1 && echo_cancellation()->stream_has_echo()) { 1324 if (capture_.stream_delay_jumps == -1 &&
1213 stream_delay_jumps_ = 0; 1325 echo_cancellation()->stream_has_echo()) {
1326 capture_.stream_delay_jumps = 0;
1214 } 1327 }
1215 if (aec_system_delay_jumps_ == -1 && 1328 if (capture_.aec_system_delay_jumps == -1 &&
1216 echo_cancellation()->stream_has_echo()) { 1329 echo_cancellation()->stream_has_echo()) {
1217 aec_system_delay_jumps_ = 0; 1330 capture_.aec_system_delay_jumps = 0;
1218 } 1331 }
1219 1332
1220 // Detect a jump in platform reported system delay and log the difference. 1333 // Detect a jump in platform reported system delay and log the difference.
1221 const int diff_stream_delay_ms = stream_delay_ms_ - last_stream_delay_ms_; 1334 const int diff_stream_delay_ms =
1222 if (diff_stream_delay_ms > kMinDiffDelayMs && last_stream_delay_ms_ != 0) { 1335 capture_nonlocked_.stream_delay_ms - capture_.last_stream_delay_ms;
1336 if (diff_stream_delay_ms > kMinDiffDelayMs &&
1337 capture_.last_stream_delay_ms != 0) {
1223 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.PlatformReportedStreamDelayJump", 1338 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.PlatformReportedStreamDelayJump",
1224 diff_stream_delay_ms, kMinDiffDelayMs, 1000, 100); 1339 diff_stream_delay_ms, kMinDiffDelayMs, 1000, 100);
1225 if (stream_delay_jumps_ == -1) { 1340 if (capture_.stream_delay_jumps == -1) {
1226 stream_delay_jumps_ = 0; // Activate counter if needed. 1341 capture_.stream_delay_jumps = 0; // Activate counter if needed.
1227 } 1342 }
1228 stream_delay_jumps_++; 1343 capture_.stream_delay_jumps++;
1229 } 1344 }
1230 last_stream_delay_ms_ = stream_delay_ms_; 1345 capture_.last_stream_delay_ms = capture_nonlocked_.stream_delay_ms;
1231 1346
1232 // Detect a jump in AEC system delay and log the difference. 1347 // Detect a jump in AEC system delay and log the difference.
1233 const int frames_per_ms = rtc::CheckedDivExact(split_rate_, 1000); 1348 const int frames_per_ms =
1349 rtc::CheckedDivExact(capture_nonlocked_.split_rate, 1000);
1234 const int aec_system_delay_ms = 1350 const int aec_system_delay_ms =
1235 WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms; 1351 WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms;
1236 const int diff_aec_system_delay_ms = 1352 const int diff_aec_system_delay_ms =
1237 aec_system_delay_ms - last_aec_system_delay_ms_; 1353 aec_system_delay_ms - capture_.last_aec_system_delay_ms;
1238 if (diff_aec_system_delay_ms > kMinDiffDelayMs && 1354 if (diff_aec_system_delay_ms > kMinDiffDelayMs &&
1239 last_aec_system_delay_ms_ != 0) { 1355 capture_.last_aec_system_delay_ms != 0) {
1240 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", 1356 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump",
1241 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, 1357 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000,
1242 100); 1358 100);
1243 if (aec_system_delay_jumps_ == -1) { 1359 if (capture_.aec_system_delay_jumps == -1) {
1244 aec_system_delay_jumps_ = 0; // Activate counter if needed. 1360 capture_.aec_system_delay_jumps = 0; // Activate counter if needed.
1245 } 1361 }
1246 aec_system_delay_jumps_++; 1362 capture_.aec_system_delay_jumps++;
1247 } 1363 }
1248 last_aec_system_delay_ms_ = aec_system_delay_ms; 1364 capture_.last_aec_system_delay_ms = aec_system_delay_ms;
1249 } 1365 }
1250 } 1366 }
1251 1367
1252 void AudioProcessingImpl::UpdateHistogramsOnCallEnd() { 1368 void AudioProcessingImpl::UpdateHistogramsOnCallEnd() {
1369 // Run in a single-threaded manner.
1370 rtc::CritScope cs_render(&crit_render_);
1371 rtc::CritScope cs_capture(&crit_capture_);
1253 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1372 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
1254 CriticalSectionScoped crit_scoped(crit_); 1373
1255 if (stream_delay_jumps_ > -1) { 1374 if (capture_.stream_delay_jumps > -1) {
1256 RTC_HISTOGRAM_ENUMERATION( 1375 RTC_HISTOGRAM_ENUMERATION(
1257 "WebRTC.Audio.NumOfPlatformReportedStreamDelayJumps", 1376 "WebRTC.Audio.NumOfPlatformReportedStreamDelayJumps",
1258 stream_delay_jumps_, 51); 1377 capture_.stream_delay_jumps, 51);
1259 } 1378 }
1260 stream_delay_jumps_ = -1; 1379 capture_.stream_delay_jumps = -1;
1261 last_stream_delay_ms_ = 0; 1380 capture_.last_stream_delay_ms = 0;
1262 1381
1263 if (aec_system_delay_jumps_ > -1) { 1382 if (capture_.aec_system_delay_jumps > -1) {
1264 RTC_HISTOGRAM_ENUMERATION("WebRTC.Audio.NumOfAecSystemDelayJumps", 1383 RTC_HISTOGRAM_ENUMERATION("WebRTC.Audio.NumOfAecSystemDelayJumps",
1265 aec_system_delay_jumps_, 51); 1384 capture_.aec_system_delay_jumps, 51);
1266 } 1385 }
1267 aec_system_delay_jumps_ = -1; 1386 capture_.aec_system_delay_jumps = -1;
1268 last_aec_system_delay_ms_ = 0; 1387 capture_.last_aec_system_delay_ms = 0;
1269 } 1388 }
1270 1389
1271 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP 1390 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
1272 int AudioProcessingImpl::WriteMessageToDebugFile() { 1391 int AudioProcessingImpl::WriteMessageToDebugFile(
1273 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1392 FileWrapper* debug_file,
1274 int32_t size = event_msg_->ByteSize(); 1393 rtc::CriticalSection* crit_debug,
1394 ApmDebugDumpThreadState* debug_state) {
1395 // Thread checker not possible due to function being static.
1396 int32_t size = debug_state->event_msg->ByteSize();
1275 if (size <= 0) { 1397 if (size <= 0) {
1276 return kUnspecifiedError; 1398 return kUnspecifiedError;
1277 } 1399 }
1278 #if defined(WEBRTC_ARCH_BIG_ENDIAN) 1400 #if defined(WEBRTC_ARCH_BIG_ENDIAN)
1279 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be 1401 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be
1280 // pretty safe in assuming little-endian. 1402 // pretty safe in assuming little-endian.
1281 #endif 1403 #endif
1282 1404
1283 if (!event_msg_->SerializeToString(&event_str_)) { 1405 if (!debug_state->event_msg->SerializeToString(&debug_state->event_str)) {
1284 return kUnspecifiedError; 1406 return kUnspecifiedError;
1285 } 1407 }
1286 1408
1287 // Write message preceded by its size. 1409 {
1288 if (!debug_file_->Write(&size, sizeof(int32_t))) { 1410 // Ensure atomic writes of the message.
1289 return kFileError; 1411 rtc::CritScope cs_capture(crit_debug);
1290 } 1412 // Write message preceded by its size.
1291 if (!debug_file_->Write(event_str_.data(), event_str_.length())) { 1413 if (!debug_file->Write(&size, sizeof(int32_t))) {
1292 return kFileError; 1414 return kFileError;
1415 }
1416 if (!debug_file->Write(debug_state->event_str.data(),
1417 debug_state->event_str.length())) {
1418 return kFileError;
1419 }
1293 } 1420 }
1294 1421
1295 event_msg_->Clear(); 1422 debug_state->event_msg->Clear();
1296 1423
1297 return kNoError; 1424 return kNoError;
1298 } 1425 }
1299 1426
1300 int AudioProcessingImpl::WriteInitMessage() { 1427 int AudioProcessingImpl::WriteInitMessage() {
1301 RTC_DCHECK(signal_thread_checker_.CalledOnValidThread() || 1428 debug_dump_.capture.event_msg->set_type(audioproc::Event::INIT);
1302 render_thread_checker_.CalledOnValidThread() || 1429 audioproc::Init* msg = debug_dump_.capture.event_msg->mutable_init();
1303 capture_thread_checker_.CalledOnValidThread()); 1430 msg->set_sample_rate(formats_.api_format.input_stream().sample_rate_hz());
1304 event_msg_->set_type(audioproc::Event::INIT); 1431
1305 audioproc::Init* msg = event_msg_->mutable_init();
1306 msg->set_sample_rate(
1307 shared_state_.api_format_.input_stream().sample_rate_hz());
1308 msg->set_num_input_channels( 1432 msg->set_num_input_channels(
1309 shared_state_.api_format_.input_stream().num_channels()); 1433 formats_.api_format.input_stream().num_channels());
1310 msg->set_num_output_channels( 1434 msg->set_num_output_channels(
1311 shared_state_.api_format_.output_stream().num_channels()); 1435 formats_.api_format.output_stream().num_channels());
1312 msg->set_num_reverse_channels( 1436 msg->set_num_reverse_channels(
1313 shared_state_.api_format_.reverse_input_stream().num_channels()); 1437 formats_.api_format.reverse_input_stream().num_channels());
1314 msg->set_reverse_sample_rate( 1438 msg->set_reverse_sample_rate(
1315 shared_state_.api_format_.reverse_input_stream().sample_rate_hz()); 1439 formats_.api_format.reverse_input_stream().sample_rate_hz());
1316 msg->set_output_sample_rate( 1440 msg->set_output_sample_rate(
1317 shared_state_.api_format_.output_stream().sample_rate_hz()); 1441 formats_.api_format.output_stream().sample_rate_hz());
1318 // TODO(ekmeyerson): Add reverse output fields to event_msg_. 1442 // TODO(ekmeyerson): Add reverse output fields to
1443 // debug_dump_.capture.event_msg.
1319 1444
1320 RETURN_ON_ERR(WriteMessageToDebugFile()); 1445 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
1446 &crit_debug_, &debug_dump_.capture));
1321 return kNoError; 1447 return kNoError;
1322 } 1448 }
1323 1449
1324 int AudioProcessingImpl::WriteConfigMessage(bool forced) { 1450 int AudioProcessingImpl::WriteConfigMessage(bool forced) {
1325 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread()); 1451 RTC_DCHECK(capture_thread_checker_.CalledOnValidThread());
1326 audioproc::Config config; 1452 audioproc::Config config;
1327 1453
1328 config.set_aec_enabled(echo_cancellation_->is_enabled()); 1454 config.set_aec_enabled(public_submodules_->echo_cancellation->is_enabled());
1329 config.set_aec_delay_agnostic_enabled( 1455 config.set_aec_delay_agnostic_enabled(
1330 echo_cancellation_->is_delay_agnostic_enabled()); 1456 public_submodules_->echo_cancellation->is_delay_agnostic_enabled());
1331 config.set_aec_drift_compensation_enabled( 1457 config.set_aec_drift_compensation_enabled(
1332 echo_cancellation_->is_drift_compensation_enabled()); 1458 public_submodules_->echo_cancellation->is_drift_compensation_enabled());
1333 config.set_aec_extended_filter_enabled( 1459 config.set_aec_extended_filter_enabled(
1334 echo_cancellation_->is_extended_filter_enabled()); 1460 public_submodules_->echo_cancellation->is_extended_filter_enabled());
1335 config.set_aec_suppression_level( 1461 config.set_aec_suppression_level(static_cast<int>(
1336 static_cast<int>(echo_cancellation_->suppression_level())); 1462 public_submodules_->echo_cancellation->suppression_level()));
1337 1463
1338 config.set_aecm_enabled(echo_control_mobile_->is_enabled()); 1464 config.set_aecm_enabled(
1465 public_submodules_->echo_control_mobile->is_enabled());
1339 config.set_aecm_comfort_noise_enabled( 1466 config.set_aecm_comfort_noise_enabled(
1340 echo_control_mobile_->is_comfort_noise_enabled()); 1467 public_submodules_->echo_control_mobile->is_comfort_noise_enabled());
1341 config.set_aecm_routing_mode( 1468 config.set_aecm_routing_mode(static_cast<int>(
1342 static_cast<int>(echo_control_mobile_->routing_mode())); 1469 public_submodules_->echo_control_mobile->routing_mode()));
1343 1470
1344 config.set_agc_enabled(gain_control_->is_enabled()); 1471 config.set_agc_enabled(public_submodules_->gain_control->is_enabled());
1345 config.set_agc_mode(static_cast<int>(gain_control_->mode())); 1472 config.set_agc_mode(
1346 config.set_agc_limiter_enabled(gain_control_->is_limiter_enabled()); 1473 static_cast<int>(public_submodules_->gain_control->mode()));
1347 config.set_noise_robust_agc_enabled(use_new_agc_); 1474 config.set_agc_limiter_enabled(
1475 public_submodules_->gain_control->is_limiter_enabled());
1476 config.set_noise_robust_agc_enabled(constants_.use_new_agc);
1348 1477
1349 config.set_hpf_enabled(high_pass_filter_->is_enabled()); 1478 config.set_hpf_enabled(public_submodules_->high_pass_filter->is_enabled());
1350 1479
1351 config.set_ns_enabled(noise_suppression_->is_enabled()); 1480 config.set_ns_enabled(public_submodules_->noise_suppression->is_enabled());
1352 config.set_ns_level(static_cast<int>(noise_suppression_->level())); 1481 config.set_ns_level(
1482 static_cast<int>(public_submodules_->noise_suppression->level()));
1353 1483
1354 config.set_transient_suppression_enabled(transient_suppressor_enabled_); 1484 config.set_transient_suppression_enabled(
1485 capture_.transient_suppressor_enabled);
1355 1486
1356 std::string serialized_config = config.SerializeAsString(); 1487 std::string serialized_config = config.SerializeAsString();
1357 if (!forced && last_serialized_config_ == serialized_config) { 1488 if (!forced &&
1489 debug_dump_.capture.last_serialized_config == serialized_config) {
1358 return kNoError; 1490 return kNoError;
1359 } 1491 }
1360 1492
1361 last_serialized_config_ = serialized_config; 1493 debug_dump_.capture.last_serialized_config = serialized_config;
1362 1494
1363 event_msg_->set_type(audioproc::Event::CONFIG); 1495 debug_dump_.capture.event_msg->set_type(audioproc::Event::CONFIG);
1364 event_msg_->mutable_config()->CopyFrom(config); 1496 debug_dump_.capture.event_msg->mutable_config()->CopyFrom(config);
1365 1497
1366 RETURN_ON_ERR(WriteMessageToDebugFile()); 1498 RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
1499 &crit_debug_, &debug_dump_.capture));
1367 return kNoError; 1500 return kNoError;
1368 } 1501 }
1369 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP 1502 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP
1370 1503
1371 } // namespace webrtc 1504 } // namespace webrtc
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698