OLD | NEW |
---|---|
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include "webrtc/modules/audio_processing/audio_processing_impl.h" | 11 #include "webrtc/modules/audio_processing/audio_processing_impl.h" |
12 | 12 |
13 #include <assert.h> | 13 #include <assert.h> |
14 #include <algorithm> | |
14 | 15 |
15 #include "webrtc/base/checks.h" | 16 #include "webrtc/base/checks.h" |
16 #include "webrtc/base/platform_file.h" | 17 #include "webrtc/base/platform_file.h" |
17 #include "webrtc/common_audio/include/audio_util.h" | 18 #include "webrtc/common_audio/include/audio_util.h" |
18 #include "webrtc/common_audio/channel_buffer.h" | 19 #include "webrtc/common_audio/channel_buffer.h" |
19 #include "webrtc/common_audio/signal_processing/include/signal_processing_librar y.h" | 20 #include "webrtc/common_audio/signal_processing/include/signal_processing_librar y.h" |
20 extern "C" { | 21 extern "C" { |
21 #include "webrtc/modules/audio_processing/aec/aec_core.h" | 22 #include "webrtc/modules/audio_processing/aec/aec_core.h" |
22 } | 23 } |
23 #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h" | 24 #include "webrtc/modules/audio_processing/agc/agc_manager_direct.h" |
(...skipping 17 matching lines...) Expand all Loading... | |
41 | 42 |
42 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 43 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
43 // Files generated at build-time by the protobuf compiler. | 44 // Files generated at build-time by the protobuf compiler. |
44 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD | 45 #ifdef WEBRTC_ANDROID_PLATFORM_BUILD |
45 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" | 46 #include "external/webrtc/webrtc/modules/audio_processing/debug.pb.h" |
46 #else | 47 #else |
47 #include "webrtc/audio_processing/debug.pb.h" | 48 #include "webrtc/audio_processing/debug.pb.h" |
48 #endif | 49 #endif |
49 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 50 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
50 | 51 |
51 #define RETURN_ON_ERR(expr) \ | 52 #define RETURN_ON_ERR(expr) \ |
52 do { \ | 53 do { \ |
53 int err = (expr); \ | 54 int err = (expr); \ |
54 if (err != kNoError) { \ | 55 if (err != kNoError) { \ |
55 return err; \ | 56 return err; \ |
56 } \ | 57 } \ |
57 } while (0) | 58 } while (0) |
58 | 59 |
59 namespace webrtc { | 60 namespace webrtc { |
61 namespace { | |
62 | |
63 static bool LayoutHasKeyboard(AudioProcessing::ChannelLayout layout) { | |
64 switch (layout) { | |
65 case AudioProcessing::kMono: | |
66 case AudioProcessing::kStereo: | |
67 return false; | |
68 case AudioProcessing::kMonoAndKeyboard: | |
69 case AudioProcessing::kStereoAndKeyboard: | |
70 return true; | |
71 } | |
72 | |
73 assert(false); | |
74 return false; | |
75 } | |
76 | |
77 } // namespace | |
60 | 78 |
61 // Throughout webrtc, it's assumed that success is represented by zero. | 79 // Throughout webrtc, it's assumed that success is represented by zero. |
62 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); | 80 static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero"); |
63 | 81 |
64 // This class has two main functionalities: | 82 // This class has two main functionalities: |
65 // | 83 // |
66 // 1) It is returned instead of the real GainControl after the new AGC has been | 84 // 1) It is returned instead of the real GainControl after the new AGC has been |
67 // enabled in order to prevent an outside user from overriding compression | 85 // enabled in order to prevent an outside user from overriding compression |
68 // settings. It doesn't do anything in its implementation, except for | 86 // settings. It doesn't do anything in its implementation, except for |
69 // delegating the const methods and Enable calls to the real GainControl, so | 87 // delegating the const methods and Enable calls to the real GainControl, so |
70 // AGC can still be disabled. | 88 // AGC can still be disabled. |
71 // | 89 // |
72 // 2) It is injected into AgcManagerDirect and implements volume callbacks for | 90 // 2) It is injected into AgcManagerDirect and implements volume callbacks for |
73 // getting and setting the volume level. It just caches this value to be used | 91 // getting and setting the volume level. It just caches this value to be used |
74 // in VoiceEngine later. | 92 // in VoiceEngine later. |
75 class GainControlForNewAgc : public GainControl, public VolumeCallbacks { | 93 class GainControlForNewAgc : public GainControl, public VolumeCallbacks { |
76 public: | 94 public: |
77 explicit GainControlForNewAgc(GainControlImpl* gain_control) | 95 explicit GainControlForNewAgc(GainControlImpl* gain_control) |
78 : real_gain_control_(gain_control), | 96 : real_gain_control_(gain_control), volume_(0) {} |
79 volume_(0) { | |
80 } | |
81 | 97 |
82 // GainControl implementation. | 98 // GainControl implementation. |
83 int Enable(bool enable) override { | 99 int Enable(bool enable) override { |
84 return real_gain_control_->Enable(enable); | 100 return real_gain_control_->Enable(enable); |
85 } | 101 } |
86 bool is_enabled() const override { return real_gain_control_->is_enabled(); } | 102 bool is_enabled() const override { return real_gain_control_->is_enabled(); } |
87 int set_stream_analog_level(int level) override { | 103 int set_stream_analog_level(int level) override { |
88 volume_ = level; | 104 volume_ = level; |
89 return AudioProcessing::kNoError; | 105 return AudioProcessing::kNoError; |
90 } | 106 } |
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
159 gain_control_(NULL), | 175 gain_control_(NULL), |
160 high_pass_filter_(NULL), | 176 high_pass_filter_(NULL), |
161 level_estimator_(NULL), | 177 level_estimator_(NULL), |
162 noise_suppression_(NULL), | 178 noise_suppression_(NULL), |
163 voice_detection_(NULL), | 179 voice_detection_(NULL), |
164 crit_(CriticalSectionWrapper::CreateCriticalSection()), | 180 crit_(CriticalSectionWrapper::CreateCriticalSection()), |
165 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 181 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
166 debug_file_(FileWrapper::Create()), | 182 debug_file_(FileWrapper::Create()), |
167 event_msg_(new audioproc::Event()), | 183 event_msg_(new audioproc::Event()), |
168 #endif | 184 #endif |
169 fwd_in_format_(kSampleRate16kHz, 1), | 185 api_format_({{{kSampleRate16kHz, 1, false}, |
186 {kSampleRate16kHz, 1, false}, | |
187 {kSampleRate16kHz, 1, false}}}), | |
170 fwd_proc_format_(kSampleRate16kHz), | 188 fwd_proc_format_(kSampleRate16kHz), |
171 fwd_out_format_(kSampleRate16kHz, 1), | |
172 rev_in_format_(kSampleRate16kHz, 1), | |
173 rev_proc_format_(kSampleRate16kHz, 1), | 189 rev_proc_format_(kSampleRate16kHz, 1), |
174 split_rate_(kSampleRate16kHz), | 190 split_rate_(kSampleRate16kHz), |
175 stream_delay_ms_(0), | 191 stream_delay_ms_(0), |
176 delay_offset_ms_(0), | 192 delay_offset_ms_(0), |
177 was_stream_delay_set_(false), | 193 was_stream_delay_set_(false), |
178 last_stream_delay_ms_(0), | 194 last_stream_delay_ms_(0), |
179 last_aec_system_delay_ms_(0), | 195 last_aec_system_delay_ms_(0), |
180 stream_delay_jumps_(-1), | 196 stream_delay_jumps_(-1), |
181 aec_system_delay_jumps_(-1), | 197 aec_system_delay_jumps_(-1), |
182 output_will_be_muted_(false), | 198 output_will_be_muted_(false), |
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
246 crit_ = NULL; | 262 crit_ = NULL; |
247 } | 263 } |
248 | 264 |
249 int AudioProcessingImpl::Initialize() { | 265 int AudioProcessingImpl::Initialize() { |
250 CriticalSectionScoped crit_scoped(crit_); | 266 CriticalSectionScoped crit_scoped(crit_); |
251 return InitializeLocked(); | 267 return InitializeLocked(); |
252 } | 268 } |
253 | 269 |
254 int AudioProcessingImpl::set_sample_rate_hz(int rate) { | 270 int AudioProcessingImpl::set_sample_rate_hz(int rate) { |
255 CriticalSectionScoped crit_scoped(crit_); | 271 CriticalSectionScoped crit_scoped(crit_); |
256 return InitializeLocked(rate, | 272 |
257 rate, | 273 ProcessingConfig processing_config = api_format_; |
258 rev_in_format_.rate(), | 274 processing_config.input_stream().set_sample_rate_hz(rate); |
259 fwd_in_format_.num_channels(), | 275 processing_config.output_stream().set_sample_rate_hz(rate); |
260 fwd_out_format_.num_channels(), | 276 return InitializeLocked(processing_config); |
261 rev_in_format_.num_channels()); | |
262 } | 277 } |
263 | 278 |
264 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, | 279 int AudioProcessingImpl::Initialize(int input_sample_rate_hz, |
265 int output_sample_rate_hz, | 280 int output_sample_rate_hz, |
266 int reverse_sample_rate_hz, | 281 int reverse_sample_rate_hz, |
267 ChannelLayout input_layout, | 282 ChannelLayout input_layout, |
268 ChannelLayout output_layout, | 283 ChannelLayout output_layout, |
269 ChannelLayout reverse_layout) { | 284 ChannelLayout reverse_layout) { |
285 const ProcessingConfig processing_config = { | |
286 {{input_sample_rate_hz, ChannelsFromLayout(input_layout), | |
287 LayoutHasKeyboard(input_layout)}, | |
288 {output_sample_rate_hz, ChannelsFromLayout(output_layout), | |
289 LayoutHasKeyboard(output_layout)}, | |
290 {reverse_sample_rate_hz, ChannelsFromLayout(reverse_layout), | |
291 LayoutHasKeyboard(reverse_layout)}}}; | |
292 | |
293 return Initialize(processing_config); | |
294 } | |
295 | |
296 int AudioProcessingImpl::Initialize(const ProcessingConfig& processing_config) { | |
270 CriticalSectionScoped crit_scoped(crit_); | 297 CriticalSectionScoped crit_scoped(crit_); |
271 return InitializeLocked(input_sample_rate_hz, | 298 return InitializeLocked(processing_config); |
272 output_sample_rate_hz, | |
273 reverse_sample_rate_hz, | |
274 ChannelsFromLayout(input_layout), | |
275 ChannelsFromLayout(output_layout), | |
276 ChannelsFromLayout(reverse_layout)); | |
277 } | 299 } |
278 | 300 |
279 int AudioProcessingImpl::InitializeLocked() { | 301 int AudioProcessingImpl::InitializeLocked() { |
280 const int fwd_audio_buffer_channels = beamformer_enabled_ ? | 302 const int fwd_audio_buffer_channels = |
281 fwd_in_format_.num_channels() : | 303 beamformer_enabled_ ? api_format_.input_stream().num_channels() |
282 fwd_out_format_.num_channels(); | 304 : api_format_.output_stream().num_channels(); |
283 render_audio_.reset(new AudioBuffer(rev_in_format_.samples_per_channel(), | 305 if (api_format_.reverse_stream().num_channels() > 0) { |
284 rev_in_format_.num_channels(), | 306 render_audio_.reset(new AudioBuffer( |
285 rev_proc_format_.samples_per_channel(), | 307 api_format_.reverse_stream().num_frames(), |
286 rev_proc_format_.num_channels(), | 308 api_format_.reverse_stream().num_channels(), |
287 rev_proc_format_.samples_per_channel())); | 309 rev_proc_format_.num_frames(), rev_proc_format_.num_channels(), |
288 capture_audio_.reset(new AudioBuffer(fwd_in_format_.samples_per_channel(), | 310 rev_proc_format_.num_frames())); |
289 fwd_in_format_.num_channels(), | 311 } else { |
290 fwd_proc_format_.samples_per_channel(), | 312 render_audio_.reset(nullptr); |
291 fwd_audio_buffer_channels, | 313 } |
292 fwd_out_format_.samples_per_channel())); | 314 capture_audio_.reset(new AudioBuffer( |
315 api_format_.input_stream().num_frames(), | |
316 api_format_.input_stream().num_channels(), fwd_proc_format_.num_frames(), | |
317 fwd_audio_buffer_channels, api_format_.output_stream().num_frames())); | |
293 | 318 |
294 // Initialize all components. | 319 // Initialize all components. |
295 for (auto item : component_list_) { | 320 for (auto item : component_list_) { |
296 int err = item->Initialize(); | 321 int err = item->Initialize(); |
297 if (err != kNoError) { | 322 if (err != kNoError) { |
298 return err; | 323 return err; |
299 } | 324 } |
300 } | 325 } |
301 | 326 |
302 InitializeExperimentalAgc(); | 327 InitializeExperimentalAgc(); |
303 | 328 |
304 InitializeTransient(); | 329 InitializeTransient(); |
305 | 330 |
306 InitializeBeamformer(); | 331 InitializeBeamformer(); |
307 | 332 |
308 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 333 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
309 if (debug_file_->Open()) { | 334 if (debug_file_->Open()) { |
310 int err = WriteInitMessage(); | 335 int err = WriteInitMessage(); |
311 if (err != kNoError) { | 336 if (err != kNoError) { |
312 return err; | 337 return err; |
313 } | 338 } |
314 } | 339 } |
315 #endif | 340 #endif |
316 | 341 |
317 return kNoError; | 342 return kNoError; |
318 } | 343 } |
319 | 344 |
320 int AudioProcessingImpl::InitializeLocked(int input_sample_rate_hz, | 345 int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) { |
321 int output_sample_rate_hz, | 346 for (const auto& stream : config.streams) { |
322 int reverse_sample_rate_hz, | 347 if (stream.num_channels() < 0) { |
323 int num_input_channels, | 348 return kBadNumberChannelsError; |
324 int num_output_channels, | 349 } |
325 int num_reverse_channels) { | 350 if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) { |
326 if (input_sample_rate_hz <= 0 || | 351 return kBadSampleRateError; |
327 output_sample_rate_hz <= 0 || | 352 } |
328 reverse_sample_rate_hz <= 0) { | |
329 return kBadSampleRateError; | |
330 } | 353 } |
331 if (num_output_channels > num_input_channels) { | 354 |
332 return kBadNumberChannelsError; | 355 const int num_in_channels = config.input_stream().num_channels(); |
333 } | 356 const int num_out_channels = config.output_stream().num_channels(); |
334 // Only mono and stereo supported currently. | 357 |
335 if (num_input_channels > 2 || num_input_channels < 1 || | 358 // Need at least one input channel. |
336 num_output_channels > 2 || num_output_channels < 1 || | 359 // Need either one output channel or as many outputs as there are inputs. |
337 num_reverse_channels > 2 || num_reverse_channels < 1) { | 360 if (num_in_channels == 0 || |
338 return kBadNumberChannelsError; | 361 !(num_out_channels == 1 || num_out_channels == num_in_channels)) { |
339 } | |
340 if (beamformer_enabled_ && | |
341 (static_cast<size_t>(num_input_channels) != array_geometry_.size() || | |
342 num_output_channels > 1)) { | |
343 return kBadNumberChannelsError; | 362 return kBadNumberChannelsError; |
344 } | 363 } |
345 | 364 |
346 fwd_in_format_.set(input_sample_rate_hz, num_input_channels); | 365 if (beamformer_enabled_ && |
347 fwd_out_format_.set(output_sample_rate_hz, num_output_channels); | 366 (static_cast<size_t>(num_in_channels) != array_geometry_.size() || |
348 rev_in_format_.set(reverse_sample_rate_hz, num_reverse_channels); | 367 num_out_channels > 1)) { |
368 return kBadNumberChannelsError; | |
369 } | |
370 | |
371 api_format_ = config; | |
349 | 372 |
350 // We process at the closest native rate >= min(input rate, output rate)... | 373 // We process at the closest native rate >= min(input rate, output rate)... |
351 int min_proc_rate = std::min(fwd_in_format_.rate(), fwd_out_format_.rate()); | 374 const int min_proc_rate = |
375 std::min(api_format_.input_stream().sample_rate_hz(), | |
376 api_format_.output_stream().sample_rate_hz()); | |
352 int fwd_proc_rate; | 377 int fwd_proc_rate; |
353 if (supports_48kHz_ && min_proc_rate > kSampleRate32kHz) { | 378 if (supports_48kHz_ && min_proc_rate > kSampleRate32kHz) { |
354 fwd_proc_rate = kSampleRate48kHz; | 379 fwd_proc_rate = kSampleRate48kHz; |
355 } else if (min_proc_rate > kSampleRate16kHz) { | 380 } else if (min_proc_rate > kSampleRate16kHz) { |
356 fwd_proc_rate = kSampleRate32kHz; | 381 fwd_proc_rate = kSampleRate32kHz; |
357 } else if (min_proc_rate > kSampleRate8kHz) { | 382 } else if (min_proc_rate > kSampleRate8kHz) { |
358 fwd_proc_rate = kSampleRate16kHz; | 383 fwd_proc_rate = kSampleRate16kHz; |
359 } else { | 384 } else { |
360 fwd_proc_rate = kSampleRate8kHz; | 385 fwd_proc_rate = kSampleRate8kHz; |
361 } | 386 } |
362 // ...with one exception. | 387 // ...with one exception. |
363 if (echo_control_mobile_->is_enabled() && min_proc_rate > kSampleRate16kHz) { | 388 if (echo_control_mobile_->is_enabled() && min_proc_rate > kSampleRate16kHz) { |
364 fwd_proc_rate = kSampleRate16kHz; | 389 fwd_proc_rate = kSampleRate16kHz; |
365 } | 390 } |
366 | 391 |
367 fwd_proc_format_.set(fwd_proc_rate); | 392 fwd_proc_format_ = StreamConfig(fwd_proc_rate); |
368 | 393 |
369 // We normally process the reverse stream at 16 kHz. Unless... | 394 // We normally process the reverse stream at 16 kHz. Unless... |
370 int rev_proc_rate = kSampleRate16kHz; | 395 int rev_proc_rate = kSampleRate16kHz; |
371 if (fwd_proc_format_.rate() == kSampleRate8kHz) { | 396 if (fwd_proc_format_.sample_rate_hz() == kSampleRate8kHz) { |
372 // ...the forward stream is at 8 kHz. | 397 // ...the forward stream is at 8 kHz. |
373 rev_proc_rate = kSampleRate8kHz; | 398 rev_proc_rate = kSampleRate8kHz; |
374 } else { | 399 } else { |
375 if (rev_in_format_.rate() == kSampleRate32kHz) { | 400 if (api_format_.reverse_stream().sample_rate_hz() == kSampleRate32kHz) { |
376 // ...or the input is at 32 kHz, in which case we use the splitting | 401 // ...or the input is at 32 kHz, in which case we use the splitting |
377 // filter rather than the resampler. | 402 // filter rather than the resampler. |
378 rev_proc_rate = kSampleRate32kHz; | 403 rev_proc_rate = kSampleRate32kHz; |
379 } | 404 } |
380 } | 405 } |
381 | 406 |
382 // Always downmix the reverse stream to mono for analysis. This has been | 407 // Always downmix the reverse stream to mono for analysis. This has been |
383 // demonstrated to work well for AEC in most practical scenarios. | 408 // demonstrated to work well for AEC in most practical scenarios. |
384 rev_proc_format_.set(rev_proc_rate, 1); | 409 rev_proc_format_ = StreamConfig(rev_proc_rate, 1); |
385 | 410 |
386 if (fwd_proc_format_.rate() == kSampleRate32kHz || | 411 if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || |
387 fwd_proc_format_.rate() == kSampleRate48kHz) { | 412 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { |
388 split_rate_ = kSampleRate16kHz; | 413 split_rate_ = kSampleRate16kHz; |
389 } else { | 414 } else { |
390 split_rate_ = fwd_proc_format_.rate(); | 415 split_rate_ = fwd_proc_format_.sample_rate_hz(); |
391 } | 416 } |
392 | 417 |
393 return InitializeLocked(); | 418 return InitializeLocked(); |
394 } | 419 } |
395 | 420 |
396 // Calls InitializeLocked() if any of the audio parameters have changed from | 421 // Calls InitializeLocked() if any of the audio parameters have changed from |
397 // their current values. | 422 // their current values. |
398 int AudioProcessingImpl::MaybeInitializeLocked(int input_sample_rate_hz, | 423 int AudioProcessingImpl::MaybeInitializeLocked( |
399 int output_sample_rate_hz, | 424 const ProcessingConfig& processing_config) { |
400 int reverse_sample_rate_hz, | 425 if (processing_config == api_format_) { |
401 int num_input_channels, | |
402 int num_output_channels, | |
403 int num_reverse_channels) { | |
404 if (input_sample_rate_hz == fwd_in_format_.rate() && | |
405 output_sample_rate_hz == fwd_out_format_.rate() && | |
406 reverse_sample_rate_hz == rev_in_format_.rate() && | |
407 num_input_channels == fwd_in_format_.num_channels() && | |
408 num_output_channels == fwd_out_format_.num_channels() && | |
409 num_reverse_channels == rev_in_format_.num_channels()) { | |
410 return kNoError; | 426 return kNoError; |
411 } | 427 } |
412 return InitializeLocked(input_sample_rate_hz, | 428 return InitializeLocked(processing_config); |
413 output_sample_rate_hz, | |
414 reverse_sample_rate_hz, | |
415 num_input_channels, | |
416 num_output_channels, | |
417 num_reverse_channels); | |
418 } | 429 } |
419 | 430 |
420 void AudioProcessingImpl::SetExtraOptions(const Config& config) { | 431 void AudioProcessingImpl::SetExtraOptions(const Config& config) { |
421 CriticalSectionScoped crit_scoped(crit_); | 432 CriticalSectionScoped crit_scoped(crit_); |
422 for (auto item : component_list_) { | 433 for (auto item : component_list_) { |
423 item->SetExtraOptions(config); | 434 item->SetExtraOptions(config); |
424 } | 435 } |
425 | 436 |
426 if (transient_suppressor_enabled_ != config.Get<ExperimentalNs>().enabled) { | 437 if (transient_suppressor_enabled_ != config.Get<ExperimentalNs>().enabled) { |
427 transient_suppressor_enabled_ = config.Get<ExperimentalNs>().enabled; | 438 transient_suppressor_enabled_ = config.Get<ExperimentalNs>().enabled; |
428 InitializeTransient(); | 439 InitializeTransient(); |
429 } | 440 } |
430 } | 441 } |
431 | 442 |
432 int AudioProcessingImpl::input_sample_rate_hz() const { | 443 int AudioProcessingImpl::input_sample_rate_hz() const { |
433 CriticalSectionScoped crit_scoped(crit_); | 444 CriticalSectionScoped crit_scoped(crit_); |
434 return fwd_in_format_.rate(); | 445 return api_format_.input_stream().sample_rate_hz(); |
435 } | 446 } |
436 | 447 |
437 int AudioProcessingImpl::sample_rate_hz() const { | 448 int AudioProcessingImpl::sample_rate_hz() const { |
438 CriticalSectionScoped crit_scoped(crit_); | 449 CriticalSectionScoped crit_scoped(crit_); |
439 return fwd_in_format_.rate(); | 450 return api_format_.input_stream().sample_rate_hz(); |
440 } | 451 } |
441 | 452 |
442 int AudioProcessingImpl::proc_sample_rate_hz() const { | 453 int AudioProcessingImpl::proc_sample_rate_hz() const { |
443 return fwd_proc_format_.rate(); | 454 return fwd_proc_format_.sample_rate_hz(); |
444 } | 455 } |
445 | 456 |
446 int AudioProcessingImpl::proc_split_sample_rate_hz() const { | 457 int AudioProcessingImpl::proc_split_sample_rate_hz() const { |
447 return split_rate_; | 458 return split_rate_; |
448 } | 459 } |
449 | 460 |
450 int AudioProcessingImpl::num_reverse_channels() const { | 461 int AudioProcessingImpl::num_reverse_channels() const { |
451 return rev_proc_format_.num_channels(); | 462 return rev_proc_format_.num_channels(); |
452 } | 463 } |
453 | 464 |
454 int AudioProcessingImpl::num_input_channels() const { | 465 int AudioProcessingImpl::num_input_channels() const { |
455 return fwd_in_format_.num_channels(); | 466 return api_format_.input_stream().num_channels(); |
456 } | 467 } |
457 | 468 |
458 int AudioProcessingImpl::num_output_channels() const { | 469 int AudioProcessingImpl::num_output_channels() const { |
459 return fwd_out_format_.num_channels(); | 470 return api_format_.output_stream().num_channels(); |
460 } | 471 } |
461 | 472 |
462 void AudioProcessingImpl::set_output_will_be_muted(bool muted) { | 473 void AudioProcessingImpl::set_output_will_be_muted(bool muted) { |
463 CriticalSectionScoped lock(crit_); | 474 CriticalSectionScoped lock(crit_); |
464 output_will_be_muted_ = muted; | 475 output_will_be_muted_ = muted; |
465 if (agc_manager_.get()) { | 476 if (agc_manager_.get()) { |
466 agc_manager_->SetCaptureMuted(output_will_be_muted_); | 477 agc_manager_->SetCaptureMuted(output_will_be_muted_); |
467 } | 478 } |
468 } | 479 } |
469 | 480 |
470 bool AudioProcessingImpl::output_will_be_muted() const { | 481 bool AudioProcessingImpl::output_will_be_muted() const { |
471 CriticalSectionScoped lock(crit_); | 482 CriticalSectionScoped lock(crit_); |
472 return output_will_be_muted_; | 483 return output_will_be_muted_; |
473 } | 484 } |
474 | 485 |
475 int AudioProcessingImpl::ProcessStream(const float* const* src, | 486 int AudioProcessingImpl::ProcessStream(const float* const* src, |
476 int samples_per_channel, | 487 int samples_per_channel, |
477 int input_sample_rate_hz, | 488 int input_sample_rate_hz, |
478 ChannelLayout input_layout, | 489 ChannelLayout input_layout, |
479 int output_sample_rate_hz, | 490 int output_sample_rate_hz, |
480 ChannelLayout output_layout, | 491 ChannelLayout output_layout, |
481 float* const* dest) { | 492 float* const* dest) { |
493 StreamConfig input_stream = api_format_.input_stream(); | |
494 input_stream.set_sample_rate_hz(input_sample_rate_hz); | |
495 input_stream.set_num_channels(ChannelsFromLayout(input_layout)); | |
496 input_stream.set_has_keyboard(LayoutHasKeyboard(input_layout)); | |
497 | |
498 StreamConfig output_stream = api_format_.output_stream(); | |
499 output_stream.set_sample_rate_hz(output_sample_rate_hz); | |
500 output_stream.set_num_channels(ChannelsFromLayout(output_layout)); | |
501 output_stream.set_has_keyboard(LayoutHasKeyboard(output_layout)); | |
502 | |
503 if (samples_per_channel != input_stream.num_frames()) { | |
504 return kBadDataLengthError; | |
505 } | |
506 return ProcessStream(src, input_stream, output_stream, dest); | |
507 } | |
508 | |
509 int AudioProcessingImpl::ProcessStream(const float* const* src, | |
510 const StreamConfig& input_config, | |
511 const StreamConfig& output_config, | |
512 float* const* dest) { | |
482 CriticalSectionScoped crit_scoped(crit_); | 513 CriticalSectionScoped crit_scoped(crit_); |
483 if (!src || !dest) { | 514 if (!src || !dest) { |
484 return kNullPointerError; | 515 return kNullPointerError; |
485 } | 516 } |
486 | 517 |
487 RETURN_ON_ERR(MaybeInitializeLocked(input_sample_rate_hz, | 518 ProcessingConfig processing_config = api_format_; |
488 output_sample_rate_hz, | 519 processing_config.input_stream() = input_config; |
489 rev_in_format_.rate(), | 520 processing_config.output_stream() = output_config; |
490 ChannelsFromLayout(input_layout), | 521 |
491 ChannelsFromLayout(output_layout), | 522 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); |
492 rev_in_format_.num_channels())); | 523 assert(processing_config.input_stream().num_frames() == |
493 if (samples_per_channel != fwd_in_format_.samples_per_channel()) { | 524 api_format_.input_stream().num_frames()); |
494 return kBadDataLengthError; | |
495 } | |
496 | 525 |
497 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 526 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
498 if (debug_file_->Open()) { | 527 if (debug_file_->Open()) { |
499 event_msg_->set_type(audioproc::Event::STREAM); | 528 event_msg_->set_type(audioproc::Event::STREAM); |
500 audioproc::Stream* msg = event_msg_->mutable_stream(); | 529 audioproc::Stream* msg = event_msg_->mutable_stream(); |
501 const size_t channel_size = | 530 const size_t channel_size = |
502 sizeof(float) * fwd_in_format_.samples_per_channel(); | 531 sizeof(float) * api_format_.input_stream().num_frames(); |
503 for (int i = 0; i < fwd_in_format_.num_channels(); ++i) | 532 for (int i = 0; i < api_format_.input_stream().num_channels(); ++i) |
504 msg->add_input_channel(src[i], channel_size); | 533 msg->add_input_channel(src[i], channel_size); |
505 } | 534 } |
506 #endif | 535 #endif |
507 | 536 |
508 capture_audio_->CopyFrom(src, samples_per_channel, input_layout); | 537 capture_audio_->CopyFrom(src, api_format_.input_stream()); |
509 RETURN_ON_ERR(ProcessStreamLocked()); | 538 RETURN_ON_ERR(ProcessStreamLocked()); |
510 capture_audio_->CopyTo(fwd_out_format_.samples_per_channel(), | 539 capture_audio_->CopyTo(api_format_.output_stream(), dest); |
511 output_layout, | |
512 dest); | |
513 | 540 |
514 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 541 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
515 if (debug_file_->Open()) { | 542 if (debug_file_->Open()) { |
516 audioproc::Stream* msg = event_msg_->mutable_stream(); | 543 audioproc::Stream* msg = event_msg_->mutable_stream(); |
517 const size_t channel_size = | 544 const size_t channel_size = |
518 sizeof(float) * fwd_out_format_.samples_per_channel(); | 545 sizeof(float) * api_format_.output_stream().num_frames(); |
mgraczyk
2015/07/23 20:59:34
The previous version of this commit incorrectly us
| |
519 for (int i = 0; i < fwd_out_format_.num_channels(); ++i) | 546 for (int i = 0; i < api_format_.output_stream().num_channels(); ++i) |
520 msg->add_output_channel(dest[i], channel_size); | 547 msg->add_output_channel(dest[i], channel_size); |
521 RETURN_ON_ERR(WriteMessageToDebugFile()); | 548 RETURN_ON_ERR(WriteMessageToDebugFile()); |
522 } | 549 } |
523 #endif | 550 #endif |
524 | 551 |
525 return kNoError; | 552 return kNoError; |
526 } | 553 } |
527 | 554 |
528 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { | 555 int AudioProcessingImpl::ProcessStream(AudioFrame* frame) { |
529 CriticalSectionScoped crit_scoped(crit_); | 556 CriticalSectionScoped crit_scoped(crit_); |
530 if (!frame) { | 557 if (!frame) { |
531 return kNullPointerError; | 558 return kNullPointerError; |
532 } | 559 } |
533 // Must be a native rate. | 560 // Must be a native rate. |
534 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 561 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
535 frame->sample_rate_hz_ != kSampleRate16kHz && | 562 frame->sample_rate_hz_ != kSampleRate16kHz && |
536 frame->sample_rate_hz_ != kSampleRate32kHz && | 563 frame->sample_rate_hz_ != kSampleRate32kHz && |
537 frame->sample_rate_hz_ != kSampleRate48kHz) { | 564 frame->sample_rate_hz_ != kSampleRate48kHz) { |
538 return kBadSampleRateError; | 565 return kBadSampleRateError; |
539 } | 566 } |
540 if (echo_control_mobile_->is_enabled() && | 567 if (echo_control_mobile_->is_enabled() && |
541 frame->sample_rate_hz_ > kSampleRate16kHz) { | 568 frame->sample_rate_hz_ > kSampleRate16kHz) { |
542 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates"; | 569 LOG(LS_ERROR) << "AECM only supports 16 or 8 kHz sample rates"; |
543 return kUnsupportedComponentError; | 570 return kUnsupportedComponentError; |
544 } | 571 } |
545 | 572 |
546 // TODO(ajm): The input and output rates and channels are currently | 573 // TODO(ajm): The input and output rates and channels are currently |
547 // constrained to be identical in the int16 interface. | 574 // constrained to be identical in the int16 interface. |
548 RETURN_ON_ERR(MaybeInitializeLocked(frame->sample_rate_hz_, | 575 ProcessingConfig processing_config = api_format_; |
549 frame->sample_rate_hz_, | 576 processing_config.input_stream().set_sample_rate_hz(frame->sample_rate_hz_); |
550 rev_in_format_.rate(), | 577 processing_config.input_stream().set_num_channels(frame->num_channels_); |
551 frame->num_channels_, | 578 processing_config.output_stream().set_sample_rate_hz(frame->sample_rate_hz_); |
552 frame->num_channels_, | 579 processing_config.output_stream().set_num_channels(frame->num_channels_); |
553 rev_in_format_.num_channels())); | 580 |
554 if (frame->samples_per_channel_ != fwd_in_format_.samples_per_channel()) { | 581 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); |
582 if (frame->samples_per_channel_ != api_format_.input_stream().num_frames()) { | |
555 return kBadDataLengthError; | 583 return kBadDataLengthError; |
556 } | 584 } |
557 | 585 |
558 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 586 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
559 if (debug_file_->Open()) { | 587 if (debug_file_->Open()) { |
560 event_msg_->set_type(audioproc::Event::STREAM); | 588 event_msg_->set_type(audioproc::Event::STREAM); |
561 audioproc::Stream* msg = event_msg_->mutable_stream(); | 589 audioproc::Stream* msg = event_msg_->mutable_stream(); |
562 const size_t data_size = sizeof(int16_t) * | 590 const size_t data_size = |
563 frame->samples_per_channel_ * | 591 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
564 frame->num_channels_; | |
565 msg->set_input_data(frame->data_, data_size); | 592 msg->set_input_data(frame->data_, data_size); |
566 } | 593 } |
567 #endif | 594 #endif |
568 | 595 |
569 capture_audio_->DeinterleaveFrom(frame); | 596 capture_audio_->DeinterleaveFrom(frame); |
570 RETURN_ON_ERR(ProcessStreamLocked()); | 597 RETURN_ON_ERR(ProcessStreamLocked()); |
571 capture_audio_->InterleaveTo(frame, output_copy_needed(is_data_processed())); | 598 capture_audio_->InterleaveTo(frame, output_copy_needed(is_data_processed())); |
572 | 599 |
573 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 600 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
574 if (debug_file_->Open()) { | 601 if (debug_file_->Open()) { |
575 audioproc::Stream* msg = event_msg_->mutable_stream(); | 602 audioproc::Stream* msg = event_msg_->mutable_stream(); |
576 const size_t data_size = sizeof(int16_t) * | 603 const size_t data_size = |
577 frame->samples_per_channel_ * | 604 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
578 frame->num_channels_; | |
579 msg->set_output_data(frame->data_, data_size); | 605 msg->set_output_data(frame->data_, data_size); |
580 RETURN_ON_ERR(WriteMessageToDebugFile()); | 606 RETURN_ON_ERR(WriteMessageToDebugFile()); |
581 } | 607 } |
582 #endif | 608 #endif |
583 | 609 |
584 return kNoError; | 610 return kNoError; |
585 } | 611 } |
586 | 612 |
587 | |
588 int AudioProcessingImpl::ProcessStreamLocked() { | 613 int AudioProcessingImpl::ProcessStreamLocked() { |
589 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 614 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
590 if (debug_file_->Open()) { | 615 if (debug_file_->Open()) { |
591 audioproc::Stream* msg = event_msg_->mutable_stream(); | 616 audioproc::Stream* msg = event_msg_->mutable_stream(); |
592 msg->set_delay(stream_delay_ms_); | 617 msg->set_delay(stream_delay_ms_); |
593 msg->set_drift(echo_cancellation_->stream_drift_samples()); | 618 msg->set_drift(echo_cancellation_->stream_drift_samples()); |
594 msg->set_level(gain_control()->stream_analog_level()); | 619 msg->set_level(gain_control()->stream_analog_level()); |
595 msg->set_keypress(key_pressed_); | 620 msg->set_keypress(key_pressed_); |
596 } | 621 } |
597 #endif | 622 #endif |
598 | 623 |
599 MaybeUpdateHistograms(); | 624 MaybeUpdateHistograms(); |
600 | 625 |
601 AudioBuffer* ca = capture_audio_.get(); // For brevity. | 626 AudioBuffer* ca = capture_audio_.get(); // For brevity. |
602 if (use_new_agc_ && gain_control_->is_enabled()) { | 627 if (use_new_agc_ && gain_control_->is_enabled()) { |
603 agc_manager_->AnalyzePreProcess(ca->channels()[0], | 628 agc_manager_->AnalyzePreProcess(ca->channels()[0], ca->num_channels(), |
604 ca->num_channels(), | 629 fwd_proc_format_.num_frames()); |
605 fwd_proc_format_.samples_per_channel()); | |
606 } | 630 } |
607 | 631 |
608 bool data_processed = is_data_processed(); | 632 bool data_processed = is_data_processed(); |
609 if (analysis_needed(data_processed)) { | 633 if (analysis_needed(data_processed)) { |
610 ca->SplitIntoFrequencyBands(); | 634 ca->SplitIntoFrequencyBands(); |
611 } | 635 } |
612 | 636 |
613 if (beamformer_enabled_) { | 637 if (beamformer_enabled_) { |
614 beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f()); | 638 beamformer_->ProcessChunk(*ca->split_data_f(), ca->split_data_f()); |
615 ca->set_num_channels(1); | 639 ca->set_num_channels(1); |
616 } | 640 } |
617 | 641 |
618 RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca)); | 642 RETURN_ON_ERR(high_pass_filter_->ProcessCaptureAudio(ca)); |
619 RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca)); | 643 RETURN_ON_ERR(gain_control_->AnalyzeCaptureAudio(ca)); |
620 RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca)); | 644 RETURN_ON_ERR(noise_suppression_->AnalyzeCaptureAudio(ca)); |
621 RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca)); | 645 RETURN_ON_ERR(echo_cancellation_->ProcessCaptureAudio(ca)); |
622 | 646 |
623 if (echo_control_mobile_->is_enabled() && noise_suppression_->is_enabled()) { | 647 if (echo_control_mobile_->is_enabled() && noise_suppression_->is_enabled()) { |
624 ca->CopyLowPassToReference(); | 648 ca->CopyLowPassToReference(); |
625 } | 649 } |
626 RETURN_ON_ERR(noise_suppression_->ProcessCaptureAudio(ca)); | 650 RETURN_ON_ERR(noise_suppression_->ProcessCaptureAudio(ca)); |
627 RETURN_ON_ERR(echo_control_mobile_->ProcessCaptureAudio(ca)); | 651 RETURN_ON_ERR(echo_control_mobile_->ProcessCaptureAudio(ca)); |
628 RETURN_ON_ERR(voice_detection_->ProcessCaptureAudio(ca)); | 652 RETURN_ON_ERR(voice_detection_->ProcessCaptureAudio(ca)); |
629 | 653 |
630 if (use_new_agc_ && | 654 if (use_new_agc_ && gain_control_->is_enabled() && |
631 gain_control_->is_enabled() && | |
632 (!beamformer_enabled_ || beamformer_->is_target_present())) { | 655 (!beamformer_enabled_ || beamformer_->is_target_present())) { |
633 agc_manager_->Process(ca->split_bands_const(0)[kBand0To8kHz], | 656 agc_manager_->Process(ca->split_bands_const(0)[kBand0To8kHz], |
634 ca->num_frames_per_band(), | 657 ca->num_frames_per_band(), split_rate_); |
635 split_rate_); | |
636 } | 658 } |
637 RETURN_ON_ERR(gain_control_->ProcessCaptureAudio(ca)); | 659 RETURN_ON_ERR(gain_control_->ProcessCaptureAudio(ca)); |
638 | 660 |
639 if (synthesis_needed(data_processed)) { | 661 if (synthesis_needed(data_processed)) { |
640 ca->MergeFrequencyBands(); | 662 ca->MergeFrequencyBands(); |
641 } | 663 } |
642 | 664 |
643 // TODO(aluebs): Investigate if the transient suppression placement should be | 665 // TODO(aluebs): Investigate if the transient suppression placement should be |
644 // before or after the AGC. | 666 // before or after the AGC. |
645 if (transient_suppressor_enabled_) { | 667 if (transient_suppressor_enabled_) { |
646 float voice_probability = | 668 float voice_probability = |
647 agc_manager_.get() ? agc_manager_->voice_probability() : 1.f; | 669 agc_manager_.get() ? agc_manager_->voice_probability() : 1.f; |
648 | 670 |
649 transient_suppressor_->Suppress(ca->channels_f()[0], | 671 transient_suppressor_->Suppress( |
650 ca->num_frames(), | 672 ca->channels_f()[0], ca->num_frames(), ca->num_channels(), |
651 ca->num_channels(), | 673 ca->split_bands_const_f(0)[kBand0To8kHz], ca->num_frames_per_band(), |
652 ca->split_bands_const_f(0)[kBand0To8kHz], | 674 ca->keyboard_data(), ca->num_keyboard_frames(), voice_probability, |
653 ca->num_frames_per_band(), | 675 key_pressed_); |
654 ca->keyboard_data(), | |
655 ca->num_keyboard_frames(), | |
656 voice_probability, | |
657 key_pressed_); | |
658 } | 676 } |
659 | 677 |
660 // The level estimator operates on the recombined data. | 678 // The level estimator operates on the recombined data. |
661 RETURN_ON_ERR(level_estimator_->ProcessStream(ca)); | 679 RETURN_ON_ERR(level_estimator_->ProcessStream(ca)); |
662 | 680 |
663 was_stream_delay_set_ = false; | 681 was_stream_delay_set_ = false; |
664 return kNoError; | 682 return kNoError; |
665 } | 683 } |
666 | 684 |
667 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, | 685 int AudioProcessingImpl::AnalyzeReverseStream(const float* const* data, |
668 int samples_per_channel, | 686 int samples_per_channel, |
669 int sample_rate_hz, | 687 int sample_rate_hz, |
670 ChannelLayout layout) { | 688 ChannelLayout layout) { |
689 const StreamConfig reverse_config = { | |
690 sample_rate_hz, ChannelsFromLayout(layout), LayoutHasKeyboard(layout), | |
691 }; | |
692 if (samples_per_channel != reverse_config.num_frames()) { | |
693 return kBadDataLengthError; | |
694 } | |
695 return AnalyzeReverseStream(data, reverse_config); | |
696 } | |
697 | |
698 int AudioProcessingImpl::AnalyzeReverseStream( | |
699 const float* const* data, | |
700 const StreamConfig& reverse_config) { | |
671 CriticalSectionScoped crit_scoped(crit_); | 701 CriticalSectionScoped crit_scoped(crit_); |
672 if (data == NULL) { | 702 if (data == NULL) { |
673 return kNullPointerError; | 703 return kNullPointerError; |
674 } | 704 } |
675 | 705 |
676 const int num_channels = ChannelsFromLayout(layout); | 706 if (reverse_config.num_channels() <= 0) { |
677 RETURN_ON_ERR(MaybeInitializeLocked(fwd_in_format_.rate(), | 707 return kBadNumberChannelsError; |
678 fwd_out_format_.rate(), | |
679 sample_rate_hz, | |
680 fwd_in_format_.num_channels(), | |
681 fwd_out_format_.num_channels(), | |
682 num_channels)); | |
683 if (samples_per_channel != rev_in_format_.samples_per_channel()) { | |
684 return kBadDataLengthError; | |
685 } | 708 } |
686 | 709 |
710 ProcessingConfig processing_config = api_format_; | |
711 processing_config.reverse_stream() = reverse_config; | |
712 | |
713 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | |
714 assert(reverse_config.num_frames() == | |
715 api_format_.reverse_stream().num_frames()); | |
716 | |
687 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 717 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
688 if (debug_file_->Open()) { | 718 if (debug_file_->Open()) { |
689 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); | 719 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); |
690 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); | 720 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); |
691 const size_t channel_size = | 721 const size_t channel_size = |
692 sizeof(float) * rev_in_format_.samples_per_channel(); | 722 sizeof(float) * api_format_.reverse_stream().num_frames(); |
693 for (int i = 0; i < num_channels; ++i) | 723 for (int i = 0; i < api_format_.reverse_stream().num_channels(); ++i) |
694 msg->add_channel(data[i], channel_size); | 724 msg->add_channel(data[i], channel_size); |
695 RETURN_ON_ERR(WriteMessageToDebugFile()); | 725 RETURN_ON_ERR(WriteMessageToDebugFile()); |
696 } | 726 } |
697 #endif | 727 #endif |
698 | 728 |
699 render_audio_->CopyFrom(data, samples_per_channel, layout); | 729 render_audio_->CopyFrom(data, api_format_.reverse_stream()); |
700 return AnalyzeReverseStreamLocked(); | 730 return AnalyzeReverseStreamLocked(); |
701 } | 731 } |
702 | 732 |
703 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { | 733 int AudioProcessingImpl::AnalyzeReverseStream(AudioFrame* frame) { |
704 CriticalSectionScoped crit_scoped(crit_); | 734 CriticalSectionScoped crit_scoped(crit_); |
705 if (frame == NULL) { | 735 if (frame == NULL) { |
706 return kNullPointerError; | 736 return kNullPointerError; |
707 } | 737 } |
708 // Must be a native rate. | 738 // Must be a native rate. |
709 if (frame->sample_rate_hz_ != kSampleRate8kHz && | 739 if (frame->sample_rate_hz_ != kSampleRate8kHz && |
710 frame->sample_rate_hz_ != kSampleRate16kHz && | 740 frame->sample_rate_hz_ != kSampleRate16kHz && |
711 frame->sample_rate_hz_ != kSampleRate32kHz && | 741 frame->sample_rate_hz_ != kSampleRate32kHz && |
712 frame->sample_rate_hz_ != kSampleRate48kHz) { | 742 frame->sample_rate_hz_ != kSampleRate48kHz) { |
713 return kBadSampleRateError; | 743 return kBadSampleRateError; |
714 } | 744 } |
715 // This interface does not tolerate different forward and reverse rates. | 745 // This interface does not tolerate different forward and reverse rates. |
716 if (frame->sample_rate_hz_ != fwd_in_format_.rate()) { | 746 if (frame->sample_rate_hz_ != api_format_.input_stream().sample_rate_hz()) { |
717 return kBadSampleRateError; | 747 return kBadSampleRateError; |
718 } | 748 } |
719 | 749 |
720 RETURN_ON_ERR(MaybeInitializeLocked(fwd_in_format_.rate(), | 750 if (frame->num_channels_ <= 0) { |
721 fwd_out_format_.rate(), | 751 return kBadNumberChannelsError; |
722 frame->sample_rate_hz_, | 752 } |
723 fwd_in_format_.num_channels(), | 753 |
724 fwd_in_format_.num_channels(), | 754 ProcessingConfig processing_config = api_format_; |
725 frame->num_channels_)); | 755 processing_config.reverse_stream().set_sample_rate_hz(frame->sample_rate_hz_); |
726 if (frame->samples_per_channel_ != rev_in_format_.samples_per_channel()) { | 756 processing_config.reverse_stream().set_num_channels(frame->num_channels_); |
757 | |
758 RETURN_ON_ERR(MaybeInitializeLocked(processing_config)); | |
759 if (frame->samples_per_channel_ != | |
760 api_format_.reverse_stream().num_frames()) { | |
727 return kBadDataLengthError; | 761 return kBadDataLengthError; |
728 } | 762 } |
729 | 763 |
730 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 764 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
731 if (debug_file_->Open()) { | 765 if (debug_file_->Open()) { |
732 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); | 766 event_msg_->set_type(audioproc::Event::REVERSE_STREAM); |
733 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); | 767 audioproc::ReverseStream* msg = event_msg_->mutable_reverse_stream(); |
734 const size_t data_size = sizeof(int16_t) * | 768 const size_t data_size = |
735 frame->samples_per_channel_ * | 769 sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_; |
736 frame->num_channels_; | |
737 msg->set_data(frame->data_, data_size); | 770 msg->set_data(frame->data_, data_size); |
738 RETURN_ON_ERR(WriteMessageToDebugFile()); | 771 RETURN_ON_ERR(WriteMessageToDebugFile()); |
739 } | 772 } |
740 #endif | 773 #endif |
741 | 774 |
742 render_audio_->DeinterleaveFrom(frame); | 775 render_audio_->DeinterleaveFrom(frame); |
743 return AnalyzeReverseStreamLocked(); | 776 return AnalyzeReverseStreamLocked(); |
744 } | 777 } |
745 | 778 |
746 int AudioProcessingImpl::AnalyzeReverseStreamLocked() { | 779 int AudioProcessingImpl::AnalyzeReverseStreamLocked() { |
747 AudioBuffer* ra = render_audio_.get(); // For brevity. | 780 AudioBuffer* ra = render_audio_.get(); // For brevity. |
748 if (rev_proc_format_.rate() == kSampleRate32kHz) { | 781 if (rev_proc_format_.sample_rate_hz() == kSampleRate32kHz) { |
749 ra->SplitIntoFrequencyBands(); | 782 ra->SplitIntoFrequencyBands(); |
750 } | 783 } |
751 | 784 |
752 RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra)); | 785 RETURN_ON_ERR(echo_cancellation_->ProcessRenderAudio(ra)); |
753 RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra)); | 786 RETURN_ON_ERR(echo_control_mobile_->ProcessRenderAudio(ra)); |
754 if (!use_new_agc_) { | 787 if (!use_new_agc_) { |
755 RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra)); | 788 RETURN_ON_ERR(gain_control_->ProcessRenderAudio(ra)); |
756 } | 789 } |
757 | 790 |
758 return kNoError; | 791 return kNoError; |
(...skipping 181 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
940 } else if (enabled_count == 2) { | 973 } else if (enabled_count == 2) { |
941 if (level_estimator_->is_enabled() && voice_detection_->is_enabled()) { | 974 if (level_estimator_->is_enabled() && voice_detection_->is_enabled()) { |
942 return false; | 975 return false; |
943 } | 976 } |
944 } | 977 } |
945 return true; | 978 return true; |
946 } | 979 } |
947 | 980 |
948 bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const { | 981 bool AudioProcessingImpl::output_copy_needed(bool is_data_processed) const { |
949 // Check if we've upmixed or downmixed the audio. | 982 // Check if we've upmixed or downmixed the audio. |
950 return ((fwd_out_format_.num_channels() != fwd_in_format_.num_channels()) || | 983 return ((api_format_.output_stream().num_channels() != |
984 api_format_.input_stream().num_channels()) || | |
951 is_data_processed || transient_suppressor_enabled_); | 985 is_data_processed || transient_suppressor_enabled_); |
952 } | 986 } |
953 | 987 |
954 bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const { | 988 bool AudioProcessingImpl::synthesis_needed(bool is_data_processed) const { |
955 return (is_data_processed && (fwd_proc_format_.rate() == kSampleRate32kHz || | 989 return (is_data_processed && |
956 fwd_proc_format_.rate() == kSampleRate48kHz)); | 990 (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || |
991 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz)); | |
957 } | 992 } |
958 | 993 |
959 bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const { | 994 bool AudioProcessingImpl::analysis_needed(bool is_data_processed) const { |
960 if (!is_data_processed && !voice_detection_->is_enabled() && | 995 if (!is_data_processed && !voice_detection_->is_enabled() && |
961 !transient_suppressor_enabled_) { | 996 !transient_suppressor_enabled_) { |
962 // Only level_estimator_ is enabled. | 997 // Only level_estimator_ is enabled. |
963 return false; | 998 return false; |
964 } else if (fwd_proc_format_.rate() == kSampleRate32kHz || | 999 } else if (fwd_proc_format_.sample_rate_hz() == kSampleRate32kHz || |
965 fwd_proc_format_.rate() == kSampleRate48kHz) { | 1000 fwd_proc_format_.sample_rate_hz() == kSampleRate48kHz) { |
966 // Something besides level_estimator_ is enabled, and we have super-wb. | 1001 // Something besides level_estimator_ is enabled, and we have super-wb. |
967 return true; | 1002 return true; |
968 } | 1003 } |
969 return false; | 1004 return false; |
970 } | 1005 } |
971 | 1006 |
972 void AudioProcessingImpl::InitializeExperimentalAgc() { | 1007 void AudioProcessingImpl::InitializeExperimentalAgc() { |
973 if (use_new_agc_) { | 1008 if (use_new_agc_) { |
974 if (!agc_manager_.get()) { | 1009 if (!agc_manager_.get()) { |
975 agc_manager_.reset(new AgcManagerDirect(gain_control_, | 1010 agc_manager_.reset(new AgcManagerDirect(gain_control_, |
976 gain_control_for_new_agc_.get(), | 1011 gain_control_for_new_agc_.get(), |
977 agc_startup_min_volume_)); | 1012 agc_startup_min_volume_)); |
978 } | 1013 } |
979 agc_manager_->Initialize(); | 1014 agc_manager_->Initialize(); |
980 agc_manager_->SetCaptureMuted(output_will_be_muted_); | 1015 agc_manager_->SetCaptureMuted(output_will_be_muted_); |
981 } | 1016 } |
982 } | 1017 } |
983 | 1018 |
984 void AudioProcessingImpl::InitializeTransient() { | 1019 void AudioProcessingImpl::InitializeTransient() { |
985 if (transient_suppressor_enabled_) { | 1020 if (transient_suppressor_enabled_) { |
986 if (!transient_suppressor_.get()) { | 1021 if (!transient_suppressor_.get()) { |
987 transient_suppressor_.reset(new TransientSuppressor()); | 1022 transient_suppressor_.reset(new TransientSuppressor()); |
988 } | 1023 } |
989 transient_suppressor_->Initialize(fwd_proc_format_.rate(), | 1024 transient_suppressor_->Initialize( |
990 split_rate_, | 1025 fwd_proc_format_.sample_rate_hz(), split_rate_, |
991 fwd_out_format_.num_channels()); | 1026 api_format_.output_stream().num_channels()); |
992 } | 1027 } |
993 } | 1028 } |
994 | 1029 |
995 void AudioProcessingImpl::InitializeBeamformer() { | 1030 void AudioProcessingImpl::InitializeBeamformer() { |
996 if (beamformer_enabled_) { | 1031 if (beamformer_enabled_) { |
997 if (!beamformer_) { | 1032 if (!beamformer_) { |
998 beamformer_.reset(new NonlinearBeamformer(array_geometry_)); | 1033 beamformer_.reset(new NonlinearBeamformer(array_geometry_)); |
999 } | 1034 } |
1000 beamformer_->Initialize(kChunkSizeMs, split_rate_); | 1035 beamformer_->Initialize(kChunkSizeMs, split_rate_); |
1001 } | 1036 } |
(...skipping 22 matching lines...) Expand all Loading... | |
1024 stream_delay_jumps_ = 0; // Activate counter if needed. | 1059 stream_delay_jumps_ = 0; // Activate counter if needed. |
1025 } | 1060 } |
1026 stream_delay_jumps_++; | 1061 stream_delay_jumps_++; |
1027 } | 1062 } |
1028 last_stream_delay_ms_ = stream_delay_ms_; | 1063 last_stream_delay_ms_ = stream_delay_ms_; |
1029 | 1064 |
1030 // Detect a jump in AEC system delay and log the difference. | 1065 // Detect a jump in AEC system delay and log the difference. |
1031 const int frames_per_ms = rtc::CheckedDivExact(split_rate_, 1000); | 1066 const int frames_per_ms = rtc::CheckedDivExact(split_rate_, 1000); |
1032 const int aec_system_delay_ms = | 1067 const int aec_system_delay_ms = |
1033 WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms; | 1068 WebRtcAec_system_delay(echo_cancellation()->aec_core()) / frames_per_ms; |
1034 const int diff_aec_system_delay_ms = aec_system_delay_ms - | 1069 const int diff_aec_system_delay_ms = |
1035 last_aec_system_delay_ms_; | 1070 aec_system_delay_ms - last_aec_system_delay_ms_; |
1036 if (diff_aec_system_delay_ms > kMinDiffDelayMs && | 1071 if (diff_aec_system_delay_ms > kMinDiffDelayMs && |
1037 last_aec_system_delay_ms_ != 0) { | 1072 last_aec_system_delay_ms_ != 0) { |
1038 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", | 1073 RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AecSystemDelayJump", |
1039 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, | 1074 diff_aec_system_delay_ms, kMinDiffDelayMs, 1000, |
1040 100); | 1075 100); |
1041 if (aec_system_delay_jumps_ == -1) { | 1076 if (aec_system_delay_jumps_ == -1) { |
1042 aec_system_delay_jumps_ = 0; // Activate counter if needed. | 1077 aec_system_delay_jumps_ = 0; // Activate counter if needed. |
1043 } | 1078 } |
1044 aec_system_delay_jumps_++; | 1079 aec_system_delay_jumps_++; |
1045 } | 1080 } |
(...skipping 19 matching lines...) Expand all Loading... | |
1065 last_aec_system_delay_ms_ = 0; | 1100 last_aec_system_delay_ms_ = 0; |
1066 } | 1101 } |
1067 | 1102 |
1068 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP | 1103 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP |
1069 int AudioProcessingImpl::WriteMessageToDebugFile() { | 1104 int AudioProcessingImpl::WriteMessageToDebugFile() { |
1070 int32_t size = event_msg_->ByteSize(); | 1105 int32_t size = event_msg_->ByteSize(); |
1071 if (size <= 0) { | 1106 if (size <= 0) { |
1072 return kUnspecifiedError; | 1107 return kUnspecifiedError; |
1073 } | 1108 } |
1074 #if defined(WEBRTC_ARCH_BIG_ENDIAN) | 1109 #if defined(WEBRTC_ARCH_BIG_ENDIAN) |
1075 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be | 1110 // TODO(ajm): Use little-endian "on the wire". For the moment, we can be |
1076 // pretty safe in assuming little-endian. | 1111 // pretty safe in assuming little-endian. |
1077 #endif | 1112 #endif |
1078 | 1113 |
1079 if (!event_msg_->SerializeToString(&event_str_)) { | 1114 if (!event_msg_->SerializeToString(&event_str_)) { |
1080 return kUnspecifiedError; | 1115 return kUnspecifiedError; |
1081 } | 1116 } |
1082 | 1117 |
1083 // Write message preceded by its size. | 1118 // Write message preceded by its size. |
1084 if (!debug_file_->Write(&size, sizeof(int32_t))) { | 1119 if (!debug_file_->Write(&size, sizeof(int32_t))) { |
1085 return kFileError; | 1120 return kFileError; |
1086 } | 1121 } |
1087 if (!debug_file_->Write(event_str_.data(), event_str_.length())) { | 1122 if (!debug_file_->Write(event_str_.data(), event_str_.length())) { |
1088 return kFileError; | 1123 return kFileError; |
1089 } | 1124 } |
1090 | 1125 |
1091 event_msg_->Clear(); | 1126 event_msg_->Clear(); |
1092 | 1127 |
1093 return kNoError; | 1128 return kNoError; |
1094 } | 1129 } |
1095 | 1130 |
1096 int AudioProcessingImpl::WriteInitMessage() { | 1131 int AudioProcessingImpl::WriteInitMessage() { |
1097 event_msg_->set_type(audioproc::Event::INIT); | 1132 event_msg_->set_type(audioproc::Event::INIT); |
1098 audioproc::Init* msg = event_msg_->mutable_init(); | 1133 audioproc::Init* msg = event_msg_->mutable_init(); |
1099 msg->set_sample_rate(fwd_in_format_.rate()); | 1134 msg->set_sample_rate(api_format_.input_stream().sample_rate_hz()); |
1100 msg->set_num_input_channels(fwd_in_format_.num_channels()); | 1135 msg->set_num_input_channels(api_format_.input_stream().num_channels()); |
1101 msg->set_num_output_channels(fwd_out_format_.num_channels()); | 1136 msg->set_num_output_channels(api_format_.output_stream().num_channels()); |
1102 msg->set_num_reverse_channels(rev_in_format_.num_channels()); | 1137 msg->set_num_reverse_channels(api_format_.reverse_stream().num_channels()); |
1103 msg->set_reverse_sample_rate(rev_in_format_.rate()); | 1138 msg->set_reverse_sample_rate(api_format_.reverse_stream().sample_rate_hz()); |
1104 msg->set_output_sample_rate(fwd_out_format_.rate()); | 1139 msg->set_output_sample_rate(api_format_.output_stream().sample_rate_hz()); |
1105 | 1140 |
1106 int err = WriteMessageToDebugFile(); | 1141 int err = WriteMessageToDebugFile(); |
1107 if (err != kNoError) { | 1142 if (err != kNoError) { |
1108 return err; | 1143 return err; |
1109 } | 1144 } |
1110 | 1145 |
1111 return kNoError; | 1146 return kNoError; |
1112 } | 1147 } |
1113 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP | 1148 #endif // WEBRTC_AUDIOPROC_DEBUG_DUMP |
1114 | 1149 |
1115 } // namespace webrtc | 1150 } // namespace webrtc |
OLD | NEW |