AecDump interface

webrtc/modules/audio_processing/audio_processing_impl.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 287 matching lines @@
 
   return apm;
 }
 
 AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config)
     : AudioProcessingImpl(config, nullptr) {}
 
 AudioProcessingImpl::AudioProcessingImpl(const webrtc::Config& config,
                                          NonlinearBeamformer* beamformer)
     : high_pass_filter_impl_(new HighPassFilterImpl(this)),
+      aec_dump_(AecDump::CreateNullDump()),
       public_submodules_(new ApmPublicSubmodules()),
       private_submodules_(new ApmPrivateSubmodules(beamformer)),
       constants_(config.Get<ExperimentalAgc>().startup_min_volume,
                  config.Get<ExperimentalAgc>().clipped_level_min,
 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
                  false),
 #else
                  config.Get<ExperimentalAgc>().enabled),
 #endif
 #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
@@ skipping 200 matching lines @@
   InitializeEchoCanceller3();
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_dump_.debug_file->is_open()) {
     int err = WriteInitMessage();
     if (err != kNoError) {
       return err;
     }
   }
 #endif
-
+  aec_dump_->WriteInitMessage(formats_.api_format);
   return kNoError;
 }
 
 int AudioProcessingImpl::InitializeLocked(const ProcessingConfig& config) {
   for (const auto& stream : config.streams) {
     if (stream.num_channels() > 0 && stream.sample_rate_hz() <= 0) {
       return kBadSampleRateError;
     }
   }
 
@@ skipping 270 matching lines @@
     debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM);
     audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
     const size_t channel_size =
         sizeof(float) * formats_.api_format.input_stream().num_frames();
     for (size_t i = 0; i < formats_.api_format.input_stream().num_channels();
          ++i)
       msg->add_input_channel(src[i], channel_size);
   }
 #endif
 
+  std::unique_ptr<AecDump::CaptureStreamInfo> stream_info =

[peah-webrtc, 2017/04/07 12:57:15]

I definitely like the aspect of the NullImplementation that it does not require
conditionals. The extra cost is, however, quite significant.

Between lines 820 and 870, for the case where no aecdumping is active
this variant actually adds: 
-9 virtual function calls
-6 ordinary function calls
-2 copies of all the data in the capture buffer.
Only the function calls are about 700 cycles. Some of these can maybe be inlined
by the compiler but as there still are the extra copies of data I don't think
this overhead warrants the not having to use conditionals.

[aleloi, 2017/04/12 11:05:29]

I agree. Now everything is wrapped with if (aec_dump_)

+      aec_dump_->GetCaptureStreamInfo();
+  const size_t channel_size =
+      sizeof(float) * formats_.api_format.input_stream().num_frames();
+
+  {
+    std::vector<rtc::ArrayView<const float>> src_view;
+    for (size_t i = 0; i < formats_.api_format.input_stream().num_channels();
+         ++i) {
+      src_view.emplace_back(src[i], channel_size);
+    }
+    stream_info->AddInput(src_view);
+  }
   capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream());
+
+  RTC_DCHECK(!(public_submodules_->echo_cancellation->is_enabled() &&
+               public_submodules_->echo_control_mobile->is_enabled()));
+
+  stream_info->set_delay(capture_nonlocked_.stream_delay_ms);
+  stream_info->set_drift(
+      public_submodules_->echo_cancellation->stream_drift_samples());
+  stream_info->set_level(gain_control()->stream_analog_level());
+  stream_info->set_keypress(capture_.key_pressed);
+
   RETURN_ON_ERR(ProcessCaptureStreamLocked());
   capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest);
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_dump_.debug_file->is_open()) {
     audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
     const size_t channel_size =
         sizeof(float) * formats_.api_format.output_stream().num_frames();
     for (size_t i = 0; i < formats_.api_format.output_stream().num_channels();
          ++i)
       msg->add_output_channel(dest[i], channel_size);
     RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
                                           &debug_dump_.num_bytes_left_for_log_,
                                           &crit_debug_, &debug_dump_.capture));
   }
 #endif
+  {
+    const size_t channel_size =
+        sizeof(float) * formats_.api_format.output_stream().num_frames();
+    std::vector<rtc::ArrayView<const float>> dest_view;
+    for (size_t i = 0; i < formats_.api_format.output_stream().num_channels();
+         ++i) {
+      dest_view.emplace_back(dest[i], channel_size);
+    }
+    stream_info->AddOutput(dest_view);
+  }
+  aec_dump_->WriteCaptureStreamMessage(std::move(stream_info));
 
   return kNoError;
 }
 
 void AudioProcessingImpl::QueueRenderAudio(AudioBuffer* audio) {
   EchoCancellationImpl::PackRenderAudioBuffer(audio, num_output_channels(),
                                               num_reverse_channels(),
                                               &aec_render_queue_buffer_);
 
   RTC_DCHECK_GE(160, audio->num_frames_per_band());
@@ skipping 217 matching lines @@
     rtc::CritScope cs_render(&crit_render_);
     RETURN_ON_ERR(
         MaybeInitializeCapture(processing_config, reinitialization_required));
   }
   rtc::CritScope cs_capture(&crit_capture_);
   if (frame->samples_per_channel_ !=
       formats_.api_format.input_stream().num_frames()) {
     return kBadDataLengthError;
   }
 
+  std::unique_ptr<AecDump::CaptureStreamInfo> stream_info =
+      aec_dump_->GetCaptureStreamInfo();

[peah-webrtc, 2017/04/07 12:57:15]

Same as above.

[aleloi, 2017/04/12 11:05:29]

Acknowledged.

+  stream_info->AddInput(*frame);
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_dump_.debug_file->is_open()) {
     RETURN_ON_ERR(WriteConfigMessage(false));
 
     debug_dump_.capture.event_msg->set_type(audioproc::Event::STREAM);
     audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
     const size_t data_size =
         sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_;
     msg->set_input_data(frame->data_, data_size);
   }
 #endif
 
   capture_.capture_audio->DeinterleaveFrom(frame);
+
+  RTC_DCHECK(!(public_submodules_->echo_cancellation->is_enabled() &&
+               public_submodules_->echo_control_mobile->is_enabled()));
+
+  stream_info->set_delay(capture_nonlocked_.stream_delay_ms);
+  stream_info->set_drift(
+      public_submodules_->echo_cancellation->stream_drift_samples());
+  stream_info->set_level(gain_control()->stream_analog_level());
+  stream_info->set_keypress(capture_.key_pressed);
+
   RETURN_ON_ERR(ProcessCaptureStreamLocked());
   capture_.capture_audio->InterleaveTo(
       frame, submodule_states_.CaptureMultiBandProcessingActive());
 
+  stream_info->AddOutput(*frame);
+  aec_dump_->WriteCaptureStreamMessage(std::move(stream_info));
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   if (debug_dump_.debug_file->is_open()) {
     audioproc::Stream* msg = debug_dump_.capture.event_msg->mutable_stream();
     const size_t data_size =
         sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_;
     msg->set_output_data(frame->data_, data_size);
     RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
                                           &debug_dump_.num_bytes_left_for_log_,
                                           &crit_debug_, &debug_dump_.capture));
   }
@@ skipping 256 matching lines @@
     const size_t channel_size =
         sizeof(float) * formats_.api_format.reverse_input_stream().num_frames();
     for (size_t i = 0;
          i < formats_.api_format.reverse_input_stream().num_channels(); ++i)
       msg->add_channel(src[i], channel_size);
     RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
                                           &debug_dump_.num_bytes_left_for_log_,
                                           &crit_debug_, &debug_dump_.render));
   }
 #endif
+  std::vector<rtc::ArrayView<const float>> src_view;
+  const size_t channel_size =

[peah-webrtc, 2017/04/07 12:57:15]

Same as above.

[aleloi, 2017/04/12 11:05:29]

Acknowledged.

+      sizeof(float) * formats_.api_format.reverse_input_stream().num_frames();
+
+  for (size_t i = 0;
+       i < formats_.api_format.reverse_input_stream().num_channels(); ++i) {
+    src_view.emplace_back(src[i], channel_size);
+  }
+  aec_dump_->WriteRenderStreamMessage(src_view);
 
   render_.render_audio->CopyFrom(src,
                                  formats_.api_format.reverse_input_stream());
   return ProcessRenderStreamLocked();
 }
 
 int AudioProcessingImpl::ProcessReverseStream(AudioFrame* frame) {
   TRACE_EVENT0("webrtc", "AudioProcessing::ProcessReverseStream_AudioFrame");
   rtc::CritScope cs(&crit_render_);
   if (frame == nullptr) {
@@ skipping 33 matching lines @@
     audioproc::ReverseStream* msg =
         debug_dump_.render.event_msg->mutable_reverse_stream();
     const size_t data_size =
         sizeof(int16_t) * frame->samples_per_channel_ * frame->num_channels_;
     msg->set_data(frame->data_, data_size);
     RETURN_ON_ERR(WriteMessageToDebugFile(debug_dump_.debug_file.get(),
                                           &debug_dump_.num_bytes_left_for_log_,
                                           &crit_debug_, &debug_dump_.render));
   }
 #endif
+  aec_dump_->WriteRenderStreamMessage(*frame);
+
   render_.render_audio->DeinterleaveFrom(frame);
   RETURN_ON_ERR(ProcessRenderStreamLocked());
   render_.render_audio->InterleaveTo(
       frame, submodule_states_.RenderMultiBandProcessingActive());
   return kNoError;
 }
 
 int AudioProcessingImpl::ProcessRenderStreamLocked() {
   AudioBuffer* render_buffer = render_.render_audio.get();  // For brevity.
   if (submodule_states_.RenderMultiBandSubModulesActive() &&
@@ skipping 63 matching lines @@
 void AudioProcessingImpl::set_delay_offset_ms(int offset) {
   rtc::CritScope cs(&crit_capture_);
   capture_.delay_offset_ms = offset;
 }
 
 int AudioProcessingImpl::delay_offset_ms() const {
   rtc::CritScope cs(&crit_capture_);
   return capture_.delay_offset_ms;
 }
 
+void AudioProcessingImpl::StartDebugRecording(
+    std::unique_ptr<AecDump> aec_dump) {
+  rtc::CritScope cs_render(&crit_render_);
+  rtc::CritScope cs_capture(&crit_capture_);
+  aec_dump_ = std::move(aec_dump);
+
+#ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
+  const int error = WriteConfigMessage(true);
+  RTC_DCHECK(error);
+#endif  // WEBRTC_AUDIOPROC_DEBUG_DUMP
+
+  aec_dump_->WriteInitMessage(formats_.api_format);
+}
+
 int AudioProcessingImpl::StartDebugRecording(
     const char filename[AudioProcessing::kMaxFilenameSize],
     int64_t max_log_size_bytes) {
   // Run in a single-threaded manner.
   rtc::CritScope cs_render(&crit_render_);
   rtc::CritScope cs_capture(&crit_capture_);
   static_assert(kMaxFilenameSize == FileWrapper::kMaxFileNameSize, "");
 
   if (filename == nullptr) {
     return kNullPointerError;
   }
 
+  aec_dump_->WriteInitMessage(formats_.api_format);

[peah-webrtc, 2017/04/07 12:57:15]

Is this needed? Nothing is anyway written until the new StartDebugRecording is
called. And at that point, the init message is written as well.

[aleloi, 2017/04/12 11:05:29]

Good point. Removed.

 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   debug_dump_.num_bytes_left_for_log_ = max_log_size_bytes;
   // Stop any ongoing recording.
   debug_dump_.debug_file->CloseFile();
 
   if (!debug_dump_.debug_file->OpenFile(filename, false)) {
     return kFileError;
   }
 
   RETURN_ON_ERR(WriteConfigMessage(true));
   RETURN_ON_ERR(WriteInitMessage());
   return kNoError;
 #else
   return kUnsupportedFunctionError;
 #endif  // WEBRTC_AUDIOPROC_DEBUG_DUMP
 }
 
 int AudioProcessingImpl::StartDebugRecording(FILE* handle,
                                              int64_t max_log_size_bytes) {
   // Run in a single-threaded manner.
   rtc::CritScope cs_render(&crit_render_);
   rtc::CritScope cs_capture(&crit_capture_);
 
   if (handle == nullptr) {
     return kNullPointerError;
   }
 
+  aec_dump_->WriteInitMessage(formats_.api_format);

[peah-webrtc, 2017/04/07 12:57:15]

Same as above.

[aleloi, 2017/04/12 11:05:29]

Acknowledged.

 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   debug_dump_.num_bytes_left_for_log_ = max_log_size_bytes;
 
   // Stop any ongoing recording.
   debug_dump_.debug_file->CloseFile();
 
   if (!debug_dump_.debug_file->OpenFromFileHandle(handle)) {
     return kFileError;
   }
 
@@ skipping 15 matching lines @@
   rtc::CritScope cs_render(&crit_render_);
   rtc::CritScope cs_capture(&crit_capture_);
   FILE* stream = rtc::FdopenPlatformFileForWriting(handle);
   return StartDebugRecording(stream, -1);
 }
 
 int AudioProcessingImpl::StopDebugRecording() {
   // Run in a single-threaded manner.
   rtc::CritScope cs_render(&crit_render_);
   rtc::CritScope cs_capture(&crit_capture_);
+  aec_dump_ = AecDump::CreateNullDump();
 
 #ifdef WEBRTC_AUDIOPROC_DEBUG_DUMP
   // We just return if recording hasn't started.
   debug_dump_.debug_file->CloseFile();
   return kNoError;
 #else
   return kUnsupportedFunctionError;
 #endif  // WEBRTC_AUDIOPROC_DEBUG_DUMP
 }
 
@@ skipping 344 matching lines @@
 
   config.set_ns_enabled(public_submodules_->noise_suppression->is_enabled());
   config.set_ns_level(
       static_cast<int>(public_submodules_->noise_suppression->level()));
 
   config.set_transient_suppression_enabled(
       capture_.transient_suppressor_enabled);
   config.set_intelligibility_enhancer_enabled(
       capture_nonlocked_.intelligibility_enabled);
 
+  InternalAPMConfig apm_config;
+
+  apm_config.aec_enabled = public_submodules_->echo_cancellation->is_enabled();
+  apm_config.aec_delay_agnostic_enabled =
+      public_submodules_->echo_cancellation->is_delay_agnostic_enabled();
+  apm_config.aec_drift_compensation_enabled =
+      public_submodules_->echo_cancellation->is_drift_compensation_enabled();
+  apm_config.aec_extended_filter_enabled =
+      public_submodules_->echo_cancellation->is_extended_filter_enabled();
+  apm_config.aec_suppression_level = static_cast<int>(
+      public_submodules_->echo_cancellation->suppression_level());
+
+  apm_config.aecm_enabled =
+      public_submodules_->echo_control_mobile->is_enabled();
+  apm_config.aecm_comfort_noise_enabled =
+      public_submodules_->echo_control_mobile->is_comfort_noise_enabled();
+  apm_config.aecm_routing_mode =
+      static_cast<int>(public_submodules_->echo_control_mobile->routing_mode());
+
+  apm_config.agc_enabled = public_submodules_->gain_control->is_enabled();
+  apm_config.agc_mode =
+      static_cast<int>(public_submodules_->gain_control->mode());
+  apm_config.agc_limiter_enabled =
+      public_submodules_->gain_control->is_limiter_enabled();
+  apm_config.noise_robust_agc_enabled = constants_.use_experimental_agc;
+
+  apm_config.hpf_enabled = config_.high_pass_filter.enabled;
+
+  apm_config.ns_enabled = public_submodules_->noise_suppression->is_enabled();
+  apm_config.ns_level =
+      static_cast<int>(public_submodules_->noise_suppression->level());
+
+  apm_config.transient_suppression_enabled =
+      capture_.transient_suppressor_enabled;
+  apm_config.intelligibility_enhancer_enabled =
+      capture_nonlocked_.intelligibility_enabled;
+
   std::string experiments_description =
       public_submodules_->echo_cancellation->GetExperimentsDescription();
   // TODO(peah): Add semicolon-separated concatenations of experiment
   // descriptions for other submodules.
   if (capture_nonlocked_.level_controller_enabled) {
     experiments_description += "LevelController;";
   }
   if (constants_.agc_clipped_level_min != kClippedLevelMin) {
     experiments_description += "AgcClippingLevelExperiment;";
   }
   if (capture_nonlocked_.echo_canceller3_enabled) {
     experiments_description += "EchoCanceller3;";
   }
   config.set_experiments_description(experiments_description);
 
+  apm_config.experiments_description = experiments_description;
+  aec_dump_->WriteConfig(apm_config, forced);
+
   std::string serialized_config = config.SerializeAsString();
   if (!forced &&
       debug_dump_.capture.last_serialized_config == serialized_config) {
     return kNoError;
   }
 
   debug_dump_.capture.last_serialized_config = serialized_config;
 
   debug_dump_.capture.event_msg->set_type(audioproc::Event::CONFIG);
   debug_dump_.capture.event_msg->mutable_config()->CopyFrom(config);
@@ skipping 23 matching lines @@
       capture_processing_format(kSampleRate16kHz),
       split_rate(kSampleRate16kHz) {}
 
 AudioProcessingImpl::ApmCaptureState::~ApmCaptureState() = default;
 
 AudioProcessingImpl::ApmRenderState::ApmRenderState() = default;
 
 AudioProcessingImpl::ApmRenderState::~ApmRenderState() = default;
 
 }  // namespace webrtc