New module for the adaptive level controlling functionality in the audio processing module

webrtc/modules/audio_processing/level_controller/level_controller_complexity_unittest.cc

+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include <vector>

[hlundin-webrtc, 2016/06/27 11:21:15]

Nit: I tend to like a blank line after the copyright notice.

[peah-webrtc, 2016/06/27 22:51:49]

Done.

+
+#include "testing/gtest/include/gtest/gtest.h"
+#include "webrtc/base/array_view.h"
+#include "webrtc/base/random.h"
+#include "webrtc/modules/audio_processing/audio_buffer.h"
+#include "webrtc/modules/audio_processing/include/audio_processing.h"
+#include "webrtc/modules/audio_processing/level_controller/level_controller.h"
+#include "webrtc/modules/audio_processing/test/audio_buffer_tools.h"
+#include "webrtc/modules/audio_processing/test/bitexactness_tools.h"
+#include "webrtc/system_wrappers/include/clock.h"
+#include "webrtc/test/testsupport/perf_test.h"
+
+namespace webrtc {
+namespace {
+
+const size_t kNumFramesToProcess = 100;
+
+struct SimulatorBuffers {
+  SimulatorBuffers(int render_input_sample_rate_hz,
+                   int capture_input_sample_rate_hz,
+                   int render_output_sample_rate_hz,
+                   int capture_output_sample_rate_hz,
+                   size_t num_render_input_channels,
+                   size_t num_capture_input_channels,
+                   size_t num_render_output_channels,
+                   size_t num_capture_output_channels) {
+    Random rand_gen(42);
+    CreateConfigAndBuffer(render_input_sample_rate_hz,
+                          num_render_input_channels, &rand_gen,
+                          &render_input_buffer, &render_input_config,
+                          &render_input, &render_input_samples);
+
+    CreateConfigAndBuffer(render_output_sample_rate_hz,
+                          num_render_output_channels, &rand_gen,
+                          &render_output_buffer, &render_output_config,
+                          &render_output, &render_output_samples);
+
+    CreateConfigAndBuffer(capture_input_sample_rate_hz,
+                          num_capture_input_channels, &rand_gen,
+                          &capture_input_buffer, &capture_input_config,
+                          &capture_input, &capture_input_samples);
+
+    CreateConfigAndBuffer(capture_output_sample_rate_hz,
+                          num_capture_output_channels, &rand_gen,
+                          &capture_output_buffer, &capture_output_config,
+                          &capture_output, &capture_output_samples);
+
+    UpdateInputBuffers();
+  }
+
+  void CreateConfigAndBuffer(int sample_rate_hz,
+                             size_t num_channels,
+                             Random* rand_gen,
+                             std::unique_ptr<AudioBuffer>* buffer,
+                             StreamConfig* config,
+                             std::vector<float*>* buffer_data,
+                             std::vector<float>* buffer_data_samples) {
+    int samples_per_channel = rtc::CheckedDivExact(sample_rate_hz, 100);
+    *config = StreamConfig(sample_rate_hz, num_channels, false);
+    buffer->reset(new AudioBuffer(config->num_frames(), config->num_channels(),
+                                  config->num_frames(), config->num_channels(),
+                                  config->num_frames()));
+
+    buffer_data_samples->resize(samples_per_channel * num_channels);
+    for (auto& v : *buffer_data_samples) {
+      v = rand_gen->Rand<float>();
+    }
+
+    buffer_data->resize(num_channels);
+    for (size_t ch = 0; ch < num_channels; ++ch) {
+      (*buffer_data)[ch] = &(*buffer_data_samples)[ch * samples_per_channel];
+    }
+  }
+
+  void UpdateInputBuffers() {
+    test::CopyVectorToAudioBuffer(capture_input_config, capture_input_samples,
+                                  capture_input_buffer.get());
+    test::CopyVectorToAudioBuffer(render_input_config, render_input_samples,
+                                  render_input_buffer.get());
+  }
+
+  std::unique_ptr<AudioBuffer> render_input_buffer;
+  std::unique_ptr<AudioBuffer> capture_input_buffer;
+  std::unique_ptr<AudioBuffer> render_output_buffer;
+  std::unique_ptr<AudioBuffer> capture_output_buffer;
+  StreamConfig render_input_config;
+  StreamConfig capture_input_config;
+  StreamConfig render_output_config;
+  StreamConfig capture_output_config;
+  std::vector<float*> render_input;
+  std::vector<float> render_input_samples;
+  std::vector<float*> capture_input;
+  std::vector<float> capture_input_samples;
+  std::vector<float*> render_output;
+  std::vector<float> render_output_samples;
+  std::vector<float*> capture_output;
+  std::vector<float> capture_output_samples;
+};
+
+class SubmodulePerformanceTimer {
+ public:
+  explicit SubmodulePerformanceTimer(size_t num_values_to_store)

[hlundin-webrtc, 2016/06/27 11:21:15]

Why have an explicit limitation on the size? Vectors resize easily and
automatically.

[peah-webrtc, 2016/06/27 22:51:48]

Good point! But since some of the timers are nested, it would be nice to not
have any overhead of a resize. I rewrote it to be slightly different though.
Could you PTAL.

Done.

+      : clock_(webrtc::Clock::GetRealTimeClock()) {
+    timestamps_.resize(num_values_to_store);

[hlundin-webrtc, 2016/06/27 11:21:15]

You can do this in the initializer list as timestamps_(num_values_to_store).

[peah-webrtc, 2016/06/27 22:51:48]

I removed the fixed size limitation so this is no longer needed.

Done.

+  }
+
+  void ResetTimer() { start_timestamp_ = clock_->TimeInMicroseconds(); }

[hlundin-webrtc, 2016/06/27 11:21:15]

Make start_timestamp_ an rtc::Optional. Initialize it to empty. Give it a value
here. CHECK that it has a value in AddTimeStamp() and reset it to empty at the
end of AddTimeStamp().

[peah-webrtc, 2016/06/27 22:51:48]

Great suggestion!

Done.

+  void AddTimeStamp() {
+    RTC_CHECK_LE(num_timestamps_stored_, timestamps_.size());
+    timestamps_[num_timestamps_stored_] =

[hlundin-webrtc, 2016/06/27 11:21:15]

You don't need num_timestamps_stored_. Use vector::push_back to insert, and it
will grow when needed.

[peah-webrtc, 2016/06/27 22:51:49]

Done.

+        clock_->TimeInMicroseconds() - start_timestamp_;
+    ++num_timestamps_stored_;
+  }
+
+  double GetDurationAverage() const {
+    RTC_DCHECK_EQ(num_timestamps_stored_, timestamps_.size());
+    int64_t durations_sum = 0;

[hlundin-webrtc, 2016/06/27 11:21:16]

One-liner method:
return timestamps_.empty() ? 0.0 :
static_cast<double>(std::accumulate(timestamps_.begin(), timestamps_.end(), 0))
/ timestamps_.size();

(Alternatively, DCHECK(!timestamps_.empty()) first, and then return the
division.)

[peah-webrtc, 2016/06/27 22:51:49]

Great suggestion! I chose the latter variant.

Done.

+    for (auto timestamp : timestamps_) {
+      durations_sum += timestamp;
+    }
+
+    RTC_DCHECK_LT(0u, timestamps_.size());
+    return static_cast<double>(durations_sum) / timestamps_.size();
+  }
+
+  double GetDurationStandardDeviationGetVarianceTime() const {

[hlundin-webrtc, 2016/06/27 11:21:16]

This is an awkward method name. What does it mean?

[peah-webrtc, 2016/06/27 22:51:48]

Agree! No idea! Changed the name.

Done.

+    int32_t average_duration = GetDurationAverage();

[hlundin-webrtc, 2016/06/27 11:21:15]

Why int32_t? GetDurationAverage() returns a double, and timestamps_ is a vector
of int64_t; use one of these types.

[peah-webrtc, 2016/06/27 22:51:49]

I changed it to double.

Done.

+    int64_t variance = 0;

[hlundin-webrtc, 2016/06/27 11:21:16]

You can make this a one-liner too, with a lambda, but it may be a bit obscure:

int64_t variance = std::accumulate(timestamps_.begin(), timestamps_.end(), 0,
[average_duration](const int64_t& a, const int64_t& b){ return a + (b -
average_duration); });

I'm not sure it improves the code.

[peah-webrtc, 2016/06/27 22:51:48]

Not sure either. But it looks great so I added it!

Done.

+    for (auto timestamp : timestamps_) {
+      variance += timestamp - average_duration;
+    }
+
+    RTC_DCHECK_LT(0u, timestamps_.size());
+    return sqrt(static_cast<double>(variance) / timestamps_.size());
+  }
+
+ private:
+  webrtc::Clock* clock_;
+  int64_t start_timestamp_ = 0;
+  size_t num_timestamps_stored_ = 0;
+  std::vector<int64_t> timestamps_;

[hlundin-webrtc, 2016/06/27 11:21:15]

timestamps_us_

[peah-webrtc, 2016/06/27 22:51:49]

Done.

+};
+
+std::string FormPerformanceMeasureString(
+    const SubmodulePerformanceTimer& timer) {
+  double average = timer.GetDurationAverage();

[hlundin-webrtc, 2016/06/27 11:21:15]

You don't need these local variables. Just plug the method calls directly into
the to_string calls.

[peah-webrtc, 2016/06/27 22:51:49]

Done.

+  double standard_dev = timer.GetDurationStandardDeviationGetVarianceTime();
+  std::string s = std::to_string(average);
+  s += ", ";
+  s += std::to_string(standard_dev);
+  return s;
+}
+
+void RunStandatoleSubmodule(int sample_rate_hz, size_t num_channels) {

[hlundin-webrtc, 2016/06/27 11:21:16]

Standatole?

[peah-webrtc, 2016/06/27 22:51:49]

Absolutely!

Done.

+  SimulatorBuffers buffers(sample_rate_hz, sample_rate_hz, sample_rate_hz,
+                           sample_rate_hz, num_channels, num_channels,
+                           num_channels, num_channels);
+  SubmodulePerformanceTimer timer(kNumFramesToProcess);
+
+  LevelController level_controller;
+  level_controller.Initialize(sample_rate_hz, num_channels);
+
+  for (size_t frame_no = 0; frame_no < kNumFramesToProcess; ++frame_no) {
+    buffers.UpdateInputBuffers();
+
+    timer.ResetTimer();

[hlundin-webrtc, 2016/06/27 11:21:15]

Based on the use case, I suggest you rename the timer methods.

ResetTimer -> StartTimer, or simply Tick (in reference to Matlab's tick-tock
functions).
AddTimeStamp -> StopTimer, StopTimerAndLog, or simply Tock.

[peah-webrtc, 2016/06/27 22:51:48]

Makes sense!

Done.

+    level_controller.Process(buffers.capture_input_buffer.get());
+    timer.AddTimeStamp();
+  }
+  webrtc::test::PrintResultMeanAndError(
+      "level_controller_call_durations",
+      "_" + std::to_string(sample_rate_hz) + "Hz_" +
+          std::to_string(num_channels) + "_channels",
+      "StandaloneLevelControl", FormPerformanceMeasureString(timer), "us",
+      false);
+}
+
+void RunTogetherWithApm(std::string test_description,
+                        int render_input_sample_rate_hz,
+                        int render_output_sample_rate_hz,
+                        int capture_input_sample_rate_hz,
+                        int capture_output_sample_rate_hz,
+                        size_t num_channels,
+                        bool use_mobile_aec,
+                        bool include_default_apm_processing) {
+  SimulatorBuffers buffers(
+      render_input_sample_rate_hz, capture_input_sample_rate_hz,
+      render_output_sample_rate_hz, capture_output_sample_rate_hz, num_channels,
+      num_channels, num_channels, num_channels);
+  SubmodulePerformanceTimer render_timer(kNumFramesToProcess);
+  SubmodulePerformanceTimer capture_timer(kNumFramesToProcess);
+  SubmodulePerformanceTimer total_timer(kNumFramesToProcess);
+
+  Config config;
+  if (include_default_apm_processing) {
+    config.Set<DelayAgnostic>(new DelayAgnostic(true));
+    config.Set<ExtendedFilter>(new ExtendedFilter(true));
+  }
+  config.Set<LevelControl>(new LevelControl(true));
+
+  std::unique_ptr<AudioProcessing> apm;
+  apm.reset(AudioProcessing::Create(config));

[hlundin-webrtc, 2016/06/27 11:21:16]

Can Create fail? If so, ASSERT_TRUE that apm is ok.

[peah-webrtc, 2016/06/27 22:51:49]

Good point! It can fail, so I'll add that.

Done.

+
+  apm->gain_control()->Enable(include_default_apm_processing);

[hlundin-webrtc, 2016/06/27 11:21:15]

All the Enable methods seem to have a return value. Use ASSERT_EQ to ensure that
they don't fail.

[peah-webrtc, 2016/06/27 22:51:48]

Done.

+  if (use_mobile_aec) {
+    apm->echo_cancellation()->Enable(false);
+    apm->echo_control_mobile()->Enable(include_default_apm_processing);
+  } else {
+    apm->echo_cancellation()->Enable(include_default_apm_processing);
+    apm->echo_control_mobile()->Enable(false);
+  }
+  apm->high_pass_filter()->Enable(include_default_apm_processing);
+  apm->noise_suppression()->Enable(include_default_apm_processing);
+  apm->voice_detection()->Enable(include_default_apm_processing);
+  apm->level_estimator()->Enable(include_default_apm_processing);
+
+  StreamConfig render_input_config(render_input_sample_rate_hz, num_channels,
+                                   false);
+  StreamConfig render_output_config(render_output_sample_rate_hz, num_channels,
+                                    false);
+  StreamConfig capture_input_config(capture_input_sample_rate_hz, num_channels,
+                                    false);
+  StreamConfig capture_output_config(capture_output_sample_rate_hz,
+                                     num_channels, false);
+
+  for (size_t frame_no = 0; frame_no < kNumFramesToProcess; ++frame_no) {
+    buffers.UpdateInputBuffers();
+
+    total_timer.ResetTimer();
+    render_timer.ResetTimer();
+    int error = apm->ProcessReverseStream(

[hlundin-webrtc, 2016/06/27 11:21:15]

Don't store the return value.
ASSERT_EQ(AudioProcessing::kNoError, apm->Process...);

Here and below.

[peah-webrtc, 2016/06/27 22:51:48]

Done.

+        &buffers.render_input[0], render_input_config, render_output_config,
+        &buffers.render_output[0]);
+
+    ASSERT_EQ(AudioProcessing::kNoError, error);
+
+    render_timer.AddTimeStamp();
+
+    capture_timer.ResetTimer();
+    apm->set_stream_delay_ms(0);

[hlundin-webrtc, 2016/06/27 11:21:15]

Can it fail? ASSERT_EQ?

[peah-webrtc, 2016/06/27 22:51:48]

Done.

+    error =
+        apm->ProcessStream(&buffers.capture_input[0], capture_input_config,
+                           capture_output_config, &buffers.capture_output[0]);
+
+    capture_timer.AddTimeStamp();
+    total_timer.AddTimeStamp();
+    ASSERT_EQ(AudioProcessing::kNoError, error);
+  }
+
+  webrtc::test::PrintResultMeanAndError(
+      "level_controller_call_durations",
+      "_" + std::to_string(render_input_sample_rate_hz) + "_" +
+          std::to_string(render_output_sample_rate_hz) + "_" +
+          std::to_string(capture_input_sample_rate_hz) + "_" +
+          std::to_string(capture_output_sample_rate_hz) + "Hz_" +
+          std::to_string(num_channels) + "_channels" + "_render",
+      test_description, FormPerformanceMeasureString(render_timer), "us",
+      false);
+  webrtc::test::PrintResultMeanAndError(
+      "level_controller_call_durations",
+      "_" + std::to_string(render_input_sample_rate_hz) + "_" +
+          std::to_string(render_output_sample_rate_hz) + "_" +
+          std::to_string(capture_input_sample_rate_hz) + "_" +
+          std::to_string(capture_output_sample_rate_hz) + "Hz_" +
+          std::to_string(num_channels) + "_channels" + "_capture",
+      test_description, FormPerformanceMeasureString(capture_timer), "us",
+      false);
+  webrtc::test::PrintResultMeanAndError(
+      "level_controller_call_durations",
+      "_" + std::to_string(render_input_sample_rate_hz) + "_" +
+          std::to_string(render_output_sample_rate_hz) + "_" +
+          std::to_string(capture_input_sample_rate_hz) + "_" +
+          std::to_string(capture_output_sample_rate_hz) + "Hz_" +
+          std::to_string(num_channels) + "_channels" + "_total",
+      test_description, FormPerformanceMeasureString(total_timer), "us", false);
+}
+
+}  // namespace
+
+TEST(LevelControllerPerformanceTest, StandaloneProcessing) {
+  int sample_rates_to_test[] = {
+      AudioProcessing::kSampleRate8kHz, AudioProcessing::kSampleRate16kHz,
+      AudioProcessing::kSampleRate32kHz, AudioProcessing::kSampleRate48kHz};

[hlundin-webrtc, 2016/06/27 11:21:15]

No 44100 Hz here?

[peah-webrtc, 2016/06/27 22:51:49]

No. The submodule only supports native rates (8, 16, 32 and 48 kHz). When the
submodule is run inside APM, APM resamples the signal to a native rate before
the submodule is called so therefore it is possible to use 44.1 kHz as an input
to the test where the level controller is run via APM.

[hlundin-webrtc, 2016/06/28 11:29:00]

Acknowledged.

[peah-webrtc, 2016/06/28 22:19:37]

Acknowledged.

+  for (auto sample_rate : sample_rates_to_test) {
+    for (size_t num_channels = 1; num_channels <= 2; ++num_channels) {
+      RunStandatoleSubmodule(sample_rate, num_channels);
+    }
+  }
+}
+
+TEST(LevelControllerPerformanceTest, ProcessingViaApm) {
+  int sample_rates_to_test[] = {AudioProcessing::kSampleRate8kHz,
+                                AudioProcessing::kSampleRate16kHz,
+                                AudioProcessing::kSampleRate32kHz,
+                                AudioProcessing::kSampleRate48kHz, 44100};
+  for (auto capture_input_sample_rate_hz : sample_rates_to_test) {
+    for (auto capture_output_sample_rate_hz : sample_rates_to_test) {
+      for (size_t num_channels = 1; num_channels <= 2; ++num_channels) {
+        RunTogetherWithApm("SimpleLevelControlViaApm", 48000, 48000,
+                           capture_input_sample_rate_hz,
+                           capture_output_sample_rate_hz, num_channels, false,
+                           false);
+      }
+    }
+  }
+}
+
+TEST(LevelControllerPerformanceTest, InteractionWithDefaultApm) {
+  int sample_rates_to_test[] = {AudioProcessing::kSampleRate8kHz,
+                                AudioProcessing::kSampleRate16kHz,
+                                AudioProcessing::kSampleRate32kHz,
+                                AudioProcessing::kSampleRate48kHz, 44100};
+  for (auto capture_input_sample_rate_hz : sample_rates_to_test) {
+    for (auto capture_output_sample_rate_hz : sample_rates_to_test) {
+      for (size_t num_channels = 1; num_channels <= 2; ++num_channels) {
+        RunTogetherWithApm("LevelControlAndDefaultDesktopApm", 48000, 48000,
+                           capture_input_sample_rate_hz,
+                           capture_output_sample_rate_hz, num_channels, false,
+                           true);
+        RunTogetherWithApm("LevelControlAndDefaultMobileApm", 48000, 48000,
+                           capture_input_sample_rate_hz,
+                           capture_output_sample_rate_hz, num_channels, true,
+                           true);
+      }
+    }
+  }
+}
+
+}  // namespace webrtc