Bitexactness test for the noise suppressor

webrtc/modules/audio_processing/noise_suppression_bitexactness_unittest.cc

Index: webrtc/modules/audio_processing/noise_suppression_bitexactness_unittest.cc
diff --git a/webrtc/modules/audio_processing/noise_suppression_bitexactness_unittest.cc b/webrtc/modules/audio_processing/noise_suppression_bitexactness_unittest.cc
new file mode 100644
index 0000000000000000000000000000000000000000..30a1ec6e8d5f2ff7c28d45fa15a3bca74798145e
--- /dev/null
+++ b/webrtc/modules/audio_processing/noise_suppression_bitexactness_unittest.cc
@@ -0,0 +1,292 @@
+/*
+ *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.

[hlundin-webrtc, 2016/03/16 12:44:28]

2016

[peah-webrtc, 2016/03/17 13:15:00]

Great find!

Done.

+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include <vector>
+
+#include "testing/gtest/include/gtest/gtest.h"
+#include "webrtc/base/array_view.h"
+#include "webrtc/base/random.h"
+#include "webrtc/modules/audio_processing/audio_buffer.h"
+#include "webrtc/modules/audio_processing/noise_suppression_impl.h"
+#include "webrtc/modules/audio_processing/test/audio_buffer_tools.h"
+#include "webrtc/modules/audio_processing/test/bitexactness_tools.h"
+
+namespace webrtc {
+namespace {
+
+// Process one frame of data and produce the output.
+void ProcessOneFrame(int sample_rate_hz,
+                     AudioBuffer* audio_buffer,
+                     NoiseSuppressionImpl* noise_suppressor,
+                     std::vector<float>* frame_output,
+                     float* speech_probability,
+                     std::vector<float>* noise_estimate) {
+  if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) {
+    audio_buffer->SplitIntoFrequencyBands();
+  }
+
+  noise_suppressor->AnalyzeCaptureAudio(audio_buffer);
+  noise_suppressor->ProcessCaptureAudio(audio_buffer);
+  *speech_probability = noise_suppressor->speech_probability();
+  *noise_estimate = noise_suppressor->NoiseEstimate();
+
+  if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) {
+    audio_buffer->MergeFrequencyBands();
+  }
+}
+
+// Forms a predefined random test vector-

[hlundin-webrtc, 2016/03/16 12:44:27]

End with '.', not '-'.

[hlundin-webrtc, 2016/03/16 12:44:28]

Please, tell me what the range of the samples is.

[peah-webrtc, 2016/03/17 13:15:00]

Done.

[peah-webrtc, 2016/03/17 13:15:00]

Good point! This part of the code is now removed.

+void ConstructTestVector(int samples_per_channel,

[hlundin-webrtc, 2016/03/16 12:44:28]

Make all int parameters size_t.

[peah-webrtc, 2016/03/17 13:15:00]

Done.

+                         int num_channels,
+                         int frame_counter,
+                         Random* rand_gen,
+                         std::vector<float>* testvector) {
+  testvector->resize(samples_per_channel * num_channels);
+
+  bool low_level = ((frame_counter / 10) > 5);

[hlundin-webrtc, 2016/03/16 12:44:28]

Why not
bool low_level = (frame_counter >= 60);
?

[peah-webrtc, 2016/03/17 13:15:00]

Good point! The code is now removed.

+  float scale = (low_level ? 0.01f : 1.0f);

[hlundin-webrtc, 2016/03/16 12:44:28]

... or even
const float scale = frame_counter >= 60 ? : 0.01f : 1.0f;

[peah-webrtc, 2016/03/17 13:15:01]

Another good point! The code is now removed.

+
+  for (auto& v : *testvector) {
+    v = scale * (2.0f * rand_gen->Rand<float>() - 1.0f);
+  }
+}
+
+void SetupNoiseSuppressor(int sample_rate_hz,
+                          int num_channels,
+                          NoiseSuppressionImpl::Level level,
+                          NoiseSuppressionImpl* noise_suppressor) {
+  noise_suppressor->Initialize(num_channels, sample_rate_hz);
+  noise_suppressor->Enable(true);
+  noise_suppressor->set_level(level);
+}
+
+// Verifies the output of the test against a reference and reports the results
+// using the gtest EXPECT_PRED_FORMAT2 functionality
+void VerifyOutput(const StreamConfig& stream_config,
+                  float speech_probability_reference,
+                  const rtc::ArrayView<const float>& noise_estimate_reference,
+                  const rtc::ArrayView<const float>& output_reference,
+                  const std::vector<float>& output,

[hlundin-webrtc, 2016/03/16 12:44:27]

Nit: the internal order of reference and test parameters should preferably be
equal.

[peah-webrtc, 2016/03/17 13:15:00]

Done.

+                  float speech_probability,
+                  const std::vector<float>& noise_estimate) {
+  // Form vectors to compare the reference to. Only the first values of the
+  // outputs are compared in order not having to specify all preceeding frames
+  // as testvectors.
+  const size_t reference_frame_length =
+      output_reference.size() / stream_config.num_channels();

[hlundin-webrtc, 2016/03/16 12:44:28]

This should be an exact division, right? Consider using rtc::CheckedDivExact(a,
b).

[peah-webrtc, 2016/03/17 13:15:00]

Good point! Thanks for the suggestion!

Done.

+  std::vector<float> output_to_verify;
+  for (size_t channel_no = 0; channel_no < stream_config.num_channels();
+       ++channel_no) {
+    output_to_verify.insert(
+        output_to_verify.end(),
+        output.begin() + channel_no * stream_config.num_frames(),
+        output.begin() + channel_no * stream_config.num_frames() +
+            reference_frame_length);
+  }
+
+  EXPECT_PRED_FORMAT2(test::AssertVectorsNotEqual, output_to_verify,
+                      output_reference);
+  EXPECT_PRED_FORMAT2(test::AssertVectorsNotEqual, noise_estimate,
+                      noise_estimate_reference);
+  EXPECT_PRED_FORMAT2(test::AssertFloatsNotEqual, speech_probability,

[hlundin-webrtc, 2016/03/16 12:44:28]

Did you consider using EXPECT_FLOAT_EQ or EXPECT_NEAR?

[peah-webrtc, 2016/03/17 13:15:00]

Great suggestion! 
Done.

+                      speech_probability_reference);
+}
+
+// Processes a specified amount of frames, verifies the results and reports
+// any errors.
+void RunBitexactnessTest(
+    int sample_rate_hz,
+    int num_channels,

[hlundin-webrtc, 2016/03/16 12:44:28]

int -> size_t

[peah-webrtc, 2016/03/17 13:15:00]

Done.

+    int num_frames_to_process,

[hlundin-webrtc, 2016/03/16 12:44:28]

int -> size_t

[peah-webrtc, 2016/03/17 13:15:00]

Done.

+    NoiseSuppressionImpl::Level level,
+    float speech_probability_reference,
+    const rtc::ArrayView<const float>& noise_estimate_reference,
+    const rtc::ArrayView<const float>& output_reference) {
+  Random rand_gen(42);
+  int samples_per_channel = 80 * sample_rate_hz / 8000;

[hlundin-webrtc, 2016/03/16 12:44:28]

I think sample_rate_hz / 100 works just as well, with a comment that this is 10
ms.

[peah-webrtc, 2016/03/17 13:15:00]

Done.

+  const StreamConfig stream_config(sample_rate_hz, num_channels, false);
+  AudioBuffer audio_buffer(
+      stream_config.num_frames(), stream_config.num_channels(),
+      stream_config.num_frames(), stream_config.num_channels(),
+      stream_config.num_frames());
+
+  rtc::CriticalSection crit;
+  NoiseSuppressionImpl noise_suppressor(&crit);
+  SetupNoiseSuppressor(sample_rate_hz, num_channels, level, &noise_suppressor);
+
+  std::vector<float> output;
+  float speech_probability = 0.0f;
+  std::vector<float> noise_estimate;
+  std::vector<float> frame_input;
+  std::vector<float> frame_output;
+  for (int frame_no = 0; frame_no < num_frames_to_process; ++frame_no) {

[hlundin-webrtc, 2016/03/16 12:44:28]

size_t frame_no

[peah-webrtc, 2016/03/17 13:15:00]

Done.

+    ConstructTestVector(samples_per_channel, num_channels, frame_no, &rand_gen,
+                        &frame_input);
+
+    test::CopyVectorToAudioBuffer(stream_config, frame_input, &audio_buffer);
+
+    ProcessOneFrame(sample_rate_hz, &audio_buffer, &noise_suppressor, &output,
+                    &speech_probability, &noise_estimate);
+
+    test::ExtractVectorFromAudioBuffer(stream_config, &audio_buffer,
+                                       &frame_output);
+  }
+
+  // Compare the output to the reference. Only the first values of the output

[hlundin-webrtc, 2016/03/16 12:44:28]

Compare ... with

[hlundin-webrtc, 2016/03/16 12:44:28]

Wonky line-breaks in this paragraph.

[peah-webrtc, 2016/03/17 13:15:00]

Done.

[peah-webrtc, 2016/03/17 13:15:01]

Done.

+  // from last frame processed
+  // is compared in order not having to specify all preceeding frames as

[hlundin-webrtc, 2016/03/16 12:44:27]

are compared

[peah-webrtc, 2016/03/17 13:15:00]

Done.

+  // testvectors.
+  // As the algorithm being tested has a memory, testing only
+  // the last frame implicitly also tests the preceeding frames.
+  VerifyOutput(stream_config, speech_probability_reference,
+               noise_estimate_reference, output_reference, frame_output,
+               speech_probability, noise_estimate);
+}
+
+}  // namespace
+
+TEST(NoiseSuppresionBitExactnessTest, Mono8kHzLowLevel) {
+#if !defined(WEBRTC_ANDROID)
+  const float kOutputReference[] = {0.000921f, 0.003884f, -0.000689f};
+  const float kNoiseEstimateReference[] = {0.028336f, 0.039530f, 0.042970f};
+  const float kSpeechProbabilityReference = 0.122579f;
+#else
+  const float kOutputReference[] = {0.000916f, 0.003876f, -0.000702f};
+  const float kNoiseEstimateReference[] = {12.392536f, 13.370509f, 11.658783f};
+  const float kSpeechProbabilityReference = -4.000000f;
+#endif
+
+  RunBitexactnessTest(8000, 1, 1000, NoiseSuppression::Level::kLow,
+                      kSpeechProbabilityReference,
+                      rtc::ArrayView<const float>(kNoiseEstimateReference),
+                      rtc::ArrayView<const float>(kOutputReference));
+}
+
+TEST(NoiseSuppresionBitExactnessTest, Mono16kHzLowLevel) {
+#if !defined(WEBRTC_ANDROID)
+  const float kOutputReference[] = {0.002048f, 0.001845f, 0.003762f};
+  const float kNoiseEstimateReference[] = {0.032741f, 0.052345f, 0.063557f};
+  const float kSpeechProbabilityReference = 0.110951f;
+#else
+  const float kOutputReference[] = {0.002014f, 0.001831f, 0.003754f};
+  const float kNoiseEstimateReference[] = {7.268418f, 8.785124f, 8.383295f};
+  const float kSpeechProbabilityReference = -4.000000f;
+#endif
+
+  RunBitexactnessTest(16000, 1, 1000, NoiseSuppression::Level::kLow,
+                      kSpeechProbabilityReference,
+                      rtc::ArrayView<const float>(kNoiseEstimateReference),
+                      rtc::ArrayView<const float>(kOutputReference));
+}
+
+TEST(NoiseSuppresionBitExactnessTest, Mono32kHzLowLevel) {
+#if !defined(WEBRTC_ANDROID)
+  const float kOutputReference[] = {-0.005249f, 0.001465f, -0.002533f};
+  const float kNoiseEstimateReference[] = {0.025969f, 0.035012f, 0.035499f};
+  const float kSpeechProbabilityReference = 0.139357f;
+#else
+  const float kOutputReference[] = {-0.005219f, 0.001373f, -0.002472f};
+  const float kNoiseEstimateReference[] = {12.616668f, 12.766106f, 11.475318f};
+  const float kSpeechProbabilityReference = -4.000000f;
+#endif
+
+  RunBitexactnessTest(32000, 1, 1000, NoiseSuppression::Level::kLow,
+                      kSpeechProbabilityReference,
+                      rtc::ArrayView<const float>(kNoiseEstimateReference),
+                      rtc::ArrayView<const float>(kOutputReference));
+}
+
+TEST(NoiseSuppresionBitExactnessTest, Mono48kHzLowLevel) {
+#if !defined(WEBRTC_ANDROID)
+  const float kOutputReference[] = {0.001224f, 0.005314f, 0.002205f};
+  const float kNoiseEstimateReference[] = {0.022175f, 0.031690f, 0.036631f};
+  const float kSpeechProbabilityReference = 0.101083f;
+#else
+  const float kOutputReference[] = {0.001181f, 0.005317f, 0.002217f};
+  const float kNoiseEstimateReference[] = {7.594315f, 9.309500f, 9.561249f};
+  const float kSpeechProbabilityReference = -4.000000f;
+#endif
+
+  RunBitexactnessTest(48000, 1, 1000, NoiseSuppression::Level::kLow,
+                      kSpeechProbabilityReference,
+                      rtc::ArrayView<const float>(kNoiseEstimateReference),
+                      rtc::ArrayView<const float>(kOutputReference));
+}
+
+TEST(NoiseSuppresionBitExactnessTest, Stereo16kHzLowLevel) {
+#if !defined(WEBRTC_ANDROID)
+  const float kOutputReference[] = {0.000954f,  0.002081f, -0.001125f,
+                                    -0.003688f, 0.004999f, -0.004168f};
+  const float kNoiseEstimateReference[] = {0.037820f, 0.054766f, 0.057829f};
+  const float kSpeechProbabilityReference = 0.106168f;
+#else
+  const float kOutputReference[] = {0.000946f,  0.002106f, -0.001099f,
+                                    -0.003693f, 0.004975f, -0.004181f};
+  const float kNoiseEstimateReference[] = {9.096287f, 8.000648f, 8.823565f};
+  const float kSpeechProbabilityReference = -4.000000f;
+#endif
+
+  RunBitexactnessTest(16000, 2, 1000, NoiseSuppression::Level::kLow,
+                      kSpeechProbabilityReference,
+                      rtc::ArrayView<const float>(kNoiseEstimateReference),
+                      rtc::ArrayView<const float>(kOutputReference));
+}
+
+TEST(NoiseSuppresionBitExactnessTest, Mono16kHzModerateLevel) {
+#if !defined(WEBRTC_ANDROID)
+  const float kOutputReference[] = {0.001036f, 0.000783f, 0.001862f};
+  const float kNoiseEstimateReference[] = {0.032994f, 0.052608f, 0.065461f};
+  const float kSpeechProbabilityReference = 0.110952f;
+#else
+  const float kOutputReference[] = {0.000977f, 0.000763f, 0.001801f};
+  const float kNoiseEstimateReference[] = {7.269972f, 8.785130f, 8.383298f};
+  const float kSpeechProbabilityReference = -4.000000f;
+#endif
+
+  RunBitexactnessTest(16000, 1, 1000, NoiseSuppression::Level::kModerate,
+                      kSpeechProbabilityReference,
+                      rtc::ArrayView<const float>(kNoiseEstimateReference),
+                      rtc::ArrayView<const float>(kOutputReference));
+}
+
+TEST(NoiseSuppresionBitExactnessTest, Mono16kHzHighLevel) {
+#if !defined(WEBRTC_ANDROID)
+  const float kOutputReference[] = {0.000533f, 0.000372f, 0.000948f};
+  const float kNoiseEstimateReference[] = {0.037815f, 0.055497f, 0.065549f};
+  const float kSpeechProbabilityReference = 0.110951f;
+#else
+  const float kOutputReference[] = {0.000519f, 0.000336f, 0.000885f};
+  const float kNoiseEstimateReference[] = {7.271456f, 8.785111f, 8.383295f};
+  const float kSpeechProbabilityReference = -4.000000f;
+#endif
+
+  RunBitexactnessTest(16000, 1, 1000, NoiseSuppression::Level::kHigh,
+                      kSpeechProbabilityReference,
+                      rtc::ArrayView<const float>(kNoiseEstimateReference),
+                      rtc::ArrayView<const float>(kOutputReference));
+}
+
+TEST(NoiseSuppresionBitExactnessTest, Mono16kHzVeryHighLevel) {
+#if !defined(WEBRTC_ANDROID)
+  const float kOutputReference[] = {0.000367f, 0.000259f, 0.000647f};
+  const float kNoiseEstimateReference[] = {0.038476f, 0.055677f, 0.065570f};
+  const float kSpeechProbabilityReference = 0.110951f;
+#else
+  const float kOutputReference[] = {0.000336f, 0.000244f, 0.000610f};
+  const float kNoiseEstimateReference[] = {7.272960f, 8.785113f, 8.383295f};
+  const float kSpeechProbabilityReference = -4.000000f;
+#endif
+
+  RunBitexactnessTest(16000, 1, 1000, NoiseSuppression::Level::kVeryHigh,
+                      kSpeechProbabilityReference,
+                      rtc::ArrayView<const float>(kNoiseEstimateReference),
+                      rtc::ArrayView<const float>(kOutputReference));
+}
+
+}  // namespace webrtc