Added a bitexactness test for the beamformer in the audio processing module

webrtc/modules/audio_processing/beamformer_unittest.cc

+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include <vector>
+
+#include "testing/gtest/include/gtest/gtest.h"
+#include "webrtc/base/array_view.h"
+#include "webrtc/modules/audio_processing/audio_buffer.h"
+#include "webrtc/modules/audio_processing/beamformer/nonlinear_beamformer.h"
+#include "webrtc/modules/audio_processing/test/audio_buffer_tools.h"
+#include "webrtc/modules/audio_processing/test/bitexactness_tools.h"
+
+namespace webrtc {
+namespace {
+
+const size_t kNumFramesToProcess = 1000;
+
+void ProcessOneFrame(int sample_rate_hz,
+                     AudioBuffer* capture_audio_buffer,
+                     Beamformer<float>* beamformer) {
+  if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) {
+    capture_audio_buffer->SplitIntoFrequencyBands();
+  }
+
+  beamformer->ProcessChunk(*capture_audio_buffer->split_data_f(),
+                           capture_audio_buffer->split_data_f());
+  capture_audio_buffer->set_num_channels(1);
+
+  if (sample_rate_hz > AudioProcessing::kSampleRate16kHz) {
+    capture_audio_buffer->MergeFrequencyBands();
+  }
+}
+
+int BeamformerSampleRate(int sample_rate_hz) {
+  return (sample_rate_hz > AudioProcessing::kSampleRate16kHz
+              ? AudioProcessing::kSampleRate16kHz
+              : sample_rate_hz);
+}
+
+void RunBitExactnessTest(int sample_rate_hz,
+                         const std::vector<Point>& array_geometry,
+                         const SphericalPointf& target_direction,
+                         rtc::ArrayView<const float> output_reference) {
+  NonlinearBeamformer beamformer(array_geometry, target_direction);
+  beamformer.Initialize(AudioProcessing::kChunkSizeMs,
+                        BeamformerSampleRate(sample_rate_hz));
+
+  const StreamConfig capture_config(sample_rate_hz, array_geometry.size(),
+                                    false);
+  AudioBuffer capture_buffer(
+      capture_config.num_frames(), capture_config.num_channels(),
+      capture_config.num_frames(), capture_config.num_channels(),
+      capture_config.num_frames());
+  test::InputAudioFile capture_file(
+      test::GetApmCaptureTestVectorFileName(sample_rate_hz));
+  std::vector<float> capture_input(capture_config.num_frames() *
+                                   capture_config.num_channels());
+  for (size_t frame_no = 0u; frame_no < kNumFramesToProcess; ++frame_no) {
+    ReadFloatSamplesFromStereoFile(capture_config.num_frames(),
+                                   capture_config.num_channels(), &capture_file,
+                                   capture_input);
+
+    test::CopyVectorToAudioBuffer(capture_config, capture_input,
+                                  &capture_buffer);
+
+    ProcessOneFrame(sample_rate_hz, &capture_buffer, &beamformer);
+  }
+
+  // Extract and verify the test results.
+  std::vector<float> capture_output;
+  test::ExtractVectorFromAudioBuffer(capture_config, &capture_buffer,
+                                     &capture_output);
+
+  const float kTolerance = 1.f / static_cast<float>(1 << 15);
+
+  // Compare the output with the reference. Only the first values of the output
+  // from last frame processed are compared in order not having to specify all
+  // preceeding frames as testvectors. As the algorithm being tested has a
+  // memory, testing only the last frame implicitly also tests the preceeding
+  // frames.
+  EXPECT_TRUE(test::BitExactFrame(
+      capture_config.num_frames(), capture_config.num_channels(),
+      output_reference, capture_output, kTolerance));
+}
+
+std::vector<Point> CreateArrayGeometry(int variant) {
+  std::vector<Point> array_geometry;
+  switch (variant) {
+    case 1:
+      array_geometry.push_back(Point(-0.025f, 0.f, 0.f));
+      array_geometry.push_back(Point(0.025f, 0.f, 0.f));
+      break;
+    case 2:
+      array_geometry.push_back(Point(-0.035f, 0.f, 0.f));
+      array_geometry.push_back(Point(0.035f, 0.f, 0.f));
+      break;
+    case 3:
+      array_geometry.push_back(Point(-0.5f, 0.f, 0.f));
+      array_geometry.push_back(Point(0.5f, 0.f, 0.f));
+      break;
+    case 4:

[aluebs-webrtc, 2016/03/22 12:15:56]

The 4th case is never called.

[peah-webrtc, 2016/03/23 22:06:20]

Good catch, missed actually using that configuration and instead just wrote that
in the test description.
Done.

[aluebs-webrtc, 2016/03/24 11:14:25]

Acknowledged.

+      array_geometry.push_back(Point(-0.5f, 0.f, 0.f));
+      array_geometry.push_back(Point(0.1f, 0.f, 0.f));
+      array_geometry.push_back(Point(0.5f, 0.f, 0.f));
+      break;
+    default:
+      RTC_CHECK(false);
+  }
+  return array_geometry;
+}
+
+const SphericalPointf TargetDirection1(static_cast<float>(M_PI) / 2.f,
+                                       0.f,
+                                       1.f);
+const SphericalPointf TargetDirection2(static_cast<float>(M_PI) / 2.f,
+                                       1.f,
+                                       2.f);
+
+}  // namespace
+
+// TODO(peah): Investigate why the nonlinear_beamformer.cc causes a DCHECK in
+// this setup.
+TEST(BeamformerBitExactnessTest,
+     DISABLED_Stereo8kHz_ArrayGeometry1_TargetDirection1) {
+  const float kOutputReference[] = {-0.000161f, 0.000171f, -0.000096f,
+                                    0.001007f,  0.000427f, 0.000977f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate8kHz, CreateArrayGeometry(1),
+                      TargetDirection1, kOutputReference);
+}
+
+TEST(BeamformerBitExactnessTest, Stereo16kHz_ArrayGeometry1_TargetDirection1) {
+  const float kOutputReference[] = {0.001144f, -0.001026f, 0.001074f,
+                                    0.001144f, -0.001026f, 0.001074f};

[aluebs-webrtc, 2016/03/22 12:15:56]

Why did these change?

[peah-webrtc, 2016/03/23 22:06:20]

I'm not sure. It may be that the beamformer testvectors were not fully updated
yet.

[aluebs-webrtc, 2016/03/24 11:14:25]

This exactly the reason I think these kind of tests are not useful. You changed
the test in a way that should be bitexact, but the test fails, but you have no
idea if there was a big or small change. So you just blindly change the values
to the new ones that make the test pass, without understanding why.

[peah-webrtc, 2016/03/24 11:51:30]

The bitexactness tests are there to identify when something happens, and not
why/what happens. And by limiting their functionality to that, they can detect
changes in a much more refined way than any other tests.

Once the test code is finished, there are only changes in the submodule that
should cause bitinexactness. If bitinexactness occurs without any apparent
reason for it, the changes done in the submodule needs to be reviewed in a more
refined manner. But if they are expected the vectors should simply be updated.

In this case I know that I changed only the test code and not the code of the
submodule, so I'm confident it is not a problem with changes done in the
submodule.

[aluebs-webrtc, 2016/03/28 23:01:17]

I am still not convinced, but let's discuss offline.

+
+  RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(1),
+                      TargetDirection1, kOutputReference);
+}
+
+TEST(BeamformerBitExactnessTest, Stereo32kHz_ArrayGeometry1_TargetDirection1) {
+  const float kOutputReference[] = {0.000732f, -0.000397f, 0.000610f,
+                                    0.000732f, -0.000397f, 0.000610f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate32kHz, CreateArrayGeometry(1),
+                      TargetDirection1, kOutputReference);
+}
+
+TEST(BeamformerBitExactnessTest, Stereo48kHz_ArrayGeometry1_TargetDirection1) {
+  const float kOutputReference[] = {0.000106f, -0.000464f, 0.000188f,
+                                    0.000106f, -0.000464f, 0.000188f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate48kHz, CreateArrayGeometry(1),
+                      TargetDirection1, kOutputReference);
+}
+
+// TODO(peah): Investigate why the nonlinear_beamformer.cc causes a DCHECK in
+// this setup.
+TEST(BeamformerBitExactnessTest,
+     DISABLED_Stereo8kHz_ArrayGeometry1_TargetDirection2) {
+  const float kOutputReference[] = {0.001144f,  -0.001026f, 0.001074f,
+                                    -0.016205f, -0.007324f, -0.015656f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate8kHz, CreateArrayGeometry(1),
+                      TargetDirection2, kOutputReference);
+}
+
+TEST(BeamformerBitExactnessTest, Stereo16kHz_ArrayGeometry1_TargetDirection2) {
+  const float kOutputReference[] = {0.001144f, -0.001026f, 0.001074f,
+                                    0.001144f, -0.001026f, 0.001074f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(1),
+                      TargetDirection2, kOutputReference);
+}
+
+TEST(BeamformerBitExactnessTest, Stereo32kHz_ArrayGeometry1_TargetDirection2) {
+  const float kOutputReference[] = {0.000732f, -0.000397f, 0.000610f,
+                                    0.000732f, -0.000397f, 0.000610f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate32kHz, CreateArrayGeometry(1),
+                      TargetDirection2, kOutputReference);
+}
+
+TEST(BeamformerBitExactnessTest, Stereo48kHz_ArrayGeometry1_TargetDirection2) {
+  const float kOutputReference[] = {0.000106f, -0.000464f, 0.000188f,
+                                    0.000106f, -0.000464f, 0.000188f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate48kHz, CreateArrayGeometry(1),
+                      TargetDirection2, kOutputReference);
+}
+
+TEST(BeamformerBitExactnessTest, Stereo8kHz_ArrayGeometry2_TargetDirection2) {
+  const float kOutputReference[] = {-0.000649f, 0.000576f, -0.000148f,
+                                    -0.000649f, 0.000576f, -0.000148f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate8kHz, CreateArrayGeometry(2),
+                      TargetDirection2, kOutputReference);
+}
+
+TEST(BeamformerBitExactnessTest, Stereo16kHz_ArrayGeometry2_TargetDirection2) {
+  const float kOutputReference[] = {0.000808f, -0.000695f, 0.000739f,
+                                    0.000808f, -0.000695f, 0.000739f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(2),
+                      TargetDirection2, kOutputReference);
+}
+
+TEST(BeamformerBitExactnessTest, Stereo32kHz_ArrayGeometry2_TargetDirection2) {
+  const float kOutputReference[] = {0.000580f, -0.000183f, 0.000458f,
+                                    0.000580f, -0.000183f, 0.000458f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate32kHz, CreateArrayGeometry(2),
+                      TargetDirection2, kOutputReference);
+}
+
+TEST(BeamformerBitExactnessTest, Stereo48kHz_ArrayGeometry2_TargetDirection2) {
+  const float kOutputReference[] = {0.000075f, -0.000288f, 0.000156f,
+                                    0.000075f, -0.000288f, 0.000156f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate48kHz, CreateArrayGeometry(2),
+                      TargetDirection2, kOutputReference);
+}
+
+TEST(BeamformerBitExactnessTest, Stereo8kHz_ArrayGeometry4_TargetDirection2) {
+  const float kOutputReference[] = {-0.000649f, 0.000576f, -0.000148f,
+                                    -0.000649f, 0.000576f, -0.000148f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate8kHz, CreateArrayGeometry(2),
+                      TargetDirection2, kOutputReference);
+}
+
+TEST(BeamformerBitExactnessTest, Stereo16kHz_ArrayGeometry4_TargetDirection2) {
+  const float kOutputReference[] = {0.000808f, -0.000695f, 0.000739f,
+                                    0.000808f, -0.000695f, 0.000739f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(2),
+                      TargetDirection2, kOutputReference);
+}
+
+TEST(BeamformerBitExactnessTest, Stereo32kHz_ArrayGeometry4_TargetDirection2) {
+  const float kOutputReference[] = {0.000580f, -0.000183f, 0.000458f,
+                                    0.000580f, -0.000183f, 0.000458f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate32kHz, CreateArrayGeometry(2),
+                      TargetDirection2, kOutputReference);
+}
+
+TEST(BeamformerBitExactnessTest, Stereo48kHz_ArrayGeometry4_TargetDirection2) {
+  const float kOutputReference[] = {0.000075f, -0.000288f, 0.000156f,
+                                    0.000075f, -0.000288f, 0.000156f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate48kHz, CreateArrayGeometry(2),
+                      TargetDirection2, kOutputReference);
+}
+
+// TODO(peah): Investigate why the nonlinear_beamformer.cc causes a DCHECK in
+// this setup.
+TEST(BeamformerBitExactnessTest,
+     DISABLED_Stereo16kHz_ArrayGeometry3_TargetDirection1) {
+  const float kOutputReference[] = {-0.000161f, 0.000171f, -0.000096f,
+                                    0.001007f,  0.000427f, 0.000977f};
+
+  RunBitExactnessTest(AudioProcessing::kSampleRate16kHz, CreateArrayGeometry(3),
+                      TargetDirection1, kOutputReference);
+}
+
+}  // namespace webrtc