Conversational speech tool, simualtor + unit tests

webrtc/modules/audio_processing/test/conversational_speech/generator_unittest.cc

 /*
  *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 24 matching lines @@
 
 // MSVC++ requires this to be set before any other includes to get M_PI.
 #define _USE_MATH_DEFINES
 
 #include <stdio.h>
 #include <cmath>
 #include <map>
 #include <memory>
 
 #include "webrtc/base/logging.h"
+#include "webrtc/base/fileutils.h"
 #include "webrtc/base/pathutils.h"
 #include "webrtc/common_audio/wav_file.h"
 #include "webrtc/modules/audio_processing/test/conversational_speech/config.h"
 #include "webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader_factory.h"
 #include "webrtc/modules/audio_processing/test/conversational_speech/multiend_call.h"
+#include "webrtc/modules/audio_processing/test/conversational_speech/simulator.h"
 #include "webrtc/modules/audio_processing/test/conversational_speech/timing.h"
 #include "webrtc/modules/audio_processing/test/conversational_speech/wavreader_factory.h"
 #include "webrtc/test/gmock.h"
 #include "webrtc/test/gtest.h"
 #include "webrtc/test/testsupport/fileutils.h"
 
 namespace webrtc {
 namespace test {
 namespace {
 
@@ skipping 15 matching lines @@
     {"B", "b2", -200},
     {"A", "a3", 0},
     {"A", "a3", 0},
 };
 const std::size_t kNumberOfTurns = expected_timing.size();
 
 // Fake audio track parameters.
 const int kDefaultSampleRate = 48000;
 const std::map<std::string, const MockWavReaderFactory::Params>
     kDefaultMockWavReaderFactoryParamsMap = {
-  {"t300", {kDefaultSampleRate, 1u, 14400u}},  // 0.3 seconds.
-  {"t500", {kDefaultSampleRate, 1u, 24000u}},  // 0.5 seconds.
-  {"t1000", {kDefaultSampleRate, 1u, 48000u}},  // 1.0 seconds.
+  {"t300", {kDefaultSampleRate, 1u, 14400u}},  // Mono, 0.3 seconds.
+  {"t500", {kDefaultSampleRate, 1u, 24000u}},  // Mono, 0.5 seconds.
+  {"t1000", {kDefaultSampleRate, 1u, 48000u}},  // Mono, 1.0 seconds.
+  {"sr8000", {8000, 1u, 8000u}},  // 8kHz sample rate, mono, 1 second.
+  {"sr16000", {16000, 1u, 16000u}},  // 16kHz sample rate, mono, 1 second.
+  {"sr16000_stereo", {16000, 2u, 16000u}},  // Like sr16000, but stereo.
 };
 
 // Default arguments for MockWavReaderFactory ctor.
 const MockWavReaderFactory::Params& kDefaultMockWavReaderFactoryParams =
     kDefaultMockWavReaderFactoryParamsMap.at("t500");
 
 std::unique_ptr<MockWavReaderFactory> CreateMockWavReaderFactory() {
   return std::unique_ptr<MockWavReaderFactory>(
       new MockWavReaderFactory(kDefaultMockWavReaderFactoryParams,
                                kDefaultMockWavReaderFactoryParamsMap));
 }
 
 void CreateSineWavFile(const std::string& filepath,
                        const MockWavReaderFactory::Params& params,
                        float frequency = 440.0f) {
   // Create samples.
   constexpr double two_pi = 2.0 * M_PI;
   std::vector<int16_t> samples(params.num_samples);
   for (std::size_t i = 0; i < params.num_samples; ++i) {
     // TODO(alessiob): the produced tone is not pure, improve.
     samples[i] = std::lround(32767.0f * std::sin(
         two_pi * i * frequency / params.sample_rate));
   }
 
   // Write samples.
   WavWriter wav_writer(filepath, params.sample_rate, params.num_channels);
   wav_writer.WriteSamples(samples.data(), params.num_samples);
 }
 
+// Parameters to generate audio tracks with CreateSineWavFile.
+struct SineAudioTrackParams {
+  MockWavReaderFactory::Params params;
+  float frequency;
+};
+
+// Creates a temporary directory in which sine audio tracks are written.
+std::string CreateTemporarySineAudioTracks(
+    const std::map<std::string, SineAudioTrackParams>& sine_tracks_params) {
+  // Create temporary directory.
+  rtc::Pathname temp_directory(OutputPath());
+  temp_directory.AppendFolder("TempConversationalSpeechAudioTracks");
+  webrtc::test::CreateDir(temp_directory.pathname());
+
+  // Create sine tracks.
+  for (auto it = sine_tracks_params.begin(); it != sine_tracks_params.end();

[minyue-webrtc, 2017/04/07 13:24:30]

for (const auto& sine_tracks_param : sine_tracks_params)

[AleBzk, 2017/04/10 08:24:48]

Nope. I need both key and value from the map.

+       ++it) {
+    const rtc::Pathname temp_filepath(temp_directory.pathname(), it->first);
+    CreateSineWavFile(
+        temp_filepath.pathname(), it->second.params, it->second.frequency);
+  }
+
+  return temp_directory.pathname();
+}
+
+void CheckAudioTrackParams(const WavReaderFactory& wav_reader_factory,
+                           const std::string& filepath,
+                           const MockWavReaderFactory::Params& expeted_params) {
+  auto wav_reader = wav_reader_factory.Create(filepath);
+  EXPECT_EQ(expeted_params.sample_rate, wav_reader->SampleRate());
+  EXPECT_EQ(expeted_params.num_channels, wav_reader->NumChannels());
+  EXPECT_EQ(expeted_params.num_samples, wav_reader->NumSamples());
+}
+
 }  // namespace
 
 class ConversationalSpeechTest : public testing::Test {
  public:
   ConversationalSpeechTest() {
     rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
   }
 };
 
 TEST_F(ConversationalSpeechTest, Settings) {
@@ skipping 36 matching lines @@
   conversational_speech::MultiEndCall multiend_call(
       expected_timing, audiotracks_path, std::move(mock_wavreader_factory));
   EXPECT_TRUE(multiend_call.valid());
 
   // Test.
   EXPECT_EQ(2u, multiend_call.speaker_names().size());
   EXPECT_EQ(5u, multiend_call.audiotrack_readers().size());
   EXPECT_EQ(6u, multiend_call.speaking_turns().size());
 }
 
+TEST_F(ConversationalSpeechTest, MultiEndCallSetupDifferentSampleRates) {
+  const std::vector<Turn> timing = {
+      {"A", "sr8000", 0},
+      {"B", "sr16000", 0},
+  };
+  auto mock_wavreader_factory = CreateMockWavReaderFactory();
+
+  // There are two unique audio tracks to read.
+  EXPECT_CALL(*mock_wavreader_factory, Create(testing::_)).Times(2);
+
+  MultiEndCall multiend_call(
+      timing, audiotracks_path, std::move(mock_wavreader_factory));
+  EXPECT_FALSE(multiend_call.valid());
+}
+
+TEST_F(ConversationalSpeechTest, MultiEndCallSetupMultipleChannels) {
+  const std::vector<Turn> timing = {
+      {"A", "sr16000_stereo", 0},
+      {"B", "sr16000_stereo", 0},
+  };
+  auto mock_wavreader_factory = CreateMockWavReaderFactory();
+
+  // There is one unique audio track to read.
+  EXPECT_CALL(*mock_wavreader_factory, Create(testing::_)).Times(1);
+
+  MultiEndCall multiend_call(
+      timing, audiotracks_path, std::move(mock_wavreader_factory));
+  EXPECT_FALSE(multiend_call.valid());
+}
+
+TEST_F(ConversationalSpeechTest,
+       MultiEndCallSetupDifferentSampleRatesAndMultipleNumChannels) {
+  const std::vector<Turn> timing = {
+      {"A", "sr8000", 0},
+      {"B", "sr16000_stereo", 0},
+  };
+  auto mock_wavreader_factory = CreateMockWavReaderFactory();
+
+  // There are two unique audio tracks to read.
+  EXPECT_CALL(*mock_wavreader_factory, Create(testing::_)).Times(2);
+
+  MultiEndCall multiend_call(
+      timing, audiotracks_path, std::move(mock_wavreader_factory));
+  EXPECT_FALSE(multiend_call.valid());
+}
+
 TEST_F(ConversationalSpeechTest, MultiEndCallSetupFirstOffsetNonNegative) {
   const std::vector<Turn> timing = {
       {"A", "t500", -100},
       {"B", "t500", 0},
   };
   auto mock_wavreader_factory = CreateMockWavReaderFactory();
 
   // There is one unique audio track to read.
   EXPECT_CALL(*mock_wavreader_factory, Create(testing::_)).Times(1);
 
@@ skipping 332 matching lines @@
 
   for (int sample_rate : sample_rates) {
     const rtc::Pathname temp_filename(
         OutputPath(), "TempSineWavFile_" + std::to_string(sample_rate)
             + ".wav");
 
     // Write wav file.
     const std::size_t num_samples = duration_seconds * sample_rate;
     MockWavReaderFactory::Params params = {sample_rate, 1u, num_samples};
     CreateSineWavFile(temp_filename.pathname(), params);
-    LOG(LS_VERBOSE) << "wav file @" << sample_rate << " Hz created ("
-        << num_samples << " samples)";
 
     // Load wav file and check if params match.
     WavReaderFactory wav_reader_factory;
-    auto wav_reader = wav_reader_factory.Create(temp_filename.pathname());
-    EXPECT_EQ(sample_rate, wav_reader->SampleRate());
-    EXPECT_EQ(1u, wav_reader->NumChannels());
-    EXPECT_EQ(num_samples, wav_reader->NumSamples());
+    MockWavReaderFactory::Params expeted_params = {
+        sample_rate, 1u, num_samples};
+    CheckAudioTrackParams(
+        wav_reader_factory, temp_filename.pathname(), expeted_params);
 
     // Clean up.
     remove(temp_filename.pathname().c_str());
   }
 }
 
+TEST_F(ConversationalSpeechTest, MultiEndCallSimulator) {
+  // Simulated call (one character corresponding to 500 ms):
+  // A 0*********...........2*********.....
+  // B ...........1*********.....3*********
+  const std::vector<Turn> expected_timing = {
+      {"A", "t5000_440.wav", 0},
+      {"B", "t5000_880.wav", 500},
+      {"A", "t5000_440.wav", 0},
+      {"B", "t5000_880.wav", -2500},
+  };
+  const std::size_t expected_duration_seconds = 18;
+
+  // Create temporary audio track files.
+  const int sample_rate = 16000;
+  const std::map<std::string, SineAudioTrackParams> sine_tracks_params = {
+      {"t5000_440.wav", {{sample_rate, 1u, sample_rate * 5}, 440.0}},
+      {"t5000_880.wav", {{sample_rate, 1u, sample_rate * 5}, 880.0}},
+  };
+  const std::string audiotracks_path = CreateTemporarySineAudioTracks(
+      sine_tracks_params);
+
+  // Set up the multi-end call.
+  auto wavreader_factory = std::unique_ptr<WavReaderFactory>(
+      new WavReaderFactory());
+  MultiEndCall multiend_call(
+      expected_timing, audiotracks_path, std::move(wavreader_factory));
+
+  // Simulate the call.
+  rtc::Pathname output_path(audiotracks_path);
+  output_path.AppendFolder("output");
+  webrtc::test::CreateDir(output_path.pathname());
+  LOG(LS_VERBOSE) << "simulator output path: " << output_path.pathname();
+  auto generated_audiotrak_pairs = conversational_speech::Simulate(
+      multiend_call, output_path.pathname());
+  EXPECT_EQ(2u, generated_audiotrak_pairs->size());
+
+  // Check the output.
+  WavReaderFactory wav_reader_factory;
+  const MockWavReaderFactory::Params expeted_params = {
+      sample_rate, 1u, sample_rate * expected_duration_seconds};
+  for (auto it = generated_audiotrak_pairs->begin();

[minyue-webrtc, 2017/04/07 13:24:30]

same here

[AleBzk, 2017/04/10 08:24:48]

Same here as well :)

+      it != generated_audiotrak_pairs->end(); ++it) {
+    LOG(LS_VERBOSE) << "checking far/near-end for <" << it->first << ">";
+    CheckAudioTrackParams(
+        wav_reader_factory, it->second.near_end, expeted_params);
+    CheckAudioTrackParams(
+        wav_reader_factory, it->second.far_end, expeted_params);
+  }
+
+  // Clean.
+  EXPECT_TRUE(rtc::Filesystem::DeleteFolderAndContents(
+    rtc::Pathname(audiotracks_path)))
+        << "Cannot delete temporary data directory " << audiotracks_path;
+}
+
 }  // namespace test
 }  // namespace webrtc