Adding audio to video_quality_test.

webrtc/video/video_quality_test.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include <stdio.h>
@@ skipping 16 matching lines @@
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_utility.h"
 #include "webrtc/system_wrappers/include/cpu_info.h"
 #include "webrtc/test/layer_filtering_transport.h"
 #include "webrtc/test/run_loop.h"
 #include "webrtc/test/statistics.h"
 #include "webrtc/test/testsupport/fileutils.h"
 #include "webrtc/test/video_renderer.h"
 #include "webrtc/video/video_quality_test.h"
+#include "webrtc/voice_engine/include/voe_base.h"
+#include "webrtc/voice_engine/include/voe_codec.h"
+
+namespace {
+
+constexpr int kSendStatsPollingIntervalMs = 1000;
+constexpr int kPayloadTypeH264 = 122;
+constexpr int kPayloadTypeVP8 = 123;
+constexpr int kPayloadTypeVP9 = 124;
+constexpr size_t kMaxComparisons = 10;
+constexpr char kSyncGroup[] = "av_sync";
+constexpr int kOpusMinBitrate = 6000;
+constexpr int kOpusBitrateFb = 32000;
+
+struct VoiceEngineState {
+  VoiceEngineState()
+      : voice_engine(nullptr),
+        base(nullptr),
+        codec(nullptr),
+        send_channel_id(-1),
+        receive_channel_id(-1) {}
+
+  webrtc::VoiceEngine* voice_engine;
+  webrtc::VoEBase* base;
+  webrtc::VoECodec* codec;
+  int send_channel_id;
+  int receive_channel_id;
+};
+
+void CreateVoiceEngine(VoiceEngineState* voe,
+                       rtc::scoped_refptr<webrtc::AudioDecoderFactory>
+                           decoder_factory) {
+  voe->voice_engine = webrtc::VoiceEngine::Create();
+  voe->base = webrtc::VoEBase::GetInterface(voe->voice_engine);
+  voe->codec = webrtc::VoECodec::GetInterface(voe->voice_engine);
+  EXPECT_EQ(0, voe->base->Init(nullptr, nullptr, decoder_factory));
+  webrtc::Config voe_config;
+  voe_config.Set<webrtc::VoicePacing>(new webrtc::VoicePacing(true));
+  voe->send_channel_id = voe->base->CreateChannel(voe_config);
+  EXPECT_GE(voe->send_channel_id, 0);
+  voe->receive_channel_id = voe->base->CreateChannel();
+  EXPECT_GE(voe->receive_channel_id, 0);
+}
+
+void DestroyVoiceEngine(VoiceEngineState* voe) {
+  voe->base->DeleteChannel(voe->send_channel_id);
+  voe->send_channel_id = -1;
+  voe->base->DeleteChannel(voe->receive_channel_id);
+  voe->receive_channel_id = -1;
+  voe->base->Release();
+  voe->base = nullptr;
+  voe->codec->Release();
+  voe->codec = nullptr;
+
+  webrtc::VoiceEngine::Delete(voe->voice_engine);
+  voe->voice_engine = nullptr;
+}
+
+}  // namespace
 
 namespace webrtc {
 
-static const int kSendStatsPollingIntervalMs = 1000;
-static const int kPayloadTypeH264 = 122;
-static const int kPayloadTypeVP8 = 123;
-static const int kPayloadTypeVP9 = 124;
-static const size_t kMaxComparisons = 10;
-
 class VideoAnalyzer : public PacketReceiver,
                       public Transport,
                       public rtc::VideoSinkInterface<VideoFrame>,
                       public VideoCaptureInput,
                       public EncodedFrameObserver {
  public:
   VideoAnalyzer(test::LayerFilteringTransport* transport,
                 const std::string& test_label,
                 double avg_psnr_threshold,
                 double avg_ssim_threshold,
@@ skipping 939 matching lines @@
       ASSERT_TRUE(capturer_) << "Could not create capturer for "
                              << params_.video.clip_name
                              << ".yuv. Is this resource file present?";
     }
   }
 }
 
 void VideoQualityTest::RunWithAnalyzer(const Params& params) {
   params_ = params;
 
+  RTC_CHECK(!params_.audio);
   // TODO(ivica): Merge with RunWithRenderer and use a flag / argument to
   // differentiate between the analyzer and the renderer case.
   CheckParams();
 
   FILE* graph_data_output_file = nullptr;
   if (!params_.analyzer.graph_data_output_filename.empty()) {
     graph_data_output_file =
         fopen(params_.analyzer.graph_data_output_filename.c_str(), "w");
     RTC_CHECK(graph_data_output_file)
         << "Can't open the file " << params_.analyzer.graph_data_output_filename
@@ skipping 77 matching lines @@
   for (VideoReceiveStream* receive_stream : video_receive_streams_)
     receive_stream->Stop();
   video_send_stream_->Stop();
 
   DestroyStreams();
 
   if (graph_data_output_file)
     fclose(graph_data_output_file);
 }
 
-void VideoQualityTest::RunWithVideoRenderer(const Params& params) {
+void VideoQualityTest::RunWithRenderers(const Params& params) {
   params_ = params;
   CheckParams();
 
   std::unique_ptr<test::VideoRenderer> local_preview(
       test::VideoRenderer::Create("Local Preview", params_.common.width,
                                   params_.common.height));
   size_t stream_id = params_.ss.selected_stream;
   std::string title = "Loopback Video";
   if (params_.ss.streams.size() > 1) {
     std::ostringstream s;
     s << stream_id;
     title += " - Stream #" + s.str();
   }
 
   std::unique_ptr<test::VideoRenderer> loopback_video(
       test::VideoRenderer::Create(title.c_str(),
                                   params_.ss.streams[stream_id].width,
                                   params_.ss.streams[stream_id].height));
 
   // TODO(ivica): Remove bitrate_config and use the default Call::Config(), to
   // match the full stack tests.
   Call::Config call_config;
   call_config.bitrate_config = params_.common.call_bitrate_config;
+
+  ::VoiceEngineState voe;
+  if (params_.audio) {
+    CreateVoiceEngine(&voe, decoder_factory_);
+    AudioState::Config audio_state_config;
+    audio_state_config.voice_engine = voe.voice_engine;
+    call_config.audio_state = AudioState::Create(audio_state_config);
+  }
+
   std::unique_ptr<Call> call(Call::Create(call_config));
 
   test::LayerFilteringTransport transport(
       params.pipe, call.get(), kPayloadTypeVP8, kPayloadTypeVP9,
       params.common.selected_tl, params_.ss.selected_sl);
   // TODO(ivica): Use two calls to be able to merge with RunWithAnalyzer or at
   // least share as much code as possible. That way this test would also match
   // the full stack tests better.
   transport.SetReceiver(call->Receiver());
 
   SetupCommon(&transport, &transport);
 
   video_send_config_.local_renderer = local_preview.get();
   video_receive_configs_[stream_id].renderer = loopback_video.get();
+  if (params_.audio && params_.audio_video_sync)
+    video_receive_configs_[stream_id].sync_group = kSyncGroup;
 
   video_send_config_.suspend_below_min_bitrate =
       params_.common.suspend_below_min_bitrate;
 
   if (params.common.fec) {
     video_send_config_.rtp.fec.red_payload_type = kRedPayloadType;
     video_send_config_.rtp.fec.ulpfec_payload_type = kUlpfecPayloadType;
     video_receive_configs_[stream_id].rtp.fec.red_payload_type =
       kRedPayloadType;
     video_receive_configs_[stream_id].rtp.fec.ulpfec_payload_type =
       kUlpfecPayloadType;
   }
 
   if (params_.screenshare.enabled)
     SetupScreenshare();
 
   video_send_stream_ =
       call->CreateVideoSendStream(video_send_config_, video_encoder_config_);
-  VideoReceiveStream* receive_stream =
+  VideoReceiveStream* video_receive_stream =
       call->CreateVideoReceiveStream(video_receive_configs_[stream_id].Copy());
   CreateCapturer(video_send_stream_->Input());
 
-  receive_stream->Start();
+  AudioReceiveStream* audio_receive_stream = nullptr;
+  if (params_.audio) {
+    audio_send_config_ = AudioSendStream::Config(&transport);
+    audio_send_config_.voe_channel_id = voe.send_channel_id;
+    audio_send_config_.rtp.ssrc = kAudioSendSsrc;
+
+    // Add extension to enable audio send side BWE, and allow audio bit rate
+    // adaptation.
+    audio_send_config_.rtp.extensions.clear();
+    if (params_.common.send_side_bwe) {
+      audio_send_config_.rtp.extensions.push_back(webrtc::RtpExtension(
+          webrtc::RtpExtension::kTransportSequenceNumberUri,
+          test::kTransportSequenceNumberExtensionId));
+      audio_send_config_.min_bitrate_kbps = kOpusMinBitrate / 1000;
+      audio_send_config_.max_bitrate_kbps = kOpusBitrateFb / 1000;
+    }
+
+    audio_send_stream_ = call->CreateAudioSendStream(audio_send_config_);
+
+    AudioReceiveStream::Config audio_config;
+    audio_config.rtp.local_ssrc = kReceiverLocalAudioSsrc;
+    audio_config.rtcp_send_transport = &transport;
+    audio_config.voe_channel_id = voe.receive_channel_id;
+    audio_config.rtp.remote_ssrc = audio_send_config_.rtp.ssrc;
+    audio_config.rtp.transport_cc = params_.common.send_side_bwe;
+    audio_config.rtp.extensions = audio_send_config_.rtp.extensions;
+    audio_config.decoder_factory = decoder_factory_;
+    if (params_.audio_video_sync)
+      audio_config.sync_group = kSyncGroup;
+
+    audio_receive_stream =call->CreateAudioReceiveStream(audio_config);
+
+    const CodecInst kOpusInst = {120, "OPUS", 48000, 960, 2, 64000};
+    EXPECT_EQ(0, voe.codec->SetSendCodec(voe.send_channel_id, kOpusInst));
+  }
+
+  // Start sending and receiving video.
+  video_receive_stream->Start();
   video_send_stream_->Start();
   capturer_->Start();
 
+  if (params_.audio) {
+    // Start receiving audio.
+    audio_receive_stream->Start();
+    EXPECT_EQ(0, voe.base->StartPlayout(voe.receive_channel_id));
+    EXPECT_EQ(0, voe.base->StartReceive(voe.receive_channel_id));
+
+    // Start sending audio.
+    audio_send_stream_->Start();
+    EXPECT_EQ(0, voe.base->StartSend(voe.send_channel_id));
+  }
+
   test::PressEnterToContinue();
 
+  if (params_.audio) {
+    // Stop sending audio.
+    EXPECT_EQ(0, voe.base->StopSend(voe.send_channel_id));
+    audio_send_stream_->Stop();
+
+    // Stop receiving audio.
+    EXPECT_EQ(0, voe.base->StopReceive(voe.receive_channel_id));
+    EXPECT_EQ(0, voe.base->StopPlayout(voe.receive_channel_id));
+    audio_receive_stream->Stop();
+  }
+
+  // Stop receiving and sending video.
   capturer_->Stop();
   video_send_stream_->Stop();
-  receive_stream->Stop();
+  video_receive_stream->Stop();
 
-  call->DestroyVideoReceiveStream(receive_stream);
+  call->DestroyVideoReceiveStream(video_receive_stream);
   call->DestroyVideoSendStream(video_send_stream_);
 
+  if (params_.audio) {
+     call->DestroyAudioSendStream(audio_send_stream_);
+     call->DestroyAudioReceiveStream(audio_receive_stream);
+  }
+
   transport.StopSending();
+  if (params_.audio)
+    DestroyVoiceEngine(&voe);
 }
 
 }  // namespace webrtc