Adding audio to video_quality_test.

webrtc/video/video_quality_test.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include <stdio.h>
@@ skipping 16 matching lines @@
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_utility.h"
 #include "webrtc/system_wrappers/include/cpu_info.h"
 #include "webrtc/test/layer_filtering_transport.h"
 #include "webrtc/test/run_loop.h"
 #include "webrtc/test/statistics.h"
 #include "webrtc/test/testsupport/fileutils.h"
 #include "webrtc/test/video_renderer.h"
 #include "webrtc/video/video_quality_test.h"
+#include "webrtc/voice_engine/include/voe_base.h"
+#include "webrtc/voice_engine/include/voe_codec.h"
 
 namespace webrtc {
 
 static const int kSendStatsPollingIntervalMs = 1000;
 static const int kPayloadTypeH264 = 122;
 static const int kPayloadTypeVP8 = 123;
 static const int kPayloadTypeVP9 = 124;
 
 class VideoAnalyzer : public PacketReceiver,
                       public Transport,
@@ skipping 1032 matching lines @@
   for (VideoReceiveStream* receive_stream : video_receive_streams_)
     receive_stream->Stop();
   video_send_stream_->Stop();
 
   DestroyStreams();
 
   if (graph_data_output_file)
     fclose(graph_data_output_file);
 }
 
 void VideoQualityTest::RunWithVideoRenderer(const Params& params) {

[stefan-webrtc, 2016/07/11 15:01:10]

The name of this method is no longer good. We should probably rename it.

[minyue-webrtc, 2016/07/12 10:09:36]

any suggestions?

[stefan-webrtc, 2016/08/08 14:11:18]

RunWithRenderer

[minyue-webrtc, 2016/08/15 15:34:31]

RunWithRenderer does not imply that it may contain audio. VideoQualityTest is a
bit limiting already. But I don't think we should change that for now.

[stefan-webrtc, 2016/08/15 15:46:24]

I think RunWithRenderers is better since it doesn't explicitly mentioned video.

Btw, will you follow up with a change which renames this file loopback.cc or
similar? Also make it possible to run with audio only?

The latter will be important, as I mentioned offline.

   params_ = params;
   CheckParams();
 
   std::unique_ptr<test::VideoRenderer> local_preview(
       test::VideoRenderer::Create("Local Preview", params_.common.width,
                                   params_.common.height));
   size_t stream_id = params_.ss.selected_stream;
   std::string title = "Loopback Video";
   if (params_.ss.streams.size() > 1) {
     std::ostringstream s;
     s << stream_id;
     title += " - Stream #" + s.str();
   }
 
   std::unique_ptr<test::VideoRenderer> loopback_video(
       test::VideoRenderer::Create(title.c_str(),
                                   params_.ss.streams[stream_id].width,
                                   params_.ss.streams[stream_id].height));
 
   // TODO(ivica): Remove bitrate_config and use the default Call::Config(), to
   // match the full stack tests.
   Call::Config call_config;
   call_config.bitrate_config = params_.common.call_bitrate_config;
+
+  CreateVoiceEngine();

[stefan-webrtc, 2016/07/11 15:01:10]

I think we should add an option to configure with or without audio in
loopback.cc which uses this class.

[minyue-webrtc, 2016/07/12 10:09:36]

Done.

+  AudioState::Config audio_state_config;
+  audio_state_config.voice_engine = voe_.voice_engine;
+  call_config.audio_state = AudioState::Create(audio_state_config);
+
   std::unique_ptr<Call> call(Call::Create(call_config));
 
   test::LayerFilteringTransport transport(
       params.pipe, call.get(), kPayloadTypeVP8, kPayloadTypeVP9,
       params.common.selected_tl, params_.ss.selected_sl);
   // TODO(ivica): Use two calls to be able to merge with RunWithAnalyzer or at
   // least share as much code as possible. That way this test would also match
   // the full stack tests better.
   transport.SetReceiver(call->Receiver());
 
   SetupCommon(&transport, &transport);
 
   video_send_config_.local_renderer = local_preview.get();
   video_receive_configs_[stream_id].renderer = loopback_video.get();
+  // video_receive_configs_[stream_id].sync_group = kSyncGroup;

[stefan-webrtc, 2016/07/11 15:01:10]

Would be good to be able to test sync. Maybe have an option to enable sync and
use the mic?

[minyue-webrtc, 2016/07/12 10:09:36]

Done. So we can use the actually device instead of fake_audio_device_

 
   video_send_config_.suspend_below_min_bitrate =
       params_.common.suspend_below_min_bitrate;
 
   if (params.common.fec) {
     video_send_config_.rtp.fec.red_payload_type = kRedPayloadType;
     video_send_config_.rtp.fec.ulpfec_payload_type = kUlpfecPayloadType;
     video_receive_configs_[stream_id].rtp.fec.red_payload_type =
       kRedPayloadType;
     video_receive_configs_[stream_id].rtp.fec.ulpfec_payload_type =
       kUlpfecPayloadType;
   }
 
   if (params_.screenshare.enabled)
     SetupScreenshare();
 
   video_send_stream_ =
       call->CreateVideoSendStream(video_send_config_, video_encoder_config_);
-  VideoReceiveStream* receive_stream =
+  VideoReceiveStream* video_receive_stream =
       call->CreateVideoReceiveStream(video_receive_configs_[stream_id].Copy());
   CreateCapturer(video_send_stream_->Input());
 
-  receive_stream->Start();
+  audio_send_config_ = AudioSendStream::Config(&transport);
+  audio_send_config_.voe_channel_id = voe_.send_channel_id;
+  audio_send_config_.rtp.ssrc = kAudioSendSsrc;
+  audio_send_stream_ = call->CreateAudioSendStream(audio_send_config_);
+
+  AudioReceiveStream::Config audio_config;
+  audio_config.rtp.local_ssrc = kReceiverLocalAudioSsrc;
+  audio_config.rtcp_send_transport = &transport;
+  audio_config.voe_channel_id = voe_.receive_channel_id;
+  audio_config.rtp.remote_ssrc = audio_send_config_.rtp.ssrc;
+  audio_config.decoder_factory = decoder_factory_;
+  audio_receive_configs_.push_back(audio_config);
+  AudioReceiveStream* audio_receive_stream =
+      call->CreateAudioReceiveStream(audio_receive_configs_[0]);
+
+  const CodecInst kOpusInst = {120, "OPUS", 48000, 960, 2, 64000};
+  EXPECT_EQ(0, voe_.codec->SetSendCodec(voe_.send_channel_id, kOpusInst));
+
+  // Start sending and receiving video.
+  video_receive_stream->Start();
   video_send_stream_->Start();
   capturer_->Start();
 
+  // Start receiving audio.
+  audio_receive_stream->Start();
+  EXPECT_EQ(0, voe_.base->StartPlayout(voe_.receive_channel_id));
+  EXPECT_EQ(0, voe_.base->StartReceive(voe_.receive_channel_id));
+
+  // Start sending audio.
+  audio_send_stream_->Start();
+  EXPECT_EQ(0, voe_.base->StartSend(voe_.send_channel_id));
+  fake_audio_device_->Start();
+
   test::PressEnterToContinue();
 
+  // Stop sending audio.
+  fake_audio_device_->Stop();
+  EXPECT_EQ(0, voe_.base->StopSend(voe_.send_channel_id));
+  audio_send_stream_->Stop();
+
+  // Stop receiving audio.
+  EXPECT_EQ(0, voe_.base->StopReceive(voe_.receive_channel_id));
+  EXPECT_EQ(0, voe_.base->StopPlayout(voe_.receive_channel_id));
+  audio_receive_stream->Stop();
+
+  // Stop receiving and sending video.
   capturer_->Stop();
   video_send_stream_->Stop();
-  receive_stream->Stop();
+  video_receive_stream->Stop();
 
-  call->DestroyVideoReceiveStream(receive_stream);
+  call->DestroyVideoReceiveStream(video_receive_stream);
   call->DestroyVideoSendStream(video_send_stream_);
+  call->DestroyAudioSendStream(audio_send_stream_);
+  call->DestroyAudioReceiveStream(audio_receive_stream);
 
   transport.StopSending();
+  DestroyVoiceEngine();
+}
+
+void VideoQualityTest::CreateVoiceEngine() {
+  fake_audio_device_.reset(new test::FakeAudioDevice(
+      clock_, test::ResourcePath("voice_engine/audio_long16", "pcm"),
+      test::DriftingClock::kNoDrift));
+  voe_.voice_engine = VoiceEngine::Create();
+  voe_.base = VoEBase::GetInterface(voe_.voice_engine);
+  voe_.codec = VoECodec::GetInterface(voe_.voice_engine);
+  EXPECT_EQ(0, voe_.base->Init(fake_audio_device_.get(), nullptr,
+                               decoder_factory_));
+  Config voe_config;
+  voe_config.Set<VoicePacing>(new VoicePacing(true));
+  voe_.send_channel_id = voe_.base->CreateChannel(voe_config);
+  EXPECT_GE(voe_.send_channel_id, 0);
+  voe_.receive_channel_id = voe_.base->CreateChannel();
+  EXPECT_GE(voe_.receive_channel_id, 0);
+}
+
+void VideoQualityTest::DestroyVoiceEngine() {
+  voe_.base->DeleteChannel(voe_.send_channel_id);
+  voe_.send_channel_id = -1;
+  voe_.base->DeleteChannel(voe_.receive_channel_id);
+  voe_.receive_channel_id = -1;
+  voe_.base->Release();
+  voe_.base = nullptr;
+  voe_.codec->Release();
+  voe_.codec = nullptr;
+
+  VoiceEngine::Delete(voe_.voice_engine);
+  voe_.voice_engine = nullptr;
 }
 
 }  // namespace webrtc