Add full stack tests for MediaCodec.

webrtc/video/video_quality_test.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 #include "webrtc/video/video_quality_test.h"
 
 #include <stdio.h>
 #include <algorithm>
 #include <deque>
 #include <map>
 #include <set>
 #include <sstream>
 #include <string>
 #include <vector>
 
+#if defined(WEBRTC_ANDROID)
+#include "webrtc/modules/video_coding/codecs/test/android_test_initializer.h"
+#include "webrtc/sdk/android/src/jni/androidmediadecoder_jni.h"
+#include "webrtc/sdk/android/src/jni/androidmediaencoder_jni.h"
+#elif defined(WEBRTC_IOS)
+#include "webrtc/modules/video_coding/codecs/test/objc_codec_h264_test.h"
+#endif
+
 #include "webrtc/api/optional.h"
 #include "webrtc/call/call.h"
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
 #include "webrtc/logging/rtc_event_log/rtc_event_log.h"
+#include "webrtc/media/engine/internalencoderfactory.h"
+#include "webrtc/media/engine/videoencodersoftwarefallbackwrapper.h"
 #include "webrtc/media/engine/webrtcvideoengine.h"
 #include "webrtc/modules/audio_mixer/audio_mixer_impl.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_format.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_utility.h"
 #include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
 #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
 #include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
 #include "webrtc/rtc_base/checks.h"
@@ skipping 1084 matching lines @@
   rtc::PlatformThread stats_polling_thread_;
   rtc::Event comparison_available_event_;
   std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
   rtc::Event done_;
 
   std::unique_ptr<test::RtpFileWriter> rtp_file_writer_;
   Clock* const clock_;
   const int64_t start_ms_;
 };
 
-class Vp8EncoderFactory : public cricket::WebRtcVideoEncoderFactory {
- public:
-  Vp8EncoderFactory() {
-    supported_codecs_.push_back(cricket::VideoCodec("VP8"));
-  }
-  ~Vp8EncoderFactory() override { RTC_CHECK(live_encoders_.empty()); }
-
-  const std::vector<cricket::VideoCodec>& supported_codecs() const override {
-    return supported_codecs_;
-  }
-
-  VideoEncoder* CreateVideoEncoder(const cricket::VideoCodec& codec) override {
-    VideoEncoder* encoder = VP8Encoder::Create();
-    live_encoders_.insert(encoder);
-    return encoder;
-  }
-
-  void DestroyVideoEncoder(VideoEncoder* encoder) override {
-    auto it = live_encoders_.find(encoder);
-    RTC_CHECK(it != live_encoders_.end());
-    live_encoders_.erase(it);
-    delete encoder;
-  }
-
- private:
-  std::vector<cricket::VideoCodec> supported_codecs_;
-  std::set<VideoEncoder*> live_encoders_;
-};
-
 VideoQualityTest::VideoQualityTest()
     : clock_(Clock::GetRealTimeClock()), receive_logs_(0), send_logs_(0) {
   payload_type_map_ = test::CallTest::payload_type_map_;
   RTC_DCHECK(payload_type_map_.find(kPayloadTypeH264) ==
              payload_type_map_.end());
   RTC_DCHECK(payload_type_map_.find(kPayloadTypeVP8) ==
              payload_type_map_.end());
   RTC_DCHECK(payload_type_map_.find(kPayloadTypeVP9) ==
              payload_type_map_.end());
   payload_type_map_[kPayloadTypeH264] = webrtc::MediaType::VIDEO;
   payload_type_map_[kPayloadTypeVP8] = webrtc::MediaType::VIDEO;
   payload_type_map_[kPayloadTypeVP9] = webrtc::MediaType::VIDEO;
+
+#if defined(WEBRTC_ANDROID)
+  InitializeAndroidObjects();
+#endif
 }
 
 VideoQualityTest::Params::Params()
     : call({false, Call::Config::BitrateConfig(), 0}),
-      video({false, 640, 480, 30, 50, 800, 800, false, "VP8", 1, -1, 0, false,
-             false, ""}),
+      video({false, 640, 480, 30, 50, 800, 800, false, "VP8", false, false, 1,
+             -1, 0, false, false, ""}),
       audio({false, false, false}),
       screenshare({false, false, 10, 0}),
       analyzer({"", 0.0, 0.0, 0, "", ""}),
       pipe(),
       ss({std::vector<VideoStream>(), 0, 0, -1, std::vector<SpatialLayer>()}),
       logging({false, "", "", ""}) {}
 
 VideoQualityTest::Params::~Params() = default;
 
 void VideoQualityTest::TestBody() {}
@@ skipping 218 matching lines @@
 
 void VideoQualityTest::SetupVideo(Transport* send_transport,
                                   Transport* recv_transport) {
   if (params_.logging.logs)
     trace_to_stderr_.reset(new test::TraceToStderr);
 
   size_t num_video_streams = params_.ss.streams.size();
   size_t num_flexfec_streams = params_.video.flexfec ? 1 : 0;
   CreateSendConfig(num_video_streams, 0, num_flexfec_streams, send_transport);
 
+  if (params_.video.hw_encoder) {
+#if defined(WEBRTC_ANDROID)
+    video_encoder_factory_.reset(new jni::MediaCodecVideoEncoderFactory());
+#elif defined(WEBRTC_IOS)
+    video_encoder_factory_ = CreateObjCEncoderFactory();
+#else
+    RTC_NOTREACHED() << "Only support HW encoder on Android and iOS.";
+#endif
+  } else {
+    video_encoder_factory_.reset(new cricket::InternalEncoderFactory());
+  }
+
+  cricket::VideoCodec codec;
   int payload_type;
   if (params_.video.codec == "H264") {
-    video_encoder_.reset(H264Encoder::Create(cricket::VideoCodec("H264")));
+    // TODO(brandtr): Add support for high profile here.
+    codec = cricket::VideoCodec(cricket::kH264CodecName);
+    video_encoder_.reset(video_encoder_factory_->CreateVideoEncoder(codec));
     payload_type = kPayloadTypeH264;
   } else if (params_.video.codec == "VP8") {
+    codec = cricket::VideoCodec(cricket::kVp8CodecName);
     if (params_.screenshare.enabled && params_.ss.streams.size() > 1) {
-      // Simulcast screenshare needs a simulcast encoder adapter to work, since
-      // encoders usually can't natively do simulcast with different frame rates
-      // for the different layers.
+      // Simulcast screenshare needs a simulcast encoder adapter to work,
+      // since encoders usually can't natively do simulcast with
+      // different frame rates for the different layers.
       video_encoder_.reset(
-          new SimulcastEncoderAdapter(new Vp8EncoderFactory()));
+          new SimulcastEncoderAdapter(video_encoder_factory_.get()));
     } else {
-      video_encoder_.reset(VP8Encoder::Create());
+      video_encoder_.reset(video_encoder_factory_->CreateVideoEncoder(codec));
     }
     payload_type = kPayloadTypeVP8;
   } else if (params_.video.codec == "VP9") {
-    video_encoder_.reset(VP9Encoder::Create());
+    codec = cricket::VideoCodec(cricket::kVp9CodecName);
+    video_encoder_.reset(video_encoder_factory_->CreateVideoEncoder(codec));
     payload_type = kPayloadTypeVP9;
   } else {
     RTC_NOTREACHED() << "Codec not supported!";
     return;
   }
+
+  if (params_.video.sw_fallback_encoder) {
+    video_encoder_ = rtc::MakeUnique<VideoEncoderSoftwareFallbackWrapper>(
+        codec, std::move(video_encoder_));
+  }
+
   video_send_config_.encoder_settings.encoder = video_encoder_.get();
   video_send_config_.encoder_settings.payload_name = params_.video.codec;
   video_send_config_.encoder_settings.payload_type = payload_type;
   video_send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
   video_send_config_.rtp.rtx.payload_type = kSendRtxPayloadType;
   for (size_t i = 0; i < num_video_streams; ++i)
     video_send_config_.rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[i]);
 
   video_send_config_.rtp.extensions.clear();
   if (params_.call.send_side_bwe) {
@@ skipping 26 matching lines @@
         new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
             params_.video.codec, params_.ss.streams[0].max_qp,
             params_.video.fps, params_.screenshare.enabled, true);
   } else {
     video_encoder_config_.video_stream_factory =
         new rtc::RefCountedObject<VideoStreamFactory>(params_.ss.streams);
   }
 
   video_encoder_config_.spatial_layers = params_.ss.spatial_layers;
 
   CreateMatchingReceiveConfigs(recv_transport);

[brandtr, 2017/09/08 11:31:14]

The decoder is instantiated during this call. Adding support for HW decode thus
requires a bit more work, and can be done in a followup CL.

 
   const bool decode_all_receive_streams =
       params_.ss.selected_stream == params_.ss.streams.size();
 
   for (size_t i = 0; i < num_video_streams; ++i) {
     video_receive_configs_[i].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
     video_receive_configs_[i].rtp.rtx_ssrc = kSendRtxSsrcs[i];
     video_receive_configs_[i]
         .rtp.rtx_associated_payload_types[kSendRtxPayloadType] = payload_type;
     video_receive_configs_[i].rtp.transport_cc = params_.call.send_side_bwe;
@@ skipping 459 matching lines @@
                              test::kTransportSequenceNumberExtensionId));
     audio_send_config_.min_bitrate_bps = kOpusMinBitrateBps;
     audio_send_config_.max_bitrate_bps = kOpusBitrateFbBps;
   }
   audio_send_config_.send_codec_spec =
       rtc::Optional<AudioSendStream::Config::SendCodecSpec>(
           {kAudioSendPayloadType,
            {"OPUS", 48000, 2,
             {{"usedtx", (params_.audio.dtx ? "1" : "0")},
               {"stereo", "1"}}}});
-  audio_send_config_.encoder_factory = encoder_factory_;
+  audio_send_config_.encoder_factory = audio_encoder_factory_;
   audio_send_stream_ = sender_call_->CreateAudioSendStream(audio_send_config_);
 
   AudioReceiveStream::Config audio_config;
   audio_config.rtp.local_ssrc = kReceiverLocalAudioSsrc;
   audio_config.rtcp_send_transport = transport;
   audio_config.voe_channel_id = receive_channel_id;
   audio_config.rtp.remote_ssrc = audio_send_config_.rtp.ssrc;
   audio_config.rtp.transport_cc = params_.call.send_side_bwe;
   audio_config.rtp.extensions = audio_send_config_.rtp.extensions;
-  audio_config.decoder_factory = decoder_factory_;
+  audio_config.decoder_factory = audio_decoder_factory_;
   audio_config.decoder_map = {{kAudioSendPayloadType, {"OPUS", 48000, 2}}};
   if (params_.video.enabled && params_.audio.sync_video)
     audio_config.sync_group = kSyncGroup;
 
   *audio_receive_stream =
       receiver_call_->CreateAudioReceiveStream(audio_config);
 }
 
 void VideoQualityTest::RunWithRenderers(const Params& params) {
   std::unique_ptr<test::LayerFilteringTransport> send_transport;
   std::unique_ptr<test::DirectTransport> recv_transport;
   ::VoiceEngineState voe;
   std::unique_ptr<test::VideoRenderer> local_preview;
   std::vector<std::unique_ptr<test::VideoRenderer>> loopback_renderers;
   AudioReceiveStream* audio_receive_stream = nullptr;
 
   task_queue_.SendTask([&]() {
     params_ = params;
     CheckParams();
 
     // TODO(ivica): Remove bitrate_config and use the default Call::Config(), to
     // match the full stack tests.
     Call::Config call_config(event_log_.get());
     call_config.bitrate_config = params_.call.call_bitrate_config;
 
     rtc::scoped_refptr<webrtc::AudioProcessing> audio_processing(
         webrtc::AudioProcessing::Create());
 
     if (params_.audio.enabled) {
-      CreateVoiceEngine(&voe, audio_processing.get(), decoder_factory_);
+      CreateVoiceEngine(&voe, audio_processing.get(), audio_decoder_factory_);
       AudioState::Config audio_state_config;
       audio_state_config.voice_engine = voe.voice_engine;
       audio_state_config.audio_mixer = AudioMixerImpl::Create();
       audio_state_config.audio_processing = audio_processing;
       call_config.audio_state = AudioState::Create(audio_state_config);
     }
 
     CreateCalls(call_config, call_config);
 
     // TODO(minyue): consider if this is a good transport even for audio only
@@ skipping 158 matching lines @@
   if (!params_.logging.encoded_frame_base_path.empty()) {
     std::ostringstream str;
     str << receive_logs_++;
     std::string path =
         params_.logging.encoded_frame_base_path + "." + str.str() + ".recv.ivf";
     stream->EnableEncodedFrameRecording(rtc::CreatePlatformFile(path),
                                         100000000);
   }
 }
 }  // namespace webrtc