Prepare for injectable SW decoders

webrtc/media/engine/webrtcvideoengine.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/media/engine/webrtcvideoengine.h"
 
 #include <stdio.h>
 #include <algorithm>
 #include <set>
 #include <string>
 #include <utility>
 
 #include "webrtc/api/video/i420_buffer.h"
 #include "webrtc/api/video_codecs/video_decoder.h"
 #include "webrtc/api/video_codecs/video_encoder.h"
 #include "webrtc/call/call.h"
 #include "webrtc/common_video/h264/profile_level_id.h"
 #include "webrtc/media/engine/constants.h"
 #include "webrtc/media/engine/internaldecoderfactory.h"
 #include "webrtc/media/engine/internalencoderfactory.h"
+#include "webrtc/media/engine/scopedvideodecoder.h"
 #include "webrtc/media/engine/scopedvideoencoder.h"
 #include "webrtc/media/engine/simulcast.h"
 #include "webrtc/media/engine/simulcast_encoder_adapter.h"
 #include "webrtc/media/engine/videodecodersoftwarefallbackwrapper.h"
 #include "webrtc/media/engine/videoencodersoftwarefallbackwrapper.h"
 #include "webrtc/media/engine/webrtcmediaengine.h"
 #include "webrtc/media/engine/webrtcvideoencoderfactory.h"
 #include "webrtc/media/engine/webrtcvoiceengine.h"
 #include "webrtc/rtc_base/copyonwritebuffer.h"
 #include "webrtc/rtc_base/logging.h"
@@ skipping 27 matching lines @@
 
   virtual AllocatedEncoder CreateVideoEncoder(
       const VideoCodec& codec,
       bool is_conference_mode_screenshare) const = 0;
 
   virtual std::vector<VideoCodec> GetSupportedCodecs() const = 0;
 
   virtual std::unique_ptr<EncoderFactoryAdapter> clone() const = 0;
 };
 
+class DecoderFactoryAdapter {
+ public:
+  struct AllocatedDecoder {
+    AllocatedDecoder() = default;
+    AllocatedDecoder(std::unique_ptr<webrtc::VideoDecoder> decoder,
+                     bool is_hardware_accelerated);
+
+    std::unique_ptr<webrtc::VideoDecoder> decoder;
+    bool is_hardware_accelerated;

[magjed_webrtc, 2017/09/01 11:15:04]

The variable 'is_hardware_accelerated' is not needed to store for decoders, so
you can remove the whole AllocatedDecoder struct and work directly with
webrtc::VideoDecoder.

[andersc, 2017/09/01 12:43:45]

Removed in CL https://codereview.webrtc.org/3011823002

[magjed_webrtc, 2017/09/01 13:04:30]

Since you are already changing this logic I think it's best if you merge that
removal into this CL.

[andersc, 2017/09/01 13:16:08]

Done.

+  };
+
+  virtual ~DecoderFactoryAdapter() {}
+
+  virtual AllocatedDecoder CreateVideoDecoder(
+      const webrtc::VideoCodecType type,
+      const StreamParams stream_params) const = 0;
+
+  virtual std::unique_ptr<DecoderFactoryAdapter> clone() const = 0;
+};
+
 namespace {
 
 // Wraps cricket::WebRtcVideoEncoderFactory* into common EncoderFactoryAdapter
 // interface.
 // TODO(magjed): Add wrapper class for future webrtc::VideoEncoderFactory
 // interface, https://bugs.chromium.org/p/webrtc/issues/detail?id=7925.
 class CricketEncoderFactoryAdapter : public EncoderFactoryAdapter {
  public:
   explicit CricketEncoderFactoryAdapter(
       WebRtcVideoEncoderFactory* external_encoder_factory)
@@ skipping 13 matching lines @@
 
   std::unique_ptr<EncoderFactoryAdapter> clone() const override {
     return std::unique_ptr<EncoderFactoryAdapter>(
         new CricketEncoderFactoryAdapter(*this));
   }
 
   const std::unique_ptr<WebRtcVideoEncoderFactory> internal_encoder_factory_;
   WebRtcVideoEncoderFactory* const external_encoder_factory_;
 };
 
+class CricketDecoderFactoryAdapter : public DecoderFactoryAdapter {
+ public:
+  explicit CricketDecoderFactoryAdapter(
+      WebRtcVideoDecoderFactory* external_decoder_factory)
+      : internal_decoder_factory_(new InternalDecoderFactory()),
+        external_decoder_factory_(external_decoder_factory) {}
+
+ private:
+  explicit CricketDecoderFactoryAdapter(
+      const CricketDecoderFactoryAdapter& other)
+      : CricketDecoderFactoryAdapter(other.external_decoder_factory_) {}
+
+  AllocatedDecoder CreateVideoDecoder(
+      const webrtc::VideoCodecType type,
+      const StreamParams stream_params) const override;
+
+  std::unique_ptr<DecoderFactoryAdapter> clone() const override {
+    return std::unique_ptr<DecoderFactoryAdapter>(
+        new CricketDecoderFactoryAdapter(*this));
+  }
+
+  const std::unique_ptr<WebRtcVideoDecoderFactory> internal_decoder_factory_;
+  WebRtcVideoDecoderFactory* const external_decoder_factory_;
+};
+
 // If this field trial is enabled, we will enable sending FlexFEC and disable
 // sending ULPFEC whenever the former has been negotiated in the SDPs.
 bool IsFlexfecFieldTrialEnabled() {
   return webrtc::field_trial::IsEnabled("WebRTC-FlexFEC-03");
 }
 
 // If this field trial is enabled, the "flexfec-03" codec may have been
 // advertised as being supported in the local SDP. That means that we must be
 // ready to receive FlexFEC packets. See internalencoderfactory.cc.
 bool IsFlexfecAdvertisedFieldTrialEnabled() {
@@ skipping 243 matching lines @@
   default_sink_ = sink;
   rtc::Optional<uint32_t> default_recv_ssrc =
       channel->GetDefaultReceiveStreamSsrc();
   if (default_recv_ssrc) {
     channel->SetSink(*default_recv_ssrc, default_sink_);
   }
 }
 
 WebRtcVideoEngine::WebRtcVideoEngine()
     : initialized_(false),
-      external_decoder_factory_(NULL),
+      decoder_factory_(new CricketDecoderFactoryAdapter(
+          nullptr /* external_decoder_factory */)),
       encoder_factory_(new CricketEncoderFactoryAdapter(
           nullptr /* external_encoder_factory */)) {
   LOG(LS_INFO) << "WebRtcVideoEngine::WebRtcVideoEngine()";
 }
 
 WebRtcVideoEngine::~WebRtcVideoEngine() {
   LOG(LS_INFO) << "WebRtcVideoEngine::~WebRtcVideoEngine";
 }
 
 void WebRtcVideoEngine::Init() {
   LOG(LS_INFO) << "WebRtcVideoEngine::Init";
   initialized_ = true;
 }
 
 WebRtcVideoChannel* WebRtcVideoEngine::CreateChannel(
     webrtc::Call* call,
     const MediaConfig& config,
     const VideoOptions& options) {
   RTC_DCHECK(initialized_);
   LOG(LS_INFO) << "CreateChannel. Options: " << options.ToString();
   return new WebRtcVideoChannel(call, config, options, *encoder_factory_,
-                                external_decoder_factory_);
+                                *decoder_factory_);
 }
 
 std::vector<VideoCodec> WebRtcVideoEngine::codecs() const {
   return encoder_factory_->GetSupportedCodecs();
 }
 
 RtpCapabilities WebRtcVideoEngine::GetCapabilities() const {
   RtpCapabilities capabilities;
   capabilities.header_extensions.push_back(
       webrtc::RtpExtension(webrtc::RtpExtension::kTimestampOffsetUri,
@@ skipping 15 matching lines @@
                            webrtc::RtpExtension::kVideoContentTypeDefaultId));
   capabilities.header_extensions.push_back(
         webrtc::RtpExtension(webrtc::RtpExtension::kVideoTimingUri,
                              webrtc::RtpExtension::kVideoTimingDefaultId));
   return capabilities;
 }
 
 void WebRtcVideoEngine::SetExternalDecoderFactory(
     WebRtcVideoDecoderFactory* decoder_factory) {
   RTC_DCHECK(!initialized_);
-  external_decoder_factory_ = decoder_factory;
+  decoder_factory_.reset(new CricketDecoderFactoryAdapter(decoder_factory));
 }
 
 void WebRtcVideoEngine::SetExternalEncoderFactory(
     WebRtcVideoEncoderFactory* encoder_factory) {
   RTC_DCHECK(!initialized_);
   encoder_factory_.reset(new CricketEncoderFactoryAdapter(encoder_factory));
 }
 
 // This function will assign dynamic payload types (in the range [96, 127]) to
 // the input codecs, and also add associated RTX codecs for recognized codecs
@@ skipping 57 matching lines @@
   }
 
   return AssignPayloadTypesAndAddAssociatedRtxCodecs(codecs);
 }
 
 WebRtcVideoChannel::WebRtcVideoChannel(
     webrtc::Call* call,
     const MediaConfig& config,
     const VideoOptions& options,
     const EncoderFactoryAdapter& encoder_factory,
-    WebRtcVideoDecoderFactory* external_decoder_factory)
+    const DecoderFactoryAdapter& decoder_factory)
     : VideoMediaChannel(config),
       call_(call),
       unsignalled_ssrc_handler_(&default_unsignalled_ssrc_handler_),
       video_config_(config.video),
       encoder_factory_(encoder_factory.clone()),
-      external_decoder_factory_(external_decoder_factory),
+      decoder_factory_(decoder_factory.clone()),
       default_send_options_(options),
       last_stats_log_ms_(-1) {
   RTC_DCHECK(thread_checker_.CalledOnValidThread());
 
   rtcp_receiver_report_ssrc_ = kDefaultRtcpReceiverReportSsrc;
   sending_ = false;
   recv_codecs_ = MapCodecs(encoder_factory_->GetSupportedCodecs());
   recv_flexfec_payload_type_ = recv_codecs_.front().flexfec_payload_type;
 }
 
@@ skipping 595 matching lines @@
 
   webrtc::VideoReceiveStream::Config config(this);
   webrtc::FlexfecReceiveStream::Config flexfec_config(this);
   ConfigureReceiverRtp(&config, &flexfec_config, sp);
 
   config.disable_prerenderer_smoothing =
       video_config_.disable_prerenderer_smoothing;
   config.sync_group = sp.sync_label;
 
   receive_streams_[ssrc] = new WebRtcVideoReceiveStream(
-      call_, sp, std::move(config), external_decoder_factory_, default_stream,
+      call_, sp, std::move(config), *decoder_factory_, default_stream,
       recv_codecs_, flexfec_config);
 
   return true;
 }
 
 void WebRtcVideoChannel::ConfigureReceiverRtp(
     webrtc::VideoReceiveStream::Config* config,
     webrtc::FlexfecReceiveStream::Config* flexfec_config,
     const StreamParams& sp) const {
   uint32_t ssrc = sp.first_ssrc();
@@ skipping 917 matching lines @@
   }
 
   // Call stream_->Start() if necessary conditions are met.
   UpdateSendState();
 }
 
 WebRtcVideoChannel::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream(
     webrtc::Call* call,
     const StreamParams& sp,
     webrtc::VideoReceiveStream::Config config,
-    WebRtcVideoDecoderFactory* external_decoder_factory,
+    const DecoderFactoryAdapter& decoder_factory,
     bool default_stream,
     const std::vector<VideoCodecSettings>& recv_codecs,
     const webrtc::FlexfecReceiveStream::Config& flexfec_config)
     : call_(call),
       stream_params_(sp),
       stream_(NULL),
       default_stream_(default_stream),
       config_(std::move(config)),
       flexfec_config_(flexfec_config),
       flexfec_stream_(nullptr),
-      external_decoder_factory_(external_decoder_factory),
+      decoder_factory_(decoder_factory.clone()),
       sink_(NULL),
       first_frame_timestamp_(-1),
       estimated_remote_start_ntp_time_ms_(0) {
   config_.renderer = this;
-  std::vector<AllocatedDecoder> old_decoders;
+  std::map<webrtc::VideoCodecType, std::unique_ptr<webrtc::VideoDecoder>>
+      old_decoders;
   ConfigureCodecs(recv_codecs, &old_decoders);
   ConfigureFlexfecCodec(flexfec_config.payload_type);
   MaybeRecreateWebRtcFlexfecStream();
   RecreateWebRtcVideoStream();
   RTC_DCHECK(old_decoders.empty());
 }
 
-WebRtcVideoChannel::WebRtcVideoReceiveStream::AllocatedDecoder::
-    AllocatedDecoder(webrtc::VideoDecoder* decoder,
-                     webrtc::VideoCodecType type,
-                     bool external)
-    : decoder(decoder),
-      external_decoder(nullptr),
-      type(type),
-      external(external) {
-  if (external) {
-    external_decoder = decoder;
-    this->decoder =
-        new webrtc::VideoDecoderSoftwareFallbackWrapper(type, external_decoder);
-  }
-}
+DecoderFactoryAdapter::AllocatedDecoder::AllocatedDecoder(
+    std::unique_ptr<webrtc::VideoDecoder> decoder,
+    bool is_hardware_accelerated)
+    : decoder(std::move(decoder)),
+      is_hardware_accelerated(is_hardware_accelerated) {}
 
 WebRtcVideoChannel::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() {
   if (flexfec_stream_) {
     MaybeDissociateFlexfecFromVideo();
     call_->DestroyFlexfecReceiveStream(flexfec_stream_);
   }
   call_->DestroyVideoReceiveStream(stream_);
-  ClearDecoders(&allocated_decoders_);
+  allocated_decoders_.clear();
 }
 
 const std::vector<uint32_t>&
 WebRtcVideoChannel::WebRtcVideoReceiveStream::GetSsrcs() const {
   return stream_params_.ssrcs;
 }
 
 rtc::Optional<uint32_t>
 WebRtcVideoChannel::WebRtcVideoReceiveStream::GetFirstPrimarySsrc() const {
   std::vector<uint32_t> primary_ssrcs;
   stream_params_.GetPrimarySsrcs(&primary_ssrcs);
 
   if (primary_ssrcs.empty()) {
     LOG(LS_WARNING) << "Empty primary ssrcs vector, returning empty optional";
     return rtc::Optional<uint32_t>();
   } else {
     return rtc::Optional<uint32_t>(primary_ssrcs[0]);
   }
 }
 
-WebRtcVideoChannel::WebRtcVideoReceiveStream::AllocatedDecoder
-WebRtcVideoChannel::WebRtcVideoReceiveStream::CreateOrReuseVideoDecoder(
-    std::vector<AllocatedDecoder>* old_decoders,
-    const VideoCodec& codec) {
-  webrtc::VideoCodecType type = webrtc::PayloadStringToCodecType(codec.name);
-
-  for (size_t i = 0; i < old_decoders->size(); ++i) {
-    if ((*old_decoders)[i].type == type) {
-      AllocatedDecoder decoder = (*old_decoders)[i];
-      (*old_decoders)[i] = old_decoders->back();
-      old_decoders->pop_back();
-      return decoder;
+DecoderFactoryAdapter::AllocatedDecoder
+CricketDecoderFactoryAdapter::CreateVideoDecoder(
+    const webrtc::VideoCodecType type,
+    const StreamParams stream_params) const {

[magjed_webrtc, 2017/09/01 11:15:04]

This should be a reference, i.e. 'const StreamParams& stream_params'. We could
also change it to pass in a 'const VideoDecoderParams&' directly, i.e. the thing
we create with {stream_params.id}.

[andersc, 2017/09/01 12:43:45]

Yes, that is nicer

+  if (external_decoder_factory_ != nullptr) {
+    std::unique_ptr<webrtc::VideoDecoder> external_decoder =
+        CreateScopedVideoDecoder(external_decoder_factory_, type,
+                                 {stream_params.id});
+    if (external_decoder) {
+      std::unique_ptr<webrtc::VideoDecoder> internal_decoder(
+          new webrtc::VideoDecoderSoftwareFallbackWrapper(
+              type, std::move(external_decoder)));
+      return AllocatedDecoder(std::move(internal_decoder),
+                              true /* is_hardware_accelerated */);
     }
   }
 
-  if (external_decoder_factory_ != NULL) {
-    webrtc::VideoDecoder* decoder =
-        external_decoder_factory_->CreateVideoDecoderWithParams(
-            type, {stream_params_.id});
-    if (decoder != NULL) {
-      return AllocatedDecoder(decoder, type, true /* is_external */);
-    }
-  }
-
-  InternalDecoderFactory internal_decoder_factory;
-  return AllocatedDecoder(internal_decoder_factory.CreateVideoDecoderWithParams(
-                              type, {stream_params_.id}),
-                          type, false /* is_external */);
+  std::unique_ptr<webrtc::VideoDecoder> internal_decoder =
+      CreateScopedVideoDecoder(internal_decoder_factory_.get(), type,

[magjed_webrtc, 2017/09/01 11:15:04]

I looked at the InternalDecoderFactory::DestroyVideoDecoder() implementation and
it's a pure 'delete decoder', so we don't have to wrap it with
CreateScopedVideoDecoder().

[andersc, 2017/09/01 12:43:45]

Acknowledged.

+                               {stream_params.id});
+  return AllocatedDecoder(std::move(internal_decoder),
+                          false /* is_hardware_accelerated */);
 }
 
 void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs(
     const std::vector<VideoCodecSettings>& recv_codecs,
-    std::vector<AllocatedDecoder>* old_decoders) {
-  *old_decoders = allocated_decoders_;
+    std::map<webrtc::VideoCodecType, std::unique_ptr<webrtc::VideoDecoder>>*
+        old_decoders) {
+  *old_decoders = std::move(allocated_decoders_);

[magjed_webrtc, 2017/09/01 11:15:04]

Hmm, this is a very interesting use case of std::move to optimize performance,
and it's valid :). std::move is mostly used for handling non-copyable objects
like std::unique_ptr.

[andersc, 2017/09/01 12:43:45]

Makes sense.

   allocated_decoders_.clear();
   config_.decoders.clear();
   for (size_t i = 0; i < recv_codecs.size(); ++i) {
-    AllocatedDecoder allocated_decoder =
-        CreateOrReuseVideoDecoder(old_decoders, recv_codecs[i].codec);
-    allocated_decoders_.push_back(allocated_decoder);
+    webrtc::VideoCodecType type =
+        webrtc::PayloadStringToCodecType(recv_codecs[i].codec.name);
+    std::unique_ptr<webrtc::VideoDecoder> new_decoder;
+
+    auto it = old_decoders->find(type);
+    if (it != old_decoders->end()) {
+      new_decoder = std::move((*old_decoders)[type]);
+      old_decoders->erase(type);

[magjed_webrtc, 2017/09/01 11:15:04]

You have an iterator to the object now, so you should do:
new_decoder = std::move(it->second);
old_decoders->erase(it);

[andersc, 2017/09/01 12:43:45]

Acknowledged.

+    }
+
+    if (!new_decoder) {
+      DecoderFactoryAdapter::AllocatedDecoder new_allocated_decoder =
+          decoder_factory_->CreateVideoDecoder(type, stream_params_);
+      new_decoder = std::unique_ptr<webrtc::VideoDecoder>(
+          std::move(new_allocated_decoder.decoder));
+      // TODO(andersc): are we using the is_hardware_accelerated field for

[magjed_webrtc, 2017/09/01 11:15:04]

Indeed this does not seem to be used.

[andersc, 2017/09/01 12:43:45]

Acknowledged.

+      // anything?
+    }
 
     webrtc::VideoReceiveStream::Decoder decoder;
-    decoder.decoder = allocated_decoder.decoder;
+    decoder.decoder = new_decoder.get();
     decoder.payload_type = recv_codecs[i].codec.id;
     decoder.payload_name = recv_codecs[i].codec.name;
     decoder.codec_params = recv_codecs[i].codec.params;
     config_.decoders.push_back(decoder);
+
+    allocated_decoders_.insert(std::make_pair(type, std::move(new_decoder)));
   }
 
   config_.rtp.rtx_associated_payload_types.clear();
   for (const VideoCodecSettings& recv_codec : recv_codecs) {
     config_.rtp.rtx_associated_payload_types[recv_codec.rtx_payload_type] =
         recv_codec.codec.id;
   }
 
   config_.rtp.ulpfec = recv_codecs.front().ulpfec;
 
@@ skipping 57 matching lines @@
       << nack_enabled << ", remb=" << remb_enabled
       << ", transport_cc=" << transport_cc_enabled;
   MaybeRecreateWebRtcFlexfecStream();
   RecreateWebRtcVideoStream();
 }
 
 void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters(
     const ChangedRecvParameters& params) {
   bool video_needs_recreation = false;
   bool flexfec_needs_recreation = false;
-  std::vector<AllocatedDecoder> old_decoders;
+  std::map<webrtc::VideoCodecType, std::unique_ptr<webrtc::VideoDecoder>>
+      old_decoders;
   if (params.codec_settings) {
     ConfigureCodecs(*params.codec_settings, &old_decoders);
     video_needs_recreation = true;
   }
   if (params.rtp_header_extensions) {
     config_.rtp.extensions = *params.rtp_header_extensions;
     flexfec_config_.rtp_header_extensions = *params.rtp_header_extensions;
     video_needs_recreation = true;
     flexfec_needs_recreation = true;
   }
   if (params.flexfec_payload_type) {
     ConfigureFlexfecCodec(*params.flexfec_payload_type);
     flexfec_needs_recreation = true;
   }
   if (flexfec_needs_recreation) {
     LOG(LS_INFO) << "MaybeRecreateWebRtcFlexfecStream (recv) because of "
                     "SetRecvParameters";
     MaybeRecreateWebRtcFlexfecStream();
   }
   if (video_needs_recreation) {
     LOG(LS_INFO)
         << "RecreateWebRtcVideoStream (recv) because of SetRecvParameters";
     RecreateWebRtcVideoStream();
-    ClearDecoders(&old_decoders);
+    old_decoders.clear();
   }
 }
 
 void WebRtcVideoChannel::WebRtcVideoReceiveStream::
     RecreateWebRtcVideoStream() {
   if (stream_) {
     MaybeDissociateFlexfecFromVideo();
     call_->DestroyVideoReceiveStream(stream_);
     stream_ = nullptr;
   }
@@ skipping 24 matching lines @@
   }
 }
 
 void WebRtcVideoChannel::WebRtcVideoReceiveStream::
     MaybeDissociateFlexfecFromVideo() {
   if (stream_ && flexfec_stream_) {
     stream_->RemoveSecondarySink(flexfec_stream_);
   }
 }
 
-void WebRtcVideoChannel::WebRtcVideoReceiveStream::ClearDecoders(
-    std::vector<AllocatedDecoder>* allocated_decoders) {
-  for (size_t i = 0; i < allocated_decoders->size(); ++i) {
-    if ((*allocated_decoders)[i].external) {
-      external_decoder_factory_->DestroyVideoDecoder(
-          (*allocated_decoders)[i].external_decoder);
-    }
-    delete (*allocated_decoders)[i].decoder;
-  }
-  allocated_decoders->clear();
-}
-
 void WebRtcVideoChannel::WebRtcVideoReceiveStream::OnFrame(
     const webrtc::VideoFrame& frame) {
   rtc::CritScope crit(&sink_lock_);
 
   if (first_frame_timestamp_ < 0)
     first_frame_timestamp_ = frame.timestamp();
   int64_t rtp_time_elapsed_since_first_frame =
       (timestamp_wraparound_handler_.Unwrap(frame.timestamp()) -
        first_frame_timestamp_);
   int64_t elapsed_time_ms = rtp_time_elapsed_since_first_frame /
@@ skipping 261 matching lines @@
     stream.temporal_layer_thresholds_bps.resize(GetDefaultVp9TemporalLayers() -
                                                 1);
   }
 
   std::vector<webrtc::VideoStream> streams;
   streams.push_back(stream);
   return streams;
 }
 
 }  // namespace cricket