Add support for a forced software encoder fallback.

webrtc/media/engine/videoencodersoftwarefallbackwrapper.cc

 /*
  *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/media/engine/videoencodersoftwarefallbackwrapper.h"
 
 #include "webrtc/media/engine/internalencoderfactory.h"
 #include "webrtc/modules/video_coding/include/video_error_codes.h"
 #include "webrtc/rtc_base/logging.h"
+#include "webrtc/rtc_base/timeutils.h"
+#include "webrtc/system_wrappers/include/field_trial.h"
 
 namespace webrtc {
+namespace {
+const char kVp8ForceFallbackEncoderFieldTrial[] =
+    "WebRTC-VP8-Forced-Fallback-Encoder";
+
+bool EnableForcedFallback(const cricket::VideoCodec& codec) {
+  if (!webrtc::field_trial::IsEnabled(kVp8ForceFallbackEncoderFieldTrial))
+    return false;
+
+  return (PayloadNameToCodecType(codec.name).value_or(kVideoCodecUnknown) ==
+          kVideoCodecVP8);
+}
+
+void GetForcedFallbackParamsFromFieldTrialGroup(
+    VideoEncoderSoftwareFallbackWrapper::ForcedFallbackParams* params) {

[brandtr, 2017/08/04 11:16:23]

RTC_DCHECK(params) ?

[åsapersson, 2017/08/08 10:40:53]

Done.

+  std::string group =
+      webrtc::field_trial::FindFullName(kVp8ForceFallbackEncoderFieldTrial);
+  if (group.empty())
+    return;
+
+  int low_kbps;
+  int high_kbps;
+  int min_low_ms;
+  if (sscanf(group.c_str(), "Enabled-%d,%d,%d", &low_kbps, &high_kbps,
+             &min_low_ms) != 3) {
+    return;

[sprang_webrtc, 2017/08/07 08:59:12]

Maybe log a warning if parsing fails?

[åsapersson, 2017/08/08 10:40:54]

Done.

+  }
+
+  if (min_low_ms <= 0 || low_kbps <= 0 || high_kbps <= low_kbps)
+    return;
+
+  params->low_kbps = low_kbps;
+  params->high_kbps = high_kbps;
+  params->min_low_ms = min_low_ms;
+}
+}  // namespace
 
 VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper(
     const cricket::VideoCodec& codec,
     webrtc::VideoEncoder* encoder)
     : number_of_cores_(0),
       max_payload_size_(0),
       rates_set_(false),
       framerate_(0),
       channel_parameters_set_(false),
       packet_loss_(0),
       rtt_(0),
       codec_(codec),
       encoder_(encoder),
-      callback_(nullptr) {}
+      callback_(nullptr),
+      forced_fallback_enabled_(EnableForcedFallback(codec)) {
+  if (forced_fallback_enabled_)

[brandtr, 2017/08/04 11:16:23]

I'd add {} here, to make it a bit more readable.

[åsapersson, 2017/08/08 10:40:53]

Done.

+    GetForcedFallbackParamsFromFieldTrialGroup(&forced_fallback_);
+}
 
 bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder() {
   cricket::InternalEncoderFactory internal_factory;
   if (!FindMatchingCodec(internal_factory.supported_codecs(), codec_)) {
     LOG(LS_WARNING)
         << "Encoder requesting fallback to codec not supported in software.";
     return false;
   }
   fallback_encoder_.reset(internal_factory.CreateVideoEncoder(codec_));
   if (fallback_encoder_->InitEncode(&codec_settings_, number_of_cores_,
@@ skipping 27 matching lines @@
     int32_t number_of_cores,
     size_t max_payload_size) {
   // Store settings, in case we need to dynamically switch to the fallback
   // encoder after a failed Encode call.
   codec_settings_ = *codec_settings;
   number_of_cores_ = number_of_cores;
   max_payload_size_ = max_payload_size;
   // Clear stored rate/channel parameters.
   rates_set_ = false;
   channel_parameters_set_ = false;
+  ValidateSettingsForForcedFallback();
+
+  if (TryReInitForcedFallbackEncoder()) {
+    return WEBRTC_VIDEO_CODEC_OK;
+  }
+  forced_fallback_.start_ms.reset();
 
   int32_t ret =
       encoder_->InitEncode(codec_settings, number_of_cores, max_payload_size);
   if (ret == WEBRTC_VIDEO_CODEC_OK || codec_.name.empty()) {
     if (fallback_encoder_)
       fallback_encoder_->Release();
     fallback_encoder_.reset();
     if (callback_)
       encoder_->RegisterEncodeCompleteCallback(callback_);
     return ret;
@@ skipping 21 matching lines @@
   // need to Release() whichever one is active.
   if (fallback_encoder_)
     return fallback_encoder_->Release();
   return encoder_->Release();
 }
 
 int32_t VideoEncoderSoftwareFallbackWrapper::Encode(
     const VideoFrame& frame,
     const CodecSpecificInfo* codec_specific_info,
     const std::vector<FrameType>* frame_types) {
+  if (TryReleaseForcedFallbackEncoder() && encoder_->SupportsNativeHandle() &&
+      frame.video_frame_buffer()->type() != VideoFrameBuffer::Type::kNative) {
+    LOG(LS_WARNING) << "Encoder supports native frames, dropping one frame "
+                    << "to avoid possible reconfig due to format change.";
+    return WEBRTC_VIDEO_CODEC_ERROR;

[sprang_webrtc, 2017/08/07 08:59:12]

I know this code isn't new, but I'm not entirely sure I understand what it does.
Could you add a comment?

[åsapersson, 2017/08/08 10:40:53]

Done.

+  }
   if (fallback_encoder_)
     return fallback_encoder_->Encode(frame, codec_specific_info, frame_types);
   int32_t ret = encoder_->Encode(frame, codec_specific_info, frame_types);
   // If requested, try a software fallback.
-  if (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE && InitFallbackEncoder()) {
+  if ((ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE ||
+       (ret == WEBRTC_VIDEO_CODEC_OK && StartForcedFallback())) &&

[brandtr, 2017/08/04 11:16:23]

These conditions are a bit complicated. Could it be expanded into several lines,
to make it more readable?

[åsapersson, 2017/08/08 10:40:54]

Done.

+      InitFallbackEncoder()) {
+    if (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE)
+      forced_fallback_.start_ms.reset();
     if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative &&
         !fallback_encoder_->SupportsNativeHandle()) {
       LOG(LS_WARNING) << "Fallback encoder doesn't support native frames, "
                       << "dropping one frame.";
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
 
     // Fallback was successful, so start using it with this frame.
     return fallback_encoder_->Encode(frame, codec_specific_info, frame_types);
   }
@@ skipping 34 matching lines @@
 VideoEncoderSoftwareFallbackWrapper::GetScalingSettings() const {
   return encoder_->GetScalingSettings();
 }
 
 const char *VideoEncoderSoftwareFallbackWrapper::ImplementationName() const {
   if (fallback_encoder_)
     return fallback_encoder_->ImplementationName();
   return encoder_->ImplementationName();
 }
 
+bool VideoEncoderSoftwareFallbackWrapper::StartForcedFallback() {

[sprang_webrtc, 2017/08/07 08:59:12]

Maybe you can make this something liek a ShouldUseFallback() const method, and
change the state outside? Would be easier to reason about this method then. It
could also be re-used in TryRelease and TryReinit

[åsapersson, 2017/08/08 10:40:53]

Changed the code a bit, ptal.

+  if (!forced_fallback_enabled_ || fallback_encoder_)
+    return false;
+
+  if (bitrate_allocation_.get_sum_kbps() > forced_fallback_.low_kbps ||
+      (codec_settings_.width * codec_settings_.height >
+       forced_fallback_.max_pixels)) {
+    forced_fallback_.start_ms.reset();
+    return false;
+  }
+  if (!forced_fallback_.start_ms) {
+    forced_fallback_.start_ms.emplace(rtc::TimeMillis());
+    return false;
+  }
+  if ((rtc::TimeMillis() - *forced_fallback_.start_ms) >=
+      forced_fallback_.min_low_ms) {
+    LOG(LS_INFO) << "Start forced sw-encoder fallback.";
+    return true;
+  }
+  return false;
+}
+
+bool VideoEncoderSoftwareFallbackWrapper::TryReleaseForcedFallbackEncoder() {
+  if (!forced_fallback_enabled_ || !fallback_encoder_ ||
+      !forced_fallback_.start_ms) {
+    return false;
+  }
+
+  if (bitrate_allocation_.get_sum_kbps() < forced_fallback_.high_kbps)
+    return false;
+
+  if (encoder_->InitEncode(&codec_settings_, number_of_cores_,
+                           max_payload_size_) == WEBRTC_VIDEO_CODEC_OK) {
+    LOG(LS_INFO) << "Stop forced sw-encoder fallback, max bitrate exceeded.";
+    fallback_encoder_->Release();
+    fallback_encoder_.reset();
+    forced_fallback_.start_ms.reset();
+    if (callback_)
+      encoder_->RegisterEncodeCompleteCallback(callback_);
+    return true;
+  }
+  return false;
+}
+
+bool VideoEncoderSoftwareFallbackWrapper::TryReInitForcedFallbackEncoder() {
+  if (!forced_fallback_enabled_ || !fallback_encoder_ ||
+      !forced_fallback_.start_ms) {

[brandtr, 2017/08/04 11:16:23]

Why do we need to check start_ms here?

[åsapersson, 2017/08/08 10:40:53]

Checks if it is the forced fallback that is in use. Added method
IsForcedFallbackActive.

+    return false;
+  }
+  if (codec_settings_.width * codec_settings_.height >
+      forced_fallback_.max_pixels) {
+    LOG(LS_INFO) << "Stop forced sw-encoder fallback, max pixels exceeded.";
+    return false;
+  }
+  if (fallback_encoder_->InitEncode(&codec_settings_, number_of_cores_,
+                                    max_payload_size_) !=
+      WEBRTC_VIDEO_CODEC_OK) {
+    LOG(LS_ERROR) << "Failed to re-init forced sw-encoder fallback.";
+    fallback_encoder_->Release();
+    fallback_encoder_.reset();
+    return false;
+  }
+  return true;
+}
+
+void VideoEncoderSoftwareFallbackWrapper::ValidateSettingsForForcedFallback() {
+  if (!forced_fallback_enabled_)
+    return;
+
+  bool valid = codec_settings_.codecType == kVideoCodecVP8 &&
+               codec_settings_.numberOfSimulcastStreams <= 1 &&
+               codec_settings_.VP8()->numberOfTemporalLayers == 1;
+
+  if (!valid) {
+    forced_fallback_enabled_ = false;
+    LOG(LS_INFO) << "Disable forced sw-encoder fallback due to settings.";
+    if (fallback_encoder_ && forced_fallback_.start_ms) {
+      fallback_encoder_->Release();
+      fallback_encoder_.reset();
+    }
+  }
+}
+
 }  // namespace webrtc