Extract simulcast rate allocation outside of video encoder.

webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h"
 
 #include <algorithm>
 
 // NOTE(ajm): Path provided by gyp.
 #include "libyuv/scale.h"  // NOLINT
 
 #include "webrtc/base/checks.h"
 #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
+#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h"
 #include "webrtc/system_wrappers/include/clock.h"
 
 namespace {
 
 const unsigned int kDefaultMinQp = 2;
 const unsigned int kDefaultMaxQp = 56;
 // Max qp for lowest spatial resolution when doing simulcast.
 const unsigned int kLowestResMaxQp = 45;
 
-uint32_t SumStreamTargetBitrate(int streams, const webrtc::VideoCodec& codec) {
-  uint32_t bitrate_sum = 0;
-  for (int i = 0; i < streams; ++i) {
-    bitrate_sum += codec.simulcastStream[i].targetBitrate;
-  }
-  return bitrate_sum;
-}
-
 uint32_t SumStreamMaxBitrate(int streams, const webrtc::VideoCodec& codec) {
   uint32_t bitrate_sum = 0;
   for (int i = 0; i < streams; ++i) {
     bitrate_sum += codec.simulcastStream[i].maxBitrate;
   }
   return bitrate_sum;
 }
 
 int NumberOfStreams(const webrtc::VideoCodec& codec) {
   int streams =
@@ skipping 38 matching lines @@
       inst->numberOfSimulcastStreams > 1) {
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
   if (inst->codecSpecific.VP8.automaticResizeOn &&
       inst->numberOfSimulcastStreams > 1) {
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
-// TL1 FrameDropper's max time to drop frames.
-const float kTl1MaxTimeToDropFrames = 20.0f;
-
 struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory {
-  ScreenshareTemporalLayersFactory()
-      : tl1_frame_dropper_(kTl1MaxTimeToDropFrames) {}
-
+  ScreenshareTemporalLayersFactory() {}
   virtual ~ScreenshareTemporalLayersFactory() {}
 
   virtual webrtc::TemporalLayers* Create(int num_temporal_layers,
                                          uint8_t initial_tl0_pic_idx) const {
     return new webrtc::ScreenshareLayers(num_temporal_layers, rand(),
                                          webrtc::Clock::GetRealTimeClock());
   }
-
-  mutable webrtc::FrameDropper tl0_frame_dropper_;
-  mutable webrtc::FrameDropper tl1_frame_dropper_;
 };
 
 // An EncodedImageCallback implementation that forwards on calls to a
 // SimulcastEncoderAdapter, but with the stream index it's registered with as
 // the first parameter to Encoded.
 class AdapterEncodedImageCallback : public webrtc::EncodedImageCallback {
  public:
   AdapterEncodedImageCallback(webrtc::SimulcastEncoderAdapter* adapter,
                               size_t stream_idx)
       : adapter_(adapter), stream_idx_(stream_idx) {}
@@ skipping 10 matching lines @@
   webrtc::SimulcastEncoderAdapter* const adapter_;
   const size_t stream_idx_;
 };
 
 }  // namespace
 
 namespace webrtc {
 
 SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory)
     : factory_(factory),
-      encoded_complete_callback_(NULL),
+      encoded_complete_callback_(nullptr),
       implementation_name_("SimulcastEncoderAdapter") {
   memset(&codec_, 0, sizeof(webrtc::VideoCodec));
+  rate_allocator_.reset(new SimulcastRateAllocator(codec_));
 }
 
 SimulcastEncoderAdapter::~SimulcastEncoderAdapter() {
   Release();
 }
 
 int SimulcastEncoderAdapter::Release() {
   // TODO(pbos): Keep the last encoder instance but call ::Release() on it, then
   // re-use this instance in ::InitEncode(). This means that changing
   // resolutions doesn't require reallocation of the first encoder, but only
@@ skipping 27 matching lines @@
   }
 
   int number_of_streams = NumberOfStreams(*inst);
   const bool doing_simulcast = (number_of_streams > 1);
 
   if (doing_simulcast && !ValidSimulcastResolutions(*inst, number_of_streams)) {
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
 
   codec_ = *inst;
+  rate_allocator_.reset(new SimulcastRateAllocator(codec_));
+  std::vector<uint32_t> start_bitrates =
+      rate_allocator_->GetAllocation(codec_.startBitrate);
 
   // Special mode when screensharing on a single stream.
   if (number_of_streams == 1 && inst->mode == kScreensharing) {
     screensharing_tl_factory_.reset(new ScreenshareTemporalLayersFactory());
     codec_.codecSpecific.VP8.tl_factory = screensharing_tl_factory_.get();
   }
 
   std::string implementation_name;
   // Create |number_of_streams| of encoder instances and init them.
   for (int i = 0; i < number_of_streams; ++i) {
     VideoCodec stream_codec;
     bool send_stream = true;
     if (!doing_simulcast) {
       stream_codec = codec_;
       stream_codec.numberOfSimulcastStreams = 1;
     } else {
       bool highest_resolution_stream = (i == (number_of_streams - 1));
-      PopulateStreamCodec(&codec_, i, number_of_streams,
+      uint32_t start_bitrate_kbps = start_bitrates[i];
+      if (!start_bitrate_kbps) {

[noahric, 2016/08/29 21:32:09]

Prefer == 0 for integral values. Most of libjingle does anyway, though it's also
because I put this comment on libjingle CLs :)

[sprang_webrtc, 2016/08/30 13:27:24]

Done.

+        // Set max bitrate of previous encoder, but don't send it.
+        // We need this in order for the multi-encoder to work.
+        start_bitrate_kbps = codec_.simulcastStream[i - 1].maxBitrate;
+      }
+      PopulateStreamCodec(&codec_, i, start_bitrate_kbps,
                           highest_resolution_stream, &stream_codec,
                           &send_stream);
     }
 
     // TODO(ronghuawu): Remove once this is handled in VP8EncoderImpl.
     if (stream_codec.qpMax < kDefaultMinQp) {
       stream_codec.qpMax = kDefaultMaxQp;
     }
 
     VideoEncoder* encoder = factory_->Create();
@@ skipping 145 matching lines @@
   }
   if (new_bitrate_kbit < codec_.minBitrate) {
     new_bitrate_kbit = codec_.minBitrate;
   }
   if (codec_.numberOfSimulcastStreams > 0 &&
       new_bitrate_kbit < codec_.simulcastStream[0].minBitrate) {
     new_bitrate_kbit = codec_.simulcastStream[0].minBitrate;
   }
   codec_.maxFramerate = new_framerate;
 
-  bool send_stream = true;
-  uint32_t stream_bitrate = 0;
+  std::vector<uint32_t> stream_bitrates =
+      rate_allocator_->GetAllocation(new_bitrate_kbit);
   for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) {
-    stream_bitrate = GetStreamBitrate(stream_idx, streaminfos_.size(),
-                                      new_bitrate_kbit, &send_stream);
+    uint32_t stream_bitrate = stream_bitrates[stream_idx];
+    bool send_stream = stream_bitrate > 0;
+    if (!send_stream) {
+      // Not enough bitrate for this stream.
+      // Return our max bitrate of |stream_idx| - 1, but we don't send it. We
+      // need to keep this resolution coding in order for the multi-encoder to
+      // work.

[noahric, 2016/08/29 21:32:09]

I see this comment goes waaaay back in webrtc (in vp8_internal_impl.cc in much
older versions), but it's not obvious to me why it is needed. Maybe there was an
assert somewhere else that the bitrate for each layer had to be >= the bitrate
of the lower layer, so max of stream_idx-1 was the lowest safest bitrate to
pick.

We should see if it works with just using 0. Internal source encoders don't
otherwise know to stop producing frames (they can't otherwise observe the
send_stream state). Today, we are wasting cycles in simulcast cases encoding
streams that aren't being sent, so there could be an interesting improvement
there as well.

If it is needed, it should move into vp8_impl.cc, because my guess is that was
the only place that ever actually needed it (no other external vp8 codec
implementation I'm aware of ever needed it).

[sprang_webrtc, 2016/08/30 13:27:24]

Gotcha, thanks.

vp8_impl.cc already has it's own internal handling of simulcast though. Looking
at WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoder, is seems like
SimulcastEncoderAdapter will only be created if it is an external encoder being
used. Otherwise  webrtc::VideoEncoder::Create(webrtc::VideoEncoder::kVp8) is
called directly, creating a VP8EncoderImpl without the wrapper.

I think I'll change this and set target bitrate to 0 for unused streams. Can you
help check that I don't break too many things upstream?

[noahric, 2016/08/30 18:14:33]

Yup! By inspection, at least, I don't expect problems, but we'll see what
happens :)

+      stream_bitrate = stream_bitrates[stream_idx - 1];
+    }
     // Need a key frame if we have not sent this stream before.
     if (send_stream && !streaminfos_[stream_idx].send_stream) {
       streaminfos_[stream_idx].key_frame_request = true;
     }
     streaminfos_[stream_idx].send_stream = send_stream;
 
     // TODO(holmer): This is a temporary hack for screensharing, where we
     // interpret the startBitrate as the encoder target bitrate. This is
     // to allow for a different max bitrate, so if the codec can't meet
     // the target we still allow it to overshoot up to the max before dropping
@@ skipping 20 matching lines @@
     const CodecSpecificInfo* codecSpecificInfo,
     const RTPFragmentationHeader* fragmentation) {
   CodecSpecificInfo stream_codec_specific = *codecSpecificInfo;
   CodecSpecificInfoVP8* vp8Info = &(stream_codec_specific.codecSpecific.VP8);
   vp8Info->simulcastIdx = stream_idx;
 
   return encoded_complete_callback_->OnEncodedImage(
       encodedImage, &stream_codec_specific, fragmentation);
 }
 
-uint32_t SimulcastEncoderAdapter::GetStreamBitrate(
-    int stream_idx,
-    size_t total_number_of_streams,
-    uint32_t new_bitrate_kbit,
-    bool* send_stream) const {
-  if (total_number_of_streams == 1) {
-    *send_stream = true;
-    return new_bitrate_kbit;
-  }
-
-  // The bitrate needed to start sending this stream is given by the
-  // minimum bitrate allowed for encoding this stream, plus the sum target
-  // rates of all lower streams.
-  uint32_t sum_target_lower_streams =
-      SumStreamTargetBitrate(stream_idx, codec_);
-  uint32_t bitrate_to_send_this_layer =
-      codec_.simulcastStream[stream_idx].minBitrate + sum_target_lower_streams;
-  if (new_bitrate_kbit >= bitrate_to_send_this_layer) {
-    // We have enough bandwidth to send this stream.
-    *send_stream = true;
-    // Bitrate for this stream is the new bitrate (|new_bitrate_kbit|) minus the
-    // sum target rates of the lower streams, and capped to a maximum bitrate.
-    // The maximum cap depends on whether we send the next higher stream.
-    // If we will be sending the next higher stream, |max_rate| is given by
-    // current stream's |targetBitrate|, otherwise it's capped by |maxBitrate|.
-    if (stream_idx < codec_.numberOfSimulcastStreams - 1) {
-      unsigned int max_rate = codec_.simulcastStream[stream_idx].maxBitrate;
-      if (new_bitrate_kbit >=
-          SumStreamTargetBitrate(stream_idx + 1, codec_) +
-              codec_.simulcastStream[stream_idx + 1].minBitrate) {
-        max_rate = codec_.simulcastStream[stream_idx].targetBitrate;
-      }
-      return std::min(new_bitrate_kbit - sum_target_lower_streams, max_rate);
-    } else {
-      // For the highest stream (highest resolution), the |targetBitRate| and
-      // |maxBitrate| are not used. Any excess bitrate (above the targets of
-      // all lower streams) is given to this (highest resolution) stream.
-      return new_bitrate_kbit - sum_target_lower_streams;
-    }
-  } else {
-    // Not enough bitrate for this stream.
-    // Return our max bitrate of |stream_idx| - 1, but we don't send it. We need
-    // to keep this resolution coding in order for the multi-encoder to work.
-    *send_stream = false;
-    return codec_.simulcastStream[stream_idx - 1].maxBitrate;
-  }
-}
-
 void SimulcastEncoderAdapter::PopulateStreamCodec(
     const webrtc::VideoCodec* inst,
     int stream_index,
-    size_t total_number_of_streams,
+    uint32_t start_bitrate_kbps,
     bool highest_resolution_stream,
     webrtc::VideoCodec* stream_codec,
     bool* send_stream) {
   *stream_codec = *inst;
 
   // Stream specific settings.
   stream_codec->codecSpecific.VP8.numberOfTemporalLayers =
       inst->simulcastStream[stream_index].numberOfTemporalLayers;
   stream_codec->numberOfSimulcastStreams = 0;
   stream_codec->width = inst->simulcastStream[stream_index].width;
@@ skipping 11 matching lines @@
     // kComplexityHigher, which maps to cpu_used = -4.
     int pixels_per_frame = stream_codec->width * stream_codec->height;
     if (pixels_per_frame < 352 * 288) {
       stream_codec->codecSpecific.VP8.complexity = webrtc::kComplexityHigher;
     }
     // Turn off denoising for all streams but the highest resolution.
     stream_codec->codecSpecific.VP8.denoisingOn = false;
   }
   // TODO(ronghuawu): what to do with targetBitrate.
 
-  int stream_bitrate = GetStreamBitrate(stream_index, total_number_of_streams,
-                                        inst->startBitrate, send_stream);
-  stream_codec->startBitrate = stream_bitrate;
+  stream_codec->startBitrate = start_bitrate_kbps;
 }
 
 bool SimulcastEncoderAdapter::Initialized() const {
   return !streaminfos_.empty();
 }
 
 void SimulcastEncoderAdapter::OnDroppedFrame() {
   streaminfos_[0].encoder->OnDroppedFrame();
 }
 
 bool SimulcastEncoderAdapter::SupportsNativeHandle() const {
   // We should not be calling this method before streaminfos_ are configured.
   RTC_DCHECK(!streaminfos_.empty());
   for (const auto& streaminfo : streaminfos_) {
     if (!streaminfo.encoder->SupportsNativeHandle())
       return false;
   }
   return true;
 }
 
 const char* SimulcastEncoderAdapter::ImplementationName() const {
   return implementation_name_.c_str();
 }
 
 }  // namespace webrtc