Extract simulcast rate allocation outside of video encoder.

webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc

Index: webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc
diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc
index eba59d0fdff8d4e280c2235f2a395f7b15e033df..0863f35dee362ed4881504f2a433e17719a8343e 100644
--- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc
+++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc
@@ -17,6 +17,7 @@
 
 #include "webrtc/base/checks.h"
 #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
+#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h"
 #include "webrtc/system_wrappers/include/clock.h"
 
 namespace {
@@ -26,14 +27,6 @@ const unsigned int kDefaultMaxQp = 56;
 // Max qp for lowest spatial resolution when doing simulcast.
 const unsigned int kLowestResMaxQp = 45;
 
-uint32_t SumStreamTargetBitrate(int streams, const webrtc::VideoCodec& codec) {
-  uint32_t bitrate_sum = 0;
-  for (int i = 0; i < streams; ++i) {
-    bitrate_sum += codec.simulcastStream[i].targetBitrate;
-  }
-  return bitrate_sum;
-}
-
 uint32_t SumStreamMaxBitrate(int streams, const webrtc::VideoCodec& codec) {
   uint32_t bitrate_sum = 0;
   for (int i = 0; i < streams; ++i) {
@@ -92,13 +85,8 @@ int VerifyCodec(const webrtc::VideoCodec* inst) {
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
-// TL1 FrameDropper's max time to drop frames.
-const float kTl1MaxTimeToDropFrames = 20.0f;
-
 struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory {
-  ScreenshareTemporalLayersFactory()
-      : tl1_frame_dropper_(kTl1MaxTimeToDropFrames) {}
-
+  ScreenshareTemporalLayersFactory() {}
   virtual ~ScreenshareTemporalLayersFactory() {}
 
   virtual webrtc::TemporalLayers* Create(int num_temporal_layers,
@@ -106,9 +94,6 @@ struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory {
     return new webrtc::ScreenshareLayers(num_temporal_layers, rand(),
                                          webrtc::Clock::GetRealTimeClock());
   }
-
-  mutable webrtc::FrameDropper tl0_frame_dropper_;
-  mutable webrtc::FrameDropper tl1_frame_dropper_;
 };
 
 // An EncodedImageCallback implementation that forwards on calls to a
@@ -139,9 +124,10 @@ namespace webrtc {
 
 SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory)
     : factory_(factory),
-      encoded_complete_callback_(NULL),
+      encoded_complete_callback_(nullptr),
       implementation_name_("SimulcastEncoderAdapter") {
   memset(&codec_, 0, sizeof(webrtc::VideoCodec));
+  rate_allocator_.reset(new SimulcastRateAllocator(codec_));
 }
 
 SimulcastEncoderAdapter::~SimulcastEncoderAdapter() {
@@ -189,6 +175,9 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst,
   }
 
   codec_ = *inst;
+  rate_allocator_.reset(new SimulcastRateAllocator(codec_));
+  std::vector<uint32_t> start_bitrates =
+      rate_allocator_->GetAllocation(codec_.startBitrate);
 
   // Special mode when screensharing on a single stream.
   if (number_of_streams == 1 && inst->mode == kScreensharing) {
@@ -206,7 +195,13 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst,
       stream_codec.numberOfSimulcastStreams = 1;
     } else {
       bool highest_resolution_stream = (i == (number_of_streams - 1));
-      PopulateStreamCodec(&codec_, i, number_of_streams,
+      uint32_t start_bitrate_kbps = start_bitrates[i];
+      if (!start_bitrate_kbps) {

[noahric, 2016/08/29 21:32:09]

Prefer == 0 for integral values. Most of libjingle does anyway, though it's also
because I put this comment on libjingle CLs :)

[sprang_webrtc, 2016/08/30 13:27:24]

Done.

+        // Set max bitrate of previous encoder, but don't send it.
+        // We need this in order for the multi-encoder to work.
+        start_bitrate_kbps = codec_.simulcastStream[i - 1].maxBitrate;
+      }
+      PopulateStreamCodec(&codec_, i, start_bitrate_kbps,
                           highest_resolution_stream, &stream_codec,
                           &send_stream);
     }
@@ -372,11 +367,18 @@ int SimulcastEncoderAdapter::SetRates(uint32_t new_bitrate_kbit,
   }
   codec_.maxFramerate = new_framerate;
 
-  bool send_stream = true;
-  uint32_t stream_bitrate = 0;
+  std::vector<uint32_t> stream_bitrates =
+      rate_allocator_->GetAllocation(new_bitrate_kbit);
   for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) {
-    stream_bitrate = GetStreamBitrate(stream_idx, streaminfos_.size(),
-                                      new_bitrate_kbit, &send_stream);
+    uint32_t stream_bitrate = stream_bitrates[stream_idx];
+    bool send_stream = stream_bitrate > 0;
+    if (!send_stream) {
+      // Not enough bitrate for this stream.
+      // Return our max bitrate of |stream_idx| - 1, but we don't send it. We
+      // need to keep this resolution coding in order for the multi-encoder to
+      // work.

[noahric, 2016/08/29 21:32:09]

I see this comment goes waaaay back in webrtc (in vp8_internal_impl.cc in much
older versions), but it's not obvious to me why it is needed. Maybe there was an
assert somewhere else that the bitrate for each layer had to be >= the bitrate
of the lower layer, so max of stream_idx-1 was the lowest safest bitrate to
pick.

We should see if it works with just using 0. Internal source encoders don't
otherwise know to stop producing frames (they can't otherwise observe the
send_stream state). Today, we are wasting cycles in simulcast cases encoding
streams that aren't being sent, so there could be an interesting improvement
there as well.

If it is needed, it should move into vp8_impl.cc, because my guess is that was
the only place that ever actually needed it (no other external vp8 codec
implementation I'm aware of ever needed it).

[sprang_webrtc, 2016/08/30 13:27:24]

Gotcha, thanks.

vp8_impl.cc already has it's own internal handling of simulcast though. Looking
at WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoder, is seems like
SimulcastEncoderAdapter will only be created if it is an external encoder being
used. Otherwise  webrtc::VideoEncoder::Create(webrtc::VideoEncoder::kVp8) is
called directly, creating a VP8EncoderImpl without the wrapper.

I think I'll change this and set target bitrate to 0 for unused streams. Can you
help check that I don't break too many things upstream?

[noahric, 2016/08/30 18:14:33]

Yup! By inspection, at least, I don't expect problems, but we'll see what
happens :)

+      stream_bitrate = stream_bitrates[stream_idx - 1];
+    }
     // Need a key frame if we have not sent this stream before.
     if (send_stream && !streaminfos_[stream_idx].send_stream) {
       streaminfos_[stream_idx].key_frame_request = true;
@@ -417,58 +419,10 @@ EncodedImageCallback::Result SimulcastEncoderAdapter::OnEncodedImage(
       encodedImage, &stream_codec_specific, fragmentation);
 }
 
-uint32_t SimulcastEncoderAdapter::GetStreamBitrate(
-    int stream_idx,
-    size_t total_number_of_streams,
-    uint32_t new_bitrate_kbit,
-    bool* send_stream) const {
-  if (total_number_of_streams == 1) {
-    *send_stream = true;
-    return new_bitrate_kbit;
-  }
-
-  // The bitrate needed to start sending this stream is given by the
-  // minimum bitrate allowed for encoding this stream, plus the sum target
-  // rates of all lower streams.
-  uint32_t sum_target_lower_streams =
-      SumStreamTargetBitrate(stream_idx, codec_);
-  uint32_t bitrate_to_send_this_layer =
-      codec_.simulcastStream[stream_idx].minBitrate + sum_target_lower_streams;
-  if (new_bitrate_kbit >= bitrate_to_send_this_layer) {
-    // We have enough bandwidth to send this stream.
-    *send_stream = true;
-    // Bitrate for this stream is the new bitrate (|new_bitrate_kbit|) minus the
-    // sum target rates of the lower streams, and capped to a maximum bitrate.
-    // The maximum cap depends on whether we send the next higher stream.
-    // If we will be sending the next higher stream, |max_rate| is given by
-    // current stream's |targetBitrate|, otherwise it's capped by |maxBitrate|.
-    if (stream_idx < codec_.numberOfSimulcastStreams - 1) {
-      unsigned int max_rate = codec_.simulcastStream[stream_idx].maxBitrate;
-      if (new_bitrate_kbit >=
-          SumStreamTargetBitrate(stream_idx + 1, codec_) +
-              codec_.simulcastStream[stream_idx + 1].minBitrate) {
-        max_rate = codec_.simulcastStream[stream_idx].targetBitrate;
-      }
-      return std::min(new_bitrate_kbit - sum_target_lower_streams, max_rate);
-    } else {
-      // For the highest stream (highest resolution), the |targetBitRate| and
-      // |maxBitrate| are not used. Any excess bitrate (above the targets of
-      // all lower streams) is given to this (highest resolution) stream.
-      return new_bitrate_kbit - sum_target_lower_streams;
-    }
-  } else {
-    // Not enough bitrate for this stream.
-    // Return our max bitrate of |stream_idx| - 1, but we don't send it. We need
-    // to keep this resolution coding in order for the multi-encoder to work.
-    *send_stream = false;
-    return codec_.simulcastStream[stream_idx - 1].maxBitrate;
-  }
-}
-
 void SimulcastEncoderAdapter::PopulateStreamCodec(
     const webrtc::VideoCodec* inst,
     int stream_index,
-    size_t total_number_of_streams,
+    uint32_t start_bitrate_kbps,
     bool highest_resolution_stream,
     webrtc::VideoCodec* stream_codec,
     bool* send_stream) {
@@ -500,9 +454,7 @@ void SimulcastEncoderAdapter::PopulateStreamCodec(
   }
   // TODO(ronghuawu): what to do with targetBitrate.
 
-  int stream_bitrate = GetStreamBitrate(stream_index, total_number_of_streams,
-                                        inst->startBitrate, send_stream);
-  stream_codec->startBitrate = stream_bitrate;
+  stream_codec->startBitrate = start_bitrate_kbps;
 }
 
 bool SimulcastEncoderAdapter::Initialized() const {