Add frame rate throttling to vp8 screenshare_layers.

webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc

 /* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */
 
 #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
@@ skipping 58 matching lines @@
       number_of_temporal_layers_(num_temporal_layers),
       last_base_layer_sync_(false),
       tl0_pic_idx_(initial_tl0_pic_idx),
       active_layer_(-1),
       last_timestamp_(-1),
       last_sync_timestamp_(-1),
       last_emitted_tl0_timestamp_(-1),
       min_qp_(-1),
       max_qp_(-1),
       max_debt_bytes_(0),
-      framerate_(-1),
+      encode_framerate_(1000.0f, 1000.0f),  // 1 second window, second scale.
       bitrate_updated_(false) {
   RTC_CHECK_GT(num_temporal_layers, 0);
   RTC_CHECK_LE(num_temporal_layers, 2);
 }
 
 ScreenshareLayers::~ScreenshareLayers() {
   UpdateHistograms();
 }
 
 int ScreenshareLayers::CurrentLayerId() const {
   // Codec does not use temporal layers for screenshare.
   return 0;
 }
 
 int ScreenshareLayers::EncodeFlags(uint32_t timestamp) {
   if (number_of_temporal_layers_ <= 1) {
     // No flags needed for 1 layer screenshare.
     return 0;
   }
 
+  const int64_t now_ms = clock_->TimeInMilliseconds();
+  if (target_framerate_.value_or(0) > 0 &&
+      encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_) {
+    // Max framerate exceeded, drop frame.
+    return -1;
+  }
+
   if (stats_.first_frame_time_ms_ == -1)
-    stats_.first_frame_time_ms_ = clock_->TimeInMilliseconds();
+    stats_.first_frame_time_ms_ = now_ms;
 
   int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
   int flags = 0;
   if (active_layer_ == -1 ||
       layers_[active_layer_].state != TemporalLayer::State::kDropped) {
     if (last_emitted_tl0_timestamp_ != -1 &&
         (unwrapped_timestamp - last_emitted_tl0_timestamp_) / 90 >
             kMaxFrameIntervalMs) {
       // Too long time has passed since the last frame was emitted, cancel
       // enough debt to allow a single frame.
@@ skipping 29 matching lines @@
       flags = -1;
       ++stats_.num_dropped_frames_;
       break;
     default:
       flags = -1;
       RTC_NOTREACHED();
   }
 
   int64_t ts_diff;
   if (last_timestamp_ == -1) {
-    ts_diff = kOneSecond90Khz / (framerate_ <= 0 ? 5 : framerate_);
+    ts_diff =
+        kOneSecond90Khz / incoming_framerate_.value_or(*target_framerate_);

[stefan-webrtc, 2016/12/19 10:58:29]

Looks like incoming_framerate_ can be zero. I'm probably missing something, but
maybe good to make it explicit with a dcheck?

[sprang_webrtc, 2016/12/19 16:46:34]

This "shouldn't happen" since we treat 0 fps as invalid in
VideoSender::UpdateEncoderParameters(). But I'll add a protection just in case
this breaks in the future.

[stefan-webrtc, 2016/12/22 09:39:34]

Yes, sounds like the check is pretty far away from here. But if we assume it
should never happen, maybe it's enough to dcheck on it? Otherwise it looks like
it actually can happen.

[sprang_webrtc, 2016/12/28 13:00:35]

Sure, added dcheck to OnRatesUpdated.

   } else {
     ts_diff = unwrapped_timestamp - last_timestamp_;
   }
   // Make sure both frame droppers leak out bits.
   layers_[0].UpdateDebt(ts_diff / 90);
   layers_[1].UpdateDebt(ts_diff / 90);
   last_timestamp_ = timestamp;
   return flags;
 }
 
 std::vector<uint32_t> ScreenshareLayers::OnRatesUpdated(int bitrate_kbps,
                                                         int max_bitrate_kbps,
                                                         int framerate) {
-  bitrate_updated_ =
-      bitrate_kbps != static_cast<int>(layers_[0].target_rate_kbps_) ||
-      max_bitrate_kbps != static_cast<int>(layers_[1].target_rate_kbps_) ||
-      framerate != framerate_;
+  if (!target_framerate_) {
+    // First OnRatesUpdated() is called during construction, with the configured
+    // targets as parameters.
+    target_framerate_.emplace(framerate);
+    incoming_framerate_ = target_framerate_;
+    bitrate_updated_ = true;
+  } else {
+    bitrate_updated_ =
+        bitrate_kbps != static_cast<int>(layers_[0].target_rate_kbps_) ||
+        max_bitrate_kbps != static_cast<int>(layers_[1].target_rate_kbps_) ||
+        (incoming_framerate_ &&
+         framerate != static_cast<int>(*incoming_framerate_));
+    if (framerate < 0) {
+      incoming_framerate_.reset();
+    } else {
+      incoming_framerate_.emplace(framerate);
+    }
+  }
+
   layers_[0].target_rate_kbps_ = bitrate_kbps;
   layers_[1].target_rate_kbps_ = max_bitrate_kbps;
-  framerate_ = framerate;
 
   std::vector<uint32_t> allocation;
   allocation.push_back(bitrate_kbps);
   if (max_bitrate_kbps > bitrate_kbps)
     allocation.push_back(max_bitrate_kbps - bitrate_kbps);
   return allocation;
 }
 
 void ScreenshareLayers::FrameEncoded(unsigned int size,
                                      uint32_t timestamp,
                                      int qp) {
+  if (size > 0)
+    encode_framerate_.Update(1, clock_->TimeInMilliseconds());
+
   if (number_of_temporal_layers_ == 1)
     return;
 
   RTC_DCHECK_NE(-1, active_layer_);
   if (size == 0) {
     layers_[active_layer_].state = TemporalLayer::State::kDropped;
     ++stats_.num_overshoots_;
     return;
   }
 
@@ skipping 101 matching lines @@
       max_qp_ = cfg->rc_max_quantizer;
       // After a dropped frame, a frame with max qp will be encoded and the
       // quality will then ramp up from there. To boost the speed of recovery,
       // encode the next frame with lower max qp. TL0 is the most important to
       // improve since the errors in this layer will propagate to TL1.
       // Currently, reduce max qp by 20% for TL0 and 15% for TL1.
       layers_[0].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 80) / 100);
       layers_[1].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 85) / 100);
     }
 
-    if (framerate_ > 0) {
-      int avg_frame_size = (target_bitrate_kbps * 1000) / (8 * framerate_);
+    if (incoming_framerate_) {
+      int avg_frame_size =
+          (target_bitrate_kbps * 1000) / (8 * *incoming_framerate_);
       max_debt_bytes_ = 4 * avg_frame_size;
     }
 
     bitrate_updated_ = false;
     cfg_updated = true;
   }
 
   // Don't try to update boosts state if not active yet.
   if (active_layer_ == -1)
     return cfg_updated;
@@ skipping 65 matching lines @@
       RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer1.Qp",
                                  stats_.tl1_qp_sum_ / stats_.num_tl1_frames_);
       RTC_HISTOGRAM_COUNTS_10000(
           "WebRTC.Video.Screenshare.Layer1.TargetBitrate",
           stats_.tl1_target_bitrate_sum_ / stats_.num_tl1_frames_);
     }
   }
 }
 
 }  // namespace webrtc