In screenshare mode, suppress VP8 bitrate overshoot and increase quality

webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc

 /* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */
 
 #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
 
 #include <stdlib.h>
 
+#include "webrtc/base/checks.h"
 #include "vpx/vpx_encoder.h"
 #include "vpx/vp8cx.h"
 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
 
 namespace webrtc {
 
-enum { kOneSecond90Khz = 90000 };
+static const int kOneSecond90Khz = 90000;
+static const int kMinTimeBetweenSyncs = kOneSecond90Khz * 5;
+static const int kMaxTimeBetweenSyncs = kOneSecond90Khz * 10;
+static const int kMaxQpDeltaForSync = 8;
 
 const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
 const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
 
 ScreenshareLayers::ScreenshareLayers(int num_temporal_layers,
-                                     uint8_t initial_tl0_pic_idx,
-                                     FrameDropper* tl0_frame_dropper,
-                                     FrameDropper* tl1_frame_dropper)
-    : tl0_frame_dropper_(tl0_frame_dropper),
-      tl1_frame_dropper_(tl1_frame_dropper),
-      number_of_temporal_layers_(num_temporal_layers),
+                                     uint8_t initial_tl0_pic_idx)
+    : number_of_temporal_layers_(num_temporal_layers),
       last_base_layer_sync_(false),
       tl0_pic_idx_(initial_tl0_pic_idx),
       active_layer_(0),
-      framerate_(5),
-      last_sync_timestamp_(-1) {
+      last_timestamp_(0),
+      last_sync_timestamp_(0),
+      min_qp_(-1),
+      max_qp_(-1),
+      max_debt_bytes_(0) {
   assert(num_temporal_layers > 0);
   assert(num_temporal_layers <= 2);
-  assert(tl0_frame_dropper && tl1_frame_dropper);
 }
 
 int ScreenshareLayers::CurrentLayerId() const {
   // Codec does not use temporal layers for screenshare.
   return 0;
 }
 
 int ScreenshareLayers::EncodeFlags(uint32_t timestamp) {
   if (number_of_temporal_layers_ <= 1) {
     // No flags needed for 1 layer screenshare.
     return 0;
   }
-  CalculateFramerate(timestamp);
+
   int flags = 0;
-  // Note that ARF on purpose isn't used in this scheme since it is allocated
-  // for the last key frame to make key frame caching possible.
-  if (tl0_frame_dropper_->DropFrame()) {
-    // Must drop TL0, encode TL1 instead.
-    if (tl1_frame_dropper_->DropFrame()) {
-      // Must drop both TL0 and TL1.
-      flags = -1;
+  int temporal_layer;
+
+  if (active_layer_ != -1 &&
+      layers_[active_layer_].state == TemporalLayer::State::kDropped) {
+    temporal_layer = active_layer_;
+  } else {
+    if (layers_[0].debt_bytes_ > max_debt_bytes_) {
+      // Must drop TL0, encode TL1 instead.
+      if (layers_[1].debt_bytes_ > max_debt_bytes_) {
+        // Must drop both TL0 and TL1.
+        temporal_layer = -1;
+      } else {
+        temporal_layer = 1;
+      }
     } else {
-      active_layer_ = 1;
+      temporal_layer = 0;
+    }
+  }
+
+  switch (temporal_layer) {
+    case 0:
+      // Since this is TL0 we only allow updating and predicting from the LAST
+      // reference frame.
+      flags = VP8_EFLAG_NO_UPD_GF;
+      flags |= VP8_EFLAG_NO_UPD_ARF;
+      flags |= VP8_EFLAG_NO_REF_GF;
+      flags |= VP8_EFLAG_NO_REF_ARF;
+      break;
+    case 1:
       if (TimeToSync(timestamp)) {
         last_sync_timestamp_ = timestamp;
         // Allow predicting from only TL0 to allow participants to switch to the
         // high bitrate stream. This means predicting only from the LAST
         // reference frame, but only updating GF to not corrupt TL0.
         flags = VP8_EFLAG_NO_REF_ARF;
         flags |= VP8_EFLAG_NO_REF_GF;
         flags |= VP8_EFLAG_NO_UPD_ARF;
         flags |= VP8_EFLAG_NO_UPD_LAST;
       } else {
         // Allow predicting from both TL0 and TL1.
         flags = VP8_EFLAG_NO_REF_ARF;
         flags |= VP8_EFLAG_NO_UPD_ARF;
         flags |= VP8_EFLAG_NO_UPD_LAST;
       }
-    }
-  } else {
-    active_layer_ = 0;
-    // Since this is TL0 we only allow updating and predicting from the LAST
-    // reference frame.
-    flags = VP8_EFLAG_NO_UPD_GF;
-    flags |= VP8_EFLAG_NO_UPD_ARF;
-    flags |= VP8_EFLAG_NO_REF_GF;
-    flags |= VP8_EFLAG_NO_REF_ARF;
+      break;
+    case -1:
+      flags = -1;
+      break;
+    default:
+      flags = -1;
+      RTC_NOTREACHED();
   }
+
+  active_layer_ = temporal_layer;
+
   // Make sure both frame droppers leak out bits.
-  tl0_frame_dropper_->Leak(framerate_);
-  tl1_frame_dropper_->Leak(framerate_);
+  if (last_timestamp_ != 0) {
+    int64_t ts_diff = timestamp - static_cast<int64_t>(last_timestamp_);
+    if (ts_diff < 0)
+      ts_diff += (1l << 32);
+
+    layers_[0].UpdateDebt(ts_diff / 90);
+    layers_[1].UpdateDebt(ts_diff / 90);
+  }
+  last_timestamp_ = timestamp;
   return flags;
 }
 
 bool ScreenshareLayers::ConfigureBitrates(int bitrate_kbit,
                                           int max_bitrate_kbit,

[stefan-webrtc, 2015/06/18 09:39:02]

Should these say kbps instead?

[sprang_webrtc, 2015/06/18 13:05:02]

Done.

                                           int framerate,
                                           vpx_codec_enc_cfg_t* cfg) {
-  if (framerate > 0)
-    framerate_ = framerate;
+  int avg_frame_size = (bitrate_kbit * 1000) / (8 * framerate);
+  max_debt_bytes_ = 5 * avg_frame_size;
 
-  tl0_frame_dropper_->SetRates(bitrate_kbit, framerate_);
-  tl1_frame_dropper_->SetRates(max_bitrate_kbit, framerate_);
+  layers_[0].target_rate_kbps_ = bitrate_kbit;
+  layers_[1].target_rate_kbps_ = max_bitrate_kbit;
 
   if (cfg != nullptr) {
     // Calculate a codec target bitrate. This may be higher than TL0, gaining
     // quality at the expense of frame rate at TL0. Constraints:
     // - TL0 frame rate should not be less than framerate / kMaxTL0FpsReduction.
     // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate.
     double target_bitrate =
         std::min(bitrate_kbit * kMaxTL0FpsReduction,
                  max_bitrate_kbit / kAcceptableTargetOvershoot);
+
+    if (layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) {
+      min_qp_ = cfg->rc_min_quantizer;
+      max_qp_ = cfg->rc_max_quantizer;
+      layers_[0].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 80) / 100);
+      layers_[1].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 85) / 100);

[stefan-webrtc, 2015/06/18 09:39:02]

80% more in the lower layer and 85% more in the higher layer? Comment

[sprang_webrtc, 2015/06/18 13:05:02]

Done.

+    }
+
     cfg->rc_target_bitrate =
         std::max(static_cast<unsigned int>(bitrate_kbit),
                  static_cast<unsigned int>(target_bitrate + 0.5));
   }
 
   return true;
 }
 
-void ScreenshareLayers::FrameEncoded(unsigned int size, uint32_t timestamp) {
-  if (active_layer_ == 0) {
-    tl0_frame_dropper_->Fill(size, true);
+void ScreenshareLayers::FrameEncoded(unsigned int size,
+                                     uint32_t timestamp,
+                                     int qp) {
+  if (size == 0) {
+    layers_[active_layer_].state = TemporalLayer::State::kDropped;
+    return;
   }
-  tl1_frame_dropper_->Fill(size, true);
+  if (layers_[active_layer_].state == TemporalLayer::State::kDropped) {
+    layers_[active_layer_].state = TemporalLayer::State::kQualityBoost;

[stefan-webrtc, 2015/06/18 09:39:02]

This means we always boost quality after a drop? Doesn't that defeat the purpose
of the drop somehow?

[sprang_webrtc, 2015/06/18 13:05:02]

The rate control will be reset to start at max qp, so the quality will start out
_very_ low, and then ramp up, after a drop. 

+  }
+
+  if (qp != -1)
+    layers_[active_layer_].last_qp = qp;
+  if (active_layer_ == 0)
+    layers_[0].debt_bytes_ += size;
+  layers_[1].debt_bytes_ += size;
 }
 
 void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync,
                                               CodecSpecificInfoVP8 *vp8_info,
                                               uint32_t timestamp) {
   if (number_of_temporal_layers_ == 1) {
     vp8_info->temporalIdx = kNoTemporalIdx;
     vp8_info->layerSync = false;
     vp8_info->tl0PicIdx = kNoTl0PicIdx;
   } else {
@@ skipping 10 matching lines @@
     if (vp8_info->temporalIdx == 0) {
       tl0_pic_idx_++;
     }
     last_base_layer_sync_ = base_layer_sync;
     vp8_info->tl0PicIdx = tl0_pic_idx_;
   }
 }
 
 bool ScreenshareLayers::TimeToSync(uint32_t timestamp) const {
   const uint32_t timestamp_diff = timestamp - last_sync_timestamp_;
-  return last_sync_timestamp_ < 0 || timestamp_diff > kOneSecond90Khz;
+  if (last_sync_timestamp_ == 0 || timestamp_diff > kMaxTimeBetweenSyncs)
+    return true;
+  if (layers_[0].last_qp == -1 || layers_[1].last_qp == -1)

[stefan-webrtc, 2015/06/18 09:39:02]

This means that the first frame should be a sync frame? Might want to comment on
what you want to achieve.

[sprang_webrtc, 2015/06/18 13:05:02]

Done.

+    return true;
+  if (timestamp_diff < kMinTimeBetweenSyncs)
+    return false;
+  if (layers_[1].last_qp - layers_[0].last_qp < kMaxQpDeltaForSync)
+    return true;
+  return false;
 }
 
-void ScreenshareLayers::CalculateFramerate(uint32_t timestamp) {
-  timestamp_list_.push_front(timestamp);
-  // Remove timestamps older than 1 second from the list.
-  uint32_t timestamp_diff = timestamp - timestamp_list_.back();
-  while (timestamp_diff > kOneSecond90Khz) {
-    timestamp_list_.pop_back();
-    timestamp_diff = timestamp - timestamp_list_.back();
+bool ScreenshareLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) {
+  if (max_qp_ == -1)
+    return false;
+
+  unsigned int adjusted_max_qp;
+  if (layers_[active_layer_].state == TemporalLayer::State::kQualityBoost &&

[stefan-webrtc, 2015/06/18 09:39:02]

Comment to describe what you are doing with this if-else.

[sprang_webrtc, 2015/06/18 13:05:02]

Done.

+      layers_[active_layer_].enhanced_max_qp != -1) {
+    adjusted_max_qp = layers_[active_layer_].enhanced_max_qp;
+    layers_[active_layer_].state = TemporalLayer::State::kNormal;
+  } else {
+    if (max_qp_ == -1)
+      return false;
+    adjusted_max_qp = max_qp_;
   }
-  // If we have encoded frames within the last second, that number of frames
-  // is a reasonable first estimate of the framerate.
-  framerate_ = timestamp_list_.size();
-  if (timestamp_diff > 0) {
-    // Estimate the framerate by dividing the number of timestamp diffs with
-    // the sum of the timestamp diffs (with rounding).
-    framerate_ = (kOneSecond90Khz * (timestamp_list_.size() - 1) +
-        timestamp_diff / 2) / timestamp_diff;
+
+  if (adjusted_max_qp == cfg->rc_max_quantizer)
+    return false;
+
+  cfg->rc_max_quantizer = adjusted_max_qp;
+  return true;
+}
+
+void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) {
+  uint32_t debt_reduction_bytes = target_rate_kbps_ * delta_ms / 8;
+  if (debt_reduction_bytes >= debt_bytes_) {
+    debt_bytes_ = 0;
+  } else {
+    debt_bytes_ -= debt_reduction_bytes;
   }
 }
 
 }  // namespace webrtc