Base screenshare layers on TemporalReferences.

webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc

 /* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */
 
 #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
@@ skipping 14 matching lines @@
 static const int kOneSecond90Khz = 90000;
 static const int kMinTimeBetweenSyncs = kOneSecond90Khz * 5;
 static const int kMaxTimeBetweenSyncs = kOneSecond90Khz * 10;
 static const int kQpDeltaThresholdForSync = 8;
 
 const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
 const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
 
 constexpr int ScreenshareLayers::kMaxNumTemporalLayers;
 
-// Since this is TL0 we only allow updating and predicting from the LAST
-// reference frame.
-const int ScreenshareLayers::kTl0Flags =
-    VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF |
-    VP8_EFLAG_NO_REF_ARF;
-
-// Allow predicting from both TL0 and TL1.
-const int ScreenshareLayers::kTl1Flags =
-    VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
-
-// Allow predicting from only TL0 to allow participants to switch to the high
-// bitrate stream. This means predicting only from the LAST reference frame, but
-// only updating GF to not corrupt TL0.
-const int ScreenshareLayers::kTl1SyncFlags =
-    VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF |
-    VP8_EFLAG_NO_UPD_LAST;
-
 // Always emit a frame with certain interval, even if bitrate targets have
 // been exceeded.
 const int ScreenshareLayers::kMaxFrameIntervalMs = 2000;
 
 webrtc::TemporalLayers* ScreenshareTemporalLayersFactory::Create(
     int simulcast_id,
     int num_temporal_layers,
     uint8_t initial_tl0_pic_idx) const {
   webrtc::TemporalLayers* tl;
   if (simulcast_id == 0) {
@@ skipping 31 matching lines @@
 
 ScreenshareLayers::~ScreenshareLayers() {
   UpdateHistograms();
 }
 
 int ScreenshareLayers::CurrentLayerId() const {
   // Codec does not use temporal layers for screenshare.
   return 0;
 }
 
-int ScreenshareLayers::EncodeFlags(uint32_t timestamp) {
+TemporalReferences ScreenshareLayers::UpdateLayerConfig(uint32_t timestamp) {
   if (number_of_temporal_layers_ <= 1) {
     // No flags needed for 1 layer screenshare.
-    return 0;
+    // TODO(pbos): Consider updating only last, and not all buffers.
+    return TemporalReferences(kReferenceAndUpdate, kReferenceAndUpdate,
+                              kReferenceAndUpdate);
   }
 
   const int64_t now_ms = clock_->TimeInMilliseconds();
   if (target_framerate_.value_or(0) > 0 &&
       encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_) {
     // Max framerate exceeded, drop frame.
-    return -1;
+    return TemporalReferences(kNone, kNone, kNone);
   }
 
   if (stats_.first_frame_time_ms_ == -1)
     stats_.first_frame_time_ms_ = now_ms;
 
   int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
-  int flags = 0;
+  enum TemporalLayerState { kDrop, kTl0, kTl1, kTl1Sync };
+  enum TemporalLayerState layer_state = kDrop;
   if (active_layer_ == -1 ||
       layers_[active_layer_].state != TemporalLayer::State::kDropped) {
     if (last_emitted_tl0_timestamp_ != -1 &&
         (unwrapped_timestamp - last_emitted_tl0_timestamp_) / 90 >
             kMaxFrameIntervalMs) {
       // Too long time has passed since the last frame was emitted, cancel
       // enough debt to allow a single frame.
       layers_[0].debt_bytes_ = max_debt_bytes_ - 1;
     }
     if (layers_[0].debt_bytes_ > max_debt_bytes_) {
       // Must drop TL0, encode TL1 instead.
       if (layers_[1].debt_bytes_ > max_debt_bytes_) {
         // Must drop both TL0 and TL1.
         active_layer_ = -1;
       } else {
         active_layer_ = 1;
       }
     } else {
       active_layer_ = 0;
     }
   }
 
   switch (active_layer_) {
     case 0:
-      flags = kTl0Flags;
+      layer_state = kTl0;
       last_emitted_tl0_timestamp_ = unwrapped_timestamp;
       break;
     case 1:
       if (TimeToSync(unwrapped_timestamp)) {
         last_sync_timestamp_ = unwrapped_timestamp;
-        flags = kTl1SyncFlags;
+        layer_state = kTl1Sync;
       } else {
-        flags = kTl1Flags;
+        layer_state = kTl1;
       }
       break;
     case -1:
-      flags = -1;
+      layer_state = kDrop;
       ++stats_.num_dropped_frames_;
       break;
     default:
-      flags = -1;
       RTC_NOTREACHED();
   }
 
   int64_t ts_diff;
   if (last_timestamp_ == -1) {
     ts_diff = kOneSecond90Khz / capture_framerate_.value_or(*target_framerate_);
   } else {
     ts_diff = unwrapped_timestamp - last_timestamp_;
   }
   // Make sure both frame droppers leak out bits.
   layers_[0].UpdateDebt(ts_diff / 90);
   layers_[1].UpdateDebt(ts_diff / 90);
   last_timestamp_ = timestamp;
-  return flags;
+  // TODO(pbos): Consider referencing but not updating the 'alt' buffer for all
+  // layers.
+  switch (layer_state) {
+    case kDrop:
+      return TemporalReferences(kNone, kNone, kNone);
+    case kTl0:
+      // TL0 only references and updates 'last'.
+      return TemporalReferences(kReferenceAndUpdate, kNone, kNone);
+    case kTl1:
+      // TL1 references both 'last' and 'golden' but only updates 'golden'.
+      return TemporalReferences(kReference, kReferenceAndUpdate, kNone);
+    case kTl1Sync:
+      // Predict from only TL0 to allow participants to switch to the high
+      // bitrate stream. Updates 'golden' so that TL1 can continue to refer to
+      // and update 'golden' from this point on.
+      return TemporalReferences(kReference, kUpdate, kNone, kLayerSync);
+  }
+  RTC_NOTREACHED();
+  return TemporalReferences(kNone, kNone, kNone);
 }
 
 std::vector<uint32_t> ScreenshareLayers::OnRatesUpdated(int bitrate_kbps,
                                                         int max_bitrate_kbps,
                                                         int framerate) {
   RTC_DCHECK_GT(framerate, 0);
   if (!target_framerate_) {
     // First OnRatesUpdated() is called during construction, with the configured
     // targets as parameters.
     target_framerate_.emplace(framerate);
@@ skipping 15 matching lines @@
   layers_[0].target_rate_kbps_ = bitrate_kbps;
   layers_[1].target_rate_kbps_ = max_bitrate_kbps;
 
   std::vector<uint32_t> allocation;
   allocation.push_back(bitrate_kbps);
   if (max_bitrate_kbps > bitrate_kbps)
     allocation.push_back(max_bitrate_kbps - bitrate_kbps);
   return allocation;
 }
 
-void ScreenshareLayers::FrameEncoded(unsigned int size,
-                                     uint32_t timestamp,
-                                     int qp) {
+void ScreenshareLayers::FrameEncoded(unsigned int size, int qp) {
   if (size > 0)
     encode_framerate_.Update(1, clock_->TimeInMilliseconds());
 
   if (number_of_temporal_layers_ == 1)
     return;
 
   RTC_DCHECK_NE(-1, active_layer_);
   if (size == 0) {
     layers_[active_layer_].state = TemporalLayer::State::kDropped;
     ++stats_.num_overshoots_;
@@ skipping 14 matching lines @@
     stats_.tl0_target_bitrate_sum_ += layers_[0].target_rate_kbps_;
     stats_.tl0_qp_sum_ += qp;
   } else if (active_layer_ == 1) {
     layers_[1].debt_bytes_ += size;
     ++stats_.num_tl1_frames_;
     stats_.tl1_target_bitrate_sum_ += layers_[1].target_rate_kbps_;
     stats_.tl1_qp_sum_ += qp;
   }
 }
 
-void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync,
+void ScreenshareLayers::PopulateCodecSpecific(bool frame_is_keyframe,
                                               CodecSpecificInfoVP8* vp8_info,
                                               uint32_t timestamp) {
   int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
   if (number_of_temporal_layers_ == 1) {
     vp8_info->temporalIdx = kNoTemporalIdx;
     vp8_info->layerSync = false;
     vp8_info->tl0PicIdx = kNoTl0PicIdx;
   } else {
     RTC_DCHECK_NE(-1, active_layer_);
     vp8_info->temporalIdx = active_layer_;
-    if (base_layer_sync) {
+    if (frame_is_keyframe) {
       vp8_info->temporalIdx = 0;
       last_sync_timestamp_ = unwrapped_timestamp;
     } else if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) {
       // Regardless of pattern the frame after a base layer sync will always
       // be a layer sync.
       last_sync_timestamp_ = unwrapped_timestamp;
     }
     vp8_info->layerSync = last_sync_timestamp_ != -1 &&
                           last_sync_timestamp_ == unwrapped_timestamp;
     if (vp8_info->temporalIdx == 0) {
       tl0_pic_idx_++;
     }
-    last_base_layer_sync_ = base_layer_sync;
+    last_base_layer_sync_ = frame_is_keyframe;
     vp8_info->tl0PicIdx = tl0_pic_idx_;
   }
 }
 
 bool ScreenshareLayers::TimeToSync(int64_t timestamp) const {
   RTC_DCHECK_EQ(1, active_layer_);
   RTC_DCHECK_NE(-1, layers_[0].last_qp);
   if (layers_[1].last_qp == -1) {
     // First frame in TL1 should only depend on TL0 since there are no
     // previous frames in TL1.
@@ skipping 133 matching lines @@
       RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer1.Qp",
                                  stats_.tl1_qp_sum_ / stats_.num_tl1_frames_);
       RTC_HISTOGRAM_COUNTS_10000(
           "WebRTC.Video.Screenshare.Layer1.TargetBitrate",
           stats_.tl1_target_bitrate_sum_ / stats_.num_tl1_frames_);
     }
   }
 }
 
 }  // namespace webrtc