Implement kBalanced degradation preference.

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 55 matching lines @@
   if (kbps > 0) {
     if (kbps < 300 /* qvga */) {
       return 320 * 240;
     } else if (kbps < 500 /* vga */) {
       return 640 * 480;
     }
   }
   return std::numeric_limits<uint32_t>::max();
 }
 
+// Limits for kBalanced degradation preference.
+//  >640x480:  |                   max                       |

[sprang_webrtc, 2017/06/13 11:58:29]

I think there could be a case for 24fps for 720p as well? But maybe in followup
in that case, this is getting complex.

[åsapersson, 2017/06/13 12:40:04]

Agree, these are initial limits.

+//  <=640x480: |         15fps                   |    max    |
+//  <=480x270: |     10fps     |        15fps                |
+//  <=320x240: |  7fps |             10fps                   |
+//  ----------------------------------------------------------
+//       kbps: 0       60     130               500          |
+
+int MinFps(int pixels) {
+  if (pixels <= 320 * 240)
+    return 7;
+  else if (pixels <= 480 * 270)
+    return 10;
+  else if (pixels <= 640 * 480)
+    return 15;
+  else
+    return std::numeric_limits<int>::max();

[sprang_webrtc, 2017/06/13 11:58:29]

nit: please use brackets for if/else statements

[åsapersson, 2017/06/13 12:40:05]

Done.

+}
+
+int MaxFps(int pixels) {
+  if (pixels <= 320 * 240)
+    return 10;
+  else if (pixels <= 480 * 270)
+    return 15;
+  else
+    return std::numeric_limits<int>::max();
+}
+
+uint32_t BitrateLimit(int pixels) {
+  if (pixels <= 320 * 240)
+    return 60000;
+  else if (pixels <= 480 * 270)
+    return 130000;
+  else if (pixels <= 640 * 480)
+    return 500000;
+  else
+    return std::numeric_limits<uint32_t>::max();
+}
+
+int MaxFpsForBitrateAndResolution(uint32_t bps, int pixels) {
+  return (bps <= BitrateLimit(pixels)) ? MinFps(pixels) : MaxFps(pixels);
+}
+
 bool IsResolutionScalingEnabled(
     VideoSendStream::DegradationPreference degradation_preference) {
   return degradation_preference ==
              VideoSendStream::DegradationPreference::kMaintainFramerate ||
          degradation_preference ==
              VideoSendStream::DegradationPreference::kBalanced;
 }
 
 bool IsFramerateScalingEnabled(
     VideoSendStream::DegradationPreference degradation_preference) {
@@ skipping 118 matching lines @@
     if (source_)
       source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
   }
 
   rtc::VideoSinkWants GetActiveSinkWants() EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
     rtc::VideoSinkWants wants = sink_wants_;
     // Clear any constraints from the current sink wants that don't apply to
     // the used degradation_preference.
     switch (degradation_preference_) {
       case VideoSendStream::DegradationPreference::kBalanced:
+        break;
       case VideoSendStream::DegradationPreference::kMaintainFramerate:
         wants.max_framerate_fps = std::numeric_limits<int>::max();
         break;
       case VideoSendStream::DegradationPreference::kMaintainResolution:
         wants.max_pixel_count = std::numeric_limits<int>::max();
         wants.target_pixel_count.reset();
         break;
       case VideoSendStream::DegradationPreference::kDegradationDisabled:
         wants.max_pixel_count = std::numeric_limits<int>::max();
         wants.target_pixel_count.reset();
@@ skipping 575 matching lines @@
       stats_proxy_->GetStats().input_frame_rate,
       AdaptationRequest::Mode::kAdaptDown};
 
   bool downgrade_requested =
       last_adaptation_request_ &&
       last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
 
   int max_downgrades = 0;
   switch (degradation_preference_) {
     case VideoSendStream::DegradationPreference::kBalanced:
+      max_downgrades = kMaxCpuResolutionDowngrades;
+      break;
     case VideoSendStream::DegradationPreference::kMaintainFramerate:
       max_downgrades = kMaxCpuResolutionDowngrades;
       if (downgrade_requested &&
           adaptation_request.input_pixel_count_ >=
               last_adaptation_request_->input_pixel_count_) {
         // Don't request lower resolution if the current resolution is not
         // lower than the last time we asked for the resolution to be lowered.
         return;
       }
       break;
@@ skipping 14 matching lines @@
     case VideoSendStream::DegradationPreference::kDegradationDisabled:
       return;
   }
 
   if (reason == kCpu) {
     if (GetConstAdaptCounter().TotalCount(kCpu) >= max_downgrades)
       return;
   }
 
   switch (degradation_preference_) {
-    case VideoSendStream::DegradationPreference::kBalanced:
+    case VideoSendStream::DegradationPreference::kBalanced: {
+      // Try scale down framerate, if lower.
+      int max_fps = MaxFpsForBitrateAndResolution(
+          last_observed_bitrate_bps_, last_frame_info_->pixel_count());
+      if (source_proxy_->RestrictFramerate(max_fps)) {
+        GetAdaptCounter().IncrementFramerate(reason);
+        break;
+      }
+      // Scale down resolution.
+      FALLTHROUGH();
+    }
     case VideoSendStream::DegradationPreference::kMaintainFramerate:
       // Scale down resolution.
       if (!source_proxy_->RequestResolutionLowerThan(
               adaptation_request.input_pixel_count_)) {
         return;
       }
-      GetAdaptCounter().IncrementResolution(reason, 1);
+      GetAdaptCounter().IncrementResolution(reason);
       break;
     case VideoSendStream::DegradationPreference::kMaintainResolution:
       // Scale down framerate.
       if (!source_proxy_->RequestFramerateLowerThan(
               adaptation_request.framerate_fps_)) {
         return;
       }
-      GetAdaptCounter().IncrementFramerate(reason, 1);
+      GetAdaptCounter().IncrementFramerate(reason);
       break;
     case VideoSendStream::DegradationPreference::kDegradationDisabled:
       RTC_NOTREACHED();
   }
 
   last_adaptation_request_.emplace(adaptation_request);
 
   UpdateAdaptationStats(reason);
 
   LOG(LS_INFO) << GetConstAdaptCounter().ToString();
@@ skipping 10 matching lines @@
 
   AdaptationRequest adaptation_request = {
       last_frame_info_->pixel_count(),
       stats_proxy_->GetStats().input_frame_rate,
       AdaptationRequest::Mode::kAdaptUp};
 
   bool adapt_up_requested =
       last_adaptation_request_ &&
       last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
 
-  switch (degradation_preference_) {
-    case VideoSendStream::DegradationPreference::kBalanced:
-    case VideoSendStream::DegradationPreference::kMaintainFramerate:
-      if (adapt_up_requested &&
-          adaptation_request.input_pixel_count_ <=
-              last_adaptation_request_->input_pixel_count_) {
-        // Don't request higher resolution if the current resolution is not
-        // higher than the last time we asked for the resolution to be higher.
-        return;
-      }
-      break;
-    case VideoSendStream::DegradationPreference::kMaintainResolution:
-      // TODO(sprang): Don't request higher framerate if we are already at
-      // max requested fps?
-      break;
-    case VideoSendStream::DegradationPreference::kDegradationDisabled:
+  if (degradation_preference_ ==
+      VideoSendStream::DegradationPreference::kMaintainFramerate) {
+    if (adapt_up_requested &&
+        adaptation_request.input_pixel_count_ <=
+            last_adaptation_request_->input_pixel_count_) {
+      // Don't request higher resolution if the current resolution is not
+      // higher than the last time we asked for the resolution to be higher.
       return;
+    }
   }
 
   switch (degradation_preference_) {
-    case VideoSendStream::DegradationPreference::kBalanced:
+    case VideoSendStream::DegradationPreference::kBalanced: {
+      // Try scale up framerate, if higher.
+      int fps = MaxFps(last_frame_info_->pixel_count());
+      if (source_proxy_->IncreaseFramerate(fps)) {
+        GetAdaptCounter().DecrementFramerate(reason);
+        break;
+      }
+      // Scale up resolution.
+      FALLTHROUGH();
+    }
     case VideoSendStream::DegradationPreference::kMaintainFramerate: {
       // Scale up resolution.
       int pixel_count = adaptation_request.input_pixel_count_;
       if (adapt_counter.ResolutionCount() == 1) {
         LOG(LS_INFO) << "Removing resolution down-scaling setting.";
         pixel_count = std::numeric_limits<int>::max();
       }
       if (!source_proxy_->RequestHigherResolutionThan(pixel_count))
         return;
-      GetAdaptCounter().IncrementResolution(reason, -1);
+      GetAdaptCounter().DecrementResolution(reason);
       break;
     }
     case VideoSendStream::DegradationPreference::kMaintainResolution: {
       // Scale up framerate.
       int fps = adaptation_request.framerate_fps_;
       if (adapt_counter.FramerateCount() == 1) {
         LOG(LS_INFO) << "Removing framerate down-scaling setting.";
         fps = std::numeric_limits<int>::max();
       }
       if (!source_proxy_->RequestHigherFramerateThan(fps))
         return;
-      GetAdaptCounter().IncrementFramerate(reason, -1);
+      GetAdaptCounter().DecrementFramerate(reason);
       break;
     }
     case VideoSendStream::DegradationPreference::kDegradationDisabled:
-      RTC_NOTREACHED();
+      return;
   }
 
   last_adaptation_request_.emplace(adaptation_request);
 
   UpdateAdaptationStats(reason);
 
   LOG(LS_INFO) << adapt_counter.ToString();
 }
 
 void ViEEncoder::UpdateAdaptationStats(AdaptReason reason) {
@@ skipping 37 matching lines @@
 }
 
 const ViEEncoder::AdaptCounter& ViEEncoder::GetConstAdaptCounter() {
   return adapt_counters_[degradation_preference_];
 }
 
 // Class holding adaptation information.
 ViEEncoder::AdaptCounter::AdaptCounter() {
   fps_counters_.resize(kScaleReasonSize);
   resolution_counters_.resize(kScaleReasonSize);
+  static_assert(kScaleReasonSize == 2, "Update MoveCount.");
 }
 
 ViEEncoder::AdaptCounter::~AdaptCounter() {}
 
 std::string ViEEncoder::AdaptCounter::ToString() const {
   std::stringstream ss;
   ss << "Downgrade counts: fps: {" << ToString(fps_counters_);
   ss << "}, resolution: {" << ToString(resolution_counters_) << "}";
   return ss.str();
 }
 
 ViEEncoder::AdaptCounts ViEEncoder::AdaptCounter::Counts(int reason) const {
   AdaptCounts counts;
   counts.fps = fps_counters_[reason];
   counts.resolution = resolution_counters_[reason];
   return counts;
 }
 
-void ViEEncoder::AdaptCounter::IncrementFramerate(int reason, int delta) {
-  fps_counters_[reason] += delta;
+void ViEEncoder::AdaptCounter::IncrementFramerate(int reason) {
+  ++(fps_counters_[reason]);
 }
 
-void ViEEncoder::AdaptCounter::IncrementResolution(int reason, int delta) {
-  resolution_counters_[reason] += delta;
+void ViEEncoder::AdaptCounter::IncrementResolution(int reason) {
+  ++(resolution_counters_[reason]);
+}
+
+void ViEEncoder::AdaptCounter::DecrementFramerate(int reason) {
+  if (fps_counters_[reason] == 0) {
+    // Balanced mode: Adapt up is in a different order, switch reason.
+    // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3).
+    // 1. Down resolution (cpu):   res={quality:0,cpu:1}, fps={quality:0,cpu:0}
+    // 2. Down fps (quality):      res={quality:0,cpu:1}, fps={quality:1,cpu:0}
+    // 3. Up fps (cpu):            res={quality:1,cpu:0}, fps={quality:0,cpu:0}
+    // 4. Up resolution (quality): res={quality:0,cpu:0}, fps={quality:0,cpu:0}
+    RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
+    RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded.";
+    MoveCount(&resolution_counters_, reason);
+    MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize);
+  }
+  --(fps_counters_[reason]);
+  RTC_DCHECK_GE(fps_counters_[reason], 0);
+}
+
+void ViEEncoder::AdaptCounter::DecrementResolution(int reason) {
+  if (resolution_counters_[reason] == 0) {
+    // Balanced mode: Adapt up is in a different order, switch reason.
+    RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
+    RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded.";
+    MoveCount(&fps_counters_, reason);
+    MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize);
+  }
+  --(resolution_counters_[reason]);
+  RTC_DCHECK_GE(resolution_counters_[reason], 0);
 }
 
 int ViEEncoder::AdaptCounter::FramerateCount() const {
   return Count(fps_counters_);
 }
 
 int ViEEncoder::AdaptCounter::ResolutionCount() const {
   return Count(resolution_counters_);
 }
 
-int ViEEncoder::AdaptCounter::TotalCount() const {
-  return FramerateCount() + ResolutionCount();
-}
-
 int ViEEncoder::AdaptCounter::FramerateCount(int reason) const {
   return fps_counters_[reason];
 }
 
 int ViEEncoder::AdaptCounter::ResolutionCount(int reason) const {
   return resolution_counters_[reason];
 }
 
 int ViEEncoder::AdaptCounter::TotalCount(int reason) const {
   return FramerateCount(reason) + ResolutionCount(reason);
 }
 
 int ViEEncoder::AdaptCounter::Count(const std::vector<int>& counters) const {
   return std::accumulate(counters.begin(), counters.end(), 0);
 }
 
+void ViEEncoder::AdaptCounter::MoveCount(std::vector<int>* counters,
+                                         int from_reason) {
+  int to_reason = (from_reason + 1) % kScaleReasonSize;
+  ++((*counters)[to_reason]);
+  --((*counters)[from_reason]);
+}
+
 std::string ViEEncoder::AdaptCounter::ToString(
     const std::vector<int>& counters) const {
   std::stringstream ss;
   for (size_t reason = 0; reason < kScaleReasonSize; ++reason) {
     ss << (reason ? " cpu" : "quality") << ":" << counters[reason];
   }
   return ss.str();
 }
 
 }  // namespace webrtc