Implement kBalanced degradation preference.

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 55 matching lines @@
   if (kbps > 0) {
     if (kbps < 300 /* qvga */) {
       return 320 * 240;
     } else if (kbps < 500 /* vga */) {
       return 640 * 480;
     }
   }
   return std::numeric_limits<uint32_t>::max();
 }
 
+// Limits for kBalanced degradation preference.
+//  >640x480:  |                   max                       |
+//  <=640x480: |         15fps                   |    max    |
+//  <=480x270: |     10fps     |        15fps                |
+//  <=320x240: |  7fps |             10fps                   |
+//  ----------------------------------------------------------
+//       kbps: 0       60     130               500          |

[kthelgason, 2017/05/19 08:34:14]

Just out of curiosity, how are these determined?

[åsapersson, 2017/06/08 13:55:32]

The bitrate limits were set roughly to a value when bits per pixel were below
some threshold (around 0.07). 

+
+int MinFps(int pixels) {
+  if (pixels <= 320 * 240)
+    return 7;
+  else if (pixels <= 480 * 270)
+    return 10;
+  else if (pixels <= 640 * 480)
+    return 15;
+  else
+    return std::numeric_limits<int>::max();
+}
+
+int MaxFps(int pixels) {
+  if (pixels <= 320 * 240)
+    return 10;
+  else if (pixels <= 480 * 270)
+    return 15;
+  else
+    return std::numeric_limits<int>::max();
+}
+
+uint32_t BitrateLimit(int pixels) {
+  if (pixels <= 320 * 240)
+    return 60000;
+  else if (pixels <= 480 * 270)
+    return 130000;
+  else if (pixels <= 640 * 480)
+    return 500000;
+  else
+    return std::numeric_limits<uint32_t>::max();
+}
+
+int MaxFpsForBitrateAndResolution(uint32_t bps, int pixels) {
+  return (bps <= BitrateLimit(pixels)) ? MinFps(pixels) : MaxFps(pixels);
+}
+
 bool IsResolutionScalingEnabled(
     VideoSendStream::DegradationPreference degradation_preference) {
   return degradation_preference ==
              VideoSendStream::DegradationPreference::kMaintainFramerate ||
          degradation_preference ==
              VideoSendStream::DegradationPreference::kBalanced;
 }
 
 bool IsFramerateScalingEnabled(
     VideoSendStream::DegradationPreference degradation_preference) {
@@ skipping 118 matching lines @@
     if (source_)
       source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
   }
 
   rtc::VideoSinkWants GetActiveSinkWants() EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
     rtc::VideoSinkWants wants = sink_wants_;
     // Clear any constraints from the current sink wants that don't apply to
     // the used degradation_preference.
     switch (degradation_preference_) {
       case VideoSendStream::DegradationPreference::kBalanced:
-        FALLTHROUGH();
+        break;
       case VideoSendStream::DegradationPreference::kMaintainFramerate:
         wants.max_framerate_fps = std::numeric_limits<int>::max();
         break;
       case VideoSendStream::DegradationPreference::kMaintainResolution:
         wants.max_pixel_count = std::numeric_limits<int>::max();
         wants.target_pixel_count.reset();
         break;
       case VideoSendStream::DegradationPreference::kDegradationDisabled:
         wants.max_pixel_count = std::numeric_limits<int>::max();
         wants.target_pixel_count.reset();
@@ skipping 82 matching lines @@
     } else {
       // The input video frame rate will be scaled up to the last step, with
       // rounding.
       const int framerate_wanted = (framerate_fps * 3) / 2;
       sink_wants_.max_framerate_fps = framerate_wanted;
     }
     if (source_)
       source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants());
   }
 
+  bool RestrictFpsOrRequestResolutionLowerThan(int max_fps,
+                                               int pixel_count,
+                                               bool* fps_changed) {
+    // Called on the encoder task queue.

[kthelgason, 2017/05/19 08:34:15]

Do you want to add a thread_checker here to make this assumption explicit?

[åsapersson, 2017/06/08 13:55:32]

Can maybe do it in a separate cl.

+    rtc::CritScope lock(&crit_);
+    // Restrict framerate if it is lower.
+    *fps_changed = false;
+    const int fps_wanted = std::max(kMinFramerateFps, max_fps);
+    if (fps_wanted < sink_wants_.max_framerate_fps &&
+        RequestFramerateLocked(fps_wanted)) {
+      *fps_changed = true;
+      return true;

[kthelgason, 2017/05/19 08:34:14]

I don't really like this weird way of signalling what changed. Can we just
return a bit-vector that's 1 if fps changed, 2 if resolution changed and 0
otherwise? Or perhaps we can do something else like return a struct or enum or
something.

[åsapersson, 2017/06/08 13:55:32]

This part has been changed in https://codereview.webrtc.org/2903563002/. 

+    }
+    // Request lower resolution.
+    if (!IsResolutionScalingEnabledLocked())
+      return false;
+
+    // The input video frame size will have a resolution less than or equal to
+    // |max_pixel_count| depending on how the source can scale the frame size.
+    const int pixels_wanted = (pixel_count * 3) / 5;
+    if (pixels_wanted < kMinPixelsPerFrame ||
+        pixels_wanted >= sink_wants_.max_pixel_count) {
+      return false;
+    }
+    sink_wants_.max_pixel_count = pixels_wanted;
+    sink_wants_.target_pixel_count = rtc::Optional<int>();
+    if (source_)

[kthelgason, 2017/05/19 08:34:14]

Maybe we should just bail out right at the start if there is no source. Isn't
all of this pointless if there is no source that can be adapted?

[åsapersson, 2017/06/08 13:55:32]

Done in https://codereview.webrtc.org/2903563002/.

+      source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants());
+    return true;
+  }
+
+  bool IncreaseFpsOrRequestHigherResolutionThan(int max_fps,
+                                                int pixel_count,
+                                                bool* fps_changed) {
+    // Called on the encoder task queue.
+    rtc::CritScope lock(&crit_);
+    // Increase framerate if it is higher.
+    *fps_changed = false;
+    const int fps_wanted = std::max(kMinFramerateFps, max_fps);
+    if (fps_wanted > sink_wants_.max_framerate_fps &&
+        RequestFramerateLocked(fps_wanted)) {
+      *fps_changed = true;
+      return true;
+    }
+    // Request higher resolution.
+    if (!IsResolutionScalingEnabledLocked())
+      return false;
+
+    int max_pixels_wanted = pixel_count;
+    if (max_pixels_wanted != std::numeric_limits<int>::max())
+      max_pixels_wanted = pixel_count * 4;
+
+    if (max_pixels_wanted <= sink_wants_.max_pixel_count)
+      return false;
+
+    sink_wants_.max_pixel_count = max_pixels_wanted;
+    if (max_pixels_wanted == std::numeric_limits<int>::max()) {
+      sink_wants_.target_pixel_count.reset();
+    } else {
+      sink_wants_.target_pixel_count =
+          rtc::Optional<int>((pixel_count * 5) / 3);
+    }
+    if (source_)
+      source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants());
+    return true;
+  }
+
  private:
+  bool RequestFramerateLocked(int framerate_fps)
+      EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
+    if (!IsFramerateScalingEnabledLocked())
+      return false;
+
+    sink_wants_.max_framerate_fps = framerate_fps;
+    if (source_)
+      source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWants());
+    return true;
+  }
+
   bool IsResolutionScalingEnabledLocked() const
       EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
     return degradation_preference_ ==
                VideoSendStream::DegradationPreference::kMaintainFramerate ||
            degradation_preference_ ==
                VideoSendStream::DegradationPreference::kBalanced;
   }
 
   bool IsFramerateScalingEnabledLocked() const
       EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
-    // TODO(sprang): Also accept kBalanced here?
     return degradation_preference_ ==
-           VideoSendStream::DegradationPreference::kMaintainResolution;
+               VideoSendStream::DegradationPreference::kMaintainResolution ||
+           degradation_preference_ ==
+               VideoSendStream::DegradationPreference::kBalanced;
   }
 
   rtc::CriticalSection crit_;
   rtc::SequencedTaskChecker main_checker_;
   ViEEncoder* const vie_encoder_;
   rtc::VideoSinkWants sink_wants_ GUARDED_BY(&crit_);
   VideoSendStream::DegradationPreference degradation_preference_
       GUARDED_BY(&crit_);
   rtc::VideoSourceInterface<VideoFrame>* source_ GUARDED_BY(&crit_);
 
@@ skipping 469 matching lines @@
       stats_proxy_->GetStats().input_frame_rate,
       AdaptationRequest::Mode::kAdaptDown};
 
   bool downgrade_requested =
       last_adaptation_request_ &&
       last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
 
   int max_downgrades = 0;
   switch (degradation_preference_) {
     case VideoSendStream::DegradationPreference::kBalanced:
-      FALLTHROUGH();
+      max_downgrades = kMaxCpuResolutionDowngrades;
+      break;
     case VideoSendStream::DegradationPreference::kMaintainFramerate:
       max_downgrades = kMaxCpuResolutionDowngrades;
       if (downgrade_requested &&
           adaptation_request.input_pixel_count_ >=
               last_adaptation_request_->input_pixel_count_) {
         // Don't request lower resolution if the current resolution is not
         // lower than the last time we asked for the resolution to be lowered.
         return;
       }
       break;
@@ skipping 14 matching lines @@
     case VideoSendStream::DegradationPreference::kDegradationDisabled:
       return;
   }
 
   if (reason == kCpu) {
     if (GetConstAdaptCounter().TotalCount(kCpu) >= max_downgrades)
       return;
   }
 
   switch (degradation_preference_) {
-    case VideoSendStream::DegradationPreference::kBalanced:
-      FALLTHROUGH();
+    case VideoSendStream::DegradationPreference::kBalanced: {
+      int max_fps = MaxFpsForBitrateAndResolution(
+          last_observed_bitrate_bps_, last_frame_info_->pixel_count());
+      bool fps_restricted;
+      if (!source_proxy_->RestrictFpsOrRequestResolutionLowerThan(
+              max_fps, adaptation_request.input_pixel_count_,
+              &fps_restricted)) {
+        return;
+      }
+      if (fps_restricted) {
+        LOG(LS_INFO) << "Scaling down framerate: " << max_fps;
+        GetAdaptCounter().IncrementFramerate(reason);
+      } else {
+        LOG(LS_INFO) << "Scaling down resolution.";
+        GetAdaptCounter().IncrementResolution(reason);
+      }
+      break;
+    }
     case VideoSendStream::DegradationPreference::kMaintainFramerate:
       if (!source_proxy_->RequestResolutionLowerThan(
               adaptation_request.input_pixel_count_)) {
         return;
       }
       LOG(LS_INFO) << "Scaling down resolution.";
-      GetAdaptCounter().IncrementResolution(reason, 1);
+      GetAdaptCounter().IncrementResolution(reason);
       break;
     case VideoSendStream::DegradationPreference::kMaintainResolution:
       source_proxy_->RequestFramerateLowerThan(
           adaptation_request.framerate_fps_);
       LOG(LS_INFO) << "Scaling down framerate.";
-      GetAdaptCounter().IncrementFramerate(reason, 1);
+      GetAdaptCounter().IncrementFramerate(reason);
       break;
     case VideoSendStream::DegradationPreference::kDegradationDisabled:
       RTC_NOTREACHED();
   }
 
   last_adaptation_request_.emplace(adaptation_request);
 
   UpdateAdaptationStats(reason);
 
   LOG(LS_INFO) << GetConstAdaptCounter().ToString();
@@ skipping 12 matching lines @@
       last_frame_info_->pixel_count(),
       stats_proxy_->GetStats().input_frame_rate,
       AdaptationRequest::Mode::kAdaptUp};
 
   bool adapt_up_requested =
       last_adaptation_request_ &&
       last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
 
   switch (degradation_preference_) {
     case VideoSendStream::DegradationPreference::kBalanced:
-      FALLTHROUGH();
+      break;
     case VideoSendStream::DegradationPreference::kMaintainFramerate:
       if (adapt_up_requested &&
           adaptation_request.input_pixel_count_ <=
               last_adaptation_request_->input_pixel_count_) {
         // Don't request higher resolution if the current resolution is not
         // higher than the last time we asked for the resolution to be higher.
         return;
       }
       break;
     case VideoSendStream::DegradationPreference::kMaintainResolution:
       // TODO(sprang): Don't request higher framerate if we are already at
       // max requested fps?
       break;
     case VideoSendStream::DegradationPreference::kDegradationDisabled:
       return;
   }
 
   switch (degradation_preference_) {

[kthelgason, 2017/05/19 08:34:14]

Why do we have these two identical switch statements right after each other?

[åsapersson, 2017/06/08 13:55:32]

Removed the one above.

-    case VideoSendStream::DegradationPreference::kBalanced:
-      FALLTHROUGH();
+    case VideoSendStream::DegradationPreference::kBalanced: {
+      int pixel_count = adaptation_request.input_pixel_count_;
+      if (adapt_counter.ResolutionCount() == 1) {
+        LOG(LS_INFO) << "Removing down-scaling setting.";
+        pixel_count = std::numeric_limits<int>::max();
+      }
+      int fps = MaxFps(last_frame_info_->pixel_count());
+      bool fps_changed;
+      if (!source_proxy_->IncreaseFpsOrRequestHigherResolutionThan(
+              fps, pixel_count, &fps_changed)) {
+        return;
+      }
+      if (fps_changed) {
+        LOG(LS_INFO) << "Scaling up framerate: " << fps;
+        GetAdaptCounter().DecrementFramerate(reason);
+      } else {
+        LOG(LS_INFO) << "Scaling up resolution.";
+        GetAdaptCounter().DecrementResolution(reason);
+      }
+      break;
+    }
     case VideoSendStream::DegradationPreference::kMaintainFramerate:
       if (adapt_counter.TotalCount() == 1) {
         LOG(LS_INFO) << "Removing resolution down-scaling setting.";
         source_proxy_->RequestHigherResolutionThan(
             std::numeric_limits<int>::max());
       } else {
         source_proxy_->RequestHigherResolutionThan(
             adaptation_request.input_pixel_count_);
         LOG(LS_INFO) << "Scaling up resolution.";
       }
-      GetAdaptCounter().IncrementResolution(reason, -1);
+      GetAdaptCounter().DecrementResolution(reason);
       break;
     case VideoSendStream::DegradationPreference::kMaintainResolution:
       if (adapt_counter.TotalCount() == 1) {
         LOG(LS_INFO) << "Removing framerate down-scaling setting.";
         source_proxy_->RequestHigherFramerateThan(
             std::numeric_limits<int>::max());
       } else {
         source_proxy_->RequestHigherFramerateThan(
             adaptation_request.framerate_fps_);
         LOG(LS_INFO) << "Scaling up framerate.";
       }
-      GetAdaptCounter().IncrementFramerate(reason, -1);
+      GetAdaptCounter().DecrementFramerate(reason);
       break;
     case VideoSendStream::DegradationPreference::kDegradationDisabled:
       RTC_NOTREACHED();
   }
 
   last_adaptation_request_.emplace(adaptation_request);
 
   UpdateAdaptationStats(reason);
 
   LOG(LS_INFO) << adapt_counter.ToString();
@@ skipping 40 matching lines @@
 }
 
 const ViEEncoder::AdaptCounter& ViEEncoder::GetConstAdaptCounter() {
   return adapt_counters_[degradation_preference_];
 }
 
 // Class holding adaptation information.
 ViEEncoder::AdaptCounter::AdaptCounter() {
   fps_counters_.resize(kScaleReasonSize);
   resolution_counters_.resize(kScaleReasonSize);
+  RTC_DCHECK_EQ(fps_counters_.size(), 2) << "Update MoveCount()";
+  RTC_DCHECK_EQ(resolution_counters_.size(), 2) << "Update MoveCount()";
 }
 
 ViEEncoder::AdaptCounter::~AdaptCounter() {}
 
 std::string ViEEncoder::AdaptCounter::ToString() const {
   std::stringstream ss;
   ss << "Downgrade counts: fps: {" << ToString(fps_counters_);
   ss << "}, resolution: {" << ToString(resolution_counters_) << "}";
   return ss.str();
 }
 
 ViEEncoder::AdaptCounts ViEEncoder::AdaptCounter::Counts(int reason) const {
   AdaptCounts counts;
   counts.fps = fps_counters_[reason];
   counts.resolution = resolution_counters_[reason];
   return counts;
 }
 
-void ViEEncoder::AdaptCounter::IncrementFramerate(int reason, int delta) {
-  fps_counters_[reason] += delta;
+void ViEEncoder::AdaptCounter::IncrementFramerate(int reason) {
+  ++(fps_counters_[reason]);
 }
 
-void ViEEncoder::AdaptCounter::IncrementResolution(int reason, int delta) {
-  resolution_counters_[reason] += delta;
+void ViEEncoder::AdaptCounter::IncrementResolution(int reason) {
+  ++(resolution_counters_[reason]);
+}
+
+void ViEEncoder::AdaptCounter::DecrementFramerate(int reason) {
+  if (fps_counters_[reason] == 0) {
+    // Adapt up is in a different order, switch reason.
+    // E.g. framerate adapt down: quality, framerate adapt up: cpu.

[kthelgason, 2017/05/19 08:34:14]

Why is this the case? This seems very confusing.

[åsapersson, 2017/06/08 13:55:32]

Updated comment.

+    MoveCount(&resolution_counters_, reason);
+    MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize);
+  }
+  --(fps_counters_[reason]);
+  RTC_DCHECK_GE(fps_counters_[reason], 0);
+}
+
+void ViEEncoder::AdaptCounter::DecrementResolution(int reason) {
+  if (resolution_counters_[reason] == 0) {
+    // Adapt up is in a different order, switch reason.
+    // E.g. resolution adapt down: quality, resolution adapt up: cpu.
+    MoveCount(&fps_counters_, reason);
+    MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize);
+  }
+  --(resolution_counters_[reason]);
+  RTC_DCHECK_GE(resolution_counters_[reason], 0);
 }
 
 int ViEEncoder::AdaptCounter::FramerateCount() const {
   return Count(fps_counters_);
 }
 
 int ViEEncoder::AdaptCounter::ResolutionCount() const {
   return Count(resolution_counters_);
 }
 
@@ skipping 10 matching lines @@
 }
 
 int ViEEncoder::AdaptCounter::TotalCount(int reason) const {
   return FramerateCount(reason) + ResolutionCount(reason);
 }
 
 int ViEEncoder::AdaptCounter::Count(const std::vector<int>& counters) const {
   return std::accumulate(counters.begin(), counters.end(), 0);
 }
 
+void ViEEncoder::AdaptCounter::MoveCount(std::vector<int>* counters,
+                                         int from_reason) {
+  int to_reason = (from_reason + 1) % kScaleReasonSize;
+  ++((*counters)[to_reason]);
+  --((*counters)[from_reason]);
+  RTC_DCHECK_GE((*counters)[from_reason], 0);
+}
+
 std::string ViEEncoder::AdaptCounter::ToString(
     const std::vector<int>& counters) const {
   std::stringstream ss;
   for (size_t reason = 0; reason < kScaleReasonSize; ++reason) {
     ss << (reason ? " cpu" : "quality") << ":" << counters[reason];
   }
   return ss.str();
 }
 
 }  // namespace webrtc