Implement kBalanced degradation preference.

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 55 matching lines @@
   if (kbps > 0) {
     if (kbps < 300 /* qvga */) {
       return 320 * 240;
     } else if (kbps < 500 /* vga */) {
       return 640 * 480;
     }
   }
   return std::numeric_limits<uint32_t>::max();
 }
 
+// Initial limits for kBalanced degradation preference.
+int MinFps(int pixels) {
+  if (pixels <= 320 * 240) {
+    return 7;
+  } else if (pixels <= 480 * 270) {
+    return 10;
+  } else if (pixels <= 640 * 480) {
+    return 15;
+  } else {
+    return std::numeric_limits<int>::max();
+  }
+}
+
+int MaxFps(int pixels) {
+  if (pixels <= 320 * 240) {
+    return 10;
+  } else if (pixels <= 480 * 270) {
+    return 15;
+  } else {
+    return std::numeric_limits<int>::max();
+  }
+}
+
 bool IsResolutionScalingEnabled(
     VideoSendStream::DegradationPreference degradation_preference) {
   return degradation_preference ==
              VideoSendStream::DegradationPreference::kMaintainFramerate ||
          degradation_preference ==
              VideoSendStream::DegradationPreference::kBalanced;
 }
 
 bool IsFramerateScalingEnabled(
     VideoSendStream::DegradationPreference degradation_preference) {
@@ skipping 118 matching lines @@
     if (source_)
       source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
   }
 
   rtc::VideoSinkWants GetActiveSinkWants() EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
     rtc::VideoSinkWants wants = sink_wants_;
     // Clear any constraints from the current sink wants that don't apply to
     // the used degradation_preference.
     switch (degradation_preference_) {
       case VideoSendStream::DegradationPreference::kBalanced:
-        FALLTHROUGH();
+        break;
       case VideoSendStream::DegradationPreference::kMaintainFramerate:
         wants.max_framerate_fps = std::numeric_limits<int>::max();
         break;
       case VideoSendStream::DegradationPreference::kMaintainResolution:
         wants.max_pixel_count = std::numeric_limits<int>::max();
         wants.target_pixel_count.reset();
         break;
       case VideoSendStream::DegradationPreference::kDegradationDisabled:
         wants.max_pixel_count = std::numeric_limits<int>::max();
         wants.target_pixel_count.reset();
         wants.max_framerate_fps = std::numeric_limits<int>::max();
     }
     return wants;
   }
 
+  void ResetPixelFpsCount() {
+    rtc::CritScope lock(&crit_);
+    sink_wants_.max_pixel_count = std::numeric_limits<int>::max();
+    sink_wants_.target_pixel_count.reset();
+    sink_wants_.max_framerate_fps = std::numeric_limits<int>::max();
+    if (source_)
+      source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
+  }
+
   bool RequestResolutionLowerThan(int pixel_count) {
     // Called on the encoder task queue.
     rtc::CritScope lock(&crit_);
     if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) {
       // This can happen since |degradation_preference_| is set on libjingle's
       // worker thread but the adaptation is done on the encoder task queue.
       return false;
     }
     // The input video frame size will have a resolution less than or equal to
     // |max_pixel_count| depending on how the source can scale the frame size.
@@ skipping 200 matching lines @@
     rtc::VideoSourceInterface<VideoFrame>* source,
     const VideoSendStream::DegradationPreference& degradation_preference) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   source_proxy_->SetSource(source, degradation_preference);
   encoder_queue_.PostTask([this, degradation_preference] {
     RTC_DCHECK_RUN_ON(&encoder_queue_);
     if (degradation_preference_ != degradation_preference) {
       // Reset adaptation state, so that we're not tricked into thinking there's
       // an already pending request of the same type.
       last_adaptation_request_.reset();
+      if (degradation_preference ==
+              VideoSendStream::DegradationPreference::kBalanced ||
+          degradation_preference_ ==
+              VideoSendStream::DegradationPreference::kBalanced) {
+        // TODO(asapersson): Consider removing |adapt_counters_| map and use one
+        // AdaptCounter for all modes.
+        source_proxy_->ResetPixelFpsCount();
+        adapt_counters_.clear();
+      }
     }
     degradation_preference_ = degradation_preference;
     bool allow_scaling = IsResolutionScalingEnabled(degradation_preference_);
     initial_rampup_ = allow_scaling ? 0 : kMaxInitialFramedrop;
     ConfigureQualityScaler();
   });
 }
 
 void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
   source_proxy_->SetWantsRotationApplied(rotation_applied);
@@ skipping 336 matching lines @@
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   AdaptationRequest adaptation_request = {
       last_frame_info_->pixel_count(),
       stats_proxy_->GetStats().input_frame_rate,
       AdaptationRequest::Mode::kAdaptDown};
 
   bool downgrade_requested =
       last_adaptation_request_ &&
       last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
 
-  int max_downgrades = 0;
   switch (degradation_preference_) {
     case VideoSendStream::DegradationPreference::kBalanced:
-      FALLTHROUGH();
+      break;
     case VideoSendStream::DegradationPreference::kMaintainFramerate:
-      max_downgrades = kMaxCpuResolutionDowngrades;
       if (downgrade_requested &&
           adaptation_request.input_pixel_count_ >=
               last_adaptation_request_->input_pixel_count_) {
         // Don't request lower resolution if the current resolution is not
         // lower than the last time we asked for the resolution to be lowered.
         return;
       }
       break;
     case VideoSendStream::DegradationPreference::kMaintainResolution:
-      max_downgrades = kMaxCpuFramerateDowngrades;
       if (adaptation_request.framerate_fps_ <= 0 ||
           (downgrade_requested &&
            adaptation_request.framerate_fps_ < kMinFramerateFps)) {
         // If no input fps estimate available, can't determine how to scale down
         // framerate. Otherwise, don't request lower framerate if we don't have
         // a valid frame rate. Since framerate, unlike resolution, is a measure
         // we have to estimate, and can fluctuate naturally over time, don't
         // make the same kind of limitations as for resolution, but trust the
         // overuse detector to not trigger too often.
         return;
       }
       break;
     case VideoSendStream::DegradationPreference::kDegradationDisabled:
       return;
   }
 
   if (reason == kCpu) {
-    if (GetConstAdaptCounter().TotalCount(kCpu) >= max_downgrades)
+    if (GetConstAdaptCounter().ResolutionCount(kCpu) >=
+            kMaxCpuResolutionDowngrades ||
+        GetConstAdaptCounter().FramerateCount(kCpu) >=
+            kMaxCpuFramerateDowngrades) {
       return;
+    }
   }
 
   switch (degradation_preference_) {
-    case VideoSendStream::DegradationPreference::kBalanced:
+    case VideoSendStream::DegradationPreference::kBalanced: {
+      // Try scale down framerate, if lower.
+      int fps = MinFps(last_frame_info_->pixel_count());
+      if (source_proxy_->RestrictFramerate(fps)) {
+        GetAdaptCounter().IncrementFramerate(reason);
+        break;
+      }
+      // Scale down resolution.
       FALLTHROUGH();
+    }
     case VideoSendStream::DegradationPreference::kMaintainFramerate:
       // Scale down resolution.
       if (!source_proxy_->RequestResolutionLowerThan(
               adaptation_request.input_pixel_count_)) {
         return;
       }
-      GetAdaptCounter().IncrementResolution(reason, 1);
+      GetAdaptCounter().IncrementResolution(reason);
       break;
     case VideoSendStream::DegradationPreference::kMaintainResolution:
       // Scale down framerate.
       if (!source_proxy_->RequestFramerateLowerThan(
               adaptation_request.framerate_fps_)) {
         return;
       }
-      GetAdaptCounter().IncrementFramerate(reason, 1);
+      GetAdaptCounter().IncrementFramerate(reason);
       break;
     case VideoSendStream::DegradationPreference::kDegradationDisabled:
       RTC_NOTREACHED();
   }
 
   last_adaptation_request_.emplace(adaptation_request);
 
   UpdateAdaptationStats(reason);
 
   LOG(LS_INFO) << GetConstAdaptCounter().ToString();
@@ skipping 10 matching lines @@
 
   AdaptationRequest adaptation_request = {
       last_frame_info_->pixel_count(),
       stats_proxy_->GetStats().input_frame_rate,
       AdaptationRequest::Mode::kAdaptUp};
 
   bool adapt_up_requested =
       last_adaptation_request_ &&
       last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
 
-  switch (degradation_preference_) {
-    case VideoSendStream::DegradationPreference::kBalanced:
-      FALLTHROUGH();
-    case VideoSendStream::DegradationPreference::kMaintainFramerate:
-      if (adapt_up_requested &&
-          adaptation_request.input_pixel_count_ <=
-              last_adaptation_request_->input_pixel_count_) {
-        // Don't request higher resolution if the current resolution is not
-        // higher than the last time we asked for the resolution to be higher.
-        return;
-      }
-      break;
-    case VideoSendStream::DegradationPreference::kMaintainResolution:
-      // TODO(sprang): Don't request higher framerate if we are already at
-      // max requested fps?
-      break;
-    case VideoSendStream::DegradationPreference::kDegradationDisabled:
+  if (degradation_preference_ ==
+      VideoSendStream::DegradationPreference::kMaintainFramerate) {
+    if (adapt_up_requested &&
+        adaptation_request.input_pixel_count_ <=
+            last_adaptation_request_->input_pixel_count_) {
+      // Don't request higher resolution if the current resolution is not
+      // higher than the last time we asked for the resolution to be higher.
       return;
+    }
   }
 
   switch (degradation_preference_) {
-    case VideoSendStream::DegradationPreference::kBalanced:
+    case VideoSendStream::DegradationPreference::kBalanced: {
+      // Try scale up framerate, if higher.
+      int fps = MaxFps(last_frame_info_->pixel_count());
+      if (source_proxy_->IncreaseFramerate(fps)) {
+        GetAdaptCounter().DecrementFramerate(reason, fps);
+        // Reset framerate in case of fewer fps steps down than up.
+        if (adapt_counter.FramerateCount() == 0 &&
+            fps != std::numeric_limits<int>::max()) {
+          LOG(LS_INFO) << "Removing framerate down-scaling setting.";
+          source_proxy_->IncreaseFramerate(std::numeric_limits<int>::max());
+        }
+        break;
+      }
+      // Scale up resolution.
       FALLTHROUGH();
+    }
     case VideoSendStream::DegradationPreference::kMaintainFramerate: {
       // Scale up resolution.
       int pixel_count = adaptation_request.input_pixel_count_;
       if (adapt_counter.ResolutionCount() == 1) {
         LOG(LS_INFO) << "Removing resolution down-scaling setting.";
         pixel_count = std::numeric_limits<int>::max();
       }
       if (!source_proxy_->RequestHigherResolutionThan(pixel_count))
         return;
-      GetAdaptCounter().IncrementResolution(reason, -1);
+      GetAdaptCounter().DecrementResolution(reason);
       break;
     }
     case VideoSendStream::DegradationPreference::kMaintainResolution: {
       // Scale up framerate.
       int fps = adaptation_request.framerate_fps_;
       if (adapt_counter.FramerateCount() == 1) {
         LOG(LS_INFO) << "Removing framerate down-scaling setting.";
         fps = std::numeric_limits<int>::max();
       }
       if (!source_proxy_->RequestHigherFramerateThan(fps))
         return;
-      GetAdaptCounter().IncrementFramerate(reason, -1);
+      GetAdaptCounter().DecrementFramerate(reason);
       break;
     }
     case VideoSendStream::DegradationPreference::kDegradationDisabled:
-      RTC_NOTREACHED();
+      return;
   }
 
   last_adaptation_request_.emplace(adaptation_request);
 
   UpdateAdaptationStats(reason);
 
   LOG(LS_INFO) << adapt_counter.ToString();
 }
 
 void ViEEncoder::UpdateAdaptationStats(AdaptReason reason) {
@@ skipping 37 matching lines @@
 }
 
 const ViEEncoder::AdaptCounter& ViEEncoder::GetConstAdaptCounter() {
   return adapt_counters_[degradation_preference_];
 }
 
 // Class holding adaptation information.
 ViEEncoder::AdaptCounter::AdaptCounter() {
   fps_counters_.resize(kScaleReasonSize);
   resolution_counters_.resize(kScaleReasonSize);
+  static_assert(kScaleReasonSize == 2, "Update MoveCount.");
 }
 
 ViEEncoder::AdaptCounter::~AdaptCounter() {}
 
 std::string ViEEncoder::AdaptCounter::ToString() const {
   std::stringstream ss;
   ss << "Downgrade counts: fps: {" << ToString(fps_counters_);
   ss << "}, resolution: {" << ToString(resolution_counters_) << "}";
   return ss.str();
 }
 
 ViEEncoder::AdaptCounts ViEEncoder::AdaptCounter::Counts(int reason) const {
   AdaptCounts counts;
   counts.fps = fps_counters_[reason];
   counts.resolution = resolution_counters_[reason];
   return counts;
 }
 
-void ViEEncoder::AdaptCounter::IncrementFramerate(int reason, int delta) {
-  fps_counters_[reason] += delta;
+void ViEEncoder::AdaptCounter::IncrementFramerate(int reason) {
+  ++(fps_counters_[reason]);
 }
 
-void ViEEncoder::AdaptCounter::IncrementResolution(int reason, int delta) {
-  resolution_counters_[reason] += delta;
+void ViEEncoder::AdaptCounter::IncrementResolution(int reason) {
+  ++(resolution_counters_[reason]);
+}
+
+void ViEEncoder::AdaptCounter::DecrementFramerate(int reason) {
+  if (fps_counters_[reason] == 0) {
+    // Balanced mode: Adapt up is in a different order, switch reason.
+    // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3).
+    // 1. Down resolution (cpu):   res={quality:0,cpu:1}, fps={quality:0,cpu:0}
+    // 2. Down fps (quality):      res={quality:0,cpu:1}, fps={quality:1,cpu:0}
+    // 3. Up fps (cpu):            res={quality:1,cpu:0}, fps={quality:0,cpu:0}
+    // 4. Up resolution (quality): res={quality:0,cpu:0}, fps={quality:0,cpu:0}
+    RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
+    RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded.";
+    MoveCount(&resolution_counters_, reason);
+    MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize);
+  }
+  --(fps_counters_[reason]);
+  RTC_DCHECK_GE(fps_counters_[reason], 0);
+}
+
+void ViEEncoder::AdaptCounter::DecrementResolution(int reason) {
+  if (resolution_counters_[reason] == 0) {
+    // Balanced mode: Adapt up is in a different order, switch reason.
+    RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
+    RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded.";
+    MoveCount(&fps_counters_, reason);
+    MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize);
+  }
+  --(resolution_counters_[reason]);
+  RTC_DCHECK_GE(resolution_counters_[reason], 0);
+}
+
+void ViEEncoder::AdaptCounter::DecrementFramerate(int reason, int cur_fps) {
+  DecrementFramerate(reason);
+  // Reset if at max fps (i.e. in case of fewer steps up than down).
+  if (cur_fps == std::numeric_limits<int>::max())
+    std::fill(fps_counters_.begin(), fps_counters_.end(), 0);
 }
 
 int ViEEncoder::AdaptCounter::FramerateCount() const {
   return Count(fps_counters_);
 }
 
 int ViEEncoder::AdaptCounter::ResolutionCount() const {
   return Count(resolution_counters_);
 }
 
-int ViEEncoder::AdaptCounter::TotalCount() const {
-  return FramerateCount() + ResolutionCount();
-}
-
 int ViEEncoder::AdaptCounter::FramerateCount(int reason) const {
   return fps_counters_[reason];
 }
 
 int ViEEncoder::AdaptCounter::ResolutionCount(int reason) const {
   return resolution_counters_[reason];
 }
 
 int ViEEncoder::AdaptCounter::TotalCount(int reason) const {
   return FramerateCount(reason) + ResolutionCount(reason);
 }
 
 int ViEEncoder::AdaptCounter::Count(const std::vector<int>& counters) const {
   return std::accumulate(counters.begin(), counters.end(), 0);
 }
 
+void ViEEncoder::AdaptCounter::MoveCount(std::vector<int>* counters,
+                                         int from_reason) {
+  int to_reason = (from_reason + 1) % kScaleReasonSize;
+  ++((*counters)[to_reason]);
+  --((*counters)[from_reason]);
+}
+
 std::string ViEEncoder::AdaptCounter::ToString(
     const std::vector<int>& counters) const {
   std::stringstream ss;
   for (size_t reason = 0; reason < kScaleReasonSize; ++reason) {
     ss << (reason ? " cpu" : "quality") << ":" << counters[reason];
   }
   return ss.str();
 }
 
 }  // namespace webrtc