Auto pause video streams based on encoder target bitrate.

webrtc/call/bitrate_allocator.cc

Index: webrtc/call/bitrate_allocator.cc
diff --git a/webrtc/call/bitrate_allocator.cc b/webrtc/call/bitrate_allocator.cc
index e82123a8b09c9069342aa4aaa62e0793d3da5190..cc29f7ed76817cdc16f842d9767731b84a92e299 100644
--- a/webrtc/call/bitrate_allocator.cc
+++ b/webrtc/call/bitrate_allocator.cc
@@ -15,7 +15,10 @@
 #include <utility>
 
 #include "webrtc/base/checks.h"
+#include "webrtc/base/logging.h"
 #include "webrtc/modules/bitrate_controller/include/bitrate_controller.h"
+#include "webrtc/system_wrappers/include/clock.h"
+#include "webrtc/system_wrappers/include/metrics.h"
 
 namespace webrtc {
 
@@ -28,13 +31,23 @@ const int kDefaultBitrateBps = 300000;
 const double kToggleFactor = 0.1;
 const uint32_t kMinToggleBitrateBps = 20000;
 
+const int64_t kBweLogIntervalMs = 5000;
+
 BitrateAllocator::BitrateAllocator(LimitObserver* limit_observer)
     : limit_observer_(limit_observer),
       bitrate_observer_configs_(),
       last_bitrate_bps_(kDefaultBitrateBps),
       last_non_zero_bitrate_bps_(kDefaultBitrateBps),
       last_fraction_loss_(0),
-      last_rtt_(0) {}
+      last_rtt_(0),
+      num_pause_events_(0),
+      clock_(Clock::GetRealTimeClock()),
+      last_bwe_log_time_(0) {}
+
+BitrateAllocator::~BitrateAllocator() {
+  RTC_LOGGED_HISTOGRAM_COUNTS_100("WebRTC.Call.NumberOfPauseEvents",
+                                  num_pause_events_);

[stefan-webrtc, 2016/06/30 16:17:38]

Good to have! Remember to add it to the uma configs too.

[mflodman, 2016/07/01 09:40:15]

Thanks for the reminder, I'll make sure this happens.

+}
 
 void BitrateAllocator::OnNetworkChanged(uint32_t target_bitrate_bps,
                                         uint8_t fraction_loss,
@@ -46,11 +59,45 @@ void BitrateAllocator::OnNetworkChanged(uint32_t target_bitrate_bps,
   last_fraction_loss_ = fraction_loss;
   last_rtt_ = rtt;
 
+  // Periodically log the incoming BWE.
+  int64_t now = clock_->TimeInMilliseconds();
+  if (now > last_bwe_log_time_ + kBweLogIntervalMs) {
+    LOG(LS_INFO) << "Current BWE " << target_bitrate_bps;
+    last_bwe_log_time_ = now;
+  }
+
   ObserverAllocation allocation = AllocateBitrates(target_bitrate_bps);
-  for (const auto& kv : allocation) {
-    kv.first->OnBitrateUpdated(kv.second, last_fraction_loss_, last_rtt_);
+
+  for (auto& config : bitrate_observer_configs_) {
+    uint32_t allocated_bitrate = allocation[config.observer];
+    uint32_t protection_bitrate = config.observer->OnBitrateUpdated(
+        allocated_bitrate, last_fraction_loss_, last_rtt_);
+
+    if (allocated_bitrate == 0 && config.allocated_bitrate_bps > 0) {
+      if (target_bitrate_bps > 0)
+        ++num_pause_events_;
+      uint32_t predicted_protection =

[stefan-webrtc, 2016/06/30 16:17:38]

_bps

Add a comment describing that this assumes the same amount of protection as was
observed before the stream was paused.

[mflodman, 2016/07/01 09:40:15]

Done.

+          (1.0 - config.media_ratio) * config.min_bitrate_bps;
+      LOG(LS_INFO) << "Pausing observer " << config.observer
+                   << " with configured min bitrate " << config.min_bitrate_bps
+                   << " and current estimate of " << target_bitrate_bps

[stefan-webrtc, 2016/06/30 16:17:38]

Should you log the allocation here too?

[mflodman, 2016/07/01 09:40:15]

Good point, added.

[stefan-webrtc, 2016/07/01 10:42:49]

Can't see that it was added?

[mflodman, 2016/07/01 10:52:28]

Sorry, bad reply from me. The allocation will be 0 for this observer, but I
added it for the case below when we have an allocation.

+                   << " and protection bitrate " << predicted_protection;
+    } else if (allocated_bitrate > 0 && config.allocated_bitrate_bps == 0) {
+      if (target_bitrate_bps > 0)
+        ++num_pause_events_;
+      LOG(LS_INFO) << "Resuming observer " << config.observer
+                   << " with configured min bitrate " << config.min_bitrate_bps
+                   << " and current allocation " << allocated_bitrate
+                   << " and protection bitrate " << protection_bitrate;
+    }
+
+    // Only update the media ratio if the observer got an allocation.
+    if (allocated_bitrate > 0) {
+      config.media_ratio = (allocated_bitrate - protection_bitrate) /

[stefan-webrtc, 2016/06/30 16:17:38]

Use a helper variable media_bitrate = allocated_bitrate - protection_bitrate to
simplify and make it clear that we assume allocated = media + protection

[mflodman, 2016/07/01 09:40:15]

Done.

+                            static_cast<double>(allocated_bitrate);
+    }
+    config.allocated_bitrate_bps = allocated_bitrate;
   }
-  last_allocation_ = allocation;
 }
 
 void BitrateAllocator::AddObserver(BitrateAllocatorObserver* observer,
@@ -77,8 +124,12 @@ void BitrateAllocator::AddObserver(BitrateAllocatorObserver* observer,
   if (last_bitrate_bps_ > 0) {
     // Calculate a new allocation and update all observers.
     allocation = AllocateBitrates(last_bitrate_bps_);
-    for (const auto& kv : allocation)
-      kv.first->OnBitrateUpdated(kv.second, last_fraction_loss_, last_rtt_);
+    for (auto& config : bitrate_observer_configs_) {
+      uint32_t allocated_bitrate = allocation[config.observer];
+      config.observer->OnBitrateUpdated(
+          allocated_bitrate, last_fraction_loss_, last_rtt_);
+      config.allocated_bitrate_bps = allocated_bitrate;

[stefan-webrtc, 2016/06/30 16:17:38]

Is there no need to update the media ratio here? I assume it could be the case
that some protection is used if last_fraction_loss_ > 0.

[mflodman, 2016/07/01 09:40:15]

That is a good question and I assumed it would be ok to wait for the first
"real" allocation. I added that now though.

[stefan-webrtc, 2016/07/01 10:42:49]

Thanks. Better to do it and avoid others asking themselves the same question, I
guess.

+    }
   } else {
     // Currently, an encoder is not allowed to produce frames.
     // But we still have to return the initial config bitrate + let the
@@ -87,8 +138,6 @@ void BitrateAllocator::AddObserver(BitrateAllocatorObserver* observer,
     observer->OnBitrateUpdated(0, last_fraction_loss_, last_rtt_);
   }
   UpdateAllocationLimits();
-
-  last_allocation_ = allocation;
 }
 
 void BitrateAllocator::UpdateAllocationLimits() {
@@ -122,17 +171,22 @@ void BitrateAllocator::RemoveObserver(BitrateAllocatorObserver* observer) {
 
 int BitrateAllocator::GetStartBitrate(BitrateAllocatorObserver* observer) {
   rtc::CritScope lock(&crit_sect_);
-  const auto& it = last_allocation_.find(observer);
-  if (it != last_allocation_.end())
-    return it->second;
-
-  // This is a new observer that has not yet been started. Assume that if it is
-  // added, all observers would split the available bitrate evenly.
-  return last_non_zero_bitrate_bps_ /
-         static_cast<int>((bitrate_observer_configs_.size() + 1));
+  const auto& it = FindObserverConfig(observer);
+  if (it == bitrate_observer_configs_.end()) {
+    // This observer hasn't been added yet, just give it its fair share.

[stefan-webrtc, 2016/06/30 16:17:38]

Not clear to me why this can happen? Is it correct usage?

[mflodman, 2016/07/01 09:40:15]

AS of now it is, the encoder is configured before the observer is added here and
we need the start bitrate before that happens.

[stefan-webrtc, 2016/07/01 10:42:49]

Acknowledged.

+    return last_non_zero_bitrate_bps_ /
+             static_cast<int>((bitrate_observer_configs_.size() + 1));
+  } else if (it->allocated_bitrate_bps == -1) {
+    // This observer hasn't received an allocation yet, so do the same.
+    return last_non_zero_bitrate_bps_ /
+             static_cast<int>(bitrate_observer_configs_.size());
+  } else {
+    // This observer already has an allocation.
+    return it->allocated_bitrate_bps;
+  }
 }
 
-BitrateAllocator::ObserverConfigList::iterator
+BitrateAllocator::ObserverConfigs::iterator
 BitrateAllocator::FindObserverConfig(
     const BitrateAllocatorObserver* observer) {
   for (auto it = bitrate_observer_configs_.begin();
@@ -202,9 +256,10 @@ BitrateAllocator::ObserverAllocation BitrateAllocator::LowRateAllocation(
           LastAllocatedBitrate(observer_config) == 0)
         continue;
 
-      if (remaining_bitrate >= observer_config.min_bitrate_bps) {
-        allocation[observer_config.observer] = observer_config.min_bitrate_bps;
-        remaining_bitrate -= observer_config.min_bitrate_bps;
+      uint32_t required_bitrate = MinBitrateWithHysteresis(observer_config);
+      if (remaining_bitrate >= required_bitrate) {
+        allocation[observer_config.observer] = required_bitrate;
+        remaining_bitrate -= required_bitrate;
       }
     }
   }
@@ -263,14 +318,11 @@ BitrateAllocator::ObserverAllocation BitrateAllocator::MaxRateAllocation(
 uint32_t BitrateAllocator::LastAllocatedBitrate(
     const ObserverConfig& observer_config) {
 
-  const auto& it = last_allocation_.find(observer_config.observer);
-  if (it != last_allocation_.end())
-    return it->second;
-
   // Return the configured minimum bitrate for newly added observers, to avoid
   // requiring an extra high bitrate for the observer to get an allocated
   // bitrate.
-  return observer_config.min_bitrate_bps;
+  return observer_config.allocated_bitrate_bps == -1 ?
+      observer_config.min_bitrate_bps : observer_config.allocated_bitrate_bps;
 }
 
 uint32_t BitrateAllocator::MinBitrateWithHysteresis(
@@ -280,6 +332,15 @@ uint32_t BitrateAllocator::MinBitrateWithHysteresis(
     min_bitrate += std::max(static_cast<uint32_t>(kToggleFactor * min_bitrate),
                             kMinToggleBitrateBps);
   }
+  // Account for protection bitrate used by this observer in the previous
+  // allocation.
+  // Note: the ratio will only be updated when the stream is active, meaning a
+  // paused stream won't get any ratio updates. This might lead to waiting a bit
+  // longer than necessary if the network condition improves, but this is to
+  // avoid too much toggling.

[stefan-webrtc, 2016/06/30 16:17:38]

I wonder if this should be moved up to where the ratio is updated instead? Not
sure, so feel free to disagree.

That said, I do think this is the right trade-off.

[mflodman, 2016/07/01 09:40:15]

As of now I prefer it this way to have it separate. Let's discuss for the future
though, if this can be simplified and done differently.

[stefan-webrtc, 2016/07/01 10:42:49]

Acknowledged.

+  if (observer_config.media_ratio > 0.0 && observer_config.media_ratio < 1.0)
+    min_bitrate += min_bitrate * (1.0 - observer_config.media_ratio);
+
   return min_bitrate;
 }