Auto pause video streams based on encoder target bitrate.

webrtc/call/bitrate_allocator.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  *
  */
 
 #include "webrtc/call/bitrate_allocator.h"
 
 #include <algorithm>
 #include <utility>
 
 #include "webrtc/base/checks.h"
+#include "webrtc/base/logging.h"
 #include "webrtc/modules/bitrate_controller/include/bitrate_controller.h"
+#include "webrtc/system_wrappers/include/clock.h"
+#include "webrtc/system_wrappers/include/metrics.h"
 
 namespace webrtc {
 
 // Allow packets to be transmitted in up to 2 times max video bitrate if the
 // bandwidth estimate allows it.
 const int kTransmissionMaxBitrateMultiplier = 2;
 const int kDefaultBitrateBps = 300000;
 
 // Require a bitrate increase of max(10%, 20kbps) to resume paused streams.
 const double kToggleFactor = 0.1;
 const uint32_t kMinToggleBitrateBps = 20000;
 
+const int64_t kBweLogIntervalMs = 5000;
+
 BitrateAllocator::BitrateAllocator(LimitObserver* limit_observer)
     : limit_observer_(limit_observer),
       bitrate_observer_configs_(),
       last_bitrate_bps_(kDefaultBitrateBps),
       last_non_zero_bitrate_bps_(kDefaultBitrateBps),
       last_fraction_loss_(0),
-      last_rtt_(0) {}
+      last_rtt_(0),
+      num_pause_events_(0),
+      clock_(Clock::GetRealTimeClock()),
+      last_bwe_log_time_(0) {}
+
+BitrateAllocator::~BitrateAllocator() {
+  RTC_LOGGED_HISTOGRAM_COUNTS_100("WebRTC.Call.NumberOfPauseEvents",
+                                  num_pause_events_);

[stefan-webrtc, 2016/06/30 16:17:38]

Good to have! Remember to add it to the uma configs too.

[mflodman, 2016/07/01 09:40:15]

Thanks for the reminder, I'll make sure this happens.

+}
 
 void BitrateAllocator::OnNetworkChanged(uint32_t target_bitrate_bps,
                                         uint8_t fraction_loss,
                                         int64_t rtt) {
   rtc::CritScope lock(&crit_sect_);
   last_bitrate_bps_ = target_bitrate_bps;
   last_non_zero_bitrate_bps_ =
       target_bitrate_bps > 0 ? target_bitrate_bps : last_non_zero_bitrate_bps_;
   last_fraction_loss_ = fraction_loss;
   last_rtt_ = rtt;
 
+  // Periodically log the incoming BWE.
+  int64_t now = clock_->TimeInMilliseconds();
+  if (now > last_bwe_log_time_ + kBweLogIntervalMs) {
+    LOG(LS_INFO) << "Current BWE " << target_bitrate_bps;
+    last_bwe_log_time_ = now;
+  }
+
   ObserverAllocation allocation = AllocateBitrates(target_bitrate_bps);
-  for (const auto& kv : allocation) {
-    kv.first->OnBitrateUpdated(kv.second, last_fraction_loss_, last_rtt_);
+
+  for (auto& config : bitrate_observer_configs_) {
+    uint32_t allocated_bitrate = allocation[config.observer];
+    uint32_t protection_bitrate = config.observer->OnBitrateUpdated(
+        allocated_bitrate, last_fraction_loss_, last_rtt_);
+
+    if (allocated_bitrate == 0 && config.allocated_bitrate_bps > 0) {
+      if (target_bitrate_bps > 0)
+        ++num_pause_events_;
+      uint32_t predicted_protection =

[stefan-webrtc, 2016/06/30 16:17:38]

_bps

Add a comment describing that this assumes the same amount of protection as was
observed before the stream was paused.

[mflodman, 2016/07/01 09:40:15]

Done.

+          (1.0 - config.media_ratio) * config.min_bitrate_bps;
+      LOG(LS_INFO) << "Pausing observer " << config.observer
+                   << " with configured min bitrate " << config.min_bitrate_bps
+                   << " and current estimate of " << target_bitrate_bps

[stefan-webrtc, 2016/06/30 16:17:38]

Should you log the allocation here too?

[mflodman, 2016/07/01 09:40:15]

Good point, added.

[stefan-webrtc, 2016/07/01 10:42:49]

Can't see that it was added?

[mflodman, 2016/07/01 10:52:28]

Sorry, bad reply from me. The allocation will be 0 for this observer, but I
added it for the case below when we have an allocation.

+                   << " and protection bitrate " << predicted_protection;
+    } else if (allocated_bitrate > 0 && config.allocated_bitrate_bps == 0) {
+      if (target_bitrate_bps > 0)
+        ++num_pause_events_;
+      LOG(LS_INFO) << "Resuming observer " << config.observer
+                   << " with configured min bitrate " << config.min_bitrate_bps
+                   << " and current allocation " << allocated_bitrate
+                   << " and protection bitrate " << protection_bitrate;
+    }
+
+    // Only update the media ratio if the observer got an allocation.
+    if (allocated_bitrate > 0) {
+      config.media_ratio = (allocated_bitrate - protection_bitrate) /

[stefan-webrtc, 2016/06/30 16:17:38]

Use a helper variable media_bitrate = allocated_bitrate - protection_bitrate to
simplify and make it clear that we assume allocated = media + protection

[mflodman, 2016/07/01 09:40:15]

Done.

+                            static_cast<double>(allocated_bitrate);
+    }
+    config.allocated_bitrate_bps = allocated_bitrate;
   }
-  last_allocation_ = allocation;
 }
 
 void BitrateAllocator::AddObserver(BitrateAllocatorObserver* observer,
                                    uint32_t min_bitrate_bps,
                                    uint32_t max_bitrate_bps,
                                    uint32_t pad_up_bitrate_bps,
                                    bool enforce_min_bitrate) {
   rtc::CritScope lock(&crit_sect_);
   auto it = FindObserverConfig(observer);
 
   // Update settings if the observer already exists, create a new one otherwise.
   if (it != bitrate_observer_configs_.end()) {
     it->min_bitrate_bps = min_bitrate_bps;
     it->max_bitrate_bps = max_bitrate_bps;
     it->pad_up_bitrate_bps = pad_up_bitrate_bps;
     it->enforce_min_bitrate = enforce_min_bitrate;
   } else {
     bitrate_observer_configs_.push_back(
         ObserverConfig(observer, min_bitrate_bps, max_bitrate_bps,
                        pad_up_bitrate_bps, enforce_min_bitrate));
   }
 
   ObserverAllocation allocation;
   if (last_bitrate_bps_ > 0) {
     // Calculate a new allocation and update all observers.
     allocation = AllocateBitrates(last_bitrate_bps_);
-    for (const auto& kv : allocation)
-      kv.first->OnBitrateUpdated(kv.second, last_fraction_loss_, last_rtt_);
+    for (auto& config : bitrate_observer_configs_) {
+      uint32_t allocated_bitrate = allocation[config.observer];
+      config.observer->OnBitrateUpdated(
+          allocated_bitrate, last_fraction_loss_, last_rtt_);
+      config.allocated_bitrate_bps = allocated_bitrate;

[stefan-webrtc, 2016/06/30 16:17:38]

Is there no need to update the media ratio here? I assume it could be the case
that some protection is used if last_fraction_loss_ > 0.

[mflodman, 2016/07/01 09:40:15]

That is a good question and I assumed it would be ok to wait for the first
"real" allocation. I added that now though.

[stefan-webrtc, 2016/07/01 10:42:49]

Thanks. Better to do it and avoid others asking themselves the same question, I
guess.

+    }
   } else {
     // Currently, an encoder is not allowed to produce frames.
     // But we still have to return the initial config bitrate + let the
     // observer know that it can not produce frames.
     allocation = AllocateBitrates(last_non_zero_bitrate_bps_);
     observer->OnBitrateUpdated(0, last_fraction_loss_, last_rtt_);
   }
   UpdateAllocationLimits();
-
-  last_allocation_ = allocation;
 }
 
 void BitrateAllocator::UpdateAllocationLimits() {
   uint32_t total_requested_padding_bitrate = 0;
   uint32_t total_requested_min_bitrate = 0;
 
   {
     rtc::CritScope lock(&crit_sect_);
     for (const auto& config : bitrate_observer_configs_) {
       if (config.enforce_min_bitrate) {
@@ skipping 13 matching lines @@
     auto it = FindObserverConfig(observer);
     if (it != bitrate_observer_configs_.end()) {
       bitrate_observer_configs_.erase(it);
     }
   }
   UpdateAllocationLimits();
 }
 
 int BitrateAllocator::GetStartBitrate(BitrateAllocatorObserver* observer) {
   rtc::CritScope lock(&crit_sect_);
-  const auto& it = last_allocation_.find(observer);
-  if (it != last_allocation_.end())
-    return it->second;
-
-  // This is a new observer that has not yet been started. Assume that if it is
-  // added, all observers would split the available bitrate evenly.
-  return last_non_zero_bitrate_bps_ /
-         static_cast<int>((bitrate_observer_configs_.size() + 1));
+  const auto& it = FindObserverConfig(observer);
+  if (it == bitrate_observer_configs_.end()) {
+    // This observer hasn't been added yet, just give it its fair share.

[stefan-webrtc, 2016/06/30 16:17:38]

Not clear to me why this can happen? Is it correct usage?

[mflodman, 2016/07/01 09:40:15]

AS of now it is, the encoder is configured before the observer is added here and
we need the start bitrate before that happens.

[stefan-webrtc, 2016/07/01 10:42:49]

Acknowledged.

+    return last_non_zero_bitrate_bps_ /
+             static_cast<int>((bitrate_observer_configs_.size() + 1));
+  } else if (it->allocated_bitrate_bps == -1) {
+    // This observer hasn't received an allocation yet, so do the same.
+    return last_non_zero_bitrate_bps_ /
+             static_cast<int>(bitrate_observer_configs_.size());
+  } else {
+    // This observer already has an allocation.
+    return it->allocated_bitrate_bps;
+  }
 }
 
-BitrateAllocator::ObserverConfigList::iterator
+BitrateAllocator::ObserverConfigs::iterator
 BitrateAllocator::FindObserverConfig(
     const BitrateAllocatorObserver* observer) {
   for (auto it = bitrate_observer_configs_.begin();
        it != bitrate_observer_configs_.end(); ++it) {
     if (it->observer == observer)
       return it;
   }
   return bitrate_observer_configs_.end();
 }
 
@@ skipping 49 matching lines @@
     remaining_bitrate -= allocated_bitrate;
   }
 
   // Allocate bitrate to all previously active streams.
   if (remaining_bitrate > 0) {
     for (const auto& observer_config : bitrate_observer_configs_) {
       if (observer_config.enforce_min_bitrate ||
           LastAllocatedBitrate(observer_config) == 0)
         continue;
 
-      if (remaining_bitrate >= observer_config.min_bitrate_bps) {
-        allocation[observer_config.observer] = observer_config.min_bitrate_bps;
-        remaining_bitrate -= observer_config.min_bitrate_bps;
+      uint32_t required_bitrate = MinBitrateWithHysteresis(observer_config);
+      if (remaining_bitrate >= required_bitrate) {
+        allocation[observer_config.observer] = required_bitrate;
+        remaining_bitrate -= required_bitrate;
       }
     }
   }
 
   // Allocate bitrate to previously paused streams.
   if (remaining_bitrate > 0) {
     for (const auto& observer_config : bitrate_observer_configs_) {
       if (LastAllocatedBitrate(observer_config) != 0)
         continue;
 
@@ skipping 38 matching lines @@
     bitrate -= observer_config.max_bitrate_bps;
   }
   DistributeBitrateEvenly(bitrate, true, kTransmissionMaxBitrateMultiplier,
                           &allocation);
   return allocation;
 }
 
 uint32_t BitrateAllocator::LastAllocatedBitrate(
     const ObserverConfig& observer_config) {
 
-  const auto& it = last_allocation_.find(observer_config.observer);
-  if (it != last_allocation_.end())
-    return it->second;
-
   // Return the configured minimum bitrate for newly added observers, to avoid
   // requiring an extra high bitrate for the observer to get an allocated
   // bitrate.
-  return observer_config.min_bitrate_bps;
+  return observer_config.allocated_bitrate_bps == -1 ?
+      observer_config.min_bitrate_bps : observer_config.allocated_bitrate_bps;
 }
 
 uint32_t BitrateAllocator::MinBitrateWithHysteresis(
     const ObserverConfig& observer_config) {
   uint32_t min_bitrate = observer_config.min_bitrate_bps;
   if (LastAllocatedBitrate(observer_config) == 0) {
     min_bitrate += std::max(static_cast<uint32_t>(kToggleFactor * min_bitrate),
                             kMinToggleBitrateBps);
   }
+  // Account for protection bitrate used by this observer in the previous
+  // allocation.
+  // Note: the ratio will only be updated when the stream is active, meaning a
+  // paused stream won't get any ratio updates. This might lead to waiting a bit
+  // longer than necessary if the network condition improves, but this is to
+  // avoid too much toggling.

[stefan-webrtc, 2016/06/30 16:17:38]

I wonder if this should be moved up to where the ratio is updated instead? Not
sure, so feel free to disagree.

That said, I do think this is the right trade-off.

[mflodman, 2016/07/01 09:40:15]

As of now I prefer it this way to have it separate. Let's discuss for the future
though, if this can be simplified and done differently.

[stefan-webrtc, 2016/07/01 10:42:49]

Acknowledged.

+  if (observer_config.media_ratio > 0.0 && observer_config.media_ratio < 1.0)
+    min_bitrate += min_bitrate * (1.0 - observer_config.media_ratio);
+
   return min_bitrate;
 }
 
 void BitrateAllocator::DistributeBitrateEvenly(uint32_t bitrate,
                                                bool include_zero_allocations,
                                                int max_multiplier,
                                                ObserverAllocation* allocation) {
   RTC_DCHECK_EQ(allocation->size(), bitrate_observer_configs_.size());
 
   ObserverSortingMap list_max_bitrates;
@@ skipping 32 matching lines @@
   uint32_t extra_bitrate_per_observer = (bitrate - sum_min_bitrates) /
       static_cast<uint32_t>(bitrate_observer_configs_.size());
   for (const auto& observer_config : bitrate_observer_configs_) {
     if (observer_config.min_bitrate_bps + extra_bitrate_per_observer <
         MinBitrateWithHysteresis(observer_config))
       return false;
   }
   return true;
 }
 }  // namespace webrtc