Frame dropper improvements & cleanup

webrtc/modules/video_coding/utility/frame_dropper.cc

 /*
  *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/modules/video_coding/utility/frame_dropper.h"
 
+#include <algorithm>
+
+#include "webrtc/base/logging.h"
 #include "webrtc/system_wrappers/include/trace.h"
 
 namespace webrtc {
 
-const float kDefaultKeyFrameSizeAvgKBits = 0.9f;
-const float kDefaultKeyFrameRatio = 0.99f;
+namespace {
+
+const float kDefaultFrameSizeAlpha = 0.9f;
+const float kDefaultKeyFrameRatioAlpha = 0.99f;
+// 1 key frame every 10th second in 30 fps.
+const float kDefaultKeyFrameRatioValue = 1 / 300.0f;
+
 const float kDefaultDropRatioAlpha = 0.9f;
-const float kDefaultDropRatioMax = 0.96f;
-const float kDefaultMaxTimeToDropFrames = 4.0f;  // In seconds.
+const float kDefaultDropRatioValue = 0.96f;
+// Maximum duration over which frames are continuously dropped.
+const float kDefaultMaxDropDurationSecs = 4.0f;
+
+// Default target bitrate.
+// TODO(isheriff): Should this be higher to avoid dropping too many packets when
+// the bandwidth is unknown at the start ?
+const float kDefaultTargetBitrateKbps = 300.0f;
+const float kDefaultIncomingFrameRate = 30;
+const float kLeakyBucketSizeSeconds = 0.5f;
+
+// Cap on the frame size accumulator to prevent excessive drops.
+const float kAccumulatorCapBufferSizeSecs = 3.0f;
+}  // namespace
 
 FrameDropper::FrameDropper()
-    : _keyFrameSizeAvgKbits(kDefaultKeyFrameSizeAvgKBits),
-      _keyFrameRatio(kDefaultKeyFrameRatio),
-      _dropRatio(kDefaultDropRatioAlpha, kDefaultDropRatioMax),
-      _enabled(true),
-      _max_time_drops(kDefaultMaxTimeToDropFrames) {
+    : key_frame_size_avg_kbits_(kDefaultFrameSizeAlpha),
+      key_frame_ratio_(kDefaultKeyFrameRatioAlpha),
+      delta_frame_size_avg_kbits_(kDefaultFrameSizeAlpha),
+      drop_ratio_(kDefaultDropRatioAlpha, kDefaultDropRatioValue),
+      enabled_(true),
+      max_drop_duration_secs_(kDefaultMaxDropDurationSecs) {
   Reset();
 }
 
-FrameDropper::FrameDropper(float max_time_drops)
-    : _keyFrameSizeAvgKbits(kDefaultKeyFrameSizeAvgKBits),
-      _keyFrameRatio(kDefaultKeyFrameRatio),
-      _dropRatio(kDefaultDropRatioAlpha, kDefaultDropRatioMax),
-      _enabled(true),
-      _max_time_drops(max_time_drops) {
+FrameDropper::FrameDropper(float max_drop_duration_secs)
+    : key_frame_size_avg_kbits_(kDefaultFrameSizeAlpha),
+      key_frame_ratio_(kDefaultKeyFrameRatioAlpha),
+      delta_frame_size_avg_kbits_(kDefaultFrameSizeAlpha),
+      drop_ratio_(kDefaultDropRatioAlpha, kDefaultDropRatioValue),
+      enabled_(true),
+      max_drop_duration_secs_(max_drop_duration_secs) {
   Reset();
 }
 
 void FrameDropper::Reset() {
-  _keyFrameRatio.Reset(0.99f);
-  _keyFrameRatio.Apply(
-      1.0f, 1.0f / 300.0f);  // 1 key frame every 10th second in 30 fps
-  _keyFrameSizeAvgKbits.Reset(0.9f);
-  _keyFrameCount = 0;
-  _accumulator = 0.0f;
-  _accumulatorMax = 150.0f;  // assume 300 kb/s and 0.5 s window
-  _targetBitRate = 300.0f;
-  _incoming_frame_rate = 30;
-  _keyFrameSpreadFrames = 0.5f * _incoming_frame_rate;
-  _dropNext = false;
-  _dropRatio.Reset(0.9f);
-  _dropRatio.Apply(0.0f, 0.0f);  // Initialize to 0
-  _dropCount = 0;
-  _windowSize = 0.5f;
-  _wasBelowMax = true;
-  _fastMode = false;  // start with normal (non-aggressive) mode
-  // Cap for the encoder buffer level/accumulator, in secs.
-  _cap_buffer_size = 3.0f;
-  // Cap on maximum amount of dropped frames between kept frames, in secs.
-  _max_time_drops = 4.0f;
+  key_frame_ratio_.Reset(kDefaultKeyFrameRatioAlpha);
+  key_frame_ratio_.Apply(1.0f, kDefaultKeyFrameRatioValue);
+  key_frame_size_avg_kbits_.Reset(kDefaultFrameSizeAlpha);
+  delta_frame_size_avg_kbits_.Reset(kDefaultFrameSizeAlpha);
+
+  accumulator_ = 0.0f;
+  accumulator_max_ = kDefaultTargetBitrateKbps / 2;
+  target_bitrate_ = kDefaultTargetBitrateKbps;
+  incoming_frame_rate_ = kDefaultIncomingFrameRate;
+
+  large_frame_accumulation_count_ = 0;
+  large_frame_accumulation_spread_ = 0.5 * kDefaultIncomingFrameRate;
+
+  drop_next_ = false;
+  drop_ratio_.Reset(0.9f);
+  drop_ratio_.Apply(0.0f, 0.0f);
+  drop_count_ = 0;
+  was_below_max_ = true;
 }
 
 void FrameDropper::Enable(bool enable) {
-  _enabled = enable;
-}
-
-void FrameDropper::Fill(size_t frameSizeBytes, bool deltaFrame) {
-  if (!_enabled) {
-    return;
-  }
-  float frameSizeKbits = 8.0f * static_cast<float>(frameSizeBytes) / 1000.0f;
-  if (!deltaFrame &&
-      !_fastMode) {  // fast mode does not treat key-frames any different
-    _keyFrameSizeAvgKbits.Apply(1, frameSizeKbits);
-    _keyFrameRatio.Apply(1.0, 1.0);
-    if (frameSizeKbits > _keyFrameSizeAvgKbits.filtered()) {
+  enabled_ = enable;
+}
+
+void FrameDropper::Fill(size_t framesize_bytes, bool delta_frame) {
+  if (!enabled_) {
+    return;
+  }
+  float framesize_kbits = 8.0f * static_cast<float>(framesize_bytes) / 1000.0f;
+  if (!delta_frame) {
+    key_frame_size_avg_kbits_.Apply(1, framesize_kbits);
+    key_frame_ratio_.Apply(1.0, 1.0);
+    if (framesize_kbits > key_frame_size_avg_kbits_.filtered()) {
       // Remove the average key frame size since we
       // compensate for key frames when adding delta
       // frames.
-      frameSizeKbits -= _keyFrameSizeAvgKbits.filtered();
+      framesize_kbits -= key_frame_size_avg_kbits_.filtered();
     } else {
       // Shouldn't be negative, so zero is the lower bound.
-      frameSizeKbits = 0;
-    }
-    if (_keyFrameRatio.filtered() > 1e-5 &&
-        1 / _keyFrameRatio.filtered() < _keyFrameSpreadFrames) {
-      // We are sending key frames more often than our upper bound for
-      // how much we allow the key frame compensation to be spread
-      // out in time. Therefor we must use the key frame ratio rather
-      // than keyFrameSpreadFrames.
-      _keyFrameCount =
-          static_cast<int32_t>(1 / _keyFrameRatio.filtered() + 0.5);
-    } else {
-      // Compensate for the key frame the following frames
-      _keyFrameCount = static_cast<int32_t>(_keyFrameSpreadFrames + 0.5);
-    }
+      framesize_kbits = 0;
+    }
+    if (key_frame_ratio_.filtered() > 1e-5 &&
+        1 / key_frame_ratio_.filtered() < large_frame_accumulation_spread_) {
+      large_frame_accumulation_count_ =
+          static_cast<int32_t>(1 / key_frame_ratio_.filtered() + 0.5);
+    } else {
+      large_frame_accumulation_count_ =
+          static_cast<int32_t>(large_frame_accumulation_spread_ + 0.5);
+    }
+    large_frame_accumulation_chunk_size_ =
+        key_frame_size_avg_kbits_.filtered() / large_frame_accumulation_count_;
   } else {
-    // Decrease the keyFrameRatio
-    _keyFrameRatio.Apply(1.0, 0.0);
+    // Identify if it is a unusually large delta frame and spread accumulation
+    // if that is the case.
+    if (delta_frame_size_avg_kbits_.filtered() != -1 &&
+        framesize_kbits > 3 * delta_frame_size_avg_kbits_.filtered() &&

[sprang_webrtc, 2016/03/03 11:50:26]

Named constant for 3.

+        large_frame_accumulation_count_ == 0) {
+      large_frame_accumulation_count_ =
+          static_cast<int32_t>(large_frame_accumulation_spread_ + 0.5);
+      large_frame_accumulation_chunk_size_ =
+          framesize_kbits / large_frame_accumulation_count_;
+      framesize_kbits = 0;
+    } else {
+      delta_frame_size_avg_kbits_.Apply(1, framesize_kbits);
+      key_frame_ratio_.Apply(1.0, 0.0);
+    }
   }
   // Change the level of the accumulator (bucket)
-  _accumulator += frameSizeKbits;
+  accumulator_ += framesize_kbits;
   CapAccumulator();
-}
-
-void FrameDropper::Leak(uint32_t inputFrameRate) {
-  if (!_enabled) {
-    return;
-  }
-  if (inputFrameRate < 1) {
-    return;
-  }
-  if (_targetBitRate < 0.0f) {
-    return;
-  }
-  _keyFrameSpreadFrames = 0.5f * inputFrameRate;
-  // T is the expected bits per frame (target). If all frames were the same
-  // size,
-  // we would get T bits per frame. Notice that T is also weighted to be able to
-  // force a lower frame rate if wanted.
-  float T = _targetBitRate / inputFrameRate;
-  if (_keyFrameCount > 0) {
-    // Perform the key frame compensation
-    if (_keyFrameRatio.filtered() > 0 &&
-        1 / _keyFrameRatio.filtered() < _keyFrameSpreadFrames) {
-      T -= _keyFrameSizeAvgKbits.filtered() * _keyFrameRatio.filtered();
-    } else {
-      T -= _keyFrameSizeAvgKbits.filtered() / _keyFrameSpreadFrames;
-    }
-    _keyFrameCount--;
-  }
-  _accumulator -= T;
-  if (_accumulator < 0.0f) {
-    _accumulator = 0.0f;
-  }
+  LOG(LS_VERBOSE) << "FILL acc " << accumulator_ << " max " << accumulator_max_
+                  << " count " << large_frame_accumulation_count_ << " chunk "
+                  << large_frame_accumulation_chunk_size_ << " spread "
+                  << large_frame_accumulation_spread_ << " key avg "
+                  << key_frame_size_avg_kbits_.filtered() << " delta avg "
+                  << delta_frame_size_avg_kbits_.filtered() << " SIZE "
+                  << framesize_kbits;
+}
+
+void FrameDropper::Leak(uint32_t input_framerate) {
+  if (!enabled_) {
+    return;
+  }
+  if (input_framerate < 1) {
+    return;
+  }
+  if (target_bitrate_ < 0.0f) {
+    return;
+  }
+  // Add lower bound for large frame accumulation spread.
+  large_frame_accumulation_spread_ = std::max(0.5 * input_framerate, 5.0);
+  // Expected bits per frame based on current input frame rate.
+  float expected_bits_per_frame = target_bitrate_ / input_framerate;
+  if (large_frame_accumulation_count_ > 0) {
+    expected_bits_per_frame -= large_frame_accumulation_chunk_size_;
+    large_frame_accumulation_count_--;

[stefan-webrtc, 2016/03/01 15:14:34]

--large_frame...;

[Irfan, 2016/03/04 07:58:49]

Done.

+  }
+  accumulator_ -= expected_bits_per_frame;
+  if (accumulator_ < 0.0f) {
+    accumulator_ = 0.0f;
+  }
+  LOG(LS_VERBOSE) << " LEAK acc " << accumulator_ << " max " << accumulator_max_

[stefan-webrtc, 2016/03/01 15:14:34]

Remove space before LEAK

[Irfan, 2016/03/04 07:58:49]

Done.

+                  << " count " << large_frame_accumulation_count_ << " spread "
+                  << large_frame_accumulation_spread_ << " key avg "
+                  << key_frame_size_avg_kbits_.filtered() << " delta avg "
+                  << delta_frame_size_avg_kbits_.filtered();
   UpdateRatio();
 }
 
-void FrameDropper::UpdateNack(uint32_t nackBytes) {
-  if (!_enabled) {
-    return;
-  }
-  _accumulator += static_cast<float>(nackBytes) * 8.0f / 1000.0f;
-}
-
-void FrameDropper::FillBucket(float inKbits, float outKbits) {
-  _accumulator += (inKbits - outKbits);
-}
-
 void FrameDropper::UpdateRatio() {
-  if (_accumulator > 1.3f * _accumulatorMax) {
+  if (accumulator_ > 1.3f * accumulator_max_) {
     // Too far above accumulator max, react faster
-    _dropRatio.UpdateBase(0.8f);
+    drop_ratio_.UpdateBase(0.8f);
   } else {
     // Go back to normal reaction
-    _dropRatio.UpdateBase(0.9f);
-  }
-  if (_accumulator > _accumulatorMax) {
+    drop_ratio_.UpdateBase(0.9f);
+  }
+  if (accumulator_ > accumulator_max_) {
     // We are above accumulator max, and should ideally
     // drop a frame. Increase the dropRatio and drop
     // the frame later.
-    if (_wasBelowMax) {
-      _dropNext = true;
-    }
-    if (_fastMode) {
-      // always drop in aggressive mode
-      _dropNext = true;
-    }
-
-    _dropRatio.Apply(1.0f, 1.0f);
-    _dropRatio.UpdateBase(0.9f);
+    if (was_below_max_) {
+      drop_next_ = true;
+    }
+    drop_ratio_.Apply(1.0f, 1.0f);
+    drop_ratio_.UpdateBase(0.9f);
   } else {
-    _dropRatio.Apply(1.0f, 0.0f);
-  }
-  _wasBelowMax = _accumulator < _accumulatorMax;
+    drop_ratio_.Apply(1.0f, 0.0f);
+  }
+  was_below_max_ = accumulator_ < accumulator_max_;
 }
 
 // This function signals when to drop frames to the caller. It makes use of the
 // dropRatio
 // to smooth out the drops over time.
 bool FrameDropper::DropFrame() {
-  if (!_enabled) {
+  if (!enabled_) {
     return false;
   }
-  if (_dropNext) {
-    _dropNext = false;
-    _dropCount = 0;
-  }
-
-  if (_dropRatio.filtered() >= 0.5f) {  // Drops per keep
+  if (drop_next_) {
+    drop_next_ = false;
+    drop_count_ = 0;
+  }
+  LOG(LS_VERBOSE) << " drop_ratio_ " << drop_ratio_.filtered()
+                  << " drop_count_ " << drop_count_;
+
+  if (drop_ratio_.filtered() >= 0.5f) {  // Drops per keep
     // limit is the number of frames we should drop between each kept frame
     // to keep our drop ratio. limit is positive in this case.
-    float denom = 1.0f - _dropRatio.filtered();
+    float denom = 1.0f - drop_ratio_.filtered();
     if (denom < 1e-5) {
       denom = 1e-5f;
     }
     int32_t limit = static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f);
     // Put a bound on the max amount of dropped frames between each kept
     // frame, in terms of frame rate and window size (secs).
-    int max_limit = static_cast<int>(_incoming_frame_rate * _max_time_drops);
+    int max_limit =
+        static_cast<int>(incoming_frame_rate_ * max_drop_duration_secs_);
     if (limit > max_limit) {
       limit = max_limit;
     }
-    if (_dropCount < 0) {
-      // Reset the _dropCount since it was negative and should be positive.
-      if (_dropRatio.filtered() > 0.4f) {
-        _dropCount = -_dropCount;
-      } else {
-        _dropCount = 0;
-      }
-    }
-    if (_dropCount < limit) {
+    if (drop_count_ < 0) {
+      // Reset the drop_count_ since it was negative and should be positive.
+      drop_count_ = -drop_count_;
+    }
+    if (drop_count_ < limit) {
       // As long we are below the limit we should drop frames.
-      _dropCount++;
+      drop_count_++;
       return true;
     } else {
-      // Only when we reset _dropCount a frame should be kept.
-      _dropCount = 0;
+      // Only when we reset drop_count_ a frame should be kept.
+      drop_count_ = 0;
       return false;
     }
-  } else if (_dropRatio.filtered() > 0.0f &&
-             _dropRatio.filtered() < 0.5f) {  // Keeps per drop
+  } else if (drop_ratio_.filtered() > 0.0f &&
+             drop_ratio_.filtered() < 0.5f) {  // Keeps per drop
     // limit is the number of frames we should keep between each drop
     // in order to keep the drop ratio. limit is negative in this case,
-    // and the _dropCount is also negative.
-    float denom = _dropRatio.filtered();
+    // and the drop_count_ is also negative.
+    float denom = drop_ratio_.filtered();
     if (denom < 1e-5) {
       denom = 1e-5f;
     }
     int32_t limit = -static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f);
-    if (_dropCount > 0) {
-      // Reset the _dropCount since we have a positive
-      // _dropCount, and it should be negative.
-      if (_dropRatio.filtered() < 0.6f) {
-        _dropCount = -_dropCount;
-      } else {
-        _dropCount = 0;
-      }
-    }
-    if (_dropCount > limit) {
-      if (_dropCount == 0) {
-        // Drop frames when we reset _dropCount.
-        _dropCount--;
+    if (drop_count_ > 0) {
+      // Reset the drop_count_ since we have a positive
+      // drop_count_, and it should be negative.
+      drop_count_ = -drop_count_;
+    }
+    if (drop_count_ > limit) {
+      if (drop_count_ == 0) {
+        // Drop frames when we reset drop_count_.
+        drop_count_--;
         return true;
       } else {
         // Keep frames as long as we haven't reached limit.
-        _dropCount--;
+        drop_count_--;
         return false;
       }
     } else {
-      _dropCount = 0;
+      drop_count_ = 0;
       return false;
     }
   }
-  _dropCount = 0;
+  drop_count_ = 0;
   return false;
-
-  // A simpler version, unfiltered and quicker
-  // bool dropNext = _dropNext;
-  // _dropNext = false;
-  // return dropNext;
-}
-
-void FrameDropper::SetRates(float bitRate, float incoming_frame_rate) {
+}
+
+void FrameDropper::SetRates(float bitrate, float incoming_frame_rate) {
   // Bit rate of -1 means infinite bandwidth.
-  _accumulatorMax = bitRate * _windowSize;  // bitRate * windowSize (in seconds)
-  if (_targetBitRate > 0.0f && bitRate < _targetBitRate &&
-      _accumulator > _accumulatorMax) {
+  accumulator_max_ = bitrate * kLeakyBucketSizeSeconds;
+  if (target_bitrate_ > 0.0f && bitrate < target_bitrate_ &&
+      accumulator_ > accumulator_max_) {
     // Rescale the accumulator level if the accumulator max decreases
-    _accumulator = bitRate / _targetBitRate * _accumulator;
-  }
-  _targetBitRate = bitRate;
+    accumulator_ = bitrate / target_bitrate_ * accumulator_;
+  }
+  target_bitrate_ = bitrate;
   CapAccumulator();
-  _incoming_frame_rate = incoming_frame_rate;
-}
-
-float FrameDropper::ActualFrameRate(uint32_t inputFrameRate) const {
-  if (!_enabled) {
-    return static_cast<float>(inputFrameRate);
-  }
-  return inputFrameRate * (1.0f - _dropRatio.filtered());
+  incoming_frame_rate_ = incoming_frame_rate;
 }
 
 // Put a cap on the accumulator, i.e., don't let it grow beyond some level.
 // This is a temporary fix for screencasting where very large frames from
 // encoder will cause very slow response (too many frame drops).
+// TODO(isheriff): Remove this now that large delta frames are also spread out ?
 void FrameDropper::CapAccumulator() {
-  float max_accumulator = _targetBitRate * _cap_buffer_size;
-  if (_accumulator > max_accumulator) {
-    _accumulator = max_accumulator;
+  float max_accumulator = target_bitrate_ * kAccumulatorCapBufferSizeSecs;
+  if (accumulator_ > max_accumulator) {
+    accumulator_ = max_accumulator;
   }
 }
 }  // namespace webrtc