Move logic for calculating needed bitrate overhead used by NACK and FEC to VideoSender.

webrtc/modules/video_coding/protection_bitrate_calculator.cc

+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <webrtc/modules/video_coding/protection_bitrate_calculator.h>
+
+namespace webrtc {
+
+using rtc::CritScope;
+
+struct ProtectionBitrateCalculator::EncodedFrameSample {
+  EncodedFrameSample(size_t size_bytes,
+                     uint32_t timestamp,
+                     int64_t time_complete_ms)
+      : size_bytes(size_bytes),
+        timestamp(timestamp),
+        time_complete_ms(time_complete_ms) {}
+  size_t size_bytes;
+  uint32_t timestamp;
+  int64_t time_complete_ms;
+};
+
+ProtectionBitrateCalculator::ProtectionBitrateCalculator(
+    Clock* clock,
+    VCMProtectionCallback* protection_callback)
+    : clock_(clock),
+      protection_callback_(protection_callback),
+      loss_prot_logic_(new media_optimization::VCMLossProtectionLogic(
+          clock_->TimeInMilliseconds())),
+      max_payload_size_(1460),
+      encoded_frame_samples_(),
+      avg_sent_bit_rate_bps_(0),
+      avg_sent_framerate_(0) {}
+
+ProtectionBitrateCalculator::~ProtectionBitrateCalculator(void) {
+  loss_prot_logic_->Release();
+}
+
+void ProtectionBitrateCalculator::SetEncodingData(uint32_t target_bitrate,
+                                                  uint16_t width,
+                                                  uint16_t height,
+                                                  uint32_t frame_rate,
+                                                  size_t num_layers,
+                                                  size_t mtu) {

[stefan-webrtc, 2016/05/19 12:52:39]

Call this max_payload_size instead.

[perkj_webrtc, 2016/06/01 20:55:23]

Done.

+  CritScope lock(&crit_sect_);
+  // Everything codec specific should be reset here since this means the codec
+  // has changed.
+  float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
+  loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
+  loss_prot_logic_->UpdateFrameRate(static_cast<float>(frame_rate));
+  loss_prot_logic_->UpdateFrameSize(width, height);
+  loss_prot_logic_->UpdateNumLayers(num_layers);
+  max_payload_size_ = mtu;
+}
+
+uint32_t ProtectionBitrateCalculator::SetTargetRates(
+    uint32_t target_bitrate,

[stefan-webrtc, 2016/05/19 12:52:39]

add unit

[perkj_webrtc, 2016/06/01 20:55:23]

Done.

+    uint8_t fraction_lost,
+    int64_t round_trip_time_ms) {
+  crit_sect_.Enter();

[stefan-webrtc, 2016/05/19 12:52:39]

Change to use a scoped crit

[perkj_webrtc, 2016/06/01 20:55:23]

ok- but I then have to move a few things around the scope so that I don't hold
the lock while calling protection_callback_->ProtectionRequest.

+
+  float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
+  loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
+  loss_prot_logic_->UpdateRtt(round_trip_time_ms);
+
+  // Get frame rate for encoder: this is the actual/sent frame rate.
+  float actual_frame_rate = SentFrameRateInternal();
+
+  // Sanity check.
+  if (actual_frame_rate < 1.0) {
+    actual_frame_rate = 1.0;
+  }
+
+  // Update frame rate for the loss protection logic class: frame rate should
+  // be the actual/sent rate.
+  loss_prot_logic_->UpdateFrameRate(actual_frame_rate);
+
+  // Returns the filtered packet loss, used for the protection setting.
+  // The filtered loss may be the received loss (no filter), or some
+  // filtered value (average or max window filter).
+  // Use max window filter for now.
+  media_optimization::FilterPacketLossMode filter_mode =
+      media_optimization::kMaxFilter;
+  uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss(
+      clock_->TimeInMilliseconds(), filter_mode, fraction_lost);
+
+  // For now use the filtered loss for computing the robustness settings.
+  loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc);
+
+  // Rate cost of the protection methods.
+  float protection_overhead_rate = 0.0f;
+
+  if (loss_prot_logic_->SelectedType() == media_optimization::kNone) {
+    crit_sect_.Leave();
+    return target_bitrate;
+  }
+
+  // Update method will compute the robustness settings for the given
+  // protection method and the overhead cost
+  // the protection method is set by the user via SetVideoProtection.
+  loss_prot_logic_->UpdateMethod();
+
+  // Update protection callback with protection settings.
+  uint32_t sent_video_rate_bps = 0;
+  uint32_t sent_nack_rate_bps = 0;
+  uint32_t sent_fec_rate_bps = 0;
+
+  // Get the bit cost of protection method, based on the amount of
+  // overhead data actually transmitted (including headers) the last
+  // second.
+  FecProtectionParams delta_fec_params;
+  FecProtectionParams key_fec_params;
+  // Get the FEC code rate for Key frames (set to 0 when NA).
+  key_fec_params.fec_rate =
+      loss_prot_logic_->SelectedMethod()->RequiredProtectionFactorK();
+
+  // Get the FEC code rate for Delta frames (set to 0 when NA).
+  delta_fec_params.fec_rate =
+      loss_prot_logic_->SelectedMethod()->RequiredProtectionFactorD();
+
+  // The RTP module currently requires the same |max_fec_frames| for both
+  // key and delta frames.
+  delta_fec_params.max_fec_frames =
+      loss_prot_logic_->SelectedMethod()->MaxFramesFec();
+  key_fec_params.max_fec_frames =
+      loss_prot_logic_->SelectedMethod()->MaxFramesFec();
+
+  // Set the FEC packet mask type. |kFecMaskBursty| is more effective for
+  // consecutive losses and little/no packet re-ordering. As we currently
+  // do not have feedback data on the degree of correlated losses and packet
+  // re-ordering, we keep default setting to |kFecMaskRandom| for now.
+  delta_fec_params.fec_mask_type = kFecMaskRandom;
+  key_fec_params.fec_mask_type = kFecMaskRandom;
+
+  crit_sect_.Leave();
+
+  // TODO(Marco): Pass FEC protection values per layer.
+  protection_callback_->ProtectionRequest(
+      &delta_fec_params, &key_fec_params, &sent_video_rate_bps,
+      &sent_nack_rate_bps, &sent_fec_rate_bps);
+
+  uint32_t sent_total_rate_bps =
+      sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps;
+  // Estimate the overhead costs of the next second as staying the same
+  // wrt the source bitrate.
+  if (sent_total_rate_bps > 0) {
+    protection_overhead_rate =
+        static_cast<float>(sent_nack_rate_bps + sent_fec_rate_bps) /
+        sent_total_rate_bps;
+  }
+  // Cap the overhead estimate to 50%.
+  if (protection_overhead_rate > 0.5)
+    protection_overhead_rate = 0.5;
+
+  // Source coding rate: total rate - protection overhead.
+  return target_bitrate * (1.0 - protection_overhead_rate);
+}
+
+void ProtectionBitrateCalculator::SetProtectionMethod(bool enable_fec,
+                                                      bool enable_nack) {
+  media_optimization::VCMProtectionMethodEnum method(media_optimization::kNone);
+  if (enable_fec && enable_nack) {
+    method = media_optimization::kNackFec;
+  } else if (enable_nack) {
+    method = media_optimization::kNack;
+  } else if (enable_fec) {
+    method = media_optimization::kFec;
+  }
+  CritScope lock(&crit_sect_);
+  loss_prot_logic_->SetMethod(method);
+}
+
+uint32_t ProtectionBitrateCalculator::SentFrameRate() {
+  CritScope lock(&crit_sect_);
+  return SentFrameRateInternal();
+}
+
+uint32_t ProtectionBitrateCalculator::SentFrameRateInternal() {
+  PurgeOldFrameSamples(clock_->TimeInMilliseconds());
+  UpdateSentFramerate();
+  return avg_sent_framerate_;
+}
+
+uint32_t ProtectionBitrateCalculator::SentBitRate() {
+  CritScope lock(&crit_sect_);
+  const int64_t now_ms = clock_->TimeInMilliseconds();
+  PurgeOldFrameSamples(now_ms);
+  UpdateSentBitrate(now_ms);
+  return avg_sent_bit_rate_bps_;
+}
+
+void ProtectionBitrateCalculator::UpdateWithEncodedData(
+    const EncodedImage& encoded_image) {
+  size_t encoded_length = encoded_image._length;
+  uint32_t timestamp = encoded_image._timeStamp;
+  CritScope lock(&crit_sect_);
+  const int64_t now_ms = clock_->TimeInMilliseconds();
+  PurgeOldFrameSamples(now_ms);
+  if (encoded_frame_samples_.size() > 0 &&
+      encoded_frame_samples_.back().timestamp == timestamp) {
+    // Frames having the same timestamp are generated from the same input
+    // frame. We don't want to double count them, but only increment the
+    // size_bytes.
+    encoded_frame_samples_.back().size_bytes += encoded_length;
+    encoded_frame_samples_.back().time_complete_ms = now_ms;
+  } else {
+    encoded_frame_samples_.push_back(
+        EncodedFrameSample(encoded_length, timestamp, now_ms));
+  }
+  UpdateSentBitrate(now_ms);
+  UpdateSentFramerate();
+  if (encoded_length > 0) {
+    const bool delta_frame = encoded_image._frameType != kVideoFrameKey;
+
+    if (max_payload_size_ > 0 && encoded_length > 0) {
+      const float min_packets_per_frame =
+          encoded_length / static_cast<float>(max_payload_size_);
+      if (delta_frame) {
+        loss_prot_logic_->UpdatePacketsPerFrame(min_packets_per_frame,
+                                                clock_->TimeInMilliseconds());
+      } else {
+        loss_prot_logic_->UpdatePacketsPerFrameKey(
+            min_packets_per_frame, clock_->TimeInMilliseconds());
+      }
+    }
+    if (!delta_frame && encoded_length > 0) {
+      loss_prot_logic_->UpdateKeyFrameSize(static_cast<float>(encoded_length));
+    }
+  }
+}
+
+void ProtectionBitrateCalculator::PurgeOldFrameSamples(int64_t now_ms) {
+  while (!encoded_frame_samples_.empty()) {
+    if (now_ms - encoded_frame_samples_.front().time_complete_ms >
+        kBitrateAverageWinMs) {
+      encoded_frame_samples_.pop_front();
+    } else {
+      break;
+    }
+  }
+}
+
+void ProtectionBitrateCalculator::UpdateSentBitrate(int64_t now_ms) {
+  if (encoded_frame_samples_.empty()) {
+    avg_sent_bit_rate_bps_ = 0;
+    return;
+  }
+  size_t framesize_sum = 0;
+  for (const auto& encoded_sample : encoded_frame_samples_) {
+    framesize_sum += encoded_sample.size_bytes;
+  }
+  float denom = static_cast<float>(
+      now_ms - encoded_frame_samples_.front().time_complete_ms);
+  if (denom >= 1.0f) {
+    avg_sent_bit_rate_bps_ =
+        static_cast<uint32_t>(framesize_sum * 8.0f * 1000.0f / denom + 0.5f);
+  } else {
+    avg_sent_bit_rate_bps_ = framesize_sum * 8;
+  }
+}
+
+void ProtectionBitrateCalculator::UpdateSentFramerate() {
+  if (encoded_frame_samples_.size() <= 1) {
+    avg_sent_framerate_ = encoded_frame_samples_.size();
+    return;
+  }
+  int denom = encoded_frame_samples_.back().timestamp -
+              encoded_frame_samples_.front().timestamp;
+  if (denom > 0) {
+    avg_sent_framerate_ =
+        (90000 * (encoded_frame_samples_.size() - 1) + denom / 2) / denom;
+  } else {
+    avg_sent_framerate_ = encoded_frame_samples_.size();
+  }
+}
+
+}  // namespace webrtc