Add flag enabling more packets to be retransmittable.

webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc

Index: webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc
diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc
index 38450742be9e29022cbabd2245952a88a1b09ddf..b9bc4f687476190cbc6dc8c620e1869f7dd69797 100644
--- a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc
+++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc
@@ -13,9 +13,10 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <limits>
 #include <memory>
-#include <vector>
 #include <utility>
+#include <vector>
 
 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
 #include "webrtc/modules/rtp_rtcp/source/byte_io.h"
@@ -33,6 +34,7 @@ namespace webrtc {
 
 namespace {
 constexpr size_t kRedForFecHeaderLength = 1;
+constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4;
 
 void BuildRedPayload(const RtpPacketToSend& media_packet,
                      RtpPacketToSend* red_packet) {
@@ -53,7 +55,8 @@ RTPSenderVideo::RTPSenderVideo(Clock* clock,
     : rtp_sender_(rtp_sender),
       clock_(clock),
       video_type_(kRtpVideoGeneric),
-      retransmission_settings_(kRetransmitBaseLayer),
+      retransmission_settings_(kRetransmitBaseLayer |
+                               kConditionallyRetransmitHigherLayers),
       last_rotation_(kVideoRotation_0),
       red_payload_type_(-1),
       ulpfec_payload_type_(-1),
@@ -292,7 +295,8 @@ bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type,
                                const uint8_t* payload_data,
                                size_t payload_size,
                                const RTPFragmentationHeader* fragmentation,
-                               const RTPVideoHeader* video_header) {
+                               const RTPVideoHeader* video_header,
+                               int64_t expected_retransmission_time_ms) {
   if (payload_size == 0)
     return false;
 
@@ -365,8 +369,12 @@ bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type,
   std::unique_ptr<RtpPacketizer> packetizer(RtpPacketizer::Create(
       video_type, max_data_payload_length, last_packet_reduction_len,
       video_header ? &(video_header->codecHeader) : nullptr, frame_type));
-  // Media packet storage.
-  StorageType storage = packetizer->GetStorageType(retransmission_settings);
+
+  const uint8_t temporal_id =
+      video_header ? GetTemporalId(*video_header) : kNoTemporalIdx;
+  StorageType storage =
+      GetStorageType(temporal_id, retransmission_settings, packetizer.get(),
+                     expected_retransmission_time_ms);
 
   // TODO(changbin): we currently don't support to configure the codec to
   // output multiple partitions for VP8. Should remove below check after the
@@ -392,7 +400,11 @@ bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type,
     if (!rtp_sender_->AssignSequenceNumber(packet.get()))
       return false;
 
-    bool protect_packet = (packetizer->GetProtectionType() == kProtectedPacket);
+    // No FEC protection for upper temporal layers, if used.
+    const uint8_t temporal_id =

[danilchap, 2017/09/04 09:03:04]

this statement can be removed - temporal_id already calculated above

[brandtr, 2017/09/04 10:29:22]

Reuse |temporal_id| from line 373?

[sprang_webrtc, 2017/09/04 11:09:28]

Done.

[sprang_webrtc, 2017/09/04 11:09:28]

Done.

+        video_header ? GetTemporalId(*video_header) : kNoTemporalIdx;
+    bool protect_packet = temporal_id == 0 || temporal_id == kNoTemporalIdx;
+
     // Put packetization finish timestamp into extension.
     if (packet->HasExtension<VideoTimingExtension>()) {
       packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds());
@@ -453,4 +465,91 @@ void RTPSenderVideo::SetSelectiveRetransmissions(uint8_t settings) {
   retransmission_settings_ = settings;
 }
 
+StorageType RTPSenderVideo::GetStorageType(
+    uint8_t temporal_id,
+    int32_t retransmission_settings,

[danilchap, 2017/09/04 09:03:04]

since retransmission_settings is a bitmask, may be unsigned type better be used.
is it possible to use RetransmissionMode directly?

[sprang_webrtc, 2017/09/04 11:09:28]

Using RetransmissionMode directly forces a lot of explicit casts to be done. It
would also be weird to mask two values together and then cast it to a
RetransmissionMode value that is not declared by the enum.

I could potentially make it unsigned (or uint8_t, which is what
RetransmissionMode is typed as), but am hesitant to change the interface at this
time. I suspect there are downstream dependencies...

+    RtpPacketizer* packetizer,

[danilchap, 2017/09/04 09:03:04]

this parameter is no longer needed

[sprang_webrtc, 2017/09/04 11:09:28]

Done.

+    int64_t expected_retransmission_time) {

[danilchap, 2017/09/04 09:03:04]

add _ms

[sprang_webrtc, 2017/09/04 11:09:28]

Done.

+  if (retransmission_settings == kRetransmitOff)
+    return StorageType::kDontRetransmit;
+  if (retransmission_settings == kRetransmitAllPackets)
+    return StorageType::kAllowRetransmission;
+
+  rtc::CritScope cs(&stats_crit_);
+  // Media packet storage.
+  if ((retransmission_settings & kConditionallyRetransmitHigherLayers) &&
+      (UpdateConditionalRetransmit(temporal_id,

[danilchap, 2017/09/04 09:03:04]

extra () around function call can be removed

[sprang_webrtc, 2017/09/04 11:09:28]

Done.

+                                   expected_retransmission_time))) {
+    retransmission_settings |= kRetransmitHigherLayers;
+  }
+
+  if (temporal_id == kNoTemporalIdx)
+    return kAllowRetransmission;
+
+  if ((retransmission_settings & kRetransmitBaseLayer) && temporal_id == 0)
+    return kAllowRetransmission;
+
+  if ((retransmission_settings & kRetransmitHigherLayers) && temporal_id > 0)
+    return kAllowRetransmission;
+
+  return kDontRetransmit;
+}
+
+uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) const {
+  switch (header.codec) {
+    case kRtpVideoVp8:
+      return header.codecHeader.VP8.temporalIdx;
+    case kRtpVideoVp9:
+      return header.codecHeader.VP9.temporal_idx;
+    default:
+      return kNoTemporalIdx;
+  }
+}
+
+bool RTPSenderVideo::UpdateConditionalRetransmit(
+    uint8_t temporal_id,
+    int64_t expected_retransmission_time_ms) {
+  int64_t now_ms = clock_->TimeInMilliseconds();
+  // Update stats for any temporal layer.
+  TemporalLayerStats* current_layer_stats =
+      &frame_stats_by_temporal_layer_[temporal_id];
+  current_layer_stats->frame_rate_fp1000s.Update(1, now_ms);
+  int64_t tl_frame_interval = now_ms - current_layer_stats->last_frame_time;
+  current_layer_stats->last_frame_time = now_ms;
+
+  // Conditional retransmit only applies to upper layers.
+  if (temporal_id != kNoTemporalIdx && temporal_id > 0) {
+    if (tl_frame_interval >= kMaxUnretransmittableFrameIntervalMs) {
+      // Too long since a retransmittable frame in this layer, enable NACK
+      // protection.
+      return true;
+    } else {
+      // Estimate when the next frame of any lower layer will be sent.
+      const int64_t kUndefined = std::numeric_limits<int64_t>::max();
+      int64_t expected_next_frame_time = kUndefined;
+      for (int i = temporal_id - 1; i >= 0; --i) {
+        TemporalLayerStats* stats = &frame_stats_by_temporal_layer_[i];
+        rtc::Optional<uint32_t> rate = stats->frame_rate_fp1000s.Rate(now_ms);
+        if (rate) {
+          int64_t tl_next = stats->last_frame_time + 1000000 / *rate;
+          if (tl_next - now_ms > -expected_retransmission_time_ms &&
+              tl_next < expected_next_frame_time) {
+            expected_next_frame_time = tl_next;
+          }
+        }
+      }
+
+      if (expected_next_frame_time == kUndefined ||
+          expected_next_frame_time - now_ms > expected_retransmission_time_ms) {
+        // The next frame in a lower layer is expected at a later time (or
+        // unable to tell due to lack of data) than a retransmission is
+        // estimated to be able to arrive, so allow this packet to be nacked.
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
 }  // namespace webrtc