Add flag enabling more packets to be retransmittable.

webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc

Index: webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc
diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc
index 38450742be9e29022cbabd2245952a88a1b09ddf..da88c9c1c3c35ffcbb620bf55c4ea5e230bcfb65 100644
--- a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc
+++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc
@@ -13,9 +13,10 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include <limits>
 #include <memory>
-#include <vector>
 #include <utility>
+#include <vector>
 
 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
 #include "webrtc/modules/rtp_rtcp/source/byte_io.h"
@@ -33,6 +34,7 @@ namespace webrtc {
 
 namespace {
 constexpr size_t kRedForFecHeaderLength = 1;
+constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4;
 
 void BuildRedPayload(const RtpPacketToSend& media_packet,
                      RtpPacketToSend* red_packet) {
@@ -53,7 +55,8 @@ RTPSenderVideo::RTPSenderVideo(Clock* clock,
     : rtp_sender_(rtp_sender),
       clock_(clock),
       video_type_(kRtpVideoGeneric),
-      retransmission_settings_(kRetransmitBaseLayer),
+      retransmission_settings_(kRetransmitBaseLayer |
+                               kConditionallyRetransmitHigherLayers),
       last_rotation_(kVideoRotation_0),
       red_payload_type_(-1),
       ulpfec_payload_type_(-1),
@@ -292,7 +295,8 @@ bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type,
                                const uint8_t* payload_data,
                                size_t payload_size,
                                const RTPFragmentationHeader* fragmentation,
-                               const RTPVideoHeader* video_header) {
+                               const RTPVideoHeader* video_header,
+                               int64_t expected_retransmission_time_ms) {
   if (payload_size == 0)
     return false;
 
@@ -365,8 +369,10 @@ bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type,
   std::unique_ptr<RtpPacketizer> packetizer(RtpPacketizer::Create(
       video_type, max_data_payload_length, last_packet_reduction_len,
       video_header ? &(video_header->codecHeader) : nullptr, frame_type));
-  // Media packet storage.
-  StorageType storage = packetizer->GetStorageType(retransmission_settings);
+
+  StorageType storage =
+      GetStorageType(*video_header, retransmission_settings, packetizer.get(),
+                     expected_retransmission_time_ms);
 
   // TODO(changbin): we currently don't support to configure the codec to
   // output multiple partitions for VP8. Should remove below check after the
@@ -453,4 +459,92 @@ void RTPSenderVideo::SetSelectiveRetransmissions(uint8_t settings) {
   retransmission_settings_ = settings;
 }
 
+RTPSenderVideo::TemporalLayerStats* RTPSenderVideo::GetTemporalLayerStats(

[danilchap, 2017/08/29 17:31:20]

since return type can't be nullptr, may return reference:
 RTPSenderVideo::TemporalLayerStats& RTPSenderVideo::GetTemporalLayerStats(int
temporal_id) {
  return frame_stats_by_temporal_layer_.emplace(temporal_id,
kWindowSizeMs).first->second;
  // If you'll add default constructor to TemporalLayerStats, even easier:
  return frame_stats_by_temporal_layer_[temporal_id];
}

[sprang_webrtc, 2017/08/31 15:54:29]

Are we ok with non-const references then?
Changed to default constructor and made the window size referencable from test.

[danilchap, 2017/09/04 09:03:04]

the rule states "All parameters passed by reference must be labeled const."

[sprang_webrtc, 2017/09/04 11:09:28]

Acknowledged.

+    int temporal_id) {
+  auto it = frame_stats_by_temporal_layer_.find(temporal_id);
+  if (it == frame_stats_by_temporal_layer_.end()) {
+    const int kWindowSizeMs = 2500;
+    it = frame_stats_by_temporal_layer_.emplace(temporal_id, kWindowSizeMs)

[danilchap, 2017/08/29 17:31:20]

you do not need to search before emplace, so can remove double lookup:
return &frame_stats_by_temporal_layer_.emplace(temporal_id,
kWindowSizeMs).first->second;

[sprang_webrtc, 2017/08/31 15:54:29]

Acknowledged.

+             .first;
+  }
+  return &it->second;
+}
+
+StorageType RTPSenderVideo::GetStorageType(

[danilchap, 2017/08/30 13:52:29]

Looks strange to have GetStorageType both in codec-dependent Packetizers and
codec-independent RtpSenderVideo

may be move all the functionality here and remove GetStorageType from
packetizers.

(e.g. your cl fixes H264: it would currently retransmit even with
kRetransmitOff)

[sprang_webrtc, 2017/08/31 15:54:29]

Yeah, that makes sense. Moved things around.

+    const RTPVideoHeader& header,
+    int32_t retransmission_settings,
+    RtpPacketizer* packetizer,
+    int64_t expected_retransmission_time) {
+  if (retransmission_settings == kRetransmitOff)
+    return StorageType::kDontRetransmit;
+  if (retransmission_settings == kRetransmitAllPackets)
+    return StorageType::kAllowRetransmission;
+
+  rtc::CritScope cs(&stats_crit_);
+  // Media packet storage.
+  int64_t now_ms = clock_->TimeInMilliseconds();
+  int pic_id = kNoPictureId;
+  if (retransmission_settings & kConditionallyRetransmitHigherLayers) {
+    int temporal_layer = 0;

[danilchap, 2017/08/30 13:52:29]

may be = kNoTemporalIdx

[sprang_webrtc, 2017/08/31 15:54:29]

Done.

+    switch (header.codec) {
+      case kRtpVideoVp8:
+        temporal_layer = header.codecHeader.VP8.temporalIdx;
+        pic_id = header.codecHeader.VP8.pictureId;
+        break;
+      case kRtpVideoVp9:
+        temporal_layer = header.codecHeader.VP9.temporal_idx;
+        pic_id = header.codecHeader.VP9.picture_id;
+        break;
+      default:
+        break;
+    }
+
+    // Update stats for any temporal layer.
+    TemporalLayerStats* current_layer_stats =
+        GetTemporalLayerStats(temporal_layer);
+    current_layer_stats->frame_rate_fp1000s_.Update(1, now_ms);
+    int64_t tl_frame_interval = now_ms - current_layer_stats->last_frame_time_;
+    current_layer_stats->last_frame_time_ = now_ms;
+
+    // Conditional retransmit only applies to upper layers.
+    if (temporal_layer != kNoTemporalIdx && temporal_layer > 0) {
+      bool enable_retransmission = false;
+      if (tl_frame_interval >= kMaxUnretransmittableFrameIntervalMs) {
+        // Too long since a retransmittable frame in this layer, enable NACK
+        // protection.
+        enable_retransmission = true;
+      } else {
+        // Estimate when the next frame of any lower layer will be sent.
+        const int64_t kUndefined = std::numeric_limits<int64_t>::max();
+        int64_t expected_next_frame_time = kUndefined;
+        for (int i = temporal_layer - 1; i >= 0; --i) {
+          TemporalLayerStats* stats = GetTemporalLayerStats(i);
+          rtc::Optional<uint32_t> rate =
+              stats->frame_rate_fp1000s_.Rate(now_ms);
+          if (rate) {
+            int64_t tl_next = stats->last_frame_time_ + 1000000 / *rate;
+            if (tl_next - now_ms > -expected_retransmission_time &&
+                tl_next < expected_next_frame_time) {
+              expected_next_frame_time = tl_next;
+            }
+          }
+        }
+
+        if (expected_next_frame_time == kUndefined ||
+            expected_next_frame_time - now_ms > expected_retransmission_time) {
+          // The next frame in a lower layer is expected at a later time (or
+          // unable to tell due to lack of data) than a retransmission is
+          // estimated to be able to arrive, so allow this packet to be nacked.
+          enable_retransmission = true;
+        }
+      }
+
+      if (enable_retransmission)
+        retransmission_settings |= kRetransmitHigherLayers;

[danilchap, 2017/08/29 17:31:20]

you already checked temporal_layer > 0.
why not return StorageType::kAllowRetransmission directly?

[sprang_webrtc, 2017/08/31 15:54:29]

Since this class doesn't know of the codec dependent bits would override that
for some reason. But this comment is now moot as I've GetStorageType() from the
packetizers.

+    }
+  }
+
+  return packetizer->GetStorageType(retransmission_settings);
+}
+
 }  // namespace webrtc