Add flag enabling more packets to be retransmittable.

webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/modules/rtp_rtcp/source/rtp_sender_video.h"
 
 #include <stdlib.h>
 #include <string.h>
 
+#include <limits>
 #include <memory>
+#include <utility>
 #include <vector>
-#include <utility>
 
 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
 #include "webrtc/modules/rtp_rtcp/source/byte_io.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_format_vp8.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_format_vp9.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h"
 #include "webrtc/rtc_base/checks.h"
 #include "webrtc/rtc_base/logging.h"
 #include "webrtc/rtc_base/ptr_util.h"
 #include "webrtc/rtc_base/trace_event.h"
 
 namespace webrtc {
 
 namespace {
 constexpr size_t kRedForFecHeaderLength = 1;
+constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4;
 
 void BuildRedPayload(const RtpPacketToSend& media_packet,
                      RtpPacketToSend* red_packet) {
   uint8_t* red_payload = red_packet->AllocatePayload(
       kRedForFecHeaderLength + media_packet.payload_size());
   RTC_DCHECK(red_payload);
   red_payload[0] = media_packet.PayloadType();
 
   auto media_payload = media_packet.payload();
   memcpy(&red_payload[kRedForFecHeaderLength], media_payload.data(),
          media_payload.size());
 }
 }  // namespace
 
 RTPSenderVideo::RTPSenderVideo(Clock* clock,
                                RTPSender* rtp_sender,
                                FlexfecSender* flexfec_sender)
     : rtp_sender_(rtp_sender),
       clock_(clock),
       video_type_(kRtpVideoGeneric),
-      retransmission_settings_(kRetransmitBaseLayer),
+      retransmission_settings_(kRetransmitBaseLayer |
+                               kConditionallyRetransmitHigherLayers),
       last_rotation_(kVideoRotation_0),
       red_payload_type_(-1),
       ulpfec_payload_type_(-1),
       flexfec_sender_(flexfec_sender),
       delta_fec_params_{0, 1, kFecMaskRandom},
       key_fec_params_{0, 1, kFecMaskRandom},
       fec_bitrate_(1000, RateStatistics::kBpsScale),
       video_bitrate_(1000, RateStatistics::kBpsScale) {}
 
 RTPSenderVideo::~RTPSenderVideo() {}
@@ skipping 218 matching lines @@
 }
 
 bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type,
                                FrameType frame_type,
                                int8_t payload_type,
                                uint32_t rtp_timestamp,
                                int64_t capture_time_ms,
                                const uint8_t* payload_data,
                                size_t payload_size,
                                const RTPFragmentationHeader* fragmentation,
-                               const RTPVideoHeader* video_header) {
+                               const RTPVideoHeader* video_header,
+                               int64_t expected_retransmission_time_ms) {
   if (payload_size == 0)
     return false;
 
   // Create header that will be reused in all packets.
   std::unique_ptr<RtpPacketToSend> rtp_header = rtp_sender_->AllocatePacket();
   rtp_header->SetPayloadType(payload_type);
   rtp_header->SetTimestamp(rtp_timestamp);
   rtp_header->set_capture_time_ms(capture_time_ms);
   auto last_packet = rtc::MakeUnique<RtpPacketToSend>(*rtp_header);
 
@@ skipping 52 matching lines @@
   RTC_DCHECK_GT(packet_capacity, rtp_header->headers_size());
   RTC_DCHECK_GT(packet_capacity, last_packet->headers_size());
   size_t max_data_payload_length = packet_capacity - rtp_header->headers_size();
   RTC_DCHECK_GE(last_packet->headers_size(), rtp_header->headers_size());
   size_t last_packet_reduction_len =
       last_packet->headers_size() - rtp_header->headers_size();
 
   std::unique_ptr<RtpPacketizer> packetizer(RtpPacketizer::Create(
       video_type, max_data_payload_length, last_packet_reduction_len,
       video_header ? &(video_header->codecHeader) : nullptr, frame_type));
-  // Media packet storage.
-  StorageType storage = packetizer->GetStorageType(retransmission_settings);
+
+  StorageType storage =
+      GetStorageType(*video_header, retransmission_settings, packetizer.get(),
+                     expected_retransmission_time_ms);
 
   // TODO(changbin): we currently don't support to configure the codec to
   // output multiple partitions for VP8. Should remove below check after the
   // issue is fixed.
   const RTPFragmentationHeader* frag =
       (video_type == kRtpVideoVp8) ? nullptr : fragmentation;
   size_t num_packets =
       packetizer->SetPayloadData(payload_data, payload_size, frag);
 
   if (num_packets == 0)
@@ skipping 66 matching lines @@
 int RTPSenderVideo::SelectiveRetransmissions() const {
   rtc::CritScope cs(&crit_);
   return retransmission_settings_;
 }
 
 void RTPSenderVideo::SetSelectiveRetransmissions(uint8_t settings) {
   rtc::CritScope cs(&crit_);
   retransmission_settings_ = settings;
 }
 
+RTPSenderVideo::TemporalLayerStats* RTPSenderVideo::GetTemporalLayerStats(

[danilchap, 2017/08/29 17:31:20]

since return type can't be nullptr, may return reference:
 RTPSenderVideo::TemporalLayerStats& RTPSenderVideo::GetTemporalLayerStats(int
temporal_id) {
  return frame_stats_by_temporal_layer_.emplace(temporal_id,
kWindowSizeMs).first->second;
  // If you'll add default constructor to TemporalLayerStats, even easier:
  return frame_stats_by_temporal_layer_[temporal_id];
}

[sprang_webrtc, 2017/08/31 15:54:29]

Are we ok with non-const references then?
Changed to default constructor and made the window size referencable from test.

[danilchap, 2017/09/04 09:03:04]

the rule states "All parameters passed by reference must be labeled const."

[sprang_webrtc, 2017/09/04 11:09:28]

Acknowledged.

+    int temporal_id) {
+  auto it = frame_stats_by_temporal_layer_.find(temporal_id);
+  if (it == frame_stats_by_temporal_layer_.end()) {
+    const int kWindowSizeMs = 2500;
+    it = frame_stats_by_temporal_layer_.emplace(temporal_id, kWindowSizeMs)

[danilchap, 2017/08/29 17:31:20]

you do not need to search before emplace, so can remove double lookup:
return &frame_stats_by_temporal_layer_.emplace(temporal_id,
kWindowSizeMs).first->second;

[sprang_webrtc, 2017/08/31 15:54:29]

Acknowledged.

+             .first;
+  }
+  return &it->second;
+}
+
+StorageType RTPSenderVideo::GetStorageType(

[danilchap, 2017/08/30 13:52:29]

Looks strange to have GetStorageType both in codec-dependent Packetizers and
codec-independent RtpSenderVideo

may be move all the functionality here and remove GetStorageType from
packetizers.

(e.g. your cl fixes H264: it would currently retransmit even with
kRetransmitOff)

[sprang_webrtc, 2017/08/31 15:54:29]

Yeah, that makes sense. Moved things around.

+    const RTPVideoHeader& header,
+    int32_t retransmission_settings,
+    RtpPacketizer* packetizer,
+    int64_t expected_retransmission_time) {
+  if (retransmission_settings == kRetransmitOff)
+    return StorageType::kDontRetransmit;
+  if (retransmission_settings == kRetransmitAllPackets)
+    return StorageType::kAllowRetransmission;
+
+  rtc::CritScope cs(&stats_crit_);
+  // Media packet storage.
+  int64_t now_ms = clock_->TimeInMilliseconds();
+  int pic_id = kNoPictureId;
+  if (retransmission_settings & kConditionallyRetransmitHigherLayers) {
+    int temporal_layer = 0;

[danilchap, 2017/08/30 13:52:29]

may be = kNoTemporalIdx

[sprang_webrtc, 2017/08/31 15:54:29]

Done.

+    switch (header.codec) {
+      case kRtpVideoVp8:
+        temporal_layer = header.codecHeader.VP8.temporalIdx;
+        pic_id = header.codecHeader.VP8.pictureId;
+        break;
+      case kRtpVideoVp9:
+        temporal_layer = header.codecHeader.VP9.temporal_idx;
+        pic_id = header.codecHeader.VP9.picture_id;
+        break;
+      default:
+        break;
+    }
+
+    // Update stats for any temporal layer.
+    TemporalLayerStats* current_layer_stats =
+        GetTemporalLayerStats(temporal_layer);
+    current_layer_stats->frame_rate_fp1000s_.Update(1, now_ms);
+    int64_t tl_frame_interval = now_ms - current_layer_stats->last_frame_time_;
+    current_layer_stats->last_frame_time_ = now_ms;
+
+    // Conditional retransmit only applies to upper layers.
+    if (temporal_layer != kNoTemporalIdx && temporal_layer > 0) {
+      bool enable_retransmission = false;
+      if (tl_frame_interval >= kMaxUnretransmittableFrameIntervalMs) {
+        // Too long since a retransmittable frame in this layer, enable NACK
+        // protection.
+        enable_retransmission = true;
+      } else {
+        // Estimate when the next frame of any lower layer will be sent.
+        const int64_t kUndefined = std::numeric_limits<int64_t>::max();
+        int64_t expected_next_frame_time = kUndefined;
+        for (int i = temporal_layer - 1; i >= 0; --i) {
+          TemporalLayerStats* stats = GetTemporalLayerStats(i);
+          rtc::Optional<uint32_t> rate =
+              stats->frame_rate_fp1000s_.Rate(now_ms);
+          if (rate) {
+            int64_t tl_next = stats->last_frame_time_ + 1000000 / *rate;
+            if (tl_next - now_ms > -expected_retransmission_time &&
+                tl_next < expected_next_frame_time) {
+              expected_next_frame_time = tl_next;
+            }
+          }
+        }
+
+        if (expected_next_frame_time == kUndefined ||
+            expected_next_frame_time - now_ms > expected_retransmission_time) {
+          // The next frame in a lower layer is expected at a later time (or
+          // unable to tell due to lack of data) than a retransmission is
+          // estimated to be able to arrive, so allow this packet to be nacked.
+          enable_retransmission = true;
+        }
+      }
+
+      if (enable_retransmission)
+        retransmission_settings |= kRetransmitHigherLayers;

[danilchap, 2017/08/29 17:31:20]

you already checked temporal_layer > 0.
why not return StorageType::kAllowRetransmission directly?

[sprang_webrtc, 2017/08/31 15:54:29]

Since this class doesn't know of the codec dependent bits would override that
for some reason. But this comment is now moot as I've GetStorageType() from the
packetizers.

+    }
+  }
+
+  return packetizer->GetStorageType(retransmission_settings);
+}
+
 }  // namespace webrtc