Add flag enabling more packets to be retransmittable.

webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/modules/rtp_rtcp/source/rtp_sender_video.h"
 
 #include <stdlib.h>
 #include <string.h>
 
+#include <limits>
 #include <memory>
+#include <utility>
 #include <vector>
-#include <utility>
 
 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
 #include "webrtc/modules/rtp_rtcp/source/byte_io.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_format_vp8.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_format_vp9.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h"
 #include "webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h"
 #include "webrtc/rtc_base/checks.h"
 #include "webrtc/rtc_base/logging.h"
 #include "webrtc/rtc_base/ptr_util.h"
 #include "webrtc/rtc_base/trace_event.h"
 
 namespace webrtc {
 
 namespace {
 constexpr size_t kRedForFecHeaderLength = 1;
+constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4;
 
 void BuildRedPayload(const RtpPacketToSend& media_packet,
                      RtpPacketToSend* red_packet) {
   uint8_t* red_payload = red_packet->AllocatePayload(
       kRedForFecHeaderLength + media_packet.payload_size());
   RTC_DCHECK(red_payload);
   red_payload[0] = media_packet.PayloadType();
 
   auto media_payload = media_packet.payload();
   memcpy(&red_payload[kRedForFecHeaderLength], media_payload.data(),
          media_payload.size());
 }
 }  // namespace
 
 RTPSenderVideo::RTPSenderVideo(Clock* clock,
                                RTPSender* rtp_sender,
                                FlexfecSender* flexfec_sender)
     : rtp_sender_(rtp_sender),
       clock_(clock),
       video_type_(kRtpVideoGeneric),
-      retransmission_settings_(kRetransmitBaseLayer),
+      retransmission_settings_(kRetransmitBaseLayer |
+                               kConditionallyRetransmitHigherLayers),
       last_rotation_(kVideoRotation_0),
       red_payload_type_(-1),
       ulpfec_payload_type_(-1),
       flexfec_sender_(flexfec_sender),
       delta_fec_params_{0, 1, kFecMaskRandom},
       key_fec_params_{0, 1, kFecMaskRandom},
       fec_bitrate_(1000, RateStatistics::kBpsScale),
       video_bitrate_(1000, RateStatistics::kBpsScale) {}
 
 RTPSenderVideo::~RTPSenderVideo() {}
@@ skipping 218 matching lines @@
 }
 
 bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type,
                                FrameType frame_type,
                                int8_t payload_type,
                                uint32_t rtp_timestamp,
                                int64_t capture_time_ms,
                                const uint8_t* payload_data,
                                size_t payload_size,
                                const RTPFragmentationHeader* fragmentation,
-                               const RTPVideoHeader* video_header) {
+                               const RTPVideoHeader* video_header,
+                               int64_t expected_retransmission_time_ms) {
   if (payload_size == 0)
     return false;
 
   // Create header that will be reused in all packets.
   std::unique_ptr<RtpPacketToSend> rtp_header = rtp_sender_->AllocatePacket();
   rtp_header->SetPayloadType(payload_type);
   rtp_header->SetTimestamp(rtp_timestamp);
   rtp_header->set_capture_time_ms(capture_time_ms);
   auto last_packet = rtc::MakeUnique<RtpPacketToSend>(*rtp_header);
 
@@ skipping 52 matching lines @@
   RTC_DCHECK_GT(packet_capacity, rtp_header->headers_size());
   RTC_DCHECK_GT(packet_capacity, last_packet->headers_size());
   size_t max_data_payload_length = packet_capacity - rtp_header->headers_size();
   RTC_DCHECK_GE(last_packet->headers_size(), rtp_header->headers_size());
   size_t last_packet_reduction_len =
       last_packet->headers_size() - rtp_header->headers_size();
 
   std::unique_ptr<RtpPacketizer> packetizer(RtpPacketizer::Create(
       video_type, max_data_payload_length, last_packet_reduction_len,
       video_header ? &(video_header->codecHeader) : nullptr, frame_type));
-  // Media packet storage.
-  StorageType storage = packetizer->GetStorageType(retransmission_settings);
+
+  const uint8_t temporal_id =
+      video_header ? GetTemporalId(*video_header) : kNoTemporalIdx;
+  StorageType storage =
+      GetStorageType(temporal_id, retransmission_settings, packetizer.get(),
+                     expected_retransmission_time_ms);
 
   // TODO(changbin): we currently don't support to configure the codec to
   // output multiple partitions for VP8. Should remove below check after the
   // issue is fixed.
   const RTPFragmentationHeader* frag =
       (video_type == kRtpVideoVp8) ? nullptr : fragmentation;
   size_t num_packets =
       packetizer->SetPayloadData(payload_data, payload_size, frag);
 
   if (num_packets == 0)
     return false;
 
   bool first_frame = first_frame_sent_();
   for (size_t i = 0; i < num_packets; ++i) {
     bool last = (i + 1) == num_packets;
     auto packet = last ? std::move(last_packet)
                        : rtc::MakeUnique<RtpPacketToSend>(*rtp_header);
     if (!packetizer->NextPacket(packet.get()))
       return false;
     RTC_DCHECK_LE(packet->payload_size(),
                   last ? max_data_payload_length - last_packet_reduction_len
                        : max_data_payload_length);
     if (!rtp_sender_->AssignSequenceNumber(packet.get()))
       return false;
 
-    bool protect_packet = (packetizer->GetProtectionType() == kProtectedPacket);
+    // No FEC protection for upper temporal layers, if used.
+    const uint8_t temporal_id =

[danilchap, 2017/09/04 09:03:04]

this statement can be removed - temporal_id already calculated above

[brandtr, 2017/09/04 10:29:22]

Reuse |temporal_id| from line 373?

[sprang_webrtc, 2017/09/04 11:09:28]

Done.

[sprang_webrtc, 2017/09/04 11:09:28]

Done.

+        video_header ? GetTemporalId(*video_header) : kNoTemporalIdx;
+    bool protect_packet = temporal_id == 0 || temporal_id == kNoTemporalIdx;
+
     // Put packetization finish timestamp into extension.
     if (packet->HasExtension<VideoTimingExtension>()) {
       packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds());
       // TODO(ilnik): Due to webrtc:7859, packets with timing extensions are not
       // protected by FEC. It reduces FEC efficiency a bit. When FEC is moved
       // below the pacer, it can be re-enabled for these packets.
       // NOTE: Any RTP stream processor in the network, modifying 'network'
       // timestamps in the timing frames extension have to be an end-point for
       // FEC, otherwise recovered by FEC packets will be corrupted.
       protect_packet = false;
@@ skipping 40 matching lines @@
 int RTPSenderVideo::SelectiveRetransmissions() const {
   rtc::CritScope cs(&crit_);
   return retransmission_settings_;
 }
 
 void RTPSenderVideo::SetSelectiveRetransmissions(uint8_t settings) {
   rtc::CritScope cs(&crit_);
   retransmission_settings_ = settings;
 }
 
+StorageType RTPSenderVideo::GetStorageType(
+    uint8_t temporal_id,
+    int32_t retransmission_settings,

[danilchap, 2017/09/04 09:03:04]

since retransmission_settings is a bitmask, may be unsigned type better be used.
is it possible to use RetransmissionMode directly?

[sprang_webrtc, 2017/09/04 11:09:28]

Using RetransmissionMode directly forces a lot of explicit casts to be done. It
would also be weird to mask two values together and then cast it to a
RetransmissionMode value that is not declared by the enum.

I could potentially make it unsigned (or uint8_t, which is what
RetransmissionMode is typed as), but am hesitant to change the interface at this
time. I suspect there are downstream dependencies...

+    RtpPacketizer* packetizer,

[danilchap, 2017/09/04 09:03:04]

this parameter is no longer needed

[sprang_webrtc, 2017/09/04 11:09:28]

Done.

+    int64_t expected_retransmission_time) {

[danilchap, 2017/09/04 09:03:04]

add _ms

[sprang_webrtc, 2017/09/04 11:09:28]

Done.

+  if (retransmission_settings == kRetransmitOff)
+    return StorageType::kDontRetransmit;
+  if (retransmission_settings == kRetransmitAllPackets)
+    return StorageType::kAllowRetransmission;
+
+  rtc::CritScope cs(&stats_crit_);
+  // Media packet storage.
+  if ((retransmission_settings & kConditionallyRetransmitHigherLayers) &&
+      (UpdateConditionalRetransmit(temporal_id,

[danilchap, 2017/09/04 09:03:04]

extra () around function call can be removed

[sprang_webrtc, 2017/09/04 11:09:28]

Done.

+                                   expected_retransmission_time))) {
+    retransmission_settings |= kRetransmitHigherLayers;
+  }
+
+  if (temporal_id == kNoTemporalIdx)
+    return kAllowRetransmission;
+
+  if ((retransmission_settings & kRetransmitBaseLayer) && temporal_id == 0)
+    return kAllowRetransmission;
+
+  if ((retransmission_settings & kRetransmitHigherLayers) && temporal_id > 0)
+    return kAllowRetransmission;
+
+  return kDontRetransmit;
+}
+
+uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) const {
+  switch (header.codec) {
+    case kRtpVideoVp8:
+      return header.codecHeader.VP8.temporalIdx;
+    case kRtpVideoVp9:
+      return header.codecHeader.VP9.temporal_idx;
+    default:
+      return kNoTemporalIdx;
+  }
+}
+
+bool RTPSenderVideo::UpdateConditionalRetransmit(
+    uint8_t temporal_id,
+    int64_t expected_retransmission_time_ms) {
+  int64_t now_ms = clock_->TimeInMilliseconds();
+  // Update stats for any temporal layer.
+  TemporalLayerStats* current_layer_stats =
+      &frame_stats_by_temporal_layer_[temporal_id];
+  current_layer_stats->frame_rate_fp1000s.Update(1, now_ms);
+  int64_t tl_frame_interval = now_ms - current_layer_stats->last_frame_time;
+  current_layer_stats->last_frame_time = now_ms;
+
+  // Conditional retransmit only applies to upper layers.
+  if (temporal_id != kNoTemporalIdx && temporal_id > 0) {
+    if (tl_frame_interval >= kMaxUnretransmittableFrameIntervalMs) {
+      // Too long since a retransmittable frame in this layer, enable NACK
+      // protection.
+      return true;
+    } else {
+      // Estimate when the next frame of any lower layer will be sent.
+      const int64_t kUndefined = std::numeric_limits<int64_t>::max();
+      int64_t expected_next_frame_time = kUndefined;
+      for (int i = temporal_id - 1; i >= 0; --i) {
+        TemporalLayerStats* stats = &frame_stats_by_temporal_layer_[i];
+        rtc::Optional<uint32_t> rate = stats->frame_rate_fp1000s.Rate(now_ms);
+        if (rate) {
+          int64_t tl_next = stats->last_frame_time + 1000000 / *rate;
+          if (tl_next - now_ms > -expected_retransmission_time_ms &&
+              tl_next < expected_next_frame_time) {
+            expected_next_frame_time = tl_next;
+          }
+        }
+      }
+
+      if (expected_next_frame_time == kUndefined ||
+          expected_next_frame_time - now_ms > expected_retransmission_time_ms) {
+        // The next frame in a lower layer is expected at a later time (or
+        // unable to tell due to lack of data) than a retransmission is
+        // estimated to be able to arrive, so allow this packet to be nacked.
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
 }  // namespace webrtc