Make FlexFEC packets paceable through RTPSender.

webrtc/modules/rtp_rtcp/source/rtp_sender.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 23 matching lines @@
 
 namespace {
 // Max in the RFC 3550 is 255 bytes, we limit it to be modulus 32 for SRTP.
 constexpr size_t kMaxPaddingLength = 224;
 constexpr int kSendSideDelayWindowMs = 1000;
 constexpr size_t kRtpHeaderLength = 12;
 constexpr uint16_t kMaxInitRtpSeqNumber = 32767;  // 2^15 -1.
 constexpr uint32_t kTimestampTicksPerMs = 90;
 constexpr int kBitrateStatisticsWindowMs = 1000;
 
+constexpr size_t kFlexfecPacketsToStoreBeforePacing = 50;

[brandtr, 2016/11/10 13:34:14]

Is this a reasonable number? We store up to 600 media packets in the main packet
history data structure:
https://cs.chromium.org/chromium/src/third_party/webrtc/video/video_send_stre...,
but the difference is that we will never retransmit a NACKed FlexFEC packet.

[danilchap, 2016/11/11 15:26:06]

this is initial size of the packet history, buffer will grow if needed.
max capacity (9000 packets) is unchangeable.
i.e the actual value is not that important.

may be add word 'Min' to variable name (similar to the constant you link to)

[brandtr, 2016/11/14 09:48:08]

Done.

+
 const char* FrameTypeToString(FrameType frame_type) {
   switch (frame_type) {
     case kEmptyFrame:
       return "empty";
     case kAudioFrameSpeech: return "audio_speech";
     case kAudioFrameCN: return "audio_cn";
     case kVideoFrameKey: return "video_key";
     case kVideoFrameDelta: return "video_delta";
   }
   return "";
@@ skipping 33 matching lines @@
       transport_sequence_number_allocator_(sequence_number_allocator),
       transport_feedback_observer_(transport_feedback_observer),
       last_capture_time_ms_sent_(0),
       transport_(transport),
       sending_media_(true),                      // Default to sending media.
       max_payload_length_(IP_PACKET_SIZE - 28),  // Default is IP-v4/UDP.
       payload_type_(-1),
       payload_type_map_(),
       rtp_header_extension_map_(),
       packet_history_(clock),
+      flexfec_packet_history_(clock),
       // Statistics
       rtp_stats_callback_(nullptr),
       total_bitrate_sent_(kBitrateStatisticsWindowMs,
                           RateStatistics::kBpsScale),
       nack_bitrate_sent_(kBitrateStatisticsWindowMs, RateStatistics::kBpsScale),
       frame_count_observer_(frame_count_observer),
       send_side_delay_observer_(send_side_delay_observer),
       event_log_(event_log),
       send_packet_observer_(send_packet_observer),
       bitrate_callback_(bitrate_callback),
@@ skipping 13 matching lines @@
   ssrc_ = ssrc_db_->CreateSSRC();
   RTC_DCHECK(ssrc_ != 0);
   ssrc_rtx_ = ssrc_db_->CreateSSRC();
   RTC_DCHECK(ssrc_rtx_ != 0);
 
   // This random initialization is not intended to be cryptographic strong.
   timestamp_offset_ = random_.Rand<uint32_t>();
   // Random start, 16 bits. Can't be 0.
   sequence_number_rtx_ = random_.Rand(1, kMaxInitRtpSeqNumber);
   sequence_number_ = random_.Rand(1, kMaxInitRtpSeqNumber);
+
+  // Store FlexFEC packets in the packet history data structure, so they can
+  // be found when paced.
+  if (flexfec_sender) {
+    flexfec_packet_history_.SetStorePacketsStatus(
+        true, kFlexfecPacketsToStoreBeforePacing);
+  }
 }
 
 RTPSender::~RTPSender() {
   // TODO(tommi): Use a thread checker to ensure the object is created and
   // deleted on the same thread.  At the moment this isn't possible due to
   // voe::ChannelOwner in voice engine.  To reproduce, run:
   // voe_auto_test --automated --gtest_filter=*MixManyChannelsForStressOpus
 
   // TODO(tommi,holmer): We don't grab locks in the dtor before accessing member
   // variables but we grab them in all other methods. (what's the design?)
@@ skipping 538 matching lines @@
     }
   }
 }
 
 void RTPSender::OnReceivedRtcpReportBlocks(
     const ReportBlockList& report_blocks) {
   playout_delay_oracle_.OnReceivedRtcpReportBlocks(report_blocks);
 }
 
 // Called from pacer when we can send the packet.
-bool RTPSender::TimeToSendPacket(uint16_t sequence_number,
+bool RTPSender::TimeToSendPacket(uint32_t ssrc,
+                                 uint16_t sequence_number,
                                  int64_t capture_time_ms,
                                  bool retransmission,
                                  int probe_cluster_id) {
-  std::unique_ptr<RtpPacketToSend> packet =
-      packet_history_.GetPacketAndSetSendTime(sequence_number, 0,
-                                              retransmission);
+  std::unique_ptr<RtpPacketToSend> packet;
+  rtc::Optional<uint32_t> flexfec_ssrc = FlexfecSsrc();
+  if (flexfec_ssrc && ssrc == flexfec_ssrc) {
+    packet = flexfec_packet_history_.GetPacketAndSetSendTime(sequence_number, 0,
+                                                             retransmission);
+  } else {
+    packet = packet_history_.GetPacketAndSetSendTime(sequence_number, 0,
+                                                     retransmission);
+  }
+
   if (!packet) {
     // Packet cannot be found. Allow sending to continue.
     return true;
   }
 
   return PrepareAndSendPacket(
       std::move(packet),
       retransmission && (RtxStatus() & kRtxRetransmitted) > 0, retransmission,
       probe_cluster_id);
 }
@@ skipping 127 matching lines @@
   } else {
     BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "AudioTotBitrate_kbps", now_ms,
                                     ActualSendBitrateKbit(), packet->Ssrc());
     BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "AudioNackBitrate_kbps", now_ms,
                                     NackOverheadRate() / 1000, packet->Ssrc());
   }
 
   if (paced_sender_) {
     uint16_t seq_no = packet->SequenceNumber();
     uint32_t ssrc = packet->Ssrc();
+    rtc::Optional<uint32_t> flexfec_ssrc = FlexfecSsrc();
     // Correct offset between implementations of millisecond time stamps in
     // TickTime and Clock.
     int64_t corrected_time_ms = packet->capture_time_ms() + clock_delta_ms_;
     size_t payload_length = packet->payload_size();
-    packet_history_.PutRtpPacket(std::move(packet), storage, false);
+    if (flexfec_ssrc && ssrc == flexfec_ssrc) {
+      flexfec_packet_history_.PutRtpPacket(std::move(packet), storage, false);
+    } else {
+      packet_history_.PutRtpPacket(std::move(packet), storage, false);
+    }
 
     paced_sender_->InsertPacket(priority, ssrc, seq_no, corrected_time_ms,
                                 payload_length, false);
     if (last_capture_time_ms_sent_ == 0 ||
         corrected_time_ms > last_capture_time_ms_sent_) {
       last_capture_time_ms_sent_ = corrected_time_ms;
       TRACE_EVENT_ASYNC_BEGIN1(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
                                "PacedSend", corrected_time_ms,
                                "capture_time_ms", corrected_time_ms);
     }
@@ skipping 18 matching lines @@
     {
       rtc::CritScope lock(&send_critsect_);
       media_has_been_sent_ = true;
     }
     UpdateRtpStats(*packet, false, false);
   }
 
   // Mark the packet as sent in the history even if send failed. Dropping a
   // packet here should be treated as any other packet drop so we should be
   // ready for a retransmission.
   packet_history_.PutRtpPacket(std::move(packet), storage, true);

[brandtr, 2016/11/10 13:34:14]

No need to store FlexFEC packets here, since at this time they have already been
sent to the transport.

[danilchap, 2016/11/10 16:15:13]

still need to check ssrc, right now you would store flexfex packets in regular
packet_history.

you do not need to store packets (both regular and flexfec) if storage ==
kDontRetransmit though.

[brandtr, 2016/11/11 10:28:16]

Good point, you are right.

Right now, I have not built support for retransmitting FlexFEC packets. I think
there _is_ support for this for ULPFEC, but we just never activate it.

 
   return sent;
 }
 
 void RTPSender::UpdateDelayStatistics(int64_t capture_time_ms, int64_t now_ms) {
   if (!send_side_delay_observer_ || capture_time_ms <= 0)
     return;
 
   uint32_t ssrc;
   int avg_delay_ms = 0;
@@ skipping 186 matching lines @@
   if (!sequence_number_forced_) {
     sequence_number_ = random_.Rand(1, kMaxInitRtpSeqNumber);
   }
 }
 
 uint32_t RTPSender::SSRC() const {
   rtc::CritScope lock(&send_critsect_);
   return ssrc_;
 }
 
+rtc::Optional<uint32_t> RTPSender::FlexfecSsrc() const {
+  if (video_) {
+    return video_->FlexfecSsrc();
+  }
+  return rtc::Optional<uint32_t>();  // No value.
+}
+
 void RTPSender::SetCsrcs(const std::vector<uint32_t>& csrcs) {
   assert(csrcs.size() <= kRtpCsrcSize);
   rtc::CritScope lock(&send_critsect_);
   csrcs_ = csrcs;
 }
 
 void RTPSender::SetSequenceNumber(uint16_t seq) {
   rtc::CritScope lock(&send_critsect_);
   sequence_number_forced_ = true;
   sequence_number_ = seq;
@@ skipping 136 matching lines @@
   rtc::CritScope lock(&send_critsect_);
 
   RtpState state;
   state.sequence_number = sequence_number_rtx_;
   state.start_timestamp = timestamp_offset_;
 
   return state;
 }
 
 }  // namespace webrtc