New jitter buffer experiment.

webrtc/video/rtp_stream_receiver.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/video/rtp_stream_receiver.h"
 
 #include <vector>
+#include <utility>
 
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/common_types.h"
 #include "webrtc/config.h"
 #include "webrtc/modules/pacing/packet_router.h"
 #include "webrtc/modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h"
 #include "webrtc/modules/rtp_rtcp/include/receive_statistics.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_cvo.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h"
 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
 #include "webrtc/modules/rtp_rtcp/include/ulpfec_receiver.h"
+#include "webrtc/modules/video_coding/frame_object.h"
+#include "webrtc/modules/video_coding/h264_sps_pps_tracker.h"
+#include "webrtc/modules/video_coding/packet_buffer.h"
 #include "webrtc/modules/video_coding/video_coding_impl.h"
+#include "webrtc/system_wrappers/include/field_trial.h"
 #include "webrtc/system_wrappers/include/metrics.h"
 #include "webrtc/system_wrappers/include/timestamp_extrapolator.h"
 #include "webrtc/system_wrappers/include/trace.h"
 #include "webrtc/video/receive_statistics_proxy.h"
 #include "webrtc/video/vie_remb.h"
 
 namespace webrtc {
 
+namespace {
+constexpr int kPacketBufferStartSize = 32;
+constexpr int kPacketBufferMaxSixe = 2048;
+}
+
 std::unique_ptr<RtpRtcp> CreateRtpRtcpModule(
     ReceiveStatistics* receive_statistics,
     Transport* outgoing_transport,
     RtcpRttStats* rtt_stats,
     RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
     RemoteBitrateEstimator* remote_bitrate_estimator,
     RtpPacketSender* paced_sender,
     TransportSequenceNumberAllocator* transport_sequence_number_allocator,
     RateLimiter* retransmission_rate_limiter) {
   RtpRtcp::Configuration configuration;
@@ skipping 30 matching lines @@
     vcm::VideoReceiver* video_receiver,
     RemoteBitrateEstimator* remote_bitrate_estimator,
     Transport* transport,
     RtcpRttStats* rtt_stats,
     PacedSender* paced_sender,
     PacketRouter* packet_router,
     VieRemb* remb,
     const VideoReceiveStream::Config* config,
     ReceiveStatisticsProxy* receive_stats_proxy,
     ProcessThread* process_thread,
-    RateLimiter* retransmission_rate_limiter)
+    RateLimiter* retransmission_rate_limiter,
+    NackSender* nack_sender,
+    KeyFrameRequestSender* keyframe_request_sender,
+    video_coding::OnCompleteFrameCallback* complete_frame_callback,
+    VCMTiming* timing)
     : clock_(Clock::GetRealTimeClock()),
       config_(*config),
       video_receiver_(video_receiver),
       remote_bitrate_estimator_(remote_bitrate_estimator),
       packet_router_(packet_router),
       remb_(remb),
       process_thread_(process_thread),
       ntp_estimator_(clock_),
       rtp_payload_registry_(RTPPayloadStrategy::CreateStrategy(false)),
       rtp_header_parser_(RtpHeaderParser::Create()),
       rtp_receiver_(RtpReceiver::CreateVideoReceiver(clock_,
                                                      this,
                                                      this,
                                                      &rtp_payload_registry_)),
       rtp_receive_statistics_(ReceiveStatistics::Create(clock_)),
       ulpfec_receiver_(UlpfecReceiver::Create(this)),
       receiving_(false),
       restored_packet_in_use_(false),
       last_packet_log_ms_(-1),
       rtp_rtcp_(CreateRtpRtcpModule(rtp_receive_statistics_.get(),
                                     transport,
                                     rtt_stats,
                                     receive_stats_proxy,
                                     remote_bitrate_estimator_,
                                     paced_sender,
                                     packet_router,
-                                    retransmission_rate_limiter)) {
+                                    retransmission_rate_limiter)),
+      complete_frame_callback_(complete_frame_callback),
+      keyframe_request_sender_(keyframe_request_sender),
+      timing_(timing) {
   packet_router_->AddRtpModule(rtp_rtcp_.get());
   rtp_receive_statistics_->RegisterRtpStatisticsCallback(receive_stats_proxy);
   rtp_receive_statistics_->RegisterRtcpStatisticsCallback(receive_stats_proxy);
 
   RTC_DCHECK(config_.rtp.rtcp_mode != RtcpMode::kOff)
       << "A stream should not be configured with RTCP disabled. This value is "
          "reserved for internal usage.";
   RTC_DCHECK(config_.rtp.remote_ssrc != 0);
   // TODO(pbos): What's an appropriate local_ssrc for receive-only streams?
   RTC_DCHECK(config_.rtp.local_ssrc != 0);
@@ skipping 61 matching lines @@
                                config_.rtp.ulpfec.ulpfec_payload_type);
   }
 
   if (config_.rtp.rtcp_xr.receiver_reference_time_report)
     rtp_rtcp_->SetRtcpXrRrtrStatus(true);
 
   // Stats callback for CNAME changes.
   rtp_rtcp_->RegisterRtcpStatisticsCallback(receive_stats_proxy);
 
   process_thread_->RegisterModule(rtp_rtcp_.get());
+
+  jitter_buffer_experiment_ =
+      field_trial::FindFullName("WebRTC-NewVideoJitterBuffer") == "Enabled";
+
+  if (jitter_buffer_experiment_) {
+    nack_module_.reset(
+        new NackModule(clock_, nack_sender, keyframe_request_sender));
+    process_thread_->RegisterModule(nack_module_.get());
+
+    packet_buffer_ = video_coding::PacketBuffer::Create(
+        clock_, kPacketBufferStartSize, kPacketBufferMaxSixe, this);
+    reference_finder_.reset(new video_coding::RtpFrameReferenceFinder(this));
+  }
 }
 
 RtpStreamReceiver::~RtpStreamReceiver() {
   process_thread_->DeRegisterModule(rtp_rtcp_.get());
 
+  if (jitter_buffer_experiment_)
+    process_thread_->DeRegisterModule(nack_module_.get());
+
   packet_router_->RemoveRtpModule(rtp_rtcp_.get());
   rtp_rtcp_->SetREMBStatus(false);
   remb_->RemoveReceiveChannel(rtp_rtcp_.get());
   UpdateHistograms();
 }
 
 bool RtpStreamReceiver::SetReceiveCodec(const VideoCodec& video_codec) {
   int8_t old_pltype = -1;
   if (rtp_payload_registry_.ReceivePayloadType(
           video_codec.plName, kVideoPayloadTypeFrequency, 0,
@@ skipping 19 matching lines @@
 }
 
 int32_t RtpStreamReceiver::OnReceivedPayloadData(
     const uint8_t* payload_data,
     size_t payload_size,
     const WebRtcRTPHeader* rtp_header) {
   RTC_DCHECK(video_receiver_);
   WebRtcRTPHeader rtp_header_with_ntp = *rtp_header;
   rtp_header_with_ntp.ntp_time_ms =
       ntp_estimator_.Estimate(rtp_header->header.timestamp);
-  if (video_receiver_->IncomingPacket(payload_data, payload_size,
-                                      rtp_header_with_ntp) != 0) {
-    // Check this...
-    return -1;
+  if (jitter_buffer_experiment_) {
+    VCMPacket packet(payload_data, payload_size, rtp_header_with_ntp);
+    timing_->IncomingTimestamp(packet.timestamp, clock_->TimeInMilliseconds());
+    packet.timesNacked = nack_module_->OnReceivedPacket(packet);
+
+    if (packet.codec == kVideoCodecH264) {
+      switch (tracker_.CopyAndFixBitstream(&packet)) {
+        case video_coding::H264SpsPpsTracker::kRequestKeyframe:
+          keyframe_request_sender_->RequestKeyFrame();
+          FALLTHROUGH();
+        case video_coding::H264SpsPpsTracker::kDrop:
+          return 0;
+        case video_coding::H264SpsPpsTracker::kInsert:
+          break;
+      }
+    } else {
+      uint8_t* data = new uint8_t[packet.sizeBytes];
+      memcpy(data, packet.dataPtr, packet.sizeBytes);
+      packet.dataPtr = data;
+    }
+
+    packet_buffer_->InsertPacket(packet);
+  } else {
+    if (video_receiver_->IncomingPacket(payload_data, payload_size,
+                                        rtp_header_with_ntp) != 0) {
+      // Check this...
+      return -1;
+    }
   }
   return 0;
 }
 
 bool RtpStreamReceiver::OnRecoveredPacket(const uint8_t* rtp_packet,
                                           size_t rtp_packet_length) {
   RTPHeader header;
   if (!rtp_header_parser_->Parse(rtp_packet, rtp_packet_length, &header)) {
     return false;
   }
@@ skipping 97 matching lines @@
 void RtpStreamReceiver::RequestPacketRetransmit(
     const std::vector<uint16_t>& sequence_numbers) {
   rtp_rtcp_->SendNack(sequence_numbers);
 }
 
 int32_t RtpStreamReceiver::ResendPackets(const uint16_t* sequence_numbers,
                                          uint16_t length) {
   return rtp_rtcp_->SendNACK(sequence_numbers, length);
 }
 
+void RtpStreamReceiver::OnReceivedFrame(
+    std::unique_ptr<video_coding::RtpFrameObject> frame) {
+  reference_finder_->ManageFrame(std::move(frame));
+}
+
+void RtpStreamReceiver::OnCompleteFrame(
+    std::unique_ptr<video_coding::FrameObject> frame) {
+  {
+    rtc::CritScope lock(&last_seq_num_cs_);
+    video_coding::RtpFrameObject* rtp_frame =
+        static_cast<video_coding::RtpFrameObject*>(frame.get());
+    last_seq_num_for_pic_id_[rtp_frame->picture_id] = rtp_frame->last_seq_num();
+  }
+  complete_frame_callback_->OnCompleteFrame(std::move(frame));
+}
+
+void RtpStreamReceiver::OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) {
+  if (jitter_buffer_experiment_)
+    nack_module_->UpdateRtt(max_rtt_ms);
+}
+
 bool RtpStreamReceiver::ReceivePacket(const uint8_t* packet,
                                       size_t packet_length,
                                       const RTPHeader& header,
                                       bool in_order) {
   if (rtp_payload_registry_.IsEncapsulated(header)) {
     return ParseAndHandleEncapsulatingHeader(packet, packet_length, header);
   }
   const uint8_t* payload = packet + header.headerLength;
   assert(packet_length >= header.headerLength);
   size_t payload_length = packet_length - header.headerLength;
@@ skipping 104 matching lines @@
   if (rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, nullptr, nullptr,
                            &rtp_timestamp) != 0) {
     // Waiting for RTCP.
     return true;
   }
   ntp_estimator_.UpdateRtcpTimestamp(rtt, ntp_secs, ntp_frac, rtp_timestamp);
 
   return true;
 }
 
+void RtpStreamReceiver::FrameContinuous(uint16_t picture_id) {
+  if (jitter_buffer_experiment_) {
+    int seq_num = -1;
+    {
+      rtc::CritScope lock(&last_seq_num_cs_);
+      auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id);
+      if (seq_num_it != last_seq_num_for_pic_id_.end())
+        seq_num = seq_num_it->second;
+    }
+    if (seq_num != -1)
+      nack_module_->ClearUpTo(seq_num);
+  }
+}
+
+void RtpStreamReceiver::FrameDecoded(uint16_t picture_id) {
+  if (jitter_buffer_experiment_) {
+    int seq_num = -1;
+    {
+      rtc::CritScope lock(&last_seq_num_cs_);
+      auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id);
+      if (seq_num_it != last_seq_num_for_pic_id_.end()) {
+        seq_num = seq_num_it->second;
+        last_seq_num_for_pic_id_.erase(last_seq_num_for_pic_id_.begin(),
+                                       ++seq_num_it);
+      }
+    }
+    if (seq_num != -1) {
+      packet_buffer_->ClearTo(seq_num);
+      reference_finder_->ClearTo(seq_num);
+    }
+  }
+}
+
 void RtpStreamReceiver::SignalNetworkState(NetworkState state) {
   rtp_rtcp_->SetRTCPStatus(state == kNetworkUp ? config_.rtp.rtcp_mode
                                                : RtcpMode::kOff);
 }
 
 void RtpStreamReceiver::StartReceive() {
   rtc::CritScope lock(&receive_cs_);
   receiving_ = true;
 }
 
@@ skipping 44 matching lines @@
     const std::string& extension, int id) {
   // One-byte-extension local identifiers are in the range 1-14 inclusive.
   RTC_DCHECK_GE(id, 1);
   RTC_DCHECK_LE(id, 14);
   RTC_DCHECK(RtpExtension::IsSupportedForVideo(extension));
   RTC_CHECK(rtp_header_parser_->RegisterRtpHeaderExtension(
       StringToRtpExtensionType(extension), id));
 }
 
 }  // namespace webrtc