New jitter buffer experiment.

webrtc/video/video_receive_stream.cc

 /*
  *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/video/video_receive_stream.h"
 
 #include <stdlib.h>
 
 #include <set>
 #include <string>
 #include <utility>
 
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
+#include "webrtc/base/optional.h"
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
 #include "webrtc/modules/congestion_controller/include/congestion_controller.h"
 #include "webrtc/modules/utility/include/process_thread.h"
+#include "webrtc/modules/video_coding/frame_object.h"
 #include "webrtc/modules/video_coding/include/video_coding.h"
+#include "webrtc/modules/video_coding/jitter_estimator.h"
+#include "webrtc/modules/video_coding/timing.h"
 #include "webrtc/modules/video_coding/utility/ivf_file_writer.h"
 #include "webrtc/system_wrappers/include/clock.h"
+#include "webrtc/system_wrappers/include/field_trial.h"
 #include "webrtc/video/call_stats.h"
 #include "webrtc/video/receive_statistics_proxy.h"
 #include "webrtc/video_receive_stream.h"
 #include "webrtc/voice_engine/include/voe_video_sync.h"
 
 namespace webrtc {
 
 static bool UseSendSideBwe(const VideoReceiveStream::Config& config) {
   if (!config.rtp.transport_cc)
     return false;
@@ skipping 153 matching lines @@
     : transport_adapter_(config.rtcp_send_transport),
       config_(std::move(config)),
       num_cpu_cores_(num_cpu_cores),
       process_thread_(process_thread),
       clock_(Clock::GetRealTimeClock()),
       decode_thread_(DecodeThreadFunction, this, "DecodingThread"),
       congestion_controller_(congestion_controller),
       call_stats_(call_stats),
       video_receiver_(clock_, nullptr, this, this, this),
       stats_proxy_(&config_, clock_),
+      timing_(new VCMTiming(clock_)),
       rtp_stream_receiver_(
           &video_receiver_,
           congestion_controller_->GetRemoteBitrateEstimator(
               UseSendSideBwe(config_)),
           &transport_adapter_,
           call_stats_->rtcp_rtt_stats(),
           congestion_controller_->pacer(),
           congestion_controller_->packet_router(),
           remb,
           &config_,
           &stats_proxy_,
           process_thread_,
-          congestion_controller_->GetRetransmissionRateLimiter()),
+          congestion_controller_->GetRetransmissionRateLimiter(),
+          this,  // NackSender
+          this,  // KeyFrameRequestSender
+          this,  // OnCompleteFrameCallback
+          timing_.get()),
       rtp_stream_sync_(&video_receiver_, &rtp_stream_receiver_) {
   LOG(LS_INFO) << "VideoReceiveStream: " << config_.ToString();
 
   RTC_DCHECK(process_thread_);
   RTC_DCHECK(congestion_controller_);
   RTC_DCHECK(call_stats_);
 
   RTC_DCHECK(!config_.decoders.empty());
   std::set<int> decoder_payload_types;
   for (const Decoder& decoder : config_.decoders) {
     RTC_CHECK(decoder.decoder);
     RTC_CHECK(decoder_payload_types.find(decoder.payload_type) ==
               decoder_payload_types.end())
         << "Duplicate payload type (" << decoder.payload_type
         << ") for different decoders.";
     decoder_payload_types.insert(decoder.payload_type);
   }
 
   video_receiver_.SetRenderDelay(config.render_delay_ms);
 
+  jitter_buffer_experiment_ =
+      field_trial::FindFullName("WebRTC-NewVideoJitterBuffer") == "Enabled";
+
+  if (jitter_buffer_experiment_) {
+    jitter_estimator_.reset(new VCMJitterEstimator(clock_));
+    frame_buffer_.reset(new video_coding::FrameBuffer(
+        clock_, jitter_estimator_.get(), timing_.get()));
+  }
+
   process_thread_->RegisterModule(&video_receiver_);
   process_thread_->RegisterModule(&rtp_stream_sync_);
 }
 
 VideoReceiveStream::~VideoReceiveStream() {
   LOG(LS_INFO) << "~VideoReceiveStream: " << config_.ToString();
   Stop();
 
   process_thread_->DeRegisterModule(&rtp_stream_sync_);
   process_thread_->DeRegisterModule(&video_receiver_);
@@ skipping 18 matching lines @@
 }
 
 bool VideoReceiveStream::OnRecoveredPacket(const uint8_t* packet,
                                            size_t length) {
   return rtp_stream_receiver_.OnRecoveredPacket(packet, length);
 }
 
 void VideoReceiveStream::Start() {
   if (decode_thread_.IsRunning())
     return;
+  if (jitter_buffer_experiment_) {
+    frame_buffer_->Start();
+    call_stats_->RegisterStatsObserver(&rtp_stream_receiver_);
+
+    if (rtp_stream_receiver_.IsRetransmissionsEnabled() &&
+        rtp_stream_receiver_.IsFecEnabled()) {
+      frame_buffer_->SetProtectionMode(kProtectionNackFEC);
+    }
+  }
   transport_adapter_.Enable();
   rtc::VideoSinkInterface<VideoFrame>* renderer = nullptr;
   if (config_.renderer) {
     if (config_.disable_prerenderer_smoothing) {
       renderer = this;
     } else {
       incoming_video_stream_.reset(
           new IncomingVideoStream(config_.render_delay_ms, this));
       renderer = incoming_video_stream_.get();
     }
@@ skipping 22 matching lines @@
   decode_thread_.SetPriority(rtc::kHighestPriority);
   rtp_stream_receiver_.StartReceive();
 }
 
 void VideoReceiveStream::Stop() {
   rtp_stream_receiver_.StopReceive();
   // TriggerDecoderShutdown will release any waiting decoder thread and make it
   // stop immediately, instead of waiting for a timeout. Needs to be called
   // before joining the decoder thread thread.
   video_receiver_.TriggerDecoderShutdown();
+
+  if (jitter_buffer_experiment_) {
+    frame_buffer_->Stop();
+    call_stats_->DeregisterStatsObserver(&rtp_stream_receiver_);
+  }
+
   if (decode_thread_.IsRunning()) {
     decode_thread_.Stop();
     // Deregister external decoders so they are no longer running during
     // destruction. This effectively stops the VCM since the decoder thread is
     // stopped, the VCM is deregistered and no asynchronous decoder threads are
     // running.
     for (const Decoder& decoder : config_.decoders)
       video_receiver_.RegisterExternalDecoder(nullptr, decoder.payload_type);
   }
+
   call_stats_->DeregisterStatsObserver(video_stream_decoder_.get());
   video_stream_decoder_.reset();
   incoming_video_stream_.reset();
   transport_adapter_.Disable();
 }
 
 void VideoReceiveStream::SetSyncChannel(VoiceEngine* voice_engine,
                                         int audio_channel_id) {
   if (voice_engine && audio_channel_id != -1) {
     VoEVideoSync* voe_sync_interface = VoEVideoSync::GetInterface(voice_engine);
@@ skipping 26 matching lines @@
     stats_proxy_.OnSyncOffsetUpdated(sync_offset_ms, estimated_freq_khz);
   }
 
   // config_.renderer must never be null if we're getting this callback.
   config_.renderer->OnFrame(video_frame);
 
   // TODO(tommi): OnRenderFrame grabs a lock too.
   stats_proxy_.OnRenderedFrame(video_frame);
 }
 
+void VideoReceiveStream::OnCompleteFrame(
+    std::unique_ptr<video_coding::FrameObject> frame) {
+  int last_continuous_pid = frame_buffer_->InsertFrame(std::move(frame));
+  if (last_continuous_pid != -1)
+    rtp_stream_receiver_.FrameContinuous(last_continuous_pid);
+}
+
 // TODO(asapersson): Consider moving callback from video_encoder.h or
 // creating a different callback.
 EncodedImageCallback::Result VideoReceiveStream::OnEncodedImage(
     const EncodedImage& encoded_image,
     const CodecSpecificInfo* codec_specific_info,
     const RTPFragmentationHeader* fragmentation) {
   stats_proxy_.OnPreDecode(encoded_image, codec_specific_info);
   if (config_.pre_decode_callback) {
     config_.pre_decode_callback->EncodedFrameCallback(
         EncodedFrame(encoded_image._buffer, encoded_image._length,
@@ skipping 12 matching lines @@
   return Result(Result::OK, encoded_image._timeStamp);
 }
 
 bool VideoReceiveStream::DecodeThreadFunction(void* ptr) {
   static_cast<VideoReceiveStream*>(ptr)->Decode();
   return true;
 }
 
 void VideoReceiveStream::Decode() {
   static const int kMaxDecodeWaitTimeMs = 50;
-  video_receiver_.Decode(kMaxDecodeWaitTimeMs);
+  if (jitter_buffer_experiment_) {
+    static const int kMaxWaitForFrameMs = 3000;
+    std::unique_ptr<video_coding::FrameObject> frame;
+    video_coding::FrameBuffer::ReturnReason res =
+        frame_buffer_->NextFrame(kMaxWaitForFrameMs, &frame);
+
+    if (res == video_coding::FrameBuffer::ReturnReason::kStopped)
+      return;
+
+    if (frame) {
+      if (video_receiver_.Decode(frame.get()) == VCM_OK)
+        rtp_stream_receiver_.FrameDecoded(frame->picture_id);
+    } else {
+      LOG(LS_WARNING) << "No decodable frame in " << kMaxWaitForFrameMs
+                      << " ms, requesting keyframe.";
+      RequestKeyFrame();
+    }
+  } else {
+    video_receiver_.Decode(kMaxDecodeWaitTimeMs);
+  }
 }
 
 void VideoReceiveStream::SendNack(
     const std::vector<uint16_t>& sequence_numbers) {
   rtp_stream_receiver_.RequestPacketRetransmit(sequence_numbers);
 }
 
 void VideoReceiveStream::EnableEncodedFrameRecording(rtc::PlatformFile file,
                                                      size_t byte_limit) {
   {
@@ skipping 11 matching lines @@
     RequestKeyFrame();
   }
 }
 
 void VideoReceiveStream::RequestKeyFrame() {
   rtp_stream_receiver_.RequestKeyFrame();
 }
 
 }  // namespace internal
 }  // namespace webrtc