Implement timing frames.

webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  *
  */
 
 #include "webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h"
 
 #include <limits>
 #include <string>
 
 #include "third_party/openh264/src/codec/api/svc/codec_api.h"
 #include "third_party/openh264/src/codec/api/svc/codec_app_def.h"
 #include "third_party/openh264/src/codec/api/svc/codec_def.h"
 #include "third_party/openh264/src/codec/api/svc/codec_ver.h"
 
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
+#include "webrtc/base/timeutils.h"
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
-#include "webrtc/media/base/mediaconstants.h"
 #include "webrtc/system_wrappers/include/metrics.h"
 
 namespace webrtc {
 
 namespace {
 
 const bool kOpenH264EncoderDetailedLogging = false;
 
 // Used by histograms. Values of entries should not be changed.
 enum H264EncoderImplEvent {
@@ skipping 124 matching lines @@
       target_bps_(0),
       max_bps_(0),
       mode_(kRealtimeVideo),
       frame_dropping_on_(false),
       key_frame_interval_(0),
       packetization_mode_(H264PacketizationMode::SingleNalUnit),
       max_payload_size_(0),
       number_of_cores_(0),
       encoded_image_callback_(nullptr),
       has_reported_init_(false),
-      has_reported_error_(false) {
+      has_reported_error_(false),
+      last_timing_frame_time_ms_(0),

[sprang_webrtc, 2017/05/31 11:12:55]

Probably a good idea to use -1 as uninitialized value (and explicitly check for
that down below). It's not uncommon for unit test to have a fake clock starting
at time 0, leading to unintuitive behavior. 

[ilnik, 2017/05/31 15:17:45]

Done.

+      timing_frames_delay_ms_(0),
+      min_frame_size_to_force_timing_frame_bytes_(0) {
   RTC_CHECK(cricket::CodecNamesEq(codec.name, cricket::kH264CodecName));
   std::string packetization_mode_string;
   if (codec.GetParam(cricket::kH264FmtpPacketizationMode,
                      &packetization_mode_string) &&
       packetization_mode_string == "1") {
     packetization_mode_ = H264PacketizationMode::NonInterleaved;
   }
 }
 
 H264EncoderImpl::~H264EncoderImpl() {
@@ skipping 43 matching lines @@
 
   number_of_cores_ = number_of_cores;
   // Set internal settings from codec_settings
   width_ = codec_settings->width;
   height_ = codec_settings->height;
   max_frame_rate_ = static_cast<float>(codec_settings->maxFramerate);
   mode_ = codec_settings->mode;
   frame_dropping_on_ = codec_settings->H264().frameDroppingOn;
   key_frame_interval_ = codec_settings->H264().keyFrameInterval;
   max_payload_size_ = max_payload_size;
+  min_frame_size_to_force_timing_frame_bytes_ =
+      codec_settings->minFrameSizeToForceTimingFrameBytes;
+  timing_frames_delay_ms_ = codec_settings->timingFramesDelayMs;
 
   // Codec_settings uses kbits/second; encoder uses bits/second.
   max_bps_ = codec_settings->maxBitrate * 1000;
   if (codec_settings->targetBitrate == 0)
     target_bps_ = codec_settings->startBitrate * 1000;
   else
     target_bps_ = codec_settings->targetBitrate * 1000;
 
   SEncParamExt encoder_params = CreateEncoderParams();
 
@@ skipping 57 matching lines @@
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t H264EncoderImpl::Encode(const VideoFrame& input_frame,
                                 const CodecSpecificInfo* codec_specific_info,
                                 const std::vector<FrameType>* frame_types) {
   if (!IsInitialized()) {
     ReportError();
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
   }
+  int64_t encode_start_ms = rtc::TimeMillis();
   if (!encoded_image_callback_) {
     LOG(LS_WARNING) << "InitEncode() has been called, but a callback function "
                     << "has not been set with RegisterEncodeCompleteCallback()";
     ReportError();
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
   }
 
   bool force_key_frame = false;
   if (frame_types != nullptr) {
     // We only support a single stream.
@@ skipping 57 matching lines @@
                  &frag_header);
 
   // Encoder can skip frames to save bandwidth in which case
   // |encoded_image_._length| == 0.
   if (encoded_image_._length > 0) {
     // Parse QP.
     h264_bitstream_parser_.ParseBitstream(encoded_image_._buffer,
                                           encoded_image_._length);
     h264_bitstream_parser_.GetLastSliceQp(&encoded_image_.qp_);
 
+    // Add timing information if needed. Timing delay is counted based on
+    // capture time to ensure that simulcast encoders will mark frames on all
+    // the streams together.
+    int64_t now_ms = rtc::TimeMillis();
+    bool is_time_for_timing_frame =
+        encoded_image_.capture_time_ms_ - last_timing_frame_time_ms_ >=
+        timing_frames_delay_ms_;
+    if (is_time_for_timing_frame) {
+      last_timing_frame_time_ms_ = encoded_image_.capture_time_ms_;
+    }
+    if (is_time_for_timing_frame ||
+        encoded_image_._length >= min_frame_size_to_force_timing_frame_bytes_) {
+      encoded_image_.timing_.is_timing_frame = true;
+      encoded_image_.timing_.encode_start_ms = encode_start_ms;
+      encoded_image_.timing_.encode_finish_ms = now_ms;
+    } else {
+      encoded_image_.timing_.is_timing_frame = false;
+    }
+

[sprang_webrtc, 2017/05/31 11:12:55]

I'm not sure you need to do this on a per codec level.
Couldn't you implement this in VCMEncodedFrameCallback::OnEncodedImage()
instead, and signal the start time to that instance from
VCMGenericEncoder::Encode()?

[ilnik, 2017/05/31 15:17:45]

Firstly, it will be very troublesome to update VCMEncodedFrameCallback
interface, as it is used in downstream projects.Secondly, I am not sure, how it
will interact with simulcast encoders, if we have several of them in Simulcast
Adapter for example. Is there a separate instance of VCMEncodedFrameCallback for
each simulcast encoder? We also want to have all encoders to mark the same
frames based on time, so decision on marking a frame as timing is still needed
to be done in each encoder in the same way in parallel.

[sprang_webrtc, 2017/06/05 14:39:20]

You can always start by having a simple method declared inline. Just something
to set a state that any frame with capture time X should be marked as timing
frame.
No, these is just a single callback be called several times. This is true
regardless of usage of simulcast encoder adapter or libvpx with built-in
simulcast support.
Not sure I follow what you mean here. If you mean outlier detection, isn't it
enough to do that on a per stream basis?

[ilnik, 2017/06/07 14:25:03]

Done.

     // Deliver encoded image.
     CodecSpecificInfo codec_specific;
     codec_specific.codecType = kVideoCodecH264;
     codec_specific.codecSpecific.H264.packetization_mode = packetization_mode_;
     encoded_image_callback_->OnEncodedImage(encoded_image_, &codec_specific,
                                             &frag_header);
   }
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
@@ skipping 99 matching lines @@
 
 int32_t H264EncoderImpl::SetPeriodicKeyFrames(bool enable) {
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 VideoEncoder::ScalingSettings H264EncoderImpl::GetScalingSettings() const {
   return VideoEncoder::ScalingSettings(true);
 }
 
 }  // namespace webrtc