Send key frame if time difference between incoming frames exceeds a certain limit.

talk/app/webrtc/java/jni/androidmediaencoder_jni.cc

 /*
  * libjingle
  * Copyright 2015 Google Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright notice,
@@ skipping 16 matching lines @@
  */
 
 #include "talk/app/webrtc/java/jni/androidmediaencoder_jni.h"
 #include "talk/app/webrtc/java/jni/classreferenceholder.h"
 #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h"
 #include "webrtc/base/bind.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/thread.h"
 #include "webrtc/base/thread_checker.h"
+#include "webrtc/common_types.h"
 #include "webrtc/modules/rtp_rtcp/source/h264_bitstream_parser.h"
 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
 #include "webrtc/modules/video_coding/utility/include/quality_scaler.h"
 #include "webrtc/modules/video_coding/utility/include/vp8_header_parser.h"
 #include "webrtc/system_wrappers/include/field_trial.h"
 #include "webrtc/system_wrappers/include/logcat_trace_context.h"
 #include "third_party/libyuv/include/libyuv/convert.h"
 #include "third_party/libyuv/include/libyuv/convert_from.h"
 #include "third_party/libyuv/include/libyuv/video_common.h"
 
@@ skipping 16 matching lines @@
 // H.264 start code length.
 #define H264_SC_LENGTH 4
 // Maximum allowed NALUs in one output frame.
 #define MAX_NALUS_PERFRAME 32
 // Maximum supported HW video encoder resolution.
 #define MAX_VIDEO_WIDTH 1280
 #define MAX_VIDEO_HEIGHT 1280
 // Maximum supported HW video encoder fps.
 #define MAX_VIDEO_FPS 30
 
+namespace {
+// Maximum time limit between incoming frames before requesting a key frame.
+const size_t kFrameDiffThresholdMs = 1100;
+const int kMinKeyFrameInterval = 2;
+}  // namespace
+
 // MediaCodecVideoEncoder is a webrtc::VideoEncoder implementation that uses
 // Android's MediaCodec SDK API behind the scenes to implement (hopefully)
 // HW-backed video encode.  This C++ class is implemented as a very thin shim,
 // delegating all of the interesting work to org.webrtc.MediaCodecVideoEncoder.
 // MediaCodecVideoEncoder is created, operated, and destroyed on a single
 // thread, currently the libjingle Worker thread.
 class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
                                public rtc::MessageHandler {
  public:
   virtual ~MediaCodecVideoEncoder();
@@ skipping 123 matching lines @@
   // and the next Encode() call being ignored.
   bool drop_next_input_frame_;
   // Global references; must be deleted in Release().
   std::vector<jobject> input_buffers_;
   webrtc::QualityScaler quality_scaler_;
   // Dynamic resolution change, off by default.
   bool scale_;
 
   // H264 bitstream parser, used to extract QP from encoded bitstreams.
   webrtc::H264BitstreamParser h264_bitstream_parser_;
+
+  // Temporary fix for VP8.
+  // Sends a key frame if frames are largely spaced apart (possibly
+  // corresponding to a large image change).
+  int64_t last_frame_received_ms_;
+  int frames_received_since_last_key_;
+  webrtc::VideoCodecMode codec_mode_;
 };
 
 MediaCodecVideoEncoder::~MediaCodecVideoEncoder() {
   // Call Release() to ensure no more callbacks to us after we are deleted.
   Release();
 }
 
 MediaCodecVideoEncoder::MediaCodecVideoEncoder(
     JNIEnv* jni, VideoCodecType codecType) :
     codecType_(codecType),
@@ skipping 74 matching lines @@
   if (codec_settings == NULL) {
     ALOGE << "NULL VideoCodec instance";
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
   // Factory should guard against other codecs being used with us.
   RTC_CHECK(codec_settings->codecType == codecType_)
       << "Unsupported codec " << codec_settings->codecType << " for "
       << codecType_;
 
   ALOGD << "InitEncode request";
+  codec_mode_ = codec_settings->mode;
   scale_ = webrtc::field_trial::FindFullName(
       "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled";
   ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled");
   if (scale_) {
     if (codecType_ == kVideoCodecVP8) {
       // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
       // (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is
       // always = 127. Note that in SW, QP is that of the user-level range [0,
       // 63].
       const int kMaxQp = 127;
@@ skipping 131 matching lines @@
   current_encoding_time_ms_ = 0;
   last_input_timestamp_ms_ = -1;
   last_output_timestamp_ms_ = -1;
   output_timestamp_ = 0;
   output_render_time_ms_ = 0;
   timestamps_.clear();
   render_times_ms_.clear();
   frame_rtc_times_ms_.clear();
   drop_next_input_frame_ = false;
   picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF;
+  last_frame_received_ms_ = -1;
+  frames_received_since_last_key_ = kMinKeyFrameInterval;
 
   // We enforce no extra stride/padding in the format creation step.
   jobject j_video_codec_enum = JavaEnumFromIndex(
       jni, "MediaCodecVideoEncoder$VideoCodecType", codecType_);
   const bool encode_status = jni->CallBooleanMethod(
       *j_media_codec_video_encoder_, j_init_encode_method_,
       j_video_codec_enum, width, height, kbps, fps);
   if (!encode_status) {
     ALOGE << "Failed to configure encoder.";
     return WEBRTC_VIDEO_CODEC_ERROR;
@@ skipping 46 matching lines @@
     const webrtc::VideoFrame& frame,
     const std::vector<webrtc::FrameType>* frame_types) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
 
   if (!inited_) {
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
   }
 
+  bool send_key_frame = false;
+  if (codecType_ == kVideoCodecVP8 && codec_mode_ == webrtc::kRealtimeVideo) {
+    ++frames_received_since_last_key_;
+    int64_t now_ms = GetCurrentTimeMs();
+    if (last_frame_received_ms_ != -1 &&
+       (now_ms - last_frame_received_ms_) > kFrameDiffThresholdMs) {
+      if (frames_received_since_last_key_ > kMinKeyFrameInterval) {

[stefan-webrtc, 2016/01/18 20:02:36]

What if there are two camera switches close to each other? Then the first one
will only generate a key frame, right? Shouldn't be a big problem since
kMinKeyFrameInterval is 2 though. Why is it 2 though?

Should we remember that the delay was large and generate a key frame a few
frames later when frames_received_since_last_key is large enough?

[åsapersson, 2016/01/19 12:22:17]

Yes will not work if there are two camera switches with only one frame in
between (but thought it should likely not be a problem?). kMinKeyFrameInterval
is set to a small value to prevent triggering a key frame for every frame for
very low framerates (for example if frame diff > kFrameDiffThresholdMs). Added a
comment.

+        ALOGD << "Send key, frame diff: " << (now_ms - last_frame_received_ms_);
+        send_key_frame = true;
+      }
+      frames_received_since_last_key_ = 0;
+    }
+    last_frame_received_ms_ = now_ms;
+  }
+
   frames_received_++;
   if (!DeliverPendingOutputs(jni)) {
     if (!ResetCodecOnCodecThread())
       return WEBRTC_VIDEO_CODEC_ERROR;
   }
 
   if (drop_next_input_frame_) {
     ALOGW << "Encoder drop frame - failed callback.";
     drop_next_input_frame_ = false;
     return WEBRTC_VIDEO_CODEC_OK;
@@ skipping 45 matching lines @@
 
   last_input_timestamp_ms_ =
       current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec;
   frames_in_queue_++;
 
   // Save input image timestamps for later output
   timestamps_.push_back(input_frame.timestamp());
   render_times_ms_.push_back(input_frame.render_time_ms());
   frame_rtc_times_ms_.push_back(GetCurrentTimeMs());
 
-  const bool key_frame = frame_types->front() != webrtc::kVideoFrameDelta;
+  const bool key_frame =
+      frame_types->front() != webrtc::kVideoFrameDelta || send_key_frame;

[stefan-webrtc, 2016/01/18 20:02:36]

Why isn't this == kVideoFrameKey? Now we will generate a key frame if it's
webrtc::kVideoFrameEmpty, which seems a bit wrong.

[åsapersson, 2016/01/19 12:22:17]

Right seems a bit wrong, not sure about the reason.

   const bool encode_status =
       EncodeByteBufferOnCodecThread(jni, key_frame, input_frame,
           j_input_buffer_index);
 
   current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
 
   if (!encode_status || !DeliverPendingOutputs(jni)) {
     ALOGE << "Failed deliver pending outputs.";
     ResetCodecOnCodecThread();
     return WEBRTC_VIDEO_CODEC_ERROR;
@@ skipping 405 matching lines @@
 }
 
 void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
     webrtc::VideoEncoder* encoder) {
   ALOGD << "Destroy video encoder.";
   delete encoder;
 }
 
 }  // namespace webrtc_jni