Send key frame if time difference between incoming frames exceeds a certain limit.

talk/app/webrtc/java/jni/androidmediaencoder_jni.cc

Index: talk/app/webrtc/java/jni/androidmediaencoder_jni.cc
diff --git a/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc b/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc
index e1793b8a162a7e872ffd315e93558c894d6fc1e3..a39c20faa434f41456e000f9f034f1cc778c3108 100644
--- a/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc
+++ b/talk/app/webrtc/java/jni/androidmediaencoder_jni.cc
@@ -34,6 +34,7 @@
 #include "webrtc/base/logging.h"
 #include "webrtc/base/thread.h"
 #include "webrtc/base/thread_checker.h"
+#include "webrtc/common_types.h"
 #include "webrtc/modules/rtp_rtcp/source/h264_bitstream_parser.h"
 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
 #include "webrtc/modules/video_coding/utility/include/quality_scaler.h"
@@ -70,6 +71,12 @@ namespace webrtc_jni {
 // Maximum supported HW video encoder fps.
 #define MAX_VIDEO_FPS 30
 
+namespace {
+// Maximum time limit between incoming frames before requesting a key frame.
+const size_t kFrameDiffThresholdMs = 1100;
+const int kMinKeyFrameInterval = 2;
+}  // namespace
+
 // MediaCodecVideoEncoder is a webrtc::VideoEncoder implementation that uses
 // Android's MediaCodec SDK API behind the scenes to implement (hopefully)
 // HW-backed video encode.  This C++ class is implemented as a very thin shim,
@@ -213,6 +220,13 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
 
   // H264 bitstream parser, used to extract QP from encoded bitstreams.
   webrtc::H264BitstreamParser h264_bitstream_parser_;
+
+  // Temporary fix for VP8.
+  // Sends a key frame if frames are largely spaced apart (possibly
+  // corresponding to a large image change).
+  int64_t last_frame_received_ms_;
+  int frames_received_since_last_key_;
+  webrtc::VideoCodecMode codec_mode_;
 };
 
 MediaCodecVideoEncoder::~MediaCodecVideoEncoder() {
@@ -307,6 +321,7 @@ int32_t MediaCodecVideoEncoder::InitEncode(
       << codecType_;
 
   ALOGD << "InitEncode request";
+  codec_mode_ = codec_settings->mode;
   scale_ = webrtc::field_trial::FindFullName(
       "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled";
   ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled");
@@ -458,6 +473,8 @@ int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread(
   frame_rtc_times_ms_.clear();
   drop_next_input_frame_ = false;
   picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF;
+  last_frame_received_ms_ = -1;
+  frames_received_since_last_key_ = kMinKeyFrameInterval;
 
   // We enforce no extra stride/padding in the format creation step.
   jobject j_video_codec_enum = JavaEnumFromIndex(
@@ -524,6 +541,21 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
   }
 
+  bool send_key_frame = false;
+  if (codecType_ == kVideoCodecVP8 && codec_mode_ == webrtc::kRealtimeVideo) {
+    ++frames_received_since_last_key_;
+    int64_t now_ms = GetCurrentTimeMs();
+    if (last_frame_received_ms_ != -1 &&
+       (now_ms - last_frame_received_ms_) > kFrameDiffThresholdMs) {
+      if (frames_received_since_last_key_ > kMinKeyFrameInterval) {

[stefan-webrtc, 2016/01/18 20:02:36]

What if there are two camera switches close to each other? Then the first one
will only generate a key frame, right? Shouldn't be a big problem since
kMinKeyFrameInterval is 2 though. Why is it 2 though?

Should we remember that the delay was large and generate a key frame a few
frames later when frames_received_since_last_key is large enough?

[åsapersson, 2016/01/19 12:22:17]

Yes will not work if there are two camera switches with only one frame in
between (but thought it should likely not be a problem?). kMinKeyFrameInterval
is set to a small value to prevent triggering a key frame for every frame for
very low framerates (for example if frame diff > kFrameDiffThresholdMs). Added a
comment.

+        ALOGD << "Send key, frame diff: " << (now_ms - last_frame_received_ms_);
+        send_key_frame = true;
+      }
+      frames_received_since_last_key_ = 0;
+    }
+    last_frame_received_ms_ = now_ms;
+  }
+
   frames_received_++;
   if (!DeliverPendingOutputs(jni)) {
     if (!ResetCodecOnCodecThread())
@@ -589,7 +621,8 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
   render_times_ms_.push_back(input_frame.render_time_ms());
   frame_rtc_times_ms_.push_back(GetCurrentTimeMs());
 
-  const bool key_frame = frame_types->front() != webrtc::kVideoFrameDelta;
+  const bool key_frame =
+      frame_types->front() != webrtc::kVideoFrameDelta || send_key_frame;

[stefan-webrtc, 2016/01/18 20:02:36]

Why isn't this == kVideoFrameKey? Now we will generate a key frame if it's
webrtc::kVideoFrameEmpty, which seems a bit wrong.

[åsapersson, 2016/01/19 12:22:17]

Right seems a bit wrong, not sure about the reason.

   const bool encode_status =
       EncodeByteBufferOnCodecThread(jni, key_frame, input_frame,
           j_input_buffer_index);