Make MediaCodecEncoder fallback to a software encoder on failure.

webrtc/api/android/jni/androidmediaencoder_jni.cc

 /*
  *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 68 matching lines @@
 #define ALOGV(...)
 #endif
 #define ALOGD LOG_TAG(rtc::LS_INFO, TAG_ENCODER)
 #define ALOGW LOG_TAG(rtc::LS_WARNING, TAG_ENCODER)
 #define ALOGE LOG_TAG(rtc::LS_ERROR, TAG_ENCODER)
 
 namespace {
 // Maximum time limit between incoming frames before requesting a key frame.
 const size_t kFrameDiffThresholdMs = 1100;
 const int kMinKeyFrameInterval = 2;
+// Maximum number of encoder resets before falling back to a software
+// implementation.
+const int kMaxEncoderResetsBeforeFallback = 3;

[magjed_webrtc, 2016/08/23 12:09:50]

Now it will fall back to software the fourth time, i.e. when we have failed to
reset the HW encoder 3 times. I think it is enough to only try resetting the HW
encoder one time, so I would like to change this value to 1.

[sakal, 2016/08/23 12:26:51]

Resetting the encoder is fairly quick so 3 resets shouldn't take a very long
time. Given that this value doesn't reset after a successful reset, I would
rather keep it at 3 in case the encoder fails for example twice during a call.

 }  // namespace
 
 // MediaCodecVideoEncoder is a webrtc::VideoEncoder implementation that uses
 // Android's MediaCodec SDK API behind the scenes to implement (hopefully)
 // HW-backed video encode.  This C++ class is implemented as a very thin shim,
 // delegating all of the interesting work to org.webrtc.MediaCodecVideoEncoder.
 // MediaCodecVideoEncoder is created, operated, and destroyed on a single
 // thread, currently the libjingle Worker thread.
 class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
                                public rtc::MessageHandler {
@@ skipping 176 matching lines @@
   // EGL context - owned by factory, should not be allocated/destroyed
   // by MediaCodecVideoEncoder.
   jobject egl_context_;
 
   // Temporary fix for VP8.
   // Sends a key frame if frames are largely spaced apart (possibly
   // corresponding to a large image change).
   int64_t last_frame_received_ms_;
   int frames_received_since_last_key_;
   webrtc::VideoCodecMode codec_mode_;
+
+  bool software_fallback_needed_;
+  int encoder_reset_counter_;
 };
 
 MediaCodecVideoEncoder::~MediaCodecVideoEncoder() {
   // Call Release() to ensure no more callbacks to us after we are deleted.
   Release();
 }
 
 MediaCodecVideoEncoder::MediaCodecVideoEncoder(
     JNIEnv* jni, VideoCodecType codecType, jobject egl_context) :
     codecType_(codecType),
     callback_(NULL),
     codec_thread_(new Thread()),
     j_media_codec_video_encoder_class_(
         jni,
         FindClass(jni, "org/webrtc/MediaCodecVideoEncoder")),
     j_media_codec_video_encoder_(
         jni,
         jni->NewObject(*j_media_codec_video_encoder_class_,
                        GetMethodID(jni,
                                    *j_media_codec_video_encoder_class_,
                                    "<init>",
                                    "()V"))),
     inited_(false),
     use_surface_(false),
     picture_id_(0),
-    egl_context_(egl_context) {
+    egl_context_(egl_context),
+    software_fallback_needed_(false),
+    encoder_reset_counter_(0) {
   ScopedLocalRefFrame local_ref_frame(jni);
   // It would be nice to avoid spinning up a new thread per MediaCodec, and
   // instead re-use e.g. the PeerConnectionFactory's |worker_thread_|, but bug
   // 2732 means that deadlocks abound.  This class synchronously trampolines
   // to |codec_thread_|, so if anything else can be coming to _us_ from
   // |codec_thread_|, or from any thread holding the |_sendCritSect| described
   // in the bug, we have a problem.  For now work around that with a dedicated
   // thread.
   codec_thread_->SetName("MediaCodecVideoEncoder", NULL);
   RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoEncoder";
@@ skipping 166 matching lines @@
   }
 
   // Call log statistics here so it's called even if no frames are being
   // delivered.
   LogStatistics(false);
 }
 
 bool MediaCodecVideoEncoder::ResetCodecOnCodecThread() {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
   ALOGE << "ResetOnCodecThread";
-  if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK ||
-      InitEncodeOnCodecThread(width_, height_, 0, 0, false) !=
-          WEBRTC_VIDEO_CODEC_OK) {
-    // TODO(fischman): wouldn't it be nice if there was a way to gracefully
-    // degrade to a SW encoder at this point?  There isn't one AFAICT :(
-    // https://code.google.com/p/webrtc/issues/detail?id=2920
+  encoder_reset_counter_++;
+  if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK) {
+    LOG(LS_ERROR) << "Releasing codec failed. Fallback to SW encoder.";
+    software_fallback_needed_ = true;

[magjed_webrtc, 2016/08/23 12:09:50]

Is it possible to remove |software_fallback_needed_| and check the return value
everywhere instead? I.e.  
if (!ResetCodecOnCodecThread())
  return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
or
return ResetCodecOnCodecThread()
    ? WEBRTC_VIDEO_CODEC_ERROR
    : WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;

[sakal, 2016/08/23 12:26:51]

There would be some trouble. The codec might not be initialized after
ResetCodecOnCodecThread fails. This makes some Java methods throw exceptions
when for example SetRatesOnCodecThread gets called.

+    return false;
+  }
+  if (encoder_reset_counter_ > kMaxEncoderResetsBeforeFallback) {
+    LOG(LS_ERROR) << "Codec has failed too many times. Fallback to SW encoder.";
+    software_fallback_needed_ = true;
+    return false;
+  }
+  if (InitEncodeOnCodecThread(width_, height_, 0, 0, false) !=
+      WEBRTC_VIDEO_CODEC_OK) {
+    LOG(LS_ERROR) << "Init encode failed. Fallback to SW encoder.";
+    software_fallback_needed_ = true;
     return false;
   }
   return true;
 }
 
 int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread(
     int width, int height, int kbps, int fps, bool use_surface) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
+  if (software_fallback_needed_) {
+    return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
+  }
   RTC_CHECK(!use_surface || egl_context_ != nullptr) << "EGL context not set.";
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
 
   ALOGD << "InitEncodeOnCodecThread Type: " <<  (int)codecType_ << ", " <<
       width << " x " << height << ". Bitrate: " << kbps <<
       " kbps. Fps: " << fps;
   if (kbps == 0) {
     kbps = last_set_bitrate_kbps_;
   }
@@ skipping 82 matching lines @@
 
   inited_ = true;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
     const webrtc::VideoFrame& frame,
     const std::vector<webrtc::FrameType>* frame_types,
     const int64_t frame_input_time_ms) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
+  if (software_fallback_needed_) {
+    return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
+  }
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
 
   if (!inited_) {
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
   }
 
   bool send_key_frame = false;
   if (codec_mode_ == webrtc::kRealtimeVideo) {
     ++frames_received_since_last_key_;
@@ skipping 222 matching lines @@
                                               handle->oes_texture_id,
                                               sampling_matrix,
                                               current_timestamp_us_);
   CHECK_EXCEPTION(jni);
   return encode_status;
 }
 
 int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread(
     webrtc::EncodedImageCallback* callback) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
+  if (software_fallback_needed_) {
+    return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
+  }
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
   callback_ = callback;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
   if (!inited_) {
     return WEBRTC_VIDEO_CODEC_OK;
@@ skipping 11 matching lines @@
   rtc::MessageQueueManager::Clear(this);
   inited_ = false;
   use_surface_ = false;
   ALOGD << "EncoderReleaseOnCodecThread done.";
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate,
                                                       uint32_t frame_rate) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
+  if (software_fallback_needed_) {
+    return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
+  }
   frame_rate = (frame_rate < MAX_ALLOWED_VIDEO_FPS) ?
       frame_rate : MAX_ALLOWED_VIDEO_FPS;
   if (last_set_bitrate_kbps_ == new_bit_rate &&
       last_set_fps_ == frame_rate) {
     return WEBRTC_VIDEO_CODEC_OK;
   }
   if (scale_) {
     quality_scaler_.ReportFramerate(frame_rate);
   }
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
@@ skipping 401 matching lines @@
   return supported_codecs_;
 }
 
 void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
     webrtc::VideoEncoder* encoder) {
   ALOGD << "Destroy video encoder.";
   delete encoder;
 }
 
 }  // namespace webrtc_jni