Make MediaCodecEncoder fallback to a software encoder on failure.

webrtc/api/android/jni/androidmediaencoder_jni.cc

 /*
  *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 68 matching lines @@
 #define ALOGV(...)
 #endif
 #define ALOGD LOG_TAG(rtc::LS_INFO, TAG_ENCODER)
 #define ALOGW LOG_TAG(rtc::LS_WARNING, TAG_ENCODER)
 #define ALOGE LOG_TAG(rtc::LS_ERROR, TAG_ENCODER)
 
 namespace {
 // Maximum time limit between incoming frames before requesting a key frame.
 const size_t kFrameDiffThresholdMs = 1100;
 const int kMinKeyFrameInterval = 2;
+// Maximum number of encoder resets before falling back to a software
+// implementation.
+const int kMaxEncoderResetsBeforeFallback = 3;

[AlexG, 2016/08/23 22:50:31]

I don't think this is necessary - similarly to decoder we can fallback as soon
as we detected any HW encoder error. This will make switch faster.

[sakal, 2016/08/25 11:52:48]

Done.

 }  // namespace
 
 // MediaCodecVideoEncoder is a webrtc::VideoEncoder implementation that uses
 // Android's MediaCodec SDK API behind the scenes to implement (hopefully)
 // HW-backed video encode.  This C++ class is implemented as a very thin shim,
 // delegating all of the interesting work to org.webrtc.MediaCodecVideoEncoder.
 // MediaCodecVideoEncoder is created, operated, and destroyed on a single
 // thread, currently the libjingle Worker thread.
 class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
                                public rtc::MessageHandler {
@@ skipping 176 matching lines @@
   // EGL context - owned by factory, should not be allocated/destroyed
   // by MediaCodecVideoEncoder.
   jobject egl_context_;
 
   // Temporary fix for VP8.
   // Sends a key frame if frames are largely spaced apart (possibly
   // corresponding to a large image change).
   int64_t last_frame_received_ms_;
   int frames_received_since_last_key_;
   webrtc::VideoCodecMode codec_mode_;
+
+  bool software_fallback_needed_;

[AlexG, 2016/08/23 22:50:31]

suggest to rename it to sw_fallback_required_ so we use same variable names both
for decoder and encoder

[sakal, 2016/08/25 11:52:48]

Done.

+  int encoder_reset_counter_;
 };
 
 MediaCodecVideoEncoder::~MediaCodecVideoEncoder() {
   // Call Release() to ensure no more callbacks to us after we are deleted.
   Release();
 }
 
 MediaCodecVideoEncoder::MediaCodecVideoEncoder(
     JNIEnv* jni, VideoCodecType codecType, jobject egl_context) :
     codecType_(codecType),
     callback_(NULL),
     codec_thread_(new Thread()),
     j_media_codec_video_encoder_class_(
         jni,
         FindClass(jni, "org/webrtc/MediaCodecVideoEncoder")),
     j_media_codec_video_encoder_(
         jni,
         jni->NewObject(*j_media_codec_video_encoder_class_,
                        GetMethodID(jni,
                                    *j_media_codec_video_encoder_class_,
                                    "<init>",
                                    "()V"))),
     inited_(false),
     use_surface_(false),
     picture_id_(0),
-    egl_context_(egl_context) {
+    egl_context_(egl_context),
+    software_fallback_needed_(false),
+    encoder_reset_counter_(0) {
   ScopedLocalRefFrame local_ref_frame(jni);
   // It would be nice to avoid spinning up a new thread per MediaCodec, and
   // instead re-use e.g. the PeerConnectionFactory's |worker_thread_|, but bug
   // 2732 means that deadlocks abound.  This class synchronously trampolines
   // to |codec_thread_|, so if anything else can be coming to _us_ from
   // |codec_thread_|, or from any thread holding the |_sendCritSect| described
   // in the bug, we have a problem.  For now work around that with a dedicated
   // thread.
   codec_thread_->SetName("MediaCodecVideoEncoder", NULL);
   RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoEncoder";
@@ skipping 166 matching lines @@
   }
 
   // Call log statistics here so it's called even if no frames are being
   // delivered.
   LogStatistics(false);
 }
 
 bool MediaCodecVideoEncoder::ResetCodecOnCodecThread() {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
   ALOGE << "ResetOnCodecThread";
-  if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK ||
-      InitEncodeOnCodecThread(width_, height_, 0, 0, false) !=
-          WEBRTC_VIDEO_CODEC_OK) {
-    // TODO(fischman): wouldn't it be nice if there was a way to gracefully
-    // degrade to a SW encoder at this point?  There isn't one AFAICT :(
-    // https://code.google.com/p/webrtc/issues/detail?id=2920
+  encoder_reset_counter_++;
+  if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK) {
+    LOG(LS_ERROR) << "Releasing codec failed. Fallback to SW encoder.";
+    software_fallback_needed_ = true;
+    return false;
+  }
+  if (encoder_reset_counter_ > kMaxEncoderResetsBeforeFallback) {
+    LOG(LS_ERROR) << "Codec has failed too many times. Fallback to SW encoder.";
+    software_fallback_needed_ = true;
+    return false;
+  }
+  if (InitEncodeOnCodecThread(width_, height_, 0, 0, false) !=
+      WEBRTC_VIDEO_CODEC_OK) {
+    LOG(LS_ERROR) << "Init encode failed. Fallback to SW encoder.";
+    software_fallback_needed_ = true;
     return false;
   }
   return true;
 }
 
 int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread(
     int width, int height, int kbps, int fps, bool use_surface) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
+  if (software_fallback_needed_) {
+    return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
+  }
   RTC_CHECK(!use_surface || egl_context_ != nullptr) << "EGL context not set.";
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
 
   ALOGD << "InitEncodeOnCodecThread Type: " <<  (int)codecType_ << ", " <<
       width << " x " << height << ". Bitrate: " << kbps <<
       " kbps. Fps: " << fps;
   if (kbps == 0) {
     kbps = last_set_bitrate_kbps_;
   }
@@ skipping 82 matching lines @@
 
   inited_ = true;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
     const webrtc::VideoFrame& frame,
     const std::vector<webrtc::FrameType>* frame_types,
     const int64_t frame_input_time_ms) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
+  if (software_fallback_needed_) {
+    return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
+  }
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
 
   if (!inited_) {
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
   }
 
   bool send_key_frame = false;
   if (codec_mode_ == webrtc::kRealtimeVideo) {
     ++frames_received_since_last_key_;
     int64_t now_ms = rtc::TimeMillis();
     if (last_frame_received_ms_ != -1 &&
         (now_ms - last_frame_received_ms_) > kFrameDiffThresholdMs) {
       // Add limit to prevent triggering a key for every frame for very low
       // framerates (e.g. if frame diff > kFrameDiffThresholdMs).
       if (frames_received_since_last_key_ > kMinKeyFrameInterval) {
         ALOGD << "Send key, frame diff: " << (now_ms - last_frame_received_ms_);
         send_key_frame = true;
       }
       frames_received_since_last_key_ = 0;
     }
     last_frame_received_ms_ = now_ms;
   }
 
   frames_received_++;
   if (!DeliverPendingOutputs(jni)) {
     if (!ResetCodecOnCodecThread())
-      return WEBRTC_VIDEO_CODEC_ERROR;
+      return software_fallback_needed_ ? WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE
+                                       : WEBRTC_VIDEO_CODEC_ERROR;
   }
   if (frames_encoded_ < kMaxEncodedLogFrames) {
     ALOGD << "Encoder frame in # " << (frames_received_ - 1)
           << ". TS: " << (int)(current_timestamp_us_ / 1000)
           << ". Q: " << input_frame_infos_.size() << ". Fps: " << last_set_fps_
           << ". Kbps: " << last_set_bitrate_kbps_;
   }
 
   if (drop_next_input_frame_) {
     ALOGW << "Encoder drop frame - failed callback.";
@@ skipping 13 matching lines @@
           << " frames in the queue, dropping"
           << ". TS: " << (int)(current_timestamp_us_ / 1000)
           << ". Fps: " << last_set_fps_
           << ". Consecutive drops: " << consecutive_full_queue_frame_drops_;
     current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
     consecutive_full_queue_frame_drops_++;
     if (consecutive_full_queue_frame_drops_ >=
         ENCODER_STALL_FRAMEDROP_THRESHOLD) {
       ALOGE << "Encoder got stuck. Reset.";
       ResetCodecOnCodecThread();
-      return WEBRTC_VIDEO_CODEC_ERROR;
+      return software_fallback_needed_ ? WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE
+                                       : WEBRTC_VIDEO_CODEC_ERROR;
     }
     frames_dropped_media_encoder_++;
     OnDroppedFrameOnCodecThread();
     return WEBRTC_VIDEO_CODEC_OK;
   }
   consecutive_full_queue_frame_drops_ = 0;
 
   rtc::scoped_refptr<webrtc::VideoFrameBuffer> input_buffer(
       frame.video_frame_buffer());
   if (scale_) {
@@ skipping 40 matching lines @@
         OnDroppedFrameOnCodecThread();
       } else {
         // Input buffers are not ready after codec initialization, HW is still
         // allocating thme - this is expected and should not result in drop
         // frame report.
         frames_received_ = 0;
       }
       return WEBRTC_VIDEO_CODEC_OK;  // TODO(fischman): see webrtc bug 2887.
     } else if (j_input_buffer_index == -2) {
       ResetCodecOnCodecThread();
-      return WEBRTC_VIDEO_CODEC_ERROR;
+      return software_fallback_needed_ ? WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE
+                                       : WEBRTC_VIDEO_CODEC_ERROR;
     }
     encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame,
         j_input_buffer_index);
   } else {
     encode_status = EncodeTextureOnCodecThread(jni, key_frame, input_frame);
   }
 
   if (!encode_status) {
     ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp();
     ResetCodecOnCodecThread();
-    return WEBRTC_VIDEO_CODEC_ERROR;
+    return software_fallback_needed_ ? WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE
+                                     : WEBRTC_VIDEO_CODEC_ERROR;
   }
 
   // Save input image timestamps for later output.
   input_frame_infos_.emplace_back(
       frame_input_time_ms, input_frame.timestamp(),
       input_frame.render_time_ms(), input_frame.rotation());
 
   last_input_timestamp_ms_ =
       current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec;
 
   current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
 
   codec_thread_->Clear(this);
   codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
 
   if (!DeliverPendingOutputs(jni)) {
     ALOGE << "Failed deliver pending outputs.";
     ResetCodecOnCodecThread();
-    return WEBRTC_VIDEO_CODEC_ERROR;
+    return software_fallback_needed_ ? WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE
+                                     : WEBRTC_VIDEO_CODEC_ERROR;
   }
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 bool MediaCodecVideoEncoder::MaybeReconfigureEncoderOnCodecThread(
     const webrtc::VideoFrame& frame) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
 
   const bool is_texture_frame =
       frame.video_frame_buffer()->native_handle() != nullptr;
@@ skipping 69 matching lines @@
                                               handle->oes_texture_id,
                                               sampling_matrix,
                                               current_timestamp_us_);
   CHECK_EXCEPTION(jni);
   return encode_status;
 }
 
 int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread(
     webrtc::EncodedImageCallback* callback) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
+  if (software_fallback_needed_) {

[AlexG, 2016/08/23 22:50:31]

is this needed?

[sakal, 2016/08/25 11:52:48]

I guess not. I just added it as a safety measure.

+    return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
+  }
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
   callback_ = callback;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
   if (!inited_) {
     return WEBRTC_VIDEO_CODEC_OK;
@@ skipping 11 matching lines @@
   rtc::MessageQueueManager::Clear(this);
   inited_ = false;
   use_surface_ = false;
   ALOGD << "EncoderReleaseOnCodecThread done.";
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate,
                                                       uint32_t frame_rate) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
+  if (software_fallback_needed_) {
+    return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
+  }
   frame_rate = (frame_rate < MAX_ALLOWED_VIDEO_FPS) ?
       frame_rate : MAX_ALLOWED_VIDEO_FPS;
   if (last_set_bitrate_kbps_ == new_bit_rate &&
       last_set_fps_ == frame_rate) {
     return WEBRTC_VIDEO_CODEC_OK;
   }
   if (scale_) {
     quality_scaler_.ReportFramerate(frame_rate);
   }
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
   if (new_bit_rate > 0) {
     last_set_bitrate_kbps_ = new_bit_rate;
   }
   if (frame_rate > 0) {
     last_set_fps_ = frame_rate;
   }
   bool ret = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
                                        j_set_rates_method_,
                                        last_set_bitrate_kbps_,
                                        last_set_fps_);
   CHECK_EXCEPTION(jni);

[AlexG, 2016/08/23 22:50:31]

Replace all CHECK_EXCEPTION(jni) with CheckException(jni) and fallback to SW if
Java exception happens - similarly to decoder implementation

[sakal, 2016/08/25 11:52:48]

Done.

   if (!ret) {
     ResetCodecOnCodecThread();
-    return WEBRTC_VIDEO_CODEC_ERROR;
+    return software_fallback_needed_ ? WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE
+                                     : WEBRTC_VIDEO_CODEC_ERROR;
   }
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int MediaCodecVideoEncoder::GetOutputBufferInfoIndex(
     JNIEnv* jni,
     jobject j_output_buffer_info) {
   return GetIntField(jni, j_output_buffer_info, j_info_index_field_);
 }
 
@@ skipping 376 matching lines @@
   return supported_codecs_;
 }
 
 void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
     webrtc::VideoEncoder* encoder) {
   ALOGD << "Destroy video encoder.";
   delete encoder;
 }
 
 }  // namespace webrtc_jni