Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(194)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: webrtc/api/android/jni/androidmediaencoder_jni.cc

Issue 2263043003: Make MediaCodecEncoder fallback to a software encoder on failure. (Closed) Base URL: https://chromium.googlesource.com/external/webrtc.git@master
Patch Set: Changes according to Stefan's comments. Created 4 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « no previous file | webrtc/video/video_encoder.cc » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: webrtc/api/android/jni/androidmediaencoder_jni.cc
diff --git a/webrtc/api/android/jni/androidmediaencoder_jni.cc b/webrtc/api/android/jni/androidmediaencoder_jni.cc
index 8d0d3b54bd512557401dcf652b94d62d930388f4..e19095725d917d79dacf857314bd38ce873150c2 100644
--- a/webrtc/api/android/jni/androidmediaencoder_jni.cc
+++ b/webrtc/api/android/jni/androidmediaencoder_jni.cc
@@ -129,8 +129,20 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
// ResetCodecOnCodecThread() calls ReleaseOnCodecThread() and
// InitEncodeOnCodecThread() in an attempt to restore the codec to an
// operable state. Necessary after all manner of OMX-layer errors.
+ // Returns true if the codec was reset successfully.
bool ResetCodecOnCodecThread();
+ // Fallback to a software encoder if one is supported else try to reset the
+ // encoder. Called with |reset_if_fallback_unavailable| equal to false from
+ // init/release encoder so that we don't go into infinite recursion.
+ // Returns true if the codec was reset successfully.
+ bool ProcessHWErrorOnCodecThread(bool reset_if_fallback_unavailable);
+
+ // Calls ProcessHWErrorOnCodecThread(true). Returns
+ // WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE if sw_fallback_required_ was set or
+ // WEBRTC_VIDEO_CODEC_ERROR otherwise.
+ int32_t ProcessHWErrorOnEncodeOnCodecThread();
+
// Implementation of webrtc::VideoEncoder methods above, all running on the
// codec thread exclusively.
//
@@ -282,6 +294,8 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
int64_t last_frame_received_ms_;
int frames_received_since_last_key_;
webrtc::VideoCodecMode codec_mode_;
+
+ bool sw_fallback_required_;
};
MediaCodecVideoEncoder::~MediaCodecVideoEncoder() {
@@ -289,25 +303,27 @@ MediaCodecVideoEncoder::~MediaCodecVideoEncoder() {
Release();
}
-MediaCodecVideoEncoder::MediaCodecVideoEncoder(
- JNIEnv* jni, VideoCodecType codecType, jobject egl_context) :
- codecType_(codecType),
- callback_(NULL),
- codec_thread_(new Thread()),
- j_media_codec_video_encoder_class_(
- jni,
- FindClass(jni, "org/webrtc/MediaCodecVideoEncoder")),
- j_media_codec_video_encoder_(
- jni,
- jni->NewObject(*j_media_codec_video_encoder_class_,
- GetMethodID(jni,
- *j_media_codec_video_encoder_class_,
- "<init>",
- "()V"))),
- inited_(false),
- use_surface_(false),
- picture_id_(0),
- egl_context_(egl_context) {
+MediaCodecVideoEncoder::MediaCodecVideoEncoder(JNIEnv* jni,
+ VideoCodecType codecType,
+ jobject egl_context)
+ : codecType_(codecType),
+ callback_(NULL),
+ codec_thread_(new Thread()),
+ j_media_codec_video_encoder_class_(
+ jni,
+ FindClass(jni, "org/webrtc/MediaCodecVideoEncoder")),
+ j_media_codec_video_encoder_(
+ jni,
+ jni->NewObject(*j_media_codec_video_encoder_class_,
+ GetMethodID(jni,
+ *j_media_codec_video_encoder_class_,
+ "<init>",
+ "()V"))),
+ inited_(false),
+ use_surface_(false),
+ picture_id_(0),
+ egl_context_(egl_context),
+ sw_fallback_required_(false) {
ScopedLocalRefFrame local_ref_frame(jni);
// It would be nice to avoid spinning up a new thread per MediaCodec, and
// instead re-use e.g. the PeerConnectionFactory's |worker_thread_|, but bug
@@ -361,7 +377,10 @@ MediaCodecVideoEncoder::MediaCodecVideoEncoder(
GetFieldID(jni, j_output_buffer_info_class, "isKeyFrame", "Z");
j_info_presentation_timestamp_us_field_ = GetFieldID(
jni, j_output_buffer_info_class, "presentationTimestampUs", "J");
- CHECK_EXCEPTION(jni) << "MediaCodecVideoEncoder ctor failed";
+ if (CheckException(jni)) {
+ ALOGW << "MediaCodecVideoEncoder ctor failed.";
+ ProcessHWErrorOnCodecThread(true /* reset_if_fallback_unavailable */);
+ }
srand(time(NULL));
AllowBlockingCalls();
}
@@ -378,7 +397,9 @@ int32_t MediaCodecVideoEncoder::InitEncode(
RTC_CHECK(codec_settings->codecType == codecType_)
<< "Unsupported codec " << codec_settings->codecType << " for "
<< codecType_;
-
+ if (sw_fallback_required_) {
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
codec_mode_ = codec_settings->mode;
int init_width = codec_settings->width;
int init_height = codec_settings->height;
@@ -494,20 +515,45 @@ void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) {
bool MediaCodecVideoEncoder::ResetCodecOnCodecThread() {
RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
ALOGE << "ResetOnCodecThread";
- if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK ||
- InitEncodeOnCodecThread(width_, height_, 0, 0, false) !=
- WEBRTC_VIDEO_CODEC_OK) {
- // TODO(fischman): wouldn't it be nice if there was a way to gracefully
- // degrade to a SW encoder at this point? There isn't one AFAICT :(
- // https://code.google.com/p/webrtc/issues/detail?id=2920
+ if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK) {
+ ALOGE << "Releasing codec failed during reset.";
+ return false;
+ }
+ if (InitEncodeOnCodecThread(width_, height_, 0, 0, false) !=
+ WEBRTC_VIDEO_CODEC_OK) {
+ ALOGE << "Initializing encoder failed during reset.";
return false;
}
return true;
}
+bool MediaCodecVideoEncoder::ProcessHWErrorOnCodecThread(
+ bool reset_if_fallback_unavailable) {
+ ALOGE << "ProcessHWErrorOnCodecThread";
+ if (VideoEncoder::IsSupportedSoftware(
+ VideoEncoder::CodecToEncoderType(codecType_))) {
+ ALOGE << "Fallback to SW encoder.";
+ sw_fallback_required_ = true;
+ return false;
+ } else if (reset_if_fallback_unavailable) {
+ ALOGE << "Reset encoder.";
+ return ResetCodecOnCodecThread();
+ }
+ return false;
+}
+
+int32_t MediaCodecVideoEncoder::ProcessHWErrorOnEncodeOnCodecThread() {
+ ProcessHWErrorOnCodecThread(true /* reset_if_fallback_unavailable */);
+ return sw_fallback_required_ ? WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE
+ : WEBRTC_VIDEO_CODEC_ERROR;
+}
+
int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread(
int width, int height, int kbps, int fps, bool use_surface) {
RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
+ if (sw_fallback_required_) {
+ return WEBRTC_VIDEO_CODEC_OK;
+ }
RTC_CHECK(!use_surface || egl_context_ != nullptr) << "EGL context not set.";
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
@@ -560,16 +606,27 @@ int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread(
(use_surface ? egl_context_ : nullptr));
if (!encode_status) {
ALOGE << "Failed to configure encoder.";
+ ProcessHWErrorOnCodecThread(false /* reset_if_fallback_unavailable */);
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+ if (CheckException(jni)) {
+ ALOGE << "Exception in init encode.";
+ ProcessHWErrorOnCodecThread(false /* reset_if_fallback_unavailable */);
return WEBRTC_VIDEO_CODEC_ERROR;
}
- CHECK_EXCEPTION(jni);
if (!use_surface) {
jobjectArray input_buffers = reinterpret_cast<jobjectArray>(
jni->CallObjectMethod(*j_media_codec_video_encoder_,
j_get_input_buffers_method_));
- CHECK_EXCEPTION(jni);
+ if (CheckException(jni)) {
+ ALOGE << "Exception in get input buffers.";
+ ProcessHWErrorOnCodecThread(false /* reset_if_fallback_unavailable */);
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
+
if (IsNull(jni, input_buffers)) {
+ ProcessHWErrorOnCodecThread(false /* reset_if_fallback_unavailable */);
return WEBRTC_VIDEO_CODEC_ERROR;
}
@@ -585,6 +642,7 @@ int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread(
break;
default:
LOG(LS_ERROR) << "Wrong color format.";
+ ProcessHWErrorOnCodecThread(false /* reset_if_fallback_unavailable */);
return WEBRTC_VIDEO_CODEC_ERROR;
}
size_t num_input_buffers = jni->GetArrayLength(input_buffers);
@@ -596,7 +654,11 @@ int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread(
jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i));
int64_t yuv_buffer_capacity =
jni->GetDirectBufferCapacity(input_buffers_[i]);
- CHECK_EXCEPTION(jni);
+ if (CheckException(jni)) {
+ ALOGE << "Exception in get direct buffer capacity.";
+ ProcessHWErrorOnCodecThread(false /* reset_if_fallback_unavailable */);
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity";
}
}
@@ -610,6 +672,8 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
const std::vector<webrtc::FrameType>* frame_types,
const int64_t frame_input_time_ms) {
RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
+ if (sw_fallback_required_)
+ return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
@@ -636,8 +700,11 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
frames_received_++;
if (!DeliverPendingOutputs(jni)) {
- if (!ResetCodecOnCodecThread())
- return WEBRTC_VIDEO_CODEC_ERROR;
+ if (!ProcessHWErrorOnCodecThread(
+ true /* reset_if_fallback_unavailable */)) {
+ return sw_fallback_required_ ? WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE
+ : WEBRTC_VIDEO_CODEC_ERROR;
+ }
}
if (frames_encoded_ < kMaxEncodedLogFrames) {
ALOGD << "Encoder frame in # " << (frames_received_ - 1)
@@ -669,9 +736,8 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
consecutive_full_queue_frame_drops_++;
if (consecutive_full_queue_frame_drops_ >=
ENCODER_STALL_FRAMEDROP_THRESHOLD) {
- ALOGE << "Encoder got stuck. Reset.";
- ResetCodecOnCodecThread();
- return WEBRTC_VIDEO_CODEC_ERROR;
+ ALOGE << "Encoder got stuck.";
+ return ProcessHWErrorOnEncodeOnCodecThread();
}
frames_dropped_media_encoder_++;
OnDroppedFrameOnCodecThread();
@@ -715,7 +781,10 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
if (!input_frame.video_frame_buffer()->native_handle()) {
int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_,
j_dequeue_input_buffer_method_);
- CHECK_EXCEPTION(jni);
+ if (CheckException(jni)) {
+ ALOGE << "Exception in dequeu input buffer.";
+ return ProcessHWErrorOnEncodeOnCodecThread();
+ }
if (j_input_buffer_index == -1) {
// Video codec falls behind - no input buffer available.
ALOGW << "Encoder drop frame - no input buffers available";
@@ -731,8 +800,7 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
}
return WEBRTC_VIDEO_CODEC_OK; // TODO(fischman): see webrtc bug 2887.
} else if (j_input_buffer_index == -2) {
- ResetCodecOnCodecThread();
- return WEBRTC_VIDEO_CODEC_ERROR;
+ return ProcessHWErrorOnEncodeOnCodecThread();
}
encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame,
j_input_buffer_index);
@@ -742,8 +810,7 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
if (!encode_status) {
ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp();
- ResetCodecOnCodecThread();
- return WEBRTC_VIDEO_CODEC_ERROR;
+ return ProcessHWErrorOnEncodeOnCodecThread();
}
// Save input image timestamps for later output.
@@ -760,9 +827,7 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
if (!DeliverPendingOutputs(jni)) {
- ALOGE << "Failed deliver pending outputs.";
- ResetCodecOnCodecThread();
- return WEBRTC_VIDEO_CODEC_ERROR;
+ return ProcessHWErrorOnEncodeOnCodecThread();
}
return WEBRTC_VIDEO_CODEC_OK;
}
@@ -810,7 +875,11 @@ bool MediaCodecVideoEncoder::EncodeByteBufferOnCodecThread(JNIEnv* jni,
jobject j_input_buffer = input_buffers_[input_buffer_index];
uint8_t* yuv_buffer =
reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer));
- CHECK_EXCEPTION(jni);
+ if (CheckException(jni)) {
+ ALOGE << "Exception in get direct buffer address.";
+ ProcessHWErrorOnCodecThread(true /* reset_if_fallback_unavailable */);
+ return false;
+ }
RTC_CHECK(yuv_buffer) << "Indirect buffer??";
RTC_CHECK(!libyuv::ConvertFromI420(
frame.video_frame_buffer()->DataY(),
@@ -828,7 +897,11 @@ bool MediaCodecVideoEncoder::EncodeByteBufferOnCodecThread(JNIEnv* jni,
input_buffer_index,
yuv_size_,
current_timestamp_us_);
- CHECK_EXCEPTION(jni);
+ if (CheckException(jni)) {
+ ALOGE << "Exception in encode buffer.";
+ ProcessHWErrorOnCodecThread(true /* reset_if_fallback_unavailable */);
+ return false;
+ }
return encode_status;
}
@@ -845,7 +918,11 @@ bool MediaCodecVideoEncoder::EncodeTextureOnCodecThread(JNIEnv* jni,
handle->oes_texture_id,
sampling_matrix,
current_timestamp_us_);
- CHECK_EXCEPTION(jni);
+ if (CheckException(jni)) {
+ ALOGE << "Exception in encode texture.";
+ ProcessHWErrorOnCodecThread(true /* reset_if_fallback_unavailable */);
+ return false;
+ }
return encode_status;
}
@@ -872,7 +949,11 @@ int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() {
jni->DeleteGlobalRef(input_buffers_[i]);
input_buffers_.clear();
jni->CallVoidMethod(*j_media_codec_video_encoder_, j_release_method_);
- CHECK_EXCEPTION(jni);
+ if (CheckException(jni)) {
+ ALOGE << "Exception in release.";
+ ProcessHWErrorOnCodecThread(false /* reset_if_fallback_unavailable */);
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
rtc::MessageQueueManager::Clear(this);
inited_ = false;
use_surface_ = false;
@@ -883,6 +964,8 @@ int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() {
int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate,
uint32_t frame_rate) {
RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
+ if (sw_fallback_required_)
+ return WEBRTC_VIDEO_CODEC_OK;
frame_rate = (frame_rate < MAX_ALLOWED_VIDEO_FPS) ?
frame_rate : MAX_ALLOWED_VIDEO_FPS;
if (last_set_bitrate_kbps_ == new_bit_rate &&
@@ -904,10 +987,10 @@ int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate,
j_set_rates_method_,
last_set_bitrate_kbps_,
last_set_fps_);
- CHECK_EXCEPTION(jni);
- if (!ret) {
- ResetCodecOnCodecThread();
- return WEBRTC_VIDEO_CODEC_ERROR;
+ if (CheckException(jni) || !ret) {
+ ProcessHWErrorOnCodecThread(true /* reset_if_fallback_unavailable */);
+ return sw_fallback_required_ ? WEBRTC_VIDEO_CODEC_OK
+ : WEBRTC_VIDEO_CODEC_ERROR;
}
return WEBRTC_VIDEO_CODEC_OK;
}
@@ -943,7 +1026,11 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) {
while (true) {
jobject j_output_buffer_info = jni->CallObjectMethod(
*j_media_codec_video_encoder_, j_dequeue_output_buffer_method_);
- CHECK_EXCEPTION(jni);
+ if (CheckException(jni)) {
+ ALOGE << "Exception in set dequeue output buffer.";
+ ProcessHWErrorOnCodecThread(true /* reset_if_fallback_unavailable */);
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
if (IsNull(jni, j_output_buffer_info)) {
break;
}
@@ -951,7 +1038,7 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) {
int output_buffer_index =
GetOutputBufferInfoIndex(jni, j_output_buffer_info);
if (output_buffer_index == -1) {
- ResetCodecOnCodecThread();
+ ProcessHWErrorOnCodecThread(true /* reset_if_fallback_unavailable */);
return false;
}
@@ -979,7 +1066,11 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) {
size_t payload_size = jni->GetDirectBufferCapacity(j_output_buffer);
uint8_t* payload = reinterpret_cast<uint8_t*>(
jni->GetDirectBufferAddress(j_output_buffer));
- CHECK_EXCEPTION(jni);
+ if (CheckException(jni)) {
+ ALOGE << "Exception in get direct buffer address.";
+ ProcessHWErrorOnCodecThread(true /* reset_if_fallback_unavailable */);
+ return WEBRTC_VIDEO_CODEC_ERROR;
+ }
// Callback - return encoded frame.
int32_t callback_status = 0;
@@ -1079,7 +1170,7 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) {
ALOGE << "Data:" << image->_buffer[0] << " " << image->_buffer[1]
<< " " << image->_buffer[2] << " " << image->_buffer[3]
<< " " << image->_buffer[4] << " " << image->_buffer[5];
- ResetCodecOnCodecThread();
+ ProcessHWErrorOnCodecThread(true /* reset_if_fallback_unavailable */);
return false;
}
scPositions[scPositionsLength] = payload_size;
@@ -1100,9 +1191,8 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) {
bool success = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
j_release_output_buffer_method_,
output_buffer_index);
- CHECK_EXCEPTION(jni);
- if (!success) {
- ResetCodecOnCodecThread();
+ if (CheckException(jni) || !success) {
+ ProcessHWErrorOnCodecThread(true /* reset_if_fallback_unavailable */);
return false;
}
« no previous file with comments | « no previous file | webrtc/video/video_encoder.cc » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698