Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(40)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: webrtc/sdk/android/src/jni/androidmediadecoder_jni.cc

Issue 2764573002: Deliver video frames on Android, on the decode thread. (Closed)
Patch Set: Created 3 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « webrtc/modules/video_coding/video_receiver.cc ('k') | webrtc/video/video_receive_stream.h » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: webrtc/sdk/android/src/jni/androidmediadecoder_jni.cc
diff --git a/webrtc/sdk/android/src/jni/androidmediadecoder_jni.cc b/webrtc/sdk/android/src/jni/androidmediadecoder_jni.cc
index d30e94c58fbbb9ed3424a2959ce52cd0253fd003..a9f1756beaa1b9634803d20a0ab65cab1b271777 100644
--- a/webrtc/sdk/android/src/jni/androidmediadecoder_jni.cc
+++ b/webrtc/sdk/android/src/jni/androidmediadecoder_jni.cc
@@ -36,7 +36,19 @@
#include "webrtc/sdk/android/src/jni/surfacetexturehelper_jni.h"
#include "webrtc/system_wrappers/include/logcat_trace_context.h"
-using rtc::Bind;
+// Logging macros.
+#define TAG_DECODER "MediaCodecVideoDecoder"
+#ifdef TRACK_BUFFER_TIMING
+#define ALOGV(...)
+__android_log_print(ANDROID_LOG_VERBOSE, TAG_DECODER, __VA_ARGS__)
+#else
+#define ALOGV(...)
+#endif
+#define ALOGD LOG_TAG(rtc::LS_INFO, TAG_DECODER)
+#define ALOGW LOG_TAG(rtc::LS_WARNING, TAG_DECODER)
+#define ALOGE LOG_TAG(rtc::LS_ERROR, TAG_DECODER)
+
+ using rtc::Bind;
sakal 2017/03/21 10:05:58 nit: indentation
tommi 2017/03/21 16:21:43 Done.
using rtc::Thread;
using rtc::ThreadManager;
@@ -53,22 +65,7 @@ using webrtc::kVideoCodecVP9;
namespace webrtc_jni {
-// Logging macros.
-#define TAG_DECODER "MediaCodecVideoDecoder"
-#ifdef TRACK_BUFFER_TIMING
-#define ALOGV(...)
- __android_log_print(ANDROID_LOG_VERBOSE, TAG_DECODER, __VA_ARGS__)
-#else
-#define ALOGV(...)
-#endif
-#define ALOGD LOG_TAG(rtc::LS_INFO, TAG_DECODER)
-#define ALOGW LOG_TAG(rtc::LS_WARNING, TAG_DECODER)
-#define ALOGE LOG_TAG(rtc::LS_ERROR, TAG_DECODER)
-
-enum { kMaxWarningLogFrames = 2 };
-
-class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
- public rtc::MessageHandler {
+class MediaCodecVideoDecoder : public webrtc::VideoDecoder {
public:
explicit MediaCodecVideoDecoder(
JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context);
@@ -83,6 +80,8 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
const CodecSpecificInfo* codecSpecificInfo = NULL,
int64_t renderTimeMs = -1) override;
+ void PollDecodedFrames() override;
+
int32_t RegisterDecodeCompleteCallback(DecodedImageCallback* callback)
override;
@@ -90,22 +89,41 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
bool PrefersLateDecoding() const override { return true; }
- // rtc::MessageHandler implementation.
- void OnMessage(rtc::Message* msg) override;
-
const char* ImplementationName() const override;
private:
- // CHECK-fail if not running on |codec_thread_|.
- void CheckOnCodecThread();
+ struct DecodedFrame {
+ DecodedFrame(VideoFrame frame,
+ int decode_time_ms,
+ int64_t timestamp,
+ int64_t ntp_timestamp,
+ rtc::Optional<uint8_t> qp)
+ : frame(std::move(frame)),
+ decode_time_ms(decode_time_ms),
+ qp(std::move(qp)) {
+ frame.set_timestamp(timestamp);
+ frame.set_ntp_time_ms(ntp_timestamp);
+ }
+
+ VideoFrame frame;
+ int decode_time_ms;
+ rtc::Optional<uint8_t> qp;
+ };
+
+ // Returns true if running on |codec_thread_|. Used for DCHECKing.
+ bool IsOnCodecThread();
int32_t InitDecodeOnCodecThread();
int32_t ResetDecodeOnCodecThread();
int32_t ReleaseOnCodecThread();
- int32_t DecodeOnCodecThread(const EncodedImage& inputImage);
+ int32_t DecodeOnCodecThread(const EncodedImage& inputImage,
+ std::vector<DecodedFrame>* frames);
+ void PollDecodedFramesOnCodecThread(std::vector<DecodedFrame>* frames);
// Deliver any outputs pending in the MediaCodec to our |callback_| and return
// true on success.
- bool DeliverPendingOutputs(JNIEnv* jni, int dequeue_timeout_us);
+ bool DeliverPendingOutputs(JNIEnv* jni,
+ int dequeue_timeout_us,
+ std::vector<DecodedFrame>* frames);
int32_t ProcessHWErrorOnCodecThread();
void EnableFrameLogOnWarning();
void ResetVariables();
@@ -179,6 +197,9 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
// Global references; must be deleted in Release().
std::vector<jobject> input_buffers_;
+
+ // Added to on the codec thread, frames are delivered on the decoder thread.
+ std::vector<DecodedFrame> decoded_frames_;
};
MediaCodecVideoDecoder::MediaCodecVideoDecoder(
@@ -201,7 +222,7 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder(
"()V"))) {
ScopedLocalRefFrame local_ref_frame(jni);
codec_thread_->SetName("MediaCodecVideoDecoder", NULL);
- RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoDecoder";
+ RTC_CHECK(codec_thread_->Start());
j_init_decode_method_ = GetMethodID(
jni, *j_media_codec_video_decoder_class_, "initDecode",
@@ -296,7 +317,7 @@ int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst,
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
// Factory should guard against other codecs being used with us.
- RTC_CHECK(inst->codecType == codecType_)
+ RTC_DCHECK(inst->codecType == codecType_)
<< "Unsupported codec " << inst->codecType << " for " << codecType_;
if (sw_fallback_required_) {
@@ -317,7 +338,7 @@ int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst,
}
void MediaCodecVideoDecoder::ResetVariables() {
- CheckOnCodecThread();
+ RTC_DCHECK(IsOnCodecThread());
key_frame_required_ = true;
frames_received_ = 0;
@@ -332,7 +353,7 @@ void MediaCodecVideoDecoder::ResetVariables() {
}
int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
- CheckOnCodecThread();
+ RTC_DCHECK(IsOnCodecThread());
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
ALOGD << "InitDecodeOnCodecThread Type: " << (int)codecType_ << ". "
@@ -406,13 +427,11 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
}
}
- codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
-
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t MediaCodecVideoDecoder::ResetDecodeOnCodecThread() {
- CheckOnCodecThread();
+ RTC_DCHECK(IsOnCodecThread());
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
ALOGD << "ResetDecodeOnCodecThread Type: " << (int)codecType_ << ". "
@@ -421,7 +440,6 @@ int32_t MediaCodecVideoDecoder::ResetDecodeOnCodecThread() {
". Frames decoded: " << frames_decoded_;
inited_ = false;
- rtc::MessageQueueManager::Clear(this);
ResetVariables();
jni->CallVoidMethod(
@@ -437,8 +455,6 @@ int32_t MediaCodecVideoDecoder::ResetDecodeOnCodecThread() {
}
inited_ = true;
- codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
-
return WEBRTC_VIDEO_CODEC_OK;
}
@@ -449,10 +465,10 @@ int32_t MediaCodecVideoDecoder::Release() {
}
int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() {
+ RTC_DCHECK(IsOnCodecThread());
if (!inited_) {
return WEBRTC_VIDEO_CODEC_OK;
}
- CheckOnCodecThread();
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ALOGD << "DecoderReleaseOnCodecThread: Frames received: " <<
frames_received_ << ". Frames decoded: " << frames_decoded_;
@@ -464,7 +480,6 @@ int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() {
jni->CallVoidMethod(*j_media_codec_video_decoder_, j_release_method_);
surface_texture_helper_ = nullptr;
inited_ = false;
- rtc::MessageQueueManager::Clear(this);
if (CheckException(jni)) {
ALOGE << "Decoder release exception";
return WEBRTC_VIDEO_CODEC_ERROR;
@@ -473,19 +488,19 @@ int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() {
return WEBRTC_VIDEO_CODEC_OK;
}
-void MediaCodecVideoDecoder::CheckOnCodecThread() {
- RTC_CHECK(codec_thread_.get() == ThreadManager::Instance()->CurrentThread())
- << "Running on wrong thread!";
+bool MediaCodecVideoDecoder::IsOnCodecThread() {
+ return codec_thread_.get() == ThreadManager::Instance()->CurrentThread();
}
void MediaCodecVideoDecoder::EnableFrameLogOnWarning() {
// Log next 2 output frames.
+ static const int kMaxWarningLogFrames = 2;
frames_decoded_logged_ = std::max(
frames_decoded_logged_, frames_decoded_ + kMaxWarningLogFrames);
}
int32_t MediaCodecVideoDecoder::ProcessHWErrorOnCodecThread() {
- CheckOnCodecThread();
+ RTC_DCHECK(IsOnCodecThread());
int ret_val = ReleaseOnCodecThread();
if (ret_val < 0) {
ALOGE << "ProcessHWError: Release failure";
@@ -516,22 +531,17 @@ int32_t MediaCodecVideoDecoder::Decode(
const RTPFragmentationHeader* fragmentation,
const CodecSpecificInfo* codecSpecificInfo,
int64_t renderTimeMs) {
+ RTC_DCHECK(callback_);
+ RTC_DCHECK(inited_);
+
if (sw_fallback_required_) {
ALOGE << "Decode() - fallback to SW codec";
return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
}
- if (callback_ == NULL) {
sakal 2017/03/21 10:05:57 Is this behavior change intended / relevant for th
tommi 2017/03/21 16:21:43 It's intended. The callback should never be null.
- ALOGE << "Decode() - callback_ is NULL";
- return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
- }
if (inputImage._buffer == NULL && inputImage._length > 0) {
ALOGE << "Decode() - inputImage is incorrect";
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
- if (!inited_) {
- ALOGE << "Decode() - decoder is not initialized";
- return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
- }
// Check if encoded frame dimension has changed.
if ((inputImage._encodedWidth * inputImage._encodedHeight > 0) &&
@@ -576,14 +586,34 @@ int32_t MediaCodecVideoDecoder::Decode(
return WEBRTC_VIDEO_CODEC_ERROR;
}
- return codec_thread_->Invoke<int32_t>(
+ ALOGD << "Decode() - about to call DecodeOnCodecThread";
sakal 2017/03/21 10:05:57 nit: Please remove excessive logging.
tommi 2017/03/21 16:21:43 Done.
+ std::vector<DecodedFrame> frames;
+ int32_t ret = codec_thread_->Invoke<int32_t>(
+ RTC_FROM_HERE, Bind(&MediaCodecVideoDecoder::DecodeOnCodecThread, this,
+ inputImage, &frames));
+ ALOGD << "Decode() - after call to DecodeOnCodecThread";
sakal 2017/03/21 10:05:58 nit: Please remove excessive logging.
tommi 2017/03/21 16:21:42 Done.
+ for (auto& f : frames)
+ callback_->Decoded(f.frame, rtc::Optional<int32_t>(f.decode_time_ms), f.qp);
+ return ret;
+}
+
+void MediaCodecVideoDecoder::PollDecodedFrames() {
+ RTC_DCHECK(callback_);
+
+ std::vector<DecodedFrame> frames;
+ codec_thread_->Invoke<void>(
RTC_FROM_HERE,
- Bind(&MediaCodecVideoDecoder::DecodeOnCodecThread, this, inputImage));
+ Bind(&MediaCodecVideoDecoder::PollDecodedFramesOnCodecThread, this,
+ &frames));
+
+ for (auto& f : frames)
+ callback_->Decoded(f.frame, rtc::Optional<int32_t>(f.decode_time_ms), f.qp);
}
int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
- const EncodedImage& inputImage) {
- CheckOnCodecThread();
+ const EncodedImage& inputImage,
+ std::vector<DecodedFrame>* frames) {
+ RTC_DCHECK(IsOnCodecThread());
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
@@ -599,7 +629,7 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
const int64 drain_start = rtc::TimeMillis();
while ((frames_received_ > frames_decoded_ + max_pending_frames_) &&
(rtc::TimeMillis() - drain_start) < kMediaCodecTimeoutMs) {
- if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) {
+ if (!DeliverPendingOutputs(jni, kMediaCodecPollMs, frames)) {
ALOGE << "DeliverPendingOutputs error. Frames received: " <<
frames_received_ << ". Frames decoded: " << frames_decoded_;
return ProcessHWErrorOnCodecThread();
@@ -619,7 +649,7 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
". Retry DeliverPendingOutputs.";
EnableFrameLogOnWarning();
// Try to drain the decoder.
- if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) {
+ if (!DeliverPendingOutputs(jni, kMediaCodecPollMs, frames)) {
ALOGE << "DeliverPendingOutputs error. Frames received: " <<
frames_received_ << ". Frames decoded: " << frames_decoded_;
return ProcessHWErrorOnCodecThread();
@@ -637,7 +667,7 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
jobject j_input_buffer = input_buffers_[j_input_buffer_index];
uint8_t* buffer =
reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer));
- RTC_CHECK(buffer) << "Indirect buffer??";
+ RTC_DCHECK(buffer) << "Indirect buffer??";
int64_t buffer_capacity = jni->GetDirectBufferCapacity(j_input_buffer);
if (CheckException(jni) || buffer_capacity < inputImage._length) {
ALOGE << "Input frame size "<< inputImage._length <<
@@ -690,7 +720,7 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
}
// Try to drain the decoder
- if (!DeliverPendingOutputs(jni, 0)) {
+ if (!DeliverPendingOutputs(jni, 0, frames)) {
ALOGE << "DeliverPendingOutputs error";
return ProcessHWErrorOnCodecThread();
}
@@ -698,9 +728,24 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
return WEBRTC_VIDEO_CODEC_OK;
}
+void MediaCodecVideoDecoder::PollDecodedFramesOnCodecThread(
+ std::vector<DecodedFrame>* frames) {
+ RTC_DCHECK(IsOnCodecThread());
+
+ JNIEnv* jni = AttachCurrentThreadIfNeeded();
+ ScopedLocalRefFrame local_ref_frame(jni);
+
+ if (!DeliverPendingOutputs(jni, 0, frames)) {
+ ALOGE << "PollDecodedFramesOnCodecThread: DeliverPendingOutputs error";
+ ProcessHWErrorOnCodecThread();
+ }
+}
+
bool MediaCodecVideoDecoder::DeliverPendingOutputs(
- JNIEnv* jni, int dequeue_timeout_ms) {
- CheckOnCodecThread();
+ JNIEnv* jni,
+ int dequeue_timeout_ms,
+ std::vector<DecodedFrame>* frames) {
+ RTC_DCHECK(IsOnCodecThread());
if (frames_received_ <= frames_decoded_) {
// No need to query for output buffers - decoder is drained.
return true;
@@ -782,7 +827,7 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs(
decode_time_ms = GetLongField(jni, j_decoder_output_buffer,
j_byte_buffer_decode_time_ms_field_);
- RTC_CHECK_GE(slice_height, height);
+ RTC_DCHECK_GE(slice_height, height);
sakal 2017/03/21 10:05:57 If this check fails it result in reading outside o
tommi 2017/03/21 16:21:42 good point, changed back to a CHECK.
if (output_buffer_size < width * height * 3 / 2) {
ALOGE << "Insufficient output buffer size: " << output_buffer_size;
@@ -809,7 +854,7 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs(
rtc::scoped_refptr<webrtc::I420Buffer> i420_buffer =
decoded_frame_pool_.CreateBuffer(width, height);
if (color_format == COLOR_FormatYUV420Planar) {
- RTC_CHECK_EQ(0, stride % 2);
+ RTC_DCHECK_EQ(0, stride % 2);
const int uv_stride = stride / 2;
const uint8_t* y_ptr = payload;
const uint8_t* u_ptr = y_ptr + stride * slice_height;
@@ -835,7 +880,7 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs(
i420_buffer->MutableDataV(), i420_buffer->StrideV(),
chroma_width, chroma_height);
if (slice_height % 2 == 1) {
- RTC_CHECK_EQ(height, slice_height);
+ RTC_DCHECK_EQ(height, slice_height);
// Duplicate the last chroma rows.
uint8_t* u_last_row_ptr = i420_buffer->MutableDataU() +
chroma_height * i420_buffer->StrideU();
@@ -904,14 +949,24 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs(
// If the frame was dropped, frame_buffer is left as nullptr.
if (frame_buffer) {
+ ALOGD << "DeliverPendingOutputs: Have frame on the codec thread.";
sakal 2017/03/21 10:05:57 This log seems a little excessive.
tommi 2017/03/21 16:21:42 Done.
VideoFrame decoded_frame(frame_buffer, 0, 0, webrtc::kVideoRotation_0);
decoded_frame.set_timestamp(output_timestamps_ms);
decoded_frame.set_ntp_time_ms(output_ntp_timestamps_ms);
rtc::Optional<uint8_t> qp = pending_frame_qps_.front();
pending_frame_qps_.pop_front();
- callback_->Decoded(decoded_frame, rtc::Optional<int32_t>(decode_time_ms),
- qp);
+ decoded_frames_.push_back(DecodedFrame(std::move(decoded_frame),
+ decode_time_ms, output_timestamps_ms,
+ output_ntp_timestamps_ms, qp));
+ ALOGD << "DeliverPendingOutputs: Decoded frame delivered.";
sakal 2017/03/21 10:05:58 Same here.
tommi 2017/03/21 16:21:43 Done.
+ }
+
+ if (frames) {
sakal 2017/03/21 10:05:57 I don't see frames ever actually being null. DCHEC
tommi 2017/03/21 16:21:43 Done.
+ frames->reserve(frames->size() + decoded_frames_.size());
+ std::move(decoded_frames_.begin(), decoded_frames_.end(),
+ std::back_inserter(*frames));
+ decoded_frames_.clear();
}
return true;
}
@@ -922,26 +977,6 @@ int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback(
return WEBRTC_VIDEO_CODEC_OK;
}
-void MediaCodecVideoDecoder::OnMessage(rtc::Message* msg) {
- JNIEnv* jni = AttachCurrentThreadIfNeeded();
- ScopedLocalRefFrame local_ref_frame(jni);
- if (!inited_) {
- return;
- }
- // We only ever send one message to |this| directly (not through a Bind()'d
- // functor), so expect no ID/data.
- RTC_CHECK(!msg->message_id) << "Unexpected message!";
- RTC_CHECK(!msg->pdata) << "Unexpected message!";
- CheckOnCodecThread();
-
- if (!DeliverPendingOutputs(jni, 0)) {
- ALOGE << "OnMessage: DeliverPendingOutputs error";
- ProcessHWErrorOnCodecThread();
- return;
- }
- codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
-}
-
MediaCodecVideoDecoderFactory::MediaCodecVideoDecoderFactory()
: egl_context_(nullptr) {
ALOGD << "MediaCodecVideoDecoderFactory ctor";
« no previous file with comments | « webrtc/modules/video_coding/video_receiver.cc ('k') | webrtc/video/video_receive_stream.h » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698