Deliver video frames on Android, on the decode thread.

webrtc/sdk/android/src/jni/androidmediadecoder_jni.cc

 /*
  *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 18 matching lines @@
 #include "webrtc/common_video/h264/h264_bitstream_parser.h"
 #include "webrtc/common_video/include/i420_buffer_pool.h"
 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
 #include "webrtc/modules/video_coding/utility/vp8_header_parser.h"
 #include "webrtc/sdk/android/src/jni/androidmediacodeccommon.h"
 #include "webrtc/sdk/android/src/jni/classreferenceholder.h"
 #include "webrtc/sdk/android/src/jni/native_handle_impl.h"
 #include "webrtc/sdk/android/src/jni/surfacetexturehelper_jni.h"
 #include "webrtc/system_wrappers/include/logcat_trace_context.h"
 
-using rtc::Bind;
+// Logging macros.
+#define TAG_DECODER "MediaCodecVideoDecoder"
+#ifdef TRACK_BUFFER_TIMING
+#define ALOGV(...)
+__android_log_print(ANDROID_LOG_VERBOSE, TAG_DECODER, __VA_ARGS__)
+#else
+#define ALOGV(...)
+#endif
+#define ALOGD LOG_TAG(rtc::LS_INFO, TAG_DECODER)
+#define ALOGW LOG_TAG(rtc::LS_WARNING, TAG_DECODER)
+#define ALOGE LOG_TAG(rtc::LS_ERROR, TAG_DECODER)
+
+    using rtc::Bind;

[sakal, 2017/03/21 10:05:58]

nit: indentation

[tommi, 2017/03/21 16:21:43]

Done.

 using rtc::Thread;
 using rtc::ThreadManager;
 
 using webrtc::CodecSpecificInfo;
 using webrtc::DecodedImageCallback;
 using webrtc::EncodedImage;
 using webrtc::VideoFrame;
 using webrtc::RTPFragmentationHeader;
 using webrtc::VideoCodec;
 using webrtc::VideoCodecType;
 using webrtc::kVideoCodecH264;
 using webrtc::kVideoCodecVP8;
 using webrtc::kVideoCodecVP9;
 
 namespace webrtc_jni {
 
-// Logging macros.
-#define TAG_DECODER "MediaCodecVideoDecoder"
-#ifdef TRACK_BUFFER_TIMING
-#define ALOGV(...)
-  __android_log_print(ANDROID_LOG_VERBOSE, TAG_DECODER, __VA_ARGS__)
-#else
-#define ALOGV(...)
-#endif
-#define ALOGD LOG_TAG(rtc::LS_INFO, TAG_DECODER)
-#define ALOGW LOG_TAG(rtc::LS_WARNING, TAG_DECODER)
-#define ALOGE LOG_TAG(rtc::LS_ERROR, TAG_DECODER)
-
-enum { kMaxWarningLogFrames = 2 };
-
-class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
-                               public rtc::MessageHandler {
+class MediaCodecVideoDecoder : public webrtc::VideoDecoder {
  public:
   explicit MediaCodecVideoDecoder(
       JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context);
   virtual ~MediaCodecVideoDecoder();
 
   int32_t InitDecode(const VideoCodec* codecSettings, int32_t numberOfCores)
       override;
 
   int32_t Decode(
       const EncodedImage& inputImage, bool missingFrames,
       const RTPFragmentationHeader* fragmentation,
       const CodecSpecificInfo* codecSpecificInfo = NULL,
       int64_t renderTimeMs = -1) override;
 
+  void PollDecodedFrames() override;
+
   int32_t RegisterDecodeCompleteCallback(DecodedImageCallback* callback)
       override;
 
   int32_t Release() override;
 
   bool PrefersLateDecoding() const override { return true; }
 
-  // rtc::MessageHandler implementation.
-  void OnMessage(rtc::Message* msg) override;
-
   const char* ImplementationName() const override;
 
  private:
-  // CHECK-fail if not running on |codec_thread_|.
-  void CheckOnCodecThread();
+  struct DecodedFrame {
+    DecodedFrame(VideoFrame frame,
+                 int decode_time_ms,
+                 int64_t timestamp,
+                 int64_t ntp_timestamp,
+                 rtc::Optional<uint8_t> qp)
+        : frame(std::move(frame)),
+          decode_time_ms(decode_time_ms),
+          qp(std::move(qp)) {
+      frame.set_timestamp(timestamp);
+      frame.set_ntp_time_ms(ntp_timestamp);
+    }
+
+    VideoFrame frame;
+    int decode_time_ms;
+    rtc::Optional<uint8_t> qp;
+  };
+
+  // Returns true if running on |codec_thread_|. Used for DCHECKing.
+  bool IsOnCodecThread();
 
   int32_t InitDecodeOnCodecThread();
   int32_t ResetDecodeOnCodecThread();
   int32_t ReleaseOnCodecThread();
-  int32_t DecodeOnCodecThread(const EncodedImage& inputImage);
+  int32_t DecodeOnCodecThread(const EncodedImage& inputImage,
+                              std::vector<DecodedFrame>* frames);
+  void PollDecodedFramesOnCodecThread(std::vector<DecodedFrame>* frames);
   // Deliver any outputs pending in the MediaCodec to our |callback_| and return
   // true on success.
-  bool DeliverPendingOutputs(JNIEnv* jni, int dequeue_timeout_us);
+  bool DeliverPendingOutputs(JNIEnv* jni,
+                             int dequeue_timeout_us,
+                             std::vector<DecodedFrame>* frames);
   int32_t ProcessHWErrorOnCodecThread();
   void EnableFrameLogOnWarning();
   void ResetVariables();
 
   // Type of video codec.
   VideoCodecType codecType_;
 
   // Render EGL context - owned by factory, should not be allocated/destroyed
   // by VideoDecoder.
   jobject render_egl_context_;
@@ skipping 53 matching lines @@
   jfieldID j_info_index_field_;
   jfieldID j_info_offset_field_;
   jfieldID j_info_size_field_;
   jfieldID j_presentation_timestamp_ms_field_;
   jfieldID j_timestamp_ms_field_;
   jfieldID j_ntp_timestamp_ms_field_;
   jfieldID j_byte_buffer_decode_time_ms_field_;
 
   // Global references; must be deleted in Release().
   std::vector<jobject> input_buffers_;
+
+  // Added to on the codec thread, frames are delivered on the decoder thread.
+  std::vector<DecodedFrame> decoded_frames_;
 };
 
 MediaCodecVideoDecoder::MediaCodecVideoDecoder(
     JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context) :
     codecType_(codecType),
     render_egl_context_(render_egl_context),
     key_frame_required_(true),
     inited_(false),
     sw_fallback_required_(false),
     codec_thread_(new Thread()),
     j_media_codec_video_decoder_class_(
         jni,
         FindClass(jni, "org/webrtc/MediaCodecVideoDecoder")),
           j_media_codec_video_decoder_(
               jni,
               jni->NewObject(*j_media_codec_video_decoder_class_,
                    GetMethodID(jni,
                               *j_media_codec_video_decoder_class_,
                               "<init>",
                               "()V"))) {
   ScopedLocalRefFrame local_ref_frame(jni);
   codec_thread_->SetName("MediaCodecVideoDecoder", NULL);
-  RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoDecoder";
+  RTC_CHECK(codec_thread_->Start());
 
   j_init_decode_method_ = GetMethodID(
       jni, *j_media_codec_video_decoder_class_, "initDecode",
       "(Lorg/webrtc/MediaCodecVideoDecoder$VideoCodecType;"
       "IILorg/webrtc/SurfaceTextureHelper;)Z");
   j_reset_method_ =
       GetMethodID(jni, *j_media_codec_video_decoder_class_, "reset", "(II)V");
   j_release_method_ =
       GetMethodID(jni, *j_media_codec_video_decoder_class_, "release", "()V");
   j_dequeue_input_buffer_method_ = GetMethodID(
@@ skipping 74 matching lines @@
 }
 
 int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst,
     int32_t numberOfCores) {
   ALOGD << "InitDecode.";
   if (inst == NULL) {
     ALOGE << "NULL VideoCodec instance";
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
   // Factory should guard against other codecs being used with us.
-  RTC_CHECK(inst->codecType == codecType_)
+  RTC_DCHECK(inst->codecType == codecType_)
       << "Unsupported codec " << inst->codecType << " for " << codecType_;
 
   if (sw_fallback_required_) {
     ALOGE << "InitDecode() - fallback to SW decoder";
     return WEBRTC_VIDEO_CODEC_OK;
   }
   // Save VideoCodec instance for later.
   if (&codec_ != inst) {
     codec_ = *inst;
   }
   // If maxFramerate is not set then assume 30 fps.
   codec_.maxFramerate = (codec_.maxFramerate >= 1) ? codec_.maxFramerate : 30;
 
   // Call Java init.
   return codec_thread_->Invoke<int32_t>(
       RTC_FROM_HERE,
       Bind(&MediaCodecVideoDecoder::InitDecodeOnCodecThread, this));
 }
 
 void MediaCodecVideoDecoder::ResetVariables() {
-  CheckOnCodecThread();
+  RTC_DCHECK(IsOnCodecThread());
 
   key_frame_required_ = true;
   frames_received_ = 0;
   frames_decoded_ = 0;
   frames_decoded_logged_ = kMaxDecodedLogFrames;
   start_time_ms_ = rtc::TimeMillis();
   current_frames_ = 0;
   current_bytes_ = 0;
   current_decoding_time_ms_ = 0;
   current_delay_time_ms_ = 0;
   pending_frame_qps_.clear();
 }
 
 int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
-  CheckOnCodecThread();
+  RTC_DCHECK(IsOnCodecThread());
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
   ALOGD << "InitDecodeOnCodecThread Type: " << (int)codecType_ << ". "
       << codec_.width << " x " << codec_.height << ". Fps: " <<
       (int)codec_.maxFramerate;
 
   // Release previous codec first if it was allocated before.
   int ret_val = ReleaseOnCodecThread();
   if (ret_val < 0) {
     ALOGE << "Release failure: " << ret_val << " - fallback to SW codec";
@@ skipping 53 matching lines @@
   for (size_t i = 0; i < num_input_buffers; ++i) {
     input_buffers_[i] =
         jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i));
     if (CheckException(jni)) {
       ALOGE << "NewGlobalRef error - fallback to SW codec.";
       sw_fallback_required_ = true;
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
   }
 
-  codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
-
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoDecoder::ResetDecodeOnCodecThread() {
-  CheckOnCodecThread();
+  RTC_DCHECK(IsOnCodecThread());
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
   ALOGD << "ResetDecodeOnCodecThread Type: " << (int)codecType_ << ". "
       << codec_.width << " x " << codec_.height;
   ALOGD << "  Frames received: " << frames_received_ <<
       ". Frames decoded: " << frames_decoded_;
 
   inited_ = false;
-  rtc::MessageQueueManager::Clear(this);
   ResetVariables();
 
   jni->CallVoidMethod(
       *j_media_codec_video_decoder_,
       j_reset_method_,
       codec_.width,
       codec_.height);
 
   if (CheckException(jni)) {
     ALOGE << "Soft reset error - fallback to SW codec.";
     sw_fallback_required_ = true;
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
   inited_ = true;
 
-  codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
-
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoDecoder::Release() {
   ALOGD << "DecoderRelease request";
   return codec_thread_->Invoke<int32_t>(
       RTC_FROM_HERE, Bind(&MediaCodecVideoDecoder::ReleaseOnCodecThread, this));
 }
 
 int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() {
+  RTC_DCHECK(IsOnCodecThread());
   if (!inited_) {
     return WEBRTC_VIDEO_CODEC_OK;
   }
-  CheckOnCodecThread();
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ALOGD << "DecoderReleaseOnCodecThread: Frames received: " <<
       frames_received_ << ". Frames decoded: " << frames_decoded_;
   ScopedLocalRefFrame local_ref_frame(jni);
   for (size_t i = 0; i < input_buffers_.size(); i++) {
     jni->DeleteGlobalRef(input_buffers_[i]);
   }
   input_buffers_.clear();
   jni->CallVoidMethod(*j_media_codec_video_decoder_, j_release_method_);
   surface_texture_helper_ = nullptr;
   inited_ = false;
-  rtc::MessageQueueManager::Clear(this);
   if (CheckException(jni)) {
     ALOGE << "Decoder release exception";
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
   ALOGD << "DecoderReleaseOnCodecThread done";
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
-void MediaCodecVideoDecoder::CheckOnCodecThread() {
-  RTC_CHECK(codec_thread_.get() == ThreadManager::Instance()->CurrentThread())
-      << "Running on wrong thread!";
+bool MediaCodecVideoDecoder::IsOnCodecThread() {
+  return codec_thread_.get() == ThreadManager::Instance()->CurrentThread();
 }
 
 void MediaCodecVideoDecoder::EnableFrameLogOnWarning() {
   // Log next 2 output frames.
+  static const int kMaxWarningLogFrames = 2;
   frames_decoded_logged_ = std::max(
       frames_decoded_logged_, frames_decoded_ + kMaxWarningLogFrames);
 }
 
 int32_t MediaCodecVideoDecoder::ProcessHWErrorOnCodecThread() {
-  CheckOnCodecThread();
+  RTC_DCHECK(IsOnCodecThread());
   int ret_val = ReleaseOnCodecThread();
   if (ret_val < 0) {
     ALOGE << "ProcessHWError: Release failure";
   }
   if (codecType_ == kVideoCodecH264) {
     // For now there is no SW H.264 which can be used as fallback codec.
     // So try to restart hw codec for now.
     ret_val = InitDecodeOnCodecThread();
     ALOGE << "Reset H.264 codec done. Status: " << ret_val;
     if (ret_val == WEBRTC_VIDEO_CODEC_OK) {
@@ skipping 10 matching lines @@
     return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
   }
 }
 
 int32_t MediaCodecVideoDecoder::Decode(
     const EncodedImage& inputImage,
     bool missingFrames,
     const RTPFragmentationHeader* fragmentation,
     const CodecSpecificInfo* codecSpecificInfo,
     int64_t renderTimeMs) {
+  RTC_DCHECK(callback_);
+  RTC_DCHECK(inited_);
+
   if (sw_fallback_required_) {
     ALOGE << "Decode() - fallback to SW codec";
     return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
   }
-  if (callback_ == NULL) {

[sakal, 2017/03/21 10:05:57]

Is this behavior change intended / relevant for the CL?

[tommi, 2017/03/21 16:21:43]

It's intended. The callback should never be null. It must be set before decoding
is started and only unset after the decoder thread has been stopped.

-    ALOGE << "Decode() - callback_ is NULL";
-    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
-  }
   if (inputImage._buffer == NULL && inputImage._length > 0) {
     ALOGE << "Decode() - inputImage is incorrect";
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
-  if (!inited_) {
-    ALOGE << "Decode() - decoder is not initialized";
-    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
-  }
 
   // Check if encoded frame dimension has changed.
   if ((inputImage._encodedWidth * inputImage._encodedHeight > 0) &&
       (inputImage._encodedWidth != codec_.width ||
       inputImage._encodedHeight != codec_.height)) {
     ALOGW << "Input resolution changed from " <<
         codec_.width << " x " << codec_.height << " to " <<
         inputImage._encodedWidth << " x " << inputImage._encodedHeight;
     codec_.width = inputImage._encodedWidth;
     codec_.height = inputImage._encodedHeight;
@@ skipping 24 matching lines @@
     if (!inputImage._completeFrame) {
       ALOGE << "Decode() - complete frame is required";
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
     key_frame_required_ = false;
   }
   if (inputImage._length == 0) {
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
 
-  return codec_thread_->Invoke<int32_t>(
+  ALOGD << "Decode() - about to call DecodeOnCodecThread";

[sakal, 2017/03/21 10:05:57]

nit: Please remove excessive logging.

[tommi, 2017/03/21 16:21:43]

Done.

+  std::vector<DecodedFrame> frames;
+  int32_t ret = codec_thread_->Invoke<int32_t>(
+      RTC_FROM_HERE, Bind(&MediaCodecVideoDecoder::DecodeOnCodecThread, this,
+                          inputImage, &frames));
+  ALOGD << "Decode() - after call to DecodeOnCodecThread";

[sakal, 2017/03/21 10:05:58]

nit: Please remove excessive logging.

[tommi, 2017/03/21 16:21:42]

Done.

+  for (auto& f : frames)
+    callback_->Decoded(f.frame, rtc::Optional<int32_t>(f.decode_time_ms), f.qp);
+  return ret;
+}
+
+void MediaCodecVideoDecoder::PollDecodedFrames() {
+  RTC_DCHECK(callback_);
+
+  std::vector<DecodedFrame> frames;
+  codec_thread_->Invoke<void>(
       RTC_FROM_HERE,
-      Bind(&MediaCodecVideoDecoder::DecodeOnCodecThread, this, inputImage));
+      Bind(&MediaCodecVideoDecoder::PollDecodedFramesOnCodecThread, this,
+           &frames));
+
+  for (auto& f : frames)
+    callback_->Decoded(f.frame, rtc::Optional<int32_t>(f.decode_time_ms), f.qp);
 }
 
 int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
-    const EncodedImage& inputImage) {
-  CheckOnCodecThread();
+    const EncodedImage& inputImage,
+    std::vector<DecodedFrame>* frames) {
+  RTC_DCHECK(IsOnCodecThread());
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
 
   // Try to drain the decoder and wait until output is not too
   // much behind the input.
   if (codecType_ == kVideoCodecH264 &&
       frames_received_ > frames_decoded_ + max_pending_frames_) {
     // Print warning for H.264 only - for VP8/VP9 one frame delay is ok.
     ALOGW << "Decoder is too far behind. Try to drain. Received: " <<
         frames_received_ << ". Decoded: " << frames_decoded_;
     EnableFrameLogOnWarning();
   }
   const int64 drain_start = rtc::TimeMillis();
   while ((frames_received_ > frames_decoded_ + max_pending_frames_) &&
          (rtc::TimeMillis() - drain_start) < kMediaCodecTimeoutMs) {
-    if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) {
+    if (!DeliverPendingOutputs(jni, kMediaCodecPollMs, frames)) {
       ALOGE << "DeliverPendingOutputs error. Frames received: " <<
           frames_received_ << ". Frames decoded: " << frames_decoded_;
       return ProcessHWErrorOnCodecThread();
     }
   }
   if (frames_received_ > frames_decoded_ + max_pending_frames_) {
     ALOGE << "Output buffer dequeue timeout. Frames received: " <<
         frames_received_ << ". Frames decoded: " << frames_decoded_;
     return ProcessHWErrorOnCodecThread();
   }
 
   // Get input buffer.
   int j_input_buffer_index = jni->CallIntMethod(
       *j_media_codec_video_decoder_, j_dequeue_input_buffer_method_);
   if (CheckException(jni) || j_input_buffer_index < 0) {
     ALOGE << "dequeueInputBuffer error: " << j_input_buffer_index <<
         ". Retry DeliverPendingOutputs.";
     EnableFrameLogOnWarning();
     // Try to drain the decoder.
-    if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) {
+    if (!DeliverPendingOutputs(jni, kMediaCodecPollMs, frames)) {
       ALOGE << "DeliverPendingOutputs error. Frames received: " <<
           frames_received_ << ". Frames decoded: " << frames_decoded_;
       return ProcessHWErrorOnCodecThread();
     }
     // Try dequeue input buffer one last time.
     j_input_buffer_index = jni->CallIntMethod(
         *j_media_codec_video_decoder_, j_dequeue_input_buffer_method_);
     if (CheckException(jni) || j_input_buffer_index < 0) {
       ALOGE << "dequeueInputBuffer critical error: " << j_input_buffer_index;
       return ProcessHWErrorOnCodecThread();
     }
   }
 
   // Copy encoded data to Java ByteBuffer.
   jobject j_input_buffer = input_buffers_[j_input_buffer_index];
   uint8_t* buffer =
       reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer));
-  RTC_CHECK(buffer) << "Indirect buffer??";
+  RTC_DCHECK(buffer) << "Indirect buffer??";
   int64_t buffer_capacity = jni->GetDirectBufferCapacity(j_input_buffer);
   if (CheckException(jni) || buffer_capacity < inputImage._length) {
     ALOGE << "Input frame size "<<  inputImage._length <<
         " is bigger than buffer size " << buffer_capacity;
     return ProcessHWErrorOnCodecThread();
   }
   jlong presentation_timestamp_us = static_cast<jlong>(
       static_cast<int64_t>(frames_received_) * 1000000 / codec_.maxFramerate);
   memcpy(buffer, inputImage._buffer, inputImage._length);
 
@@ skipping 32 matching lines @@
       inputImage._length,
       presentation_timestamp_us,
       static_cast<int64_t> (inputImage._timeStamp),
       inputImage.ntp_time_ms_);
   if (CheckException(jni) || !success) {
     ALOGE << "queueInputBuffer error";
     return ProcessHWErrorOnCodecThread();
   }
 
   // Try to drain the decoder
-  if (!DeliverPendingOutputs(jni, 0)) {
+  if (!DeliverPendingOutputs(jni, 0, frames)) {
     ALOGE << "DeliverPendingOutputs error";
     return ProcessHWErrorOnCodecThread();
   }
 
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
+void MediaCodecVideoDecoder::PollDecodedFramesOnCodecThread(
+    std::vector<DecodedFrame>* frames) {
+  RTC_DCHECK(IsOnCodecThread());
+
+  JNIEnv* jni = AttachCurrentThreadIfNeeded();
+  ScopedLocalRefFrame local_ref_frame(jni);
+
+  if (!DeliverPendingOutputs(jni, 0, frames)) {
+    ALOGE << "PollDecodedFramesOnCodecThread: DeliverPendingOutputs error";
+    ProcessHWErrorOnCodecThread();
+  }
+}
+
 bool MediaCodecVideoDecoder::DeliverPendingOutputs(
-    JNIEnv* jni, int dequeue_timeout_ms) {
-  CheckOnCodecThread();
+    JNIEnv* jni,
+    int dequeue_timeout_ms,
+    std::vector<DecodedFrame>* frames) {
+  RTC_DCHECK(IsOnCodecThread());
   if (frames_received_ <= frames_decoded_) {
     // No need to query for output buffers - decoder is drained.
     return true;
   }
   // Get decoder output.
   jobject j_decoder_output_buffer =
       jni->CallObjectMethod(*j_media_codec_video_decoder_,
           use_surface_ ? j_dequeue_texture_buffer_method_
                        : j_dequeue_byte_buffer_method_,
           dequeue_timeout_ms);
@@ skipping 61 matching lines @@
         jni, j_decoder_output_buffer, j_info_size_field_);
     presentation_timestamps_ms = GetLongField(
         jni, j_decoder_output_buffer, j_presentation_timestamp_ms_field_);
     output_timestamps_ms = GetLongField(
         jni, j_decoder_output_buffer, j_timestamp_ms_field_);
     output_ntp_timestamps_ms = GetLongField(
         jni, j_decoder_output_buffer, j_ntp_timestamp_ms_field_);
 
     decode_time_ms = GetLongField(jni, j_decoder_output_buffer,
                                   j_byte_buffer_decode_time_ms_field_);
-    RTC_CHECK_GE(slice_height, height);
+    RTC_DCHECK_GE(slice_height, height);

[sakal, 2017/03/21 10:05:57]

If this check fails it result in reading outside of a buffer. Should we really
change it to a DCHECK?

[tommi, 2017/03/21 16:21:42]

good point, changed back to a CHECK.

 
     if (output_buffer_size < width * height * 3 / 2) {
       ALOGE << "Insufficient output buffer size: " << output_buffer_size;
       return false;
     }
     if (output_buffer_size < stride * height * 3 / 2 &&
         slice_height == height && stride > width) {
       // Some codecs (Exynos) incorrectly report stride information for
       // output byte buffer, so actual stride value need to be corrected.
       stride = output_buffer_size * 2 / (height * 3);
     }
     jobjectArray output_buffers = reinterpret_cast<jobjectArray>(GetObjectField(
         jni, *j_media_codec_video_decoder_, j_output_buffers_field_));
     jobject output_buffer =
         jni->GetObjectArrayElement(output_buffers, output_buffer_index);
     uint8_t* payload = reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(
         output_buffer));
     if (CheckException(jni)) {
       return false;
     }
     payload += output_buffer_offset;
 
     // Create yuv420 frame.
     rtc::scoped_refptr<webrtc::I420Buffer> i420_buffer =
         decoded_frame_pool_.CreateBuffer(width, height);
     if (color_format == COLOR_FormatYUV420Planar) {
-      RTC_CHECK_EQ(0, stride % 2);
+      RTC_DCHECK_EQ(0, stride % 2);
       const int uv_stride = stride / 2;
       const uint8_t* y_ptr = payload;
       const uint8_t* u_ptr = y_ptr + stride * slice_height;
 
       // Note that the case with odd |slice_height| is handled in a special way.
       // The chroma height contained in the payload is rounded down instead of
       // up, making it one row less than what we expect in WebRTC. Therefore, we
       // have to duplicate the last chroma rows for this case. Also, the offset
       // between the Y plane and the U plane is unintuitive for this case. See
       // http://bugs.webrtc.org/6651 for more info.
       const int chroma_width = (width + 1) / 2;
       const int chroma_height =
           (slice_height % 2 == 0) ? (height + 1) / 2 : height / 2;
       const int u_offset = uv_stride * slice_height / 2;
       const uint8_t* v_ptr = u_ptr + u_offset;
       libyuv::CopyPlane(y_ptr, stride,
                         i420_buffer->MutableDataY(), i420_buffer->StrideY(),
                         width, height);
       libyuv::CopyPlane(u_ptr, uv_stride,
                         i420_buffer->MutableDataU(), i420_buffer->StrideU(),
                         chroma_width, chroma_height);
       libyuv::CopyPlane(v_ptr, uv_stride,
                         i420_buffer->MutableDataV(), i420_buffer->StrideV(),
                         chroma_width, chroma_height);
       if (slice_height % 2 == 1) {
-        RTC_CHECK_EQ(height, slice_height);
+        RTC_DCHECK_EQ(height, slice_height);
         // Duplicate the last chroma rows.
         uint8_t* u_last_row_ptr = i420_buffer->MutableDataU() +
                                   chroma_height * i420_buffer->StrideU();
         memcpy(u_last_row_ptr, u_last_row_ptr - i420_buffer->StrideU(),
                i420_buffer->StrideU());
         uint8_t* v_last_row_ptr = i420_buffer->MutableDataV() +
                                   chroma_height * i420_buffer->StrideV();
         memcpy(v_last_row_ptr, v_last_row_ptr - i420_buffer->StrideV(),
                i420_buffer->StrideV());
       }
@@ skipping 48 matching lines @@
         " for last " << statistic_time_ms << " ms.";
     start_time_ms_ = rtc::TimeMillis();
     current_frames_ = 0;
     current_bytes_ = 0;
     current_decoding_time_ms_ = 0;
     current_delay_time_ms_ = 0;
   }
 
   // If the frame was dropped, frame_buffer is left as nullptr.
   if (frame_buffer) {
+    ALOGD << "DeliverPendingOutputs: Have frame on the codec thread.";

[sakal, 2017/03/21 10:05:57]

This log seems a little excessive.

[tommi, 2017/03/21 16:21:42]

Done.

     VideoFrame decoded_frame(frame_buffer, 0, 0, webrtc::kVideoRotation_0);
     decoded_frame.set_timestamp(output_timestamps_ms);
     decoded_frame.set_ntp_time_ms(output_ntp_timestamps_ms);
 
     rtc::Optional<uint8_t> qp = pending_frame_qps_.front();
     pending_frame_qps_.pop_front();
-    callback_->Decoded(decoded_frame, rtc::Optional<int32_t>(decode_time_ms),
-                       qp);
+    decoded_frames_.push_back(DecodedFrame(std::move(decoded_frame),
+                                           decode_time_ms, output_timestamps_ms,
+                                           output_ntp_timestamps_ms, qp));
+    ALOGD << "DeliverPendingOutputs: Decoded frame delivered.";

[sakal, 2017/03/21 10:05:58]

Same here.

[tommi, 2017/03/21 16:21:43]

Done.

+  }
+
+  if (frames) {

[sakal, 2017/03/21 10:05:57]

I don't see frames ever actually being null. DCHECK instead?

[tommi, 2017/03/21 16:21:43]

Done.

+    frames->reserve(frames->size() + decoded_frames_.size());
+    std::move(decoded_frames_.begin(), decoded_frames_.end(),
+              std::back_inserter(*frames));
+    decoded_frames_.clear();
   }
   return true;
 }
 
 int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback(
     DecodedImageCallback* callback) {
   callback_ = callback;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
-void MediaCodecVideoDecoder::OnMessage(rtc::Message* msg) {
-  JNIEnv* jni = AttachCurrentThreadIfNeeded();
-  ScopedLocalRefFrame local_ref_frame(jni);
-  if (!inited_) {
-    return;
-  }
-  // We only ever send one message to |this| directly (not through a Bind()'d
-  // functor), so expect no ID/data.
-  RTC_CHECK(!msg->message_id) << "Unexpected message!";
-  RTC_CHECK(!msg->pdata) << "Unexpected message!";
-  CheckOnCodecThread();
-
-  if (!DeliverPendingOutputs(jni, 0)) {
-    ALOGE << "OnMessage: DeliverPendingOutputs error";
-    ProcessHWErrorOnCodecThread();
-    return;
-  }
-  codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
-}
-
 MediaCodecVideoDecoderFactory::MediaCodecVideoDecoderFactory()
     : egl_context_(nullptr) {
   ALOGD << "MediaCodecVideoDecoderFactory ctor";
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
   jclass j_decoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoDecoder");
   supported_codec_types_.clear();
 
   bool is_vp8_hw_supported = jni->CallStaticBooleanMethod(
       j_decoder_class,
@@ skipping 71 matching lines @@
     webrtc::VideoDecoder* decoder) {
   ALOGD << "Destroy video decoder.";
   delete decoder;
 }
 
 const char* MediaCodecVideoDecoder::ImplementationName() const {
   return "MediaCodec";
 }
 
 }  // namespace webrtc_jni