wip Move SurfaceTexture.updateTexImage() from video renderers into MediaCodecVideoDecoder

talk/app/webrtc/java/jni/androidmediadecoder_jni.cc

 /*
  * libjingle
  * Copyright 2015 Google Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright notice,
@@ skipping 18 matching lines @@
 #include <vector>
 
 #include "talk/app/webrtc/java/jni/androidmediadecoder_jni.h"
 #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h"
 #include "talk/app/webrtc/java/jni/classreferenceholder.h"
 #include "talk/app/webrtc/java/jni/native_handle_impl.h"
 #include "webrtc/base/bind.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/thread.h"
+#include "webrtc/common_video/interface/video_frame_buffer.h"
 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
 #include "webrtc/system_wrappers/interface/logcat_trace_context.h"
 #include "webrtc/system_wrappers/interface/tick_util.h"
 #include "third_party/libyuv/include/libyuv/convert.h"
 #include "third_party/libyuv/include/libyuv/convert_from.h"
 #include "third_party/libyuv/include/libyuv/video_common.h"
 
 using rtc::Bind;
 using rtc::Thread;
 using rtc::ThreadManager;
@@ skipping 30 matching lines @@
 
   int32_t RegisterDecodeCompleteCallback(DecodedImageCallback* callback)
       override;
 
   int32_t Release() override;
 
   int32_t Reset() override;
   // rtc::MessageHandler implementation.
   void OnMessage(rtc::Message* msg) override;
 
+  void OnTextureFrame(int width,
+                      int height,
+                      int64_t timestamp_ns,
+                      const NativeHandleImpl& native_handle);
+  void ReturnTextureFrame();
+
  private:
   // CHECK-fail if not running on |codec_thread_|.
   void CheckOnCodecThread();
 
   int32_t InitDecodeOnCodecThread();
   int32_t ReleaseOnCodecThread();
   int32_t DecodeOnCodecThread(const EncodedImage& inputImage);
   // Deliver any outputs pending in the MediaCodec to our |callback_| and return
   // true on success.
   bool DeliverPendingOutputs(JNIEnv* jni, int dequeue_timeout_us);
+  void DeliverFrame(VideoFrame* frame);
+  void OnTextureFrameOnCodecThread(int width,
+                                   int height,
+                                   int64_t timestamp_ns,
+                                   const NativeHandleImpl& native_handle);
+  void ReturnTextureFrameOnCodecThread();
   int32_t ProcessHWErrorOnCodecThread();
 
   // Type of video codec.
   VideoCodecType codecType_;
 
   bool key_frame_required_;
   bool inited_;
   bool sw_fallback_required_;
   bool use_surface_;
   VideoCodec codec_;
+  // TODO(magjed): Use frame pool instead of |decoded_image_|.

[perkj_webrtc, 2015/09/22 12:40:37]

I would skip this comment or do it now. 

   VideoFrame decoded_image_;
-  NativeHandleImpl native_handle_;
   DecodedImageCallback* callback_;
   int frames_received_;  // Number of frames received by decoder.
   int frames_decoded_;  // Number of frames decoded by decoder.
   int64_t start_time_ms_;  // Start time for statistics.
   int current_frames_;  // Number of frames in the current statistics interval.
   int current_bytes_;  // Encoded bytes in the current statistics interval.
   int current_decoding_time_ms_;  // Overall decoding time in the current second
   uint32_t max_pending_frames_;  // Maximum number of pending input frames
   std::vector<int32_t> timestamps_;
   std::vector<int64_t> ntp_times_ms_;
   std::vector<int64_t> frame_rtc_times_ms_;  // Time when video frame is sent to
                                              // decoder input.
-  int32_t output_timestamp_;  // Last output frame timestamp from timestamps_ Q.
-  int64_t output_ntp_time_ms_; // Last output frame ntp time from
-                               // ntp_times_ms_ queue.
 
   // State that is constant for the lifetime of this object once the ctor
   // returns.
   scoped_ptr<Thread> codec_thread_;  // Thread on which to operate MediaCodec.
   ScopedGlobalRef<jclass> j_media_codec_video_decoder_class_;
   ScopedGlobalRef<jobject> j_media_codec_video_decoder_;
   jmethodID j_init_decode_method_;
   jmethodID j_release_method_;
   jmethodID j_dequeue_input_buffer_method_;
   jmethodID j_queue_input_buffer_method_;
   jmethodID j_dequeue_output_buffer_method_;
+  jmethodID j_return_texture_frame_method_;
   jmethodID j_release_output_buffer_method_;
   // MediaCodecVideoDecoder fields.
   jfieldID j_input_buffers_field_;
   jfieldID j_output_buffers_field_;
   jfieldID j_color_format_field_;
   jfieldID j_width_field_;
   jfieldID j_height_field_;
   jfieldID j_stride_field_;
   jfieldID j_slice_height_field_;
-  jfieldID j_surface_texture_field_;
-  jfieldID j_textureID_field_;
   // MediaCodecVideoDecoder.DecoderOutputBufferInfo fields.
   jfieldID j_info_index_field_;
   jfieldID j_info_offset_field_;
   jfieldID j_info_size_field_;
   jfieldID j_info_presentation_timestamp_us_field_;
 
   // Global references; must be deleted in Release().
   std::vector<jobject> input_buffers_;
-  jobject surface_texture_;
-  jobject previous_surface_texture_;
 
   // Render EGL context - owned by factory, should not be allocated/destroyed
   // by VideoDecoder.
   jobject render_egl_context_;
 };
 
+namespace {
+
+class AndroidTextureBuffer : public webrtc::NativeHandleBuffer {
+ public:
+  AndroidTextureBuffer(int width,
+                       int height,
+                       MediaCodecVideoDecoder* decoder,
+                       const NativeHandleImpl& native_handle)
+      : webrtc::NativeHandleBuffer(&native_handle_, width, height),
+        native_handle_(native_handle),
+        decoder_(decoder) {}
+
+  ~AndroidTextureBuffer() {
+    decoder_->ReturnTextureFrame();
+  }
+
+  rtc::scoped_refptr<VideoFrameBuffer> NativeToI420Buffer() override {
+    // TODO(magjed): Implement this.

[perkj_webrtc, 2015/09/22 12:40:37]

CHECK(false) - should never be called.

+    return nullptr;
+  }
+
+ private:
+  NativeHandleImpl native_handle_;
+  MediaCodecVideoDecoder*const decoder_;

[perkj_webrtc, 2015/09/22 12:40:37]

There is nothing that guarantee the lifetime of decoder_;

+};
+
+}  // anonymous namespace
+
 MediaCodecVideoDecoder::MediaCodecVideoDecoder(
     JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context) :
     codecType_(codecType),
     render_egl_context_(render_egl_context),
     key_frame_required_(true),
     inited_(false),
     sw_fallback_required_(false),
-    surface_texture_(NULL),
-    previous_surface_texture_(NULL),
     codec_thread_(new Thread()),
     j_media_codec_video_decoder_class_(
         jni,
         FindClass(jni, "org/webrtc/MediaCodecVideoDecoder")),
           j_media_codec_video_decoder_(
               jni,
               jni->NewObject(*j_media_codec_video_decoder_class_,
                    GetMethodID(jni,
                               *j_media_codec_video_decoder_class_,
                               "<init>",
                               "()V"))) {
   ScopedLocalRefFrame local_ref_frame(jni);
   codec_thread_->SetName("MediaCodecVideoDecoder", NULL);
   RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoDecoder";
 
   j_init_decode_method_ = GetMethodID(
       jni, *j_media_codec_video_decoder_class_, "initDecode",
-      "(Lorg/webrtc/MediaCodecVideoDecoder$VideoCodecType;"
+      "(JLorg/webrtc/MediaCodecVideoDecoder$VideoCodecType;"
       "IILandroid/opengl/EGLContext;)Z");
   j_release_method_ =
       GetMethodID(jni, *j_media_codec_video_decoder_class_, "release", "()V");
   j_dequeue_input_buffer_method_ = GetMethodID(
       jni, *j_media_codec_video_decoder_class_, "dequeueInputBuffer", "()I");
   j_queue_input_buffer_method_ = GetMethodID(
       jni, *j_media_codec_video_decoder_class_, "queueInputBuffer", "(IIJ)Z");
   j_dequeue_output_buffer_method_ = GetMethodID(
       jni, *j_media_codec_video_decoder_class_, "dequeueOutputBuffer",
       "(I)Lorg/webrtc/MediaCodecVideoDecoder$DecoderOutputBufferInfo;");
+  j_return_texture_frame_method_ = GetMethodID(
+      jni, *j_media_codec_video_decoder_class_, "returnTextureFrame", "()V");
   j_release_output_buffer_method_ = GetMethodID(
       jni, *j_media_codec_video_decoder_class_, "releaseOutputBuffer", "(I)Z");
 
   j_input_buffers_field_ = GetFieldID(
       jni, *j_media_codec_video_decoder_class_,
       "inputBuffers", "[Ljava/nio/ByteBuffer;");
   j_output_buffers_field_ = GetFieldID(
       jni, *j_media_codec_video_decoder_class_,
       "outputBuffers", "[Ljava/nio/ByteBuffer;");
   j_color_format_field_ = GetFieldID(
       jni, *j_media_codec_video_decoder_class_, "colorFormat", "I");
   j_width_field_ = GetFieldID(
       jni, *j_media_codec_video_decoder_class_, "width", "I");
   j_height_field_ = GetFieldID(
       jni, *j_media_codec_video_decoder_class_, "height", "I");
   j_stride_field_ = GetFieldID(
       jni, *j_media_codec_video_decoder_class_, "stride", "I");
   j_slice_height_field_ = GetFieldID(
       jni, *j_media_codec_video_decoder_class_, "sliceHeight", "I");
-  j_textureID_field_ = GetFieldID(
-      jni, *j_media_codec_video_decoder_class_, "textureID", "I");
-  j_surface_texture_field_ = GetFieldID(
-      jni, *j_media_codec_video_decoder_class_, "surfaceTexture",
-      "Landroid/graphics/SurfaceTexture;");
 
   jclass j_decoder_output_buffer_info_class = FindClass(jni,
       "org/webrtc/MediaCodecVideoDecoder$DecoderOutputBufferInfo");
   j_info_index_field_ = GetFieldID(
       jni, j_decoder_output_buffer_info_class, "index", "I");
   j_info_offset_field_ = GetFieldID(
       jni, j_decoder_output_buffer_info_class, "offset", "I");
   j_info_size_field_ = GetFieldID(
       jni, j_decoder_output_buffer_info_class, "size", "I");
   j_info_presentation_timestamp_us_field_ = GetFieldID(
       jni, j_decoder_output_buffer_info_class, "presentationTimestampUs", "J");
 
   CHECK_EXCEPTION(jni) << "MediaCodecVideoDecoder ctor failed";
   use_surface_ = (render_egl_context_ != NULL);
   ALOGD("MediaCodecVideoDecoder ctor. Use surface: %d", use_surface_);
   memset(&codec_, 0, sizeof(codec_));
   AllowBlockingCalls();
 }
 
 MediaCodecVideoDecoder::~MediaCodecVideoDecoder() {
   // Call Release() to ensure no more callbacks to us after we are deleted.
   Release();
-  // Delete global references.
-  JNIEnv* jni = AttachCurrentThreadIfNeeded();
-  if (previous_surface_texture_ != NULL) {
-    jni->DeleteGlobalRef(previous_surface_texture_);
-  }
-  if (surface_texture_ != NULL) {
-    jni->DeleteGlobalRef(surface_texture_);
-  }
 }
 
 int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst,
     int32_t numberOfCores) {
   ALOGD("InitDecode.");
   if (inst == NULL) {
     ALOGE("NULL VideoCodec instance");
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
   // Factory should guard against other codecs being used with us.
@@ skipping 34 matching lines @@
   // Always start with a complete key frame.
   key_frame_required_ = true;
   frames_received_ = 0;
   frames_decoded_ = 0;
 
   jobject j_video_codec_enum = JavaEnumFromIndex(
       jni, "MediaCodecVideoDecoder$VideoCodecType", codecType_);
   bool success = jni->CallBooleanMethod(
       *j_media_codec_video_decoder_,
       j_init_decode_method_,
+      jlongFromPointer(this),
       j_video_codec_enum,
       codec_.width,
       codec_.height,
       use_surface_ ? render_egl_context_ : nullptr);
   if (CheckException(jni) || !success) {
     ALOGE("Codec initialization error - fallback to SW codec.");
     sw_fallback_required_ = true;
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
   inited_ = true;
 
   switch (codecType_) {
     case kVideoCodecVP8:
       max_pending_frames_ = kMaxPendingFramesVp8;
       break;
     case kVideoCodecH264:
       max_pending_frames_ = kMaxPendingFramesH264;
       break;
     default:
       max_pending_frames_ = 0;
   }
   start_time_ms_ = GetCurrentTimeMs();
   current_frames_ = 0;
   current_bytes_ = 0;
   current_decoding_time_ms_ = 0;
-  output_timestamp_ = 0;
-  output_ntp_time_ms_ = 0;
+  decoded_image_.set_timestamp(0);
+  decoded_image_.set_ntp_time_ms(0);
   timestamps_.clear();
   ntp_times_ms_.clear();
   frame_rtc_times_ms_.clear();
 
   jobjectArray input_buffers = (jobjectArray)GetObjectField(
       jni, *j_media_codec_video_decoder_, j_input_buffers_field_);
   size_t num_input_buffers = jni->GetArrayLength(input_buffers);
   input_buffers_.resize(num_input_buffers);
   for (size_t i = 0; i < num_input_buffers; ++i) {
     input_buffers_[i] =
         jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i));
     if (CheckException(jni)) {
       ALOGE("NewGlobalRef error - fallback to SW codec.");
       sw_fallback_required_ = true;
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
   }
 
-  if (use_surface_) {
-    jobject surface_texture = GetObjectField(
-        jni, *j_media_codec_video_decoder_, j_surface_texture_field_);
-    if (previous_surface_texture_ != NULL) {
-      jni->DeleteGlobalRef(previous_surface_texture_);
-    }
-    previous_surface_texture_ = surface_texture_;
-    surface_texture_ = jni->NewGlobalRef(surface_texture);
-  }
   codec_thread_->PostDelayed(kMediaCodecPollMs, this);
 
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoDecoder::Release() {
   ALOGD("DecoderRelease request");
   return codec_thread_->Invoke<int32_t>(
         Bind(&MediaCodecVideoDecoder::ReleaseOnCodecThread, this));
 }
@@ skipping 174 matching lines @@
 
   // Try to drain the decoder
   if (!DeliverPendingOutputs(jni, 0)) {
     ALOGE("DeliverPendingOutputs error");
     return ProcessHWErrorOnCodecThread();
   }
 
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
+void MediaCodecVideoDecoder::DeliverFrame(VideoFrame* frame) {
+  // Pop timestamps from queues and update |frame|.

[perkj_webrtc, 2015/09/22 12:40:37]

please CheckOnCodecThread();

+  if (!timestamps_.empty()) {
+    frame->set_timestamp(timestamps_.front());
+    timestamps_.erase(timestamps_.begin());
+  }
+  if (!ntp_times_ms_.empty()) {
+    frame->set_ntp_time_ms(ntp_times_ms_.front());
+    ntp_times_ms_.erase(ntp_times_ms_.begin());
+  }
+
+  const int32_t callback_status = callback_->Decoded(*frame);
+  if (callback_status > 0) {
+    ALOGE("callback error");
+  }
+}
+
 bool MediaCodecVideoDecoder::DeliverPendingOutputs(
     JNIEnv* jni, int dequeue_timeout_us) {
   if (frames_received_ <= frames_decoded_) {
     // No need to query for output buffers - decoder is drained.
     return true;
   }
   // Get decoder output.
   jobject j_decoder_output_buffer_info = jni->CallObjectMethod(
       *j_media_codec_video_decoder_,
       j_dequeue_output_buffer_method_,
@@ skipping 23 matching lines @@
   }
 
   // Get decoded video frame properties.
   int color_format = GetIntField(jni, *j_media_codec_video_decoder_,
       j_color_format_field_);
   int width = GetIntField(jni, *j_media_codec_video_decoder_, j_width_field_);
   int height = GetIntField(jni, *j_media_codec_video_decoder_, j_height_field_);
   int stride = GetIntField(jni, *j_media_codec_video_decoder_, j_stride_field_);
   int slice_height = GetIntField(jni, *j_media_codec_video_decoder_,
       j_slice_height_field_);
-  int texture_id = GetIntField(jni, *j_media_codec_video_decoder_,
-      j_textureID_field_);
 
   // Extract data from Java ByteBuffer and create output yuv420 frame -
   // for non surface decoding only.
   if (!use_surface_) {
     if (output_buffer_size < width * height * 3 / 2) {
       ALOGE("Insufficient output buffer size: %d", output_buffer_size);
       return false;
     }
     jobjectArray output_buffers = reinterpret_cast<jobjectArray>(GetObjectField(
         jni, *j_media_codec_video_decoder_, j_output_buffers_field_));
@@ skipping 24 matching lines @@
           decoded_image_.buffer(webrtc::kYPlane),
           decoded_image_.stride(webrtc::kYPlane),
           decoded_image_.buffer(webrtc::kUPlane),
           decoded_image_.stride(webrtc::kUPlane),
           decoded_image_.buffer(webrtc::kVPlane),
           decoded_image_.stride(webrtc::kVPlane),
           width, height);
     }
   }
 
-  // Get frame timestamps from a queue.
-  if (timestamps_.size() > 0) {
-    output_timestamp_ = timestamps_.front();
-    timestamps_.erase(timestamps_.begin());
-  }
-  if (ntp_times_ms_.size() > 0) {
-    output_ntp_time_ms_ = ntp_times_ms_.front();
-    ntp_times_ms_.erase(ntp_times_ms_.begin());
-  }
   int64_t frame_decoding_time_ms = 0;
   if (frame_rtc_times_ms_.size() > 0) {
     frame_decoding_time_ms = GetCurrentTimeMs() - frame_rtc_times_ms_.front();
     frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin());
   }
   ALOGV("Decoder frame out # %d. %d x %d. %d x %d. Color: 0x%x. TS: %ld."
       " DecTime: %lld", frames_decoded_, width, height, stride, slice_height,
       color_format, output_timestamps_ms, frame_decoding_time_ms);
 
   // Return output buffer back to codec.
@@ skipping 17 matching lines @@
         current_bytes_ * 8 / statistic_time_ms,
         (current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms,
         current_decoding_time_ms_ / current_frames_, statistic_time_ms);
     start_time_ms_ = GetCurrentTimeMs();
     current_frames_ = 0;
     current_bytes_ = 0;
     current_decoding_time_ms_ = 0;
   }
 
   // Callback - output decoded frame.
-  int32_t callback_status = WEBRTC_VIDEO_CODEC_OK;
   if (use_surface_) {
-    native_handle_.SetTextureObject(surface_texture_, texture_id);
-    VideoFrame texture_image(new rtc::RefCountedObject<JniNativeHandleBuffer>(
-                                 &native_handle_, width, height),
-                             output_timestamp_, 0, webrtc::kVideoRotation_0);
-    texture_image.set_ntp_time_ms(output_ntp_time_ms_);
-    callback_status = callback_->Decoded(texture_image);
+    // We will receive a callback in OnTextureFrame() when the texture frame is
+    // ready to deliver.
   } else {
-    decoded_image_.set_timestamp(output_timestamp_);
-    decoded_image_.set_ntp_time_ms(output_ntp_time_ms_);
-    callback_status = callback_->Decoded(decoded_image_);
-  }
-  if (callback_status > 0) {
-    ALOGE("callback error");
+    DeliverFrame(&decoded_image_);
   }
 
   return true;
 }
 
+void MediaCodecVideoDecoder::OnTextureFrame(
+    int width,
+    int height,
+    int64_t timestamp_ns,
+    const NativeHandleImpl& native_handle) {
+  codec_thread_->Invoke<void>(

[perkj_webrtc, 2015/09/22 12:40:37]

CheckOnCodecThread ?

Can we skip this communication ? I.e keep the polling strategy. Do not mix
strategies.

[AlexG, 2015/09/28 20:06:21]

I agree with perkj@ - keep polling strategy - it will make logic simpler without
adding excessive JNI - Java interaction

+      Bind(&MediaCodecVideoDecoder::OnTextureFrameOnCodecThread, this,
+           width, height, timestamp_ns, native_handle));
+}
+
+void MediaCodecVideoDecoder::OnTextureFrameOnCodecThread(
+    int width,
+    int height,
+    int64_t timestamp_ns,
+    const NativeHandleImpl& native_handle) {
+  VideoFrame texture_frame(new rtc::RefCountedObject<AndroidTextureBuffer>(
+                               width, height, this, native_handle),
+                           0, 0, webrtc::kVideoRotation_0);
+  DeliverFrame(&texture_frame);
+}
+
+void MediaCodecVideoDecoder::ReturnTextureFrame() {
+  codec_thread_->Invoke<void>(
+      Bind(&MediaCodecVideoDecoder::ReturnTextureFrameOnCodecThread, this));
+}
+
+void MediaCodecVideoDecoder::ReturnTextureFrameOnCodecThread() {
+  JNIEnv* jni = AttachCurrentThreadIfNeeded();
+  jni->CallVoidMethod(*j_media_codec_video_decoder_,
+                      j_return_texture_frame_method_);
+}
+
 int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback(
     DecodedImageCallback* callback) {
   callback_ = callback;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoDecoder::Reset() {
   ALOGD("DecoderReset");
   if (!inited_) {
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
@@ skipping 102 matching lines @@
   }
   ALOGE("Can not find HW video decoder for type %d.", (int)type);
   return NULL;
 }
 
 void MediaCodecVideoDecoderFactory::DestroyVideoDecoder(
     webrtc::VideoDecoder* decoder) {
   delete decoder;
 }
 
+JOW(void, MediaCodecVideoDecoder_nativeOnTextureFrame)(
+    JNIEnv* jni,
+    jclass,
+    jlong j_decoder_ptr,
+    jint j_width,
+    jint j_height,
+    jint j_oes_texture_id,
+    jfloatArray j_transform_matrix,
+    jlong j_timestamp_ns) {
+  reinterpret_cast<MediaCodecVideoDecoder*>(j_decoder_ptr)

[AlexG, 2015/09/28 20:06:21]

How do you ensure that j_decoder_ptr is still valid (native C++ is not
destroyed)?

+      ->OnTextureFrame(j_width, j_height, j_timestamp_ns,
+                       NativeHandleImpl(jni, j_oes_texture_id,
+                                        j_transform_matrix));
+}
+
 }  // namespace webrtc_jni