wip Move SurfaceTexture.updateTexImage() from video renderers into MediaCodecVideoDecoder

talk/app/webrtc/java/jni/androidmediadecoder_jni.cc

Index: talk/app/webrtc/java/jni/androidmediadecoder_jni.cc
diff --git a/talk/app/webrtc/java/jni/androidmediadecoder_jni.cc b/talk/app/webrtc/java/jni/androidmediadecoder_jni.cc
index a67dd502ffc433dac2e5b63e2d5a8f079aee66c9..1ca8b8190042ea135b57289ba876517d2e881084 100644
--- a/talk/app/webrtc/java/jni/androidmediadecoder_jni.cc
+++ b/talk/app/webrtc/java/jni/androidmediadecoder_jni.cc
@@ -36,6 +36,7 @@
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/thread.h"
+#include "webrtc/common_video/interface/video_frame_buffer.h"
 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
 #include "webrtc/system_wrappers/interface/logcat_trace_context.h"
 #include "webrtc/system_wrappers/interface/tick_util.h"
@@ -86,6 +87,12 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
   // rtc::MessageHandler implementation.
   void OnMessage(rtc::Message* msg) override;
 
+  void OnTextureFrame(int width,
+                      int height,
+                      int64_t timestamp_ns,
+                      const NativeHandleImpl& native_handle);
+  void ReturnTextureFrame();

[magjed_webrtc, 2015/09/21 18:29:13]

Self review: Make ReturnTextureFrame() private.

[magjed_webrtc, 2015/09/22 09:47:36]

MediaCodecVideoDecoder is an implementation detail and not exposed in the header
file, so making OnTextureFrame() and ReturnTextureFrame() private is not worth
the trouble.

+
  private:
   // CHECK-fail if not running on |codec_thread_|.
   void CheckOnCodecThread();
@@ -96,6 +103,11 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
   // Deliver any outputs pending in the MediaCodec to our |callback_| and return
   // true on success.
   bool DeliverPendingOutputs(JNIEnv* jni, int dequeue_timeout_us);
+  void OnTextureFrameOnCodecThread(int width,
+                                   int height,
+                                   int64_t timestamp_ns,
+                                   const NativeHandleImpl& native_handle);
+  void ReturnTextureFrameOnCodecThread();
   int32_t ProcessHWErrorOnCodecThread();
 
   // Type of video codec.
@@ -107,7 +119,6 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
   bool use_surface_;
   VideoCodec codec_;
   VideoFrame decoded_image_;
-  NativeHandleImpl native_handle_;
   DecodedImageCallback* callback_;
   int frames_received_;  // Number of frames received by decoder.
   int frames_decoded_;  // Number of frames decoded by decoder.
@@ -134,6 +145,7 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
   jmethodID j_dequeue_input_buffer_method_;
   jmethodID j_queue_input_buffer_method_;
   jmethodID j_dequeue_output_buffer_method_;
+  jmethodID j_return_texture_frame_method_;
   jmethodID j_release_output_buffer_method_;
   // MediaCodecVideoDecoder fields.
   jfieldID j_input_buffers_field_;
@@ -143,8 +155,6 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
   jfieldID j_height_field_;
   jfieldID j_stride_field_;
   jfieldID j_slice_height_field_;
-  jfieldID j_surface_texture_field_;
-  jfieldID j_textureID_field_;
   // MediaCodecVideoDecoder.DecoderOutputBufferInfo fields.
   jfieldID j_info_index_field_;
   jfieldID j_info_offset_field_;
@@ -153,8 +163,6 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
 
   // Global references; must be deleted in Release().
   std::vector<jobject> input_buffers_;
-  jobject surface_texture_;
-  jobject previous_surface_texture_;
 
   // Render EGL context - owned by factory, should not be allocated/destroyed
   // by VideoDecoder.
@@ -168,8 +176,6 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder(
     key_frame_required_(true),
     inited_(false),
     sw_fallback_required_(false),
-    surface_texture_(NULL),
-    previous_surface_texture_(NULL),
     codec_thread_(new Thread()),
     j_media_codec_video_decoder_class_(
         jni,
@@ -187,7 +193,7 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder(
 
   j_init_decode_method_ = GetMethodID(
       jni, *j_media_codec_video_decoder_class_, "initDecode",
-      "(Lorg/webrtc/MediaCodecVideoDecoder$VideoCodecType;"
+      "(JLorg/webrtc/MediaCodecVideoDecoder$VideoCodecType;"
       "IILandroid/opengl/EGLContext;)Z");
   j_release_method_ =
       GetMethodID(jni, *j_media_codec_video_decoder_class_, "release", "()V");
@@ -198,6 +204,8 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder(
   j_dequeue_output_buffer_method_ = GetMethodID(
       jni, *j_media_codec_video_decoder_class_, "dequeueOutputBuffer",
       "(I)Lorg/webrtc/MediaCodecVideoDecoder$DecoderOutputBufferInfo;");
+  j_return_texture_frame_method_ = GetMethodID(
+      jni, *j_media_codec_video_decoder_class_, "returnTextureFrame", "()V");
   j_release_output_buffer_method_ = GetMethodID(
       jni, *j_media_codec_video_decoder_class_, "releaseOutputBuffer", "(I)Z");
 
@@ -217,11 +225,6 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder(
       jni, *j_media_codec_video_decoder_class_, "stride", "I");
   j_slice_height_field_ = GetFieldID(
       jni, *j_media_codec_video_decoder_class_, "sliceHeight", "I");
-  j_textureID_field_ = GetFieldID(
-      jni, *j_media_codec_video_decoder_class_, "textureID", "I");
-  j_surface_texture_field_ = GetFieldID(
-      jni, *j_media_codec_video_decoder_class_, "surfaceTexture",
-      "Landroid/graphics/SurfaceTexture;");
 
   jclass j_decoder_output_buffer_info_class = FindClass(jni,
       "org/webrtc/MediaCodecVideoDecoder$DecoderOutputBufferInfo");
@@ -244,14 +247,6 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder(
 MediaCodecVideoDecoder::~MediaCodecVideoDecoder() {
   // Call Release() to ensure no more callbacks to us after we are deleted.
   Release();
-  // Delete global references.
-  JNIEnv* jni = AttachCurrentThreadIfNeeded();
-  if (previous_surface_texture_ != NULL) {
-    jni->DeleteGlobalRef(previous_surface_texture_);
-  }
-  if (surface_texture_ != NULL) {
-    jni->DeleteGlobalRef(surface_texture_);
-  }
 }
 
 int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst,
@@ -306,6 +301,7 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
   bool success = jni->CallBooleanMethod(
       *j_media_codec_video_decoder_,
       j_init_decode_method_,
+      jlongFromPointer(this),
       j_video_codec_enum,
       codec_.width,
       codec_.height,
@@ -351,15 +347,6 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
     }
   }
 
-  if (use_surface_) {
-    jobject surface_texture = GetObjectField(
-        jni, *j_media_codec_video_decoder_, j_surface_texture_field_);
-    if (previous_surface_texture_ != NULL) {
-      jni->DeleteGlobalRef(previous_surface_texture_);
-    }
-    previous_surface_texture_ = surface_texture_;
-    surface_texture_ = jni->NewGlobalRef(surface_texture);
-  }
   codec_thread_->PostDelayed(kMediaCodecPollMs, this);
 
   return WEBRTC_VIDEO_CODEC_OK;
@@ -597,8 +584,6 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs(
   int stride = GetIntField(jni, *j_media_codec_video_decoder_, j_stride_field_);
   int slice_height = GetIntField(jni, *j_media_codec_video_decoder_,
       j_slice_height_field_);
-  int texture_id = GetIntField(jni, *j_media_codec_video_decoder_,
-      j_textureID_field_);
 
   // Extract data from Java ByteBuffer and create output yuv420 frame -
   // for non surface decoding only.
@@ -689,24 +674,82 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs(
   }
 
   // Callback - output decoded frame.
-  int32_t callback_status = WEBRTC_VIDEO_CODEC_OK;
   if (use_surface_) {
-    native_handle_.SetTextureObject(surface_texture_, texture_id);
-    VideoFrame texture_image(new rtc::RefCountedObject<JniNativeHandleBuffer>(
-                                 &native_handle_, width, height),
-                             output_timestamp_, 0, webrtc::kVideoRotation_0);
-    texture_image.set_ntp_time_ms(output_ntp_time_ms_);
-    callback_status = callback_->Decoded(texture_image);
+    // We will receive a callback in OnTextureFrame() when the texture frame is
+    // ready.
+    // TODO: Should probably move the timestamp updates (output_timestamp_,
+    // output_ntp_time_ms_, etc) into a separate function, and call it for
+    // !use_surface_ in this function, and otherwise in OnTextureFrame().
   } else {
     decoded_image_.set_timestamp(output_timestamp_);
     decoded_image_.set_ntp_time_ms(output_ntp_time_ms_);
-    callback_status = callback_->Decoded(decoded_image_);
+    const int32_t callback_status = callback_->Decoded(decoded_image_);
+    if (callback_status > 0) {
+      ALOGE("callback error");
+    }
+  }
+
+  return true;
+}
+
+class AndroidTextureBuffer : public webrtc::NativeHandleBuffer {
+ public:
+  AndroidTextureBuffer(int width,
+                       int height,
+                       MediaCodecVideoDecoder* decoder,
+                       const NativeHandleImpl& native_handle)
+      : webrtc::NativeHandleBuffer(&native_handle_, width, height),
+        native_handle_(native_handle),
+        decoder_(decoder) {}
+
+  ~AndroidTextureBuffer() {
+    decoder_->ReturnTextureFrame();
   }
+
+  rtc::scoped_refptr<VideoFrameBuffer> NativeToI420Buffer() override {
+    // TODO(magjed): Implement this.
+    return nullptr;
+  }
+
+ private:
+  NativeHandleImpl native_handle_;
+  MediaCodecVideoDecoder*const decoder_;
+};
+
+void MediaCodecVideoDecoder::OnTextureFrame(
+    int width,
+    int height,
+    int64_t timestamp_ns,
+    const NativeHandleImpl& native_handle) {
+  codec_thread_->Invoke<void>(
+      Bind(&MediaCodecVideoDecoder::OnTextureFrameOnCodecThread, this,
+           width, height, timestamp_ns, native_handle));
+}
+
+void MediaCodecVideoDecoder::OnTextureFrameOnCodecThread(
+    int width,
+    int height,
+    int64_t timestamp_ns,
+    const NativeHandleImpl& native_handle) {
+  VideoFrame texture_image(new rtc::RefCountedObject<AndroidTextureBuffer>(
+                               width, height, this, native_handle),
+                           output_timestamp_, 0, webrtc::kVideoRotation_0);
+  texture_image.set_ntp_time_ms(output_ntp_time_ms_);
+  const int32_t callback_status = callback_->Decoded(texture_image);
   if (callback_status > 0) {
     ALOGE("callback error");
   }
+}
 
-  return true;
+void MediaCodecVideoDecoder::ReturnTextureFrame() {
+  codec_thread_->Invoke<void>(
+      Bind(&MediaCodecVideoDecoder::ReturnTextureFrameOnCodecThread, this));
+}
+
+void MediaCodecVideoDecoder::ReturnTextureFrameOnCodecThread() {
+  JNIEnv* jni = AttachCurrentThreadIfNeeded();
+  jni->CallVoidMethod(*j_media_codec_video_decoder_,
+                      j_return_texture_frame_method_);
 }
 
 int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback(
@@ -831,5 +874,20 @@ void MediaCodecVideoDecoderFactory::DestroyVideoDecoder(
   delete decoder;
 }
 
+JOW(void, MediaCodecVideoDecoder_nativeOnTextureFrame)(
+    JNIEnv* jni,
+    jclass,
+    jlong j_decoder_ptr,
+    jint j_width,
+    jint j_height,
+    jint j_oes_texture_id,
+    jfloatArray j_transform_matrix,
+    jlong j_timestamp_ns) {
+  reinterpret_cast<MediaCodecVideoDecoder*>(j_decoder_ptr)
+      ->OnTextureFrame(j_width, j_height, j_timestamp_ns,
+                       NativeHandleImpl(jni, j_oes_texture_id,
+                                        j_transform_matrix));
+}
+
 }  // namespace webrtc_jni