Support Java VideoFrames in MediaCodecVideoEncoder.

webrtc/sdk/android/src/jni/androidmediaencoder_jni.cc

 /*
  *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 106 matching lines @@
       webrtc::EncodedImageCallback* callback) override;
   int32_t Release() override;
   int32_t SetChannelParameters(uint32_t /* packet_loss */,
                                int64_t /* rtt */) override;
   int32_t SetRateAllocation(const webrtc::BitrateAllocation& rate_allocation,
                             uint32_t frame_rate) override;
 
   bool SupportsNativeHandle() const override { return egl_context_ != nullptr; }
   const char* ImplementationName() const override;
 
+  // Fills the input buffer with data from the buffers passed as parameters.
+  bool FillInputBuffer(JNIEnv* jni,
+                       int input_buffer_index,
+                       uint8_t const* buffer_y,
+                       int stride_y,
+                       uint8_t const* buffer_u,
+                       int stride_u,
+                       uint8_t const* buffer_v,
+                       int stride_v);
+
  private:
   class EncodeTask : public rtc::QueuedTask {
    public:
     explicit EncodeTask(rtc::WeakPtr<MediaCodecVideoEncoder> encoder);
     bool Run() override;
 
    private:
     rtc::WeakPtr<MediaCodecVideoEncoder> encoder_;
   };
 
@@ skipping 16 matching lines @@
   // previously-current values are reused instead of the passed parameters
   // (makes it easier to reason about thread-safety).
   int32_t InitEncodeInternal(int width,
                              int height,
                              int kbps,
                              int fps,
                              bool use_surface);
   // Reconfigure to match |frame| in width, height. Also reconfigures the
   // encoder if |frame| is a texture/byte buffer and the encoder is initialized
   // for byte buffer/texture. Returns false if reconfiguring fails.
-  bool MaybeReconfigureEncoder(const webrtc::VideoFrame& frame);
+  bool MaybeReconfigureEncoder(JNIEnv* jni, const webrtc::VideoFrame& frame);
+
+  // Returns true if the frame is a texture frame and we should use surface
+  // based encoding.
+  bool IsTextureFrame(JNIEnv* jni, const webrtc::VideoFrame& frame);
+
   bool EncodeByteBuffer(JNIEnv* jni,
                         bool key_frame,
                         const webrtc::VideoFrame& frame,
                         int input_buffer_index);
   bool EncodeTexture(JNIEnv* jni,
                      bool key_frame,
                      const webrtc::VideoFrame& frame);
+  // Encodes a new style org.webrtc.VideoFrame. Might be a I420 or a texture
+  // frame.
+  bool EncodeJavaFrame(JNIEnv* jni,
+                       bool key_frame,
+                       jobject frame,
+                       int input_buffer_index);
 
   // Helper accessors for MediaCodecVideoEncoder$OutputBufferInfo members.
   int GetOutputBufferInfoIndex(JNIEnv* jni, jobject j_output_buffer_info);
   jobject GetOutputBufferInfoBuffer(JNIEnv* jni, jobject j_output_buffer_info);
   bool GetOutputBufferInfoIsKeyFrame(JNIEnv* jni, jobject j_output_buffer_info);
   jlong GetOutputBufferInfoPresentationTimestampUs(
       JNIEnv* jni, jobject j_output_buffer_info);
 
   // Deliver any outputs pending in the MediaCodec to our |callback_| and return
   // true on success.
@@ skipping 22 matching lines @@
   // State that is constant for the lifetime of this object once the ctor
   // returns.
   rtc::SequencedTaskChecker encoder_queue_checker_;
   ScopedGlobalRef<jclass> j_media_codec_video_encoder_class_;
   ScopedGlobalRef<jobject> j_media_codec_video_encoder_;
   jmethodID j_init_encode_method_;
   jmethodID j_get_input_buffers_method_;
   jmethodID j_dequeue_input_buffer_method_;
   jmethodID j_encode_buffer_method_;
   jmethodID j_encode_texture_method_;
+  jmethodID j_encode_frame_method_;
   jmethodID j_release_method_;
   jmethodID j_set_rates_method_;
   jmethodID j_dequeue_output_buffer_method_;
   jmethodID j_release_output_buffer_method_;
   jfieldID j_color_format_field_;
   jfieldID j_info_index_field_;
   jfieldID j_info_buffer_field_;
   jfieldID j_info_is_key_frame_field_;
   jfieldID j_info_presentation_timestamp_us_field_;
 
+  const JavaVideoFrameFactory video_frame_factory_;
+  ScopedGlobalRef<jclass> j_video_frame_texture_buffer_class_;
+
   // State that is valid only between InitEncode() and the next Release().
   int width_;   // Frame width in pixels.
   int height_;  // Frame height in pixels.
   bool inited_;
   bool use_surface_;
   enum libyuv::FourCC encoder_fourcc_;  // Encoder color space format.
   int last_set_bitrate_kbps_;  // Last-requested bitrate in kbps.
   int last_set_fps_;  // Last-requested frame rate.
   int64_t current_timestamp_us_;  // Current frame timestamps in us.
   int frames_received_;  // Number of frames received by encoder.
@@ skipping 89 matching lines @@
       j_media_codec_video_encoder_class_(
           jni,
           FindClass(jni, "org/webrtc/MediaCodecVideoEncoder")),
       j_media_codec_video_encoder_(
           jni,
           jni->NewObject(*j_media_codec_video_encoder_class_,
                          GetMethodID(jni,
                                      *j_media_codec_video_encoder_class_,
                                      "<init>",
                                      "()V"))),
+      video_frame_factory_(jni),
+      j_video_frame_texture_buffer_class_(
+          jni,
+          FindClass(jni, "org/webrtc/VideoFrame$TextureBuffer")),
       inited_(false),
       use_surface_(false),
       egl_context_(egl_context),
       sw_fallback_required_(false) {
   encoder_queue_checker_.Detach();
 
   jclass j_output_buffer_info_class =
       FindClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo");
   j_init_encode_method_ =
       GetMethodID(jni, *j_media_codec_video_encoder_class_, "initEncode",
                   "(Lorg/webrtc/MediaCodecVideoEncoder$VideoCodecType;"
                   "IIIIILorg/webrtc/EglBase14$Context;)Z");
   j_get_input_buffers_method_ = GetMethodID(
       jni,
       *j_media_codec_video_encoder_class_,
       "getInputBuffers",
       "()[Ljava/nio/ByteBuffer;");
   j_dequeue_input_buffer_method_ = GetMethodID(
       jni, *j_media_codec_video_encoder_class_, "dequeueInputBuffer", "()I");
   j_encode_buffer_method_ = GetMethodID(
       jni, *j_media_codec_video_encoder_class_, "encodeBuffer", "(ZIIJ)Z");
   j_encode_texture_method_ = GetMethodID(
         jni, *j_media_codec_video_encoder_class_, "encodeTexture",
         "(ZI[FJ)Z");
+  j_encode_frame_method_ =
+      GetMethodID(jni, *j_media_codec_video_encoder_class_, "encodeFrame",
+                  "(JZLorg/webrtc/VideoFrame;I)Z");
   j_release_method_ =
       GetMethodID(jni, *j_media_codec_video_encoder_class_, "release", "()V");
   j_set_rates_method_ = GetMethodID(
       jni, *j_media_codec_video_encoder_class_, "setRates", "(II)Z");
   j_dequeue_output_buffer_method_ = GetMethodID(
       jni,
       *j_media_codec_video_encoder_class_,
       "dequeueOutputBuffer",
       "()Lorg/webrtc/MediaCodecVideoEncoder$OutputBufferInfo;");
   j_release_output_buffer_method_ = GetMethodID(
@@ skipping 363 matching lines @@
     return WEBRTC_VIDEO_CODEC_OK;
   }
   consecutive_full_queue_frame_drops_ = 0;
 
   rtc::scoped_refptr<webrtc::VideoFrameBuffer> input_buffer(
       frame.video_frame_buffer());
 
   VideoFrame input_frame(input_buffer, frame.timestamp(),
                          frame.render_time_ms(), frame.rotation());
 
-  if (!MaybeReconfigureEncoder(input_frame)) {
+  if (!MaybeReconfigureEncoder(jni, input_frame)) {
     ALOGE << "Failed to reconfigure encoder.";
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
 
   const bool key_frame =
       frame_types->front() != webrtc::kVideoFrameDelta || send_key_frame;
   bool encode_status = true;
-  if (input_frame.video_frame_buffer()->type() !=
-      webrtc::VideoFrameBuffer::Type::kNative) {
-    int j_input_buffer_index = jni->CallIntMethod(
-        *j_media_codec_video_encoder_, j_dequeue_input_buffer_method_);
+
+  int j_input_buffer_index = -1;
+  if (!use_surface_) {
+    j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_,
+                                              j_dequeue_input_buffer_method_);
     if (CheckException(jni)) {
       ALOGE << "Exception in dequeu input buffer.";
       return ProcessHWErrorOnEncode();
     }
     if (j_input_buffer_index == -1) {
       // Video codec falls behind - no input buffer available.
       ALOGW << "Encoder drop frame - no input buffers available";
       if (frames_received_ > 1) {
         current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
         frames_dropped_media_encoder_++;
       } else {
         // Input buffers are not ready after codec initialization, HW is still
         // allocating thme - this is expected and should not result in drop
         // frame report.
         frames_received_ = 0;
       }
       return WEBRTC_VIDEO_CODEC_OK;  // TODO(fischman): see webrtc bug 2887.
     } else if (j_input_buffer_index == -2) {
       return ProcessHWErrorOnEncode();
     }
+  }
+
+  if (input_frame.video_frame_buffer()->type() !=
+      webrtc::VideoFrameBuffer::Type::kNative) {
     encode_status =
         EncodeByteBuffer(jni, key_frame, input_frame, j_input_buffer_index);
   } else {
-    encode_status = EncodeTexture(jni, key_frame, input_frame);
+    AndroidVideoFrameBuffer* android_buffer =
+        static_cast<AndroidVideoFrameBuffer*>(
+            input_frame.video_frame_buffer().get());
+    switch (android_buffer->android_type()) {
+      case AndroidVideoFrameBuffer::AndroidType::kTextureBuffer:
+        encode_status = EncodeTexture(jni, key_frame, input_frame);
+        break;
+      case AndroidVideoFrameBuffer::AndroidType::kJavaBuffer:
+        encode_status = EncodeJavaFrame(
+            jni, key_frame, video_frame_factory_.ToJavaFrame(jni, input_frame),
+            j_input_buffer_index);
+        break;
+      default:
+        RTC_NOTREACHED();
+        return WEBRTC_VIDEO_CODEC_ERROR;
+    }
   }
 
   if (!encode_status) {
     ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp();
     return ProcessHWErrorOnEncode();
   }
 
   // Save input image timestamps for later output.
   input_frame_infos_.emplace_back(frame_input_time_ms, input_frame.timestamp(),
                                   input_frame.render_time_ms(),
@@ skipping 10 matching lines @@
                                                kMediaCodecPollMs);
   }
 
   if (!DeliverPendingOutputs(jni)) {
     return ProcessHWErrorOnEncode();
   }
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 bool MediaCodecVideoEncoder::MaybeReconfigureEncoder(
+    JNIEnv* jni,
     const webrtc::VideoFrame& frame) {
   RTC_DCHECK_CALLED_SEQUENTIALLY(&encoder_queue_checker_);
 
-  const bool reconfigure_due_to_format = frame.is_texture() != use_surface_;
+  bool is_texture = IsTextureFrame(jni, frame);
+  const bool reconfigure_due_to_format = is_texture != use_surface_;
   const bool reconfigure_due_to_size =
       frame.width() != width_ || frame.height() != height_;
 
   if (reconfigure_due_to_format) {
       ALOGD << "Reconfigure encoder due to format change. "
             << (use_surface_ ?
                 "Reconfiguring to encode from byte buffer." :
                 "Reconfiguring to encode from texture.");
       LogStatistics(true);
   }
   if (reconfigure_due_to_size) {
     ALOGW << "Reconfigure encoder due to frame resolution change from "
         << width_ << " x " << height_ << " to " << frame.width() << " x "
         << frame.height();
     LogStatistics(true);
     width_ = frame.width();
     height_ = frame.height();
   }
 
   if (!reconfigure_due_to_format && !reconfigure_due_to_size)
     return true;
 
   Release();
 
-  return InitEncodeInternal(width_, height_, 0, 0, frame.is_texture()) ==
+  return InitEncodeInternal(width_, height_, 0, 0, is_texture) ==
          WEBRTC_VIDEO_CODEC_OK;
 }
 
+bool MediaCodecVideoEncoder::IsTextureFrame(JNIEnv* jni,
+                                            const webrtc::VideoFrame& frame) {
+  if (frame.video_frame_buffer()->type() !=
+      webrtc::VideoFrameBuffer::Type::kNative) {
+    return false;
+  }
+
+  AndroidVideoFrameBuffer* android_buffer =
+      static_cast<AndroidVideoFrameBuffer*>(frame.video_frame_buffer().get());
+  switch (android_buffer->android_type()) {
+    case AndroidVideoFrameBuffer::AndroidType::kTextureBuffer:
+      return true;
+    case AndroidVideoFrameBuffer::AndroidType::kJavaBuffer:
+      return jni->IsInstanceOf(static_cast<AndroidVideoBuffer*>(android_buffer)
+                                   ->video_frame_buffer(),
+                               *j_video_frame_texture_buffer_class_);
+    default:
+      RTC_NOTREACHED();
+      return false;
+  }
+}
+
 bool MediaCodecVideoEncoder::EncodeByteBuffer(JNIEnv* jni,
                                               bool key_frame,
                                               const webrtc::VideoFrame& frame,
                                               int input_buffer_index) {
   RTC_DCHECK_CALLED_SEQUENTIALLY(&encoder_queue_checker_);
   RTC_CHECK(!use_surface_);
 
+  rtc::scoped_refptr<webrtc::I420BufferInterface> i420_buffer =
+      frame.video_frame_buffer()->ToI420();
+  if (!FillInputBuffer(jni, input_buffer_index, i420_buffer->DataY(),
+                       i420_buffer->StrideY(), i420_buffer->DataU(),
+                       i420_buffer->StrideU(), i420_buffer->DataV(),
+                       i420_buffer->StrideV())) {
+    return false;
+  }
+  bool encode_status = jni->CallBooleanMethod(
+      *j_media_codec_video_encoder_, j_encode_buffer_method_, key_frame,
+      input_buffer_index, yuv_size_, current_timestamp_us_);
+  if (CheckException(jni)) {
+    ALOGE << "Exception in encode buffer.";
+    ProcessHWError(true /* reset_if_fallback_unavailable */);
+    return false;
+  }
+  return encode_status;
+}
+
+bool MediaCodecVideoEncoder::FillInputBuffer(JNIEnv* jni,
+                                             int input_buffer_index,
+                                             uint8_t const* buffer_y,
+                                             int stride_y,
+                                             uint8_t const* buffer_u,
+                                             int stride_u,
+                                             uint8_t const* buffer_v,
+                                             int stride_v) {
   jobject j_input_buffer = input_buffers_[input_buffer_index];
   uint8_t* yuv_buffer =
       reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer));
   if (CheckException(jni)) {
     ALOGE << "Exception in get direct buffer address.";
     ProcessHWError(true /* reset_if_fallback_unavailable */);
     return false;
   }
   RTC_CHECK(yuv_buffer) << "Indirect buffer??";
-  rtc::scoped_refptr<webrtc::I420BufferInterface> i420_buffer =
-      frame.video_frame_buffer()->ToI420();
-  RTC_CHECK(!libyuv::ConvertFromI420(
-      i420_buffer->DataY(), i420_buffer->StrideY(), i420_buffer->DataU(),
-      i420_buffer->StrideU(), i420_buffer->DataV(), i420_buffer->StrideV(),
-      yuv_buffer, width_, width_, height_, encoder_fourcc_))
+
+  RTC_CHECK(!libyuv::ConvertFromI420(buffer_y, stride_y, buffer_u, stride_u,
+                                     buffer_v, stride_v, yuv_buffer, width_,
+                                     width_, height_, encoder_fourcc_))
       << "ConvertFromI420 failed";
-
-  bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
-                                              j_encode_buffer_method_,
-                                              key_frame,
-                                              input_buffer_index,
-                                              yuv_size_,
-                                              current_timestamp_us_);
-  if (CheckException(jni)) {
-    ALOGE << "Exception in encode buffer.";
-    ProcessHWError(true /* reset_if_fallback_unavailable */);
-    return false;
-  }
-  return encode_status;
+  return true;
 }
 
 bool MediaCodecVideoEncoder::EncodeTexture(JNIEnv* jni,
                                            bool key_frame,
                                            const webrtc::VideoFrame& frame) {
   RTC_DCHECK_CALLED_SEQUENTIALLY(&encoder_queue_checker_);
   RTC_CHECK(use_surface_);
   NativeHandleImpl handle =
       static_cast<AndroidTextureBuffer*>(frame.video_frame_buffer().get())
           ->native_handle_impl();
 
   jfloatArray sampling_matrix = handle.sampling_matrix.ToJava(jni);
   bool encode_status = jni->CallBooleanMethod(
       *j_media_codec_video_encoder_, j_encode_texture_method_, key_frame,
       handle.oes_texture_id, sampling_matrix, current_timestamp_us_);
   if (CheckException(jni)) {
     ALOGE << "Exception in encode texture.";
     ProcessHWError(true /* reset_if_fallback_unavailable */);
     return false;
   }
   return encode_status;
 }
 
+bool MediaCodecVideoEncoder::EncodeJavaFrame(JNIEnv* jni,
+                                             bool key_frame,
+                                             jobject frame,
+                                             int input_buffer_index) {
+  bool encode_status = jni->CallBooleanMethod(
+      *j_media_codec_video_encoder_, j_encode_frame_method_,
+      jlongFromPointer(this), key_frame, frame, input_buffer_index);
+  if (CheckException(jni)) {
+    ALOGE << "Exception in encode frame.";
+    ProcessHWError(true /* reset_if_fallback_unavailable */);
+    return false;
+  }
+  return encode_status;
+}
+
 int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallback(
     webrtc::EncodedImageCallback* callback) {
   RTC_DCHECK_CALLED_SEQUENTIALLY(&encoder_queue_checker_);
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
   callback_ = callback;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoEncoder::Release() {
@@ skipping 460 matching lines @@
     return supported_codecs_;
   }
 }
 
 void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
     webrtc::VideoEncoder* encoder) {
   ALOGD << "Destroy video encoder.";
   delete encoder;
 }
 
+JOW(void, MediaCodecVideoEncoder_nativeFillBuffer)
+(JNIEnv* jni,
+ jlong native_encoder,
+ jint input_buffer,
+ jobject j_buffer_y,
+ jint stride_y,
+ jobject j_buffer_u,
+ jint stride_u,
+ jobject j_buffer_v,
+ jint stride_v) {
+  uint8_t* buffer_y =
+      static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_buffer_y));
+  uint8_t* buffer_u =
+      static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_buffer_u));
+  uint8_t* buffer_v =
+      static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_buffer_v));
+
+  reinterpret_cast<MediaCodecVideoEncoder*>(native_encoder)
+      ->FillInputBuffer(jni, input_buffer, buffer_y, stride_y, buffer_u,
+                        stride_u, buffer_v, stride_v);
+}
+
 }  // namespace webrtc_jni