Android MediaDecoder: Use frame pool to avoid allocations for non-surface decoding

talk/app/webrtc/java/jni/androidmediadecoder_jni.cc

 /*
  * libjingle
  * Copyright 2015 Google Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright notice,
@@ skipping 17 matching lines @@
 
 #include <vector>
 
 #include "talk/app/webrtc/java/jni/androidmediadecoder_jni.h"
 #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h"
 #include "talk/app/webrtc/java/jni/classreferenceholder.h"
 #include "talk/app/webrtc/java/jni/native_handle_impl.h"
 #include "webrtc/base/bind.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
+#include "webrtc/base/scoped_ref_ptr.h"
 #include "webrtc/base/thread.h"
+#include "webrtc/common_video/interface/i420_buffer_pool.h"
 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
 #include "webrtc/system_wrappers/interface/logcat_trace_context.h"
 #include "webrtc/system_wrappers/interface/tick_util.h"
 #include "third_party/libyuv/include/libyuv/convert.h"
 #include "third_party/libyuv/include/libyuv/convert_from.h"
 #include "third_party/libyuv/include/libyuv/video_common.h"
 
 using rtc::Bind;
 using rtc::Thread;
 using rtc::ThreadManager;
@@ skipping 50 matching lines @@
   int32_t ProcessHWErrorOnCodecThread();
 
   // Type of video codec.
   VideoCodecType codecType_;
 
   bool key_frame_required_;
   bool inited_;
   bool sw_fallback_required_;
   bool use_surface_;
   VideoCodec codec_;
-  VideoFrame decoded_image_;
+  webrtc::I420BufferPool decoded_frame_pool_;
   NativeHandleImpl native_handle_;
   DecodedImageCallback* callback_;
   int frames_received_;  // Number of frames received by decoder.
   int frames_decoded_;  // Number of frames decoded by decoder.
   int64_t start_time_ms_;  // Start time for statistics.
   int current_frames_;  // Number of frames in the current statistics interval.
   int current_bytes_;  // Encoded bytes in the current statistics interval.
   int current_decoding_time_ms_;  // Overall decoding time in the current second
   uint32_t max_pending_frames_;  // Maximum number of pending input frames
   std::vector<int32_t> timestamps_;
   std::vector<int64_t> ntp_times_ms_;
   std::vector<int64_t> frame_rtc_times_ms_;  // Time when video frame is sent to
                                              // decoder input.
-  int32_t output_timestamp_;  // Last output frame timestamp from timestamps_ Q.
-  int64_t output_ntp_time_ms_; // Last output frame ntp time from
-                               // ntp_times_ms_ queue.
 
   // State that is constant for the lifetime of this object once the ctor
   // returns.
   scoped_ptr<Thread> codec_thread_;  // Thread on which to operate MediaCodec.
   ScopedGlobalRef<jclass> j_media_codec_video_decoder_class_;
   ScopedGlobalRef<jobject> j_media_codec_video_decoder_;
   jmethodID j_init_decode_method_;
   jmethodID j_release_method_;
   jmethodID j_dequeue_input_buffer_method_;
   jmethodID j_queue_input_buffer_method_;
@@ skipping 188 matching lines @@
     case kVideoCodecH264:
       max_pending_frames_ = kMaxPendingFramesH264;
       break;
     default:
       max_pending_frames_ = 0;
   }
   start_time_ms_ = GetCurrentTimeMs();
   current_frames_ = 0;
   current_bytes_ = 0;
   current_decoding_time_ms_ = 0;
-  output_timestamp_ = 0;
-  output_ntp_time_ms_ = 0;
   timestamps_.clear();
   ntp_times_ms_.clear();
   frame_rtc_times_ms_.clear();
 
   jobjectArray input_buffers = (jobjectArray)GetObjectField(
       jni, *j_media_codec_video_decoder_, j_input_buffers_field_);
   size_t num_input_buffers = jni->GetArrayLength(input_buffers);
   input_buffers_.resize(num_input_buffers);
   for (size_t i = 0; i < num_input_buffers; ++i) {
     input_buffers_[i] =
@@ skipping 247 matching lines @@
   int color_format = GetIntField(jni, *j_media_codec_video_decoder_,
       j_color_format_field_);
   int width = GetIntField(jni, *j_media_codec_video_decoder_, j_width_field_);
   int height = GetIntField(jni, *j_media_codec_video_decoder_, j_height_field_);
   int stride = GetIntField(jni, *j_media_codec_video_decoder_, j_stride_field_);
   int slice_height = GetIntField(jni, *j_media_codec_video_decoder_,
       j_slice_height_field_);
   int texture_id = GetIntField(jni, *j_media_codec_video_decoder_,
       j_textureID_field_);
 
-  // Extract data from Java ByteBuffer and create output yuv420 frame -
-  // for non surface decoding only.
-  if (!use_surface_) {
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer;
+  if (use_surface_) {
+    native_handle_.SetTextureObject(surface_texture_, texture_id);
+    frame_buffer = new rtc::RefCountedObject<JniNativeHandleBuffer>(
+        &native_handle_, width, height);
+  } else {
+    // Extract data from Java ByteBuffer and create output yuv420 frame -
+    // for non surface decoding only.
     if (output_buffer_size < width * height * 3 / 2) {
       ALOGE("Insufficient output buffer size: %d", output_buffer_size);
       return false;
     }
     jobjectArray output_buffers = reinterpret_cast<jobjectArray>(GetObjectField(
         jni, *j_media_codec_video_decoder_, j_output_buffers_field_));
     jobject output_buffer =
         jni->GetObjectArrayElement(output_buffers, output_buffer_index);
     uint8_t* payload = reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(
         output_buffer));
     if (CheckException(jni)) {
       return false;
     }
     payload += output_buffer_offset;
 
     // Create yuv420 frame.
+    frame_buffer = decoded_frame_pool_.CreateBuffer(width, height);
     if (color_format == COLOR_FormatYUV420Planar) {
-      decoded_image_.CreateFrame(
-          payload,
-          payload + (stride * slice_height),
-          payload + (5 * stride * slice_height / 4),
-          width, height,
-          stride, stride / 2, stride / 2);
+      RTC_CHECK_EQ(0, stride % 2);
+      RTC_CHECK_EQ(0, slice_height % 2);
+      const int uv_stride = stride / 2;
+      const int u_slice_height = slice_height / 2;
+      const uint8_t* y_ptr = payload;
+      const uint8_t* u_ptr = y_ptr + stride * slice_height;
+      const uint8_t* v_ptr = u_ptr + uv_stride * u_slice_height;
+      libyuv::I420Copy(y_ptr, stride,
+                       u_ptr, uv_stride,
+                       v_ptr, uv_stride,
+                       frame_buffer->MutableData(webrtc::kYPlane),
+                       frame_buffer->stride(webrtc::kYPlane),
+                       frame_buffer->MutableData(webrtc::kUPlane),
+                       frame_buffer->stride(webrtc::kUPlane),
+                       frame_buffer->MutableData(webrtc::kVPlane),
+                       frame_buffer->stride(webrtc::kVPlane),
+                       width, height);
     } else {
       // All other supported formats are nv12.
-      decoded_image_.CreateEmptyFrame(width, height, width,
-          width / 2, width / 2);
+      const uint8_t* y_ptr = payload;
+      const uint8_t* uv_ptr = y_ptr + stride * slice_height;
       libyuv::NV12ToI420(
-          payload, stride,
-          payload + stride * slice_height, stride,
-          decoded_image_.buffer(webrtc::kYPlane),
-          decoded_image_.stride(webrtc::kYPlane),
-          decoded_image_.buffer(webrtc::kUPlane),
-          decoded_image_.stride(webrtc::kUPlane),
-          decoded_image_.buffer(webrtc::kVPlane),
-          decoded_image_.stride(webrtc::kVPlane),
+          y_ptr, stride,
+          uv_ptr, stride,
+          frame_buffer->MutableData(webrtc::kYPlane),
+          frame_buffer->stride(webrtc::kYPlane),
+          frame_buffer->MutableData(webrtc::kUPlane),
+          frame_buffer->stride(webrtc::kUPlane),
+          frame_buffer->MutableData(webrtc::kVPlane),
+          frame_buffer->stride(webrtc::kVPlane),
           width, height);
     }
   }
+  VideoFrame decoded_frame(frame_buffer, 0, 0, webrtc::kVideoRotation_0);
 
   // Get frame timestamps from a queue.
   if (timestamps_.size() > 0) {
-    output_timestamp_ = timestamps_.front();
+    decoded_frame.set_timestamp(timestamps_.front());
     timestamps_.erase(timestamps_.begin());
   }
   if (ntp_times_ms_.size() > 0) {
-    output_ntp_time_ms_ = ntp_times_ms_.front();
+    decoded_frame.set_ntp_time_ms(ntp_times_ms_.front());
     ntp_times_ms_.erase(ntp_times_ms_.begin());
   }
   int64_t frame_decoding_time_ms = 0;
   if (frame_rtc_times_ms_.size() > 0) {
     frame_decoding_time_ms = GetCurrentTimeMs() - frame_rtc_times_ms_.front();
     frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin());
   }
   ALOGV("Decoder frame out # %d. %d x %d. %d x %d. Color: 0x%x. TS: %ld."
       " DecTime: %lld", frames_decoded_, width, height, stride, slice_height,
       color_format, output_timestamps_ms, frame_decoding_time_ms);
@@ skipping 19 matching lines @@
         current_bytes_ * 8 / statistic_time_ms,
         (current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms,
         current_decoding_time_ms_ / current_frames_, statistic_time_ms);
     start_time_ms_ = GetCurrentTimeMs();
     current_frames_ = 0;
     current_bytes_ = 0;
     current_decoding_time_ms_ = 0;
   }
 
   // Callback - output decoded frame.
-  int32_t callback_status = WEBRTC_VIDEO_CODEC_OK;
-  if (use_surface_) {
-    native_handle_.SetTextureObject(surface_texture_, texture_id);
-    VideoFrame texture_image(new rtc::RefCountedObject<JniNativeHandleBuffer>(
-                                 &native_handle_, width, height),
-                             output_timestamp_, 0, webrtc::kVideoRotation_0);
-    texture_image.set_ntp_time_ms(output_ntp_time_ms_);
-    callback_status = callback_->Decoded(texture_image);
-  } else {
-    decoded_image_.set_timestamp(output_timestamp_);
-    decoded_image_.set_ntp_time_ms(output_ntp_time_ms_);
-    callback_status = callback_->Decoded(decoded_image_);
-  }
+  const int32_t callback_status = callback_->Decoded(decoded_frame);
   if (callback_status > 0) {
     ALOGE("callback error");
   }
 
   return true;
 }
 
 int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback(
     DecodedImageCallback* callback) {
   callback_ = callback;
@@ skipping 111 matching lines @@
   return NULL;
 }
 
 void MediaCodecVideoDecoderFactory::DestroyVideoDecoder(
     webrtc::VideoDecoder* decoder) {
   delete decoder;
 }
 
 }  // namespace webrtc_jni