Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(48)

Side by Side Diff: talk/app/webrtc/java/jni/androidmediadecoder_jni.cc

Issue 1440343002: Reland again Android MediaCodecVideoDecoder: Manage lifetime of texture frames (Closed) Base URL: https://chromium.googlesource.com/external/webrtc.git@master
Patch Set: Drop frames if texture is not returned Created 5 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 /* 1 /*
2 * libjingle 2 * libjingle
3 * Copyright 2015 Google Inc. 3 * Copyright 2015 Google Inc.
4 * 4 *
5 * Redistribution and use in source and binary forms, with or without 5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met: 6 * modification, are permitted provided that the following conditions are met:
7 * 7 *
8 * 1. Redistributions of source code must retain the above copyright notice, 8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer. 9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice, 10 * 2. Redistributions in binary form must reproduce the above copyright notice,
(...skipping 15 matching lines...) Expand all
26 * 26 *
27 */ 27 */
28 28
29 #include <algorithm> 29 #include <algorithm>
30 #include <vector> 30 #include <vector>
31 31
32 #include "talk/app/webrtc/java/jni/androidmediadecoder_jni.h" 32 #include "talk/app/webrtc/java/jni/androidmediadecoder_jni.h"
33 #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h" 33 #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h"
34 #include "talk/app/webrtc/java/jni/classreferenceholder.h" 34 #include "talk/app/webrtc/java/jni/classreferenceholder.h"
35 #include "talk/app/webrtc/java/jni/native_handle_impl.h" 35 #include "talk/app/webrtc/java/jni/native_handle_impl.h"
36 #include "talk/app/webrtc/java/jni/surfacetexturehelper_jni.h"
36 #include "webrtc/base/bind.h" 37 #include "webrtc/base/bind.h"
37 #include "webrtc/base/checks.h" 38 #include "webrtc/base/checks.h"
38 #include "webrtc/base/logging.h" 39 #include "webrtc/base/logging.h"
39 #include "webrtc/base/scoped_ref_ptr.h" 40 #include "webrtc/base/scoped_ref_ptr.h"
40 #include "webrtc/base/thread.h" 41 #include "webrtc/base/thread.h"
41 #include "webrtc/base/timeutils.h" 42 #include "webrtc/base/timeutils.h"
42 #include "webrtc/common_video/interface/i420_buffer_pool.h" 43 #include "webrtc/common_video/interface/i420_buffer_pool.h"
43 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" 44 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
44 #include "webrtc/system_wrappers/include/logcat_trace_context.h" 45 #include "webrtc/system_wrappers/include/logcat_trace_context.h"
45 #include "webrtc/system_wrappers/include/tick_util.h" 46 #include "webrtc/system_wrappers/include/tick_util.h"
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after
105 106
106 // Type of video codec. 107 // Type of video codec.
107 VideoCodecType codecType_; 108 VideoCodecType codecType_;
108 109
109 bool key_frame_required_; 110 bool key_frame_required_;
110 bool inited_; 111 bool inited_;
111 bool sw_fallback_required_; 112 bool sw_fallback_required_;
112 bool use_surface_; 113 bool use_surface_;
113 VideoCodec codec_; 114 VideoCodec codec_;
114 webrtc::I420BufferPool decoded_frame_pool_; 115 webrtc::I420BufferPool decoded_frame_pool_;
115 NativeHandleImpl native_handle_; 116 rtc::scoped_refptr<SurfaceTextureHelper> surface_texture_helper_;
116 DecodedImageCallback* callback_; 117 DecodedImageCallback* callback_;
117 int frames_received_; // Number of frames received by decoder. 118 int frames_received_; // Number of frames received by decoder.
118 int frames_decoded_; // Number of frames decoded by decoder. 119 int frames_decoded_; // Number of frames decoded by decoder.
119 int64_t start_time_ms_; // Start time for statistics. 120 int64_t start_time_ms_; // Start time for statistics.
120 int current_frames_; // Number of frames in the current statistics interval. 121 int current_frames_; // Number of frames in the current statistics interval.
121 int current_bytes_; // Encoded bytes in the current statistics interval. 122 int current_bytes_; // Encoded bytes in the current statistics interval.
122 int current_decoding_time_ms_; // Overall decoding time in the current second 123 int current_decoding_time_ms_; // Overall decoding time in the current second
123 uint32_t max_pending_frames_; // Maximum number of pending input frames 124 uint32_t max_pending_frames_; // Maximum number of pending input frames
124 std::vector<int32_t> timestamps_; 125 std::vector<int32_t> timestamps_;
125 std::vector<int64_t> ntp_times_ms_; 126 std::vector<int64_t> ntp_times_ms_;
126 std::vector<int64_t> frame_rtc_times_ms_; // Time when video frame is sent to
127 // decoder input.
128 127
129 // State that is constant for the lifetime of this object once the ctor 128 // State that is constant for the lifetime of this object once the ctor
130 // returns. 129 // returns.
131 scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec. 130 scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec.
132 ScopedGlobalRef<jclass> j_media_codec_video_decoder_class_; 131 ScopedGlobalRef<jclass> j_media_codec_video_decoder_class_;
133 ScopedGlobalRef<jobject> j_media_codec_video_decoder_; 132 ScopedGlobalRef<jobject> j_media_codec_video_decoder_;
134 jmethodID j_init_decode_method_; 133 jmethodID j_init_decode_method_;
135 jmethodID j_release_method_; 134 jmethodID j_release_method_;
136 jmethodID j_dequeue_input_buffer_method_; 135 jmethodID j_dequeue_input_buffer_method_;
137 jmethodID j_queue_input_buffer_method_; 136 jmethodID j_queue_input_buffer_method_;
138 jmethodID j_dequeue_output_buffer_method_; 137 jmethodID j_dequeue_byte_buffer_method_;
138 jmethodID j_dequeue_texture_buffer_method_;
139 jmethodID j_return_decoded_byte_buffer_method_; 139 jmethodID j_return_decoded_byte_buffer_method_;
140 // MediaCodecVideoDecoder fields. 140 // MediaCodecVideoDecoder fields.
141 jfieldID j_input_buffers_field_; 141 jfieldID j_input_buffers_field_;
142 jfieldID j_output_buffers_field_; 142 jfieldID j_output_buffers_field_;
143 jfieldID j_color_format_field_; 143 jfieldID j_color_format_field_;
144 jfieldID j_width_field_; 144 jfieldID j_width_field_;
145 jfieldID j_height_field_; 145 jfieldID j_height_field_;
146 jfieldID j_stride_field_; 146 jfieldID j_stride_field_;
147 jfieldID j_slice_height_field_; 147 jfieldID j_slice_height_field_;
148 jfieldID j_surface_texture_field_;
149 // MediaCodecVideoDecoder.DecodedTextureBuffer fields. 148 // MediaCodecVideoDecoder.DecodedTextureBuffer fields.
150 jfieldID j_textureID_field_; 149 jfieldID j_texture_id_field_;
150 jfieldID j_transform_matrix_field_;
151 jfieldID j_texture_presentation_timestamp_us_field_; 151 jfieldID j_texture_presentation_timestamp_us_field_;
152 // MediaCodecVideoDecoder.DecodedByteBuffer fields. 152 jfieldID j_texture_decode_time_ms_field_;
153 jfieldID j_texture_frame_delay_ms_field_;
154 // MediaCodecVideoDecoder.DecodedOutputBuffer fields.
153 jfieldID j_info_index_field_; 155 jfieldID j_info_index_field_;
154 jfieldID j_info_offset_field_; 156 jfieldID j_info_offset_field_;
155 jfieldID j_info_size_field_; 157 jfieldID j_info_size_field_;
156 jfieldID j_info_presentation_timestamp_us_field_; 158 jfieldID j_info_presentation_timestamp_us_field_;
159 jfieldID j_byte_buffer_decode_time_ms_field_;
157 160
158 // Global references; must be deleted in Release(). 161 // Global references; must be deleted in Release().
159 std::vector<jobject> input_buffers_; 162 std::vector<jobject> input_buffers_;
160 jobject surface_texture_;
161 jobject previous_surface_texture_;
162 163
163 // Render EGL context - owned by factory, should not be allocated/destroyed 164 // Render EGL context - owned by factory, should not be allocated/destroyed
164 // by VideoDecoder. 165 // by VideoDecoder.
165 jobject render_egl_context_; 166 jobject render_egl_context_;
166 }; 167 };
167 168
168 MediaCodecVideoDecoder::MediaCodecVideoDecoder( 169 MediaCodecVideoDecoder::MediaCodecVideoDecoder(
169 JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context) : 170 JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context) :
170 codecType_(codecType), 171 codecType_(codecType),
171 render_egl_context_(render_egl_context), 172 render_egl_context_(render_egl_context),
172 key_frame_required_(true), 173 key_frame_required_(true),
173 inited_(false), 174 inited_(false),
174 sw_fallback_required_(false), 175 sw_fallback_required_(false),
175 surface_texture_(NULL),
176 previous_surface_texture_(NULL),
177 codec_thread_(new Thread()), 176 codec_thread_(new Thread()),
178 j_media_codec_video_decoder_class_( 177 j_media_codec_video_decoder_class_(
179 jni, 178 jni,
180 FindClass(jni, "org/webrtc/MediaCodecVideoDecoder")), 179 FindClass(jni, "org/webrtc/MediaCodecVideoDecoder")),
181 j_media_codec_video_decoder_( 180 j_media_codec_video_decoder_(
182 jni, 181 jni,
183 jni->NewObject(*j_media_codec_video_decoder_class_, 182 jni->NewObject(*j_media_codec_video_decoder_class_,
184 GetMethodID(jni, 183 GetMethodID(jni,
185 *j_media_codec_video_decoder_class_, 184 *j_media_codec_video_decoder_class_,
186 "<init>", 185 "<init>",
187 "()V"))) { 186 "()V"))) {
188 ScopedLocalRefFrame local_ref_frame(jni); 187 ScopedLocalRefFrame local_ref_frame(jni);
189 codec_thread_->SetName("MediaCodecVideoDecoder", NULL); 188 codec_thread_->SetName("MediaCodecVideoDecoder", NULL);
190 RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoDecoder"; 189 RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoDecoder";
191 190
192 j_init_decode_method_ = GetMethodID( 191 j_init_decode_method_ = GetMethodID(
193 jni, *j_media_codec_video_decoder_class_, "initDecode", 192 jni, *j_media_codec_video_decoder_class_, "initDecode",
194 "(Lorg/webrtc/MediaCodecVideoDecoder$VideoCodecType;" 193 "(Lorg/webrtc/MediaCodecVideoDecoder$VideoCodecType;"
195 "IILjavax/microedition/khronos/egl/EGLContext;)Z"); 194 "IILorg/webrtc/SurfaceTextureHelper;)Z");
196 j_release_method_ = 195 j_release_method_ =
197 GetMethodID(jni, *j_media_codec_video_decoder_class_, "release", "()V"); 196 GetMethodID(jni, *j_media_codec_video_decoder_class_, "release", "()V");
198 j_dequeue_input_buffer_method_ = GetMethodID( 197 j_dequeue_input_buffer_method_ = GetMethodID(
199 jni, *j_media_codec_video_decoder_class_, "dequeueInputBuffer", "()I"); 198 jni, *j_media_codec_video_decoder_class_, "dequeueInputBuffer", "()I");
200 j_queue_input_buffer_method_ = GetMethodID( 199 j_queue_input_buffer_method_ = GetMethodID(
201 jni, *j_media_codec_video_decoder_class_, "queueInputBuffer", "(IIJ)Z"); 200 jni, *j_media_codec_video_decoder_class_, "queueInputBuffer", "(IIJ)Z");
202 j_dequeue_output_buffer_method_ = GetMethodID( 201 j_dequeue_byte_buffer_method_ = GetMethodID(
203 jni, *j_media_codec_video_decoder_class_, "dequeueOutputBuffer", 202 jni, *j_media_codec_video_decoder_class_, "dequeueOutputBuffer",
204 "(I)Ljava/lang/Object;"); 203 "(I)Lorg/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer;");
204 j_dequeue_texture_buffer_method_ = GetMethodID(
205 jni, *j_media_codec_video_decoder_class_, "dequeueTextureBuffer",
206 "(I)Lorg/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer;");
205 j_return_decoded_byte_buffer_method_ = 207 j_return_decoded_byte_buffer_method_ =
206 GetMethodID(jni, *j_media_codec_video_decoder_class_, 208 GetMethodID(jni, *j_media_codec_video_decoder_class_,
207 "returnDecodedByteBuffer", "(I)V"); 209 "returnDecodedOutputBuffer", "(I)V");
208 210
209 j_input_buffers_field_ = GetFieldID( 211 j_input_buffers_field_ = GetFieldID(
210 jni, *j_media_codec_video_decoder_class_, 212 jni, *j_media_codec_video_decoder_class_,
211 "inputBuffers", "[Ljava/nio/ByteBuffer;"); 213 "inputBuffers", "[Ljava/nio/ByteBuffer;");
212 j_output_buffers_field_ = GetFieldID( 214 j_output_buffers_field_ = GetFieldID(
213 jni, *j_media_codec_video_decoder_class_, 215 jni, *j_media_codec_video_decoder_class_,
214 "outputBuffers", "[Ljava/nio/ByteBuffer;"); 216 "outputBuffers", "[Ljava/nio/ByteBuffer;");
215 j_color_format_field_ = GetFieldID( 217 j_color_format_field_ = GetFieldID(
216 jni, *j_media_codec_video_decoder_class_, "colorFormat", "I"); 218 jni, *j_media_codec_video_decoder_class_, "colorFormat", "I");
217 j_width_field_ = GetFieldID( 219 j_width_field_ = GetFieldID(
218 jni, *j_media_codec_video_decoder_class_, "width", "I"); 220 jni, *j_media_codec_video_decoder_class_, "width", "I");
219 j_height_field_ = GetFieldID( 221 j_height_field_ = GetFieldID(
220 jni, *j_media_codec_video_decoder_class_, "height", "I"); 222 jni, *j_media_codec_video_decoder_class_, "height", "I");
221 j_stride_field_ = GetFieldID( 223 j_stride_field_ = GetFieldID(
222 jni, *j_media_codec_video_decoder_class_, "stride", "I"); 224 jni, *j_media_codec_video_decoder_class_, "stride", "I");
223 j_slice_height_field_ = GetFieldID( 225 j_slice_height_field_ = GetFieldID(
224 jni, *j_media_codec_video_decoder_class_, "sliceHeight", "I"); 226 jni, *j_media_codec_video_decoder_class_, "sliceHeight", "I");
225 j_surface_texture_field_ = GetFieldID(
226 jni, *j_media_codec_video_decoder_class_, "surfaceTexture",
227 "Landroid/graphics/SurfaceTexture;");
228 227
229 jclass j_decoder_decoded_texture_buffer_class = FindClass(jni, 228 jclass j_decoded_texture_buffer_class = FindClass(jni,
230 "org/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer"); 229 "org/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer");
231 j_textureID_field_ = GetFieldID( 230 j_texture_id_field_ = GetFieldID(
232 jni, j_decoder_decoded_texture_buffer_class, "textureID", "I"); 231 jni, j_decoded_texture_buffer_class, "textureID", "I");
233 j_texture_presentation_timestamp_us_field_ = 232 j_transform_matrix_field_ = GetFieldID(
234 GetFieldID(jni, j_decoder_decoded_texture_buffer_class, 233 jni, j_decoded_texture_buffer_class, "transformMatrix", "[F");
235 "presentationTimestampUs", "J"); 234 j_texture_presentation_timestamp_us_field_ = GetFieldID(
235 jni, j_decoded_texture_buffer_class, "presentationTimestampUs", "J");
236 j_texture_decode_time_ms_field_ = GetFieldID(
237 jni, j_decoded_texture_buffer_class, "decodeTimeMs", "J");
238 j_texture_frame_delay_ms_field_ = GetFieldID(
239 jni, j_decoded_texture_buffer_class, "frameDelayMs", "J");
236 240
237 jclass j_decoder_decoded_byte_buffer_class = FindClass(jni, 241 jclass j_decoded_output_buffer_class = FindClass(jni,
238 "org/webrtc/MediaCodecVideoDecoder$DecodedByteBuffer"); 242 "org/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer");
239 j_info_index_field_ = GetFieldID( 243 j_info_index_field_ = GetFieldID(
240 jni, j_decoder_decoded_byte_buffer_class, "index", "I"); 244 jni, j_decoded_output_buffer_class, "index", "I");
241 j_info_offset_field_ = GetFieldID( 245 j_info_offset_field_ = GetFieldID(
242 jni, j_decoder_decoded_byte_buffer_class, "offset", "I"); 246 jni, j_decoded_output_buffer_class, "offset", "I");
243 j_info_size_field_ = GetFieldID( 247 j_info_size_field_ = GetFieldID(
244 jni, j_decoder_decoded_byte_buffer_class, "size", "I"); 248 jni, j_decoded_output_buffer_class, "size", "I");
245 j_info_presentation_timestamp_us_field_ = GetFieldID( 249 j_info_presentation_timestamp_us_field_ = GetFieldID(
246 jni, j_decoder_decoded_byte_buffer_class, "presentationTimestampUs", "J"); 250 jni, j_decoded_output_buffer_class, "presentationTimestampUs", "J");
251 j_byte_buffer_decode_time_ms_field_ = GetFieldID(
252 jni, j_decoded_output_buffer_class, "decodeTimeMs", "J");
247 253
248 CHECK_EXCEPTION(jni) << "MediaCodecVideoDecoder ctor failed"; 254 CHECK_EXCEPTION(jni) << "MediaCodecVideoDecoder ctor failed";
249 use_surface_ = (render_egl_context_ != NULL); 255 use_surface_ = (render_egl_context_ != NULL);
250 ALOGD << "MediaCodecVideoDecoder ctor. Use surface: " << use_surface_; 256 ALOGD << "MediaCodecVideoDecoder ctor. Use surface: " << use_surface_;
251 memset(&codec_, 0, sizeof(codec_)); 257 memset(&codec_, 0, sizeof(codec_));
252 AllowBlockingCalls(); 258 AllowBlockingCalls();
253 } 259 }
254 260
255 MediaCodecVideoDecoder::~MediaCodecVideoDecoder() { 261 MediaCodecVideoDecoder::~MediaCodecVideoDecoder() {
256 // Call Release() to ensure no more callbacks to us after we are deleted. 262 // Call Release() to ensure no more callbacks to us after we are deleted.
257 Release(); 263 Release();
258 // Delete global references.
259 JNIEnv* jni = AttachCurrentThreadIfNeeded();
260 if (previous_surface_texture_ != NULL) {
261 jni->DeleteGlobalRef(previous_surface_texture_);
262 }
263 if (surface_texture_ != NULL) {
264 jni->DeleteGlobalRef(surface_texture_);
265 }
266 } 264 }
267 265
268 int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst, 266 int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst,
269 int32_t numberOfCores) { 267 int32_t numberOfCores) {
270 ALOGD << "InitDecode."; 268 ALOGD << "InitDecode.";
271 if (inst == NULL) { 269 if (inst == NULL) {
272 ALOGE << "NULL VideoCodec instance"; 270 ALOGE << "NULL VideoCodec instance";
273 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; 271 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
274 } 272 }
275 // Factory should guard against other codecs being used with us. 273 // Factory should guard against other codecs being used with us.
(...skipping 30 matching lines...) Expand all
306 ALOGE << "Release failure: " << ret_val << " - fallback to SW codec"; 304 ALOGE << "Release failure: " << ret_val << " - fallback to SW codec";
307 sw_fallback_required_ = true; 305 sw_fallback_required_ = true;
308 return WEBRTC_VIDEO_CODEC_ERROR; 306 return WEBRTC_VIDEO_CODEC_ERROR;
309 } 307 }
310 308
311 // Always start with a complete key frame. 309 // Always start with a complete key frame.
312 key_frame_required_ = true; 310 key_frame_required_ = true;
313 frames_received_ = 0; 311 frames_received_ = 0;
314 frames_decoded_ = 0; 312 frames_decoded_ = 0;
315 313
314 if (use_surface_) {
315 surface_texture_helper_ = new rtc::RefCountedObject<SurfaceTextureHelper>(
316 jni, render_egl_context_);
317 }
318
316 jobject j_video_codec_enum = JavaEnumFromIndex( 319 jobject j_video_codec_enum = JavaEnumFromIndex(
317 jni, "MediaCodecVideoDecoder$VideoCodecType", codecType_); 320 jni, "MediaCodecVideoDecoder$VideoCodecType", codecType_);
318 bool success = jni->CallBooleanMethod( 321 bool success = jni->CallBooleanMethod(
319 *j_media_codec_video_decoder_, 322 *j_media_codec_video_decoder_,
320 j_init_decode_method_, 323 j_init_decode_method_,
321 j_video_codec_enum, 324 j_video_codec_enum,
322 codec_.width, 325 codec_.width,
323 codec_.height, 326 codec_.height,
324 use_surface_ ? render_egl_context_ : nullptr); 327 use_surface_ ? surface_texture_helper_->GetJavaSurfaceTextureHelper()
328 : nullptr);
325 if (CheckException(jni) || !success) { 329 if (CheckException(jni) || !success) {
326 ALOGE << "Codec initialization error - fallback to SW codec."; 330 ALOGE << "Codec initialization error - fallback to SW codec.";
327 sw_fallback_required_ = true; 331 sw_fallback_required_ = true;
328 return WEBRTC_VIDEO_CODEC_ERROR; 332 return WEBRTC_VIDEO_CODEC_ERROR;
329 } 333 }
330 inited_ = true; 334 inited_ = true;
331 335
332 switch (codecType_) { 336 switch (codecType_) {
333 case kVideoCodecVP8: 337 case kVideoCodecVP8:
334 max_pending_frames_ = kMaxPendingFramesVp8; 338 max_pending_frames_ = kMaxPendingFramesVp8;
335 break; 339 break;
336 case kVideoCodecVP9: 340 case kVideoCodecVP9:
337 max_pending_frames_ = kMaxPendingFramesVp9; 341 max_pending_frames_ = kMaxPendingFramesVp9;
338 break; 342 break;
339 case kVideoCodecH264: 343 case kVideoCodecH264:
340 max_pending_frames_ = kMaxPendingFramesH264; 344 max_pending_frames_ = kMaxPendingFramesH264;
341 break; 345 break;
342 default: 346 default:
343 max_pending_frames_ = 0; 347 max_pending_frames_ = 0;
344 } 348 }
345 start_time_ms_ = GetCurrentTimeMs(); 349 start_time_ms_ = GetCurrentTimeMs();
346 current_frames_ = 0; 350 current_frames_ = 0;
347 current_bytes_ = 0; 351 current_bytes_ = 0;
348 current_decoding_time_ms_ = 0; 352 current_decoding_time_ms_ = 0;
349 timestamps_.clear(); 353 timestamps_.clear();
350 ntp_times_ms_.clear(); 354 ntp_times_ms_.clear();
351 frame_rtc_times_ms_.clear();
352 355
353 jobjectArray input_buffers = (jobjectArray)GetObjectField( 356 jobjectArray input_buffers = (jobjectArray)GetObjectField(
354 jni, *j_media_codec_video_decoder_, j_input_buffers_field_); 357 jni, *j_media_codec_video_decoder_, j_input_buffers_field_);
355 size_t num_input_buffers = jni->GetArrayLength(input_buffers); 358 size_t num_input_buffers = jni->GetArrayLength(input_buffers);
356 ALOGD << "Maximum amount of pending frames: " << max_pending_frames_; 359 ALOGD << "Maximum amount of pending frames: " << max_pending_frames_;
357 input_buffers_.resize(num_input_buffers); 360 input_buffers_.resize(num_input_buffers);
358 for (size_t i = 0; i < num_input_buffers; ++i) { 361 for (size_t i = 0; i < num_input_buffers; ++i) {
359 input_buffers_[i] = 362 input_buffers_[i] =
360 jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i)); 363 jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i));
361 if (CheckException(jni)) { 364 if (CheckException(jni)) {
362 ALOGE << "NewGlobalRef error - fallback to SW codec."; 365 ALOGE << "NewGlobalRef error - fallback to SW codec.";
363 sw_fallback_required_ = true; 366 sw_fallback_required_ = true;
364 return WEBRTC_VIDEO_CODEC_ERROR; 367 return WEBRTC_VIDEO_CODEC_ERROR;
365 } 368 }
366 } 369 }
367 370
368 if (use_surface_) {
369 jobject surface_texture = GetObjectField(
370 jni, *j_media_codec_video_decoder_, j_surface_texture_field_);
371 if (previous_surface_texture_ != NULL) {
372 jni->DeleteGlobalRef(previous_surface_texture_);
373 }
374 previous_surface_texture_ = surface_texture_;
375 surface_texture_ = jni->NewGlobalRef(surface_texture);
376 }
377 codec_thread_->PostDelayed(kMediaCodecPollMs, this); 371 codec_thread_->PostDelayed(kMediaCodecPollMs, this);
378 372
379 return WEBRTC_VIDEO_CODEC_OK; 373 return WEBRTC_VIDEO_CODEC_OK;
380 } 374 }
381 375
382 int32_t MediaCodecVideoDecoder::Release() { 376 int32_t MediaCodecVideoDecoder::Release() {
383 ALOGD << "DecoderRelease request"; 377 ALOGD << "DecoderRelease request";
384 return codec_thread_->Invoke<int32_t>( 378 return codec_thread_->Invoke<int32_t>(
385 Bind(&MediaCodecVideoDecoder::ReleaseOnCodecThread, this)); 379 Bind(&MediaCodecVideoDecoder::ReleaseOnCodecThread, this));
386 } 380 }
387 381
388 int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() { 382 int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() {
389 if (!inited_) { 383 if (!inited_) {
390 return WEBRTC_VIDEO_CODEC_OK; 384 return WEBRTC_VIDEO_CODEC_OK;
391 } 385 }
392 CheckOnCodecThread(); 386 CheckOnCodecThread();
393 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 387 JNIEnv* jni = AttachCurrentThreadIfNeeded();
394 ALOGD << "DecoderReleaseOnCodecThread: Frames received: " << 388 ALOGD << "DecoderReleaseOnCodecThread: Frames received: " <<
395 frames_received_ << ". Frames decoded: " << frames_decoded_; 389 frames_received_ << ". Frames decoded: " << frames_decoded_;
396 ScopedLocalRefFrame local_ref_frame(jni); 390 ScopedLocalRefFrame local_ref_frame(jni);
397 for (size_t i = 0; i < input_buffers_.size(); i++) { 391 for (size_t i = 0; i < input_buffers_.size(); i++) {
398 jni->DeleteGlobalRef(input_buffers_[i]); 392 jni->DeleteGlobalRef(input_buffers_[i]);
399 } 393 }
400 input_buffers_.clear(); 394 input_buffers_.clear();
401 jni->CallVoidMethod(*j_media_codec_video_decoder_, j_release_method_); 395 jni->CallVoidMethod(*j_media_codec_video_decoder_, j_release_method_);
396 surface_texture_helper_ = nullptr;
402 inited_ = false; 397 inited_ = false;
403 rtc::MessageQueueManager::Clear(this); 398 rtc::MessageQueueManager::Clear(this);
404 if (CheckException(jni)) { 399 if (CheckException(jni)) {
405 ALOGE << "Decoder release exception"; 400 ALOGE << "Decoder release exception";
406 return WEBRTC_VIDEO_CODEC_ERROR; 401 return WEBRTC_VIDEO_CODEC_ERROR;
407 } 402 }
408 ALOGD << "DecoderReleaseOnCodecThread done"; 403 ALOGD << "DecoderReleaseOnCodecThread done";
409 return WEBRTC_VIDEO_CODEC_OK; 404 return WEBRTC_VIDEO_CODEC_OK;
410 } 405 }
411 406
(...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after
501 const EncodedImage& inputImage) { 496 const EncodedImage& inputImage) {
502 CheckOnCodecThread(); 497 CheckOnCodecThread();
503 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 498 JNIEnv* jni = AttachCurrentThreadIfNeeded();
504 ScopedLocalRefFrame local_ref_frame(jni); 499 ScopedLocalRefFrame local_ref_frame(jni);
505 500
506 // Try to drain the decoder and wait until output is not too 501 // Try to drain the decoder and wait until output is not too
507 // much behind the input. 502 // much behind the input.
508 if (frames_received_ > frames_decoded_ + max_pending_frames_) { 503 if (frames_received_ > frames_decoded_ + max_pending_frames_) {
509 ALOGV("Received: %d. Decoded: %d. Wait for output...", 504 ALOGV("Received: %d. Decoded: %d. Wait for output...",
510 frames_received_, frames_decoded_); 505 frames_received_, frames_decoded_);
511 if (!DeliverPendingOutputs(jni, kMediaCodecTimeoutMs * 1000)) { 506 if (!DeliverPendingOutputs(jni, kMediaCodecTimeoutMs)) {
512 ALOGE << "DeliverPendingOutputs error. Frames received: " << 507 ALOGE << "DeliverPendingOutputs error. Frames received: " <<
513 frames_received_ << ". Frames decoded: " << frames_decoded_; 508 frames_received_ << ". Frames decoded: " << frames_decoded_;
514 return ProcessHWErrorOnCodecThread(); 509 return ProcessHWErrorOnCodecThread();
515 } 510 }
516 if (frames_received_ > frames_decoded_ + max_pending_frames_) { 511 if (frames_received_ > frames_decoded_ + max_pending_frames_) {
517 ALOGE << "Output buffer dequeue timeout. Frames received: " << 512 ALOGE << "Output buffer dequeue timeout. Frames received: " <<
518 frames_received_ << ". Frames decoded: " << frames_decoded_; 513 frames_received_ << ". Frames decoded: " << frames_decoded_;
519 return ProcessHWErrorOnCodecThread(); 514 return ProcessHWErrorOnCodecThread();
520 } 515 }
521 } 516 }
(...skipping 24 matching lines...) Expand all
546 j_input_buffer_index << ". TS: " << (int)(timestamp_us / 1000) 541 j_input_buffer_index << ". TS: " << (int)(timestamp_us / 1000)
547 << ". Size: " << inputImage._length; 542 << ". Size: " << inputImage._length;
548 } 543 }
549 memcpy(buffer, inputImage._buffer, inputImage._length); 544 memcpy(buffer, inputImage._buffer, inputImage._length);
550 545
551 // Save input image timestamps for later output. 546 // Save input image timestamps for later output.
552 frames_received_++; 547 frames_received_++;
553 current_bytes_ += inputImage._length; 548 current_bytes_ += inputImage._length;
554 timestamps_.push_back(inputImage._timeStamp); 549 timestamps_.push_back(inputImage._timeStamp);
555 ntp_times_ms_.push_back(inputImage.ntp_time_ms_); 550 ntp_times_ms_.push_back(inputImage.ntp_time_ms_);
556 frame_rtc_times_ms_.push_back(GetCurrentTimeMs());
557 551
558 // Feed input to decoder. 552 // Feed input to decoder.
559 bool success = jni->CallBooleanMethod(*j_media_codec_video_decoder_, 553 bool success = jni->CallBooleanMethod(*j_media_codec_video_decoder_,
560 j_queue_input_buffer_method_, 554 j_queue_input_buffer_method_,
561 j_input_buffer_index, 555 j_input_buffer_index,
562 inputImage._length, 556 inputImage._length,
563 timestamp_us); 557 timestamp_us);
564 if (CheckException(jni) || !success) { 558 if (CheckException(jni) || !success) {
565 ALOGE << "queueInputBuffer error"; 559 ALOGE << "queueInputBuffer error";
566 return ProcessHWErrorOnCodecThread(); 560 return ProcessHWErrorOnCodecThread();
567 } 561 }
568 562
569 // Try to drain the decoder 563 // Try to drain the decoder
570 if (!DeliverPendingOutputs(jni, 0)) { 564 if (!DeliverPendingOutputs(jni, 0)) {
571 ALOGE << "DeliverPendingOutputs error"; 565 ALOGE << "DeliverPendingOutputs error";
572 return ProcessHWErrorOnCodecThread(); 566 return ProcessHWErrorOnCodecThread();
573 } 567 }
574 568
575 return WEBRTC_VIDEO_CODEC_OK; 569 return WEBRTC_VIDEO_CODEC_OK;
576 } 570 }
577 571
578 bool MediaCodecVideoDecoder::DeliverPendingOutputs( 572 bool MediaCodecVideoDecoder::DeliverPendingOutputs(
579 JNIEnv* jni, int dequeue_timeout_us) { 573 JNIEnv* jni, int dequeue_timeout_ms) {
580 if (frames_received_ <= frames_decoded_) { 574 if (frames_received_ <= frames_decoded_) {
581 // No need to query for output buffers - decoder is drained. 575 // No need to query for output buffers - decoder is drained.
582 return true; 576 return true;
583 } 577 }
584 // Get decoder output. 578 // Get decoder output.
585 jobject j_decoder_output_buffer = jni->CallObjectMethod( 579 jobject j_decoder_output_buffer =
586 *j_media_codec_video_decoder_, 580 jni->CallObjectMethod(*j_media_codec_video_decoder_,
587 j_dequeue_output_buffer_method_, 581 use_surface_ ? j_dequeue_texture_buffer_method_
588 dequeue_timeout_us); 582 : j_dequeue_byte_buffer_method_,
583 dequeue_timeout_ms);
584
589 if (CheckException(jni)) { 585 if (CheckException(jni)) {
590 ALOGE << "dequeueOutputBuffer() error"; 586 ALOGE << "dequeueOutputBuffer() error";
591 return false; 587 return false;
592 } 588 }
593 if (IsNull(jni, j_decoder_output_buffer)) { 589 if (IsNull(jni, j_decoder_output_buffer)) {
594 // No decoded frame ready. 590 // No decoded frame ready.
595 return true; 591 return true;
596 } 592 }
597 593
598 // Get decoded video frame properties. 594 // Get decoded video frame properties.
599 int color_format = GetIntField(jni, *j_media_codec_video_decoder_, 595 int color_format = GetIntField(jni, *j_media_codec_video_decoder_,
600 j_color_format_field_); 596 j_color_format_field_);
601 int width = GetIntField(jni, *j_media_codec_video_decoder_, j_width_field_); 597 int width = GetIntField(jni, *j_media_codec_video_decoder_, j_width_field_);
602 int height = GetIntField(jni, *j_media_codec_video_decoder_, j_height_field_); 598 int height = GetIntField(jni, *j_media_codec_video_decoder_, j_height_field_);
603 int stride = GetIntField(jni, *j_media_codec_video_decoder_, j_stride_field_); 599 int stride = GetIntField(jni, *j_media_codec_video_decoder_, j_stride_field_);
604 int slice_height = GetIntField(jni, *j_media_codec_video_decoder_, 600 int slice_height = GetIntField(jni, *j_media_codec_video_decoder_,
605 j_slice_height_field_); 601 j_slice_height_field_);
606 602
607 rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer; 603 rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer;
608 long output_timestamps_ms = 0; 604 int64_t output_timestamps_ms = 0;
605 int decode_time_ms = 0;
606 int64_t frame_delayed_ms = 0;
609 if (use_surface_) { 607 if (use_surface_) {
610 // Extract data from Java DecodedTextureBuffer. 608 // Extract data from Java DecodedTextureBuffer.
611 const int texture_id = 609 const int texture_id =
612 GetIntField(jni, j_decoder_output_buffer, j_textureID_field_); 610 GetIntField(jni, j_decoder_output_buffer, j_texture_id_field_);
613 const int64_t timestamp_us = 611 if (texture_id != 0) { // |texture_id| == 0 represents a dropped frame.
614 GetLongField(jni, j_decoder_output_buffer, 612 const jfloatArray j_transform_matrix =
615 j_texture_presentation_timestamp_us_field_); 613 reinterpret_cast<jfloatArray>(GetObjectField(
616 output_timestamps_ms = timestamp_us / rtc::kNumMicrosecsPerMillisec; 614 jni, j_decoder_output_buffer, j_transform_matrix_field_));
617 // Create webrtc::VideoFrameBuffer with native texture handle. 615 const int64_t timestamp_us =
618 native_handle_.SetTextureObject(surface_texture_, texture_id); 616 GetLongField(jni, j_decoder_output_buffer,
619 frame_buffer = new rtc::RefCountedObject<JniNativeHandleBuffer>( 617 j_texture_presentation_timestamp_us_field_);
620 &native_handle_, width, height); 618 output_timestamps_ms = timestamp_us / rtc::kNumMicrosecsPerMillisec;
619 decode_time_ms = GetLongField(jni, j_decoder_output_buffer,
620 j_texture_decode_time_ms_field_);
621 frame_delayed_ms = GetLongField(jni, j_decoder_output_buffer,
622 j_texture_frame_delay_ms_field_);
623
624 // Create webrtc::VideoFrameBuffer with native texture handle.
625 frame_buffer = surface_texture_helper_->CreateTextureFrame(
626 width, height, NativeHandleImpl(jni, texture_id, j_transform_matrix));
627 }
621 } else { 628 } else {
622 // Extract data from Java ByteBuffer and create output yuv420 frame - 629 // Extract data from Java ByteBuffer and create output yuv420 frame -
623 // for non surface decoding only. 630 // for non surface decoding only.
624 const int output_buffer_index = 631 const int output_buffer_index =
625 GetIntField(jni, j_decoder_output_buffer, j_info_index_field_); 632 GetIntField(jni, j_decoder_output_buffer, j_info_index_field_);
626 const int output_buffer_offset = 633 const int output_buffer_offset =
627 GetIntField(jni, j_decoder_output_buffer, j_info_offset_field_); 634 GetIntField(jni, j_decoder_output_buffer, j_info_offset_field_);
628 const int output_buffer_size = 635 const int output_buffer_size =
629 GetIntField(jni, j_decoder_output_buffer, j_info_size_field_); 636 GetIntField(jni, j_decoder_output_buffer, j_info_size_field_);
630 const int64_t timestamp_us = GetLongField( 637 const int64_t timestamp_us = GetLongField(
631 jni, j_decoder_output_buffer, j_info_presentation_timestamp_us_field_); 638 jni, j_decoder_output_buffer, j_info_presentation_timestamp_us_field_);
632 output_timestamps_ms = timestamp_us / rtc::kNumMicrosecsPerMillisec; 639 output_timestamps_ms = timestamp_us / rtc::kNumMicrosecsPerMillisec;
640 decode_time_ms = GetLongField(jni, j_decoder_output_buffer,
641 j_byte_buffer_decode_time_ms_field_);
633 642
634 if (output_buffer_size < width * height * 3 / 2) { 643 if (output_buffer_size < width * height * 3 / 2) {
635 ALOGE << "Insufficient output buffer size: " << output_buffer_size; 644 ALOGE << "Insufficient output buffer size: " << output_buffer_size;
636 return false; 645 return false;
637 } 646 }
638 jobjectArray output_buffers = reinterpret_cast<jobjectArray>(GetObjectField( 647 jobjectArray output_buffers = reinterpret_cast<jobjectArray>(GetObjectField(
639 jni, *j_media_codec_video_decoder_, j_output_buffers_field_)); 648 jni, *j_media_codec_video_decoder_, j_output_buffers_field_));
640 jobject output_buffer = 649 jobject output_buffer =
641 jni->GetObjectArrayElement(output_buffers, output_buffer_index); 650 jni->GetObjectArrayElement(output_buffers, output_buffer_index);
642 uint8_t* payload = reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress( 651 uint8_t* payload = reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after
680 frame_buffer->MutableData(webrtc::kVPlane), 689 frame_buffer->MutableData(webrtc::kVPlane),
681 frame_buffer->stride(webrtc::kVPlane), 690 frame_buffer->stride(webrtc::kVPlane),
682 width, height); 691 width, height);
683 } 692 }
684 // Return output byte buffer back to codec. 693 // Return output byte buffer back to codec.
685 jni->CallVoidMethod( 694 jni->CallVoidMethod(
686 *j_media_codec_video_decoder_, 695 *j_media_codec_video_decoder_,
687 j_return_decoded_byte_buffer_method_, 696 j_return_decoded_byte_buffer_method_,
688 output_buffer_index); 697 output_buffer_index);
689 if (CheckException(jni)) { 698 if (CheckException(jni)) {
690 ALOGE << "returnDecodedByteBuffer error"; 699 ALOGE << "returnDecodedOutputBuffer error";
691 return false; 700 return false;
692 } 701 }
693 } 702 }
694 VideoFrame decoded_frame(frame_buffer, 0, 0, webrtc::kVideoRotation_0); 703 VideoFrame decoded_frame(frame_buffer, 0, 0, webrtc::kVideoRotation_0);
695 704
696 // Get frame timestamps from a queue. 705 // Get frame timestamps from a queue.
697 if (timestamps_.size() > 0) { 706 if (timestamps_.size() > 0) {
698 decoded_frame.set_timestamp(timestamps_.front()); 707 decoded_frame.set_timestamp(timestamps_.front());
699 timestamps_.erase(timestamps_.begin()); 708 timestamps_.erase(timestamps_.begin());
700 } 709 }
701 if (ntp_times_ms_.size() > 0) { 710 if (ntp_times_ms_.size() > 0) {
702 decoded_frame.set_ntp_time_ms(ntp_times_ms_.front()); 711 decoded_frame.set_ntp_time_ms(ntp_times_ms_.front());
703 ntp_times_ms_.erase(ntp_times_ms_.begin()); 712 ntp_times_ms_.erase(ntp_times_ms_.begin());
704 } 713 }
705 int64_t frame_decoding_time_ms = 0; 714
706 if (frame_rtc_times_ms_.size() > 0) {
707 frame_decoding_time_ms = GetCurrentTimeMs() - frame_rtc_times_ms_.front();
708 frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin());
709 }
710 if (frames_decoded_ < kMaxDecodedLogFrames) { 715 if (frames_decoded_ < kMaxDecodedLogFrames) {
711 ALOGD << "Decoder frame out # " << frames_decoded_ << ". " << width << 716 ALOGD << "Decoder frame out # " << frames_decoded_ << ". " << width <<
712 " x " << height << ". " << stride << " x " << slice_height << 717 " x " << height << ". " << stride << " x " << slice_height <<
713 ". Color: " << color_format << ". TS:" << (int)output_timestamps_ms << 718 ". Color: " << color_format << ". TS:" << (int)output_timestamps_ms <<
714 ". DecTime: " << (int)frame_decoding_time_ms; 719 ". DecTime: " << (int)decode_time_ms <<
720 ". DelayTime: " << (int)frame_delayed_ms;
715 } 721 }
716 722
717 // Calculate and print decoding statistics - every 3 seconds. 723 // Calculate and print decoding statistics - every 3 seconds.
718 frames_decoded_++; 724 frames_decoded_++;
719 current_frames_++; 725 current_frames_++;
720 current_decoding_time_ms_ += frame_decoding_time_ms; 726 current_decoding_time_ms_ += decode_time_ms;
721 int statistic_time_ms = GetCurrentTimeMs() - start_time_ms_; 727 int statistic_time_ms = GetCurrentTimeMs() - start_time_ms_;
722 if (statistic_time_ms >= kMediaCodecStatisticsIntervalMs && 728 if (statistic_time_ms >= kMediaCodecStatisticsIntervalMs &&
723 current_frames_ > 0) { 729 current_frames_ > 0) {
724 ALOGD << "Decoded frames: " << frames_decoded_ << ". Bitrate: " << 730 ALOGD << "Decoded frames: " << frames_decoded_ << ". Received frames: "
731 << frames_received_ << ". Bitrate: " <<
725 (current_bytes_ * 8 / statistic_time_ms) << " kbps, fps: " << 732 (current_bytes_ * 8 / statistic_time_ms) << " kbps, fps: " <<
726 ((current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms) 733 ((current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms)
727 << ". decTime: " << (current_decoding_time_ms_ / current_frames_) << 734 << ". decTime: " << (current_decoding_time_ms_ / current_frames_) <<
728 " for last " << statistic_time_ms << " ms."; 735 " for last " << statistic_time_ms << " ms.";
729 start_time_ms_ = GetCurrentTimeMs(); 736 start_time_ms_ = GetCurrentTimeMs();
730 current_frames_ = 0; 737 current_frames_ = 0;
731 current_bytes_ = 0; 738 current_bytes_ = 0;
732 current_decoding_time_ms_ = 0; 739 current_decoding_time_ms_ = 0;
733 } 740 }
734 741
735 // Callback - output decoded frame. 742 // |.IsZeroSize())| returns true when a frame has been dropped.
736 const int32_t callback_status = callback_->Decoded(decoded_frame); 743 if (!decoded_frame.IsZeroSize()) {
737 if (callback_status > 0) { 744 // Callback - output decoded frame.
738 ALOGE << "callback error"; 745 const int32_t callback_status =
746 callback_->Decoded(decoded_frame, decode_time_ms);
747 if (callback_status > 0) {
748 ALOGE << "callback error";
749 }
739 } 750 }
740
741 return true; 751 return true;
742 } 752 }
743 753
744 int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback( 754 int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback(
745 DecodedImageCallback* callback) { 755 DecodedImageCallback* callback) {
746 callback_ = callback; 756 callback_ = callback;
747 return WEBRTC_VIDEO_CODEC_OK; 757 return WEBRTC_VIDEO_CODEC_OK;
748 } 758 }
749 759
750 int32_t MediaCodecVideoDecoder::Reset() { 760 int32_t MediaCodecVideoDecoder::Reset() {
(...skipping 120 matching lines...) Expand 10 before | Expand all | Expand 10 after
871 } 881 }
872 882
873 void MediaCodecVideoDecoderFactory::DestroyVideoDecoder( 883 void MediaCodecVideoDecoderFactory::DestroyVideoDecoder(
874 webrtc::VideoDecoder* decoder) { 884 webrtc::VideoDecoder* decoder) {
875 ALOGD << "Destroy video decoder."; 885 ALOGD << "Destroy video decoder.";
876 delete decoder; 886 delete decoder;
877 } 887 }
878 888
879 } // namespace webrtc_jni 889 } // namespace webrtc_jni
880 890
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698