Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(308)

Side by Side Diff: talk/app/webrtc/java/jni/androidmediadecoder_jni.cc

Issue 1422963003: Android MediaCodecVideoDecoder: Manage lifetime of texture frames (Closed) Base URL: https://chromium.googlesource.com/external/webrtc.git@master
Patch Set: Addressed magjeds comments. Created 5 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 /* 1 /*
2 * libjingle 2 * libjingle
3 * Copyright 2015 Google Inc. 3 * Copyright 2015 Google Inc.
4 * 4 *
5 * Redistribution and use in source and binary forms, with or without 5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met: 6 * modification, are permitted provided that the following conditions are met:
7 * 7 *
8 * 1. Redistributions of source code must retain the above copyright notice, 8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer. 9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice, 10 * 2. Redistributions in binary form must reproduce the above copyright notice,
(...skipping 15 matching lines...) Expand all
26 * 26 *
27 */ 27 */
28 28
29 #include <algorithm> 29 #include <algorithm>
30 #include <vector> 30 #include <vector>
31 31
32 #include "talk/app/webrtc/java/jni/androidmediadecoder_jni.h" 32 #include "talk/app/webrtc/java/jni/androidmediadecoder_jni.h"
33 #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h" 33 #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h"
34 #include "talk/app/webrtc/java/jni/classreferenceholder.h" 34 #include "talk/app/webrtc/java/jni/classreferenceholder.h"
35 #include "talk/app/webrtc/java/jni/native_handle_impl.h" 35 #include "talk/app/webrtc/java/jni/native_handle_impl.h"
36 #include "talk/app/webrtc/java/jni/surfacetexturehelper_jni.h"
36 #include "webrtc/base/bind.h" 37 #include "webrtc/base/bind.h"
37 #include "webrtc/base/checks.h" 38 #include "webrtc/base/checks.h"
38 #include "webrtc/base/logging.h" 39 #include "webrtc/base/logging.h"
39 #include "webrtc/base/scoped_ref_ptr.h" 40 #include "webrtc/base/scoped_ref_ptr.h"
40 #include "webrtc/base/thread.h" 41 #include "webrtc/base/thread.h"
41 #include "webrtc/base/timeutils.h" 42 #include "webrtc/base/timeutils.h"
42 #include "webrtc/common_video/interface/i420_buffer_pool.h" 43 #include "webrtc/common_video/interface/i420_buffer_pool.h"
43 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h" 44 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
44 #include "webrtc/system_wrappers/include/logcat_trace_context.h" 45 #include "webrtc/system_wrappers/include/logcat_trace_context.h"
45 #include "webrtc/system_wrappers/include/tick_util.h" 46 #include "webrtc/system_wrappers/include/tick_util.h"
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after
104 105
105 // Type of video codec. 106 // Type of video codec.
106 VideoCodecType codecType_; 107 VideoCodecType codecType_;
107 108
108 bool key_frame_required_; 109 bool key_frame_required_;
109 bool inited_; 110 bool inited_;
110 bool sw_fallback_required_; 111 bool sw_fallback_required_;
111 bool use_surface_; 112 bool use_surface_;
112 VideoCodec codec_; 113 VideoCodec codec_;
113 webrtc::I420BufferPool decoded_frame_pool_; 114 webrtc::I420BufferPool decoded_frame_pool_;
114 NativeHandleImpl native_handle_; 115 rtc::scoped_refptr<SurfaceTextureHelper> surface_texture_helper_;
115 DecodedImageCallback* callback_; 116 DecodedImageCallback* callback_;
116 int frames_received_; // Number of frames received by decoder. 117 int frames_received_; // Number of frames received by decoder.
117 int frames_decoded_; // Number of frames decoded by decoder. 118 int frames_decoded_; // Number of frames decoded by decoder.
118 int64_t start_time_ms_; // Start time for statistics. 119 int64_t start_time_ms_; // Start time for statistics.
119 int current_frames_; // Number of frames in the current statistics interval. 120 int current_frames_; // Number of frames in the current statistics interval.
120 int current_bytes_; // Encoded bytes in the current statistics interval. 121 int current_bytes_; // Encoded bytes in the current statistics interval.
121 int current_decoding_time_ms_; // Overall decoding time in the current second 122 int current_decoding_time_ms_; // Overall decoding time in the current second
122 uint32_t max_pending_frames_; // Maximum number of pending input frames 123 uint32_t max_pending_frames_; // Maximum number of pending input frames
123 std::vector<int32_t> timestamps_; 124 std::vector<int32_t> timestamps_;
124 std::vector<int64_t> ntp_times_ms_; 125 std::vector<int64_t> ntp_times_ms_;
125 std::vector<int64_t> frame_rtc_times_ms_; // Time when video frame is sent to
126 // decoder input.
127 126
128 // State that is constant for the lifetime of this object once the ctor 127 // State that is constant for the lifetime of this object once the ctor
129 // returns. 128 // returns.
130 scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec. 129 scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec.
131 ScopedGlobalRef<jclass> j_media_codec_video_decoder_class_; 130 ScopedGlobalRef<jclass> j_media_codec_video_decoder_class_;
132 ScopedGlobalRef<jobject> j_media_codec_video_decoder_; 131 ScopedGlobalRef<jobject> j_media_codec_video_decoder_;
133 jmethodID j_init_decode_method_; 132 jmethodID j_init_decode_method_;
134 jmethodID j_release_method_; 133 jmethodID j_release_method_;
135 jmethodID j_dequeue_input_buffer_method_; 134 jmethodID j_dequeue_input_buffer_method_;
136 jmethodID j_queue_input_buffer_method_; 135 jmethodID j_queue_input_buffer_method_;
137 jmethodID j_dequeue_output_buffer_method_; 136 jmethodID j_dequeue_byte_buffer_method_;
137 jmethodID j_dequeue_texture_buffer_method_;
138 jmethodID j_return_decoded_byte_buffer_method_; 138 jmethodID j_return_decoded_byte_buffer_method_;
139 // MediaCodecVideoDecoder fields. 139 // MediaCodecVideoDecoder fields.
140 jfieldID j_input_buffers_field_; 140 jfieldID j_input_buffers_field_;
141 jfieldID j_output_buffers_field_; 141 jfieldID j_output_buffers_field_;
142 jfieldID j_color_format_field_; 142 jfieldID j_color_format_field_;
143 jfieldID j_width_field_; 143 jfieldID j_width_field_;
144 jfieldID j_height_field_; 144 jfieldID j_height_field_;
145 jfieldID j_stride_field_; 145 jfieldID j_stride_field_;
146 jfieldID j_slice_height_field_; 146 jfieldID j_slice_height_field_;
147 jfieldID j_surface_texture_field_;
148 // MediaCodecVideoDecoder.DecodedTextureBuffer fields. 147 // MediaCodecVideoDecoder.DecodedTextureBuffer fields.
149 jfieldID j_textureID_field_; 148 jfieldID j_texture_id_field_;
149 jfieldID j_transform_matrix_field_;
150 jfieldID j_texture_presentation_timestamp_us_field_; 150 jfieldID j_texture_presentation_timestamp_us_field_;
151 // MediaCodecVideoDecoder.DecodedByteBuffer fields. 151 jfieldID j_texture_decode_time_ms_field_;
152 jfieldID j_texture_frame_delay_ms_field_;
153 // MediaCodecVideoDecoder.DecodedOutputBuffer fields.
152 jfieldID j_info_index_field_; 154 jfieldID j_info_index_field_;
153 jfieldID j_info_offset_field_; 155 jfieldID j_info_offset_field_;
154 jfieldID j_info_size_field_; 156 jfieldID j_info_size_field_;
155 jfieldID j_info_presentation_timestamp_us_field_; 157 jfieldID j_info_presentation_timestamp_us_field_;
158 jfieldID j_byte_buffer_decode_time_ms_field_;
156 159
157 // Global references; must be deleted in Release(). 160 // Global references; must be deleted in Release().
158 std::vector<jobject> input_buffers_; 161 std::vector<jobject> input_buffers_;
159 jobject surface_texture_;
160 jobject previous_surface_texture_;
161 162
162 // Render EGL context - owned by factory, should not be allocated/destroyed 163 // Render EGL context - owned by factory, should not be allocated/destroyed
163 // by VideoDecoder. 164 // by VideoDecoder.
164 jobject render_egl_context_; 165 jobject render_egl_context_;
165 }; 166 };
166 167
167 MediaCodecVideoDecoder::MediaCodecVideoDecoder( 168 MediaCodecVideoDecoder::MediaCodecVideoDecoder(
168 JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context) : 169 JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context) :
169 codecType_(codecType), 170 codecType_(codecType),
170 render_egl_context_(render_egl_context), 171 render_egl_context_(render_egl_context),
171 key_frame_required_(true), 172 key_frame_required_(true),
172 inited_(false), 173 inited_(false),
173 sw_fallback_required_(false), 174 sw_fallback_required_(false),
174 surface_texture_(NULL),
175 previous_surface_texture_(NULL),
176 codec_thread_(new Thread()), 175 codec_thread_(new Thread()),
177 j_media_codec_video_decoder_class_( 176 j_media_codec_video_decoder_class_(
178 jni, 177 jni,
179 FindClass(jni, "org/webrtc/MediaCodecVideoDecoder")), 178 FindClass(jni, "org/webrtc/MediaCodecVideoDecoder")),
180 j_media_codec_video_decoder_( 179 j_media_codec_video_decoder_(
181 jni, 180 jni,
182 jni->NewObject(*j_media_codec_video_decoder_class_, 181 jni->NewObject(*j_media_codec_video_decoder_class_,
183 GetMethodID(jni, 182 GetMethodID(jni,
184 *j_media_codec_video_decoder_class_, 183 *j_media_codec_video_decoder_class_,
185 "<init>", 184 "<init>",
186 "()V"))) { 185 "()V"))) {
187 ScopedLocalRefFrame local_ref_frame(jni); 186 ScopedLocalRefFrame local_ref_frame(jni);
188 codec_thread_->SetName("MediaCodecVideoDecoder", NULL); 187 codec_thread_->SetName("MediaCodecVideoDecoder", NULL);
189 RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoDecoder"; 188 RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoDecoder";
190 189
191 j_init_decode_method_ = GetMethodID( 190 j_init_decode_method_ = GetMethodID(
192 jni, *j_media_codec_video_decoder_class_, "initDecode", 191 jni, *j_media_codec_video_decoder_class_, "initDecode",
193 "(Lorg/webrtc/MediaCodecVideoDecoder$VideoCodecType;" 192 "(Lorg/webrtc/MediaCodecVideoDecoder$VideoCodecType;"
194 "IILjavax/microedition/khronos/egl/EGLContext;)Z"); 193 "IILorg/webrtc/SurfaceTextureHelper;)Z");
195 j_release_method_ = 194 j_release_method_ =
196 GetMethodID(jni, *j_media_codec_video_decoder_class_, "release", "()V"); 195 GetMethodID(jni, *j_media_codec_video_decoder_class_, "release", "()V");
197 j_dequeue_input_buffer_method_ = GetMethodID( 196 j_dequeue_input_buffer_method_ = GetMethodID(
198 jni, *j_media_codec_video_decoder_class_, "dequeueInputBuffer", "()I"); 197 jni, *j_media_codec_video_decoder_class_, "dequeueInputBuffer", "()I");
199 j_queue_input_buffer_method_ = GetMethodID( 198 j_queue_input_buffer_method_ = GetMethodID(
200 jni, *j_media_codec_video_decoder_class_, "queueInputBuffer", "(IIJ)Z"); 199 jni, *j_media_codec_video_decoder_class_, "queueInputBuffer", "(IIJ)Z");
201 j_dequeue_output_buffer_method_ = GetMethodID( 200 j_dequeue_byte_buffer_method_ = GetMethodID(
202 jni, *j_media_codec_video_decoder_class_, "dequeueOutputBuffer", 201 jni, *j_media_codec_video_decoder_class_, "dequeueOutputBuffer",
203 "(I)Ljava/lang/Object;"); 202 "(I)Lorg/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer;");
203 j_dequeue_texture_buffer_method_ = GetMethodID(
204 jni, *j_media_codec_video_decoder_class_, "dequeueTextureBuffer",
205 "(I)Lorg/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer;");
204 j_return_decoded_byte_buffer_method_ = 206 j_return_decoded_byte_buffer_method_ =
205 GetMethodID(jni, *j_media_codec_video_decoder_class_, 207 GetMethodID(jni, *j_media_codec_video_decoder_class_,
206 "returnDecodedByteBuffer", "(I)V"); 208 "returnDecodedOutputBuffer", "(I)V");
207 209
208 j_input_buffers_field_ = GetFieldID( 210 j_input_buffers_field_ = GetFieldID(
209 jni, *j_media_codec_video_decoder_class_, 211 jni, *j_media_codec_video_decoder_class_,
210 "inputBuffers", "[Ljava/nio/ByteBuffer;"); 212 "inputBuffers", "[Ljava/nio/ByteBuffer;");
211 j_output_buffers_field_ = GetFieldID( 213 j_output_buffers_field_ = GetFieldID(
212 jni, *j_media_codec_video_decoder_class_, 214 jni, *j_media_codec_video_decoder_class_,
213 "outputBuffers", "[Ljava/nio/ByteBuffer;"); 215 "outputBuffers", "[Ljava/nio/ByteBuffer;");
214 j_color_format_field_ = GetFieldID( 216 j_color_format_field_ = GetFieldID(
215 jni, *j_media_codec_video_decoder_class_, "colorFormat", "I"); 217 jni, *j_media_codec_video_decoder_class_, "colorFormat", "I");
216 j_width_field_ = GetFieldID( 218 j_width_field_ = GetFieldID(
217 jni, *j_media_codec_video_decoder_class_, "width", "I"); 219 jni, *j_media_codec_video_decoder_class_, "width", "I");
218 j_height_field_ = GetFieldID( 220 j_height_field_ = GetFieldID(
219 jni, *j_media_codec_video_decoder_class_, "height", "I"); 221 jni, *j_media_codec_video_decoder_class_, "height", "I");
220 j_stride_field_ = GetFieldID( 222 j_stride_field_ = GetFieldID(
221 jni, *j_media_codec_video_decoder_class_, "stride", "I"); 223 jni, *j_media_codec_video_decoder_class_, "stride", "I");
222 j_slice_height_field_ = GetFieldID( 224 j_slice_height_field_ = GetFieldID(
223 jni, *j_media_codec_video_decoder_class_, "sliceHeight", "I"); 225 jni, *j_media_codec_video_decoder_class_, "sliceHeight", "I");
224 j_surface_texture_field_ = GetFieldID(
225 jni, *j_media_codec_video_decoder_class_, "surfaceTexture",
226 "Landroid/graphics/SurfaceTexture;");
227 226
228 jclass j_decoder_decoded_texture_buffer_class = FindClass(jni, 227 jclass j_decoded_texture_buffer_class = FindClass(jni,
229 "org/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer"); 228 "org/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer");
230 j_textureID_field_ = GetFieldID( 229 j_texture_id_field_ = GetFieldID(
231 jni, j_decoder_decoded_texture_buffer_class, "textureID", "I"); 230 jni, j_decoded_texture_buffer_class, "textureID", "I");
232 j_texture_presentation_timestamp_us_field_ = 231 j_transform_matrix_field_ = GetFieldID(
233 GetFieldID(jni, j_decoder_decoded_texture_buffer_class, 232 jni, j_decoded_texture_buffer_class, "transformMatrix", "[F");
234 "presentationTimestampUs", "J"); 233 j_texture_presentation_timestamp_us_field_ = GetFieldID(
234 jni, j_decoded_texture_buffer_class, "presentationTimestampUs", "J");
235 j_texture_decode_time_ms_field_ = GetFieldID(
236 jni, j_decoded_texture_buffer_class, "decodeTimeMs", "J");
237 j_texture_frame_delay_ms_field_ = GetFieldID(
238 jni, j_decoded_texture_buffer_class, "frameDelayMs", "J");
235 239
236 jclass j_decoder_decoded_byte_buffer_class = FindClass(jni, 240 jclass j_decoded_output_buffer_class = FindClass(jni,
237 "org/webrtc/MediaCodecVideoDecoder$DecodedByteBuffer"); 241 "org/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer");
238 j_info_index_field_ = GetFieldID( 242 j_info_index_field_ = GetFieldID(
239 jni, j_decoder_decoded_byte_buffer_class, "index", "I"); 243 jni, j_decoded_output_buffer_class, "index", "I");
240 j_info_offset_field_ = GetFieldID( 244 j_info_offset_field_ = GetFieldID(
241 jni, j_decoder_decoded_byte_buffer_class, "offset", "I"); 245 jni, j_decoded_output_buffer_class, "offset", "I");
242 j_info_size_field_ = GetFieldID( 246 j_info_size_field_ = GetFieldID(
243 jni, j_decoder_decoded_byte_buffer_class, "size", "I"); 247 jni, j_decoded_output_buffer_class, "size", "I");
244 j_info_presentation_timestamp_us_field_ = GetFieldID( 248 j_info_presentation_timestamp_us_field_ = GetFieldID(
245 jni, j_decoder_decoded_byte_buffer_class, "presentationTimestampUs", "J"); 249 jni, j_decoded_output_buffer_class, "presentationTimestampUs", "J");
250 j_byte_buffer_decode_time_ms_field_ = GetFieldID(
251 jni, j_decoded_output_buffer_class, "decodeTimeMs", "J");
246 252
247 CHECK_EXCEPTION(jni) << "MediaCodecVideoDecoder ctor failed"; 253 CHECK_EXCEPTION(jni) << "MediaCodecVideoDecoder ctor failed";
248 use_surface_ = (render_egl_context_ != NULL); 254 use_surface_ = (render_egl_context_ != NULL);
249 ALOGD << "MediaCodecVideoDecoder ctor. Use surface: " << use_surface_; 255 ALOGD << "MediaCodecVideoDecoder ctor. Use surface: " << use_surface_;
250 memset(&codec_, 0, sizeof(codec_)); 256 memset(&codec_, 0, sizeof(codec_));
251 AllowBlockingCalls(); 257 AllowBlockingCalls();
252 } 258 }
253 259
254 MediaCodecVideoDecoder::~MediaCodecVideoDecoder() { 260 MediaCodecVideoDecoder::~MediaCodecVideoDecoder() {
255 // Call Release() to ensure no more callbacks to us after we are deleted. 261 // Call Release() to ensure no more callbacks to us after we are deleted.
256 Release(); 262 Release();
257 // Delete global references.
258 JNIEnv* jni = AttachCurrentThreadIfNeeded();
259 if (previous_surface_texture_ != NULL) {
260 jni->DeleteGlobalRef(previous_surface_texture_);
261 }
262 if (surface_texture_ != NULL) {
263 jni->DeleteGlobalRef(surface_texture_);
264 }
265 } 263 }
266 264
267 int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst, 265 int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst,
268 int32_t numberOfCores) { 266 int32_t numberOfCores) {
269 ALOGD << "InitDecode."; 267 ALOGD << "InitDecode.";
270 if (inst == NULL) { 268 if (inst == NULL) {
271 ALOGE << "NULL VideoCodec instance"; 269 ALOGE << "NULL VideoCodec instance";
272 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; 270 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
273 } 271 }
274 // Factory should guard against other codecs being used with us. 272 // Factory should guard against other codecs being used with us.
(...skipping 30 matching lines...) Expand all
305 ALOGE << "Release failure: " << ret_val << " - fallback to SW codec"; 303 ALOGE << "Release failure: " << ret_val << " - fallback to SW codec";
306 sw_fallback_required_ = true; 304 sw_fallback_required_ = true;
307 return WEBRTC_VIDEO_CODEC_ERROR; 305 return WEBRTC_VIDEO_CODEC_ERROR;
308 } 306 }
309 307
310 // Always start with a complete key frame. 308 // Always start with a complete key frame.
311 key_frame_required_ = true; 309 key_frame_required_ = true;
312 frames_received_ = 0; 310 frames_received_ = 0;
313 frames_decoded_ = 0; 311 frames_decoded_ = 0;
314 312
313 if (use_surface_) {
314 surface_texture_helper_ = new rtc::RefCountedObject<SurfaceTextureHelper>(
315 jni, render_egl_context_);
316 }
317
315 jobject j_video_codec_enum = JavaEnumFromIndex( 318 jobject j_video_codec_enum = JavaEnumFromIndex(
316 jni, "MediaCodecVideoDecoder$VideoCodecType", codecType_); 319 jni, "MediaCodecVideoDecoder$VideoCodecType", codecType_);
317 bool success = jni->CallBooleanMethod( 320 bool success = jni->CallBooleanMethod(
318 *j_media_codec_video_decoder_, 321 *j_media_codec_video_decoder_,
319 j_init_decode_method_, 322 j_init_decode_method_,
320 j_video_codec_enum, 323 j_video_codec_enum,
321 codec_.width, 324 codec_.width,
322 codec_.height, 325 codec_.height,
323 use_surface_ ? render_egl_context_ : nullptr); 326 use_surface_ ? surface_texture_helper_->GetJavaSurfaceTextureHelper()
327 : nullptr);
324 if (CheckException(jni) || !success) { 328 if (CheckException(jni) || !success) {
325 ALOGE << "Codec initialization error - fallback to SW codec."; 329 ALOGE << "Codec initialization error - fallback to SW codec.";
326 sw_fallback_required_ = true; 330 sw_fallback_required_ = true;
327 return WEBRTC_VIDEO_CODEC_ERROR; 331 return WEBRTC_VIDEO_CODEC_ERROR;
328 } 332 }
329 inited_ = true; 333 inited_ = true;
330 334
331 switch (codecType_) { 335 switch (codecType_) {
332 case kVideoCodecVP8: 336 case kVideoCodecVP8:
333 max_pending_frames_ = kMaxPendingFramesVp8; 337 max_pending_frames_ = kMaxPendingFramesVp8;
334 break; 338 break;
335 case kVideoCodecH264: 339 case kVideoCodecH264:
336 max_pending_frames_ = kMaxPendingFramesH264; 340 max_pending_frames_ = kMaxPendingFramesH264;
337 break; 341 break;
338 default: 342 default:
339 max_pending_frames_ = 0; 343 max_pending_frames_ = 0;
340 } 344 }
341 start_time_ms_ = GetCurrentTimeMs(); 345 start_time_ms_ = GetCurrentTimeMs();
342 current_frames_ = 0; 346 current_frames_ = 0;
343 current_bytes_ = 0; 347 current_bytes_ = 0;
344 current_decoding_time_ms_ = 0; 348 current_decoding_time_ms_ = 0;
345 timestamps_.clear(); 349 timestamps_.clear();
346 ntp_times_ms_.clear(); 350 ntp_times_ms_.clear();
347 frame_rtc_times_ms_.clear();
348 351
349 jobjectArray input_buffers = (jobjectArray)GetObjectField( 352 jobjectArray input_buffers = (jobjectArray)GetObjectField(
350 jni, *j_media_codec_video_decoder_, j_input_buffers_field_); 353 jni, *j_media_codec_video_decoder_, j_input_buffers_field_);
351 size_t num_input_buffers = jni->GetArrayLength(input_buffers); 354 size_t num_input_buffers = jni->GetArrayLength(input_buffers);
352 ALOGD << "Maximum amount of pending frames: " << max_pending_frames_; 355 ALOGD << "Maximum amount of pending frames: " << max_pending_frames_;
353 input_buffers_.resize(num_input_buffers); 356 input_buffers_.resize(num_input_buffers);
354 for (size_t i = 0; i < num_input_buffers; ++i) { 357 for (size_t i = 0; i < num_input_buffers; ++i) {
355 input_buffers_[i] = 358 input_buffers_[i] =
356 jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i)); 359 jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i));
357 if (CheckException(jni)) { 360 if (CheckException(jni)) {
358 ALOGE << "NewGlobalRef error - fallback to SW codec."; 361 ALOGE << "NewGlobalRef error - fallback to SW codec.";
359 sw_fallback_required_ = true; 362 sw_fallback_required_ = true;
360 return WEBRTC_VIDEO_CODEC_ERROR; 363 return WEBRTC_VIDEO_CODEC_ERROR;
361 } 364 }
362 } 365 }
363 366
364 if (use_surface_) {
365 jobject surface_texture = GetObjectField(
366 jni, *j_media_codec_video_decoder_, j_surface_texture_field_);
367 if (previous_surface_texture_ != NULL) {
368 jni->DeleteGlobalRef(previous_surface_texture_);
369 }
370 previous_surface_texture_ = surface_texture_;
371 surface_texture_ = jni->NewGlobalRef(surface_texture);
372 }
373 codec_thread_->PostDelayed(kMediaCodecPollMs, this); 367 codec_thread_->PostDelayed(kMediaCodecPollMs, this);
374 368
375 return WEBRTC_VIDEO_CODEC_OK; 369 return WEBRTC_VIDEO_CODEC_OK;
376 } 370 }
377 371
378 int32_t MediaCodecVideoDecoder::Release() { 372 int32_t MediaCodecVideoDecoder::Release() {
379 ALOGD << "DecoderRelease request"; 373 ALOGD << "DecoderRelease request";
380 return codec_thread_->Invoke<int32_t>( 374 return codec_thread_->Invoke<int32_t>(
381 Bind(&MediaCodecVideoDecoder::ReleaseOnCodecThread, this)); 375 Bind(&MediaCodecVideoDecoder::ReleaseOnCodecThread, this));
382 } 376 }
383 377
384 int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() { 378 int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() {
385 if (!inited_) { 379 if (!inited_) {
386 return WEBRTC_VIDEO_CODEC_OK; 380 return WEBRTC_VIDEO_CODEC_OK;
387 } 381 }
388 CheckOnCodecThread(); 382 CheckOnCodecThread();
389 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 383 JNIEnv* jni = AttachCurrentThreadIfNeeded();
390 ALOGD << "DecoderReleaseOnCodecThread: Frames received: " << 384 ALOGD << "DecoderReleaseOnCodecThread: Frames received: " <<
391 frames_received_ << ". Frames decoded: " << frames_decoded_; 385 frames_received_ << ". Frames decoded: " << frames_decoded_;
392 ScopedLocalRefFrame local_ref_frame(jni); 386 ScopedLocalRefFrame local_ref_frame(jni);
393 for (size_t i = 0; i < input_buffers_.size(); i++) { 387 for (size_t i = 0; i < input_buffers_.size(); i++) {
394 jni->DeleteGlobalRef(input_buffers_[i]); 388 jni->DeleteGlobalRef(input_buffers_[i]);
395 } 389 }
396 input_buffers_.clear(); 390 input_buffers_.clear();
397 jni->CallVoidMethod(*j_media_codec_video_decoder_, j_release_method_); 391 jni->CallVoidMethod(*j_media_codec_video_decoder_, j_release_method_);
392 surface_texture_helper_ = nullptr;
398 inited_ = false; 393 inited_ = false;
399 rtc::MessageQueueManager::Clear(this); 394 rtc::MessageQueueManager::Clear(this);
400 if (CheckException(jni)) { 395 if (CheckException(jni)) {
401 ALOGE << "Decoder release exception"; 396 ALOGE << "Decoder release exception";
402 return WEBRTC_VIDEO_CODEC_ERROR; 397 return WEBRTC_VIDEO_CODEC_ERROR;
403 } 398 }
404 ALOGD << "DecoderReleaseOnCodecThread done"; 399 ALOGD << "DecoderReleaseOnCodecThread done";
405 return WEBRTC_VIDEO_CODEC_OK; 400 return WEBRTC_VIDEO_CODEC_OK;
406 } 401 }
407 402
(...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after
497 const EncodedImage& inputImage) { 492 const EncodedImage& inputImage) {
498 CheckOnCodecThread(); 493 CheckOnCodecThread();
499 JNIEnv* jni = AttachCurrentThreadIfNeeded(); 494 JNIEnv* jni = AttachCurrentThreadIfNeeded();
500 ScopedLocalRefFrame local_ref_frame(jni); 495 ScopedLocalRefFrame local_ref_frame(jni);
501 496
502 // Try to drain the decoder and wait until output is not too 497 // Try to drain the decoder and wait until output is not too
503 // much behind the input. 498 // much behind the input.
504 if (frames_received_ > frames_decoded_ + max_pending_frames_) { 499 if (frames_received_ > frames_decoded_ + max_pending_frames_) {
505 ALOGV("Received: %d. Decoded: %d. Wait for output...", 500 ALOGV("Received: %d. Decoded: %d. Wait for output...",
506 frames_received_, frames_decoded_); 501 frames_received_, frames_decoded_);
507 if (!DeliverPendingOutputs(jni, kMediaCodecTimeoutMs * 1000)) { 502 if (!DeliverPendingOutputs(jni, kMediaCodecTimeoutMs)) {
508 ALOGE << "DeliverPendingOutputs error. Frames received: " << 503 ALOGE << "DeliverPendingOutputs error. Frames received: " <<
509 frames_received_ << ". Frames decoded: " << frames_decoded_; 504 frames_received_ << ". Frames decoded: " << frames_decoded_;
510 return ProcessHWErrorOnCodecThread(); 505 return ProcessHWErrorOnCodecThread();
511 } 506 }
512 if (frames_received_ > frames_decoded_ + max_pending_frames_) { 507 if (frames_received_ > frames_decoded_ + max_pending_frames_) {
513 ALOGE << "Output buffer dequeue timeout. Frames received: " << 508 ALOGE << "Output buffer dequeue timeout. Frames received: " <<
514 frames_received_ << ". Frames decoded: " << frames_decoded_; 509 frames_received_ << ". Frames decoded: " << frames_decoded_;
515 return ProcessHWErrorOnCodecThread(); 510 return ProcessHWErrorOnCodecThread();
516 } 511 }
517 } 512 }
(...skipping 24 matching lines...) Expand all
542 j_input_buffer_index << ". TS: " << (int)(timestamp_us / 1000) 537 j_input_buffer_index << ". TS: " << (int)(timestamp_us / 1000)
543 << ". Size: " << inputImage._length; 538 << ". Size: " << inputImage._length;
544 } 539 }
545 memcpy(buffer, inputImage._buffer, inputImage._length); 540 memcpy(buffer, inputImage._buffer, inputImage._length);
546 541
547 // Save input image timestamps for later output. 542 // Save input image timestamps for later output.
548 frames_received_++; 543 frames_received_++;
549 current_bytes_ += inputImage._length; 544 current_bytes_ += inputImage._length;
550 timestamps_.push_back(inputImage._timeStamp); 545 timestamps_.push_back(inputImage._timeStamp);
551 ntp_times_ms_.push_back(inputImage.ntp_time_ms_); 546 ntp_times_ms_.push_back(inputImage.ntp_time_ms_);
552 frame_rtc_times_ms_.push_back(GetCurrentTimeMs());
553 547
554 // Feed input to decoder. 548 // Feed input to decoder.
555 bool success = jni->CallBooleanMethod(*j_media_codec_video_decoder_, 549 bool success = jni->CallBooleanMethod(*j_media_codec_video_decoder_,
556 j_queue_input_buffer_method_, 550 j_queue_input_buffer_method_,
557 j_input_buffer_index, 551 j_input_buffer_index,
558 inputImage._length, 552 inputImage._length,
559 timestamp_us); 553 timestamp_us);
560 if (CheckException(jni) || !success) { 554 if (CheckException(jni) || !success) {
561 ALOGE << "queueInputBuffer error"; 555 ALOGE << "queueInputBuffer error";
562 return ProcessHWErrorOnCodecThread(); 556 return ProcessHWErrorOnCodecThread();
563 } 557 }
564 558
565 // Try to drain the decoder 559 // Try to drain the decoder
566 if (!DeliverPendingOutputs(jni, 0)) { 560 if (!DeliverPendingOutputs(jni, 0)) {
567 ALOGE << "DeliverPendingOutputs error"; 561 ALOGE << "DeliverPendingOutputs error";
568 return ProcessHWErrorOnCodecThread(); 562 return ProcessHWErrorOnCodecThread();
569 } 563 }
570 564
571 return WEBRTC_VIDEO_CODEC_OK; 565 return WEBRTC_VIDEO_CODEC_OK;
572 } 566 }
573 567
574 bool MediaCodecVideoDecoder::DeliverPendingOutputs( 568 bool MediaCodecVideoDecoder::DeliverPendingOutputs(
575 JNIEnv* jni, int dequeue_timeout_us) { 569 JNIEnv* jni, int dequeue_timeout_ms) {
576 if (frames_received_ <= frames_decoded_) { 570 if (frames_received_ <= frames_decoded_) {
577 // No need to query for output buffers - decoder is drained. 571 // No need to query for output buffers - decoder is drained.
578 return true; 572 return true;
579 } 573 }
580 // Get decoder output. 574 // Get decoder output.
581 jobject j_decoder_output_buffer = jni->CallObjectMethod( 575 jobject j_decoder_output_buffer =
582 *j_media_codec_video_decoder_, 576 jni->CallObjectMethod(*j_media_codec_video_decoder_,
583 j_dequeue_output_buffer_method_, 577 use_surface_ ? j_dequeue_texture_buffer_method_
584 dequeue_timeout_us); 578 : j_dequeue_byte_buffer_method_,
579 dequeue_timeout_ms);
580
585 if (CheckException(jni)) { 581 if (CheckException(jni)) {
586 ALOGE << "dequeueOutputBuffer() error"; 582 ALOGE << "dequeueOutputBuffer() error";
587 return false; 583 return false;
588 } 584 }
589 if (IsNull(jni, j_decoder_output_buffer)) { 585 if (IsNull(jni, j_decoder_output_buffer)) {
590 // No decoded frame ready. 586 // No decoded frame ready.
591 return true; 587 return true;
592 } 588 }
593 589
594 // Get decoded video frame properties. 590 // Get decoded video frame properties.
595 int color_format = GetIntField(jni, *j_media_codec_video_decoder_, 591 int color_format = GetIntField(jni, *j_media_codec_video_decoder_,
596 j_color_format_field_); 592 j_color_format_field_);
597 int width = GetIntField(jni, *j_media_codec_video_decoder_, j_width_field_); 593 int width = GetIntField(jni, *j_media_codec_video_decoder_, j_width_field_);
598 int height = GetIntField(jni, *j_media_codec_video_decoder_, j_height_field_); 594 int height = GetIntField(jni, *j_media_codec_video_decoder_, j_height_field_);
599 int stride = GetIntField(jni, *j_media_codec_video_decoder_, j_stride_field_); 595 int stride = GetIntField(jni, *j_media_codec_video_decoder_, j_stride_field_);
600 int slice_height = GetIntField(jni, *j_media_codec_video_decoder_, 596 int slice_height = GetIntField(jni, *j_media_codec_video_decoder_,
601 j_slice_height_field_); 597 j_slice_height_field_);
602 598
603 rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer; 599 rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer;
604 long output_timestamps_ms = 0; 600 int64_t output_timestamps_ms = 0;
601 int decode_time_ms = 0;
602 int64_t frame_delayed_ms = 0;
605 if (use_surface_) { 603 if (use_surface_) {
606 // Extract data from Java DecodedTextureBuffer. 604 // Extract data from Java DecodedTextureBuffer.
607 const int texture_id = 605 const int texture_id =
608 GetIntField(jni, j_decoder_output_buffer, j_textureID_field_); 606 GetIntField(jni, j_decoder_output_buffer, j_texture_id_field_);
609 const int64_t timestamp_us = 607 if (texture_id != 0) { // |texture_id| == 0 represents a dropped frame.
610 GetLongField(jni, j_decoder_output_buffer, 608 const jfloatArray j_transform_matrix =
611 j_texture_presentation_timestamp_us_field_); 609 reinterpret_cast<jfloatArray>(GetObjectField(
612 output_timestamps_ms = timestamp_us / rtc::kNumMicrosecsPerMillisec; 610 jni, j_decoder_output_buffer, j_transform_matrix_field_));
613 // Create webrtc::VideoFrameBuffer with native texture handle. 611 const int64_t timestamp_us =
614 native_handle_.SetTextureObject(surface_texture_, texture_id); 612 GetLongField(jni, j_decoder_output_buffer,
615 frame_buffer = new rtc::RefCountedObject<JniNativeHandleBuffer>( 613 j_texture_presentation_timestamp_us_field_);
616 &native_handle_, width, height); 614 output_timestamps_ms = timestamp_us / rtc::kNumMicrosecsPerMillisec;
615 decode_time_ms = GetLongField(jni, j_decoder_output_buffer,
616 j_texture_decode_time_ms_field_);
617 frame_delayed_ms = GetLongField(jni, j_decoder_output_buffer,
618 j_texture_frame_delay_ms_field_);
619
620 // Create webrtc::VideoFrameBuffer with native texture handle.
621 frame_buffer = surface_texture_helper_->CreateTextureFrame(
622 width, height, NativeHandleImpl(jni, texture_id, j_transform_matrix));
623 }
617 } else { 624 } else {
618 // Extract data from Java ByteBuffer and create output yuv420 frame - 625 // Extract data from Java ByteBuffer and create output yuv420 frame -
619 // for non surface decoding only. 626 // for non surface decoding only.
620 const int output_buffer_index = 627 const int output_buffer_index =
621 GetIntField(jni, j_decoder_output_buffer, j_info_index_field_); 628 GetIntField(jni, j_decoder_output_buffer, j_info_index_field_);
622 const int output_buffer_offset = 629 const int output_buffer_offset =
623 GetIntField(jni, j_decoder_output_buffer, j_info_offset_field_); 630 GetIntField(jni, j_decoder_output_buffer, j_info_offset_field_);
624 const int output_buffer_size = 631 const int output_buffer_size =
625 GetIntField(jni, j_decoder_output_buffer, j_info_size_field_); 632 GetIntField(jni, j_decoder_output_buffer, j_info_size_field_);
626 const int64_t timestamp_us = GetLongField( 633 const int64_t timestamp_us = GetLongField(
627 jni, j_decoder_output_buffer, j_info_presentation_timestamp_us_field_); 634 jni, j_decoder_output_buffer, j_info_presentation_timestamp_us_field_);
628 output_timestamps_ms = timestamp_us / rtc::kNumMicrosecsPerMillisec; 635 output_timestamps_ms = timestamp_us / rtc::kNumMicrosecsPerMillisec;
636 decode_time_ms = GetLongField(jni, j_decoder_output_buffer,
637 j_byte_buffer_decode_time_ms_field_);
629 638
630 if (output_buffer_size < width * height * 3 / 2) { 639 if (output_buffer_size < width * height * 3 / 2) {
631 ALOGE << "Insufficient output buffer size: " << output_buffer_size; 640 ALOGE << "Insufficient output buffer size: " << output_buffer_size;
632 return false; 641 return false;
633 } 642 }
634 jobjectArray output_buffers = reinterpret_cast<jobjectArray>(GetObjectField( 643 jobjectArray output_buffers = reinterpret_cast<jobjectArray>(GetObjectField(
635 jni, *j_media_codec_video_decoder_, j_output_buffers_field_)); 644 jni, *j_media_codec_video_decoder_, j_output_buffers_field_));
636 jobject output_buffer = 645 jobject output_buffer =
637 jni->GetObjectArrayElement(output_buffers, output_buffer_index); 646 jni->GetObjectArrayElement(output_buffers, output_buffer_index);
638 uint8_t* payload = reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress( 647 uint8_t* payload = reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after
676 frame_buffer->MutableData(webrtc::kVPlane), 685 frame_buffer->MutableData(webrtc::kVPlane),
677 frame_buffer->stride(webrtc::kVPlane), 686 frame_buffer->stride(webrtc::kVPlane),
678 width, height); 687 width, height);
679 } 688 }
680 // Return output byte buffer back to codec. 689 // Return output byte buffer back to codec.
681 jni->CallVoidMethod( 690 jni->CallVoidMethod(
682 *j_media_codec_video_decoder_, 691 *j_media_codec_video_decoder_,
683 j_return_decoded_byte_buffer_method_, 692 j_return_decoded_byte_buffer_method_,
684 output_buffer_index); 693 output_buffer_index);
685 if (CheckException(jni)) { 694 if (CheckException(jni)) {
686 ALOGE << "returnDecodedByteBuffer error"; 695 ALOGE << "returnDecodedOutputBuffer error";
687 return false; 696 return false;
688 } 697 }
689 } 698 }
690 VideoFrame decoded_frame(frame_buffer, 0, 0, webrtc::kVideoRotation_0); 699 VideoFrame decoded_frame(frame_buffer, 0, 0, webrtc::kVideoRotation_0);
691 700
692 // Get frame timestamps from a queue. 701 // Get frame timestamps from a queue.
693 if (timestamps_.size() > 0) { 702 if (timestamps_.size() > 0) {
694 decoded_frame.set_timestamp(timestamps_.front()); 703 decoded_frame.set_timestamp(timestamps_.front());
695 timestamps_.erase(timestamps_.begin()); 704 timestamps_.erase(timestamps_.begin());
696 } 705 }
697 if (ntp_times_ms_.size() > 0) { 706 if (ntp_times_ms_.size() > 0) {
698 decoded_frame.set_ntp_time_ms(ntp_times_ms_.front()); 707 decoded_frame.set_ntp_time_ms(ntp_times_ms_.front());
699 ntp_times_ms_.erase(ntp_times_ms_.begin()); 708 ntp_times_ms_.erase(ntp_times_ms_.begin());
700 } 709 }
701 int64_t frame_decoding_time_ms = 0; 710
702 if (frame_rtc_times_ms_.size() > 0) {
703 frame_decoding_time_ms = GetCurrentTimeMs() - frame_rtc_times_ms_.front();
704 frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin());
705 }
706 if (frames_decoded_ < kMaxDecodedLogFrames) { 711 if (frames_decoded_ < kMaxDecodedLogFrames) {
707 ALOGD << "Decoder frame out # " << frames_decoded_ << ". " << width << 712 ALOGD << "Decoder frame out # " << frames_decoded_ << ". " << width <<
708 " x " << height << ". " << stride << " x " << slice_height << 713 " x " << height << ". " << stride << " x " << slice_height <<
709 ". Color: " << color_format << ". TS:" << (int)output_timestamps_ms << 714 ". Color: " << color_format << ". TS:" << (int)output_timestamps_ms <<
710 ". DecTime: " << (int)frame_decoding_time_ms; 715 ". DecTime: " << (int)decode_time_ms <<
716 ". DelayTime: " << (int)frame_delayed_ms;
711 } 717 }
712 718
713 // Calculate and print decoding statistics - every 3 seconds. 719 // Calculate and print decoding statistics - every 3 seconds.
714 frames_decoded_++; 720 frames_decoded_++;
715 current_frames_++; 721 current_frames_++;
716 current_decoding_time_ms_ += frame_decoding_time_ms; 722 current_decoding_time_ms_ += decode_time_ms;
717 int statistic_time_ms = GetCurrentTimeMs() - start_time_ms_; 723 int statistic_time_ms = GetCurrentTimeMs() - start_time_ms_;
718 if (statistic_time_ms >= kMediaCodecStatisticsIntervalMs && 724 if (statistic_time_ms >= kMediaCodecStatisticsIntervalMs &&
719 current_frames_ > 0) { 725 current_frames_ > 0) {
720 ALOGD << "Decoded frames: " << frames_decoded_ << ". Bitrate: " << 726 ALOGD << "Decoded frames: " << frames_decoded_ << ". Received frames: "
727 << frames_received_ << ". Bitrate: " <<
721 (current_bytes_ * 8 / statistic_time_ms) << " kbps, fps: " << 728 (current_bytes_ * 8 / statistic_time_ms) << " kbps, fps: " <<
722 ((current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms) 729 ((current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms)
723 << ". decTime: " << (current_decoding_time_ms_ / current_frames_) << 730 << ". decTime: " << (current_decoding_time_ms_ / current_frames_) <<
724 " for last " << statistic_time_ms << " ms."; 731 " for last " << statistic_time_ms << " ms.";
725 start_time_ms_ = GetCurrentTimeMs(); 732 start_time_ms_ = GetCurrentTimeMs();
726 current_frames_ = 0; 733 current_frames_ = 0;
727 current_bytes_ = 0; 734 current_bytes_ = 0;
728 current_decoding_time_ms_ = 0; 735 current_decoding_time_ms_ = 0;
729 } 736 }
730 737
731 // Callback - output decoded frame. 738 // |.IsZeroSize())| returns true when a frame has been dropped.
732 const int32_t callback_status = callback_->Decoded(decoded_frame); 739 if (!decoded_frame.IsZeroSize()) {
733 if (callback_status > 0) { 740 // Callback - output decoded frame.
734 ALOGE << "callback error"; 741 const int32_t callback_status =
742 callback_->Decoded(decoded_frame, decode_time_ms);
743 if (callback_status > 0) {
744 ALOGE << "callback error";
745 }
735 } 746 }
736
737 return true; 747 return true;
738 } 748 }
739 749
740 int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback( 750 int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback(
741 DecodedImageCallback* callback) { 751 DecodedImageCallback* callback) {
742 callback_ = callback; 752 callback_ = callback;
743 return WEBRTC_VIDEO_CODEC_OK; 753 return WEBRTC_VIDEO_CODEC_OK;
744 } 754 }
745 755
746 int32_t MediaCodecVideoDecoder::Reset() { 756 int32_t MediaCodecVideoDecoder::Reset() {
(...skipping 109 matching lines...) Expand 10 before | Expand all | Expand 10 after
856 } 866 }
857 867
858 void MediaCodecVideoDecoderFactory::DestroyVideoDecoder( 868 void MediaCodecVideoDecoderFactory::DestroyVideoDecoder(
859 webrtc::VideoDecoder* decoder) { 869 webrtc::VideoDecoder* decoder) {
860 ALOGD << "Destroy video decoder."; 870 ALOGD << "Destroy video decoder.";
861 delete decoder; 871 delete decoder;
862 } 872 }
863 873
864 } // namespace webrtc_jni 874 } // namespace webrtc_jni
865 875
OLDNEW
« no previous file with comments | « talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java ('k') | talk/app/webrtc/java/jni/androidvideocapturer_jni.h » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698