OLD | NEW |
---|---|
1 /* | 1 /* |
2 * libjingle | 2 * libjingle |
3 * Copyright 2014 Google Inc. | 3 * Copyright 2014 Google Inc. |
4 * | 4 * |
5 * Redistribution and use in source and binary forms, with or without | 5 * Redistribution and use in source and binary forms, with or without |
6 * modification, are permitted provided that the following conditions are met: | 6 * modification, are permitted provided that the following conditions are met: |
7 * | 7 * |
8 * 1. Redistributions of source code must retain the above copyright notice, | 8 * 1. Redistributions of source code must retain the above copyright notice, |
9 * this list of conditions and the following disclaimer. | 9 * this list of conditions and the following disclaimer. |
10 * 2. Redistributions in binary form must reproduce the above copyright notice, | 10 * 2. Redistributions in binary form must reproduce the above copyright notice, |
11 * this list of conditions and the following disclaimer in the documentation | 11 * this list of conditions and the following disclaimer in the documentation |
12 * and/or other materials provided with the distribution. | 12 * and/or other materials provided with the distribution. |
13 * 3. The name of the author may not be used to endorse or promote products | 13 * 3. The name of the author may not be used to endorse or promote products |
14 * derived from this software without specific prior written permission. | 14 * derived from this software without specific prior written permission. |
15 * | 15 * |
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED | 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED |
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF | 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF |
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO | 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO |
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, | 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; | 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; |
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, |
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR | 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR |
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF | 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
26 */ | 26 */ |
27 | 27 |
28 package org.webrtc; | 28 package org.webrtc; |
29 | 29 |
30 import android.graphics.SurfaceTexture; | |
31 import android.media.MediaCodec; | 30 import android.media.MediaCodec; |
32 import android.media.MediaCodecInfo; | 31 import android.media.MediaCodecInfo; |
33 import android.media.MediaCodecInfo.CodecCapabilities; | 32 import android.media.MediaCodecInfo.CodecCapabilities; |
34 import android.media.MediaCodecList; | 33 import android.media.MediaCodecList; |
35 import android.media.MediaFormat; | 34 import android.media.MediaFormat; |
36 import android.opengl.GLES11Ext; | |
37 import android.opengl.GLES20; | |
38 import android.os.Build; | 35 import android.os.Build; |
36 import android.os.SystemClock; | |
39 import android.view.Surface; | 37 import android.view.Surface; |
40 | 38 |
41 import org.webrtc.Logging; | 39 import org.webrtc.Logging; |
42 | 40 |
43 import java.nio.ByteBuffer; | 41 import java.nio.ByteBuffer; |
44 import java.util.Arrays; | 42 import java.util.Arrays; |
43 import java.util.LinkedList; | |
45 import java.util.List; | 44 import java.util.List; |
46 import java.util.concurrent.CountDownLatch; | 45 import java.util.concurrent.CountDownLatch; |
47 | 46 import java.util.Queue; |
48 import javax.microedition.khronos.egl.EGLContext; | 47 import java.util.concurrent.TimeUnit; |
49 | 48 |
50 // Java-side of peerconnection_jni.cc:MediaCodecVideoDecoder. | 49 // Java-side of peerconnection_jni.cc:MediaCodecVideoDecoder. |
51 // This class is an implementation detail of the Java PeerConnection API. | 50 // This class is an implementation detail of the Java PeerConnection API. |
52 public class MediaCodecVideoDecoder { | 51 public class MediaCodecVideoDecoder { |
53 // This class is constructed, operated, and destroyed by its C++ incarnation, | 52 // This class is constructed, operated, and destroyed by its C++ incarnation, |
54 // so the class and its methods have non-public visibility. The API this | 53 // so the class and its methods have non-public visibility. The API this |
55 // class exposes aims to mimic the webrtc::VideoDecoder API as closely as | 54 // class exposes aims to mimic the webrtc::VideoDecoder API as closely as |
56 // possibly to minimize the amount of translation work necessary. | 55 // possibly to minimize the amount of translation work necessary. |
57 | 56 |
58 private static final String TAG = "MediaCodecVideoDecoder"; | 57 private static final String TAG = "MediaCodecVideoDecoder"; |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
92 private static final List<Integer> supportedColorList = Arrays.asList( | 91 private static final List<Integer> supportedColorList = Arrays.asList( |
93 CodecCapabilities.COLOR_FormatYUV420Planar, | 92 CodecCapabilities.COLOR_FormatYUV420Planar, |
94 CodecCapabilities.COLOR_FormatYUV420SemiPlanar, | 93 CodecCapabilities.COLOR_FormatYUV420SemiPlanar, |
95 CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, | 94 CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, |
96 COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m); | 95 COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m); |
97 private int colorFormat; | 96 private int colorFormat; |
98 private int width; | 97 private int width; |
99 private int height; | 98 private int height; |
100 private int stride; | 99 private int stride; |
101 private int sliceHeight; | 100 private int sliceHeight; |
101 private boolean hasDecodedFirstFrame; | |
102 private final Queue<Long> decodeStartTimeMs = new LinkedList<Long>(); | |
102 private boolean useSurface; | 103 private boolean useSurface; |
103 private int textureID = 0; | 104 |
104 private SurfaceTexture surfaceTexture = null; | 105 // The below variables are only used when decoding to a Surface. |
106 private TextureListener textureListener; | |
107 private int droppedFrames; | |
108 // |isWaitingForTexture| is true when waiting for the transition: | |
109 // MediaCodec.releaseOutputBuffer() -> onTextureFrameAvailable(). | |
110 private boolean isWaitingForTexture; | |
105 private Surface surface = null; | 111 private Surface surface = null; |
106 private EglBase eglBase; | 112 private final Queue<DecodedOutputBuffer> |
107 | 113 dequeuedSurfaceOutputBuffers = new LinkedList<DecodedOutputBuffer>(); |
108 private MediaCodecVideoDecoder() { | |
109 } | |
110 | 114 |
111 // MediaCodec error handler - invoked when critical error happens which may pr event | 115 // MediaCodec error handler - invoked when critical error happens which may pr event |
112 // further use of media codec API. Now it means that one of media codec instan ces | 116 // further use of media codec API. Now it means that one of media codec instan ces |
113 // is hanging and can no longer be used in the next call. | 117 // is hanging and can no longer be used in the next call. |
114 public static interface MediaCodecVideoDecoderErrorCallback { | 118 public static interface MediaCodecVideoDecoderErrorCallback { |
115 void onMediaCodecVideoDecoderCriticalError(int codecErrors); | 119 void onMediaCodecVideoDecoderCriticalError(int codecErrors); |
116 } | 120 } |
117 | 121 |
118 public static void setErrorCallback(MediaCodecVideoDecoderErrorCallback errorC allback) { | 122 public static void setErrorCallback(MediaCodecVideoDecoderErrorCallback errorC allback) { |
119 Logging.d(TAG, "Set error callback"); | 123 Logging.d(TAG, "Set error callback"); |
(...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
205 } | 209 } |
206 | 210 |
207 private void checkOnMediaCodecThread() throws IllegalStateException { | 211 private void checkOnMediaCodecThread() throws IllegalStateException { |
208 if (mediaCodecThread.getId() != Thread.currentThread().getId()) { | 212 if (mediaCodecThread.getId() != Thread.currentThread().getId()) { |
209 throw new IllegalStateException( | 213 throw new IllegalStateException( |
210 "MediaCodecVideoDecoder previously operated on " + mediaCodecThread + | 214 "MediaCodecVideoDecoder previously operated on " + mediaCodecThread + |
211 " but is now called on " + Thread.currentThread()); | 215 " but is now called on " + Thread.currentThread()); |
212 } | 216 } |
213 } | 217 } |
214 | 218 |
215 // Pass null in |sharedContext| to configure the codec for ByteBuffer output. | 219 // Pass null in |surfaceTextureHelper| to configure the codec for ByteBuffer o utput. |
216 private boolean initDecode(VideoCodecType type, int width, int height, EGLCont ext sharedContext) { | 220 private boolean initDecode( |
221 VideoCodecType type, int width, int height, SurfaceTextureHelper surfaceTe xtureHelper) { | |
217 if (mediaCodecThread != null) { | 222 if (mediaCodecThread != null) { |
218 throw new RuntimeException("Forgot to release()?"); | 223 throw new RuntimeException("Forgot to release()?"); |
219 } | 224 } |
220 useSurface = (sharedContext != null); | 225 useSurface = (surfaceTextureHelper != null); |
221 String mime = null; | 226 String mime = null; |
222 String[] supportedCodecPrefixes = null; | 227 String[] supportedCodecPrefixes = null; |
223 if (type == VideoCodecType.VIDEO_CODEC_VP8) { | 228 if (type == VideoCodecType.VIDEO_CODEC_VP8) { |
224 mime = VP8_MIME_TYPE; | 229 mime = VP8_MIME_TYPE; |
225 supportedCodecPrefixes = supportedVp8HwCodecPrefixes; | 230 supportedCodecPrefixes = supportedVp8HwCodecPrefixes; |
226 } else if (type == VideoCodecType.VIDEO_CODEC_H264) { | 231 } else if (type == VideoCodecType.VIDEO_CODEC_H264) { |
227 mime = H264_MIME_TYPE; | 232 mime = H264_MIME_TYPE; |
228 supportedCodecPrefixes = supportedH264HwCodecPrefixes; | 233 supportedCodecPrefixes = supportedH264HwCodecPrefixes; |
229 } else { | 234 } else { |
230 throw new RuntimeException("Non supported codec " + type); | 235 throw new RuntimeException("Non supported codec " + type); |
231 } | 236 } |
232 DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes); | 237 DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes); |
233 if (properties == null) { | 238 if (properties == null) { |
234 throw new RuntimeException("Cannot find HW decoder for " + type); | 239 throw new RuntimeException("Cannot find HW decoder for " + type); |
235 } | 240 } |
236 Logging.d(TAG, "Java initDecode: " + type + " : "+ width + " x " + height + | 241 Logging.d(TAG, "Java initDecode: " + type + " : "+ width + " x " + height + |
237 ". Color: 0x" + Integer.toHexString(properties.colorFormat) + | 242 ". Color: 0x" + Integer.toHexString(properties.colorFormat) + |
238 ". Use Surface: " + useSurface); | 243 ". Use Surface: " + useSurface); |
239 if (sharedContext != null) { | |
240 Logging.d(TAG, "Decoder shared EGL Context: " + sharedContext); | |
241 } | |
242 runningInstance = this; // Decoder is now running and can be queried for sta ck traces. | 244 runningInstance = this; // Decoder is now running and can be queried for sta ck traces. |
243 mediaCodecThread = Thread.currentThread(); | 245 mediaCodecThread = Thread.currentThread(); |
244 try { | 246 try { |
245 this.width = width; | 247 this.width = width; |
246 this.height = height; | 248 this.height = height; |
247 stride = width; | 249 stride = width; |
248 sliceHeight = height; | 250 sliceHeight = height; |
249 | 251 |
250 if (useSurface) { | 252 if (useSurface) { |
251 // Create shared EGL context. | 253 textureListener = new TextureListener(surfaceTextureHelper); |
252 eglBase = new EglBase(sharedContext, EglBase.ConfigType.PIXEL_BUFFER); | 254 surface = new Surface(surfaceTextureHelper.getSurfaceTexture()); |
253 eglBase.createDummyPbufferSurface(); | |
254 eglBase.makeCurrent(); | |
255 | |
256 // Create output surface | |
257 textureID = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); | |
258 Logging.d(TAG, "Video decoder TextureID = " + textureID); | |
259 surfaceTexture = new SurfaceTexture(textureID); | |
260 surface = new Surface(surfaceTexture); | |
261 } | 255 } |
262 | 256 |
263 MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); | 257 MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); |
264 if (!useSurface) { | 258 if (!useSurface) { |
265 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat); | 259 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat); |
266 } | 260 } |
267 Logging.d(TAG, " Format: " + format); | 261 Logging.d(TAG, " Format: " + format); |
268 mediaCodec = | 262 mediaCodec = |
269 MediaCodecVideoEncoder.createByCodecName(properties.codecName); | 263 MediaCodecVideoEncoder.createByCodecName(properties.codecName); |
270 if (mediaCodec == null) { | 264 if (mediaCodec == null) { |
271 Logging.e(TAG, "Can not create media decoder"); | 265 Logging.e(TAG, "Can not create media decoder"); |
272 return false; | 266 return false; |
273 } | 267 } |
274 mediaCodec.configure(format, surface, null, 0); | 268 mediaCodec.configure(format, surface, null, 0); |
275 mediaCodec.start(); | 269 mediaCodec.start(); |
276 colorFormat = properties.colorFormat; | 270 colorFormat = properties.colorFormat; |
277 outputBuffers = mediaCodec.getOutputBuffers(); | 271 outputBuffers = mediaCodec.getOutputBuffers(); |
278 inputBuffers = mediaCodec.getInputBuffers(); | 272 inputBuffers = mediaCodec.getInputBuffers(); |
273 decodeStartTimeMs.clear(); | |
274 hasDecodedFirstFrame = false; | |
275 dequeuedSurfaceOutputBuffers.clear(); | |
276 droppedFrames = 0; | |
277 isWaitingForTexture = false; | |
279 Logging.d(TAG, "Input buffers: " + inputBuffers.length + | 278 Logging.d(TAG, "Input buffers: " + inputBuffers.length + |
280 ". Output buffers: " + outputBuffers.length); | 279 ". Output buffers: " + outputBuffers.length); |
281 return true; | 280 return true; |
282 } catch (IllegalStateException e) { | 281 } catch (IllegalStateException e) { |
283 Logging.e(TAG, "initDecode failed", e); | 282 Logging.e(TAG, "initDecode failed", e); |
284 return false; | 283 return false; |
285 } | 284 } |
286 } | 285 } |
287 | 286 |
288 private void release() { | 287 private void release() { |
289 Logging.d(TAG, "Java releaseDecoder"); | 288 Logging.d(TAG, "Java releaseDecoder. Total number of dropped frames: " + dro ppedFrames); |
290 checkOnMediaCodecThread(); | 289 checkOnMediaCodecThread(); |
291 | 290 |
292 // Run Mediacodec stop() and release() on separate thread since sometime | 291 // Run Mediacodec stop() and release() on separate thread since sometime |
293 // Mediacodec.stop() may hang. | 292 // Mediacodec.stop() may hang. |
294 final CountDownLatch releaseDone = new CountDownLatch(1); | 293 final CountDownLatch releaseDone = new CountDownLatch(1); |
295 | 294 |
296 Runnable runMediaCodecRelease = new Runnable() { | 295 Runnable runMediaCodecRelease = new Runnable() { |
297 @Override | 296 @Override |
298 public void run() { | 297 public void run() { |
299 try { | 298 try { |
(...skipping 17 matching lines...) Expand all Loading... | |
317 errorCallback.onMediaCodecVideoDecoderCriticalError(codecErrors); | 316 errorCallback.onMediaCodecVideoDecoderCriticalError(codecErrors); |
318 } | 317 } |
319 } | 318 } |
320 | 319 |
321 mediaCodec = null; | 320 mediaCodec = null; |
322 mediaCodecThread = null; | 321 mediaCodecThread = null; |
323 runningInstance = null; | 322 runningInstance = null; |
324 if (useSurface) { | 323 if (useSurface) { |
325 surface.release(); | 324 surface.release(); |
326 surface = null; | 325 surface = null; |
327 Logging.d(TAG, "Delete video decoder TextureID " + textureID); | 326 textureListener.release(); |
328 GLES20.glDeleteTextures(1, new int[] {textureID}, 0); | |
329 textureID = 0; | |
330 eglBase.release(); | |
331 eglBase = null; | |
332 } | 327 } |
333 Logging.d(TAG, "Java releaseDecoder done"); | 328 Logging.d(TAG, "Java releaseDecoder done"); |
334 } | 329 } |
335 | 330 |
336 // Dequeue an input buffer and return its index, -1 if no input buffer is | 331 // Dequeue an input buffer and return its index, -1 if no input buffer is |
337 // available, or -2 if the codec is no longer operative. | 332 // available, or -2 if the codec is no longer operative. |
338 private int dequeueInputBuffer() { | 333 private int dequeueInputBuffer() { |
339 checkOnMediaCodecThread(); | 334 checkOnMediaCodecThread(); |
340 try { | 335 try { |
341 return mediaCodec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT); | 336 return mediaCodec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT); |
342 } catch (IllegalStateException e) { | 337 } catch (IllegalStateException e) { |
343 Logging.e(TAG, "dequeueIntputBuffer failed", e); | 338 Logging.e(TAG, "dequeueIntputBuffer failed", e); |
344 return -2; | 339 return -2; |
345 } | 340 } |
346 } | 341 } |
347 | 342 |
348 private boolean queueInputBuffer( | 343 private boolean queueInputBuffer( |
349 int inputBufferIndex, int size, long timestampUs) { | 344 int inputBufferIndex, int size, long timestampUs) { |
350 checkOnMediaCodecThread(); | 345 checkOnMediaCodecThread(); |
351 try { | 346 try { |
352 inputBuffers[inputBufferIndex].position(0); | 347 inputBuffers[inputBufferIndex].position(0); |
353 inputBuffers[inputBufferIndex].limit(size); | 348 inputBuffers[inputBufferIndex].limit(size); |
349 decodeStartTimeMs.add(SystemClock.elapsedRealtime()); | |
354 mediaCodec.queueInputBuffer(inputBufferIndex, 0, size, timestampUs, 0); | 350 mediaCodec.queueInputBuffer(inputBufferIndex, 0, size, timestampUs, 0); |
355 return true; | 351 return true; |
356 } | 352 } |
357 catch (IllegalStateException e) { | 353 catch (IllegalStateException e) { |
358 Logging.e(TAG, "decode failed", e); | 354 Logging.e(TAG, "decode failed", e); |
359 return false; | 355 return false; |
360 } | 356 } |
361 } | 357 } |
362 | 358 |
363 // Helper structs for dequeueOutputBuffer() below. | 359 // Helper struct for dequeueOutputBuffer() below. |
364 private static class DecodedByteBuffer { | 360 private static class DecodedOutputBuffer { |
365 public DecodedByteBuffer(int index, int offset, int size, long presentationT imestampUs) { | 361 public DecodedOutputBuffer(int index, int offset, int size, long presentatio nTimestampUs, |
362 long decodeTime, long endDecodeTime) { | |
366 this.index = index; | 363 this.index = index; |
367 this.offset = offset; | 364 this.offset = offset; |
368 this.size = size; | 365 this.size = size; |
369 this.presentationTimestampUs = presentationTimestampUs; | 366 this.presentationTimestampUs = presentationTimestampUs; |
367 this.decodeTimeMs = decodeTime; | |
368 this.endDecodeTimeMs = endDecodeTime; | |
370 } | 369 } |
371 | 370 |
372 private final int index; | 371 private final int index; |
373 private final int offset; | 372 private final int offset; |
374 private final int size; | 373 private final int size; |
375 private final long presentationTimestampUs; | 374 private final long presentationTimestampUs; |
375 // Number of ms it took to decode this frame. | |
376 private final long decodeTimeMs; | |
377 // System time when this frame finished decoding. | |
378 private final long endDecodeTimeMs; | |
376 } | 379 } |
377 | 380 |
381 // Helper struct for dequeueTextureBuffer() below. | |
378 private static class DecodedTextureBuffer { | 382 private static class DecodedTextureBuffer { |
379 private final int textureID; | 383 private final int textureID; |
384 private final float[] transformMatrix; | |
380 private final long presentationTimestampUs; | 385 private final long presentationTimestampUs; |
386 private final long decodeTimeMs; | |
387 // Interval from when the frame finished decoding until this buffer has been created. | |
388 // Since there there is only one texture, this interval depend on the time f rom when | |
magjed_webrtc
2015/11/11 10:09:53
Remove extra 'there'
perkj_webrtc
2015/11/11 10:34:30
Done.
| |
389 // a frame is decoded and provided to C++ and until that frame is returned t o the MediaCodec | |
390 // so that the texture can be updated with the next decoded frame. | |
391 private final long frameDelayMs; | |
381 | 392 |
382 public DecodedTextureBuffer(int textureID, long presentationTimestampUs) { | 393 // A DecodedTextureBuffer with zero |textureID| has special meaning and repr esents a frame |
394 // that was dropped. | |
395 public DecodedTextureBuffer(int textureID, float[] transformMatrix, | |
396 long presentationTimestampUs, long decodeTimeMs, long frameDelay) { | |
383 this.textureID = textureID; | 397 this.textureID = textureID; |
398 this.transformMatrix = transformMatrix; | |
384 this.presentationTimestampUs = presentationTimestampUs; | 399 this.presentationTimestampUs = presentationTimestampUs; |
400 this.decodeTimeMs = decodeTimeMs; | |
401 this.frameDelayMs = frameDelay; | |
385 } | 402 } |
386 } | 403 } |
387 | 404 |
388 // Returns null if no decoded buffer is available, and otherwise either a Deco dedByteBuffer or | 405 // Poll based texture listener. |
389 // DecodedTexturebuffer depending on |useSurface| configuration. | 406 private static class TextureListener |
407 implements SurfaceTextureHelper.OnTextureFrameAvailableListener { | |
408 public static class TextureInfo { | |
409 private final int textureID; | |
410 private final float[] transformMatrix; | |
411 | |
412 TextureInfo(int textureId, float[] transformMatrix) { | |
413 this.textureID = textureId; | |
414 this.transformMatrix = transformMatrix; | |
415 } | |
416 } | |
417 private final SurfaceTextureHelper surfaceTextureHelper; | |
418 private TextureInfo textureInfo; | |
419 // |newFrameLock| is used to synchronize arrival of new frames with wait()/n otifyAll(). | |
420 private final Object newFrameLock = new Object(); | |
421 | |
422 public TextureListener(SurfaceTextureHelper surfaceTextureHelper) { | |
423 this.surfaceTextureHelper = surfaceTextureHelper; | |
424 surfaceTextureHelper.setListener(this); | |
425 } | |
426 | |
427 // Callback from |surfaceTextureHelper|. May be called on an arbitrary threa d. | |
428 @Override | |
429 public void onTextureFrameAvailable( | |
430 int oesTextureId, float[] transformMatrix, long timestampNs) { | |
431 synchronized (newFrameLock) { | |
432 if (textureInfo != null) { | |
433 Logging.e(TAG, | |
434 "Unexpected onTextureFrameAvailable() called while already holding a texture."); | |
435 throw new IllegalStateException("Already holding a texture."); | |
436 } | |
437 // |timestampNs| is always zero on some Android versions. | |
438 textureInfo = new TextureInfo(oesTextureId, transformMatrix); | |
439 newFrameLock.notifyAll(); | |
440 } | |
441 } | |
442 | |
443 // Dequeues and returns a TextureInfo if available, or null otherwise. | |
444 public TextureInfo dequeueTextureInfo(int timeoutMs) { | |
445 synchronized (newFrameLock) { | |
446 if (textureInfo == null && timeoutMs > 0) { | |
447 try { | |
448 newFrameLock.wait(timeoutMs); | |
449 } catch(InterruptedException e) { | |
450 // Restore the interrupted status by reinterrupting the thread. | |
451 Thread.currentThread().interrupt(); | |
452 } | |
453 } | |
454 TextureInfo returnedInfo = textureInfo; | |
455 textureInfo = null; | |
456 return returnedInfo; | |
457 } | |
458 } | |
459 | |
460 public void release() { | |
461 // SurfaceTextureHelper.disconnect() will block until any onTextureFrameAv ailable() in | |
462 // progress is done. Therefore, the call to disconnect() must be outside a ny synchronized | |
463 // statement that is also used in the onTextureFrameAvailable() above to a void deadlocks. | |
464 surfaceTextureHelper.disconnect(); | |
465 synchronized (newFrameLock) { | |
466 if (textureInfo != null) { | |
467 surfaceTextureHelper.returnTextureFrame(); | |
468 textureInfo = null; | |
469 } | |
470 } | |
471 } | |
472 } | |
473 | |
474 // Returns null if no decoded buffer is available, and otherwise a DecodedByte Buffer. | |
390 // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an | 475 // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an |
391 // unsupported format, or if |mediaCodec| is not in the Executing state. Throw s CodecException | 476 // unsupported format, or if |mediaCodec| is not in the Executing state. Throw s CodecException |
392 // upon codec error. | 477 // upon codec error. |
393 private Object dequeueOutputBuffer(int dequeueTimeoutUs) | 478 private DecodedOutputBuffer dequeueOutputBuffer(int dequeueTimeoutMs) { |
394 throws IllegalStateException, MediaCodec.CodecException { | |
395 checkOnMediaCodecThread(); | 479 checkOnMediaCodecThread(); |
396 | 480 if (decodeStartTimeMs.isEmpty()) { |
481 return null; | |
482 } | |
397 // Drain the decoder until receiving a decoded buffer or hitting | 483 // Drain the decoder until receiving a decoded buffer or hitting |
398 // MediaCodec.INFO_TRY_AGAIN_LATER. | 484 // MediaCodec.INFO_TRY_AGAIN_LATER. |
399 final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); | 485 final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); |
400 while (true) { | 486 while (true) { |
401 final int result = mediaCodec.dequeueOutputBuffer(info, dequeueTimeoutUs); | 487 final int result = mediaCodec.dequeueOutputBuffer( |
488 info, TimeUnit.MILLISECONDS.toMicros(dequeueTimeoutMs)); | |
402 switch (result) { | 489 switch (result) { |
403 case MediaCodec.INFO_TRY_AGAIN_LATER: | |
404 return null; | |
405 case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: | 490 case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: |
406 outputBuffers = mediaCodec.getOutputBuffers(); | 491 outputBuffers = mediaCodec.getOutputBuffers(); |
407 Logging.d(TAG, "Decoder output buffers changed: " + outputBuffers.leng th); | 492 Logging.d(TAG, "Decoder output buffers changed: " + outputBuffers.leng th); |
493 if (hasDecodedFirstFrame) { | |
494 throw new RuntimeException("Unexpected output buffer change event.") ; | |
495 } | |
408 break; | 496 break; |
409 case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: | 497 case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: |
410 MediaFormat format = mediaCodec.getOutputFormat(); | 498 MediaFormat format = mediaCodec.getOutputFormat(); |
411 Logging.d(TAG, "Decoder format changed: " + format.toString()); | 499 Logging.d(TAG, "Decoder format changed: " + format.toString()); |
500 int new_width = format.getInteger(MediaFormat.KEY_WIDTH); | |
501 int new_height = format.getInteger(MediaFormat.KEY_HEIGHT); | |
502 if (hasDecodedFirstFrame && (new_width != width || new_height != heigh t)) { | |
503 throw new RuntimeException("Unexpected size change. Configured " + w idth + "*" + | |
504 height + ". New " + new_width + "*" + new_height); | |
505 } | |
412 width = format.getInteger(MediaFormat.KEY_WIDTH); | 506 width = format.getInteger(MediaFormat.KEY_WIDTH); |
413 height = format.getInteger(MediaFormat.KEY_HEIGHT); | 507 height = format.getInteger(MediaFormat.KEY_HEIGHT); |
508 | |
414 if (!useSurface && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) { | 509 if (!useSurface && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) { |
415 colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT); | 510 colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT); |
416 Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat)); | 511 Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat)); |
417 if (!supportedColorList.contains(colorFormat)) { | 512 if (!supportedColorList.contains(colorFormat)) { |
418 throw new IllegalStateException("Non supported color format: " + c olorFormat); | 513 throw new IllegalStateException("Non supported color format: " + c olorFormat); |
419 } | 514 } |
420 } | 515 } |
421 if (format.containsKey("stride")) { | 516 if (format.containsKey("stride")) { |
422 stride = format.getInteger("stride"); | 517 stride = format.getInteger("stride"); |
423 } | 518 } |
424 if (format.containsKey("slice-height")) { | 519 if (format.containsKey("slice-height")) { |
425 sliceHeight = format.getInteger("slice-height"); | 520 sliceHeight = format.getInteger("slice-height"); |
426 } | 521 } |
427 Logging.d(TAG, "Frame stride and slice height: " + stride + " x " + sl iceHeight); | 522 Logging.d(TAG, "Frame stride and slice height: " + stride + " x " + sl iceHeight); |
428 stride = Math.max(width, stride); | 523 stride = Math.max(width, stride); |
429 sliceHeight = Math.max(height, sliceHeight); | 524 sliceHeight = Math.max(height, sliceHeight); |
430 break; | 525 break; |
526 case MediaCodec.INFO_TRY_AGAIN_LATER: | |
527 return null; | |
431 default: | 528 default: |
432 // Output buffer decoded. | 529 hasDecodedFirstFrame = true; |
433 if (useSurface) { | 530 return new DecodedOutputBuffer(result, info.offset, info.size, info.pr esentationTimeUs, |
434 mediaCodec.releaseOutputBuffer(result, true /* render */); | 531 SystemClock.elapsedRealtime() - decodeStartTimeMs.remove(), |
435 // TODO(magjed): Wait for SurfaceTexture.onFrameAvailable() before r eturning a texture | 532 SystemClock.elapsedRealtime()); |
436 // frame. | 533 } |
437 return new DecodedTextureBuffer(textureID, info.presentationTimeUs); | |
438 } else { | |
439 return new DecodedByteBuffer(result, info.offset, info.size, info.pr esentationTimeUs); | |
440 } | |
441 } | |
442 } | 534 } |
443 } | 535 } |
444 | 536 |
537 // Returns null if no decoded buffer is available, and otherwise a DecodedText ureBuffer. | |
538 // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an | |
539 // unsupported format, or if |mediaCodec| is not in the Executing state. Throw s CodecException | |
540 // upon codec error. | |
541 private DecodedTextureBuffer dequeueTextureBuffer(int dequeueTimeoutMs) { | |
542 checkOnMediaCodecThread(); | |
543 if (!useSurface) { | |
544 throw new IllegalStateException("dequeueTexture() called for byte buffer d ecoding."); | |
545 } | |
546 | |
547 DecodedOutputBuffer outputBuffer = dequeueOutputBuffer(dequeueTimeoutMs); | |
548 if (outputBuffer != null) { | |
549 if (dequeuedSurfaceOutputBuffers.size() > Math.min(2, outputBuffers.length - 1)) { | |
magjed_webrtc
2015/11/11 10:09:53
I would prefer:
dequeuedSurfaceOutputBuffers.size(
perkj_webrtc
2015/11/11 10:34:30
Done.
| |
550 ++droppedFrames; | |
551 Logging.w(TAG, "Too many output buffers. Dropping frame. Total number of dropped frames: " | |
552 + droppedFrames); | |
553 // Drop the newest frame. Don't drop the oldest since if |isWaitingForTe xture| | |
554 // releaseOutputBuffer has already been called. Dropping the newest fram e will lead to a | |
555 // shift of timestamps by one frame in MediaCodecVideoDecoder::DeliverP endingOutputs. | |
556 mediaCodec.releaseOutputBuffer(outputBuffer.index, false /* render */); | |
557 return new DecodedTextureBuffer(0, null, outputBuffer.presentationTimest ampUs, | |
558 outputBuffer.decodeTimeMs, | |
559 SystemClock.elapsedRealtime() - outputBuffer.endDecodeTimeMs); | |
560 } | |
561 dequeuedSurfaceOutputBuffers.add(outputBuffer); | |
562 } | |
563 | |
564 if (dequeuedSurfaceOutputBuffers.isEmpty()) { | |
565 return null; | |
566 } | |
567 | |
568 if (!isWaitingForTexture) { | |
569 // Get the first frame in the queue and render to the decoder output surfa ce. | |
570 mediaCodec.releaseOutputBuffer(dequeuedSurfaceOutputBuffers.peek().index, true /* render */); | |
571 isWaitingForTexture = true; | |
572 } | |
573 | |
574 // We are waiting for a frame to be rendered to the decoder surface. | |
575 // Check if it is ready now by waiting max |dequeueTimeoutMs|. There can onl y be one frame | |
576 // rendered at a time. | |
577 TextureListener.TextureInfo info = textureListener.dequeueTextureInfo(dequeu eTimeoutMs); | |
578 if (info != null) { | |
579 isWaitingForTexture = false; | |
580 final DecodedOutputBuffer renderedBuffer = | |
581 dequeuedSurfaceOutputBuffers.remove(); | |
582 if (!dequeuedSurfaceOutputBuffers.isEmpty()) { | |
583 // Get the next frame in the queue and render to the decoder output surf ace. | |
584 mediaCodec.releaseOutputBuffer( | |
585 dequeuedSurfaceOutputBuffers.peek().index, true /* render */); | |
586 isWaitingForTexture = true; | |
587 } | |
588 | |
589 return new DecodedTextureBuffer(info.textureID, info.transformMatrix, | |
590 renderedBuffer.presentationTimestampUs, renderedBuffer.decodeTimeMs, | |
591 SystemClock.elapsedRealtime() - renderedBuffer.endDecodeTimeMs); | |
592 } | |
593 return null; | |
594 } | |
595 | |
445 // Release a dequeued output byte buffer back to the codec for re-use. Should only be called for | 596 // Release a dequeued output byte buffer back to the codec for re-use. Should only be called for |
446 // non-surface decoding. | 597 // non-surface decoding. |
447 // Throws IllegalStateException if the call is made on the wrong thread, if co dec is configured | 598 // Throws IllegalStateException if the call is made on the wrong thread, if co dec is configured |
448 // for surface decoding, or if |mediaCodec| is not in the Executing state. Thr ows | 599 // for surface decoding, or if |mediaCodec| is not in the Executing state. Thr ows |
449 // MediaCodec.CodecException upon codec error. | 600 // MediaCodec.CodecException upon codec error. |
450 private void returnDecodedByteBuffer(int index) | 601 private void returnDecodedOutputBuffer(int index) |
451 throws IllegalStateException, MediaCodec.CodecException { | 602 throws IllegalStateException, MediaCodec.CodecException { |
452 checkOnMediaCodecThread(); | 603 checkOnMediaCodecThread(); |
453 if (useSurface) { | 604 if (useSurface) { |
454 throw new IllegalStateException("returnDecodedByteBuffer() called for surf ace decoding."); | 605 throw new IllegalStateException("returnDecodedOutputBuffer() called for su rface decoding."); |
455 } | 606 } |
456 mediaCodec.releaseOutputBuffer(index, false /* render */); | 607 mediaCodec.releaseOutputBuffer(index, false /* render */); |
457 } | 608 } |
458 } | 609 } |
OLD | NEW |