OLD | NEW |
1 /* | 1 /* |
2 * libjingle | 2 * libjingle |
3 * Copyright 2014 Google Inc. | 3 * Copyright 2014 Google Inc. |
4 * | 4 * |
5 * Redistribution and use in source and binary forms, with or without | 5 * Redistribution and use in source and binary forms, with or without |
6 * modification, are permitted provided that the following conditions are met: | 6 * modification, are permitted provided that the following conditions are met: |
7 * | 7 * |
8 * 1. Redistributions of source code must retain the above copyright notice, | 8 * 1. Redistributions of source code must retain the above copyright notice, |
9 * this list of conditions and the following disclaimer. | 9 * this list of conditions and the following disclaimer. |
10 * 2. Redistributions in binary form must reproduce the above copyright notice, | 10 * 2. Redistributions in binary form must reproduce the above copyright notice, |
11 * this list of conditions and the following disclaimer in the documentation | 11 * this list of conditions and the following disclaimer in the documentation |
12 * and/or other materials provided with the distribution. | 12 * and/or other materials provided with the distribution. |
13 * 3. The name of the author may not be used to endorse or promote products | 13 * 3. The name of the author may not be used to endorse or promote products |
14 * derived from this software without specific prior written permission. | 14 * derived from this software without specific prior written permission. |
15 * | 15 * |
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED | 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED |
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF | 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF |
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO | 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO |
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, | 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; | 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; |
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, |
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR | 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR |
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF | 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
26 */ | 26 */ |
27 | 27 |
28 package org.webrtc; | 28 package org.webrtc; |
29 | 29 |
30 import android.graphics.SurfaceTexture; | |
31 import android.media.MediaCodec; | 30 import android.media.MediaCodec; |
32 import android.media.MediaCodecInfo; | 31 import android.media.MediaCodecInfo; |
33 import android.media.MediaCodecInfo.CodecCapabilities; | 32 import android.media.MediaCodecInfo.CodecCapabilities; |
34 import android.media.MediaCodecList; | 33 import android.media.MediaCodecList; |
35 import android.media.MediaFormat; | 34 import android.media.MediaFormat; |
36 import android.opengl.GLES11Ext; | |
37 import android.opengl.GLES20; | |
38 import android.os.Build; | 35 import android.os.Build; |
| 36 import android.os.SystemClock; |
39 import android.view.Surface; | 37 import android.view.Surface; |
40 | 38 |
41 import org.webrtc.Logging; | 39 import org.webrtc.Logging; |
42 | 40 |
43 import java.nio.ByteBuffer; | 41 import java.nio.ByteBuffer; |
44 import java.util.Arrays; | 42 import java.util.Arrays; |
| 43 import java.util.LinkedList; |
45 import java.util.List; | 44 import java.util.List; |
46 import java.util.concurrent.CountDownLatch; | 45 import java.util.concurrent.CountDownLatch; |
47 | 46 import java.util.Queue; |
48 import javax.microedition.khronos.egl.EGLContext; | 47 import java.util.concurrent.TimeUnit; |
49 | 48 |
50 // Java-side of peerconnection_jni.cc:MediaCodecVideoDecoder. | 49 // Java-side of peerconnection_jni.cc:MediaCodecVideoDecoder. |
51 // This class is an implementation detail of the Java PeerConnection API. | 50 // This class is an implementation detail of the Java PeerConnection API. |
52 public class MediaCodecVideoDecoder { | 51 public class MediaCodecVideoDecoder { |
53 // This class is constructed, operated, and destroyed by its C++ incarnation, | 52 // This class is constructed, operated, and destroyed by its C++ incarnation, |
54 // so the class and its methods have non-public visibility. The API this | 53 // so the class and its methods have non-public visibility. The API this |
55 // class exposes aims to mimic the webrtc::VideoDecoder API as closely as | 54 // class exposes aims to mimic the webrtc::VideoDecoder API as closely as |
56 // possibly to minimize the amount of translation work necessary. | 55 // possibly to minimize the amount of translation work necessary. |
57 | 56 |
58 private static final String TAG = "MediaCodecVideoDecoder"; | 57 private static final String TAG = "MediaCodecVideoDecoder"; |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
92 private static final List<Integer> supportedColorList = Arrays.asList( | 91 private static final List<Integer> supportedColorList = Arrays.asList( |
93 CodecCapabilities.COLOR_FormatYUV420Planar, | 92 CodecCapabilities.COLOR_FormatYUV420Planar, |
94 CodecCapabilities.COLOR_FormatYUV420SemiPlanar, | 93 CodecCapabilities.COLOR_FormatYUV420SemiPlanar, |
95 CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, | 94 CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, |
96 COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m); | 95 COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m); |
97 private int colorFormat; | 96 private int colorFormat; |
98 private int width; | 97 private int width; |
99 private int height; | 98 private int height; |
100 private int stride; | 99 private int stride; |
101 private int sliceHeight; | 100 private int sliceHeight; |
| 101 private boolean hasDecodedFirstFrame; |
| 102 private final Queue<Long> decodeStartTimeMs = new LinkedList<Long>(); |
102 private boolean useSurface; | 103 private boolean useSurface; |
103 private int textureID = 0; | 104 |
104 private SurfaceTexture surfaceTexture = null; | 105 // The below variables are only used when decoding to a Surface. |
| 106 private TextureListener textureListener; |
| 107 // Max number of output buffers queued before starting to drop decoded frames. |
| 108 private static final int MAX_QUEUED_OUTPUTBUFFERS = 3; |
| 109 private int droppedFrames; |
| 110 // |isWaitingForTexture| is true when waiting for the transition: |
| 111 // MediaCodec.releaseOutputBuffer() -> onTextureFrameAvailable(). |
| 112 private boolean isWaitingForTexture; |
105 private Surface surface = null; | 113 private Surface surface = null; |
106 private EglBase eglBase; | 114 private final Queue<DecodedOutputBuffer> |
107 | 115 dequeuedSurfaceOutputBuffers = new LinkedList<DecodedOutputBuffer>(); |
108 private MediaCodecVideoDecoder() { | |
109 } | |
110 | 116 |
111 // MediaCodec error handler - invoked when critical error happens which may pr
event | 117 // MediaCodec error handler - invoked when critical error happens which may pr
event |
112 // further use of media codec API. Now it means that one of media codec instan
ces | 118 // further use of media codec API. Now it means that one of media codec instan
ces |
113 // is hanging and can no longer be used in the next call. | 119 // is hanging and can no longer be used in the next call. |
114 public static interface MediaCodecVideoDecoderErrorCallback { | 120 public static interface MediaCodecVideoDecoderErrorCallback { |
115 void onMediaCodecVideoDecoderCriticalError(int codecErrors); | 121 void onMediaCodecVideoDecoderCriticalError(int codecErrors); |
116 } | 122 } |
117 | 123 |
118 public static void setErrorCallback(MediaCodecVideoDecoderErrorCallback errorC
allback) { | 124 public static void setErrorCallback(MediaCodecVideoDecoderErrorCallback errorC
allback) { |
119 Logging.d(TAG, "Set error callback"); | 125 Logging.d(TAG, "Set error callback"); |
(...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
205 } | 211 } |
206 | 212 |
207 private void checkOnMediaCodecThread() throws IllegalStateException { | 213 private void checkOnMediaCodecThread() throws IllegalStateException { |
208 if (mediaCodecThread.getId() != Thread.currentThread().getId()) { | 214 if (mediaCodecThread.getId() != Thread.currentThread().getId()) { |
209 throw new IllegalStateException( | 215 throw new IllegalStateException( |
210 "MediaCodecVideoDecoder previously operated on " + mediaCodecThread + | 216 "MediaCodecVideoDecoder previously operated on " + mediaCodecThread + |
211 " but is now called on " + Thread.currentThread()); | 217 " but is now called on " + Thread.currentThread()); |
212 } | 218 } |
213 } | 219 } |
214 | 220 |
215 // Pass null in |sharedContext| to configure the codec for ByteBuffer output. | 221 // Pass null in |surfaceTextureHelper| to configure the codec for ByteBuffer o
utput. |
216 private boolean initDecode(VideoCodecType type, int width, int height, EGLCont
ext sharedContext) { | 222 private boolean initDecode( |
| 223 VideoCodecType type, int width, int height, SurfaceTextureHelper surfaceTe
xtureHelper) { |
217 if (mediaCodecThread != null) { | 224 if (mediaCodecThread != null) { |
218 throw new RuntimeException("Forgot to release()?"); | 225 throw new RuntimeException("Forgot to release()?"); |
219 } | 226 } |
220 useSurface = (sharedContext != null); | 227 useSurface = (surfaceTextureHelper != null); |
221 String mime = null; | 228 String mime = null; |
222 String[] supportedCodecPrefixes = null; | 229 String[] supportedCodecPrefixes = null; |
223 if (type == VideoCodecType.VIDEO_CODEC_VP8) { | 230 if (type == VideoCodecType.VIDEO_CODEC_VP8) { |
224 mime = VP8_MIME_TYPE; | 231 mime = VP8_MIME_TYPE; |
225 supportedCodecPrefixes = supportedVp8HwCodecPrefixes; | 232 supportedCodecPrefixes = supportedVp8HwCodecPrefixes; |
226 } else if (type == VideoCodecType.VIDEO_CODEC_H264) { | 233 } else if (type == VideoCodecType.VIDEO_CODEC_H264) { |
227 mime = H264_MIME_TYPE; | 234 mime = H264_MIME_TYPE; |
228 supportedCodecPrefixes = supportedH264HwCodecPrefixes; | 235 supportedCodecPrefixes = supportedH264HwCodecPrefixes; |
229 } else { | 236 } else { |
230 throw new RuntimeException("Non supported codec " + type); | 237 throw new RuntimeException("Non supported codec " + type); |
231 } | 238 } |
232 DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes); | 239 DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes); |
233 if (properties == null) { | 240 if (properties == null) { |
234 throw new RuntimeException("Cannot find HW decoder for " + type); | 241 throw new RuntimeException("Cannot find HW decoder for " + type); |
235 } | 242 } |
236 Logging.d(TAG, "Java initDecode: " + type + " : "+ width + " x " + height + | 243 Logging.d(TAG, "Java initDecode: " + type + " : "+ width + " x " + height + |
237 ". Color: 0x" + Integer.toHexString(properties.colorFormat) + | 244 ". Color: 0x" + Integer.toHexString(properties.colorFormat) + |
238 ". Use Surface: " + useSurface); | 245 ". Use Surface: " + useSurface); |
239 if (sharedContext != null) { | |
240 Logging.d(TAG, "Decoder shared EGL Context: " + sharedContext); | |
241 } | |
242 runningInstance = this; // Decoder is now running and can be queried for sta
ck traces. | 246 runningInstance = this; // Decoder is now running and can be queried for sta
ck traces. |
243 mediaCodecThread = Thread.currentThread(); | 247 mediaCodecThread = Thread.currentThread(); |
244 try { | 248 try { |
245 this.width = width; | 249 this.width = width; |
246 this.height = height; | 250 this.height = height; |
247 stride = width; | 251 stride = width; |
248 sliceHeight = height; | 252 sliceHeight = height; |
249 | 253 |
250 if (useSurface) { | 254 if (useSurface) { |
251 // Create shared EGL context. | 255 textureListener = new TextureListener(surfaceTextureHelper); |
252 eglBase = new EglBase(sharedContext, EglBase.ConfigType.PIXEL_BUFFER); | 256 surface = new Surface(surfaceTextureHelper.getSurfaceTexture()); |
253 eglBase.createDummyPbufferSurface(); | |
254 eglBase.makeCurrent(); | |
255 | |
256 // Create output surface | |
257 textureID = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); | |
258 Logging.d(TAG, "Video decoder TextureID = " + textureID); | |
259 surfaceTexture = new SurfaceTexture(textureID); | |
260 surface = new Surface(surfaceTexture); | |
261 } | 257 } |
262 | 258 |
263 MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); | 259 MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); |
264 if (!useSurface) { | 260 if (!useSurface) { |
265 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat); | 261 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat); |
266 } | 262 } |
267 Logging.d(TAG, " Format: " + format); | 263 Logging.d(TAG, " Format: " + format); |
268 mediaCodec = | 264 mediaCodec = |
269 MediaCodecVideoEncoder.createByCodecName(properties.codecName); | 265 MediaCodecVideoEncoder.createByCodecName(properties.codecName); |
270 if (mediaCodec == null) { | 266 if (mediaCodec == null) { |
271 Logging.e(TAG, "Can not create media decoder"); | 267 Logging.e(TAG, "Can not create media decoder"); |
272 return false; | 268 return false; |
273 } | 269 } |
274 mediaCodec.configure(format, surface, null, 0); | 270 mediaCodec.configure(format, surface, null, 0); |
275 mediaCodec.start(); | 271 mediaCodec.start(); |
276 colorFormat = properties.colorFormat; | 272 colorFormat = properties.colorFormat; |
277 outputBuffers = mediaCodec.getOutputBuffers(); | 273 outputBuffers = mediaCodec.getOutputBuffers(); |
278 inputBuffers = mediaCodec.getInputBuffers(); | 274 inputBuffers = mediaCodec.getInputBuffers(); |
| 275 decodeStartTimeMs.clear(); |
| 276 hasDecodedFirstFrame = false; |
| 277 dequeuedSurfaceOutputBuffers.clear(); |
| 278 droppedFrames = 0; |
| 279 isWaitingForTexture = false; |
279 Logging.d(TAG, "Input buffers: " + inputBuffers.length + | 280 Logging.d(TAG, "Input buffers: " + inputBuffers.length + |
280 ". Output buffers: " + outputBuffers.length); | 281 ". Output buffers: " + outputBuffers.length); |
281 return true; | 282 return true; |
282 } catch (IllegalStateException e) { | 283 } catch (IllegalStateException e) { |
283 Logging.e(TAG, "initDecode failed", e); | 284 Logging.e(TAG, "initDecode failed", e); |
284 return false; | 285 return false; |
285 } | 286 } |
286 } | 287 } |
287 | 288 |
288 private void release() { | 289 private void release() { |
289 Logging.d(TAG, "Java releaseDecoder"); | 290 Logging.d(TAG, "Java releaseDecoder. Total number of dropped frames: " + dro
ppedFrames); |
290 checkOnMediaCodecThread(); | 291 checkOnMediaCodecThread(); |
291 | 292 |
292 // Run Mediacodec stop() and release() on separate thread since sometime | 293 // Run Mediacodec stop() and release() on separate thread since sometime |
293 // Mediacodec.stop() may hang. | 294 // Mediacodec.stop() may hang. |
294 final CountDownLatch releaseDone = new CountDownLatch(1); | 295 final CountDownLatch releaseDone = new CountDownLatch(1); |
295 | 296 |
296 Runnable runMediaCodecRelease = new Runnable() { | 297 Runnable runMediaCodecRelease = new Runnable() { |
297 @Override | 298 @Override |
298 public void run() { | 299 public void run() { |
299 try { | 300 try { |
(...skipping 17 matching lines...) Expand all Loading... |
317 errorCallback.onMediaCodecVideoDecoderCriticalError(codecErrors); | 318 errorCallback.onMediaCodecVideoDecoderCriticalError(codecErrors); |
318 } | 319 } |
319 } | 320 } |
320 | 321 |
321 mediaCodec = null; | 322 mediaCodec = null; |
322 mediaCodecThread = null; | 323 mediaCodecThread = null; |
323 runningInstance = null; | 324 runningInstance = null; |
324 if (useSurface) { | 325 if (useSurface) { |
325 surface.release(); | 326 surface.release(); |
326 surface = null; | 327 surface = null; |
327 Logging.d(TAG, "Delete video decoder TextureID " + textureID); | 328 textureListener.release(); |
328 GLES20.glDeleteTextures(1, new int[] {textureID}, 0); | |
329 textureID = 0; | |
330 eglBase.release(); | |
331 eglBase = null; | |
332 } | 329 } |
333 Logging.d(TAG, "Java releaseDecoder done"); | 330 Logging.d(TAG, "Java releaseDecoder done"); |
334 } | 331 } |
335 | 332 |
336 // Dequeue an input buffer and return its index, -1 if no input buffer is | 333 // Dequeue an input buffer and return its index, -1 if no input buffer is |
337 // available, or -2 if the codec is no longer operative. | 334 // available, or -2 if the codec is no longer operative. |
338 private int dequeueInputBuffer() { | 335 private int dequeueInputBuffer() { |
339 checkOnMediaCodecThread(); | 336 checkOnMediaCodecThread(); |
340 try { | 337 try { |
341 return mediaCodec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT); | 338 return mediaCodec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT); |
342 } catch (IllegalStateException e) { | 339 } catch (IllegalStateException e) { |
343 Logging.e(TAG, "dequeueIntputBuffer failed", e); | 340 Logging.e(TAG, "dequeueIntputBuffer failed", e); |
344 return -2; | 341 return -2; |
345 } | 342 } |
346 } | 343 } |
347 | 344 |
348 private boolean queueInputBuffer( | 345 private boolean queueInputBuffer( |
349 int inputBufferIndex, int size, long timestampUs) { | 346 int inputBufferIndex, int size, long timestampUs) { |
350 checkOnMediaCodecThread(); | 347 checkOnMediaCodecThread(); |
351 try { | 348 try { |
352 inputBuffers[inputBufferIndex].position(0); | 349 inputBuffers[inputBufferIndex].position(0); |
353 inputBuffers[inputBufferIndex].limit(size); | 350 inputBuffers[inputBufferIndex].limit(size); |
| 351 decodeStartTimeMs.add(SystemClock.elapsedRealtime()); |
354 mediaCodec.queueInputBuffer(inputBufferIndex, 0, size, timestampUs, 0); | 352 mediaCodec.queueInputBuffer(inputBufferIndex, 0, size, timestampUs, 0); |
355 return true; | 353 return true; |
356 } | 354 } |
357 catch (IllegalStateException e) { | 355 catch (IllegalStateException e) { |
358 Logging.e(TAG, "decode failed", e); | 356 Logging.e(TAG, "decode failed", e); |
359 return false; | 357 return false; |
360 } | 358 } |
361 } | 359 } |
362 | 360 |
363 // Helper structs for dequeueOutputBuffer() below. | 361 // Helper struct for dequeueOutputBuffer() below. |
364 private static class DecodedByteBuffer { | 362 private static class DecodedOutputBuffer { |
365 public DecodedByteBuffer(int index, int offset, int size, long presentationT
imestampUs) { | 363 public DecodedOutputBuffer(int index, int offset, int size, long presentatio
nTimestampUs, |
| 364 long decodeTime, long endDecodeTime) { |
366 this.index = index; | 365 this.index = index; |
367 this.offset = offset; | 366 this.offset = offset; |
368 this.size = size; | 367 this.size = size; |
369 this.presentationTimestampUs = presentationTimestampUs; | 368 this.presentationTimestampUs = presentationTimestampUs; |
| 369 this.decodeTimeMs = decodeTime; |
| 370 this.endDecodeTimeMs = endDecodeTime; |
370 } | 371 } |
371 | 372 |
372 private final int index; | 373 private final int index; |
373 private final int offset; | 374 private final int offset; |
374 private final int size; | 375 private final int size; |
375 private final long presentationTimestampUs; | 376 private final long presentationTimestampUs; |
| 377 // Number of ms it took to decode this frame. |
| 378 private final long decodeTimeMs; |
| 379 // System time when this frame finished decoding. |
| 380 private final long endDecodeTimeMs; |
376 } | 381 } |
377 | 382 |
| 383 // Helper struct for dequeueTextureBuffer() below. |
378 private static class DecodedTextureBuffer { | 384 private static class DecodedTextureBuffer { |
379 private final int textureID; | 385 private final int textureID; |
| 386 private final float[] transformMatrix; |
380 private final long presentationTimestampUs; | 387 private final long presentationTimestampUs; |
| 388 private final long decodeTimeMs; |
| 389 // Interval from when the frame finished decoding until this buffer has been
created. |
| 390 // Since there is only one texture, this interval depend on the time from wh
en |
| 391 // a frame is decoded and provided to C++ and until that frame is returned t
o the MediaCodec |
| 392 // so that the texture can be updated with the next decoded frame. |
| 393 private final long frameDelayMs; |
381 | 394 |
382 public DecodedTextureBuffer(int textureID, long presentationTimestampUs) { | 395 // A DecodedTextureBuffer with zero |textureID| has special meaning and repr
esents a frame |
| 396 // that was dropped. |
| 397 public DecodedTextureBuffer(int textureID, float[] transformMatrix, |
| 398 long presentationTimestampUs, long decodeTimeMs, long frameDelay) { |
383 this.textureID = textureID; | 399 this.textureID = textureID; |
| 400 this.transformMatrix = transformMatrix; |
384 this.presentationTimestampUs = presentationTimestampUs; | 401 this.presentationTimestampUs = presentationTimestampUs; |
| 402 this.decodeTimeMs = decodeTimeMs; |
| 403 this.frameDelayMs = frameDelay; |
385 } | 404 } |
386 } | 405 } |
387 | 406 |
388 // Returns null if no decoded buffer is available, and otherwise either a Deco
dedByteBuffer or | 407 // Poll based texture listener. |
389 // DecodedTexturebuffer depending on |useSurface| configuration. | 408 private static class TextureListener |
| 409 implements SurfaceTextureHelper.OnTextureFrameAvailableListener { |
| 410 public static class TextureInfo { |
| 411 private final int textureID; |
| 412 private final float[] transformMatrix; |
| 413 |
| 414 TextureInfo(int textureId, float[] transformMatrix) { |
| 415 this.textureID = textureId; |
| 416 this.transformMatrix = transformMatrix; |
| 417 } |
| 418 } |
| 419 private final SurfaceTextureHelper surfaceTextureHelper; |
| 420 private TextureInfo textureInfo; |
| 421 // |newFrameLock| is used to synchronize arrival of new frames with wait()/n
otifyAll(). |
| 422 private final Object newFrameLock = new Object(); |
| 423 |
| 424 public TextureListener(SurfaceTextureHelper surfaceTextureHelper) { |
| 425 this.surfaceTextureHelper = surfaceTextureHelper; |
| 426 surfaceTextureHelper.setListener(this); |
| 427 } |
| 428 |
| 429 // Callback from |surfaceTextureHelper|. May be called on an arbitrary threa
d. |
| 430 @Override |
| 431 public void onTextureFrameAvailable( |
| 432 int oesTextureId, float[] transformMatrix, long timestampNs) { |
| 433 synchronized (newFrameLock) { |
| 434 if (textureInfo != null) { |
| 435 Logging.e(TAG, |
| 436 "Unexpected onTextureFrameAvailable() called while already holding
a texture."); |
| 437 throw new IllegalStateException("Already holding a texture."); |
| 438 } |
| 439 // |timestampNs| is always zero on some Android versions. |
| 440 textureInfo = new TextureInfo(oesTextureId, transformMatrix); |
| 441 newFrameLock.notifyAll(); |
| 442 } |
| 443 } |
| 444 |
| 445 // Dequeues and returns a TextureInfo if available, or null otherwise. |
| 446 public TextureInfo dequeueTextureInfo(int timeoutMs) { |
| 447 synchronized (newFrameLock) { |
| 448 if (textureInfo == null && timeoutMs > 0) { |
| 449 try { |
| 450 newFrameLock.wait(timeoutMs); |
| 451 } catch(InterruptedException e) { |
| 452 // Restore the interrupted status by reinterrupting the thread. |
| 453 Thread.currentThread().interrupt(); |
| 454 } |
| 455 } |
| 456 TextureInfo returnedInfo = textureInfo; |
| 457 textureInfo = null; |
| 458 return returnedInfo; |
| 459 } |
| 460 } |
| 461 |
| 462 public void release() { |
| 463 // SurfaceTextureHelper.disconnect() will block until any onTextureFrameAv
ailable() in |
| 464 // progress is done. Therefore, the call to disconnect() must be outside a
ny synchronized |
| 465 // statement that is also used in the onTextureFrameAvailable() above to a
void deadlocks. |
| 466 surfaceTextureHelper.disconnect(); |
| 467 synchronized (newFrameLock) { |
| 468 if (textureInfo != null) { |
| 469 surfaceTextureHelper.returnTextureFrame(); |
| 470 textureInfo = null; |
| 471 } |
| 472 } |
| 473 } |
| 474 } |
| 475 |
| 476 // Returns null if no decoded buffer is available, and otherwise a DecodedByte
Buffer. |
390 // Throws IllegalStateException if call is made on the wrong thread, if color
format changes to an | 477 // Throws IllegalStateException if call is made on the wrong thread, if color
format changes to an |
391 // unsupported format, or if |mediaCodec| is not in the Executing state. Throw
s CodecException | 478 // unsupported format, or if |mediaCodec| is not in the Executing state. Throw
s CodecException |
392 // upon codec error. | 479 // upon codec error. |
393 private Object dequeueOutputBuffer(int dequeueTimeoutUs) | 480 private DecodedOutputBuffer dequeueOutputBuffer(int dequeueTimeoutMs) { |
394 throws IllegalStateException, MediaCodec.CodecException { | |
395 checkOnMediaCodecThread(); | 481 checkOnMediaCodecThread(); |
396 | 482 if (decodeStartTimeMs.isEmpty()) { |
| 483 return null; |
| 484 } |
397 // Drain the decoder until receiving a decoded buffer or hitting | 485 // Drain the decoder until receiving a decoded buffer or hitting |
398 // MediaCodec.INFO_TRY_AGAIN_LATER. | 486 // MediaCodec.INFO_TRY_AGAIN_LATER. |
399 final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); | 487 final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); |
400 while (true) { | 488 while (true) { |
401 final int result = mediaCodec.dequeueOutputBuffer(info, dequeueTimeoutUs); | 489 final int result = mediaCodec.dequeueOutputBuffer( |
| 490 info, TimeUnit.MILLISECONDS.toMicros(dequeueTimeoutMs)); |
402 switch (result) { | 491 switch (result) { |
403 case MediaCodec.INFO_TRY_AGAIN_LATER: | |
404 return null; | |
405 case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: | 492 case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: |
406 outputBuffers = mediaCodec.getOutputBuffers(); | 493 outputBuffers = mediaCodec.getOutputBuffers(); |
407 Logging.d(TAG, "Decoder output buffers changed: " + outputBuffers.leng
th); | 494 Logging.d(TAG, "Decoder output buffers changed: " + outputBuffers.leng
th); |
| 495 if (hasDecodedFirstFrame) { |
| 496 throw new RuntimeException("Unexpected output buffer change event.")
; |
| 497 } |
408 break; | 498 break; |
409 case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: | 499 case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: |
410 MediaFormat format = mediaCodec.getOutputFormat(); | 500 MediaFormat format = mediaCodec.getOutputFormat(); |
411 Logging.d(TAG, "Decoder format changed: " + format.toString()); | 501 Logging.d(TAG, "Decoder format changed: " + format.toString()); |
| 502 int new_width = format.getInteger(MediaFormat.KEY_WIDTH); |
| 503 int new_height = format.getInteger(MediaFormat.KEY_HEIGHT); |
| 504 if (hasDecodedFirstFrame && (new_width != width || new_height != heigh
t)) { |
| 505 throw new RuntimeException("Unexpected size change. Configured " + w
idth + "*" + |
| 506 height + ". New " + new_width + "*" + new_height); |
| 507 } |
412 width = format.getInteger(MediaFormat.KEY_WIDTH); | 508 width = format.getInteger(MediaFormat.KEY_WIDTH); |
413 height = format.getInteger(MediaFormat.KEY_HEIGHT); | 509 height = format.getInteger(MediaFormat.KEY_HEIGHT); |
| 510 |
414 if (!useSurface && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) { | 511 if (!useSurface && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) { |
415 colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT); | 512 colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT); |
416 Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat)); | 513 Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat)); |
417 if (!supportedColorList.contains(colorFormat)) { | 514 if (!supportedColorList.contains(colorFormat)) { |
418 throw new IllegalStateException("Non supported color format: " + c
olorFormat); | 515 throw new IllegalStateException("Non supported color format: " + c
olorFormat); |
419 } | 516 } |
420 } | 517 } |
421 if (format.containsKey("stride")) { | 518 if (format.containsKey("stride")) { |
422 stride = format.getInteger("stride"); | 519 stride = format.getInteger("stride"); |
423 } | 520 } |
424 if (format.containsKey("slice-height")) { | 521 if (format.containsKey("slice-height")) { |
425 sliceHeight = format.getInteger("slice-height"); | 522 sliceHeight = format.getInteger("slice-height"); |
426 } | 523 } |
427 Logging.d(TAG, "Frame stride and slice height: " + stride + " x " + sl
iceHeight); | 524 Logging.d(TAG, "Frame stride and slice height: " + stride + " x " + sl
iceHeight); |
428 stride = Math.max(width, stride); | 525 stride = Math.max(width, stride); |
429 sliceHeight = Math.max(height, sliceHeight); | 526 sliceHeight = Math.max(height, sliceHeight); |
430 break; | 527 break; |
| 528 case MediaCodec.INFO_TRY_AGAIN_LATER: |
| 529 return null; |
431 default: | 530 default: |
432 // Output buffer decoded. | 531 hasDecodedFirstFrame = true; |
433 if (useSurface) { | 532 return new DecodedOutputBuffer(result, info.offset, info.size, info.pr
esentationTimeUs, |
434 mediaCodec.releaseOutputBuffer(result, true /* render */); | 533 SystemClock.elapsedRealtime() - decodeStartTimeMs.remove(), |
435 // TODO(magjed): Wait for SurfaceTexture.onFrameAvailable() before r
eturning a texture | 534 SystemClock.elapsedRealtime()); |
436 // frame. | 535 } |
437 return new DecodedTextureBuffer(textureID, info.presentationTimeUs); | |
438 } else { | |
439 return new DecodedByteBuffer(result, info.offset, info.size, info.pr
esentationTimeUs); | |
440 } | |
441 } | |
442 } | 536 } |
443 } | 537 } |
444 | 538 |
| 539 // Returns null if no decoded buffer is available, and otherwise a DecodedText
ureBuffer. |
| 540 // Throws IllegalStateException if call is made on the wrong thread, if color
format changes to an |
| 541 // unsupported format, or if |mediaCodec| is not in the Executing state. Throw
s CodecException |
| 542 // upon codec error. |
| 543 private DecodedTextureBuffer dequeueTextureBuffer(int dequeueTimeoutMs) { |
| 544 checkOnMediaCodecThread(); |
| 545 if (!useSurface) { |
| 546 throw new IllegalStateException("dequeueTexture() called for byte buffer d
ecoding."); |
| 547 } |
| 548 |
| 549 DecodedOutputBuffer outputBuffer = dequeueOutputBuffer(dequeueTimeoutMs); |
| 550 if (outputBuffer != null) { |
| 551 if (dequeuedSurfaceOutputBuffers.size() >= Math.min( |
| 552 MAX_QUEUED_OUTPUTBUFFERS, outputBuffers.length)) { |
| 553 ++droppedFrames; |
| 554 Logging.w(TAG, "Too many output buffers. Dropping frame. Total number of
dropped frames: " |
| 555 + droppedFrames); |
| 556 // Drop the newest frame. Don't drop the oldest since if |isWaitingForTe
xture| |
| 557 // releaseOutputBuffer has already been called. Dropping the newest fram
e will lead to a |
| 558 // shift of timestamps by one frame in MediaCodecVideoDecoder::DeliverP
endingOutputs. |
| 559 mediaCodec.releaseOutputBuffer(outputBuffer.index, false /* render */); |
| 560 return new DecodedTextureBuffer(0, null, outputBuffer.presentationTimest
ampUs, |
| 561 outputBuffer.decodeTimeMs, |
| 562 SystemClock.elapsedRealtime() - outputBuffer.endDecodeTimeMs); |
| 563 } |
| 564 dequeuedSurfaceOutputBuffers.add(outputBuffer); |
| 565 } |
| 566 |
| 567 if (dequeuedSurfaceOutputBuffers.isEmpty()) { |
| 568 return null; |
| 569 } |
| 570 |
| 571 if (!isWaitingForTexture) { |
| 572 // Get the first frame in the queue and render to the decoder output surfa
ce. |
| 573 mediaCodec.releaseOutputBuffer(dequeuedSurfaceOutputBuffers.peek().index,
true /* render */); |
| 574 isWaitingForTexture = true; |
| 575 } |
| 576 |
| 577 // We are waiting for a frame to be rendered to the decoder surface. |
| 578 // Check if it is ready now by waiting max |dequeueTimeoutMs|. There can onl
y be one frame |
| 579 // rendered at a time. |
| 580 TextureListener.TextureInfo info = textureListener.dequeueTextureInfo(dequeu
eTimeoutMs); |
| 581 if (info != null) { |
| 582 isWaitingForTexture = false; |
| 583 final DecodedOutputBuffer renderedBuffer = |
| 584 dequeuedSurfaceOutputBuffers.remove(); |
| 585 if (!dequeuedSurfaceOutputBuffers.isEmpty()) { |
| 586 // Get the next frame in the queue and render to the decoder output surf
ace. |
| 587 mediaCodec.releaseOutputBuffer( |
| 588 dequeuedSurfaceOutputBuffers.peek().index, true /* render */); |
| 589 isWaitingForTexture = true; |
| 590 } |
| 591 |
| 592 return new DecodedTextureBuffer(info.textureID, info.transformMatrix, |
| 593 renderedBuffer.presentationTimestampUs, renderedBuffer.decodeTimeMs, |
| 594 SystemClock.elapsedRealtime() - renderedBuffer.endDecodeTimeMs); |
| 595 } |
| 596 return null; |
| 597 } |
| 598 |
445 // Release a dequeued output byte buffer back to the codec for re-use. Should
only be called for | 599 // Release a dequeued output byte buffer back to the codec for re-use. Should
only be called for |
446 // non-surface decoding. | 600 // non-surface decoding. |
447 // Throws IllegalStateException if the call is made on the wrong thread, if co
dec is configured | 601 // Throws IllegalStateException if the call is made on the wrong thread, if co
dec is configured |
448 // for surface decoding, or if |mediaCodec| is not in the Executing state. Thr
ows | 602 // for surface decoding, or if |mediaCodec| is not in the Executing state. Thr
ows |
449 // MediaCodec.CodecException upon codec error. | 603 // MediaCodec.CodecException upon codec error. |
450 private void returnDecodedByteBuffer(int index) | 604 private void returnDecodedOutputBuffer(int index) |
451 throws IllegalStateException, MediaCodec.CodecException { | 605 throws IllegalStateException, MediaCodec.CodecException { |
452 checkOnMediaCodecThread(); | 606 checkOnMediaCodecThread(); |
453 if (useSurface) { | 607 if (useSurface) { |
454 throw new IllegalStateException("returnDecodedByteBuffer() called for surf
ace decoding."); | 608 throw new IllegalStateException("returnDecodedOutputBuffer() called for su
rface decoding."); |
455 } | 609 } |
456 mediaCodec.releaseOutputBuffer(index, false /* render */); | 610 mediaCodec.releaseOutputBuffer(index, false /* render */); |
457 } | 611 } |
458 } | 612 } |
OLD | NEW |