OLD | NEW |
---|---|
1 /* | 1 /* |
2 * libjingle | 2 * libjingle |
3 * Copyright 2014 Google Inc. | 3 * Copyright 2014 Google Inc. |
4 * | 4 * |
5 * Redistribution and use in source and binary forms, with or without | 5 * Redistribution and use in source and binary forms, with or without |
6 * modification, are permitted provided that the following conditions are met: | 6 * modification, are permitted provided that the following conditions are met: |
7 * | 7 * |
8 * 1. Redistributions of source code must retain the above copyright notice, | 8 * 1. Redistributions of source code must retain the above copyright notice, |
9 * this list of conditions and the following disclaimer. | 9 * this list of conditions and the following disclaimer. |
10 * 2. Redistributions in binary form must reproduce the above copyright notice, | 10 * 2. Redistributions in binary form must reproduce the above copyright notice, |
11 * this list of conditions and the following disclaimer in the documentation | 11 * this list of conditions and the following disclaimer in the documentation |
12 * and/or other materials provided with the distribution. | 12 * and/or other materials provided with the distribution. |
13 * 3. The name of the author may not be used to endorse or promote products | 13 * 3. The name of the author may not be used to endorse or promote products |
14 * derived from this software without specific prior written permission. | 14 * derived from this software without specific prior written permission. |
15 * | 15 * |
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED | 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED |
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF | 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF |
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO | 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO |
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, | 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; | 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; |
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, |
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR | 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR |
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF | 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
26 */ | 26 */ |
27 | 27 |
28 package org.webrtc; | 28 package org.webrtc; |
29 | 29 |
30 import android.graphics.SurfaceTexture; | |
31 import android.media.MediaCodec; | 30 import android.media.MediaCodec; |
32 import android.media.MediaCodecInfo; | 31 import android.media.MediaCodecInfo; |
33 import android.media.MediaCodecInfo.CodecCapabilities; | 32 import android.media.MediaCodecInfo.CodecCapabilities; |
34 import android.media.MediaCodecList; | 33 import android.media.MediaCodecList; |
35 import android.media.MediaFormat; | 34 import android.media.MediaFormat; |
36 import android.opengl.GLES11Ext; | |
37 import android.opengl.GLES20; | |
38 import android.os.Build; | 35 import android.os.Build; |
36 import android.os.SystemClock; | |
39 import android.view.Surface; | 37 import android.view.Surface; |
40 | 38 |
41 import org.webrtc.Logging; | 39 import org.webrtc.Logging; |
42 | 40 |
43 import java.nio.ByteBuffer; | 41 import java.nio.ByteBuffer; |
44 import java.util.Arrays; | 42 import java.util.Arrays; |
43 import java.util.LinkedList; | |
45 import java.util.List; | 44 import java.util.List; |
46 import java.util.concurrent.CountDownLatch; | 45 import java.util.concurrent.CountDownLatch; |
47 | 46 import java.util.Queue; |
48 import javax.microedition.khronos.egl.EGLContext; | 47 import java.util.concurrent.TimeUnit; |
49 | 48 |
50 // Java-side of peerconnection_jni.cc:MediaCodecVideoDecoder. | 49 // Java-side of peerconnection_jni.cc:MediaCodecVideoDecoder. |
51 // This class is an implementation detail of the Java PeerConnection API. | 50 // This class is an implementation detail of the Java PeerConnection API. |
52 public class MediaCodecVideoDecoder { | 51 public class MediaCodecVideoDecoder { |
53 // This class is constructed, operated, and destroyed by its C++ incarnation, | 52 // This class is constructed, operated, and destroyed by its C++ incarnation, |
54 // so the class and its methods have non-public visibility. The API this | 53 // so the class and its methods have non-public visibility. The API this |
55 // class exposes aims to mimic the webrtc::VideoDecoder API as closely as | 54 // class exposes aims to mimic the webrtc::VideoDecoder API as closely as |
56 // possibly to minimize the amount of translation work necessary. | 55 // possibly to minimize the amount of translation work necessary. |
57 | 56 |
58 private static final String TAG = "MediaCodecVideoDecoder"; | 57 private static final String TAG = "MediaCodecVideoDecoder"; |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
92 private static final List<Integer> supportedColorList = Arrays.asList( | 91 private static final List<Integer> supportedColorList = Arrays.asList( |
93 CodecCapabilities.COLOR_FormatYUV420Planar, | 92 CodecCapabilities.COLOR_FormatYUV420Planar, |
94 CodecCapabilities.COLOR_FormatYUV420SemiPlanar, | 93 CodecCapabilities.COLOR_FormatYUV420SemiPlanar, |
95 CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, | 94 CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, |
96 COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m); | 95 COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m); |
97 private int colorFormat; | 96 private int colorFormat; |
98 private int width; | 97 private int width; |
99 private int height; | 98 private int height; |
100 private int stride; | 99 private int stride; |
101 private int sliceHeight; | 100 private int sliceHeight; |
101 private boolean hasDecodedFirstFrame; | |
102 private final Queue<Long> decodeStartTimeMs = new LinkedList<Long>(); | |
102 private boolean useSurface; | 103 private boolean useSurface; |
103 private int textureID = 0; | 104 |
104 private SurfaceTexture surfaceTexture = null; | 105 // The below variables are only used when decoding to a Surface. |
106 private TextureListener textureListener; | |
107 // |isWaitingForTexture| is true when waiting for the transition: | |
108 // MediaCodec.releaseOutputBuffer() -> onTextureFrameAvailable(). | |
109 private boolean isWaitingForTexture; | |
105 private Surface surface = null; | 110 private Surface surface = null; |
106 private EglBase eglBase; | 111 private final Queue<DecodedOutputBuffer> |
107 | 112 dequeuedSurfaceOutputBuffers = new LinkedList<DecodedOutputBuffer>(); |
108 private MediaCodecVideoDecoder() { | |
109 } | |
110 | 113 |
111 // MediaCodec error handler - invoked when critical error happens which may pr event | 114 // MediaCodec error handler - invoked when critical error happens which may pr event |
112 // further use of media codec API. Now it means that one of media codec instan ces | 115 // further use of media codec API. Now it means that one of media codec instan ces |
113 // is hanging and can no longer be used in the next call. | 116 // is hanging and can no longer be used in the next call. |
114 public static interface MediaCodecVideoDecoderErrorCallback { | 117 public static interface MediaCodecVideoDecoderErrorCallback { |
115 void onMediaCodecVideoDecoderCriticalError(int codecErrors); | 118 void onMediaCodecVideoDecoderCriticalError(int codecErrors); |
116 } | 119 } |
117 | 120 |
118 public static void setErrorCallback(MediaCodecVideoDecoderErrorCallback errorC allback) { | 121 public static void setErrorCallback(MediaCodecVideoDecoderErrorCallback errorC allback) { |
119 Logging.d(TAG, "Set error callback"); | 122 Logging.d(TAG, "Set error callback"); |
(...skipping 85 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
205 } | 208 } |
206 | 209 |
207 private void checkOnMediaCodecThread() throws IllegalStateException { | 210 private void checkOnMediaCodecThread() throws IllegalStateException { |
208 if (mediaCodecThread.getId() != Thread.currentThread().getId()) { | 211 if (mediaCodecThread.getId() != Thread.currentThread().getId()) { |
209 throw new IllegalStateException( | 212 throw new IllegalStateException( |
210 "MediaCodecVideoDecoder previously operated on " + mediaCodecThread + | 213 "MediaCodecVideoDecoder previously operated on " + mediaCodecThread + |
211 " but is now called on " + Thread.currentThread()); | 214 " but is now called on " + Thread.currentThread()); |
212 } | 215 } |
213 } | 216 } |
214 | 217 |
215 // Pass null in |sharedContext| to configure the codec for ByteBuffer output. | 218 // Pass null in |surfaceTextureHelper| to configure the codec for ByteBuffer o utput. |
216 private boolean initDecode(VideoCodecType type, int width, int height, EGLCont ext sharedContext) { | 219 private boolean initDecode( |
220 VideoCodecType type, int width, int height, SurfaceTextureHelper surfaceTe xtureHelper) { | |
217 if (mediaCodecThread != null) { | 221 if (mediaCodecThread != null) { |
218 throw new RuntimeException("Forgot to release()?"); | 222 throw new RuntimeException("Forgot to release()?"); |
219 } | 223 } |
220 useSurface = (sharedContext != null); | 224 useSurface = (surfaceTextureHelper != null); |
221 String mime = null; | 225 String mime = null; |
222 String[] supportedCodecPrefixes = null; | 226 String[] supportedCodecPrefixes = null; |
223 if (type == VideoCodecType.VIDEO_CODEC_VP8) { | 227 if (type == VideoCodecType.VIDEO_CODEC_VP8) { |
224 mime = VP8_MIME_TYPE; | 228 mime = VP8_MIME_TYPE; |
225 supportedCodecPrefixes = supportedVp8HwCodecPrefixes; | 229 supportedCodecPrefixes = supportedVp8HwCodecPrefixes; |
226 } else if (type == VideoCodecType.VIDEO_CODEC_H264) { | 230 } else if (type == VideoCodecType.VIDEO_CODEC_H264) { |
227 mime = H264_MIME_TYPE; | 231 mime = H264_MIME_TYPE; |
228 supportedCodecPrefixes = supportedH264HwCodecPrefixes; | 232 supportedCodecPrefixes = supportedH264HwCodecPrefixes; |
229 } else { | 233 } else { |
230 throw new RuntimeException("Non supported codec " + type); | 234 throw new RuntimeException("Non supported codec " + type); |
231 } | 235 } |
232 DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes); | 236 DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes); |
233 if (properties == null) { | 237 if (properties == null) { |
234 throw new RuntimeException("Cannot find HW decoder for " + type); | 238 throw new RuntimeException("Cannot find HW decoder for " + type); |
235 } | 239 } |
236 Logging.d(TAG, "Java initDecode: " + type + " : "+ width + " x " + height + | 240 Logging.d(TAG, "Java initDecode: " + type + " : "+ width + " x " + height + |
237 ". Color: 0x" + Integer.toHexString(properties.colorFormat) + | 241 ". Color: 0x" + Integer.toHexString(properties.colorFormat) + |
238 ". Use Surface: " + useSurface); | 242 ". Use Surface: " + useSurface); |
239 if (sharedContext != null) { | |
240 Logging.d(TAG, "Decoder shared EGL Context: " + sharedContext); | |
241 } | |
242 runningInstance = this; // Decoder is now running and can be queried for sta ck traces. | 243 runningInstance = this; // Decoder is now running and can be queried for sta ck traces. |
243 mediaCodecThread = Thread.currentThread(); | 244 mediaCodecThread = Thread.currentThread(); |
244 try { | 245 try { |
245 this.width = width; | 246 this.width = width; |
246 this.height = height; | 247 this.height = height; |
247 stride = width; | 248 stride = width; |
248 sliceHeight = height; | 249 sliceHeight = height; |
249 | 250 |
250 if (useSurface) { | 251 if (useSurface) { |
251 // Create shared EGL context. | 252 textureListener = new TextureListener(surfaceTextureHelper); |
252 eglBase = new EglBase(sharedContext, EglBase.ConfigType.PIXEL_BUFFER); | 253 surface = new Surface(surfaceTextureHelper.getSurfaceTexture()); |
253 eglBase.createDummyPbufferSurface(); | |
254 eglBase.makeCurrent(); | |
255 | |
256 // Create output surface | |
257 textureID = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); | |
258 Logging.d(TAG, "Video decoder TextureID = " + textureID); | |
259 surfaceTexture = new SurfaceTexture(textureID); | |
260 surface = new Surface(surfaceTexture); | |
261 } | 254 } |
262 | 255 |
263 MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); | 256 MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); |
264 if (!useSurface) { | 257 if (!useSurface) { |
265 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat); | 258 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat); |
266 } | 259 } |
267 Logging.d(TAG, " Format: " + format); | 260 Logging.d(TAG, " Format: " + format); |
268 mediaCodec = | 261 mediaCodec = |
269 MediaCodecVideoEncoder.createByCodecName(properties.codecName); | 262 MediaCodecVideoEncoder.createByCodecName(properties.codecName); |
270 if (mediaCodec == null) { | 263 if (mediaCodec == null) { |
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
313 Logging.e(TAG, "Media decoder release timeout"); | 306 Logging.e(TAG, "Media decoder release timeout"); |
314 codecErrors++; | 307 codecErrors++; |
315 if (errorCallback != null) { | 308 if (errorCallback != null) { |
316 Logging.e(TAG, "Invoke codec error callback. Errors: " + codecErrors); | 309 Logging.e(TAG, "Invoke codec error callback. Errors: " + codecErrors); |
317 errorCallback.onMediaCodecVideoDecoderCriticalError(codecErrors); | 310 errorCallback.onMediaCodecVideoDecoderCriticalError(codecErrors); |
318 } | 311 } |
319 } | 312 } |
320 | 313 |
321 mediaCodec = null; | 314 mediaCodec = null; |
322 mediaCodecThread = null; | 315 mediaCodecThread = null; |
316 decodeStartTimeMs.clear(); | |
AlexG
2015/11/10 01:21:39
nit: it looks a little more clear for me to call c
perkj_webrtc
2015/11/10 13:58:15
Done.
| |
317 hasDecodedFirstFrame = false; | |
323 runningInstance = null; | 318 runningInstance = null; |
319 dequeuedSurfaceOutputBuffers.clear(); | |
320 isWaitingForTexture = false; | |
324 if (useSurface) { | 321 if (useSurface) { |
325 surface.release(); | 322 surface.release(); |
326 surface = null; | 323 surface = null; |
327 Logging.d(TAG, "Delete video decoder TextureID " + textureID); | 324 textureListener.release(); |
328 GLES20.glDeleteTextures(1, new int[] {textureID}, 0); | |
329 textureID = 0; | |
330 eglBase.release(); | |
331 eglBase = null; | |
332 } | 325 } |
333 Logging.d(TAG, "Java releaseDecoder done"); | 326 Logging.d(TAG, "Java releaseDecoder done"); |
334 } | 327 } |
335 | 328 |
336 // Dequeue an input buffer and return its index, -1 if no input buffer is | 329 // Dequeue an input buffer and return its index, -1 if no input buffer is |
337 // available, or -2 if the codec is no longer operative. | 330 // available, or -2 if the codec is no longer operative. |
338 private int dequeueInputBuffer() { | 331 private int dequeueInputBuffer() { |
339 checkOnMediaCodecThread(); | 332 checkOnMediaCodecThread(); |
340 try { | 333 try { |
341 return mediaCodec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT); | 334 return mediaCodec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT); |
342 } catch (IllegalStateException e) { | 335 } catch (IllegalStateException e) { |
343 Logging.e(TAG, "dequeueIntputBuffer failed", e); | 336 Logging.e(TAG, "dequeueIntputBuffer failed", e); |
344 return -2; | 337 return -2; |
345 } | 338 } |
346 } | 339 } |
347 | 340 |
348 private boolean queueInputBuffer( | 341 private boolean queueInputBuffer( |
349 int inputBufferIndex, int size, long timestampUs) { | 342 int inputBufferIndex, int size, long timestampUs) { |
350 checkOnMediaCodecThread(); | 343 checkOnMediaCodecThread(); |
351 try { | 344 try { |
352 inputBuffers[inputBufferIndex].position(0); | 345 inputBuffers[inputBufferIndex].position(0); |
353 inputBuffers[inputBufferIndex].limit(size); | 346 inputBuffers[inputBufferIndex].limit(size); |
347 decodeStartTimeMs.add(SystemClock.elapsedRealtime()); | |
354 mediaCodec.queueInputBuffer(inputBufferIndex, 0, size, timestampUs, 0); | 348 mediaCodec.queueInputBuffer(inputBufferIndex, 0, size, timestampUs, 0); |
355 return true; | 349 return true; |
356 } | 350 } |
357 catch (IllegalStateException e) { | 351 catch (IllegalStateException e) { |
358 Logging.e(TAG, "decode failed", e); | 352 Logging.e(TAG, "decode failed", e); |
359 return false; | 353 return false; |
360 } | 354 } |
361 } | 355 } |
362 | 356 |
363 // Helper structs for dequeueOutputBuffer() below. | 357 // Helper struct for dequeueOutputBuffer() below. |
364 private static class DecodedByteBuffer { | 358 private static class DecodedOutputBuffer { |
365 public DecodedByteBuffer(int index, int offset, int size, long presentationT imestampUs) { | 359 public DecodedOutputBuffer(int index, int offset, int size, long presentatio nTimestampUs, |
360 long decodeTime) { | |
366 this.index = index; | 361 this.index = index; |
367 this.offset = offset; | 362 this.offset = offset; |
368 this.size = size; | 363 this.size = size; |
369 this.presentationTimestampUs = presentationTimestampUs; | 364 this.presentationTimestampUs = presentationTimestampUs; |
365 this.decodeTimeMs = decodeTime; | |
370 } | 366 } |
371 | 367 |
372 private final int index; | 368 private final int index; |
373 private final int offset; | 369 private final int offset; |
374 private final int size; | 370 private final int size; |
375 private final long presentationTimestampUs; | 371 private final long presentationTimestampUs; |
372 private final long decodeTimeMs; | |
376 } | 373 } |
377 | 374 |
375 // Helper struct for dequeueTextureBuffer() below. | |
378 private static class DecodedTextureBuffer { | 376 private static class DecodedTextureBuffer { |
379 private final int textureID; | 377 private final int textureID; |
378 private final float[] transformMatrix; | |
380 private final long presentationTimestampUs; | 379 private final long presentationTimestampUs; |
380 private final long decodeTimeMs; | |
381 | 381 |
382 public DecodedTextureBuffer(int textureID, long presentationTimestampUs) { | 382 // A DecodedTextureBuffer with zero |textureID| has special meaning and repr esents a frame |
383 // that was dropped. | |
384 public DecodedTextureBuffer(int textureID, float[] transformMatrix, | |
385 long presentationTimestampUs, long decodeTimeMs) { | |
383 this.textureID = textureID; | 386 this.textureID = textureID; |
387 this.transformMatrix = transformMatrix; | |
384 this.presentationTimestampUs = presentationTimestampUs; | 388 this.presentationTimestampUs = presentationTimestampUs; |
389 this.decodeTimeMs = decodeTimeMs; | |
385 } | 390 } |
386 } | 391 } |
387 | 392 |
388 // Returns null if no decoded buffer is available, and otherwise either a Deco dedByteBuffer or | 393 // Poll based texture listener. |
389 // DecodedTexturebuffer depending on |useSurface| configuration. | 394 private static class TextureListener |
395 implements SurfaceTextureHelper.OnTextureFrameAvailableListener { | |
396 public static class TextureInfo { | |
397 private final int textureID; | |
398 private final float[] transformMatrix; | |
399 | |
400 TextureInfo(int textureId, float[] transformMatrix) { | |
401 this.textureID = textureId; | |
402 this.transformMatrix = transformMatrix; | |
403 } | |
404 } | |
405 private final SurfaceTextureHelper surfaceTextureHelper; | |
406 private TextureInfo textureInfo; | |
407 // |newFrameLock| is used to synchronize arrival of new frames with wait()/n otifyAll(). | |
408 private final Object newFrameLock = new Object(); | |
409 | |
410 public TextureListener(SurfaceTextureHelper surfaceTextureHelper) { | |
411 this.surfaceTextureHelper = surfaceTextureHelper; | |
412 surfaceTextureHelper.setListener(this); | |
413 } | |
414 | |
415 // Callback from |surfaceTextureHelper|. May be called on an arbitrary threa d. | |
416 @Override | |
417 public void onTextureFrameAvailable( | |
418 int oesTextureId, float[] transformMatrix, long timestampNs) { | |
419 synchronized (newFrameLock) { | |
420 if (textureInfo != null) { | |
421 Logging.e(TAG, | |
422 "Unexpected onTextureFrameAvailable() called while already holding a texture."); | |
423 throw new IllegalStateException("Already holding a texture."); | |
424 } | |
425 // |timestampNs| is always zero on some Android versions. | |
426 textureInfo = new TextureInfo(oesTextureId, transformMatrix); | |
427 newFrameLock.notifyAll(); | |
428 } | |
429 } | |
430 | |
431 // Dequeues and returns a TextureInfo if available, or null otherwise. | |
432 public TextureInfo dequeueTextureInfo(int timeoutMs) { | |
433 synchronized (newFrameLock) { | |
434 if (textureInfo == null && timeoutMs > 0) { | |
435 try { | |
436 newFrameLock.wait(timeoutMs); | |
AlexG
2015/11/10 01:21:39
This may be not a safe wait() as I was recently le
perkj_webrtc
2015/11/10 13:58:15
Discussed with magnus. Spurious wake up is not dan
| |
437 } catch(InterruptedException e) { | |
438 // Restore the interrupted status by reinterrupting the thread. | |
439 Thread.currentThread().interrupt(); | |
440 } | |
441 } | |
442 TextureInfo returnedInfo = textureInfo; | |
443 textureInfo = null; | |
444 return returnedInfo; | |
445 } | |
446 } | |
447 | |
448 public void release() { | |
449 // SurfaceTextureHelper.disconnect() will block until any onTextureFrameAv ailable() in | |
450 // progress is done. Therefore, the call to disconnect() must be outside a ny synchronized | |
451 // statement that is also used in the onTextureFrameAvailable() above to a void deadlocks. | |
452 surfaceTextureHelper.disconnect(); | |
453 synchronized (newFrameLock) { | |
454 if (textureInfo != null) { | |
455 surfaceTextureHelper.returnTextureFrame(); | |
456 textureInfo = null; | |
457 } | |
458 } | |
459 } | |
460 } | |
461 | |
462 // Returns null if no decoded buffer is available, and otherwise a DecodedByte Buffer. | |
390 // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an | 463 // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an |
391 // unsupported format, or if |mediaCodec| is not in the Executing state. Throw s CodecException | 464 // unsupported format, or if |mediaCodec| is not in the Executing state. Throw s CodecException |
392 // upon codec error. | 465 // upon codec error. |
393 private Object dequeueOutputBuffer(int dequeueTimeoutUs) | 466 private DecodedOutputBuffer dequeueOutputBuffer(int dequeueTimeoutMs) { |
394 throws IllegalStateException, MediaCodec.CodecException { | |
395 checkOnMediaCodecThread(); | 467 checkOnMediaCodecThread(); |
396 | 468 if (decodeStartTimeMs.isEmpty()) { |
469 return null; | |
470 } | |
397 // Drain the decoder until receiving a decoded buffer or hitting | 471 // Drain the decoder until receiving a decoded buffer or hitting |
398 // MediaCodec.INFO_TRY_AGAIN_LATER. | 472 // MediaCodec.INFO_TRY_AGAIN_LATER. |
399 final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); | 473 final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); |
400 while (true) { | 474 while (true) { |
401 final int result = mediaCodec.dequeueOutputBuffer(info, dequeueTimeoutUs); | 475 final int result = mediaCodec.dequeueOutputBuffer( |
476 info, TimeUnit.MILLISECONDS.toMicros(dequeueTimeoutMs)); | |
402 switch (result) { | 477 switch (result) { |
403 case MediaCodec.INFO_TRY_AGAIN_LATER: | |
404 return null; | |
405 case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: | 478 case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: |
406 outputBuffers = mediaCodec.getOutputBuffers(); | 479 outputBuffers = mediaCodec.getOutputBuffers(); |
407 Logging.d(TAG, "Decoder output buffers changed: " + outputBuffers.leng th); | 480 Logging.d(TAG, "Decoder output buffers changed: " + outputBuffers.leng th); |
481 if (hasDecodedFirstFrame) { | |
482 throw new RuntimeException("Unexpected output buffer change event.") ; | |
483 } | |
408 break; | 484 break; |
409 case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: | 485 case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: |
410 MediaFormat format = mediaCodec.getOutputFormat(); | 486 MediaFormat format = mediaCodec.getOutputFormat(); |
411 Logging.d(TAG, "Decoder format changed: " + format.toString()); | 487 Logging.d(TAG, "Decoder format changed: " + format.toString()); |
488 int new_width = format.getInteger(MediaFormat.KEY_WIDTH); | |
489 int new_height = format.getInteger(MediaFormat.KEY_HEIGHT); | |
490 if (hasDecodedFirstFrame && (new_width != width || new_height != heigh t)) { | |
491 throw new RuntimeException("Unexpected size change. Configured " + w idth + "*" + | |
492 height + ". New " + new_width + "*" + new_height); | |
493 } | |
412 width = format.getInteger(MediaFormat.KEY_WIDTH); | 494 width = format.getInteger(MediaFormat.KEY_WIDTH); |
413 height = format.getInteger(MediaFormat.KEY_HEIGHT); | 495 height = format.getInteger(MediaFormat.KEY_HEIGHT); |
496 | |
414 if (!useSurface && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) { | 497 if (!useSurface && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) { |
415 colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT); | 498 colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT); |
416 Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat)); | 499 Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat)); |
417 if (!supportedColorList.contains(colorFormat)) { | 500 if (!supportedColorList.contains(colorFormat)) { |
418 throw new IllegalStateException("Non supported color format: " + c olorFormat); | 501 throw new IllegalStateException("Non supported color format: " + c olorFormat); |
419 } | 502 } |
420 } | 503 } |
421 if (format.containsKey("stride")) { | 504 if (format.containsKey("stride")) { |
422 stride = format.getInteger("stride"); | 505 stride = format.getInteger("stride"); |
423 } | 506 } |
424 if (format.containsKey("slice-height")) { | 507 if (format.containsKey("slice-height")) { |
425 sliceHeight = format.getInteger("slice-height"); | 508 sliceHeight = format.getInteger("slice-height"); |
426 } | 509 } |
427 Logging.d(TAG, "Frame stride and slice height: " + stride + " x " + sl iceHeight); | 510 Logging.d(TAG, "Frame stride and slice height: " + stride + " x " + sl iceHeight); |
428 stride = Math.max(width, stride); | 511 stride = Math.max(width, stride); |
429 sliceHeight = Math.max(height, sliceHeight); | 512 sliceHeight = Math.max(height, sliceHeight); |
430 break; | 513 break; |
514 case MediaCodec.INFO_TRY_AGAIN_LATER: | |
515 return null; | |
431 default: | 516 default: |
432 // Output buffer decoded. | 517 hasDecodedFirstFrame = true; |
433 if (useSurface) { | 518 return new DecodedOutputBuffer(result, info.offset, info.size, info.pr esentationTimeUs, |
434 mediaCodec.releaseOutputBuffer(result, true /* render */); | 519 SystemClock.elapsedRealtime() - decodeStartTimeMs.remove()); |
435 // TODO(magjed): Wait for SurfaceTexture.onFrameAvailable() before r eturning a texture | 520 } |
436 // frame. | |
437 return new DecodedTextureBuffer(textureID, info.presentationTimeUs); | |
438 } else { | |
439 return new DecodedByteBuffer(result, info.offset, info.size, info.pr esentationTimeUs); | |
440 } | |
441 } | |
442 } | 521 } |
443 } | 522 } |
444 | 523 |
524 // Returns null if no decoded buffer is available, and otherwise a DecodedText ureBuffer. | |
525 // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an | |
526 // unsupported format, or if |mediaCodec| is not in the Executing state. Throw s CodecException | |
527 // upon codec error. | |
528 private DecodedTextureBuffer dequeueTextureBuffer(int dequeueTimeoutMs) { | |
529 checkOnMediaCodecThread(); | |
530 if (!useSurface) { | |
531 throw new IllegalStateException("dequeueTexture() called for byte buffer d ecoding."); | |
532 } | |
533 | |
534 DecodedOutputBuffer outputBuffer = dequeueOutputBuffer(dequeueTimeoutMs); | |
535 if (outputBuffer != null) { | |
536 if (dequeuedSurfaceOutputBuffers.size() > 2) { | |
AlexG
2015/11/10 01:21:39
Make limitation dependent on output buffers amount
perkj_webrtc
2015/11/10 13:58:15
Done.
| |
537 Logging.w(TAG, "Too many output buffers. Dropping frame."); | |
AlexG
2015/11/10 01:21:39
I think it is worth to add stat variable on how ma
perkj_webrtc
2015/11/10 13:58:15
Done.
| |
538 // Drop the newest frame. Don't drop the oldest since if |isWaitingForTe xture| | |
539 // releaseOutputBuffer has already been called. Dropping the newest fram e will lead to a | |
540 // shift of timestamps by one frame in MediaCodecVideoDecoder::DeliverP endingOutputs. | |
541 mediaCodec.releaseOutputBuffer(outputBuffer.index, false /* render */); | |
542 return new DecodedTextureBuffer(0, null, outputBuffer.presentationTimest ampUs, | |
543 outputBuffer.decodeTimeMs); | |
544 } | |
545 dequeuedSurfaceOutputBuffers.add(outputBuffer); | |
546 } | |
547 | |
548 if (dequeuedSurfaceOutputBuffers.isEmpty()) { | |
549 return null; | |
550 } | |
551 | |
AlexG
2015/11/10 01:21:39
nit: extra line
perkj_webrtc
2015/11/10 13:58:15
Done.
| |
552 | |
553 if (!isWaitingForTexture) { | |
554 // Get the first frame in the queue and render to the decoder output surfa ce. | |
555 mediaCodec.releaseOutputBuffer(dequeuedSurfaceOutputBuffers.peek().index, true /* render */); | |
556 isWaitingForTexture = true; | |
557 } | |
558 | |
559 // We are waiting for a frame to be rendered to the decoder surface. | |
560 // Check if it is ready now by waiting max |dequeueTimeoutMs|. There can onl y be one frame | |
561 // rendered at a time. | |
562 TextureListener.TextureInfo info = textureListener.dequeueTextureInfo(dequeu eTimeoutMs); | |
563 if (info != null) { | |
564 isWaitingForTexture = false; | |
565 final DecodedOutputBuffer renderedBuffer = | |
566 dequeuedSurfaceOutputBuffers.remove(); | |
AlexG
2015/11/10 01:21:39
call mediaCodec.releaseOutputBuffer(true) for the
perkj_webrtc
2015/11/10 13:58:15
:-) I did a few rounds back but Magnus wanted me t
| |
567 return new DecodedTextureBuffer(info.textureID, info.transformMatrix, | |
568 renderedBuffer.presentationTimestampUs, renderedBuffer.decodeTimeMs); | |
569 } | |
570 return null; | |
571 } | |
572 | |
445 // Release a dequeued output byte buffer back to the codec for re-use. Should only be called for | 573 // Release a dequeued output byte buffer back to the codec for re-use. Should only be called for |
446 // non-surface decoding. | 574 // non-surface decoding. |
447 // Throws IllegalStateException if the call is made on the wrong thread, if co dec is configured | 575 // Throws IllegalStateException if the call is made on the wrong thread, if co dec is configured |
448 // for surface decoding, or if |mediaCodec| is not in the Executing state. Thr ows | 576 // for surface decoding, or if |mediaCodec| is not in the Executing state. Thr ows |
449 // MediaCodec.CodecException upon codec error. | 577 // MediaCodec.CodecException upon codec error. |
450 private void returnDecodedByteBuffer(int index) | 578 private void returnDecodedOutputBuffer(int index) |
451 throws IllegalStateException, MediaCodec.CodecException { | 579 throws IllegalStateException, MediaCodec.CodecException { |
452 checkOnMediaCodecThread(); | 580 checkOnMediaCodecThread(); |
453 if (useSurface) { | 581 if (useSurface) { |
454 throw new IllegalStateException("returnDecodedByteBuffer() called for surf ace decoding."); | 582 throw new IllegalStateException("returnDecodedOutputBuffer() called for su rface decoding."); |
455 } | 583 } |
456 mediaCodec.releaseOutputBuffer(index, false /* render */); | 584 mediaCodec.releaseOutputBuffer(index, false /* render */); |
457 } | 585 } |
458 } | 586 } |
OLD | NEW |