OLD | NEW |
---|---|
1 /* | 1 /* |
2 * libjingle | 2 * libjingle |
3 * Copyright 2014 Google Inc. | 3 * Copyright 2014 Google Inc. |
4 * | 4 * |
5 * Redistribution and use in source and binary forms, with or without | 5 * Redistribution and use in source and binary forms, with or without |
6 * modification, are permitted provided that the following conditions are met: | 6 * modification, are permitted provided that the following conditions are met: |
7 * | 7 * |
8 * 1. Redistributions of source code must retain the above copyright notice, | 8 * 1. Redistributions of source code must retain the above copyright notice, |
9 * this list of conditions and the following disclaimer. | 9 * this list of conditions and the following disclaimer. |
10 * 2. Redistributions in binary form must reproduce the above copyright notice, | 10 * 2. Redistributions in binary form must reproduce the above copyright notice, |
11 * this list of conditions and the following disclaimer in the documentation | 11 * this list of conditions and the following disclaimer in the documentation |
12 * and/or other materials provided with the distribution. | 12 * and/or other materials provided with the distribution. |
13 * 3. The name of the author may not be used to endorse or promote products | 13 * 3. The name of the author may not be used to endorse or promote products |
14 * derived from this software without specific prior written permission. | 14 * derived from this software without specific prior written permission. |
15 * | 15 * |
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED | 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED |
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF | 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF |
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO | 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO |
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, | 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; | 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; |
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, |
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR | 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR |
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF | 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
26 */ | 26 */ |
27 | 27 |
28 package org.webrtc; | 28 package org.webrtc; |
29 | 29 |
30 import android.graphics.SurfaceTexture; | |
31 import android.media.MediaCodec; | 30 import android.media.MediaCodec; |
32 import android.media.MediaCodecInfo; | 31 import android.media.MediaCodecInfo; |
33 import android.media.MediaCodecInfo.CodecCapabilities; | 32 import android.media.MediaCodecInfo.CodecCapabilities; |
34 import android.media.MediaCodecList; | 33 import android.media.MediaCodecList; |
35 import android.media.MediaFormat; | 34 import android.media.MediaFormat; |
36 import android.opengl.EGLContext; | 35 import android.opengl.EGLContext; |
37 import android.opengl.GLES11Ext; | 36 import android.opengl.GLES11Ext; |
38 import android.opengl.GLES20; | 37 import android.opengl.GLES20; |
39 import android.os.Build; | 38 import android.os.Build; |
40 import android.view.Surface; | 39 import android.view.Surface; |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
86 CodecCapabilities.COLOR_FormatYUV420Planar, | 85 CodecCapabilities.COLOR_FormatYUV420Planar, |
87 CodecCapabilities.COLOR_FormatYUV420SemiPlanar, | 86 CodecCapabilities.COLOR_FormatYUV420SemiPlanar, |
88 CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, | 87 CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, |
89 COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m); | 88 COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m); |
90 private int colorFormat; | 89 private int colorFormat; |
91 private int width; | 90 private int width; |
92 private int height; | 91 private int height; |
93 private int stride; | 92 private int stride; |
94 private int sliceHeight; | 93 private int sliceHeight; |
95 private boolean useSurface; | 94 private boolean useSurface; |
96 private int textureID = 0; | 95 // |isWaitingForTexture| is true when waiting for the transition: |
97 private SurfaceTexture surfaceTexture = null; | 96 // MediaCodec.releaseOutputBuffer() -> onTextureFrameAvailable(). |
97 private boolean isWaitingForTexture = false; | |
98 private TextureListener textureListener; | |
98 private Surface surface = null; | 99 private Surface surface = null; |
99 private EglBase eglBase; | 100 |
101 // Poll based texture listener. | |
102 private static class TextureListener | |
perkj_webrtc
2015/10/05 07:28:34
move to below DecodedTextureBuffer ?
magjed_webrtc
2015/10/05 09:49:55
Done.
| |
103 implements SurfaceTextureHelper.OnTextureFrameAvailableListener { | |
104 private final SurfaceTextureHelper surfaceTextureHelper; | |
105 private DecodedTextureBuffer textureBuffer; | |
106 | |
107 public TextureListener(SurfaceTextureHelper surfaceTextureHelper) { | |
108 this.surfaceTextureHelper = surfaceTextureHelper; | |
109 surfaceTextureHelper.setListener(this); | |
110 } | |
111 | |
112 // Callback from |surfaceTextureHelper|. May be called on an arbitrary threa d. | |
113 @Override | |
114 public synchronized void onTextureFrameAvailable( | |
115 int oesTextureId, float[] transformMatrix, long timestampNs) { | |
116 if (textureBuffer != null) { | |
117 throw new IllegalStateException("Already holding a texture."); | |
118 } | |
119 textureBuffer = new DecodedTextureBuffer(oesTextureId, transformMatrix, ti mestampNs); | |
120 notifyAll(); | |
121 } | |
122 | |
123 // Dequeues and returns a texture buffer if available, or null otherwise. | |
124 public synchronized DecodedTextureBuffer dequeueTextureFrame(int timeoutUs) { | |
perkj_webrtc
2015/10/05 07:28:34
timesoutMs - why timeoutUs? Fix in jni class?
magjed_webrtc
2015/10/05 09:49:55
Done.
| |
125 final int timeoutMs = timeoutUs / 1000; | |
126 if (textureBuffer == null && timeoutMs > 0) { | |
127 try { | |
128 wait(timeoutMs); | |
129 } catch(InterruptedException e) { | |
130 Thread.currentThread().interrupt(); | |
131 } | |
132 } | |
133 final DecodedTextureBuffer textureBuffer = this.textureBuffer; | |
134 this.textureBuffer = null; | |
135 return textureBuffer; | |
136 } | |
137 | |
138 public void release() { | |
139 surfaceTextureHelper.disconnect(); | |
140 synchronized (this) { | |
141 if (textureBuffer != null) { | |
142 surfaceTextureHelper.returnTextureFrame(); | |
143 textureBuffer = null; | |
144 } | |
145 } | |
146 } | |
147 } | |
100 | 148 |
101 private MediaCodecVideoDecoder() { } | 149 private MediaCodecVideoDecoder() { } |
102 | 150 |
103 // Helper struct for findVp8Decoder() below. | 151 // Helper struct for findVp8Decoder() below. |
104 private static class DecoderProperties { | 152 private static class DecoderProperties { |
105 public DecoderProperties(String codecName, int colorFormat) { | 153 public DecoderProperties(String codecName, int colorFormat) { |
106 this.codecName = codecName; | 154 this.codecName = codecName; |
107 this.colorFormat = colorFormat; | 155 this.colorFormat = colorFormat; |
108 } | 156 } |
109 public final String codecName; // OpenMax component name for VP8 codec. | 157 public final String codecName; // OpenMax component name for VP8 codec. |
(...skipping 63 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
173 } | 221 } |
174 | 222 |
175 private void checkOnMediaCodecThread() throws IllegalStateException { | 223 private void checkOnMediaCodecThread() throws IllegalStateException { |
176 if (mediaCodecThread.getId() != Thread.currentThread().getId()) { | 224 if (mediaCodecThread.getId() != Thread.currentThread().getId()) { |
177 throw new IllegalStateException( | 225 throw new IllegalStateException( |
178 "MediaCodecVideoDecoder previously operated on " + mediaCodecThread + | 226 "MediaCodecVideoDecoder previously operated on " + mediaCodecThread + |
179 " but is now called on " + Thread.currentThread()); | 227 " but is now called on " + Thread.currentThread()); |
180 } | 228 } |
181 } | 229 } |
182 | 230 |
183 // Pass null in |sharedContext| to configure the codec for ByteBuffer output. | 231 // Pass null in |surfaceTextureHelper| to configure the codec for ByteBuffer o utput. |
184 private boolean initDecode(VideoCodecType type, int width, int height, EGLCont ext sharedContext) { | 232 private boolean initDecode( |
233 VideoCodecType type, int width, int height, SurfaceTextureHelper surfaceTe xtureHelper) { | |
185 if (mediaCodecThread != null) { | 234 if (mediaCodecThread != null) { |
186 throw new RuntimeException("Forgot to release()?"); | 235 throw new RuntimeException("Forgot to release()?"); |
187 } | 236 } |
188 useSurface = (sharedContext != null); | 237 useSurface = (surfaceTextureHelper != null); |
189 String mime = null; | 238 String mime = null; |
190 String[] supportedCodecPrefixes = null; | 239 String[] supportedCodecPrefixes = null; |
191 if (type == VideoCodecType.VIDEO_CODEC_VP8) { | 240 if (type == VideoCodecType.VIDEO_CODEC_VP8) { |
192 mime = VP8_MIME_TYPE; | 241 mime = VP8_MIME_TYPE; |
193 supportedCodecPrefixes = supportedVp8HwCodecPrefixes; | 242 supportedCodecPrefixes = supportedVp8HwCodecPrefixes; |
194 } else if (type == VideoCodecType.VIDEO_CODEC_H264) { | 243 } else if (type == VideoCodecType.VIDEO_CODEC_H264) { |
195 mime = H264_MIME_TYPE; | 244 mime = H264_MIME_TYPE; |
196 supportedCodecPrefixes = supportedH264HwCodecPrefixes; | 245 supportedCodecPrefixes = supportedH264HwCodecPrefixes; |
197 } else { | 246 } else { |
198 throw new RuntimeException("Non supported codec " + type); | 247 throw new RuntimeException("Non supported codec " + type); |
199 } | 248 } |
200 DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes); | 249 DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes); |
201 if (properties == null) { | 250 if (properties == null) { |
202 throw new RuntimeException("Cannot find HW decoder for " + type); | 251 throw new RuntimeException("Cannot find HW decoder for " + type); |
203 } | 252 } |
204 Logging.d(TAG, "Java initDecode: " + type + " : "+ width + " x " + height + | 253 Logging.d(TAG, "Java initDecode: " + type + " : "+ width + " x " + height + |
205 ". Color: 0x" + Integer.toHexString(properties.colorFormat) + | 254 ". Color: 0x" + Integer.toHexString(properties.colorFormat) + |
206 ". Use Surface: " + useSurface); | 255 ". Use Surface: " + useSurface); |
207 if (sharedContext != null) { | |
208 Logging.d(TAG, "Decoder shared EGL Context: " + sharedContext); | |
209 } | |
210 mediaCodecThread = Thread.currentThread(); | 256 mediaCodecThread = Thread.currentThread(); |
211 try { | 257 try { |
212 this.width = width; | 258 this.width = width; |
213 this.height = height; | 259 this.height = height; |
214 stride = width; | 260 stride = width; |
215 sliceHeight = height; | 261 sliceHeight = height; |
216 | 262 |
217 if (useSurface) { | 263 if (useSurface) { |
218 // Create shared EGL context. | 264 textureListener = new TextureListener(surfaceTextureHelper); |
219 eglBase = new EglBase(sharedContext, EglBase.ConfigType.PIXEL_BUFFER); | 265 surface = new Surface(surfaceTextureHelper.getSurfaceTexture()); |
220 eglBase.createDummyPbufferSurface(); | |
221 eglBase.makeCurrent(); | |
222 | |
223 // Create output surface | |
224 textureID = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); | |
225 Logging.d(TAG, "Video decoder TextureID = " + textureID); | |
226 surfaceTexture = new SurfaceTexture(textureID); | |
227 surface = new Surface(surfaceTexture); | |
228 } | 266 } |
229 | 267 |
230 MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); | 268 MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); |
231 if (!useSurface) { | 269 if (!useSurface) { |
232 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat); | 270 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat); |
233 } | 271 } |
234 Logging.d(TAG, " Format: " + format); | 272 Logging.d(TAG, " Format: " + format); |
235 mediaCodec = | 273 mediaCodec = |
236 MediaCodecVideoEncoder.createByCodecName(properties.codecName); | 274 MediaCodecVideoEncoder.createByCodecName(properties.codecName); |
237 if (mediaCodec == null) { | 275 if (mediaCodec == null) { |
(...skipping 20 matching lines...) Expand all Loading... | |
258 mediaCodec.stop(); | 296 mediaCodec.stop(); |
259 mediaCodec.release(); | 297 mediaCodec.release(); |
260 } catch (IllegalStateException e) { | 298 } catch (IllegalStateException e) { |
261 Logging.e(TAG, "release failed", e); | 299 Logging.e(TAG, "release failed", e); |
262 } | 300 } |
263 mediaCodec = null; | 301 mediaCodec = null; |
264 mediaCodecThread = null; | 302 mediaCodecThread = null; |
265 if (useSurface) { | 303 if (useSurface) { |
266 surface.release(); | 304 surface.release(); |
267 surface = null; | 305 surface = null; |
268 Logging.d(TAG, "Delete video decoder TextureID " + textureID); | 306 textureListener.release(); |
269 GLES20.glDeleteTextures(1, new int[] {textureID}, 0); | |
270 textureID = 0; | |
271 eglBase.release(); | |
272 eglBase = null; | |
273 } | 307 } |
274 } | 308 } |
275 | 309 |
276 // Dequeue an input buffer and return its index, -1 if no input buffer is | 310 // Dequeue an input buffer and return its index, -1 if no input buffer is |
277 // available, or -2 if the codec is no longer operative. | 311 // available, or -2 if the codec is no longer operative. |
278 private int dequeueInputBuffer() { | 312 private int dequeueInputBuffer() { |
279 checkOnMediaCodecThread(); | 313 checkOnMediaCodecThread(); |
280 try { | 314 try { |
281 return mediaCodec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT); | 315 return mediaCodec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT); |
282 } catch (IllegalStateException e) { | 316 } catch (IllegalStateException e) { |
(...skipping 27 matching lines...) Expand all Loading... | |
310 } | 344 } |
311 | 345 |
312 private final int index; | 346 private final int index; |
313 private final int offset; | 347 private final int offset; |
314 private final int size; | 348 private final int size; |
315 private final long presentationTimestampUs; | 349 private final long presentationTimestampUs; |
316 } | 350 } |
317 | 351 |
318 private static class DecodedTextureBuffer { | 352 private static class DecodedTextureBuffer { |
319 private final int textureID; | 353 private final int textureID; |
320 private final long presentationTimestampUs; | 354 private final float[] transformMatrix; |
355 private final long timestampNs; | |
321 | 356 |
322 public DecodedTextureBuffer(int textureID, long presentationTimestampUs) { | 357 public DecodedTextureBuffer(int textureID, float[] transformMatrix, long tim estampNs) { |
323 this.textureID = textureID; | 358 this.textureID = textureID; |
324 this.presentationTimestampUs = presentationTimestampUs; | 359 this.transformMatrix = transformMatrix; |
360 this.timestampNs = timestampNs; | |
325 } | 361 } |
326 } | 362 } |
327 | 363 |
328 // Returns null if no decoded buffer is available, and otherwise either a Deco dedByteBuffer or | 364 // Returns null if no decoded buffer is available, and otherwise either a Deco dedByteBuffer or |
329 // DecodedTexturebuffer depending on |useSurface| configuration. | 365 // DecodedTexturebuffer depending on |useSurface| configuration. |
330 // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an | 366 // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an |
331 // unsupported format, or if |mediaCodec| is not in the Executing state. Throw s CodecException | 367 // unsupported format, or if |mediaCodec| is not in the Executing state. Throw s CodecException |
332 // upon codec error. | 368 // upon codec error. |
333 private Object dequeueOutputBuffer(int dequeueTimeoutUs) | 369 private Object dequeueOutputBuffer(int dequeueTimeoutUs) |
334 throws IllegalStateException, MediaCodec.CodecException { | 370 throws IllegalStateException, MediaCodec.CodecException { |
335 checkOnMediaCodecThread(); | 371 checkOnMediaCodecThread(); |
372 // Calling multiple MediaCodec.releaseOutputBuffer() with render=true in a r ow will result in | |
373 // dropped texture frames. Therefore, wait for any pending onTextureFrameAva ilable() before | |
374 // proceeding. | |
375 if (isWaitingForTexture) { | |
376 final DecodedTextureBuffer textureBuffer = | |
377 textureListener.dequeueTextureFrame(dequeueTimeoutUs); | |
378 isWaitingForTexture = (textureBuffer == null); | |
379 return textureBuffer; | |
380 } | |
381 | |
336 // Drain the decoder until receiving a decoded buffer or hitting | 382 // Drain the decoder until receiving a decoded buffer or hitting |
337 // MediaCodec.INFO_TRY_AGAIN_LATER. | 383 // MediaCodec.INFO_TRY_AGAIN_LATER. |
338 final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); | 384 final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); |
339 while (true) { | 385 while (true) { |
340 final int result = mediaCodec.dequeueOutputBuffer(info, dequeueTimeoutUs); | 386 final int result = mediaCodec.dequeueOutputBuffer(info, dequeueTimeoutUs); |
341 switch (result) { | 387 switch (result) { |
342 case MediaCodec.INFO_TRY_AGAIN_LATER: | 388 case MediaCodec.INFO_TRY_AGAIN_LATER: |
343 return null; | 389 return null; |
344 case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: | 390 case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: |
345 outputBuffers = mediaCodec.getOutputBuffers(); | 391 outputBuffers = mediaCodec.getOutputBuffers(); |
(...skipping 18 matching lines...) Expand all Loading... | |
364 sliceHeight = format.getInteger("slice-height"); | 410 sliceHeight = format.getInteger("slice-height"); |
365 } | 411 } |
366 Logging.d(TAG, "Frame stride and slice height: " + stride + " x " + sl iceHeight); | 412 Logging.d(TAG, "Frame stride and slice height: " + stride + " x " + sl iceHeight); |
367 stride = Math.max(width, stride); | 413 stride = Math.max(width, stride); |
368 sliceHeight = Math.max(height, sliceHeight); | 414 sliceHeight = Math.max(height, sliceHeight); |
369 break; | 415 break; |
370 default: | 416 default: |
371 // Output buffer decoded. | 417 // Output buffer decoded. |
372 if (useSurface) { | 418 if (useSurface) { |
373 mediaCodec.releaseOutputBuffer(result, true /* render */); | 419 mediaCodec.releaseOutputBuffer(result, true /* render */); |
374 // TODO(magjed): Wait for SurfaceTexture.onFrameAvailable() before r eturning a texture | 420 final DecodedTextureBuffer textureBuffer = |
375 // frame. | 421 textureListener.dequeueTextureFrame(dequeueTimeoutUs); |
376 return new DecodedTextureBuffer(textureID, info.presentationTimeUs); | 422 isWaitingForTexture = (textureBuffer == null); |
423 return textureBuffer; | |
377 } else { | 424 } else { |
378 return new DecodedByteBuffer(result, info.offset, info.size, info.pr esentationTimeUs); | 425 return new DecodedByteBuffer(result, info.offset, info.size, info.pr esentationTimeUs); |
379 } | 426 } |
380 } | 427 } |
381 } | 428 } |
382 } | 429 } |
383 | 430 |
384 // Release a dequeued output byte buffer back to the codec for re-use. Should only be called for | 431 // Release a dequeued output byte buffer back to the codec for re-use. Should only be called for |
385 // non-surface decoding. | 432 // non-surface decoding. |
386 // Throws IllegalStateException if the call is made on the wrong thread, if co dec is configured | 433 // Throws IllegalStateException if the call is made on the wrong thread, if co dec is configured |
387 // for surface decoding, or if |mediaCodec| is not in the Executing state. Thr ows | 434 // for surface decoding, or if |mediaCodec| is not in the Executing state. Thr ows |
388 // MediaCodec.CodecException upon codec error. | 435 // MediaCodec.CodecException upon codec error. |
389 private void returnDecodedByteBuffer(int index) | 436 private void returnDecodedByteBuffer(int index) |
390 throws IllegalStateException, MediaCodec.CodecException { | 437 throws IllegalStateException, MediaCodec.CodecException { |
391 checkOnMediaCodecThread(); | 438 checkOnMediaCodecThread(); |
392 if (useSurface) { | 439 if (useSurface) { |
393 throw new IllegalStateException("returnDecodedByteBuffer() called for surf ace decoding."); | 440 throw new IllegalStateException("returnDecodedByteBuffer() called for surf ace decoding."); |
394 } | 441 } |
395 mediaCodec.releaseOutputBuffer(index, false /* render */); | 442 mediaCodec.releaseOutputBuffer(index, false /* render */); |
396 } | 443 } |
397 } | 444 } |
OLD | NEW |