OLD | NEW |
| (Empty) |
1 /* | |
2 * Copyright 2015 The WebRTC project authors. All Rights Reserved. | |
3 * | |
4 * Use of this source code is governed by a BSD-style license | |
5 * that can be found in the LICENSE file in the root of the source | |
6 * tree. An additional intellectual property rights grant can be found | |
7 * in the file PATENTS. All contributing project authors may | |
8 * be found in the AUTHORS file in the root of the source tree. | |
9 */ | |
10 | |
11 package org.webrtc; | |
12 | |
13 import android.graphics.SurfaceTexture; | |
14 import android.opengl.GLES11Ext; | |
15 import android.opengl.GLES20; | |
16 import android.os.Build; | |
17 import android.os.Handler; | |
18 import android.os.HandlerThread; | |
19 import android.os.SystemClock; | |
20 | |
21 import java.nio.ByteBuffer; | |
22 import java.nio.FloatBuffer; | |
23 import java.util.concurrent.Callable; | |
24 import java.util.concurrent.CountDownLatch; | |
25 import java.util.concurrent.TimeUnit; | |
26 | |
27 /** | |
28 * Helper class to create and synchronize access to a SurfaceTexture. The caller
will get notified | |
29 * of new frames in onTextureFrameAvailable(), and should call returnTextureFram
e() when done with | |
30 * the frame. Only one texture frame can be in flight at once, so returnTextureF
rame() must be | |
31 * called in order to receive a new frame. Call stopListening() to stop receivei
ng new frames. Call | |
32 * dispose to release all resources once the texture frame is returned. | |
33 * Note that there is a C++ counter part of this class that optionally can be us
ed. It is used for | |
34 * wrapping texture frames into webrtc::VideoFrames and also handles calling ret
urnTextureFrame() | |
35 * when the webrtc::VideoFrame is no longer used. | |
36 */ | |
37 class SurfaceTextureHelper { | |
38 private static final String TAG = "SurfaceTextureHelper"; | |
39 /** | |
40 * Callback interface for being notified that a new texture frame is available
. The calls will be | |
41 * made on a dedicated thread with a bound EGLContext. The thread will be the
same throughout the | |
42 * lifetime of the SurfaceTextureHelper instance, but different from the threa
d calling the | |
43 * SurfaceTextureHelper constructor. The callee is not allowed to make another
EGLContext current | |
44 * on the calling thread. | |
45 */ | |
46 public interface OnTextureFrameAvailableListener { | |
47 abstract void onTextureFrameAvailable( | |
48 int oesTextureId, float[] transformMatrix, long timestampNs); | |
49 } | |
50 | |
51 /** | |
52 * Construct a new SurfaceTextureHelper sharing OpenGL resources with |sharedC
ontext|. A dedicated | |
53 * thread and handler is created for handling the SurfaceTexture. May return n
ull if EGL fails to | |
54 * initialize a pixel buffer surface and make it current. | |
55 */ | |
56 public static SurfaceTextureHelper create( | |
57 final String threadName, final EglBase.Context sharedContext) { | |
58 final HandlerThread thread = new HandlerThread(threadName); | |
59 thread.start(); | |
60 final Handler handler = new Handler(thread.getLooper()); | |
61 | |
62 // The onFrameAvailable() callback will be executed on the SurfaceTexture ct
or thread. See: | |
63 // http://grepcode.com/file/repository.grepcode.com/java/ext/com.google.andr
oid/android/5.1.1_r1/android/graphics/SurfaceTexture.java#195. | |
64 // Therefore, in order to control the callback thread on API lvl < 21, the S
urfaceTextureHelper | |
65 // is constructed on the |handler| thread. | |
66 return ThreadUtils.invokeAtFrontUninterruptibly(handler, new Callable<Surfac
eTextureHelper>() { | |
67 @Override | |
68 public SurfaceTextureHelper call() { | |
69 try { | |
70 return new SurfaceTextureHelper(sharedContext, handler); | |
71 } catch (RuntimeException e) { | |
72 Logging.e(TAG, threadName + " create failure", e); | |
73 return null; | |
74 } | |
75 } | |
76 }); | |
77 } | |
78 | |
79 // State for YUV conversion, instantiated on demand. | |
80 static private class YuvConverter { | |
81 private final EglBase eglBase; | |
82 private final GlShader shader; | |
83 private boolean released = false; | |
84 | |
85 // Vertex coordinates in Normalized Device Coordinates, i.e. | |
86 // (-1, -1) is bottom-left and (1, 1) is top-right. | |
87 private static final FloatBuffer DEVICE_RECTANGLE = | |
88 GlUtil.createFloatBuffer(new float[] { | |
89 -1.0f, -1.0f, // Bottom left. | |
90 1.0f, -1.0f, // Bottom right. | |
91 -1.0f, 1.0f, // Top left. | |
92 1.0f, 1.0f, // Top right. | |
93 }); | |
94 | |
95 // Texture coordinates - (0, 0) is bottom-left and (1, 1) is top-right. | |
96 private static final FloatBuffer TEXTURE_RECTANGLE = | |
97 GlUtil.createFloatBuffer(new float[] { | |
98 0.0f, 0.0f, // Bottom left. | |
99 1.0f, 0.0f, // Bottom right. | |
100 0.0f, 1.0f, // Top left. | |
101 1.0f, 1.0f // Top right. | |
102 }); | |
103 | |
104 private static final String VERTEX_SHADER = | |
105 "varying vec2 interp_tc;\n" | |
106 + "attribute vec4 in_pos;\n" | |
107 + "attribute vec4 in_tc;\n" | |
108 + "\n" | |
109 + "uniform mat4 texMatrix;\n" | |
110 + "\n" | |
111 + "void main() {\n" | |
112 + " gl_Position = in_pos;\n" | |
113 + " interp_tc = (texMatrix * in_tc).xy;\n" | |
114 + "}\n"; | |
115 | |
116 private static final String FRAGMENT_SHADER = | |
117 "#extension GL_OES_EGL_image_external : require\n" | |
118 + "precision mediump float;\n" | |
119 + "varying vec2 interp_tc;\n" | |
120 + "\n" | |
121 + "uniform samplerExternalOES oesTex;\n" | |
122 // Difference in texture coordinate corresponding to one | |
123 // sub-pixel in the x direction. | |
124 + "uniform vec2 xUnit;\n" | |
125 // Color conversion coefficients, including constant term | |
126 + "uniform vec4 coeffs;\n" | |
127 + "\n" | |
128 + "void main() {\n" | |
129 // Since the alpha read from the texture is always 1, this could | |
130 // be written as a mat4 x vec4 multiply. However, that seems to | |
131 // give a worse framerate, possibly because the additional | |
132 // multiplies by 1.0 consume resources. TODO(nisse): Could also | |
133 // try to do it as a vec3 x mat3x4, followed by an add in of a | |
134 // constant vector. | |
135 + " gl_FragColor.r = coeffs.a + dot(coeffs.rgb,\n" | |
136 + " texture2D(oesTex, interp_tc - 1.5 * xUnit).rgb);\n" | |
137 + " gl_FragColor.g = coeffs.a + dot(coeffs.rgb,\n" | |
138 + " texture2D(oesTex, interp_tc - 0.5 * xUnit).rgb);\n" | |
139 + " gl_FragColor.b = coeffs.a + dot(coeffs.rgb,\n" | |
140 + " texture2D(oesTex, interp_tc + 0.5 * xUnit).rgb);\n" | |
141 + " gl_FragColor.a = coeffs.a + dot(coeffs.rgb,\n" | |
142 + " texture2D(oesTex, interp_tc + 1.5 * xUnit).rgb);\n" | |
143 + "}\n"; | |
144 | |
145 private int texMatrixLoc; | |
146 private int xUnitLoc; | |
147 private int coeffsLoc;; | |
148 | |
149 YuvConverter (EglBase.Context sharedContext) { | |
150 eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_RGBA_BUFFER); | |
151 eglBase.createDummyPbufferSurface(); | |
152 eglBase.makeCurrent(); | |
153 | |
154 shader = new GlShader(VERTEX_SHADER, FRAGMENT_SHADER); | |
155 shader.useProgram(); | |
156 texMatrixLoc = shader.getUniformLocation("texMatrix"); | |
157 xUnitLoc = shader.getUniformLocation("xUnit"); | |
158 coeffsLoc = shader.getUniformLocation("coeffs"); | |
159 GLES20.glUniform1i(shader.getUniformLocation("oesTex"), 0); | |
160 GlUtil.checkNoGLES2Error("Initialize fragment shader uniform values."); | |
161 // Initialize vertex shader attributes. | |
162 shader.setVertexAttribArray("in_pos", 2, DEVICE_RECTANGLE); | |
163 // If the width is not a multiple of 4 pixels, the texture | |
164 // will be scaled up slightly and clipped at the right border. | |
165 shader.setVertexAttribArray("in_tc", 2, TEXTURE_RECTANGLE); | |
166 eglBase.detachCurrent(); | |
167 } | |
168 | |
169 synchronized void convert(ByteBuffer buf, | |
170 int width, int height, int stride, int textureId, float [] transformMatr
ix) { | |
171 if (released) { | |
172 throw new IllegalStateException( | |
173 "YuvConverter.convert called on released object"); | |
174 } | |
175 | |
176 // We draw into a buffer laid out like | |
177 // | |
178 // +---------+ | |
179 // | | | |
180 // | Y | | |
181 // | | | |
182 // | | | |
183 // +----+----+ | |
184 // | U | V | | |
185 // | | | | |
186 // +----+----+ | |
187 // | |
188 // In memory, we use the same stride for all of Y, U and V. The | |
189 // U data starts at offset |height| * |stride| from the Y data, | |
190 // and the V data starts at at offset |stride/2| from the U | |
191 // data, with rows of U and V data alternating. | |
192 // | |
193 // Now, it would have made sense to allocate a pixel buffer with | |
194 // a single byte per pixel (EGL10.EGL_COLOR_BUFFER_TYPE, | |
195 // EGL10.EGL_LUMINANCE_BUFFER,), but that seems to be | |
196 // unsupported by devices. So do the following hack: Allocate an | |
197 // RGBA buffer, of width |stride|/4. To render each of these | |
198 // large pixels, sample the texture at 4 different x coordinates | |
199 // and store the results in the four components. | |
200 // | |
201 // Since the V data needs to start on a boundary of such a | |
202 // larger pixel, it is not sufficient that |stride| is even, it | |
203 // has to be a multiple of 8 pixels. | |
204 | |
205 if (stride % 8 != 0) { | |
206 throw new IllegalArgumentException( | |
207 "Invalid stride, must be a multiple of 8"); | |
208 } | |
209 if (stride < width){ | |
210 throw new IllegalArgumentException( | |
211 "Invalid stride, must >= width"); | |
212 } | |
213 | |
214 int y_width = (width+3) / 4; | |
215 int uv_width = (width+7) / 8; | |
216 int uv_height = (height+1)/2; | |
217 int total_height = height + uv_height; | |
218 int size = stride * total_height; | |
219 | |
220 if (buf.capacity() < size) { | |
221 throw new IllegalArgumentException("YuvConverter.convert called with too
small buffer"); | |
222 } | |
223 // Produce a frame buffer starting at top-left corner, not | |
224 // bottom-left. | |
225 transformMatrix = | |
226 RendererCommon.multiplyMatrices(transformMatrix, | |
227 RendererCommon.verticalFlipMatrix()); | |
228 | |
229 // Create new pBuffferSurface with the correct size if needed. | |
230 if (eglBase.hasSurface()) { | |
231 if (eglBase.surfaceWidth() != stride/4 || | |
232 eglBase.surfaceHeight() != total_height){ | |
233 eglBase.releaseSurface(); | |
234 eglBase.createPbufferSurface(stride/4, total_height); | |
235 } | |
236 } else { | |
237 eglBase.createPbufferSurface(stride/4, total_height); | |
238 } | |
239 | |
240 eglBase.makeCurrent(); | |
241 | |
242 GLES20.glActiveTexture(GLES20.GL_TEXTURE0); | |
243 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId); | |
244 GLES20.glUniformMatrix4fv(texMatrixLoc, 1, false, transformMatrix, 0); | |
245 | |
246 // Draw Y | |
247 GLES20.glViewport(0, 0, y_width, height); | |
248 // Matrix * (1;0;0;0) / width. Note that opengl uses column major order. | |
249 GLES20.glUniform2f(xUnitLoc, | |
250 transformMatrix[0] / width, | |
251 transformMatrix[1] / width); | |
252 // Y'UV444 to RGB888, see | |
253 // https://en.wikipedia.org/wiki/YUV#Y.27UV444_to_RGB888_conversion. | |
254 // We use the ITU-R coefficients for U and V */ | |
255 GLES20.glUniform4f(coeffsLoc, 0.299f, 0.587f, 0.114f, 0.0f); | |
256 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); | |
257 | |
258 // Draw U | |
259 GLES20.glViewport(0, height, uv_width, uv_height); | |
260 // Matrix * (1;0;0;0) / (width / 2). Note that opengl uses column major or
der. | |
261 GLES20.glUniform2f(xUnitLoc, | |
262 2.0f * transformMatrix[0] / width, | |
263 2.0f * transformMatrix[1] / width); | |
264 GLES20.glUniform4f(coeffsLoc, -0.169f, -0.331f, 0.499f, 0.5f); | |
265 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); | |
266 | |
267 // Draw V | |
268 GLES20.glViewport(stride/8, height, uv_width, uv_height); | |
269 GLES20.glUniform4f(coeffsLoc, 0.499f, -0.418f, -0.0813f, 0.5f); | |
270 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); | |
271 | |
272 GLES20.glReadPixels(0, 0, stride/4, total_height, GLES20.GL_RGBA, | |
273 GLES20.GL_UNSIGNED_BYTE, buf); | |
274 | |
275 GlUtil.checkNoGLES2Error("YuvConverter.convert"); | |
276 | |
277 // Unbind texture. Reportedly needed on some devices to get | |
278 // the texture updated from the camera. | |
279 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0); | |
280 eglBase.detachCurrent(); | |
281 } | |
282 | |
283 synchronized void release() { | |
284 released = true; | |
285 eglBase.makeCurrent(); | |
286 shader.release(); | |
287 eglBase.release(); | |
288 } | |
289 } | |
290 | |
291 private final Handler handler; | |
292 private final EglBase eglBase; | |
293 private final SurfaceTexture surfaceTexture; | |
294 private final int oesTextureId; | |
295 private YuvConverter yuvConverter; | |
296 | |
297 // These variables are only accessed from the |handler| thread. | |
298 private OnTextureFrameAvailableListener listener; | |
299 // The possible states of this class. | |
300 private boolean hasPendingTexture = false; | |
301 private volatile boolean isTextureInUse = false; | |
302 private boolean isQuitting = false; | |
303 // |pendingListener| is set in setListener() and the runnable is posted to the
handler thread. | |
304 // setListener() is not allowed to be called again before stopListening(), so
this is thread safe. | |
305 private OnTextureFrameAvailableListener pendingListener; | |
306 final Runnable setListenerRunnable = new Runnable() { | |
307 @Override | |
308 public void run() { | |
309 Logging.d(TAG, "Setting listener to " + pendingListener); | |
310 listener = pendingListener; | |
311 pendingListener = null; | |
312 // May have a pending frame from the previous capture session - drop it. | |
313 if (hasPendingTexture) { | |
314 // Calling updateTexImage() is neccessary in order to receive new frames
. | |
315 updateTexImage(); | |
316 hasPendingTexture = false; | |
317 } | |
318 } | |
319 }; | |
320 | |
321 private SurfaceTextureHelper(EglBase.Context sharedContext, Handler handler) { | |
322 if (handler.getLooper().getThread() != Thread.currentThread()) { | |
323 throw new IllegalStateException("SurfaceTextureHelper must be created on t
he handler thread"); | |
324 } | |
325 this.handler = handler; | |
326 | |
327 eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_BUFFER); | |
328 try { | |
329 // Both these statements have been observed to fail on rare occasions, see
BUG=webrtc:5682. | |
330 eglBase.createDummyPbufferSurface(); | |
331 eglBase.makeCurrent(); | |
332 } catch (RuntimeException e) { | |
333 // Clean up before rethrowing the exception. | |
334 eglBase.release(); | |
335 handler.getLooper().quit(); | |
336 throw e; | |
337 } | |
338 | |
339 oesTextureId = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); | |
340 surfaceTexture = new SurfaceTexture(oesTextureId); | |
341 surfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailab
leListener() { | |
342 @Override | |
343 public void onFrameAvailable(SurfaceTexture surfaceTexture) { | |
344 hasPendingTexture = true; | |
345 tryDeliverTextureFrame(); | |
346 } | |
347 }); | |
348 } | |
349 | |
350 private YuvConverter getYuvConverter() { | |
351 // yuvConverter is assigned once | |
352 if (yuvConverter != null) | |
353 return yuvConverter; | |
354 | |
355 synchronized(this) { | |
356 if (yuvConverter == null) | |
357 yuvConverter = new YuvConverter(eglBase.getEglBaseContext()); | |
358 return yuvConverter; | |
359 } | |
360 } | |
361 | |
362 /** | |
363 * Start to stream textures to the given |listener|. If you need to change lis
tener, you need to | |
364 * call stopListening() first. | |
365 */ | |
366 public void startListening(final OnTextureFrameAvailableListener listener) { | |
367 if (this.listener != null || this.pendingListener != null) { | |
368 throw new IllegalStateException("SurfaceTextureHelper listener has already
been set."); | |
369 } | |
370 this.pendingListener = listener; | |
371 handler.post(setListenerRunnable); | |
372 } | |
373 | |
374 /** | |
375 * Stop listening. The listener set in startListening() is guaranteded to not
receive any more | |
376 * onTextureFrameAvailable() callbacks after this function returns. | |
377 */ | |
378 public void stopListening() { | |
379 Logging.d(TAG, "stopListening()"); | |
380 handler.removeCallbacks(setListenerRunnable); | |
381 ThreadUtils.invokeAtFrontUninterruptibly(handler, new Runnable() { | |
382 @Override | |
383 public void run() { | |
384 listener = null; | |
385 pendingListener = null; | |
386 } | |
387 }); | |
388 } | |
389 | |
390 /** | |
391 * Retrieve the underlying SurfaceTexture. The SurfaceTexture should be passed
in to a video | |
392 * producer such as a camera or decoder. | |
393 */ | |
394 public SurfaceTexture getSurfaceTexture() { | |
395 return surfaceTexture; | |
396 } | |
397 | |
398 /** | |
399 * Retrieve the handler that calls onTextureFrameAvailable(). This handler is
valid until | |
400 * dispose() is called. | |
401 */ | |
402 public Handler getHandler() { | |
403 return handler; | |
404 } | |
405 | |
406 /** | |
407 * Call this function to signal that you are done with the frame received in | |
408 * onTextureFrameAvailable(). Only one texture frame can be in flight at once,
so you must call | |
409 * this function in order to receive a new frame. | |
410 */ | |
411 public void returnTextureFrame() { | |
412 handler.post(new Runnable() { | |
413 @Override public void run() { | |
414 isTextureInUse = false; | |
415 if (isQuitting) { | |
416 release(); | |
417 } else { | |
418 tryDeliverTextureFrame(); | |
419 } | |
420 } | |
421 }); | |
422 } | |
423 | |
424 public boolean isTextureInUse() { | |
425 return isTextureInUse; | |
426 } | |
427 | |
428 /** | |
429 * Call disconnect() to stop receiving frames. OpenGL resources are released a
nd the handler is | |
430 * stopped when the texture frame has been returned by a call to returnTexture
Frame(). You are | |
431 * guaranteed to not receive any more onTextureFrameAvailable() after this fun
ction returns. | |
432 */ | |
433 public void dispose() { | |
434 Logging.d(TAG, "dispose()"); | |
435 ThreadUtils.invokeAtFrontUninterruptibly(handler, new Runnable() { | |
436 @Override | |
437 public void run() { | |
438 isQuitting = true; | |
439 if (!isTextureInUse) { | |
440 release(); | |
441 } | |
442 } | |
443 }); | |
444 } | |
445 | |
446 public void textureToYUV(ByteBuffer buf, | |
447 int width, int height, int stride, int textureId, float [] transformMatrix
) { | |
448 if (textureId != oesTextureId) | |
449 throw new IllegalStateException("textureToByteBuffer called with unexpecte
d textureId"); | |
450 | |
451 getYuvConverter().convert(buf, width, height, stride, textureId, transformMa
trix); | |
452 } | |
453 | |
454 private void updateTexImage() { | |
455 // SurfaceTexture.updateTexImage apparently can compete and deadlock with eg
lSwapBuffers, | |
456 // as observed on Nexus 5. Therefore, synchronize it with the EGL functions. | |
457 // See https://bugs.chromium.org/p/webrtc/issues/detail?id=5702 for more inf
o. | |
458 synchronized (EglBase.lock) { | |
459 surfaceTexture.updateTexImage(); | |
460 } | |
461 } | |
462 | |
463 private void tryDeliverTextureFrame() { | |
464 if (handler.getLooper().getThread() != Thread.currentThread()) { | |
465 throw new IllegalStateException("Wrong thread."); | |
466 } | |
467 if (isQuitting || !hasPendingTexture || isTextureInUse || listener == null)
{ | |
468 return; | |
469 } | |
470 isTextureInUse = true; | |
471 hasPendingTexture = false; | |
472 | |
473 updateTexImage(); | |
474 | |
475 final float[] transformMatrix = new float[16]; | |
476 surfaceTexture.getTransformMatrix(transformMatrix); | |
477 final long timestampNs = (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_C
REAM_SANDWICH) | |
478 ? surfaceTexture.getTimestamp() | |
479 : TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime()); | |
480 listener.onTextureFrameAvailable(oesTextureId, transformMatrix, timestampNs)
; | |
481 } | |
482 | |
483 private void release() { | |
484 if (handler.getLooper().getThread() != Thread.currentThread()) { | |
485 throw new IllegalStateException("Wrong thread."); | |
486 } | |
487 if (isTextureInUse || !isQuitting) { | |
488 throw new IllegalStateException("Unexpected release."); | |
489 } | |
490 synchronized (this) { | |
491 if (yuvConverter != null) | |
492 yuvConverter.release(); | |
493 } | |
494 GLES20.glDeleteTextures(1, new int[] {oesTextureId}, 0); | |
495 surfaceTexture.release(); | |
496 eglBase.release(); | |
497 handler.getLooper().quit(); | |
498 } | |
499 } | |
OLD | NEW |