OLD | NEW |
---|---|
1 /* | 1 /* |
2 * libjingle | 2 * libjingle |
3 * Copyright 2015 Google Inc. | 3 * Copyright 2015 Google Inc. |
4 * | 4 * |
5 * Redistribution and use in source and binary forms, with or without | 5 * Redistribution and use in source and binary forms, with or without |
6 * modification, are permitted provided that the following conditions are met: | 6 * modification, are permitted provided that the following conditions are met: |
7 * | 7 * |
8 * 1. Redistributions of source code must retain the above copyright notice, | 8 * 1. Redistributions of source code must retain the above copyright notice, |
9 * this list of conditions and the following disclaimer. | 9 * this list of conditions and the following disclaimer. |
10 * 2. Redistributions in binary form must reproduce the above copyright notice, | 10 * 2. Redistributions in binary form must reproduce the above copyright notice, |
(...skipping 17 matching lines...) Expand all Loading... | |
28 package org.webrtc; | 28 package org.webrtc; |
29 | 29 |
30 import android.graphics.SurfaceTexture; | 30 import android.graphics.SurfaceTexture; |
31 import android.opengl.GLES11Ext; | 31 import android.opengl.GLES11Ext; |
32 import android.opengl.GLES20; | 32 import android.opengl.GLES20; |
33 import android.os.Build; | 33 import android.os.Build; |
34 import android.os.Handler; | 34 import android.os.Handler; |
35 import android.os.HandlerThread; | 35 import android.os.HandlerThread; |
36 import android.os.SystemClock; | 36 import android.os.SystemClock; |
37 | 37 |
38 import java.nio.ByteBuffer; | |
39 import java.nio.FloatBuffer; | |
38 import java.util.concurrent.Callable; | 40 import java.util.concurrent.Callable; |
39 import java.util.concurrent.CountDownLatch; | 41 import java.util.concurrent.CountDownLatch; |
40 import java.util.concurrent.TimeUnit; | 42 import java.util.concurrent.TimeUnit; |
41 | 43 |
42 /** | 44 /** |
43 * Helper class to create and synchronize access to a SurfaceTexture. The caller will get notified | 45 * Helper class to create and synchronize access to a SurfaceTexture. The caller will get notified |
44 * of new frames in onTextureFrameAvailable(), and should call returnTextureFram e() when done with | 46 * of new frames in onTextureFrameAvailable(), and should call returnTextureFram e() when done with |
45 * the frame. Only one texture frame can be in flight at once, so returnTextureF rame() must be | 47 * the frame. Only one texture frame can be in flight at once, so returnTextureF rame() must be |
46 * called in order to receive a new frame. Call disconnect() to stop receiveing new frames and | 48 * called in order to receive a new frame. Call disconnect() to stop receiveing new frames and |
47 * release all resources. | 49 * release all resources. |
(...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
86 // http://grepcode.com/file/repository.grepcode.com/java/ext/com.google.andr oid/android/5.1.1_r1/android/graphics/SurfaceTexture.java#195. | 88 // http://grepcode.com/file/repository.grepcode.com/java/ext/com.google.andr oid/android/5.1.1_r1/android/graphics/SurfaceTexture.java#195. |
87 // Therefore, in order to control the callback thread on API lvl < 21, the S urfaceTextureHelper | 89 // Therefore, in order to control the callback thread on API lvl < 21, the S urfaceTextureHelper |
88 // is constructed on the |handler| thread. | 90 // is constructed on the |handler| thread. |
89 return ThreadUtils.invokeUninterruptibly(finalHandler, new Callable<SurfaceT extureHelper>() { | 91 return ThreadUtils.invokeUninterruptibly(finalHandler, new Callable<SurfaceT extureHelper>() { |
90 @Override public SurfaceTextureHelper call() { | 92 @Override public SurfaceTextureHelper call() { |
91 return new SurfaceTextureHelper(sharedContext, finalHandler, (handler == null)); | 93 return new SurfaceTextureHelper(sharedContext, finalHandler, (handler == null)); |
92 } | 94 } |
93 }); | 95 }); |
94 } | 96 } |
95 | 97 |
98 // State for YUV conversion, instantiated on demand. | |
99 static private class YuvConverter { | |
100 private final EglBase eglBase; | |
101 private final GlShader shader; | |
102 private boolean released = false; | |
103 | |
104 // Vertex coordinates in Normalized Device Coordinates, i.e. | |
105 // (-1, -1) is bottom-left and (1, 1) is top-right. | |
106 private static final FloatBuffer DEVICE_RECTANGLE = | |
107 GlUtil.createFloatBuffer(new float[] { | |
108 -1.0f, -1.0f, // Bottom left. | |
109 1.0f, -1.0f, // Bottom right. | |
110 -1.0f, 1.0f, // Top left. | |
111 1.0f, 1.0f, // Top right. | |
112 }); | |
113 | |
114 // Texture coordinates - (0, 0) is bottom-left and (1, 1) is top-right. | |
115 private static final FloatBuffer TEXTURE_RECTANGLE = | |
116 GlUtil.createFloatBuffer(new float[] { | |
117 0.0f, 0.0f, // Bottom left. | |
118 1.0f, 0.0f, // Bottom right. | |
119 0.0f, 1.0f, // Top left. | |
120 1.0f, 1.0f // Top right. | |
121 }); | |
122 | |
123 private static final String VERTEX_SHADER = | |
124 "varying vec2 interp_tc;\n" | |
125 + "attribute vec4 in_pos;\n" | |
126 + "attribute vec4 in_tc;\n" | |
127 + "\n" | |
128 + "uniform mat4 texMatrix;\n" | |
129 + "\n" | |
130 + "void main() {\n" | |
131 + " gl_Position = in_pos;\n" | |
132 + " interp_tc = (texMatrix * in_tc).xy;\n" | |
133 + "}\n"; | |
134 | |
135 private static final String FRAGMENT_SHADER = | |
136 "#extension GL_OES_EGL_image_external : require\n" | |
137 + "precision mediump float;\n" | |
138 + "varying vec2 interp_tc;\n" | |
139 + "\n" | |
140 + "uniform samplerExternalOES oesTex;\n" | |
141 // Difference in texture coordinate corresponding to one | |
142 // sub-pixel in the x direction. | |
143 + "uniform vec2 xUnit;\n" | |
144 // Color conversion coefficients, including constant term | |
145 + "uniform vec4 coeffs;\n" | |
146 + "\n" | |
147 + "void main() {\n" | |
148 // Since the alpha read from the texture is always 1, this could | |
149 // be written as a mat4 x vec4 multiply. However, that seems to | |
150 // give a worse framerate, possibly because the additional | |
151 // multiplies by 1.0 consume resources. TODO(nisse): Could also | |
152 // try to do it as a vec3 x mat3x4, followed by an add in of a | |
153 // constant vector. | |
154 + " gl_FragColor.r = coeffs.a + dot(coeffs.rgb,\n" | |
155 + " texture2D(oesTex, interp_tc - 1.5 * xUnit).rgb);\n" | |
156 + " gl_FragColor.g = coeffs.a + dot(coeffs.rgb,\n" | |
157 + " texture2D(oesTex, interp_tc - 0.5 * xUnit).rgb);\n" | |
158 + " gl_FragColor.b = coeffs.a + dot(coeffs.rgb,\n" | |
159 + " texture2D(oesTex, interp_tc + 0.5 * xUnit).rgb);\n" | |
160 + " gl_FragColor.a = coeffs.a + dot(coeffs.rgb,\n" | |
161 + " texture2D(oesTex, interp_tc + 1.5 * xUnit).rgb);\n" | |
162 + "}\n"; | |
163 | |
164 private int texMatrixLoc; | |
165 private int xUnitLoc; | |
166 private int coeffsLoc;; | |
167 | |
168 YuvConverter (EglBase.Context sharedContext) { | |
169 eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_RGBA_BUFFER); | |
170 eglBase.createDummyPbufferSurface(); | |
171 eglBase.makeCurrent(); | |
172 | |
173 shader = new GlShader(VERTEX_SHADER, FRAGMENT_SHADER); | |
174 shader.useProgram(); | |
175 texMatrixLoc = shader.getUniformLocation("texMatrix"); | |
176 xUnitLoc = shader.getUniformLocation("xUnit"); | |
177 coeffsLoc = shader.getUniformLocation("coeffs"); | |
178 GLES20.glUniform1i(shader.getUniformLocation("oesTex"), 0); | |
179 GlUtil.checkNoGLES2Error("Initialize fragment shader uniform values."); | |
180 // Initialize vertex shader attributes. | |
181 shader.setVertexAttribArray("in_pos", 2, DEVICE_RECTANGLE); | |
182 // If the width is not a multiple of 4 pixels, the texture | |
183 // will be scaled up slightly and clipped at the right border. | |
184 shader.setVertexAttribArray("in_tc", 2, TEXTURE_RECTANGLE); | |
185 eglBase.detachCurrent(); | |
186 } | |
187 | |
188 synchronized void convert(ByteBuffer buf, | |
189 int width, int height, int stride, int textureId, float [] transformMatr ix) { | |
190 if (released) { | |
191 throw new IllegalStateException( | |
192 "YuvConverter.convert called on released object"); | |
193 } | |
194 | |
195 // We draw into a buffer laid out like | |
196 // | |
197 // +---------+ | |
198 // | | | |
199 // | Y | | |
200 // | | | |
201 // | | | |
202 // +----+----+ | |
203 // | U | V | | |
204 // | | | | |
205 // +----+----+ | |
206 // | |
207 // In memory, we use the same stride for all of Y, U and V. The | |
208 // U data starts at offset |height| * |stride| from the Y data, | |
209 // and the V data starts at at offset |stride/2| from the U | |
210 // data, with rows of U and V data alternating. | |
211 // | |
212 // Now, it would have made sense to allocate a pixel buffer with | |
213 // a single byte per pixel (EGL10.EGL_COLOR_BUFFER_TYPE, | |
214 // EGL10.EGL_LUMINANCE_BUFFER,), but that seems to be | |
215 // unsupported by devices. So do the following hack: Allocate an | |
216 // RGBA buffer, of width |stride|/4. To render each of these | |
217 // large pixels, sample the texture at 4 different x coordinates | |
218 // and store the results in the four components. | |
219 // | |
220 // Since the V data needs to start on a boundary of such a | |
221 // larger pixel, it is not sufficient that |stride| is even, it | |
222 // has to be a multiple of 8 pixels. | |
223 | |
224 if (stride % 8 != 0) { | |
225 throw new IllegalArgumentException( | |
226 "Invalid stride, must be a multiple of 8"); | |
227 } | |
228 if (stride < width){ | |
229 throw new IllegalArgumentException( | |
230 "Invalid stride, must >= width"); | |
231 } | |
232 | |
233 int y_width = (width+3) / 4; | |
234 int uv_width = (width+7) / 8; | |
235 int uv_height = (height+1)/2; | |
236 int total_height = height + uv_height; | |
237 int size = stride * total_height; | |
238 | |
239 if (buf.capacity() < size) { | |
240 throw new IllegalArgumentException("YuvConverter.convert called with too small buffer"); | |
241 } | |
242 // Produce a frame buffer starting at top-left corner, not | |
243 // bottom-left. | |
244 transformMatrix = | |
245 RendererCommon.multiplyMatrices(transformMatrix, | |
246 RendererCommon.verticalFlipMatrix()); | |
247 | |
248 // Reuse surface, if possible. | |
perkj_webrtc
2015/12/10 12:33:35
s Create new pBuffferSurface with the correct size
| |
249 if (eglBase.hasSurface()) { | |
250 if (eglBase.surfaceWidth() != stride/4 || | |
251 eglBase.surfaceHeight() != total_height){ | |
252 eglBase.releaseSurface(); | |
253 eglBase.createPbufferSurface(stride/4, total_height); | |
254 } | |
255 } else { | |
256 eglBase.createPbufferSurface(stride/4, total_height); | |
257 } | |
258 | |
259 eglBase.makeCurrent(); | |
260 | |
261 GLES20.glActiveTexture(GLES20.GL_TEXTURE0); | |
262 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId); | |
263 GLES20.glUniformMatrix4fv(texMatrixLoc, 1, false, transformMatrix, 0); | |
264 | |
265 // Draw Y | |
266 GLES20.glViewport(0, 0, y_width, height); | |
267 // Matrix * (1;0;0;0) / width. Note that opengl uses column major order. | |
268 GLES20.glUniform2f(xUnitLoc, | |
269 transformMatrix[0] / width, | |
270 transformMatrix[1] / width); | |
271 // Y'UV444 to RGB888, see | |
272 // https://en.wikipedia.org/wiki/YUV#Y.27UV444_to_RGB888_conversion. | |
273 // We use the ITU-R coefficients for U and V */ | |
274 GLES20.glUniform4f(coeffsLoc, 0.299f, 0.587f, 0.114f, 0.0f); | |
275 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); | |
276 | |
277 // Draw U | |
278 GLES20.glViewport(0, height, uv_width, uv_height); | |
279 // Matrix * (1;0;0;0) / (2*width). Note that opengl uses column major orde r. | |
280 GLES20.glUniform2f(xUnitLoc, | |
281 transformMatrix[0] / (2.0f*width), | |
282 transformMatrix[1] / (2.0f*width)); | |
283 GLES20.glUniform4f(coeffsLoc, -0.169f, -0.331f, 0.499f, 0.5f); | |
284 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); | |
285 | |
286 // Draw V | |
287 GLES20.glViewport(stride/8, height, uv_width, uv_height); | |
288 GLES20.glUniform4f(coeffsLoc, 0.499f, -0.418f, -0.0813f, 0.5f); | |
289 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); | |
290 | |
291 GLES20.glReadPixels(0, 0, stride/4, total_height, GLES20.GL_RGBA, | |
292 GLES20.GL_UNSIGNED_BYTE, buf); | |
293 | |
294 GlUtil.checkNoGLES2Error("YuvConverter.convert"); | |
295 | |
296 // Unbind texture. Reportedly needed on some devices to get | |
297 // the texture updated from the camera. | |
298 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0); | |
299 eglBase.detachCurrent(); | |
300 } | |
301 | |
302 synchronized void release() { | |
303 released = true; | |
304 eglBase.makeCurrent(); | |
305 shader.release(); | |
306 eglBase.release(); | |
307 } | |
308 } | |
309 | |
96 private final Handler handler; | 310 private final Handler handler; |
97 private boolean isOwningThread; | 311 private boolean isOwningThread; |
98 private final EglBase eglBase; | 312 private final EglBase eglBase; |
99 private final SurfaceTexture surfaceTexture; | 313 private final SurfaceTexture surfaceTexture; |
100 private final int oesTextureId; | 314 private final int oesTextureId; |
315 private YuvConverter yuvConverter; | |
316 | |
101 private OnTextureFrameAvailableListener listener; | 317 private OnTextureFrameAvailableListener listener; |
102 // The possible states of this class. | 318 // The possible states of this class. |
103 private boolean hasPendingTexture = false; | 319 private boolean hasPendingTexture = false; |
104 private boolean isTextureInUse = false; | 320 private boolean isTextureInUse = false; |
105 private boolean isQuitting = false; | 321 private boolean isQuitting = false; |
106 | 322 |
107 private SurfaceTextureHelper(EglBase.Context sharedContext, | 323 private SurfaceTextureHelper(EglBase.Context sharedContext, |
108 Handler handler, boolean isOwningThread) { | 324 Handler handler, boolean isOwningThread) { |
109 if (handler.getLooper().getThread() != Thread.currentThread()) { | 325 if (handler.getLooper().getThread() != Thread.currentThread()) { |
110 throw new IllegalStateException("SurfaceTextureHelper must be created on t he handler thread"); | 326 throw new IllegalStateException("SurfaceTextureHelper must be created on t he handler thread"); |
111 } | 327 } |
112 this.handler = handler; | 328 this.handler = handler; |
113 this.isOwningThread = isOwningThread; | 329 this.isOwningThread = isOwningThread; |
114 | 330 |
115 eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_BUFFER); | 331 eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_BUFFER); |
116 eglBase.createDummyPbufferSurface(); | 332 eglBase.createDummyPbufferSurface(); |
117 eglBase.makeCurrent(); | 333 eglBase.makeCurrent(); |
118 | 334 |
119 oesTextureId = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); | 335 oesTextureId = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); |
120 surfaceTexture = new SurfaceTexture(oesTextureId); | 336 surfaceTexture = new SurfaceTexture(oesTextureId); |
121 } | 337 } |
122 | 338 |
339 private YuvConverter getYuvConverter() { | |
340 // yuvConverter is assigned once | |
341 if (yuvConverter != null) | |
342 return yuvConverter; | |
343 | |
344 synchronized(this) { | |
345 if (yuvConverter == null) | |
346 yuvConverter = new YuvConverter(eglBase.getEglBaseContext()); | |
347 return yuvConverter; | |
348 } | |
349 } | |
350 | |
123 /** | 351 /** |
124 * Start to stream textures to the given |listener|. | 352 * Start to stream textures to the given |listener|. |
125 * A Listener can only be set once. | 353 * A Listener can only be set once. |
126 */ | 354 */ |
127 public void setListener(OnTextureFrameAvailableListener listener) { | 355 public void setListener(OnTextureFrameAvailableListener listener) { |
128 if (this.listener != null) { | 356 if (this.listener != null) { |
129 throw new IllegalStateException("SurfaceTextureHelper listener has already been set."); | 357 throw new IllegalStateException("SurfaceTextureHelper listener has already been set."); |
130 } | 358 } |
131 this.listener = listener; | 359 this.listener = listener; |
132 surfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailab leListener() { | 360 surfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailab leListener() { |
(...skipping 67 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
200 * onTextureFrameAvailable() after this function returns. | 428 * onTextureFrameAvailable() after this function returns. |
201 */ | 429 */ |
202 public void disconnect(Handler handler) { | 430 public void disconnect(Handler handler) { |
203 if (this.handler != handler) { | 431 if (this.handler != handler) { |
204 throw new IllegalStateException("Wrong handler."); | 432 throw new IllegalStateException("Wrong handler."); |
205 } | 433 } |
206 isOwningThread = true; | 434 isOwningThread = true; |
207 disconnect(); | 435 disconnect(); |
208 } | 436 } |
209 | 437 |
438 public void textureToYUV(ByteBuffer buf, | |
439 int width, int height, int stride, int textureId, float [] transformMatrix ) { | |
440 if (textureId != oesTextureId) | |
441 throw new IllegalStateException("textureToByteBuffer called with unexpecte d textureId"); | |
442 | |
443 getYuvConverter().convert(buf, width, height, stride, textureId, transformMa trix); | |
444 } | |
445 | |
210 private void tryDeliverTextureFrame() { | 446 private void tryDeliverTextureFrame() { |
211 if (handler.getLooper().getThread() != Thread.currentThread()) { | 447 if (handler.getLooper().getThread() != Thread.currentThread()) { |
212 throw new IllegalStateException("Wrong thread."); | 448 throw new IllegalStateException("Wrong thread."); |
213 } | 449 } |
214 if (isQuitting || !hasPendingTexture || isTextureInUse) { | 450 if (isQuitting || !hasPendingTexture || isTextureInUse) { |
215 return; | 451 return; |
216 } | 452 } |
217 isTextureInUse = true; | 453 isTextureInUse = true; |
218 hasPendingTexture = false; | 454 hasPendingTexture = false; |
219 | 455 |
220 eglBase.makeCurrent(); | 456 eglBase.makeCurrent(); |
221 surfaceTexture.updateTexImage(); | 457 surfaceTexture.updateTexImage(); |
222 | 458 |
223 final float[] transformMatrix = new float[16]; | 459 final float[] transformMatrix = new float[16]; |
224 surfaceTexture.getTransformMatrix(transformMatrix); | 460 surfaceTexture.getTransformMatrix(transformMatrix); |
225 final long timestampNs = (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_C REAM_SANDWICH) | 461 final long timestampNs = (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_C REAM_SANDWICH) |
226 ? surfaceTexture.getTimestamp() | 462 ? surfaceTexture.getTimestamp() |
227 : TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime()); | 463 : TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime()); |
228 listener.onTextureFrameAvailable(oesTextureId, transformMatrix, timestampNs) ; | 464 listener.onTextureFrameAvailable(oesTextureId, transformMatrix, timestampNs) ; |
229 } | 465 } |
230 | 466 |
231 private void release() { | 467 private void release() { |
232 if (handler.getLooper().getThread() != Thread.currentThread()) { | 468 if (handler.getLooper().getThread() != Thread.currentThread()) { |
233 throw new IllegalStateException("Wrong thread."); | 469 throw new IllegalStateException("Wrong thread."); |
234 } | 470 } |
235 if (isTextureInUse || !isQuitting) { | 471 if (isTextureInUse || !isQuitting) { |
236 throw new IllegalStateException("Unexpected release."); | 472 throw new IllegalStateException("Unexpected release."); |
237 } | 473 } |
474 synchronized (this) { | |
475 if (yuvConverter != null) | |
476 yuvConverter.release(); | |
477 } | |
238 eglBase.makeCurrent(); | 478 eglBase.makeCurrent(); |
239 GLES20.glDeleteTextures(1, new int[] {oesTextureId}, 0); | 479 GLES20.glDeleteTextures(1, new int[] {oesTextureId}, 0); |
240 surfaceTexture.release(); | 480 surfaceTexture.release(); |
241 eglBase.release(); | 481 eglBase.release(); |
242 handler.getLooper().quit(); | 482 handler.getLooper().quit(); |
243 } | 483 } |
244 } | 484 } |
OLD | NEW |