OLD | NEW |
---|---|
1 /* | 1 /* |
2 * libjingle | 2 * libjingle |
3 * Copyright 2015 Google Inc. | 3 * Copyright 2015 Google Inc. |
4 * | 4 * |
5 * Redistribution and use in source and binary forms, with or without | 5 * Redistribution and use in source and binary forms, with or without |
6 * modification, are permitted provided that the following conditions are met: | 6 * modification, are permitted provided that the following conditions are met: |
7 * | 7 * |
8 * 1. Redistributions of source code must retain the above copyright notice, | 8 * 1. Redistributions of source code must retain the above copyright notice, |
9 * this list of conditions and the following disclaimer. | 9 * this list of conditions and the following disclaimer. |
10 * 2. Redistributions in binary form must reproduce the above copyright notice, | 10 * 2. Redistributions in binary form must reproduce the above copyright notice, |
(...skipping 17 matching lines...) Expand all Loading... | |
28 package org.webrtc; | 28 package org.webrtc; |
29 | 29 |
30 import android.graphics.SurfaceTexture; | 30 import android.graphics.SurfaceTexture; |
31 import android.opengl.GLES11Ext; | 31 import android.opengl.GLES11Ext; |
32 import android.opengl.GLES20; | 32 import android.opengl.GLES20; |
33 import android.os.Build; | 33 import android.os.Build; |
34 import android.os.Handler; | 34 import android.os.Handler; |
35 import android.os.HandlerThread; | 35 import android.os.HandlerThread; |
36 import android.os.SystemClock; | 36 import android.os.SystemClock; |
37 | 37 |
38 import java.nio.ByteBuffer; | |
39 import java.nio.FloatBuffer; | |
38 import java.util.concurrent.Callable; | 40 import java.util.concurrent.Callable; |
39 import java.util.concurrent.CountDownLatch; | 41 import java.util.concurrent.CountDownLatch; |
40 import java.util.concurrent.TimeUnit; | 42 import java.util.concurrent.TimeUnit; |
41 | 43 |
42 /** | 44 /** |
43 * Helper class to create and synchronize access to a SurfaceTexture. The caller will get notified | 45 * Helper class to create and synchronize access to a SurfaceTexture. The caller will get notified |
44 * of new frames in onTextureFrameAvailable(), and should call returnTextureFram e() when done with | 46 * of new frames in onTextureFrameAvailable(), and should call returnTextureFram e() when done with |
45 * the frame. Only one texture frame can be in flight at once, so returnTextureF rame() must be | 47 * the frame. Only one texture frame can be in flight at once, so returnTextureF rame() must be |
46 * called in order to receive a new frame. Call disconnect() to stop receiveing new frames and | 48 * called in order to receive a new frame. Call disconnect() to stop receiveing new frames and |
47 * release all resources. | 49 * release all resources. |
(...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
86 // http://grepcode.com/file/repository.grepcode.com/java/ext/com.google.andr oid/android/5.1.1_r1/android/graphics/SurfaceTexture.java#195. | 88 // http://grepcode.com/file/repository.grepcode.com/java/ext/com.google.andr oid/android/5.1.1_r1/android/graphics/SurfaceTexture.java#195. |
87 // Therefore, in order to control the callback thread on API lvl < 21, the S urfaceTextureHelper | 89 // Therefore, in order to control the callback thread on API lvl < 21, the S urfaceTextureHelper |
88 // is constructed on the |handler| thread. | 90 // is constructed on the |handler| thread. |
89 return ThreadUtils.invokeUninterruptibly(finalHandler, new Callable<SurfaceT extureHelper>() { | 91 return ThreadUtils.invokeUninterruptibly(finalHandler, new Callable<SurfaceT extureHelper>() { |
90 @Override public SurfaceTextureHelper call() { | 92 @Override public SurfaceTextureHelper call() { |
91 return new SurfaceTextureHelper(sharedContext, finalHandler, (handler == null)); | 93 return new SurfaceTextureHelper(sharedContext, finalHandler, (handler == null)); |
92 } | 94 } |
93 }); | 95 }); |
94 } | 96 } |
95 | 97 |
98 // State for YUV conversion, instantiated on demand. | |
99 static private class YuvConverter { | |
100 private final EglBase eglBase; | |
101 private final GlShader shader; | |
102 private boolean released = false; | |
103 | |
104 // Vertex coordinates in Normalized Device Coordinates, i.e. | |
105 // (-1, -1) is bottom-left and (1, 1) is top-right. | |
106 private static final FloatBuffer DEVICE_RECTANGLE = | |
107 GlUtil.createFloatBuffer(new float[] { | |
108 -1.0f, -1.0f, // Bottom left. | |
109 1.0f, -1.0f, // Bottom right. | |
110 -1.0f, 1.0f, // Top left. | |
111 1.0f, 1.0f, // Top right. | |
112 }); | |
113 | |
114 // Texture coordinates - (0, 0) is bottom-left and (1, 1) is top-right. | |
115 private static final FloatBuffer TEXTURE_RECTANGLE = | |
116 GlUtil.createFloatBuffer(new float[] { | |
117 0.0f, 0.0f, // Bottom left. | |
118 1.0f, 0.0f, // Bottom right. | |
119 0.0f, 1.0f, // Top left. | |
120 1.0f, 1.0f // Top right. | |
121 }); | |
122 | |
123 private static final String VERTEX_SHADER = | |
124 "varying vec2 interp_tc;\n" | |
125 + "attribute vec4 in_pos;\n" | |
126 + "attribute vec4 in_tc;\n" | |
127 + "\n" | |
128 + "uniform mat4 texMatrix;\n" | |
129 + "\n" | |
130 + "void main() {\n" | |
131 + " gl_Position = in_pos;\n" | |
132 + " interp_tc = (texMatrix * in_tc).xy;\n" | |
133 + "}\n"; | |
134 | |
135 private static final String FRAGMENT_SHADER = | |
136 "#extension GL_OES_EGL_image_external : require\n" | |
137 + "precision mediump float;\n" | |
138 + "varying vec2 interp_tc;\n" | |
139 + "\n" | |
140 + "uniform samplerExternalOES oesTex;\n" | |
141 // Difference in texture coordinate corresponding to one | |
142 // sub-pixel in the x direction. | |
143 + "uniform vec2 xUnit;\n" | |
144 // Color conversion coefficients, including constant term | |
145 + "uniform vec4 coeffs;\n" | |
146 + "\n" | |
147 + "void main() {\n" | |
148 // Since the alpha read from the texture is always 1, this could | |
149 // be written as a mat4 x vec4 multiply. However, that seems to | |
150 // give a worse framerate, possibly because the additional | |
151 // multiplies by 1.0 consume resources. TODO(nisse): Could also | |
152 // try to do it as a vec3 x mat3x4, followed by an add in of a | |
153 // constant vector. | |
154 + " gl_FragColor.r = coeffs.a + dot(coeffs.rgb,\n" | |
155 + " texture2D(oesTex, interp_tc - 1.5 * xUnit).rgb);\n" | |
156 + " gl_FragColor.g = coeffs.a + dot(coeffs.rgb,\n" | |
157 + " texture2D(oesTex, interp_tc - 0.5 * xUnit).rgb);\n" | |
158 + " gl_FragColor.b = coeffs.a + dot(coeffs.rgb,\n" | |
159 + " texture2D(oesTex, interp_tc + 0.5 * xUnit).rgb);\n" | |
160 + " gl_FragColor.a = coeffs.a + dot(coeffs.rgb,\n" | |
161 + " texture2D(oesTex, interp_tc + 1.5 * xUnit).rgb);\n" | |
162 + "}\n"; | |
163 | |
164 private int texMatrixLoc; | |
165 private int xUnitLoc; | |
166 private int coeffsLoc;; | |
167 | |
168 YuvConverter (EglBase.Context sharedContext) { | |
169 eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_RGBA_BUFFER); | |
170 eglBase.createDummyPbufferSurface(); | |
171 eglBase.makeCurrent(); | |
172 | |
173 shader = new GlShader(VERTEX_SHADER, FRAGMENT_SHADER); | |
174 shader.useProgram(); | |
175 texMatrixLoc = shader.getUniformLocation("texMatrix"); | |
176 xUnitLoc = shader.getUniformLocation("xUnit"); | |
177 coeffsLoc = shader.getUniformLocation("coeffs"); | |
178 GLES20.glUniform1i(shader.getUniformLocation("oesTex"), 0); | |
179 GlUtil.checkNoGLES2Error("Initialize fragment shader uniform values."); | |
180 // Initialize vertex shader attributes. | |
181 shader.setVertexAttribArray("in_pos", 2, DEVICE_RECTANGLE); | |
182 // If the width is not a multiple of 4 pixels, the texture | |
183 // will be scaled up slightly and clipped at the right border. | |
184 shader.setVertexAttribArray("in_tc", 2, TEXTURE_RECTANGLE); | |
185 } | |
186 | |
187 synchronized void convert(ByteBuffer buf, | |
188 int width, int height, int stride, int textureId, float [] transformMatr ix) { | |
189 if (released) { | |
190 throw new IllegalStateException( | |
191 "YuvConverter.convert called on released object"); | |
192 } | |
193 if (stride % 8 != 0) { | |
194 throw new IllegalArgumentException( | |
195 "Invalid stride, must be a multiple of 8"); | |
196 } | |
197 if (stride < width){ | |
198 throw new IllegalArgumentException( | |
199 "Invalid stride, must >= width"); | |
200 } | |
201 | |
202 int y_width = (width+3) / 4; | |
203 int uv_width = (width+7) / 8; | |
204 int uv_height = (height+1)/2; | |
205 int total_height = height + uv_height; | |
206 int size = stride * total_height; | |
207 | |
208 if (buf.capacity() < size) { | |
209 throw new IllegalArgumentException("YuvConverter.convert called with too small buffer"); | |
210 } | |
211 // Produce a frame buffer starting at top-left corner, not | |
212 // bottom-left. | |
213 transformMatrix = | |
214 RendererCommon.multiplyMatrices(transformMatrix, | |
215 RendererCommon.verticalFlipMatrix()); | |
216 | |
217 try { | |
218 // Reuse surface, if possible. TODO(nisse): Add an eglBase | |
perkj_webrtc
2015/12/09 09:28:28
I would recommend to do this now or remove the tod
nisse-webrtc
2015/12/09 12:02:21
I'm adding a helper function, withPbufferSurface.
| |
219 // helper function, say, makeCurrentWithSize. | |
220 if (eglBase.hasSurface()) { | |
221 if (eglBase.surfaceWidth() != stride/4 || | |
222 eglBase.surfaceHeight() != total_height){ | |
223 eglBase.releaseSurface(); | |
224 eglBase.createPbufferSurface(stride/4, total_height); | |
225 } | |
226 } else { | |
227 eglBase.createPbufferSurface(stride/4, total_height); | |
228 } | |
229 | |
230 eglBase.makeCurrent(); | |
231 | |
232 GlUtil.checkNoGLES2Error("Initialize fragment shader uniform values."); | |
perkj_webrtc
2015/12/09 09:28:28
Can we skip checking for error on every line. Just
nisse-webrtc
2015/12/09 12:02:21
Done.
| |
233 | |
234 GLES20.glActiveTexture(GLES20.GL_TEXTURE0); | |
235 GlUtil.checkNoGLES2Error("Active texture"); | |
236 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId); | |
237 GLES20.glUniformMatrix4fv(texMatrixLoc, 1, false, transformMatrix, 0); | |
238 | |
239 // Draw Y */ | |
240 GLES20.glViewport(0, 0, y_width, height); | |
241 GlUtil.checkNoGLES2Error("Viewport"); | |
242 // Matrix * (1;0;0;0) / width. Note that opengl uses column major order. | |
243 GLES20.glUniform2f(xUnitLoc, | |
244 transformMatrix[0] / width, | |
245 transformMatrix[1] / width); | |
246 // Y'UV444 to RGB888, see | |
247 // https://en.wikipedia.org/wiki/YUV#Y.27UV444_to_RGB888_conversion | |
248 GLES20.glUniform4f(coeffsLoc, 0.299f, 0.587f, 0.114f, 0.0f); | |
249 GlUtil.checkNoGLES2Error("Binding texture and matrix"); | |
250 | |
251 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); | |
252 GlUtil.checkNoGLES2Error("glDrawArrays, Y"); | |
253 | |
254 // Draw U */ | |
255 GLES20.glViewport(0, height, uv_width, uv_height); | |
256 GlUtil.checkNoGLES2Error("Viewport"); | |
257 // Matrix * (1;0;0;0) / (2*width). Note that opengl uses column major or der. | |
258 GLES20.glUniform2f(xUnitLoc, | |
259 transformMatrix[0] / (2.0f*width), | |
260 transformMatrix[1] / (2.0f*width)); | |
261 /* Use ITU-R coefficients for U and V */ | |
262 GLES20.glUniform4f(coeffsLoc, -0.169f, -0.331f, 0.499f, 0.5f); | |
263 GlUtil.checkNoGLES2Error("Binding texture and matrix"); | |
264 | |
265 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); | |
266 GlUtil.checkNoGLES2Error("glDrawArrays, U"); | |
267 | |
268 // Draw V */ | |
269 GLES20.glViewport(stride/8, height, uv_width, uv_height); | |
270 GlUtil.checkNoGLES2Error("Viewport"); | |
271 /* Use ITU-R coefficients for U and V */ | |
272 GLES20.glUniform4f(coeffsLoc, 0.499f, -0.418f, -0.0813f, 0.5f); | |
273 GlUtil.checkNoGLES2Error("Binding texture and matrix"); | |
274 | |
275 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); | |
276 GlUtil.checkNoGLES2Error("glDrawArrays, V"); | |
277 | |
278 GLES20.glReadPixels(0, 0, stride/4, total_height, GLES20.GL_RGBA, | |
279 GLES20.GL_UNSIGNED_BYTE, buf); | |
280 GlUtil.checkNoGLES2Error("glReadPixels"); | |
281 | |
282 // Unbind texture. Reportedly needed on some devices to get | |
283 // the texture updated from the camera. | |
284 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0); | |
285 } | |
286 finally { | |
287 eglBase.detachCurrent(); | |
288 } | |
289 } | |
290 | |
291 synchronized public void release() { | |
292 released = true; | |
293 eglBase.makeCurrent(); | |
294 shader.release(); | |
295 eglBase.release(); | |
296 } | |
297 } | |
298 | |
96 private final Handler handler; | 299 private final Handler handler; |
97 private boolean isOwningThread; | 300 private boolean isOwningThread; |
98 private final EglBase eglBase; | 301 private final EglBase eglBase; |
99 private final SurfaceTexture surfaceTexture; | 302 private final SurfaceTexture surfaceTexture; |
100 private final int oesTextureId; | 303 private final int oesTextureId; |
304 private YuvConverter yuvConverter; | |
305 | |
101 private OnTextureFrameAvailableListener listener; | 306 private OnTextureFrameAvailableListener listener; |
102 // The possible states of this class. | 307 // The possible states of this class. |
103 private boolean hasPendingTexture = false; | 308 private boolean hasPendingTexture = false; |
104 private boolean isTextureInUse = false; | 309 private boolean isTextureInUse = false; |
105 private boolean isQuitting = false; | 310 private boolean isQuitting = false; |
106 | 311 |
107 private SurfaceTextureHelper(EglBase.Context sharedContext, | 312 private SurfaceTextureHelper(EglBase.Context sharedContext, |
108 Handler handler, boolean isOwningThread) { | 313 Handler handler, boolean isOwningThread) { |
109 if (handler.getLooper().getThread() != Thread.currentThread()) { | 314 if (handler.getLooper().getThread() != Thread.currentThread()) { |
110 throw new IllegalStateException("SurfaceTextureHelper must be created on t he handler thread"); | 315 throw new IllegalStateException("SurfaceTextureHelper must be created on t he handler thread"); |
111 } | 316 } |
112 this.handler = handler; | 317 this.handler = handler; |
113 this.isOwningThread = isOwningThread; | 318 this.isOwningThread = isOwningThread; |
114 | 319 |
115 eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_BUFFER); | 320 eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_BUFFER); |
116 eglBase.createDummyPbufferSurface(); | 321 eglBase.createDummyPbufferSurface(); |
117 eglBase.makeCurrent(); | 322 eglBase.makeCurrent(); |
118 | 323 |
119 oesTextureId = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); | 324 oesTextureId = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); |
120 surfaceTexture = new SurfaceTexture(oesTextureId); | 325 surfaceTexture = new SurfaceTexture(oesTextureId); |
121 } | 326 } |
122 | 327 |
328 private YuvConverter getYuvConverter() { | |
329 // yuvConverter is assign once | |
330 if (yuvConverter != null) | |
331 return yuvConverter; | |
332 | |
333 synchronized(this) { | |
334 if (yuvConverter == null) | |
335 yuvConverter = new YuvConverter(eglBase.getEglBaseContext()); | |
336 return yuvConverter; | |
337 } | |
338 } | |
339 | |
123 /** | 340 /** |
124 * Start to stream textures to the given |listener|. | 341 * Start to stream textures to the given |listener|. |
125 * A Listener can only be set once. | 342 * A Listener can only be set once. |
126 */ | 343 */ |
127 public void setListener(OnTextureFrameAvailableListener listener) { | 344 public void setListener(OnTextureFrameAvailableListener listener) { |
128 if (this.listener != null) { | 345 if (this.listener != null) { |
129 throw new IllegalStateException("SurfaceTextureHelper listener has already been set."); | 346 throw new IllegalStateException("SurfaceTextureHelper listener has already been set."); |
130 } | 347 } |
131 this.listener = listener; | 348 this.listener = listener; |
132 surfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailab leListener() { | 349 surfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailab leListener() { |
(...skipping 67 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
200 * onTextureFrameAvailable() after this function returns. | 417 * onTextureFrameAvailable() after this function returns. |
201 */ | 418 */ |
202 public void disconnect(Handler handler) { | 419 public void disconnect(Handler handler) { |
203 if (this.handler != handler) { | 420 if (this.handler != handler) { |
204 throw new IllegalStateException("Wrong handler."); | 421 throw new IllegalStateException("Wrong handler."); |
205 } | 422 } |
206 isOwningThread = true; | 423 isOwningThread = true; |
207 disconnect(); | 424 disconnect(); |
208 } | 425 } |
209 | 426 |
427 public void textureToYUV(ByteBuffer buf, | |
428 int width, int height, int stride, int textureId, float [] transformMatrix ) { | |
429 if (textureId != oesTextureId) | |
430 throw new IllegalStateException("textureToByteBuffer called with unexpecte d textureId"); | |
431 | |
432 getYuvConverter().convert(buf, width, height, stride, textureId, transformMa trix); | |
433 } | |
434 | |
210 private void tryDeliverTextureFrame() { | 435 private void tryDeliverTextureFrame() { |
211 if (handler.getLooper().getThread() != Thread.currentThread()) { | 436 if (handler.getLooper().getThread() != Thread.currentThread()) { |
212 throw new IllegalStateException("Wrong thread."); | 437 throw new IllegalStateException("Wrong thread."); |
213 } | 438 } |
214 if (isQuitting || !hasPendingTexture || isTextureInUse) { | 439 if (isQuitting || !hasPendingTexture || isTextureInUse) { |
215 return; | 440 return; |
216 } | 441 } |
217 isTextureInUse = true; | 442 isTextureInUse = true; |
218 hasPendingTexture = false; | 443 hasPendingTexture = false; |
219 | 444 |
220 eglBase.makeCurrent(); | 445 eglBase.makeCurrent(); |
221 surfaceTexture.updateTexImage(); | 446 surfaceTexture.updateTexImage(); |
222 | 447 |
223 final float[] transformMatrix = new float[16]; | 448 final float[] transformMatrix = new float[16]; |
224 surfaceTexture.getTransformMatrix(transformMatrix); | 449 surfaceTexture.getTransformMatrix(transformMatrix); |
225 final long timestampNs = (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_C REAM_SANDWICH) | 450 final long timestampNs = (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_C REAM_SANDWICH) |
226 ? surfaceTexture.getTimestamp() | 451 ? surfaceTexture.getTimestamp() |
227 : TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime()); | 452 : TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime()); |
228 listener.onTextureFrameAvailable(oesTextureId, transformMatrix, timestampNs) ; | 453 listener.onTextureFrameAvailable(oesTextureId, transformMatrix, timestampNs) ; |
229 } | 454 } |
230 | 455 |
231 private void release() { | 456 private void release() { |
232 if (handler.getLooper().getThread() != Thread.currentThread()) { | 457 if (handler.getLooper().getThread() != Thread.currentThread()) { |
233 throw new IllegalStateException("Wrong thread."); | 458 throw new IllegalStateException("Wrong thread."); |
234 } | 459 } |
235 if (isTextureInUse || !isQuitting) { | 460 if (isTextureInUse || !isQuitting) { |
236 throw new IllegalStateException("Unexpected release."); | 461 throw new IllegalStateException("Unexpected release."); |
237 } | 462 } |
463 synchronized (this) { | |
464 if (yuvConverter != null) | |
465 yuvConverter.release(); | |
466 } | |
238 eglBase.makeCurrent(); | 467 eglBase.makeCurrent(); |
239 GLES20.glDeleteTextures(1, new int[] {oesTextureId}, 0); | 468 GLES20.glDeleteTextures(1, new int[] {oesTextureId}, 0); |
240 surfaceTexture.release(); | 469 surfaceTexture.release(); |
241 eglBase.release(); | 470 eglBase.release(); |
242 handler.getLooper().quit(); | 471 handler.getLooper().quit(); |
243 } | 472 } |
244 } | 473 } |
OLD | NEW |