OLD | NEW |
1 /* | 1 /* |
2 * libjingle | 2 * libjingle |
3 * Copyright 2015 Google Inc. | 3 * Copyright 2015 Google Inc. |
4 * | 4 * |
5 * Redistribution and use in source and binary forms, with or without | 5 * Redistribution and use in source and binary forms, with or without |
6 * modification, are permitted provided that the following conditions are met: | 6 * modification, are permitted provided that the following conditions are met: |
7 * | 7 * |
8 * 1. Redistributions of source code must retain the above copyright notice, | 8 * 1. Redistributions of source code must retain the above copyright notice, |
9 * this list of conditions and the following disclaimer. | 9 * this list of conditions and the following disclaimer. |
10 * 2. Redistributions in binary form must reproduce the above copyright notice, | 10 * 2. Redistributions in binary form must reproduce the above copyright notice, |
(...skipping 17 matching lines...) Expand all Loading... |
28 package org.webrtc; | 28 package org.webrtc; |
29 | 29 |
30 import android.graphics.SurfaceTexture; | 30 import android.graphics.SurfaceTexture; |
31 import android.opengl.GLES11Ext; | 31 import android.opengl.GLES11Ext; |
32 import android.opengl.GLES20; | 32 import android.opengl.GLES20; |
33 import android.os.Build; | 33 import android.os.Build; |
34 import android.os.Handler; | 34 import android.os.Handler; |
35 import android.os.HandlerThread; | 35 import android.os.HandlerThread; |
36 import android.os.SystemClock; | 36 import android.os.SystemClock; |
37 | 37 |
| 38 import java.nio.ByteBuffer; |
| 39 import java.nio.FloatBuffer; |
38 import java.util.concurrent.Callable; | 40 import java.util.concurrent.Callable; |
39 import java.util.concurrent.CountDownLatch; | 41 import java.util.concurrent.CountDownLatch; |
40 import java.util.concurrent.TimeUnit; | 42 import java.util.concurrent.TimeUnit; |
41 | 43 |
42 /** | 44 /** |
43 * Helper class to create and synchronize access to a SurfaceTexture. The caller
will get notified | 45 * Helper class to create and synchronize access to a SurfaceTexture. The caller
will get notified |
44 * of new frames in onTextureFrameAvailable(), and should call returnTextureFram
e() when done with | 46 * of new frames in onTextureFrameAvailable(), and should call returnTextureFram
e() when done with |
45 * the frame. Only one texture frame can be in flight at once, so returnTextureF
rame() must be | 47 * the frame. Only one texture frame can be in flight at once, so returnTextureF
rame() must be |
46 * called in order to receive a new frame. Call disconnect() to stop receiveing
new frames and | 48 * called in order to receive a new frame. Call disconnect() to stop receiveing
new frames and |
47 * release all resources. | 49 * release all resources. |
(...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
86 // http://grepcode.com/file/repository.grepcode.com/java/ext/com.google.andr
oid/android/5.1.1_r1/android/graphics/SurfaceTexture.java#195. | 88 // http://grepcode.com/file/repository.grepcode.com/java/ext/com.google.andr
oid/android/5.1.1_r1/android/graphics/SurfaceTexture.java#195. |
87 // Therefore, in order to control the callback thread on API lvl < 21, the S
urfaceTextureHelper | 89 // Therefore, in order to control the callback thread on API lvl < 21, the S
urfaceTextureHelper |
88 // is constructed on the |handler| thread. | 90 // is constructed on the |handler| thread. |
89 return ThreadUtils.invokeUninterruptibly(finalHandler, new Callable<SurfaceT
extureHelper>() { | 91 return ThreadUtils.invokeUninterruptibly(finalHandler, new Callable<SurfaceT
extureHelper>() { |
90 @Override public SurfaceTextureHelper call() { | 92 @Override public SurfaceTextureHelper call() { |
91 return new SurfaceTextureHelper(sharedContext, finalHandler, (handler ==
null)); | 93 return new SurfaceTextureHelper(sharedContext, finalHandler, (handler ==
null)); |
92 } | 94 } |
93 }); | 95 }); |
94 } | 96 } |
95 | 97 |
| 98 // State for YUV conversion, instantiated on demand. |
| 99 static private class YuvConverter { |
| 100 private final EglBase eglBase; |
| 101 private final GlShader shader; |
| 102 private boolean released = false; |
| 103 |
| 104 // Vertex coordinates in Normalized Device Coordinates, i.e. |
| 105 // (-1, -1) is bottom-left and (1, 1) is top-right. |
| 106 private static final FloatBuffer DEVICE_RECTANGLE = |
| 107 GlUtil.createFloatBuffer(new float[] { |
| 108 -1.0f, -1.0f, // Bottom left. |
| 109 1.0f, -1.0f, // Bottom right. |
| 110 -1.0f, 1.0f, // Top left. |
| 111 1.0f, 1.0f, // Top right. |
| 112 }); |
| 113 |
| 114 // Texture coordinates - (0, 0) is bottom-left and (1, 1) is top-right. |
| 115 private static final FloatBuffer TEXTURE_RECTANGLE = |
| 116 GlUtil.createFloatBuffer(new float[] { |
| 117 0.0f, 0.0f, // Bottom left. |
| 118 1.0f, 0.0f, // Bottom right. |
| 119 0.0f, 1.0f, // Top left. |
| 120 1.0f, 1.0f // Top right. |
| 121 }); |
| 122 |
| 123 private static final String VERTEX_SHADER = |
| 124 "varying vec2 interp_tc;\n" |
| 125 + "attribute vec4 in_pos;\n" |
| 126 + "attribute vec4 in_tc;\n" |
| 127 + "\n" |
| 128 + "uniform mat4 texMatrix;\n" |
| 129 + "\n" |
| 130 + "void main() {\n" |
| 131 + " gl_Position = in_pos;\n" |
| 132 + " interp_tc = (texMatrix * in_tc).xy;\n" |
| 133 + "}\n"; |
| 134 |
| 135 private static final String FRAGMENT_SHADER = |
| 136 "#extension GL_OES_EGL_image_external : require\n" |
| 137 + "precision mediump float;\n" |
| 138 + "varying vec2 interp_tc;\n" |
| 139 + "\n" |
| 140 + "uniform samplerExternalOES oesTex;\n" |
| 141 // Difference in texture coordinate corresponding to one |
| 142 // sub-pixel in the x direction. |
| 143 + "uniform vec2 xUnit;\n" |
| 144 // Color conversion coefficients, including constant term |
| 145 + "uniform vec4 coeffs;\n" |
| 146 + "\n" |
| 147 + "void main() {\n" |
| 148 // Since the alpha read from the texture is always 1, this could |
| 149 // be written as a mat4 x vec4 multiply. However, that seems to |
| 150 // give a worse framerate, possibly because the additional |
| 151 // multiplies by 1.0 consume resources. TODO(nisse): Could also |
| 152 // try to do it as a vec3 x mat3x4, followed by an add in of a |
| 153 // constant vector. |
| 154 + " gl_FragColor.r = coeffs.a + dot(coeffs.rgb,\n" |
| 155 + " texture2D(oesTex, interp_tc - 1.5 * xUnit).rgb);\n" |
| 156 + " gl_FragColor.g = coeffs.a + dot(coeffs.rgb,\n" |
| 157 + " texture2D(oesTex, interp_tc - 0.5 * xUnit).rgb);\n" |
| 158 + " gl_FragColor.b = coeffs.a + dot(coeffs.rgb,\n" |
| 159 + " texture2D(oesTex, interp_tc + 0.5 * xUnit).rgb);\n" |
| 160 + " gl_FragColor.a = coeffs.a + dot(coeffs.rgb,\n" |
| 161 + " texture2D(oesTex, interp_tc + 1.5 * xUnit).rgb);\n" |
| 162 + "}\n"; |
| 163 |
| 164 private int texMatrixLoc; |
| 165 private int xUnitLoc; |
| 166 private int coeffsLoc;; |
| 167 |
| 168 YuvConverter (EglBase.Context sharedContext) { |
| 169 eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_RGBA_BUFFER); |
| 170 eglBase.createDummyPbufferSurface(); |
| 171 eglBase.makeCurrent(); |
| 172 |
| 173 shader = new GlShader(VERTEX_SHADER, FRAGMENT_SHADER); |
| 174 shader.useProgram(); |
| 175 texMatrixLoc = shader.getUniformLocation("texMatrix"); |
| 176 xUnitLoc = shader.getUniformLocation("xUnit"); |
| 177 coeffsLoc = shader.getUniformLocation("coeffs"); |
| 178 GLES20.glUniform1i(shader.getUniformLocation("oesTex"), 0); |
| 179 GlUtil.checkNoGLES2Error("Initialize fragment shader uniform values."); |
| 180 // Initialize vertex shader attributes. |
| 181 shader.setVertexAttribArray("in_pos", 2, DEVICE_RECTANGLE); |
| 182 // If the width is not a multiple of 4 pixels, the texture |
| 183 // will be scaled up slightly and clipped at the right border. |
| 184 shader.setVertexAttribArray("in_tc", 2, TEXTURE_RECTANGLE); |
| 185 eglBase.detachCurrent(); |
| 186 } |
| 187 |
| 188 synchronized void convert(ByteBuffer buf, |
| 189 int width, int height, int stride, int textureId, float [] transformMatr
ix) { |
| 190 if (released) { |
| 191 throw new IllegalStateException( |
| 192 "YuvConverter.convert called on released object"); |
| 193 } |
| 194 |
| 195 // We draw into a buffer laid out like |
| 196 // |
| 197 // +---------+ |
| 198 // | | |
| 199 // | Y | |
| 200 // | | |
| 201 // | | |
| 202 // +----+----+ |
| 203 // | U | V | |
| 204 // | | | |
| 205 // +----+----+ |
| 206 // |
| 207 // In memory, we use the same stride for all of Y, U and V. The |
| 208 // U data starts at offset |height| * |stride| from the Y data, |
| 209 // and the V data starts at at offset |stride/2| from the U |
| 210 // data, with rows of U and V data alternating. |
| 211 // |
| 212 // Now, it would have made sense to allocate a pixel buffer with |
| 213 // a single byte per pixel (EGL10.EGL_COLOR_BUFFER_TYPE, |
| 214 // EGL10.EGL_LUMINANCE_BUFFER,), but that seems to be |
| 215 // unsupported by devices. So do the following hack: Allocate an |
| 216 // RGBA buffer, of width |stride|/4. To render each of these |
| 217 // large pixels, sample the texture at 4 different x coordinates |
| 218 // and store the results in the four components. |
| 219 // |
| 220 // Since the V data needs to start on a boundary of such a |
| 221 // larger pixel, it is not sufficient that |stride| is even, it |
| 222 // has to be a multiple of 8 pixels. |
| 223 |
| 224 if (stride % 8 != 0) { |
| 225 throw new IllegalArgumentException( |
| 226 "Invalid stride, must be a multiple of 8"); |
| 227 } |
| 228 if (stride < width){ |
| 229 throw new IllegalArgumentException( |
| 230 "Invalid stride, must >= width"); |
| 231 } |
| 232 |
| 233 int y_width = (width+3) / 4; |
| 234 int uv_width = (width+7) / 8; |
| 235 int uv_height = (height+1)/2; |
| 236 int total_height = height + uv_height; |
| 237 int size = stride * total_height; |
| 238 |
| 239 if (buf.capacity() < size) { |
| 240 throw new IllegalArgumentException("YuvConverter.convert called with too
small buffer"); |
| 241 } |
| 242 // Produce a frame buffer starting at top-left corner, not |
| 243 // bottom-left. |
| 244 transformMatrix = |
| 245 RendererCommon.multiplyMatrices(transformMatrix, |
| 246 RendererCommon.verticalFlipMatrix()); |
| 247 |
| 248 // Create new pBuffferSurface with the correct size if needed. |
| 249 if (eglBase.hasSurface()) { |
| 250 if (eglBase.surfaceWidth() != stride/4 || |
| 251 eglBase.surfaceHeight() != total_height){ |
| 252 eglBase.releaseSurface(); |
| 253 eglBase.createPbufferSurface(stride/4, total_height); |
| 254 } |
| 255 } else { |
| 256 eglBase.createPbufferSurface(stride/4, total_height); |
| 257 } |
| 258 |
| 259 eglBase.makeCurrent(); |
| 260 |
| 261 GLES20.glActiveTexture(GLES20.GL_TEXTURE0); |
| 262 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId); |
| 263 GLES20.glUniformMatrix4fv(texMatrixLoc, 1, false, transformMatrix, 0); |
| 264 |
| 265 // Draw Y |
| 266 GLES20.glViewport(0, 0, y_width, height); |
| 267 // Matrix * (1;0;0;0) / width. Note that opengl uses column major order. |
| 268 GLES20.glUniform2f(xUnitLoc, |
| 269 transformMatrix[0] / width, |
| 270 transformMatrix[1] / width); |
| 271 // Y'UV444 to RGB888, see |
| 272 // https://en.wikipedia.org/wiki/YUV#Y.27UV444_to_RGB888_conversion. |
| 273 // We use the ITU-R coefficients for U and V */ |
| 274 GLES20.glUniform4f(coeffsLoc, 0.299f, 0.587f, 0.114f, 0.0f); |
| 275 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); |
| 276 |
| 277 // Draw U |
| 278 GLES20.glViewport(0, height, uv_width, uv_height); |
| 279 // Matrix * (1;0;0;0) / (2*width). Note that opengl uses column major orde
r. |
| 280 GLES20.glUniform2f(xUnitLoc, |
| 281 transformMatrix[0] / (2.0f*width), |
| 282 transformMatrix[1] / (2.0f*width)); |
| 283 GLES20.glUniform4f(coeffsLoc, -0.169f, -0.331f, 0.499f, 0.5f); |
| 284 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); |
| 285 |
| 286 // Draw V |
| 287 GLES20.glViewport(stride/8, height, uv_width, uv_height); |
| 288 GLES20.glUniform4f(coeffsLoc, 0.499f, -0.418f, -0.0813f, 0.5f); |
| 289 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); |
| 290 |
| 291 GLES20.glReadPixels(0, 0, stride/4, total_height, GLES20.GL_RGBA, |
| 292 GLES20.GL_UNSIGNED_BYTE, buf); |
| 293 |
| 294 GlUtil.checkNoGLES2Error("YuvConverter.convert"); |
| 295 |
| 296 // Unbind texture. Reportedly needed on some devices to get |
| 297 // the texture updated from the camera. |
| 298 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0); |
| 299 eglBase.detachCurrent(); |
| 300 } |
| 301 |
| 302 synchronized void release() { |
| 303 released = true; |
| 304 eglBase.makeCurrent(); |
| 305 shader.release(); |
| 306 eglBase.release(); |
| 307 } |
| 308 } |
| 309 |
96 private final Handler handler; | 310 private final Handler handler; |
97 private boolean isOwningThread; | 311 private boolean isOwningThread; |
98 private final EglBase eglBase; | 312 private final EglBase eglBase; |
99 private final SurfaceTexture surfaceTexture; | 313 private final SurfaceTexture surfaceTexture; |
100 private final int oesTextureId; | 314 private final int oesTextureId; |
| 315 private YuvConverter yuvConverter; |
| 316 |
101 private OnTextureFrameAvailableListener listener; | 317 private OnTextureFrameAvailableListener listener; |
102 // The possible states of this class. | 318 // The possible states of this class. |
103 private boolean hasPendingTexture = false; | 319 private boolean hasPendingTexture = false; |
104 private boolean isTextureInUse = false; | 320 private boolean isTextureInUse = false; |
105 private boolean isQuitting = false; | 321 private boolean isQuitting = false; |
106 | 322 |
107 private SurfaceTextureHelper(EglBase.Context sharedContext, | 323 private SurfaceTextureHelper(EglBase.Context sharedContext, |
108 Handler handler, boolean isOwningThread) { | 324 Handler handler, boolean isOwningThread) { |
109 if (handler.getLooper().getThread() != Thread.currentThread()) { | 325 if (handler.getLooper().getThread() != Thread.currentThread()) { |
110 throw new IllegalStateException("SurfaceTextureHelper must be created on t
he handler thread"); | 326 throw new IllegalStateException("SurfaceTextureHelper must be created on t
he handler thread"); |
111 } | 327 } |
112 this.handler = handler; | 328 this.handler = handler; |
113 this.isOwningThread = isOwningThread; | 329 this.isOwningThread = isOwningThread; |
114 | 330 |
115 eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_BUFFER); | 331 eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_BUFFER); |
116 eglBase.createDummyPbufferSurface(); | 332 eglBase.createDummyPbufferSurface(); |
117 eglBase.makeCurrent(); | 333 eglBase.makeCurrent(); |
118 | 334 |
119 oesTextureId = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); | 335 oesTextureId = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); |
120 surfaceTexture = new SurfaceTexture(oesTextureId); | 336 surfaceTexture = new SurfaceTexture(oesTextureId); |
121 } | 337 } |
122 | 338 |
| 339 private YuvConverter getYuvConverter() { |
| 340 // yuvConverter is assigned once |
| 341 if (yuvConverter != null) |
| 342 return yuvConverter; |
| 343 |
| 344 synchronized(this) { |
| 345 if (yuvConverter == null) |
| 346 yuvConverter = new YuvConverter(eglBase.getEglBaseContext()); |
| 347 return yuvConverter; |
| 348 } |
| 349 } |
| 350 |
123 /** | 351 /** |
124 * Start to stream textures to the given |listener|. | 352 * Start to stream textures to the given |listener|. |
125 * A Listener can only be set once. | 353 * A Listener can only be set once. |
126 */ | 354 */ |
127 public void setListener(OnTextureFrameAvailableListener listener) { | 355 public void setListener(OnTextureFrameAvailableListener listener) { |
128 if (this.listener != null) { | 356 if (this.listener != null) { |
129 throw new IllegalStateException("SurfaceTextureHelper listener has already
been set."); | 357 throw new IllegalStateException("SurfaceTextureHelper listener has already
been set."); |
130 } | 358 } |
131 this.listener = listener; | 359 this.listener = listener; |
132 surfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailab
leListener() { | 360 surfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailab
leListener() { |
(...skipping 67 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
200 * onTextureFrameAvailable() after this function returns. | 428 * onTextureFrameAvailable() after this function returns. |
201 */ | 429 */ |
202 public void disconnect(Handler handler) { | 430 public void disconnect(Handler handler) { |
203 if (this.handler != handler) { | 431 if (this.handler != handler) { |
204 throw new IllegalStateException("Wrong handler."); | 432 throw new IllegalStateException("Wrong handler."); |
205 } | 433 } |
206 isOwningThread = true; | 434 isOwningThread = true; |
207 disconnect(); | 435 disconnect(); |
208 } | 436 } |
209 | 437 |
| 438 public void textureToYUV(ByteBuffer buf, |
| 439 int width, int height, int stride, int textureId, float [] transformMatrix
) { |
| 440 if (textureId != oesTextureId) |
| 441 throw new IllegalStateException("textureToByteBuffer called with unexpecte
d textureId"); |
| 442 |
| 443 getYuvConverter().convert(buf, width, height, stride, textureId, transformMa
trix); |
| 444 } |
| 445 |
210 private void tryDeliverTextureFrame() { | 446 private void tryDeliverTextureFrame() { |
211 if (handler.getLooper().getThread() != Thread.currentThread()) { | 447 if (handler.getLooper().getThread() != Thread.currentThread()) { |
212 throw new IllegalStateException("Wrong thread."); | 448 throw new IllegalStateException("Wrong thread."); |
213 } | 449 } |
214 if (isQuitting || !hasPendingTexture || isTextureInUse) { | 450 if (isQuitting || !hasPendingTexture || isTextureInUse) { |
215 return; | 451 return; |
216 } | 452 } |
217 isTextureInUse = true; | 453 isTextureInUse = true; |
218 hasPendingTexture = false; | 454 hasPendingTexture = false; |
219 | 455 |
220 eglBase.makeCurrent(); | 456 eglBase.makeCurrent(); |
221 surfaceTexture.updateTexImage(); | 457 surfaceTexture.updateTexImage(); |
222 | 458 |
223 final float[] transformMatrix = new float[16]; | 459 final float[] transformMatrix = new float[16]; |
224 surfaceTexture.getTransformMatrix(transformMatrix); | 460 surfaceTexture.getTransformMatrix(transformMatrix); |
225 final long timestampNs = (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_C
REAM_SANDWICH) | 461 final long timestampNs = (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_C
REAM_SANDWICH) |
226 ? surfaceTexture.getTimestamp() | 462 ? surfaceTexture.getTimestamp() |
227 : TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime()); | 463 : TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime()); |
228 listener.onTextureFrameAvailable(oesTextureId, transformMatrix, timestampNs)
; | 464 listener.onTextureFrameAvailable(oesTextureId, transformMatrix, timestampNs)
; |
229 } | 465 } |
230 | 466 |
231 private void release() { | 467 private void release() { |
232 if (handler.getLooper().getThread() != Thread.currentThread()) { | 468 if (handler.getLooper().getThread() != Thread.currentThread()) { |
233 throw new IllegalStateException("Wrong thread."); | 469 throw new IllegalStateException("Wrong thread."); |
234 } | 470 } |
235 if (isTextureInUse || !isQuitting) { | 471 if (isTextureInUse || !isQuitting) { |
236 throw new IllegalStateException("Unexpected release."); | 472 throw new IllegalStateException("Unexpected release."); |
237 } | 473 } |
| 474 synchronized (this) { |
| 475 if (yuvConverter != null) |
| 476 yuvConverter.release(); |
| 477 } |
238 eglBase.makeCurrent(); | 478 eglBase.makeCurrent(); |
239 GLES20.glDeleteTextures(1, new int[] {oesTextureId}, 0); | 479 GLES20.glDeleteTextures(1, new int[] {oesTextureId}, 0); |
240 surfaceTexture.release(); | 480 surfaceTexture.release(); |
241 eglBase.release(); | 481 eglBase.release(); |
242 handler.getLooper().quit(); | 482 handler.getLooper().quit(); |
243 } | 483 } |
244 } | 484 } |
OLD | NEW |