OLD | NEW |
(Empty) | |
| 1 /* |
| 2 * libjingle |
| 3 * Copyright 2015 Google Inc. |
| 4 * |
| 5 * Redistribution and use in source and binary forms, with or without |
| 6 * modification, are permitted provided that the following conditions are met: |
| 7 * |
| 8 * 1. Redistributions of source code must retain the above copyright notice, |
| 9 * this list of conditions and the following disclaimer. |
| 10 * 2. Redistributions in binary form must reproduce the above copyright notice, |
| 11 * this list of conditions and the following disclaimer in the documentation |
| 12 * and/or other materials provided with the distribution. |
| 13 * 3. The name of the author may not be used to endorse or promote products |
| 14 * derived from this software without specific prior written permission. |
| 15 * |
| 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED |
| 17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF |
| 18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO |
| 19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; |
| 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, |
| 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR |
| 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
| 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 26 */ |
| 27 |
| 28 package org.webrtc; |
| 29 |
| 30 import java.nio.ByteBuffer; |
| 31 |
| 32 import android.content.Context; |
| 33 import android.graphics.Point; |
| 34 import android.graphics.SurfaceTexture; |
| 35 import android.opengl.EGLContext; |
| 36 import android.opengl.GLES20; |
| 37 import android.os.Handler; |
| 38 import android.os.HandlerThread; |
| 39 import android.util.AttributeSet; |
| 40 import android.util.Log; |
| 41 import android.view.SurfaceHolder; |
| 42 import android.view.SurfaceView; |
| 43 |
| 44 /** |
| 45 * Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream
on a SurfaceView. |
| 46 * renderFrame() is asynchronous to avoid blocking the calling thread. Instead,
a shallow copy of |
| 47 * the frame is posted to a dedicated render thread. |
| 48 * This class is thread safe and handles access from potentially four different
threads: |
| 49 * Interaction from the main app in init, release, setMirror, and setScalingtype
. |
| 50 * Interaction from C++ webrtc::VideoRendererInterface in renderFrame and canApp
lyRotation. |
| 51 * Interaction from the Activity lifecycle in surfaceCreated, surfaceChanged, an
d surfaceDestroyed. |
| 52 * Interaction with the layout framework in onMeasure and onSizeChanged. |
| 53 */ |
| 54 public class SurfaceViewRenderer extends SurfaceView |
| 55 implements SurfaceHolder.Callback, VideoRenderer.Callbacks { |
| 56 private static final String TAG = "SurfaceViewRenderer"; |
| 57 |
| 58 // Dedicated render thread. Synchronized on |this|. |
| 59 private HandlerThread renderThread; |
| 60 // Handler for inter-thread communication. Synchronized on |this|. |
| 61 private Handler renderThreadHandler; |
| 62 // Pending frame to render. Serves as a queue with size 1. Synchronized on |th
is|. |
| 63 private VideoRenderer.I420Frame pendingFrame; |
| 64 |
| 65 // EGL and GL resources for drawing YUV/OES textures. After initilization, the
se are only accessed |
| 66 // from the render thread. |
| 67 private EglBase eglBase; |
| 68 private GlRectDrawer drawer; |
| 69 // Texture ids for YUV frames. Allocated on first arrival of a YUV frame. |
| 70 private int[] yuvTextures = null; |
| 71 // Intermediate copy buffers in case yuv frames are not packed, i.e. stride >
plane width. One for |
| 72 // Y, and one for U and V. |
| 73 private final ByteBuffer[] copyBuffer = new ByteBuffer[2]; |
| 74 |
| 75 // These variables are synchronized on |layoutLock|. |
| 76 private final Object layoutLock = new Object(); |
| 77 // Current surface size. |
| 78 public int surfaceWidth; |
| 79 public int surfaceHeight; |
| 80 // Most recent measurement specification from onMeasure(). |
| 81 private int widthSpec; |
| 82 private int heightSpec; |
| 83 // Current size on screen in pixels. |
| 84 public int layoutWidth; |
| 85 public int layoutHeight; |
| 86 // Desired layout size, or 0 if no frame has arrived yet. The desired size is
updated before |
| 87 // rendering a new frame, and is enforced in onMeasure(). Rendering is blocked
until layout is |
| 88 // updated to the desired size. |
| 89 public int desiredLayoutWidth; |
| 90 public int desiredLayoutHeight; |
| 91 // |scalingType| determines how the video will fill the allowed layout area in
onMeasure(). |
| 92 private RendererCommon.ScalingType scalingType = RendererCommon.ScalingType.SC
ALE_ASPECT_BALANCED; |
| 93 // If true, mirrors the video stream horizontally. |
| 94 private boolean mirror; |
| 95 |
| 96 // These variables are synchronized on |statisticsLock|. |
| 97 private final Object statisticsLock = new Object(); |
| 98 // Total number of video frames received in renderFrame() call. |
| 99 private int framesReceived; |
| 100 // Number of video frames dropped by renderFrame() because previous frame has
not been rendered |
| 101 // yet. |
| 102 private int framesDropped; |
| 103 // Number of rendered video frames. |
| 104 private int framesRendered; |
| 105 // Time in ns when the first video frame was rendered. |
| 106 private long firstFrameTimeNs; |
| 107 // Time in ns spent in renderFrameOnRenderThread() function. |
| 108 private long renderTimeNs; |
| 109 |
| 110 // Runnable for posting frames to render thread.. |
| 111 private final Runnable renderFrameRunnable = new Runnable() { |
| 112 @Override public void run() { |
| 113 renderFrameOnRenderThread(); |
| 114 } |
| 115 }; |
| 116 |
| 117 /** |
| 118 * Standard View constructor. In order to render something, you must first cal
l init(). |
| 119 */ |
| 120 public SurfaceViewRenderer(Context context) { |
| 121 super(context); |
| 122 } |
| 123 |
| 124 /** |
| 125 * Standard View constructor. In order to render something, you must first cal
l init(). |
| 126 */ |
| 127 public SurfaceViewRenderer(Context context, AttributeSet attrs) { |
| 128 super(context, attrs); |
| 129 } |
| 130 |
| 131 /** |
| 132 * Initialize this class, sharing resources with |sharedContext|. |
| 133 */ |
| 134 public synchronized void init(EGLContext sharedContext) { |
| 135 if (renderThreadHandler != null) { |
| 136 throw new IllegalStateException("Already initialized"); |
| 137 } |
| 138 Log.d(TAG, "Initializing"); |
| 139 renderThread = new HandlerThread(TAG); |
| 140 renderThread.start(); |
| 141 renderThreadHandler = new Handler(renderThread.getLooper()); |
| 142 eglBase = new EglBase(sharedContext, EglBase.ConfigType.PLAIN); |
| 143 drawer = new GlRectDrawer(); |
| 144 getHolder().addCallback(this); |
| 145 } |
| 146 |
| 147 /** |
| 148 * Release all resources. This needs to be done manually, otherwise the resour
ces are leaked. |
| 149 */ |
| 150 public synchronized void release() { |
| 151 if (renderThreadHandler == null) { |
| 152 Log.d(TAG, "Already released"); |
| 153 return; |
| 154 } |
| 155 // Release EGL and GL resources on render thread. |
| 156 renderThreadHandler.post(new Runnable() { |
| 157 @Override public void run() { |
| 158 drawer.release(); |
| 159 drawer = null; |
| 160 if (yuvTextures != null) { |
| 161 GLES20.glDeleteTextures(3, yuvTextures, 0); |
| 162 yuvTextures = null; |
| 163 } |
| 164 eglBase.release(); |
| 165 eglBase = null; |
| 166 } |
| 167 }); |
| 168 // Don't accept any more messages to the render thread. |
| 169 renderThreadHandler = null; |
| 170 // Quit safely to make sure the EGL/GL cleanup posted above is executed. |
| 171 renderThread.quitSafely(); |
| 172 renderThread = null; |
| 173 |
| 174 getHolder().removeCallback(this); |
| 175 if (pendingFrame != null) { |
| 176 VideoRenderer.renderFrameDone(pendingFrame); |
| 177 pendingFrame = null; |
| 178 } |
| 179 } |
| 180 |
| 181 /** |
| 182 * Set if the video stream should be mirrored or not. |
| 183 */ |
| 184 public void setMirror(final boolean mirror) { |
| 185 synchronized (layoutLock) { |
| 186 this.mirror = mirror; |
| 187 } |
| 188 } |
| 189 |
| 190 /** |
| 191 * Set how the video will fill the allowed layout area. |
| 192 */ |
| 193 public void setScalingType(RendererCommon.ScalingType scalingType) { |
| 194 synchronized (layoutLock) { |
| 195 this.scalingType = scalingType; |
| 196 } |
| 197 } |
| 198 |
| 199 // VideoRenderer.Callbacks interface. |
| 200 @Override |
| 201 public void renderFrame(VideoRenderer.I420Frame frame) { |
| 202 synchronized (statisticsLock) { |
| 203 ++framesReceived; |
| 204 } |
| 205 synchronized (this) { |
| 206 if (renderThreadHandler == null) { |
| 207 Log.d(TAG, "Dropping frame - SurfaceViewRenderer not initialized or alre
ady released."); |
| 208 VideoRenderer.renderFrameDone(frame); |
| 209 return; |
| 210 } |
| 211 if (pendingFrame != null) { |
| 212 synchronized (statisticsLock) { |
| 213 ++framesDropped; |
| 214 } |
| 215 Log.d(TAG, "Dropping frame - previous frame has not been rendered yet.")
; |
| 216 VideoRenderer.renderFrameDone(frame); |
| 217 return; |
| 218 } |
| 219 pendingFrame = frame; |
| 220 renderThreadHandler.post(renderFrameRunnable); |
| 221 } |
| 222 } |
| 223 |
| 224 @Override |
| 225 public boolean canApplyRotation() { |
| 226 return true; |
| 227 } |
| 228 |
| 229 // View layout interface. |
| 230 @Override |
| 231 protected void onMeasure(int widthSpec, int heightSpec) { |
| 232 synchronized (layoutLock) { |
| 233 this.widthSpec = widthSpec; |
| 234 this.heightSpec = heightSpec; |
| 235 if (desiredLayoutWidth == 0 || desiredLayoutHeight == 0) { |
| 236 super.onMeasure(widthSpec, heightSpec); |
| 237 } else { |
| 238 setMeasuredDimension(desiredLayoutWidth, desiredLayoutHeight); |
| 239 } |
| 240 } |
| 241 } |
| 242 |
| 243 @Override |
| 244 protected void onLayout(boolean changed, int left, int top, int right, int bot
tom) { |
| 245 synchronized (layoutLock) { |
| 246 layoutWidth = right - left; |
| 247 layoutHeight = bottom - top; |
| 248 } |
| 249 // Might have a pending frame waiting for a layout of correct size. |
| 250 runOnRenderThread(renderFrameRunnable); |
| 251 } |
| 252 |
| 253 // SurfaceHolder.Callback interface. |
| 254 @Override |
| 255 public void surfaceCreated(final SurfaceHolder holder) { |
| 256 Log.d(TAG, "Surface created"); |
| 257 runOnRenderThread(new Runnable() { |
| 258 @Override public void run() { |
| 259 eglBase.createSurface(holder.getSurface()); |
| 260 eglBase.makeCurrent(); |
| 261 // Necessary for YUV frames with odd width. |
| 262 GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); |
| 263 } |
| 264 }); |
| 265 } |
| 266 |
| 267 @Override |
| 268 public void surfaceDestroyed(SurfaceHolder holder) { |
| 269 Log.d(TAG, "Surface destroyed"); |
| 270 synchronized (layoutLock) { |
| 271 surfaceWidth = 0; |
| 272 surfaceHeight = 0; |
| 273 } |
| 274 runOnRenderThread(new Runnable() { |
| 275 @Override public void run() { |
| 276 eglBase.releaseSurface(); |
| 277 } |
| 278 }); |
| 279 } |
| 280 |
| 281 @Override |
| 282 public void surfaceChanged(SurfaceHolder holder, int format, int width, int he
ight) { |
| 283 Log.d(TAG, "Surface changed: " + width + "x" + height); |
| 284 synchronized (layoutLock) { |
| 285 surfaceWidth = width; |
| 286 surfaceHeight = height; |
| 287 } |
| 288 // Might have a pending frame waiting for a surface of correct size. |
| 289 runOnRenderThread(renderFrameRunnable); |
| 290 } |
| 291 |
| 292 /** |
| 293 * Private helper function to post tasks safely. |
| 294 */ |
| 295 private synchronized void runOnRenderThread(Runnable runnable) { |
| 296 if (renderThreadHandler != null) { |
| 297 renderThreadHandler.post(runnable); |
| 298 } |
| 299 } |
| 300 |
| 301 private synchronized void runOnRenderThreadDelayed(Runnable runnable, long ms)
{ |
| 302 if (renderThreadHandler != null) { |
| 303 renderThreadHandler.postDelayed(runnable, ms); |
| 304 } |
| 305 } |
| 306 |
| 307 /** |
| 308 * Renders and releases |pendingFrame|. |
| 309 */ |
| 310 private void renderFrameOnRenderThread() { |
| 311 if (eglBase == null || !eglBase.hasSurface()) { |
| 312 Log.d(TAG, "No surface to draw on"); |
| 313 return; |
| 314 } |
| 315 final float videoAspectRatio; |
| 316 synchronized (this) { |
| 317 if (pendingFrame == null) { |
| 318 return; |
| 319 } |
| 320 videoAspectRatio = (float) pendingFrame.rotatedWidth() / pendingFrame.rota
tedHeight(); |
| 321 } |
| 322 // Request new layout if necessary. Don't continue until layout and surface
size are in a good |
| 323 // state. |
| 324 synchronized (layoutLock) { |
| 325 final int maxWidth = getDefaultSize(Integer.MAX_VALUE, widthSpec); |
| 326 final int maxHeight = getDefaultSize(Integer.MAX_VALUE, heightSpec); |
| 327 final Point suggestedSize = |
| 328 RendererCommon.getDisplaySize(scalingType, videoAspectRatio, maxWidth,
maxHeight); |
| 329 desiredLayoutWidth = |
| 330 MeasureSpec.getMode(widthSpec) == MeasureSpec.EXACTLY ? maxWidth : sug
gestedSize.x; |
| 331 desiredLayoutHeight = |
| 332 MeasureSpec.getMode(heightSpec) == MeasureSpec.EXACTLY ? maxHeight : s
uggestedSize.y; |
| 333 if (desiredLayoutWidth != layoutWidth || desiredLayoutHeight != layoutHeig
ht) { |
| 334 Log.d(TAG, "Requesting new layout with size: " |
| 335 + desiredLayoutWidth + "x" + desiredLayoutHeight); |
| 336 // Output an intermediate black frame while the layout is updated. |
| 337 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); |
| 338 eglBase.swapBuffers(); |
| 339 // Request layout update on UI thread. |
| 340 post(new Runnable() { |
| 341 @Override public void run() { |
| 342 requestLayout(); |
| 343 } |
| 344 }); |
| 345 return; |
| 346 } |
| 347 if (surfaceWidth != layoutWidth || surfaceHeight != layoutHeight) { |
| 348 Log.d(TAG, "Postponing rendering until surface size is updated."); |
| 349 return; |
| 350 } |
| 351 } |
| 352 |
| 353 // The EGLSurface might have a buffer of the old size in the pipeline, even
after |
| 354 // surfaceChanged() has been called. Querying the EGLSurface will show if th
e underlying buffer |
| 355 // dimensions haven't yet changed. |
| 356 if (eglBase.surfaceWidth() != surfaceWidth || eglBase.surfaceHeight() != sur
faceHeight) { |
| 357 Log.d(TAG, "Flushing old egl surface buffer with incorrect size."); |
| 358 // There is no way to display the old buffer correctly, so just make it bl
ack, and immediately |
| 359 // render |pendingFrame| on the next buffer. |
| 360 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); |
| 361 eglBase.swapBuffers(); |
| 362 // In some rare cases, the next buffer is not updated either. In those cas
es, wait 1 ms and |
| 363 // try again. |
| 364 if (eglBase.surfaceWidth() != surfaceWidth || eglBase.surfaceHeight() != s
urfaceHeight) { |
| 365 Log.e(TAG, "Unexpected buffer size even after swapBuffers() has been cal
led."); |
| 366 runOnRenderThreadDelayed(renderFrameRunnable, 1); |
| 367 return; |
| 368 } |
| 369 } |
| 370 |
| 371 // Finally, layout, surface, and EGLSurface are in a good state. Fetch and r
ender pendingFrame|. |
| 372 final VideoRenderer.I420Frame frame; |
| 373 synchronized (this) { |
| 374 if (pendingFrame == null) { |
| 375 return; |
| 376 } |
| 377 frame = pendingFrame; |
| 378 pendingFrame = null; |
| 379 } |
| 380 |
| 381 final long startTimeNs = System.nanoTime(); |
| 382 final float[] texMatrix = new float[16]; |
| 383 synchronized (layoutLock) { |
| 384 RendererCommon.getTextureMatrix(texMatrix, frame.rotationDegree, mirror, v
ideoAspectRatio, |
| 385 (float) layoutWidth / layoutHeight); |
| 386 } |
| 387 |
| 388 GLES20.glViewport(0, 0, surfaceWidth, surfaceHeight); |
| 389 if (frame.yuvFrame) { |
| 390 uploadYuvData(frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes
); |
| 391 drawer.drawYuv(frame.width, frame.height, yuvTextures, texMatrix); |
| 392 } else { |
| 393 SurfaceTexture surfaceTexture = (SurfaceTexture) frame.textureObject; |
| 394 // TODO(magjed): Move updateTexImage() to the video source instead. |
| 395 surfaceTexture.updateTexImage(); |
| 396 drawer.drawOes(frame.textureId, texMatrix); |
| 397 } |
| 398 |
| 399 eglBase.swapBuffers(); |
| 400 VideoRenderer.renderFrameDone(frame); |
| 401 synchronized (statisticsLock) { |
| 402 if (framesRendered == 0) { |
| 403 firstFrameTimeNs = startTimeNs; |
| 404 } |
| 405 ++framesRendered; |
| 406 renderTimeNs += (System.nanoTime() - startTimeNs); |
| 407 if (framesRendered % 300 == 0) { |
| 408 logStatistics(); |
| 409 } |
| 410 } |
| 411 } |
| 412 |
| 413 private void uploadYuvData(int width, int height, int[] strides, ByteBuffer[]
planes) { |
| 414 // Make sure YUV textures are allocated. |
| 415 if (yuvTextures == null) { |
| 416 yuvTextures = new int[3]; |
| 417 // Generate 3 texture ids for Y/U/V and place them into |yuvTextures|. |
| 418 GLES20.glGenTextures(3, yuvTextures, 0); |
| 419 for (int i = 0; i < 3; i++) { |
| 420 GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); |
| 421 GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); |
| 422 GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, |
| 423 GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST); |
| 424 GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, |
| 425 GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); |
| 426 GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, |
| 427 GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); |
| 428 GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, |
| 429 GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); |
| 430 } |
| 431 GlUtil.checkNoGLES2Error("y/u/v glGenTextures"); |
| 432 } |
| 433 // Upload each plane. |
| 434 for (int i = 0; i < 3; ++i) { |
| 435 GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); |
| 436 GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); |
| 437 final int planeWidth = (i == 0) ? width : width / 2; |
| 438 final int planeHeight = (i == 0) ? height : height / 2; |
| 439 final int bufferIndex = (i == 0) ? 0 : 1; |
| 440 // GLES only accepts packed data, i.e. stride == planeWidth. |
| 441 final ByteBuffer packedByteBuffer; |
| 442 if (strides[i] == planeWidth) { |
| 443 // Input is packed already. |
| 444 packedByteBuffer = planes[i]; |
| 445 } else { |
| 446 // Make an intermediate packed copy. |
| 447 final int capacityNeeded = planeWidth * planeHeight; |
| 448 if (copyBuffer[bufferIndex] == null |
| 449 || copyBuffer[bufferIndex].capacity() != capacityNeeded) { |
| 450 copyBuffer[bufferIndex] = ByteBuffer.allocateDirect(capacityNeeded); |
| 451 } |
| 452 packedByteBuffer = copyBuffer[bufferIndex]; |
| 453 VideoRenderer.nativeCopyPlane( |
| 454 planes[i], planeWidth, planeHeight, strides[i], packedByteBuffer, pl
aneWidth); |
| 455 } |
| 456 GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, planeWid
th, planeHeight, 0, |
| 457 GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, packedByteBuffer); |
| 458 } |
| 459 } |
| 460 |
| 461 private void logStatistics() { |
| 462 synchronized (statisticsLock) { |
| 463 Log.d(TAG, "ID: " + getResources().getResourceEntryName(getId()) + ". Fram
es received: " |
| 464 + framesReceived + ". Dropped: " + framesDropped + ". Rendered: " + fr
amesRendered); |
| 465 if (framesReceived > 0 && framesRendered > 0) { |
| 466 final long timeSinceFirstFrameNs = System.nanoTime() - firstFrameTimeNs; |
| 467 Log.d(TAG, "Duration: " + (int) (timeSinceFirstFrameNs / 1e6) + |
| 468 " ms. FPS: " + (float) framesRendered * 1e9 / timeSinceFirstFrameNs)
; |
| 469 Log.d(TAG, "Average render time: " |
| 470 + (int) (renderTimeNs / (1000 * framesRendered)) + " us."); |
| 471 } |
| 472 } |
| 473 } |
| 474 } |
OLD | NEW |