AppRTCDemo: Render each video in a separate SurfaceView

talk/app/webrtc/java/android/org/webrtc/SurfaceViewRenderer.java

+/*
+ * libjingle
+ * Copyright 2015 Google Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.webrtc;
+
+import android.content.Context;
+import android.graphics.Point;
+import android.graphics.SurfaceTexture;
+import android.opengl.EGLContext;
+import android.opengl.GLES20;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.util.AttributeSet;
+import android.util.Log;
+import android.view.SurfaceHolder;
+import android.view.SurfaceView;
+
+/**
+ * Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream on a SurfaceView.
+ * renderFrame() is asynchronous to avoid blocking the calling thread. Instead, a shallow copy of
+ * the frame is posted to a dedicated render thread.
+ * This class is thread safe and handles access from potentially four different threads:
+ * Interaction from the main app in init, release, setMirror, and setScalingtype.
+ * Interaction from C++ webrtc::VideoRendererInterface in renderFrame and canApplyRotation.
+ * Interaction from the Activity lifecycle in surfaceCreated, surfaceChanged, and surfaceDestroyed.
+ * Interaction with the layout framework in onMeasure and onSizeChanged.
+ */
+public class SurfaceViewRenderer extends SurfaceView
+    implements SurfaceHolder.Callback, VideoRenderer.Callbacks {
+  private static final String TAG = "SurfaceViewRenderer";
+
+  // Dedicated render thread. Synchronized on |this|.
+  private HandlerThread renderThread;
+  // Handler for inter-thread communication. Synchronized on |this|.
+  private Handler renderThreadHandler;
+  // Pending frame to render. Serves as a queue with size 1. Synchronized on |this|.
+  private VideoRenderer.I420Frame pendingFrame;
+
+  // EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed
+  // from the render thread.
+  private EglBase eglBase;
+  private GlRectDrawer drawer;
+  // Texture ids for YUV frames. Allocated on first arrival of a YUV frame.
+  private int[] yuvTextures = null;
+
+  // These variables are synchronized on |layoutLock|.
+  private final Object layoutLock = new Object();
+  // Aspect ratio on screen.
+  private float layoutAspectRatio;
+  // Aspect ratio of the most recent video frame.
+  private float videoAspectRatio;
+  // |scalingType| determines how the video will fill the allowed layout area in onMeasure().
+  private VideoRendererGui.ScalingType scalingType =
+      VideoRendererGui.ScalingType.SCALE_ASPECT_BALANCED;
+  // If true, mirrors the video stream horizontally.
+  private boolean mirror;
+
+  // These variables are synchronized on |statisticsLock|.
+  private final Object statisticsLock = new Object();
+  // Total number of video frames received in renderFrame() call.
+  private int framesReceived;
+  // Number of video frames dropped by renderFrame() because previous frame has not been rendered
+  // yet.
+  private int framesDropped;
+  // Number of rendered video frames.
+  private int framesRendered;
+  // Time in ns when the first video frame was rendered.
+  private long firstFrameTimeNs;
+  // Time in ns spent in renderFrameOnRenderThread() function.
+  private long renderTimeNs;
+
+  /**
+   * Standard View constructor. In order to render something, you must first call init().
+   */
+  public SurfaceViewRenderer(Context context) {
+    super(context);
+  }
+
+  /**
+   * Standard View constructor. In order to render something, you must first call init().
+   */
+  public SurfaceViewRenderer(Context context, AttributeSet attrs) {
+    super(context, attrs);
+  }
+
+  /**
+   * Initialize this class, sharing resources with |sharedContext|.
+   */
+  public synchronized void init(EGLContext sharedContext) {
+    if (renderThreadHandler != null) {
+      throw new IllegalStateException("Already initialized");
+    }
+    Log.d(TAG, "Initializing");
+    renderThread = new HandlerThread(TAG);
+    renderThread.start();
+    renderThreadHandler = new Handler(renderThread.getLooper());
+    eglBase = new EglBase(sharedContext, EglBase.ConfigType.PLAIN);
+    drawer = new GlRectDrawer();
+    getHolder().addCallback(this);
+  }
+
+  /**
+   * Release all resources. This needs to be done manually, otherwise the resources are leaked.
+   */
+  public synchronized void release() {
+    if (renderThreadHandler == null) {
+      Log.d(TAG, "Already released");
+      return;
+    }
+    // Release EGL and GL resources on render thread.
+    renderThreadHandler.post(new Runnable() {
+      @Override public void run() {
+        drawer.release();
+        drawer = null;
+        if (yuvTextures != null) {
+          GLES20.glDeleteTextures(3, yuvTextures, 0);
+          yuvTextures = null;
+        }
+        eglBase.release();
+        eglBase = null;
+      }
+    });
+    // Don't accept any more messages to the render thread.
+    renderThreadHandler = null;
+    renderThread.quitSafely();

[AlexG, 2015/08/04 00:27:23]

use quit() here to reduce exit latency - we don't need to wait for pending frame
rendering complete.

[magjed_webrtc, 2015/08/04 17:05:05]

I want to make sure that the EGL/GL cleanup posted above is executed. I'm
stopping renderFrame() from posting any more frames by setting
|renderThreadHandler| to null and then releasing |pendingFrame|. So in those
cases, renderFrameOnRenderThread() will return immediately and there won't be
any extra latency.

+    renderThread = null;
+
+    getHolder().removeCallback(this);
+    if (pendingFrame != null) {
+      pendingFrame.release();
+      pendingFrame = null;
+    }
+  }
+
+  /**
+   * Set if the video stream should be mirrored or not.
+   */
+  public void setMirror(final boolean mirror) {
+    synchronized (layoutLock) {
+      this.mirror = mirror;
+    }
+  }
+
+  /**
+   * Set how the video will fill the allowed layout area.
+   */
+  public void setScalingType(VideoRendererGui.ScalingType scalingType) {
+    synchronized (layoutLock) {
+      this.scalingType = scalingType;
+    }
+  }
+
+  // VideoRenderer.Callbacks interface.
+  @Override
+  public void renderFrame(VideoRenderer.I420Frame frame) {
+    synchronized (statisticsLock) {
+      ++framesReceived;
+    }
+    // Trigger layout update if video aspect ratio changed.
+    synchronized (layoutLock) {

[AlexG, 2015/08/04 00:27:23]

Do you think it's worth skip rendering of the frame until layout change is done?
I am worried about some corner cases when camera is running at low fps due to
high load or application is paused and the last frame is displayed when
application is in background - for these cases the last rendered may stick for
some time and may have incorrect layout.

[magjed_webrtc, 2015/08/04 17:05:05]

Do you mean dropping the frame or delay renderFrameOnRenderThread() until layout
change is done? One problem is that I don't know of any good way to get notified
when layout change is done. onSizeChanged() is not guaranteed to get called
after requestLayout(), so we could get stuck with a pending frame that will
never be rendered. onLayout() might get called several times during the layout
process, so I don't think it's a good place either. onDraw() is not called at
all for SurfaceViews.

Also, this requestLayout() is typically only triggered by an orientation change
so it's an uncommon corner case.

[AlexG, 2015/08/05 00:47:11]

I often seen the cases when for bad connection remote frame freezes for quite a
while. If user will either rotate the device or even swipe to get a system menu
the layout will be updated.
Will onSizeChanged() callback indicate that layout update is done? May be it is
possible to use this callback to block any frame from rendering until layout
update is fully completed. Yes, I think we should not redraw the view and block
rendering thread while pending layout update.

[magjed_webrtc, 2015/08/07 17:14:50]

Done. I block rendering while a layout update is pending. Layout size is
determined when a frame is about to be rendered, and onMeasure() will refuse to
stretch the current frame.

+      final float videoAspectRatio = (float) frame.rotatedWidth() / frame.rotatedHeight();
+      if (videoAspectRatio != this.videoAspectRatio) {
+        this.videoAspectRatio = videoAspectRatio;
+        // Need to request layout update from the UI thread.
+        post(new Runnable() {
+          @Override public void run() {
+            requestLayout();
+          }
+        });
+      }
+    }
+    synchronized (this) {
+      if (renderThreadHandler == null) {
+        Log.d(TAG, "Dropping frame - SurfaceViewRenderer not initialized or already released.");
+        return;
+      }
+      if (pendingFrame != null) {
+        synchronized (statisticsLock) {
+          ++framesDropped;
+        }
+        Log.d(TAG, "Dropping frame - previous frame has not been rendered yet.");
+        return;
+      }
+      pendingFrame = frame.shallowCopy();
+      renderThreadHandler.post(new Runnable() {
+        @Override public void run() {
+          renderFrameOnRenderThread();
+        }
+      });
+    }
+  }
+
+  @Override
+  public boolean canApplyRotation() {
+    return true;
+  }
+
+  // View layout interface.
+  @Override
+  protected void onMeasure(int widthSpec, int heightSpec) {
+    final int maxWidth = getDefaultSize(Integer.MAX_VALUE, widthSpec);
+    final int maxHeight = getDefaultSize(Integer.MAX_VALUE, heightSpec);
+    final Point suggestedSize;
+    synchronized (layoutLock) {
+      suggestedSize = VideoRendererGui.getDisplaySize(
+          VideoRendererGui.convertScalingTypeToVisibleFraction(scalingType),
+          videoAspectRatio, maxWidth, maxHeight);
+    }
+    setMeasuredDimension(
+        MeasureSpec.getMode(widthSpec) == MeasureSpec.EXACTLY ? maxWidth : suggestedSize.x,
+        MeasureSpec.getMode(heightSpec) == MeasureSpec.EXACTLY ? maxHeight : suggestedSize.y);
+  }
+
+  @Override
+  protected void onSizeChanged(int w, int h, int oldw, int oldh) {
+    synchronized (layoutLock) {
+      layoutAspectRatio = (float) w / h;
+    }
+  }
+
+  // SurfaceHolder.Callback interface.
+  @Override
+  public void surfaceCreated(final SurfaceHolder holder) {
+    Log.d(TAG, "Surface created");
+    runOnRenderThread(new Runnable() {
+      @Override public void run() {
+        eglBase.createSurface(holder.getSurface());
+        eglBase.makeCurrent();
+        // Might have a pending frame waiting for a surface to be created.
+        renderFrameOnRenderThread();
+      }
+    });
+  }
+
+  @Override
+  public void surfaceDestroyed(SurfaceHolder holder) {
+    Log.d(TAG, "Surface destroyed");
+    runOnRenderThread(new Runnable() {
+      @Override public void run() {
+        eglBase.releaseSurface();
+      }
+    });
+  }
+
+  @Override
+  public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
+  }
+
+  /**
+   * Private helper function to post tasks safely.
+   */
+  private synchronized void runOnRenderThread(Runnable runnable) {
+    if (renderThreadHandler != null) {
+      renderThreadHandler.post(runnable);
+    }
+  }
+
+  /**
+   * Renders and releases |pendingFrame|.
+   */
+  private void renderFrameOnRenderThread() {
+    if (eglBase == null || !eglBase.hasSurface()) {
+      Log.d(TAG, "No surface to draw on");
+      return;
+    }
+    // Synchronized fetch of |pendingFrame|.
+    final VideoRenderer.I420Frame frame;
+    synchronized (this) {
+      if (pendingFrame == null) {
+        return;
+      }
+      frame = pendingFrame;
+      pendingFrame = null;
+    }
+
+    final long startTimeNs = System.nanoTime();
+    final float[] texMatrix = new float[16];
+    synchronized (layoutLock) {
+      VideoRendererGui.getTextureMatrix(
+          texMatrix, frame.rotationDegree, mirror, videoAspectRatio, layoutAspectRatio);
+    }
+
+    GLES20.glViewport(0, 0, eglBase.surfaceWidth(), eglBase.surfaceHeight());
+    if (frame.yuvFrame) {
+      // Make sure YUV textures are allocated.
+      if (yuvTextures == null) {
+        yuvTextures = new int[3];
+        // Generate 3 texture ids for Y/U/V and place them into |yuvTextures|.
+        GLES20.glGenTextures(3, yuvTextures, 0);
+        for (int i = 0; i < 3; i++)  {
+          GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
+          GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
+          GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
+              GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
+          GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
+              GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
+          GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
+              GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
+          GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
+              GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
+        }
+        GlUtil.checkNoGLES2Error("y/u/v glGenTextures");
+      }
+
+      // Upload YUV data.
+      for (int i = 0; i < 3; ++i) {
+        GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
+        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
+        int w = (i == 0) ? frame.width : frame.width / 2;
+        int h = (i == 0) ? frame.height : frame.height / 2;
+        GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, w, h, 0,
+            GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, frame.yuvPlanes[i]);
+      }
+
+      drawer.drawYuv(frame.width, frame.height, yuvTextures, texMatrix);
+    } else {
+      SurfaceTexture surfaceTexture = (SurfaceTexture) frame.textureObject;
+      // TODO(magjed): Move updateTexImage() to the video source instead.
+      surfaceTexture.updateTexImage();
+      drawer.drawOes(frame.textureId, texMatrix);
+    }
+
+    eglBase.swapBuffers();
+    frame.release();
+    synchronized (statisticsLock) {
+      if (framesRendered == 0) {
+        firstFrameTimeNs = startTimeNs;
+      }
+      ++framesRendered;
+      renderTimeNs += (System.nanoTime() - startTimeNs);
+      if (framesRendered % 300 == 0) {
+        logStatistics();
+      }
+    }
+  }
+
+  private void logStatistics() {
+    synchronized (statisticsLock) {
+      Log.d(TAG, "ID: " + getResources().getResourceEntryName(getId()) + ". Frames received: "
+          + framesReceived + ". Dropped: " + framesDropped + ". Rendered: " + framesRendered);
+      if (framesReceived > 0 && framesRendered > 0) {
+        final long timeSinceFirstFrameNs = System.nanoTime() - firstFrameTimeNs;
+        Log.d(TAG, "Duration: " + (int) (timeSinceFirstFrameNs / 1e6) +
+            " ms. FPS: " + (float) framesRendered * 1e9 / timeSinceFirstFrameNs);
+        Log.d(TAG, "Average render time: "
+            + (int) (renderTimeNs / (1000 * framesRendered)) + " us.");
+      }
+    }
+  }
+}