Android EglRenderer: Add Bitmap frame listener functionality

webrtc/api/android/java/src/org/webrtc/EglRenderer.java

 /*
  *  Copyright 2016 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 package org.webrtc;
 
+import android.graphics.Bitmap;
 import android.graphics.SurfaceTexture;
 import android.opengl.GLES20;
 import android.os.Handler;
 import android.os.HandlerThread;
 import android.os.Looper;
 import android.view.Surface;
+import java.nio.IntBuffer;
+import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
 /**
  * Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream on an EGL Surface.
  * This class is intended to be used as a helper class for rendering on SurfaceViews and
  * TextureViews.
  */
 public class EglRenderer implements VideoRenderer.Callbacks {
   private static final String TAG = "EglRenderer";
   private static final long LOG_INTERVAL_SEC = 4;
   private static final int MAX_SURFACE_CLEAR_COUNT = 3;
 
+  public interface FrameListener { void onFrame(Bitmap frame); }
+
+  private static class ScaleAndFrameListener {
+    public final float scale;
+    public final FrameListener listener;
+
+    public ScaleAndFrameListener(float scale, FrameListener listener) {
+      this.scale = scale;
+      this.listener = listener;
+    }
+  }
+
   private class EglSurfaceCreation implements Runnable {
     private Object surface;
 
     public synchronized void setSurface(Object surface) {
       this.surface = surface;
     }
 
     @Override
     public synchronized void run() {
       if (surface != null && eglBase != null && !eglBase.hasSurface()) {
@@ skipping 11 matching lines @@
     }
   }
 
   private final String name;
 
   // |renderThreadHandler| is a handler for communicating with |renderThread|, and is synchronized
   // on |handlerLock|.
   private final Object handlerLock = new Object();
   private Handler renderThreadHandler;
 
+  private final Object frameListenerLock = new Object();
+  private final ArrayList<ScaleAndFrameListener> frameListeners = new ArrayList<>();
+
   // Variables for fps reduction.
   private final Object fpsReductionLock = new Object();
   // Time for when next frame should be rendered.
   private long nextFrameTimeNs;
   // Minimum duration between frames when fps reduction is active, or -1 if video is completely
   // paused.
   private long minRenderPeriodNs;
 
   // EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed
   // from the render thread.
   private EglBase eglBase;
   private final RendererCommon.YuvUploader yuvUploader = new RendererCommon.YuvUploader();
+  private YuvConverter yuvConverter;

[magjed_webrtc, 2016/11/01 18:06:46]

Remove this now since you didn't end up using it.

[sakal, 2016/11/02 08:34:29]

Done.

   private RendererCommon.GlDrawer drawer;
   // Texture ids for YUV frames. Allocated on first arrival of a YUV frame.
   private int[] yuvTextures = null;
 
   // Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|.
   private final Object frameLock = new Object();
   private VideoRenderer.I420Frame pendingFrame;
 
   // These variables are synchronized on |layoutLock|.
   private final Object layoutLock = new Object();
@@ skipping 12 matching lines @@
   private int framesDropped;
   // Number of rendered video frames.
   private int framesRendered;
   // Start time for counting these statistics, or 0 if we haven't started measuring yet.
   private long statisticsStartTimeNs;
   // Time in ns spent in renderFrameOnRenderThread() function.
   private long renderTimeNs;
   // Time in ns spent by the render thread in the swapBuffers() function.
   private long renderSwapBufferTimeNs;
 
+  // Used for bitmap capturing.
+  private final int[] bitmapTexture = new int[1];

[magjed_webrtc, 2016/11/01 18:06:46]

Keep this as an int instead of int[1]. Same for bitmapFramebuffer.

[sakal, 2016/11/02 08:34:29]

Done.

+  private final int[] bitmapFramebuffer = new int[1];
+  private int bitmapTextureWidth;
+  private int bitmapTextureHeight;
+
   // Runnable for posting frames to render thread.
   private final Runnable renderFrameRunnable = new Runnable() {
     @Override
     public void run() {
       renderFrameOnRenderThread();
     }
   };
 
   private final Runnable logStatisticsRunnable = new Runnable() {
     @Override
@@ skipping 92 matching lines @@
       }
       renderThreadHandler.removeCallbacks(logStatisticsRunnable);
       // Release EGL and GL resources on render thread.
       renderThreadHandler.postAtFrontOfQueue(new Runnable() {
         @Override
         public void run() {
           if (drawer != null) {
             drawer.release();
             drawer = null;
           }
+          if (yuvConverter != null) {
+            yuvConverter.release();
+            yuvConverter = null;
+          }
           if (yuvTextures != null) {
             GLES20.glDeleteTextures(3, yuvTextures, 0);
             yuvTextures = null;
           }
+          GLES20.glDeleteFramebuffers(1, bitmapFramebuffer, 0);

[magjed_webrtc, 2016/11/01 18:06:46]

Add 'if (bitmapFramebuffer[0] != 0)' check here. Same for bitmapTexture.

[sakal, 2016/11/02 08:34:29]

"glDeleteFramebuffers silently ignores 0's and names that do not correspond to
existing framebuffer objects."

Anyway, it is irrelevant now since I changed the code to use
GlTextureFrameBuffer.

+          bitmapFramebuffer[0] = 0;
+          GLES20.glDeleteTextures(1, bitmapTexture, 0);
+          bitmapTexture[0] = 0;
           if (eglBase != null) {
             logD("eglBase detach and release.");
             eglBase.detachCurrent();
             eglBase.release();
             eglBase = null;
           }
           eglCleanupBarrier.countDown();
         }
       });
       final Looper renderLooper = renderThreadHandler.getLooper();
@@ skipping 93 matching lines @@
   }
 
   public void disableFpsReduction() {
     setFpsReduction(Float.POSITIVE_INFINITY /* fps */);
   }
 
   public void pauseVideo() {
     setFpsReduction(0 /* fps */);
   }
 
+  /**
+   * Register a callback to be invoked when a new video frame has been received.
+   *
+   * @param listener The callback to be invoked.
+   * @param scale    The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
+   *                 required.
+   */
+  public void addFrameListener(FrameListener listener, float scale) {
+    synchronized (frameListenerLock) {
+      frameListeners.add(new ScaleAndFrameListener(scale, listener));
+    }
+  }
+
+  /**
+   * Remove any pending callback that was added with addFrameListener. If the callback is not in
+   * the queue, nothing happens.
+   *
+   * @param runnable The callback to remove.
+   */
+  public void removeFrameListener(FrameListener listener) {
+    synchronized (frameListenerLock) {
+      final Iterator<ScaleAndFrameListener> iter = frameListeners.iterator();
+      while (iter.hasNext()) {
+        if (iter.next().listener == listener) {
+          iter.remove();
+        }
+      }
+    }
+  }
+
   // VideoRenderer.Callbacks interface.
   @Override
   public void renderFrame(VideoRenderer.I420Frame frame) {
     synchronized (statisticsLock) {
       ++framesReceived;
     }
     final boolean dropOldFrame;
     synchronized (handlerLock) {
       if (renderThreadHandler == null) {
         logD("Dropping frame - Not initialized or already released.");
@@ skipping 119 matching lines @@
       frame = pendingFrame;
       pendingFrame = null;
     }
     if (eglBase == null || !eglBase.hasSurface()) {
       logD("Dropping frame - No surface");
       VideoRenderer.renderFrameDone(frame);
       return;
     }
 
     final long startTimeNs = System.nanoTime();
-    float[] texMatrix =
+    final float[] texMatrix =
         RendererCommon.rotateTextureMatrix(frame.samplingMatrix, frame.rotationDegree);
+    final float[] drawMatrix;
 
     // After a surface size change, the EGLSurface might still have a buffer of the old size in the
     // pipeline. Querying the EGLSurface will show if the underlying buffer dimensions haven't yet
     // changed. Such a buffer will be rendered incorrectly, so flush it with a black frame.
     synchronized (layoutLock) {
       int surfaceClearCount = 0;
       while (eglBase.surfaceWidth() != surfaceWidth || eglBase.surfaceHeight() != surfaceHeight) {
         ++surfaceClearCount;
         if (surfaceClearCount > MAX_SURFACE_CLEAR_COUNT) {
           logD("Failed to get surface of expected size - dropping frame.");
           VideoRenderer.renderFrameDone(frame);
           return;
         }
-        logD("Surface size mismatch - clearing surface.");
+        logD("Surface size mismatch - clearing surface. Size: " + eglBase.surfaceWidth() + "x"
+            + eglBase.surfaceHeight() + " Expected: " + surfaceWidth + "x" + surfaceHeight);
         clearSurfaceOnRenderThread();
       }
       final float[] layoutMatrix;
       if (layoutAspectRatio > 0) {
         layoutMatrix = RendererCommon.getLayoutMatrix(
             mirror, frame.rotatedWidth() / (float) frame.rotatedHeight(), layoutAspectRatio);
       } else {
         layoutMatrix =
             mirror ? RendererCommon.horizontalFlipMatrix() : RendererCommon.identityMatrix();
       }
-      texMatrix = RendererCommon.multiplyMatrices(texMatrix, layoutMatrix);
+      drawMatrix = RendererCommon.multiplyMatrices(texMatrix, layoutMatrix);
     }
 
     GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
     GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
     if (frame.yuvFrame) {
       // Make sure YUV textures are allocated.
       if (yuvTextures == null) {
         yuvTextures = new int[3];
         for (int i = 0; i < 3; i++) {
           yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D);
         }
       }
       yuvUploader.uploadYuvData(
           yuvTextures, frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes);
-      drawer.drawYuv(yuvTextures, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
+      drawer.drawYuv(yuvTextures, drawMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
           surfaceWidth, surfaceHeight);
     } else {
-      drawer.drawOes(frame.textureId, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
+      drawer.drawOes(frame.textureId, drawMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
           surfaceWidth, surfaceHeight);
     }
 
+    // Notify callbacks. Make temporary copy of callback list to avoid

[magjed_webrtc, 2016/11/01 18:06:46]

Move this block of code into a helper function. Also abort early if
frameListeners is empty. Also, you need to synchronize on frameListenerLock.

[sakal, 2016/11/02 08:34:29]

Done.

+    // ConcurrentModificationException, in case callbacks call addFramelistener or
+    // removeFrameListener.
+    final ArrayList<ScaleAndFrameListener> tmpList = new ArrayList<>(frameListeners);
+    frameListeners.clear();
+    for (ScaleAndFrameListener scaleAndListener : tmpList) {
+      final int scaledWidth = (int) (scaleAndListener.scale * frame.rotatedWidth());
+      final int scaledHeight = (int) (scaleAndListener.scale * frame.rotatedHeight());
+      final float[] bitmapMatrix = RendererCommon.multiplyMatrices(

[magjed_webrtc, 2016/11/01 18:06:46]

Move this outside the loop.

[sakal, 2016/11/02 08:34:29]

Done.

+          RendererCommon.multiplyMatrices(texMatrix,
+              mirror ? RendererCommon.horizontalFlipMatrix() : RendererCommon.identityMatrix()),
+          RendererCommon.verticalFlipMatrix());
+
+      if (scaledWidth == 0 || scaledHeight == 0) {
+        scaleAndListener.listener.onFrame(null);
+        continue;
+      }
+      if (bitmapTexture[0] == 0) {
+        bitmapTexture[0] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D);
+        bitmapTextureWidth = bitmapTextureHeight = 0;
+      }
+      if (bitmapFramebuffer[0] == 0) {
+        GLES20.glGenFramebuffers(1, bitmapFramebuffer, 0);
+      }
+
+      GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexture[0]);
+      if (scaledWidth != bitmapTextureWidth || scaledHeight != bitmapTextureHeight) {

[magjed_webrtc, 2016/11/01 18:06:46]

Wait a minute... Maybe you can use the helper class GlTextureFrameBuffer for
this?

[sakal, 2016/11/02 08:34:29]

Done.

+        GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, scaledWidth, scaledHeight, 0,
+            GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, null);
+        bitmapTextureWidth = scaledWidth;
+        bitmapTextureHeight = scaledHeight;
+      }
+      GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, bitmapFramebuffer[0]);
+      GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
+          GLES20.GL_TEXTURE_2D, bitmapTexture[0], 0);
+
+      if (frame.yuvFrame) {
+        drawer.drawYuv(yuvTextures, bitmapMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
+            scaledWidth, scaledHeight);
+      } else {
+        drawer.drawOes(frame.textureId, bitmapMatrix, frame.rotatedWidth(), frame.rotatedHeight(),
+            0, 0, scaledWidth, scaledHeight);
+      }
+
+      final IntBuffer bitmapBuffer = IntBuffer.allocate(scaledWidth * scaledHeight);
+      GLES20.glViewport(0, 0, scaledWidth, scaledHeight);
+      GLES20.glReadPixels(
+          0, 0, scaledWidth, scaledHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, bitmapBuffer);
+
+      GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
+      GlUtil.checkNoGLES2Error("EglRenderer.renderFrameOnRenderThread - bitmap capture");
+
+      // RGBA to ARGB conversion.
+      final int[] bitmapArray = bitmapBuffer.array();
+      nativeABGRToARGB(bitmapArray, scaledWidth, scaledHeight);
+
+      final Bitmap bitmap = Bitmap.createBitmap(
+          bitmapBuffer.array(), scaledWidth, scaledHeight, Bitmap.Config.ARGB_8888);
+      scaleAndListener.listener.onFrame(bitmap);
+    }
+
     final long swapBuffersStartTimeNs = System.nanoTime();
     eglBase.swapBuffers();
     VideoRenderer.renderFrameDone(frame);
 
     final long currentTimeNs = System.nanoTime();
     synchronized (statisticsLock) {
       ++framesRendered;
       renderTimeNs += (currentTimeNs - startTimeNs);
       renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs);
     }
@@ skipping 19 matching lines @@
           + " Average render time: " + averageTimeAsString(renderTimeNs, framesRendered) + "."
           + " Average swapBuffer time: "
           + averageTimeAsString(renderSwapBufferTimeNs, framesRendered) + ".");
       resetStatistics(currentTimeNs);
     }
   }
 
   private void logD(String string) {
     Logging.d(TAG, name + string);
   }
+
+  private static native void nativeABGRToARGB(int[] array, int width, int height);
 }