Android EglRenderer: Add Bitmap frame listener functionality

webrtc/api/android/java/src/org/webrtc/EglRenderer.java

Index: webrtc/api/android/java/src/org/webrtc/EglRenderer.java
diff --git a/webrtc/api/android/java/src/org/webrtc/EglRenderer.java b/webrtc/api/android/java/src/org/webrtc/EglRenderer.java
index d5c29e77404ba26ef4318d2e63eba382507a018e..2a56fdbb9beb3530907cb698305a27ac7faa561c 100644
--- a/webrtc/api/android/java/src/org/webrtc/EglRenderer.java
+++ b/webrtc/api/android/java/src/org/webrtc/EglRenderer.java
@@ -10,12 +10,16 @@
 
 package org.webrtc;
 
+import android.graphics.Bitmap;
 import android.graphics.SurfaceTexture;
 import android.opengl.GLES20;
 import android.os.Handler;
 import android.os.HandlerThread;
 import android.os.Looper;
 import android.view.Surface;
+import java.nio.IntBuffer;
+import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
@@ -29,6 +33,18 @@ public class EglRenderer implements VideoRenderer.Callbacks {
   private static final long LOG_INTERVAL_SEC = 4;
   private static final int MAX_SURFACE_CLEAR_COUNT = 3;
 
+  public interface FrameListener { void onFrame(Bitmap frame); }
+
+  private static class ScaleAndFrameListener {
+    public final float scale;
+    public final FrameListener listener;
+
+    public ScaleAndFrameListener(float scale, FrameListener listener) {
+      this.scale = scale;
+      this.listener = listener;
+    }
+  }
+
   private class EglSurfaceCreation implements Runnable {
     private Object surface;
 
@@ -60,6 +76,9 @@ public class EglRenderer implements VideoRenderer.Callbacks {
   private final Object handlerLock = new Object();
   private Handler renderThreadHandler;
 
+  private final Object frameListenerLock = new Object();
+  private final ArrayList<ScaleAndFrameListener> frameListeners = new ArrayList<>();
+
   // Variables for fps reduction.
   private final Object fpsReductionLock = new Object();
   // Time for when next frame should be rendered.
@@ -72,6 +91,7 @@ public class EglRenderer implements VideoRenderer.Callbacks {
   // from the render thread.
   private EglBase eglBase;
   private final RendererCommon.YuvUploader yuvUploader = new RendererCommon.YuvUploader();
+  private YuvConverter yuvConverter;

[magjed_webrtc, 2016/11/01 18:06:46]

Remove this now since you didn't end up using it.

[sakal, 2016/11/02 08:34:29]

Done.

   private RendererCommon.GlDrawer drawer;
   // Texture ids for YUV frames. Allocated on first arrival of a YUV frame.
   private int[] yuvTextures = null;
@@ -104,6 +124,12 @@ public class EglRenderer implements VideoRenderer.Callbacks {
   // Time in ns spent by the render thread in the swapBuffers() function.
   private long renderSwapBufferTimeNs;
 
+  // Used for bitmap capturing.
+  private final int[] bitmapTexture = new int[1];

[magjed_webrtc, 2016/11/01 18:06:46]

Keep this as an int instead of int[1]. Same for bitmapFramebuffer.

[sakal, 2016/11/02 08:34:29]

Done.

+  private final int[] bitmapFramebuffer = new int[1];
+  private int bitmapTextureWidth;
+  private int bitmapTextureHeight;
+
   // Runnable for posting frames to render thread.
   private final Runnable renderFrameRunnable = new Runnable() {
     @Override
@@ -216,10 +242,18 @@ public class EglRenderer implements VideoRenderer.Callbacks {
             drawer.release();
             drawer = null;
           }
+          if (yuvConverter != null) {
+            yuvConverter.release();
+            yuvConverter = null;
+          }
           if (yuvTextures != null) {
             GLES20.glDeleteTextures(3, yuvTextures, 0);
             yuvTextures = null;
           }
+          GLES20.glDeleteFramebuffers(1, bitmapFramebuffer, 0);

[magjed_webrtc, 2016/11/01 18:06:46]

Add 'if (bitmapFramebuffer[0] != 0)' check here. Same for bitmapTexture.

[sakal, 2016/11/02 08:34:29]

"glDeleteFramebuffers silently ignores 0's and names that do not correspond to
existing framebuffer objects."

Anyway, it is irrelevant now since I changed the code to use
GlTextureFrameBuffer.

+          bitmapFramebuffer[0] = 0;
+          GLES20.glDeleteTextures(1, bitmapTexture, 0);
+          bitmapTexture[0] = 0;
           if (eglBase != null) {
             logD("eglBase detach and release.");
             eglBase.detachCurrent();
@@ -333,6 +367,36 @@ public class EglRenderer implements VideoRenderer.Callbacks {
     setFpsReduction(0 /* fps */);
   }
 
+  /**
+   * Register a callback to be invoked when a new video frame has been received.
+   *
+   * @param listener The callback to be invoked.
+   * @param scale    The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
+   *                 required.
+   */
+  public void addFrameListener(FrameListener listener, float scale) {
+    synchronized (frameListenerLock) {
+      frameListeners.add(new ScaleAndFrameListener(scale, listener));
+    }
+  }
+
+  /**
+   * Remove any pending callback that was added with addFrameListener. If the callback is not in
+   * the queue, nothing happens.
+   *
+   * @param runnable The callback to remove.
+   */
+  public void removeFrameListener(FrameListener listener) {
+    synchronized (frameListenerLock) {
+      final Iterator<ScaleAndFrameListener> iter = frameListeners.iterator();
+      while (iter.hasNext()) {
+        if (iter.next().listener == listener) {
+          iter.remove();
+        }
+      }
+    }
+  }
+
   // VideoRenderer.Callbacks interface.
   @Override
   public void renderFrame(VideoRenderer.I420Frame frame) {
@@ -472,8 +536,9 @@ public class EglRenderer implements VideoRenderer.Callbacks {
     }
 
     final long startTimeNs = System.nanoTime();
-    float[] texMatrix =
+    final float[] texMatrix =
         RendererCommon.rotateTextureMatrix(frame.samplingMatrix, frame.rotationDegree);
+    final float[] drawMatrix;
 
     // After a surface size change, the EGLSurface might still have a buffer of the old size in the
     // pipeline. Querying the EGLSurface will show if the underlying buffer dimensions haven't yet
@@ -487,7 +552,8 @@ public class EglRenderer implements VideoRenderer.Callbacks {
           VideoRenderer.renderFrameDone(frame);
           return;
         }
-        logD("Surface size mismatch - clearing surface.");
+        logD("Surface size mismatch - clearing surface. Size: " + eglBase.surfaceWidth() + "x"
+            + eglBase.surfaceHeight() + " Expected: " + surfaceWidth + "x" + surfaceHeight);
         clearSurfaceOnRenderThread();
       }
       final float[] layoutMatrix;
@@ -498,7 +564,7 @@ public class EglRenderer implements VideoRenderer.Callbacks {
         layoutMatrix =
             mirror ? RendererCommon.horizontalFlipMatrix() : RendererCommon.identityMatrix();
       }
-      texMatrix = RendererCommon.multiplyMatrices(texMatrix, layoutMatrix);
+      drawMatrix = RendererCommon.multiplyMatrices(texMatrix, layoutMatrix);
     }
 
     GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
@@ -513,13 +579,74 @@ public class EglRenderer implements VideoRenderer.Callbacks {
       }
       yuvUploader.uploadYuvData(
           yuvTextures, frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes);
-      drawer.drawYuv(yuvTextures, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
+      drawer.drawYuv(yuvTextures, drawMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
           surfaceWidth, surfaceHeight);
     } else {
-      drawer.drawOes(frame.textureId, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
+      drawer.drawOes(frame.textureId, drawMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
           surfaceWidth, surfaceHeight);
     }
 
+    // Notify callbacks. Make temporary copy of callback list to avoid

[magjed_webrtc, 2016/11/01 18:06:46]

Move this block of code into a helper function. Also abort early if
frameListeners is empty. Also, you need to synchronize on frameListenerLock.

[sakal, 2016/11/02 08:34:29]

Done.

+    // ConcurrentModificationException, in case callbacks call addFramelistener or
+    // removeFrameListener.
+    final ArrayList<ScaleAndFrameListener> tmpList = new ArrayList<>(frameListeners);
+    frameListeners.clear();
+    for (ScaleAndFrameListener scaleAndListener : tmpList) {
+      final int scaledWidth = (int) (scaleAndListener.scale * frame.rotatedWidth());
+      final int scaledHeight = (int) (scaleAndListener.scale * frame.rotatedHeight());
+      final float[] bitmapMatrix = RendererCommon.multiplyMatrices(

[magjed_webrtc, 2016/11/01 18:06:46]

Move this outside the loop.

[sakal, 2016/11/02 08:34:29]

Done.

+          RendererCommon.multiplyMatrices(texMatrix,
+              mirror ? RendererCommon.horizontalFlipMatrix() : RendererCommon.identityMatrix()),
+          RendererCommon.verticalFlipMatrix());
+
+      if (scaledWidth == 0 || scaledHeight == 0) {
+        scaleAndListener.listener.onFrame(null);
+        continue;
+      }
+      if (bitmapTexture[0] == 0) {
+        bitmapTexture[0] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D);
+        bitmapTextureWidth = bitmapTextureHeight = 0;
+      }
+      if (bitmapFramebuffer[0] == 0) {
+        GLES20.glGenFramebuffers(1, bitmapFramebuffer, 0);
+      }
+
+      GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexture[0]);
+      if (scaledWidth != bitmapTextureWidth || scaledHeight != bitmapTextureHeight) {

[magjed_webrtc, 2016/11/01 18:06:46]

Wait a minute... Maybe you can use the helper class GlTextureFrameBuffer for
this?

[sakal, 2016/11/02 08:34:29]

Done.

+        GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, scaledWidth, scaledHeight, 0,
+            GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, null);
+        bitmapTextureWidth = scaledWidth;
+        bitmapTextureHeight = scaledHeight;
+      }
+      GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, bitmapFramebuffer[0]);
+      GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
+          GLES20.GL_TEXTURE_2D, bitmapTexture[0], 0);
+
+      if (frame.yuvFrame) {
+        drawer.drawYuv(yuvTextures, bitmapMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
+            scaledWidth, scaledHeight);
+      } else {
+        drawer.drawOes(frame.textureId, bitmapMatrix, frame.rotatedWidth(), frame.rotatedHeight(),
+            0, 0, scaledWidth, scaledHeight);
+      }
+
+      final IntBuffer bitmapBuffer = IntBuffer.allocate(scaledWidth * scaledHeight);
+      GLES20.glViewport(0, 0, scaledWidth, scaledHeight);
+      GLES20.glReadPixels(
+          0, 0, scaledWidth, scaledHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, bitmapBuffer);
+
+      GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
+      GlUtil.checkNoGLES2Error("EglRenderer.renderFrameOnRenderThread - bitmap capture");
+
+      // RGBA to ARGB conversion.
+      final int[] bitmapArray = bitmapBuffer.array();
+      nativeABGRToARGB(bitmapArray, scaledWidth, scaledHeight);
+
+      final Bitmap bitmap = Bitmap.createBitmap(
+          bitmapBuffer.array(), scaledWidth, scaledHeight, Bitmap.Config.ARGB_8888);
+      scaleAndListener.listener.onFrame(bitmap);
+    }
+
     final long swapBuffersStartTimeNs = System.nanoTime();
     eglBase.swapBuffers();
     VideoRenderer.renderFrameDone(frame);
@@ -559,4 +686,6 @@ public class EglRenderer implements VideoRenderer.Callbacks {
   private void logD(String string) {
     Logging.d(TAG, name + string);
   }
+
+  private static native void nativeABGRToARGB(int[] array, int width, int height);
 }