Android EglRenderer: Add Bitmap frame listener functionality

webrtc/api/android/java/src/org/webrtc/EglRenderer.java

Index: webrtc/api/android/java/src/org/webrtc/EglRenderer.java
diff --git a/webrtc/api/android/java/src/org/webrtc/EglRenderer.java b/webrtc/api/android/java/src/org/webrtc/EglRenderer.java
index d5c29e77404ba26ef4318d2e63eba382507a018e..9fb035b34b9edaf37997e87276c6900e8b374dc1 100644
--- a/webrtc/api/android/java/src/org/webrtc/EglRenderer.java
+++ b/webrtc/api/android/java/src/org/webrtc/EglRenderer.java
@@ -10,12 +10,16 @@
 
 package org.webrtc;
 
+import android.graphics.Bitmap;
 import android.graphics.SurfaceTexture;
 import android.opengl.GLES20;
 import android.os.Handler;
 import android.os.HandlerThread;
 import android.os.Looper;
 import android.view.Surface;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
@@ -29,6 +33,18 @@ public class EglRenderer implements VideoRenderer.Callbacks {
   private static final long LOG_INTERVAL_SEC = 4;
   private static final int MAX_SURFACE_CLEAR_COUNT = 3;
 
+  public interface FrameListener { void onFrame(Bitmap frame); }
+
+  private static class ScaleAndFrameListener {
+    public final float scale;
+    public final FrameListener listener;
+
+    public ScaleAndFrameListener(float scale, FrameListener listener) {
+      this.scale = scale;
+      this.listener = listener;
+    }
+  }
+
   private class EglSurfaceCreation implements Runnable {
     private Object surface;
 
@@ -60,6 +76,9 @@ public class EglRenderer implements VideoRenderer.Callbacks {
   private final Object handlerLock = new Object();
   private Handler renderThreadHandler;
 
+  private final Object frameListenerLock = new Object();
+  private final ArrayList<ScaleAndFrameListener> frameListeners = new ArrayList<>();
+
   // Variables for fps reduction.
   private final Object fpsReductionLock = new Object();
   // Time for when next frame should be rendered.
@@ -104,6 +123,9 @@ public class EglRenderer implements VideoRenderer.Callbacks {
   // Time in ns spent by the render thread in the swapBuffers() function.
   private long renderSwapBufferTimeNs;
 
+  // Used for bitmap capturing.
+  private GlTextureFrameBuffer bitmapTextureFramebuffer;
+
   // Runnable for posting frames to render thread.
   private final Runnable renderFrameRunnable = new Runnable() {
     @Override
@@ -220,6 +242,10 @@ public class EglRenderer implements VideoRenderer.Callbacks {
             GLES20.glDeleteTextures(3, yuvTextures, 0);
             yuvTextures = null;
           }
+          if (bitmapTextureFramebuffer != null) {
+            bitmapTextureFramebuffer.release();
+            bitmapTextureFramebuffer = null;
+          }
           if (eglBase != null) {
             logD("eglBase detach and release.");
             eglBase.detachCurrent();
@@ -333,6 +359,36 @@ public class EglRenderer implements VideoRenderer.Callbacks {
     setFpsReduction(0 /* fps */);
   }
 
+  /**
+   * Register a callback to be invoked when a new video frame has been received.
+   *
+   * @param listener The callback to be invoked.
+   * @param scale    The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
+   *                 required.
+   */
+  public void addFrameListener(FrameListener listener, float scale) {
+    synchronized (frameListenerLock) {
+      frameListeners.add(new ScaleAndFrameListener(scale, listener));
+    }
+  }
+
+  /**
+   * Remove any pending callback that was added with addFrameListener. If the callback is not in
+   * the queue, nothing happens.
+   *
+   * @param runnable The callback to remove.
+   */
+  public void removeFrameListener(FrameListener listener) {
+    synchronized (frameListenerLock) {
+      final Iterator<ScaleAndFrameListener> iter = frameListeners.iterator();
+      while (iter.hasNext()) {
+        if (iter.next().listener == listener) {
+          iter.remove();
+        }
+      }
+    }
+  }
+
   // VideoRenderer.Callbacks interface.
   @Override
   public void renderFrame(VideoRenderer.I420Frame frame) {
@@ -472,8 +528,9 @@ public class EglRenderer implements VideoRenderer.Callbacks {
     }
 
     final long startTimeNs = System.nanoTime();
-    float[] texMatrix =
+    final float[] texMatrix =
         RendererCommon.rotateTextureMatrix(frame.samplingMatrix, frame.rotationDegree);
+    final float[] drawMatrix;
 
     // After a surface size change, the EGLSurface might still have a buffer of the old size in the
     // pipeline. Querying the EGLSurface will show if the underlying buffer dimensions haven't yet
@@ -487,7 +544,8 @@ public class EglRenderer implements VideoRenderer.Callbacks {
           VideoRenderer.renderFrameDone(frame);
           return;
         }
-        logD("Surface size mismatch - clearing surface.");
+        logD("Surface size mismatch - clearing surface. Size: " + eglBase.surfaceWidth() + "x"
+            + eglBase.surfaceHeight() + " Expected: " + surfaceWidth + "x" + surfaceHeight);
         clearSurfaceOnRenderThread();
       }
       final float[] layoutMatrix;
@@ -498,7 +556,7 @@ public class EglRenderer implements VideoRenderer.Callbacks {
         layoutMatrix =
             mirror ? RendererCommon.horizontalFlipMatrix() : RendererCommon.identityMatrix();
       }
-      texMatrix = RendererCommon.multiplyMatrices(texMatrix, layoutMatrix);
+      drawMatrix = RendererCommon.multiplyMatrices(texMatrix, layoutMatrix);
     }
 
     GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
@@ -513,16 +571,15 @@ public class EglRenderer implements VideoRenderer.Callbacks {
       }
       yuvUploader.uploadYuvData(
           yuvTextures, frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes);
-      drawer.drawYuv(yuvTextures, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
+      drawer.drawYuv(yuvTextures, drawMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
           surfaceWidth, surfaceHeight);
     } else {
-      drawer.drawOes(frame.textureId, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
+      drawer.drawOes(frame.textureId, drawMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
           surfaceWidth, surfaceHeight);
     }
 
     final long swapBuffersStartTimeNs = System.nanoTime();
     eglBase.swapBuffers();
-    VideoRenderer.renderFrameDone(frame);
 
     final long currentTimeNs = System.nanoTime();
     synchronized (statisticsLock) {
@@ -530,6 +587,65 @@ public class EglRenderer implements VideoRenderer.Callbacks {
       renderTimeNs += (currentTimeNs - startTimeNs);
       renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs);
     }
+
+    notifyCallbacks(frame, texMatrix);
+    VideoRenderer.renderFrameDone(frame);
+  }
+
+  private void notifyCallbacks(VideoRenderer.I420Frame frame, float[] texMatrix) {
+    // Make temporary copy of callback list to avoid ConcurrentModificationException, in case
+    // callbacks call addFramelistener or removeFrameListener.
+    final ArrayList<ScaleAndFrameListener> tmpList;
+    synchronized (frameListenerLock) {
+      if (frameListeners.isEmpty())
+        return;
+      tmpList = new ArrayList<>(frameListeners);
+      frameListeners.clear();
+    }
+
+    final float[] bitmapMatrix = RendererCommon.multiplyMatrices(
+        RendererCommon.multiplyMatrices(texMatrix,
+            mirror ? RendererCommon.horizontalFlipMatrix() : RendererCommon.identityMatrix()),
+        RendererCommon.verticalFlipMatrix());
+
+    for (ScaleAndFrameListener scaleAndListener : tmpList) {
+      final int scaledWidth = (int) (scaleAndListener.scale * frame.rotatedWidth());
+      final int scaledHeight = (int) (scaleAndListener.scale * frame.rotatedHeight());
+
+      if (scaledWidth == 0 || scaledHeight == 0) {
+        scaleAndListener.listener.onFrame(null);
+        continue;
+      }
+
+      if (bitmapTextureFramebuffer == null) {
+        bitmapTextureFramebuffer = new GlTextureFrameBuffer(GLES20.GL_RGBA);
+      }
+      bitmapTextureFramebuffer.setSize(scaledWidth, scaledHeight);
+
+      GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, bitmapTextureFramebuffer.getFrameBufferId());
+      GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
+          GLES20.GL_TEXTURE_2D, bitmapTextureFramebuffer.getTextureId(), 0);
+
+      if (frame.yuvFrame) {
+        drawer.drawYuv(yuvTextures, bitmapMatrix, frame.rotatedWidth(), frame.rotatedHeight(),
+            0 /* viewportX */, 0 /* viewportY */, scaledWidth, scaledHeight);
+      } else {
+        drawer.drawOes(frame.textureId, bitmapMatrix, frame.rotatedWidth(), frame.rotatedHeight(),
+            0 /* viewportX */, 0 /* viewportY */, scaledWidth, scaledHeight);
+      }
+
+      final ByteBuffer bitmapBuffer = ByteBuffer.allocateDirect(scaledWidth * scaledHeight * 4);
+      GLES20.glViewport(0, 0, scaledWidth, scaledHeight);
+      GLES20.glReadPixels(
+          0, 0, scaledWidth, scaledHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, bitmapBuffer);
+
+      GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
+      GlUtil.checkNoGLES2Error("EglRenderer.notifyCallbacks");
+
+      final Bitmap bitmap = Bitmap.createBitmap(scaledWidth, scaledHeight, Bitmap.Config.ARGB_8888);
+      bitmap.copyPixelsFromBuffer(bitmapBuffer);
+      scaleAndListener.listener.onFrame(bitmap);
+    }
   }
 
   private String averageTimeAsString(long sumTimeNs, int count) {