Android EglRenderer: Add Bitmap frame listener functionality

webrtc/api/androidtests/src/org/webrtc/EglRendererTest.java

Index: webrtc/api/androidtests/src/org/webrtc/EglRendererTest.java
diff --git a/webrtc/api/androidtests/src/org/webrtc/EglRendererTest.java b/webrtc/api/androidtests/src/org/webrtc/EglRendererTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..1ac577c64f7ed2fe44a7b12ae1c57913988acd64
--- /dev/null
+++ b/webrtc/api/androidtests/src/org/webrtc/EglRendererTest.java
@@ -0,0 +1,251 @@
+/*
+ *  Copyright 2016 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import android.graphics.Bitmap;
+import android.test.ActivityInstrumentationTestCase2;
+import android.test.suitebuilder.annotation.SmallTest;
+import android.view.SurfaceHolder;
+import android.view.SurfaceView;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.concurrent.CountDownLatch;
+import org.webrtc.test.EmptyActivity;
+
+public class EglRendererTest extends ActivityInstrumentationTestCase2<EmptyActivity> {
+  final String TAG = "EglRendererTest";
+  final int RENDER_WAIT_MS = 100;
+  final int TEST_FRAME_WIDTH = 4;
+  final int TEST_FRAME_HEIGHT = 4;
+  // Some arbitrary frames
+  final ByteBuffer[][] TEST_FRAMES = {
+      {
+          ByteBuffer.wrap(
+              new byte[] {11, 12, 13, 14, 15, 16, 17, 18, 19, 110, 111, 112, 113, 114, 115, 116}),
+          ByteBuffer.wrap(new byte[] {117, 118, 119, 120}),
+          ByteBuffer.wrap(new byte[] {121, 122, 123, 124}),
+      },
+      {
+          ByteBuffer.wrap(new byte[] {-11, -12, -13, -14, -15, -16, -17, -18, -19, -110, -111, -112,
+              -113, -114, -115, -116}),
+          ByteBuffer.wrap(new byte[] {-121, -122, -123, -124}),
+          ByteBuffer.wrap(new byte[] {-117, -118, -119, -120}),
+      },
+      {
+          ByteBuffer.wrap(new byte[] {-11, -12, -13, -14, -15, -16, -17, -18, -19, -110, -111, -112,
+              -113, -114, -115, -116}),
+          ByteBuffer.wrap(new byte[] {117, 118, 119, 120}),
+          ByteBuffer.wrap(new byte[] {121, 122, 123, 124}),
+      },
+  };
+
+  private static class TestFrameListener implements EglRenderer.FrameListener {
+    final private ArrayList<Bitmap> bitmaps = new ArrayList<Bitmap>();
+
+    @Override
+    public synchronized void onFrame(Bitmap bitmap) {
+      bitmaps.add(bitmap);
+    }
+
+    public synchronized ArrayList<Bitmap> getBitmaps() {
+      return new ArrayList<Bitmap>(bitmaps);
+    }
+  }
+
+  final TestFrameListener testFrameListener = new TestFrameListener();
+
+  EglRenderer eglRenderer;
+  SurfaceView surfaceView;
+  CountDownLatch surfaceReadyLatch = new CountDownLatch(1);
+
+  public EglRendererTest() {
+    super(EmptyActivity.class);
+  }
+
+  @Override
+  protected void setUp() throws Exception {
+    PeerConnectionFactory.initializeAndroidGlobals(getInstrumentation().getTargetContext(),
+        true /* initializeAudio */, true /* initializeVideo */, true /* videoHwAcceleration */);
+    eglRenderer = new EglRenderer("TestRenderer: ");
+    eglRenderer.init(null, EglBase.CONFIG_RGBA, new GlRectDrawer());
+    getActivity().runOnUiThread(new Runnable() {
+      @Override
+      public void run() {
+        surfaceView = new SurfaceView(getInstrumentation().getTargetContext());
+        surfaceView.getHolder().addCallback(new SurfaceHolder.Callback() {
+          @Override
+          public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
+            eglRenderer.surfaceSizeChanged(width, height);
+          }
+
+          @Override
+          public void surfaceCreated(SurfaceHolder holder) {
+            eglRenderer.createEglSurface(holder.getSurface());
+            surfaceReadyLatch.countDown();
+          }
+
+          @Override
+          public void surfaceDestroyed(SurfaceHolder holder) {
+            eglRenderer.releaseEglSurface();
+          }
+        });
+        getActivity().setContentView(surfaceView);
+      }
+    });
+  }
+
+  @Override
+  public void tearDown() {
+    eglRenderer.release();
+  }
+
+  private void waitForSurfaceReady() throws InterruptedException {
+    surfaceReadyLatch.await();
+  }
+
+  private void checkBitmap(Bitmap bitmap, float scale) {
+    assertNotNull(bitmap);
+    assertEquals((int) (TEST_FRAME_WIDTH * scale), bitmap.getWidth());
+    assertEquals((int) (TEST_FRAME_HEIGHT * scale), bitmap.getHeight());
+  }
+
+  private float linearSample(ByteBuffer data, int x, int y) {
+    final int stride = TEST_FRAME_WIDTH / 2;
+
+    float coordX = (x + 0.5f) / TEST_FRAME_WIDTH * (TEST_FRAME_WIDTH / 2f);
+    float coordY = (y + 0.5f) / TEST_FRAME_HEIGHT * (TEST_FRAME_HEIGHT / 2f);
+
+    int lowX = (int) Math.floor(coordX - 0.5f);
+    int lowY = (int) Math.floor(coordY - 0.5f);
+    int highX = lowX + 1;
+    int highY = lowY + 1;
+
+    float highSampleX = coordX - lowX - 0.5f;
+    float highSampleY = coordY - lowY - 0.5f;
+    float lowSampleX = 1.f - highSampleX;
+    float lowSampleY = 1.f - highSampleY;
+
+    lowX = Math.max(0, lowY);
+    lowY = Math.max(0, lowY);
+    highX = Math.min(TEST_FRAME_WIDTH / 2 - 1, highX);
+    highY = Math.min(TEST_FRAME_HEIGHT / 2 - 1, highY);
+
+    float lowYValue = (data.get(lowY * stride + lowX) & 0xFF) * lowSampleX
+        + (data.get(lowY * stride + highX) & 0xFF) * highSampleX;
+    float highYValue = (data.get(highY * stride + lowX) & 0xFF) * lowSampleX
+        + (data.get(highY * stride + highX) & 0xFF) * highSampleX;
+
+    return (lowSampleY * lowYValue + highSampleY * highYValue) / 255.f - 0.5f;
+  }
+
+  private byte[] convertYUVFrameToRBG(ByteBuffer[] yuvFrame) {
+    final byte[] argbFrame = new byte[TEST_FRAME_WIDTH * TEST_FRAME_HEIGHT * 4];
+    final int argbStride = TEST_FRAME_WIDTH * 4;
+    final int yStride = TEST_FRAME_WIDTH;
+
+    final int vStride = TEST_FRAME_WIDTH / 2;
+
+    for (int y = 0; y < TEST_FRAME_HEIGHT; y++) {
+      for (int x = 0; x < TEST_FRAME_WIDTH; x++) {
+        final int x2 = x / 2;
+        final int y2 = y / 2;
+
+        final float yC = (yuvFrame[0].get(y * yStride + x) & 0xFF) / 255f;
+        final float uC = linearSample(yuvFrame[1], x, y);
+        final float vC = linearSample(yuvFrame[2], x, y);
+        final float rC = Math.max(0, Math.min(1, yC + 1.403f * vC));
+        final float gC = Math.max(0, Math.min(1, yC - 0.344f * uC - 0.714f * vC));
+        final float bC = Math.max(0, Math.min(1, yC + 1.77f * uC));
+
+        argbFrame[y * argbStride + x * 4 + 0] = (byte) (int) Math.round(rC * 255f);
+        argbFrame[y * argbStride + x * 4 + 1] = (byte) (int) Math.round(gC * 255f);
+        argbFrame[y * argbStride + x * 4 + 2] = (byte) (int) Math.round(bC * 255f);
+        argbFrame[y * argbStride + x * 4 + 3] = (byte) 255;
+      }
+    }
+
+    return argbFrame;
+  }
+
+  private void checkBitmapContent(Bitmap bitmap, int frame) {
+    checkBitmap(bitmap, 1.f);
+
+    byte[] expectedARGB = convertYUVFrameToRBG(TEST_FRAMES[frame]);
+    ByteBuffer bitmapBuffer = ByteBuffer.allocateDirect(bitmap.getByteCount());
+    bitmap.copyPixelsToBuffer(bitmapBuffer);
+
+    Logging.d(TAG, "Expected bitmap content: " + Arrays.toString(expectedARGB));
+    Logging.d(TAG, "Bitmap content: " + Arrays.toString(bitmapBuffer.array()));
+    for (int i = 0; i < expectedARGB.length; i++) {
+      int expected = expectedARGB[i] & 0xFF;
+      int value = bitmapBuffer.get(i) & 0xFF;
+      // Due to unknown conversion differences check value matches +-2.
+      if (value < expected - 2 || value > expected + 2) {
+        fail("Frame doesn't match original frame on byte " + i + ". Expected: " + expected
+            + " Result: " + value);
+      }
+    }
+  }
+
+  private void feedFrame(int i) {
+    eglRenderer.renderFrame(new VideoRenderer.I420Frame(TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT, 0,
+        new int[] {TEST_FRAME_WIDTH, TEST_FRAME_WIDTH / 2, TEST_FRAME_WIDTH / 2}, TEST_FRAMES[i],
+        0));
+  }
+
+  @SmallTest
+  public void testAddFrameListener() throws Exception {
+    waitForSurfaceReady();
+
+    eglRenderer.addFrameListener(testFrameListener, 0.f);
+    feedFrame(0);
+    feedFrame(1);
+    Thread.sleep(RENDER_WAIT_MS);
+    assertEquals(1, testFrameListener.getBitmaps().size());
+    assertNull(testFrameListener.getBitmaps().get(0));
+    eglRenderer.addFrameListener(testFrameListener, 0.f);
+    feedFrame(2);
+    Thread.sleep(RENDER_WAIT_MS);
+    assertEquals(2, testFrameListener.getBitmaps().size());
+    assertNull(testFrameListener.getBitmaps().get(1));
+  }
+
+  @SmallTest
+  public void testAddFrameListenerBitmap() throws Exception {
+    waitForSurfaceReady();
+
+    eglRenderer.addFrameListener(testFrameListener, 1.f);
+    feedFrame(0);
+    Thread.sleep(RENDER_WAIT_MS);
+    assertEquals(1, testFrameListener.getBitmaps().size());
+    checkBitmapContent(testFrameListener.getBitmaps().get(0), 0);
+    eglRenderer.addFrameListener(testFrameListener, 1.f);
+    feedFrame(1);
+    Thread.sleep(RENDER_WAIT_MS);
+    assertEquals(2, testFrameListener.getBitmaps().size());
+    checkBitmapContent(testFrameListener.getBitmaps().get(1), 1);
+  }
+
+  @SmallTest
+  public void testAddFrameListenerBitmapScale() throws Exception {
+    waitForSurfaceReady();
+
+    for (int i = 0; i < 3; ++i) {
+      float scale = i * 0.5f + 0.5f;
+      eglRenderer.addFrameListener(testFrameListener, scale);
+      feedFrame(i);
+      Thread.sleep(RENDER_WAIT_MS);
+      assertEquals(i + 1, testFrameListener.getBitmaps().size());
+      checkBitmap(testFrameListener.getBitmaps().get(i), scale);
+    }
+  }
+}