Android: Move YuvConverter to its own file

webrtc/api/android/java/src/org/webrtc/YuvConverter.java

Index: webrtc/api/android/java/src/org/webrtc/YuvConverter.java
diff --git a/webrtc/api/android/java/src/org/webrtc/GlRectDrawer.java b/webrtc/api/android/java/src/org/webrtc/YuvConverter.java
similarity index 19%
copy from webrtc/api/android/java/src/org/webrtc/GlRectDrawer.java
copy to webrtc/api/android/java/src/org/webrtc/YuvConverter.java
index c81e6e8d938591c490947b097e03ac444e9802ae..1203d86515840db873750cf555b3196844ec3812 100644
--- a/webrtc/api/android/java/src/org/webrtc/GlRectDrawer.java
+++ b/webrtc/api/android/java/src/org/webrtc/YuvConverter.java
@@ -12,22 +12,36 @@ package org.webrtc;
 
 import android.opengl.GLES11Ext;
 import android.opengl.GLES20;
-
+import java.nio.ByteBuffer;
 import java.nio.FloatBuffer;
-import java.util.IdentityHashMap;
-import java.util.Map;
 
 /**
- * Helper class to draw an opaque quad on the target viewport location. Rotation, mirror, and
- * cropping is specified using a 4x4 texture coordinate transform matrix. The frame input can either
- * be an OES texture or YUV textures in I420 format. The GL state must be preserved between draw
- * calls, this is intentional to maximize performance. The function release() must be called
- * manually to free the resources held by this object.
+ * Class for converting OES textures to a YUV ByteBuffer.
  */
-public class GlRectDrawer implements RendererCommon.GlDrawer {
+class YuvConverter {
+  private final EglBase eglBase;
+  private final GlShader shader;
+  private boolean released = false;
+
+  // Vertex coordinates in Normalized Device Coordinates, i.e.
+  // (-1, -1) is bottom-left and (1, 1) is top-right.
+  private static final FloatBuffer DEVICE_RECTANGLE = GlUtil.createFloatBuffer(new float[] {
+      -1.0f, -1.0f, // Bottom left.
+      1.0f, -1.0f, // Bottom right.
+      -1.0f, 1.0f, // Top left.
+      1.0f, 1.0f, // Top right.
+  });
+
+  // Texture coordinates - (0, 0) is bottom-left and (1, 1) is top-right.
+  private static final FloatBuffer TEXTURE_RECTANGLE = GlUtil.createFloatBuffer(new float[] {
+      0.0f, 0.0f, // Bottom left.
+      1.0f, 0.0f, // Bottom right.
+      0.0f, 1.0f, // Top left.
+      1.0f, 1.0f // Top right.
+  });
+
   // clang-format off
-  // Simple vertex shader, used for both YUV and OES.
-  private static final String VERTEX_SHADER_STRING =
+  private static final String VERTEX_SHADER =
         "varying vec2 interp_tc;\n"
       + "attribute vec4 in_pos;\n"
       + "attribute vec4 in_tc;\n"
@@ -39,173 +53,170 @@ public class GlRectDrawer implements RendererCommon.GlDrawer {
       + "    interp_tc = (texMatrix * in_tc).xy;\n"
       + "}\n";
 
-  private static final String YUV_FRAGMENT_SHADER_STRING =
-        "precision mediump float;\n"
-      + "varying vec2 interp_tc;\n"
-      + "\n"
-      + "uniform sampler2D y_tex;\n"
-      + "uniform sampler2D u_tex;\n"
-      + "uniform sampler2D v_tex;\n"
-      + "\n"
-      + "void main() {\n"
-      // CSC according to http://www.fourcc.org/fccyvrgb.php
-      + "  float y = texture2D(y_tex, interp_tc).r;\n"
-      + "  float u = texture2D(u_tex, interp_tc).r - 0.5;\n"
-      + "  float v = texture2D(v_tex, interp_tc).r - 0.5;\n"
-      + "  gl_FragColor = vec4(y + 1.403 * v, "
-      + "                      y - 0.344 * u - 0.714 * v, "
-      + "                      y + 1.77 * u, 1);\n"
-      + "}\n";
-
-  private static final String RGB_FRAGMENT_SHADER_STRING =
-        "precision mediump float;\n"
-      + "varying vec2 interp_tc;\n"
-      + "\n"
-      + "uniform sampler2D rgb_tex;\n"
-      + "\n"
-      + "void main() {\n"
-      + "  gl_FragColor = texture2D(rgb_tex, interp_tc);\n"
-      + "}\n";
-
-  private static final String OES_FRAGMENT_SHADER_STRING =
+  private static final String FRAGMENT_SHADER =
         "#extension GL_OES_EGL_image_external : require\n"
       + "precision mediump float;\n"
       + "varying vec2 interp_tc;\n"
       + "\n"
-      + "uniform samplerExternalOES oes_tex;\n"
+      + "uniform samplerExternalOES oesTex;\n"
+      // Difference in texture coordinate corresponding to one
+      // sub-pixel in the x direction.
+      + "uniform vec2 xUnit;\n"
+      // Color conversion coefficients, including constant term
+      + "uniform vec4 coeffs;\n"
       + "\n"
       + "void main() {\n"
-      + "  gl_FragColor = texture2D(oes_tex, interp_tc);\n"
+      // Since the alpha read from the texture is always 1, this could
+      // be written as a mat4 x vec4 multiply. However, that seems to
+      // give a worse framerate, possibly because the additional
+      // multiplies by 1.0 consume resources. TODO(nisse): Could also
+      // try to do it as a vec3 x mat3x4, followed by an add in of a
+      // constant vector.
+      + "  gl_FragColor.r = coeffs.a + dot(coeffs.rgb,\n"
+      + "      texture2D(oesTex, interp_tc - 1.5 * xUnit).rgb);\n"
+      + "  gl_FragColor.g = coeffs.a + dot(coeffs.rgb,\n"
+      + "      texture2D(oesTex, interp_tc - 0.5 * xUnit).rgb);\n"
+      + "  gl_FragColor.b = coeffs.a + dot(coeffs.rgb,\n"
+      + "      texture2D(oesTex, interp_tc + 0.5 * xUnit).rgb);\n"
+      + "  gl_FragColor.a = coeffs.a + dot(coeffs.rgb,\n"
+      + "      texture2D(oesTex, interp_tc + 1.5 * xUnit).rgb);\n"
       + "}\n";
   // clang-format on
 
-  // Vertex coordinates in Normalized Device Coordinates, i.e. (-1, -1) is bottom-left and (1, 1) is
-  // top-right.
-  private static final FloatBuffer FULL_RECTANGLE_BUF = GlUtil.createFloatBuffer(new float[] {
-      -1.0f, -1.0f, // Bottom left.
-      1.0f, -1.0f, // Bottom right.
-      -1.0f, 1.0f, // Top left.
-      1.0f, 1.0f, // Top right.
-  });
+  private int texMatrixLoc;
+  private int xUnitLoc;
+  private int coeffsLoc;
+
+  public YuvConverter(EglBase.Context sharedContext) {
+    eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_RGBA_BUFFER);
+    eglBase.createDummyPbufferSurface();
+    eglBase.makeCurrent();
+
+    shader = new GlShader(VERTEX_SHADER, FRAGMENT_SHADER);
+    shader.useProgram();
+    texMatrixLoc = shader.getUniformLocation("texMatrix");
+    xUnitLoc = shader.getUniformLocation("xUnit");
+    coeffsLoc = shader.getUniformLocation("coeffs");
+    GLES20.glUniform1i(shader.getUniformLocation("oesTex"), 0);
+    GlUtil.checkNoGLES2Error("Initialize fragment shader uniform values.");
+    // Initialize vertex shader attributes.
+    shader.setVertexAttribArray("in_pos", 2, DEVICE_RECTANGLE);
+    // If the width is not a multiple of 4 pixels, the texture
+    // will be scaled up slightly and clipped at the right border.
+    shader.setVertexAttribArray("in_tc", 2, TEXTURE_RECTANGLE);
+    eglBase.detachCurrent();
+  }
 
-  // Texture coordinates - (0, 0) is bottom-left and (1, 1) is top-right.
-  private static final FloatBuffer FULL_RECTANGLE_TEX_BUF = GlUtil.createFloatBuffer(new float[] {
-      0.0f, 0.0f, // Bottom left.
-      1.0f, 0.0f, // Bottom right.
-      0.0f, 1.0f, // Top left.
-      1.0f, 1.0f // Top right.
-  });
+  synchronized public void convert(
+      ByteBuffer buf, int width, int height, int stride, int textureId, float[] transformMatrix) {
+    if (released) {
+      throw new IllegalStateException("YuvConverter.convert called on released object");
+    }
+
+    // We draw into a buffer laid out like
+    //
+    //    +---------+
+    //    |         |
+    //    |  Y      |
+    //    |         |
+    //    |         |
+    //    +----+----+
+    //    | U  | V  |
+    //    |    |    |
+    //    +----+----+
+    //
+    // In memory, we use the same stride for all of Y, U and V. The
+    // U data starts at offset |height| * |stride| from the Y data,
+    // and the V data starts at at offset |stride/2| from the U
+    // data, with rows of U and V data alternating.
+    //
+    // Now, it would have made sense to allocate a pixel buffer with
+    // a single byte per pixel (EGL10.EGL_COLOR_BUFFER_TYPE,
+    // EGL10.EGL_LUMINANCE_BUFFER,), but that seems to be
+    // unsupported by devices. So do the following hack: Allocate an
+    // RGBA buffer, of width |stride|/4. To render each of these
+    // large pixels, sample the texture at 4 different x coordinates
+    // and store the results in the four components.
+    //
+    // Since the V data needs to start on a boundary of such a
+    // larger pixel, it is not sufficient that |stride| is even, it
+    // has to be a multiple of 8 pixels.
+
+    if (stride % 8 != 0) {
+      throw new IllegalArgumentException("Invalid stride, must be a multiple of 8");
+    }
+    if (stride < width) {
+      throw new IllegalArgumentException("Invalid stride, must >= width");
+    }
 
-  private static class Shader {
-    public final GlShader glShader;
-    public final int texMatrixLocation;
+    int y_width = (width + 3) / 4;
+    int uv_width = (width + 7) / 8;
+    int uv_height = (height + 1) / 2;
+    int total_height = height + uv_height;
+    int size = stride * total_height;
 
-    public Shader(String fragmentShader) {
-      this.glShader = new GlShader(VERTEX_SHADER_STRING, fragmentShader);
-      this.texMatrixLocation = glShader.getUniformLocation("texMatrix");
+    if (buf.capacity() < size) {
+      throw new IllegalArgumentException("YuvConverter.convert called with too small buffer");
+    }
+    // Produce a frame buffer starting at top-left corner, not
+    // bottom-left.
+    transformMatrix =
+        RendererCommon.multiplyMatrices(transformMatrix, RendererCommon.verticalFlipMatrix());
+
+    // Create new pBuffferSurface with the correct size if needed.
+    if (eglBase.hasSurface()) {
+      if (eglBase.surfaceWidth() != stride / 4 || eglBase.surfaceHeight() != total_height) {
+        eglBase.releaseSurface();
+        eglBase.createPbufferSurface(stride / 4, total_height);
+      }
+    } else {
+      eglBase.createPbufferSurface(stride / 4, total_height);
     }
-  }
 
-  // The keys are one of the fragments shaders above.
-  private final Map<String, Shader> shaders = new IdentityHashMap<String, Shader>();
-
-  /**
-   * Draw an OES texture frame with specified texture transformation matrix. Required resources are
-   * allocated at the first call to this function.
-   */
-  @Override
-  public void drawOes(int oesTextureId, float[] texMatrix, int frameWidth, int frameHeight,
-      int viewportX, int viewportY, int viewportWidth, int viewportHeight) {
-    prepareShader(OES_FRAGMENT_SHADER_STRING, texMatrix);
-    GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
-    // updateTexImage() may be called from another thread in another EGL context, so we need to
-    // bind/unbind the texture in each draw call so that GLES understads it's a new texture.
-    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, oesTextureId);
-    drawRectangle(viewportX, viewportY, viewportWidth, viewportHeight);
-    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0);
-  }
+    eglBase.makeCurrent();
 
-  /**
-   * Draw a RGB(A) texture frame with specified texture transformation matrix. Required resources
-   * are allocated at the first call to this function.
-   */
-  @Override
-  public void drawRgb(int textureId, float[] texMatrix, int frameWidth, int frameHeight,
-      int viewportX, int viewportY, int viewportWidth, int viewportHeight) {
-    prepareShader(RGB_FRAGMENT_SHADER_STRING, texMatrix);
     GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
-    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId);
-    drawRectangle(viewportX, viewportY, viewportWidth, viewportHeight);
-    // Unbind the texture as a precaution.
-    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
-  }
+    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId);
+    GLES20.glUniformMatrix4fv(texMatrixLoc, 1, false, transformMatrix, 0);
+
+    // Draw Y
+    GLES20.glViewport(0, 0, y_width, height);
+    // Matrix * (1;0;0;0) / width. Note that opengl uses column major order.
+    GLES20.glUniform2f(xUnitLoc, transformMatrix[0] / width, transformMatrix[1] / width);
+    // Y'UV444 to RGB888, see
+    // https://en.wikipedia.org/wiki/YUV#Y.27UV444_to_RGB888_conversion.
+    // We use the ITU-R coefficients for U and V */
+    GLES20.glUniform4f(coeffsLoc, 0.299f, 0.587f, 0.114f, 0.0f);
+    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
 
-  /**
-   * Draw a YUV frame with specified texture transformation matrix. Required resources are
-   * allocated at the first call to this function.
-   */
-  @Override
-  public void drawYuv(int[] yuvTextures, float[] texMatrix, int frameWidth, int frameHeight,
-      int viewportX, int viewportY, int viewportWidth, int viewportHeight) {
-    prepareShader(YUV_FRAGMENT_SHADER_STRING, texMatrix);
-    // Bind the textures.
-    for (int i = 0; i < 3; ++i) {
-      GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
-      GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
-    }
-    drawRectangle(viewportX, viewportY, viewportWidth, viewportHeight);
-    // Unbind the textures as a precaution..
-    for (int i = 0; i < 3; ++i) {
-      GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
-      GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
-    }
-  }
+    // Draw U
+    GLES20.glViewport(0, height, uv_width, uv_height);
+    // Matrix * (1;0;0;0) / (width / 2). Note that opengl uses column major order.
+    GLES20.glUniform2f(
+        xUnitLoc, 2.0f * transformMatrix[0] / width, 2.0f * transformMatrix[1] / width);
+    GLES20.glUniform4f(coeffsLoc, -0.169f, -0.331f, 0.499f, 0.5f);
+    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
 
-  private void drawRectangle(int x, int y, int width, int height) {
-    // Draw quad.
-    GLES20.glViewport(x, y, width, height);
+    // Draw V
+    GLES20.glViewport(stride / 8, height, uv_width, uv_height);
+    GLES20.glUniform4f(coeffsLoc, 0.499f, -0.418f, -0.0813f, 0.5f);
     GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
-  }
 
-  private void prepareShader(String fragmentShader, float[] texMatrix) {
-    final Shader shader;
-    if (shaders.containsKey(fragmentShader)) {
-      shader = shaders.get(fragmentShader);
-    } else {
-      // Lazy allocation.
-      shader = new Shader(fragmentShader);
-      shaders.put(fragmentShader, shader);
-      shader.glShader.useProgram();
-      // Initialize fragment shader uniform values.
-      if (fragmentShader == YUV_FRAGMENT_SHADER_STRING) {
-        GLES20.glUniform1i(shader.glShader.getUniformLocation("y_tex"), 0);
-        GLES20.glUniform1i(shader.glShader.getUniformLocation("u_tex"), 1);
-        GLES20.glUniform1i(shader.glShader.getUniformLocation("v_tex"), 2);
-      } else if (fragmentShader == RGB_FRAGMENT_SHADER_STRING) {
-        GLES20.glUniform1i(shader.glShader.getUniformLocation("rgb_tex"), 0);
-      } else if (fragmentShader == OES_FRAGMENT_SHADER_STRING) {
-        GLES20.glUniform1i(shader.glShader.getUniformLocation("oes_tex"), 0);
-      } else {
-        throw new IllegalStateException("Unknown fragment shader: " + fragmentShader);
-      }
-      GlUtil.checkNoGLES2Error("Initialize fragment shader uniform values.");
-      // Initialize vertex shader attributes.
-      shader.glShader.setVertexAttribArray("in_pos", 2, FULL_RECTANGLE_BUF);
-      shader.glShader.setVertexAttribArray("in_tc", 2, FULL_RECTANGLE_TEX_BUF);
-    }
-    shader.glShader.useProgram();
-    // Copy the texture transformation matrix over.
-    GLES20.glUniformMatrix4fv(shader.texMatrixLocation, 1, false, texMatrix, 0);
+    GLES20.glReadPixels(
+        0, 0, stride / 4, total_height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, buf);
+
+    GlUtil.checkNoGLES2Error("YuvConverter.convert");
+
+    // Unbind texture. Reportedly needed on some devices to get
+    // the texture updated from the camera.
+    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0);
+    eglBase.detachCurrent();
   }
 
-  /**
-   * Release all GLES resources. This needs to be done manually, otherwise the resources are leaked.
-   */
-  @Override
-  public void release() {
-    for (Shader shader : shaders.values()) {
-      shader.glShader.release();
-    }
-    shaders.clear();
+  synchronized public void release() {
+    released = true;
+    eglBase.makeCurrent();
+    shader.release();
+    eglBase.release();
   }
 }