Android video rendering: Apply SurfaceTexture.getTransformationMatrix()

talk/app/webrtc/java/android/org/webrtc/RendererCommon.java

 /*
  * libjingle
  * Copyright 2015 Google Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright notice,
@@ skipping 10 matching lines @@
  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 package org.webrtc;
 
 import android.graphics.Point;
+import android.graphics.SurfaceTexture;
 import android.opengl.Matrix;
 
 /**
  * Static helper functions for renderer implementations.
  */
 public class RendererCommon {
   /** Interface for reporting rendering events. */
   public static interface RendererEvents {
     /**
      * Callback fired once first frame is rendered.
@@ skipping 12 matching lines @@
   // SCALE_ASPECT_FILL - video frame is scaled to fill the size of the view by
   //    maintaining the aspect ratio. Some portion of the video frame may be
   //    clipped.
   // SCALE_ASPECT_BALANCED - Compromise between FIT and FILL. Video frame will fill as much as
   // possible of the view while maintaining aspect ratio, under the constraint that at least
   // |BALANCED_VISIBLE_FRACTION| of the frame content will be shown.
   public static enum ScalingType { SCALE_ASPECT_FIT, SCALE_ASPECT_FILL, SCALE_ASPECT_BALANCED }
   // The minimum fraction of the frame content that will be shown for |SCALE_ASPECT_BALANCED|.
   // This limits excessive cropping when adjusting display size.
   private static float BALANCED_VISIBLE_FRACTION = 0.5625f;
+  // Matrix with transform y' = 1 - y.
+  private static float[] VERTICAL_FLIP = new float[] {
+      1,  0, 0, 0,
+      0, -1, 0, 0,
+      0,  0, 1, 0,
+      0,  1, 0, 1};

[hbos, 2015/09/08 14:08:47]

nit: Could make this final (even though it wouldn't make its contents const)

[magjed_webrtc, 2015/09/08 15:24:28]

Done.

 
   /**
-   * Calculates a texture transformation matrix based on rotation, mirror, and video vs display
-   * aspect ratio.
+   * Returns matrix that transforms standard coordinates to their proper sampling locations in
+   * the texture. This transform compensates for any properties of the video source that
+   * cause it to appear different from a normalized texture. If the video source is based on
+   * ByteBuffers, pass null in |surfaceTexture|.
    */
-  public static void getTextureMatrix(float[] outputTextureMatrix, float rotationDegree,
-      boolean mirror, float videoAspectRatio, float displayAspectRatio) {
-    // The matrix stack is using post-multiplication, which means that matrix operations:
-    // A; B; C; will end up as A * B * C. When you apply this to a vertex, it will result in:
-    // v' = A * B * C * v, i.e. the last matrix operation is the first thing that affects the
-    // vertex. This is the opposite of what you might expect.
-    Matrix.setIdentityM(outputTextureMatrix, 0);
-    // Move coordinates back to [0,1]x[0,1].
-    Matrix.translateM(outputTextureMatrix, 0, 0.5f, 0.5f, 0.0f);
-    // Rotate frame clockwise in the XY-plane (around the Z-axis).
-    Matrix.rotateM(outputTextureMatrix, 0, -rotationDegree, 0, 0, 1);
-    // Scale one dimension until video and display size have same aspect ratio.
-    if (displayAspectRatio > videoAspectRatio) {
-      Matrix.scaleM(outputTextureMatrix, 0, 1, videoAspectRatio / displayAspectRatio, 1);
+  public static float[] getSamplingMatrix(SurfaceTexture surfaceTexture, float rotationDegree) {
+    final float[] samplingMatrix;
+    if (surfaceTexture == null) {
+      // For ByteBuffers, row 0 specifies the top row, but for a texture, row 0 specifies the
+      // bottom row. Flip the image vertically to compensate for this.
+      samplingMatrix = VERTICAL_FLIP;
     } else {
-      Matrix.scaleM(outputTextureMatrix, 0, displayAspectRatio / videoAspectRatio, 1, 1);
+      samplingMatrix = new float[16];
+      surfaceTexture.getTransformMatrix(samplingMatrix);
     }
-    // TODO(magjed): We currently ignore the texture transform matrix from the SurfaceTexture.
-    // It contains a vertical flip that is hardcoded here instead.
-    Matrix.scaleM(outputTextureMatrix, 0, 1, -1, 1);
-    // Apply optional horizontal flip.
-    if (mirror) {
-      Matrix.scaleM(outputTextureMatrix, 0, -1, 1, 1);
-    }
-    // Center coordinates around origin.
-    Matrix.translateM(outputTextureMatrix, 0, -0.5f, -0.5f, 0.0f);
+    // Clockwise rotation matrix in the XY-plane (around the Z-axis).
+    final float[] rotationMatrix = new float[16];
+    Matrix.setRotateM(rotationMatrix, 0, -rotationDegree, 0, 0, 1);
+    adjustOrigin(rotationMatrix);
+    // Multiply matrices together.
+    final float[] tmpMatrix = new float[16];
+    Matrix.multiplyMM(tmpMatrix, 0, rotationMatrix, 0, samplingMatrix, 0);
+    return tmpMatrix;
   }
 
   /**
+   * Returns layout transformation matrix that applies an optional mirror effect and compensates
+   * for video vs display aspect ratio.
+   */
+  public static float[] getLayoutMatrix(
+      boolean mirror, float videoAspectRatio, float displayAspectRatio) {
+    float scaleX = 1;
+    float scaleY = 1;
+    // Scale X or Y dimension so that video and display size have same aspect ratio.
+    if (displayAspectRatio > videoAspectRatio) {
+      scaleY = videoAspectRatio / displayAspectRatio;
+    } else {
+      scaleX = displayAspectRatio / videoAspectRatio;
+    }
+    // Apply optional horizontal flip.
+    if (mirror) {
+      scaleX *= -1;
+    }
+    final float matrix[] = new float[16];
+    Matrix.setIdentityM(matrix, 0);
+    Matrix.scaleM(matrix, 0, scaleX, scaleY, 1);
+    adjustOrigin(matrix);

[hbos, 2015/09/08 14:08:47]

Does "adjustOrigin" have any effect on a scale matrix?

[magjed_webrtc, 2015/09/08 15:24:28]

Yes. Otherwise it will zoom in/out around the corner of the texture instead of
the center of the texture.

[hbos, 2015/09/09 08:27:54]

Acknowledged.

+    return matrix;
+  }
+
+  /**
    * Calculate display size based on scaling type, video aspect ratio, and maximum display size.
    */
   public static Point getDisplaySize(ScalingType scalingType, float videoAspectRatio,
       int maxDisplayWidth, int maxDisplayHeight) {
     return getDisplaySize(convertScalingTypeToVisibleFraction(scalingType), videoAspectRatio,
         maxDisplayWidth, maxDisplayHeight);
   }
 
   /**
+   * Move |matrix| transformation origin to (0.5, 0.5). This is the origin for texture coordinates
+   * that are in the range 0 to 1.
+   */
+  private static void adjustOrigin(float[] matrix) {
+    // Note that OpenGL is using column-major order.
+    // Pre translate with -0.5 to move coordinates to range [-0.5, 0.5].
+    matrix[12] -= 0.5f * (matrix[0] + matrix[4]);
+    matrix[13] -= 0.5f * (matrix[1] + matrix[5]);
+    // Post translate with 0.5 to move coordinates to range [0, 1].
+    matrix[12] += 0.5f;
+    matrix[13] += 0.5f;
+  }

[hbos, 2015/09/08 14:08:47]

What assumptions, if any, about |matrix| enable this method to do the
translation transform without a fullblown matrix multiplication?

[magjed_webrtc, 2015/09/08 15:24:28]

No assumptions really, this is just two inlined and simplified matrix
multiplications.

[hbos, 2015/09/09 08:27:54]

Acknowledged.

+
+  /**
    * Each scaling type has a one-to-one correspondence to a numeric minimum fraction of the video
    * that must remain visible.
    */
   private static float convertScalingTypeToVisibleFraction(ScalingType scalingType) {
     switch (scalingType) {
       case SCALE_ASPECT_FIT:
         return 1.0f;
       case SCALE_ASPECT_FILL:
         return 0.0f;
       case SCALE_ASPECT_BALANCED:
@@ skipping 14 matching lines @@
       return new Point(maxDisplayWidth, maxDisplayHeight);
     }
     // Each dimension is constrained on max display size and how much we are allowed to crop.
     final int width = Math.min(maxDisplayWidth,
         (int) (maxDisplayHeight / minVisibleFraction * videoAspectRatio));
     final int height = Math.min(maxDisplayHeight,
         (int) (maxDisplayWidth / minVisibleFraction / videoAspectRatio));
     return new Point(width, height);
   }
 }