AppRTCDemo: Render each video in a separate SurfaceView

talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java

 /*
  * libjingle
  * Copyright 2014 Google Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright notice,
@@ skipping 84 matching lines @@
     // Create an OpenGL ES 2.0 context.
     surface.setPreserveEGLContextOnPause(true);
     surface.setEGLContextClientVersion(2);
     surface.setRenderer(this);
     surface.setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY);
 
     yuvImageRenderers = new ArrayList<YuvImageRenderer>();
   }
 
   /**
+   * Each scaling type has a one-to-one correspondence to a numeric minimum fraction of the video
+   * that must remain visible.
+   */
+  public static float convertScalingTypeToVisibleFraction(ScalingType scalingType) {

[AlexG, 2015/08/05 00:47:11]

I think these helpers need to me moved to a separate class - VideoRendererGui
and SurfaceViewRenderer are two independent video renderers and VideoRendererGui
may be gone then. Now new renderer still uses helpers from old renderer.

[magjed_webrtc, 2015/08/07 17:14:50]

Done.

+    switch (scalingType) {
+      case SCALE_ASPECT_FIT:
+        return 1.0f;
+      case SCALE_ASPECT_FILL:
+        return 0.0f;
+      case SCALE_ASPECT_BALANCED:
+        return BALANCED_VISIBLE_FRACTION;
+      default:
+        throw new IllegalArgumentException();
+    }
+  }
+
+  /**
+   * Calculates display size based on minimum fraction of the video that must remain visible,
+   * video aspect ratio, and maximum display size.
+   */
+  public static Point getDisplaySize(float minVisibleFraction, float videoAspectRatio,
+      int maxDisplayWidth, int maxDisplayHeight) {
+    // If there is no constraint on the amount of cropping, fill the allowed display area.
+    if (minVisibleFraction == 0 || videoAspectRatio == 0) {
+      return new Point(maxDisplayWidth, maxDisplayHeight);
+    }
+    // Each dimension is constrained on max display size and how much we are allowed to crop.
+    final int width = Math.min(maxDisplayWidth,
+        (int) (maxDisplayHeight / minVisibleFraction * videoAspectRatio));
+    final int height = Math.min(maxDisplayHeight,
+        (int) (maxDisplayWidth / minVisibleFraction / videoAspectRatio));
+    return new Point(width, height);
+  }
+
+  /**
+   * Calculates a texture transformation matrix based on rotation, mirror, and video vs display
+   * aspect ratio.
+   */
+  public static void getTextureMatrix(float[] outputTextureMatrix, float rotationDegree,
+      boolean mirror, float videoAspectRatio, float displayAspectRatio) {
+    // The matrix stack is using post-multiplication, which means that matrix operations:
+    // A; B; C; will end up as A * B * C. When you apply this to a vertex, it will result in:
+    // v' = A * B * C * v, i.e. the last matrix operation is the first thing that affects the
+    // vertex. This is the opposite of what you might expect.
+    Matrix.setIdentityM(outputTextureMatrix, 0);
+    // Move coordinates back to [0,1]x[0,1].
+    Matrix.translateM(outputTextureMatrix, 0, 0.5f, 0.5f, 0.0f);
+    // Rotate frame clockwise in the XY-plane (around the Z-axis).
+    Matrix.rotateM(outputTextureMatrix, 0, -rotationDegree, 0, 0, 1);
+    // Scale one dimension until video and display size have same aspect ratio.
+    if (displayAspectRatio > videoAspectRatio) {
+      Matrix.scaleM(outputTextureMatrix, 0, 1, videoAspectRatio / displayAspectRatio, 1);
+    } else {
+      Matrix.scaleM(outputTextureMatrix, 0, displayAspectRatio / videoAspectRatio, 1, 1);
+    }
+    // TODO(magjed): We currently ignore the texture transform matrix from the SurfaceTexture.
+    // It contains a vertical flip that is hardcoded here instead.
+    Matrix.scaleM(outputTextureMatrix, 0, 1, -1, 1);
+    // Apply optional horizontal flip.
+    if (mirror) {
+      Matrix.scaleM(outputTextureMatrix, 0, -1, 1, 1);
+    }
+    // Center coordinates around origin.
+    Matrix.translateM(outputTextureMatrix, 0, -0.5f, -0.5f, 0.0f);
+  }
+
+  /**
    * Class used to display stream of YUV420 frames at particular location
    * on a screen. New video frames are sent to display using renderFrame()
    * call.
    */
   private static class YuvImageRenderer implements VideoRenderer.Callbacks {
     private GLSurfaceView surface;
     private int id;
     private int[] yuvTextures = { -1, -1, -1 };
     private int oesTexture = -1;
 
@@ skipping 77 matching lines @@
         GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
             GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
         GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
             GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
         GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
             GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
       }
       GlUtil.checkNoGLES2Error("y/u/v glGenTextures");
     }
 
-    private static float convertScalingTypeToVisibleFraction(ScalingType scalingType) {
-      switch (scalingType) {
-        case SCALE_ASPECT_FIT:
-          return 1.0f;
-        case SCALE_ASPECT_FILL:
-          return 0.0f;
-        case SCALE_ASPECT_BALANCED:
-          return BALANCED_VISIBLE_FRACTION;
-        default:
-          throw new IllegalArgumentException();
-      }
-    }
-
-    private static Point getDisplaySize(float minVisibleFraction, float videoAspectRatio,
-        int maxDisplayWidth, int maxDisplayHeight) {
-      // If there is no constraint on the amount of cropping, fill the allowed display area.
-      if (minVisibleFraction == 0) {
-        return new Point(maxDisplayWidth, maxDisplayHeight);
-      }
-      // Each dimension is constrained on max display size and how much we are allowed to crop.
-      final int width = Math.min(maxDisplayWidth,
-          (int) (maxDisplayHeight / minVisibleFraction * videoAspectRatio));
-      final int height = Math.min(maxDisplayHeight,
-          (int) (maxDisplayWidth / minVisibleFraction / videoAspectRatio));
-      return new Point(width, height);
-    }
-
     private void checkAdjustTextureCoords() {
       synchronized(updateTextureLock) {
         if (!updateTextureProperties) {
           return;
         }
         // Initialize to maximum allowed area. Round to integer coordinates inwards the layout
         // bounding box (ceil left/top and floor right/bottom) to not break constraints.
         displayLayout.set(
             (screenWidth * layoutInPercentage.left + 99) / 100,
             (screenHeight * layoutInPercentage.top + 99) / 100,
             (screenWidth * layoutInPercentage.right) / 100,
             (screenHeight * layoutInPercentage.bottom) / 100);
         Log.d(TAG, "ID: "  + id + ". AdjustTextureCoords. Allowed display size: "
             + displayLayout.width() + " x " + displayLayout.height() + ". Video: " + videoWidth
             + " x " + videoHeight + ". Rotation: " + rotationDegree + ". Mirror: " + mirror);
         final float videoAspectRatio = (rotationDegree % 180 == 0)
             ? (float) videoWidth / videoHeight
             : (float) videoHeight / videoWidth;
         // Adjust display size based on |scalingType|.
         final float minVisibleFraction = convertScalingTypeToVisibleFraction(scalingType);
         final Point displaySize = getDisplaySize(minVisibleFraction, videoAspectRatio,
             displayLayout.width(), displayLayout.height());
         displayLayout.inset((displayLayout.width() - displaySize.x) / 2,
                             (displayLayout.height() - displaySize.y) / 2);
         Log.d(TAG, "  Adjusted display size: " + displayLayout.width() + " x "
             + displayLayout.height());
-        // The matrix stack is using post-multiplication, which means that matrix operations:
-        // A; B; C; will end up as A * B * C. When you apply this to a vertex, it will result in:
-        // v' = A * B * C * v, i.e. the last matrix operation is the first thing that affects the
-        // vertex. This is the opposite of what you might expect.
-        Matrix.setIdentityM(texMatrix, 0);
-        // Move coordinates back to [0,1]x[0,1].
-        Matrix.translateM(texMatrix, 0, 0.5f, 0.5f, 0.0f);
-        // Rotate frame clockwise in the XY-plane (around the Z-axis).
-        Matrix.rotateM(texMatrix, 0, -rotationDegree, 0, 0, 1);
-        // Scale one dimension until video and display size have same aspect ratio.
-        final float displayAspectRatio = (float) displayLayout.width() / displayLayout.height();
-        if (displayAspectRatio > videoAspectRatio) {
-            Matrix.scaleM(texMatrix, 0, 1, videoAspectRatio / displayAspectRatio, 1);
-        } else {
-            Matrix.scaleM(texMatrix, 0, displayAspectRatio / videoAspectRatio, 1, 1);
-        }
-        // TODO(magjed): We currently ignore the texture transform matrix from the SurfaceTexture.
-        // It contains a vertical flip that is hardcoded here instead.
-        Matrix.scaleM(texMatrix, 0, 1, -1, 1);
-        // Apply optional horizontal flip.
-        if (mirror) {
-          Matrix.scaleM(texMatrix, 0, -1, 1, 1);
-        }
-        // Center coordinates around origin.
-        Matrix.translateM(texMatrix, 0, -0.5f, -0.5f, 0.0f);
+        getTextureMatrix(texMatrix, rotationDegree, mirror, videoAspectRatio,
+            (float) displayLayout.width() / displayLayout.height());
         updateTextureProperties = false;
         Log.d(TAG, "  AdjustTextureCoords done");
       }
     }
 
     private void draw(GlRectDrawer drawer) {
       if (!seenFrame) {
         // No frame received yet - nothing to render.
         return;
       }
@@ skipping 118 matching lines @@
 
         this.videoWidth = videoWidth;
         this.videoHeight = videoHeight;
         rotationDegree = rotation;
         int[] strides = { videoWidth, videoWidth / 2, videoWidth / 2  };
 
         // Clear rendering queue.
         frameToRenderQueue.poll();
         // Re-allocate / allocate the frame.
         yuvFrameToRender = new I420Frame(videoWidth, videoHeight, rotationDegree,
-                                         strides, null);
+                                         strides, null, 0);
         textureFrameToRender = new I420Frame(videoWidth, videoHeight, rotationDegree,
-                                             null, -1);
+                                             null, -1, 0);
         updateTextureProperties = true;
         Log.d(TAG, "  YuvImageRenderer.setSize done.");
       }
     }
 
     @Override
     public synchronized void renderFrame(I420Frame frame) {
       setSize(frame.width, frame.height, frame.rotationDegree);
       long now = System.nanoTime();
       framesReceived++;
       // Skip rendering of this frame if setSize() was not called.
       if (yuvFrameToRender == null || textureFrameToRender == null) {
         framesDropped++;
+        VideoRenderer.renderFrameDone(frame);
         return;
       }
       // Check input frame parameters.
       if (frame.yuvFrame) {
         if (frame.yuvStrides[0] < frame.width ||
             frame.yuvStrides[1] < frame.width / 2 ||
             frame.yuvStrides[2] < frame.width / 2) {
           Log.e(TAG, "Incorrect strides " + frame.yuvStrides[0] + ", " +
               frame.yuvStrides[1] + ", " + frame.yuvStrides[2]);
+          VideoRenderer.renderFrameDone(frame);
           return;
         }
         // Check incoming frame dimensions.
         if (frame.width != yuvFrameToRender.width ||
             frame.height != yuvFrameToRender.height) {
           throw new RuntimeException("Wrong frame size " +
               frame.width + " x " + frame.height);
         }
       }
 
       if (frameToRenderQueue.size() > 0) {
         // Skip rendering of this frame if previous frame was not rendered yet.
         framesDropped++;
+        VideoRenderer.renderFrameDone(frame);
         return;
       }
 
       // Create a local copy of the frame.
       if (frame.yuvFrame) {
         yuvFrameToRender.copyFrom(frame);
         rendererType = RendererType.RENDERER_YUV;
         frameToRenderQueue.offer(yuvFrameToRender);
       } else {
         textureFrameToRender.copyFrom(frame);
         rendererType = RendererType.RENDERER_TEXTURE;
         frameToRenderQueue.offer(textureFrameToRender);
       }
       copyTimeNs += (System.nanoTime() - now);
       seenFrame = true;
+      VideoRenderer.renderFrameDone(frame);
 
       // Request rendering.
       surface.requestRender();
     }
 
     // TODO(guoweis): Remove this once chrome code base is updated.
     @Override
     public boolean canApplyRotation() {
       return true;
     }
@@ skipping 153 matching lines @@
     GLES20.glViewport(0, 0, screenWidth, screenHeight);
     GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
     synchronized (yuvImageRenderers) {
       for (YuvImageRenderer yuvImageRenderer : yuvImageRenderers) {
         yuvImageRenderer.draw(drawer);
       }
     }
   }
 
 }