AppRTCDemo: Render each video in a separate SurfaceView

talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java

 /*
  * libjingle
  * Copyright 2014 Google Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright notice,
@@ skipping 24 matching lines @@
 import javax.microedition.khronos.opengles.GL10;
 
 import android.annotation.SuppressLint;
 import android.graphics.Point;
 import android.graphics.Rect;
 import android.graphics.SurfaceTexture;
 import android.opengl.EGL14;
 import android.opengl.EGLContext;
 import android.opengl.GLES20;
 import android.opengl.GLSurfaceView;
-import android.opengl.Matrix;
 import android.util.Log;
 
 import org.webrtc.VideoRenderer.I420Frame;
 
 /**
  * Efficiently renders YUV frames using the GPU for CSC.
  * Clients will want first to call setView() to pass GLSurfaceView
  * and then for each video stream either create instance of VideoRenderer using
  * createGui() call or VideoRenderer.Callbacks interface using create() call.
  * Only one instance of the class can be created.
  */
 public class VideoRendererGui implements GLSurfaceView.Renderer {
   private static VideoRendererGui instance = null;
   private static Runnable eglContextReady = null;
   private static final String TAG = "VideoRendererGui";
   private GLSurfaceView surface;
   private static EGLContext eglContext = null;
   // Indicates if SurfaceView.Renderer.onSurfaceCreated was called.
   // If true then for every newly created yuv image renderer createTexture()
   // should be called. The variable is accessed on multiple threads and
   // all accesses are synchronized on yuvImageRenderers' object lock.
   private boolean onSurfaceCreatedCalled;
   private int screenWidth;
   private int screenHeight;
   // List of yuv renderers.
   private ArrayList<YuvImageRenderer> yuvImageRenderers;
   private GlRectDrawer drawer;
-  // The minimum fraction of the frame content that will be shown for |SCALE_ASPECT_BALANCED|.
-  // This limits excessive cropping when adjusting display size.
-  private static float BALANCED_VISIBLE_FRACTION = 0.56f;
-  // Types of video scaling:
-  // SCALE_ASPECT_FIT - video frame is scaled to fit the size of the view by
-  //    maintaining the aspect ratio (black borders may be displayed).
-  // SCALE_ASPECT_FILL - video frame is scaled to fill the size of the view by
-  //    maintaining the aspect ratio. Some portion of the video frame may be
-  //    clipped.
-  // SCALE_ASPECT_BALANCED - Compromise between FIT and FILL. Video frame will fill as much as
-  // possible of the view while maintaining aspect ratio, under the constraint that at least
-  // |BALANCED_VISIBLE_FRACTION| of the frame content will be shown.
-  public static enum ScalingType
-      { SCALE_ASPECT_FIT, SCALE_ASPECT_FILL, SCALE_ASPECT_BALANCED }
   private static final int EGL14_SDK_VERSION =
       android.os.Build.VERSION_CODES.JELLY_BEAN_MR1;
   // Current SDK version.
   private static final int CURRENT_SDK_VERSION =
       android.os.Build.VERSION.SDK_INT;
 
   private VideoRendererGui(GLSurfaceView surface) {
     this.surface = surface;
     // Create an OpenGL ES 2.0 context.
     surface.setPreserveEGLContextOnPause(true);
@@ skipping 19 matching lines @@
     // an offer (writing I420Frame to render) and early-returns (recording
     // a dropped frame) if that queue is full. draw() call does a peek(),
     // copies frame to texture and then removes it from a queue using poll().
     LinkedBlockingQueue<I420Frame> frameToRenderQueue;
     // Local copy of incoming video frame.
     private I420Frame yuvFrameToRender;
     private I420Frame textureFrameToRender;
     // Type of video frame used for recent frame rendering.
     private static enum RendererType { RENDERER_YUV, RENDERER_TEXTURE };
     private RendererType rendererType;
-    private ScalingType scalingType;
+    private RendererCommon.ScalingType scalingType;
     private boolean mirror;
     // Flag if renderFrame() was ever called.
     boolean seenFrame;
     // Total number of video frames received in renderFrame() call.
     private int framesReceived;
     // Number of video frames dropped by renderFrame() because previous
     // frame has not been rendered yet.
     private int framesDropped;
     // Number of rendered video frames.
     private int framesRendered;
@@ skipping 22 matching lines @@
     private int videoWidth;
     private int videoHeight;
 
     // This is the degree that the frame should be rotated clockwisely to have
     // it rendered up right.
     private int rotationDegree;
 
     private YuvImageRenderer(
         GLSurfaceView surface, int id,
         int x, int y, int width, int height,
-        ScalingType scalingType, boolean mirror) {
+        RendererCommon.ScalingType scalingType, boolean mirror) {
       Log.d(TAG, "YuvImageRenderer.Create id: " + id);
       this.surface = surface;
       this.id = id;
       this.scalingType = scalingType;
       this.mirror = mirror;
       frameToRenderQueue = new LinkedBlockingQueue<I420Frame>(1);
       layoutInPercentage = new Rect(x, y, Math.min(100, x + width), Math.min(100, y + height));
       updateTextureProperties = false;
       rotationDegree = 0;
     }
@@ skipping 12 matching lines @@
         GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
             GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
         GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
             GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
         GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,
             GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
       }
       GlUtil.checkNoGLES2Error("y/u/v glGenTextures");
     }
 
-    private static float convertScalingTypeToVisibleFraction(ScalingType scalingType) {
-      switch (scalingType) {
-        case SCALE_ASPECT_FIT:
-          return 1.0f;
-        case SCALE_ASPECT_FILL:
-          return 0.0f;
-        case SCALE_ASPECT_BALANCED:
-          return BALANCED_VISIBLE_FRACTION;
-        default:
-          throw new IllegalArgumentException();
-      }
-    }
-
-    private static Point getDisplaySize(float minVisibleFraction, float videoAspectRatio,
-        int maxDisplayWidth, int maxDisplayHeight) {
-      // If there is no constraint on the amount of cropping, fill the allowed display area.
-      if (minVisibleFraction == 0) {
-        return new Point(maxDisplayWidth, maxDisplayHeight);
-      }
-      // Each dimension is constrained on max display size and how much we are allowed to crop.
-      final int width = Math.min(maxDisplayWidth,
-          (int) (maxDisplayHeight / minVisibleFraction * videoAspectRatio));
-      final int height = Math.min(maxDisplayHeight,
-          (int) (maxDisplayWidth / minVisibleFraction / videoAspectRatio));
-      return new Point(width, height);
-    }
-
     private void checkAdjustTextureCoords() {
       synchronized(updateTextureLock) {
         if (!updateTextureProperties) {
           return;
         }
         // Initialize to maximum allowed area. Round to integer coordinates inwards the layout
         // bounding box (ceil left/top and floor right/bottom) to not break constraints.
         displayLayout.set(
             (screenWidth * layoutInPercentage.left + 99) / 100,
             (screenHeight * layoutInPercentage.top + 99) / 100,
             (screenWidth * layoutInPercentage.right) / 100,
             (screenHeight * layoutInPercentage.bottom) / 100);
         Log.d(TAG, "ID: "  + id + ". AdjustTextureCoords. Allowed display size: "
             + displayLayout.width() + " x " + displayLayout.height() + ". Video: " + videoWidth
             + " x " + videoHeight + ". Rotation: " + rotationDegree + ". Mirror: " + mirror);
         final float videoAspectRatio = (rotationDegree % 180 == 0)
             ? (float) videoWidth / videoHeight
             : (float) videoHeight / videoWidth;
         // Adjust display size based on |scalingType|.
-        final float minVisibleFraction = convertScalingTypeToVisibleFraction(scalingType);
-        final Point displaySize = getDisplaySize(minVisibleFraction, videoAspectRatio,
-            displayLayout.width(), displayLayout.height());
+        final Point displaySize = RendererCommon.getDisplaySize(scalingType,
+            videoAspectRatio, displayLayout.width(), displayLayout.height());
         displayLayout.inset((displayLayout.width() - displaySize.x) / 2,
                             (displayLayout.height() - displaySize.y) / 2);
         Log.d(TAG, "  Adjusted display size: " + displayLayout.width() + " x "
             + displayLayout.height());
-        // The matrix stack is using post-multiplication, which means that matrix operations:
-        // A; B; C; will end up as A * B * C. When you apply this to a vertex, it will result in:
-        // v' = A * B * C * v, i.e. the last matrix operation is the first thing that affects the
-        // vertex. This is the opposite of what you might expect.
-        Matrix.setIdentityM(texMatrix, 0);
-        // Move coordinates back to [0,1]x[0,1].
-        Matrix.translateM(texMatrix, 0, 0.5f, 0.5f, 0.0f);
-        // Rotate frame clockwise in the XY-plane (around the Z-axis).
-        Matrix.rotateM(texMatrix, 0, -rotationDegree, 0, 0, 1);
-        // Scale one dimension until video and display size have same aspect ratio.
-        final float displayAspectRatio = (float) displayLayout.width() / displayLayout.height();
-        if (displayAspectRatio > videoAspectRatio) {
-            Matrix.scaleM(texMatrix, 0, 1, videoAspectRatio / displayAspectRatio, 1);
-        } else {
-            Matrix.scaleM(texMatrix, 0, displayAspectRatio / videoAspectRatio, 1, 1);
-        }
-        // TODO(magjed): We currently ignore the texture transform matrix from the SurfaceTexture.
-        // It contains a vertical flip that is hardcoded here instead.
-        Matrix.scaleM(texMatrix, 0, 1, -1, 1);
-        // Apply optional horizontal flip.
-        if (mirror) {
-          Matrix.scaleM(texMatrix, 0, -1, 1, 1);
-        }
-        // Center coordinates around origin.
-        Matrix.translateM(texMatrix, 0, -0.5f, -0.5f, 0.0f);
+        RendererCommon.getTextureMatrix(texMatrix, rotationDegree, mirror, videoAspectRatio,
+            (float) displayLayout.width() / displayLayout.height());
         updateTextureProperties = false;
         Log.d(TAG, "  AdjustTextureCoords done");
       }
     }
 
     private void draw(GlRectDrawer drawer) {
       if (!seenFrame) {
         // No frame received yet - nothing to render.
         return;
       }
@@ skipping 78 matching lines @@
         }
         Log.d(TAG, "ID: " + id + ". YuvImageRenderer.setScreenSize: " +
             screenWidth + " x " + screenHeight);
         this.screenWidth = screenWidth;
         this.screenHeight = screenHeight;
         updateTextureProperties = true;
       }
     }
 
     public void setPosition(int x, int y, int width, int height,
-        ScalingType scalingType, boolean mirror) {
+        RendererCommon.ScalingType scalingType, boolean mirror) {
       final Rect layoutInPercentage =
           new Rect(x, y, Math.min(100, x + width), Math.min(100, y + height));
       synchronized(updateTextureLock) {
         if (layoutInPercentage.equals(this.layoutInPercentage) && scalingType == this.scalingType
             && mirror == this.mirror) {
           return;
         }
         Log.d(TAG, "ID: " + id + ". YuvImageRenderer.setPosition: (" + x + ", " + y +
             ") " +  width + " x " + height + ". Scaling: " + scalingType +
             ". Mirror: " + mirror);
@@ skipping 19 matching lines @@
 
         this.videoWidth = videoWidth;
         this.videoHeight = videoHeight;
         rotationDegree = rotation;
         int[] strides = { videoWidth, videoWidth / 2, videoWidth / 2  };
 
         // Clear rendering queue.
         frameToRenderQueue.poll();
         // Re-allocate / allocate the frame.
         yuvFrameToRender = new I420Frame(videoWidth, videoHeight, rotationDegree,
-                                         strides, null);
+                                         strides, null, 0);
         textureFrameToRender = new I420Frame(videoWidth, videoHeight, rotationDegree,
-                                             null, -1);
+                                             null, -1, 0);
         updateTextureProperties = true;
         Log.d(TAG, "  YuvImageRenderer.setSize done.");
       }
     }
 
     @Override
     public synchronized void renderFrame(I420Frame frame) {
       setSize(frame.width, frame.height, frame.rotationDegree);
       long now = System.nanoTime();
       framesReceived++;
       // Skip rendering of this frame if setSize() was not called.
       if (yuvFrameToRender == null || textureFrameToRender == null) {
         framesDropped++;
+        VideoRenderer.renderFrameDone(frame);
         return;
       }
       // Check input frame parameters.
       if (frame.yuvFrame) {
         if (frame.yuvStrides[0] < frame.width ||
             frame.yuvStrides[1] < frame.width / 2 ||
             frame.yuvStrides[2] < frame.width / 2) {
           Log.e(TAG, "Incorrect strides " + frame.yuvStrides[0] + ", " +
               frame.yuvStrides[1] + ", " + frame.yuvStrides[2]);
+          VideoRenderer.renderFrameDone(frame);
           return;
         }
         // Check incoming frame dimensions.
         if (frame.width != yuvFrameToRender.width ||
             frame.height != yuvFrameToRender.height) {
           throw new RuntimeException("Wrong frame size " +
               frame.width + " x " + frame.height);
         }
       }
 
       if (frameToRenderQueue.size() > 0) {
         // Skip rendering of this frame if previous frame was not rendered yet.
         framesDropped++;
+        VideoRenderer.renderFrameDone(frame);
         return;
       }
 
       // Create a local copy of the frame.
       if (frame.yuvFrame) {
         yuvFrameToRender.copyFrom(frame);
         rendererType = RendererType.RENDERER_YUV;
         frameToRenderQueue.offer(yuvFrameToRender);
       } else {
         textureFrameToRender.copyFrom(frame);
         rendererType = RendererType.RENDERER_TEXTURE;
         frameToRenderQueue.offer(textureFrameToRender);
       }
       copyTimeNs += (System.nanoTime() - now);
       seenFrame = true;
+      VideoRenderer.renderFrameDone(frame);
 
       // Request rendering.
       surface.requestRender();
     }
 
     // TODO(guoweis): Remove this once chrome code base is updated.
     @Override
     public boolean canApplyRotation() {
       return true;
     }
   }
 
   /** Passes GLSurfaceView to video renderer. */
   public static void setView(GLSurfaceView surface,
       Runnable eglContextReadyCallback) {
     Log.d(TAG, "VideoRendererGui.setView");
     instance = new VideoRendererGui(surface);
     eglContextReady = eglContextReadyCallback;
   }
 
   public static EGLContext getEGLContext() {
     return eglContext;
   }
 
   /**
    * Creates VideoRenderer with top left corner at (x, y) and resolution
    * (width, height). All parameters are in percentage of screen resolution.
    */
   public static VideoRenderer createGui(int x, int y, int width, int height,
-      ScalingType scalingType, boolean mirror) throws Exception {
+      RendererCommon.ScalingType scalingType, boolean mirror) throws Exception {
     YuvImageRenderer javaGuiRenderer = create(
         x, y, width, height, scalingType, mirror);
     return new VideoRenderer(javaGuiRenderer);
   }
 
   public static VideoRenderer.Callbacks createGuiRenderer(
       int x, int y, int width, int height,
-      ScalingType scalingType, boolean mirror) {
+      RendererCommon.ScalingType scalingType, boolean mirror) {
     return create(x, y, width, height, scalingType, mirror);
   }
 
   /**
    * Creates VideoRenderer.Callbacks with top left corner at (x, y) and
    * resolution (width, height). All parameters are in percentage of
    * screen resolution.
    */
   public static YuvImageRenderer create(int x, int y, int width, int height,
-      ScalingType scalingType, boolean mirror) {
+      RendererCommon.ScalingType scalingType, boolean mirror) {
     // Check display region parameters.
     if (x < 0 || x > 100 || y < 0 || y > 100 ||
         width < 0 || width > 100 || height < 0 || height > 100 ||
         x + width > 100 || y + height > 100) {
       throw new RuntimeException("Incorrect window parameters.");
     }
 
     if (instance == null) {
       throw new RuntimeException(
           "Attempt to create yuv renderer before setting GLSurfaceView");
@@ skipping 23 matching lines @@
         }
       }
       // Add yuv renderer to rendering list.
       instance.yuvImageRenderers.add(yuvImageRenderer);
     }
     return yuvImageRenderer;
   }
 
   public static void update(
       VideoRenderer.Callbacks renderer,
-      int x, int y, int width, int height, ScalingType scalingType, boolean mirror) {
+      int x, int y, int width, int height, RendererCommon.ScalingType scalingType, boolean mirror) {
     Log.d(TAG, "VideoRendererGui.update");
     if (instance == null) {
       throw new RuntimeException(
           "Attempt to update yuv renderer before setting GLSurfaceView");
     }
     synchronized (instance.yuvImageRenderers) {
       for (YuvImageRenderer yuvImageRenderer : instance.yuvImageRenderers) {
         if (yuvImageRenderer == renderer) {
           yuvImageRenderer.setPosition(x, y, width, height, scalingType, mirror);
         }
@@ skipping 61 matching lines @@
     GLES20.glViewport(0, 0, screenWidth, screenHeight);
     GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
     synchronized (yuvImageRenderers) {
       for (YuvImageRenderer yuvImageRenderer : yuvImageRenderers) {
         yuvImageRenderer.draw(drawer);
       }
     }
   }
 
 }