AppRTCDemo: Render each video in a separate SurfaceView

talk/examples/android/src/org/appspot/apprtc/CallActivity.java

 /*
  * libjingle
  * Copyright 2015 Google Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright notice,
@@ skipping 29 matching lines @@
 import android.content.pm.PackageManager;
 import android.net.Uri;
 import android.opengl.GLSurfaceView;
 import android.os.Bundle;
 import android.util.Log;
 import android.view.View;
 import android.view.Window;
 import android.view.WindowManager.LayoutParams;
 import android.widget.Toast;
 
+import org.webrtc.EglBase;
 import org.webrtc.IceCandidate;
 import org.webrtc.SessionDescription;
 import org.webrtc.StatsReport;
-import org.webrtc.VideoRenderer;
-import org.webrtc.VideoRendererGui;
+import org.webrtc.SurfaceViewRenderer;
 import org.webrtc.VideoRendererGui.ScalingType;
 
 /**
  * Activity for peer connection call setup, call waiting
  * and call view.
  */
 public class CallActivity extends Activity
     implements AppRTCClient.SignalingEvents,
       PeerConnectionClient.PeerConnectionEvents,
       CallFragment.OnCallEvents {
@@ skipping 54 matching lines @@
   // Remote video screen position
   private static final int REMOTE_X = 0;
   private static final int REMOTE_Y = 0;
   private static final int REMOTE_WIDTH = 100;
   private static final int REMOTE_HEIGHT = 100;
 
   private PeerConnectionClient peerConnectionClient = null;
   private AppRTCClient appRtcClient;
   private SignalingParameters signalingParameters;
   private AppRTCAudioManager audioManager = null;
-  private VideoRenderer.Callbacks localRender;
-  private VideoRenderer.Callbacks remoteRender;
+  private EglBase rootEglBase;
+  private SurfaceViewRenderer localRender;
+  private SurfaceViewRenderer remoteRender;
+  private PercentFrameLayout localRenderLayout;
+  private PercentFrameLayout remoteRenderLayout;
   private ScalingType scalingType;
   private Toast logToast;
   private boolean commandLineRun;
   private int runTimeMs;
   private boolean activityRunning;
   private RoomConnectionParameters roomConnectionParameters;
   private PeerConnectionParameters peerConnectionParameters;
   private boolean iceConnected;
   private boolean isError;
   private boolean callControlFragmentVisible = true;
   private long callStartedTimeMs = 0;
 
   // Controls
-  private GLSurfaceView videoView;
   CallFragment callFragment;
   HudFragment hudFragment;
 
   @Override
   public void onCreate(Bundle savedInstanceState) {
     super.onCreate(savedInstanceState);
     Thread.setDefaultUncaughtExceptionHandler(
         new UnhandledExceptionHandler(this));
 
     // Set window styles for fullscreen-window size. Needs to be done before
     // adding content.
     requestWindowFeature(Window.FEATURE_NO_TITLE);
     getWindow().addFlags(
         LayoutParams.FLAG_FULLSCREEN
         | LayoutParams.FLAG_KEEP_SCREEN_ON
         | LayoutParams.FLAG_DISMISS_KEYGUARD
         | LayoutParams.FLAG_SHOW_WHEN_LOCKED
         | LayoutParams.FLAG_TURN_SCREEN_ON);
     getWindow().getDecorView().setSystemUiVisibility(
         View.SYSTEM_UI_FLAG_HIDE_NAVIGATION
         | View.SYSTEM_UI_FLAG_FULLSCREEN
         | View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY);
     setContentView(R.layout.activity_call);
 
     iceConnected = false;
     signalingParameters = null;
     scalingType = ScalingType.SCALE_ASPECT_FILL;
 
     // Create UI controls.
-    videoView = (GLSurfaceView) findViewById(R.id.glview_call);
+    localRender = (SurfaceViewRenderer) findViewById(R.id.local_video_view);
+    remoteRender = (SurfaceViewRenderer) findViewById(R.id.remote_video_view);
+    localRenderLayout = (PercentFrameLayout) findViewById(R.id.local_video_layout);
+    remoteRenderLayout = (PercentFrameLayout) findViewById(R.id.remote_video_layout);
     callFragment = new CallFragment();
     hudFragment = new HudFragment();
 
-    // Create video renderers.
-    VideoRendererGui.setView(videoView, new Runnable() {
-      @Override
-      public void run() {
-        createPeerConnectionFactory();
-      }
-    });
-    remoteRender = VideoRendererGui.create(
-        REMOTE_X, REMOTE_Y,
-        REMOTE_WIDTH, REMOTE_HEIGHT, scalingType, false);
-    localRender = VideoRendererGui.create(
-        LOCAL_X_CONNECTING, LOCAL_Y_CONNECTING,
-        LOCAL_WIDTH_CONNECTING, LOCAL_HEIGHT_CONNECTING, scalingType, true);
-
     // Show/hide call control fragment on view click.
-    videoView.setOnClickListener(new View.OnClickListener() {
+    View.OnClickListener listener = new View.OnClickListener() {
       @Override
       public void onClick(View view) {
         toggleCallControlFragmentVisibility();
       }
-    });
+    };
+    localRender.setOnClickListener(listener);
+    remoteRender.setOnClickListener(listener);
+
+    // Create video renderers.
+    rootEglBase = new EglBase();
+    localRender.init(rootEglBase.getContext());
+    remoteRender.init(rootEglBase.getContext());
+    localRender.setZOrderMediaOverlay(true);
+    updateVideoView();
 
     // Check for mandatory permissions.
     for (String permission : MANDATORY_PERMISSIONS) {
       if (checkCallingOrSelfPermission(permission) != PackageManager.PERMISSION_GRANTED) {
         logAndToast("Permission " + permission + " is not granted");
         setResult(RESULT_CANCELED);
         finish();
         return;
       }
     }
@@ skipping 43 matching lines @@
     hudFragment.setArguments(intent.getExtras());
     // Activate call and HUD fragments and start the call.
     FragmentTransaction ft = getFragmentManager().beginTransaction();
     ft.add(R.id.call_fragment_container, callFragment);
     ft.add(R.id.hud_fragment_container, hudFragment);
     ft.commit();
     startCall();
 
     // For command line execution run connection for <runTimeMs> and exit.
     if (commandLineRun && runTimeMs > 0) {
-      videoView.postDelayed(new Runnable() {
+      localRender.postDelayed(new Runnable() {

[AlexG, 2015/08/04 00:27:23]

nit: this was probably not the best way from the beginning, now it looks a
little strange why we are posting disconnect() request to local renderer - may
be use:
(new Handler()).postDelayed(...) here?

[magjed_webrtc, 2015/08/04 17:05:05]

Done.

         public void run() {
           disconnect();
         }
       }, runTimeMs);
     }
+    createPeerConnectionFactory();
   }
 
   // Activity interfaces
   @Override
   public void onPause() {
     super.onPause();
-    videoView.onPause();
     activityRunning = false;
     if (peerConnectionClient != null) {
       peerConnectionClient.stopVideoSource();
     }
   }
 
   @Override
   public void onResume() {
     super.onResume();
-    videoView.onResume();
     activityRunning = true;
     if (peerConnectionClient != null) {
       peerConnectionClient.startVideoSource();
     }
   }
 
   @Override
   protected void onDestroy() {
     disconnect();
     super.onDestroy();
     if (logToast != null) {
       logToast.cancel();
     }
     activityRunning = false;
+    localRender.release();
+    localRender = null;
+    remoteRender.release();
+    remoteRender = null;
+    rootEglBase.release();
+    rootEglBase = null;
   }
 
   // CallFragment.OnCallEvents interface implementation.
   @Override
   public void onCallHangUp() {
     disconnect();
   }
 
   @Override
   public void onCameraSwitch() {
@@ skipping 21 matching lines @@
       ft.show(hudFragment);
     } else {
       ft.hide(callFragment);
       ft.hide(hudFragment);
     }
     ft.setTransition(FragmentTransaction.TRANSIT_FRAGMENT_FADE);
     ft.commit();
   }
 
   private void updateVideoView() {
-    VideoRendererGui.update(remoteRender,
-        REMOTE_X, REMOTE_Y,
-        REMOTE_WIDTH, REMOTE_HEIGHT, scalingType, false);
+    remoteRenderLayout.setPosition(REMOTE_X, REMOTE_Y, REMOTE_WIDTH, REMOTE_HEIGHT);

[AlexG, 2015/08/04 00:27:23]

Could it cause race condition when say position is updated, but scaling type is
not and renderer thread will render frame with updated position, but not updated
scaling. May be merge all layout specific set function into one set function
with one parameter structure containing position, mirror and scaling type
fields?

[magjed_webrtc, 2015/08/04 17:05:05]

There is no race between setPosition() and setScalingType() because they are
just setters, and the actual layout change is triggered by requestLayout().
However, making a layout update that changes the display aspect ratio will
stretch the currently shown frame, and it will remain stretched until the next
frame arrives and is rendered. I don't know any way around this. I think
animations that preserve aspect ratio will look fine, and otherwise they will
look stretched. To avoid this, we could hide the SurfaceView during layout
transitions, show some generic animation instead, and then unhide the
SurfaceView once it has been rendered correctly in the new layout.

+    remoteRender.setScalingType(scalingType);
+    remoteRender.setMirror(false);
+
     if (iceConnected) {
-      VideoRendererGui.update(localRender,
-          LOCAL_X_CONNECTED, LOCAL_Y_CONNECTED,
-          LOCAL_WIDTH_CONNECTED, LOCAL_HEIGHT_CONNECTED,
-          ScalingType.SCALE_ASPECT_FIT, true);
+      localRenderLayout.setPosition(
+          LOCAL_X_CONNECTED, LOCAL_Y_CONNECTED, LOCAL_WIDTH_CONNECTED, LOCAL_HEIGHT_CONNECTED);
+      localRender.setScalingType(ScalingType.SCALE_ASPECT_FIT);
     } else {
-      VideoRendererGui.update(localRender,
-          LOCAL_X_CONNECTING, LOCAL_Y_CONNECTING,
-          LOCAL_WIDTH_CONNECTING, LOCAL_HEIGHT_CONNECTING, scalingType, true);
+      localRenderLayout.setPosition(
+          LOCAL_X_CONNECTING, LOCAL_Y_CONNECTING, LOCAL_WIDTH_CONNECTING, LOCAL_HEIGHT_CONNECTING);
+      localRender.setScalingType(scalingType);
     }
+    localRender.setMirror(true);
+
+    localRender.requestLayout();
+    remoteRender.requestLayout();
   }
 
   private void startCall() {
     if (appRtcClient == null) {
       Log.e(TAG, "AppRTC client is not allocated for a call.");
       return;
     }
     callStartedTimeMs = System.currentTimeMillis();
 
     // Start room connection.
@@ skipping 37 matching lines @@
   // Create peer connection factory when EGL context is ready.
   private void createPeerConnectionFactory() {
     runOnUiThread(new Runnable() {
       @Override
       public void run() {
         if (peerConnectionClient == null) {
           final long delta = System.currentTimeMillis() - callStartedTimeMs;
           Log.d(TAG, "Creating peer connection factory, delay=" + delta + "ms");
           peerConnectionClient = PeerConnectionClient.getInstance();
           peerConnectionClient.createPeerConnectionFactory(CallActivity.this,
-              VideoRendererGui.getEGLContext(), peerConnectionParameters,
+              rootEglBase.getContext(), peerConnectionParameters,
               CallActivity.this);
         }
         if (signalingParameters != null) {
           Log.w(TAG, "EGL context is ready after room connection.");
           onConnectedToRoomInternal(signalingParameters);
         }
       }
     });
   }
 
@@ skipping 234 matching lines @@
         }
       }
     });
   }
 
   @Override
   public void onPeerConnectionError(final String description) {
     reportError(description);
   }
 }