Add support for swapping feeds in Android AppRTCMobile.

webrtc/examples/androidapp/src/org/appspot/apprtc/CallActivity.java

 /*
  *  Copyright 2015 The WebRTC Project Authors. All rights reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 116 matching lines @@
   public static final String EXTRA_ID = "org.appspot.apprtc.ID";
 
   private static final int CAPTURE_PERMISSION_REQUEST_CODE = 1;
 
   // List of mandatory application permissions.
   private static final String[] MANDATORY_PERMISSIONS = {"android.permission.MODIFY_AUDIO_SETTINGS",
       "android.permission.RECORD_AUDIO", "android.permission.INTERNET"};
 
   // Peer connection statistics callback period in ms.
   private static final int STAT_CALLBACK_PERIOD = 1000;
-  // Local preview screen position before call is connected.
-  private static final int LOCAL_X_CONNECTING = 0;
-  private static final int LOCAL_Y_CONNECTING = 0;
-  private static final int LOCAL_WIDTH_CONNECTING = 100;
-  private static final int LOCAL_HEIGHT_CONNECTING = 100;
-  // Local preview screen position after call is connected.
-  private static final int LOCAL_X_CONNECTED = 72;
-  private static final int LOCAL_Y_CONNECTED = 72;
-  private static final int LOCAL_WIDTH_CONNECTED = 25;
-  private static final int LOCAL_HEIGHT_CONNECTED = 25;
-  // Remote video screen position
-  private static final int REMOTE_X = 0;
-  private static final int REMOTE_Y = 0;
-  private static final int REMOTE_WIDTH = 100;
-  private static final int REMOTE_HEIGHT = 100;
+
+  private class ProxyRenderer implements VideoRenderer.Callbacks {
+    private VideoRenderer.Callbacks target;
+
+    synchronized public void renderFrame(VideoRenderer.I420Frame frame) {
+      if (target == null) {
+        Logging.d(TAG, "Dropping frame in proxy because target is null.");
+        VideoRenderer.renderFrameDone(frame);
+        return;
+      }
+
+      target.renderFrame(frame);
+    }
+
+    synchronized public void setTarget(VideoRenderer.Callbacks target) {
+      this.target = target;
+    }
+  }
+
+  private final ProxyRenderer remoteProxyRenderer = new ProxyRenderer();
+  private final ProxyRenderer localProxyRenderer = new ProxyRenderer();
   private PeerConnectionClient peerConnectionClient = null;
   private AppRTCClient appRtcClient;
   private SignalingParameters signalingParameters;
   private AppRTCAudioManager audioManager = null;
   private EglBase rootEglBase;
-  private SurfaceViewRenderer localRender;
-  private SurfaceViewRenderer remoteRenderScreen;
+  private SurfaceViewRenderer pipRenderer;
+  private SurfaceViewRenderer fullscreenRenderer;
   private VideoFileRenderer videoFileRenderer;
   private final List<VideoRenderer.Callbacks> remoteRenderers =
       new ArrayList<VideoRenderer.Callbacks>();
-  private PercentFrameLayout localRenderLayout;
-  private PercentFrameLayout remoteRenderLayout;
-  private ScalingType scalingType;
   private Toast logToast;
   private boolean commandLineRun;
   private int runTimeMs;
   private boolean activityRunning;
   private RoomConnectionParameters roomConnectionParameters;
   private PeerConnectionParameters peerConnectionParameters;
   private boolean iceConnected;
   private boolean isError;
   private boolean callControlFragmentVisible = true;
   private long callStartedTimeMs = 0;
   private boolean micEnabled = true;
   private boolean screencaptureEnabled = false;
   private static Intent mediaProjectionPermissionResultData;
   private static int mediaProjectionPermissionResultCode;
+  private boolean isSwappedFeeds;

[magjed_webrtc, 2017/02/10 13:23:38]

Can you comment what it means when this is true vs false?

[sakal, 2017/02/10 14:05:36]

Done.

 
   // Controls
   private CallFragment callFragment;
   private HudFragment hudFragment;
   private CpuMonitor cpuMonitor;
 
+

[magjed_webrtc, 2017/02/10 13:23:38]

nit: remove unnecessary empty line

[sakal, 2017/02/10 14:05:36]

Done.

   @Override
   public void onCreate(Bundle savedInstanceState) {
     super.onCreate(savedInstanceState);
     Thread.setDefaultUncaughtExceptionHandler(new UnhandledExceptionHandler(this));
 
     // Set window styles for fullscreen-window size. Needs to be done before
     // adding content.
     requestWindowFeature(Window.FEATURE_NO_TITLE);
     getWindow().addFlags(LayoutParams.FLAG_FULLSCREEN | LayoutParams.FLAG_KEEP_SCREEN_ON
         | LayoutParams.FLAG_DISMISS_KEYGUARD | LayoutParams.FLAG_SHOW_WHEN_LOCKED
         | LayoutParams.FLAG_TURN_SCREEN_ON);
     getWindow().getDecorView().setSystemUiVisibility(getSystemUiVisibility());
     setContentView(R.layout.activity_call);
 
     iceConnected = false;
     signalingParameters = null;
-    scalingType = ScalingType.SCALE_ASPECT_FILL;
 
     // Create UI controls.
-    localRender = (SurfaceViewRenderer) findViewById(R.id.local_video_view);
-    remoteRenderScreen = (SurfaceViewRenderer) findViewById(R.id.remote_video_view);
-    localRenderLayout = (PercentFrameLayout) findViewById(R.id.local_video_layout);
-    remoteRenderLayout = (PercentFrameLayout) findViewById(R.id.remote_video_layout);
+    pipRenderer = (SurfaceViewRenderer) findViewById(R.id.pip_video_view);
+    fullscreenRenderer = (SurfaceViewRenderer) findViewById(R.id.fullscreen_video_view);
     callFragment = new CallFragment();
     hudFragment = new HudFragment();
 
     // Show/hide call control fragment on view click.
     View.OnClickListener listener = new View.OnClickListener() {
       @Override
       public void onClick(View view) {
         toggleCallControlFragmentVisibility();
       }
     };
 
-    localRender.setOnClickListener(listener);
-    remoteRenderScreen.setOnClickListener(listener);
-    remoteRenderers.add(remoteRenderScreen);
+    // Swap feeds on pip view click.
+    pipRenderer.setOnClickListener(new View.OnClickListener() {
+      @Override
+      public void onClick(View view) {
+        setSwappedFeeds(!isSwappedFeeds);
+      }
+    });
+
+    fullscreenRenderer.setOnClickListener(listener);
+    remoteRenderers.add(remoteProxyRenderer);
 
     final Intent intent = getIntent();
 
     // Create video renderers.
     rootEglBase = EglBase.create();
-    localRender.init(rootEglBase.getEglBaseContext(), null);
+    pipRenderer.init(rootEglBase.getEglBaseContext(), null);
+    pipRenderer.setScalingType(ScalingType.SCALE_ASPECT_FIT);
     String saveRemoteVideoToFile = intent.getStringExtra(EXTRA_SAVE_REMOTE_VIDEO_TO_FILE);
 
     // When saveRemoteVideoToFile is set we save the video from the remote to a file.
     if (saveRemoteVideoToFile != null) {
       int videoOutWidth = intent.getIntExtra(EXTRA_SAVE_REMOTE_VIDEO_TO_FILE_WIDTH, 0);
       int videoOutHeight = intent.getIntExtra(EXTRA_SAVE_REMOTE_VIDEO_TO_FILE_HEIGHT, 0);
       try {
         videoFileRenderer = new VideoFileRenderer(
             saveRemoteVideoToFile, videoOutWidth, videoOutHeight, rootEglBase.getEglBaseContext());
         remoteRenderers.add(videoFileRenderer);
       } catch (IOException e) {
         throw new RuntimeException(
             "Failed to open video file for output: " + saveRemoteVideoToFile, e);
       }
     }
-    remoteRenderScreen.init(rootEglBase.getEglBaseContext(), null);
+    fullscreenRenderer.init(rootEglBase.getEglBaseContext(), null);
+    fullscreenRenderer.setScalingType(ScalingType.SCALE_ASPECT_FILL);
 
-    localRender.setZOrderMediaOverlay(true);
-    localRender.setEnableHardwareScaler(true /* enabled */);
-    remoteRenderScreen.setEnableHardwareScaler(true /* enabled */);
-    updateVideoView();
+    pipRenderer.setZOrderMediaOverlay(true);
+    pipRenderer.setEnableHardwareScaler(true /* enabled */);
+    fullscreenRenderer.setEnableHardwareScaler(true /* enabled */);
+    setSwappedFeeds(true /* isSwappedFeeds */);

[magjed_webrtc, 2017/02/10 13:23:38]

Add a comment why we start with local in fullscreen.

[sakal, 2017/02/10 14:05:36]

Done.

 
     // Check for mandatory permissions.
     for (String permission : MANDATORY_PERMISSIONS) {
       if (checkCallingOrSelfPermission(permission) != PackageManager.PERMISSION_GRANTED) {
         logAndToast("Permission " + permission + " is not granted");
         setResult(RESULT_CANCELED);
         finish();
         return;
       }
     }
@@ skipping 242 matching lines @@
 
   @Override
   public void onCameraSwitch() {
     if (peerConnectionClient != null) {
       peerConnectionClient.switchCamera();
     }
   }
 
   @Override
   public void onVideoScalingSwitch(ScalingType scalingType) {
-    this.scalingType = scalingType;
-    updateVideoView();
+    fullscreenRenderer.setScalingType(scalingType);
   }
 
   @Override
   public void onCaptureFormatChange(int width, int height, int framerate) {
     if (peerConnectionClient != null) {
       peerConnectionClient.changeCaptureFormat(width, height, framerate);
     }
   }
 
   @Override
@@ skipping 17 matching lines @@
       ft.show(callFragment);
       ft.show(hudFragment);
     } else {
       ft.hide(callFragment);
       ft.hide(hudFragment);
     }
     ft.setTransition(FragmentTransaction.TRANSIT_FRAGMENT_FADE);
     ft.commit();
   }
 
-  private void updateVideoView() {
-    remoteRenderLayout.setPosition(REMOTE_X, REMOTE_Y, REMOTE_WIDTH, REMOTE_HEIGHT);
-    remoteRenderScreen.setScalingType(scalingType);
-    remoteRenderScreen.setMirror(false);
-
-    if (iceConnected) {
-      localRenderLayout.setPosition(
-          LOCAL_X_CONNECTED, LOCAL_Y_CONNECTED, LOCAL_WIDTH_CONNECTED, LOCAL_HEIGHT_CONNECTED);
-      localRender.setScalingType(ScalingType.SCALE_ASPECT_FIT);
-    } else {
-      localRenderLayout.setPosition(
-          LOCAL_X_CONNECTING, LOCAL_Y_CONNECTING, LOCAL_WIDTH_CONNECTING, LOCAL_HEIGHT_CONNECTING);
-      localRender.setScalingType(scalingType);
-    }
-    localRender.setMirror(true);
-
-    localRender.requestLayout();
-    remoteRenderScreen.requestLayout();
-  }
-
   private void startCall() {
     if (appRtcClient == null) {
       Log.e(TAG, "AppRTC client is not allocated for a call.");
       return;
     }
     callStartedTimeMs = System.currentTimeMillis();
 
     // Start room connection.
     logAndToast(getString(R.string.connecting_to, roomConnectionParameters.roomUrl));
     appRtcClient.connectToRoom(roomConnectionParameters);
@@ skipping 16 matching lines @@
   }
 
   // Should be called from UI thread
   private void callConnected() {
     final long delta = System.currentTimeMillis() - callStartedTimeMs;
     Log.i(TAG, "Call connected: delay=" + delta + "ms");
     if (peerConnectionClient == null || isError) {
       Log.w(TAG, "Call is connected in closed or error state");
       return;
     }
-    // Update video view.
-    updateVideoView();
     // Enable statistics callback.
     peerConnectionClient.enableStatsEvents(true, STAT_CALLBACK_PERIOD);
+    setSwappedFeeds(false /* isSwappedFeeds */);
   }
 
   // This method is called when the audio manager reports audio device change,
   // e.g. from wired headset to speakerphone.
   private void onAudioManagerDevicesChanged(
       final AudioDevice device, final Set<AudioDevice> availableDevices) {
     Log.d(TAG, "onAudioManagerDevicesChanged: " + availableDevices + ", "
             + "selected: " + device);
     // TODO(henrika): add callback handler.
   }
 
   // Disconnect from remote resources, dispose of local resources, and exit.
   private void disconnect() {
     activityRunning = false;
+    remoteProxyRenderer.setTarget(null);
+    localProxyRenderer.setTarget(null);
     if (appRtcClient != null) {
       appRtcClient.disconnectFromRoom();
       appRtcClient = null;
     }
     if (peerConnectionClient != null) {
       peerConnectionClient.close();
       peerConnectionClient = null;
     }
-    if (localRender != null) {
-      localRender.release();
-      localRender = null;
+    if (pipRenderer != null) {
+      pipRenderer.release();
+      pipRenderer = null;
     }
     if (videoFileRenderer != null) {
       videoFileRenderer.release();
       videoFileRenderer = null;
     }
-    if (remoteRenderScreen != null) {
-      remoteRenderScreen.release();
-      remoteRenderScreen = null;
+    if (fullscreenRenderer != null) {
+      fullscreenRenderer.release();
+      fullscreenRenderer = null;
     }
     if (audioManager != null) {
       audioManager.stop();
       audioManager = null;
     }
     if (iceConnected && !isError) {
       setResult(RESULT_OK);
     } else {
       setResult(RESULT_CANCELED);
     }
@@ skipping 68 matching lines @@
       Logging.d(TAG, "Creating capturer using camera1 API.");
       videoCapturer = createCameraCapturer(new Camera1Enumerator(captureToTexture()));
     }
     if (videoCapturer == null) {
       reportError("Failed to open camera");
       return null;
     }
     return videoCapturer;
   }
 
+  private void setSwappedFeeds(boolean isSwappedFeeds) {
+    Logging.d(TAG, "setSwappedFeeds: " + isSwappedFeeds);
+    this.isSwappedFeeds = isSwappedFeeds;
+    localProxyRenderer.setTarget(isSwappedFeeds ? fullscreenRenderer : pipRenderer);
+    remoteProxyRenderer.setTarget(isSwappedFeeds ? pipRenderer : fullscreenRenderer);
+    fullscreenRenderer.setMirror(isSwappedFeeds);
+    pipRenderer.setMirror(!isSwappedFeeds);
+  }
+
   // -----Implementation of AppRTCClient.AppRTCSignalingEvents ---------------
   // All callbacks are invoked from websocket signaling looper thread and
   // are routed to UI thread.
   private void onConnectedToRoomInternal(final SignalingParameters params) {
     final long delta = System.currentTimeMillis() - callStartedTimeMs;
 
     signalingParameters = params;
     logAndToast("Creating peer connection, delay=" + delta + "ms");
     VideoCapturer videoCapturer = null;
     if (peerConnectionParameters.videoCallEnabled) {
       videoCapturer = createVideoCapturer();
     }
-    peerConnectionClient.createPeerConnection(rootEglBase.getEglBaseContext(), localRender,
+    peerConnectionClient.createPeerConnection(rootEglBase.getEglBaseContext(), localProxyRenderer,
         remoteRenderers, videoCapturer, signalingParameters);
 
     if (signalingParameters.initiator) {
       logAndToast("Creating OFFER...");
       // Create offer. Offer SDP will be sent to answering client in
       // PeerConnectionEvents.onLocalDescription event.
       peerConnectionClient.createOffer();
     } else {
       if (params.offerSdp != null) {
         peerConnectionClient.setRemoteDescription(params.offerSdp);
@@ skipping 175 matching lines @@
         }
       }
     });
   }
 
   @Override
   public void onPeerConnectionError(final String description) {
     reportError(description);
   }
 }