MediaCodecVideoEncoder add support to encode from textures

talk/app/webrtc/java/src/org/webrtc/MediaCodecVideoEncoder.java

 /*
  * libjingle
  * Copyright 2013 Google Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright notice,
@@ skipping 14 matching lines @@
  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 package org.webrtc;
 
 import android.media.MediaCodec;
 import android.media.MediaCodecInfo.CodecCapabilities;
 import android.media.MediaCodecInfo;
 import android.media.MediaCodecList;
 import android.media.MediaFormat;
+import android.opengl.GLES20;
 import android.os.Build;
 import android.os.Bundle;
+import android.view.Surface;
 
 import org.webrtc.Logging;
 
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 
+import javax.microedition.khronos.egl.EGLContext;
+
 // Java-side of peerconnection_jni.cc:MediaCodecVideoEncoder.
 // This class is an implementation detail of the Java PeerConnection API.
 public class MediaCodecVideoEncoder {
   // This class is constructed, operated, and destroyed by its C++ incarnation,
   // so the class and its methods have non-public visibility.  The API this
   // class exposes aims to mimic the webrtc::VideoEncoder API as closely as
   // possibly to minimize the amount of translation work necessary.
 
   private static final String TAG = "MediaCodecVideoEncoder";
 
   // Tracks webrtc::VideoCodecType.
   public enum VideoCodecType {
     VIDEO_CODEC_VP8,
     VIDEO_CODEC_VP9,
     VIDEO_CODEC_H264
   }
 
   private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000; // Timeout for codec releasing.
   private static final int DEQUEUE_TIMEOUT = 0;  // Non-blocking, no wait.
   // Active running encoder instance. Set in initEncode() (called from native code)
   // and reset to null in release() call.
   private static MediaCodecVideoEncoder runningInstance = null;
   private static MediaCodecVideoEncoderErrorCallback errorCallback = null;
   private static int codecErrors = 0;
 
   private Thread mediaCodecThread;
   private MediaCodec mediaCodec;
   private ByteBuffer[] outputBuffers;
+  private EglBase eglBase;
+  private Surface inputSurface;
+  private GlRectDrawer drawer;
   private static final String VP8_MIME_TYPE = "video/x-vnd.on2.vp8";
   private static final String H264_MIME_TYPE = "video/avc";
   // List of supported HW VP8 codecs.
   private static final String[] supportedVp8HwCodecPrefixes =
     {"OMX.qcom.", "OMX.Intel." };
   // List of supported HW H.264 codecs.
   private static final String[] supportedH264HwCodecPrefixes =
     {"OMX.qcom." };
   // List of devices with poor H.264 encoder quality.
   private static final String[] H264_HW_EXCEPTION_MODELS = new String[] {
@@ skipping 41 matching lines @@
   private static class EncoderProperties {
     public EncoderProperties(String codecName, int colorFormat) {
       this.codecName = codecName;
       this.colorFormat = colorFormat;
     }
     public final String codecName; // OpenMax component name for HW codec.
     public final int colorFormat;  // Color format supported by codec.
   }
 
   private static EncoderProperties findHwEncoder(
-      String mime, String[] supportedHwCodecPrefixes) {
+      String mime, String[] supportedHwCodecPrefixes, boolean useSurface) {
     // MediaCodec.setParameters is missing for JB and below, so bitrate
     // can not be adjusted dynamically.
     if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) {
       return null;
     }
 
     // Check if device is in H.264 exception list.
     if (mime.equals(H264_MIME_TYPE)) {
       List<String> exceptionModels = Arrays.asList(H264_HW_EXCEPTION_MODELS);
       if (exceptionModels.contains(Build.MODEL)) {
@@ skipping 29 matching lines @@
       }
       if (!supportedCodec) {
         continue;
       }
 
       CodecCapabilities capabilities = info.getCapabilitiesForType(mime);
       for (int colorFormat : capabilities.colorFormats) {
         Logging.v(TAG, "   Color: 0x" + Integer.toHexString(colorFormat));
       }
 
+      if (useSurface) {
+        for (int codecColorFormat : capabilities.colorFormats) {

[magjed_webrtc, 2015/11/18 13:12:52]

This looks like duplicated code. I suggest replacing the |useSurface| argument
with a |supportedColorList| argument instead. Rename static |supportedColorList|
to |supportedYuvColorList| and add
private static final int[] supportedSurfaceColorList = {
  CodecCapabilities.COLOR_FormatSurface
}
Then in isVp8HwSupported(), pass |supportedYuvColorList|, and in
isVp8HwSupportedUsingTextures(), pass |supportedSurfaceColorList|.

[perkj_webrtc, 2015/11/18 14:51:21]

nice. yes

+          if (codecColorFormat == CodecCapabilities.COLOR_FormatSurface) {
+            // Found supported HW encoder.
+            Logging.d(TAG, "Found target encoder for mime " + mime + " : " + name +
+                ". Using Surface as input");
+            return new EncoderProperties(name, codecColorFormat);
+          }
+        }
+        return null;
+      }
+
       // Check if codec supports either yuv420 or nv12.
       for (int supportedColorFormat : supportedColorList) {
         for (int codecColorFormat : capabilities.colorFormats) {
           if (codecColorFormat == supportedColorFormat) {
             // Found supported HW encoder.
             Logging.d(TAG, "Found target encoder for mime " + mime + " : " + name +
                 ". Color: 0x" + Integer.toHexString(codecColorFormat));
             return new EncoderProperties(name, codecColorFormat);
           }
         }
       }
     }
     return null;  // No HW VP8 encoder.
   }
 
   public static boolean isVp8HwSupported() {
-    return findHwEncoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes) != null;
+    return findHwEncoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes, false) != null;
   }
 
   public static boolean isH264HwSupported() {
-    return findHwEncoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes) != null;
+    return findHwEncoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes, false) != null;
+  }
+
+  public static boolean isVp8HwSupportedUsingTextures() {
+    return findHwEncoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes, true) != null;
+  }
+
+  public static boolean isH264HwSupportedUsingTextures() {
+    return findHwEncoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes, true) != null;
   }
 
   private void checkOnMediaCodecThread() {
     if (mediaCodecThread.getId() != Thread.currentThread().getId()) {
       throw new RuntimeException(
           "MediaCodecVideoEncoder previously operated on " + mediaCodecThread +
           " but is now called on " + Thread.currentThread());
     }
   }
 
@@ skipping 12 matching lines @@
   static MediaCodec createByCodecName(String codecName) {
     try {
       // In the L-SDK this call can throw IOException so in order to work in
       // both cases catch an exception.
       return MediaCodec.createByCodecName(codecName);
     } catch (Exception e) {
       return null;
     }
   }
 
-  // Returns false if the hardware encoder currently can't be used.
-  boolean initEncode(VideoCodecType type, int width, int height, int kbps, int fps) {
+  boolean initEncode(VideoCodecType type, int width, int height, int kbps, int fps,
+      EGLContext sharedContext) {
+    final boolean useSurface = sharedContext != null;
     Logging.d(TAG, "Java initEncode: " + type + " : " + width + " x " + height +
-        ". @ " + kbps + " kbps. Fps: " + fps + ".");
+        ". @ " + kbps + " kbps. Fps: " + fps + ". Encode from texture : " +
+        (useSurface ? "True" : "False"));

[magjed_webrtc, 2015/11/18 13:12:52]

Just print useSurface, you don't need to convert it to a string manually.

[perkj_webrtc, 2015/11/18 14:51:21]

Done.

 
     if (mediaCodecThread != null) {
       throw new RuntimeException("Forgot to release()?");
     }
     EncoderProperties properties = null;
     String mime = null;
     int keyFrameIntervalSec = 0;
     if (type == VideoCodecType.VIDEO_CODEC_VP8) {
       mime = VP8_MIME_TYPE;
-      properties = findHwEncoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes);
+      properties = findHwEncoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes, useSurface);
       keyFrameIntervalSec = 100;
     } else if (type == VideoCodecType.VIDEO_CODEC_H264) {
       mime = H264_MIME_TYPE;
-      properties = findHwEncoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes);
+      properties = findHwEncoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes, useSurface);
       keyFrameIntervalSec = 20;
     }
     if (properties == null) {
       throw new RuntimeException("Can not find HW encoder for " + type);
     }
     runningInstance = this; // Encoder is now running and can be queried for stack traces.
     colorFormat = properties.colorFormat;
     Logging.d(TAG, "Color format: " + colorFormat);
 
     mediaCodecThread = Thread.currentThread();
     try {
       MediaFormat format = MediaFormat.createVideoFormat(mime, width, height);
       format.setInteger(MediaFormat.KEY_BIT_RATE, 1000 * kbps);
       format.setInteger("bitrate-mode", VIDEO_ControlRateConstant);
       format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat);
       format.setInteger(MediaFormat.KEY_FRAME_RATE, fps);
       format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, keyFrameIntervalSec);
       Logging.d(TAG, "  Format: " + format);
       mediaCodec = createByCodecName(properties.codecName);
       this.type = type;
       if (mediaCodec == null) {
         Logging.e(TAG, "Can not create media encoder");
         return false;
       }
       mediaCodec.configure(
           format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
 
+      if (useSurface) {
+        eglBase = new EglBase(sharedContext, EglBase.ConfigType.RECORDABLE);
+        // Create an input surface and keep a reference since we must release the surface when done.
+        inputSurface = mediaCodec.createInputSurface();
+        eglBase.createSurface(inputSurface);
+        drawer = new GlRectDrawer();
+      }
       mediaCodec.start();
       outputBuffers = mediaCodec.getOutputBuffers();
       Logging.d(TAG, "Output buffers: " + outputBuffers.length);
 
     } catch (IllegalStateException e) {
       Logging.e(TAG, "initEncode failed", e);
       return false;
     }
     return true;
   }
@@ skipping 22 matching lines @@
       mediaCodec.queueInputBuffer(
           inputBuffer, 0, size, presentationTimestampUs, 0);
       return true;
     }
     catch (IllegalStateException e) {
       Logging.e(TAG, "encodeBuffer failed", e);
       return false;
     }
   }
 
+  boolean encodeTexture(boolean isKeyframe, int oesTextureId, float[] transformationMatrix,
+      long presentationTimestampUs) {
+    checkOnMediaCodecThread();
+    try {
+      if (isKeyframe) {
+        Logging.d(TAG, "Sync frame request");
+        Bundle b = new Bundle();
+        b.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0);
+        mediaCodec.setParameters(b);
+      }
+      eglBase.makeCurrent();
+      drawer.drawOes(oesTextureId, transformationMatrix);
+      // TODO(perkj): Do we have to call EGLExt.eglPresentationTimeANDROID ?
+      // If not, remove |presentationTimestampUs|.
+      eglBase.swapBuffers();
+      return true;
+    }
+    catch (RuntimeException e) {
+      Logging.e(TAG, "encodeTexture failed", e);
+      return false;
+    }
+  }
+
   void release() {
     Logging.d(TAG, "Java releaseEncoder");
     checkOnMediaCodecThread();
 
     // Run Mediacodec stop() and release() on separate thread since sometime
     // Mediacodec.stop() may hang.
     final CountDownLatch releaseDone = new CountDownLatch(1);
 
     Runnable runMediaCodecRelease = new Runnable() {
       @Override
@@ skipping 15 matching lines @@
       Logging.e(TAG, "Media encoder release timeout");
       codecErrors++;
       if (errorCallback != null) {
         Logging.e(TAG, "Invoke codec error callback. Errors: " + codecErrors);
         errorCallback.onMediaCodecVideoEncoderCriticalError(codecErrors);
       }
     }
 
     mediaCodec = null;
     mediaCodecThread = null;
+    if (drawer != null) {
+      drawer.release();
+      drawer = null;
+    }
+    if (eglBase != null) {
+      eglBase.release();
+      eglBase = null;
+    }
+    if (inputSurface != null) {
+      inputSurface.release();
+      inputSurface = null;
+    }
     runningInstance = null;
     Logging.d(TAG, "Java releaseEncoder done");
   }
 
   private boolean setRates(int kbps, int frameRateIgnored) {
     // frameRate argument is ignored - HW encoder is supposed to use
     // video frame timestamps for bit allocation.
     checkOnMediaCodecThread();
     Logging.v(TAG, "setRates: " + kbps + " kbps. Fps: " + frameRateIgnored);
     try {
@@ skipping 111 matching lines @@
     checkOnMediaCodecThread();
     try {
       mediaCodec.releaseOutputBuffer(index, false);
       return true;
     } catch (IllegalStateException e) {
       Logging.e(TAG, "releaseOutputBuffer failed", e);
       return false;
     }
   }
 }