MediaCodecVideoEncoder add support to encode from textures

talk/app/webrtc/java/src/org/webrtc/MediaCodecVideoEncoder.java

 /*
  * libjingle
  * Copyright 2013 Google Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright notice,
@@ skipping 15 matching lines @@
  */
 
 
 package org.webrtc;
 
 import android.media.MediaCodec;
 import android.media.MediaCodecInfo.CodecCapabilities;
 import android.media.MediaCodecInfo;
 import android.media.MediaCodecList;
 import android.media.MediaFormat;
+import android.opengl.EGLContext;
+import android.opengl.GLES20;
 import android.os.Build;
 import android.os.Bundle;
+import android.view.Surface;
 
 import org.webrtc.Logging;
 
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
 
 // Java-side of peerconnection_jni.cc:MediaCodecVideoEncoder.
 // This class is an implementation detail of the Java PeerConnection API.
 // MediaCodec is thread-hostile so this class must be operated on a single
@@ skipping 10 matching lines @@
   public enum VideoCodecType {
     VIDEO_CODEC_VP8,
     VIDEO_CODEC_VP9,
     VIDEO_CODEC_H264
   }
 
   private static final int DEQUEUE_TIMEOUT = 0;  // Non-blocking, no wait.
   private Thread mediaCodecThread;
   private MediaCodec mediaCodec;
   private ByteBuffer[] outputBuffers;
+  private EglBase eglBase;
+  private Surface inputSurface;
+  private GlRectDrawer drawer;
   private static final String VP8_MIME_TYPE = "video/x-vnd.on2.vp8";
   private static final String H264_MIME_TYPE = "video/avc";
   // List of supported HW VP8 codecs.
   private static final String[] supportedVp8HwCodecPrefixes =
     {"OMX.qcom.", "OMX.Intel." };
   // List of supported HW H.264 codecs.
   private static final String[] supportedH264HwCodecPrefixes =
     {"OMX.qcom." };
   // List of devices with poor H.264 encoder quality.
   private static final String[] H264_HW_EXCEPTION_MODELS = new String[] {
@@ skipping 32 matching lines @@
   private static class EncoderProperties {
     public EncoderProperties(String codecName, int colorFormat) {
       this.codecName = codecName;
       this.colorFormat = colorFormat;
     }
     public final String codecName; // OpenMax component name for HW codec.
     public final int colorFormat;  // Color format supported by codec.
   }
 
   private static EncoderProperties findHwEncoder(
-      String mime, String[] supportedHwCodecPrefixes) {
+      String mime, String[] supportedHwCodecPrefixes, boolean useSurface) {
     // MediaCodec.setParameters is missing for JB and below, so bitrate
     // can not be adjusted dynamically.
     if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) {
       return null;
     }
 
     // Check if device is in H.264 exception list.
     if (mime.equals(H264_MIME_TYPE)) {
       List<String> exceptionModels = Arrays.asList(H264_HW_EXCEPTION_MODELS);
       if (exceptionModels.contains(Build.MODEL)) {
@@ skipping 30 matching lines @@
       }
       if (!supportedCodec) {
         continue;
       }
 
       CodecCapabilities capabilities = info.getCapabilitiesForType(mime);
       for (int colorFormat : capabilities.colorFormats) {
         Logging.v(TAG, "   Color: 0x" + Integer.toHexString(colorFormat));
       }
 
+      if (useSurface) {
+        for (int codecColorFormat : capabilities.colorFormats) {
+          if (codecColorFormat == CodecCapabilities.COLOR_FormatSurface) {
+            // Found supported HW encoder.
+            Logging.d(TAG, "Found target encoder for mime " + mime + " : " + name +
+                ". Using Surface as input");
+            return new EncoderProperties(name, codecColorFormat);
+          }
+        }
+        return null;
+      }
+
       // Check if codec supports either yuv420 or nv12.
       for (int supportedColorFormat : supportedColorList) {
         for (int codecColorFormat : capabilities.colorFormats) {
           if (codecColorFormat == supportedColorFormat) {
             // Found supported HW encoder.
             Logging.d(TAG, "Found target encoder for mime " + mime + " : " + name +
                 ". Color: 0x" + Integer.toHexString(codecColorFormat));
             return new EncoderProperties(name, codecColorFormat);
           }
         }
       }
     }
     return null;  // No HW VP8 encoder.
   }
 
   public static boolean isVp8HwSupported() {
-    return findHwEncoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes) != null;
+    return findHwEncoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes, false) != null;
   }
 
   public static boolean isH264HwSupported() {
-    return findHwEncoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes) != null;
+    return findHwEncoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes, false) != null;
+  }
+
+  public static boolean isVp8HwSupportedUsingTextures() {

[AlexG, 2015/10/14 23:48:27]

Are these 2 functions called from native code or from any app code or you are
planning to use them in the future?

[perkj_webrtc, 2015/11/16 13:08:51]

they are here so that an app can ask if the hw codec can use with textures. If
we don't add them - is there a risk that a hw encoder does not support encoding
from textures? I guess not and in that case we can remove them and rely on
isH264HwSupported and the vp8 version. Wdyt? 

+    return findHwEncoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes, true) != null;
+  }
+
+  public static boolean isH264HwSupportedUsingTextures() {
+    return findHwEncoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes, true) != null;
   }
 
   private void checkOnMediaCodecThread() {
     if (mediaCodecThread.getId() != Thread.currentThread().getId()) {
       throw new RuntimeException(
           "MediaCodecVideoEncoder previously operated on " + mediaCodecThread +
           " but is now called on " + Thread.currentThread());
     }
   }
 
   static MediaCodec createByCodecName(String codecName) {
     try {
       // In the L-SDK this call can throw IOException so in order to work in
       // both cases catch an exception.
       return MediaCodec.createByCodecName(codecName);
     } catch (Exception e) {
       return null;
     }
   }
 
-  // Returns false if the hardware encoder currently can't be used.
-  boolean initEncode(VideoCodecType type, int width, int height, int kbps, int fps) {
+  boolean initEncode(VideoCodecType type, int width, int height, int kbps, int fps,
+      EGLContext sharedContext) {
+    final boolean useSurface = sharedContext != null;
     Logging.d(TAG, "Java initEncode: " + type + " : " + width + " x " + height +
-        ". @ " + kbps + " kbps. Fps: " + fps + ".");
+        ". @ " + kbps + " kbps. Fps: " + fps + ". Encode from texture : " +
+        (useSurface ? "True" : "False"));
 
     if (mediaCodecThread != null) {
       throw new RuntimeException("Forgot to release()?");
     }
     EncoderProperties properties = null;
     String mime = null;
     int keyFrameIntervalSec = 0;
     if (type == VideoCodecType.VIDEO_CODEC_VP8) {
       mime = VP8_MIME_TYPE;
-      properties = findHwEncoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes);
+      properties = findHwEncoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes, useSurface);
       keyFrameIntervalSec = 100;
     } else if (type == VideoCodecType.VIDEO_CODEC_H264) {
       mime = H264_MIME_TYPE;
-      properties = findHwEncoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes);
+      properties = findHwEncoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes, useSurface);
       keyFrameIntervalSec = 20;
     }
     if (properties == null) {
       throw new RuntimeException("Can not find HW encoder for " + type);
     }
     colorFormat = properties.colorFormat;
     mediaCodecThread = Thread.currentThread();
     try {
       MediaFormat format = MediaFormat.createVideoFormat(mime, width, height);
       format.setInteger(MediaFormat.KEY_BIT_RATE, 1000 * kbps);
       format.setInteger("bitrate-mode", VIDEO_ControlRateConstant);
       format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat);
       format.setInteger(MediaFormat.KEY_FRAME_RATE, fps);
       format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, keyFrameIntervalSec);
       Logging.d(TAG, "  Format: " + format);
       mediaCodec = createByCodecName(properties.codecName);
       this.type = type;
       if (mediaCodec == null) {
         return false;
       }
       mediaCodec.configure(
           format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
 
+      if (useSurface) {
+        eglBase = new EglBase(sharedContext, EglBase.ConfigType.RECORDABLE);
+        // Create an input surface and keep a reference since we must release the surface when done.
+        inputSurface = mediaCodec.createInputSurface();
+        eglBase.createSurface(inputSurface);
+        drawer = new GlRectDrawer();
+      }
       mediaCodec.start();
       outputBuffers = mediaCodec.getOutputBuffers();
 
     } catch (IllegalStateException e) {
       Logging.e(TAG, "initEncode failed", e);
       return false;
     }
     return true;
   }
 
@@ skipping 21 matching lines @@
       mediaCodec.queueInputBuffer(
           inputBuffer, 0, size, presentationTimestampUs, 0);
       return true;
     }
     catch (IllegalStateException e) {
       Logging.e(TAG, "encodeBuffer failed", e);
       return false;
     }
   }
 
+  boolean encodeTexture(boolean isKeyframe, int oesTextureId, float[] transformationMatrix,
+      long presentationTimestampUs) {
+    checkOnMediaCodecThread();
+    try {
+      if (isKeyframe) {
+        Logging.d(TAG, "Sync frame request");
+        Bundle b = new Bundle();
+        b.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0);
+        mediaCodec.setParameters(b);
+      }
+      eglBase.makeCurrent();
+      drawer.drawOes(oesTextureId, transformationMatrix);
+      eglBase.swapBuffers(presentationTimestampUs * 1000);
+      return true;
+    }
+    catch (RuntimeException e) {
+      Logging.e(TAG, "encodeTexture failed", e);
+      return false;
+    }
+  }
+
   void release() {
     Logging.d(TAG, "Java releaseEncoder");
     checkOnMediaCodecThread();
     try {
       mediaCodec.stop();
       mediaCodec.release();
     } catch (IllegalStateException e) {
       Logging.e(TAG, "release failed", e);
     }
     mediaCodec = null;
     mediaCodecThread = null;
+    if (drawer != null) {
+      drawer.release();

[AlexG, 2015/10/14 23:48:27]

nit: drawer = null and same for 2 checks below?

[perkj_webrtc, 2015/11/16 13:08:51]

Done.

+    }
+    if (eglBase != null) {
+      eglBase.release();
+    }
+    if (inputSurface != null) {
+      inputSurface.release();
+    }
   }
 
   private boolean setRates(int kbps, int frameRateIgnored) {
     // frameRate argument is ignored - HW encoder is supposed to use
     // video frame timestamps for bit allocation.
     checkOnMediaCodecThread();
     Logging.v(TAG, "setRates: " + kbps + " kbps. Fps: " + frameRateIgnored);
     try {
       Bundle params = new Bundle();
       params.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, 1000 * kbps);
@@ skipping 109 matching lines @@
     checkOnMediaCodecThread();
     try {
       mediaCodec.releaseOutputBuffer(index, false);
       return true;
     } catch (IllegalStateException e) {
       Logging.e(TAG, "releaseOutputBuffer failed", e);
       return false;
     }
   }
 }