Add texture support to HardwareVideoDecoder.

webrtc/sdk/android/src/java/org/webrtc/HardwareVideoDecoder.java

Index: webrtc/sdk/android/src/java/org/webrtc/HardwareVideoDecoder.java
diff --git a/webrtc/sdk/android/src/java/org/webrtc/HardwareVideoDecoder.java b/webrtc/sdk/android/src/java/org/webrtc/HardwareVideoDecoder.java
index 99a0a4b06fb2ab1355a1a5552152e085c7c14237..a2713b5a5138373389bd189d9eb619efceafb10e 100644
--- a/webrtc/sdk/android/src/java/org/webrtc/HardwareVideoDecoder.java
+++ b/webrtc/sdk/android/src/java/org/webrtc/HardwareVideoDecoder.java
@@ -16,18 +16,22 @@ import android.media.MediaCodec;
 import android.media.MediaCodecInfo.CodecCapabilities;
 import android.media.MediaFormat;
 import android.os.SystemClock;
+import android.view.Surface;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Deque;
+import java.util.concurrent.BlockingDeque;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.atomic.AtomicReference;
 import org.webrtc.ThreadUtils.ThreadChecker;
 
 /** Android hardware video decoder. */
 @TargetApi(16)
 @SuppressWarnings("deprecation") // Cannot support API 16 without using deprecated methods.
-class HardwareVideoDecoder implements VideoDecoder {
+class HardwareVideoDecoder
+    implements VideoDecoder, SurfaceTextureHelper.OnTextureFrameAvailableListener {
   private static final String TAG = "HardwareVideoDecoder";
 
   // TODO(magjed): Use MediaFormat.KEY_* constants when part of the public API.
@@ -52,6 +56,9 @@ class HardwareVideoDecoder implements VideoDecoder {
   // MediaCodec.
   private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000;
 
+  // Max number of output buffers queued before starting to drop decoded frames.
+  private static final int MAX_QUEUED_OUTPUTBUFFERS = 3;
+
   private final String codecName;
   private final VideoCodecType codecType;
 
@@ -100,18 +107,49 @@ class HardwareVideoDecoder implements VideoDecoder {
   // Whether the decoder has seen a key frame.  The first frame must be a key frame.
   private boolean keyFrameRequired;
 
+  private final SurfaceTextureHelper surfaceTextureHelper;
+  private Surface surface = null;
+
+  private static class DecodedTextureBuffer {
+    final int index;
+    final int width;
+    final int height;
+    final int rotation;
+    final long presentationTimestampUs;
+    final Integer decodeTimeMs;
+
+    DecodedTextureBuffer(int index, int width, int height, int rotation,
+        long presentationTimestampUs, Integer decodeTimeMs) {
+      this.index = index;
+      this.width = width;
+      this.height = height;
+      this.rotation = rotation;
+      this.presentationTimestampUs = presentationTimestampUs;
+      this.decodeTimeMs = decodeTimeMs;
+    }
+  }
+
+  private final BlockingDeque<DecodedTextureBuffer> decodedTextureBuffers;
+  private final AtomicReference<DecodedTextureBuffer> renderedTextureBuffer;
+
+  private int droppedFrames = 0;
+
   // Decoding proceeds asynchronously.  This callback returns decoded frames to the caller.
   private Callback callback;
 
   private MediaCodec codec = null;
 
-  HardwareVideoDecoder(String codecName, VideoCodecType codecType, int colorFormat) {
+  HardwareVideoDecoder(String codecName, VideoCodecType codecType, int colorFormat,
+      SurfaceTextureHelper surfaceTextureHelper) {

[sakal, 2017/07/17 13:44:56]

I don't think we have to pass in the SurfaceTextureHelper. It should be able to
live in the decoder. We should just pass in the shared context.

[mellem, 2017/07/17 21:57:15]

Done.

     if (!isSupportedColorFormat(colorFormat)) {
       throw new IllegalArgumentException("Unsupported color format: " + colorFormat);
     }
     this.codecName = codecName;
     this.codecType = codecType;
     this.colorFormat = colorFormat;
+    this.surfaceTextureHelper = surfaceTextureHelper;
+    this.decodedTextureBuffers = new LinkedBlockingDeque<>();
+    this.renderedTextureBuffer = new AtomicReference<>();
     this.frameInfos = new LinkedBlockingDeque<>();
   }
 
@@ -138,6 +176,7 @@ class HardwareVideoDecoder implements VideoDecoder {
     sliceHeight = height;
     hasDecodedFirstFrame = false;
     keyFrameRequired = true;
+    droppedFrames = 0;
 
     try {
       codec = MediaCodec.createByCodecName(codecName);
@@ -147,8 +186,13 @@ class HardwareVideoDecoder implements VideoDecoder {
     }
     try {
       MediaFormat format = MediaFormat.createVideoFormat(codecType.mimeType(), width, height);
-      format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
-      codec.configure(format, null, null, 0);
+      if (surfaceTextureHelper == null) {
+        format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
+      } else {
+        surface = new Surface(surfaceTextureHelper.getSurfaceTexture());
+        surfaceTextureHelper.startListening(this);
+      }
+      codec.configure(format, surface, null, 0);
       codec.start();
     } catch (IllegalStateException e) {
       Logging.e(TAG, "initDecode failed", e);
@@ -209,7 +253,6 @@ class HardwareVideoDecoder implements VideoDecoder {
       }
     }
 
-    // TODO(mellem):  Support textures.
     int index;
     try {
       index = codec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT_US);
@@ -288,6 +331,14 @@ class HardwareVideoDecoder implements VideoDecoder {
       callback = null;
       outputThread = null;
       frameInfos.clear();
+      if (surface != null) {
+        surface.release();
+        surface = null;
+        surfaceTextureHelper.stopListening();
+        surfaceTextureHelper.returnTextureFrame();

[sakal, 2017/07/17 13:44:56]

I think we should wait that all frames are released and then release the surface
texture helper.

[mellem, 2017/07/17 21:57:15]

Now that the decoder owns the SurfaceTextureHelper, dispose() does this for us. 
It waits until any outstanding frame is returned before releasing the helper.

+      }
+      decodedTextureBuffers.clear();
+      renderedTextureBuffer.set(null);
     }
     return VideoCodecStatus.OK;
   }
@@ -308,6 +359,7 @@ class HardwareVideoDecoder implements VideoDecoder {
         outputThreadChecker = new ThreadChecker();
         while (running) {
           deliverDecodedFrame();
+          maybeRenderTextureBuffers();
         }
         releaseCodecOnOutputThread();
       }
@@ -343,62 +395,145 @@ class HardwareVideoDecoder implements VideoDecoder {
 
       hasDecodedFirstFrame = true;
 
-      // Load dimensions from shared memory under the dimension lock.
-      int width, height, stride, sliceHeight;
-      synchronized (dimensionLock) {
-        width = this.width;
-        height = this.height;
-        stride = this.stride;
-        sliceHeight = this.sliceHeight;
+      if (surfaceTextureHelper != null) {
+        deliverTextureFrame(result, info, rotation, decodeTimeMs);
+      } else {
+        deliverByteFrame(result, info, rotation, decodeTimeMs);
       }
 
-      // Output must be at least width * height bytes for Y channel, plus (width / 2) * (height / 2)
-      // bytes for each of the U and V channels.
-      if (info.size < width * height * 3 / 2) {
-        Logging.e(TAG, "Insufficient output buffer size: " + info.size);
-        return;
-      }
+    } catch (IllegalStateException e) {
+      Logging.e(TAG, "deliverDecodedFrame failed", e);
+    }
+  }
 
-      if (info.size < stride * height * 3 / 2 && sliceHeight == height && stride > width) {
-        // Some codecs (Exynos) report an incorrect stride.  Correct it here.
-        // Expected size == stride * height * 3 / 2.  A bit of algebra gives the correct stride as
-        // 2 * size / (3 * height).
-        stride = info.size * 2 / (height * 3);
-      }
+  private void deliverTextureFrame(
+      int index, MediaCodec.BufferInfo info, int rotation, Integer decodeTimeMs) {
+    // Load dimensions from shared memory under the dimension lock.
+    int width, height;
+    synchronized (dimensionLock) {
+      width = this.width;
+      height = this.height;
+    }
 
-      ByteBuffer buffer = codec.getOutputBuffers()[result];
-      buffer.position(info.offset);
-      buffer.limit(info.size);
-
-      final VideoFrame.I420Buffer frameBuffer;
-
-      // TODO(mellem):  As an optimization, use libyuv via JNI to copy/reformatting data.
-      if (colorFormat == CodecCapabilities.COLOR_FormatYUV420Planar) {
-        if (sliceHeight % 2 == 0) {
-          frameBuffer =
-              createBufferFromI420(buffer, result, info.offset, stride, sliceHeight, width, height);
-        } else {
-          frameBuffer = new I420BufferImpl(width, height);
-          // Optimal path is not possible because we have to copy the last rows of U- and V-planes.
-          copyI420(buffer, info.offset, frameBuffer, stride, sliceHeight, width, height);
-          codec.releaseOutputBuffer(result, false);
-        }
+    DecodedTextureBuffer buffer = new DecodedTextureBuffer(
+        index, width, height, rotation, info.presentationTimeUs, decodeTimeMs);
+    decodedTextureBuffers.offerLast(buffer);
+
+    maybeRenderTextureBuffers();
+    maybeDropTextureBuffer();
+  }
+
+  private void maybeRenderTextureBuffers() {
+    // Possibly render more than one buffer, to allow the callee to catch up.
+    while (!decodedTextureBuffers.isEmpty() && !surfaceTextureHelper.isTextureInUse()) {

[sakal, 2017/07/17 13:44:56]

I don't think we are going to get more than one loop with this strategy. We can
render even if the surface is in use. SurfaceTextureHelper will call
updateTexImage only after the texture is free. However, if we render two frames
in this period, we will only get the last one.

Overall, I don't think SurfaceTextureHelper is a good fit here. If you want to
implement serializing access to the texture yourself, we should create a
SurfaceTexture in single buffer mode: https://goo.gl/FuXqV7

Alternatively, we could use SurfaceTextureHelper and just always render the
frame. However, then we will drop frames if the renderer doesn't keep up.

[mellem, 2017/07/17 21:57:15]

I need to call updateTexImage() whenever I get a texture frame callback, to
update the texture to the right contents.  updateTexImage() must be called on
the thread with the EGL context holding the texture.  SurfaceTexture's callbacks
are on an arbitrary thread.  SurfaceTextureHelper owns the thread, EGL context,
and texture, and calls updateTexImage() on the right thread.

We can't call updateTexImage() while a frame is in flight, or the texture will
change out from under it.  SurfaceTextureHelper's synchronization prevents that.

We can only convert a texture frame to YUV on the thread that owns the texture. 
SurfaceTextureHelper performs that operation on the correct thread.

As far as I can tell, I need almost everything SurfaceTextureHelper does.

But I don't need to buffer before rendering to the texture.  I can render
everything, as long as I'm careful to keep renderedTextureMetadata in sync with
the last thing I rendered.  (Otherwise we could attach the wrong metadata to an
image.)

[sakal, 2017/07/18 08:53:40]

Yeah, we of course would have had to implement drawing the new frame only after
the last frame has been released. SurfaceTexture gives callbacks on a given
handler if you supply it with one.

However, I think the current approach looks good as well.

+      DecodedTextureBuffer buffer = decodedTextureBuffers.pollFirst();
+      renderedTextureBuffer.set(buffer);
+      codec.releaseOutputBuffer(buffer.index, true);
+    }
+  }
+
+  private void maybeDropTextureBuffer() {
+    if (decodedTextureBuffers.size() > MAX_QUEUED_OUTPUTBUFFERS) {
+      ++droppedFrames;
+      DecodedTextureBuffer buffer = decodedTextureBuffers.pollFirst();
+      Logging.w(TAG,
+          "Draining the decoder.  Dropped frame with timestamp : " + buffer.presentationTimestampUs
+              + "(us).  Total dropped frames: " + droppedFrames);
+      codec.releaseOutputBuffer(buffer.index, false);
+    }
+  }
+
+  @Override
+  public void onTextureFrameAvailable(int oesTextureId, float[] transformMatrix, long timestampNs) {
+    DecodedTextureBuffer info = renderedTextureBuffer.getAndSet(null);
+
+    OesTextureBuffer oesBuffer = new OesTextureBuffer(
+        oesTextureId, info.width, info.height, transformMatrix, surfaceTextureHelper);
+
+    // SurfaceTexture's matrix looks like this:

[sakal, 2017/07/17 13:44:56]

Please copy the helper method from my other CL here:
https://chromium-review.googlesource.com/c/555516/4/webrtc/sdk/android/api/or...

[mellem, 2017/07/17 21:57:15]

Done.

+    // [x1 y1  0 w1]
+    // [x2 y2  0 w2]
+    // [ 0  0  1  0]
+    // [x3 y3  0 w3]
+    // The android.graphics.Matrix looks like this:
+    // [x1 y1 w1]
+    // [x2 y2 w2]
+    // [x3 y3 w3]
+    float[] matrix3x3 = new float[9];
+    matrix3x3[0 * 3 + 0] = transformMatrix[0 * 4 + 0];
+    matrix3x3[0 * 3 + 1] = transformMatrix[0 * 4 + 1];
+    matrix3x3[0 * 3 + 2] = transformMatrix[0 * 4 + 3];
+    matrix3x3[1 * 3 + 0] = transformMatrix[1 * 4 + 0];
+    matrix3x3[1 * 3 + 1] = transformMatrix[1 * 4 + 1];
+    matrix3x3[1 * 3 + 2] = transformMatrix[1 * 4 + 3];
+    matrix3x3[2 * 3 + 0] = transformMatrix[3 * 4 + 0];
+    matrix3x3[2 * 3 + 1] = transformMatrix[3 * 4 + 1];
+    matrix3x3[2 * 3 + 2] = transformMatrix[3 * 4 + 3];
+
+    Matrix matrix = new Matrix();
+    matrix.setValues(matrix3x3);
+
+    VideoFrame frame =
+        new VideoFrame(oesBuffer, info.rotation, info.presentationTimestampUs * 1000, matrix);
+    callback.onDecodedFrame(frame, info.decodeTimeMs, null /* qp */);
+    frame.release();
+  }
+
+  private void deliverByteFrame(
+      int result, MediaCodec.BufferInfo info, int rotation, Integer decodeTimeMs) {
+    // Load dimensions from shared memory under the dimension lock.
+    int width, height, stride, sliceHeight;
+    synchronized (dimensionLock) {
+      width = this.width;
+      height = this.height;
+      stride = this.stride;
+      sliceHeight = this.sliceHeight;
+    }
+
+    // Output must be at least width * height bytes for Y channel, plus (width / 2) * (height / 2)
+    // bytes for each of the U and V channels.
+    if (info.size < width * height * 3 / 2) {
+      Logging.e(TAG, "Insufficient output buffer size: " + info.size);
+      return;
+    }
+
+    if (info.size < stride * height * 3 / 2 && sliceHeight == height && stride > width) {
+      // Some codecs (Exynos) report an incorrect stride.  Correct it here.
+      // Expected size == stride * height * 3 / 2.  A bit of algebra gives the correct stride as
+      // 2 * size / (3 * height).
+      stride = info.size * 2 / (height * 3);
+    }
+
+    ByteBuffer buffer = codec.getOutputBuffers()[result];
+    buffer.position(info.offset);
+    buffer.limit(info.size);
+
+    final VideoFrame.I420Buffer frameBuffer;
+
+    // TODO(mellem):  As an optimization, use libyuv via JNI to copy/reformatting data.
+    if (colorFormat == CodecCapabilities.COLOR_FormatYUV420Planar) {
+      if (sliceHeight % 2 == 0) {
+        frameBuffer =
+            createBufferFromI420(buffer, result, info.offset, stride, sliceHeight, width, height);
       } else {
         frameBuffer = new I420BufferImpl(width, height);
-        // All other supported color formats are NV12.
-        nv12ToI420(buffer, info.offset, frameBuffer, stride, sliceHeight, width, height);
+        // Optimal path is not possible because we have to copy the last rows of U- and V-planes.
+        copyI420(buffer, info.offset, frameBuffer, stride, sliceHeight, width, height);
         codec.releaseOutputBuffer(result, false);
       }
+    } else {
+      frameBuffer = new I420BufferImpl(width, height);
+      // All other supported color formats are NV12.
+      nv12ToI420(buffer, info.offset, frameBuffer, stride, sliceHeight, width, height);
+      codec.releaseOutputBuffer(result, false);
+    }
 
-      long presentationTimeNs = info.presentationTimeUs * 1000;
-      VideoFrame frame = new VideoFrame(frameBuffer, rotation, presentationTimeNs, new Matrix());
+    long presentationTimeNs = info.presentationTimeUs * 1000;
+    VideoFrame frame = new VideoFrame(frameBuffer, rotation, presentationTimeNs, new Matrix());
 
-      // Note that qp is parsed on the C++ side.
-      callback.onDecodedFrame(frame, decodeTimeMs, null /* qp */);
-      frame.release();
-    } catch (IllegalStateException e) {
-      Logging.e(TAG, "deliverDecodedFrame failed", e);
-    }
+    // Note that qp is parsed on the C++ side.
+    callback.onDecodedFrame(frame, decodeTimeMs, null /* qp */);
+    frame.release();
   }
 
   private void reformat(MediaFormat format) {
@@ -429,7 +564,9 @@ class HardwareVideoDecoder implements VideoDecoder {
       height = newHeight;
     }
 
-    if (format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) {
+    // Note:  texture mode ignores colorFormat.  Hence, if the texture helper is non-null, skip
+    // color format updates.
+    if (surfaceTextureHelper == null && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) {
       colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT);
       Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat));
       if (!isSupportedColorFormat(colorFormat)) {
@@ -519,81 +656,20 @@ class HardwareVideoDecoder implements VideoDecoder {
     synchronized (activeOutputBuffersLock) {
       activeOutputBuffers++;
     }
-    return new VideoFrame.I420Buffer() {
-      private int refCount = 1;
-
-      @Override
-      public ByteBuffer getDataY() {
-        ByteBuffer data = buffer.slice();
-        data.position(yPos);
-        data.limit(yPos + getStrideY() * height);
-        return data;
-      }
 
+    I420BufferImpl.ReleaseCallback callback = new I420BufferImpl.ReleaseCallback() {
       @Override
-      public ByteBuffer getDataU() {
-        ByteBuffer data = buffer.slice();
-        data.position(uPos);
-        data.limit(uPos + getStrideU() * chromaHeight);
-        return data;
-      }
-
-      @Override
-      public ByteBuffer getDataV() {
-        ByteBuffer data = buffer.slice();
-        data.position(vPos);
-        data.limit(vPos + getStrideV() * chromaHeight);
-        return data;
-      }
-
-      @Override
-      public int getStrideY() {
-        return stride;
-      }
-
-      @Override
-      public int getStrideU() {
-        return uvStride;
-      }
-
-      @Override
-      public int getStrideV() {
-        return uvStride;
-      }
-
-      @Override
-      public int getWidth() {
-        return width;
-      }
-
-      @Override
-      public int getHeight() {
-        return height;
-      }
-
-      @Override
-      public VideoFrame.I420Buffer toI420() {
-        return this;
-      }
-
-      @Override
-      public void retain() {
-        refCount++;
-      }
-
-      @Override
-      public void release() {
-        refCount--;
-
-        if (refCount == 0) {
-          codec.releaseOutputBuffer(outputBufferIndex, false);
-          synchronized (activeOutputBuffersLock) {
-            activeOutputBuffers--;
-            activeOutputBuffersLock.notifyAll();
-          }
+      public void onRelease() {
+        codec.releaseOutputBuffer(outputBufferIndex, false);
+        synchronized (activeOutputBuffersLock) {
+          activeOutputBuffers--;
+          activeOutputBuffersLock.notifyAll();
         }
       }
     };
+
+    return new I420BufferImpl(
+        buffer, width, height, yPos, stride, uPos, uvStride, vPos, uvStride, callback);
   }
 
   private static void copyI420(ByteBuffer src, int offset, VideoFrame.I420Buffer frameBuffer,