Add texture support to HardwareVideoDecoder.

webrtc/sdk/android/src/java/org/webrtc/HardwareVideoDecoder.java

 /*
  *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 package org.webrtc;
 
 import android.annotation.TargetApi;
 import android.graphics.Matrix;
 import android.media.MediaCodec;
 import android.media.MediaCodecInfo.CodecCapabilities;
 import android.media.MediaFormat;
 import android.os.SystemClock;
+import android.view.Surface;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Deque;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingDeque;
 import org.webrtc.ThreadUtils.ThreadChecker;
 
 /** Android hardware video decoder. */
 @TargetApi(16)
 @SuppressWarnings("deprecation") // Cannot support API 16 without using deprecated methods.
-class HardwareVideoDecoder implements VideoDecoder {
+class HardwareVideoDecoder
+    implements VideoDecoder, SurfaceTextureHelper.OnTextureFrameAvailableListener {
   private static final String TAG = "HardwareVideoDecoder";
 
   // TODO(magjed): Use MediaFormat.KEY_* constants when part of the public API.
   private static final String MEDIA_FORMAT_KEY_STRIDE = "stride";
   private static final String MEDIA_FORMAT_KEY_SLICE_HEIGHT = "slice-height";
   private static final String MEDIA_FORMAT_KEY_CROP_LEFT = "crop-left";
   private static final String MEDIA_FORMAT_KEY_CROP_RIGHT = "crop-right";
   private static final String MEDIA_FORMAT_KEY_CROP_TOP = "crop-top";
   private static final String MEDIA_FORMAT_KEY_CROP_BOTTOM = "crop-bottom";
 
   // MediaCodec.release() occasionally hangs.  Release stops waiting and reports failure after
   // this timeout.
   private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000;
 
   // WebRTC queues input frames quickly in the beginning on the call. Wait for input buffers with a
   // long timeout (500 ms) to prevent this from causing the codec to return an error.
   private static final int DEQUEUE_INPUT_TIMEOUT_US = 500000;
 
   // Dequeuing an output buffer will block until a buffer is available (up to 100 milliseconds).
   // If this timeout is exceeded, the output thread will unblock and check if the decoder is still
   // running.  If it is, it will block on dequeue again.  Otherwise, it will stop and release the
   // MediaCodec.
   private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000;
 
+  // Max number of output buffers queued before starting to drop decoded frames.

[sakal, 2017/07/18 08:53:40]

Not used anymore?

[mellem, 2017/07/18 16:50:03]

Done.

+  private static final int MAX_QUEUED_OUTPUTBUFFERS = 3;
+
   private final String codecName;
   private final VideoCodecType codecType;
 
   private static class FrameInfo {
     final long decodeStartTimeMs;
     final int rotation;
 
     FrameInfo(long decodeStartTimeMs, int rotation) {
       this.decodeStartTimeMs = decodeStartTimeMs;
       this.rotation = rotation;
@@ skipping 28 matching lines @@
   private int height;
   private int stride;
   private int sliceHeight;
 
   // Whether the decoder has finished the first frame.  The codec may not change output dimensions
   // after delivering the first frame.
   private boolean hasDecodedFirstFrame;
   // Whether the decoder has seen a key frame.  The first frame must be a key frame.
   private boolean keyFrameRequired;
 
+  private final EglBase.Context sharedContext;
+  private SurfaceTextureHelper surfaceTextureHelper;
+  private Surface surface = null;
+
+  private static class DecodedTextureMetadata {
+    final int width;
+    final int height;
+    final int rotation;
+    final long presentationTimestampUs;
+    final Integer decodeTimeMs;
+
+    DecodedTextureMetadata(
+        int width, int height, int rotation, long presentationTimestampUs, Integer decodeTimeMs) {
+      this.width = width;
+      this.height = height;
+      this.rotation = rotation;
+      this.presentationTimestampUs = presentationTimestampUs;
+      this.decodeTimeMs = decodeTimeMs;
+    }
+  }
+
+  // Metadata for the last frame rendered to the texture.  Only accessed on the texture helper's
+  // thread.
+  private DecodedTextureMetadata renderedTextureMetadata;
+
+  private int droppedFrames = 0;

[sakal, 2017/07/18 08:53:40]

Not used?

[mellem, 2017/07/18 16:50:03]

Done.

+
   // Decoding proceeds asynchronously.  This callback returns decoded frames to the caller.
   private Callback callback;
 
   private MediaCodec codec = null;
 
-  HardwareVideoDecoder(String codecName, VideoCodecType codecType, int colorFormat) {
+  HardwareVideoDecoder(
+      String codecName, VideoCodecType codecType, int colorFormat, EglBase.Context sharedContext) {
     if (!isSupportedColorFormat(colorFormat)) {
       throw new IllegalArgumentException("Unsupported color format: " + colorFormat);
     }
     this.codecName = codecName;
     this.codecType = codecType;
     this.colorFormat = colorFormat;
+    this.sharedContext = sharedContext;
     this.frameInfos = new LinkedBlockingDeque<>();
   }
 
   @Override
   public VideoCodecStatus initDecode(Settings settings, Callback callback) {
     this.decoderThreadChecker = new ThreadChecker();
     return initDecodeInternal(settings.width, settings.height, callback);
   }
 
   private VideoCodecStatus initDecodeInternal(int width, int height, Callback callback) {
     decoderThreadChecker.checkIsOnValidThread();
     if (outputThread != null) {
       Logging.e(TAG, "initDecodeInternal called while the codec is already running");
       return VideoCodecStatus.ERROR;
     }
 
     // Note:  it is not necessary to initialize dimensions under the lock, since the output thread
     // is not running.
     this.callback = callback;
     this.width = width;
     this.height = height;
 
     stride = width;
     sliceHeight = height;
     hasDecodedFirstFrame = false;
     keyFrameRequired = true;
+    droppedFrames = 0;
 
     try {
       codec = MediaCodec.createByCodecName(codecName);
     } catch (IOException | IllegalArgumentException e) {
       Logging.e(TAG, "Cannot create media decoder " + codecName);
       return VideoCodecStatus.ERROR;
     }
     try {
       MediaFormat format = MediaFormat.createVideoFormat(codecType.mimeType(), width, height);
-      format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
-      codec.configure(format, null, null, 0);
+      if (sharedContext == null) {
+        format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
+      } else {
+        surfaceTextureHelper = SurfaceTextureHelper.create("decoder-texture-thread", sharedContext);
+        surface = new Surface(surfaceTextureHelper.getSurfaceTexture());
+        surfaceTextureHelper.startListening(this);
+      }
+      codec.configure(format, surface, null, 0);
       codec.start();
     } catch (IllegalStateException e) {
       Logging.e(TAG, "initDecode failed", e);
       release();
       return VideoCodecStatus.ERROR;
     }
 
     running = true;
     outputThread = createOutputThread();
     outputThread.start();
@@ skipping 40 matching lines @@
       if (frame.frameType != EncodedImage.FrameType.VideoFrameKey) {
         Logging.e(TAG, "decode() - key frame required first");
         return VideoCodecStatus.ERROR;
       }
       if (!frame.completeFrame) {
         Logging.e(TAG, "decode() - complete frame required first");
         return VideoCodecStatus.ERROR;
       }
     }
 
-    // TODO(mellem):  Support textures.
     int index;
     try {
       index = codec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT_US);
     } catch (IllegalStateException e) {
       Logging.e(TAG, "dequeueInputBuffer failed", e);
       return VideoCodecStatus.ERROR;
     }
     if (index < 0) {
       // Decoder is falling behind.  No input buffers available.
       // The decoder can't simply drop frames; it might lose a key frame.
@@ skipping 58 matching lines @@
         // capture both the output thread's stack trace and this thread's stack trace.
         Logging.e(TAG, "Media encoder release error", new RuntimeException(shutdownException));
         shutdownException = null;
         return VideoCodecStatus.ERROR;
       }
     } finally {
       codec = null;
       callback = null;
       outputThread = null;
       frameInfos.clear();
+      if (surface != null) {
+        surface.release();
+        surface = null;
+        surfaceTextureHelper.stopListening();
+        surfaceTextureHelper.dispose();
+        surfaceTextureHelper = null;
+      }
     }
     return VideoCodecStatus.OK;
   }
 
   private VideoCodecStatus reinitDecode(int newWidth, int newHeight) {
     decoderThreadChecker.checkIsOnValidThread();
     VideoCodecStatus status = release();
     if (status != VideoCodecStatus.OK) {
       return status;
     }
@@ skipping 35 matching lines @@
       FrameInfo frameInfo = frameInfos.poll();
       Integer decodeTimeMs = null;
       int rotation = 0;
       if (frameInfo != null) {
         decodeTimeMs = (int) (SystemClock.elapsedRealtime() - frameInfo.decodeStartTimeMs);
         rotation = frameInfo.rotation;
       }
 
       hasDecodedFirstFrame = true;
 
-      // Load dimensions from shared memory under the dimension lock.
-      int width, height, stride, sliceHeight;
-      synchronized (dimensionLock) {
-        width = this.width;
-        height = this.height;
-        stride = this.stride;
-        sliceHeight = this.sliceHeight;
+      if (surfaceTextureHelper != null) {
+        deliverTextureFrame(result, info, rotation, decodeTimeMs);
+      } else {
+        deliverByteFrame(result, info, rotation, decodeTimeMs);
       }
 
-      // Output must be at least width * height bytes for Y channel, plus (width / 2) * (height / 2)
-      // bytes for each of the U and V channels.
-      if (info.size < width * height * 3 / 2) {
-        Logging.e(TAG, "Insufficient output buffer size: " + info.size);
-        return;
-      }
-
-      if (info.size < stride * height * 3 / 2 && sliceHeight == height && stride > width) {
-        // Some codecs (Exynos) report an incorrect stride.  Correct it here.
-        // Expected size == stride * height * 3 / 2.  A bit of algebra gives the correct stride as
-        // 2 * size / (3 * height).
-        stride = info.size * 2 / (height * 3);
-      }
-
-      ByteBuffer buffer = codec.getOutputBuffers()[result];
-      buffer.position(info.offset);
-      buffer.limit(info.size);
-
-      final VideoFrame.I420Buffer frameBuffer;
-
-      // TODO(mellem):  As an optimization, use libyuv via JNI to copy/reformatting data.
-      if (colorFormat == CodecCapabilities.COLOR_FormatYUV420Planar) {
-        if (sliceHeight % 2 == 0) {
-          frameBuffer =
-              createBufferFromI420(buffer, result, info.offset, stride, sliceHeight, width, height);
-        } else {
-          frameBuffer = new I420BufferImpl(width, height);
-          // Optimal path is not possible because we have to copy the last rows of U- and V-planes.
-          copyI420(buffer, info.offset, frameBuffer, stride, sliceHeight, width, height);
-          codec.releaseOutputBuffer(result, false);
-        }
-      } else {
-        frameBuffer = new I420BufferImpl(width, height);
-        // All other supported color formats are NV12.
-        nv12ToI420(buffer, info.offset, frameBuffer, stride, sliceHeight, width, height);
-        codec.releaseOutputBuffer(result, false);
-      }
-
-      long presentationTimeNs = info.presentationTimeUs * 1000;
-      VideoFrame frame = new VideoFrame(frameBuffer, rotation, presentationTimeNs, new Matrix());
-
-      // Note that qp is parsed on the C++ side.
-      callback.onDecodedFrame(frame, decodeTimeMs, null /* qp */);
-      frame.release();
     } catch (IllegalStateException e) {
       Logging.e(TAG, "deliverDecodedFrame failed", e);
     }
   }
 
+  private void deliverTextureFrame(final int index, final MediaCodec.BufferInfo info,
+      final int rotation, final Integer decodeTimeMs) {
+    // Load dimensions from shared memory under the dimension lock.
+    final int width, height;
+    synchronized (dimensionLock) {
+      width = this.width;
+      height = this.height;
+    }
+
+    surfaceTextureHelper.getHandler().post(new Runnable() {
+      @Override
+      public void run() {
+        renderedTextureMetadata = new DecodedTextureMetadata(
+            width, height, rotation, info.presentationTimeUs, decodeTimeMs);
+        codec.releaseOutputBuffer(index, true);
+      }
+    });
+  }
+
+  @Override
+  public void onTextureFrameAvailable(int oesTextureId, float[] transformMatrix, long timestampNs) {
+    VideoFrame.TextureBuffer oesBuffer = surfaceTextureHelper.createTextureBuffer(
+        renderedTextureMetadata.width, renderedTextureMetadata.height, transformMatrix);
+
+    Matrix matrix = RendererCommon.convertMatrixToAndroidGraphicsMatrix(transformMatrix);
+
+    VideoFrame frame = new VideoFrame(oesBuffer, renderedTextureMetadata.rotation,
+        renderedTextureMetadata.presentationTimestampUs * 1000, matrix);
+    callback.onDecodedFrame(frame, renderedTextureMetadata.decodeTimeMs, null /* qp */);
+    frame.release();
+  }
+
+  private void deliverByteFrame(
+      int result, MediaCodec.BufferInfo info, int rotation, Integer decodeTimeMs) {
+    // Load dimensions from shared memory under the dimension lock.
+    int width, height, stride, sliceHeight;
+    synchronized (dimensionLock) {
+      width = this.width;
+      height = this.height;
+      stride = this.stride;
+      sliceHeight = this.sliceHeight;
+    }
+
+    // Output must be at least width * height bytes for Y channel, plus (width / 2) * (height / 2)
+    // bytes for each of the U and V channels.
+    if (info.size < width * height * 3 / 2) {
+      Logging.e(TAG, "Insufficient output buffer size: " + info.size);
+      return;
+    }
+
+    if (info.size < stride * height * 3 / 2 && sliceHeight == height && stride > width) {
+      // Some codecs (Exynos) report an incorrect stride.  Correct it here.
+      // Expected size == stride * height * 3 / 2.  A bit of algebra gives the correct stride as
+      // 2 * size / (3 * height).
+      stride = info.size * 2 / (height * 3);
+    }
+
+    ByteBuffer buffer = codec.getOutputBuffers()[result];
+    buffer.position(info.offset);
+    buffer.limit(info.size);
+
+    final VideoFrame.I420Buffer frameBuffer;
+
+    // TODO(mellem):  As an optimization, use libyuv via JNI to copy/reformatting data.
+    if (colorFormat == CodecCapabilities.COLOR_FormatYUV420Planar) {
+      if (sliceHeight % 2 == 0) {
+        frameBuffer =
+            createBufferFromI420(buffer, result, info.offset, stride, sliceHeight, width, height);
+      } else {
+        frameBuffer = I420BufferImpl.allocate(width, height);
+        // Optimal path is not possible because we have to copy the last rows of U- and V-planes.
+        copyI420(buffer, info.offset, frameBuffer, stride, sliceHeight, width, height);
+        codec.releaseOutputBuffer(result, false);
+      }
+    } else {
+      frameBuffer = I420BufferImpl.allocate(width, height);
+      // All other supported color formats are NV12.
+      nv12ToI420(buffer, info.offset, frameBuffer, stride, sliceHeight, width, height);
+      codec.releaseOutputBuffer(result, false);
+    }
+
+    long presentationTimeNs = info.presentationTimeUs * 1000;
+    VideoFrame frame = new VideoFrame(frameBuffer, rotation, presentationTimeNs, new Matrix());
+
+    // Note that qp is parsed on the C++ side.
+    callback.onDecodedFrame(frame, decodeTimeMs, null /* qp */);
+    frame.release();
+  }
+
   private void reformat(MediaFormat format) {
     outputThreadChecker.checkIsOnValidThread();
     Logging.d(TAG, "Decoder format changed: " + format.toString());
     final int newWidth;
     final int newHeight;
     if (format.containsKey(MEDIA_FORMAT_KEY_CROP_LEFT)
         && format.containsKey(MEDIA_FORMAT_KEY_CROP_RIGHT)
         && format.containsKey(MEDIA_FORMAT_KEY_CROP_BOTTOM)
         && format.containsKey(MEDIA_FORMAT_KEY_CROP_TOP)) {
       newWidth = 1 + format.getInteger(MEDIA_FORMAT_KEY_CROP_RIGHT)
           - format.getInteger(MEDIA_FORMAT_KEY_CROP_LEFT);
       newHeight = 1 + format.getInteger(MEDIA_FORMAT_KEY_CROP_BOTTOM)
           - format.getInteger(MEDIA_FORMAT_KEY_CROP_TOP);
     } else {
       newWidth = format.getInteger(MediaFormat.KEY_WIDTH);
       newHeight = format.getInteger(MediaFormat.KEY_HEIGHT);
     }
     // Compare to existing width, height, and save values under the dimension lock.
     synchronized (dimensionLock) {
       if (hasDecodedFirstFrame && (width != newWidth || height != newHeight)) {
         stopOnOutputThread(new RuntimeException("Unexpected size change. Configured " + width + "*"
             + height + ". New " + newWidth + "*" + newHeight));
         return;
       }
       width = newWidth;
       height = newHeight;
     }
 
-    if (format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) {
+    // Note:  texture mode ignores colorFormat.  Hence, if the texture helper is non-null, skip
+    // color format updates.
+    if (surfaceTextureHelper == null && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) {
       colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT);
       Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat));
       if (!isSupportedColorFormat(colorFormat)) {
         stopOnOutputThread(new IllegalStateException("Unsupported color format: " + colorFormat));
         return;
       }
     }
 
     // Save stride and sliceHeight under the dimension lock.
     synchronized (dimensionLock) {
@@ skipping 69 matching lines @@
     final int chromaWidth = (width + 1) / 2;
     final int chromaHeight = (height + 1) / 2;
 
     final int yPos = offset;
     final int uPos = yPos + stride * sliceHeight;
     final int vPos = uPos + uvStride * sliceHeight / 2;
 
     synchronized (activeOutputBuffersLock) {
       activeOutputBuffers++;
     }
-    return new VideoFrame.I420Buffer() {
-      private int refCount = 1;
 
+    I420BufferImpl.ReleaseCallback callback = new I420BufferImpl.ReleaseCallback() {
       @Override
-      public ByteBuffer getDataY() {
-        ByteBuffer data = buffer.slice();
-        data.position(yPos);
-        data.limit(yPos + getStrideY() * height);
-        return data;
-      }
-
-      @Override
-      public ByteBuffer getDataU() {
-        ByteBuffer data = buffer.slice();
-        data.position(uPos);
-        data.limit(uPos + getStrideU() * chromaHeight);
-        return data;
-      }
-
-      @Override
-      public ByteBuffer getDataV() {
-        ByteBuffer data = buffer.slice();
-        data.position(vPos);
-        data.limit(vPos + getStrideV() * chromaHeight);
-        return data;
-      }
-
-      @Override
-      public int getStrideY() {
-        return stride;
-      }
-
-      @Override
-      public int getStrideU() {
-        return uvStride;
-      }
-
-      @Override
-      public int getStrideV() {
-        return uvStride;
-      }
-
-      @Override
-      public int getWidth() {
-        return width;
-      }
-
-      @Override
-      public int getHeight() {
-        return height;
-      }
-
-      @Override
-      public VideoFrame.I420Buffer toI420() {
-        return this;
-      }
-
-      @Override
-      public void retain() {
-        refCount++;
-      }
-
-      @Override
-      public void release() {
-        refCount--;
-
-        if (refCount == 0) {
-          codec.releaseOutputBuffer(outputBufferIndex, false);
-          synchronized (activeOutputBuffersLock) {
-            activeOutputBuffers--;
-            activeOutputBuffersLock.notifyAll();
-          }
+      public void onRelease() {
+        codec.releaseOutputBuffer(outputBufferIndex, false);
+        synchronized (activeOutputBuffersLock) {
+          activeOutputBuffers--;
+          activeOutputBuffersLock.notifyAll();
         }
       }
     };
+
+    return new I420BufferImpl(
+        buffer, width, height, yPos, stride, uPos, uvStride, vPos, uvStride, callback);
   }
 
   private static void copyI420(ByteBuffer src, int offset, VideoFrame.I420Buffer frameBuffer,
       int stride, int sliceHeight, int width, int height) {
     int uvStride = stride / 2;
     int chromaWidth = (width + 1) / 2;
     // Note that hardware truncates instead of rounding.  WebRTC expects rounding, so the last
     // row will be duplicated if the sliceHeight is odd.
     int chromaHeight = (sliceHeight % 2 == 0) ? (height + 1) / 2 : height / 2;
 
@@ skipping 52 matching lines @@
       dstPos += dstStride;
     }
   }
 
   private static void copyRow(ByteBuffer src, int srcPos, ByteBuffer dst, int dstPos, int width) {
     for (int i = 0; i < width; ++i) {
       dst.put(dstPos + i, src.get(srcPos + i));
     }
   }
 }