Change capture time format to nanoseconds in EncodedImage.

webrtc/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java

 /*
  *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 package org.webrtc;
 
 import android.annotation.TargetApi;
 import android.graphics.Matrix;
 import android.media.MediaCodec;
 import android.media.MediaCodecInfo;
 import android.media.MediaFormat;
 import android.opengl.GLES20;
 import android.os.Bundle;
 import android.view.Surface;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Deque;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.TimeUnit;
 
 /** Android hardware video encoder. */
 @TargetApi(19)
 @SuppressWarnings("deprecation") // Cannot support API level 19 without using deprecated methods.
 class HardwareVideoEncoder implements VideoEncoder {
   private static final String TAG = "HardwareVideoEncoder";
 
   // Bitrate modes - should be in sync with OMX_VIDEO_CONTROLRATETYPE defined
   // in OMX_Video.h
   private static final int VIDEO_ControlRateConstant = 2;
@@ skipping 10 matching lines @@
   private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000;
 
   private final String codecName;
   private final VideoCodecType codecType;
   private final int colorFormat;
   private final ColorFormat inputColorFormat;
   // Base interval for generating key frames.
   private final int keyFrameIntervalSec;
   // Interval at which to force a key frame. Used to reduce color distortions caused by some
   // Qualcomm video encoders.
-  private final long forcedKeyFrameMs;
+  private final long forcedKeyFrameNs;
   // Presentation timestamp of the last requested (or forced) key frame.
-  private long lastKeyFrameMs;
+  private long lastKeyFrameNs;
 
   private final BitrateAdjuster bitrateAdjuster;
   private int adjustedBitrate;
 
   // A queue of EncodedImage.Builders that correspond to frames in the codec.  These builders are
   // pre-populated with all the information that can't be sent through MediaCodec.
   private final Deque<EncodedImage.Builder> outputBuilders;
 
   // Thread that delivers encoded frames to the user callback.
   private Thread outputThread;
@@ skipping 47 matching lines @@
     this.codecName = codecName;
     this.codecType = codecType;
     this.colorFormat = colorFormat;
     if (textureContext == null) {
       this.inputColorFormat = ColorFormat.valueOf(colorFormat);
     } else {
       // ColorFormat copies bytes between buffers.  It is not used in texture mode.
       this.inputColorFormat = null;
     }
     this.keyFrameIntervalSec = keyFrameIntervalSec;
-    this.forcedKeyFrameMs = forceKeyFrameIntervalMs;
+    this.forcedKeyFrameNs = TimeUnit.MILLISECONDS.toNanos(forceKeyFrameIntervalMs);
     this.bitrateAdjuster = bitrateAdjuster;
     this.outputBuilders = new LinkedBlockingDeque<>();
     this.textureContext = textureContext;
   }
 
   @Override
   public VideoCodecStatus initEncode(Settings settings, Callback callback) {
     return initEncodeInternal(
         settings.width, settings.height, settings.startBitrate, settings.maxFramerate, callback);
   }
 
   private VideoCodecStatus initEncodeInternal(
       int width, int height, int bitrateKbps, int fps, Callback callback) {
     Logging.d(
         TAG, "initEncode: " + width + " x " + height + ". @ " + bitrateKbps + "kbps. Fps: " + fps);
     this.width = width;
     this.height = height;
     if (bitrateKbps != 0 && fps != 0) {
       bitrateAdjuster.setTargets(bitrateKbps * 1000, fps);
     }
     adjustedBitrate = bitrateAdjuster.getAdjustedBitrateBps();
 
     this.callback = callback;
 
-    lastKeyFrameMs = -1;
+    lastKeyFrameNs = -1;
 
     try {
       codec = MediaCodec.createByCodecName(codecName);
     } catch (IOException | IllegalArgumentException e) {
       Logging.e(TAG, "Cannot create media encoder " + codecName);
       return VideoCodecStatus.ERROR;
     }
     try {
       MediaFormat format = MediaFormat.createVideoFormat(codecType.mimeType(), width, height);
       format.setInteger(MediaFormat.KEY_BIT_RATE, adjustedBitrate);
@@ skipping 86 matching lines @@
       return VideoCodecStatus.OK; // See webrtc bug 2887.
     }
 
     boolean requestedKeyFrame = false;
     for (EncodedImage.FrameType frameType : encodeInfo.frameTypes) {
       if (frameType == EncodedImage.FrameType.VideoFrameKey) {
         requestedKeyFrame = true;
       }
     }
 
-    // Frame timestamp rounded to the nearest microsecond and millisecond.
-    long presentationTimestampUs = (videoFrame.getTimestampNs() + 500) / 1000;
-    long presentationTimestampMs = (presentationTimestampUs + 500) / 1000;
-    if (requestedKeyFrame || shouldForceKeyFrame(presentationTimestampMs)) {
-      requestKeyFrame(presentationTimestampMs);
+    if (requestedKeyFrame || shouldForceKeyFrame(videoFrame.getTimestampNs())) {
+      requestKeyFrame(videoFrame.getTimestampNs());
     }
 
     VideoFrame.Buffer videoFrameBuffer = videoFrame.getBuffer();
     // Number of bytes in the video buffer. Y channel is sampled at one byte per pixel; U and V are
     // subsampled at one byte per four pixels.
     int bufferSize = videoFrameBuffer.getHeight() * videoFrameBuffer.getWidth() * 3 / 2;
     EncodedImage.Builder builder = EncodedImage.builder()
-                                       .setCaptureTimeMs(presentationTimestampMs)
+                                       .setCaptureTimeNs(videoFrame.getTimestampNs())
                                        .setCompleteFrame(true)
                                        .setEncodedWidth(videoFrame.getBuffer().getWidth())
                                        .setEncodedHeight(videoFrame.getBuffer().getHeight())
                                        .setRotation(videoFrame.getRotation());
     outputBuilders.offer(builder);
 
     if (textureContext != null) {
       if (!(videoFrameBuffer instanceof VideoFrame.TextureBuffer)) {
         Logging.e(TAG, "Cannot encode non-texture buffer in texture mode");
         return VideoCodecStatus.ERROR;
       }
       VideoFrame.TextureBuffer textureBuffer = (VideoFrame.TextureBuffer) videoFrameBuffer;
       return encodeTextureBuffer(videoFrame, textureBuffer);
     } else {
       if (videoFrameBuffer instanceof VideoFrame.TextureBuffer) {
         Logging.w(TAG, "Encoding texture buffer in byte mode; this may be inefficient");
       }
-      return encodeByteBuffer(videoFrame, videoFrameBuffer, bufferSize, presentationTimestampUs);
+      return encodeByteBuffer(videoFrame, videoFrameBuffer, bufferSize);
     }
   }
 
   private VideoCodecStatus encodeTextureBuffer(
       VideoFrame videoFrame, VideoFrame.TextureBuffer textureBuffer) {
     Matrix matrix = textureBuffer.getTransformMatrix();
     float[] transformationMatrix = RendererCommon.convertMatrixFromAndroidGraphicsMatrix(matrix);
 
     try {
       textureEglBase.makeCurrent();
@@ skipping 13 matching lines @@
       textureEglBase.swapBuffers(videoFrame.getTimestampNs());
     } catch (RuntimeException e) {
       Logging.e(TAG, "encodeTexture failed", e);
       // Keep the output builders in sync with buffers in the codec.
       outputBuilders.pollLast();
       return VideoCodecStatus.ERROR;
     }
     return VideoCodecStatus.OK;
   }
 
-  private VideoCodecStatus encodeByteBuffer(VideoFrame videoFrame,
-      VideoFrame.Buffer videoFrameBuffer, int bufferSize, long presentationTimestampUs) {
+  private VideoCodecStatus encodeByteBuffer(
+      VideoFrame videoFrame, VideoFrame.Buffer videoFrameBuffer, int bufferSize) {
+    // Frame timestamp rounded to the nearest microsecond.
+    long presentationTimestampUs = (videoFrame.getTimestampNs() + 500) / 1000;
+
     // No timeout.  Don't block for an input buffer, drop frames if the encoder falls behind.
     int index;
     try {
       index = codec.dequeueInputBuffer(0 /* timeout */);
     } catch (IllegalStateException e) {
       Logging.e(TAG, "dequeueInputBuffer failed", e);
       return VideoCodecStatus.FALLBACK_SOFTWARE;
     }
 
     if (index == -1) {
@@ skipping 54 matching lines @@
 
   private VideoCodecStatus resetCodec(int newWidth, int newHeight) {
     VideoCodecStatus status = release();
     if (status != VideoCodecStatus.OK) {
       return status;
     }
     // Zero bitrate and framerate indicate not to change the targets.
     return initEncodeInternal(newWidth, newHeight, 0, 0, callback);
   }
 
-  private boolean shouldForceKeyFrame(long presentationTimestampMs) {
-    return forcedKeyFrameMs > 0 && presentationTimestampMs > lastKeyFrameMs + forcedKeyFrameMs;
+  private boolean shouldForceKeyFrame(long presentationTimestampNs) {
+    return forcedKeyFrameNs > 0 && presentationTimestampNs > lastKeyFrameNs + forcedKeyFrameNs;
   }
 
-  private void requestKeyFrame(long presentationTimestampMs) {
+  private void requestKeyFrame(long presentationTimestampNs) {
     // Ideally MediaCodec would honor BUFFER_FLAG_SYNC_FRAME so we could
     // indicate this in queueInputBuffer() below and guarantee _this_ frame
     // be encoded as a key frame, but sadly that flag is ignored.  Instead,
     // we request a key frame "soon".
     try {
       Bundle b = new Bundle();
       b.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0);
       codec.setParameters(b);
     } catch (IllegalStateException e) {
       Logging.e(TAG, "requestKeyFrame failed", e);
       return;
     }
-    lastKeyFrameMs = presentationTimestampMs;
+    lastKeyFrameNs = presentationTimestampNs;
   }
 
   private Thread createOutputThread() {
     return new Thread() {
       @Override
       public void run() {
         while (running) {
           deliverEncodedImage();
         }
         releaseCodecOnOutputThread();
@@ skipping 122 matching lines @@
         case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
         case MediaCodecInfo.CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar:
         case MediaCodecUtils.COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m:
           return NV12;
         default:
           throw new IllegalArgumentException("Unsupported colorFormat: " + colorFormat);
       }
     }
   }
 }