Add dynamic bitrate tracker and adjustment for Exynos VP8 HW encoder.

webrtc/api/android/java/src/org/webrtc/MediaCodecVideoEncoder.java

Index: webrtc/api/android/java/src/org/webrtc/MediaCodecVideoEncoder.java
diff --git a/webrtc/api/android/java/src/org/webrtc/MediaCodecVideoEncoder.java b/webrtc/api/android/java/src/org/webrtc/MediaCodecVideoEncoder.java
index 72544bd83f0babb186b3ba1f1c1d90102d19433a..923ea91441631f1bba155def2c2acd55394c3d85 100644
--- a/webrtc/api/android/java/src/org/webrtc/MediaCodecVideoEncoder.java
+++ b/webrtc/api/android/java/src/org/webrtc/MediaCodecVideoEncoder.java
@@ -52,6 +52,12 @@ public class MediaCodecVideoEncoder {
   private static final int DEQUEUE_TIMEOUT = 0;  // Non-blocking, no wait.
   private static final int BITRATE_ADJUSTMENT_FPS = 30;
   private static final int MAXIMUM_INITIAL_FPS = 30;
+  private static final double BITRATE_CORRECTION_SEC = 3.0;
+  // Maximum bitrate correction scale - no more than 2 times.
+  private static final double BITRATE_CORRECTION_MAX_SCALE = 2;
+  // Amount of correction steps to reach correction maximum scale.
+  private static final int BITRATE_CORRECTION_STEPS = 10;
+
   // Active running encoder instance. Set in initEncode() (called from native code)
   // and reset to null in release() call.
   private static MediaCodecVideoEncoder runningInstance = null;
@@ -73,6 +79,19 @@ public class MediaCodecVideoEncoder {
   private static final String VP9_MIME_TYPE = "video/x-vnd.on2.vp9";
   private static final String H264_MIME_TYPE = "video/avc";
 
+  // Type of bitrate adjustment for video encoder.
+  public enum BitrateAdjustmentType {
+    // No adjustment - video encoder has no known bitrate problem.
+    NO_ADJUSTMENT,
+    // Framerate based bitrate adjustment is required - HW encoder does not use frame
+    // timestamps to calculate frame bitrate budget and instead is relying on initial
+    // fps configuration assuming that all frames are coming at fixed initial frame rate.
+    FRAMERATE_ADJUSTMENT,
+    // Dynamic bitrate adjustment is required - HW encoder used frame timestamps, but actual
+    // bitrate deviates too much from the target value.
+    DYNAMIC_ADJUSTMENT
+  }
+
   // Class describing supported media codec properties.
   private static class MediaCodecProperties {
     public final String codecPrefix;
@@ -81,39 +100,39 @@ public class MediaCodecVideoEncoder {
     // Flag if encoder implementation does not use frame timestamps to calculate frame bitrate
     // budget and instead is relying on initial fps configuration assuming that all frames are
     // coming at fixed initial frame rate. Bitrate adjustment is required for this case.
-    public final boolean bitrateAdjustmentRequired;
+    public final BitrateAdjustmentType bitrateAdjustmentType;
 
     MediaCodecProperties(
-        String codecPrefix, int minSdk, boolean bitrateAdjustmentRequired) {
+        String codecPrefix, int minSdk, BitrateAdjustmentType bitrateAdjustmentType) {
       this.codecPrefix = codecPrefix;
       this.minSdk = minSdk;
-      this.bitrateAdjustmentRequired = bitrateAdjustmentRequired;
+      this.bitrateAdjustmentType = bitrateAdjustmentType;
     }
   }
 
   // List of supported HW VP8 encoders.
   private static final MediaCodecProperties qcomVp8HwProperties = new MediaCodecProperties(
-      "OMX.qcom.", Build.VERSION_CODES.KITKAT, false /* bitrateAdjustmentRequired */);
+      "OMX.qcom.", Build.VERSION_CODES.KITKAT, BitrateAdjustmentType.NO_ADJUSTMENT);
   private static final MediaCodecProperties exynosVp8HwProperties = new MediaCodecProperties(
-      "OMX.Exynos.", Build.VERSION_CODES.M, false /* bitrateAdjustmentRequired */);
+      "OMX.Exynos.", Build.VERSION_CODES.M, BitrateAdjustmentType.DYNAMIC_ADJUSTMENT);
   private static final MediaCodecProperties[] vp8HwList = new MediaCodecProperties[] {
     qcomVp8HwProperties, exynosVp8HwProperties
   };
 
   // List of supported HW VP9 encoders.
   private static final MediaCodecProperties qcomVp9HwProperties = new MediaCodecProperties(
-      "OMX.qcom.", Build.VERSION_CODES.M, false /* bitrateAdjustmentRequired */);
+      "OMX.qcom.", Build.VERSION_CODES.M, BitrateAdjustmentType.NO_ADJUSTMENT);
   private static final MediaCodecProperties exynosVp9HwProperties = new MediaCodecProperties(
-      "OMX.Exynos.", Build.VERSION_CODES.M, false /* bitrateAdjustmentRequired */);
+      "OMX.Exynos.", Build.VERSION_CODES.M, BitrateAdjustmentType.NO_ADJUSTMENT);
   private static final MediaCodecProperties[] vp9HwList = new MediaCodecProperties[] {
     qcomVp9HwProperties, exynosVp9HwProperties
   };
 
   // List of supported HW H.264 encoders.
   private static final MediaCodecProperties qcomH264HwProperties = new MediaCodecProperties(
-      "OMX.qcom.", Build.VERSION_CODES.KITKAT, false /* bitrateAdjustmentRequired */);
+      "OMX.qcom.", Build.VERSION_CODES.KITKAT, BitrateAdjustmentType.NO_ADJUSTMENT);
   private static final MediaCodecProperties exynosH264HwProperties = new MediaCodecProperties(
-      "OMX.Exynos.", Build.VERSION_CODES.LOLLIPOP, true /* bitrateAdjustmentRequired */);
+      "OMX.Exynos.", Build.VERSION_CODES.LOLLIPOP, BitrateAdjustmentType.FRAMERATE_ADJUSTMENT);
   private static final MediaCodecProperties[] h264HwList = new MediaCodecProperties[] {
     qcomH264HwProperties, exynosH264HwProperties
   };
@@ -146,7 +165,15 @@ public class MediaCodecVideoEncoder {
   };
   private VideoCodecType type;
   private int colorFormat;  // Used by native code.
-  private boolean bitrateAdjustmentRequired;
+
+  // Variables used for dynamic bitrate adjustment.
+  private BitrateAdjustmentType bitrateAdjustmentType = BitrateAdjustmentType.NO_ADJUSTMENT;
+  private double bitrateAccumulator;
+  private double bitrateAccumulatorMax;
+  private double bitrateObservationTimeMs;
+  private int bitrateAdjustmentScaleExp;
+  private int targetBitrateBps;
+  private int targetFps;
 
   // SPS and PPS NALs (Config frame) for H.264.
   private ByteBuffer configData = null;
@@ -213,14 +240,15 @@ public class MediaCodecVideoEncoder {
 
   // Helper struct for findHwEncoder() below.
   private static class EncoderProperties {
-    public EncoderProperties(String codecName, int colorFormat, boolean bitrateAdjustment) {
+    public EncoderProperties(
+        String codecName, int colorFormat, BitrateAdjustmentType bitrateAdjustmentType) {
       this.codecName = codecName;
       this.colorFormat = colorFormat;
-      this.bitrateAdjustment = bitrateAdjustment;
+      this.bitrateAdjustmentType = bitrateAdjustmentType;
     }
     public final String codecName; // OpenMax component name for HW codec.
     public final int colorFormat;  // Color format supported by codec.
-    public final boolean bitrateAdjustment; // true if bitrate adjustment workaround is required.
+    public final BitrateAdjustmentType bitrateAdjustmentType; // Bitrate adjustment type
   }
 
   private static EncoderProperties findHwEncoder(
@@ -264,7 +292,7 @@ public class MediaCodecVideoEncoder {
 
       // Check if this is supported HW encoder.
       boolean supportedCodec = false;
-      boolean bitrateAdjustmentRequired = false;
+      BitrateAdjustmentType bitrateAdjustmentType = BitrateAdjustmentType.NO_ADJUSTMENT;
       for (MediaCodecProperties codecProperties : supportedHwCodecProperties) {
         if (name.startsWith(codecProperties.codecPrefix)) {
           if (Build.VERSION.SDK_INT < codecProperties.minSdk) {
@@ -272,9 +300,10 @@ public class MediaCodecVideoEncoder {
                 Build.VERSION.SDK_INT);
             continue;
           }
-          if (codecProperties.bitrateAdjustmentRequired) {
-            Logging.w(TAG, "Codec " + name + " does not use frame timestamps.");
-            bitrateAdjustmentRequired = true;
+          if (codecProperties.bitrateAdjustmentType != BitrateAdjustmentType.NO_ADJUSTMENT) {
+            bitrateAdjustmentType = codecProperties.bitrateAdjustmentType;
+            Logging.w(TAG, "Codec " + name
+                + " requires bitrate adjustment: " + bitrateAdjustmentType);
           }
           supportedCodec = true;
           break;
@@ -300,9 +329,10 @@ public class MediaCodecVideoEncoder {
         for (int codecColorFormat : capabilities.colorFormats) {
           if (codecColorFormat == supportedColorFormat) {
             // Found supported HW encoder.
-            Logging.d(TAG, "Found target encoder for mime " + mime + " : " + name +
-                ". Color: 0x" + Integer.toHexString(codecColorFormat));
-            return new EncoderProperties(name, codecColorFormat, bitrateAdjustmentRequired);
+            Logging.d(TAG, "Found target encoder for mime " + mime + " : " + name
+                + ". Color: 0x" + Integer.toHexString(codecColorFormat)
+                + ". Bitrate adjustment: " + bitrateAdjustmentType);
+            return new EncoderProperties(name, codecColorFormat, bitrateAdjustmentType);
           }
         }
       }
@@ -375,23 +405,29 @@ public class MediaCodecVideoEncoder {
     }
     runningInstance = this; // Encoder is now running and can be queried for stack traces.
     colorFormat = properties.colorFormat;
-    bitrateAdjustmentRequired = properties.bitrateAdjustment;
-    if (bitrateAdjustmentRequired) {
+    bitrateAdjustmentType = properties.bitrateAdjustmentType;
+    if (bitrateAdjustmentType == BitrateAdjustmentType.FRAMERATE_ADJUSTMENT) {
       fps = BITRATE_ADJUSTMENT_FPS;
     } else  {
       fps = Math.min(fps, MAXIMUM_INITIAL_FPS);
     }
     Logging.d(TAG, "Color format: " + colorFormat +
-        ". Bitrate adjustment: " + bitrateAdjustmentRequired +
+        ". Bitrate adjustment: " + bitrateAdjustmentType +
         ". Initial fps: " + fps);
+    targetBitrateBps = 1000 * kbps;
+    targetFps = fps;
+    bitrateAccumulatorMax = targetBitrateBps / 8.0;
+    bitrateAccumulator = 0;
+    bitrateObservationTimeMs = 0;
+    bitrateAdjustmentScaleExp = 0;
 
     mediaCodecThread = Thread.currentThread();
     try {
       MediaFormat format = MediaFormat.createVideoFormat(mime, width, height);
-      format.setInteger(MediaFormat.KEY_BIT_RATE, 1000 * kbps);
+      format.setInteger(MediaFormat.KEY_BIT_RATE, targetBitrateBps);
       format.setInteger("bitrate-mode", VIDEO_ControlRateConstant);
       format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat);
-      format.setInteger(MediaFormat.KEY_FRAME_RATE, fps);
+      format.setInteger(MediaFormat.KEY_FRAME_RATE, targetFps);
       format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, keyFrameIntervalSec);
       Logging.d(TAG, "  Format: " + format);
       mediaCodec = createByCodecName(properties.codecName);
@@ -529,17 +565,36 @@ public class MediaCodecVideoEncoder {
 
   private boolean setRates(int kbps, int frameRate) {
     checkOnMediaCodecThread();
-    int codecBitrate = 1000 * kbps;
-    if (bitrateAdjustmentRequired && frameRate > 0) {
-      codecBitrate = BITRATE_ADJUSTMENT_FPS * codecBitrate / frameRate;
-      Logging.v(TAG, "setRates: " + kbps + " -> " + (codecBitrate / 1000)
-          + " kbps. Fps: " + frameRate);
+
+    int codecBitrateBps = 1000 * kbps;
+    if (bitrateAdjustmentType == BitrateAdjustmentType.DYNAMIC_ADJUSTMENT) {
+      bitrateAccumulatorMax = codecBitrateBps / 8.0;
+      if (targetBitrateBps > 0 && codecBitrateBps < targetBitrateBps) {
+        // Rescale the accumulator level if the accumulator max decreases
+        bitrateAccumulator = bitrateAccumulator * codecBitrateBps / targetBitrateBps;
+      }
+    }
+    targetBitrateBps = codecBitrateBps;
+    targetFps = frameRate;
+
+    // Adjust actual encoder bitrate based on bitrate adjustment type.
+    if (bitrateAdjustmentType == BitrateAdjustmentType.FRAMERATE_ADJUSTMENT && targetFps > 0) {
+      codecBitrateBps = BITRATE_ADJUSTMENT_FPS * targetBitrateBps / targetFps;
+      Logging.v(TAG, "setRates: " + kbps + " -> " + (codecBitrateBps / 1000)
+          + " kbps. Fps: " + targetFps);
+    } else if (bitrateAdjustmentType == BitrateAdjustmentType.DYNAMIC_ADJUSTMENT) {
+      Logging.v(TAG, "setRates: " + kbps + " kbps. Fps: " + targetFps
+          + ". ExpScale: " + bitrateAdjustmentScaleExp);
+      if (bitrateAdjustmentScaleExp != 0) {
+        codecBitrateBps = (int)(codecBitrateBps * getBitrateScale(bitrateAdjustmentScaleExp));
+      }
     } else {
-      Logging.v(TAG, "setRates: " + kbps + " kbps. Fps: " + frameRate);
+      Logging.v(TAG, "setRates: " + kbps + " kbps. Fps: " + targetFps);
     }
+
     try {
       Bundle params = new Bundle();
-      params.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, codecBitrate);
+      params.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, codecBitrateBps);
       mediaCodec.setParameters(params);
       return true;
     } catch (IllegalStateException e) {
@@ -608,6 +663,8 @@ public class MediaCodecVideoEncoder {
         ByteBuffer outputBuffer = outputBuffers[result].duplicate();
         outputBuffer.position(info.offset);
         outputBuffer.limit(info.offset + info.size);
+        reportEncodedFrame(info.size);
+
         // Check key frame flag.
         boolean isKeyFrame =
             (info.flags & MediaCodec.BUFFER_FLAG_SYNC_FRAME) != 0;
@@ -646,6 +703,56 @@ public class MediaCodecVideoEncoder {
     }
   }
 
+  private double getBitrateScale(int bitrateAdjustmentScaleExp) {
+    return Math.pow(BITRATE_CORRECTION_MAX_SCALE,
+        (double)bitrateAdjustmentScaleExp / BITRATE_CORRECTION_STEPS);
+  }
+
+  private void reportEncodedFrame(int size) {
+    if (targetFps == 0 || bitrateAdjustmentType != BitrateAdjustmentType.DYNAMIC_ADJUSTMENT) {
+      return;
+    }
+
+    // Accumulate the difference between actial and expected frame sizes.
+    double expectedBytesPerFrame = targetBitrateBps / (8.0 * targetFps);
+    bitrateAccumulator += (size - expectedBytesPerFrame);
+    bitrateObservationTimeMs += 1000.0 / targetFps;
+
+    // Put a cap on the accumulator, i.e., don't let it grow beyond some level to avoid
+    // using too old data for bitrate adjustment.
+    double bitrateAccumulatorCap = BITRATE_CORRECTION_SEC * bitrateAccumulatorMax;
+    bitrateAccumulator = Math.min(bitrateAccumulator, bitrateAccumulatorCap);
+    bitrateAccumulator = Math.max(bitrateAccumulator, -bitrateAccumulatorCap);
+
+    // Do bitrate adjustment every 3 seconds if actual encoder bitrate deviates too much
+    // form the target value.
+    if (bitrateObservationTimeMs > 1000 * BITRATE_CORRECTION_SEC) {
+      Logging.d(TAG, "Acc: " + (int)bitrateAccumulator
+          + ". Max: " + (int)bitrateAccumulatorMax
+          + ". ExpScale: " + bitrateAdjustmentScaleExp);
+      boolean bitrateAdjustmentScaleChanged = false;
+      if (bitrateAccumulator > bitrateAccumulatorMax) {
+        // Encoder generates too high bitrate - need to reduce the scale.
+        bitrateAccumulator = bitrateAccumulatorMax;
+        bitrateAdjustmentScaleExp--;
+        bitrateAdjustmentScaleChanged = true;
+      } else if (bitrateAccumulator < -bitrateAccumulatorMax) {
+        // Encoder generates too low bitrate - need to increase the scale.
+        bitrateAdjustmentScaleExp++;
+        bitrateAccumulator = -bitrateAccumulatorMax;
+        bitrateAdjustmentScaleChanged = true;
+      }
+      if (bitrateAdjustmentScaleChanged) {
+        bitrateAdjustmentScaleExp = Math.min(bitrateAdjustmentScaleExp, BITRATE_CORRECTION_STEPS);
+        bitrateAdjustmentScaleExp = Math.max(bitrateAdjustmentScaleExp, -BITRATE_CORRECTION_STEPS);
+        Logging.d(TAG, "Adjusting bitrate scale to " + bitrateAdjustmentScaleExp
+            + ". Value: " + getBitrateScale(bitrateAdjustmentScaleExp));
+        setRates(targetBitrateBps / 1000, targetFps);
+      }
+      bitrateObservationTimeMs = 0;
+    }
+  }
+
   // Release a dequeued output buffer back to the codec for re-use.  Return
   // false if the codec is no longer operable.
   boolean releaseOutputBuffer(int index) {