Wire up QualityScaler for H.264 on Android.

talk/app/webrtc/java/jni/androidmediaencoder_jni.cc

 /*
  * libjingle
  * Copyright 2015 Google Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright notice,
@@ skipping 15 matching lines @@
  *
  */
 
 #include "talk/app/webrtc/java/jni/androidmediaencoder_jni.h"
 #include "talk/app/webrtc/java/jni/classreferenceholder.h"
 #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h"
 #include "webrtc/base/bind.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/thread.h"
+#include "webrtc/modules/rtp_rtcp/source/h264_bitstream_parser.h"
 #include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
 #include "webrtc/modules/video_coding/utility/include/quality_scaler.h"
 #include "webrtc/modules/video_coding/utility/include/vp8_header_parser.h"
 #include "webrtc/system_wrappers/interface/field_trial.h"
 #include "webrtc/system_wrappers/interface/logcat_trace_context.h"
 #include "third_party/libyuv/include/libyuv/convert.h"
 #include "third_party/libyuv/include/libyuv/convert_from.h"
 #include "third_party/libyuv/include/libyuv/video_common.h"
 
 using rtc::Bind;
@@ skipping 146 matching lines @@
   int32_t output_timestamp_;  // Last output frame timestamp from timestamps_ Q.
   int64_t output_render_time_ms_; // Last output frame render time from
                                   // render_times_ms_ queue.
   // Frame size in bytes fed to MediaCodec.
   int yuv_size_;
   // True only when between a callback_->Encoded() call return a positive value
   // and the next Encode() call being ignored.
   bool drop_next_input_frame_;
   // Global references; must be deleted in Release().
   std::vector<jobject> input_buffers_;
-  scoped_ptr<webrtc::QualityScaler> quality_scaler_;
+  webrtc::QualityScaler quality_scaler_;
   // Dynamic resolution change, off by default.
   bool scale_;
-  int updated_framerate_;
+
+  // H264 bitstream parser, used to extract QP from encoded bitstreams.
+  webrtc::H264BitstreamParser h264_bitstream_parser_;
 };
 
 MediaCodecVideoEncoder::~MediaCodecVideoEncoder() {
   // Call Release() to ensure no more callbacks to us after we are deleted.
   Release();
 }
 
 MediaCodecVideoEncoder::MediaCodecVideoEncoder(
     JNIEnv* jni, VideoCodecType codecType) :
     codecType_(codecType),
     callback_(NULL),
     inited_(false),
     picture_id_(0),
     codec_thread_(new Thread()),
-    quality_scaler_(new webrtc::QualityScaler()),
     j_media_codec_video_encoder_class_(
         jni,
         FindClass(jni, "org/webrtc/MediaCodecVideoEncoder")),
     j_media_codec_video_encoder_(
         jni,
         jni->NewObject(*j_media_codec_video_encoder_class_,
                        GetMethodID(jni,
                                    *j_media_codec_video_encoder_class_,
                                    "<init>",
                                    "()V"))) {
@@ skipping 45 matching lines @@
   CHECK_EXCEPTION(jni) << "MediaCodecVideoEncoder ctor failed";
   AllowBlockingCalls();
 }
 
 int32_t MediaCodecVideoEncoder::InitEncode(
     const webrtc::VideoCodec* codec_settings,
     int32_t /* number_of_cores */,
     size_t /* max_payload_size */) {
   const int kMinWidth = 320;
   const int kMinHeight = 180;
-  // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
-  // (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is
-  // always = 127. Note that in SW, QP is that of the user-level range [0, 63].
-  const int kMaxQP = 127;
   const int kLowQpThresholdDenominator = 3;
   if (codec_settings == NULL) {
     ALOGE("NULL VideoCodec instance");
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
   // Factory should guard against other codecs being used with us.
   RTC_CHECK(codec_settings->codecType == codecType_)
       << "Unsupported codec " << codec_settings->codecType << " for "
       << codecType_;
 
   ALOGD("InitEncode request");
-
   scale_ = webrtc::field_trial::FindFullName(
       "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled";
   ALOGD("Automatic resize: %s", scale_ ? "enabled" : "disabled");
-
-  if (scale_ && codecType_ == kVideoCodecVP8) {
-    quality_scaler_->Init(kMaxQP / kLowQpThresholdDenominator, true);
-    quality_scaler_->SetMinResolution(kMinWidth, kMinHeight);
-    quality_scaler_->ReportFramerate(codec_settings->maxFramerate);
-    updated_framerate_ = codec_settings->maxFramerate;
-  } else {
-    updated_framerate_ = -1;
+  if (scale_) {
+    if (codecType_ == kVideoCodecVP8) {
+      // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
+      // (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is
+      // always = 127. Note that in SW, QP is that of the user-level range [0,
+      // 63].
+      const int kMaxQp = 127;
+      quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator, true);
+    } else if (codecType_ == kVideoCodecH264) {
+      // H264 QP is in the range [0, 51].
+      const int kMaxQp = 51;
+      quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator, true);
+    } else {
+      // When adding codec support to additional hardware codecs, also configure
+      // their QP thresholds for scaling.
+      RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds.";
+    }
+    quality_scaler_.SetMinResolution(kMinWidth, kMinHeight);
+    quality_scaler_.ReportFramerate(codec_settings->maxFramerate);
   }
   return codec_thread_->Invoke<int32_t>(
       Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread,
            this,
            codec_settings->width,
            codec_settings->height,
            codec_settings->startBitrate,
            codec_settings->maxFramerate));
 }
 
@@ skipping 19 matching lines @@
       Bind(&MediaCodecVideoEncoder::ReleaseOnCodecThread, this));
 }
 
 int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */,
                                                      int64_t /* rtt */) {
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate,
                                          uint32_t frame_rate) {
-  if (scale_ && codecType_ == kVideoCodecVP8) {
-    quality_scaler_->ReportFramerate(frame_rate);
-  }
+  if (scale_)
+    quality_scaler_.ReportFramerate(frame_rate);
+
   return codec_thread_->Invoke<int32_t>(
       Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread,
            this,
            new_bit_rate,
            frame_rate));
 }
 
 void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) {
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
@@ skipping 135 matching lines @@
   }
 
   if (drop_next_input_frame_) {
     ALOGV("Encoder drop frame - failed callback.");
     drop_next_input_frame_ = false;
     return WEBRTC_VIDEO_CODEC_OK;
   }
 
   RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count";
   // Check framerate before spatial resolution change.
-  if (scale_ && codecType_ == kVideoCodecVP8) {
-    quality_scaler_->OnEncodeFrame(frame);
-    updated_framerate_ = quality_scaler_->GetTargetFramerate();
-  }
-  const VideoFrame& input_frame = (scale_ && codecType_ == kVideoCodecVP8) ?
-      quality_scaler_->GetScaledFrame(frame) : frame;
+  if (scale_)
+    quality_scaler_.OnEncodeFrame(frame);
+
+  const VideoFrame& input_frame =
+      scale_ ? quality_scaler_.GetScaledFrame(frame) : frame;
 
   if (input_frame.width() != width_ || input_frame.height() != height_) {
     ALOGD("Frame resolution change from %d x %d to %d x %d",
           width_, height_, input_frame.width(), input_frame.height());
     width_ = input_frame.width();
     height_ = input_frame.height();
     ResetCodec();
     return WEBRTC_VIDEO_CODEC_OK;
   }
 
@@ skipping 190 matching lines @@
         jni->GetDirectBufferAddress(j_output_buffer));
     CHECK_EXCEPTION(jni);
 
     ALOGV("Encoder frame out # %d. Key: %d. Size: %d. TS: %lld."
         " Latency: %lld. EncTime: %lld",
         frames_encoded_, key_frame, payload_size,
         last_output_timestamp_ms_,
         last_input_timestamp_ms_ - last_output_timestamp_ms_,
         frame_encoding_time_ms);
 
-    if (payload_size && scale_ && codecType_ == kVideoCodecVP8)
-      quality_scaler_->ReportQP(webrtc::vp8::GetQP(payload));
-
     // Calculate and print encoding statistics - every 3 seconds.
     frames_encoded_++;
     current_frames_++;
     current_bytes_ += payload_size;
     current_encoding_time_ms_ += frame_encoding_time_ms;
     int statistic_time_ms = GetCurrentTimeMs() - start_time_ms_;
     if (statistic_time_ms >= kMediaCodecStatisticsIntervalMs &&
         current_frames_ > 0) {
       ALOGD("Encoder bitrate: %d, target: %d kbps, fps: %d,"
           " encTime: %d for last %d ms",
@@ skipping 35 matching lines @@
 
       // Generate a header describing a single fragment.
       webrtc::RTPFragmentationHeader header;
       memset(&header, 0, sizeof(header));
       if (codecType_ == kVideoCodecVP8) {
         header.VerifyAndAllocateFragmentationHeader(1);
         header.fragmentationOffset[0] = 0;
         header.fragmentationLength[0] = image->_length;
         header.fragmentationPlType[0] = 0;
         header.fragmentationTimeDiff[0] = 0;
+        if (scale_)
+          quality_scaler_.ReportQP(webrtc::vp8::GetQP(payload));
       } else if (codecType_ == kVideoCodecH264) {
+        if (scale_) {
+          h264_bitstream_parser_.ParseBitstream(payload, payload_size);
+          int qp;
+          if (h264_bitstream_parser_.GetLastSliceQp(&qp))
+            quality_scaler_.ReportQP(qp);
+        }
         // For H.264 search for start codes.
         int32_t scPositions[MAX_NALUS_PERFRAME + 1] = {};
         int32_t scPositionsLength = 0;
         int32_t scPosition = 0;
         while (scPositionsLength < MAX_NALUS_PERFRAME) {
           int32_t naluPosition = NextNaluPosition(
               payload + scPosition, payload_size - scPosition);
           if (naluPosition < 0) {
             break;
           }
@@ skipping 69 matching lines @@
     if (head[3] != 0x01) { // got 000000xx
       head++; // xx != 1, continue searching.
       continue;
     }
     return (int32_t)(head - buffer);
   }
   return -1;
 }
 
 void MediaCodecVideoEncoder::OnDroppedFrame() {
-  if (scale_ && codecType_ == kVideoCodecVP8)
-    quality_scaler_->ReportDroppedFrame();
+  if (scale_)
+    quality_scaler_.ReportDroppedFrame();
 }
 
 int MediaCodecVideoEncoder::GetTargetFramerate() {
-  return updated_framerate_;
+  return scale_ ? quality_scaler_.GetTargetFramerate() : -1;
 }
 
 MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() {
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
   jclass j_encoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoEncoder");
   supported_codecs_.clear();
 
   bool is_vp8_hw_supported = jni->CallStaticBooleanMethod(
       j_encoder_class,
@@ skipping 40 matching lines @@
 }
 
 void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
     webrtc::VideoEncoder* encoder) {
   ALOGD("Destroy video encoder.");
   delete encoder;
 }
 
 }  // namespace webrtc_jni