Move usage of QualityScaler to ViEEncoder.

webrtc/api/android/jni/androidmediaencoder_jni.cc

 /*
  *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 13 matching lines @@
 #include "webrtc/api/android/jni/native_handle_impl.h"
 #include "webrtc/base/bind.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/thread.h"
 #include "webrtc/base/thread_checker.h"
 #include "webrtc/base/timeutils.h"
 #include "webrtc/common_types.h"
 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
 #include "webrtc/modules/video_coding/utility/h264_bitstream_parser.h"
+#include "webrtc/modules/video_coding/utility/vp8_header_parser.h"

[magjed_webrtc, 2016/10/26 14:28:36]

nit: revert this change. It was right before.

[kthelgason, 2016/10/26 19:02:49]

Alphabet is hard :(

 #include "webrtc/modules/video_coding/utility/quality_scaler.h"
-#include "webrtc/modules/video_coding/utility/vp8_header_parser.h"
 #include "webrtc/system_wrappers/include/field_trial.h"
 #include "webrtc/system_wrappers/include/logcat_trace_context.h"
 
 using rtc::Bind;
 using rtc::Thread;
 using rtc::ThreadManager;
 
 using webrtc::CodecSpecificInfo;
 using webrtc::EncodedImage;
 using webrtc::VideoFrame;
@@ skipping 103 matching lines @@
       const int64_t frame_input_time_ms);
   bool EncodeByteBufferOnCodecThread(JNIEnv* jni,
       bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index);
   bool EncodeTextureOnCodecThread(JNIEnv* jni,
       bool key_frame, const webrtc::VideoFrame& frame);
 
   int32_t RegisterEncodeCompleteCallbackOnCodecThread(
       webrtc::EncodedImageCallback* callback);
   int32_t ReleaseOnCodecThread();
   int32_t SetRatesOnCodecThread(uint32_t new_bit_rate, uint32_t frame_rate);
-  void OnDroppedFrameOnCodecThread();
 
   // Helper accessors for MediaCodecVideoEncoder$OutputBufferInfo members.
   int GetOutputBufferInfoIndex(JNIEnv* jni, jobject j_output_buffer_info);
   jobject GetOutputBufferInfoBuffer(JNIEnv* jni, jobject j_output_buffer_info);
   bool GetOutputBufferInfoIsKeyFrame(JNIEnv* jni, jobject j_output_buffer_info);
   jlong GetOutputBufferInfoPresentationTimestampUs(
       JNIEnv* jni, jobject j_output_buffer_info);
 
   // Deliver any outputs pending in the MediaCodec to our |callback_| and return
   // true on success.
   bool DeliverPendingOutputs(JNIEnv* jni);
 
   // Search for H.264 start codes.
   int32_t NextNaluPosition(uint8_t *buffer, size_t buffer_size);
 
+  QualityScaler::Settings GetQPThresholds() const override;
+
   // Displays encoder statistics.
   void LogStatistics(bool force_log);
 
   // Type of video codec.
   VideoCodecType codecType_;
 
   // Valid all the time since RegisterEncodeCompleteCallback() Invoke()s to
   // |codec_thread_| synchronously.
   webrtc::EncodedImageCallback* callback_;
 
@@ skipping 65 matching lines @@
                                   // |input_frame_infos_|.
   int64_t output_render_time_ms_; // Last output frame render time from
                                   // |input_frame_infos_|.
   webrtc::VideoRotation output_rotation_;  // Last output frame rotation from
                                            // |input_frame_infos_|.
   // Frame size in bytes fed to MediaCodec.
   int yuv_size_;
   // True only when between a callback_->Encoded() call return a positive value
   // and the next Encode() call being ignored.
   bool drop_next_input_frame_;
+  bool scale_;
   // Global references; must be deleted in Release().
   std::vector<jobject> input_buffers_;
-  QualityScaler quality_scaler_;
-  // Dynamic resolution change, off by default.
-  bool scale_;
-
-  // H264 bitstream parser, used to extract QP from encoded bitstreams.
   webrtc::H264BitstreamParser h264_bitstream_parser_;
 
   // VP9 variables to populate codec specific structure.
   webrtc::GofInfoVP9 gof_; // Contains each frame's temporal information for
                            // non-flexible VP9 mode.
   uint8_t tl0_pic_idx_;
   size_t gof_idx_;
 
   // EGL context - owned by factory, should not be allocated/destroyed
   // by MediaCodecVideoEncoder.
@@ skipping 96 matching lines @@
   if (codec_settings == NULL) {
     ALOGE << "NULL VideoCodec instance";
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
   // Factory should guard against other codecs being used with us.
   RTC_CHECK(codec_settings->codecType == codecType_)
       << "Unsupported codec " << codec_settings->codecType << " for "
       << codecType_;
 
   codec_mode_ = codec_settings->mode;
+  scale_ =
+      codecType_ == (kVideoCodecVP8 &&

[magjed_webrtc, 2016/10/26 14:28:36]

The parentheses are misplaced so it's doing something crazy right now. It's also
conecptually wrong because '|| codecType_ != kVideoCodecVP9' will always be true
for VP8. I would recommend keeping exactly the same code for now and not touch
this logic.

[kthelgason, 2016/10/26 19:02:49]

Oops. good catch, thanks. I've replaced it to the way it was.

+                     codec_settings->codecSpecific.VP8.automaticResizeOn) ||
+      codecType_ != kVideoCodecVP9;
   int init_width = codec_settings->width;
   int init_height = codec_settings->height;
-  // Scaling is disabled for VP9, but optionally enabled for VP8.
-  // TODO(pbos): Extract automaticResizeOn out of VP8 settings.
-  scale_ = false;
-  if (codecType_ == kVideoCodecVP8) {
-    scale_ = codec_settings->codecSpecific.VP8.automaticResizeOn;
-  } else if (codecType_ != kVideoCodecVP9) {
-    scale_ = true;
-  }
-
   ALOGD << "InitEncode request: " << init_width << " x " << init_height;
-  ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled");
-
-  if (scale_) {
-    if (codecType_ == kVideoCodecVP8 || codecType_ == kVideoCodecH264) {
-      quality_scaler_.Init(codecType_, codec_settings->startBitrate,
-                           codec_settings->width, codec_settings->height,
-                           codec_settings->maxFramerate);
-    } else {
-      // When adding codec support to additional hardware codecs, also configure
-      // their QP thresholds for scaling.
-      RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds.";
-      scale_ = false;
-    }
-    QualityScaler::Resolution res = quality_scaler_.GetScaledResolution();
-    init_width = res.width;
-    init_height = res.height;
-    ALOGD << "Scaled resolution: " << init_width << " x " << init_height;
-  }
 
   return codec_thread_->Invoke<int32_t>(
       RTC_FROM_HERE,
       Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread, this, init_width,
            init_height, codec_settings->startBitrate,
            codec_settings->maxFramerate,
            codec_settings->expect_encode_from_texture));
 }
 
 int32_t MediaCodecVideoEncoder::Encode(
@@ skipping 214 matching lines @@
           << ". TS: " << (int)(current_timestamp_us_ / 1000)
           << ". Q: " << input_frame_infos_.size() << ". Fps: " << last_set_fps_
           << ". Kbps: " << last_set_bitrate_kbps_;
   }
 
   if (drop_next_input_frame_) {
     ALOGW << "Encoder drop frame - failed callback.";
     drop_next_input_frame_ = false;
     current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
     frames_dropped_media_encoder_++;
-    OnDroppedFrameOnCodecThread();
     return WEBRTC_VIDEO_CODEC_OK;
   }
 
   RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count";
 
   // Check if we accumulated too many frames in encoder input buffers and drop
   // frame if so.
   if (input_frame_infos_.size() > MAX_ENCODER_Q_SIZE) {
     ALOGD << "Already " << input_frame_infos_.size()
           << " frames in the queue, dropping"
           << ". TS: " << (int)(current_timestamp_us_ / 1000)
           << ". Fps: " << last_set_fps_
           << ". Consecutive drops: " << consecutive_full_queue_frame_drops_;
     current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
     consecutive_full_queue_frame_drops_++;
     if (consecutive_full_queue_frame_drops_ >=
         ENCODER_STALL_FRAMEDROP_THRESHOLD) {
       ALOGE << "Encoder got stuck. Reset.";
       ResetCodecOnCodecThread();
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
     frames_dropped_media_encoder_++;
-    OnDroppedFrameOnCodecThread();
     return WEBRTC_VIDEO_CODEC_OK;
   }
   consecutive_full_queue_frame_drops_ = 0;
 
   rtc::scoped_refptr<webrtc::VideoFrameBuffer> input_buffer(
       frame.video_frame_buffer());
-  if (scale_) {
-    // Check framerate before spatial resolution change.
-    quality_scaler_.OnEncodeFrame(frame.width(), frame.height());
-    const webrtc::QualityScaler::Resolution scaled_resolution =
-        quality_scaler_.GetScaledResolution();
-    if (scaled_resolution.width != frame.width() ||
-        scaled_resolution.height != frame.height()) {
-      if (input_buffer->native_handle() != nullptr) {
-        input_buffer = static_cast<AndroidTextureBuffer*>(input_buffer.get())
-                           ->CropScaleAndRotate(frame.width(), frame.height(),
-                                                0, 0,
-                                                scaled_resolution.width,
-                                                scaled_resolution.height,
-                                                webrtc::kVideoRotation_0);
-      } else {
-        input_buffer = quality_scaler_.GetScaledBuffer(input_buffer);
-      }
-    }
-  }
 
   VideoFrame input_frame(input_buffer, frame.timestamp(),
                          frame.render_time_ms(), frame.rotation());
 
   if (!MaybeReconfigureEncoderOnCodecThread(input_frame)) {
     ALOGE << "Failed to reconfigure encoder.";
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
 
   const bool key_frame =
       frame_types->front() != webrtc::kVideoFrameDelta || send_key_frame;
   bool encode_status = true;
   if (!input_frame.video_frame_buffer()->native_handle()) {
     int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_,
         j_dequeue_input_buffer_method_);
     CHECK_EXCEPTION(jni);
     if (j_input_buffer_index == -1) {
       // Video codec falls behind - no input buffer available.
       ALOGW << "Encoder drop frame - no input buffers available";
       if (frames_received_ > 1) {
         current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
         frames_dropped_media_encoder_++;
-        OnDroppedFrameOnCodecThread();
       } else {
         // Input buffers are not ready after codec initialization, HW is still
         // allocating thme - this is expected and should not result in drop
         // frame report.
         frames_received_ = 0;
       }
       return WEBRTC_VIDEO_CODEC_OK;  // TODO(fischman): see webrtc bug 2887.
     } else if (j_input_buffer_index == -2) {
       ResetCodecOnCodecThread();
       return WEBRTC_VIDEO_CODEC_ERROR;
@@ skipping 146 matching lines @@
 
 int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate,
                                                       uint32_t frame_rate) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
   frame_rate = (frame_rate < MAX_ALLOWED_VIDEO_FPS) ?
       frame_rate : MAX_ALLOWED_VIDEO_FPS;
   if (last_set_bitrate_kbps_ == new_bit_rate &&
       last_set_fps_ == frame_rate) {
     return WEBRTC_VIDEO_CODEC_OK;
   }
-  if (scale_) {
-    quality_scaler_.ReportFramerate(frame_rate);
-  }
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
   if (new_bit_rate > 0) {
     last_set_bitrate_kbps_ = new_bit_rate;
   }
   if (frame_rate > 0) {
     last_set_fps_ = frame_rate;
   }
   bool ret = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
                                        j_set_rates_method_,
@@ skipping 82 matching lines @@
       std::unique_ptr<webrtc::EncodedImage> image(
           new webrtc::EncodedImage(payload, payload_size, payload_size));
       image->_encodedWidth = width_;
       image->_encodedHeight = height_;
       image->_timeStamp = output_timestamp_;
       image->capture_time_ms_ = output_render_time_ms_;
       image->rotation_ = output_rotation_;
       image->_frameType =
           (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta);
       image->_completeFrame = true;
-      image->adapt_reason_.quality_resolution_downscales =
-          scale_ ? quality_scaler_.downscale_shift() : -1;
-
       webrtc::CodecSpecificInfo info;
       memset(&info, 0, sizeof(info));
       info.codecType = codecType_;
       if (codecType_ == kVideoCodecVP8) {
         info.codecSpecific.VP8.pictureId = picture_id_;
         info.codecSpecific.VP8.nonReference = false;
         info.codecSpecific.VP8.simulcastIdx = 0;
         info.codecSpecific.VP8.temporalIdx = webrtc::kNoTemporalIdx;
         info.codecSpecific.VP8.layerSync = false;
         info.codecSpecific.VP8.tl0PicIdx = webrtc::kNoTl0PicIdx;
@@ skipping 26 matching lines @@
 
       // Generate a header describing a single fragment.
       webrtc::RTPFragmentationHeader header;
       memset(&header, 0, sizeof(header));
       if (codecType_ == kVideoCodecVP8 || codecType_ == kVideoCodecVP9) {
         header.VerifyAndAllocateFragmentationHeader(1);
         header.fragmentationOffset[0] = 0;
         header.fragmentationLength[0] = image->_length;
         header.fragmentationPlType[0] = 0;
         header.fragmentationTimeDiff[0] = 0;
-        if (codecType_ == kVideoCodecVP8 && scale_) {
+        if (codecType_ == kVideoCodecVP8) {
           int qp;
           if (webrtc::vp8::GetQp(payload, payload_size, &qp)) {
             current_acc_qp_ += qp;
-            quality_scaler_.ReportQP(qp);
             image->qp_ = qp;
           }
         }
       } else if (codecType_ == kVideoCodecH264) {
-        if (scale_) {
-          h264_bitstream_parser_.ParseBitstream(payload, payload_size);
-          int qp;
-          if (h264_bitstream_parser_.GetLastSliceQp(&qp)) {
-            current_acc_qp_ += qp;
-            quality_scaler_.ReportQP(qp);
-          }
+        h264_bitstream_parser_.ParseBitstream(payload, payload_size);
+        int qp;
+        if (h264_bitstream_parser_.GetLastSliceQp(&qp)) {
+          current_acc_qp_ += qp;
+          image->qp_ = qp;
         }
         // For H.264 search for start codes.
         int32_t scPositions[MAX_NALUS_PERFRAME + 1] = {};
         int32_t scPositionsLength = 0;
         int32_t scPosition = 0;
         while (scPositionsLength < MAX_NALUS_PERFRAME) {
           int32_t naluPosition = NextNaluPosition(
               payload + scPosition, payload_size - scPosition);
           if (naluPosition < 0) {
             break;
@@ skipping 83 matching lines @@
         ". QP: " << (current_acc_qp_ / current_frames_divider) <<
         " for last " << statistic_time_ms << " ms.";
     stat_start_time_ms_ = rtc::TimeMillis();
     current_frames_ = 0;
     current_bytes_ = 0;
     current_acc_qp_ = 0;
     current_encoding_time_ms_ = 0;
   }
 }
 
+QualityScaler::Settings MediaCodecVideoEncoder::GetQPThresholds() const {
+  return QualityScaler::Settings(scale_);
+}
+
 int32_t MediaCodecVideoEncoder::NextNaluPosition(
     uint8_t *buffer, size_t buffer_size) {
   if (buffer_size < H264_SC_LENGTH) {
     return -1;
   }
   uint8_t *head = buffer;
   // Set end buffer pointer to 4 bytes before actual buffer end so we can
   // access head[1], head[2] and head[3] in a loop without buffer overrun.
   uint8_t *end = buffer + buffer_size - H264_SC_LENGTH;
 
@@ skipping 13 matching lines @@
     if (head[3] != 0x01) { // got 000000xx
       head++; // xx != 1, continue searching.
       continue;
     }
     return (int32_t)(head - buffer);
   }
   return -1;
 }
 
 void MediaCodecVideoEncoder::OnDroppedFrame() {
-  // Methods running on the codec thread should call OnDroppedFrameOnCodecThread
-  // directly.
   RTC_DCHECK(!codec_thread_checker_.CalledOnValidThread());
-  codec_thread_->Invoke<void>(
-      RTC_FROM_HERE,
-      Bind(&MediaCodecVideoEncoder::OnDroppedFrameOnCodecThread, this));
-}
-
-void MediaCodecVideoEncoder::OnDroppedFrameOnCodecThread() {
-  RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
-  // Report dropped frame to quality_scaler_.
-  if (scale_)
-    quality_scaler_.ReportDroppedFrame();
 }
 
 const char* MediaCodecVideoEncoder::ImplementationName() const {
   return "MediaCodec";
 }
 
 MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory()
     : egl_context_(nullptr) {
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
@@ skipping 76 matching lines @@
   return supported_codecs_;
 }
 
 void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
     webrtc::VideoEncoder* encoder) {
   ALOGD << "Destroy video encoder.";
   delete encoder;
 }
 
 }  // namespace webrtc_jni