Use high QP threshold for HW VP8 encoder frame downscaling.

talk/app/webrtc/java/jni/androidmediaencoder_jni.cc

 /*
  * libjingle
  * Copyright 2015 Google Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright notice,
@@ skipping 59 matching lines @@
 #define MAX_VIDEO_WIDTH 1280
 #define MAX_VIDEO_HEIGHT 1280
 // Maximum supported HW video encoder fps.
 #define MAX_VIDEO_FPS 30
 // Maximum allowed fps value in SetRates() call.
 #define MAX_ALLOWED_VIDEO_FPS 60
 // Maximum allowed frames in encoder input queue.
 #define MAX_ENCODER_Q_SIZE 2
 // Maximum allowed latency in ms.
 #define MAX_ENCODER_LATENCY_MS 70
-
+// Maximum amount of dropped frames caused by full encoder queue - exceeding
+// this threshold means that encoder probably got stuck and need to be reset.
+#define ENCODER_STALL_FRAMEDROP_THRESHOLD 60
 
 // Logging macros.
 #define TAG_ENCODER "MediaCodecVideoEncoder"
 #ifdef TRACK_BUFFER_TIMING
 #define ALOGV(...)
   __android_log_print(ANDROID_LOG_VERBOSE, TAG_ENCODER, __VA_ARGS__)
 #else
 #define ALOGV(...)
 #endif
 #define ALOGD LOG_TAG(rtc::LS_INFO, TAG_ENCODER)
@@ skipping 123 matching lines @@
   bool inited_;
   bool use_surface_;
   uint16_t picture_id_;
   enum libyuv::FourCC encoder_fourcc_;  // Encoder color space format.
   int last_set_bitrate_kbps_;  // Last-requested bitrate in kbps.
   int last_set_fps_;  // Last-requested frame rate.
   int64_t current_timestamp_us_;  // Current frame timestamps in us.
   int frames_received_;  // Number of frames received by encoder.
   int frames_encoded_;  // Number of frames encoded by encoder.
   int frames_dropped_;  // Number of frames dropped by encoder.
+  // Number of dropped frames caused by full queue.
+  int consecutive_full_queue_frame_drops_;
   int frames_in_queue_;  // Number of frames in encoder queue.
   int64_t start_time_ms_;  // Start time for statistics.
   int current_frames_;  // Number of frames in the current statistics interval.
   int current_bytes_;  // Encoded bytes in the current statistics interval.
   int current_acc_qp_; // Accumulated QP in the current statistics interval.
   int current_encoding_time_ms_;  // Overall encoding time in the current second
   int64_t last_input_timestamp_ms_;  // Timestamp of last received yuv frame.
   int64_t last_output_timestamp_ms_;  // Timestamp of last encoded frame.
   std::vector<int32_t> timestamps_;  // Video frames timestamp queue.
   std::vector<int64_t> render_times_ms_;  // Video frames render time queue.
@@ skipping 129 matching lines @@
   scale_ = (codecType_ != kVideoCodecVP9) && (webrtc::field_trial::FindFullName(
         "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled");
   ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled");
   if (scale_) {
     if (codecType_ == kVideoCodecVP8) {
       // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
       // (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is
       // always = 127. Note that in SW, QP is that of the user-level range [0,
       // 63].
       const int kMaxQp = 127;
-      // TODO(pbos): Investigate whether high-QP thresholds make sense for VP8.
-      // This effectively disables high QP as VP8 QP can't go above this
-      // threshold.
-      const int kDisabledBadQpThreshold = kMaxQp + 1;
-      quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator,
-                           kDisabledBadQpThreshold, true);
+      const int kBadQpThreshold = 95;
+      quality_scaler_.Init(
+          kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold, false);

[jackychen_, 2016/01/26 15:03:57]

LGTM. 

You found framerate reduction bad for VP8 HW encoder or you want to do it after
scaledown to the minimum resolution as you mentioned in one post? Since I will
tune QP threshold also, do you want me to experiment and change in another CL?

[AlexG, 2016/01/26 20:34:18]

It is not bad, but I thought there is less difference if we scale from 720p 30
fps to VGA 30 fps than when reducing 720p 30 fps to 15 fps. Scaling down
resolution is not very visible for remote side, but frame rate reduction is
quite visible - at least for me. Yes, I think we can start frame rate reduction
only after we scale down to minimum possible resolution or may be at some
resolution close to minimum - may be QVGA?  This is something worth to
experiment.
QP thresholds are preliminary, they can definitely be changed in follow  up CL.

     } else if (codecType_ == kVideoCodecH264) {
       // H264 QP is in the range [0, 51].
       const int kMaxQp = 51;
       const int kBadQpThreshold = 40;
-      quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold,
-                           false);
+      quality_scaler_.Init(
+          kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold, false);
     } else {
       // When adding codec support to additional hardware codecs, also configure
       // their QP thresholds for scaling.
       RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds.";
     }
     quality_scaler_.SetMinResolution(kMinWidth, kMinHeight);
     quality_scaler_.ReportFramerate(codec_settings->maxFramerate);
   }
   return codec_thread_->Invoke<int32_t>(
       Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread,
@@ skipping 27 matching lines @@
       Bind(&MediaCodecVideoEncoder::ReleaseOnCodecThread, this));
 }
 
 int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */,
                                                      int64_t /* rtt */) {
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate,
                                          uint32_t frame_rate) {
-  if (scale_)
-    quality_scaler_.ReportFramerate(frame_rate);
-
   return codec_thread_->Invoke<int32_t>(
       Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread,
            this,
            new_bit_rate,
            frame_rate));
 }
 
 void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
@@ skipping 46 matching lines @@
   }
 
   width_ = width;
   height_ = height;
   last_set_bitrate_kbps_ = kbps;
   last_set_fps_ = (fps < MAX_VIDEO_FPS) ? fps : MAX_VIDEO_FPS;
   yuv_size_ = width_ * height_ * 3 / 2;
   frames_received_ = 0;
   frames_encoded_ = 0;
   frames_dropped_ = 0;
+  consecutive_full_queue_frame_drops_ = 0;
   frames_in_queue_ = 0;
   current_timestamp_us_ = 0;
   start_time_ms_ = GetCurrentTimeMs();
   current_frames_ = 0;
   current_bytes_ = 0;
   current_acc_qp_ = 0;
   current_encoding_time_ms_ = 0;
   last_input_timestamp_ms_ = -1;
   last_output_timestamp_ms_ = -1;
   output_timestamp_ = 0;
@@ skipping 82 matching lines @@
   if (frames_encoded_ < kMaxEncodedLogFrames) {
     ALOGD << "Encoder frame in # " << (frames_received_ - 1) << ". TS: " <<
         (int)(current_timestamp_us_ / 1000) << ". Q: " << frames_in_queue_ <<
         ". Fps: " << last_set_fps_ << ". Kbps: " << last_set_bitrate_kbps_;
   }
 
   if (drop_next_input_frame_) {
     ALOGW << "Encoder drop frame - failed callback.";
     drop_next_input_frame_ = false;
     current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
+    frames_dropped_++;

[pbos-webrtc, 2016/01/26 14:48:57]

move frames_dropped_ into OnDroppedFrame() ?

[AlexG, 2016/01/26 20:34:18]

OnDroppedFrame is an override - I think it may be called by some other module,
so frames_dropped_ will be incremented by other module. Originally I thought
frames_dropped_ statistics to count frame drops initiated by Android specific
implementation only - like input queue overflow or encoder falling behind.

[pbos-webrtc, 2016/01/27 15:55:49]

Should we rename frames_dropped_ to frames_dropped_before_media_codec_ or
something if it's not useful to know all dropped frames?

     OnDroppedFrame();
     return WEBRTC_VIDEO_CODEC_OK;
   }
 
   RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count";
 
   // Check if we accumulated too many frames in encoder input buffers
   // or the encoder latency exceeds 70 ms and drop frame if so.
   if (frames_in_queue_ > 0 && last_input_timestamp_ms_ >= 0) {
     int encoder_latency_ms = last_input_timestamp_ms_ -
         last_output_timestamp_ms_;
     if (frames_in_queue_ > MAX_ENCODER_Q_SIZE ||
         encoder_latency_ms > MAX_ENCODER_LATENCY_MS) {
       ALOGD << "Drop frame - encoder is behind by " << encoder_latency_ms <<
-          " ms. Q size: " << frames_in_queue_;
+          " ms. Q size: " << frames_in_queue_ << ". Consecutive drops: " <<
+          consecutive_full_queue_frame_drops_;
       current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
+      consecutive_full_queue_frame_drops_++;
+      if (consecutive_full_queue_frame_drops_ >=
+          ENCODER_STALL_FRAMEDROP_THRESHOLD) {
+        ALOGE << "Encoder got stuck. Reset.";
+        ResetCodecOnCodecThread();
+        return WEBRTC_VIDEO_CODEC_ERROR;
+      }
+      frames_dropped_++;
       OnDroppedFrame();
       return WEBRTC_VIDEO_CODEC_OK;
     }
   }
+  consecutive_full_queue_frame_drops_ = 0;
 
   VideoFrame input_frame = frame;
   if (scale_) {
     // Check framerate before spatial resolution change.
     quality_scaler_.OnEncodeFrame(frame);
     const webrtc::QualityScaler::Resolution scaled_resolution =
         quality_scaler_.GetScaledResolution();
     if (scaled_resolution.width != frame.width() ||
         scaled_resolution.height != frame.height()) {
       if (frame.native_handle() != nullptr) {
@@ skipping 20 matching lines @@
 
   const bool key_frame = frame_types->front() != webrtc::kVideoFrameDelta;
   bool encode_status = true;
   if (!input_frame.native_handle()) {
     int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_,
         j_dequeue_input_buffer_method_);
     CHECK_EXCEPTION(jni);
     if (j_input_buffer_index == -1) {
       // Video codec falls behind - no input buffer available.
       ALOGW << "Encoder drop frame - no input buffers available";
+      frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin());
       current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
-      frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin());
+      frames_dropped_++;
+      OnDroppedFrame();
       return WEBRTC_VIDEO_CODEC_OK;  // TODO(fischman): see webrtc bug 2887.
     }
     if (j_input_buffer_index == -2) {
       ResetCodecOnCodecThread();
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
     encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame,
         j_input_buffer_index);
   } else {
     encode_status = EncodeTextureOnCodecThread(jni, key_frame, input_frame);
@@ skipping 133 matching lines @@
 
 int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate,
                                                       uint32_t frame_rate) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
   frame_rate = (frame_rate < MAX_ALLOWED_VIDEO_FPS) ?
       frame_rate : MAX_ALLOWED_VIDEO_FPS;
   if (last_set_bitrate_kbps_ == new_bit_rate &&
       last_set_fps_ == frame_rate) {
     return WEBRTC_VIDEO_CODEC_OK;
   }
+  if (scale_) {
+    quality_scaler_.ReportFramerate(frame_rate);
+  }
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
   if (new_bit_rate > 0) {
     last_set_bitrate_kbps_ = new_bit_rate;
   }
   if (frame_rate > 0) {
     last_set_fps_ = frame_rate;
   }
   bool ret = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
                                        j_set_rates_method_,
@@ skipping 265 matching lines @@
     if (head[3] != 0x01) { // got 000000xx
       head++; // xx != 1, continue searching.
       continue;
     }
     return (int32_t)(head - buffer);
   }
   return -1;
 }
 
 void MediaCodecVideoEncoder::OnDroppedFrame() {
-  frames_dropped_++;
   // Report dropped frame to quality_scaler_.
   if (scale_)
     quality_scaler_.ReportDroppedFrame();
 }
 
 int MediaCodecVideoEncoder::GetTargetFramerate() {
   return scale_ ? quality_scaler_.GetTargetFramerate() : -1;
 }
 
 const char* MediaCodecVideoEncoder::ImplementationName() const {
@@ skipping 92 matching lines @@
 }
 
 void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
     webrtc::VideoEncoder* encoder) {
   ALOGD << "Destroy video encoder.";
   delete encoder;
 }
 
 }  // namespace webrtc_jni