Use high QP threshold for HW VP8 encoder frame downscaling.

talk/app/webrtc/java/jni/androidmediaencoder_jni.cc

 /*
  * libjingle
  * Copyright 2015 Google Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright notice,
@@ skipping 59 matching lines @@
 #define MAX_VIDEO_WIDTH 1280
 #define MAX_VIDEO_HEIGHT 1280
 // Maximum supported HW video encoder fps.
 #define MAX_VIDEO_FPS 30
 // Maximum allowed fps value in SetRates() call.
 #define MAX_ALLOWED_VIDEO_FPS 60
 // Maximum allowed frames in encoder input queue.
 #define MAX_ENCODER_Q_SIZE 2
 // Maximum allowed latency in ms.
 #define MAX_ENCODER_LATENCY_MS 70
-
+// Maximum amount of dropped frames caused by full encoder queue - exceeding
+// this threshold means that encoder probably got stuck and need to be reset.
+#define MAX_DROPPED_FRAMES 60

[pbos-webrtc, 2016/01/18 10:33:07]

ENCODER_STALL_FRAMEDROP_THRESHOLD maybe, something indicating that it's
detecting stalls.

[jackychen_, 2016/01/19 02:21:01]

In the comment, "reset the encoder when it drops more than 70 frames in a raw",
but 60 here? Maybe remove concrete number in the comment.

[AlexG, 2016/01/19 22:13:41]

Done.

[AlexG, 2016/01/19 22:13:41]

Done.

 
 // Logging macros.
 #define TAG_ENCODER "MediaCodecVideoEncoder"
 #ifdef TRACK_BUFFER_TIMING
 #define ALOGV(...)
   __android_log_print(ANDROID_LOG_VERBOSE, TAG_ENCODER, __VA_ARGS__)
 #else
 #define ALOGV(...)
 #endif
 #define ALOGD LOG_TAG(rtc::LS_INFO, TAG_ENCODER)
@@ skipping 123 matching lines @@
   bool inited_;
   bool use_surface_;
   uint16_t picture_id_;
   enum libyuv::FourCC encoder_fourcc_;  // Encoder color space format.
   int last_set_bitrate_kbps_;  // Last-requested bitrate in kbps.
   int last_set_fps_;  // Last-requested frame rate.
   int64_t current_timestamp_us_;  // Current frame timestamps in us.
   int frames_received_;  // Number of frames received by encoder.
   int frames_encoded_;  // Number of frames encoded by encoder.
   int frames_dropped_;  // Number of frames dropped by encoder.
+  int frames_dropped_q_full_;  // Number of dropped frames caused by full queue.

[pbos-webrtc, 2016/01/18 10:33:07]

consecutive_full_queue_frame_drops_

[AlexG, 2016/01/19 22:13:41]

Done.

   int frames_in_queue_;  // Number of frames in encoder queue.
   int64_t start_time_ms_;  // Start time for statistics.
   int current_frames_;  // Number of frames in the current statistics interval.
   int current_bytes_;  // Encoded bytes in the current statistics interval.
   int current_acc_qp_; // Accumulated QP in the current statistics interval.
   int current_encoding_time_ms_;  // Overall encoding time in the current second
   int64_t last_input_timestamp_ms_;  // Timestamp of last received yuv frame.
   int64_t last_output_timestamp_ms_;  // Timestamp of last encoded frame.
   std::vector<int32_t> timestamps_;  // Video frames timestamp queue.
   std::vector<int64_t> render_times_ms_;  // Video frames render time queue.
@@ skipping 129 matching lines @@
   scale_ = (codecType_ != kVideoCodecVP9) && (webrtc::field_trial::FindFullName(
         "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled");
   ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled");
   if (scale_) {
     if (codecType_ == kVideoCodecVP8) {
       // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
       // (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is
       // always = 127. Note that in SW, QP is that of the user-level range [0,
       // 63].
       const int kMaxQp = 127;
-      // TODO(pbos): Investigate whether high-QP thresholds make sense for VP8.
-      // This effectively disables high QP as VP8 QP can't go above this
-      // threshold.
-      const int kDisabledBadQpThreshold = kMaxQp + 1;
-      quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator,
-                           kDisabledBadQpThreshold, true);
+      const int kBadQpThreshold = kMaxQp  - kMaxQp / 4;

[pbos-webrtc, 2016/01/18 10:33:07]

I think this should just be hard-coded as another value, max_qp / 4 doesn't
really mean anything quantifyable, right?

[AlexG, 2016/01/19 22:13:41]

Done.

+      quality_scaler_.Init(
+          kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold, false);
     } else if (codecType_ == kVideoCodecH264) {
       // H264 QP is in the range [0, 51].
       const int kMaxQp = 51;
       const int kBadQpThreshold = 40;
-      quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold,
-                           false);
+      quality_scaler_.Init(
+          kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold, false);
     } else {
       // When adding codec support to additional hardware codecs, also configure
       // their QP thresholds for scaling.
       RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds.";
     }
     quality_scaler_.SetMinResolution(kMinWidth, kMinHeight);
     quality_scaler_.ReportFramerate(codec_settings->maxFramerate);
   }
   return codec_thread_->Invoke<int32_t>(
       Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread,
@@ skipping 27 matching lines @@
       Bind(&MediaCodecVideoEncoder::ReleaseOnCodecThread, this));
 }
 
 int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */,
                                                      int64_t /* rtt */) {
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate,
                                          uint32_t frame_rate) {
-  if (scale_)
-    quality_scaler_.ReportFramerate(frame_rate);
-
   return codec_thread_->Invoke<int32_t>(
       Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread,
            this,
            new_bit_rate,
            frame_rate));
 }
 
 void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
@@ skipping 46 matching lines @@
   }
 
   width_ = width;
   height_ = height;
   last_set_bitrate_kbps_ = kbps;
   last_set_fps_ = (fps < MAX_VIDEO_FPS) ? fps : MAX_VIDEO_FPS;
   yuv_size_ = width_ * height_ * 3 / 2;
   frames_received_ = 0;
   frames_encoded_ = 0;
   frames_dropped_ = 0;
+  frames_dropped_q_full_ = 0;
   frames_in_queue_ = 0;
   current_timestamp_us_ = 0;
   start_time_ms_ = GetCurrentTimeMs();
   current_frames_ = 0;
   current_bytes_ = 0;
   current_acc_qp_ = 0;
   current_encoding_time_ms_ = 0;
   last_input_timestamp_ms_ = -1;
   last_output_timestamp_ms_ = -1;
   output_timestamp_ = 0;
@@ skipping 96 matching lines @@
   RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count";
 
   // Check if we accumulated too many frames in encoder input buffers
   // or the encoder latency exceeds 70 ms and drop frame if so.
   if (frames_in_queue_ > 0 && last_input_timestamp_ms_ >= 0) {
     int encoder_latency_ms = last_input_timestamp_ms_ -
         last_output_timestamp_ms_;
     if (frames_in_queue_ > MAX_ENCODER_Q_SIZE ||
         encoder_latency_ms > MAX_ENCODER_LATENCY_MS) {
       ALOGD << "Drop frame - encoder is behind by " << encoder_latency_ms <<
-          " ms. Q size: " << frames_in_queue_;
+          " ms. Q size: " << frames_in_queue_ << ". Consecutive drops: " <<
+          frames_dropped_q_full_;
       current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
+      frames_dropped_q_full_++;
+      if (frames_dropped_q_full_ >= MAX_DROPPED_FRAMES) {
+        ALOGE << "Encoder got stuck. Reset.";
+        ResetCodecOnCodecThread();
+        return WEBRTC_VIDEO_CODEC_ERROR;
+      }
       OnDroppedFrame();
       return WEBRTC_VIDEO_CODEC_OK;
     }
   }
+  frames_dropped_q_full_ = 0;
 
   VideoFrame input_frame = frame;
   if (scale_) {
     // Check framerate before spatial resolution change.
     quality_scaler_.OnEncodeFrame(frame);
     const webrtc::QualityScaler::Resolution scaled_resolution =
         quality_scaler_.GetScaledResolution();
     if (scaled_resolution.width != frame.width() ||
         scaled_resolution.height != frame.height()) {
       if (frame.native_handle() != nullptr) {
@@ skipping 22 matching lines @@
   bool encode_status = true;
   if (!input_frame.native_handle()) {
     int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_,
         j_dequeue_input_buffer_method_);
     CHECK_EXCEPTION(jni);
     if (j_input_buffer_index == -1) {
       // Video codec falls behind - no input buffer available.
       ALOGW << "Encoder drop frame - no input buffers available";
       current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
       frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin());
+      OnDroppedFrame();
       return WEBRTC_VIDEO_CODEC_OK;  // TODO(fischman): see webrtc bug 2887.
     }
     if (j_input_buffer_index == -2) {
       ResetCodecOnCodecThread();
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
     encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame,
         j_input_buffer_index);
   } else {
     encode_status = EncodeTextureOnCodecThread(jni, key_frame, input_frame);
@@ skipping 133 matching lines @@
 
 int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate,
                                                       uint32_t frame_rate) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
   frame_rate = (frame_rate < MAX_ALLOWED_VIDEO_FPS) ?
       frame_rate : MAX_ALLOWED_VIDEO_FPS;
   if (last_set_bitrate_kbps_ == new_bit_rate &&
       last_set_fps_ == frame_rate) {
     return WEBRTC_VIDEO_CODEC_OK;
   }
+  if (scale_) {
+    quality_scaler_.ReportFramerate(frame_rate);
+  }
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
   if (new_bit_rate > 0) {
     last_set_bitrate_kbps_ = new_bit_rate;
   }
   if (frame_rate > 0) {
     last_set_fps_ = frame_rate;
   }
   bool ret = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
                                        j_set_rates_method_,
@@ skipping 378 matching lines @@
 }
 
 void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
     webrtc::VideoEncoder* encoder) {
   ALOGD << "Destroy video encoder.";
   delete encoder;
 }
 
 }  // namespace webrtc_jni