Use high QP threshold for HW VP8 encoder frame downscaling.

talk/app/webrtc/java/jni/androidmediaencoder_jni.cc

 /*
  * libjingle
  * Copyright 2015 Google Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  *
  *  1. Redistributions of source code must retain the above copyright notice,
  *     this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright notice,
@@ skipping 60 matching lines @@
 #define MAX_VIDEO_WIDTH 1280
 #define MAX_VIDEO_HEIGHT 1280
 // Maximum supported HW video encoder fps.
 #define MAX_VIDEO_FPS 30
 // Maximum allowed fps value in SetRates() call.
 #define MAX_ALLOWED_VIDEO_FPS 60
 // Maximum allowed frames in encoder input queue.
 #define MAX_ENCODER_Q_SIZE 2
 // Maximum allowed latency in ms.
 #define MAX_ENCODER_LATENCY_MS 70
-
+// Maximum amount of dropped frames caused by full encoder queue - exceeding
+// this threshold means that encoder probably got stuck and need to be reset.
+#define ENCODER_STALL_FRAMEDROP_THRESHOLD 60
 
 // Logging macros.
 #define TAG_ENCODER "MediaCodecVideoEncoder"
 #ifdef TRACK_BUFFER_TIMING
 #define ALOGV(...)
   __android_log_print(ANDROID_LOG_VERBOSE, TAG_ENCODER, __VA_ARGS__)
 #else
 #define ALOGV(...)
 #endif
 #define ALOGD LOG_TAG(rtc::LS_INFO, TAG_ENCODER)
@@ skipping 128 matching lines @@
   int height_;  // Frame height in pixels.
   bool inited_;
   bool use_surface_;
   uint16_t picture_id_;
   enum libyuv::FourCC encoder_fourcc_;  // Encoder color space format.
   int last_set_bitrate_kbps_;  // Last-requested bitrate in kbps.
   int last_set_fps_;  // Last-requested frame rate.
   int64_t current_timestamp_us_;  // Current frame timestamps in us.
   int frames_received_;  // Number of frames received by encoder.
   int frames_encoded_;  // Number of frames encoded by encoder.
-  int frames_dropped_;  // Number of frames dropped by encoder.
+  int frames_dropped_media_encoder_;  // Number of frames dropped by encoder.
+  // Number of dropped frames caused by full queue.
+  int consecutive_full_queue_frame_drops_;
   int frames_in_queue_;  // Number of frames in encoder queue.
   int64_t start_time_ms_;  // Start time for statistics.
   int current_frames_;  // Number of frames in the current statistics interval.
   int current_bytes_;  // Encoded bytes in the current statistics interval.
   int current_acc_qp_; // Accumulated QP in the current statistics interval.
   int current_encoding_time_ms_;  // Overall encoding time in the current second
   int64_t last_input_timestamp_ms_;  // Timestamp of last received yuv frame.
   int64_t last_output_timestamp_ms_;  // Timestamp of last encoded frame.
   std::vector<int32_t> timestamps_;  // Video frames timestamp queue.
   std::vector<int64_t> render_times_ms_;  // Video frames render time queue.
@@ skipping 137 matching lines @@
   scale_ = (codecType_ != kVideoCodecVP9) && (webrtc::field_trial::FindFullName(
         "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled");
   ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled");
   if (scale_) {
     if (codecType_ == kVideoCodecVP8) {
       // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
       // (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is
       // always = 127. Note that in SW, QP is that of the user-level range [0,
       // 63].
       const int kMaxQp = 127;
-      // TODO(pbos): Investigate whether high-QP thresholds make sense for VP8.
-      // This effectively disables high QP as VP8 QP can't go above this
-      // threshold.
-      const int kDisabledBadQpThreshold = kMaxQp + 1;
-      quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator,
-                           kDisabledBadQpThreshold, true);
+      const int kBadQpThreshold = 95;
+      quality_scaler_.Init(
+          kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold, false);
     } else if (codecType_ == kVideoCodecH264) {
       // H264 QP is in the range [0, 51].
       const int kMaxQp = 51;
       const int kBadQpThreshold = 40;
-      quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold,
-                           false);
+      quality_scaler_.Init(
+          kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold, false);
     } else {
       // When adding codec support to additional hardware codecs, also configure
       // their QP thresholds for scaling.
       RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds.";
     }
     quality_scaler_.SetMinResolution(kMinWidth, kMinHeight);
     quality_scaler_.ReportFramerate(codec_settings->maxFramerate);
   }
   return codec_thread_->Invoke<int32_t>(
       Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread,
@@ skipping 27 matching lines @@
       Bind(&MediaCodecVideoEncoder::ReleaseOnCodecThread, this));
 }
 
 int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */,
                                                      int64_t /* rtt */) {
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate,
                                          uint32_t frame_rate) {
-  if (scale_)
-    quality_scaler_.ReportFramerate(frame_rate);
-
   return codec_thread_->Invoke<int32_t>(
       Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread,
            this,
            new_bit_rate,
            frame_rate));
 }
 
 void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
@@ skipping 45 matching lines @@
     fps = MAX_VIDEO_FPS;
   }
 
   width_ = width;
   height_ = height;
   last_set_bitrate_kbps_ = kbps;
   last_set_fps_ = (fps < MAX_VIDEO_FPS) ? fps : MAX_VIDEO_FPS;
   yuv_size_ = width_ * height_ * 3 / 2;
   frames_received_ = 0;
   frames_encoded_ = 0;
-  frames_dropped_ = 0;
+  frames_dropped_media_encoder_ = 0;
+  consecutive_full_queue_frame_drops_ = 0;
   frames_in_queue_ = 0;
   current_timestamp_us_ = 0;
   start_time_ms_ = GetCurrentTimeMs();
   current_frames_ = 0;
   current_bytes_ = 0;
   current_acc_qp_ = 0;
   current_encoding_time_ms_ = 0;
   last_input_timestamp_ms_ = -1;
   last_output_timestamp_ms_ = -1;
   output_timestamp_ = 0;
@@ skipping 101 matching lines @@
   if (frames_encoded_ < kMaxEncodedLogFrames) {
     ALOGD << "Encoder frame in # " << (frames_received_ - 1) << ". TS: " <<
         (int)(current_timestamp_us_ / 1000) << ". Q: " << frames_in_queue_ <<
         ". Fps: " << last_set_fps_ << ". Kbps: " << last_set_bitrate_kbps_;
   }
 
   if (drop_next_input_frame_) {
     ALOGW << "Encoder drop frame - failed callback.";
     drop_next_input_frame_ = false;
     current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
+    frames_dropped_media_encoder_++;
     OnDroppedFrame();
     return WEBRTC_VIDEO_CODEC_OK;
   }
 
   RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count";
 
   // Check if we accumulated too many frames in encoder input buffers
   // or the encoder latency exceeds 70 ms and drop frame if so.
   if (frames_in_queue_ > 0 && last_input_timestamp_ms_ >= 0) {
     int encoder_latency_ms = last_input_timestamp_ms_ -
         last_output_timestamp_ms_;
     if (frames_in_queue_ > MAX_ENCODER_Q_SIZE ||
         encoder_latency_ms > MAX_ENCODER_LATENCY_MS) {
       ALOGD << "Drop frame - encoder is behind by " << encoder_latency_ms <<
-          " ms. Q size: " << frames_in_queue_;
+          " ms. Q size: " << frames_in_queue_ << ". Consecutive drops: " <<
+          consecutive_full_queue_frame_drops_;
       current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
+      consecutive_full_queue_frame_drops_++;
+      if (consecutive_full_queue_frame_drops_ >=
+          ENCODER_STALL_FRAMEDROP_THRESHOLD) {
+        ALOGE << "Encoder got stuck. Reset.";
+        ResetCodecOnCodecThread();
+        return WEBRTC_VIDEO_CODEC_ERROR;
+      }
+      frames_dropped_media_encoder_++;
       OnDroppedFrame();
       return WEBRTC_VIDEO_CODEC_OK;
     }
   }
+  consecutive_full_queue_frame_drops_ = 0;
 
   VideoFrame input_frame = frame;
   if (scale_) {
     // Check framerate before spatial resolution change.
     quality_scaler_.OnEncodeFrame(frame);
     const webrtc::QualityScaler::Resolution scaled_resolution =
         quality_scaler_.GetScaledResolution();
     if (scaled_resolution.width != frame.width() ||
         scaled_resolution.height != frame.height()) {
       if (frame.native_handle() != nullptr) {
@@ skipping 21 matching lines @@
   const bool key_frame =
       frame_types->front() != webrtc::kVideoFrameDelta || send_key_frame;
   bool encode_status = true;
   if (!input_frame.native_handle()) {
     int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_,
         j_dequeue_input_buffer_method_);
     CHECK_EXCEPTION(jni);
     if (j_input_buffer_index == -1) {
       // Video codec falls behind - no input buffer available.
       ALOGW << "Encoder drop frame - no input buffers available";
+      frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin());
       current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
-      frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin());
+      frames_dropped_media_encoder_++;
+      OnDroppedFrame();
       return WEBRTC_VIDEO_CODEC_OK;  // TODO(fischman): see webrtc bug 2887.
     }
     if (j_input_buffer_index == -2) {
       ResetCodecOnCodecThread();
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
     encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame,
         j_input_buffer_index);
   } else {
     encode_status = EncodeTextureOnCodecThread(jni, key_frame, input_frame);
@@ skipping 110 matching lines @@
 }
 
 int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
   if (!inited_) {
     return WEBRTC_VIDEO_CODEC_OK;
   }
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ALOGD << "EncoderReleaseOnCodecThread: Frames received: " <<
       frames_received_ << ". Encoded: " << frames_encoded_ <<
-      ". Dropped: " << frames_dropped_;
+      ". Dropped: " << frames_dropped_media_encoder_;
   ScopedLocalRefFrame local_ref_frame(jni);
   for (size_t i = 0; i < input_buffers_.size(); ++i)
     jni->DeleteGlobalRef(input_buffers_[i]);
   input_buffers_.clear();
   jni->CallVoidMethod(*j_media_codec_video_encoder_, j_release_method_);
   CHECK_EXCEPTION(jni);
   rtc::MessageQueueManager::Clear(this);
   inited_ = false;
   use_surface_ = false;
   ALOGD << "EncoderReleaseOnCodecThread done.";
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate,
                                                       uint32_t frame_rate) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
   frame_rate = (frame_rate < MAX_ALLOWED_VIDEO_FPS) ?
       frame_rate : MAX_ALLOWED_VIDEO_FPS;
   if (last_set_bitrate_kbps_ == new_bit_rate &&
       last_set_fps_ == frame_rate) {
     return WEBRTC_VIDEO_CODEC_OK;
   }
+  if (scale_) {
+    quality_scaler_.ReportFramerate(frame_rate);
+  }
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
   if (new_bit_rate > 0) {
     last_set_bitrate_kbps_ = new_bit_rate;
   }
   if (frame_rate > 0) {
     last_set_fps_ = frame_rate;
   }
   bool ret = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
                                        j_set_rates_method_,
@@ skipping 265 matching lines @@
     if (head[3] != 0x01) { // got 000000xx
       head++; // xx != 1, continue searching.
       continue;
     }
     return (int32_t)(head - buffer);
   }
   return -1;
 }
 
 void MediaCodecVideoEncoder::OnDroppedFrame() {
-  frames_dropped_++;
   // Report dropped frame to quality_scaler_.
   if (scale_)
     quality_scaler_.ReportDroppedFrame();
 }
 
 int MediaCodecVideoEncoder::GetTargetFramerate() {
   return scale_ ? quality_scaler_.GetTargetFramerate() : -1;
 }
 
 const char* MediaCodecVideoEncoder::ImplementationName() const {
@@ skipping 92 matching lines @@
 }
 
 void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
     webrtc::VideoEncoder* encoder) {
   ALOGD << "Destroy video encoder.";
   delete encoder;
 }
 
 }  // namespace webrtc_jni