Improve encoding time calculation for Android HW encoder.

webrtc/api/java/jni/androidmediaencoder_jni.cc

 /*
  *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 127 matching lines @@
   // previously-current values are reused instead of the passed parameters
   // (makes it easier to reason about thread-safety).
   int32_t InitEncodeOnCodecThread(int width, int height, int kbps, int fps,
       bool use_surface);
   // Reconfigure to match |frame| in width, height. Also reconfigures the
   // encoder if |frame| is a texture/byte buffer and the encoder is initialized
   // for byte buffer/texture. Returns false if reconfiguring fails.
   bool MaybeReconfigureEncoderOnCodecThread(const webrtc::VideoFrame& frame);
   int32_t EncodeOnCodecThread(
       const webrtc::VideoFrame& input_image,
-      const std::vector<webrtc::FrameType>* frame_types);
+      const std::vector<webrtc::FrameType>* frame_types,
+      const int64_t frame_input_time_ms);
   bool EncodeByteBufferOnCodecThread(JNIEnv* jni,
       bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index);
   bool EncodeTextureOnCodecThread(JNIEnv* jni,
       bool key_frame, const webrtc::VideoFrame& frame);
 
   int32_t RegisterEncodeCompleteCallbackOnCodecThread(
       webrtc::EncodedImageCallback* callback);
   int32_t ReleaseOnCodecThread();
   int32_t SetRatesOnCodecThread(uint32_t new_bit_rate, uint32_t frame_rate);
   void OnDroppedFrameOnCodecThread();
@@ skipping 257 matching lines @@
            init_height, codec_settings->startBitrate,
            codec_settings->maxFramerate, false /* use_surface */));
 }
 
 int32_t MediaCodecVideoEncoder::Encode(
     const webrtc::VideoFrame& frame,
     const webrtc::CodecSpecificInfo* /* codec_specific_info */,
     const std::vector<webrtc::FrameType>* frame_types) {
   return codec_thread_->Invoke<int32_t>(
       RTC_FROM_HERE, Bind(&MediaCodecVideoEncoder::EncodeOnCodecThread, this,
-                          frame, frame_types));
+                          frame, frame_types, rtc::TimeMillis()));
 }
 
 int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallback(
     webrtc::EncodedImageCallback* callback) {
   return codec_thread_->Invoke<int32_t>(
       RTC_FROM_HERE,
       Bind(&MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread,
            this, callback));
 }
 
@@ skipping 154 matching lines @@
       RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity";
     }
   }
 
   inited_ = true;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
     const webrtc::VideoFrame& frame,
-    const std::vector<webrtc::FrameType>* frame_types) {
+    const std::vector<webrtc::FrameType>* frame_types,
+    const int64_t frame_input_time_ms) {
   RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
 
   if (!inited_) {
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
   }
 
   bool send_key_frame = false;
   if (codec_mode_ == webrtc::kRealtimeVideo) {
@@ skipping 78 matching lines @@
             quality_scaler_.GetScaledBuffer(frame.video_frame_buffer()));
       }
     }
   }
 
   if (!MaybeReconfigureEncoderOnCodecThread(input_frame)) {
     ALOGE << "Failed to reconfigure encoder.";
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
 
-  const int64_t time_before_calling_encode = rtc::TimeMillis();
   const bool key_frame =
       frame_types->front() != webrtc::kVideoFrameDelta || send_key_frame;
   bool encode_status = true;
   if (!input_frame.video_frame_buffer()->native_handle()) {
     int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_,
         j_dequeue_input_buffer_method_);
     CHECK_EXCEPTION(jni);
     if (j_input_buffer_index == -1) {
       // Video codec falls behind - no input buffer available.
       ALOGW << "Encoder drop frame - no input buffers available";
-      current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
-      frames_dropped_media_encoder_++;
-      OnDroppedFrameOnCodecThread();
+      if (frames_received_ > 1) {
+        current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
+        frames_dropped_media_encoder_++;
+        OnDroppedFrameOnCodecThread();
+      } else {
+        // Input buffers are not ready after codec initialization, HW is still
+        // allocating thme - this is expected and should not result in drop
+        // frame report.
+        frames_received_ = 0;
+      }
       return WEBRTC_VIDEO_CODEC_OK;  // TODO(fischman): see webrtc bug 2887.
-    }
-    if (j_input_buffer_index == -2) {
+    } else if (j_input_buffer_index == -2) {
       ResetCodecOnCodecThread();
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
     encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame,
         j_input_buffer_index);
   } else {
     encode_status = EncodeTextureOnCodecThread(jni, key_frame, input_frame);
   }
 
   if (!encode_status) {
     ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp();
     ResetCodecOnCodecThread();
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
 
   // Save input image timestamps for later output.
   input_frame_infos_.emplace_back(
-      time_before_calling_encode, input_frame.timestamp(),
+      frame_input_time_ms, input_frame.timestamp(),
       input_frame.render_time_ms(), input_frame.rotation());
 
   last_input_timestamp_ms_ =
       current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec;
 
   current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
 
   codec_thread_->Clear(this);
   codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
 
@@ skipping 192 matching lines @@
       ResetCodecOnCodecThread();
       return false;
     }
 
     // Get key and config frame flags.
     jobject j_output_buffer =
         GetOutputBufferInfoBuffer(jni, j_output_buffer_info);
     bool key_frame = GetOutputBufferInfoIsKeyFrame(jni, j_output_buffer_info);
 
     // Get frame timestamps from a queue - for non config frames only.
+    int64_t encoding_start_time_ms = 0;
     int64_t frame_encoding_time_ms = 0;
     last_output_timestamp_ms_ =
         GetOutputBufferInfoPresentationTimestampUs(jni, j_output_buffer_info) /
         rtc::kNumMicrosecsPerMillisec;
     if (!input_frame_infos_.empty()) {
       const InputFrameInfo& frame_info = input_frame_infos_.front();
       output_timestamp_ = frame_info.frame_timestamp;
       output_render_time_ms_ = frame_info.frame_render_time_ms;
       output_rotation_ = frame_info.rotation;
-      frame_encoding_time_ms =
-          rtc::TimeMillis() - frame_info.encode_start_time;
+      encoding_start_time_ms = frame_info.encode_start_time;
       input_frame_infos_.pop_front();
     }
 
     // Extract payload.
     size_t payload_size = jni->GetDirectBufferCapacity(j_output_buffer);
     uint8_t* payload = reinterpret_cast<uint8_t*>(
         jni->GetDirectBufferAddress(j_output_buffer));
     CHECK_EXCEPTION(jni);
 
-    if (frames_encoded_ < kMaxEncodedLogFrames) {
-      int current_latency =
-          (int)(last_input_timestamp_ms_ - last_output_timestamp_ms_);
-      ALOGD << "Encoder frame out # " << frames_encoded_ <<
-          ". Key: " << key_frame <<
-          ". Size: " << payload_size <<
-          ". TS: " << (int)last_output_timestamp_ms_ <<
-          ". Latency: " << current_latency <<
-          ". EncTime: " << frame_encoding_time_ms;
-    }
-
     // Callback - return encoded frame.
     int32_t callback_status = 0;
     if (callback_) {
       std::unique_ptr<webrtc::EncodedImage> image(
           new webrtc::EncodedImage(payload, payload_size, payload_size));
       image->_encodedWidth = width_;
       image->_encodedHeight = height_;
       image->_timeStamp = output_timestamp_;
       image->capture_time_ms_ = output_render_time_ms_;
       image->rotation_ = output_rotation_;
@@ skipping 105 matching lines @@
     // Return output buffer back to the encoder.
     bool success = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
                                           j_release_output_buffer_method_,
                                           output_buffer_index);
     CHECK_EXCEPTION(jni);
     if (!success) {
       ResetCodecOnCodecThread();
       return false;
     }
 
+    // Print per frame statistics.
+    if (encoding_start_time_ms > 0) {
+      frame_encoding_time_ms = rtc::TimeMillis() - encoding_start_time_ms;
+    }
+    if (frames_encoded_ < kMaxEncodedLogFrames) {
+      int current_latency =
+          (int)(last_input_timestamp_ms_ - last_output_timestamp_ms_);
+      ALOGD << "Encoder frame out # " << frames_encoded_ <<
+          ". Key: " << key_frame <<
+          ". Size: " << payload_size <<
+          ". TS: " << (int)last_output_timestamp_ms_ <<
+          ". Latency: " << current_latency <<
+          ". EncTime: " << frame_encoding_time_ms;
+    }
+
     // Calculate and print encoding statistics - every 3 seconds.
     frames_encoded_++;
     current_frames_++;
     current_bytes_ += payload_size;
     current_encoding_time_ms_ += frame_encoding_time_ms;
     LogStatistics(false);
 
     if (callback_status > 0) {
       drop_next_input_frame_ = true;
       // Theoretically could handle callback_status<0 here, but unclear what
@@ skipping 163 matching lines @@
   return supported_codecs_;
 }
 
 void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
     webrtc::VideoEncoder* encoder) {
   ALOGD << "Destroy video encoder.";
   delete encoder;
 }
 
 }  // namespace webrtc_jni