Add QP for MediaCodec decoder.

webrtc/sdk/android/src/jni/androidmediadecoder_jni.cc

 /*
  *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include <algorithm>
 #include <memory>
 #include <vector>
+#include <deque>

[magjed_webrtc, 2017/02/16 20:22:20]

sort the list

[sakal, 2017/02/17 09:23:43]

Done.

 
 // NOTICE: androidmediadecoder_jni.h must be included before
 // androidmediacodeccommon.h to avoid build errors.
 #include "webrtc/sdk/android/src/jni/androidmediadecoder_jni.h"
 
 #include "third_party/libyuv/include/libyuv/convert.h"
 #include "third_party/libyuv/include/libyuv/convert_from.h"
 #include "third_party/libyuv/include/libyuv/video_common.h"
 #include "webrtc/sdk/android/src/jni/androidmediacodeccommon.h"
 #include "webrtc/sdk/android/src/jni/classreferenceholder.h"
 #include "webrtc/sdk/android/src/jni/native_handle_impl.h"
 #include "webrtc/sdk/android/src/jni/surfacetexturehelper_jni.h"
 #include "webrtc/base/bind.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/scoped_ref_ptr.h"
 #include "webrtc/base/thread.h"
 #include "webrtc/base/timeutils.h"
+#include "webrtc/common_video/h264/h264_bitstream_parser.h"
 #include "webrtc/common_video/include/i420_buffer_pool.h"
+#include "webrtc/modules/video_coding/utility/vp8_header_parser.h"

[magjed_webrtc, 2017/02/16 20:22:20]

sort the list

[sakal, 2017/02/17 09:23:43]

Done.

 #include "webrtc/modules/video_coding/include/video_codec_interface.h"
 #include "webrtc/system_wrappers/include/logcat_trace_context.h"
 
 using rtc::Bind;
 using rtc::Thread;
 using rtc::ThreadManager;
 
 using webrtc::CodecSpecificInfo;
 using webrtc::DecodedImageCallback;
 using webrtc::EncodedImage;
@@ skipping 82 matching lines @@
   int frames_received_;  // Number of frames received by decoder.
   int frames_decoded_;  // Number of frames decoded by decoder.
   // Number of decoded frames for which log information is displayed.
   int frames_decoded_logged_;
   int64_t start_time_ms_;  // Start time for statistics.
   int current_frames_;  // Number of frames in the current statistics interval.
   int current_bytes_;  // Encoded bytes in the current statistics interval.
   int current_decoding_time_ms_;  // Overall decoding time in the current second
   int current_delay_time_ms_;  // Overall delay time in the current second.
   uint32_t max_pending_frames_;  // Maximum number of pending input frames.
+  webrtc::H264BitstreamParser h264_bitstream_parser_;
+  std::deque<int> pending_frame_qps_;

[magjed_webrtc, 2017/02/16 20:22:20]

Why not a regular queue? You don't need it to be double-ended.

[sakal, 2017/02/17 09:23:43]

queue type doesn't have clear method that is required to empty the queue.

 
   // State that is constant for the lifetime of this object once the ctor
   // returns.
   std::unique_ptr<Thread>
       codec_thread_;  // Thread on which to operate MediaCodec.
   ScopedGlobalRef<jclass> j_media_codec_video_decoder_class_;
   ScopedGlobalRef<jobject> j_media_codec_video_decoder_;
   jmethodID j_init_decode_method_;
   jmethodID j_reset_method_;
   jmethodID j_release_method_;
@@ skipping 171 matching lines @@
 
   key_frame_required_ = true;
   frames_received_ = 0;
   frames_decoded_ = 0;
   frames_decoded_logged_ = kMaxDecodedLogFrames;
   start_time_ms_ = rtc::TimeMillis();
   current_frames_ = 0;
   current_bytes_ = 0;
   current_decoding_time_ms_ = 0;
   current_delay_time_ms_ = 0;
+  pending_frame_qps_.clear();
 }
 
 int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
   CheckOnCodecThread();
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
   ScopedLocalRefFrame local_ref_frame(jni);
   ALOGD << "InitDecodeOnCodecThread Type: " << (int)codecType_ << ". "
       << codec_.width << " x " << codec_.height << ". Fps: " <<
       (int)codec_.maxFramerate;
 
@@ skipping 310 matching lines @@
     ALOGD << "Decoder frame in # " << frames_received_ <<
         ". Type: " << inputImage._frameType <<
         ". Buffer # " << j_input_buffer_index <<
         ". TS: " << presentation_timestamp_us / 1000 <<
         ". Size: " << inputImage._length;
   }
 
   // Save input image timestamps for later output.
   frames_received_++;
   current_bytes_ += inputImage._length;
+  int qp = -1;

[magjed_webrtc, 2017/02/16 20:22:20]

Use a 'rtc::Optional<uint8_t> qp' directly instead of magic -1 value.

[sakal, 2017/02/17 09:23:43]

Done.

+  if (codecType_ == kVideoCodecVP8) {
+    webrtc::vp8::GetQp(inputImage._buffer, inputImage._length, &qp);
+  } else if (codecType_ == kVideoCodecH264) {
+    h264_bitstream_parser_.ParseBitstream(inputImage._buffer,
+                                          inputImage._length);
+    h264_bitstream_parser_.GetLastSliceQp(&qp);
+  }
+  pending_frame_qps_.push_back(qp);
 
   // Feed input to decoder.
   bool success = jni->CallBooleanMethod(
       *j_media_codec_video_decoder_,
       j_queue_input_buffer_method_,
       j_input_buffer_index,
       inputImage._length,
       presentation_timestamp_us,
       static_cast<int64_t> (inputImage._timeStamp),
       inputImage.ntp_time_ms_);
   if (CheckException(jni) || !success) {
     ALOGE << "queueInputBuffer error";
     return ProcessHWErrorOnCodecThread();
   }
 
   // Try to drain the decoder
   if (!DeliverPendingOutputs(jni, 0)) {
     ALOGE << "DeliverPendingOutputs error";
     return ProcessHWErrorOnCodecThread();
   }
 
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 bool MediaCodecVideoDecoder::DeliverPendingOutputs(
     JNIEnv* jni, int dequeue_timeout_ms) {
+  CheckOnCodecThread();
   if (frames_received_ <= frames_decoded_) {
     // No need to query for output buffers - decoder is drained.
     return true;
   }
   // Get decoder output.
   jobject j_decoder_output_buffer =
       jni->CallObjectMethod(*j_media_codec_video_decoder_,
           use_surface_ ? j_dequeue_texture_buffer_method_
                        : j_dequeue_byte_buffer_method_,
           dequeue_timeout_ms);
@@ skipping 162 matching lines @@
     current_decoding_time_ms_ = 0;
     current_delay_time_ms_ = 0;
   }
 
   // If the frame was dropped, frame_buffer is left as nullptr.
   if (frame_buffer) {
     VideoFrame decoded_frame(frame_buffer, 0, 0, webrtc::kVideoRotation_0);
     decoded_frame.set_timestamp(output_timestamps_ms);
     decoded_frame.set_ntp_time_ms(output_ntp_timestamps_ms);
 
-    const int32_t callback_status =
-        callback_->Decoded(decoded_frame, decode_time_ms);
-    if (callback_status > 0) {
-      ALOGE << "callback error";
-    }
+    int qp = pending_frame_qps_.front();
+    pending_frame_qps_.pop_front();
+    callback_->Decoded(

[magjed_webrtc, 2017/02/16 20:22:20]

Why did you remove the return value check?

[sakal, 2017/02/17 09:23:43]

The new callback doesn't include a return value because it always returned
success anyway.

+        decoded_frame, rtc::Optional<int32_t>(decode_time_ms),
+        qp != -1 ? rtc::Optional<uint8_t>(qp) : rtc::Optional<uint8_t>());
   }
   return true;
 }
 
 int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback(
     DecodedImageCallback* callback) {
   callback_ = callback;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
@@ skipping 101 matching lines @@
     webrtc::VideoDecoder* decoder) {
   ALOGD << "Destroy video decoder.";
   delete decoder;
 }
 
 const char* MediaCodecVideoDecoder::ImplementationName() const {
   return "MediaCodec";
 }
 
 }  // namespace webrtc_jni