Add content type information to encoded images and corresponding rtp extension header

webrtc/sdk/android/src/jni/androidmediaencoder_jni.cc

 /*
  *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 224 matching lines @@
   int current_encoding_time_ms_;  // Overall encoding time in the current second
   int64_t last_input_timestamp_ms_;  // Timestamp of last received yuv frame.
   int64_t last_output_timestamp_ms_;  // Timestamp of last encoded frame.
   // Holds the task while the polling loop is paused.
   std::unique_ptr<rtc::QueuedTask> encode_task_;
 
   struct InputFrameInfo {
     InputFrameInfo(int64_t encode_start_time,
                    int32_t frame_timestamp,
                    int64_t frame_render_time_ms,
-                   webrtc::VideoRotation rotation)
+                   webrtc::VideoRotation rotation,
+                   webrtc::VideoContentTypeId content_type)
         : encode_start_time(encode_start_time),
           frame_timestamp(frame_timestamp),
           frame_render_time_ms(frame_render_time_ms),
-          rotation(rotation) {}
+          rotation(rotation),
+          content_type(content_type) {}
     // Time when video frame is sent to encoder input.
     const int64_t encode_start_time;
 
     // Input frame information.
     const int32_t frame_timestamp;
     const int64_t frame_render_time_ms;
     const webrtc::VideoRotation rotation;
+    const webrtc::VideoContentTypeId content_type;
   };
   std::list<InputFrameInfo> input_frame_infos_;
   int32_t output_timestamp_;      // Last output frame timestamp from
                                   // |input_frame_infos_|.
   int64_t output_render_time_ms_; // Last output frame render time from
                                   // |input_frame_infos_|.
   webrtc::VideoRotation output_rotation_;  // Last output frame rotation from
                                            // |input_frame_infos_|.
+  webrtc::VideoContentTypeId output_content_type_;
   // Frame size in bytes fed to MediaCodec.
   int yuv_size_;
   // True only when between a callback_->OnEncodedImage() call return a positive
   // value and the next Encode() call being ignored.
   bool drop_next_input_frame_;
   bool scale_;
   // Global references; must be deleted in Release().
   std::vector<jobject> input_buffers_;
   webrtc::H264BitstreamParser h264_bitstream_parser_;
 
@@ skipping 427 matching lines @@
     }
     frames_dropped_media_encoder_++;
     return WEBRTC_VIDEO_CODEC_OK;
   }
   consecutive_full_queue_frame_drops_ = 0;
 
   rtc::scoped_refptr<webrtc::VideoFrameBuffer> input_buffer(
       frame.video_frame_buffer());
 
   VideoFrame input_frame(input_buffer, frame.timestamp(),
-                         frame.render_time_ms(), frame.rotation());
+                         frame.render_time_ms(), frame.rotation(),
+                         frame.content_type());
 
   if (!MaybeReconfigureEncoder(input_frame)) {
     ALOGE << "Failed to reconfigure encoder.";
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
 
   const bool key_frame =
       frame_types->front() != webrtc::kVideoFrameDelta || send_key_frame;
   bool encode_status = true;
   if (!input_frame.video_frame_buffer()->native_handle()) {
@@ skipping 26 matching lines @@
   }
 
   if (!encode_status) {
     ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp();
     return ProcessHWErrorOnEncode();
   }
 
   // Save input image timestamps for later output.
   input_frame_infos_.emplace_back(frame_input_time_ms, input_frame.timestamp(),
                                   input_frame.render_time_ms(),
-                                  input_frame.rotation());
+                                  input_frame.rotation(),
+                                  input_frame.content_type());
 
   last_input_timestamp_ms_ =
       current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec;
 
   current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
 
   // Start the polling loop if it is not started.
   if (encode_task_) {
     rtc::TaskQueue::Current()->PostDelayedTask(std::move(encode_task_),
                                                kMediaCodecPollMs);
@@ skipping 230 matching lines @@
     int64_t encoding_start_time_ms = 0;
     int64_t frame_encoding_time_ms = 0;
     last_output_timestamp_ms_ =
         GetOutputBufferInfoPresentationTimestampUs(jni, j_output_buffer_info) /
         rtc::kNumMicrosecsPerMillisec;
     if (!input_frame_infos_.empty()) {
       const InputFrameInfo& frame_info = input_frame_infos_.front();
       output_timestamp_ = frame_info.frame_timestamp;
       output_render_time_ms_ = frame_info.frame_render_time_ms;
       output_rotation_ = frame_info.rotation;
+      output_content_type_ = frame_info.content_type;
       encoding_start_time_ms = frame_info.encode_start_time;
       input_frame_infos_.pop_front();
     }
 
     // Extract payload.
     size_t payload_size = jni->GetDirectBufferCapacity(j_output_buffer);
     uint8_t* payload = reinterpret_cast<uint8_t*>(
         jni->GetDirectBufferAddress(j_output_buffer));
     if (CheckException(jni)) {
       ALOGE << "Exception in get direct buffer address.";
       ProcessHWError(true /* reset_if_fallback_unavailable */);
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
 
     // Callback - return encoded frame.
     const VideoCodecType codec_type =
         webrtc::PayloadNameToCodecType(codec_.name)
             .value_or(webrtc::kVideoCodecUnknown);
     webrtc::EncodedImageCallback::Result callback_result(
         webrtc::EncodedImageCallback::Result::OK);
     if (callback_) {
       std::unique_ptr<webrtc::EncodedImage> image(
           new webrtc::EncodedImage(payload, payload_size, payload_size));
       image->_encodedWidth = width_;
       image->_encodedHeight = height_;
       image->_timeStamp = output_timestamp_;
       image->capture_time_ms_ = output_render_time_ms_;
       image->rotation_ = output_rotation_;
+      image->content_type_ = output_content_type_;
       image->_frameType =
           (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta);
       image->_completeFrame = true;
       webrtc::CodecSpecificInfo info;
       memset(&info, 0, sizeof(info));
       info.codecType = codec_type;
       if (codec_type == kVideoCodecVP8) {
         info.codecSpecific.VP8.pictureId = picture_id_;
         info.codecSpecific.VP8.nonReference = false;
         info.codecSpecific.VP8.simulcastIdx = 0;
@@ skipping 263 matching lines @@
   }
 }
 
 void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
     webrtc::VideoEncoder* encoder) {
   ALOGD << "Destroy video encoder.";
   delete encoder;
 }
 
 }  // namespace webrtc_jni