Add content type information to encoded images and corresponding rtp extension header

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 238 matching lines @@
   DegradationPreference degradation_preference_ GUARDED_BY(&crit_);
   rtc::VideoSourceInterface<VideoFrame>* source_ GUARDED_BY(&crit_);
 
   RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy);
 };
 
 ViEEncoder::ViEEncoder(uint32_t number_of_cores,
                        SendStatisticsProxy* stats_proxy,
                        const VideoSendStream::Config::EncoderSettings& settings,
                        rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback,
-                       EncodedFrameObserver* encoder_timing)
+                       EncodedFrameObserver* encoder_timing,
+                       VideoEncoderConfig::ContentType content_type)
     : shutdown_event_(true /* manual_reset */, false),
       number_of_cores_(number_of_cores),
       initial_rampup_(0),
       source_proxy_(new VideoSourceProxy(this)),
       sink_(nullptr),
       settings_(settings),
       codec_type_(PayloadNameToCodecType(settings.payload_name)
                       .value_or(VideoCodecType::kVideoCodecUnknown)),
+      content_type_(content_type),
       video_sender_(Clock::GetRealTimeClock(), this, this),
       overuse_detector_(GetCpuOveruseOptions(settings.full_overuse_time),
                         this,
                         encoder_timing,
                         stats_proxy),
       stats_proxy_(stats_proxy),
       pre_encode_callback_(pre_encode_callback),
       module_process_thread_(nullptr),
       pending_encoder_reconfiguration_(false),
       encoder_start_bitrate_bps_(0),
@@ skipping 119 matching lines @@
   encoder_config_ = std::move(config);
   pending_encoder_reconfiguration_ = true;
 
   // Reconfigure the encoder now if the encoder has an internal source or
   // if the frame resolution is known. Otherwise, the reconfiguration is
   // deferred until the next frame to minimize the number of reconfigurations.
   // The codec configuration depends on incoming video frame size.
   if (last_frame_info_) {
     ReconfigureEncoder();
   } else if (settings_.internal_source) {
-    last_frame_info_ = rtc::Optional<VideoFrameInfo>(
-        VideoFrameInfo(176, 144, kVideoRotation_0, false));
+    last_frame_info_ = rtc::Optional<VideoFrameInfo>(VideoFrameInfo(
+        176, 144, kVideoRotation_0, VideoContentType::kDefault, false));
     ReconfigureEncoder();
   }
 }
 
 void ViEEncoder::ReconfigureEncoder() {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   RTC_DCHECK(pending_encoder_reconfiguration_);
   std::vector<VideoStream> streams =
       encoder_config_.video_stream_factory->CreateEncoderStreams(
           last_frame_info_->width, last_frame_info_->height, encoder_config_);
@@ skipping 59 matching lines @@
     }
   } else {
     quality_scaler_.reset(nullptr);
     initial_rampup_ = kMaxInitialFramedrop;
   }
 }
 
 void ViEEncoder::OnFrame(const VideoFrame& video_frame) {
   RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_);
   VideoFrame incoming_frame = video_frame;
-
+  if (content_type_ == VideoEncoderConfig::ContentType::kScreen)
+    incoming_frame.set_content_type(VideoContentType::kScreenshare);
   // Local time in webrtc time base.
   int64_t current_time_us = clock_->TimeInMicroseconds();
   int64_t current_time_ms = current_time_us / rtc::kNumMicrosecsPerMillisec;
   // TODO(nisse): This always overrides the incoming timestamp. Don't
   // do that, trust the frame source.
   incoming_frame.set_timestamp_us(current_time_us);
 
   // Capture time may come from clock with an offset and drift from clock_.
   int64_t capture_ntp_time_ms;
   if (video_frame.ntp_time_ms() > 0) {
@@ skipping 63 matching lines @@
   RTC_DCHECK_RUN_ON(&encoder_queue_);
 
   if (pre_encode_callback_)
     pre_encode_callback_->OnFrame(video_frame);
 
   if (!last_frame_info_ || video_frame.width() != last_frame_info_->width ||
       video_frame.height() != last_frame_info_->height ||
       video_frame.rotation() != last_frame_info_->rotation ||
       video_frame.is_texture() != last_frame_info_->is_texture) {
     pending_encoder_reconfiguration_ = true;
-    last_frame_info_ = rtc::Optional<VideoFrameInfo>(
-        VideoFrameInfo(video_frame.width(), video_frame.height(),
-                       video_frame.rotation(), video_frame.is_texture()));
+    last_frame_info_ = rtc::Optional<VideoFrameInfo>(VideoFrameInfo(
+        video_frame.width(), video_frame.height(), video_frame.rotation(),
+        video_frame.content_type(), video_frame.is_texture()));
     LOG(LS_INFO) << "Video frame parameters changed: dimensions="
                  << last_frame_info_->width << "x" << last_frame_info_->height
                  << ", rotation=" << last_frame_info_->rotation
                  << ", texture=" << last_frame_info_->is_texture;
   }
 
   if (initial_rampup_ < kMaxInitialFramedrop &&
       video_frame.size() >
           MaximumFrameSizeForBitrate(encoder_start_bitrate_bps_ / 1000)) {
     LOG(LS_INFO) << "Dropping frame. Too large for target bitrate.";
@@ skipping 192 matching lines @@
   --scale_counter_[reason];
   source_proxy_->RequestHigherResolutionThan(current_pixel_count);
   LOG(LS_INFO) << "Scaling up resolution.";
   for (size_t i = 0; i < kScaleReasonSize; ++i) {
     LOG(LS_INFO) << "Scaled " << scale_counter_[i]
                  << " times for reason: " << (i ? "cpu" : "quality");
   }
 }
 
 }  // namespace webrtc