Let ViEEncoder express resolution requests as Sinkwants

webrtc/media/engine/webrtcvideoengine2.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 13 matching lines @@
 #include "webrtc/call.h"
 #include "webrtc/media/engine/constants.h"
 #include "webrtc/media/engine/simulcast.h"
 #include "webrtc/media/engine/webrtcmediaengine.h"
 #include "webrtc/media/engine/webrtcvideoencoderfactory.h"
 #include "webrtc/media/engine/webrtcvoiceengine.h"
 #include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
 #include "webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h"
 #include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
 #include "webrtc/system_wrappers/include/field_trial.h"
-#include "webrtc/system_wrappers/include/metrics.h"
 #include "webrtc/video_decoder.h"
 #include "webrtc/video_encoder.h"
 
 namespace cricket {
 namespace {
 
 // Wrap cricket::WebRtcVideoEncoderFactory as a webrtc::VideoEncoderFactory.
 class EncoderFactoryAdapter : public webrtc::VideoEncoderFactory {
  public:
   // EncoderFactoryAdapter doesn't take ownership of |factory|, which is owned
@@ skipping 200 matching lines @@
   if (!rtx_ssrcs.empty() && primary_ssrcs.size() != rtx_ssrcs.size()) {
     LOG(LS_ERROR)
         << "RTX SSRCs exist, but don't cover all SSRCs (unsupported): "
         << sp.ToString();
     return false;
   }
 
   return true;
 }
 
-inline bool ContainsHeaderExtension(
-    const std::vector<webrtc::RtpExtension>& extensions,
-    const std::string& uri) {
-  for (const auto& kv : extensions) {
-    if (kv.uri == uri) {
-      return true;
-    }
-  }
-  return false;
-}
-
 // Returns true if the given codec is disallowed from doing simulcast.
 bool IsCodecBlacklistedForSimulcast(const std::string& codec_name) {
   return CodecNamesEq(codec_name, kH264CodecName) ||
          CodecNamesEq(codec_name, kVp9CodecName);
 }
 
 // The selected thresholds for QVGA and VGA corresponded to a QP around 10.
 // The change in QP declined above the selected bitrates.
 static int GetMaxDefaultVideoBitrateKbps(int width, int height) {
   if (width * height <= 320 * 240) {
@@ skipping 127 matching lines @@
 const int kVideoRtpBufferSize = 65536;
 
 // This constant is really an on/off, lower-level configurable NACK history
 // duration hasn't been implemented.
 static const int kNackHistoryMs = 1000;
 
 static const int kDefaultQpMax = 56;
 
 static const int kDefaultRtcpReceiverReportSsrc = 1;
 
-// Down grade resolution at most 2 times for CPU reasons.
-static const int kMaxCpuDowngrades = 2;
-
 // Minimum time interval for logging stats.
 static const int64_t kStatsLogIntervalMs = 10000;
 
 // Adds |codec| to |list|, and also adds an RTX codec if |codec|'s name is
 // recognized.
 // TODO(deadbeef): Should we add RTX codecs for external codecs whose names we
 // don't recognize?
 void AddCodecAndMaybeRtxCodec(const VideoCodec& codec,
                               std::vector<VideoCodec>* codecs) {
   codecs->push_back(codec);
@@ skipping 1162 matching lines @@
     int max_bitrate_bps,
     const rtc::Optional<VideoCodecSettings>& codec_settings,
     const rtc::Optional<std::vector<webrtc::RtpExtension>>& rtp_extensions,
     // TODO(deadbeef): Don't duplicate information between send_params,
     // rtp_extensions, options, etc.
     const VideoSendParameters& send_params)
     : worker_thread_(rtc::Thread::Current()),
       ssrcs_(sp.ssrcs),
       ssrc_groups_(sp.ssrc_groups),
       call_(call),
-      cpu_restricted_counter_(0),
-      number_of_cpu_adapt_changes_(0),
-      frame_count_(0),
-      cpu_restricted_frame_count_(0),
+      enable_cpu_overuse_detection_(enable_cpu_overuse_detection),
       source_(nullptr),
       external_encoder_factory_(external_encoder_factory),
       stream_(nullptr),
       encoder_sink_(nullptr),
       parameters_(std::move(config), options, max_bitrate_bps, codec_settings),
       rtp_parameters_(CreateRtpParametersWithOneEncoding()),
       allocated_encoder_(nullptr, webrtc::kVideoCodecUnknown, false),
       sending_(false),
       last_frame_timestamp_us_(0) {
   parameters_.config.rtp.max_packet_size = kVideoMtu;
   parameters_.conference_mode = send_params.conference_mode;
 
   sp.GetPrimarySsrcs(&parameters_.config.rtp.ssrcs);
   sp.GetFidSsrcs(parameters_.config.rtp.ssrcs,
                  &parameters_.config.rtp.rtx.ssrcs);
   parameters_.config.rtp.c_name = sp.cname;
   if (rtp_extensions) {
     parameters_.config.rtp.extensions = *rtp_extensions;
   }
   parameters_.config.rtp.rtcp_mode = send_params.rtcp.reduced_size
                                          ? webrtc::RtcpMode::kReducedSize
                                          : webrtc::RtcpMode::kCompound;
-  parameters_.config.overuse_callback =
-      enable_cpu_overuse_detection ? this : nullptr;
-
-  // Only request rotation at the source when we positively know that the remote
-  // side doesn't support the rotation extension. This allows us to prepare the
-  // encoder in the expectation that rotation is supported - which is the common
-  // case.
-  sink_wants_.rotation_applied =
-      rtp_extensions &&
-      !ContainsHeaderExtension(*rtp_extensions,
-                               webrtc::RtpExtension::kVideoRotationUri);
-
   if (codec_settings) {
     SetCodec(*codec_settings);
   }
 }
 
 WebRtcVideoChannel2::WebRtcVideoSendStream::~WebRtcVideoSendStream() {
-  DisconnectSource();
   if (stream_ != NULL) {
     call_->DestroyVideoSendStream(stream_);
   }
   DestroyVideoEncoder(&allocated_encoder_);
-  UpdateHistograms();
-}
-
-void WebRtcVideoChannel2::WebRtcVideoSendStream::UpdateHistograms() const {
-  const int kMinRequiredFrames = 200;
-  if (frame_count_ > kMinRequiredFrames) {
-    RTC_HISTOGRAM_PERCENTAGE("WebRTC.Video.CpuLimitedResolutionInPercent",
-                             cpu_restricted_frame_count_ * 100 / frame_count_);
-  }
 }
 
 void WebRtcVideoChannel2::WebRtcVideoSendStream::OnFrame(
     const VideoFrame& frame) {
   TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::OnFrame");
   webrtc::VideoFrame video_frame(frame.video_frame_buffer(),
                                  frame.rotation(),
                                  frame.timestamp_us());
 
   rtc::CritScope cs(&lock_);
@@ skipping 13 matching lines @@
                  << ", texture=" << last_frame_info_.is_texture;
   }
 
   if (encoder_sink_ == NULL) {
     // Frame input before send codecs are configured, dropping frame.
     return;
   }
 
   last_frame_timestamp_us_ = video_frame.timestamp_us();
 
-  ++frame_count_;
-  if (cpu_restricted_counter_ > 0)
-    ++cpu_restricted_frame_count_;
-
   // Forward frame to the encoder regardless if we are sending or not. This is
   // to ensure that the encoder can be reconfigured with the correct frame size
   // as quickly as possible.
   encoder_sink_->OnFrame(video_frame);
 }
 
 bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetVideoSend(
     bool enable,
     const VideoOptions* options,
     rtc::VideoSourceInterface<cricket::VideoFrame>* source) {
   TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::SetVideoSend");
   RTC_DCHECK_RUN_ON(&thread_checker_);
 
   // Ignore |options| pointer if |enable| is false.
   bool options_present = enable && options;
   bool source_changing = source_ != source;
-  if (source_changing) {
-    DisconnectSource();
-  }
 
   if (options_present) {
     VideoOptions old_options = parameters_.options;
     parameters_.options.SetAll(*options);
     if (parameters_.options != old_options) {
       ReconfigureEncoder();
     }
   }
 
   if (source_changing) {
     rtc::CritScope cs(&lock_);
-    if (source == nullptr && encoder_sink_ != nullptr &&
-        last_frame_info_.width > 0) {
+    if (source == nullptr && last_frame_info_.width > 0 && encoder_sink_) {
       LOG(LS_VERBOSE) << "Disabling capturer, sending black frame.";
       // Force this black frame not to be dropped due to timestamp order
       // check. As IncomingCapturedFrame will drop the frame if this frame's
       // timestamp is less than or equal to last frame's timestamp, it is
       // necessary to give this black frame a larger timestamp than the
       // previous one.
       last_frame_timestamp_us_ += rtc::kNumMicrosecsPerMillisec;
       rtc::scoped_refptr<webrtc::I420Buffer> black_buffer(
           webrtc::I420Buffer::Create(last_frame_info_.width,
                                      last_frame_info_.height));
       black_buffer->SetToBlack();
 
       encoder_sink_->OnFrame(webrtc::VideoFrame(
           black_buffer, last_frame_info_.rotation, last_frame_timestamp_us_));
     }
-    source_ = source;
   }
 
-  if (source_changing && source_) {
-    // |source_->AddOrUpdateSink| may not be called while holding |lock_| since
-    // that might cause a lock order inversion.
-    source_->AddOrUpdateSink(this, sink_wants_);
+  // TODO(perkj, nisse): Remove |source_| and directly call
+  // |stream_|->SetSource(source) once the video frame types have been
+  // merged.

[nisse-webrtc, 2016/10/28 10:53:44]

That time is now. ;-)

[perkj_webrtc, 2016/10/28 14:05:34]

Great- but it is a little bit more to it so I would like to do it in a separate
cl.

[nisse-webrtc, 2016/10/31 07:52:46]

Sure. I'd also really like to see this change landed sooner than later. Maybe
update TODO comment, though?

+  if (source_ && stream_) {
+    stream_->SetSource(
+        nullptr, webrtc::VideoSendStream::DegradationPreference::kBalanced);
+  }
+  // Switch to the new source.
+  source_ = source;
+  if (source && stream_) {
+    // Do not adapt resolution for screen content as this will likely
+    // result in blurry and unreadable text.
+    stream_->SetSource(
+        this, enable_cpu_overuse_detection_ &&
+                      !parameters_.options.is_screencast.value_or(false)
+                  ? webrtc::VideoSendStream::DegradationPreference::kBalanced
+                  : webrtc::VideoSendStream::DegradationPreference::
+                        kMaintainResolution);
   }
   return true;
 }
 
-void WebRtcVideoChannel2::WebRtcVideoSendStream::DisconnectSource() {
-  RTC_DCHECK_RUN_ON(&thread_checker_);
-  if (source_ == nullptr) {
-    return;
-  }
-
-  // |source_->RemoveSink| may not be called while holding |lock_| since
-  // that might cause a lock order inversion.
-  source_->RemoveSink(this);
-  source_ = nullptr;
-  // Reset |cpu_restricted_counter_| if the source is changed. It is not
-  // possible to know if the video resolution is restricted by CPU usage after
-  // the source is changed since the next source might be screen capture
-  // with another resolution and frame rate.
-  cpu_restricted_counter_ = 0;
-}
-
 const std::vector<uint32_t>&
 WebRtcVideoChannel2::WebRtcVideoSendStream::GetSsrcs() const {
   return ssrcs_;
 }
 
 webrtc::VideoCodecType CodecTypeFromName(const std::string& name) {
   if (CodecNamesEq(name, kVp8CodecName)) {
     return webrtc::kVideoCodecVP8;
   } else if (CodecNamesEq(name, kVp9CodecName)) {
     return webrtc::kVideoCodecVP9;
@@ skipping 118 matching lines @@
     SetCodec(*params.codec);
     recreate_stream = false;  // SetCodec has already recreated the stream.
   } else if (params.conference_mode && parameters_.codec_settings) {
     SetCodec(*parameters_.codec_settings);
     recreate_stream = false;  // SetCodec has already recreated the stream.
   }
   if (recreate_stream) {
     LOG(LS_INFO) << "RecreateWebRtcStream (send) because of SetSendParameters";
     RecreateWebRtcStream();
   }
-
-  // |source_->AddOrUpdateSink| may not be called while holding |lock_| since
-  // that might cause a lock order inversion.
-  if (params.rtp_header_extensions) {
-    sink_wants_.rotation_applied = !ContainsHeaderExtension(
-        *params.rtp_header_extensions, webrtc::RtpExtension::kVideoRotationUri);
-    if (source_) {
-      source_->AddOrUpdateSink(this, sink_wants_);
-    }
-  }
 }
 
 bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetRtpParameters(
     const webrtc::RtpParameters& new_parameters) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   if (!ValidateRtpParameters(new_parameters)) {
     return false;
   }
 
   bool reconfigure_encoder = new_parameters.encodings[0].max_bitrate_bps !=
@@ skipping 108 matching lines @@
 
   parameters_.encoder_config = std::move(encoder_config);
 }
 
 void WebRtcVideoChannel2::WebRtcVideoSendStream::SetSend(bool send) {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   sending_ = send;
   UpdateSendState();
 }
 
+void WebRtcVideoChannel2::WebRtcVideoSendStream::RemoveSink(
+    VideoSinkInterface<webrtc::VideoFrame>* sink) {
+  RTC_DCHECK_RUN_ON(&thread_checker_);
+  {
+    rtc::CritScope cs(&lock_);
+    RTC_DCHECK(encoder_sink_ == sink);
+    encoder_sink_ = nullptr;
+  }
+  source_->RemoveSink(this);
+}
+
 void WebRtcVideoChannel2::WebRtcVideoSendStream::AddOrUpdateSink(
     VideoSinkInterface<webrtc::VideoFrame>* sink,
     const rtc::VideoSinkWants& wants) {
-  // TODO(perkj): Actually consider the encoder |wants| and remove
-  // WebRtcVideoSendStream::OnLoadUpdate(Load load).
-  rtc::CritScope cs(&lock_);
-  RTC_DCHECK(!encoder_sink_ || encoder_sink_ == sink);
-  encoder_sink_ = sink;
-}
-
-void WebRtcVideoChannel2::WebRtcVideoSendStream::RemoveSink(
-    VideoSinkInterface<webrtc::VideoFrame>* sink) {
-  rtc::CritScope cs(&lock_);
-  RTC_DCHECK_EQ(encoder_sink_, sink);
-  encoder_sink_ = nullptr;
-}
-
-void WebRtcVideoChannel2::WebRtcVideoSendStream::OnLoadUpdate(Load load) {
-  if (worker_thread_ != rtc::Thread::Current()) {
-    invoker_.AsyncInvoke<void>(
-        RTC_FROM_HERE, worker_thread_,
-        rtc::Bind(&WebRtcVideoChannel2::WebRtcVideoSendStream::OnLoadUpdate,
-                  this, load));
-    return;
+  if (worker_thread_ == rtc::Thread::Current()) {
+    // AddOrUpdateSink is called on |worker_thread_| if this is the first
+    // registration of |sink|.
+    RTC_DCHECK_RUN_ON(&thread_checker_);
+    {
+      rtc::CritScope cs(&lock_);
+      encoder_sink_ = sink;
+    }
+    source_->AddOrUpdateSink(this, wants);
+  } else {
+    // Subsequent calls to AddOrUpdateSink will happen on the encoder task
+    // queue.
+    invoker_.AsyncInvoke<void>(RTC_FROM_HERE, worker_thread_, [this, wants] {
+      RTC_DCHECK_RUN_ON(&thread_checker_);
+      bool encoder_sink_valid = true;
+      {
+        rtc::CritScope cs(&lock_);
+        encoder_sink_valid = encoder_sink_ != nullptr;
+      }
+      // Since |source_| is still valid after a call to RemoveSink, check if
+      // |encoder_sink_| is still valid to check if this call should be
+      // cancelled.
+      if (source_ && encoder_sink_valid) {
+        source_->AddOrUpdateSink(this, wants);
+      }
+    });
   }
-  RTC_DCHECK_RUN_ON(&thread_checker_);
-  if (!source_) {
-    return;
-  }
-
-  LOG(LS_INFO) << "OnLoadUpdate " << load << ", is_screencast: "
-               << (parameters_.options.is_screencast
-                       ? (*parameters_.options.is_screencast ? "true" : "false")
-                       : "unset");
-  // Do not adapt resolution for screen content as this will likely result in
-  // blurry and unreadable text.
-  if (parameters_.options.is_screencast.value_or(false))
-    return;
-
-  rtc::Optional<int> max_pixel_count;
-  rtc::Optional<int> max_pixel_count_step_up;
-  if (load == kOveruse) {
-    rtc::CritScope cs(&lock_);
-    if (cpu_restricted_counter_ >= kMaxCpuDowngrades) {
-      return;
-    }
-    // The input video frame size will have a resolution with less than or
-    // equal to |max_pixel_count| depending on how the source can scale the
-    // input frame size.
-    max_pixel_count = rtc::Optional<int>(
-        (last_frame_info_.height * last_frame_info_.width * 3) / 5);
-    // Increase |number_of_cpu_adapt_changes_| if
-    // sink_wants_.max_pixel_count will be changed since
-    // last time |source_->AddOrUpdateSink| was called. That is, this will
-    // result in a new request for the source to change resolution.
-    if (!sink_wants_.max_pixel_count ||
-        *sink_wants_.max_pixel_count > *max_pixel_count) {
-      ++number_of_cpu_adapt_changes_;
-      ++cpu_restricted_counter_;
-    }
-  } else {
-    RTC_DCHECK(load == kUnderuse);
-    rtc::CritScope cs(&lock_);
-    // The input video frame size will have a resolution with "one step up"
-    // pixels than |max_pixel_count_step_up| where "one step up" depends on
-    // how the source can scale the input frame size.
-    max_pixel_count_step_up =
-        rtc::Optional<int>(last_frame_info_.height * last_frame_info_.width);
-    // Increase |number_of_cpu_adapt_changes_| if
-    // sink_wants_.max_pixel_count_step_up will be changed since
-    // last time |source_->AddOrUpdateSink| was called. That is, this will
-    // result in a new request for the source to change resolution.
-    if (sink_wants_.max_pixel_count ||
-        (sink_wants_.max_pixel_count_step_up &&
-         *sink_wants_.max_pixel_count_step_up < *max_pixel_count_step_up)) {
-      ++number_of_cpu_adapt_changes_;
-      --cpu_restricted_counter_;
-    }
-  }
-  sink_wants_.max_pixel_count = max_pixel_count;
-  sink_wants_.max_pixel_count_step_up = max_pixel_count_step_up;
-  // |source_->AddOrUpdateSink| may not be called while holding |lock_| since
-  // that might cause a lock order inversion.
-  source_->AddOrUpdateSink(this, sink_wants_);
 }
 
 VideoSenderInfo WebRtcVideoChannel2::WebRtcVideoSendStream::GetVideoSenderInfo(
     bool log_stats) {
   VideoSenderInfo info;
   RTC_DCHECK_RUN_ON(&thread_checker_);
   for (uint32_t ssrc : parameters_.config.rtp.ssrcs)
     info.add_ssrc(ssrc);
 
   if (parameters_.codec_settings)
     info.codec_name = parameters_.codec_settings->codec.name;
 
   if (stream_ == NULL)
     return info;
 
   webrtc::VideoSendStream::Stats stats = stream_->GetStats();
 
   if (log_stats)
     LOG(LS_INFO) << stats.ToString(rtc::TimeMillis());
 
-  info.adapt_changes = number_of_cpu_adapt_changes_;
+  info.adapt_changes = stats.number_of_cpu_adapt_changes;
   info.adapt_reason =
-      cpu_restricted_counter_ <= 0 ? ADAPTREASON_NONE : ADAPTREASON_CPU;
+      stats.cpu_limited_resolution ? ADAPTREASON_CPU : ADAPTREASON_NONE;
 
   // Get bandwidth limitation info from stream_->GetStats().
   // Input resolution (output from video_adapter) can be further scaled down or
   // higher video layer(s) can be dropped due to bitrate constraints.
   // Note, adapt_changes only include changes from the video_adapter.
   if (stats.bw_limited_resolution)
     info.adapt_reason |= ADAPTREASON_BANDWIDTH;
 
   info.encoder_implementation_name = stats.encoder_implementation_name;
   info.ssrc_groups = ssrc_groups_;
@@ skipping 71 matching lines @@
       ConfigureVideoEncoderSettings(parameters_.codec_settings->codec);
 
   webrtc::VideoSendStream::Config config = parameters_.config.Copy();
   if (!config.rtp.rtx.ssrcs.empty() && config.rtp.rtx.payload_type == -1) {
     LOG(LS_WARNING) << "RTX SSRCs configured but there's no configured RTX "
                        "payload type the set codec. Ignoring RTX.";
     config.rtp.rtx.ssrcs.clear();
   }
   stream_ = call_->CreateVideoSendStream(std::move(config),
                                          parameters_.encoder_config.Copy());
-  stream_->SetSource(this);
 
   parameters_.encoder_config.encoder_specific_settings = NULL;
 
+  if (source_) {
+    // TODO(perkj, nisse): Remove |source_| and directly call
+    // |stream_|->SetSource(source) once the video frame types have been
+    // merged and |stream_| internally reconfigure the encoder on frame
+    // resolution change.
+    // Do not adapt resolution for screen content as this will likely result in
+    // blurry and unreadable text.
+    stream_->SetSource(
+        this, enable_cpu_overuse_detection_ &&
+                      !parameters_.options.is_screencast.value_or(false)
+                  ? webrtc::VideoSendStream::DegradationPreference::kBalanced
+                  : webrtc::VideoSendStream::DegradationPreference::
+                        kMaintainResolution);
+  }
+
   // Call stream_->Start() if necessary conditions are met.
   UpdateSendState();
 }
 
 WebRtcVideoChannel2::WebRtcVideoReceiveStream::WebRtcVideoReceiveStream(
     webrtc::Call* call,
     const StreamParams& sp,
     webrtc::VideoReceiveStream::Config config,
     WebRtcVideoDecoderFactory* external_decoder_factory,
     bool default_stream,
@@ skipping 432 matching lines @@
         rtx_mapping[video_codecs[i].codec.id] !=
             ulpfec_config.red_payload_type) {
       video_codecs[i].rtx_payload_type = rtx_mapping[video_codecs[i].codec.id];
     }
   }
 
   return video_codecs;
 }
 
 }  // namespace cricket