Move direct use of VideoCapturer::VideoAdapter to VideoSinkWants.

webrtc/media/engine/webrtcvideoengine2.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 296 matching lines @@
   if (width * height <= 320 * 240) {
     return 600;
   } else if (width * height <= 640 * 480) {
     return 1700;
   } else if (width * height <= 960 * 540) {
     return 2000;
   } else {
     return 2500;
   }
 }
+
 }  // namespace
 
 // Constants defined in webrtc/media/engine/constants.h
 // TODO(pbos): Move these to a separate constants.cc file.
 const int kMinVideoBitrate = 30;
 const int kStartVideoBitrate = 300;
 
 const int kVideoMtu = 1200;
 const int kVideoRtpBufferSize = 65536;
 
@@ skipping 666 matching lines @@
 
   rtc::CritScope stream_lock(&stream_crit_);
 
   if (!ValidateSendSsrcAvailability(sp))
     return false;
 
   for (uint32_t used_ssrc : sp.ssrcs)
     send_ssrcs_.insert(used_ssrc);
 
   webrtc::VideoSendStream::Config config(this);
-  config.overuse_callback = this;
-
-  WebRtcVideoSendStream* stream =
-      new WebRtcVideoSendStream(call_, sp, config, external_encoder_factory_,
-                                bitrate_config_.max_bitrate_bps, send_codec_,
-                                send_rtp_extensions_, send_params_);
+  WebRtcVideoSendStream* stream = new WebRtcVideoSendStream(
+      call_, sp, config, external_encoder_factory_, signal_cpu_adaptation_,
+      bitrate_config_.max_bitrate_bps, send_codec_, send_rtp_extensions_,
+      send_params_);
 
   uint32_t ssrc = sp.first_ssrc();
   RTC_DCHECK(ssrc != 0);
   send_streams_[ssrc] = stream;
 
   if (rtcp_receiver_report_ssrc_ == kDefaultRtcpReceiverReportSsrc) {
     rtcp_receiver_report_ssrc_ = ssrc;
     LOG(LS_INFO) << "SetLocalSsrc on all the receive streams because we added "
                     "a send stream.";
     for (auto& kv : receive_streams_)
@@ skipping 257 matching lines @@
     rtc::CritScope stream_lock(&stream_crit_);
     const auto& kv = send_streams_.find(ssrc);
     if (kv == send_streams_.end()) {
       LOG(LS_ERROR) << "No sending stream on ssrc " << ssrc;
       return false;
     }
     if (!kv->second->SetCapturer(capturer)) {
       return false;
     }
   }
-  {
-    rtc::CritScope lock(&capturer_crit_);
-    capturers_[ssrc] = capturer;
-  }
   return true;
 }
 
 void WebRtcVideoChannel2::OnPacketReceived(
     rtc::Buffer* packet,
     const rtc::PacketTime& packet_time) {
   const webrtc::PacketTime webrtc_packet_time(packet_time.timestamp,
                                               packet_time.not_before);
   const webrtc::PacketReceiver::DeliveryStatus delivery_result =
       call_->Receiver()->DeliverPacket(
@@ skipping 105 matching lines @@
 
   // Speculative change to increase the outbound socket buffer size.
   // In b/15152257, we are seeing a significant number of packets discarded
   // due to lack of socket buffer space, although it's not yet clear what the
   // ideal value should be.
   MediaChannel::SetOption(NetworkInterface::ST_RTP,
                           rtc::Socket::OPT_SNDBUF,
                           kVideoRtpBufferSize);
 }
 
-void WebRtcVideoChannel2::OnLoadUpdate(Load load) {
-  // OnLoadUpdate can not take any locks that are held while creating streams
-  // etc. Doing so establishes lock-order inversions between the webrtc process
-  // thread on stream creation and locks such as stream_crit_ while calling out.
-  rtc::CritScope stream_lock(&capturer_crit_);
-  if (!signal_cpu_adaptation_)
-    return;
-  // Do not adapt resolution for screen content as this will likely result in
-  // blurry and unreadable text.
-  for (auto& kv : capturers_) {
-    if (kv.second != nullptr
-        && !kv.second->IsScreencast()
-        && kv.second->video_adapter() != nullptr) {
-      kv.second->video_adapter()->OnCpuResolutionRequest(
-          load == kOveruse ? CoordinatedVideoAdapter::DOWNGRADE
-                           : CoordinatedVideoAdapter::UPGRADE);
-    }
-  }
-}
-
 bool WebRtcVideoChannel2::SendRtp(const uint8_t* data,
                                   size_t len,
                                   const webrtc::PacketOptions& options) {
   rtc::Buffer packet(data, len, kMaxRtpPacketLen);
   rtc::PacketOptions rtc_options;
   rtc_options.packet_id = options.packet_id;
   return MediaChannel::SendPacket(&packet, rtc_options);
 }
 
 bool WebRtcVideoChannel2::SendRtcp(const uint8_t* data, size_t len) {
@@ skipping 43 matching lines @@
     this->encoder =
         new webrtc::VideoEncoderSoftwareFallbackWrapper(type, encoder);
   }
 }
 
 WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream(
     webrtc::Call* call,
     const StreamParams& sp,
     const webrtc::VideoSendStream::Config& config,
     WebRtcVideoEncoderFactory* external_encoder_factory,
+    bool enable_cpu_overuse_detection,
     int max_bitrate_bps,
     const rtc::Optional<VideoCodecSettings>& codec_settings,
     const std::vector<webrtc::RtpExtension>& rtp_extensions,
     // TODO(deadbeef): Don't duplicate information between send_params,
     // rtp_extensions, options, etc.
     const VideoSendParameters& send_params)
-    : ssrcs_(sp.ssrcs),
+    : worker_thread_(rtc::Thread::Current()),
+      ssrcs_(sp.ssrcs),
       ssrc_groups_(sp.ssrc_groups),
       call_(call),
+      cpu_restricted_counter_(0),
+      number_of_cpu_adapt_changes_(0),
+      capturer_(nullptr),
       external_encoder_factory_(external_encoder_factory),
-      stream_(NULL),
+      stream_(nullptr),
       parameters_(config, send_params.options, max_bitrate_bps, codec_settings),
       pending_encoder_reconfiguration_(false),
-      allocated_encoder_(NULL, webrtc::kVideoCodecUnknown, false),
-      capturer_(NULL),
+      allocated_encoder_(nullptr, webrtc::kVideoCodecUnknown, false),
+      capturer_is_screencast_(false),
       sending_(false),
       muted_(false),
-      old_adapt_changes_(0),
       first_frame_timestamp_ms_(0),
       last_frame_timestamp_ms_(0) {
   parameters_.config.rtp.max_packet_size = kVideoMtu;
   parameters_.conference_mode = send_params.conference_mode;
 
   sp.GetPrimarySsrcs(&parameters_.config.rtp.ssrcs);
   sp.GetFidSsrcs(parameters_.config.rtp.ssrcs,
                  &parameters_.config.rtp.rtx.ssrcs);
   parameters_.config.rtp.c_name = sp.cname;
   parameters_.config.rtp.extensions = rtp_extensions;
   parameters_.config.rtp.rtcp_mode = send_params.rtcp.reduced_size
                                          ? webrtc::RtcpMode::kReducedSize
                                          : webrtc::RtcpMode::kCompound;
+  parameters_.config.overuse_callback =
+      enable_cpu_overuse_detection ? this : nullptr;
 
   if (codec_settings) {
     SetCodecAndOptions(*codec_settings, parameters_.options);
   }
 }
 
 WebRtcVideoChannel2::WebRtcVideoSendStream::~WebRtcVideoSendStream() {
   DisconnectCapturer();
   if (stream_ != NULL) {
     call_->DestroyVideoSendStream(stream_);
@@ skipping 43 matching lines @@
   int64_t frame_delta_ms = frame.GetTimeStamp() / rtc::kNumNanosecsPerMillisec;
   // frame->GetTimeStamp() is essentially a delta, align to webrtc time
   if (first_frame_timestamp_ms_ == 0) {
     first_frame_timestamp_ms_ = rtc::Time() - frame_delta_ms;
   }
 
   last_frame_timestamp_ms_ = first_frame_timestamp_ms_ + frame_delta_ms;
   video_frame.set_render_time_ms(last_frame_timestamp_ms_);
   // Reconfigure codec if necessary.
   SetDimensions(video_frame.width(), video_frame.height(),
-                capturer_->IsScreencast());
+                capturer_is_screencast_);
   last_rotation_ = video_frame.rotation();
 
   stream_->Input()->IncomingCapturedFrame(video_frame);
 }
 
 bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetCapturer(
     VideoCapturer* capturer) {
   TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::SetCapturer");
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
   if (!DisconnectCapturer() && capturer == NULL) {
     return false;
   }
 
   {
     rtc::CritScope cs(&lock_);
 
     // Reset timestamps to realign new incoming frames to a webrtc timestamp. A
     // new capturer may have a different timestamp delta than the previous one.
     first_frame_timestamp_ms_ = 0;
@@ skipping 12 matching lines @@
         // necessary to give this black frame a larger timestamp than the
         // previous one.
         last_frame_timestamp_ms_ += 1;
         black_frame.set_render_time_ms(last_frame_timestamp_ms_);
         stream_->Input()->IncomingCapturedFrame(black_frame);
       }
 
       capturer_ = NULL;
       return true;
     }
-
-    capturer_ = capturer;
-    capturer_->AddOrUpdateSink(this, sink_wants_);
+    capturer_is_screencast_ = capturer->IsScreencast();
   }
+  capturer_ = capturer;
+  capturer_->AddOrUpdateSink(this, sink_wants_);
   return true;
 }
 
 void WebRtcVideoChannel2::WebRtcVideoSendStream::MuteStream(bool mute) {
   rtc::CritScope cs(&lock_);
   muted_ = mute;
 }
 
 bool WebRtcVideoChannel2::WebRtcVideoSendStream::DisconnectCapturer() {
-  cricket::VideoCapturer* capturer;
-  {
-    rtc::CritScope cs(&lock_);
-    if (capturer_ == NULL)
-      return false;
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  if (capturer_ == NULL) {
+    return false;
+  }
 
-    if (capturer_->video_adapter() != nullptr)
-      old_adapt_changes_ += capturer_->video_adapter()->adaptation_changes();
-
-    capturer = capturer_;
-    capturer_ = NULL;
-  }
-  capturer->RemoveSink(this);
-
+  capturer_->RemoveSink(this);
+  capturer_ = NULL;
+  // Reset |cpu_restricted_counter_| if the capturer is changed. It is not
+  // possible to know if the video resolution is restricted by CPU usage after
+  // the capturer is changed since the next capturer might be screen capture
+  // with another resolution and frame rate.
+  cpu_restricted_counter_ = 0;
   return true;
 }
 
 const std::vector<uint32_t>&
 WebRtcVideoChannel2::WebRtcVideoSendStream::GetSsrcs() const {
   return ssrcs_;
 }
 
 void WebRtcVideoChannel2::WebRtcVideoSendStream::SetOptions(
     const VideoOptions& options) {
@@ skipping 145 matching lines @@
                        params.options ? *params.options : parameters_.options);
     return;
   } else if (params.options) {
     // Reconfigure if codecs are already set.
     if (parameters_.codec_settings) {
       SetCodecAndOptions(*parameters_.codec_settings, *params.options);
       return;
     } else {
       parameters_.options = *params.options;
     }
-  }
-  else if (params.conference_mode && parameters_.codec_settings) {
+  } else if (params.conference_mode && parameters_.codec_settings) {
     SetCodecAndOptions(*parameters_.codec_settings, parameters_.options);
     return;
   }
   if (recreate_stream) {
     LOG(LS_INFO) << "RecreateWebRtcStream (send) because of SetSendParameters";
     RecreateWebRtcStream();
   }
 }
 
 webrtc::VideoEncoderConfig
@@ skipping 106 matching lines @@
 }
 
 void WebRtcVideoChannel2::WebRtcVideoSendStream::Stop() {
   rtc::CritScope cs(&lock_);
   if (stream_ != NULL) {
     stream_->Stop();
   }
   sending_ = false;
 }
 
+void WebRtcVideoChannel2::WebRtcVideoSendStream::OnLoadUpdate(Load load) {
+  if (worker_thread_ != rtc::Thread::Current()) {
+    invoker_.AsyncInvoke<void>(
+        worker_thread_,
+        rtc::Bind(&WebRtcVideoChannel2::WebRtcVideoSendStream::OnLoadUpdate,
+                  this, load));
+    return;
+  }
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  LOG(LS_INFO) << "OnLoadUpdate " << load;
+  if (!capturer_) {
+    return;
+  }
+  {
+    rtc::CritScope cs(&lock_);
+    // Do not adapt resolution for screen content as this will likely result in
+    // blurry and unreadable text.
+    if (capturer_is_screencast_)
+      return;
+
+    rtc::Optional<int> max_pixel_count;
+    rtc::Optional<int> max_pixel_count_step_up;
+    if (load == kOveruse) {
+        max_pixel_count = rtc::Optional<int>(
+            (last_dimensions_.height * last_dimensions_.width) / 2);
+      // Increase |number_of_cpu_adapt_changes_| if
+      // sink_wants_.max_pixel_count will be changed since
+      // last time |capturer_->AddOrUpdateSink| was called. That is, this will
+      // result in a new request for the capturer to change resolution.
+      if (!sink_wants_.max_pixel_count ||
+          *sink_wants_.max_pixel_count > *max_pixel_count) {
+        ++number_of_cpu_adapt_changes_;
+        ++cpu_restricted_counter_;
+      }
+    } else {
+      RTC_DCHECK(load == kUnderuse);
+      max_pixel_count_step_up = rtc::Optional<int>(last_dimensions_.height *
+                                                   last_dimensions_.width);
+      // Increase |number_of_cpu_adapt_changes_| if
+      // sink_wants_.max_pixel_count_step_up will be changed since
+      // last time |capturer_->AddOrUpdateSink| was called. That is, this will
+      // result in a new request for the capturer to change resolution.
+      if (sink_wants_.max_pixel_count ||
+          (sink_wants_.max_pixel_count_step_up &&
+           *sink_wants_.max_pixel_count_step_up < *max_pixel_count_step_up)) {
+        ++number_of_cpu_adapt_changes_;
+        --cpu_restricted_counter_;
+      }
+    }
+    sink_wants_.max_pixel_count = max_pixel_count;
+    sink_wants_.max_pixel_count_step_up = max_pixel_count_step_up;
+  }
+  capturer_->AddOrUpdateSink(this, sink_wants_);
+}
+
 VideoSenderInfo
 WebRtcVideoChannel2::WebRtcVideoSendStream::GetVideoSenderInfo() {
   VideoSenderInfo info;
   webrtc::VideoSendStream::Stats stats;
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
   {
     rtc::CritScope cs(&lock_);
     for (uint32_t ssrc : parameters_.config.rtp.ssrcs)
       info.add_ssrc(ssrc);
 
     if (parameters_.codec_settings)
       info.codec_name = parameters_.codec_settings->codec.name;
     for (size_t i = 0; i < parameters_.encoder_config.streams.size(); ++i) {
       if (i == parameters_.encoder_config.streams.size() - 1) {
         info.preferred_bitrate +=
             parameters_.encoder_config.streams[i].max_bitrate_bps;
       } else {
         info.preferred_bitrate +=
             parameters_.encoder_config.streams[i].target_bitrate_bps;
       }
     }
 
     if (stream_ == NULL)
       return info;
 
     stats = stream_->GetStats();
+  }
+  info.adapt_changes = number_of_cpu_adapt_changes_;
+  info.adapt_reason = cpu_restricted_counter_ <= 0
+                          ? CoordinatedVideoAdapter::ADAPTREASON_NONE
+                          : CoordinatedVideoAdapter::ADAPTREASON_CPU;
 
-    info.adapt_changes = old_adapt_changes_;
-    info.adapt_reason = CoordinatedVideoAdapter::ADAPTREASON_NONE;
-
-    if (capturer_ != NULL) {
-      if (!capturer_->IsMuted()) {
-        VideoFormat last_captured_frame_format;
-        capturer_->GetStats(&info.adapt_frame_drops, &info.effects_frame_drops,
-                            &info.capturer_frame_time,
-                            &last_captured_frame_format);
-        info.input_frame_width = last_captured_frame_format.width;
-        info.input_frame_height = last_captured_frame_format.height;
-      }
-      if (capturer_->video_adapter() != nullptr) {
-        info.adapt_changes += capturer_->video_adapter()->adaptation_changes();
-        info.adapt_reason = capturer_->video_adapter()->adapt_reason();
-      }
+  if (capturer_) {
+    if (!capturer_->IsMuted()) {
+      VideoFormat last_captured_frame_format;
+      capturer_->GetStats(&info.adapt_frame_drops, &info.effects_frame_drops,
+                          &info.capturer_frame_time,
+                          &last_captured_frame_format);
+      info.input_frame_width = last_captured_frame_format.width;
+      info.input_frame_height = last_captured_frame_format.height;
     }
   }
 
   // Get bandwidth limitation info from stream_->GetStats().
   // Input resolution (output from video_adapter) can be further scaled down or
   // higher video layer(s) can be dropped due to bitrate constraints.
   // Note, adapt_changes only include changes from the video_adapter.
   if (stats.bw_limited_resolution)
     info.adapt_reason |= CoordinatedVideoAdapter::ADAPTREASON_BANDWIDTH;
 
@@ skipping 499 matching lines @@
         rtx_mapping[video_codecs[i].codec.id] !=
             fec_settings.red_payload_type) {
       video_codecs[i].rtx_payload_type = rtx_mapping[video_codecs[i].codec.id];
     }
   }
 
   return video_codecs;
 }
 
 }  // namespace cricket