Move direct use of VideoCapturer::VideoAdapter to VideoSinkWants.

webrtc/media/engine/webrtcvideoengine2.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/media/engine/webrtcvideoengine2.h"
 
 #include <algorithm>
 #include <set>
 #include <string>
 
+#include "webrtc/base/asyncinvoker.h"
 #include "webrtc/base/buffer.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/stringutils.h"
 #include "webrtc/base/timeutils.h"
 #include "webrtc/base/trace_event.h"
 #include "webrtc/call.h"
 #include "webrtc/media/base/videocapturer.h"
 #include "webrtc/media/base/videorenderer.h"
 #include "webrtc/media/engine/constants.h"
 #include "webrtc/media/engine/simulcast.h"
@@ skipping 280 matching lines @@
   if (width * height <= 320 * 240) {
     return 600;
   } else if (width * height <= 640 * 480) {
     return 1700;
   } else if (width * height <= 960 * 540) {
     return 2000;
   } else {
     return 2500;
   }
 }
+
 }  // namespace
 
 // Constants defined in webrtc/media/engine/constants.h
 // TODO(pbos): Move these to a separate constants.cc file.
 const int kMinVideoBitrate = 30;
 const int kStartVideoBitrate = 300;
 
 const int kVideoMtu = 1200;
 const int kVideoRtpBufferSize = 65536;
 
@@ skipping 666 matching lines @@
 
   rtc::CritScope stream_lock(&stream_crit_);
 
   if (!ValidateSendSsrcAvailability(sp))
     return false;
 
   for (uint32_t used_ssrc : sp.ssrcs)
     send_ssrcs_.insert(used_ssrc);
 
   webrtc::VideoSendStream::Config config(this);
-  config.overuse_callback = this;
-
-  WebRtcVideoSendStream* stream =
-      new WebRtcVideoSendStream(call_, sp, config, external_encoder_factory_,
-                                bitrate_config_.max_bitrate_bps, send_codec_,
-                                send_rtp_extensions_, send_params_);
+  WebRtcVideoSendStream* stream = new WebRtcVideoSendStream(
+      call_, sp, config, external_encoder_factory_, signal_cpu_adaptation_,
+      bitrate_config_.max_bitrate_bps, send_codec_, send_rtp_extensions_,
+      send_params_);
 
   uint32_t ssrc = sp.first_ssrc();
   RTC_DCHECK(ssrc != 0);
   send_streams_[ssrc] = stream;
 
   if (rtcp_receiver_report_ssrc_ == kDefaultRtcpReceiverReportSsrc) {
     rtcp_receiver_report_ssrc_ = ssrc;
     LOG(LS_INFO) << "SetLocalSsrc on all the receive streams because we added "
                     "a send stream.";
     for (auto& kv : receive_streams_)
@@ skipping 386 matching lines @@
 
   // Speculative change to increase the outbound socket buffer size.
   // In b/15152257, we are seeing a significant number of packets discarded
   // due to lack of socket buffer space, although it's not yet clear what the
   // ideal value should be.
   MediaChannel::SetOption(NetworkInterface::ST_RTP,
                           rtc::Socket::OPT_SNDBUF,
                           kVideoRtpBufferSize);
 }
 
-void WebRtcVideoChannel2::OnLoadUpdate(Load load) {
-  // OnLoadUpdate can not take any locks that are held while creating streams
-  // etc. Doing so establishes lock-order inversions between the webrtc process
-  // thread on stream creation and locks such as stream_crit_ while calling out.
-  rtc::CritScope stream_lock(&capturer_crit_);
-  if (!signal_cpu_adaptation_)
-    return;
-  // Do not adapt resolution for screen content as this will likely result in
-  // blurry and unreadable text.
-  for (auto& kv : capturers_) {
-    if (kv.second != nullptr
-        && !kv.second->IsScreencast()
-        && kv.second->video_adapter() != nullptr) {
-      kv.second->video_adapter()->OnCpuResolutionRequest(
-          load == kOveruse ? CoordinatedVideoAdapter::DOWNGRADE
-                           : CoordinatedVideoAdapter::UPGRADE);
-    }
-  }
-}
-
 bool WebRtcVideoChannel2::SendRtp(const uint8_t* data,
                                   size_t len,
                                   const webrtc::PacketOptions& options) {
   rtc::Buffer packet(data, len, kMaxRtpPacketLen);
   rtc::PacketOptions rtc_options;
   rtc_options.packet_id = options.packet_id;
   return MediaChannel::SendPacket(&packet, rtc_options);
 }
 
 bool WebRtcVideoChannel2::SendRtcp(const uint8_t* data, size_t len) {
@@ skipping 38 matching lines @@
       external_encoder(nullptr),
       type(type),
       external(external) {
   if (external) {
     external_encoder = encoder;
     this->encoder =
         new webrtc::VideoEncoderSoftwareFallbackWrapper(type, encoder);
   }
 }
 
+// Proxy class used for marshalling calls to webrtc::LoadObserver::OnLoadUpdate
+// from a media engine thread to the worker thread.
+class WebRtcVideoChannel2::WebRtcVideoSendStream::LoadObserverProxy {

[pthatcher1, 2016/02/25 07:40:30]

This still looks too complex to me, but maybe I'm just not smart enough to
understand it.

[perkj_webrtc, 2016/02/25 13:57:07]

I am sure that is not true.
But I am not sure how do do this more to your liking. I think we need to post
async to the worker thread. We can have SendStream (or this proxy) inherit
MessageHandlerand do a normal Post directly in OnloadUpdate I assume and use
Thread::Clear to remove anny unhandled messages on destruction. ie- no ref
counting and no asyncinvoker.

+ public:
+  explicit LoadObserverProxy(webrtc::LoadObserver* observer) {
+    helper_ = new rtc::RefCountedObject<Helper>(observer);
+  }
+  ~LoadObserverProxy() { helper_->Detach(); }
+
+  webrtc::LoadObserver* proxy() { return helper_; }
+
+ private:
+  class Helper : public webrtc::LoadObserver, public rtc::RefCountInterface {
+   public:
+    explicit Helper(webrtc::LoadObserver* observer)
+        : thread_(rtc::Thread::Current()), observer_(observer) {}
+    void Detach() {
+      RTC_DCHECK(thread_checker_.CalledOnValidThread());
+      observer_ = nullptr;
+    }
+    void OnLoadUpdate(webrtc::LoadObserver::Load load) override {
+      if (rtc::Thread::Current() == thread_) {
+        observer_->OnLoadUpdate(load);
+        return;
+      }
+      invoker_.AsyncInvoke<void>(
+          thread_, rtc::Bind(&Helper::OnLoadUpdateOnCorrectThread, this, load));
+    }
+    void OnLoadUpdateOnCorrectThread(webrtc::LoadObserver::Load load) {
+      RTC_DCHECK(thread_checker_.CalledOnValidThread());
+      if (observer_) {
+        observer_->OnLoadUpdate(load);
+      }
+    }
+
+   private:
+    rtc::ThreadChecker thread_checker_;
+    rtc::AsyncInvoker invoker_;
+    rtc::Thread* thread_;
+    webrtc::LoadObserver* observer_;
+  };
+
+  rtc::scoped_refptr<Helper> helper_;
+};
+
 WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream(
     webrtc::Call* call,
     const StreamParams& sp,
     const webrtc::VideoSendStream::Config& config,
     WebRtcVideoEncoderFactory* external_encoder_factory,
+    bool enable_cpu_overuse_detection,
     int max_bitrate_bps,
     const rtc::Optional<VideoCodecSettings>& codec_settings,
     const std::vector<webrtc::RtpExtension>& rtp_extensions,
     // TODO(deadbeef): Don't duplicate information between send_params,
     // rtp_extensions, options, etc.
     const VideoSendParameters& send_params)
     : ssrcs_(sp.ssrcs),
       ssrc_groups_(sp.ssrc_groups),
       call_(call),
+      cpu_adapted_(false),
+      number_of_cpu_adapt_changes_(0),
+      load_proxy_(new LoadObserverProxy(this)),
       external_encoder_factory_(external_encoder_factory),
       stream_(NULL),
       parameters_(config, send_params.options, max_bitrate_bps, codec_settings),
       pending_encoder_reconfiguration_(false),
       allocated_encoder_(NULL, webrtc::kVideoCodecUnknown, false),
       capturer_(NULL),
       sending_(false),
       muted_(false),
-      old_adapt_changes_(0),
       first_frame_timestamp_ms_(0),
       last_frame_timestamp_ms_(0) {
   parameters_.config.rtp.max_packet_size = kVideoMtu;
   parameters_.conference_mode = send_params.conference_mode;
 
   sp.GetPrimarySsrcs(&parameters_.config.rtp.ssrcs);
   sp.GetFidSsrcs(parameters_.config.rtp.ssrcs,
                  &parameters_.config.rtp.rtx.ssrcs);
   parameters_.config.rtp.c_name = sp.cname;
   parameters_.config.rtp.extensions = rtp_extensions;
   parameters_.config.rtp.rtcp_mode = send_params.rtcp.reduced_size
                                          ? webrtc::RtcpMode::kReducedSize
                                          : webrtc::RtcpMode::kCompound;
+  parameters_.config.overuse_callback =
+      enable_cpu_overuse_detection ? load_proxy_->proxy() : nullptr;
 
   if (codec_settings) {
     SetCodecAndOptions(*codec_settings, parameters_.options);
   }
 }
 
 WebRtcVideoChannel2::WebRtcVideoSendStream::~WebRtcVideoSendStream() {
   DisconnectCapturer();
   if (stream_ != NULL) {
     call_->DestroyVideoSendStream(stream_);
@@ skipping 103 matching lines @@
   muted_ = mute;
 }
 
 bool WebRtcVideoChannel2::WebRtcVideoSendStream::DisconnectCapturer() {
   cricket::VideoCapturer* capturer;
   {
     rtc::CritScope cs(&lock_);
     if (capturer_ == NULL)
       return false;
 
-    if (capturer_->video_adapter() != nullptr)
-      old_adapt_changes_ += capturer_->video_adapter()->adaptation_changes();
-
     capturer = capturer_;
     capturer_ = NULL;
   }
   capturer->RemoveSink(this);
 
   return true;
 }
 
 const std::vector<uint32_t>&
 WebRtcVideoChannel2::WebRtcVideoSendStream::GetSsrcs() const {
@@ skipping 150 matching lines @@
                        params.options ? *params.options : parameters_.options);
     return;
   } else if (params.options) {
     // Reconfigure if codecs are already set.
     if (parameters_.codec_settings) {
       SetCodecAndOptions(*parameters_.codec_settings, *params.options);
       return;
     } else {
       parameters_.options = *params.options;
     }
-  }
-  else if (params.conference_mode && parameters_.codec_settings) {
+  } else if (params.conference_mode && parameters_.codec_settings) {
     SetCodecAndOptions(*parameters_.codec_settings, parameters_.options);
     return;
   }
   if (recreate_stream) {
     LOG(LS_INFO) << "RecreateWebRtcStream (send) because of SetSendParameters";
     RecreateWebRtcStream();
   }
 }
 
 webrtc::VideoEncoderConfig
@@ skipping 106 matching lines @@
 }
 
 void WebRtcVideoChannel2::WebRtcVideoSendStream::Stop() {
   rtc::CritScope cs(&lock_);
   if (stream_ != NULL) {
     stream_->Stop();
   }
   sending_ = false;
 }
 
+void WebRtcVideoChannel2::WebRtcVideoSendStream::OnLoadUpdate(Load load) {
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  LOG(LS_INFO) << "OnLoadUpdate " << load;
+  rtc::CritScope cs(&lock_);
+  if (!capturer_) {
+    return;
+  }
+
+  if (load == kOveruse) {
+    rtc::Optional<int> max_pixel_count(
+        (last_dimensions_.height * last_dimensions_.width) / 2);
+    if (!sink_wants_.max_pixel_count ||
+        *sink_wants_.max_pixel_count != *max_pixel_count) {
+      ++number_of_cpu_adapt_changes_;
+      cpu_adapted_ = true;
+    }
+    sink_wants_.max_pixel_count = rtc::Optional<int>(
+        last_dimensions_.height * last_dimensions_.width / 2);

[pthatcher1, 2016/02/25 07:40:30]

Why can't use you just do:

sink_wants_.max_pixel_count = max_pixel_count;

on this line?

[pthatcher1, 2016/02/25 07:40:30]

Shouldn't we clear the sink_wants_.max_pixel_count_step_up here?

[perkj_webrtc, 2016/02/25 13:57:07]

Done.

+  } else {
+    RTC_DCHECK(load == kUnderuse);
+    cpu_adapted_ = false;

[pthatcher1, 2016/02/25 07:40:30]

It's still cpu_adapted sometimes, right?  Just slightly less?  Wouldn't we have
to clear both max_pixel_counts to be sure it's not adapted any more?

[perkj_webrtc, 2016/02/25 13:57:07]

Yes. This is just used for the info.adapt_reason stats. 
I can count the number of ups / downs and that might be slightly better. The
problem might be screen share- that we don't adapt anyway.... Ie, say we adapt
down twice and then up once and then the screen share window is enlarged outside
our control and we don't get any more OnLoadUpdate. Are we then cpu constrained
or not? 

[pthatcher1, 2016/02/25 20:20:05]

It seems tricky.  Only the Source really knows if it constrained or not.  But it
doesn't know why (different sinks could have different reasons).  

The real question is: how does the source know when it's safe to remove the
max_pixel_count_step_up?  It would have to know that a given VideoFrame is one
step up from that value set.  Then it could clear it.  Could it assume so as
soon as it gets a frame that is larger than max_pixel_count_step_up?

[perkj_webrtc, 2016/02/26 14:08:25]

Can't the source just ignore max_pixel_value_count_step_up if it produce a
resolution larger than this value? If the sink wanted a max it should set
max_resolution_count. 

+    rtc::Optional<int> pixel_count_next_step_larger_than(
+        last_dimensions_.height * last_dimensions_.width);

[pthatcher1, 2016/02/25 07:40:30]

Why not just call this max_pixel_count_step_up?

[perkj_webrtc, 2016/02/25 13:57:07]

Done.

+    if (sink_wants_.max_pixel_count ||
+        (sink_wants_.max_pixel_count_step_up &&
+         *sink_wants_.max_pixel_count_step_up !=
+             *pixel_count_next_step_larger_than)) {
+      ++number_of_cpu_adapt_changes_;
+    }
+
+    sink_wants_.max_pixel_count_step_up =
+        pixel_count_next_step_larger_than;
+    sink_wants_.max_pixel_count = rtc::Optional<int>();
+  }
+  capturer_->AddOrUpdateSink(this, sink_wants_);
+}
+
 VideoSenderInfo
 WebRtcVideoChannel2::WebRtcVideoSendStream::GetVideoSenderInfo() {
   VideoSenderInfo info;
   webrtc::VideoSendStream::Stats stats;
   {
     rtc::CritScope cs(&lock_);
     for (uint32_t ssrc : parameters_.config.rtp.ssrcs)
       info.add_ssrc(ssrc);
 
     if (parameters_.codec_settings)
       info.codec_name = parameters_.codec_settings->codec.name;
     for (size_t i = 0; i < parameters_.encoder_config.streams.size(); ++i) {
       if (i == parameters_.encoder_config.streams.size() - 1) {
         info.preferred_bitrate +=
             parameters_.encoder_config.streams[i].max_bitrate_bps;
       } else {
         info.preferred_bitrate +=
             parameters_.encoder_config.streams[i].target_bitrate_bps;
       }
     }
 
     if (stream_ == NULL)
       return info;
 
     stats = stream_->GetStats();
-
-    info.adapt_changes = old_adapt_changes_;
-    info.adapt_reason = CoordinatedVideoAdapter::ADAPTREASON_NONE;
+    info.adapt_changes = number_of_cpu_adapt_changes_;
+    info.adapt_reason = cpu_adapted_
+                            ? CoordinatedVideoAdapter::ADAPTREASON_CPU
+                            : CoordinatedVideoAdapter::ADAPTREASON_NONE;
 
     if (capturer_ != NULL) {
       if (!capturer_->IsMuted()) {
         VideoFormat last_captured_frame_format;
         capturer_->GetStats(&info.adapt_frame_drops, &info.effects_frame_drops,
                             &info.capturer_frame_time,
                             &last_captured_frame_format);
         info.input_frame_width = last_captured_frame_format.width;
         info.input_frame_height = last_captured_frame_format.height;
       }
-      if (capturer_->video_adapter() != nullptr) {
-        info.adapt_changes += capturer_->video_adapter()->adaptation_changes();
-        info.adapt_reason = capturer_->video_adapter()->adapt_reason();
-      }
     }
   }
 
   // Get bandwidth limitation info from stream_->GetStats().
   // Input resolution (output from video_adapter) can be further scaled down or
   // higher video layer(s) can be dropped due to bitrate constraints.
   // Note, adapt_changes only include changes from the video_adapter.
   if (stats.bw_limited_resolution)
     info.adapt_reason |= CoordinatedVideoAdapter::ADAPTREASON_BANDWIDTH;
 
@@ skipping 499 matching lines @@
         rtx_mapping[video_codecs[i].codec.id] !=
             fec_settings.red_payload_type) {
       video_codecs[i].rtx_payload_type = rtx_mapping[video_codecs[i].codec.id];
     }
   }
 
   return video_codecs;
 }
 
 }  // namespace cricket