Move direct use of VideoCapturer::VideoAdapter to VideoSinkWants.

webrtc/media/engine/webrtcvideoengine2.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/media/engine/webrtcvideoengine2.h"
 
 #include <algorithm>
 #include <set>
 #include <string>
 
+#include "webrtc/base/asyncinvoker.h"
 #include "webrtc/base/buffer.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/stringutils.h"
 #include "webrtc/base/timeutils.h"
 #include "webrtc/base/trace_event.h"
 #include "webrtc/call.h"
 #include "webrtc/media/base/videocapturer.h"
 #include "webrtc/media/base/videorenderer.h"
 #include "webrtc/media/engine/constants.h"
 #include "webrtc/media/engine/simulcast.h"
@@ skipping 280 matching lines @@
   if (width * height <= 320 * 240) {
     return 600;
   } else if (width * height <= 640 * 480) {
     return 1700;
   } else if (width * height <= 960 * 540) {
     return 2000;
   } else {
     return 2500;
   }
 }
+
 }  // namespace
 
 // Constants defined in webrtc/media/engine/constants.h
 // TODO(pbos): Move these to a separate constants.cc file.
 const int kMinVideoBitrate = 30;
 const int kStartVideoBitrate = 300;
 
 const int kVideoMtu = 1200;
 const int kVideoRtpBufferSize = 65536;
 
@@ skipping 660 matching lines @@
 
   rtc::CritScope stream_lock(&stream_crit_);
 
   if (!ValidateSendSsrcAvailability(sp))
     return false;
 
   for (uint32_t used_ssrc : sp.ssrcs)
     send_ssrcs_.insert(used_ssrc);
 
   webrtc::VideoSendStream::Config config(this);
-  config.overuse_callback = this;
-
   WebRtcVideoSendStream* stream = new WebRtcVideoSendStream(
       call_, sp, config, external_encoder_factory_, options_,
-      bitrate_config_.max_bitrate_bps, send_codec_, send_rtp_extensions_,
-      send_params_);
-
+      signal_cpu_adaptation_, bitrate_config_.max_bitrate_bps, send_codec_,
+      send_rtp_extensions_, send_params_);
   uint32_t ssrc = sp.first_ssrc();
   RTC_DCHECK(ssrc != 0);
   send_streams_[ssrc] = stream;
 
   if (rtcp_receiver_report_ssrc_ == kDefaultRtcpReceiverReportSsrc) {
     rtcp_receiver_report_ssrc_ = ssrc;
     LOG(LS_INFO) << "SetLocalSsrc on all the receive streams because we added "
                     "a send stream.";
     for (auto& kv : receive_streams_)
       kv.second->SetLocalSsrc(ssrc);
@@ skipping 377 matching lines @@
 
   // Speculative change to increase the outbound socket buffer size.
   // In b/15152257, we are seeing a significant number of packets discarded
   // due to lack of socket buffer space, although it's not yet clear what the
   // ideal value should be.
   MediaChannel::SetOption(NetworkInterface::ST_RTP,
                           rtc::Socket::OPT_SNDBUF,
                           kVideoRtpBufferSize);
 }
 
-void WebRtcVideoChannel2::OnLoadUpdate(Load load) {
-  // OnLoadUpdate can not take any locks that are held while creating streams
-  // etc. Doing so establishes lock-order inversions between the webrtc process
-  // thread on stream creation and locks such as stream_crit_ while calling out.
-  rtc::CritScope stream_lock(&capturer_crit_);
-  if (!signal_cpu_adaptation_)
-    return;
-  // Do not adapt resolution for screen content as this will likely result in
-  // blurry and unreadable text.
-  for (auto& kv : capturers_) {
-    if (kv.second != nullptr
-        && !kv.second->IsScreencast()
-        && kv.second->video_adapter() != nullptr) {
-      kv.second->video_adapter()->OnCpuResolutionRequest(
-          load == kOveruse ? CoordinatedVideoAdapter::DOWNGRADE
-                           : CoordinatedVideoAdapter::UPGRADE);
-    }
-  }
-}
-
 bool WebRtcVideoChannel2::SendRtp(const uint8_t* data,
                                   size_t len,
                                   const webrtc::PacketOptions& options) {
   rtc::Buffer packet(data, len, kMaxRtpPacketLen);
   rtc::PacketOptions rtc_options;
   rtc_options.packet_id = options.packet_id;
   return MediaChannel::SendPacket(&packet, rtc_options);
 }
 
 bool WebRtcVideoChannel2::SendRtcp(const uint8_t* data, size_t len) {
@@ skipping 38 matching lines @@
       external_encoder(nullptr),
       type(type),
       external(external) {
   if (external) {
     external_encoder = encoder;
     this->encoder =
         new webrtc::VideoEncoderSoftwareFallbackWrapper(type, encoder);
   }
 }
 
+// Proxy class used for marshalling calls to webrtc::LoadObserver::OnLoadUpdate
+// from a media engine thread to the worker thread.
+class WebRtcVideoChannel2::WebRtcVideoSendStream::LoadObserverProxy {
+ public:
+  explicit LoadObserverProxy(webrtc::LoadObserver* observer) {
+    helper_ = new rtc::RefCountedObject<Helper>(observer);
+  }
+  ~LoadObserverProxy() { helper_->Detach(); }
+
+  webrtc::LoadObserver* proxy() { return helper_; }
+
+ private:
+  class Helper : public webrtc::LoadObserver, public rtc::RefCountInterface {
+   public:
+    explicit Helper(webrtc::LoadObserver* observer)
+        : thread_(rtc::Thread::Current()), observer_(observer) {}
+    void Detach() {
+      RTC_DCHECK(thread_checker_.CalledOnValidThread());
+      observer_ = nullptr;
+    }
+    void OnLoadUpdate(webrtc::LoadObserver::Load load) override {
+      if (rtc::Thread::Current() == thread_) {
+        observer_->OnLoadUpdate(load);
+        return;
+      }
+      invoker_.AsyncInvoke<void>(
+          thread_, rtc::Bind(&Helper::OnLoadUpdateOnCorrectThread, this, load));
+    }
+    void OnLoadUpdateOnCorrectThread(webrtc::LoadObserver::Load load) {
+      RTC_DCHECK(thread_checker_.CalledOnValidThread());
+      if (observer_) {
+        observer_->OnLoadUpdate(load);
+      }
+    }
+
+   private:
+    rtc::ThreadChecker thread_checker_;
+    rtc::AsyncInvoker invoker_;
+    rtc::Thread* thread_;
+    webrtc::LoadObserver* observer_;
+  };
+
+  rtc::scoped_refptr<Helper> helper_;
+};
+
 WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream(
     webrtc::Call* call,
     const StreamParams& sp,
     const webrtc::VideoSendStream::Config& config,
     WebRtcVideoEncoderFactory* external_encoder_factory,
     const VideoOptions& options,
+    bool enable_cpu_overuse_detection,
     int max_bitrate_bps,
     const rtc::Optional<VideoCodecSettings>& codec_settings,
     const std::vector<webrtc::RtpExtension>& rtp_extensions,
     // TODO(deadbeef): Don't duplicate information between send_params,
     // rtp_extensions, options, etc.
     const VideoSendParameters& send_params)
     : ssrcs_(sp.ssrcs),
       ssrc_groups_(sp.ssrc_groups),
       call_(call),
+      load_proxy_(new LoadObserverProxy(this)),
       external_encoder_factory_(external_encoder_factory),
       stream_(NULL),
       parameters_(config, options, max_bitrate_bps, codec_settings),
       pending_encoder_reconfiguration_(false),
       allocated_encoder_(NULL, webrtc::kVideoCodecUnknown, false),
       capturer_(NULL),
       sending_(false),
       muted_(false),
       old_adapt_changes_(0),
       first_frame_timestamp_ms_(0),
       last_frame_timestamp_ms_(0) {
   parameters_.config.rtp.max_packet_size = kVideoMtu;
 
   sp.GetPrimarySsrcs(&parameters_.config.rtp.ssrcs);
   sp.GetFidSsrcs(parameters_.config.rtp.ssrcs,
                  &parameters_.config.rtp.rtx.ssrcs);
   parameters_.config.rtp.c_name = sp.cname;
   parameters_.config.rtp.extensions = rtp_extensions;
   parameters_.config.rtp.rtcp_mode = send_params.rtcp.reduced_size
                                          ? webrtc::RtcpMode::kReducedSize
                                          : webrtc::RtcpMode::kCompound;
+  parameters_.config.overuse_callback =
+      enable_cpu_overuse_detection ? load_proxy_->proxy() : nullptr;
 
   if (codec_settings) {
     SetCodecAndOptions(*codec_settings, parameters_.options);
   }
 }
 
 WebRtcVideoChannel2::WebRtcVideoSendStream::~WebRtcVideoSendStream() {
   DisconnectCapturer();
   if (stream_ != NULL) {
     call_->DestroyVideoSendStream(stream_);
@@ skipping 103 matching lines @@
   muted_ = mute;
 }
 
 bool WebRtcVideoChannel2::WebRtcVideoSendStream::DisconnectCapturer() {
   cricket::VideoCapturer* capturer;
   {
     rtc::CritScope cs(&lock_);
     if (capturer_ == NULL)
       return false;
 
-    if (capturer_->video_adapter() != nullptr)
-      old_adapt_changes_ += capturer_->video_adapter()->adaptation_changes();
-
     capturer = capturer_;
     capturer_ = NULL;
   }
   capturer->RemoveSink(this);
 
   return true;
 }
 
 const std::vector<uint32_t>&
 WebRtcVideoChannel2::WebRtcVideoSendStream::GetSsrcs() const {
@@ skipping 271 matching lines @@
 }
 
 void WebRtcVideoChannel2::WebRtcVideoSendStream::Stop() {
   rtc::CritScope cs(&lock_);
   if (stream_ != NULL) {
     stream_->Stop();
   }
   sending_ = false;
 }
 
+void WebRtcVideoChannel2::WebRtcVideoSendStream::OnLoadUpdate(Load load) {
+  RTC_DCHECK(thread_checker_.CalledOnValidThread());
+  LOG(LS_INFO) << "OnLoadUpdate " << load;
+  rtc::CritScope cs(&lock_);
+  if (!capturer_) {
+    return;
+  }
+
+  rtc::VideoSinkWants::ResolutionRequest request;
+  request.request = load == kOveruse
+                        ? rtc::VideoSinkWants::ResolutionRequest::kSmallerThan
+                        : rtc::VideoSinkWants::ResolutionRequest::kLargerThan;
+  request.seen_number_of_pixels =
+      last_dimensions_.height * last_dimensions_.width;
+
+  sink_wants_.resolution =
+      rtc::Optional<rtc::VideoSinkWants::ResolutionRequest>(request);

[pthatcher1, 2016/02/19 06:08:40]

I think I liked what we had better before :(.  This just seems too complex.

But let's step back for a moment an think of what we're trying to accomplish. 
We want the following:

1.  When we detect CPU overuse at resolution X, start scaling down to resolution
X/A.  This can happen multiple times down to some limit of X/Z.

2.  When we detect CPU underuse at resolution X/B, scale down less, to only X/A.
 This can happen multiple times all the way up to X/1 == X.

3.  If the video source changes resolution (and perhaps aspect ratio) on its
own, from X to Y, adjust the scale factor from A to B so that we have X/A =~
Y/B.  If Y > X, then B > A.  If Y < X, then B < A.

Once we know the scale factor A, then the scaler has a really easy job: just
scale with that factor.


The simplest thing would be for the encoder sink to tell the source "I want
frames scaled down by factor A".  When overuse is detected, we scale down
further (to B > A).  When underuse is detected, we scale down less (to B < A). 
The video broadcaster simply chooses the sink with the largest factor.  That is
simple and easy.  

However, it has the downside that when the video source changes resolution from
X to Y, the frame the encoder sink sees will change from X/A to Y/A.  Then we'll
have to wait until another overuse/underuse cycle to change Y/A to Y/B, which
would be undesirable.

A slight tweak to this would be for the encoder sink to tell the source "I want
frames scaled down to X/A".  When overuse is detected, we scale down further (to
X/B < X/A).  When underuse is detected, we scale down less (to X/B > X/A).  The
video broadcaster simply chooses the sink with the smallest scaled resolution. 
That is also simple and easy.

However, it has the upside that when the video source changes resolutions from X
to Y, the source can say to itself "I should scale down to Y/B == X/A".  Then
the encoder sink will see the resolution Y/B, which should be about the same as
it wanted with X/A.  And if it sees underuse or overuse again, it will tell the
source "I want to see Z/C".


And thus the big question is: how do we represent X/A?  And here are the
options:

A.  As scale factor A or B or C.  Really easy, but doesn't react well to
changing resolutions.  However, because we don't adapt for screencast and we
also change the source object when we switch cameras, I think we don't have a
current use case for this.  So, it would meet our current needs and be very
simple.

B.  As pixel_count(X)/A, or scaled pixel count, or max_pixel_count.   This is
what you had before, and now that I think about it more, I think your code you
had before is correct, and pretty simple.   I'd like to hear pbos's objections
to it.  Like I said, at first glance, it seems better (much more simple) than
this updated patch.

C.  As pixel_count + direction request, which you have here. This model seems
more complex because it puts the source in charge of how much to go up or down
and it makes the aggregation of wants so much more complicated.  Can someone
explain to me the advantage of option C over option B?

By the way, I think this is equivalent to having two values in SinkWants:
less_than_pixel_count and more_than_pixel_count, which I'd greatly prefer over
the pixel_count + direction if we go with C.   

D.  As resolution + scale.  Basically "if the resolution is X, scale down by A".
 Then it's clear what the source should do (scale down by A), and if it does
happen to change resolution (if we ever adapt for screencast, for example), then
the source can figure out what to do.  The aggregation might still be a bit
complex, though.


I think I prefer B > A > D > C.

+  capturer_->AddOrUpdateSink(this, sink_wants_);
+}
+
 VideoSenderInfo
 WebRtcVideoChannel2::WebRtcVideoSendStream::GetVideoSenderInfo() {
   VideoSenderInfo info;
   webrtc::VideoSendStream::Stats stats;
   {
     rtc::CritScope cs(&lock_);
     for (uint32_t ssrc : parameters_.config.rtp.ssrcs)
       info.add_ssrc(ssrc);
 
     if (parameters_.codec_settings)
@@ skipping 18 matching lines @@
 
     if (capturer_ != NULL) {
       if (!capturer_->IsMuted()) {
         VideoFormat last_captured_frame_format;
         capturer_->GetStats(&info.adapt_frame_drops, &info.effects_frame_drops,
                             &info.capturer_frame_time,
                             &last_captured_frame_format);
         info.input_frame_width = last_captured_frame_format.width;
         info.input_frame_height = last_captured_frame_format.height;
       }
-      if (capturer_->video_adapter() != nullptr) {
-        info.adapt_changes += capturer_->video_adapter()->adaptation_changes();
-        info.adapt_reason = capturer_->video_adapter()->adapt_reason();
-      }
     }
   }
 
   // Get bandwidth limitation info from stream_->GetStats().
   // Input resolution (output from video_adapter) can be further scaled down or
   // higher video layer(s) can be dropped due to bitrate constraints.
   // Note, adapt_changes only include changes from the video_adapter.
   if (stats.bw_limited_resolution)
     info.adapt_reason |= CoordinatedVideoAdapter::ADAPTREASON_BANDWIDTH;
 
@@ skipping 499 matching lines @@
         rtx_mapping[video_codecs[i].codec.id] !=
             fec_settings.red_payload_type) {
       video_codecs[i].rtx_payload_type = rtx_mapping[video_codecs[i].codec.id];
     }
   }
 
   return video_codecs;
 }
 
 }  // namespace cricket