Add framerate to VideoSinkWants and ability to signal on overuse

webrtc/video/vie_encoder.h

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #ifndef WEBRTC_VIDEO_VIE_ENCODER_H_
 #define WEBRTC_VIDEO_VIE_ENCODER_H_
 
+#include <map>
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "webrtc/api/video/video_rotation.h"
 #include "webrtc/base/criticalsection.h"
 #include "webrtc/base/event.h"
 #include "webrtc/base/sequenced_task_checker.h"
 #include "webrtc/base/task_queue.h"
 #include "webrtc/call/call.h"
@@ skipping 31 matching lines @@
   // Interface for receiving encoded video frames and notifications about
   // configuration changes.
   class EncoderSink : public EncodedImageCallback {
    public:
     virtual void OnEncoderConfigurationChanged(
         std::vector<VideoStream> streams,
         int min_transmit_bitrate_bps) = 0;
   };
 
   // Downscale resolution at most 2 times for CPU reasons.
-  static const int kMaxCpuDowngrades = 2;
+  static const int kMaxCpuResolutionDowngrades = 2;
+  // Downscale framerate at most 4 times.
+  static const int kMaxCpuFramerateDowngrades = 4;
 
   ViEEncoder(uint32_t number_of_cores,
              SendStatisticsProxy* stats_proxy,
              const VideoSendStream::Config::EncoderSettings& settings,
              rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback,
              EncodedFrameObserver* encoder_timing);
   ~ViEEncoder();
   // RegisterProcessThread register |module_process_thread| with those objects
   // that use it. Registration has to happen on the thread where
   // |module_process_thread| was created (libjingle's worker thread).
@@ skipping 91 matching lines @@
       const EncodedImage& encoded_image,
       const CodecSpecificInfo* codec_specific_info,
       const RTPFragmentationHeader* fragmentation) override;
 
   void OnDroppedFrame() override;
 
   bool EncoderPaused() const;
   void TraceFrameDropStart();
   void TraceFrameDropEnd();
 
+  std::vector<int>* GetScaleCounters()
+      EXCLUSIVE_LOCKS_REQUIRED(&encoder_queue_);
+
   rtc::Event shutdown_event_;
 
   const uint32_t number_of_cores_;
   // Counts how many frames we've dropped in the initial rampup phase.
   int initial_rampup_;
 
   const std::unique_ptr<VideoSourceProxy> source_proxy_;
   EncoderSink* sink_;
   const VideoSendStream::Config::EncoderSettings settings_;
   const VideoCodecType codec_type_;
@@ skipping 22 matching lines @@
   size_t max_data_payload_length_ ACCESS_ON(&encoder_queue_);
   bool nack_enabled_ ACCESS_ON(&encoder_queue_);
   uint32_t last_observed_bitrate_bps_ ACCESS_ON(&encoder_queue_);
   bool encoder_paused_and_dropped_frame_ ACCESS_ON(&encoder_queue_);
   bool has_received_sli_ ACCESS_ON(&encoder_queue_);
   uint8_t picture_id_sli_ ACCESS_ON(&encoder_queue_);
   bool has_received_rpsi_ ACCESS_ON(&encoder_queue_);
   uint64_t picture_id_rpsi_ ACCESS_ON(&encoder_queue_);
   Clock* const clock_;
   // Counters used for deciding if the video resolution is currently
-  // restricted, and if so, why.
-  std::vector<int> scale_counter_ ACCESS_ON(&encoder_queue_);
+  // restricted, and if so, why, on a per degradation preference basis.
+  // TODO(sprang): Replace this with a state holding a relative overuse measure
+  // instead, that can be translated into suitable down-scale or fps limit.
+  std::map<const VideoSendStream::DegradationPreference, std::vector<int>>
+      scale_counters_ ACCESS_ON(&encoder_queue_);
   // Set depending on degradation preferences
   VideoSendStream::DegradationPreference degradation_preference_
       ACCESS_ON(&encoder_queue_);
 
   struct AdaptationRequest {
     // The pixel count produced by the source at the time of the adaptation.
     int input_pixel_count_;
+    // Framerate received from the source at the time of the adaptation.
+    int framerate_fps_;

[nisse-webrtc, 2017/03/14 09:00:29]

How is that measured? Or is it requested/configured fps?

[sprang_webrtc, 2017/03/14 14:15:03]

This is the measured input fps to the encoder, the same value that is used to
update the encoder instance.

     // Indicates if request was to adapt up or down.
     enum class Mode { kAdaptUp, kAdaptDown } mode_;
   };
   // Stores a snapshot of the last adaptation request triggered by an AdaptUp
   // or AdaptDown signal.
   rtc::Optional<AdaptationRequest> last_adaptation_request_
       ACCESS_ON(&encoder_queue_);
 
   rtc::RaceChecker incoming_frame_race_checker_
       GUARDED_BY(incoming_frame_race_checker_);
@@ skipping 13 matching lines @@
   // All public methods are proxied to |encoder_queue_|. It must must be
   // destroyed first to make sure no tasks are run that use other members.
   rtc::TaskQueue encoder_queue_;
 
   RTC_DISALLOW_COPY_AND_ASSIGN(ViEEncoder);
 };
 
 }  // namespace webrtc
 
 #endif  // WEBRTC_VIDEO_VIE_ENCODER_H_