Move usage of QualityScaler to ViEEncoder.

webrtc/video/vie_encoder.h

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #ifndef WEBRTC_VIDEO_VIE_ENCODER_H_
 #define WEBRTC_VIDEO_VIE_ENCODER_H_
 
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "webrtc/base/criticalsection.h"
 #include "webrtc/base/event.h"
 #include "webrtc/base/sequenced_task_checker.h"
 #include "webrtc/base/task_queue.h"
 #include "webrtc/call.h"
 #include "webrtc/common_types.h"
 #include "webrtc/common_video/include/video_bitrate_allocator.h"
 #include "webrtc/common_video/rotation.h"
 #include "webrtc/media/base/videosinkinterface.h"
 #include "webrtc/modules/video_coding/include/video_coding_defines.h"
+#include "webrtc/modules/video_coding/utility/quality_scaler.h"
 #include "webrtc/modules/video_coding/video_coding_impl.h"
 #include "webrtc/modules/video_processing/include/video_processing.h"
 #include "webrtc/system_wrappers/include/atomic32.h"
 #include "webrtc/video/overuse_frame_detector.h"
 #include "webrtc/video_encoder.h"
 #include "webrtc/video_send_stream.h"
 #include "webrtc/typedefs.h"
 
 namespace webrtc {
 
 class ProcessThread;
 class SendStatisticsProxy;
 
 // VieEncoder represent a video encoder that accepts raw video frames as input
 // and produces an encoded bit stream.
 // Usage:
 //  Instantiate.
 //  Call SetSink.
 //  Call SetSource.
 //  Call ConfigureEncoder with the codec settings.
 //  Call Stop() when done.
 class ViEEncoder : public rtc::VideoSinkInterface<VideoFrame>,
                    public EncodedImageCallback,
                    public VCMSendStatisticsCallback,
-                   public CpuOveruseObserver {
+                   public ScalingObserverInterface {
  public:
   // Interface for receiving encoded video frames and notifications about
   // configuration changes.
   class EncoderSink : public EncodedImageCallback {
    public:
     virtual void OnEncoderConfigurationChanged(
         std::vector<VideoStream> streams,
         int min_transmit_bitrate_bps) = 0;
   };
 
-  // Down grade resolution at most 2 times for CPU reasons.
+  // Downscale resolution at most 2 times for CPU reasons.
   static const int kMaxCpuDowngrades = 2;
+  // Downscale resolution at most 2 times for low-quality reasons.
+  static const int kMaxQualityDowngrades = 2;
 
   ViEEncoder(uint32_t number_of_cores,
              SendStatisticsProxy* stats_proxy,
              const VideoSendStream::Config::EncoderSettings& settings,
              rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback,
              EncodedFrameObserver* encoder_timing);
   ~ViEEncoder();
   // RegisterProcessThread register |module_process_thread| with those objects
   // that use it. Registration has to happen on the thread where
   // |module_process_thread| was created (libjingle's worker thread).
@@ skipping 28 matching lines @@
   // virtual to test EncoderStateFeedback with mocks.
   virtual void OnReceivedIntraFrameRequest(size_t stream_index);
   virtual void OnReceivedSLI(uint8_t picture_id);
   virtual void OnReceivedRPSI(uint64_t picture_id);
 
   void OnBitrateUpdated(uint32_t bitrate_bps,
                         uint8_t fraction_lost,
                         int64_t round_trip_time_ms);
 
  protected:
-  // Used for testing. For example the |CpuOveruseObserver| methods must be
-  // called on |encoder_queue_|.
+  // Used for testing. For example the |ScalingObserverInterface| methods must
+  // be called on |encoder_queue_|.
   rtc::TaskQueue* encoder_queue() { return &encoder_queue_; }
 
-  // webrtc::CpuOveruseObserver implementation.
+  // webrtc::ScalingObserverInterface implementation.
   // These methods are protected for easier testing.
-  void OveruseDetected() override;
-  void NormalUsage() override;
+  void ScaleUp(ScaleReason reason) override;
+  void ScaleDown(ScaleReason reason) override;
 
  private:
   class ConfigureEncoderTask;
   class EncodeTask;
   class VideoSourceProxy;
 
   struct VideoFrameInfo {
     VideoFrameInfo(int width,
                    int height,
                    VideoRotation rotation,
@@ skipping 21 matching lines @@
 
   void EncodeVideoFrame(const VideoFrame& frame,
                         int64_t time_when_posted_in_ms);
 
   // Implements EncodedImageCallback.
   EncodedImageCallback::Result OnEncodedImage(
       const EncodedImage& encoded_image,
       const CodecSpecificInfo* codec_specific_info,
       const RTPFragmentationHeader* fragmentation) override;
 
+  void OnDroppedFrame() override;

[stefan-webrtc, 2016/11/25 10:45:25]

Should / can we add tests for this which would have been a regression if I
hadn't seen it?

[kthelgason, 2016/11/25 12:34:01]

As to whether we should, I think the answer is yes. I looked at adding tests but
it seems quite hard, especially if we want to isolate this feature. Since most
dependencies in this chain are not injected it's difficult to break it in a test
and insert a mock. Testing this in a sensible way would require a pretty big
refactoring, at least I don't see another way. I'd love to be wrong though!

+
   bool EncoderPaused() const;
   void TraceFrameDropStart();
   void TraceFrameDropEnd();
 
   rtc::Event shutdown_event_;
 
   const uint32_t number_of_cores_;
 
   const std::unique_ptr<VideoSourceProxy> source_proxy_;
   EncoderSink* sink_;
   const VideoSendStream::Config::EncoderSettings settings_;
   const VideoCodecType codec_type_;
 
   vcm::VideoSender video_sender_ ACCESS_ON(&encoder_queue_);
   OveruseFrameDetector overuse_detector_ ACCESS_ON(&encoder_queue_);
+  std::unique_ptr<QualityScaler> quality_scaler_ ACCESS_ON(&encoder_queue_);
 
   SendStatisticsProxy* const stats_proxy_;
   rtc::VideoSinkInterface<VideoFrame>* const pre_encode_callback_;
   ProcessThread* module_process_thread_;
   rtc::ThreadChecker module_process_thread_checker_;
   // |thread_checker_| checks that public methods that are related to lifetime
   // of ViEEncoder are called on the same thread.
   rtc::ThreadChecker thread_checker_;
 
   VideoEncoderConfig encoder_config_ ACCESS_ON(&encoder_queue_);
   std::unique_ptr<VideoBitrateAllocator> rate_allocator_
       ACCESS_ON(&encoder_queue_);
 
   // Set when ConfigureEncoder has been called in order to lazy reconfigure the
   // encoder on the next frame.
   bool pending_encoder_reconfiguration_ ACCESS_ON(&encoder_queue_);
   rtc::Optional<VideoFrameInfo> last_frame_info_ ACCESS_ON(&encoder_queue_);
   uint32_t encoder_start_bitrate_bps_ ACCESS_ON(&encoder_queue_);
   size_t max_data_payload_length_ ACCESS_ON(&encoder_queue_);
   uint32_t last_observed_bitrate_bps_ ACCESS_ON(&encoder_queue_);
   bool encoder_paused_and_dropped_frame_ ACCESS_ON(&encoder_queue_);
   bool has_received_sli_ ACCESS_ON(&encoder_queue_);
   uint8_t picture_id_sli_ ACCESS_ON(&encoder_queue_);
   bool has_received_rpsi_ ACCESS_ON(&encoder_queue_);
   uint64_t picture_id_rpsi_ ACCESS_ON(&encoder_queue_);
   Clock* const clock_;
-
-  VideoSendStream::DegradationPreference degradation_preference_
-      ACCESS_ON(&encoder_queue_);
-  // Counter used for deciding if the video resolution is currently
-  // restricted by CPU usage.
-  int cpu_restricted_counter_ ACCESS_ON(&encoder_queue_);
+  // Counters used for deciding if the video resolution is currently
+  // restricted, and if so, why.
+  int scale_counter_[kScaleReasonSize] ACCESS_ON(&encoder_queue_) = {0};
+  // Set depending on degradation preferences
+  bool scaling_enabled_ ACCESS_ON(&encoder_queue_) = false;
 
   int last_frame_width_ ACCESS_ON(&encoder_queue_);
   int last_frame_height_ ACCESS_ON(&encoder_queue_);
   // Pixel count last time the resolution was requested to be changed down.
   rtc::Optional<int> max_pixel_count_ ACCESS_ON(&encoder_queue_);
   // Pixel count last time the resolution was requested to be changed up.
   rtc::Optional<int> max_pixel_count_step_up_ ACCESS_ON(&encoder_queue_);
 
   rtc::RaceChecker incoming_frame_race_checker_
       GUARDED_BY(incoming_frame_race_checker_);
@@ skipping 10 matching lines @@
   // All public methods are proxied to |encoder_queue_|. It must must be
   // destroyed first to make sure no tasks are run that use other members.
   rtc::TaskQueue encoder_queue_;
 
   RTC_DISALLOW_COPY_AND_ASSIGN(ViEEncoder);
 };
 
 }  // namespace webrtc
 
 #endif  // WEBRTC_VIDEO_VIE_ENCODER_H_