OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #ifndef WEBRTC_VIDEO_VIE_ENCODER_H_ | 11 #ifndef WEBRTC_VIDEO_VIE_ENCODER_H_ |
12 #define WEBRTC_VIDEO_VIE_ENCODER_H_ | 12 #define WEBRTC_VIDEO_VIE_ENCODER_H_ |
13 | 13 |
14 #include <memory> | 14 #include <memory> |
15 #include <string> | 15 #include <string> |
16 #include <vector> | 16 #include <vector> |
17 | 17 |
18 #include "webrtc/base/criticalsection.h" | 18 #include "webrtc/base/criticalsection.h" |
19 #include "webrtc/base/event.h" | 19 #include "webrtc/base/event.h" |
20 #include "webrtc/base/sequenced_task_checker.h" | 20 #include "webrtc/base/sequenced_task_checker.h" |
21 #include "webrtc/base/task_queue.h" | 21 #include "webrtc/base/task_queue.h" |
22 #include "webrtc/call.h" | 22 #include "webrtc/call.h" |
23 #include "webrtc/common_types.h" | 23 #include "webrtc/common_types.h" |
24 #include "webrtc/common_video/include/video_bitrate_allocator.h" | 24 #include "webrtc/common_video/include/video_bitrate_allocator.h" |
25 #include "webrtc/common_video/rotation.h" | 25 #include "webrtc/common_video/rotation.h" |
26 #include "webrtc/media/base/videosinkinterface.h" | 26 #include "webrtc/media/base/videosinkinterface.h" |
27 #include "webrtc/modules/video_coding/include/video_coding_defines.h" | 27 #include "webrtc/modules/video_coding/include/video_coding_defines.h" |
| 28 #include "webrtc/modules/video_coding/utility/quality_scaler.h" |
28 #include "webrtc/modules/video_coding/video_coding_impl.h" | 29 #include "webrtc/modules/video_coding/video_coding_impl.h" |
29 #include "webrtc/modules/video_processing/include/video_processing.h" | 30 #include "webrtc/modules/video_processing/include/video_processing.h" |
30 #include "webrtc/system_wrappers/include/atomic32.h" | 31 #include "webrtc/system_wrappers/include/atomic32.h" |
31 #include "webrtc/video/overuse_frame_detector.h" | 32 #include "webrtc/video/overuse_frame_detector.h" |
32 #include "webrtc/video_encoder.h" | 33 #include "webrtc/video_encoder.h" |
33 #include "webrtc/video_send_stream.h" | 34 #include "webrtc/video_send_stream.h" |
34 #include "webrtc/typedefs.h" | 35 #include "webrtc/typedefs.h" |
35 | 36 |
36 namespace webrtc { | 37 namespace webrtc { |
37 | 38 |
38 class ProcessThread; | 39 class ProcessThread; |
39 class SendStatisticsProxy; | 40 class SendStatisticsProxy; |
40 | 41 |
41 // VieEncoder represent a video encoder that accepts raw video frames as input | 42 // VieEncoder represent a video encoder that accepts raw video frames as input |
42 // and produces an encoded bit stream. | 43 // and produces an encoded bit stream. |
43 // Usage: | 44 // Usage: |
44 // Instantiate. | 45 // Instantiate. |
45 // Call SetSink. | 46 // Call SetSink. |
46 // Call SetSource. | 47 // Call SetSource. |
47 // Call ConfigureEncoder with the codec settings. | 48 // Call ConfigureEncoder with the codec settings. |
48 // Call Stop() when done. | 49 // Call Stop() when done. |
49 class ViEEncoder : public rtc::VideoSinkInterface<VideoFrame>, | 50 class ViEEncoder : public rtc::VideoSinkInterface<VideoFrame>, |
50 public EncodedImageCallback, | 51 public EncodedImageCallback, |
51 public VCMSendStatisticsCallback, | 52 public VCMSendStatisticsCallback, |
52 public CpuOveruseObserver { | 53 public ScalingObserverInterface { |
53 public: | 54 public: |
54 // Interface for receiving encoded video frames and notifications about | 55 // Interface for receiving encoded video frames and notifications about |
55 // configuration changes. | 56 // configuration changes. |
56 class EncoderSink : public EncodedImageCallback { | 57 class EncoderSink : public EncodedImageCallback { |
57 public: | 58 public: |
58 virtual void OnEncoderConfigurationChanged( | 59 virtual void OnEncoderConfigurationChanged( |
59 std::vector<VideoStream> streams, | 60 std::vector<VideoStream> streams, |
60 int min_transmit_bitrate_bps) = 0; | 61 int min_transmit_bitrate_bps) = 0; |
61 }; | 62 }; |
62 | 63 |
63 // Down grade resolution at most 2 times for CPU reasons. | 64 // Downscale resolution at most 2 times for CPU reasons. |
64 static const int kMaxCpuDowngrades = 2; | 65 static const int kMaxCpuDowngrades = 2; |
| 66 // Downscale resolution at most 2 times for low-quality reasons. |
| 67 static const int kMaxQualityDowngrades = 2; |
65 | 68 |
66 ViEEncoder(uint32_t number_of_cores, | 69 ViEEncoder(uint32_t number_of_cores, |
67 SendStatisticsProxy* stats_proxy, | 70 SendStatisticsProxy* stats_proxy, |
68 const VideoSendStream::Config::EncoderSettings& settings, | 71 const VideoSendStream::Config::EncoderSettings& settings, |
69 rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback, | 72 rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback, |
70 EncodedFrameObserver* encoder_timing); | 73 EncodedFrameObserver* encoder_timing); |
71 ~ViEEncoder(); | 74 ~ViEEncoder(); |
72 // RegisterProcessThread register |module_process_thread| with those objects | 75 // RegisterProcessThread register |module_process_thread| with those objects |
73 // that use it. Registration has to happen on the thread where | 76 // that use it. Registration has to happen on the thread where |
74 // |module_process_thread| was created (libjingle's worker thread). | 77 // |module_process_thread| was created (libjingle's worker thread). |
(...skipping 29 matching lines...) Expand all Loading... |
104 // virtual to test EncoderStateFeedback with mocks. | 107 // virtual to test EncoderStateFeedback with mocks. |
105 virtual void OnReceivedIntraFrameRequest(size_t stream_index); | 108 virtual void OnReceivedIntraFrameRequest(size_t stream_index); |
106 virtual void OnReceivedSLI(uint8_t picture_id); | 109 virtual void OnReceivedSLI(uint8_t picture_id); |
107 virtual void OnReceivedRPSI(uint64_t picture_id); | 110 virtual void OnReceivedRPSI(uint64_t picture_id); |
108 | 111 |
109 void OnBitrateUpdated(uint32_t bitrate_bps, | 112 void OnBitrateUpdated(uint32_t bitrate_bps, |
110 uint8_t fraction_lost, | 113 uint8_t fraction_lost, |
111 int64_t round_trip_time_ms); | 114 int64_t round_trip_time_ms); |
112 | 115 |
113 protected: | 116 protected: |
114 // Used for testing. For example the |CpuOveruseObserver| methods must be | 117 // Used for testing. For example the |ScalingObserverInterface| methods must |
115 // called on |encoder_queue_|. | 118 // be called on |encoder_queue_|. |
116 rtc::TaskQueue* encoder_queue() { return &encoder_queue_; } | 119 rtc::TaskQueue* encoder_queue() { return &encoder_queue_; } |
117 | 120 |
118 // webrtc::CpuOveruseObserver implementation. | 121 // webrtc::ScalingObserverInterface implementation. |
119 // These methods are protected for easier testing. | 122 // These methods are protected for easier testing. |
120 void OveruseDetected() override; | 123 void ScaleUp(ScaleReason reason) override; |
121 void NormalUsage() override; | 124 void ScaleDown(ScaleReason reason) override; |
122 | 125 |
123 private: | 126 private: |
124 class ConfigureEncoderTask; | 127 class ConfigureEncoderTask; |
125 class EncodeTask; | 128 class EncodeTask; |
126 class VideoSourceProxy; | 129 class VideoSourceProxy; |
127 | 130 |
128 struct VideoFrameInfo { | 131 struct VideoFrameInfo { |
129 VideoFrameInfo(int width, | 132 VideoFrameInfo(int width, |
130 int height, | 133 int height, |
131 VideoRotation rotation, | 134 VideoRotation rotation, |
(...skipping 22 matching lines...) Expand all Loading... |
154 | 157 |
155 void EncodeVideoFrame(const VideoFrame& frame, | 158 void EncodeVideoFrame(const VideoFrame& frame, |
156 int64_t time_when_posted_in_ms); | 159 int64_t time_when_posted_in_ms); |
157 | 160 |
158 // Implements EncodedImageCallback. | 161 // Implements EncodedImageCallback. |
159 EncodedImageCallback::Result OnEncodedImage( | 162 EncodedImageCallback::Result OnEncodedImage( |
160 const EncodedImage& encoded_image, | 163 const EncodedImage& encoded_image, |
161 const CodecSpecificInfo* codec_specific_info, | 164 const CodecSpecificInfo* codec_specific_info, |
162 const RTPFragmentationHeader* fragmentation) override; | 165 const RTPFragmentationHeader* fragmentation) override; |
163 | 166 |
| 167 void OnDroppedFrame() override; |
| 168 |
164 bool EncoderPaused() const; | 169 bool EncoderPaused() const; |
165 void TraceFrameDropStart(); | 170 void TraceFrameDropStart(); |
166 void TraceFrameDropEnd(); | 171 void TraceFrameDropEnd(); |
167 | 172 |
168 rtc::Event shutdown_event_; | 173 rtc::Event shutdown_event_; |
169 | 174 |
170 const uint32_t number_of_cores_; | 175 const uint32_t number_of_cores_; |
171 | 176 |
172 const std::unique_ptr<VideoSourceProxy> source_proxy_; | 177 const std::unique_ptr<VideoSourceProxy> source_proxy_; |
173 EncoderSink* sink_; | 178 EncoderSink* sink_; |
174 const VideoSendStream::Config::EncoderSettings settings_; | 179 const VideoSendStream::Config::EncoderSettings settings_; |
175 const VideoCodecType codec_type_; | 180 const VideoCodecType codec_type_; |
176 | 181 |
177 vcm::VideoSender video_sender_ ACCESS_ON(&encoder_queue_); | 182 vcm::VideoSender video_sender_ ACCESS_ON(&encoder_queue_); |
178 OveruseFrameDetector overuse_detector_ ACCESS_ON(&encoder_queue_); | 183 OveruseFrameDetector overuse_detector_ ACCESS_ON(&encoder_queue_); |
| 184 std::unique_ptr<QualityScaler> quality_scaler_ ACCESS_ON(&encoder_queue_); |
179 | 185 |
180 SendStatisticsProxy* const stats_proxy_; | 186 SendStatisticsProxy* const stats_proxy_; |
181 rtc::VideoSinkInterface<VideoFrame>* const pre_encode_callback_; | 187 rtc::VideoSinkInterface<VideoFrame>* const pre_encode_callback_; |
182 ProcessThread* module_process_thread_; | 188 ProcessThread* module_process_thread_; |
183 rtc::ThreadChecker module_process_thread_checker_; | 189 rtc::ThreadChecker module_process_thread_checker_; |
184 // |thread_checker_| checks that public methods that are related to lifetime | 190 // |thread_checker_| checks that public methods that are related to lifetime |
185 // of ViEEncoder are called on the same thread. | 191 // of ViEEncoder are called on the same thread. |
186 rtc::ThreadChecker thread_checker_; | 192 rtc::ThreadChecker thread_checker_; |
187 | 193 |
188 VideoEncoderConfig encoder_config_ ACCESS_ON(&encoder_queue_); | 194 VideoEncoderConfig encoder_config_ ACCESS_ON(&encoder_queue_); |
189 std::unique_ptr<VideoBitrateAllocator> rate_allocator_ | 195 std::unique_ptr<VideoBitrateAllocator> rate_allocator_ |
190 ACCESS_ON(&encoder_queue_); | 196 ACCESS_ON(&encoder_queue_); |
191 | 197 |
192 // Set when ConfigureEncoder has been called in order to lazy reconfigure the | 198 // Set when ConfigureEncoder has been called in order to lazy reconfigure the |
193 // encoder on the next frame. | 199 // encoder on the next frame. |
194 bool pending_encoder_reconfiguration_ ACCESS_ON(&encoder_queue_); | 200 bool pending_encoder_reconfiguration_ ACCESS_ON(&encoder_queue_); |
195 rtc::Optional<VideoFrameInfo> last_frame_info_ ACCESS_ON(&encoder_queue_); | 201 rtc::Optional<VideoFrameInfo> last_frame_info_ ACCESS_ON(&encoder_queue_); |
196 uint32_t encoder_start_bitrate_bps_ ACCESS_ON(&encoder_queue_); | 202 uint32_t encoder_start_bitrate_bps_ ACCESS_ON(&encoder_queue_); |
197 size_t max_data_payload_length_ ACCESS_ON(&encoder_queue_); | 203 size_t max_data_payload_length_ ACCESS_ON(&encoder_queue_); |
198 bool nack_enabled_ ACCESS_ON(&encoder_queue_); | 204 bool nack_enabled_ ACCESS_ON(&encoder_queue_); |
199 uint32_t last_observed_bitrate_bps_ ACCESS_ON(&encoder_queue_); | 205 uint32_t last_observed_bitrate_bps_ ACCESS_ON(&encoder_queue_); |
200 bool encoder_paused_and_dropped_frame_ ACCESS_ON(&encoder_queue_); | 206 bool encoder_paused_and_dropped_frame_ ACCESS_ON(&encoder_queue_); |
201 bool has_received_sli_ ACCESS_ON(&encoder_queue_); | 207 bool has_received_sli_ ACCESS_ON(&encoder_queue_); |
202 uint8_t picture_id_sli_ ACCESS_ON(&encoder_queue_); | 208 uint8_t picture_id_sli_ ACCESS_ON(&encoder_queue_); |
203 bool has_received_rpsi_ ACCESS_ON(&encoder_queue_); | 209 bool has_received_rpsi_ ACCESS_ON(&encoder_queue_); |
204 uint64_t picture_id_rpsi_ ACCESS_ON(&encoder_queue_); | 210 uint64_t picture_id_rpsi_ ACCESS_ON(&encoder_queue_); |
205 Clock* const clock_; | 211 Clock* const clock_; |
206 | 212 // Counters used for deciding if the video resolution is currently |
207 VideoSendStream::DegradationPreference degradation_preference_ | 213 // restricted, and if so, why. |
208 ACCESS_ON(&encoder_queue_); | 214 int scale_counter_[kScaleReasonSize] ACCESS_ON(&encoder_queue_) = {0}; |
209 // Counter used for deciding if the video resolution is currently | 215 // Set depending on degradation preferences |
210 // restricted by CPU usage. | 216 bool scaling_enabled_ ACCESS_ON(&encoder_queue_) = false; |
211 int cpu_restricted_counter_ ACCESS_ON(&encoder_queue_); | |
212 | 217 |
213 int last_frame_width_ ACCESS_ON(&encoder_queue_); | 218 int last_frame_width_ ACCESS_ON(&encoder_queue_); |
214 int last_frame_height_ ACCESS_ON(&encoder_queue_); | 219 int last_frame_height_ ACCESS_ON(&encoder_queue_); |
215 // Pixel count last time the resolution was requested to be changed down. | 220 // Pixel count last time the resolution was requested to be changed down. |
216 rtc::Optional<int> max_pixel_count_ ACCESS_ON(&encoder_queue_); | 221 rtc::Optional<int> max_pixel_count_ ACCESS_ON(&encoder_queue_); |
217 // Pixel count last time the resolution was requested to be changed up. | 222 // Pixel count last time the resolution was requested to be changed up. |
218 rtc::Optional<int> max_pixel_count_step_up_ ACCESS_ON(&encoder_queue_); | 223 rtc::Optional<int> max_pixel_count_step_up_ ACCESS_ON(&encoder_queue_); |
219 | 224 |
220 rtc::RaceChecker incoming_frame_race_checker_ | 225 rtc::RaceChecker incoming_frame_race_checker_ |
221 GUARDED_BY(incoming_frame_race_checker_); | 226 GUARDED_BY(incoming_frame_race_checker_); |
(...skipping 10 matching lines...) Expand all Loading... |
232 // All public methods are proxied to |encoder_queue_|. It must must be | 237 // All public methods are proxied to |encoder_queue_|. It must must be |
233 // destroyed first to make sure no tasks are run that use other members. | 238 // destroyed first to make sure no tasks are run that use other members. |
234 rtc::TaskQueue encoder_queue_; | 239 rtc::TaskQueue encoder_queue_; |
235 | 240 |
236 RTC_DISALLOW_COPY_AND_ASSIGN(ViEEncoder); | 241 RTC_DISALLOW_COPY_AND_ASSIGN(ViEEncoder); |
237 }; | 242 }; |
238 | 243 |
239 } // namespace webrtc | 244 } // namespace webrtc |
240 | 245 |
241 #endif // WEBRTC_VIDEO_VIE_ENCODER_H_ | 246 #endif // WEBRTC_VIDEO_VIE_ENCODER_H_ |
OLD | NEW |