OLD | NEW |
| (Empty) |
1 /* | |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | |
3 * | |
4 * Use of this source code is governed by a BSD-style license | |
5 * that can be found in the LICENSE file in the root of the source | |
6 * tree. An additional intellectual property rights grant can be found | |
7 * in the file PATENTS. All contributing project authors may | |
8 * be found in the AUTHORS file in the root of the source tree. | |
9 */ | |
10 | |
11 #ifndef WEBRTC_VIDEO_VIE_ENCODER_H_ | |
12 #define WEBRTC_VIDEO_VIE_ENCODER_H_ | |
13 | |
14 #include <map> | |
15 #include <memory> | |
16 #include <string> | |
17 #include <vector> | |
18 | |
19 #include "webrtc/api/video/video_rotation.h" | |
20 #include "webrtc/api/video_codecs/video_encoder.h" | |
21 #include "webrtc/call/call.h" | |
22 #include "webrtc/common_types.h" | |
23 #include "webrtc/common_video/include/video_bitrate_allocator.h" | |
24 #include "webrtc/media/base/videosinkinterface.h" | |
25 #include "webrtc/modules/video_coding/include/video_coding_defines.h" | |
26 #include "webrtc/modules/video_coding/utility/quality_scaler.h" | |
27 #include "webrtc/modules/video_coding/video_coding_impl.h" | |
28 #include "webrtc/rtc_base/criticalsection.h" | |
29 #include "webrtc/rtc_base/event.h" | |
30 #include "webrtc/rtc_base/sequenced_task_checker.h" | |
31 #include "webrtc/rtc_base/task_queue.h" | |
32 #include "webrtc/system_wrappers/include/atomic32.h" | |
33 #include "webrtc/typedefs.h" | |
34 #include "webrtc/video/overuse_frame_detector.h" | |
35 #include "webrtc/video_send_stream.h" | |
36 | |
37 namespace webrtc { | |
38 | |
39 class ProcessThread; | |
40 class SendStatisticsProxy; | |
41 class VideoBitrateAllocationObserver; | |
42 | |
43 // VieEncoder represent a video encoder that accepts raw video frames as input | |
44 // and produces an encoded bit stream. | |
45 // Usage: | |
46 // Instantiate. | |
47 // Call SetSink. | |
48 // Call SetSource. | |
49 // Call ConfigureEncoder with the codec settings. | |
50 // Call Stop() when done. | |
51 class ViEEncoder : public rtc::VideoSinkInterface<VideoFrame>, | |
52 public EncodedImageCallback, | |
53 public VCMSendStatisticsCallback, | |
54 public AdaptationObserverInterface { | |
55 public: | |
56 // Interface for receiving encoded video frames and notifications about | |
57 // configuration changes. | |
58 class EncoderSink : public EncodedImageCallback { | |
59 public: | |
60 virtual void OnEncoderConfigurationChanged( | |
61 std::vector<VideoStream> streams, | |
62 int min_transmit_bitrate_bps) = 0; | |
63 }; | |
64 | |
65 // Number of resolution and framerate reductions (-1: disabled). | |
66 struct AdaptCounts { | |
67 int resolution = 0; | |
68 int fps = 0; | |
69 }; | |
70 | |
71 // Downscale resolution at most 2 times for CPU reasons. | |
72 static const int kMaxCpuResolutionDowngrades = 2; | |
73 // Downscale framerate at most 4 times. | |
74 static const int kMaxCpuFramerateDowngrades = 4; | |
75 | |
76 ViEEncoder(uint32_t number_of_cores, | |
77 SendStatisticsProxy* stats_proxy, | |
78 const VideoSendStream::Config::EncoderSettings& settings, | |
79 rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback, | |
80 EncodedFrameObserver* encoder_timing, | |
81 std::unique_ptr<OveruseFrameDetector> overuse_detector); | |
82 ~ViEEncoder(); | |
83 // RegisterProcessThread register |module_process_thread| with those objects | |
84 // that use it. Registration has to happen on the thread where | |
85 // |module_process_thread| was created (libjingle's worker thread). | |
86 // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue. | |
87 void RegisterProcessThread(ProcessThread* module_process_thread); | |
88 void DeRegisterProcessThread(); | |
89 | |
90 // Sets the source that will provide I420 video frames. | |
91 // |degradation_preference| control whether or not resolution or frame rate | |
92 // may be reduced. | |
93 void SetSource( | |
94 rtc::VideoSourceInterface<VideoFrame>* source, | |
95 const VideoSendStream::DegradationPreference& degradation_preference); | |
96 | |
97 // Sets the |sink| that gets the encoded frames. |rotation_applied| means | |
98 // that the source must support rotation. Only set |rotation_applied| if the | |
99 // remote side does not support the rotation extension. | |
100 void SetSink(EncoderSink* sink, bool rotation_applied); | |
101 | |
102 // TODO(perkj): Can we remove VideoCodec.startBitrate ? | |
103 void SetStartBitrate(int start_bitrate_bps); | |
104 | |
105 void SetBitrateObserver(VideoBitrateAllocationObserver* bitrate_observer); | |
106 | |
107 void ConfigureEncoder(VideoEncoderConfig config, | |
108 size_t max_data_payload_length, | |
109 bool nack_enabled); | |
110 | |
111 // Permanently stop encoding. After this method has returned, it is | |
112 // guaranteed that no encoded frames will be delivered to the sink. | |
113 void Stop(); | |
114 | |
115 void SendKeyFrame(); | |
116 | |
117 // virtual to test EncoderStateFeedback with mocks. | |
118 virtual void OnReceivedIntraFrameRequest(size_t stream_index); | |
119 | |
120 void OnBitrateUpdated(uint32_t bitrate_bps, | |
121 uint8_t fraction_lost, | |
122 int64_t round_trip_time_ms); | |
123 | |
124 protected: | |
125 // Used for testing. For example the |ScalingObserverInterface| methods must | |
126 // be called on |encoder_queue_|. | |
127 rtc::TaskQueue* encoder_queue() { return &encoder_queue_; } | |
128 | |
129 // webrtc::ScalingObserverInterface implementation. | |
130 // These methods are protected for easier testing. | |
131 void AdaptUp(AdaptReason reason) override; | |
132 void AdaptDown(AdaptReason reason) override; | |
133 static CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time); | |
134 | |
135 private: | |
136 class ConfigureEncoderTask; | |
137 class EncodeTask; | |
138 class VideoSourceProxy; | |
139 | |
140 class VideoFrameInfo { | |
141 public: | |
142 VideoFrameInfo(int width, | |
143 int height, | |
144 bool is_texture) | |
145 : width(width), | |
146 height(height), | |
147 is_texture(is_texture) {} | |
148 int width; | |
149 int height; | |
150 bool is_texture; | |
151 int pixel_count() const { return width * height; } | |
152 }; | |
153 | |
154 void ConfigureEncoderOnTaskQueue(VideoEncoderConfig config, | |
155 size_t max_data_payload_length, | |
156 bool nack_enabled); | |
157 void ReconfigureEncoder(); | |
158 | |
159 void ConfigureQualityScaler(); | |
160 | |
161 // Implements VideoSinkInterface. | |
162 void OnFrame(const VideoFrame& video_frame) override; | |
163 | |
164 // Implements VideoSendStatisticsCallback. | |
165 void SendStatistics(uint32_t bit_rate, | |
166 uint32_t frame_rate) override; | |
167 | |
168 void EncodeVideoFrame(const VideoFrame& frame, | |
169 int64_t time_when_posted_in_ms); | |
170 | |
171 // Implements EncodedImageCallback. | |
172 EncodedImageCallback::Result OnEncodedImage( | |
173 const EncodedImage& encoded_image, | |
174 const CodecSpecificInfo* codec_specific_info, | |
175 const RTPFragmentationHeader* fragmentation) override; | |
176 | |
177 void OnDroppedFrame() override; | |
178 | |
179 bool EncoderPaused() const; | |
180 void TraceFrameDropStart(); | |
181 void TraceFrameDropEnd(); | |
182 | |
183 // Class holding adaptation information. | |
184 class AdaptCounter final { | |
185 public: | |
186 AdaptCounter(); | |
187 ~AdaptCounter(); | |
188 | |
189 // Get number of adaptation downscales for |reason|. | |
190 AdaptCounts Counts(int reason) const; | |
191 | |
192 std::string ToString() const; | |
193 | |
194 void IncrementFramerate(int reason); | |
195 void IncrementResolution(int reason); | |
196 void DecrementFramerate(int reason); | |
197 void DecrementResolution(int reason); | |
198 void DecrementFramerate(int reason, int cur_fps); | |
199 | |
200 // Gets the total number of downgrades (for all adapt reasons). | |
201 int FramerateCount() const; | |
202 int ResolutionCount() const; | |
203 | |
204 // Gets the total number of downgrades for |reason|. | |
205 int FramerateCount(int reason) const; | |
206 int ResolutionCount(int reason) const; | |
207 int TotalCount(int reason) const; | |
208 | |
209 private: | |
210 std::string ToString(const std::vector<int>& counters) const; | |
211 int Count(const std::vector<int>& counters) const; | |
212 void MoveCount(std::vector<int>* counters, int from_reason); | |
213 | |
214 // Degradation counters holding number of framerate/resolution reductions | |
215 // per adapt reason. | |
216 std::vector<int> fps_counters_; | |
217 std::vector<int> resolution_counters_; | |
218 }; | |
219 | |
220 AdaptCounter& GetAdaptCounter() RUN_ON(&encoder_queue_); | |
221 const AdaptCounter& GetConstAdaptCounter() RUN_ON(&encoder_queue_); | |
222 void UpdateAdaptationStats(AdaptReason reason) RUN_ON(&encoder_queue_); | |
223 AdaptCounts GetActiveCounts(AdaptReason reason) RUN_ON(&encoder_queue_); | |
224 | |
225 rtc::Event shutdown_event_; | |
226 | |
227 const uint32_t number_of_cores_; | |
228 // Counts how many frames we've dropped in the initial rampup phase. | |
229 int initial_rampup_; | |
230 | |
231 const std::unique_ptr<VideoSourceProxy> source_proxy_; | |
232 EncoderSink* sink_; | |
233 const VideoSendStream::Config::EncoderSettings settings_; | |
234 const VideoCodecType codec_type_; | |
235 | |
236 vcm::VideoSender video_sender_ ACCESS_ON(&encoder_queue_); | |
237 std::unique_ptr<OveruseFrameDetector> overuse_detector_ | |
238 ACCESS_ON(&encoder_queue_); | |
239 std::unique_ptr<QualityScaler> quality_scaler_ ACCESS_ON(&encoder_queue_); | |
240 | |
241 SendStatisticsProxy* const stats_proxy_; | |
242 rtc::VideoSinkInterface<VideoFrame>* const pre_encode_callback_; | |
243 ProcessThread* module_process_thread_; | |
244 rtc::ThreadChecker module_process_thread_checker_; | |
245 // |thread_checker_| checks that public methods that are related to lifetime | |
246 // of ViEEncoder are called on the same thread. | |
247 rtc::ThreadChecker thread_checker_; | |
248 | |
249 VideoEncoderConfig encoder_config_ ACCESS_ON(&encoder_queue_); | |
250 std::unique_ptr<VideoBitrateAllocator> rate_allocator_ | |
251 ACCESS_ON(&encoder_queue_); | |
252 // The maximum frame rate of the current codec configuration, as determined | |
253 // at the last ReconfigureEncoder() call. | |
254 int max_framerate_ ACCESS_ON(&encoder_queue_); | |
255 | |
256 // Set when ConfigureEncoder has been called in order to lazy reconfigure the | |
257 // encoder on the next frame. | |
258 bool pending_encoder_reconfiguration_ ACCESS_ON(&encoder_queue_); | |
259 rtc::Optional<VideoFrameInfo> last_frame_info_ ACCESS_ON(&encoder_queue_); | |
260 int crop_width_ ACCESS_ON(&encoder_queue_); | |
261 int crop_height_ ACCESS_ON(&encoder_queue_); | |
262 uint32_t encoder_start_bitrate_bps_ ACCESS_ON(&encoder_queue_); | |
263 size_t max_data_payload_length_ ACCESS_ON(&encoder_queue_); | |
264 bool nack_enabled_ ACCESS_ON(&encoder_queue_); | |
265 uint32_t last_observed_bitrate_bps_ ACCESS_ON(&encoder_queue_); | |
266 bool encoder_paused_and_dropped_frame_ ACCESS_ON(&encoder_queue_); | |
267 Clock* const clock_; | |
268 // Counters used for deciding if the video resolution or framerate is | |
269 // currently restricted, and if so, why, on a per degradation preference | |
270 // basis. | |
271 // TODO(sprang): Replace this with a state holding a relative overuse measure | |
272 // instead, that can be translated into suitable down-scale or fps limit. | |
273 std::map<const VideoSendStream::DegradationPreference, AdaptCounter> | |
274 adapt_counters_ ACCESS_ON(&encoder_queue_); | |
275 // Set depending on degradation preferences. | |
276 VideoSendStream::DegradationPreference degradation_preference_ | |
277 ACCESS_ON(&encoder_queue_); | |
278 | |
279 struct AdaptationRequest { | |
280 // The pixel count produced by the source at the time of the adaptation. | |
281 int input_pixel_count_; | |
282 // Framerate received from the source at the time of the adaptation. | |
283 int framerate_fps_; | |
284 // Indicates if request was to adapt up or down. | |
285 enum class Mode { kAdaptUp, kAdaptDown } mode_; | |
286 }; | |
287 // Stores a snapshot of the last adaptation request triggered by an AdaptUp | |
288 // or AdaptDown signal. | |
289 rtc::Optional<AdaptationRequest> last_adaptation_request_ | |
290 ACCESS_ON(&encoder_queue_); | |
291 | |
292 rtc::RaceChecker incoming_frame_race_checker_ | |
293 GUARDED_BY(incoming_frame_race_checker_); | |
294 Atomic32 posted_frames_waiting_for_encode_; | |
295 // Used to make sure incoming time stamp is increasing for every frame. | |
296 int64_t last_captured_timestamp_ GUARDED_BY(incoming_frame_race_checker_); | |
297 // Delta used for translating between NTP and internal timestamps. | |
298 const int64_t delta_ntp_internal_ms_ GUARDED_BY(incoming_frame_race_checker_); | |
299 | |
300 int64_t last_frame_log_ms_ GUARDED_BY(incoming_frame_race_checker_); | |
301 int captured_frame_count_ ACCESS_ON(&encoder_queue_); | |
302 int dropped_frame_count_ ACCESS_ON(&encoder_queue_); | |
303 | |
304 VideoBitrateAllocationObserver* bitrate_observer_ ACCESS_ON(&encoder_queue_); | |
305 rtc::Optional<int64_t> last_parameters_update_ms_ ACCESS_ON(&encoder_queue_); | |
306 | |
307 // All public methods are proxied to |encoder_queue_|. It must must be | |
308 // destroyed first to make sure no tasks are run that use other members. | |
309 rtc::TaskQueue encoder_queue_; | |
310 | |
311 RTC_DISALLOW_COPY_AND_ASSIGN(ViEEncoder); | |
312 }; | |
313 | |
314 } // namespace webrtc | |
315 | |
316 #endif // WEBRTC_VIDEO_VIE_ENCODER_H_ | |
OLD | NEW |