| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | |
| 3 * | |
| 4 * Use of this source code is governed by a BSD-style license | |
| 5 * that can be found in the LICENSE file in the root of the source | |
| 6 * tree. An additional intellectual property rights grant can be found | |
| 7 * in the file PATENTS. All contributing project authors may | |
| 8 * be found in the AUTHORS file in the root of the source tree. | |
| 9 */ | |
| 10 | |
| 11 #ifndef WEBRTC_VIDEO_VIE_ENCODER_H_ | |
| 12 #define WEBRTC_VIDEO_VIE_ENCODER_H_ | |
| 13 | |
| 14 #include <map> | |
| 15 #include <memory> | |
| 16 #include <string> | |
| 17 #include <vector> | |
| 18 | |
| 19 #include "webrtc/api/video/video_rotation.h" | |
| 20 #include "webrtc/api/video_codecs/video_encoder.h" | |
| 21 #include "webrtc/call/call.h" | |
| 22 #include "webrtc/common_types.h" | |
| 23 #include "webrtc/common_video/include/video_bitrate_allocator.h" | |
| 24 #include "webrtc/media/base/videosinkinterface.h" | |
| 25 #include "webrtc/modules/video_coding/include/video_coding_defines.h" | |
| 26 #include "webrtc/modules/video_coding/utility/quality_scaler.h" | |
| 27 #include "webrtc/modules/video_coding/video_coding_impl.h" | |
| 28 #include "webrtc/rtc_base/criticalsection.h" | |
| 29 #include "webrtc/rtc_base/event.h" | |
| 30 #include "webrtc/rtc_base/sequenced_task_checker.h" | |
| 31 #include "webrtc/rtc_base/task_queue.h" | |
| 32 #include "webrtc/system_wrappers/include/atomic32.h" | |
| 33 #include "webrtc/typedefs.h" | |
| 34 #include "webrtc/video/overuse_frame_detector.h" | |
| 35 #include "webrtc/video_send_stream.h" | |
| 36 | |
| 37 namespace webrtc { | |
| 38 | |
| 39 class ProcessThread; | |
| 40 class SendStatisticsProxy; | |
| 41 class VideoBitrateAllocationObserver; | |
| 42 | |
| 43 // VieEncoder represent a video encoder that accepts raw video frames as input | |
| 44 // and produces an encoded bit stream. | |
| 45 // Usage: | |
| 46 // Instantiate. | |
| 47 // Call SetSink. | |
| 48 // Call SetSource. | |
| 49 // Call ConfigureEncoder with the codec settings. | |
| 50 // Call Stop() when done. | |
| 51 class ViEEncoder : public rtc::VideoSinkInterface<VideoFrame>, | |
| 52 public EncodedImageCallback, | |
| 53 public VCMSendStatisticsCallback, | |
| 54 public AdaptationObserverInterface { | |
| 55 public: | |
| 56 // Interface for receiving encoded video frames and notifications about | |
| 57 // configuration changes. | |
| 58 class EncoderSink : public EncodedImageCallback { | |
| 59 public: | |
| 60 virtual void OnEncoderConfigurationChanged( | |
| 61 std::vector<VideoStream> streams, | |
| 62 int min_transmit_bitrate_bps) = 0; | |
| 63 }; | |
| 64 | |
| 65 // Number of resolution and framerate reductions (-1: disabled). | |
| 66 struct AdaptCounts { | |
| 67 int resolution = 0; | |
| 68 int fps = 0; | |
| 69 }; | |
| 70 | |
| 71 // Downscale resolution at most 2 times for CPU reasons. | |
| 72 static const int kMaxCpuResolutionDowngrades = 2; | |
| 73 // Downscale framerate at most 4 times. | |
| 74 static const int kMaxCpuFramerateDowngrades = 4; | |
| 75 | |
| 76 ViEEncoder(uint32_t number_of_cores, | |
| 77 SendStatisticsProxy* stats_proxy, | |
| 78 const VideoSendStream::Config::EncoderSettings& settings, | |
| 79 rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback, | |
| 80 EncodedFrameObserver* encoder_timing, | |
| 81 std::unique_ptr<OveruseFrameDetector> overuse_detector); | |
| 82 ~ViEEncoder(); | |
| 83 // RegisterProcessThread register |module_process_thread| with those objects | |
| 84 // that use it. Registration has to happen on the thread where | |
| 85 // |module_process_thread| was created (libjingle's worker thread). | |
| 86 // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue. | |
| 87 void RegisterProcessThread(ProcessThread* module_process_thread); | |
| 88 void DeRegisterProcessThread(); | |
| 89 | |
| 90 // Sets the source that will provide I420 video frames. | |
| 91 // |degradation_preference| control whether or not resolution or frame rate | |
| 92 // may be reduced. | |
| 93 void SetSource( | |
| 94 rtc::VideoSourceInterface<VideoFrame>* source, | |
| 95 const VideoSendStream::DegradationPreference& degradation_preference); | |
| 96 | |
| 97 // Sets the |sink| that gets the encoded frames. |rotation_applied| means | |
| 98 // that the source must support rotation. Only set |rotation_applied| if the | |
| 99 // remote side does not support the rotation extension. | |
| 100 void SetSink(EncoderSink* sink, bool rotation_applied); | |
| 101 | |
| 102 // TODO(perkj): Can we remove VideoCodec.startBitrate ? | |
| 103 void SetStartBitrate(int start_bitrate_bps); | |
| 104 | |
| 105 void SetBitrateObserver(VideoBitrateAllocationObserver* bitrate_observer); | |
| 106 | |
| 107 void ConfigureEncoder(VideoEncoderConfig config, | |
| 108 size_t max_data_payload_length, | |
| 109 bool nack_enabled); | |
| 110 | |
| 111 // Permanently stop encoding. After this method has returned, it is | |
| 112 // guaranteed that no encoded frames will be delivered to the sink. | |
| 113 void Stop(); | |
| 114 | |
| 115 void SendKeyFrame(); | |
| 116 | |
| 117 // virtual to test EncoderStateFeedback with mocks. | |
| 118 virtual void OnReceivedIntraFrameRequest(size_t stream_index); | |
| 119 | |
| 120 void OnBitrateUpdated(uint32_t bitrate_bps, | |
| 121 uint8_t fraction_lost, | |
| 122 int64_t round_trip_time_ms); | |
| 123 | |
| 124 protected: | |
| 125 // Used for testing. For example the |ScalingObserverInterface| methods must | |
| 126 // be called on |encoder_queue_|. | |
| 127 rtc::TaskQueue* encoder_queue() { return &encoder_queue_; } | |
| 128 | |
| 129 // webrtc::ScalingObserverInterface implementation. | |
| 130 // These methods are protected for easier testing. | |
| 131 void AdaptUp(AdaptReason reason) override; | |
| 132 void AdaptDown(AdaptReason reason) override; | |
| 133 static CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time); | |
| 134 | |
| 135 private: | |
| 136 class ConfigureEncoderTask; | |
| 137 class EncodeTask; | |
| 138 class VideoSourceProxy; | |
| 139 | |
| 140 class VideoFrameInfo { | |
| 141 public: | |
| 142 VideoFrameInfo(int width, | |
| 143 int height, | |
| 144 bool is_texture) | |
| 145 : width(width), | |
| 146 height(height), | |
| 147 is_texture(is_texture) {} | |
| 148 int width; | |
| 149 int height; | |
| 150 bool is_texture; | |
| 151 int pixel_count() const { return width * height; } | |
| 152 }; | |
| 153 | |
| 154 void ConfigureEncoderOnTaskQueue(VideoEncoderConfig config, | |
| 155 size_t max_data_payload_length, | |
| 156 bool nack_enabled); | |
| 157 void ReconfigureEncoder(); | |
| 158 | |
| 159 void ConfigureQualityScaler(); | |
| 160 | |
| 161 // Implements VideoSinkInterface. | |
| 162 void OnFrame(const VideoFrame& video_frame) override; | |
| 163 | |
| 164 // Implements VideoSendStatisticsCallback. | |
| 165 void SendStatistics(uint32_t bit_rate, | |
| 166 uint32_t frame_rate) override; | |
| 167 | |
| 168 void EncodeVideoFrame(const VideoFrame& frame, | |
| 169 int64_t time_when_posted_in_ms); | |
| 170 | |
| 171 // Implements EncodedImageCallback. | |
| 172 EncodedImageCallback::Result OnEncodedImage( | |
| 173 const EncodedImage& encoded_image, | |
| 174 const CodecSpecificInfo* codec_specific_info, | |
| 175 const RTPFragmentationHeader* fragmentation) override; | |
| 176 | |
| 177 void OnDroppedFrame() override; | |
| 178 | |
| 179 bool EncoderPaused() const; | |
| 180 void TraceFrameDropStart(); | |
| 181 void TraceFrameDropEnd(); | |
| 182 | |
| 183 // Class holding adaptation information. | |
| 184 class AdaptCounter final { | |
| 185 public: | |
| 186 AdaptCounter(); | |
| 187 ~AdaptCounter(); | |
| 188 | |
| 189 // Get number of adaptation downscales for |reason|. | |
| 190 AdaptCounts Counts(int reason) const; | |
| 191 | |
| 192 std::string ToString() const; | |
| 193 | |
| 194 void IncrementFramerate(int reason); | |
| 195 void IncrementResolution(int reason); | |
| 196 void DecrementFramerate(int reason); | |
| 197 void DecrementResolution(int reason); | |
| 198 void DecrementFramerate(int reason, int cur_fps); | |
| 199 | |
| 200 // Gets the total number of downgrades (for all adapt reasons). | |
| 201 int FramerateCount() const; | |
| 202 int ResolutionCount() const; | |
| 203 | |
| 204 // Gets the total number of downgrades for |reason|. | |
| 205 int FramerateCount(int reason) const; | |
| 206 int ResolutionCount(int reason) const; | |
| 207 int TotalCount(int reason) const; | |
| 208 | |
| 209 private: | |
| 210 std::string ToString(const std::vector<int>& counters) const; | |
| 211 int Count(const std::vector<int>& counters) const; | |
| 212 void MoveCount(std::vector<int>* counters, int from_reason); | |
| 213 | |
| 214 // Degradation counters holding number of framerate/resolution reductions | |
| 215 // per adapt reason. | |
| 216 std::vector<int> fps_counters_; | |
| 217 std::vector<int> resolution_counters_; | |
| 218 }; | |
| 219 | |
| 220 AdaptCounter& GetAdaptCounter() RUN_ON(&encoder_queue_); | |
| 221 const AdaptCounter& GetConstAdaptCounter() RUN_ON(&encoder_queue_); | |
| 222 void UpdateAdaptationStats(AdaptReason reason) RUN_ON(&encoder_queue_); | |
| 223 AdaptCounts GetActiveCounts(AdaptReason reason) RUN_ON(&encoder_queue_); | |
| 224 | |
| 225 rtc::Event shutdown_event_; | |
| 226 | |
| 227 const uint32_t number_of_cores_; | |
| 228 // Counts how many frames we've dropped in the initial rampup phase. | |
| 229 int initial_rampup_; | |
| 230 | |
| 231 const std::unique_ptr<VideoSourceProxy> source_proxy_; | |
| 232 EncoderSink* sink_; | |
| 233 const VideoSendStream::Config::EncoderSettings settings_; | |
| 234 const VideoCodecType codec_type_; | |
| 235 | |
| 236 vcm::VideoSender video_sender_ ACCESS_ON(&encoder_queue_); | |
| 237 std::unique_ptr<OveruseFrameDetector> overuse_detector_ | |
| 238 ACCESS_ON(&encoder_queue_); | |
| 239 std::unique_ptr<QualityScaler> quality_scaler_ ACCESS_ON(&encoder_queue_); | |
| 240 | |
| 241 SendStatisticsProxy* const stats_proxy_; | |
| 242 rtc::VideoSinkInterface<VideoFrame>* const pre_encode_callback_; | |
| 243 ProcessThread* module_process_thread_; | |
| 244 rtc::ThreadChecker module_process_thread_checker_; | |
| 245 // |thread_checker_| checks that public methods that are related to lifetime | |
| 246 // of ViEEncoder are called on the same thread. | |
| 247 rtc::ThreadChecker thread_checker_; | |
| 248 | |
| 249 VideoEncoderConfig encoder_config_ ACCESS_ON(&encoder_queue_); | |
| 250 std::unique_ptr<VideoBitrateAllocator> rate_allocator_ | |
| 251 ACCESS_ON(&encoder_queue_); | |
| 252 // The maximum frame rate of the current codec configuration, as determined | |
| 253 // at the last ReconfigureEncoder() call. | |
| 254 int max_framerate_ ACCESS_ON(&encoder_queue_); | |
| 255 | |
| 256 // Set when ConfigureEncoder has been called in order to lazy reconfigure the | |
| 257 // encoder on the next frame. | |
| 258 bool pending_encoder_reconfiguration_ ACCESS_ON(&encoder_queue_); | |
| 259 rtc::Optional<VideoFrameInfo> last_frame_info_ ACCESS_ON(&encoder_queue_); | |
| 260 int crop_width_ ACCESS_ON(&encoder_queue_); | |
| 261 int crop_height_ ACCESS_ON(&encoder_queue_); | |
| 262 uint32_t encoder_start_bitrate_bps_ ACCESS_ON(&encoder_queue_); | |
| 263 size_t max_data_payload_length_ ACCESS_ON(&encoder_queue_); | |
| 264 bool nack_enabled_ ACCESS_ON(&encoder_queue_); | |
| 265 uint32_t last_observed_bitrate_bps_ ACCESS_ON(&encoder_queue_); | |
| 266 bool encoder_paused_and_dropped_frame_ ACCESS_ON(&encoder_queue_); | |
| 267 Clock* const clock_; | |
| 268 // Counters used for deciding if the video resolution or framerate is | |
| 269 // currently restricted, and if so, why, on a per degradation preference | |
| 270 // basis. | |
| 271 // TODO(sprang): Replace this with a state holding a relative overuse measure | |
| 272 // instead, that can be translated into suitable down-scale or fps limit. | |
| 273 std::map<const VideoSendStream::DegradationPreference, AdaptCounter> | |
| 274 adapt_counters_ ACCESS_ON(&encoder_queue_); | |
| 275 // Set depending on degradation preferences. | |
| 276 VideoSendStream::DegradationPreference degradation_preference_ | |
| 277 ACCESS_ON(&encoder_queue_); | |
| 278 | |
| 279 struct AdaptationRequest { | |
| 280 // The pixel count produced by the source at the time of the adaptation. | |
| 281 int input_pixel_count_; | |
| 282 // Framerate received from the source at the time of the adaptation. | |
| 283 int framerate_fps_; | |
| 284 // Indicates if request was to adapt up or down. | |
| 285 enum class Mode { kAdaptUp, kAdaptDown } mode_; | |
| 286 }; | |
| 287 // Stores a snapshot of the last adaptation request triggered by an AdaptUp | |
| 288 // or AdaptDown signal. | |
| 289 rtc::Optional<AdaptationRequest> last_adaptation_request_ | |
| 290 ACCESS_ON(&encoder_queue_); | |
| 291 | |
| 292 rtc::RaceChecker incoming_frame_race_checker_ | |
| 293 GUARDED_BY(incoming_frame_race_checker_); | |
| 294 Atomic32 posted_frames_waiting_for_encode_; | |
| 295 // Used to make sure incoming time stamp is increasing for every frame. | |
| 296 int64_t last_captured_timestamp_ GUARDED_BY(incoming_frame_race_checker_); | |
| 297 // Delta used for translating between NTP and internal timestamps. | |
| 298 const int64_t delta_ntp_internal_ms_ GUARDED_BY(incoming_frame_race_checker_); | |
| 299 | |
| 300 int64_t last_frame_log_ms_ GUARDED_BY(incoming_frame_race_checker_); | |
| 301 int captured_frame_count_ ACCESS_ON(&encoder_queue_); | |
| 302 int dropped_frame_count_ ACCESS_ON(&encoder_queue_); | |
| 303 | |
| 304 VideoBitrateAllocationObserver* bitrate_observer_ ACCESS_ON(&encoder_queue_); | |
| 305 rtc::Optional<int64_t> last_parameters_update_ms_ ACCESS_ON(&encoder_queue_); | |
| 306 | |
| 307 // All public methods are proxied to |encoder_queue_|. It must must be | |
| 308 // destroyed first to make sure no tasks are run that use other members. | |
| 309 rtc::TaskQueue encoder_queue_; | |
| 310 | |
| 311 RTC_DISALLOW_COPY_AND_ASSIGN(ViEEncoder); | |
| 312 }; | |
| 313 | |
| 314 } // namespace webrtc | |
| 315 | |
| 316 #endif // WEBRTC_VIDEO_VIE_ENCODER_H_ | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2995433002:
Rename ViEEncoder to VideoStreamEncoder. (Closed)
