OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #ifndef WEBRTC_VIDEO_VIE_ENCODER_H_ | 11 #ifndef WEBRTC_VIDEO_VIE_ENCODER_H_ |
12 #define WEBRTC_VIDEO_VIE_ENCODER_H_ | 12 #define WEBRTC_VIDEO_VIE_ENCODER_H_ |
13 | 13 |
14 #include <memory> | 14 #include <memory> |
15 #include <string> | 15 #include <string> |
16 #include <vector> | 16 #include <vector> |
17 | 17 |
18 #include "webrtc/base/criticalsection.h" | 18 #include "webrtc/base/criticalsection.h" |
19 #include "webrtc/base/event.h" | 19 #include "webrtc/base/scoped_ref_ptr.h" |
20 #include "webrtc/base/sequenced_task_checker.h" | 20 #include "webrtc/base/thread_annotations.h" |
21 #include "webrtc/base/task_queue.h" | |
22 #include "webrtc/call.h" | |
23 #include "webrtc/common_types.h" | 21 #include "webrtc/common_types.h" |
| 22 #include "webrtc/video_encoder.h" |
24 #include "webrtc/media/base/videosinkinterface.h" | 23 #include "webrtc/media/base/videosinkinterface.h" |
| 24 #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h" |
25 #include "webrtc/modules/video_coding/include/video_coding_defines.h" | 25 #include "webrtc/modules/video_coding/include/video_coding_defines.h" |
26 #include "webrtc/modules/video_coding/video_coding_impl.h" | 26 #include "webrtc/modules/video_coding/video_coding_impl.h" |
27 #include "webrtc/modules/video_processing/include/video_processing.h" | 27 #include "webrtc/modules/video_processing/include/video_processing.h" |
28 #include "webrtc/system_wrappers/include/atomic32.h" | |
29 #include "webrtc/video/overuse_frame_detector.h" | |
30 #include "webrtc/video_encoder.h" | |
31 #include "webrtc/video_send_stream.h" | |
32 #include "webrtc/typedefs.h" | 28 #include "webrtc/typedefs.h" |
33 | 29 |
34 namespace webrtc { | 30 namespace webrtc { |
35 | 31 |
| 32 class Config; |
| 33 class EncodedImageCallback; |
| 34 class OveruseFrameDetector; |
| 35 class PacedSender; |
36 class ProcessThread; | 36 class ProcessThread; |
37 class SendStatisticsProxy; | 37 class SendStatisticsProxy; |
| 38 class ViEBitrateObserver; |
| 39 class ViEEffectFilter; |
| 40 class VideoEncoder; |
38 | 41 |
39 // VieEncoder represent a video encoder that accepts raw video frames as input | 42 // VieEncoder represent a video encoder that accepts raw video frames as input |
40 // and produces an encoded bit stream. | 43 // and produces an encoded bit stream. |
41 // Usage: | 44 // Usage: |
42 // Instantiate. | 45 // 1. Instantiate |
43 // Call SetStartRate and SetSink. | 46 // 2. Call Init |
44 // Call ConfigureEncoder with the codec settings. | 47 // 3. Call RegisterExternalEncoder if available. |
45 // Provide frames to encode by calling IncomingCapturedFrame. | 48 // 4. Call SetEncoder with the codec settings and the object that shall receive |
46 // Call Stop() when done. | 49 // the encoded bit stream. |
47 class ViEEncoder : public VideoCaptureInput, | 50 // 5. For each available raw video frame call EncodeVideoFrame. |
48 public EncodedImageCallback, | 51 class ViEEncoder : public EncodedImageCallback, |
49 public VCMSendStatisticsCallback, | 52 public VCMSendStatisticsCallback { |
50 public CpuOveruseObserver { | |
51 public: | 53 public: |
| 54 friend class ViEBitrateObserver; |
| 55 |
52 ViEEncoder(uint32_t number_of_cores, | 56 ViEEncoder(uint32_t number_of_cores, |
| 57 ProcessThread* module_process_thread, |
53 SendStatisticsProxy* stats_proxy, | 58 SendStatisticsProxy* stats_proxy, |
54 const webrtc::VideoSendStream::Config::EncoderSettings& settings, | 59 OveruseFrameDetector* overuse_detector, |
55 rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback, | 60 EncodedImageCallback* sink); |
56 LoadObserver* overuse_callback, | |
57 EncodedFrameObserver* encoder_timing); | |
58 ~ViEEncoder(); | 61 ~ViEEncoder(); |
59 // RegisterProcessThread register |module_process_thread| with those objects | |
60 // that use it. Registration has to happen on the thread where | |
61 // |module_process_thread| was created (libjingle's worker thread). | |
62 // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue. | |
63 void RegisterProcessThread(ProcessThread* module_process_thread); | |
64 void DeRegisterProcessThread(); | |
65 | 62 |
66 void SetSink(EncodedImageCallback* sink); | 63 vcm::VideoSender* video_sender(); |
67 | 64 |
68 // TODO(perkj): Can we remove VideoCodec.startBitrate ? | 65 // Returns the id of the owning channel. |
69 void SetStartBitrate(int start_bitrate_bps); | 66 int Owner() const; |
70 | 67 |
71 void ConfigureEncoder(const VideoEncoderConfig& config, | 68 // Codec settings. |
72 size_t max_data_payload_length); | 69 int32_t RegisterExternalEncoder(VideoEncoder* encoder, |
| 70 uint8_t pl_type, |
| 71 bool internal_source); |
| 72 int32_t DeRegisterExternalEncoder(uint8_t pl_type); |
| 73 void SetEncoder(const VideoCodec& video_codec, |
| 74 size_t max_data_payload_length); |
73 | 75 |
74 // Permanently stop encoding. After this method has returned, it is | 76 void EncodeVideoFrame(const VideoFrame& video_frame); |
75 // guaranteed that no encoded frames will be delivered to the sink. | 77 void SendKeyFrame(); |
76 void Stop(); | |
77 | 78 |
78 // Implements VideoCaptureInput. | 79 // Returns the time when the encoder last received an input frame or produced |
79 // TODO(perkj): Refactor ViEEncoder to inherit rtc::VideoSink instead of | 80 // an encoded frame. |
80 // VideoCaptureInput. | 81 int64_t time_of_last_frame_activity_ms(); |
81 void IncomingCapturedFrame(const VideoFrame& video_frame) override; | |
82 | 82 |
83 void SendKeyFrame(); | 83 |
| 84 // Implements EncodedImageCallback. |
| 85 EncodedImageCallback::Result OnEncodedImage( |
| 86 const EncodedImage& encoded_image, |
| 87 const CodecSpecificInfo* codec_specific_info, |
| 88 const RTPFragmentationHeader* fragmentation) override; |
| 89 |
| 90 // Implements VideoSendStatisticsCallback. |
| 91 void SendStatistics(uint32_t bit_rate, |
| 92 uint32_t frame_rate, |
| 93 const std::string& encoder_name) override; |
84 | 94 |
85 // virtual to test EncoderStateFeedback with mocks. | 95 // virtual to test EncoderStateFeedback with mocks. |
86 virtual void OnReceivedIntraFrameRequest(size_t stream_index); | 96 virtual void OnReceivedIntraFrameRequest(size_t stream_index); |
87 virtual void OnReceivedSLI(uint8_t picture_id); | 97 virtual void OnReceivedSLI(uint8_t picture_id); |
88 virtual void OnReceivedRPSI(uint64_t picture_id); | 98 virtual void OnReceivedRPSI(uint64_t picture_id); |
89 | 99 |
90 void OnBitrateUpdated(uint32_t bitrate_bps, | 100 void OnBitrateUpdated(uint32_t bitrate_bps, |
91 uint8_t fraction_lost, | 101 uint8_t fraction_lost, |
92 int64_t round_trip_time_ms); | 102 int64_t round_trip_time_ms); |
93 | 103 |
94 private: | 104 private: |
95 class EncodeTask; | 105 bool EncoderPaused() const EXCLUSIVE_LOCKS_REQUIRED(data_cs_); |
96 | 106 void TraceFrameDropStart() EXCLUSIVE_LOCKS_REQUIRED(data_cs_); |
97 void ConfigureEncoderInternal(const VideoCodec& video_codec, | 107 void TraceFrameDropEnd() EXCLUSIVE_LOCKS_REQUIRED(data_cs_); |
98 size_t max_data_payload_length); | |
99 | |
100 // Implements VideoSendStatisticsCallback. | |
101 void SendStatistics(uint32_t bit_rate, | |
102 uint32_t frame_rate, | |
103 const std::string& encoder_name) override; | |
104 | |
105 void EncodeVideoFrame(const VideoFrame& frame); | |
106 | |
107 // Implements EncodedImageCallback. | |
108 EncodedImageCallback::Result OnEncodedImage( | |
109 const EncodedImage& encoded_image, | |
110 const CodecSpecificInfo* codec_specific_info, | |
111 const RTPFragmentationHeader* fragmentation) override; | |
112 | |
113 // webrtc::CpuOveruseObserver implementation. | |
114 void OveruseDetected() override; | |
115 void NormalUsage() override; | |
116 | |
117 bool EncoderPaused() const; | |
118 void TraceFrameDropStart(); | |
119 void TraceFrameDropEnd(); | |
120 | |
121 rtc::Event shutdown_event_; | |
122 | 108 |
123 const uint32_t number_of_cores_; | 109 const uint32_t number_of_cores_; |
124 EncodedImageCallback* sink_; | 110 EncodedImageCallback* const sink_; |
125 const VideoSendStream::Config::EncoderSettings settings_; | |
126 | 111 |
127 const std::unique_ptr<VideoProcessing> vp_; | 112 const std::unique_ptr<VideoProcessing> vp_; |
128 vcm::VideoSender video_sender_ ACCESS_ON(&encoder_queue_); | 113 vcm::VideoSender video_sender_; |
129 OveruseFrameDetector overuse_detector_; | 114 |
130 LoadObserver* const load_observer_ ACCESS_ON(&module_process_thread_checker_); | 115 rtc::CriticalSection data_cs_; |
131 | 116 |
132 SendStatisticsProxy* const stats_proxy_; | 117 SendStatisticsProxy* const stats_proxy_; |
133 rtc::VideoSinkInterface<VideoFrame>* const pre_encode_callback_; | 118 OveruseFrameDetector* const overuse_detector_; |
| 119 |
| 120 // The time we last received an input frame or encoded frame. This is used to |
| 121 // track when video is stopped long enough that we also want to stop sending |
| 122 // padding. |
| 123 int64_t time_of_last_frame_activity_ms_ GUARDED_BY(data_cs_); |
| 124 VideoCodec encoder_config_ GUARDED_BY(data_cs_); |
| 125 uint32_t last_observed_bitrate_bps_ GUARDED_BY(data_cs_); |
| 126 bool encoder_paused_and_dropped_frame_ GUARDED_BY(data_cs_); |
| 127 |
134 ProcessThread* module_process_thread_; | 128 ProcessThread* module_process_thread_; |
135 rtc::ThreadChecker module_process_thread_checker_; | |
136 | 129 |
137 VideoCodec encoder_config_ ACCESS_ON(&encoder_queue_); | 130 bool has_received_sli_ GUARDED_BY(data_cs_); |
| 131 uint8_t picture_id_sli_ GUARDED_BY(data_cs_); |
| 132 bool has_received_rpsi_ GUARDED_BY(data_cs_); |
| 133 uint64_t picture_id_rpsi_ GUARDED_BY(data_cs_); |
138 | 134 |
139 int encoder_start_bitrate_bps_ ACCESS_ON(&encoder_queue_); | 135 bool video_suspended_ GUARDED_BY(data_cs_); |
140 uint32_t last_observed_bitrate_bps_ ACCESS_ON(&encoder_queue_); | |
141 bool encoder_paused_and_dropped_frame_ ACCESS_ON(&encoder_queue_); | |
142 bool has_received_sli_ ACCESS_ON(&encoder_queue_); | |
143 uint8_t picture_id_sli_ ACCESS_ON(&encoder_queue_); | |
144 bool has_received_rpsi_ ACCESS_ON(&encoder_queue_); | |
145 uint64_t picture_id_rpsi_ ACCESS_ON(&encoder_queue_); | |
146 Clock* const clock_; | |
147 | |
148 rtc::RaceChecker incoming_frame_race_checker_; | |
149 Atomic32 posted_frames_waiting_for_encode_; | |
150 // Used to make sure incoming time stamp is increasing for every frame. | |
151 int64_t last_captured_timestamp_ GUARDED_BY(incoming_frame_race_checker_); | |
152 // Delta used for translating between NTP and internal timestamps. | |
153 const int64_t delta_ntp_internal_ms_; | |
154 | |
155 // All public methods are proxied to |encoder_queue_|. It must must be | |
156 // destroyed first to make sure no tasks are run that use other members. | |
157 rtc::TaskQueue encoder_queue_; | |
158 }; | 136 }; |
159 | 137 |
160 } // namespace webrtc | 138 } // namespace webrtc |
161 | 139 |
162 #endif // WEBRTC_VIDEO_VIE_ENCODER_H_ | 140 #endif // WEBRTC_VIDEO_VIE_ENCODER_H_ |
OLD | NEW |