OLD | NEW |
---|---|
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include "webrtc/modules/video_coding/generic_encoder.h" | 11 #include "webrtc/modules/video_coding/generic_encoder.h" |
12 | 12 |
13 #include <vector> | 13 #include <vector> |
14 | 14 |
15 #include "webrtc/api/video/i420_buffer.h" | 15 #include "webrtc/api/video/i420_buffer.h" |
16 #include "webrtc/base/checks.h" | 16 #include "webrtc/base/checks.h" |
17 #include "webrtc/base/logging.h" | 17 #include "webrtc/base/logging.h" |
18 #include "webrtc/base/timeutils.h" | |
18 #include "webrtc/base/trace_event.h" | 19 #include "webrtc/base/trace_event.h" |
19 #include "webrtc/modules/video_coding/encoded_frame.h" | 20 #include "webrtc/modules/video_coding/encoded_frame.h" |
20 #include "webrtc/modules/video_coding/media_optimization.h" | 21 #include "webrtc/modules/video_coding/media_optimization.h" |
21 | 22 |
22 namespace webrtc { | 23 namespace webrtc { |
23 | 24 |
24 VCMGenericEncoder::VCMGenericEncoder( | 25 VCMGenericEncoder::VCMGenericEncoder( |
25 VideoEncoder* encoder, | 26 VideoEncoder* encoder, |
26 VCMEncodedFrameCallback* encoded_frame_callback, | 27 VCMEncodedFrameCallback* encoded_frame_callback, |
27 bool internal_source) | 28 bool internal_source) |
28 : encoder_(encoder), | 29 : encoder_(encoder), |
29 vcm_encoded_frame_callback_(encoded_frame_callback), | 30 vcm_encoded_frame_callback_(encoded_frame_callback), |
30 internal_source_(internal_source), | 31 internal_source_(internal_source), |
31 encoder_params_({BitrateAllocation(), 0, 0, 0}), | 32 encoder_params_({BitrateAllocation(), 0, 0, 0}), |
32 is_screenshare_(false) {} | 33 is_screenshare_(false), |
34 streams_or_svc_num_(0) {} | |
33 | 35 |
34 VCMGenericEncoder::~VCMGenericEncoder() {} | 36 VCMGenericEncoder::~VCMGenericEncoder() {} |
35 | 37 |
36 int32_t VCMGenericEncoder::Release() { | 38 int32_t VCMGenericEncoder::Release() { |
37 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); | 39 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); |
38 TRACE_EVENT0("webrtc", "VCMGenericEncoder::Release"); | 40 TRACE_EVENT0("webrtc", "VCMGenericEncoder::Release"); |
39 return encoder_->Release(); | 41 return encoder_->Release(); |
40 } | 42 } |
41 | 43 |
42 int32_t VCMGenericEncoder::InitEncode(const VideoCodec* settings, | 44 int32_t VCMGenericEncoder::InitEncode(const VideoCodec* settings, |
43 int32_t number_of_cores, | 45 int32_t number_of_cores, |
44 size_t max_payload_size) { | 46 size_t max_payload_size) { |
45 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); | 47 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); |
46 TRACE_EVENT0("webrtc", "VCMGenericEncoder::InitEncode"); | 48 TRACE_EVENT0("webrtc", "VCMGenericEncoder::InitEncode"); |
47 is_screenshare_ = settings->mode == VideoCodecMode::kScreensharing; | 49 is_screenshare_ = settings->mode == VideoCodecMode::kScreensharing; |
50 streams_or_svc_num_ = settings->numberOfSimulcastStreams; | |
51 if (settings->codecType == kVideoCodecVP9) { | |
52 streams_or_svc_num_ = settings->VP9().numberOfSpatialLayers; | |
53 } | |
54 if (streams_or_svc_num_ == 0) | |
55 streams_or_svc_num_ = 1; | |
56 | |
57 vcm_encoded_frame_callback_->SetTimingFramesThresholds( | |
58 settings->timingFrameTriggerThresholds); | |
59 vcm_encoded_frame_callback_->OnFrameRateChanged(settings->maxFramerate); | |
60 | |
48 if (encoder_->InitEncode(settings, number_of_cores, max_payload_size) != 0) { | 61 if (encoder_->InitEncode(settings, number_of_cores, max_payload_size) != 0) { |
49 LOG(LS_ERROR) << "Failed to initialize the encoder associated with " | 62 LOG(LS_ERROR) << "Failed to initialize the encoder associated with " |
50 "payload name: " | 63 "payload name: " |
51 << settings->plName; | 64 << settings->plName; |
52 return -1; | 65 return -1; |
53 } | 66 } |
54 encoder_->RegisterEncodeCompleteCallback(vcm_encoded_frame_callback_); | 67 encoder_->RegisterEncodeCompleteCallback(vcm_encoded_frame_callback_); |
55 return 0; | 68 return 0; |
56 } | 69 } |
57 | 70 |
58 int32_t VCMGenericEncoder::Encode(const VideoFrame& frame, | 71 int32_t VCMGenericEncoder::Encode(const VideoFrame& frame, |
59 const CodecSpecificInfo* codec_specific, | 72 const CodecSpecificInfo* codec_specific, |
60 const std::vector<FrameType>& frame_types) { | 73 const std::vector<FrameType>& frame_types) { |
61 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); | 74 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); |
62 TRACE_EVENT1("webrtc", "VCMGenericEncoder::Encode", "timestamp", | 75 TRACE_EVENT1("webrtc", "VCMGenericEncoder::Encode", "timestamp", |
63 frame.timestamp()); | 76 frame.timestamp()); |
64 | 77 |
65 for (FrameType frame_type : frame_types) | 78 for (FrameType frame_type : frame_types) |
66 RTC_DCHECK(frame_type == kVideoFrameKey || frame_type == kVideoFrameDelta); | 79 RTC_DCHECK(frame_type == kVideoFrameKey || frame_type == kVideoFrameDelta); |
67 | 80 |
81 for (size_t i = 0; i < streams_or_svc_num_; ++i) | |
82 vcm_encoded_frame_callback_->OnEncodeStarted(frame.render_time_ms(), i); | |
68 int32_t result = encoder_->Encode(frame, codec_specific, &frame_types); | 83 int32_t result = encoder_->Encode(frame, codec_specific, &frame_types); |
69 | 84 |
70 if (is_screenshare_ && | 85 if (is_screenshare_ && |
71 result == WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT) { | 86 result == WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT) { |
72 // Target bitrate exceeded, encoder state has been reset - try again. | 87 // Target bitrate exceeded, encoder state has been reset - try again. |
73 return encoder_->Encode(frame, codec_specific, &frame_types); | 88 return encoder_->Encode(frame, codec_specific, &frame_types); |
74 } | 89 } |
75 | 90 |
76 return result; | 91 return result; |
77 } | 92 } |
(...skipping 22 matching lines...) Expand all Loading... | |
100 } | 115 } |
101 if (rates_have_changed) { | 116 if (rates_have_changed) { |
102 int res = encoder_->SetRateAllocation(params.target_bitrate, | 117 int res = encoder_->SetRateAllocation(params.target_bitrate, |
103 params.input_frame_rate); | 118 params.input_frame_rate); |
104 if (res != 0) { | 119 if (res != 0) { |
105 LOG(LS_WARNING) << "Error set encoder rate (total bitrate bps = " | 120 LOG(LS_WARNING) << "Error set encoder rate (total bitrate bps = " |
106 << params.target_bitrate.get_sum_bps() | 121 << params.target_bitrate.get_sum_bps() |
107 << ", framerate = " << params.input_frame_rate | 122 << ", framerate = " << params.input_frame_rate |
108 << "): " << res; | 123 << "): " << res; |
109 } | 124 } |
125 vcm_encoded_frame_callback_->OnFrameRateChanged(params.input_frame_rate); | |
126 for (size_t i = 0; i < streams_or_svc_num_; ++i) { | |
127 size_t layer_bitrate_bytes_per_sec = | |
128 params.target_bitrate.GetSpatialLayerSum(i) / 8; | |
129 // VP9 rate control is not yet moved out of VP9Impl. Due to that rates | |
130 // are not split among spatial layers. | |
131 if (layer_bitrate_bytes_per_sec == 0) | |
132 layer_bitrate_bytes_per_sec = params.target_bitrate.get_sum_bps() / 8; | |
133 vcm_encoded_frame_callback_->OnTargetBitrateChanged( | |
134 layer_bitrate_bytes_per_sec, i); | |
135 } | |
110 } | 136 } |
111 } | 137 } |
112 | 138 |
113 EncoderParameters VCMGenericEncoder::GetEncoderParameters() const { | 139 EncoderParameters VCMGenericEncoder::GetEncoderParameters() const { |
114 rtc::CritScope lock(¶ms_lock_); | 140 rtc::CritScope lock(¶ms_lock_); |
115 return encoder_params_; | 141 return encoder_params_; |
116 } | 142 } |
117 | 143 |
118 int32_t VCMGenericEncoder::SetPeriodicKeyFrames(bool enable) { | 144 int32_t VCMGenericEncoder::SetPeriodicKeyFrames(bool enable) { |
119 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); | 145 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); |
120 return encoder_->SetPeriodicKeyFrames(enable); | 146 return encoder_->SetPeriodicKeyFrames(enable); |
121 } | 147 } |
122 | 148 |
123 int32_t VCMGenericEncoder::RequestFrame( | 149 int32_t VCMGenericEncoder::RequestFrame( |
124 const std::vector<FrameType>& frame_types) { | 150 const std::vector<FrameType>& frame_types) { |
125 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); | 151 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); |
126 | 152 |
153 for (size_t i = 0; i < streams_or_svc_num_; ++i) | |
154 vcm_encoded_frame_callback_->OnEncodeStarted(0, i); | |
127 // TODO(nisse): Used only with internal source. Delete as soon as | 155 // TODO(nisse): Used only with internal source. Delete as soon as |
128 // that feature is removed. The only implementation I've been able | 156 // that feature is removed. The only implementation I've been able |
129 // to find ignores what's in the frame. With one exception: It seems | 157 // to find ignores what's in the frame. With one exception: It seems |
130 // a few test cases, e.g., | 158 // a few test cases, e.g., |
131 // VideoSendStreamTest.VideoSendStreamStopSetEncoderRateToZero, set | 159 // VideoSendStreamTest.VideoSendStreamStopSetEncoderRateToZero, set |
132 // internal_source to true and use FakeEncoder. And the latter will | 160 // internal_source to true and use FakeEncoder. And the latter will |
133 // happily encode this 1x1 frame and pass it on down the pipeline. | 161 // happily encode this 1x1 frame and pass it on down the pipeline. |
134 return encoder_->Encode(VideoFrame(I420Buffer::Create(1, 1), | 162 return encoder_->Encode(VideoFrame(I420Buffer::Create(1, 1), |
135 kVideoRotation_0, 0), | 163 kVideoRotation_0, 0), |
136 NULL, &frame_types); | 164 NULL, &frame_types); |
137 return 0; | 165 return 0; |
138 } | 166 } |
139 | 167 |
140 bool VCMGenericEncoder::InternalSource() const { | 168 bool VCMGenericEncoder::InternalSource() const { |
141 return internal_source_; | 169 return internal_source_; |
142 } | 170 } |
143 | 171 |
144 bool VCMGenericEncoder::SupportsNativeHandle() const { | 172 bool VCMGenericEncoder::SupportsNativeHandle() const { |
145 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); | 173 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); |
146 return encoder_->SupportsNativeHandle(); | 174 return encoder_->SupportsNativeHandle(); |
147 } | 175 } |
148 | 176 |
149 VCMEncodedFrameCallback::VCMEncodedFrameCallback( | 177 VCMEncodedFrameCallback::VCMEncodedFrameCallback( |
150 EncodedImageCallback* post_encode_callback, | 178 EncodedImageCallback* post_encode_callback, |
151 media_optimization::MediaOptimization* media_opt) | 179 media_optimization::MediaOptimization* media_opt) |
152 : internal_source_(false), | 180 : internal_source_(false), |
153 post_encode_callback_(post_encode_callback), | 181 post_encode_callback_(post_encode_callback), |
154 media_opt_(media_opt) {} | 182 media_opt_(media_opt), |
183 framerate_(1), | |
184 last_timing_frame_time_ms_(-1), | |
185 timing_frames_thresholds_({-1, 0}) {} | |
155 | 186 |
156 VCMEncodedFrameCallback::~VCMEncodedFrameCallback() {} | 187 VCMEncodedFrameCallback::~VCMEncodedFrameCallback() {} |
157 | 188 |
189 void VCMEncodedFrameCallback::OnTargetBitrateChanged( | |
190 size_t bitrate_bytes_per_second, | |
191 size_t simulcast_svc_idx) { | |
192 if (timing_frames_info_.size() < simulcast_svc_idx + 1) | |
193 timing_frames_info_.resize(simulcast_svc_idx + 1); | |
194 timing_frames_info_[simulcast_svc_idx].target_bitrate_bytes_per_sec = | |
195 bitrate_bytes_per_second; | |
196 } | |
197 | |
198 void VCMEncodedFrameCallback::OnFrameRateChanged(size_t framerate) { | |
199 framerate_ = framerate; | |
200 } | |
201 | |
202 void VCMEncodedFrameCallback::OnEncodeStarted(int64_t capture_time_ms, | |
203 size_t simulcast_svc_idx) { | |
204 if (timing_frames_info_.size() < simulcast_svc_idx + 1) | |
205 timing_frames_info_.resize(simulcast_svc_idx + 1); | |
206 timing_frames_info_[simulcast_svc_idx].encode_start_time_ms[capture_time_ms] = | |
207 rtc::TimeMillis(); | |
208 } | |
209 | |
158 EncodedImageCallback::Result VCMEncodedFrameCallback::OnEncodedImage( | 210 EncodedImageCallback::Result VCMEncodedFrameCallback::OnEncodedImage( |
159 const EncodedImage& encoded_image, | 211 const EncodedImage& encoded_image, |
160 const CodecSpecificInfo* codec_specific, | 212 const CodecSpecificInfo* codec_specific, |
161 const RTPFragmentationHeader* fragmentation_header) { | 213 const RTPFragmentationHeader* fragmentation_header) { |
162 TRACE_EVENT_INSTANT1("webrtc", "VCMEncodedFrameCallback::Encoded", | 214 TRACE_EVENT_INSTANT1("webrtc", "VCMEncodedFrameCallback::Encoded", |
163 "timestamp", encoded_image._timeStamp); | 215 "timestamp", encoded_image._timeStamp); |
216 | |
217 bool is_timing_frame = false; | |
218 size_t simulcast_svc_idx = 0; | |
219 if (codec_specific->codecType == kVideoCodecVP9) { | |
220 if (codec_specific->codecSpecific.VP9.num_spatial_layers > 1) | |
221 simulcast_svc_idx = codec_specific->codecSpecific.VP9.spatial_idx; | |
222 } else if (codec_specific->codecType == kVideoCodecVP8) { | |
223 simulcast_svc_idx = codec_specific->codecSpecific.VP8.simulcastIdx; | |
224 } else if (codec_specific->codecType == kVideoCodecGeneric) { | |
225 simulcast_svc_idx = codec_specific->codecSpecific.generic.simulcast_idx; | |
226 } else if (codec_specific->codecType == kVideoCodecH264) { | |
227 // TODO(ilnik): When h264 simulcast is landed, extract simulcast idx here. | |
228 } | |
229 RTC_CHECK_LT(simulcast_svc_idx, timing_frames_info_.size()); | |
230 | |
231 int64_t encode_start_ms = -1; | |
232 auto it = timing_frames_info_[simulcast_svc_idx].encode_start_time_ms.find( | |
233 encoded_image.capture_time_ms_); | |
sprang_webrtc
2017/06/09 11:02:01
nit: maybe you could store timing_frames_info_[sim
ilnik
2017/06/09 12:06:05
Done.
| |
234 if (it != timing_frames_info_[simulcast_svc_idx].encode_start_time_ms.end()) { | |
235 encode_start_ms = it->second; | |
236 // Assuming all encoders do not reorder frames within single stream, | |
237 // there may be some dropped frames with smaller timestamps. These should be | |
238 // purged. | |
239 timing_frames_info_[simulcast_svc_idx].encode_start_time_ms.erase( | |
240 timing_frames_info_[simulcast_svc_idx].encode_start_time_ms.begin(), | |
241 it); | |
242 timing_frames_info_[simulcast_svc_idx].encode_start_time_ms.erase(it); | |
243 } | |
244 RTC_DCHECK_NE(encode_start_ms, -1); | |
245 | |
246 int64_t timing_frame_delay_ms = | |
247 encoded_image.capture_time_ms_ - last_timing_frame_time_ms_; | |
248 if (last_timing_frame_time_ms_ == -1 || | |
249 timing_frame_delay_ms >= timing_frames_thresholds_.delay_ms || | |
250 timing_frame_delay_ms == 0) { | |
251 is_timing_frame = true; | |
252 last_timing_frame_time_ms_ = encoded_image.capture_time_ms_; | |
253 } | |
254 RTC_CHECK_GT(framerate_, 0); | |
255 size_t outlier_frame_size = | |
256 timing_frames_thresholds_.outlier_ratio_percent / 100.0 * | |
257 timing_frames_info_[simulcast_svc_idx].target_bitrate_bytes_per_sec / | |
258 framerate_; | |
sprang_webrtc
2017/06/09 11:02:01
Could you clarify this by adding explicit parenthe
ilnik
2017/06/09 12:06:05
Split in two steps now. No float is needed now.
| |
259 if (encoded_image._length >= outlier_frame_size) { | |
260 is_timing_frame = true; | |
261 } | |
262 if (is_timing_frame) { | |
263 encoded_image.SetEncodeTime(encode_start_ms, rtc::TimeMillis()); | |
264 } | |
265 | |
164 Result result = post_encode_callback_->OnEncodedImage( | 266 Result result = post_encode_callback_->OnEncodedImage( |
165 encoded_image, codec_specific, fragmentation_header); | 267 encoded_image, codec_specific, fragmentation_header); |
166 if (result.error != Result::OK) | 268 if (result.error != Result::OK) |
167 return result; | 269 return result; |
168 | 270 |
169 if (media_opt_) { | 271 if (media_opt_) { |
170 media_opt_->UpdateWithEncodedData(encoded_image); | 272 media_opt_->UpdateWithEncodedData(encoded_image); |
171 if (internal_source_) { | 273 if (internal_source_) { |
172 // Signal to encoder to drop next frame. | 274 // Signal to encoder to drop next frame. |
173 result.drop_next_frame = media_opt_->DropFrame(); | 275 result.drop_next_frame = media_opt_->DropFrame(); |
174 } | 276 } |
175 } | 277 } |
176 return result; | 278 return result; |
177 } | 279 } |
178 | 280 |
179 } // namespace webrtc | 281 } // namespace webrtc |
OLD | NEW |