OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #include "webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h" | 11 #include "webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h" |
12 | 12 |
13 #include <algorithm> | 13 #include <algorithm> |
14 | 14 |
15 // NOTE(ajm): Path provided by gyp. | 15 // NOTE(ajm): Path provided by gyp. |
16 #include "libyuv/scale.h" // NOLINT | 16 #include "libyuv/scale.h" // NOLINT |
17 | 17 |
18 #include "webrtc/base/checks.h" | 18 #include "webrtc/base/checks.h" |
19 #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h" | 19 #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h" |
20 #include "webrtc/system_wrappers/include/clock.h" | 20 #include "webrtc/system_wrappers/include/clock.h" |
21 | 21 |
22 namespace { | 22 namespace { |
23 | 23 |
24 const unsigned int kDefaultMinQp = 2; | 24 const unsigned int kDefaultMinQp = 2; |
25 const unsigned int kDefaultMaxQp = 56; | 25 const unsigned int kDefaultMaxQp = 56; |
26 // Max qp for lowest spatial resolution when doing simulcast. | 26 // Max qp for lowest spatial resolution when doing simulcast. |
27 const unsigned int kLowestResMaxQp = 45; | 27 const unsigned int kLowestResMaxQp = 45; |
28 | 28 |
29 uint32_t SumStreamTargetBitrate(int streams, const webrtc::VideoCodec& codec) { | 29 static const char* kDefaultImplementationName = |
30 uint32_t bitrate_sum = 0; | 30 "SimulcastEncoderAdapter (uninitialized)"; |
31 for (int i = 0; i < streams; ++i) { | |
32 bitrate_sum += codec.simulcastStream[i].targetBitrate; | |
33 } | |
34 return bitrate_sum; | |
35 } | |
36 | |
37 uint32_t SumStreamMaxBitrate(int streams, const webrtc::VideoCodec& codec) { | |
38 uint32_t bitrate_sum = 0; | |
39 for (int i = 0; i < streams; ++i) { | |
40 bitrate_sum += codec.simulcastStream[i].maxBitrate; | |
41 } | |
42 return bitrate_sum; | |
43 } | |
44 | |
45 int NumberOfStreams(const webrtc::VideoCodec& codec) { | |
46 int streams = | |
47 codec.numberOfSimulcastStreams < 1 ? 1 : codec.numberOfSimulcastStreams; | |
48 uint32_t simulcast_max_bitrate = SumStreamMaxBitrate(streams, codec); | |
49 if (simulcast_max_bitrate == 0) { | |
50 streams = 1; | |
51 } | |
52 return streams; | |
53 } | |
54 | 31 |
55 bool ValidSimulcastResolutions(const webrtc::VideoCodec& codec, | 32 bool ValidSimulcastResolutions(const webrtc::VideoCodec& codec, |
56 int num_streams) { | 33 int num_streams) { |
57 if (codec.width != codec.simulcastStream[num_streams - 1].width || | 34 if (codec.width != codec.simulcastStream[num_streams - 1].width || |
58 codec.height != codec.simulcastStream[num_streams - 1].height) { | 35 codec.height != codec.simulcastStream[num_streams - 1].height) { |
59 return false; | 36 return false; |
60 } | 37 } |
61 for (int i = 0; i < num_streams; ++i) { | 38 for (int i = 0; i < num_streams; ++i) { |
62 if (codec.width * codec.simulcastStream[i].height != | 39 if (codec.width * codec.simulcastStream[i].height != |
63 codec.height * codec.simulcastStream[i].width) { | 40 codec.height * codec.simulcastStream[i].width) { |
(...skipping 21 matching lines...) Expand all Loading... |
85 inst->numberOfSimulcastStreams > 1) { | 62 inst->numberOfSimulcastStreams > 1) { |
86 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; | 63 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
87 } | 64 } |
88 if (inst->codecSpecific.VP8.automaticResizeOn && | 65 if (inst->codecSpecific.VP8.automaticResizeOn && |
89 inst->numberOfSimulcastStreams > 1) { | 66 inst->numberOfSimulcastStreams > 1) { |
90 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; | 67 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
91 } | 68 } |
92 return WEBRTC_VIDEO_CODEC_OK; | 69 return WEBRTC_VIDEO_CODEC_OK; |
93 } | 70 } |
94 | 71 |
95 // TL1 FrameDropper's max time to drop frames. | |
96 const float kTl1MaxTimeToDropFrames = 20.0f; | |
97 | |
98 struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory { | 72 struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory { |
99 ScreenshareTemporalLayersFactory() | 73 ScreenshareTemporalLayersFactory() {} |
100 : tl1_frame_dropper_(kTl1MaxTimeToDropFrames) {} | |
101 | |
102 virtual ~ScreenshareTemporalLayersFactory() {} | 74 virtual ~ScreenshareTemporalLayersFactory() {} |
103 | 75 |
104 virtual webrtc::TemporalLayers* Create(int num_temporal_layers, | 76 virtual webrtc::TemporalLayers* Create(int num_temporal_layers, |
105 uint8_t initial_tl0_pic_idx) const { | 77 uint8_t initial_tl0_pic_idx) const { |
106 return new webrtc::ScreenshareLayers(num_temporal_layers, rand(), | 78 return new webrtc::ScreenshareLayers(num_temporal_layers, rand(), |
107 webrtc::Clock::GetRealTimeClock()); | 79 webrtc::Clock::GetRealTimeClock()); |
108 } | 80 } |
109 | |
110 mutable webrtc::FrameDropper tl0_frame_dropper_; | |
111 mutable webrtc::FrameDropper tl1_frame_dropper_; | |
112 }; | 81 }; |
113 | 82 |
114 // An EncodedImageCallback implementation that forwards on calls to a | 83 // An EncodedImageCallback implementation that forwards on calls to a |
115 // SimulcastEncoderAdapter, but with the stream index it's registered with as | 84 // SimulcastEncoderAdapter, but with the stream index it's registered with as |
116 // the first parameter to Encoded. | 85 // the first parameter to Encoded. |
117 class AdapterEncodedImageCallback : public webrtc::EncodedImageCallback { | 86 class AdapterEncodedImageCallback : public webrtc::EncodedImageCallback { |
118 public: | 87 public: |
119 AdapterEncodedImageCallback(webrtc::SimulcastEncoderAdapter* adapter, | 88 AdapterEncodedImageCallback(webrtc::SimulcastEncoderAdapter* adapter, |
120 size_t stream_idx) | 89 size_t stream_idx) |
121 : adapter_(adapter), stream_idx_(stream_idx) {} | 90 : adapter_(adapter), stream_idx_(stream_idx) {} |
122 | 91 |
123 int32_t Encoded( | 92 int32_t Encoded( |
124 const webrtc::EncodedImage& encodedImage, | 93 const webrtc::EncodedImage& encodedImage, |
125 const webrtc::CodecSpecificInfo* codecSpecificInfo = NULL, | 94 const webrtc::CodecSpecificInfo* codecSpecificInfo = NULL, |
126 const webrtc::RTPFragmentationHeader* fragmentation = NULL) override { | 95 const webrtc::RTPFragmentationHeader* fragmentation = NULL) override { |
127 return adapter_->Encoded(stream_idx_, encodedImage, codecSpecificInfo, | 96 return adapter_->Encoded(stream_idx_, encodedImage, codecSpecificInfo, |
128 fragmentation); | 97 fragmentation); |
129 } | 98 } |
130 | 99 |
131 private: | 100 private: |
132 webrtc::SimulcastEncoderAdapter* const adapter_; | 101 webrtc::SimulcastEncoderAdapter* const adapter_; |
133 const size_t stream_idx_; | 102 const size_t stream_idx_; |
134 }; | 103 }; |
135 | 104 |
| 105 webrtc::VideoCodec DefaultCodec() { |
| 106 webrtc::VideoCodec codec; |
| 107 memset(&codec, 0, sizeof(webrtc::VideoCodec)); |
| 108 return codec; |
| 109 } |
| 110 |
136 } // namespace | 111 } // namespace |
137 | 112 |
138 namespace webrtc { | 113 namespace webrtc { |
139 | 114 |
140 SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory) | 115 SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory) |
141 : factory_(factory), | 116 : factory_(factory), |
142 encoded_complete_callback_(NULL), | 117 codec_(DefaultCodec()), |
143 implementation_name_("SimulcastEncoderAdapter") { | 118 stream_states_(codec_), |
144 memset(&codec_, 0, sizeof(webrtc::VideoCodec)); | 119 encoded_complete_callback_(nullptr), |
145 } | 120 implementation_name_(kDefaultImplementationName) {} |
146 | 121 |
147 SimulcastEncoderAdapter::~SimulcastEncoderAdapter() { | 122 SimulcastEncoderAdapter::~SimulcastEncoderAdapter() { |
148 Release(); | 123 Release(); |
149 } | 124 } |
150 | 125 |
151 int SimulcastEncoderAdapter::Release() { | 126 int SimulcastEncoderAdapter::Release() { |
152 // TODO(pbos): Keep the last encoder instance but call ::Release() on it, then | 127 // TODO(pbos): Keep the last encoder instance but call ::Release() on it, then |
153 // re-use this instance in ::InitEncode(). This means that changing | 128 // re-use this instance in ::InitEncode(). This means that changing |
154 // resolutions doesn't require reallocation of the first encoder, but only | 129 // resolutions doesn't require reallocation of the first encoder, but only |
155 // reinitialization, which makes sense. Then Destroy this instance instead in | 130 // reinitialization, which makes sense. Then Destroy this instance instead in |
156 // ~SimulcastEncoderAdapter(). | 131 // ~SimulcastEncoderAdapter(). |
157 while (!streaminfos_.empty()) { | 132 stream_states_ = SimulcastState(codec_); |
158 VideoEncoder* encoder = streaminfos_.back().encoder; | 133 for (auto& it : encoders_) |
159 EncodedImageCallback* callback = streaminfos_.back().callback; | 134 factory_->Destroy(it.second.first); |
160 factory_->Destroy(encoder); | 135 encoders_.clear(); |
161 delete callback; | 136 |
162 streaminfos_.pop_back(); | |
163 } | |
164 return WEBRTC_VIDEO_CODEC_OK; | 137 return WEBRTC_VIDEO_CODEC_OK; |
165 } | 138 } |
166 | 139 |
167 int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, | 140 int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, |
168 int number_of_cores, | 141 int number_of_cores, |
169 size_t max_payload_size) { | 142 size_t max_payload_size) { |
170 if (number_of_cores < 1) { | 143 if (number_of_cores < 1) { |
171 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; | 144 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
172 } | 145 } |
173 | 146 |
174 int ret = VerifyCodec(inst); | 147 int ret = VerifyCodec(inst); |
175 if (ret < 0) { | 148 if (ret < 0) { |
176 return ret; | 149 return ret; |
177 } | 150 } |
178 | 151 |
179 ret = Release(); | 152 ret = Release(); |
180 if (ret < 0) { | 153 if (ret < 0) { |
181 return ret; | 154 return ret; |
182 } | 155 } |
183 | 156 |
184 int number_of_streams = NumberOfStreams(*inst); | 157 std::unique_ptr<SimulcastState> simulcast(new SimulcastState(*inst)); |
| 158 |
| 159 int number_of_streams = simulcast->NumStreams(); |
185 const bool doing_simulcast = (number_of_streams > 1); | 160 const bool doing_simulcast = (number_of_streams > 1); |
186 | 161 |
187 if (doing_simulcast && !ValidSimulcastResolutions(*inst, number_of_streams)) { | 162 if (doing_simulcast && !ValidSimulcastResolutions(*inst, number_of_streams)) { |
188 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; | 163 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
189 } | 164 } |
190 | 165 |
191 codec_ = *inst; | 166 codec_ = *inst; |
192 | 167 |
193 // Special mode when screensharing on a single stream. | 168 // Special mode when screensharing on a single stream. |
194 if (number_of_streams == 1 && inst->mode == kScreensharing) { | 169 if (number_of_streams == 1 && inst->mode == kScreensharing) { |
195 screensharing_tl_factory_.reset(new ScreenshareTemporalLayersFactory()); | 170 screensharing_tl_factory_.reset(new ScreenshareTemporalLayersFactory()); |
196 codec_.codecSpecific.VP8.tl_factory = screensharing_tl_factory_.get(); | 171 codec_.codecSpecific.VP8.tl_factory = screensharing_tl_factory_.get(); |
197 } | 172 } |
198 | 173 |
199 std::string implementation_name; | 174 std::string implementation_name; |
200 // Create |number_of_streams| of encoder instances and init them. | 175 // Create |number_of_streams| of encoder instances and init them. |
201 for (int i = 0; i < number_of_streams; ++i) { | 176 stream_states_ = *simulcast; |
| 177 for (const SimulcastState::Stream& stream : stream_states_.Streams()) { |
202 VideoCodec stream_codec; | 178 VideoCodec stream_codec; |
203 bool send_stream = true; | |
204 if (!doing_simulcast) { | 179 if (!doing_simulcast) { |
205 stream_codec = codec_; | 180 stream_codec = codec_; |
206 stream_codec.numberOfSimulcastStreams = 1; | 181 stream_codec.numberOfSimulcastStreams = 1; |
207 } else { | 182 } else { |
208 bool highest_resolution_stream = (i == (number_of_streams - 1)); | 183 stream_codec = stream.AsCodec(); |
209 PopulateStreamCodec(&codec_, i, number_of_streams, | 184 if (stream.idx == 0) { |
210 highest_resolution_stream, &stream_codec, | 185 // Settings for lowest spatial resolutions. |
211 &send_stream); | 186 stream_codec.qpMax = kLowestResMaxQp; |
| 187 } |
| 188 if (stream.idx < codec_.numberOfSimulcastStreams - 1) { |
| 189 // For resolutions below CIF, set the codec |complexity| parameter to |
| 190 // kComplexityHigher, which maps to cpu_used = -4. |
| 191 if (stream_codec.width * stream_codec.height < 352 * 288) { |
| 192 stream_codec.codecSpecific.VP8.complexity = webrtc::kComplexityHigher; |
| 193 } |
| 194 // Turn off denoising for all streams but the highest resolution. |
| 195 stream_codec.codecSpecific.VP8.denoisingOn = false; |
| 196 } |
212 } | 197 } |
213 | 198 |
214 // TODO(ronghuawu): Remove once this is handled in VP8EncoderImpl. | 199 // TODO(ronghuawu): Remove once this is handled in VP8EncoderImpl. |
215 if (stream_codec.qpMax < kDefaultMinQp) { | 200 if (stream_codec.qpMax < kDefaultMinQp) { |
216 stream_codec.qpMax = kDefaultMaxQp; | 201 stream_codec.qpMax = kDefaultMaxQp; |
217 } | 202 } |
218 | 203 |
219 VideoEncoder* encoder = factory_->Create(); | 204 VideoEncoder* encoder = factory_->Create(); |
220 ret = encoder->InitEncode(&stream_codec, number_of_cores, max_payload_size); | 205 ret = encoder->InitEncode(&stream_codec, number_of_cores, max_payload_size); |
221 if (ret < 0) { | 206 if (ret < 0) { |
222 Release(); | 207 Release(); |
223 return ret; | 208 return ret; |
224 } | 209 } |
225 EncodedImageCallback* callback = new AdapterEncodedImageCallback(this, i); | 210 EncodedImageCallback* callback = |
| 211 new AdapterEncodedImageCallback(this, stream.idx); |
226 encoder->RegisterEncodeCompleteCallback(callback); | 212 encoder->RegisterEncodeCompleteCallback(callback); |
227 streaminfos_.push_back(StreamInfo(encoder, callback, stream_codec.width, | 213 encoders_[stream.idx] = EncoderContext( |
228 stream_codec.height, send_stream)); | 214 encoder, std::unique_ptr<EncodedImageCallback>(callback)); |
229 if (i != 0) | 215 if (stream.idx != 0) |
230 implementation_name += ", "; | 216 implementation_name += ", "; |
231 implementation_name += streaminfos_[i].encoder->ImplementationName(); | 217 implementation_name += encoder->ImplementationName(); |
232 } | 218 } |
| 219 |
233 if (doing_simulcast) { | 220 if (doing_simulcast) { |
234 implementation_name_ = | 221 implementation_name_ = |
235 "SimulcastEncoderAdapter (" + implementation_name + ")"; | 222 "SimulcastEncoderAdapter (" + implementation_name + ")"; |
236 } else { | 223 } else { |
237 implementation_name_ = implementation_name; | 224 implementation_name_ = implementation_name; |
238 } | 225 } |
239 return WEBRTC_VIDEO_CODEC_OK; | 226 return WEBRTC_VIDEO_CODEC_OK; |
240 } | 227 } |
241 | 228 |
242 int SimulcastEncoderAdapter::Encode( | 229 int SimulcastEncoderAdapter::Encode( |
243 const VideoFrame& input_image, | 230 const VideoFrame& input_image, |
244 const CodecSpecificInfo* codec_specific_info, | 231 const CodecSpecificInfo* codec_specific_info, |
245 const std::vector<FrameType>* frame_types) { | 232 const std::vector<FrameType>* frame_types) { |
246 if (!Initialized()) { | 233 if (!Initialized()) { |
247 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; | 234 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
248 } | 235 } |
249 if (encoded_complete_callback_ == NULL) { | 236 if (encoded_complete_callback_ == NULL) { |
250 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; | 237 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
251 } | 238 } |
252 | 239 |
253 // All active streams should generate a key frame if | 240 // TODO(sprang): Do we really need to force key frames on all streams? |
254 // a key frame is requested by any stream. | 241 // All active streams should generate a key frame if a key frame is requested |
255 bool send_key_frame = false; | 242 // by any stream. |
256 if (frame_types) { | 243 bool force_key_frame = frame_types && |
257 for (size_t i = 0; i < frame_types->size(); ++i) { | 244 std::find(frame_types->begin(), frame_types->end(), |
258 if (frame_types->at(i) == kVideoFrameKey) { | 245 kVideoFrameKey) != frame_types->end(); |
259 send_key_frame = true; | 246 int active_streams = stream_states_.NumSendingStreams(); |
260 break; | 247 for (int i = 0; i < active_streams; ++i) |
261 } | 248 force_key_frame |= stream_states_.GetAndResetKeyFrameRequest(i); |
262 } | |
263 } | |
264 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { | |
265 if (streaminfos_[stream_idx].key_frame_request && | |
266 streaminfos_[stream_idx].send_stream) { | |
267 send_key_frame = true; | |
268 break; | |
269 } | |
270 } | |
271 | 249 |
272 int src_width = input_image.width(); | 250 int src_width = input_image.width(); |
273 int src_height = input_image.height(); | 251 int src_height = input_image.height(); |
274 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { | 252 for (const SimulcastState::Stream& stream : stream_states_.Streams()) { |
275 // Don't encode frames in resolutions that we don't intend to send. | 253 if (!stream.sending) |
276 if (!streaminfos_[stream_idx].send_stream) | |
277 continue; | 254 continue; |
278 | 255 |
279 std::vector<FrameType> stream_frame_types; | 256 std::vector<FrameType> stream_frame_types; |
280 if (send_key_frame) { | 257 if (force_key_frame) { |
281 stream_frame_types.push_back(kVideoFrameKey); | 258 stream_frame_types.push_back(kVideoFrameKey); |
282 streaminfos_[stream_idx].key_frame_request = false; | |
283 } else { | 259 } else { |
284 stream_frame_types.push_back(kVideoFrameDelta); | 260 stream_frame_types.push_back(kVideoFrameDelta); |
285 } | 261 } |
286 | 262 |
287 int dst_width = streaminfos_[stream_idx].width; | 263 int dst_width = stream.config.width; |
288 int dst_height = streaminfos_[stream_idx].height; | 264 int dst_height = stream.config.height; |
289 // If scaling isn't required, because the input resolution | 265 // If scaling isn't required, because the input resolution matches the |
290 // matches the destination or the input image is empty (e.g. | 266 // destination or the input image is empty (e.g. a keyframe request for |
291 // a keyframe request for encoders with internal camera | 267 // encoders with internal camera sources), pass the image on directly. |
292 // sources), pass the image on directly. Otherwise, we'll | 268 // Otherwise, we'll scale it to match what the encoder expects (below). |
293 // scale it to match what the encoder expects (below). | |
294 if ((dst_width == src_width && dst_height == src_height) || | 269 if ((dst_width == src_width && dst_height == src_height) || |
295 input_image.IsZeroSize()) { | 270 input_image.IsZeroSize()) { |
296 streaminfos_[stream_idx].encoder->Encode(input_image, codec_specific_info, | 271 encoders_[stream.idx].first->Encode(input_image, codec_specific_info, |
297 &stream_frame_types); | 272 &stream_frame_types); |
298 } else { | 273 } else { |
299 VideoFrame dst_frame; | 274 VideoFrame dst_frame; |
300 // Making sure that destination frame is of sufficient size. | 275 // Making sure that destination frame is of sufficient size. |
301 // Aligning stride values based on width. | 276 // Aligning stride values based on width. |
302 dst_frame.CreateEmptyFrame(dst_width, dst_height, dst_width, | 277 dst_frame.CreateEmptyFrame(dst_width, dst_height, dst_width, |
303 (dst_width + 1) / 2, (dst_width + 1) / 2); | 278 (dst_width + 1) / 2, (dst_width + 1) / 2); |
304 libyuv::I420Scale( | 279 libyuv::I420Scale( |
305 input_image.buffer(kYPlane), input_image.stride(kYPlane), | 280 input_image.buffer(kYPlane), input_image.stride(kYPlane), |
306 input_image.buffer(kUPlane), input_image.stride(kUPlane), | 281 input_image.buffer(kUPlane), input_image.stride(kUPlane), |
307 input_image.buffer(kVPlane), input_image.stride(kVPlane), src_width, | 282 input_image.buffer(kVPlane), input_image.stride(kVPlane), src_width, |
308 src_height, dst_frame.buffer(kYPlane), dst_frame.stride(kYPlane), | 283 src_height, dst_frame.buffer(kYPlane), dst_frame.stride(kYPlane), |
309 dst_frame.buffer(kUPlane), dst_frame.stride(kUPlane), | 284 dst_frame.buffer(kUPlane), dst_frame.stride(kUPlane), |
310 dst_frame.buffer(kVPlane), dst_frame.stride(kVPlane), dst_width, | 285 dst_frame.buffer(kVPlane), dst_frame.stride(kVPlane), dst_width, |
311 dst_height, libyuv::kFilterBilinear); | 286 dst_height, libyuv::kFilterBilinear); |
312 dst_frame.set_timestamp(input_image.timestamp()); | 287 dst_frame.set_timestamp(input_image.timestamp()); |
313 dst_frame.set_render_time_ms(input_image.render_time_ms()); | 288 dst_frame.set_render_time_ms(input_image.render_time_ms()); |
314 streaminfos_[stream_idx].encoder->Encode(dst_frame, codec_specific_info, | 289 encoders_[stream.idx].first->Encode(dst_frame, codec_specific_info, |
315 &stream_frame_types); | 290 &stream_frame_types); |
316 } | 291 } |
317 } | 292 } |
318 | 293 |
319 return WEBRTC_VIDEO_CODEC_OK; | 294 return WEBRTC_VIDEO_CODEC_OK; |
320 } | 295 } |
321 | 296 |
322 int SimulcastEncoderAdapter::RegisterEncodeCompleteCallback( | 297 int SimulcastEncoderAdapter::RegisterEncodeCompleteCallback( |
323 EncodedImageCallback* callback) { | 298 EncodedImageCallback* callback) { |
324 encoded_complete_callback_ = callback; | 299 encoded_complete_callback_ = callback; |
325 return WEBRTC_VIDEO_CODEC_OK; | 300 return WEBRTC_VIDEO_CODEC_OK; |
326 } | 301 } |
327 | 302 |
328 int SimulcastEncoderAdapter::SetChannelParameters(uint32_t packet_loss, | 303 int SimulcastEncoderAdapter::SetChannelParameters(uint32_t packet_loss, |
329 int64_t rtt) { | 304 int64_t rtt) { |
330 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { | 305 for (const auto& it : encoders_) |
331 streaminfos_[stream_idx].encoder->SetChannelParameters(packet_loss, rtt); | 306 it.second.first->SetChannelParameters(packet_loss, rtt); |
332 } | |
333 return WEBRTC_VIDEO_CODEC_OK; | 307 return WEBRTC_VIDEO_CODEC_OK; |
334 } | 308 } |
335 | 309 |
336 int SimulcastEncoderAdapter::SetRates(uint32_t new_bitrate_kbit, | 310 int SimulcastEncoderAdapter::SetRates(uint32_t new_bitrate_kbit, |
337 uint32_t new_framerate) { | 311 uint32_t new_framerate) { |
338 if (!Initialized()) { | 312 if (!Initialized()) |
339 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; | 313 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
340 } | 314 |
341 if (new_framerate < 1) { | 315 if (new_framerate < 1) |
342 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; | 316 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
343 } | 317 |
344 if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) { | 318 if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) |
345 new_bitrate_kbit = codec_.maxBitrate; | 319 new_bitrate_kbit = codec_.maxBitrate; |
346 } | 320 |
347 if (new_bitrate_kbit < codec_.minBitrate) { | 321 if (new_bitrate_kbit < codec_.minBitrate) |
348 new_bitrate_kbit = codec_.minBitrate; | 322 new_bitrate_kbit = codec_.minBitrate; |
349 } | 323 |
350 if (codec_.numberOfSimulcastStreams > 0 && | 324 if (codec_.numberOfSimulcastStreams > 0 && |
351 new_bitrate_kbit < codec_.simulcastStream[0].minBitrate) { | 325 new_bitrate_kbit < codec_.simulcastStream[0].minBitrate) { |
352 new_bitrate_kbit = codec_.simulcastStream[0].minBitrate; | 326 new_bitrate_kbit = codec_.simulcastStream[0].minBitrate; |
353 } | 327 } |
| 328 |
354 codec_.maxFramerate = new_framerate; | 329 codec_.maxFramerate = new_framerate; |
355 | 330 stream_states_.AllocateBitrate(new_bitrate_kbit * 1000); |
356 bool send_stream = true; | 331 for (const SimulcastState::Stream& stream : stream_states_.Streams()) { |
357 uint32_t stream_bitrate = 0; | 332 encoders_[stream.idx].first->SetRates(stream.allocated_rate_bps / 1000, |
358 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { | 333 new_framerate); |
359 stream_bitrate = GetStreamBitrate(stream_idx, streaminfos_.size(), | |
360 new_bitrate_kbit, &send_stream); | |
361 // Need a key frame if we have not sent this stream before. | |
362 if (send_stream && !streaminfos_[stream_idx].send_stream) { | |
363 streaminfos_[stream_idx].key_frame_request = true; | |
364 } | |
365 streaminfos_[stream_idx].send_stream = send_stream; | |
366 | |
367 // TODO(holmer): This is a temporary hack for screensharing, where we | |
368 // interpret the startBitrate as the encoder target bitrate. This is | |
369 // to allow for a different max bitrate, so if the codec can't meet | |
370 // the target we still allow it to overshoot up to the max before dropping | |
371 // frames. This hack should be improved. | |
372 if (codec_.targetBitrate > 0 && | |
373 (codec_.codecSpecific.VP8.numberOfTemporalLayers == 2 || | |
374 codec_.simulcastStream[0].numberOfTemporalLayers == 2)) { | |
375 stream_bitrate = std::min(codec_.maxBitrate, stream_bitrate); | |
376 // TODO(ronghuawu): Can't change max bitrate via the VideoEncoder | |
377 // interface. And VP8EncoderImpl doesn't take negative framerate. | |
378 // max_bitrate = std::min(codec_.maxBitrate, stream_bitrate); | |
379 // new_framerate = -1; | |
380 } | |
381 | |
382 streaminfos_[stream_idx].encoder->SetRates(stream_bitrate, new_framerate); | |
383 } | 334 } |
384 | 335 |
385 return WEBRTC_VIDEO_CODEC_OK; | 336 return WEBRTC_VIDEO_CODEC_OK; |
386 } | 337 } |
387 | 338 |
388 int32_t SimulcastEncoderAdapter::Encoded( | 339 int32_t SimulcastEncoderAdapter::Encoded( |
389 size_t stream_idx, | 340 size_t stream_idx, |
390 const EncodedImage& encodedImage, | 341 const EncodedImage& encodedImage, |
391 const CodecSpecificInfo* codecSpecificInfo, | 342 const CodecSpecificInfo* codecSpecificInfo, |
392 const RTPFragmentationHeader* fragmentation) { | 343 const RTPFragmentationHeader* fragmentation) { |
393 CodecSpecificInfo stream_codec_specific = *codecSpecificInfo; | 344 CodecSpecificInfo stream_codec_specific = *codecSpecificInfo; |
394 CodecSpecificInfoVP8* vp8Info = &(stream_codec_specific.codecSpecific.VP8); | 345 CodecSpecificInfoVP8* vp8Info = &(stream_codec_specific.codecSpecific.VP8); |
395 vp8Info->simulcastIdx = stream_idx; | 346 vp8Info->simulcastIdx = stream_idx; |
396 | 347 |
397 return encoded_complete_callback_->Encoded( | 348 return encoded_complete_callback_->Encoded( |
398 encodedImage, &stream_codec_specific, fragmentation); | 349 encodedImage, &stream_codec_specific, fragmentation); |
399 } | 350 } |
400 | 351 |
401 uint32_t SimulcastEncoderAdapter::GetStreamBitrate( | |
402 int stream_idx, | |
403 size_t total_number_of_streams, | |
404 uint32_t new_bitrate_kbit, | |
405 bool* send_stream) const { | |
406 if (total_number_of_streams == 1) { | |
407 *send_stream = true; | |
408 return new_bitrate_kbit; | |
409 } | |
410 | |
411 // The bitrate needed to start sending this stream is given by the | |
412 // minimum bitrate allowed for encoding this stream, plus the sum target | |
413 // rates of all lower streams. | |
414 uint32_t sum_target_lower_streams = | |
415 SumStreamTargetBitrate(stream_idx, codec_); | |
416 uint32_t bitrate_to_send_this_layer = | |
417 codec_.simulcastStream[stream_idx].minBitrate + sum_target_lower_streams; | |
418 if (new_bitrate_kbit >= bitrate_to_send_this_layer) { | |
419 // We have enough bandwidth to send this stream. | |
420 *send_stream = true; | |
421 // Bitrate for this stream is the new bitrate (|new_bitrate_kbit|) minus the | |
422 // sum target rates of the lower streams, and capped to a maximum bitrate. | |
423 // The maximum cap depends on whether we send the next higher stream. | |
424 // If we will be sending the next higher stream, |max_rate| is given by | |
425 // current stream's |targetBitrate|, otherwise it's capped by |maxBitrate|. | |
426 if (stream_idx < codec_.numberOfSimulcastStreams - 1) { | |
427 unsigned int max_rate = codec_.simulcastStream[stream_idx].maxBitrate; | |
428 if (new_bitrate_kbit >= | |
429 SumStreamTargetBitrate(stream_idx + 1, codec_) + | |
430 codec_.simulcastStream[stream_idx + 1].minBitrate) { | |
431 max_rate = codec_.simulcastStream[stream_idx].targetBitrate; | |
432 } | |
433 return std::min(new_bitrate_kbit - sum_target_lower_streams, max_rate); | |
434 } else { | |
435 // For the highest stream (highest resolution), the |targetBitRate| and | |
436 // |maxBitrate| are not used. Any excess bitrate (above the targets of | |
437 // all lower streams) is given to this (highest resolution) stream. | |
438 return new_bitrate_kbit - sum_target_lower_streams; | |
439 } | |
440 } else { | |
441 // Not enough bitrate for this stream. | |
442 // Return our max bitrate of |stream_idx| - 1, but we don't send it. We need | |
443 // to keep this resolution coding in order for the multi-encoder to work. | |
444 *send_stream = false; | |
445 return codec_.simulcastStream[stream_idx - 1].maxBitrate; | |
446 } | |
447 } | |
448 | |
449 void SimulcastEncoderAdapter::PopulateStreamCodec( | |
450 const webrtc::VideoCodec* inst, | |
451 int stream_index, | |
452 size_t total_number_of_streams, | |
453 bool highest_resolution_stream, | |
454 webrtc::VideoCodec* stream_codec, | |
455 bool* send_stream) { | |
456 *stream_codec = *inst; | |
457 | |
458 // Stream specific settings. | |
459 stream_codec->codecSpecific.VP8.numberOfTemporalLayers = | |
460 inst->simulcastStream[stream_index].numberOfTemporalLayers; | |
461 stream_codec->numberOfSimulcastStreams = 0; | |
462 stream_codec->width = inst->simulcastStream[stream_index].width; | |
463 stream_codec->height = inst->simulcastStream[stream_index].height; | |
464 stream_codec->maxBitrate = inst->simulcastStream[stream_index].maxBitrate; | |
465 stream_codec->minBitrate = inst->simulcastStream[stream_index].minBitrate; | |
466 stream_codec->qpMax = inst->simulcastStream[stream_index].qpMax; | |
467 // Settings that are based on stream/resolution. | |
468 if (stream_index == 0) { | |
469 // Settings for lowest spatial resolutions. | |
470 stream_codec->qpMax = kLowestResMaxQp; | |
471 } | |
472 if (!highest_resolution_stream) { | |
473 // For resolutions below CIF, set the codec |complexity| parameter to | |
474 // kComplexityHigher, which maps to cpu_used = -4. | |
475 int pixels_per_frame = stream_codec->width * stream_codec->height; | |
476 if (pixels_per_frame < 352 * 288) { | |
477 stream_codec->codecSpecific.VP8.complexity = webrtc::kComplexityHigher; | |
478 } | |
479 // Turn off denoising for all streams but the highest resolution. | |
480 stream_codec->codecSpecific.VP8.denoisingOn = false; | |
481 } | |
482 // TODO(ronghuawu): what to do with targetBitrate. | |
483 | |
484 int stream_bitrate = GetStreamBitrate(stream_index, total_number_of_streams, | |
485 inst->startBitrate, send_stream); | |
486 stream_codec->startBitrate = stream_bitrate; | |
487 } | |
488 | |
489 bool SimulcastEncoderAdapter::Initialized() const { | 352 bool SimulcastEncoderAdapter::Initialized() const { |
490 return !streaminfos_.empty(); | 353 return !encoders_.empty(); |
491 } | 354 } |
492 | 355 |
493 void SimulcastEncoderAdapter::OnDroppedFrame() { | 356 void SimulcastEncoderAdapter::OnDroppedFrame() { |
494 streaminfos_[0].encoder->OnDroppedFrame(); | 357 for (const auto& it : encoders_) |
| 358 it.second.first->OnDroppedFrame(); |
495 } | 359 } |
496 | 360 |
497 bool SimulcastEncoderAdapter::SupportsNativeHandle() const { | 361 bool SimulcastEncoderAdapter::SupportsNativeHandle() const { |
498 // We should not be calling this method before streaminfos_ are configured. | 362 // We should not be calling this method before streaminfos_ are configured. |
499 RTC_DCHECK(!streaminfos_.empty()); | 363 RTC_DCHECK(Initialized()); |
500 // TODO(pbos): Support textures when using more than one encoder. | 364 // TODO(pbos): Support textures when using more than one encoder. |
501 if (streaminfos_.size() != 1) | 365 if (encoders_.size() != 1) |
502 return false; | 366 return false; |
503 return streaminfos_[0].encoder->SupportsNativeHandle(); | 367 return encoders_.begin()->second.first->SupportsNativeHandle(); |
504 } | 368 } |
505 | 369 |
506 const char* SimulcastEncoderAdapter::ImplementationName() const { | 370 const char* SimulcastEncoderAdapter::ImplementationName() const { |
507 return implementation_name_.c_str(); | 371 return implementation_name_.c_str(); |
508 } | 372 } |
509 | 373 |
510 } // namespace webrtc | 374 } // namespace webrtc |
OLD | NEW |