Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(235)

Side by Side Diff: webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc

Issue 1913073002: Extract common simulcast logic from VP8 wrapper and simulcast adapter (Closed) Base URL: https://chromium.googlesource.com/external/webrtc.git@master
Patch Set: Address comments, added tests Created 4 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 /* 1 /*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 * 3 *
4 * Use of this source code is governed by a BSD-style license 4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source 5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found 6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may 7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree. 8 * be found in the AUTHORS file in the root of the source tree.
9 */ 9 */
10 10
11 #include "webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h" 11 #include "webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h"
12 12
13 #include <algorithm> 13 #include <algorithm>
14 14
15 // NOTE(ajm): Path provided by gyp. 15 // NOTE(ajm): Path provided by gyp.
16 #include "libyuv/scale.h" // NOLINT 16 #include "libyuv/scale.h" // NOLINT
17 17
18 #include "webrtc/base/checks.h" 18 #include "webrtc/base/checks.h"
19 #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h" 19 #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
20 #include "webrtc/system_wrappers/include/clock.h" 20 #include "webrtc/system_wrappers/include/clock.h"
21 21
22 namespace { 22 namespace {
23 23
24 const unsigned int kDefaultMinQp = 2; 24 const unsigned int kDefaultMinQp = 2;
25 const unsigned int kDefaultMaxQp = 56; 25 const unsigned int kDefaultMaxQp = 56;
26 // Max qp for lowest spatial resolution when doing simulcast. 26 // Max qp for lowest spatial resolution when doing simulcast.
27 const unsigned int kLowestResMaxQp = 45; 27 const unsigned int kLowestResMaxQp = 45;
28 28
29 uint32_t SumStreamTargetBitrate(int streams, const webrtc::VideoCodec& codec) { 29 static const char* kDefaultImplementationName =
30 uint32_t bitrate_sum = 0; 30 "SimulcastEncoderAdapter (uninitialized)";
31 for (int i = 0; i < streams; ++i) {
32 bitrate_sum += codec.simulcastStream[i].targetBitrate;
33 }
34 return bitrate_sum;
35 }
36
37 uint32_t SumStreamMaxBitrate(int streams, const webrtc::VideoCodec& codec) {
38 uint32_t bitrate_sum = 0;
39 for (int i = 0; i < streams; ++i) {
40 bitrate_sum += codec.simulcastStream[i].maxBitrate;
41 }
42 return bitrate_sum;
43 }
44
45 int NumberOfStreams(const webrtc::VideoCodec& codec) {
46 int streams =
47 codec.numberOfSimulcastStreams < 1 ? 1 : codec.numberOfSimulcastStreams;
48 uint32_t simulcast_max_bitrate = SumStreamMaxBitrate(streams, codec);
49 if (simulcast_max_bitrate == 0) {
50 streams = 1;
51 }
52 return streams;
53 }
54 31
55 bool ValidSimulcastResolutions(const webrtc::VideoCodec& codec, 32 bool ValidSimulcastResolutions(const webrtc::VideoCodec& codec,
56 int num_streams) { 33 int num_streams) {
57 if (codec.width != codec.simulcastStream[num_streams - 1].width || 34 if (codec.width != codec.simulcastStream[num_streams - 1].width ||
58 codec.height != codec.simulcastStream[num_streams - 1].height) { 35 codec.height != codec.simulcastStream[num_streams - 1].height) {
59 return false; 36 return false;
60 } 37 }
61 for (int i = 0; i < num_streams; ++i) { 38 for (int i = 0; i < num_streams; ++i) {
62 if (codec.width * codec.simulcastStream[i].height != 39 if (codec.width * codec.simulcastStream[i].height !=
63 codec.height * codec.simulcastStream[i].width) { 40 codec.height * codec.simulcastStream[i].width) {
(...skipping 21 matching lines...) Expand all
85 inst->numberOfSimulcastStreams > 1) { 62 inst->numberOfSimulcastStreams > 1) {
86 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; 63 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
87 } 64 }
88 if (inst->codecSpecific.VP8.automaticResizeOn && 65 if (inst->codecSpecific.VP8.automaticResizeOn &&
89 inst->numberOfSimulcastStreams > 1) { 66 inst->numberOfSimulcastStreams > 1) {
90 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; 67 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
91 } 68 }
92 return WEBRTC_VIDEO_CODEC_OK; 69 return WEBRTC_VIDEO_CODEC_OK;
93 } 70 }
94 71
95 // TL1 FrameDropper's max time to drop frames.
96 const float kTl1MaxTimeToDropFrames = 20.0f;
97
98 struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory { 72 struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory {
99 ScreenshareTemporalLayersFactory() 73 ScreenshareTemporalLayersFactory() {}
100 : tl1_frame_dropper_(kTl1MaxTimeToDropFrames) {}
101
102 virtual ~ScreenshareTemporalLayersFactory() {} 74 virtual ~ScreenshareTemporalLayersFactory() {}
103 75
104 virtual webrtc::TemporalLayers* Create(int num_temporal_layers, 76 virtual webrtc::TemporalLayers* Create(int num_temporal_layers,
105 uint8_t initial_tl0_pic_idx) const { 77 uint8_t initial_tl0_pic_idx) const {
106 return new webrtc::ScreenshareLayers(num_temporal_layers, rand(), 78 return new webrtc::ScreenshareLayers(num_temporal_layers, rand(),
107 webrtc::Clock::GetRealTimeClock()); 79 webrtc::Clock::GetRealTimeClock());
108 } 80 }
109
110 mutable webrtc::FrameDropper tl0_frame_dropper_;
111 mutable webrtc::FrameDropper tl1_frame_dropper_;
112 }; 81 };
113 82
114 // An EncodedImageCallback implementation that forwards on calls to a 83 // An EncodedImageCallback implementation that forwards on calls to a
115 // SimulcastEncoderAdapter, but with the stream index it's registered with as 84 // SimulcastEncoderAdapter, but with the stream index it's registered with as
116 // the first parameter to Encoded. 85 // the first parameter to Encoded.
117 class AdapterEncodedImageCallback : public webrtc::EncodedImageCallback { 86 class AdapterEncodedImageCallback : public webrtc::EncodedImageCallback {
118 public: 87 public:
119 AdapterEncodedImageCallback(webrtc::SimulcastEncoderAdapter* adapter, 88 AdapterEncodedImageCallback(webrtc::SimulcastEncoderAdapter* adapter,
120 size_t stream_idx) 89 size_t stream_idx)
121 : adapter_(adapter), stream_idx_(stream_idx) {} 90 : adapter_(adapter), stream_idx_(stream_idx) {}
122 91
123 int32_t Encoded( 92 int32_t Encoded(
124 const webrtc::EncodedImage& encodedImage, 93 const webrtc::EncodedImage& encodedImage,
125 const webrtc::CodecSpecificInfo* codecSpecificInfo = NULL, 94 const webrtc::CodecSpecificInfo* codecSpecificInfo = NULL,
126 const webrtc::RTPFragmentationHeader* fragmentation = NULL) override { 95 const webrtc::RTPFragmentationHeader* fragmentation = NULL) override {
127 return adapter_->Encoded(stream_idx_, encodedImage, codecSpecificInfo, 96 return adapter_->Encoded(stream_idx_, encodedImage, codecSpecificInfo,
128 fragmentation); 97 fragmentation);
129 } 98 }
130 99
131 private: 100 private:
132 webrtc::SimulcastEncoderAdapter* const adapter_; 101 webrtc::SimulcastEncoderAdapter* const adapter_;
133 const size_t stream_idx_; 102 const size_t stream_idx_;
134 }; 103 };
135 104
105 webrtc::VideoCodec DefaultCodec() {
106 webrtc::VideoCodec codec;
107 memset(&codec, 0, sizeof(webrtc::VideoCodec));
108 return codec;
109 }
110
136 } // namespace 111 } // namespace
137 112
138 namespace webrtc { 113 namespace webrtc {
139 114
140 SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory) 115 SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory)
141 : factory_(factory), 116 : factory_(factory),
142 encoded_complete_callback_(NULL), 117 codec_(DefaultCodec()),
143 implementation_name_("SimulcastEncoderAdapter") { 118 stream_states_(codec_),
144 memset(&codec_, 0, sizeof(webrtc::VideoCodec)); 119 encoded_complete_callback_(nullptr),
145 } 120 implementation_name_(kDefaultImplementationName) {}
146 121
147 SimulcastEncoderAdapter::~SimulcastEncoderAdapter() { 122 SimulcastEncoderAdapter::~SimulcastEncoderAdapter() {
148 Release(); 123 Release();
149 } 124 }
150 125
151 int SimulcastEncoderAdapter::Release() { 126 int SimulcastEncoderAdapter::Release() {
152 // TODO(pbos): Keep the last encoder instance but call ::Release() on it, then 127 // TODO(pbos): Keep the last encoder instance but call ::Release() on it, then
153 // re-use this instance in ::InitEncode(). This means that changing 128 // re-use this instance in ::InitEncode(). This means that changing
154 // resolutions doesn't require reallocation of the first encoder, but only 129 // resolutions doesn't require reallocation of the first encoder, but only
155 // reinitialization, which makes sense. Then Destroy this instance instead in 130 // reinitialization, which makes sense. Then Destroy this instance instead in
156 // ~SimulcastEncoderAdapter(). 131 // ~SimulcastEncoderAdapter().
157 while (!streaminfos_.empty()) { 132 stream_states_ = SimulcastState(codec_);
158 VideoEncoder* encoder = streaminfos_.back().encoder; 133 for (auto& it : encoders_)
159 EncodedImageCallback* callback = streaminfos_.back().callback; 134 factory_->Destroy(it.second.first);
160 factory_->Destroy(encoder); 135 encoders_.clear();
161 delete callback; 136
162 streaminfos_.pop_back();
163 }
164 return WEBRTC_VIDEO_CODEC_OK; 137 return WEBRTC_VIDEO_CODEC_OK;
165 } 138 }
166 139
167 int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, 140 int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst,
168 int number_of_cores, 141 int number_of_cores,
169 size_t max_payload_size) { 142 size_t max_payload_size) {
170 if (number_of_cores < 1) { 143 if (number_of_cores < 1) {
171 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; 144 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
172 } 145 }
173 146
174 int ret = VerifyCodec(inst); 147 int ret = VerifyCodec(inst);
175 if (ret < 0) { 148 if (ret < 0) {
176 return ret; 149 return ret;
177 } 150 }
178 151
179 ret = Release(); 152 ret = Release();
180 if (ret < 0) { 153 if (ret < 0) {
181 return ret; 154 return ret;
182 } 155 }
183 156
184 int number_of_streams = NumberOfStreams(*inst); 157 std::unique_ptr<SimulcastState> simulcast(new SimulcastState(*inst));
158
159 int number_of_streams = simulcast->NumStreams();
185 const bool doing_simulcast = (number_of_streams > 1); 160 const bool doing_simulcast = (number_of_streams > 1);
186 161
187 if (doing_simulcast && !ValidSimulcastResolutions(*inst, number_of_streams)) { 162 if (doing_simulcast && !ValidSimulcastResolutions(*inst, number_of_streams)) {
188 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; 163 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
189 } 164 }
190 165
191 codec_ = *inst; 166 codec_ = *inst;
192 167
193 // Special mode when screensharing on a single stream. 168 // Special mode when screensharing on a single stream.
194 if (number_of_streams == 1 && inst->mode == kScreensharing) { 169 if (number_of_streams == 1 && inst->mode == kScreensharing) {
195 screensharing_tl_factory_.reset(new ScreenshareTemporalLayersFactory()); 170 screensharing_tl_factory_.reset(new ScreenshareTemporalLayersFactory());
196 codec_.codecSpecific.VP8.tl_factory = screensharing_tl_factory_.get(); 171 codec_.codecSpecific.VP8.tl_factory = screensharing_tl_factory_.get();
197 } 172 }
198 173
199 std::string implementation_name; 174 std::string implementation_name;
200 // Create |number_of_streams| of encoder instances and init them. 175 // Create |number_of_streams| of encoder instances and init them.
201 for (int i = 0; i < number_of_streams; ++i) { 176 stream_states_ = *simulcast;
177 for (const SimulcastState::Stream& stream : stream_states_.Streams()) {
202 VideoCodec stream_codec; 178 VideoCodec stream_codec;
203 bool send_stream = true;
204 if (!doing_simulcast) { 179 if (!doing_simulcast) {
205 stream_codec = codec_; 180 stream_codec = codec_;
206 stream_codec.numberOfSimulcastStreams = 1; 181 stream_codec.numberOfSimulcastStreams = 1;
207 } else { 182 } else {
208 bool highest_resolution_stream = (i == (number_of_streams - 1)); 183 stream_codec = stream.AsCodec();
209 PopulateStreamCodec(&codec_, i, number_of_streams, 184 if (stream.idx == 0) {
210 highest_resolution_stream, &stream_codec, 185 // Settings for lowest spatial resolutions.
211 &send_stream); 186 stream_codec.qpMax = kLowestResMaxQp;
187 }
188 if (stream.idx < codec_.numberOfSimulcastStreams - 1) {
189 // For resolutions below CIF, set the codec |complexity| parameter to
190 // kComplexityHigher, which maps to cpu_used = -4.
191 if (stream_codec.width * stream_codec.height < 352 * 288) {
192 stream_codec.codecSpecific.VP8.complexity = webrtc::kComplexityHigher;
193 }
194 // Turn off denoising for all streams but the highest resolution.
195 stream_codec.codecSpecific.VP8.denoisingOn = false;
196 }
212 } 197 }
213 198
214 // TODO(ronghuawu): Remove once this is handled in VP8EncoderImpl. 199 // TODO(ronghuawu): Remove once this is handled in VP8EncoderImpl.
215 if (stream_codec.qpMax < kDefaultMinQp) { 200 if (stream_codec.qpMax < kDefaultMinQp) {
216 stream_codec.qpMax = kDefaultMaxQp; 201 stream_codec.qpMax = kDefaultMaxQp;
217 } 202 }
218 203
219 VideoEncoder* encoder = factory_->Create(); 204 VideoEncoder* encoder = factory_->Create();
220 ret = encoder->InitEncode(&stream_codec, number_of_cores, max_payload_size); 205 ret = encoder->InitEncode(&stream_codec, number_of_cores, max_payload_size);
221 if (ret < 0) { 206 if (ret < 0) {
222 Release(); 207 Release();
223 return ret; 208 return ret;
224 } 209 }
225 EncodedImageCallback* callback = new AdapterEncodedImageCallback(this, i); 210 EncodedImageCallback* callback =
211 new AdapterEncodedImageCallback(this, stream.idx);
226 encoder->RegisterEncodeCompleteCallback(callback); 212 encoder->RegisterEncodeCompleteCallback(callback);
227 streaminfos_.push_back(StreamInfo(encoder, callback, stream_codec.width, 213 encoders_[stream.idx] = EncoderContext(
228 stream_codec.height, send_stream)); 214 encoder, std::unique_ptr<EncodedImageCallback>(callback));
229 if (i != 0) 215 if (stream.idx != 0)
230 implementation_name += ", "; 216 implementation_name += ", ";
231 implementation_name += streaminfos_[i].encoder->ImplementationName(); 217 implementation_name += encoder->ImplementationName();
232 } 218 }
219
233 if (doing_simulcast) { 220 if (doing_simulcast) {
234 implementation_name_ = 221 implementation_name_ =
235 "SimulcastEncoderAdapter (" + implementation_name + ")"; 222 "SimulcastEncoderAdapter (" + implementation_name + ")";
236 } else { 223 } else {
237 implementation_name_ = implementation_name; 224 implementation_name_ = implementation_name;
238 } 225 }
239 return WEBRTC_VIDEO_CODEC_OK; 226 return WEBRTC_VIDEO_CODEC_OK;
240 } 227 }
241 228
242 int SimulcastEncoderAdapter::Encode( 229 int SimulcastEncoderAdapter::Encode(
243 const VideoFrame& input_image, 230 const VideoFrame& input_image,
244 const CodecSpecificInfo* codec_specific_info, 231 const CodecSpecificInfo* codec_specific_info,
245 const std::vector<FrameType>* frame_types) { 232 const std::vector<FrameType>* frame_types) {
246 if (!Initialized()) { 233 if (!Initialized()) {
247 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; 234 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
248 } 235 }
249 if (encoded_complete_callback_ == NULL) { 236 if (encoded_complete_callback_ == NULL) {
250 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; 237 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
251 } 238 }
252 239
253 // All active streams should generate a key frame if 240 // TODO(sprang): Do we really need to force key frames on all streams?
254 // a key frame is requested by any stream. 241 // All active streams should generate a key frame if a key frame is requested
255 bool send_key_frame = false; 242 // by any stream.
256 if (frame_types) { 243 bool force_key_frame = frame_types &&
257 for (size_t i = 0; i < frame_types->size(); ++i) { 244 std::find(frame_types->begin(), frame_types->end(),
258 if (frame_types->at(i) == kVideoFrameKey) { 245 kVideoFrameKey) != frame_types->end();
259 send_key_frame = true; 246 int active_streams = stream_states_.NumSendingStreams();
260 break; 247 for (int i = 0; i < active_streams; ++i)
261 } 248 force_key_frame |= stream_states_.GetAndResetKeyFrameRequest(i);
262 }
263 }
264 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) {
265 if (streaminfos_[stream_idx].key_frame_request &&
266 streaminfos_[stream_idx].send_stream) {
267 send_key_frame = true;
268 break;
269 }
270 }
271 249
272 int src_width = input_image.width(); 250 int src_width = input_image.width();
273 int src_height = input_image.height(); 251 int src_height = input_image.height();
274 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { 252 for (const SimulcastState::Stream& stream : stream_states_.Streams()) {
275 // Don't encode frames in resolutions that we don't intend to send. 253 if (!stream.sending)
276 if (!streaminfos_[stream_idx].send_stream)
277 continue; 254 continue;
278 255
279 std::vector<FrameType> stream_frame_types; 256 std::vector<FrameType> stream_frame_types;
280 if (send_key_frame) { 257 if (force_key_frame) {
281 stream_frame_types.push_back(kVideoFrameKey); 258 stream_frame_types.push_back(kVideoFrameKey);
282 streaminfos_[stream_idx].key_frame_request = false;
283 } else { 259 } else {
284 stream_frame_types.push_back(kVideoFrameDelta); 260 stream_frame_types.push_back(kVideoFrameDelta);
285 } 261 }
286 262
287 int dst_width = streaminfos_[stream_idx].width; 263 int dst_width = stream.config.width;
288 int dst_height = streaminfos_[stream_idx].height; 264 int dst_height = stream.config.height;
289 // If scaling isn't required, because the input resolution 265 // If scaling isn't required, because the input resolution matches the
290 // matches the destination or the input image is empty (e.g. 266 // destination or the input image is empty (e.g. a keyframe request for
291 // a keyframe request for encoders with internal camera 267 // encoders with internal camera sources), pass the image on directly.
292 // sources), pass the image on directly. Otherwise, we'll 268 // Otherwise, we'll scale it to match what the encoder expects (below).
293 // scale it to match what the encoder expects (below).
294 if ((dst_width == src_width && dst_height == src_height) || 269 if ((dst_width == src_width && dst_height == src_height) ||
295 input_image.IsZeroSize()) { 270 input_image.IsZeroSize()) {
296 streaminfos_[stream_idx].encoder->Encode(input_image, codec_specific_info, 271 encoders_[stream.idx].first->Encode(input_image, codec_specific_info,
297 &stream_frame_types); 272 &stream_frame_types);
298 } else { 273 } else {
299 VideoFrame dst_frame; 274 VideoFrame dst_frame;
300 // Making sure that destination frame is of sufficient size. 275 // Making sure that destination frame is of sufficient size.
301 // Aligning stride values based on width. 276 // Aligning stride values based on width.
302 dst_frame.CreateEmptyFrame(dst_width, dst_height, dst_width, 277 dst_frame.CreateEmptyFrame(dst_width, dst_height, dst_width,
303 (dst_width + 1) / 2, (dst_width + 1) / 2); 278 (dst_width + 1) / 2, (dst_width + 1) / 2);
304 libyuv::I420Scale( 279 libyuv::I420Scale(
305 input_image.buffer(kYPlane), input_image.stride(kYPlane), 280 input_image.buffer(kYPlane), input_image.stride(kYPlane),
306 input_image.buffer(kUPlane), input_image.stride(kUPlane), 281 input_image.buffer(kUPlane), input_image.stride(kUPlane),
307 input_image.buffer(kVPlane), input_image.stride(kVPlane), src_width, 282 input_image.buffer(kVPlane), input_image.stride(kVPlane), src_width,
308 src_height, dst_frame.buffer(kYPlane), dst_frame.stride(kYPlane), 283 src_height, dst_frame.buffer(kYPlane), dst_frame.stride(kYPlane),
309 dst_frame.buffer(kUPlane), dst_frame.stride(kUPlane), 284 dst_frame.buffer(kUPlane), dst_frame.stride(kUPlane),
310 dst_frame.buffer(kVPlane), dst_frame.stride(kVPlane), dst_width, 285 dst_frame.buffer(kVPlane), dst_frame.stride(kVPlane), dst_width,
311 dst_height, libyuv::kFilterBilinear); 286 dst_height, libyuv::kFilterBilinear);
312 dst_frame.set_timestamp(input_image.timestamp()); 287 dst_frame.set_timestamp(input_image.timestamp());
313 dst_frame.set_render_time_ms(input_image.render_time_ms()); 288 dst_frame.set_render_time_ms(input_image.render_time_ms());
314 streaminfos_[stream_idx].encoder->Encode(dst_frame, codec_specific_info, 289 encoders_[stream.idx].first->Encode(dst_frame, codec_specific_info,
315 &stream_frame_types); 290 &stream_frame_types);
316 } 291 }
317 } 292 }
318 293
319 return WEBRTC_VIDEO_CODEC_OK; 294 return WEBRTC_VIDEO_CODEC_OK;
320 } 295 }
321 296
322 int SimulcastEncoderAdapter::RegisterEncodeCompleteCallback( 297 int SimulcastEncoderAdapter::RegisterEncodeCompleteCallback(
323 EncodedImageCallback* callback) { 298 EncodedImageCallback* callback) {
324 encoded_complete_callback_ = callback; 299 encoded_complete_callback_ = callback;
325 return WEBRTC_VIDEO_CODEC_OK; 300 return WEBRTC_VIDEO_CODEC_OK;
326 } 301 }
327 302
328 int SimulcastEncoderAdapter::SetChannelParameters(uint32_t packet_loss, 303 int SimulcastEncoderAdapter::SetChannelParameters(uint32_t packet_loss,
329 int64_t rtt) { 304 int64_t rtt) {
330 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { 305 for (const auto& it : encoders_)
331 streaminfos_[stream_idx].encoder->SetChannelParameters(packet_loss, rtt); 306 it.second.first->SetChannelParameters(packet_loss, rtt);
332 }
333 return WEBRTC_VIDEO_CODEC_OK; 307 return WEBRTC_VIDEO_CODEC_OK;
334 } 308 }
335 309
336 int SimulcastEncoderAdapter::SetRates(uint32_t new_bitrate_kbit, 310 int SimulcastEncoderAdapter::SetRates(uint32_t new_bitrate_kbit,
337 uint32_t new_framerate) { 311 uint32_t new_framerate) {
338 if (!Initialized()) { 312 if (!Initialized())
339 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; 313 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
340 } 314
341 if (new_framerate < 1) { 315 if (new_framerate < 1)
342 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; 316 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
343 } 317
344 if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) { 318 if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate)
345 new_bitrate_kbit = codec_.maxBitrate; 319 new_bitrate_kbit = codec_.maxBitrate;
346 } 320
347 if (new_bitrate_kbit < codec_.minBitrate) { 321 if (new_bitrate_kbit < codec_.minBitrate)
348 new_bitrate_kbit = codec_.minBitrate; 322 new_bitrate_kbit = codec_.minBitrate;
349 } 323
350 if (codec_.numberOfSimulcastStreams > 0 && 324 if (codec_.numberOfSimulcastStreams > 0 &&
351 new_bitrate_kbit < codec_.simulcastStream[0].minBitrate) { 325 new_bitrate_kbit < codec_.simulcastStream[0].minBitrate) {
352 new_bitrate_kbit = codec_.simulcastStream[0].minBitrate; 326 new_bitrate_kbit = codec_.simulcastStream[0].minBitrate;
353 } 327 }
328
354 codec_.maxFramerate = new_framerate; 329 codec_.maxFramerate = new_framerate;
355 330 stream_states_.AllocateBitrate(new_bitrate_kbit * 1000);
356 bool send_stream = true; 331 for (const SimulcastState::Stream& stream : stream_states_.Streams()) {
357 uint32_t stream_bitrate = 0; 332 encoders_[stream.idx].first->SetRates(stream.allocated_rate_bps / 1000,
358 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { 333 new_framerate);
359 stream_bitrate = GetStreamBitrate(stream_idx, streaminfos_.size(),
360 new_bitrate_kbit, &send_stream);
361 // Need a key frame if we have not sent this stream before.
362 if (send_stream && !streaminfos_[stream_idx].send_stream) {
363 streaminfos_[stream_idx].key_frame_request = true;
364 }
365 streaminfos_[stream_idx].send_stream = send_stream;
366
367 // TODO(holmer): This is a temporary hack for screensharing, where we
368 // interpret the startBitrate as the encoder target bitrate. This is
369 // to allow for a different max bitrate, so if the codec can't meet
370 // the target we still allow it to overshoot up to the max before dropping
371 // frames. This hack should be improved.
372 if (codec_.targetBitrate > 0 &&
373 (codec_.codecSpecific.VP8.numberOfTemporalLayers == 2 ||
374 codec_.simulcastStream[0].numberOfTemporalLayers == 2)) {
375 stream_bitrate = std::min(codec_.maxBitrate, stream_bitrate);
376 // TODO(ronghuawu): Can't change max bitrate via the VideoEncoder
377 // interface. And VP8EncoderImpl doesn't take negative framerate.
378 // max_bitrate = std::min(codec_.maxBitrate, stream_bitrate);
379 // new_framerate = -1;
380 }
381
382 streaminfos_[stream_idx].encoder->SetRates(stream_bitrate, new_framerate);
383 } 334 }
384 335
385 return WEBRTC_VIDEO_CODEC_OK; 336 return WEBRTC_VIDEO_CODEC_OK;
386 } 337 }
387 338
388 int32_t SimulcastEncoderAdapter::Encoded( 339 int32_t SimulcastEncoderAdapter::Encoded(
389 size_t stream_idx, 340 size_t stream_idx,
390 const EncodedImage& encodedImage, 341 const EncodedImage& encodedImage,
391 const CodecSpecificInfo* codecSpecificInfo, 342 const CodecSpecificInfo* codecSpecificInfo,
392 const RTPFragmentationHeader* fragmentation) { 343 const RTPFragmentationHeader* fragmentation) {
393 CodecSpecificInfo stream_codec_specific = *codecSpecificInfo; 344 CodecSpecificInfo stream_codec_specific = *codecSpecificInfo;
394 CodecSpecificInfoVP8* vp8Info = &(stream_codec_specific.codecSpecific.VP8); 345 CodecSpecificInfoVP8* vp8Info = &(stream_codec_specific.codecSpecific.VP8);
395 vp8Info->simulcastIdx = stream_idx; 346 vp8Info->simulcastIdx = stream_idx;
396 347
397 return encoded_complete_callback_->Encoded( 348 return encoded_complete_callback_->Encoded(
398 encodedImage, &stream_codec_specific, fragmentation); 349 encodedImage, &stream_codec_specific, fragmentation);
399 } 350 }
400 351
401 uint32_t SimulcastEncoderAdapter::GetStreamBitrate(
402 int stream_idx,
403 size_t total_number_of_streams,
404 uint32_t new_bitrate_kbit,
405 bool* send_stream) const {
406 if (total_number_of_streams == 1) {
407 *send_stream = true;
408 return new_bitrate_kbit;
409 }
410
411 // The bitrate needed to start sending this stream is given by the
412 // minimum bitrate allowed for encoding this stream, plus the sum target
413 // rates of all lower streams.
414 uint32_t sum_target_lower_streams =
415 SumStreamTargetBitrate(stream_idx, codec_);
416 uint32_t bitrate_to_send_this_layer =
417 codec_.simulcastStream[stream_idx].minBitrate + sum_target_lower_streams;
418 if (new_bitrate_kbit >= bitrate_to_send_this_layer) {
419 // We have enough bandwidth to send this stream.
420 *send_stream = true;
421 // Bitrate for this stream is the new bitrate (|new_bitrate_kbit|) minus the
422 // sum target rates of the lower streams, and capped to a maximum bitrate.
423 // The maximum cap depends on whether we send the next higher stream.
424 // If we will be sending the next higher stream, |max_rate| is given by
425 // current stream's |targetBitrate|, otherwise it's capped by |maxBitrate|.
426 if (stream_idx < codec_.numberOfSimulcastStreams - 1) {
427 unsigned int max_rate = codec_.simulcastStream[stream_idx].maxBitrate;
428 if (new_bitrate_kbit >=
429 SumStreamTargetBitrate(stream_idx + 1, codec_) +
430 codec_.simulcastStream[stream_idx + 1].minBitrate) {
431 max_rate = codec_.simulcastStream[stream_idx].targetBitrate;
432 }
433 return std::min(new_bitrate_kbit - sum_target_lower_streams, max_rate);
434 } else {
435 // For the highest stream (highest resolution), the |targetBitRate| and
436 // |maxBitrate| are not used. Any excess bitrate (above the targets of
437 // all lower streams) is given to this (highest resolution) stream.
438 return new_bitrate_kbit - sum_target_lower_streams;
439 }
440 } else {
441 // Not enough bitrate for this stream.
442 // Return our max bitrate of |stream_idx| - 1, but we don't send it. We need
443 // to keep this resolution coding in order for the multi-encoder to work.
444 *send_stream = false;
445 return codec_.simulcastStream[stream_idx - 1].maxBitrate;
446 }
447 }
448
449 void SimulcastEncoderAdapter::PopulateStreamCodec(
450 const webrtc::VideoCodec* inst,
451 int stream_index,
452 size_t total_number_of_streams,
453 bool highest_resolution_stream,
454 webrtc::VideoCodec* stream_codec,
455 bool* send_stream) {
456 *stream_codec = *inst;
457
458 // Stream specific settings.
459 stream_codec->codecSpecific.VP8.numberOfTemporalLayers =
460 inst->simulcastStream[stream_index].numberOfTemporalLayers;
461 stream_codec->numberOfSimulcastStreams = 0;
462 stream_codec->width = inst->simulcastStream[stream_index].width;
463 stream_codec->height = inst->simulcastStream[stream_index].height;
464 stream_codec->maxBitrate = inst->simulcastStream[stream_index].maxBitrate;
465 stream_codec->minBitrate = inst->simulcastStream[stream_index].minBitrate;
466 stream_codec->qpMax = inst->simulcastStream[stream_index].qpMax;
467 // Settings that are based on stream/resolution.
468 if (stream_index == 0) {
469 // Settings for lowest spatial resolutions.
470 stream_codec->qpMax = kLowestResMaxQp;
471 }
472 if (!highest_resolution_stream) {
473 // For resolutions below CIF, set the codec |complexity| parameter to
474 // kComplexityHigher, which maps to cpu_used = -4.
475 int pixels_per_frame = stream_codec->width * stream_codec->height;
476 if (pixels_per_frame < 352 * 288) {
477 stream_codec->codecSpecific.VP8.complexity = webrtc::kComplexityHigher;
478 }
479 // Turn off denoising for all streams but the highest resolution.
480 stream_codec->codecSpecific.VP8.denoisingOn = false;
481 }
482 // TODO(ronghuawu): what to do with targetBitrate.
483
484 int stream_bitrate = GetStreamBitrate(stream_index, total_number_of_streams,
485 inst->startBitrate, send_stream);
486 stream_codec->startBitrate = stream_bitrate;
487 }
488
489 bool SimulcastEncoderAdapter::Initialized() const { 352 bool SimulcastEncoderAdapter::Initialized() const {
490 return !streaminfos_.empty(); 353 return !encoders_.empty();
491 } 354 }
492 355
493 void SimulcastEncoderAdapter::OnDroppedFrame() { 356 void SimulcastEncoderAdapter::OnDroppedFrame() {
494 streaminfos_[0].encoder->OnDroppedFrame(); 357 for (const auto& it : encoders_)
358 it.second.first->OnDroppedFrame();
495 } 359 }
496 360
497 bool SimulcastEncoderAdapter::SupportsNativeHandle() const { 361 bool SimulcastEncoderAdapter::SupportsNativeHandle() const {
498 // We should not be calling this method before streaminfos_ are configured. 362 // We should not be calling this method before streaminfos_ are configured.
499 RTC_DCHECK(!streaminfos_.empty()); 363 RTC_DCHECK(Initialized());
500 // TODO(pbos): Support textures when using more than one encoder. 364 // TODO(pbos): Support textures when using more than one encoder.
501 if (streaminfos_.size() != 1) 365 if (encoders_.size() != 1)
502 return false; 366 return false;
503 return streaminfos_[0].encoder->SupportsNativeHandle(); 367 return encoders_.begin()->second.first->SupportsNativeHandle();
504 } 368 }
505 369
506 const char* SimulcastEncoderAdapter::ImplementationName() const { 370 const char* SimulcastEncoderAdapter::ImplementationName() const {
507 return implementation_name_.c_str(); 371 return implementation_name_.c_str();
508 } 372 }
509 373
510 } // namespace webrtc 374 } // namespace webrtc
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698