| OLD | NEW |
| 1 /* | 1 /* |
| 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. |
| 3 * | 3 * |
| 4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 */ | 9 */ |
| 10 | 10 |
| 11 #include "webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h" | 11 #include "webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h" |
| 12 | 12 |
| 13 #include <algorithm> | 13 #include <algorithm> |
| 14 | 14 |
| 15 // NOTE(ajm): Path provided by gyp. | 15 // NOTE(ajm): Path provided by gyp. |
| 16 #include "libyuv/scale.h" // NOLINT | 16 #include "libyuv/scale.h" // NOLINT |
| 17 | 17 |
| 18 #include "webrtc/base/checks.h" | 18 #include "webrtc/base/checks.h" |
| 19 #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h" | 19 #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h" |
| 20 #include "webrtc/system_wrappers/include/clock.h" | 20 #include "webrtc/system_wrappers/include/clock.h" |
| 21 | 21 |
| 22 namespace { | 22 namespace { |
| 23 | 23 |
| 24 const unsigned int kDefaultMinQp = 2; | 24 const unsigned int kDefaultMinQp = 2; |
| 25 const unsigned int kDefaultMaxQp = 56; | 25 const unsigned int kDefaultMaxQp = 56; |
| 26 // Max qp for lowest spatial resolution when doing simulcast. | 26 // Max qp for lowest spatial resolution when doing simulcast. |
| 27 const unsigned int kLowestResMaxQp = 45; | 27 const unsigned int kLowestResMaxQp = 45; |
| 28 | 28 |
| 29 uint32_t SumStreamTargetBitrate(int streams, const webrtc::VideoCodec& codec) { | 29 static const char* kDefaultImplementationName = |
| 30 uint32_t bitrate_sum = 0; | 30 "SimulcastEncoderAdapter (uninitialized)"; |
| 31 for (int i = 0; i < streams; ++i) { | |
| 32 bitrate_sum += codec.simulcastStream[i].targetBitrate; | |
| 33 } | |
| 34 return bitrate_sum; | |
| 35 } | |
| 36 | |
| 37 uint32_t SumStreamMaxBitrate(int streams, const webrtc::VideoCodec& codec) { | |
| 38 uint32_t bitrate_sum = 0; | |
| 39 for (int i = 0; i < streams; ++i) { | |
| 40 bitrate_sum += codec.simulcastStream[i].maxBitrate; | |
| 41 } | |
| 42 return bitrate_sum; | |
| 43 } | |
| 44 | |
| 45 int NumberOfStreams(const webrtc::VideoCodec& codec) { | |
| 46 int streams = | |
| 47 codec.numberOfSimulcastStreams < 1 ? 1 : codec.numberOfSimulcastStreams; | |
| 48 uint32_t simulcast_max_bitrate = SumStreamMaxBitrate(streams, codec); | |
| 49 if (simulcast_max_bitrate == 0) { | |
| 50 streams = 1; | |
| 51 } | |
| 52 return streams; | |
| 53 } | |
| 54 | 31 |
| 55 bool ValidSimulcastResolutions(const webrtc::VideoCodec& codec, | 32 bool ValidSimulcastResolutions(const webrtc::VideoCodec& codec, |
| 56 int num_streams) { | 33 int num_streams) { |
| 57 if (codec.width != codec.simulcastStream[num_streams - 1].width || | 34 if (codec.width != codec.simulcastStream[num_streams - 1].width || |
| 58 codec.height != codec.simulcastStream[num_streams - 1].height) { | 35 codec.height != codec.simulcastStream[num_streams - 1].height) { |
| 59 return false; | 36 return false; |
| 60 } | 37 } |
| 61 for (int i = 0; i < num_streams; ++i) { | 38 for (int i = 0; i < num_streams; ++i) { |
| 62 if (codec.width * codec.simulcastStream[i].height != | 39 if (codec.width * codec.simulcastStream[i].height != |
| 63 codec.height * codec.simulcastStream[i].width) { | 40 codec.height * codec.simulcastStream[i].width) { |
| (...skipping 21 matching lines...) Expand all Loading... |
| 85 inst->numberOfSimulcastStreams > 1) { | 62 inst->numberOfSimulcastStreams > 1) { |
| 86 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; | 63 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 87 } | 64 } |
| 88 if (inst->codecSpecific.VP8.automaticResizeOn && | 65 if (inst->codecSpecific.VP8.automaticResizeOn && |
| 89 inst->numberOfSimulcastStreams > 1) { | 66 inst->numberOfSimulcastStreams > 1) { |
| 90 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; | 67 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 91 } | 68 } |
| 92 return WEBRTC_VIDEO_CODEC_OK; | 69 return WEBRTC_VIDEO_CODEC_OK; |
| 93 } | 70 } |
| 94 | 71 |
| 95 // TL1 FrameDropper's max time to drop frames. | |
| 96 const float kTl1MaxTimeToDropFrames = 20.0f; | |
| 97 | |
| 98 struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory { | 72 struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory { |
| 99 ScreenshareTemporalLayersFactory() | 73 ScreenshareTemporalLayersFactory() {} |
| 100 : tl1_frame_dropper_(kTl1MaxTimeToDropFrames) {} | |
| 101 | |
| 102 virtual ~ScreenshareTemporalLayersFactory() {} | 74 virtual ~ScreenshareTemporalLayersFactory() {} |
| 103 | 75 |
| 104 virtual webrtc::TemporalLayers* Create(int num_temporal_layers, | 76 virtual webrtc::TemporalLayers* Create(int num_temporal_layers, |
| 105 uint8_t initial_tl0_pic_idx) const { | 77 uint8_t initial_tl0_pic_idx) const { |
| 106 return new webrtc::ScreenshareLayers(num_temporal_layers, rand(), | 78 return new webrtc::ScreenshareLayers(num_temporal_layers, rand(), |
| 107 webrtc::Clock::GetRealTimeClock()); | 79 webrtc::Clock::GetRealTimeClock()); |
| 108 } | 80 } |
| 109 | |
| 110 mutable webrtc::FrameDropper tl0_frame_dropper_; | |
| 111 mutable webrtc::FrameDropper tl1_frame_dropper_; | |
| 112 }; | 81 }; |
| 113 | 82 |
| 114 // An EncodedImageCallback implementation that forwards on calls to a | 83 // An EncodedImageCallback implementation that forwards on calls to a |
| 115 // SimulcastEncoderAdapter, but with the stream index it's registered with as | 84 // SimulcastEncoderAdapter, but with the stream index it's registered with as |
| 116 // the first parameter to Encoded. | 85 // the first parameter to Encoded. |
| 117 class AdapterEncodedImageCallback : public webrtc::EncodedImageCallback { | 86 class AdapterEncodedImageCallback : public webrtc::EncodedImageCallback { |
| 118 public: | 87 public: |
| 119 AdapterEncodedImageCallback(webrtc::SimulcastEncoderAdapter* adapter, | 88 AdapterEncodedImageCallback(webrtc::SimulcastEncoderAdapter* adapter, |
| 120 size_t stream_idx) | 89 size_t stream_idx) |
| 121 : adapter_(adapter), stream_idx_(stream_idx) {} | 90 : adapter_(adapter), stream_idx_(stream_idx) {} |
| 122 | 91 |
| 123 int32_t Encoded( | 92 int32_t Encoded( |
| 124 const webrtc::EncodedImage& encodedImage, | 93 const webrtc::EncodedImage& encodedImage, |
| 125 const webrtc::CodecSpecificInfo* codecSpecificInfo = NULL, | 94 const webrtc::CodecSpecificInfo* codecSpecificInfo = NULL, |
| 126 const webrtc::RTPFragmentationHeader* fragmentation = NULL) override { | 95 const webrtc::RTPFragmentationHeader* fragmentation = NULL) override { |
| 127 return adapter_->Encoded(stream_idx_, encodedImage, codecSpecificInfo, | 96 return adapter_->Encoded(stream_idx_, encodedImage, codecSpecificInfo, |
| 128 fragmentation); | 97 fragmentation); |
| 129 } | 98 } |
| 130 | 99 |
| 131 private: | 100 private: |
| 132 webrtc::SimulcastEncoderAdapter* const adapter_; | 101 webrtc::SimulcastEncoderAdapter* const adapter_; |
| 133 const size_t stream_idx_; | 102 const size_t stream_idx_; |
| 134 }; | 103 }; |
| 135 | 104 |
| 105 webrtc::VideoCodec DefaultCodec() { |
| 106 webrtc::VideoCodec codec; |
| 107 memset(&codec, 0, sizeof(webrtc::VideoCodec)); |
| 108 return codec; |
| 109 } |
| 110 |
| 136 } // namespace | 111 } // namespace |
| 137 | 112 |
| 138 namespace webrtc { | 113 namespace webrtc { |
| 139 | 114 |
| 140 SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory) | 115 SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory) |
| 141 : factory_(factory), | 116 : factory_(factory), |
| 142 encoded_complete_callback_(NULL), | 117 codec_(DefaultCodec()), |
| 143 implementation_name_("SimulcastEncoderAdapter") { | 118 stream_states_(codec_), |
| 144 memset(&codec_, 0, sizeof(webrtc::VideoCodec)); | 119 encoded_complete_callback_(nullptr), |
| 145 } | 120 implementation_name_(kDefaultImplementationName) {} |
| 146 | 121 |
| 147 SimulcastEncoderAdapter::~SimulcastEncoderAdapter() { | 122 SimulcastEncoderAdapter::~SimulcastEncoderAdapter() { |
| 148 Release(); | 123 Release(); |
| 149 } | 124 } |
| 150 | 125 |
| 151 int SimulcastEncoderAdapter::Release() { | 126 int SimulcastEncoderAdapter::Release() { |
| 152 // TODO(pbos): Keep the last encoder instance but call ::Release() on it, then | 127 // TODO(pbos): Keep the last encoder instance but call ::Release() on it, then |
| 153 // re-use this instance in ::InitEncode(). This means that changing | 128 // re-use this instance in ::InitEncode(). This means that changing |
| 154 // resolutions doesn't require reallocation of the first encoder, but only | 129 // resolutions doesn't require reallocation of the first encoder, but only |
| 155 // reinitialization, which makes sense. Then Destroy this instance instead in | 130 // reinitialization, which makes sense. Then Destroy this instance instead in |
| 156 // ~SimulcastEncoderAdapter(). | 131 // ~SimulcastEncoderAdapter(). |
| 157 while (!streaminfos_.empty()) { | 132 stream_states_ = SimulcastState(codec_); |
| 158 VideoEncoder* encoder = streaminfos_.back().encoder; | 133 for (auto& it : encoders_) |
| 159 EncodedImageCallback* callback = streaminfos_.back().callback; | 134 factory_->Destroy(it.second.first); |
| 160 factory_->Destroy(encoder); | 135 encoders_.clear(); |
| 161 delete callback; | 136 |
| 162 streaminfos_.pop_back(); | |
| 163 } | |
| 164 return WEBRTC_VIDEO_CODEC_OK; | 137 return WEBRTC_VIDEO_CODEC_OK; |
| 165 } | 138 } |
| 166 | 139 |
| 167 int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, | 140 int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, |
| 168 int number_of_cores, | 141 int number_of_cores, |
| 169 size_t max_payload_size) { | 142 size_t max_payload_size) { |
| 170 if (number_of_cores < 1) { | 143 if (number_of_cores < 1) { |
| 171 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; | 144 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 172 } | 145 } |
| 173 | 146 |
| 174 int ret = VerifyCodec(inst); | 147 int ret = VerifyCodec(inst); |
| 175 if (ret < 0) { | 148 if (ret < 0) { |
| 176 return ret; | 149 return ret; |
| 177 } | 150 } |
| 178 | 151 |
| 179 ret = Release(); | 152 ret = Release(); |
| 180 if (ret < 0) { | 153 if (ret < 0) { |
| 181 return ret; | 154 return ret; |
| 182 } | 155 } |
| 183 | 156 |
| 184 int number_of_streams = NumberOfStreams(*inst); | 157 std::unique_ptr<SimulcastState> simulcast(new SimulcastState(*inst)); |
| 158 |
| 159 int number_of_streams = simulcast->NumStreams(); |
| 185 const bool doing_simulcast = (number_of_streams > 1); | 160 const bool doing_simulcast = (number_of_streams > 1); |
| 186 | 161 |
| 187 if (doing_simulcast && !ValidSimulcastResolutions(*inst, number_of_streams)) { | 162 if (doing_simulcast && !ValidSimulcastResolutions(*inst, number_of_streams)) { |
| 188 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; | 163 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 189 } | 164 } |
| 190 | 165 |
| 191 codec_ = *inst; | 166 codec_ = *inst; |
| 192 | 167 |
| 193 // Special mode when screensharing on a single stream. | 168 // Special mode when screensharing on a single stream. |
| 194 if (number_of_streams == 1 && inst->mode == kScreensharing) { | 169 if (number_of_streams == 1 && inst->mode == kScreensharing) { |
| 195 screensharing_tl_factory_.reset(new ScreenshareTemporalLayersFactory()); | 170 screensharing_tl_factory_.reset(new ScreenshareTemporalLayersFactory()); |
| 196 codec_.codecSpecific.VP8.tl_factory = screensharing_tl_factory_.get(); | 171 codec_.codecSpecific.VP8.tl_factory = screensharing_tl_factory_.get(); |
| 197 } | 172 } |
| 198 | 173 |
| 199 std::string implementation_name; | 174 std::string implementation_name; |
| 200 // Create |number_of_streams| of encoder instances and init them. | 175 // Create |number_of_streams| of encoder instances and init them. |
| 201 for (int i = 0; i < number_of_streams; ++i) { | 176 stream_states_ = *simulcast; |
| 177 for (const SimulcastState::Stream& stream : stream_states_.Streams()) { |
| 202 VideoCodec stream_codec; | 178 VideoCodec stream_codec; |
| 203 bool send_stream = true; | |
| 204 if (!doing_simulcast) { | 179 if (!doing_simulcast) { |
| 205 stream_codec = codec_; | 180 stream_codec = codec_; |
| 206 stream_codec.numberOfSimulcastStreams = 1; | 181 stream_codec.numberOfSimulcastStreams = 1; |
| 207 } else { | 182 } else { |
| 208 bool highest_resolution_stream = (i == (number_of_streams - 1)); | 183 stream_codec = stream.AsCodec(); |
| 209 PopulateStreamCodec(&codec_, i, number_of_streams, | 184 if (stream.idx == 0) { |
| 210 highest_resolution_stream, &stream_codec, | 185 // Settings for lowest spatial resolutions. |
| 211 &send_stream); | 186 stream_codec.qpMax = kLowestResMaxQp; |
| 187 } |
| 188 if (stream.idx < codec_.numberOfSimulcastStreams - 1) { |
| 189 // For resolutions below CIF, set the codec |complexity| parameter to |
| 190 // kComplexityHigher, which maps to cpu_used = -4. |
| 191 if (stream_codec.width * stream_codec.height < 352 * 288) { |
| 192 stream_codec.codecSpecific.VP8.complexity = webrtc::kComplexityHigher; |
| 193 } |
| 194 // Turn off denoising for all streams but the highest resolution. |
| 195 stream_codec.codecSpecific.VP8.denoisingOn = false; |
| 196 } |
| 212 } | 197 } |
| 213 | 198 |
| 214 // TODO(ronghuawu): Remove once this is handled in VP8EncoderImpl. | 199 // TODO(ronghuawu): Remove once this is handled in VP8EncoderImpl. |
| 215 if (stream_codec.qpMax < kDefaultMinQp) { | 200 if (stream_codec.qpMax < kDefaultMinQp) { |
| 216 stream_codec.qpMax = kDefaultMaxQp; | 201 stream_codec.qpMax = kDefaultMaxQp; |
| 217 } | 202 } |
| 218 | 203 |
| 219 VideoEncoder* encoder = factory_->Create(); | 204 VideoEncoder* encoder = factory_->Create(); |
| 220 ret = encoder->InitEncode(&stream_codec, number_of_cores, max_payload_size); | 205 ret = encoder->InitEncode(&stream_codec, number_of_cores, max_payload_size); |
| 221 if (ret < 0) { | 206 if (ret < 0) { |
| 222 Release(); | 207 Release(); |
| 223 return ret; | 208 return ret; |
| 224 } | 209 } |
| 225 EncodedImageCallback* callback = new AdapterEncodedImageCallback(this, i); | 210 EncodedImageCallback* callback = |
| 211 new AdapterEncodedImageCallback(this, stream.idx); |
| 226 encoder->RegisterEncodeCompleteCallback(callback); | 212 encoder->RegisterEncodeCompleteCallback(callback); |
| 227 streaminfos_.push_back(StreamInfo(encoder, callback, stream_codec.width, | 213 encoders_[stream.idx] = EncoderContext( |
| 228 stream_codec.height, send_stream)); | 214 encoder, std::unique_ptr<EncodedImageCallback>(callback)); |
| 229 if (i != 0) | 215 if (stream.idx != 0) |
| 230 implementation_name += ", "; | 216 implementation_name += ", "; |
| 231 implementation_name += streaminfos_[i].encoder->ImplementationName(); | 217 implementation_name += encoder->ImplementationName(); |
| 232 } | 218 } |
| 219 |
| 233 if (doing_simulcast) { | 220 if (doing_simulcast) { |
| 234 implementation_name_ = | 221 implementation_name_ = |
| 235 "SimulcastEncoderAdapter (" + implementation_name + ")"; | 222 "SimulcastEncoderAdapter (" + implementation_name + ")"; |
| 236 } else { | 223 } else { |
| 237 implementation_name_ = implementation_name; | 224 implementation_name_ = implementation_name; |
| 238 } | 225 } |
| 239 return WEBRTC_VIDEO_CODEC_OK; | 226 return WEBRTC_VIDEO_CODEC_OK; |
| 240 } | 227 } |
| 241 | 228 |
| 242 int SimulcastEncoderAdapter::Encode( | 229 int SimulcastEncoderAdapter::Encode( |
| 243 const VideoFrame& input_image, | 230 const VideoFrame& input_image, |
| 244 const CodecSpecificInfo* codec_specific_info, | 231 const CodecSpecificInfo* codec_specific_info, |
| 245 const std::vector<FrameType>* frame_types) { | 232 const std::vector<FrameType>* frame_types) { |
| 246 if (!Initialized()) { | 233 if (!Initialized()) { |
| 247 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; | 234 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 248 } | 235 } |
| 249 if (encoded_complete_callback_ == NULL) { | 236 if (encoded_complete_callback_ == NULL) { |
| 250 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; | 237 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 251 } | 238 } |
| 252 | 239 |
| 253 // All active streams should generate a key frame if | 240 // TODO(sprang): Do we really need to force key frames on all streams? |
| 254 // a key frame is requested by any stream. | 241 // All active streams should generate a key frame if a key frame is requested |
| 255 bool send_key_frame = false; | 242 // by any stream. |
| 256 if (frame_types) { | 243 bool force_key_frame = frame_types && |
| 257 for (size_t i = 0; i < frame_types->size(); ++i) { | 244 std::find(frame_types->begin(), frame_types->end(), |
| 258 if (frame_types->at(i) == kVideoFrameKey) { | 245 kVideoFrameKey) != frame_types->end(); |
| 259 send_key_frame = true; | 246 int active_streams = stream_states_.NumSendingStreams(); |
| 260 break; | 247 for (int i = 0; i < active_streams; ++i) |
| 261 } | 248 force_key_frame |= stream_states_.GetAndResetKeyFrameRequest(i); |
| 262 } | |
| 263 } | |
| 264 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { | |
| 265 if (streaminfos_[stream_idx].key_frame_request && | |
| 266 streaminfos_[stream_idx].send_stream) { | |
| 267 send_key_frame = true; | |
| 268 break; | |
| 269 } | |
| 270 } | |
| 271 | 249 |
| 272 int src_width = input_image.width(); | 250 int src_width = input_image.width(); |
| 273 int src_height = input_image.height(); | 251 int src_height = input_image.height(); |
| 274 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { | 252 for (const SimulcastState::Stream& stream : stream_states_.Streams()) { |
| 275 // Don't encode frames in resolutions that we don't intend to send. | 253 if (!stream.sending) |
| 276 if (!streaminfos_[stream_idx].send_stream) | |
| 277 continue; | 254 continue; |
| 278 | 255 |
| 279 std::vector<FrameType> stream_frame_types; | 256 std::vector<FrameType> stream_frame_types; |
| 280 if (send_key_frame) { | 257 if (force_key_frame) { |
| 281 stream_frame_types.push_back(kVideoFrameKey); | 258 stream_frame_types.push_back(kVideoFrameKey); |
| 282 streaminfos_[stream_idx].key_frame_request = false; | |
| 283 } else { | 259 } else { |
| 284 stream_frame_types.push_back(kVideoFrameDelta); | 260 stream_frame_types.push_back(kVideoFrameDelta); |
| 285 } | 261 } |
| 286 | 262 |
| 287 int dst_width = streaminfos_[stream_idx].width; | 263 int dst_width = stream.config.width; |
| 288 int dst_height = streaminfos_[stream_idx].height; | 264 int dst_height = stream.config.height; |
| 289 // If scaling isn't required, because the input resolution | 265 // If scaling isn't required, because the input resolution matches the |
| 290 // matches the destination or the input image is empty (e.g. | 266 // destination or the input image is empty (e.g. a keyframe request for |
| 291 // a keyframe request for encoders with internal camera | 267 // encoders with internal camera sources), pass the image on directly. |
| 292 // sources), pass the image on directly. Otherwise, we'll | 268 // Otherwise, we'll scale it to match what the encoder expects (below). |
| 293 // scale it to match what the encoder expects (below). | |
| 294 if ((dst_width == src_width && dst_height == src_height) || | 269 if ((dst_width == src_width && dst_height == src_height) || |
| 295 input_image.IsZeroSize()) { | 270 input_image.IsZeroSize()) { |
| 296 streaminfos_[stream_idx].encoder->Encode(input_image, codec_specific_info, | 271 encoders_[stream.idx].first->Encode(input_image, codec_specific_info, |
| 297 &stream_frame_types); | 272 &stream_frame_types); |
| 298 } else { | 273 } else { |
| 299 VideoFrame dst_frame; | 274 VideoFrame dst_frame; |
| 300 // Making sure that destination frame is of sufficient size. | 275 // Making sure that destination frame is of sufficient size. |
| 301 // Aligning stride values based on width. | 276 // Aligning stride values based on width. |
| 302 dst_frame.CreateEmptyFrame(dst_width, dst_height, dst_width, | 277 dst_frame.CreateEmptyFrame(dst_width, dst_height, dst_width, |
| 303 (dst_width + 1) / 2, (dst_width + 1) / 2); | 278 (dst_width + 1) / 2, (dst_width + 1) / 2); |
| 304 libyuv::I420Scale( | 279 libyuv::I420Scale( |
| 305 input_image.buffer(kYPlane), input_image.stride(kYPlane), | 280 input_image.buffer(kYPlane), input_image.stride(kYPlane), |
| 306 input_image.buffer(kUPlane), input_image.stride(kUPlane), | 281 input_image.buffer(kUPlane), input_image.stride(kUPlane), |
| 307 input_image.buffer(kVPlane), input_image.stride(kVPlane), src_width, | 282 input_image.buffer(kVPlane), input_image.stride(kVPlane), src_width, |
| 308 src_height, dst_frame.buffer(kYPlane), dst_frame.stride(kYPlane), | 283 src_height, dst_frame.buffer(kYPlane), dst_frame.stride(kYPlane), |
| 309 dst_frame.buffer(kUPlane), dst_frame.stride(kUPlane), | 284 dst_frame.buffer(kUPlane), dst_frame.stride(kUPlane), |
| 310 dst_frame.buffer(kVPlane), dst_frame.stride(kVPlane), dst_width, | 285 dst_frame.buffer(kVPlane), dst_frame.stride(kVPlane), dst_width, |
| 311 dst_height, libyuv::kFilterBilinear); | 286 dst_height, libyuv::kFilterBilinear); |
| 312 dst_frame.set_timestamp(input_image.timestamp()); | 287 dst_frame.set_timestamp(input_image.timestamp()); |
| 313 dst_frame.set_render_time_ms(input_image.render_time_ms()); | 288 dst_frame.set_render_time_ms(input_image.render_time_ms()); |
| 314 streaminfos_[stream_idx].encoder->Encode(dst_frame, codec_specific_info, | 289 encoders_[stream.idx].first->Encode(dst_frame, codec_specific_info, |
| 315 &stream_frame_types); | 290 &stream_frame_types); |
| 316 } | 291 } |
| 317 } | 292 } |
| 318 | 293 |
| 319 return WEBRTC_VIDEO_CODEC_OK; | 294 return WEBRTC_VIDEO_CODEC_OK; |
| 320 } | 295 } |
| 321 | 296 |
| 322 int SimulcastEncoderAdapter::RegisterEncodeCompleteCallback( | 297 int SimulcastEncoderAdapter::RegisterEncodeCompleteCallback( |
| 323 EncodedImageCallback* callback) { | 298 EncodedImageCallback* callback) { |
| 324 encoded_complete_callback_ = callback; | 299 encoded_complete_callback_ = callback; |
| 325 return WEBRTC_VIDEO_CODEC_OK; | 300 return WEBRTC_VIDEO_CODEC_OK; |
| 326 } | 301 } |
| 327 | 302 |
| 328 int SimulcastEncoderAdapter::SetChannelParameters(uint32_t packet_loss, | 303 int SimulcastEncoderAdapter::SetChannelParameters(uint32_t packet_loss, |
| 329 int64_t rtt) { | 304 int64_t rtt) { |
| 330 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { | 305 for (const auto& it : encoders_) |
| 331 streaminfos_[stream_idx].encoder->SetChannelParameters(packet_loss, rtt); | 306 it.second.first->SetChannelParameters(packet_loss, rtt); |
| 332 } | |
| 333 return WEBRTC_VIDEO_CODEC_OK; | 307 return WEBRTC_VIDEO_CODEC_OK; |
| 334 } | 308 } |
| 335 | 309 |
| 336 int SimulcastEncoderAdapter::SetRates(uint32_t new_bitrate_kbit, | 310 int SimulcastEncoderAdapter::SetRates(uint32_t new_bitrate_kbit, |
| 337 uint32_t new_framerate) { | 311 uint32_t new_framerate) { |
| 338 if (!Initialized()) { | 312 if (!Initialized()) |
| 339 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; | 313 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 340 } | 314 |
| 341 if (new_framerate < 1) { | 315 if (new_framerate < 1) |
| 342 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; | 316 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 343 } | 317 |
| 344 if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) { | 318 if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) |
| 345 new_bitrate_kbit = codec_.maxBitrate; | 319 new_bitrate_kbit = codec_.maxBitrate; |
| 346 } | 320 |
| 347 if (new_bitrate_kbit < codec_.minBitrate) { | 321 if (new_bitrate_kbit < codec_.minBitrate) |
| 348 new_bitrate_kbit = codec_.minBitrate; | 322 new_bitrate_kbit = codec_.minBitrate; |
| 349 } | 323 |
| 350 if (codec_.numberOfSimulcastStreams > 0 && | 324 if (codec_.numberOfSimulcastStreams > 0 && |
| 351 new_bitrate_kbit < codec_.simulcastStream[0].minBitrate) { | 325 new_bitrate_kbit < codec_.simulcastStream[0].minBitrate) { |
| 352 new_bitrate_kbit = codec_.simulcastStream[0].minBitrate; | 326 new_bitrate_kbit = codec_.simulcastStream[0].minBitrate; |
| 353 } | 327 } |
| 328 |
| 354 codec_.maxFramerate = new_framerate; | 329 codec_.maxFramerate = new_framerate; |
| 355 | 330 stream_states_.AllocateBitrate(new_bitrate_kbit * 1000); |
| 356 bool send_stream = true; | 331 for (const SimulcastState::Stream& stream : stream_states_.Streams()) { |
| 357 uint32_t stream_bitrate = 0; | 332 encoders_[stream.idx].first->SetRates(stream.allocated_rate_bps / 1000, |
| 358 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { | 333 new_framerate); |
| 359 stream_bitrate = GetStreamBitrate(stream_idx, streaminfos_.size(), | |
| 360 new_bitrate_kbit, &send_stream); | |
| 361 // Need a key frame if we have not sent this stream before. | |
| 362 if (send_stream && !streaminfos_[stream_idx].send_stream) { | |
| 363 streaminfos_[stream_idx].key_frame_request = true; | |
| 364 } | |
| 365 streaminfos_[stream_idx].send_stream = send_stream; | |
| 366 | |
| 367 // TODO(holmer): This is a temporary hack for screensharing, where we | |
| 368 // interpret the startBitrate as the encoder target bitrate. This is | |
| 369 // to allow for a different max bitrate, so if the codec can't meet | |
| 370 // the target we still allow it to overshoot up to the max before dropping | |
| 371 // frames. This hack should be improved. | |
| 372 if (codec_.targetBitrate > 0 && | |
| 373 (codec_.codecSpecific.VP8.numberOfTemporalLayers == 2 || | |
| 374 codec_.simulcastStream[0].numberOfTemporalLayers == 2)) { | |
| 375 stream_bitrate = std::min(codec_.maxBitrate, stream_bitrate); | |
| 376 // TODO(ronghuawu): Can't change max bitrate via the VideoEncoder | |
| 377 // interface. And VP8EncoderImpl doesn't take negative framerate. | |
| 378 // max_bitrate = std::min(codec_.maxBitrate, stream_bitrate); | |
| 379 // new_framerate = -1; | |
| 380 } | |
| 381 | |
| 382 streaminfos_[stream_idx].encoder->SetRates(stream_bitrate, new_framerate); | |
| 383 } | 334 } |
| 384 | 335 |
| 385 return WEBRTC_VIDEO_CODEC_OK; | 336 return WEBRTC_VIDEO_CODEC_OK; |
| 386 } | 337 } |
| 387 | 338 |
| 388 int32_t SimulcastEncoderAdapter::Encoded( | 339 int32_t SimulcastEncoderAdapter::Encoded( |
| 389 size_t stream_idx, | 340 size_t stream_idx, |
| 390 const EncodedImage& encodedImage, | 341 const EncodedImage& encodedImage, |
| 391 const CodecSpecificInfo* codecSpecificInfo, | 342 const CodecSpecificInfo* codecSpecificInfo, |
| 392 const RTPFragmentationHeader* fragmentation) { | 343 const RTPFragmentationHeader* fragmentation) { |
| 393 CodecSpecificInfo stream_codec_specific = *codecSpecificInfo; | 344 CodecSpecificInfo stream_codec_specific = *codecSpecificInfo; |
| 394 CodecSpecificInfoVP8* vp8Info = &(stream_codec_specific.codecSpecific.VP8); | 345 CodecSpecificInfoVP8* vp8Info = &(stream_codec_specific.codecSpecific.VP8); |
| 395 vp8Info->simulcastIdx = stream_idx; | 346 vp8Info->simulcastIdx = stream_idx; |
| 396 | 347 |
| 397 return encoded_complete_callback_->Encoded( | 348 return encoded_complete_callback_->Encoded( |
| 398 encodedImage, &stream_codec_specific, fragmentation); | 349 encodedImage, &stream_codec_specific, fragmentation); |
| 399 } | 350 } |
| 400 | 351 |
| 401 uint32_t SimulcastEncoderAdapter::GetStreamBitrate( | |
| 402 int stream_idx, | |
| 403 size_t total_number_of_streams, | |
| 404 uint32_t new_bitrate_kbit, | |
| 405 bool* send_stream) const { | |
| 406 if (total_number_of_streams == 1) { | |
| 407 *send_stream = true; | |
| 408 return new_bitrate_kbit; | |
| 409 } | |
| 410 | |
| 411 // The bitrate needed to start sending this stream is given by the | |
| 412 // minimum bitrate allowed for encoding this stream, plus the sum target | |
| 413 // rates of all lower streams. | |
| 414 uint32_t sum_target_lower_streams = | |
| 415 SumStreamTargetBitrate(stream_idx, codec_); | |
| 416 uint32_t bitrate_to_send_this_layer = | |
| 417 codec_.simulcastStream[stream_idx].minBitrate + sum_target_lower_streams; | |
| 418 if (new_bitrate_kbit >= bitrate_to_send_this_layer) { | |
| 419 // We have enough bandwidth to send this stream. | |
| 420 *send_stream = true; | |
| 421 // Bitrate for this stream is the new bitrate (|new_bitrate_kbit|) minus the | |
| 422 // sum target rates of the lower streams, and capped to a maximum bitrate. | |
| 423 // The maximum cap depends on whether we send the next higher stream. | |
| 424 // If we will be sending the next higher stream, |max_rate| is given by | |
| 425 // current stream's |targetBitrate|, otherwise it's capped by |maxBitrate|. | |
| 426 if (stream_idx < codec_.numberOfSimulcastStreams - 1) { | |
| 427 unsigned int max_rate = codec_.simulcastStream[stream_idx].maxBitrate; | |
| 428 if (new_bitrate_kbit >= | |
| 429 SumStreamTargetBitrate(stream_idx + 1, codec_) + | |
| 430 codec_.simulcastStream[stream_idx + 1].minBitrate) { | |
| 431 max_rate = codec_.simulcastStream[stream_idx].targetBitrate; | |
| 432 } | |
| 433 return std::min(new_bitrate_kbit - sum_target_lower_streams, max_rate); | |
| 434 } else { | |
| 435 // For the highest stream (highest resolution), the |targetBitRate| and | |
| 436 // |maxBitrate| are not used. Any excess bitrate (above the targets of | |
| 437 // all lower streams) is given to this (highest resolution) stream. | |
| 438 return new_bitrate_kbit - sum_target_lower_streams; | |
| 439 } | |
| 440 } else { | |
| 441 // Not enough bitrate for this stream. | |
| 442 // Return our max bitrate of |stream_idx| - 1, but we don't send it. We need | |
| 443 // to keep this resolution coding in order for the multi-encoder to work. | |
| 444 *send_stream = false; | |
| 445 return codec_.simulcastStream[stream_idx - 1].maxBitrate; | |
| 446 } | |
| 447 } | |
| 448 | |
| 449 void SimulcastEncoderAdapter::PopulateStreamCodec( | |
| 450 const webrtc::VideoCodec* inst, | |
| 451 int stream_index, | |
| 452 size_t total_number_of_streams, | |
| 453 bool highest_resolution_stream, | |
| 454 webrtc::VideoCodec* stream_codec, | |
| 455 bool* send_stream) { | |
| 456 *stream_codec = *inst; | |
| 457 | |
| 458 // Stream specific settings. | |
| 459 stream_codec->codecSpecific.VP8.numberOfTemporalLayers = | |
| 460 inst->simulcastStream[stream_index].numberOfTemporalLayers; | |
| 461 stream_codec->numberOfSimulcastStreams = 0; | |
| 462 stream_codec->width = inst->simulcastStream[stream_index].width; | |
| 463 stream_codec->height = inst->simulcastStream[stream_index].height; | |
| 464 stream_codec->maxBitrate = inst->simulcastStream[stream_index].maxBitrate; | |
| 465 stream_codec->minBitrate = inst->simulcastStream[stream_index].minBitrate; | |
| 466 stream_codec->qpMax = inst->simulcastStream[stream_index].qpMax; | |
| 467 // Settings that are based on stream/resolution. | |
| 468 if (stream_index == 0) { | |
| 469 // Settings for lowest spatial resolutions. | |
| 470 stream_codec->qpMax = kLowestResMaxQp; | |
| 471 } | |
| 472 if (!highest_resolution_stream) { | |
| 473 // For resolutions below CIF, set the codec |complexity| parameter to | |
| 474 // kComplexityHigher, which maps to cpu_used = -4. | |
| 475 int pixels_per_frame = stream_codec->width * stream_codec->height; | |
| 476 if (pixels_per_frame < 352 * 288) { | |
| 477 stream_codec->codecSpecific.VP8.complexity = webrtc::kComplexityHigher; | |
| 478 } | |
| 479 // Turn off denoising for all streams but the highest resolution. | |
| 480 stream_codec->codecSpecific.VP8.denoisingOn = false; | |
| 481 } | |
| 482 // TODO(ronghuawu): what to do with targetBitrate. | |
| 483 | |
| 484 int stream_bitrate = GetStreamBitrate(stream_index, total_number_of_streams, | |
| 485 inst->startBitrate, send_stream); | |
| 486 stream_codec->startBitrate = stream_bitrate; | |
| 487 } | |
| 488 | |
| 489 bool SimulcastEncoderAdapter::Initialized() const { | 352 bool SimulcastEncoderAdapter::Initialized() const { |
| 490 return !streaminfos_.empty(); | 353 return !encoders_.empty(); |
| 491 } | 354 } |
| 492 | 355 |
| 493 void SimulcastEncoderAdapter::OnDroppedFrame() { | 356 void SimulcastEncoderAdapter::OnDroppedFrame() { |
| 494 streaminfos_[0].encoder->OnDroppedFrame(); | 357 for (const auto& it : encoders_) |
| 358 it.second.first->OnDroppedFrame(); |
| 495 } | 359 } |
| 496 | 360 |
| 497 bool SimulcastEncoderAdapter::SupportsNativeHandle() const { | 361 bool SimulcastEncoderAdapter::SupportsNativeHandle() const { |
| 498 // We should not be calling this method before streaminfos_ are configured. | 362 // We should not be calling this method before streaminfos_ are configured. |
| 499 RTC_DCHECK(!streaminfos_.empty()); | 363 RTC_DCHECK(Initialized()); |
| 500 // TODO(pbos): Support textures when using more than one encoder. | 364 // TODO(pbos): Support textures when using more than one encoder. |
| 501 if (streaminfos_.size() != 1) | 365 if (encoders_.size() != 1) |
| 502 return false; | 366 return false; |
| 503 return streaminfos_[0].encoder->SupportsNativeHandle(); | 367 return encoders_.begin()->second.first->SupportsNativeHandle(); |
| 504 } | 368 } |
| 505 | 369 |
| 506 const char* SimulcastEncoderAdapter::ImplementationName() const { | 370 const char* SimulcastEncoderAdapter::ImplementationName() const { |
| 507 return implementation_name_.c_str(); | 371 return implementation_name_.c_str(); |
| 508 } | 372 } |
| 509 | 373 |
| 510 } // namespace webrtc | 374 } // namespace webrtc |
| OLD | NEW |