OLD | NEW |
(Empty) | |
| 1 /* |
| 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. |
| 3 * |
| 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 * |
| 10 */ |
| 11 |
| 12 #include "webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h" |
| 13 |
| 14 #include <limits> |
| 15 |
| 16 #include "third_party/openh264/src/codec/api/svc/codec_api.h" |
| 17 #include "third_party/openh264/src/codec/api/svc/codec_app_def.h" |
| 18 #include "third_party/openh264/src/codec/api/svc/codec_def.h" |
| 19 |
| 20 #include "webrtc/base/checks.h" |
| 21 #include "webrtc/base/logging.h" |
| 22 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" |
| 23 |
| 24 namespace webrtc { |
| 25 |
| 26 namespace { |
| 27 |
| 28 const bool kOpenH264EncoderDetailedLogging = false; |
| 29 |
| 30 int NumberOfThreads(int width, int height, int number_of_cores) { |
| 31 if (width * height >= 1920 * 1080 && number_of_cores > 8) { |
| 32 return 8; // 8 threads for 1080p on high perf machines. |
| 33 } else if (width * height > 1280 * 960 && number_of_cores >= 6) { |
| 34 return 3; // 3 threads for 1080p. |
| 35 } else if (width * height > 640 * 480 && number_of_cores >= 3) { |
| 36 return 2; // 2 threads for qHD/HD. |
| 37 } else { |
| 38 return 1; // 1 thread for VGA or less. |
| 39 } |
| 40 } |
| 41 |
| 42 } // namespace |
| 43 |
| 44 static FrameType EVideoFrameType_to_FrameType(EVideoFrameType type) { |
| 45 switch (type) { |
| 46 case videoFrameTypeInvalid: |
| 47 return kEmptyFrame; |
| 48 case videoFrameTypeIDR: |
| 49 return kVideoFrameKey; |
| 50 case videoFrameTypeSkip: |
| 51 case videoFrameTypeI: |
| 52 case videoFrameTypeP: |
| 53 case videoFrameTypeIPMixed: |
| 54 return kVideoFrameDelta; |
| 55 default: |
| 56 LOG(LS_WARNING) << "Unknown EVideoFrameType: " << type; |
| 57 return kVideoFrameDelta; |
| 58 } |
| 59 } |
| 60 |
| 61 // Helper method used by H264EncoderImpl::Encode. |
| 62 // Copies the encoded bytes from |info| to |encoded_image| and updates the |
| 63 // fragmentation information of |frag_header|. The |encoded_image->_buffer| may |
| 64 // be deleted and reallocated if a bigger buffer is required. |
| 65 // |
| 66 // After OpenH264 encoding, the encoded bytes are stored in |info| spread out |
| 67 // over a number of layers and "NAL units". Each NAL unit is a fragment starting |
| 68 // with the four-byte start code {0,0,0,1}. All of this data (including the |
| 69 // start codes) is copied to the |encoded_image->_buffer| and the |frag_header| |
| 70 // is updated to point to each fragment, with offsets and lengths set as to |
| 71 // exclude the start codes. |
| 72 static void RtpFragmentize(EncodedImage* encoded_image, |
| 73 rtc::scoped_ptr<uint8_t[]>* encoded_image_buffer, |
| 74 const VideoFrame& frame, |
| 75 SFrameBSInfo* info, |
| 76 RTPFragmentationHeader* frag_header) { |
| 77 // Calculate minimum buffer size required to hold encoded data. |
| 78 size_t required_size = 0; |
| 79 size_t fragments_count = 0; |
| 80 for (int layer = 0; layer < info->iLayerNum; ++layer) { |
| 81 const SLayerBSInfo& layerInfo = info->sLayerInfo[layer]; |
| 82 for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++fragments_count) { |
| 83 RTC_CHECK_GE(layerInfo.pNalLengthInByte[nal], 0); |
| 84 // Ensure |required_size| will not overflow. |
| 85 RTC_CHECK_LE(static_cast<size_t>(layerInfo.pNalLengthInByte[nal]), |
| 86 std::numeric_limits<size_t>::max() - required_size); |
| 87 required_size += layerInfo.pNalLengthInByte[nal]; |
| 88 } |
| 89 } |
| 90 if (encoded_image->_size < required_size) { |
| 91 // Increase buffer size. Allocate enough to hold an unencoded image, this |
| 92 // should be more than enough to hold any encoded data of future frames of |
| 93 // the same size (avoiding possible future reallocation due to variations in |
| 94 // required size). |
| 95 encoded_image->_size = CalcBufferSize(kI420, frame.width(), frame.height()); |
| 96 if (encoded_image->_size < required_size) { |
| 97 // Encoded data > unencoded data. Allocate required bytes. |
| 98 LOG(LS_WARNING) << "Encoding produced more bytes than the original image " |
| 99 << "data! Original bytes: " << encoded_image->_size |
| 100 << ", encoded bytes: " << required_size << "."; |
| 101 encoded_image->_size = required_size; |
| 102 } |
| 103 encoded_image->_buffer = new uint8_t[encoded_image->_size]; |
| 104 encoded_image_buffer->reset(encoded_image->_buffer); |
| 105 } |
| 106 |
| 107 // Iterate layers and NAL units, note each NAL unit as a fragment and copy |
| 108 // the data to |encoded_image->_buffer|. |
| 109 const uint8_t start_code[4] = {0, 0, 0, 1}; |
| 110 frag_header->VerifyAndAllocateFragmentationHeader(fragments_count); |
| 111 size_t frag = 0; |
| 112 encoded_image->_length = 0; |
| 113 for (int layer = 0; layer < info->iLayerNum; ++layer) { |
| 114 const SLayerBSInfo& layerInfo = info->sLayerInfo[layer]; |
| 115 // Iterate NAL units making up this layer, noting fragments. |
| 116 size_t layer_len = 0; |
| 117 for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++frag) { |
| 118 // Because the sum of all layer lengths, |required_size|, fits in a |
| 119 // |size_t|, we know that any indices in-between will not overflow. |
| 120 RTC_DCHECK_GE(layerInfo.pNalLengthInByte[nal], 4); |
| 121 RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len+0], start_code[0]); |
| 122 RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len+1], start_code[1]); |
| 123 RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len+2], start_code[2]); |
| 124 RTC_DCHECK_EQ(layerInfo.pBsBuf[layer_len+3], start_code[3]); |
| 125 frag_header->fragmentationOffset[frag] = |
| 126 encoded_image->_length + layer_len + sizeof(start_code); |
| 127 frag_header->fragmentationLength[frag] = |
| 128 layerInfo.pNalLengthInByte[nal] - sizeof(start_code); |
| 129 layer_len += layerInfo.pNalLengthInByte[nal]; |
| 130 } |
| 131 // Copy the entire layer's data (including start codes). |
| 132 memcpy(encoded_image->_buffer + encoded_image->_length, |
| 133 layerInfo.pBsBuf, |
| 134 layer_len); |
| 135 encoded_image->_length += layer_len; |
| 136 } |
| 137 } |
| 138 |
| 139 H264EncoderImpl::H264EncoderImpl() |
| 140 : openh264_encoder_(nullptr), |
| 141 encoded_image_callback_(nullptr) { |
| 142 } |
| 143 |
| 144 H264EncoderImpl::~H264EncoderImpl() { |
| 145 Release(); |
| 146 } |
| 147 |
| 148 int32_t H264EncoderImpl::InitEncode(const VideoCodec* codec_settings, |
| 149 int32_t number_of_cores, |
| 150 size_t /*max_payload_size*/) { |
| 151 if (!codec_settings || |
| 152 codec_settings->codecType != kVideoCodecH264) { |
| 153 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 154 } |
| 155 if (codec_settings->maxFramerate == 0) |
| 156 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 157 if (codec_settings->width < 1 || codec_settings->height < 1) |
| 158 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 159 |
| 160 int32_t release_ret = Release(); |
| 161 if (release_ret != WEBRTC_VIDEO_CODEC_OK) |
| 162 return release_ret; |
| 163 RTC_DCHECK(!openh264_encoder_); |
| 164 |
| 165 // Create encoder. |
| 166 if (WelsCreateSVCEncoder(&openh264_encoder_) != 0) { |
| 167 // Failed to create encoder. |
| 168 LOG(LS_ERROR) << "Failed to create OpenH264 encoder"; |
| 169 RTC_DCHECK(!openh264_encoder_); |
| 170 return WEBRTC_VIDEO_CODEC_ERROR; |
| 171 } |
| 172 RTC_DCHECK(openh264_encoder_); |
| 173 if (kOpenH264EncoderDetailedLogging) { |
| 174 int trace_level = WELS_LOG_DETAIL; |
| 175 openh264_encoder_->SetOption(ENCODER_OPTION_TRACE_LEVEL, |
| 176 &trace_level); |
| 177 } |
| 178 // else WELS_LOG_DEFAULT is used by default. |
| 179 |
| 180 codec_settings_ = *codec_settings; |
| 181 if (codec_settings_.targetBitrate == 0) |
| 182 codec_settings_.targetBitrate = codec_settings_.startBitrate; |
| 183 |
| 184 // Initialization parameters. |
| 185 // There are two ways to initialize. There is SEncParamBase (cleared with |
| 186 // memset(&p, 0, sizeof(SEncParamBase)) used in Initialize, and SEncParamExt |
| 187 // which is a superset of SEncParamBase (cleared with GetDefaultParams) used |
| 188 // in InitializeExt. |
| 189 SEncParamExt init_params; |
| 190 openh264_encoder_->GetDefaultParams(&init_params); |
| 191 if (codec_settings_.mode == kRealtimeVideo) { |
| 192 init_params.iUsageType = CAMERA_VIDEO_REAL_TIME; |
| 193 } else if (codec_settings_.mode == kScreensharing) { |
| 194 init_params.iUsageType = SCREEN_CONTENT_REAL_TIME; |
| 195 } else { |
| 196 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 197 } |
| 198 init_params.iPicWidth = codec_settings_.width; |
| 199 init_params.iPicHeight = codec_settings_.height; |
| 200 // |init_params| uses bit/s, |codec_settings_| uses kbit/s. |
| 201 init_params.iTargetBitrate = codec_settings_.targetBitrate * 1000; |
| 202 init_params.iMaxBitrate = codec_settings_.maxBitrate * 1000; |
| 203 // Rate Control mode |
| 204 init_params.iRCMode = RC_BITRATE_MODE; |
| 205 init_params.fMaxFrameRate = static_cast<float>(codec_settings_.maxFramerate); |
| 206 |
| 207 // The following parameters are extension parameters (they're in SEncParamExt, |
| 208 // not in SEncParamBase). |
| 209 init_params.bEnableFrameSkip = |
| 210 codec_settings_.codecSpecific.H264.frameDroppingOn; |
| 211 // |uiIntraPeriod| - multiple of GOP size |
| 212 // |keyFrameInterval| - number of frames |
| 213 init_params.uiIntraPeriod = |
| 214 codec_settings_.codecSpecific.H264.keyFrameInterval; |
| 215 init_params.uiMaxNalSize = 0; |
| 216 // Threading model: use auto. |
| 217 // 0: auto (dynamic imp. internal encoder) |
| 218 // 1: single thread (default value) |
| 219 // >1: number of threads |
| 220 init_params.iMultipleThreadIdc = NumberOfThreads(init_params.iPicWidth, |
| 221 init_params.iPicHeight, |
| 222 number_of_cores); |
| 223 // The base spatial layer 0 is the only one we use. |
| 224 init_params.sSpatialLayers[0].iVideoWidth = init_params.iPicWidth; |
| 225 init_params.sSpatialLayers[0].iVideoHeight = init_params.iPicHeight; |
| 226 init_params.sSpatialLayers[0].fFrameRate = init_params.fMaxFrameRate; |
| 227 init_params.sSpatialLayers[0].iSpatialBitrate = init_params.iTargetBitrate; |
| 228 init_params.sSpatialLayers[0].iMaxSpatialBitrate = init_params.iMaxBitrate; |
| 229 // Slice num according to number of threads. |
| 230 init_params.sSpatialLayers[0].sSliceCfg.uiSliceMode = SM_AUTO_SLICE; |
| 231 |
| 232 // Initialize. |
| 233 if (openh264_encoder_->InitializeExt(&init_params) != 0) { |
| 234 LOG(LS_ERROR) << "Failed to initialize OpenH264 encoder"; |
| 235 Release(); |
| 236 return WEBRTC_VIDEO_CODEC_ERROR; |
| 237 } |
| 238 int video_format = EVideoFormatType::videoFormatI420; |
| 239 openh264_encoder_->SetOption(ENCODER_OPTION_DATAFORMAT, |
| 240 &video_format); |
| 241 |
| 242 // Initialize encoded image. Default buffer size: size of unencoded data. |
| 243 encoded_image_._size = CalcBufferSize( |
| 244 kI420, codec_settings_.width, codec_settings_.height); |
| 245 encoded_image_._buffer = new uint8_t[encoded_image_._size]; |
| 246 encoded_image_buffer_.reset(encoded_image_._buffer); |
| 247 encoded_image_._completeFrame = true; |
| 248 encoded_image_._encodedWidth = 0; |
| 249 encoded_image_._encodedHeight = 0; |
| 250 encoded_image_._length = 0; |
| 251 return WEBRTC_VIDEO_CODEC_OK; |
| 252 } |
| 253 |
| 254 int32_t H264EncoderImpl::Release() { |
| 255 if (openh264_encoder_) { |
| 256 int uninit_ret = openh264_encoder_->Uninitialize(); |
| 257 if (uninit_ret != 0) { |
| 258 LOG(LS_WARNING) << "OpenH264 encoder's Uninitialize() returned " |
| 259 << "unsuccessful: " << uninit_ret; |
| 260 } |
| 261 WelsDestroySVCEncoder(openh264_encoder_); |
| 262 openh264_encoder_ = nullptr; |
| 263 } |
| 264 if (encoded_image_._buffer != nullptr) { |
| 265 encoded_image_._buffer = nullptr; |
| 266 encoded_image_buffer_.reset(); |
| 267 } |
| 268 return WEBRTC_VIDEO_CODEC_OK; |
| 269 } |
| 270 |
| 271 int32_t H264EncoderImpl::RegisterEncodeCompleteCallback( |
| 272 EncodedImageCallback* callback) { |
| 273 encoded_image_callback_ = callback; |
| 274 return WEBRTC_VIDEO_CODEC_OK; |
| 275 } |
| 276 |
| 277 int32_t H264EncoderImpl::SetRates(uint32_t bitrate, uint32_t framerate) { |
| 278 if (bitrate <= 0 || framerate <= 0) { |
| 279 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 280 } |
| 281 codec_settings_.targetBitrate = bitrate; |
| 282 codec_settings_.maxFramerate = framerate; |
| 283 |
| 284 SBitrateInfo target_bitrate; |
| 285 memset(&target_bitrate, 0, sizeof(SBitrateInfo)); |
| 286 target_bitrate.iLayer = SPATIAL_LAYER_ALL, |
| 287 target_bitrate.iBitrate = codec_settings_.targetBitrate * 1000; |
| 288 openh264_encoder_->SetOption(ENCODER_OPTION_BITRATE, |
| 289 &target_bitrate); |
| 290 float max_framerate = static_cast<float>(codec_settings_.maxFramerate); |
| 291 openh264_encoder_->SetOption(ENCODER_OPTION_FRAME_RATE, |
| 292 &max_framerate); |
| 293 return WEBRTC_VIDEO_CODEC_OK; |
| 294 } |
| 295 |
| 296 int32_t H264EncoderImpl::Encode( |
| 297 const VideoFrame& frame, const CodecSpecificInfo* codec_specific_info, |
| 298 const std::vector<FrameType>* frame_types) { |
| 299 if (!IsInitialized()) |
| 300 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 301 if (frame.IsZeroSize()) |
| 302 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 303 if (!encoded_image_callback_) { |
| 304 LOG(LS_WARNING) << "InitEncode() has been called, but a callback function " |
| 305 << "has not been set with RegisterEncodeCompleteCallback()"; |
| 306 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 307 } |
| 308 if (frame.width() != codec_settings_.width || |
| 309 frame.height() != codec_settings_.height) { |
| 310 LOG(LS_WARNING) << "Encoder initialized for " << codec_settings_.width |
| 311 << "x" << codec_settings_.height << " but trying to encode " |
| 312 << frame.width() << "x" << frame.height() << " frame."; |
| 313 return WEBRTC_VIDEO_CODEC_ERR_SIZE; |
| 314 } |
| 315 |
| 316 bool force_key_frame = false; |
| 317 if (frame_types != nullptr) { |
| 318 // We only support a single stream. |
| 319 RTC_DCHECK_EQ(frame_types->size(), static_cast<size_t>(1)); |
| 320 // Skip frame? |
| 321 if ((*frame_types)[0] == kEmptyFrame) { |
| 322 return WEBRTC_VIDEO_CODEC_OK; |
| 323 } |
| 324 // Force key frame? |
| 325 force_key_frame = (*frame_types)[0] == kVideoFrameKey; |
| 326 } |
| 327 if (force_key_frame) { |
| 328 // API doc says ForceIntraFrame(false) does nothing, but calling this |
| 329 // function forces a key frame regardless of the |bIDR| argument's value. |
| 330 // (If every frame is a key frame we get lag/delays.) |
| 331 openh264_encoder_->ForceIntraFrame(true); |
| 332 } |
| 333 |
| 334 // EncodeFrame input. |
| 335 SSourcePicture picture; |
| 336 memset(&picture, 0, sizeof(SSourcePicture)); |
| 337 picture.iPicWidth = frame.width(); |
| 338 picture.iPicHeight = frame.height(); |
| 339 picture.iColorFormat = EVideoFormatType::videoFormatI420; |
| 340 picture.uiTimeStamp = frame.ntp_time_ms(); |
| 341 picture.iStride[0] = frame.stride(kYPlane); |
| 342 picture.iStride[1] = frame.stride(kUPlane); |
| 343 picture.iStride[2] = frame.stride(kVPlane); |
| 344 picture.pData[0] = const_cast<uint8_t*>(frame.buffer(kYPlane)); |
| 345 picture.pData[1] = const_cast<uint8_t*>(frame.buffer(kUPlane)); |
| 346 picture.pData[2] = const_cast<uint8_t*>(frame.buffer(kVPlane)); |
| 347 |
| 348 // EncodeFrame output. |
| 349 SFrameBSInfo info; |
| 350 memset(&info, 0, sizeof(SFrameBSInfo)); |
| 351 |
| 352 // Encode! |
| 353 int enc_ret = openh264_encoder_->EncodeFrame(&picture, &info); |
| 354 if (enc_ret != 0) { |
| 355 LOG(LS_ERROR) << "OpenH264 frame encoding failed, EncodeFrame returned " |
| 356 << enc_ret << "."; |
| 357 return WEBRTC_VIDEO_CODEC_ERROR; |
| 358 } |
| 359 |
| 360 encoded_image_._encodedWidth = frame.width(); |
| 361 encoded_image_._encodedHeight = frame.height(); |
| 362 encoded_image_._timeStamp = frame.timestamp(); |
| 363 encoded_image_.ntp_time_ms_ = frame.ntp_time_ms(); |
| 364 encoded_image_.capture_time_ms_ = frame.render_time_ms(); |
| 365 encoded_image_._frameType = EVideoFrameType_to_FrameType(info.eFrameType); |
| 366 |
| 367 // Split encoded image up into fragments. This also updates |encoded_image_|. |
| 368 RTPFragmentationHeader frag_header; |
| 369 RtpFragmentize(&encoded_image_, &encoded_image_buffer_, frame, &info, |
| 370 &frag_header); |
| 371 |
| 372 // Encoder can skip frames to save bandwidth in which case |
| 373 // |encoded_image_._length| == 0. |
| 374 if (encoded_image_._length > 0) { |
| 375 // Deliver encoded image. |
| 376 CodecSpecificInfo codec_specific; |
| 377 codec_specific.codecType = kVideoCodecH264; |
| 378 encoded_image_callback_->Encoded(encoded_image_, |
| 379 &codec_specific, |
| 380 &frag_header); |
| 381 } |
| 382 return WEBRTC_VIDEO_CODEC_OK; |
| 383 } |
| 384 |
| 385 bool H264EncoderImpl::IsInitialized() const { |
| 386 return openh264_encoder_ != nullptr; |
| 387 } |
| 388 |
| 389 int32_t H264EncoderImpl::SetChannelParameters( |
| 390 uint32_t packet_loss, int64_t rtt) { |
| 391 return WEBRTC_VIDEO_CODEC_OK; |
| 392 } |
| 393 |
| 394 int32_t H264EncoderImpl::SetPeriodicKeyFrames(bool enable) { |
| 395 return WEBRTC_VIDEO_CODEC_OK; |
| 396 } |
| 397 |
| 398 void H264EncoderImpl::OnDroppedFrame() { |
| 399 } |
| 400 |
| 401 } // namespace webrtc |
OLD | NEW |