OLD | NEW |
(Empty) | |
| 1 /* |
| 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. |
| 3 * |
| 4 * Use of this source code is governed by a BSD-style license |
| 5 * that can be found in the LICENSE file in the root of the source |
| 6 * tree. An additional intellectual property rights grant can be found |
| 7 * in the file PATENTS. All contributing project authors may |
| 8 * be found in the AUTHORS file in the root of the source tree. |
| 9 * |
| 10 */ |
| 11 |
| 12 #include "webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h" |
| 13 |
| 14 // OpenH264 |
| 15 #include "codec_api.h" |
| 16 #include "codec_app_def.h" |
| 17 #include "codec_def.h" |
| 18 |
| 19 #include "webrtc/base/checks.h" |
| 20 #include "webrtc/base/logging.h" |
| 21 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" |
| 22 |
| 23 namespace webrtc { |
| 24 |
| 25 namespace { |
| 26 const bool kOpenH264EncoderDetailedLogging = false; |
| 27 } // namespace |
| 28 |
| 29 static VideoFrameType EVideoFrameType_to_VideoFrameType( |
| 30 EVideoFrameType type) { |
| 31 switch (type) { |
| 32 case videoFrameTypeInvalid: |
| 33 return kSkipFrame; |
| 34 case videoFrameTypeSkip: |
| 35 return kDeltaFrame; |
| 36 case videoFrameTypeIDR: |
| 37 return kKeyFrame; |
| 38 case videoFrameTypeI: |
| 39 case videoFrameTypeP: |
| 40 case videoFrameTypeIPMixed: |
| 41 return kDeltaFrame; |
| 42 default: |
| 43 LOG(LS_WARNING) << "Unknown EVideoFrameType: " << type; |
| 44 return kDeltaFrame; |
| 45 } |
| 46 } |
| 47 // Helper method used by H264EncoderImpl::Encode. |
| 48 // Copies the encoded bytes from |info| to |encoded_image| and updates the |
| 49 // fragmentation information of |frag_header|. The |encoded_image->_buffer| may |
| 50 // be deleted and reallocated if a bigger buffer is required. |
| 51 // After OpenH264 encoding, the encoded bytes are stored in |info| spread out |
| 52 // over a number of layers and "NAL units". Each NAL unit is a fragment starting |
| 53 // with the four-byte start code {0,0,0,1}. All of this data (including the |
| 54 // start codes) is copied to the |encoded_image->_buffer| and the |frag_header| |
| 55 // is updated to point to each fragment, with offsets and lengths set as to |
| 56 // exclude the start codes. |
| 57 static void RtpFragmentize(EncodedImage* encoded_image, |
| 58 rtc::scoped_ptr<uint8_t>* encoded_image_buffer, |
| 59 const VideoFrame& frame, |
| 60 SFrameBSInfo* info, |
| 61 RTPFragmentationHeader* frag_header) { |
| 62 // Calculate minimum buffer size required to hold encoded data. |
| 63 size_t required_size = 0; |
| 64 size_t fragments_count = 0; |
| 65 for (int iLayer = 0; iLayer < info->iLayerNum; ++iLayer) { |
| 66 const SLayerBSInfo& layerInfo = info->sLayerInfo[iLayer]; |
| 67 for (int iNal = 0; iNal < layerInfo.iNalCount; ++iNal) { |
| 68 required_size += layerInfo.pNalLengthInByte[iNal]; |
| 69 ++fragments_count; |
| 70 } |
| 71 } |
| 72 if (encoded_image->_size < required_size) { |
| 73 // Increase buffer size. Allocate enough to hold an unencoded image, this |
| 74 // should be more than enough to hold any encoded data of future frames of |
| 75 // the same size (avoiding possible future reallocation due to variations in |
| 76 // required size). |
| 77 encoded_image->_size = CalcBufferSize( |
| 78 VideoType::kI420, frame.width(), frame.height()); |
| 79 if (encoded_image->_size < required_size) { |
| 80 // Encoded data > unencoded data, wtf? Allocate required bytes. |
| 81 LOG(LS_WARNING) << "Encoding produced more bytes than the original image " |
| 82 << "data! Original bytes: " << encoded_image->_size |
| 83 << ", encoded bytes: " << required_size << "."; |
| 84 encoded_image->_size = required_size; |
| 85 } |
| 86 encoded_image->_buffer = new uint8_t[encoded_image->_size]; |
| 87 encoded_image_buffer->reset(encoded_image->_buffer); |
| 88 } |
| 89 |
| 90 // Iterate layers and NAL units, note each NAL unit as a fragment and copy |
| 91 // the data to |encoded_image->_buffer|. |
| 92 frag_header->VerifyAndAllocateFragmentationHeader(fragments_count); |
| 93 size_t frag_i = 0; |
| 94 encoded_image->_length = 0; |
| 95 for (int iLayer = 0; iLayer < info->iLayerNum; ++iLayer) { |
| 96 const SLayerBSInfo& layerInfo = info->sLayerInfo[iLayer]; |
| 97 // Iterate NAL units making up this layer, noting fragments. |
| 98 size_t iLayerLen = 0; |
| 99 for (int iNal = 0; iNal < layerInfo.iNalCount; ++iNal, ++frag_i) { |
| 100 // Expecting start code constant {0,0,0,1}. |
| 101 RTC_DCHECK_EQ(layerInfo.pBsBuf[iLayerLen+0], |
| 102 static_cast<unsigned char>(0)); |
| 103 RTC_DCHECK_EQ(layerInfo.pBsBuf[iLayerLen+1], |
| 104 static_cast<unsigned char>(0)); |
| 105 RTC_DCHECK_EQ(layerInfo.pBsBuf[iLayerLen+2], |
| 106 static_cast<unsigned char>(0)); |
| 107 RTC_DCHECK_EQ(layerInfo.pBsBuf[iLayerLen+3], |
| 108 static_cast<unsigned char>(1)); |
| 109 // Fragment: +4/-4 is for excluding the start code. |
| 110 frag_header->fragmentationOffset[frag_i] = |
| 111 encoded_image->_length + iLayerLen + 4; |
| 112 frag_header->fragmentationLength[frag_i] = |
| 113 layerInfo.pNalLengthInByte[iNal] - 4; |
| 114 iLayerLen += layerInfo.pNalLengthInByte[iNal]; |
| 115 } |
| 116 // Copy the entire layer's data (including start codes). |
| 117 memcpy(encoded_image->_buffer + encoded_image->_length, |
| 118 layerInfo.pBsBuf, |
| 119 iLayerLen * sizeof(unsigned char)); |
| 120 encoded_image->_length += iLayerLen; |
| 121 } |
| 122 } |
| 123 |
| 124 H264EncoderImpl::H264EncoderImpl() |
| 125 : openh264_encoder_(nullptr), |
| 126 encoded_image_callback_(nullptr) { |
| 127 } |
| 128 |
| 129 H264EncoderImpl::~H264EncoderImpl() { |
| 130 Release(); |
| 131 } |
| 132 |
| 133 int32_t H264EncoderImpl::InitEncode(const VideoCodec* codec_settings, |
| 134 int32_t /*number_of_cores*/, |
| 135 size_t /*max_payload_size*/) { |
| 136 if (!codec_settings || |
| 137 codec_settings->codecType != VideoCodecType::kVideoCodecH264) { |
| 138 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 139 } |
| 140 if (codec_settings->maxFramerate == 0) |
| 141 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 142 if (codec_settings->width < 1 || codec_settings->height < 1) |
| 143 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 144 |
| 145 int release_ret = Release(); |
| 146 if (release_ret != WEBRTC_VIDEO_CODEC_OK) |
| 147 return release_ret; |
| 148 RTC_DCHECK(!openh264_encoder_); |
| 149 |
| 150 // Create encoder. |
| 151 if (WelsCreateSVCEncoder(&openh264_encoder_) != 0) { |
| 152 // Failed to create encoder. |
| 153 LOG(LS_ERROR) << "Failed to create OpenH264 encoder"; |
| 154 RTC_DCHECK(!openh264_encoder_); |
| 155 return WEBRTC_VIDEO_CODEC_ERROR; |
| 156 } |
| 157 RTC_DCHECK(openh264_encoder_); |
| 158 if (kOpenH264EncoderDetailedLogging) { |
| 159 int trace_level = WELS_LOG_DETAIL; |
| 160 openh264_encoder_->SetOption(ENCODER_OPTION_TRACE_LEVEL, |
| 161 &trace_level); |
| 162 } |
| 163 // else WELS_LOG_DEFAULT is used by default. |
| 164 |
| 165 codec_settings_ = *codec_settings; |
| 166 if (codec_settings_.targetBitrate == 0) |
| 167 codec_settings_.targetBitrate = codec_settings_.startBitrate; |
| 168 |
| 169 // Initialization parameters. |
| 170 // There are two ways to initialize. There is SEncParamBase (cleared with |
| 171 // memset(&p, 0, sizeof(SEncParamBase)) used in Initialize, and SEncParamExt |
| 172 // which is a superset of SEncParamBase (cleared with GetDefaultParams) used |
| 173 // in InitializeExt. |
| 174 SEncParamExt init_params; |
| 175 openh264_encoder_->GetDefaultParams(&init_params); |
| 176 if (codec_settings_.mode == kRealtimeVideo) { |
| 177 init_params.iUsageType = CAMERA_VIDEO_REAL_TIME; |
| 178 } else if (codec_settings_.mode == kScreensharing) { |
| 179 init_params.iUsageType = SCREEN_CONTENT_REAL_TIME; |
| 180 } else { |
| 181 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 182 } |
| 183 init_params.iPicWidth = codec_settings_.width; |
| 184 init_params.iPicHeight = codec_settings_.height; |
| 185 // |init_params| uses bit/s, |codec_settings_| uses kbit/s. |
| 186 init_params.iTargetBitrate = codec_settings_.targetBitrate * 1000; |
| 187 init_params.iMaxBitrate = codec_settings_.maxBitrate * 1000; |
| 188 // Rate Control mode |
| 189 // TODO(hbos): Switch to RC_TIMESTAMP_MODE? VideoProcessorImpl::ProcessFrame |
| 190 // need to be updated to use proper time stamps or else it will not work and |
| 191 // VideoProcessorIntegrationTest.Process0PercentPacketLossH264 will fail. |
| 192 init_params.iRCMode = RC_BITRATE_MODE; |
| 193 init_params.fMaxFrameRate = static_cast<float>(codec_settings_.maxFramerate); |
| 194 |
| 195 // The following parameters are extension parameters (they're in SEncParamExt, |
| 196 // not in SEncParamBase). |
| 197 init_params.bEnableFrameSkip = |
| 198 codec_settings_.codecSpecific.H264.frameDroppingOn; |
| 199 // |uiIntraPeriod| - multiple of GOP size |
| 200 // |keyFrameInterval| - ? number of frames? multiple of GOP size? |
| 201 init_params.uiIntraPeriod = |
| 202 codec_settings_.codecSpecific.H264.keyFrameInterval; |
| 203 init_params.uiMaxNalSize = 0; |
| 204 // Threading model: use auto. |
| 205 // 0: auto (dynamic imp. internal encoder) |
| 206 // 1: single thread (default value) |
| 207 // >1: number of threads |
| 208 init_params.iMultipleThreadIdc = 0; |
| 209 // The base spatial layer 0 is the only one we use. |
| 210 init_params.sSpatialLayers[0].iVideoWidth = init_params.iPicWidth; |
| 211 init_params.sSpatialLayers[0].iVideoHeight = init_params.iPicHeight; |
| 212 init_params.sSpatialLayers[0].fFrameRate = init_params.fMaxFrameRate; |
| 213 init_params.sSpatialLayers[0].iSpatialBitrate = init_params.iTargetBitrate; |
| 214 init_params.sSpatialLayers[0].iMaxSpatialBitrate = init_params.iMaxBitrate; |
| 215 // Slice num according to number of threads. |
| 216 init_params.sSpatialLayers[0].sSliceCfg.uiSliceMode = SM_AUTO_SLICE; |
| 217 |
| 218 // Initialize. |
| 219 if (openh264_encoder_->InitializeExt(&init_params) != 0) { |
| 220 LOG(LS_ERROR) << "Failed to initialize OpenH264 encoder"; |
| 221 Release(); |
| 222 return WEBRTC_VIDEO_CODEC_ERROR; |
| 223 } |
| 224 int video_format = EVideoFormatType::videoFormatI420; |
| 225 openh264_encoder_->SetOption(ENCODER_OPTION_DATAFORMAT, |
| 226 &video_format); |
| 227 |
| 228 // Initialize encoded image. Default buffer size: size of unencoded data. |
| 229 encoded_image_._size = CalcBufferSize( |
| 230 VideoType::kI420, codec_settings_.width, codec_settings_.height); |
| 231 encoded_image_._buffer = new uint8_t[encoded_image_._size]; |
| 232 encoded_image_buffer_.reset(encoded_image_._buffer); |
| 233 encoded_image_._completeFrame = true; |
| 234 encoded_image_._encodedWidth = 0; |
| 235 encoded_image_._encodedHeight = 0; |
| 236 encoded_image_._length = 0; |
| 237 return WEBRTC_VIDEO_CODEC_OK; |
| 238 } |
| 239 |
| 240 int32_t H264EncoderImpl::Release() { |
| 241 if (openh264_encoder_) { |
| 242 int uninit_ret = openh264_encoder_->Uninitialize(); |
| 243 if (uninit_ret != 0) { |
| 244 LOG(LS_WARNING) << "OpenH264 encoder's Uninitialize() returned " |
| 245 << "unsuccessful: " << uninit_ret; |
| 246 } |
| 247 WelsDestroySVCEncoder(openh264_encoder_); |
| 248 openh264_encoder_ = nullptr; |
| 249 } |
| 250 if (encoded_image_._buffer != nullptr) { |
| 251 encoded_image_._buffer = nullptr; |
| 252 encoded_image_buffer_.reset(nullptr); |
| 253 } |
| 254 return WEBRTC_VIDEO_CODEC_OK; |
| 255 } |
| 256 |
| 257 int32_t H264EncoderImpl::RegisterEncodeCompleteCallback( |
| 258 EncodedImageCallback* callback) { |
| 259 encoded_image_callback_ = callback; |
| 260 return WEBRTC_VIDEO_CODEC_OK; |
| 261 } |
| 262 |
| 263 int32_t H264EncoderImpl::SetRates(uint32_t bitrate, uint32_t framerate) { |
| 264 if (bitrate <= 0 || framerate <= 0) { |
| 265 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 266 } |
| 267 codec_settings_.targetBitrate = bitrate; |
| 268 codec_settings_.maxFramerate = framerate; |
| 269 |
| 270 SBitrateInfo target_bitrate; |
| 271 memset(&target_bitrate, 0, sizeof(SBitrateInfo)); |
| 272 target_bitrate.iLayer = SPATIAL_LAYER_ALL, |
| 273 target_bitrate.iBitrate = codec_settings_.targetBitrate * 1000; |
| 274 openh264_encoder_->SetOption(ENCODER_OPTION_BITRATE, |
| 275 &target_bitrate); |
| 276 float max_framerate = static_cast<float>(codec_settings_.maxFramerate); |
| 277 openh264_encoder_->SetOption(ENCODER_OPTION_FRAME_RATE, |
| 278 &max_framerate); |
| 279 return WEBRTC_VIDEO_CODEC_OK; |
| 280 } |
| 281 |
| 282 int32_t H264EncoderImpl::Encode( |
| 283 const VideoFrame& frame, const CodecSpecificInfo* codec_specific_info, |
| 284 const std::vector<VideoFrameType>* frame_types) { |
| 285 if (!IsInitialized()) |
| 286 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 287 if (frame.IsZeroSize()) |
| 288 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 289 if (!encoded_image_callback_) { |
| 290 LOG(LS_WARNING) << "InitEncode() has been called, but a callback function " |
| 291 << "has not been set with RegisterEncodeCompleteCallback()"; |
| 292 return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 293 } |
| 294 if (frame.width() != codec_settings_.width || |
| 295 frame.height() != codec_settings_.height) { |
| 296 LOG(LS_WARNING) << "Encoder initialized for " << codec_settings_.width |
| 297 << "x" << codec_settings_.height << " but trying to encode " |
| 298 << frame.width() << "x" << frame.height() << " frame."; |
| 299 return WEBRTC_VIDEO_CODEC_ERR_SIZE; |
| 300 } |
| 301 |
| 302 bool force_key_frame = false; |
| 303 if (frame_types != nullptr) { |
| 304 // Skip frame? |
| 305 if (std::find(frame_types->begin(), frame_types->end(), |
| 306 kSkipFrame) != frame_types->end()) { |
| 307 return WEBRTC_VIDEO_CODEC_OK; |
| 308 } |
| 309 // Force key frame? |
| 310 force_key_frame = std::find(frame_types->begin(), frame_types->end(), |
| 311 kKeyFrame) != frame_types->end(); |
| 312 } |
| 313 if (force_key_frame) { |
| 314 // Only need to call ForceIntraFrame when true. API doc says |
| 315 // ForceIntraFrame(false) does nothing but really if you call it for every |
| 316 // frame it introduces massive delays and lag in the video stream. |
| 317 openh264_encoder_->ForceIntraFrame(true); |
| 318 } |
| 319 |
| 320 // EncodeFrame input. |
| 321 SSourcePicture picture; |
| 322 memset(&picture, 0, sizeof(SSourcePicture)); |
| 323 picture.iPicWidth = frame.width(); |
| 324 picture.iPicHeight = frame.height(); |
| 325 picture.iColorFormat = EVideoFormatType::videoFormatI420; |
| 326 // 90 kHz -> milliseconds (1 kHz) |
| 327 picture.uiTimeStamp = frame.timestamp() / 90; |
| 328 picture.iStride[0] = frame.stride(kYPlane); |
| 329 picture.iStride[1] = frame.stride(kUPlane); |
| 330 picture.iStride[2] = frame.stride(kVPlane); |
| 331 picture.pData[0] = const_cast<uint8_t*>(frame.buffer(kYPlane)); |
| 332 picture.pData[1] = const_cast<uint8_t*>(frame.buffer(kUPlane)); |
| 333 picture.pData[2] = const_cast<uint8_t*>(frame.buffer(kVPlane)); |
| 334 |
| 335 // EncodeFrame output. |
| 336 SFrameBSInfo info; |
| 337 memset(&info, 0, sizeof(SFrameBSInfo)); |
| 338 |
| 339 // Encode! |
| 340 int enc_ret = openh264_encoder_->EncodeFrame(&picture, &info); |
| 341 if (enc_ret != 0) { |
| 342 LOG(LS_ERROR) << "OpenH264 frame encoding failed, EncodeFrame returned " |
| 343 << enc_ret << "."; |
| 344 return WEBRTC_VIDEO_CODEC_ERROR; |
| 345 } |
| 346 |
| 347 encoded_image_._encodedWidth = frame.width(); |
| 348 encoded_image_._encodedHeight = frame.height(); |
| 349 encoded_image_._timeStamp = frame.timestamp(); |
| 350 encoded_image_.ntp_time_ms_ = frame.ntp_time_ms(); |
| 351 encoded_image_.capture_time_ms_ = frame.render_time_ms(); |
| 352 encoded_image_._frameType = EVideoFrameType_to_VideoFrameType( |
| 353 info.eFrameType); |
| 354 |
| 355 // Split encoded image up into fragments. This also updates |encoded_image_|. |
| 356 RTPFragmentationHeader frag_header; |
| 357 RtpFragmentize(&encoded_image_, &encoded_image_buffer_, |
| 358 frame, &info, &frag_header); |
| 359 |
| 360 // Encoder can skip frames to save bandwidth in which case |
| 361 // |encoded_image_._length| == 0. |
| 362 if (encoded_image_._length > 0) { |
| 363 // Deliver encoded image. |
| 364 encoded_image_callback_->Encoded(encoded_image_, codec_specific_info, |
| 365 &frag_header); |
| 366 } |
| 367 return WEBRTC_VIDEO_CODEC_OK; |
| 368 } |
| 369 |
| 370 bool H264EncoderImpl::IsInitialized() { |
| 371 return openh264_encoder_ != nullptr; |
| 372 } |
| 373 |
| 374 int32_t H264EncoderImpl::SetChannelParameters( |
| 375 uint32_t packet_loss, int64_t rtt) { |
| 376 return WEBRTC_VIDEO_CODEC_OK; |
| 377 } |
| 378 |
| 379 int32_t H264EncoderImpl::SetPeriodicKeyFrames(bool enable) { |
| 380 return WEBRTC_VIDEO_CODEC_OK; |
| 381 } |
| 382 |
| 383 int32_t H264EncoderImpl::CodecConfigParameters(uint8_t* buffer, int32_t size) { |
| 384 return WEBRTC_VIDEO_CODEC_OK; |
| 385 } |
| 386 |
| 387 void H264EncoderImpl::OnDroppedFrame() { |
| 388 } |
| 389 |
| 390 } // namespace webrtc |
OLD | NEW |