In screenshare mode, suppress VP8 bitrate overshoot and increase quality

webrtc/modules/video_coding/main/source/generic_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 42 matching lines @@
 
 VCMGenericEncoder::VCMGenericEncoder(VideoEncoder* encoder,
                                      VideoEncoderRateObserver* rate_observer,
                                      bool internalSource)
     : encoder_(encoder),
       rate_observer_(rate_observer),
       vcm_encoded_frame_callback_(nullptr),
       bit_rate_(0),
       frame_rate_(0),
       internal_source_(internalSource),
-      rotation_(kVideoRotation_0) {
+      rotation_(kVideoRotation_0),
+      is_screenshare_(false) {
 }
 
 VCMGenericEncoder::~VCMGenericEncoder()
 {
 }
 
 int32_t VCMGenericEncoder::Release()
 {
     {
       rtc::CritScope lock(&rates_lock_);
       bit_rate_ = 0;
       frame_rate_ = 0;
       vcm_encoded_frame_callback_ = nullptr;
     }
 
     return encoder_->Release();
 }
 
 int32_t
 VCMGenericEncoder::InitEncode(const VideoCodec* settings,
                               int32_t numberOfCores,
                               size_t maxPayloadSize)
 {
     {
       rtc::CritScope lock(&rates_lock_);
       bit_rate_ = settings->startBitrate * 1000;
       frame_rate_ = settings->maxFramerate;
     }
 
+    is_screenshare_ = settings->mode == VideoCodecMode::kScreensharing;
     if (encoder_->InitEncode(settings, numberOfCores, maxPayloadSize) != 0) {
       LOG(LS_ERROR) << "Failed to initialize the encoder associated with "
                        "payload name: " << settings->plName;
       return -1;
     }
     return 0;
 }
 
 int32_t VCMGenericEncoder::Encode(const VideoFrame& inputFrame,
                                   const CodecSpecificInfo* codecSpecificInfo,
                                   const std::vector<FrameType>& frameTypes) {
   std::vector<VideoFrameType> video_frame_types(frameTypes.size(),
                                                 kDeltaFrame);
   VCMEncodedFrame::ConvertFrameTypes(frameTypes, &video_frame_types);
 
   rotation_ = inputFrame.rotation();
 
   if (vcm_encoded_frame_callback_) {
     // Keep track of the current frame rotation and apply to the output of the
     // encoder. There might not be exact as the encoder could have one frame
     // delay but it should be close enough.
     vcm_encoded_frame_callback_->SetRotation(rotation_);
   }
 
-  return encoder_->Encode(inputFrame, codecSpecificInfo, &video_frame_types);
+  int32_t result =
+      encoder_->Encode(inputFrame, codecSpecificInfo, &video_frame_types);
+  if (is_screenshare_ && result == WEBRTC_VIDEO_CODEC_OK &&

[stefan-webrtc, 2015/06/18 09:39:02]

This is really hard to understand. What Does it do, and why? Seems to rely on
the underlying encoder being synchronous and finish encoding/return a frame
before it the Encode() call returns?

[sprang_webrtc, 2015/06/18 13:05:02]

Yes, this was an ad-hoc way to discover that the encoder had dropped a frame a
we needed to re-encode it. I've changed it so the encoder returns a special code
for this case instead.

[stefan-webrtc, 2015/06/23 14:28:32]

I'm wondering if it would be better to have this logic in vp8_impl.cc instead?
Or is there a reason to have it generically?

[sprang_webrtc, 2015/06/23 15:07:30]

It's more difficult since we don't know until after GetEncodedPartitions if this
happpened or not. We would need to wrap Encode inside a method just for this or
risk using recursion.
While not needed for any other impl, I think it is not inconceivable that we
would like to use this elsewhere? 

+      vcm_encoded_frame_callback_ != nullptr &&
+      !vcm_encoded_frame_callback_->GetAndResetFrameProduced()) {
+    result =
+        encoder_->Encode(inputFrame, codecSpecificInfo, &video_frame_types);
+    vcm_encoded_frame_callback_->GetAndResetFrameProduced();
+  }
+
+  return result;
 }
 
 int32_t
 VCMGenericEncoder::SetChannelParameters(int32_t packetLoss, int64_t rtt)
 {
     return encoder_->SetChannelParameters(packetLoss, rtt);
 }
 
 int32_t
 VCMGenericEncoder::SetRates(uint32_t newBitRate, uint32_t frameRate)
@@ skipping 79 matching lines @@
  /***************************
   * Callback Implementation
   ***************************/
 VCMEncodedFrameCallback::VCMEncodedFrameCallback(
     EncodedImageCallback* post_encode_callback)
     : _sendCallback(),
       _mediaOpt(NULL),
       _payloadType(0),
       _internalSource(false),
       _rotation(kVideoRotation_0),
-      post_encode_callback_(post_encode_callback)
+      post_encode_callback_(post_encode_callback),
+      frame_produced_(false)
 #ifdef DEBUG_ENCODER_BIT_STREAM
       ,
       _bitStreamAfterEncoder(NULL)
 #endif
 {
 #ifdef DEBUG_ENCODER_BIT_STREAM
     _bitStreamAfterEncoder = fopen("encoderBitStream.bit", "wb");
 #endif
 }
 
@@ skipping 10 matching lines @@
     _sendCallback = transport;
     return VCM_OK;
 }
 
 int32_t VCMEncodedFrameCallback::Encoded(
     const EncodedImage& encodedImage,
     const CodecSpecificInfo* codecSpecificInfo,
     const RTPFragmentationHeader* fragmentationHeader) {
   post_encode_callback_->Encoded(encodedImage, NULL, NULL);
 
+  frame_produced_ = true;
+
   if (_sendCallback == NULL) {
     return VCM_UNINITIALIZED;
   }
 
 #ifdef DEBUG_ENCODER_BIT_STREAM
   if (_bitStreamAfterEncoder != NULL) {
     fwrite(encodedImage._buffer, 1, encodedImage._length,
            _bitStreamAfterEncoder);
   }
 #endif
@@ skipping 20 matching lines @@
   return VCM_OK;
 }
 
 void
 VCMEncodedFrameCallback::SetMediaOpt(
     media_optimization::MediaOptimization *mediaOpt)
 {
     _mediaOpt = mediaOpt;
 }
 
+bool VCMEncodedFrameCallback::GetAndResetFrameProduced() {
+  bool frame_produced = frame_produced_;
+  frame_produced_ = false;
+  return frame_produced;
+}
+
 }  // namespace webrtc