In screenshare mode, suppress VP8 bitrate overshoot and increase quality

webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 260 matching lines @@
     // interpret the startBitrate as the encoder target bitrate. This is
     // to allow for a different max bitrate, so if the codec can't meet
     // the target we still allow it to overshoot up to the max before dropping
     // frames. This hack should be improved.
     if (codec_.targetBitrate > 0 &&
         (codec_.codecSpecific.VP8.numberOfTemporalLayers == 2 ||
          codec_.simulcastStream[0].numberOfTemporalLayers == 2)) {
       int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate);
       max_bitrate = std::min(codec_.maxBitrate, target_bitrate);
       target_bitrate = tl0_bitrate;
-      framerate = -1;
     }
     configurations_[i].rc_target_bitrate = target_bitrate;
     temporal_layers_[stream_idx]->ConfigureBitrates(target_bitrate,
                                                     max_bitrate,
                                                     framerate,
                                                     &configurations_[i]);
     if (vpx_codec_enc_config_set(&encoders_[i], &configurations_[i])) {
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
   }
@@ skipping 13 matching lines @@
 void VP8EncoderImpl::SetupTemporalLayers(int num_streams,
                                                  int num_temporal_layers,
                                                  const VideoCodec& codec) {
   const Config default_options;
   const TemporalLayers::Factory& tl_factory =
       (codec.extra_options ? codec.extra_options : &default_options)
           ->Get<TemporalLayers::Factory>();
   if (num_streams == 1) {
     if (codec.mode == kScreensharing) {
       // Special mode when screensharing on a single stream.
-      temporal_layers_.push_back(new ScreenshareLayers(num_temporal_layers,
-                                                       rand(),
-                                                       &tl0_frame_dropper_,
-                                                       &tl1_frame_dropper_));
+      temporal_layers_.push_back(
+          new ScreenshareLayers(num_temporal_layers, rand()));
     } else {
       temporal_layers_.push_back(
           tl_factory.Create(num_temporal_layers, rand()));
     }
   } else {
     for (int i = 0; i < num_streams; ++i) {
       // TODO(andresp): crash if layers is invalid.
       int layers = codec.simulcastStream[i].numberOfTemporalLayers;
       if (layers < 1) layers = 1;
       temporal_layers_.push_back(tl_factory.Create(layers, rand()));
@@ skipping 335 matching lines @@
                       denoiser_state : kDenoiserOff);
   }
   for (size_t i = 0; i < encoders_.size(); ++i) {
     vpx_codec_control(&(encoders_[i]), VP8E_SET_STATIC_THRESHOLD, 1);
     vpx_codec_control(&(encoders_[i]), VP8E_SET_CPUUSED, cpu_speed_[i]);
     vpx_codec_control(&(encoders_[i]), VP8E_SET_TOKEN_PARTITIONS,
                       static_cast<vp8e_token_partitions>(token_partitions_));
     vpx_codec_control(&(encoders_[i]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
                       rc_max_intra_target_);
     vpx_codec_control(&(encoders_[i]), VP8E_SET_SCREEN_CONTENT_MODE,
-                      codec_.mode == kScreensharing);
+                      codec_.mode == kScreensharing ? 2 : 0);

[stefan-webrtc, 2015/06/18 09:39:02]

What does 2 mean?

[sprang_webrtc, 2015/06/18 13:05:02]

Commented

[stefan-webrtc, 2015/06/23 14:28:32]

Still doesn't say what 2 means

[sprang_webrtc, 2015/06/23 15:07:30]

You mean you miss the actual 2? :)

   }
   inited_ = true;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 uint32_t VP8EncoderImpl::MaxIntraTarget(uint32_t optimalBuffersize) {
   // Set max to the optimal buffer level (normalized by target BR),
   // and scaled by a scalePar.
   // Max target size = scalePar * optimalBufferSize * targetBR[Kbps].
   // This values is presented in percentage of perFrameBw:
   // perFrameBw = targetBR[Kbps] * 1000 / frameRate.
   // The target in % is as follows:
 
   float scalePar = 0.5;
   uint32_t targetPct = optimalBuffersize * scalePar * codec_.maxFramerate / 10;
 
   // Don't go below 3 times the per frame bandwidth.
   const uint32_t minIntraTh = 300;
   return (targetPct < minIntraTh) ? minIntraTh: targetPct;
 }
 
 int VP8EncoderImpl::Encode(const VideoFrame& frame,
                            const CodecSpecificInfo* codec_specific_info,
                            const std::vector<VideoFrameType>* frame_types) {
   TRACE_EVENT1("webrtc", "VP8::Encode", "timestamp", frame.timestamp());
 
-  if (!inited_) {
+  if (!inited_)
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
-  }
-  if (frame.IsZeroSize()) {
+  if (frame.IsZeroSize())
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
-  }
-  if (encoded_complete_callback_ == NULL) {
+  if (encoded_complete_callback_ == NULL)
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
-  }
 
   // Only apply scaling to improve for single-layer streams. The scaling metrics
   // use frame drops as a signal and is only applicable when we drop frames.
   const bool use_quality_scaler = encoders_.size() == 1 &&
                                   configurations_[0].rc_dropframe_thresh > 0 &&
                                   codec_.codecSpecific.VP8.automaticResizeOn;
   const VideoFrame& input_image =
       use_quality_scaler ? quality_scaler_.GetScaledFrame(frame) : frame;
 
   if (use_quality_scaler && (input_image.width() != codec_.width ||
@@ skipping 124 matching lines @@
         }
       }
     }
   }
   // Set the encoder frame flags and temporal layer_id for each spatial stream.
   // Note that |temporal_layers_| are defined starting from lowest resolution at
   // position 0 to highest resolution at position |encoders_.size() - 1|,
   // whereas |encoder_| is from highest to lowest resolution.
   size_t stream_idx = encoders_.size() - 1;
   for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) {
+    vpx_codec_enc_cfg_t temp_config;

[stefan-webrtc, 2015/06/18 09:39:02]

What is temporary with this config? Isn't it a screencast config?

[sprang_webrtc, 2015/06/18 13:05:02]

It's not stored in configurations_ and will be reset on the next update. Added
comment.

+    memcpy(&temp_config, &configurations_[i], sizeof(vpx_codec_enc_cfg_t));
+    if (temporal_layers_[stream_idx]->UpdateConfiguration(&temp_config)) {
+      if (vpx_codec_enc_config_set(&encoders_[i], &temp_config))
+        return WEBRTC_VIDEO_CODEC_ERROR;
+    }
+
     vpx_codec_control(&encoders_[i], VP8E_SET_FRAME_FLAGS, flags[stream_idx]);
     vpx_codec_control(&encoders_[i],
                       VP8E_SET_TEMPORAL_LAYER_ID,
                       temporal_layers_[stream_idx]->CurrentLayerId());
   }
   // TODO(holmer): Ideally the duration should be the timestamp diff of this
   // frame and the next frame to be encoded, which we don't have. Instead we
   // would like to use the duration of the previous frame. Unfortunately the
   // rate control seems to be off with that setup. Using the average input
   // frame rate to calculate an average duration for now.
   assert(codec_.maxFramerate > 0);
   uint32_t duration = 90000 / codec_.maxFramerate;
 
   // Note we must pass 0 for |flags| field in encode call below since they are
   // set above in |vpx_codec_control| function for each encoder/spatial layer.
   int error = vpx_codec_encode(&encoders_[0], &raw_images_[0], timestamp_,
                                duration, 0, VPX_DL_REALTIME);
   // Reset specific intra frame thresholds, following the key frame.
   if (send_key_frame) {
     vpx_codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
         rc_max_intra_target_);
   }
-  if (error) {
+  if (error)
     return WEBRTC_VIDEO_CODEC_ERROR;
-  }
   timestamp_ += duration;
   return GetEncodedPartitions(input_image, only_predict_from_key_frame);
 }
 
 // TODO(pbos): Make sure this works for properly for >1 encoders.
 int VP8EncoderImpl::UpdateCodecFrameSize(const VideoFrame& input_image) {
   codec_.width = input_image.width();
   codec_.height = input_image.height();
   // Update the cpu_speed setting for resolution change.
   vpx_codec_control(&(encoders_[0]),
@@ skipping 85 matching lines @@
         }
         PopulateCodecSpecific(&codec_specific, *pkt, stream_idx,
                               input_image.timestamp(),
                               only_predicting_from_key_frame);
         break;
       }
     }
     encoded_images_[encoder_idx]._timeStamp = input_image.timestamp();
     encoded_images_[encoder_idx].capture_time_ms_ =
         input_image.render_time_ms();
+
+    int qp = -1;
+    vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp);
     temporal_layers_[stream_idx]->FrameEncoded(
         encoded_images_[encoder_idx]._length,
-        encoded_images_[encoder_idx]._timeStamp);
+        encoded_images_[encoder_idx]._timeStamp, qp);
     if (send_stream_[stream_idx]) {
       if (encoded_images_[encoder_idx]._length > 0) {
         TRACE_COUNTER_ID1("webrtc", "EncodedFrameSize", encoder_idx,
                           encoded_images_[encoder_idx]._length);
         encoded_images_[encoder_idx]._encodedHeight =
             codec_.simulcastStream[stream_idx].height;
         encoded_images_[encoder_idx]._encodedWidth =
             codec_.simulcastStream[stream_idx].width;
         encoded_complete_callback_->Encoded(encoded_images_[encoder_idx],
                                             &codec_specific, &frag_info);
@@ skipping 378 matching lines @@
     return -1;
   }
   if (vpx_codec_control(copy->decoder_, VP8_SET_REFERENCE, ref_frame_)
       != VPX_CODEC_OK) {
     return -1;
   }
   return 0;
 }
 
 }  // namespace webrtc