In screenshare mode, suppress VP8 bitrate overshoot and increase quality

webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 260 matching lines @@
     // interpret the startBitrate as the encoder target bitrate. This is
     // to allow for a different max bitrate, so if the codec can't meet
     // the target we still allow it to overshoot up to the max before dropping
     // frames. This hack should be improved.
     if (codec_.targetBitrate > 0 &&
         (codec_.codecSpecific.VP8.numberOfTemporalLayers == 2 ||
          codec_.simulcastStream[0].numberOfTemporalLayers == 2)) {
       int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate);
       max_bitrate = std::min(codec_.maxBitrate, target_bitrate);
       target_bitrate = tl0_bitrate;
-      framerate = -1;
     }
     configurations_[i].rc_target_bitrate = target_bitrate;
     temporal_layers_[stream_idx]->ConfigureBitrates(target_bitrate,
                                                     max_bitrate,
                                                     framerate,
                                                     &configurations_[i]);
     if (vpx_codec_enc_config_set(&encoders_[i], &configurations_[i])) {
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
   }
@@ skipping 13 matching lines @@
 void VP8EncoderImpl::SetupTemporalLayers(int num_streams,
                                                  int num_temporal_layers,
                                                  const VideoCodec& codec) {
   const Config default_options;
   const TemporalLayers::Factory& tl_factory =
       (codec.extra_options ? codec.extra_options : &default_options)
           ->Get<TemporalLayers::Factory>();
   if (num_streams == 1) {
     if (codec.mode == kScreensharing) {
       // Special mode when screensharing on a single stream.
-      temporal_layers_.push_back(new ScreenshareLayers(num_temporal_layers,
-                                                       rand(),
-                                                       &tl0_frame_dropper_,
-                                                       &tl1_frame_dropper_));
+      temporal_layers_.push_back(
+          new ScreenshareLayers(num_temporal_layers, rand()));
     } else {
       temporal_layers_.push_back(
           tl_factory.Create(num_temporal_layers, rand()));
     }
   } else {
     for (int i = 0; i < num_streams; ++i) {
       // TODO(andresp): crash if layers is invalid.
       int layers = codec.simulcastStream[i].numberOfTemporalLayers;
       if (layers < 1) layers = 1;
       temporal_layers_.push_back(tl_factory.Create(layers, rand()));
@@ skipping 334 matching lines @@
                       codec_.codecSpecific.VP8.denoisingOn ?
                       denoiser_state : kDenoiserOff);
   }
   for (size_t i = 0; i < encoders_.size(); ++i) {
     vpx_codec_control(&(encoders_[i]), VP8E_SET_STATIC_THRESHOLD, 1);
     vpx_codec_control(&(encoders_[i]), VP8E_SET_CPUUSED, cpu_speed_[i]);
     vpx_codec_control(&(encoders_[i]), VP8E_SET_TOKEN_PARTITIONS,
                       static_cast<vp8e_token_partitions>(token_partitions_));
     vpx_codec_control(&(encoders_[i]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
                       rc_max_intra_target_);
+    // VP8E_SET_SCREEN_CONTENT_MODE 2 = screen content with more aggressive
+    // rate control (drop frames on large target bitrate overshoot)
     vpx_codec_control(&(encoders_[i]), VP8E_SET_SCREEN_CONTENT_MODE,
-                      codec_.mode == kScreensharing);
+                      codec_.mode == kScreensharing ? 2 : 0);
   }
   inited_ = true;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 uint32_t VP8EncoderImpl::MaxIntraTarget(uint32_t optimalBuffersize) {
   // Set max to the optimal buffer level (normalized by target BR),
   // and scaled by a scalePar.
   // Max target size = scalePar * optimalBufferSize * targetBR[Kbps].
   // This values is presented in percentage of perFrameBw:
   // perFrameBw = targetBR[Kbps] * 1000 / frameRate.
   // The target in % is as follows:
 
   float scalePar = 0.5;
   uint32_t targetPct = optimalBuffersize * scalePar * codec_.maxFramerate / 10;
 
   // Don't go below 3 times the per frame bandwidth.
   const uint32_t minIntraTh = 300;
   return (targetPct < minIntraTh) ? minIntraTh: targetPct;
 }
 
 int VP8EncoderImpl::Encode(const VideoFrame& frame,
                            const CodecSpecificInfo* codec_specific_info,
                            const std::vector<VideoFrameType>* frame_types) {
   TRACE_EVENT1("webrtc", "VP8::Encode", "timestamp", frame.timestamp());
 
-  if (!inited_) {
+  if (!inited_)
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
-  }
-  if (frame.IsZeroSize()) {
+  if (frame.IsZeroSize())
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
-  }
-  if (encoded_complete_callback_ == NULL) {
+  if (encoded_complete_callback_ == NULL)
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
-  }
 
   // Only apply scaling to improve for single-layer streams. The scaling metrics
   // use frame drops as a signal and is only applicable when we drop frames.
   const bool use_quality_scaler = encoders_.size() == 1 &&
                                   configurations_[0].rc_dropframe_thresh > 0 &&
                                   codec_.codecSpecific.VP8.automaticResizeOn;
   const VideoFrame& input_image =
       use_quality_scaler ? quality_scaler_.GetScaledFrame(frame) : frame;
 
   if (use_quality_scaler && (input_image.width() != codec_.width ||
@@ skipping 124 matching lines @@
         }
       }
     }
   }
   // Set the encoder frame flags and temporal layer_id for each spatial stream.
   // Note that |temporal_layers_| are defined starting from lowest resolution at
   // position 0 to highest resolution at position |encoders_.size() - 1|,
   // whereas |encoder_| is from highest to lowest resolution.
   size_t stream_idx = encoders_.size() - 1;
   for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) {
+    // Allow the layers adapter to temporarily modify the configuration. This
+    // change isn't stored in configurations_ so change will be discarded at
+    // the next update.
+    vpx_codec_enc_cfg_t temp_config;
+    memcpy(&temp_config, &configurations_[i], sizeof(vpx_codec_enc_cfg_t));
+    if (temporal_layers_[stream_idx]->UpdateConfiguration(&temp_config)) {
+      if (vpx_codec_enc_config_set(&encoders_[i], &temp_config))
+        return WEBRTC_VIDEO_CODEC_ERROR;
+    }
+
     vpx_codec_control(&encoders_[i], VP8E_SET_FRAME_FLAGS, flags[stream_idx]);
     vpx_codec_control(&encoders_[i],
                       VP8E_SET_TEMPORAL_LAYER_ID,
                       temporal_layers_[stream_idx]->CurrentLayerId());
   }
   // TODO(holmer): Ideally the duration should be the timestamp diff of this
   // frame and the next frame to be encoded, which we don't have. Instead we
   // would like to use the duration of the previous frame. Unfortunately the
   // rate control seems to be off with that setup. Using the average input
   // frame rate to calculate an average duration for now.
   assert(codec_.maxFramerate > 0);
   uint32_t duration = 90000 / codec_.maxFramerate;
 
   // Note we must pass 0 for |flags| field in encode call below since they are
   // set above in |vpx_codec_control| function for each encoder/spatial layer.
   int error = vpx_codec_encode(&encoders_[0], &raw_images_[0], timestamp_,
                                duration, 0, VPX_DL_REALTIME);
   // Reset specific intra frame thresholds, following the key frame.
   if (send_key_frame) {
     vpx_codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
         rc_max_intra_target_);
   }
-  if (error) {
+  if (error)
     return WEBRTC_VIDEO_CODEC_ERROR;
-  }
   timestamp_ += duration;
   return GetEncodedPartitions(input_image, only_predict_from_key_frame);
 }
 
 // TODO(pbos): Make sure this works for properly for >1 encoders.
 int VP8EncoderImpl::UpdateCodecFrameSize(const VideoFrame& input_image) {
   codec_.width = input_image.width();
   codec_.height = input_image.height();
   // Update the cpu_speed setting for resolution change.
   vpx_codec_control(&(encoders_[0]),
@@ skipping 37 matching lines @@
   temporal_layers_[stream_idx]->PopulateCodecSpecific(base_layer_sync_point,
                                                       vp8Info,
                                                       timestamp);
   // Prepare next.
   picture_id_[stream_idx] = (picture_id_[stream_idx] + 1) & 0x7FFF;
 }
 
 int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image,
                                          bool only_predicting_from_key_frame) {
   int stream_idx = static_cast<int>(encoders_.size()) - 1;
+  int result = WEBRTC_VIDEO_CODEC_OK;
   for (size_t encoder_idx = 0; encoder_idx < encoders_.size();
       ++encoder_idx, --stream_idx) {
     vpx_codec_iter_t iter = NULL;
     int part_idx = 0;
     encoded_images_[encoder_idx]._length = 0;
     encoded_images_[encoder_idx]._frameType = kDeltaFrame;
     RTPFragmentationHeader frag_info;
     // token_partitions_ is number of bits used.
     frag_info.VerifyAndAllocateFragmentationHeader((1 << token_partitions_)
                                                    + 1);
@@ skipping 28 matching lines @@
         }
         PopulateCodecSpecific(&codec_specific, *pkt, stream_idx,
                               input_image.timestamp(),
                               only_predicting_from_key_frame);
         break;
       }
     }
     encoded_images_[encoder_idx]._timeStamp = input_image.timestamp();
     encoded_images_[encoder_idx].capture_time_ms_ =
         input_image.render_time_ms();
+
+    int qp = -1;
+    vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp);
     temporal_layers_[stream_idx]->FrameEncoded(
         encoded_images_[encoder_idx]._length,
-        encoded_images_[encoder_idx]._timeStamp);
+        encoded_images_[encoder_idx]._timeStamp, qp);
     if (send_stream_[stream_idx]) {
       if (encoded_images_[encoder_idx]._length > 0) {
         TRACE_COUNTER_ID1("webrtc", "EncodedFrameSize", encoder_idx,
                           encoded_images_[encoder_idx]._length);
         encoded_images_[encoder_idx]._encodedHeight =
             codec_.simulcastStream[stream_idx].height;
         encoded_images_[encoder_idx]._encodedWidth =
             codec_.simulcastStream[stream_idx].width;
         encoded_complete_callback_->Encoded(encoded_images_[encoder_idx],
                                             &codec_specific, &frag_info);
+      } else if (codec_.mode == kScreensharing) {
+        result = WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT;
       }
     } else {
       // Required in case padding is applied to dropped frames.
       encoded_images_[encoder_idx]._length = 0;
       encoded_images_[encoder_idx]._frameType = kSkipFrame;
       codec_specific.codecType = kVideoCodecVP8;
       CodecSpecificInfoVP8* vp8Info = &(codec_specific.codecSpecific.VP8);
       vp8Info->pictureId = picture_id_[stream_idx];
       vp8Info->simulcastIdx = stream_idx;
       vp8Info->keyIdx = kNoKeyIdx;
       encoded_complete_callback_->Encoded(encoded_images_[encoder_idx],
                                           &codec_specific, NULL);
     }
   }
   if (encoders_.size() == 1 && send_stream_[0]) {
     if (encoded_images_[0]._length > 0) {
       int qp;
       vpx_codec_control(&encoders_[0], VP8E_GET_LAST_QUANTIZER_64, &qp);
       quality_scaler_.ReportQP(qp);
     } else {
       quality_scaler_.ReportDroppedFrame();
     }
   }
-  return WEBRTC_VIDEO_CODEC_OK;
+  return result;
 }
 
 int VP8EncoderImpl::SetChannelParameters(uint32_t packetLoss, int64_t rtt) {
   rps_.SetRtt(rtt);
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int VP8EncoderImpl::RegisterEncodeCompleteCallback(
     EncodedImageCallback* callback) {
   encoded_complete_callback_ = callback;
@@ skipping 344 matching lines @@
     return -1;
   }
   if (vpx_codec_control(copy->decoder_, VP8_SET_REFERENCE, ref_frame_)
       != VPX_CODEC_OK) {
     return -1;
   }
   return 0;
 }
 
 }  // namespace webrtc