Avoid undefined behavior in vp8 screenshare_layers

webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc

 /* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */
 
 #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
@@ skipping 129 matching lines @@
       // - TL0 frame rate no less than framerate / kMaxTL0FpsReduction.
       // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate.
       target_bitrate_kbps =
           std::min(bitrate_kbps * kMaxTL0FpsReduction,
                    max_bitrate_kbps / kAcceptableTargetOvershoot);
 
       cfg->rc_target_bitrate = std::max(bitrate_kbps, target_bitrate_kbps);
     }
 
     // Don't reconfigure qp limits during quality boost frames.
-    if (layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) {
+    if (active_layer_ != -1 &&

[pbos-webrtc, 2016/02/02 21:16:14]

Can you amend the comment above to include what -1 as active_layer_ means in
this function?

[sprang, 2016/02/03 10:05:37]

Done.

+        layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) {
       min_qp_ = cfg->rc_min_quantizer;
       max_qp_ = cfg->rc_max_quantizer;
       // After a dropped frame, a frame with max qp will be encoded and the
       // quality will then ramp up from there. To boost the speed of recovery,
       // encode the next frame with lower max qp. TL0 is the most important to
       // improve since the errors in this layer will propagate to TL1.
       // Currently, reduce max qp by 20% for TL0 and 15% for TL1.
       layers_[0].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 80) / 100);
       layers_[1].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 85) / 100);
     }
   }
 
   int avg_frame_size = (target_bitrate_kbps * 1000) / (8 * framerate);
   max_debt_bytes_ = 4 * avg_frame_size;
 
   return true;
 }
 
 void ScreenshareLayers::FrameEncoded(unsigned int size,
                                      uint32_t timestamp,
                                      int qp) {
+  RTC_DCHECK_NE(-1, active_layer_);
   if (size == 0) {
     layers_[active_layer_].state = TemporalLayer::State::kDropped;
     return;
   }
 
   if (layers_[active_layer_].state == TemporalLayer::State::kDropped) {
     layers_[active_layer_].state = TemporalLayer::State::kQualityBoost;
   }
 
   if (qp != -1)
     layers_[active_layer_].last_qp = qp;
 
   if (active_layer_ == 0) {
     layers_[0].debt_bytes_ += size;
     layers_[1].debt_bytes_ += size;
   } else if (active_layer_ == 1) {
     layers_[1].debt_bytes_ += size;
   }
 }
 
 void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync,
                                               CodecSpecificInfoVP8* vp8_info,
                                               uint32_t timestamp) {
   int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
   if (number_of_temporal_layers_ == 1) {
     vp8_info->temporalIdx = kNoTemporalIdx;
     vp8_info->layerSync = false;
     vp8_info->tl0PicIdx = kNoTl0PicIdx;
   } else {
+    RTC_DCHECK_NE(-1, active_layer_);
     vp8_info->temporalIdx = active_layer_;
     if (base_layer_sync) {
       vp8_info->temporalIdx = 0;
       last_sync_timestamp_ = unwrapped_timestamp;
     } else if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) {
       // Regardless of pattern the frame after a base layer sync will always
       // be a layer sync.
       last_sync_timestamp_ = unwrapped_timestamp;
     }
     vp8_info->layerSync = last_sync_timestamp_ != -1 &&
                           last_sync_timestamp_ == unwrapped_timestamp;
     if (vp8_info->temporalIdx == 0) {
       tl0_pic_idx_++;
     }
     last_base_layer_sync_ = base_layer_sync;
     vp8_info->tl0PicIdx = tl0_pic_idx_;
   }
 }
 
 bool ScreenshareLayers::TimeToSync(int64_t timestamp) const {
   if (active_layer_ != 1) {

[pbos-webrtc, 2016/02/02 21:16:14]

Can you DCHECK_NE here as well?

[sprang, 2016/02/03 10:05:37]

Done.

     RTC_NOTREACHED();
     return false;
   }
   RTC_DCHECK_NE(-1, layers_[0].last_qp);
   if (layers_[1].last_qp == -1) {
     // First frame in TL1 should only depend on TL0 since there are no
     // previous frames in TL1.
     return true;
   }
 
   RTC_DCHECK_NE(-1, last_sync_timestamp_);
   int64_t timestamp_diff = timestamp - last_sync_timestamp_;
   if (timestamp_diff > kMaxTimeBetweenSyncs) {
     // After a certain time, force a sync frame.
     return true;
   } else if (timestamp_diff < kMinTimeBetweenSyncs) {
     // If too soon from previous sync frame, don't issue a new one.
     return false;
   }
   // Issue a sync frame if difference in quality between TL0 and TL1 isn't too
   // large.
   if (layers_[0].last_qp - layers_[1].last_qp < kQpDeltaThresholdForSync)
     return true;
   return false;
 }
 
 bool ScreenshareLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) {
   if (max_qp_ == -1 || number_of_temporal_layers_ <= 1)
     return false;
+  RTC_DCHECK_NE(-1, active_layer_);
 
   // If layer is in the quality boost state (following a dropped frame), update
   // the configuration with the adjusted (lower) qp and set the state back to
   // normal.
   unsigned int adjusted_max_qp;
   if (layers_[active_layer_].state == TemporalLayer::State::kQualityBoost &&
       layers_[active_layer_].enhanced_max_qp != -1) {
     adjusted_max_qp = layers_[active_layer_].enhanced_max_qp;
     layers_[active_layer_].state = TemporalLayer::State::kNormal;
   } else {
@@ skipping 12 matching lines @@
 void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) {
   uint32_t debt_reduction_bytes = target_rate_kbps_ * delta_ms / 8;
   if (debt_reduction_bytes >= debt_bytes_) {
     debt_bytes_ = 0;
   } else {
     debt_bytes_ -= debt_reduction_bytes;
   }
 }
 
 }  // namespace webrtc