Work on flexible mode and screen sharing.

webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc

 /*
  *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  *
  */
 
 #include "webrtc/modules/video_coding/codecs/vp9/vp9_impl.h"
 
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
 #include <vector>
 
 #include "vpx/vpx_encoder.h"
 #include "vpx/vpx_decoder.h"
 #include "vpx/vp8cx.h"
 #include "vpx/vp8dx.h"
 
 #include "webrtc/base/bind.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/common.h"
 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
 #include "webrtc/modules/interface/module_common_types.h"
+#include "webrtc/modules/video_coding/codecs/vp9/screenshare_layers.h"
 #include "webrtc/system_wrappers/interface/logging.h"
 #include "webrtc/system_wrappers/interface/tick_util.h"
 #include "webrtc/system_wrappers/interface/trace_event.h"
 
 namespace {
 
 // VP9DecoderImpl::ReturnFrame helper function used with WrappedI420Buffer.
 static void WrappedI420BufferNoLongerUsedCb(
     webrtc::Vp9FrameBufferPool::Vp9FrameBuffer* img_buffer) {
   img_buffer->Release();
@@ skipping 30 matching lines @@
       inited_(false),
       timestamp_(0),
       picture_id_(0),
       cpu_speed_(3),
       rc_max_intra_target_(0),
       encoder_(NULL),
       config_(NULL),
       raw_(NULL),
       input_image_(NULL),
       tl0_pic_idx_(0),
-      gof_idx_(0),
+      frames_since_kf_(0),
       num_temporal_layers_(0),
-      num_spatial_layers_(0) {
+      num_spatial_layers_(0),
+      frames_encoded_(0),
+      spatial_layer_(new ScreenshareLayersVP9()) {
   memset(&codec_, 0, sizeof(codec_));
   uint32_t seed = static_cast<uint32_t>(TickTime::MillisecondTimestamp());
   srand(seed);
 }
 
 VP9EncoderImpl::~VP9EncoderImpl() {
   Release();
 }
 
 int VP9EncoderImpl::Release() {
@@ skipping 79 matching lines @@
   }
   if (new_framerate < 1) {
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
   // Update bit rate
   if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) {
     new_bitrate_kbit = codec_.maxBitrate;
   }
   config_->rc_target_bitrate = new_bitrate_kbit;
   codec_.maxFramerate = new_framerate;
+  spatial_layer_->ConfigureBitrate(new_bitrate_kbit);
 
   if (!SetSvcRates()) {
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
 
   // Update encoder context
   if (vpx_codec_enc_config_set(encoder_, config_)) {
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
   return WEBRTC_VIDEO_CODEC_OK;
@@ skipping 18 matching lines @@
   if (number_of_cores < 1) {
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
   if (inst->codecSpecific.VP9.numberOfTemporalLayers > 3) {
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
   // libvpx currently supports only one or two spatial layers.
   if (inst->codecSpecific.VP9.numberOfSpatialLayers > 2) {
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
+
   int retVal = Release();
   if (retVal < 0) {
     return retVal;
   }
   if (encoder_ == NULL) {
     encoder_ = new vpx_codec_ctx_t;
   }
   if (config_ == NULL) {
     config_ = new vpx_codec_enc_cfg_t;
   }
@@ skipping 56 matching lines @@
   }
   // Determine number of threads based on the image size and #cores.
   config_->g_threads = NumberOfThreads(config_->g_w,
                                        config_->g_h,
                                        number_of_cores);
 
   cpu_speed_ = GetCpuSpeed(config_->g_w, config_->g_h);
 
   // TODO(asapersson): Check configuration of temporal switch up and increase
   // pattern length.
-  if (num_temporal_layers_ == 1) {
+  is_flexible_mode_ = inst->codecSpecific.VP9.flexibleMode;
+  if (is_flexible_mode_) {
+    config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_BYPASS;
+    config_->ts_number_layers = num_temporal_layers_;
+
+    if (codec_.mode == kScreensharing) {
+      spatial_layer_->ConfigureBitrate(inst->startBitrate);
+    }
+  } else if (num_temporal_layers_ == 1) {
     gof_.SetGofInfoVP9(kTemporalStructureMode1);
     config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_NOLAYERING;
     config_->ts_number_layers = 1;
     config_->ts_rate_decimator[0] = 1;
     config_->ts_periodicity = 1;
     config_->ts_layer_id[0] = 0;
   } else if (num_temporal_layers_ == 2) {
     gof_.SetGofInfoVP9(kTemporalStructureMode2);
     config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_0101;
     config_->ts_number_layers = 2;
@@ skipping 47 matching lines @@
 int VP9EncoderImpl::InitAndSetControlSettings(const VideoCodec* inst) {
   config_->ss_number_layers = num_spatial_layers_;
 
   int scaling_factor_num = 256;
   for (int i = num_spatial_layers_ - 1; i >= 0; --i) {
     svc_internal_.svc_params.max_quantizers[i] = config_->rc_max_quantizer;
     svc_internal_.svc_params.min_quantizers[i] = config_->rc_min_quantizer;
     // 1:2 scaling in each dimension.
     svc_internal_.svc_params.scaling_factor_num[i] = scaling_factor_num;
     svc_internal_.svc_params.scaling_factor_den[i] = 256;
-    scaling_factor_num /= 2;
+    if (!is_flexible_mode_)

[sprang_webrtc, 2015/09/15 15:41:22]

This seems very specific. This only works because of the assumption that it's
flexible mode iff it's screenshare...

I think we need to extend the api for the vp9 wrapper so that we can configure
these things explicitly. Check if there is a bug for this, add a todo and
reference.

[philipel, 2015/09/16 09:35:54]

Yes, this assumption is completely wrong. It should check if we are using
screensharing or not, which I have now done.

+      scaling_factor_num /= 2;
   }
 
   if (!SetSvcRates()) {
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
 
   if (vpx_codec_enc_init(encoder_, vpx_codec_vp9_cx(), config_, 0)) {
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
   }
   vpx_codec_control(encoder_, VP8E_SET_CPUUSED, cpu_speed_);
@@ skipping 79 matching lines @@
 
   // Image in vpx_image_t format.
   // Input image is const. VPX's raw image is not defined as const.
   raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>(input_image.buffer(kYPlane));
   raw_->planes[VPX_PLANE_U] = const_cast<uint8_t*>(input_image.buffer(kUPlane));
   raw_->planes[VPX_PLANE_V] = const_cast<uint8_t*>(input_image.buffer(kVPlane));
   raw_->stride[VPX_PLANE_Y] = input_image.stride(kYPlane);
   raw_->stride[VPX_PLANE_U] = input_image.stride(kUPlane);
   raw_->stride[VPX_PLANE_V] = input_image.stride(kVPlane);
 
-  int flags = 0;
+  vpx_enc_frame_flags_t flags = 0;
   bool send_keyframe = (frame_type == kKeyFrame);
   if (send_keyframe) {
     // Key frame request from caller.
     flags = VPX_EFLAG_FORCE_KF;
   }
+
+  if (is_flexible_mode_) {
+    SuperFrameRefSettings settings;
+    vpx_svc_ref_frame_config enc_layer_conf;
+    if (codec_.mode == kRealtimeVideo) {
+      // Real time video not yet implemented in flexible mode.
+      CHECK(false);

[sprang_webrtc, 2015/09/15 15:41:22]

RTC_NOTREACHED();

[philipel, 2015/09/16 09:35:54]

Done.

+    } else {
+      settings =
+          spatial_layer_->SfSettings(input_image.timestamp(), send_keyframe);
+    }
+    enc_layer_conf = GenerateRefsAndFlags(settings);
+    vpx_codec_control(encoder_, VP9E_SET_FIRST_SPATIAL_LAYER,
+                      settings.start_layer);
+    vpx_codec_control(encoder_, VP9E_SET_SVC_REF_FRAME_CONFIG, &enc_layer_conf);
+  }
+
   assert(codec_.maxFramerate > 0);
   uint32_t duration = 90000 / codec_.maxFramerate;
   if (vpx_codec_encode(encoder_, raw_, timestamp_, duration, flags,
                        VPX_DL_REALTIME)) {
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
   timestamp_ += duration;
 
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
                                        const vpx_codec_cx_pkt& pkt,
                                        uint32_t timestamp) {
   assert(codec_specific != NULL);
   codec_specific->codecType = kVideoCodecVP9;
   CodecSpecificInfoVP9 *vp9_info = &(codec_specific->codecSpecific.VP9);
   // TODO(asapersson): Set correct values.
   vp9_info->inter_pic_predicted =
       (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? false : true;
   vp9_info->flexible_mode = codec_.codecSpecific.VP9.flexibleMode;
   vp9_info->ss_data_available = ((pkt.data.frame.flags & VPX_FRAME_IS_KEY) &&
                                  !codec_.codecSpecific.VP9.flexibleMode)
                                     ? true
                                     : false;
-  if (pkt.data.frame.flags & VPX_FRAME_IS_KEY) {
-    gof_idx_ = 0;
-  }
+  if (pkt.data.frame.flags & VPX_FRAME_IS_KEY)
+    frames_since_kf_ = 0;
 
   vpx_svc_layer_id_t layer_id = {0};
   vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
 
   assert(num_temporal_layers_ > 0);
   assert(num_spatial_layers_ > 0);
   if (num_temporal_layers_ == 1) {
     assert(layer_id.temporal_layer_id == 0);
     vp9_info->temporal_idx = kNoTemporalIdx;
   } else {
     vp9_info->temporal_idx = layer_id.temporal_layer_id;
   }
   if (num_spatial_layers_ == 1) {
     assert(layer_id.spatial_layer_id == 0);
     vp9_info->spatial_idx = kNoSpatialIdx;
   } else {
     vp9_info->spatial_idx = layer_id.spatial_layer_id;
   }
   if (layer_id.spatial_layer_id != 0) {
     vp9_info->ss_data_available = false;
   }
 
-  if (vp9_info->flexible_mode) {
-    vp9_info->gof_idx = kNoGofIdx;
-  } else {
-    vp9_info->gof_idx =
-        static_cast<uint8_t>(gof_idx_++ % gof_.num_frames_in_gof);
-  }
-
   // TODO(asapersson): this info has to be obtained from the encoder.
   vp9_info->temporal_up_switch = true;
 
-  if (layer_id.spatial_layer_id == 0) {
+  if (layer_id.spatial_layer_id == spatial_layer_->CurrentLayer()) {
     picture_id_ = (picture_id_ + 1) & 0x7FFF;
     // TODO(asapersson): this info has to be obtained from the encoder.
     vp9_info->inter_layer_predicted = false;
   } else {
     // TODO(asapersson): this info has to be obtained from the encoder.
     vp9_info->inter_layer_predicted = true;
   }
 
   vp9_info->picture_id = picture_id_;
 
   if (!vp9_info->flexible_mode) {
     if (layer_id.temporal_layer_id == 0 && layer_id.spatial_layer_id == 0) {
       tl0_pic_idx_++;
     }
     vp9_info->tl0_pic_idx = tl0_pic_idx_;
   }
 
   // Always populate this, so that the packetizer can properly set the marker
   // bit.
   vp9_info->num_spatial_layers = num_spatial_layers_;
+
+  vp9_info->num_ref_pics = 0;
+  if (vp9_info->flexible_mode) {
+    vp9_info->gof_idx = kNoGofIdx;
+    if (!(pkt.data.frame.flags & VPX_FRAME_IS_KEY)) {
+      vp9_info->num_ref_pics = num_ref_pics_[layer_id.spatial_layer_id];
+      for (int i = 0; i < num_ref_pics_[layer_id.spatial_layer_id]; ++i) {
+        vp9_info->p_diff[i] = p_diff_[layer_id.spatial_layer_id][i];
+      }
+    }
+  } else {
+    vp9_info->gof_idx =
+        static_cast<uint8_t>(frames_since_kf_ % gof_.num_frames_in_gof);
+  }
+  ++frames_since_kf_;
+
   if (vp9_info->ss_data_available) {
     vp9_info->spatial_layer_resolution_present = true;
     for (size_t i = 0; i < vp9_info->num_spatial_layers; ++i) {
       vp9_info->width[i] = codec_.width *
                            svc_internal_.svc_params.scaling_factor_num[i] /
                            svc_internal_.svc_params.scaling_factor_den[i];
       vp9_info->height[i] = codec_.height *
                             svc_internal_.svc_params.scaling_factor_num[i] /
                             svc_internal_.svc_params.scaling_factor_den[i];
     }
@@ skipping 15 matching lines @@
 
   assert(pkt->kind == VPX_CODEC_CX_FRAME_PKT);
   memcpy(&encoded_image_._buffer[encoded_image_._length], pkt->data.frame.buf,
          pkt->data.frame.sz);
   frag_info.fragmentationOffset[part_idx] = encoded_image_._length;
   frag_info.fragmentationLength[part_idx] =
       static_cast<uint32_t>(pkt->data.frame.sz);
   frag_info.fragmentationPlType[part_idx] = 0;
   frag_info.fragmentationTimeDiff[part_idx] = 0;
   encoded_image_._length += static_cast<uint32_t>(pkt->data.frame.sz);
+
+  vpx_svc_layer_id_t layer_id = {0};
+  vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
+  if (is_flexible_mode_ && codec_.mode == kScreensharing)
+    spatial_layer_->LayerFrameEncoded(encoded_image_._length,
+                                      layer_id.spatial_layer_id);
+
   assert(encoded_image_._length <= encoded_image_._size);
 
   // End of frame.
   // Check if encoded frame is a key frame.
   if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) {
     encoded_image_._frameType = kKeyFrame;
   }
   PopulateCodecSpecific(&codec_specific, *pkt, input_image_->timestamp());
 
   if (encoded_image_._length > 0) {
     TRACE_COUNTER1("webrtc", "EncodedFrameSize", encoded_image_._length);
     encoded_image_._timeStamp = input_image_->timestamp();
     encoded_image_.capture_time_ms_ = input_image_->render_time_ms();
     encoded_image_._encodedHeight = raw_->d_h;
     encoded_image_._encodedWidth = raw_->d_w;
     encoded_complete_callback_->Encoded(encoded_image_, &codec_specific,
                                         &frag_info);
   }
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
+vpx_svc_ref_frame_config VP9EncoderImpl::GenerateRefsAndFlags(
+    const SuperFrameRefSettings& settings) {
+  static const vpx_enc_frame_flags_t all_flags =
+      VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_LAST |
+      VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF;
+  vpx_svc_ref_frame_config sf_conf;
+  if (settings.is_keyframe) {
+    // Used later on to make sure we don't make any invalid references.
+    memset(buf_upd_at_frame_, -1, sizeof(buf_upd_at_frame_));
+    for (int l = settings.start_layer; l <= settings.stop_layer; ++l) {

[sprang_webrtc, 2015/09/15 15:41:22]

More descriptive names please! layer instead of l? same below.

[philipel, 2015/09/16 09:35:53]

Done.

+      buf_upd_at_frame_[settings.layer[l].upd_buf] = frames_encoded_;
+      sf_conf.lst_fb_idx[l] = settings.layer[l].upd_buf;
+    }
+  } else {
+    for (int l = settings.start_layer; l <= settings.stop_layer; ++l) {
+      vpx_enc_frame_flags_t layer_flags = all_flags;
+      num_ref_pics_[l] = 0;
+      int8_t refs[3] = {settings.layer[l].ref_buf1, settings.layer[l].ref_buf2,
+                        settings.layer[l].ref_buf3};
+
+      for (unsigned int r = 0; r < kMaxVp9RefPics; ++r) {

[sprang_webrtc, 2015/09/15 15:41:22]

name

[philipel, 2015/09/16 09:35:54]

Done.

+        if (refs[r] == -1)
+          continue;
+
+        DCHECK_GE(refs[r], 0);
+        DCHECK_LE(refs[r], 7);
+        switch (num_ref_pics_[l]) {
+          case 0: {
+            sf_conf.lst_fb_idx[l] = refs[r];
+            layer_flags ^= VP8_EFLAG_NO_REF_LAST;

[sprang_webrtc, 2015/09/15 15:41:22]

Not sure I follow why you xor this flag. Comment?

[philipel, 2015/09/16 09:35:54]

If we want to reference the LAST buffer then we should use the flags
VP8_EFLAG_NO_REF_GF and VP8_EFLAG_NO_REF_ARF. So what I do is to start with all
the flags set (line 667) and then simply xor out the flag for the buffer that
should be referenced.

Lets say we want to reference two buffers, only looking at the reference flags
the following will happen:

Before first iteration:
layer_flags = NO_REF_LAST ^ NO_REF_GF ^ NO_REF_ARF
After first iteration:
layer_flags = (NO_REF_LAST ^ NO_REF_LAST) ^ NO_REF_GF ^ NO_REF_ARF = NO_REF_GF ^
NO_REF_ARF

Before second iteration:
layer_flags = NO_REF_GF ^ NO_REF_ARF
After second iteration:
layer_flags = (NO_REF_GF ^ NO_REF_GF) ^ NO_REF_ARF = NO_REF_ARF

+            break;
+          }
+          case 1: {
+            sf_conf.gld_fb_idx[l] = refs[r];
+            layer_flags ^= VP8_EFLAG_NO_REF_GF;
+            break;
+          }
+          case 2: {
+            sf_conf.alt_fb_idx[l] = refs[r];
+            layer_flags ^= VP8_EFLAG_NO_REF_ARF;
+            break;
+          }
+        }
+        // Make sure we don't reference a buffer that hasn't been
+        // used at all or hasn't been used since a keyframe.
+        DCHECK_NE(buf_upd_at_frame_[refs[r]], -1);
+
+        p_diff_[l][num_ref_pics_[l]] =
+            frames_encoded_ - buf_upd_at_frame_[refs[r]];
+        num_ref_pics_[l]++;
+      }
+
+      if (settings.layer[l].upd_buf != -1) {
+        for (unsigned int r = 0; r < kMaxVp9RefPics; ++r) {
+          if (settings.layer[l].upd_buf == refs[r]) {
+            switch (r) {
+              case 0: {
+                layer_flags ^= VP8_EFLAG_NO_UPD_LAST;
+                break;
+              }
+              case 1: {
+                layer_flags ^= VP8_EFLAG_NO_UPD_GF;
+                break;
+              }
+              case 2: {
+                layer_flags ^= VP8_EFLAG_NO_UPD_ARF;
+                break;
+              }
+            }
+            goto done;

[sprang_webrtc, 2015/09/15 15:41:22]

Please no goto's!

Use a descriptively named temp bool flag instead to determine what to do after
the break. 

[philipel, 2015/09/16 09:35:54]

Done.

+          }
+        }
+        // If we have three references and a buffer is specified to be updated,
+        // then that buffer must be the same as one of the three references.
+        CHECK_LT(num_ref_pics_[l], kMaxVp9RefPics);
+
+        sf_conf.alt_fb_idx[l] = settings.layer[l].upd_buf;
+        layer_flags ^= VP8_EFLAG_NO_UPD_ARF;
+
+      done:
+        buf_upd_at_frame_[settings.layer[l].upd_buf] = frames_encoded_;
+        sf_conf.frame_flags[l] = layer_flags;
+      }
+    }
+  }
+  frames_encoded_++;
+  return sf_conf;
+}
+
 int VP9EncoderImpl::SetChannelParameters(uint32_t packet_loss, int64_t rtt) {
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int VP9EncoderImpl::RegisterEncodeCompleteCallback(
     EncodedImageCallback* callback) {
   encoded_complete_callback_ = callback;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
@@ skipping 164 matching lines @@
     decoder_ = NULL;
   }
   // Releases buffers from the pool. Any buffers not in use are deleted. Buffers
   // still referenced externally are deleted once fully released, not returning
   // to the pool.
   frame_buffer_pool_.ClearPool();
   inited_ = false;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 }  // namespace webrtc