Add cropping to VIEEncoder to match simulcast streams resolution

webrtc/video/vie_encoder.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include "webrtc/video/vie_encoder.h"
 
 #include <algorithm>
 #include <limits>
 #include <numeric>
 #include <utility>
 
+#include "webrtc/api/video/i420_buffer.h"
 #include "webrtc/base/arraysize.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/location.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/timeutils.h"
 #include "webrtc/base/trace_event.h"
 #include "webrtc/common_video/include/video_bitrate_allocator.h"
 #include "webrtc/common_video/include/video_frame.h"
 #include "webrtc/modules/pacing/paced_sender.h"
 #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
@@ skipping 540 matching lines @@
   }
 }
 
 void ViEEncoder::ReconfigureEncoder() {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   RTC_DCHECK(pending_encoder_reconfiguration_);
   std::vector<VideoStream> streams =
       encoder_config_.video_stream_factory->CreateEncoderStreams(
           last_frame_info_->width, last_frame_info_->height, encoder_config_);
 
+  // Stream dimensions may be not equal to given because of a simulcast
+  // restrictions.
+  int highest_stream_width = static_cast<int>(streams.back().width);
+  int highest_stream_height = static_cast<int>(streams.back().height);
+  // Dimension may be reduced to be, e.g. divisible by 4.
+  RTC_CHECK_GE(last_frame_info_->width, highest_stream_width);
+  RTC_CHECK_GE(last_frame_info_->height, highest_stream_height);
+  crop_width_ = last_frame_info_->width - highest_stream_width;
+  crop_height_ = last_frame_info_->height - highest_stream_height;

[sprang_webrtc, 2017/06/16 08:57:23]

Hm, I'm not always cropping here is a good idea.. 

Normalizing to the closest resolution divisible by 4 is only one of the changes
the video stream factory can request. It may for instance also choose to create
streams of lower resolution if the number of assigned ssrcs isn't high enough.
Though I guess that case won't really work today either?

[ilnik, 2017/06/16 11:22:49]

Before, that case wouldn't work in the same way, dropping all the frames: codec
resolutions are set as a maximum from all the streams in
https://cs.chromium.org/chromium/src/third_party/webrtc/modules/video_coding/...
. So, if the upper stream is not matching the input frame resolution, frame will
be dropped.

Now, it will work by cropping the image, instead of scaling. Should I add
additional logic to make it scale if, e.g crop area is more than some percentage
of image area?

[sprang_webrtc, 2017/06/16 11:58:47]

Yeah, maybe that's a working stopgap solution. Crop if delta < 4 pixels,
otherwise scale?
But add a TODO about using VideoSinkWants signaling instead.

[ilnik, 2017/06/16 12:53:30]

Done.

+

[sprang_webrtc, 2017/06/16 08:57:23]

The full stack test is great, but would probably be good if you also added a
unit test on vie_encoder level, using a test factory design to trigger this
scenario.

[ilnik, 2017/06/16 11:22:48]

Done.

   VideoCodec codec;
   if (!VideoCodecInitializer::SetupCodec(encoder_config_, settings_, streams,
                                          nack_enabled_, &codec,
                                          &rate_allocator_)) {
     LOG(LS_ERROR) << "Failed to create encoder configuration.";
   }
 
   codec.startBitrate =
       std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate);
   codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate);
@@ skipping 175 matching lines @@
                                           bitrate_observer_);
   }
   last_parameters_update_ms_.emplace(now_ms);
 
   if (EncoderPaused()) {
     TraceFrameDropStart();
     return;
   }
   TraceFrameDropEnd();
 
+  VideoFrame out_frame(video_frame);
+  // Crop frame if needed.
+  if (crop_width_ > 0 || crop_height_ > 0) {
+    int cropped_width = video_frame.width() - crop_width_;
+    int cropped_height = video_frame.height() - crop_height_;
+    rtc::scoped_refptr<I420Buffer> cropped_buffer =
+        I420Buffer::Create(cropped_width, cropped_height);
+    cropped_buffer->CropAndScaleFrom(
+        *video_frame.video_frame_buffer()->ToI420(), crop_width_ / 2,
+        crop_height_ / 2, cropped_width, cropped_height);
+    out_frame =
+        VideoFrame(cropped_buffer, video_frame.timestamp(),
+                   video_frame.render_time_ms(), video_frame.rotation());
+  }
+
   TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(),
                           "Encode");
 
-  overuse_detector_->FrameCaptured(video_frame, time_when_posted_us);
+  overuse_detector_->FrameCaptured(out_frame, time_when_posted_us);
 
-  video_sender_.AddVideoFrame(video_frame, nullptr);
+  video_sender_.AddVideoFrame(out_frame, nullptr);
 }
 
 void ViEEncoder::SendKeyFrame() {
   if (!encoder_queue_.IsCurrent()) {
     encoder_queue_.PostTask([this] { SendKeyFrame(); });
     return;
   }
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   video_sender_.IntraFrameRequest(0);
 }
@@ skipping 406 matching lines @@
 std::string ViEEncoder::AdaptCounter::ToString(
     const std::vector<int>& counters) const {
   std::stringstream ss;
   for (size_t reason = 0; reason < kScaleReasonSize; ++reason) {
     ss << (reason ? " cpu" : "quality") << ":" << counters[reason];
   }
   return ss.str();
 }
 
 }  // namespace webrtc