Fix VideoToolbox backgrounding issues

webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  *
  */
 
 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.h"
 
 #if defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)
 
 #include <memory>
 #include <string>
 #include <vector>
 
 #include "libyuv/convert_from.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
+#if defined(WEBRTC_IOS)
+#include "webrtc/base/objc/RTCUIApplication.h"
+#endif
 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h"
 #include "webrtc/system_wrappers/include/clock.h"
 
 namespace internal {
 
 // Convenience function for creating a dictionary.
 inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys,
                                           CFTypeRef* values,
                                           size_t size) {
   return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size,
@@ skipping 198 matching lines @@
   if (input_image.IsZeroSize()) {
     // It's possible to get zero sizes as a signal to produce keyframes (this
     // happens for internal sources). But this shouldn't happen in
     // webrtcvideoengine2.
     RTC_NOTREACHED();
     return WEBRTC_VIDEO_CODEC_OK;
   }
   if (!callback_ || !compression_session_) {
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
   }
-
+#if defined(WEBRTC_IOS)
+  bool is_app_active = RTCIsUIApplicationActive();
+  if (!is_app_active) {
+    // Ignore all encode requests when app isn't active. In this state, the
+    // hardware encoder has been invalidated by the OS.
+    return WEBRTC_VIDEO_CODEC_OK;
+  }
+#endif
   // Get a pixel buffer from the pool and copy frame data over.
   CVPixelBufferPoolRef pixel_buffer_pool =
       VTCompressionSessionGetPixelBufferPool(compression_session_);
+  if (!pixel_buffer_pool) {
+    // Kind of a hack. On backgrounding, the compression session seems to get
+    // invalidated, which causes this pool call to fail when the application
+    // is foregrounded and frames are being sent for encoding again.
+    // Resetting the session when this happens fixes the issue.
+    ResetCompressionSession();
+    pixel_buffer_pool =
+        VTCompressionSessionGetPixelBufferPool(compression_session_);
+  }
+  if (!pixel_buffer_pool) {
+    LOG(LS_ERROR) << "Failed to get pixel buffer pool.";
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
   CVPixelBufferRef pixel_buffer = nullptr;
   CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool,
                                                     &pixel_buffer);
   if (ret != kCVReturnSuccess) {
     LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret;
     // We probably want to drop frames here, since failure probably means
     // that the pool is empty.
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
   RTC_DCHECK(pixel_buffer);
@@ skipping 23 matching lines @@
     frame_properties = internal::CreateCFDictionary(keys, values, 1);
   }
   std::unique_ptr<internal::FrameEncodeParams> encode_params;
   encode_params.reset(new internal::FrameEncodeParams(
       this, codec_specific_info, width_, height_, input_image.render_time_ms(),
       input_image.timestamp()));
 
   // Update the bitrate if needed.
   SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
 
-  VTCompressionSessionEncodeFrame(
+  OSStatus status = VTCompressionSessionEncodeFrame(
       compression_session_, pixel_buffer, presentation_time_stamp,
       kCMTimeInvalid, frame_properties, encode_params.release(), nullptr);
   if (frame_properties) {
     CFRelease(frame_properties);
   }
   if (pixel_buffer) {
     CVBufferRelease(pixel_buffer);
   }
+  if (status != noErr) {
+    LOG(LS_ERROR) << "Failed to encode frame with code: " << status;
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback(
     EncodedImageCallback* callback) {
   callback_ = callback;
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss,
@@ skipping 179 matching lines @@
   if (result != 0) {
     LOG(LS_ERROR) << "Encode callback failed: " << result;
     return;
   }
   bitrate_adjuster_.Update(frame._size);
 }
 
 }  // namespace webrtc
 
 #endif  // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)