Fix VideoToolbox backgrounding issues

webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.cc

 /*
  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  *
  */
 
 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.h"
 
 #if defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)
 
 #include <memory>
 
 #include "libyuv/convert.h"
 #include "webrtc/base/checks.h"
 #include "webrtc/base/logging.h"
+#if defined(WEBRTC_IOS)
+#include "webrtc/base/objc/RTCUIApplication.h"
+#endif
 #include "webrtc/common_video/include/video_frame_buffer.h"
 #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h"
 #include "webrtc/video_frame.h"
 
 namespace internal {
 
 // Convenience function for creating a dictionary.
 inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys,
                                           CFTypeRef* values,
                                           size_t size) {
@@ skipping 90 matching lines @@
 }
 
 int H264VideoToolboxDecoder::Decode(
     const EncodedImage& input_image,
     bool missing_frames,
     const RTPFragmentationHeader* fragmentation,
     const CodecSpecificInfo* codec_specific_info,
     int64_t render_time_ms) {
   RTC_DCHECK(input_image._buffer);
 
+#if defined(WEBRTC_IOS)
+  if (!RTCIsUIApplicationActive()) {
+    // Ignore all decode requests when app isn't active. In this state, the
+    // hardware decoder has been invalidated by the OS.
+    // Reset video format so that we won't process frames until the next
+    // keyframe.
+    SetVideoFormat(nullptr);
+    return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
+  }
+#endif
+  CMVideoFormatDescriptionRef input_format = nullptr;
+  if (H264AnnexBBufferHasVideoFormatDescription(input_image._buffer,
+                                                input_image._length)) {
+    input_format = CreateVideoFormatDescription(input_image._buffer,
+                                                input_image._length);
+    if (input_format) {
+      // Check if the video format has changed, and reinitialize decoder if
+      // needed.
+      if (!CMFormatDescriptionEqual(input_format, video_format_)) {
+        SetVideoFormat(input_format);
+        ResetDecompressionSession();
+      }
+      CFRelease(input_format);
+    }
+  }
+  if (!video_format_) {
+    // We received a frame but we don't have format information so we can't
+    // decode it.
+    // This can happen after backgrounding. We need to wait for the next
+    // sps/pps before we can resume so we request a keyframe by returning an
+    // error.
+    LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required.";
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
   CMSampleBufferRef sample_buffer = nullptr;
   if (!H264AnnexBBufferToCMSampleBuffer(input_image._buffer,
                                         input_image._length, video_format_,
                                         &sample_buffer)) {
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
   RTC_DCHECK(sample_buffer);
-  // Check if the video format has changed, and reinitialize decoder if needed.
-  CMVideoFormatDescriptionRef description =
-      CMSampleBufferGetFormatDescription(sample_buffer);
-  if (!CMFormatDescriptionEqual(description, video_format_)) {
-    SetVideoFormat(description);
-    ResetDecompressionSession();
-  }
   VTDecodeFrameFlags decode_flags =
       kVTDecodeFrame_EnableAsynchronousDecompression;
   std::unique_ptr<internal::FrameDecodeParams> frame_decode_params;
   frame_decode_params.reset(
       new internal::FrameDecodeParams(callback_, input_image._timeStamp));
   OSStatus status = VTDecompressionSessionDecodeFrame(
       decompression_session_, sample_buffer, decode_flags,
       frame_decode_params.release(), nullptr);
+#if defined(WEBRTC_IOS)
+  // Re-initialize the decoder if we have an invalid session while the app is
+  // active and retry the decode request.
+  if (status == kVTInvalidSessionErr &&
+      ResetDecompressionSession() == WEBRTC_VIDEO_CODEC_OK) {
+    frame_decode_params.reset(
+        new internal::FrameDecodeParams(callback_, input_image._timeStamp));
+    status = VTDecompressionSessionDecodeFrame(
+        decompression_session_, sample_buffer, decode_flags,
+        frame_decode_params.release(), nullptr);
+  }
+#endif
   CFRelease(sample_buffer);
   if (status != noErr) {
     LOG(LS_ERROR) << "Failed to decode frame with code: " << status;
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int H264VideoToolboxDecoder::RegisterDecodeCompleteCallback(
     DecodedImageCallback* callback) {
@@ skipping 74 matching lines @@
   RTC_DCHECK(decompression_session_);
 #if defined(WEBRTC_IOS)
   VTSessionSetProperty(decompression_session_,
                        kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
 #endif
 }
 
 void H264VideoToolboxDecoder::DestroyDecompressionSession() {
   if (decompression_session_) {
     VTDecompressionSessionInvalidate(decompression_session_);
+    CFRelease(decompression_session_);

[stefan-webrtc, 2016/03/07 15:00:02]

This isn't mentioned in the CL description, AFAICT. Does it fix a memory leak?

[tkchin_webrtc, 2016/03/08 04:51:41]

Yes, fixes a leak.

     decompression_session_ = nullptr;
   }
 }
 
 void H264VideoToolboxDecoder::SetVideoFormat(
     CMVideoFormatDescriptionRef video_format) {
   if (video_format_ == video_format) {
     return;
   }
   if (video_format_) {
     CFRelease(video_format_);
   }
   video_format_ = video_format;
   if (video_format_) {
     CFRetain(video_format_);
   }
 }
 
 const char* H264VideoToolboxDecoder::ImplementationName() const {
   return "VideoToolbox";
 }
 
 }  // namespace webrtc
 
 #endif  // defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)