iOS/MacOS:Refactor metal rendering by extracting common implementation

webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer.mm

Index: webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer.mm
diff --git a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer.mm
similarity index 65%
copy from webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm
copy to webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer.mm
index 45e5905ec808718a7c9248edbdc438788dd4b637..bd07eb21558ff67bf49e5c51e692d95264a6e41e 100644
--- a/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLNV12Renderer.mm
+++ b/webrtc/sdk/objc/Framework/Classes/Metal/RTCMTLRenderer.mm
@@ -8,7 +8,7 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#import "RTCMTLNV12Renderer.h"
+#import "RTCMTLRenderer+Private.h"
 
 #import <Metal/Metal.h>
 #import <MetalKit/MetalKit.h>
@@ -17,8 +17,7 @@
 #import "WebRTC/RTCVideoFrame.h"
 
 #include "webrtc/api/video/video_rotation.h"
-
-#define MTL_STRINGIFY(s) @ #s
+#include "webrtc/base/checks.h"
 
 // As defined in shaderSource.
 static NSString *const vertexFunctionName = @"vertexPassthrough";
@@ -56,49 +55,12 @@ static inline int offsetForRotation(webrtc::VideoRotation rotation) {
   return 0;
 }
 
-static NSString *const shaderSource = MTL_STRINGIFY(
-    using namespace metal; typedef struct {
-      packed_float2 position;
-      packed_float2 texcoord;
-    } Vertex;
-
-    typedef struct {
-      float4 position[[position]];
-      float2 texcoord;
-    } Varyings;
-
-    vertex Varyings vertexPassthrough(device Vertex * verticies[[buffer(0)]],
-                                      unsigned int vid[[vertex_id]]) {
-      Varyings out;
-      device Vertex &v = verticies[vid];
-      out.position = float4(float2(v.position), 0.0, 1.0);
-      out.texcoord = v.texcoord;
-
-      return out;
-    }
-
-    // Receiving YCrCb textures.
-    fragment half4 fragmentColorConversion(
-        Varyings in[[stage_in]], texture2d<float, access::sample> textureY[[texture(0)]],
-        texture2d<float, access::sample> textureCbCr[[texture(1)]]) {
-      constexpr sampler s(address::clamp_to_edge, filter::linear);
-      float y;
-      float2 uv;
-      y = textureY.sample(s, in.texcoord).r;
-      uv = textureCbCr.sample(s, in.texcoord).rg - float2(0.5, 0.5);
-
-      // Conversion for YUV to rgb from http://www.fourcc.org/fccyvrgb.php
-      float4 out = float4(y + 1.403 * uv.y, y - 0.344 * uv.x - 0.714 * uv.y, y + 1.770 * uv.x, 1.0);
-
-      return half4(out);
-    });
-
 // The max number of command buffers in flight (submitted to GPU).
 // For now setting it up to 1.
 // In future we might use triple buffering method if it improves performance.
 static const NSInteger kMaxInflightBuffers = 1;
 
-@implementation RTCMTLNV12Renderer {
+@implementation RTCMTLRenderer {
   __kindof MTKView *_view;
 
   // Controller.
@@ -110,11 +72,6 @@ static const NSInteger kMaxInflightBuffers = 1;
   id<MTLLibrary> _defaultLibrary;
   id<MTLRenderPipelineState> _pipelineState;
 
-  // Textures.
-  CVMetalTextureCacheRef _textureCache;
-  id<MTLTexture> _yTexture;
-  id<MTLTexture> _CrCbTexture;
-
   // Buffers.
   id<MTLBuffer> _vertexBuffer;
 
@@ -144,11 +101,29 @@ static const NSInteger kMaxInflightBuffers = 1;
     [self setupView:view];
     [self loadAssets];
     [self setupBuffers];
-    [self initializeTextureCache];
     success = YES;
   }
   return success;
 }
+#pragma mark - Inheritance
+
+- (id<MTLDevice>)currentMetalDevice {
+  return _device;
+}
+
+- (NSString *)shaderSource {
+  RTC_CHECK(0) << "Virtual method not implemented in subclass.";

[magjed_webrtc, 2017/04/04 08:07:36]

nit: Use RTC_NOTREACHED instead.

[daniela-webrtc, 2017/04/05 11:49:31]

Done.

+  return nil;
+}
+
+- (void)uploadTexturesToRenderEncoder:(id<MTLRenderCommandEncoder>)renderEncoder {
+  RTC_CHECK(0) << "Virtual method not implemented in subclass.";
+}
+
+- (BOOL)setupTexturesForFrame:(nonnull RTCVideoFrame *)frame {
+  _offset = offsetForRotation((webrtc::VideoRotation)frame.rotation);

[magjed_webrtc, 2017/04/04 08:07:36]

Use static_cast<webrtc::VideoRotation> instead. Or we should possibly introduce
convert functions in RTCVideoFrame for RTCVideoRotation <->
webrtc::VideoRotation that switches on the enum values.

[daniela-webrtc, 2017/04/05 11:49:31]

Actually I'll use RTCVideoRotation directly instead of webrtc::VideoRotation.
This was done (I think) before RTCVideoRotation was added.
Great catch!

+  return YES;
+}
 
 #pragma mark - GPU methods
 
@@ -166,7 +141,7 @@ static const NSInteger kMaxInflightBuffers = 1;
   NSError *libraryError = nil;
 
   id<MTLLibrary> sourceLibrary =
-      [_device newLibraryWithSource:shaderSource options:NULL error:&libraryError];
+      [_device newLibraryWithSource:[self shaderSource] options:NULL error:&libraryError];
 
   if (libraryError) {
     RTCLogError(@"Metal: Library with source failed\n%@", libraryError);
@@ -194,8 +169,7 @@ static const NSInteger kMaxInflightBuffers = 1;
 
 - (void)loadAssets {
   id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:vertexFunctionName];
-  id<MTLFunction> fragmentFunction =
-      [_defaultLibrary newFunctionWithName:fragmentFunctionName];
+  id<MTLFunction> fragmentFunction = [_defaultLibrary newFunctionWithName:fragmentFunctionName];
 
   MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
   pipelineDescriptor.label = pipelineDescriptorLabel;
@@ -217,14 +191,6 @@ static const NSInteger kMaxInflightBuffers = 1;
                                       options:MTLResourceOptionCPUCacheModeDefault];
 }
 
-- (void)initializeTextureCache {
-  CVReturn status =
-      CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, _device, nil, &_textureCache);
-  if (status != kCVReturnSuccess) {
-    RTCLogError(@"Metal: Failed to initialize metal texture cache. Return status is %d", status);
-  }
-}
-
 - (void)render {
   // Wait until the inflight (curently sent to GPU) command buffer
   // has completed the GPU work.
@@ -249,8 +215,7 @@ static const NSInteger kMaxInflightBuffers = 1;
     [renderEncoder pushDebugGroup:renderEncoderDebugGroup];
     [renderEncoder setRenderPipelineState:_pipelineState];
     [renderEncoder setVertexBuffer:_vertexBuffer offset:_offset * sizeof(float) atIndex:0];
-    [renderEncoder setFragmentTexture:_yTexture atIndex:0];
-    [renderEncoder setFragmentTexture:_CrCbTexture atIndex:1];
+    [self uploadTexturesToRenderEncoder:renderEncoder];
 
     [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
                       vertexStart:0
@@ -270,52 +235,10 @@ static const NSInteger kMaxInflightBuffers = 1;
 
 - (void)drawFrame:(RTCVideoFrame *)frame {
   @autoreleasepool {
-    if ([self setupTexturesForFrame:frame])
+    if ([self setupTexturesForFrame:frame]) {
       [self render];
+    }
   }
 }
 
-- (BOOL)setupTexturesForFrame:(nonnull RTCVideoFrame *)frame {
-  CVPixelBufferRef pixelBuffer = frame.nativeHandle;
-
-  id<MTLTexture> lumaTexture = nil;
-  id<MTLTexture> chromaTexture = nil;
-  CVMetalTextureRef outTexture = nullptr;
-
-  // Luma (y) texture.
-  int lumaWidth = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);
-  int lumaHeight = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);
-
-  int indexPlane = 0;
-  CVReturn result = CVMetalTextureCacheCreateTextureFromImage(
-      kCFAllocatorDefault, _textureCache, pixelBuffer, nil, MTLPixelFormatR8Unorm, lumaWidth,
-      lumaHeight, indexPlane, &outTexture);
-
-  if (result == kCVReturnSuccess) {
-    lumaTexture = CVMetalTextureGetTexture(outTexture);
-  }
-
-  // Same as CFRelease except it can be passed NULL without crashing.
-  CVBufferRelease(outTexture);
-  outTexture = nullptr;
-
-  // Chroma (CrCb) texture.
-  indexPlane = 1;
-  result = CVMetalTextureCacheCreateTextureFromImage(
-      kCFAllocatorDefault, _textureCache, pixelBuffer, nil, MTLPixelFormatRG8Unorm, lumaWidth / 2,
-      lumaHeight / 2, indexPlane, &outTexture);
-  if (result == kCVReturnSuccess) {
-    chromaTexture = CVMetalTextureGetTexture(outTexture);
-  }
-  CVBufferRelease(outTexture);
-
-  if (lumaTexture != nil && chromaTexture != nil) {
-    _yTexture = lumaTexture;
-    _CrCbTexture = chromaTexture;
-    _offset = offsetForRotation((webrtc::VideoRotation)frame.rotation);
-    return YES;
-  }
-  return NO;
-}
-
 @end