Add metal view, shaders and renderer.

webrtc/sdk/objc/Framework/Classes/Metal/RTCNV12Renderer.mm

+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import "RTCNV12Renderer.h"
+
+#import <Metal/Metal.h>
+#import <MetalKit/MetalKit.h>
+
+#import <WebRTC/RTCMTLVideoView.h>
+#import "WebRTC/RTCLogging.h"
+#import "WebRTC/RTCVideoFrame.h"
+
+#include "webrtc/api/video/video_rotation.h"
+
+
+#define MTL_STRINGIFY(s) @#s
+
+static const float cubeVertexData[64] = {
+    -1.0, -1.0, 0.0, 1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0,
+
+    // rotation = 90, offset = 16.
+    -1.0, -1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 0.0, -1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0,
+
+    // rotation = 180, offset = 32.
+    -1.0, -1.0, 1.0, 0.0, 1.0, -1.0, 0.0, 0.0, -1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 1.0,
+
+    // rotation = 270, offset = 48.
+    -1.0, -1.0, 0.0, 0.0, 1.0, -1.0, 0.0, 1.0, -1.0, 1.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0,
+};
+
+static inline int offsetForRotation(webrtc::VideoRotation rotation) {
+  switch (rotation) {
+    case webrtc::kVideoRotation_0:
+      return 0;
+    case webrtc::kVideoRotation_90:
+      return 16;
+    case webrtc::kVideoRotation_180:
+      return 32;
+    case webrtc::kVideoRotation_270:
+      return 48;
+  }
+  return 0;
+}
+
+static NSString *const shaderSource = MTL_STRINGIFY(
+    using namespace metal; typedef struct {
+      packed_float2 position;
+      packed_float2 texcoord;
+    } Vertex;
+
+    typedef struct {
+      float4 position[[position]];
+      float2 texcoord;
+    } Varyings;
+
+    vertex Varyings vertexPassthrough(device Vertex * verticies[[buffer(0)]],
+                                      unsigned int vid[[vertex_id]]) {
+      Varyings out;
+      device Vertex &v = verticies[vid];
+      out.position = float4(float2(v.position), 0.0, 1.0);
+      out.texcoord = v.texcoord;
+
+      return out;
+    }
+
+    fragment half4 fragmentColorConversion(
+        Varyings in[[stage_in]], texture2d<float, access::sample> textureY[[texture(0)]],
+        texture2d<float, access::sample> textureCbCr[[texture(1)]]) {
+      constexpr sampler s(address::clamp_to_edge, filter::linear);
+      float y;
+      float2 uv;
+      y = textureY.sample(s, in.texcoord).r;
+      uv = textureCbCr.sample(s, in.texcoord).rg - float2(0.5, 0.5);
+
+      float4 out = float4(y + 1.403 * uv.y, y - 0.344 * uv.x - 0.714 * uv.y, y + 1.770 * uv.x, 1.0);
+
+      return half4(out);
+    });
+
+// The max number of command buffers in flight.
+// For now setting it up to 1.
+// In future we might use triple buffering method if it improves performance.
+
+static const NSInteger kMaxInflightBuffers = 1;
+
+@interface RTCNV12Renderer ()
+@end
+
+@implementation RTCNV12Renderer {
+  __kindof MTKView *_view;
+
+  // Controller.
+  dispatch_semaphore_t _inflight_semaphore;
+
+  // Renderer.
+  id<MTLDevice> _device;
+  id<MTLCommandQueue> _commandQueue;
+  id<MTLLibrary> _defaultLibrary;
+  id<MTLRenderPipelineState> _pipelineState;
+
+  // Textures.
+  CVMetalTextureCacheRef textureCache;
+  id<MTLTexture> yTexture;
+  id<MTLTexture> CrCbTexture;

[tkchin_webrtc, 2017/02/16 23:28:04]

nit  _textureCache, _yTexture: _crCbTexture

[daniela-webrtc, 2017/02/20 15:17:08]

Done.

+
+  // Buffers.
+  id<MTLBuffer> _vertexBuffer;
+
+  // RTC Frame parameters.
+  int offset;
+}
+
+- (instancetype)init {
+  if (self = [super init]) {
+    // Offset of 0 is equal to rotation of 0.
+    offset = 0;
+    _inflight_semaphore = dispatch_semaphore_create(kMaxInflightBuffers);
+  }
+
+  return self;
+}
+
+- (BOOL)addRenderingDestination:(__kindof MTKView *)view {
+  return [self setupWithView:view];
+}
+
+#pragma mark - Private
+
+- (BOOL)setupWithView:(__kindof MTKView *)view {
+  BOOL success = NO;
+  if ([self setupMetal]) {
+    [self setupView:view];
+    [self loadAssets];
+    [self setupBuffers];
+    [self initializeTextureCache];
+    success = YES;
+  }
+  return success;
+}
+
+#pragma mark - GPU methods
+
+- (BOOL)setupMetal {
+  // Set the view to use the default device.
+  _device = MTLCreateSystemDefaultDevice();
+  if (!_device) {
+    return NO;
+  }
+
+  // Create a new command queue.
+  _commandQueue = [_device newCommandQueue];
+
+  // Load metal library from source.
+  NSError *libraryError = nil;
+
+  id<MTLLibrary> sourceLibrary =
+      [_device newLibraryWithSource:shaderSource options:NULL error:&libraryError];
+
+  if (libraryError) {
+    RTCLog(@"Metal Error: Library with source failed\n%@", libraryError);

[tkchin_webrtc, 2017/02/16 23:28:04]

RTCLogError for errors. Here and elsewhere.

[daniela-webrtc, 2017/02/20 15:17:07]

Done.

+    return NO;
+  }
+
+  if (!sourceLibrary) {
+    RTCLog(@"Metal error: Failed to load library. %@", libraryError);
+    return NO;
+  }
+  _defaultLibrary = sourceLibrary;
+
+  return YES;
+}
+
+- (void)setupView:(__kindof MTKView *)view {
+  view.device = _device;
+
+  view.preferredFramesPerSecond = 30;
+  view.autoResizeDrawable = NO;
+
+  // We need to keep reference to the view as it's needed down the rendering pipeline.
+  _view = view;
+}
+
+- (void)loadAssets {
+  // As defined in shaderSource.
+  id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:@"vertexPassthrough"];
+  id<MTLFunction> fragmentFunction =
+      [_defaultLibrary newFunctionWithName:@"fragmentColorConversion"];
+
+  MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
+  pipelineDescriptor.label = @"Pipeline";
+  pipelineDescriptor.vertexFunction = vertexFunction;
+  pipelineDescriptor.fragmentFunction = fragmentFunction;
+  pipelineDescriptor.colorAttachments[0].pixelFormat = _view.colorPixelFormat;
+  pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid;
+  NSError *error = nil;
+  _pipelineState = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error];
+
+  if (!_pipelineState) {
+    RTCLog(@"Metal error: Failed to create pipeline state. %@", error);
+  }
+}
+
+- (void)setupBuffers {
+  _vertexBuffer = [_device newBufferWithBytes:cubeVertexData
+                                       length:sizeof(cubeVertexData)
+                                      options:MTLResourceOptionCPUCacheModeDefault];
+}
+
+- (void)initializeTextureCache {
+  CVReturn status =
+      CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, _device, nil, &textureCache);
+  if (status != kCVReturnSuccess) {
+    RTCLog(@"Metal error: Failed to initialize metal texture cache. Return status is %d", status);
+  }
+}
+
+- (void)render {
+  dispatch_semaphore_wait(_inflight_semaphore, DISPATCH_TIME_FOREVER);
+
+  id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
+  commandBuffer.label = @"RTCCommandBuffer";

[tkchin_webrtc, 2017/02/16 23:28:04]

stuff these constants into globals at the top of the file.
e.g
static NSString const *kRTCMTLNV12RendererCommandBufferLabel

[daniela-webrtc, 2017/02/20 15:17:07]

Done.

+
+  __block dispatch_semaphore_t block_semaphore = _inflight_semaphore;
+  [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> _Nonnull) {
+    dispatch_semaphore_signal(block_semaphore);
+  }];
+
+  MTLRenderPassDescriptor *_renderPassDescriptor = _view.currentRenderPassDescriptor;
+  if (_renderPassDescriptor) {  // Valid drawable.
+    id<MTLRenderCommandEncoder> renderEncoder =
+        [commandBuffer renderCommandEncoderWithDescriptor:_renderPassDescriptor];
+    renderEncoder.label = @"RTCEncoder";
+
+    // Set context state.
+    [renderEncoder pushDebugGroup:@"DrawFrame"];
+    [renderEncoder setRenderPipelineState:_pipelineState];
+    [renderEncoder setVertexBuffer:_vertexBuffer offset:offset * sizeof(float) atIndex:0];
+    [renderEncoder setFragmentTexture:yTexture atIndex:0];
+    [renderEncoder setFragmentTexture:CrCbTexture atIndex:1];
+
+    [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
+                      vertexStart:0
+                      vertexCount:4
+                    instanceCount:1];
+    [renderEncoder popDebugGroup];
+    [renderEncoder endEncoding];
+
+    [commandBuffer presentDrawable:_view.currentDrawable];
+  }
+
+  [commandBuffer commit];
+}
+
+#pragma mark - RTCMTLRenderer
+
+- (void)draw {
+  if (!yTexture && !CrCbTexture) {
+    NSLog(@"No current frame. Aborting drawinFrame:");

[tkchin_webrtc, 2017/02/16 23:28:04]

RTCLog, don't use NSLog (otherwise won't show up in WebRTC logstream)

[daniela-webrtc, 2017/02/20 15:17:07]

Done.

+    return;
+  }
+  @autoreleasepool {

[tkchin_webrtc, 2017/02/16 23:28:04]

why is an autoreleasepool required here?

+    [self render];
+  }
+}
+
+#pragma mark - RTCVideoRenderer
+
+- (void)setSize:(CGSize)size {

[tkchin_webrtc, 2017/02/16 23:28:04]

I think this is a legacy API fyi. It'd be nice to get rid of needing to have it,
but not required in this CL.

+  _view.drawableSize = size;
+  [_view draw];
+}
+
+- (void)renderFrame:(nullable RTCVideoFrame *)frame {
+  if (frame == NULL) {

[tkchin_webrtc, 2017/02/16 23:28:04]

use nil instead of NULL

for non-objC object pointers, use nullptr
for ObjC object pointers, use nil

[daniela-webrtc, 2017/02/20 15:17:07]

Done.

+    return;
+  }
+  [self setupSyncVariablesForFrame:frame];
+}
+
+- (void)setupSyncVariablesForFrame:(nonnull RTCVideoFrame *)frame {
+  CVPixelBufferRef pixelBuffer = frame.nativeHandle;
+
+  id<MTLTexture> lumaTexture = nil;
+  id<MTLTexture> chromaTexture = nil;
+  CVMetalTextureRef outTexture;

[tkchin_webrtc, 2017/02/16 23:28:04]

nit: initialize to nullptr

[daniela-webrtc, 2017/02/20 15:17:07]

Done.

+
+  // Luma (y) texture.
+  int lumaWidth = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);
+  int lumaHeight = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);
+
+  int indexPlane = 0;
+  CVReturn result = CVMetalTextureCacheCreateTextureFromImage(
+      kCFAllocatorDefault, textureCache, pixelBuffer, nil, MTLPixelFormatR8Unorm, lumaWidth,
+      lumaHeight, indexPlane, &outTexture);
+
+  if (result == kCVReturnSuccess) {
+    lumaTexture = CVMetalTextureGetTexture(outTexture);
+  }
+
+  CVBufferRelease(outTexture);

[tkchin_webrtc, 2017/02/16 23:28:04]

nit: consider using a guard anyway to make the code more explicit despite
CVBufferRelease being able to handle taking nullptr without crashing because
most people wouldn't know that without checking the doco. up to you.

[daniela-webrtc, 2017/02/20 15:17:07]

Added code comment to clarify.

+  outTexture = nil;

[tkchin_webrtc, 2017/02/16 23:28:04]

nullptr for refs

[daniela-webrtc, 2017/02/20 15:17:07]

Done.

+
+  // Chroma (CrCb) texture.
+  indexPlane = 1;
+  result = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer,
+                                                     nil, MTLPixelFormatRG8Unorm, lumaWidth / 2,
+                                                     lumaHeight / 2, indexPlane, &outTexture);
+  if (result == kCVReturnSuccess) {
+    chromaTexture = CVMetalTextureGetTexture(outTexture);
+  }
+  CVBufferRelease(outTexture);
+
+  if (lumaTexture != nil && chromaTexture != nil) {
+    dispatch_async(dispatch_get_main_queue(), ^{

[tkchin_webrtc, 2017/02/16 23:28:04]

seems expensive to dispatch at 30fps since this is called within renderFrame

Shouldn't we also be rendering at screen refresh frequency, as opposed to
decoder output frequency?

+      yTexture = lumaTexture;
+      CrCbTexture = chromaTexture;
+      offset = offsetForRotation((webrtc::VideoRotation)frame.rotation);
+    });
+  }
+}
+
+@end