Add metal view, shaders and renderer.

webrtc/sdk/objc/Framework/Classes/metal/RTCMTLRenderer.mm

+/*
+ *  Copyright 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import "WebRTC/RTCMTLRenderer.h"
+
+#import <Metal/Metal.h>
+#import <MetalKit/MetalKit.h>
+
+#import "WebRTC/RTCLogging.h"
+
+float cubeVertexData[64] =

[kthelgason, 2017/01/24 14:17:59]

static const

[daniela-webrtc, 2017/01/25 08:46:19]

Done.

+{
+  -1.0, -1.0,  0.0, 1.0,
+   1.0, -1.0,  1.0, 1.0,
+  -1.0,  1.0,  0.0, 0.0,
+   1.0,  1.0,  1.0, 0.0,
+
+  // rotation = 90, offset = 16.
+  -1.0, -1.0,  1.0, 1.0,
+   1.0, -1.0,  1.0, 0.0,
+  -1.0,  1.0,  0.0, 1.0,
+   1.0,  1.0,  0.0, 0.0,
+
+  // rotation = 180, offset = 32.
+  -1.0, -1.0, 1.0, 0.0,
+   1.0, -1.0, 0.0, 0.0,
+  -1.0,  1.0, 1.0, 1.0,
+   1.0,  1.0, 0.0, 1.0,
+
+  // rotation = 270, offset = 48.
+  -1.0, -1.0, 0.0, 0.0,
+   1.0, -1.0, 0.0, 1.0,
+  -1.0,  1.0, 1.0, 0.0,
+   1.0,  1.0, 1.0, 1.0,
+};
+
+static inline int offsetForRotation(int rotation) {
+  switch (rotation) {
+    case 0:
+      return 0;
+      break;
+    case 90:
+      return 16;
+      break;
+    case 180:
+      return 32;
+      break;
+    case 270:
+      return 48;
+      break;
+    default:
+      return 0;
+      break;
+  }
+}
+
+// The max number of command buffers in flight.
+// For now setting it up to 1.
+// In future we might opt in to use tripple buffering method if it improves performance.

[kthelgason, 2017/01/24 14:18:00]

I think this sentence could be made more concise:
In future we might use triple buffering if it improves performance.

[daniela-webrtc, 2017/01/25 08:46:19]

Done.

+
+static const NSInteger kMaxInflightBuffers = 1;
+
+@interface RTCMTLRenderer ()<MTKViewDelegate>
+@end
+
+@implementation RTCMTLRenderer {
+  __kindof MTKView *_view;
+
+  // Controller.
+  dispatch_semaphore_t _inflight_semaphore;
+
+  // Renderer.
+  id<MTLDevice> _device;
+  id<MTLCommandQueue> _commandQueue;
+  id<MTLLibrary> _defaultLibrary;
+  id<MTLRenderPipelineState> _pipelineState;
+
+  // Textures.
+  CVMetalTextureCacheRef textureCache;
+  id<MTLTexture> yTexture;
+  id<MTLTexture> CrCbTexture;
+
+  // Buffers.
+  id<MTLBuffer> _vertexBuffer;
+
+  // RTC Frame parameters.
+  int rotation;
+  int offset;
+}
+
+- (instancetype)initWithView:(__kindof MTKView *)view {
+  if (self = [super init]) {
+    _view = view;
+
+    // Offset of 0 is equal to rotation of 0.
+    rotation = 0;
+    offset = 0;
+    _inflight_semaphore = dispatch_semaphore_create(kMaxInflightBuffers);
+  }
+
+  return self;
+}
+
+- (BOOL)setup {
+  BOOL success = NO;
+  if ([self _setupMetal] && _device) {
+    [self _setupView];
+    [self _loadAssets];
+    [self _setupBuffers];
+    [self initializeTextureCache];
+    success = YES;
+  }
+  return success;
+}
+
+- (BOOL)_setupMetal {

[kthelgason, 2017/01/24 14:17:59]

We haven't really been doing the _methodName. For methods not part of public api
put them at the bottom of the file below a #pragma mark - private

[daniela-webrtc, 2017/01/25 08:46:19]

Yes it's not common. I saw this in Apple's  Metal examples, where they use the
_method to indicate low level methods involving the GPU. I think it's a good
practice to follow for Metal code because it makes it really obvious what
methods involve GPU.
Do you think adding a comment to clarify this would make it better? 

+  // Set the view to use the default device.
+  _device = MTLCreateSystemDefaultDevice();
+
+  // Create a new command queue.
+  _commandQueue = [_device newCommandQueue];
+
+  // Load all the shader files with a metal file extension in the project.
+  NSError *libraryError = nil;
+  NSString *libraryFile = [[NSBundle mainBundle] pathForResource:@"rtc_shaders" ofType:@"metallib"];
+  if (!libraryFile) {
+    RTCLog(@"Metal Error: library not found in main bundle.");
+    return NO;
+  }
+
+  _defaultLibrary = [_device newLibraryWithFile:libraryFile error:&libraryError];
+  if (!_defaultLibrary) {
+    RTCLog(@"Metal error: Failed to load library. %@", libraryError);
+    return NO;
+  }
+
+  return YES;
+}
+
+- (void)_setupView {
+  _view.device = _device;
+  _view.delegate = self;
+  // VERY VERY MUCHISIMO IMPORTANTE.

[kthelgason, 2017/01/24 14:18:00]

Maybe a less flamboyant comment? :D although I do like this

[daniela-webrtc, 2017/01/25 08:46:19]

Ooops! This one slipped :D was meant to be part of the hacking code. 

+  _view.contentMode = UIViewContentModeScaleAspectFit;
+
+  _view.preferredFramesPerSecond = 30;
+  _view.autoResizeDrawable = NO;
+}
+
+- (void)_loadAssets {
+  // As defined in Shaders.metal.
+  id<MTLFunction> vertexFunction = [_defaultLibrary newFunctionWithName:@"vertexPassthrough"];
+  id<MTLFunction> fragmentFunction =
+      [_defaultLibrary newFunctionWithName:@"fragmentColorConversion"];
+
+  MTLRenderPipelineDescriptor *pipelineDescriptor = [[MTLRenderPipelineDescriptor alloc] init];
+  pipelineDescriptor.label = @"Pipeline";
+  pipelineDescriptor.vertexFunction = vertexFunction;
+  pipelineDescriptor.fragmentFunction = fragmentFunction;
+  pipelineDescriptor.colorAttachments[0].pixelFormat = _view.colorPixelFormat;
+  pipelineDescriptor.depthAttachmentPixelFormat = MTLPixelFormatInvalid;
+  NSError *error = nil;
+  _pipelineState = [_device newRenderPipelineStateWithDescriptor:pipelineDescriptor error:&error];
+
+  if (!_pipelineState) {
+    RTCLog(@"Metal error: Failed to create pipeline state. %@", error);
+  }
+}
+
+- (void)_setupBuffers {
+  _vertexBuffer = [_device newBufferWithBytes:cubeVertexData
+                                       length:sizeof(cubeVertexData)
+                                      options:MTLResourceOptionCPUCacheModeDefault];
+}
+
+- (void)initializeTextureCache {
+  CVReturn status =
+      CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, _device, nil, &textureCache);
+  if (status != kCVReturnSuccess) {
+    RTCLog(@"Metal error: Falied to initialize metal texture cache. Return status is %d", status);

[magjed_webrtc, 2017/01/24 14:01:56]

spelling nit: Failed

[daniela-webrtc, 2017/01/25 08:46:19]

Done.

+  }
+}
+
+- (void)_render {
+  dispatch_semaphore_wait(_inflight_semaphore, DISPATCH_TIME_FOREVER);
+
+  id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
+  commandBuffer.label = @"RTCCommandBuffer";
+
+  __block dispatch_semaphore_t block_semaphore = _inflight_semaphore;
+  [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> _Nonnull) {
+    dispatch_semaphore_signal(block_semaphore);
+  }];
+
+  MTLRenderPassDescriptor *_renderPassDescriptor = _view.currentRenderPassDescriptor;
+  if (_renderPassDescriptor) {  // valid drawable.
+
+    id<MTLRenderCommandEncoder> renderEncoder =
+        [commandBuffer renderCommandEncoderWithDescriptor:_renderPassDescriptor];
+    renderEncoder.label = @"RTCEncoder";
+
+    // Set context state.
+    [renderEncoder pushDebugGroup:@"DrawFrame"];
+    [renderEncoder setRenderPipelineState:_pipelineState];
+    [renderEncoder setVertexBuffer:_vertexBuffer offset:sizeof(float[offset]) atIndex:0];
+    [renderEncoder setFragmentTexture:yTexture atIndex:0];
+    [renderEncoder setFragmentTexture:CrCbTexture atIndex:1];
+
+    [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip
+                      vertexStart:0
+                      vertexCount:4
+                    instanceCount:1];
+    [renderEncoder popDebugGroup];
+    [renderEncoder endEncoding];
+
+    [commandBuffer presentDrawable:_view.currentDrawable];
+  }
+
+  [commandBuffer commit];
+}
+
+#pragma mark - MTKViewDelegate
+
+- (void)drawInMTKView:(MTKView *)view {
+  if (!yTexture && !CrCbTexture) {
+    NSLog(@"No current frame. Aborting drawinFrame:");
+    return;
+  }
+  @autoreleasepool {
+    [self _render];
+  }
+}
+
+- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size {
+}
+
+#pragma mark - RTCVideoRenderer
+- (void)setSize:(CGSize)size {
+  _view.drawableSize = size;
+  [_view draw];
+}
+
+- (void)renderFrame:(nullable RTCVideoFrame *)frame {
+  if (frame == NULL) {
+    return;
+  }
+  [self setupSyncVariablesForFrame:frame];

[kthelgason, 2017/01/24 14:18:00]

I don't see the point of extracting this into a method. Can't it just be inlined
here?

[daniela-webrtc, 2017/01/25 08:46:19]

It can be inlined but it was my personal preference as I prefer separate methods
over code comments (for instance). I think it nicely conveys the intention that
this is part of the code dealing with synchronization of variables.

+}
+
+- (void)setupSyncVariablesForFrame:(nonnull RTCVideoFrame *)frame {
+  CVPixelBufferRef pixelBuffer = frame.nativeHandle;
+
+  id<MTLTexture> lumaTexture = nil;
+  id<MTLTexture> chromaTexture = nil;
+  CVMetalTextureRef outTexture;
+
+  // Luma (y) texture.
+  int lumaWidth = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);
+  int lumaHeight = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);
+
+  CVReturn result = CVMetalTextureCacheCreateTextureFromImage(
+      kCFAllocatorDefault, textureCache, pixelBuffer, nil, MTLPixelFormatR8Unorm, lumaWidth,
+      lumaHeight, 0, &outTexture);

[magjed_webrtc, 2017/01/24 14:01:56]

Maybe add a comment for the literal 0: /* indexPlane */?

[daniela-webrtc, 2017/01/25 08:46:19]

Good idea. I've actually added a separate variable int indexPlane that will have
values 0 and 1 respectively. I think that would make the intent even clearer.
Let me know what you think.

+
+  if (result == kCVReturnSuccess) {
+    lumaTexture = CVMetalTextureGetTexture(outTexture);
+  }
+
+  CVBufferRelease(outTexture);

[magjed_webrtc, 2017/01/24 14:01:56]

Is it ok to release |outTexture| here and keep using lumaTexture?

[daniela-webrtc, 2017/01/25 08:46:19]

Not sure I understand the question. Are you asking if lumaTexture will be
available after outTexture is released?
If that's the question the answer is yes. CVMetalTextyreGetTexture returns
retained object  (id<MTLTexture) that will be released when it gets out of scope
(i.e when this method returns). So it's perfectly safe to use it even after
outTexture has been released.

+  outTexture = nil;
+
+  // Chroma (CrCb) texture.
+  result = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer,
+                                                     nil, MTLPixelFormatRG8Unorm, lumaWidth / 2,
+                                                     lumaHeight / 2, 1, &outTexture);

[magjed_webrtc, 2017/01/24 14:01:56]

ditto: Maybe add a comment for the literal 1: /* indexPlane */?

[daniela-webrtc, 2017/01/25 08:46:19]

^

+  if (result == kCVReturnSuccess) {
+    chromaTexture = CVMetalTextureGetTexture(outTexture);
+  }
+  CVBufferRelease(outTexture);
+
+  if (lumaTexture != nil && chromaTexture != nil) {
+    dispatch_async(dispatch_get_main_queue(), ^{
+      yTexture = lumaTexture;
+      CrCbTexture = chromaTexture;
+      if (rotation != frame.rotation) {
+        rotation = frame.rotation;
+        offset = offsetForRotation(rotation);
+      }
+    });
+  }
+}
+
+- (nullable id<MTLTexture>)transformFrame:(RTCVideoFrame *)videoFrame

[magjed_webrtc, 2017/01/24 14:01:56]

Is this function used?

[daniela-webrtc, 2017/01/25 08:46:19]

Great catch. It was part of a previous implementation I made. Will remove it.

+                      toTextureIndexPlane:(int)indexPlane {
+  CVPixelBufferRef pixelBuffer = videoFrame.nativeHandle;
+
+  id<MTLTexture> mtlTexture = nil;
+  int width = CVPixelBufferGetWidthOfPlane(pixelBuffer, indexPlane);
+  int height = CVPixelBufferGetHeightOfPlane(pixelBuffer, indexPlane);
+
+  CVMetalTextureRef texture;
+
+  MTLPixelFormat format = MTLPixelFormatRG8Unorm;
+  if (indexPlane == 0) {
+    format = MTLPixelFormatR8Unorm;
+  }
+  CVReturn status =
+      CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil,
+                                                format, width, height, indexPlane, &texture);
+  if (status != kCVReturnSuccess) {
+    RTCLog(@"Metal errror: Failed to create metal texture for plane index %d", indexPlane);
+    return nil;
+  }
+
+  mtlTexture = CVMetalTextureGetTexture(texture);
+
+  // Same like CFRelease except it can be passed NULL without crashing.
+  CVBufferRelease(texture);
+  return mtlTexture;
+}
+@end