iOS: Add support for rendering native CVPixelBuffers directly

webrtc/sdk/objc/Framework/Classes/RTCI420Shader.mm

+/*
+ *  Copyright 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#import "RTCShader.h"
+
+#include <vector>
+
+#import "RTCShader+Private.h"
+#import "WebRTC/RTCVideoFrame.h"
+
+// |kNumTextures| must not exceed 8, which is the limit in OpenGLES2. Two sets
+// of 3 textures are used here, one for each of the Y, U and V planes. Having
+// two sets alleviates CPU blockage in the event that the GPU is asked to render
+// to a texture that is already in use.
+static const GLsizei kNumTextureSets = 2;
+static const GLsizei kNumTextures = 3 * kNumTextureSets;

[sakal, 2016/08/03 10:52:59]

I would prefer 3 be a constant like kNumTexturesPerSet.

[tkchin_webrtc, 2016/08/03 17:53:19]

Done.

+
+// Fragment shader converts YUV values from input textures into a final RGB
+// pixel. The conversion formula is from http://www.fourcc.org/fccyvrgb.php.
+static const char kI420FragmentShaderSource[] =
+  SHADER_VERSION
+  "precision highp float;"
+  FRAGMENT_SHADER_IN " vec2 v_texcoord;\n"
+  "uniform lowp sampler2D s_textureY;\n"
+  "uniform lowp sampler2D s_textureU;\n"
+  "uniform lowp sampler2D s_textureV;\n"
+  FRAGMENT_SHADER_OUT
+  "void main() {\n"
+  "    float y, u, v, r, g, b;\n"
+  "    y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n"
+  "    u = " FRAGMENT_SHADER_TEXTURE "(s_textureU, v_texcoord).r;\n"
+  "    v = " FRAGMENT_SHADER_TEXTURE "(s_textureV, v_texcoord).r;\n"
+  "    u = u - 0.5;\n"
+  "    v = v - 0.5;\n"
+  "    r = y + 1.403 * v;\n"
+  "    g = y - 0.344 * u - 0.714 * v;\n"
+  "    b = y + 1.770 * u;\n"
+  "    " FRAGMENT_SHADER_COLOR " = vec4(r, g, b, 1.0);\n"
+  "  }\n";
+
+@implementation RTCI420Shader {
+  BOOL _hasUnpackRowLength;
+  GLint _currentTextureSet;
+  // Handles for OpenGL constructs.
+  GLuint _textures[kNumTextures];
+  GLuint _i420Program;
+  GLuint _vertexArray;
+  GLuint _vertexBuffer;
+  GLint _ySampler;
+  GLint _uSampler;
+  GLint _vSampler;
+  // Used to create a non-padded plane for GPU upload when we receive padded
+  // frames.
+  std::vector<uint8_t> _planeBuffer;
+}
+
+- (instancetype)initWithContext:(GlContextType *)context {
+  if (self = [super init]) {
+#if TARGET_OS_IPHONE
+    _hasUnpackRowLength = (context.API == kEAGLRenderingAPIOpenGLES3);
+#else
+    _hasUnpackRowLength = YES;
+#endif
+    glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
+    if (![self setupI420Program] || ![self setupTextures] ||
+        !RTCSetupVerticesForProgram(_i420Program, &_vertexBuffer, &_vertexArray)) {
+      self = nil;
+    }
+  }
+  return self;
+}
+
+- (void)dealloc {
+  if (_i420Program != 0) {

[sakal, 2016/08/03 10:52:59]

These if statements are unnecessary. OpenGL silently ignores trying to delete
something with index 0.

[tkchin_webrtc, 2016/08/03 17:53:19]

Done.

+    glDeleteProgram(_i420Program);
+  }
+  if (_textures[0] != 0) {
+    glDeleteTextures(kNumTextures, _textures);
+  }
+  if (_vertexBuffer != 0) {
+    glDeleteBuffers(1, &_vertexBuffer);
+  }
+  if (_vertexArray != 0) {
+    glDeleteVertexArrays(1, &_vertexArray);
+  }
+}
+
+- (BOOL)setupI420Program {
+  _i420Program = RTCCreateProgramFromFragmentSource(kI420FragmentShaderSource);
+  if (!_i420Program) {
+    return NO;
+  }
+  _ySampler = glGetUniformLocation(_i420Program, "s_textureY");
+  _uSampler = glGetUniformLocation(_i420Program, "s_textureU");
+  _vSampler = glGetUniformLocation(_i420Program, "s_textureV");
+
+  return (_ySampler >= 0 && _uSampler >= 0 && _vSampler >= 0);
+}
+
+- (BOOL)setupTextures {
+  glGenTextures(kNumTextures, _textures);
+  // Set parameters for each of the textures we created.
+  for (GLsizei i = 0; i < kNumTextures; i++) {
+    glActiveTexture(GL_TEXTURE0 + i);

[sakal, 2016/08/03 10:52:59]

I don't think this is needed. We can just set up the textures using any active
texture.

[tkchin_webrtc, 2016/08/03 17:53:19]

Done.

+    glBindTexture(GL_TEXTURE_2D, _textures[i]);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+  }
+  return YES;
+}
+
+- (BOOL)drawFrame:(RTCVideoFrame*)frame {
+  glUseProgram(_i420Program);
+  if (![self updateTextureDataForFrame:frame]) {
+    return NO;
+  }
+#if !TARGET_OS_IPHONE
+  glBindVertexArray(_vertexArray);
+#endif
+  glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer);
+  glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
+
+  return YES;
+}
+
+- (void)uploadPlane:(const uint8_t *)plane
+            sampler:(GLint)sampler
+             offset:(GLint)offset
+              width:(size_t)width
+             height:(size_t)height
+             stride:(int32_t)stride {
+  glActiveTexture(static_cast<GLenum>(GL_TEXTURE0 + offset));

[sakal, 2016/08/03 10:52:59]

Is it not possible to just always use GL_TEXTURE0-2 or does that cause CPU
blockage? If we only used GL_TEXTURE0-2 that would allow kNumTextures be over 8,
right?

[tkchin_webrtc, 2016/08/03 17:53:19]

See comment at top of file.

+  glBindTexture(GL_TEXTURE_2D, _textures[offset]);
+
+  // When setting texture sampler uniforms, the texture index is used not
+  // the texture handle.
+  glUniform1i(sampler, offset);
+  const uint8_t *uploadPlane = plane;
+  if ((size_t)stride != width) {
+   if (_hasUnpackRowLength) {
+      // GLES3 allows us to specify stride.
+      glPixelStorei(GL_UNPACK_ROW_LENGTH, stride);
+      glTexImage2D(GL_TEXTURE_2D,
+                   0,
+                   RTC_PIXEL_FORMAT,
+                   static_cast<GLsizei>(width),
+                   static_cast<GLsizei>(height),
+                   0,
+                   RTC_PIXEL_FORMAT,
+                   GL_UNSIGNED_BYTE,
+                   uploadPlane);
+      glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
+      return;
+    } else {
+      // Make an unpadded copy and upload that instead. Quick profiling showed
+      // that this is faster than uploading row by row using glTexSubImage2D.
+      uint8_t *unpaddedPlane = _planeBuffer.data();
+      for (size_t y = 0; y < height; ++y) {
+        memcpy(unpaddedPlane + y * width, plane + y * stride, width);
+      }
+      uploadPlane = unpaddedPlane;
+    }
+  }
+  glTexImage2D(GL_TEXTURE_2D,
+               0,
+               RTC_PIXEL_FORMAT,
+               static_cast<GLsizei>(width),
+               static_cast<GLsizei>(height),
+               0,
+               RTC_PIXEL_FORMAT,
+               GL_UNSIGNED_BYTE,
+               uploadPlane);
+}
+
+- (BOOL)updateTextureDataForFrame:(RTCVideoFrame *)frame {
+  GLint textureOffset = _currentTextureSet * 3;
+  NSAssert(textureOffset + 3 <= kNumTextures, @"invalid offset");
+
+  if (frame.yPitch != static_cast<int32_t>(frame.width) ||
+      frame.uPitch != static_cast<int32_t>(frame.chromaWidth) ||
+      frame.vPitch != static_cast<int32_t>(frame.chromaWidth)) {
+    _planeBuffer.resize(frame.width * frame.height);
+  }
+
+  [self uploadPlane:frame.yPlane
+            sampler:_ySampler
+             offset:textureOffset
+              width:frame.width
+             height:frame.height
+             stride:frame.yPitch];
+
+  [self uploadPlane:frame.uPlane
+            sampler:_uSampler
+             offset:textureOffset + 1
+              width:frame.chromaWidth
+             height:frame.chromaHeight
+             stride:frame.uPitch];
+
+  [self uploadPlane:frame.vPlane
+            sampler:_vSampler
+             offset:textureOffset + 2
+              width:frame.chromaWidth
+             height:frame.chromaHeight
+             stride:frame.vPitch];
+
+  _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets;
+  return YES;
+}
+
+@end