OLD | NEW |
| (Empty) |
1 /* | |
2 * Copyright 2015 The WebRTC project authors. All Rights Reserved. | |
3 * | |
4 * Use of this source code is governed by a BSD-style license | |
5 * that can be found in the LICENSE file in the root of the source | |
6 * tree. An additional intellectual property rights grant can be found | |
7 * in the file PATENTS. All contributing project authors may | |
8 * be found in the AUTHORS file in the root of the source tree. | |
9 */ | |
10 | |
11 #import "RTCOpenGLVideoRenderer.h" | |
12 | |
13 #include <string.h> | |
14 | |
15 #include "webrtc/base/scoped_ptr.h" | |
16 | |
17 #if TARGET_OS_IPHONE | |
18 #import <OpenGLES/ES3/gl.h> | |
19 #else | |
20 #import <OpenGL/gl3.h> | |
21 #endif | |
22 | |
23 #import "RTCVideoFrame.h" | |
24 | |
25 // TODO(tkchin): check and log openGL errors. Methods here return BOOLs in | |
26 // anticipation of that happening in the future. | |
27 | |
28 #if TARGET_OS_IPHONE | |
29 #define RTC_PIXEL_FORMAT GL_LUMINANCE | |
30 #define SHADER_VERSION | |
31 #define VERTEX_SHADER_IN "attribute" | |
32 #define VERTEX_SHADER_OUT "varying" | |
33 #define FRAGMENT_SHADER_IN "varying" | |
34 #define FRAGMENT_SHADER_OUT | |
35 #define FRAGMENT_SHADER_COLOR "gl_FragColor" | |
36 #define FRAGMENT_SHADER_TEXTURE "texture2D" | |
37 #else | |
38 #define RTC_PIXEL_FORMAT GL_RED | |
39 #define SHADER_VERSION "#version 150\n" | |
40 #define VERTEX_SHADER_IN "in" | |
41 #define VERTEX_SHADER_OUT "out" | |
42 #define FRAGMENT_SHADER_IN "in" | |
43 #define FRAGMENT_SHADER_OUT "out vec4 fragColor;\n" | |
44 #define FRAGMENT_SHADER_COLOR "fragColor" | |
45 #define FRAGMENT_SHADER_TEXTURE "texture" | |
46 #endif | |
47 | |
48 // Vertex shader doesn't do anything except pass coordinates through. | |
49 static const char kVertexShaderSource[] = | |
50 SHADER_VERSION | |
51 VERTEX_SHADER_IN " vec2 position;\n" | |
52 VERTEX_SHADER_IN " vec2 texcoord;\n" | |
53 VERTEX_SHADER_OUT " vec2 v_texcoord;\n" | |
54 "void main() {\n" | |
55 " gl_Position = vec4(position.x, position.y, 0.0, 1.0);\n" | |
56 " v_texcoord = texcoord;\n" | |
57 "}\n"; | |
58 | |
59 // Fragment shader converts YUV values from input textures into a final RGB | |
60 // pixel. The conversion formula is from http://www.fourcc.org/fccyvrgb.php. | |
61 static const char kFragmentShaderSource[] = | |
62 SHADER_VERSION | |
63 "precision highp float;" | |
64 FRAGMENT_SHADER_IN " vec2 v_texcoord;\n" | |
65 "uniform lowp sampler2D s_textureY;\n" | |
66 "uniform lowp sampler2D s_textureU;\n" | |
67 "uniform lowp sampler2D s_textureV;\n" | |
68 FRAGMENT_SHADER_OUT | |
69 "void main() {\n" | |
70 " float y, u, v, r, g, b;\n" | |
71 " y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n" | |
72 " u = " FRAGMENT_SHADER_TEXTURE "(s_textureU, v_texcoord).r;\n" | |
73 " v = " FRAGMENT_SHADER_TEXTURE "(s_textureV, v_texcoord).r;\n" | |
74 " u = u - 0.5;\n" | |
75 " v = v - 0.5;\n" | |
76 " r = y + 1.403 * v;\n" | |
77 " g = y - 0.344 * u - 0.714 * v;\n" | |
78 " b = y + 1.770 * u;\n" | |
79 " " FRAGMENT_SHADER_COLOR " = vec4(r, g, b, 1.0);\n" | |
80 " }\n"; | |
81 | |
82 // Compiles a shader of the given |type| with GLSL source |source| and returns | |
83 // the shader handle or 0 on error. | |
84 GLuint CreateShader(GLenum type, const GLchar *source) { | |
85 GLuint shader = glCreateShader(type); | |
86 if (!shader) { | |
87 return 0; | |
88 } | |
89 glShaderSource(shader, 1, &source, NULL); | |
90 glCompileShader(shader); | |
91 GLint compileStatus = GL_FALSE; | |
92 glGetShaderiv(shader, GL_COMPILE_STATUS, &compileStatus); | |
93 if (compileStatus == GL_FALSE) { | |
94 glDeleteShader(shader); | |
95 shader = 0; | |
96 } | |
97 return shader; | |
98 } | |
99 | |
100 // Links a shader program with the given vertex and fragment shaders and | |
101 // returns the program handle or 0 on error. | |
102 GLuint CreateProgram(GLuint vertexShader, GLuint fragmentShader) { | |
103 if (vertexShader == 0 || fragmentShader == 0) { | |
104 return 0; | |
105 } | |
106 GLuint program = glCreateProgram(); | |
107 if (!program) { | |
108 return 0; | |
109 } | |
110 glAttachShader(program, vertexShader); | |
111 glAttachShader(program, fragmentShader); | |
112 glLinkProgram(program); | |
113 GLint linkStatus = GL_FALSE; | |
114 glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); | |
115 if (linkStatus == GL_FALSE) { | |
116 glDeleteProgram(program); | |
117 program = 0; | |
118 } | |
119 return program; | |
120 } | |
121 | |
122 // When modelview and projection matrices are identity (default) the world is | |
123 // contained in the square around origin with unit size 2. Drawing to these | |
124 // coordinates is equivalent to drawing to the entire screen. The texture is | |
125 // stretched over that square using texture coordinates (u, v) that range | |
126 // from (0, 0) to (1, 1) inclusive. Texture coordinates are flipped vertically | |
127 // here because the incoming frame has origin in upper left hand corner but | |
128 // OpenGL expects origin in bottom left corner. | |
129 const GLfloat gVertices[] = { | |
130 // X, Y, U, V. | |
131 -1, -1, 0, 1, // Bottom left. | |
132 1, -1, 1, 1, // Bottom right. | |
133 1, 1, 1, 0, // Top right. | |
134 -1, 1, 0, 0, // Top left. | |
135 }; | |
136 | |
137 // |kNumTextures| must not exceed 8, which is the limit in OpenGLES2. Two sets | |
138 // of 3 textures are used here, one for each of the Y, U and V planes. Having | |
139 // two sets alleviates CPU blockage in the event that the GPU is asked to render | |
140 // to a texture that is already in use. | |
141 static const GLsizei kNumTextureSets = 2; | |
142 static const GLsizei kNumTextures = 3 * kNumTextureSets; | |
143 | |
144 @implementation RTCOpenGLVideoRenderer { | |
145 #if TARGET_OS_IPHONE | |
146 EAGLContext *_context; | |
147 #else | |
148 NSOpenGLContext *_context; | |
149 #endif | |
150 BOOL _isInitialized; | |
151 GLint _currentTextureSet; | |
152 // Handles for OpenGL constructs. | |
153 GLuint _textures[kNumTextures]; | |
154 GLuint _program; | |
155 #if !TARGET_OS_IPHONE | |
156 GLuint _vertexArray; | |
157 #endif | |
158 GLuint _vertexBuffer; | |
159 GLint _position; | |
160 GLint _texcoord; | |
161 GLint _ySampler; | |
162 GLint _uSampler; | |
163 GLint _vSampler; | |
164 // Used to create a non-padded plane for GPU upload when we receive padded | |
165 // frames. | |
166 rtc::scoped_ptr<uint8_t[]> _planeBuffer; | |
167 } | |
168 | |
169 @synthesize lastDrawnFrame = _lastDrawnFrame; | |
170 | |
171 + (void)initialize { | |
172 // Disable dithering for performance. | |
173 glDisable(GL_DITHER); | |
174 } | |
175 | |
176 #if TARGET_OS_IPHONE | |
177 - (instancetype)initWithContext:(EAGLContext *)context { | |
178 #else | |
179 - (instancetype)initWithContext:(NSOpenGLContext *)context { | |
180 #endif | |
181 NSAssert(context != nil, @"context cannot be nil"); | |
182 if (self = [super init]) { | |
183 _context = context; | |
184 } | |
185 return self; | |
186 } | |
187 | |
188 - (BOOL)drawFrame:(RTCVideoFrame *)frame { | |
189 if (!_isInitialized) { | |
190 return NO; | |
191 } | |
192 if (_lastDrawnFrame == frame) { | |
193 return NO; | |
194 } | |
195 [self ensureGLContext]; | |
196 glClear(GL_COLOR_BUFFER_BIT); | |
197 if (frame) { | |
198 if (![self updateTextureSizesForFrame:frame] || | |
199 ![self updateTextureDataForFrame:frame]) { | |
200 return NO; | |
201 } | |
202 #if !TARGET_OS_IPHONE | |
203 glBindVertexArray(_vertexArray); | |
204 #endif | |
205 glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); | |
206 glDrawArrays(GL_TRIANGLE_FAN, 0, 4); | |
207 } | |
208 #if !TARGET_OS_IPHONE | |
209 [_context flushBuffer]; | |
210 #endif | |
211 _lastDrawnFrame = frame; | |
212 return YES; | |
213 } | |
214 | |
215 - (void)setupGL { | |
216 if (_isInitialized) { | |
217 return; | |
218 } | |
219 [self ensureGLContext]; | |
220 if (![self setupProgram]) { | |
221 return; | |
222 } | |
223 if (![self setupTextures]) { | |
224 return; | |
225 } | |
226 if (![self setupVertices]) { | |
227 return; | |
228 } | |
229 glUseProgram(_program); | |
230 glPixelStorei(GL_UNPACK_ALIGNMENT, 1); | |
231 _isInitialized = YES; | |
232 } | |
233 | |
234 - (void)teardownGL { | |
235 if (!_isInitialized) { | |
236 return; | |
237 } | |
238 [self ensureGLContext]; | |
239 glDeleteProgram(_program); | |
240 _program = 0; | |
241 glDeleteTextures(kNumTextures, _textures); | |
242 glDeleteBuffers(1, &_vertexBuffer); | |
243 _vertexBuffer = 0; | |
244 #if !TARGET_OS_IPHONE | |
245 glDeleteVertexArrays(1, &_vertexArray); | |
246 #endif | |
247 _isInitialized = NO; | |
248 } | |
249 | |
250 #pragma mark - Private | |
251 | |
252 - (void)ensureGLContext { | |
253 NSAssert(_context, @"context shouldn't be nil"); | |
254 #if TARGET_OS_IPHONE | |
255 if ([EAGLContext currentContext] != _context) { | |
256 [EAGLContext setCurrentContext:_context]; | |
257 } | |
258 #else | |
259 if ([NSOpenGLContext currentContext] != _context) { | |
260 [_context makeCurrentContext]; | |
261 } | |
262 #endif | |
263 } | |
264 | |
265 - (BOOL)setupProgram { | |
266 NSAssert(!_program, @"program already set up"); | |
267 GLuint vertexShader = CreateShader(GL_VERTEX_SHADER, kVertexShaderSource); | |
268 NSAssert(vertexShader, @"failed to create vertex shader"); | |
269 GLuint fragmentShader = | |
270 CreateShader(GL_FRAGMENT_SHADER, kFragmentShaderSource); | |
271 NSAssert(fragmentShader, @"failed to create fragment shader"); | |
272 _program = CreateProgram(vertexShader, fragmentShader); | |
273 // Shaders are created only to generate program. | |
274 if (vertexShader) { | |
275 glDeleteShader(vertexShader); | |
276 } | |
277 if (fragmentShader) { | |
278 glDeleteShader(fragmentShader); | |
279 } | |
280 if (!_program) { | |
281 return NO; | |
282 } | |
283 _position = glGetAttribLocation(_program, "position"); | |
284 _texcoord = glGetAttribLocation(_program, "texcoord"); | |
285 _ySampler = glGetUniformLocation(_program, "s_textureY"); | |
286 _uSampler = glGetUniformLocation(_program, "s_textureU"); | |
287 _vSampler = glGetUniformLocation(_program, "s_textureV"); | |
288 if (_position < 0 || _texcoord < 0 || _ySampler < 0 || _uSampler < 0 || | |
289 _vSampler < 0) { | |
290 return NO; | |
291 } | |
292 return YES; | |
293 } | |
294 | |
295 - (BOOL)setupTextures { | |
296 glGenTextures(kNumTextures, _textures); | |
297 // Set parameters for each of the textures we created. | |
298 for (GLsizei i = 0; i < kNumTextures; i++) { | |
299 glActiveTexture(GL_TEXTURE0 + i); | |
300 glBindTexture(GL_TEXTURE_2D, _textures[i]); | |
301 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); | |
302 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); | |
303 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); | |
304 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); | |
305 } | |
306 return YES; | |
307 } | |
308 | |
309 - (BOOL)updateTextureSizesForFrame:(RTCVideoFrame *)frame { | |
310 if (frame.height == _lastDrawnFrame.height && | |
311 frame.width == _lastDrawnFrame.width && | |
312 frame.chromaWidth == _lastDrawnFrame.chromaWidth && | |
313 frame.chromaHeight == _lastDrawnFrame.chromaHeight) { | |
314 return YES; | |
315 } | |
316 GLsizei lumaWidth = static_cast<GLsizei>(frame.width); | |
317 GLsizei lumaHeight = static_cast<GLsizei>(frame.height); | |
318 GLsizei chromaWidth = static_cast<GLsizei>(frame.chromaWidth); | |
319 GLsizei chromaHeight = static_cast<GLsizei>(frame.chromaHeight); | |
320 for (GLint i = 0; i < kNumTextureSets; i++) { | |
321 glActiveTexture(GL_TEXTURE0 + i * 3); | |
322 glTexImage2D(GL_TEXTURE_2D, | |
323 0, | |
324 RTC_PIXEL_FORMAT, | |
325 lumaWidth, | |
326 lumaHeight, | |
327 0, | |
328 RTC_PIXEL_FORMAT, | |
329 GL_UNSIGNED_BYTE, | |
330 0); | |
331 glActiveTexture(GL_TEXTURE0 + i * 3 + 1); | |
332 glTexImage2D(GL_TEXTURE_2D, | |
333 0, | |
334 RTC_PIXEL_FORMAT, | |
335 chromaWidth, | |
336 chromaHeight, | |
337 0, | |
338 RTC_PIXEL_FORMAT, | |
339 GL_UNSIGNED_BYTE, | |
340 0); | |
341 glActiveTexture(GL_TEXTURE0 + i * 3 + 2); | |
342 glTexImage2D(GL_TEXTURE_2D, | |
343 0, | |
344 RTC_PIXEL_FORMAT, | |
345 chromaWidth, | |
346 chromaHeight, | |
347 0, | |
348 RTC_PIXEL_FORMAT, | |
349 GL_UNSIGNED_BYTE, | |
350 0); | |
351 } | |
352 if ((NSUInteger)frame.yPitch != frame.width || | |
353 (NSUInteger)frame.uPitch != frame.chromaWidth || | |
354 (NSUInteger)frame.vPitch != frame.chromaWidth) { | |
355 _planeBuffer.reset(new uint8_t[frame.width * frame.height]); | |
356 } else { | |
357 _planeBuffer.reset(); | |
358 } | |
359 return YES; | |
360 } | |
361 | |
362 - (void)uploadPlane:(const uint8_t *)plane | |
363 sampler:(GLint)sampler | |
364 offset:(GLint)offset | |
365 width:(size_t)width | |
366 height:(size_t)height | |
367 stride:(int32_t)stride { | |
368 glActiveTexture(static_cast<GLenum>(GL_TEXTURE0 + offset)); | |
369 // When setting texture sampler uniforms, the texture index is used not | |
370 // the texture handle. | |
371 glUniform1i(sampler, offset); | |
372 #if TARGET_OS_IPHONE | |
373 BOOL hasUnpackRowLength = _context.API == kEAGLRenderingAPIOpenGLES3; | |
374 #else | |
375 BOOL hasUnpackRowLength = YES; | |
376 #endif | |
377 const uint8_t *uploadPlane = plane; | |
378 if ((size_t)stride != width) { | |
379 if (hasUnpackRowLength) { | |
380 // GLES3 allows us to specify stride. | |
381 glPixelStorei(GL_UNPACK_ROW_LENGTH, stride); | |
382 glTexImage2D(GL_TEXTURE_2D, | |
383 0, | |
384 RTC_PIXEL_FORMAT, | |
385 static_cast<GLsizei>(width), | |
386 static_cast<GLsizei>(height), | |
387 0, | |
388 RTC_PIXEL_FORMAT, | |
389 GL_UNSIGNED_BYTE, | |
390 uploadPlane); | |
391 glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); | |
392 return; | |
393 } else { | |
394 // Make an unpadded copy and upload that instead. Quick profiling showed | |
395 // that this is faster than uploading row by row using glTexSubImage2D. | |
396 uint8_t *unpaddedPlane = _planeBuffer.get(); | |
397 for (size_t y = 0; y < height; ++y) { | |
398 memcpy(unpaddedPlane + y * width, plane + y * stride, width); | |
399 } | |
400 uploadPlane = unpaddedPlane; | |
401 } | |
402 } | |
403 glTexImage2D(GL_TEXTURE_2D, | |
404 0, | |
405 RTC_PIXEL_FORMAT, | |
406 static_cast<GLsizei>(width), | |
407 static_cast<GLsizei>(height), | |
408 0, | |
409 RTC_PIXEL_FORMAT, | |
410 GL_UNSIGNED_BYTE, | |
411 uploadPlane); | |
412 } | |
413 | |
414 - (BOOL)updateTextureDataForFrame:(RTCVideoFrame *)frame { | |
415 GLint textureOffset = _currentTextureSet * 3; | |
416 NSAssert(textureOffset + 3 <= kNumTextures, @"invalid offset"); | |
417 | |
418 [self uploadPlane:frame.yPlane | |
419 sampler:_ySampler | |
420 offset:textureOffset | |
421 width:frame.width | |
422 height:frame.height | |
423 stride:frame.yPitch]; | |
424 | |
425 [self uploadPlane:frame.uPlane | |
426 sampler:_uSampler | |
427 offset:textureOffset + 1 | |
428 width:frame.chromaWidth | |
429 height:frame.chromaHeight | |
430 stride:frame.uPitch]; | |
431 | |
432 [self uploadPlane:frame.vPlane | |
433 sampler:_vSampler | |
434 offset:textureOffset + 2 | |
435 width:frame.chromaWidth | |
436 height:frame.chromaHeight | |
437 stride:frame.vPitch]; | |
438 | |
439 _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets; | |
440 return YES; | |
441 } | |
442 | |
443 - (BOOL)setupVertices { | |
444 #if !TARGET_OS_IPHONE | |
445 NSAssert(!_vertexArray, @"vertex array already set up"); | |
446 glGenVertexArrays(1, &_vertexArray); | |
447 if (!_vertexArray) { | |
448 return NO; | |
449 } | |
450 glBindVertexArray(_vertexArray); | |
451 #endif | |
452 NSAssert(!_vertexBuffer, @"vertex buffer already set up"); | |
453 glGenBuffers(1, &_vertexBuffer); | |
454 if (!_vertexBuffer) { | |
455 #if !TARGET_OS_IPHONE | |
456 glDeleteVertexArrays(1, &_vertexArray); | |
457 _vertexArray = 0; | |
458 #endif | |
459 return NO; | |
460 } | |
461 glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); | |
462 glBufferData(GL_ARRAY_BUFFER, sizeof(gVertices), gVertices, GL_DYNAMIC_DRAW); | |
463 | |
464 // Read position attribute from |gVertices| with size of 2 and stride of 4 | |
465 // beginning at the start of the array. The last argument indicates offset | |
466 // of data within |gVertices| as supplied to the vertex buffer. | |
467 glVertexAttribPointer( | |
468 _position, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (void *)0); | |
469 glEnableVertexAttribArray(_position); | |
470 | |
471 // Read texcoord attribute from |gVertices| with size of 2 and stride of 4 | |
472 // beginning at the first texcoord in the array. The last argument indicates | |
473 // offset of data within |gVertices| as supplied to the vertex buffer. | |
474 glVertexAttribPointer(_texcoord, | |
475 2, | |
476 GL_FLOAT, | |
477 GL_FALSE, | |
478 4 * sizeof(GLfloat), | |
479 (void *)(2 * sizeof(GLfloat))); | |
480 glEnableVertexAttribArray(_texcoord); | |
481 | |
482 return YES; | |
483 } | |
484 | |
485 @end | |
OLD | NEW |