OLD | NEW |
| (Empty) |
1 /* | |
2 * libjingle | |
3 * Copyright 2014 Google Inc. | |
4 * | |
5 * Redistribution and use in source and binary forms, with or without | |
6 * modification, are permitted provided that the following conditions are met: | |
7 * | |
8 * 1. Redistributions of source code must retain the above copyright notice, | |
9 * this list of conditions and the following disclaimer. | |
10 * 2. Redistributions in binary form must reproduce the above copyright notice, | |
11 * this list of conditions and the following disclaimer in the documentation | |
12 * and/or other materials provided with the distribution. | |
13 * 3. The name of the author may not be used to endorse or promote products | |
14 * derived from this software without specific prior written permission. | |
15 * | |
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED | |
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF | |
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO | |
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, | |
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; | |
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR | |
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF | |
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
26 */ | |
27 | |
28 #if !defined(__has_feature) || !__has_feature(objc_arc) | |
29 #error "This file requires ARC support." | |
30 #endif | |
31 | |
32 #import "RTCOpenGLVideoRenderer.h" | |
33 | |
34 #include <string.h> | |
35 | |
36 #include <memory> | |
37 | |
38 #if TARGET_OS_IPHONE | |
39 #import <OpenGLES/ES3/gl.h> | |
40 #else | |
41 #import <OpenGL/gl3.h> | |
42 #endif | |
43 | |
44 #import "RTCI420Frame.h" | |
45 | |
46 // TODO(tkchin): check and log openGL errors. Methods here return BOOLs in | |
47 // anticipation of that happening in the future. | |
48 | |
49 #if TARGET_OS_IPHONE | |
50 #define RTC_PIXEL_FORMAT GL_LUMINANCE | |
51 #define SHADER_VERSION | |
52 #define VERTEX_SHADER_IN "attribute" | |
53 #define VERTEX_SHADER_OUT "varying" | |
54 #define FRAGMENT_SHADER_IN "varying" | |
55 #define FRAGMENT_SHADER_OUT | |
56 #define FRAGMENT_SHADER_COLOR "gl_FragColor" | |
57 #define FRAGMENT_SHADER_TEXTURE "texture2D" | |
58 #else | |
59 #define RTC_PIXEL_FORMAT GL_RED | |
60 #define SHADER_VERSION "#version 150\n" | |
61 #define VERTEX_SHADER_IN "in" | |
62 #define VERTEX_SHADER_OUT "out" | |
63 #define FRAGMENT_SHADER_IN "in" | |
64 #define FRAGMENT_SHADER_OUT "out vec4 fragColor;\n" | |
65 #define FRAGMENT_SHADER_COLOR "fragColor" | |
66 #define FRAGMENT_SHADER_TEXTURE "texture" | |
67 #endif | |
68 | |
69 // Vertex shader doesn't do anything except pass coordinates through. | |
70 static const char kVertexShaderSource[] = | |
71 SHADER_VERSION | |
72 VERTEX_SHADER_IN " vec2 position;\n" | |
73 VERTEX_SHADER_IN " vec2 texcoord;\n" | |
74 VERTEX_SHADER_OUT " vec2 v_texcoord;\n" | |
75 "void main() {\n" | |
76 " gl_Position = vec4(position.x, position.y, 0.0, 1.0);\n" | |
77 " v_texcoord = texcoord;\n" | |
78 "}\n"; | |
79 | |
80 // Fragment shader converts YUV values from input textures into a final RGB | |
81 // pixel. The conversion formula is from http://www.fourcc.org/fccyvrgb.php. | |
82 static const char kFragmentShaderSource[] = | |
83 SHADER_VERSION | |
84 "precision highp float;" | |
85 FRAGMENT_SHADER_IN " vec2 v_texcoord;\n" | |
86 "uniform lowp sampler2D s_textureY;\n" | |
87 "uniform lowp sampler2D s_textureU;\n" | |
88 "uniform lowp sampler2D s_textureV;\n" | |
89 FRAGMENT_SHADER_OUT | |
90 "void main() {\n" | |
91 " float y, u, v, r, g, b;\n" | |
92 " y = " FRAGMENT_SHADER_TEXTURE "(s_textureY, v_texcoord).r;\n" | |
93 " u = " FRAGMENT_SHADER_TEXTURE "(s_textureU, v_texcoord).r;\n" | |
94 " v = " FRAGMENT_SHADER_TEXTURE "(s_textureV, v_texcoord).r;\n" | |
95 " u = u - 0.5;\n" | |
96 " v = v - 0.5;\n" | |
97 " r = y + 1.403 * v;\n" | |
98 " g = y - 0.344 * u - 0.714 * v;\n" | |
99 " b = y + 1.770 * u;\n" | |
100 " " FRAGMENT_SHADER_COLOR " = vec4(r, g, b, 1.0);\n" | |
101 " }\n"; | |
102 | |
103 // Compiles a shader of the given |type| with GLSL source |source| and returns | |
104 // the shader handle or 0 on error. | |
105 GLuint CreateShader(GLenum type, const GLchar* source) { | |
106 GLuint shader = glCreateShader(type); | |
107 if (!shader) { | |
108 return 0; | |
109 } | |
110 glShaderSource(shader, 1, &source, NULL); | |
111 glCompileShader(shader); | |
112 GLint compileStatus = GL_FALSE; | |
113 glGetShaderiv(shader, GL_COMPILE_STATUS, &compileStatus); | |
114 if (compileStatus == GL_FALSE) { | |
115 glDeleteShader(shader); | |
116 shader = 0; | |
117 } | |
118 return shader; | |
119 } | |
120 | |
121 // Links a shader program with the given vertex and fragment shaders and | |
122 // returns the program handle or 0 on error. | |
123 GLuint CreateProgram(GLuint vertexShader, GLuint fragmentShader) { | |
124 if (vertexShader == 0 || fragmentShader == 0) { | |
125 return 0; | |
126 } | |
127 GLuint program = glCreateProgram(); | |
128 if (!program) { | |
129 return 0; | |
130 } | |
131 glAttachShader(program, vertexShader); | |
132 glAttachShader(program, fragmentShader); | |
133 glLinkProgram(program); | |
134 GLint linkStatus = GL_FALSE; | |
135 glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); | |
136 if (linkStatus == GL_FALSE) { | |
137 glDeleteProgram(program); | |
138 program = 0; | |
139 } | |
140 return program; | |
141 } | |
142 | |
143 // When modelview and projection matrices are identity (default) the world is | |
144 // contained in the square around origin with unit size 2. Drawing to these | |
145 // coordinates is equivalent to drawing to the entire screen. The texture is | |
146 // stretched over that square using texture coordinates (u, v) that range | |
147 // from (0, 0) to (1, 1) inclusive. Texture coordinates are flipped vertically | |
148 // here because the incoming frame has origin in upper left hand corner but | |
149 // OpenGL expects origin in bottom left corner. | |
150 const GLfloat gVertices[] = { | |
151 // X, Y, U, V. | |
152 -1, -1, 0, 1, // Bottom left. | |
153 1, -1, 1, 1, // Bottom right. | |
154 1, 1, 1, 0, // Top right. | |
155 -1, 1, 0, 0, // Top left. | |
156 }; | |
157 | |
158 // |kNumTextures| must not exceed 8, which is the limit in OpenGLES2. Two sets | |
159 // of 3 textures are used here, one for each of the Y, U and V planes. Having | |
160 // two sets alleviates CPU blockage in the event that the GPU is asked to render | |
161 // to a texture that is already in use. | |
162 static const GLsizei kNumTextureSets = 2; | |
163 static const GLsizei kNumTextures = 3 * kNumTextureSets; | |
164 | |
165 @implementation RTCOpenGLVideoRenderer { | |
166 #if TARGET_OS_IPHONE | |
167 EAGLContext* _context; | |
168 #else | |
169 NSOpenGLContext* _context; | |
170 #endif | |
171 BOOL _isInitialized; | |
172 NSUInteger _currentTextureSet; | |
173 // Handles for OpenGL constructs. | |
174 GLuint _textures[kNumTextures]; | |
175 GLuint _program; | |
176 #if !TARGET_OS_IPHONE | |
177 GLuint _vertexArray; | |
178 #endif | |
179 GLuint _vertexBuffer; | |
180 GLint _position; | |
181 GLint _texcoord; | |
182 GLint _ySampler; | |
183 GLint _uSampler; | |
184 GLint _vSampler; | |
185 // Used to create a non-padded plane for GPU upload when we receive padded | |
186 // frames. | |
187 std::unique_ptr<uint8_t[]> _planeBuffer; | |
188 } | |
189 | |
190 + (void)initialize { | |
191 // Disable dithering for performance. | |
192 glDisable(GL_DITHER); | |
193 } | |
194 | |
195 #if TARGET_OS_IPHONE | |
196 - (instancetype)initWithContext:(EAGLContext*)context { | |
197 #else | |
198 - (instancetype)initWithContext:(NSOpenGLContext*)context { | |
199 #endif | |
200 NSAssert(context != nil, @"context cannot be nil"); | |
201 if (self = [super init]) { | |
202 _context = context; | |
203 } | |
204 return self; | |
205 } | |
206 | |
207 - (BOOL)drawFrame:(RTCI420Frame*)frame { | |
208 if (!_isInitialized) { | |
209 return NO; | |
210 } | |
211 if (_lastDrawnFrame == frame) { | |
212 return NO; | |
213 } | |
214 [self ensureGLContext]; | |
215 glClear(GL_COLOR_BUFFER_BIT); | |
216 if (frame) { | |
217 if (![self updateTextureSizesForFrame:frame] || | |
218 ![self updateTextureDataForFrame:frame]) { | |
219 return NO; | |
220 } | |
221 #if !TARGET_OS_IPHONE | |
222 glBindVertexArray(_vertexArray); | |
223 #endif | |
224 glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); | |
225 glDrawArrays(GL_TRIANGLE_FAN, 0, 4); | |
226 } | |
227 #if !TARGET_OS_IPHONE | |
228 [_context flushBuffer]; | |
229 #endif | |
230 _lastDrawnFrame = frame; | |
231 return YES; | |
232 } | |
233 | |
234 - (void)setupGL { | |
235 if (_isInitialized) { | |
236 return; | |
237 } | |
238 [self ensureGLContext]; | |
239 if (![self setupProgram]) { | |
240 return; | |
241 } | |
242 if (![self setupTextures]) { | |
243 return; | |
244 } | |
245 if (![self setupVertices]) { | |
246 return; | |
247 } | |
248 glUseProgram(_program); | |
249 glPixelStorei(GL_UNPACK_ALIGNMENT, 1); | |
250 _isInitialized = YES; | |
251 } | |
252 | |
253 - (void)teardownGL { | |
254 if (!_isInitialized) { | |
255 return; | |
256 } | |
257 [self ensureGLContext]; | |
258 glDeleteProgram(_program); | |
259 _program = 0; | |
260 glDeleteTextures(kNumTextures, _textures); | |
261 glDeleteBuffers(1, &_vertexBuffer); | |
262 _vertexBuffer = 0; | |
263 #if !TARGET_OS_IPHONE | |
264 glDeleteVertexArrays(1, &_vertexArray); | |
265 #endif | |
266 _isInitialized = NO; | |
267 } | |
268 | |
269 #pragma mark - Private | |
270 | |
271 - (void)ensureGLContext { | |
272 NSAssert(_context, @"context shouldn't be nil"); | |
273 #if TARGET_OS_IPHONE | |
274 if ([EAGLContext currentContext] != _context) { | |
275 [EAGLContext setCurrentContext:_context]; | |
276 } | |
277 #else | |
278 if ([NSOpenGLContext currentContext] != _context) { | |
279 [_context makeCurrentContext]; | |
280 } | |
281 #endif | |
282 } | |
283 | |
284 - (BOOL)setupProgram { | |
285 NSAssert(!_program, @"program already set up"); | |
286 GLuint vertexShader = CreateShader(GL_VERTEX_SHADER, kVertexShaderSource); | |
287 NSAssert(vertexShader, @"failed to create vertex shader"); | |
288 GLuint fragmentShader = | |
289 CreateShader(GL_FRAGMENT_SHADER, kFragmentShaderSource); | |
290 NSAssert(fragmentShader, @"failed to create fragment shader"); | |
291 _program = CreateProgram(vertexShader, fragmentShader); | |
292 // Shaders are created only to generate program. | |
293 if (vertexShader) { | |
294 glDeleteShader(vertexShader); | |
295 } | |
296 if (fragmentShader) { | |
297 glDeleteShader(fragmentShader); | |
298 } | |
299 if (!_program) { | |
300 return NO; | |
301 } | |
302 _position = glGetAttribLocation(_program, "position"); | |
303 _texcoord = glGetAttribLocation(_program, "texcoord"); | |
304 _ySampler = glGetUniformLocation(_program, "s_textureY"); | |
305 _uSampler = glGetUniformLocation(_program, "s_textureU"); | |
306 _vSampler = glGetUniformLocation(_program, "s_textureV"); | |
307 if (_position < 0 || _texcoord < 0 || _ySampler < 0 || _uSampler < 0 || | |
308 _vSampler < 0) { | |
309 return NO; | |
310 } | |
311 return YES; | |
312 } | |
313 | |
314 - (BOOL)setupTextures { | |
315 glGenTextures(kNumTextures, _textures); | |
316 // Set parameters for each of the textures we created. | |
317 for (GLsizei i = 0; i < kNumTextures; i++) { | |
318 glActiveTexture(GL_TEXTURE0 + i); | |
319 glBindTexture(GL_TEXTURE_2D, _textures[i]); | |
320 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); | |
321 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); | |
322 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); | |
323 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); | |
324 } | |
325 return YES; | |
326 } | |
327 | |
328 - (BOOL)updateTextureSizesForFrame:(RTCI420Frame*)frame { | |
329 if (frame.height == _lastDrawnFrame.height && | |
330 frame.width == _lastDrawnFrame.width && | |
331 frame.chromaWidth == _lastDrawnFrame.chromaWidth && | |
332 frame.chromaHeight == _lastDrawnFrame.chromaHeight) { | |
333 return YES; | |
334 } | |
335 GLsizei lumaWidth = frame.width; | |
336 GLsizei lumaHeight = frame.height; | |
337 GLsizei chromaWidth = frame.chromaWidth; | |
338 GLsizei chromaHeight = frame.chromaHeight; | |
339 for (GLint i = 0; i < kNumTextureSets; i++) { | |
340 glActiveTexture(GL_TEXTURE0 + i * 3); | |
341 glTexImage2D(GL_TEXTURE_2D, | |
342 0, | |
343 RTC_PIXEL_FORMAT, | |
344 lumaWidth, | |
345 lumaHeight, | |
346 0, | |
347 RTC_PIXEL_FORMAT, | |
348 GL_UNSIGNED_BYTE, | |
349 0); | |
350 glActiveTexture(GL_TEXTURE0 + i * 3 + 1); | |
351 glTexImage2D(GL_TEXTURE_2D, | |
352 0, | |
353 RTC_PIXEL_FORMAT, | |
354 chromaWidth, | |
355 chromaHeight, | |
356 0, | |
357 RTC_PIXEL_FORMAT, | |
358 GL_UNSIGNED_BYTE, | |
359 0); | |
360 glActiveTexture(GL_TEXTURE0 + i * 3 + 2); | |
361 glTexImage2D(GL_TEXTURE_2D, | |
362 0, | |
363 RTC_PIXEL_FORMAT, | |
364 chromaWidth, | |
365 chromaHeight, | |
366 0, | |
367 RTC_PIXEL_FORMAT, | |
368 GL_UNSIGNED_BYTE, | |
369 0); | |
370 } | |
371 if (frame.yPitch != frame.width || frame.uPitch != frame.chromaWidth || | |
372 frame.vPitch != frame.chromaWidth) { | |
373 _planeBuffer.reset(new uint8_t[frame.width * frame.height]); | |
374 } else { | |
375 _planeBuffer.reset(); | |
376 } | |
377 return YES; | |
378 } | |
379 | |
380 - (void)uploadPlane:(const uint8_t*)plane | |
381 sampler:(GLint)sampler | |
382 offset:(NSUInteger)offset | |
383 width:(NSUInteger)width | |
384 height:(NSUInteger)height | |
385 stride:(NSInteger)stride { | |
386 glActiveTexture(GL_TEXTURE0 + offset); | |
387 // When setting texture sampler uniforms, the texture index is used not | |
388 // the texture handle. | |
389 glUniform1i(sampler, offset); | |
390 #if TARGET_OS_IPHONE | |
391 BOOL hasUnpackRowLength = _context.API == kEAGLRenderingAPIOpenGLES3; | |
392 #else | |
393 BOOL hasUnpackRowLength = YES; | |
394 #endif | |
395 const uint8_t* uploadPlane = plane; | |
396 if (stride != width) { | |
397 if (hasUnpackRowLength) { | |
398 // GLES3 allows us to specify stride. | |
399 glPixelStorei(GL_UNPACK_ROW_LENGTH, stride); | |
400 glTexImage2D(GL_TEXTURE_2D, | |
401 0, | |
402 RTC_PIXEL_FORMAT, | |
403 width, | |
404 height, | |
405 0, | |
406 RTC_PIXEL_FORMAT, | |
407 GL_UNSIGNED_BYTE, | |
408 uploadPlane); | |
409 glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); | |
410 return; | |
411 } else { | |
412 // Make an unpadded copy and upload that instead. Quick profiling showed | |
413 // that this is faster than uploading row by row using glTexSubImage2D. | |
414 uint8_t* unpaddedPlane = _planeBuffer.get(); | |
415 for (NSUInteger y = 0; y < height; ++y) { | |
416 memcpy(unpaddedPlane + y * width, plane + y * stride, width); | |
417 } | |
418 uploadPlane = unpaddedPlane; | |
419 } | |
420 } | |
421 glTexImage2D(GL_TEXTURE_2D, | |
422 0, | |
423 RTC_PIXEL_FORMAT, | |
424 width, | |
425 height, | |
426 0, | |
427 RTC_PIXEL_FORMAT, | |
428 GL_UNSIGNED_BYTE, | |
429 uploadPlane); | |
430 } | |
431 | |
432 - (BOOL)updateTextureDataForFrame:(RTCI420Frame*)frame { | |
433 NSUInteger textureOffset = _currentTextureSet * 3; | |
434 NSAssert(textureOffset + 3 <= kNumTextures, @"invalid offset"); | |
435 | |
436 [self uploadPlane:frame.yPlane | |
437 sampler:_ySampler | |
438 offset:textureOffset | |
439 width:frame.width | |
440 height:frame.height | |
441 stride:frame.yPitch]; | |
442 | |
443 [self uploadPlane:frame.uPlane | |
444 sampler:_uSampler | |
445 offset:textureOffset + 1 | |
446 width:frame.chromaWidth | |
447 height:frame.chromaHeight | |
448 stride:frame.uPitch]; | |
449 | |
450 [self uploadPlane:frame.vPlane | |
451 sampler:_vSampler | |
452 offset:textureOffset + 2 | |
453 width:frame.chromaWidth | |
454 height:frame.chromaHeight | |
455 stride:frame.vPitch]; | |
456 | |
457 _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets; | |
458 return YES; | |
459 } | |
460 | |
461 - (BOOL)setupVertices { | |
462 #if !TARGET_OS_IPHONE | |
463 NSAssert(!_vertexArray, @"vertex array already set up"); | |
464 glGenVertexArrays(1, &_vertexArray); | |
465 if (!_vertexArray) { | |
466 return NO; | |
467 } | |
468 glBindVertexArray(_vertexArray); | |
469 #endif | |
470 NSAssert(!_vertexBuffer, @"vertex buffer already set up"); | |
471 glGenBuffers(1, &_vertexBuffer); | |
472 if (!_vertexBuffer) { | |
473 #if !TARGET_OS_IPHONE | |
474 glDeleteVertexArrays(1, &_vertexArray); | |
475 _vertexArray = 0; | |
476 #endif | |
477 return NO; | |
478 } | |
479 glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); | |
480 glBufferData(GL_ARRAY_BUFFER, sizeof(gVertices), gVertices, GL_DYNAMIC_DRAW); | |
481 | |
482 // Read position attribute from |gVertices| with size of 2 and stride of 4 | |
483 // beginning at the start of the array. The last argument indicates offset | |
484 // of data within |gVertices| as supplied to the vertex buffer. | |
485 glVertexAttribPointer( | |
486 _position, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), (void*)0); | |
487 glEnableVertexAttribArray(_position); | |
488 | |
489 // Read texcoord attribute from |gVertices| with size of 2 and stride of 4 | |
490 // beginning at the first texcoord in the array. The last argument indicates | |
491 // offset of data within |gVertices| as supplied to the vertex buffer. | |
492 glVertexAttribPointer(_texcoord, | |
493 2, | |
494 GL_FLOAT, | |
495 GL_FALSE, | |
496 4 * sizeof(GLfloat), | |
497 (void*)(2 * sizeof(GLfloat))); | |
498 glEnableVertexAttribArray(_texcoord); | |
499 | |
500 return YES; | |
501 } | |
502 | |
503 @end | |
OLD | NEW |