OLD | NEW |
---|---|
(Empty) | |
1 /* | |
2 * libjingle | |
3 * Copyright 2015 Google Inc. | |
4 * | |
5 * Redistribution and use in source and binary forms, with or without | |
6 * modification, are permitted provided that the following conditions are met: | |
7 * | |
8 * 1. Redistributions of source code must retain the above copyright notice, | |
9 * this list of conditions and the following disclaimer. | |
10 * 2. Redistributions in binary form must reproduce the above copyright notice, | |
11 * this list of conditions and the following disclaimer in the documentation | |
12 * and/or other materials provided with the distribution. | |
13 * 3. The name of the author may not be used to endorse or promote products | |
14 * derived from this software without specific prior written permission. | |
15 * | |
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED | |
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF | |
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO | |
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, | |
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; | |
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, | |
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR | |
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF | |
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
26 */ | |
27 | |
28 package org.webrtc; | |
29 | |
30 import android.content.Context; | |
31 import android.graphics.Point; | |
32 import android.graphics.SurfaceTexture; | |
33 import android.opengl.EGLContext; | |
34 import android.opengl.GLES20; | |
35 import android.os.Handler; | |
36 import android.os.HandlerThread; | |
37 import android.util.AttributeSet; | |
38 import android.util.Log; | |
39 import android.view.SurfaceHolder; | |
40 import android.view.SurfaceView; | |
41 | |
42 /** | |
43 * Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream on a SurfaceView. | |
44 * renderFrame() is asynchronous to avoid blocking the calling thread. Instead, a shallow copy of | |
45 * the frame is posted to a dedicated render thread. | |
46 * This class is thread safe and handles access from potentially four different threads: | |
47 * Interaction from the main app in init, release, setMirror, and setScalingtype . | |
48 * Interaction from C++ webrtc::VideoRendererInterface in renderFrame and canApp lyRotation. | |
49 * Interaction from the Activity lifecycle in surfaceCreated, surfaceChanged, an d surfaceDestroyed. | |
50 * Interaction with the layout framework in onMeasure and onSizeChanged. | |
51 */ | |
52 public class SurfaceViewRenderer extends SurfaceView | |
53 implements SurfaceHolder.Callback, VideoRenderer.Callbacks { | |
54 private static final String TAG = "SurfaceViewRenderer"; | |
55 | |
56 // Dedicated render thread. Synchronized on |this|. | |
57 private HandlerThread renderThread; | |
58 // Handler for inter-thread communication. Synchronized on |this|. | |
59 private Handler renderThreadHandler; | |
60 // Pending frame to render. Serves as a queue with size 1. Synchronized on |th is|. | |
61 private VideoRenderer.I420Frame pendingFrame; | |
62 | |
63 // EGL and GL resources for drawing YUV/OES textures. After initilization, the se are only accessed | |
64 // from the render thread. | |
65 private EglBase eglBase; | |
66 private GlRectDrawer drawer; | |
67 // Texture ids for YUV frames. Allocated on first arrival of a YUV frame. | |
68 private int[] yuvTextures = null; | |
69 | |
70 // These variables are synchronized on |layoutLock|. | |
71 private final Object layoutLock = new Object(); | |
72 // Aspect ratio on screen. | |
73 private float layoutAspectRatio; | |
74 // Aspect ratio of the most recent video frame. | |
75 private float videoAspectRatio; | |
76 // |scalingType| determines how the video will fill the allowed layout area in onMeasure(). | |
77 private VideoRendererGui.ScalingType scalingType = | |
78 VideoRendererGui.ScalingType.SCALE_ASPECT_BALANCED; | |
79 // If true, mirrors the video stream horizontally. | |
80 private boolean mirror; | |
81 | |
82 // These variables are synchronized on |statisticsLock|. | |
83 private final Object statisticsLock = new Object(); | |
84 // Total number of video frames received in renderFrame() call. | |
85 private int framesReceived; | |
86 // Number of video frames dropped by renderFrame() because previous frame has not been rendered | |
87 // yet. | |
88 private int framesDropped; | |
89 // Number of rendered video frames. | |
90 private int framesRendered; | |
91 // Time in ns when the first video frame was rendered. | |
92 private long firstFrameTimeNs; | |
93 // Time in ns spent in renderFrameOnRenderThread() function. | |
94 private long renderTimeNs; | |
95 | |
96 /** | |
97 * Standard View constructor. In order to render something, you must first cal l init(). | |
98 */ | |
99 public SurfaceViewRenderer(Context context) { | |
100 super(context); | |
101 } | |
102 | |
103 /** | |
104 * Standard View constructor. In order to render something, you must first cal l init(). | |
105 */ | |
106 public SurfaceViewRenderer(Context context, AttributeSet attrs) { | |
107 super(context, attrs); | |
108 } | |
109 | |
110 /** | |
111 * Initialize this class, sharing resources with |sharedContext|. | |
112 */ | |
113 public synchronized void init(EGLContext sharedContext) { | |
114 if (renderThreadHandler != null) { | |
115 throw new IllegalStateException("Already initialized"); | |
116 } | |
117 Log.d(TAG, "Initializing"); | |
118 renderThread = new HandlerThread(TAG); | |
119 renderThread.start(); | |
120 renderThreadHandler = new Handler(renderThread.getLooper()); | |
121 eglBase = new EglBase(sharedContext, EglBase.ConfigType.PLAIN); | |
122 drawer = new GlRectDrawer(); | |
123 getHolder().addCallback(this); | |
124 } | |
125 | |
126 /** | |
127 * Release all resources. This needs to be done manually, otherwise the resour ces are leaked. | |
128 */ | |
129 public synchronized void release() { | |
130 if (renderThreadHandler == null) { | |
131 Log.d(TAG, "Already released"); | |
132 return; | |
133 } | |
134 // Release EGL and GL resources on render thread. | |
135 renderThreadHandler.post(new Runnable() { | |
136 @Override public void run() { | |
137 drawer.release(); | |
138 drawer = null; | |
139 if (yuvTextures != null) { | |
140 GLES20.glDeleteTextures(3, yuvTextures, 0); | |
141 yuvTextures = null; | |
142 } | |
143 eglBase.release(); | |
144 eglBase = null; | |
145 } | |
146 }); | |
147 // Don't accept any more messages to the render thread. | |
148 renderThreadHandler = null; | |
149 // Quit safely to make sure the EGL/GL cleanup posted above is executed. | |
150 renderThread.quitSafely(); | |
151 renderThread = null; | |
152 | |
153 getHolder().removeCallback(this); | |
154 if (pendingFrame != null) { | |
155 VideoRenderer.renderFrameDone(pendingFrame); | |
156 pendingFrame = null; | |
157 } | |
158 } | |
159 | |
160 /** | |
161 * Set if the video stream should be mirrored or not. | |
162 */ | |
163 public void setMirror(final boolean mirror) { | |
164 synchronized (layoutLock) { | |
165 this.mirror = mirror; | |
166 } | |
167 } | |
168 | |
169 /** | |
170 * Set how the video will fill the allowed layout area. | |
171 */ | |
172 public void setScalingType(VideoRendererGui.ScalingType scalingType) { | |
173 synchronized (layoutLock) { | |
174 this.scalingType = scalingType; | |
175 } | |
176 } | |
177 | |
178 // VideoRenderer.Callbacks interface. | |
179 @Override | |
180 public void renderFrame(VideoRenderer.I420Frame frame) { | |
181 synchronized (statisticsLock) { | |
182 ++framesReceived; | |
183 } | |
184 // Trigger layout update if video aspect ratio changed. | |
185 synchronized (layoutLock) { | |
186 final float videoAspectRatio = (float) frame.rotatedWidth() / frame.rotate dHeight(); | |
187 if (videoAspectRatio != this.videoAspectRatio) { | |
188 this.videoAspectRatio = videoAspectRatio; | |
189 // Need to request layout update from the UI thread. | |
190 post(new Runnable() { | |
191 @Override public void run() { | |
192 requestLayout(); | |
193 } | |
194 }); | |
195 } | |
196 } | |
197 synchronized (this) { | |
198 if (renderThreadHandler == null) { | |
199 Log.d(TAG, "Dropping frame - SurfaceViewRenderer not initialized or alre ady released."); | |
200 VideoRenderer.renderFrameDone(frame); | |
201 return; | |
202 } | |
203 if (pendingFrame != null) { | |
204 synchronized (statisticsLock) { | |
205 ++framesDropped; | |
206 } | |
207 Log.d(TAG, "Dropping frame - previous frame has not been rendered yet.") ; | |
208 VideoRenderer.renderFrameDone(frame); | |
209 return; | |
210 } | |
211 pendingFrame = frame; | |
212 renderThreadHandler.post(new Runnable() { | |
213 @Override public void run() { | |
214 renderFrameOnRenderThread(); | |
215 } | |
216 }); | |
217 } | |
218 } | |
219 | |
220 @Override | |
221 public boolean canApplyRotation() { | |
222 return true; | |
223 } | |
224 | |
225 // View layout interface. | |
226 @Override | |
227 protected void onMeasure(int widthSpec, int heightSpec) { | |
228 final int maxWidth = getDefaultSize(Integer.MAX_VALUE, widthSpec); | |
229 final int maxHeight = getDefaultSize(Integer.MAX_VALUE, heightSpec); | |
230 final Point suggestedSize; | |
231 synchronized (layoutLock) { | |
232 suggestedSize = VideoRendererGui.getDisplaySize( | |
AlexG
2015/08/05 00:47:11
Move this to helper class?
magjed_webrtc
2015/08/07 17:14:50
Done.
| |
233 VideoRendererGui.convertScalingTypeToVisibleFraction(scalingType), | |
234 videoAspectRatio, maxWidth, maxHeight); | |
235 } | |
236 setMeasuredDimension( | |
237 MeasureSpec.getMode(widthSpec) == MeasureSpec.EXACTLY ? maxWidth : sugge stedSize.x, | |
238 MeasureSpec.getMode(heightSpec) == MeasureSpec.EXACTLY ? maxHeight : sug gestedSize.y); | |
239 } | |
240 | |
241 @Override | |
242 protected void onSizeChanged(int w, int h, int oldw, int oldh) { | |
243 synchronized (layoutLock) { | |
244 layoutAspectRatio = (float) w / h; | |
245 } | |
246 } | |
247 | |
248 // SurfaceHolder.Callback interface. | |
249 @Override | |
250 public void surfaceCreated(final SurfaceHolder holder) { | |
251 Log.d(TAG, "Surface created"); | |
252 runOnRenderThread(new Runnable() { | |
253 @Override public void run() { | |
254 eglBase.createSurface(holder.getSurface()); | |
255 eglBase.makeCurrent(); | |
256 // Might have a pending frame waiting for a surface to be created. | |
257 renderFrameOnRenderThread(); | |
258 } | |
259 }); | |
260 } | |
261 | |
262 @Override | |
263 public void surfaceDestroyed(SurfaceHolder holder) { | |
264 Log.d(TAG, "Surface destroyed"); | |
265 runOnRenderThread(new Runnable() { | |
266 @Override public void run() { | |
267 eglBase.releaseSurface(); | |
268 } | |
269 }); | |
270 } | |
271 | |
272 @Override | |
273 public void surfaceChanged(SurfaceHolder holder, int format, int width, int he ight) { | |
274 } | |
275 | |
276 /** | |
277 * Private helper function to post tasks safely. | |
278 */ | |
279 private synchronized void runOnRenderThread(Runnable runnable) { | |
280 if (renderThreadHandler != null) { | |
281 renderThreadHandler.post(runnable); | |
282 } | |
283 } | |
284 | |
285 /** | |
286 * Renders and releases |pendingFrame|. | |
287 */ | |
288 private void renderFrameOnRenderThread() { | |
289 if (eglBase == null || !eglBase.hasSurface()) { | |
290 Log.d(TAG, "No surface to draw on"); | |
291 return; | |
292 } | |
293 // Synchronized fetch of |pendingFrame|. | |
294 final VideoRenderer.I420Frame frame; | |
295 synchronized (this) { | |
296 if (pendingFrame == null) { | |
297 return; | |
298 } | |
299 frame = pendingFrame; | |
300 pendingFrame = null; | |
301 } | |
302 | |
303 final long startTimeNs = System.nanoTime(); | |
304 final float[] texMatrix = new float[16]; | |
305 synchronized (layoutLock) { | |
306 VideoRendererGui.getTextureMatrix( | |
AlexG
2015/08/05 00:47:11
Move this to helper class?
magjed_webrtc
2015/08/07 17:14:50
Done.
| |
307 texMatrix, frame.rotationDegree, mirror, videoAspectRatio, layoutAspec tRatio); | |
308 } | |
309 | |
310 GLES20.glViewport(0, 0, eglBase.surfaceWidth(), eglBase.surfaceHeight()); | |
311 if (frame.yuvFrame) { | |
312 // Make sure YUV textures are allocated. | |
313 if (yuvTextures == null) { | |
314 yuvTextures = new int[3]; | |
315 // Generate 3 texture ids for Y/U/V and place them into |yuvTextures|. | |
316 GLES20.glGenTextures(3, yuvTextures, 0); | |
317 for (int i = 0; i < 3; i++) { | |
318 GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); | |
319 GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); | |
320 GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, | |
321 GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST); | |
322 GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, | |
323 GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); | |
324 GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, | |
325 GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); | |
326 GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, | |
327 GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); | |
328 } | |
329 GlUtil.checkNoGLES2Error("y/u/v glGenTextures"); | |
330 } | |
331 | |
332 // Upload YUV data. | |
333 for (int i = 0; i < 3; ++i) { | |
334 GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); | |
335 GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); | |
336 int w = (i == 0) ? frame.width : frame.width / 2; | |
337 int h = (i == 0) ? frame.height : frame.height / 2; | |
338 GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, w, h, 0, | |
339 GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, frame.yuvPlanes[i]); | |
340 } | |
341 | |
342 drawer.drawYuv(frame.width, frame.height, yuvTextures, texMatrix); | |
343 } else { | |
344 SurfaceTexture surfaceTexture = (SurfaceTexture) frame.textureObject; | |
345 // TODO(magjed): Move updateTexImage() to the video source instead. | |
346 surfaceTexture.updateTexImage(); | |
347 drawer.drawOes(frame.textureId, texMatrix); | |
348 } | |
349 | |
350 eglBase.swapBuffers(); | |
351 VideoRenderer.renderFrameDone(frame); | |
352 synchronized (statisticsLock) { | |
353 if (framesRendered == 0) { | |
354 firstFrameTimeNs = startTimeNs; | |
355 } | |
356 ++framesRendered; | |
357 renderTimeNs += (System.nanoTime() - startTimeNs); | |
358 if (framesRendered % 300 == 0) { | |
359 logStatistics(); | |
360 } | |
361 } | |
362 } | |
363 | |
364 private void logStatistics() { | |
365 synchronized (statisticsLock) { | |
366 Log.d(TAG, "ID: " + getResources().getResourceEntryName(getId()) + ". Fram es received: " | |
367 + framesReceived + ". Dropped: " + framesDropped + ". Rendered: " + fr amesRendered); | |
368 if (framesReceived > 0 && framesRendered > 0) { | |
369 final long timeSinceFirstFrameNs = System.nanoTime() - firstFrameTimeNs; | |
370 Log.d(TAG, "Duration: " + (int) (timeSinceFirstFrameNs / 1e6) + | |
371 " ms. FPS: " + (float) framesRendered * 1e9 / timeSinceFirstFrameNs) ; | |
372 Log.d(TAG, "Average render time: " | |
373 + (int) (renderTimeNs / (1000 * framesRendered)) + " us."); | |
374 } | |
375 } | |
376 } | |
377 } | |
OLD | NEW |