Drawing on top of previous frame with an offscreen texture
I am very new to OpenGL ES 2.0.
I'm trying to write a fingerpaint app using OpenGL ES 2.0. The idea is to draw from touches each frame onto a texture incrementally (without calling glClear(int) ), and sampling the texture onto a full-screen quad.
Referring to my code below, when I draw the GlCircle and GlLine onto the default Framebuffer , everything works fine.
But when I try to draw on top of the previous frame by using an offscreen texture, the coordinate on the rendered texture seems to be off:
The screenshot below should visually show what's wrong (the red/blue outline shows the actual touch coordinates on the screen, white dots are drawn to/from texture):
What am I doing wrong? Is there a better way of achieving this?
Here's my GLSurfaceView.Renderer :
package com.oaskamay.whiteboard.opengl;
import android.opengl.GLES20;
import android.opengl.Matrix;
import android.os.Bundle;
import android.util.Log;
import android.view.MotionEvent;
import com.oaskamay.whiteboard.opengl.base.GlSurfaceView;
import com.oaskamay.whiteboard.opengl.drawable.GlCircle;
import com.oaskamay.whiteboard.opengl.drawable.GlLine;
import com.oaskamay.whiteboard.opengl.drawable.GlTexturedQuad;
import java.util.ArrayList;
import java.util.List;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;
public class GlDrawingRenderer implements GlSurfaceView.Renderer {
/*
* Keys used to store/restore the state of this renderer.
*/
private static final String EXTRA_MOTION_EVENTS = "extra_motion_events";
private static final float[] COLOR_BG = new float[]{0.0f, 0.0f, 0.0f, 1.0f};
private static final float[] COLOR_BRUSH = new float[]{1.0f, 1.0f, 1.0f, 1.0f};
/*
* Model-view-projection matrix used to map normalized GL coordinates to the screen's.
*/
private final float[] mMvpMatrix;
private final float[] mViewMatrix;
private final float[] mProjectionMatrix;
private final float[] mTextureProjectionMatrix;
private final float[] mTextureMvpMatrix;
/*
* Offscreen texture rendering handles.
*/
private int[] mFrameBufferHandle;
private int[] mRenderTextureHandle;
/*
* Lists of vertices to draw each frame.
*/
private List<Float> mLineVertexData;
private List<Float> mCircleVertexData;
/*
* List of stored MotionEvents and PacketData, required to store/restore state of Renderer.
*/
private ArrayList<MotionEvent> mMotionEvents;
private boolean mRestoreMotionEvents = false;
private GlLine mLine;
private GlCircle mCircle;
private GlTexturedQuad mTexturedQuad;
/*
* Variables to calculate FPS throughput.
*/
private long mStartTime = System.nanoTime();
private int mFrameCount = 0;
public GlDrawingRenderer() {
mMvpMatrix = new float[16];
mViewMatrix = new float[16];
mProjectionMatrix = new float[16];
mTextureProjectionMatrix = new float[16];
mTextureMvpMatrix = new float[16];
mFrameBufferHandle = new int[1];
mRenderTextureHandle = new int[1];
mLineVertexData = new ArrayList<>();
mCircleVertexData = new ArrayList<>();
mMotionEvents = new ArrayList<>();
}
@Override
public void onSurfaceCreated(GL10 unused, EGLConfig config) {
// one time feature initializations
GLES20.glDisable(GLES20.GL_DEPTH_TEST);
GLES20.glDisable(GLES20.GL_DITHER);
// clear attachment buffers
GLES20.glClearColor(COLOR_BG[0], COLOR_BG[1], COLOR_BG[2],
COLOR_BG[3]);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
// initialize drawables
mLine = new GlLine();
mCircle = new GlCircle(5.0f);
mTexturedQuad = new GlTexturedQuad();
}
@Override
public void onSurfaceChanged(GL10 unused, int width, int height) {
GLES20.glViewport(0, 0, width, height);
// calculate projection, camera matrix and MVP matrix for touch events
Matrix.setLookAtM(mViewMatrix, 0, 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f, 0.0f);
Matrix.orthoM(mProjectionMatrix, 0, 0.0f, width, height, 0.0f, 0.0f, 1.0f);
Matrix.multiplyMM(mMvpMatrix, 0, mProjectionMatrix, 0, mViewMatrix, 0);
mLine.setMvpMatrix(mMvpMatrix);
mCircle.setMvpMatrix(mMvpMatrix);
// calculate projection and MVP matrix for texture
Matrix.setIdentityM(mTextureProjectionMatrix, 0);
Matrix.multiplyMM(mTextureMvpMatrix, 0, mTextureProjectionMatrix, 0, mViewMatrix, 0);
mTexturedQuad.setMvpMatrix(mTextureMvpMatrix);
// setup buffers for offscreen texture
GLES20.glGenFramebuffers(1, mFrameBufferHandle, 0);
GLES20.glGenTextures(1, mRenderTextureHandle, 0);
mTexturedQuad.initTexture(width, height, mRenderTextureHandle[0]);
}
@Override
public void onDrawFrame(GL10 unused) {
// use offscreen texture frame buffer
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, mFrameBufferHandle[0]);
GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
GLES20.GL_TEXTURE_2D, mRenderTextureHandle[0], 0);
GlUtil.glCheckFramebufferStatus();
// restore and draw saved MotionEvents onto texture if they exist
if (mRestoreMotionEvents) {
mRestoreMotionEvents = false;
processStoredMotionEvents();
}
// draw current MotionEvents onto texture
drawObjects();
// use window frame buffer
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
GLES20.glClearColor(COLOR_BG[0], COLOR_BG[1], COLOR_BG[2], COLOR_BG[3]);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
// draw texture onto full-screen quad onto the window surface
drawTexturedQuad();
logFps();
}
/**
* Draws any available line and circle vertex data. Objects including {@code GlCircle} and
* {@code GlLine} are to be drawn on the offscreen texture. The offscreen texture will then be
* drawn onto a fullscreen quad in the default window framebuffer.
*/
private void drawObjects() {
if (!mLineVertexData.isEmpty()) {
drawLines();
}
if (!mCircleVertexData.isEmpty()) {
drawCircles();
}
}
/**
* Draws circles. OpenGL points cannot have radii, hence we draw circles on down key events
* instead of points.
*/
private void drawCircles() {
GLES20.glUseProgram(mCircle.getProgramHandle());
// read offsets
float dx = mCircleVertexData.remove(0);
float dy = mCircleVertexData.remove(0);
float dz = mCircleVertexData.remove(0);
mCircle.setTranslateMatrix(dx, dy, dz);
// read color
float r = mCircleVertexData.remove(0);
float g = mCircleVertexData.remove(0);
float b = mCircleVertexData.remove(0);
float a = mCircleVertexData.remove(0);
mCircle.setColor(r, g, b, a);
mCircle.draw();
}
/**
* Draws lines from touch start points to touch end points.
*/
private void drawLines() {
GLES20.glUseProgram(mLine.getProgramHandle());
// read offsets
float x1 = mLineVertexData.remove(0);
float y1 = mLineVertexData.remove(0);
float z1 = mLineVertexData.remove(0);
float x2 = mLineVertexData.remove(0);
float y2 = mLineVertexData.remove(0);
float z2 = mLineVertexData.remove(0);
mLine.setTranslateMatrix(x1, y1, z1, x2, y2, z2);
// read color
float r = mLineVertexData.remove(0);
float g = mLineVertexData.remove(0);
float b = mLineVertexData.remove(0);
float a = mLineVertexData.remove(0);
mLine.setColor(r, g, b, a);
mLine.draw();
}
/**
* Draws the offscreen texture onto the fullscreen quad, and draws the quad onto the default
* window framebuffer.
*/
private void drawTexturedQuad() {
GLES20.glUseProgram(mTexturedQuad.getProgramHandle());
mTexturedQuad.draw();
}
/**
* Processes MotionEvent.
* Sets vertex and color data based on MotionEvent information.
*
* @param event MotionEvent to process.
* @param store Pass true when processing fresh MotionEvents to store them to support parent
* activity recreations, pass false otherwise.
*/
public void processMotionEvent(MotionEvent event, boolean store) {
if (store) {
mMotionEvents.add(MotionEvent.obtain(event));
}
int action = event.getActionMasked();
switch (action) {
case MotionEvent.ACTION_POINTER_DOWN:
case MotionEvent.ACTION_DOWN:
case MotionEvent.ACTION_MOVE:
// set centroid
mCircleVertexData.add(event.getX());
mCircleVertexData.add(event.getY());
mCircleVertexData.add(0.0f);
// set color
mCircleVertexData.add(COLOR_BRUSH[0]);
mCircleVertexData.add(COLOR_BRUSH[1]);
mCircleVertexData.add(COLOR_BRUSH[2]);
mCircleVertexData.add(COLOR_BRUSH[3]);
break;
}
}
/**
* Draws stored MotionEvents.
* Required to be able to restore state of this Renderer.
*/
private void processStoredMotionEvents() {
for (MotionEvent event : mMotionEvents) {
processMotionEvent(event, false);
drawObjects();
}
}
/**
* Prints out current frames-per-second throughput.
*/
private void logFps() {
mFrameCount++;
if (System.nanoTime() - mStartTime >= 1000000000L) {
Log.d("GlDrawingRenderer", "FPS: " + mFrameCount);
mFrameCount = 0;
mStartTime = System.nanoTime();
}
}
/**
* Saves line and circle vertex data into the {@code Bundle} argument. Call when the parent
* {@code GLSurfaceView} calls its corresponding {@code onSaveInstanceState()} method.
*
* @param bundle Destination {@code Bundle} to save the renderer state into.
*/
public void onSaveInstanceState(Bundle bundle) {
bundle.putParcelableArrayList(EXTRA_MOTION_EVENTS, mMotionEvents);
}
/**
* Restores line and circle vertex data from the {@code Bundle} argument. Call when the parent
* {@code GLSurfaceView} calls its corresponding {@code onRestoreInstanceState(Parcelable)}
* method.
*
* @param bundle Source {@code Bundle} to save the renderer state from.
*/
public void onRestoreInstanceState(Bundle bundle) {
ArrayList<MotionEvent> motionEvents = bundle.getParcelableArrayList(EXTRA_MOTION_EVENTS);
if (motionEvents != null && !motionEvents.isEmpty()) {
mMotionEvents.addAll(motionEvents);
mRestoreMotionEvents = true;
}
}
}
And here's the GlTexturedQuad class:
package com.oaskamay.whiteboard.opengl.drawable;
import android.opengl.GLES20;
import com.oaskamay.whiteboard.opengl.GlUtil;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.nio.ShortBuffer;
public class GlTexturedQuad {
/*
* Vertex metadata: we have 3 coordinates per vertex, and a quad can be drawn with 2 triangles.
*/
private static final int VERTEX_COORDS = 3;
private static final String VERTEX_SHADER_SOURCE =
"uniform mat4 u_MvpMatrix; n" +
"attribute vec4 a_Position; n" +
"attribute vec2 a_TextureCoord; n" +
"varying vec2 v_TextureCoord; n" +
" n" +
"void main() { n" +
" v_TextureCoord = a_TextureCoord; n" +
" gl_Position = u_MvpMatrix * a_Position; n" +
"} n";
private static final String FRAGMENT_SHADER_SOURCE =
"uniform sampler2D u_Texture; n" +
"varying vec2 v_TextureCoord; n" +
" n" +
"void main() { n" +
" gl_FragColor = texture2D(u_Texture, v_TextureCoord);n" +
"} n";
/*
* Vertex locations. The quad will cover the whole screen, and is in normalized device
* coordinates. The projection matrix for this quad should be identity.
*/
private static final float[] VERTICES = {
-1.0f, +1.0f, 0.0f,
-1.0f, -1.0f, 0.0f,
+1.0f, -1.0f, 0.0f,
+1.0f, +1.0f, 0.0f
};
/*
* Describes the order in which vertices are to be rendered.
*/
private static final short[] VERTICES_ORDER = {
0, 1, 2,
0, 2, 3
};
/*
* (u, v) texture coordinates to be sent to the vertex and fragment shaders.
*/
private static final float[] TEXTURE_COORDS = {
0.0f, 0.0f,
0.0f, 1.0f,
1.0f, 1.0f,
1.0f, 0.0f
};
private float mMvpMatrix[];
private int mRenderTexture;
/*
* FloatBuffers used to store vertices and their order to draw.
*/
private final FloatBuffer mVertexBuffer;
private final ShortBuffer mVertexOrderBuffer;
private final FloatBuffer mTextureCoordsBuffer;
/*
* OpenGL handles to shader program, attributes, and uniforms.
*/
private final int mProgramHandle;
private final int mMvpMatrixHandle;
private final int mPositionHandle;
private final int mTextureHandle;
private final int mTextureCoordHandle;
/**
* Default constructor. Refrain from calling this multiple times as it may be expensive due to
* compilation of shader sources.
*/
public GlTexturedQuad() {
// initialize vertex buffer
ByteBuffer vertexBuffer = ByteBuffer.allocateDirect(VERTICES.length * 4);
vertexBuffer.order(ByteOrder.nativeOrder());
mVertexBuffer = vertexBuffer.asFloatBuffer();
mVertexBuffer.put(VERTICES);
mVertexBuffer.position(0);
// initialize vertex order buffer
ByteBuffer vertexOrderBuffer = ByteBuffer.allocateDirect(VERTICES_ORDER.length * 2);
vertexOrderBuffer.order(ByteOrder.nativeOrder());
mVertexOrderBuffer = vertexOrderBuffer.asShortBuffer();
mVertexOrderBuffer.put(VERTICES_ORDER);
mVertexOrderBuffer.position(0);
// initialize texture coordinates
ByteBuffer textureCoordsBuffer = ByteBuffer.allocateDirect(TEXTURE_COORDS.length * 4);
textureCoordsBuffer.order(ByteOrder.nativeOrder());
mTextureCoordsBuffer = textureCoordsBuffer.asFloatBuffer();
mTextureCoordsBuffer.put(TEXTURE_COORDS);
mTextureCoordsBuffer.position(0);
// compile vertex and fragment shader sources
int vertexShader = GlUtil.glLoadShader(GLES20.GL_VERTEX_SHADER,
VERTEX_SHADER_SOURCE);
int fragmentShader = GlUtil.glLoadShader(GLES20.GL_FRAGMENT_SHADER,
FRAGMENT_SHADER_SOURCE);
// create shader program and attach compiled sources
mProgramHandle = GLES20.glCreateProgram();
GLES20.glAttachShader(mProgramHandle, vertexShader);
GLES20.glAttachShader(mProgramHandle, fragmentShader);
GLES20.glLinkProgram(mProgramHandle);
// store attribute / uniform handles
mMvpMatrixHandle = GLES20.glGetUniformLocation(mProgramHandle, "u_MvpMatrix");
mTextureHandle = GLES20.glGetUniformLocation(mProgramHandle, "u_Texture");
mPositionHandle = GLES20.glGetAttribLocation(mProgramHandle, "a_Position");
mTextureCoordHandle = GLES20.glGetAttribLocation(mProgramHandle, "a_TextureCoord");
}
/**
* Initializes texture components.
*
* @param width Width of texture in pixels.
* @param height Height of texture in pixels.
*/
public void initTexture(int width, int height, int renderTexture) {
mRenderTexture = renderTexture;
// allocate pixel buffer for texture
ByteBuffer byteBuffer = ByteBuffer.allocateDirect(width * height * 4);
byteBuffer.order(ByteOrder.nativeOrder());
IntBuffer texturePixelBuffer = byteBuffer.asIntBuffer();
// initialize texture
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mRenderTexture);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S,
GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T,
GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_LINEAR);
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGB, width, height,
0, GLES20.GL_RGB, GLES20.GL_UNSIGNED_SHORT_5_6_5, texturePixelBuffer);
}
/**
* Draws this object. The model-view-projection matrix must be set with
* {@link #setMvpMatrix(float[])}.
*/
public final void draw() {
GLES20.glEnableVertexAttribArray(mPositionHandle);
GLES20.glEnableVertexAttribArray(mTextureCoordHandle);
// set vertex position and MVP matrix in shader
GLES20.glVertexAttribPointer(mPositionHandle, VERTEX_COORDS, GLES20.GL_FLOAT,
false, VERTEX_COORDS * 4, mVertexBuffer);
GLES20.glUniformMatrix4fv(mMvpMatrixHandle, 1, false, mMvpMatrix, 0);
// bind texture
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mRenderTexture);
// set texture data and coordinate
GLES20.glVertexAttribPointer(mTextureCoordHandle, 2, GLES20.GL_FLOAT, false, 0,
mTextureCoordsBuffer);
GLES20.glUniform1i(mTextureHandle, 0);
GLES20.glDrawElements(GLES20.GL_TRIANGLES, VERTICES_ORDER.length, GLES20.GL_UNSIGNED_SHORT,
mVertexOrderBuffer);
GLES20.glDisableVertexAttribArray(mPositionHandle);
GLES20.glDisableVertexAttribArray(mTextureCoordHandle);
}
/**
* Sets the model-view-projection matrix in the vertex shader. Necessary to map the normalized
* GL coordinate system to that of the display.
*
* @param mvpMatrix Matrix to use as the model-view-projection matrix.
*/
public void setMvpMatrix(float[] mvpMatrix) {
mMvpMatrix = mvpMatrix;
}
public int getProgramHandle() {
return mProgramHandle;
}
}
EDIT (12/11/2015) :
@reto-koradi suggested a much better solution. Invert the V-axis by changing the texture coordinates. This fix is also simple:
Change this (initialization of TEXTURE_COORDS array in GlTexturedQuad ):
/*
* (u, v) texture coordinates to be sent to the vertex and fragment shaders.
*/
private static final float[] TEXTURE_COORDS = {
0.0f, 0.0f,
0.0f, 1.0f,
1.0f, 1.0f,
1.0f, 0.0f
};
To this:
/*
* (u, v) texture coordinates to be sent to the vertex and fragment shaders.
*/
private static final float[] TEXTURE_COORDS = {
0.0f, 1.0f,
0.0f, 0.0f,
1.0f, 0.0f,
1.0f, 1.0f
};
I've fixed the issue. The problem was with the projection matrix used for the GlTexturedQuad . The fix was simple:
I changed this (in onSurfaceChanged(GL10, int, int) in GlDrawingRenderer ):
// calculate projection and MVP matrix for texture
Matrix.setIdentityM(mTextureProjectionMatrix, 0);
Matrix.multiplyMM(mTextureMvpMatrix, 0, mTextureProjectionMatrix, 0, mViewMatrix, 0);
mTexturedQuad.setMvpMatrix(mTextureMvpMatrix);
To this:
// calculate projection and MVP matrix for texture
Matrix.orthoM(mTextureProjectionMatrix, 0, -1.0f, 1.0f, 1.0f, -1.0f, 0.0f, 1.0f);
Matrix.multiplyMM(mTextureMvpMatrix, 0, mTextureProjectionMatrix, 0, mViewMatrix, 0);
mTexturedQuad.setMvpMatrix(mTextureMvpMatrix);
So now mTextureProjectionMatrix takes into account the V-axis inversion of the texture. Again, I'm an OpenGL ES 2.0 beginner, my explanation might be wrong. But it works :)
I hope this post helped someone out there!
Although there seem to be many solutions to fix the inverted screen you should understand what happens in the background, why is it even inverted in your case and incidentally why your solution is not general.
The openGL buffers follow the legacy desktop coordinate system where bottom-left point is the origin and height increases upwards. So the raw buffer data will have the first pixel data at bottom-left part and not at top-left as you would expect due to how image data is used. So if you want to draw to the top-left part of the image you actually need to draw to the bottom-left part of the buffer (respecting the presentation).
So your issue is not in how you present the drawn texture but how you actually draw to the texture itself. Your coordinate system is inverted while drawing the points. But what difference does it make where I invert?
There is a huge difference actually. Since you inverted the coordinate system when drawing to the FBO and then inverted again when drawing to the presentation buffer to get the correct result your inversion equation is kind of (-1 * -1 = 1) . Then what will happen if you add some post processing by adding another FBO: (-1 * -1 * -1 = -1) which means you will need to change the presentation coordinates back to normal as these will appear inverted again.
Another issue is if you try to read pixels to generate an image. In all cases if you try to read it from the presentation buffer it will be inverted. But if you use the FBO and read pixels from that buffer the data should be correct (which is not your case).
So the true general solution is to respect the orientation when drawing to anything but the presentation buffer. The FBO matrix should not invert the Y coordinate, Y should increase upwards. In your case the best thing to do is use a separate ortho call: For the FBO simply flip the top and bottom compared to the presentation values.
上一篇: OpenGL纹理加载问题
下一篇: 在以前的框架顶部绘制一个离屏纹理