eglglessink: Add support for GstVideoGLTextureUploadMeta

3 files changed, 127 insertions, 39 deletions

diff --git a/ext/eglgles/gsteglglessink.c b/ext/eglgles/gsteglglessink.c
index 2583ce420..fb0b7b068 100644
--- a/ext/eglgles/gsteglglessink.c
+++ b/ext/eglgles/gsteglglessink.c
@@ -192,7 +192,7 @@ static const char *frag_BLACK_prog = {
       "}"
 };
 
-/* Direct fragments copy */
+/* Direct fragments copy with stride-scaling */
 static const char *frag_COPY_prog = {
   "precision mediump float;"
       "varying vec2 opos;"
@@ -207,6 +207,18 @@ static const char *frag_COPY_prog = {
       "}"
 };
 
+/* Direct fragments copy without stride-scaling */
+static const char *frag_COPY_DIRECT_prog = {
+  "precision mediump float;"
+      "varying vec2 opos;"
+      "uniform sampler2D tex;"
+      "void main(void)"
+      "{"
+      " vec4 t = texture2D(tex, opos);"
+      " gl_FragColor = vec4(t.rgb, 1.0);"
+      "}"
+};
+
 /* Channel reordering for XYZ <-> ZYX conversion */
 static const char *frag_REORDER_prog = {
   "precision mediump float;"
@@ -643,13 +655,13 @@ gst_eglglessink_wipe_eglglesctx (GstEglGlesSink * eglglessink)
   }
 
   if (eglglessink->have_texture) {
-    glDeleteTextures (eglglessink->eglglesctx.n_textures,
+    glDeleteTextures (eglglessink->eglglesctx.n_textures + 1,
         eglglessink->eglglesctx.texture);
     eglglessink->have_texture = FALSE;
     eglglessink->eglglesctx.n_textures = 0;
   }
 
-  for (i = 0; i < 2; i++) {
+  for (i = 0; i < 3; i++) {
     if (eglglessink->eglglesctx.glslprogram[i]) {
       glDetachShader (eglglessink->eglglesctx.glslprogram[i],
           eglglessink->eglglesctx.fragshader[i]);
@@ -1414,6 +1426,34 @@ gst_eglglessink_init_egl_surface (GstEglGlesSink * eglglessink)
         texnames[i]);
   }
 
+  /* custom rendering shader */
+
+  if (!create_shader_program (eglglessink,
+          &eglglessink->eglglesctx.glslprogram[2],
+          &eglglessink->eglglesctx.vertshader[2],
+          &eglglessink->eglglesctx.fragshader[2], vert_COPY_prog,
+          frag_COPY_DIRECT_prog)) {
+    if (free_frag_prog)
+      g_free (frag_prog);
+    frag_prog = NULL;
+    goto HANDLE_ERROR;
+  }
+  if (free_frag_prog)
+    g_free (frag_prog);
+  frag_prog = NULL;
+
+  eglglessink->eglglesctx.position_loc[2] =
+      glGetAttribLocation (eglglessink->eglglesctx.glslprogram[2], "position");
+  eglglessink->eglglesctx.texpos_loc[1] =
+      glGetAttribLocation (eglglessink->eglglesctx.glslprogram[2], "texpos");
+
+  glEnableVertexAttribArray (eglglessink->eglglesctx.position_loc[2]);
+  if (got_gl_error ("glEnableVertexAttribArray"))
+    goto HANDLE_ERROR;
+
+  eglglessink->eglglesctx.tex_loc[1][0] =
+      glGetUniformLocation (eglglessink->eglglesctx.glslprogram[2], "tex");
+
   if (!eglglessink->eglglesctx.buffer_preserved) {
     /* Build shader program for black borders */
     if (!create_shader_program (eglglessink,
@@ -2119,9 +2159,12 @@ gst_eglglessink_upload (GstEglGlesSink * eglglessink, GstBuffer * buf)
     GST_DEBUG_OBJECT (eglglessink, "Rendering previous buffer again");
   } else if (buf) {
     GstMemory *mem;
+    GstVideoGLTextureUploadMeta *upload_meta;
 
     crop = gst_buffer_get_video_crop_meta (buf);
 
+    upload_meta = gst_buffer_get_video_gl_texture_upload_meta (buf);
+
     if (gst_eglglessink_crop_changed (eglglessink, crop)) {
       if (crop) {
         eglglessink->crop.x = crop->x;
@@ -2137,11 +2180,24 @@ gst_eglglessink_upload (GstEglGlesSink * eglglessink, GstBuffer * buf)
       eglglessink->crop_changed = TRUE;
     }
 
-    if (gst_buffer_n_memory (buf) >= 1 &&
+    if (upload_meta) {
+      glActiveTexture (GL_TEXTURE0);
+      glBindTexture (GL_TEXTURE_2D, eglglessink->eglglesctx.texture[0]);
+      if (!gst_video_gl_texture_upload_meta_upload (upload_meta,
+              (eglglessink->configured_info.finfo->format ==
+                  GST_VIDEO_FORMAT_RGBA ? GL_RGBA : GL_RGB),
+              eglglessink->eglglesctx.texture[0]))
+        goto HANDLE_ERROR;
+
+      eglglessink->orientation = GST_EGL_IMAGE_ORIENTATION_X_NORMAL_Y_NORMAL;
+      eglglessink->custom_format = TRUE;
+    } else if (gst_buffer_n_memory (buf) >= 1 &&
         (mem = gst_buffer_peek_memory (buf, 0))
         && gst_is_egl_image_memory (mem)) {
       guint n, i;
 
+      eglglessink->custom_format = FALSE;
+
       n = gst_buffer_n_memory (buf);
 
       for (i = 0; i < n; i++) {
@@ -2175,6 +2231,8 @@ gst_eglglessink_upload (GstEglGlesSink * eglglessink, GstBuffer * buf)
       eglglessink->stride[1] = 1;
       eglglessink->stride[2] = 1;
     } else {
+      eglglessink->custom_format = FALSE;
+
       eglglessink->orientation = GST_EGL_IMAGE_ORIENTATION_X_NORMAL_Y_NORMAL;
       if (!gst_eglglessink_fill_texture (eglglessink, buf))
         goto HANDLE_ERROR;
@@ -2314,45 +2372,67 @@ gst_eglglessink_render (GstEglGlesSink * eglglessink)
 
   /* Draw video frame */
   GST_DEBUG_OBJECT (eglglessink, "Drawing video frame");
-  glUseProgram (eglglessink->eglglesctx.glslprogram[0]);
 
-  glUniform2f (eglglessink->eglglesctx.tex_scale_loc[0][0],
-      eglglessink->stride[0], 1);
-  glUniform2f (eglglessink->eglglesctx.tex_scale_loc[0][1],
-      eglglessink->stride[1], 1);
-  glUniform2f (eglglessink->eglglesctx.tex_scale_loc[0][2],
-      eglglessink->stride[2], 1);
+  if (eglglessink->custom_format) {
+    glUseProgram (eglglessink->eglglesctx.glslprogram[2]);
 
-  for (i = 0; i < eglglessink->eglglesctx.n_textures; i++) {
-    glUniform1i (eglglessink->eglglesctx.tex_loc[0][i], i);
+    glUniform1i (eglglessink->eglglesctx.tex_loc[1][0], 0);
     if (got_gl_error ("glUniform1i"))
       goto HANDLE_ERROR;
-  }
 
-  if (eglglessink->orientation == GST_EGL_IMAGE_ORIENTATION_X_NORMAL_Y_NORMAL) {
-    glVertexAttribPointer (eglglessink->eglglesctx.position_loc[0], 3,
-        GL_FLOAT, GL_FALSE, sizeof (coord5), (gpointer) (0 * sizeof (coord5)));
+    glVertexAttribPointer (eglglessink->eglglesctx.position_loc[2], 3,
+        GL_FLOAT, GL_FALSE, sizeof (coord5), (gpointer) (0));
     if (got_gl_error ("glVertexAttribPointer"))
       goto HANDLE_ERROR;
 
-    glVertexAttribPointer (eglglessink->eglglesctx.texpos_loc[0], 2,
+    glVertexAttribPointer (eglglessink->eglglesctx.texpos_loc[1], 2,
         GL_FLOAT, GL_FALSE, sizeof (coord5), (gpointer) (3 * sizeof (gfloat)));
     if (got_gl_error ("glVertexAttribPointer"))
       goto HANDLE_ERROR;
-  } else if (eglglessink->orientation ==
-      GST_EGL_IMAGE_ORIENTATION_X_NORMAL_Y_FLIP) {
-    glVertexAttribPointer (eglglessink->eglglesctx.position_loc[0], 3, GL_FLOAT,
-        GL_FALSE, sizeof (coord5), (gpointer) (4 * sizeof (coord5)));
-    if (got_gl_error ("glVertexAttribPointer"))
-      goto HANDLE_ERROR;
-
-    glVertexAttribPointer (eglglessink->eglglesctx.texpos_loc[0], 2,
-        GL_FLOAT, GL_FALSE, sizeof (coord5),
-        (gpointer) (4 * sizeof (coord5) + 3 * sizeof (gfloat)));
-    if (got_gl_error ("glVertexAttribPointer"))
-      goto HANDLE_ERROR;
   } else {
-    g_assert_not_reached ();
+    glUseProgram (eglglessink->eglglesctx.glslprogram[0]);
+
+    glUniform2f (eglglessink->eglglesctx.tex_scale_loc[0][0],
+        eglglessink->stride[0], 1);
+    glUniform2f (eglglessink->eglglesctx.tex_scale_loc[0][1],
+        eglglessink->stride[1], 1);
+    glUniform2f (eglglessink->eglglesctx.tex_scale_loc[0][2],
+        eglglessink->stride[2], 1);
+
+    for (i = 0; i < eglglessink->eglglesctx.n_textures; i++) {
+      glUniform1i (eglglessink->eglglesctx.tex_loc[0][i], i);
+      if (got_gl_error ("glUniform1i"))
+        goto HANDLE_ERROR;
+    }
+
+    if (eglglessink->orientation == GST_EGL_IMAGE_ORIENTATION_X_NORMAL_Y_NORMAL) {
+      glVertexAttribPointer (eglglessink->eglglesctx.position_loc[0], 3,
+          GL_FLOAT, GL_FALSE, sizeof (coord5),
+          (gpointer) (0 * sizeof (coord5)));
+      if (got_gl_error ("glVertexAttribPointer"))
+        goto HANDLE_ERROR;
+
+      glVertexAttribPointer (eglglessink->eglglesctx.texpos_loc[0], 2,
+          GL_FLOAT, GL_FALSE, sizeof (coord5),
+          (gpointer) (3 * sizeof (gfloat)));
+      if (got_gl_error ("glVertexAttribPointer"))
+        goto HANDLE_ERROR;
+    } else if (eglglessink->orientation ==
+        GST_EGL_IMAGE_ORIENTATION_X_NORMAL_Y_FLIP) {
+      glVertexAttribPointer (eglglessink->eglglesctx.position_loc[0], 3,
+          GL_FLOAT, GL_FALSE, sizeof (coord5),
+          (gpointer) (4 * sizeof (coord5)));
+      if (got_gl_error ("glVertexAttribPointer"))
+        goto HANDLE_ERROR;
+
+      glVertexAttribPointer (eglglessink->eglglesctx.texpos_loc[0], 2,
+          GL_FLOAT, GL_FALSE, sizeof (coord5),
+          (gpointer) (4 * sizeof (coord5) + 3 * sizeof (gfloat)));
+      if (got_gl_error ("glVertexAttribPointer"))
+        goto HANDLE_ERROR;
+    } else {
+      g_assert_not_reached ();
+    }
   }
 
   glDrawElements (GL_TRIANGLE_STRIP, 4, GL_UNSIGNED_SHORT, 0);
@@ -2438,6 +2518,7 @@ gst_eglglessink_propose_allocation (GstBaseSink * bsink, GstQuery * query)
   GstBufferPool *pool;
   GstStructure *config;
   GstCaps *caps;
+  GstVideoInfo info;
   gboolean need_pool;
   guint size;
   GstAllocator *allocator;
@@ -2453,6 +2534,11 @@ gst_eglglessink_propose_allocation (GstBaseSink * bsink, GstQuery * query)
     return FALSE;
   }
 
+  if (!gst_video_info_from_caps (&info, caps)) {
+    GST_ERROR_OBJECT (eglglessink, "allocation query with invalid caps");
+    return FALSE;
+  }
+
   GST_OBJECT_LOCK (eglglessink);
   pool = eglglessink->pool ? gst_object_ref (eglglessink->pool) : NULL;
   GST_OBJECT_UNLOCK (eglglessink);
@@ -2523,6 +2609,8 @@ gst_eglglessink_propose_allocation (GstBaseSink * bsink, GstQuery * query)
 
   gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL);
   gst_query_add_allocation_meta (query, GST_VIDEO_CROP_META_API_TYPE, NULL);
+  gst_query_add_allocation_meta (query,
+      GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, NULL);
 
   return TRUE;
 }
diff --git a/ext/eglgles/gsteglglessink.h b/ext/eglgles/gsteglglessink.h
index 32351ec63..d1cfbb736 100644
--- a/ext/eglgles/gsteglglessink.h
+++ b/ext/eglgles/gsteglglessink.h
@@ -124,10 +124,10 @@ struct _GstEglGlesRenderContext
   EGLNativeWindowType window, used_window;
   EGLSurface surface;
   gboolean buffer_preserved;
-  GLuint fragshader[2]; /* frame, border */
-  GLuint vertshader[2]; /* frame, border */
-  GLuint glslprogram[2]; /* frame, border */
-  GLuint texture[3]; /* RGB/Y, U/UV, V */
+  GLuint fragshader[3]; /* frame, border, frame-custom */
+  GLuint vertshader[3]; /* frame, border, frame-custom */
+  GLuint glslprogram[3]; /* frame, border, frame-custom */
+  GLuint texture[4]; /* RGB/Y, U/UV, V, custom */
   EGLint surface_width;
   EGLint surface_height;
   EGLint pixel_aspect_ratio;
@@ -135,10 +135,10 @@ struct _GstEglGlesRenderContext
   gint n_textures;
 
   /* shader vars */
-  GLuint position_loc[2]; /* frame, border */
-  GLuint texpos_loc[1]; /* frame */
+  GLuint position_loc[3]; /* frame, border, frame-custom */
+  GLuint texpos_loc[2]; /* frame, frame-custom */
   GLuint tex_scale_loc[1][3]; /* [frame] RGB/Y, U/UV, V */
-  GLuint tex_loc[1][3]; /* [frame] RGB/Y, U/UV, V */
+  GLuint tex_loc[2][3]; /* [frame,frame-custom] RGB/Y, U/UV, V */
   coord5 position_array[16];    /* 4 x Frame x-normal,y-normal, 4x Frame x-normal,y-flip, 4 x Border1, 4 x Border2 */
   unsigned short index_array[4];
   unsigned int position_buffer, index_buffer;
@@ -185,6 +185,7 @@ struct _GstEglGlesSink
   GstVideoInfo configured_info;
   gfloat stride[3];
   GstEGLImageOrientation orientation;
+  gboolean custom_format; /* If it's a single texture that is just copied */
   GstBufferPool *pool;
 
   GstEglGlesRenderContext eglglesctx;
diff --git a/ext/eglgles/video_platform_wrapper.c b/ext/eglgles/video_platform_wrapper.c
index e8fcdbfa9..0cc1eb499 100644
--- a/ext/eglgles/video_platform_wrapper.c
+++ b/ext/eglgles/video_platform_wrapper.c
@@ -689,4 +689,3 @@ platform_destroy_native_window (EGLNativeDisplayType display,
   return TRUE;
 }
 #endif
-