libs: wayland: implement video overlay API

The Wayland sub-surfaces API is used to embed the video into an application
window.

See Appendix A. Wayland Protocol Specification as the following.

"""
The aim of sub-surfaces is to offload some of the compositing work
within a window from clients to the compositor. A prime example is
a video player with decorations and video in separate wl_surface
objects.

This should allow the compositor to pass YUV video buffer processing to
dedicated overlay hardware when possible.
"""

Added new method gst_vaapi_window_wayland_new_with_surface()

Original-Patch-By: Víctor Manuel Jáquez Leal <vjaquez@igalia.com>
                   Zhao Halley <halley.zhao@intel.com>
                   changzhix.wei@intel.com
                   Hyunjun Ko <zzoon@igalia.com>

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer-vaapi/-/merge_requests/342>

5 files changed, 89 insertions, 8 deletions

diff --git a/gst-libs/gst/vaapi/gstvaapidisplay_wayland.c b/gst-libs/gst/vaapi/gstvaapidisplay_wayland.c
index 7dfe8578..e6e3f394 100644
--- a/gst-libs/gst/vaapi/gstvaapidisplay_wayland.c
+++ b/gst-libs/gst/vaapi/gstvaapidisplay_wayland.c
@@ -151,6 +151,9 @@ registry_handle_global (void *data,
   if (strcmp (interface, "wl_compositor") == 0)
     priv->compositor =
         wl_registry_bind (registry, id, &wl_compositor_interface, 1);
+  else if (strcmp (interface, "wl_subcompositor") == 0)
+    priv->subcompositor =
+        wl_registry_bind (registry, id, &wl_subcompositor_interface, 1);
   else if (strcmp (interface, "wl_shell") == 0)
     priv->wl_shell = wl_registry_bind (registry, id, &wl_shell_interface, 1);
   else if (strcmp (interface, "xdg_wm_base") == 0) {
@@ -254,6 +257,7 @@ gst_vaapi_display_wayland_close_display (GstVaapiDisplay * display)
   g_clear_pointer (&priv->output, wl_output_destroy);
   g_clear_pointer (&priv->wl_shell, wl_shell_destroy);
   g_clear_pointer (&priv->xdg_wm_base, xdg_wm_base_destroy);
+  g_clear_pointer (&priv->subcompositor, wl_subcompositor_destroy);
   g_clear_pointer (&priv->compositor, wl_compositor_destroy);
   g_clear_pointer (&priv->registry, wl_registry_destroy);
 
diff --git a/gst-libs/gst/vaapi/gstvaapidisplay_wayland_priv.h b/gst-libs/gst/vaapi/gstvaapidisplay_wayland_priv.h
index 6822d822..a4e9992e 100644
--- a/gst-libs/gst/vaapi/gstvaapidisplay_wayland_priv.h
+++ b/gst-libs/gst/vaapi/gstvaapidisplay_wayland_priv.h
@@ -72,6 +72,7 @@ struct _GstVaapiDisplayWaylandPrivate
   struct wl_compositor *compositor;
   struct wl_shell *wl_shell;
   struct xdg_wm_base *xdg_wm_base;
+  struct wl_subcompositor *subcompositor;
   struct wl_output *output;
   struct zwp_linux_dmabuf_v1 *dmabuf;
   struct wl_registry *registry;
diff --git a/gst-libs/gst/vaapi/gstvaapiwindow.c b/gst-libs/gst/vaapi/gstvaapiwindow.c
index 19085408..01514038 100644
--- a/gst-libs/gst/vaapi/gstvaapiwindow.c
+++ b/gst-libs/gst/vaapi/gstvaapiwindow.c
@@ -544,10 +544,12 @@ gst_vaapi_window_set_size (GstVaapiWindow * window, guint width, guint height)
   if (!GST_VAAPI_WINDOW_GET_CLASS (window)->resize (window, width, height))
     return;
 
+  GST_VAAPI_WINDOW_LOCK_DISPLAY (window);
   gst_vaapi_video_pool_replace (&window->surface_pool, NULL);
 
   window->width = width;
   window->height = height;
+  GST_VAAPI_WINDOW_UNLOCK_DISPLAY (window);
 }
 
 static inline void
diff --git a/gst-libs/gst/vaapi/gstvaapiwindow_wayland.c b/gst-libs/gst/vaapi/gstvaapiwindow_wayland.c
index 4a0338ba..4cf76a15 100644
--- a/gst-libs/gst/vaapi/gstvaapiwindow_wayland.c
+++ b/gst-libs/gst/vaapi/gstvaapiwindow_wayland.c
@@ -40,6 +40,7 @@
 #include <unistd.h>
 
 GST_DEBUG_CATEGORY_EXTERN (gst_debug_vaapi_window);
+GST_DEBUG_CATEGORY_EXTERN (gst_debug_vaapi);
 #define GST_CAT_DEFAULT gst_debug_vaapi_window
 
 #define GST_VAAPI_WINDOW_WAYLAND_CAST(obj) \
@@ -104,6 +105,7 @@ struct _GstVaapiWindowWaylandPrivate
   struct xdg_toplevel *xdg_toplevel;
   struct wl_shell_surface *wl_shell_surface;
   struct wl_surface *surface;
+  struct wl_subsurface *video_subsurface;
   struct wl_region *opaque_region;
   struct wl_event_queue *event_queue;
   FrameState *last_frame;
@@ -350,6 +352,9 @@ gst_vaapi_window_wayland_set_fullscreen (GstVaapiWindow * window,
   GstVaapiWindowWaylandPrivate *const priv =
       GST_VAAPI_WINDOW_WAYLAND_GET_PRIVATE (window);
 
+  if (window->use_foreign_window)
+    return TRUE;
+
   if (!priv->is_shown) {
     priv->fullscreen_on_show = fullscreen;
     return TRUE;
@@ -403,8 +408,34 @@ gst_vaapi_window_wayland_create (GstVaapiWindow * window,
     return FALSE;
   wl_proxy_set_queue ((struct wl_proxy *) priv->surface, priv->event_queue);
 
-  /* Prefer XDG-shell over deprecated wl_shell (if available) */
-  if (priv_display->xdg_wm_base) {
+  if (window->use_foreign_window) {
+    struct wl_surface *wl_surface;
+
+    if (priv_display->subcompositor) {
+      if (GST_VAAPI_SURFACE_ID (window) == VA_INVALID_ID) {
+        GST_ERROR ("Invalid window");
+        return FALSE;
+      }
+
+      wl_surface = (struct wl_surface *) GST_VAAPI_WINDOW_ID (window);
+      GST_VAAPI_WINDOW_LOCK_DISPLAY (window);
+      priv->video_subsurface =
+          wl_subcompositor_get_subsurface (priv_display->subcompositor,
+          priv->surface, wl_surface);
+      GST_VAAPI_WINDOW_UNLOCK_DISPLAY (window);
+      if (!priv->video_subsurface)
+        return FALSE;
+
+      wl_proxy_set_queue ((struct wl_proxy *) priv->video_subsurface,
+          priv->event_queue);
+
+      wl_subsurface_set_desync (priv->video_subsurface);
+    } else {
+      GST_ERROR ("Wayland server does not support subsurfaces");
+      window->use_foreign_window = FALSE;
+    }
+    /* Prefer XDG-shell over deprecated wl_shell (if available) */
+  } else if (priv_display->xdg_wm_base) {
     /* Create the XDG surface. We make the toplevel on VaapiWindow::show() */
     GST_VAAPI_WINDOW_LOCK_DISPLAY (window);
     priv->xdg_surface = xdg_wm_base_get_xdg_surface (priv_display->xdg_wm_base,
@@ -467,6 +498,7 @@ gst_vaapi_window_wayland_finalize (GObject * object)
 
   g_clear_pointer (&priv->xdg_surface, xdg_surface_destroy);
   g_clear_pointer (&priv->wl_shell_surface, wl_shell_surface_destroy);
+  g_clear_pointer (&priv->video_subsurface, wl_subsurface_destroy);
   g_clear_pointer (&priv->surface, wl_surface_destroy);
   g_clear_pointer (&priv->event_queue, wl_event_queue_destroy);
 
@@ -484,6 +516,9 @@ gst_vaapi_window_wayland_resize (GstVaapiWindow * window,
   GstVaapiDisplayWaylandPrivate *const priv_display =
       GST_VAAPI_DISPLAY_WAYLAND_GET_PRIVATE (GST_VAAPI_WINDOW_DISPLAY (window));
 
+  if (window->use_foreign_window)
+    return TRUE;
+
   GST_DEBUG ("resize window, new size %ux%u", width, height);
 
   if (priv->opaque_region)
@@ -630,8 +665,7 @@ choose_next_format (GstVaapiDisplay * const display, gint * next_index)
 
 static GstVaapiDmabufStatus
 dmabuf_buffer_from_surface (GstVaapiWindow * window, GstVaapiSurface * surface,
-    const GstVaapiRectangle * rect, guint va_flags,
-    struct wl_buffer **out_buffer)
+    guint va_flags, struct wl_buffer **out_buffer)
 {
   GstVaapiDisplay *const display = GST_VAAPI_WINDOW_DISPLAY (window);
   GstVaapiDisplayWaylandPrivate *const priv_display =
@@ -690,8 +724,8 @@ dmabuf_buffer_from_surface (GstVaapiWindow * window, GstVaapiSurface * surface,
     }
   }
 
-  buffer = zwp_linux_buffer_params_v1_create_immed (params, rect->width,
-      rect->height, format, 0);
+  buffer = zwp_linux_buffer_params_v1_create_immed (params, window->width,
+      window->height, format, 0);
 
   if (!buffer)
     ret = GST_VAAPI_DMABUF_NOT_SUPPORTED;
@@ -754,8 +788,7 @@ again:
     }
   }
   if (!priv->dmabuf_broken) {
-    ret = dmabuf_buffer_from_surface (window, surface, dst_rect, va_flags,
-        buffer);
+    ret = dmabuf_buffer_from_surface (window, surface, va_flags, buffer);
     switch (ret) {
       case GST_VAAPI_DMABUF_SUCCESS:
         goto out;
@@ -843,6 +876,11 @@ gst_vaapi_window_wayland_render (GstVaapiWindow * window,
   guint width, height;
   gboolean ret;
 
+  /* Skip rendering without valid window size. This can happen with a foreign
+     window if the render rectangle is not yet set. */
+  if (window->width == 0 || window->height == 0)
+    return TRUE;
+
   /* Check that we don't need to crop source VA surface */
   gst_vaapi_surface_get_size (surface, &width, &height);
   if (src_rect->x != 0 || src_rect->y != 0)
@@ -856,6 +894,11 @@ gst_vaapi_window_wayland_render (GstVaapiWindow * window,
   if (dst_rect->width != window->width || dst_rect->height != window->height)
     priv->need_vpp = TRUE;
 
+  /* Check that the surface has the correct size for the window */
+  if (dst_rect->width != src_rect->width ||
+      dst_rect->height != src_rect->height)
+    priv->need_vpp = TRUE;
+
   ret = buffer_from_surface (window, &surface, src_rect, dst_rect, flags,
       &buffer);
   if (!ret)
@@ -980,3 +1023,30 @@ gst_vaapi_window_wayland_new (GstVaapiDisplay * display,
   return gst_vaapi_window_new_internal (GST_TYPE_VAAPI_WINDOW_WAYLAND, display,
       GST_VAAPI_ID_INVALID, width, height);
 }
+
+/**
+ * gst_vaapi_window_wayland_new_with_surface:
+ * @display: a #GstVaapiDisplay
+ * @wl_surface: a Wayland surface pointer
+ *
+ * Creates a window with the specified @wl_surface. The window
+ * will be attached to the @display and remains invisible to the user
+ * until gst_vaapi_window_show() is called.
+ *
+ * Return value (transfer full): the newly allocated #GstVaapiWindow object
+ *
+ * Since: 1.18
+ */
+GstVaapiWindow *
+gst_vaapi_window_wayland_new_with_surface (GstVaapiDisplay * display,
+    guintptr wl_surface)
+{
+  g_return_val_if_fail (GST_VAAPI_IS_DISPLAY_WAYLAND (display), NULL);
+  g_return_val_if_fail (wl_surface, NULL);
+
+  GST_CAT_DEBUG (gst_debug_vaapi, "new window from surface 0x%"
+      G_GINTPTR_MODIFIER "x", wl_surface);
+
+  return gst_vaapi_window_new_internal (GST_TYPE_VAAPI_WINDOW_WAYLAND, display,
+      wl_surface, 0, 0);
+}
diff --git a/gst-libs/gst/vaapi/gstvaapiwindow_wayland.h b/gst-libs/gst/vaapi/gstvaapiwindow_wayland.h
index d4eb427a..016f5825 100644
--- a/gst-libs/gst/vaapi/gstvaapiwindow_wayland.h
+++ b/gst-libs/gst/vaapi/gstvaapiwindow_wayland.h
@@ -45,6 +45,10 @@ GstVaapiWindow *
 gst_vaapi_window_wayland_new (GstVaapiDisplay * display, guint width,
     guint height);
 
+GstVaapiWindow *
+gst_vaapi_window_wayland_new_with_surface (GstVaapiDisplay * display,
+    guintptr wl_surface);
+
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstVaapiWindowWayland, gst_object_unref)
 
 G_END_DECLS