plugins: fix display type selection and propagation.

If vaapisink is in the GStreamer pipeline, then we shall allocate a
unique GstVaapiDisplay and propagate it upstream. i.e. subsequent
queries from vaapidecode shall get a valid answer from vaapisink.

7 files changed, 68 insertions, 17 deletions

diff --git a/gst/vaapi/gstvaapidecode.c b/gst/vaapi/gstvaapidecode.c
index 6c303f9f..253874ca 100644
--- a/gst/vaapi/gstvaapidecode.c
+++ b/gst/vaapi/gstvaapidecode.c
@@ -286,6 +286,13 @@ error_commit_buffer:
     }
 }
 
+static inline gboolean
+gst_vaapidecode_ensure_display(GstVaapiDecode *decode)
+{
+    return gst_vaapi_ensure_display(decode, GST_VAAPI_DISPLAY_TYPE_ANY,
+        &decode->display);
+}
+
 static gboolean
 gst_vaapidecode_create(GstVaapiDecode *decode, GstCaps *caps)
 {
@@ -293,7 +300,7 @@ gst_vaapidecode_create(GstVaapiDecode *decode, GstCaps *caps)
     GstStructure *structure;
     int version;
 
-    if (!gst_vaapi_ensure_display(decode, &decode->display, NULL))
+    if (!gst_vaapidecode_ensure_display(decode))
         return FALSE;
     dpy = decode->display;
 
@@ -526,7 +533,7 @@ gst_vaapidecode_ensure_allowed_caps(GstVaapiDecode *decode)
     if (decode->allowed_caps)
         return TRUE;
 
-    if (!gst_vaapi_ensure_display(decode, &decode->display, NULL))
+    if (!gst_vaapidecode_ensure_display(decode))
         goto error_no_display;
 
     decode_caps = gst_vaapi_display_get_decode_caps(decode->display);
diff --git a/gst/vaapi/gstvaapidownload.c b/gst/vaapi/gstvaapidownload.c
index f3e9caa4..69ce5d19 100644
--- a/gst/vaapi/gstvaapidownload.c
+++ b/gst/vaapi/gstvaapidownload.c
@@ -290,12 +290,19 @@ gst_vaapidownload_init(GstVaapiDownload *download)
     gst_object_unref(srcpad);
 }
 
+static inline gboolean
+gst_vaapidownload_ensure_display(GstVaapiDownload *download)
+{
+    return gst_vaapi_ensure_display(download, GST_VAAPI_DISPLAY_TYPE_ANY,
+        &download->display);
+}
+
 static gboolean
 gst_vaapidownload_start(GstBaseTransform *trans)
 {
     GstVaapiDownload * const download = GST_VAAPIDOWNLOAD(trans);
 
-    if (!gst_vaapi_ensure_display(download, &download->display, NULL))
+    if (!gst_vaapidownload_ensure_display(download))
         return FALSE;
     return TRUE;
 }
@@ -448,7 +455,7 @@ gst_vaapidownload_transform_caps(
     if (direction == GST_PAD_SINK) {
         if (!gst_structure_has_name(structure, GST_VAAPI_SURFACE_CAPS_NAME))
             return NULL;
-        if (!gst_vaapi_ensure_display(download, &download->display, NULL))
+        if (!gst_vaapidownload_ensure_display(download))
             return NULL;
         out_caps = gst_caps_from_string(gst_vaapidownload_yuv_caps_str);
 
diff --git a/gst/vaapi/gstvaapipluginutil.c b/gst/vaapi/gstvaapipluginutil.c
index ad8ce88c..6c731a69 100644
--- a/gst/vaapi/gstvaapipluginutil.c
+++ b/gst/vaapi/gstvaapipluginutil.c
@@ -66,12 +66,10 @@ static const DisplayMap g_display_map[] = {
 gboolean
 gst_vaapi_ensure_display(
     gpointer             element,
-    GstVaapiDisplay    **display_ptr,
-    GstVaapiDisplayType *display_type_ptr
+    GstVaapiDisplayType  display_type,
+    GstVaapiDisplay    **display_ptr
 )
 {
-    GstVaapiDisplayType display_type =
-        display_type_ptr ? *display_type_ptr : GST_VAAPI_DISPLAY_TYPE_ANY;
     GstVaapiDisplay *display;
     GstVideoContext *context;
     const DisplayMap *m;
@@ -85,8 +83,14 @@ gst_vaapi_ensure_display(
         return TRUE;
 
     context = GST_VIDEO_CONTEXT(element);
+    g_return_val_if_fail(context != NULL, FALSE);
+
     gst_video_context_prepare(context, display_types);
 
+    /* Neighbour found and it updated the display */
+    if (*display_ptr)
+        return TRUE;
+
     /* If no neighboor, or application not interested, use system default */
     for (m = g_display_map; m->type_str != NULL; m++) {
         if (display_type != GST_VAAPI_DISPLAY_TYPE_ANY &&
@@ -110,8 +114,6 @@ gst_vaapi_ensure_display(
 
     if (display_ptr)
         *display_ptr = display;
-    if (display_type_ptr)
-        *display_type_ptr = display_type;
     return display != NULL;
 }
 
diff --git a/gst/vaapi/gstvaapipluginutil.h b/gst/vaapi/gstvaapipluginutil.h
index 64aa8e9e..b55374be 100644
--- a/gst/vaapi/gstvaapipluginutil.h
+++ b/gst/vaapi/gstvaapipluginutil.h
@@ -30,8 +30,8 @@ G_GNUC_INTERNAL
 gboolean
 gst_vaapi_ensure_display(
     gpointer             element,
-    GstVaapiDisplay    **display,
-    GstVaapiDisplayType *display_type_ptr
+    GstVaapiDisplayType  display_type,
+    GstVaapiDisplay    **display
 );
 
 G_GNUC_INTERNAL
diff --git a/gst/vaapi/gstvaapipostproc.c b/gst/vaapi/gstvaapipostproc.c
index 59bd1b3d..8565a5e4 100644
--- a/gst/vaapi/gstvaapipostproc.c
+++ b/gst/vaapi/gstvaapipostproc.c
@@ -199,10 +199,17 @@ gst_video_context_interface_init(GstVideoContextInterface *iface)
     iface->set_context = gst_vaapipostproc_set_video_context;
 }
 
+static inline gboolean
+gst_vaapipostproc_ensure_display(GstVaapiPostproc *postproc)
+{
+    return gst_vaapi_ensure_display(postproc, GST_VAAPI_DISPLAY_TYPE_ANY,
+        &postproc->display);
+}
+
 static gboolean
 gst_vaapipostproc_create(GstVaapiPostproc *postproc, GstCaps *caps)
 {
-    if (!gst_vaapi_ensure_display(postproc, &postproc->display, NULL))
+    if (!gst_vaapipostproc_ensure_display(postproc))
         return FALSE;
 
     gst_caps_replace(&postproc->postproc_caps, caps);
@@ -231,7 +238,7 @@ gst_vaapipostproc_reset(GstVaapiPostproc *postproc, GstCaps *caps)
 static gboolean
 gst_vaapipostproc_start(GstVaapiPostproc *postproc)
 {
-    if (!gst_vaapi_ensure_display(postproc, &postproc->display, NULL))
+    if (!gst_vaapipostproc_ensure_display(postproc))
         return FALSE;
     return TRUE;
 }
diff --git a/gst/vaapi/gstvaapisink.c b/gst/vaapi/gstvaapisink.c
index ff43a065..f43b27fd 100644
--- a/gst/vaapi/gstvaapisink.c
+++ b/gst/vaapi/gstvaapisink.c
@@ -260,10 +260,32 @@ configure_notify_event_pending(
     return args.match;
 }
 
+static const gchar *
+get_display_type_name(GstVaapiDisplayType display_type)
+{
+    gpointer const klass = g_type_class_peek(GST_VAAPI_TYPE_DISPLAY_TYPE);
+    GEnumValue * const e = g_enum_get_value(klass, display_type);
+
+    if (e)
+        return e->value_name;
+    return "<unknown-type>";
+}
+
 static inline gboolean
 gst_vaapisink_ensure_display(GstVaapiSink *sink)
 {
-    return gst_vaapi_ensure_display(sink, &sink->display, &sink->display_type);
+    GstVaapiDisplayType display_type;
+
+    if (!gst_vaapi_ensure_display(sink, sink->display_type, &sink->display))
+        return FALSE;
+
+    display_type = gst_vaapi_display_get_display_type(sink->display);
+    if (display_type != sink->display_type) {
+        GST_INFO("created %s %p", get_display_type_name(display_type),
+            sink->display);
+        sink->display_type = display_type;
+    }
+    return TRUE;
 }
 
 static gboolean
diff --git a/gst/vaapi/gstvaapiupload.c b/gst/vaapi/gstvaapiupload.c
index d66b21d5..b4bc9fde 100644
--- a/gst/vaapi/gstvaapiupload.c
+++ b/gst/vaapi/gstvaapiupload.c
@@ -360,14 +360,20 @@ gst_vaapiupload_init(GstVaapiUpload *upload)
     g_object_unref(srcpad);
 }
 
+static inline gboolean
+gst_vaapiupload_ensure_display(GstVaapiUpload *upload)
+{
+    return gst_vaapi_ensure_display(upload, GST_VAAPI_DISPLAY_TYPE_ANY,
+        &upload->display);
+}
+
 static gboolean
 gst_vaapiupload_start(GstBaseTransform *trans)
 {
     GstVaapiUpload * const upload = GST_VAAPIUPLOAD(trans);
 
-    if (!gst_vaapi_ensure_display(upload, &upload->display, NULL))
+    if (!gst_vaapiupload_ensure_display(upload))
         return FALSE;
-
     return TRUE;
 }