libs: display, context: handle broken jpeg decoder for i965 driver

JPEG decoding in i965 driver is pretty much broken, and the driver is
deprecated which mean authors only accept trivial fixes.

Surfaces for JPEG decoder context in i965 only handle IMC3[1] color
format which is not a common format in GStreamer. It can export it to
I420 at mapping raw bytes, but DMABuf exporting is problematic.

This patch artificially adds NV12 to the context format list when it's
JPEG decoder for i965 and force the usage of old VA-API for surface
creation without specifying color format. Also it artificially
disables the DMABuf announcement.

1. https://docs.microsoft.com/en-us/windows/win32/medfound/recommended-8-bit-yuv-formats-for-video-rendering#420-formats-16-bits-per-pixel

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer-vaapi/-/merge_requests/369>

3 files changed, 29 insertions, 1 deletions

diff --git a/gst-libs/gst/vaapi/gstvaapicontext.c b/gst-libs/gst/vaapi/gstvaapicontext.c
index 5283a13b..52fd751b 100644
--- a/gst-libs/gst/vaapi/gstvaapicontext.c
+++ b/gst-libs/gst/vaapi/gstvaapicontext.c
@@ -61,6 +61,17 @@ _init_vaapi_context_debug (void)
 #endif
 }
 
+static inline gboolean
+_context_is_broken_jpeg_decoder (GstVaapiContext * context)
+{
+  GstVaapiDisplay *const display = GST_VAAPI_CONTEXT_DISPLAY (context);
+
+  return (context->info.profile == GST_VAAPI_PROFILE_JPEG_BASELINE
+      && context->info.entrypoint == GST_VAAPI_ENTRYPOINT_VLD
+      && gst_vaapi_display_has_driver_quirks (display,
+          GST_VAAPI_DRIVER_QUIRK_JPEG_DEC_BROKEN_FORMATS));
+}
+
 static gboolean
 ensure_attributes (GstVaapiContext * context)
 {
@@ -70,7 +81,17 @@ ensure_attributes (GstVaapiContext * context)
   context->attribs =
       gst_vaapi_config_surface_attributes_get (GST_VAAPI_CONTEXT_DISPLAY
       (context), context->va_config);
-  return (context->attribs != NULL);
+
+  if (!context->attribs)
+    return FALSE;
+
+  if (_context_is_broken_jpeg_decoder (context)) {
+    GstVideoFormat fmt = GST_VIDEO_FORMAT_NV12;
+    g_array_prepend_val (context->attribs->formats, fmt);
+
+    context->attribs->mem_types &= ~VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME;
+  }
+  return TRUE;
 }
 
 /* XXX(victor): verify the preferred video format concords with the
@@ -87,6 +108,9 @@ ensure_preferred_format (GstVaapiContext * context)
   if (context->preferred_format != GST_VIDEO_FORMAT_UNKNOWN)
     return;
 
+  if (_context_is_broken_jpeg_decoder (context))
+    return;
+
   if (!ensure_attributes (context) || !context->attribs->formats)
     return;
 
diff --git a/gst-libs/gst/vaapi/gstvaapidisplay.c b/gst-libs/gst/vaapi/gstvaapidisplay.c
index 840e6c56..3a6841d0 100644
--- a/gst-libs/gst/vaapi/gstvaapidisplay.c
+++ b/gst-libs/gst/vaapi/gstvaapidisplay.c
@@ -808,6 +808,7 @@ set_driver_quirks (GstVaapiDisplay * display)
     { "i965", GST_VAAPI_DRIVER_QUIRK_MISSING_RGBA_IMAGE_FORMAT },
     { "iHD", GST_VAAPI_DRIVER_QUIRK_JPEG_ENC_SHIFT_VALUE_BY_50 },
     { "iHD", GST_VAAPI_DRIVER_QUIRK_HEVC_ENC_SLICE_NOT_SPAN_TILE },
+    { "i965", GST_VAAPI_DRIVER_QUIRK_JPEG_DEC_BROKEN_FORMATS },
   };
   /* *INDENT-ON* */
 
diff --git a/gst-libs/gst/vaapi/gstvaapidisplay.h b/gst-libs/gst/vaapi/gstvaapidisplay.h
index 9e690cb3..3eca38aa 100644
--- a/gst-libs/gst/vaapi/gstvaapidisplay.h
+++ b/gst-libs/gst/vaapi/gstvaapidisplay.h
@@ -109,6 +109,8 @@ typedef struct _GstVaapiDisplay                 GstVaapiDisplay;
  *   the value by 50 when calculating quantization from quality level
  * @GST_VAAPI_DRIVER_QUIRK_HEVC_ENC_SLICE_NOT_SPAN_TILE: The requirement
  *   that one slice should not span tiles when tile is enabled.
+ * @GST_VAAPI_DRIVER_QUIRK_JPEG_DEC_BROKEN_FORMATS: i965 driver does not
+ *   report all the handled formats for JPEG decoding.
  */
 typedef enum
 {
@@ -118,6 +120,7 @@ typedef enum
   GST_VAAPI_DRIVER_QUIRK_MISSING_RGBA_IMAGE_FORMAT = (1U << 3),
   GST_VAAPI_DRIVER_QUIRK_JPEG_ENC_SHIFT_VALUE_BY_50 = (1U << 4),
   GST_VAAPI_DRIVER_QUIRK_HEVC_ENC_SLICE_NOT_SPAN_TILE = (1U << 5),
+  GST_VAAPI_DRIVER_QUIRK_JPEG_DEC_BROKEN_FORMATS = (1U << 6),
 } GstVaapiDriverQuirks;
 
 /**