frontends/va: Return an error if non-interlaced buffer is not supported

Add a check to vaDeriveImage to see if a non-interlaced buffer was
created successfully. Otherwise, return an error, since we won't be able
to derive an image from the interlaced buffer.

Prevents a null pointer dereference from occuring on some nVidia cards,
reported by Alexander Kapshuk.

v2: Check for PIPE_VIDEO_CAP_SUPPORTS_PROGRESSIVE support (Ilia)

Fixes: fcb558321e6 ("frontends/va: Derive image from interlaced buffers")
Signed-off-by: Thong Thai <thong.thai@amd.com>
Tested-by: Alexander Kapshuk <alexander.kapshuk@gmail.com>
Reviewed-by: Leo Liu <leo.liu@amd.com>
Reviewed-by: Ilia Mirkin <imirkin@alum.mit.edu>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/8320>

1 files changed, 13 insertions, 2 deletions

diff --git a/src/gallium/frontends/va/image.c b/src/gallium/frontends/va/image.c
index 667b6293ec5..7a0d391e470 100644
--- a/src/gallium/frontends/va/image.c
+++ b/src/gallium/frontends/va/image.c
@@ -212,8 +212,8 @@ vlVaDeriveImage(VADriverContextP ctx, VASurfaceID surface, VAImage *image)
    /* This function is used by some programs to test for hardware decoding, but on
     * AMD devices, the buffers default to interlaced, which causes this function to fail.
     * Some programs expect this function to fail, while others, assume this means
-    * hardware acceleration is not available and give up without trying the fall-back 
-    * vaCreateImage + vaPutImage 
+    * hardware acceleration is not available and give up without trying the fall-back
+    * vaCreateImage + vaPutImage
     */
    const char *proc = util_get_process_name();
    const char *derive_interlaced_allowlist[] = {
@@ -245,6 +245,10 @@ vlVaDeriveImage(VADriverContextP ctx, VASurfaceID surface, VAImage *image)
 
       if (i >= ARRAY_SIZE(derive_interlaced_allowlist))
          return VA_STATUS_ERROR_OPERATION_FAILED;
+
+      if (!screen->get_video_param(screen, PIPE_VIDEO_PROFILE_UNKNOWN, PIPE_VIDEO_ENTRYPOINT_BITSTREAM,
+                                   PIPE_VIDEO_CAP_SUPPORTS_PROGRESSIVE))
+         return VA_STATUS_ERROR_OPERATION_FAILED;
    }
 
    surfaces = surf->buffer->get_surfaces(surf->buffer);
@@ -312,6 +316,13 @@ vlVaDeriveImage(VADriverContextP ctx, VASurfaceID surface, VAImage *image)
          new_template.interlaced = false;
          new_buffer = drv->pipe->create_video_buffer(drv->pipe, &new_template);
 
+         /* not all devices support non-interlaced buffers */
+         if (!new_buffer) {
+            FREE(img);
+            mtx_unlock(&drv->mutex);
+            return VA_STATUS_ERROR_OPERATION_FAILED;
+         }
+
          /* convert the interlaced to the progressive */
          src_rect.x0 = dst_rect.x0 = 0;
          src_rect.x1 = dst_rect.x1 = surf->templat.width;