xwayland: Check status in GBM pixmap creation

The current code in `xwl_glamor_gbm_create_pixmap_for_bo()` may fail in
several cases that are not checked for:

 - `eglCreateImageKHR()` may have failed to create the image,
 - `glEGLImageTargetTexture2DOES()` may fail and set an error,
 - `glamor_set_pixmap_texture()` may fail for very large pixmaps
    because the corresponding FBO could not be created.

Trying to upload content to a pixmap with no texture will crash Mesa,
glamor and Xwayland, e.g.:

  XXX fail to create fbo.
  (EE)
  (EE) Backtrace:
  (EE) 0: Xwayland (OsSigHandler+0x29)
  (EE) 1: libpthread.so.0 (funlockfile+0x50)
  (EE) 2: libc.so.6 (__memmove_avx_unaligned_erms+0x215)
  (EE) 3: dri/i965_dri.so (_mesa_format_convert+0xab3)
  (EE) 4: dri/i965_dri.so (_mesa_texstore+0x205)
  (EE) 5: dri/i965_dri.so (store_texsubimage+0x28c)
  (EE) 6: dri/i965_dri.so (intel_upload_tex+0x13b)
  (EE) 7: dri/i965_dri.so (texture_sub_image+0x134)
  (EE) 8: dri/i965_dri.so (texsubimage_err+0x150)
  (EE) 9: dri/i965_dri.so (_mesa_TexSubImage2D+0x48)
  (EE) 10: Xwayland (glamor_upload_boxes+0x246)
  (EE) 11: Xwayland (glamor_copy+0x4d1)
  (EE) 12: Xwayland (miCopyRegion+0x96)
  (EE) 13: Xwayland (miDoCopy+0x43c)
  (EE) 14: Xwayland (glamor_copy_area+0x24)
  (EE) 15: Xwayland (damageCopyArea+0xba)
  (EE) 16: Xwayland (compCopyWindow+0x31c)
  (EE) 17: Xwayland (damageCopyWindow+0xd3)
  (EE) 18: Xwayland (miResizeWindow+0x7b7)
  (EE) 19: Xwayland (compResizeWindow+0x3a)
  (EE) 20: Xwayland (ConfigureWindow+0xa96)
  (EE) 21: Xwayland (ProcConfigureWindow+0x7d)
  (EE) 22: Xwayland (Dispatch+0x320)
  (EE) 23: Xwayland (dix_main+0x366)
  (EE) 24: libc.so.6 (__libc_start_main+0xf3)
  (EE) 25: Xwayland (_start+0x2e)
  (EE)
  Fatal server error:
  (EE) Caught signal 11 (Segmentation fault). Server aborting
  (EE)

Check for the possible cases of failure above and fallback to the
regular glamor pixmap creation when an error is detected.

Signed-off-by: Olivier Fourdan <ofourdan@redhat.com>
Closes: https://gitlab.freedesktop.org/xorg/xserver/issues/661
(cherry picked from commit fc6380a11be4c6202ed72f241dd9ee8c7c24671d)

1 files changed, 30 insertions, 4 deletions

diff --git a/hw/xwayland/xwayland-glamor-gbm.c b/hw/xwayland/xwayland-glamor-gbm.c
index a211e0915..80146ab6e 100644
--- a/hw/xwayland/xwayland-glamor-gbm.c
+++ b/hw/xwayland/xwayland-glamor-gbm.c
@@ -169,6 +169,8 @@ xwl_glamor_gbm_create_pixmap_for_bo(ScreenPtr screen, struct gbm_bo *bo,
                                           xwl_screen->egl_context,
                                           EGL_NATIVE_PIXMAP_KHR,
                                           xwl_pixmap->bo, NULL);
+    if (xwl_pixmap->image == EGL_NO_IMAGE_KHR)
+      goto error;
 
     glGenTextures(1, &xwl_pixmap->texture);
     glBindTexture(GL_TEXTURE_2D, xwl_pixmap->texture);
@@ -176,14 +178,31 @@ xwl_glamor_gbm_create_pixmap_for_bo(ScreenPtr screen, struct gbm_bo *bo,
     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
 
     glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, xwl_pixmap->image);
-    glBindTexture(GL_TEXTURE_2D, 0);
+    if (eglGetError() != EGL_SUCCESS)
+      goto error;
 
-    xwl_pixmap_set_private(pixmap, xwl_pixmap);
+    glBindTexture(GL_TEXTURE_2D, 0);
 
     glamor_set_pixmap_texture(pixmap, xwl_pixmap->texture);
+    /* `set_pixmap_texture()` may fail silently if the FBO creation failed,
+     * so we check again the texture to be sure it worked.
+     */
+    if (!glamor_get_pixmap_texture(pixmap))
+      goto error;
+
     glamor_set_pixmap_type(pixmap, GLAMOR_TEXTURE_DRM);
+    xwl_pixmap_set_private(pixmap, xwl_pixmap);
 
     return pixmap;
+
+error:
+    if (xwl_pixmap->image != EGL_NO_IMAGE_KHR)
+      eglDestroyImageKHR(xwl_screen->egl_display, xwl_pixmap->image);
+    if (pixmap)
+      glamor_destroy_pixmap(pixmap);
+    free(xwl_pixmap);
+
+    return NULL;
 }
 
 static PixmapPtr
@@ -194,6 +213,7 @@ xwl_glamor_gbm_create_pixmap(ScreenPtr screen,
     struct xwl_screen *xwl_screen = xwl_screen_get(screen);
     struct xwl_gbm_private *xwl_gbm = xwl_gbm_get(xwl_screen);
     struct gbm_bo *bo;
+    PixmapPtr pixmap = NULL;
 
     if (width > 0 && height > 0 && depth >= 15 &&
         (hint == 0 ||
@@ -219,10 +239,16 @@ xwl_glamor_gbm_create_pixmap(ScreenPtr screen,
         }
 
         if (bo)
-            return xwl_glamor_gbm_create_pixmap_for_bo(screen, bo, depth);
+            pixmap = xwl_glamor_gbm_create_pixmap_for_bo(screen, bo, depth);
+
+        if (!pixmap)
+            gbm_bo_destroy(bo);
     }
 
-    return glamor_create_pixmap(screen, width, height, depth, hint);
+    if (!pixmap)
+        pixmap = glamor_create_pixmap(screen, width, height, depth, hint);
+
+    return pixmap;
 }
 
 static Bool