xwayland: clear pixmaps after creation in rootless mode

When a pixmap is created with a backing FBO, the FBO should be cleared
to avoid rendering uninitialized memory. This could happen when the
pixmap is rendered without being filled in its entirety.

One example is when a top-level window without a background is
resized. The pixmap would be reallocated to prepare for more pixels,
but uninitialized memory would be rendered in the resize offset until
the client sends a frame that fills these additional pixels.

Another example is when a new top-level window is created without a
background. Uninitialized memory would be rendered after the pixmap is
allocated and before the client sends its first frame.

This issue is only apparent in OpenGL implementations that don't zero
the VRAM of allocated buffers by default, such as RadeonSI.

Signed-off-by: Dor Askayo <dor.askayo@gmail.com>
Closes: https://gitlab.freedesktop.org/xorg/xserver/issues/636
Reviewed-by: Michel Dänzer <mdaenzer@redhat.com>
(cherry picked from commit 0e9a0c203c2ae4eae12bdbb95428f398211c7bee)

[ Michel Dänzer:
* Squashed in commit ebf549db2d9341d99e0d0847b948dd798d98f7dc
* Dropped code related to glamor_format, which only exists on master ]

5 files changed, 36 insertions, 1 deletions

diff --git a/glamor/glamor.c b/glamor/glamor.c
index a6cc798f8..abefef614 100644
--- a/glamor/glamor.c
+++ b/glamor/glamor.c
@@ -128,6 +128,21 @@ glamor_set_pixmap_texture(PixmapPtr pixmap, unsigned int tex)
     glamor_pixmap_attach_fbo(pixmap, fbo);
 }
 
+_X_EXPORT void
+glamor_clear_pixmap(PixmapPtr pixmap)
+{
+    ScreenPtr screen = pixmap->drawable.pScreen;
+    glamor_screen_private *glamor_priv;
+    glamor_pixmap_private *pixmap_priv;
+
+    glamor_priv = glamor_get_screen_private(screen);
+    pixmap_priv = glamor_get_pixmap_private(pixmap);
+
+    assert(pixmap_priv->fbo != NULL);
+
+    glamor_pixmap_clear_fbo(glamor_priv, pixmap_priv->fbo);
+}
+
 uint32_t
 glamor_get_pixmap_texture(PixmapPtr pixmap)
 {
diff --git a/glamor/glamor.h b/glamor/glamor.h
index 3f9d265db..be04bf858 100644
--- a/glamor/glamor.h
+++ b/glamor/glamor.h
@@ -115,6 +115,9 @@ extern _X_EXPORT void glamor_set_pixmap_texture(PixmapPtr pixmap,
 
 extern _X_EXPORT void glamor_set_pixmap_type(PixmapPtr pixmap,
                                              glamor_pixmap_type_t type);
+
+extern _X_EXPORT void glamor_clear_pixmap(PixmapPtr pixmap);
+
 extern _X_EXPORT void glamor_block_handler(ScreenPtr screen);
 
 extern _X_EXPORT PixmapPtr glamor_create_pixmap(ScreenPtr screen, int w, int h,
diff --git a/glamor/glamor_fbo.c b/glamor/glamor_fbo.c
index e8c4330b3..f939a6c2f 100644
--- a/glamor/glamor_fbo.c
+++ b/glamor/glamor_fbo.c
@@ -239,6 +239,18 @@ glamor_create_fbo_array(glamor_screen_private *glamor_priv,
     return NULL;
 }
 
+void
+glamor_pixmap_clear_fbo(glamor_screen_private *glamor_priv, glamor_pixmap_fbo *fbo)
+{
+    glamor_make_current(glamor_priv);
+
+    assert(fbo->fb != 0 && fbo->tex != 0);
+
+    glamor_set_destination_pixmap_fbo(glamor_priv, fbo, 0, 0, fbo->width, fbo->height);
+    glClearColor(0.0, 0.0, 0.0, 0.0);
+    glClear(GL_COLOR_BUFFER_BIT);
+}
+
 glamor_pixmap_fbo *
 glamor_pixmap_detach_fbo(glamor_pixmap_private *pixmap_priv)
 {
diff --git a/glamor/glamor_priv.h b/glamor/glamor_priv.h
index 1686ef5a4..4353a99f1 100644
--- a/glamor/glamor_priv.h
+++ b/glamor/glamor_priv.h
@@ -539,6 +539,7 @@ void glamor_destroy_fbo(glamor_screen_private *glamor_priv,
                         glamor_pixmap_fbo *fbo);
 void glamor_pixmap_destroy_fbo(PixmapPtr pixmap);
 Bool glamor_pixmap_fbo_fixup(ScreenPtr screen, PixmapPtr pixmap);
+void glamor_pixmap_clear_fbo(glamor_screen_private *glamor_priv, glamor_pixmap_fbo *fbo);
 
 /* Return whether 'picture' is alpha-only */
 static inline Bool glamor_picture_is_alpha(PicturePtr picture)
diff --git a/hw/xwayland/xwayland-glamor-gbm.c b/hw/xwayland/xwayland-glamor-gbm.c
index febc0b910..c02ba7363 100644
--- a/hw/xwayland/xwayland-glamor-gbm.c
+++ b/hw/xwayland/xwayland-glamor-gbm.c
@@ -242,8 +242,12 @@ xwl_glamor_gbm_create_pixmap(ScreenPtr screen,
         if (bo) {
             pixmap = xwl_glamor_gbm_create_pixmap_for_bo(screen, bo, depth);
 
-            if (!pixmap)
+            if (!pixmap) {
                 gbm_bo_destroy(bo);
+            }
+            else if (xwl_screen->rootless && hint == CREATE_PIXMAP_USAGE_BACKING_PIXMAP) {
+                glamor_clear_pixmap(pixmap);
+            }
         }
     }