rootless: Protect alpha channel for Render operationsXQuartz-2.8.6_beta3

The Render extension operates on PictFormats rather than GC, so it
bypasses the rootless layer's GC-based alpha protection
(ROOTLESS_PROTECT_ALPHA).  Depth-24 windows use PICT_x8r8g8b8 where
the 'x' tells pixman the high byte is padding it may freely zero.
The macOS compositor needs this byte to be 0xFF (opaque).

Before each Render operation that targets a rootless window,
temporarily upgrade the destination Picture's format from
PICT_x8r8g8b8 to PICT_a8r8g8b8.  This tells pixman that the alpha
channel is significant and must not be optimized away.  The format
is restored after the operation completes.

This parallels how ROOTLESS_PROTECT_ALPHA handles GC operations (by
masking alpha out of the planemask) and how ROOTLESS_SAFEALPHA
handles PaintWindow (by forcing alpha in solid fills).  The
save/restore approach is analogous to rootlessGC.c's GC_SAVE /
GC_RESTORE pattern.

Fixes [2/2]: https://github.com/XQuartz/XQuartz/issues/31

Signed-off-by: Jeremy Huddleston Sequoia <jeremyhu@apple.com>

1 files changed, 34 insertions, 0 deletions

diff --git a/miext/rootless/rootlessScreen.c b/miext/rootless/rootlessScreen.c
index 6f6ab89b4..1d8a3a21c 100644
--- a/miext/rootless/rootlessScreen.c
+++ b/miext/rootless/rootlessScreen.c
@@ -52,6 +52,30 @@
 #include "rootlessCommon.h"
 #include "rootlessWindow.h"
 
+/*
+ * Render operations use PictFormat to describe pixel layout.  Depth-24
+ * windows use PICT_x8r8g8b8, where 'x' tells pixman the high byte is
+ * padding it may freely zero.  The compositor needs this byte to be 0xFF
+ * (opaque).  Temporarily upgrading the destination format from 'x' to 'a'
+ * prevents pixman from optimizing away the alpha channel, paralleling how
+ * ROOTLESS_PROTECT_ALPHA prevents fb from doing the same for GC ops.
+ */
+
+#if ROOTLESS_PROTECT_ALPHA
+#define RL_RENDER_SAVE_FORMAT(pict)                              \
+    CARD32 _saved_format = (pict)->format;                       \
+    if ((pict)->pDrawable->type == DRAWABLE_WINDOW &&            \
+        (pict)->format == PICT_x8r8g8b8)                         \
+        (pict)->format = PICT_a8r8g8b8
+
+#define RL_RENDER_RESTORE_FORMAT(pict) \
+    (pict)->format = _saved_format
+
+#else
+#define RL_RENDER_SAVE_FORMAT(pict)
+#define RL_RENDER_RESTORE_FORMAT(pict)
+#endif
+
 extern int RootlessMiValidateTree(WindowPtr pRoot, WindowPtr pChild,
                                   VTKind kind);
 extern Bool RootlessCreateGC(GCPtr pGC);
@@ -258,8 +282,10 @@ RootlessComposite(CARD8 op, PicturePtr pSrc, PicturePtr pMask, PicturePtr pDst,
     if (dstWin && IsFramedWindow(dstWin))
         RootlessStartDrawing(dstWin);
 
+    RL_RENDER_SAVE_FORMAT(pDst);
     ps->Composite(op, pSrc, pMask, pDst,
                   xSrc, ySrc, xMask, yMask, xDst, yDst, width, height);
+    RL_RENDER_RESTORE_FORMAT(pDst);
 
     if (dstWin && IsFramedWindow(dstWin)) {
         RootlessDamageRect(dstWin, xDst, yDst, width, height);
@@ -293,7 +319,9 @@ RootlessGlyphs(CARD8 op, PicturePtr pSrc, PicturePtr pDst,
 
     //SCREEN_UNWRAP(ps, Glyphs);
     ps->Glyphs = SCREENREC(pScreen)->Glyphs;
+    RL_RENDER_SAVE_FORMAT(pDst);
     ps->Glyphs(op, pSrc, pDst, maskFormat, xSrc, ySrc, nlist, list, glyphs);
+    RL_RENDER_RESTORE_FORMAT(pDst);
     ps->Glyphs = RootlessGlyphs;
     //SCREEN_WRAP(ps, Glyphs);
 
@@ -375,7 +403,9 @@ RootlessTrapezoids(CARD8 op, PicturePtr pSrc, PicturePtr pDst,
     if (dstWin && IsFramedWindow(dstWin))
         RootlessStartDrawing(dstWin);
 
+    RL_RENDER_SAVE_FORMAT(pDst);
     ps->Trapezoids(op, pSrc, pDst, maskFormat, xSrc, ySrc, ntrap, traps);
+    RL_RENDER_RESTORE_FORMAT(pDst);
 
     if (dstWin && IsFramedWindow(dstWin) && ntrap > 0) {
         BoxRec box;
@@ -415,7 +445,9 @@ RootlessTriangles(CARD8 op, PicturePtr pSrc, PicturePtr pDst,
     if (dstWin && IsFramedWindow(dstWin))
         RootlessStartDrawing(dstWin);
 
+    RL_RENDER_SAVE_FORMAT(pDst);
     ps->Triangles(op, pSrc, pDst, maskFormat, xSrc, ySrc, ntri, tris);
+    RL_RENDER_RESTORE_FORMAT(pDst);
 
     if (dstWin && IsFramedWindow(dstWin) && ntri > 0) {
         BoxRec box;
@@ -450,7 +482,9 @@ RootlessCompositeRects(CARD8 op, PicturePtr pDst, xRenderColor *color,
     if (dstWin && IsFramedWindow(dstWin))
         RootlessStartDrawing(dstWin);
 
+    RL_RENDER_SAVE_FORMAT(pDst);
     ps->CompositeRects(op, pDst, color, nRect, rects);
+    RL_RENDER_RESTORE_FORMAT(pDst);
 
     if (dstWin && IsFramedWindow(dstWin) && nRect > 0) {
         int i;