sna/composite: Fix incorrect operator reduction for RenderFillRectangles

As exemplified by KDE (using Kate) on gen3, it would attempt to render a
large set of boxes using OVER and a transparent colour. As gen3 copied
across some of the BLT assumptions, it was incorrectly reducing that to
a CLEAR and thus rendering incorrectly.

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>

7 files changed, 89 insertions, 48 deletions

diff --git a/src/sna/gen2_render.c b/src/sna/gen2_render.c
index e76876c8..f8e71e7e 100644
--- a/src/sna/gen2_render.c
+++ b/src/sna/gen2_render.c
@@ -1938,20 +1938,19 @@ gen2_render_fill_boxes(struct sna *sna,
 					   box, n))
 		return TRUE;
 
-	if (!sna_get_pixel_from_rgba(&pixel,
-				     color->red,
-				     color->green,
-				     color->blue,
-				     color->alpha,
-				     PICT_a8r8g8b8))
+	if (op == PictOpClear)
+		pixel = 0;
+	else if (!sna_get_pixel_from_rgba(&pixel,
+					  color->red,
+					  color->green,
+					  color->blue,
+					  color->alpha,
+					  PICT_a8r8g8b8))
 		return FALSE;
 
 	DBG(("%s: using shader for op=%d, format=%x, pixel=%x\n",
 	     __FUNCTION__, op, (int)format, pixel));
 
-	if (pixel == 0)
-		op = PictOpClear;
-
 	memset(&tmp, 0, sizeof(tmp));
 	tmp.op = op;
 	tmp.dst.pixmap = dst;
diff --git a/src/sna/gen3_render.c b/src/sna/gen3_render.c
index efd5d7a2..e21ea6c1 100644
--- a/src/sna/gen3_render.c
+++ b/src/sna/gen3_render.c
@@ -3675,6 +3675,23 @@ static inline Bool prefer_fill_blt(struct sna *sna)
 #endif
 }
 
+static inline void set_fill_shader(struct sna_composite_channel *c,
+				   uint32_t pixel)
+{
+	if (pixel == 0)
+		c->u.gen3.type = SHADER_ZERO;
+	else if (pixel == 0xff000000)
+		c->u.gen3.type = SHADER_BLACK;
+	else if (pixel == 0xffffffff)
+		c->u.gen3.type = SHADER_WHITE;
+	else
+		c->u.gen3.type = SHADER_CONSTANT;
+	c->u.gen3.mode = pixel;
+	c->is_affine = 1;
+	c->alpha_fixup = 0;
+	c->rb_reversed = 0;
+}
+
 static Bool
 gen3_render_fill_boxes(struct sna *sna,
 		       CARD8 op,
@@ -3716,20 +3733,20 @@ gen3_render_fill_boxes(struct sna *sna,
 					   box, n))
 		return TRUE;
 
-	if (!sna_get_pixel_from_rgba(&pixel,
-				     color->red,
-				     color->green,
-				     color->blue,
-				     color->alpha,
-				     PICT_a8r8g8b8))
-		return FALSE;
-
+	if (op == PictOpClear) {
+		pixel = 0;
+	} else {
+		if (!sna_get_pixel_from_rgba(&pixel,
+					     color->red,
+					     color->green,
+					     color->blue,
+					     color->alpha,
+					     PICT_a8r8g8b8))
+			return FALSE;
+	}
 	DBG(("%s: using shader for op=%d, format=%x, pixel=%x\n",
 	     __FUNCTION__, op, (int)format, pixel));
 
-	if (pixel == 0)
-		op = PictOpClear;
-
 	tmp.op = op;
 	tmp.dst.pixmap = dst;
 	tmp.dst.width = dst->drawable.width;
@@ -3740,11 +3757,7 @@ gen3_render_fill_boxes(struct sna *sna,
 	tmp.floats_per_rect = 6;
 	tmp.rb_reversed = 0;
 
-	tmp.src.u.gen3.type = op == PictOpClear ? SHADER_ZERO : SHADER_CONSTANT;
-	tmp.src.u.gen3.mode = pixel;
-	tmp.src.is_affine = 0;
-	tmp.src.alpha_fixup = 0;
-	tmp.src.rb_reversed = 0;
+	set_fill_shader(&tmp.src, pixel);
 	tmp.mask.u.gen3.type = SHADER_NONE;
 	tmp.u.gen3.num_constants = 0;
 
@@ -3897,12 +3910,8 @@ gen3_render_fill(struct sna *sna, uint8_t alu,
 	tmp->base.has_component_alpha = 0;
 	tmp->base.rb_reversed = 0;
 
-	tmp->base.src.u.gen3.type = SHADER_CONSTANT;
-	tmp->base.src.u.gen3.mode =
-		sna_rgba_for_color(color, dst->drawable.depth);
-	tmp->base.src.is_affine = 0;
-	tmp->base.src.alpha_fixup = 0;
-	tmp->base.src.rb_reversed = 0;
+	set_fill_shader(&tmp->base.src,
+			sna_rgba_for_color(color, dst->drawable.depth));
 	tmp->base.mask.u.gen3.type = SHADER_NONE;
 	tmp->base.u.gen3.num_constants = 0;
 
@@ -3979,19 +3988,8 @@ gen3_render_fill_one(struct sna *sna, PixmapPtr dst, struct kgem_bo *bo,
 	tmp.has_component_alpha = 0;
 	tmp.rb_reversed = 0;
 
-	color = sna_rgba_for_color(color, dst->drawable.depth);
-	if (color == 0)
-		tmp.src.u.gen3.type = SHADER_ZERO;
-	else if (color == 0xff000000)
-		tmp.src.u.gen3.type = SHADER_BLACK;
-	else if (color == 0xffffffff)
-		tmp.src.u.gen3.type = SHADER_WHITE;
-	else
-		tmp.src.u.gen3.type = SHADER_CONSTANT;
-	tmp.src.u.gen3.mode = color;
-	tmp.src.is_affine = 0;
-	tmp.src.alpha_fixup = 0;
-	tmp.src.rb_reversed = 0;
+	set_fill_shader(&tmp.src,
+			sna_rgba_for_color(color, dst->drawable.depth));
 	tmp.mask.u.gen3.type = SHADER_NONE;
 	tmp.u.gen3.num_constants = 0;
 
diff --git a/src/sna/gen4_render.c b/src/sna/gen4_render.c
index 18171c45..9fa22782 100644
--- a/src/sna/gen4_render.c
+++ b/src/sna/gen4_render.c
@@ -2451,7 +2451,9 @@ gen4_render_fill_boxes(struct sna *sna,
 	return FALSE;
 #endif
 
-	if (!sna_get_pixel_from_rgba(&pixel,
+	if (op == PictOpClear)
+		pixel = 0;
+	else if (!sna_get_pixel_from_rgba(&pixel,
 				     color->red,
 				     color->green,
 				     color->blue,
diff --git a/src/sna/gen5_render.c b/src/sna/gen5_render.c
index b9512c77..b33125ca 100644
--- a/src/sna/gen5_render.c
+++ b/src/sna/gen5_render.c
@@ -2420,7 +2420,9 @@ gen5_render_fill_boxes(struct sna *sna,
 			return FALSE;
 	}
 
-	if (!sna_get_pixel_from_rgba(&pixel,
+	if (op == PictOpClear)
+		pixel = 0;
+	else if (!sna_get_pixel_from_rgba(&pixel,
 				     color->red,
 				     color->green,
 				     color->blue,
diff --git a/src/sna/gen6_render.c b/src/sna/gen6_render.c
index 1fc971fe..6a132ada 100644
--- a/src/sna/gen6_render.c
+++ b/src/sna/gen6_render.c
@@ -2628,7 +2628,9 @@ gen6_render_fill_boxes(struct sna *sna,
 	return FALSE;
 #endif
 
-	if (!sna_get_pixel_from_rgba(&pixel,
+	if (op == PictOpClear)
+		pixel = 0;
+	else if (!sna_get_pixel_from_rgba(&pixel,
 				     color->red,
 				     color->green,
 				     color->blue,
diff --git a/src/sna/gen7_render.c b/src/sna/gen7_render.c
index ecdab6e7..3f9e59a3 100644
--- a/src/sna/gen7_render.c
+++ b/src/sna/gen7_render.c
@@ -2776,7 +2776,9 @@ gen7_render_fill_boxes(struct sna *sna,
 	return FALSE;
 #endif
 
-	if (!sna_get_pixel_from_rgba(&pixel,
+	if (op ==PictOpClear)
+		pixel = 0;
+	else if (!sna_get_pixel_from_rgba(&pixel,
 				     color->red,
 				     color->green,
 				     color->blue,
diff --git a/src/sna/sna_composite.c b/src/sna/sna_composite.c
index e9dc965f..0373bad4 100644
--- a/src/sna/sna_composite.c
+++ b/src/sna/sna_composite.c
@@ -601,6 +601,42 @@ sna_composite_rectangles(CARD8		 op,
 	if (!num_rects)
 		return;
 
+	if (color->alpha <= 0x00ff) {
+		switch (op) {
+		case PictOpOver:
+		case PictOpOutReverse:
+		case PictOpAdd:
+			return;
+		case  PictOpInReverse:
+		case  PictOpSrc:
+			op = PictOpClear;
+			break;
+		case  PictOpAtopReverse:
+			op = PictOpOut;
+			break;
+		case  PictOpXor:
+			op = PictOpOverReverse;
+			break;
+		}
+	} else if (color->alpha >= 0xff00) {
+		switch (op) {
+		case PictOpOver:
+			op = PictOpSrc;
+			break;
+		case PictOpInReverse:
+			return;
+		case PictOpOutReverse:
+			op = PictOpClear;
+			break;
+		case  PictOpAtopReverse:
+			op = PictOpOverReverse;
+			break;
+		case  PictOpXor:
+			op = PictOpOut;
+			break;
+		}
+	}
+
 	if (!pixman_region_not_empty(dst->pCompositeClip)) {
 		DBG(("%s: empty clip, skipping\n", __FUNCTION__));
 		return;