Pre-clip panning coordinates to keep crtc within panning region

There is a separate panning region check, but that doesn't work under
transformation, so just pre-clip the mouse coordinates when computing the
panning offsets. This leaves the case where panning constants are changing
unresolved.

Signed-off-by: Keith Packard <keithp@keithp.com>

1 files changed, 233 insertions, 21 deletions

diff --git a/hw/xfree86/modes/xf86RandR12.c b/hw/xfree86/modes/xf86RandR12.c
index ab39329bf..dacb5d27d 100644
--- a/hw/xfree86/modes/xf86RandR12.c
+++ b/hw/xfree86/modes/xf86RandR12.c
@@ -172,11 +172,167 @@ xf86RandR13VerifyPanningArea (xf86CrtcPtr crtc, int screenWidth, int screenHeigh
     return ret;
 }
 
+/*
+ * The heart of the panning operation:
+ *
+ * Given a frame buffer position (fb_x, fb_y),
+ * and a crtc position (crtc_x, crtc_y),
+ * and a transform matrix which maps frame buffer to crtc,
+ * compute a panning position (pan_x, pan_y) that
+ * makes the resulting transform line those two up
+ */
+
+static void
+xf86ComputeCrtcPan (Bool transform_in_use,
+		    struct pixman_f_transform *m,
+		    double screen_x, double screen_y,
+		    double crtc_x, double crtc_y,
+		    int old_pan_x, int old_pan_y,
+		    int *new_pan_x, int *new_pan_y)
+{
+    if (transform_in_use) {
+	/*
+	 * Given the current transform, M, the current position
+	 * on the Screen, S, and the desired position on the CRTC,
+	 * C, compute a translation, T, such that:
+	 *
+	 * M T S = C
+	 *
+	 * where T is of the form
+	 *
+	 * | 1 0 dx |
+	 * | 0 1 dy |
+	 * | 0 0 1  |
+	 *
+	 * M T S =
+	 *   | M00 Sx + M01 Sy + M00 dx + M01 dy + M02 |   | Cx F |
+	 *   | M10 Sx + M11 Sy + M10 dx + M11 dy + M12 | = | Cy F |
+	 *   | M20 Sx + M21 Sy + M20 dx + M21 dy + M22 |   |  F   |
+	 *
+	 * R = M S
+	 *
+	 *   Cx F = M00 dx + M01 dy + R0
+	 *   Cy F = M10 dx + M11 dy + R1
+	 *      F = M20 dx + M21 dy + R2
+	 *
+	 * Zero out dx, then dy
+	 *
+	 * F (Cx M10 - Cy M00) =
+	 *	    (M10 M01 - M00 M11) dy + M10 R0 - M00 R1
+	 * F (M10 - Cy M20) =
+	 *	    (M10 M21 - M20 M11) dy + M10 R2 - M20 R1
+	 *
+	 * F (Cx M11 - Cy M01) =
+	 *	    (M11 M00 - M01 M10) dx + M11 R0 - M01 R1
+	 * F (M11 - Cy M21) =
+	 *	    (M11 M20 - M21 M10) dx + M11 R2 - M21 R1
+	 *
+	 * Make some temporaries
+	 *
+	 * T = | Cx M10 - Cy M00 |
+	 *     | Cx M11 - Cy M01 |
+	 *
+	 * U = | M10 M01 - M00 M11 |
+	 *     | M11 M00 - M01 M10 |
+	 *
+	 * Q = | M10 R0 - M00 R1 |
+	 *     | M11 R0 - M01 R1 |
+	 *
+	 * P = | M10 - Cy M20 |
+	 *     | M11 - Cy M21 |
+	 *
+	 * W = | M10 M21 - M20 M11 |
+	 *     | M11 M20 - M21 M10 |
+	 *
+	 * V = | M10 R2 - M20 R1 |
+	 *	   | M11 R2 - M21 R1 |
+	 *
+	 * Rewrite:
+	 *
+	 * F T0 = U0 dy + Q0
+	 * F P0 = W0 dy + V0
+	 * F T1 = U1 dx + Q1
+	 * F P1 = W1 dx + V1
+	 *
+	 * Solve for F (two ways)
+	 *
+	 * F (W0 T0 - U0 P0)  = W0 Q0 - U0 V0
+	 *
+	 *     W0 Q0 - U0 V0
+	 * F = -------------
+	 *     W0 T0 - U0 P0
+	 *
+	 * F (W1 T1 - U1 P1) = W1 Q1 - U1 V1
+	 *
+	 *     W1 Q1 - U1 V1
+	 * F = -------------
+	 *     W1 T1 - U1 P1
+	 *
+	 * We'll use which ever solution works (denominator != 0)
+	 *
+	 * Finally, solve for dx and dy:
+	 *
+	 * dx = (F T1 - Q1) / U1
+	 * dx = (F P1 - V1) / W1
+	 *
+	 * dy = (F T0 - Q0) / U0
+	 * dy = (F P0 - V0) / W0
+	 */
+	double			    r[3];
+	double			    q[2], u[2], t[2], v[2], w[2], p[2];
+	double			    f;
+	struct pict_f_vector	    d;
+	int			    i;
+
+	/* Get the un-normalized crtc coordinates again */
+	for (i = 0; i < 3; i++)
+	    r[i] = m->m[i][0] * screen_x + m->m[i][1] * screen_y + m->m[i][2];
+
+	/* Combine values into temporaries */
+	for (i = 0; i < 2; i++) {
+	    q[i] = m->m[1][i] * r[0] - m->m[0][i] * r[1];
+	    u[i] = m->m[1][i] * m->m[0][1-i] - m->m[0][i] * m->m[1][1-i];
+	    t[i] = m->m[1][i] * crtc_x - m->m[0][i] * crtc_y;
+
+	    v[i] = m->m[1][i] * r[2] - m->m[2][i] * r[1];
+	    w[i] = m->m[1][i] * m->m[2][1-i] - m->m[2][i] * m->m[1][1-i];
+	    p[i] = m->m[1][i] - m->m[2][i] * crtc_y;
+	}
+
+	/* Find a way to compute f */
+	f = 0;
+	for (i = 0; i < 2; i++) {
+	    double a = w[i] * q[i] - u[i] * v[i];
+	    double b = w[i] * t[i] - u[i] * p[i];
+	    if (b != 0) {
+		f = a/b;
+		break;
+	    }
+	}
+
+	/* Solve for the resulting transform vector */
+	for (i = 0; i < 2; i++) {
+	    if (u[i])
+		d.v[1-i] = (t[i] * f - q[i]) / u[i];
+	    else if (w[1])
+		d.v[1-i] = (p[i] * f - v[i]) / w[i];
+	    else
+		d.v[1-i] = 0;
+	}
+	*new_pan_x = old_pan_x - floor (d.v[0] + 0.5);
+	*new_pan_y = old_pan_y - floor (d.v[1] + 0.5);
+    } else {
+	*new_pan_x = screen_x - crtc_x;
+	*new_pan_y = screen_y - crtc_y;
+    }
+}
+
 static void
 xf86RandR13Pan (xf86CrtcPtr crtc, int x, int y)
 {
     int newX, newY;
     int width, height;
+    Bool panned = FALSE;
 
     if (crtc->version < 2)
 	return;
@@ -194,32 +350,88 @@ xf86RandR13Pan (xf86CrtcPtr crtc, int x, int y)
     if ((crtc->panningTrackingArea.x2 <= crtc->panningTrackingArea.x1 ||
 	 (x >= crtc->panningTrackingArea.x1 && x < crtc->panningTrackingArea.x2)) &&
 	(crtc->panningTrackingArea.y2 <= crtc->panningTrackingArea.y1 ||
-	 (y >= crtc->panningTrackingArea.y1 && y < crtc->panningTrackingArea.y2))) {
+	 (y >= crtc->panningTrackingArea.y1 && y < crtc->panningTrackingArea.y2)))
+    {
+	struct pict_f_vector    c;
+
+	/*
+	 * Pre-clip the mouse position to the panning area so that we don't
+	 * push the crtc outside. This doesn't deal with changes to the
+	 * panning values, only mouse position changes.
+	 */
+	if (crtc->panningTotalArea.x2 > crtc->panningTotalArea.x1)
+	{
+	    if (x < crtc->panningTotalArea.x1)
+		x = crtc->panningTotalArea.x1;
+	    if (x >= crtc->panningTotalArea.x2)
+		x = crtc->panningTotalArea.x2 - 1;
+	}
+	if (crtc->panningTotalArea.y2 > crtc->panningTotalArea.y1)
+	{
+	    if (y < crtc->panningTotalArea.y1)
+		y = crtc->panningTotalArea.y1;
+	    if (y >= crtc->panningTotalArea.y2)
+		y = crtc->panningTotalArea.y2 - 1;
+	}
+
+	c.v[0] = x;
+	c.v[1] = y;
+	c.v[2] = 1.0;
+	if (crtc->transform_in_use) {
+	    pixman_f_transform_point(&crtc->f_framebuffer_to_crtc, &c);
+	} else {
+	    c.v[0] -= crtc->x;
+	    c.v[1] -= crtc->y;
+	}
+
 	if (crtc->panningTotalArea.x2 > crtc->panningTotalArea.x1) {
-	    if (x < crtc->x + crtc->panningBorder[0])
-		newX = x - crtc->panningBorder[0];
-	    if (x >= crtc->x + width - crtc->panningBorder[2])
-		newX = x - width + crtc->panningBorder[2] + 1;
+	    if (c.v[0] < crtc->panningBorder[0]) {
+		c.v[0] = crtc->panningBorder[0];
+		panned = TRUE;
+	    }
+	    if (c.v[0] >= width - crtc->panningBorder[2]) {
+		c.v[0] = width - crtc->panningBorder[2] - 1;
+		panned = TRUE;
+	    }
 	}
 	if (crtc->panningTotalArea.y2 > crtc->panningTotalArea.y1) {
-	    if (y < crtc->y + crtc->panningBorder[1])
-		newY = y - crtc->panningBorder[1];
-	    if (y >= crtc->y + height - crtc->panningBorder[3])
-		newY = y - height + crtc->panningBorder[3] + 1;
+	    if (c.v[1] < crtc->panningBorder[1]) {
+		c.v[1] = crtc->panningBorder[1];
+		panned = TRUE;
+	    }
+	    if (c.v[1] >= height - crtc->panningBorder[3]) {
+		c.v[1] = height - crtc->panningBorder[3] - 1;
+		panned = TRUE;
+	    }
 	}
+	if (panned)
+	    xf86ComputeCrtcPan (crtc->transform_in_use,
+				&crtc->f_framebuffer_to_crtc,
+				x, y, c.v[0], c.v[1],
+				newX, newY, &newX, &newY);
     }
-    /* Validate against [xy]1 after [xy]2, to be sure that results are > 0 for [xy]1 > 0 */
-    if (crtc->panningTotalArea.x2 > crtc->panningTotalArea.x1) {
-	if (newX > crtc->panningTotalArea.x2 - width)
-	    newX =  crtc->panningTotalArea.x2 - width;
-	if (newX <  crtc->panningTotalArea.x1)
-	    newX =  crtc->panningTotalArea.x1;
-    }
-    if (crtc->panningTotalArea.y2 > crtc->panningTotalArea.y1) {
-	if (newY > crtc->panningTotalArea.y2 - height)
-	    newY =  crtc->panningTotalArea.y2 - height;
-	if (newY <  crtc->panningTotalArea.y1)
-	    newY =  crtc->panningTotalArea.y1;
+
+    /*
+     * Ensure that the crtc is within the panning region.
+     *
+     * XXX This computation only works when we do not have a transform
+     * in use.
+     */
+    if (!crtc->transform_in_use)
+    {
+	/* Validate against [xy]1 after [xy]2, to be sure that results are > 0 for [xy]1 > 0 */
+	if (crtc->panningTotalArea.x2 > crtc->panningTotalArea.x1) {
+	    if (newX > crtc->panningTotalArea.x2 - width)
+		newX =  crtc->panningTotalArea.x2 - width;
+	    if (newX <  crtc->panningTotalArea.x1)
+		newX =  crtc->panningTotalArea.x1;
+	}
+	if (crtc->panningTotalArea.y2 > crtc->panningTotalArea.y1) {
+	    if (newY > crtc->panningTotalArea.y2 - height)
+		newY =  crtc->panningTotalArea.y2 - height;
+	    if (newY <  crtc->panningTotalArea.y1)
+		newY =  crtc->panningTotalArea.y1;
+	}
     }
     if (newX != crtc->x || newY != crtc->y)
 	xf86CrtcSetOrigin (crtc, newX, newY);