xwayland: Do not round non-standard modes

Currently, Xwayland uses libxcvt to generate the mode info and then
passes that to RRModeGet() to generate a RRMode.

However, libxcvt may round down the width to match the horizontal
granularity (8), and that's a problem when the Wayland compositor is
running a non-standard size (like, e.g. running nested with a custom
size) because XRandR would report a width smaller than the actual size.

To avoid that, check whether the CVT computed size differs from the
expected size, and fallback to a simpler computation not doing any
rounding if that's the case.

Signed-off-by: Olivier Fourdan <ofourdan@redhat.com>
Closes: https://gitlab.freedesktop.org/xorg/xserver/-/issues/1540
Reviewed-by: Michel Dänzer <mdaenzer@redhat.com>

1 files changed, 46 insertions, 16 deletions

diff --git a/hw/xwayland/xwayland-cvt.c b/hw/xwayland/xwayland-cvt.c
index ba8cbc9d1..0fc4ae82a 100644
--- a/hw/xwayland/xwayland-cvt.c
+++ b/hw/xwayland/xwayland-cvt.c
@@ -29,30 +29,60 @@
 
 #include "xwayland-cvt.h"
 
-RRModePtr
-xwayland_cvt(int hdisplay, int vdisplay, float vrefresh, Bool reduced,
-             Bool interlaced)
+static void
+xwayland_modinfo_from_cvt(xRRModeInfo *modeinfo,
+                          int hdisplay, int vdisplay, float vrefresh,
+                          Bool reduced, Bool interlaced)
 {
     struct libxcvt_mode_info *libxcvt_mode_info;
-    char name[128];
-    xRRModeInfo modeinfo;
 
     libxcvt_mode_info =
         libxcvt_gen_mode_info(hdisplay, vdisplay, vrefresh, reduced, interlaced);
 
-    memset(&modeinfo, 0, sizeof modeinfo);
-    modeinfo.width      = libxcvt_mode_info->hdisplay;
-    modeinfo.height     = libxcvt_mode_info->vdisplay;
-    modeinfo.dotClock   = libxcvt_mode_info->dot_clock * 1000.0;
-    modeinfo.hSyncStart = libxcvt_mode_info->hsync_start;
-    modeinfo.hSyncEnd   = libxcvt_mode_info->hsync_end;
-    modeinfo.hTotal     = libxcvt_mode_info->htotal;
-    modeinfo.vSyncStart = libxcvt_mode_info->vsync_start;
-    modeinfo.vSyncEnd   = libxcvt_mode_info->vsync_end;
-    modeinfo.vTotal     = libxcvt_mode_info->vtotal;
-    modeinfo.modeFlags  = libxcvt_mode_info->mode_flags;
+    modeinfo->width      = libxcvt_mode_info->hdisplay;
+    modeinfo->height     = libxcvt_mode_info->vdisplay;
+    modeinfo->dotClock   = libxcvt_mode_info->dot_clock * 1000.0;
+    modeinfo->hSyncStart = libxcvt_mode_info->hsync_start;
+    modeinfo->hSyncEnd   = libxcvt_mode_info->hsync_end;
+    modeinfo->hTotal     = libxcvt_mode_info->htotal;
+    modeinfo->vSyncStart = libxcvt_mode_info->vsync_start;
+    modeinfo->vSyncEnd   = libxcvt_mode_info->vsync_end;
+    modeinfo->vTotal     = libxcvt_mode_info->vtotal;
+    modeinfo->modeFlags  = libxcvt_mode_info->mode_flags;
 
     free(libxcvt_mode_info);
+}
+
+static void
+xwayland_modinfo_from_values(xRRModeInfo *modeinfo,
+                             int hdisplay, int vdisplay, float vrefresh)
+{
+    modeinfo->width  = hdisplay;
+    modeinfo->height = vdisplay;
+    modeinfo->hTotal = hdisplay;
+    modeinfo->vTotal = vdisplay;
+    modeinfo->dotClock = hdisplay * vdisplay * vrefresh;
+}
+
+RRModePtr
+xwayland_cvt(int hdisplay, int vdisplay, float vrefresh, Bool reduced,
+             Bool interlaced)
+{
+    char name[128];
+    xRRModeInfo modeinfo = { 0, };
+
+    xwayland_modinfo_from_cvt(&modeinfo,
+                              hdisplay, vdisplay, vrefresh, reduced, interlaced);
+
+    /* Horizontal granularity in libxcvt is 8, so if our horizontal size is not
+     * divisible by 8, libxcvt will round it down, and we will advertise a wrong
+     * size to our XRandR clients. Fallback to a simpler method in that case.
+     */
+    if (modeinfo.width != hdisplay || modeinfo.height != vdisplay) {
+        memset(&modeinfo, 0, sizeof(xRRModeInfo));
+        xwayland_modinfo_from_values(&modeinfo,
+                                     hdisplay, vdisplay, vrefresh);
+    }
 
     snprintf(name, sizeof name, "%dx%d",
              modeinfo.width, modeinfo.height);