integer overflow in XShapeGetRectangles() [CVE-2013-1982 5/6]

If the number of rectangles reported by the server is large enough that
it overflows when multiplied by the size of the appropriate struct, then
memory corruption can occur when more bytes are read from the X server
than the size of the buffer we allocated to hold them.

Reported-by: Ilja Van Sprundel <ivansprundel@ioactive.com>
Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>

1 files changed, 14 insertions, 10 deletions

diff --git a/src/XShape.c b/src/XShape.c
index 3987876..d025020 100644
--- a/src/XShape.c
+++ b/src/XShape.c
@@ -35,6 +35,7 @@ in this Software without prior written authorization from The Open Group.
 #include <X11/extensions/extutil.h>
 #include <X11/extensions/shape.h>
 #include <X11/extensions/shapeproto.h>
+#include <limits.h>
 #include "eat.h"
 
 static XExtensionInfo _shape_info_data;
@@ -443,7 +444,7 @@ XRectangle *XShapeGetRectangles (
     xShapeGetRectanglesReply	    rep;
     XRectangle			    *rects;
     xRectangle			    *xrects;
-    int				    i;
+    unsigned int		    i;
 
     ShapeCheckExtension (dpy, info, (XRectangle *)NULL);
 
@@ -461,20 +462,23 @@ XRectangle *XShapeGetRectangles (
     *count = rep.nrects;
     *ordering = rep.ordering;
     rects = NULL;
-    if (*count) {
-	xrects = (xRectangle *) Xmalloc (*count * sizeof (xRectangle));
-	rects = (XRectangle *) Xmalloc (*count * sizeof (XRectangle));
+    if (rep.nrects) {
+	if (rep.nrects < (INT_MAX / sizeof (XRectangle))) {
+	    xrects = Xmalloc (rep.nrects * sizeof (xRectangle));
+	    rects = Xmalloc (rep.nrects * sizeof (XRectangle));
+	} else {
+	    xrects = NULL;
+	    rects = NULL;
+	}
 	if (!xrects || !rects) {
-	    if (xrects)
-		Xfree (xrects);
-	    if (rects)
-		Xfree (rects);
+	    Xfree (xrects);
+	    Xfree (rects);
 	    _XEatDataWords (dpy, rep.length);
 	    rects = NULL;
 	    *count = 0;
 	} else {
-	    _XRead (dpy, (char *) xrects, *count * sizeof (xRectangle));
-	    for (i = 0; i < *count; i++) {
+	    _XRead (dpy, (char *) xrects, rep.nrects * sizeof (xRectangle));
+	    for (i = 0; i < rep.nrects; i++) {
 	    	rects[i].x = (short) cvtINT16toInt (xrects[i].x);
 	    	rects[i].y = (short) cvtINT16toInt (xrects[i].y);
 	    	rects[i].width = xrects[i].width;