CVE-2008-2360 - RENDER Extension heap buffer overflow

An integer overflow may occur in the computation of the size of the
glyph to be allocated by the AllocateGlyph() function which will cause
less memory to be allocated than expected, leading to later heap
overflow.

On systems where the X  SIGSEGV handler includes a stack trace, more
malloc()-type functions are called, which may lead to other
exploitable issues.
(cherry picked from commit b1a4a96885bf191d5f4afcfb2b41a88631b8412b)

1 files changed, 12 insertions, 2 deletions

diff --git a/render/glyph.c b/render/glyph.c
index 583a52ba3..42ae65d6b 100644
--- a/render/glyph.c
+++ b/render/glyph.c
@@ -42,6 +42,12 @@
 #include "picturestr.h"
 #include "glyphstr.h"
 
+#if HAVE_STDINT_H
+#include <stdint.h>
+#elif !defined(UINT32_MAX)
+#define UINT32_MAX 0xffffffffU
+#endif
+
 /*
  * From Knuth -- a good choice for hash/rehash values is p, p-2 where
  * p and p-2 are both prime.  These tables are sized to have an extra 10%
@@ -626,8 +632,12 @@ AllocateGlyph (xGlyphInfo *gi, int fdepth)
     int		     size;
     GlyphPtr	     glyph;
     int		     i;
-
-    size = gi->height * PixmapBytePad (gi->width, glyphDepths[fdepth]);
+    size_t	     padded_width;
+    
+    padded_width = PixmapBytePad (gi->width, glyphDepths[fdepth]);
+    if (gi->height && padded_width > (UINT32_MAX - sizeof(GlyphRec))/gi->height)
+	return 0;
+    size = gi->height * padded_width;
     glyph = (GlyphPtr) xalloc (size + sizeof (GlyphRec));
     if (!glyph)
 	return 0;