glx: Be more strict about rejecting invalid image sizes [CVE-2014-8093 2/6]

Before this we'd just clamp the image size to 0, which was just
hideously stupid; if the parameters were such that they'd overflow an
integer, you'd allocate a small buffer, then pass huge values into (say)
ReadPixels, and now you're scribbling over arbitrary server memory.

Reviewed-by: Keith Packard <keithp@keithp.com>
Reviewed-by: Julien Cristau <jcristau@debian.org>
Reviewed-by: Michal Srb <msrb@suse.com>
Reviewed-by: Andy Ritger <aritger@nvidia.com>
Signed-off-by: Adam Jackson <ajax@redhat.com>
Signed-off-by: Alan Coopersmith <alan.coopersmith@oracle.com>
(cherry picked from commit ab2ba9338aa5e85b4487bc7fbe69985c76483e01)
Signed-off-by: Julien Cristau <jcristau@debian.org>

2 files changed, 16 insertions, 16 deletions

diff --git a/glx/singlepix.c b/glx/singlepix.c
index 506fdaad5..8b6c26187 100644
--- a/glx/singlepix.c
+++ b/glx/singlepix.c
@@ -65,7 +65,7 @@ __glXDisp_ReadPixels(__GLXclientState * cl, GLbyte * pc)
     lsbFirst = *(GLboolean *) (pc + 25);
     compsize = __glReadPixels_size(format, type, width, height);
     if (compsize < 0)
-        compsize = 0;
+        return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, swapBytes);
     glPixelStorei(GL_PACK_LSB_FIRST, lsbFirst);
@@ -124,7 +124,7 @@ __glXDisp_GetTexImage(__GLXclientState * cl, GLbyte * pc)
     compsize =
         __glGetTexImage_size(target, level, format, type, width, height, depth);
     if (compsize < 0)
-        compsize = 0;
+        return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer, cl, compsize, 1);
@@ -218,9 +218,9 @@ GetSeparableFilter(__GLXclientState * cl, GLbyte * pc, GLXContextTag tag)
     compsize2 = __glGetTexImage_size(target, 1, format, type, height, 1, 1);
 
     if (compsize < 0)
-        compsize = 0;
+        return BadLength;
     if (compsize2 < 0)
-        compsize2 = 0;
+        return BadLength;
     compsize = __GLX_PAD(compsize);
     compsize2 = __GLX_PAD(compsize2);
 
@@ -296,7 +296,7 @@ GetConvolutionFilter(__GLXclientState * cl, GLbyte * pc, GLXContextTag tag)
      */
     compsize = __glGetTexImage_size(target, 1, format, type, width, height, 1);
     if (compsize < 0)
-        compsize = 0;
+        return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer, cl, compsize, 1);
@@ -365,7 +365,7 @@ GetHistogram(__GLXclientState * cl, GLbyte * pc, GLXContextTag tag)
      */
     compsize = __glGetTexImage_size(target, 1, format, type, width, 1, 1);
     if (compsize < 0)
-        compsize = 0;
+        return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer, cl, compsize, 1);
@@ -426,7 +426,7 @@ GetMinmax(__GLXclientState * cl, GLbyte * pc, GLXContextTag tag)
 
     compsize = __glGetTexImage_size(target, 1, format, type, 2, 1, 1);
     if (compsize < 0)
-        compsize = 0;
+        return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer, cl, compsize, 1);
@@ -491,7 +491,7 @@ GetColorTable(__GLXclientState * cl, GLbyte * pc, GLXContextTag tag)
      */
     compsize = __glGetTexImage_size(target, 1, format, type, width, 1, 1);
     if (compsize < 0)
-        compsize = 0;
+        return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer, cl, compsize, 1);
diff --git a/glx/singlepixswap.c b/glx/singlepixswap.c
index 846910153..8dc304fa7 100644
--- a/glx/singlepixswap.c
+++ b/glx/singlepixswap.c
@@ -75,7 +75,7 @@ __glXDispSwap_ReadPixels(__GLXclientState * cl, GLbyte * pc)
     lsbFirst = *(GLboolean *) (pc + 25);
     compsize = __glReadPixels_size(format, type, width, height);
     if (compsize < 0)
-        compsize = 0;
+        return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, !swapBytes);
     glPixelStorei(GL_PACK_LSB_FIRST, lsbFirst);
@@ -144,7 +144,7 @@ __glXDispSwap_GetTexImage(__GLXclientState * cl, GLbyte * pc)
     compsize =
         __glGetTexImage_size(target, level, format, type, width, height, depth);
     if (compsize < 0)
-        compsize = 0;
+        return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, !swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer, cl, compsize, 1);
@@ -252,9 +252,9 @@ GetSeparableFilter(__GLXclientState * cl, GLbyte * pc, GLXContextTag tag)
     compsize2 = __glGetTexImage_size(target, 1, format, type, height, 1, 1);
 
     if (compsize < 0)
-        compsize = 0;
+        return BadLength;
     if (compsize2 < 0)
-        compsize2 = 0;
+        return BadLength;
     compsize = __GLX_PAD(compsize);
     compsize2 = __GLX_PAD(compsize2);
 
@@ -338,7 +338,7 @@ GetConvolutionFilter(__GLXclientState * cl, GLbyte * pc, GLXContextTag tag)
      */
     compsize = __glGetTexImage_size(target, 1, format, type, width, height, 1);
     if (compsize < 0)
-        compsize = 0;
+        return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, !swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer, cl, compsize, 1);
@@ -415,7 +415,7 @@ GetHistogram(__GLXclientState * cl, GLbyte * pc, GLXContextTag tag)
      */
     compsize = __glGetTexImage_size(target, 1, format, type, width, 1, 1);
     if (compsize < 0)
-        compsize = 0;
+        return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, !swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer, cl, compsize, 1);
@@ -483,7 +483,7 @@ GetMinmax(__GLXclientState * cl, GLbyte * pc, GLXContextTag tag)
 
     compsize = __glGetTexImage_size(target, 1, format, type, 2, 1, 1);
     if (compsize < 0)
-        compsize = 0;
+        return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, !swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer, cl, compsize, 1);
@@ -554,7 +554,7 @@ GetColorTable(__GLXclientState * cl, GLbyte * pc, GLXContextTag tag)
      */
     compsize = __glGetTexImage_size(target, 1, format, type, width, 1, 1);
     if (compsize < 0)
-        compsize = 0;
+        return BadLength;
 
     glPixelStorei(GL_PACK_SWAP_BYTES, !swapBytes);
     __GLX_GET_ANSWER_BUFFER(answer, cl, compsize, 1);