CVE-2007-6429: Don't spuriously reject <8bpp shm pixmaps.

Move size validation after depth validation, and only validate size if
the bpp of the pixmap format is > 8.  If bpp < 8 then we're already
protected from overflow by the width and height checks.

1 files changed, 20 insertions, 16 deletions

diff --git a/Xext/shm.c b/Xext/shm.c
index c545e4919..e46f6fcde 100644
--- a/Xext/shm.c
+++ b/Xext/shm.c
@@ -783,14 +783,6 @@ ProcPanoramiXShmCreatePixmap(
     }
     if (width > 32767 || height > 32767)
         return BadAlloc;
-    size = PixmapBytePad(width, depth) * height;
-    if (sizeof(size) == 4) {
-        if (size < width * height)
-            return BadAlloc;
-        /* thankfully, offset is unsigned */
-        if (stuff->offset + size < size)
-            return BadAlloc;
-    }
 
     if (stuff->depth != 1)
     {
@@ -801,7 +793,17 @@ ProcPanoramiXShmCreatePixmap(
 	client->errorValue = stuff->depth;
         return BadValue;
     }
+
 CreatePmap:
+    size = PixmapBytePad(width, depth) * height;
+    if (sizeof(size) == 4 && BitsPerPixel(depth) > 8) {
+        if (size < width * height)
+            return BadAlloc;
+        /* thankfully, offset is unsigned */
+        if (stuff->offset + size < size)
+            return BadAlloc;
+    }
+
     VERIFY_SHMSIZE(shmdesc, stuff->offset, size, client);
 
     if(!(newPix = (PanoramiXRes *) xalloc(sizeof(PanoramiXRes))))
@@ -1126,14 +1128,6 @@ ProcShmCreatePixmap(client)
     }
     if (width > 32767 || height > 32767)
 	return BadAlloc;
-    size = PixmapBytePad(width, depth) * height;
-    if (sizeof(size) == 4) {
-	if (size < width * height)
-	    return BadAlloc;
-	/* thankfully, offset is unsigned */
-	if (stuff->offset + size < size)
-	    return BadAlloc;
-    }
 
     if (stuff->depth != 1)
     {
@@ -1144,7 +1138,17 @@ ProcShmCreatePixmap(client)
 	client->errorValue = stuff->depth;
         return BadValue;
     }
+
 CreatePmap:
+    size = PixmapBytePad(width, depth) * height;
+    if (sizeof(size) == 4 && BitsPerPixel(depth) > 8) {
+	if (size < width * height)
+	    return BadAlloc;
+	/* thankfully, offset is unsigned */
+	if (stuff->offset + size < size)
+	    return BadAlloc;
+    }
+
     VERIFY_SHMSIZE(shmdesc, stuff->offset, size, client);
     pMap = (*shmFuncs[pDraw->pScreen->myNum]->CreatePixmap)(
 			    pDraw->pScreen, stuff->width,