CVE-2007-6429: Don't spuriously reject <8bpp shm pixmaps.

Move size validation after depth validation, and only validate size if
the bpp of the pixmap format is > 8.  If bpp < 8 then we're already
protected from overflow by the width and height checks.
(cherry picked from commit e9fa7c1c88a8130a48f772c92b186b8b777986b5)

1 files changed, 20 insertions, 16 deletions

diff --git a/Xext/shm.c b/Xext/shm.c
index 5633be904..6f99e9064 100644
--- a/Xext/shm.c
+++ b/Xext/shm.c
@@ -737,14 +737,6 @@ ProcPanoramiXShmCreatePixmap(
     }
     if (width > 32767 || height > 32767)
         return BadAlloc;
-    size = PixmapBytePad(width, depth) * height;
-    if (sizeof(size) == 4) {
-        if (size < width * height)
-            return BadAlloc;
-        /* thankfully, offset is unsigned */
-        if (stuff->offset + size < size)
-            return BadAlloc;
-    }
 
     if (stuff->depth != 1)
     {
@@ -755,7 +747,17 @@ ProcPanoramiXShmCreatePixmap(
 	client->errorValue = stuff->depth;
         return BadValue;
     }
+
 CreatePmap:
+    size = PixmapBytePad(width, depth) * height;
+    if (sizeof(size) == 4 && BitsPerPixel(depth) > 8) {
+        if (size < width * height)
+            return BadAlloc;
+        /* thankfully, offset is unsigned */
+        if (stuff->offset + size < size)
+            return BadAlloc;
+    }
+
     VERIFY_SHMSIZE(shmdesc, stuff->offset, size, client);
 
     if(!(newPix = (PanoramiXRes *) xalloc(sizeof(PanoramiXRes))))
@@ -1080,14 +1082,6 @@ ProcShmCreatePixmap(client)
     }
     if (width > 32767 || height > 32767)
 	return BadAlloc;
-    size = PixmapBytePad(width, depth) * height;
-    if (sizeof(size) == 4) {
-	if (size < width * height)
-	    return BadAlloc;
-	/* thankfully, offset is unsigned */
-	if (stuff->offset + size < size)
-	    return BadAlloc;
-    }
 
     if (stuff->depth != 1)
     {
@@ -1098,7 +1092,17 @@ ProcShmCreatePixmap(client)
 	client->errorValue = stuff->depth;
         return BadValue;
     }
+
 CreatePmap:
+    size = PixmapBytePad(width, depth) * height;
+    if (sizeof(size) == 4 && BitsPerPixel(depth) > 8) {
+	if (size < width * height)
+	    return BadAlloc;
+	/* thankfully, offset is unsigned */
+	if (stuff->offset + size < size)
+	    return BadAlloc;
+    }
+
     VERIFY_SHMSIZE(shmdesc, stuff->offset, size, client);
     pMap = (*shmFuncs[pDraw->pScreen->myNum]->CreatePixmap)(
 			    pDraw->pScreen, stuff->width,