Bug #594: CAN-2005-2495: Fix exploitable integer overflow in pixmap

    creation, where we could create a far smaller pixmap than we thought,
    allowing changes to arbitrary chunks of memory. (Søren Sandmann
    Pedersen)

16 files changed, 73 insertions, 13 deletions

diff --git a/afb/afbpixmap.c b/afb/afbpixmap.c
index a155c101b..c6ae8481c 100644
--- a/afb/afbpixmap.c
+++ b/afb/afbpixmap.c
@@ -77,10 +77,14 @@ afbCreatePixmap(pScreen, width, height, depth)
 	int				depth;
 {
 	PixmapPtr pPixmap;
-	int datasize;
-	int paddedWidth;
+	size_t datasize;
+	size_t paddedWidth;
 
 	paddedWidth = BitmapBytePad(width);
+
+	if (paddedWidth > 32767 || height > 32767 || depth > 4)
+	    return NullPixmap;
+	
 	datasize = height * paddedWidth * depth;
 	pPixmap = AllocatePixmap(pScreen, datasize);
 	if (!pPixmap)
diff --git a/cfb/cfbpixmap.c b/cfb/cfbpixmap.c
index f7e06bd13..704ab4eda 100644
--- a/cfb/cfbpixmap.c
+++ b/cfb/cfbpixmap.c
@@ -72,10 +72,13 @@ cfbCreatePixmap (pScreen, width, height, depth)
     int		depth;
 {
     PixmapPtr pPixmap;
-    int datasize;
-    int paddedWidth;
+    size_t datasize;
+    size_t paddedWidth;
 
     paddedWidth = PixmapBytePad(width, depth);
+
+    if (paddedWidth / 4 > 32767 || height > 32767)
+	return NullPixmap;
     datasize = height * paddedWidth;
     pPixmap = AllocatePixmap(pScreen, datasize);
     if (!pPixmap)
diff --git a/dix/dispatch.c b/dix/dispatch.c
index 99103ae65..ccbe06419 100644
--- a/dix/dispatch.c
+++ b/dix/dispatch.c
@@ -1483,6 +1483,23 @@ ProcCreatePixmap(register ClientPtr client)
 	client->errorValue = 0;
         return BadValue;
     }
+    if (stuff->width > 32767 || stuff->height > 32767)
+    {
+	/* It is allowed to try and allocate a pixmap which is larger than
+	 * 32767 in either dimension. However, all of the framebuffer code
+	 * is buggy and does not reliably draw to such big pixmaps, basically
+	 * because the Region data structure operates with signed shorts
+	 * for the rectangles in it.
+	 *
+	 * Furthermore, several places in the X server computes the
+	 * size in bytes of the pixmap and tries to store it in an
+	 * integer. This integer can overflow and cause the allocated size
+	 * to be much smaller.
+	 *
+	 * So, such big pixmaps are rejected here with a BadAlloc
+	 */
+	return BadAlloc;
+    }
     if (stuff->depth != 1)
     {
         pDepth = pDraw->pScreen->allowedDepths;
diff --git a/dix/pixmap.c b/dix/pixmap.c
index f76c557f4..78ce2a8c6 100644
--- a/dix/pixmap.c
+++ b/dix/pixmap.c
@@ -118,6 +118,9 @@ AllocatePixmap(ScreenPtr pScreen, int pixDataSize)
     unsigned size;
     int i;
 
+    if (pScreen->totalPixmapSize > ((size_t)-1) - pixDataSize)
+	return NullPixmap;
+    
     pPixmap = (PixmapPtr)xalloc(pScreen->totalPixmapSize + pixDataSize);
     if (!pPixmap)
 	return NullPixmap;
diff --git a/exa/exa.c b/exa/exa.c
index 92ff394e2..22f5edd37 100644
--- a/exa/exa.c
+++ b/exa/exa.c
@@ -376,6 +376,9 @@ exaCreatePixmap(ScreenPtr pScreen, int w, int h, int depth)
     ScrnInfoPtr pScrn = XF86SCRNINFO(pScreen);
     ExaScreenPriv(pScreen);
 
+    if (w > 32767 || h > 32767)
+	return NullPixmap;
+    
     if (!pScrn->vtSema || pExaScr->swappedOut) {
         pPixmap = pExaScr->SavedCreatePixmap(pScreen, w, h, depth);
     } else {
diff --git a/exa/exa_accel.c b/exa/exa_accel.c
index 92ff394e2..22f5edd37 100644
--- a/exa/exa_accel.c
+++ b/exa/exa_accel.c
@@ -376,6 +376,9 @@ exaCreatePixmap(ScreenPtr pScreen, int w, int h, int depth)
     ScrnInfoPtr pScrn = XF86SCRNINFO(pScreen);
     ExaScreenPriv(pScreen);
 
+    if (w > 32767 || h > 32767)
+	return NullPixmap;
+    
     if (!pScrn->vtSema || pExaScr->swappedOut) {
         pPixmap = pExaScr->SavedCreatePixmap(pScreen, w, h, depth);
     } else {
diff --git a/exa/exa_migration.c b/exa/exa_migration.c
index 92ff394e2..22f5edd37 100644
--- a/exa/exa_migration.c
+++ b/exa/exa_migration.c
@@ -376,6 +376,9 @@ exaCreatePixmap(ScreenPtr pScreen, int w, int h, int depth)
     ScrnInfoPtr pScrn = XF86SCRNINFO(pScreen);
     ExaScreenPriv(pScreen);
 
+    if (w > 32767 || h > 32767)
+	return NullPixmap;
+    
     if (!pScrn->vtSema || pExaScr->swappedOut) {
         pPixmap = pExaScr->SavedCreatePixmap(pScreen, w, h, depth);
     } else {
diff --git a/fb/fbpixmap.c b/fb/fbpixmap.c
index 1cb34e4d9..decc07ba6 100644
--- a/fb/fbpixmap.c
+++ b/fb/fbpixmap.c
@@ -36,12 +36,14 @@ PixmapPtr
 fbCreatePixmapBpp (ScreenPtr pScreen, int width, int height, int depth, int bpp)
 {
     PixmapPtr	pPixmap;
-    int		datasize;
-    int		paddedWidth;
+    size_t	datasize;
+    size_t	paddedWidth;
     int		adjust;
     int		base;
 
     paddedWidth = ((width * bpp + FB_MASK) >> FB_SHIFT) * sizeof (FbBits);
+    if (paddedWidth / 4 > 32767 || height > 32767)
+	return NullPixmap;
     datasize = height * paddedWidth;
 #ifdef PIXPRIV
     base = pScreen->totalPixmapSize;
diff --git a/hw/xfree86/exa/exa.c b/hw/xfree86/exa/exa.c
index 92ff394e2..22f5edd37 100644
--- a/hw/xfree86/exa/exa.c
+++ b/hw/xfree86/exa/exa.c
@@ -376,6 +376,9 @@ exaCreatePixmap(ScreenPtr pScreen, int w, int h, int depth)
     ScrnInfoPtr pScrn = XF86SCRNINFO(pScreen);
     ExaScreenPriv(pScreen);
 
+    if (w > 32767 || h > 32767)
+	return NullPixmap;
+    
     if (!pScrn->vtSema || pExaScr->swappedOut) {
         pPixmap = pExaScr->SavedCreatePixmap(pScreen, w, h, depth);
     } else {
diff --git a/hw/xfree86/exa/exa_accel.c b/hw/xfree86/exa/exa_accel.c
index 92ff394e2..22f5edd37 100644
--- a/hw/xfree86/exa/exa_accel.c
+++ b/hw/xfree86/exa/exa_accel.c
@@ -376,6 +376,9 @@ exaCreatePixmap(ScreenPtr pScreen, int w, int h, int depth)
     ScrnInfoPtr pScrn = XF86SCRNINFO(pScreen);
     ExaScreenPriv(pScreen);
 
+    if (w > 32767 || h > 32767)
+	return NullPixmap;
+    
     if (!pScrn->vtSema || pExaScr->swappedOut) {
         pPixmap = pExaScr->SavedCreatePixmap(pScreen, w, h, depth);
     } else {
diff --git a/hw/xfree86/exa/exa_migration.c b/hw/xfree86/exa/exa_migration.c
index 92ff394e2..22f5edd37 100644
--- a/hw/xfree86/exa/exa_migration.c
+++ b/hw/xfree86/exa/exa_migration.c
@@ -376,6 +376,9 @@ exaCreatePixmap(ScreenPtr pScreen, int w, int h, int depth)
     ScrnInfoPtr pScrn = XF86SCRNINFO(pScreen);
     ExaScreenPriv(pScreen);
 
+    if (w > 32767 || h > 32767)
+	return NullPixmap;
+    
     if (!pScrn->vtSema || pExaScr->swappedOut) {
         pPixmap = pExaScr->SavedCreatePixmap(pScreen, w, h, depth);
     } else {
diff --git a/hw/xfree86/xaa/xaaInit.c b/hw/xfree86/xaa/xaaInit.c
index 35ee34c6c..15132560d 100644
--- a/hw/xfree86/xaa/xaaInit.c
+++ b/hw/xfree86/xaa/xaaInit.c
@@ -502,6 +502,9 @@ XAACreatePixmap(ScreenPtr pScreen, int w, int h, int depth)
     XAAPixmapPtr pPriv;
     PixmapPtr pPix = NULL;
     int size = w * h;
+
+    if (w > 32767 || h > 32767)
+	return NullPixmap;
     
     if (!infoRec->offscreenDepthsInitialized)
 	XAAInitializeOffscreenDepths (pScreen);
diff --git a/hw/xfree86/xf4bpp/ppcPixmap.c b/hw/xfree86/xf4bpp/ppcPixmap.c
index 1f66c5850..011dc56ff 100644
--- a/hw/xfree86/xf4bpp/ppcPixmap.c
+++ b/hw/xfree86/xf4bpp/ppcPixmap.c
@@ -89,7 +89,7 @@ xf4bppCreatePixmap( pScreen, width, height, depth )
     int		depth ;
 {
     register PixmapPtr pPixmap  = (PixmapPtr)NULL;
-    int size ;
+    size_t size ;
     
     TRACE(("xf4bppCreatePixmap(pScreen=0x%x, width=%d, height=%d, depth=%d)\n", pScreen, width, height, depth)) ;
 
@@ -97,6 +97,10 @@ xf4bppCreatePixmap( pScreen, width, height, depth )
 	return (PixmapPtr) NULL ;
 
     size = PixmapBytePad(width, depth);
+
+    if (size / 4 > 32767 || height > 32767)
+	return (PixmapPtr) NULL ;
+    
     pPixmap = AllocatePixmap (pScreen, (height * size));
     
     if ( !pPixmap )
diff --git a/ilbm/ilbmpixmap.c b/ilbm/ilbmpixmap.c
index bd977e627..33c317155 100644
--- a/ilbm/ilbmpixmap.c
+++ b/ilbm/ilbmpixmap.c
@@ -79,10 +79,12 @@ ilbmCreatePixmap(pScreen, width, height, depth)
 	int				depth;
 {
 	PixmapPtr pPixmap;
-	int datasize;
-	int paddedWidth;
+	size_t datasize;
+	size_t paddedWidth;
 
 	paddedWidth = BitmapBytePad(width);
+	if (paddedWidth > 32767 || height > 32767 || depth > 4)
+		return NullPixmap;
 	datasize = height * paddedWidth * depth;
 	pPixmap = AllocatePixmap(pScreen, datasize);
 	if (!pPixmap)
diff --git a/iplan2p4/iplpixmap.c b/iplan2p4/iplpixmap.c
index 324b161e2..7bfb49551 100644
--- a/iplan2p4/iplpixmap.c
+++ b/iplan2p4/iplpixmap.c
@@ -78,12 +78,14 @@ iplCreatePixmap (pScreen, width, height, depth)
     int		depth;
 {
     PixmapPtr pPixmap;
-    int datasize;
-    int paddedWidth;
+    size_t datasize;
+    size_t paddedWidth;
     int ipad=INTER_PLANES*2 - 1;
 
     paddedWidth = PixmapBytePad(width, depth);
     paddedWidth = (paddedWidth + ipad) & ~ipad;
+    if (paddedWidth / 4 > 32767 || height > 32767)
+	return NullPixmap;
     datasize = height * paddedWidth;
     pPixmap = AllocatePixmap(pScreen, datasize);
     if (!pPixmap)
diff --git a/mfb/mfbpixmap.c b/mfb/mfbpixmap.c
index c3d5d5ea6..ad0dfe79f 100644
--- a/mfb/mfbpixmap.c
+++ b/mfb/mfbpixmap.c
@@ -75,12 +75,14 @@ mfbCreatePixmap (pScreen, width, height, depth)
     int		depth;
 {
     PixmapPtr pPixmap;
-    int datasize;
-    int paddedWidth;
+    size_t datasize;
+    size_t paddedWidth;
 
     if (depth != 1)
 	return NullPixmap;
     paddedWidth = BitmapBytePad(width);
+    if (paddedWidth / 4 > 32767 || height > 32767)
+	return NullPixmap;
     datasize = height * paddedWidth;
     pPixmap = AllocatePixmap(pScreen, datasize);
     if (!pPixmap)