dri2: Fix xserver crash if dri2 buffer allocation fails.

If driver fails to allocate memory for dri2 buffer server would crash
in send_buffers_reply.

Solution is to handle the allocation failure and return BadAlloc to
client.

Signed-off-by: Pauli Nieminen <ext-pauli.nieminen@nokia.com>
Reviewed-by: Michel Dänzer <michel@daenzer.net>
Reviewed-by: Kristian Høgsberg <krh@bitplanet.net>
Signed-off-by: Keith Packard <keithp@keithp.com>

2 files changed, 58 insertions, 24 deletions

diff --git a/hw/xfree86/dri2/dri2.c b/hw/xfree86/dri2/dri2.c
index e3bec3336..68982fe3e 100644
--- a/hw/xfree86/dri2/dri2.c
+++ b/hw/xfree86/dri2/dri2.c
@@ -326,6 +326,31 @@ allocate_or_reuse_buffer(DrawablePtr pDraw, DRI2ScreenPtr ds,
     }
 }
 
+static void
+update_dri2_drawable_buffers(DRI2DrawablePtr pPriv, DrawablePtr pDraw,
+			     DRI2BufferPtr *buffers, int *out_count, int *width, int *height)
+{
+    DRI2ScreenPtr   ds = DRI2GetScreen(pDraw->pScreen);
+    int i;
+
+    if (pPriv->buffers != NULL) {
+	for (i = 0; i < pPriv->bufferCount; i++) {
+	    if (pPriv->buffers[i] != NULL) {
+		(*ds->DestroyBuffer)(pDraw, pPriv->buffers[i]);
+	    }
+	}
+
+	free(pPriv->buffers);
+    }
+
+    pPriv->buffers = buffers;
+    pPriv->bufferCount = *out_count;
+    pPriv->width = pDraw->width;
+    pPriv->height = pDraw->height;
+    *width = pPriv->width;
+    *height = pPriv->height;
+}
+
 static DRI2BufferPtr *
 do_get_buffers(DrawablePtr pDraw, int *width, int *height,
 	       unsigned int *attachments, int count, int *out_count,
@@ -363,6 +388,9 @@ do_get_buffers(DrawablePtr pDraw, int *width, int *height,
 				     &buffers[i]))
 		buffers_changed = 1;
 
+	if (buffers[i] == NULL)
+	    goto err_out;
+
 	/* If the drawable is a window and the front-buffer is requested,
 	 * silently add the fake front-buffer to the list of requested
 	 * attachments.  The counting logic in the loop accounts for the case
@@ -395,6 +423,9 @@ do_get_buffers(DrawablePtr pDraw, int *width, int *height,
 				     front_format, dimensions_match,
 				     &buffers[i++]))
 	    buffers_changed = 1;
+
+	if (buffers[i] == NULL)
+	    goto err_out;
     }
 
     if (need_fake_front > 0) {
@@ -403,29 +434,15 @@ do_get_buffers(DrawablePtr pDraw, int *width, int *height,
 				     &buffers[i++]))
 	    buffers_changed = 1;
 
+	if (buffers[i] == NULL)
+	    goto err_out;
+
 	have_fake_front = 1;
     }
 
     *out_count = i;
 
-
-    if (pPriv->buffers != NULL) {
-	for (i = 0; i < pPriv->bufferCount; i++) {
-	    if (pPriv->buffers[i] != NULL) {
-		(*ds->DestroyBuffer)(pDraw, pPriv->buffers[i]);
-	    }
-	}
-
-	free(pPriv->buffers);
-    }
-
-    pPriv->buffers = buffers;
-    pPriv->bufferCount = *out_count;
-    pPriv->width = pDraw->width;
-    pPriv->height = pDraw->height;
-    *width = pPriv->width;
-    *height = pPriv->height;
-
+    update_dri2_drawable_buffers(pPriv, pDraw, buffers, out_count, width, height);
 
     /* If the client is getting a fake front-buffer, pre-fill it with the
      * contents of the real front-buffer.  This ensures correct operation of
@@ -446,6 +463,22 @@ do_get_buffers(DrawablePtr pDraw, int *width, int *height,
     }
 
     return pPriv->buffers;
+
+err_out:
+
+    *out_count = 0;
+
+    for (i = 0; i < count; i++) {
+	    if (buffers[i] != NULL)
+		    (*ds->DestroyBuffer)(pDraw, buffers[i]);
+    }
+
+    free(buffers);
+    buffers = NULL;
+
+    update_dri2_drawable_buffers(pPriv, pDraw, buffers, out_count, width, height);
+
+    return buffers;
 }
 
 DRI2BufferPtr *
diff --git a/hw/xfree86/dri2/dri2ext.c b/hw/xfree86/dri2/dri2ext.c
index 602eb6653..44a47cceb 100644
--- a/hw/xfree86/dri2/dri2ext.c
+++ b/hw/xfree86/dri2/dri2ext.c
@@ -207,7 +207,7 @@ ProcDRI2DestroyDrawable(ClientPtr client)
 }
 
 
-static void
+static int
 send_buffers_reply(ClientPtr client, DrawablePtr pDrawable,
 		   DRI2BufferPtr *buffers, int count, int width, int height)
 {
@@ -215,6 +215,9 @@ send_buffers_reply(ClientPtr client, DrawablePtr pDrawable,
     int skip = 0;
     int i;
 
+    if (buffers == NULL)
+	    return BadAlloc;
+
     if (pDrawable->type == DRAWABLE_WINDOW) {
 	for (i = 0; i < count; i++) {
 	    /* Do not send the real front buffer of a window to the client.
@@ -251,6 +254,7 @@ send_buffers_reply(ClientPtr client, DrawablePtr pDrawable,
 	buffer.flags = buffers[i]->flags;
 	WriteToClient(client, sizeof(xDRI2Buffer), &buffer);
     }
+    return Success;
 }
 
 
@@ -276,9 +280,8 @@ ProcDRI2GetBuffers(ClientPtr client)
 			     attachments, stuff->count, &count);
 
 
-    send_buffers_reply(client, pDrawable, buffers, count, width, height);
+    return send_buffers_reply(client, pDrawable, buffers, count, width, height);
 
-    return Success;
 }
 
 static int
@@ -302,9 +305,7 @@ ProcDRI2GetBuffersWithFormat(ClientPtr client)
     buffers = DRI2GetBuffersWithFormat(pDrawable, &width, &height,
 				       attachments, stuff->count, &count);
 
-    send_buffers_reply(client, pDrawable, buffers, count, width, height);
-
-    return Success;
+    return send_buffers_reply(client, pDrawable, buffers, count, width, height);
 }
 
 static int