gl: Return surface in error when creating oversized texture surfaces

When creating a texture surface that is larger than the maximum
framebuffer or texture dimensions of the context, return a surface in
error. Previously the code failed an assertion, but this prevents an
application from easily detecting when to fall back.

Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>

3 files changed, 111 insertions, 11 deletions

diff --git a/src/cairo-gl-surface.c b/src/cairo-gl-surface.c
index 1506f2010..c287a01e6 100644
--- a/src/cairo-gl-surface.c
+++ b/src/cairo-gl-surface.c
@@ -395,6 +395,23 @@ _cairo_gl_surface_init (cairo_device_t *device,
     _cairo_gl_surface_embedded_operand_init (surface);
 }
 
+static cairo_bool_t
+_cairo_gl_surface_size_valid_for_context (cairo_gl_context_t *ctx,
+					  int width, int height)
+{
+    return width > 0 && height > 0 &&
+	width <= ctx->max_framebuffer_size &&
+	height <= ctx->max_framebuffer_size;
+}
+
+static cairo_bool_t
+_cairo_gl_surface_size_valid (cairo_gl_surface_t *surface,
+			      int width, int height)
+{
+    cairo_gl_context_t *ctx = (cairo_gl_context_t *)surface->base.device;
+    return _cairo_gl_surface_size_valid_for_context (ctx, width, height);
+}
+
 static cairo_surface_t *
 _cairo_gl_surface_create_scratch_for_texture (cairo_gl_context_t   *ctx,
 					      cairo_content_t	    content,
@@ -404,7 +421,6 @@ _cairo_gl_surface_create_scratch_for_texture (cairo_gl_context_t   *ctx,
 {
     cairo_gl_surface_t *surface;
 
-    assert (width <= ctx->max_framebuffer_size && height <= ctx->max_framebuffer_size);
     surface = calloc (1, sizeof (cairo_gl_surface_t));
     if (unlikely (surface == NULL))
 	return _cairo_surface_create_in_error (_cairo_error (CAIRO_STATUS_NO_MEMORY));
@@ -588,6 +604,11 @@ cairo_gl_surface_create (cairo_device_t		*abstract_device,
     if (unlikely (status))
 	return _cairo_surface_create_in_error (status);
 
+    if (! _cairo_gl_surface_size_valid_for_context (ctx, width, height)) {
+	status = _cairo_gl_context_release (ctx, status);
+	return _cairo_surface_create_in_error (_cairo_error (CAIRO_STATUS_INVALID_SIZE));
+    }
+
     surface = (cairo_gl_surface_t *)
 	_cairo_gl_surface_create_and_clear_scratch (ctx, content, width, height);
     if (unlikely (surface->base.status)) {
@@ -761,16 +782,6 @@ cairo_gl_surface_swapbuffers (cairo_surface_t *abstract_surface)
     }
 }
 
-static cairo_bool_t
-_cairo_gl_surface_size_valid (cairo_gl_surface_t *surface,
-			      int width, int height)
-{
-    cairo_gl_context_t *ctx = (cairo_gl_context_t *)surface->base.device;
-    return width > 0 && height > 0 &&
-	width <= ctx->max_framebuffer_size &&
-	height <= ctx->max_framebuffer_size;
-}
-
 static cairo_surface_t *
 _cairo_gl_surface_create_similar (void		 *abstract_surface,
 				  cairo_content_t  content,
diff --git a/test/Makefile.sources b/test/Makefile.sources
index d44edb4f5..509c11ce8 100644
--- a/test/Makefile.sources
+++ b/test/Makefile.sources
@@ -394,6 +394,7 @@ ft_font_test_sources = \
 
 gl_surface_test_sources = \
 	gl-device-release.c \
+	gl-oversized-surface.c \
 	gl-surface-source.c
 
 quartz_surface_test_sources = quartz-surface-source.c
diff --git a/test/gl-oversized-surface.c b/test/gl-oversized-surface.c
new file mode 100644
index 000000000..4c46efd80
--- /dev/null
+++ b/test/gl-oversized-surface.c
@@ -0,0 +1,88 @@
+/*
+ * Copyright © 2012 Igalia S.L.
+ * Copyright © 2009 Eric Anholt
+ * Copyright © 2009 Chris Wilson
+ * Copyright © 2005 Red Hat, Inc
+ *
+ * Permission to use, copy, modify, distribute, and sell this software
+ * and its documentation for any purpose is hereby granted without
+ * fee, provided that the above copyright notice appear in all copies
+ * and that both that copyright notice and this permission notice
+ * appear in supporting documentation, and that the name of
+ * Chris Wilson not be used in advertising or publicity pertaining to
+ * distribution of the software without specific, written prior
+ * permission. Chris Wilson makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * IGALIA S.L. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL CHRIS WILSON BE LIABLE FOR ANY SPECIAL,
+ * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+ * RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
+ * IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * Author: Martin Robinson <mrobinson@igalia.com>
+ */
+
+#include "cairo-test.h"
+#include <cairo-gl.h>
+#include <assert.h>
+#include <limits.h>
+
+static cairo_test_status_t
+preamble (cairo_test_context_t *test_ctx)
+{
+    int rgba_attribs[] = {
+	GLX_RGBA,
+	GLX_RED_SIZE, 1,
+	GLX_GREEN_SIZE, 1,
+	GLX_BLUE_SIZE, 1,
+	GLX_ALPHA_SIZE, 1,
+	GLX_DOUBLEBUFFER,
+	None
+    };
+
+    Display *display;
+    XVisualInfo *visual_info;
+    GLXContext glx_context;
+    cairo_device_t *device;
+    cairo_surface_t *oversized_surface;
+    cairo_test_status_t test_status = CAIRO_TEST_SUCCESS;
+
+    display = XOpenDisplay (NULL);
+    if (display == NULL)
+	return CAIRO_TEST_UNTESTED;
+
+    visual_info = glXChooseVisual (display, DefaultScreen (display), rgba_attribs);
+    if (visual_info == NULL) {
+	XCloseDisplay (display);
+	return CAIRO_TEST_UNTESTED;
+    }
+
+    glx_context = glXCreateContext (display, visual_info, NULL, True);
+    if (glx_context == NULL) {
+	XCloseDisplay (display);
+	return CAIRO_TEST_UNTESTED;
+    }
+
+    device = cairo_glx_device_create (display, glx_context);
+
+    oversized_surface = cairo_gl_surface_create (device, CAIRO_CONTENT_COLOR_ALPHA, INT_MAX, INT_MAX);
+    if (cairo_surface_status (oversized_surface) != CAIRO_STATUS_INVALID_SIZE)
+        test_status = CAIRO_TEST_FAILURE;
+
+    cairo_device_destroy (device);
+    glXDestroyContext(display, glx_context);
+    XCloseDisplay (display);
+
+    return test_status;
+}
+
+CAIRO_TEST (gl_oversized_surface,
+	    "Test that creating a surface beyond texture limits results in an error surface",
+	    "gl", /* keywords */
+	    NULL, /* requirements */
+	    0, 0,
+	    preamble, NULL)