Fallback to system memory when fail to allocate one big fbo.

Even when create a pixmap which smaller than the max_fbo_size,
it may fail due to some low level driver limitation. If that is
the case, we don't need to crash the xserver. We just need to
fallback to system memory.

See the related bug at:
https://bugs.freedesktop.org/show_bug.cgi?id=71190

Signed-off-by: Zhigang Gong <zhigang.gong@intel.com>
Reviewed-by: Michel Danzer  <michel@daenzer.net>
Tested-by: Kai Wasserbach <kai@dev.carbon-project.org>
Tested-by: Erich Seifert <eseifert@error-reports.org>

2 files changed, 24 insertions, 16 deletions

diff --git a/src/glamor.c b/src/glamor.c
index 93d3c5e..4a2f5ad 100644
--- a/src/glamor.c
+++ b/src/glamor.c
@@ -134,7 +134,7 @@ glamor_create_pixmap(ScreenPtr screen, int w, int h, int depth,
 	glamor_pixmap_private *pixmap_priv;
 	glamor_screen_private *glamor_priv =
 	    glamor_get_screen_private(screen);
-	glamor_pixmap_fbo *fbo;
+	glamor_pixmap_fbo *fbo = NULL;
 	int pitch;
 	GLenum format;
 
@@ -173,13 +173,13 @@ glamor_create_pixmap(ScreenPtr screen, int w, int h, int depth,
 	if (type == GLAMOR_MEMORY_MAP || glamor_check_fbo_size(glamor_priv, w, h)) {
 		pixmap_priv->type = type;
 		fbo = glamor_create_fbo(glamor_priv, w, h, format, usage);
-	}
-	else {
-		DEBUGF("Create LARGE pixmap %p width %d height %d\n", pixmap, w, h);
+	} else {
+	    int tile_size = glamor_priv->max_fbo_size;
+		DEBUGF("Create LARGE pixmap %p width %d height %d, tile size %d\n", pixmap, w, h, tile_size);
 		pixmap_priv->type = GLAMOR_TEXTURE_LARGE;
 		fbo = glamor_create_fbo_array(glamor_priv, w, h, format, usage,
-					      glamor_priv->max_fbo_size,
-					      glamor_priv->max_fbo_size,
+					      tile_size,
+					      tile_size,
 					      pixmap_priv);
 	}
 
@@ -589,7 +589,8 @@ glamor_dri3_fd_from_pixmap (ScreenPtr screen,
 	{
 		case GLAMOR_TEXTURE_DRM:
 		case GLAMOR_TEXTURE_ONLY:
-			glamor_pixmap_ensure_fbo(pixmap, GL_RGBA, 0);
+			if (!glamor_pixmap_ensure_fbo(pixmap, GL_RGBA, 0))
+				return -1;
 			return glamor_egl_dri3_fd_name_from_tex(screen,
 								pixmap,
 								pixmap_priv->base.fbo->tex,
@@ -615,7 +616,8 @@ glamor_dri3_name_from_pixmap (PixmapPtr pixmap)
 	{
 		case GLAMOR_TEXTURE_DRM:
 		case GLAMOR_TEXTURE_ONLY:
-			glamor_pixmap_ensure_fbo(pixmap, GL_RGBA, 0);
+			if (!glamor_pixmap_ensure_fbo(pixmap, GL_RGBA, 0))
+				return -1;
 			return glamor_egl_dri3_fd_name_from_tex(pixmap->drawable.pScreen,
 								pixmap,
 								pixmap_priv->base.fbo->tex,
diff --git a/src/glamor_fbo.c b/src/glamor_fbo.c
index d1b087e..111d36c 100644
--- a/src/glamor_fbo.c
+++ b/src/glamor_fbo.c
@@ -169,16 +169,17 @@ glamor_pixmap_fbo_cache_put(glamor_pixmap_fbo *fbo)
 #endif
 }
 
-static void
+static int
 glamor_pixmap_ensure_fb(glamor_pixmap_fbo *fbo)
 {
 	glamor_gl_dispatch *dispatch;
-	int status;
+	int status, err = 0;
 
 	dispatch = glamor_get_dispatch(fbo->glamor_priv);
 
 	if (fbo->fb == 0)
 		dispatch->glGenFramebuffers(1, &fbo->fb);
+
 	assert(fbo->tex != 0);
 	dispatch->glBindFramebuffer(GL_FRAMEBUFFER, fbo->fb);
 	dispatch->glFramebufferTexture2D(GL_FRAMEBUFFER,
@@ -211,11 +212,11 @@ glamor_pixmap_ensure_fb(glamor_pixmap_fbo *fbo)
 			str = "unknown error";
 			break;
 		}
-
-		FatalError("destination is framebuffer incomplete: %s [%x]\n",
-			   str, status);
+		glamor_fallback("glamor: Failed to create fbo, %s\n", str);
+		err = -1;
 	}
 	glamor_put_dispatch(fbo->glamor_priv);
+	return err;
 }
 
 glamor_pixmap_fbo *
@@ -244,8 +245,12 @@ glamor_create_fbo_from_tex(glamor_screen_private *glamor_priv,
 		goto done;
 	}
 
-	if (flag != GLAMOR_CREATE_FBO_NO_FBO)
-		glamor_pixmap_ensure_fb(fbo);
+	if (flag != GLAMOR_CREATE_FBO_NO_FBO) {
+		if (glamor_pixmap_ensure_fb(fbo) != 0) {
+			glamor_purge_fbo(fbo);
+			fbo = NULL;
+		}
+        }
 
 done:
 	return fbo;
@@ -570,7 +575,8 @@ glamor_pixmap_ensure_fbo(PixmapPtr pixmap, GLenum format, int flag)
 								   pixmap->drawable.height, format);
 
 		if (flag != GLAMOR_CREATE_FBO_NO_FBO && pixmap_priv->base.fbo->fb == 0)
-			glamor_pixmap_ensure_fb(pixmap_priv->base.fbo);
+			if (glamor_pixmap_ensure_fb(pixmap_priv->base.fbo) != 0)
+				return FALSE;
 	}
 
 	return TRUE;