glamor: Fallback to system memory when fail to allocate one big fbo.

Even when create a pixmap which smaller than the max_fbo_size,
it may fail due to some low level driver limitation. If that is
the case, we don't need to crash the xserver. We just need to
fallback to system memory.

See the related bug at:
https://bugs.freedesktop.org/show_bug.cgi?id=71190

Signed-off-by: Zhigang Gong <zhigang.gong@intel.com>
Reviewed-by: Michel Dänzer <michel.daenzer@amd.com>
Tested-by: Kai Wasserbach <kai@dev.carbon-project.org>
Tested-by: Erich Seifert <eseifert@error-reports.org>
Signed-off-by: Michel Dänzer <michel.daenzer@amd.com>
Signed-off-by: Eric Anholt <eric@anholt.net>

2 files changed, 23 insertions, 15 deletions

diff --git a/glamor/glamor.c b/glamor/glamor.c
index 994e92358..08f6ba174 100644
--- a/glamor/glamor.c
+++ b/glamor/glamor.c
@@ -146,7 +146,7 @@ glamor_create_pixmap(ScreenPtr screen, int w, int h, int depth,
     glamor_pixmap_type_t type = GLAMOR_TEXTURE_ONLY;
     glamor_pixmap_private *pixmap_priv;
     glamor_screen_private *glamor_priv = glamor_get_screen_private(screen);
-    glamor_pixmap_fbo *fbo;
+    glamor_pixmap_fbo *fbo = NULL;
     int pitch;
     GLenum format;
 
@@ -199,13 +199,12 @@ glamor_create_pixmap(ScreenPtr screen, int w, int h, int depth,
         pixmap_priv->base.box.x2 = w;
         pixmap_priv->base.box.y2 = h;
         fbo = glamor_create_fbo(glamor_priv, w, h, format, usage);
-    }
-    else {
-        DEBUGF("Create LARGE pixmap %p width %d height %d\n", pixmap, w, h);
+    } else {
+        int tile_size = glamor_priv->max_fbo_size;
+        DEBUGF("Create LARGE pixmap %p width %d height %d, tile size %d\n", pixmap, w, h, tile_size);
         pixmap_priv->type = GLAMOR_TEXTURE_LARGE;
         fbo = glamor_create_fbo_array(glamor_priv, w, h, format, usage,
-                                      glamor_priv->max_fbo_size,
-                                      glamor_priv->max_fbo_size, pixmap_priv);
+                                      tile_size, tile_size, pixmap_priv);
     }
 
     if (fbo == NULL) {
@@ -658,7 +657,8 @@ glamor_fd_from_pixmap(ScreenPtr screen,
     switch (pixmap_priv->type) {
     case GLAMOR_TEXTURE_DRM:
     case GLAMOR_TEXTURE_ONLY:
-        glamor_pixmap_ensure_fbo(pixmap, GL_RGBA, 0);
+        if (!glamor_pixmap_ensure_fbo(pixmap, GL_RGBA, 0))
+            return -1;
         return glamor_egl_dri3_fd_name_from_tex(screen,
                                                 pixmap,
                                                 pixmap_priv->base.fbo->tex,
@@ -682,7 +682,8 @@ glamor_name_from_pixmap(PixmapPtr pixmap, CARD16 *stride, CARD32 *size)
     switch (pixmap_priv->type) {
     case GLAMOR_TEXTURE_DRM:
     case GLAMOR_TEXTURE_ONLY:
-        glamor_pixmap_ensure_fbo(pixmap, GL_RGBA, 0);
+        if (!glamor_pixmap_ensure_fbo(pixmap, GL_RGBA, 0))
+            return -1;
         return glamor_egl_dri3_fd_name_from_tex(pixmap->drawable.pScreen,
                                                 pixmap,
                                                 pixmap_priv->base.fbo->tex,
diff --git a/glamor/glamor_fbo.c b/glamor/glamor_fbo.c
index ed4e348c1..552168381 100644
--- a/glamor/glamor_fbo.c
+++ b/glamor/glamor_fbo.c
@@ -174,10 +174,10 @@ glamor_pixmap_fbo_cache_put(glamor_pixmap_fbo *fbo)
 #endif
 }
 
-static void
+static int
 glamor_pixmap_ensure_fb(glamor_pixmap_fbo *fbo)
 {
-    int status;
+    int status, err = 0;
 
     glamor_make_current(fbo->glamor_priv);
 
@@ -215,9 +215,11 @@ glamor_pixmap_ensure_fb(glamor_pixmap_fbo *fbo)
             break;
         }
 
-        FatalError("destination is framebuffer incomplete: %s [%x]\n",
-                   str, status);
+        glamor_fallback("glamor: Failed to create fbo, %s\n", str);
+        err = -1;
     }
+
+    return err;
 }
 
 glamor_pixmap_fbo *
@@ -244,8 +246,12 @@ glamor_create_fbo_from_tex(glamor_screen_private *glamor_priv,
         goto done;
     }
 
-    if (flag != GLAMOR_CREATE_FBO_NO_FBO)
-        glamor_pixmap_ensure_fb(fbo);
+    if (flag != GLAMOR_CREATE_FBO_NO_FBO) {
+        if (glamor_pixmap_ensure_fb(fbo) != 0) {
+            glamor_purge_fbo(fbo);
+            fbo = NULL;
+        }
+    }
 
  done:
     return fbo;
@@ -562,7 +568,8 @@ glamor_pixmap_ensure_fbo(PixmapPtr pixmap, GLenum format, int flag)
                                    pixmap->drawable.height, format);
 
         if (flag != GLAMOR_CREATE_FBO_NO_FBO && pixmap_priv->base.fbo->fb == 0)
-            glamor_pixmap_ensure_fb(pixmap_priv->base.fbo);
+            if (glamor_pixmap_ensure_fb(pixmap_priv->base.fbo) != 0)
+                return FALSE;
     }
 
     return TRUE;