vbo: better handling of VBO allocation failures

Previously, if we failed to allocate a VBO (either for display list
compilation or immediate mode rendering) we'd eventually segfault
when trying to map the non-existant buffer or in a glVertex/Color/etc
call when we hit a null pointer.

Now we don't try to map non-existant buffers and if we do fail to
allocate a VBO we plug in no-op functions for glVertex/Color/etc
so we don't segfault.

5 files changed, 125 insertions, 39 deletions

diff --git a/src/mesa/vbo/vbo_exec.h b/src/mesa/vbo/vbo_exec.h
index 0b72579a82c..8d6b8f95a7f 100644
--- a/src/mesa/vbo/vbo_exec.h
+++ b/src/mesa/vbo/vbo_exec.h
@@ -82,6 +82,7 @@ struct vbo_exec_context
 {
    struct gl_context *ctx;   
    GLvertexformat vtxfmt;
+   GLvertexformat vtxfmt_noop;
 
    struct {
       struct gl_buffer_object *bufferobj;
diff --git a/src/mesa/vbo/vbo_exec_api.c b/src/mesa/vbo/vbo_exec_api.c
index 9e8ae7d7a41..62e7d03f38a 100644
--- a/src/mesa/vbo/vbo_exec_api.c
+++ b/src/mesa/vbo/vbo_exec_api.c
@@ -45,6 +45,8 @@ USE OR OTHER DEALINGS IN THE SOFTWARE.
 #include "main/dispatch.h"
 
 #include "vbo_context.h"
+#include "vbo_noop.h"
+
 
 #ifdef ERROR
 #undef ERROR
@@ -1028,7 +1030,9 @@ void vbo_use_buffer_objects(struct gl_context *ctx)
    /* Allocate a real buffer object now */
    _mesa_reference_buffer_object(ctx, &exec->vtx.bufferobj, NULL);
    exec->vtx.bufferobj = ctx->Driver.NewBufferObject(ctx, bufName, target);
-   ctx->Driver.BufferData(ctx, target, size, NULL, usage, exec->vtx.bufferobj);
+   if (!ctx->Driver.BufferData(ctx, target, size, NULL, usage, exec->vtx.bufferobj)) {
+      _mesa_error(ctx, GL_OUT_OF_MEMORY, "VBO allocation");
+   }
 }
 
 
@@ -1065,6 +1069,7 @@ void vbo_exec_vtx_init( struct vbo_exec_context *exec )
    exec->vtx.buffer_ptr = exec->vtx.buffer_map;
 
    vbo_exec_vtxfmt_init( exec );
+   _mesa_noop_vtxfmt_init(&exec->vtxfmt_noop);
 
    /* Hook our functions into the dispatch table.
     */
diff --git a/src/mesa/vbo/vbo_exec_draw.c b/src/mesa/vbo/vbo_exec_draw.c
index 8ffaaaa4876..efb6dd10a80 100644
--- a/src/mesa/vbo/vbo_exec_draw.c
+++ b/src/mesa/vbo/vbo_exec_draw.c
@@ -28,11 +28,14 @@
 #include "main/glheader.h"
 #include "main/bufferobj.h"
 #include "main/compiler.h"
+#include "main/context.h"
 #include "main/enums.h"
 #include "main/mfeatures.h"
 #include "main/state.h"
+#include "main/vtxfmt.h"
 
 #include "vbo_context.h"
+#include "vbo_noop.h"
 
 
 #if FEATURE_beginend
@@ -308,32 +311,55 @@ vbo_exec_vtx_map( struct vbo_exec_context *exec )
 
    if (VBO_VERT_BUFFER_SIZE > exec->vtx.buffer_used + 1024) {
       /* The VBO exists and there's room for more */
-      exec->vtx.buffer_map = 
-         (GLfloat *)ctx->Driver.MapBufferRange(ctx, 
-                                               exec->vtx.buffer_used,
-                                               (VBO_VERT_BUFFER_SIZE - 
-                                                exec->vtx.buffer_used),
-                                               accessRange,
-                                               exec->vtx.bufferobj);
-      exec->vtx.buffer_ptr = exec->vtx.buffer_map;
+      if (exec->vtx.bufferobj->Size > 0) {
+         exec->vtx.buffer_map =
+            (GLfloat *)ctx->Driver.MapBufferRange(ctx, 
+                                                  exec->vtx.buffer_used,
+                                                  (VBO_VERT_BUFFER_SIZE - 
+                                                   exec->vtx.buffer_used),
+                                                  accessRange,
+                                                  exec->vtx.bufferobj);
+         exec->vtx.buffer_ptr = exec->vtx.buffer_map;
+      }
+      else {
+         exec->vtx.buffer_ptr = exec->vtx.buffer_map = NULL;
+      }
    }
    
    if (!exec->vtx.buffer_map) {
       /* Need to allocate a new VBO */
       exec->vtx.buffer_used = 0;
 
-      ctx->Driver.BufferData(ctx, GL_ARRAY_BUFFER_ARB,
-                             VBO_VERT_BUFFER_SIZE, 
-                             NULL, usage, exec->vtx.bufferobj);
+      if (ctx->Driver.BufferData(ctx, GL_ARRAY_BUFFER_ARB,
+                                  VBO_VERT_BUFFER_SIZE, 
+                                  NULL, usage, exec->vtx.bufferobj)) {
+         /* buffer allocation worked, now map the buffer */
+         exec->vtx.buffer_map =
+            (GLfloat *)ctx->Driver.MapBufferRange(ctx,
+                                                  0, VBO_VERT_BUFFER_SIZE,
+                                                  accessRange,
+                                                  exec->vtx.bufferobj);
+      }
+      else {
+         _mesa_error(ctx, GL_OUT_OF_MEMORY, "VBO allocation");
+         exec->vtx.buffer_map = NULL;
+      }
+   }
 
+   exec->vtx.buffer_ptr = exec->vtx.buffer_map;
 
-      exec->vtx.buffer_map =
-	 (GLfloat *)ctx->Driver.MapBufferRange(ctx,
-					       0, VBO_VERT_BUFFER_SIZE,
-					       accessRange,
-					       exec->vtx.bufferobj);
-      assert(exec->vtx.buffer_map);
-      exec->vtx.buffer_ptr = exec->vtx.buffer_map;
+   if (!exec->vtx.buffer_map) {
+      /* out of memory */
+      _mesa_install_exec_vtxfmt( ctx, &exec->vtxfmt_noop );
+   }
+   else {
+      if (_mesa_using_noop_vtxfmt(ctx->Exec)) {
+         /* The no-op functions are installed so switch back to regular
+          * functions.  We do this test just to avoid frequent and needless
+          * calls to _mesa_install_exec_vtxfmt().
+          */
+         _mesa_install_exec_vtxfmt(ctx, &exec->vtxfmt);
+      }
    }
 
    if (0)
diff --git a/src/mesa/vbo/vbo_save.h b/src/mesa/vbo/vbo_save.h
index a85a7cbf631..4d4a5bf1710 100644
--- a/src/mesa/vbo/vbo_save.h
+++ b/src/mesa/vbo/vbo_save.h
@@ -122,6 +122,7 @@ struct vbo_save_primitive_store {
 struct vbo_save_context {
    struct gl_context *ctx;
    GLvertexformat vtxfmt;
+   GLvertexformat vtxfmt_noop;  /**< Used if out_of_memory is true */
    struct gl_client_array arrays[VBO_ATTRIB_MAX];
    const struct gl_client_array *inputs[VBO_ATTRIB_MAX];
 
@@ -129,6 +130,8 @@ struct vbo_save_context {
    GLubyte active_sz[VBO_ATTRIB_MAX];
    GLuint vertex_size;
 
+   GLboolean out_of_memory;  /**< True if last VBO allocation failed */
+
    GLfloat *buffer;
    GLuint count;
    GLuint wrap_count;
diff --git a/src/mesa/vbo/vbo_save_api.c b/src/mesa/vbo/vbo_save_api.c
index eba64b3b346..64da7ac4993 100644
--- a/src/mesa/vbo/vbo_save_api.c
+++ b/src/mesa/vbo/vbo_save_api.c
@@ -81,6 +81,7 @@ USE OR OTHER DEALINGS IN THE SOFTWARE.
 #include "main/dispatch.h"
 
 #include "vbo_context.h"
+#include "vbo_noop.h"
 
 
 #if FEATURE_dlist
@@ -184,6 +185,7 @@ _save_copy_vertices(struct gl_context *ctx,
 static struct vbo_save_vertex_store *
 alloc_vertex_store(struct gl_context *ctx)
 {
+   struct vbo_save_context *save = &vbo_context(ctx)->save;
    struct vbo_save_vertex_store *vertex_store =
       CALLOC_STRUCT(vbo_save_vertex_store);
 
@@ -196,11 +198,22 @@ alloc_vertex_store(struct gl_context *ctx)
    vertex_store->bufferobj = ctx->Driver.NewBufferObject(ctx,
                                                          VBO_BUF_ID,
                                                          GL_ARRAY_BUFFER_ARB);
+   if (vertex_store->bufferobj) {
+      save->out_of_memory =
+         !ctx->Driver.BufferData(ctx,
+                                 GL_ARRAY_BUFFER_ARB,
+                                 VBO_SAVE_BUFFER_SIZE * sizeof(GLfloat),
+                                 NULL, GL_STATIC_DRAW_ARB,
+                                 vertex_store->bufferobj);
+   }
+   else {
+      save->out_of_memory = GL_TRUE;
+   }
 
-   ctx->Driver.BufferData(ctx,
-                          GL_ARRAY_BUFFER_ARB,
-                          VBO_SAVE_BUFFER_SIZE * sizeof(GLfloat),
-                          NULL, GL_STATIC_DRAW_ARB, vertex_store->bufferobj);
+   if (save->out_of_memory) {
+      _mesa_error(ctx, GL_OUT_OF_MEMORY, "internal VBO allocation");
+      _mesa_install_save_vtxfmt(ctx, &save->vtxfmt_noop);
+   }
 
    vertex_store->buffer = NULL;
    vertex_store->used = 0;
@@ -230,14 +243,19 @@ map_vertex_store(struct gl_context *ctx,
 {
    assert(vertex_store->bufferobj);
    assert(!vertex_store->buffer);
-   vertex_store->buffer =
-      (GLfloat *) ctx->Driver.MapBufferRange(ctx, 0,
-					     vertex_store->bufferobj->Size,
-					     GL_MAP_WRITE_BIT,    /* not used */
-					     vertex_store->bufferobj);
-
-   assert(vertex_store->buffer);
-   return vertex_store->buffer + vertex_store->used;
+   if (vertex_store->bufferobj->Size > 0) {
+      vertex_store->buffer =
+         (GLfloat *) ctx->Driver.MapBufferRange(ctx, 0,
+                                                vertex_store->bufferobj->Size,
+                                                GL_MAP_WRITE_BIT,  /* not used */
+                                                vertex_store->bufferobj);
+      assert(vertex_store->buffer);
+      return vertex_store->buffer + vertex_store->used;
+   }
+   else {
+      /* probably ran out of memory for buffers */
+      return NULL;
+   }
 }
 
 
@@ -245,7 +263,9 @@ static void
 unmap_vertex_store(struct gl_context *ctx,
                    struct vbo_save_vertex_store *vertex_store)
 {
-   ctx->Driver.UnmapBuffer(ctx, vertex_store->bufferobj);
+   if (vertex_store->bufferobj->Size > 0) {
+      ctx->Driver.UnmapBuffer(ctx, vertex_store->bufferobj);
+   }
    vertex_store->buffer = NULL;
 }
 
@@ -399,6 +419,7 @@ _save_compile_vertex_list(struct gl_context *ctx)
        */
       save->vertex_store = alloc_vertex_store(ctx);
       save->buffer_ptr = map_vertex_store(ctx, save->vertex_store);
+      save->out_of_memory = save->buffer_ptr == NULL;
    }
 
    if (save->prim_store->used > VBO_SAVE_PRIM_SIZE - 6) {
@@ -732,7 +753,12 @@ dlist_fallback(struct gl_context *ctx)
    _save_copy_to_current(ctx);
    _save_reset_vertex(ctx);
    _save_reset_counters(ctx);
-   _mesa_install_save_vtxfmt(ctx, &ctx->ListState.ListVtxfmt);
+   if (save->out_of_memory) {
+      _mesa_install_save_vtxfmt(ctx, &save->vtxfmt_noop);
+   }
+   else {
+      _mesa_install_save_vtxfmt(ctx, &ctx->ListState.ListVtxfmt);
+   }
    ctx->Driver.SaveNeedFlush = 0;
 }
 
@@ -825,7 +851,12 @@ vbo_save_NotifyBegin(struct gl_context *ctx, GLenum mode)
    save->prim[i].count = 0;
    save->prim[i].num_instances = 1;
 
-   _mesa_install_save_vtxfmt(ctx, &save->vtxfmt);
+   if (save->out_of_memory) {
+      _mesa_install_save_vtxfmt(ctx, &save->vtxfmt_noop);
+   }
+   else {
+      _mesa_install_save_vtxfmt(ctx, &save->vtxfmt);
+   }
    ctx->Driver.SaveNeedFlush = 1;
    return GL_TRUE;
 }
@@ -851,7 +882,12 @@ _save_End(void)
     * etc. received between here and the next begin will be compiled
     * as opcodes.
     */
-   _mesa_install_save_vtxfmt(ctx, &ctx->ListState.ListVtxfmt);
+   if (save->out_of_memory) {
+      _mesa_install_save_vtxfmt(ctx, &save->vtxfmt_noop);
+   }
+   else {
+      _mesa_install_save_vtxfmt(ctx, &ctx->ListState.ListVtxfmt);
+   }
 }
 
 
@@ -1042,11 +1078,15 @@ static void GLAPIENTRY
 _save_OBE_DrawArrays(GLenum mode, GLint start, GLsizei count)
 {
    GET_CURRENT_CONTEXT(ctx);
+   struct vbo_save_context *save = &vbo_context(ctx)->save;
    GLint i;
 
    if (!_mesa_validate_DrawArrays(ctx, mode, start, count))
       return;
 
+   if (save->out_of_memory)
+      return;
+
    _ae_map_vbos(ctx);
 
    vbo_save_NotifyBegin(ctx, (mode | VBO_SAVE_PRIM_WEAK
@@ -1068,11 +1108,15 @@ _save_OBE_DrawElements(GLenum mode, GLsizei count, GLenum type,
                        const GLvoid * indices)
 {
    GET_CURRENT_CONTEXT(ctx);
+   struct vbo_save_context *save = &vbo_context(ctx)->save;
    GLint i;
 
    if (!_mesa_validate_DrawElements(ctx, mode, count, type, indices, 0))
       return;
 
+   if (save->out_of_memory)
+      return;
+
    _ae_map_vbos(ctx);
 
    if (_mesa_is_bufferobj(ctx->Array.ElementArrayBufferObj))
@@ -1112,10 +1156,16 @@ _save_OBE_DrawRangeElements(GLenum mode, GLuint start, GLuint end,
                             const GLvoid * indices)
 {
    GET_CURRENT_CONTEXT(ctx);
-   if (_mesa_validate_DrawRangeElements(ctx, mode,
-                                        start, end, count, type, indices, 0)) {
-      _save_OBE_DrawElements(mode, count, type, indices);
-   }
+   struct vbo_save_context *save = &vbo_context(ctx)->save;
+
+   if (!_mesa_validate_DrawRangeElements(ctx, mode,
+                                         start, end, count, type, indices, 0))
+      return;
+
+   if (save->out_of_memory)
+      return;
+
+   _save_OBE_DrawElements(mode, count, type, indices);
 }
 
 
@@ -1487,6 +1537,7 @@ vbo_save_api_init(struct vbo_save_context *save)
 
    _save_vtxfmt_init(ctx);
    _save_current_init(ctx);
+   _mesa_noop_vtxfmt_init(&save->vtxfmt_noop);
 
    /* These will actually get set again when binding/drawing */
    for (i = 0; i < VBO_ATTRIB_MAX; i++)