zink: add a suballocator

this is an aux/pipebuffer implementation borrowing heavily from the
one in radeonsi. it currently has the following limitations, which
will be resolved in a followup series:
* 32bit address space still explodes
* swapchain images still have separate memory handling

performance in games like Tomb Raider has been observed to increase by
over 1000%

SQUASHED: simplify get_memory_type_index()

now that the heaps are enumerated, this can be reduced to a simple
array index with a fallback

Reviewed-by: Dave Airlie <airlied@redhat.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/12146>

8 files changed, 1425 insertions, 250 deletions

diff --git a/src/gallium/drivers/zink/meson.build b/src/gallium/drivers/zink/meson.build
index 28aae9cdfc4..87501df180e 100644
--- a/src/gallium/drivers/zink/meson.build
+++ b/src/gallium/drivers/zink/meson.build
@@ -24,6 +24,7 @@ files_libzink = files(
   'nir_to_spirv/spirv_builder.c',
   'zink_batch.c',
   'zink_blit.c',
+  'zink_bo.c',
   'zink_clear.c',
   'zink_compiler.c',
   'zink_context.c',
diff --git a/src/gallium/drivers/zink/zink_bo.c b/src/gallium/drivers/zink/zink_bo.c
new file mode 100644
index 00000000000..b73d6cfbb9d
--- /dev/null
+++ b/src/gallium/drivers/zink/zink_bo.c
@@ -0,0 +1,1014 @@
+/*
+ * Copyright © 2011 Marek Olšák <maraeo@gmail.com>
+ * Copyright © 2015 Advanced Micro Devices, Inc.
+ * Copyright © 2021 Valve Corporation
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
+ * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ * 
+ * Authors:
+ *    Mike Blumenkrantz <michael.blumenkrantz@gmail.com>
+ */
+
+#include "zink_bo.h"
+#include "zink_resource.h"
+#include "zink_screen.h"
+#include "util/u_hash_table.h"
+
+struct zink_bo;
+
+struct zink_sparse_backing_chunk {
+   uint32_t begin, end;
+};
+
+
+/*
+ * Sub-allocation information for a real buffer used as backing memory of a
+ * sparse buffer.
+ */
+struct zink_sparse_backing {
+   struct list_head list;
+
+   struct zink_bo *bo;
+
+   /* Sorted list of free chunks. */
+   struct zink_sparse_backing_chunk *chunks;
+   uint32_t max_chunks;
+   uint32_t num_chunks;
+};
+
+struct zink_sparse_commitment {
+   struct zink_sparse_backing *backing;
+   uint32_t page;
+};
+
+struct zink_slab {
+   struct pb_slab base;
+   unsigned entry_size;
+   struct zink_bo *buffer;
+   struct zink_bo *entries;
+};
+
+
+ALWAYS_INLINE static struct zink_slab *
+zink_slab(struct pb_slab *pslab)
+{
+   return (struct zink_slab*)pslab;
+}
+
+static struct pb_slabs *
+get_slabs(struct zink_screen *screen, uint64_t size, enum zink_alloc_flag flags)
+{
+   //struct pb_slabs *bo_slabs = ((flags & RADEON_FLAG_ENCRYPTED) && screen->info.has_tmz_support) ?
+      //screen->bo_slabs_encrypted : screen->bo_slabs;
+
+   struct pb_slabs *bo_slabs = screen->pb.bo_slabs;
+   /* Find the correct slab allocator for the given size. */
+   for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
+      struct pb_slabs *slabs = &bo_slabs[i];
+
+      if (size <= 1ULL << (slabs->min_order + slabs->num_orders - 1))
+         return slabs;
+   }
+
+   assert(0);
+   return NULL;
+}
+
+/* Return the power of two size of a slab entry matching the input size. */
+static unsigned
+get_slab_pot_entry_size(struct zink_screen *screen, unsigned size)
+{
+   unsigned entry_size = util_next_power_of_two(size);
+   unsigned min_entry_size = 1 << screen->pb.bo_slabs[0].min_order;
+
+   return MAX2(entry_size, min_entry_size);
+}
+
+/* Return the slab entry alignment. */
+static unsigned get_slab_entry_alignment(struct zink_screen *screen, unsigned size)
+{
+   unsigned entry_size = get_slab_pot_entry_size(screen, size);
+
+   if (size <= entry_size * 3 / 4)
+      return entry_size / 4;
+
+   return entry_size;
+}
+
+static void
+bo_destroy(struct zink_screen *screen, struct pb_buffer *pbuf)
+{
+   struct zink_bo *bo = zink_bo(pbuf);
+
+   simple_mtx_lock(&screen->pb.bo_export_table_lock);
+   _mesa_hash_table_remove_key(screen->pb.bo_export_table, bo);
+   simple_mtx_unlock(&screen->pb.bo_export_table_lock);
+
+   if (!bo->u.real.is_user_ptr && bo->u.real.cpu_ptr) {
+      bo->u.real.map_count = 1;
+      bo->u.real.cpu_ptr = NULL;
+      zink_bo_unmap(screen, bo);
+   }
+
+   vkFreeMemory(screen->dev, bo->mem, NULL);
+
+   simple_mtx_destroy(&bo->lock);
+   FREE(bo);
+}
+
+static bool
+bo_can_reclaim(struct zink_screen *screen, struct pb_buffer *pbuf)
+{
+   struct zink_bo *bo = zink_bo(pbuf);
+
+   return zink_screen_usage_check_completion(screen, bo->reads) && zink_screen_usage_check_completion(screen, bo->writes);
+}
+
+static bool
+bo_can_reclaim_slab(void *priv, struct pb_slab_entry *entry)
+{
+   struct zink_bo *bo = container_of(entry, struct zink_bo, u.slab.entry);
+
+   return bo_can_reclaim(priv, &bo->base);
+}
+
+static void
+bo_slab_free(struct zink_screen *screen, struct pb_slab *pslab)
+{
+   struct zink_slab *slab = zink_slab(pslab);
+   ASSERTED unsigned slab_size = slab->buffer->base.size;
+
+   assert(slab->base.num_entries * slab->entry_size <= slab_size);
+   FREE(slab->entries);
+   zink_bo_unref(screen, slab->buffer);
+   FREE(slab);
+}
+
+static void
+bo_slab_destroy(struct zink_screen *screen, struct pb_buffer *pbuf)
+{
+   struct zink_bo *bo = zink_bo(pbuf);
+
+   assert(!bo->mem);
+
+   //if (bo->base.usage & RADEON_FLAG_ENCRYPTED)
+      //pb_slab_free(get_slabs(screen, bo->base.size, RADEON_FLAG_ENCRYPTED), &bo->u.slab.entry);
+   //else
+      pb_slab_free(get_slabs(screen, bo->base.size, 0), &bo->u.slab.entry);
+}
+
+static void
+clean_up_buffer_managers(struct zink_screen *screen)
+{
+   for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
+      pb_slabs_reclaim(&screen->pb.bo_slabs[i]);
+      //if (screen->info.has_tmz_support)
+         //pb_slabs_reclaim(&screen->bo_slabs_encrypted[i]);
+   }
+
+   pb_cache_release_all_buffers(&screen->pb.bo_cache);
+}
+
+static unsigned
+get_optimal_alignment(struct zink_screen *screen, uint64_t size, unsigned alignment)
+{
+   /* Increase the alignment for faster address translation and better memory
+    * access pattern.
+    */
+   if (size >= 4096) {
+      alignment = MAX2(alignment, 4096);
+   } else if (size) {
+      unsigned msb = util_last_bit(size);
+
+      alignment = MAX2(alignment, 1u << (msb - 1));
+   }
+   return alignment;
+}
+
+static void
+bo_destroy_or_cache(struct zink_screen *screen, struct pb_buffer *pbuf)
+{
+   struct zink_bo *bo = zink_bo(pbuf);
+
+   assert(bo->mem); /* slab buffers have a separate vtbl */
+   bo->reads = NULL;
+   bo->writes = NULL;
+
+   if (bo->u.real.use_reusable_pool)
+      pb_cache_add_buffer(bo->cache_entry);
+   else
+      bo_destroy(screen, pbuf);
+}
+
+static const struct pb_vtbl bo_vtbl = {
+   /* Cast to void* because one of the function parameters is a struct pointer instead of void*. */
+   (void*)bo_destroy_or_cache
+   /* other functions are never called */
+};
+
+static struct zink_bo *
+bo_create_internal(struct zink_screen *screen,
+                   uint64_t size,
+                   unsigned alignment,
+                   enum zink_heap heap,
+                   unsigned flags)
+{
+   struct zink_bo *bo;
+   bool init_pb_cache;
+
+   /* too big for vk alloc */
+   if (size > UINT32_MAX)
+      return NULL;
+
+   alignment = get_optimal_alignment(screen, size, alignment);
+
+   /* all non-suballocated bo can cache */
+   init_pb_cache = true;
+
+   bo = CALLOC(1, sizeof(struct zink_bo) + init_pb_cache * sizeof(struct pb_cache_entry));
+   if (!bo) {
+      return NULL;
+   }
+
+   if (init_pb_cache) {
+      bo->u.real.use_reusable_pool = true;
+      pb_cache_init_entry(&screen->pb.bo_cache, bo->cache_entry, &bo->base, heap);
+   }
+
+   VkMemoryAllocateInfo mai = {0};
+   mai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
+   mai.allocationSize = size;
+   mai.memoryTypeIndex = screen->heap_map[heap];
+   if (screen->info.mem_props.memoryTypes[mai.memoryTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
+      alignment = MAX2(alignment, screen->info.props.limits.minMemoryMapAlignment);
+      mai.allocationSize = align(mai.allocationSize, screen->info.props.limits.minMemoryMapAlignment);
+   }
+   VkResult ret = vkAllocateMemory(screen->dev, &mai, NULL, &bo->mem);
+   if (!zink_screen_handle_vkresult(screen, ret))
+      goto fail;
+
+   simple_mtx_init(&bo->lock, mtx_plain);
+   pipe_reference_init(&bo->base.reference, 1);
+   bo->base.alignment_log2 = util_logbase2(alignment);
+   bo->base.size = size;
+   bo->base.vtbl = &bo_vtbl;
+   bo->base.placement = vk_domain_from_heap(heap);
+   bo->base.usage = flags;
+   bo->unique_id = p_atomic_inc_return(&screen->pb.next_bo_unique_id);
+
+   return bo;
+
+fail:
+   bo_destroy(screen, (void*)bo);
+   return NULL;
+}
+
+/*
+ * Attempt to allocate the given number of backing pages. Fewer pages may be
+ * allocated (depending on the fragmentation of existing backing buffers),
+ * which will be reflected by a change to *pnum_pages.
+ */
+static struct zink_sparse_backing *
+sparse_backing_alloc(struct zink_screen *screen, struct zink_bo *bo,
+                     uint32_t *pstart_page, uint32_t *pnum_pages)
+{
+   struct zink_sparse_backing *best_backing;
+   unsigned best_idx;
+   uint32_t best_num_pages;
+
+   best_backing = NULL;
+   best_idx = 0;
+   best_num_pages = 0;
+
+   /* This is a very simple and inefficient best-fit algorithm. */
+   list_for_each_entry(struct zink_sparse_backing, backing, &bo->u.sparse.backing, list) {
+      for (unsigned idx = 0; idx < backing->num_chunks; ++idx) {
+         uint32_t cur_num_pages = backing->chunks[idx].end - backing->chunks[idx].begin;
+         if ((best_num_pages < *pnum_pages && cur_num_pages > best_num_pages) ||
+            (best_num_pages > *pnum_pages && cur_num_pages < best_num_pages)) {
+            best_backing = backing;
+            best_idx = idx;
+            best_num_pages = cur_num_pages;
+         }
+      }
+   }
+
+   /* Allocate a new backing buffer if necessary. */
+   if (!best_backing) {
+      struct pb_buffer *buf;
+      uint64_t size;
+      uint32_t pages;
+
+      best_backing = CALLOC_STRUCT(zink_sparse_backing);
+      if (!best_backing)
+         return NULL;
+
+      best_backing->max_chunks = 4;
+      best_backing->chunks = CALLOC(best_backing->max_chunks,
+                                    sizeof(*best_backing->chunks));
+      if (!best_backing->chunks) {
+         FREE(best_backing);
+         return NULL;
+      }
+
+      assert(bo->u.sparse.num_backing_pages < DIV_ROUND_UP(bo->base.size, ZINK_SPARSE_BUFFER_PAGE_SIZE));
+
+      size = MIN3(bo->base.size / 16,
+                  8 * 1024 * 1024,
+                  bo->base.size - (uint64_t)bo->u.sparse.num_backing_pages * ZINK_SPARSE_BUFFER_PAGE_SIZE);
+      size = MAX2(size, ZINK_SPARSE_BUFFER_PAGE_SIZE);
+
+      buf = zink_bo_create(screen, size, ZINK_SPARSE_BUFFER_PAGE_SIZE,
+                           bo->base.placement, ZINK_ALLOC_NO_SUBALLOC);
+      if (!buf) {
+         FREE(best_backing->chunks);
+         FREE(best_backing);
+         return NULL;
+      }
+
+      /* We might have gotten a bigger buffer than requested via caching. */
+      pages = buf->size / ZINK_SPARSE_BUFFER_PAGE_SIZE;
+
+      best_backing->bo = zink_bo(buf);
+      best_backing->num_chunks = 1;
+      best_backing->chunks[0].begin = 0;
+      best_backing->chunks[0].end = pages;
+
+      list_add(&best_backing->list, &bo->u.sparse.backing);
+      bo->u.sparse.num_backing_pages += pages;
+
+      best_idx = 0;
+      best_num_pages = pages;
+   }
+
+   *pnum_pages = MIN2(*pnum_pages, best_num_pages);
+   *pstart_page = best_backing->chunks[best_idx].begin;
+   best_backing->chunks[best_idx].begin += *pnum_pages;
+
+   if (best_backing->chunks[best_idx].begin >= best_backing->chunks[best_idx].end) {
+      memmove(&best_backing->chunks[best_idx], &best_backing->chunks[best_idx + 1],
+              sizeof(*best_backing->chunks) * (best_backing->num_chunks - best_idx - 1));
+      best_backing->num_chunks--;
+   }
+
+   return best_backing;
+}
+
+static void
+sparse_free_backing_buffer(struct zink_screen *screen, struct zink_bo *bo,
+                           struct zink_sparse_backing *backing)
+{
+   bo->u.sparse.num_backing_pages -= backing->bo->base.size / ZINK_SPARSE_BUFFER_PAGE_SIZE;
+
+   list_del(&backing->list);
+   zink_bo_unref(screen, backing->bo);
+   FREE(backing->chunks);
+   FREE(backing);
+}
+
+/*
+ * Return a range of pages from the given backing buffer back into the
+ * free structure.
+ */
+static bool
+sparse_backing_free(struct zink_screen *screen, struct zink_bo *bo,
+                    struct zink_sparse_backing *backing,
+                    uint32_t start_page, uint32_t num_pages)
+{
+   uint32_t end_page = start_page + num_pages;
+   unsigned low = 0;
+   unsigned high = backing->num_chunks;
+
+   /* Find the first chunk with begin >= start_page. */
+   while (low < high) {
+      unsigned mid = low + (high - low) / 2;
+
+      if (backing->chunks[mid].begin >= start_page)
+         high = mid;
+      else
+         low = mid + 1;
+   }
+
+   assert(low >= backing->num_chunks || end_page <= backing->chunks[low].begin);
+   assert(low == 0 || backing->chunks[low - 1].end <= start_page);
+
+   if (low > 0 && backing->chunks[low - 1].end == start_page) {
+      backing->chunks[low - 1].end = end_page;
+
+      if (low < backing->num_chunks && end_page == backing->chunks[low].begin) {
+         backing->chunks[low - 1].end = backing->chunks[low].end;
+         memmove(&backing->chunks[low], &backing->chunks[low + 1],
+                 sizeof(*backing->chunks) * (backing->num_chunks - low - 1));
+         backing->num_chunks--;
+      }
+   } else if (low < backing->num_chunks && end_page == backing->chunks[low].begin) {
+      backing->chunks[low].begin = start_page;
+   } else {
+      if (backing->num_chunks >= backing->max_chunks) {
+         unsigned new_max_chunks = 2 * backing->max_chunks;
+         struct zink_sparse_backing_chunk *new_chunks =
+            REALLOC(backing->chunks,
+                    sizeof(*backing->chunks) * backing->max_chunks,
+                    sizeof(*backing->chunks) * new_max_chunks);
+         if (!new_chunks)
+            return false;
+
+         backing->max_chunks = new_max_chunks;
+         backing->chunks = new_chunks;
+      }
+
+      memmove(&backing->chunks[low + 1], &backing->chunks[low],
+              sizeof(*backing->chunks) * (backing->num_chunks - low));
+      backing->chunks[low].begin = start_page;
+      backing->chunks[low].end = end_page;
+      backing->num_chunks++;
+   }
+
+   if (backing->num_chunks == 1 && backing->chunks[0].begin == 0 &&
+       backing->chunks[0].end == backing->bo->base.size / ZINK_SPARSE_BUFFER_PAGE_SIZE)
+      sparse_free_backing_buffer(screen, bo, backing);
+
+   return true;
+}
+
+static void
+bo_sparse_destroy(struct zink_screen *screen, struct pb_buffer *pbuf)
+{
+   struct zink_bo *bo = zink_bo(pbuf);
+
+   assert(!bo->mem && bo->base.usage & ZINK_ALLOC_SPARSE);
+
+   while (!list_is_empty(&bo->u.sparse.backing)) {
+      sparse_free_backing_buffer(screen, bo,
+                                 container_of(bo->u.sparse.backing.next,
+                                              struct zink_sparse_backing, list));
+   }
+
+   FREE(bo->u.sparse.commitments);
+   simple_mtx_destroy(&bo->lock);
+   FREE(bo);
+}
+
+static const struct pb_vtbl bo_sparse_vtbl = {
+   /* Cast to void* because one of the function parameters is a struct pointer instead of void*. */
+   (void*)bo_sparse_destroy
+   /* other functions are never called */
+};
+
+static struct pb_buffer *
+bo_sparse_create(struct zink_screen *screen, uint64_t size)
+{
+   struct zink_bo *bo;
+
+   /* We use 32-bit page numbers; refuse to attempt allocating sparse buffers
+    * that exceed this limit. This is not really a restriction: we don't have
+    * that much virtual address space anyway.
+    */
+   if (size > (uint64_t)INT32_MAX * ZINK_SPARSE_BUFFER_PAGE_SIZE)
+      return NULL;
+
+   bo = CALLOC_STRUCT(zink_bo);
+   if (!bo)
+      return NULL;
+
+   simple_mtx_init(&bo->lock, mtx_plain);
+   pipe_reference_init(&bo->base.reference, 1);
+   bo->base.alignment_log2 = util_logbase2(ZINK_SPARSE_BUFFER_PAGE_SIZE);
+   bo->base.size = size;
+   bo->base.vtbl = &bo_sparse_vtbl;
+   bo->base.placement = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
+   bo->unique_id = p_atomic_inc_return(&screen->pb.next_bo_unique_id);
+   bo->base.usage = ZINK_ALLOC_SPARSE;
+
+   bo->u.sparse.num_va_pages = DIV_ROUND_UP(size, ZINK_SPARSE_BUFFER_PAGE_SIZE);
+   bo->u.sparse.commitments = CALLOC(bo->u.sparse.num_va_pages,
+                                     sizeof(*bo->u.sparse.commitments));
+   if (!bo->u.sparse.commitments)
+      goto error_alloc_commitments;
+
+   list_inithead(&bo->u.sparse.backing);
+
+   return &bo->base;
+
+error_alloc_commitments:
+   simple_mtx_destroy(&bo->lock);
+   FREE(bo);
+   return NULL;
+}
+
+struct pb_buffer *
+zink_bo_create(struct zink_screen *screen, uint64_t size, unsigned alignment, enum zink_heap heap, enum zink_alloc_flag flags)
+{
+   struct zink_bo *bo;
+   /* pull in sparse flag */
+   flags |= zink_alloc_flags_from_heap(heap);
+
+   //struct pb_slabs *slabs = ((flags & RADEON_FLAG_ENCRYPTED) && screen->info.has_tmz_support) ?
+      //screen->bo_slabs_encrypted : screen->bo_slabs;
+   struct pb_slabs *slabs = screen->pb.bo_slabs;
+
+   struct pb_slabs *last_slab = &slabs[NUM_SLAB_ALLOCATORS - 1];
+   unsigned max_slab_entry_size = 1 << (last_slab->min_order + last_slab->num_orders - 1);
+
+   /* Sub-allocate small buffers from slabs. */
+   if (!(flags & (ZINK_ALLOC_NO_SUBALLOC | ZINK_ALLOC_SPARSE)) &&
+       size <= max_slab_entry_size) {
+      struct pb_slab_entry *entry;
+
+      if (heap < 0 || heap >= ZINK_HEAP_MAX)
+         goto no_slab;
+
+      unsigned alloc_size = size;
+
+      /* Always use slabs for sizes less than 4 KB because the kernel aligns
+       * everything to 4 KB.
+       */
+      if (size < alignment && alignment <= 4 * 1024)
+         alloc_size = alignment;
+
+      if (alignment > get_slab_entry_alignment(screen, alloc_size)) {
+         /* 3/4 allocations can return too small alignment. Try again with a power of two
+          * allocation size.
+          */
+         unsigned pot_size = get_slab_pot_entry_size(screen, alloc_size);
+
+         if (alignment <= pot_size) {
+            /* This size works but wastes some memory to fulfil the alignment. */
+            alloc_size = pot_size;
+         } else {
+            goto no_slab; /* can't fulfil alignment requirements */
+         }
+      }
+
+      struct pb_slabs *slabs = get_slabs(screen, alloc_size, flags);
+      entry = pb_slab_alloc(slabs, alloc_size, heap);
+      if (!entry) {
+         /* Clean up buffer managers and try again. */
+         clean_up_buffer_managers(screen);
+
+         entry = pb_slab_alloc(slabs, alloc_size, heap);
+      }
+      if (!entry)
+         return NULL;
+
+      bo = container_of(entry, struct zink_bo, u.slab.entry);
+      pipe_reference_init(&bo->base.reference, 1);
+      bo->base.size = size;
+      assert(alignment <= 1 << bo->base.alignment_log2);
+
+      return &bo->base;
+   }
+no_slab:
+
+   if (flags & ZINK_ALLOC_SPARSE) {
+      assert(ZINK_SPARSE_BUFFER_PAGE_SIZE % alignment == 0);
+
+      return bo_sparse_create(screen, size);
+   }
+
+   /* Align size to page size. This is the minimum alignment for normal
+    * BOs. Aligning this here helps the cached bufmgr. Especially small BOs,
+    * like constant/uniform buffers, can benefit from better and more reuse.
+    */
+   if (heap == ZINK_HEAP_DEVICE_LOCAL_VISIBLE) {
+      size = align64(size, screen->info.props.limits.minMemoryMapAlignment);
+      alignment = align(alignment, screen->info.props.limits.minMemoryMapAlignment);
+   }
+
+   bool use_reusable_pool = !(flags & ZINK_ALLOC_NO_SUBALLOC);
+
+   if (use_reusable_pool) {
+       /* Get a buffer from the cache. */
+       bo = (struct zink_bo*)
+            pb_cache_reclaim_buffer(&screen->pb.bo_cache, size, alignment, 0, heap);
+       if (bo)
+          return &bo->base;
+   }
+
+   /* Create a new one. */
+   bo = bo_create_internal(screen, size, alignment, heap, flags);
+   if (!bo) {
+      /* Clean up buffer managers and try again. */
+      clean_up_buffer_managers(screen);
+
+      bo = bo_create_internal(screen, size, alignment, heap, flags);
+      if (!bo)
+         return NULL;
+   }
+
+   return &bo->base;
+}
+
+void *
+zink_bo_map(struct zink_screen *screen, struct zink_bo *bo)
+{
+   void *cpu = NULL;
+   uint64_t offset = 0;
+   struct zink_bo *real;
+
+   if (bo->mem) {
+      real = bo;
+   } else {
+      real = bo->u.slab.real;
+      offset = bo->offset - real->offset;
+   }
+
+   cpu = p_atomic_read(&real->u.real.cpu_ptr);
+   if (!cpu) {
+      simple_mtx_lock(&real->lock);
+      /* Must re-check due to the possibility of a race. Re-check need not
+       * be atomic thanks to the lock. */
+      cpu = real->u.real.cpu_ptr;
+      if (!cpu) {
+         VkResult result = vkMapMemory(screen->dev, real->mem, 0, real->base.size, 0, &cpu);
+         if (result != VK_SUCCESS) {
+            simple_mtx_unlock(&real->lock);
+            return NULL;
+         }
+         p_atomic_set(&real->u.real.cpu_ptr, cpu);
+      }
+      simple_mtx_unlock(&real->lock);
+   }
+   p_atomic_inc(&real->u.real.map_count);
+
+   return (uint8_t*)cpu + offset;
+}
+
+void
+zink_bo_unmap(struct zink_screen *screen, struct zink_bo *bo)
+{
+   struct zink_bo *real = bo->mem ? bo : bo->u.slab.real;
+
+   assert(real->u.real.map_count != 0 && "too many unmaps");
+
+   if (p_atomic_dec_zero(&real->u.real.map_count)) {
+      p_atomic_set(&real->u.real.cpu_ptr, NULL);
+      vkUnmapMemory(screen->dev, real->mem);
+   }
+}
+
+
+static inline struct zink_screen **
+get_screen_ptr_for_commit(uint8_t *mem)
+{
+   return (struct zink_screen**)(mem + sizeof(VkBindSparseInfo) + sizeof(VkSparseBufferMemoryBindInfo) + sizeof(VkSparseMemoryBind));
+}
+
+static bool
+resource_commit(struct zink_screen *screen, VkBindSparseInfo *sparse)
+{
+   VkQueue queue = screen->threaded ? screen->thread_queue : screen->queue;
+
+   VkResult ret = vkQueueBindSparse(queue, 1, sparse, VK_NULL_HANDLE);
+   return zink_screen_handle_vkresult(screen, ret);
+}
+
+static void
+submit_resource_commit(void *data, void *gdata, int thread_index)
+{
+   struct zink_screen **screen = get_screen_ptr_for_commit(data);
+   resource_commit(*screen, data);
+   free(data);
+}
+
+static bool
+do_commit_single(struct zink_screen *screen, struct zink_resource *res, struct zink_bo *bo, uint32_t offset, uint32_t size, bool commit)
+{
+
+   uint8_t *mem = malloc(sizeof(VkBindSparseInfo) + sizeof(VkSparseBufferMemoryBindInfo) + sizeof(VkSparseMemoryBind) + sizeof(void*));
+   if (!mem)
+      return false;
+   VkBindSparseInfo *sparse = (void*)mem;
+   sparse->sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO;
+   sparse->pNext = NULL;
+   sparse->waitSemaphoreCount = 0;
+   sparse->bufferBindCount = 1;
+   sparse->imageOpaqueBindCount = 0;
+   sparse->imageBindCount = 0;
+   sparse->signalSemaphoreCount = 0;
+
+   VkSparseBufferMemoryBindInfo *sparse_bind = (void*)(mem + sizeof(VkBindSparseInfo));
+   sparse_bind->buffer = res->obj->buffer;
+   sparse_bind->bindCount = 1;
+   sparse->pBufferBinds = sparse_bind;
+
+   VkSparseMemoryBind *mem_bind = (void*)(mem + sizeof(VkBindSparseInfo) + sizeof(VkSparseBufferMemoryBindInfo));
+   mem_bind->resourceOffset = offset;
+   mem_bind->size = MIN2(res->base.b.width0 - offset, size);
+   mem_bind->memory = commit ? bo->mem : VK_NULL_HANDLE;
+   mem_bind->memoryOffset = 0;
+   mem_bind->flags = 0;
+   sparse_bind->pBinds = mem_bind;
+
+   struct zink_screen **ptr = get_screen_ptr_for_commit(mem);
+   *ptr = screen;
+
+   if (screen->threaded) {
+      /* this doesn't need any kind of fencing because any access to this resource
+       * will be automagically synchronized by queue dispatch */
+      util_queue_add_job(&screen->flush_queue, mem, NULL, submit_resource_commit, NULL, 0);
+   } else {
+      bool ret = resource_commit(screen, sparse);
+      free(sparse);
+      return ret;
+   }
+   return true;
+}
+
+bool
+zink_bo_commit(struct zink_screen *screen, struct zink_resource *res, uint32_t offset, uint32_t size, bool commit)
+{
+   bool ok = true;
+   struct zink_bo *bo = res->obj->bo;
+   assert(offset % ZINK_SPARSE_BUFFER_PAGE_SIZE == 0);
+   assert(offset <= bo->base.size);
+   assert(size <= bo->base.size - offset);
+   assert(size % ZINK_SPARSE_BUFFER_PAGE_SIZE == 0 || offset + size == bo->base.size);
+
+   struct zink_sparse_commitment *comm = bo->u.sparse.commitments;
+
+   uint32_t va_page = offset / ZINK_SPARSE_BUFFER_PAGE_SIZE;
+   uint32_t end_va_page = va_page + DIV_ROUND_UP(size, ZINK_SPARSE_BUFFER_PAGE_SIZE);
+
+   simple_mtx_lock(&bo->lock);
+
+   if (commit) {
+      while (va_page < end_va_page) {
+         uint32_t span_va_page;
+
+         /* Skip pages that are already committed. */
+         if (comm[va_page].backing) {
+            va_page++;
+            continue;
+         }
+
+         /* Determine length of uncommitted span. */
+         span_va_page = va_page;
+         while (va_page < end_va_page && !comm[va_page].backing)
+            va_page++;
+
+         /* Fill the uncommitted span with chunks of backing memory. */
+         while (span_va_page < va_page) {
+            struct zink_sparse_backing *backing;
+            uint32_t backing_start, backing_size;
+
+            backing_size = va_page - span_va_page;
+            backing = sparse_backing_alloc(screen, bo, &backing_start, &backing_size);
+            if (!backing) {
+               ok = false;
+               goto out;
+            }
+            if (!do_commit_single(screen, res, backing->bo,
+                                  (uint64_t)span_va_page * ZINK_SPARSE_BUFFER_PAGE_SIZE,
+                                  (uint64_t)backing_size * ZINK_SPARSE_BUFFER_PAGE_SIZE, true)) {
+
+               ok = sparse_backing_free(screen, bo, backing, backing_start, backing_size);
+               assert(ok && "sufficient memory should already be allocated");
+
+               ok = false;
+               goto out;
+            }
+
+            while (backing_size) {
+               comm[span_va_page].backing = backing;
+               comm[span_va_page].page = backing_start;
+               span_va_page++;
+               backing_start++;
+               backing_size--;
+            }
+         }
+      }
+   } else {
+      if (!do_commit_single(screen, res, NULL,
+                            (uint64_t)va_page * ZINK_SPARSE_BUFFER_PAGE_SIZE,
+                            (uint64_t)(end_va_page - va_page) * ZINK_SPARSE_BUFFER_PAGE_SIZE, false)) {
+         ok = false;
+         goto out;
+      }
+
+      while (va_page < end_va_page) {
+         struct zink_sparse_backing *backing;
+         uint32_t backing_start;
+         uint32_t span_pages;
+
+         /* Skip pages that are already uncommitted. */
+         if (!comm[va_page].backing) {
+            va_page++;
+            continue;
+         }
+
+         /* Group contiguous spans of pages. */
+         backing = comm[va_page].backing;
+         backing_start = comm[va_page].page;
+         comm[va_page].backing = NULL;
+
+         span_pages = 1;
+         va_page++;
+
+         while (va_page < end_va_page &&
+                comm[va_page].backing == backing &&
+                comm[va_page].page == backing_start + span_pages) {
+            comm[va_page].backing = NULL;
+            va_page++;
+            span_pages++;
+         }
+
+         if (!sparse_backing_free(screen, bo, backing, backing_start, span_pages)) {
+            /* Couldn't allocate tracking data structures, so we have to leak */
+            fprintf(stderr, "zink: leaking sparse backing memory\n");
+            ok = false;
+         }
+      }
+   }
+out:
+
+   simple_mtx_unlock(&bo->lock);
+   return ok;
+}
+
+static const struct pb_vtbl bo_slab_vtbl = {
+   /* Cast to void* because one of the function parameters is a struct pointer instead of void*. */
+   (void*)bo_slab_destroy
+   /* other functions are never called */
+};
+
+static struct pb_slab *
+bo_slab_alloc(void *priv, unsigned heap, unsigned entry_size, unsigned group_index, bool encrypted)
+{
+   struct zink_screen *screen = priv;
+   VkMemoryPropertyFlags domains = vk_domain_from_heap(heap);
+   uint32_t base_id;
+   unsigned slab_size = 0;
+   struct zink_slab *slab = CALLOC_STRUCT(zink_slab);
+
+   if (!slab)
+      return NULL;
+
+   //struct pb_slabs *slabs = ((flags & RADEON_FLAG_ENCRYPTED) && screen->info.has_tmz_support) ?
+      //screen->bo_slabs_encrypted : screen->bo_slabs;
+   struct pb_slabs *slabs = screen->pb.bo_slabs;
+
+   /* Determine the slab buffer size. */
+   for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
+      unsigned max_entry_size = 1 << (slabs[i].min_order + slabs[i].num_orders - 1);
+
+      if (entry_size <= max_entry_size) {
+         /* The slab size is twice the size of the largest possible entry. */
+         slab_size = max_entry_size * 2;
+
+         if (!util_is_power_of_two_nonzero(entry_size)) {
+            assert(util_is_power_of_two_nonzero(entry_size * 4 / 3));
+
+            /* If the entry size is 3/4 of a power of two, we would waste space and not gain
+             * anything if we allocated only twice the power of two for the backing buffer:
+             *   2 * 3/4 = 1.5 usable with buffer size 2
+             *
+             * Allocating 5 times the entry size leads us to the next power of two and results
+             * in a much better memory utilization:
+             *   5 * 3/4 = 3.75 usable with buffer size 4
+             */
+            if (entry_size * 5 > slab_size)
+               slab_size = util_next_power_of_two(entry_size * 5);
+         }
+
+         break;
+      }
+   }
+   assert(slab_size != 0);
+
+   slab->buffer = zink_bo(zink_bo_create(screen, slab_size, slab_size, heap, 0));
+   if (!slab->buffer)
+      goto fail;
+
+   slab_size = slab->buffer->base.size;
+
+   slab->base.num_entries = slab_size / entry_size;
+   slab->base.num_free = slab->base.num_entries;
+   slab->entry_size = entry_size;
+   slab->entries = CALLOC(slab->base.num_entries, sizeof(*slab->entries));
+   if (!slab->entries)
+      goto fail_buffer;
+
+   list_inithead(&slab->base.free);
+
+#ifdef _MSC_VER
+   /* C11 too hard for msvc, no __sync_fetch_and_add */
+   base_id = p_atomic_add_return(&screen->pb.next_bo_unique_id, slab->base.num_entries) - slab->base.num_entries;
+#else
+   base_id = __sync_fetch_and_add(&screen->pb.next_bo_unique_id, slab->base.num_entries);
+#endif
+   for (unsigned i = 0; i < slab->base.num_entries; ++i) {
+      struct zink_bo *bo = &slab->entries[i];
+
+      simple_mtx_init(&bo->lock, mtx_plain);
+      bo->base.alignment_log2 = util_logbase2(get_slab_entry_alignment(screen, entry_size));
+      bo->base.size = entry_size;
+      bo->base.vtbl = &bo_slab_vtbl;
+      bo->offset = slab->buffer->offset + i * entry_size;
+      bo->base.placement = domains;
+      bo->unique_id = base_id + i;
+      bo->u.slab.entry.slab = &slab->base;
+      bo->u.slab.entry.group_index = group_index;
+      bo->u.slab.entry.entry_size = entry_size;
+
+      if (slab->buffer->mem) {
+         /* The slab is not suballocated. */
+         bo->u.slab.real = slab->buffer;
+      } else {
+         /* The slab is allocated out of a bigger slab. */
+         bo->u.slab.real = slab->buffer->u.slab.real;
+         assert(bo->u.slab.real->mem);
+      }
+
+      list_addtail(&bo->u.slab.entry.head, &slab->base.free);
+   }
+
+   /* Wasted alignment due to slabs with 3/4 allocations being aligned to a power of two. */
+   assert(slab->base.num_entries * entry_size <= slab_size);
+
+   return &slab->base;
+
+fail_buffer:
+   zink_bo_unref(screen, slab->buffer);
+fail:
+   FREE(slab);
+   return NULL;
+}
+
+static struct pb_slab *
+bo_slab_alloc_normal(void *priv, unsigned heap, unsigned entry_size, unsigned group_index)
+{
+   return bo_slab_alloc(priv, heap, entry_size, group_index, false);
+}
+
+bool
+zink_bo_init(struct zink_screen *screen)
+{
+   uint64_t total_mem = 0;
+   for (uint32_t i = 0; i < screen->info.mem_props.memoryHeapCount; ++i)
+      total_mem += screen->info.mem_props.memoryHeaps[i].size;
+   /* Create managers. */
+   pb_cache_init(&screen->pb.bo_cache, ZINK_HEAP_MAX,
+                 500000, 2.0f, 0,
+                 total_mem / 8, screen,
+                 (void*)bo_destroy, (void*)bo_can_reclaim);
+
+   unsigned min_slab_order = 8;  /* 256 bytes */
+   unsigned max_slab_order = 20; /* 1 MB (slab size = 2 MB) */
+   unsigned num_slab_orders_per_allocator = (max_slab_order - min_slab_order) /
+                                            NUM_SLAB_ALLOCATORS;
+
+   /* Divide the size order range among slab managers. */
+   for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
+      unsigned min_order = min_slab_order;
+      unsigned max_order = MIN2(min_order + num_slab_orders_per_allocator,
+                                max_slab_order);
+
+      if (!pb_slabs_init(&screen->pb.bo_slabs[i],
+                         min_order, max_order,
+                         ZINK_HEAP_MAX, true,
+                         screen,
+                         bo_can_reclaim_slab,
+                         bo_slab_alloc_normal,
+                         (void*)bo_slab_free)) {
+         return false;
+      }
+      min_slab_order = max_order + 1;
+   }
+   screen->pb.min_alloc_size = 1 << screen->pb.bo_slabs[0].min_order;
+   screen->pb.bo_export_table = util_hash_table_create_ptr_keys();
+   simple_mtx_init(&screen->pb.bo_export_table_lock, mtx_plain);
+   return true;
+}
+
+void
+zink_bo_deinit(struct zink_screen *screen)
+{
+   for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
+      if (screen->pb.bo_slabs[i].groups)
+         pb_slabs_deinit(&screen->pb.bo_slabs[i]);
+   }
+   pb_cache_deinit(&screen->pb.bo_cache);
+   _mesa_hash_table_destroy(screen->pb.bo_export_table, NULL);
+   simple_mtx_destroy(&screen->pb.bo_export_table_lock);
+}
diff --git a/src/gallium/drivers/zink/zink_bo.h b/src/gallium/drivers/zink/zink_bo.h
new file mode 100644
index 00000000000..f189a89ec41
--- /dev/null
+++ b/src/gallium/drivers/zink/zink_bo.h
@@ -0,0 +1,270 @@
+/*
+ * Copyright © 2021 Valve Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ * 
+ * Authors:
+ *    Mike Blumenkrantz <michael.blumenkrantz@gmail.com>
+ */
+
+#ifndef ZINK_BO_H
+#define ZINK_BO_H
+#include <vulkan/vulkan.h>
+#include "pipebuffer/pb_cache.h"
+#include "pipebuffer/pb_slab.h"
+#include "zink_batch.h"
+
+#define VK_VIS_VRAM (VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)
+enum zink_resource_access {
+   ZINK_RESOURCE_ACCESS_READ = 1,
+   ZINK_RESOURCE_ACCESS_WRITE = 32,
+   ZINK_RESOURCE_ACCESS_RW = ZINK_RESOURCE_ACCESS_READ | ZINK_RESOURCE_ACCESS_WRITE,
+};
+
+
+enum zink_heap {
+   ZINK_HEAP_DEVICE_LOCAL,
+   ZINK_HEAP_DEVICE_LOCAL_SPARSE,
+   ZINK_HEAP_DEVICE_LOCAL_VISIBLE,
+   ZINK_HEAP_HOST_VISIBLE_ANY,
+   ZINK_HEAP_HOST_VISIBLE_COHERENT,
+   ZINK_HEAP_HOST_VISIBLE_CACHED,
+   ZINK_HEAP_MAX,
+};
+
+enum zink_alloc_flag {
+   ZINK_ALLOC_SPARSE = 1<<0,
+   ZINK_ALLOC_NO_SUBALLOC = 1<<1,
+};
+
+
+struct zink_bo {
+   struct pb_buffer base;
+
+   union {
+      struct {
+         void *cpu_ptr; /* for user_ptr and permanent maps */
+         int map_count;
+
+         bool is_user_ptr;
+         bool use_reusable_pool;
+
+         /* Whether buffer_get_handle or buffer_from_handle has been called,
+          * it can only transition from false to true. Protected by lock.
+          */
+         bool is_shared;
+      } real;
+      struct {
+         struct pb_slab_entry entry;
+         struct zink_bo *real;
+      } slab;
+      struct {
+         uint32_t num_va_pages;
+         uint32_t num_backing_pages;
+
+         struct list_head backing;
+
+         /* Commitment information for each page of the virtual memory area. */
+         struct zink_sparse_commitment *commitments;
+      } sparse;
+   } u;
+
+   VkDeviceMemory mem;
+   uint64_t offset;
+
+   uint32_t unique_id;
+
+   simple_mtx_t lock;
+
+   struct zink_batch_usage *reads;
+   struct zink_batch_usage *writes;
+
+   struct pb_cache_entry cache_entry[];
+};
+
+static inline struct zink_bo *
+zink_bo(struct pb_buffer *pbuf)
+{
+   return (struct zink_bo*)pbuf;
+}
+
+static inline enum zink_alloc_flag
+zink_alloc_flags_from_heap(enum zink_heap heap)
+{
+   enum zink_alloc_flag flags = 0;
+   switch (heap) {
+   case ZINK_HEAP_DEVICE_LOCAL_SPARSE:
+      flags |= ZINK_ALLOC_SPARSE;
+      break;
+   default:
+      break;
+   }
+   return flags;
+}
+
+static inline VkMemoryPropertyFlags
+vk_domain_from_heap(enum zink_heap heap)
+{
+   VkMemoryPropertyFlags domains = 0;
+
+   switch (heap) {
+   case ZINK_HEAP_DEVICE_LOCAL:
+   case ZINK_HEAP_DEVICE_LOCAL_SPARSE:
+      domains = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
+      break;
+   case ZINK_HEAP_DEVICE_LOCAL_VISIBLE:
+      domains = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
+      break;
+   case ZINK_HEAP_HOST_VISIBLE_ANY:
+      domains = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
+      break;
+   case ZINK_HEAP_HOST_VISIBLE_COHERENT:
+      domains = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
+      break;
+   case ZINK_HEAP_HOST_VISIBLE_CACHED:
+      domains = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
+      break;
+   default:
+      break;
+   }
+   return domains;
+}
+
+static inline enum zink_heap
+zink_heap_from_domain_flags(VkMemoryPropertyFlags domains, enum zink_alloc_flag flags)
+{
+   if (flags & ZINK_ALLOC_SPARSE)
+      return ZINK_HEAP_DEVICE_LOCAL_SPARSE;
+
+   if ((domains & VK_VIS_VRAM) == VK_VIS_VRAM)
+      return ZINK_HEAP_DEVICE_LOCAL_VISIBLE;
+
+   if (domains & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)
+      return ZINK_HEAP_DEVICE_LOCAL;
+
+   if (domains & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)
+      return ZINK_HEAP_HOST_VISIBLE_COHERENT;
+
+   if (domains & VK_MEMORY_PROPERTY_HOST_CACHED_BIT)
+      return ZINK_HEAP_HOST_VISIBLE_CACHED;
+
+   return ZINK_HEAP_HOST_VISIBLE_ANY;
+}
+
+bool
+zink_bo_init(struct zink_screen *screen);
+
+void
+zink_bo_deinit(struct zink_screen *screen);
+
+struct pb_buffer *
+zink_bo_create(struct zink_screen *screen, uint64_t size, unsigned alignment, enum zink_heap heap, enum zink_alloc_flag flags);
+
+static inline uint64_t
+zink_bo_get_offset(const struct zink_bo *bo)
+{
+   return bo->offset;
+}
+
+static inline VkDeviceMemory
+zink_bo_get_mem(const struct zink_bo *bo)
+{
+   return bo->mem ? bo->mem : bo->u.slab.real->mem;
+}
+
+static inline VkDeviceSize
+zink_bo_get_size(const struct zink_bo *bo)
+{
+   return bo->mem ? bo->base.size : bo->u.slab.real->base.size;
+}
+
+void *
+zink_bo_map(struct zink_screen *screen, struct zink_bo *bo);
+void
+zink_bo_unmap(struct zink_screen *screen, struct zink_bo *bo);
+
+bool
+zink_bo_commit(struct zink_screen *screen, struct zink_resource *res, uint32_t offset, uint32_t size, bool commit);
+
+static inline bool
+zink_bo_has_unflushed_usage(const struct zink_bo *bo)
+{
+   return zink_batch_usage_is_unflushed(bo->reads) ||
+          zink_batch_usage_is_unflushed(bo->writes);
+}
+
+static inline bool
+zink_bo_has_usage(const struct zink_bo *bo)
+{
+   return zink_batch_usage_exists(bo->reads) ||
+          zink_batch_usage_exists(bo->writes);
+}
+
+static inline bool
+zink_bo_usage_matches(const struct zink_bo *bo, const struct zink_batch_state *bs)
+{
+   return zink_batch_usage_matches(bo->reads, bs) ||
+          zink_batch_usage_matches(bo->writes, bs);
+}
+
+static inline bool
+zink_bo_usage_check_completion(struct zink_screen *screen, struct zink_bo *bo, enum zink_resource_access access)
+{
+   if (access & ZINK_RESOURCE_ACCESS_READ && !zink_screen_usage_check_completion(screen, bo->reads))
+      return false;
+   if (access & ZINK_RESOURCE_ACCESS_WRITE && !zink_screen_usage_check_completion(screen, bo->writes))
+      return false;
+   return true;
+}
+
+static inline void
+zink_bo_usage_wait(struct zink_context *ctx, struct zink_bo *bo, enum zink_resource_access access)
+{
+   if (access & ZINK_RESOURCE_ACCESS_READ)
+      zink_batch_usage_wait(ctx, bo->reads);
+   if (access & ZINK_RESOURCE_ACCESS_WRITE)
+      zink_batch_usage_wait(ctx, bo->writes);
+}
+
+static inline void
+zink_bo_usage_set(struct zink_bo *bo, struct zink_batch_state *bs, bool write)
+{
+   if (write)
+      zink_batch_usage_set(&bo->writes, bs);
+   else
+      zink_batch_usage_set(&bo->reads, bs);
+}
+
+static inline void
+zink_bo_usage_unset(struct zink_bo *bo, struct zink_batch_state *bs)
+{
+   zink_batch_usage_unset(&bo->reads, bs);
+   zink_batch_usage_unset(&bo->writes, bs);
+}
+
+
+static inline void
+zink_bo_unref(struct zink_screen *screen, struct zink_bo *bo)
+{
+   struct pb_buffer *pbuf = &bo->base;
+   pb_reference_with_winsys(screen, &pbuf, NULL);
+}
+
+#endif
diff --git a/src/gallium/drivers/zink/zink_context.c b/src/gallium/drivers/zink/zink_context.c
index 4e5734787d1..b279c3563a1 100644
--- a/src/gallium/drivers/zink/zink_context.c
+++ b/src/gallium/drivers/zink/zink_context.c
@@ -3201,29 +3201,6 @@ rebind_buffer(struct zink_context *ctx, struct zink_resource *res)
    zink_batch_resource_usage_set(&ctx->batch, res, has_write);
 }
 
-static inline struct zink_screen **
-get_screen_ptr_for_commit(uint8_t *mem)
-{
-   return (struct zink_screen**)(mem + sizeof(VkBindSparseInfo) + sizeof(VkSparseBufferMemoryBindInfo) + sizeof(VkSparseMemoryBind));
-}
-
-static bool
-resource_commit(struct zink_screen *screen, VkBindSparseInfo *sparse)
-{
-   VkQueue queue = screen->threaded ? screen->thread_queue : screen->queue;
-
-   VkResult ret = vkQueueBindSparse(queue, 1, sparse, VK_NULL_HANDLE);
-   return zink_screen_handle_vkresult(screen, ret);
-}
-
-static void
-submit_resource_commit(void *data, void *gdata, int thread_index)
-{
-   struct zink_screen **screen = get_screen_ptr_for_commit(data);
-   resource_commit(*screen, data);
-   free(data);
-}
-
 static bool
 zink_resource_commit(struct pipe_context *pctx, struct pipe_resource *pres, unsigned level, struct pipe_box *box, bool commit)
 {
@@ -3235,50 +3212,11 @@ zink_resource_commit(struct pipe_context *pctx, struct pipe_resource *pres, unsi
    if (zink_resource_has_unflushed_usage(res))
       zink_flush_queue(ctx);
 
-   uint8_t *mem = malloc(sizeof(VkBindSparseInfo) + sizeof(VkSparseBufferMemoryBindInfo) + sizeof(VkSparseMemoryBind) + sizeof(void*));
-   if (!mem)
-      return false;
-   VkBindSparseInfo *sparse = (void*)mem;
-   sparse->sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO;
-   sparse->pNext = NULL;
-   sparse->waitSemaphoreCount = 0;
-   sparse->bufferBindCount = 1;
-   sparse->imageOpaqueBindCount = 0;
-   sparse->imageBindCount = 0;
-   sparse->signalSemaphoreCount = 0;
-
-   VkSparseBufferMemoryBindInfo *sparse_bind = (void*)(mem + sizeof(VkBindSparseInfo));
-   sparse_bind->buffer = res->obj->buffer;
-   sparse_bind->bindCount = 1;
-   sparse->pBufferBinds = sparse_bind;
-
-   VkSparseMemoryBind *mem_bind = (void*)(mem + sizeof(VkBindSparseInfo) + sizeof(VkSparseBufferMemoryBindInfo));
-   mem_bind->resourceOffset = box->x;
-   mem_bind->size = box->width;
-   mem_bind->memory = commit ? res->obj->mem : VK_NULL_HANDLE;
-   /* currently sparse buffers allocate memory 1:1 for the max sparse size,
-    * but probably it should dynamically allocate the committed regions;
-    * if this ever changes, update the below line
-    */
-   mem_bind->memoryOffset = box->x;
-   mem_bind->flags = 0;
-   sparse_bind->pBinds = mem_bind;
-
-   struct zink_screen **ptr = get_screen_ptr_for_commit(mem);
-   *ptr = screen;
+   bool ret = zink_bo_commit(screen, res, box->x, box->width, commit);
+   if (!ret)
+      check_device_lost(ctx);
 
-   if (screen->threaded) {
-      /* this doesn't need any kind of fencing because any access to this resource
-       * will be automagically synchronized by queue dispatch */
-      util_queue_add_job(&screen->flush_queue, mem, NULL, submit_resource_commit, NULL, 0);
-   } else {
-      bool ret = resource_commit(screen, sparse);
-      if (!ret)
-         check_device_lost(ctx);
-      free(sparse);
-      return ret;
-   }
-   return true;
+   return ret;
 }
 
 static void
diff --git a/src/gallium/drivers/zink/zink_resource.c b/src/gallium/drivers/zink/zink_resource.c
index 3a706a9fc9c..5080cc13429 100644
--- a/src/gallium/drivers/zink/zink_resource.c
+++ b/src/gallium/drivers/zink/zink_resource.c
@@ -78,49 +78,6 @@ debug_describe_zink_resource_object(char *buf, const struct zink_resource_object
    sprintf(buf, "zink_resource_object");
 }
 
-static uint32_t
-mem_hash(const void *key)
-{
-   const struct mem_key *mkey = key;
-   return _mesa_hash_data(&mkey->key, sizeof(mkey->key));
-}
-
-static bool
-mem_equals(const void *a, const void *b)
-{
-   const struct mem_key *ma = a;
-   const struct mem_key *mb = b;
-   return !memcmp(&ma->key, &mb->key, sizeof(ma->key));
-}
-
-static void
-cache_or_free_mem(struct zink_screen *screen, struct zink_resource_object *obj)
-{
-   if (obj->mkey.key.heap_index != UINT32_MAX) {
-      simple_mtx_lock(&screen->mem[obj->mkey.key.heap_index].mem_cache_mtx);
-      struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&screen->mem[obj->mkey.key.heap_index].resource_mem_cache, obj->mem_hash, &obj->mkey);
-      assert(he);
-      struct util_dynarray *array = he->data;
-      struct mem_key *mkey = (void*)he->key;
-
-      unsigned seen = mkey->seen_count;
-      mkey->seen_count--;
-      if (util_dynarray_num_elements(array, struct mem_cache_entry) < seen) {
-         struct mem_cache_entry mc = { obj->mem, obj->map };
-         screen->mem[obj->mkey.key.heap_index].mem_cache_size += obj->size;
-         if (sizeof(void*) == 4 && obj->map) {
-            vkUnmapMemory(screen->dev, obj->mem);
-            mc.map = NULL;
-         }
-         util_dynarray_append(array, struct mem_cache_entry, mc);
-         simple_mtx_unlock(&screen->mem[obj->mkey.key.heap_index].mem_cache_mtx);
-         return;
-      }
-      simple_mtx_unlock(&screen->mem[obj->mkey.key.heap_index].mem_cache_mtx);
-   }
-   vkFreeMemory(screen->dev, obj->mem, NULL);
-}
-
 void
 zink_destroy_resource_object(struct zink_screen *screen, struct zink_resource_object *obj)
 {
@@ -134,7 +91,10 @@ zink_destroy_resource_object(struct zink_screen *screen, struct zink_resource_ob
 
    util_dynarray_fini(&obj->tmp);
    zink_descriptor_set_refs_clear(&obj->desc_set_refs, obj);
-   cache_or_free_mem(screen, obj);
+   if (obj->dedicated)
+      vkFreeMemory(screen->dev, obj->mem, NULL);
+   else
+      zink_bo_unref(screen, obj->bo);
    FREE(obj);
 }
 
@@ -155,38 +115,6 @@ zink_resource_destroy(struct pipe_screen *pscreen,
    FREE(res);
 }
 
-static uint32_t
-get_memory_type_index(struct zink_screen *screen,
-                      const VkMemoryRequirements *reqs,
-                      VkMemoryPropertyFlags props)
-{
-   int32_t idx = -1;
-   for (uint32_t i = 0u; i < VK_MAX_MEMORY_TYPES; i++) {
-      if (((reqs->memoryTypeBits >> i) & 1) == 1) {
-         if ((screen->info.mem_props.memoryTypes[i].propertyFlags & props) == props) {
-            if (!(props & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) &&
-                screen->info.mem_props.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) {
-               idx = i;
-            } else
-               return i;
-         }
-      }
-   }
-   if (idx >= 0)
-      return idx;
-
-   if (props & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) {
-      /* if no suitable cached memory can be found, fall back
-       * to non-cached memory instead.
-       */
-      return get_memory_type_index(screen, reqs,
-         props & ~VK_MEMORY_PROPERTY_HOST_CACHED_BIT);
-   }
-
-   unreachable("Unsupported memory-type");
-   return 0;
-}
-
 static VkImageAspectFlags
 aspect_from_format(enum pipe_format fmt)
 {
@@ -636,17 +564,31 @@ resource_object_create(struct zink_screen *screen, const struct pipe_resource *t
       flags |= VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
 
    VkMemoryAllocateInfo mai = {0};
+   enum zink_alloc_flag aflags = templ->flags & PIPE_RESOURCE_FLAG_SPARSE ? ZINK_ALLOC_SPARSE : 0;
    mai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
    mai.allocationSize = reqs.size;
-   mai.memoryTypeIndex = get_memory_type_index(screen, &reqs, flags);
+   enum zink_heap heap = zink_heap_from_domain_flags(flags, aflags);
+   mai.memoryTypeIndex = screen->heap_map[heap];
+   if (unlikely(!(reqs.memoryTypeBits & BITFIELD_BIT(mai.memoryTypeIndex)))) {
+      /* not valid based on reqs; demote to more compatible type */
+      switch (heap) {
+      case ZINK_HEAP_DEVICE_LOCAL_VISIBLE:
+         heap = ZINK_HEAP_DEVICE_LOCAL;
+         break;
+      case ZINK_HEAP_HOST_VISIBLE_CACHED:
+         heap = ZINK_HEAP_HOST_VISIBLE_ANY;
+         break;
+      default:
+         break;
+      }
+      mai.memoryTypeIndex = screen->heap_map[heap];
+      assert(reqs.memoryTypeBits & BITFIELD_BIT(mai.memoryTypeIndex));
+   }
 
    VkMemoryType mem_type = screen->info.mem_props.memoryTypes[mai.memoryTypeIndex];
    obj->coherent = mem_type.propertyFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
    if (!(templ->flags & PIPE_RESOURCE_FLAG_SPARSE))
       obj->host_visible = mem_type.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
-   if (templ->target == PIPE_BUFFER && !obj->coherent && obj->host_visible) {
-      mai.allocationSize = reqs.size = align(reqs.size, screen->info.props.limits.nonCoherentAtomSize);
-   }
 
    VkMemoryDedicatedAllocateInfo ded_alloc_info = {
       .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
@@ -695,46 +637,33 @@ resource_object_create(struct zink_screen *screen, const struct pipe_resource *t
       mai.pNext = &memory_wsi_info;
    }
 
-   if (!mai.pNext && !(templ->flags & (PIPE_RESOURCE_FLAG_MAP_COHERENT | PIPE_RESOURCE_FLAG_SPARSE))) {
-      obj->mkey.key.reqs = reqs;
-      obj->mkey.key.heap_index = mai.memoryTypeIndex;
-      obj->mem_hash = mem_hash(&obj->mkey);
-      simple_mtx_lock(&screen->mem[mai.memoryTypeIndex].mem_cache_mtx);
-
-      struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&screen->mem[mai.memoryTypeIndex].resource_mem_cache, obj->mem_hash, &obj->mkey);
-      struct mem_key *mkey;
-      if (he) {
-         struct util_dynarray *array = he->data;
-         mkey = (void*)he->key;
-         if (array && util_dynarray_num_elements(array, struct mem_cache_entry)) {
-            struct mem_cache_entry mc = util_dynarray_pop(array, struct mem_cache_entry);
-            obj->mem = mc.mem;
-            obj->map = mc.map;
-            screen->mem[mai.memoryTypeIndex].mem_cache_size -= reqs.size;
-            screen->mem[mai.memoryTypeIndex].mem_cache_count--;
-         }
+   if (!mai.pNext) {
+      unsigned alignment = MAX2(reqs.alignment, 256);
+      if (templ->usage == PIPE_USAGE_STAGING && obj->is_buffer)
+         alignment = MAX2(alignment, screen->info.props.limits.minMemoryMapAlignment);
+      obj->alignment = alignment;
+      obj->bo = zink_bo(zink_bo_create(screen, reqs.size, alignment, heap, 0));
+      if (!obj->bo)
+         goto fail2;
+      if (aflags == ZINK_ALLOC_SPARSE) {
+         obj->size = templ->width0;
       } else {
-         mkey = ralloc(screen, struct mem_key);
-         memcpy(&mkey->key, &obj->mkey.key, sizeof(obj->mkey.key));
-         mkey->seen_count = 0;
-         struct util_dynarray *array = rzalloc(screen, struct util_dynarray);
-         util_dynarray_init(array, screen);
-         _mesa_hash_table_insert_pre_hashed(&screen->mem[mai.memoryTypeIndex].resource_mem_cache, obj->mem_hash, mkey, array);
+         obj->offset = zink_bo_get_offset(obj->bo);
+         obj->mem = zink_bo_get_mem(obj->bo);
+         obj->size = zink_bo_get_size(obj->bo);
       }
-      mkey->seen_count++;
-      simple_mtx_unlock(&screen->mem[mai.memoryTypeIndex].mem_cache_mtx);
-   } else
-      obj->mkey.key.heap_index = UINT32_MAX;
+   } else {
+      obj->dedicated = true;
+      obj->offset = 0;
+      obj->size = reqs.size;
+   }
 
    /* TODO: sparse buffers should probably allocate multiple regions of memory instead of giant blobs? */
-   if (!obj->mem && vkAllocateMemory(screen->dev, &mai, NULL, &obj->mem) != VK_SUCCESS) {
+   if (obj->dedicated && vkAllocateMemory(screen->dev, &mai, NULL, &obj->mem) != VK_SUCCESS) {
       debug_printf("vkAllocateMemory failed\n");
       goto fail2;
    }
 
-   obj->offset = 0;
-   obj->size = reqs.size;
-
    if (templ->target == PIPE_BUFFER) {
       if (!(templ->flags & PIPE_RESOURCE_FLAG_SPARSE))
          if (vkBindBufferMemory(screen->dev, obj->buffer, obj->mem, obj->offset) != VK_SUCCESS)
@@ -1109,6 +1038,8 @@ map_resource(struct zink_screen *screen, struct zink_resource *res)
    if (res->obj->map)
       return res->obj->map;
    assert(res->obj->host_visible);
+   if (!res->obj->dedicated)
+      return zink_bo_map(screen, res->obj->bo);
    result = vkMapMemory(screen->dev, res->obj->mem, res->obj->offset,
                         res->obj->size, 0, &res->obj->map);
    if (zink_screen_handle_vkresult(screen, result))
@@ -1120,7 +1051,10 @@ static void
 unmap_resource(struct zink_screen *screen, struct zink_resource *res)
 {
    res->obj->map = NULL;
-   vkUnmapMemory(screen->dev, res->obj->mem);
+   if (!res->obj->dedicated)
+      zink_bo_unmap(screen, res->obj->bo);
+   else
+      vkUnmapMemory(screen->dev, res->obj->mem);
 }
 
 static void *
@@ -1672,14 +1606,6 @@ zink_screen_resource_init(struct pipe_screen *pscreen)
       pscreen->resource_from_handle = zink_resource_from_handle;
    }
    pscreen->resource_get_param = zink_resource_get_param;
-
-   screen->mem = rzalloc_array(screen, struct zink_mem_cache, screen->info.mem_props.memoryTypeCount);
-   if (!screen->mem)
-      return false;
-   for (uint32_t i = 0; i < screen->info.mem_props.memoryTypeCount; ++i) {
-      simple_mtx_init(&screen->mem[i].mem_cache_mtx, mtx_plain);
-      _mesa_hash_table_init(&screen->mem[i].resource_mem_cache, screen, mem_hash, mem_equals);
-   }
    return true;
 }
 
diff --git a/src/gallium/drivers/zink/zink_resource.h b/src/gallium/drivers/zink/zink_resource.h
index 15533fed611..3887f0a308d 100644
--- a/src/gallium/drivers/zink/zink_resource.h
+++ b/src/gallium/drivers/zink/zink_resource.h
@@ -28,7 +28,7 @@ struct pipe_screen;
 struct sw_displaytarget;
 struct zink_batch;
 struct zink_context;
-
+struct zink_bo;
 #define ZINK_RESOURCE_USAGE_STREAMOUT (1 << 10) //much greater than ZINK_DESCRIPTOR_TYPES
 
 #include "util/simple_mtx.h"
@@ -44,12 +44,6 @@ struct zink_context;
 
 #define ZINK_MAP_TEMPORARY (PIPE_MAP_DRV_PRV << 0)
 
-enum zink_resource_access {
-   ZINK_RESOURCE_ACCESS_READ = 1,
-   ZINK_RESOURCE_ACCESS_WRITE = 32,
-   ZINK_RESOURCE_ACCESS_RW = ZINK_RESOURCE_ACCESS_READ | ZINK_RESOURCE_ACCESS_WRITE,
-};
-
 struct mem_key {
    unsigned seen_count;
    struct {
@@ -70,9 +64,9 @@ struct zink_resource_object {
    bool transfer_dst;
    VkImageAspectFlags modifier_aspect;
 
+   bool dedicated;
+   struct zink_bo *bo;
    VkDeviceMemory mem;
-   uint32_t mem_hash;
-   struct mem_key mkey;
    VkDeviceSize offset, size, alignment;
 
    VkSampleLocationsInfoEXT zs_evaluate;
@@ -187,75 +181,98 @@ bool
 zink_resource_object_init_storage(struct zink_context *ctx, struct zink_resource *res);
 
 #ifndef __cplusplus
+#include "zink_bo.h"
 
 static inline bool
 zink_resource_usage_is_unflushed(const struct zink_resource *res)
 {
-   return zink_batch_usage_is_unflushed(res->obj->reads) ||
-          zink_batch_usage_is_unflushed(res->obj->writes);
+   if (res->obj->dedicated)
+      return zink_batch_usage_is_unflushed(res->obj->reads) ||
+             zink_batch_usage_is_unflushed(res->obj->writes);
+   return zink_bo_has_unflushed_usage(res->obj->bo);
 }
 
 static inline bool
 zink_resource_usage_is_unflushed_write(const struct zink_resource *res)
 {
-   return zink_batch_usage_is_unflushed(res->obj->writes);
+   if (res->obj->dedicated)
+      return zink_batch_usage_is_unflushed(res->obj->writes);
+   return zink_batch_usage_is_unflushed(res->obj->bo->writes);
 }
 
 
 static inline bool
 zink_resource_usage_matches(const struct zink_resource *res, const struct zink_batch_state *bs)
 {
-   return zink_batch_usage_matches(res->obj->reads, bs) ||
-          zink_batch_usage_matches(res->obj->writes, bs);
+   if (res->obj->dedicated)
+      return zink_batch_usage_matches(res->obj->reads, bs) ||
+             zink_batch_usage_matches(res->obj->writes, bs);
+   return zink_bo_usage_matches(res->obj->bo, bs);
 }
 
 static inline bool
 zink_resource_has_usage(const struct zink_resource *res)
 {
-   return zink_batch_usage_exists(res->obj->reads) ||
-          zink_batch_usage_exists(res->obj->writes);
+   if (res->obj->dedicated)
+      return zink_batch_usage_exists(res->obj->reads) ||
+             zink_batch_usage_exists(res->obj->writes);
+   return zink_bo_has_usage(res->obj->bo);
 }
 
 static inline bool
 zink_resource_has_unflushed_usage(const struct zink_resource *res)
 {
-   return zink_batch_usage_is_unflushed(res->obj->reads) ||
-          zink_batch_usage_is_unflushed(res->obj->writes);
+   if (res->obj->dedicated)
+      return zink_batch_usage_is_unflushed(res->obj->reads) ||
+             zink_batch_usage_is_unflushed(res->obj->writes);
+   return zink_bo_has_unflushed_usage(res->obj->bo);
 }
 
 static inline bool
 zink_resource_usage_check_completion(struct zink_screen *screen, struct zink_resource *res, enum zink_resource_access access)
 {
-   if (access & ZINK_RESOURCE_ACCESS_READ && !zink_screen_usage_check_completion(screen, res->obj->reads))
-      return false;
-   if (access & ZINK_RESOURCE_ACCESS_WRITE && !zink_screen_usage_check_completion(screen, res->obj->writes))
-      return false;
-   return true;
+   if (res->obj->dedicated) {
+      if (access & ZINK_RESOURCE_ACCESS_READ && !zink_screen_usage_check_completion(screen, res->obj->reads))
+         return false;
+      if (access & ZINK_RESOURCE_ACCESS_WRITE && !zink_screen_usage_check_completion(screen, res->obj->writes))
+         return false;
+      return true;
+   }
+   return zink_bo_usage_check_completion(screen, res->obj->bo, access);
 }
 
 static inline void
 zink_resource_usage_wait(struct zink_context *ctx, struct zink_resource *res, enum zink_resource_access access)
 {
-   if (access & ZINK_RESOURCE_ACCESS_READ)
-      zink_batch_usage_wait(ctx, res->obj->reads);
-   if (access & ZINK_RESOURCE_ACCESS_WRITE)
-      zink_batch_usage_wait(ctx, res->obj->writes);
+   if (res->obj->dedicated) {
+      if (access & ZINK_RESOURCE_ACCESS_READ)
+         zink_batch_usage_wait(ctx, res->obj->reads);
+      if (access & ZINK_RESOURCE_ACCESS_WRITE)
+         zink_batch_usage_wait(ctx, res->obj->writes);
+   } else
+      zink_bo_usage_wait(ctx, res->obj->bo, access);
 }
 
 static inline void
 zink_resource_usage_set(struct zink_resource *res, struct zink_batch_state *bs, bool write)
 {
-   if (write)
-      zink_batch_usage_set(&res->obj->writes, bs);
-   else
-      zink_batch_usage_set(&res->obj->reads, bs);
+   if (res->obj->dedicated) {
+      if (write)
+         zink_batch_usage_set(&res->obj->writes, bs);
+      else
+         zink_batch_usage_set(&res->obj->reads, bs);
+   } else
+      zink_bo_usage_set(res->obj->bo, bs, write);
 }
 
 static inline void
 zink_resource_object_usage_unset(struct zink_resource_object *obj, struct zink_batch_state *bs)
 {
-   zink_batch_usage_unset(&obj->reads, bs);
-   zink_batch_usage_unset(&obj->writes, bs);
+   if (obj->dedicated) {
+      zink_batch_usage_unset(&obj->reads, bs);
+      zink_batch_usage_unset(&obj->writes, bs);
+   } else
+      zink_bo_usage_unset(obj->bo, bs);
 }
 
 #endif
diff --git a/src/gallium/drivers/zink/zink_screen.c b/src/gallium/drivers/zink/zink_screen.c
index 16a68195cf7..c6434fa5682 100644
--- a/src/gallium/drivers/zink/zink_screen.c
+++ b/src/gallium/drivers/zink/zink_screen.c
@@ -585,8 +585,8 @@ zink_get_param(struct pipe_screen *pscreen, enum pipe_cap param)
       return screen->info.feats.features.shaderCullDistance;
 
    case PIPE_CAP_SPARSE_BUFFER_PAGE_SIZE:
-      /* this is the spec minimum */
-      return screen->info.feats.features.sparseBinding ? 64 * 1024 : 0;
+      
+      return screen->info.feats.features.sparseBinding ? ZINK_SPARSE_BUFFER_PAGE_SIZE : 0;
 
    case PIPE_CAP_VIEWPORT_SUBPIXEL_BITS:
       return screen->info.props.limits.viewportSubPixelBits;
@@ -1038,16 +1038,6 @@ zink_is_format_supported(struct pipe_screen *pscreen,
 }
 
 static void
-resource_cache_entry_destroy(struct zink_screen *screen, struct hash_entry *he)
-{
-   struct util_dynarray *array = (void*)he->data;
-   util_dynarray_foreach(array, struct mem_cache_entry, mc) {
-      vkFreeMemory(screen->dev, mc->mem, NULL);
-   }
-   util_dynarray_fini(array);
-}
-
-static void
 zink_destroy_screen(struct pipe_screen *pscreen)
 {
    struct zink_screen *screen = zink_screen(pscreen);
@@ -1087,15 +1077,7 @@ zink_destroy_screen(struct pipe_screen *pscreen)
    }
 #endif
    disk_cache_destroy(screen->disk_cache);
-
-   for (uint32_t i = 0; i < screen->info.mem_props.memoryHeapCount; ++i) {
-      simple_mtx_lock(&screen->mem[i].mem_cache_mtx);
-      hash_table_foreach(&screen->mem[i].resource_mem_cache, he)
-         resource_cache_entry_destroy(screen, he);
-      simple_mtx_unlock(&screen->mem[i].mem_cache_mtx);
-      simple_mtx_destroy(&screen->mem[i].mem_cache_mtx);
-   }
-
+   zink_bo_deinit(screen);
    util_live_shader_cache_deinit(&screen->shaders);
 
    if (screen->sem)
@@ -1892,6 +1874,7 @@ zink_internal_create_screen(const struct pipe_screen_config *config)
 
    if (!zink_screen_resource_init(&screen->base))
       goto fail;
+   zink_bo_init(screen);
    zink_screen_fence_init(&screen->base);
 
    zink_screen_init_compiler(screen);
@@ -1921,6 +1904,25 @@ zink_internal_create_screen(const struct pipe_screen_config *config)
    if (screen->info.have_KHR_timeline_semaphore)
       zink_screen_init_semaphore(screen);
 
+   memset(&screen->heap_map, UINT8_MAX, sizeof(screen->heap_map));
+   for (enum zink_heap i = 0; i < ZINK_HEAP_MAX; i++) {
+      for (unsigned j = 0; j < screen->info.mem_props.memoryTypeCount; j++) {
+         VkMemoryPropertyFlags domains = vk_domain_from_heap(i);
+         if ((screen->info.mem_props.memoryTypes[j].propertyFlags & domains) == domains) {
+            assert(screen->heap_map[i] == UINT8_MAX);
+            screen->heap_map[i] = j;
+            break;
+         }
+      }
+
+      /* not found: use compatible heap */
+      if (screen->heap_map[i] == UINT8_MAX) {
+         /* only cached mem has a failure case for now */
+         assert(i == ZINK_HEAP_HOST_VISIBLE_CACHED);
+         screen->heap_map[i] = screen->heap_map[ZINK_HEAP_HOST_VISIBLE_ANY];
+      }
+   }
+
    simple_mtx_init(&screen->surface_mtx, mtx_plain);
    simple_mtx_init(&screen->bufferview_mtx, mtx_plain);
    simple_mtx_init(&screen->framebuffer_mtx, mtx_plain);
diff --git a/src/gallium/drivers/zink/zink_screen.h b/src/gallium/drivers/zink/zink_screen.h
index e74c514b336..082ac2d23a6 100644
--- a/src/gallium/drivers/zink/zink_screen.h
+++ b/src/gallium/drivers/zink/zink_screen.h
@@ -37,7 +37,8 @@
 #include "util/simple_mtx.h"
 #include "util/u_queue.h"
 #include "util/u_live_shader_cache.h"
-
+#include "pipebuffer/pb_cache.h"
+#include "pipebuffer/pb_slab.h"
 #include <vulkan/vulkan.h>
 
 extern uint32_t zink_debug;
@@ -50,11 +51,16 @@ struct zink_program;
 struct zink_shader;
 enum zink_descriptor_type;
 
+/* this is the spec minimum */
+#define ZINK_SPARSE_BUFFER_PAGE_SIZE (64 * 1024)
+
 #define ZINK_DEBUG_NIR 0x1
 #define ZINK_DEBUG_SPIRV 0x2
 #define ZINK_DEBUG_TGSI 0x4
 #define ZINK_DEBUG_VALIDATION 0x8
 
+#define NUM_SLAB_ALLOCATORS 3
+
 enum zink_descriptor_mode {
    ZINK_DESCRIPTOR_MODE_AUTO,
    ZINK_DESCRIPTOR_MODE_LAZY,
@@ -66,13 +72,6 @@ struct zink_modifier_prop {
     VkDrmFormatModifierPropertiesEXT*    pDrmFormatModifierProperties;
 };
 
-struct zink_mem_cache {
-   simple_mtx_t mem_cache_mtx;
-   struct hash_table resource_mem_cache;
-   uint64_t mem_cache_size;
-   unsigned mem_cache_count;
-};
-
 struct zink_screen {
    struct pipe_screen base;
    bool threaded;
@@ -99,7 +98,15 @@ struct zink_screen {
 
    struct util_live_shader_cache shaders;
 
-   struct zink_mem_cache *mem;
+   struct {
+      struct pb_cache bo_cache;
+      struct pb_slabs bo_slabs[NUM_SLAB_ALLOCATORS];
+      unsigned min_alloc_size;
+      struct hash_table *bo_export_table;
+      simple_mtx_t bo_export_table_lock;
+      uint32_t next_bo_unique_id;
+   } pb;
+   uint8_t heap_map[VK_MAX_MEMORY_TYPES];
 
    uint64_t total_video_mem;
    uint64_t clamp_video_mem;