zink: add a mechanism to track current resource usage in batches

this is really primitive, but it at least gives an idea of whether a
resource has been submitted for writing in a pending batch

Reviewed-by: Erik Faye-Lun <erik.faye-lund@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6924>

10 files changed, 88 insertions, 31 deletions

diff --git a/src/gallium/drivers/zink/zink_batch.c b/src/gallium/drivers/zink/zink_batch.c
index aad6ef958bb..ee0fa994cdc 100644
--- a/src/gallium/drivers/zink/zink_batch.c
+++ b/src/gallium/drivers/zink/zink_batch.c
@@ -9,6 +9,7 @@
 #include "zink_resource.h"
 #include "zink_screen.h"
 
+#include "util/hash_table.h"
 #include "util/u_debug.h"
 #include "util/set.h"
 
@@ -104,23 +105,32 @@ zink_end_batch(struct zink_context *ctx, struct zink_batch *batch)
 }
 
 void
-zink_batch_reference_resoure(struct zink_batch *batch,
-                             struct zink_resource *res)
+zink_batch_reference_resource_rw(struct zink_batch *batch, struct zink_resource *res, bool write)
 {
+   unsigned mask = write ? ZINK_RESOURCE_ACCESS_WRITE : ZINK_RESOURCE_ACCESS_READ;
+
+   /* u_transfer_helper unrefs the stencil buffer when the depth buffer is unrefed,
+    * so we add an extra ref here to the stencil buffer to compensate
+    */
+   struct zink_resource *stencil;
+
+   zink_get_depth_stencil_resources((struct pipe_resource*)res, NULL, &stencil);
+
+
    struct set_entry *entry = _mesa_set_search(batch->resources, res);
    if (!entry) {
       entry = _mesa_set_add(batch->resources, res);
       pipe_reference(NULL, &res->base.reference);
-
-      /* u_transfer_helper unrefs the stencil buffer when the depth buffer is unrefed,
-       * so we add an extra ref here to the stencil buffer to compensate
-       */
-      struct zink_resource *stencil;
-
-      zink_get_depth_stencil_resources((struct pipe_resource*)res, NULL, &stencil);
       if (stencil)
          pipe_reference(NULL, &stencil->base.reference);
    }
+   /* the batch_uses value for this batch is guaranteed to not be in use now because
+    * reset_batch() waits on the fence and removes access before resetting
+    */
+   res->batch_uses[batch->batch_id] |= mask;
+
+   if (stencil)
+      stencil->batch_uses[batch->batch_id] |= mask;
 }
 
 void
diff --git a/src/gallium/drivers/zink/zink_batch.h b/src/gallium/drivers/zink/zink_batch.h
index 7b5ffe9fc95..9bb3294e871 100644
--- a/src/gallium/drivers/zink/zink_batch.h
+++ b/src/gallium/drivers/zink/zink_batch.h
@@ -40,6 +40,7 @@ struct zink_sampler_view;
 #define ZINK_BATCH_DESC_SIZE 1000
 
 struct zink_batch {
+   unsigned batch_id : 2;
    VkCommandBuffer cmdbuf;
    VkDescriptorPool descpool;
    int descs_left;
@@ -64,8 +65,9 @@ void
 zink_end_batch(struct zink_context *ctx, struct zink_batch *batch);
 
 void
-zink_batch_reference_resoure(struct zink_batch *batch,
-                             struct zink_resource *res);
+zink_batch_reference_resource_rw(struct zink_batch *batch,
+                                 struct zink_resource *res,
+                                 bool write);
 
 void
 zink_batch_reference_sampler_view(struct zink_batch *batch,
diff --git a/src/gallium/drivers/zink/zink_blit.c b/src/gallium/drivers/zink/zink_blit.c
index 891ba599f74..9701f200dd3 100644
--- a/src/gallium/drivers/zink/zink_blit.c
+++ b/src/gallium/drivers/zink/zink_blit.c
@@ -28,8 +28,8 @@ blit_resolve(struct zink_context *ctx, const struct pipe_blit_info *info)
 
    struct zink_batch *batch = zink_batch_no_rp(ctx);
 
-   zink_batch_reference_resoure(batch, src);
-   zink_batch_reference_resoure(batch, dst);
+   zink_batch_reference_resource_rw(batch, src, false);
+   zink_batch_reference_resource_rw(batch, dst, true);
 
    zink_resource_setup_transfer_layouts(batch, src, dst);
 
@@ -88,8 +88,8 @@ blit_native(struct zink_context *ctx, const struct pipe_blit_info *info)
       return false;
 
    struct zink_batch *batch = zink_batch_no_rp(ctx);
-   zink_batch_reference_resoure(batch, src);
-   zink_batch_reference_resoure(batch, dst);
+   zink_batch_reference_resource_rw(batch, src, false);
+   zink_batch_reference_resource_rw(batch, dst, true);
 
    zink_resource_setup_transfer_layouts(batch, src, dst);
 
diff --git a/src/gallium/drivers/zink/zink_context.c b/src/gallium/drivers/zink/zink_context.c
index 669fb52ec02..6043ff461ee 100644
--- a/src/gallium/drivers/zink/zink_context.c
+++ b/src/gallium/drivers/zink/zink_context.c
@@ -971,8 +971,8 @@ zink_resource_copy_region(struct pipe_context *pctx,
       region.extent.height = src_box->height;
 
       struct zink_batch *batch = zink_batch_no_rp(ctx);
-      zink_batch_reference_resoure(batch, src);
-      zink_batch_reference_resoure(batch, dst);
+      zink_batch_reference_resource_rw(batch, src, false);
+      zink_batch_reference_resource_rw(batch, dst, true);
 
       zink_resource_setup_transfer_layouts(batch, src, dst);
       vkCmdCopyImage(batch->cmdbuf, src->image, src->layout,
@@ -986,8 +986,8 @@ zink_resource_copy_region(struct pipe_context *pctx,
       region.size = src_box->width;
 
       struct zink_batch *batch = zink_batch_no_rp(ctx);
-      zink_batch_reference_resoure(batch, src);
-      zink_batch_reference_resoure(batch, dst);
+      zink_batch_reference_resource_rw(batch, src, false);
+      zink_batch_reference_resource_rw(batch, dst, true);
 
       vkCmdCopyBuffer(batch->cmdbuf, src->buffer, dst->buffer, 1, &region);
    } else
@@ -1180,6 +1180,8 @@ zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags)
       if (vkCreateDescriptorPool(screen->dev, &dpci, 0,
                                  &ctx->batches[i].descpool) != VK_SUCCESS)
          goto fail;
+
+      ctx->batches[i].batch_id = i;
    }
 
    vkGetDeviceQueue(screen->dev, screen->gfx_queue, 0, &ctx->queue);
diff --git a/src/gallium/drivers/zink/zink_draw.c b/src/gallium/drivers/zink/zink_draw.c
index 64e0e7058e5..5877ba0ed02 100644
--- a/src/gallium/drivers/zink/zink_draw.c
+++ b/src/gallium/drivers/zink/zink_draw.c
@@ -95,7 +95,7 @@ zink_emit_xfb_vertex_input_barrier(struct zink_context *ctx, struct zink_resourc
    barriers[0].buffer = res->buffer;
    barriers[0].size = VK_WHOLE_SIZE;
    struct zink_batch *batch = zink_batch_no_rp(ctx);
-   zink_batch_reference_resoure(batch, res);
+   zink_batch_reference_resource_rw(batch, res, false);
    vkCmdPipelineBarrier(batch->cmdbuf,
       VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT,
       VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
@@ -120,7 +120,7 @@ zink_emit_stream_output_targets(struct pipe_context *pctx)
    for (unsigned i = 0; i < ctx->num_so_targets; i++) {
       struct zink_so_target *t = (struct zink_so_target *)ctx->so_targets[i];
       buffers[i] = zink_resource(t->base.buffer)->buffer;
-      zink_batch_reference_resoure(batch, zink_resource(t->base.buffer));
+      zink_batch_reference_resource_rw(batch, zink_resource(t->base.buffer), true);
       buffer_offsets[i] = t->base.buffer_offset;
       buffer_sizes[i] = t->base.buffer_size;
    }
@@ -144,7 +144,7 @@ zink_bind_vertex_buffers(struct zink_batch *batch, struct zink_context *ctx)
          struct zink_resource *res = zink_resource(vb->buffer.resource);
          buffers[i] = res->buffer;
          buffer_offsets[i] = vb->buffer_offset;
-         zink_batch_reference_resoure(batch, res);
+         zink_batch_reference_resource_rw(batch, res, false);
       } else {
          buffers[i] = zink_resource(ctx->dummy_buffer)->buffer;
          buffer_offsets[i] = 0;
@@ -386,7 +386,7 @@ zink_draw_vbo(struct pipe_context *pctx,
          int index = shader->bindings[j].index;
          if (shader->bindings[j].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER) {
             struct zink_resource *res = zink_resource(ctx->ubos[i][index].buffer);
-            zink_batch_reference_resoure(batch, res);
+            zink_batch_reference_resource_rw(batch, res, false);
          } else {
             struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->image_views[i][index]);
             zink_batch_reference_sampler_view(batch, sampler_view);
@@ -450,7 +450,7 @@ zink_draw_vbo(struct pipe_context *pctx,
          struct zink_so_target *t = zink_so_target(ctx->so_targets[i]);
          struct zink_resource *res = zink_resource(t->counter_buffer);
          if (t->counter_buffer_valid) {
-            zink_batch_reference_resoure(batch, zink_resource(t->counter_buffer));
+            zink_batch_reference_resource_rw(batch, res, true);
             counter_buffers[i] = res->buffer;
             counter_buffer_offsets[i] = t->counter_buffer_offset;
          } else
@@ -481,13 +481,13 @@ zink_draw_vbo(struct pipe_context *pctx,
       }
       struct zink_resource *res = zink_resource(index_buffer);
       vkCmdBindIndexBuffer(batch->cmdbuf, res->buffer, index_offset, index_type);
-      zink_batch_reference_resoure(batch, res);
+      zink_batch_reference_resource_rw(batch, res, false);
       vkCmdDrawIndexed(batch->cmdbuf,
          dinfo->count, dinfo->instance_count,
          need_index_buffer_unref ? 0 : dinfo->start, dinfo->index_bias, dinfo->start_instance);
    } else {
       if (so_target && screen->info.tf_props.transformFeedbackDraw) {
-         zink_batch_reference_resoure(batch, zink_resource(so_target->counter_buffer));
+         zink_batch_reference_resource_rw(batch, zink_resource(so_target->counter_buffer), true);
          screen->vk_CmdDrawIndirectByteCountEXT(batch->cmdbuf, dinfo->instance_count, dinfo->start_instance,
                                        zink_resource(so_target->counter_buffer)->buffer, so_target->counter_buffer_offset, 0,
                                        MIN2(so_target->stride, screen->info.tf_props.maxTransformFeedbackBufferDataStride));
diff --git a/src/gallium/drivers/zink/zink_fence.c b/src/gallium/drivers/zink/zink_fence.c
index 86679f2caf7..e6f07045cae 100644
--- a/src/gallium/drivers/zink/zink_fence.c
+++ b/src/gallium/drivers/zink/zink_fence.c
@@ -25,8 +25,10 @@
 #include "zink_fence.h"
 
 #include "zink_query.h"
+#include "zink_resource.h"
 #include "zink_screen.h"
 
+#include "util/set.h"
 #include "util/u_memory.h"
 
 static void
@@ -34,6 +36,7 @@ destroy_fence(struct zink_screen *screen, struct zink_fence *fence)
 {
    if (fence->fence)
       vkDestroyFence(screen->dev, fence->fence, NULL);
+   util_dynarray_fini(&fence->resources);
    FREE(fence);
 }
 
@@ -58,6 +61,17 @@ zink_create_fence(struct pipe_screen *pscreen, struct zink_batch *batch)
    ret->active_queries = batch->active_queries;
    batch->active_queries = NULL;
 
+   ret->batch_id = batch->batch_id;
+   util_dynarray_init(&ret->resources, NULL);
+   set_foreach(batch->resources, entry) {
+      /* the fence needs its own reference to ensure it can safely access lifetime-dependent
+       * resource members
+       */
+      struct pipe_resource *r = NULL, *pres = (struct pipe_resource *)entry->key;
+      pipe_resource_reference(&r, pres);
+      util_dynarray_append(&ret->resources, struct pipe_resource*, pres);
+   }
+
    pipe_reference_init(&ret->reference, 1);
    return ret;
 
@@ -86,14 +100,35 @@ fence_reference(struct pipe_screen *pscreen,
                         zink_fence(pfence));
 }
 
+static inline void
+fence_remove_resource_access(struct zink_fence *fence, struct zink_resource *res)
+{
+   p_atomic_set(&res->batch_uses[fence->batch_id], 0);
+}
+
 bool
 zink_fence_finish(struct zink_screen *screen, struct zink_fence *fence,
                   uint64_t timeout_ns)
 {
    bool success = vkWaitForFences(screen->dev, 1, &fence->fence, VK_TRUE,
                                   timeout_ns) == VK_SUCCESS;
-   if (success && fence->active_queries)
-      zink_prune_queries(screen, fence);
+   if (success) {
+      if (fence->active_queries)
+         zink_prune_queries(screen, fence);
+
+      /* unref all used resources */
+      util_dynarray_foreach(&fence->resources, struct pipe_resource*, pres) {
+         struct zink_resource *stencil, *res = zink_resource(*pres);
+         fence_remove_resource_access(fence, res);
+
+         /* we still hold a ref, so this doesn't need to be atomic */
+         zink_get_depth_stencil_resources((struct pipe_resource*)res, NULL, &stencil);
+         if (stencil)
+            fence_remove_resource_access(fence, stencil);
+         pipe_resource_reference(pres, NULL);
+      }
+      util_dynarray_clear(&fence->resources);
+   }
    return success;
 }
 
diff --git a/src/gallium/drivers/zink/zink_fence.h b/src/gallium/drivers/zink/zink_fence.h
index dab6f5ece8d..06ef49947e6 100644
--- a/src/gallium/drivers/zink/zink_fence.h
+++ b/src/gallium/drivers/zink/zink_fence.h
@@ -25,6 +25,7 @@
 #define ZINK_FENCE_H
 
 #include "util/u_inlines.h"
+#include "util/u_dynarray.h"
 
 #include <vulkan/vulkan.h>
 
@@ -33,8 +34,10 @@ struct zink_screen;
 
 struct zink_fence {
    struct pipe_reference reference;
+   unsigned batch_id : 2;
    VkFence fence;
    struct set *active_queries; /* zink_query objects which were active at some point in this batch */
+   struct util_dynarray resources;
 };
 
 static inline struct zink_fence *
diff --git a/src/gallium/drivers/zink/zink_query.c b/src/gallium/drivers/zink/zink_query.c
index 0d1e8117cbc..7d913815813 100644
--- a/src/gallium/drivers/zink/zink_query.c
+++ b/src/gallium/drivers/zink/zink_query.c
@@ -472,7 +472,7 @@ zink_render_condition(struct pipe_context *pctx,
    begin_info.flags = begin_flags;
    screen->vk_CmdBeginConditionalRenderingEXT(batch->cmdbuf, &begin_info);
 
-   zink_batch_reference_resoure(batch, res);
+   zink_batch_reference_resource_rw(batch, res, true);
 
    pipe_resource_reference(&pres, NULL);
 }
diff --git a/src/gallium/drivers/zink/zink_resource.c b/src/gallium/drivers/zink/zink_resource.c
index 561afeaf590..6e41fdd8be4 100644
--- a/src/gallium/drivers/zink/zink_resource.c
+++ b/src/gallium/drivers/zink/zink_resource.c
@@ -388,8 +388,8 @@ zink_transfer_copy_bufimage(struct zink_context *ctx,
    copyRegion.imageExtent.width = trans->base.box.width;
    copyRegion.imageExtent.height = trans->base.box.height;
 
-   zink_batch_reference_resoure(batch, res);
-   zink_batch_reference_resoure(batch, staging_res);
+   zink_batch_reference_resource_rw(batch, res, buf2img);
+   zink_batch_reference_resource_rw(batch, staging_res, !buf2img);
 
    /* we're using u_transfer_helper_deinterleave, which means we'll be getting PIPE_MAP_* usage
     * to indicate whether to copy either the depth or stencil aspects
diff --git a/src/gallium/drivers/zink/zink_resource.h b/src/gallium/drivers/zink/zink_resource.h
index 98ae7be9318..04e8d8c7395 100644
--- a/src/gallium/drivers/zink/zink_resource.h
+++ b/src/gallium/drivers/zink/zink_resource.h
@@ -32,6 +32,9 @@ struct zink_batch;
 
 #include <vulkan/vulkan.h>
 
+#define ZINK_RESOURCE_ACCESS_READ 1
+#define ZINK_RESOURCE_ACCESS_WRITE 16
+
 struct zink_resource {
    struct pipe_resource base;
 
@@ -53,6 +56,8 @@ struct zink_resource {
    struct sw_displaytarget *dt;
    unsigned dt_stride;
 
+   /* this has to be atomic for fence access, so we can't use a bitmask and make everything neat */
+   uint8_t batch_uses[4];
    bool needs_xfb_barrier;
 };