radeonsi: Add displayable DCC flushing without explicit flushes.

Flushes non-explicit shared textures that need retiling on

* glFlush
* glSync
* glSignalSemaphoreEXT
* DRI fences.
* The first time we create a non-explicit handle for it.

Reviewed-by: Marek Olšák <marek.olsak@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6176>

6 files changed, 61 insertions, 10 deletions

diff --git a/src/gallium/drivers/radeonsi/si_blit.c b/src/gallium/drivers/radeonsi/si_blit.c
index 71037726863..96c4d8b9733 100644
--- a/src/gallium/drivers/radeonsi/si_blit.c
+++ b/src/gallium/drivers/radeonsi/si_blit.c
@@ -1258,6 +1258,15 @@ static void si_flush_resource(struct pipe_context *ctx, struct pipe_resource *re
    }
 }
 
+void si_flush_implicit_resources(struct si_context *sctx)
+{
+   hash_table_foreach(sctx->dirty_implicit_resources, entry) {
+      si_flush_resource(&sctx->b, entry->data);
+      pipe_resource_reference((struct pipe_resource **)&entry->data, NULL);
+   }
+   _mesa_hash_table_clear(sctx->dirty_implicit_resources, NULL);
+}
+
 void si_decompress_dcc(struct si_context *sctx, struct si_texture *tex)
 {
    /* If graphics is disabled, we can't decompress DCC, but it shouldn't
diff --git a/src/gallium/drivers/radeonsi/si_fence.c b/src/gallium/drivers/radeonsi/si_fence.c
index 419a26d969f..24cc7ad04f9 100644
--- a/src/gallium/drivers/radeonsi/si_fence.c
+++ b/src/gallium/drivers/radeonsi/si_fence.c
@@ -458,8 +458,9 @@ static int si_fence_get_fd(struct pipe_screen *screen, struct pipe_fence_handle
    return gfx_fd;
 }
 
-static void si_flush_from_st(struct pipe_context *ctx, struct pipe_fence_handle **fence,
-                             unsigned flags)
+static void si_flush_all_queues(struct pipe_context *ctx,
+                                struct pipe_fence_handle **fence,
+                                unsigned flags, bool force_flush)
 {
    struct pipe_screen *screen = ctx->screen;
    struct si_context *sctx = (struct si_context *)ctx;
@@ -470,6 +471,10 @@ static void si_flush_from_st(struct pipe_context *ctx, struct pipe_fence_handle
    struct si_fine_fence fine = {};
    unsigned rflags = PIPE_FLUSH_ASYNC;
 
+   if (!(flags & PIPE_FLUSH_DEFERRED)) {
+      si_flush_implicit_resources(sctx);
+   }
+
    if (flags & PIPE_FLUSH_END_OF_FRAME)
       rflags |= PIPE_FLUSH_END_OF_FRAME;
 
@@ -484,6 +489,10 @@ static void si_flush_from_st(struct pipe_context *ctx, struct pipe_fence_handle
    if (sctx->sdma_cs)
       si_flush_dma_cs(sctx, rflags, fence ? &sdma_fence : NULL);
 
+   if (force_flush) {
+      sctx->initial_gfx_cs_size = 0;
+   }
+
    if (!radeon_emitted(sctx->gfx_cs, sctx->initial_gfx_cs_size)) {
       if (fence)
          ws->fence_reference(&gfx_fence, sctx->last_gfx_fence);
@@ -549,6 +558,12 @@ finish:
    }
 }
 
+static void si_flush_from_st(struct pipe_context *ctx, struct pipe_fence_handle **fence,
+                             unsigned flags)
+{
+   return si_flush_all_queues(ctx, fence, flags, false);
+}
+
 static void si_fence_server_signal(struct pipe_context *ctx, struct pipe_fence_handle *fence)
 {
    struct si_context *sctx = (struct si_context *)ctx;
@@ -572,11 +587,9 @@ static void si_fence_server_signal(struct pipe_context *ctx, struct pipe_fence_h
     * new work being emitted and getting executed before the signal
     * operation.
     *
-    * Set sctx->initial_gfx_cs_size to force IB submission even if
-    * it is empty.
+    * Forces a flush even if the GFX CS is empty.
     */
-   sctx->initial_gfx_cs_size = 0;
-   si_flush_from_st(ctx, NULL, PIPE_FLUSH_ASYNC);
+   si_flush_all_queues(ctx, NULL, PIPE_FLUSH_ASYNC, true);
 }
 
 static void si_fence_server_sync(struct pipe_context *ctx, struct pipe_fence_handle *fence)
diff --git a/src/gallium/drivers/radeonsi/si_pipe.c b/src/gallium/drivers/radeonsi/si_pipe.c
index cee842fc69b..24e87804988 100644
--- a/src/gallium/drivers/radeonsi/si_pipe.c
+++ b/src/gallium/drivers/radeonsi/si_pipe.c
@@ -166,6 +166,12 @@ static void si_destroy_compiler(struct ac_llvm_compiler *compiler)
    ac_destroy_llvm_compiler(compiler);
 }
 
+
+static void decref_implicit_resource(struct hash_entry *entry)
+{
+   pipe_resource_reference((struct pipe_resource**)&entry->data, NULL);
+}
+
 /*
  * pipe_context
  */
@@ -286,6 +292,10 @@ static void si_destroy_context(struct pipe_context *context)
    if (sctx->ctx)
       sctx->ws->ctx_destroy(sctx->ctx);
 
+   if (sctx->dirty_implicit_resources)
+      _mesa_hash_table_destroy(sctx->dirty_implicit_resources,
+                               decref_implicit_resource);
+
    if (sctx->b.stream_uploader)
       u_upload_destroy(sctx->b.stream_uploader);
    if (sctx->b.const_uploader)
@@ -707,6 +717,10 @@ static struct pipe_context *si_create_context(struct pipe_screen *screen, unsign
    util_dynarray_init(&sctx->resident_img_needs_color_decompress, NULL);
    util_dynarray_init(&sctx->resident_tex_needs_depth_decompress, NULL);
 
+   sctx->dirty_implicit_resources = _mesa_pointer_hash_table_create(NULL);
+   if (!sctx->dirty_implicit_resources)
+      goto fail;
+
    sctx->sample_pos_buffer =
       pipe_buffer_create(sctx->b.screen, 0, PIPE_USAGE_DEFAULT, sizeof(sctx->sample_positions));
    pipe_buffer_write(&sctx->b, sctx->sample_pos_buffer, 0, sizeof(sctx->sample_positions),
diff --git a/src/gallium/drivers/radeonsi/si_pipe.h b/src/gallium/drivers/radeonsi/si_pipe.h
index a5c7daeacdb..acf0aa65fe1 100644
--- a/src/gallium/drivers/radeonsi/si_pipe.h
+++ b/src/gallium/drivers/radeonsi/si_pipe.h
@@ -1282,6 +1282,12 @@ struct si_context {
                     unsigned src_level, const struct pipe_box *src_box);
 
    struct si_tracked_regs tracked_regs;
+
+   /* Resources that need to be flushed, but will not get an explicit
+    * flush_resource from the frontend and that will need to get flushed during
+    * a context flush.
+    */
+   struct hash_table *dirty_implicit_resources;
 };
 
 /* cik_sdma.c */
@@ -1307,6 +1313,7 @@ void si_resource_copy_region(struct pipe_context *ctx, struct pipe_resource *dst
                              struct pipe_resource *src, unsigned src_level,
                              const struct pipe_box *src_box);
 void si_decompress_dcc(struct si_context *sctx, struct si_texture *tex);
+void si_flush_implicit_resources(struct si_context *sctx);
 
 /* si_buffer.c */
 bool si_rings_is_buffer_referenced(struct si_context *sctx, struct pb_buffer *buf,
diff --git a/src/gallium/drivers/radeonsi/si_state.c b/src/gallium/drivers/radeonsi/si_state.c
index 0352345bbc9..b7b2bb2c209 100644
--- a/src/gallium/drivers/radeonsi/si_state.c
+++ b/src/gallium/drivers/radeonsi/si_state.c
@@ -2576,9 +2576,17 @@ static void si_update_display_dcc_dirty(struct si_context *sctx)
       surf = (struct si_surface *)state->cbufs[i];
       tex = (struct si_texture *)surf->base.texture;
 
-      if (!tex->surface.display_dcc_offset)
+      if (!tex->surface.display_dcc_offset || tex->displayable_dcc_dirty)
          continue;
 
+      if (!(tex->buffer.external_usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH)) {
+         struct hash_entry *entry = _mesa_hash_table_search(sctx->dirty_implicit_resources, tex);
+         if (!entry) {
+            struct pipe_resource *dummy = NULL;
+            pipe_resource_reference(&dummy, &tex->buffer.b.b);
+            _mesa_hash_table_insert(sctx->dirty_implicit_resources, tex, tex);
+         }
+      }
       tex->displayable_dcc_dirty = true;
    }
 }
diff --git a/src/gallium/drivers/radeonsi/si_texture.c b/src/gallium/drivers/radeonsi/si_texture.c
index bfc76b744ab..3fea3b2eebd 100644
--- a/src/gallium/drivers/radeonsi/si_texture.c
+++ b/src/gallium/drivers/radeonsi/si_texture.c
@@ -784,9 +784,6 @@ static bool si_texture_get_handle(struct pipe_screen *screen, struct pipe_contex
 
    si_texture_get_info(screen, resource, &stride, &offset);
 
-   if (flush)
-      sctx->b.flush(&sctx->b, NULL, 0);
-
    if (res->b.is_shared) {
       /* USAGE_EXPLICIT_FLUSH must be cleared if at least one user
        * doesn't set it.
@@ -799,6 +796,9 @@ static bool si_texture_get_handle(struct pipe_screen *screen, struct pipe_contex
       res->external_usage = usage;
    }
 
+   if (flush)
+      sctx->b.flush(&sctx->b, NULL, 0);
+
    whandle->stride = stride;
    whandle->offset = offset + slice_size * whandle->layer;