gallium/u_blitter: do MSAA copies in 1 pass using sample shading

Acked-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/13602>

3 files changed, 116 insertions, 59 deletions

diff --git a/src/gallium/auxiliary/util/u_blitter.c b/src/gallium/auxiliary/util/u_blitter.c
index b7b7e531d70..f23474325e1 100644
--- a/src/gallium/auxiliary/util/u_blitter.c
+++ b/src/gallium/auxiliary/util/u_blitter.c
@@ -95,9 +95,9 @@ struct blitter_context_priv
 
    /* FS which outputs one sample from a multisample texture. */
    void *fs_texfetch_col_msaa[5][PIPE_MAX_TEXTURE_TYPES];
-   void *fs_texfetch_depth_msaa[PIPE_MAX_TEXTURE_TYPES];
-   void *fs_texfetch_depthstencil_msaa[PIPE_MAX_TEXTURE_TYPES];
-   void *fs_texfetch_stencil_msaa[PIPE_MAX_TEXTURE_TYPES];
+   void *fs_texfetch_depth_msaa[PIPE_MAX_TEXTURE_TYPES][2];
+   void *fs_texfetch_depthstencil_msaa[PIPE_MAX_TEXTURE_TYPES][2];
+   void *fs_texfetch_stencil_msaa[PIPE_MAX_TEXTURE_TYPES][2];
 
    /* FS which outputs an average of all samples. */
    void *fs_resolve[PIPE_MAX_TEXTURE_TYPES][NUM_RESOLVE_FRAG_SHADERS][2];
@@ -149,6 +149,7 @@ struct blitter_context_priv
    bool has_texture_multisample;
    bool has_tex_lz;
    bool has_txf;
+   bool has_sample_shading;
    bool cube_as_2darray;
    bool cached_all_shaders;
 
@@ -214,6 +215,8 @@ struct blitter_context *util_blitter_create(struct pipe_context *pipe)
                                              PIPE_CAP_TGSI_TEX_TXF_LZ);
    ctx->has_txf = pipe->screen->get_param(pipe->screen,
                                           PIPE_CAP_GLSL_FEATURE_LEVEL) > 130;
+   ctx->has_sample_shading = pipe->screen->get_param(pipe->screen,
+                                                     PIPE_CAP_SAMPLE_SHADING);
    ctx->cube_as_2darray = pipe->screen->get_param(pipe->screen,
                                                   PIPE_CAP_SAMPLER_VIEW_TARGET);
 
@@ -547,12 +550,14 @@ void util_blitter_destroy(struct blitter_context *blitter)
             ctx->delete_fs_state(pipe, ctx->fs_texfetch_stencil[i][inst]);
       }
 
-      if (ctx->fs_texfetch_depth_msaa[i])
-         ctx->delete_fs_state(pipe, ctx->fs_texfetch_depth_msaa[i]);
-      if (ctx->fs_texfetch_depthstencil_msaa[i])
-         ctx->delete_fs_state(pipe, ctx->fs_texfetch_depthstencil_msaa[i]);
-      if (ctx->fs_texfetch_stencil_msaa[i])
-         ctx->delete_fs_state(pipe, ctx->fs_texfetch_stencil_msaa[i]);
+      for (unsigned ss = 0; ss < 2; ss++) {
+         if (ctx->fs_texfetch_depth_msaa[i][ss])
+            ctx->delete_fs_state(pipe, ctx->fs_texfetch_depth_msaa[i][ss]);
+         if (ctx->fs_texfetch_depthstencil_msaa[i][ss])
+            ctx->delete_fs_state(pipe, ctx->fs_texfetch_depthstencil_msaa[i][ss]);
+         if (ctx->fs_texfetch_stencil_msaa[i][ss])
+            ctx->delete_fs_state(pipe, ctx->fs_texfetch_stencil_msaa[i][ss]);
+      }
 
       for (j = 0; j< ARRAY_SIZE(ctx->fs_resolve[i]); j++)
          for (f = 0; f < 2; f++)
@@ -1020,7 +1025,8 @@ static void *blitter_get_fs_texfetch_col(struct blitter_context_priv *ctx,
          /* Create the fragment shader on-demand. */
          if (!*shader) {
             assert(!ctx->cached_all_shaders);
-            *shader = util_make_fs_blit_msaa_color(pipe, tgsi_tex, stype, dtype);
+            *shader = util_make_fs_blit_msaa_color(pipe, tgsi_tex, stype, dtype,
+                                                   ctx->has_sample_shading);
          }
       }
 
@@ -1087,22 +1093,24 @@ void *blitter_get_fs_pack_color_zs(struct blitter_context_priv *ctx,
 static inline
 void *blitter_get_fs_texfetch_depth(struct blitter_context_priv *ctx,
                                     enum pipe_texture_target target,
-                                    unsigned nr_samples,
+                                    unsigned src_samples, unsigned dst_samples,
                                     bool use_txf)
 {
    struct pipe_context *pipe = ctx->base.pipe;
 
    assert(target < PIPE_MAX_TEXTURE_TYPES);
 
-   if (nr_samples > 1) {
-      void **shader = &ctx->fs_texfetch_depth_msaa[target];
+   if (src_samples > 1) {
+      bool sample_shading = ctx->has_sample_shading && src_samples > 1 &&
+                            src_samples == dst_samples;
+      void **shader = &ctx->fs_texfetch_depth_msaa[target][sample_shading];
 
       /* Create the fragment shader on-demand. */
       if (!*shader) {
          enum tgsi_texture_type tgsi_tex;
          assert(!ctx->cached_all_shaders);
-         tgsi_tex = util_pipe_tex_to_tgsi_tex(target, nr_samples);
-         *shader = util_make_fs_blit_msaa_depth(pipe, tgsi_tex);
+         tgsi_tex = util_pipe_tex_to_tgsi_tex(target, src_samples);
+         *shader = util_make_fs_blit_msaa_depth(pipe, tgsi_tex, sample_shading);
       }
 
       return *shader;
@@ -1130,22 +1138,25 @@ void *blitter_get_fs_texfetch_depth(struct blitter_context_priv *ctx,
 static inline
 void *blitter_get_fs_texfetch_depthstencil(struct blitter_context_priv *ctx,
                                            enum pipe_texture_target target,
-                                           unsigned nr_samples,
-                                           bool use_txf)
+                                           unsigned src_samples,
+                                           unsigned dst_samples, bool use_txf)
 {
    struct pipe_context *pipe = ctx->base.pipe;
 
    assert(target < PIPE_MAX_TEXTURE_TYPES);
 
-   if (nr_samples > 1) {
-      void **shader = &ctx->fs_texfetch_depthstencil_msaa[target];
+   if (src_samples > 1) {
+      bool sample_shading = ctx->has_sample_shading && src_samples > 1 &&
+                            src_samples == dst_samples;
+      void **shader = &ctx->fs_texfetch_depthstencil_msaa[target][sample_shading];
 
       /* Create the fragment shader on-demand. */
       if (!*shader) {
          enum tgsi_texture_type tgsi_tex;
          assert(!ctx->cached_all_shaders);
-         tgsi_tex = util_pipe_tex_to_tgsi_tex(target, nr_samples);
-         *shader = util_make_fs_blit_msaa_depthstencil(pipe, tgsi_tex);
+         tgsi_tex = util_pipe_tex_to_tgsi_tex(target, src_samples);
+         *shader = util_make_fs_blit_msaa_depthstencil(pipe, tgsi_tex,
+                                                       sample_shading);
       }
 
       return *shader;
@@ -1173,22 +1184,25 @@ void *blitter_get_fs_texfetch_depthstencil(struct blitter_context_priv *ctx,
 static inline
 void *blitter_get_fs_texfetch_stencil(struct blitter_context_priv *ctx,
                                       enum pipe_texture_target target,
-                                      unsigned nr_samples,
+                                      unsigned src_samples, unsigned dst_samples,
                                       bool use_txf)
 {
    struct pipe_context *pipe = ctx->base.pipe;
 
    assert(target < PIPE_MAX_TEXTURE_TYPES);
 
-   if (nr_samples > 1) {
-      void **shader = &ctx->fs_texfetch_stencil_msaa[target];
+   if (src_samples > 1) {
+      bool sample_shading = ctx->has_sample_shading && src_samples > 1 &&
+                            src_samples == dst_samples;
+      void **shader = &ctx->fs_texfetch_stencil_msaa[target][sample_shading];
 
       /* Create the fragment shader on-demand. */
       if (!*shader) {
          enum tgsi_texture_type tgsi_tex;
          assert(!ctx->cached_all_shaders);
-         tgsi_tex = util_pipe_tex_to_tgsi_tex(target, nr_samples);
-         *shader = util_make_fs_blit_msaa_stencil(pipe, tgsi_tex);
+         tgsi_tex = util_pipe_tex_to_tgsi_tex(target, src_samples);
+         *shader = util_make_fs_blit_msaa_stencil(pipe, tgsi_tex,
+                                                  sample_shading);
       }
 
       return *shader;
@@ -1273,10 +1287,18 @@ void util_blitter_cache_all_shaders(struct blitter_context *blitter)
             blitter_get_fs_texfetch_col(ctx, PIPE_FORMAT_R32_SINT,
                                         PIPE_FORMAT_R32_UINT, target,
                                         samples, samples, 0, use_txf);
-            blitter_get_fs_texfetch_depth(ctx, target, samples, use_txf);
+            blitter_get_fs_texfetch_depth(ctx, target, samples, samples, use_txf);
             if (ctx->has_stencil_export) {
-               blitter_get_fs_texfetch_depthstencil(ctx, target, samples, use_txf);
-               blitter_get_fs_texfetch_stencil(ctx, target, samples, use_txf);
+               blitter_get_fs_texfetch_depthstencil(ctx, target, samples, samples, use_txf);
+               blitter_get_fs_texfetch_stencil(ctx, target, samples, samples, use_txf);
+            }
+
+            if (samples == 2) {
+               blitter_get_fs_texfetch_depth(ctx, target, samples, 1, use_txf);
+               if (ctx->has_stencil_export) {
+                  blitter_get_fs_texfetch_depthstencil(ctx, target, samples, 1, use_txf);
+                  blitter_get_fs_texfetch_stencil(ctx, target, samples, 1, use_txf);
+               }
             }
 
             if (samples == 1)
@@ -1798,6 +1820,8 @@ static void do_blits(struct blitter_context_priv *ctx,
    struct pipe_context *pipe = ctx->base.pipe;
    unsigned src_samples = src->texture->nr_samples;
    unsigned dst_samples = dst->texture->nr_samples;
+   bool sample_shading = ctx->has_sample_shading && src_samples > 1 &&
+                         src_samples == dst_samples && !sample0_only;
    enum pipe_texture_target src_target = src->target;
    struct pipe_framebuffer_state fb_state = {0};
 
@@ -1811,7 +1835,7 @@ static void do_blits(struct blitter_context_priv *ctx,
    if ((src_target == PIPE_TEXTURE_1D ||
         src_target == PIPE_TEXTURE_2D ||
         src_target == PIPE_TEXTURE_RECT) &&
-       src_samples <= 1) {
+       (src_samples <= 1 || sample_shading)) {
       /* Set framebuffer state. */
       if (is_zsbuf) {
          fb_state.zsbuf = dst;
@@ -1823,7 +1847,7 @@ static void do_blits(struct blitter_context_priv *ctx,
       /* Draw. */
       pipe->set_sample_mask(pipe, ~0);
       if (pipe->set_min_samples)
-         pipe->set_min_samples(pipe, 1);
+         pipe->set_min_samples(pipe, sample_shading ? dst_samples : 1);
       blitter_draw_tex(ctx, dstbox->x, dstbox->y,
                        dstbox->x + dstbox->width,
                        dstbox->y + dstbox->height,
@@ -1879,11 +1903,10 @@ static void do_blits(struct blitter_context_priv *ctx,
             /* MSAA copy. */
             unsigned i, max_sample = sample0_only ? 0 : dst_samples - 1;
 
-            if (pipe->set_min_samples)
-               pipe->set_min_samples(pipe, 1);
-
-            for (i = 0; i <= max_sample; i++) {
-               pipe->set_sample_mask(pipe, 1 << i);
+            if (sample_shading) {
+               pipe->set_sample_mask(pipe, ~0);
+               if (pipe->set_min_samples)
+                  pipe->set_min_samples(pipe, max_sample);
                blitter_draw_tex(ctx, dstbox->x, dstbox->y,
                                 dstbox->x + dstbox->width,
                                 dstbox->y + dstbox->height,
@@ -1891,8 +1914,24 @@ static void do_blits(struct blitter_context_priv *ctx,
                                 srcbox->x, srcbox->y,
                                 srcbox->x + srcbox->width,
                                 srcbox->y + srcbox->height,
-                                srcbox->z + src_z, i, uses_txf,
+                                srcbox->z + src_z, 0, uses_txf,
                                 UTIL_BLITTER_ATTRIB_TEXCOORD_XYZW);
+            } else {
+               if (pipe->set_min_samples)
+                  pipe->set_min_samples(pipe, 1);
+
+               for (i = 0; i <= max_sample; i++) {
+                  pipe->set_sample_mask(pipe, 1 << i);
+                  blitter_draw_tex(ctx, dstbox->x, dstbox->y,
+                                   dstbox->x + dstbox->width,
+                                   dstbox->y + dstbox->height,
+                                   src, src_width0, src_height0,
+                                   srcbox->x, srcbox->y,
+                                   srcbox->x + srcbox->width,
+                                   srcbox->y + srcbox->height,
+                                   srcbox->z + src_z, i, uses_txf,
+                                   UTIL_BLITTER_ATTRIB_TEXCOORD_XYZW);
+               }
             }
          } else {
             /* Normal copy, MSAA upsampling, or MSAA resolve. */
@@ -2024,8 +2063,8 @@ void util_blitter_blit_generic(struct blitter_context *blitter,
                                          src_samples, dst->format, false));
       } else {
          ctx->bind_fs_state(pipe,
-            blitter_get_fs_texfetch_depthstencil(ctx, src_target,
-                                                 src_samples, use_txf));
+            blitter_get_fs_texfetch_depthstencil(ctx, src_target, src_samples,
+                                                 dst_samples, use_txf));
       }
    } else if (dst_has_depth) {
       pipe->bind_blend_state(pipe, ctx->blend[0][0]);
@@ -2040,8 +2079,8 @@ void util_blitter_blit_generic(struct blitter_context *blitter,
                                          src_samples, dst->format, false));
       } else {
          ctx->bind_fs_state(pipe,
-            blitter_get_fs_texfetch_depth(ctx, src_target,
-                                          src_samples, use_txf));
+            blitter_get_fs_texfetch_depth(ctx, src_target, src_samples,
+                                          dst_samples, use_txf));
       }
    } else if (dst_has_stencil) {
       pipe->bind_blend_state(pipe, ctx->blend[0][0]);
@@ -2050,8 +2089,8 @@ void util_blitter_blit_generic(struct blitter_context *blitter,
 
       assert(src_has_stencil); /* unpacking from color is unsupported */
       ctx->bind_fs_state(pipe,
-         blitter_get_fs_texfetch_stencil(ctx, src_target,
-                                         src_samples, use_txf));
+         blitter_get_fs_texfetch_stencil(ctx, src_target, src_samples,
+                                         dst_samples, use_txf));
    } else {
       unsigned colormask = mask & PIPE_MASK_RGBA;
 
@@ -2223,7 +2262,7 @@ void util_blitter_generate_mipmap(struct blitter_context *blitter,
       pipe->bind_depth_stencil_alpha_state(pipe,
                                            ctx->dsa_write_depth_keep_stencil);
       ctx->bind_fs_state(pipe,
-                         blitter_get_fs_texfetch_depth(ctx, target, 1, false));
+                         blitter_get_fs_texfetch_depth(ctx, target, 1, 1, false));
    } else {
       pipe->bind_blend_state(pipe, ctx->blend[PIPE_MASK_RGBA][0]);
       pipe->bind_depth_stencil_alpha_state(pipe, ctx->dsa_keep_depth_stencil);
diff --git a/src/gallium/auxiliary/util/u_simple_shaders.c b/src/gallium/auxiliary/util/u_simple_shaders.c
index 7a9c5cf435f..f46b319789d 100644
--- a/src/gallium/auxiliary/util/u_simple_shaders.c
+++ b/src/gallium/auxiliary/util/u_simple_shaders.c
@@ -527,6 +527,7 @@ util_make_fragment_cloneinput_shader(struct pipe_context *pipe, int num_cbufs,
 static void *
 util_make_fs_blit_msaa_gen(struct pipe_context *pipe,
                            enum tgsi_texture_type tgsi_tex,
+                           bool sample_shading,
                            const char *samp_type,
                            const char *output_semantic,
                            const char *output_mask,
@@ -541,15 +542,17 @@ util_make_fs_blit_msaa_gen(struct pipe_context *pipe,
          "DCL OUT[0], %s\n"
          "DCL TEMP[0]\n"
          "%s"
+         "%s"
 
          "F2U TEMP[0], IN[0]\n"
+         "%s"
          "TXF TEMP[0], TEMP[0], SAMP[0], %s\n"
          "%s"
          "MOV OUT[0]%s, TEMP[0]\n"
          "END\n";
 
    const char *type = tgsi_texture_names[tgsi_tex];
-   char text[sizeof(shader_templ)+100];
+   char text[sizeof(shader_templ)+400];
    struct tgsi_token tokens[1000];
    struct pipe_shader_state state = {0};
 
@@ -557,7 +560,9 @@ util_make_fs_blit_msaa_gen(struct pipe_context *pipe,
           tgsi_tex == TGSI_TEXTURE_2D_ARRAY_MSAA);
 
    snprintf(text, sizeof(text), shader_templ, type, samp_type,
-            output_semantic, conversion_decl, type, conversion, output_mask);
+            output_semantic, sample_shading ? "DCL SV[0], SAMPLEID\n" : "",
+            conversion_decl, sample_shading ? "MOV TEMP[0].w, SV[0].xxxx\n" : "",
+            type, conversion, output_mask);
 
    if (!tgsi_text_translate(text, tokens, ARRAY_SIZE(tokens))) {
       puts(text);
@@ -582,7 +587,8 @@ void *
 util_make_fs_blit_msaa_color(struct pipe_context *pipe,
                              enum tgsi_texture_type tgsi_tex,
                              enum tgsi_return_type stype,
-                             enum tgsi_return_type dtype)
+                             enum tgsi_return_type dtype,
+                             bool sample_shading)
 {
    const char *samp_type;
    const char *conversion_decl = "";
@@ -607,7 +613,7 @@ util_make_fs_blit_msaa_color(struct pipe_context *pipe,
       samp_type = "FLOAT";
    }
 
-   return util_make_fs_blit_msaa_gen(pipe, tgsi_tex, samp_type,
+   return util_make_fs_blit_msaa_gen(pipe, tgsi_tex, sample_shading, samp_type,
                                      "COLOR[0]", "", conversion_decl,
                                      conversion);
 }
@@ -620,9 +626,10 @@ util_make_fs_blit_msaa_color(struct pipe_context *pipe,
  */
 void *
 util_make_fs_blit_msaa_depth(struct pipe_context *pipe,
-                             enum tgsi_texture_type tgsi_tex)
+                             enum tgsi_texture_type tgsi_tex,
+                             bool sample_shading)
 {
-   return util_make_fs_blit_msaa_gen(pipe, tgsi_tex, "FLOAT",
+   return util_make_fs_blit_msaa_gen(pipe, tgsi_tex, sample_shading, "FLOAT",
                                      "POSITION", ".z", "",
                                      "MOV TEMP[0].z, TEMP[0].xxxx\n");
 }
@@ -635,9 +642,10 @@ util_make_fs_blit_msaa_depth(struct pipe_context *pipe,
  */
 void *
 util_make_fs_blit_msaa_stencil(struct pipe_context *pipe,
-                               enum tgsi_texture_type tgsi_tex)
+                               enum tgsi_texture_type tgsi_tex,
+                               bool sample_shading)
 {
-   return util_make_fs_blit_msaa_gen(pipe, tgsi_tex, "UINT",
+   return util_make_fs_blit_msaa_gen(pipe, tgsi_tex, sample_shading, "UINT",
                                      "STENCIL", ".y", "",
                                      "MOV TEMP[0].y, TEMP[0].xxxx\n");
 }
@@ -652,7 +660,8 @@ util_make_fs_blit_msaa_stencil(struct pipe_context *pipe,
  */
 void *
 util_make_fs_blit_msaa_depthstencil(struct pipe_context *pipe,
-                                    enum tgsi_texture_type tgsi_tex)
+                                    enum tgsi_texture_type tgsi_tex,
+                                    bool sample_shading)
 {
    static const char shader_templ[] =
          "FRAG\n"
@@ -663,21 +672,26 @@ util_make_fs_blit_msaa_depthstencil(struct pipe_context *pipe,
          "DCL OUT[0], POSITION\n"
          "DCL OUT[1], STENCIL\n"
          "DCL TEMP[0]\n"
+         "%s"
 
          "F2U TEMP[0], IN[0]\n"
+         "%s"
          "TXF OUT[0].z, TEMP[0], SAMP[0], %s\n"
          "TXF OUT[1].y, TEMP[0], SAMP[1], %s\n"
          "END\n";
 
    const char *type = tgsi_texture_names[tgsi_tex];
-   char text[sizeof(shader_templ)+100];
+   char text[sizeof(shader_templ)+400];
    struct tgsi_token tokens[1000];
    struct pipe_shader_state state = {0};
 
    assert(tgsi_tex == TGSI_TEXTURE_2D_MSAA ||
           tgsi_tex == TGSI_TEXTURE_2D_ARRAY_MSAA);
 
-   sprintf(text, shader_templ, type, type, type, type);
+   sprintf(text, shader_templ, type, type,
+           sample_shading ? "DCL SV[0], SAMPLEID\n" : "",
+           sample_shading ? "MOV TEMP[0].w, SV[0].xxxx\n" : "",
+           type, type);
 
    if (!tgsi_text_translate(text, tokens, ARRAY_SIZE(tokens))) {
       assert(0);
diff --git a/src/gallium/auxiliary/util/u_simple_shaders.h b/src/gallium/auxiliary/util/u_simple_shaders.h
index 6bc79401847..dfd16b9d11f 100644
--- a/src/gallium/auxiliary/util/u_simple_shaders.h
+++ b/src/gallium/auxiliary/util/u_simple_shaders.h
@@ -117,22 +117,26 @@ extern void *
 util_make_fs_blit_msaa_color(struct pipe_context *pipe,
                              enum tgsi_texture_type tgsi_tex,
                              enum tgsi_return_type stype,
-                             enum tgsi_return_type dtype);
+                             enum tgsi_return_type dtype,
+                             bool sample_shading);
 
 
 extern void *
 util_make_fs_blit_msaa_depth(struct pipe_context *pipe,
-                             enum tgsi_texture_type tgsi_tex);
+                             enum tgsi_texture_type tgsi_tex,
+                             bool sample_shading);
 
 
 extern void *
 util_make_fs_blit_msaa_depthstencil(struct pipe_context *pipe,
-                                    enum tgsi_texture_type tgsi_tex);
+                                    enum tgsi_texture_type tgsi_tex,
+                                    bool sample_shading);
 
 
 void *
 util_make_fs_blit_msaa_stencil(struct pipe_context *pipe,
-                               enum tgsi_texture_type tgsi_tex);
+                               enum tgsi_texture_type tgsi_tex,
+                               bool sample_shading);
 
 
 void *