summaryrefslogtreecommitdiff
path: root/src/gallium/drivers/iris/iris_blit.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/gallium/drivers/iris/iris_blit.c')
-rw-r--r--src/gallium/drivers/iris/iris_blit.c315
1 files changed, 161 insertions, 154 deletions
diff --git a/src/gallium/drivers/iris/iris_blit.c b/src/gallium/drivers/iris/iris_blit.c
index 461da6cb947..921e9ef3f43 100644
--- a/src/gallium/drivers/iris/iris_blit.c
+++ b/src/gallium/drivers/iris/iris_blit.c
@@ -27,6 +27,7 @@
#include "pipe/p_screen.h"
#include "util/format/u_format.h"
#include "util/u_inlines.h"
+#include "util/u_surface.h"
#include "util/ralloc.h"
#include "intel/blorp/blorp.h"
#include "iris_context.h"
@@ -229,24 +230,26 @@ apply_blit_scissor(const struct pipe_scissor_state *scissor,
}
void
-iris_blorp_surf_for_resource(struct isl_device *isl_dev,
+iris_blorp_surf_for_resource(struct iris_batch *batch,
struct blorp_surf *surf,
struct pipe_resource *p_res,
enum isl_aux_usage aux_usage,
unsigned level,
- bool is_render_target)
+ bool is_dest)
{
+ const struct isl_device *isl_dev = &batch->screen->isl_dev;
struct iris_resource *res = (void *) p_res;
+ const struct intel_device_info *devinfo = isl_dev->info;
*surf = (struct blorp_surf) {
.surf = &res->surf,
.addr = (struct blorp_address) {
.buffer = res->bo,
.offset = res->offset,
- .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0,
+ .reloc_flags = is_dest ? IRIS_BLORP_RELOC_FLAGS_EXEC_OBJECT_WRITE : 0,
.mocs = iris_mocs(res->bo, isl_dev,
- is_render_target ? ISL_SURF_USAGE_RENDER_TARGET_BIT
- : ISL_SURF_USAGE_TEXTURE_BIT),
+ iris_blorp_batch_usage(batch, is_dest)),
+ .local_hint = iris_bo_likely_local(res->bo),
},
.aux_usage = aux_usage,
};
@@ -256,8 +259,10 @@ iris_blorp_surf_for_resource(struct isl_device *isl_dev,
surf->aux_addr = (struct blorp_address) {
.buffer = res->aux.bo,
.offset = res->aux.offset,
- .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0,
+ .reloc_flags = is_dest ? IRIS_BLORP_RELOC_FLAGS_EXEC_OBJECT_WRITE : 0,
.mocs = iris_mocs(res->bo, isl_dev, 0),
+ .local_hint = devinfo->has_flat_ccs ||
+ iris_bo_likely_local(res->aux.bo),
};
surf->clear_color = res->aux.clear_color;
surf->clear_color_addr = (struct blorp_address) {
@@ -265,6 +270,8 @@ iris_blorp_surf_for_resource(struct isl_device *isl_dev,
.offset = res->aux.clear_color_offset,
.reloc_flags = 0,
.mocs = iris_mocs(res->aux.clear_color_bo, isl_dev, 0),
+ .local_hint = devinfo->has_flat_ccs ||
+ iris_bo_likely_local(res->aux.clear_color_bo),
};
}
}
@@ -272,8 +279,7 @@ iris_blorp_surf_for_resource(struct isl_device *isl_dev,
static bool
is_astc(enum isl_format format)
{
- return format != ISL_FORMAT_UNSUPPORTED &&
- isl_format_get_layout(format)->txc == ISL_TXC_ASTC;
+ return isl_format_get_layout(format)->txc == ISL_TXC_ASTC;
}
static void
@@ -281,7 +287,7 @@ tex_cache_flush_hack(struct iris_batch *batch,
enum isl_format view_format,
enum isl_format surf_format)
{
- const struct intel_device_info *devinfo = &batch->screen->devinfo;
+ const struct intel_device_info *devinfo = batch->screen->devinfo;
/* The WaSamplerCacheFlushBetweenRedescribedSurfaceReads workaround says:
*
@@ -337,6 +343,16 @@ pipe_format_for_aspect(enum pipe_format format, unsigned pipe_mask)
}
}
+static bool
+clear_color_is_fully_zero(const struct iris_resource *res)
+{
+ return !res->aux.clear_color_unknown &&
+ res->aux.clear_color.u32[0] == 0 &&
+ res->aux.clear_color.u32[1] == 0 &&
+ res->aux.clear_color.u32[2] == 0 &&
+ res->aux.clear_color.u32[3] == 0;
+}
+
/**
* The pipe->blit() driver hook.
*
@@ -348,9 +364,9 @@ iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info)
{
struct iris_context *ice = (void *) ctx;
struct iris_screen *screen = (struct iris_screen *)ctx->screen;
- const struct intel_device_info *devinfo = &screen->devinfo;
+ const struct intel_device_info *devinfo = screen->devinfo;
struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
- enum blorp_batch_flags blorp_flags = 0;
+ enum blorp_batch_flags blorp_flags = iris_blorp_flags_for_batch(batch);
/* We don't support color masking. */
assert((info->mask & PIPE_MASK_RGBA) == PIPE_MASK_RGBA ||
@@ -385,6 +401,19 @@ iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info)
return;
}
+ /* Do DRI PRIME blits on the hardware blitter on Gfx12+ */
+ if (devinfo->ver >= 12 &&
+ (info->dst.resource->bind & PIPE_BIND_PRIME_BLIT_DST)) {
+ assert(!info->render_condition_enable);
+ assert(util_can_blit_via_copy_region(info, false, false));
+ iris_copy_region(&ice->blorp, &ice->batches[IRIS_BATCH_BLITTER],
+ info->dst.resource, info->dst.level,
+ info->dst.box.x, info->dst.box.y, info->dst.box.z,
+ info->src.resource, info->src.level,
+ &info->src.box);
+ return;
+ }
+
if (abs(info->dst.box.width) == abs(info->src.box.width) &&
abs(info->dst.box.height) == abs(info->src.box.height)) {
if (info->src.resource->nr_samples > 1 &&
@@ -463,35 +492,36 @@ iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info)
struct iris_format_info src_fmt =
iris_format_for_usage(devinfo, src_pfmt, ISL_SURF_USAGE_TEXTURE_BIT);
enum isl_aux_usage src_aux_usage =
- iris_resource_texture_aux_usage(ice, src_res, src_fmt.fmt);
+ iris_resource_texture_aux_usage(ice, src_res, src_fmt.fmt,
+ info->src.level, 1);
iris_resource_prepare_texture(ice, src_res, src_fmt.fmt,
info->src.level, 1, info->src.box.z,
info->src.box.depth);
iris_emit_buffer_barrier_for(batch, src_res->bo,
- IRIS_DOMAIN_OTHER_READ);
+ IRIS_DOMAIN_SAMPLER_READ);
struct iris_format_info dst_fmt =
iris_format_for_usage(devinfo, dst_pfmt,
ISL_SURF_USAGE_RENDER_TARGET_BIT);
enum isl_aux_usage dst_aux_usage =
- iris_resource_render_aux_usage(ice, dst_res, info->dst.level,
- dst_fmt.fmt, false);
+ iris_resource_render_aux_usage(ice, dst_res, dst_fmt.fmt,
+ info->dst.level, false);
+
+ iris_resource_prepare_render(ice, dst_res, dst_fmt.fmt, info->dst.level,
+ info->dst.box.z, info->dst.box.depth,
+ dst_aux_usage);
+ iris_emit_buffer_barrier_for(batch, dst_res->bo,
+ IRIS_DOMAIN_RENDER_WRITE);
struct blorp_surf src_surf, dst_surf;
- iris_blorp_surf_for_resource(&screen->isl_dev, &src_surf,
+ iris_blorp_surf_for_resource(batch, &src_surf,
&src_res->base.b, src_aux_usage,
info->src.level, false);
- iris_blorp_surf_for_resource(&screen->isl_dev, &dst_surf,
+ iris_blorp_surf_for_resource(batch, &dst_surf,
&dst_res->base.b, dst_aux_usage,
info->dst.level, true);
- iris_resource_prepare_render(ice, dst_res, info->dst.level,
- info->dst.box.z, info->dst.box.depth,
- dst_aux_usage);
- iris_emit_buffer_barrier_for(batch, dst_res->bo,
- IRIS_DOMAIN_RENDER_WRITE);
-
if (iris_batch_references(batch, src_res->bo))
tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format);
@@ -529,76 +559,76 @@ iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info)
blorp_batch_finish(&blorp_batch);
- iris_flush_and_dirty_for_history(ice, batch, (struct iris_resource *)
- info->dst.resource,
- PIPE_CONTROL_RENDER_TARGET_FLUSH,
- "cache history: post-blit");
+ iris_dirty_for_history(ice, (struct iris_resource *)info->dst.resource);
}
-static void
-get_copy_region_aux_settings(struct iris_context *ice,
- struct iris_resource *res,
- unsigned level,
- enum isl_aux_usage *out_aux_usage,
- bool *out_clear_supported,
- bool is_render_target)
+static enum isl_aux_usage
+copy_region_aux_usage(struct iris_context *ice,
+ const struct iris_batch *batch,
+ struct iris_resource *res,
+ enum isl_format view_format,
+ unsigned level,
+ bool is_dest)
{
struct iris_screen *screen = (void *) ice->ctx.screen;
- struct intel_device_info *devinfo = &screen->devinfo;
-
- switch (res->aux.usage) {
- case ISL_AUX_USAGE_HIZ:
- case ISL_AUX_USAGE_HIZ_CCS:
- case ISL_AUX_USAGE_HIZ_CCS_WT:
- case ISL_AUX_USAGE_STC_CCS:
- if (is_render_target) {
- *out_aux_usage = iris_resource_render_aux_usage(ice, res, level,
- res->surf.format,
- false);
+ const struct intel_device_info *devinfo = screen->devinfo;
+
+ if (batch->name == IRIS_BATCH_RENDER) {
+ if (is_dest) {
+ return iris_resource_render_aux_usage(ice, res, view_format, level,
+ false);
} else {
- *out_aux_usage = iris_resource_texture_aux_usage(ice, res,
- res->surf.format);
+ return iris_resource_texture_aux_usage(ice, res, view_format, level,
+ 1);
}
- *out_clear_supported = isl_aux_usage_has_fast_clears(*out_aux_usage);
- break;
- case ISL_AUX_USAGE_MCS:
- case ISL_AUX_USAGE_MCS_CCS:
- if (!is_render_target &&
- !iris_can_sample_mcs_with_clear(devinfo, res)) {
- *out_aux_usage = res->aux.usage;
- *out_clear_supported = false;
- break;
+ } else {
+ assert(batch->name == IRIS_BATCH_BLITTER);
+
+ if (is_dest) {
+
+ /* We only blit to images created with PIPE_BIND_PRIME_BLIT_DST.
+ * These are only created with dri3_alloc_render_buffer. That
+ * function also makes them linear, so they lack compression.
+ *
+ * Note: this code block could be substituted with the one below, but
+ * this setup clarifies that no additional handling of FCV is
+ * necessary.
+ */
+ assert(res->base.b.bind & PIPE_BIND_PRIME_BLIT_DST);
+ assert(res->surf.tiling == ISL_TILING_LINEAR);
+ assert(res->aux.usage == ISL_AUX_USAGE_NONE);
+
+ return ISL_AUX_USAGE_NONE;
+ } else {
+ return devinfo->verx10 >= 125 ? res->aux.usage : ISL_AUX_USAGE_NONE;
}
- FALLTHROUGH;
- case ISL_AUX_USAGE_CCS_E:
- case ISL_AUX_USAGE_GFX12_CCS_E:
- *out_aux_usage = res->aux.usage;
-
- /* blorp_copy may reinterpret the surface format and has limited support
- * for adjusting the clear color, so clear support may only be enabled
- * in some cases:
- *
- * - On gfx11+, the clear color is indirect and comes in two forms: a
- * 32bpc representation used for rendering and a pixel representation
- * used for sampling. blorp_copy doesn't change indirect clear colors,
- * so clears are only supported in the sampling case.
- *
- * - A clear color of zeroes holds the same meaning regardless of the
- * format. Although it could avoid more resolves, we don't use
- * isl_color_value_is_zero because the surface format used by
- * blorp_copy isn't guaranteed to access the same components as the
- * original format (e.g. A8_UNORM/R8_UINT).
- */
- *out_clear_supported = (devinfo->ver >= 11 && !is_render_target) ||
- (res->aux.clear_color.u32[0] == 0 &&
- res->aux.clear_color.u32[1] == 0 &&
- res->aux.clear_color.u32[2] == 0 &&
- res->aux.clear_color.u32[3] == 0);
- break;
- default:
- *out_aux_usage = ISL_AUX_USAGE_NONE;
- *out_clear_supported = false;
- break;
+ }
+}
+
+static void
+prepare_copy_region(struct iris_context *ice,
+ const struct iris_batch *batch,
+ struct iris_resource *res,
+ enum isl_format view_format,
+ uint32_t level,
+ uint32_t start_layer, uint32_t layer_count,
+ enum isl_aux_usage aux_usage,
+ bool is_dest)
+{
+ if (batch->name == IRIS_BATCH_RENDER) {
+ if (is_dest) {
+ iris_resource_prepare_render(ice, res, view_format, level,
+ start_layer, layer_count, aux_usage);
+ } else {
+ iris_resource_prepare_texture(ice, res, view_format, level, 1,
+ start_layer, layer_count);
+ }
+ } else {
+ assert(batch->name == IRIS_BATCH_BLITTER);
+ iris_resource_prepare_access(ice, res, level, 1,
+ start_layer, layer_count, aux_usage,
+ isl_aux_usage_has_fast_clears(aux_usage) &&
+ clear_color_is_fully_zero(res));
}
}
@@ -626,62 +656,70 @@ iris_copy_region(struct blorp_context *blorp,
struct iris_resource *src_res = (void *) src;
struct iris_resource *dst_res = (void *) dst;
- enum isl_aux_usage src_aux_usage, dst_aux_usage;
- bool src_clear_supported, dst_clear_supported;
- get_copy_region_aux_settings(ice, src_res, src_level, &src_aux_usage,
- &src_clear_supported, false);
- get_copy_region_aux_settings(ice, dst_res, dst_level, &dst_aux_usage,
- &dst_clear_supported, true);
+ enum iris_domain write_domain =
+ batch->name == IRIS_BATCH_BLITTER ? IRIS_DOMAIN_OTHER_WRITE
+ : IRIS_DOMAIN_RENDER_WRITE;
+
+ enum isl_format src_fmt, dst_fmt;
+ blorp_copy_get_formats(&screen->isl_dev, &src_res->surf, &dst_res->surf,
+ &src_fmt, &dst_fmt);
+
+ enum isl_aux_usage src_aux_usage =
+ copy_region_aux_usage(ice, batch, src_res, src_fmt, src_level, false);
+ enum isl_aux_usage dst_aux_usage =
+ copy_region_aux_usage(ice, batch, dst_res, dst_fmt, dst_level, true);
if (iris_batch_references(batch, src_res->bo))
- tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format);
+ tex_cache_flush_hack(batch, src_fmt, src_res->surf.format);
if (dst->target == PIPE_BUFFER)
util_range_add(&dst_res->base.b, &dst_res->valid_buffer_range, dstx, dstx + src_box->width);
+ enum blorp_batch_flags blorp_flags = iris_blorp_flags_for_batch(batch);
+
+ blorp_batch_init(blorp, &blorp_batch, batch, blorp_flags);
+
if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
struct blorp_address src_addr = {
- .buffer = iris_resource_bo(src), .offset = src_box->x,
+ .buffer = src_res->bo, .offset = src_res->offset + src_box->x,
+ .mocs = iris_mocs(src_res->bo, &screen->isl_dev,
+ iris_blorp_batch_usage(batch, false /* is_dest */)),
+ .local_hint = iris_bo_likely_local(src_res->bo),
};
struct blorp_address dst_addr = {
- .buffer = iris_resource_bo(dst), .offset = dstx,
- .reloc_flags = EXEC_OBJECT_WRITE,
+ .buffer = dst_res->bo, .offset = dst_res->offset + dstx,
+ .reloc_flags = IRIS_BLORP_RELOC_FLAGS_EXEC_OBJECT_WRITE,
+ .mocs = iris_mocs(dst_res->bo, &screen->isl_dev,
+ iris_blorp_batch_usage(batch, true /* is_dest */)),
+ .local_hint = iris_bo_likely_local(dst_res->bo),
};
- iris_emit_buffer_barrier_for(batch, iris_resource_bo(src),
- IRIS_DOMAIN_OTHER_READ);
- iris_emit_buffer_barrier_for(batch, iris_resource_bo(dst),
- IRIS_DOMAIN_RENDER_WRITE);
+ iris_emit_buffer_barrier_for(batch, src_res->bo,
+ IRIS_DOMAIN_SAMPLER_READ);
+ iris_emit_buffer_barrier_for(batch, dst_res->bo, write_domain);
iris_batch_maybe_flush(batch, 1500);
iris_batch_sync_region_start(batch);
- blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
blorp_buffer_copy(&blorp_batch, src_addr, dst_addr, src_box->width);
- blorp_batch_finish(&blorp_batch);
iris_batch_sync_region_end(batch);
} else {
// XXX: what about one surface being a buffer and not the other?
- struct blorp_surf src_surf, dst_surf;
- iris_blorp_surf_for_resource(&screen->isl_dev, &src_surf,
- src, src_aux_usage, src_level, false);
- iris_blorp_surf_for_resource(&screen->isl_dev, &dst_surf,
- dst, dst_aux_usage, dst_level, true);
-
- iris_resource_prepare_access(ice, src_res, src_level, 1,
- src_box->z, src_box->depth,
- src_aux_usage, src_clear_supported);
- iris_resource_prepare_access(ice, dst_res, dst_level, 1,
- dstz, src_box->depth,
- dst_aux_usage, dst_clear_supported);
-
- iris_emit_buffer_barrier_for(batch, iris_resource_bo(src),
- IRIS_DOMAIN_OTHER_READ);
- iris_emit_buffer_barrier_for(batch, iris_resource_bo(dst),
- IRIS_DOMAIN_RENDER_WRITE);
+ prepare_copy_region(ice, batch, src_res, src_fmt, src_level,
+ src_box->z, src_box->depth, src_aux_usage, false);
+ prepare_copy_region(ice, batch, dst_res, dst_fmt, dst_level,
+ dstz, src_box->depth, dst_aux_usage, true);
+
+ iris_emit_buffer_barrier_for(batch, src_res->bo,
+ IRIS_DOMAIN_SAMPLER_READ);
+ iris_emit_buffer_barrier_for(batch, dst_res->bo, write_domain);
- blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
+ struct blorp_surf src_surf, dst_surf;
+ iris_blorp_surf_for_resource(batch, &src_surf, src,
+ src_aux_usage, src_level, false);
+ iris_blorp_surf_for_resource(batch, &dst_surf, dst,
+ dst_aux_usage, dst_level, true);
for (int slice = 0; slice < src_box->depth; slice++) {
iris_batch_maybe_flush(batch, 1500);
@@ -693,29 +731,16 @@ iris_copy_region(struct blorp_context *blorp,
src_box->width, src_box->height);
iris_batch_sync_region_end(batch);
}
- blorp_batch_finish(&blorp_batch);
iris_resource_finish_write(ice, dst_res, dst_level, dstz,
src_box->depth, dst_aux_usage);
}
- tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format);
-}
-
-static struct iris_batch *
-get_preferred_batch(struct iris_context *ice, struct iris_bo *bo)
-{
- /* If the compute batch is already using this buffer, we'd prefer to
- * continue queueing in the compute batch.
- */
- if (iris_batch_references(&ice->batches[IRIS_BATCH_COMPUTE], bo))
- return &ice->batches[IRIS_BATCH_COMPUTE];
+ blorp_batch_finish(&blorp_batch);
- /* Otherwise default to the render batch. */
- return &ice->batches[IRIS_BATCH_RENDER];
+ tex_cache_flush_hack(batch, src_fmt, src_res->surf.format);
}
-
/**
* The pipe->resource_copy_region() driver hook.
*
@@ -732,24 +757,8 @@ iris_resource_copy_region(struct pipe_context *ctx,
const struct pipe_box *src_box)
{
struct iris_context *ice = (void *) ctx;
- struct iris_screen *screen = (void *) ctx->screen;
struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
- /* Use MI_COPY_MEM_MEM for tiny (<= 16 byte, % 4) buffer copies. */
- if (p_src->target == PIPE_BUFFER && p_dst->target == PIPE_BUFFER &&
- dstx % 4 == 0 && src_box->x % 4 == 0 &&
- src_box->width % 4 == 0 && src_box->width <= 16) {
- struct iris_bo *dst_bo = iris_resource_bo(p_dst);
- batch = get_preferred_batch(ice, dst_bo);
- iris_batch_maybe_flush(batch, 24 + 5 * (src_box->width / 4));
- iris_emit_pipe_control_flush(batch,
- "stall for MI_COPY_MEM_MEM copy_region",
- PIPE_CONTROL_CS_STALL);
- screen->vtbl.copy_mem_mem(batch, dst_bo, dstx, iris_resource_bo(p_src),
- src_box->x, src_box->width);
- return;
- }
-
iris_copy_region(&ice->blorp, batch, p_dst, dst_level, dstx, dsty, dstz,
p_src, src_level, src_box);
@@ -763,9 +772,7 @@ iris_resource_copy_region(struct pipe_context *ctx,
dsty, dstz, &s_src_res->base.b, src_level, src_box);
}
- iris_flush_and_dirty_for_history(ice, batch, (struct iris_resource *)p_dst,
- PIPE_CONTROL_RENDER_TARGET_FLUSH | PIPE_CONTROL_TILE_CACHE_FLUSH,
- "cache history: post copy_region");
+ iris_dirty_for_history(ice, (struct iris_resource *) p_dst);
}
void
generated by cgit v1.2.3 (git 2.39.1) at 2024-04-25 02:03:55 +0000