summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorConnor Abbott <cwabbott0@gmail.com>2021-06-17 17:04:41 +0200
committerMarge Bot <eric+marge@anholt.net>2021-06-23 17:20:29 +0000
commit0f329ba10ae64a6c58d7945d8b1b3f19959878e2 (patch)
tree2bb2840fbff9409939ddd10ee2d1e506a5021eac /src
parentcc649453364dd1ea3a8efff8f3bc4290ba80e34b (diff)
ir3: Split read-modify-write array dests in two
Instructions that operate on an array read the previous state of the array, modify it, and write a new array, at least conceptually before RA. Previously the same register specified the previous state and acted as the new state, but this meant that it was both a source and destination which meant that it was getting in the way of splitting up sources and destinations. Break out the source into a separate register, and use the new tied-src infrastructure to share code with a6xx atomics. With this, there are basically no more special cases for arrays in RA. Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11469>
Diffstat (limited to 'src')
-rw-r--r--src/freedreno/ir3/ir3.c24
-rw-r--r--src/freedreno/ir3/ir3.h5
-rw-r--r--src/freedreno/ir3/ir3_array_to_ssa.c23
-rw-r--r--src/freedreno/ir3/ir3_context.c11
-rw-r--r--src/freedreno/ir3/ir3_cp.c6
-rw-r--r--src/freedreno/ir3/ir3_print.c7
-rw-r--r--src/freedreno/ir3/ir3_ra.c15
-rw-r--r--src/freedreno/ir3/ir3_ra.h10
-rw-r--r--src/freedreno/ir3/ir3_spill.c8
-rw-r--r--src/freedreno/ir3/ir3_validate.c6
10 files changed, 56 insertions, 59 deletions
diff --git a/src/freedreno/ir3/ir3.c b/src/freedreno/ir3/ir3.c
index 0a1df2a1dd8..fdc6184728f 100644
--- a/src/freedreno/ir3/ir3.c
+++ b/src/freedreno/ir3/ir3.c
@@ -387,8 +387,12 @@ unsigned ir3_block_get_pred_index(struct ir3_block *block, struct ir3_block *pre
unreachable("ir3_block_get_pred_index() invalid predecessor");
}
-static struct ir3_instruction *instr_create(struct ir3_block *block, int nreg)
+static struct ir3_instruction *instr_create(struct ir3_block *block,
+ opc_t opc, int nreg)
{
+ /* Add an extra source for array destinations */
+ if (1 <= opc_cat(opc))
+ nreg++;
struct ir3_instruction *instr;
unsigned sz = sizeof(*instr) + (nreg * sizeof(instr->regs[0]));
char *ptr = ir3_alloc(block->shader, sz);
@@ -407,7 +411,7 @@ static struct ir3_instruction *instr_create(struct ir3_block *block, int nreg)
struct ir3_instruction * ir3_instr_create(struct ir3_block *block,
opc_t opc, int nreg)
{
- struct ir3_instruction *instr = instr_create(block, nreg);
+ struct ir3_instruction *instr = instr_create(block, opc, nreg);
instr->block = block;
instr->opc = opc;
insert_instr(block, instr);
@@ -416,7 +420,7 @@ struct ir3_instruction * ir3_instr_create(struct ir3_block *block,
struct ir3_instruction * ir3_instr_clone(struct ir3_instruction *instr)
{
- struct ir3_instruction *new_instr = instr_create(instr->block,
+ struct ir3_instruction *new_instr = instr_create(instr->block, instr->opc,
instr->regs_count);
struct ir3_register **regs;
unsigned i;
@@ -470,6 +474,20 @@ struct ir3_register * ir3_reg_clone(struct ir3 *shader,
return new_reg;
}
+
+void ir3_reg_set_last_array(struct ir3_instruction *instr,
+ struct ir3_register *reg,
+ struct ir3_register *last_write)
+{
+ assert(reg->flags & IR3_REG_ARRAY);
+ assert(reg->flags & IR3_REG_DEST);
+ struct ir3_register *new_reg = ir3_reg_create(instr, 0, 0);
+ *new_reg = *reg;
+ new_reg->flags &= ~IR3_REG_DEST;
+ new_reg->def = last_write;
+ ir3_reg_tie(reg, new_reg);
+}
+
void
ir3_instr_set_address(struct ir3_instruction *instr,
struct ir3_instruction *addr)
diff --git a/src/freedreno/ir3/ir3.h b/src/freedreno/ir3/ir3.h
index 97c36f53bd8..487dd4b4b1a 100644
--- a/src/freedreno/ir3/ir3.h
+++ b/src/freedreno/ir3/ir3.h
@@ -611,10 +611,15 @@ struct ir3_register * ir3_reg_clone(struct ir3 *shader,
static inline void ir3_reg_tie(struct ir3_register *dst, struct ir3_register *src)
{
+ assert(!dst->tied && !src->tied);
dst->tied = src;
src->tied = dst;
}
+void ir3_reg_set_last_array(struct ir3_instruction *instr,
+ struct ir3_register *reg,
+ struct ir3_register *last_write);
+
void ir3_instr_set_address(struct ir3_instruction *instr,
struct ir3_instruction *addr);
diff --git a/src/freedreno/ir3/ir3_array_to_ssa.c b/src/freedreno/ir3/ir3_array_to_ssa.c
index 53ea2d96942..af0a86be3cb 100644
--- a/src/freedreno/ir3/ir3_array_to_ssa.c
+++ b/src/freedreno/ir3/ir3_array_to_ssa.c
@@ -240,8 +240,8 @@ ir3_array_to_ssa(struct ir3 *ir)
for (unsigned i = 0; i < instr->regs_count; i++) {
struct ir3_register *reg = instr->regs[i];
if ((reg->flags & IR3_REG_ARRAY) &&
- (!reg->def || reg->def->instr->block != block)) {
- reg->def = NULL;
+ (((reg->flags & IR3_REG_DEST) && !reg->tied) ||
+ (!(reg->flags & IR3_REG_DEST) && !reg->def))) {
struct ir3_array *arr = ir3_lookup_array(ir, reg->array.id);
/* Construct any phi nodes necessary to read this value */
@@ -276,15 +276,18 @@ ir3_array_to_ssa(struct ir3 *ir)
for (unsigned i = 0; i < instr->regs_count; i++) {
struct ir3_register *reg = instr->regs[i];
if ((reg->flags & IR3_REG_ARRAY)) {
- if (!reg->def) {
- /* It is assumed that before calling
- * ir3_array_to_ssa(), reg->def was set to the
- * previous writer of the array within the current
- * block (if any). The reg->def of the first write
- * to an array within a block was cleared in the
- * loop calling read_value_beginning() above.
- */
+ /* It is assumed that before calling
+ * ir3_array_to_ssa(), reg->def was set to the
+ * previous writer of the array within the current
+ * block or NULL if none.
+ */
+ if (!(reg->flags & IR3_REG_DEST) && !reg->def) {
reg->def = lookup_live_in(&ctx, block, reg->array.id);
+ } else if ((reg->flags & IR3_REG_DEST) && !reg->tied) {
+ struct ir3_register *def =
+ lookup_live_in(&ctx, block, reg->array.id);
+ if (def)
+ ir3_reg_set_last_array(instr, reg, def);
}
reg->flags |= IR3_REG_SSA;
}
diff --git a/src/freedreno/ir3/ir3_context.c b/src/freedreno/ir3/ir3_context.c
index b41293671c2..8fdaa239c4b 100644
--- a/src/freedreno/ir3/ir3_context.c
+++ b/src/freedreno/ir3/ir3_context.c
@@ -588,7 +588,8 @@ ir3_create_array_load(struct ir3_context *ctx, struct ir3_array *arr, int n,
__ssa_dst(mov)->flags |= flags;
src = ir3_reg_create(mov, 0, IR3_REG_ARRAY |
COND(address, IR3_REG_RELATIV) | flags);
- src->def = arr->last_write;
+ src->def = (arr->last_write && arr->last_write->instr->block == block) ?
+ arr->last_write : NULL;
src->size = arr->length;
src->array.id = arr->id;
src->array.offset = n;
@@ -623,12 +624,14 @@ ir3_create_array_store(struct ir3_context *ctx, struct ir3_array *arr, int n,
src->barrier_conflict |= IR3_BARRIER_ARRAY_R | IR3_BARRIER_ARRAY_W;
dst->flags |= IR3_REG_ARRAY;
- dst->def = arr->last_write;
dst->size = arr->length;
dst->array.id = arr->id;
dst->array.offset = n;
dst->array.base = INVALID_REG;
+ if (arr->last_write && arr->last_write->instr->block == src->block)
+ ir3_reg_set_last_array(src, dst, arr->last_write);
+
arr->last_write = dst;
array_insert(block, block->keeps, src);
@@ -650,7 +653,6 @@ ir3_create_array_store(struct ir3_context *ctx, struct ir3_array *arr, int n,
dst = ir3_reg_create(mov, 0, IR3_REG_DEST | IR3_REG_SSA | IR3_REG_ARRAY |
flags |
COND(address, IR3_REG_RELATIV));
- dst->def = arr->last_write;
dst->instr = mov;
dst->size = arr->length;
dst->array.id = arr->id;
@@ -658,6 +660,9 @@ ir3_create_array_store(struct ir3_context *ctx, struct ir3_array *arr, int n,
dst->array.base = INVALID_REG;
ir3_reg_create(mov, 0, IR3_REG_SSA | flags)->def = src->regs[0];
+ if (arr->last_write && arr->last_write->instr->block == block)
+ ir3_reg_set_last_array(mov, dst, arr->last_write);
+
if (address)
ir3_instr_set_address(mov, address);
diff --git a/src/freedreno/ir3/ir3_cp.c b/src/freedreno/ir3/ir3_cp.c
index fe0cc4695ca..bed9efe082f 100644
--- a/src/freedreno/ir3/ir3_cp.c
+++ b/src/freedreno/ir3/ir3_cp.c
@@ -531,12 +531,6 @@ instr_cp(struct ir3_cp_ctx *ctx, struct ir3_instruction *instr)
}
} while (progress);
- if (instr->regs[0]->flags & IR3_REG_ARRAY) {
- struct ir3_instruction *src = ssa(instr->regs[0]);
- if (src)
- instr_cp(ctx, src);
- }
-
if (instr->address) {
instr_cp(ctx, instr->address);
ir3_instr_set_address(instr, eliminate_output_mov(ctx, instr->address));
diff --git a/src/freedreno/ir3/ir3_print.c b/src/freedreno/ir3/ir3_print.c
index 9c6cb284acd..2ae241ba30f 100644
--- a/src/freedreno/ir3/ir3_print.c
+++ b/src/freedreno/ir3/ir3_print.c
@@ -222,13 +222,8 @@ static void print_reg_name(struct log_stream *stream, struct ir3_instruction *in
print_ssa_name(stream, reg, reg->flags & IR3_REG_DEST);
mesa_log_stream_printf(stream, ":");
}
- mesa_log_stream_printf(stream, SYN_ARRAY("arr[id=%u, offset=%d, size=%u"), reg->array.id,
+ mesa_log_stream_printf(stream, SYN_ARRAY("arr[id=%u, offset=%d, size=%u]"), reg->array.id,
reg->array.offset, reg->size);
- if (reg->flags & IR3_REG_DEST) {
- mesa_log_stream_printf(stream, SYN_ARRAY(", "));
- print_ssa_name(stream, reg, false);
- }
- mesa_log_stream_printf(stream, SYN_ARRAY("]"));
if (reg->array.base != INVALID_REG)
mesa_log_stream_printf(stream, "("SYN_REG("r%u.%c")")", reg->array.base >> 2,
"xyzw"[reg->array.base & 0x3]);
diff --git a/src/freedreno/ir3/ir3_ra.c b/src/freedreno/ir3/ir3_ra.c
index 34540fe2a58..eb53d65fb1b 100644
--- a/src/freedreno/ir3/ir3_ra.c
+++ b/src/freedreno/ir3/ir3_ra.c
@@ -1136,8 +1136,6 @@ assign_src(struct ra_ctx *ctx, struct ir3_instruction *instr, struct ir3_registe
struct ra_interval *interval = &ctx->intervals[src->def->name];
struct ra_file *file = ra_get_file(ctx, src);
- bool array_rmw = ra_reg_is_array_rmw(src);
-
struct ir3_register *tied = src->tied;
physreg_t physreg;
if (tied) {
@@ -1151,15 +1149,6 @@ assign_src(struct ra_ctx *ctx, struct ir3_instruction *instr, struct ir3_registe
if (src->flags & IR3_REG_FIRST_KILL)
ra_file_remove(file, interval);
-
- /* This source is also a destination. */
- if (array_rmw) {
- struct ra_interval *dst_interval = &ctx->intervals[src->name];
- ra_interval_init(dst_interval, src);
- dst_interval->physreg_start = physreg;
- dst_interval->physreg_end = physreg + src->size * reg_elem_size(src);
- ra_file_insert(file, dst_interval);
- }
}
/* Insert a parallel copy instruction before the instruction with the parallel
@@ -1209,8 +1198,6 @@ handle_normal_instr(struct ra_ctx *ctx, struct ir3_instruction *instr)
/* Allocate the destination. */
ra_foreach_dst(dst, instr) {
- if (ra_reg_is_array_rmw(dst))
- continue;
allocate_dst(ctx, dst);
}
@@ -1224,8 +1211,6 @@ handle_normal_instr(struct ra_ctx *ctx, struct ir3_instruction *instr)
/* Now finally insert the destination into the map. */
ra_foreach_dst(dst, instr) {
- if (ra_reg_is_array_rmw(dst))
- continue;
insert_dst(ctx, dst);
}
diff --git a/src/freedreno/ir3/ir3_ra.h b/src/freedreno/ir3/ir3_ra.h
index a3404df052c..7efb6584591 100644
--- a/src/freedreno/ir3/ir3_ra.h
+++ b/src/freedreno/ir3/ir3_ra.h
@@ -91,16 +91,6 @@ ra_reg_is_src(const struct ir3_register *reg)
def_is_gpr(reg->def);
}
-/* Array destinations can act as a source, reading the previous array and then
- * modifying it. Return true when the register is an array destination that
- * acts like a source.
- */
-static inline bool
-ra_reg_is_array_rmw(const struct ir3_register *reg)
-{
- return ((reg->flags & IR3_REG_ARRAY) && (reg->flags & IR3_REG_DEST) && reg->def);
-}
-
static inline bool
ra_reg_is_dst(const struct ir3_register *reg)
{
diff --git a/src/freedreno/ir3/ir3_spill.c b/src/freedreno/ir3/ir3_spill.c
index 94bd85e9b9e..1205dcc5b4b 100644
--- a/src/freedreno/ir3/ir3_spill.c
+++ b/src/freedreno/ir3/ir3_spill.c
@@ -250,11 +250,9 @@ handle_instr(struct ra_spill_ctx *ctx, struct ir3_instruction *instr)
*/
ra_foreach_dst(dst, instr) {
- if (!ra_reg_is_array_rmw(dst)) {
- struct ir3_register *tied_src = dst->tied;
- if (tied_src && !(tied_src->flags & IR3_REG_FIRST_KILL))
- insert_dst(ctx, dst);
- }
+ struct ir3_register *tied_src = dst->tied;
+ if (tied_src && !(tied_src->flags & IR3_REG_FIRST_KILL))
+ insert_dst(ctx, dst);
}
update_max_pressure(ctx);
diff --git a/src/freedreno/ir3/ir3_validate.c b/src/freedreno/ir3/ir3_validate.c
index 779edc2d563..68e931dd619 100644
--- a/src/freedreno/ir3/ir3_validate.c
+++ b/src/freedreno/ir3/ir3_validate.c
@@ -150,7 +150,11 @@ validate_instr(struct ir3_validate_ctx *ctx, struct ir3_instruction *instr)
* bindless, irrespective of the precision of other srcs. The
* tex/samp src is the first src reg when .s2en is set
*/
- if ((instr->flags & IR3_INSTR_S2EN) && (n < 2)) {
+ if (reg->tied) {
+ /* must have the same size as the destination, handled in
+ * validate_reg().
+ */
+ } else if ((instr->flags & IR3_INSTR_S2EN) && (n < 2)) {
if (n == 0) {
if (instr->flags & IR3_INSTR_B)
validate_assert(ctx, !(reg->flags & IR3_REG_HALF));