ir3: add terminators to blocks

Instead of using brtype and condition fields in blocks, explicitly add
terminator branches in their instruction lists. This makes it more
uniform to deal with branches in passes. This will be especially useful
for passes that need to deal with predicate registers like the new
register allocator.

Note that only a single terminator is added to blocks: either an
unconditional jump in case of a single successor, or a conditional
branch in case of two successors. In the latter case, the unconditional
jump to the second successor is implicit. This makes it slightly easier
to handle terminators since there is always exactly one.

Signed-off-by: Job Noorman <jnoorman@igalia.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/27411>

13 files changed, 284 insertions, 155 deletions

diff --git a/src/freedreno/ir3/disasm-a3xx.c b/src/freedreno/ir3/disasm-a3xx.c
index 10fbbd8fdc6..6a48d503cca 100644
--- a/src/freedreno/ir3/disasm-a3xx.c
+++ b/src/freedreno/ir3/disasm-a3xx.c
@@ -154,6 +154,11 @@ static const struct opc_info {
    /* category 0: */
    OPC(0, OPC_NOP,          nop),
    OPC(0, OPC_B,            b),
+   OPC(0, OPC_BR,           br),
+   OPC(0, OPC_BRAA,         braa),
+   OPC(0, OPC_BRAO,         brao),
+   OPC(0, OPC_BALL,         ball),
+   OPC(0, OPC_BANY,         bany),
    OPC(0, OPC_JUMP,         jump),
    OPC(0, OPC_CALL,         call),
    OPC(0, OPC_RET,          ret),
diff --git a/src/freedreno/ir3/ir3.c b/src/freedreno/ir3/ir3.c
index 5b9b51399fe..862703718b9 100644
--- a/src/freedreno/ir3/ir3.c
+++ b/src/freedreno/ir3/ir3.c
@@ -483,14 +483,19 @@ reg_create(struct ir3 *shader, int num, int flags)
 }
 
 static void
-insert_instr(struct ir3_block *block, struct ir3_instruction *instr)
+insert_instr(struct ir3_block *block, struct ir3_instruction *instr,
+             bool at_end)
 {
    struct ir3 *shader = block->shader;
 
    instr->serialno = ++shader->instr_count;
 
+   struct ir3_instruction *terminator = ir3_block_get_terminator(block);
    list_addtail(&instr->node, &block->instr_list);
 
+   if (!at_end && terminator)
+      ir3_instr_move_before(instr, terminator);
+
    if (is_input(instr))
       array_insert(shader, shader->baryfs, instr);
 }
@@ -508,6 +513,53 @@ ir3_block_create(struct ir3 *shader)
    return block;
 }
 
+static struct ir3_instruction *
+block_get_last_instruction(struct ir3_block *block)
+{
+   if (list_is_empty(&block->instr_list))
+      return NULL;
+   return list_last_entry(&block->instr_list, struct ir3_instruction, node);
+}
+
+struct ir3_instruction *
+ir3_block_get_terminator(struct ir3_block *block)
+{
+   struct ir3_instruction *last = block_get_last_instruction(block);
+
+   if (last && is_terminator(last))
+      return last;
+
+   return NULL;
+}
+
+struct ir3_instruction *
+ir3_block_take_terminator(struct ir3_block *block)
+{
+   struct ir3_instruction *terminator = ir3_block_get_terminator(block);
+
+   if (terminator)
+      list_delinit(&terminator->node);
+
+   return terminator;
+}
+
+struct ir3_instruction *
+ir3_block_get_last_non_terminator(struct ir3_block *block)
+{
+   struct ir3_instruction *last = block_get_last_instruction(block);
+
+   if (!last)
+      return NULL;
+
+   if (!is_terminator(last))
+      return last;
+
+   if (last->node.prev != &block->instr_list)
+      return list_entry(last->node.prev, struct ir3_instruction, node);
+
+   return NULL;
+}
+
 void
 ir3_block_add_predecessor(struct ir3_block *block, struct ir3_block *pred)
 {
@@ -574,13 +626,14 @@ instr_create(struct ir3_block *block, opc_t opc, int ndst, int nsrc)
    return instr;
 }
 
-struct ir3_instruction *
-ir3_instr_create(struct ir3_block *block, opc_t opc, int ndst, int nsrc)
+static struct ir3_instruction *
+instr_create_impl(struct ir3_block *block, opc_t opc, int ndst, int nsrc,
+                  bool at_end)
 {
    struct ir3_instruction *instr = instr_create(block, opc, ndst, nsrc);
    instr->block = block;
    instr->opc = opc;
-   insert_instr(block, instr);
+   insert_instr(block, instr, at_end);
    return instr;
 }
 
@@ -602,6 +655,18 @@ add_to_address_users(struct ir3_instruction *instr)
 }
 
 struct ir3_instruction *
+ir3_instr_create(struct ir3_block *block, opc_t opc, int ndst, int nsrc)
+{
+   return instr_create_impl(block, opc, ndst, nsrc, false);
+}
+
+struct ir3_instruction *
+ir3_instr_create_at_end(struct ir3_block *block, opc_t opc, int ndst, int nsrc)
+{
+   return instr_create_impl(block, opc, ndst, nsrc, true);
+}
+
+struct ir3_instruction *
 ir3_instr_clone(struct ir3_instruction *instr)
 {
    struct ir3_instruction *new_instr = instr_create(
@@ -614,7 +679,7 @@ ir3_instr_clone(struct ir3_instruction *instr)
    new_instr->dsts = dsts;
    new_instr->srcs = srcs;
 
-   insert_instr(instr->block, new_instr);
+   insert_instr(instr->block, new_instr, false);
 
    /* clone registers: */
    new_instr->dsts_count = 0;
@@ -741,6 +806,21 @@ ir3_count_instructions(struct ir3 *ir)
    return cnt;
 }
 
+unsigned
+ir3_count_instructions_sched(struct ir3 *ir)
+{
+   unsigned cnt = 1;
+   foreach_block (block, &ir->block_list) {
+      block->start_ip = cnt;
+      foreach_instr (instr, &block->instr_list) {
+         if (!is_terminator(instr))
+            instr->ip = cnt++;
+      }
+      block->end_ip = cnt;
+   }
+   return cnt;
+}
+
 /* When counting instructions for RA, we insert extra fake instructions at the
  * beginning of each block, where values become live, and at the end where
  * values die. This prevents problems where values live-in at the beginning or
diff --git a/src/freedreno/ir3/ir3.h b/src/freedreno/ir3/ir3.h
index 45d28d007bf..0c99f068abc 100644
--- a/src/freedreno/ir3/ir3.h
+++ b/src/freedreno/ir3/ir3.h
@@ -618,15 +618,6 @@ struct ir3_array {
 
 struct ir3_array *ir3_lookup_array(struct ir3 *ir, unsigned id);
 
-enum ir3_branch_type {
-   IR3_BRANCH_COND,    /* condition */
-   IR3_BRANCH_ANY,     /* subgroupAny(condition) */
-   IR3_BRANCH_ALL,     /* subgroupAll(condition) */
-   IR3_BRANCH_GETONE,  /* subgroupElect() */
-   IR3_BRANCH_GETLAST, /* getlast.w8 */
-   IR3_BRANCH_SHPS,    /* preamble start */
-};
-
 struct ir3_block {
    struct list_head node;
    struct ir3 *shader;
@@ -635,9 +626,6 @@ struct ir3_block {
 
    struct list_head instr_list; /* list of ir3_instruction */
 
-   /* The actual branch condition, if there are two successors */
-   enum ir3_branch_type brtype;
-
    /* each block has either one or two successors.. in case of two
     * successors, 'condition' decides which one to follow.  A block preceding
     * an if/else has two successors.
@@ -652,7 +640,6 @@ struct ir3_block {
     * physical successors which includes the fallthrough edge from the if to
     * the else.
     */
-   struct ir3_instruction *condition;
    struct ir3_block *successors[2];
 
    DECLARE_ARRAY(struct ir3_block *, predecessors);
@@ -711,6 +698,13 @@ ir3_end_block(struct ir3 *ir)
    return list_last_entry(&ir->block_list, struct ir3_block, node);
 }
 
+struct ir3_instruction *ir3_block_get_terminator(struct ir3_block *block);
+
+struct ir3_instruction *ir3_block_take_terminator(struct ir3_block *block);
+
+struct ir3_instruction *
+ir3_block_get_last_non_terminator(struct ir3_block *block);
+
 static inline struct ir3_block *
 ir3_after_preamble(struct ir3 *ir)
 {
@@ -718,7 +712,8 @@ ir3_after_preamble(struct ir3 *ir)
    /* The preamble will have a usually-empty else branch, and we want to skip
     * that to get to the block after the preamble.
     */
-   if (block->brtype == IR3_BRANCH_SHPS)
+   struct ir3_instruction *terminator = ir3_block_get_terminator(block);
+   if (terminator && (terminator->opc == OPC_SHPS))
       return block->successors[1]->successors[0];
    else
       return block;
@@ -757,6 +752,8 @@ struct ir3_block *ir3_block_create(struct ir3 *shader);
 
 struct ir3_instruction *ir3_instr_create(struct ir3_block *block, opc_t opc,
                                          int ndst, int nsrc);
+struct ir3_instruction *ir3_instr_create_at_end(struct ir3_block *block,
+                                                opc_t opc, int ndst, int nsrc);
 struct ir3_instruction *ir3_instr_clone(struct ir3_instruction *instr);
 void ir3_instr_add_dep(struct ir3_instruction *instr,
                        struct ir3_instruction *dep);
@@ -797,6 +794,7 @@ void ir3_block_clear_mark(struct ir3_block *block);
 void ir3_clear_mark(struct ir3 *shader);
 
 unsigned ir3_count_instructions(struct ir3 *ir);
+unsigned ir3_count_instructions_sched(struct ir3 *ir);
 unsigned ir3_count_instructions_ra(struct ir3 *ir);
 
 /**
@@ -869,6 +867,26 @@ is_flow(struct ir3_instruction *instr)
 }
 
 static inline bool
+is_terminator(struct ir3_instruction *instr)
+{
+   switch (instr->opc) {
+   case OPC_B:
+   case OPC_BR:
+   case OPC_JUMP:
+   case OPC_BANY:
+   case OPC_BALL:
+   case OPC_BRAA:
+   case OPC_BRAO:
+   case OPC_SHPS:
+   case OPC_GETONE:
+   case OPC_GETLAST:
+      return true;
+   default:
+      return false;
+   }
+}
+
+static inline bool
 is_kill_or_demote(struct ir3_instruction *instr)
 {
    return instr->opc == OPC_KILL || instr->opc == OPC_DEMOTE;
@@ -2319,6 +2337,11 @@ static inline struct ir3_instruction *ir3_##name(                              \
 
 /* cat0 instructions: */
 INSTR1NODST(B)
+INSTR1NODST(BR)
+INSTR1NODST(BALL)
+INSTR1NODST(BANY)
+INSTR2NODST(BRAA)
+INSTR2NODST(BRAO)
 INSTR0(JUMP)
 INSTR1NODST(KILL)
 INSTR1NODST(DEMOTE)
diff --git a/src/freedreno/ir3/ir3_compiler_nir.c b/src/freedreno/ir3/ir3_compiler_nir.c
index e97ec561f37..1c5d253e433 100644
--- a/src/freedreno/ir3/ir3_compiler_nir.c
+++ b/src/freedreno/ir3/ir3_compiler_nir.c
@@ -3802,6 +3802,12 @@ emit_block(struct ir3_context *ctx, nir_block *nblock)
       }
    }
 
+   /* Emit unconditional branch if we only have one successor. Conditional
+    * branches are emitted in emit_if.
+    */
+   if (ctx->block->successors[0] && !ctx->block->successors[1])
+      ir3_JUMP(ctx->block);
+
    _mesa_hash_table_clear(ctx->sel_cond_conversions, NULL);
 }
 
@@ -3813,27 +3819,25 @@ emit_if(struct ir3_context *ctx, nir_if *nif)
    struct ir3_instruction *condition = ir3_get_src(ctx, &nif->condition)[0];
 
    if (condition->opc == OPC_ANY_MACRO && condition->block == ctx->block) {
-      ctx->block->condition = ssa(condition->srcs[0]);
-      ctx->block->brtype = IR3_BRANCH_ANY;
+      struct ir3_instruction *pred = ssa(condition->srcs[0]);
+      ir3_BANY(ctx->block, pred, 0);
    } else if (condition->opc == OPC_ALL_MACRO &&
               condition->block == ctx->block) {
-      ctx->block->condition = ssa(condition->srcs[0]);
-      ctx->block->brtype = IR3_BRANCH_ALL;
+      struct ir3_instruction *pred = ssa(condition->srcs[0]);
+      ir3_BALL(ctx->block, pred, 0);
    } else if (condition->opc == OPC_ELECT_MACRO &&
               condition->block == ctx->block) {
-      ctx->block->condition = NULL;
-      ctx->block->brtype = IR3_BRANCH_GETONE;
+      ir3_GETONE(ctx->block);
    } else if (condition->opc == OPC_SHPS_MACRO &&
               condition->block == ctx->block) {
       /* TODO: technically this only works if the block is the only user of the
        * shps, but we only use it in very constrained scenarios so this should
        * be ok.
        */
-      ctx->block->condition = NULL;
-      ctx->block->brtype = IR3_BRANCH_SHPS;
+      ir3_SHPS(ctx->block);
    } else {
-      ctx->block->condition = ir3_get_predicate(ctx, condition);
-      ctx->block->brtype = IR3_BRANCH_COND;
+      struct ir3_instruction *pred = ir3_get_predicate(ctx, condition);
+      ir3_BR(ctx->block, pred, 0);
    }
 
    emit_cf_list(ctx, &nif->then_list);
@@ -3864,6 +3868,7 @@ emit_loop(struct ir3_context *ctx, nir_loop *nloop)
 
    if (continue_blk) {
       struct ir3_block *start = get_block(ctx, nstart);
+      ir3_JUMP(continue_blk);
       continue_blk->successors[0] = start;
       continue_blk->loop_id = ctx->loop_id;
       continue_blk->loop_depth = ctx->loop_depth;
@@ -3962,7 +3967,7 @@ emit_stream_out(struct ir3_context *ctx)
     * since it is used to pick which of the two successor
     * paths to take:
     */
-   orig_end_block->condition = cond;
+   ir3_BR(orig_end_block, cond, 0);
 
    /* switch to stream_out_block to generate the stream-out
     * instructions:
@@ -4006,6 +4011,8 @@ emit_stream_out(struct ir3_context *ctx)
       }
    }
 
+   ir3_JUMP(ctx->block);
+
    /* and finally switch to the new_end_block: */
    ctx->block = new_end_block;
 }
diff --git a/src/freedreno/ir3/ir3_cp.c b/src/freedreno/ir3/ir3_cp.c
index d4165bb3461..6a872dc904b 100644
--- a/src/freedreno/ir3/ir3_cp.c
+++ b/src/freedreno/ir3/ir3_cp.c
@@ -704,10 +704,9 @@ ir3_cp(struct ir3 *ir, struct ir3_shader_variant *so)
    ir3_clear_mark(ir);
 
    foreach_block (block, &ir->block_list) {
-      if (block->condition) {
-         instr_cp(&ctx, block->condition);
-         block->condition = eliminate_output_mov(&ctx, block->condition);
-      }
+      struct ir3_instruction *terminator = ir3_block_get_terminator(block);
+      if (terminator)
+         instr_cp(&ctx, terminator);
 
       for (unsigned i = 0; i < block->keeps_count; i++) {
          instr_cp(&ctx, block->keeps[i]);
diff --git a/src/freedreno/ir3/ir3_dce.c b/src/freedreno/ir3/ir3_dce.c
index 66a4c4c48ab..c640f6e5c35 100644
--- a/src/freedreno/ir3/ir3_dce.c
+++ b/src/freedreno/ir3/ir3_dce.c
@@ -130,8 +130,16 @@ find_and_remove_unused(struct ir3 *ir, struct ir3_shader_variant *so)
          instr_dce(block->keeps[i], false);
 
       /* We also need to account for if-condition: */
-      if (block->condition)
-         instr_dce(block->condition, false);
+      struct ir3_instruction *terminator = ir3_block_get_terminator(block);
+      if (terminator) {
+         instr_dce(terminator, false);
+
+         /* Temporary workaround for predicates not being SSA. Won't be
+          * necessary anymore once we have RA for predicates.
+          */
+         foreach_src (src, terminator)
+            instr_dce(src->def->instr, false);
+      }
    }
 
    /* remove un-used instructions: */
diff --git a/src/freedreno/ir3/ir3_legalize.c b/src/freedreno/ir3/ir3_legalize.c
index f5f4fc6259a..10bde848273 100644
--- a/src/freedreno/ir3/ir3_legalize.c
+++ b/src/freedreno/ir3/ir3_legalize.c
@@ -698,70 +698,73 @@ static void
 block_sched(struct ir3 *ir)
 {
    foreach_block (block, &ir->block_list) {
+      struct ir3_instruction *terminator = ir3_block_get_terminator(block);
+
       if (block->successors[1]) {
          /* if/else, conditional branches to "then" or "else": */
          struct ir3_instruction *br1, *br2;
 
-         if (block->brtype == IR3_BRANCH_GETONE ||
-             block->brtype == IR3_BRANCH_GETLAST ||
-             block->brtype == IR3_BRANCH_SHPS) {
+         assert(terminator);
+         unsigned opc = terminator->opc;
+
+         if (opc == OPC_GETONE || opc == OPC_SHPS || opc == OPC_GETLAST) {
             /* getone/shps can't be inverted, and it wouldn't even make sense
              * to follow it with an inverted branch, so follow it by an
              * unconditional branch.
              */
-            assert(!block->condition);
-            if (block->brtype == IR3_BRANCH_GETONE)
-               br1 = ir3_GETONE(block);
-            else if (block->brtype == IR3_BRANCH_GETLAST)
-               br1 = ir3_GETLAST(block);
-            else
-               br1 = ir3_SHPS(block);
+            assert(terminator->srcs_count == 0);
+            br1 = terminator;
             br1->cat0.target = block->successors[1];
 
             br2 = ir3_JUMP(block);
             br2->cat0.target = block->successors[0];
          } else {
-            assert(block->condition);
+            assert(terminator->srcs_count == 1);
 
             /* create "else" branch first (since "then" block should
              * frequently/always end up being a fall-thru):
              */
-            br1 = ir3_instr_create(block, OPC_B, 0, 1);
-            ir3_src_create(br1, regid(REG_P0, 0), 0)->def =
-               block->condition->dsts[0];
+            br1 = terminator;
+            br1->opc = OPC_B;
+            br1->srcs[0]->num = regid(REG_P0, 0);
             br1->cat0.inv1 = true;
             br1->cat0.target = block->successors[1];
 
             /* "then" branch: */
             br2 = ir3_instr_create(block, OPC_B, 0, 1);
             ir3_src_create(br2, regid(REG_P0, 0), 0)->def =
-               block->condition->dsts[0];
+               terminator->srcs[0]->def;
             br2->cat0.target = block->successors[0];
 
-            switch (block->brtype) {
-            case IR3_BRANCH_COND:
+            switch (opc) {
+            case OPC_B:
+            case OPC_BR:
                br1->cat0.brtype = br2->cat0.brtype = BRANCH_PLAIN;
                break;
-            case IR3_BRANCH_ALL:
+            case OPC_BALL:
                br1->cat0.brtype = BRANCH_ANY;
                br2->cat0.brtype = BRANCH_ALL;
                break;
-            case IR3_BRANCH_ANY:
+            case OPC_BANY:
                br1->cat0.brtype = BRANCH_ALL;
                br2->cat0.brtype = BRANCH_ANY;
                break;
-            case IR3_BRANCH_GETONE:
-            case IR3_BRANCH_GETLAST:
-            case IR3_BRANCH_SHPS:
+            default:
                unreachable("can't get here");
             }
          }
-      } else if (block->successors[0]) {
-         /* otherwise unconditional jump to next block: */
-         struct ir3_instruction *jmp;
 
-         jmp = ir3_JUMP(block);
-         jmp->cat0.target = block->successors[0];
+         /* Creating br2 caused it to be moved before the terminator b1, move it
+          * back.
+          */
+         ir3_instr_move_after(br2, br1);
+      } else if (block->successors[0]) {
+         /* otherwise unconditional jump to next block which should already have
+          * been inserted.
+          */
+         assert(terminator);
+         assert(terminator->opc == OPC_JUMP);
+         terminator->cat0.target = block->successors[0];
       }
    }
 }
@@ -957,11 +960,13 @@ helper_sched(struct ir3_legalize_ctx *ctx, struct ir3 *ir,
          }
       }
 
-      if (block->brtype == IR3_BRANCH_ALL ||
-          block->brtype == IR3_BRANCH_ANY ||
-          block->brtype == IR3_BRANCH_GETONE) {
-         bd->uses_helpers_beginning = true;
-         bd->uses_helpers_end = true;
+      struct ir3_instruction *terminator = ir3_block_get_terminator(block);
+      if (terminator) {
+         if (terminator->opc == OPC_BALL || terminator->opc == OPC_BANY ||
+             terminator->opc == OPC_GETONE) {
+            bd->uses_helpers_beginning = true;
+            bd->uses_helpers_end = true;
+         }
       }
 
       block->data = bd;
@@ -1175,6 +1180,14 @@ ir3_legalize(struct ir3 *ir, struct ir3_shader_variant *so, int *max_bary)
 
    *max_bary = ctx->max_bary;
 
+   foreach_block (block, &ir->block_list) {
+      struct ir3_instruction *terminator = ir3_block_get_terminator(block);
+      if (terminator && terminator->opc == OPC_GETONE) {
+         apply_push_consts_load_macro(ctx, block->successors[0]);
+         break;
+      }
+   }
+
    block_sched(ir);
    if (so->type == MESA_SHADER_FRAGMENT)
       kill_sched(ir, so);
@@ -1183,13 +1196,6 @@ ir3_legalize(struct ir3 *ir, struct ir3_shader_variant *so, int *max_bary)
       progress |= apply_fine_deriv_macro(ctx, block);
    }
 
-   foreach_block (block, &ir->block_list) {
-      if (block->brtype == IR3_BRANCH_GETONE) {
-         apply_push_consts_load_macro(ctx, block->successors[0]);
-         break;
-      }
-   }
-
    nop_sched(ir, so);
 
    if (ir3_shader_debug & IR3_DBG_FULLSYNC) {
diff --git a/src/freedreno/ir3/ir3_lower_subgroups.c b/src/freedreno/ir3/ir3_lower_subgroups.c
index d95d7fcb7a8..57680e9c646 100644
--- a/src/freedreno/ir3/ir3_lower_subgroups.c
+++ b/src/freedreno/ir3/ir3_lower_subgroups.c
@@ -203,9 +203,6 @@ split_block(struct ir3 *ir, struct ir3_block *before_block,
       rem_instr->block = after_block;
    }
 
-   after_block->brtype = before_block->brtype;
-   after_block->condition = before_block->condition;
-
    return after_block;
 }
 
@@ -217,16 +214,42 @@ link_blocks(struct ir3_block *pred, struct ir3_block *succ, unsigned index)
    ir3_block_link_physical(pred, succ);
 }
 
+static void
+link_blocks_jump(struct ir3_block *pred, struct ir3_block *succ)
+{
+   ir3_JUMP(pred);
+   link_blocks(pred, succ, 0);
+}
+
+static void
+link_blocks_branch(struct ir3_block *pred, struct ir3_block *target,
+                   struct ir3_block *fallthrough, unsigned opc,
+                   struct ir3_instruction *condition)
+{
+   unsigned nsrc = condition ? 1 : 0;
+   struct ir3_instruction *branch = ir3_instr_create(pred, opc, 0, nsrc);
+
+   if (condition) {
+      struct ir3_register *cond_dst = condition->dsts[0];
+      struct ir3_register *src =
+         ir3_src_create(branch, cond_dst->num, cond_dst->flags);
+      src->def = cond_dst;
+   }
+
+   link_blocks(pred, target, 0);
+   link_blocks(pred, fallthrough, 1);
+}
+
 static struct ir3_block *
 create_if(struct ir3 *ir, struct ir3_block *before_block,
-          struct ir3_block *after_block)
+          struct ir3_block *after_block, unsigned opc,
+          struct ir3_instruction *condition)
 {
    struct ir3_block *then_block = ir3_block_create(ir);
    list_add(&then_block->node, &before_block->node);
 
-   link_blocks(before_block, then_block, 0);
-   link_blocks(before_block, after_block, 1);
-   link_blocks(then_block, after_block, 0);
+   link_blocks_branch(before_block, then_block, after_block, opc, condition);
+   link_blocks_jump(then_block, after_block);
 
    return then_block;
 }
@@ -296,16 +319,14 @@ lower_instr(struct ir3 *ir, struct ir3_block **block, struct ir3_instruction *in
 
       after_block->reconvergence_point = true;
 
-      link_blocks(before_block, header, 0);
+      link_blocks_jump(before_block, header);
 
-      link_blocks(header, exit, 0);
-      link_blocks(header, footer, 1);
-      header->brtype = IR3_BRANCH_GETONE;
+      link_blocks_branch(header, exit, footer, OPC_GETONE, NULL);
 
-      link_blocks(exit, after_block, 0);
+      link_blocks_jump(exit, after_block);
       ir3_block_link_physical(exit, footer);
 
-      link_blocks(footer, header, 0);
+      link_blocks_jump(footer, header);
 
       struct ir3_register *exclusive = instr->dsts[0];
       struct ir3_register *inclusive = instr->dsts[1];
@@ -347,15 +368,11 @@ lower_instr(struct ir3 *ir, struct ir3_block **block, struct ir3_instruction *in
       body->reconvergence_point = true;
       after_block->reconvergence_point = true;
 
-      link_blocks(before_block, body, 0);
+      link_blocks_jump(before_block, body);
 
-      link_blocks(body, store, 0);
-      link_blocks(body, after_block, 1);
-      body->brtype = IR3_BRANCH_GETLAST;
+      link_blocks_branch(body, store, after_block, OPC_GETLAST, NULL);
 
-      link_blocks(store, after_block, 0);
-      link_blocks(store, body, 1);
-      store->brtype = IR3_BRANCH_GETONE;
+      link_blocks_branch(store, after_block, body, OPC_GETONE, NULL);
 
       struct ir3_register *reduce = instr->dsts[0];
       struct ir3_register *inclusive = instr->dsts[1];
@@ -392,56 +409,58 @@ lower_instr(struct ir3 *ir, struct ir3_block **block, struct ir3_instruction *in
 
       mov_reg(store, reduce, inclusive);
    } else {
-      struct ir3_block *then_block = create_if(ir, before_block, after_block);
-
       /* For ballot, the destination must be initialized to 0 before we do
        * the movmsk because the condition may be 0 and then the movmsk will
        * be skipped. Because it's a shared register we have to wrap the
        * initialization in a getone block.
        */
       if (instr->opc == OPC_BALLOT_MACRO) {
-         before_block->brtype = IR3_BRANCH_GETONE;
-         before_block->condition = NULL;
+         struct ir3_block *then_block =
+            create_if(ir, before_block, after_block, OPC_GETONE, NULL);
          mov_immed(instr->dsts[0], then_block, 0);
          after_block->reconvergence_point = true;
          before_block = after_block;
          after_block = split_block(ir, before_block, instr);
-         then_block = create_if(ir, before_block, after_block);
       }
 
+      struct ir3_instruction *condition = NULL;
+      unsigned branch_opc = 0;
+
       switch (instr->opc) {
       case OPC_BALLOT_MACRO:
       case OPC_READ_COND_MACRO:
       case OPC_ANY_MACRO:
       case OPC_ALL_MACRO:
-         before_block->condition = instr->srcs[0]->def->instr;
+         condition = instr->srcs[0]->def->instr;
          break;
       default:
-         before_block->condition = NULL;
          break;
       }
 
       switch (instr->opc) {
       case OPC_BALLOT_MACRO:
       case OPC_READ_COND_MACRO:
-         before_block->brtype = IR3_BRANCH_COND;
          after_block->reconvergence_point = true;
+         branch_opc = OPC_B;
          break;
       case OPC_ANY_MACRO:
-         before_block->brtype = IR3_BRANCH_ANY;
+         branch_opc = OPC_BANY;
          break;
       case OPC_ALL_MACRO:
-         before_block->brtype = IR3_BRANCH_ALL;
+         branch_opc = OPC_BALL;
          break;
       case OPC_ELECT_MACRO:
       case OPC_SWZ_SHARED_MACRO:
-         before_block->brtype = IR3_BRANCH_GETONE;
          after_block->reconvergence_point = true;
+         branch_opc = OPC_GETONE;
          break;
       default:
          unreachable("bad opcode");
       }
 
+      struct ir3_block *then_block =
+         create_if(ir, before_block, after_block, branch_opc, condition);
+
       switch (instr->opc) {
       case OPC_ALL_MACRO:
       case OPC_ANY_MACRO:
diff --git a/src/freedreno/ir3/ir3_parser.y b/src/freedreno/ir3/ir3_parser.y
index 264d371edad..4a09bc90a37 100644
--- a/src/freedreno/ir3/ir3_parser.y
+++ b/src/freedreno/ir3/ir3_parser.y
@@ -102,7 +102,7 @@ static void new_label(const char *name)
 
 static struct ir3_instruction * new_instr(opc_t opc)
 {
-	instr = ir3_instr_create(block, opc, 4, 6);
+	instr = ir3_instr_create_at_end(block, opc, 4, 6);
 	instr->flags = iflags.flags;
 	instr->repeat = iflags.repeat;
 	instr->nop = iflags.nop;
diff --git a/src/freedreno/ir3/ir3_postsched.c b/src/freedreno/ir3/ir3_postsched.c
index 29bf3a64847..4213af11331 100644
--- a/src/freedreno/ir3/ir3_postsched.c
+++ b/src/freedreno/ir3/ir3_postsched.c
@@ -650,6 +650,12 @@ sched_block(struct ir3_postsched_ctx *ctx, struct ir3_block *block)
    ctx->sy_delay = 0;
    ctx->ss_delay = 0;
 
+   /* The terminator has to stay at the end. Instead of trying to set up
+    * dependencies to achieve this, it's easier to just remove it now and add it
+    * back after scheduling.
+    */
+   struct ir3_instruction *terminator = ir3_block_take_terminator(block);
+
    /* move all instructions to the unscheduled list, and
     * empty the block's instruction list (to which we will
     * be inserting).
@@ -707,6 +713,9 @@ sched_block(struct ir3_postsched_ctx *ctx, struct ir3_block *block)
    }
 
    sched_dag_destroy(ctx);
+
+   if (terminator)
+      list_addtail(&terminator->node, &block->instr_list);
 }
 
 static bool
diff --git a/src/freedreno/ir3/ir3_print.c b/src/freedreno/ir3/ir3_print.c
index d3a86bc64a5..ee6ce376a01 100644
--- a/src/freedreno/ir3/ir3_print.c
+++ b/src/freedreno/ir3/ir3_print.c
@@ -533,39 +533,15 @@ print_block(struct ir3_block *block, int lvl)
    tab(stream, lvl + 1);
    mesa_log_stream_printf(stream, " */\n");
 
-   if (block->successors[1]) {
-      /* leading into if/else: */
+   if (block->successors[0]) {
       tab(stream, lvl + 1);
-      mesa_log_stream_printf(stream, "/* succs: if ");
-      switch (block->brtype) {
-      case IR3_BRANCH_COND:
-         break;
-      case IR3_BRANCH_ANY:
-         mesa_log_stream_printf(stream, "any ");
-         break;
-      case IR3_BRANCH_ALL:
-         mesa_log_stream_printf(stream, "all ");
-         break;
-      case IR3_BRANCH_GETONE:
-         mesa_log_stream_printf(stream, "getone ");
-         break;
-      case IR3_BRANCH_GETLAST:
-         mesa_log_stream_printf(stream, "getlast ");
-         break;
-      case IR3_BRANCH_SHPS:
-         mesa_log_stream_printf(stream, "shps ");
-         break;
-      }
-      if (block->condition)
-         mesa_log_stream_printf(stream, SYN_SSA("ssa_%u") " ",
-                                block->condition->serialno);
-      mesa_log_stream_printf(stream, "block%u; else block%u; */\n",
-                             block_id(block->successors[0]),
-                             block_id(block->successors[1]));
-   } else if (block->successors[0]) {
-      tab(stream, lvl + 1);
-      mesa_log_stream_printf(stream, "/* succs: block%u; */\n",
+      mesa_log_stream_printf(stream, "/* succs: block%u",
                              block_id(block->successors[0]));
+      if (block->successors[1]) {
+         mesa_log_stream_printf(stream, ", block%u",
+                                block_id(block->successors[1]));
+      }
+      mesa_log_stream_printf(stream, " */\n");
    }
    if (block->physical_successors_count > 0) {
       tab(stream, lvl + 1);
diff --git a/src/freedreno/ir3/ir3_ra.c b/src/freedreno/ir3/ir3_ra.c
index 89432f78cb8..ddc3da4b76b 100644
--- a/src/freedreno/ir3/ir3_ra.c
+++ b/src/freedreno/ir3/ir3_ra.c
@@ -2070,12 +2070,10 @@ insert_liveout_copy(struct ir3_block *block, physreg_t dst, physreg_t src,
                     struct ir3_register *reg)
 {
    struct ir3_instruction *old_pcopy = NULL;
-   if (!list_is_empty(&block->instr_list)) {
-      struct ir3_instruction *last =
-         list_entry(block->instr_list.prev, struct ir3_instruction, node);
-      if (last->opc == OPC_META_PARALLEL_COPY)
-         old_pcopy = last;
-   }
+   struct ir3_instruction *last = ir3_block_get_last_non_terminator(block);
+
+   if (last && last->opc == OPC_META_PARALLEL_COPY)
+      old_pcopy = last;
 
    unsigned old_pcopy_srcs = old_pcopy ? old_pcopy->srcs_count : 0;
    struct ir3_instruction *pcopy = ir3_instr_create(
diff --git a/src/freedreno/ir3/ir3_sched.c b/src/freedreno/ir3/ir3_sched.c
index 31f709b52d8..3991c0c6861 100644
--- a/src/freedreno/ir3/ir3_sched.c
+++ b/src/freedreno/ir3/ir3_sched.c
@@ -1001,16 +1001,6 @@ split_pred(struct ir3_sched_ctx *ctx)
       }
    }
 
-   if (ctx->block->condition == ctx->pred) {
-      if (!new_pred) {
-         new_pred = split_instr(ctx, ctx->pred);
-         /* original pred is scheduled, but new one isn't: */
-         new_pred->flags &= ~IR3_INSTR_MARK;
-      }
-      ctx->block->condition = new_pred;
-      d("new branch condition");
-   }
-
    /* all remaining predicated remapped to new pred: */
    ctx->pred = NULL;
 
@@ -1198,6 +1188,12 @@ sched_block(struct ir3_sched_ctx *ctx, struct ir3_block *block)
    ctx->sy_index = ctx->first_outstanding_sy_index = 0;
    ctx->ss_index = ctx->first_outstanding_ss_index = 0;
 
+   /* The terminator has to stay at the end. Instead of trying to set up
+    * dependencies to achieve this, it's easier to just remove it now and add it
+    * back after scheduling.
+    */
+   struct ir3_instruction *terminator = ir3_block_take_terminator(block);
+
    /* move all instructions to the unscheduled list, and
     * empty the block's instruction list (to which we will
     * be inserting).
@@ -1295,6 +1291,9 @@ sched_block(struct ir3_sched_ctx *ctx, struct ir3_block *block)
    }
 
    sched_dag_destroy(ctx);
+
+   if (terminator)
+      list_addtail(&terminator->node, &block->instr_list);
 }
 
 int
@@ -1308,7 +1307,7 @@ ir3_sched(struct ir3 *ir)
       }
    }
 
-   ir3_count_instructions(ir);
+   ir3_count_instructions_sched(ir);
    ir3_clear_mark(ir);
    ir3_find_ssa_uses(ir, ctx, false);