R600/SI: Custom lower SI_IF and SI_ELSE to avoid machine verifier errors

SI_IF and SI_ELSE are terminators which also produce a value.  For
these instructions ISel always inserts a COPY to move their value
to another basic block.  This COPY ends up between SI_(IF|ELSE)
and the S_BRANCH* instruction at the end of the block.

This breaks MachineBasicBlock::getFirstTerminator() and also the
machine verifier which assumes that terminators are grouped together at
the end of blocks.

To solve this we coalesce the copy away right after ISel to make sure
there are no instructions in between terminators at the end of blocks.

2 files changed, 63 insertions, 15 deletions

diff --git a/lib/Target/R600/SIISelLowering.cpp b/lib/Target/R600/SIISelLowering.cpp
index b7c35d71aa1..848e0453939 100644
--- a/lib/Target/R600/SIISelLowering.cpp
+++ b/lib/Target/R600/SIISelLowering.cpp
@@ -411,19 +411,48 @@ SDValue SITargetLowering::LowerFormalArguments(
   return Chain;
 }
 
+/// Usually ISel will insert a copy between terminator insturction that output
+/// a value and the S_BRANCH* at the end of the block.  This causes
+/// MachineBasicBlock::getFirstTerminator() to return the incorrect value,
+/// so we want to make sure there are no copies between terminators at the
+/// end of blocks.
+static void LowerTerminatorWithOutput(unsigned Opcode, MachineBasicBlock *BB,
+                                      MachineInstr *MI,
+                                      const TargetInstrInfo *TII,
+                                      MachineRegisterInfo &MRI) {
+  unsigned DstReg = MI->getOperand(0).getReg();
+  // Usually ISel will insert a copy between the SI_IF_NON_TERM instruction
+  // and the S_BRANCH* terminator.  We want to replace SI_IF_NO_TERM with
+  // SI_IF and we can't have any instructions between S_BRANCH* and SI_IF,
+  // since they are both terminators
+  assert(MRI.hasOneUse(DstReg));
+  MachineOperand &Use = *MRI.use_begin(DstReg);
+  MachineInstr *UseMI = Use.getParent();
+  assert(UseMI->getOpcode() == AMDGPU::COPY);
+
+  MRI.replaceRegWith(UseMI->getOperand(0).getReg(), DstReg);
+  UseMI->eraseFromParent();
+  BuildMI(*BB, BB->getFirstTerminator(), MI->getDebugLoc(),
+          TII->get(Opcode))
+          .addOperand(MI->getOperand(0))
+          .addOperand(MI->getOperand(1))
+          .addOperand(MI->getOperand(2));
+  MI->eraseFromParent();
+}
+
 MachineBasicBlock * SITargetLowering::EmitInstrWithCustomInserter(
     MachineInstr * MI, MachineBasicBlock * BB) const {
 
   MachineBasicBlock::iterator I = *MI;
+  const SIInstrInfo *TII =
+    static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
+  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
 
   switch (MI->getOpcode()) {
   default:
     return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB);
   case AMDGPU::BRANCH: return BB;
   case AMDGPU::SI_ADDR64_RSRC: {
-    const SIInstrInfo *TII =
-      static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
-    MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
     unsigned SuperReg = MI->getOperand(0).getReg();
     unsigned SubRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass);
     unsigned SubRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass);
@@ -448,9 +477,13 @@ MachineBasicBlock * SITargetLowering::EmitInstrWithCustomInserter(
     MI->eraseFromParent();
     break;
   }
-  case AMDGPU::V_SUB_F64: {
-    const SIInstrInfo *TII =
-      static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
+  case AMDGPU::SI_IF_NON_TERM:
+    LowerTerminatorWithOutput(AMDGPU::SI_IF, BB, MI, TII, MRI);
+    break;
+  case AMDGPU::SI_ELSE_NON_TERM:
+    LowerTerminatorWithOutput(AMDGPU::SI_ELSE, BB, MI, TII, MRI);
+    break;
+  case AMDGPU::V_SUB_F64:
     BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::V_ADD_F64),
             MI->getOperand(0).getReg())
             .addReg(MI->getOperand(1).getReg())
@@ -462,11 +495,9 @@ MachineBasicBlock * SITargetLowering::EmitInstrWithCustomInserter(
             .addImm(2); /* NEG */
     MI->eraseFromParent();
     break;
-  }
+
   case AMDGPU::SI_RegisterStorePseudo: {
     MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
-    const SIInstrInfo *TII =
-      static_cast<const SIInstrInfo*>(getTargetMachine().getInstrInfo());
     unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
     MachineInstrBuilder MIB =
         BuildMI(*BB, I, MI->getDebugLoc(), TII->get(AMDGPU::SI_RegisterStore),
diff --git a/lib/Target/R600/SIInstructions.td b/lib/Target/R600/SIInstructions.td
index 80897f2f416..01013f6f43f 100644
--- a/lib/Target/R600/SIInstructions.td
+++ b/lib/Target/R600/SIInstructions.td
@@ -1375,21 +1375,38 @@ def LOAD_CONST : AMDGPUShaderInst <
 let mayLoad = 1, mayStore = 1, hasSideEffects = 1,
     Uses = [EXEC], Defs = [EXEC] in {
 
+let usesCustomInserter = 1 in {
+
+def SI_IF_NON_TERM : InstSI <
+  (outs SReg_64:$dst),
+  (ins SReg_64:$vcc, brtarget:$target), "",
+  [(set i64:$dst, (int_SI_if i1:$vcc, bb:$target))]
+>;
+
+def SI_ELSE_NON_TERM : InstSI <
+  (outs SReg_64:$dst),
+  (ins SReg_64:$src, brtarget:$target),
+  "",
+  [(set i64:$dst, (int_SI_else i64:$src, bb:$target))]
+> {
+  let Constraints = "$src = $dst";
+}
+
+} // usesCustomInserter = 1
+
 let isBranch = 1, isTerminator = 1 in {
 
-def SI_IF : InstSI <
+def SI_IF: InstSI <
   (outs SReg_64:$dst),
   (ins SReg_64:$vcc, brtarget:$target),
-  "SI_IF $dst, $vcc, $target",
-  [(set i64:$dst, (int_SI_if i1:$vcc, bb:$target))]
+  "", []
 >;
 
 def SI_ELSE : InstSI <
   (outs SReg_64:$dst),
   (ins SReg_64:$src, brtarget:$target),
-  "SI_ELSE $dst, $src, $target",
-  [(set i64:$dst, (int_SI_else i64:$src, bb:$target))]> {
-
+  "", []
+> {
   let Constraints = "$src = $dst";
 }