Teach MachineLICM to unfold loads from constant memory from

otherwise unhoistable instructions in order to allow the loads
to be hoisted.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@85364 91177308-0d34-0410-b5e6-96231b3b80d8

lib/CodeGen/MachineLICM.cpp

@@ -24,7 +24,9 @@
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/PseudoSourceValue.h"
 #include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetMachine.h"
@@ -106,7 +108,7 @@ namespace {
     /// Hoist - When an instruction is found to only use loop invariant operands
     /// that is safe to hoist, this instruction is called to do the dirty work.
     ///
-    void Hoist(MachineInstr &MI);
+    void Hoist(MachineInstr *MI);
   };
 } // end anonymous namespace
 
@@ -185,7 +187,7 @@ void MachineLICM::HoistRegion(MachineDomTreeNode *N) {
     MachineBasicBlock::iterator NextMII = MII; ++NextMII;
     MachineInstr &MI = *MII;
 
-    Hoist(MI);
+    Hoist(&MI);
 
     MII = NextMII;
   }
@@ -370,39 +372,103 @@ static const MachineInstr *LookForDuplicate(const MachineInstr *MI,
 /// Hoist - When an instruction is found to use only loop invariant operands
 /// that are safe to hoist, this instruction is called to do the dirty work.
 ///
-void MachineLICM::Hoist(MachineInstr &MI) {
+void MachineLICM::Hoist(MachineInstr *MI) {
-  if (!IsLoopInvariantInst(MI)) return;
+  // First check whether we should hoist this instruction.
-  if (!IsProfitableToHoist(MI)) return;
+  if (!IsLoopInvariantInst(*MI) || !IsProfitableToHoist(*MI)) {
+    // If not, we may be able to unfold a load and hoist that.
+    // First test whether the instruction is loading from an amenable
+    // memory location.
+    if (!MI->getDesc().mayLoad()) return;
+    if (!MI->hasOneMemOperand()) return;
+    MachineMemOperand *MMO = *MI->memoperands_begin();
+    if (MMO->isVolatile()) return;
+    MachineFunction &MF = *MI->getParent()->getParent();
+    if (!MMO->getValue()) return;
+    if (const PseudoSourceValue *PSV =
+          dyn_cast<PseudoSourceValue>(MMO->getValue())) {
+      if (!PSV->isConstant(MF.getFrameInfo())) return;
+    } else {
+      if (!AA->pointsToConstantMemory(MMO->getValue())) return;
+    }
+    // Next determine the register class for a temporary register.
+    unsigned NewOpc =
+      TII->getOpcodeAfterMemoryUnfold(MI->getOpcode(),
+                                      /*UnfoldLoad=*/true,
+                                      /*UnfoldStore=*/false);
+    if (NewOpc == 0) return;
+    const TargetInstrDesc &TID = TII->get(NewOpc);
+    if (TID.getNumDefs() != 1) return;
+    const TargetRegisterClass *RC = TID.OpInfo[0].getRegClass(TRI);
+    // Ok, we're unfolding. Create a temporary register and do the unfold.
+    unsigned Reg = RegInfo->createVirtualRegister(RC);
+    SmallVector<MachineInstr *, 1> NewMIs;
+    bool Success =
+      TII->unfoldMemoryOperand(MF, MI, Reg,
+                               /*UnfoldLoad=*/true, /*UnfoldStore=*/false,
+                               NewMIs);
+    (void)Success;
+    assert(Success &&
+           "unfoldMemoryOperand failed when getOpcodeAfterMemoryUnfold "
+           "succeeded!");
+    assert(NewMIs.size() == 2 &&
+           "Unfolded a load into multiple instructions!");
+    MachineBasicBlock *MBB = MI->getParent();
+    MBB->insert(MI, NewMIs[0]);
+    MBB->insert(MI, NewMIs[1]);
+    MI->eraseFromParent();
+    // If unfolding produced a load that wasn't loop-invariant or profitable to
+    // hoist, re-fold it to undo the damage.
+    if (!IsLoopInvariantInst(*NewMIs[0]) || !IsProfitableToHoist(*NewMIs[0])) {
+      SmallVector<unsigned, 1> Ops;
+      for (unsigned i = 0, e = NewMIs[1]->getNumOperands(); i != e; ++i) {
+        MachineOperand &MO = NewMIs[1]->getOperand(i);
+        if (MO.isReg() && MO.getReg() == Reg) {
+          assert(MO.isUse() &&
+                 "Register defined by unfolded load is redefined "
+                 "instead of just used!");
+          Ops.push_back(i);
+        }
+      }
+      MI = TII->foldMemoryOperand(MF, NewMIs[1], Ops, NewMIs[0]);
+      assert(MI && "Re-fold failed!");
+      MBB->insert(NewMIs[1], MI);
+      NewMIs[0]->eraseFromParent();
+      NewMIs[1]->eraseFromParent();
+      return;
+    }
+    // Otherwise we successfully unfolded a load that we can hoist.
+    MI = NewMIs[0];
+  }
 
   // Now move the instructions to the predecessor, inserting it before any
   // terminator instructions.
   DEBUG({
-      errs() << "Hoisting " << MI;
+      errs() << "Hoisting " << *MI;
       if (CurPreheader->getBasicBlock())
         errs() << " to MachineBasicBlock "
                << CurPreheader->getBasicBlock()->getName();
-      if (MI.getParent()->getBasicBlock())
+      if (MI->getParent()->getBasicBlock())
         errs() << " from MachineBasicBlock "
-               << MI.getParent()->getBasicBlock()->getName();
+               << MI->getParent()->getBasicBlock()->getName();
       errs() << "\n";
     });
 
   // Look for opportunity to CSE the hoisted instruction.
   std::pair<unsigned, unsigned> BBOpcPair =
-    std::make_pair(CurPreheader->getNumber(), MI.getOpcode());
+    std::make_pair(CurPreheader->getNumber(), MI->getOpcode());
   DenseMap<std::pair<unsigned, unsigned>,
     std::vector<const MachineInstr*> >::iterator CI = CSEMap.find(BBOpcPair);
   bool DoneCSE = false;
   if (CI != CSEMap.end()) {
-    const MachineInstr *Dup = LookForDuplicate(&MI, CI->second, RegInfo);
+    const MachineInstr *Dup = LookForDuplicate(MI, CI->second, RegInfo);
     if (Dup) {
-      DEBUG(errs() << "CSEing " << MI << " with " << *Dup);
+      DEBUG(errs() << "CSEing " << *MI << " with " << *Dup);
-      for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
+      for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
-        const MachineOperand &MO = MI.getOperand(i);
+        const MachineOperand &MO = MI->getOperand(i);
         if (MO.isReg() && MO.isDef())
           RegInfo->replaceRegWith(MO.getReg(), Dup->getOperand(i).getReg());
       }
-      MI.eraseFromParent();
+      MI->eraseFromParent();
       DoneCSE = true;
       ++NumCSEed;
     }
@@ -411,13 +477,13 @@ void MachineLICM::Hoist(MachineInstr &MI) {
   // Otherwise, splice the instruction to the preheader.
   if (!DoneCSE) {
     CurPreheader->splice(CurPreheader->getFirstTerminator(),
-                         MI.getParent(), &MI);
+                         MI->getParent(), MI);
     // Add to the CSE map.
     if (CI != CSEMap.end())
-      CI->second.push_back(&MI);
+      CI->second.push_back(MI);
     else {
       std::vector<const MachineInstr*> CSEMIs;
-      CSEMIs.push_back(&MI);
+      CSEMIs.push_back(MI);
       CSEMap.insert(std::make_pair(BBOpcPair, CSEMIs));
     }
   }

test/CodeGen/X86/pic-load-remat.ll

@@ -1,4 +1,10 @@
 ; RUN: llc < %s -mtriple=i686-apple-darwin -mattr=+sse2 -relocation-model=pic | grep psllw | grep pb
+; XFAIL: *
+
+; This is XFAIL'd because MachineLICM is now hoisting all of the loads, and the pic
+; base appears killed in the entry block when remat is making its decisions. Remat's
+; simple heuristic decides against rematting because it doesn't want to extend the
+; live-range of the pic base; this isn't necessarily optimal.
 
 define void @f() nounwind  {
 entry:

test/CodeGen/X86/sink-hoist.ll

@@ -44,6 +44,7 @@ return:
 
 ; Sink instructions with dead EFLAGS defs.
 
+; CHECK: zzz:
 ; CHECK:      je
 ; CHECK-NEXT: orb
 
@@ -56,3 +57,66 @@ entry:
   %b_addr.0 = select i1 %tmp2, i8 %tmp4, i8 %tmp3 ; <i8> [#uses=1]
   ret i8 %b_addr.0
 }
+
+; Codegen should hoist and CSE these constants.
+
+; CHECK: vv:
+; CHECK: LCPI4_0(%rip), %xmm0
+; CHECK: LCPI4_1(%rip), %xmm1
+; CHECK: LCPI4_2(%rip), %xmm2
+; CHECK: align
+; CHECK-NOT: LCPI
+; CHECK: ret
+
+@_minusZero.6007 = internal constant <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00> ; <<4 x float>*> [#uses=0]
+@twoTo23.6008 = internal constant <4 x float> <float 8.388608e+06, float 8.388608e+06, float 8.388608e+06, float 8.388608e+06> ; <<4 x float>*> [#uses=0]
+
+define void @vv(float* %y, float* %x, i32* %n) nounwind ssp {
+entry:
+  br label %bb60
+
+bb:                                               ; preds = %bb60
+  %0 = bitcast float* %x_addr.0 to <4 x float>*   ; <<4 x float>*> [#uses=1]
+  %1 = load <4 x float>* %0, align 16             ; <<4 x float>> [#uses=4]
+  %tmp20 = bitcast <4 x float> %1 to <4 x i32>    ; <<4 x i32>> [#uses=1]
+  %tmp22 = and <4 x i32> %tmp20, <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647> ; <<4 x i32>> [#uses=1]
+  %tmp23 = bitcast <4 x i32> %tmp22 to <4 x float> ; <<4 x float>> [#uses=1]
+  %tmp25 = bitcast <4 x float> %1 to <4 x i32>    ; <<4 x i32>> [#uses=1]
+  %tmp27 = and <4 x i32> %tmp25, <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648> ; <<4 x i32>> [#uses=2]
+  %tmp30 = call <4 x float> @llvm.x86.sse.cmp.ps(<4 x float> %tmp23, <4 x float> <float 8.388608e+06, float 8.388608e+06, float 8.388608e+06, float 8.388608e+06>, i8 5) ; <<4 x float>> [#uses=1]
+  %tmp34 = bitcast <4 x float> %tmp30 to <4 x i32> ; <<4 x i32>> [#uses=1]
+  %tmp36 = xor <4 x i32> %tmp34, <i32 -1, i32 -1, i32 -1, i32 -1> ; <<4 x i32>> [#uses=1]
+  %tmp37 = and <4 x i32> %tmp36, <i32 1258291200, i32 1258291200, i32 1258291200, i32 1258291200> ; <<4 x i32>> [#uses=1]
+  %tmp42 = or <4 x i32> %tmp37, %tmp27            ; <<4 x i32>> [#uses=1]
+  %tmp43 = bitcast <4 x i32> %tmp42 to <4 x float> ; <<4 x float>> [#uses=2]
+  %tmp45 = fadd <4 x float> %1, %tmp43            ; <<4 x float>> [#uses=1]
+  %tmp47 = fsub <4 x float> %tmp45, %tmp43        ; <<4 x float>> [#uses=2]
+  %tmp49 = call <4 x float> @llvm.x86.sse.cmp.ps(<4 x float> %1, <4 x float> %tmp47, i8 1) ; <<4 x float>> [#uses=1]
+  %2 = bitcast <4 x float> %tmp49 to <4 x i32>    ; <<4 x i32>> [#uses=1]
+  %3 = call <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32> %2) nounwind readnone ; <<4 x float>> [#uses=1]
+  %tmp53 = fadd <4 x float> %tmp47, %3            ; <<4 x float>> [#uses=1]
+  %tmp55 = bitcast <4 x float> %tmp53 to <4 x i32> ; <<4 x i32>> [#uses=1]
+  %tmp57 = or <4 x i32> %tmp55, %tmp27            ; <<4 x i32>> [#uses=1]
+  %tmp58 = bitcast <4 x i32> %tmp57 to <4 x float> ; <<4 x float>> [#uses=1]
+  %4 = bitcast float* %y_addr.0 to <4 x float>*   ; <<4 x float>*> [#uses=1]
+  store <4 x float> %tmp58, <4 x float>* %4, align 16
+  %5 = getelementptr float* %x_addr.0, i64 4      ; <float*> [#uses=1]
+  %6 = getelementptr float* %y_addr.0, i64 4      ; <float*> [#uses=1]
+  %7 = add i32 %i.0, 4                            ; <i32> [#uses=1]
+  br label %bb60
+
+bb60:                                             ; preds = %bb, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %7, %bb ]       ; <i32> [#uses=2]
+  %x_addr.0 = phi float* [ %x, %entry ], [ %5, %bb ] ; <float*> [#uses=2]
+  %y_addr.0 = phi float* [ %y, %entry ], [ %6, %bb ] ; <float*> [#uses=2]
+  %8 = load i32* %n, align 4                      ; <i32> [#uses=1]
+  %9 = icmp sgt i32 %8, %i.0                      ; <i1> [#uses=1]
+  br i1 %9, label %bb, label %return
+
+return:                                           ; preds = %bb60
+  ret void
+}
+
+declare <4 x float> @llvm.x86.sse.cmp.ps(<4 x float>, <4 x float>, i8) nounwind readnone
+
+declare <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32>) nounwind readnone