diff --git a/lib/CodeGen/MachineLICM.cpp b/lib/CodeGen/MachineLICM.cpp index 10ee8d6d137..76d5f377949 100644 --- a/lib/CodeGen/MachineLICM.cpp +++ b/lib/CodeGen/MachineLICM.cpp @@ -28,6 +28,7 @@ #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Statistic.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" @@ -36,6 +37,7 @@ using namespace llvm; STATISTIC(NumHoisted, "Number of machine instructions hoisted out of loops"); +STATISTIC(NumCSEed, "Number of hoisted machine instructions CSEed"); namespace { class VISIBILITY_HIDDEN MachineLICM : public MachineFunctionPass { @@ -51,6 +53,10 @@ namespace { bool Changed; // True if a loop is changed. MachineLoop *CurLoop; // The current loop we are working on. MachineBasicBlock *CurPreheader; // The preheader for CurLoop. + + // For each BB and opcode pair, keep a list of hoisted instructions. + DenseMap, + std::vector > CSEMap; public: static char ID; // Pass identification, replacement for typeid MachineLICM() : MachineFunctionPass(&ID) {} @@ -68,6 +74,11 @@ namespace { AU.addPreserved(); MachineFunctionPass::getAnalysisUsage(AU); } + + virtual void releaseMemory() { + CSEMap.clear(); + } + private: /// IsLoopInvariantInst - Returns true if the instruction is loop /// invariant. I.e., all virtual register operands are defined outside of @@ -163,13 +174,13 @@ void MachineLICM::HoistRegion(MachineDomTreeNode *N) { if (!CurLoop->contains(BB)) return; for (MachineBasicBlock::iterator - I = BB->begin(), E = BB->end(); I != E; ) { - MachineInstr &MI = *I++; + MII = BB->begin(), E = BB->end(); MII != E; ) { + MachineBasicBlock::iterator NextMII = MII; ++NextMII; + MachineInstr &MI = *MII; - // Try hoisting the instruction out of the loop. We can only do this if - // all of the operands of the instruction are loop invariant and if it is - // safe to hoist the instruction. Hoist(MI); + + MII = NextMII; } const std::vector &Children = N->getChildren(); @@ -258,17 +269,16 @@ bool MachineLICM::IsLoopInvariantInst(MachineInstr &I) { return true; } -/// HasOnlyPHIUses - Return true if the only uses of Reg are PHIs. -static bool HasOnlyPHIUses(unsigned Reg, MachineRegisterInfo *RegInfo) { - bool OnlyPHIUse = false; + +/// HasPHIUses - Return true if the specified register has any PHI use. +static bool HasPHIUses(unsigned Reg, MachineRegisterInfo *RegInfo) { for (MachineRegisterInfo::use_iterator UI = RegInfo->use_begin(Reg), UE = RegInfo->use_end(); UI != UE; ++UI) { MachineInstr *UseMI = &*UI; - if (UseMI->getOpcode() != TargetInstrInfo::PHI) - return false; - OnlyPHIUse = true; + if (UseMI->getOpcode() == TargetInstrInfo::PHI) + return true; } - return OnlyPHIUse; + return false; } /// IsProfitableToHoist - Return true if it is potentially profitable to hoist @@ -283,21 +293,42 @@ bool MachineLICM::IsProfitableToHoist(MachineInstr &MI) { !TII->isTriviallyReMaterializable(&MI))) return false; - if (!TID.isAsCheapAsAMove()) - return true; - - // If the instruction is "cheap" and the only uses of the register(s) defined - // by this MI are PHIs, then don't hoist it. Otherwise we just end up with a - // cheap instruction (e.g. constant) with long live interval feeeding into - // copies that are not always coalesced away. - bool OnlyPHIUses = false; + // If result(s) of this instruction is used by PHIs, then don't hoist it. + // The presence of joins makes it difficult for current register allocator + // implementation to perform remat. for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { const MachineOperand &MO = MI.getOperand(i); if (!MO.isReg() || !MO.isDef()) continue; - OnlyPHIUses |= HasOnlyPHIUses(MO.getReg(), RegInfo); + if (HasPHIUses(MO.getReg(), RegInfo)) + return false; } - return !OnlyPHIUses; + + return true; +} + +static const MachineInstr *LookForDuplicate(const MachineInstr *MI, + std::vector &PrevMIs) { + unsigned NumOps = MI->getNumOperands(); + for (unsigned i = 0, e = PrevMIs.size(); i != e; ++i) { + const MachineInstr *PrevMI = PrevMIs[i]; + unsigned NumOps2 = PrevMI->getNumOperands(); + if (NumOps != NumOps2) + continue; + bool IsSame = true; + for (unsigned j = 0; j != NumOps; ++j) { + const MachineOperand &MO = MI->getOperand(j); + if (MO.isReg() && MO.isDef()) + continue; + if (!MO.isIdenticalTo(PrevMI->getOperand(j))) { + IsSame = false; + break; + } + } + if (IsSame) + return PrevMI; + } + return 0; } /// Hoist - When an instruction is found to use only loop invariant operands @@ -320,7 +351,41 @@ void MachineLICM::Hoist(MachineInstr &MI) { DOUT << "\n"; }); - CurPreheader->splice(CurPreheader->getFirstTerminator(), MI.getParent(), &MI); + // Look for opportunity to CSE the hoisted instruction. + std::pair BBOpcPair = + std::make_pair(CurPreheader->getNumber(), MI.getOpcode()); + DenseMap, + std::vector >::iterator CI = CSEMap.find(BBOpcPair); + bool DoneCSE = false; + if (CI != CSEMap.end()) { + const MachineInstr *Dup = LookForDuplicate(&MI, CI->second); + if (Dup) { + DOUT << "CSEing " << MI; + DOUT << " with " << *Dup; + for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { + const MachineOperand &MO = MI.getOperand(i); + if (MO.isReg() && MO.isDef()) + RegInfo->replaceRegWith(MO.getReg(), Dup->getOperand(i).getReg()); + } + MI.eraseFromParent(); + DoneCSE = true; + ++NumCSEed; + } + } + + // Otherwise, splice the instruction to the preheader. + if (!DoneCSE) { + CurPreheader->splice(CurPreheader->getFirstTerminator(), + MI.getParent(), &MI); + // Add to the CSE map. + if (CI != CSEMap.end()) + CI->second.push_back(&MI); + else { + std::vector CSEMIs; + CSEMIs.push_back(&MI); + CSEMap.insert(std::make_pair(BBOpcPair, CSEMIs)); + } + } ++NumHoisted; Changed = true;