//===-- LICM.cpp - Loop Invariant Code Motion Pass ------------------------===// // // This pass is a simple loop invariant code motion pass. An interesting aspect // of this pass is that it uses alias analysis for two purposes: // // 1. Moving loop invariant loads out of loops. If we can determine that a // load inside of a loop never aliases anything stored to, we can hoist it // like any other instruction. // 2. Scalar Promotion of Memory - If there is a store instruction inside of // the loop, we try to move the store to happen AFTER the loop instead of // inside of the loop. This can only happen if a few conditions are true: // A. The pointer stored through is loop invariant // B. There are no stores or loads in the loop which _may_ alias the // pointer. There are no calls in the loop which mod/ref the pointer. // If these conditions are true, we can promote the loads and stores in the // loop of the pointer to use a temporary alloca'd variable. We then use // the mem2reg functionality to construct the appropriate SSA form for the // variable. // //===----------------------------------------------------------------------===// #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Utils/PromoteMemToReg.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/Dominators.h" #include "llvm/Instructions.h" #include "llvm/DerivedTypes.h" #include "llvm/Support/InstVisitor.h" #include "llvm/Support/CFG.h" #include "Support/Statistic.h" #include "Support/CommandLine.h" #include "llvm/Assembly/Writer.h" #include namespace { cl::opt DisablePromotion("disable-licm-promotion", cl::Hidden, cl::desc("Disable memory promotion in LICM pass")); Statistic<> NumHoisted("licm", "Number of instructions hoisted out of loop"); Statistic<> NumHoistedLoads("licm", "Number of load insts hoisted"); Statistic<> NumPromoted("licm", "Number of memory locations promoted to registers"); /// LoopBodyInfo - We recursively traverse loops from most-deeply-nested to /// least-deeply-nested. For all of the loops nested within the current one, /// we keep track of information so that we don't have to repeat queries. /// struct LoopBodyInfo { std::vector Calls; // Call instructions in loop std::vector Invokes; // Invoke instructions in loop // StoredPointers - Targets of store instructions... std::set StoredPointers; // LoadedPointers - Source pointers for load instructions... std::set LoadedPointers; enum PointerClass { PointerUnknown = 0, // Nothing is known about this pointer yet PointerMustStore, // Memory is stored to ONLY through this pointer PointerMayStore, // Memory is stored to through this or other pointers PointerNoStore // Memory is not modified in this loop }; // PointerIsModified - Keep track of information as we find out about it in // the loop body... // std::map PointerIsModified; /// CantModifyAnyPointers - Return true if no memory modifying instructions /// occur in this loop. This is just a conservative approximation, because /// a call may not actually store anything. bool CantModifyAnyPointers() const { return Calls.empty() && Invokes.empty() && StoredPointers.empty(); } /// incorporate - Incorporate information about a subloop into the current /// loop. void incorporate(const LoopBodyInfo &OtherLBI); void incorporate(BasicBlock &BB); // do the same for a basic block PointerClass getPointerInfo(Value *V, AliasAnalysis &AA) { PointerClass &VInfo = PointerIsModified[V]; if (VInfo == PointerUnknown) VInfo = calculatePointerInfo(V, AA); return VInfo; } private: /// calculatePointerInfo - Calculate information about the specified /// pointer. PointerClass calculatePointerInfo(Value *V, AliasAnalysis &AA) const; }; } /// incorporate - Incorporate information about a subloop into the current loop. void LoopBodyInfo::incorporate(const LoopBodyInfo &OtherLBI) { // Do not incorporate NonModifiedPointers (which is just a cache) because it // is too much trouble to make sure it's still valid. Calls.insert (Calls.end(), OtherLBI.Calls.begin(), OtherLBI.Calls.end()); Invokes.insert(Invokes.end(),OtherLBI.Invokes.begin(),OtherLBI.Invokes.end()); StoredPointers.insert(OtherLBI.StoredPointers.begin(), OtherLBI.StoredPointers.end()); LoadedPointers.insert(OtherLBI.LoadedPointers.begin(), OtherLBI.LoadedPointers.end()); } void LoopBodyInfo::incorporate(BasicBlock &BB) { for (BasicBlock::iterator I = BB.begin(), E = --BB.end(); I != E; ++I) if (CallInst *CI = dyn_cast(&*I)) Calls.push_back(CI); else if (StoreInst *SI = dyn_cast(&*I)) StoredPointers.insert(SI->getOperand(1)); else if (LoadInst *LI = dyn_cast(&*I)) LoadedPointers.insert(LI->getOperand(0)); if (InvokeInst *II = dyn_cast(BB.getTerminator())) Invokes.push_back(II); } // calculatePointerInfo - Calculate information about the specified pointer. LoopBodyInfo::PointerClass LoopBodyInfo::calculatePointerInfo(Value *V, AliasAnalysis &AA) const { for (unsigned i = 0, e = Calls.size(); i != e; ++i) if (AA.getModRefInfo(Calls[i], V, ~0)) return PointerMayStore; for (unsigned i = 0, e = Invokes.size(); i != e; ++i) if (AA.getModRefInfo(Invokes[i], V, ~0)) return PointerMayStore; PointerClass Result = PointerNoStore; for (std::set::const_iterator I = StoredPointers.begin(), E = StoredPointers.end(); I != E; ++I) if (AA.alias(V, ~0, *I, ~0)) if (V == *I) Result = PointerMustStore; // If this is the only alias, return must else return PointerMayStore; // We have to return may return Result; } namespace { struct LICM : public FunctionPass, public InstVisitor { virtual bool runOnFunction(Function &F); /// This transformation requires natural loop information & requires that /// loop preheaders be inserted into the CFG... /// virtual void getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesCFG(); AU.addRequiredID(LoopPreheadersID); AU.addRequired(); AU.addRequired(); AU.addRequired(); AU.addRequired(); } private: LoopInfo *LI; // Current LoopInfo AliasAnalysis *AA; // Current AliasAnalysis information bool Changed; // Set to true when we change anything. BasicBlock *Preheader; // The preheader block of the current loop... Loop *CurLoop; // The current loop we are working on... LoopBodyInfo *CurLBI; // Information about the current loop... /// visitLoop - Hoist expressions out of the specified loop... /// void visitLoop(Loop *L, LoopBodyInfo &LBI); /// HoistRegion - Walk the specified region of the CFG (defined by all /// blocks dominated by the specified block, and that are in the current /// loop) in depth first order w.r.t the DominatorTree. This allows us to /// visit defintions before uses, allowing us to hoist a loop body in one /// pass without iteration. /// void HoistRegion(DominatorTree::Node *N); /// inSubLoop - Little predicate that returns true if the specified basic /// block is in a subloop of the current one, not the current one itself. /// bool inSubLoop(BasicBlock *BB) { assert(CurLoop->contains(BB) && "Only valid if BB is IN the loop"); for (unsigned i = 0, e = CurLoop->getSubLoops().size(); i != e; ++i) if (CurLoop->getSubLoops()[i]->contains(BB)) return true; // A subloop actually contains this block! return false; } /// hoist - When an instruction is found to only use loop invariant operands /// that is safe to hoist, this instruction is called to do the dirty work. /// void hoist(Instruction &I); /// pointerInvalidatedByLoop - Return true if the body of this loop may /// store into the memory location pointed to by V. /// bool pointerInvalidatedByLoop(Value *V) { // Check to see if any of the basic blocks in CurLoop invalidate V. return CurLBI->getPointerInfo(V, *AA) != LoopBodyInfo::PointerNoStore; } /// isLoopInvariant - Return true if the specified value is loop invariant /// inline bool isLoopInvariant(Value *V) { if (Instruction *I = dyn_cast(V)) return !CurLoop->contains(I->getParent()); return true; // All non-instructions are loop invariant } /// PromoteValuesInLoop - Look at the stores in the loop and promote as many /// to scalars as we can. /// void PromoteValuesInLoop(); /// findPromotableValuesInLoop - Check the current loop for stores to /// definate pointers, which are not loaded and stored through may aliases. /// If these are found, create an alloca for the value, add it to the /// PromotedValues list, and keep track of the mapping from value to /// alloca... /// void findPromotableValuesInLoop( std::vector > &PromotedValues, std::map &Val2AlMap); /// Instruction visitation handlers... these basically control whether or /// not the specified instruction types are hoisted. /// friend class InstVisitor; void visitBinaryOperator(Instruction &I) { if (isLoopInvariant(I.getOperand(0)) && isLoopInvariant(I.getOperand(1))) hoist(I); } void visitCastInst(CastInst &CI) { Instruction &I = (Instruction&)CI; if (isLoopInvariant(I.getOperand(0))) hoist(I); } void visitShiftInst(ShiftInst &I) { visitBinaryOperator((Instruction&)I); } void visitLoadInst(LoadInst &LI); void visitGetElementPtrInst(GetElementPtrInst &GEPI) { Instruction &I = (Instruction&)GEPI; for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) if (!isLoopInvariant(I.getOperand(i))) return; hoist(I); } }; RegisterOpt X("licm", "Loop Invariant Code Motion"); } Pass *createLICMPass() { return new LICM(); } /// runOnFunction - For LICM, this simply traverses the loop structure of the /// function, hoisting expressions out of loops if possible. /// bool LICM::runOnFunction(Function &) { Changed = false; // Get our Loop and Alias Analysis information... LI = &getAnalysis(); AA = &getAnalysis(); // Hoist expressions out of all of the top-level loops. const std::vector &TopLevelLoops = LI->getTopLevelLoops(); for (std::vector::const_iterator I = TopLevelLoops.begin(), E = TopLevelLoops.end(); I != E; ++I) { LoopBodyInfo LBI; LICM::visitLoop(*I, LBI); } return Changed; } /// visitLoop - Hoist expressions out of the specified loop... /// void LICM::visitLoop(Loop *L, LoopBodyInfo &LBI) { // Recurse through all subloops before we process this loop... for (std::vector::const_iterator I = L->getSubLoops().begin(), E = L->getSubLoops().end(); I != E; ++I) { LoopBodyInfo SubLBI; LICM::visitLoop(*I, SubLBI); // Incorporate information about the subloops into this loop... LBI.incorporate(SubLBI); } CurLoop = L; CurLBI = &LBI; // Get the preheader block to move instructions into... Preheader = L->getLoopPreheader(); assert(Preheader&&"Preheader insertion pass guarantees we have a preheader!"); // Loop over the body of this loop, looking for calls, invokes, and stores. // Because subloops have already been incorporated into LBI, we skip blocks in // subloops. // const std::vector &LoopBBs = L->getBlocks(); for (std::vector::const_iterator I = LoopBBs.begin(), E = LoopBBs.end(); I != E; ++I) if (LI->getLoopFor(*I) == L) // Ignore blocks in subloops... LBI.incorporate(**I); // Incorporate the specified basic block // We want to visit all of the instructions in this loop... that are not parts // of our subloops (they have already had their invariants hoisted out of // their loop, into this loop, so there is no need to process the BODIES of // the subloops). // // Traverse the body of the loop in depth first order on the dominator tree so // that we are guaranteed to see definitions before we see uses. This allows // us to perform the LICM transformation in one pass, without iteration. // HoistRegion(getAnalysis()[L->getHeader()]); // Now that all loop invariants have been removed from the loop, promote any // memory references to scalars that we can... if (!DisablePromotion) PromoteValuesInLoop(); // Clear out loops state information for the next iteration CurLoop = 0; Preheader = 0; } /// HoistRegion - Walk the specified region of the CFG (defined by all blocks /// dominated by the specified block, and that are in the current loop) in depth /// first order w.r.t the DominatorTree. This allows us to visit defintions /// before uses, allowing us to hoist a loop body in one pass without iteration. /// void LICM::HoistRegion(DominatorTree::Node *N) { assert(N != 0 && "Null dominator tree node?"); // If this subregion is not in the top level loop at all, exit. if (!CurLoop->contains(N->getNode())) return; // Only need to hoist the contents of this block if it is not part of a // subloop (which would already have been hoisted) if (!inSubLoop(N->getNode())) visit(*N->getNode()); const std::vector &Children = N->getChildren(); for (unsigned i = 0, e = Children.size(); i != e; ++i) HoistRegion(Children[i]); } /// hoist - When an instruction is found to only use loop invariant operands /// that is safe to hoist, this instruction is called to do the dirty work. /// void LICM::hoist(Instruction &Inst) { DEBUG(std::cerr << "LICM hoisting to"; WriteAsOperand(std::cerr, Preheader, false); std::cerr << ": " << Inst); // Remove the instruction from its current basic block... but don't delete the // instruction. Inst.getParent()->getInstList().remove(&Inst); // Insert the new node in Preheader, before the terminator. Preheader->getInstList().insert(Preheader->getTerminator(), &Inst); ++NumHoisted; Changed = true; } void LICM::visitLoadInst(LoadInst &LI) { if (isLoopInvariant(LI.getOperand(0)) && !pointerInvalidatedByLoop(LI.getOperand(0))) { hoist(LI); ++NumHoistedLoads; } } /// PromoteValuesInLoop - Try to promote memory values to scalars by sinking /// stores out of the loop and moving loads to before the loop. We do this by /// looping over the stores in the loop, looking for stores to Must pointers /// which are loop invariant. We promote these memory locations to use allocas /// instead. These allocas can easily be raised to register values by the /// PromoteMem2Reg functionality. /// void LICM::PromoteValuesInLoop() { // PromotedValues - List of values that are promoted out of the loop. Each // value has an alloca instruction for it, and a cannonical version of the // pointer. std::vector > PromotedValues; std::map ValueToAllocaMap; // Map of ptr to alloca findPromotableValuesInLoop(PromotedValues, ValueToAllocaMap); if (ValueToAllocaMap.empty()) return; // If there are values to promote... Changed = true; NumPromoted += PromotedValues.size(); // Emit a copy from the value into the alloca'd value in the loop preheader TerminatorInst *LoopPredInst = Preheader->getTerminator(); for (unsigned i = 0, e = PromotedValues.size(); i != e; ++i) { // Load from the memory we are promoting... LoadInst *LI = new LoadInst(PromotedValues[i].second, PromotedValues[i].second->getName()+".promoted", LoopPredInst); // Store into the temporary alloca... new StoreInst(LI, PromotedValues[i].first, LoopPredInst); } // Scan the basic blocks in the loop, replacing uses of our pointers with // uses of the allocas in question. If we find a branch that exits the // loop, make sure to put reload code into all of the successors of the // loop. // const std::vector &LoopBBs = CurLoop->getBlocks(); for (std::vector::const_iterator I = LoopBBs.begin(), E = LoopBBs.end(); I != E; ++I) { // Rewrite all loads and stores in the block of the pointer... for (BasicBlock::iterator II = (*I)->begin(), E = (*I)->end(); II != E; ++II) { if (LoadInst *L = dyn_cast(&*II)) { std::map::iterator I = ValueToAllocaMap.find(L->getOperand(0)); if (I != ValueToAllocaMap.end()) L->setOperand(0, I->second); // Rewrite load instruction... } else if (StoreInst *S = dyn_cast(&*II)) { std::map::iterator I = ValueToAllocaMap.find(S->getOperand(1)); if (I != ValueToAllocaMap.end()) S->setOperand(1, I->second); // Rewrite store instruction... } } // Check to see if any successors of this block are outside of the loop. // If so, we need to copy the value from the alloca back into the memory // location... // for (succ_iterator SI = succ_begin(*I), SE = succ_end(*I); SI != SE; ++SI) if (!CurLoop->contains(*SI)) { // Copy all of the allocas into their memory locations... BasicBlock::iterator BI = (*SI)->begin(); while (isa(*BI)) ++BI; // Skip over all of the phi nodes in the block... Instruction *InsertPos = BI; for (unsigned i = 0, e = PromotedValues.size(); i != e; ++i) { // Load from the alloca... LoadInst *LI = new LoadInst(PromotedValues[i].first, "", InsertPos); // Store into the memory we promoted... new StoreInst(LI, PromotedValues[i].second, InsertPos); } } } // Now that we have done the deed, use the mem2reg functionality to promote // all of the new allocas we just created into real SSA registers... // std::vector PromotedAllocas; PromotedAllocas.reserve(PromotedValues.size()); for (unsigned i = 0, e = PromotedValues.size(); i != e; ++i) PromotedAllocas.push_back(PromotedValues[i].first); PromoteMemToReg(PromotedAllocas, getAnalysis()); } /// findPromotableValuesInLoop - Check the current loop for stores to definate /// pointers, which are not loaded and stored through may aliases. If these are /// found, create an alloca for the value, add it to the PromotedValues list, /// and keep track of the mapping from value to alloca... /// void LICM::findPromotableValuesInLoop( std::vector > &PromotedValues, std::map &ValueToAllocaMap) { Instruction *FnStart = CurLoop->getHeader()->getParent()->begin()->begin(); for (std::set::iterator I = CurLBI->StoredPointers.begin(), E = CurLBI->StoredPointers.end(); I != E; ++I) { Value *V = *I; if (isLoopInvariant(V) && CurLBI->getPointerInfo(V, *AA) == LoopBodyInfo::PointerMustStore) { // Don't add a new entry for this stored pointer if it aliases something // we have already processed. std::map::iterator V2AMI = ValueToAllocaMap.lower_bound(V); if (V2AMI == ValueToAllocaMap.end() || V2AMI->first != V) { // Check to make sure that any loads in the loop are either NO or MUST // aliases. We cannot rewrite loads that _might_ come from this memory // location. bool PointerOk = true; for (std::set::const_iterator I =CurLBI->LoadedPointers.begin(), E = CurLBI->LoadedPointers.end(); PointerOk && I != E; ++I) switch (AA->alias(V, ~0, *I, ~0)) { case AliasAnalysis::MustAlias: if (V->getType() != (*I)->getType()) PointerOk = false; break; case AliasAnalysis::MayAlias: PointerOk = false; case AliasAnalysis::NoAlias: break; } if (PointerOk) { const Type *Ty = cast(V->getType())->getElementType(); AllocaInst *AI = new AllocaInst(Ty, 0, V->getName()+".tmp", FnStart); PromotedValues.push_back(std::make_pair(AI, V)); ValueToAllocaMap.insert(V2AMI, std::make_pair(V, AI)); DEBUG(std::cerr << "LICM: Promoting value: " << *V << "\n"); // Loop over all of the loads and stores that alias this pointer, // adding them to the Value2AllocaMap as well... for (std::set::const_iterator I = CurLBI->LoadedPointers.begin(), E = CurLBI->LoadedPointers.end(); I != E; ++I) if (AA->alias(V, ~0, *I, ~0) == AliasAnalysis::MustAlias) ValueToAllocaMap[*I] = AI; for (std::set::const_iterator I = CurLBI->StoredPointers.begin(), E = CurLBI->StoredPointers.end(); I != E; ++I) if (AA->alias(V, ~0, *I, ~0) == AliasAnalysis::MustAlias) ValueToAllocaMap[*I] = AI; } } } } }