diff --git a/include/llvm/Transforms/Utils/SSAUpdater.h b/include/llvm/Transforms/Utils/SSAUpdater.h index 9e79aca50e6..b4048b9b440 100644 --- a/include/llvm/Transforms/Utils/SSAUpdater.h +++ b/include/llvm/Transforms/Utils/SSAUpdater.h @@ -118,15 +118,17 @@ private: /// virtual methods. /// class LoadAndStorePromoter { +protected: + SSAUpdater &SSA; public: - LoadAndStorePromoter() {} + LoadAndStorePromoter(const SmallVectorImpl &Insts, + SSAUpdater &S, StringRef Name = StringRef()); virtual ~LoadAndStorePromoter() {} /// run - This does the promotion. Insts is a list of loads and stores to /// promote, and Name is the basename for the PHIs to insert. After this is /// complete, the loads and stores are removed from the code. - void run(StringRef Name, const SmallVectorImpl &Insts, - SSAUpdater *SSA = 0); + void run(const SmallVectorImpl &Insts) const; /// Return true if the specified instruction is in the Inst list (which was @@ -139,6 +141,21 @@ public: return true; return false; } + + /// doExtraRewritesBeforeFinalDeletion - This hook is invoked after all the + /// stores are found and inserted as available values, but + virtual void doExtraRewritesBeforeFinalDeletion() const { + } + + /// replaceLoadWithValue - Clients can choose to implement this to get + /// notified right before a load is RAUW'd another value. + virtual void replaceLoadWithValue(LoadInst *LI, Value *V) const { + } + + /// This is called before each instruction is deleted. + virtual void instructionDeleted(Instruction *I) const { + } + }; } // End llvm namespace diff --git a/lib/Transforms/Scalar/LICM.cpp b/lib/Transforms/Scalar/LICM.cpp index ba75c6f55ce..07867933d08 100644 --- a/lib/Transforms/Scalar/LICM.cpp +++ b/lib/Transforms/Scalar/LICM.cpp @@ -595,6 +595,53 @@ bool LICM::isSafeToExecuteUnconditionally(Instruction &Inst) { return true; } +namespace { + class LoopPromoter : public LoadAndStorePromoter { + Value *SomePtr; // Designated pointer to store to. + SmallPtrSet &PointerMustAliases; + SmallVectorImpl &LoopExitBlocks; + AliasSetTracker * + public: + LoopPromoter(Value *SP, + const SmallVectorImpl &Insts, SSAUpdater &S, + SmallPtrSet &PMA, + SmallVectorImpl &LEB, AliasSetTracker &ast) + : LoadAndStorePromoter(Insts, S), SomePtr(SP), PointerMustAliases(PMA), + LoopExitBlocks(LEB), AST(ast) {} + + virtual bool isInstInList(Instruction *I, + const SmallVectorImpl &) const { + Value *Ptr; + if (LoadInst *LI = dyn_cast(I)) + Ptr = LI->getOperand(0); + else + Ptr = cast(I)->getPointerOperand(); + return PointerMustAliases.count(Ptr); + } + + virtual void doExtraRewritesBeforeFinalDeletion() const { + // Insert stores after in the loop exit blocks. Each exit block gets a + // store of the live-out values that feed them. Since we've already told + // the SSA updater about the defs in the loop and the preheader + // definition, it is all set and we can start using it. + for (unsigned i = 0, e = LoopExitBlocks.size(); i != e; ++i) { + BasicBlock *ExitBlock = LoopExitBlocks[i]; + Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock); + Instruction *InsertPos = ExitBlock->getFirstNonPHI(); + new StoreInst(LiveInValue, SomePtr, InsertPos); + } + } + + virtual void replaceLoadWithValue(LoadInst *LI, Value *V) const { + // Update alias analysis. + AST.copyValue(LI, V); + } + virtual void instructionDeleted(Instruction *I) const { + AST.deleteValue(I); + } + }; +} // end anon namespace + /// PromoteAliasSet - Try to promote memory values to scalars by sinking /// stores out of the loop and moving loads to before the loop. We do this by /// looping over the stores in the loop, looking for stores to Must pointers @@ -679,179 +726,43 @@ void LICM::PromoteAliasSet(AliasSet &AS) { Changed = true; ++NumPromoted; + SmallVector ExitBlocks; + CurLoop->getUniqueExitBlocks(ExitBlocks); + // We use the SSAUpdater interface to insert phi nodes as required. SmallVector NewPHIs; SSAUpdater SSA(&NewPHIs); + LoopPromoter Promoter(SomePtr, LoopUses, SSA, PointerMustAliases, ExitBlocks, + *CurAST); - // It wants to know some value of the same type as what we'll be inserting. - Value *SomeValue; - if (isa(LoopUses[0])) - SomeValue = LoopUses[0]; - else - SomeValue = cast(LoopUses[0])->getOperand(0); - SSA.Initialize(SomeValue->getType(), SomeValue->getName()); - - // First step: bucket up uses of the pointers by the block they occur in. - // This is important because we have to handle multiple defs/uses in a block - // ourselves: SSAUpdater is purely for cross-block references. - // FIXME: Want a TinyVector since there is usually 0/1 element. - DenseMap > UsesByBlock; - for (unsigned i = 0, e = LoopUses.size(); i != e; ++i) { - Instruction *User = LoopUses[i]; - UsesByBlock[User->getParent()].push_back(User); - } - - // Okay, now we can iterate over all the blocks in the loop with uses, - // processing them. Keep track of which loads are loading a live-in value. - SmallVector LiveInLoads; - DenseMap ReplacedLoads; - - for (unsigned LoopUse = 0, e = LoopUses.size(); LoopUse != e; ++LoopUse) { - Instruction *User = LoopUses[LoopUse]; - std::vector &BlockUses = UsesByBlock[User->getParent()]; - - // If this block has already been processed, ignore this repeat use. - if (BlockUses.empty()) continue; - - // Okay, this is the first use in the block. If this block just has a - // single user in it, we can rewrite it trivially. - if (BlockUses.size() == 1) { - // If it is a store, it is a trivial def of the value in the block. - if (isa(User)) { - SSA.AddAvailableValue(User->getParent(), - cast(User)->getOperand(0)); - } else { - // Otherwise it is a load, queue it to rewrite as a live-in load. - LiveInLoads.push_back(cast(User)); - } - BlockUses.clear(); - continue; - } - - // Otherwise, check to see if this block is all loads. If so, we can queue - // them all as live in loads. - bool HasStore = false; - for (unsigned i = 0, e = BlockUses.size(); i != e; ++i) { - if (isa(BlockUses[i])) { - HasStore = true; - break; - } - } - - if (!HasStore) { - for (unsigned i = 0, e = BlockUses.size(); i != e; ++i) - LiveInLoads.push_back(cast(BlockUses[i])); - BlockUses.clear(); - continue; - } - - // Otherwise, we have mixed loads and stores (or just a bunch of stores). - // Since SSAUpdater is purely for cross-block values, we need to determine - // the order of these instructions in the block. If the first use in the - // block is a load, then it uses the live in value. The last store defines - // the live out value. We handle this by doing a linear scan of the block. - BasicBlock *BB = User->getParent(); - Value *StoredValue = 0; - for (BasicBlock::iterator II = BB->begin(), E = BB->end(); II != E; ++II) { - if (LoadInst *L = dyn_cast(II)) { - // If this is a load from an unrelated pointer, ignore it. - if (!PointerMustAliases.count(L->getOperand(0))) continue; - - // If we haven't seen a store yet, this is a live in use, otherwise - // use the stored value. - if (StoredValue) { - L->replaceAllUsesWith(StoredValue); - ReplacedLoads[L] = StoredValue; - } else { - LiveInLoads.push_back(L); - } - continue; - } - - if (StoreInst *S = dyn_cast(II)) { - // If this is a store to an unrelated pointer, ignore it. - if (!PointerMustAliases.count(S->getOperand(1))) continue; - - // Remember that this is the active value in the block. - StoredValue = S->getOperand(0); - } - } - - // The last stored value that happened is the live-out for the block. - assert(StoredValue && "Already checked that there is a store in block"); - SSA.AddAvailableValue(BB, StoredValue); - BlockUses.clear(); - } - - // Now that all the intra-loop values are classified, set up the preheader. - // It gets a load of the pointer we're promoting, and it is the live-out value - // from the preheader. - LoadInst *PreheaderLoad = new LoadInst(SomePtr,SomePtr->getName()+".promoted", - Preheader->getTerminator()); + // Set up the preheader to have a definition of the value. It is the live-out + // value from the preheader that uses in the loop will use. + LoadInst *PreheaderLoad = + new LoadInst(SomePtr, SomePtr->getName()+".promoted", + Preheader->getTerminator()); SSA.AddAvailableValue(Preheader, PreheaderLoad); - // Now that the preheader is good to go, set up the exit blocks. Each exit - // block gets a store of the live-out values that feed them. Since we've - // already told the SSA updater about the defs in the loop and the preheader - // definition, it is all set and we can start using it. - SmallVector ExitBlocks; - CurLoop->getUniqueExitBlocks(ExitBlocks); - for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) { - BasicBlock *ExitBlock = ExitBlocks[i]; - Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock); - Instruction *InsertPos = ExitBlock->getFirstNonPHI(); - new StoreInst(LiveInValue, SomePtr, InsertPos); + // Copy any value stored to or loaded from a must-alias of the pointer. + if (PreheaderLoad->getType()->isPointerTy()) { + Value *SomeValue; + if (LoadInst *LI = dyn_cast(LoopUses[0])) + SomeValue = LI; + else + SomeValue = cast(LoopUses[0])->getValueOperand(); + + CurAST->copyValue(SomeValue, PreheaderLoad); } - // Okay, now we rewrite all loads that use live-in values in the loop, - // inserting PHI nodes as necessary. - for (unsigned i = 0, e = LiveInLoads.size(); i != e; ++i) { - LoadInst *ALoad = LiveInLoads[i]; - Value *NewVal = SSA.GetValueInMiddleOfBlock(ALoad->getParent()); - ALoad->replaceAllUsesWith(NewVal); - CurAST->copyValue(ALoad, NewVal); - ReplacedLoads[ALoad] = NewVal; - } + // Rewrite all the loads in the loop and remember all the definitions from + // stores in the loop. + Promoter.run(LoopUses); // If the preheader load is itself a pointer, we need to tell alias analysis // about the new pointer we created in the preheader block and about any PHI // nodes that just got inserted. if (PreheaderLoad->getType()->isPointerTy()) { - // Copy any value stored to or loaded from a must-alias of the pointer. - CurAST->copyValue(SomeValue, PreheaderLoad); - for (unsigned i = 0, e = NewPHIs.size(); i != e; ++i) - CurAST->copyValue(SomeValue, NewPHIs[i]); - } - - // Now that everything is rewritten, delete the old instructions from the body - // of the loop. They should all be dead now. - for (unsigned i = 0, e = LoopUses.size(); i != e; ++i) { - Instruction *User = LoopUses[i]; - - // If this is a load that still has uses, then the load must have been added - // as a live value in the SSAUpdate data structure for a block (e.g. because - // the loaded value was stored later). In this case, we need to recursively - // propagate the updates until we get to the real value. - if (!User->use_empty()) { - Value *NewVal = ReplacedLoads[User]; - assert(NewVal && "not a replaced load?"); - - // Propagate down to the ultimate replacee. The intermediately loads - // could theoretically already have been deleted, so we don't want to - // dereference the Value*'s. - DenseMap::iterator RLI = ReplacedLoads.find(NewVal); - while (RLI != ReplacedLoads.end()) { - NewVal = RLI->second; - RLI = ReplacedLoads.find(NewVal); - } - - User->replaceAllUsesWith(NewVal); - CurAST->copyValue(User, NewVal); - } - - CurAST->deleteValue(User); - User->eraseFromParent(); + CurAST->copyValue(PreheaderLoad, NewPHIs[i]); } // fwew, we're done! diff --git a/lib/Transforms/Scalar/ScalarReplAggregates.cpp b/lib/Transforms/Scalar/ScalarReplAggregates.cpp index 07d9787e663..13902552ebd 100644 --- a/lib/Transforms/Scalar/ScalarReplAggregates.cpp +++ b/lib/Transforms/Scalar/ScalarReplAggregates.cpp @@ -844,20 +844,13 @@ namespace { class AllocaPromoter : public LoadAndStorePromoter { AllocaInst *AI; public: - AllocaPromoter() : AI(0) {} + AllocaPromoter(const SmallVectorImpl &Insts, SSAUpdater &S) + : LoadAndStorePromoter(Insts, S), AI(0) {} - void run(AllocaInst *AI, SSAUpdater &SSA) { + void run(AllocaInst *AI, const SmallVectorImpl &Insts) { // Remember which alloca we're promoting (for isInstInList). this->AI = AI; - - // Build the list of instructions to promote. - SmallVector Insts; - for (Value::use_iterator UI = AI->use_begin(), E = AI->use_end(); - UI != E; ++UI) - Insts.push_back(cast(*UI)); - - LoadAndStorePromoter::run(AI->getName(), Insts, &SSA); - + LoadAndStorePromoter::run(Insts); AI->eraseFromParent(); } @@ -882,7 +875,7 @@ bool SROA::performPromotion(Function &F) { BasicBlock &BB = F.getEntryBlock(); // Get the entry node for the function bool Changed = false; - + SmallVector Insts; while (1) { Allocas.clear(); @@ -899,9 +892,17 @@ bool SROA::performPromotion(Function &F) { PromoteMemToReg(Allocas, *DT, *DF); else { SSAUpdater SSA; - AllocaPromoter Promoter; - for (unsigned i = 0, e = Allocas.size(); i != e; ++i) - Promoter.run(Allocas[i], SSA); + for (unsigned i = 0, e = Allocas.size(); i != e; ++i) { + AllocaInst *AI = Allocas[i]; + + // Build list of instructions to promote. + for (Value::use_iterator UI = AI->use_begin(), E = AI->use_end(); + UI != E; ++UI) + Insts.push_back(cast(*UI)); + + AllocaPromoter(Insts, SSA).run(AI, Insts); + Insts.clear(); + } } NumPromoted += Allocas.size(); Changed = true; diff --git a/lib/Transforms/Utils/SSAUpdater.cpp b/lib/Transforms/Utils/SSAUpdater.cpp index 2ecae356ff9..0474a7842d1 100644 --- a/lib/Transforms/Utils/SSAUpdater.cpp +++ b/lib/Transforms/Utils/SSAUpdater.cpp @@ -348,23 +348,25 @@ Value *SSAUpdater::GetValueAtEndOfBlockInternal(BasicBlock *BB) { // LoadAndStorePromoter Implementation //===----------------------------------------------------------------------===// -void LoadAndStorePromoter::run(StringRef BaseName, - const SmallVectorImpl &Insts, - SSAUpdater *SSA) { +LoadAndStorePromoter:: +LoadAndStorePromoter(const SmallVectorImpl &Insts, + SSAUpdater &S, StringRef BaseName) : SSA(S) { if (Insts.empty()) return; - // If no SSAUpdater was provided, use a default one. This allows the client - // to capture inserted PHI nodes etc if they want. - SSAUpdater DefaultSSA; - if (SSA == 0) SSA = &DefaultSSA; - - const Type *ValTy; + Value *SomeVal; if (LoadInst *LI = dyn_cast(Insts[0])) - ValTy = LI->getType(); + SomeVal = LI; else - ValTy = cast(Insts[0])->getOperand(0)->getType(); - - SSA->Initialize(ValTy, BaseName); + SomeVal = cast(Insts[0])->getOperand(0); + + if (BaseName.empty()) + BaseName = SomeVal->getName(); + SSA.Initialize(SomeVal->getType(), BaseName); +} + + +void LoadAndStorePromoter:: +run(const SmallVectorImpl &Insts) const { // First step: bucket up uses of the alloca by the block they occur in. // This is important because we have to handle multiple defs/uses in a block @@ -396,7 +398,7 @@ void LoadAndStorePromoter::run(StringRef BaseName, if (BlockUses.size() == 1) { // If it is a store, it is a trivial def of the value in the block. if (StoreInst *SI = dyn_cast(User)) - SSA->AddAvailableValue(BB, SI->getOperand(0)); + SSA.AddAvailableValue(BB, SI->getOperand(0)); else // Otherwise it is a load, queue it to rewrite as a live-in load. LiveInLoads.push_back(cast(User)); @@ -437,6 +439,7 @@ void LoadAndStorePromoter::run(StringRef BaseName, // If we haven't seen a store yet, this is a live in use, otherwise // use the stored value. if (StoredValue) { + replaceLoadWithValue(L, StoredValue); L->replaceAllUsesWith(StoredValue); ReplacedLoads[L] = StoredValue; } else { @@ -456,7 +459,7 @@ void LoadAndStorePromoter::run(StringRef BaseName, // The last stored value that happened is the live-out for the block. assert(StoredValue && "Already checked that there is a store in block"); - SSA->AddAvailableValue(BB, StoredValue); + SSA.AddAvailableValue(BB, StoredValue); BlockUses.clear(); } @@ -464,11 +467,15 @@ void LoadAndStorePromoter::run(StringRef BaseName, // inserting PHI nodes as necessary. for (unsigned i = 0, e = LiveInLoads.size(); i != e; ++i) { LoadInst *ALoad = LiveInLoads[i]; - Value *NewVal = SSA->GetValueInMiddleOfBlock(ALoad->getParent()); + Value *NewVal = SSA.GetValueInMiddleOfBlock(ALoad->getParent()); + replaceLoadWithValue(ALoad, NewVal); ALoad->replaceAllUsesWith(NewVal); ReplacedLoads[ALoad] = NewVal; } + // Allow the client to do stuff before we start nuking things. + doExtraRewritesBeforeFinalDeletion(); + // Now that everything is rewritten, delete the old instructions from the // function. They should all be dead now. for (unsigned i = 0, e = Insts.size(); i != e; ++i) { @@ -491,9 +498,11 @@ void LoadAndStorePromoter::run(StringRef BaseName, RLI = ReplacedLoads.find(NewVal); } + replaceLoadWithValue(cast(User), NewVal); User->replaceAllUsesWith(NewVal); } + instructionDeleted(User); User->eraseFromParent(); } }