diff --git a/lib/Transforms/Scalar/LoopRotation.cpp b/lib/Transforms/Scalar/LoopRotation.cpp index 3bc2011a47f..199c42009d2 100644 --- a/lib/Transforms/Scalar/LoopRotation.cpp +++ b/lib/Transforms/Scalar/LoopRotation.cpp @@ -80,6 +80,71 @@ bool LoopRotate::runOnLoop(Loop *L, LPPassManager &LPM) { return MadeChange; } +/// RewriteUsesOfClonedInstructions - We just cloned the instructions from the +/// old header into the preheader. If there were uses of the values produced by +/// these instruction that were outside of the loop, we have to insert PHI nodes +/// to merge the two values. Do this now. +static void RewriteUsesOfClonedInstructions(BasicBlock *OrigHeader, + BasicBlock *OrigPreheader, + ValueToValueMapTy &ValueMap) { + // Remove PHI node entries that are no longer live. + BasicBlock::iterator I, E = OrigHeader->end(); + for (I = OrigHeader->begin(); PHINode *PN = dyn_cast(I); ++I) + PN->removeIncomingValue(PN->getBasicBlockIndex(OrigPreheader)); + + // Now fix up users of the instructions in OrigHeader, inserting PHI nodes + // as necessary. + SSAUpdater SSA; + for (I = OrigHeader->begin(); I != E; ++I) { + Value *OrigHeaderVal = I; + + // If there are no uses of the value (e.g. because it returns void), there + // is nothing to rewrite. + if (OrigHeaderVal->use_empty()) + continue; + + Value *OrigPreHeaderVal = ValueMap[OrigHeaderVal]; + + // The value now exits in two versions: the initial value in the preheader + // and the loop "next" value in the original header. + SSA.Initialize(OrigHeaderVal->getType(), OrigHeaderVal->getName()); + SSA.AddAvailableValue(OrigHeader, OrigHeaderVal); + SSA.AddAvailableValue(OrigPreheader, OrigPreHeaderVal); + + // Visit each use of the OrigHeader instruction. + for (Value::use_iterator UI = OrigHeaderVal->use_begin(), + UE = OrigHeaderVal->use_end(); UI != UE; ) { + // Grab the use before incrementing the iterator. + Use &U = UI.getUse(); + + // Increment the iterator before removing the use from the list. + ++UI; + + // SSAUpdater can't handle a non-PHI use in the same block as an + // earlier def. We can easily handle those cases manually. + Instruction *UserInst = cast(U.getUser()); + if (!isa(UserInst)) { + BasicBlock *UserBB = UserInst->getParent(); + + // The original users in the OrigHeader are already using the + // original definitions. + if (UserBB == OrigHeader) + continue; + + // Users in the OrigPreHeader need to use the value to which the + // original definitions are mapped. + if (UserBB == OrigPreheader) { + U = OrigPreHeaderVal; + continue; + } + } + + // Anything else can be handled by SSAUpdater. + SSA.RewriteUse(U); + } + } +} + /// Rotate loop LP. Return true if the loop is rotated. bool LoopRotate::rotateLoop(Loop *L) { // If the loop has only one block then there is not much to rotate. @@ -116,9 +181,9 @@ bool LoopRotate::rotateLoop(Loop *L) { } // Now, this loop is suitable for rotation. - BasicBlock *OrigPreHeader = L->getLoopPreheader(); + BasicBlock *OrigPreheader = L->getLoopPreheader(); BasicBlock *OrigLatch = L->getLoopLatch(); - assert(OrigPreHeader && OrigLatch && "Loop not in canonical form?"); + assert(OrigPreheader && OrigLatch && "Loop not in canonical form?"); // Anything ScalarEvolution may know about this loop or the PHI nodes // in its header will soon be invalidated. @@ -150,11 +215,11 @@ bool LoopRotate::rotateLoop(Loop *L) { // For PHI nodes, the value available in OldPreHeader is just the // incoming value from OldPreHeader. for (; PHINode *PN = dyn_cast(I); ++I) - ValueMap[PN] = PN->getIncomingValue(PN->getBasicBlockIndex(OrigPreHeader)); + ValueMap[PN] = PN->getIncomingValue(PN->getBasicBlockIndex(OrigPreheader)); // For the rest of the instructions, either hoist to the OrigPreheader if // possible or create a clone in the OldPreHeader if not. - TerminatorInst *LoopEntryBranch = OrigPreHeader->getTerminator(); + TerminatorInst *LoopEntryBranch = OrigPreheader->getTerminator(); while (I != E) { Instruction *Inst = I++; @@ -202,90 +267,28 @@ bool LoopRotate::rotateLoop(Loop *L) { for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) for (BasicBlock::iterator BI = TI->getSuccessor(i)->begin(); PHINode *PN = dyn_cast(BI); ++BI) - PN->addIncoming(PN->getIncomingValueForBlock(OrigHeader), OrigPreHeader); + PN->addIncoming(PN->getIncomingValueForBlock(OrigHeader), OrigPreheader); // Now that OrigPreHeader has a clone of OrigHeader's terminator, remove // OrigPreHeader's old terminator (the original branch into the loop), and // remove the corresponding incoming values from the PHI nodes in OrigHeader. LoopEntryBranch->eraseFromParent(); - for (I = OrigHeader->begin(); PHINode *PN = dyn_cast(I); ++I) - PN->removeIncomingValue(PN->getBasicBlockIndex(OrigPreHeader)); - // Now fix up users of the instructions in OrigHeader, inserting PHI nodes - // as necessary. - SSAUpdater SSA; - for (I = OrigHeader->begin(); I != E; ++I) { - Value *OrigHeaderVal = I; - Value *OrigPreHeaderVal = ValueMap[OrigHeaderVal]; - - // If there are no uses of the value (e.g. because it returns void), there - // is nothing to rewrite. - if (OrigHeaderVal->use_empty() && OrigPreHeaderVal->use_empty()) - continue; - - // The value now exits in two versions: the initial value in the preheader - // and the loop "next" value in the original header. - SSA.Initialize(OrigHeaderVal->getType(), OrigHeaderVal->getName()); - SSA.AddAvailableValue(OrigHeader, OrigHeaderVal); - SSA.AddAvailableValue(OrigPreHeader, OrigPreHeaderVal); - - // Visit each use of the OrigHeader instruction. - for (Value::use_iterator UI = OrigHeaderVal->use_begin(), - UE = OrigHeaderVal->use_end(); UI != UE; ) { - // Grab the use before incrementing the iterator. - Use &U = UI.getUse(); - - // Increment the iterator before removing the use from the list. - ++UI; - - // SSAUpdater can't handle a non-PHI use in the same block as an - // earlier def. We can easily handle those cases manually. - Instruction *UserInst = cast(U.getUser()); - if (!isa(UserInst)) { - BasicBlock *UserBB = UserInst->getParent(); - - // The original users in the OrigHeader are already using the - // original definitions. - if (UserBB == OrigHeader) - continue; - - // Users in the OrigPreHeader need to use the value to which the - // original definitions are mapped. - if (UserBB == OrigPreHeader) { - U = OrigPreHeaderVal; - continue; - } - } - - // Anything else can be handled by SSAUpdater. - SSA.RewriteUse(U); - } - } + // If there were any uses of instructions in the duplicated block outside the + // loop, update them, inserting PHI nodes as required + RewriteUsesOfClonedInstructions(OrigHeader, OrigPreheader, ValueMap); // NewHeader is now the header of the loop. L->moveToHeader(NewHeader); assert(L->getHeader() == NewHeader && "Latch block is our new header"); - // Move the original header to the bottom of the loop, where it now more - // naturally belongs. This isn't necessary for correctness, and CodeGen can - // usually reorder blocks on its own to fix things like this up, but it's - // still nice to keep the IR readable. - // - // The original header should have only one predecessor at this point, since - // we checked that the loop had a proper preheader and unique backedge before - // we started. - assert(OrigHeader->getSinglePredecessor() && - "Original loop header has too many predecessors after loop rotation!"); - OrigHeader->moveAfter(OrigHeader->getSinglePredecessor()); - - // Update DominatorTree to reflect the CFG change we just made. Then split // edges as necessary to preserve LoopSimplify form. if (DominatorTree *DT = getAnalysisIfAvailable()) { // Since OrigPreheader now has the conditional branch to Exit block, it is // the dominator of Exit. - DT->changeImmediateDominator(Exit, OrigPreHeader); - DT->changeImmediateDominator(NewHeader, OrigPreHeader); + DT->changeImmediateDominator(Exit, OrigPreheader); + DT->changeImmediateDominator(NewHeader, OrigPreheader); // Update OrigHeader to be dominated by the new header block. DT->changeImmediateDominator(OrigHeader, OrigLatch); @@ -293,12 +296,13 @@ bool LoopRotate::rotateLoop(Loop *L) { // Right now OrigPreHeader has two successors, NewHeader and ExitBlock, and // thus is not a preheader anymore. Split the edge to form a real preheader. - BasicBlock *NewPH = SplitCriticalEdge(OrigPreHeader, NewHeader, this); + BasicBlock *NewPH = SplitCriticalEdge(OrigPreheader, NewHeader, this); NewPH->setName(NewHeader->getName() + ".lr.ph"); // Preserve canonical loop form, which means that 'Exit' should have only one // predecessor. - SplitCriticalEdge(L->getLoopLatch(), Exit, this); + BasicBlock *ExitSplit = SplitCriticalEdge(L->getLoopLatch(), Exit, this); + ExitSplit->moveBefore(Exit); assert(L->getLoopPreheader() == NewPH && "Invalid loop preheader after loop rotation");