diff --git a/lib/CodeGen/MachineSink.cpp b/lib/CodeGen/MachineSink.cpp index aed0e500d44..01eb87225d9 100644 --- a/lib/CodeGen/MachineSink.cpp +++ b/lib/CodeGen/MachineSink.cpp @@ -73,6 +73,11 @@ namespace { SparseBitVector<> RegsToClearKillFlags; + // Cache all successors to a MachineBasicBlock, sorted by frequency info and + // loop depth. AllSuccessors is lazily populated. + std::map> + AllSuccessors; + public: static char ID; // Pass identification MachineSinking() : MachineFunctionPass(ID) { @@ -132,6 +137,9 @@ namespace { bool PerformTrivialForwardCoalescing(MachineInstr *MI, MachineBasicBlock *MBB); + + SmallVector & + GetAllSortedSuccessors(MachineInstr *MI, MachineBasicBlock *MBB); }; } // end anonymous namespace @@ -269,9 +277,8 @@ bool MachineSinking::runOnMachineFunction(MachineFunction &MF) { // Process all basic blocks. CEBCandidates.clear(); ToSplit.clear(); - for (MachineFunction::iterator I = MF.begin(), E = MF.end(); - I != E; ++I) - MadeChange |= ProcessBlock(*I); + for (auto &MBB: MF) + MadeChange |= ProcessBlock(MBB); // If we have anything we marked as toSplit, split it now. for (auto &Pair : ToSplit) { @@ -310,6 +317,9 @@ bool MachineSinking::ProcessBlock(MachineBasicBlock &MBB) { bool MadeChange = false; + // MBB changed, reset all cached information. + AllSuccessors.clear(); + // Walk the basic block bottom-up. Remember if we saw a store. MachineBasicBlock::iterator I = MBB.end(); --I; @@ -522,6 +532,51 @@ bool MachineSinking::isProfitableToSinkTo(unsigned Reg, MachineInstr *MI, return false; } +/// Get the sorted sequence of successors for this MachineBasicBlock, possibly +/// computing it if it was not already cached. +SmallVector & +MachineSinking::GetAllSortedSuccessors(MachineInstr *MI, MachineBasicBlock *MBB) { + + // Do we have the sorted successors in cache ? + auto Succs = AllSuccessors.find(MBB); + if (Succs != AllSuccessors.end()) + return Succs->second; + + SmallVector AllSuccs(MBB->succ_begin(), + MBB->succ_end()); + + // Handle cases where sinking can happen but where the sink point isn't a + // successor. For example: + // + // x = computation + // if () {} else {} + // use x + // + const std::vector &Children = + DT->getNode(MBB)->getChildren(); + for (const auto &DTChild : Children) + // DomTree children of MBB that have MBB as immediate dominator are added. + if (DTChild->getIDom()->getBlock() == MI->getParent() && + // Skip MBBs already added to the AllSuccs vector above. + !MBB->isSuccessor(DTChild->getBlock())) + AllSuccs.push_back(DTChild->getBlock()); + + // Sort Successors according to their loop depth or block frequency info. + std::stable_sort( + AllSuccs.begin(), AllSuccs.end(), + [this](const MachineBasicBlock *L, const MachineBasicBlock *R) { + uint64_t LHSFreq = MBFI ? MBFI->getBlockFreq(L).getFrequency() : 0; + uint64_t RHSFreq = MBFI ? MBFI->getBlockFreq(R).getFrequency() : 0; + bool HasBlockFreq = LHSFreq != 0 && RHSFreq != 0; + return HasBlockFreq ? LHSFreq < RHSFreq + : LI->getLoopDepth(L) < LI->getLoopDepth(R); + }); + + auto it = AllSuccessors.insert(std::make_pair(MBB, AllSuccs)); + + return it.first->second; +} + /// FindSuccToSinkTo - Find a successor to sink this instruction to. MachineBasicBlock *MachineSinking::FindSuccToSinkTo(MachineInstr *MI, MachineBasicBlock *MBB, @@ -579,38 +634,7 @@ MachineBasicBlock *MachineSinking::FindSuccToSinkTo(MachineInstr *MI, // we should sink to. If we have reliable block frequency information // (frequency != 0) available, give successors with smaller frequencies // higher priority, otherwise prioritize smaller loop depths. - SmallVector Succs(MBB->succ_begin(), - MBB->succ_end()); - - // Handle cases where sinking can happen but where the sink point isn't a - // successor. For example: - // - // x = computation - // if () {} else {} - // use x - // - const std::vector &Children = - DT->getNode(MBB)->getChildren(); - for (const auto &DTChild : Children) - // DomTree children of MBB that have MBB as immediate dominator are added. - if (DTChild->getIDom()->getBlock() == MI->getParent() && - // Skip MBBs already added to the Succs vector above. - !MBB->isSuccessor(DTChild->getBlock())) - Succs.push_back(DTChild->getBlock()); - - // Sort Successors according to their loop depth or block frequency info. - std::stable_sort( - Succs.begin(), Succs.end(), - [this](const MachineBasicBlock *L, const MachineBasicBlock *R) { - uint64_t LHSFreq = MBFI ? MBFI->getBlockFreq(L).getFrequency() : 0; - uint64_t RHSFreq = MBFI ? MBFI->getBlockFreq(R).getFrequency() : 0; - bool HasBlockFreq = LHSFreq != 0 && RHSFreq != 0; - return HasBlockFreq ? LHSFreq < RHSFreq - : LI->getLoopDepth(L) < LI->getLoopDepth(R); - }); - for (SmallVectorImpl::iterator SI = Succs.begin(), - E = Succs.end(); SI != E; ++SI) { - MachineBasicBlock *SuccBlock = *SI; + for (MachineBasicBlock *SuccBlock : GetAllSortedSuccessors(MI, MBB)) { bool LocalUse = false; if (AllUsesDominatedByBlock(Reg, SuccBlock, MBB, BreakPHIEdge, LocalUse)) {