From 953af2c3c560a13bd5eeb676c128b7e362dca684 Mon Sep 17 00:00:00 2001 From: Jakob Stoklund Olesen Date: Tue, 7 Dec 2010 23:18:47 +0000 Subject: [PATCH] Switch LiveIntervalUnion from std::set to IntervalMap. This speeds up RegAllocBasic by 20%, not counting releaseMemory which becomes way faster. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@121201 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/CodeGen/LiveIntervalUnion.cpp | 179 +++++++++--------------------- lib/CodeGen/LiveIntervalUnion.h | 92 ++++----------- lib/CodeGen/RegAllocBase.h | 9 +- lib/CodeGen/RegAllocBasic.cpp | 26 +++-- 4 files changed, 97 insertions(+), 209 deletions(-) diff --git a/lib/CodeGen/LiveIntervalUnion.cpp b/lib/CodeGen/LiveIntervalUnion.cpp index 9fb2a458428..bedf22b5bad 100644 --- a/lib/CodeGen/LiveIntervalUnion.cpp +++ b/lib/CodeGen/LiveIntervalUnion.cpp @@ -21,98 +21,50 @@ #include using namespace llvm; -// Find the first segment in the range [SegBegin,Segments.end()) that -// intersects with LS. If no intersection is found, return the first SI -// such that SI.start >= LS.End. -// -// This logic is tied to the underlying LiveSegments data structure. For now, we -// use set::upper_bound to find the nearest starting position, -// then reverse iterate to find the first overlap. -// -// Upon entry we have SegBegin.Start < LS.End -// SegBegin |--... -// \ . -// LS ...-| -// -// After set::upper_bound, we have SI.start >= LS.start: -// SI |--... -// / -// LS |--... -// -// Assuming intervals are disjoint, if an intersection exists, it must be the -// segment found or the one immediately preceeding it. We continue reverse -// iterating to return the first overlapping segment. -LiveIntervalUnion::SegmentIter -LiveIntervalUnion::upperBound(SegmentIter SegBegin, - const LiveSegment &LS) { - assert(LS.End > SegBegin->Start && "segment iterator precondition"); - - // Get the next LIU segment such that segI->Start is not less than seg.Start - // - // FIXME: Once we have a B+tree, we can make good use of SegBegin as a hint to - // upper_bound. For now, we're forced to search again from the root each time. - SegmentIter SI = Segments.upper_bound(LS); - while (SI != SegBegin) { - --SI; - if (LS.Start >= SI->End) - return ++SI; - } - return SI; -} // Merge a LiveInterval's segments. Guarantee no overlaps. -// -// After implementing B+tree, segments will be coalesced. void LiveIntervalUnion::unify(LiveInterval &VirtReg) { + if (VirtReg.empty()) + return; // Insert each of the virtual register's live segments into the map. - SegmentIter SegPos = Segments.begin(); - for (LiveInterval::iterator VirtRegI = VirtReg.begin(), - VirtRegEnd = VirtReg.end(); - VirtRegI != VirtRegEnd; ++VirtRegI ) { + LiveInterval::iterator RegPos = VirtReg.begin(); + LiveInterval::iterator RegEnd = VirtReg.end(); + SegmentIter SegPos = Segments.find(RegPos->start); - LiveSegment Seg(*VirtRegI, &VirtReg); - SegPos = Segments.insert(SegPos, Seg); - - assert(*SegPos == Seg && "need equal val for equal key"); -#ifndef NDEBUG - // Check for overlap (inductively). - if (SegPos != Segments.begin()) { - assert(llvm::prior(SegPos)->End <= Seg.Start && "overlapping segments" ); - } - SegmentIter NextPos = llvm::next(SegPos); - if (NextPos != Segments.end()) - assert(Seg.End <= NextPos->Start && "overlapping segments" ); -#endif // NDEBUG + for (;;) { + SegPos.insert(RegPos->start, RegPos->end, &VirtReg); + if (++RegPos == RegEnd) + return; + SegPos.advanceTo(RegPos->start); } } // Remove a live virtual register's segments from this union. -void LiveIntervalUnion::extract(const LiveInterval &VirtReg) { +void LiveIntervalUnion::extract(LiveInterval &VirtReg) { + if (VirtReg.empty()) + return; // Remove each of the virtual register's live segments from the map. - SegmentIter SegPos = Segments.begin(); - for (LiveInterval::const_iterator VirtRegI = VirtReg.begin(), - VirtRegEnd = VirtReg.end(); - VirtRegI != VirtRegEnd; ++VirtRegI) { + LiveInterval::iterator RegPos = VirtReg.begin(); + LiveInterval::iterator RegEnd = VirtReg.end(); + SegmentIter SegPos = Segments.find(RegPos->start); - LiveSegment Seg(*VirtRegI, const_cast(&VirtReg)); - SegPos = upperBound(SegPos, Seg); - assert(SegPos != Segments.end() && "missing VirtReg segment"); + for (;;) { + assert(SegPos.value() == &VirtReg && "Inconsistent LiveInterval"); + SegPos.erase(); + if (!SegPos.valid()) + return; - Segments.erase(SegPos++); + // Skip all segments that may have been coalesced. + RegPos = VirtReg.advanceTo(RegPos, SegPos.start()); + if (RegPos == RegEnd) + return; + + SegPos.advanceTo(RegPos->start); } } -raw_ostream& llvm::operator<<(raw_ostream& OS, const LiveSegment &LS) { - return OS << '[' << LS.Start << ',' << LS.End << ':' << - LS.VirtReg->reg << ")"; -} - -void LiveSegment::dump() const { - dbgs() << *this << "\n"; -} - void LiveIntervalUnion::print(raw_ostream &OS, const AbstractRegisterDescription *RegDesc) const { @@ -122,10 +74,9 @@ LiveIntervalUnion::print(raw_ostream &OS, else { OS << RepReg; } - for (LiveSegments::const_iterator SI = Segments.begin(), - SegEnd = Segments.end(); SI != SegEnd; ++SI) { - dbgs() << " " << *SI; - } + for (LiveSegments::const_iterator SI = Segments.begin(); SI.valid(); ++SI) + dbgs() << " [" << SI.start() << ' ' << SI.stop() << "):%reg" + << SI.value()->reg; OS << "\n"; } @@ -136,14 +87,8 @@ void LiveIntervalUnion::dump(const AbstractRegisterDescription *RegDesc) const { #ifndef NDEBUG // Verify the live intervals in this union and add them to the visited set. void LiveIntervalUnion::verify(LiveVirtRegBitSet& VisitedVRegs) { - SegmentIter SI = Segments.begin(); - SegmentIter SegEnd = Segments.end(); - if (SI == SegEnd) return; - VisitedVRegs.set(SI->VirtReg->reg); - for (++SI; SI != SegEnd; ++SI) { - VisitedVRegs.set(SI->VirtReg->reg); - assert(llvm::prior(SI)->End <= SI->Start && "overlapping segments" ); - } + for (SegmentIter SI = Segments.begin(); SI.valid(); ++SI) + VisitedVRegs.set(SI.value()->reg); } #endif //!NDEBUG @@ -169,36 +114,30 @@ void LiveIntervalUnion::Query::findIntersection(InterferenceResult &IR) const { // Search until reaching the end of the LiveUnion segments. LiveInterval::iterator VirtRegEnd = VirtReg->end(); - SegmentIter LiveUnionEnd = LiveUnion->end(); - while (IR.LiveUnionI != LiveUnionEnd) { - + while (IR.LiveUnionI.valid()) { // Slowly advance the live virtual reg iterator until we surpass the next // segment in LiveUnion. // // Note: If this is ever used for coalescing of fixed registers and we have // a live vreg with thousands of segments, then change this code to use // upperBound instead. - while (IR.VirtRegI != VirtRegEnd && - IR.VirtRegI->end <= IR.LiveUnionI->Start) - ++IR.VirtRegI; + IR.VirtRegI = VirtReg->advanceTo(IR.VirtRegI, IR.LiveUnionI.start()); if (IR.VirtRegI == VirtRegEnd) break; // Retain current (nonoverlapping) LiveUnionI - // VirtRegI may have advanced far beyond LiveUnionI, - // do a fast intersection test to "catch up" - LiveSegment Seg(*IR.VirtRegI, VirtReg); - IR.LiveUnionI = LiveUnion->upperBound(IR.LiveUnionI, Seg); + // VirtRegI may have advanced far beyond LiveUnionI, catch up. + IR.LiveUnionI.advanceTo(IR.VirtRegI->start); // Check if no LiveUnionI exists with VirtRegI->Start < LiveUnionI.end - if (IR.LiveUnionI == LiveUnionEnd) + if (!IR.LiveUnionI.valid()) break; - if (IR.LiveUnionI->Start < IR.VirtRegI->end) { - assert(overlap(*IR.VirtRegI, *IR.LiveUnionI) && + if (IR.LiveUnionI.start() < IR.VirtRegI->end) { + assert(overlap(*IR.VirtRegI, IR.LiveUnionI) && "upperBound postcondition"); break; } } - if (IR.LiveUnionI == LiveUnionEnd) + if (!IR.LiveUnionI.valid()) IR.VirtRegI = VirtRegEnd; } @@ -221,18 +160,18 @@ bool LiveIntervalUnion::Query::nextInterference(InterferenceResult &IR) const { // Advance either the VirtReg or LiveUnion segment to ensure that we visit all // unique overlapping pairs. - if (IR.VirtRegI->end < IR.LiveUnionI->End) { + if (IR.VirtRegI->end < IR.LiveUnionI.stop()) { if (++IR.VirtRegI == VirtReg->end()) return false; } else { - if (++IR.LiveUnionI == LiveUnion->end()) { + if (!(++IR.LiveUnionI).valid()) { IR.VirtRegI = VirtReg->end(); return false; } } // Short-circuit findIntersection() if possible. - if (overlap(*IR.VirtRegI, *IR.LiveUnionI)) + if (overlap(*IR.VirtRegI, IR.LiveUnionI)) return true; // Find the next intersection. @@ -261,55 +200,47 @@ unsigned LiveIntervalUnion::Query:: collectInterferingVRegs(unsigned MaxInterferingRegs) { InterferenceResult IR = firstInterference(); LiveInterval::iterator VirtRegEnd = VirtReg->end(); - SegmentIter LiveUnionEnd = LiveUnion->end(); LiveInterval *RecentInterferingVReg = NULL; - while (IR.LiveUnionI != LiveUnionEnd) { + while (IR.LiveUnionI.valid()) { // Advance the union's iterator to reach an unseen interfering vreg. do { - if (IR.LiveUnionI->VirtReg == RecentInterferingVReg) + if (IR.LiveUnionI.value() == RecentInterferingVReg) continue; - if (!isSeenInterference(IR.LiveUnionI->VirtReg)) + if (!isSeenInterference(IR.LiveUnionI.value())) break; // Cache the most recent interfering vreg to bypass isSeenInterference. - RecentInterferingVReg = IR.LiveUnionI->VirtReg; + RecentInterferingVReg = IR.LiveUnionI.value(); - } while( ++IR.LiveUnionI != LiveUnionEnd); - if (IR.LiveUnionI == LiveUnionEnd) + } while ((++IR.LiveUnionI).valid()); + if (!IR.LiveUnionI.valid()) break; // Advance the VirtReg iterator until surpassing the next segment in // LiveUnion. - // - // Note: If this is ever used for coalescing of fixed registers and we have - // a live virtual register with thousands of segments, then use upperBound - // instead. - while (IR.VirtRegI != VirtRegEnd && - IR.VirtRegI->end <= IR.LiveUnionI->Start) - ++IR.VirtRegI; + IR.VirtRegI = VirtReg->advanceTo(IR.VirtRegI, IR.LiveUnionI.start()); if (IR.VirtRegI == VirtRegEnd) break; // Check for intersection with the union's segment. - if (overlap(*IR.VirtRegI, *IR.LiveUnionI)) { + if (overlap(*IR.VirtRegI, IR.LiveUnionI)) { - if (!IR.LiveUnionI->VirtReg->isSpillable()) + if (!IR.LiveUnionI.value()->isSpillable()) SeenUnspillableVReg = true; - InterferingVRegs.push_back(IR.LiveUnionI->VirtReg); + InterferingVRegs.push_back(IR.LiveUnionI.value()); if (InterferingVRegs.size() == MaxInterferingRegs) return MaxInterferingRegs; // Cache the most recent interfering vreg to bypass isSeenInterference. - RecentInterferingVReg = IR.LiveUnionI->VirtReg; + RecentInterferingVReg = IR.LiveUnionI.value(); ++IR.LiveUnionI; continue; } // VirtRegI may have advanced far beyond LiveUnionI, // do a fast intersection test to "catch up" - LiveSegment Seg(*IR.VirtRegI, VirtReg); - IR.LiveUnionI = LiveUnion->upperBound(IR.LiveUnionI, Seg); + IR.LiveUnionI.advanceTo(IR.VirtRegI->start); } SeenAllInterferences = true; return InterferingVRegs.size(); diff --git a/lib/CodeGen/LiveIntervalUnion.h b/lib/CodeGen/LiveIntervalUnion.h index 445e7b3bf18..2068149ca0c 100644 --- a/lib/CodeGen/LiveIntervalUnion.h +++ b/lib/CodeGen/LiveIntervalUnion.h @@ -17,8 +17,8 @@ #ifndef LLVM_CODEGEN_LIVEINTERVALUNION #define LLVM_CODEGEN_LIVEINTERVALUNION +#include "llvm/ADT/IntervalMap.h" #include "llvm/CodeGen/LiveInterval.h" -#include namespace llvm { @@ -28,58 +28,6 @@ template class SparseBitVector; typedef SparseBitVector<128> LiveVirtRegBitSet; #endif -/// A LiveSegment is a copy of a LiveRange object used within -/// LiveIntervalUnion. LiveSegment additionally contains a pointer to its -/// original live virtual register (LiveInterval). This allows quick lookup of -/// the live virtual register as we iterate over live segments in a union. Note -/// that LiveRange is misnamed and actually represents only a single contiguous -/// interval within a virtual register's liveness. To limit confusion, in this -/// file we refer it as a live segment. -/// -/// Note: This currently represents a half-open interval [Start,End). -/// If LiveRange is modified to represent a closed interval, so should this. -struct LiveSegment { - SlotIndex Start; - SlotIndex End; - LiveInterval *VirtReg; - - LiveSegment(const LiveRange& LR, LiveInterval *VReg) - : Start(LR.start), End(LR.end), VirtReg(VReg) {} - - bool operator==(const LiveSegment &LS) const { - return Start == LS.Start && End == LS.End && VirtReg == LS.VirtReg; - } - - bool operator!=(const LiveSegment &LS) const { - return !operator==(LS); - } - - // Order segments by starting point only--we expect them to be disjoint. - bool operator<(const LiveSegment &LS) const { return Start < LS.Start; } - - void dump() const; - void print(raw_ostream &OS) const; -}; - -inline bool operator<(SlotIndex Idx, const LiveSegment &LS) { - return Idx < LS.Start; -} - -inline bool operator<(const LiveSegment &LS, SlotIndex Idx) { - return LS.Start < Idx; -} - -/// Compare a live virtual register segment to a LiveIntervalUnion segment. -inline bool overlap(const LiveRange &VirtRegSegment, - const LiveSegment &LiveUnionSegment) { - return VirtRegSegment.start < LiveUnionSegment.End && - LiveUnionSegment.Start < VirtRegSegment.end; -} - -template <> struct isPodLike { static const bool value = true; }; - -raw_ostream& operator<<(raw_ostream& OS, const LiveSegment &LS); - /// Abstraction to provide info for the representative register. class AbstractRegisterDescription { public: @@ -87,6 +35,13 @@ public: virtual ~AbstractRegisterDescription() {} }; +/// Compare a live virtual register segment to a LiveIntervalUnion segment. +inline bool +overlap(const LiveRange &VRSeg, + const IntervalMap::const_iterator &LUSeg) { + return VRSeg.start < LUSeg.stop() && LUSeg.start() < VRSeg.end; +} + /// Union of live intervals that are strong candidates for coalescing into a /// single register (either physical or virtual depending on the context). We /// expect the constituent live intervals to be disjoint, although we may @@ -94,10 +49,8 @@ public: class LiveIntervalUnion { // A set of live virtual register segments that supports fast insertion, // intersection, and removal. - // - // FIXME: std::set is a placeholder until we decide how to - // efficiently represent it. Probably need to roll our own B-tree. - typedef std::set LiveSegments; + // Mapping SlotIndex intervals to virtual register numbers. + typedef IntervalMap LiveSegments; public: // SegmentIter can advance to the next segment ordered by starting position @@ -105,36 +58,29 @@ public: // to reach the current segment's containing virtual register. typedef LiveSegments::iterator SegmentIter; + // LiveIntervalUnions share an external allocator. + typedef LiveSegments::Allocator Allocator; + class InterferenceResult; class Query; private: - unsigned RepReg; // representative register number - LiveSegments Segments; // union of virtual reg segements + const unsigned RepReg; // representative register number + LiveSegments Segments; // union of virtual reg segments public: - // default ctor avoids placement new - LiveIntervalUnion() : RepReg(0) {} - - // Initialize the union by associating it with a representative register - // number. - void init(unsigned Reg) { RepReg = Reg; } + LiveIntervalUnion(unsigned r, Allocator &a) : RepReg(r), Segments(a) {} // Iterate over all segments in the union of live virtual registers ordered // by their starting position. SegmentIter begin() { return Segments.begin(); } SegmentIter end() { return Segments.end(); } - // Return an iterator to the first segment after or including begin that - // intersects with LS. - SegmentIter upperBound(SegmentIter SegBegin, const LiveSegment &LS); - // Add a live virtual register to this union and merge its segments. - // Holds a nonconst reference to the VirtReg for later maniplution. void unify(LiveInterval &VirtReg); // Remove a live virtual register's segments from this union. - void extract(const LiveInterval &VirtReg); + void extract(LiveInterval &VirtReg); void dump(const AbstractRegisterDescription *RegDesc) const; @@ -171,7 +117,7 @@ public: LiveInterval::iterator virtRegPos() const { return VirtRegI; } // Access the LiveUnion segment. - SegmentIter liveUnionPos() const { return LiveUnionI; } + const SegmentIter &liveUnionPos() const { return LiveUnionI; } bool operator==(const InterferenceResult &IR) const { return VirtRegI == IR.VirtRegI && LiveUnionI == IR.LiveUnionI; @@ -228,7 +174,7 @@ public: bool isInterference(const InterferenceResult &IR) const { if (IR.VirtRegI != VirtReg->end()) { - assert(overlap(*IR.VirtRegI, *IR.LiveUnionI) && + assert(overlap(*IR.VirtRegI, IR.LiveUnionI) && "invalid segment iterators"); return true; } diff --git a/lib/CodeGen/RegAllocBase.h b/lib/CodeGen/RegAllocBase.h index 8044a192385..32f5e0870ed 100644 --- a/lib/CodeGen/RegAllocBase.h +++ b/lib/CodeGen/RegAllocBase.h @@ -38,6 +38,7 @@ #define LLVM_CODEGEN_REGALLOCBASE #include "llvm/ADT/OwningPtr.h" +#include "LiveIntervalUnion.h" namespace llvm { @@ -69,17 +70,19 @@ class LiveVirtRegQueue; /// live range splitting. LessSpillWeightPriority is provided as a standard /// comparator, but we may add an interface to override it if necessary. class RegAllocBase { + LiveIntervalUnion::Allocator UnionAllocator; protected: // Array of LiveIntervalUnions indexed by physical register. class LiveUnionArray { unsigned NumRegs; - OwningArrayPtr Array; + LiveIntervalUnion *Array; public: - LiveUnionArray(): NumRegs(0) {} + LiveUnionArray(): NumRegs(0), Array(0) {} + ~LiveUnionArray() { clear(); } unsigned numRegs() const { return NumRegs; } - void init(unsigned NRegs); + void init(LiveIntervalUnion::Allocator &, unsigned NRegs); void clear(); diff --git a/lib/CodeGen/RegAllocBasic.cpp b/lib/CodeGen/RegAllocBasic.cpp index 96c8076c2d6..f8eafe42009 100644 --- a/lib/CodeGen/RegAllocBasic.cpp +++ b/lib/CodeGen/RegAllocBasic.cpp @@ -19,6 +19,7 @@ #include "Spiller.h" #include "VirtRegMap.h" #include "VirtRegRewriter.h" +#include "llvm/ADT/OwningPtr.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Function.h" #include "llvm/PassAnalysisSupport.h" @@ -44,6 +45,7 @@ #include #include +#include using namespace llvm; @@ -198,12 +200,13 @@ void RegAllocBase::verify() { //===----------------------------------------------------------------------===// // Instantiate a LiveIntervalUnion for each physical register. -void RegAllocBase::LiveUnionArray::init(unsigned NRegs) { - Array.reset(new LiveIntervalUnion[NRegs]); +void RegAllocBase::LiveUnionArray::init(LiveIntervalUnion::Allocator &allocator, + unsigned NRegs) { NumRegs = NRegs; - for (unsigned RegNum = 0; RegNum < NRegs; ++RegNum) { - Array[RegNum].init(RegNum); - } + Array = + static_cast(malloc(sizeof(LiveIntervalUnion)*NRegs)); + for (unsigned r = 0; r != NRegs; ++r) + new(Array + r) LiveIntervalUnion(r, allocator); } void RegAllocBase::init(const TargetRegisterInfo &tri, VirtRegMap &vrm, @@ -211,14 +214,19 @@ void RegAllocBase::init(const TargetRegisterInfo &tri, VirtRegMap &vrm, TRI = &tri; VRM = &vrm; LIS = &lis; - PhysReg2LiveUnion.init(TRI->getNumRegs()); + PhysReg2LiveUnion.init(UnionAllocator, TRI->getNumRegs()); // Cache an interferece query for each physical reg Queries.reset(new LiveIntervalUnion::Query[PhysReg2LiveUnion.numRegs()]); } void RegAllocBase::LiveUnionArray::clear() { + if (!Array) + return; + for (unsigned r = 0; r != NumRegs; ++r) + Array[r].~LiveIntervalUnion(); + free(Array); NumRegs = 0; - Array.reset(0); + Array = 0; } void RegAllocBase::releaseMemory() { @@ -427,7 +435,7 @@ unsigned RABasic::selectOrSplit(LiveInterval &VirtReg, return PhysReg; } LiveInterval *interferingVirtReg = - Queries[interfReg].firstInterference().liveUnionPos()->VirtReg; + Queries[interfReg].firstInterference().liveUnionPos().value(); // The current VirtReg must either spillable, or one of its interferences // must have less spill weight. @@ -474,7 +482,7 @@ void RABasic::addMBBLiveIns() { // Find the set of basic blocks which this range is live into... liveInMBBs.clear(); - if (!LIS->findLiveInMBBs(SI->Start, SI->End, liveInMBBs)) continue; + if (!LIS->findLiveInMBBs(SI.start(), SI.stop(), liveInMBBs)) continue; // And add the physreg for this interval to their live-in sets. for (MBBVec::iterator I = liveInMBBs.begin(), E = liveInMBBs.end();