diff --git a/lib/CodeGen/LiveIntervalUnion.cpp b/lib/CodeGen/LiveIntervalUnion.cpp index ec502cd51ec..9d4de6a7c24 100644 --- a/lib/CodeGen/LiveIntervalUnion.cpp +++ b/lib/CodeGen/LiveIntervalUnion.cpp @@ -15,6 +15,7 @@ #define DEBUG_TYPE "regalloc" #include "LiveIntervalUnion.h" +#include "llvm/ADT/SparseBitVector.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include @@ -73,12 +74,9 @@ void LiveIntervalUnion::unify(LiveInterval &lvr) { #ifndef NDEBUG // check for overlap (inductively) if (segPos != segments_.begin()) { - SegmentIter prevPos = segPos; - --prevPos; - assert(prevPos->end <= segment.start && "overlapping segments" ); + assert(prior(segPos)->end <= segment.start && "overlapping segments" ); } - SegmentIter nextPos = segPos; - ++nextPos; + SegmentIter nextPos = next(segPos); if (nextPos != segments_.end()) assert(segment.end <= nextPos->start && "overlapping segments" ); #endif // NDEBUG @@ -98,6 +96,49 @@ void LiveIntervalUnion::extract(const LiveInterval &lvr) { } } +raw_ostream& llvm::operator<<(raw_ostream& os, const LiveSegment &ls) { + return os << '[' << ls.start << ',' << ls.end << ':' << + ls.liveVirtReg->reg << ")"; +} + +void LiveSegment::dump() const { + dbgs() << *this << "\n"; +} + +void +LiveIntervalUnion::print(raw_ostream &os, + const AbstractRegisterDescription *rdesc) const { + os << "LIU "; + if (rdesc != NULL) + os << rdesc->getName(repReg_); + else { + os << repReg_; + } + for (SegmentIter segI = segments_.begin(), segEnd = segments_.end(); + segI != segEnd; ++segI) { + dbgs() << " " << *segI; + } + os << "\n"; +} + +void LiveIntervalUnion::dump(const AbstractRegisterDescription *rdesc) const { + print(dbgs(), rdesc); +} + +#ifndef NDEBUG +// Verify the live intervals in this union and add them to the visited set. +void LiveIntervalUnion::verify(LvrBitSet& visitedVRegs) { + SegmentIter segI = segments_.begin(); + SegmentIter segEnd = segments_.end(); + if (segI == segEnd) return; + visitedVRegs.set(segI->liveVirtReg->reg); + for (++segI; segI != segEnd; ++segI) { + visitedVRegs.set(segI->liveVirtReg->reg); + assert(prior(segI)->end <= segI->start && "overlapping segments" ); + } +} +#endif //!NDEBUG + // Private interface accessed by Query. // // Find a pair of segments that intersect, one in the live virtual register diff --git a/lib/CodeGen/LiveIntervalUnion.h b/lib/CodeGen/LiveIntervalUnion.h index f0bce470875..4d379cfa115 100644 --- a/lib/CodeGen/LiveIntervalUnion.h +++ b/lib/CodeGen/LiveIntervalUnion.h @@ -23,6 +23,12 @@ namespace llvm { +#ifndef NDEBUG +// forward declaration +template class SparseBitVector; +typedef SparseBitVector<128> LvrBitSet; +#endif + /// A LiveSegment is a copy of a LiveRange object used within /// LiveIntervalUnion. LiveSegment additionally contains a pointer to its /// original live virtual register (LiveInterval). This allows quick lookup of @@ -51,6 +57,9 @@ struct LiveSegment { // Order segments by starting point only--we expect them to be disjoint. bool operator<(const LiveSegment &ls) const { return start < ls.start; } + + void dump() const; + void print(raw_ostream &os) const; }; inline bool operator<(SlotIndex V, const LiveSegment &ls) { @@ -66,6 +75,16 @@ inline bool overlap(const LiveRange &lvrSeg, const LiveSegment &liuSeg) { return lvrSeg.start < liuSeg.end && liuSeg.start < lvrSeg.end; } +template <> struct isPodLike { static const bool value = true; }; + +raw_ostream& operator<<(raw_ostream& os, const LiveSegment &ls); + +/// Abstraction to provide info for the representative register. +class AbstractRegisterDescription { +public: + virtual const char *getName(unsigned reg) const = 0; +}; + /// Union of live intervals that are strong candidates for coalescing into a /// single register (either physical or virtual depending on the context). We /// expect the constituent live intervals to be disjoint, although we may @@ -122,6 +141,16 @@ public: // Remove a live virtual register's segments from this union. void extract(const LiveInterval &lvr); + void dump(const AbstractRegisterDescription *regInfo) const; + + // If tri != NULL, use it to decode repReg_ + void print(raw_ostream &os, const AbstractRegisterDescription *rdesc) const; + +#ifndef NDEBUG + // Verify the live intervals in this union and add them to the visited set. + void verify(LvrBitSet& visitedVRegs); +#endif + /// Cache a single interference test result in the form of two intersecting /// segments. This allows efficiently iterating over the interferences. The /// iteration logic is handled by LiveIntervalUnion::Query which may diff --git a/lib/CodeGen/RegAllocBase.h b/lib/CodeGen/RegAllocBase.h index f4ca9727384..fa595fed8b5 100644 --- a/lib/CodeGen/RegAllocBase.h +++ b/lib/CodeGen/RegAllocBase.h @@ -128,6 +128,11 @@ protected: // exists, return the interfering register, which may be preg or an alias. unsigned checkPhysRegInterference(LiveInterval& lvr, unsigned preg); +#ifndef NDEBUG + // Verify each LiveIntervalUnion. + void verify(); +#endif + // Helper that spills all live virtual registers currently unified under preg // that interfere with the most recently queried lvr. void spillInterferences(unsigned preg, diff --git a/lib/CodeGen/RegAllocBasic.cpp b/lib/CodeGen/RegAllocBasic.cpp index a2f9ea274b9..cd77d136407 100644 --- a/lib/CodeGen/RegAllocBasic.cpp +++ b/lib/CodeGen/RegAllocBasic.cpp @@ -34,6 +34,9 @@ #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetRegisterInfo.h" +#ifndef NDEBUG +#include "llvm/ADT/SparseBitVector.h" +#endif #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" @@ -46,6 +49,19 @@ using namespace llvm; static RegisterRegAlloc basicRegAlloc("basic", "basic register allocator", createBasicRegisterAllocator); +// Temporary verification option until we can put verification inside +// MachineVerifier. +static cl::opt +VerifyRegAlloc("verify-regalloc", + cl::desc("Verify live intervals before renaming")); + +class PhysicalRegisterDescription : public AbstractRegisterDescription { + const TargetRegisterInfo *tri_; +public: + PhysicalRegisterDescription(const TargetRegisterInfo *tri): tri_(tri) {} + virtual const char *getName(unsigned reg) const { return tri_->getName(reg); } +}; + namespace { /// RABasic provides a minimal implementation of the basic register allocation @@ -153,6 +169,40 @@ void RABasic::releaseMemory() { RegAllocBase::releaseMemory(); } +#ifndef NDEBUG +// Verify each LiveIntervalUnion. +void RegAllocBase::verify() { + LvrBitSet visitedVRegs; + OwningArrayPtr unionVRegs(new LvrBitSet[physReg2liu_.numRegs()]); + // Verify disjoint unions. + for (unsigned preg = 0; preg < physReg2liu_.numRegs(); ++preg) { + DEBUG(PhysicalRegisterDescription prd(tri_); physReg2liu_[preg].dump(&prd)); + LvrBitSet &vregs = unionVRegs[preg]; + physReg2liu_[preg].verify(vregs); + // Union + intersection test could be done efficiently in one pass, but + // don't add a method to SparseBitVector unless we really need it. + assert(!visitedVRegs.intersects(vregs) && "vreg in multiple unions"); + visitedVRegs |= vregs; + } + // Verify vreg coverage. + for (LiveIntervals::iterator liItr = lis_->begin(), liEnd = lis_->end(); + liItr != liEnd; ++liItr) { + unsigned reg = liItr->first; + LiveInterval &li = *liItr->second; + if (li.empty() ) continue; + if (TargetRegisterInfo::isPhysicalRegister(reg)) continue; + if (!vrm_->hasPhys(reg)) continue; // spilled? + unsigned preg = vrm_->getPhys(reg); + if (!unionVRegs[preg].test(reg)) { + dbgs() << "LiveVirtReg " << reg << " not in union " << + tri_->getName(preg) << "\n"; + llvm_unreachable("unallocated live vreg"); + } + } + // FIXME: I'm not sure how to verify spilled intervals. +} +#endif //!NDEBUG + //===----------------------------------------------------------------------===// // RegAllocBase Implementation //===----------------------------------------------------------------------===// @@ -222,6 +272,7 @@ void RegAllocBase::seedLiveVirtRegs(LiveVirtRegQueue &lvrQ) { liItr != liEnd; ++liItr) { unsigned reg = liItr->first; LiveInterval &li = *liItr->second; + if (li.empty()) continue; if (TargetRegisterInfo::isPhysicalRegister(reg)) { physReg2liu_[reg].unify(li); } @@ -243,13 +294,14 @@ void RegAllocBase::allocatePhysRegs() { unsigned availablePhysReg = selectOrSplit(*lvr, splitLVRs); if (availablePhysReg) { DEBUG(dbgs() << "allocating: " << tri_->getName(availablePhysReg) << - " " << lvr << '\n'); + " " << *lvr << '\n'); assert(!vrm_->hasPhys(lvr->reg) && "duplicate vreg in interval unions"); vrm_->assignVirt2Phys(lvr->reg, availablePhysReg); physReg2liu_[availablePhysReg].unify(*lvr); } for (LVRVec::iterator lvrI = splitLVRs.begin(), lvrEnd = splitLVRs.end(); lvrI != lvrEnd; ++lvrI) { + if ((*lvrI)->empty()) continue; DEBUG(dbgs() << "queuing new interval: " << **lvrI << "\n"); assert(TargetRegisterInfo::isVirtualRegister((*lvrI)->reg) && "expect split value in virtual register"); @@ -274,26 +326,32 @@ unsigned RegAllocBase::checkPhysRegInterference(LiveInterval &lvr, return 0; } -// Spill all live virtual registers currently unified under preg that interfere -// with lvr. +// Spill or split all live virtual registers currently unified under preg that +// interfere with lvr. The newly spilled or split live intervals are returned by +// appending them to splitLVRs. void RABasic::spillInterferences(unsigned preg, SmallVectorImpl &splitLVRs) { SmallPtrSet spilledLVRs; LiveIntervalUnion::Query &query = queries_[preg]; + // Record each interference before mutating either the union or live + // intervals. LiveIntervalUnion::InterferenceResult ir = query.firstInterference(); assert(query.isInterference(ir) && "expect interference"); do { - LiveInterval *lvr = ir.liuSegPos()->liveVirtReg; - if (!spilledLVRs.insert(lvr)) continue; - // Spill the previously allocated lvr. - SmallVector spillIs; // ignored - spiller_->spill(lvr, splitLVRs, spillIs); + spilledLVRs.insert(ir.liuSegPos()->liveVirtReg); } while (query.nextInterference(ir)); for (SmallPtrSetIterator lvrI = spilledLVRs.begin(), lvrEnd = spilledLVRs.end(); lvrI != lvrEnd; ++lvrI ) { + LiveInterval& lvr = **lvrI; + // Spill the previously allocated lvr. + DEBUG(dbgs() << "extracting from " << preg << " " << lvr << '\n'); // Deallocate the interfering lvr by removing it from the preg union. - physReg2liu_[preg].extract(**lvrI); + // Live intervals may not be in a union during modification. + physReg2liu_[preg].extract(lvr); + // Spill the extracted interval. + SmallVector spillIs; + spiller_->spill(&lvr, splitLVRs, spillIs); } // After extracting segments, the query's results are invalid. query.clear(); @@ -399,6 +457,24 @@ bool RABasic::runOnMachineFunction(MachineFunction &mf) { // optional HTML output DEBUG(rmf_->renderMachineFunction("After basic register allocation.", vrm_)); + // FIXME: Verification currently must run before VirtRegRewriter. We should + // make the rewriter a separate pass and override verifyAnalysis instead. When + // that happens, verification naturally falls under VerifyMachineCode. +#ifndef NDEBUG + if (VerifyRegAlloc) { + // Verify accuracy of LiveIntervals. The standard machine code verifier + // ensures that each LiveIntervals covers all uses of the virtual reg. + + // FIXME: MachineVerifier is currently broken when using the standard + // spiller. Enable it for InlineSpiller only. + // mf_->verify(this); + + // Verify that LiveIntervals are partitioned into unions and disjoint within + // the unions. + verify(); + } +#endif // !NDEBUG + // Run rewriter std::auto_ptr rewriter(createVirtRegRewriter()); rewriter->runOnMachineFunction(*mf_, *vrm_, lis_);