diff --git a/include/llvm/CodeGen/LiveInterval.h b/include/llvm/CodeGen/LiveInterval.h index 058f95e24e6..38d07e4612b 100644 --- a/include/llvm/CodeGen/LiveInterval.h +++ b/include/llvm/CodeGen/LiveInterval.h @@ -296,6 +296,12 @@ namespace llvm { return r != end() && r->end.getBaseIndex() == BaseIdx; } + /// Return true if a live range starts at the instruction at this index. + bool isDefinedByInstr(SlotIndex index) const { + const_iterator r = find(index.getDeadSlot()); + return r != end() && r->end.getBaseIndex() == index.getBaseIndex(); + } + /// getLiveRangeContaining - Return the live range that contains the /// specified index, or null if there is none. const LiveRange *getLiveRangeContaining(SlotIndex Idx) const { diff --git a/include/llvm/CodeGen/MachineScheduler.h b/include/llvm/CodeGen/MachineScheduler.h index 8f307f91a54..69c648c8c1a 100644 --- a/include/llvm/CodeGen/MachineScheduler.h +++ b/include/llvm/CodeGen/MachineScheduler.h @@ -221,7 +221,9 @@ protected: MachineBasicBlock::iterator LiveRegionEnd; - // Map each SU to its summary of pressure changes. + // Map each SU to its summary of pressure changes. This array is updated for + // liveness during bottom-up scheduling. Top-down scheduling may proceed but + // has no affect on the pressure diffs. PressureDiffs SUPressureDiffs; /// Register pressure in this region computed by initRegPressure. @@ -376,6 +378,8 @@ protected: void initRegPressure(); + void updatePressureDiffs(ArrayRef LiveUses); + void updateScheduledPressure(const std::vector &NewMaxPressure); bool checkSchedLimit(); diff --git a/include/llvm/CodeGen/RegisterPressure.h b/include/llvm/CodeGen/RegisterPressure.h index 68f2ac23494..e01e4ec2166 100644 --- a/include/llvm/CodeGen/RegisterPressure.h +++ b/include/llvm/CodeGen/RegisterPressure.h @@ -311,7 +311,7 @@ public: SlotIndex getCurrSlot() const; /// Recede across the previous instruction. - bool recede(PressureDiff *PDiff = 0); + bool recede(SmallVectorImpl *LiveUses = 0, PressureDiff *PDiff = 0); /// Advance across the current instruction. bool advance(); diff --git a/lib/CodeGen/MachineScheduler.cpp b/lib/CodeGen/MachineScheduler.cpp index e233d4af440..a7a3bb80730 100644 --- a/lib/CodeGen/MachineScheduler.cpp +++ b/lib/CodeGen/MachineScheduler.cpp @@ -156,8 +156,9 @@ static ScheduleDAGInstrs *createConvergingSched(MachineSchedContext *C); /// Decrement this iterator until reaching the top or a non-debug instr. -static MachineBasicBlock::iterator -priorNonDebug(MachineBasicBlock::iterator I, MachineBasicBlock::iterator Beg) { +static MachineBasicBlock::const_iterator +priorNonDebug(MachineBasicBlock::const_iterator I, + MachineBasicBlock::const_iterator Beg) { assert(I != Beg && "reached the top of the region, cannot decrement"); while (--I != Beg) { if (!I->isDebugValue()) @@ -166,6 +167,14 @@ priorNonDebug(MachineBasicBlock::iterator I, MachineBasicBlock::iterator Beg) { return I; } +/// Non-const version. +static MachineBasicBlock::iterator +priorNonDebug(MachineBasicBlock::iterator I, + MachineBasicBlock::const_iterator Beg) { + return const_cast( + &*priorNonDebug(MachineBasicBlock::const_iterator(I), Beg)); +} + /// If this iterator is a debug value, increment until reaching the End or a /// non-debug instruction. static MachineBasicBlock::iterator @@ -488,9 +497,16 @@ void ScheduleDAGMI::initRegPressure() { dumpRegSetPressure(BotRPTracker.getLiveThru(), TRI)); }; + // For each live out vreg reduce the pressure change associated with other + // uses of the same vreg below the live-out reaching def. + updatePressureDiffs(RPTracker.getPressure().LiveOutRegs); + // Account for liveness generated by the region boundary. - if (LiveRegionEnd != RegionEnd) - BotRPTracker.recede(); + if (LiveRegionEnd != RegionEnd) { + SmallVector LiveUses; + BotRPTracker.recede(&LiveUses); + updatePressureDiffs(LiveUses); + } assert(BotRPTracker.getPos() == RegionEnd && "Can't find the region bottom"); @@ -535,6 +551,42 @@ updateScheduledPressure(const std::vector &NewMaxPressure) { }); } +/// Update the PressureDiff array for liveness after scheduling this +/// instruction. +void ScheduleDAGMI::updatePressureDiffs(ArrayRef LiveUses) { + for (unsigned LUIdx = 0, LUEnd = LiveUses.size(); LUIdx != LUEnd; ++LUIdx) { + /// FIXME: Currently assuming single-use physregs. + unsigned Reg = LiveUses[LUIdx]; + if (!TRI->isVirtualRegister(Reg)) + continue; + // This may be called before CurrentBottom has been initialized. However, + // BotRPTracker must have a valid position. We want the value live into the + // instruction or live out of the block, so ask for the previous + // instruction's live-out. + const LiveInterval &LI = LIS->getInterval(Reg); + VNInfo *VNI; + if (BotRPTracker.getPos() == BB->end()) + VNI = LI.getVNInfoBefore(LIS->getMBBEndIdx(BB)); + else { + LiveRangeQuery LRQ(LI, LIS->getInstructionIndex(BotRPTracker.getPos())); + VNI = LRQ.valueIn(); + } + // RegisterPressureTracker guarantees that readsReg is true for LiveUses. + assert(VNI && "No live value at use."); + for (VReg2UseMap::iterator + UI = VRegUses.find(Reg); UI != VRegUses.end(); ++UI) { + SUnit *SU = UI->SU; + // If this use comes before the reaching def, it cannot be a last use, so + // descrease its pressure change. + if (!SU->isScheduled && SU != &ExitSU) { + LiveRangeQuery LRQ(LI, LIS->getInstructionIndex(SU->getInstr())); + if (LRQ.valueIn() == VNI) + getPressureDiff(SU).addPressureChange(Reg, true, &MRI); + } + } + } +} + /// schedule - Called back from MachineScheduler::runOnMachineFunction /// after setting up the current scheduling region. [RegionBegin, RegionEnd) /// only includes instructions that have DAG nodes, not scheduling boundaries. @@ -794,8 +846,10 @@ void ScheduleDAGMI::scheduleMI(SUnit *SU, bool IsTopNode) { CurrentBottom = MI; } // Update bottom scheduled pressure. - BotRPTracker.recede(); + SmallVector LiveUses; + BotRPTracker.recede(&LiveUses); assert(BotRPTracker.getPos() == CurrentBottom && "out of sync"); + updatePressureDiffs(LiveUses); updateScheduledPressure(BotRPTracker.getPressure().MaxSetPressure); } } diff --git a/lib/CodeGen/RegisterPressure.cpp b/lib/CodeGen/RegisterPressure.cpp index 8328b500af1..1be203f36dd 100644 --- a/lib/CodeGen/RegisterPressure.cpp +++ b/lib/CodeGen/RegisterPressure.cpp @@ -399,10 +399,9 @@ static void collectPDiff(PressureDiff &PDiff, RegisterOperands &RegOpers, const MachineRegisterInfo *MRI) { assert(!PDiff.begin()->isValid() && "stale PDiff"); - for (unsigned i = 0, e = RegOpers.Defs.size(); i != e; ++i) { - if (!containsReg(RegOpers.Uses, RegOpers.Defs[i])) - PDiff.addPressureChange(RegOpers.Defs[i], true, MRI); - } + for (unsigned i = 0, e = RegOpers.Defs.size(); i != e; ++i) + PDiff.addPressureChange(RegOpers.Defs[i], true, MRI); + for (unsigned i = 0, e = RegOpers.Uses.size(); i != e; ++i) PDiff.addPressureChange(RegOpers.Uses[i], false, MRI); } @@ -437,9 +436,13 @@ void RegPressureTracker::discoverLiveOut(unsigned Reg) { increaseSetPressure(P.MaxSetPressure, MRI->getPressureSets(Reg)); } -/// Recede across the previous instruction. -/// Record the pressure difference if it is provided. -bool RegPressureTracker::recede(PressureDiff *PDiff) { +/// Recede across the previous instruction. If LiveUses is provided, record any +/// RegUnits that are made live by the current instruction's uses. This includes +/// registers that are both defined and used by the instruction. If a pressure +/// difference pointer is provided record the changes is pressure caused by this +/// instruction independent of liveness. +bool RegPressureTracker::recede(SmallVectorImpl *LiveUses, + PressureDiff *PDiff) { // Check for the top of the analyzable region. if (CurrPos == MBB->begin()) { closeRegion(); @@ -496,11 +499,16 @@ bool RegPressureTracker::recede(PressureDiff *PDiff) { // Adjust liveouts if LiveIntervals are available. if (RequireIntervals) { const LiveInterval *LI = getInterval(Reg); - if (LI && !LI->isKilledAtInstr(SlotIdx)) - discoverLiveOut(Reg); + // Check if this LR is killed and not redefined here. + if (LI && !LI->isKilledAtInstr(SlotIdx) + && !LI->isDefinedByInstr(SlotIdx)) { + discoverLiveOut(Reg); + } } increaseRegPressure(Reg); LiveRegs.insert(Reg); + if (LiveUses && !containsReg(*LiveUses, Reg)) + LiveUses->push_back(Reg); } } if (TrackUntiedDefs) { @@ -773,22 +781,10 @@ getMaxUpwardPressureDelta(const MachineInstr *MI, PressureDiff *PDiff, /// @param MaxPressureLimit Is the max pressure within the region, not /// necessarily at the current position. void RegPressureTracker:: -getUpwardPressureDelta(const MachineInstr *MI, /*const*/ PressureDiff &PDiff1, +getUpwardPressureDelta(const MachineInstr *MI, /*const*/ PressureDiff &PDiff, RegPressureDelta &Delta, ArrayRef CriticalPSets, ArrayRef MaxPressureLimit) const { - RegisterOperands RegOpers(TRI, MRI, /*IgnoreDead=*/true); - collectOperands(MI, RegOpers); - - // Decrease the pressure change for live uses. - PressureDiff PDiff = PDiff1; - for (unsigned i = 0, e = RegOpers.Uses.size(); i != e; ++i) { - if (LiveRegs.contains(RegOpers.Uses[i])) - PDiff.addPressureChange(RegOpers.Uses[i], true, MRI); - } - - // Now directly query pressure from PDiff. Everything above this can be - // cached and updated independent of the query. unsigned CritIdx = 0, CritEnd = CriticalPSets.size(); for (PressureDiff::const_iterator PDiffI = PDiff.begin(), PDiffE = PDiff.end(); diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp index 9b317e30a98..33cdea6f19c 100644 --- a/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -408,7 +408,13 @@ void ScheduleDAGInstrs::addVRegUseDeps(SUnit *SU, unsigned OperIdx) { unsigned Reg = MI->getOperand(OperIdx).getReg(); // Record this local VReg use. - VRegUses.insert(VReg2SUnit(Reg, SU)); + VReg2UseMap::iterator UI = VRegUses.find(Reg); + for (; UI != VRegUses.end(); ++UI) { + if (UI->SU == SU) + break; + } + if (UI == VRegUses.end()) + VRegUses.insert(VReg2SUnit(Reg, SU)); // Lookup this operand's reaching definition. assert(LIS && "vreg dependencies requires LiveIntervals"); @@ -755,7 +761,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, if (RPTracker) { PressureDiff *PDiff = PDiffs ? &(*PDiffs)[SU->NodeNum] : 0; - RPTracker->recede(PDiff); + RPTracker->recede(/*LiveUses=*/0, PDiff); assert(RPTracker->getPos() == prior(MII) && "RPTracker can't find MI"); }