RegisterPressureTracker: unify virtual registers and physical regunits.

Now that live register units are tracked individually, the code can be simplified.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@169426 91177308-0d34-0410-b5e6-96231b3b80d8

include/llvm/CodeGen/LiveIntervalAnalysis.h

@@ -347,6 +347,10 @@ namespace llvm {
       return RegUnitIntervals[Unit];
     }
 
+    const LiveInterval *getCachedRegUnit(unsigned Unit) const {
+      return RegUnitIntervals[Unit];
+    }
+
   private:
     /// computeIntervals - Compute live intervals.
     void computeIntervals();

include/llvm/CodeGen/RegisterPressure.h

@@ -22,6 +22,7 @@
 namespace llvm {
 
 class LiveIntervals;
+class LiveInterval;
 class RegisterClassInfo;
 class MachineInstr;
 
@@ -37,17 +38,17 @@ struct RegisterPressure {
   /// Increase register pressure for each pressure set impacted by this register
   /// class. Normally called by RegPressureTracker, but may be called manually
   /// to account for live through (global liveness).
-  void increase(const TargetRegisterClass *RC, const TargetRegisterInfo *TRI);
+  ///
-
+  /// \param Reg is either a virtual register number or register unit number.
-  /// Increase pressure for each pressure set impacted by this register unit.
+  void increase(unsigned Reg, const TargetRegisterInfo *TRI,
-  void increase(unsigned RU, const TargetRegisterInfo *TRI);
+                const MachineRegisterInfo *MRI);
 
   /// Decrease register pressure for each pressure set impacted by this register
   /// class. This is only useful to account for spilling or rematerialization.
-  void decrease(const TargetRegisterClass *RC, const TargetRegisterInfo *TRI);
+  ///
-
+  /// \param Reg is either a virtual register number or register unit number.
-  /// Decrease pressure for each pressure set impacted by this register unit.
+  void decrease(unsigned Reg, const TargetRegisterInfo *TRI,
-  void decrease(unsigned RU, const TargetRegisterInfo *TRI);
+                const MachineRegisterInfo *MRI);
 
   void dump(const TargetRegisterInfo *TRI) const;
 };
@@ -122,6 +123,33 @@ struct RegPressureDelta {
   RegPressureDelta() {}
 };
 
+/// \brief A set of live virtual registers and physical register units.
+///
+/// Virtual and physical register numbers require separate sparse sets, but most
+/// of the RegisterPressureTracker handles them uniformly.
+struct LiveRegSet {
+  SparseSet<unsigned> PhysRegs;
+  SparseSet<unsigned, VirtReg2IndexFunctor> VirtRegs;
+
+  bool contains(unsigned Reg) {
+    if (TargetRegisterInfo::isVirtualRegister(Reg))
+      return VirtRegs.count(Reg);
+    return PhysRegs.count(Reg);
+  }
+
+  bool insert(unsigned Reg) {
+    if (TargetRegisterInfo::isVirtualRegister(Reg))
+      return VirtRegs.insert(Reg).second;
+    return PhysRegs.insert(Reg).second;
+  }
+
+  bool erase(unsigned Reg) {
+    if (TargetRegisterInfo::isVirtualRegister(Reg))
+      return VirtRegs.erase(Reg);
+    return PhysRegs.erase(Reg);
+  }
+};
+
 /// Track the current register pressure at some position in the instruction
 /// stream, and remember the high water mark within the region traversed. This
 /// does not automatically consider live-through ranges. The client may
@@ -163,9 +191,8 @@ class RegPressureTracker {
   /// Pressure map indexed by pressure set ID, not class ID.
   std::vector<unsigned> CurrSetPressure;
 
-  /// List of live registers.
+  /// Set of live registers.
-  SparseSet<unsigned> LivePhysRegs;
+  LiveRegSet LiveRegs;
-  SparseSet<unsigned, VirtReg2IndexFunctor> LiveVirtRegs;
 
 public:
   RegPressureTracker(IntervalPressure &rp) :
@@ -215,11 +242,8 @@ public:
   /// than the pressure across the traversed region.
   std::vector<unsigned> &getRegSetPressureAtPos() { return CurrSetPressure; }
 
-  void discoverPhysLiveIn(unsigned Reg);
+  void discoverLiveOut(unsigned Reg);
-  void discoverPhysLiveOut(unsigned Reg);
+  void discoverLiveIn(unsigned Reg);
-
-  void discoverVirtLiveIn(unsigned Reg);
-  void discoverVirtLiveOut(unsigned Reg);
 
   bool isTopClosed() const;
   bool isBottomClosed() const;
@@ -283,11 +307,10 @@ public:
   void dump(const TargetRegisterInfo *TRI) const;
 
 protected:
-  void increasePhysRegPressure(ArrayRef<unsigned> Regs);
+  const LiveInterval *getInterval(unsigned Reg) const;
-  void decreasePhysRegPressure(ArrayRef<unsigned> Regs);
 
-  void increaseVirtRegPressure(ArrayRef<unsigned> Regs);
+  void increaseRegPressure(ArrayRef<unsigned> Regs);
-  void decreaseVirtRegPressure(ArrayRef<unsigned> Regs);
+  void decreaseRegPressure(ArrayRef<unsigned> Regs);
 
   void bumpUpwardPressure(const MachineInstr *MI);
   void bumpDownwardPressure(const MachineInstr *MI);

lib/CodeGen/RegisterPressure.cpp

@@ -46,31 +46,33 @@ static void decreaseSetPressure(std::vector<unsigned> &CurrSetPressure,
 }
 
 /// Directly increase pressure only within this RegisterPressure result.
-void RegisterPressure::increase(const TargetRegisterClass *RC,
+void RegisterPressure::increase(unsigned Reg, const TargetRegisterInfo *TRI,
-                                const TargetRegisterInfo *TRI) {
+                                const MachineRegisterInfo *MRI) {
-  increaseSetPressure(MaxSetPressure, MaxSetPressure,
+  if (TargetRegisterInfo::isVirtualRegister(Reg)) {
-                      TRI->getRegClassPressureSets(RC),
+    const TargetRegisterClass *RC = MRI->getRegClass(Reg);
-                      TRI->getRegClassWeight(RC).RegWeight);
+    increaseSetPressure(MaxSetPressure, MaxSetPressure,
-}
+                        TRI->getRegClassPressureSets(RC),
-
+                        TRI->getRegClassWeight(RC).RegWeight);
-/// Directly increase pressure only within this RegisterPressure result.
+  }
-void RegisterPressure::increase(unsigned RU, const TargetRegisterInfo *TRI) {
+  else {
-  increaseSetPressure(MaxSetPressure, MaxSetPressure,
+    increaseSetPressure(MaxSetPressure, MaxSetPressure,
-                      TRI->getRegUnitPressureSets(RU),
+                        TRI->getRegUnitPressureSets(Reg),
-                      TRI->getRegUnitWeight(RU));
+                        TRI->getRegUnitWeight(Reg));
+  }
 }
 
 /// Directly decrease pressure only within this RegisterPressure result.
-void RegisterPressure::decrease(const TargetRegisterClass *RC,
+void RegisterPressure::decrease(unsigned Reg, const TargetRegisterInfo *TRI,
-                                const TargetRegisterInfo *TRI) {
+                                const MachineRegisterInfo *MRI) {
-  decreaseSetPressure(MaxSetPressure, TRI->getRegClassPressureSets(RC),
+  if (TargetRegisterInfo::isVirtualRegister(Reg)) {
-                      TRI->getRegClassWeight(RC).RegWeight);
+    const TargetRegisterClass *RC = MRI->getRegClass(Reg);
-}
+    decreaseSetPressure(MaxSetPressure, TRI->getRegClassPressureSets(RC),
-
+                        TRI->getRegClassWeight(RC).RegWeight);
-/// Directly decrease pressure only within this RegisterPressure result.
+  }
-void RegisterPressure::decrease(unsigned RU, const TargetRegisterInfo *TRI) {
+  else {
-  decreaseSetPressure(MaxSetPressure, TRI->getRegUnitPressureSets(RU),
+    decreaseSetPressure(MaxSetPressure, TRI->getRegUnitPressureSets(Reg),
-                      TRI->getRegUnitWeight(RU));
+                        TRI->getRegUnitWeight(Reg));
+  }
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -102,41 +104,37 @@ void RegPressureTracker::dump(const TargetRegisterInfo *TRI) const {
 }
 #endif
 
-/// Increase the current pressure as impacted by these register units and bump
+/// Increase the current pressure as impacted by these registers and bump
 /// the high water mark if needed.
-void RegPressureTracker::increasePhysRegPressure(ArrayRef<unsigned> Regs) {
+void RegPressureTracker::increaseRegPressure(ArrayRef<unsigned> Regs) {
-  for (unsigned I = 0, E = Regs.size(); I != E; ++I)
-    increaseSetPressure(CurrSetPressure, P.MaxSetPressure,
-                        TRI->getRegUnitPressureSets(Regs[I]),
-                        TRI->getRegUnitWeight(Regs[I]));
-}
-
-/// Simply decrease the current pressure as impacted by these physcial
-/// registers.
-void RegPressureTracker::decreasePhysRegPressure(ArrayRef<unsigned> Regs) {
-  for (unsigned I = 0, E = Regs.size(); I != E; ++I)
-    decreaseSetPressure(CurrSetPressure, TRI->getRegUnitPressureSets(Regs[I]),
-                        TRI->getRegUnitWeight(Regs[I]));
-}
-
-/// Increase the current pressure as impacted by these virtual registers and
-/// bump the high water mark if needed.
-void RegPressureTracker::increaseVirtRegPressure(ArrayRef<unsigned> Regs) {
   for (unsigned I = 0, E = Regs.size(); I != E; ++I) {
-    const TargetRegisterClass *RC = MRI->getRegClass(Regs[I]);
+    if (TargetRegisterInfo::isVirtualRegister(Regs[I])) {
-    increaseSetPressure(CurrSetPressure, P.MaxSetPressure,
+      const TargetRegisterClass *RC = MRI->getRegClass(Regs[I]);
-                        TRI->getRegClassPressureSets(RC),
+      increaseSetPressure(CurrSetPressure, P.MaxSetPressure,
-                        TRI->getRegClassWeight(RC).RegWeight);
+                          TRI->getRegClassPressureSets(RC),
+                          TRI->getRegClassWeight(RC).RegWeight);
+    }
+    else {
+      increaseSetPressure(CurrSetPressure, P.MaxSetPressure,
+                          TRI->getRegUnitPressureSets(Regs[I]),
+                          TRI->getRegUnitWeight(Regs[I]));
+    }
   }
 }
 
-/// Simply decrease the current pressure as impacted by these virtual registers.
+/// Simply decrease the current pressure as impacted by these registers.
-void RegPressureTracker::decreaseVirtRegPressure(ArrayRef<unsigned> Regs) {
+void RegPressureTracker::decreaseRegPressure(ArrayRef<unsigned> Regs) {
   for (unsigned I = 0, E = Regs.size(); I != E; ++I) {
-    const TargetRegisterClass *RC = MRI->getRegClass(Regs[I]);
+    if (TargetRegisterInfo::isVirtualRegister(Regs[I])) {
-    decreaseSetPressure(CurrSetPressure,
+      const TargetRegisterClass *RC = MRI->getRegClass(Regs[I]);
-                        TRI->getRegClassPressureSets(RC),
+      decreaseSetPressure(CurrSetPressure,
-                        TRI->getRegClassWeight(RC).RegWeight);
+                          TRI->getRegClassPressureSets(RC),
+                          TRI->getRegClassWeight(RC).RegWeight);
+    }
+    else {
+      decreaseSetPressure(CurrSetPressure, TRI->getRegUnitPressureSets(Regs[I]),
+                          TRI->getRegUnitWeight(Regs[I]));
+    }
   }
 }
 
@@ -189,6 +187,12 @@ void RegionPressure::openBottom(MachineBasicBlock::const_iterator PrevBottom) {
   LiveInRegs.clear();
 }
 
+const LiveInterval *RegPressureTracker::getInterval(unsigned Reg) const {
+  if (TargetRegisterInfo::isVirtualRegister(Reg))
+    return &LIS->getInterval(Reg);
+  return LIS->getCachedRegUnit(Reg);
+}
+
 /// Setup the RegPressureTracker.
 ///
 /// TODO: Add support for pressure without LiveIntervals.
@@ -218,10 +222,10 @@ void RegPressureTracker::init(const MachineFunction *mf,
     static_cast<RegionPressure&>(P).reset();
   P.MaxSetPressure = CurrSetPressure;
 
-  LivePhysRegs.clear();
+  LiveRegs.PhysRegs.clear();
-  LivePhysRegs.setUniverse(TRI->getNumRegs());
+  LiveRegs.PhysRegs.setUniverse(TRI->getNumRegs());
-  LiveVirtRegs.clear();
+  LiveRegs.VirtRegs.clear();
-  LiveVirtRegs.setUniverse(MRI->getNumVirtRegs());
+  LiveRegs.VirtRegs.setUniverse(MRI->getNumVirtRegs());
 }
 
 /// Does this pressure result have a valid top position and live ins.
@@ -258,10 +262,10 @@ void RegPressureTracker::closeTop() {
     static_cast<RegionPressure&>(P).TopPos = CurrPos;
 
   assert(P.LiveInRegs.empty() && "inconsistent max pressure result");
-  P.LiveInRegs.reserve(LivePhysRegs.size() + LiveVirtRegs.size());
+  P.LiveInRegs.reserve(LiveRegs.PhysRegs.size() + LiveRegs.VirtRegs.size());
-  P.LiveInRegs.append(LivePhysRegs.begin(), LivePhysRegs.end());
+  P.LiveInRegs.append(LiveRegs.PhysRegs.begin(), LiveRegs.PhysRegs.end());
   for (SparseSet<unsigned>::const_iterator I =
-         LiveVirtRegs.begin(), E = LiveVirtRegs.end(); I != E; ++I)
+         LiveRegs.VirtRegs.begin(), E = LiveRegs.VirtRegs.end(); I != E; ++I)
     P.LiveInRegs.push_back(*I);
   std::sort(P.LiveInRegs.begin(), P.LiveInRegs.end());
   P.LiveInRegs.erase(std::unique(P.LiveInRegs.begin(), P.LiveInRegs.end()),
@@ -276,10 +280,10 @@ void RegPressureTracker::closeBottom() {
     static_cast<RegionPressure&>(P).BottomPos = CurrPos;
 
   assert(P.LiveOutRegs.empty() && "inconsistent max pressure result");
-  P.LiveOutRegs.reserve(LivePhysRegs.size() + LiveVirtRegs.size());
+  P.LiveOutRegs.reserve(LiveRegs.PhysRegs.size() + LiveRegs.VirtRegs.size());
-  P.LiveOutRegs.append(LivePhysRegs.begin(), LivePhysRegs.end());
+  P.LiveOutRegs.append(LiveRegs.PhysRegs.begin(), LiveRegs.PhysRegs.end());
   for (SparseSet<unsigned>::const_iterator I =
-         LiveVirtRegs.begin(), E = LiveVirtRegs.end(); I != E; ++I)
+         LiveRegs.VirtRegs.begin(), E = LiveRegs.VirtRegs.end(); I != E; ++I)
     P.LiveOutRegs.push_back(*I);
   std::sort(P.LiveOutRegs.begin(), P.LiveOutRegs.end());
   P.LiveOutRegs.erase(std::unique(P.LiveOutRegs.begin(), P.LiveOutRegs.end()),
@@ -289,7 +293,7 @@ void RegPressureTracker::closeBottom() {
 /// Finalize the region boundaries and record live ins and live outs.
 void RegPressureTracker::closeRegion() {
   if (!isTopClosed() && !isBottomClosed()) {
-    assert(LivePhysRegs.empty() && LiveVirtRegs.empty() &&
+    assert(LiveRegs.PhysRegs.empty() && LiveRegs.VirtRegs.empty() &&
            "no region boundary");
     return;
   }
@@ -307,34 +311,40 @@ static bool containsReg(ArrayRef<unsigned> Regs, unsigned Reg) {
 
 /// Collect this instruction's unique uses and defs into SmallVectors for
 /// processing defs and uses in order.
-template<bool isVReg>
 class RegisterOperands {
+  const TargetRegisterInfo *TRI;
+  const MachineRegisterInfo *MRI;
+
 public:
   SmallVector<unsigned, 8> Uses;
   SmallVector<unsigned, 8> Defs;
   SmallVector<unsigned, 8> DeadDefs;
 
+  RegisterOperands(const TargetRegisterInfo *tri,
+                   const MachineRegisterInfo *mri): TRI(tri), MRI(mri) {}
+
   /// Push this operand's register onto the correct vector.
-  void collect(const MachineOperand &MO, const TargetRegisterInfo *TRI) {
+  void collect(const MachineOperand &MO) {
+    if (!MO.isReg() || !MO.getReg())
+      return;
     if (MO.readsReg())
-      pushRegUnits(MO.getReg(), Uses, TRI);
+      pushRegUnits(MO.getReg(), Uses);
     if (MO.isDef()) {
       if (MO.isDead())
-        pushRegUnits(MO.getReg(), DeadDefs, TRI);
+        pushRegUnits(MO.getReg(), DeadDefs);
       else
-        pushRegUnits(MO.getReg(), Defs, TRI);
+        pushRegUnits(MO.getReg(), Defs);
     }
   }
 
 protected:
-  void pushRegUnits(unsigned Reg, SmallVectorImpl<unsigned> &Regs,
+  void pushRegUnits(unsigned Reg, SmallVectorImpl<unsigned> &Regs) {
-                    const TargetRegisterInfo *TRI) {
+    if (TargetRegisterInfo::isVirtualRegister(Reg)) {
-    if (isVReg) {
       if (containsReg(Regs, Reg))
         return;
       Regs.push_back(Reg);
     }
-    else {
+    else if (MRI->isAllocatable(Reg)) {
       for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units) {
         if (containsReg(Regs, *Units))
           continue;
@@ -343,89 +353,49 @@ protected:
     }
   }
 };
-typedef RegisterOperands<false> PhysRegOperands;
-typedef RegisterOperands<true> VirtRegOperands;
 
 /// Collect physical and virtual register operands.
 static void collectOperands(const MachineInstr *MI,
-                            PhysRegOperands &PhysRegOpers,
+                            RegisterOperands &RegOpers) {
-                            VirtRegOperands &VirtRegOpers,
+  for(ConstMIBundleOperands OperI(MI); OperI.isValid(); ++OperI)
-                            const TargetRegisterInfo *TRI,
+    RegOpers.collect(*OperI);
-                            const MachineRegisterInfo *MRI) {
-  for(ConstMIBundleOperands OperI(MI); OperI.isValid(); ++OperI) {
-    const MachineOperand &MO = *OperI;
-    if (!MO.isReg() || !MO.getReg())
-      continue;
 
-    if (TargetRegisterInfo::isVirtualRegister(MO.getReg()))
-      VirtRegOpers.collect(MO, TRI);
-    else if (MRI->isAllocatable(MO.getReg()))
-      PhysRegOpers.collect(MO, TRI);
-  }
   // Remove redundant physreg dead defs.
-  for (unsigned i = PhysRegOpers.DeadDefs.size(); i > 0; --i) {
+  for (unsigned i = RegOpers.DeadDefs.size(); i > 0; --i) {
-    unsigned Reg = PhysRegOpers.DeadDefs[i-1];
+    unsigned Reg = RegOpers.DeadDefs[i-1];
-    if (containsReg(PhysRegOpers.Defs, Reg))
+    if (containsReg(RegOpers.Defs, Reg))
-      PhysRegOpers.DeadDefs.erase(&PhysRegOpers.DeadDefs[i-1]);
+      RegOpers.DeadDefs.erase(&RegOpers.DeadDefs[i-1]);
   }
 }
 
 /// Force liveness of registers.
 void RegPressureTracker::addLiveRegs(ArrayRef<unsigned> Regs) {
   for (unsigned i = 0, e = Regs.size(); i != e; ++i) {
-    if (TargetRegisterInfo::isVirtualRegister(Regs[i])) {
+    if (LiveRegs.insert(Regs[i]))
-      if (LiveVirtRegs.insert(Regs[i]).second)
+      increaseRegPressure(Regs[i]);
-        increaseVirtRegPressure(Regs[i]);
-    }
-    else  {
-      if (LivePhysRegs.insert(Regs[i]).second)
-        increasePhysRegPressure(Regs[i]);
-    }
   }
 }
 
-/// Add PhysReg to the live in set and increase max pressure.
+/// Add Reg to the live in set and increase max pressure.
-void RegPressureTracker::discoverPhysLiveIn(unsigned Reg) {
+void RegPressureTracker::discoverLiveIn(unsigned Reg) {
-  assert(!LivePhysRegs.count(Reg) && "avoid bumping max pressure twice");
+  assert(!LiveRegs.contains(Reg) && "avoid bumping max pressure twice");
   if (containsReg(P.LiveInRegs, Reg))
     return;
 
   // At live in discovery, unconditionally increase the high water mark.
   P.LiveInRegs.push_back(Reg);
-  P.increase(Reg, TRI);
+  P.increase(Reg, TRI, MRI);
 }
 
-/// Add PhysReg to the live out set and increase max pressure.
+/// Add Reg to the live out set and increase max pressure.
-void RegPressureTracker::discoverPhysLiveOut(unsigned Reg) {
+void RegPressureTracker::discoverLiveOut(unsigned Reg) {
-  assert(!LivePhysRegs.count(Reg) && "avoid bumping max pressure twice");
+  assert(!LiveRegs.contains(Reg) && "avoid bumping max pressure twice");
   if (containsReg(P.LiveOutRegs, Reg))
     return;
 
   // At live out discovery, unconditionally increase the high water mark.
   P.LiveOutRegs.push_back(Reg);
-  P.increase(Reg, TRI);
+  P.increase(Reg, TRI, MRI);
-}
-
-/// Add VirtReg to the live in set and increase max pressure.
-void RegPressureTracker::discoverVirtLiveIn(unsigned Reg) {
-  assert(!LiveVirtRegs.count(Reg) && "avoid bumping max pressure twice");
-  if (containsReg(P.LiveInRegs, Reg))
-    return;
-
-  // At live in discovery, unconditionally increase the high water mark.
-  P.LiveInRegs.push_back(Reg);
-  P.increase(MRI->getRegClass(Reg), TRI);
-}
-
-/// Add VirtReg to the live out set and increase max pressure.
-void RegPressureTracker::discoverVirtLiveOut(unsigned Reg) {
-  assert(!LiveVirtRegs.count(Reg) && "avoid bumping max pressure twice");
-  if (containsReg(P.LiveOutRegs, Reg))
-    return;
-
-  // At live out discovery, unconditionally increase the high water mark.
-  P.LiveOutRegs.push_back(Reg);
-  P.increase(MRI->getRegClass(Reg), TRI);
 }
 
 /// Recede across the previous instruction.
@@ -459,50 +429,35 @@ bool RegPressureTracker::recede() {
   if (RequireIntervals && isTopClosed())
     static_cast<IntervalPressure&>(P).openTop(SlotIdx);
 
-  PhysRegOperands PhysRegOpers;
+  RegisterOperands RegOpers(TRI, MRI);
-  VirtRegOperands VirtRegOpers;
+  collectOperands(CurrPos, RegOpers);
-  collectOperands(CurrPos, PhysRegOpers, VirtRegOpers, TRI, MRI);
 
   // Boost pressure for all dead defs together.
-  increasePhysRegPressure(PhysRegOpers.DeadDefs);
+  increaseRegPressure(RegOpers.DeadDefs);
-  increaseVirtRegPressure(VirtRegOpers.DeadDefs);
+  decreaseRegPressure(RegOpers.DeadDefs);
-  decreasePhysRegPressure(PhysRegOpers.DeadDefs);
-  decreaseVirtRegPressure(VirtRegOpers.DeadDefs);
 
   // Kill liveness at live defs.
   // TODO: consider earlyclobbers?
-  for (unsigned i = 0, e = PhysRegOpers.Defs.size(); i < e; ++i) {
+  for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) {
-    unsigned Reg = PhysRegOpers.Defs[i];
+    unsigned Reg = RegOpers.Defs[i];
-    if (LivePhysRegs.erase(Reg))
+    if (LiveRegs.erase(Reg))
-      decreasePhysRegPressure(Reg);
+      decreaseRegPressure(Reg);
     else
-      discoverPhysLiveOut(Reg);
+      discoverLiveOut(Reg);
-  }
-  for (unsigned i = 0, e = VirtRegOpers.Defs.size(); i < e; ++i) {
-    unsigned Reg = VirtRegOpers.Defs[i];
-    if (LiveVirtRegs.erase(Reg))
-      decreaseVirtRegPressure(Reg);
-    else
-      discoverVirtLiveOut(Reg);
   }
 
   // Generate liveness for uses.
-  for (unsigned i = 0, e = PhysRegOpers.Uses.size(); i < e; ++i) {
+  for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) {
-    unsigned Reg = PhysRegOpers.Uses[i];
+    unsigned Reg = RegOpers.Uses[i];
-    if (LivePhysRegs.insert(Reg).second)
+    if (!LiveRegs.contains(Reg)) {
-      increasePhysRegPressure(Reg);
-  }
-  for (unsigned i = 0, e = VirtRegOpers.Uses.size(); i < e; ++i) {
-    unsigned Reg = VirtRegOpers.Uses[i];
-    if (!LiveVirtRegs.count(Reg)) {
       // Adjust liveouts if LiveIntervals are available.
       if (RequireIntervals) {
-        const LiveInterval *LI = &LIS->getInterval(Reg);
+        const LiveInterval *LI = getInterval(Reg);
-        if (!LI->killedAt(SlotIdx))
+        if (LI && !LI->killedAt(SlotIdx))
-          discoverVirtLiveOut(Reg);
+          discoverLiveOut(Reg);
       }
-      increaseVirtRegPressure(Reg);
+      increaseRegPressure(Reg);
-      LiveVirtRegs.insert(Reg);
+      LiveRegs.insert(Reg);
     }
   }
   return true;
@@ -530,53 +485,43 @@ bool RegPressureTracker::advance() {
       static_cast<RegionPressure&>(P).openBottom(CurrPos);
   }
 
-  PhysRegOperands PhysRegOpers;
+  RegisterOperands RegOpers(TRI, MRI);
-  VirtRegOperands VirtRegOpers;
+  collectOperands(CurrPos, RegOpers);
-  collectOperands(CurrPos, PhysRegOpers, VirtRegOpers, TRI, MRI);
 
-  // Kill liveness at last uses.
+  for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) {
-  for (unsigned i = 0, e = PhysRegOpers.Uses.size(); i < e; ++i) {
+    unsigned Reg = RegOpers.Uses[i];
-    unsigned Reg = PhysRegOpers.Uses[i];
+    // Discover live-ins.
-    // Allocatable physregs are always single-use before register rewriting.
+    bool isLive = LiveRegs.contains(Reg);
-    if (LivePhysRegs.erase(Reg))
+    if (!isLive)
-      decreasePhysRegPressure(Reg);
+      discoverLiveIn(Reg);
-    else
+    // Kill liveness at last uses.
-      discoverPhysLiveIn(Reg);
+    bool lastUse = false;
-  }
-  for (unsigned i = 0, e = VirtRegOpers.Uses.size(); i < e; ++i) {
-    unsigned Reg = VirtRegOpers.Uses[i];
     if (RequireIntervals) {
-      const LiveInterval *LI = &LIS->getInterval(Reg);
+      const LiveInterval *LI = getInterval(Reg);
-      if (LI->killedAt(SlotIdx)) {
+      lastUse = LI && LI->killedAt(SlotIdx);
-        if (LiveVirtRegs.erase(Reg))
-          decreaseVirtRegPressure(Reg);
-        else
-          discoverVirtLiveIn(Reg);
-      }
     }
-    else if (!LiveVirtRegs.count(Reg)) {
+    else {
-      discoverVirtLiveIn(Reg);
+      // Allocatable physregs are always single-use before register rewriting.
-      increaseVirtRegPressure(Reg);
+      lastUse = !TargetRegisterInfo::isVirtualRegister(Reg);
     }
+    if (lastUse && isLive) {
+      LiveRegs.erase(Reg);
+      decreaseRegPressure(Reg);
+    }
+    else if (!lastUse && !isLive)
+      increaseRegPressure(Reg);
   }
 
   // Generate liveness for defs.
-  for (unsigned i = 0, e = PhysRegOpers.Defs.size(); i < e; ++i) {
+  for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) {
-    unsigned Reg = PhysRegOpers.Defs[i];
+    unsigned Reg = RegOpers.Defs[i];
-    if (LivePhysRegs.insert(Reg).second)
+    if (LiveRegs.insert(Reg))
-      increasePhysRegPressure(Reg);
+      increaseRegPressure(Reg);
-  }
-  for (unsigned i = 0, e = VirtRegOpers.Defs.size(); i < e; ++i) {
-    unsigned Reg = VirtRegOpers.Defs[i];
-    if (LiveVirtRegs.insert(Reg).second)
-      increaseVirtRegPressure(Reg);
   }
 
   // Boost pressure for all dead defs together.
-  increasePhysRegPressure(PhysRegOpers.DeadDefs);
+  increaseRegPressure(RegOpers.DeadDefs);
-  increaseVirtRegPressure(VirtRegOpers.DeadDefs);
+  decreaseRegPressure(RegOpers.DeadDefs);
-  decreasePhysRegPressure(PhysRegOpers.DeadDefs);
-  decreaseVirtRegPressure(VirtRegOpers.DeadDefs);
 
   // Find the next instruction.
   do
@@ -667,39 +612,28 @@ static void computeMaxPressureDelta(ArrayRef<unsigned> OldMaxPressureVec,
 /// This is intended for speculative queries. It leaves pressure inconsistent
 /// with the current position, so must be restored by the caller.
 void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) {
+  assert(!MI->isDebugValue() && "Expect a nondebug instruction.");
+
   // Account for register pressure similar to RegPressureTracker::recede().
-  PhysRegOperands PhysRegOpers;
+  RegisterOperands RegOpers(TRI, MRI);
-  VirtRegOperands VirtRegOpers;
+  collectOperands(MI, RegOpers);
-  collectOperands(MI, PhysRegOpers, VirtRegOpers, TRI, MRI);
 
   // Boost max pressure for all dead defs together.
   // Since CurrSetPressure and MaxSetPressure
-  increasePhysRegPressure(PhysRegOpers.DeadDefs);
+  increaseRegPressure(RegOpers.DeadDefs);
-  increaseVirtRegPressure(VirtRegOpers.DeadDefs);
+  decreaseRegPressure(RegOpers.DeadDefs);
-  decreasePhysRegPressure(PhysRegOpers.DeadDefs);
-  decreaseVirtRegPressure(VirtRegOpers.DeadDefs);
 
   // Kill liveness at live defs.
-  for (unsigned i = 0, e = PhysRegOpers.Defs.size(); i < e; ++i) {
+  for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) {
-    unsigned Reg = PhysRegOpers.Defs[i];
+    unsigned Reg = RegOpers.Defs[i];
-    if (!containsReg(PhysRegOpers.Uses, Reg))
+    if (!containsReg(RegOpers.Uses, Reg))
-      decreasePhysRegPressure(Reg);
+      decreaseRegPressure(Reg);
-  }
-  for (unsigned i = 0, e = VirtRegOpers.Defs.size(); i < e; ++i) {
-    unsigned Reg = VirtRegOpers.Defs[i];
-    if (!containsReg(VirtRegOpers.Uses, Reg))
-      decreaseVirtRegPressure(Reg);
   }
   // Generate liveness for uses.
-  for (unsigned i = 0, e = PhysRegOpers.Uses.size(); i < e; ++i) {
+  for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) {
-    unsigned Reg = PhysRegOpers.Uses[i];
+    unsigned Reg = RegOpers.Uses[i];
-    if (!LivePhysRegs.count(Reg))
+    if (!LiveRegs.contains(Reg))
-      increasePhysRegPressure(Reg);
+      increaseRegPressure(Reg);
-  }
-  for (unsigned i = 0, e = VirtRegOpers.Uses.size(); i < e; ++i) {
-    unsigned Reg = VirtRegOpers.Uses[i];
-    if (!LiveVirtRegs.count(Reg))
-      increaseVirtRegPressure(Reg);
   }
 }
 
@@ -760,38 +694,42 @@ static bool findUseBetween(unsigned Reg,
 /// This is intended for speculative queries. It leaves pressure inconsistent
 /// with the current position, so must be restored by the caller.
 void RegPressureTracker::bumpDownwardPressure(const MachineInstr *MI) {
+  assert(!MI->isDebugValue() && "Expect a nondebug instruction.");
+
   // Account for register pressure similar to RegPressureTracker::recede().
-  PhysRegOperands PhysRegOpers;
+  RegisterOperands RegOpers(TRI, MRI);
-  VirtRegOperands VirtRegOpers;
+  collectOperands(MI, RegOpers);
-  collectOperands(MI, PhysRegOpers, VirtRegOpers, TRI, MRI);
 
   // Kill liveness at last uses. Assume allocatable physregs are single-use
   // rather than checking LiveIntervals.
-  decreasePhysRegPressure(PhysRegOpers.Uses);
+  SlotIndex SlotIdx;
-  if (RequireIntervals) {
+  if (RequireIntervals)
-    SlotIndex SlotIdx = LIS->getInstructionIndex(MI).getRegSlot();
+    SlotIdx = LIS->getInstructionIndex(MI).getRegSlot();
-    for (unsigned i = 0, e = VirtRegOpers.Uses.size(); i < e; ++i) {
+
-      unsigned Reg = VirtRegOpers.Uses[i];
+  for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) {
-      const LiveInterval *LI = &LIS->getInterval(Reg);
+    unsigned Reg = RegOpers.Uses[i];
-      // FIXME: allow the caller to pass in the list of vreg uses that remain to
+    if (RequireIntervals) {
-      // be bottom-scheduled to avoid searching uses at each query.
+      // FIXME: allow the caller to pass in the list of vreg uses that remain
+      // to be bottom-scheduled to avoid searching uses at each query.
       SlotIndex CurrIdx = getCurrSlot();
-      if (LI->killedAt(SlotIdx)
+      const LiveInterval *LI = getInterval(Reg);
+      if (LI && LI->killedAt(SlotIdx)
           && !findUseBetween(Reg, CurrIdx, SlotIdx, MRI, LIS)) {
-        decreaseVirtRegPressure(Reg);
+        decreaseRegPressure(Reg);
       }
     }
+    else if (!TargetRegisterInfo::isVirtualRegister(Reg)) {
+      // Allocatable physregs are always single-use before register rewriting.
+      decreaseRegPressure(Reg);
+    }
   }
 
   // Generate liveness for defs.
-  increasePhysRegPressure(PhysRegOpers.Defs);
+  increaseRegPressure(RegOpers.Defs);
-  increaseVirtRegPressure(VirtRegOpers.Defs);
 
   // Boost pressure for all dead defs together.
-  increasePhysRegPressure(PhysRegOpers.DeadDefs);
+  increaseRegPressure(RegOpers.DeadDefs);
-  increaseVirtRegPressure(VirtRegOpers.DeadDefs);
+  decreaseRegPressure(RegOpers.DeadDefs);
-  decreasePhysRegPressure(PhysRegOpers.DeadDefs);
-  decreaseVirtRegPressure(VirtRegOpers.DeadDefs);
 }
 
 /// Consider the pressure increase caused by traversing this instruction