RegisterPresssureTracker: Track live physical register by unit.

This is much simpler to reason about, more efficient, and
fixes some corner cases involving implicit super-register defs.
Fixed rdar://12797931.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@169425 91177308-0d34-0410-b5e6-96231b3b80d8

include/llvm/CodeGen/RegisterPressure.h

@@ -30,7 +30,7 @@ struct RegisterPressure {
   /// Map of max reg pressure indexed by pressure set ID, not class ID.
   std::vector<unsigned> MaxSetPressure;
 
-  /// List of live in registers.
+  /// List of live in virtual registers or physical register units.
   SmallVector<unsigned,8> LiveInRegs;
   SmallVector<unsigned,8> LiveOutRegs;
 
@@ -39,10 +39,16 @@ struct RegisterPressure {
   /// to account for live through (global liveness).
   void increase(const TargetRegisterClass *RC, const TargetRegisterInfo *TRI);
 
+  /// Increase pressure for each pressure set impacted by this register unit.
+  void increase(unsigned RU, const TargetRegisterInfo *TRI);
+
   /// Decrease register pressure for each pressure set impacted by this register
   /// class. This is only useful to account for spilling or rematerialization.
   void decrease(const TargetRegisterClass *RC, const TargetRegisterInfo *TRI);
 
+  /// Decrease pressure for each pressure set impacted by this register unit.
+  void decrease(unsigned RU, const TargetRegisterInfo *TRI);
+
   void dump(const TargetRegisterInfo *TRI) const;
 };
 
@@ -172,8 +178,9 @@ public:
             const LiveIntervals *lis, const MachineBasicBlock *mbb,
             MachineBasicBlock::const_iterator pos);
 
-  /// Force liveness of registers. Particularly useful to initialize the
+  /// Force liveness of virtual registers or physical register
-  /// livein/out state of the tracker before the first call to advance/recede.
+  /// units. Particularly useful to initialize the livein/out state of the
+  /// tracker before the first call to advance/recede.
   void addLiveRegs(ArrayRef<unsigned> Regs);
 
   /// Get the MI position corresponding to this register pressure.

lib/CodeGen/RegisterPressure.cpp

@@ -23,14 +23,11 @@
 
 using namespace llvm;
 
-/// Increase register pressure for each set impacted by this register class.
+/// Increase pressure for each pressure set provided by TargetRegisterInfo.
 static void increaseSetPressure(std::vector<unsigned> &CurrSetPressure,
                                 std::vector<unsigned> &MaxSetPressure,
-                                const TargetRegisterClass *RC,
+                                const int *PSet, unsigned Weight) {
-                                const TargetRegisterInfo *TRI) {
+  for (; *PSet != -1; ++PSet) {
-  unsigned Weight = TRI->getRegClassWeight(RC).RegWeight;
-  for (const int *PSet = TRI->getRegClassPressureSets(RC);
-       *PSet != -1; ++PSet) {
     CurrSetPressure[*PSet] += Weight;
     if (&CurrSetPressure != &MaxSetPressure
         && CurrSetPressure[*PSet] > MaxSetPressure[*PSet]) {
@@ -39,13 +36,10 @@ static void increaseSetPressure(std::vector<unsigned> &CurrSetPressure,
   }
 }
 
-/// Decrease register pressure for each set impacted by this register class.
+/// Decrease pressure for each pressure set provided by TargetRegisterInfo.
 static void decreaseSetPressure(std::vector<unsigned> &CurrSetPressure,
-                                const TargetRegisterClass *RC,
+                                const int *PSet, unsigned Weight) {
-                                const TargetRegisterInfo *TRI) {
+  for (; *PSet != -1; ++PSet) {
-  unsigned Weight = TRI->getRegClassWeight(RC).RegWeight;
-  for (const int *PSet = TRI->getRegClassPressureSets(RC);
-       *PSet != -1; ++PSet) {
     assert(CurrSetPressure[*PSet] >= Weight && "register pressure underflow");
     CurrSetPressure[*PSet] -= Weight;
   }
@@ -54,13 +48,29 @@ static void decreaseSetPressure(std::vector<unsigned> &CurrSetPressure,
 /// Directly increase pressure only within this RegisterPressure result.
 void RegisterPressure::increase(const TargetRegisterClass *RC,
                                 const TargetRegisterInfo *TRI) {
-  increaseSetPressure(MaxSetPressure, MaxSetPressure, RC, TRI);
+  increaseSetPressure(MaxSetPressure, MaxSetPressure,
+                      TRI->getRegClassPressureSets(RC),
+                      TRI->getRegClassWeight(RC).RegWeight);
+}
+
+/// Directly increase pressure only within this RegisterPressure result.
+void RegisterPressure::increase(unsigned RU, const TargetRegisterInfo *TRI) {
+  increaseSetPressure(MaxSetPressure, MaxSetPressure,
+                      TRI->getRegUnitPressureSets(RU),
+                      TRI->getRegUnitWeight(RU));
 }
 
 /// Directly decrease pressure only within this RegisterPressure result.
 void RegisterPressure::decrease(const TargetRegisterClass *RC,
                                 const TargetRegisterInfo *TRI) {
-  decreaseSetPressure(MaxSetPressure, RC, TRI);
+  decreaseSetPressure(MaxSetPressure, TRI->getRegClassPressureSets(RC),
+                      TRI->getRegClassWeight(RC).RegWeight);
+}
+
+/// Directly decrease pressure only within this RegisterPressure result.
+void RegisterPressure::decrease(unsigned RU, const TargetRegisterInfo *TRI) {
+  decreaseSetPressure(MaxSetPressure, TRI->getRegUnitPressureSets(RU),
+                      TRI->getRegUnitWeight(RU));
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -92,34 +102,42 @@ void RegPressureTracker::dump(const TargetRegisterInfo *TRI) const {
 }
 #endif
 
-/// Increase the current pressure as impacted by these physical registers and
+/// Increase the current pressure as impacted by these register units and bump
-/// bump the high water mark if needed.
+/// the high water mark if needed.
 void RegPressureTracker::increasePhysRegPressure(ArrayRef<unsigned> Regs) {
   for (unsigned I = 0, E = Regs.size(); I != E; ++I)
     increaseSetPressure(CurrSetPressure, P.MaxSetPressure,
-                        TRI->getMinimalPhysRegClass(Regs[I]), TRI);
+                        TRI->getRegUnitPressureSets(Regs[I]),
+                        TRI->getRegUnitWeight(Regs[I]));
 }
 
 /// Simply decrease the current pressure as impacted by these physcial
 /// registers.
 void RegPressureTracker::decreasePhysRegPressure(ArrayRef<unsigned> Regs) {
   for (unsigned I = 0, E = Regs.size(); I != E; ++I)
-    decreaseSetPressure(CurrSetPressure, TRI->getMinimalPhysRegClass(Regs[I]),
+    decreaseSetPressure(CurrSetPressure, TRI->getRegUnitPressureSets(Regs[I]),
-                        TRI);
+                        TRI->getRegUnitWeight(Regs[I]));
 }
 
 /// Increase the current pressure as impacted by these virtual registers and
 /// bump the high water mark if needed.
 void RegPressureTracker::increaseVirtRegPressure(ArrayRef<unsigned> Regs) {
-  for (unsigned I = 0, E = Regs.size(); I != E; ++I)
+  for (unsigned I = 0, E = Regs.size(); I != E; ++I) {
+    const TargetRegisterClass *RC = MRI->getRegClass(Regs[I]);
     increaseSetPressure(CurrSetPressure, P.MaxSetPressure,
-                        MRI->getRegClass(Regs[I]), TRI);
+                        TRI->getRegClassPressureSets(RC),
+                        TRI->getRegClassWeight(RC).RegWeight);
+  }
 }
 
 /// Simply decrease the current pressure as impacted by these virtual registers.
 void RegPressureTracker::decreaseVirtRegPressure(ArrayRef<unsigned> Regs) {
-  for (unsigned I = 0, E = Regs.size(); I != E; ++I)
+  for (unsigned I = 0, E = Regs.size(); I != E; ++I) {
-    decreaseSetPressure(CurrSetPressure, MRI->getRegClass(Regs[I]), TRI);
+    const TargetRegisterClass *RC = MRI->getRegClass(Regs[I]);
+    decreaseSetPressure(CurrSetPressure,
+                        TRI->getRegClassPressureSets(RC),
+                        TRI->getRegClassWeight(RC).RegWeight);
+  }
 }
 
 /// Clear the result so it can be used for another round of pressure tracking.
@@ -282,61 +300,46 @@ void RegPressureTracker::closeRegion() {
   // If both top and bottom are closed, do nothing.
 }
 
-/// Return true if Reg aliases a register in Regs SparseSet.
+/// \brief Convenient wrapper for checking membership in RegisterOperands.
-static bool hasRegAlias(unsigned Reg, SparseSet<unsigned> &Regs,
+static bool containsReg(ArrayRef<unsigned> Regs, unsigned Reg) {
-                        const TargetRegisterInfo *TRI) {
+  return std::find(Regs.begin(), Regs.end(), Reg) != Regs.end();
-  assert(!TargetRegisterInfo::isVirtualRegister(Reg) && "only for physregs");
-  for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
-    if (Regs.count(*AI))
-      return true;
-  return false;
-}
-
-/// Return true if Reg aliases a register in unsorted Regs SmallVector.
-/// This is only valid for physical registers.
-static SmallVectorImpl<unsigned>::iterator
-findRegAlias(unsigned Reg, SmallVectorImpl<unsigned> &Regs,
-             const TargetRegisterInfo *TRI) {
-  for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) {
-    SmallVectorImpl<unsigned>::iterator I =
-      std::find(Regs.begin(), Regs.end(), *AI);
-    if (I != Regs.end())
-      return I;
-  }
-  return Regs.end();
-}
-
-/// Return true if Reg can be inserted into Regs SmallVector. For virtual
-/// register, do a linear search. For physical registers check for aliases.
-static SmallVectorImpl<unsigned>::iterator
-findReg(unsigned Reg, bool isVReg, SmallVectorImpl<unsigned> &Regs,
-        const TargetRegisterInfo *TRI) {
-  if(isVReg)
-    return std::find(Regs.begin(), Regs.end(), Reg);
-  return findRegAlias(Reg, Regs, TRI);
 }
 
 /// Collect this instruction's unique uses and defs into SmallVectors for
 /// processing defs and uses in order.
 template<bool isVReg>
-struct RegisterOperands {
+class RegisterOperands {
+public:
   SmallVector<unsigned, 8> Uses;
   SmallVector<unsigned, 8> Defs;
   SmallVector<unsigned, 8> DeadDefs;
 
   /// Push this operand's register onto the correct vector.
   void collect(const MachineOperand &MO, const TargetRegisterInfo *TRI) {
-    if (MO.readsReg()) {
+    if (MO.readsReg())
-      if (findReg(MO.getReg(), isVReg, Uses, TRI) == Uses.end())
+      pushRegUnits(MO.getReg(), Uses, TRI);
-      Uses.push_back(MO.getReg());
-    }
     if (MO.isDef()) {
-      if (MO.isDead()) {
+      if (MO.isDead())
-        if (findReg(MO.getReg(), isVReg, DeadDefs, TRI) == DeadDefs.end())
+        pushRegUnits(MO.getReg(), DeadDefs, TRI);
-          DeadDefs.push_back(MO.getReg());
+      else
+        pushRegUnits(MO.getReg(), Defs, TRI);
+    }
+  }
+
+protected:
+  void pushRegUnits(unsigned Reg, SmallVectorImpl<unsigned> &Regs,
+                    const TargetRegisterInfo *TRI) {
+    if (isVReg) {
+      if (containsReg(Regs, Reg))
+        return;
+      Regs.push_back(Reg);
+    }
+    else {
+      for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units) {
+        if (containsReg(Regs, *Units))
+          continue;
+        Regs.push_back(*Units);
       }
-      else if (findReg(MO.getReg(), isVReg, Defs, TRI) == Defs.end())
-        Defs.push_back(MO.getReg());
     }
   }
 };
@@ -362,7 +365,7 @@ static void collectOperands(const MachineInstr *MI,
   // Remove redundant physreg dead defs.
   for (unsigned i = PhysRegOpers.DeadDefs.size(); i > 0; --i) {
     unsigned Reg = PhysRegOpers.DeadDefs[i-1];
-    if (findRegAlias(Reg, PhysRegOpers.Defs, TRI) != PhysRegOpers.Defs.end())
+    if (containsReg(PhysRegOpers.Defs, Reg))
       PhysRegOpers.DeadDefs.erase(&PhysRegOpers.DeadDefs[i-1]);
   }
 }
@@ -375,41 +378,38 @@ void RegPressureTracker::addLiveRegs(ArrayRef<unsigned> Regs) {
         increaseVirtRegPressure(Regs[i]);
     }
     else  {
-      if (!hasRegAlias(Regs[i], LivePhysRegs, TRI)) {
+      if (LivePhysRegs.insert(Regs[i]).second)
-        LivePhysRegs.insert(Regs[i]);
         increasePhysRegPressure(Regs[i]);
     }
   }
 }
-}
 
 /// Add PhysReg to the live in set and increase max pressure.
 void RegPressureTracker::discoverPhysLiveIn(unsigned Reg) {
   assert(!LivePhysRegs.count(Reg) && "avoid bumping max pressure twice");
-  if (findRegAlias(Reg, P.LiveInRegs, TRI) != P.LiveInRegs.end())
+  if (containsReg(P.LiveInRegs, Reg))
     return;
 
   // At live in discovery, unconditionally increase the high water mark.
   P.LiveInRegs.push_back(Reg);
-  P.increase(TRI->getMinimalPhysRegClass(Reg), TRI);
+  P.increase(Reg, TRI);
 }
 
 /// Add PhysReg to the live out set and increase max pressure.
 void RegPressureTracker::discoverPhysLiveOut(unsigned Reg) {
   assert(!LivePhysRegs.count(Reg) && "avoid bumping max pressure twice");
-  if (findRegAlias(Reg, P.LiveOutRegs, TRI) != P.LiveOutRegs.end())
+  if (containsReg(P.LiveOutRegs, Reg))
     return;
 
   // At live out discovery, unconditionally increase the high water mark.
   P.LiveOutRegs.push_back(Reg);
-  P.increase(TRI->getMinimalPhysRegClass(Reg), TRI);
+  P.increase(Reg, TRI);
 }
 
 /// Add VirtReg to the live in set and increase max pressure.
 void RegPressureTracker::discoverVirtLiveIn(unsigned Reg) {
   assert(!LiveVirtRegs.count(Reg) && "avoid bumping max pressure twice");
-  if (std::find(P.LiveInRegs.begin(), P.LiveInRegs.end(), Reg) !=
+  if (containsReg(P.LiveInRegs, Reg))
-      P.LiveInRegs.end())
     return;
 
   // At live in discovery, unconditionally increase the high water mark.
@@ -420,8 +420,7 @@ void RegPressureTracker::discoverVirtLiveIn(unsigned Reg) {
 /// Add VirtReg to the live out set and increase max pressure.
 void RegPressureTracker::discoverVirtLiveOut(unsigned Reg) {
   assert(!LiveVirtRegs.count(Reg) && "avoid bumping max pressure twice");
-  if (std::find(P.LiveOutRegs.begin(), P.LiveOutRegs.end(), Reg) !=
+  if (containsReg(P.LiveOutRegs, Reg))
-      P.LiveOutRegs.end())
     return;
 
   // At live out discovery, unconditionally increase the high water mark.
@@ -490,10 +489,8 @@ bool RegPressureTracker::recede() {
   // Generate liveness for uses.
   for (unsigned i = 0, e = PhysRegOpers.Uses.size(); i < e; ++i) {
     unsigned Reg = PhysRegOpers.Uses[i];
-    if (!hasRegAlias(Reg, LivePhysRegs, TRI)) {
+    if (LivePhysRegs.insert(Reg).second)
       increasePhysRegPressure(Reg);
-      LivePhysRegs.insert(Reg);
-    }
   }
   for (unsigned i = 0, e = VirtRegOpers.Uses.size(); i < e; ++i) {
     unsigned Reg = VirtRegOpers.Uses[i];
@@ -540,13 +537,11 @@ bool RegPressureTracker::advance() {
   // Kill liveness at last uses.
   for (unsigned i = 0, e = PhysRegOpers.Uses.size(); i < e; ++i) {
     unsigned Reg = PhysRegOpers.Uses[i];
-    if (!hasRegAlias(Reg, LivePhysRegs, TRI))
+    // Allocatable physregs are always single-use before register rewriting.
-      discoverPhysLiveIn(Reg);
+    if (LivePhysRegs.erase(Reg))
-    else {
-      // Allocatable physregs are always single-use before regalloc.
       decreasePhysRegPressure(Reg);
-      LivePhysRegs.erase(Reg);
+    else
-    }
+      discoverPhysLiveIn(Reg);
   }
   for (unsigned i = 0, e = VirtRegOpers.Uses.size(); i < e; ++i) {
     unsigned Reg = VirtRegOpers.Uses[i];
@@ -568,10 +563,8 @@ bool RegPressureTracker::advance() {
   // Generate liveness for defs.
   for (unsigned i = 0, e = PhysRegOpers.Defs.size(); i < e; ++i) {
     unsigned Reg = PhysRegOpers.Defs[i];
-    if (!hasRegAlias(Reg, LivePhysRegs, TRI)) {
+    if (LivePhysRegs.insert(Reg).second)
       increasePhysRegPressure(Reg);
-      LivePhysRegs.insert(Reg);
-    }
   }
   for (unsigned i = 0, e = VirtRegOpers.Defs.size(); i < e; ++i) {
     unsigned Reg = VirtRegOpers.Defs[i];
@@ -689,18 +682,18 @@ void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) {
   // Kill liveness at live defs.
   for (unsigned i = 0, e = PhysRegOpers.Defs.size(); i < e; ++i) {
     unsigned Reg = PhysRegOpers.Defs[i];
-    if (!findReg(Reg, false, PhysRegOpers.Uses, TRI))
+    if (!containsReg(PhysRegOpers.Uses, Reg))
-      decreasePhysRegPressure(PhysRegOpers.Defs);
+      decreasePhysRegPressure(Reg);
   }
   for (unsigned i = 0, e = VirtRegOpers.Defs.size(); i < e; ++i) {
     unsigned Reg = VirtRegOpers.Defs[i];
-    if (!findReg(Reg, true, VirtRegOpers.Uses, TRI))
+    if (!containsReg(VirtRegOpers.Uses, Reg))
-      decreaseVirtRegPressure(VirtRegOpers.Defs);
+      decreaseVirtRegPressure(Reg);
   }
   // Generate liveness for uses.
   for (unsigned i = 0, e = PhysRegOpers.Uses.size(); i < e; ++i) {
     unsigned Reg = PhysRegOpers.Uses[i];
-    if (!hasRegAlias(Reg, LivePhysRegs, TRI))
+    if (!LivePhysRegs.count(Reg))
       increasePhysRegPressure(Reg);
   }
   for (unsigned i = 0, e = VirtRegOpers.Uses.size(); i < e; ++i) {

test/CodeGen/X86/misched-new.ll

@@ -1,6 +1,9 @@
 ; RUN: llc < %s -march=x86-64 -mcpu=core2 -x86-early-ifcvt -enable-misched \
 ; RUN:          -misched=shuffle -misched-bottomup -verify-machineinstrs \
 ; RUN:     | FileCheck %s
+; RUN: llc < %s -march=x86-64 -mcpu=core2 -x86-early-ifcvt -enable-misched \
+; RUN:          -misched=shuffle -misched-topdown -verify-machineinstrs \
+; RUN:     | FileCheck %s --check-prefix TOPDOWN
 ; REQUIRES: asserts
 ;
 ; Interesting MachineScheduler cases.
@@ -77,3 +80,30 @@ define void @hasundef() unnamed_addr uwtable ssp align 2 {
 ; <label>:5                                       ; preds = %3
   ret void
 }
+
+; Test top-down subregister liveness tracking. Self-verification
+; catches any pressure set underflow.
+; rdar://12797931.
+;
+; TOPDOWN: @testSubregTracking
+; TOPDOWN: divb
+; TOPDOWN: movzbl %al
+; TOPDOWN: ret
+define void @testSubregTracking() nounwind uwtable ssp align 2 {
+  %tmp = load i8* undef, align 1
+  %tmp6 = sub i8 0, %tmp
+  %tmp7 = load i8* undef, align 1
+  %tmp8 = udiv i8 %tmp6, %tmp7
+  %tmp9 = zext i8 %tmp8 to i64
+  %tmp10 = load i8* undef, align 1
+  %tmp11 = zext i8 %tmp10 to i64
+  %tmp12 = mul i64 %tmp11, %tmp9
+  %tmp13 = urem i8 %tmp6, %tmp7
+  %tmp14 = zext i8 %tmp13 to i32
+  %tmp15 = add nsw i32 %tmp14, 0
+  %tmp16 = add i32 %tmp15, 0
+  store i32 %tmp16, i32* undef, align 4
+  %tmp17 = add i64 0, %tmp12
+  store i64 %tmp17, i64* undef, align 8
+  ret void
+}

utils/TableGen/RegisterInfoEmitter.cpp

@@ -195,7 +195,9 @@ EmitRegUnitPressure(raw_ostream &OS, const CodeGenRegBank &RegBank,
   }
   OS << "/// Get the weight in units of pressure for this register unit.\n"
      << "unsigned " << ClassName << "::\n"
-     << "getRegUnitWeight(unsigned RegUnit) const {\n";
+     << "getRegUnitWeight(unsigned RegUnit) const {\n"
+     << "  assert(RegUnit < " << RegBank.getNumNativeRegUnits()
+     << " && \"invalid register unit\");\n";
   if (!RegUnitsHaveUnitWeight) {
     OS << "  static const uint8_t RUWeightTable[] = {\n    ";
     for (unsigned UnitIdx = 0, UnitEnd = RegBank.getNumNativeRegUnits();
@@ -290,7 +292,9 @@ EmitRegUnitPressure(raw_ostream &OS, const CodeGenRegBank &RegBank,
      << "register unit.\n"
      << "/// Returns a -1 terminated array of pressure set IDs\n"
      << "const int* " << ClassName << "::\n"
-     << "getRegUnitPressureSets(unsigned RegUnit) const {\n";
+     << "getRegUnitPressureSets(unsigned RegUnit) const {\n"
+     << "  assert(RegUnit < " << RegBank.getNumNativeRegUnits()
+     << " && \"invalid register unit\");\n";
   OS << "  static const unsigned RUSetStartTable[] = {\n    ";
   for (unsigned UnitIdx = 0, UnitEnd = RegBank.getNumNativeRegUnits();
        UnitIdx < UnitEnd; ++UnitIdx) {