Changed the liveness tracking in the RegisterScavenger

to use register units instead of registers.

reviewed by Jakob Stoklund Olesen.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@214798 91177308-0d34-0410-b5e6-96231b3b80d8

include/llvm/CodeGen/MachineRegisterInfo.h

@@ -516,8 +516,12 @@ public:
   ///
   /// That function will return NULL if the virtual registers have incompatible
   /// constraints.
+  ///
+  /// Note that if ToReg is a physical register the function will replace and
+  /// apply sub registers to ToReg in order to obtain a final/proper physical
+  /// register.
   void replaceRegWith(unsigned FromReg, unsigned ToReg);
-
+  
   /// getVRegDef - Return the machine instr that defines the specified virtual
   /// register or null if none is found.  This assumes that the code is in SSA
   /// form, so there should only be one definition.

include/llvm/CodeGen/RegisterScavenging.h

@@ -34,7 +34,7 @@ class RegScavenger {
   MachineRegisterInfo* MRI;
   MachineBasicBlock *MBB;
   MachineBasicBlock::iterator MBBI;
-  unsigned NumPhysRegs;
+  unsigned NumRegUnits;
 
   /// Tracking - True if RegScavenger is currently tracking the liveness of 
   /// registers.
@@ -58,22 +58,19 @@ class RegScavenger {
   /// A vector of information on scavenged registers.
   SmallVector<ScavengedInfo, 2> Scavenged;
 
-  /// CalleeSavedrRegs - A bitvector of callee saved registers for the target.
+  /// RegUnitsAvailable - The current state of each reg unit immediatelly
-  ///
+  /// before MBBI. One bit per register unit. If bit is not set it means any
-  BitVector CalleeSavedRegs;
+  /// register containing that register unit is currently being used.
-
+  BitVector RegUnitsAvailable;
-  /// RegsAvailable - The current state of all the physical registers immediately
-  /// before MBBI. One bit per physical register. If bit is set that means it's
-  /// available, unset means the register is currently being used.
-  BitVector RegsAvailable;
 
   // These BitVectors are only used internally to forward(). They are members
   // to avoid frequent reallocations.
-  BitVector KillRegs, DefRegs;
+  BitVector KillRegUnits, DefRegUnits;
+  BitVector TmpRegUnits;
 
 public:
   RegScavenger()
-    : MBB(nullptr), NumPhysRegs(0), Tracking(false) {}
+    : MBB(nullptr), NumRegUnits(0), Tracking(false) {}
 
   /// enterBasicBlock - Start tracking liveness from the begin of the specific
   /// basic block.
@@ -112,9 +109,9 @@ public:
   MachineBasicBlock::iterator getCurrentPosition() const {
     return MBBI;
   }
-
+  
-  /// getRegsUsed - return all registers currently in use in used.
+  /// isRegUsed - return if a specific register is currently used.
-  void getRegsUsed(BitVector &used, bool includeReserved);
+  bool isRegUsed(unsigned Reg, bool includeReserved = true) const;
 
   /// getRegsAvailable - Return all available registers in the register class
   /// in Mask.
@@ -157,40 +154,29 @@ public:
     return scavengeRegister(RegClass, MBBI, SPAdj);
   }
 
-  /// setUsed - Tell the scavenger a register is used.
+  /// setRegUsed - Tell the scavenger a register is used.
   ///
-  void setUsed(unsigned Reg);
+  void setRegUsed(unsigned Reg);
 private:
   /// isReserved - Returns true if a register is reserved. It is never "unused".
   bool isReserved(unsigned Reg) const { return MRI->isReserved(Reg); }
 
-  /// isUsed - Test if a register is currently being used.  When called by the
+  /// setUsed / setUnused - Mark the state of one or a number of register units.
-  /// isAliasUsed function, we only check isReserved if this is the original
-  /// register, not an alias register.
   ///
-  bool isUsed(unsigned Reg, bool CheckReserved = true) const   {
+  void setUsed(BitVector &RegUnits) {
-    return !RegsAvailable.test(Reg) || (CheckReserved && isReserved(Reg));
+    RegUnitsAvailable.reset(RegUnits);
+  }
+  void setUnused(BitVector &RegUnits) {
+    RegUnitsAvailable |= RegUnits;
   }
 
-  /// isAliasUsed - Is Reg or an alias currently in use?
+  /// Processes the current instruction and fill the KillRegUnits and
-  bool isAliasUsed(unsigned Reg) const;
+  /// DefRegUnits bit vectors.
-
-  /// setUsed / setUnused - Mark the state of one or a number of registers.
-  ///
-  void setUsed(BitVector &Regs) {
-    RegsAvailable.reset(Regs);
-  }
-  void setUnused(BitVector &Regs) {
-    RegsAvailable |= Regs;
-  }
-
-  /// Processes the current instruction and fill the KillRegs and DefRegs bit
-  /// vectors.
   void determineKillsAndDefs();
-
+  
-  /// Add Reg and all its sub-registers to BV.
+  /// Add all Reg Units that Reg contains to BV.
-  void addRegWithSubRegs(BitVector &BV, unsigned Reg);
+  void addRegUnits(BitVector &BV, unsigned Reg);
-
+  
   /// findSurvivorReg - Return the candidate register that is unused for the
   /// longest after StartMI. UseMI is set to the instruction where the search
   /// stopped.

lib/CodeGen/BranchFolding.cpp

@@ -389,10 +389,8 @@ void BranchFolder::MaintainLiveIns(MachineBasicBlock *CurMBB,
     RS->enterBasicBlock(CurMBB);
     if (!CurMBB->empty())
       RS->forward(std::prev(CurMBB->end()));
-    BitVector RegsLiveAtExit(TRI->getNumRegs());
+    for (unsigned int i = 1, e = TRI->getNumRegs(); i != e; i++)
-    RS->getRegsUsed(RegsLiveAtExit, false);
+      if (RS->isRegUsed(i, false))
-    for (unsigned int i = 0, e = TRI->getNumRegs(); i != e; i++)
-      if (RegsLiveAtExit[i])
         NewMBB->addLiveIn(i);
   }
 }

lib/CodeGen/MachineRegisterInfo.cpp

@@ -284,18 +284,25 @@ void MachineRegisterInfo::moveOperands(MachineOperand *Dst,
 /// replaceRegWith - Replace all instances of FromReg with ToReg in the
 /// machine function.  This is like llvm-level X->replaceAllUsesWith(Y),
 /// except that it also changes any definitions of the register as well.
+/// If ToReg is a physical register we apply the sub register to obtain the
+/// final/proper physical register.
 void MachineRegisterInfo::replaceRegWith(unsigned FromReg, unsigned ToReg) {
   assert(FromReg != ToReg && "Cannot replace a reg with itself");
 
+  const TargetRegisterInfo *TRI = getTargetRegisterInfo();
+  
   // TODO: This could be more efficient by bulk changing the operands.
   for (reg_iterator I = reg_begin(FromReg), E = reg_end(); I != E; ) {
     MachineOperand &O = *I;
     ++I;
-    O.setReg(ToReg);
+    if (TargetRegisterInfo::isPhysicalRegister(ToReg)) {
+      O.substPhysReg(ToReg, *TRI);
+    } else {
+      O.setReg(ToReg);
+    }
   }
 }
 
-
 /// getVRegDef - Return the machine instr that defines the specified virtual
 /// register or null if none is found.  This assumes that the code is in SSA
 /// form, so there should only be one definition.

lib/CodeGen/PrologEpilogInserter.cpp

@@ -852,7 +852,8 @@ void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn,
 /// FIXME: Iterating over the instruction stream is unnecessary. We can simply
 /// iterate over the vreg use list, which at this point only contains machine
 /// operands for which eliminateFrameIndex need a new scratch reg.
-void PEI::scavengeFrameVirtualRegs(MachineFunction &Fn) {
+void
+PEI::scavengeFrameVirtualRegs(MachineFunction &Fn) {
   // Run through the instructions and find any virtual registers.
   for (MachineFunction::iterator BB = Fn.begin(),
        E = Fn.end(); BB != E; ++BB) {
@@ -903,12 +904,16 @@ void PEI::scavengeFrameVirtualRegs(MachineFunction &Fn) {
           // Replace this reference to the virtual register with the
           // scratch register.
           assert (ScratchReg && "Missing scratch register!");
+          MachineRegisterInfo &MRI = Fn.getRegInfo();
           Fn.getRegInfo().replaceRegWith(Reg, ScratchReg);
+          
+          // Make sure MRI now accounts this register as used.
+          MRI.setPhysRegUsed(ScratchReg);
 
           // Because this instruction was processed by the RS before this
           // register was allocated, make sure that the RS now records the
           // register as being used.
-          RS->setUsed(ScratchReg);
+          RS->setRegUsed(ScratchReg);
         }
       }
 

lib/CodeGen/RegisterScavenging.cpp

@@ -31,18 +31,10 @@ using namespace llvm;
 
 #define DEBUG_TYPE "reg-scavenging"
 
-/// setUsed - Set the register and its sub-registers as being used.
+/// setUsed - Set the register units of this register as used.
-void RegScavenger::setUsed(unsigned Reg) {
+void RegScavenger::setRegUsed(unsigned Reg) {
-  for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true);
+  for (MCRegUnitIterator RUI(Reg, TRI); RUI.isValid(); ++RUI)
-       SubRegs.isValid(); ++SubRegs)
+    RegUnitsAvailable.reset(*RUI);
-    RegsAvailable.reset(*SubRegs);
-}
-
-bool RegScavenger::isAliasUsed(unsigned Reg) const {
-  for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
-    if (isUsed(*AI, *AI == Reg))
-      return true;
-  return false;
 }
 
 void RegScavenger::initRegState() {
@@ -52,8 +44,8 @@ void RegScavenger::initRegState() {
     I->Restore = nullptr;
   }
 
-  // All registers started out unused.
+  // All register units start out unused.
-  RegsAvailable.set();
+  RegUnitsAvailable.set();
 
   if (!MBB)
     return;
@@ -61,12 +53,12 @@ void RegScavenger::initRegState() {
   // Live-in registers are in use.
   for (MachineBasicBlock::livein_iterator I = MBB->livein_begin(),
          E = MBB->livein_end(); I != E; ++I)
-    setUsed(*I);
+    setRegUsed(*I);
 
   // Pristine CSRs are also unavailable.
   BitVector PR = MBB->getParent()->getFrameInfo()->getPristineRegs(MBB);
   for (int I = PR.find_first(); I>0; I = PR.find_next(I))
-    setUsed(I);
+    setRegUsed(I);
 }
 
 void RegScavenger::enterBasicBlock(MachineBasicBlock *mbb) {
@@ -76,7 +68,7 @@ void RegScavenger::enterBasicBlock(MachineBasicBlock *mbb) {
   TRI = TM.getSubtargetImpl()->getRegisterInfo();
   MRI = &MF.getRegInfo();
 
-  assert((NumPhysRegs == 0 || NumPhysRegs == TRI->getNumRegs()) &&
+  assert((NumRegUnits == 0 || NumRegUnits == TRI->getNumRegUnits()) &&
          "Target changed?");
 
   // It is not possible to use the register scavenger after late optimization
@@ -86,17 +78,11 @@ void RegScavenger::enterBasicBlock(MachineBasicBlock *mbb) {
 
   // Self-initialize.
   if (!MBB) {
-    NumPhysRegs = TRI->getNumRegs();
+    NumRegUnits = TRI->getNumRegUnits();
-    RegsAvailable.resize(NumPhysRegs);
+    RegUnitsAvailable.resize(NumRegUnits);
-    KillRegs.resize(NumPhysRegs);
+    KillRegUnits.resize(NumRegUnits);
-    DefRegs.resize(NumPhysRegs);
+    DefRegUnits.resize(NumRegUnits);
-
+    TmpRegUnits.resize(NumRegUnits);
-    // Create callee-saved registers bitvector.
-    CalleeSavedRegs.resize(NumPhysRegs);
-    const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(&MF);
-    if (CSRegs != nullptr)
-      for (unsigned i = 0; CSRegs[i]; ++i)
-        CalleeSavedRegs.set(CSRegs[i]);
   }
 
   MBB = mbb;
@@ -105,10 +91,9 @@ void RegScavenger::enterBasicBlock(MachineBasicBlock *mbb) {
   Tracking = false;
 }
 
-void RegScavenger::addRegWithSubRegs(BitVector &BV, unsigned Reg) {
+void RegScavenger::addRegUnits(BitVector &BV, unsigned Reg) {
-  for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true);
+  for (MCRegUnitIterator RUI(Reg, TRI); RUI.isValid(); ++RUI)
-       SubRegs.isValid(); ++SubRegs)
+    BV.set(*RUI);
-    BV.set(*SubRegs);
 }
 
 void RegScavenger::determineKillsAndDefs() {
@@ -123,12 +108,25 @@ void RegScavenger::determineKillsAndDefs() {
   // predicated, conservatively assume "kill" markers do not actually kill the
   // register. Similarly ignores "dead" markers.
   bool isPred = TII->isPredicated(MI);
-  KillRegs.reset();
+  KillRegUnits.reset();
-  DefRegs.reset();
+  DefRegUnits.reset();
   for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
     const MachineOperand &MO = MI->getOperand(i);
-    if (MO.isRegMask())
+    if (MO.isRegMask()) {
-      (isPred ? DefRegs : KillRegs).setBitsNotInMask(MO.getRegMask());
+      
+      TmpRegUnits.clear();
+      for (unsigned RU = 0, RUEnd = TRI->getNumRegUnits(); RU != RUEnd; ++RU) {
+        for (MCRegUnitRootIterator RURI(RU, TRI); RURI.isValid(); ++RURI) {
+          if (MO.clobbersPhysReg(*RURI)) {
+            TmpRegUnits.set(RU);
+            break;
+          }
+        }
+      }
+      
+      // Apply the mask.
+      (isPred ? DefRegUnits : KillRegUnits) |= TmpRegUnits;
+    }
     if (!MO.isReg())
       continue;
     unsigned Reg = MO.getReg();
@@ -140,13 +138,13 @@ void RegScavenger::determineKillsAndDefs() {
       if (MO.isUndef())
         continue;
       if (!isPred && MO.isKill())
-        addRegWithSubRegs(KillRegs, Reg);
+        addRegUnits(KillRegUnits, Reg);
     } else {
       assert(MO.isDef());
       if (!isPred && MO.isDead())
-        addRegWithSubRegs(KillRegs, Reg);
+        addRegUnits(KillRegUnits, Reg);
       else
-        addRegWithSubRegs(DefRegs, Reg);
+        addRegUnits(DefRegUnits, Reg);
     }
   }
 }
@@ -159,8 +157,8 @@ void RegScavenger::unprocess() {
     determineKillsAndDefs();
 
     // Commit the changes.
-    setUsed(KillRegs);
+    setUsed(KillRegUnits);
-    setUnused(DefRegs);
+    setUnused(DefRegUnits);
   }
 
   if (MBBI == MBB->begin()) {
@@ -209,7 +207,7 @@ void RegScavenger::forward() {
     if (MO.isUse()) {
       if (MO.isUndef())
         continue;
-      if (!isUsed(Reg)) {
+      if (!isRegUsed(Reg)) {
         // Check if it's partial live: e.g.
         // D0 = insert_subreg D0<undef>, S0
         // ... D0
@@ -220,15 +218,23 @@ void RegScavenger::forward() {
         // insert_subreg around causes both correctness and performance issues.
         bool SubUsed = false;
         for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs)
-          if (isUsed(*SubRegs)) {
+          if (isRegUsed(*SubRegs)) {
             SubUsed = true;
             break;
           }
-        if (!SubUsed) {
+        bool SuperUsed = false;
+        for (MCSuperRegIterator SR(Reg, TRI); SR.isValid(); ++SR) {
+          if (isRegUsed(*SR)) {
+            SuperUsed = true;
+            break;
+          }
+        }
+        if (!SubUsed && !SuperUsed) {
           MBB->getParent()->verify(nullptr, "In Register Scavenger");
           llvm_unreachable("Using an undefined register!");
         }
         (void)SubUsed;
+        (void)SuperUsed;
       }
     } else {
       assert(MO.isDef());
@@ -244,23 +250,23 @@ void RegScavenger::forward() {
 #endif // NDEBUG
 
   // Commit the changes.
-  setUnused(KillRegs);
+  setUnused(KillRegUnits);
-  setUsed(DefRegs);
+  setUsed(DefRegUnits);
 }
 
-void RegScavenger::getRegsUsed(BitVector &used, bool includeReserved) {
+bool RegScavenger::isRegUsed(unsigned Reg, bool includeReserved) const {
-  used = RegsAvailable;
+  if (includeReserved && isReserved(Reg))
-  used.flip();
+    return true;
-  if (includeReserved)
+  for (MCRegUnitIterator RUI(Reg, TRI); RUI.isValid(); ++RUI)
-    used |= MRI->getReservedRegs();
+    if (!RegUnitsAvailable.test(*RUI))
-  else
+      return true;
-    used.reset(MRI->getReservedRegs());
+  return false;
 }
 
 unsigned RegScavenger::FindUnusedReg(const TargetRegisterClass *RC) const {
   for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end();
        I != E; ++I)
-    if (!isAliasUsed(*I)) {
+    if (!isRegUsed(*I)) {
       DEBUG(dbgs() << "Scavenger found unused reg: " << TRI->getName(*I) <<
             "\n");
       return *I;
@@ -274,13 +280,13 @@ BitVector RegScavenger::getRegsAvailable(const TargetRegisterClass *RC) {
   BitVector Mask(TRI->getNumRegs());
   for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end();
        I != E; ++I)
-    if (!isAliasUsed(*I))
+    if (!isRegUsed(*I))
       Mask.set(*I);
   return Mask;
 }
 
 /// findSurvivorReg - Return the candidate register that is unused for the
-/// longest after StargMII. UseMI is set to the instruction where the search
+/// longest after StartMII. UseMI is set to the instruction where the search
 /// stopped.
 ///
 /// No more than InstrLimit instructions are inspected.
@@ -376,9 +382,7 @@ unsigned RegScavenger::scavengeRegister(const TargetRegisterClass *RC,
   }
 
   // Try to find a register that's unused if there is one, as then we won't
-  // have to spill. Search explicitly rather than masking out based on
+  // have to spill.
-  // RegsAvailable, as RegsAvailable does not take aliases into account.
-  // That's what getRegsAvailable() is for.
   BitVector Available = getRegsAvailable(RC);
   Available &= Candidates;
   if (Available.any())
@@ -389,7 +393,7 @@ unsigned RegScavenger::scavengeRegister(const TargetRegisterClass *RC,
   unsigned SReg = findSurvivorReg(I, Candidates, 25, UseMI);
 
   // If we found an unused register there is no reason to spill it.
-  if (!isAliasUsed(SReg)) {
+  if (!isRegUsed(SReg)) {
     DEBUG(dbgs() << "Scavenged register: " << TRI->getName(SReg) << "\n");
     return SReg;
   }

lib/CodeGen/TailDuplication.cpp

@@ -799,11 +799,9 @@ TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB,
       RS->enterBasicBlock(PredBB);
       if (!PredBB->empty())
         RS->forward(std::prev(PredBB->end()));
-      BitVector RegsLiveAtExit(TRI->getNumRegs());
-      RS->getRegsUsed(RegsLiveAtExit, false);
       for (MachineBasicBlock::livein_iterator I = TailBB->livein_begin(),
              E = TailBB->livein_end(); I != E; ++I) {
-        if (!RegsLiveAtExit[*I])
+        if (!RS->isRegUsed(*I, false))
           // If a register is previously livein to the tail but it's not live
           // at the end of predecessor BB, then it should be added to its
           // livein list.

lib/Target/XCore/XCoreRegisterInfo.cpp

@@ -99,7 +99,7 @@ static void InsertFPConstInst(MachineBasicBlock::iterator II,
   MachineBasicBlock &MBB = *MI.getParent();
   DebugLoc dl = MI.getDebugLoc();
   unsigned ScratchOffset = RS->scavengeRegister(&XCore::GRRegsRegClass, II, 0);
-  RS->setUsed(ScratchOffset);
+  RS->setRegUsed(ScratchOffset);
   TII.loadImmediate(MBB, II, ScratchOffset, Offset);
 
   switch (MI.getOpcode()) {
@@ -171,12 +171,12 @@ static void InsertSPConstInst(MachineBasicBlock::iterator II,
   unsigned ScratchBase;
   if (OpCode==XCore::STWFI) {
     ScratchBase = RS->scavengeRegister(&XCore::GRRegsRegClass, II, 0);
-    RS->setUsed(ScratchBase);
+    RS->setRegUsed(ScratchBase);
   } else
     ScratchBase = Reg;
   BuildMI(MBB, II, dl, TII.get(XCore::LDAWSP_ru6), ScratchBase).addImm(0);
   unsigned ScratchOffset = RS->scavengeRegister(&XCore::GRRegsRegClass, II, 0);
-  RS->setUsed(ScratchOffset);
+  RS->setRegUsed(ScratchOffset);
   TII.loadImmediate(MBB, II, ScratchOffset, Offset);
 
   switch (OpCode) {