* Improve comments/documentation substantially

* Eliminate the releaseMemory method, this is not an analysis
* Change the fixed, active, and inactive lists of intervals to maintain an
  iterator for the current position in the interval.  This allows us to do
  constant time increments of the iterator instead of having to do a binary
  search to find our liverange in our liveinterval all of the time, which
  substantially speeds up cases where LiveIntervals have many LiveRanges
  - which is very common for physical registers.  On targets with many
  physregs, this can make a noticable difference.

  With a release build of LLC for PPC, this halves the time in
  processInactiveIntervals and processActiveIntervals, from 1.5s to .75s.

  This also lays the ground for more to come.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@17933 91177308-0d34-0410-b5e6-96231b3b80d8

lib/CodeGen/RegAllocLinearScan.cpp

@@ -29,6 +29,7 @@
 #include <cmath>
 #include <set>
 #include <queue>
+
 using namespace llvm;
 
 namespace {
@@ -39,16 +40,33 @@ namespace {
   static unsigned numIterations = 0;
   static unsigned numIntervals = 0;
 
-  class RA : public MachineFunctionPass {
+  struct RA : public MachineFunctionPass {
+    typedef std::pair<LiveInterval*, LiveInterval::iterator> IntervalPtr;
+    typedef std::vector<IntervalPtr> IntervalPtrs;
+  private:
     MachineFunction* mf_;
     const TargetMachine* tm_;
     const MRegisterInfo* mri_;
     LiveIntervals* li_;
-    typedef LiveInterval* IntervalPtr;
+
-    typedef std::vector<IntervalPtr> IntervalPtrs;
+    /// handled_ - Intervals are added to the handled_ set in the order of their
-    IntervalPtrs handled_, fixed_, active_, inactive_;
+    /// start value.  This is uses for backtracking.
+    std::vector<LiveInterval*> handled_;
+
+    /// fixed_ - Intervals that correspond to machine registers.
+    ///
+    IntervalPtrs fixed_;
+
+    /// active_ - Intervals that are currently being processed, and which have a
+    /// live range active for the current point.
+    IntervalPtrs active_;
+
+    /// inactive_ - Intervals that are currently being processed, but which have
+    /// a hold at the current point.
+    IntervalPtrs inactive_;
+
     typedef std::priority_queue<LiveInterval*,
-                                IntervalPtrs,
+                                std::vector<LiveInterval*>,
                                 greater_ptr<LiveInterval> > IntervalHeap;
     IntervalHeap unhandled_;
     std::auto_ptr<PhysRegTracker> prt_;
@@ -71,26 +89,24 @@ namespace {
     /// runOnMachineFunction - register allocate the whole function
     bool runOnMachineFunction(MachineFunction&);
 
-    void releaseMemory();
-
   private:
     /// linearScan - the linear scan algorithm
     void linearScan();
 
-    /// initIntervalSets - initializa the four interval sets:
+    /// initIntervalSets - initialize the interval sets.
-    /// unhandled, fixed, active and inactive
+    ///
     void initIntervalSets();
 
-    /// processActiveIntervals - expire old intervals and move
+    /// processActiveIntervals - expire old intervals and move non-overlapping
-    /// non-overlapping ones to the incative list
+    /// ones to the inactive list.
-    void processActiveIntervals(LiveInterval* cur);
+    void processActiveIntervals(unsigned CurPoint);
 
-    /// processInactiveIntervals - expire old intervals and move
+    /// processInactiveIntervals - expire old intervals and move overlapping
-    /// overlapping ones to the active list
+    /// ones to the active list.
-    void processInactiveIntervals(LiveInterval* cur);
+    void processInactiveIntervals(unsigned CurPoint);
 
     /// updateSpillWeights - updates the spill weights of the
-    /// specifed physical register and its weight
+    /// specifed physical register and its weight.
     void updateSpillWeights(unsigned reg, SpillWeights::value_type weight);
 
     /// assignRegOrStackSlotAtInterval - assign a register if one
@@ -101,9 +117,8 @@ namespace {
     /// register handling helpers
     ///
 
-    /// getFreePhysReg - return a free physical register for this
+    /// getFreePhysReg - return a free physical register for this virtual
-    /// virtual register interval if we have one, otherwise return
+    /// register interval if we have one, otherwise return 0.
-    /// 0
     unsigned getFreePhysReg(LiveInterval* cur);
 
     /// assignVirt2StackSlot - assigns this virtual register to a
@@ -114,8 +129,8 @@ namespace {
     void printIntervals(const char* const str, ItTy i, ItTy e) const {
       if (str) std::cerr << str << " intervals:\n";
       for (; i != e; ++i) {
-        std::cerr << "\t" << **i << " -> ";
+        std::cerr << "\t" << *i->first << " -> ";
-        unsigned reg = (*i)->reg;
+        unsigned reg = i->first->reg;
         if (MRegisterInfo::isVirtualRegister(reg)) {
           reg = vrm_->getPhys(reg);
         }
@@ -125,15 +140,6 @@ namespace {
   };
 }
 
-void RA::releaseMemory()
-{
-  while (!unhandled_.empty()) unhandled_.pop();
-  fixed_.clear();
-  active_.clear();
-  inactive_.clear();
-  handled_.clear();
-}
-
 bool RA::runOnMachineFunction(MachineFunction &fn) {
   mf_ = &fn;
   tm_ = &fn.getTarget();
@@ -150,6 +156,14 @@ bool RA::runOnMachineFunction(MachineFunction &fn) {
   spiller_->runOnMachineFunction(*mf_, *vrm_);
 
   vrm_.reset();  // Free the VirtRegMap
+
+
+  while (!unhandled_.empty()) unhandled_.pop();
+  fixed_.clear();
+  active_.clear();
+  inactive_.clear();
+  handled_.clear();
+
   return true;
 }
 
@@ -172,19 +186,18 @@ void RA::linearScan()
     ++numIterations;
     DEBUG(std::cerr << "\n*** CURRENT ***: " << *cur << '\n');
 
-    processActiveIntervals(cur);
+    processActiveIntervals(cur->beginNumber());
-    processInactiveIntervals(cur);
+    processInactiveIntervals(cur->beginNumber());
 
     // if this register is fixed we are done
     if (MRegisterInfo::isPhysicalRegister(cur->reg)) {
       prt_->addRegUse(cur->reg);
-      active_.push_back(cur);
+      active_.push_back(std::make_pair(cur, cur->begin()));
       handled_.push_back(cur);
-    }
+    } else {
-    // otherwise we are allocating a virtual register. try to find
+      // otherwise we are allocating a virtual register. try to find a free
-    // a free physical register or spill an interval in order to
+      // physical register or spill an interval in order to assign it one (we
-    // assign it one (we could spill the current though).
+      // could spill the current though).
-    else {
       assignRegOrStackSlotAtInterval(cur);
     }
 
@@ -197,8 +210,8 @@ void RA::linearScan()
   // expire any remaining active intervals
   for (IntervalPtrs::reverse_iterator
          i = active_.rbegin(); i != active_.rend(); ) {
-    unsigned reg = (*i)->reg;
+    unsigned reg = i->first->reg;
-    DEBUG(std::cerr << "\tinterval " << **i << " expired\n");
+    DEBUG(std::cerr << "\tinterval " << *i->first << " expired\n");
     if (MRegisterInfo::isVirtualRegister(reg))
       reg = vrm_->getPhys(reg);
     prt_->delRegUse(reg);
@@ -208,13 +221,15 @@ void RA::linearScan()
   // expire any remaining inactive intervals
   for (IntervalPtrs::reverse_iterator
          i = inactive_.rbegin(); i != inactive_.rend(); ) {
-    DEBUG(std::cerr << "\tinterval " << **i << " expired\n");
+    DEBUG(std::cerr << "\tinterval " << *i->first << " expired\n");
     i = IntervalPtrs::reverse_iterator(inactive_.erase(i.base()-1));
   }
 
   DEBUG(std::cerr << *vrm_);
 }
 
+/// initIntervalSets - initialize the interval sets.
+///
 void RA::initIntervalSets()
 {
   assert(unhandled_.empty() && fixed_.empty() &&
@@ -224,79 +239,95 @@ void RA::initIntervalSets()
   for (LiveIntervals::iterator i = li_->begin(), e = li_->end(); i != e; ++i){
     unhandled_.push(&i->second);
     if (MRegisterInfo::isPhysicalRegister(i->second.reg))
-      fixed_.push_back(&i->second);
+      fixed_.push_back(std::make_pair(&i->second, i->second.begin()));
   }
 }
 
-void RA::processActiveIntervals(IntervalPtrs::value_type cur)
+/// processActiveIntervals - expire old intervals and move non-overlapping ones
+/// to the inactive list.
+void RA::processActiveIntervals(unsigned CurPoint)
 {
   DEBUG(std::cerr << "\tprocessing active intervals:\n");
 
-  IntervalPtrs::iterator ii = active_.begin(), ie = active_.end();
+  for (unsigned i = 0, e = active_.size(); i != e; ++i) {
-  while (ii != ie) {
+    LiveInterval *Interval = active_[i].first;
-    LiveInterval* i = *ii;
+    LiveInterval::iterator IntervalPos = active_[i].second;
-    unsigned reg = i->reg;
+    unsigned reg = Interval->reg;
 
-    // remove expired intervals
+    IntervalPos = Interval->advanceTo(IntervalPos, CurPoint);
-    if (i->expiredAt(cur->beginNumber())) {
+
-      DEBUG(std::cerr << "\t\tinterval " << *i << " expired\n");
+    if (IntervalPos == Interval->end()) {     // Remove expired intervals.
+      DEBUG(std::cerr << "\t\tinterval " << *Interval << " expired\n");
       if (MRegisterInfo::isVirtualRegister(reg))
         reg = vrm_->getPhys(reg);
       prt_->delRegUse(reg);
-      // swap with last element and move end iterator back one position
+
-      std::iter_swap(ii, --ie);
+      // Pop off the end of the list.
-    }
+      active_[i] = active_.back();
-    // move inactive intervals to inactive list
+      active_.pop_back();
-    else if (!i->liveAt(cur->beginNumber())) {
+      --i; --e;
-      DEBUG(std::cerr << "\t\tinterval " << *i << " inactive\n");
+      
+    } else if (IntervalPos->start > CurPoint) {
+      // Move inactive intervals to inactive list.
+      DEBUG(std::cerr << "\t\tinterval " << *Interval << " inactive\n");
       if (MRegisterInfo::isVirtualRegister(reg))
         reg = vrm_->getPhys(reg);
       prt_->delRegUse(reg);
-      // add to inactive
+      // add to inactive.
-      inactive_.push_back(i);
+      inactive_.push_back(std::make_pair(Interval, IntervalPos));
-      // swap with last element and move end iterator back one postion
+
-      std::iter_swap(ii, --ie);
+      // Pop off the end of the list.
-    }
+      active_[i] = active_.back();
-    else {
+      active_.pop_back();
-      ++ii;
+      --i; --e;
+    } else {
+      // Otherwise, just update the iterator position.
+      active_[i].second = IntervalPos;
     }
   }
-  active_.erase(ie, active_.end());
 }
 
-void RA::processInactiveIntervals(IntervalPtrs::value_type cur)
+/// processInactiveIntervals - expire old intervals and move overlapping
+/// ones to the active list.
+void RA::processInactiveIntervals(unsigned CurPoint)
 {
   DEBUG(std::cerr << "\tprocessing inactive intervals:\n");
-  IntervalPtrs::iterator ii = inactive_.begin(), ie = inactive_.end();
+  for (unsigned i = 0, e = inactive_.size(); i != e; ++i) {
+    LiveInterval *Interval = inactive_[i].first;
+    LiveInterval::iterator IntervalPos = inactive_[i].second;
+    unsigned reg = Interval->reg;
 
-  while (ii != ie) {
+    IntervalPos = Interval->advanceTo(IntervalPos, CurPoint);
-    LiveInterval* i = *ii;
+    
-    unsigned reg = i->reg;
+    if (IntervalPos == Interval->end()) {       // remove expired intervals.
+      DEBUG(std::cerr << "\t\tinterval " << *Interval << " expired\n");
 
-    // remove expired intervals
+      // Pop off the end of the list.
-    if (i->expiredAt(cur->beginNumber())) {
+      inactive_[i] = inactive_.back();
-      DEBUG(std::cerr << "\t\tinterval " << *i << " expired\n");
+      inactive_.pop_back();
-      // swap with last element and move end iterator back one position
+      --i; --e;
-      std::iter_swap(ii, --ie);
+    } else if (IntervalPos->start <= CurPoint) {
-    }
+      // move re-activated intervals in active list
-    // move re-activated intervals in active list
+      DEBUG(std::cerr << "\t\tinterval " << *Interval << " active\n");
-    else if (i->liveAt(cur->beginNumber())) {
-      DEBUG(std::cerr << "\t\tinterval " << *i << " active\n");
       if (MRegisterInfo::isVirtualRegister(reg))
         reg = vrm_->getPhys(reg);
       prt_->addRegUse(reg);
       // add to active
-      active_.push_back(i);
+      active_.push_back(std::make_pair(Interval, IntervalPos));
-      // swap with last element and move end iterator back one position
+
-      std::iter_swap(ii, --ie);
+      // Pop off the end of the list.
-    }
+      inactive_[i] = inactive_.back();
-    else {
+      inactive_.pop_back();
-      ++ii;
+      --i; --e;
+    } else {
+      // Otherwise, just update the iterator position.
+      inactive_[i].second = IntervalPos;
     }
   }
-  inactive_.erase(ie, inactive_.end());
 }
 
+/// updateSpillWeights - updates the spill weights of the specifed physical
+/// register and its weight.
 void RA::updateSpillWeights(unsigned reg, SpillWeights::value_type weight)
 {
   spillWeights_[reg] += weight;
@@ -304,6 +335,17 @@ void RA::updateSpillWeights(unsigned reg, SpillWeights::value_type weight)
     spillWeights_[*as] += weight;
 }
 
+static RA::IntervalPtrs::iterator FindIntervalInVector(RA::IntervalPtrs &IP,
+                                                       LiveInterval *LI) {
+  for (RA::IntervalPtrs::iterator I = IP.begin(), E = IP.end(); I != E; ++I)
+    if (I->first == LI) return I;
+  return IP.end();
+}
+
+
+
+/// assignRegOrStackSlotAtInterval - assign a register if one is available, or
+/// spill.
 void RA::assignRegOrStackSlotAtInterval(LiveInterval* cur)
 {
   DEBUG(std::cerr << "\tallocating current interval: ");
@@ -315,22 +357,22 @@ void RA::assignRegOrStackSlotAtInterval(LiveInterval* cur)
   // for each interval in active update spill weights
   for (IntervalPtrs::const_iterator i = active_.begin(), e = active_.end();
        i != e; ++i) {
-    unsigned reg = (*i)->reg;
+    unsigned reg = i->first->reg;
     if (MRegisterInfo::isVirtualRegister(reg))
       reg = vrm_->getPhys(reg);
-    updateSpillWeights(reg, (*i)->weight);
+    updateSpillWeights(reg, i->first->weight);
   }
 
   // for every interval in inactive we overlap with, mark the
   // register as not free and update spill weights
   for (IntervalPtrs::const_iterator i = inactive_.begin(),
          e = inactive_.end(); i != e; ++i) {
-    if (cur->overlaps(**i)) {
+    if (cur->overlaps(*i->first)) {
-      unsigned reg = (*i)->reg;
+      unsigned reg = i->first->reg;
       if (MRegisterInfo::isVirtualRegister(reg))
         reg = vrm_->getPhys(reg);
       prt_->addRegUse(reg);
-      updateSpillWeights(reg, (*i)->weight);
+      updateSpillWeights(reg, i->first->weight);
     }
   }
 
@@ -338,10 +380,10 @@ void RA::assignRegOrStackSlotAtInterval(LiveInterval* cur)
   // mark the register as not free and update spill weights
   for (IntervalPtrs::const_iterator i = fixed_.begin(),
          e = fixed_.end(); i != e; ++i) {
-    if (cur->overlaps(**i)) {
+    if (cur->overlaps(*i->first)) {
-      unsigned reg = (*i)->reg;
+      unsigned reg = i->first->reg;
       prt_->addRegUse(reg);
-      updateSpillWeights(reg, (*i)->weight);
+      updateSpillWeights(reg, i->first->weight);
     }
   }
 
@@ -355,7 +397,7 @@ void RA::assignRegOrStackSlotAtInterval(LiveInterval* cur)
     DEBUG(std::cerr <<  mri_->getName(physReg) << '\n');
     vrm_->assignVirt2Phys(cur->reg, physReg);
     prt_->addRegUse(physReg);
-    active_.push_back(cur);
+    active_.push_back(std::make_pair(cur, cur->begin()));
     handled_.push_back(cur);
     return;
   }
@@ -428,64 +470,63 @@ void RA::assignRegOrStackSlotAtInterval(LiveInterval* cur)
   // point
   for (IntervalPtrs::iterator
          i = active_.begin(); i != active_.end(); ++i) {
-    unsigned reg = (*i)->reg;
+    unsigned reg = i->first->reg;
     if (MRegisterInfo::isVirtualRegister(reg) &&
         toSpill[vrm_->getPhys(reg)] &&
-        cur->overlaps(**i)) {
+        cur->overlaps(*i->first)) {
-      DEBUG(std::cerr << "\t\t\tspilling(a): " << **i << '\n');
+      DEBUG(std::cerr << "\t\t\tspilling(a): " << *i->first << '\n');
-      earliestStart = std::min(earliestStart, (*i)->beginNumber());
+      earliestStart = std::min(earliestStart, i->first->beginNumber());
-      int slot = vrm_->assignVirt2StackSlot((*i)->reg);
+      int slot = vrm_->assignVirt2StackSlot(i->first->reg);
       std::vector<LiveInterval*> newIs =
-        li_->addIntervalsForSpills(**i, *vrm_, slot);
+        li_->addIntervalsForSpills(*i->first, *vrm_, slot);
       std::copy(newIs.begin(), newIs.end(), std::back_inserter(added));
       spilled.insert(reg);
     }
   }
   for (IntervalPtrs::iterator
          i = inactive_.begin(); i != inactive_.end(); ++i) {
-    unsigned reg = (*i)->reg;
+    unsigned reg = i->first->reg;
     if (MRegisterInfo::isVirtualRegister(reg) &&
         toSpill[vrm_->getPhys(reg)] &&
-        cur->overlaps(**i)) {
+        cur->overlaps(*i->first)) {
-      DEBUG(std::cerr << "\t\t\tspilling(i): " << **i << '\n');
+      DEBUG(std::cerr << "\t\t\tspilling(i): " << *i->first << '\n');
-      earliestStart = std::min(earliestStart, (*i)->beginNumber());
+      earliestStart = std::min(earliestStart, i->first->beginNumber());
-      int slot = vrm_->assignVirt2StackSlot((*i)->reg);
+      int slot = vrm_->assignVirt2StackSlot(reg);
       std::vector<LiveInterval*> newIs =
-        li_->addIntervalsForSpills(**i, *vrm_, slot);
+        li_->addIntervalsForSpills(*i->first, *vrm_, slot);
       std::copy(newIs.begin(), newIs.end(), std::back_inserter(added));
       spilled.insert(reg);
     }
   }
 
   DEBUG(std::cerr << "\t\trolling back to: " << earliestStart << '\n');
-  // scan handled in reverse order up to the earliaset start of a
+
+  // Scan handled in reverse order up to the earliest start of a
   // spilled live interval and undo each one, restoring the state of
-  // unhandled
+  // unhandled.
   while (!handled_.empty()) {
     LiveInterval* i = handled_.back();
-    // if this interval starts before t we are done
+    // If this interval starts before t we are done.
     if (i->beginNumber() < earliestStart)
       break;
     DEBUG(std::cerr << "\t\t\tundo changes for: " << *i << '\n');
     handled_.pop_back();
-    // when undoing a live interval allocation we must know if it
+
-    // is active or inactive to properly update the PhysRegTracker
+    // When undoing a live interval allocation we must know if it is active or
-    // and the VirtRegMap
+    // inactive to properly update the PhysRegTracker and the VirtRegMap.
     IntervalPtrs::iterator it;
-    if ((it = std::find(active_.begin(), active_.end(), i)) != active_.end()) {
+    if ((it = FindIntervalInVector(active_, i)) != active_.end()) {
       active_.erase(it);
       if (MRegisterInfo::isPhysicalRegister(i->reg)) {
         prt_->delRegUse(i->reg);
         unhandled_.push(i);
-      }
+      } else {
-      else {
         if (!spilled.count(i->reg))
           unhandled_.push(i);
         prt_->delRegUse(vrm_->getPhys(i->reg));
         vrm_->clearVirt(i->reg);
       }
-    }
+    } else if ((it = FindIntervalInVector(inactive_, i)) != inactive_.end()) {
-    else if ((it = std::find(inactive_.begin(), inactive_.end(), i)) != inactive_.end()) {
       inactive_.erase(it);
       if (MRegisterInfo::isPhysicalRegister(i->reg))
         unhandled_.push(i);
@@ -505,15 +546,16 @@ void RA::assignRegOrStackSlotAtInterval(LiveInterval* cur)
   // scan the rest and undo each interval that expired after t and
   // insert it in active (the next iteration of the algorithm will
   // put it in inactive if required)
-  for (IntervalPtrs::iterator i = handled_.begin(), e = handled_.end(); 
+  for (unsigned i = 0, e = handled_.size(); i != e; ++i) {
-       i != e; ++i) {
+    LiveInterval *HI = handled_[i];
-    if (!(*i)->expiredAt(earliestStart) && (*i)->expiredAt(cur->beginNumber())){
+    if (!HI->expiredAt(earliestStart) &&
-      DEBUG(std::cerr << "\t\t\tundo changes for: " << **i << '\n');
+        HI->expiredAt(cur->beginNumber())) {
-      active_.push_back(*i);
+      DEBUG(std::cerr << "\t\t\tundo changes for: " << *HI << '\n');
-      if (MRegisterInfo::isPhysicalRegister((*i)->reg))
+      active_.push_back(std::make_pair(HI, HI->begin()));
-        prt_->addRegUse((*i)->reg);
+      if (MRegisterInfo::isPhysicalRegister(HI->reg))
+        prt_->addRegUse(HI->reg);
       else
-        prt_->addRegUse(vrm_->getPhys((*i)->reg));
+        prt_->addRegUse(vrm_->getPhys(HI->reg));
     }
   }
 
@@ -522,12 +564,14 @@ void RA::assignRegOrStackSlotAtInterval(LiveInterval* cur)
     unhandled_.push(added[i]);
 }
 
+/// getFreePhysReg - return a free physical register for this virtual register
+/// interval if we have one, otherwise return 0.
 unsigned RA::getFreePhysReg(LiveInterval* cur)
 {
   std::vector<unsigned> inactiveCounts(mri_->getNumRegs(), 0);
   for (IntervalPtrs::iterator i = inactive_.begin(), e = inactive_.end();
        i != e; ++i) {
-    unsigned reg = (*i)->reg;
+    unsigned reg = i->first->reg;
     if (MRegisterInfo::isVirtualRegister(reg))
       reg = vrm_->getPhys(reg);
     ++inactiveCounts[reg];