Finally committing to the new scheduler. Still -sched=none by default.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@23702 91177308-0d34-0410-b5e6-96231b3b80d8

lib/CodeGen/SelectionDAG/ScheduleDAG.cpp

@@ -190,13 +190,16 @@ public:
 };
 //===----------------------------------------------------------------------===//
 
+// Forward
+class NodeInfo;
+typedef std::vector<NodeInfo *>           NIVector;
+typedef std::vector<NodeInfo *>::iterator NIIterator;
 
 //===----------------------------------------------------------------------===//
 ///
 /// Node group -  This struct is used to manage flagged node groups.
 ///
-class NodeInfo;
+class NodeGroup : public NIVector {
-class NodeGroup : public std::vector<NodeInfo *> {
 private:
   int           Pending;                // Number of visits pending before
                                         //    adding to order  
@@ -232,7 +235,10 @@ public:
   unsigned      Slot;                   // Node's time slot
   NodeGroup     *Group;                 // Grouping information
   unsigned      VRBase;                 // Virtual register base
-  
+#ifndef NDEBUG
+  unsigned      Preorder;               // Index before scheduling
+#endif
+
   // Ctor.
   NodeInfo(SDNode *N = NULL)
   : Pending(0)
@@ -264,7 +270,6 @@ public:
     else       return Pending += I;
   }
 };
-typedef std::vector<NodeInfo *>::iterator NIIterator;
 //===----------------------------------------------------------------------===//
 
 
@@ -383,10 +388,9 @@ private:
   unsigned NodeCount;                   // Number of nodes in DAG
   NodeInfo *Info;                       // Info for nodes being scheduled
   std::map<SDNode *, NodeInfo *> Map;   // Map nodes to info
-  std::vector<NodeInfo*> Ordering;      // Emit ordering of nodes
+  NIVector Ordering;                    // Emit ordering of nodes
   ResourceTally<unsigned> Tally;        // Resource usage tally
   unsigned NSlots;                      // Total latency
-  std::map<SDNode *, unsigned> VRMap;   // Node to VR map
   static const unsigned NotFound = ~0U; // Search marker
   
 public:
@@ -396,7 +400,7 @@ public:
     : BB(bb), DAG(D), TM(D.getTarget()), TII(*TM.getInstrInfo()),
       MRI(*TM.getRegisterInfo()), RegMap(BB->getParent()->getSSARegMap()),
       ConstPool(BB->getParent()->getConstantPool()),
-      NSlots(0) {
+      NodeCount(0), Info(NULL), Map(), Tally(), NSlots(0) {
     assert(&TII && "Target doesn't provide instr info?");
     assert(&MRI && "Target doesn't provide register info?");
   }
@@ -427,7 +431,9 @@ private:
   void IncludeNode(NodeInfo *NI);
   void VisitAll();
   void Schedule();
-  void GatherNodeInfo();
+  void IdentifyGroups();
+  void GatherSchedulingInfo();
+  void PrepareNodeInfo();
   bool isStrongDependency(NodeInfo *A, NodeInfo *B);
   bool isWeakDependency(NodeInfo *A, NodeInfo *B);
   void ScheduleBackward();
@@ -439,8 +445,8 @@ private:
   unsigned CreateVirtualRegisters(MachineInstr *MI,
                                   unsigned NumResults,
                                   const TargetInstrDescriptor &II);
-  unsigned EmitDAG(SDOperand A);
 
+  void printChanges(unsigned Index);
   void printSI(std::ostream &O, NodeInfo *NI) const;
   void print(std::ostream &O) const;
   inline void dump(const char *tag) const { std::cerr << tag; dump(); }
@@ -635,28 +641,34 @@ void SimpleSched::VisitAll() {
   }
 }
 
-/// GatherNodeInfo - Get latency and resource information about each node.
+/// IdentifyGroups - Put flagged nodes into groups.
-/// 
+///
-void SimpleSched::GatherNodeInfo() {
+void SimpleSched::IdentifyGroups() {
-  // Allocate node information
-  Info = new NodeInfo[NodeCount];
-  // Get base of all nodes table
-  SelectionDAG::allnodes_iterator AllNodes = DAG.allnodes_begin();
-  
-  // For each node being scheduled
   for (unsigned i = 0, N = NodeCount; i < N; i++) {
-    // Get next node from DAG all nodes table
-    SDNode *Node = AllNodes[i];
-    // Fast reference to node schedule info
     NodeInfo* NI = &Info[i];
-    // Set up map
+    SDNode *Node = NI->Node;
-    Map[Node] = NI;
+
-    // Set node
+    // For each operand (in reverse to only look at flags)
-    NI->Node = Node;
+    for (unsigned N = Node->getNumOperands(); 0 < N--;) {
-    // Set pending visit count
+      // Get operand
-    NI->setPending(Node->use_size());    
+      SDOperand Op = Node->getOperand(N);
-    
+      // No more flags to walk
+      if (Op.getValueType() != MVT::Flag) break;
+      // Add to node group
+      NodeGroup::Add(getNI(Op.Val), NI);
+    }
+  }
+}
+
+/// GatherSchedulingInfo - Get latency and resource information about each node.
+///
+void SimpleSched::GatherSchedulingInfo() {
+  for (unsigned i = 0, N = NodeCount; i < N; i++) {
+    NodeInfo* NI = &Info[i];
+    SDNode *Node = NI->Node;
+
     MVT::ValueType VT = Node->getValueType(0);
+    
     if (Node->isTargetOpcode()) {
       MachineOpCode TOpc = Node->getTargetOpcode();
       // FIXME: This is an ugly (but temporary!) hack to test the scheduler
@@ -701,21 +713,28 @@ void SimpleSched::GatherNodeInfo() {
     // Sum up all the latencies for max tally size
     NSlots += NI->Latency;
   }
+}
 
-  // Put flagged nodes into groups
+/// PrepareNodeInfo - Set up the basic minimum node info for scheduling.
+/// 
+void SimpleSched::PrepareNodeInfo() {
+  // Allocate node information
+  Info = new NodeInfo[NodeCount];
+  // Get base of all nodes table
+  SelectionDAG::allnodes_iterator AllNodes = DAG.allnodes_begin();
+  
+  // For each node being scheduled
   for (unsigned i = 0, N = NodeCount; i < N; i++) {
+    // Get next node from DAG all nodes table
+    SDNode *Node = AllNodes[i];
+    // Fast reference to node schedule info
     NodeInfo* NI = &Info[i];
-    SDNode *Node = NI->Node;
+    // Set up map
-
+    Map[Node] = NI;
-    // For each operand (in reverse to only look at flags)
+    // Set node
-    for (unsigned N = Node->getNumOperands(); 0 < N--;) {
+    NI->Node = Node;
-      // Get operand
+    // Set pending visit count
-      SDOperand Op = Node->getOperand(N);
+    NI->setPending(Node->use_size());    
-      // No more flags to walk
-      if (Op.getValueType() != MVT::Flag) break;
-      // Add to node group
-      NodeGroup::Add(getNI(Op.Val), NI);
-    }
   }
 }
 
@@ -784,7 +803,7 @@ void SimpleSched::ScheduleBackward() {
     for (; j < N; j++) {
       // Get following instruction
       NodeInfo *Other = Ordering[j];
-      // Should we look further
+      // Should we look further (remember slots are in reverse time)
       if (Slot >= Other->Slot) break;
       // Shuffle other into ordering
       Ordering[j - 1] = Other;
@@ -837,7 +856,7 @@ void SimpleSched::ScheduleForward() {
     // Insert sort based on slot
     j = i;
     for (; 0 < j--;) {
-      // Get following instruction
+      // Get prior instruction
       NodeInfo *Other = Ordering[j];
       // Should we look further
       if (Slot >= Other->Slot) break;
@@ -856,11 +875,8 @@ void SimpleSched::EmitAll() {
   for (unsigned i = 0, N = Ordering.size(); i < N; i++) {
     // Get the scheduling info
     NodeInfo *NI = Ordering[i];
-#if 0
     // Iterate through nodes
     NodeGroupIterator NGI(Ordering[i]);
-    while (NodeInfo *NI = NGI.next()) EmitNode(NI);
-#else
     if (NI->isInGroup()) {
       if (NI->isGroupLeader()) {
         NodeGroupIterator NGI(Ordering[i]);
@@ -869,7 +885,6 @@ void SimpleSched::EmitAll() {
     } else {
       EmitNode(NI);
     }
-#endif
   }
 }
 
@@ -1055,228 +1070,89 @@ void SimpleSched::EmitNode(NodeInfo *NI) {
   NI->VRBase = VRBase;
 }
 
-/// EmitDag - Generate machine code for an operand and needed dependencies.
-///
-unsigned SimpleSched::EmitDAG(SDOperand Op) {
-  std::map<SDNode *, unsigned>::iterator OpI = VRMap.lower_bound(Op.Val);
-  if (OpI != VRMap.end() && OpI->first == Op.Val)
-    return OpI->second + Op.ResNo;
-  unsigned &OpSlot = VRMap.insert(OpI, std::make_pair(Op.Val, 0))->second;
-  
-  unsigned ResultReg = 0;
-  if (Op.isTargetOpcode()) {
-    unsigned Opc = Op.getTargetOpcode();
-    const TargetInstrDescriptor &II = TII.get(Opc);
-
-    unsigned NumResults = CountResults(Op.Val);
-    unsigned NodeOperands = CountOperands(Op.Val);
-    unsigned NumMIOperands = NodeOperands + NumResults;
-#ifndef NDEBUG
-    assert((unsigned(II.numOperands) == NumMIOperands || II.numOperands == -1)&&
-           "#operands for dag node doesn't match .td file!"); 
-#endif
-
-    // Create the new machine instruction.
-    MachineInstr *MI = new MachineInstr(Opc, NumMIOperands, true, true);
-    
-    // Add result register values for things that are defined by this
-    // instruction.
-    if (NumResults) ResultReg = CreateVirtualRegisters(MI, NumResults, II);
-    
-    // If there is a token chain operand, emit it first, as a hack to get avoid
-    // really bad cases.
-    if (Op.getNumOperands() > NodeOperands &&
-        Op.getOperand(NodeOperands).getValueType() == MVT::Other) {
-      EmitDAG(Op.getOperand(NodeOperands));
-    }
-    
-    // Emit all of the actual operands of this instruction, adding them to the
-    // instruction as appropriate.
-    for (unsigned i = 0; i != NodeOperands; ++i) {
-      if (Op.getOperand(i).isTargetOpcode()) {
-        // Note that this case is redundant with the final else block, but we
-        // include it because it is the most common and it makes the logic
-        // simpler here.
-        assert(Op.getOperand(i).getValueType() != MVT::Other &&
-               Op.getOperand(i).getValueType() != MVT::Flag &&
-               "Chain and flag operands should occur at end of operand list!");
-        
-        unsigned VReg = EmitDAG(Op.getOperand(i));
-        MI->addRegOperand(VReg, MachineOperand::Use);
-        
-        // Verify that it is right.
-        assert(MRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?");
-        assert(II.OpInfo[i+NumResults].RegClass &&
-               "Don't have operand info for this instruction!");
-#ifndef NDEBUG
-        if (RegMap->getRegClass(VReg) != II.OpInfo[i+NumResults].RegClass) {
-          std::cerr << "OP:  ";
-          Op.getOperand(i).Val->dump(&DAG); std::cerr << "\nUSE: ";
-          Op.Val->dump(&DAG); std::cerr << "\n";
-        }
-#endif
-        assert(RegMap->getRegClass(VReg) == II.OpInfo[i+NumResults].RegClass &&
-               "Register class of operand and regclass of use don't agree!");
-      } else if (ConstantSDNode *C =
-                                   dyn_cast<ConstantSDNode>(Op.getOperand(i))) {
-        MI->addZeroExtImm64Operand(C->getValue());
-      } else if (RegisterSDNode*R =dyn_cast<RegisterSDNode>(Op.getOperand(i))) {
-        MI->addRegOperand(R->getReg(), MachineOperand::Use);
-      } else if (GlobalAddressSDNode *TGA =
-                       dyn_cast<GlobalAddressSDNode>(Op.getOperand(i))) {
-        MI->addGlobalAddressOperand(TGA->getGlobal(), false, 0);
-      } else if (BasicBlockSDNode *BB =
-                       dyn_cast<BasicBlockSDNode>(Op.getOperand(i))) {
-        MI->addMachineBasicBlockOperand(BB->getBasicBlock());
-      } else if (FrameIndexSDNode *FI =
-                       dyn_cast<FrameIndexSDNode>(Op.getOperand(i))) {
-        MI->addFrameIndexOperand(FI->getIndex());
-      } else if (ConstantPoolSDNode *CP = 
-                    dyn_cast<ConstantPoolSDNode>(Op.getOperand(i))) {
-        unsigned Idx = ConstPool->getConstantPoolIndex(CP->get());
-        MI->addConstantPoolIndexOperand(Idx);
-      } else if (ExternalSymbolSDNode *ES = 
-                 dyn_cast<ExternalSymbolSDNode>(Op.getOperand(i))) {
-        MI->addExternalSymbolOperand(ES->getSymbol(), false);
-      } else {
-        assert(Op.getOperand(i).getValueType() != MVT::Other &&
-               Op.getOperand(i).getValueType() != MVT::Flag &&
-               "Chain and flag operands should occur at end of operand list!");
-        unsigned VReg = EmitDAG(Op.getOperand(i));
-        MI->addRegOperand(VReg, MachineOperand::Use);
-        
-        // Verify that it is right.
-        assert(MRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?");
-        assert(II.OpInfo[i+NumResults].RegClass &&
-               "Don't have operand info for this instruction!");
-        assert(RegMap->getRegClass(VReg) == II.OpInfo[i+NumResults].RegClass &&
-               "Register class of operand and regclass of use don't agree!");
-      }
-    }
-
-    // Finally, if this node has any flag operands, we *must* emit them last, to
-    // avoid emitting operations that might clobber the flags.
-    if (Op.getNumOperands() > NodeOperands) {
-      unsigned i = NodeOperands;
-      if (Op.getOperand(i).getValueType() == MVT::Other)
-        ++i;  // the chain is already selected.
-      for (unsigned N = Op.getNumOperands(); i < N; i++) {
-        assert(Op.getOperand(i).getValueType() == MVT::Flag &&
-               "Must be flag operands!");
-        EmitDAG(Op.getOperand(i));
-      }
-    }
-    
-    // Now that we have emitted all operands, emit this instruction itself.
-    if ((II.Flags & M_USES_CUSTOM_DAG_SCHED_INSERTION) == 0) {
-      BB->insert(BB->end(), MI);
-    } else {
-      // Insert this instruction into the end of the basic block, potentially
-      // taking some custom action.
-      BB = DAG.getTargetLoweringInfo().InsertAtEndOfBasicBlock(MI, BB);
-    }
-  } else {
-    switch (Op.getOpcode()) {
-    default:
-      Op.Val->dump(); 
-      assert(0 && "This target-independent node should have been selected!");
-    case ISD::EntryToken: break;
-    case ISD::TokenFactor:
-      for (unsigned i = 0, N = Op.getNumOperands(); i < N; i++) {
-        EmitDAG(Op.getOperand(i));
-      }
-      break;
-    case ISD::CopyToReg: {
-      SDOperand FlagOp;  FlagOp.ResNo = 0;
-      if (Op.getNumOperands() == 4) {
-        FlagOp = Op.getOperand(3);
-      }
-      if (Op.getOperand(0).Val != FlagOp.Val) {
-        EmitDAG(Op.getOperand(0));   // Emit the chain.
-      }
-      unsigned Val = EmitDAG(Op.getOperand(2));
-      if (FlagOp.Val) {
-        EmitDAG(FlagOp);
-      }
-      MRI.copyRegToReg(*BB, BB->end(),
-                       cast<RegisterSDNode>(Op.getOperand(1))->getReg(), Val,
-                       RegMap->getRegClass(Val));
-      break;
-    }
-    case ISD::CopyFromReg: {
-      EmitDAG(Op.getOperand(0));   // Emit the chain.
-      unsigned SrcReg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();
-      
-      // If the input is already a virtual register, just use it.
-      if (MRegisterInfo::isVirtualRegister(SrcReg)) {
-        ResultReg = SrcReg;
-        break;
-      }        
-
-      // Figure out the register class to create for the destreg.
-      const TargetRegisterClass *TRC = 0;
-
-      // Pick the register class of the right type that contains this physreg.
-      for (MRegisterInfo::regclass_iterator I = MRI.regclass_begin(),
-             E = MRI.regclass_end(); I != E; ++I)
-        if ((*I)->getType() == Op.Val->getValueType(0) &&
-            (*I)->contains(SrcReg)) {
-          TRC = *I;
-          break;
-        }
-      assert(TRC && "Couldn't find register class for reg copy!");
-      
-      // Create the reg, emit the copy.
-      ResultReg = RegMap->createVirtualRegister(TRC);
-      MRI.copyRegToReg(*BB, BB->end(), ResultReg, SrcReg, TRC);
-      break;
-    }
-    }
-  }
-
-  OpSlot = ResultReg;
-  return ResultReg+Op.ResNo;
-}
-
 /// Schedule - Order nodes according to selected style.
 ///
 void SimpleSched::Schedule() {
-  switch (ScheduleStyle) {
+  // Number the nodes
-  case simpleScheduling:
+  NodeCount = DAG.allnodes_size();
-    // Number the nodes
+  // Set up minimum info for scheduling.
-    NodeCount = DAG.allnodes_size();
+  PrepareNodeInfo();
-    // Don't waste time if is only entry and return
+  // Construct node groups for flagged nodes
-    if (NodeCount > 3) {
+  IdentifyGroups();
-      // Get latency and resource requirements
+  // Breadth first walk of DAG
-      GatherNodeInfo();
+  VisitAll();
-      // Breadth first walk of DAG
+
-      VisitAll();
+#ifndef NDEBUG
-      DEBUG(dump("Pre-"));
+  static unsigned Count = 0;
-      // Push back long instructions and critical path
+  Count++;
-      ScheduleBackward();
+  for (unsigned i = 0, N = Ordering.size(); i < N; i++) {
-      DEBUG(dump("Mid-"));
+    NodeInfo *NI = Ordering[i];
-      // Pack instructions to maximize resource utilization
+    NI->Preorder = i;
-      ScheduleForward();
-      DEBUG(dump("Post-"));
-      // Emit in scheduled order
-      EmitAll();
-      break;
-    } // fall thru
-  case noScheduling:
-    // Emit instructions in using a DFS from the exit root
-    EmitDAG(DAG.getRoot());
-    break;
   }
+#endif  
+  
+  // Don't waste time if is only entry and return
+  if (NodeCount > 3 && ScheduleStyle != noScheduling) {
+    // Get latency and resource requirements
+    GatherSchedulingInfo();
+    
+    // Push back long instructions and critical path
+    ScheduleBackward();
+    
+    // Pack instructions to maximize resource utilization
+    ScheduleForward();
+  }
+  
+  DEBUG(printChanges(Count));
+  
+  // Emit in scheduled order
+  EmitAll();
+}
+
+/// printChanges - Hilight changes in order caused by scheduling.
+///
+void SimpleSched::printChanges(unsigned Index) {
+#ifndef NDEBUG
+  // Get the ordered node count
+  unsigned N = Ordering.size();
+  // Determine if any changes
+  unsigned i = 0;
+  for (; i < N; i++) {
+    NodeInfo *NI = Ordering[i];
+    if (NI->Preorder != i) break;
+  }
+  
+  if (i < N) {
+    std::cerr << Index << ". New Ordering\n";
+    
+    for (i = 0; i < N; i++) {
+      NodeInfo *NI = Ordering[i];
+      std::cerr << "  " << NI->Preorder << ". ";
+      printSI(std::cerr, NI);
+      std::cerr << "\n";
+      if (NI->isGroupLeader()) {
+        NodeGroup *Group = NI->Group;
+        for (NIIterator NII = Group->begin(), E = Group->end();
+             NII != E; NII++) {
+          std::cerr << "      ";
+          printSI(std::cerr, *NII);
+          std::cerr << "\n";
+        }
+      }
+    }
+  } else {
+    std::cerr << Index << ". No Changes\n";
+  }
+#endif
 }
 
 /// printSI - Print schedule info.
 ///
 void SimpleSched::printSI(std::ostream &O, NodeInfo *NI) const {
 #ifndef NDEBUG
-  using namespace std;
   SDNode *Node = NI->Node;
   O << " "
-    << hex << Node
+    << std::hex << Node << std::dec
     << ", RS=" << NI->ResourceSet
     << ", Lat=" << NI->Latency
     << ", Slot=" << NI->Slot