From b6d4c2cf2d3e5173fb16c24d93b18eb13ba04f37 Mon Sep 17 00:00:00 2001 From: Jim Laskey Date: Fri, 30 Sep 2005 19:15:27 +0000 Subject: [PATCH] 1. Made things node-centric (from operand). 2. Added node groups to handle flagged nodes. 3. Started weaning simple scheduling off existing emitter. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@23566 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/CodeGen/SelectionDAG/ScheduleDAG.cpp | 881 ++++++++++++++++------- 1 file changed, 623 insertions(+), 258 deletions(-) diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp index be383a3fe3a..38bb2ffc294 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAG.cpp @@ -192,25 +192,177 @@ public: //===----------------------------------------------------------------------===// -// This struct tracks information used to schedule the a node. -struct ScheduleInfo { - SDOperand Op; // Operand information - unsigned Latency; // Cycles to complete instruction - unsigned ResourceSet; // Bit vector of usable resources - unsigned Slot; // Operand's time slot - +/// +/// Node group - This struct is used to manage flagged node groups. +/// +class NodeInfo; +class NodeGroup : public std::vector { +private: + int Pending; // Number of visits pending before + // adding to order + +public: // Ctor. - ScheduleInfo(SDOperand op) - : Op(op) - , Latency(0) - , ResourceSet(0) - , Slot(0) - {} + NodeGroup() : Pending(0) {} + + // Accessors + inline NodeInfo *getLeader() { return empty() ? NULL : front(); } + inline int getPending() const { return Pending; } + inline void setPending(int P) { Pending = P; } + inline int addPending(int I) { return Pending += I; } + + static void Add(NodeInfo *D, NodeInfo *U); + static unsigned CountInternalUses(NodeInfo *D, NodeInfo *U); }; //===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===// +/// +/// NodeInfo - This struct tracks information used to schedule the a node. +/// +class NodeInfo { +private: + int Pending; // Number of visits pending before + // adding to order +public: + SDNode *Node; // DAG node + unsigned Latency; // Cycles to complete instruction + unsigned ResourceSet; // Bit vector of usable resources + unsigned Slot; // Node's time slot + NodeGroup *Group; // Grouping information + unsigned VRBase; // Virtual register base + + // Ctor. + NodeInfo(SDNode *N = NULL) + : Pending(0) + , Node(N) + , Latency(0) + , ResourceSet(0) + , Slot(0) + , Group(NULL) + , VRBase(0) + {} + + // Accessors + inline bool isInGroup() const { + assert(!Group || !Group->empty() && "Group with no members"); + return Group != NULL; + } + inline bool isGroupLeader() const { + return isInGroup() && Group->getLeader() == this; + } + inline int getPending() const { + return Group ? Group->getPending() : Pending; + } + inline void setPending(int P) { + if (Group) Group->setPending(P); + else Pending = P; + } + inline int addPending(int I) { + if (Group) return Group->addPending(I); + else return Pending += I; + } +}; +typedef std::vector::iterator NIIterator; +//===----------------------------------------------------------------------===// + + +//===----------------------------------------------------------------------===// +/// +/// NodeGroupIterator - Iterates over all the nodes indicated by the node info. +/// If the node is in a group then iterate over the members of the group, +/// otherwise just the node info. +/// +class NodeGroupIterator { +private: + NodeInfo *NI; // Node info + NIIterator NGI; // Node group iterator + NIIterator NGE; // Node group iterator end + +public: + // Ctor. + NodeGroupIterator(NodeInfo *N) : NI(N) { + // If the node is in a group then set up the group iterator. Otherwise + // the group iterators will trip first time out. + if (N->isInGroup()) { + // get Group + NodeGroup *Group = NI->Group; + NGI = Group->begin(); + NGE = Group->end(); + // Prevent this node from being used (will be in members list + NI = NULL; + } + } + + /// next - Return the next node info, otherwise NULL. + /// + NodeInfo *next() { + // If members list + if (NGI != NGE) return *NGI++; + // Use node as the result (may be NULL) + NodeInfo *Result = NI; + // Only use once + NI = NULL; + // Return node or NULL + return Result; + } +}; +//===----------------------------------------------------------------------===// + + +//===----------------------------------------------------------------------===// +/// +/// NodeGroupOpIterator - Iterates over all the operands of a node. If the node +/// is a member of a group, this iterates over all the operands of all the +/// members of the group. +/// +class NodeGroupOpIterator { +private: + NodeInfo *NI; // Node containing operands + NodeGroupIterator GI; // Node group iterator + SDNode::op_iterator OI; // Operand iterator + SDNode::op_iterator OE; // Operand iterator end + + /// CheckNode - Test if node has more operands. If not get the next node + /// skipping over nodes that have no operands. + void CheckNode() { + // Only if operands are exhausted first + while (OI == OE) { + // Get next node info + NodeInfo *NI = GI.next(); + // Exit if nodes are exhausted + if (!NI) return; + // Get node itself + SDNode *Node = NI->Node; + // Set up the operand iterators + OI = Node->op_begin(); + OE = Node->op_end(); + } + } + +public: + // Ctor. + NodeGroupOpIterator(NodeInfo *N) : NI(N), GI(N) {} + + /// isEnd - Returns true when not more operands are available. + /// + inline bool isEnd() { CheckNode(); return OI == OE; } + + /// next - Returns the next available operand. + /// + inline SDOperand next() { + assert(OI != OE && "Not checking for end of NodeGroupOpIterator correctly"); + return *OI++; + } +}; +//===----------------------------------------------------------------------===// + + +//===----------------------------------------------------------------------===// +/// +/// SimpleSched - Simple two pass scheduler. +/// class SimpleSched { private: // TODO - get ResourceSet from TII @@ -228,12 +380,13 @@ private: const MRegisterInfo &MRI; // Target processor register information SSARegMap *RegMap; // Virtual/real register map MachineConstantPool *ConstPool; // Target constant pool - std::vector Operands; // All operands to be scheduled - std::vector Ordering; // Emit ordering of operands - std::map Visited; // Operands that have been visited + unsigned NodeCount; // Number of nodes in DAG + NodeInfo *Info; // Info for nodes being scheduled + std::map Map; // Map nodes to info + std::vector Ordering; // Emit ordering of nodes ResourceTally Tally; // Resource usage tally unsigned NSlots; // Total latency - std::mapVRMap; // Operand to VR map + std::map VRMap; // Node to VR map static const unsigned NotFound = ~0U; // Search marker public: @@ -255,37 +408,41 @@ public: } private: - static bool isFlagDefiner(SDOperand Op) { return isFlagDefiner(Op.Val); } - static bool isFlagUser(SDOperand Op) { return isFlagUser(Op.Val); } + /// getNI - Returns the node info for the specified node. + /// + inline NodeInfo *getNI(SDNode *Node) { return Map[Node]; } + + /// getVR - Returns the virtual register number of the node. + /// + inline unsigned getVR(SDOperand Op) { + NodeInfo *NI = getNI(Op.Val); + assert(NI->VRBase != 0 && "Node emitted out of order - late"); + return NI->VRBase + Op.ResNo; + } + static bool isFlagDefiner(SDNode *A); static bool isFlagUser(SDNode *A); - static bool isDefiner(SDNode *A, SDNode *B); - static bool isPassiveOperand(SDOperand Op); - void IncludeOperand(SDOperand Op); + static bool isDefiner(NodeInfo *A, NodeInfo *B); + static bool isPassiveNode(SDNode *Node); + void IncludeNode(NodeInfo *NI); void VisitAll(); void Schedule(); - void GatherOperandInfo(); - bool isStrongDependency(SDOperand A, SDOperand B) { - return isStrongDependency(A.Val, B.Val); - } - bool isWeakDependency(SDOperand A, SDOperand B) { - return isWeakDependency(A.Val, B.Val); - } - static bool isStrongDependency(SDNode *A, SDNode *B); - static bool isWeakDependency(SDNode *A, SDNode *B); + void GatherNodeInfo(); + bool isStrongDependency(NodeInfo *A, NodeInfo *B); + bool isWeakDependency(NodeInfo *A, NodeInfo *B); void ScheduleBackward(); void ScheduleForward(); void EmitAll(); - void EmitFlagUsers(SDOperand Op); - static unsigned CountResults(SDOperand Op); - static unsigned CountOperands(SDOperand Op); - unsigned CreateVirtualRegisters(SDOperand Op, MachineInstr *MI, + void EmitNode(NodeInfo *NI); + static unsigned CountResults(SDNode *Node); + static unsigned CountOperands(SDNode *Node); + unsigned CreateVirtualRegisters(MachineInstr *MI, unsigned NumResults, const TargetInstrDescriptor &II); - unsigned Emit(SDOperand A); + unsigned EmitDAG(SDOperand A); - void printSI(std::ostream &O, ScheduleInfo *SI) const ; - void print(std::ostream &O) const ; + void printSI(std::ostream &O, NodeInfo *NI) const; + void print(std::ostream &O) const; inline void dump(const char *tag) const { std::cerr << tag; dump(); } void dump() const; }; @@ -341,193 +498,284 @@ public: }; } // namespace +//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===// -/// isFlagDefiner - Returns true if the operand defines a flag result. +/// Add - Adds a definer and user pair to a node group. +/// +void NodeGroup::Add(NodeInfo *D, NodeInfo *U) { + // Get current groups + NodeGroup *DGroup = D->Group; + NodeGroup *UGroup = U->Group; + // If both are members of groups + if (DGroup && UGroup) { + // There may have been another edge connecting + if (DGroup == UGroup) return; + // Add the pending users count + DGroup->addPending(UGroup->getPending()); + // For each member of the users group + NodeGroupIterator UNGI(U); + while (NodeInfo *UNI = UNGI.next() ) { + // Change the group + UNI->Group = DGroup; + // For each member of the definers group + NodeGroupIterator DNGI(D); + while (NodeInfo *DNI = DNGI.next() ) { + // Remove internal edges + DGroup->addPending(-CountInternalUses(DNI, UNI)); + } + } + // Merge the two lists + DGroup->insert(DGroup->end(), UGroup->begin(), UGroup->end()); + } else if (DGroup) { + // Make user member of definers group + U->Group = DGroup; + // Add users uses to definers group pending + DGroup->addPending(U->Node->use_size()); + // For each member of the definers group + NodeGroupIterator DNGI(D); + while (NodeInfo *DNI = DNGI.next() ) { + // Remove internal edges + DGroup->addPending(-CountInternalUses(DNI, U)); + } + DGroup->push_back(U); + } else if (UGroup) { + // Make definer member of users group + D->Group = UGroup; + // Add definers uses to users group pending + UGroup->addPending(D->Node->use_size()); + // For each member of the users group + NodeGroupIterator UNGI(U); + while (NodeInfo *UNI = UNGI.next() ) { + // Remove internal edges + UGroup->addPending(-CountInternalUses(D, UNI)); + } + UGroup->insert(UGroup->begin(), D); + } else { + D->Group = U->Group = DGroup = new NodeGroup(); + DGroup->addPending(D->Node->use_size() + U->Node->use_size() - + CountInternalUses(D, U)); + DGroup->push_back(D); + DGroup->push_back(U); + } +} + +/// CountInternalUses - Returns the number of edges between the two nodes. +/// +unsigned NodeGroup::CountInternalUses(NodeInfo *D, NodeInfo *U) { + unsigned N = 0; + for (SDNode:: use_iterator UI = D->Node->use_begin(), + E = D->Node->use_end(); UI != E; UI++) { + if (*UI == U->Node) N++; + } + return N; +} +//===----------------------------------------------------------------------===// + + +//===----------------------------------------------------------------------===// +/// isFlagDefiner - Returns true if the node defines a flag result. bool SimpleSched::isFlagDefiner(SDNode *A) { unsigned N = A->getNumValues(); return N && A->getValueType(N - 1) == MVT::Flag; } -/// isFlagUser - Returns true if the operand uses a flag result. +/// isFlagUser - Returns true if the node uses a flag result. /// bool SimpleSched::isFlagUser(SDNode *A) { unsigned N = A->getNumOperands(); return N && A->getOperand(N - 1).getValueType() == MVT::Flag; } -/// isDefiner - Return true if Node A is a definder for B. +/// isDefiner - Return true if node A is a definer for B. /// -bool SimpleSched::isDefiner(SDNode *A, SDNode *B) { - for (unsigned i = 0, N = B->getNumOperands(); i < N; i++) { - if (B->getOperand(i).Val == A) return true; +bool SimpleSched::isDefiner(NodeInfo *A, NodeInfo *B) { + // While there are A nodes + NodeGroupIterator NII(A); + while (NodeInfo *NI = NII.next()) { + // Extract node + SDNode *Node = NI->Node; + // While there operands in nodes of B + NodeGroupOpIterator NGOI(B); + while (!NGOI.isEnd()) { + SDOperand Op = NGOI.next(); + // If node from A defines a node in B + if (Node == Op.Val) return true; + } } return false; } -/// isPassiveOperand - Return true if the operand is a non-scheduled leaf -/// operand. -bool SimpleSched::isPassiveOperand(SDOperand Op) { - if (isa(Op)) return true; - if (isa(Op)) return true; - if (isa(Op)) return true; - if (isa(Op)) return true; - if (isa(Op)) return true; - if (isa(Op)) return true; - if (isa(Op)) return true; +/// isPassiveNode - Return true if the node is a non-scheduled leaf. +/// +bool SimpleSched::isPassiveNode(SDNode *Node) { + if (isa(Node)) return true; + if (isa(Node)) return true; + if (isa(Node)) return true; + if (isa(Node)) return true; + if (isa(Node)) return true; + if (isa(Node)) return true; + if (isa(Node)) return true; return false; } -/// IncludeOperand - Add operand to ScheduleInfo vector. +/// IncludeNode - Add node to NodeInfo vector. /// -void SimpleSched::IncludeOperand(SDOperand Op) { +void SimpleSched::IncludeNode(NodeInfo *NI) { + // Get node + SDNode *Node = NI->Node; // Ignore entry node - if (Op.getOpcode() == ISD::EntryToken) return; - // Check current count for operand - int Count = Visited[Op.Val]; - // If the operand is already in list +if (Node->getOpcode() == ISD::EntryToken) return; + // Check current count for node + int Count = NI->getPending(); + // If the node is already in list if (Count < 0) return; - // If this the first time then get count - if (!Count) Count = Op.Val->use_size(); // Decrement count to indicate a visit Count--; - // If count has gone to zero then add operand to list + // If count has gone to zero then add node to list if (!Count) { - // Add operand - Operands.push_back(ScheduleInfo(Op)); - // indicate operand has been added + // Add node + if (NI->isInGroup()) { + Ordering.push_back(NI->Group->getLeader()); + } else { + Ordering.push_back(NI); + } + // indicate node has been added Count--; } // Mark as visited with new count - Visited[Op.Val] = Count; + NI->setPending(Count); } -/// VisitAll - Visit each operand breadth-wise to produce an initial ordering. -/// Note that the ordering in the Operands vector is reversed. +/// VisitAll - Visit each node breadth-wise to produce an initial ordering. +/// Note that the ordering in the Nodes vector is reversed. void SimpleSched::VisitAll() { // Add first element to list - Operands.push_back(DAG.getRoot()); - for (unsigned i = 0; i < Operands.size(); i++) { // note: size() varies - // Get next operand. Need copy because Operands vector is growing and - // addresses can be ScheduleInfo changing. - SDOperand Op = Operands[i].Op; - // Get the number of real operands - unsigned NodeOperands = CountOperands(Op); - // Get the total number of operands - unsigned NumOperands = Op.getNumOperands(); - - // Visit all operands skipping the Other operand if present - for (unsigned i = NumOperands; 0 < i--;) { - SDOperand OpI = Op.getOperand(i); - // Ignore passive operands - if (isPassiveOperand(OpI)) continue; - // Check out operand - IncludeOperand(OpI); + Ordering.push_back(getNI(DAG.getRoot().Val)); + + // Iterate through all nodes that have been added + for (unsigned i = 0; i < Ordering.size(); i++) { // note: size() varies + // Visit all operands + NodeGroupOpIterator NGI(Ordering[i]); + while (!NGI.isEnd()) { + // Get next operand + SDOperand Op = NGI.next(); + // Get node + SDNode *Node = Op.Val; + // Ignore passive nodes + if (isPassiveNode(Node)) continue; + // Check out node + IncludeNode(getNI(Node)); } } - // Add entry node last (IncludeOperand filters entry nodes) + // Add entry node last (IncludeNode filters entry nodes) if (DAG.getEntryNode().Val != DAG.getRoot().Val) - Operands.push_back(DAG.getEntryNode()); + Ordering.push_back(getNI(DAG.getEntryNode().Val)); + + // FIXME - Reverse the order + for (unsigned i = 0, N = Ordering.size(), Half = N >> 1; i < Half; i++) { + unsigned j = N - i - 1; + NodeInfo *tmp = Ordering[i]; + Ordering[i] = Ordering[j]; + Ordering[j] = tmp; + } } -/// GatherOperandInfo - Get latency and resource information about each operand. -/// -void SimpleSched::GatherOperandInfo() { - // Add addresses of operand info to ordering vector - // Get number of operands - unsigned N = Operands.size(); - // FIXME: This is an ugly (but temporary!) hack to test the scheduler before - // we have real target info. +/// GatherNodeInfo - Get latency and resource information about each node. +/// +void SimpleSched::GatherNodeInfo() { + // Allocate node information + Info = new NodeInfo[NodeCount]; + // Get base of all nodes table + SelectionDAG::allnodes_iterator AllNodes = DAG.allnodes_begin(); - // For each operand being scheduled - for (unsigned i = 0; i < N; i++) { - ScheduleInfo* SI = &Operands[N - i - 1]; - SDOperand Op = SI->Op; - MVT::ValueType VT = Op.Val->getValueType(0); - if (Op.isTargetOpcode()) { - MachineOpCode TOpc = Op.getTargetOpcode(); - // FIXME SI->Latency = std::max(1, TII.maxLatency(TOpc)); - // FIXME SI->ResourceSet = TII.resources(TOpc); + // For each node being scheduled + for (unsigned i = 0, N = NodeCount; i < N; i++) { + // Get next node from DAG all nodes table + SDNode *Node = AllNodes[i]; + // Fast reference to node schedule info + NodeInfo* NI = &Info[i]; + // Set up map + Map[Node] = NI; + // Set node + NI->Node = Node; + // Set pending visit count + NI->setPending(Node->use_size()); + + MVT::ValueType VT = Node->getValueType(0); + if (Node->isTargetOpcode()) { + MachineOpCode TOpc = Node->getTargetOpcode(); + // FIXME: This is an ugly (but temporary!) hack to test the scheduler + // before we have real target info. + // FIXME NI->Latency = std::max(1, TII.maxLatency(TOpc)); + // FIXME NI->ResourceSet = TII.resources(TOpc); if (TII.isCall(TOpc)) { - SI->ResourceSet = RSInteger; - SI->Latency = 40; + NI->ResourceSet = RSInteger; + NI->Latency = 40; } else if (TII.isLoad(TOpc)) { - SI->ResourceSet = RSLoadStore; - SI->Latency = 5; + NI->ResourceSet = RSLoadStore; + NI->Latency = 5; } else if (TII.isStore(TOpc)) { - SI->ResourceSet = RSLoadStore; - SI->Latency = 2; + NI->ResourceSet = RSLoadStore; + NI->Latency = 2; } else if (MVT::isInteger(VT)) { - SI->ResourceSet = RSInteger; - SI->Latency = 2; + NI->ResourceSet = RSInteger; + NI->Latency = 2; } else if (MVT::isFloatingPoint(VT)) { - SI->ResourceSet = RSFloat; - SI->Latency = 3; + NI->ResourceSet = RSFloat; + NI->Latency = 3; } else { - SI->ResourceSet = RSOther; - SI->Latency = 0; + NI->ResourceSet = RSOther; + NI->Latency = 0; } } else { if (MVT::isInteger(VT)) { - SI->ResourceSet = RSInteger; - SI->Latency = 2; + NI->ResourceSet = RSInteger; + NI->Latency = 2; } else if (MVT::isFloatingPoint(VT)) { - SI->ResourceSet = RSFloat; - SI->Latency = 3; + NI->ResourceSet = RSFloat; + NI->Latency = 3; } else { - SI->ResourceSet = RSOther; - SI->Latency = 0; + NI->ResourceSet = RSOther; + NI->Latency = 0; } } // Add one slot for the instruction itself - SI->Latency++; + NI->Latency++; // Sum up all the latencies for max tally size - NSlots += SI->Latency; - - // Place in initial sorted order - // FIXME - PUNT - ignore flag users - if (!isFlagUser(Op)) Ordering.push_back(SI); + NSlots += NI->Latency; + } + + // Put flagged nodes into groups + for (unsigned i = 0, N = NodeCount; i < N; i++) { + NodeInfo* NI = &Info[i]; + SDNode *Node = NI->Node; + if (isFlagDefiner(Node)) { + FlagUserIterator FI(Node); + while (SDNode *User = FI.next()) NodeGroup::Add(NI, getNI(User)); + } } } -/// isStrongDependency - Return true if operand A has results used by operand B. +/// isStrongDependency - Return true if node A has results used by node B. /// I.E., B must wait for latency of A. -bool SimpleSched::isStrongDependency(SDNode *A, SDNode *B) { +bool SimpleSched::isStrongDependency(NodeInfo *A, NodeInfo *B) { // If A defines for B then it's a strong dependency - if (isDefiner(A, B)) return true; - // If A defines a flag then it's users are part of the dependency - if (isFlagDefiner(A)) { - // Check each flag user - FlagUserIterator FI(A); - while (SDNode *User = FI.next()) { - // If flag user has strong dependency so does B - if (isStrongDependency(User, B)) return true; - } - } - // If B defines a flag then it's users are part of the dependency - if (isFlagDefiner(B)) { - // Check each flag user - FlagUserIterator FI(B); - while (SDNode *User = FI.next()) { - // If flag user has strong dependency so does B - if (isStrongDependency(A, User)) return true; - } - } - return false; + return isDefiner(A, B); } -/// isWeakDependency Return true if operand A produces a result that will +/// isWeakDependency Return true if node A produces a result that will /// conflict with operands of B. -bool SimpleSched::isWeakDependency(SDNode *A, SDNode *B) { +bool SimpleSched::isWeakDependency(NodeInfo *A, NodeInfo *B) { // TODO check for conflicting real registers and aliases -#if 0 // Since we are in SSA form and not checking register aliasing - return A->getOpcode() == ISD::EntryToken || isStrongDependency(B, A); +#if 0 // FIXME - Since we are in SSA form and not checking register aliasing + return A->Node->getOpcode() == ISD::EntryToken || isStrongDependency(B, A); #else - return A->getOpcode() == ISD::EntryToken; + return A->Node->getOpcode() == ISD::EntryToken; #endif } @@ -538,26 +786,26 @@ bool SimpleSched::isWeakDependency(SDNode *A, SDNode *B) { void SimpleSched::ScheduleBackward() { // Size and clear the resource tally Tally.Initialize(NSlots); - // Get number of operands to schedule + // Get number of nodes to schedule unsigned N = Ordering.size(); - // For each operand being scheduled + // For each node being scheduled for (unsigned i = N; 0 < i--;) { - ScheduleInfo *SI = Ordering[i]; + NodeInfo *NI = Ordering[i]; // Track insertion unsigned Slot = NotFound; - // Compare against those previously scheduled operands + // Compare against those previously scheduled nodes unsigned j = i + 1; for (; j < N; j++) { // Get following instruction - ScheduleInfo *Other = Ordering[j]; + NodeInfo *Other = Ordering[j]; - // Check dependency against previously inserted operands - if (isStrongDependency(SI->Op, Other->Op)) { + // Check dependency against previously inserted nodes + if (isStrongDependency(NI, Other)) { Slot = Other->Slot + Other->Latency; break; - } else if (isWeakDependency(SI->Op, Other->Op)) { + } else if (isWeakDependency(NI, Other)) { Slot = Other->Slot; break; } @@ -567,24 +815,24 @@ void SimpleSched::ScheduleBackward() { if (Slot == NotFound) Slot = 0; // Find a slot where the needed resources are available - if (SI->ResourceSet) - Slot = Tally.FindAndReserve(Slot, SI->Latency, SI->ResourceSet); + if (NI->ResourceSet) + Slot = Tally.FindAndReserve(Slot, NI->Latency, NI->ResourceSet); - // Set operand slot - SI->Slot = Slot; + // Set node slot + NI->Slot = Slot; // Insert sort based on slot j = i + 1; for (; j < N; j++) { // Get following instruction - ScheduleInfo *Other = Ordering[j]; + NodeInfo *Other = Ordering[j]; // Should we look further if (Slot >= Other->Slot) break; // Shuffle other into ordering Ordering[j - 1] = Other; } - // Insert operand in proper slot - if (j != i + 1) Ordering[j - 1] = SI; + // Insert node in proper slot + if (j != i + 1) Ordering[j - 1] = NI; } } @@ -593,26 +841,26 @@ void SimpleSched::ScheduleBackward() { void SimpleSched::ScheduleForward() { // Size and clear the resource tally Tally.Initialize(NSlots); - // Get number of operands to schedule + // Get number of nodes to schedule unsigned N = Ordering.size(); - // For each operand being scheduled + // For each node being scheduled for (unsigned i = 0; i < N; i++) { - ScheduleInfo *SI = Ordering[i]; + NodeInfo *NI = Ordering[i]; // Track insertion unsigned Slot = NotFound; - // Compare against those previously scheduled operands + // Compare against those previously scheduled nodes unsigned j = i; for (; 0 < j--;) { // Get following instruction - ScheduleInfo *Other = Ordering[j]; + NodeInfo *Other = Ordering[j]; - // Check dependency against previously inserted operands - if (isStrongDependency(Other->Op, SI->Op)) { + // Check dependency against previously inserted nodes + if (isStrongDependency(Other, NI)) { Slot = Other->Slot + Other->Latency; break; - } else if (isWeakDependency(Other->Op, SI->Op)) { + } else if (isWeakDependency(Other, NI)) { Slot = Other->Slot; break; } @@ -622,66 +870,59 @@ void SimpleSched::ScheduleForward() { if (Slot == NotFound) Slot = 0; // Find a slot where the needed resources are available - if (SI->ResourceSet) - Slot = Tally.FindAndReserve(Slot, SI->Latency, SI->ResourceSet); + if (NI->ResourceSet) + Slot = Tally.FindAndReserve(Slot, NI->Latency, NI->ResourceSet); - // Set operand slot - SI->Slot = Slot; + // Set node slot + NI->Slot = Slot; // Insert sort based on slot j = i; for (; 0 < j--;) { // Get following instruction - ScheduleInfo *Other = Ordering[j]; + NodeInfo *Other = Ordering[j]; // Should we look further if (Slot >= Other->Slot) break; // Shuffle other into ordering Ordering[j + 1] = Other; } - // Insert operand in proper slot - if (j != i) Ordering[j + 1] = SI; + // Insert node in proper slot + if (j != i) Ordering[j + 1] = NI; } } -/// EmitAll - Emit all operands in schedule sorted order. +/// EmitAll - Emit all nodes in schedule sorted order. /// void SimpleSched::EmitAll() { - // For each operand in the ordering + // For each node in the ordering for (unsigned i = 0, N = Ordering.size(); i < N; i++) { // Get the scheduling info - ScheduleInfo *SI = Ordering[i]; - // Get the operand - SDOperand Op = SI->Op; - // Emit the operand - Emit(Op); - // FIXME - PUNT - If Op defines a flag then it's users need to be emitted now - if (isFlagDefiner(Op)) EmitFlagUsers(Op); - } -} - -/// EmitFlagUsers - Emit users of operands flag. -/// -void SimpleSched::EmitFlagUsers(SDOperand Op) { - // Check each flag user - FlagUserIterator FI(Op.Val); - while (SDNode *User = FI.next()) { - // Construct user node as operand - SDOperand OpU(User, 0); - // Emit user node - Emit(OpU); - // If user defines a flag then it's users need to be emitted now - if (isFlagDefiner(User)) EmitFlagUsers(OpU); + NodeInfo *NI = Ordering[i]; +#if 0 + // Iterate through nodes + NodeGroupIterator NGI(Ordering[i]); + while (NodeInfo *NI = NGI.next()) EmitNode(NI); +#else + if (NI->isInGroup()) { + if (NI->isGroupLeader()) { + NodeGroupIterator NGI(Ordering[i]); + while (NodeInfo *NI = NGI.next()) EmitNode(NI); + } + } else { + EmitNode(NI); + } +#endif } } /// CountResults - The results of target nodes have register or immediate /// operands first, then an optional chain, and optional flag operands (which do /// not go into the machine instrs.) -unsigned SimpleSched::CountResults(SDOperand Op) { - unsigned N = Op.Val->getNumValues(); - while (N && Op.Val->getValueType(N - 1) == MVT::Flag) +unsigned SimpleSched::CountResults(SDNode *Node) { + unsigned N = Node->getNumValues(); + while (N && Node->getValueType(N - 1) == MVT::Flag) --N; - if (N && Op.Val->getValueType(N - 1) == MVT::Other) + if (N && Node->getValueType(N - 1) == MVT::Other) --N; // Skip over chain result. return N; } @@ -689,18 +930,18 @@ unsigned SimpleSched::CountResults(SDOperand Op) { /// CountOperands The inputs to target nodes have any actual inputs first, /// followed by an optional chain operand, then flag operands. Compute the /// number of actual operands that will go into the machine instr. -unsigned SimpleSched::CountOperands(SDOperand Op) { - unsigned N = Op.getNumOperands(); - while (N && Op.getOperand(N - 1).getValueType() == MVT::Flag) +unsigned SimpleSched::CountOperands(SDNode *Node) { + unsigned N = Node->getNumOperands(); + while (N && Node->getOperand(N - 1).getValueType() == MVT::Flag) --N; - if (N && Op.getOperand(N - 1).getValueType() == MVT::Other) + if (N && Node->getOperand(N - 1).getValueType() == MVT::Other) --N; // Ignore chain if it exists. return N; } /// CreateVirtualRegisters - Add result register values for things that are /// defined by this instruction. -unsigned SimpleSched::CreateVirtualRegisters(SDOperand Op, MachineInstr *MI, +unsigned SimpleSched::CreateVirtualRegisters(MachineInstr *MI, unsigned NumResults, const TargetInstrDescriptor &II) { // Create the result registers for this node and add the result regs to @@ -716,21 +957,19 @@ unsigned SimpleSched::CreateVirtualRegisters(SDOperand Op, MachineInstr *MI, return ResultReg; } -/// Emit - Generate machine code for an operand and needed dependencies. +/// EmitNode - Generate machine code for an node and needed dependencies. /// -unsigned SimpleSched::Emit(SDOperand Op) { - std::map::iterator OpI = VRMap.lower_bound(Op.Val); - if (OpI != VRMap.end() && OpI->first == Op.Val) - return OpI->second + Op.ResNo; - unsigned &OpSlot = VRMap.insert(OpI, std::make_pair(Op.Val, 0))->second; +void SimpleSched::EmitNode(NodeInfo *NI) { + unsigned VRBase = 0; // First virtual register for node + SDNode *Node = NI->Node; - unsigned ResultReg = 0; - if (Op.isTargetOpcode()) { - unsigned Opc = Op.getTargetOpcode(); + // If machine instruction + if (Node->isTargetOpcode()) { + unsigned Opc = Node->getTargetOpcode(); const TargetInstrDescriptor &II = TII.get(Opc); - unsigned NumResults = CountResults(Op); - unsigned NodeOperands = CountOperands(Op); + unsigned NumResults = CountResults(Node); + unsigned NodeOperands = CountOperands(Node); unsigned NumMIOperands = NodeOperands + NumResults; #ifndef NDEBUG assert((unsigned(II.numOperands) == NumMIOperands || II.numOperands == -1)&& @@ -742,13 +981,137 @@ unsigned SimpleSched::Emit(SDOperand Op) { // Add result register values for things that are defined by this // instruction. - if (NumResults) ResultReg = CreateVirtualRegisters(Op, MI, NumResults, II); + if (NumResults) VRBase = CreateVirtualRegisters(MI, NumResults, II); + + // Emit all of the actual operands of this instruction, adding them to the + // instruction as appropriate. + for (unsigned i = 0; i != NodeOperands; ++i) { + if (Node->getOperand(i).isTargetOpcode()) { + // Note that this case is redundant with the final else block, but we + // include it because it is the most common and it makes the logic + // simpler here. + assert(Node->getOperand(i).getValueType() != MVT::Other && + Node->getOperand(i).getValueType() != MVT::Flag && + "Chain and flag operands should occur at end of operand list!"); + + MI->addRegOperand(getVR(Node->getOperand(i)), MachineOperand::Use); + } else if (ConstantSDNode *C = + dyn_cast(Node->getOperand(i))) { + MI->addZeroExtImm64Operand(C->getValue()); + } else if (RegisterSDNode*R = + dyn_cast(Node->getOperand(i))) { + MI->addRegOperand(R->getReg(), MachineOperand::Use); + } else if (GlobalAddressSDNode *TGA = + dyn_cast(Node->getOperand(i))) { + MI->addGlobalAddressOperand(TGA->getGlobal(), false, 0); + } else if (BasicBlockSDNode *BB = + dyn_cast(Node->getOperand(i))) { + MI->addMachineBasicBlockOperand(BB->getBasicBlock()); + } else if (FrameIndexSDNode *FI = + dyn_cast(Node->getOperand(i))) { + MI->addFrameIndexOperand(FI->getIndex()); + } else if (ConstantPoolSDNode *CP = + dyn_cast(Node->getOperand(i))) { + unsigned Idx = ConstPool->getConstantPoolIndex(CP->get()); + MI->addConstantPoolIndexOperand(Idx); + } else if (ExternalSymbolSDNode *ES = + dyn_cast(Node->getOperand(i))) { + MI->addExternalSymbolOperand(ES->getSymbol(), false); + } else { + assert(Node->getOperand(i).getValueType() != MVT::Other && + Node->getOperand(i).getValueType() != MVT::Flag && + "Chain and flag operands should occur at end of operand list!"); + MI->addRegOperand(getVR(Node->getOperand(i)), MachineOperand::Use); + } + } + + // Now that we have emitted all operands, emit this instruction itself. + if ((II.Flags & M_USES_CUSTOM_DAG_SCHED_INSERTION) == 0) { + BB->insert(BB->end(), MI); + } else { + // Insert this instruction into the end of the basic block, potentially + // taking some custom action. + BB = DAG.getTargetLoweringInfo().InsertAtEndOfBasicBlock(MI, BB); + } + } else { + switch (Node->getOpcode()) { + default: + Node->dump(); + assert(0 && "This target-independent node should have been selected!"); + case ISD::EntryToken: // fall thru + case ISD::TokenFactor: + break; + case ISD::CopyToReg: { + unsigned Val = getVR(Node->getOperand(2)); + MRI.copyRegToReg(*BB, BB->end(), + cast(Node->getOperand(1))->getReg(), Val, + RegMap->getRegClass(Val)); + break; + } + case ISD::CopyFromReg: { + unsigned SrcReg = cast(Node->getOperand(1))->getReg(); + + // Figure out the register class to create for the destreg. + const TargetRegisterClass *TRC = 0; + if (MRegisterInfo::isVirtualRegister(SrcReg)) { + TRC = RegMap->getRegClass(SrcReg); + } else { + // FIXME: we don't know what register class to generate this for. Do + // a brute force search and pick the first match. :( + for (MRegisterInfo::regclass_iterator I = MRI.regclass_begin(), + E = MRI.regclass_end(); I != E; ++I) + if ((*I)->contains(SrcReg)) { + TRC = *I; + break; + } + assert(TRC && "Couldn't find register class for reg copy!"); + } + + // Create the reg, emit the copy. + VRBase = RegMap->createVirtualRegister(TRC); + MRI.copyRegToReg(*BB, BB->end(), VRBase, SrcReg, TRC); + break; + } + } + } + + assert(NI->VRBase == 0 && "Node emitted out of order - early"); + NI->VRBase = VRBase; +} + +/// EmitDag - Generate machine code for an operand and needed dependencies. +/// +unsigned SimpleSched::EmitDAG(SDOperand Op) { + std::map::iterator OpI = VRMap.lower_bound(Op.Val); + if (OpI != VRMap.end() && OpI->first == Op.Val) + return OpI->second + Op.ResNo; + unsigned &OpSlot = VRMap.insert(OpI, std::make_pair(Op.Val, 0))->second; + + unsigned ResultReg = 0; + if (Op.isTargetOpcode()) { + unsigned Opc = Op.getTargetOpcode(); + const TargetInstrDescriptor &II = TII.get(Opc); + + unsigned NumResults = CountResults(Op.Val); + unsigned NodeOperands = CountOperands(Op.Val); + unsigned NumMIOperands = NodeOperands + NumResults; +#ifndef NDEBUG + assert((unsigned(II.numOperands) == NumMIOperands || II.numOperands == -1)&& + "#operands for dag node doesn't match .td file!"); +#endif + + // Create the new machine instruction. + MachineInstr *MI = new MachineInstr(Opc, NumMIOperands, true, true); + + // Add result register values for things that are defined by this + // instruction. + if (NumResults) ResultReg = CreateVirtualRegisters(MI, NumResults, II); // If there is a token chain operand, emit it first, as a hack to get avoid // really bad cases. if (Op.getNumOperands() > NodeOperands && Op.getOperand(NodeOperands).getValueType() == MVT::Other) { - Emit(Op.getOperand(NodeOperands)); + EmitDAG(Op.getOperand(NodeOperands)); } // Emit all of the actual operands of this instruction, adding them to the @@ -762,7 +1125,7 @@ unsigned SimpleSched::Emit(SDOperand Op) { Op.getOperand(i).getValueType() != MVT::Flag && "Chain and flag operands should occur at end of operand list!"); - MI->addRegOperand(Emit(Op.getOperand(i)), MachineOperand::Use); + MI->addRegOperand(EmitDAG(Op.getOperand(i)), MachineOperand::Use); } else if (ConstantSDNode *C = dyn_cast(Op.getOperand(i))) { MI->addZeroExtImm64Operand(C->getValue()); @@ -788,7 +1151,7 @@ unsigned SimpleSched::Emit(SDOperand Op) { assert(Op.getOperand(i).getValueType() != MVT::Other && Op.getOperand(i).getValueType() != MVT::Flag && "Chain and flag operands should occur at end of operand list!"); - MI->addRegOperand(Emit(Op.getOperand(i)), MachineOperand::Use); + MI->addRegOperand(EmitDAG(Op.getOperand(i)), MachineOperand::Use); } } @@ -801,7 +1164,7 @@ unsigned SimpleSched::Emit(SDOperand Op) { for (unsigned N = Op.getNumOperands(); i < N; i++) { assert(Op.getOperand(i).getValueType() == MVT::Flag && "Must be flag operands!"); - Emit(Op.getOperand(i)); + EmitDAG(Op.getOperand(i)); } } @@ -821,7 +1184,7 @@ unsigned SimpleSched::Emit(SDOperand Op) { case ISD::EntryToken: break; case ISD::TokenFactor: for (unsigned i = 0, N = Op.getNumOperands(); i < N; i++) { - Emit(Op.getOperand(i)); + EmitDAG(Op.getOperand(i)); } break; case ISD::CopyToReg: { @@ -830,11 +1193,11 @@ unsigned SimpleSched::Emit(SDOperand Op) { FlagOp = Op.getOperand(3); } if (Op.getOperand(0).Val != FlagOp.Val) { - Emit(Op.getOperand(0)); // Emit the chain. + EmitDAG(Op.getOperand(0)); // Emit the chain. } - unsigned Val = Emit(Op.getOperand(2)); + unsigned Val = EmitDAG(Op.getOperand(2)); if (FlagOp.Val) { - Emit(FlagOp); + EmitDAG(FlagOp); } MRI.copyRegToReg(*BB, BB->end(), cast(Op.getOperand(1))->getReg(), Val, @@ -842,7 +1205,7 @@ unsigned SimpleSched::Emit(SDOperand Op) { break; } case ISD::CopyFromReg: { - Emit(Op.getOperand(0)); // Emit the chain. + EmitDAG(Op.getOperand(0)); // Emit the chain. unsigned SrcReg = cast(Op.getOperand(1))->getReg(); // Figure out the register class to create for the destreg. @@ -873,17 +1236,19 @@ unsigned SimpleSched::Emit(SDOperand Op) { return ResultReg+Op.ResNo; } -/// Schedule - Order operands according to selected style. +/// Schedule - Order nodes according to selected style. /// void SimpleSched::Schedule() { switch (ScheduleStyle) { case simpleScheduling: - // Breadth first walk of DAG - VisitAll(); - // Get latency and resource requirements - GatherOperandInfo(); + // Number the nodes + NodeCount = DAG.allnodes_size(); // Don't waste time if is only entry and return - if (Operands.size() > 2) { + if (NodeCount > 3) { + // Get latency and resource requirements + GatherNodeInfo(); + // Breadth first walk of DAG + VisitAll(); DEBUG(dump("Pre-")); // Push back long instructions and critical path ScheduleBackward(); @@ -897,27 +1262,27 @@ void SimpleSched::Schedule() { } // fall thru case noScheduling: // Emit instructions in using a DFS from the exit root - Emit(DAG.getRoot()); + EmitDAG(DAG.getRoot()); break; } } /// printSI - Print schedule info. /// -void SimpleSched::printSI(std::ostream &O, ScheduleInfo *SI) const { +void SimpleSched::printSI(std::ostream &O, NodeInfo *NI) const { #ifndef NDEBUG using namespace std; - SDOperand Op = SI->Op; + SDNode *Node = NI->Node; O << " " - << hex << Op.Val - << ", RS=" << SI->ResourceSet - << ", Lat=" << SI->Latency - << ", Slot=" << SI->Slot - << ", ARITY=(" << Op.getNumOperands() << "," - << Op.Val->getNumValues() << ")" - << " " << Op.Val->getOperationName(&DAG); - if (isFlagDefiner(Op)) O << "<#"; - if (isFlagUser(Op)) O << ">#"; + << hex << Node + << ", RS=" << NI->ResourceSet + << ", Lat=" << NI->Latency + << ", Slot=" << NI->Slot + << ", ARITY=(" << Node->getNumOperands() << "," + << Node->getNumValues() << ")" + << " " << Node->getOperationName(&DAG); + if (isFlagDefiner(Node)) O << "<#"; + if (isFlagUser(Node)) O << ">#"; #endif }