From f06c835f769aa1cf67801ed1f6bd366a447c18b1 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 30 Sep 2008 18:30:35 +0000 Subject: [PATCH] Optimize SelectionDAG's AssignTopologicalOrder even further. Completely eliminate the TopOrder std::vector. Instead, sort the AllNodes list in place. This also eliminates the need to call AllNodes.size(), a linear-time operation, before performing the sort. Also, eliminate the Sources temporary std::vector, since it essentially duplicates the sorted result as it is being built. This also changes the direction of the topological sort from bottom-up to top-down. The AllNodes list starts out in roughly top-down order, so this reduces the amount of reordering needed. Top-down is also more convenient for Legalize, and ISel needed only minor adjustments. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@56867 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/llvm/CodeGen/DAGISelHeader.h | 8 +- include/llvm/CodeGen/SelectionDAG.h | 8 +- include/llvm/CodeGen/SelectionDAGISel.h | 3 +- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 9 +-- lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 92 ++++++++++++++++------- lib/Target/X86/X86ISelDAGToDAG.cpp | 2 +- 6 files changed, 78 insertions(+), 44 deletions(-) diff --git a/include/llvm/CodeGen/DAGISelHeader.h b/include/llvm/CodeGen/DAGISelHeader.h index 257c59583fc..3542ebcbd02 100644 --- a/include/llvm/CodeGen/DAGISelHeader.h +++ b/include/llvm/CodeGen/DAGISelHeader.h @@ -18,7 +18,7 @@ #define LLVM_CODEGEN_DAGISEL_HEADER_H /// ISelQueue - Instruction selector priority queue sorted -/// in the order of increasing NodeId() values. +/// in the order of decreasing NodeId() values. std::vector ISelQueue; /// Keep track of nodes which have already been added to queue. @@ -43,10 +43,10 @@ static bool IsChainCompatible(SDNode *Chain, SDNode *Op) { } /// isel_sort - Sorting functions for the selection queue in the -/// increasing NodeId order. +/// decreasing NodeId order. struct isel_sort : public std::binary_function { bool operator()(const SDNode* left, const SDNode* right) const { - return (left->getNodeId() > right->getNodeId()); + return left->getNodeId() < right->getNodeId(); } }; @@ -108,7 +108,7 @@ class VISIBILITY_HIDDEN ISelQueueUpdater : }; /// UpdateQueue - update the instruction selction queue to maintain -/// the increasing NodeId() ordering property. +/// the decreasing NodeId() ordering property. inline void UpdateQueue(const ISelQueueUpdater &ISQU) { if (ISQU.hadDelete()) std::make_heap(ISelQueue.begin(), ISelQueue.end(),isel_sort()); diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 3c09460205a..91c9e378a3a 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -663,10 +663,10 @@ public: unsigned Num, DAGUpdateListener *UpdateListener = 0); - /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG - /// based on their topological order. It returns the maximum id and a vector - /// of the SDNodes* in assigned order by reference. - unsigned AssignTopologicalOrder(std::vector &TopOrder); + /// AssignTopologicalOrder - Topological-sort the AllNodes list and a + /// assign a unique node id for each node in the DAG based on their + /// topological order. Returns the number of nodes. + unsigned AssignTopologicalOrder(); /// isCommutativeBinOp - Returns true if the opcode is a commutative binary /// operation. diff --git a/include/llvm/CodeGen/SelectionDAGISel.h b/include/llvm/CodeGen/SelectionDAGISel.h index a1c6f5b79d7..b091fc2ff97 100644 --- a/include/llvm/CodeGen/SelectionDAGISel.h +++ b/include/llvm/CodeGen/SelectionDAGISel.h @@ -48,7 +48,6 @@ public: AliasAnalysis *AA; GCFunctionInfo *GFI; bool Fast; - std::vector TopOrder; static char ID; explicit SelectionDAGISel(TargetLowering &tli, bool fast = false); @@ -67,7 +66,7 @@ public: virtual void InstructionSelectPostProcessing() {} void SelectRootInit() { - DAGSize = CurDAG->AssignTopologicalOrder(TopOrder); + DAGSize = CurDAG->AssignTopologicalOrder(); } /// SelectInlineAsmMemoryOperand - Select the specified address as a target diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index c3ab3680b22..d6bb1426461 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -280,11 +280,10 @@ void SelectionDAGLegalize::LegalizeDAG() { // practice however, this causes us to run out of stack space on large basic // blocks. To avoid this problem, compute an ordering of the nodes where each // node is only legalized after all of its operands are legalized. - std::vector TopOrder; - unsigned N = DAG.AssignTopologicalOrder(TopOrder); - for (unsigned i = N; i != 0; --i) - HandleOp(SDValue(TopOrder[i-1], 0)); - TopOrder.clear(); + DAG.AssignTopologicalOrder(); + for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), + E = prior(DAG.allnodes_end()); I != next(E); ++I) + HandleOp(SDValue(I, 0)); // Finally, it's possible the root changed. Get the new root. SDValue OldRoot = DAG.getRoot(); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 4c4a90a2ed4..004fc4c1565 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -587,7 +587,7 @@ void SelectionDAG::DeleteNode(SDNode *N) { void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { - // Drop all of the operands and decrement used nodes use counts. + // Drop all of the operands and decrement used node's use counts. for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) I->getVal()->removeUser(std::distance(N->op_begin(), I), N); if (N->OperandsNeedDelete) @@ -4569,38 +4569,74 @@ void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG /// based on their topological order. It returns the maximum id and a vector /// of the SDNodes* in assigned order by reference. -unsigned SelectionDAG::AssignTopologicalOrder(std::vector &TopOrder) { - unsigned DAGSize = AllNodes.size(); - std::vector Sources; +unsigned SelectionDAG::AssignTopologicalOrder() { - for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I){ - SDNode *N = I; - unsigned Degree = N->use_size(); - // Temporarily use the Node Id as scratch space for the degree count. - N->setNodeId(Degree); - if (Degree == 0) - Sources.push_back(N); - } + unsigned DAGSize = 0; - TopOrder.clear(); - TopOrder.reserve(DAGSize); - int Id = 0; - while (!Sources.empty()) { - SDNode *N = Sources.back(); - Sources.pop_back(); - TopOrder.push_back(N); - N->setNodeId(Id++); - for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { - SDNode *P = I->getVal(); - unsigned Degree = P->getNodeId(); - --Degree; - P->setNodeId(Degree); - if (Degree == 0) - Sources.push_back(P); + // SortedPos tracks the progress of the algorithm. Nodes before it are + // sorted, nodes after it are unsorted. When the algorithm completes + // it is at the end of the list. + allnodes_iterator SortedPos = allnodes_begin(); + + // Visit all the nodes. Add nodes with no operands to the TopOrder result + // array immediately. Annotate nodes that do have operands with their + // operand count. Before we do this, the Node Id fields of the nodes + // may contain arbitrary values. After, the Node Id fields for nodes + // before SortedPos will contain the topological sort index, and the + // Node Id fields for nodes At SortedPos and after will contain the + // count of outstanding operands. + for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) { + SDNode *N = I++; + unsigned Degree = N->getNumOperands(); + if (Degree == 0) { + // A node with no uses, add it to the result array immediately. + N->setNodeId(DAGSize++); + allnodes_iterator Q = N; + if (Q != SortedPos) + SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); + ++SortedPos; + } else { + // Temporarily use the Node Id as scratch space for the degree count. + N->setNodeId(Degree); } } - return Id; + // Visit all the nodes. As we iterate, moves nodes into sorted order, + // such that by the time the end is reached all nodes will be sorted. + for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) { + SDNode *N = I; + for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); + UI != UE; ++UI) { + SDNode *P = *UI; + unsigned Degree = P->getNodeId(); + --Degree; + if (Degree == 0) { + // All of P's operands are sorted, so P may sorted now. + P->setNodeId(DAGSize++); + if (P != SortedPos) + SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); + ++SortedPos; + } else { + // Update P's outstanding operand count. + P->setNodeId(Degree); + } + } + } + + assert(SortedPos == AllNodes.end() && + "Topological sort incomplete!"); + assert(AllNodes.front().getOpcode() == ISD::EntryToken && + "First node in topological sort is not the entry token!"); + assert(AllNodes.front().getNodeId() == 0 && + "First node in topological sort has non-zero id!"); + assert(AllNodes.front().getNumOperands() == 0 && + "First node in topological sort has operands!"); + assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && + "Last node in topologic sort has unexpected id!"); + assert(AllNodes.back().use_empty() && + "Last node in topologic sort has users!"); + assert(DAGSize == allnodes_size() && "TopOrder result count mismatch!"); + return DAGSize; } diff --git a/lib/Target/X86/X86ISelDAGToDAG.cpp b/lib/Target/X86/X86ISelDAGToDAG.cpp index 692f4deb7d8..2b7bf15d845 100644 --- a/lib/Target/X86/X86ISelDAGToDAG.cpp +++ b/lib/Target/X86/X86ISelDAGToDAG.cpp @@ -269,7 +269,7 @@ static void findNonImmUse(SDNode *Use, SDNode* Def, SDNode *ImmedUse, SDNode *Root, bool &found, SmallPtrSet &Visited) { if (found || - Use->getNodeId() > Def->getNodeId() || + Use->getNodeId() < Def->getNodeId() || !Visited.insert(Use)) return;