diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h index 6f53d344e7a..53aee7a9c9f 100644 --- a/include/llvm/CodeGen/Passes.h +++ b/include/llvm/CodeGen/Passes.h @@ -88,6 +88,11 @@ namespace llvm { /// register allocators. extern char &TwoAddressInstructionPassID; + /// SpillPlacement analysis. Suggest optimal placement of spill code between + /// basic blocks. + /// + extern char &SpillPlacementID; + /// UnreachableMachineBlockElimination pass - This pass removes unreachable /// machine basic blocks. extern char &UnreachableMachineBlockElimID; diff --git a/include/llvm/InitializePasses.h b/include/llvm/InitializePasses.h index dc457163285..07812809764 100644 --- a/include/llvm/InitializePasses.h +++ b/include/llvm/InitializePasses.h @@ -203,6 +203,7 @@ void initializeSimplifyLibCallsPass(PassRegistry&); void initializeSingleLoopExtractorPass(PassRegistry&); void initializeSinkingPass(PassRegistry&); void initializeSlotIndexesPass(PassRegistry&); +void initializeSpillPlacementPass(PassRegistry&); void initializeStackProtectorPass(PassRegistry&); void initializeStackSlotColoringPass(PassRegistry&); void initializeStripDeadDebugInfoPass(PassRegistry&); diff --git a/lib/CodeGen/CMakeLists.txt b/lib/CodeGen/CMakeLists.txt index b4da151a3a8..977edcb12cf 100644 --- a/lib/CodeGen/CMakeLists.txt +++ b/lib/CodeGen/CMakeLists.txt @@ -80,6 +80,7 @@ add_llvm_library(LLVMCodeGen SjLjEHPrepare.cpp SlotIndexes.cpp Spiller.cpp + SpillPlacement.cpp SplitKit.cpp Splitter.cpp StackProtector.cpp diff --git a/lib/CodeGen/SpillPlacement.cpp b/lib/CodeGen/SpillPlacement.cpp new file mode 100644 index 00000000000..d8cc3e889e2 --- /dev/null +++ b/lib/CodeGen/SpillPlacement.cpp @@ -0,0 +1,354 @@ +//===-- SpillPlacement.cpp - Optimal Spill Code Placement -----------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the spill code placement analysis. +// +// Each edge bundle corresponds to a node in a Hopfield network. Constraints on +// basic blocks are weighted by the block frequency and added to become the node +// bias. +// +// Transparent basic blocks have the variable live through, but don't care if it +// is spilled or in a register. These blocks become connections in the Hopfield +// network, again weighted by block frequency. +// +// The Hopfield network minimizes (possibly locally) its energy function: +// +// E = -sum_n V_n * ( B_n + sum_{n, m linked by b} V_m * F_b ) +// +// The energy function represents the expected spill code execution frequency, +// or the cost of spilling. This is a Lyapunov function which never increases +// when a node is updated. It is guaranteed to converge to a local minimum. +// +//===----------------------------------------------------------------------===// + +#include "SpillPlacement.h" +#include "llvm/CodeGen/EdgeBundles.h" +#include "llvm/CodeGen/LiveIntervalAnalysis.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Format.h" + +using namespace llvm; + +char SpillPlacement::ID = 0; +INITIALIZE_PASS_BEGIN(SpillPlacement, "spill-code-placement", + "Spill Code Placement Analysis", true, true) +INITIALIZE_PASS_DEPENDENCY(EdgeBundles) +INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) +INITIALIZE_PASS_END(SpillPlacement, "spill-code-placement", + "Spill Code Placement Analysis", true, true) + +char &llvm::SpillPlacementID = SpillPlacement::ID; + +void SpillPlacement::getAnalysisUsage(AnalysisUsage &AU) const { + AU.setPreservesAll(); + AU.addRequiredTransitive(); + AU.addRequiredTransitive(); + MachineFunctionPass::getAnalysisUsage(AU); +} + +/// Node - Each edge bundle corresponds to a Hopfield node. +/// +/// The node contains precomputed frequency data that only depends on the CFG, +/// but Bias and Links are computed each time placeSpills is called. +/// +/// The node Value is positive when the variable should be in a register. The +/// value can change when linked nodes change, but convergence is very fast +/// because all weights are positive. +/// +struct SpillPlacement::Node { + /// Frequency - Total block frequency feeding into[0] or out of[1] the bundle. + /// Ideally, these two numbers should be identical, but inaccuracies in the + /// block frequency estimates means that we need to normalize ingoing and + /// outgoing frequencies separately so they are commensurate. + float Frequency[2]; + + /// Bias - Normalized contributions from non-transparent blocks. + /// A bundle connected to a MustSpill block has a huge negative bias, + /// otherwise it is a number in the range [-2;2]. + float Bias; + + /// Value - Output value of this node computed from the Bias and links. + /// This is always in the range [-1;1]. A positive number means the variable + /// should go in a register through this bundle. + float Value; + + typedef SmallVector, 4> LinkVector; + + /// Links - (Weight, BundleNo) for all transparent blocks connecting to other + /// bundles. The weights are all positive and add up to at most 2, weights + /// from ingoing and outgoing nodes separately add up to a most 1. The weight + /// sum can be less than 2 when the variable is not live into / out of some + /// connected basic blocks. + LinkVector Links; + + /// preferReg - Return true when this node prefers to be in a register. + bool preferReg() const { + // Undecided nodes (Value==0) go on the stack. + return Value > 0; + } + + /// mustSpill - Return True if this node is so biased that it must spill. + bool mustSpill() const { + // Actually, we must spill if Bias < sum(weights). + // It may be worth it to compute the weight sum here? + return Bias < -2.0f; + } + + /// Node - Create a blank Node. + Node() { + Frequency[0] = Frequency[1] = 0; + } + + /// clear - Reset per-query data, but preserve frequencies that only depend on + // the CFG. + void clear() { + Bias = Value = 0; + Links.clear(); + } + + /// addLink - Add a link to bundle b with weight w. + /// out=0 for an ingoing link, and 1 for an outgoing link. + void addLink(unsigned b, float w, bool out) { + // Normalize w relative to all connected blocks from that direction. + w /= Frequency[out]; + + // There can be multiple links to the same bundle, add them up. + for (LinkVector::iterator I = Links.begin(), E = Links.end(); I != E; ++I) + if (I->second == b) { + I->first += w; + return; + } + // This must be the first link to b. + Links.push_back(std::make_pair(w, b)); + } + + /// addBias - Bias this node from an ingoing[0] or outgoing[1] link. + void addBias(float w, bool out) { + // Normalize w relative to all connected blocks from that direction. + w /= Frequency[out]; + Bias += w; + } + + /// update - Recompute Value from Bias and Links. Return true when node + /// preference changes. + bool update(const Node nodes[]) { + // Compute the weighted sum of inputs. + float Sum = Bias; + for (LinkVector::iterator I = Links.begin(), E = Links.end(); I != E; ++I) + Sum += I->first * nodes[I->second].Value; + + // The weighted sum is going to be in the range [-2;2]. Ideally, we should + // simply set Value = sign(Sum), but we will add a dead zone around 0 for + // two reasons: + // 1. It avoids arbitrary bias when all links are 0 as is possible during + // initial iterations. + // 2. It helps tame rounding errors when the links nominally sum to 0. + const float Thres = 1e-4; + bool Before = preferReg(); + if (Sum < -Thres) + Value = -1; + else if (Sum > Thres) + Value = 1; + else + Value = 0; + return Before != preferReg(); + } +}; + +bool SpillPlacement::runOnMachineFunction(MachineFunction &mf) { + MF = &mf; + bundles = &getAnalysis(); + loops = &getAnalysis(); + + assert(!nodes && "Leaking node array"); + nodes = new Node[bundles->getNumBundles()]; + + // Compute total ingoing and outgoing block frequencies for all bundles. + for (MachineFunction::iterator I = mf.begin(), E = mf.end(); I != E; ++I) { + float Freq = getBlockFrequency(I); + unsigned Num = I->getNumber(); + nodes[bundles->getBundle(Num, 1)].Frequency[0] += Freq; + nodes[bundles->getBundle(Num, 0)].Frequency[1] += Freq; + } + + // We never change the function. + return false; +} + +void SpillPlacement::releaseMemory() { + delete[] nodes; + nodes = 0; +} + +/// activate - mark node n as active if it wasn't already. +void SpillPlacement::activate(unsigned n) { + if (ActiveNodes->test(n)) + return; + ActiveNodes->set(n); + nodes[n].clear(); +} + + +/// prepareNodes - Compute node biases and weights from a set of constraints. +/// Set a bit in NodeMask for each active node. +void SpillPlacement:: +prepareNodes(const SmallVectorImpl &LiveBlocks) { + DEBUG(dbgs() << "Building Hopfield network from " << LiveBlocks.size() + << " constraint blocks:\n"); + for (SmallVectorImpl::const_iterator I = LiveBlocks.begin(), + E = LiveBlocks.end(); I != E; ++I) { + MachineBasicBlock *MBB = MF->getBlockNumbered(I->Number); + float Freq = getBlockFrequency(MBB); + DEBUG(dbgs() << " BB#" << I->Number << format(", Freq = %.1f", Freq)); + + // Is this a transparent block? Link ingoing and outgoing bundles. + if (I->Entry == DontCare && I->Exit == DontCare) { + unsigned ib = bundles->getBundle(I->Number, 0); + unsigned ob = bundles->getBundle(I->Number, 1); + DEBUG(dbgs() << ", transparent EB#" << ib << " -> EB#" << ob << '\n'); + + // Ignore self-loops. + if (ib == ob) + continue; + activate(ib); + activate(ob); + nodes[ib].addLink(ob, Freq, 1); + nodes[ob].addLink(ib, Freq, 0); + continue; + } + + // This block is not transparent, but it can still add bias. + const float Bias[] = { + 0, // DontCare, + 1, // PrefReg, + -1, // PrefSpill + -HUGE_VALF // MustSpill + }; + + // Live-in to block? + if (I->Entry != DontCare) { + unsigned ib = bundles->getBundle(I->Number, 0); + activate(ib); + nodes[ib].addBias(Freq * Bias[I->Entry], 1); + DEBUG(dbgs() << format(", entry EB#%u %+.1f", ib, Freq * Bias[I->Entry])); + } + + // Live-out from block? + if (I->Exit != DontCare) { + unsigned ob = bundles->getBundle(I->Number, 1); + activate(ob); + nodes[ob].addBias(Freq * Bias[I->Exit], 0); + DEBUG(dbgs() << format(", exit EB#%u %+.1f", ob, Freq * Bias[I->Exit])); + } + + DEBUG(dbgs() << '\n'); + } +} + +/// iterate - Repeatedly update the Hopfield nodes until stability or the +/// maximum number of iterations is reached. +/// @param Linked - Numbers of linked nodes that need updating. +void SpillPlacement::iterate(const SmallVectorImpl &Linked) { + DEBUG(dbgs() << "Iterating over " << Linked.size() << " linked nodes:\n"); + if (Linked.empty()) + return; + + // Run up to 10 iterations. The edge bundle numbering is closely related to + // basic block numbering, so there is a strong tendency towards chains of + // linked nodes with sequential numbers. By scanning the linked nodes + // backwards and forwards, we make it very likely that a single node can + // affect the entire network in a single iteration. That means very fast + // convergence, usually in a single iteration. + for (unsigned iteration = 0; iteration != 10; ++iteration) { + // Scan backwards, skipping the last node which was just updated. + bool Changed = false; + for (SmallVectorImpl::const_reverse_iterator I = + llvm::next(Linked.rbegin()), E = Linked.rend(); I != E; ++I) { + unsigned n = *I; + bool C = nodes[n].update(nodes); + Changed |= C; + DEBUG(dbgs() << " \\EB#" << n << format(" = %+2.0f", nodes[n].Value) + << (C ? " *\n" : "\n")); + } + if (!Changed) + return; + + // Scan forwards, skipping the first node which was just updated. + Changed = false; + for (SmallVectorImpl::const_iterator I = + llvm::next(Linked.begin()), E = Linked.end(); I != E; ++I) { + unsigned n = *I; + bool C = nodes[n].update(nodes); + Changed |= C; + DEBUG(dbgs() << " /EB#" << n << format(" = %+2.0f", nodes[n].Value) + << (C ? " *\n" : "\n")); + } + if (!Changed) + return; + } +} + +bool +SpillPlacement::placeSpills(const SmallVectorImpl &LiveBlocks, + BitVector &RegBundles) { + // Reuse RegBundles as our ActiveNodes vector. + ActiveNodes = &RegBundles; + ActiveNodes->clear(); + ActiveNodes->resize(bundles->getNumBundles()); + + // Compute active nodes, links and biases. + prepareNodes(LiveBlocks); + + // Update all active nodes, and find the ones that are actually linked to + // something so their value may change when iterating. + DEBUG(dbgs() << "Network has " << RegBundles.count() << " active nodes:\n"); + SmallVector Linked; + for (int n = RegBundles.find_first(); n>=0; n = RegBundles.find_next(n)) { + nodes[n].update(nodes); + // A node that must spill, or a node without any links is not going to + // change its value ever again, so exclude it from iterations. + if (!nodes[n].Links.empty() && !nodes[n].mustSpill()) + Linked.push_back(n); + + DEBUG({ + dbgs() << " EB#" << n << format(" = %+2.0f", nodes[n].Value) + << format(", Bias %+.2f", nodes[n].Bias) + << format(", Freq %.1f/%.1f", nodes[n].Frequency[0], + nodes[n].Frequency[1]); + for (unsigned i = 0, e = nodes[n].Links.size(); i != e; ++i) + dbgs() << format(", %.2f -> EB#%u", nodes[n].Links[i].first, + nodes[n].Links[i].second); + dbgs() << '\n'; + }); + } + + // Iterate the network to convergence. + iterate(Linked); + + // Write preferences back to RegBundles. + bool Perfect = true; + for (int n = RegBundles.find_first(); n>=0; n = RegBundles.find_next(n)) + if (!nodes[n].preferReg()) { + RegBundles.reset(n); + Perfect = false; + } + return Perfect; +} + +/// getBlockFrequency - Return our best estimate of the block frequency which is +/// the expected number of block executions per function invocation. +float SpillPlacement::getBlockFrequency(const MachineBasicBlock *MBB) { + // Use the unnormalized spill weight for real block frequencies. + return LiveIntervals::getSpillWeight(true, false, loops->getLoopDepth(MBB)); +} + diff --git a/lib/CodeGen/SpillPlacement.h b/lib/CodeGen/SpillPlacement.h new file mode 100644 index 00000000000..0611180ef4a --- /dev/null +++ b/lib/CodeGen/SpillPlacement.h @@ -0,0 +1,105 @@ +//===-- SpillPlacement.h - Optimal Spill Code Placement --------*- C++ -*--===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This analysis computes the optimal spill code placement between basic blocks. +// +// The runOnMachineFunction() method only precomputes some profiling information +// about the CFG. The real work is done by placeSpills() which is called by the +// register allocator. +// +// Given a variable that is live across multiple basic blocks, and given +// constraints on the basic blocks where the variable is live, determine which +// edge bundles should have the variable in a register and which edge bundles +// should have the variable in a stack slot. +// +// The returned bit vector can be used to place optimal spill code at basic +// block entries and exits. Spill code placement inside a basic block is not +// considered. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_SPILLPLACEMENT_H +#define LLVM_CODEGEN_SPILLPLACEMENT_H + +#include "llvm/CodeGen/MachineFunctionPass.h" + +namespace llvm { + +class BitVector; +class EdgeBundles; +class MachineBasicBlock; +class MachineLoopInfo; +template class SmallVectorImpl; + +class SpillPlacement : public MachineFunctionPass { + struct Node; + const MachineFunction *MF; + const EdgeBundles *bundles; + const MachineLoopInfo *loops; + Node *nodes; + + // Nodes that are active in the current computation. Owned by the placeSpills + // caller. + BitVector *ActiveNodes; + +public: + static char ID; // Pass identification, replacement for typeid. + + SpillPlacement() : MachineFunctionPass(ID), nodes(0) {} + ~SpillPlacement() { releaseMemory(); } + + /// BorderConstraint - A basic block has separate constraints for entry and + /// exit. + enum BorderConstraint { + DontCare, ///< Block doesn't care / variable not live. + PrefReg, ///< Block entry/exit prefers a register. + PrefSpill, ///< Block entry/exit prefers a stack slot. + MustSpill ///< A register is impossible, variable must be spilled. + }; + + /// BlockConstraint - Entry and exit constraints for a basic block. + struct BlockConstraint { + unsigned Number; ///< Basic block number (from MBB::getNumber()). + BorderConstraint Entry : 8; ///< Constraint on block entry. + BorderConstraint Exit : 8; ///< Constraint on block exit. + }; + + /// placeSpills - Compute the optimal spill code placement given the + /// constraints. No MustSpill constraints will be violated, and the smallest + /// possible number of PrefX constraints will be violated, weighted by + /// expected execution frequencies. + /// @param LiveBlocks Constraints for blocks that have the variable live in or + /// live out. DontCare/DontCare means the variable is live + /// through the block. DontCare/X means the variable is live + /// out, but not live in. + /// @param RegBundles Bit vector to receive the edge bundles where the + /// variable should be kept in a register. Each bit + /// corresponds to an edge bundle, a set bit means the + /// variable should be kept in a register through the + /// bundle. A clear bit means the variable should be + /// spilled. + /// @return True if a perfect solution was found, allowing the variable to be + /// in a register through all relevant bundles. + bool placeSpills(const SmallVectorImpl &LiveBlocks, + BitVector &RegBundles); + +private: + virtual bool runOnMachineFunction(MachineFunction&); + virtual void getAnalysisUsage(AnalysisUsage&) const; + virtual void releaseMemory(); + + void activate(unsigned); + float getBlockFrequency(const MachineBasicBlock*); + void prepareNodes(const SmallVectorImpl&); + void iterate(const SmallVectorImpl&); +}; + +} // end namespace llvm + +#endif