2013-07-12 18:11:14 +00:00
|
|
|
//===-- BlockFrequencyImpl.h - Block Frequency Implementation --*- C++ -*--===//
|
2011-06-23 21:56:59 +00:00
|
|
|
//
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
// Shared implementation of BlockFrequency for IR and Machine Instructions.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
#ifndef LLVM_ANALYSIS_BLOCKFREQUENCYIMPL_H
|
|
|
|
#define LLVM_ANALYSIS_BLOCKFREQUENCYIMPL_H
|
|
|
|
|
|
|
|
#include "llvm/ADT/DenseMap.h"
|
|
|
|
#include "llvm/ADT/PostOrderIterator.h"
|
|
|
|
#include "llvm/CodeGen/MachineBasicBlock.h"
|
2011-07-16 20:23:20 +00:00
|
|
|
#include "llvm/CodeGen/MachineFunction.h"
|
2013-01-02 11:36:10 +00:00
|
|
|
#include "llvm/IR/BasicBlock.h"
|
2011-07-27 22:05:51 +00:00
|
|
|
#include "llvm/Support/BlockFrequency.h"
|
2011-06-23 21:56:59 +00:00
|
|
|
#include "llvm/Support/BranchProbability.h"
|
|
|
|
#include "llvm/Support/Debug.h"
|
2011-07-16 20:23:20 +00:00
|
|
|
#include "llvm/Support/raw_ostream.h"
|
2011-06-23 21:56:59 +00:00
|
|
|
#include <string>
|
2012-12-03 17:02:12 +00:00
|
|
|
#include <vector>
|
2011-06-23 21:56:59 +00:00
|
|
|
|
|
|
|
namespace llvm {
|
|
|
|
|
|
|
|
|
2011-07-25 19:25:40 +00:00
|
|
|
class BlockFrequencyInfo;
|
|
|
|
class MachineBlockFrequencyInfo;
|
2011-07-16 20:23:20 +00:00
|
|
|
|
2011-06-23 21:56:59 +00:00
|
|
|
/// BlockFrequencyImpl implements block frequency algorithm for IR and
|
2013-06-25 13:34:40 +00:00
|
|
|
/// Machine Instructions. Algorithm starts with value ENTRY_FREQ
|
2011-06-23 21:56:59 +00:00
|
|
|
/// for the entry block and then propagates frequencies using branch weights
|
|
|
|
/// from (Machine)BranchProbabilityInfo. LoopInfo is not required because
|
|
|
|
/// algorithm can find "backedges" by itself.
|
|
|
|
template<class BlockT, class FunctionT, class BlockProbInfoT>
|
|
|
|
class BlockFrequencyImpl {
|
|
|
|
|
2011-12-20 20:03:10 +00:00
|
|
|
DenseMap<const BlockT *, BlockFrequency> Freqs;
|
2011-06-23 21:56:59 +00:00
|
|
|
|
|
|
|
BlockProbInfoT *BPI;
|
|
|
|
|
|
|
|
FunctionT *Fn;
|
|
|
|
|
|
|
|
typedef GraphTraits< Inverse<BlockT *> > GT;
|
|
|
|
|
2011-08-01 15:22:10 +00:00
|
|
|
const uint32_t EntryFreq;
|
2011-06-23 21:56:59 +00:00
|
|
|
|
|
|
|
std::string getBlockName(BasicBlock *BB) const {
|
2011-11-15 16:27:03 +00:00
|
|
|
return BB->getName().str();
|
2011-06-23 21:56:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
std::string getBlockName(MachineBasicBlock *MBB) const {
|
2011-11-15 16:27:03 +00:00
|
|
|
std::string str;
|
|
|
|
raw_string_ostream ss(str);
|
2011-06-23 21:56:59 +00:00
|
|
|
ss << "BB#" << MBB->getNumber();
|
|
|
|
|
2011-07-16 20:23:20 +00:00
|
|
|
if (const BasicBlock *BB = MBB->getBasicBlock())
|
2011-11-15 16:27:03 +00:00
|
|
|
ss << " derived from LLVM BB " << BB->getName();
|
2011-06-23 21:56:59 +00:00
|
|
|
|
|
|
|
return ss.str();
|
|
|
|
}
|
|
|
|
|
2011-07-27 22:05:51 +00:00
|
|
|
void setBlockFreq(BlockT *BB, BlockFrequency Freq) {
|
2011-06-23 21:56:59 +00:00
|
|
|
Freqs[BB] = Freq;
|
|
|
|
DEBUG(dbgs() << "Frequency(" << getBlockName(BB) << ") = " << Freq << "\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
/// getEdgeFreq - Return edge frequency based on SRC frequency and Src -> Dst
|
|
|
|
/// edge probability.
|
2011-07-27 22:05:51 +00:00
|
|
|
BlockFrequency getEdgeFreq(BlockT *Src, BlockT *Dst) const {
|
2011-06-23 21:56:59 +00:00
|
|
|
BranchProbability Prob = BPI->getEdgeProbability(Src, Dst);
|
2011-07-27 22:05:51 +00:00
|
|
|
return getBlockFreq(Src) * Prob;
|
2011-06-23 21:56:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/// incBlockFreq - Increase BB block frequency by FREQ.
|
|
|
|
///
|
2011-07-27 22:05:51 +00:00
|
|
|
void incBlockFreq(BlockT *BB, BlockFrequency Freq) {
|
2011-06-23 21:56:59 +00:00
|
|
|
Freqs[BB] += Freq;
|
|
|
|
DEBUG(dbgs() << "Frequency(" << getBlockName(BB) << ") += " << Freq
|
|
|
|
<< " --> " << Freqs[BB] << "\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
// All blocks in postorder.
|
|
|
|
std::vector<BlockT *> POT;
|
|
|
|
|
|
|
|
// Map Block -> Position in reverse-postorder list.
|
|
|
|
DenseMap<BlockT *, unsigned> RPO;
|
|
|
|
|
2013-06-28 22:40:43 +00:00
|
|
|
// For each loop header, record the per-iteration probability of exiting the
|
|
|
|
// loop. This is the reciprocal of the expected number of loop iterations.
|
|
|
|
typedef DenseMap<BlockT*, BranchProbability> LoopExitProbMap;
|
|
|
|
LoopExitProbMap LoopExitProb;
|
2011-06-23 21:56:59 +00:00
|
|
|
|
|
|
|
// (reverse-)postorder traversal iterators.
|
|
|
|
typedef typename std::vector<BlockT *>::iterator pot_iterator;
|
|
|
|
typedef typename std::vector<BlockT *>::reverse_iterator rpot_iterator;
|
|
|
|
|
|
|
|
pot_iterator pot_begin() { return POT.begin(); }
|
|
|
|
pot_iterator pot_end() { return POT.end(); }
|
|
|
|
|
|
|
|
rpot_iterator rpot_begin() { return POT.rbegin(); }
|
|
|
|
rpot_iterator rpot_end() { return POT.rend(); }
|
|
|
|
|
|
|
|
rpot_iterator rpot_at(BlockT *BB) {
|
|
|
|
rpot_iterator I = rpot_begin();
|
2013-06-25 23:32:10 +00:00
|
|
|
unsigned idx = RPO.lookup(BB);
|
2011-06-23 21:56:59 +00:00
|
|
|
assert(idx);
|
|
|
|
std::advance(I, idx - 1);
|
|
|
|
|
|
|
|
assert(*I == BB);
|
|
|
|
return I;
|
|
|
|
}
|
|
|
|
|
2013-06-25 23:32:10 +00:00
|
|
|
/// isBackedge - Return if edge Src -> Dst is a reachable backedge.
|
2011-06-23 21:56:59 +00:00
|
|
|
///
|
2013-08-20 23:04:15 +00:00
|
|
|
bool isBackedge(BlockT *Src, BlockT *Dst) const {
|
2013-06-25 23:32:10 +00:00
|
|
|
unsigned a = RPO.lookup(Src);
|
|
|
|
if (!a)
|
|
|
|
return false;
|
|
|
|
unsigned b = RPO.lookup(Dst);
|
|
|
|
assert(b && "Destination block should be reachable");
|
2011-07-27 15:42:09 +00:00
|
|
|
return a >= b;
|
2011-06-23 21:56:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/// getSingleBlockPred - return single BB block predecessor or NULL if
|
|
|
|
/// BB has none or more predecessors.
|
|
|
|
BlockT *getSingleBlockPred(BlockT *BB) {
|
|
|
|
typename GT::ChildIteratorType
|
|
|
|
PI = GraphTraits< Inverse<BlockT *> >::child_begin(BB),
|
|
|
|
PE = GraphTraits< Inverse<BlockT *> >::child_end(BB);
|
|
|
|
|
|
|
|
if (PI == PE)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
BlockT *Pred = *PI;
|
|
|
|
|
|
|
|
++PI;
|
|
|
|
if (PI != PE)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return Pred;
|
|
|
|
}
|
|
|
|
|
|
|
|
void doBlock(BlockT *BB, BlockT *LoopHead,
|
|
|
|
SmallPtrSet<BlockT *, 8> &BlocksInLoop) {
|
|
|
|
|
|
|
|
DEBUG(dbgs() << "doBlock(" << getBlockName(BB) << ")\n");
|
|
|
|
setBlockFreq(BB, 0);
|
|
|
|
|
|
|
|
if (BB == LoopHead) {
|
2011-08-01 15:22:10 +00:00
|
|
|
setBlockFreq(BB, EntryFreq);
|
2011-06-23 21:56:59 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if(BlockT *Pred = getSingleBlockPred(BB)) {
|
|
|
|
if (BlocksInLoop.count(Pred))
|
|
|
|
setBlockFreq(BB, getEdgeFreq(Pred, BB));
|
|
|
|
// TODO: else? irreducible, ignore it for now.
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool isInLoop = false;
|
|
|
|
bool isLoopHead = false;
|
|
|
|
|
|
|
|
for (typename GT::ChildIteratorType
|
|
|
|
PI = GraphTraits< Inverse<BlockT *> >::child_begin(BB),
|
|
|
|
PE = GraphTraits< Inverse<BlockT *> >::child_end(BB);
|
|
|
|
PI != PE; ++PI) {
|
|
|
|
BlockT *Pred = *PI;
|
|
|
|
|
2013-06-25 23:32:10 +00:00
|
|
|
if (isBackedge(Pred, BB)) {
|
2011-06-23 21:56:59 +00:00
|
|
|
isLoopHead = true;
|
|
|
|
} else if (BlocksInLoop.count(Pred)) {
|
|
|
|
incBlockFreq(BB, getEdgeFreq(Pred, BB));
|
|
|
|
isInLoop = true;
|
|
|
|
}
|
|
|
|
// TODO: else? irreducible.
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!isInLoop)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (!isLoopHead)
|
|
|
|
return;
|
|
|
|
|
2013-06-28 22:40:43 +00:00
|
|
|
// This block is a loop header, so boost its frequency by the expected
|
|
|
|
// number of loop iterations. The loop blocks will be revisited so they all
|
|
|
|
// get this boost.
|
|
|
|
typename LoopExitProbMap::const_iterator I = LoopExitProb.find(BB);
|
|
|
|
assert(I != LoopExitProb.end() && "Loop header missing from table");
|
|
|
|
Freqs[BB] /= I->second;
|
|
|
|
DEBUG(dbgs() << "Loop header scaled to " << Freqs[BB] << ".\n");
|
2011-06-23 21:56:59 +00:00
|
|
|
}
|
|
|
|
|
2012-06-02 10:20:22 +00:00
|
|
|
/// doLoop - Propagate block frequency down through the loop.
|
2011-06-23 21:56:59 +00:00
|
|
|
void doLoop(BlockT *Head, BlockT *Tail) {
|
|
|
|
DEBUG(dbgs() << "doLoop(" << getBlockName(Head) << ", "
|
|
|
|
<< getBlockName(Tail) << ")\n");
|
|
|
|
|
|
|
|
SmallPtrSet<BlockT *, 8> BlocksInLoop;
|
|
|
|
|
2011-07-28 20:09:31 +00:00
|
|
|
for (rpot_iterator I = rpot_at(Head), E = rpot_at(Tail); ; ++I) {
|
2011-06-23 21:56:59 +00:00
|
|
|
BlockT *BB = *I;
|
|
|
|
doBlock(BB, Head, BlocksInLoop);
|
|
|
|
|
|
|
|
BlocksInLoop.insert(BB);
|
2011-07-28 20:17:18 +00:00
|
|
|
if (I == E)
|
2011-07-28 20:09:31 +00:00
|
|
|
break;
|
2011-06-23 21:56:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Compute loop's cyclic probability using backedges probabilities.
|
2013-06-28 22:40:43 +00:00
|
|
|
BlockFrequency BackFreq;
|
2011-06-23 21:56:59 +00:00
|
|
|
for (typename GT::ChildIteratorType
|
|
|
|
PI = GraphTraits< Inverse<BlockT *> >::child_begin(Head),
|
|
|
|
PE = GraphTraits< Inverse<BlockT *> >::child_end(Head);
|
|
|
|
PI != PE; ++PI) {
|
|
|
|
BlockT *Pred = *PI;
|
|
|
|
assert(Pred);
|
2013-06-28 22:40:43 +00:00
|
|
|
if (isBackedge(Pred, Head))
|
|
|
|
BackFreq += getEdgeFreq(Pred, Head);
|
|
|
|
}
|
|
|
|
|
|
|
|
// The cyclic probability is freq(BackEdges) / freq(Head), where freq(Head)
|
|
|
|
// only counts edges entering the loop, not the loop backedges.
|
|
|
|
// The probability of leaving the loop on each iteration is:
|
|
|
|
//
|
|
|
|
// ExitProb = 1 - CyclicProb
|
|
|
|
//
|
|
|
|
// The Expected number of loop iterations is:
|
|
|
|
//
|
|
|
|
// Iterations = 1 / ExitProb
|
|
|
|
//
|
2013-06-28 22:54:16 +00:00
|
|
|
uint64_t D = std::max(getBlockFreq(Head).getFrequency(), UINT64_C(1));
|
|
|
|
uint64_t N = std::max(BackFreq.getFrequency(), UINT64_C(1));
|
2013-06-28 22:40:43 +00:00
|
|
|
if (N < D)
|
|
|
|
N = D - N;
|
|
|
|
else
|
|
|
|
// We'd expect N < D, but rounding and saturation means that can't be
|
|
|
|
// guaranteed.
|
|
|
|
N = 1;
|
|
|
|
|
|
|
|
// Now ExitProb = N / D, make sure it fits in an i32/i32 fraction.
|
|
|
|
assert(N <= D);
|
|
|
|
if (D > UINT32_MAX) {
|
|
|
|
unsigned Shift = 32 - countLeadingZeros(D);
|
|
|
|
D >>= Shift;
|
|
|
|
N >>= Shift;
|
|
|
|
if (N == 0)
|
|
|
|
N = 1;
|
2011-06-23 21:56:59 +00:00
|
|
|
}
|
2013-06-28 22:40:43 +00:00
|
|
|
BranchProbability LEP = BranchProbability(N, D);
|
|
|
|
LoopExitProb.insert(std::make_pair(Head, LEP));
|
|
|
|
DEBUG(dbgs() << "LoopExitProb[" << getBlockName(Head) << "] = " << LEP
|
|
|
|
<< " from 1 - " << BackFreq << " / " << getBlockFreq(Head)
|
|
|
|
<< ".\n");
|
2011-06-23 21:56:59 +00:00
|
|
|
}
|
|
|
|
|
2011-07-25 19:25:40 +00:00
|
|
|
friend class BlockFrequencyInfo;
|
|
|
|
friend class MachineBlockFrequencyInfo;
|
2011-06-23 21:56:59 +00:00
|
|
|
|
2011-08-01 15:22:10 +00:00
|
|
|
BlockFrequencyImpl() : EntryFreq(BlockFrequency::getEntryFrequency()) { }
|
|
|
|
|
2011-06-23 21:56:59 +00:00
|
|
|
void doFunction(FunctionT *fn, BlockProbInfoT *bpi) {
|
|
|
|
Fn = fn;
|
|
|
|
BPI = bpi;
|
|
|
|
|
|
|
|
// Clear everything.
|
|
|
|
RPO.clear();
|
|
|
|
POT.clear();
|
2013-06-28 22:40:43 +00:00
|
|
|
LoopExitProb.clear();
|
2011-06-23 21:56:59 +00:00
|
|
|
Freqs.clear();
|
|
|
|
|
|
|
|
BlockT *EntryBlock = fn->begin();
|
|
|
|
|
2013-03-20 21:00:22 +00:00
|
|
|
std::copy(po_begin(EntryBlock), po_end(EntryBlock), std::back_inserter(POT));
|
2011-06-23 21:56:59 +00:00
|
|
|
|
|
|
|
unsigned RPOidx = 0;
|
|
|
|
for (rpot_iterator I = rpot_begin(), E = rpot_end(); I != E; ++I) {
|
|
|
|
BlockT *BB = *I;
|
|
|
|
RPO[BB] = ++RPOidx;
|
|
|
|
DEBUG(dbgs() << "RPO[" << getBlockName(BB) << "] = " << RPO[BB] << "\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
// Travel over all blocks in postorder.
|
|
|
|
for (pot_iterator I = pot_begin(), E = pot_end(); I != E; ++I) {
|
|
|
|
BlockT *BB = *I;
|
|
|
|
BlockT *LastTail = 0;
|
|
|
|
DEBUG(dbgs() << "POT: " << getBlockName(BB) << "\n");
|
|
|
|
|
|
|
|
for (typename GT::ChildIteratorType
|
|
|
|
PI = GraphTraits< Inverse<BlockT *> >::child_begin(BB),
|
|
|
|
PE = GraphTraits< Inverse<BlockT *> >::child_end(BB);
|
|
|
|
PI != PE; ++PI) {
|
|
|
|
|
|
|
|
BlockT *Pred = *PI;
|
2013-06-25 23:32:10 +00:00
|
|
|
if (isBackedge(Pred, BB) && (!LastTail || RPO[Pred] > RPO[LastTail]))
|
2011-06-23 21:56:59 +00:00
|
|
|
LastTail = Pred;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (LastTail)
|
|
|
|
doLoop(BB, LastTail);
|
|
|
|
}
|
|
|
|
|
|
|
|
// At the end assume the whole function as a loop, and travel over it once
|
|
|
|
// again.
|
|
|
|
doLoop(*(rpot_begin()), *(pot_begin()));
|
|
|
|
}
|
|
|
|
|
|
|
|
public:
|
2011-07-22 02:24:57 +00:00
|
|
|
/// getBlockFreq - Return block frequency. Return 0 if we don't have it.
|
2011-12-20 20:03:10 +00:00
|
|
|
BlockFrequency getBlockFreq(const BlockT *BB) const {
|
|
|
|
typename DenseMap<const BlockT *, BlockFrequency>::const_iterator
|
|
|
|
I = Freqs.find(BB);
|
2011-06-23 21:56:59 +00:00
|
|
|
if (I != Freqs.end())
|
2011-07-22 02:24:57 +00:00
|
|
|
return I->second;
|
|
|
|
return 0;
|
2011-06-23 21:56:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void print(raw_ostream &OS) const {
|
|
|
|
OS << "\n\n---- Block Freqs ----\n";
|
|
|
|
for (typename FunctionT::iterator I = Fn->begin(), E = Fn->end(); I != E;) {
|
|
|
|
BlockT *BB = I++;
|
|
|
|
OS << " " << getBlockName(BB) << " = " << getBlockFreq(BB) << "\n";
|
|
|
|
|
|
|
|
for (typename GraphTraits<BlockT *>::ChildIteratorType
|
|
|
|
SI = GraphTraits<BlockT *>::child_begin(BB),
|
|
|
|
SE = GraphTraits<BlockT *>::child_end(BB); SI != SE; ++SI) {
|
|
|
|
BlockT *Succ = *SI;
|
|
|
|
OS << " " << getBlockName(BB) << " -> " << getBlockName(Succ)
|
|
|
|
<< " = " << getEdgeFreq(BB, Succ) << "\n";
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void dump() const {
|
|
|
|
print(dbgs());
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|