From f289df2d9544bd3a0934651daa20e589544413ba Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Sat, 11 Jun 2011 01:05:22 +0000 Subject: [PATCH] Branch profiling: floating-point avoidance. Patch by: Jakub Staszak! Introduces BranchProbability. Changes unsigned to uint32_t all over and uint64_t only when overflow is expected. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@132867 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/llvm/Analysis/BranchProbabilityInfo.h | 26 ++-- include/llvm/Support/BranchProbability.h | 50 ++++++++ lib/Analysis/BranchProbabilityInfo.cpp | 117 ++++++++++-------- lib/Support/BranchProbability.cpp | 44 +++++++ lib/Support/CMakeLists.txt | 1 + 5 files changed, 176 insertions(+), 62 deletions(-) create mode 100644 include/llvm/Support/BranchProbability.h create mode 100644 lib/Support/BranchProbability.cpp diff --git a/include/llvm/Analysis/BranchProbabilityInfo.h b/include/llvm/Analysis/BranchProbabilityInfo.h index bf2a6a765f6..91f289da7c4 100644 --- a/include/llvm/Analysis/BranchProbabilityInfo.h +++ b/include/llvm/Analysis/BranchProbabilityInfo.h @@ -15,19 +15,24 @@ #define LLVM_ANALYSIS_BRANCHPROBABILITYINFO_H #include "llvm/InitializePasses.h" +#include "llvm/Support/BranchProbability.h" #include "llvm/Analysis/LoopInfo.h" -#include "llvm/Support/Debug.h" namespace llvm { +class raw_ostream; + class BranchProbabilityInfo : public FunctionPass { // Default weight value. Used when we don't have information about the edge. - static const unsigned int DEFAULT_WEIGHT = 16; + static const uint32_t DEFAULT_WEIGHT = 16; typedef std::pair Edge; - DenseMap Weights; + DenseMap Weights; + + // Get sum of the block successors' weights. + uint32_t getSumForBlock(BasicBlock *BB) const; public: static char ID; @@ -43,12 +48,12 @@ public: bool runOnFunction(Function &F); - // Returned value is between 1 and UINT_MAX. Look at BranchProbabilityInfo.cpp - // for details. - unsigned getEdgeWeight(BasicBlock *Src, BasicBlock *Dst) const; + // Returned value is between 1 and UINT32_MAX. Look at + // BranchProbabilityInfo.cpp for details. + uint32_t getEdgeWeight(BasicBlock *Src, BasicBlock *Dst) const; // Look at BranchProbabilityInfo.cpp for details. Use it with caution! - void setEdgeWeight(BasicBlock *Src, BasicBlock *Dst, unsigned Weight); + void setEdgeWeight(BasicBlock *Src, BasicBlock *Dst, uint32_t Weight); // A 'Hot' edge is an edge which probability is >= 80%. bool isEdgeHot(BasicBlock *Src, BasicBlock *Dst) const; @@ -56,11 +61,16 @@ public: // Return a hot successor for the block BB or null if there isn't one. BasicBlock *getHotSucc(BasicBlock *BB) const; + // Return a probability as a fraction between 0 (0% probability) and + // 1 (100% probability), however the value is never equal to 0, and can be 1 + // only iff SRC block has only one successor. + BranchProbability getEdgeProbability(BasicBlock *Src, BasicBlock *Dst) const; + // Print value between 0 (0% probability) and 1 (100% probability), // however the value is never equal to 0, and can be 1 only iff SRC block // has only one successor. raw_ostream &printEdgeProbability(raw_ostream &OS, BasicBlock *Src, - BasicBlock *Dst) const; + BasicBlock *Dst) const; }; } diff --git a/include/llvm/Support/BranchProbability.h b/include/llvm/Support/BranchProbability.h new file mode 100644 index 00000000000..7ba649133b2 --- /dev/null +++ b/include/llvm/Support/BranchProbability.h @@ -0,0 +1,50 @@ +//===- BranchProbability.h - Branch Probability Analysis --------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Definition of BranchProbability shared by IR and Machine Instructions. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_BRANCHPROBABILITY_H +#define LLVM_SUPPORT_BRANCHPROBABILITY_H + +#include "llvm/Support/DataTypes.h" + +namespace llvm { + +class raw_ostream; +class BranchProbabilityInfo; +class MachineBranchProbabilityInfo; +class MachineBasicBlock; + +// This class represents Branch Probability as a non-negative fraction. +class BranchProbability { + friend class BranchProbabilityInfo; + friend class MachineBranchProbabilityInfo; + friend class MachineBasicBlock; + + // Numerator + uint32_t N; + + // Denominator + uint32_t D; + + BranchProbability(uint32_t n, uint32_t d); + +public: + raw_ostream &print(raw_ostream &OS) const; + + void dump() const; +}; + +raw_ostream &operator<<(raw_ostream &OS, const BranchProbability &Prob); + +} + +#endif diff --git a/lib/Analysis/BranchProbabilityInfo.cpp b/lib/Analysis/BranchProbabilityInfo.cpp index d69c0120bb4..812fac0bb75 100644 --- a/lib/Analysis/BranchProbabilityInfo.cpp +++ b/lib/Analysis/BranchProbabilityInfo.cpp @@ -13,7 +13,7 @@ #include "llvm/Instructions.h" #include "llvm/Analysis/BranchProbabilityInfo.h" -#include +#include "llvm/Support/Debug.h" using namespace llvm; @@ -34,7 +34,7 @@ class BranchProbabilityAnalysis { typedef std::pair Edge; - DenseMap *Weights; + DenseMap *Weights; BranchProbabilityInfo *BP; @@ -62,15 +62,15 @@ class BranchProbabilityAnalysis { // Probability of the edge BB2->BB1 = 128 / (128 + 4) = 0.9696.. // Probability of the edge BB2->BB3 = 4 / (128 + 4) = 0.0303.. - static const unsigned int LBH_TAKEN_WEIGHT = 128; - static const unsigned int LBH_NONTAKEN_WEIGHT = 4; + static const uint32_t LBH_TAKEN_WEIGHT = 128; + static const uint32_t LBH_NONTAKEN_WEIGHT = 4; // Standard weight value. Used when none of the heuristics set weight for // the edge. - static const unsigned int NORMAL_WEIGHT = 16; + static const uint32_t NORMAL_WEIGHT = 16; // Minimum weight of an edge. Please note, that weight is NEVER 0. - static const unsigned int MIN_WEIGHT = 1; + static const uint32_t MIN_WEIGHT = 1; // Return TRUE if BB leads directly to a Return Instruction. static bool isReturningBlock(BasicBlock *BB) { @@ -101,8 +101,8 @@ class BranchProbabilityAnalysis { // Multiply Edge Weight by two. void incEdgeWeight(BasicBlock *Src, BasicBlock *Dst) { - unsigned Weight = BP->getEdgeWeight(Src, Dst); - unsigned MaxWeight = getMaxWeightFor(Src); + uint32_t Weight = BP->getEdgeWeight(Src, Dst); + uint32_t MaxWeight = getMaxWeightFor(Src); if (Weight * 2 > MaxWeight) BP->setEdgeWeight(Src, Dst, MaxWeight); @@ -112,7 +112,7 @@ class BranchProbabilityAnalysis { // Divide Edge Weight by two. void decEdgeWeight(BasicBlock *Src, BasicBlock *Dst) { - unsigned Weight = BP->getEdgeWeight(Src, Dst); + uint32_t Weight = BP->getEdgeWeight(Src, Dst); assert(Weight > 0); if (Weight / 2 < MIN_WEIGHT) @@ -122,12 +122,12 @@ class BranchProbabilityAnalysis { } - unsigned getMaxWeightFor(BasicBlock *BB) const { - return UINT_MAX / BB->getTerminator()->getNumSuccessors(); + uint32_t getMaxWeightFor(BasicBlock *BB) const { + return UINT32_MAX / BB->getTerminator()->getNumSuccessors(); } public: - BranchProbabilityAnalysis(DenseMap *W, + BranchProbabilityAnalysis(DenseMap *W, BranchProbabilityInfo *BP, LoopInfo *LI) : Weights(W), BP(BP), LI(LI) { } @@ -195,7 +195,7 @@ void BranchProbabilityAnalysis::calcPointerHeuristics(BasicBlock *BB) { // Calculate Edge Weights using "Loop Branch Heuristics". Predict backedges // as taken, exiting edges as not-taken. void BranchProbabilityAnalysis::calcLoopBranchHeuristics(BasicBlock *BB) { - unsigned numSuccs = BB->getTerminator()->getNumSuccessors(); + uint32_t numSuccs = BB->getTerminator()->getNumSuccessors(); Loop *L = LI->getLoopFor(BB); if (!L) @@ -213,8 +213,8 @@ void BranchProbabilityAnalysis::calcLoopBranchHeuristics(BasicBlock *BB) { BackEdges.push_back(Succ); } - if (unsigned numBackEdges = BackEdges.size()) { - unsigned backWeight = LBH_TAKEN_WEIGHT / numBackEdges; + if (uint32_t numBackEdges = BackEdges.size()) { + uint32_t backWeight = LBH_TAKEN_WEIGHT / numBackEdges; if (backWeight < NORMAL_WEIGHT) backWeight = NORMAL_WEIGHT; @@ -225,9 +225,9 @@ void BranchProbabilityAnalysis::calcLoopBranchHeuristics(BasicBlock *BB) { } } - unsigned numExitingEdges = ExitingEdges.size(); - if (unsigned numNonExitingEdges = numSuccs - numExitingEdges) { - unsigned exitWeight = LBH_NONTAKEN_WEIGHT / numNonExitingEdges; + uint32_t numExitingEdges = ExitingEdges.size(); + if (uint32_t numNonExitingEdges = numSuccs - numExitingEdges) { + uint32_t exitWeight = LBH_NONTAKEN_WEIGHT / numNonExitingEdges; if (exitWeight < MIN_WEIGHT) exitWeight = MIN_WEIGHT; @@ -260,36 +260,43 @@ bool BranchProbabilityAnalysis::runOnFunction(Function &F) { bool BranchProbabilityInfo::runOnFunction(Function &F) { LoopInfo &LI = getAnalysis(); BranchProbabilityAnalysis BPA(&Weights, this, &LI); - bool ret = BPA.runOnFunction(F); - return ret; + return BPA.runOnFunction(F); } -// TODO: This currently hardcodes 80% as a fraction 4/5. We will soon add a -// BranchProbability class to encapsulate the fractional probability and -// define a few static instances of the class for use as predefined thresholds. -bool BranchProbabilityInfo::isEdgeHot(BasicBlock *Src, BasicBlock *Dst) const { - unsigned Sum = 0; - for (succ_iterator I = succ_begin(Src), E = succ_end(Src); I != E; ++I) { +uint32_t BranchProbabilityInfo::getSumForBlock(BasicBlock *BB) const { + uint32_t Sum = 0; + + for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) { BasicBlock *Succ = *I; - unsigned Weight = getEdgeWeight(Src, Succ); - unsigned PrevSum = Sum; + uint32_t Weight = getEdgeWeight(BB, Succ); + uint32_t PrevSum = Sum; Sum += Weight; assert(Sum > PrevSum); (void) PrevSum; } - return getEdgeWeight(Src, Dst) * 5 > Sum * 4; + return Sum; +} + +bool BranchProbabilityInfo::isEdgeHot(BasicBlock *Src, BasicBlock *Dst) const { + // Hot probability is at least 4/5 = 80% + uint32_t Weight = getEdgeWeight(Src, Dst); + uint32_t Sum = getSumForBlock(Src); + + // FIXME: Implement BranchProbability::compare then change this code to + // compare this BranchProbability against a static "hot" BranchProbability. + return (uint64_t)Weight * 5 > (uint64_t)Sum * 4; } BasicBlock *BranchProbabilityInfo::getHotSucc(BasicBlock *BB) const { - unsigned Sum = 0; - unsigned MaxWeight = 0; + uint32_t Sum = 0; + uint32_t MaxWeight = 0; BasicBlock *MaxSucc = 0; for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) { BasicBlock *Succ = *I; - unsigned Weight = getEdgeWeight(BB, Succ); - unsigned PrevSum = Sum; + uint32_t Weight = getEdgeWeight(BB, Succ); + uint32_t PrevSum = Sum; Sum += Weight; assert(Sum > PrevSum); (void) PrevSum; @@ -300,17 +307,18 @@ BasicBlock *BranchProbabilityInfo::getHotSucc(BasicBlock *BB) const { } } - if (MaxWeight * 5 > Sum * 4) + // FIXME: Use BranchProbability::compare. + if ((uint64_t)MaxWeight * 5 > (uint64_t)Sum * 4) return MaxSucc; return 0; } // Return edge's weight. If can't find it, return DEFAULT_WEIGHT value. -unsigned +uint32_t BranchProbabilityInfo::getEdgeWeight(BasicBlock *Src, BasicBlock *Dst) const { Edge E(Src, Dst); - DenseMap::const_iterator I = Weights.find(E); + DenseMap::const_iterator I = Weights.find(E); if (I != Weights.end()) return I->second; @@ -319,30 +327,31 @@ BranchProbabilityInfo::getEdgeWeight(BasicBlock *Src, BasicBlock *Dst) const { } void BranchProbabilityInfo::setEdgeWeight(BasicBlock *Src, BasicBlock *Dst, - unsigned Weight) { + uint32_t Weight) { Weights[std::make_pair(Src, Dst)] = Weight; - DEBUG(dbgs() << "setEdgeWeight: " << Src->getNameStr() << " -> " - << Dst->getNameStr() << " to " << Weight - << (isEdgeHot(Src, Dst) ? " [is HOT now]\n" : "\n")); + DEBUG(dbgs() << "set edge " << Src->getNameStr() << " -> " + << Dst->getNameStr() << " weight to " << Weight + << (isEdgeHot(Src, Dst) ? " [is HOT now]\n" : "\n")); +} + + +BranchProbability BranchProbabilityInfo:: +getEdgeProbability(BasicBlock *Src, BasicBlock *Dst) const { + + uint32_t N = getEdgeWeight(Src, Dst); + uint32_t D = getSumForBlock(Src); + + return BranchProbability(N, D); } raw_ostream & BranchProbabilityInfo::printEdgeProbability(raw_ostream &OS, BasicBlock *Src, - BasicBlock *Dst) const { + BasicBlock *Dst) const { + BranchProbability Prob = getEdgeProbability(Src, Dst); - unsigned Sum = 0; - for (succ_iterator I = succ_begin(Src), E = succ_end(Src); I != E; ++I) { - BasicBlock *Succ = *I; - unsigned Weight = getEdgeWeight(Src, Succ); - unsigned PrevSum = Sum; - - Sum += Weight; - assert(Sum > PrevSum); (void) PrevSum; - } - - double Prob = (double)getEdgeWeight(Src, Dst) / Sum; - OS << "probability (" << Src->getNameStr() << " --> " << Dst->getNameStr() - << ") = " << Prob << "\n"; + OS << "edge " << Src->getNameStr() << " -> " << Dst->getNameStr() + << " probability is " << Prob + << (isEdgeHot(Src, Dst) ? " [HOT edge]\n" : "\n"); return OS; } diff --git a/lib/Support/BranchProbability.cpp b/lib/Support/BranchProbability.cpp new file mode 100644 index 00000000000..97342da3f07 --- /dev/null +++ b/lib/Support/BranchProbability.cpp @@ -0,0 +1,44 @@ +//===-------------- lib/Support/BranchProbability.cpp -----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements Branch Probability class. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Support/BranchProbability.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; + +BranchProbability::BranchProbability(uint32_t n, uint32_t d) { + assert(d > 0 && "Denomiator cannot be 0!"); + assert(n <= d && "Probability cannot be bigger than 1!"); + N = n; + D = d; +} + +raw_ostream &BranchProbability::print(raw_ostream &OS) const { + OS << N << " / " << D << " = " << ((double)N / D); + return OS; +} + +void BranchProbability::dump() const { + print(dbgs()); + dbgs() << "\n"; +} + +namespace llvm { + +raw_ostream &operator<<(raw_ostream &OS, const BranchProbability &Prob) { + Prob.print(OS); + return OS; +} + +} diff --git a/lib/Support/CMakeLists.txt b/lib/Support/CMakeLists.txt index a0e997d349f..867d930a0f9 100644 --- a/lib/Support/CMakeLists.txt +++ b/lib/Support/CMakeLists.txt @@ -9,6 +9,7 @@ add_llvm_library(LLVMSupport APInt.cpp APSInt.cpp Allocator.cpp + BranchProbability.cpp circular_raw_ostream.cpp CommandLine.cpp ConstantRange.cpp