2002-05-10 22:44:58 +00:00
|
|
|
//===-- LICM.cpp - Loop Invariant Code Motion Pass ------------------------===//
|
2005-04-21 23:48:37 +00:00
|
|
|
//
|
2003-10-20 19:43:21 +00:00
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
2007-12-29 20:36:04 +00:00
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
2005-04-21 23:48:37 +00:00
|
|
|
//
|
2003-10-20 19:43:21 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
2002-05-10 22:44:58 +00:00
|
|
|
//
|
2003-12-09 17:18:00 +00:00
|
|
|
// This pass performs loop invariant code motion, attempting to remove as much
|
|
|
|
// code from the body of a loop as possible. It does this by either hoisting
|
|
|
|
// code into the preheader block, or by sinking code to the exit blocks if it is
|
|
|
|
// safe. This pass also promotes must-aliased memory locations in the loop to
|
2003-12-19 07:22:45 +00:00
|
|
|
// live in registers, thus hoisting and sinking "invariant" loads and stores.
|
2003-12-09 17:18:00 +00:00
|
|
|
//
|
|
|
|
// This pass uses alias analysis for two purposes:
|
2003-02-24 03:52:32 +00:00
|
|
|
//
|
2004-05-23 21:20:19 +00:00
|
|
|
// 1. Moving loop invariant loads and calls out of loops. If we can determine
|
|
|
|
// that a load or call inside of a loop never aliases anything stored to,
|
|
|
|
// we can hoist it or sink it like any other instruction.
|
2003-02-24 03:52:32 +00:00
|
|
|
// 2. Scalar Promotion of Memory - If there is a store instruction inside of
|
|
|
|
// the loop, we try to move the store to happen AFTER the loop instead of
|
|
|
|
// inside of the loop. This can only happen if a few conditions are true:
|
|
|
|
// A. The pointer stored through is loop invariant
|
|
|
|
// B. There are no stores or loads in the loop which _may_ alias the
|
|
|
|
// pointer. There are no calls in the loop which mod/ref the pointer.
|
|
|
|
// If these conditions are true, we can promote the loads and stores in the
|
|
|
|
// loop of the pointer to use a temporary alloca'd variable. We then use
|
|
|
|
// the mem2reg functionality to construct the appropriate SSA form for the
|
|
|
|
// variable.
|
2002-05-10 22:44:58 +00:00
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2005-03-23 21:00:12 +00:00
|
|
|
#define DEBUG_TYPE "licm"
|
2002-05-10 22:44:58 +00:00
|
|
|
#include "llvm/Transforms/Scalar.h"
|
2007-01-30 23:46:24 +00:00
|
|
|
#include "llvm/Constants.h"
|
2004-05-23 21:20:19 +00:00
|
|
|
#include "llvm/DerivedTypes.h"
|
|
|
|
#include "llvm/Instructions.h"
|
2009-07-03 00:54:20 +00:00
|
|
|
#include "llvm/LLVMContext.h"
|
2004-05-23 21:20:19 +00:00
|
|
|
#include "llvm/Target/TargetData.h"
|
2002-05-10 22:44:58 +00:00
|
|
|
#include "llvm/Analysis/LoopInfo.h"
|
2007-03-07 04:41:30 +00:00
|
|
|
#include "llvm/Analysis/LoopPass.h"
|
2002-08-22 21:39:55 +00:00
|
|
|
#include "llvm/Analysis/AliasAnalysis.h"
|
2003-03-03 23:32:45 +00:00
|
|
|
#include "llvm/Analysis/AliasSetTracker.h"
|
2002-09-29 21:46:09 +00:00
|
|
|
#include "llvm/Analysis/Dominators.h"
|
2007-07-30 20:19:59 +00:00
|
|
|
#include "llvm/Analysis/ScalarEvolution.h"
|
2004-05-23 21:20:19 +00:00
|
|
|
#include "llvm/Transforms/Utils/PromoteMemToReg.h"
|
2007-02-05 23:32:05 +00:00
|
|
|
#include "llvm/Support/CFG.h"
|
|
|
|
#include "llvm/Support/Compiler.h"
|
2004-09-01 22:55:40 +00:00
|
|
|
#include "llvm/Support/CommandLine.h"
|
|
|
|
#include "llvm/Support/Debug.h"
|
|
|
|
#include "llvm/ADT/Statistic.h"
|
2002-05-10 22:44:58 +00:00
|
|
|
#include <algorithm>
|
2003-12-09 17:18:00 +00:00
|
|
|
using namespace llvm;
|
2003-11-11 22:41:34 +00:00
|
|
|
|
2006-12-19 21:40:18 +00:00
|
|
|
STATISTIC(NumSunk , "Number of instructions sunk out of loop");
|
|
|
|
STATISTIC(NumHoisted , "Number of instructions hoisted out of loop");
|
|
|
|
STATISTIC(NumMovedLoads, "Number of load insts hoisted or sunk");
|
|
|
|
STATISTIC(NumMovedCalls, "Number of call insts hoisted or sunk");
|
|
|
|
STATISTIC(NumPromoted , "Number of memory locations promoted to registers");
|
|
|
|
|
2008-05-13 00:00:25 +00:00
|
|
|
static cl::opt<bool>
|
|
|
|
DisablePromotion("disable-licm-promotion", cl::Hidden,
|
|
|
|
cl::desc("Disable memory promotion in LICM pass"));
|
2003-02-24 03:52:32 +00:00
|
|
|
|
2009-03-09 05:11:09 +00:00
|
|
|
// This feature is currently disabled by default because CodeGen is not yet
|
|
|
|
// capable of rematerializing these constants in PIC mode, so it can lead to
|
|
|
|
// degraded performance. Compile test/CodeGen/X86/remat-constant.ll with
|
2008-07-24 23:57:25 +00:00
|
|
|
// -relocation-model=pic to see an example of this.
|
|
|
|
static cl::opt<bool>
|
|
|
|
EnableLICMConstantMotion("enable-licm-constant-variables", cl::Hidden,
|
|
|
|
cl::desc("Enable hoisting/sinking of constant "
|
|
|
|
"global variables"));
|
|
|
|
|
2008-05-13 00:00:25 +00:00
|
|
|
namespace {
|
2007-03-07 04:41:30 +00:00
|
|
|
struct VISIBILITY_HIDDEN LICM : public LoopPass {
|
2007-05-06 13:37:16 +00:00
|
|
|
static char ID; // Pass identification, replacement for typeid
|
2008-09-04 17:05:41 +00:00
|
|
|
LICM() : LoopPass(&ID) {}
|
2007-05-01 21:15:47 +00:00
|
|
|
|
2007-03-07 04:41:30 +00:00
|
|
|
virtual bool runOnLoop(Loop *L, LPPassManager &LPM);
|
2002-05-10 22:44:58 +00:00
|
|
|
|
2002-09-26 16:52:07 +00:00
|
|
|
/// This transformation requires natural loop information & requires that
|
|
|
|
/// loop preheaders be inserted into the CFG...
|
|
|
|
///
|
2002-05-10 22:44:58 +00:00
|
|
|
virtual void getAnalysisUsage(AnalysisUsage &AU) const {
|
2002-10-21 20:00:28 +00:00
|
|
|
AU.setPreservesCFG();
|
2003-10-12 21:52:28 +00:00
|
|
|
AU.addRequiredID(LoopSimplifyID);
|
2002-08-08 19:01:30 +00:00
|
|
|
AU.addRequired<LoopInfo>();
|
2007-04-24 06:40:39 +00:00
|
|
|
AU.addRequired<DominatorTree>();
|
2003-03-03 23:32:45 +00:00
|
|
|
AU.addRequired<DominanceFrontier>(); // For scalar promotion (mem2reg)
|
2002-08-22 21:39:55 +00:00
|
|
|
AU.addRequired<AliasAnalysis>();
|
2007-07-30 20:19:59 +00:00
|
|
|
AU.addPreserved<ScalarEvolution>();
|
|
|
|
AU.addPreserved<DominanceFrontier>();
|
2002-05-10 22:44:58 +00:00
|
|
|
}
|
|
|
|
|
2007-04-17 18:21:36 +00:00
|
|
|
bool doFinalization() {
|
2007-11-25 23:52:02 +00:00
|
|
|
// Free the values stored in the map
|
|
|
|
for (std::map<Loop *, AliasSetTracker *>::iterator
|
|
|
|
I = LoopToAliasMap.begin(), E = LoopToAliasMap.end(); I != E; ++I)
|
|
|
|
delete I->second;
|
|
|
|
|
2007-03-07 04:41:30 +00:00
|
|
|
LoopToAliasMap.clear();
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2002-05-10 22:44:58 +00:00
|
|
|
private:
|
2003-12-09 17:18:00 +00:00
|
|
|
// Various analyses that we use...
|
2003-02-24 03:52:32 +00:00
|
|
|
AliasAnalysis *AA; // Current AliasAnalysis information
|
2003-12-09 17:18:00 +00:00
|
|
|
LoopInfo *LI; // Current LoopInfo
|
2007-04-24 06:40:39 +00:00
|
|
|
DominatorTree *DT; // Dominator Tree for the current Loop...
|
2003-10-05 21:20:13 +00:00
|
|
|
DominanceFrontier *DF; // Current Dominance Frontier
|
2003-12-09 17:18:00 +00:00
|
|
|
|
|
|
|
// State that is updated as we process loops
|
2003-02-24 03:52:32 +00:00
|
|
|
bool Changed; // Set to true when we change anything.
|
|
|
|
BasicBlock *Preheader; // The preheader block of the current loop...
|
|
|
|
Loop *CurLoop; // The current loop we are working on...
|
2003-03-03 23:32:45 +00:00
|
|
|
AliasSetTracker *CurAST; // AliasSet information for the current loop...
|
2007-03-07 04:41:30 +00:00
|
|
|
std::map<Loop *, AliasSetTracker *> LoopToAliasMap;
|
2002-05-10 22:44:58 +00:00
|
|
|
|
2007-07-31 08:01:41 +00:00
|
|
|
/// cloneBasicBlockAnalysis - Simple Analysis hook. Clone alias set info.
|
|
|
|
void cloneBasicBlockAnalysis(BasicBlock *From, BasicBlock *To, Loop *L);
|
|
|
|
|
|
|
|
/// deleteAnalysisValue - Simple Analysis hook. Delete value V from alias
|
|
|
|
/// set.
|
|
|
|
void deleteAnalysisValue(Value *V, Loop *L);
|
|
|
|
|
2003-12-19 07:22:45 +00:00
|
|
|
/// SinkRegion - Walk the specified region of the CFG (defined by all blocks
|
|
|
|
/// dominated by the specified block, and that are in the current loop) in
|
2007-04-24 06:40:39 +00:00
|
|
|
/// reverse depth first order w.r.t the DominatorTree. This allows us to
|
2003-12-19 07:22:45 +00:00
|
|
|
/// visit uses before definitions, allowing us to sink a loop body in one
|
|
|
|
/// pass without iteration.
|
|
|
|
///
|
2007-06-04 00:32:22 +00:00
|
|
|
void SinkRegion(DomTreeNode *N);
|
2003-12-19 07:22:45 +00:00
|
|
|
|
2002-09-29 21:46:09 +00:00
|
|
|
/// HoistRegion - Walk the specified region of the CFG (defined by all
|
|
|
|
/// blocks dominated by the specified block, and that are in the current
|
2007-04-24 06:40:39 +00:00
|
|
|
/// loop) in depth first order w.r.t the DominatorTree. This allows us to
|
2003-10-10 17:57:28 +00:00
|
|
|
/// visit definitions before uses, allowing us to hoist a loop body in one
|
2002-09-29 21:46:09 +00:00
|
|
|
/// pass without iteration.
|
|
|
|
///
|
2007-06-04 00:32:22 +00:00
|
|
|
void HoistRegion(DomTreeNode *N);
|
2002-09-29 21:46:09 +00:00
|
|
|
|
2002-09-29 22:26:07 +00:00
|
|
|
/// inSubLoop - Little predicate that returns true if the specified basic
|
|
|
|
/// block is in a subloop of the current one, not the current one itself.
|
2002-09-26 16:52:07 +00:00
|
|
|
///
|
2002-09-29 22:26:07 +00:00
|
|
|
bool inSubLoop(BasicBlock *BB) {
|
|
|
|
assert(CurLoop->contains(BB) && "Only valid if BB is IN the loop");
|
2004-01-08 00:09:44 +00:00
|
|
|
for (Loop::iterator I = CurLoop->begin(), E = CurLoop->end(); I != E; ++I)
|
|
|
|
if ((*I)->contains(BB))
|
2002-09-29 22:26:07 +00:00
|
|
|
return true; // A subloop actually contains this block!
|
|
|
|
return false;
|
2002-05-10 22:44:58 +00:00
|
|
|
}
|
|
|
|
|
2003-12-10 06:41:05 +00:00
|
|
|
/// isExitBlockDominatedByBlockInLoop - This method checks to see if the
|
|
|
|
/// specified exit block of the loop is dominated by the specified block
|
|
|
|
/// that is in the body of the loop. We use these constraints to
|
|
|
|
/// dramatically limit the amount of the dominator tree that needs to be
|
|
|
|
/// searched.
|
|
|
|
bool isExitBlockDominatedByBlockInLoop(BasicBlock *ExitBlock,
|
|
|
|
BasicBlock *BlockInLoop) const {
|
|
|
|
// If the block in the loop is the loop header, it must be dominated!
|
|
|
|
BasicBlock *LoopHeader = CurLoop->getHeader();
|
|
|
|
if (BlockInLoop == LoopHeader)
|
|
|
|
return true;
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2007-06-04 00:32:22 +00:00
|
|
|
DomTreeNode *BlockInLoopNode = DT->getNode(BlockInLoop);
|
|
|
|
DomTreeNode *IDom = DT->getNode(ExitBlock);
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2003-12-10 06:41:05 +00:00
|
|
|
// Because the exit block is not in the loop, we know we have to get _at
|
2004-05-23 21:20:19 +00:00
|
|
|
// least_ its immediate dominator.
|
2003-12-10 06:41:05 +00:00
|
|
|
do {
|
|
|
|
// Get next Immediate Dominator.
|
2007-04-24 06:40:39 +00:00
|
|
|
IDom = IDom->getIDom();
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2003-12-10 06:41:05 +00:00
|
|
|
// If we have got to the header of the loop, then the instructions block
|
|
|
|
// did not dominate the exit node, so we can't hoist it.
|
2007-04-24 06:40:39 +00:00
|
|
|
if (IDom->getBlock() == LoopHeader)
|
2003-12-10 06:41:05 +00:00
|
|
|
return false;
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2007-04-24 06:40:39 +00:00
|
|
|
} while (IDom != BlockInLoopNode);
|
2003-12-10 06:41:05 +00:00
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// sink - When an instruction is found to only be used outside of the loop,
|
|
|
|
/// this function moves it to the exit blocks and patches up SSA form as
|
|
|
|
/// needed.
|
|
|
|
///
|
|
|
|
void sink(Instruction &I);
|
|
|
|
|
2002-09-26 16:52:07 +00:00
|
|
|
/// hoist - When an instruction is found to only use loop invariant operands
|
|
|
|
/// that is safe to hoist, this instruction is called to do the dirty work.
|
|
|
|
///
|
2002-06-25 16:13:24 +00:00
|
|
|
void hoist(Instruction &I);
|
2002-05-10 22:44:58 +00:00
|
|
|
|
2003-12-10 06:41:05 +00:00
|
|
|
/// isSafeToExecuteUnconditionally - Only sink or hoist an instruction if it
|
|
|
|
/// is not a trapping instruction or if it is a trapping instruction and is
|
|
|
|
/// guaranteed to execute.
|
2003-08-05 18:45:46 +00:00
|
|
|
///
|
2003-12-10 06:41:05 +00:00
|
|
|
bool isSafeToExecuteUnconditionally(Instruction &I);
|
2003-08-05 18:45:46 +00:00
|
|
|
|
2002-09-26 16:52:07 +00:00
|
|
|
/// pointerInvalidatedByLoop - Return true if the body of this loop may
|
|
|
|
/// store into the memory location pointed to by V.
|
2005-04-21 23:48:37 +00:00
|
|
|
///
|
2004-11-26 21:20:09 +00:00
|
|
|
bool pointerInvalidatedByLoop(Value *V, unsigned Size) {
|
2003-03-03 23:32:45 +00:00
|
|
|
// Check to see if any of the basic blocks in CurLoop invalidate *V.
|
2004-11-26 21:20:09 +00:00
|
|
|
return CurAST->getAliasSetForPointer(V, Size).isMod();
|
2003-02-24 03:52:32 +00:00
|
|
|
}
|
2002-08-22 21:39:55 +00:00
|
|
|
|
2003-12-10 06:41:05 +00:00
|
|
|
bool canSinkOrHoistInst(Instruction &I);
|
|
|
|
bool isLoopInvariantInst(Instruction &I);
|
|
|
|
bool isNotUsedInLoop(Instruction &I);
|
2002-05-10 22:44:58 +00:00
|
|
|
|
2003-02-24 03:52:32 +00:00
|
|
|
/// PromoteValuesInLoop - Look at the stores in the loop and promote as many
|
|
|
|
/// to scalars as we can.
|
|
|
|
///
|
|
|
|
void PromoteValuesInLoop();
|
|
|
|
|
2004-09-15 01:04:07 +00:00
|
|
|
/// FindPromotableValuesInLoop - Check the current loop for stores to
|
2003-09-11 15:32:37 +00:00
|
|
|
/// definite pointers, which are not loaded and stored through may aliases.
|
2003-02-24 03:52:32 +00:00
|
|
|
/// If these are found, create an alloca for the value, add it to the
|
|
|
|
/// PromotedValues list, and keep track of the mapping from value to
|
|
|
|
/// alloca...
|
|
|
|
///
|
2004-09-15 01:04:07 +00:00
|
|
|
void FindPromotableValuesInLoop(
|
2003-02-24 03:52:32 +00:00
|
|
|
std::vector<std::pair<AllocaInst*, Value*> > &PromotedValues,
|
|
|
|
std::map<Value*, AllocaInst*> &Val2AlMap);
|
2002-05-10 22:44:58 +00:00
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2008-05-13 00:00:25 +00:00
|
|
|
char LICM::ID = 0;
|
|
|
|
static RegisterPass<LICM> X("licm", "Loop Invariant Code Motion");
|
|
|
|
|
2008-10-22 23:32:42 +00:00
|
|
|
Pass *llvm::createLICMPass() { return new LICM(); }
|
2002-05-10 22:44:58 +00:00
|
|
|
|
2007-07-31 16:52:25 +00:00
|
|
|
/// Hoist expressions out of the specified loop. Note, alias info for inner
|
|
|
|
/// loop is not preserved so it is not a good idea to run LICM multiple
|
|
|
|
/// times on one loop.
|
2002-09-26 16:52:07 +00:00
|
|
|
///
|
2007-03-07 04:41:30 +00:00
|
|
|
bool LICM::runOnLoop(Loop *L, LPPassManager &LPM) {
|
2003-02-24 03:52:32 +00:00
|
|
|
Changed = false;
|
2002-05-10 22:44:58 +00:00
|
|
|
|
2003-02-24 03:52:32 +00:00
|
|
|
// Get our Loop and Alias Analysis information...
|
|
|
|
LI = &getAnalysis<LoopInfo>();
|
2002-08-22 21:39:55 +00:00
|
|
|
AA = &getAnalysis<AliasAnalysis>();
|
2003-10-05 21:20:13 +00:00
|
|
|
DF = &getAnalysis<DominanceFrontier>();
|
2007-04-24 06:40:39 +00:00
|
|
|
DT = &getAnalysis<DominatorTree>();
|
2002-08-22 21:39:55 +00:00
|
|
|
|
2007-03-07 04:41:30 +00:00
|
|
|
CurAST = new AliasSetTracker(*AA);
|
2007-05-30 15:29:37 +00:00
|
|
|
// Collect Alias info from subloops
|
2007-03-07 04:41:30 +00:00
|
|
|
for (Loop::iterator LoopItr = L->begin(), LoopItrE = L->end();
|
|
|
|
LoopItr != LoopItrE; ++LoopItr) {
|
|
|
|
Loop *InnerL = *LoopItr;
|
|
|
|
AliasSetTracker *InnerAST = LoopToAliasMap[InnerL];
|
|
|
|
assert (InnerAST && "Where is my AST?");
|
2002-09-26 16:52:07 +00:00
|
|
|
|
2007-03-07 04:41:30 +00:00
|
|
|
// What if InnerLoop was modified by other passes ?
|
|
|
|
CurAST->add(*InnerAST);
|
2003-02-24 03:52:32 +00:00
|
|
|
}
|
2007-03-07 04:41:30 +00:00
|
|
|
|
2002-05-10 22:44:58 +00:00
|
|
|
CurLoop = L;
|
|
|
|
|
2002-09-26 19:40:25 +00:00
|
|
|
// Get the preheader block to move instructions into...
|
|
|
|
Preheader = L->getLoopPreheader();
|
|
|
|
assert(Preheader&&"Preheader insertion pass guarantees we have a preheader!");
|
|
|
|
|
2003-02-24 03:52:32 +00:00
|
|
|
// Loop over the body of this loop, looking for calls, invokes, and stores.
|
2003-03-03 23:32:45 +00:00
|
|
|
// Because subloops have already been incorporated into AST, we skip blocks in
|
2003-02-24 03:52:32 +00:00
|
|
|
// subloops.
|
|
|
|
//
|
2008-06-22 20:18:58 +00:00
|
|
|
for (Loop::block_iterator I = L->block_begin(), E = L->block_end();
|
|
|
|
I != E; ++I) {
|
|
|
|
BasicBlock *BB = *I;
|
|
|
|
if (LI->getLoopFor(BB) == L) // Ignore blocks in subloops...
|
|
|
|
CurAST->add(*BB); // Incorporate the specified basic block
|
|
|
|
}
|
2003-02-24 03:52:32 +00:00
|
|
|
|
2002-05-10 22:44:58 +00:00
|
|
|
// We want to visit all of the instructions in this loop... that are not parts
|
|
|
|
// of our subloops (they have already had their invariants hoisted out of
|
|
|
|
// their loop, into this loop, so there is no need to process the BODIES of
|
|
|
|
// the subloops).
|
|
|
|
//
|
2002-09-29 21:46:09 +00:00
|
|
|
// Traverse the body of the loop in depth first order on the dominator tree so
|
|
|
|
// that we are guaranteed to see definitions before we see uses. This allows
|
2007-08-18 15:08:56 +00:00
|
|
|
// us to sink instructions in one pass, without iteration. After sinking
|
2003-12-19 07:22:45 +00:00
|
|
|
// instructions, we perform another pass to hoist them out of the loop.
|
2002-09-29 21:46:09 +00:00
|
|
|
//
|
2007-04-24 06:40:39 +00:00
|
|
|
SinkRegion(DT->getNode(L->getHeader()));
|
|
|
|
HoistRegion(DT->getNode(L->getHeader()));
|
2002-05-10 22:44:58 +00:00
|
|
|
|
2003-02-24 03:52:32 +00:00
|
|
|
// Now that all loop invariants have been removed from the loop, promote any
|
|
|
|
// memory references to scalars that we can...
|
|
|
|
if (!DisablePromotion)
|
|
|
|
PromoteValuesInLoop();
|
|
|
|
|
2002-05-10 22:44:58 +00:00
|
|
|
// Clear out loops state information for the next iteration
|
|
|
|
CurLoop = 0;
|
2002-09-26 19:40:25 +00:00
|
|
|
Preheader = 0;
|
2007-03-07 04:41:30 +00:00
|
|
|
|
|
|
|
LoopToAliasMap[L] = CurAST;
|
|
|
|
return Changed;
|
2002-05-10 22:44:58 +00:00
|
|
|
}
|
|
|
|
|
2003-12-19 07:22:45 +00:00
|
|
|
/// SinkRegion - Walk the specified region of the CFG (defined by all blocks
|
|
|
|
/// dominated by the specified block, and that are in the current loop) in
|
2007-04-24 06:40:39 +00:00
|
|
|
/// reverse depth first order w.r.t the DominatorTree. This allows us to visit
|
2003-12-19 07:22:45 +00:00
|
|
|
/// uses before definitions, allowing us to sink a loop body in one pass without
|
|
|
|
/// iteration.
|
|
|
|
///
|
2007-06-04 00:32:22 +00:00
|
|
|
void LICM::SinkRegion(DomTreeNode *N) {
|
2007-04-24 06:40:39 +00:00
|
|
|
assert(N != 0 && "Null dominator tree node?");
|
|
|
|
BasicBlock *BB = N->getBlock();
|
2003-12-19 07:22:45 +00:00
|
|
|
|
|
|
|
// If this subregion is not in the top level loop at all, exit.
|
|
|
|
if (!CurLoop->contains(BB)) return;
|
|
|
|
|
|
|
|
// We are processing blocks in reverse dfo, so process children first...
|
2007-06-04 00:32:22 +00:00
|
|
|
const std::vector<DomTreeNode*> &Children = N->getChildren();
|
2003-12-19 07:22:45 +00:00
|
|
|
for (unsigned i = 0, e = Children.size(); i != e; ++i)
|
|
|
|
SinkRegion(Children[i]);
|
|
|
|
|
|
|
|
// Only need to process the contents of this block if it is not part of a
|
|
|
|
// subloop (which would already have been processed).
|
|
|
|
if (inSubLoop(BB)) return;
|
|
|
|
|
2003-12-19 08:18:16 +00:00
|
|
|
for (BasicBlock::iterator II = BB->end(); II != BB->begin(); ) {
|
|
|
|
Instruction &I = *--II;
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2003-12-19 07:22:45 +00:00
|
|
|
// Check to see if we can sink this instruction to the exit blocks
|
|
|
|
// of the loop. We can do this if the all users of the instruction are
|
|
|
|
// outside of the loop. In this case, it doesn't even matter if the
|
|
|
|
// operands of the instruction are loop invariant.
|
|
|
|
//
|
2005-03-25 00:22:36 +00:00
|
|
|
if (isNotUsedInLoop(I) && canSinkOrHoistInst(I)) {
|
2003-12-19 08:18:16 +00:00
|
|
|
++II;
|
2003-12-19 07:22:45 +00:00
|
|
|
sink(I);
|
2003-12-19 08:18:16 +00:00
|
|
|
}
|
2003-12-19 07:22:45 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2002-09-29 21:46:09 +00:00
|
|
|
/// HoistRegion - Walk the specified region of the CFG (defined by all blocks
|
|
|
|
/// dominated by the specified block, and that are in the current loop) in depth
|
2007-04-24 06:40:39 +00:00
|
|
|
/// first order w.r.t the DominatorTree. This allows us to visit definitions
|
2002-09-29 21:46:09 +00:00
|
|
|
/// before uses, allowing us to hoist a loop body in one pass without iteration.
|
|
|
|
///
|
2007-06-04 00:32:22 +00:00
|
|
|
void LICM::HoistRegion(DomTreeNode *N) {
|
2007-04-24 06:40:39 +00:00
|
|
|
assert(N != 0 && "Null dominator tree node?");
|
|
|
|
BasicBlock *BB = N->getBlock();
|
2002-09-29 21:46:09 +00:00
|
|
|
|
2002-09-29 22:26:07 +00:00
|
|
|
// If this subregion is not in the top level loop at all, exit.
|
2003-12-09 19:32:44 +00:00
|
|
|
if (!CurLoop->contains(BB)) return;
|
2002-09-29 21:46:09 +00:00
|
|
|
|
2003-12-10 06:41:05 +00:00
|
|
|
// Only need to process the contents of this block if it is not part of a
|
|
|
|
// subloop (which would already have been processed).
|
2003-12-09 19:32:44 +00:00
|
|
|
if (!inSubLoop(BB))
|
2003-12-10 06:41:05 +00:00
|
|
|
for (BasicBlock::iterator II = BB->begin(), E = BB->end(); II != E; ) {
|
|
|
|
Instruction &I = *II++;
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2003-12-19 07:22:45 +00:00
|
|
|
// Try hoisting the instruction out to the preheader. We can only do this
|
|
|
|
// if all of the operands of the instruction are loop invariant and if it
|
|
|
|
// is safe to hoist the instruction.
|
|
|
|
//
|
2005-04-21 23:48:37 +00:00
|
|
|
if (isLoopInvariantInst(I) && canSinkOrHoistInst(I) &&
|
2003-12-19 07:22:45 +00:00
|
|
|
isSafeToExecuteUnconditionally(I))
|
2006-06-26 19:10:05 +00:00
|
|
|
hoist(I);
|
2003-12-10 06:41:05 +00:00
|
|
|
}
|
2002-09-29 21:46:09 +00:00
|
|
|
|
2007-06-04 00:32:22 +00:00
|
|
|
const std::vector<DomTreeNode*> &Children = N->getChildren();
|
2002-09-29 21:46:09 +00:00
|
|
|
for (unsigned i = 0, e = Children.size(); i != e; ++i)
|
2004-06-19 20:23:35 +00:00
|
|
|
HoistRegion(Children[i]);
|
2002-09-29 21:46:09 +00:00
|
|
|
}
|
|
|
|
|
2003-12-10 06:41:05 +00:00
|
|
|
/// canSinkOrHoistInst - Return true if the hoister and sinker can handle this
|
|
|
|
/// instruction.
|
|
|
|
///
|
|
|
|
bool LICM::canSinkOrHoistInst(Instruction &I) {
|
2003-12-09 19:32:44 +00:00
|
|
|
// Loads have extra constraints we have to verify before we can hoist them.
|
|
|
|
if (LoadInst *LI = dyn_cast<LoadInst>(&I)) {
|
|
|
|
if (LI->isVolatile())
|
|
|
|
return false; // Don't hoist volatile loads!
|
|
|
|
|
2008-07-23 05:06:28 +00:00
|
|
|
// Loads from constant memory are always safe to move, even if they end up
|
|
|
|
// in the same alias set as something that ends up being modified.
|
2008-07-24 23:57:25 +00:00
|
|
|
if (EnableLICMConstantMotion &&
|
|
|
|
AA->pointsToConstantMemory(LI->getOperand(0)))
|
2008-07-23 05:06:28 +00:00
|
|
|
return true;
|
|
|
|
|
2003-12-09 19:32:44 +00:00
|
|
|
// Don't hoist loads which have may-aliased stores in loop.
|
2004-11-26 21:20:09 +00:00
|
|
|
unsigned Size = 0;
|
|
|
|
if (LI->getType()->isSized())
|
Executive summary: getTypeSize -> getTypeStoreSize / getABITypeSize.
The meaning of getTypeSize was not clear - clarifying it is important
now that we have x86 long double and arbitrary precision integers.
The issue with long double is that it requires 80 bits, and this is
not a multiple of its alignment. This gives a primitive type for
which getTypeSize differed from getABITypeSize. For arbitrary precision
integers it is even worse: there is the minimum number of bits needed to
hold the type (eg: 36 for an i36), the maximum number of bits that will
be overwriten when storing the type (40 bits for i36) and the ABI size
(i.e. the storage size rounded up to a multiple of the alignment; 64 bits
for i36).
This patch removes getTypeSize (not really - it is still there but
deprecated to allow for a gradual transition). Instead there is:
(1) getTypeSizeInBits - a number of bits that suffices to hold all
values of the type. For a primitive type, this is the minimum number
of bits. For an i36 this is 36 bits. For x86 long double it is 80.
This corresponds to gcc's TYPE_PRECISION.
(2) getTypeStoreSizeInBits - the maximum number of bits that is
written when storing the type (or read when reading it). For an
i36 this is 40 bits, for an x86 long double it is 80 bits. This
is the size alias analysis is interested in (getTypeStoreSize
returns the number of bytes). There doesn't seem to be anything
corresponding to this in gcc.
(3) getABITypeSizeInBits - this is getTypeStoreSizeInBits rounded
up to a multiple of the alignment. For an i36 this is 64, for an
x86 long double this is 96 or 128 depending on the OS. This is the
spacing between consecutive elements when you form an array out of
this type (getABITypeSize returns the number of bytes). This is
TYPE_SIZE in gcc.
Since successive elements in a SequentialType (arrays, pointers
and vectors) need to be aligned, the spacing between them will be
given by getABITypeSize. This means that the size of an array
is the length times the getABITypeSize. It also means that GEP
computations need to use getABITypeSize when computing offsets.
Furthermore, if an alloca allocates several elements at once then
these too need to be aligned, so the size of the alloca has to be
the number of elements multiplied by getABITypeSize. Logically
speaking this doesn't have to be the case when allocating just
one element, but it is simpler to also use getABITypeSize in this
case. So alloca's and mallocs should use getABITypeSize. Finally,
since gcc's only notion of size is that given by getABITypeSize, if
you want to output assembler etc the same as gcc then getABITypeSize
is the size you want.
Since a store will overwrite no more than getTypeStoreSize bytes,
and a read will read no more than that many bytes, this is the
notion of size appropriate for alias analysis calculations.
In this patch I have corrected all type size uses except some of
those in ScalarReplAggregates, lib/Codegen, lib/Target (the hard
cases). I will get around to auditing these too at some point,
but I could do with some help.
Finally, I made one change which I think wise but others might
consider pointless and suboptimal: in an unpacked struct the
amount of space allocated for a field is now given by the ABI
size rather than getTypeStoreSize. I did this because every
other place that reserves memory for a type (eg: alloca) now
uses getABITypeSize, and I didn't want to make an exception
for unpacked structs, i.e. I did it to make things more uniform.
This only effects structs containing long doubles and arbitrary
precision integers. If someone wants to pack these types more
tightly they can always use a packed struct.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@43620 91177308-0d34-0410-b5e6-96231b3b80d8
2007-11-01 20:53:16 +00:00
|
|
|
Size = AA->getTargetData().getTypeStoreSize(LI->getType());
|
2004-11-26 21:20:09 +00:00
|
|
|
return !pointerInvalidatedByLoop(LI->getOperand(0), Size);
|
2004-03-15 04:11:30 +00:00
|
|
|
} else if (CallInst *CI = dyn_cast<CallInst>(&I)) {
|
|
|
|
// Handle obvious cases efficiently.
|
2007-12-01 07:51:45 +00:00
|
|
|
AliasAnalysis::ModRefBehavior Behavior = AA->getModRefBehavior(CI);
|
|
|
|
if (Behavior == AliasAnalysis::DoesNotAccessMemory)
|
|
|
|
return true;
|
|
|
|
else if (Behavior == AliasAnalysis::OnlyReadsMemory) {
|
|
|
|
// If this call only reads from memory and there are no writes to memory
|
|
|
|
// in the loop, we can hoist or sink the call as appropriate.
|
|
|
|
bool FoundMod = false;
|
|
|
|
for (AliasSetTracker::iterator I = CurAST->begin(), E = CurAST->end();
|
|
|
|
I != E; ++I) {
|
|
|
|
AliasSet &AS = *I;
|
|
|
|
if (!AS.isForwardingAliasSet() && AS.isMod()) {
|
|
|
|
FoundMod = true;
|
|
|
|
break;
|
2004-03-15 04:11:30 +00:00
|
|
|
}
|
|
|
|
}
|
2007-12-01 07:51:45 +00:00
|
|
|
if (!FoundMod) return true;
|
2004-03-15 04:11:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// FIXME: This should use mod/ref information to see if we can hoist or sink
|
|
|
|
// the call.
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2004-03-15 04:11:30 +00:00
|
|
|
return false;
|
2003-12-09 19:32:44 +00:00
|
|
|
}
|
|
|
|
|
2006-11-27 01:05:10 +00:00
|
|
|
// Otherwise these instructions are hoistable/sinkable
|
2007-02-02 02:16:23 +00:00
|
|
|
return isa<BinaryOperator>(I) || isa<CastInst>(I) ||
|
2007-06-05 16:05:55 +00:00
|
|
|
isa<SelectInst>(I) || isa<GetElementPtrInst>(I) || isa<CmpInst>(I) ||
|
|
|
|
isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) ||
|
|
|
|
isa<ShuffleVectorInst>(I);
|
2003-12-10 06:41:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/// isNotUsedInLoop - Return true if the only users of this instruction are
|
|
|
|
/// outside of the loop. If this is true, we can sink the instruction to the
|
|
|
|
/// exit blocks of the loop.
|
|
|
|
///
|
|
|
|
bool LICM::isNotUsedInLoop(Instruction &I) {
|
2003-12-11 22:23:32 +00:00
|
|
|
for (Value::use_iterator UI = I.use_begin(), E = I.use_end(); UI != E; ++UI) {
|
|
|
|
Instruction *User = cast<Instruction>(*UI);
|
|
|
|
if (PHINode *PN = dyn_cast<PHINode>(User)) {
|
|
|
|
// PHI node uses occur in predecessor blocks!
|
|
|
|
for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
|
|
|
|
if (PN->getIncomingValue(i) == &I)
|
|
|
|
if (CurLoop->contains(PN->getIncomingBlock(i)))
|
|
|
|
return false;
|
|
|
|
} else if (CurLoop->contains(User->getParent())) {
|
2003-12-10 06:41:05 +00:00
|
|
|
return false;
|
2003-12-11 22:23:32 +00:00
|
|
|
}
|
|
|
|
}
|
2003-12-10 06:41:05 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/// isLoopInvariantInst - Return true if all operands of this instruction are
|
|
|
|
/// loop invariant. We also filter out non-hoistable instructions here just for
|
|
|
|
/// efficiency.
|
|
|
|
///
|
|
|
|
bool LICM::isLoopInvariantInst(Instruction &I) {
|
|
|
|
// The instruction is loop invariant if all of its operands are loop-invariant
|
|
|
|
for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i)
|
2004-04-18 22:46:08 +00:00
|
|
|
if (!CurLoop->isLoopInvariant(I.getOperand(i)))
|
2003-12-10 06:41:05 +00:00
|
|
|
return false;
|
|
|
|
|
2003-12-09 19:32:44 +00:00
|
|
|
// If we got this far, the instruction is loop invariant!
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2003-12-10 06:41:05 +00:00
|
|
|
/// sink - When an instruction is found to only be used outside of the loop,
|
2003-12-19 08:18:16 +00:00
|
|
|
/// this function moves it to the exit blocks and patches up SSA form as needed.
|
|
|
|
/// This method is guaranteed to remove the original instruction from its
|
|
|
|
/// position, and may either delete it or move it to outside of the loop.
|
2003-12-10 06:41:05 +00:00
|
|
|
///
|
|
|
|
void LICM::sink(Instruction &I) {
|
2006-11-26 09:46:52 +00:00
|
|
|
DOUT << "LICM sinking instruction: " << I;
|
2003-12-10 06:41:05 +00:00
|
|
|
|
2007-08-21 00:31:24 +00:00
|
|
|
SmallVector<BasicBlock*, 8> ExitBlocks;
|
2004-04-18 22:15:13 +00:00
|
|
|
CurLoop->getExitBlocks(ExitBlocks);
|
2003-12-10 20:43:29 +00:00
|
|
|
|
|
|
|
if (isa<LoadInst>(I)) ++NumMovedLoads;
|
2004-03-15 04:11:30 +00:00
|
|
|
else if (isa<CallInst>(I)) ++NumMovedCalls;
|
2003-12-10 20:43:29 +00:00
|
|
|
++NumSunk;
|
|
|
|
Changed = true;
|
|
|
|
|
2003-12-10 06:41:05 +00:00
|
|
|
// The case where there is only a single exit node of this loop is common
|
|
|
|
// enough that we handle it as a special (more efficient) case. It is more
|
|
|
|
// efficient to handle because there are no PHI nodes that need to be placed.
|
|
|
|
if (ExitBlocks.size() == 1) {
|
|
|
|
if (!isExitBlockDominatedByBlockInLoop(ExitBlocks[0], I.getParent())) {
|
|
|
|
// Instruction is not used, just delete it.
|
2004-05-23 21:20:19 +00:00
|
|
|
CurAST->deleteValue(&I);
|
2006-09-12 19:17:09 +00:00
|
|
|
if (!I.use_empty()) // If I has users in unreachable blocks, eliminate.
|
2009-07-03 00:54:20 +00:00
|
|
|
I.replaceAllUsesWith(Context->getUndef(I.getType()));
|
2006-06-26 19:10:05 +00:00
|
|
|
I.eraseFromParent();
|
2003-12-10 06:41:05 +00:00
|
|
|
} else {
|
|
|
|
// Move the instruction to the start of the exit block, after any PHI
|
|
|
|
// nodes in it.
|
2006-06-26 19:10:05 +00:00
|
|
|
I.removeFromParent();
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2008-05-23 21:05:58 +00:00
|
|
|
BasicBlock::iterator InsertPt = ExitBlocks[0]->getFirstNonPHI();
|
2003-12-10 06:41:05 +00:00
|
|
|
ExitBlocks[0]->getInstList().insert(InsertPt, &I);
|
|
|
|
}
|
2008-01-29 13:02:09 +00:00
|
|
|
} else if (ExitBlocks.empty()) {
|
2003-12-10 06:41:05 +00:00
|
|
|
// The instruction is actually dead if there ARE NO exit blocks.
|
2004-05-23 21:20:19 +00:00
|
|
|
CurAST->deleteValue(&I);
|
2006-09-12 19:17:09 +00:00
|
|
|
if (!I.use_empty()) // If I has users in unreachable blocks, eliminate.
|
2009-07-03 00:54:20 +00:00
|
|
|
I.replaceAllUsesWith(Context->getUndef(I.getType()));
|
2006-06-26 19:10:05 +00:00
|
|
|
I.eraseFromParent();
|
2003-12-10 06:41:05 +00:00
|
|
|
} else {
|
|
|
|
// Otherwise, if we have multiple exits, use the PromoteMem2Reg function to
|
|
|
|
// do all of the hard work of inserting PHI nodes as necessary. We convert
|
|
|
|
// the value into a stack object to get it to do this.
|
|
|
|
|
|
|
|
// Firstly, we create a stack object to hold the value...
|
2004-07-27 07:38:32 +00:00
|
|
|
AllocaInst *AI = 0;
|
2003-12-10 06:41:05 +00:00
|
|
|
|
2007-06-01 22:15:31 +00:00
|
|
|
if (I.getType() != Type::VoidTy) {
|
2009-07-15 23:53:25 +00:00
|
|
|
AI = new AllocaInst(I.getType(), 0, I.getName(),
|
2007-03-22 16:38:57 +00:00
|
|
|
I.getParent()->getParent()->getEntryBlock().begin());
|
2007-06-01 22:15:31 +00:00
|
|
|
CurAST->add(AI);
|
|
|
|
}
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2003-12-10 06:41:05 +00:00
|
|
|
// Secondly, insert load instructions for each use of the instruction
|
|
|
|
// outside of the loop.
|
|
|
|
while (!I.use_empty()) {
|
|
|
|
Instruction *U = cast<Instruction>(I.use_back());
|
|
|
|
|
|
|
|
// If the user is a PHI Node, we actually have to insert load instructions
|
|
|
|
// in all predecessor blocks, not in the PHI block itself!
|
|
|
|
if (PHINode *UPN = dyn_cast<PHINode>(U)) {
|
|
|
|
// Only insert into each predecessor once, so that we don't have
|
|
|
|
// different incoming values from the same block!
|
|
|
|
std::map<BasicBlock*, Value*> InsertedBlocks;
|
|
|
|
for (unsigned i = 0, e = UPN->getNumIncomingValues(); i != e; ++i)
|
|
|
|
if (UPN->getIncomingValue(i) == &I) {
|
|
|
|
BasicBlock *Pred = UPN->getIncomingBlock(i);
|
|
|
|
Value *&PredVal = InsertedBlocks[Pred];
|
|
|
|
if (!PredVal) {
|
|
|
|
// Insert a new load instruction right before the terminator in
|
|
|
|
// the predecessor block.
|
|
|
|
PredVal = new LoadInst(AI, "", Pred->getTerminator());
|
2007-06-01 22:15:31 +00:00
|
|
|
CurAST->add(cast<LoadInst>(PredVal));
|
2003-12-10 06:41:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
UPN->setIncomingValue(i, PredVal);
|
|
|
|
}
|
|
|
|
|
|
|
|
} else {
|
|
|
|
LoadInst *L = new LoadInst(AI, "", U);
|
|
|
|
U->replaceUsesOfWith(&I, L);
|
2007-06-01 22:15:31 +00:00
|
|
|
CurAST->add(L);
|
2003-12-10 06:41:05 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Thirdly, insert a copy of the instruction in each exit block of the loop
|
|
|
|
// that is dominated by the instruction, storing the result into the memory
|
|
|
|
// location. Be careful not to insert the instruction into any particular
|
|
|
|
// basic block more than once.
|
|
|
|
std::set<BasicBlock*> InsertedBlocks;
|
|
|
|
BasicBlock *InstOrigBB = I.getParent();
|
|
|
|
|
|
|
|
for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
|
|
|
|
BasicBlock *ExitBlock = ExitBlocks[i];
|
|
|
|
|
|
|
|
if (isExitBlockDominatedByBlockInLoop(ExitBlock, InstOrigBB)) {
|
|
|
|
// If we haven't already processed this exit block, do so now.
|
2003-12-10 16:58:24 +00:00
|
|
|
if (InsertedBlocks.insert(ExitBlock).second) {
|
2003-12-10 06:41:05 +00:00
|
|
|
// Insert the code after the last PHI node...
|
2008-05-23 21:05:58 +00:00
|
|
|
BasicBlock::iterator InsertPt = ExitBlock->getFirstNonPHI();
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2003-12-10 06:41:05 +00:00
|
|
|
// If this is the first exit block processed, just move the original
|
|
|
|
// instruction, otherwise clone the original instruction and insert
|
|
|
|
// the copy.
|
|
|
|
Instruction *New;
|
2003-12-10 22:35:56 +00:00
|
|
|
if (InsertedBlocks.size() == 1) {
|
2006-06-26 19:10:05 +00:00
|
|
|
I.removeFromParent();
|
2003-12-10 06:41:05 +00:00
|
|
|
ExitBlock->getInstList().insert(InsertPt, &I);
|
|
|
|
New = &I;
|
|
|
|
} else {
|
2009-07-09 23:48:35 +00:00
|
|
|
New = I.clone(*Context);
|
2005-03-25 00:22:36 +00:00
|
|
|
CurAST->copyValue(&I, New);
|
2004-07-27 07:38:32 +00:00
|
|
|
if (!I.getName().empty())
|
|
|
|
New->setName(I.getName()+".le");
|
2003-12-10 06:41:05 +00:00
|
|
|
ExitBlock->getInstList().insert(InsertPt, New);
|
|
|
|
}
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2003-12-10 06:41:05 +00:00
|
|
|
// Now that we have inserted the instruction, store it into the alloca
|
2004-07-27 07:38:32 +00:00
|
|
|
if (AI) new StoreInst(New, AI, InsertPt);
|
2003-12-10 06:41:05 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2003-12-19 08:18:16 +00:00
|
|
|
|
|
|
|
// If the instruction doesn't dominate any exit blocks, it must be dead.
|
|
|
|
if (InsertedBlocks.empty()) {
|
2004-05-23 21:20:19 +00:00
|
|
|
CurAST->deleteValue(&I);
|
2006-06-26 19:10:05 +00:00
|
|
|
I.eraseFromParent();
|
2003-12-19 08:18:16 +00:00
|
|
|
}
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2003-12-10 06:41:05 +00:00
|
|
|
// Finally, promote the fine value to SSA form.
|
2004-07-27 07:38:32 +00:00
|
|
|
if (AI) {
|
|
|
|
std::vector<AllocaInst*> Allocas;
|
|
|
|
Allocas.push_back(AI);
|
2009-07-05 22:41:43 +00:00
|
|
|
PromoteMemToReg(Allocas, *DT, *DF, Context, CurAST);
|
2004-07-27 07:38:32 +00:00
|
|
|
}
|
2003-12-10 06:41:05 +00:00
|
|
|
}
|
|
|
|
}
|
2002-09-29 21:46:09 +00:00
|
|
|
|
2002-09-26 16:52:07 +00:00
|
|
|
/// hoist - When an instruction is found to only use loop invariant operands
|
|
|
|
/// that is safe to hoist, this instruction is called to do the dirty work.
|
|
|
|
///
|
2003-12-10 06:41:05 +00:00
|
|
|
void LICM::hoist(Instruction &I) {
|
2006-11-26 09:46:52 +00:00
|
|
|
DOUT << "LICM hoisting to " << Preheader->getName() << ": " << I;
|
2003-12-09 19:32:44 +00:00
|
|
|
|
2002-09-26 19:40:25 +00:00
|
|
|
// Remove the instruction from its current basic block... but don't delete the
|
|
|
|
// instruction.
|
2006-06-26 19:10:05 +00:00
|
|
|
I.removeFromParent();
|
2002-05-10 22:44:58 +00:00
|
|
|
|
2002-09-26 16:38:03 +00:00
|
|
|
// Insert the new node in Preheader, before the terminator.
|
2003-12-10 06:41:05 +00:00
|
|
|
Preheader->getInstList().insert(Preheader->getTerminator(), &I);
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2003-12-10 06:41:05 +00:00
|
|
|
if (isa<LoadInst>(I)) ++NumMovedLoads;
|
2004-03-15 04:11:30 +00:00
|
|
|
else if (isa<CallInst>(I)) ++NumMovedCalls;
|
2002-09-26 16:38:03 +00:00
|
|
|
++NumHoisted;
|
2002-05-10 22:44:58 +00:00
|
|
|
Changed = true;
|
|
|
|
}
|
|
|
|
|
2003-12-10 06:41:05 +00:00
|
|
|
/// isSafeToExecuteUnconditionally - Only sink or hoist an instruction if it is
|
|
|
|
/// not a trapping instruction or if it is a trapping instruction and is
|
|
|
|
/// guaranteed to execute.
|
2003-08-05 18:45:46 +00:00
|
|
|
///
|
2003-12-10 06:41:05 +00:00
|
|
|
bool LICM::isSafeToExecuteUnconditionally(Instruction &Inst) {
|
2003-12-09 17:18:00 +00:00
|
|
|
// If it is not a trapping instruction, it is always safe to hoist.
|
|
|
|
if (!Inst.isTrapping()) return true;
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2003-12-09 17:18:00 +00:00
|
|
|
// Otherwise we have to check to make sure that the instruction dominates all
|
|
|
|
// of the exit blocks. If it doesn't, then there is a path out of the loop
|
|
|
|
// which does not execute this instruction, so we can't hoist it.
|
|
|
|
|
|
|
|
// If the instruction is in the header block for the loop (which is very
|
|
|
|
// common), it is always guaranteed to dominate the exit blocks. Since this
|
|
|
|
// is a common case, and can save some work, check it now.
|
2003-12-10 06:41:05 +00:00
|
|
|
if (Inst.getParent() == CurLoop->getHeader())
|
2003-12-09 17:18:00 +00:00
|
|
|
return true;
|
|
|
|
|
2004-11-29 21:26:12 +00:00
|
|
|
// It's always safe to load from a global or alloca.
|
|
|
|
if (isa<LoadInst>(Inst))
|
|
|
|
if (isa<AllocationInst>(Inst.getOperand(0)) ||
|
|
|
|
isa<GlobalVariable>(Inst.getOperand(0)))
|
|
|
|
return true;
|
|
|
|
|
2003-12-09 17:18:00 +00:00
|
|
|
// Get the exit blocks for the current loop.
|
2007-08-21 00:31:24 +00:00
|
|
|
SmallVector<BasicBlock*, 8> ExitBlocks;
|
2004-04-18 22:15:13 +00:00
|
|
|
CurLoop->getExitBlocks(ExitBlocks);
|
2003-08-05 18:45:46 +00:00
|
|
|
|
2007-04-24 06:40:39 +00:00
|
|
|
// For each exit block, get the DT node and walk up the DT until the
|
2003-12-09 17:18:00 +00:00
|
|
|
// instruction's basic block is found or we exit the loop.
|
2003-12-10 06:41:05 +00:00
|
|
|
for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i)
|
|
|
|
if (!isExitBlockDominatedByBlockInLoop(ExitBlocks[i], Inst.getParent()))
|
|
|
|
return false;
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2003-08-05 18:45:46 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2002-09-26 16:19:31 +00:00
|
|
|
|
2003-02-24 03:52:32 +00:00
|
|
|
/// PromoteValuesInLoop - Try to promote memory values to scalars by sinking
|
|
|
|
/// stores out of the loop and moving loads to before the loop. We do this by
|
|
|
|
/// looping over the stores in the loop, looking for stores to Must pointers
|
|
|
|
/// which are loop invariant. We promote these memory locations to use allocas
|
|
|
|
/// instead. These allocas can easily be raised to register values by the
|
|
|
|
/// PromoteMem2Reg functionality.
|
|
|
|
///
|
|
|
|
void LICM::PromoteValuesInLoop() {
|
|
|
|
// PromotedValues - List of values that are promoted out of the loop. Each
|
2003-09-10 05:29:43 +00:00
|
|
|
// value has an alloca instruction for it, and a canonical version of the
|
2003-02-24 03:52:32 +00:00
|
|
|
// pointer.
|
|
|
|
std::vector<std::pair<AllocaInst*, Value*> > PromotedValues;
|
|
|
|
std::map<Value*, AllocaInst*> ValueToAllocaMap; // Map of ptr to alloca
|
|
|
|
|
2004-09-15 01:04:07 +00:00
|
|
|
FindPromotableValuesInLoop(PromotedValues, ValueToAllocaMap);
|
|
|
|
if (ValueToAllocaMap.empty()) return; // If there are values to promote.
|
2003-02-24 03:52:32 +00:00
|
|
|
|
|
|
|
Changed = true;
|
|
|
|
NumPromoted += PromotedValues.size();
|
|
|
|
|
2004-09-15 01:04:07 +00:00
|
|
|
std::vector<Value*> PointerValueNumbers;
|
|
|
|
|
2003-02-24 03:52:32 +00:00
|
|
|
// Emit a copy from the value into the alloca'd value in the loop preheader
|
|
|
|
TerminatorInst *LoopPredInst = Preheader->getTerminator();
|
|
|
|
for (unsigned i = 0, e = PromotedValues.size(); i != e; ++i) {
|
2004-09-15 01:04:07 +00:00
|
|
|
Value *Ptr = PromotedValues[i].second;
|
|
|
|
|
|
|
|
// If we are promoting a pointer value, update alias information for the
|
|
|
|
// inserted load.
|
|
|
|
Value *LoadValue = 0;
|
|
|
|
if (isa<PointerType>(cast<PointerType>(Ptr->getType())->getElementType())) {
|
|
|
|
// Locate a load or store through the pointer, and assign the same value
|
|
|
|
// to LI as we are loading or storing. Since we know that the value is
|
|
|
|
// stored in this loop, this will always succeed.
|
|
|
|
for (Value::use_iterator UI = Ptr->use_begin(), E = Ptr->use_end();
|
|
|
|
UI != E; ++UI)
|
|
|
|
if (LoadInst *LI = dyn_cast<LoadInst>(*UI)) {
|
|
|
|
LoadValue = LI;
|
|
|
|
break;
|
|
|
|
} else if (StoreInst *SI = dyn_cast<StoreInst>(*UI)) {
|
2004-09-15 02:34:40 +00:00
|
|
|
if (SI->getOperand(1) == Ptr) {
|
2004-09-15 01:04:07 +00:00
|
|
|
LoadValue = SI->getOperand(0);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
assert(LoadValue && "No store through the pointer found!");
|
|
|
|
PointerValueNumbers.push_back(LoadValue); // Remember this for later.
|
|
|
|
}
|
|
|
|
|
|
|
|
// Load from the memory we are promoting.
|
|
|
|
LoadInst *LI = new LoadInst(Ptr, Ptr->getName()+".promoted", LoopPredInst);
|
|
|
|
|
|
|
|
if (LoadValue) CurAST->copyValue(LoadValue, LI);
|
|
|
|
|
|
|
|
// Store into the temporary alloca.
|
2003-02-24 03:52:32 +00:00
|
|
|
new StoreInst(LI, PromotedValues[i].first, LoopPredInst);
|
|
|
|
}
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2003-02-24 03:52:32 +00:00
|
|
|
// Scan the basic blocks in the loop, replacing uses of our pointers with
|
2003-12-10 15:56:24 +00:00
|
|
|
// uses of the allocas in question.
|
2003-02-24 03:52:32 +00:00
|
|
|
//
|
2008-06-22 20:18:58 +00:00
|
|
|
for (Loop::block_iterator I = CurLoop->block_begin(),
|
|
|
|
E = CurLoop->block_end(); I != E; ++I) {
|
|
|
|
BasicBlock *BB = *I;
|
2003-02-24 03:52:32 +00:00
|
|
|
// Rewrite all loads and stores in the block of the pointer...
|
2008-06-22 20:18:58 +00:00
|
|
|
for (BasicBlock::iterator II = BB->begin(), E = BB->end(); II != E; ++II) {
|
2003-04-23 16:37:45 +00:00
|
|
|
if (LoadInst *L = dyn_cast<LoadInst>(II)) {
|
2003-02-24 03:52:32 +00:00
|
|
|
std::map<Value*, AllocaInst*>::iterator
|
|
|
|
I = ValueToAllocaMap.find(L->getOperand(0));
|
|
|
|
if (I != ValueToAllocaMap.end())
|
|
|
|
L->setOperand(0, I->second); // Rewrite load instruction...
|
2003-04-23 16:37:45 +00:00
|
|
|
} else if (StoreInst *S = dyn_cast<StoreInst>(II)) {
|
2003-02-24 03:52:32 +00:00
|
|
|
std::map<Value*, AllocaInst*>::iterator
|
|
|
|
I = ValueToAllocaMap.find(S->getOperand(1));
|
|
|
|
if (I != ValueToAllocaMap.end())
|
|
|
|
S->setOperand(1, I->second); // Rewrite store instruction...
|
|
|
|
}
|
|
|
|
}
|
2003-12-10 15:56:24 +00:00
|
|
|
}
|
2003-02-24 03:52:32 +00:00
|
|
|
|
2003-12-10 15:56:24 +00:00
|
|
|
// Now that the body of the loop uses the allocas instead of the original
|
|
|
|
// memory locations, insert code to copy the alloca value back into the
|
|
|
|
// original memory location on all exits from the loop. Note that we only
|
|
|
|
// want to insert one copy of the code in each exit block, though the loop may
|
|
|
|
// exit to the same block more than once.
|
|
|
|
//
|
2008-05-22 03:22:42 +00:00
|
|
|
SmallPtrSet<BasicBlock*, 16> ProcessedBlocks;
|
2003-12-10 15:56:24 +00:00
|
|
|
|
2007-08-21 00:31:24 +00:00
|
|
|
SmallVector<BasicBlock*, 8> ExitBlocks;
|
2004-04-18 22:15:13 +00:00
|
|
|
CurLoop->getExitBlocks(ExitBlocks);
|
2008-05-22 03:22:42 +00:00
|
|
|
for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
|
|
|
|
if (!ProcessedBlocks.insert(ExitBlocks[i]))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
// Copy all of the allocas into their memory locations.
|
2008-05-23 21:05:58 +00:00
|
|
|
BasicBlock::iterator BI = ExitBlocks[i]->getFirstNonPHI();
|
2008-05-22 03:22:42 +00:00
|
|
|
Instruction *InsertPos = BI;
|
|
|
|
unsigned PVN = 0;
|
|
|
|
for (unsigned i = 0, e = PromotedValues.size(); i != e; ++i) {
|
|
|
|
// Load from the alloca.
|
|
|
|
LoadInst *LI = new LoadInst(PromotedValues[i].first, "", InsertPos);
|
|
|
|
|
|
|
|
// If this is a pointer type, update alias info appropriately.
|
|
|
|
if (isa<PointerType>(LI->getType()))
|
|
|
|
CurAST->copyValue(PointerValueNumbers[PVN++], LI);
|
|
|
|
|
|
|
|
// Store into the memory we promoted.
|
|
|
|
new StoreInst(LI, PromotedValues[i].second, InsertPos);
|
2003-12-10 15:56:24 +00:00
|
|
|
}
|
2008-05-22 03:22:42 +00:00
|
|
|
}
|
2003-02-24 03:52:32 +00:00
|
|
|
|
|
|
|
// Now that we have done the deed, use the mem2reg functionality to promote
|
2004-09-15 01:04:07 +00:00
|
|
|
// all of the new allocas we just created into real SSA registers.
|
2003-02-24 03:52:32 +00:00
|
|
|
//
|
|
|
|
std::vector<AllocaInst*> PromotedAllocas;
|
|
|
|
PromotedAllocas.reserve(PromotedValues.size());
|
|
|
|
for (unsigned i = 0, e = PromotedValues.size(); i != e; ++i)
|
|
|
|
PromotedAllocas.push_back(PromotedValues[i].first);
|
2009-07-05 22:41:43 +00:00
|
|
|
PromoteMemToReg(PromotedAllocas, *DT, *DF, Context, CurAST);
|
2003-02-24 03:52:32 +00:00
|
|
|
}
|
|
|
|
|
2004-09-15 01:04:07 +00:00
|
|
|
/// FindPromotableValuesInLoop - Check the current loop for stores to definite
|
2007-09-19 20:18:51 +00:00
|
|
|
/// pointers, which are not loaded and stored through may aliases and are safe
|
|
|
|
/// for promotion. If these are found, create an alloca for the value, add it
|
|
|
|
/// to the PromotedValues list, and keep track of the mapping from value to
|
|
|
|
/// alloca.
|
2004-09-15 01:04:07 +00:00
|
|
|
void LICM::FindPromotableValuesInLoop(
|
2003-02-24 03:52:32 +00:00
|
|
|
std::vector<std::pair<AllocaInst*, Value*> > &PromotedValues,
|
|
|
|
std::map<Value*, AllocaInst*> &ValueToAllocaMap) {
|
|
|
|
Instruction *FnStart = CurLoop->getHeader()->getParent()->begin()->begin();
|
|
|
|
|
2004-09-15 01:04:07 +00:00
|
|
|
// Loop over all of the alias sets in the tracker object.
|
2003-03-03 23:32:45 +00:00
|
|
|
for (AliasSetTracker::iterator I = CurAST->begin(), E = CurAST->end();
|
|
|
|
I != E; ++I) {
|
|
|
|
AliasSet &AS = *I;
|
|
|
|
// We can promote this alias set if it has a store, if it is a "Must" alias
|
2004-09-15 01:04:07 +00:00
|
|
|
// set, if the pointer is loop invariant, and if we are not eliminating any
|
2004-03-15 04:11:30 +00:00
|
|
|
// volatile loads or stores.
|
2008-05-22 00:53:38 +00:00
|
|
|
if (AS.isForwardingAliasSet() || !AS.isMod() || !AS.isMustAlias() ||
|
2009-03-09 05:11:09 +00:00
|
|
|
AS.isVolatile() || !CurLoop->isLoopInvariant(AS.begin()->getValue()))
|
2008-05-22 00:53:38 +00:00
|
|
|
continue;
|
|
|
|
|
|
|
|
assert(!AS.empty() &&
|
|
|
|
"Must alias set should have at least one pointer element in it!");
|
2009-03-09 05:11:09 +00:00
|
|
|
Value *V = AS.begin()->getValue();
|
2008-05-22 00:53:38 +00:00
|
|
|
|
|
|
|
// Check that all of the pointers in the alias set have the same type. We
|
|
|
|
// cannot (yet) promote a memory location that is loaded and stored in
|
|
|
|
// different sizes.
|
|
|
|
{
|
2003-03-03 23:32:45 +00:00
|
|
|
bool PointerOk = true;
|
|
|
|
for (AliasSet::iterator I = AS.begin(), E = AS.end(); I != E; ++I)
|
2009-03-09 05:11:09 +00:00
|
|
|
if (V->getType() != I->getValue()->getType()) {
|
2003-03-03 23:32:45 +00:00
|
|
|
PointerOk = false;
|
|
|
|
break;
|
2003-02-24 03:52:32 +00:00
|
|
|
}
|
2008-05-22 00:53:38 +00:00
|
|
|
if (!PointerOk)
|
|
|
|
continue;
|
|
|
|
}
|
2003-03-03 23:32:45 +00:00
|
|
|
|
2008-05-22 03:22:42 +00:00
|
|
|
// It isn't safe to promote a load/store from the loop if the load/store is
|
|
|
|
// conditional. For example, turning:
|
|
|
|
//
|
|
|
|
// for () { if (c) *P += 1; }
|
|
|
|
//
|
|
|
|
// into:
|
|
|
|
//
|
|
|
|
// tmp = *P; for () { if (c) tmp +=1; } *P = tmp;
|
|
|
|
//
|
|
|
|
// is not safe, because *P may only be valid to access if 'c' is true.
|
|
|
|
//
|
|
|
|
// It is safe to promote P if all uses are direct load/stores and if at
|
|
|
|
// least one is guaranteed to be executed.
|
|
|
|
bool GuaranteedToExecute = false;
|
|
|
|
bool InvalidInst = false;
|
|
|
|
for (Value::use_iterator UI = V->use_begin(), UE = V->use_end();
|
|
|
|
UI != UE; ++UI) {
|
|
|
|
// Ignore instructions not in this loop.
|
2008-05-22 00:53:38 +00:00
|
|
|
Instruction *Use = dyn_cast<Instruction>(*UI);
|
|
|
|
if (!Use || !CurLoop->contains(Use->getParent()))
|
|
|
|
continue;
|
2007-10-01 18:12:58 +00:00
|
|
|
|
2008-05-22 03:22:42 +00:00
|
|
|
if (!isa<LoadInst>(Use) && !isa<StoreInst>(Use)) {
|
|
|
|
InvalidInst = true;
|
2008-05-22 00:53:38 +00:00
|
|
|
break;
|
2008-05-22 03:22:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!GuaranteedToExecute)
|
|
|
|
GuaranteedToExecute = isSafeToExecuteUnconditionally(*Use);
|
2008-05-22 00:53:38 +00:00
|
|
|
}
|
2007-10-01 18:12:58 +00:00
|
|
|
|
2008-05-22 03:22:42 +00:00
|
|
|
// If there is an non-load/store instruction in the loop, we can't promote
|
|
|
|
// it. If there isn't a guaranteed-to-execute instruction, we can't
|
|
|
|
// promote.
|
|
|
|
if (InvalidInst || !GuaranteedToExecute)
|
2008-05-22 00:53:38 +00:00
|
|
|
continue;
|
|
|
|
|
|
|
|
const Type *Ty = cast<PointerType>(V->getType())->getElementType();
|
2009-07-15 23:53:25 +00:00
|
|
|
AllocaInst *AI = new AllocaInst(Ty, 0, V->getName()+".tmp", FnStart);
|
2008-05-22 00:53:38 +00:00
|
|
|
PromotedValues.push_back(std::make_pair(AI, V));
|
2004-09-15 01:04:07 +00:00
|
|
|
|
2008-05-22 00:53:38 +00:00
|
|
|
// Update the AST and alias analysis.
|
|
|
|
CurAST->copyValue(V, AI);
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2008-05-22 00:53:38 +00:00
|
|
|
for (AliasSet::iterator I = AS.begin(), E = AS.end(); I != E; ++I)
|
2009-03-09 05:11:09 +00:00
|
|
|
ValueToAllocaMap.insert(std::make_pair(I->getValue(), AI));
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2008-05-22 00:53:38 +00:00
|
|
|
DOUT << "LICM: Promoting value: " << *V << "\n";
|
2003-02-24 03:52:32 +00:00
|
|
|
}
|
2002-08-22 21:39:55 +00:00
|
|
|
}
|
2007-07-31 08:01:41 +00:00
|
|
|
|
|
|
|
/// cloneBasicBlockAnalysis - Simple Analysis hook. Clone alias set info.
|
|
|
|
void LICM::cloneBasicBlockAnalysis(BasicBlock *From, BasicBlock *To, Loop *L) {
|
|
|
|
AliasSetTracker *AST = LoopToAliasMap[L];
|
|
|
|
if (!AST)
|
|
|
|
return;
|
|
|
|
|
|
|
|
AST->copyValue(From, To);
|
|
|
|
}
|
|
|
|
|
|
|
|
/// deleteAnalysisValue - Simple Analysis hook. Delete value V from alias
|
|
|
|
/// set.
|
|
|
|
void LICM::deleteAnalysisValue(Value *V, Loop *L) {
|
|
|
|
AliasSetTracker *AST = LoopToAliasMap[L];
|
|
|
|
if (!AST)
|
|
|
|
return;
|
|
|
|
|
|
|
|
AST->deleteValue(V);
|
|
|
|
}
|