2004-10-18 21:08:22 +00:00
|
|
|
//===- LoopStrengthReduce.cpp - Strength Reduce GEPs in Loops -------------===//
|
|
|
|
//
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
|
|
|
// This file was developed by Nate Begeman and is distributed under the
|
|
|
|
// University of Illinois Open Source License. See LICENSE.TXT for details.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
// This pass performs a strength reduction on array references inside loops that
|
|
|
|
// have as one or more of their components the loop induction variable. This is
|
|
|
|
// accomplished by creating a new Value to hold the initial value of the array
|
|
|
|
// access for the first iteration, and then creating a new GEP instruction in
|
|
|
|
// the loop to increment the value by the appropriate amount.
|
|
|
|
//
|
|
|
|
// There are currently several deficiencies in the implementation, marked with
|
|
|
|
// FIXME in the code.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
#include "llvm/Transforms/Scalar.h"
|
|
|
|
#include "llvm/Constants.h"
|
|
|
|
#include "llvm/Instructions.h"
|
|
|
|
#include "llvm/Type.h"
|
2005-03-04 04:04:26 +00:00
|
|
|
#include "llvm/DerivedTypes.h"
|
2004-10-18 21:08:22 +00:00
|
|
|
#include "llvm/Analysis/Dominators.h"
|
|
|
|
#include "llvm/Analysis/LoopInfo.h"
|
|
|
|
#include "llvm/Support/CFG.h"
|
|
|
|
#include "llvm/Transforms/Utils/Local.h"
|
2005-03-04 04:04:26 +00:00
|
|
|
#include "llvm/Target/TargetData.h"
|
2004-10-18 21:08:22 +00:00
|
|
|
#include "llvm/ADT/Statistic.h"
|
|
|
|
#include <set>
|
|
|
|
using namespace llvm;
|
|
|
|
|
|
|
|
namespace {
|
|
|
|
Statistic<> NumReduced ("loop-reduce", "Number of GEPs strength reduced");
|
|
|
|
|
2005-03-06 21:58:22 +00:00
|
|
|
class GEPCache {
|
2005-03-05 22:40:34 +00:00
|
|
|
public:
|
|
|
|
GEPCache() : CachedPHINode(0), Map() {}
|
|
|
|
|
2005-03-06 21:58:22 +00:00
|
|
|
GEPCache *get(Value *v) {
|
2005-03-05 22:40:34 +00:00
|
|
|
std::map<Value *, GEPCache>::iterator I = Map.find(v);
|
|
|
|
if (I == Map.end())
|
|
|
|
I = Map.insert(std::pair<Value *, GEPCache>(v, GEPCache())).first;
|
2005-03-06 21:58:22 +00:00
|
|
|
return &I->second;
|
2005-03-05 22:40:34 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
PHINode *CachedPHINode;
|
|
|
|
std::map<Value *, GEPCache> Map;
|
|
|
|
};
|
|
|
|
|
2004-10-18 21:08:22 +00:00
|
|
|
class LoopStrengthReduce : public FunctionPass {
|
|
|
|
LoopInfo *LI;
|
|
|
|
DominatorSet *DS;
|
|
|
|
bool Changed;
|
2005-03-04 04:04:26 +00:00
|
|
|
unsigned MaxTargetAMSize;
|
2004-10-18 21:08:22 +00:00
|
|
|
public:
|
2005-03-04 04:04:26 +00:00
|
|
|
LoopStrengthReduce(unsigned MTAMS = 1)
|
|
|
|
: MaxTargetAMSize(MTAMS) {
|
|
|
|
}
|
|
|
|
|
2004-10-18 21:08:22 +00:00
|
|
|
virtual bool runOnFunction(Function &) {
|
|
|
|
LI = &getAnalysis<LoopInfo>();
|
|
|
|
DS = &getAnalysis<DominatorSet>();
|
|
|
|
Changed = false;
|
|
|
|
|
|
|
|
for (LoopInfo::iterator I = LI->begin(), E = LI->end(); I != E; ++I)
|
|
|
|
runOnLoop(*I);
|
|
|
|
return Changed;
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual void getAnalysisUsage(AnalysisUsage &AU) const {
|
|
|
|
AU.setPreservesCFG();
|
2005-02-27 19:37:07 +00:00
|
|
|
AU.addRequiredID(LoopSimplifyID);
|
2004-10-18 21:08:22 +00:00
|
|
|
AU.addRequired<LoopInfo>();
|
|
|
|
AU.addRequired<DominatorSet>();
|
2005-03-04 04:04:26 +00:00
|
|
|
AU.addRequired<TargetData>();
|
2004-10-18 21:08:22 +00:00
|
|
|
}
|
|
|
|
private:
|
|
|
|
void runOnLoop(Loop *L);
|
|
|
|
void strengthReduceGEP(GetElementPtrInst *GEPI, Loop *L,
|
2005-03-05 22:40:34 +00:00
|
|
|
GEPCache* GEPCache,
|
2004-10-18 21:08:22 +00:00
|
|
|
Instruction *InsertBefore,
|
|
|
|
std::set<Instruction*> &DeadInsts);
|
|
|
|
void DeleteTriviallyDeadInstructions(std::set<Instruction*> &Insts);
|
|
|
|
};
|
|
|
|
RegisterOpt<LoopStrengthReduce> X("loop-reduce",
|
|
|
|
"Strength Reduce GEP Uses of Ind. Vars");
|
|
|
|
}
|
|
|
|
|
2005-03-04 04:04:26 +00:00
|
|
|
FunctionPass *llvm::createLoopStrengthReducePass(unsigned MaxTargetAMSize) {
|
|
|
|
return new LoopStrengthReduce(MaxTargetAMSize);
|
2004-10-18 21:08:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/// DeleteTriviallyDeadInstructions - If any of the instructions is the
|
|
|
|
/// specified set are trivially dead, delete them and see if this makes any of
|
|
|
|
/// their operands subsequently dead.
|
|
|
|
void LoopStrengthReduce::
|
|
|
|
DeleteTriviallyDeadInstructions(std::set<Instruction*> &Insts) {
|
|
|
|
while (!Insts.empty()) {
|
|
|
|
Instruction *I = *Insts.begin();
|
|
|
|
Insts.erase(Insts.begin());
|
|
|
|
if (isInstructionTriviallyDead(I)) {
|
2005-03-01 03:46:11 +00:00
|
|
|
for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i)
|
|
|
|
if (Instruction *U = dyn_cast<Instruction>(I->getOperand(i)))
|
|
|
|
Insts.insert(U);
|
2004-10-18 21:08:22 +00:00
|
|
|
I->getParent()->getInstList().erase(I);
|
|
|
|
Changed = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void LoopStrengthReduce::strengthReduceGEP(GetElementPtrInst *GEPI, Loop *L,
|
2005-03-05 22:40:34 +00:00
|
|
|
GEPCache *Cache,
|
2004-10-18 21:08:22 +00:00
|
|
|
Instruction *InsertBefore,
|
|
|
|
std::set<Instruction*> &DeadInsts) {
|
|
|
|
// We will strength reduce the GEP by splitting it into two parts. The first
|
|
|
|
// is a GEP to hold the initial value of the non-strength-reduced GEP upon
|
|
|
|
// entering the loop, which we will insert at the end of the loop preheader.
|
|
|
|
// The second is a GEP to hold the incremented value of the initial GEP.
|
|
|
|
// The LoopIndVarSimplify pass guarantees that loop counts start at zero, so
|
|
|
|
// we will replace the indvar with a constant zero value to create the first
|
|
|
|
// GEP.
|
|
|
|
//
|
|
|
|
// We currently only handle GEP instructions that consist of zero or more
|
2005-02-27 21:08:04 +00:00
|
|
|
// constants or loop invariable expressions prior to an instance of the
|
|
|
|
// canonical induction variable.
|
2005-02-27 19:37:07 +00:00
|
|
|
unsigned indvar = 0;
|
2004-10-18 21:08:22 +00:00
|
|
|
std::vector<Value *> pre_op_vector;
|
|
|
|
std::vector<Value *> inc_op_vector;
|
2005-03-04 04:04:26 +00:00
|
|
|
const Type *ty = GEPI->getOperand(0)->getType();
|
2004-10-18 21:08:22 +00:00
|
|
|
Value *CanonicalIndVar = L->getCanonicalInductionVariable();
|
2005-02-27 19:37:07 +00:00
|
|
|
BasicBlock *Header = L->getHeader();
|
2005-03-01 03:46:11 +00:00
|
|
|
BasicBlock *Preheader = L->getLoopPreheader();
|
|
|
|
bool AllConstantOperands = true;
|
2005-03-06 21:58:22 +00:00
|
|
|
Cache = Cache->get(GEPI->getOperand(0));
|
2005-02-27 19:37:07 +00:00
|
|
|
|
2004-10-18 21:08:22 +00:00
|
|
|
for (unsigned op = 1, e = GEPI->getNumOperands(); op != e; ++op) {
|
|
|
|
Value *operand = GEPI->getOperand(op);
|
2005-03-04 04:04:26 +00:00
|
|
|
if (ty->getTypeID() == Type::StructTyID) {
|
|
|
|
assert(isa<ConstantUInt>(operand));
|
|
|
|
ConstantUInt *c = dyn_cast<ConstantUInt>(operand);
|
|
|
|
ty = ty->getContainedType(unsigned(c->getValue()));
|
|
|
|
} else {
|
|
|
|
ty = ty->getContainedType(0);
|
|
|
|
}
|
|
|
|
|
2004-10-18 21:08:22 +00:00
|
|
|
if (operand == CanonicalIndVar) {
|
|
|
|
// FIXME: use getCanonicalInductionVariableIncrement to choose between
|
|
|
|
// one and neg one maybe? We need to support int *foo = GEP base, -1
|
|
|
|
const Type *Ty = CanonicalIndVar->getType();
|
|
|
|
pre_op_vector.push_back(Constant::getNullValue(Ty));
|
|
|
|
inc_op_vector.push_back(ConstantInt::get(Ty, 1));
|
2005-02-27 19:37:07 +00:00
|
|
|
indvar = op;
|
|
|
|
break;
|
2005-03-06 22:52:29 +00:00
|
|
|
} else if (isa<Argument>(operand)) {
|
|
|
|
pre_op_vector.push_back(operand);
|
|
|
|
AllConstantOperands = false;
|
|
|
|
} else if (isa<Constant>(operand)) {
|
2004-10-18 21:08:22 +00:00
|
|
|
pre_op_vector.push_back(operand);
|
2005-02-27 19:37:07 +00:00
|
|
|
} else if (Instruction *inst = dyn_cast<Instruction>(operand)) {
|
2005-03-01 03:46:11 +00:00
|
|
|
if (!DS->dominates(inst, Preheader->getTerminator()))
|
2005-02-27 19:37:07 +00:00
|
|
|
return;
|
|
|
|
pre_op_vector.push_back(operand);
|
2005-03-01 03:46:11 +00:00
|
|
|
AllConstantOperands = false;
|
2005-03-06 22:52:29 +00:00
|
|
|
} else {
|
|
|
|
return; // Cannot handle this.
|
|
|
|
}
|
2005-03-06 21:58:22 +00:00
|
|
|
Cache = Cache->get(operand);
|
2004-10-18 21:08:22 +00:00
|
|
|
}
|
2005-02-27 19:37:07 +00:00
|
|
|
assert(indvar > 0 && "Indvar used by GEP not found in operand list");
|
2004-10-18 21:08:22 +00:00
|
|
|
|
2005-02-27 19:37:07 +00:00
|
|
|
// Ensure the pointer base is loop invariant. While strength reduction
|
|
|
|
// makes sense even if the pointer changed on every iteration, there is no
|
|
|
|
// realistic way of handling it unless GEPs were completely decomposed into
|
|
|
|
// their constituent operations so we have explicit multiplications to work
|
|
|
|
// with.
|
2004-10-18 21:08:22 +00:00
|
|
|
if (Instruction *GepPtrOp = dyn_cast<Instruction>(GEPI->getOperand(0)))
|
2005-03-01 03:46:11 +00:00
|
|
|
if (!DS->dominates(GepPtrOp, Preheader->getTerminator()))
|
2004-10-18 21:08:22 +00:00
|
|
|
return;
|
2005-03-04 04:04:26 +00:00
|
|
|
|
2005-03-05 22:40:34 +00:00
|
|
|
// Don't reduce multiplies that the target can handle via addressing modes.
|
2005-03-04 04:04:26 +00:00
|
|
|
uint64_t sz = getAnalysis<TargetData>().getTypeSize(ty);
|
2005-03-06 21:58:22 +00:00
|
|
|
if (sz && (sz & (sz-1)) == 0) // Power of two?
|
|
|
|
if (sz <= (1ULL << (MaxTargetAMSize-1)))
|
2005-03-04 04:04:26 +00:00
|
|
|
return;
|
2004-10-18 21:08:22 +00:00
|
|
|
|
|
|
|
// If all operands of the GEP we are going to insert into the preheader
|
|
|
|
// are constants, generate a GEP ConstantExpr instead.
|
|
|
|
//
|
|
|
|
// If there is only one operand after the initial non-constant one, we know
|
|
|
|
// that it was the induction variable, and has been replaced by a constant
|
|
|
|
// null value. In this case, replace the GEP with a use of pointer directly.
|
2005-03-05 22:40:34 +00:00
|
|
|
PHINode *NewPHI;
|
2005-03-06 22:36:12 +00:00
|
|
|
if (Cache->CachedPHINode == 0) {
|
2005-03-05 22:40:34 +00:00
|
|
|
Value *PreGEP;
|
|
|
|
if (AllConstantOperands && isa<Constant>(GEPI->getOperand(0))) {
|
|
|
|
Constant *C = dyn_cast<Constant>(GEPI->getOperand(0));
|
|
|
|
PreGEP = ConstantExpr::getGetElementPtr(C, pre_op_vector);
|
|
|
|
} else if (pre_op_vector.size() == 1) {
|
|
|
|
PreGEP = GEPI->getOperand(0);
|
|
|
|
} else {
|
|
|
|
PreGEP = new GetElementPtrInst(GEPI->getOperand(0),
|
|
|
|
pre_op_vector, GEPI->getName()+".pre",
|
|
|
|
Preheader->getTerminator());
|
|
|
|
}
|
|
|
|
|
2005-03-05 22:45:40 +00:00
|
|
|
// The next step of the strength reduction is to create a PHI that will
|
|
|
|
// choose between the initial GEP we created and inserted into the
|
|
|
|
// preheader, and the incremented GEP that we will create below and insert
|
|
|
|
// into the loop body.
|
2005-03-05 22:40:34 +00:00
|
|
|
NewPHI = new PHINode(PreGEP->getType(),
|
|
|
|
GEPI->getName()+".str", InsertBefore);
|
|
|
|
NewPHI->addIncoming(PreGEP, Preheader);
|
|
|
|
|
2005-03-05 22:45:40 +00:00
|
|
|
// Now, create the GEP instruction to increment by one the value selected
|
|
|
|
// by the PHI instruction we just created above, and add it as the second
|
|
|
|
// incoming Value/BasicBlock pair to the PHINode. It is inserted before
|
|
|
|
// the increment of the canonical induction variable.
|
2005-03-05 22:40:34 +00:00
|
|
|
Instruction *IncrInst =
|
|
|
|
const_cast<Instruction*>(L->getCanonicalInductionVariableIncrement());
|
|
|
|
GetElementPtrInst *StrGEP = new GetElementPtrInst(NewPHI, inc_op_vector,
|
|
|
|
GEPI->getName()+".inc",
|
|
|
|
IncrInst);
|
|
|
|
pred_iterator PI = pred_begin(Header);
|
|
|
|
if (*PI == Preheader)
|
|
|
|
++PI;
|
|
|
|
NewPHI->addIncoming(StrGEP, *PI);
|
|
|
|
Cache->CachedPHINode = NewPHI;
|
2004-10-18 21:08:22 +00:00
|
|
|
} else {
|
2005-03-05 22:40:34 +00:00
|
|
|
// Reuse previously created pointer, as it is identical to the one we were
|
|
|
|
// about to create.
|
|
|
|
NewPHI = Cache->CachedPHINode;
|
2004-10-18 21:08:22 +00:00
|
|
|
}
|
|
|
|
|
2005-02-27 19:37:07 +00:00
|
|
|
if (GEPI->getNumOperands() - 1 == indvar) {
|
|
|
|
// If there were no operands following the induction variable, replace all
|
|
|
|
// uses of the old GEP instruction with the new PHI.
|
|
|
|
GEPI->replaceAllUsesWith(NewPHI);
|
|
|
|
} else {
|
|
|
|
// Create a new GEP instruction using the new PHI as the base. The
|
|
|
|
// operands of the original GEP past the induction variable become
|
|
|
|
// operands of this new GEP.
|
|
|
|
std::vector<Value *> op_vector;
|
|
|
|
const Type *Ty = CanonicalIndVar->getType();
|
|
|
|
op_vector.push_back(Constant::getNullValue(Ty));
|
|
|
|
for (unsigned op = indvar + 1; op < GEPI->getNumOperands(); op++)
|
|
|
|
op_vector.push_back(GEPI->getOperand(op));
|
|
|
|
GetElementPtrInst *newGEP = new GetElementPtrInst(NewPHI, op_vector,
|
|
|
|
GEPI->getName() + ".lsr",
|
|
|
|
GEPI);
|
|
|
|
GEPI->replaceAllUsesWith(newGEP);
|
2005-03-06 21:58:22 +00:00
|
|
|
}
|
2004-10-18 21:08:22 +00:00
|
|
|
|
|
|
|
// The old GEP is now dead.
|
|
|
|
DeadInsts.insert(GEPI);
|
|
|
|
++NumReduced;
|
|
|
|
}
|
|
|
|
|
|
|
|
void LoopStrengthReduce::runOnLoop(Loop *L) {
|
|
|
|
// First step, transform all loops nesting inside of this loop.
|
|
|
|
for (LoopInfo::iterator I = L->begin(), E = L->end(); I != E; ++I)
|
|
|
|
runOnLoop(*I);
|
|
|
|
|
|
|
|
// Next, get the first PHINode since it is guaranteed to be the canonical
|
|
|
|
// induction variable for the loop by the preceding IndVarSimplify pass.
|
|
|
|
PHINode *PN = L->getCanonicalInductionVariable();
|
|
|
|
if (0 == PN)
|
|
|
|
return;
|
|
|
|
|
|
|
|
// FIXME: Need to use SCEV to detect GEP uses of the indvar, since indvars
|
|
|
|
// pass creates code like this, which we can't currently detect:
|
|
|
|
// %tmp.1 = sub uint 2000, %indvar
|
|
|
|
// %tmp.8 = getelementptr int* %y, uint %tmp.1
|
|
|
|
|
2005-02-27 21:08:04 +00:00
|
|
|
// Strength reduce all GEPs in the Loop. Insert secondary PHI nodes for the
|
|
|
|
// strength reduced pointers we'll be creating after the canonical induction
|
|
|
|
// variable's PHI.
|
2004-10-18 21:08:22 +00:00
|
|
|
std::set<Instruction*> DeadInsts;
|
2005-03-05 22:40:34 +00:00
|
|
|
GEPCache Cache;
|
2004-10-18 21:08:22 +00:00
|
|
|
for (Value::use_iterator UI = PN->use_begin(), UE = PN->use_end();
|
|
|
|
UI != UE; ++UI)
|
|
|
|
if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(*UI))
|
2005-03-05 22:40:34 +00:00
|
|
|
strengthReduceGEP(GEPI, L, &Cache, PN->getNext(), DeadInsts);
|
2004-10-18 21:08:22 +00:00
|
|
|
|
|
|
|
// Clean up after ourselves
|
|
|
|
if (!DeadInsts.empty()) {
|
|
|
|
DeleteTriviallyDeadInstructions(DeadInsts);
|
|
|
|
|
|
|
|
// At this point, we know that we have killed one or more GEP instructions.
|
|
|
|
// It is worth checking to see if the cann indvar is also dead, so that we
|
|
|
|
// can remove it as well. The requirements for the cann indvar to be
|
|
|
|
// considered dead are:
|
|
|
|
// 1. the cann indvar has one use
|
|
|
|
// 2. the use is an add instruction
|
|
|
|
// 3. the add has one use
|
|
|
|
// 4. the add is used by the cann indvar
|
|
|
|
// If all four cases above are true, then we can remove both the add and
|
|
|
|
// the cann indvar.
|
2005-03-01 03:46:11 +00:00
|
|
|
// FIXME: this needs to eliminate an induction variable even if it's being
|
|
|
|
// compared against some value to decide loop termination.
|
2004-10-18 21:08:22 +00:00
|
|
|
if (PN->hasOneUse()) {
|
|
|
|
BinaryOperator *BO = dyn_cast<BinaryOperator>(*(PN->use_begin()));
|
|
|
|
if (BO && BO->getOpcode() == Instruction::Add)
|
|
|
|
if (BO->hasOneUse()) {
|
2005-03-01 03:46:11 +00:00
|
|
|
if (PN == dyn_cast<PHINode>(*(BO->use_begin()))) {
|
2004-10-18 21:08:22 +00:00
|
|
|
DeadInsts.insert(BO);
|
2005-03-01 03:46:11 +00:00
|
|
|
// Break the cycle, then delete the PHI.
|
|
|
|
PN->replaceAllUsesWith(UndefValue::get(PN->getType()));
|
|
|
|
PN->eraseFromParent();
|
2004-10-18 21:08:22 +00:00
|
|
|
DeleteTriviallyDeadInstructions(DeadInsts);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|