2005-07-30 00:12:19 +00:00
|
|
|
//===- ScalarEvolutionExpander.cpp - Scalar Evolution Analysis --*- C++ -*-===//
|
|
|
|
//
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
2007-12-29 20:36:04 +00:00
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
2005-07-30 00:12:19 +00:00
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
// This file contains the implementation of the scalar evolution expander,
|
|
|
|
// which is used to generate the code corresponding to a given scalar evolution
|
|
|
|
// expression.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
|
|
|
#include "llvm/Analysis/ScalarEvolutionExpander.h"
|
2006-12-07 01:30:32 +00:00
|
|
|
#include "llvm/Analysis/LoopInfo.h"
|
2009-05-19 02:15:55 +00:00
|
|
|
#include "llvm/Target/TargetData.h"
|
2005-07-30 00:12:19 +00:00
|
|
|
using namespace llvm;
|
|
|
|
|
2006-02-04 09:51:53 +00:00
|
|
|
/// InsertCastOfTo - Insert a cast of V to the specified type, doing what
|
|
|
|
/// we can to share the casts.
|
2006-12-13 08:06:42 +00:00
|
|
|
Value *SCEVExpander::InsertCastOfTo(Instruction::CastOps opcode, Value *V,
|
|
|
|
const Type *Ty) {
|
2009-04-16 03:18:22 +00:00
|
|
|
// Short-circuit unnecessary bitcasts.
|
|
|
|
if (opcode == Instruction::BitCast && V->getType() == Ty)
|
|
|
|
return V;
|
|
|
|
|
2009-04-16 15:52:57 +00:00
|
|
|
// Short-circuit unnecessary inttoptr<->ptrtoint casts.
|
2009-04-21 01:07:12 +00:00
|
|
|
if ((opcode == Instruction::PtrToInt || opcode == Instruction::IntToPtr) &&
|
2009-05-01 17:00:00 +00:00
|
|
|
SE.getTypeSizeInBits(Ty) == SE.getTypeSizeInBits(V->getType())) {
|
2009-04-21 01:07:12 +00:00
|
|
|
if (CastInst *CI = dyn_cast<CastInst>(V))
|
|
|
|
if ((CI->getOpcode() == Instruction::PtrToInt ||
|
|
|
|
CI->getOpcode() == Instruction::IntToPtr) &&
|
|
|
|
SE.getTypeSizeInBits(CI->getType()) ==
|
|
|
|
SE.getTypeSizeInBits(CI->getOperand(0)->getType()))
|
|
|
|
return CI->getOperand(0);
|
2009-05-01 17:00:00 +00:00
|
|
|
if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
|
|
|
|
if ((CE->getOpcode() == Instruction::PtrToInt ||
|
|
|
|
CE->getOpcode() == Instruction::IntToPtr) &&
|
|
|
|
SE.getTypeSizeInBits(CE->getType()) ==
|
|
|
|
SE.getTypeSizeInBits(CE->getOperand(0)->getType()))
|
|
|
|
return CE->getOperand(0);
|
|
|
|
}
|
2009-04-16 15:52:57 +00:00
|
|
|
|
2006-02-04 09:51:53 +00:00
|
|
|
// FIXME: keep track of the cast instruction.
|
|
|
|
if (Constant *C = dyn_cast<Constant>(V))
|
2006-12-12 23:36:14 +00:00
|
|
|
return ConstantExpr::getCast(opcode, C, Ty);
|
2006-02-04 09:51:53 +00:00
|
|
|
|
|
|
|
if (Argument *A = dyn_cast<Argument>(V)) {
|
|
|
|
// Check to see if there is already a cast!
|
|
|
|
for (Value::use_iterator UI = A->use_begin(), E = A->use_end();
|
|
|
|
UI != E; ++UI) {
|
|
|
|
if ((*UI)->getType() == Ty)
|
2008-02-09 18:30:13 +00:00
|
|
|
if (CastInst *CI = dyn_cast<CastInst>(cast<Instruction>(*UI)))
|
|
|
|
if (CI->getOpcode() == opcode) {
|
|
|
|
// If the cast isn't the first instruction of the function, move it.
|
|
|
|
if (BasicBlock::iterator(CI) !=
|
|
|
|
A->getParent()->getEntryBlock().begin()) {
|
2009-04-22 16:11:16 +00:00
|
|
|
// If the CastInst is the insert point, change the insert point.
|
|
|
|
if (CI == InsertPt) ++InsertPt;
|
|
|
|
// Splice the cast at the beginning of the entry block.
|
2008-02-09 18:30:13 +00:00
|
|
|
CI->moveBefore(A->getParent()->getEntryBlock().begin());
|
|
|
|
}
|
|
|
|
return CI;
|
2006-02-04 09:51:53 +00:00
|
|
|
}
|
|
|
|
}
|
2009-05-01 17:13:31 +00:00
|
|
|
Instruction *I = CastInst::Create(opcode, V, Ty, V->getName(),
|
|
|
|
A->getParent()->getEntryBlock().begin());
|
|
|
|
InsertedValues.insert(I);
|
|
|
|
return I;
|
2006-02-04 09:51:53 +00:00
|
|
|
}
|
2008-02-09 18:30:13 +00:00
|
|
|
|
2006-02-04 09:51:53 +00:00
|
|
|
Instruction *I = cast<Instruction>(V);
|
2008-02-09 18:30:13 +00:00
|
|
|
|
2006-02-04 09:51:53 +00:00
|
|
|
// Check to see if there is already a cast. If there is, use it.
|
|
|
|
for (Value::use_iterator UI = I->use_begin(), E = I->use_end();
|
|
|
|
UI != E; ++UI) {
|
|
|
|
if ((*UI)->getType() == Ty)
|
2008-02-09 18:30:13 +00:00
|
|
|
if (CastInst *CI = dyn_cast<CastInst>(cast<Instruction>(*UI)))
|
|
|
|
if (CI->getOpcode() == opcode) {
|
|
|
|
BasicBlock::iterator It = I; ++It;
|
|
|
|
if (isa<InvokeInst>(I))
|
|
|
|
It = cast<InvokeInst>(I)->getNormalDest()->begin();
|
|
|
|
while (isa<PHINode>(It)) ++It;
|
|
|
|
if (It != BasicBlock::iterator(CI)) {
|
2009-04-22 16:11:16 +00:00
|
|
|
// If the CastInst is the insert point, change the insert point.
|
|
|
|
if (CI == InsertPt) ++InsertPt;
|
2008-02-09 18:30:13 +00:00
|
|
|
// Splice the cast immediately after the operand in question.
|
|
|
|
CI->moveBefore(It);
|
|
|
|
}
|
|
|
|
return CI;
|
2006-02-04 09:51:53 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
BasicBlock::iterator IP = I; ++IP;
|
|
|
|
if (InvokeInst *II = dyn_cast<InvokeInst>(I))
|
|
|
|
IP = II->getNormalDest()->begin();
|
|
|
|
while (isa<PHINode>(IP)) ++IP;
|
2009-05-01 17:13:31 +00:00
|
|
|
Instruction *CI = CastInst::Create(opcode, V, Ty, V->getName(), IP);
|
|
|
|
InsertedValues.insert(CI);
|
|
|
|
return CI;
|
2006-02-04 09:51:53 +00:00
|
|
|
}
|
|
|
|
|
2009-04-21 01:07:12 +00:00
|
|
|
/// InsertNoopCastOfTo - Insert a cast of V to the specified type,
|
|
|
|
/// which must be possible with a noop cast.
|
|
|
|
Value *SCEVExpander::InsertNoopCastOfTo(Value *V, const Type *Ty) {
|
|
|
|
Instruction::CastOps Op = CastInst::getCastOpcode(V, false, Ty, false);
|
|
|
|
assert((Op == Instruction::BitCast ||
|
2009-04-22 18:51:05 +00:00
|
|
|
Op == Instruction::PtrToInt ||
|
|
|
|
Op == Instruction::IntToPtr) &&
|
2009-04-21 01:07:12 +00:00
|
|
|
"InsertNoopCastOfTo cannot perform non-noop casts!");
|
|
|
|
assert(SE.getTypeSizeInBits(V->getType()) == SE.getTypeSizeInBits(Ty) &&
|
|
|
|
"InsertNoopCastOfTo cannot change sizes!");
|
|
|
|
return InsertCastOfTo(Op, V, Ty);
|
|
|
|
}
|
|
|
|
|
2007-04-13 05:04:18 +00:00
|
|
|
/// InsertBinop - Insert the specified binary operator, doing a small amount
|
|
|
|
/// of work to avoid inserting an obviously redundant operation.
|
|
|
|
Value *SCEVExpander::InsertBinop(Instruction::BinaryOps Opcode, Value *LHS,
|
2009-04-22 16:05:50 +00:00
|
|
|
Value *RHS, BasicBlock::iterator InsertPt) {
|
2007-06-15 19:21:55 +00:00
|
|
|
// Fold a binop with constant operands.
|
|
|
|
if (Constant *CLHS = dyn_cast<Constant>(LHS))
|
|
|
|
if (Constant *CRHS = dyn_cast<Constant>(RHS))
|
|
|
|
return ConstantExpr::get(Opcode, CLHS, CRHS);
|
|
|
|
|
2007-04-13 05:04:18 +00:00
|
|
|
// Do a quick scan to see if we have this binop nearby. If so, reuse it.
|
|
|
|
unsigned ScanLimit = 6;
|
2008-06-15 19:07:39 +00:00
|
|
|
BasicBlock::iterator BlockBegin = InsertPt->getParent()->begin();
|
|
|
|
if (InsertPt != BlockBegin) {
|
|
|
|
// Scanning starts from the last instruction before InsertPt.
|
|
|
|
BasicBlock::iterator IP = InsertPt;
|
|
|
|
--IP;
|
|
|
|
for (; ScanLimit; --IP, --ScanLimit) {
|
2009-05-19 02:15:55 +00:00
|
|
|
if (IP->getOpcode() == (unsigned)Opcode && IP->getOperand(0) == LHS &&
|
|
|
|
IP->getOperand(1) == RHS)
|
|
|
|
return IP;
|
2008-06-15 19:07:39 +00:00
|
|
|
if (IP == BlockBegin) break;
|
|
|
|
}
|
2007-04-13 05:04:18 +00:00
|
|
|
}
|
2008-06-15 19:07:39 +00:00
|
|
|
|
|
|
|
// If we haven't found this binop, insert it.
|
2009-05-01 17:13:31 +00:00
|
|
|
Instruction *BO = BinaryOperator::Create(Opcode, LHS, RHS, "tmp", InsertPt);
|
|
|
|
InsertedValues.insert(BO);
|
|
|
|
return BO;
|
2007-04-13 05:04:18 +00:00
|
|
|
}
|
|
|
|
|
2009-05-19 02:15:55 +00:00
|
|
|
/// expandAddToGEP - Expand a SCEVAddExpr with a pointer type into a GEP
|
|
|
|
/// instead of using ptrtoint+arithmetic+inttoptr.
|
|
|
|
Value *SCEVExpander::expandAddToGEP(const SCEVAddExpr *S,
|
|
|
|
const PointerType *PTy,
|
|
|
|
const Type *Ty,
|
|
|
|
Value *V) {
|
|
|
|
const Type *ElTy = PTy->getElementType();
|
|
|
|
SmallVector<Value *, 4> GepIndices;
|
|
|
|
std::vector<SCEVHandle> Ops = S->getOperands();
|
|
|
|
bool AnyNonZeroIndices = false;
|
|
|
|
Ops.pop_back();
|
|
|
|
|
|
|
|
// Decend down the pointer's type and attempt to convert the other
|
|
|
|
// operands into GEP indices, at each level. The first index in a GEP
|
|
|
|
// indexes into the array implied by the pointer operand; the rest of
|
|
|
|
// the indices index into the element or field type selected by the
|
|
|
|
// preceding index.
|
|
|
|
for (;;) {
|
|
|
|
APInt ElSize = APInt(SE.getTypeSizeInBits(Ty),
|
|
|
|
ElTy->isSized() ? SE.TD->getTypeAllocSize(ElTy) : 0);
|
|
|
|
std::vector<SCEVHandle> NewOps;
|
|
|
|
std::vector<SCEVHandle> ScaledOps;
|
|
|
|
for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
|
|
|
|
if (ElSize != 0) {
|
|
|
|
if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i]))
|
|
|
|
if (!C->getValue()->getValue().srem(ElSize)) {
|
|
|
|
ConstantInt *CI =
|
|
|
|
ConstantInt::get(C->getValue()->getValue().sdiv(ElSize));
|
|
|
|
SCEVHandle Div = SE.getConstant(CI);
|
|
|
|
ScaledOps.push_back(Div);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Ops[i]))
|
|
|
|
if (const SCEVConstant *C = dyn_cast<SCEVConstant>(M->getOperand(0)))
|
|
|
|
if (C->getValue()->getValue() == ElSize) {
|
|
|
|
for (unsigned j = 1, f = M->getNumOperands(); j != f; ++j)
|
|
|
|
ScaledOps.push_back(M->getOperand(j));
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Ops[i]))
|
|
|
|
if (BinaryOperator *BO = dyn_cast<BinaryOperator>(U->getValue()))
|
|
|
|
if (BO->getOpcode() == Instruction::Mul)
|
|
|
|
if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->getOperand(1)))
|
|
|
|
if (CI->getValue() == ElSize) {
|
|
|
|
ScaledOps.push_back(SE.getUnknown(BO->getOperand(0)));
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (ElSize == 1) {
|
|
|
|
ScaledOps.push_back(Ops[i]);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
NewOps.push_back(Ops[i]);
|
|
|
|
}
|
|
|
|
Ops = NewOps;
|
|
|
|
AnyNonZeroIndices |= !ScaledOps.empty();
|
|
|
|
Value *Scaled = ScaledOps.empty() ?
|
|
|
|
Constant::getNullValue(Ty) :
|
|
|
|
expandCodeFor(SE.getAddExpr(ScaledOps), Ty);
|
|
|
|
GepIndices.push_back(Scaled);
|
|
|
|
|
|
|
|
// Collect struct field index operands.
|
|
|
|
if (!Ops.empty())
|
|
|
|
while (const StructType *STy = dyn_cast<StructType>(ElTy)) {
|
|
|
|
if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[0]))
|
|
|
|
if (SE.getTypeSizeInBits(C->getType()) <= 64) {
|
|
|
|
const StructLayout &SL = *SE.TD->getStructLayout(STy);
|
|
|
|
uint64_t FullOffset = C->getValue()->getZExtValue();
|
|
|
|
if (FullOffset < SL.getSizeInBytes()) {
|
|
|
|
unsigned ElIdx = SL.getElementContainingOffset(FullOffset);
|
|
|
|
GepIndices.push_back(ConstantInt::get(Type::Int32Ty, ElIdx));
|
|
|
|
ElTy = STy->getTypeAtIndex(ElIdx);
|
|
|
|
Ops[0] =
|
|
|
|
SE.getConstant(ConstantInt::get(Ty,
|
|
|
|
FullOffset -
|
|
|
|
SL.getElementOffset(ElIdx)));
|
|
|
|
AnyNonZeroIndices = true;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (const ArrayType *ATy = dyn_cast<ArrayType>(ElTy)) {
|
|
|
|
ElTy = ATy->getElementType();
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
// If none of the operands were convertable to proper GEP indices, cast
|
|
|
|
// the base to i8* and do an ugly getelementptr with that. It's still
|
|
|
|
// better than ptrtoint+arithmetic+inttoptr at least.
|
|
|
|
if (!AnyNonZeroIndices) {
|
|
|
|
V = InsertNoopCastOfTo(V,
|
|
|
|
Type::Int8Ty->getPointerTo(PTy->getAddressSpace()));
|
|
|
|
Value *Idx = expand(SE.getAddExpr(Ops));
|
|
|
|
Idx = InsertNoopCastOfTo(Idx, Ty);
|
|
|
|
|
|
|
|
// Fold a GEP with constant operands.
|
|
|
|
if (Constant *CLHS = dyn_cast<Constant>(V))
|
|
|
|
if (Constant *CRHS = dyn_cast<Constant>(Idx))
|
|
|
|
return ConstantExpr::get(Instruction::GetElementPtr, CLHS, CRHS);
|
|
|
|
|
|
|
|
// Do a quick scan to see if we have this GEP nearby. If so, reuse it.
|
|
|
|
unsigned ScanLimit = 6;
|
|
|
|
BasicBlock::iterator BlockBegin = InsertPt->getParent()->begin();
|
|
|
|
if (InsertPt != BlockBegin) {
|
|
|
|
// Scanning starts from the last instruction before InsertPt.
|
|
|
|
BasicBlock::iterator IP = InsertPt;
|
|
|
|
--IP;
|
|
|
|
for (; ScanLimit; --IP, --ScanLimit) {
|
|
|
|
if (IP->getOpcode() == Instruction::GetElementPtr &&
|
|
|
|
IP->getOperand(0) == V && IP->getOperand(1) == Idx)
|
|
|
|
return IP;
|
|
|
|
if (IP == BlockBegin) break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Value *GEP = GetElementPtrInst::Create(V, Idx, "scevgep", InsertPt);
|
|
|
|
InsertedValues.insert(GEP);
|
|
|
|
return GEP;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Insert a pretty getelementptr.
|
|
|
|
Value *GEP = GetElementPtrInst::Create(V,
|
|
|
|
GepIndices.begin(),
|
|
|
|
GepIndices.end(),
|
|
|
|
"scevgep", InsertPt);
|
|
|
|
Ops.push_back(SE.getUnknown(GEP));
|
|
|
|
InsertedValues.insert(GEP);
|
|
|
|
return expand(SE.getAddExpr(Ops));
|
|
|
|
}
|
|
|
|
|
2009-04-18 17:56:28 +00:00
|
|
|
Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) {
|
2009-04-21 01:07:12 +00:00
|
|
|
const Type *Ty = SE.getEffectiveSCEVType(S->getType());
|
2008-06-18 16:37:11 +00:00
|
|
|
Value *V = expand(S->getOperand(S->getNumOperands()-1));
|
2009-05-19 02:15:55 +00:00
|
|
|
|
|
|
|
// Turn things like ptrtoint+arithmetic+inttoptr into GEP. This helps
|
|
|
|
// BasicAliasAnalysis analyze the result. However, it suffers from the
|
|
|
|
// underlying bug described in PR2831. Addition in LLVM currently always
|
|
|
|
// has two's complement wrapping guaranteed. However, the semantics for
|
|
|
|
// getelementptr overflow are ambiguous. In the common case though, this
|
|
|
|
// expansion gets used when a GEP in the original code has been converted
|
|
|
|
// into integer arithmetic, in which case the resulting code will be no
|
|
|
|
// more undefined than it was originally.
|
|
|
|
if (SE.TD)
|
|
|
|
if (const PointerType *PTy = dyn_cast<PointerType>(V->getType()))
|
|
|
|
return expandAddToGEP(S, PTy, Ty, V);
|
|
|
|
|
2009-04-21 01:07:12 +00:00
|
|
|
V = InsertNoopCastOfTo(V, Ty);
|
2008-06-18 16:37:11 +00:00
|
|
|
|
|
|
|
// Emit a bunch of add instructions
|
2009-04-16 03:18:22 +00:00
|
|
|
for (int i = S->getNumOperands()-2; i >= 0; --i) {
|
|
|
|
Value *W = expand(S->getOperand(i));
|
2009-04-21 01:07:12 +00:00
|
|
|
W = InsertNoopCastOfTo(W, Ty);
|
2009-04-16 03:18:22 +00:00
|
|
|
V = InsertBinop(Instruction::Add, V, W, InsertPt);
|
|
|
|
}
|
2008-06-18 16:37:11 +00:00
|
|
|
return V;
|
|
|
|
}
|
2009-05-19 02:15:55 +00:00
|
|
|
|
2009-04-18 17:56:28 +00:00
|
|
|
Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) {
|
2009-04-21 01:07:12 +00:00
|
|
|
const Type *Ty = SE.getEffectiveSCEVType(S->getType());
|
2005-07-30 00:12:19 +00:00
|
|
|
int FirstOp = 0; // Set if we should emit a subtract.
|
2009-04-18 17:56:28 +00:00
|
|
|
if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(S->getOperand(0)))
|
2005-07-30 00:12:19 +00:00
|
|
|
if (SC->getValue()->isAllOnesValue())
|
|
|
|
FirstOp = 1;
|
|
|
|
|
|
|
|
int i = S->getNumOperands()-2;
|
2007-06-15 14:38:12 +00:00
|
|
|
Value *V = expand(S->getOperand(i+1));
|
2009-04-21 01:07:12 +00:00
|
|
|
V = InsertNoopCastOfTo(V, Ty);
|
2005-07-30 00:12:19 +00:00
|
|
|
|
|
|
|
// Emit a bunch of multiply instructions
|
2009-04-16 03:18:22 +00:00
|
|
|
for (; i >= FirstOp; --i) {
|
|
|
|
Value *W = expand(S->getOperand(i));
|
2009-04-21 01:07:12 +00:00
|
|
|
W = InsertNoopCastOfTo(W, Ty);
|
2009-04-16 03:18:22 +00:00
|
|
|
V = InsertBinop(Instruction::Mul, V, W, InsertPt);
|
|
|
|
}
|
|
|
|
|
2005-07-30 00:12:19 +00:00
|
|
|
// -1 * ... ---> 0 - ...
|
|
|
|
if (FirstOp == 1)
|
2009-04-16 03:18:22 +00:00
|
|
|
V = InsertBinop(Instruction::Sub, Constant::getNullValue(Ty), V, InsertPt);
|
2005-07-30 00:12:19 +00:00
|
|
|
return V;
|
|
|
|
}
|
|
|
|
|
2009-04-18 17:56:28 +00:00
|
|
|
Value *SCEVExpander::visitUDivExpr(const SCEVUDivExpr *S) {
|
2009-04-21 01:07:12 +00:00
|
|
|
const Type *Ty = SE.getEffectiveSCEVType(S->getType());
|
2009-04-16 03:18:22 +00:00
|
|
|
|
2008-07-08 05:05:37 +00:00
|
|
|
Value *LHS = expand(S->getLHS());
|
2009-04-21 01:07:12 +00:00
|
|
|
LHS = InsertNoopCastOfTo(LHS, Ty);
|
2009-04-18 17:56:28 +00:00
|
|
|
if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(S->getRHS())) {
|
2008-07-08 05:05:37 +00:00
|
|
|
const APInt &RHS = SC->getValue()->getValue();
|
|
|
|
if (RHS.isPowerOf2())
|
|
|
|
return InsertBinop(Instruction::LShr, LHS,
|
2009-04-16 03:18:22 +00:00
|
|
|
ConstantInt::get(Ty, RHS.logBase2()),
|
2008-07-08 05:05:37 +00:00
|
|
|
InsertPt);
|
|
|
|
}
|
|
|
|
|
|
|
|
Value *RHS = expand(S->getRHS());
|
2009-04-21 01:07:12 +00:00
|
|
|
RHS = InsertNoopCastOfTo(RHS, Ty);
|
2008-07-08 05:05:37 +00:00
|
|
|
return InsertBinop(Instruction::UDiv, LHS, RHS, InsertPt);
|
|
|
|
}
|
|
|
|
|
2009-04-18 17:56:28 +00:00
|
|
|
Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
|
2009-04-21 01:07:12 +00:00
|
|
|
const Type *Ty = SE.getEffectiveSCEVType(S->getType());
|
2005-07-30 00:12:19 +00:00
|
|
|
const Loop *L = S->getLoop();
|
|
|
|
|
|
|
|
// {X,+,F} --> X + {0,+,F}
|
2008-06-18 16:23:07 +00:00
|
|
|
if (!S->getStart()->isZero()) {
|
2009-05-19 02:15:55 +00:00
|
|
|
std::vector<SCEVHandle> NewOps(S->getOperands());
|
2007-10-22 18:31:58 +00:00
|
|
|
NewOps[0] = SE.getIntegerSCEV(0, Ty);
|
|
|
|
Value *Rest = expand(SE.getAddRecExpr(NewOps, L));
|
2009-05-19 02:15:55 +00:00
|
|
|
return expand(SE.getAddExpr(S->getStart(), SE.getUnknown(Rest)));
|
2005-07-30 00:12:19 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// {0,+,1} --> Insert a canonical induction variable into the loop!
|
2008-06-22 19:23:09 +00:00
|
|
|
if (S->isAffine() &&
|
2007-10-22 18:31:58 +00:00
|
|
|
S->getOperand(1) == SE.getIntegerSCEV(1, Ty)) {
|
2005-07-30 00:12:19 +00:00
|
|
|
// Create and insert the PHI node for the induction variable in the
|
|
|
|
// specified loop.
|
|
|
|
BasicBlock *Header = L->getHeader();
|
2008-04-06 20:25:17 +00:00
|
|
|
PHINode *PN = PHINode::Create(Ty, "indvar", Header->begin());
|
2009-05-01 17:13:31 +00:00
|
|
|
InsertedValues.insert(PN);
|
2005-07-30 00:12:19 +00:00
|
|
|
PN->addIncoming(Constant::getNullValue(Ty), L->getLoopPreheader());
|
|
|
|
|
|
|
|
pred_iterator HPI = pred_begin(Header);
|
|
|
|
assert(HPI != pred_end(Header) && "Loop with zero preds???");
|
|
|
|
if (!L->contains(*HPI)) ++HPI;
|
|
|
|
assert(HPI != pred_end(Header) && L->contains(*HPI) &&
|
|
|
|
"No backedge in loop?");
|
|
|
|
|
|
|
|
// Insert a unit add instruction right before the terminator corresponding
|
|
|
|
// to the back-edge.
|
2007-01-21 00:29:26 +00:00
|
|
|
Constant *One = ConstantInt::get(Ty, 1);
|
2008-05-16 19:29:10 +00:00
|
|
|
Instruction *Add = BinaryOperator::CreateAdd(PN, One, "indvar.next",
|
2005-07-30 00:12:19 +00:00
|
|
|
(*HPI)->getTerminator());
|
2009-05-01 17:13:31 +00:00
|
|
|
InsertedValues.insert(Add);
|
2005-07-30 00:12:19 +00:00
|
|
|
|
|
|
|
pred_iterator PI = pred_begin(Header);
|
|
|
|
if (*PI == L->getLoopPreheader())
|
|
|
|
++PI;
|
|
|
|
PN->addIncoming(Add, *PI);
|
|
|
|
return PN;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get the canonical induction variable I for this loop.
|
|
|
|
Value *I = getOrInsertCanonicalInductionVariable(L, Ty);
|
|
|
|
|
Fix a problem that Nate noticed with LSR:
When inserting code for an addrec expression with a non-unit stride, be
more careful where we insert the multiply. In particular, insert the multiply
in the outermost loop we can, instead of the requested insertion point.
This allows LSR to notice the mul in the right loop, reducing it when it gets
to it. This allows it to reduce the multiply, where before it missed it.
This happens quite a bit in the test suite, for example, eliminating 2
multiplies in art, 3 in ammp, 4 in apsi, reducing from 1050 multiplies to
910 muls in galgel (!), from 877 to 859 in applu, and 36 to 30 in bzip2.
This speeds up galgel from 16.45s to 16.01s, applu from 14.21 to 13.94s and
fourinarow from 66.67s to 63.48s.
This implements Transforms/LoopStrengthReduce/nested-reduce.ll
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@24102 91177308-0d34-0410-b5e6-96231b3b80d8
2005-10-30 06:24:33 +00:00
|
|
|
// If this is a simple linear addrec, emit it now as a special case.
|
2008-06-22 19:23:09 +00:00
|
|
|
if (S->isAffine()) { // {0,+,F} --> i*F
|
2007-06-15 14:38:12 +00:00
|
|
|
Value *F = expand(S->getOperand(1));
|
2009-04-21 01:07:12 +00:00
|
|
|
F = InsertNoopCastOfTo(F, Ty);
|
Fix a problem that Nate noticed with LSR:
When inserting code for an addrec expression with a non-unit stride, be
more careful where we insert the multiply. In particular, insert the multiply
in the outermost loop we can, instead of the requested insertion point.
This allows LSR to notice the mul in the right loop, reducing it when it gets
to it. This allows it to reduce the multiply, where before it missed it.
This happens quite a bit in the test suite, for example, eliminating 2
multiplies in art, 3 in ammp, 4 in apsi, reducing from 1050 multiplies to
910 muls in galgel (!), from 877 to 859 in applu, and 36 to 30 in bzip2.
This speeds up galgel from 16.45s to 16.01s, applu from 14.21 to 13.94s and
fourinarow from 66.67s to 63.48s.
This implements Transforms/LoopStrengthReduce/nested-reduce.ll
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@24102 91177308-0d34-0410-b5e6-96231b3b80d8
2005-10-30 06:24:33 +00:00
|
|
|
|
|
|
|
// IF the step is by one, just return the inserted IV.
|
2007-01-11 12:24:14 +00:00
|
|
|
if (ConstantInt *CI = dyn_cast<ConstantInt>(F))
|
2007-03-01 19:45:00 +00:00
|
|
|
if (CI->getValue() == 1)
|
Fix a problem that Nate noticed with LSR:
When inserting code for an addrec expression with a non-unit stride, be
more careful where we insert the multiply. In particular, insert the multiply
in the outermost loop we can, instead of the requested insertion point.
This allows LSR to notice the mul in the right loop, reducing it when it gets
to it. This allows it to reduce the multiply, where before it missed it.
This happens quite a bit in the test suite, for example, eliminating 2
multiplies in art, 3 in ammp, 4 in apsi, reducing from 1050 multiplies to
910 muls in galgel (!), from 877 to 859 in applu, and 36 to 30 in bzip2.
This speeds up galgel from 16.45s to 16.01s, applu from 14.21 to 13.94s and
fourinarow from 66.67s to 63.48s.
This implements Transforms/LoopStrengthReduce/nested-reduce.ll
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@24102 91177308-0d34-0410-b5e6-96231b3b80d8
2005-10-30 06:24:33 +00:00
|
|
|
return I;
|
|
|
|
|
|
|
|
// If the insert point is directly inside of the loop, emit the multiply at
|
|
|
|
// the insert point. Otherwise, L is a loop that is a parent of the insert
|
|
|
|
// point loop. If we can, move the multiply to the outer most loop that it
|
|
|
|
// is safe to be in.
|
2009-04-22 16:05:50 +00:00
|
|
|
BasicBlock::iterator MulInsertPt = getInsertionPoint();
|
2009-05-19 02:15:55 +00:00
|
|
|
Loop *InsertPtLoop = SE.LI->getLoopFor(MulInsertPt->getParent());
|
Fix a problem that Nate noticed with LSR:
When inserting code for an addrec expression with a non-unit stride, be
more careful where we insert the multiply. In particular, insert the multiply
in the outermost loop we can, instead of the requested insertion point.
This allows LSR to notice the mul in the right loop, reducing it when it gets
to it. This allows it to reduce the multiply, where before it missed it.
This happens quite a bit in the test suite, for example, eliminating 2
multiplies in art, 3 in ammp, 4 in apsi, reducing from 1050 multiplies to
910 muls in galgel (!), from 877 to 859 in applu, and 36 to 30 in bzip2.
This speeds up galgel from 16.45s to 16.01s, applu from 14.21 to 13.94s and
fourinarow from 66.67s to 63.48s.
This implements Transforms/LoopStrengthReduce/nested-reduce.ll
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@24102 91177308-0d34-0410-b5e6-96231b3b80d8
2005-10-30 06:24:33 +00:00
|
|
|
if (InsertPtLoop != L && InsertPtLoop &&
|
|
|
|
L->contains(InsertPtLoop->getHeader())) {
|
2008-06-14 16:48:22 +00:00
|
|
|
do {
|
Fix a problem that Nate noticed with LSR:
When inserting code for an addrec expression with a non-unit stride, be
more careful where we insert the multiply. In particular, insert the multiply
in the outermost loop we can, instead of the requested insertion point.
This allows LSR to notice the mul in the right loop, reducing it when it gets
to it. This allows it to reduce the multiply, where before it missed it.
This happens quite a bit in the test suite, for example, eliminating 2
multiplies in art, 3 in ammp, 4 in apsi, reducing from 1050 multiplies to
910 muls in galgel (!), from 877 to 859 in applu, and 36 to 30 in bzip2.
This speeds up galgel from 16.45s to 16.01s, applu from 14.21 to 13.94s and
fourinarow from 66.67s to 63.48s.
This implements Transforms/LoopStrengthReduce/nested-reduce.ll
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@24102 91177308-0d34-0410-b5e6-96231b3b80d8
2005-10-30 06:24:33 +00:00
|
|
|
// If we cannot hoist the multiply out of this loop, don't.
|
|
|
|
if (!InsertPtLoop->isLoopInvariant(F)) break;
|
|
|
|
|
2008-06-14 16:48:22 +00:00
|
|
|
BasicBlock *InsertPtLoopPH = InsertPtLoop->getLoopPreheader();
|
|
|
|
|
|
|
|
// If this loop hasn't got a preheader, we aren't able to hoist the
|
|
|
|
// multiply.
|
|
|
|
if (!InsertPtLoopPH)
|
|
|
|
break;
|
|
|
|
|
|
|
|
// Otherwise, move the insert point to the preheader.
|
|
|
|
MulInsertPt = InsertPtLoopPH->getTerminator();
|
Fix a problem that Nate noticed with LSR:
When inserting code for an addrec expression with a non-unit stride, be
more careful where we insert the multiply. In particular, insert the multiply
in the outermost loop we can, instead of the requested insertion point.
This allows LSR to notice the mul in the right loop, reducing it when it gets
to it. This allows it to reduce the multiply, where before it missed it.
This happens quite a bit in the test suite, for example, eliminating 2
multiplies in art, 3 in ammp, 4 in apsi, reducing from 1050 multiplies to
910 muls in galgel (!), from 877 to 859 in applu, and 36 to 30 in bzip2.
This speeds up galgel from 16.45s to 16.01s, applu from 14.21 to 13.94s and
fourinarow from 66.67s to 63.48s.
This implements Transforms/LoopStrengthReduce/nested-reduce.ll
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@24102 91177308-0d34-0410-b5e6-96231b3b80d8
2005-10-30 06:24:33 +00:00
|
|
|
InsertPtLoop = InsertPtLoop->getParentLoop();
|
2008-06-14 16:48:22 +00:00
|
|
|
} while (InsertPtLoop != L);
|
Fix a problem that Nate noticed with LSR:
When inserting code for an addrec expression with a non-unit stride, be
more careful where we insert the multiply. In particular, insert the multiply
in the outermost loop we can, instead of the requested insertion point.
This allows LSR to notice the mul in the right loop, reducing it when it gets
to it. This allows it to reduce the multiply, where before it missed it.
This happens quite a bit in the test suite, for example, eliminating 2
multiplies in art, 3 in ammp, 4 in apsi, reducing from 1050 multiplies to
910 muls in galgel (!), from 877 to 859 in applu, and 36 to 30 in bzip2.
This speeds up galgel from 16.45s to 16.01s, applu from 14.21 to 13.94s and
fourinarow from 66.67s to 63.48s.
This implements Transforms/LoopStrengthReduce/nested-reduce.ll
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@24102 91177308-0d34-0410-b5e6-96231b3b80d8
2005-10-30 06:24:33 +00:00
|
|
|
}
|
|
|
|
|
2007-04-13 05:04:18 +00:00
|
|
|
return InsertBinop(Instruction::Mul, I, F, MulInsertPt);
|
2005-07-30 00:12:19 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// If this is a chain of recurrences, turn it into a closed form, using the
|
|
|
|
// folders, then expandCodeFor the closed form. This allows the folders to
|
|
|
|
// simplify the expression without having to build a bunch of special code
|
|
|
|
// into this folder.
|
2007-10-22 18:31:58 +00:00
|
|
|
SCEVHandle IH = SE.getUnknown(I); // Get I as a "symbolic" SCEV.
|
2005-07-30 00:12:19 +00:00
|
|
|
|
2007-10-22 18:31:58 +00:00
|
|
|
SCEVHandle V = S->evaluateAtIteration(IH, SE);
|
2006-12-07 01:30:32 +00:00
|
|
|
//cerr << "Evaluated: " << *this << "\n to: " << *V << "\n";
|
2005-07-30 00:12:19 +00:00
|
|
|
|
2007-06-15 14:38:12 +00:00
|
|
|
return expand(V);
|
2005-07-30 00:12:19 +00:00
|
|
|
}
|
2007-08-20 21:17:26 +00:00
|
|
|
|
2009-04-18 17:56:28 +00:00
|
|
|
Value *SCEVExpander::visitTruncateExpr(const SCEVTruncateExpr *S) {
|
2009-04-21 01:07:12 +00:00
|
|
|
const Type *Ty = SE.getEffectiveSCEVType(S->getType());
|
2008-06-22 19:09:18 +00:00
|
|
|
Value *V = expand(S->getOperand());
|
2009-04-21 01:07:12 +00:00
|
|
|
V = InsertNoopCastOfTo(V, SE.getEffectiveSCEVType(V->getType()));
|
2009-05-01 17:13:31 +00:00
|
|
|
Instruction *I = new TruncInst(V, Ty, "tmp.", InsertPt);
|
|
|
|
InsertedValues.insert(I);
|
|
|
|
return I;
|
2008-06-22 19:09:18 +00:00
|
|
|
}
|
|
|
|
|
2009-04-18 17:56:28 +00:00
|
|
|
Value *SCEVExpander::visitZeroExtendExpr(const SCEVZeroExtendExpr *S) {
|
2009-04-21 01:07:12 +00:00
|
|
|
const Type *Ty = SE.getEffectiveSCEVType(S->getType());
|
2008-06-22 19:09:18 +00:00
|
|
|
Value *V = expand(S->getOperand());
|
2009-04-21 01:07:12 +00:00
|
|
|
V = InsertNoopCastOfTo(V, SE.getEffectiveSCEVType(V->getType()));
|
2009-05-01 17:13:31 +00:00
|
|
|
Instruction *I = new ZExtInst(V, Ty, "tmp.", InsertPt);
|
|
|
|
InsertedValues.insert(I);
|
|
|
|
return I;
|
2008-06-22 19:09:18 +00:00
|
|
|
}
|
|
|
|
|
2009-04-18 17:56:28 +00:00
|
|
|
Value *SCEVExpander::visitSignExtendExpr(const SCEVSignExtendExpr *S) {
|
2009-04-21 01:07:12 +00:00
|
|
|
const Type *Ty = SE.getEffectiveSCEVType(S->getType());
|
2008-06-22 19:09:18 +00:00
|
|
|
Value *V = expand(S->getOperand());
|
2009-04-21 01:07:12 +00:00
|
|
|
V = InsertNoopCastOfTo(V, SE.getEffectiveSCEVType(V->getType()));
|
2009-05-01 17:13:31 +00:00
|
|
|
Instruction *I = new SExtInst(V, Ty, "tmp.", InsertPt);
|
|
|
|
InsertedValues.insert(I);
|
|
|
|
return I;
|
2008-06-22 19:09:18 +00:00
|
|
|
}
|
|
|
|
|
2009-04-18 17:56:28 +00:00
|
|
|
Value *SCEVExpander::visitSMaxExpr(const SCEVSMaxExpr *S) {
|
2009-04-21 01:07:12 +00:00
|
|
|
const Type *Ty = SE.getEffectiveSCEVType(S->getType());
|
2007-11-25 22:41:31 +00:00
|
|
|
Value *LHS = expand(S->getOperand(0));
|
2009-04-21 01:07:12 +00:00
|
|
|
LHS = InsertNoopCastOfTo(LHS, Ty);
|
2007-11-25 22:41:31 +00:00
|
|
|
for (unsigned i = 1; i < S->getNumOperands(); ++i) {
|
|
|
|
Value *RHS = expand(S->getOperand(i));
|
2009-04-21 01:07:12 +00:00
|
|
|
RHS = InsertNoopCastOfTo(RHS, Ty);
|
2009-05-01 17:13:31 +00:00
|
|
|
Instruction *ICmp =
|
|
|
|
new ICmpInst(ICmpInst::ICMP_SGT, LHS, RHS, "tmp", InsertPt);
|
|
|
|
InsertedValues.insert(ICmp);
|
|
|
|
Instruction *Sel = SelectInst::Create(ICmp, LHS, RHS, "smax", InsertPt);
|
|
|
|
InsertedValues.insert(Sel);
|
|
|
|
LHS = Sel;
|
2007-11-25 22:41:31 +00:00
|
|
|
}
|
|
|
|
return LHS;
|
|
|
|
}
|
|
|
|
|
2009-04-18 17:56:28 +00:00
|
|
|
Value *SCEVExpander::visitUMaxExpr(const SCEVUMaxExpr *S) {
|
2009-04-21 01:07:12 +00:00
|
|
|
const Type *Ty = SE.getEffectiveSCEVType(S->getType());
|
2008-02-20 06:48:22 +00:00
|
|
|
Value *LHS = expand(S->getOperand(0));
|
2009-04-21 01:07:12 +00:00
|
|
|
LHS = InsertNoopCastOfTo(LHS, Ty);
|
2008-02-20 06:48:22 +00:00
|
|
|
for (unsigned i = 1; i < S->getNumOperands(); ++i) {
|
|
|
|
Value *RHS = expand(S->getOperand(i));
|
2009-04-21 01:07:12 +00:00
|
|
|
RHS = InsertNoopCastOfTo(RHS, Ty);
|
2009-05-01 17:13:31 +00:00
|
|
|
Instruction *ICmp =
|
|
|
|
new ICmpInst(ICmpInst::ICMP_UGT, LHS, RHS, "tmp", InsertPt);
|
|
|
|
InsertedValues.insert(ICmp);
|
|
|
|
Instruction *Sel = SelectInst::Create(ICmp, LHS, RHS, "umax", InsertPt);
|
|
|
|
InsertedValues.insert(Sel);
|
|
|
|
LHS = Sel;
|
2008-02-20 06:48:22 +00:00
|
|
|
}
|
|
|
|
return LHS;
|
|
|
|
}
|
|
|
|
|
2009-04-23 15:16:49 +00:00
|
|
|
Value *SCEVExpander::expandCodeFor(SCEVHandle SH, const Type *Ty) {
|
2008-06-22 19:09:18 +00:00
|
|
|
// Expand the code for this SCEV.
|
2009-04-16 03:18:22 +00:00
|
|
|
Value *V = expand(SH);
|
2009-05-19 02:15:55 +00:00
|
|
|
if (Ty) {
|
|
|
|
assert(SE.getTypeSizeInBits(Ty) == SE.getTypeSizeInBits(SH->getType()) &&
|
|
|
|
"non-trivial casts should be done with the SCEVs directly!");
|
|
|
|
V = InsertNoopCastOfTo(V, Ty);
|
|
|
|
}
|
|
|
|
return V;
|
2008-06-22 19:09:18 +00:00
|
|
|
}
|
|
|
|
|
2009-04-18 17:56:28 +00:00
|
|
|
Value *SCEVExpander::expand(const SCEV *S) {
|
2007-08-20 21:17:26 +00:00
|
|
|
// Check to see if we already expanded this.
|
|
|
|
std::map<SCEVHandle, Value*>::iterator I = InsertedExpressions.find(S);
|
|
|
|
if (I != InsertedExpressions.end())
|
|
|
|
return I->second;
|
|
|
|
|
|
|
|
Value *V = visit(S);
|
|
|
|
InsertedExpressions[S] = V;
|
|
|
|
return V;
|
|
|
|
}
|