mirror of
https://github.com/c64scene-ar/llvm-6502.git
synced 2024-09-07 16:54:48 +00:00
Make use of vector load and store operations to implement memcpy, memmove, and memset. Currently only X86 target is taking advantage of these.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@51140 91177308-0d34-0410-b5e6-96231b3b80d8
This commit is contained in:
parent
38503d4046
commit
f0df03134e
@ -210,6 +210,13 @@ public:
|
|||||||
return Objects[ObjectIdx+NumFixedObjects].Alignment;
|
return Objects[ObjectIdx+NumFixedObjects].Alignment;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// setObjectAlignment - Change the alignment of the spcified stack object...
|
||||||
|
void setObjectAlignment(int ObjectIdx, unsigned Align) {
|
||||||
|
assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
|
||||||
|
"Invalid Object Idx!");
|
||||||
|
Objects[ObjectIdx+NumFixedObjects].Alignment = Align;
|
||||||
|
}
|
||||||
|
|
||||||
/// getObjectOffset - Return the assigned stack offset of the specified object
|
/// getObjectOffset - Return the assigned stack offset of the specified object
|
||||||
/// from the incoming stack pointer.
|
/// from the incoming stack pointer.
|
||||||
///
|
///
|
||||||
|
@ -511,6 +511,15 @@ public:
|
|||||||
return allowUnalignedMemoryAccesses;
|
return allowUnalignedMemoryAccesses;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// getOptimalMemOpType - Returns the target specific optimal type for load
|
||||||
|
/// store operations as result of memset, memcpy, and memmove lowering.
|
||||||
|
/// It returns MVT::iAny if SelectionDAG should be responsible for
|
||||||
|
/// determining it.
|
||||||
|
virtual MVT::ValueType getOptimalMemOpType(uint64_t Size, unsigned Align,
|
||||||
|
bool isSrcConst, bool isSrcStr) const {
|
||||||
|
return MVT::iAny;
|
||||||
|
}
|
||||||
|
|
||||||
/// usesUnderscoreSetJmp - Determine if we should use _setjmp or setjmp
|
/// usesUnderscoreSetJmp - Determine if we should use _setjmp or setjmp
|
||||||
/// to implement llvm.setjmp.
|
/// to implement llvm.setjmp.
|
||||||
bool usesUnderscoreSetJmp() const {
|
bool usesUnderscoreSetJmp() const {
|
||||||
|
@ -2505,41 +2505,42 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT,
|
|||||||
/// operand.
|
/// operand.
|
||||||
static SDOperand getMemsetValue(SDOperand Value, MVT::ValueType VT,
|
static SDOperand getMemsetValue(SDOperand Value, MVT::ValueType VT,
|
||||||
SelectionDAG &DAG) {
|
SelectionDAG &DAG) {
|
||||||
MVT::ValueType CurVT = VT;
|
unsigned NumBits = MVT::isVector(VT) ?
|
||||||
|
MVT::getSizeInBits(MVT::getVectorElementType(VT)) : MVT::getSizeInBits(VT);
|
||||||
if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
|
if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
|
||||||
uint64_t Val = C->getValue() & 255;
|
APInt Val = APInt(NumBits, C->getValue() & 255);
|
||||||
unsigned Shift = 8;
|
unsigned Shift = 8;
|
||||||
while (CurVT != MVT::i8) {
|
for (unsigned i = NumBits; i > 8; i >>= 1) {
|
||||||
Val = (Val << Shift) | Val;
|
Val = (Val << Shift) | Val;
|
||||||
Shift <<= 1;
|
Shift <<= 1;
|
||||||
CurVT = (MVT::ValueType)((unsigned)CurVT - 1);
|
|
||||||
}
|
}
|
||||||
|
if (MVT::isInteger(VT))
|
||||||
return DAG.getConstant(Val, VT);
|
return DAG.getConstant(Val, VT);
|
||||||
} else {
|
return DAG.getConstantFP(APFloat(Val), VT);
|
||||||
|
}
|
||||||
|
|
||||||
Value = DAG.getNode(ISD::ZERO_EXTEND, VT, Value);
|
Value = DAG.getNode(ISD::ZERO_EXTEND, VT, Value);
|
||||||
unsigned Shift = 8;
|
unsigned Shift = 8;
|
||||||
while (CurVT != MVT::i8) {
|
for (unsigned i = NumBits; i > 8; i >>= 1) {
|
||||||
Value =
|
Value = DAG.getNode(ISD::OR, VT,
|
||||||
DAG.getNode(ISD::OR, VT,
|
|
||||||
DAG.getNode(ISD::SHL, VT, Value,
|
DAG.getNode(ISD::SHL, VT, Value,
|
||||||
DAG.getConstant(Shift, MVT::i8)), Value);
|
DAG.getConstant(Shift, MVT::i8)), Value);
|
||||||
Shift <<= 1;
|
Shift <<= 1;
|
||||||
CurVT = (MVT::ValueType)((unsigned)CurVT - 1);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return Value;
|
return Value;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
|
/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
|
||||||
/// used when a memcpy is turned into a memset when the source is a constant
|
/// used when a memcpy is turned into a memset when the source is a constant
|
||||||
/// string ptr.
|
/// string ptr.
|
||||||
static SDOperand getMemsetStringVal(MVT::ValueType VT,
|
static SDOperand getMemsetStringVal(MVT::ValueType VT, SelectionDAG &DAG,
|
||||||
SelectionDAG &DAG,
|
|
||||||
const TargetLowering &TLI,
|
const TargetLowering &TLI,
|
||||||
std::string &Str, unsigned Offset) {
|
std::string &Str, unsigned Offset) {
|
||||||
|
assert(!MVT::isVector(VT) && "Can't handle vector type here!");
|
||||||
|
unsigned NumBits = MVT::getSizeInBits(VT);
|
||||||
|
unsigned MSB = NumBits / 8;
|
||||||
uint64_t Val = 0;
|
uint64_t Val = 0;
|
||||||
unsigned MSB = MVT::getSizeInBits(VT) / 8;
|
|
||||||
if (TLI.isLittleEndian())
|
if (TLI.isLittleEndian())
|
||||||
Offset = Offset + MSB - 1;
|
Offset = Offset + MSB - 1;
|
||||||
for (unsigned i = 0; i != MSB; ++i) {
|
for (unsigned i = 0; i != MSB; ++i) {
|
||||||
@ -2550,37 +2551,92 @@ static SDOperand getMemsetStringVal(MVT::ValueType VT,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// getMemBasePlusOffset - Returns base and offset node for the
|
/// getMemBasePlusOffset - Returns base and offset node for the
|
||||||
|
///
|
||||||
static SDOperand getMemBasePlusOffset(SDOperand Base, unsigned Offset,
|
static SDOperand getMemBasePlusOffset(SDOperand Base, unsigned Offset,
|
||||||
SelectionDAG &DAG) {
|
SelectionDAG &DAG) {
|
||||||
MVT::ValueType VT = Base.getValueType();
|
MVT::ValueType VT = Base.getValueType();
|
||||||
return DAG.getNode(ISD::ADD, VT, Base, DAG.getConstant(Offset, VT));
|
return DAG.getNode(ISD::ADD, VT, Base, DAG.getConstant(Offset, VT));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// isMemSrcFromString - Returns true if memcpy source is a string constant.
|
||||||
|
///
|
||||||
|
static bool isMemSrcFromString(SDOperand Src, std::string &Str,
|
||||||
|
uint64_t &SrcOff) {
|
||||||
|
unsigned SrcDelta = 0;
|
||||||
|
GlobalAddressSDNode *G = NULL;
|
||||||
|
if (Src.getOpcode() == ISD::GlobalAddress)
|
||||||
|
G = cast<GlobalAddressSDNode>(Src);
|
||||||
|
else if (Src.getOpcode() == ISD::ADD &&
|
||||||
|
Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
|
||||||
|
Src.getOperand(1).getOpcode() == ISD::Constant) {
|
||||||
|
G = cast<GlobalAddressSDNode>(Src.getOperand(0));
|
||||||
|
SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getValue();
|
||||||
|
}
|
||||||
|
if (!G)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
GlobalVariable *GV = dyn_cast<GlobalVariable>(G->getGlobal());
|
||||||
|
if (GV && GV->isConstant()) {
|
||||||
|
Str = GV->getStringValue(false);
|
||||||
|
if (!Str.empty()) {
|
||||||
|
SrcOff += SrcDelta;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
/// MeetsMaxMemopRequirement - Determines if the number of memory ops required
|
/// MeetsMaxMemopRequirement - Determines if the number of memory ops required
|
||||||
/// to replace the memset / memcpy is below the threshold. It also returns the
|
/// to replace the memset / memcpy is below the threshold. It also returns the
|
||||||
/// types of the sequence of memory ops to perform memset / memcpy.
|
/// types of the sequence of memory ops to perform memset / memcpy.
|
||||||
static bool MeetsMaxMemopRequirement(std::vector<MVT::ValueType> &MemOps,
|
static
|
||||||
unsigned Limit, uint64_t Size,
|
bool MeetsMaxMemopRequirement(std::vector<MVT::ValueType> &MemOps,
|
||||||
unsigned Align,
|
SDOperand Dst, SDOperand Src,
|
||||||
|
unsigned Limit, uint64_t Size, unsigned &Align,
|
||||||
|
SelectionDAG &DAG,
|
||||||
const TargetLowering &TLI) {
|
const TargetLowering &TLI) {
|
||||||
MVT::ValueType VT;
|
bool AllowUnalign = TLI.allowsUnalignedMemoryAccesses();
|
||||||
|
|
||||||
if (TLI.allowsUnalignedMemoryAccesses()) {
|
std::string Str;
|
||||||
|
uint64_t SrcOff = 0;
|
||||||
|
bool isSrcStr = isMemSrcFromString(Src, Str, SrcOff);
|
||||||
|
bool isSrcConst = isa<ConstantSDNode>(Src);
|
||||||
|
MVT::ValueType VT= TLI.getOptimalMemOpType(Size, Align, isSrcConst, isSrcStr);
|
||||||
|
if (VT != MVT::iAny) {
|
||||||
|
unsigned NewAlign = (unsigned)
|
||||||
|
TLI.getTargetData()->getABITypeAlignment(MVT::getTypeForValueType(VT));
|
||||||
|
// If source is a string constant, this will require an unaligned load.
|
||||||
|
if (NewAlign > Align && (isSrcConst || AllowUnalign)) {
|
||||||
|
if (Dst.getOpcode() != ISD::FrameIndex) {
|
||||||
|
// Can't change destination alignment. It requires a unaligned store.
|
||||||
|
if (AllowUnalign)
|
||||||
|
VT = MVT::iAny;
|
||||||
|
} else {
|
||||||
|
int FI = cast<FrameIndexSDNode>(Dst)->getIndex();
|
||||||
|
MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
|
||||||
|
if (MFI->isFixedObjectIndex(FI)) {
|
||||||
|
// Can't change destination alignment. It requires a unaligned store.
|
||||||
|
if (AllowUnalign)
|
||||||
|
VT = MVT::iAny;
|
||||||
|
} else {
|
||||||
|
// Give the stack frame object a larger alignment.
|
||||||
|
MFI->setObjectAlignment(FI, NewAlign);
|
||||||
|
Align = NewAlign;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (VT == MVT::iAny) {
|
||||||
|
if (AllowUnalign) {
|
||||||
VT = MVT::i64;
|
VT = MVT::i64;
|
||||||
} else {
|
} else {
|
||||||
switch (Align & 7) {
|
switch (Align & 7) {
|
||||||
case 0:
|
case 0: VT = MVT::i64; break;
|
||||||
VT = MVT::i64;
|
case 4: VT = MVT::i32; break;
|
||||||
break;
|
case 2: VT = MVT::i16; break;
|
||||||
case 4:
|
default: VT = MVT::i8; break;
|
||||||
VT = MVT::i32;
|
|
||||||
break;
|
|
||||||
case 2:
|
|
||||||
VT = MVT::i16;
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
VT = MVT::i8;
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2591,15 +2647,23 @@ static bool MeetsMaxMemopRequirement(std::vector<MVT::ValueType> &MemOps,
|
|||||||
|
|
||||||
if (VT > LVT)
|
if (VT > LVT)
|
||||||
VT = LVT;
|
VT = LVT;
|
||||||
|
}
|
||||||
|
|
||||||
unsigned NumMemOps = 0;
|
unsigned NumMemOps = 0;
|
||||||
while (Size != 0) {
|
while (Size != 0) {
|
||||||
unsigned VTSize = MVT::getSizeInBits(VT) / 8;
|
unsigned VTSize = MVT::getSizeInBits(VT) / 8;
|
||||||
while (VTSize > Size) {
|
while (VTSize > Size) {
|
||||||
|
// For now, only use non-vector load / store's for the left-over pieces.
|
||||||
|
if (MVT::isVector(VT)) {
|
||||||
|
VT = MVT::i64;
|
||||||
|
while (!TLI.isTypeLegal(VT))
|
||||||
|
VT = (MVT::ValueType)((unsigned)VT - 1);
|
||||||
|
VTSize = MVT::getSizeInBits(VT) / 8;
|
||||||
|
} else {
|
||||||
VT = (MVT::ValueType)((unsigned)VT - 1);
|
VT = (MVT::ValueType)((unsigned)VT - 1);
|
||||||
VTSize >>= 1;
|
VTSize >>= 1;
|
||||||
}
|
}
|
||||||
assert(MVT::isInteger(VT));
|
}
|
||||||
|
|
||||||
if (++NumMemOps > Limit)
|
if (++NumMemOps > Limit)
|
||||||
return false;
|
return false;
|
||||||
@ -2613,8 +2677,7 @@ static bool MeetsMaxMemopRequirement(std::vector<MVT::ValueType> &MemOps,
|
|||||||
static SDOperand getMemcpyLoadsAndStores(SelectionDAG &DAG,
|
static SDOperand getMemcpyLoadsAndStores(SelectionDAG &DAG,
|
||||||
SDOperand Chain, SDOperand Dst,
|
SDOperand Chain, SDOperand Dst,
|
||||||
SDOperand Src, uint64_t Size,
|
SDOperand Src, uint64_t Size,
|
||||||
unsigned Align,
|
unsigned Align, bool AlwaysInline,
|
||||||
bool AlwaysInline,
|
|
||||||
const Value *DstSV, uint64_t DstSVOff,
|
const Value *DstSV, uint64_t DstSVOff,
|
||||||
const Value *SrcSV, uint64_t SrcSVOff){
|
const Value *SrcSV, uint64_t SrcSVOff){
|
||||||
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
|
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
|
||||||
@ -2625,56 +2688,38 @@ static SDOperand getMemcpyLoadsAndStores(SelectionDAG &DAG,
|
|||||||
uint64_t Limit = -1;
|
uint64_t Limit = -1;
|
||||||
if (!AlwaysInline)
|
if (!AlwaysInline)
|
||||||
Limit = TLI.getMaxStoresPerMemcpy();
|
Limit = TLI.getMaxStoresPerMemcpy();
|
||||||
if (!MeetsMaxMemopRequirement(MemOps, Limit, Size, Align, TLI))
|
unsigned DstAlign = Align; // Destination alignment can change.
|
||||||
|
if (!MeetsMaxMemopRequirement(MemOps, Dst, Src, Limit, Size, DstAlign,
|
||||||
|
DAG, TLI))
|
||||||
return SDOperand();
|
return SDOperand();
|
||||||
|
|
||||||
SmallVector<SDOperand, 8> OutChains;
|
|
||||||
|
|
||||||
unsigned NumMemOps = MemOps.size();
|
|
||||||
unsigned SrcDelta = 0;
|
|
||||||
GlobalAddressSDNode *G = NULL;
|
|
||||||
std::string Str;
|
std::string Str;
|
||||||
bool CopyFromStr = false;
|
|
||||||
uint64_t SrcOff = 0, DstOff = 0;
|
uint64_t SrcOff = 0, DstOff = 0;
|
||||||
|
bool CopyFromStr = isMemSrcFromString(Src, Str, SrcOff);
|
||||||
|
|
||||||
if (Src.getOpcode() == ISD::GlobalAddress)
|
SmallVector<SDOperand, 8> OutChains;
|
||||||
G = cast<GlobalAddressSDNode>(Src);
|
unsigned NumMemOps = MemOps.size();
|
||||||
else if (Src.getOpcode() == ISD::ADD &&
|
|
||||||
Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
|
|
||||||
Src.getOperand(1).getOpcode() == ISD::Constant) {
|
|
||||||
G = cast<GlobalAddressSDNode>(Src.getOperand(0));
|
|
||||||
SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getValue();
|
|
||||||
}
|
|
||||||
if (G) {
|
|
||||||
GlobalVariable *GV = dyn_cast<GlobalVariable>(G->getGlobal());
|
|
||||||
if (GV && GV->isConstant()) {
|
|
||||||
Str = GV->getStringValue(false);
|
|
||||||
if (!Str.empty()) {
|
|
||||||
CopyFromStr = true;
|
|
||||||
SrcOff += SrcDelta;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (unsigned i = 0; i < NumMemOps; i++) {
|
for (unsigned i = 0; i < NumMemOps; i++) {
|
||||||
MVT::ValueType VT = MemOps[i];
|
MVT::ValueType VT = MemOps[i];
|
||||||
unsigned VTSize = MVT::getSizeInBits(VT) / 8;
|
unsigned VTSize = MVT::getSizeInBits(VT) / 8;
|
||||||
SDOperand Value, Store;
|
SDOperand Value, Store;
|
||||||
|
|
||||||
if (CopyFromStr) {
|
if (CopyFromStr && !MVT::isVector(VT)) {
|
||||||
|
// It's unlikely a store of a vector immediate can be done in a single
|
||||||
|
// instruction. It would require a load from a constantpool first.
|
||||||
|
// FIXME: Handle cases where store of vector immediate is done in a
|
||||||
|
// single instruction.
|
||||||
Value = getMemsetStringVal(VT, DAG, TLI, Str, SrcOff);
|
Value = getMemsetStringVal(VT, DAG, TLI, Str, SrcOff);
|
||||||
Store =
|
Store = DAG.getStore(Chain, Value,
|
||||||
DAG.getStore(Chain, Value,
|
|
||||||
getMemBasePlusOffset(Dst, DstOff, DAG),
|
getMemBasePlusOffset(Dst, DstOff, DAG),
|
||||||
DstSV, DstSVOff + DstOff);
|
DstSV, DstSVOff + DstOff);
|
||||||
} else {
|
} else {
|
||||||
Value = DAG.getLoad(VT, Chain,
|
Value = DAG.getLoad(VT, Chain,
|
||||||
getMemBasePlusOffset(Src, SrcOff, DAG),
|
getMemBasePlusOffset(Src, SrcOff, DAG),
|
||||||
SrcSV, SrcSVOff + SrcOff, false, Align);
|
SrcSV, SrcSVOff + SrcOff, false, Align);
|
||||||
Store =
|
Store = DAG.getStore(Chain, Value,
|
||||||
DAG.getStore(Chain, Value,
|
|
||||||
getMemBasePlusOffset(Dst, DstOff, DAG),
|
getMemBasePlusOffset(Dst, DstOff, DAG),
|
||||||
DstSV, DstSVOff + DstOff, false, Align);
|
DstSV, DstSVOff + DstOff, false, DstAlign);
|
||||||
}
|
}
|
||||||
OutChains.push_back(Store);
|
OutChains.push_back(Store);
|
||||||
SrcOff += VTSize;
|
SrcOff += VTSize;
|
||||||
@ -2695,8 +2740,8 @@ static SDOperand getMemsetStores(SelectionDAG &DAG,
|
|||||||
// Expand memset to a series of load/store ops if the size operand
|
// Expand memset to a series of load/store ops if the size operand
|
||||||
// falls below a certain threshold.
|
// falls below a certain threshold.
|
||||||
std::vector<MVT::ValueType> MemOps;
|
std::vector<MVT::ValueType> MemOps;
|
||||||
if (!MeetsMaxMemopRequirement(MemOps, TLI.getMaxStoresPerMemset(),
|
if (!MeetsMaxMemopRequirement(MemOps, Dst, Src, TLI.getMaxStoresPerMemset(),
|
||||||
Size, Align, TLI))
|
Size, Align, DAG, TLI))
|
||||||
return SDOperand();
|
return SDOperand();
|
||||||
|
|
||||||
SmallVector<SDOperand, 8> OutChains;
|
SmallVector<SDOperand, 8> OutChains;
|
||||||
|
@ -787,6 +787,23 @@ unsigned X86TargetLowering::getByValTypeAlignment(const Type *Ty) const {
|
|||||||
return Align;
|
return Align;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// getOptimalMemOpType - Returns the target specific optimal type for load
|
||||||
|
/// store operations as result of memset, memcpy, and memmove lowering.
|
||||||
|
/// It returns MVT::iAny if SelectionDAG should be responsible for
|
||||||
|
/// determining it.
|
||||||
|
MVT::ValueType
|
||||||
|
X86TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned Align,
|
||||||
|
bool isSrcConst, bool isSrcStr) const {
|
||||||
|
if ((isSrcConst || isSrcStr) && Subtarget->hasSSE2() && Size >= 16)
|
||||||
|
return MVT::v4i32;
|
||||||
|
if ((isSrcConst || isSrcStr) && Subtarget->hasSSE1() && Size >= 16)
|
||||||
|
return MVT::v4f32;
|
||||||
|
if (Subtarget->is64Bit() && Size >= 8)
|
||||||
|
return MVT::i64;
|
||||||
|
return MVT::i32;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/// getPICJumpTableRelocaBase - Returns relocation base for the given PIC
|
/// getPICJumpTableRelocaBase - Returns relocation base for the given PIC
|
||||||
/// jumptable.
|
/// jumptable.
|
||||||
SDOperand X86TargetLowering::getPICJumpTableRelocBase(SDOperand Table,
|
SDOperand X86TargetLowering::getPICJumpTableRelocBase(SDOperand Table,
|
||||||
@ -2738,17 +2755,23 @@ static bool isZeroShuffle(SDNode *N) {
|
|||||||
|
|
||||||
/// getZeroVector - Returns a vector of specified type with all zero elements.
|
/// getZeroVector - Returns a vector of specified type with all zero elements.
|
||||||
///
|
///
|
||||||
static SDOperand getZeroVector(MVT::ValueType VT, SelectionDAG &DAG) {
|
static SDOperand getZeroVector(MVT::ValueType VT, bool HasSSE2,
|
||||||
|
SelectionDAG &DAG) {
|
||||||
assert(MVT::isVector(VT) && "Expected a vector type");
|
assert(MVT::isVector(VT) && "Expected a vector type");
|
||||||
|
|
||||||
// Always build zero vectors as <4 x i32> or <2 x i32> bitcasted to their dest
|
// Always build zero vectors as <4 x i32> or <2 x i32> bitcasted to their dest
|
||||||
// type. This ensures they get CSE'd.
|
// type. This ensures they get CSE'd.
|
||||||
SDOperand Cst = DAG.getTargetConstant(0, MVT::i32);
|
|
||||||
SDOperand Vec;
|
SDOperand Vec;
|
||||||
if (MVT::getSizeInBits(VT) == 64) // MMX
|
if (MVT::getSizeInBits(VT) == 64) { // MMX
|
||||||
|
SDOperand Cst = DAG.getTargetConstant(0, MVT::i32);
|
||||||
Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst);
|
Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v2i32, Cst, Cst);
|
||||||
else // SSE
|
} else if (HasSSE2) { // SSE2
|
||||||
|
SDOperand Cst = DAG.getTargetConstant(0, MVT::i32);
|
||||||
Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst);
|
Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4i32, Cst, Cst, Cst, Cst);
|
||||||
|
} else { // SSE1
|
||||||
|
SDOperand Cst = DAG.getTargetConstantFP(+0.0, MVT::f32);
|
||||||
|
Vec = DAG.getNode(ISD::BUILD_VECTOR, MVT::v4f32, Cst, Cst, Cst, Cst);
|
||||||
|
}
|
||||||
return DAG.getNode(ISD::BIT_CONVERT, VT, Vec);
|
return DAG.getNode(ISD::BIT_CONVERT, VT, Vec);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2866,7 +2889,7 @@ static SDOperand PromoteSplat(SDOperand Op, SelectionDAG &DAG, bool HasSSE2) {
|
|||||||
V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, Mask);
|
V1 = DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, V1, Mask);
|
||||||
NumElems >>= 1;
|
NumElems >>= 1;
|
||||||
}
|
}
|
||||||
Mask = getZeroVector(MVT::v4i32, DAG);
|
Mask = getZeroVector(MVT::v4i32, true, DAG);
|
||||||
}
|
}
|
||||||
|
|
||||||
V1 = DAG.getNode(ISD::BIT_CONVERT, PVT, V1);
|
V1 = DAG.getNode(ISD::BIT_CONVERT, PVT, V1);
|
||||||
@ -2880,9 +2903,11 @@ static SDOperand PromoteSplat(SDOperand Op, SelectionDAG &DAG, bool HasSSE2) {
|
|||||||
/// element of V2 is swizzled into the zero/undef vector, landing at element
|
/// element of V2 is swizzled into the zero/undef vector, landing at element
|
||||||
/// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).
|
/// Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).
|
||||||
static SDOperand getShuffleVectorZeroOrUndef(SDOperand V2, unsigned Idx,
|
static SDOperand getShuffleVectorZeroOrUndef(SDOperand V2, unsigned Idx,
|
||||||
bool isZero, SelectionDAG &DAG) {
|
bool isZero, bool HasSSE2,
|
||||||
|
SelectionDAG &DAG) {
|
||||||
MVT::ValueType VT = V2.getValueType();
|
MVT::ValueType VT = V2.getValueType();
|
||||||
SDOperand V1 = isZero ? getZeroVector(VT, DAG) : DAG.getNode(ISD::UNDEF, VT);
|
SDOperand V1 = isZero
|
||||||
|
? getZeroVector(VT, HasSSE2, DAG) : DAG.getNode(ISD::UNDEF, VT);
|
||||||
unsigned NumElems = MVT::getVectorNumElements(V2.getValueType());
|
unsigned NumElems = MVT::getVectorNumElements(V2.getValueType());
|
||||||
MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
|
MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
|
||||||
MVT::ValueType EVT = MVT::getVectorElementType(MaskVT);
|
MVT::ValueType EVT = MVT::getVectorElementType(MaskVT);
|
||||||
@ -2911,7 +2936,7 @@ static SDOperand LowerBuildVectorv16i8(SDOperand Op, unsigned NonZeros,
|
|||||||
bool ThisIsNonZero = (NonZeros & (1 << i)) != 0;
|
bool ThisIsNonZero = (NonZeros & (1 << i)) != 0;
|
||||||
if (ThisIsNonZero && First) {
|
if (ThisIsNonZero && First) {
|
||||||
if (NumZero)
|
if (NumZero)
|
||||||
V = getZeroVector(MVT::v8i16, DAG);
|
V = getZeroVector(MVT::v8i16, true, DAG);
|
||||||
else
|
else
|
||||||
V = DAG.getNode(ISD::UNDEF, MVT::v8i16);
|
V = DAG.getNode(ISD::UNDEF, MVT::v8i16);
|
||||||
First = false;
|
First = false;
|
||||||
@ -2956,7 +2981,7 @@ static SDOperand LowerBuildVectorv8i16(SDOperand Op, unsigned NonZeros,
|
|||||||
if (isNonZero) {
|
if (isNonZero) {
|
||||||
if (First) {
|
if (First) {
|
||||||
if (NumZero)
|
if (NumZero)
|
||||||
V = getZeroVector(MVT::v8i16, DAG);
|
V = getZeroVector(MVT::v8i16, true, DAG);
|
||||||
else
|
else
|
||||||
V = DAG.getNode(ISD::UNDEF, MVT::v8i16);
|
V = DAG.getNode(ISD::UNDEF, MVT::v8i16);
|
||||||
First = false;
|
First = false;
|
||||||
@ -2981,7 +3006,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
|
|||||||
|
|
||||||
if (ISD::isBuildVectorAllOnes(Op.Val))
|
if (ISD::isBuildVectorAllOnes(Op.Val))
|
||||||
return getOnesVector(Op.getValueType(), DAG);
|
return getOnesVector(Op.getValueType(), DAG);
|
||||||
return getZeroVector(Op.getValueType(), DAG);
|
return getZeroVector(Op.getValueType(), Subtarget->hasSSE2(), DAG);
|
||||||
}
|
}
|
||||||
|
|
||||||
MVT::ValueType VT = Op.getValueType();
|
MVT::ValueType VT = Op.getValueType();
|
||||||
@ -3036,7 +3061,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
|
|||||||
// convert it to a vector with movd (S2V+shuffle to zero extend).
|
// convert it to a vector with movd (S2V+shuffle to zero extend).
|
||||||
Item = DAG.getNode(ISD::TRUNCATE, MVT::i32, Item);
|
Item = DAG.getNode(ISD::TRUNCATE, MVT::i32, Item);
|
||||||
Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VecVT, Item);
|
Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VecVT, Item);
|
||||||
Item = getShuffleVectorZeroOrUndef(Item, 0, true, DAG);
|
Item = getShuffleVectorZeroOrUndef(Item, 0, true,
|
||||||
|
Subtarget->hasSSE2(), DAG);
|
||||||
|
|
||||||
// Now we have our 32-bit value zero extended in the low element of
|
// Now we have our 32-bit value zero extended in the low element of
|
||||||
// a vector. If Idx != 0, swizzle it into place.
|
// a vector. If Idx != 0, swizzle it into place.
|
||||||
@ -3061,7 +3087,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
|
|||||||
(EVT != MVT::i64 || Subtarget->is64Bit())) {
|
(EVT != MVT::i64 || Subtarget->is64Bit())) {
|
||||||
Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item);
|
Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item);
|
||||||
// Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector.
|
// Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector.
|
||||||
return getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, DAG);
|
return getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0,
|
||||||
|
Subtarget->hasSSE2(), DAG);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (IsAllConstants) // Otherwise, it's better to do a constpool load.
|
if (IsAllConstants) // Otherwise, it's better to do a constpool load.
|
||||||
@ -3076,7 +3103,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
|
|||||||
Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item);
|
Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Item);
|
||||||
|
|
||||||
// Turn it into a shuffle of zero and zero-extended scalar to vector.
|
// Turn it into a shuffle of zero and zero-extended scalar to vector.
|
||||||
Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0, DAG);
|
Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0,
|
||||||
|
Subtarget->hasSSE2(), DAG);
|
||||||
MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
|
MVT::ValueType MaskVT = MVT::getIntVectorWithNumElements(NumElems);
|
||||||
MVT::ValueType MaskEVT = MVT::getVectorElementType(MaskVT);
|
MVT::ValueType MaskEVT = MVT::getVectorElementType(MaskVT);
|
||||||
SmallVector<SDOperand, 8> MaskVec;
|
SmallVector<SDOperand, 8> MaskVec;
|
||||||
@ -3105,7 +3133,8 @@ X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
|
|||||||
unsigned Idx = CountTrailingZeros_32(NonZeros);
|
unsigned Idx = CountTrailingZeros_32(NonZeros);
|
||||||
SDOperand V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT,
|
SDOperand V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT,
|
||||||
Op.getOperand(Idx));
|
Op.getOperand(Idx));
|
||||||
return getShuffleVectorZeroOrUndef(V2, Idx, true, DAG);
|
return getShuffleVectorZeroOrUndef(V2, Idx, true,
|
||||||
|
Subtarget->hasSSE2(), DAG);
|
||||||
}
|
}
|
||||||
return SDOperand();
|
return SDOperand();
|
||||||
}
|
}
|
||||||
@ -3130,7 +3159,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDOperand Op, SelectionDAG &DAG) {
|
|||||||
for (unsigned i = 0; i < 4; ++i) {
|
for (unsigned i = 0; i < 4; ++i) {
|
||||||
bool isZero = !(NonZeros & (1 << i));
|
bool isZero = !(NonZeros & (1 << i));
|
||||||
if (isZero)
|
if (isZero)
|
||||||
V[i] = getZeroVector(VT, DAG);
|
V[i] = getZeroVector(VT, Subtarget->hasSSE2(), DAG);
|
||||||
else
|
else
|
||||||
V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i));
|
V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, VT, Op.getOperand(i));
|
||||||
}
|
}
|
||||||
@ -3542,7 +3571,7 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDOperand Op, SelectionDAG &DAG) {
|
|||||||
return DAG.getNode(ISD::UNDEF, VT);
|
return DAG.getNode(ISD::UNDEF, VT);
|
||||||
|
|
||||||
if (isZeroShuffle(Op.Val))
|
if (isZeroShuffle(Op.Val))
|
||||||
return getZeroVector(VT, DAG);
|
return getZeroVector(VT, Subtarget->hasSSE2(), DAG);
|
||||||
|
|
||||||
if (isIdentityMask(PermMask.Val))
|
if (isIdentityMask(PermMask.Val))
|
||||||
return V1;
|
return V1;
|
||||||
|
@ -337,6 +337,14 @@ namespace llvm {
|
|||||||
/// 4-byte boundaries.
|
/// 4-byte boundaries.
|
||||||
virtual unsigned getByValTypeAlignment(const Type *Ty) const;
|
virtual unsigned getByValTypeAlignment(const Type *Ty) const;
|
||||||
|
|
||||||
|
/// getOptimalMemOpType - Returns the target specific optimal type for load
|
||||||
|
/// store operations as result of memset, memcpy, and memmove lowering.
|
||||||
|
/// It returns MVT::iAny if SelectionDAG should be responsible for
|
||||||
|
/// determining it.
|
||||||
|
virtual
|
||||||
|
MVT::ValueType getOptimalMemOpType(uint64_t Size, unsigned Align,
|
||||||
|
bool isSrcConst, bool isSrcStr) const;
|
||||||
|
|
||||||
/// LowerOperation - Provide custom lowering hooks for some operations.
|
/// LowerOperation - Provide custom lowering hooks for some operations.
|
||||||
///
|
///
|
||||||
virtual SDOperand LowerOperation(SDOperand Op, SelectionDAG &DAG);
|
virtual SDOperand LowerOperation(SDOperand Op, SelectionDAG &DAG);
|
||||||
|
Loading…
Reference in New Issue
Block a user