mirror of
https://github.com/c64scene-ar/llvm-6502.git
synced 2024-07-21 02:29:22 +00:00
Masked gather and scatter - added DAGCombine visitors
and AVX-512 instruction selection patterns. All other patches, including tests will follow. http://reviews.llvm.org/D7665 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@236211 91177308-0d34-0410-b5e6-96231b3b80d8
This commit is contained in:
Elena Demikhovsky
parent
c8ee30be4f
commit
4a25fbea03
@ -208,6 +208,16 @@ def SDTMaskedLoad: SDTypeProfile<1, 3, [ // masked load
|
||||
SDTCisVec<0>, SDTCisPtrTy<1>, SDTCisVec<2>, SDTCisSameAs<0, 3>
|
||||
]>;
|
||||
|
||||
def SDTMaskedGather: SDTypeProfile<2, 3, [ // masked gather
|
||||
SDTCisVec<0>, SDTCisVec<1>, SDTCisSameAs<0, 2>, SDTCisSameAs<1, 3>,
|
||||
SDTCisPtrTy<4>, SDTCVecEltisVT<1, i1>, SDTCisSameNumEltsAs<0, 1>
|
||||
]>;
|
||||
|
||||
def SDTMaskedScatter: SDTypeProfile<1, 3, [ // masked scatter
|
||||
SDTCisVec<0>, SDTCisVec<1>, SDTCisSameAs<0, 2>, SDTCisSameNumEltsAs<0, 1>,
|
||||
SDTCVecEltisVT<0, i1>, SDTCisPtrTy<3>
|
||||
]>;
|
||||
|
||||
def SDTVecShuffle : SDTypeProfile<1, 2, [
|
||||
SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>
|
||||
]>;
|
||||
@ -480,6 +490,10 @@ def masked_store : SDNode<"ISD::MSTORE", SDTMaskedStore,
|
||||
[SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
|
||||
def masked_load : SDNode<"ISD::MLOAD", SDTMaskedLoad,
|
||||
[SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
|
||||
def masked_scatter : SDNode<"ISD::MSCATTER", SDTMaskedScatter,
|
||||
[SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
|
||||
def masked_gather : SDNode<"ISD::MGATHER", SDTMaskedGather,
|
||||
[SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
|
||||
|
||||
// Do not use ld, st directly. Use load, extload, sextload, zextload, store,
|
||||
// and truncst (see below).
|
||||
|
||||
@ -307,6 +307,8 @@ namespace {
|
||||
SDValue visitINSERT_SUBVECTOR(SDNode *N);
|
||||
SDValue visitMLOAD(SDNode *N);
|
||||
SDValue visitMSTORE(SDNode *N);
|
||||
SDValue visitMGATHER(SDNode *N);
|
||||
SDValue visitMSCATTER(SDNode *N);
|
||||
SDValue visitFP_TO_FP16(SDNode *N);
|
||||
|
||||
SDValue visitFADDForFMACombine(SDNode *N);
|
||||
@ -1382,7 +1384,9 @@ SDValue DAGCombiner::visit(SDNode *N) {
|
||||
case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
|
||||
case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N);
|
||||
case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N);
|
||||
case ISD::MGATHER: return visitMGATHER(N);
|
||||
case ISD::MLOAD: return visitMLOAD(N);
|
||||
case ISD::MSCATTER: return visitMSCATTER(N);
|
||||
case ISD::MSTORE: return visitMSTORE(N);
|
||||
case ISD::FP_TO_FP16: return visitFP_TO_FP16(N);
|
||||
}
|
||||
@ -5073,6 +5077,67 @@ static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG) {
|
||||
TopHalf->isNullValue() ? RHS->getOperand(1) : LHS->getOperand(1));
|
||||
}
|
||||
|
||||
SDValue DAGCombiner::visitMSCATTER(SDNode *N) {
|
||||
|
||||
if (Level >= AfterLegalizeTypes)
|
||||
return SDValue();
|
||||
|
||||
MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
|
||||
SDValue Mask = MSC->getMask();
|
||||
SDValue Data = MSC->getValue();
|
||||
SDLoc DL(N);
|
||||
|
||||
// If the MSCATTER data type requires splitting and the mask is provided by a
|
||||
// SETCC, then split both nodes and its operands before legalization. This
|
||||
// prevents the type legalizer from unrolling SETCC into scalar comparisons
|
||||
// and enables future optimizations (e.g. min/max pattern matching on X86).
|
||||
if (Mask.getOpcode() != ISD::SETCC)
|
||||
return SDValue();
|
||||
|
||||
// Check if any splitting is required.
|
||||
if (TLI.getTypeAction(*DAG.getContext(), Data.getValueType()) !=
|
||||
TargetLowering::TypeSplitVector)
|
||||
return SDValue();
|
||||
SDValue MaskLo, MaskHi, Lo, Hi;
|
||||
std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
|
||||
|
||||
EVT LoVT, HiVT;
|
||||
std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MSC->getValueType(0));
|
||||
|
||||
SDValue Chain = MSC->getChain();
|
||||
|
||||
EVT MemoryVT = MSC->getMemoryVT();
|
||||
unsigned Alignment = MSC->getOriginalAlignment();
|
||||
|
||||
EVT LoMemVT, HiMemVT;
|
||||
std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
|
||||
|
||||
SDValue DataLo, DataHi;
|
||||
std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
|
||||
|
||||
SDValue BasePtr = MSC->getBasePtr();
|
||||
SDValue IndexLo, IndexHi;
|
||||
std::tie(IndexLo, IndexHi) = DAG.SplitVector(MSC->getIndex(), DL);
|
||||
|
||||
MachineMemOperand *MMO = DAG.getMachineFunction().
|
||||
getMachineMemOperand(MSC->getPointerInfo(),
|
||||
MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
|
||||
Alignment, MSC->getAAInfo(), MSC->getRanges());
|
||||
|
||||
SDValue OpsLo[] = { Chain, DataLo, MaskLo, BasePtr, IndexLo };
|
||||
Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
|
||||
DL, OpsLo, MMO);
|
||||
|
||||
SDValue OpsHi[] = {Chain, DataHi, MaskHi, BasePtr, IndexHi};
|
||||
Hi = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
|
||||
DL, OpsHi, MMO);
|
||||
|
||||
AddToWorklist(Lo.getNode());
|
||||
AddToWorklist(Hi.getNode());
|
||||
|
||||
return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
|
||||
}
|
||||
|
||||
SDValue DAGCombiner::visitMSTORE(SDNode *N) {
|
||||
|
||||
if (Level >= AfterLegalizeTypes)
|
||||
@ -5147,6 +5212,83 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) {
|
||||
return SDValue();
|
||||
}
|
||||
|
||||
SDValue DAGCombiner::visitMGATHER(SDNode *N) {
|
||||
|
||||
if (Level >= AfterLegalizeTypes)
|
||||
return SDValue();
|
||||
|
||||
MaskedGatherSDNode *MGT = dyn_cast<MaskedGatherSDNode>(N);
|
||||
SDValue Mask = MGT->getMask();
|
||||
SDLoc DL(N);
|
||||
|
||||
// If the MGATHER result requires splitting and the mask is provided by a
|
||||
// SETCC, then split both nodes and its operands before legalization. This
|
||||
// prevents the type legalizer from unrolling SETCC into scalar comparisons
|
||||
// and enables future optimizations (e.g. min/max pattern matching on X86).
|
||||
|
||||
if (Mask.getOpcode() != ISD::SETCC)
|
||||
return SDValue();
|
||||
|
||||
EVT VT = N->getValueType(0);
|
||||
|
||||
// Check if any splitting is required.
|
||||
if (TLI.getTypeAction(*DAG.getContext(), VT) !=
|
||||
TargetLowering::TypeSplitVector)
|
||||
return SDValue();
|
||||
|
||||
SDValue MaskLo, MaskHi, Lo, Hi;
|
||||
std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG);
|
||||
|
||||
SDValue Src0 = MGT->getValue();
|
||||
SDValue Src0Lo, Src0Hi;
|
||||
std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, DL);
|
||||
|
||||
EVT LoVT, HiVT;
|
||||
std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
|
||||
|
||||
SDValue Chain = MGT->getChain();
|
||||
EVT MemoryVT = MGT->getMemoryVT();
|
||||
unsigned Alignment = MGT->getOriginalAlignment();
|
||||
|
||||
EVT LoMemVT, HiMemVT;
|
||||
std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
|
||||
|
||||
SDValue BasePtr = MGT->getBasePtr();
|
||||
SDValue Index = MGT->getIndex();
|
||||
SDValue IndexLo, IndexHi;
|
||||
std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
|
||||
|
||||
MachineMemOperand *MMO = DAG.getMachineFunction().
|
||||
getMachineMemOperand(MGT->getPointerInfo(),
|
||||
MachineMemOperand::MOLoad, LoMemVT.getStoreSize(),
|
||||
Alignment, MGT->getAAInfo(), MGT->getRanges());
|
||||
|
||||
SDValue OpsLo[] = { Chain, Src0Lo, MaskLo, BasePtr, IndexLo };
|
||||
Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, DL, OpsLo,
|
||||
MMO);
|
||||
|
||||
SDValue OpsHi[] = {Chain, Src0Hi, MaskHi, BasePtr, IndexHi};
|
||||
Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, DL, OpsHi,
|
||||
MMO);
|
||||
|
||||
AddToWorklist(Lo.getNode());
|
||||
AddToWorklist(Hi.getNode());
|
||||
|
||||
// Build a factor node to remember that this load is independent of the
|
||||
// other one.
|
||||
Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
|
||||
Hi.getValue(1));
|
||||
|
||||
// Legalized the chain result - switch anything that used the old chain to
|
||||
// use the new one.
|
||||
DAG.ReplaceAllUsesOfValueWith(SDValue(MGT, 1), Chain);
|
||||
|
||||
SDValue GatherRes = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
|
||||
|
||||
SDValue RetOps[] = { GatherRes, Chain };
|
||||
return DAG.getMergeValues(RetOps, DL);
|
||||
}
|
||||
|
||||
SDValue DAGCombiner::visitMLOAD(SDNode *N) {
|
||||
|
||||
if (Level >= AfterLegalizeTypes)
|
||||
|
||||
@ -273,6 +273,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
|
||||
case ISD::STORE: return "store";
|
||||
case ISD::MLOAD: return "masked_load";
|
||||
case ISD::MSTORE: return "masked_store";
|
||||
case ISD::MGATHER: return "masked_gather";
|
||||
case ISD::MSCATTER: return "masked_scatter";
|
||||
case ISD::VAARG: return "vaarg";
|
||||
case ISD::VACOPY: return "vacopy";
|
||||
case ISD::VAEND: return "vaend";
|
||||
|
||||
@ -204,6 +204,9 @@ namespace {
|
||||
bool SelectAddr(SDNode *Parent, SDValue N, SDValue &Base,
|
||||
SDValue &Scale, SDValue &Index, SDValue &Disp,
|
||||
SDValue &Segment);
|
||||
bool SelectVectorAddr(SDNode *Parent, SDValue N, SDValue &Base,
|
||||
SDValue &Scale, SDValue &Index, SDValue &Disp,
|
||||
SDValue &Segment);
|
||||
bool SelectMOV64Imm32(SDValue N, SDValue &Imm);
|
||||
bool SelectLEAAddr(SDValue N, SDValue &Base,
|
||||
SDValue &Scale, SDValue &Index, SDValue &Disp,
|
||||
@ -1319,6 +1322,41 @@ bool X86DAGToDAGISel::MatchAddressBase(SDValue N, X86ISelAddressMode &AM) {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool X86DAGToDAGISel::SelectVectorAddr(SDNode *Parent, SDValue N, SDValue &Base,
|
||||
SDValue &Scale, SDValue &Index,
|
||||
SDValue &Disp, SDValue &Segment) {
|
||||
|
||||
MaskedGatherScatterSDNode *Mgs = dyn_cast<MaskedGatherScatterSDNode>(Parent);
|
||||
if (!Mgs)
|
||||
return false;
|
||||
X86ISelAddressMode AM;
|
||||
unsigned AddrSpace = Mgs->getPointerInfo().getAddrSpace();
|
||||
// AddrSpace 256 -> GS, 257 -> FS.
|
||||
if (AddrSpace == 256)
|
||||
AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
|
||||
if (AddrSpace == 257)
|
||||
AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
|
||||
|
||||
SDLoc DL(N);
|
||||
Base = Mgs->getBasePtr();
|
||||
Index = Mgs->getIndex();
|
||||
unsigned ScalarSize = Mgs->getValue().getValueType().getScalarSizeInBits();
|
||||
Scale = getI8Imm(ScalarSize/8, DL);
|
||||
|
||||
// If Base is 0, the whole address is in index and the Scale is 1
|
||||
if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Base)) {
|
||||
assert(C->isNullValue() && "Unexpected base in gather/scatter");
|
||||
Scale = getI8Imm(1, DL);
|
||||
Base = CurDAG->getRegister(0, MVT::i32);
|
||||
}
|
||||
if (AM.Segment.getNode())
|
||||
Segment = AM.Segment;
|
||||
else
|
||||
Segment = CurDAG->getRegister(0, MVT::i32);
|
||||
Disp = CurDAG->getTargetConstant(0, DL, MVT::i32);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// SelectAddr - returns true if it is able pattern match an addressing mode.
|
||||
/// It returns the operands which make up the maximal addressing mode it can
|
||||
/// match by reference.
|
||||
|
||||
@ -5177,74 +5177,80 @@ defm VPMOVSXDQZ: avx512_extend<0x25, "vpmovsxdq", VK8WM, VR512, VR256X, X86vsext
|
||||
//===----------------------------------------------------------------------===//
|
||||
// GATHER - SCATTER Operations
|
||||
|
||||
multiclass avx512_gather<bits<8> opc, string OpcodeStr, RegisterClass KRC,
|
||||
RegisterClass RC, X86MemOperand memop> {
|
||||
let mayLoad = 1,
|
||||
Constraints = "@earlyclobber $dst, $src1 = $dst, $mask = $mask_wb" in
|
||||
def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst, KRC:$mask_wb),
|
||||
(ins RC:$src1, KRC:$mask, memop:$src2),
|
||||
multiclass avx512_gather<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
|
||||
X86MemOperand memop, PatFrag GatherNode> {
|
||||
let Constraints = "@earlyclobber $dst, $src1 = $dst, $mask = $mask_wb" in
|
||||
def rm : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst, _.KRCWM:$mask_wb),
|
||||
(ins _.RC:$src1, _.KRCWM:$mask, memop:$src2),
|
||||
!strconcat(OpcodeStr,
|
||||
"\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"),
|
||||
[]>, EVEX, EVEX_K;
|
||||
[(set _.RC:$dst, _.KRCWM:$mask_wb,
|
||||
(GatherNode (_.VT _.RC:$src1), _.KRCWM:$mask,
|
||||
vectoraddr:$src2))]>, EVEX, EVEX_K,
|
||||
EVEX_CD8<_.EltSize, CD8VT1>;
|
||||
}
|
||||
|
||||
let ExeDomain = SSEPackedDouble in {
|
||||
defm VGATHERDPDZ : avx512_gather<0x92, "vgatherdpd", VK8WM, VR512, vy64xmem>,
|
||||
EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
|
||||
defm VGATHERQPDZ : avx512_gather<0x93, "vgatherqpd", VK8WM, VR512, vz64mem>,
|
||||
EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
|
||||
defm VGATHERDPDZ : avx512_gather<0x92, "vgatherdpd", v8f64_info, vy64xmem,
|
||||
mgatherv8i32>, EVEX_V512, VEX_W;
|
||||
defm VGATHERQPDZ : avx512_gather<0x93, "vgatherqpd", v8f64_info, vz64mem,
|
||||
mgatherv8i64>, EVEX_V512, VEX_W;
|
||||
}
|
||||
|
||||
let ExeDomain = SSEPackedSingle in {
|
||||
defm VGATHERDPSZ : avx512_gather<0x92, "vgatherdps", VK16WM, VR512, vz32mem>,
|
||||
EVEX_V512, EVEX_CD8<32, CD8VT1>;
|
||||
defm VGATHERQPSZ : avx512_gather<0x93, "vgatherqps", VK8WM, VR256X, vz64mem>,
|
||||
EVEX_V512, EVEX_CD8<32, CD8VT1>;
|
||||
defm VGATHERDPSZ : avx512_gather<0x92, "vgatherdps", v16f32_info, vz32mem,
|
||||
mgatherv16i32>, EVEX_V512;
|
||||
defm VGATHERQPSZ : avx512_gather<0x93, "vgatherqps", v8f32x_info, vz64mem,
|
||||
mgatherv8i64>, EVEX_V512;
|
||||
}
|
||||
|
||||
defm VPGATHERDQZ : avx512_gather<0x90, "vpgatherdq", VK8WM, VR512, vy64xmem>,
|
||||
EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
|
||||
defm VPGATHERDDZ : avx512_gather<0x90, "vpgatherdd", VK16WM, VR512, vz32mem>,
|
||||
EVEX_V512, EVEX_CD8<32, CD8VT1>;
|
||||
defm VPGATHERDQZ : avx512_gather<0x90, "vpgatherdq", v8i64_info, vy64xmem,
|
||||
mgatherv8i32>, EVEX_V512, VEX_W;
|
||||
defm VPGATHERDDZ : avx512_gather<0x90, "vpgatherdd", v16i32_info, vz32mem,
|
||||
mgatherv16i32>, EVEX_V512;
|
||||
|
||||
defm VPGATHERQQZ : avx512_gather<0x91, "vpgatherqq", VK8WM, VR512, vz64mem>,
|
||||
EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
|
||||
defm VPGATHERQDZ : avx512_gather<0x91, "vpgatherqd", VK8WM, VR256X, vz64mem>,
|
||||
EVEX_V512, EVEX_CD8<32, CD8VT1>;
|
||||
defm VPGATHERQQZ : avx512_gather<0x91, "vpgatherqq", v8i64_info, vz64mem,
|
||||
mgatherv8i64>, EVEX_V512, VEX_W;
|
||||
defm VPGATHERQDZ : avx512_gather<0x91, "vpgatherqd", v8i32x_info, vz64mem,
|
||||
mgatherv8i64>, EVEX_V512;
|
||||
|
||||
multiclass avx512_scatter<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
|
||||
X86MemOperand memop, PatFrag ScatterNode> {
|
||||
|
||||
multiclass avx512_scatter<bits<8> opc, string OpcodeStr, RegisterClass KRC,
|
||||
RegisterClass RC, X86MemOperand memop> {
|
||||
let mayStore = 1, Constraints = "$mask = $mask_wb" in
|
||||
def mr : AVX5128I<opc, MRMDestMem, (outs KRC:$mask_wb),
|
||||
(ins memop:$dst, KRC:$mask, RC:$src2),
|
||||
|
||||
def mr : AVX5128I<opc, MRMDestMem, (outs _.KRCWM:$mask_wb),
|
||||
(ins memop:$dst, _.KRCWM:$mask, _.RC:$src),
|
||||
!strconcat(OpcodeStr,
|
||||
"\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"),
|
||||
[]>, EVEX, EVEX_K;
|
||||
"\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"),
|
||||
[(set _.KRCWM:$mask_wb, (ScatterNode (_.VT _.RC:$src),
|
||||
_.KRCWM:$mask, vectoraddr:$dst))]>,
|
||||
EVEX, EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>;
|
||||
}
|
||||
|
||||
let ExeDomain = SSEPackedDouble in {
|
||||
defm VSCATTERDPDZ : avx512_scatter<0xA2, "vscatterdpd", VK8WM, VR512, vy64xmem>,
|
||||
EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
|
||||
defm VSCATTERQPDZ : avx512_scatter<0xA3, "vscatterqpd", VK8WM, VR512, vz64mem>,
|
||||
EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
|
||||
defm VSCATTERDPDZ : avx512_scatter<0xA2, "vscatterdpd", v8f64_info, vy64xmem,
|
||||
mscatterv8i32>, EVEX_V512, VEX_W;
|
||||
defm VSCATTERQPDZ : avx512_scatter<0xA3, "vscatterqpd", v8f64_info, vz64mem,
|
||||
mscatterv8i64>, EVEX_V512, VEX_W;
|
||||
}
|
||||
|
||||
let ExeDomain = SSEPackedSingle in {
|
||||
defm VSCATTERDPSZ : avx512_scatter<0xA2, "vscatterdps", VK16WM, VR512, vz32mem>,
|
||||
EVEX_V512, EVEX_CD8<32, CD8VT1>;
|
||||
defm VSCATTERQPSZ : avx512_scatter<0xA3, "vscatterqps", VK8WM, VR256X, vz64mem>,
|
||||
EVEX_V512, EVEX_CD8<32, CD8VT1>;
|
||||
defm VSCATTERDPSZ : avx512_scatter<0xA2, "vscatterdps", v16f32_info, vz32mem,
|
||||
mscatterv16i32>, EVEX_V512;
|
||||
defm VSCATTERQPSZ : avx512_scatter<0xA3, "vscatterqps", v8f32x_info, vz64mem,
|
||||
mscatterv8i64>, EVEX_V512;
|
||||
}
|
||||
|
||||
defm VPSCATTERDQZ : avx512_scatter<0xA0, "vpscatterdq", VK8WM, VR512, vy64xmem>,
|
||||
EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
|
||||
defm VPSCATTERDDZ : avx512_scatter<0xA0, "vpscatterdd", VK16WM, VR512, vz32mem>,
|
||||
EVEX_V512, EVEX_CD8<32, CD8VT1>;
|
||||
defm VPSCATTERDQZ : avx512_scatter<0xA0, "vpscatterdq", v8i64_info, vy64xmem,
|
||||
mscatterv8i32>, EVEX_V512, VEX_W;
|
||||
defm VPSCATTERDDZ : avx512_scatter<0xA0, "vpscatterdd", v16i32_info, vz32mem,
|
||||
mscatterv16i32>, EVEX_V512;
|
||||
|
||||
defm VPSCATTERQQZ : avx512_scatter<0xA1, "vpscatterqq", VK8WM, VR512, vz64mem>,
|
||||
EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
|
||||
defm VPSCATTERQDZ : avx512_scatter<0xA1, "vpscatterqd", VK8WM, VR256X, vz64mem>,
|
||||
EVEX_V512, EVEX_CD8<32, CD8VT1>;
|
||||
defm VPSCATTERQQZ : avx512_scatter<0xA1, "vpscatterqq", v8i64_info, vz64mem,
|
||||
mscatterv8i64>, EVEX_V512, VEX_W;
|
||||
defm VPSCATTERQDZ : avx512_scatter<0xA1, "vpscatterqd", v8i32x_info, vz64mem,
|
||||
mscatterv8i64>, EVEX_V512;
|
||||
|
||||
// prefetch
|
||||
multiclass avx512_gather_scatter_prefetch<bits<8> opc, Format F, string OpcodeStr,
|
||||
|
||||
@ -528,6 +528,52 @@ def unalignednontemporalstore : PatFrag<(ops node:$val, node:$ptr),
|
||||
return false;
|
||||
}]>;
|
||||
|
||||
def mgatherv8i32 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
|
||||
(masked_gather node:$src1, node:$src2, node:$src3) , [{
|
||||
if (MaskedGatherSDNode *Mgt = dyn_cast<MaskedGatherSDNode>(N))
|
||||
return (Mgt->getIndex().getValueType() == MVT::v8i32 ||
|
||||
Mgt->getBasePtr().getValueType() == MVT::v8i32);
|
||||
return false;
|
||||
}]>;
|
||||
|
||||
def mgatherv8i64 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
|
||||
(masked_gather node:$src1, node:$src2, node:$src3) , [{
|
||||
if (MaskedGatherSDNode *Mgt = dyn_cast<MaskedGatherSDNode>(N))
|
||||
return (Mgt->getIndex().getValueType() == MVT::v8i64 ||
|
||||
Mgt->getBasePtr().getValueType() == MVT::v8i64);
|
||||
return false;
|
||||
}]>;
|
||||
def mgatherv16i32 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
|
||||
(masked_gather node:$src1, node:$src2, node:$src3) , [{
|
||||
if (MaskedGatherSDNode *Mgt = dyn_cast<MaskedGatherSDNode>(N))
|
||||
return (Mgt->getIndex().getValueType() == MVT::v16i32 ||
|
||||
Mgt->getBasePtr().getValueType() == MVT::v16i32);
|
||||
return false;
|
||||
}]>;
|
||||
|
||||
def mscatterv8i32 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
|
||||
(masked_scatter node:$src1, node:$src2, node:$src3) , [{
|
||||
if (MaskedScatterSDNode *Sc = dyn_cast<MaskedScatterSDNode>(N))
|
||||
return (Sc->getIndex().getValueType() == MVT::v8i32 ||
|
||||
Sc->getBasePtr().getValueType() == MVT::v8i32);
|
||||
return false;
|
||||
}]>;
|
||||
|
||||
def mscatterv8i64 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
|
||||
(masked_scatter node:$src1, node:$src2, node:$src3) , [{
|
||||
if (MaskedScatterSDNode *Sc = dyn_cast<MaskedScatterSDNode>(N))
|
||||
return (Sc->getIndex().getValueType() == MVT::v8i64 ||
|
||||
Sc->getBasePtr().getValueType() == MVT::v8i64);
|
||||
return false;
|
||||
}]>;
|
||||
def mscatterv16i32 : PatFrag<(ops node:$src1, node:$src2, node:$src3),
|
||||
(masked_scatter node:$src1, node:$src2, node:$src3) , [{
|
||||
if (MaskedScatterSDNode *Sc = dyn_cast<MaskedScatterSDNode>(N))
|
||||
return (Sc->getIndex().getValueType() == MVT::v16i32 ||
|
||||
Sc->getBasePtr().getValueType() == MVT::v16i32);
|
||||
return false;
|
||||
}]>;
|
||||
|
||||
// 128-bit bitconvert pattern fragments
|
||||
def bc_v4f32 : PatFrag<(ops node:$in), (v4f32 (bitconvert node:$in))>;
|
||||
def bc_v2f64 : PatFrag<(ops node:$in), (v2f64 (bitconvert node:$in))>;
|
||||
|
||||
@ -716,6 +716,8 @@ def tls64addr : ComplexPattern<i64, 5, "SelectTLSADDRAddr",
|
||||
def tls64baseaddr : ComplexPattern<i64, 5, "SelectTLSADDRAddr",
|
||||
[tglobaltlsaddr], []>;
|
||||
|
||||
def vectoraddr : ComplexPattern<iPTR, 5, "SelectVectorAddr", [],[SDNPWantParent]>;
|
||||
|
||||
//===----------------------------------------------------------------------===//
|
||||
// X86 Instruction Predicate Definitions.
|
||||
def HasCMov : Predicate<"Subtarget->hasCMov()">;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user