mirror of
https://github.com/c64scene-ar/llvm-6502.git
synced 2024-10-01 10:57:21 +00:00
AVX-512: Optimized vector shuffle for v16f32 and v16i32 types.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@238743 91177308-0d34-0410-b5e6-96231b3b80d8
This commit is contained in:
Elena Demikhovsky
parent
68f83e59f7
commit
bbd7cab2b9
@ -10167,42 +10167,16 @@ static SDValue lowerV8X64VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
|
||||
return DAG.getNode(X86ISD::VPERMV3, DL, VT, MaskNode, V1, V2);
|
||||
}
|
||||
|
||||
/// \brief Handle lowering of 16-lane 32-bit floating point shuffles.
|
||||
static SDValue lowerV16F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
|
||||
const X86Subtarget *Subtarget,
|
||||
SelectionDAG &DAG) {
|
||||
SDLoc DL(Op);
|
||||
assert(V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");
|
||||
assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");
|
||||
ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
|
||||
ArrayRef<int> Mask = SVOp->getMask();
|
||||
assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
|
||||
|
||||
// Use dedicated unpack instructions for masks that match their pattern.
|
||||
if (isShuffleEquivalent(V1, V2, Mask,
|
||||
{// First 128-bit lane.
|
||||
0, 16, 1, 17, 4, 20, 5, 21,
|
||||
// Second 128-bit lane.
|
||||
8, 24, 9, 25, 12, 28, 13, 29}))
|
||||
return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16f32, V1, V2);
|
||||
if (isShuffleEquivalent(V1, V2, Mask,
|
||||
{// First 128-bit lane.
|
||||
2, 18, 3, 19, 6, 22, 7, 23,
|
||||
// Second 128-bit lane.
|
||||
10, 26, 11, 27, 14, 30, 15, 31}))
|
||||
return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16f32, V1, V2);
|
||||
|
||||
// FIXME: Implement direct support for this type!
|
||||
return splitAndLowerVectorShuffle(DL, MVT::v16f32, V1, V2, Mask, DAG);
|
||||
}
|
||||
|
||||
/// \brief Handle lowering of 16-lane 32-bit integer shuffles.
|
||||
static SDValue lowerV16I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
|
||||
static SDValue lowerV16X32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
|
||||
const X86Subtarget *Subtarget,
|
||||
SelectionDAG &DAG) {
|
||||
MVT VT = Op.getSimpleValueType();
|
||||
SDLoc DL(Op);
|
||||
assert(V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");
|
||||
assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");
|
||||
assert((V1.getSimpleValueType() == MVT::v16i32 ||
|
||||
V1.getSimpleValueType() == MVT::v16f32) && "Bad operand type!");
|
||||
assert((V2.getSimpleValueType() == MVT::v16i32 ||
|
||||
V2.getSimpleValueType() == MVT::v16f32) && "Bad operand type!");
|
||||
ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
|
||||
ArrayRef<int> Mask = SVOp->getMask();
|
||||
assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
|
||||
@ -10213,16 +10187,55 @@ static SDValue lowerV16I32VectorShuffle(SDValue Op, SDValue V1, SDValue V2,
|
||||
0, 16, 1, 17, 4, 20, 5, 21,
|
||||
// Second 128-bit lane.
|
||||
8, 24, 9, 25, 12, 28, 13, 29}))
|
||||
return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i32, V1, V2);
|
||||
return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);
|
||||
if (isShuffleEquivalent(V1, V2, Mask,
|
||||
{// First 128-bit lane.
|
||||
2, 18, 3, 19, 6, 22, 7, 23,
|
||||
// Second 128-bit lane.
|
||||
10, 26, 11, 27, 14, 30, 15, 31}))
|
||||
return DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i32, V1, V2);
|
||||
return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);
|
||||
|
||||
// FIXME: Implement direct support for this type!
|
||||
return splitAndLowerVectorShuffle(DL, MVT::v16i32, V1, V2, Mask, DAG);
|
||||
if (isShuffleEquivalent(V1, V2, Mask, {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10,
|
||||
12, 12, 14, 14}))
|
||||
return DAG.getNode(X86ISD::MOVSLDUP, DL, VT, V1);
|
||||
if (isShuffleEquivalent(V1, V2, Mask, {1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11,
|
||||
13, 13, 15, 15}))
|
||||
return DAG.getNode(X86ISD::MOVSHDUP, DL, VT, V1);
|
||||
|
||||
SmallVector<int, 4> RepeatedMask;
|
||||
if (is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask)) {
|
||||
unsigned Immediate = 0;
|
||||
for (int i = 0; i < 4; ++i)
|
||||
if (RepeatedMask[i] > 0)
|
||||
Immediate |= (RepeatedMask[i] & 3) << (i*2);
|
||||
|
||||
if (isSingleInputShuffleMask(Mask)) {
|
||||
unsigned Opc = VT.isInteger() ? X86ISD::PSHUFD : X86ISD::VPERMILPI;
|
||||
return DAG.getNode(Opc, DL, VT, V1,
|
||||
DAG.getConstant(Immediate, DL, MVT::i8));
|
||||
}
|
||||
|
||||
// VSHUFPS pattern: 0-3, 0-3, 16-19, 16-19, 4-7, 4-7, 20-23, 20-23 ..
|
||||
bool InterleavedMask = true;
|
||||
for (int i = 0; i < 4; ++i)
|
||||
if (RepeatedMask[i] >= 0 &&
|
||||
((i < 2 && RepeatedMask[i] > 2) || ( i >=2 && RepeatedMask[i] < 16)))
|
||||
InterleavedMask = false;
|
||||
|
||||
if (InterleavedMask)
|
||||
return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
|
||||
DAG.getConstant(Immediate, DL, MVT::i8));
|
||||
}
|
||||
SDValue VPermMask[16];
|
||||
for (int i = 0; i < 16; ++i)
|
||||
VPermMask[i] = Mask[i] < 0 ? DAG.getUNDEF(MVT::i32)
|
||||
: DAG.getConstant(Mask[i], DL, MVT::i32);
|
||||
SDValue MaskNode = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i32,
|
||||
VPermMask);
|
||||
if (V2.getOpcode() == ISD::UNDEF)
|
||||
return DAG.getNode(X86ISD::VPERMV, DL, VT, MaskNode, V1);
|
||||
|
||||
return DAG.getNode(X86ISD::VPERMV3, DL, VT, MaskNode, V1, V2);
|
||||
}
|
||||
|
||||
/// \brief Handle lowering of 32-lane 16-bit integer shuffles.
|
||||
@ -10285,9 +10298,8 @@ static SDValue lower512BitVectorShuffle(SDValue Op, SDValue V1, SDValue V2,
|
||||
case MVT::v8i64:
|
||||
return lowerV8X64VectorShuffle(Op, V1, V2, Subtarget, DAG);
|
||||
case MVT::v16f32:
|
||||
return lowerV16F32VectorShuffle(Op, V1, V2, Subtarget, DAG);
|
||||
case MVT::v16i32:
|
||||
return lowerV16I32VectorShuffle(Op, V1, V2, Subtarget, DAG);
|
||||
return lowerV16X32VectorShuffle(Op, V1, V2, Subtarget, DAG);
|
||||
case MVT::v32i16:
|
||||
if (Subtarget->hasBWI())
|
||||
return lowerV32I16VectorShuffle(Op, V1, V2, Subtarget, DAG);
|
||||
|
||||
@ -2,13 +2,9 @@
|
||||
|
||||
define <16 x i32> @test1(i32* %x) {
|
||||
; CHECK-LABEL: test1:
|
||||
; CHECK: ## BB#0:
|
||||
; CHECK-NEXT: vmovd (%rdi), %xmm0
|
||||
; CHECK-NEXT: vinserti128 $1, %xmm0, %ymm0, %ymm0
|
||||
; CHECK-NEXT: vpxor %ymm1, %ymm1, %ymm1
|
||||
; CHECK-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7]
|
||||
; CHECK-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
|
||||
; CHECK-NEXT: retq
|
||||
; CHECK: vmovd (%rdi), %xmm
|
||||
; CHECK: vmovdqa32
|
||||
; CHECK: vpermt2d %zmm
|
||||
%y = load i32, i32* %x, align 4
|
||||
%res = insertelement <16 x i32>zeroinitializer, i32 %y, i32 4
|
||||
ret <16 x i32>%res
|
||||
|
||||
@ -56,6 +56,48 @@ define <8 x i64> @test7(<8 x i64> %a, <8 x i64> %b) nounwind {
|
||||
ret <8 x i64> %c
|
||||
}
|
||||
|
||||
; CHECK-LABEL: test8:
|
||||
; CHECK: vpermt2d
|
||||
; CHECK: ret
|
||||
define <16 x i32> @test8(<16 x i32> %a, <16 x i32> %b) nounwind {
|
||||
%c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 15, i32 31, i32 14, i32 22, i32 13, i32 29, i32 4, i32 28, i32 11, i32 27, i32 10, i32 26, i32 9, i32 25, i32 8, i32 24>
|
||||
ret <16 x i32> %c
|
||||
}
|
||||
|
||||
; CHECK-LABEL: test9:
|
||||
; CHECK: vpermt2ps
|
||||
; CHECK: ret
|
||||
define <16 x float> @test9(<16 x float> %a, <16 x float> %b) nounwind {
|
||||
%c = shufflevector <16 x float> %a, <16 x float> %b, <16 x i32> <i32 15, i32 31, i32 14, i32 22, i32 13, i32 29, i32 4, i32 28, i32 11, i32 27, i32 10, i32 26, i32 9, i32 25, i32 8, i32 24>
|
||||
ret <16 x float> %c
|
||||
}
|
||||
|
||||
; CHECK-LABEL: test10:
|
||||
; CHECK: vpermt2ps (
|
||||
; CHECK: ret
|
||||
define <16 x float> @test10(<16 x float> %a, <16 x float>* %b) nounwind {
|
||||
%c = load <16 x float>, <16 x float>* %b
|
||||
%d = shufflevector <16 x float> %a, <16 x float> %c, <16 x i32> <i32 15, i32 31, i32 14, i32 22, i32 13, i32 29, i32 4, i32 28, i32 11, i32 27, i32 10, i32 26, i32 9, i32 25, i32 8, i32 24>
|
||||
ret <16 x float> %d
|
||||
}
|
||||
|
||||
; CHECK-LABEL: test11:
|
||||
; CHECK: vpermt2d
|
||||
; CHECK: ret
|
||||
define <16 x i32> @test11(<16 x i32> %a, <16 x i32>* %b) nounwind {
|
||||
%c = load <16 x i32>, <16 x i32>* %b
|
||||
%d = shufflevector <16 x i32> %a, <16 x i32> %c, <16 x i32> <i32 15, i32 31, i32 14, i32 22, i32 13, i32 29, i32 4, i32 28, i32 11, i32 27, i32 10, i32 26, i32 9, i32 25, i32 8, i32 24>
|
||||
ret <16 x i32> %d
|
||||
}
|
||||
|
||||
; CHECK-LABEL: test13
|
||||
; CHECK: vpermilps $177, %zmm
|
||||
; CHECK: ret
|
||||
define <16 x float> @test13(<16 x float> %a) {
|
||||
%b = shufflevector <16 x float> %a, <16 x float> undef, <16 x i32><i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
|
||||
ret <16 x float> %b
|
||||
}
|
||||
|
||||
; CHECK-LABEL: test14
|
||||
; CHECK: vpermilpd $203, %zmm
|
||||
; CHECK: ret
|
||||
@ -64,6 +106,15 @@ define <8 x double> @test14(<8 x double> %a) {
|
||||
ret <8 x double> %b
|
||||
}
|
||||
|
||||
; CHECK-LABEL: test15
|
||||
; CHECK: vpshufd $177, %zmm
|
||||
; CHECK: ret
|
||||
define <16 x i32> @test15(<16 x i32> %a) {
|
||||
; mask 1-0-3-2 = 10110001 = 0xb1 = 177
|
||||
%b = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32><i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
|
||||
ret <16 x i32> %b
|
||||
}
|
||||
|
||||
; CHECK-LABEL: test17
|
||||
; CHECK: vshufpd $19, %zmm1, %zmm0
|
||||
; CHECK: ret
|
||||
@ -72,6 +123,22 @@ define <8 x double> @test17(<8 x double> %a, <8 x double> %b) nounwind {
|
||||
ret <8 x double> %c
|
||||
}
|
||||
|
||||
; CHECK-LABEL: test18
|
||||
; CHECK: vpunpckhdq %zmm
|
||||
; CHECK: ret
|
||||
define <16 x i32> @test18(<16 x i32> %a, <16 x i32> %c) {
|
||||
%b = shufflevector <16 x i32> %a, <16 x i32> %c, <16 x i32><i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
|
||||
ret <16 x i32> %b
|
||||
}
|
||||
|
||||
; CHECK-LABEL: test19
|
||||
; CHECK: vpunpckldq %zmm
|
||||
; CHECK: ret
|
||||
define <16 x i32> @test19(<16 x i32> %a, <16 x i32> %c) {
|
||||
%b = shufflevector <16 x i32> %a, <16 x i32> %c, <16 x i32><i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
|
||||
ret <16 x i32> %b
|
||||
}
|
||||
|
||||
; CHECK-LABEL: test20
|
||||
; CHECK: vpunpckhqdq %zmm
|
||||
; CHECK: ret
|
||||
@ -96,3 +163,58 @@ define <8 x i64> @test22(<8 x i64> %a, <8 x i64> %b) {
|
||||
ret <8 x i64> %shuffle
|
||||
}
|
||||
|
||||
; CHECK-LABEL: @test24
|
||||
; CHECK: vpermt2d
|
||||
; CHECK: ret
|
||||
define <16 x i32> @test24(<16 x i32> %a, <16 x i32> %b) nounwind {
|
||||
%c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 19, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
|
||||
ret <16 x i32> %c
|
||||
}
|
||||
|
||||
; CHECK-LABEL: @test25
|
||||
; CHECK: vshufps $52
|
||||
; CHECK: ret
|
||||
define <16 x i32> @test25(<16 x i32> %a, <16 x i32> %b) nounwind {
|
||||
; mask - 0-1-3-undef 00110100 = 0x34 = 52
|
||||
%c = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 19, i32 undef, i32 4, i32 5, i32 23, i32 undef, i32 8, i32 9, i32 27, i32 undef, i32 12, i32 13, i32 undef, i32 undef>
|
||||
ret <16 x i32> %c
|
||||
}
|
||||
|
||||
; CHECK-LABEL: @test26
|
||||
; CHECK: vmovshdup
|
||||
; CHECK: ret
|
||||
define <16 x i32> @test26(<16 x i32> %a) nounwind {
|
||||
%c = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 undef, i32 9, i32 9, i32 undef, i32 11, i32 13, i32 undef, i32 undef, i32 undef>
|
||||
ret <16 x i32> %c
|
||||
}
|
||||
|
||||
; CHECK-LABEL: @test27
|
||||
; CHECK: ret
|
||||
define <16 x i32> @test27(<4 x i32>%a) {
|
||||
%res = shufflevector <4 x i32> %a, <4 x i32> undef, <16 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
|
||||
ret <16 x i32> %res
|
||||
}
|
||||
|
||||
; CHECK-LABEL: test28
|
||||
; CHECK: vpshufhw $177, %ymm
|
||||
; CHECK: ret
|
||||
define <16 x i16> @test28(<16 x i16> %a) {
|
||||
%b = shufflevector <16 x i16> %a, <16 x i16> undef, <16 x i32><i32 0, i32 1, i32 2, i32 3, i32 5, i32 4, i32 7, i32 6, i32 8, i32 9, i32 10, i32 11, i32 13, i32 12, i32 15, i32 14>
|
||||
ret <16 x i16> %b
|
||||
}
|
||||
|
||||
; CHECK-LABEL: test29
|
||||
; CHECK: vunpcklps %zmm
|
||||
; CHECK: ret
|
||||
define <16 x float> @test29(<16 x float> %a, <16 x float> %c) {
|
||||
%b = shufflevector <16 x float> %a, <16 x float> %c, <16 x i32><i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
|
||||
ret <16 x float> %b
|
||||
}
|
||||
|
||||
; CHECK-LABEL: @test30
|
||||
; CHECK: vshufps $144, %zmm
|
||||
; CHECK: ret
|
||||
define <16 x float> @test30(<16 x float> %a, <16 x float> %c) {
|
||||
%b = shufflevector <16 x float> %a, <16 x float> %c, <16 x i32><i32 0, i32 0, i32 17, i32 18, i32 4, i32 4, i32 21, i32 22, i32 8, i32 8, i32 25, i32 26, i32 12, i32 12, i32 29, i32 30>
|
||||
ret <16 x float> %b
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user