From 37bb4b0365ac299849fbd2683748bcf955a3cabf Mon Sep 17 00:00:00 2001 From: Chandler Carruth Date: Sun, 21 Sep 2014 23:46:13 +0000 Subject: [PATCH] [x86] Teach the new vector shuffle lowering how to cleverly lower single input v8f32 shuffles which are not 128-bit lane crossing but have different shuffle patterns in the low and high lanes. This removes most of the extract/insert traffic that was unnecessary and is particularly good at lowering cases where only one of the two lanes is shuffled at all. I've also added a collection of test cases with undef lanes because this lowering is somewhat more sensitive to undef lanes than others. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@218226 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Target/X86/X86ISelLowering.cpp | 25 +++- test/CodeGen/X86/vector-shuffle-256-v8.ll | 143 +++++++++++++++------- 2 files changed, 118 insertions(+), 50 deletions(-) diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index c2f8427df0a..0bd4e998999 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -9391,9 +9391,28 @@ static SDValue lowerV8F32VectorShuffle(SDValue Op, SDValue V1, SDValue V2, return lowerVectorShuffleWithSHUPFS(DL, MVT::v8f32, SHUFPSMask, V1, V2, DAG); } - if (isSingleInputShuffleMask(Mask)) - // FIXME: We can do better than just falling back blindly. - return splitAndLower256BitVectorShuffle(Op, V1, V2, Subtarget, DAG); + // If we have a single input shuffle with different shuffle patterns in the + // two 128-bit lanes, just do two shuffles and blend them together. This will + // be faster than extracting the high 128-bit lane, shuffling it, and + // re-inserting it. Especially on newer processors where blending is *the* + // fastest operation. + if (isSingleInputShuffleMask(Mask)) { + int LoMask[4] = {Mask[0], Mask[1], Mask[2], Mask[3]}; + int HiMask[4] = {Mask[4], Mask[5], Mask[6], Mask[7]}; + for (int &M : HiMask) + if (M >= 0) + M -= 4; + SDValue Lo = V1, Hi = V1; + if (!isNoopShuffleMask(LoMask)) + Lo = DAG.getNode(X86ISD::VPERMILP, DL, MVT::v8f32, Lo, + getV4X86ShuffleImm8ForMask(LoMask, DAG)); + if (!isNoopShuffleMask(HiMask)) + Hi = DAG.getNode(X86ISD::VPERMILP, DL, MVT::v8f32, Hi, + getV4X86ShuffleImm8ForMask(HiMask, DAG)); + unsigned BlendMask = 1 << 4 | 1 << 5 | 1 << 6 | 1 << 7; + return DAG.getNode(X86ISD::BLENDI, DL, MVT::v8f32, Lo, Hi, + DAG.getConstant(BlendMask, MVT::i8)); + } // Shuffle the input elements into the desired positions in V1 and V2 and // blend them together. diff --git a/test/CodeGen/X86/vector-shuffle-256-v8.ll b/test/CodeGen/X86/vector-shuffle-256-v8.ll index 34215806fb1..1335a53a32a 100644 --- a/test/CodeGen/X86/vector-shuffle-256-v8.ll +++ b/test/CodeGen/X86/vector-shuffle-256-v8.ll @@ -381,10 +381,9 @@ define <8 x float> @shuffle_v8f32_10225466(<8 x float> %a, <8 x float> %b) { define <8 x float> @shuffle_v8f32_00015444(<8 x float> %a, <8 x float> %b) { ; ALL-LABEL: @shuffle_v8f32_00015444 ; ALL: # BB#0: -; ALL-NEXT: vpermilps {{.*}} # xmm1 = xmm0[0,0,0,1] -; ALL-NEXT: vextractf128 $1, %ymm0, %xmm0 -; ALL-NEXT: vpermilps {{.*}} # xmm0 = xmm0[1,0,0,0] -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[1,0,0,0,5,4,4,4] +; ALL-NEXT: vpermilps {{.*}} # ymm0 = ymm0[0,0,0,1,4,4,4,5] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; ALL-NEXT: retq %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> ret <8 x float> %shuffle @@ -393,10 +392,9 @@ define <8 x float> @shuffle_v8f32_00015444(<8 x float> %a, <8 x float> %b) { define <8 x float> @shuffle_v8f32_00204644(<8 x float> %a, <8 x float> %b) { ; ALL-LABEL: @shuffle_v8f32_00204644 ; ALL: # BB#0: -; ALL-NEXT: vpermilps {{.*}} # xmm1 = xmm0[0,0,2,0] -; ALL-NEXT: vextractf128 $1, %ymm0, %xmm0 -; ALL-NEXT: vpermilps {{.*}} # xmm0 = xmm0[0,2,0,0] -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[0,2,0,0,4,6,4,4] +; ALL-NEXT: vpermilps {{.*}} # ymm0 = ymm0[0,0,2,0,4,4,6,4] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; ALL-NEXT: retq %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> ret <8 x float> %shuffle @@ -405,10 +403,9 @@ define <8 x float> @shuffle_v8f32_00204644(<8 x float> %a, <8 x float> %b) { define <8 x float> @shuffle_v8f32_03004474(<8 x float> %a, <8 x float> %b) { ; ALL-LABEL: @shuffle_v8f32_03004474 ; ALL: # BB#0: -; ALL-NEXT: vpermilps {{.*}} # xmm1 = xmm0[0,3,0,0] -; ALL-NEXT: vextractf128 $1, %ymm0, %xmm0 -; ALL-NEXT: vpermilps {{.*}} # xmm0 = xmm0[0,0,3,0] -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[0,0,3,0,4,4,7,4] +; ALL-NEXT: vpermilps {{.*}} # ymm0 = ymm0[0,3,0,0,4,7,4,4] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; ALL-NEXT: retq %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> ret <8 x float> %shuffle @@ -417,10 +414,9 @@ define <8 x float> @shuffle_v8f32_03004474(<8 x float> %a, <8 x float> %b) { define <8 x float> @shuffle_v8f32_10004444(<8 x float> %a, <8 x float> %b) { ; ALL-LABEL: @shuffle_v8f32_10004444 ; ALL: # BB#0: -; ALL-NEXT: vpermilps {{.*}} # xmm1 = xmm0[1,0,0,0] -; ALL-NEXT: vextractf128 $1, %ymm0, %xmm0 -; ALL-NEXT: vpermilps {{.*}} # xmm0 = xmm0[0,0,0,0] -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[0,0,0,0,4,4,4,4] +; ALL-NEXT: vpermilps {{.*}} # ymm0 = ymm0[1,0,0,0,5,4,4,4] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; ALL-NEXT: retq %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> ret <8 x float> %shuffle @@ -429,10 +425,9 @@ define <8 x float> @shuffle_v8f32_10004444(<8 x float> %a, <8 x float> %b) { define <8 x float> @shuffle_v8f32_22006446(<8 x float> %a, <8 x float> %b) { ; ALL-LABEL: @shuffle_v8f32_22006446 ; ALL: # BB#0: -; ALL-NEXT: vpermilps {{.*}} # xmm1 = xmm0[2,2,0,0] -; ALL-NEXT: vextractf128 $1, %ymm0, %xmm0 -; ALL-NEXT: vpermilps {{.*}} # xmm0 = xmm0[2,0,0,2] -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[2,0,0,2,6,4,4,6] +; ALL-NEXT: vpermilps {{.*}} # ymm0 = ymm0[2,2,0,0,6,6,4,4] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; ALL-NEXT: retq %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> ret <8 x float> %shuffle @@ -441,10 +436,9 @@ define <8 x float> @shuffle_v8f32_22006446(<8 x float> %a, <8 x float> %b) { define <8 x float> @shuffle_v8f32_33307474(<8 x float> %a, <8 x float> %b) { ; ALL-LABEL: @shuffle_v8f32_33307474 ; ALL: # BB#0: -; ALL-NEXT: vpermilps {{.*}} # xmm1 = xmm0[3,3,3,0] -; ALL-NEXT: vextractf128 $1, %ymm0, %xmm0 -; ALL-NEXT: vpermilps {{.*}} # xmm0 = xmm0[3,0,3,0] -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[3,0,3,0,7,4,7,4] +; ALL-NEXT: vpermilps {{.*}} # ymm0 = ymm0[3,3,3,0,7,7,7,4] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; ALL-NEXT: retq %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> ret <8 x float> %shuffle @@ -453,9 +447,8 @@ define <8 x float> @shuffle_v8f32_33307474(<8 x float> %a, <8 x float> %b) { define <8 x float> @shuffle_v8f32_32104567(<8 x float> %a, <8 x float> %b) { ; ALL-LABEL: @shuffle_v8f32_32104567 ; ALL: # BB#0: -; ALL-NEXT: vpermilps {{.*}} # xmm1 = xmm0[3,2,1,0] -; ALL-NEXT: vextractf128 $1, %ymm0, %xmm0 -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[3,2,1,0,7,6,5,4] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; ALL-NEXT: retq %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> ret <8 x float> %shuffle @@ -464,10 +457,9 @@ define <8 x float> @shuffle_v8f32_32104567(<8 x float> %a, <8 x float> %b) { define <8 x float> @shuffle_v8f32_00236744(<8 x float> %a, <8 x float> %b) { ; ALL-LABEL: @shuffle_v8f32_00236744 ; ALL: # BB#0: -; ALL-NEXT: vpermilps {{.*}} # xmm1 = xmm0[0,0,2,3] -; ALL-NEXT: vextractf128 $1, %ymm0, %xmm0 -; ALL-NEXT: vpermilps {{.*}} # xmm0 = xmm0[2,3,0,0] -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[2,3,0,0,6,7,4,4] +; ALL-NEXT: vpermilps {{.*}} # ymm0 = ymm0[0,0,2,3,4,4,6,7] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; ALL-NEXT: retq %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> ret <8 x float> %shuffle @@ -476,10 +468,9 @@ define <8 x float> @shuffle_v8f32_00236744(<8 x float> %a, <8 x float> %b) { define <8 x float> @shuffle_v8f32_00226644(<8 x float> %a, <8 x float> %b) { ; ALL-LABEL: @shuffle_v8f32_00226644 ; ALL: # BB#0: -; ALL-NEXT: vmovsldup {{.*}} # xmm1 = xmm0[0,0,2,2] -; ALL-NEXT: vextractf128 $1, %ymm0, %xmm0 -; ALL-NEXT: vpermilps {{.*}} # xmm0 = xmm0[2,2,0,0] -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[2,2,0,0,6,6,4,4] +; ALL-NEXT: vpermilps {{.*}} # ymm0 = ymm0[0,0,2,2,4,4,6,6] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; ALL-NEXT: retq %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> ret <8 x float> %shuffle @@ -488,9 +479,8 @@ define <8 x float> @shuffle_v8f32_00226644(<8 x float> %a, <8 x float> %b) { define <8 x float> @shuffle_v8f32_10324567(<8 x float> %a, <8 x float> %b) { ; ALL-LABEL: @shuffle_v8f32_10324567 ; ALL: # BB#0: -; ALL-NEXT: vpermilps {{.*}} # xmm1 = xmm0[1,0,3,2] -; ALL-NEXT: vextractf128 $1, %ymm0, %xmm0 -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[1,0,3,2,5,4,7,6] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; ALL-NEXT: retq %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> ret <8 x float> %shuffle @@ -499,9 +489,8 @@ define <8 x float> @shuffle_v8f32_10324567(<8 x float> %a, <8 x float> %b) { define <8 x float> @shuffle_v8f32_11334567(<8 x float> %a, <8 x float> %b) { ; ALL-LABEL: @shuffle_v8f32_11334567 ; ALL: # BB#0: -; ALL-NEXT: vmovshdup {{.*}} # xmm1 = xmm0[1,1,3,3] -; ALL-NEXT: vextractf128 $1, %ymm0, %xmm0 -; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[1,1,3,3,5,5,7,7] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] ; ALL-NEXT: retq %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> ret <8 x float> %shuffle @@ -510,9 +499,8 @@ define <8 x float> @shuffle_v8f32_11334567(<8 x float> %a, <8 x float> %b) { define <8 x float> @shuffle_v8f32_01235467(<8 x float> %a, <8 x float> %b) { ; ALL-LABEL: @shuffle_v8f32_01235467 ; ALL: # BB#0: -; ALL-NEXT: vextractf128 $1, %ymm0, %xmm1 -; ALL-NEXT: vpermilps {{.*}} # xmm1 = xmm1[1,0,2,3] -; ALL-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[1,0,2,3,5,4,6,7] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; ALL-NEXT: retq %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> ret <8 x float> %shuffle @@ -521,10 +509,71 @@ define <8 x float> @shuffle_v8f32_01235467(<8 x float> %a, <8 x float> %b) { define <8 x float> @shuffle_v8f32_01235466(<8 x float> %a, <8 x float> %b) { ; ALL-LABEL: @shuffle_v8f32_01235466 ; ALL: # BB#0: -; ALL-NEXT: vextractf128 $1, %ymm0, %xmm1 -; ALL-NEXT: vpermilps {{.*}} # xmm1 = xmm1[1,0,2,2] -; ALL-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[1,0,2,2,5,4,6,6] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] ; ALL-NEXT: retq %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> ret <8 x float> %shuffle } + +define <8 x float> @shuffle_v8f32_002u6u44(<8 x float> %a, <8 x float> %b) { +; ALL-LABEL: @shuffle_v8f32_002u6u44 +; ALL: # BB#0: +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[2,1,0,0,6,5,4,4] +; ALL-NEXT: vpermilps {{.*}} # ymm0 = ymm0[0,0,2,3,4,4,6,7] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; ALL-NEXT: retq + %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> + ret <8 x float> %shuffle +} + +define <8 x float> @shuffle_v8f32_00uu66uu(<8 x float> %a, <8 x float> %b) { +; ALL-LABEL: @shuffle_v8f32_00uu66uu +; ALL: # BB#0: +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[2,2,2,3,6,6,6,7] +; ALL-NEXT: vpermilps {{.*}} # ymm0 = ymm0[0,0,2,3,4,4,6,7] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; ALL-NEXT: retq + %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> + ret <8 x float> %shuffle +} + +define <8 x float> @shuffle_v8f32_103245uu(<8 x float> %a, <8 x float> %b) { +; ALL-LABEL: @shuffle_v8f32_103245uu +; ALL: # BB#0: +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[1,0,3,2,5,4,7,6] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; ALL-NEXT: retq + %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> + ret <8 x float> %shuffle +} + +define <8 x float> @shuffle_v8f32_1133uu67(<8 x float> %a, <8 x float> %b) { +; ALL-LABEL: @shuffle_v8f32_1133uu67 +; ALL: # BB#0: +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[1,1,3,3,5,5,7,7] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7] +; ALL-NEXT: retq + %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> + ret <8 x float> %shuffle +} + +define <8 x float> @shuffle_v8f32_0uu354uu(<8 x float> %a, <8 x float> %b) { +; ALL-LABEL: @shuffle_v8f32_0uu354uu +; ALL: # BB#0: +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[1,0,2,3,5,4,6,7] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; ALL-NEXT: retq + %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> + ret <8 x float> %shuffle +} + +define <8 x float> @shuffle_v8f32_uuu3uu66(<8 x float> %a, <8 x float> %b) { +; ALL-LABEL: @shuffle_v8f32_uuu3uu66 +; ALL: # BB#0: +; ALL-NEXT: vpermilps {{.*}} # ymm1 = ymm0[0,1,2,2,4,5,6,6] +; ALL-NEXT: vblendps {{.*}} # ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7] +; ALL-NEXT: retq + %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> + ret <8 x float> %shuffle +}