From 596516bef8425bbce7e6363d3a002e908747ec6f Mon Sep 17 00:00:00 2001 From: Quentin Colombet Date: Wed, 26 Mar 2014 00:10:22 +0000 Subject: [PATCH] [X86] Add broadcast instructions to the table used by ExeDepsFix pass. Adds the different broadcast instructions to the ReplaceableInstrsAVX2 table. That way the ExeDepsFix pass can take better decisions when AVX2 broadcasts are across domain (int <-> float). In particular, prior to this patch we were generating: vpbroadcastd LCPI1_0(%rip), %ymm2 vpand %ymm2, %ymm0, %ymm0 vmaxps %ymm1, %ymm0, %ymm0 ## <- domain change penalty Now, we generate the following nice sequence where everything is in the float domain: vbroadcastss LCPI1_0(%rip), %ymm2 vandps %ymm2, %ymm0, %ymm0 vmaxps %ymm1, %ymm0, %ymm0 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@204770 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Target/X86/X86InstrInfo.cpp | 8 +- test/CodeGen/X86/avx2-intrinsics-x86.ll | 6 +- test/CodeGen/X86/avx2-vbroadcast.ll | 12 +-- test/CodeGen/X86/exedepsfix-broadcast.ll | 128 +++++++++++++++++++++++ 4 files changed, 144 insertions(+), 10 deletions(-) create mode 100644 test/CodeGen/X86/exedepsfix-broadcast.ll diff --git a/lib/Target/X86/X86InstrInfo.cpp b/lib/Target/X86/X86InstrInfo.cpp index 8eb4a17269b..d131791dfd7 100644 --- a/lib/Target/X86/X86InstrInfo.cpp +++ b/lib/Target/X86/X86InstrInfo.cpp @@ -5125,7 +5125,13 @@ static const uint16_t ReplaceableInstrsAVX2[][3] = { { X86::VINSERTF128rm, X86::VINSERTF128rm, X86::VINSERTI128rm }, { X86::VINSERTF128rr, X86::VINSERTF128rr, X86::VINSERTI128rr }, { X86::VPERM2F128rm, X86::VPERM2F128rm, X86::VPERM2I128rm }, - { X86::VPERM2F128rr, X86::VPERM2F128rr, X86::VPERM2I128rr } + { X86::VPERM2F128rr, X86::VPERM2F128rr, X86::VPERM2I128rr }, + { X86::VBROADCASTSSrm, X86::VBROADCASTSSrm, X86::VPBROADCASTDrm}, + { X86::VBROADCASTSSrr, X86::VBROADCASTSSrr, X86::VPBROADCASTDrr}, + { X86::VBROADCASTSSYrr, X86::VBROADCASTSSYrr, X86::VPBROADCASTDYrr}, + { X86::VBROADCASTSSYrm, X86::VBROADCASTSSYrm, X86::VPBROADCASTDYrm}, + { X86::VBROADCASTSDYrr, X86::VBROADCASTSDYrr, X86::VPBROADCASTQYrr}, + { X86::VBROADCASTSDYrm, X86::VBROADCASTSDYrm, X86::VPBROADCASTQYrm} }; // FIXME: Some shuffle and unpack instructions have equivalents in different diff --git a/test/CodeGen/X86/avx2-intrinsics-x86.ll b/test/CodeGen/X86/avx2-intrinsics-x86.ll index 7ee944b7c92..ab3d591e1d9 100644 --- a/test/CodeGen/X86/avx2-intrinsics-x86.ll +++ b/test/CodeGen/X86/avx2-intrinsics-x86.ll @@ -753,7 +753,7 @@ declare <16 x i16> @llvm.x86.avx2.pbroadcastw.256(<8 x i16>) nounwind readonly define <4 x i32> @test_x86_avx2_pbroadcastd_128(<4 x i32> %a0) { - ; CHECK: vpbroadcastd + ; CHECK: vbroadcastss %res = call <4 x i32> @llvm.x86.avx2.pbroadcastd.128(<4 x i32> %a0) ; <<4 x i32>> [#uses=1] ret <4 x i32> %res } @@ -761,7 +761,7 @@ declare <4 x i32> @llvm.x86.avx2.pbroadcastd.128(<4 x i32>) nounwind readonly define <8 x i32> @test_x86_avx2_pbroadcastd_256(<4 x i32> %a0) { - ; CHECK: vpbroadcastd + ; CHECK: vbroadcastss {{[^,]+}}, %ymm{{[0-9]+}} %res = call <8 x i32> @llvm.x86.avx2.pbroadcastd.256(<4 x i32> %a0) ; <<8 x i32>> [#uses=1] ret <8 x i32> %res } @@ -777,7 +777,7 @@ declare <2 x i64> @llvm.x86.avx2.pbroadcastq.128(<2 x i64>) nounwind readonly define <4 x i64> @test_x86_avx2_pbroadcastq_256(<2 x i64> %a0) { - ; CHECK: vpbroadcastq + ; CHECK: vbroadcastsd {{[^,]+}}, %ymm{{[0-9]+}} %res = call <4 x i64> @llvm.x86.avx2.pbroadcastq.256(<2 x i64> %a0) ; <<4 x i64>> [#uses=1] ret <4 x i64> %res } diff --git a/test/CodeGen/X86/avx2-vbroadcast.ll b/test/CodeGen/X86/avx2-vbroadcast.ll index bac9c66c3ae..66f586d23d1 100644 --- a/test/CodeGen/X86/avx2-vbroadcast.ll +++ b/test/CodeGen/X86/avx2-vbroadcast.ll @@ -98,7 +98,7 @@ entry: %qf = insertelement <16 x i16> %qe, i16 %q, i32 15 ret <16 x i16> %qf } -; CHECK: vpbroadcastd (% +; CHECK: vbroadcastss (% define <4 x i32> @D32(i32* %ptr) nounwind uwtable readnone ssp { entry: %q = load i32* %ptr, align 4 @@ -108,7 +108,7 @@ entry: %q3 = insertelement <4 x i32> %q2, i32 %q, i32 3 ret <4 x i32> %q3 } -; CHECK: vpbroadcastd (% +; CHECK: vbroadcastss (% define <8 x i32> @DD32(i32* %ptr) nounwind uwtable readnone ssp { entry: %q = load i32* %ptr, align 4 @@ -130,7 +130,7 @@ entry: %q1 = insertelement <2 x i64> %q0, i64 %q, i32 1 ret <2 x i64> %q1 } -; CHECK: vpbroadcastq (% +; CHECK: vbroadcastsd (% define <4 x i64> @QQ64(i64* %ptr) nounwind uwtable readnone ssp { entry: %q = load i64* %ptr, align 4 @@ -293,7 +293,7 @@ define <8 x i16> @_inreg8xi16(<8 x i16> %a) { ;CHECK-LABEL: _inreg4xi64: -;CHECK: vpbroadcastq +;CHECK: vbroadcastsd ;CHECK: ret define <4 x i64> @_inreg4xi64(<4 x i64> %a) { %b = shufflevector <4 x i64> %a, <4 x i64> undef, <4 x i32> zeroinitializer @@ -325,7 +325,7 @@ define <2 x double> @_inreg2xdouble(<2 x double> %a) { } ;CHECK-LABEL: _inreg8xi32: -;CHECK: vpbroadcastd +;CHECK: vbroadcastss ;CHECK: ret define <8 x i32> @_inreg8xi32(<8 x i32> %a) { %b = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> zeroinitializer @@ -333,7 +333,7 @@ define <8 x i32> @_inreg8xi32(<8 x i32> %a) { } ;CHECK-LABEL: _inreg4xi32: -;CHECK: vpbroadcastd +;CHECK: vbroadcastss ;CHECK: ret define <4 x i32> @_inreg4xi32(<4 x i32> %a) { %b = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> zeroinitializer diff --git a/test/CodeGen/X86/exedepsfix-broadcast.ll b/test/CodeGen/X86/exedepsfix-broadcast.ll new file mode 100644 index 00000000000..a18f7519563 --- /dev/null +++ b/test/CodeGen/X86/exedepsfix-broadcast.ll @@ -0,0 +1,128 @@ +; RUN: llc -O3 -mtriple=x86_64-apple-macosx -o - < %s -mattr=+avx2 -enable-unsafe-fp-math -mcpu=core2 | FileCheck %s +; Check that the ExeDepsFix pass correctly fixes the domain for broadcast instructions. +; + +; CHECK-LABEL: ExeDepsFix_broadcastss +; CHECK: broadcastss +; CHECK: vandps +; CHECK: vmaxps +; CHECK: ret +define <4 x float> @ExeDepsFix_broadcastss(<4 x float> %arg, <4 x float> %arg2) { + %bitcast = bitcast <4 x float> %arg to <4 x i32> + %and = and <4 x i32> %bitcast, + %floatcast = bitcast <4 x i32> %and to <4 x float> + %max_is_x = fcmp oge <4 x float> %floatcast, %arg2 + %max = select <4 x i1> %max_is_x, <4 x float> %floatcast, <4 x float> %arg2 + ret <4 x float> %max +} + +; CHECK-LABEL: ExeDepsFix_broadcastss256 +; CHECK: broadcastss +; CHECK: vandps +; CHECK: vmaxps +; CHECK: ret +define <8 x float> @ExeDepsFix_broadcastss256(<8 x float> %arg, <8 x float> %arg2) { + %bitcast = bitcast <8 x float> %arg to <8 x i32> + %and = and <8 x i32> %bitcast, + %floatcast = bitcast <8 x i32> %and to <8 x float> + %max_is_x = fcmp oge <8 x float> %floatcast, %arg2 + %max = select <8 x i1> %max_is_x, <8 x float> %floatcast, <8 x float> %arg2 + ret <8 x float> %max +} + + +; CHECK-LABEL: ExeDepsFix_broadcastss_inreg +; CHECK: broadcastss +; CHECK: vandps +; CHECK: vmaxps +; CHECK: ret +define <4 x float> @ExeDepsFix_broadcastss_inreg(<4 x float> %arg, <4 x float> %arg2, i32 %broadcastvalue) { + %bitcast = bitcast <4 x float> %arg to <4 x i32> + %in = insertelement <4 x i32> undef, i32 %broadcastvalue, i32 0 + %mask = shufflevector <4 x i32> %in, <4 x i32> undef, <4 x i32> zeroinitializer + %and = and <4 x i32> %bitcast, %mask + %floatcast = bitcast <4 x i32> %and to <4 x float> + %max_is_x = fcmp oge <4 x float> %floatcast, %arg2 + %max = select <4 x i1> %max_is_x, <4 x float> %floatcast, <4 x float> %arg2 + ret <4 x float> %max +} + +; CHECK-LABEL: ExeDepsFix_broadcastss256_inreg +; CHECK: broadcastss +; CHECK: vandps +; CHECK: vmaxps +; CHECK: ret +define <8 x float> @ExeDepsFix_broadcastss256_inreg(<8 x float> %arg, <8 x float> %arg2, i32 %broadcastvalue) { + %bitcast = bitcast <8 x float> %arg to <8 x i32> + %in = insertelement <8 x i32> undef, i32 %broadcastvalue, i32 0 + %mask = shufflevector <8 x i32> %in, <8 x i32> undef, <8 x i32> zeroinitializer + %and = and <8 x i32> %bitcast, %mask + %floatcast = bitcast <8 x i32> %and to <8 x float> + %max_is_x = fcmp oge <8 x float> %floatcast, %arg2 + %max = select <8 x i1> %max_is_x, <8 x float> %floatcast, <8 x float> %arg2 + ret <8 x float> %max +} + +; CHECK-LABEL: ExeDepsFix_broadcastsd +; In that case the broadcast is directly folded into vandpd. +; CHECK: vandpd +; CHECK: vmaxpd +; CHECK:ret +define <2 x double> @ExeDepsFix_broadcastsd(<2 x double> %arg, <2 x double> %arg2) { + %bitcast = bitcast <2 x double> %arg to <2 x i64> + %and = and <2 x i64> %bitcast, + %floatcast = bitcast <2 x i64> %and to <2 x double> + %max_is_x = fcmp oge <2 x double> %floatcast, %arg2 + %max = select <2 x i1> %max_is_x, <2 x double> %floatcast, <2 x double> %arg2 + ret <2 x double> %max +} + +; CHECK-LABEL: ExeDepsFix_broadcastsd256 +; CHECK: broadcastsd +; CHECK: vandpd +; CHECK: vmaxpd +; CHECK: ret +define <4 x double> @ExeDepsFix_broadcastsd256(<4 x double> %arg, <4 x double> %arg2) { + %bitcast = bitcast <4 x double> %arg to <4 x i64> + %and = and <4 x i64> %bitcast, + %floatcast = bitcast <4 x i64> %and to <4 x double> + %max_is_x = fcmp oge <4 x double> %floatcast, %arg2 + %max = select <4 x i1> %max_is_x, <4 x double> %floatcast, <4 x double> %arg2 + ret <4 x double> %max +} + + +; CHECK-LABEL: ExeDepsFix_broadcastsd_inreg +; ExeDepsFix works top down, thus it coalesces vmovlhps domain with +; vandps and there is nothing more you can do to match vmaxpd. +; CHECK: vmovlhps +; CHECK: vandps +; CHECK: vmaxpd +; CHECK: ret +define <2 x double> @ExeDepsFix_broadcastsd_inreg(<2 x double> %arg, <2 x double> %arg2, i64 %broadcastvalue) { + %bitcast = bitcast <2 x double> %arg to <2 x i64> + %in = insertelement <2 x i64> undef, i64 %broadcastvalue, i32 0 + %mask = shufflevector <2 x i64> %in, <2 x i64> undef, <2 x i32> zeroinitializer + %and = and <2 x i64> %bitcast, %mask + %floatcast = bitcast <2 x i64> %and to <2 x double> + %max_is_x = fcmp oge <2 x double> %floatcast, %arg2 + %max = select <2 x i1> %max_is_x, <2 x double> %floatcast, <2 x double> %arg2 + ret <2 x double> %max +} + +; CHECK-LABEL: ExeDepsFix_broadcastsd256_inreg +; CHECK: broadcastsd +; CHECK: vandpd +; CHECK: vmaxpd +; CHECK: ret +define <4 x double> @ExeDepsFix_broadcastsd256_inreg(<4 x double> %arg, <4 x double> %arg2, i64 %broadcastvalue) { + %bitcast = bitcast <4 x double> %arg to <4 x i64> + %in = insertelement <4 x i64> undef, i64 %broadcastvalue, i32 0 + %mask = shufflevector <4 x i64> %in, <4 x i64> undef, <4 x i32> zeroinitializer + %and = and <4 x i64> %bitcast, %mask + %floatcast = bitcast <4 x i64> %and to <4 x double> + %max_is_x = fcmp oge <4 x double> %floatcast, %arg2 + %max = select <4 x i1> %max_is_x, <4 x double> %floatcast, <4 x double> %arg2 + ret <4 x double> %max +} +