diff --git a/lib/Target/X86/X86InstrInfo.cpp b/lib/Target/X86/X86InstrInfo.cpp index 8eb4a17269b..d131791dfd7 100644 --- a/lib/Target/X86/X86InstrInfo.cpp +++ b/lib/Target/X86/X86InstrInfo.cpp @@ -5125,7 +5125,13 @@ static const uint16_t ReplaceableInstrsAVX2[][3] = { { X86::VINSERTF128rm, X86::VINSERTF128rm, X86::VINSERTI128rm }, { X86::VINSERTF128rr, X86::VINSERTF128rr, X86::VINSERTI128rr }, { X86::VPERM2F128rm, X86::VPERM2F128rm, X86::VPERM2I128rm }, - { X86::VPERM2F128rr, X86::VPERM2F128rr, X86::VPERM2I128rr } + { X86::VPERM2F128rr, X86::VPERM2F128rr, X86::VPERM2I128rr }, + { X86::VBROADCASTSSrm, X86::VBROADCASTSSrm, X86::VPBROADCASTDrm}, + { X86::VBROADCASTSSrr, X86::VBROADCASTSSrr, X86::VPBROADCASTDrr}, + { X86::VBROADCASTSSYrr, X86::VBROADCASTSSYrr, X86::VPBROADCASTDYrr}, + { X86::VBROADCASTSSYrm, X86::VBROADCASTSSYrm, X86::VPBROADCASTDYrm}, + { X86::VBROADCASTSDYrr, X86::VBROADCASTSDYrr, X86::VPBROADCASTQYrr}, + { X86::VBROADCASTSDYrm, X86::VBROADCASTSDYrm, X86::VPBROADCASTQYrm} }; // FIXME: Some shuffle and unpack instructions have equivalents in different diff --git a/test/CodeGen/X86/avx2-intrinsics-x86.ll b/test/CodeGen/X86/avx2-intrinsics-x86.ll index 7ee944b7c92..ab3d591e1d9 100644 --- a/test/CodeGen/X86/avx2-intrinsics-x86.ll +++ b/test/CodeGen/X86/avx2-intrinsics-x86.ll @@ -753,7 +753,7 @@ declare <16 x i16> @llvm.x86.avx2.pbroadcastw.256(<8 x i16>) nounwind readonly define <4 x i32> @test_x86_avx2_pbroadcastd_128(<4 x i32> %a0) { - ; CHECK: vpbroadcastd + ; CHECK: vbroadcastss %res = call <4 x i32> @llvm.x86.avx2.pbroadcastd.128(<4 x i32> %a0) ; <<4 x i32>> [#uses=1] ret <4 x i32> %res } @@ -761,7 +761,7 @@ declare <4 x i32> @llvm.x86.avx2.pbroadcastd.128(<4 x i32>) nounwind readonly define <8 x i32> @test_x86_avx2_pbroadcastd_256(<4 x i32> %a0) { - ; CHECK: vpbroadcastd + ; CHECK: vbroadcastss {{[^,]+}}, %ymm{{[0-9]+}} %res = call <8 x i32> @llvm.x86.avx2.pbroadcastd.256(<4 x i32> %a0) ; <<8 x i32>> [#uses=1] ret <8 x i32> %res } @@ -777,7 +777,7 @@ declare <2 x i64> @llvm.x86.avx2.pbroadcastq.128(<2 x i64>) nounwind readonly define <4 x i64> @test_x86_avx2_pbroadcastq_256(<2 x i64> %a0) { - ; CHECK: vpbroadcastq + ; CHECK: vbroadcastsd {{[^,]+}}, %ymm{{[0-9]+}} %res = call <4 x i64> @llvm.x86.avx2.pbroadcastq.256(<2 x i64> %a0) ; <<4 x i64>> [#uses=1] ret <4 x i64> %res } diff --git a/test/CodeGen/X86/avx2-vbroadcast.ll b/test/CodeGen/X86/avx2-vbroadcast.ll index bac9c66c3ae..66f586d23d1 100644 --- a/test/CodeGen/X86/avx2-vbroadcast.ll +++ b/test/CodeGen/X86/avx2-vbroadcast.ll @@ -98,7 +98,7 @@ entry: %qf = insertelement <16 x i16> %qe, i16 %q, i32 15 ret <16 x i16> %qf } -; CHECK: vpbroadcastd (% +; CHECK: vbroadcastss (% define <4 x i32> @D32(i32* %ptr) nounwind uwtable readnone ssp { entry: %q = load i32* %ptr, align 4 @@ -108,7 +108,7 @@ entry: %q3 = insertelement <4 x i32> %q2, i32 %q, i32 3 ret <4 x i32> %q3 } -; CHECK: vpbroadcastd (% +; CHECK: vbroadcastss (% define <8 x i32> @DD32(i32* %ptr) nounwind uwtable readnone ssp { entry: %q = load i32* %ptr, align 4 @@ -130,7 +130,7 @@ entry: %q1 = insertelement <2 x i64> %q0, i64 %q, i32 1 ret <2 x i64> %q1 } -; CHECK: vpbroadcastq (% +; CHECK: vbroadcastsd (% define <4 x i64> @QQ64(i64* %ptr) nounwind uwtable readnone ssp { entry: %q = load i64* %ptr, align 4 @@ -293,7 +293,7 @@ define <8 x i16> @_inreg8xi16(<8 x i16> %a) { ;CHECK-LABEL: _inreg4xi64: -;CHECK: vpbroadcastq +;CHECK: vbroadcastsd ;CHECK: ret define <4 x i64> @_inreg4xi64(<4 x i64> %a) { %b = shufflevector <4 x i64> %a, <4 x i64> undef, <4 x i32> zeroinitializer @@ -325,7 +325,7 @@ define <2 x double> @_inreg2xdouble(<2 x double> %a) { } ;CHECK-LABEL: _inreg8xi32: -;CHECK: vpbroadcastd +;CHECK: vbroadcastss ;CHECK: ret define <8 x i32> @_inreg8xi32(<8 x i32> %a) { %b = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> zeroinitializer @@ -333,7 +333,7 @@ define <8 x i32> @_inreg8xi32(<8 x i32> %a) { } ;CHECK-LABEL: _inreg4xi32: -;CHECK: vpbroadcastd +;CHECK: vbroadcastss ;CHECK: ret define <4 x i32> @_inreg4xi32(<4 x i32> %a) { %b = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> zeroinitializer diff --git a/test/CodeGen/X86/exedepsfix-broadcast.ll b/test/CodeGen/X86/exedepsfix-broadcast.ll new file mode 100644 index 00000000000..a18f7519563 --- /dev/null +++ b/test/CodeGen/X86/exedepsfix-broadcast.ll @@ -0,0 +1,128 @@ +; RUN: llc -O3 -mtriple=x86_64-apple-macosx -o - < %s -mattr=+avx2 -enable-unsafe-fp-math -mcpu=core2 | FileCheck %s +; Check that the ExeDepsFix pass correctly fixes the domain for broadcast instructions. +; + +; CHECK-LABEL: ExeDepsFix_broadcastss +; CHECK: broadcastss +; CHECK: vandps +; CHECK: vmaxps +; CHECK: ret +define <4 x float> @ExeDepsFix_broadcastss(<4 x float> %arg, <4 x float> %arg2) { + %bitcast = bitcast <4 x float> %arg to <4 x i32> + %and = and <4 x i32> %bitcast, + %floatcast = bitcast <4 x i32> %and to <4 x float> + %max_is_x = fcmp oge <4 x float> %floatcast, %arg2 + %max = select <4 x i1> %max_is_x, <4 x float> %floatcast, <4 x float> %arg2 + ret <4 x float> %max +} + +; CHECK-LABEL: ExeDepsFix_broadcastss256 +; CHECK: broadcastss +; CHECK: vandps +; CHECK: vmaxps +; CHECK: ret +define <8 x float> @ExeDepsFix_broadcastss256(<8 x float> %arg, <8 x float> %arg2) { + %bitcast = bitcast <8 x float> %arg to <8 x i32> + %and = and <8 x i32> %bitcast, + %floatcast = bitcast <8 x i32> %and to <8 x float> + %max_is_x = fcmp oge <8 x float> %floatcast, %arg2 + %max = select <8 x i1> %max_is_x, <8 x float> %floatcast, <8 x float> %arg2 + ret <8 x float> %max +} + + +; CHECK-LABEL: ExeDepsFix_broadcastss_inreg +; CHECK: broadcastss +; CHECK: vandps +; CHECK: vmaxps +; CHECK: ret +define <4 x float> @ExeDepsFix_broadcastss_inreg(<4 x float> %arg, <4 x float> %arg2, i32 %broadcastvalue) { + %bitcast = bitcast <4 x float> %arg to <4 x i32> + %in = insertelement <4 x i32> undef, i32 %broadcastvalue, i32 0 + %mask = shufflevector <4 x i32> %in, <4 x i32> undef, <4 x i32> zeroinitializer + %and = and <4 x i32> %bitcast, %mask + %floatcast = bitcast <4 x i32> %and to <4 x float> + %max_is_x = fcmp oge <4 x float> %floatcast, %arg2 + %max = select <4 x i1> %max_is_x, <4 x float> %floatcast, <4 x float> %arg2 + ret <4 x float> %max +} + +; CHECK-LABEL: ExeDepsFix_broadcastss256_inreg +; CHECK: broadcastss +; CHECK: vandps +; CHECK: vmaxps +; CHECK: ret +define <8 x float> @ExeDepsFix_broadcastss256_inreg(<8 x float> %arg, <8 x float> %arg2, i32 %broadcastvalue) { + %bitcast = bitcast <8 x float> %arg to <8 x i32> + %in = insertelement <8 x i32> undef, i32 %broadcastvalue, i32 0 + %mask = shufflevector <8 x i32> %in, <8 x i32> undef, <8 x i32> zeroinitializer + %and = and <8 x i32> %bitcast, %mask + %floatcast = bitcast <8 x i32> %and to <8 x float> + %max_is_x = fcmp oge <8 x float> %floatcast, %arg2 + %max = select <8 x i1> %max_is_x, <8 x float> %floatcast, <8 x float> %arg2 + ret <8 x float> %max +} + +; CHECK-LABEL: ExeDepsFix_broadcastsd +; In that case the broadcast is directly folded into vandpd. +; CHECK: vandpd +; CHECK: vmaxpd +; CHECK:ret +define <2 x double> @ExeDepsFix_broadcastsd(<2 x double> %arg, <2 x double> %arg2) { + %bitcast = bitcast <2 x double> %arg to <2 x i64> + %and = and <2 x i64> %bitcast, + %floatcast = bitcast <2 x i64> %and to <2 x double> + %max_is_x = fcmp oge <2 x double> %floatcast, %arg2 + %max = select <2 x i1> %max_is_x, <2 x double> %floatcast, <2 x double> %arg2 + ret <2 x double> %max +} + +; CHECK-LABEL: ExeDepsFix_broadcastsd256 +; CHECK: broadcastsd +; CHECK: vandpd +; CHECK: vmaxpd +; CHECK: ret +define <4 x double> @ExeDepsFix_broadcastsd256(<4 x double> %arg, <4 x double> %arg2) { + %bitcast = bitcast <4 x double> %arg to <4 x i64> + %and = and <4 x i64> %bitcast, + %floatcast = bitcast <4 x i64> %and to <4 x double> + %max_is_x = fcmp oge <4 x double> %floatcast, %arg2 + %max = select <4 x i1> %max_is_x, <4 x double> %floatcast, <4 x double> %arg2 + ret <4 x double> %max +} + + +; CHECK-LABEL: ExeDepsFix_broadcastsd_inreg +; ExeDepsFix works top down, thus it coalesces vmovlhps domain with +; vandps and there is nothing more you can do to match vmaxpd. +; CHECK: vmovlhps +; CHECK: vandps +; CHECK: vmaxpd +; CHECK: ret +define <2 x double> @ExeDepsFix_broadcastsd_inreg(<2 x double> %arg, <2 x double> %arg2, i64 %broadcastvalue) { + %bitcast = bitcast <2 x double> %arg to <2 x i64> + %in = insertelement <2 x i64> undef, i64 %broadcastvalue, i32 0 + %mask = shufflevector <2 x i64> %in, <2 x i64> undef, <2 x i32> zeroinitializer + %and = and <2 x i64> %bitcast, %mask + %floatcast = bitcast <2 x i64> %and to <2 x double> + %max_is_x = fcmp oge <2 x double> %floatcast, %arg2 + %max = select <2 x i1> %max_is_x, <2 x double> %floatcast, <2 x double> %arg2 + ret <2 x double> %max +} + +; CHECK-LABEL: ExeDepsFix_broadcastsd256_inreg +; CHECK: broadcastsd +; CHECK: vandpd +; CHECK: vmaxpd +; CHECK: ret +define <4 x double> @ExeDepsFix_broadcastsd256_inreg(<4 x double> %arg, <4 x double> %arg2, i64 %broadcastvalue) { + %bitcast = bitcast <4 x double> %arg to <4 x i64> + %in = insertelement <4 x i64> undef, i64 %broadcastvalue, i32 0 + %mask = shufflevector <4 x i64> %in, <4 x i64> undef, <4 x i32> zeroinitializer + %and = and <4 x i64> %bitcast, %mask + %floatcast = bitcast <4 x i64> %and to <4 x double> + %max_is_x = fcmp oge <4 x double> %floatcast, %arg2 + %max = select <4 x i1> %max_is_x, <4 x double> %floatcast, <4 x double> %arg2 + ret <4 x double> %max +} +