2015-03-06 20:57:40 +00:00
|
|
|
; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX1
|
|
|
|
|
; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX2
|
2014-10-02 20:11:11 +00:00
|
|
|
|
|
|
|
|
define <8 x float> @A(<8 x float> %a, <8 x float> %b) nounwind uwtable readnone ssp {
|
2014-10-05 11:41:36 +00:00
|
|
|
; ALL-LABEL: A:
|
|
|
|
|
; ALL: ## BB#0: ## %entry
|
|
|
|
|
; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
|
|
|
|
|
; ALL-NEXT: retq
|
2014-10-02 20:11:11 +00:00
|
|
|
entry:
|
|
|
|
|
%shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
|
|
|
|
|
ret <8 x float> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <8 x float> @B(<8 x float> %a, <8 x float> %b) nounwind uwtable readnone ssp {
|
|
|
|
|
; ALL-LABEL: B:
|
|
|
|
|
; ALL: ## BB#0: ## %entry
|
[x86] Enable the new vector shuffle lowering by default.
Update the entire regression test suite for the new shuffles. Remove
most of the old testing which was devoted to the old shuffle lowering
path and is no longer relevant really. Also remove a few other random
tests that only really exercised shuffles and only incidently or without
any interesting aspects to them.
Benchmarking that I have done shows a few small regressions with this on
LNT, zero measurable regressions on real, large applications, and for
several benchmarks where the loop vectorizer fires in the hot path it
shows 5% to 40% improvements for SSE2 and SSE3 code running on Sandy
Bridge machines. Running on AMD machines shows even more dramatic
improvements.
When using newer ISA vector extensions the gains are much more modest,
but the code is still better on the whole. There are a few regressions
being tracked (PR21137, PR21138, PR21139) but by and large this is
expected to be a win for x86 generated code performance.
It is also more correct than the code it replaces. I have fuzz tested
this extensively with ISA extensions up through AVX2 and found no
crashes or miscompiles (yet...). The old lowering had a few miscompiles
and crashers after a somewhat smaller amount of fuzz testing.
There is one significant area where the new code path lags behind and
that is in AVX-512 support. However, there was *extremely little*
support for that already and so this isn't a significant step backwards
and the new framework will probably make it easier to implement lowering
that uses the full power of AVX-512's table-based shuffle+blend (IMO).
Many thanks to Quentin, Andrea, Robert, and others for benchmarking
assistance. Thanks to Adam and others for help with AVX-512. Thanks to
Hal, Eric, and *many* others for answering my incessant questions about
how the backend actually works. =]
I will leave the old code path in the tree until the 3 PRs above are at
least resolved to folks' satisfaction. Then I will rip it (and 1000s of
lines of code) out. =] I don't expect this flag to stay around for very
long. It may not survive next week.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@219046 91177308-0d34-0410-b5e6-96231b3b80d8
2014-10-04 03:52:55 +00:00
|
|
|
; ALL-NEXT: vblendpd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3]
|
2014-10-02 20:11:11 +00:00
|
|
|
; ALL-NEXT: retq
|
|
|
|
|
entry:
|
|
|
|
|
%shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
|
|
|
|
|
ret <8 x float> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <8 x float> @C(<8 x float> %a, <8 x float> %b) nounwind uwtable readnone ssp {
|
2014-10-05 11:41:36 +00:00
|
|
|
; ALL-LABEL: C:
|
|
|
|
|
; ALL: ## BB#0: ## %entry
|
|
|
|
|
; ALL-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
|
|
|
|
; ALL-NEXT: retq
|
2014-10-02 20:11:11 +00:00
|
|
|
entry:
|
|
|
|
|
%shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
|
|
|
|
|
ret <8 x float> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <8 x float> @D(<8 x float> %a, <8 x float> %b) nounwind uwtable readnone ssp {
|
2014-10-05 11:41:36 +00:00
|
|
|
; ALL-LABEL: D:
|
|
|
|
|
; ALL: ## BB#0: ## %entry
|
|
|
|
|
; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
|
|
|
|
|
; ALL-NEXT: retq
|
2014-10-02 20:11:11 +00:00
|
|
|
entry:
|
|
|
|
|
%shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
|
|
|
|
|
ret <8 x float> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <32 x i8> @E(<32 x i8> %a, <32 x i8> %b) nounwind uwtable readnone ssp {
|
2014-10-05 11:41:36 +00:00
|
|
|
; ALL-LABEL: E:
|
|
|
|
|
; ALL: ## BB#0: ## %entry
|
|
|
|
|
; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
|
|
|
|
|
; ALL-NEXT: retq
|
2014-10-02 20:11:11 +00:00
|
|
|
entry:
|
|
|
|
|
%shuffle = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
|
|
|
|
|
ret <32 x i8> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <4 x i64> @E2(<4 x i64> %a, <4 x i64> %b) nounwind uwtable readnone ssp {
|
2014-10-05 11:41:36 +00:00
|
|
|
; ALL-LABEL: E2:
|
|
|
|
|
; ALL: ## BB#0: ## %entry
|
|
|
|
|
; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[0,1]
|
|
|
|
|
; ALL-NEXT: retq
|
2014-10-02 20:11:11 +00:00
|
|
|
entry:
|
|
|
|
|
%shuffle = shufflevector <4 x i64> %a, <4 x i64> %b, <4 x i32> <i32 6, i32 7, i32 0, i32 1>
|
|
|
|
|
ret <4 x i64> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <32 x i8> @Ei(<32 x i8> %a, <32 x i8> %b) nounwind uwtable readnone ssp {
|
|
|
|
|
; AVX1-LABEL: Ei:
|
|
|
|
|
; AVX1: ## BB#0: ## %entry
|
2014-10-05 19:14:34 +00:00
|
|
|
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
|
|
|
|
|
; AVX1-NEXT: vpaddb {{.*}}(%rip), %xmm0, %xmm0
|
|
|
|
|
; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
2014-10-05 11:41:36 +00:00
|
|
|
; AVX1-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
|
2014-10-02 20:11:11 +00:00
|
|
|
; AVX1-NEXT: retq
|
|
|
|
|
;
|
|
|
|
|
; AVX2-LABEL: Ei:
|
|
|
|
|
; AVX2: ## BB#0: ## %entry
|
[x86] Enable the new vector shuffle lowering by default.
Update the entire regression test suite for the new shuffles. Remove
most of the old testing which was devoted to the old shuffle lowering
path and is no longer relevant really. Also remove a few other random
tests that only really exercised shuffles and only incidently or without
any interesting aspects to them.
Benchmarking that I have done shows a few small regressions with this on
LNT, zero measurable regressions on real, large applications, and for
several benchmarks where the loop vectorizer fires in the hot path it
shows 5% to 40% improvements for SSE2 and SSE3 code running on Sandy
Bridge machines. Running on AMD machines shows even more dramatic
improvements.
When using newer ISA vector extensions the gains are much more modest,
but the code is still better on the whole. There are a few regressions
being tracked (PR21137, PR21138, PR21139) but by and large this is
expected to be a win for x86 generated code performance.
It is also more correct than the code it replaces. I have fuzz tested
this extensively with ISA extensions up through AVX2 and found no
crashes or miscompiles (yet...). The old lowering had a few miscompiles
and crashers after a somewhat smaller amount of fuzz testing.
There is one significant area where the new code path lags behind and
that is in AVX-512 support. However, there was *extremely little*
support for that already and so this isn't a significant step backwards
and the new framework will probably make it easier to implement lowering
that uses the full power of AVX-512's table-based shuffle+blend (IMO).
Many thanks to Quentin, Andrea, Robert, and others for benchmarking
assistance. Thanks to Adam and others for help with AVX-512. Thanks to
Hal, Eric, and *many* others for answering my incessant questions about
how the backend actually works. =]
I will leave the old code path in the tree until the 3 PRs above are at
least resolved to folks' satisfaction. Then I will rip it (and 1000s of
lines of code) out. =] I don't expect this flag to stay around for very
long. It may not survive next week.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@219046 91177308-0d34-0410-b5e6-96231b3b80d8
2014-10-04 03:52:55 +00:00
|
|
|
; AVX2-NEXT: vpaddb {{.*}}(%rip), %ymm0, %ymm0
|
2014-10-05 11:41:36 +00:00
|
|
|
; AVX2-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
|
2014-10-02 20:11:11 +00:00
|
|
|
; AVX2-NEXT: retq
|
|
|
|
|
entry:
|
|
|
|
|
; add forces execution domain
|
|
|
|
|
%a2 = add <32 x i8> %a, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
|
|
|
|
|
%shuffle = shufflevector <32 x i8> %a2, <32 x i8> %b, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
|
|
|
|
|
ret <32 x i8> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <4 x i64> @E2i(<4 x i64> %a, <4 x i64> %b) nounwind uwtable readnone ssp {
|
|
|
|
|
; AVX1-LABEL: E2i:
|
|
|
|
|
; AVX1: ## BB#0: ## %entry
|
2014-10-05 19:14:34 +00:00
|
|
|
; AVX1-NEXT: vpaddq {{.*}}(%rip), %xmm0, %xmm0
|
2014-10-05 11:41:36 +00:00
|
|
|
; AVX1-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[0,1]
|
2014-10-02 20:11:11 +00:00
|
|
|
; AVX1-NEXT: retq
|
|
|
|
|
;
|
|
|
|
|
; AVX2-LABEL: E2i:
|
|
|
|
|
; AVX2: ## BB#0: ## %entry
|
[x86] Enable the new vector shuffle lowering by default.
Update the entire regression test suite for the new shuffles. Remove
most of the old testing which was devoted to the old shuffle lowering
path and is no longer relevant really. Also remove a few other random
tests that only really exercised shuffles and only incidently or without
any interesting aspects to them.
Benchmarking that I have done shows a few small regressions with this on
LNT, zero measurable regressions on real, large applications, and for
several benchmarks where the loop vectorizer fires in the hot path it
shows 5% to 40% improvements for SSE2 and SSE3 code running on Sandy
Bridge machines. Running on AMD machines shows even more dramatic
improvements.
When using newer ISA vector extensions the gains are much more modest,
but the code is still better on the whole. There are a few regressions
being tracked (PR21137, PR21138, PR21139) but by and large this is
expected to be a win for x86 generated code performance.
It is also more correct than the code it replaces. I have fuzz tested
this extensively with ISA extensions up through AVX2 and found no
crashes or miscompiles (yet...). The old lowering had a few miscompiles
and crashers after a somewhat smaller amount of fuzz testing.
There is one significant area where the new code path lags behind and
that is in AVX-512 support. However, there was *extremely little*
support for that already and so this isn't a significant step backwards
and the new framework will probably make it easier to implement lowering
that uses the full power of AVX-512's table-based shuffle+blend (IMO).
Many thanks to Quentin, Andrea, Robert, and others for benchmarking
assistance. Thanks to Adam and others for help with AVX-512. Thanks to
Hal, Eric, and *many* others for answering my incessant questions about
how the backend actually works. =]
I will leave the old code path in the tree until the 3 PRs above are at
least resolved to folks' satisfaction. Then I will rip it (and 1000s of
lines of code) out. =] I don't expect this flag to stay around for very
long. It may not survive next week.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@219046 91177308-0d34-0410-b5e6-96231b3b80d8
2014-10-04 03:52:55 +00:00
|
|
|
; AVX2-NEXT: vpbroadcastq {{.*}}(%rip), %ymm2
|
2014-10-02 20:11:11 +00:00
|
|
|
; AVX2-NEXT: vpaddq %ymm2, %ymm0, %ymm0
|
2014-10-05 11:41:36 +00:00
|
|
|
; AVX2-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[0,1]
|
2014-10-02 20:11:11 +00:00
|
|
|
; AVX2-NEXT: retq
|
|
|
|
|
entry:
|
|
|
|
|
; add forces execution domain
|
|
|
|
|
%a2 = add <4 x i64> %a, <i64 1, i64 1, i64 1, i64 1>
|
|
|
|
|
%shuffle = shufflevector <4 x i64> %a2, <4 x i64> %b, <4 x i32> <i32 6, i32 7, i32 0, i32 1>
|
|
|
|
|
ret <4 x i64> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <8 x i32> @E3i(<8 x i32> %a, <8 x i32> %b) nounwind uwtable readnone ssp {
|
|
|
|
|
; AVX1-LABEL: E3i:
|
|
|
|
|
; AVX1: ## BB#0: ## %entry
|
2014-10-05 19:14:34 +00:00
|
|
|
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
|
|
|
|
|
; AVX1-NEXT: vpaddd {{.*}}(%rip), %xmm0, %xmm0
|
|
|
|
|
; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
|
2014-10-05 11:41:36 +00:00
|
|
|
; AVX1-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
|
2014-10-02 20:11:11 +00:00
|
|
|
; AVX1-NEXT: retq
|
|
|
|
|
;
|
|
|
|
|
; AVX2-LABEL: E3i:
|
|
|
|
|
; AVX2: ## BB#0: ## %entry
|
[x86] Enable the new vector shuffle lowering by default.
Update the entire regression test suite for the new shuffles. Remove
most of the old testing which was devoted to the old shuffle lowering
path and is no longer relevant really. Also remove a few other random
tests that only really exercised shuffles and only incidently or without
any interesting aspects to them.
Benchmarking that I have done shows a few small regressions with this on
LNT, zero measurable regressions on real, large applications, and for
several benchmarks where the loop vectorizer fires in the hot path it
shows 5% to 40% improvements for SSE2 and SSE3 code running on Sandy
Bridge machines. Running on AMD machines shows even more dramatic
improvements.
When using newer ISA vector extensions the gains are much more modest,
but the code is still better on the whole. There are a few regressions
being tracked (PR21137, PR21138, PR21139) but by and large this is
expected to be a win for x86 generated code performance.
It is also more correct than the code it replaces. I have fuzz tested
this extensively with ISA extensions up through AVX2 and found no
crashes or miscompiles (yet...). The old lowering had a few miscompiles
and crashers after a somewhat smaller amount of fuzz testing.
There is one significant area where the new code path lags behind and
that is in AVX-512 support. However, there was *extremely little*
support for that already and so this isn't a significant step backwards
and the new framework will probably make it easier to implement lowering
that uses the full power of AVX-512's table-based shuffle+blend (IMO).
Many thanks to Quentin, Andrea, Robert, and others for benchmarking
assistance. Thanks to Adam and others for help with AVX-512. Thanks to
Hal, Eric, and *many* others for answering my incessant questions about
how the backend actually works. =]
I will leave the old code path in the tree until the 3 PRs above are at
least resolved to folks' satisfaction. Then I will rip it (and 1000s of
lines of code) out. =] I don't expect this flag to stay around for very
long. It may not survive next week.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@219046 91177308-0d34-0410-b5e6-96231b3b80d8
2014-10-04 03:52:55 +00:00
|
|
|
; AVX2-NEXT: vpbroadcastd {{.*}}(%rip), %ymm2
|
2014-10-02 20:11:11 +00:00
|
|
|
; AVX2-NEXT: vpaddd %ymm2, %ymm0, %ymm0
|
2014-10-05 11:41:36 +00:00
|
|
|
; AVX2-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
|
2014-10-02 20:11:11 +00:00
|
|
|
; AVX2-NEXT: retq
|
|
|
|
|
entry:
|
|
|
|
|
; add forces execution domain
|
|
|
|
|
%a2 = add <8 x i32> %a, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
|
|
|
|
|
%shuffle = shufflevector <8 x i32> %a2, <8 x i32> %b, <8 x i32> <i32 undef, i32 5, i32 undef, i32 7, i32 12, i32 13, i32 14, i32 15>
|
|
|
|
|
ret <8 x i32> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <16 x i16> @E4i(<16 x i16> %a, <16 x i16> %b) nounwind uwtable readnone ssp {
|
|
|
|
|
; AVX1-LABEL: E4i:
|
|
|
|
|
; AVX1: ## BB#0: ## %entry
|
[x86] Enable the new vector shuffle lowering by default.
Update the entire regression test suite for the new shuffles. Remove
most of the old testing which was devoted to the old shuffle lowering
path and is no longer relevant really. Also remove a few other random
tests that only really exercised shuffles and only incidently or without
any interesting aspects to them.
Benchmarking that I have done shows a few small regressions with this on
LNT, zero measurable regressions on real, large applications, and for
several benchmarks where the loop vectorizer fires in the hot path it
shows 5% to 40% improvements for SSE2 and SSE3 code running on Sandy
Bridge machines. Running on AMD machines shows even more dramatic
improvements.
When using newer ISA vector extensions the gains are much more modest,
but the code is still better on the whole. There are a few regressions
being tracked (PR21137, PR21138, PR21139) but by and large this is
expected to be a win for x86 generated code performance.
It is also more correct than the code it replaces. I have fuzz tested
this extensively with ISA extensions up through AVX2 and found no
crashes or miscompiles (yet...). The old lowering had a few miscompiles
and crashers after a somewhat smaller amount of fuzz testing.
There is one significant area where the new code path lags behind and
that is in AVX-512 support. However, there was *extremely little*
support for that already and so this isn't a significant step backwards
and the new framework will probably make it easier to implement lowering
that uses the full power of AVX-512's table-based shuffle+blend (IMO).
Many thanks to Quentin, Andrea, Robert, and others for benchmarking
assistance. Thanks to Adam and others for help with AVX-512. Thanks to
Hal, Eric, and *many* others for answering my incessant questions about
how the backend actually works. =]
I will leave the old code path in the tree until the 3 PRs above are at
least resolved to folks' satisfaction. Then I will rip it (and 1000s of
lines of code) out. =] I don't expect this flag to stay around for very
long. It may not survive next week.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@219046 91177308-0d34-0410-b5e6-96231b3b80d8
2014-10-04 03:52:55 +00:00
|
|
|
; AVX1-NEXT: vpaddw {{.*}}(%rip), %xmm0, %xmm0
|
2014-10-05 11:41:36 +00:00
|
|
|
; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
|
2014-10-02 20:11:11 +00:00
|
|
|
; AVX1-NEXT: retq
|
|
|
|
|
;
|
|
|
|
|
; AVX2-LABEL: E4i:
|
|
|
|
|
; AVX2: ## BB#0: ## %entry
|
[x86] Enable the new vector shuffle lowering by default.
Update the entire regression test suite for the new shuffles. Remove
most of the old testing which was devoted to the old shuffle lowering
path and is no longer relevant really. Also remove a few other random
tests that only really exercised shuffles and only incidently or without
any interesting aspects to them.
Benchmarking that I have done shows a few small regressions with this on
LNT, zero measurable regressions on real, large applications, and for
several benchmarks where the loop vectorizer fires in the hot path it
shows 5% to 40% improvements for SSE2 and SSE3 code running on Sandy
Bridge machines. Running on AMD machines shows even more dramatic
improvements.
When using newer ISA vector extensions the gains are much more modest,
but the code is still better on the whole. There are a few regressions
being tracked (PR21137, PR21138, PR21139) but by and large this is
expected to be a win for x86 generated code performance.
It is also more correct than the code it replaces. I have fuzz tested
this extensively with ISA extensions up through AVX2 and found no
crashes or miscompiles (yet...). The old lowering had a few miscompiles
and crashers after a somewhat smaller amount of fuzz testing.
There is one significant area where the new code path lags behind and
that is in AVX-512 support. However, there was *extremely little*
support for that already and so this isn't a significant step backwards
and the new framework will probably make it easier to implement lowering
that uses the full power of AVX-512's table-based shuffle+blend (IMO).
Many thanks to Quentin, Andrea, Robert, and others for benchmarking
assistance. Thanks to Adam and others for help with AVX-512. Thanks to
Hal, Eric, and *many* others for answering my incessant questions about
how the backend actually works. =]
I will leave the old code path in the tree until the 3 PRs above are at
least resolved to folks' satisfaction. Then I will rip it (and 1000s of
lines of code) out. =] I don't expect this flag to stay around for very
long. It may not survive next week.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@219046 91177308-0d34-0410-b5e6-96231b3b80d8
2014-10-04 03:52:55 +00:00
|
|
|
; AVX2-NEXT: vpaddw {{.*}}(%rip), %ymm0, %ymm0
|
2014-10-05 11:41:36 +00:00
|
|
|
; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
|
2014-10-02 20:11:11 +00:00
|
|
|
; AVX2-NEXT: retq
|
|
|
|
|
entry:
|
|
|
|
|
; add forces execution domain
|
|
|
|
|
%a2 = add <16 x i16> %a, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
|
|
|
|
|
%shuffle = shufflevector <16 x i16> %a2, <16 x i16> %b, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
|
|
|
|
ret <16 x i16> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <16 x i16> @E5i(<16 x i16>* %a, <16 x i16>* %b) nounwind uwtable readnone ssp {
|
|
|
|
|
; AVX1-LABEL: E5i:
|
|
|
|
|
; AVX1: ## BB#0: ## %entry
|
[x86] Enable the new vector shuffle lowering by default.
Update the entire regression test suite for the new shuffles. Remove
most of the old testing which was devoted to the old shuffle lowering
path and is no longer relevant really. Also remove a few other random
tests that only really exercised shuffles and only incidently or without
any interesting aspects to them.
Benchmarking that I have done shows a few small regressions with this on
LNT, zero measurable regressions on real, large applications, and for
several benchmarks where the loop vectorizer fires in the hot path it
shows 5% to 40% improvements for SSE2 and SSE3 code running on Sandy
Bridge machines. Running on AMD machines shows even more dramatic
improvements.
When using newer ISA vector extensions the gains are much more modest,
but the code is still better on the whole. There are a few regressions
being tracked (PR21137, PR21138, PR21139) but by and large this is
expected to be a win for x86 generated code performance.
It is also more correct than the code it replaces. I have fuzz tested
this extensively with ISA extensions up through AVX2 and found no
crashes or miscompiles (yet...). The old lowering had a few miscompiles
and crashers after a somewhat smaller amount of fuzz testing.
There is one significant area where the new code path lags behind and
that is in AVX-512 support. However, there was *extremely little*
support for that already and so this isn't a significant step backwards
and the new framework will probably make it easier to implement lowering
that uses the full power of AVX-512's table-based shuffle+blend (IMO).
Many thanks to Quentin, Andrea, Robert, and others for benchmarking
assistance. Thanks to Adam and others for help with AVX-512. Thanks to
Hal, Eric, and *many* others for answering my incessant questions about
how the backend actually works. =]
I will leave the old code path in the tree until the 3 PRs above are at
least resolved to folks' satisfaction. Then I will rip it (and 1000s of
lines of code) out. =] I don't expect this flag to stay around for very
long. It may not survive next week.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@219046 91177308-0d34-0410-b5e6-96231b3b80d8
2014-10-04 03:52:55 +00:00
|
|
|
; AVX1-NEXT: vmovdqa (%rdi), %ymm0
|
|
|
|
|
; AVX1-NEXT: vpaddw {{.*}}(%rip), %xmm0, %xmm0
|
2014-10-05 11:41:36 +00:00
|
|
|
; AVX1-NEXT: vmovaps (%rsi), %ymm1
|
|
|
|
|
; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
|
2014-10-02 20:11:11 +00:00
|
|
|
; AVX1-NEXT: retq
|
|
|
|
|
;
|
|
|
|
|
; AVX2-LABEL: E5i:
|
|
|
|
|
; AVX2: ## BB#0: ## %entry
|
|
|
|
|
; AVX2-NEXT: vmovdqa (%rdi), %ymm0
|
[x86] Enable the new vector shuffle lowering by default.
Update the entire regression test suite for the new shuffles. Remove
most of the old testing which was devoted to the old shuffle lowering
path and is no longer relevant really. Also remove a few other random
tests that only really exercised shuffles and only incidently or without
any interesting aspects to them.
Benchmarking that I have done shows a few small regressions with this on
LNT, zero measurable regressions on real, large applications, and for
several benchmarks where the loop vectorizer fires in the hot path it
shows 5% to 40% improvements for SSE2 and SSE3 code running on Sandy
Bridge machines. Running on AMD machines shows even more dramatic
improvements.
When using newer ISA vector extensions the gains are much more modest,
but the code is still better on the whole. There are a few regressions
being tracked (PR21137, PR21138, PR21139) but by and large this is
expected to be a win for x86 generated code performance.
It is also more correct than the code it replaces. I have fuzz tested
this extensively with ISA extensions up through AVX2 and found no
crashes or miscompiles (yet...). The old lowering had a few miscompiles
and crashers after a somewhat smaller amount of fuzz testing.
There is one significant area where the new code path lags behind and
that is in AVX-512 support. However, there was *extremely little*
support for that already and so this isn't a significant step backwards
and the new framework will probably make it easier to implement lowering
that uses the full power of AVX-512's table-based shuffle+blend (IMO).
Many thanks to Quentin, Andrea, Robert, and others for benchmarking
assistance. Thanks to Adam and others for help with AVX-512. Thanks to
Hal, Eric, and *many* others for answering my incessant questions about
how the backend actually works. =]
I will leave the old code path in the tree until the 3 PRs above are at
least resolved to folks' satisfaction. Then I will rip it (and 1000s of
lines of code) out. =] I don't expect this flag to stay around for very
long. It may not survive next week.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@219046 91177308-0d34-0410-b5e6-96231b3b80d8
2014-10-04 03:52:55 +00:00
|
|
|
; AVX2-NEXT: vmovdqa (%rsi), %ymm1
|
|
|
|
|
; AVX2-NEXT: vpaddw {{.*}}(%rip), %ymm0, %ymm0
|
2014-10-05 11:41:36 +00:00
|
|
|
; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
|
2014-10-02 20:11:11 +00:00
|
|
|
; AVX2-NEXT: retq
|
|
|
|
|
entry:
|
2015-02-27 21:17:42 +00:00
|
|
|
%c = load <16 x i16>, <16 x i16>* %a
|
|
|
|
|
%d = load <16 x i16>, <16 x i16>* %b
|
2014-10-02 20:11:11 +00:00
|
|
|
%c2 = add <16 x i16> %c, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
|
|
|
|
|
%shuffle = shufflevector <16 x i16> %c2, <16 x i16> %d, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
|
|
|
|
|
ret <16 x i16> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
;;;; Cases with undef indicies mixed in the mask
|
|
|
|
|
|
|
|
|
|
define <8 x float> @F(<8 x float> %a, <8 x float> %b) nounwind uwtable readnone ssp {
|
2014-10-05 11:41:36 +00:00
|
|
|
; ALL-LABEL: F:
|
|
|
|
|
; ALL: ## BB#0: ## %entry
|
[X86][AVX] Fix wrong lowering of VPERM2X128 nodes
There were cases where the backend computed a wrong permute mask for a VPERM2X128 node.
Example:
\code
define <8 x float> @foo(<8 x float> %a, <8 x float> %b) {
%shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 undef, i32 undef, i32 6, i32 7, i32 undef, i32 undef, i32 6, i32 7>
ret <8 x float> %shuffle
}
\code end
Before this patch, llc (with -mattr=+avx) emitted the following vperm2f128:
vperm2f128 $0, %ymm0, %ymm0, %ymm0 # ymm0 = ymm0[0,1,0,1]
With this patch, llc emits a vperm2f128 with a correct permute mask:
vperm2f128 $17, %ymm0, %ymm0, %ymm0 # ymm0 = ymm0[2,3,2,3]
Differential Revision: http://reviews.llvm.org/D8119
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@231601 91177308-0d34-0410-b5e6-96231b3b80d8
2015-03-08 16:28:47 +00:00
|
|
|
; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1]
|
2014-10-05 11:41:36 +00:00
|
|
|
; ALL-NEXT: retq
|
2014-10-02 20:11:11 +00:00
|
|
|
entry:
|
|
|
|
|
%shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 undef, i32 undef, i32 6, i32 7, i32 undef, i32 9, i32 undef, i32 11>
|
|
|
|
|
ret <8 x float> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
[X86][AVX] Fix wrong lowering of VPERM2X128 nodes
There were cases where the backend computed a wrong permute mask for a VPERM2X128 node.
Example:
\code
define <8 x float> @foo(<8 x float> %a, <8 x float> %b) {
%shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 undef, i32 undef, i32 6, i32 7, i32 undef, i32 undef, i32 6, i32 7>
ret <8 x float> %shuffle
}
\code end
Before this patch, llc (with -mattr=+avx) emitted the following vperm2f128:
vperm2f128 $0, %ymm0, %ymm0, %ymm0 # ymm0 = ymm0[0,1,0,1]
With this patch, llc emits a vperm2f128 with a correct permute mask:
vperm2f128 $17, %ymm0, %ymm0, %ymm0 # ymm0 = ymm0[2,3,2,3]
Differential Revision: http://reviews.llvm.org/D8119
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@231601 91177308-0d34-0410-b5e6-96231b3b80d8
2015-03-08 16:28:47 +00:00
|
|
|
define <8 x float> @F2(<8 x float> %a, <8 x float> %b) nounwind uwtable readnone ssp {
|
|
|
|
|
; ALL-LABEL: F2:
|
|
|
|
|
; ALL: ## BB#0: ## %entry
|
|
|
|
|
; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
|
|
|
|
|
; ALL-NEXT: retq
|
|
|
|
|
entry:
|
|
|
|
|
%shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 undef, i32 undef, i32 6, i32 7, i32 undef, i32 undef, i32 6, i32 7>
|
|
|
|
|
ret <8 x float> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <8 x float> @F3(<8 x float> %a, <8 x float> %b) nounwind uwtable readnone ssp {
|
|
|
|
|
; ALL-LABEL: F3:
|
|
|
|
|
; ALL: ## BB#0: ## %entry
|
|
|
|
|
; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1]
|
|
|
|
|
; ALL-NEXT: retq
|
|
|
|
|
entry:
|
|
|
|
|
%shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 undef, i32 undef, i32 6, i32 7, i32 undef, i32 undef, i32 10, i32 11>
|
|
|
|
|
ret <8 x float> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <8 x float> @F4(<8 x float> %a, <8 x float> %b) nounwind uwtable readnone ssp {
|
|
|
|
|
; ALL-LABEL: F4:
|
|
|
|
|
; ALL: ## BB#0: ## %entry
|
|
|
|
|
; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
|
|
|
|
|
; ALL-NEXT: retq
|
|
|
|
|
entry:
|
|
|
|
|
%shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 undef, i32 undef, i32 6, i32 7, i32 undef, i32 undef, i32 14, i32 15>
|
|
|
|
|
ret <8 x float> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <8 x float> @F5(<8 x float> %a, <8 x float> %b) nounwind uwtable readnone ssp {
|
|
|
|
|
; ALL-LABEL: F5:
|
|
|
|
|
; ALL: ## BB#0: ## %entry
|
|
|
|
|
; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
|
|
|
|
|
; ALL-NEXT: retq
|
|
|
|
|
entry:
|
|
|
|
|
%shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 undef, i32 undef, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
|
|
|
|
|
ret <8 x float> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <8 x float> @F6(<8 x float> %a, <8 x float> %b) nounwind uwtable readnone ssp {
|
|
|
|
|
; ALL-LABEL: F6:
|
|
|
|
|
; ALL: ## BB#0: ## %entry
|
|
|
|
|
; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1]
|
|
|
|
|
; ALL-NEXT: retq
|
|
|
|
|
entry:
|
|
|
|
|
%shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 undef, i32 undef, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
|
|
|
|
|
ret <8 x float> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <8 x float> @F7(<8 x float> %a, <8 x float> %b) nounwind uwtable readnone ssp {
|
|
|
|
|
; ALL-LABEL: F7:
|
|
|
|
|
; ALL: ## BB#0: ## %entry
|
|
|
|
|
; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,2,3]
|
|
|
|
|
; ALL-NEXT: retq
|
|
|
|
|
entry:
|
|
|
|
|
%shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 6, i32 7>
|
|
|
|
|
ret <8 x float> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
define <8 x float> @F8(<8 x float> %a, <8 x float> %b) nounwind uwtable readnone ssp {
|
|
|
|
|
; ALL-LABEL: F8:
|
|
|
|
|
; ALL: ## BB#0: ## %entry
|
|
|
|
|
; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
|
|
|
|
|
; ALL-NEXT: retq
|
|
|
|
|
entry:
|
|
|
|
|
%shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 14, i32 15>
|
|
|
|
|
ret <8 x float> %shuffle
|
|
|
|
|
}
|
|
|
|
|
|
2014-10-02 20:11:11 +00:00
|
|
|
;;;; Cases we must not select vperm2f128
|
|
|
|
|
|
|
|
|
|
define <8 x float> @G(<8 x float> %a, <8 x float> %b) nounwind uwtable readnone ssp {
|
2014-11-21 13:56:05 +00:00
|
|
|
; ALL-LABEL: G:
|
|
|
|
|
; ALL: ## BB#0: ## %entry
|
|
|
|
|
; ALL-NEXT: vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3]
|
|
|
|
|
; ALL-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,2,3,4,4,6,7]
|
|
|
|
|
; ALL-NEXT: retq
|
2014-10-02 20:11:11 +00:00
|
|
|
entry:
|
|
|
|
|
%shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 undef, i32 undef, i32 6, i32 7, i32 undef, i32 12, i32 undef, i32 15>
|
|
|
|
|
ret <8 x float> %shuffle
|
|
|
|
|
}
|
