From 685b0d9315c0e4305113e8a92b1891908c3f5fe6 Mon Sep 17 00:00:00 2001 From: Quentin Colombet Date: Wed, 26 Feb 2014 21:39:12 +0000 Subject: [PATCH] Lower unsigned vsetcc to psubus in certain cases The current approach to lower a vsetult is to flip the sign bit of the operands, swap the operands and then use a (signed) pcmpgt. psubus (unsigned saturating subtract) can be used to emulate a vsetult more efficiently: + case ISD::SETULT: { + // If the comparison is against a constant we can turn this into a + // setule. With psubus, setule does not require a swap. This is + // beneficial because the constant in the register is no longer + // destructed as the destination so it can be hoisted out of a loop. I also enable lowering via psubus in a few other cases where it's clearly beneficial: setule and setuge if minu/maxu cannot be used. rdar://problem/14338765 Patch by Adam Nemet . git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@202301 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Target/X86/X86ISelLowering.cpp | 70 ++++++++++++++++++++++ test/CodeGen/X86/vec_setcc-2.ll | 95 ++++++++++++++++++++++++++++++ test/CodeGen/X86/vec_setcc.ll | 18 ++---- 3 files changed, 171 insertions(+), 12 deletions(-) create mode 100644 test/CodeGen/X86/vec_setcc-2.ll diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index f038580c5f9..71aee4ff3de 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -10009,6 +10009,37 @@ static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG, DAG.getConstant(SSECC, MVT::i8)); } +/// \brief Try to turn a VSETULT into a VSETULE by modifying its second +/// operand \p Op1. If non-trivial (for example because it's not constant) +/// return an empty value. +static SDValue ChangeVSETULTtoVSETULE(SDValue Op1, SelectionDAG &DAG) +{ + BuildVectorSDNode *BV = dyn_cast(Op1.getNode()); + if (!BV) + return SDValue(); + + MVT VT = Op1.getSimpleValueType(); + MVT EVT = VT.getVectorElementType(); + unsigned n = VT.getVectorNumElements(); + SmallVector ULTOp1; + + for (unsigned i = 0; i < n; ++i) { + ConstantSDNode *Elt = dyn_cast(BV->getOperand(i)); + if (!Elt || Elt->isOpaque() || Elt->getValueType(0) != EVT) + return SDValue(); + + // Avoid underflow. + APInt Val = Elt->getAPIntValue(); + if (Val == 0) + return SDValue(); + + ULTOp1.push_back(DAG.getConstant(Val - 1, EVT)); + } + + return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op1), VT, ULTOp1.data(), + ULTOp1.size()); +} + static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) { SDValue Op0 = Op.getOperand(0); @@ -10080,6 +10111,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget, // operations may be required for some comparisons. unsigned Opc; bool Swap = false, Invert = false, FlipSigns = false, MinMax = false; + bool Subus = false; switch (SetCCOpcode) { default: llvm_unreachable("Unexpected SETCC condition"); @@ -10114,6 +10146,40 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget, if (MinMax) { Swap = false; Invert = false; FlipSigns = false; } } + bool hasSubus = Subtarget->hasSSE2() && (VET == MVT::i8 || VET == MVT::i16); + if (!MinMax && hasSubus) { + // As another special case, use PSUBUS[BW] when it's profitable. E.g. for + // Op0 u<= Op1: + // t = psubus Op0, Op1 + // pcmpeq t, <0..0> + switch (SetCCOpcode) { + default: break; + case ISD::SETULT: { + // If the comparison is against a constant we can turn this into a + // setule. With psubus, setule does not require a swap. This is + // beneficial because the constant in the register is no longer + // destructed as the destination so it can be hoisted out of a loop. + // Only do this pre-AVX since vpcmp* is no longer destructive. + if (Subtarget->hasAVX()) + break; + SDValue ULEOp1 = ChangeVSETULTtoVSETULE(Op1, DAG); + if (ULEOp1.getNode()) { + Op1 = ULEOp1; + Subus = true; Invert = false; Swap = false; + } + break; + } + // Psubus is better than flip-sign because it requires no inversion. + case ISD::SETUGE: Subus = true; Invert = false; Swap = true; break; + case ISD::SETULE: Subus = true; Invert = false; Swap = false; break; + } + + if (Subus) { + Opc = X86ISD::SUBUS; + FlipSigns = false; + } + } + if (Swap) std::swap(Op0, Op1); @@ -10204,6 +10270,10 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget, if (MinMax) Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result); + if (Subus) + Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result, + getZeroVector(VT, Subtarget, DAG, dl)); + return Result; } diff --git a/test/CodeGen/X86/vec_setcc-2.ll b/test/CodeGen/X86/vec_setcc-2.ll new file mode 100644 index 00000000000..7173ad69ec5 --- /dev/null +++ b/test/CodeGen/X86/vec_setcc-2.ll @@ -0,0 +1,95 @@ +; RUN: llc < %s -o - -mcpu=generic -mtriple=x86_64-apple-darwin -mattr=+sse2 | FileCheck %s +; RUN: llc < %s -o - -mcpu=generic -mtriple=x86_64-apple-darwin -mattr=+sse4.2 | FileCheck %s + +; For a setult against a constant, turn it into a setule and lower via psubusw. + +define void @loop_no_const_reload(<2 x i64>* %in, <2 x i64>* %out, i32 %n) { +; CHECK: .short 25 +; CHECK-NEXT: .short 25 +; CHECK-NEXT: .short 25 +; CHECK-NEXT: .short 25 +; CHECK-NEXT: .short 25 +; CHECK-NEXT: .short 25 +; CHECK-NEXT: .short 25 +; CHECK-NEXT: .short 25 +; CHECK-LABEL: loop_no_const_reload: +; CHECK: psubusw + +; Constant is no longer clobbered so no need to reload it in the loop. + +; CHECK-NOT: movdqa {{%xmm[0-9]+}}, {{%xmm[0-9]+}} +entry: + %cmp9 = icmp eq i32 %n, 0 + br i1 %cmp9, label %for.end, label %for.body + +for.body: ; preds = %for.body, %entry + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx1 = getelementptr inbounds <2 x i64>* %in, i64 %indvars.iv + %arrayidx1.val = load <2 x i64>* %arrayidx1, align 16 + %0 = bitcast <2 x i64> %arrayidx1.val to <8 x i16> + %cmp.i.i = icmp ult <8 x i16> %0, + %sext.i.i = sext <8 x i1> %cmp.i.i to <8 x i16> + %1 = bitcast <8 x i16> %sext.i.i to <2 x i64> + %arrayidx5 = getelementptr inbounds <2 x i64>* %out, i64 %indvars.iv + store <2 x i64> %1, <2 x i64>* %arrayidx5, align 16 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +; Be careful if decrementing the constant would undeflow. + +define void @loop_const_folding_underflow(<2 x i64>* %in, <2 x i64>* %out, i32 %n) { +; CHECK-NOT: .short 25 +; CHECK-LABEL: loop_const_folding_underflow: +; CHECK-NOT: psubusw +entry: + %cmp9 = icmp eq i32 %n, 0 + br i1 %cmp9, label %for.end, label %for.body + +for.body: ; preds = %for.body, %entry + %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] + %arrayidx1 = getelementptr inbounds <2 x i64>* %in, i64 %indvars.iv + %arrayidx1.val = load <2 x i64>* %arrayidx1, align 16 + %0 = bitcast <2 x i64> %arrayidx1.val to <8 x i16> + %cmp.i.i = icmp ult <8 x i16> %0, + %sext.i.i = sext <8 x i1> %cmp.i.i to <8 x i16> + %1 = bitcast <8 x i16> %sext.i.i to <2 x i64> + %arrayidx5 = getelementptr inbounds <2 x i64>* %out, i64 %indvars.iv + store <2 x i64> %1, <2 x i64>* %arrayidx5, align 16 + %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 + %lftr.wideiv = trunc i64 %indvars.iv.next to i32 + %exitcond = icmp eq i32 %lftr.wideiv, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +} + +; Test for PSUBUSB + +define <16 x i8> @test_ult_byte(<16 x i8> %a) { +; CHECK: .space 16,10 +; CHECK-LABEL: test_ult_byte: +; CHECK: psubus +entry: + %icmp = icmp ult <16 x i8> %a, + %sext = sext <16 x i1> %icmp to <16 x i8> + ret <16 x i8> %sext +} + +; Only do this when we can turn the comparison into a setule. I.e. not for +; register operands. + +define <8 x i16> @test_ult_register(<8 x i16> %a, <8 x i16> %b) { +; CHECK-LABEL: test_ult_register: +; CHECK-NOT: psubus +entry: + %icmp = icmp ult <8 x i16> %a, %b + %sext = sext <8 x i1> %icmp to <8 x i16> + ret <8 x i16> %sext +} diff --git a/test/CodeGen/X86/vec_setcc.ll b/test/CodeGen/X86/vec_setcc.ll index fc8a56de791..322dbae0c89 100644 --- a/test/CodeGen/X86/vec_setcc.ll +++ b/test/CodeGen/X86/vec_setcc.ll @@ -42,12 +42,9 @@ define <8 x i16> @v8i16_icmp_uge(<8 x i16> %a, <8 x i16> %b) nounwind readnone s %2 = sext <8 x i1> %1 to <8 x i16> ret <8 x i16> %2 ; SSE2-LABEL: v8i16_icmp_uge: -; SSE2: movdqa {{.*}}(%rip), %xmm2 -; SEE2: pxor %xmm2, %xmm0 -; SSE2: pxor %xmm1, %xmm2 -; SSE2: pcmpgtw %xmm0, %xmm2 -; SSE2: pcmpeqd %xmm0, %xmm0 -; SSE2: pxor %xmm2, %xmm0 +; SSE2: psubusw %xmm0, %xmm1 +; SEE2: pxor %xmm0, %xmm0 +; SSE2: pcmpeqw %xmm1, %xmm0 ; SSE41-LABEL: v8i16_icmp_uge: ; SSE41: pmaxuw %xmm0, %xmm1 @@ -63,12 +60,9 @@ define <8 x i16> @v8i16_icmp_ule(<8 x i16> %a, <8 x i16> %b) nounwind readnone s %2 = sext <8 x i1> %1 to <8 x i16> ret <8 x i16> %2 ; SSE2-LABEL: v8i16_icmp_ule: -; SSE2: movdqa {{.*}}(%rip), %xmm2 -; SSE2: pxor %xmm2, %xmm1 -; SSE2: pxor %xmm2, %xmm0 -; SSE2: pcmpgtw %xmm1, %xmm0 -; SSE2: pcmpeqd %xmm1, %xmm1 -; SSE2: pxor %xmm0, %xmm1 +; SSE2: psubusw %xmm1, %xmm0 +; SSE2: pxor %xmm1, %xmm1 +; SSE2: pcmpeqw %xmm0, %xmm1 ; SSE2: movdqa %xmm1, %xmm0 ; SSE41-LABEL: v8i16_icmp_ule: