mirror of
https://github.com/c64scene-ar/llvm-6502.git
synced 2025-04-13 15:37:24 +00:00
ARM custom lower ctpop for vector types. Patch by Pete Couperus.
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@169325 91177308-0d34-0410-b5e6-96231b3b80d8
This commit is contained in:
Evan Cheng
parent
e570dee4b0
commit
c8e7045c8a
@ -546,6 +546,14 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM)
|
||||
setOperationAction(ISD::FP_ROUND, MVT::v2f32, Expand);
|
||||
setOperationAction(ISD::FP_EXTEND, MVT::v2f64, Expand);
|
||||
|
||||
// NEON does not have single instruction CTPOP for vectors with element
|
||||
// types wider than 8-bits. However, custom lowering can leverage the
|
||||
// v8i8/v16i8 vcnt instruction.
|
||||
setOperationAction(ISD::CTPOP, MVT::v2i32, Custom);
|
||||
setOperationAction(ISD::CTPOP, MVT::v4i32, Custom);
|
||||
setOperationAction(ISD::CTPOP, MVT::v4i16, Custom);
|
||||
setOperationAction(ISD::CTPOP, MVT::v8i16, Custom);
|
||||
|
||||
setTargetDAGCombine(ISD::INTRINSIC_VOID);
|
||||
setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
|
||||
setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
|
||||
@ -3554,6 +3562,114 @@ static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG,
|
||||
return DAG.getNode(ISD::CTLZ, dl, VT, rbit);
|
||||
}
|
||||
|
||||
/// getCTPOP16BitCounts - Returns a v8i8/v16i8 vector containing the bit-count
|
||||
/// for each 16-bit element from operand, repeated. The basic idea is to
|
||||
/// leverage vcnt to get the 8-bit counts, gather and add the results.
|
||||
///
|
||||
/// Trace for v4i16:
|
||||
/// input = [v0 v1 v2 v3 ] (vi 16-bit element)
|
||||
/// cast: N0 = [w0 w1 w2 w3 w4 w5 w6 w7] (v0 = [w0 w1], wi 8-bit element)
|
||||
/// vcnt: N1 = [b0 b1 b2 b3 b4 b5 b6 b7] (bi = bit-count of 8-bit element wi)
|
||||
/// vrev: N2 = [b1 b0 b3 b2 b5 b4 b7 b6]
|
||||
/// [b0 b1 b2 b3 b4 b5 b6 b7]
|
||||
/// +[b1 b0 b3 b2 b5 b4 b7 b6]
|
||||
/// N3=N1+N2 = [k0 k0 k1 k1 k2 k2 k3 k3] (k0 = b0+b1 = bit-count of 16-bit v0,
|
||||
/// vuzp: = [k0 k1 k2 k3 k0 k1 k2 k3] each ki is 8-bits)
|
||||
static SDValue getCTPOP16BitCounts(SDNode *N, SelectionDAG &DAG) {
|
||||
EVT VT = N->getValueType(0);
|
||||
DebugLoc DL = N->getDebugLoc();
|
||||
|
||||
EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
|
||||
SDValue N0 = DAG.getNode(ISD::BITCAST, DL, VT8Bit, N->getOperand(0));
|
||||
SDValue N1 = DAG.getNode(ISD::CTPOP, DL, VT8Bit, N0);
|
||||
SDValue N2 = DAG.getNode(ARMISD::VREV16, DL, VT8Bit, N1);
|
||||
SDValue N3 = DAG.getNode(ISD::ADD, DL, VT8Bit, N1, N2);
|
||||
return DAG.getNode(ARMISD::VUZP, DL, VT8Bit, N3, N3);
|
||||
}
|
||||
|
||||
/// lowerCTPOP16BitElements - Returns a v4i16/v8i16 vector containing the
|
||||
/// bit-count for each 16-bit element from the operand. We need slightly
|
||||
/// different sequencing for v4i16 and v8i16 to stay within NEON's available
|
||||
/// 64/128-bit registers.
|
||||
///
|
||||
/// Trace for v4i16:
|
||||
/// input = [v0 v1 v2 v3 ] (vi 16-bit element)
|
||||
/// v8i8: BitCounts = [k0 k1 k2 k3 k0 k1 k2 k3 ] (ki is the bit-count of vi)
|
||||
/// v8i16:Extended = [k0 k1 k2 k3 k0 k1 k2 k3 ]
|
||||
/// v4i16:Extracted = [k0 k1 k2 k3 ]
|
||||
static SDValue lowerCTPOP16BitElements(SDNode *N, SelectionDAG &DAG) {
|
||||
EVT VT = N->getValueType(0);
|
||||
DebugLoc DL = N->getDebugLoc();
|
||||
|
||||
SDValue BitCounts = getCTPOP16BitCounts(N, DAG);
|
||||
if (VT.is64BitVector()) {
|
||||
SDValue Extended = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, BitCounts);
|
||||
return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, Extended,
|
||||
DAG.getIntPtrConstant(0));
|
||||
} else {
|
||||
SDValue Extracted = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8,
|
||||
BitCounts, DAG.getIntPtrConstant(0));
|
||||
return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, Extracted);
|
||||
}
|
||||
}
|
||||
|
||||
/// lowerCTPOP32BitElements - Returns a v2i32/v4i32 vector containing the
|
||||
/// bit-count for each 32-bit element from the operand. The idea here is
|
||||
/// to split the vector into 16-bit elements, leverage the 16-bit count
|
||||
/// routine, and then combine the results.
|
||||
///
|
||||
/// Trace for v2i32 (v4i32 similar with Extracted/Extended exchanged):
|
||||
/// input = [v0 v1 ] (vi: 32-bit elements)
|
||||
/// Bitcast = [w0 w1 w2 w3 ] (wi: 16-bit elements, v0 = [w0 w1])
|
||||
/// Counts16 = [k0 k1 k2 k3 ] (ki: 16-bit elements, bit-count of wi)
|
||||
/// vrev: N0 = [k1 k0 k3 k2 ]
|
||||
/// [k0 k1 k2 k3 ]
|
||||
/// N1 =+[k1 k0 k3 k2 ]
|
||||
/// [k0 k2 k1 k3 ]
|
||||
/// N2 =+[k1 k3 k0 k2 ]
|
||||
/// [k0 k2 k1 k3 ]
|
||||
/// Extended =+[k1 k3 k0 k2 ]
|
||||
/// [k0 k2 ]
|
||||
/// Extracted=+[k1 k3 ]
|
||||
///
|
||||
static SDValue lowerCTPOP32BitElements(SDNode *N, SelectionDAG &DAG) {
|
||||
EVT VT = N->getValueType(0);
|
||||
DebugLoc DL = N->getDebugLoc();
|
||||
|
||||
EVT VT16Bit = VT.is64BitVector() ? MVT::v4i16 : MVT::v8i16;
|
||||
|
||||
SDValue Bitcast = DAG.getNode(ISD::BITCAST, DL, VT16Bit, N->getOperand(0));
|
||||
SDValue Counts16 = lowerCTPOP16BitElements(Bitcast.getNode(), DAG);
|
||||
SDValue N0 = DAG.getNode(ARMISD::VREV32, DL, VT16Bit, Counts16);
|
||||
SDValue N1 = DAG.getNode(ISD::ADD, DL, VT16Bit, Counts16, N0);
|
||||
SDValue N2 = DAG.getNode(ARMISD::VUZP, DL, VT16Bit, N1, N1);
|
||||
|
||||
if (VT.is64BitVector()) {
|
||||
SDValue Extended = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i32, N2);
|
||||
return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i32, Extended,
|
||||
DAG.getIntPtrConstant(0));
|
||||
} else {
|
||||
SDValue Extracted = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, N2,
|
||||
DAG.getIntPtrConstant(0));
|
||||
return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i32, Extracted);
|
||||
}
|
||||
}
|
||||
|
||||
static SDValue LowerCTPOP(SDNode *N, SelectionDAG &DAG,
|
||||
const ARMSubtarget *ST) {
|
||||
EVT VT = N->getValueType(0);
|
||||
|
||||
assert(ST->hasNEON() && "Custom ctpop lowering requires NEON.");
|
||||
assert((VT == MVT::v2i32 || VT == MVT::v4i32) ||
|
||||
(VT == MVT::v4i16 || VT == MVT::v8i16) &&
|
||||
"Unexpected type for custom ctpop lowering");
|
||||
|
||||
if (VT.getVectorElementType() == MVT::i32)
|
||||
return lowerCTPOP32BitElements(N, DAG);
|
||||
else
|
||||
return lowerCTPOP16BitElements(N, DAG);
|
||||
}
|
||||
|
||||
static SDValue LowerShift(SDNode *N, SelectionDAG &DAG,
|
||||
const ARMSubtarget *ST) {
|
||||
EVT VT = N->getValueType(0);
|
||||
@ -5411,6 +5527,7 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
|
||||
case ISD::SRL_PARTS:
|
||||
case ISD::SRA_PARTS: return LowerShiftRightParts(Op, DAG);
|
||||
case ISD::CTTZ: return LowerCTTZ(Op.getNode(), DAG, Subtarget);
|
||||
case ISD::CTPOP: return LowerCTPOP(Op.getNode(), DAG, Subtarget);
|
||||
case ISD::SETCC: return LowerVSETCC(Op, DAG);
|
||||
case ISD::ConstantFP: return LowerConstantFP(Op, DAG, Subtarget);
|
||||
case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG, Subtarget);
|
||||
|
||||
191
test/CodeGen/ARM/popcnt.ll
Normal file
191
test/CodeGen/ARM/popcnt.ll
Normal file
@ -0,0 +1,191 @@
|
||||
; RUN: llc < %s -march=arm -mattr=+neon | FileCheck %s
|
||||
; Implement ctpop with vcnt
|
||||
|
||||
define <8 x i8> @vcnt8(<8 x i8>* %A) nounwind {
|
||||
;CHECK: vcnt8:
|
||||
;CHECK: vcnt.8 {{d[0-9]+}}, {{d[0-9]+}}
|
||||
%tmp1 = load <8 x i8>* %A
|
||||
%tmp2 = call <8 x i8> @llvm.ctpop.v8i8(<8 x i8> %tmp1)
|
||||
ret <8 x i8> %tmp2
|
||||
}
|
||||
|
||||
define <16 x i8> @vcntQ8(<16 x i8>* %A) nounwind {
|
||||
;CHECK: vcntQ8:
|
||||
;CHECK: vcnt.8 {{q[0-9]+}}, {{q[0-9]+}}
|
||||
%tmp1 = load <16 x i8>* %A
|
||||
%tmp2 = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %tmp1)
|
||||
ret <16 x i8> %tmp2
|
||||
}
|
||||
|
||||
define <4 x i16> @vcnt16(<4 x i16>* %A) nounwind {
|
||||
; CHECK: vcnt16:
|
||||
; CHECK: vcnt.8 {{d[0-9]+}}, {{d[0-9]+}}
|
||||
; CHECK: vrev16.8 {{d[0-9]+}}, {{d[0-9]+}}
|
||||
; CHECK: vadd.i8 {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}
|
||||
; CHECK: vuzp.8 {{d[0-9]+}}, {{d[0-9]+}}
|
||||
; CHECK: vmovl.u8 {{q[0-9]+}}, {{d[0-9]+}}
|
||||
%tmp1 = load <4 x i16>* %A
|
||||
%tmp2 = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %tmp1)
|
||||
ret <4 x i16> %tmp2
|
||||
}
|
||||
|
||||
define <8 x i16> @vcntQ16(<8 x i16>* %A) nounwind {
|
||||
; CHECK: vcntQ16:
|
||||
; CHECK: vcnt.8 {{q[0-9]+}}, {{q[0-9]+}}
|
||||
; CHECK: vrev16.8 {{q[0-9]+}}, {{q[0-9]+}}
|
||||
; CHECK: vadd.i8 {{q[0-9]+}}, {{q[0-9]+}}, {{q[0-9]+}}
|
||||
; CHECK: vuzp.8 {{q[0-9]+}}, {{q[0-9]+}}
|
||||
; CHECK: vmovl.u8 {{q[0-9]+}}, {{d[0-9]+}}
|
||||
%tmp1 = load <8 x i16>* %A
|
||||
%tmp2 = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %tmp1)
|
||||
ret <8 x i16> %tmp2
|
||||
}
|
||||
|
||||
define <2 x i32> @vcnt32(<2 x i32>* %A) nounwind {
|
||||
; CHECK: vcnt32:
|
||||
; CHECK: vcnt.8 {{d[0-9]+}}, {{d[0-9]+}}
|
||||
; CHECK: vrev16.8 {{d[0-9]+}}, {{d[0-9]+}}
|
||||
; CHECK: vadd.i8 {{d[0-9]+}}, {{d[0-9]+}}, {{d[0-9]+}}
|
||||
; CHECK: vuzp.8 {{d[0-9]+}}, {{d[0-9]+}}
|
||||
; CHECK: vmovl.u8 {{q[0-9]+}}, {{d[0-9]+}}
|
||||
; CHECK: vrev32.16 {{d[0-9]+}}, {{d[0-9]+}}
|
||||
; CHECK: vuzp.16 {{d[0-9]+}}, {{d[0-9]+}}
|
||||
; CHECK: vmovl.u16 {{q[0-9]+}}, {{d[0-9]+}}
|
||||
%tmp1 = load <2 x i32>* %A
|
||||
%tmp2 = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %tmp1)
|
||||
ret <2 x i32> %tmp2
|
||||
}
|
||||
|
||||
define <4 x i32> @vcntQ32(<4 x i32>* %A) nounwind {
|
||||
; CHECK: vcntQ32:
|
||||
; CHECK: vcnt.8 {{q[0-9]+}}, {{q[0-9]+}}
|
||||
; CHECK: vrev16.8 {{q[0-9]+}}, {{q[0-9]+}}
|
||||
; CHECK: vadd.i8 {{q[0-9]+}}, {{q[0-9]+}}, {{q[0-9]+}}
|
||||
; CHECK: vuzp.8 {{q[0-9]+}}, {{q[0-9]+}}
|
||||
; CHECK: vmovl.u8 {{q[0-9]+}}, {{d[0-9]+}}
|
||||
; CHECK: vrev32.16 {{q[0-9]+}}, {{q[0-9]+}}
|
||||
; CHECK: vuzp.16 {{q[0-9]+}}, {{q[0-9]+}}
|
||||
; CHECK: vmovl.u16 {{q[0-9]+}}, {{d[0-9]+}}
|
||||
%tmp1 = load <4 x i32>* %A
|
||||
%tmp2 = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %tmp1)
|
||||
ret <4 x i32> %tmp2
|
||||
}
|
||||
|
||||
declare <8 x i8> @llvm.ctpop.v8i8(<8 x i8>) nounwind readnone
|
||||
declare <16 x i8> @llvm.ctpop.v16i8(<16 x i8>) nounwind readnone
|
||||
declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>) nounwind readnone
|
||||
declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>) nounwind readnone
|
||||
declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>) nounwind readnone
|
||||
declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>) nounwind readnone
|
||||
|
||||
define <8 x i8> @vclz8(<8 x i8>* %A) nounwind {
|
||||
;CHECK: vclz8:
|
||||
;CHECK: vclz.i8 {{d[0-9]+}}, {{d[0-9]+}}
|
||||
%tmp1 = load <8 x i8>* %A
|
||||
%tmp2 = call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %tmp1, i1 0)
|
||||
ret <8 x i8> %tmp2
|
||||
}
|
||||
|
||||
define <4 x i16> @vclz16(<4 x i16>* %A) nounwind {
|
||||
;CHECK: vclz16:
|
||||
;CHECK: vclz.i16 {{d[0-9]+}}, {{d[0-9]+}}
|
||||
%tmp1 = load <4 x i16>* %A
|
||||
%tmp2 = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %tmp1, i1 0)
|
||||
ret <4 x i16> %tmp2
|
||||
}
|
||||
|
||||
define <2 x i32> @vclz32(<2 x i32>* %A) nounwind {
|
||||
;CHECK: vclz32:
|
||||
;CHECK: vclz.i32 {{d[0-9]+}}, {{d[0-9]+}}
|
||||
%tmp1 = load <2 x i32>* %A
|
||||
%tmp2 = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %tmp1, i1 0)
|
||||
ret <2 x i32> %tmp2
|
||||
}
|
||||
|
||||
define <16 x i8> @vclzQ8(<16 x i8>* %A) nounwind {
|
||||
;CHECK: vclzQ8:
|
||||
;CHECK: vclz.i8 {{q[0-9]+}}, {{q[0-9]+}}
|
||||
%tmp1 = load <16 x i8>* %A
|
||||
%tmp2 = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %tmp1, i1 0)
|
||||
ret <16 x i8> %tmp2
|
||||
}
|
||||
|
||||
define <8 x i16> @vclzQ16(<8 x i16>* %A) nounwind {
|
||||
;CHECK: vclzQ16:
|
||||
;CHECK: vclz.i16 {{q[0-9]+}}, {{q[0-9]+}}
|
||||
%tmp1 = load <8 x i16>* %A
|
||||
%tmp2 = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %tmp1, i1 0)
|
||||
ret <8 x i16> %tmp2
|
||||
}
|
||||
|
||||
define <4 x i32> @vclzQ32(<4 x i32>* %A) nounwind {
|
||||
;CHECK: vclzQ32:
|
||||
;CHECK: vclz.i32 {{q[0-9]+}}, {{q[0-9]+}}
|
||||
%tmp1 = load <4 x i32>* %A
|
||||
%tmp2 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %tmp1, i1 0)
|
||||
ret <4 x i32> %tmp2
|
||||
}
|
||||
|
||||
declare <8 x i8> @llvm.ctlz.v8i8(<8 x i8>, i1) nounwind readnone
|
||||
declare <4 x i16> @llvm.ctlz.v4i16(<4 x i16>, i1) nounwind readnone
|
||||
declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1) nounwind readnone
|
||||
|
||||
declare <16 x i8> @llvm.ctlz.v16i8(<16 x i8>, i1) nounwind readnone
|
||||
declare <8 x i16> @llvm.ctlz.v8i16(<8 x i16>, i1) nounwind readnone
|
||||
declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1) nounwind readnone
|
||||
|
||||
define <8 x i8> @vclss8(<8 x i8>* %A) nounwind {
|
||||
;CHECK: vclss8:
|
||||
;CHECK: vcls.s8
|
||||
%tmp1 = load <8 x i8>* %A
|
||||
%tmp2 = call <8 x i8> @llvm.arm.neon.vcls.v8i8(<8 x i8> %tmp1)
|
||||
ret <8 x i8> %tmp2
|
||||
}
|
||||
|
||||
define <4 x i16> @vclss16(<4 x i16>* %A) nounwind {
|
||||
;CHECK: vclss16:
|
||||
;CHECK: vcls.s16
|
||||
%tmp1 = load <4 x i16>* %A
|
||||
%tmp2 = call <4 x i16> @llvm.arm.neon.vcls.v4i16(<4 x i16> %tmp1)
|
||||
ret <4 x i16> %tmp2
|
||||
}
|
||||
|
||||
define <2 x i32> @vclss32(<2 x i32>* %A) nounwind {
|
||||
;CHECK: vclss32:
|
||||
;CHECK: vcls.s32
|
||||
%tmp1 = load <2 x i32>* %A
|
||||
%tmp2 = call <2 x i32> @llvm.arm.neon.vcls.v2i32(<2 x i32> %tmp1)
|
||||
ret <2 x i32> %tmp2
|
||||
}
|
||||
|
||||
define <16 x i8> @vclsQs8(<16 x i8>* %A) nounwind {
|
||||
;CHECK: vclsQs8:
|
||||
;CHECK: vcls.s8
|
||||
%tmp1 = load <16 x i8>* %A
|
||||
%tmp2 = call <16 x i8> @llvm.arm.neon.vcls.v16i8(<16 x i8> %tmp1)
|
||||
ret <16 x i8> %tmp2
|
||||
}
|
||||
|
||||
define <8 x i16> @vclsQs16(<8 x i16>* %A) nounwind {
|
||||
;CHECK: vclsQs16:
|
||||
;CHECK: vcls.s16
|
||||
%tmp1 = load <8 x i16>* %A
|
||||
%tmp2 = call <8 x i16> @llvm.arm.neon.vcls.v8i16(<8 x i16> %tmp1)
|
||||
ret <8 x i16> %tmp2
|
||||
}
|
||||
|
||||
define <4 x i32> @vclsQs32(<4 x i32>* %A) nounwind {
|
||||
;CHECK: vclsQs32:
|
||||
;CHECK: vcls.s32
|
||||
%tmp1 = load <4 x i32>* %A
|
||||
%tmp2 = call <4 x i32> @llvm.arm.neon.vcls.v4i32(<4 x i32> %tmp1)
|
||||
ret <4 x i32> %tmp2
|
||||
}
|
||||
|
||||
declare <8 x i8> @llvm.arm.neon.vcls.v8i8(<8 x i8>) nounwind readnone
|
||||
declare <4 x i16> @llvm.arm.neon.vcls.v4i16(<4 x i16>) nounwind readnone
|
||||
declare <2 x i32> @llvm.arm.neon.vcls.v2i32(<2 x i32>) nounwind readnone
|
||||
|
||||
declare <16 x i8> @llvm.arm.neon.vcls.v16i8(<16 x i8>) nounwind readnone
|
||||
declare <8 x i16> @llvm.arm.neon.vcls.v8i16(<8 x i16>) nounwind readnone
|
||||
declare <4 x i32> @llvm.arm.neon.vcls.v4i32(<4 x i32>) nounwind readnone
|
||||
Loading…
x
Reference in New Issue
Block a user