diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp index 2a4c57d118f..cd183131f17 100644 --- a/lib/Target/X86/X86ISelLowering.cpp +++ b/lib/Target/X86/X86ISelLowering.cpp @@ -1383,9 +1383,7 @@ bool X86::isPSHUFDMask(SDNode *N) { return false; // Check if the value doesn't reference the second vector. - SDOperand Elt = N->getOperand(0); - assert(isa(Elt) && "Invalid VECTOR_SHUFFLE mask!"); - for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { + for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) { assert(isa(N->getOperand(i)) && "Invalid VECTOR_SHUFFLE mask!"); if (cast(N->getOperand(i))->getValue() >= 4) return false; @@ -1394,6 +1392,38 @@ bool X86::isPSHUFDMask(SDNode *N) { return true; } +/// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand +/// specifies a shuffle of elements that is suitable for input to SHUFP*. +bool X86::isSHUFPMask(SDNode *N) { + assert(N->getOpcode() == ISD::BUILD_VECTOR); + + unsigned NumOperands = N->getNumOperands(); + if (NumOperands != 2 && NumOperands != 4) + return false; + + // Each half must refer to only one of the vector. + SDOperand Elt = N->getOperand(0); + assert(isa(Elt) && "Invalid VECTOR_SHUFFLE mask!"); + for (unsigned i = 1; i != NumOperands / 2; ++i) { + assert(isa(N->getOperand(i)) && + "Invalid VECTOR_SHUFFLE mask!"); + if (cast(N->getOperand(i))->getValue() != + cast(Elt)->getValue()) + return false; + } + Elt = N->getOperand(NumOperands / 2); + assert(isa(Elt) && "Invalid VECTOR_SHUFFLE mask!"); + for (unsigned i = NumOperands / 2; i != NumOperands; ++i) { + assert(isa(N->getOperand(i)) && + "Invalid VECTOR_SHUFFLE mask!"); + if (cast(N->getOperand(i))->getValue() != + cast(Elt)->getValue()) + return false; + } + + return true; +} + /// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies /// a splat of a single element. bool X86::isSplatMask(SDNode *N) { @@ -1426,30 +1456,10 @@ unsigned X86::getShuffleSHUFImmediate(SDNode *N) { unsigned Mask = 0; unsigned i = NumOperands - 1; do { - Mask |= cast(N->getOperand(i))->getValue(); - Mask <<= Shift; - --i; - } while (i != 0); - - return Mask; -} - -/// getShufflePSHUFDImmediate - Return the appropriate immediate to shuffle -/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFD instruction. -unsigned X86::getShufflePSHUFDImmediate(SDNode *N) { - unsigned NumOperands = N->getNumOperands(); - unsigned Mask = 0; - - assert(NumOperands == 4 && "Expect v4f32 / v4i32 vector operand"); - - unsigned i = NumOperands - 1; - do { - uint64_t Val = cast(N->getOperand(i))->getValue(); - // Second vector operand must be undef. We can have it point to anything - // we want. - if (Val >= NumOperands) Val = 0; + unsigned Val = cast(N->getOperand(i))->getValue(); + if (Val >= NumOperands) Val -= NumOperands; Mask |= Val; - Mask <<= 2; + Mask <<= Shift; --i; } while (i != 0); @@ -2246,7 +2256,7 @@ SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, DAG.getNode(ISD::UNDEF, V1.getValueType()), PermMask); - } else if (Subtarget->hasSSE2() && X86::isPSHUFDMask(PermMask.Val)) { + } else if (X86::isPSHUFDMask(PermMask.Val)) { if (V2.getOpcode() == ISD::UNDEF) // Leave the VECTOR_SHUFFLE alone. It matches PSHUFD. return SDOperand(); @@ -2255,6 +2265,21 @@ SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1, DAG.getNode(ISD::UNDEF, V1.getValueType()), PermMask); + } else if (X86::isSHUFPMask(PermMask.Val)) { + unsigned NumElems = PermMask.getNumOperands(); + SDOperand Elt = PermMask.getOperand(0); + if (cast(Elt)->getValue() >= NumElems) { + // Swap the operands and change mask. + std::vector MaskVec; + for (unsigned i = NumElems / 2; i != NumElems; ++i) + MaskVec.push_back(PermMask.getOperand(i)); + for (unsigned i = 0; i != NumElems / 2; ++i) + MaskVec.push_back(PermMask.getOperand(i)); + PermMask = + DAG.getNode(ISD::BUILD_VECTOR, PermMask.getValueType(), MaskVec); + return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V2, V1, PermMask); + } + return SDOperand(); } // TODO. @@ -2382,5 +2407,6 @@ X86TargetLowering::isShuffleMaskLegal(SDOperand Mask, MVT::ValueType VT) const { // Only do shuffles on 128-bit vector types for now. if (MVT::getSizeInBits(VT) == 64) return false; return (X86::isSplatMask(Mask.Val) || - (Subtarget->hasSSE2() && X86::isPSHUFDMask(Mask.Val))); + X86::isPSHUFDMask(Mask.Val) || + X86::isSHUFPMask(Mask.Val)); } diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h index 4a3f7a65cb2..d60ad39d9b6 100644 --- a/lib/Target/X86/X86ISelLowering.h +++ b/lib/Target/X86/X86ISelLowering.h @@ -183,6 +183,10 @@ namespace llvm { /// specifies a shuffle of elements that is suitable for input to PSHUFD. bool isPSHUFDMask(SDNode *N); + /// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand + /// specifies a shuffle of elements that is suitable for input to SHUFP*. + bool isSHUFPMask(SDNode *N); + /// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand /// specifies a splat of a single element. bool isSplatMask(SDNode *N); @@ -191,10 +195,6 @@ namespace llvm { /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP* /// instructions. unsigned getShuffleSHUFImmediate(SDNode *N); - - /// getShufflePSHUFDImmediate - Return the appropriate immediate to shuffle - /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFD instruction. - unsigned getShufflePSHUFDImmediate(SDNode *N); } //===----------------------------------------------------------------------===// diff --git a/lib/Target/X86/X86InstrSSE.td b/lib/Target/X86/X86InstrSSE.td index 0828b61b057..ed686313a0f 100644 --- a/lib/Target/X86/X86InstrSSE.td +++ b/lib/Target/X86/X86InstrSSE.td @@ -51,10 +51,6 @@ def SHUFFLE_get_shuf_imm : SDNodeXForm; -def SHUFFLE_get_pshufd_imm : SDNodeXForm; - def SHUFP_splat_mask : PatLeaf<(build_vector), [{ return X86::isSplatMask(N); }], SHUFFLE_get_shuf_imm>; @@ -66,8 +62,11 @@ def MOVLHPS_splat_mask : PatLeaf<(build_vector), [{ // Only use PSHUF if it is not a splat. def PSHUFD_shuffle_mask : PatLeaf<(build_vector), [{ return !X86::isSplatMask(N) && X86::isPSHUFDMask(N); -}], SHUFFLE_get_pshufd_imm>; +}], SHUFFLE_get_shuf_imm>; +def SHUFP_shuffle_mask : PatLeaf<(build_vector), [{ + return X86::isSHUFPMask(N); +}], SHUFFLE_get_shuf_imm>; //===----------------------------------------------------------------------===// // SSE scalar FP Instructions @@ -442,10 +441,12 @@ def MOVHPDrm : PDI<0x16, MRMSrcMem, (ops VR128:$dst, f64mem:$src), def MOVHPDmr : PDI<0x17, MRMDestMem, (ops f64mem:$dst, VR128:$src), "movhpd {$src, $dst|$dst, $src}", []>; -def MOVLHPSrr : PSI<0x16, MRMSrcReg, (ops VR128:$dst, VR128:$src), - "movlhps {$src, $dst|$dst, $src}", []>; -def MOVHLPSrr : PSI<0x12, MRMSrcReg, (ops VR128:$dst, VR128:$src), - "movlhps {$src, $dst|$dst, $src}", []>; +let isTwoAddress = 1 in { +def MOVLHPSrr : PSI<0x16, MRMSrcReg, (ops VR128:$dst, VR128:$src1, VR128:$src2), + "movlhps {$src2, $dst|$dst, $src2}", []>; +def MOVHLPSrr : PSI<0x12, MRMSrcReg, (ops VR128:$dst, VR128:$src1, VR128:$src2), + "movlhps {$src2, $dst|$dst, $src2}", []>; +} def MOVMSKPSrr : PSI<0x50, MRMSrcReg, (ops R32:$dst, VR128:$src), "movmskps {$src, $dst|$dst, $src}", []>; @@ -711,13 +712,19 @@ def PSHUFDrm : PDIi8<0x70, MRMSrcMem, let isTwoAddress = 1 in { def SHUFPSrr : PSIi8<0xC6, MRMSrcReg, (ops VR128:$dst, VR128:$src1, VR128:$src2, i8imm:$src3), - "shufps {$src3, $src2, $dst|$dst, $src2, $src3}", []>; + "shufps {$src3, $src2, $dst|$dst, $src2, $src3}", + [(set VR128:$dst, (vector_shuffle + (v4f32 VR128:$src1), (v4f32 VR128:$src2), + SHUFP_shuffle_mask:$src3))]>; def SHUFPSrm : PSIi8<0xC6, MRMSrcMem, (ops VR128:$dst, VR128:$src1, f128mem:$src2, i8imm:$src3), "shufps {$src3, $src2, $dst|$dst, $src2, $src3}", []>; def SHUFPDrr : PDIi8<0xC6, MRMSrcReg, (ops VR128:$dst, VR128:$src1, VR128:$src2, i8imm:$src3), - "shufpd {$src3, $src2, $dst|$dst, $src2, $src3}", []>; + "shufpd {$src3, $src2, $dst|$dst, $src2, $src3}", + [(set VR128:$dst, (vector_shuffle + (v2f64 VR128:$src1), (v2f64 VR128:$src2), + SHUFP_shuffle_mask:$src3))]>; def SHUFPDrm : PDIi8<0xC6, MRMSrcMem, (ops VR128:$dst, VR128:$src1, f128mem:$src2, i8imm:$src3), "shufpd {$src3, $src2, $dst|$dst, $src2, $src3}", []>; @@ -873,9 +880,9 @@ def : Pat<(vector_shuffle (v4i32 VR128:$src), (undef), SHUFP_splat_mask:$sm), // Splat v2f64 / v2i64 def : Pat<(vector_shuffle (v2f64 VR128:$src), (undef), MOVLHPS_splat_mask:$sm), - (v2f64 (MOVLHPSrr VR128:$src))>, Requires<[HasSSE1]>; + (v2f64 (MOVLHPSrr VR128:$src, VR128:$src))>, Requires<[HasSSE1]>; def : Pat<(vector_shuffle (v2i64 VR128:$src), (undef), MOVLHPS_splat_mask:$sm), - (v2i64 (MOVLHPSrr VR128:$src))>, Requires<[HasSSE1]>; + (v2i64 (MOVLHPSrr VR128:$src, VR128:$src))>, Requires<[HasSSE1]>; // Shuffle v4f32 / v4i32, undef. These should only match if splat cases do not. def : Pat<(vector_shuffle (v4f32 VR128:$src), (undef), PSHUFD_shuffle_mask:$sm),