Handle more shuffle cases with SHUFP* instructions.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@27024 91177308-0d34-0410-b5e6-96231b3b80d8

lib/Target/X86/X86ISelLowering.cpp

@@ -1383,9 +1383,7 @@ bool X86::isPSHUFDMask(SDNode *N) {
     return false;
 
   // Check if the value doesn't reference the second vector.
-  SDOperand Elt = N->getOperand(0);
+  for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) {
-  assert(isa<ConstantSDNode>(Elt) && "Invalid VECTOR_SHUFFLE mask!");
-  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
     assert(isa<ConstantSDNode>(N->getOperand(i)) &&
            "Invalid VECTOR_SHUFFLE mask!");
     if (cast<ConstantSDNode>(N->getOperand(i))->getValue() >= 4) return false;
@@ -1394,6 +1392,38 @@ bool X86::isPSHUFDMask(SDNode *N) {
   return true;
 }
 
+/// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand
+/// specifies a shuffle of elements that is suitable for input to SHUFP*.
+bool X86::isSHUFPMask(SDNode *N) {
+  assert(N->getOpcode() == ISD::BUILD_VECTOR);
+
+  unsigned NumOperands = N->getNumOperands();
+  if (NumOperands != 2 && NumOperands != 4)
+    return false;
+
+  // Each half must refer to only one of the vector.
+  SDOperand Elt = N->getOperand(0);
+  assert(isa<ConstantSDNode>(Elt) && "Invalid VECTOR_SHUFFLE mask!");
+  for (unsigned i = 1; i != NumOperands / 2; ++i) {
+    assert(isa<ConstantSDNode>(N->getOperand(i)) &&
+           "Invalid VECTOR_SHUFFLE mask!");
+    if (cast<ConstantSDNode>(N->getOperand(i))->getValue() != 
+        cast<ConstantSDNode>(Elt)->getValue())
+      return false;
+  }
+  Elt = N->getOperand(NumOperands / 2);
+  assert(isa<ConstantSDNode>(Elt) && "Invalid VECTOR_SHUFFLE mask!");
+  for (unsigned i = NumOperands / 2; i != NumOperands; ++i) {
+    assert(isa<ConstantSDNode>(N->getOperand(i)) &&
+           "Invalid VECTOR_SHUFFLE mask!");
+    if (cast<ConstantSDNode>(N->getOperand(i))->getValue() != 
+        cast<ConstantSDNode>(Elt)->getValue())
+      return false;
+  }
+
+  return true;
+}
+
 /// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand specifies
 /// a splat of a single element.
 bool X86::isSplatMask(SDNode *N) {
@@ -1426,30 +1456,10 @@ unsigned X86::getShuffleSHUFImmediate(SDNode *N) {
   unsigned Mask = 0;
   unsigned i = NumOperands - 1;
   do {
-    Mask |= cast<ConstantSDNode>(N->getOperand(i))->getValue();
+    unsigned Val = cast<ConstantSDNode>(N->getOperand(i))->getValue();
-    Mask <<= Shift;
+    if (Val >= NumOperands) Val -= NumOperands;
-    --i;
-  } while (i != 0);
-
-  return Mask;
-}
-
-/// getShufflePSHUFDImmediate - Return the appropriate immediate to shuffle
-/// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFD instruction.
-unsigned X86::getShufflePSHUFDImmediate(SDNode *N) {
-  unsigned NumOperands = N->getNumOperands();
-  unsigned Mask = 0;
-
-  assert(NumOperands == 4 && "Expect v4f32 / v4i32 vector operand");
-
-  unsigned i = NumOperands - 1;
-  do {
-    uint64_t Val = cast<ConstantSDNode>(N->getOperand(i))->getValue();
-    // Second vector operand must be undef. We can have it point to anything
-    // we want.
-    if (Val >= NumOperands) Val = 0;
     Mask |= Val;
-    Mask <<= 2;
+    Mask <<= Shift;
     --i;
   } while (i != 0);
 
@@ -2246,7 +2256,7 @@ SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) {
         return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1,
                            DAG.getNode(ISD::UNDEF, V1.getValueType()),
                            PermMask);
-    } else if (Subtarget->hasSSE2() && X86::isPSHUFDMask(PermMask.Val)) {
+    } else if (X86::isPSHUFDMask(PermMask.Val)) {
       if (V2.getOpcode() == ISD::UNDEF)
         // Leave the VECTOR_SHUFFLE alone. It matches PSHUFD.
         return SDOperand();
@@ -2255,6 +2265,21 @@ SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) {
         return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V1,
                            DAG.getNode(ISD::UNDEF, V1.getValueType()),
                            PermMask);
+    } else if (X86::isSHUFPMask(PermMask.Val)) {
+      unsigned NumElems = PermMask.getNumOperands();
+      SDOperand Elt = PermMask.getOperand(0);
+      if (cast<ConstantSDNode>(Elt)->getValue() >= NumElems) {
+        // Swap the operands and change mask.
+        std::vector<SDOperand> MaskVec;
+        for (unsigned i = NumElems / 2; i != NumElems; ++i)
+          MaskVec.push_back(PermMask.getOperand(i));
+        for (unsigned i = 0; i != NumElems / 2; ++i)
+          MaskVec.push_back(PermMask.getOperand(i));
+        PermMask =
+          DAG.getNode(ISD::BUILD_VECTOR, PermMask.getValueType(), MaskVec);
+        return DAG.getNode(ISD::VECTOR_SHUFFLE, VT, V2, V1, PermMask);
+      }
+      return SDOperand();
     }
 
     // TODO.
@@ -2382,5 +2407,6 @@ X86TargetLowering::isShuffleMaskLegal(SDOperand Mask, MVT::ValueType VT) const {
   // Only do shuffles on 128-bit vector types for now.
   if (MVT::getSizeInBits(VT) == 64) return false;
   return (X86::isSplatMask(Mask.Val) ||
-          (Subtarget->hasSSE2() && X86::isPSHUFDMask(Mask.Val)));
+          X86::isPSHUFDMask(Mask.Val) ||
+          X86::isSHUFPMask(Mask.Val));
 }

lib/Target/X86/X86ISelLowering.h

@@ -183,6 +183,10 @@ namespace llvm {
    /// specifies a shuffle of elements that is suitable for input to PSHUFD.
    bool isPSHUFDMask(SDNode *N);
 
+   /// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand
+   /// specifies a shuffle of elements that is suitable for input to SHUFP*.
+   bool isSHUFPMask(SDNode *N);
+
    /// isSplatMask - Return true if the specified VECTOR_SHUFFLE operand
    /// specifies a splat of a single element.
    bool isSplatMask(SDNode *N);
@@ -191,10 +195,6 @@ namespace llvm {
    /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUF* and SHUFP*
    /// instructions.
    unsigned getShuffleSHUFImmediate(SDNode *N);
-
-   /// getShufflePSHUFDImmediate - Return the appropriate immediate to shuffle
-   /// the specified isShuffleMask VECTOR_SHUFFLE mask with PSHUFD instruction.
-   unsigned getShufflePSHUFDImmediate(SDNode *N);
  }
 
   //===----------------------------------------------------------------------===//

lib/Target/X86/X86InstrSSE.td

@@ -51,10 +51,6 @@ def SHUFFLE_get_shuf_imm : SDNodeXForm<build_vector, [{
   return getI8Imm(X86::getShuffleSHUFImmediate(N));
 }]>;
 
-def SHUFFLE_get_pshufd_imm : SDNodeXForm<build_vector, [{
-  return getI8Imm(X86::getShufflePSHUFDImmediate(N));
-}]>;
-
 def SHUFP_splat_mask : PatLeaf<(build_vector), [{
   return X86::isSplatMask(N);
 }], SHUFFLE_get_shuf_imm>;
@@ -66,8 +62,11 @@ def MOVLHPS_splat_mask : PatLeaf<(build_vector), [{
 // Only use PSHUF if it is not a splat.
 def PSHUFD_shuffle_mask : PatLeaf<(build_vector), [{
   return !X86::isSplatMask(N) && X86::isPSHUFDMask(N);
-}], SHUFFLE_get_pshufd_imm>;
+}], SHUFFLE_get_shuf_imm>;
 
+def SHUFP_shuffle_mask : PatLeaf<(build_vector), [{
+  return X86::isSHUFPMask(N);
+}], SHUFFLE_get_shuf_imm>;
 
 //===----------------------------------------------------------------------===//
 // SSE scalar FP Instructions
@@ -442,10 +441,12 @@ def MOVHPDrm : PDI<0x16, MRMSrcMem, (ops VR128:$dst, f64mem:$src),
 def MOVHPDmr : PDI<0x17, MRMDestMem, (ops f64mem:$dst, VR128:$src),
                    "movhpd {$src, $dst|$dst, $src}", []>;
 
-def MOVLHPSrr : PSI<0x16, MRMSrcReg, (ops VR128:$dst, VR128:$src),
+let isTwoAddress = 1 in {
-                    "movlhps {$src, $dst|$dst, $src}", []>;
+def MOVLHPSrr : PSI<0x16, MRMSrcReg, (ops VR128:$dst, VR128:$src1, VR128:$src2),
-def MOVHLPSrr : PSI<0x12, MRMSrcReg, (ops VR128:$dst, VR128:$src),
+                    "movlhps {$src2, $dst|$dst, $src2}", []>;
-                    "movlhps {$src, $dst|$dst, $src}", []>;
+def MOVHLPSrr : PSI<0x12, MRMSrcReg, (ops VR128:$dst, VR128:$src1, VR128:$src2),
+                    "movlhps {$src2, $dst|$dst, $src2}", []>;
+}
 
 def MOVMSKPSrr : PSI<0x50, MRMSrcReg, (ops R32:$dst, VR128:$src),
                      "movmskps {$src, $dst|$dst, $src}", []>;
@@ -711,13 +712,19 @@ def PSHUFDrm : PDIi8<0x70, MRMSrcMem,
 let isTwoAddress = 1 in {
 def SHUFPSrr : PSIi8<0xC6, MRMSrcReg, 
                      (ops VR128:$dst, VR128:$src1, VR128:$src2, i8imm:$src3),
-                     "shufps {$src3, $src2, $dst|$dst, $src2, $src3}", []>;
+                     "shufps {$src3, $src2, $dst|$dst, $src2, $src3}",
+                    [(set VR128:$dst, (vector_shuffle
+                                       (v4f32 VR128:$src1), (v4f32 VR128:$src2),
+                                       SHUFP_shuffle_mask:$src3))]>;
 def SHUFPSrm : PSIi8<0xC6, MRMSrcMem, 
                      (ops VR128:$dst, VR128:$src1, f128mem:$src2, i8imm:$src3),
                      "shufps {$src3, $src2, $dst|$dst, $src2, $src3}", []>;
 def SHUFPDrr : PDIi8<0xC6, MRMSrcReg, 
                      (ops VR128:$dst, VR128:$src1, VR128:$src2, i8imm:$src3),
-                     "shufpd {$src3, $src2, $dst|$dst, $src2, $src3}", []>;
+                     "shufpd {$src3, $src2, $dst|$dst, $src2, $src3}",
+                    [(set VR128:$dst, (vector_shuffle
+                                       (v2f64 VR128:$src1), (v2f64 VR128:$src2),
+                                       SHUFP_shuffle_mask:$src3))]>;
 def SHUFPDrm : PDIi8<0xC6, MRMSrcMem, 
                      (ops VR128:$dst, VR128:$src1, f128mem:$src2, i8imm:$src3),
                      "shufpd {$src3, $src2, $dst|$dst, $src2, $src3}", []>;
@@ -873,9 +880,9 @@ def : Pat<(vector_shuffle (v4i32 VR128:$src), (undef), SHUFP_splat_mask:$sm),
 
 // Splat v2f64 / v2i64
 def : Pat<(vector_shuffle (v2f64 VR128:$src), (undef), MOVLHPS_splat_mask:$sm),
-          (v2f64 (MOVLHPSrr VR128:$src))>, Requires<[HasSSE1]>;
+          (v2f64 (MOVLHPSrr VR128:$src, VR128:$src))>, Requires<[HasSSE1]>;
 def : Pat<(vector_shuffle (v2i64 VR128:$src), (undef), MOVLHPS_splat_mask:$sm),
-          (v2i64 (MOVLHPSrr VR128:$src))>, Requires<[HasSSE1]>;
+          (v2i64 (MOVLHPSrr VR128:$src, VR128:$src))>, Requires<[HasSSE1]>;
 
 // Shuffle v4f32 / v4i32, undef. These should only match if splat cases do not.
 def : Pat<(vector_shuffle (v4f32 VR128:$src), (undef), PSHUFD_shuffle_mask:$sm),