Change SelectionDAG type legalization to allow BUILD_VECTOR operands to be

promoted to legal types without changing the type of the vector.  This is
following a suggestion from Duncan
(http://lists.cs.uiuc.edu/pipermail/llvmdev/2009-February/019923.html).
The transformation that used to be done during type legalization is now
postponed to DAG legalization.  This allows the BUILD_VECTORs to be optimized
and potentially handled specially by target-specific code.

It turns out that this is also consistent with an optimization done by the
DAG combiner: a BUILD_VECTOR and INSERT_VECTOR_ELT may be combined by
replacing one of the BUILD_VECTOR operands with the newly inserted element;
but INSERT_VECTOR_ELT allows its scalar operand to be larger than the
element type, with any extra high bits being implicitly truncated.  The
result is a BUILD_VECTOR where one of the operands has a type larger the
the vector element type.

Any code that operates on BUILD_VECTORs may now need to be aware of the
potential type discrepancy between the vector element type and the
BUILD_VECTOR operands.  This patch updates all of the places that I could
find to handle that case.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@68996 91177308-0d34-0410-b5e6-96231b3b80d8

include/llvm/CodeGen/SelectionDAGNodes.h

@@ -290,7 +290,11 @@ namespace ISD {
 
     /// BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector
     /// with the specified, possibly variable, elements.  The number of elements
-    /// is required to be a power of two.
+    /// is required to be a power of two.  The types of the operands must
+    /// all be the same.  They must match the vector element type, except if an 
+    /// integer element type is not legal for the target, the operands may 
+    /// be promoted to a legal type, in which case the operands are implicitly
+    /// truncated to the vector element types.
     BUILD_VECTOR,
 
     /// INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element

lib/CodeGen/SelectionDAG/DAGCombiner.cpp

@@ -3795,7 +3795,7 @@ SDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) {
 /// destination element value type.
 SDValue DAGCombiner::
 ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT DstEltVT) {
-  MVT SrcEltVT = BV->getOperand(0).getValueType();
+  MVT SrcEltVT = BV->getValueType(0).getVectorElementType();
 
   // If this is already the right type, we're done.
   if (SrcEltVT == DstEltVT) return SDValue(BV, 0);
@@ -3808,8 +3808,17 @@ ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT DstEltVT) {
   if (SrcBitSize == DstBitSize) {
     SmallVector<SDValue, 8> Ops;
     for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
+      SDValue Op = BV->getOperand(i);
+      // If the vector element type is not legal, the BUILD_VECTOR operands
+      // are promoted and implicitly truncated.  Make that explicit here.
+      if (Op.getValueType() != SrcEltVT) {
+        if (Op.getOpcode() == ISD::UNDEF)
+          Op = DAG.getUNDEF(SrcEltVT);
+        else
+          Op = DAG.getNode(ISD::TRUNCATE, BV->getDebugLoc(), SrcEltVT, Op);
+      }
       Ops.push_back(DAG.getNode(ISD::BIT_CONVERT, BV->getDebugLoc(),
-                                DstEltVT, BV->getOperand(i)));
+                                DstEltVT, Op));
       AddToWorkList(Ops.back().getNode());
     }
     MVT VT = MVT::getVectorVT(DstEltVT,
@@ -3860,8 +3869,8 @@ ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT DstEltVT) {
         if (Op.getOpcode() == ISD::UNDEF) continue;
         EltIsUndef = false;
 
-        NewBits |=
+        NewBits |= (APInt(cast<ConstantSDNode>(Op)->getAPIntValue()).
-          APInt(cast<ConstantSDNode>(Op)->getAPIntValue()).zext(DstBitSize);
+                    zextOrTrunc(SrcBitSize).zext(DstBitSize));
       }
 
       if (EltIsUndef)
@@ -3889,7 +3898,8 @@ ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT DstEltVT) {
       continue;
     }
 
-    APInt OpVal = cast<ConstantSDNode>(BV->getOperand(i))->getAPIntValue();
+    APInt OpVal = APInt(cast<ConstantSDNode>(BV->getOperand(i))->
+                        getAPIntValue()).zextOrTrunc(SrcBitSize);
 
     for (unsigned j = 0; j != NumOutputsPerInput; ++j) {
       APInt ThisVal = APInt(OpVal).trunc(DstBitSize);

lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

@@ -5483,6 +5483,41 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
   MVT OpVT = SplatValue.getValueType();
   MVT EltVT = VT.getVectorElementType();
 
+  // Check if the BUILD_VECTOR operands were promoted to legalize their types.
+  if (OpVT != EltVT) {
+    // Now that the DAG combiner and target-specific lowering have had a
+    // chance to optimize/recognize the BUILD_VECTOR with promoted operands,
+    // transform it so the operand types match the vector.  Build a vector of
+    // half the length out of elements of twice the bitwidth.
+    // For example <4 x i16> -> <2 x i32>.
+    MVT NewVT = MVT::getIntegerVT(2 * EltVT.getSizeInBits());
+    assert(OpVT.isSimple() && NewVT.isSimple());
+    SmallVector<SDValue, 16> NewElts;
+
+    for (unsigned i = 0; i < NumElems; i += 2) {
+      // Combine two successive elements into one promoted element.
+      SDValue Lo = Node->getOperand(i);
+      SDValue Hi = Node->getOperand(i+1);
+      if (TLI.isBigEndian())
+        std::swap(Lo, Hi);
+      Lo = DAG.getZeroExtendInReg(Lo, dl, EltVT);
+      Hi = DAG.getNode(ISD::SHL, dl, OpVT, Hi,
+                       DAG.getConstant(EltVT.getSizeInBits(),
+                                       TLI.getPointerTy()));
+      NewElts.push_back(DAG.getNode(ISD::OR, dl, OpVT, Lo, Hi));
+    }
+
+    SDValue NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl,
+                                 MVT::getVectorVT(NewVT, NewElts.size()),
+                                 &NewElts[0], NewElts.size());
+
+    // Recurse
+    NewVec = ExpandBUILD_VECTOR(NewVec.getNode());
+
+    // Convert the new vector to the old vector type.
+    return DAG.getNode(ISD::BIT_CONVERT, dl, VT, NewVec);
+  }
+
   // If the only non-undef value is the low element, turn this into a
   // SCALAR_TO_VECTOR node.  If this is { X, X, X, X }, determine X.
   bool isOnlyLowElement = true;

lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

@@ -799,32 +799,20 @@ SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
   MVT VecVT = N->getValueType(0);
   unsigned NumElts = VecVT.getVectorNumElements();
   assert(!(NumElts & 1) && "Legal vector of one illegal element?");
-  DebugLoc dl = N->getDebugLoc();
 
-  // Build a vector of half the length out of elements of twice the bitwidth.
+  // Promote the inserted value.  The type does not need to match the
-  // For example <4 x i16> -> <2 x i32>.
+  // vector element type.  Check that any extra bits introduced will be
-  MVT OldVT = N->getOperand(0).getValueType();
+  // truncated away.
-  MVT NewVT = MVT::getIntegerVT(2 * OldVT.getSizeInBits());
+  assert(N->getOperand(0).getValueType().getSizeInBits() >=
-  assert(OldVT.isSimple() && NewVT.isSimple());
+         N->getValueType(0).getVectorElementType().getSizeInBits() &&
+         "Type of inserted value narrower than vector element type!");
 
-  std::vector<SDValue> NewElts;
+  SmallVector<SDValue, 16> NewOps;
-  NewElts.reserve(NumElts/2);
+  for (unsigned i = 0; i < NumElts; ++i) {
-
+    NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
-  for (unsigned i = 0; i < NumElts; i += 2) {
-    // Combine two successive elements into one promoted element.
-    SDValue Lo = N->getOperand(i);
-    SDValue Hi = N->getOperand(i+1);
-    if (TLI.isBigEndian())
-      std::swap(Lo, Hi);
-    NewElts.push_back(JoinIntegers(Lo, Hi));
   }
 
-  SDValue NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl,
+  return DAG.UpdateNodeOperands(SDValue(N, 0), &NewOps[0], NumElts);
-                                 MVT::getVectorVT(NewVT, NewElts.size()),
-                                 &NewElts[0], NewElts.size());
-
-  // Convert the new vector to the old vector type.
-  return DAG.getNode(ISD::BIT_CONVERT, dl, VecVT, NewVec);
 }
 
 SDValue DAGTypeLegalizer::PromoteIntOp_CONVERT_RNDSAT(SDNode *N) {

lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp

@@ -278,6 +278,9 @@ SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) {
   MVT NewVT = TLI.getTypeToTransformTo(OldVT);
   DebugLoc dl = N->getDebugLoc();
 
+  assert(OldVT == VecVT.getVectorElementType() &&
+         "BUILD_VECTOR operand type doesn't match vector element type!");
+
   // Build a vector of twice the length out of the expanded elements.
   // For example <3 x i64> -> <6 x i32>.
   std::vector<SDValue> NewElts;

lib/CodeGen/SelectionDAG/SelectionDAG.cpp

@@ -768,7 +768,8 @@ void SelectionDAG::VerifyNode(SDNode *N) {
     // following checks at least makes it possible to legalize most of the time.
 //    MVT EltVT = N->getValueType(0).getVectorElementType();
 //    for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I)
-//      assert(I->getValueType() == EltVT &&
+//      assert((I->getValueType() == EltVT ||
+//              I->getValueType() == TLI.getTypeToTransformTo(EltVT)) &&
 //             "Wrong operand type!");
     break;
   }
@@ -2550,8 +2551,17 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, MVT VT,
 
     // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
     // expanding large vector constants.
-    if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR)
+    if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
-      return N1.getOperand(N2C->getZExtValue());
+      SDValue Elt = N1.getOperand(N2C->getZExtValue());
+      if (Elt.getValueType() != VT) {
+        // If the vector element type is not legal, the BUILD_VECTOR operands
+        // are promoted and implicitly truncated.  Make that explicit here.
+        assert(Elt.getValueType() == TLI.getTypeToTransformTo(VT) &&
+               "Bad type for BUILD_VECTOR operand");
+        Elt = getNode(ISD::TRUNCATE, DL, VT, Elt);
+      }
+      return Elt;
+    }
 
     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
     // operations are lowered to scalars.
@@ -5569,7 +5579,8 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue,
     if (OpVal.getOpcode() == ISD::UNDEF)
       SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos +EltBitSize);
     else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal))
-      SplatValue |= APInt(CN->getAPIntValue()).zextOrTrunc(sz) << BitPos;
+      SplatValue |= (APInt(CN->getAPIntValue()).zextOrTrunc(EltBitSize).
+                     zextOrTrunc(sz) << BitPos);
     else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal))
       SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos;
      else