diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index 515634908af..aeaf42f03db 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -290,7 +290,11 @@ namespace ISD { /// BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector /// with the specified, possibly variable, elements. The number of elements - /// is required to be a power of two. + /// is required to be a power of two. The types of the operands must + /// all be the same. They must match the vector element type, except if an + /// integer element type is not legal for the target, the operands may + /// be promoted to a legal type, in which case the operands are implicitly + /// truncated to the vector element types. BUILD_VECTOR, /// INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 5af12eb1745..dd4fda1e842 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -3795,7 +3795,7 @@ SDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) { /// destination element value type. SDValue DAGCombiner:: ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT DstEltVT) { - MVT SrcEltVT = BV->getOperand(0).getValueType(); + MVT SrcEltVT = BV->getValueType(0).getVectorElementType(); // If this is already the right type, we're done. if (SrcEltVT == DstEltVT) return SDValue(BV, 0); @@ -3808,8 +3808,17 @@ ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT DstEltVT) { if (SrcBitSize == DstBitSize) { SmallVector Ops; for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) { + SDValue Op = BV->getOperand(i); + // If the vector element type is not legal, the BUILD_VECTOR operands + // are promoted and implicitly truncated. Make that explicit here. + if (Op.getValueType() != SrcEltVT) { + if (Op.getOpcode() == ISD::UNDEF) + Op = DAG.getUNDEF(SrcEltVT); + else + Op = DAG.getNode(ISD::TRUNCATE, BV->getDebugLoc(), SrcEltVT, Op); + } Ops.push_back(DAG.getNode(ISD::BIT_CONVERT, BV->getDebugLoc(), - DstEltVT, BV->getOperand(i))); + DstEltVT, Op)); AddToWorkList(Ops.back().getNode()); } MVT VT = MVT::getVectorVT(DstEltVT, @@ -3860,8 +3869,8 @@ ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT DstEltVT) { if (Op.getOpcode() == ISD::UNDEF) continue; EltIsUndef = false; - NewBits |= - APInt(cast(Op)->getAPIntValue()).zext(DstBitSize); + NewBits |= (APInt(cast(Op)->getAPIntValue()). + zextOrTrunc(SrcBitSize).zext(DstBitSize)); } if (EltIsUndef) @@ -3889,7 +3898,8 @@ ConstantFoldBIT_CONVERTofBUILD_VECTOR(SDNode *BV, MVT DstEltVT) { continue; } - APInt OpVal = cast(BV->getOperand(i))->getAPIntValue(); + APInt OpVal = APInt(cast(BV->getOperand(i))-> + getAPIntValue()).zextOrTrunc(SrcBitSize); for (unsigned j = 0; j != NumOutputsPerInput; ++j) { APInt ThisVal = APInt(OpVal).trunc(DstBitSize); diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 40785da8cfd..55a52b600a7 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -5483,6 +5483,41 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { MVT OpVT = SplatValue.getValueType(); MVT EltVT = VT.getVectorElementType(); + // Check if the BUILD_VECTOR operands were promoted to legalize their types. + if (OpVT != EltVT) { + // Now that the DAG combiner and target-specific lowering have had a + // chance to optimize/recognize the BUILD_VECTOR with promoted operands, + // transform it so the operand types match the vector. Build a vector of + // half the length out of elements of twice the bitwidth. + // For example <4 x i16> -> <2 x i32>. + MVT NewVT = MVT::getIntegerVT(2 * EltVT.getSizeInBits()); + assert(OpVT.isSimple() && NewVT.isSimple()); + SmallVector NewElts; + + for (unsigned i = 0; i < NumElems; i += 2) { + // Combine two successive elements into one promoted element. + SDValue Lo = Node->getOperand(i); + SDValue Hi = Node->getOperand(i+1); + if (TLI.isBigEndian()) + std::swap(Lo, Hi); + Lo = DAG.getZeroExtendInReg(Lo, dl, EltVT); + Hi = DAG.getNode(ISD::SHL, dl, OpVT, Hi, + DAG.getConstant(EltVT.getSizeInBits(), + TLI.getPointerTy())); + NewElts.push_back(DAG.getNode(ISD::OR, dl, OpVT, Lo, Hi)); + } + + SDValue NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, + MVT::getVectorVT(NewVT, NewElts.size()), + &NewElts[0], NewElts.size()); + + // Recurse + NewVec = ExpandBUILD_VECTOR(NewVec.getNode()); + + // Convert the new vector to the old vector type. + return DAG.getNode(ISD::BIT_CONVERT, dl, VT, NewVec); + } + // If the only non-undef value is the low element, turn this into a // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. bool isOnlyLowElement = true; diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index 66a57dc4320..115c9f8924f 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -799,32 +799,20 @@ SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { MVT VecVT = N->getValueType(0); unsigned NumElts = VecVT.getVectorNumElements(); assert(!(NumElts & 1) && "Legal vector of one illegal element?"); - DebugLoc dl = N->getDebugLoc(); - // Build a vector of half the length out of elements of twice the bitwidth. - // For example <4 x i16> -> <2 x i32>. - MVT OldVT = N->getOperand(0).getValueType(); - MVT NewVT = MVT::getIntegerVT(2 * OldVT.getSizeInBits()); - assert(OldVT.isSimple() && NewVT.isSimple()); + // Promote the inserted value. The type does not need to match the + // vector element type. Check that any extra bits introduced will be + // truncated away. + assert(N->getOperand(0).getValueType().getSizeInBits() >= + N->getValueType(0).getVectorElementType().getSizeInBits() && + "Type of inserted value narrower than vector element type!"); - std::vector NewElts; - NewElts.reserve(NumElts/2); - - for (unsigned i = 0; i < NumElts; i += 2) { - // Combine two successive elements into one promoted element. - SDValue Lo = N->getOperand(i); - SDValue Hi = N->getOperand(i+1); - if (TLI.isBigEndian()) - std::swap(Lo, Hi); - NewElts.push_back(JoinIntegers(Lo, Hi)); + SmallVector NewOps; + for (unsigned i = 0; i < NumElts; ++i) { + NewOps.push_back(GetPromotedInteger(N->getOperand(i))); } - SDValue NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl, - MVT::getVectorVT(NewVT, NewElts.size()), - &NewElts[0], NewElts.size()); - - // Convert the new vector to the old vector type. - return DAG.getNode(ISD::BIT_CONVERT, dl, VecVT, NewVec); + return DAG.UpdateNodeOperands(SDValue(N, 0), &NewOps[0], NumElts); } SDValue DAGTypeLegalizer::PromoteIntOp_CONVERT_RNDSAT(SDNode *N) { diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp index 9333120de72..f1d1f5b077e 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp @@ -278,6 +278,9 @@ SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) { MVT NewVT = TLI.getTypeToTransformTo(OldVT); DebugLoc dl = N->getDebugLoc(); + assert(OldVT == VecVT.getVectorElementType() && + "BUILD_VECTOR operand type doesn't match vector element type!"); + // Build a vector of twice the length out of the expanded elements. // For example <3 x i64> -> <6 x i32>. std::vector NewElts; diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 0f156238985..022b961331a 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -768,7 +768,8 @@ void SelectionDAG::VerifyNode(SDNode *N) { // following checks at least makes it possible to legalize most of the time. // MVT EltVT = N->getValueType(0).getVectorElementType(); // for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) -// assert(I->getValueType() == EltVT && +// assert((I->getValueType() == EltVT || +// I->getValueType() == TLI.getTypeToTransformTo(EltVT)) && // "Wrong operand type!"); break; } @@ -2550,8 +2551,17 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, MVT VT, // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is // expanding large vector constants. - if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) - return N1.getOperand(N2C->getZExtValue()); + if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) { + SDValue Elt = N1.getOperand(N2C->getZExtValue()); + if (Elt.getValueType() != VT) { + // If the vector element type is not legal, the BUILD_VECTOR operands + // are promoted and implicitly truncated. Make that explicit here. + assert(Elt.getValueType() == TLI.getTypeToTransformTo(VT) && + "Bad type for BUILD_VECTOR operand"); + Elt = getNode(ISD::TRUNCATE, DL, VT, Elt); + } + return Elt; + } // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector // operations are lowered to scalars. @@ -5569,7 +5579,8 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, if (OpVal.getOpcode() == ISD::UNDEF) SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos +EltBitSize); else if (ConstantSDNode *CN = dyn_cast(OpVal)) - SplatValue |= APInt(CN->getAPIntValue()).zextOrTrunc(sz) << BitPos; + SplatValue |= (APInt(CN->getAPIntValue()).zextOrTrunc(EltBitSize). + zextOrTrunc(sz) << BitPos); else if (ConstantFPSDNode *CN = dyn_cast(OpVal)) SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <