diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 63255ae2ebd..a8bccda291b 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -1369,89 +1369,8 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { break; } - // If this is a promoted vector load, and the vector element types are - // legal, then scalarize it. - if (ExtType == ISD::EXTLOAD && SrcVT.isVector() && - TLI.isTypeLegal(Node->getValueType(0).getScalarType())) { - SmallVector LoadVals; - SmallVector LoadChains; - unsigned NumElem = SrcVT.getVectorNumElements(); - unsigned Stride = SrcVT.getScalarType().getSizeInBits()/8; - - for (unsigned Idx=0; IdxgetValueType(0).getScalarType(), - Tmp1, Tmp2, LD->getPointerInfo().getWithOffset(Idx * Stride), - SrcVT.getScalarType(), - LD->isVolatile(), LD->isNonTemporal(), - LD->getAlignment()); - - LoadVals.push_back(ScalarLoad.getValue(0)); - LoadChains.push_back(ScalarLoad.getValue(1)); - } - Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, - &LoadChains[0], LoadChains.size()); - SDValue ValRes = DAG.getNode(ISD::BUILD_VECTOR, dl, - Node->getValueType(0), &LoadVals[0], LoadVals.size()); - - Tmp1 = LegalizeOp(ValRes); // Relegalize new nodes. - Tmp2 = LegalizeOp(Result.getValue(0)); // Relegalize new nodes. - break; - } - - // If this is a promoted vector load, and the vector element types are - // illegal, create the promoted vector from bitcasted segments. - if (ExtType == ISD::EXTLOAD && SrcVT.isVector()) { - EVT MemElemTy = Node->getValueType(0).getScalarType(); - EVT SrcSclrTy = SrcVT.getScalarType(); - unsigned SizeRatio = - (MemElemTy.getSizeInBits() / SrcSclrTy.getSizeInBits()); - - SmallVector LoadVals; - SmallVector LoadChains; - unsigned NumElem = SrcVT.getVectorNumElements(); - unsigned Stride = SrcVT.getScalarType().getSizeInBits()/8; - - for (unsigned Idx=0; IdxgetPointerInfo().getWithOffset(Idx * Stride), - SrcVT.getScalarType(), - LD->isVolatile(), LD->isNonTemporal(), - LD->getAlignment()); - if (TLI.isBigEndian()) { - // MSB (which is garbage, comes first) - LoadVals.push_back(ScalarLoad.getValue(0)); - for (unsigned i = 0; igetValueType(0), ValRes); - - Tmp1 = LegalizeOp(ValRes); // Relegalize new nodes. - Tmp2 = LegalizeOp(Result.getValue(0)); // Relegalize new nodes. - break; - - } + assert(!SrcVT.isVector() && + "Vector Loads are handled in LegalizeVectorOps"); // FIXME: This does not work for vectors on most targets. Sign- and // zero-extend operations are currently folded into extending loads, @@ -1628,106 +1547,8 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { Result = TLI.LowerOperation(Result, DAG); break; case TargetLowering::Expand: - - EVT WideScalarVT = Tmp3.getValueType().getScalarType(); - EVT NarrowScalarVT = StVT.getScalarType(); - - if (StVT.isVector()) { - unsigned NumElem = StVT.getVectorNumElements(); - // The type of the data we want to save - EVT RegVT = Tmp3.getValueType(); - EVT RegSclVT = RegVT.getScalarType(); - // The type of data as saved in memory. - EVT MemSclVT = StVT.getScalarType(); - - bool RegScalarLegal = TLI.isTypeLegal(RegSclVT); - bool MemScalarLegal = TLI.isTypeLegal(MemSclVT); - - // We need to expand this store. If the register element type - // is legal then we can scalarize the vector and use - // truncating stores. - if (RegScalarLegal) { - // Cast floats into integers - unsigned ScalarSize = MemSclVT.getSizeInBits(); - EVT EltVT = EVT::getIntegerVT(*DAG.getContext(), ScalarSize); - - // Round odd types to the next pow of two. - if (!isPowerOf2_32(ScalarSize)) - ScalarSize = NextPowerOf2(ScalarSize); - - // Store Stride in bytes - unsigned Stride = ScalarSize/8; - // Extract each of the elements from the original vector - // and save them into memory individually. - SmallVector Stores; - for (unsigned Idx = 0; Idx < NumElem; Idx++) { - SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, - RegSclVT, Tmp3, DAG.getIntPtrConstant(Idx)); - - Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, - DAG.getIntPtrConstant(Stride)); - - // This scalar TruncStore may be illegal, but we lehalize it - // later. - SDValue Store = DAG.getTruncStore(Tmp1, dl, Ex, Tmp2, - ST->getPointerInfo().getWithOffset(Idx*Stride), MemSclVT, - isVolatile, isNonTemporal, Alignment); - - Stores.push_back(Store); - } - - Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, - &Stores[0], Stores.size()); - break; - } - - // The scalar register type is illegal. - // For example saving <2 x i64> -> <2 x i32> on a x86. - // In here we bitcast the value into a vector of smaller parts and - // save it using smaller scalars. - if (!RegScalarLegal && MemScalarLegal) { - // Store Stride in bytes - unsigned Stride = MemSclVT.getSizeInBits()/8; - - unsigned SizeRatio = - (RegSclVT.getSizeInBits() / MemSclVT.getSizeInBits()); - - EVT CastValueVT = EVT::getVectorVT(*DAG.getContext(), - MemSclVT, - SizeRatio * NumElem); - - // Cast the wide elem vector to wider vec with smaller elem type. - // Example <2 x i64> -> <4 x i32> - Tmp3 = DAG.getNode(ISD::BITCAST, dl, CastValueVT, Tmp3); - - SmallVector Stores; - for (unsigned Idx=0; Idx < NumElem * SizeRatio; Idx++) { - // Extract the Ith element. - SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, - NarrowScalarVT, Tmp3, DAG.getIntPtrConstant(Idx)); - // Bump pointer. - Tmp2 = DAG.getNode(ISD::ADD, dl, Tmp2.getValueType(), Tmp2, - DAG.getIntPtrConstant(Stride)); - - // Store if, this element is: - // - First element on big endian, or - // - Last element on little endian - if (( TLI.isBigEndian() && (Idx % SizeRatio == 0)) || - ((!TLI.isBigEndian() && (Idx % SizeRatio == SizeRatio-1)))) { - SDValue Store = DAG.getStore(Tmp1, dl, Ex, Tmp2, - ST->getPointerInfo().getWithOffset(Idx*Stride), - isVolatile, isNonTemporal, Alignment); - Stores.push_back(Store); - } - } - Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, - &Stores[0], Stores.size()); - break; - } - - assert(false && "Unable to legalize the vector trunc store!"); - }// is vector - + assert(!StVT.isVector() && + "Vector Stores are handled in LegalizeVectorOps"); // TRUNCSTORE:i16 i32 -> STORE i16 assert(TLI.isTypeLegal(StVT) && "Do not know how to expand this store!"); diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index 80b4c6053a6..ca67c22f50a 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -64,6 +64,8 @@ class VectorLegalizer { // Implement vselect in terms of XOR, AND, OR when blend is not supported // by the target. SDValue ExpandVSELECT(SDValue Op); + SDValue ExpandLoad(SDValue Op); + SDValue ExpandStore(SDValue Op); SDValue ExpandFNEG(SDValue Op); // Implements vector promotion; this is essentially just bitcasting the // operands to a different type and bitcasting the result back to the @@ -124,6 +126,33 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { SDValue Result = SDValue(DAG.UpdateNodeOperands(Op.getNode(), Ops.data(), Ops.size()), 0); + if (Op.getOpcode() == ISD::LOAD) { + LoadSDNode *LD = cast(Op.getNode()); + ISD::LoadExtType ExtType = LD->getExtensionType(); + if (LD->getMemoryVT().isVector() && ExtType != ISD::NON_EXTLOAD) { + if (TLI.isLoadExtLegal(LD->getExtensionType(), LD->getMemoryVT())) + return TranslateLegalizeResults(Op, Result); + Changed = true; + return LegalizeOp(ExpandLoad(Op)); + } + } else if (Op.getOpcode() == ISD::STORE) { + StoreSDNode *ST = cast(Op.getNode()); + EVT StVT = ST->getMemoryVT(); + EVT ValVT = ST->getValue().getValueType(); + if (StVT.isVector() && ST->isTruncatingStore()) + switch (TLI.getTruncStoreAction(ValVT, StVT)) { + default: assert(0 && "This action is not supported yet!"); + case TargetLowering::Legal: + return TranslateLegalizeResults(Op, Result); + case TargetLowering::Custom: + Changed = true; + return LegalizeOp(TLI.LowerOperation(Result, DAG)); + case TargetLowering::Expand: + Changed = true; + return LegalizeOp(ExpandStore(Op)); + } + } + bool HasVectorValue = false; for (SDNode::value_iterator J = Node->value_begin(), E = Node->value_end(); J != E; @@ -262,6 +291,96 @@ SDValue VectorLegalizer::PromoteVectorOp(SDValue Op) { return DAG.getNode(ISD::BITCAST, dl, VT, Op); } + +SDValue VectorLegalizer::ExpandLoad(SDValue Op) { + DebugLoc dl = Op.getDebugLoc(); + LoadSDNode *LD = cast(Op.getNode()); + SDValue Chain = LD->getChain(); + SDValue BasePTR = LD->getBasePtr(); + EVT SrcVT = LD->getMemoryVT(); + + SmallVector LoadVals; + SmallVector LoadChains; + unsigned NumElem = SrcVT.getVectorNumElements(); + unsigned Stride = SrcVT.getScalarType().getSizeInBits()/8; + + for (unsigned Idx=0; IdxgetValueType(0).getScalarType(), + Chain, BasePTR, LD->getPointerInfo().getWithOffset(Idx * Stride), + SrcVT.getScalarType(), + LD->isVolatile(), LD->isNonTemporal(), + LD->getAlignment()); + + LoadVals.push_back(ScalarLoad.getValue(0)); + LoadChains.push_back(ScalarLoad.getValue(1)); + } + + SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, + &LoadChains[0], LoadChains.size()); + SDValue Value = DAG.getNode(ISD::BUILD_VECTOR, dl, + Op.getNode()->getValueType(0), &LoadVals[0], LoadVals.size()); + + AddLegalizedOperand(Op.getValue(0), Value); + AddLegalizedOperand(Op.getValue(1), NewChain); + + return (Op.getResNo() ? NewChain : Value); +} + +SDValue VectorLegalizer::ExpandStore(SDValue Op) { + DebugLoc dl = Op.getDebugLoc(); + StoreSDNode *ST = cast(Op.getNode()); + SDValue Chain = ST->getChain(); + SDValue BasePTR = ST->getBasePtr(); + SDValue Value = ST->getValue(); + EVT StVT = ST->getMemoryVT(); + + unsigned Alignment = ST->getAlignment(); + bool isVolatile = ST->isVolatile(); + bool isNonTemporal = ST->isNonTemporal(); + + unsigned NumElem = StVT.getVectorNumElements(); + // The type of the data we want to save + EVT RegVT = Value.getValueType(); + EVT RegSclVT = RegVT.getScalarType(); + // The type of data as saved in memory. + EVT MemSclVT = StVT.getScalarType(); + + // Cast floats into integers + unsigned ScalarSize = MemSclVT.getSizeInBits(); + EVT EltVT = EVT::getIntegerVT(*DAG.getContext(), ScalarSize); + + // Round odd types to the next pow of two. + if (!isPowerOf2_32(ScalarSize)) + ScalarSize = NextPowerOf2(ScalarSize); + + // Store Stride in bytes + unsigned Stride = ScalarSize/8; + // Extract each of the elements from the original vector + // and save them into memory individually. + SmallVector Stores; + for (unsigned Idx = 0; Idx < NumElem; Idx++) { + SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, + RegSclVT, Value, DAG.getIntPtrConstant(Idx)); + + BasePTR = DAG.getNode(ISD::ADD, dl, BasePTR.getValueType(), BasePTR, + DAG.getIntPtrConstant(Stride)); + + // This scalar TruncStore may be illegal, but we legalize it later. + SDValue Store = DAG.getTruncStore(Chain, dl, Ex, BasePTR, + ST->getPointerInfo().getWithOffset(Idx*Stride), MemSclVT, + isVolatile, isNonTemporal, Alignment); + + Stores.push_back(Store); + } + SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, + &Stores[0], Stores.size()); + AddLegalizedOperand(Op, TF); + return TF; +} + SDValue VectorLegalizer::ExpandVSELECT(SDValue Op) { // Implement VSELECT in terms of XOR, AND, OR // on platforms which do not support blend natively.