diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index d594cfe7efa..370772faab1 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -160,12 +160,15 @@ private: void LegalizeSetCCOperands(SDOperand &LHS, SDOperand &RHS, SDOperand &CC); + SDOperand CreateStackTemporary(MVT::ValueType VT); + SDOperand ExpandLibCall(const char *Name, SDNode *Node, SDOperand &Hi); SDOperand ExpandIntToFP(bool isSigned, MVT::ValueType DestTy, SDOperand Source); SDOperand ExpandBIT_CONVERT(MVT::ValueType DestVT, SDOperand SrcOp); + SDOperand ExpandBUILD_VECTOR(SDNode *Node); SDOperand ExpandLegalINT_TO_FP(bool isSigned, SDOperand LegalOp, MVT::ValueType DestVT); @@ -737,91 +740,10 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { break; } // FALLTHROUGH - case TargetLowering::Expand: { - // We assume that built vectors are not legal, and will be immediately - // spilled to memory. If the values are all constants, turn this into a - // load from the constant pool. - bool isConstant = true; - for (SDNode::op_iterator I = Node->op_begin(), E = Node->op_end(); - I != E; ++I) { - if (!isa(I) && !isa(I) && - I->getOpcode() != ISD::UNDEF) { - isConstant = false; - break; - } - } - - // Create a ConstantPacked, and put it in the constant pool. - if (isConstant) { - MVT::ValueType VT = Node->getValueType(0); - const Type *OpNTy = - MVT::getTypeForValueType(Node->getOperand(0).getValueType()); - std::vector CV; - for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { - if (ConstantFPSDNode *V = - dyn_cast(Node->getOperand(i))) { - CV.push_back(ConstantFP::get(OpNTy, V->getValue())); - } else if (ConstantSDNode *V = - dyn_cast(Node->getOperand(i))) { - CV.push_back(ConstantUInt::get(OpNTy, V->getValue())); - } else { - assert(Node->getOperand(i).getOpcode() == ISD::UNDEF); - CV.push_back(UndefValue::get(OpNTy)); - } - } - Constant *CP = ConstantPacked::get(CV); - SDOperand CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy()); - Result = DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, - DAG.getSrcValue(NULL)); - break; - } - - // Otherwise, this isn't a constant entry. Allocate a sufficiently - // aligned object on the stack, store each element into it, then load - // the result as a vector. - MVT::ValueType VT = Node->getValueType(0); - // Create the stack frame object. - MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo(); - unsigned ByteSize = MVT::getSizeInBits(VT)/8; - int FrameIdx = FrameInfo->CreateStackObject(ByteSize, ByteSize); - SDOperand FIPtr = DAG.getFrameIndex(FrameIdx, TLI.getPointerTy()); - - // Emit a store of each element to the stack slot. - std::vector Stores; - bool isLittleEndian = TLI.isLittleEndian(); - unsigned TypeByteSize = - MVT::getSizeInBits(Node->getOperand(0).getValueType())/8; - unsigned VectorSize = MVT::getSizeInBits(VT)/8; - // Store (in the right endianness) the elements to memory. - for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { - // Ignore undef elements. - if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue; - - unsigned Offset; - if (isLittleEndian) - Offset = TypeByteSize*i; - else - Offset = TypeByteSize*(e-i-1); - - SDOperand Idx = DAG.getConstant(Offset, FIPtr.getValueType()); - Idx = DAG.getNode(ISD::ADD, FIPtr.getValueType(), FIPtr, Idx); - - Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, DAG.getEntryNode(), - Node->getOperand(i), Idx, - DAG.getSrcValue(NULL))); - } - - SDOperand StoreChain; - if (!Stores.empty()) // Not all undef elements? - StoreChain = DAG.getNode(ISD::TokenFactor, MVT::Other, Stores); - else - StoreChain = DAG.getEntryNode(); - - // Result is a load from the stack slot. - Result = DAG.getLoad(VT, StoreChain, FIPtr, DAG.getSrcValue(0)); + case TargetLowering::Expand: + Result = ExpandBUILD_VECTOR(Result.Val); break; } - } break; case ISD::INSERT_VECTOR_ELT: Tmp1 = LegalizeOp(Node->getOperand(0)); // InVec @@ -853,6 +775,29 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) { } } break; + case ISD::SCALAR_TO_VECTOR: + Tmp1 = LegalizeOp(Node->getOperand(0)); // InVal + Result = DAG.UpdateNodeOperands(Result, Tmp1); + switch (TLI.getOperationAction(ISD::SCALAR_TO_VECTOR, + Node->getValueType(0))) { + default: assert(0 && "This action is not supported yet!"); + case TargetLowering::Legal: + break; + case TargetLowering::Expand: { + // If the target doesn't support this, store the value to a temporary + // stack slot, then EXTLOAD the vector back out. + SDOperand StackPtr = + CreateStackTemporary(Node->getOperand(0).getValueType()); + SDOperand Ch = DAG.getNode(ISD::STORE, MVT::Other, DAG.getEntryNode(), + Node->getOperand(0), StackPtr, + DAG.getSrcValue(NULL)); + Result = DAG.getExtLoad(ISD::EXTLOAD, Node->getValueType(0), Ch, StackPtr, + DAG.getSrcValue(NULL), + Node->getOperand(0).getValueType()); + break; + } + } + break; case ISD::CALLSEQ_START: { SDNode *CallEnd = FindCallEndFromCallStart(Node); @@ -3005,10 +2950,7 @@ void SelectionDAGLegalize::LegalizeSetCCOperands(SDOperand &LHS, SDOperand SelectionDAGLegalize::ExpandBIT_CONVERT(MVT::ValueType DestVT, SDOperand SrcOp) { // Create the stack frame object. - MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo(); - unsigned ByteSize = MVT::getSizeInBits(DestVT)/8; - int FrameIdx = FrameInfo->CreateStackObject(ByteSize, ByteSize); - SDOperand FIPtr = DAG.getFrameIndex(FrameIdx, TLI.getPointerTy()); + SDOperand FIPtr = CreateStackTemporary(DestVT); // Emit a store to the stack slot. SDOperand Store = DAG.getNode(ISD::STORE, MVT::Other, DAG.getEntryNode(), @@ -3017,6 +2959,117 @@ SDOperand SelectionDAGLegalize::ExpandBIT_CONVERT(MVT::ValueType DestVT, return DAG.getLoad(DestVT, Store, FIPtr, DAG.getSrcValue(0)); } +/// ExpandBUILD_VECTOR - Expand a BUILD_VECTOR node on targets that don't +/// support the operation, but do support the resultant packed vector type. +SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { + + // If the only non-undef value is the low element, turn this into a + // SCALAR_TO_VECTOR node. + bool isOnlyLowElement = true; + for (SDNode::op_iterator I = Node->op_begin()+1, E = Node->op_end(); + I != E; ++I) { + if (I->getOpcode() != ISD::UNDEF) { + isOnlyLowElement = false; + break; + } + } + + if (isOnlyLowElement) { + // If the low element is an undef too, then this whole things is an undef. + if (Node->getOperand(0).getOpcode() == ISD::UNDEF) + return DAG.getNode(ISD::UNDEF, Node->getValueType(0)); + // Otherwise, turn this into a scalar_to_vector node. + return DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0), + Node->getOperand(0)); + } + + // If the elements are all constants, turn this into a load from the constant + // pool. + bool isConstant = true; + for (SDNode::op_iterator I = Node->op_begin(), E = Node->op_end(); + I != E; ++I) { + if (!isa(I) && !isa(I) && + I->getOpcode() != ISD::UNDEF) { + isConstant = false; + break; + } + } + + // Create a ConstantPacked, and put it in the constant pool. + if (isConstant) { + MVT::ValueType VT = Node->getValueType(0); + const Type *OpNTy = + MVT::getTypeForValueType(Node->getOperand(0).getValueType()); + std::vector CV; + for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { + if (ConstantFPSDNode *V = + dyn_cast(Node->getOperand(i))) { + CV.push_back(ConstantFP::get(OpNTy, V->getValue())); + } else if (ConstantSDNode *V = + dyn_cast(Node->getOperand(i))) { + CV.push_back(ConstantUInt::get(OpNTy, V->getValue())); + } else { + assert(Node->getOperand(i).getOpcode() == ISD::UNDEF); + CV.push_back(UndefValue::get(OpNTy)); + } + } + Constant *CP = ConstantPacked::get(CV); + SDOperand CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy()); + return DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, + DAG.getSrcValue(NULL)); + } + + // Otherwise, we can't handle this case efficiently. Allocate a sufficiently + // aligned object on the stack, store each element into it, then load + // the result as a vector. + MVT::ValueType VT = Node->getValueType(0); + // Create the stack frame object. + SDOperand FIPtr = CreateStackTemporary(VT); + + // Emit a store of each element to the stack slot. + std::vector Stores; + bool isLittleEndian = TLI.isLittleEndian(); + unsigned TypeByteSize = + MVT::getSizeInBits(Node->getOperand(0).getValueType())/8; + unsigned VectorSize = MVT::getSizeInBits(VT)/8; + // Store (in the right endianness) the elements to memory. + for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { + // Ignore undef elements. + if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue; + + unsigned Offset; + if (isLittleEndian) + Offset = TypeByteSize*i; + else + Offset = TypeByteSize*(e-i-1); + + SDOperand Idx = DAG.getConstant(Offset, FIPtr.getValueType()); + Idx = DAG.getNode(ISD::ADD, FIPtr.getValueType(), FIPtr, Idx); + + Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, DAG.getEntryNode(), + Node->getOperand(i), Idx, + DAG.getSrcValue(NULL))); + } + + SDOperand StoreChain; + if (!Stores.empty()) // Not all undef elements? + StoreChain = DAG.getNode(ISD::TokenFactor, MVT::Other, Stores); + else + StoreChain = DAG.getEntryNode(); + + // Result is a load from the stack slot. + return DAG.getLoad(VT, StoreChain, FIPtr, DAG.getSrcValue(0)); +} + +/// CreateStackTemporary - Create a stack temporary, suitable for holding the +/// specified value type. +SDOperand SelectionDAGLegalize::CreateStackTemporary(MVT::ValueType VT) { + MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo(); + unsigned ByteSize = MVT::getSizeInBits(VT)/8; + int FrameIdx = FrameInfo->CreateStackObject(ByteSize, ByteSize); + return DAG.getFrameIndex(FrameIdx, TLI.getPointerTy()); +} + void SelectionDAGLegalize::ExpandShiftParts(unsigned NodeOp, SDOperand Op, SDOperand Amt, SDOperand &Lo, SDOperand &Hi) { diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 985d6bd8ba0..57c423e6f99 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -1092,6 +1092,11 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, MVT::ValueType VT, if (OpOpcode == ISD::BIT_CONVERT) // bitconv(bitconv(x)) -> bitconv(x) return getNode(ISD::BIT_CONVERT, VT, Operand.getOperand(0)); break; + case ISD::SCALAR_TO_VECTOR: + assert(MVT::isVector(VT) && !MVT::isVector(Operand.getValueType()) && + MVT::getVectorBaseType(VT) == Operand.getValueType() && + "Illegal SCALAR_TO_VECTOR node!"); + break; case ISD::FNEG: if (OpOpcode == ISD::FSUB) // -(X-Y) -> (Y-X) return getNode(ISD::FSUB, VT, Operand.Val->getOperand(1), @@ -1555,10 +1560,15 @@ SDOperand SelectionDAG::getNode(unsigned Opcode, // normal load. if (ResultTys[0] == EVT) return getLoad(ResultTys[0], Ops[0], Ops[1], Ops[2]); - assert(EVT < ResultTys[0] && - "Should only be an extending load, not truncating!"); + if (MVT::isVector(ResultTys[0])) { + assert(EVT == MVT::getVectorBaseType(ResultTys[0]) && + "Invalid vector extload!"); + } else { + assert(EVT < ResultTys[0] && + "Should only be an extending load, not truncating!"); + } assert((Opcode == ISD::EXTLOAD || MVT::isInteger(ResultTys[0])) && - "Cannot sign/zero extend a FP load!"); + "Cannot sign/zero extend a FP/Vector load!"); assert(MVT::isInteger(ResultTys[0]) == MVT::isInteger(EVT) && "Cannot convert from FP to Int or Int -> FP!"); break; @@ -2654,6 +2664,7 @@ const char *SDNode::getOperationName(const SelectionDAG *G) const { case ISD::SELECT_CC: return "select_cc"; case ISD::INSERT_VECTOR_ELT: return "insert_vector_elt"; case ISD::VINSERT_VECTOR_ELT: return "vinsert_vector_elt"; + case ISD::SCALAR_TO_VECTOR: return "scalar_to_vector"; case ISD::ADDC: return "addc"; case ISD::ADDE: return "adde"; case ISD::SUBC: return "subc";