diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index c80b125228d..44bf18bef92 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -69,11 +69,6 @@ class VISIBILITY_HIDDEN SelectionDAGLegalize { /// being legalized (which could lead to non-serialized call sequences). bool IsLegalizingCall; - /// IsLegalizingCallArguments - This member is used only for the purpose - /// of providing assert to check for LegalizeTypes because legalizing an - /// operation might introduce call nodes that might need type legalization. - bool IsLegalizingCallArgs; - enum LegalizeAction { Legal, // The target natively supports this operation. Promote, // This operation should be executed in a larger type. @@ -90,51 +85,12 @@ class VISIBILITY_HIDDEN SelectionDAGLegalize { /// allows us to avoid legalizing the same thing more than once. DenseMap LegalizedNodes; - /// PromotedNodes - For nodes that are below legal width, and that have more - /// than one use, this map indicates what promoted value to use. This allows - /// us to avoid promoting the same thing more than once. - DenseMap PromotedNodes; - - /// ExpandedNodes - For nodes that need to be expanded this map indicates - /// which operands are the expanded version of the input. This allows - /// us to avoid expanding the same node more than once. - DenseMap > ExpandedNodes; - - /// SplitNodes - For vector nodes that need to be split, this map indicates - /// which operands are the split version of the input. This allows us - /// to avoid splitting the same node more than once. - std::map > SplitNodes; - - /// ScalarizedNodes - For nodes that need to be converted from vector types to - /// scalar types, this contains the mapping of ones we have already - /// processed to the result. - std::map ScalarizedNodes; - - /// WidenNodes - For nodes that need to be widened from one vector type to - /// another, this contains the mapping of those that we have already widen. - /// This allows us to avoid widening more than once. - std::map WidenNodes; - void AddLegalizedOperand(SDValue From, SDValue To) { LegalizedNodes.insert(std::make_pair(From, To)); // If someone requests legalization of the new node, return itself. if (From != To) LegalizedNodes.insert(std::make_pair(To, To)); } - void AddPromotedOperand(SDValue From, SDValue To) { - bool isNew = PromotedNodes.insert(std::make_pair(From, To)).second; - assert(isNew && "Got into the map somehow?"); - isNew = isNew; - // If someone requests legalization of the new node, return itself. - LegalizedNodes.insert(std::make_pair(To, To)); - } - void AddWidenedOperand(SDValue From, SDValue To) { - bool isNew = WidenNodes.insert(std::make_pair(From, To)).second; - assert(isNew && "Got into the map somehow?"); - isNew = isNew; - // If someone requests legalization of the new node, return itself. - LegalizedNodes.insert(std::make_pair(To, To)); - } public: SelectionDAGLegalize(SelectionDAG &DAG, CodeGenOpt::Level ol); @@ -164,12 +120,6 @@ private: /// result. SDValue LegalizeOp(SDValue O); - /// UnrollVectorOp - We know that the given vector has a legal type, however - /// the operation it performs is not legal and is an operation that we have - /// no way of lowering. "Unroll" the vector, splitting out the scalars and - /// operating on each element individually. - SDValue UnrollVectorOp(SDValue O); - /// PerformInsertVectorEltInMemory - Some target cannot handle a variable /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it /// is necessary to spill the vector being inserted into to memory, perform @@ -177,94 +127,9 @@ private: SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, DebugLoc dl); - /// PromoteOp - Given an operation that produces a value in an invalid type, - /// promote it to compute the value into a larger type. The produced value - /// will have the correct bits for the low portion of the register, but no - /// guarantee is made about the top bits: it may be zero, sign-extended, or - /// garbage. - SDValue PromoteOp(SDValue O); - - /// ExpandOp - Expand the specified SDValue into its two component pieces - /// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this, - /// the LegalizedNodes map is filled in for any results that are not expanded, - /// the ExpandedNodes map is filled in for any results that are expanded, and - /// the Lo/Hi values are returned. This applies to integer types and Vector - /// types. - void ExpandOp(SDValue O, SDValue &Lo, SDValue &Hi); - - /// WidenVectorOp - Widen a vector operation to a wider type given by WidenVT - /// (e.g., v3i32 to v4i32). The produced value will have the correct value - /// for the existing elements but no guarantee is made about the new elements - /// at the end of the vector: it may be zero, ones, or garbage. This is useful - /// when we have an instruction operating on an illegal vector type and we - /// want to widen it to do the computation on a legal wider vector type. - SDValue WidenVectorOp(SDValue Op, MVT WidenVT); - - /// SplitVectorOp - Given an operand of vector type, break it down into - /// two smaller values. - void SplitVectorOp(SDValue O, SDValue &Lo, SDValue &Hi); - - /// ScalarizeVectorOp - Given an operand of single-element vector type - /// (e.g. v1f32), convert it into the equivalent operation that returns a - /// scalar (e.g. f32) value. - SDValue ScalarizeVectorOp(SDValue O); - /// Useful 16 element vector type that is used to pass operands for widening. typedef SmallVector SDValueVector; - /// LoadWidenVectorOp - Load a vector for a wider type. Returns true if - /// the LdChain contains a single load and false if it contains a token - /// factor for multiple loads. It takes - /// Result: location to return the result - /// LdChain: location to return the load chain - /// Op: load operation to widen - /// NVT: widen vector result type we want for the load - bool LoadWidenVectorOp(SDValue& Result, SDValue& LdChain, - SDValue Op, MVT NVT); - - /// Helper genWidenVectorLoads - Helper function to generate a set of - /// loads to load a vector with a resulting wider type. It takes - /// LdChain: list of chains for the load we have generated - /// Chain: incoming chain for the ld vector - /// BasePtr: base pointer to load from - /// SV: memory disambiguation source value - /// SVOffset: memory disambiugation offset - /// Alignment: alignment of the memory - /// isVolatile: volatile load - /// LdWidth: width of memory that we want to load - /// ResType: the wider result result type for the resulting loaded vector - SDValue genWidenVectorLoads(SDValueVector& LdChain, SDValue Chain, - SDValue BasePtr, const Value *SV, - int SVOffset, unsigned Alignment, - bool isVolatile, unsigned LdWidth, - MVT ResType, DebugLoc dl); - - /// StoreWidenVectorOp - Stores a widen vector into non widen memory - /// location. It takes - /// ST: store node that we want to replace - /// Chain: incoming store chain - /// BasePtr: base address of where we want to store into - SDValue StoreWidenVectorOp(StoreSDNode *ST, SDValue Chain, - SDValue BasePtr); - - /// Helper genWidenVectorStores - Helper function to generate a set of - /// stores to store a widen vector into non widen memory - // It takes - // StChain: list of chains for the stores we have generated - // Chain: incoming chain for the ld vector - // BasePtr: base pointer to load from - // SV: memory disambiguation source value - // SVOffset: memory disambiugation offset - // Alignment: alignment of the memory - // isVolatile: volatile lod - // ValOp: value to store - // StWidth: width of memory that we want to store - void genWidenVectorStores(SDValueVector& StChain, SDValue Chain, - SDValue BasePtr, const Value *SV, - int SVOffset, unsigned Alignment, - bool isVolatile, SDValue ValOp, - unsigned StWidth, DebugLoc dl); - /// ShuffleWithNarrowerEltType - Return a vector shuffle operation which /// performs the same shuffe in terms of order or result bytes, but on a type /// whose vector element type is narrower than the original shuffle type. @@ -288,7 +153,6 @@ private: SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned, SDValue &Hi); - SDValue ExpandIntToFP(bool isSigned, MVT DestTy, SDValue Source, DebugLoc dl); SDValue EmitStackConvert(SDValue SrcOp, MVT SlotVT, MVT DestVT, DebugLoc dl); SDValue ExpandBUILD_VECTOR(SDNode *Node); @@ -304,12 +168,7 @@ private: SDValue ExpandBSWAP(SDValue Op, DebugLoc dl); SDValue ExpandBitCount(unsigned Opc, SDValue Op, DebugLoc dl); - bool ExpandShift(unsigned Opc, SDValue Op, SDValue Amt, - SDValue &Lo, SDValue &Hi, DebugLoc dl); - void ExpandShiftParts(unsigned NodeOp, SDValue Op, SDValue Amt, - SDValue &Lo, SDValue &Hi, DebugLoc dl); - SDValue ExpandEXTRACT_SUBVECTOR(SDValue Op); SDValue ExpandEXTRACT_VECTOR_ELT(SDValue Op); SDValue ExpandExtractFromVectorThroughStack(SDValue Op); }; @@ -359,7 +218,6 @@ SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag, void SelectionDAGLegalize::LegalizeDAG() { LastCALLSEQ_END = DAG.getEntryNode(); IsLegalizingCall = false; - IsLegalizingCallArgs = false; // The legalize process is inherently a bottom-up recursive process (users // legalize their uses before themselves). Given infinite stack space, we @@ -377,12 +235,7 @@ void SelectionDAGLegalize::LegalizeDAG() { assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?"); DAG.setRoot(LegalizedNodes[OldRoot]); - ExpandedNodes.clear(); LegalizedNodes.clear(); - PromotedNodes.clear(); - SplitNodes.clear(); - ScalarizedNodes.clear(); - WidenNodes.clear(); // Remove dead nodes now. DAG.RemoveDeadNodes(); @@ -457,17 +310,7 @@ bool SelectionDAGLegalize::LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest, // If the first result of this node has been already legalized, then it cannot // reach N. - switch (getTypeAction(N->getValueType(0))) { - case Legal: - if (LegalizedNodes.count(SDValue(N, 0))) return false; - break; - case Promote: - if (PromotedNodes.count(SDValue(N, 0))) return false; - break; - case Expand: - if (ExpandedNodes.count(SDValue(N, 0))) return false; - break; - } + if (LegalizedNodes.count(SDValue(N, 0))) return false; // Okay, this node has not already been legalized. Check and legalize all // operands. If none lead to Dest, then we can legalize this node. @@ -495,43 +338,7 @@ void SelectionDAGLegalize::HandleOp(SDValue Op) { MVT VT = Op.getValueType(); // We should never see any illegal result types here. assert(isTypeLegal(VT) && "Illegal type introduced after type legalization?"); - switch (getTypeAction(VT)) { - default: assert(0 && "Bad type action!"); - case Legal: (void)LegalizeOp(Op); break; - case Promote: - if (!VT.isVector()) { - (void)PromoteOp(Op); - break; - } - else { - // See if we can widen otherwise use Expand to either scalarize or split - MVT WidenVT = TLI.getWidenVectorType(VT); - if (WidenVT != MVT::Other) { - (void) WidenVectorOp(Op, WidenVT); - break; - } - // else fall thru to expand since we can't widen the vector - } - case Expand: - if (!VT.isVector()) { - // If this is an illegal scalar, expand it into its two component - // pieces. - SDValue X, Y; - if (Op.getOpcode() == ISD::TargetConstant) - break; // Allow illegal target nodes. - ExpandOp(Op, X, Y); - } else if (VT.getVectorNumElements() == 1) { - // If this is an illegal single element vector, convert it to a - // scalar operation. - (void)ScalarizeVectorOp(Op); - } else { - // This is an illegal multiple element vector. - // Split it in half and legalize both parts. - SDValue X, Y; - SplitVectorOp(Op, X, Y); - } - break; - } + (void)LegalizeOp(Op); } /// ExpandConstantFP - Expands the ConstantFP node to an integer constant or @@ -582,53 +389,6 @@ static SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP, PseudoSourceValue::getConstantPool(), 0, false, Alignment); } - -/// ExpandFCOPYSIGNToBitwiseOps - Expands fcopysign to a series of bitwise -/// operations. -static -SDValue ExpandFCOPYSIGNToBitwiseOps(SDNode *Node, MVT NVT, - SelectionDAG &DAG, - const TargetLowering &TLI) { - DebugLoc dl = Node->getDebugLoc(); - MVT VT = Node->getValueType(0); - MVT SrcVT = Node->getOperand(1).getValueType(); - assert((SrcVT == MVT::f32 || SrcVT == MVT::f64) && - "fcopysign expansion only supported for f32 and f64"); - MVT SrcNVT = (SrcVT == MVT::f64) ? MVT::i64 : MVT::i32; - - // First get the sign bit of second operand. - SDValue Mask1 = (SrcVT == MVT::f64) - ? DAG.getConstantFP(BitsToDouble(1ULL << 63), SrcVT) - : DAG.getConstantFP(BitsToFloat(1U << 31), SrcVT); - Mask1 = DAG.getNode(ISD::BIT_CONVERT, dl, SrcNVT, Mask1); - SDValue SignBit= DAG.getNode(ISD::BIT_CONVERT, dl, SrcNVT, - Node->getOperand(1)); - SignBit = DAG.getNode(ISD::AND, dl, SrcNVT, SignBit, Mask1); - // Shift right or sign-extend it if the two operands have different types. - int SizeDiff = SrcNVT.getSizeInBits() - NVT.getSizeInBits(); - if (SizeDiff > 0) { - SignBit = DAG.getNode(ISD::SRL, dl, SrcNVT, SignBit, - DAG.getConstant(SizeDiff, TLI.getShiftAmountTy())); - SignBit = DAG.getNode(ISD::TRUNCATE, dl, NVT, SignBit); - } else if (SizeDiff < 0) { - SignBit = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, SignBit); - SignBit = DAG.getNode(ISD::SHL, dl, NVT, SignBit, - DAG.getConstant(-SizeDiff, TLI.getShiftAmountTy())); - } - - // Clear the sign bit of first operand. - SDValue Mask2 = (VT == MVT::f64) - ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT) - : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT); - Mask2 = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Mask2); - SDValue Result = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Node->getOperand(0)); - Result = DAG.getNode(ISD::AND, dl, NVT, Result, Mask2); - - // Or the value with the sign bit. - Result = DAG.getNode(ISD::OR, dl, NVT, Result, SignBit); - return Result; -} - /// ExpandUnalignedStore - Expands an unaligned store to 2 half-size stores. static SDValue ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, @@ -864,58 +624,6 @@ SDValue ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, return DAG.getMergeValues(Ops, 2, dl); } -/// UnrollVectorOp - We know that the given vector has a legal type, however -/// the operation it performs is not legal and is an operation that we have -/// no way of lowering. "Unroll" the vector, splitting out the scalars and -/// operating on each element individually. -SDValue SelectionDAGLegalize::UnrollVectorOp(SDValue Op) { - MVT VT = Op.getValueType(); - assert(isTypeLegal(VT) && - "Caller should expand or promote operands that are not legal!"); - assert(Op.getNode()->getNumValues() == 1 && - "Can't unroll a vector with multiple results!"); - unsigned NE = VT.getVectorNumElements(); - MVT EltVT = VT.getVectorElementType(); - DebugLoc dl = Op.getDebugLoc(); - - SmallVector Scalars; - SmallVector Operands(Op.getNumOperands()); - for (unsigned i = 0; i != NE; ++i) { - for (unsigned j = 0; j != Op.getNumOperands(); ++j) { - SDValue Operand = Op.getOperand(j); - MVT OperandVT = Operand.getValueType(); - if (OperandVT.isVector()) { - // A vector operand; extract a single element. - MVT OperandEltVT = OperandVT.getVectorElementType(); - Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, - OperandEltVT, - Operand, - DAG.getConstant(i, MVT::i32)); - } else { - // A scalar operand; just use it as is. - Operands[j] = Operand; - } - } - - switch (Op.getOpcode()) { - default: - Scalars.push_back(DAG.getNode(Op.getOpcode(), dl, EltVT, - &Operands[0], Operands.size())); - break; - case ISD::SHL: - case ISD::SRA: - case ISD::SRL: - case ISD::ROTL: - case ISD::ROTR: - Scalars.push_back(DAG.getNode(Op.getOpcode(), dl, EltVT, Operands[0], - DAG.getShiftAmountOperand(Operands[1]))); - break; - } - } - - return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Scalars[0], Scalars.size()); -} - /// GetFPLibCall - Return the right libcall for the given floating point type. static RTLIB::Libcall GetFPLibCall(MVT VT, RTLIB::Libcall Call_F32, @@ -1291,20 +999,13 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { if (Result.getNode()) break; case TargetLowering::Legal: { - LegalizeAction Action = getTypeAction(Node->getOperand(1).getValueType()); - if (Action == Legal && Tmp1 == Node->getOperand(0)) + if (Tmp1 == Node->getOperand(0)) break; SmallVector Ops; Ops.push_back(Tmp1); - if (Action == Legal) { - Ops.push_back(Node->getOperand(1)); // line # must be legal. - Ops.push_back(Node->getOperand(2)); // col # must be legal. - } else { - // Otherwise promote them. - Ops.push_back(PromoteOp(Node->getOperand(1))); - Ops.push_back(PromoteOp(Node->getOperand(2))); - } + Ops.push_back(Node->getOperand(1)); // line # must be legal. + Ops.push_back(Node->getOperand(2)); // col # must be legal. Ops.push_back(Node->getOperand(3)); // filename must be legal. Ops.push_back(Node->getOperand(4)); // working dir # must be legal. Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size()); @@ -1784,13 +1485,11 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { // Recursively Legalize all of the inputs of the call end that do not lead // to this call start. This ensures that any libcalls that need be inserted // are inserted *before* the CALLSEQ_START. - IsLegalizingCallArgs = true; {SmallPtrSet NodesLeadingTo; for (unsigned i = 0, e = CallEnd->getNumOperands(); i != e; ++i) LegalizeAllNodesNotLeadingTo(CallEnd->getOperand(i).getNode(), Node, NodesLeadingTo); } - IsLegalizingCallArgs = false; // Now that we legalized all of the inputs (which may have inserted // libcalls) create the new CALLSEQ_START node. @@ -3086,10 +2785,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { break; } - assert(Node->getValueType(0).isVector() && - "Cannot expand this binary operator!"); - // Expand the operation into a bunch of nasty scalar code. - Result = LegalizeOp(UnrollVectorOp(Op)); + assert(0 && "Cannot expand this binary operator!"); break; } case TargetLowering::Promote: { @@ -3336,10 +3032,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { break; } - assert(VT.isVector() && - "Cannot expand this binary operator!"); - // Expand the operation into a bunch of nasty scalar code. - Result = LegalizeOp(UnrollVectorOp(Op)); + assert(0 && "Cannot expand this binary operator!"); break; } } @@ -3620,11 +3313,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { case ISD::FNEARBYINT: { MVT VT = Node->getValueType(0); - // Expand unsupported unary vector operators by unrolling them. - if (VT.isVector()) { - Result = LegalizeOp(UnrollVectorOp(Op)); - break; - } + assert(!VT.isVector() && "Vector shouldn't get here!"); RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; switch(Node->getOpcode()) { @@ -3695,10 +3384,7 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { MVT VT = Node->getValueType(0); // Expand unsupported unary vector operators by unrolling them. - if (VT.isVector()) { - Result = LegalizeOp(UnrollVectorOp(Op)); - break; - } + assert(!VT.isVector() && "Vector shouldn't get here!"); // We always lower FPOWI into a libcall. No target support for it yet. RTLIB::Libcall LC = GetFPLibCall(VT, RTLIB::POWI_F32, RTLIB::POWI_F64, @@ -3774,11 +3460,6 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { if (Tmp1.getNode()) Result = Tmp1; } break; - case TargetLowering::Expand: - assert(Result.getValueType().isVector() && "must be vector type"); - // Unroll the truncate. We should do better. - Result = LegalizeOp(UnrollVectorOp(Result)); - break; } break; @@ -4093,497 +3774,6 @@ SDValue SelectionDAGLegalize::LegalizeOp(SDValue Op) { return Result; } -/// PromoteOp - Given an operation that produces a value in an invalid type, -/// promote it to compute the value into a larger type. The produced value will -/// have the correct bits for the low portion of the register, but no guarantee -/// is made about the top bits: it may be zero, sign-extended, or garbage. -SDValue SelectionDAGLegalize::PromoteOp(SDValue Op) { - assert(0 && "This should be dead!"); - MVT VT = Op.getValueType(); - MVT NVT = TLI.getTypeToTransformTo(VT); - assert(getTypeAction(VT) == Promote && - "Caller should expand or legalize operands that are not promotable!"); - assert(NVT.bitsGT(VT) && NVT.isInteger() == VT.isInteger() && - "Cannot promote to smaller type!"); - - SDValue Tmp1, Tmp2, Tmp3; - SDValue Result; - SDNode *Node = Op.getNode(); - DebugLoc dl = Node->getDebugLoc(); - - DenseMap::iterator I = PromotedNodes.find(Op); - if (I != PromotedNodes.end()) return I->second; - - switch (Node->getOpcode()) { - case ISD::CopyFromReg: - assert(0 && "CopyFromReg must be legal!"); - default: -#ifndef NDEBUG - cerr << "NODE: "; Node->dump(&DAG); cerr << "\n"; -#endif - assert(0 && "Do not know how to promote this operator!"); - abort(); - case ISD::UNDEF: - Result = DAG.getUNDEF(NVT); - break; - case ISD::Constant: - if (VT != MVT::i1) - Result = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, Op); - else - Result = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Op); - assert(isa(Result) && "Didn't constant fold zext?"); - break; - case ISD::ConstantFP: - Result = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Op); - assert(isa(Result) && "Didn't constant fold fp_extend?"); - break; - - case ISD::SETCC: { - MVT VT0 = Node->getOperand(0).getValueType(); - assert(isTypeLegal(TLI.getSetCCResultType(VT0)) - && "SetCC type is not legal??"); - Result = DAG.getNode(ISD::SETCC, dl, TLI.getSetCCResultType(VT0), - Node->getOperand(0), Node->getOperand(1), - Node->getOperand(2)); - break; - } - case ISD::TRUNCATE: - switch (getTypeAction(Node->getOperand(0).getValueType())) { - case Legal: - Result = LegalizeOp(Node->getOperand(0)); - assert(Result.getValueType().bitsGE(NVT) && - "This truncation doesn't make sense!"); - if (Result.getValueType().bitsGT(NVT)) // Truncate to NVT instead of VT - Result = DAG.getNode(ISD::TRUNCATE, dl, NVT, Result); - break; - case Promote: - // The truncation is not required, because we don't guarantee anything - // about high bits anyway. - Result = PromoteOp(Node->getOperand(0)); - break; - case Expand: - ExpandOp(Node->getOperand(0), Tmp1, Tmp2); - // Truncate the low part of the expanded value to the result type - Result = DAG.getNode(ISD::TRUNCATE, dl, NVT, Tmp1); - } - break; - case ISD::SIGN_EXTEND: - case ISD::ZERO_EXTEND: - case ISD::ANY_EXTEND: - switch (getTypeAction(Node->getOperand(0).getValueType())) { - case Expand: assert(0 && "BUG: Smaller reg should have been promoted!"); - case Legal: - // Input is legal? Just do extend all the way to the larger type. - Result = DAG.getNode(Node->getOpcode(), dl, NVT, Node->getOperand(0)); - break; - case Promote: - // Promote the reg if it's smaller. - Result = PromoteOp(Node->getOperand(0)); - // The high bits are not guaranteed to be anything. Insert an extend. - if (Node->getOpcode() == ISD::SIGN_EXTEND) - Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Result, - DAG.getValueType(Node->getOperand(0).getValueType())); - else if (Node->getOpcode() == ISD::ZERO_EXTEND) - Result = DAG.getZeroExtendInReg(Result, dl, - Node->getOperand(0).getValueType()); - break; - } - break; - case ISD::CONVERT_RNDSAT: { - ISD::CvtCode CvtCode = cast(Node)->getCvtCode(); - assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU || - CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU || - CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) && - "can only promote integers"); - Result = DAG.getConvertRndSat(NVT, dl, Node->getOperand(0), - Node->getOperand(1), Node->getOperand(2), - Node->getOperand(3), Node->getOperand(4), - CvtCode); - break; - - } - case ISD::BIT_CONVERT: - Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), - Node->getValueType(0), dl); - Result = PromoteOp(Result); - break; - - case ISD::FP_EXTEND: - assert(0 && "Case not implemented. Dynamically dead with 2 FP types!"); - case ISD::FP_ROUND: - switch (getTypeAction(Node->getOperand(0).getValueType())) { - case Expand: assert(0 && "BUG: Cannot expand FP regs!"); - case Promote: assert(0 && "Unreachable with 2 FP types!"); - case Legal: - if (Node->getConstantOperandVal(1) == 0) { - // Input is legal? Do an FP_ROUND_INREG. - Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Node->getOperand(0), - DAG.getValueType(VT)); - } else { - // Just remove the truncate, it isn't affecting the value. - Result = DAG.getNode(ISD::FP_ROUND, dl, NVT, Node->getOperand(0), - Node->getOperand(1)); - } - break; - } - break; - case ISD::SINT_TO_FP: - case ISD::UINT_TO_FP: - switch (getTypeAction(Node->getOperand(0).getValueType())) { - case Legal: - // No extra round required here. - Result = DAG.getNode(Node->getOpcode(), dl, NVT, Node->getOperand(0)); - break; - - case Promote: - Result = PromoteOp(Node->getOperand(0)); - if (Node->getOpcode() == ISD::SINT_TO_FP) - Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Result.getValueType(), - Result, - DAG.getValueType(Node->getOperand(0).getValueType())); - else - Result = DAG.getZeroExtendInReg(Result, dl, - Node->getOperand(0).getValueType()); - // No extra round required here. - Result = DAG.getNode(Node->getOpcode(), dl, NVT, Result); - break; - case Expand: - Result = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, NVT, - Node->getOperand(0), dl); - // Round if we cannot tolerate excess precision. - if (NoExcessFPPrecision) - Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, - DAG.getValueType(VT)); - break; - } - break; - - case ISD::SIGN_EXTEND_INREG: - Result = PromoteOp(Node->getOperand(0)); - Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Result, - Node->getOperand(1)); - break; - case ISD::FP_TO_SINT: - case ISD::FP_TO_UINT: - switch (getTypeAction(Node->getOperand(0).getValueType())) { - case Legal: - case Expand: - Tmp1 = Node->getOperand(0); - break; - case Promote: - // The input result is prerounded, so we don't have to do anything - // special. - Tmp1 = PromoteOp(Node->getOperand(0)); - break; - } - // If we're promoting a UINT to a larger size, check to see if the new node - // will be legal. If it isn't, check to see if FP_TO_SINT is legal, since - // we can use that instead. This allows us to generate better code for - // FP_TO_UINT for small destination sizes on targets where FP_TO_UINT is not - // legal, such as PowerPC. - if (Node->getOpcode() == ISD::FP_TO_UINT && - !TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NVT) && - (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT) || - TLI.getOperationAction(ISD::FP_TO_SINT, NVT)==TargetLowering::Custom)){ - Result = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Tmp1); - } else { - Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); - } - break; - - case ISD::FABS: - case ISD::FNEG: - Tmp1 = PromoteOp(Node->getOperand(0)); - assert(Tmp1.getValueType() == NVT); - Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); - // NOTE: we do not have to do any extra rounding here for - // NoExcessFPPrecision, because we know the input will have the appropriate - // precision, and these operations don't modify precision at all. - break; - - case ISD::FLOG: - case ISD::FLOG2: - case ISD::FLOG10: - case ISD::FEXP: - case ISD::FEXP2: - case ISD::FSQRT: - case ISD::FSIN: - case ISD::FCOS: - case ISD::FTRUNC: - case ISD::FFLOOR: - case ISD::FCEIL: - case ISD::FRINT: - case ISD::FNEARBYINT: - Tmp1 = PromoteOp(Node->getOperand(0)); - assert(Tmp1.getValueType() == NVT); - Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); - if (NoExcessFPPrecision) - Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, - DAG.getValueType(VT)); - break; - - case ISD::FPOW: - case ISD::FPOWI: { - // Promote f32 pow(i) to f64 pow(i). Note that this could insert a libcall - // directly as well, which may be better. - Tmp1 = PromoteOp(Node->getOperand(0)); - Tmp2 = Node->getOperand(1); - if (Node->getOpcode() == ISD::FPOW) - Tmp2 = PromoteOp(Tmp2); - assert(Tmp1.getValueType() == NVT); - Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); - if (NoExcessFPPrecision) - Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, - DAG.getValueType(VT)); - break; - } - - case ISD::ATOMIC_CMP_SWAP: { - AtomicSDNode* AtomNode = cast(Node); - Tmp2 = PromoteOp(Node->getOperand(2)); - Tmp3 = PromoteOp(Node->getOperand(3)); - Result = DAG.getAtomic(Node->getOpcode(), dl, AtomNode->getMemoryVT(), - AtomNode->getChain(), - AtomNode->getBasePtr(), Tmp2, Tmp3, - AtomNode->getSrcValue(), - AtomNode->getAlignment()); - // Remember that we legalized the chain. - AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); - break; - } - case ISD::ATOMIC_LOAD_ADD: - case ISD::ATOMIC_LOAD_SUB: - case ISD::ATOMIC_LOAD_AND: - case ISD::ATOMIC_LOAD_OR: - case ISD::ATOMIC_LOAD_XOR: - case ISD::ATOMIC_LOAD_NAND: - case ISD::ATOMIC_LOAD_MIN: - case ISD::ATOMIC_LOAD_MAX: - case ISD::ATOMIC_LOAD_UMIN: - case ISD::ATOMIC_LOAD_UMAX: - case ISD::ATOMIC_SWAP: { - AtomicSDNode* AtomNode = cast(Node); - Tmp2 = PromoteOp(Node->getOperand(2)); - Result = DAG.getAtomic(Node->getOpcode(), dl, AtomNode->getMemoryVT(), - AtomNode->getChain(), - AtomNode->getBasePtr(), Tmp2, - AtomNode->getSrcValue(), - AtomNode->getAlignment()); - // Remember that we legalized the chain. - AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); - break; - } - - case ISD::AND: - case ISD::OR: - case ISD::XOR: - case ISD::ADD: - case ISD::SUB: - case ISD::MUL: - // The input may have strange things in the top bits of the registers, but - // these operations don't care. They may have weird bits going out, but - // that too is okay if they are integer operations. - Tmp1 = PromoteOp(Node->getOperand(0)); - Tmp2 = PromoteOp(Node->getOperand(1)); - assert(Tmp1.getValueType() == NVT && Tmp2.getValueType() == NVT); - Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); - break; - case ISD::FADD: - case ISD::FSUB: - case ISD::FMUL: - Tmp1 = PromoteOp(Node->getOperand(0)); - Tmp2 = PromoteOp(Node->getOperand(1)); - assert(Tmp1.getValueType() == NVT && Tmp2.getValueType() == NVT); - Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); - - // Floating point operations will give excess precision that we may not be - // able to tolerate. If we DO allow excess precision, just leave it, - // otherwise excise it. - // FIXME: Why would we need to round FP ops more than integer ones? - // Is Round(Add(Add(A,B),C)) != Round(Add(Round(Add(A,B)), C)) - if (NoExcessFPPrecision) - Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, - DAG.getValueType(VT)); - break; - - case ISD::SDIV: - case ISD::SREM: - // These operators require that their input be sign extended. - Tmp1 = PromoteOp(Node->getOperand(0)); - Tmp2 = PromoteOp(Node->getOperand(1)); - if (NVT.isInteger()) { - Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Tmp1, - DAG.getValueType(VT)); - Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Tmp2, - DAG.getValueType(VT)); - } - Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); - - // Perform FP_ROUND: this is probably overly pessimistic. - if (NVT.isFloatingPoint() && NoExcessFPPrecision) - Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, - DAG.getValueType(VT)); - break; - case ISD::FDIV: - case ISD::FREM: - case ISD::FCOPYSIGN: - // These operators require that their input be fp extended. - switch (getTypeAction(Node->getOperand(0).getValueType())) { - case Expand: assert(0 && "not implemented"); - case Legal: Tmp1 = LegalizeOp(Node->getOperand(0)); break; - case Promote: Tmp1 = PromoteOp(Node->getOperand(0)); break; - } - switch (getTypeAction(Node->getOperand(1).getValueType())) { - case Expand: assert(0 && "not implemented"); - case Legal: Tmp2 = LegalizeOp(Node->getOperand(1)); break; - case Promote: Tmp2 = PromoteOp(Node->getOperand(1)); break; - } - Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); - - // Perform FP_ROUND: this is probably overly pessimistic. - if (NoExcessFPPrecision && Node->getOpcode() != ISD::FCOPYSIGN) - Result = DAG.getNode(ISD::FP_ROUND_INREG, dl, NVT, Result, - DAG.getValueType(VT)); - break; - - case ISD::UDIV: - case ISD::UREM: - // These operators require that their input be zero extended. - Tmp1 = PromoteOp(Node->getOperand(0)); - Tmp2 = PromoteOp(Node->getOperand(1)); - assert(NVT.isInteger() && "Operators don't apply to FP!"); - Tmp1 = DAG.getZeroExtendInReg(Tmp1, dl, VT); - Tmp2 = DAG.getZeroExtendInReg(Tmp2, dl, VT); - Result = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); - break; - - case ISD::SHL: - Tmp1 = PromoteOp(Node->getOperand(0)); - Result = DAG.getNode(ISD::SHL, dl, NVT, Tmp1, Node->getOperand(1)); - break; - case ISD::SRA: - // The input value must be properly sign extended. - Tmp1 = PromoteOp(Node->getOperand(0)); - Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Tmp1, - DAG.getValueType(VT)); - Result = DAG.getNode(ISD::SRA, dl, NVT, Tmp1, Node->getOperand(1)); - break; - case ISD::SRL: - // The input value must be properly zero extended. - Tmp1 = PromoteOp(Node->getOperand(0)); - Tmp1 = DAG.getZeroExtendInReg(Tmp1, dl, VT); - Result = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, Node->getOperand(1)); - break; - - case ISD::VAARG: - Tmp1 = Node->getOperand(0); // Get the chain. - Tmp2 = Node->getOperand(1); // Get the pointer. - if (TLI.getOperationAction(ISD::VAARG, VT) == TargetLowering::Custom) { - Tmp3 = DAG.getVAArg(VT, dl, Tmp1, Tmp2, Node->getOperand(2)); - Result = TLI.LowerOperation(Tmp3, DAG); - } else { - const Value *V = cast(Node->getOperand(2))->getValue(); - SDValue VAList = DAG.getLoad(TLI.getPointerTy(), dl, Tmp1, Tmp2, V, 0); - // Increment the pointer, VAList, to the next vaarg - Tmp3 = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList, - DAG.getConstant(VT.getSizeInBits()/8, - TLI.getPointerTy())); - // Store the incremented VAList to the legalized pointer - Tmp3 = DAG.getStore(VAList.getValue(1), dl, Tmp3, Tmp2, V, 0); - // Load the actual argument out of the pointer VAList - Result = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Tmp3, VAList, NULL, 0, VT); - } - // Remember that we legalized the chain. - AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); - break; - - case ISD::LOAD: { - LoadSDNode *LD = cast(Node); - ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(Node) - ? ISD::EXTLOAD : LD->getExtensionType(); - Result = DAG.getExtLoad(ExtType, dl, NVT, - LD->getChain(), LD->getBasePtr(), - LD->getSrcValue(), LD->getSrcValueOffset(), - LD->getMemoryVT(), - LD->isVolatile(), - LD->getAlignment()); - // Remember that we legalized the chain. - AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); - break; - } - case ISD::SELECT: { - Tmp2 = PromoteOp(Node->getOperand(1)); // Legalize the op0 - Tmp3 = PromoteOp(Node->getOperand(2)); // Legalize the op1 - - MVT VT2 = Tmp2.getValueType(); - assert(VT2 == Tmp3.getValueType() - && "PromoteOp SELECT: Operands 2 and 3 ValueTypes don't match"); - // Ensure that the resulting node is at least the same size as the operands' - // value types, because we cannot assume that TLI.getSetCCValueType() is - // constant. - Result = DAG.getNode(ISD::SELECT, dl, VT2, Node->getOperand(0), Tmp2, Tmp3); - break; - } - case ISD::SELECT_CC: - Tmp2 = PromoteOp(Node->getOperand(2)); // True - Tmp3 = PromoteOp(Node->getOperand(3)); // False - Result = DAG.getNode(ISD::SELECT_CC, dl, NVT, Node->getOperand(0), - Node->getOperand(1), Tmp2, Tmp3, Node->getOperand(4)); - break; - case ISD::BSWAP: - Tmp1 = Node->getOperand(0); - Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Tmp1); - Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1); - Result = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, - DAG.getConstant(NVT.getSizeInBits() - - VT.getSizeInBits(), - TLI.getShiftAmountTy())); - break; - case ISD::CTPOP: - case ISD::CTTZ: - case ISD::CTLZ: - // Zero extend the argument - Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); - // Perform the larger operation, then subtract if needed. - Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); - switch(Node->getOpcode()) { - case ISD::CTPOP: - Result = Tmp1; - break; - case ISD::CTTZ: - // if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT) - Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(Tmp1.getValueType()), Tmp1, - DAG.getConstant(NVT.getSizeInBits(), NVT), - ISD::SETEQ); - Result = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2, - DAG.getConstant(VT.getSizeInBits(), NVT), Tmp1); - break; - case ISD::CTLZ: - //Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) - Result = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, - DAG.getConstant(NVT.getSizeInBits() - - VT.getSizeInBits(), NVT)); - break; - } - break; - case ISD::EXTRACT_SUBVECTOR: - Result = PromoteOp(ExpandEXTRACT_SUBVECTOR(Op)); - break; - case ISD::EXTRACT_VECTOR_ELT: - Result = PromoteOp(ExpandEXTRACT_VECTOR_ELT(Op)); - break; - } - - assert(Result.getNode() && "Didn't set a result!"); - - // Make sure the result is itself legal. - Result = LegalizeOp(Result); - - // Remember that we promoted this! - AddPromotedOperand(Op, Result); - return Result; -} - /// ExpandEXTRACT_VECTOR_ELT - Expand an EXTRACT_VECTOR_ELT operation into /// a legal EXTRACT_VECTOR_ELT operation, scalar code, or memory traffic, /// based on the vector type. The return type of this matches the element type @@ -4654,37 +3844,6 @@ SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, NULL, 0); } -/// ExpandEXTRACT_SUBVECTOR - Expand a EXTRACT_SUBVECTOR operation. For now -/// we assume the operation can be split if it is not already legal. -SDValue SelectionDAGLegalize::ExpandEXTRACT_SUBVECTOR(SDValue Op) { - // We know that operand #0 is the Vec vector. For now we assume the index - // is a constant and that the extracted result is a supported hardware type. - SDValue Vec = Op.getOperand(0); - SDValue Idx = LegalizeOp(Op.getOperand(1)); - - unsigned NumElems = Vec.getValueType().getVectorNumElements(); - - if (NumElems == Op.getValueType().getVectorNumElements()) { - // This must be an access of the desired vector length. Return it. - return Vec; - } - - ConstantSDNode *CIdx = cast(Idx); - SDValue Lo, Hi; - SplitVectorOp(Vec, Lo, Hi); - if (CIdx->getZExtValue() < NumElems/2) { - Vec = Lo; - } else { - Vec = Hi; - Idx = DAG.getConstant(CIdx->getZExtValue() - NumElems/2, - Idx.getValueType()); - } - - // It's now an extract from the appropriate high or low part. Recurse. - Op = DAG.UpdateNodeOperands(Op, Vec, Idx); - return ExpandEXTRACT_SUBVECTOR(Op); -} - /// LegalizeSetCCOperands - Attempts to create a legal LHS and RHS for a SETCC /// with condition CC on the current target. This usually involves legalizing /// or promoting the arguments. In the case where LHS and RHS must be expanded, @@ -4695,259 +3854,8 @@ void SelectionDAGLegalize::LegalizeSetCCOperands(SDValue &LHS, SDValue &RHS, SDValue &CC, DebugLoc dl) { - SDValue Tmp1, Tmp2, Tmp3, Result; - - switch (getTypeAction(LHS.getValueType())) { - case Legal: - Tmp1 = LegalizeOp(LHS); // LHS - Tmp2 = LegalizeOp(RHS); // RHS - break; - case Promote: - Tmp1 = PromoteOp(LHS); // LHS - Tmp2 = PromoteOp(RHS); // RHS - - // If this is an FP compare, the operands have already been extended. - if (LHS.getValueType().isInteger()) { - MVT VT = LHS.getValueType(); - MVT NVT = TLI.getTypeToTransformTo(VT); - - // Otherwise, we have to insert explicit sign or zero extends. Note - // that we could insert sign extends for ALL conditions, but zero extend - // is cheaper on many machines (an AND instead of two shifts), so prefer - // it. - switch (cast(CC)->get()) { - default: assert(0 && "Unknown integer comparison!"); - case ISD::SETEQ: - case ISD::SETNE: - case ISD::SETUGE: - case ISD::SETUGT: - case ISD::SETULE: - case ISD::SETULT: - // ALL of these operations will work if we either sign or zero extend - // the operands (including the unsigned comparisons!). Zero extend is - // usually a simpler/cheaper operation, so prefer it. - Tmp1 = DAG.getZeroExtendInReg(Tmp1, dl, VT); - Tmp2 = DAG.getZeroExtendInReg(Tmp2, dl, VT); - break; - case ISD::SETGE: - case ISD::SETGT: - case ISD::SETLT: - case ISD::SETLE: - Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Tmp1, - DAG.getValueType(VT)); - Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Tmp2, - DAG.getValueType(VT)); - Tmp1 = LegalizeOp(Tmp1); // Relegalize new nodes. - Tmp2 = LegalizeOp(Tmp2); // Relegalize new nodes. - break; - } - } - break; - case Expand: { - MVT VT = LHS.getValueType(); - if (VT == MVT::f32 || VT == MVT::f64) { - // Expand into one or more soft-fp libcall(s). - RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL; - switch (cast(CC)->get()) { - case ISD::SETEQ: - case ISD::SETOEQ: - LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64; - break; - case ISD::SETNE: - case ISD::SETUNE: - LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 : RTLIB::UNE_F64; - break; - case ISD::SETGE: - case ISD::SETOGE: - LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64; - break; - case ISD::SETLT: - case ISD::SETOLT: - LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64; - break; - case ISD::SETLE: - case ISD::SETOLE: - LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64; - break; - case ISD::SETGT: - case ISD::SETOGT: - LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64; - break; - case ISD::SETUO: - LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64; - break; - case ISD::SETO: - LC1 = (VT == MVT::f32) ? RTLIB::O_F32 : RTLIB::O_F64; - break; - default: - LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64; - switch (cast(CC)->get()) { - case ISD::SETONE: - // SETONE = SETOLT | SETOGT - LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64; - // Fallthrough - case ISD::SETUGT: - LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64; - break; - case ISD::SETUGE: - LC2 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64; - break; - case ISD::SETULT: - LC2 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64; - break; - case ISD::SETULE: - LC2 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64; - break; - case ISD::SETUEQ: - LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64; - break; - default: assert(0 && "Unsupported FP setcc!"); - } - } - - SDValue Dummy; - SDValue Ops[2] = { LHS, RHS }; - Tmp1 = ExpandLibCall(LC1, DAG.getMergeValues(Ops, 2, dl).getNode(), - false /*sign irrelevant*/, Dummy); - Tmp2 = DAG.getConstant(0, MVT::i32); - CC = DAG.getCondCode(TLI.getCmpLibcallCC(LC1)); - if (LC2 != RTLIB::UNKNOWN_LIBCALL) { - Tmp1 = DAG.getNode(ISD::SETCC, dl, - TLI.getSetCCResultType(Tmp1.getValueType()), - Tmp1, Tmp2, CC); - LHS = ExpandLibCall(LC2, DAG.getMergeValues(Ops, 2, dl).getNode(), - false /*sign irrelevant*/, Dummy); - Tmp2 = DAG.getNode(ISD::SETCC, dl, - TLI.getSetCCResultType(LHS.getValueType()), LHS, - Tmp2, DAG.getCondCode(TLI.getCmpLibcallCC(LC2))); - Tmp1 = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp2); - Tmp2 = SDValue(); - } - LHS = LegalizeOp(Tmp1); - RHS = Tmp2; - return; - } - - SDValue LHSLo, LHSHi, RHSLo, RHSHi; - ExpandOp(LHS, LHSLo, LHSHi); - ExpandOp(RHS, RHSLo, RHSHi); - ISD::CondCode CCCode = cast(CC)->get(); - - if (VT==MVT::ppcf128) { - // FIXME: This generated code sucks. We want to generate - // FCMPU crN, hi1, hi2 - // BNE crN, L: - // FCMPU crN, lo1, lo2 - // The following can be improved, but not that much. - Tmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), - LHSHi, RHSHi, ISD::SETOEQ); - Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSLo.getValueType()), - LHSLo, RHSLo, CCCode); - Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); - Tmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), - LHSHi, RHSHi, ISD::SETUNE); - Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), - LHSHi, RHSHi, CCCode); - Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); - Tmp1 = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3); - Tmp2 = SDValue(); - break; - } - - switch (CCCode) { - case ISD::SETEQ: - case ISD::SETNE: - if (RHSLo == RHSHi) - if (ConstantSDNode *RHSCST = dyn_cast(RHSLo)) - if (RHSCST->isAllOnesValue()) { - // Comparison to -1. - Tmp1 = DAG.getNode(ISD::AND, dl,LHSLo.getValueType(), LHSLo, LHSHi); - Tmp2 = RHSLo; - break; - } - - Tmp1 = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); - Tmp2 = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); - Tmp1 = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp2); - Tmp2 = DAG.getConstant(0, Tmp1.getValueType()); - break; - default: - // If this is a comparison of the sign bit, just look at the top part. - // X > -1, x < 0 - if (ConstantSDNode *CST = dyn_cast(RHS)) - if ((cast(CC)->get() == ISD::SETLT && - CST->isNullValue()) || // X < 0 - (cast(CC)->get() == ISD::SETGT && - CST->isAllOnesValue())) { // X > -1 - Tmp1 = LHSHi; - Tmp2 = RHSHi; - break; - } - - // FIXME: This generated code sucks. - ISD::CondCode LowCC; - switch (CCCode) { - default: assert(0 && "Unknown integer setcc!"); - case ISD::SETLT: - case ISD::SETULT: LowCC = ISD::SETULT; break; - case ISD::SETGT: - case ISD::SETUGT: LowCC = ISD::SETUGT; break; - case ISD::SETLE: - case ISD::SETULE: LowCC = ISD::SETULE; break; - case ISD::SETGE: - case ISD::SETUGE: LowCC = ISD::SETUGE; break; - } - - // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison - // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands - // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2; - - // NOTE: on targets without efficient SELECT of bools, we can always use - // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) - TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, NULL); - Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo.getValueType()), - LHSLo, RHSLo, LowCC, false, DagCombineInfo, dl); - if (!Tmp1.getNode()) - Tmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(LHSLo.getValueType()), - LHSLo, RHSLo, LowCC); - Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), - LHSHi, RHSHi, CCCode, false, DagCombineInfo, dl); - if (!Tmp2.getNode()) - Tmp2 = DAG.getNode(ISD::SETCC, dl, - TLI.getSetCCResultType(LHSHi.getValueType()), - LHSHi, RHSHi, CC); - - ConstantSDNode *Tmp1C = dyn_cast(Tmp1.getNode()); - ConstantSDNode *Tmp2C = dyn_cast(Tmp2.getNode()); - if ((Tmp1C && Tmp1C->isNullValue()) || - (Tmp2C && Tmp2C->isNullValue() && - (CCCode == ISD::SETLE || CCCode == ISD::SETGE || - CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) || - (Tmp2C && Tmp2C->getAPIntValue() == 1 && - (CCCode == ISD::SETLT || CCCode == ISD::SETGT || - CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) { - // low part is known false, returns high part. - // For LE / GE, if high part is known false, ignore the low part. - // For LT / GT, if high part is known true, ignore the low part. - Tmp1 = Tmp2; - Tmp2 = SDValue(); - } else { - Result = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi.getValueType()), - LHSHi, RHSHi, ISD::SETEQ, false, - DagCombineInfo, dl); - if (!Result.getNode()) - Result=DAG.getSetCC(dl, TLI.getSetCCResultType(LHSHi.getValueType()), - LHSHi, RHSHi, ISD::SETEQ); - Result = LegalizeOp(DAG.getNode(ISD::SELECT, dl, Tmp1.getValueType(), - Result, Tmp1, Tmp2)); - Tmp1 = Result; - Tmp2 = SDValue(); - } - } - } - } - LHS = Tmp1; - RHS = Tmp2; + LHS = LegalizeOp(LHS); + RHS = LegalizeOp(RHS); } /// LegalizeSetCCCondCode - Legalize a SETCC with given LHS and RHS and @@ -5222,177 +4130,6 @@ SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { return DAG.getLoad(VT, dl, StoreChain, FIPtr, SV, 0); } -void SelectionDAGLegalize::ExpandShiftParts(unsigned NodeOp, - SDValue Op, SDValue Amt, - SDValue &Lo, SDValue &Hi, - DebugLoc dl) { - // Expand the subcomponents. - SDValue LHSL, LHSH; - ExpandOp(Op, LHSL, LHSH); - - SDValue Ops[] = { LHSL, LHSH, Amt }; - MVT VT = LHSL.getValueType(); - Lo = DAG.getNode(NodeOp, dl, DAG.getVTList(VT, VT), Ops, 3); - Hi = Lo.getValue(1); -} - - -/// ExpandShift - Try to find a clever way to expand this shift operation out to -/// smaller elements. If we can't find a way that is more efficient than a -/// libcall on this target, return false. Otherwise, return true with the -/// low-parts expanded into Lo and Hi. -bool SelectionDAGLegalize::ExpandShift(unsigned Opc, SDValue Op, SDValue Amt, - SDValue &Lo, SDValue &Hi, - DebugLoc dl) { - assert((Opc == ISD::SHL || Opc == ISD::SRA || Opc == ISD::SRL) && - "This is not a shift!"); - - MVT NVT = TLI.getTypeToTransformTo(Op.getValueType()); - SDValue ShAmt = LegalizeOp(Amt); - MVT ShTy = ShAmt.getValueType(); - unsigned ShBits = ShTy.getSizeInBits(); - unsigned VTBits = Op.getValueType().getSizeInBits(); - unsigned NVTBits = NVT.getSizeInBits(); - - // Handle the case when Amt is an immediate. - if (ConstantSDNode *CN = dyn_cast(Amt.getNode())) { - unsigned Cst = CN->getZExtValue(); - // Expand the incoming operand to be shifted, so that we have its parts - SDValue InL, InH; - ExpandOp(Op, InL, InH); - switch(Opc) { - case ISD::SHL: - if (Cst > VTBits) { - Lo = DAG.getConstant(0, NVT); - Hi = DAG.getConstant(0, NVT); - } else if (Cst > NVTBits) { - Lo = DAG.getConstant(0, NVT); - Hi = DAG.getNode(ISD::SHL, dl, - NVT, InL, DAG.getConstant(Cst-NVTBits, ShTy)); - } else if (Cst == NVTBits) { - Lo = DAG.getConstant(0, NVT); - Hi = InL; - } else { - Lo = DAG.getNode(ISD::SHL, dl, NVT, InL, DAG.getConstant(Cst, ShTy)); - Hi = DAG.getNode(ISD::OR, dl, NVT, - DAG.getNode(ISD::SHL, dl, NVT, InH, DAG.getConstant(Cst, ShTy)), - DAG.getNode(ISD::SRL, dl, NVT, InL, - DAG.getConstant(NVTBits-Cst, ShTy))); - } - return true; - case ISD::SRL: - if (Cst > VTBits) { - Lo = DAG.getConstant(0, NVT); - Hi = DAG.getConstant(0, NVT); - } else if (Cst > NVTBits) { - Lo = DAG.getNode(ISD::SRL, dl, NVT, - InH, DAG.getConstant(Cst-NVTBits, ShTy)); - Hi = DAG.getConstant(0, NVT); - } else if (Cst == NVTBits) { - Lo = InH; - Hi = DAG.getConstant(0, NVT); - } else { - Lo = DAG.getNode(ISD::OR, dl, NVT, - DAG.getNode(ISD::SRL, dl, NVT, InL, DAG.getConstant(Cst, ShTy)), - DAG.getNode(ISD::SHL, dl, NVT, InH, - DAG.getConstant(NVTBits-Cst, ShTy))); - Hi = DAG.getNode(ISD::SRL, dl, NVT, InH, DAG.getConstant(Cst, ShTy)); - } - return true; - case ISD::SRA: - if (Cst > VTBits) { - Hi = Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, - DAG.getConstant(NVTBits-1, ShTy)); - } else if (Cst > NVTBits) { - Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, - DAG.getConstant(Cst-NVTBits, ShTy)); - Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, - DAG.getConstant(NVTBits-1, ShTy)); - } else if (Cst == NVTBits) { - Lo = InH; - Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, - DAG.getConstant(NVTBits-1, ShTy)); - } else { - Lo = DAG.getNode(ISD::OR, dl, NVT, - DAG.getNode(ISD::SRL, dl, NVT, InL, DAG.getConstant(Cst, ShTy)), - DAG.getNode(ISD::SHL, dl, - NVT, InH, DAG.getConstant(NVTBits-Cst, ShTy))); - Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, DAG.getConstant(Cst, ShTy)); - } - return true; - } - } - - // Okay, the shift amount isn't constant. However, if we can tell that it is - // >= 32 or < 32, we can still simplify it, without knowing the actual value. - APInt Mask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); - APInt KnownZero, KnownOne; - DAG.ComputeMaskedBits(Amt, Mask, KnownZero, KnownOne); - - // If we know that if any of the high bits of the shift amount are one, then - // we can do this as a couple of simple shifts. - if (KnownOne.intersects(Mask)) { - // Mask out the high bit, which we know is set. - Amt = DAG.getNode(ISD::AND, dl, Amt.getValueType(), Amt, - DAG.getConstant(~Mask, Amt.getValueType())); - - // Expand the incoming operand to be shifted, so that we have its parts - SDValue InL, InH; - ExpandOp(Op, InL, InH); - switch(Opc) { - case ISD::SHL: - Lo = DAG.getConstant(0, NVT); // Low part is zero. - Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. - return true; - case ISD::SRL: - Hi = DAG.getConstant(0, NVT); // Hi part is zero. - Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. - return true; - case ISD::SRA: - Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. - DAG.getConstant(NVTBits-1, Amt.getValueType())); - Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. - return true; - } - } - - // If we know that the high bits of the shift amount are all zero, then we can - // do this as a couple of simple shifts. - if ((KnownZero & Mask) == Mask) { - // Compute 32-amt. - SDValue Amt2 = DAG.getNode(ISD::SUB, dl, Amt.getValueType(), - DAG.getConstant(NVTBits, Amt.getValueType()), - Amt); - - // Expand the incoming operand to be shifted, so that we have its parts - SDValue InL, InH; - ExpandOp(Op, InL, InH); - switch(Opc) { - case ISD::SHL: - Lo = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); - Hi = DAG.getNode(ISD::OR, dl, NVT, - DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), - DAG.getNode(ISD::SRL, dl, NVT, InL, Amt2)); - return true; - case ISD::SRL: - Hi = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); - Lo = DAG.getNode(ISD::OR, dl, NVT, - DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), - DAG.getNode(ISD::SHL, dl, NVT, InH, Amt2)); - return true; - case ISD::SRA: - Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); - Lo = DAG.getNode(ISD::OR, dl, NVT, - DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), - DAG.getNode(ISD::SHL, dl, NVT, InH, Amt2)); - return true; - } - } - - return false; -} - - // ExpandLibCall - Expand a node into a call to a libcall. If the result value // does not fit into a register, return the lo part and set the hi part to the // by-reg argument. If it does fit into a single register, return the result @@ -5429,17 +4166,7 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that // was added by LowerCallTo (guaranteeing proper serialization of calls). LegalizeOp(CallInfo.second); - SDValue Result; - switch (getTypeAction(CallInfo.first.getValueType())) { - default: assert(0 && "Unknown thing"); - case Legal: - Result = CallInfo.first; - break; - case Expand: - ExpandOp(CallInfo.first, Result, Hi); - break; - } - return Result; + return CallInfo.first; } /// LegalizeINT_TO_FP - Legalize a [US]INT_TO_FP operation. @@ -5449,200 +4176,34 @@ LegalizeINT_TO_FP(SDValue Result, bool isSigned, MVT DestTy, SDValue Op, DebugLoc dl) { bool isCustom = false; SDValue Tmp1; - switch (getTypeAction(Op.getValueType())) { - case Legal: - switch (TLI.getOperationAction(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, - Op.getValueType())) { - default: assert(0 && "Unknown operation action!"); - case TargetLowering::Custom: - isCustom = true; - // FALLTHROUGH - case TargetLowering::Legal: - Tmp1 = LegalizeOp(Op); - if (Result.getNode()) - Result = DAG.UpdateNodeOperands(Result, Tmp1); - else - Result = DAG.getNode(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, dl, - DestTy, Tmp1); - if (isCustom) { - Tmp1 = TLI.LowerOperation(Result, DAG); - if (Tmp1.getNode()) Result = Tmp1; - } - break; - case TargetLowering::Expand: - Result = ExpandLegalINT_TO_FP(isSigned, LegalizeOp(Op), DestTy, dl); - break; - case TargetLowering::Promote: - Result = PromoteLegalINT_TO_FP(LegalizeOp(Op), DestTy, isSigned, dl); - break; - } - break; - case Expand: - Result = ExpandIntToFP(isSigned, DestTy, Op, dl) ; - break; - case Promote: - Tmp1 = PromoteOp(Op); - if (isSigned) { - Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Tmp1.getValueType(), - Tmp1, DAG.getValueType(Op.getValueType())); - } else { - Tmp1 = DAG.getZeroExtendInReg(Tmp1, dl, Op.getValueType()); - } + switch (TLI.getOperationAction(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, + Op.getValueType())) { + default: assert(0 && "Unknown operation action!"); + case TargetLowering::Custom: + isCustom = true; + // FALLTHROUGH + case TargetLowering::Legal: + Tmp1 = LegalizeOp(Op); if (Result.getNode()) Result = DAG.UpdateNodeOperands(Result, Tmp1); else Result = DAG.getNode(isSigned ? ISD::SINT_TO_FP : ISD::UINT_TO_FP, dl, DestTy, Tmp1); - Result = LegalizeOp(Result); // The 'op' is not necessarily legal! + if (isCustom) { + Tmp1 = TLI.LowerOperation(Result, DAG); + if (Tmp1.getNode()) Result = Tmp1; + } + break; + case TargetLowering::Expand: + Result = ExpandLegalINT_TO_FP(isSigned, LegalizeOp(Op), DestTy, dl); + break; + case TargetLowering::Promote: + Result = PromoteLegalINT_TO_FP(LegalizeOp(Op), DestTy, isSigned, dl); break; } return Result; } -/// ExpandIntToFP - Expand a [US]INT_TO_FP operation. -/// -SDValue SelectionDAGLegalize:: -ExpandIntToFP(bool isSigned, MVT DestTy, SDValue Source, DebugLoc dl) { - MVT SourceVT = Source.getValueType(); - bool ExpandSource = getTypeAction(SourceVT) == Expand; - - // Expand unsupported int-to-fp vector casts by unrolling them. - if (DestTy.isVector()) { - if (!ExpandSource) - return LegalizeOp(UnrollVectorOp(Source)); - MVT DestEltTy = DestTy.getVectorElementType(); - if (DestTy.getVectorNumElements() == 1) { - SDValue Scalar = ScalarizeVectorOp(Source); - SDValue Result = LegalizeINT_TO_FP(SDValue(), isSigned, - DestEltTy, Scalar, dl); - return DAG.getNode(ISD::BUILD_VECTOR, dl, DestTy, Result); - } - SDValue Lo, Hi; - SplitVectorOp(Source, Lo, Hi); - MVT SplitDestTy = MVT::getVectorVT(DestEltTy, - DestTy.getVectorNumElements() / 2); - SDValue LoResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, - Lo, dl); - SDValue HiResult = LegalizeINT_TO_FP(SDValue(), isSigned, SplitDestTy, - Hi, dl); - return LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, dl, DestTy, LoResult, - HiResult)); - } - - // Special case for i32 source to take advantage of UINTTOFP_I32_F32, etc. - if (!isSigned && SourceVT != MVT::i32) { - // The integer value loaded will be incorrectly if the 'sign bit' of the - // incoming integer is set. To handle this, we dynamically test to see if - // it is set, and, if so, add a fudge factor. - SDValue Hi; - if (ExpandSource) { - SDValue Lo; - ExpandOp(Source, Lo, Hi); - Source = DAG.getNode(ISD::BUILD_PAIR, dl, SourceVT, Lo, Hi); - } else { - // The comparison for the sign bit will use the entire operand. - Hi = Source; - } - - // Check to see if the target has a custom way to lower this. If so, use - // it. (Note we've already expanded the operand in this case.) - switch (TLI.getOperationAction(ISD::UINT_TO_FP, SourceVT)) { - default: assert(0 && "This action not implemented for this operation!"); - case TargetLowering::Legal: - case TargetLowering::Expand: - break; // This case is handled below. - case TargetLowering::Custom: { - SDValue NV = TLI.LowerOperation(DAG.getNode(ISD::UINT_TO_FP, dl, DestTy, - Source), DAG); - if (NV.getNode()) - return LegalizeOp(NV); - break; // The target decided this was legal after all - } - } - - // If this is unsigned, and not supported, first perform the conversion to - // signed, then adjust the result if the sign bit is set. - SDValue SignedConv = ExpandIntToFP(true, DestTy, Source, dl); - - SDValue SignSet = DAG.getSetCC(dl, - TLI.getSetCCResultType(Hi.getValueType()), - Hi, DAG.getConstant(0, Hi.getValueType()), - ISD::SETLT); - SDValue Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4); - SDValue CstOffset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), - SignSet, Four, Zero); - uint64_t FF = 0x5f800000ULL; - if (TLI.isLittleEndian()) FF <<= 32; - Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF); - - SDValue CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy()); - unsigned Alignment = cast(CPIdx)->getAlignment(); - CPIdx = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), CPIdx, CstOffset); - Alignment = std::min(Alignment, 4u); - SDValue FudgeInReg; - if (DestTy == MVT::f32) - FudgeInReg = DAG.getLoad(MVT::f32, dl, DAG.getEntryNode(), CPIdx, - PseudoSourceValue::getConstantPool(), 0, - false, Alignment); - else if (DestTy.bitsGT(MVT::f32)) - // FIXME: Avoid the extend by construction the right constantpool? - FudgeInReg = DAG.getExtLoad(ISD::EXTLOAD, dl, DestTy, DAG.getEntryNode(), - CPIdx, - PseudoSourceValue::getConstantPool(), 0, - MVT::f32, false, Alignment); - else - assert(0 && "Unexpected conversion"); - - MVT SCVT = SignedConv.getValueType(); - if (SCVT != DestTy) { - // Destination type needs to be expanded as well. The FADD now we are - // constructing will be expanded into a libcall. - if (SCVT.getSizeInBits() != DestTy.getSizeInBits()) { - assert(SCVT.getSizeInBits() * 2 == DestTy.getSizeInBits()); - SignedConv = DAG.getNode(ISD::BUILD_PAIR, dl, DestTy, - SignedConv, SignedConv.getValue(1)); - } - SignedConv = DAG.getNode(ISD::BIT_CONVERT, dl, DestTy, SignedConv); - } - return DAG.getNode(ISD::FADD, dl, DestTy, SignedConv, FudgeInReg); - } - - // Check to see if the target has a custom way to lower this. If so, use it. - switch (TLI.getOperationAction(ISD::SINT_TO_FP, SourceVT)) { - default: assert(0 && "This action not implemented for this operation!"); - case TargetLowering::Legal: - case TargetLowering::Expand: - break; // This case is handled below. - case TargetLowering::Custom: { - SDValue NV = TLI.LowerOperation(DAG.getNode(ISD::SINT_TO_FP, dl, DestTy, - Source), DAG); - if (NV.getNode()) - return LegalizeOp(NV); - break; // The target decided this was legal after all - } - } - - // Expand the source, then glue it back together for the call. We must expand - // the source in case it is shared (this pass of legalize must traverse it). - if (ExpandSource) { - SDValue SrcLo, SrcHi; - ExpandOp(Source, SrcLo, SrcHi); - Source = DAG.getNode(ISD::BUILD_PAIR, dl, SourceVT, SrcLo, SrcHi); - } - - RTLIB::Libcall LC = isSigned ? - RTLIB::getSINTTOFP(SourceVT, DestTy) : - RTLIB::getUINTTOFP(SourceVT, DestTy); - assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unknown int value type"); - - Source = DAG.getNode(ISD::SINT_TO_FP, dl, DestTy, Source); - SDValue HiPart; - SDValue Result = ExpandLibCall(LC, Source.getNode(), isSigned, HiPart); - if (Result.getValueType() != DestTy && HiPart.getNode()) - Result = DAG.getNode(ISD::BUILD_PAIR, dl, DestTy, Result, HiPart); - return Result; -} - /// ExpandLegalINT_TO_FP - This function is responsible for legalizing a /// INT_TO_FP operation of the specified operand when the target requests that /// we expand it. At this point, we know that the result and operand types are @@ -5990,2205 +4551,6 @@ SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, } } -/// ExpandOp - Expand the specified SDValue into its two component pieces -/// Lo&Hi. Note that the Op MUST be an expanded type. As a result of this, the -/// LegalizedNodes map is filled in for any results that are not expanded, the -/// ExpandedNodes map is filled in for any results that are expanded, and the -/// Lo/Hi values are returned. -void SelectionDAGLegalize::ExpandOp(SDValue Op, SDValue &Lo, SDValue &Hi){ - assert(0 && "This should be dead!"); - MVT VT = Op.getValueType(); - MVT NVT = TLI.getTypeToTransformTo(VT); - SDNode *Node = Op.getNode(); - DebugLoc dl = Node->getDebugLoc(); - assert(getTypeAction(VT) == Expand && "Not an expanded type!"); - assert(((NVT.isInteger() && NVT.bitsLT(VT)) || VT.isFloatingPoint() || - VT.isVector()) && "Cannot expand to FP value or to larger int value!"); - - // See if we already expanded it. - DenseMap >::iterator I - = ExpandedNodes.find(Op); - if (I != ExpandedNodes.end()) { - Lo = I->second.first; - Hi = I->second.second; - return; - } - - switch (Node->getOpcode()) { - case ISD::CopyFromReg: - assert(0 && "CopyFromReg must be legal!"); - case ISD::FP_ROUND_INREG: - if (VT == MVT::ppcf128 && - TLI.getOperationAction(ISD::FP_ROUND_INREG, VT) == - TargetLowering::Custom) { - SDValue SrcLo, SrcHi, Src; - ExpandOp(Op.getOperand(0), SrcLo, SrcHi); - Src = DAG.getNode(ISD::BUILD_PAIR, dl, VT, SrcLo, SrcHi); - SDValue Result = - TLI.LowerOperation(DAG.getNode(ISD::FP_ROUND_INREG, dl, VT, Src, - Op.getOperand(1)), DAG); - assert(Result.getNode()->getOpcode() == ISD::BUILD_PAIR); - Lo = Result.getNode()->getOperand(0); - Hi = Result.getNode()->getOperand(1); - break; - } - // fall through - default: -#ifndef NDEBUG - cerr << "NODE: "; Node->dump(&DAG); cerr << "\n"; -#endif - assert(0 && "Do not know how to expand this operator!"); - abort(); - case ISD::EXTRACT_ELEMENT: - ExpandOp(Node->getOperand(0), Lo, Hi); - if (cast(Node->getOperand(1))->getZExtValue()) - return ExpandOp(Hi, Lo, Hi); - return ExpandOp(Lo, Lo, Hi); - case ISD::EXTRACT_VECTOR_ELT: - // ExpandEXTRACT_VECTOR_ELT tolerates invalid result types. - Lo = ExpandEXTRACT_VECTOR_ELT(Op); - return ExpandOp(Lo, Lo, Hi); - case ISD::UNDEF: - Lo = DAG.getUNDEF(NVT); - Hi = DAG.getUNDEF(NVT); - break; - case ISD::Constant: { - unsigned NVTBits = NVT.getSizeInBits(); - const APInt &Cst = cast(Node)->getAPIntValue(); - Lo = DAG.getConstant(APInt(Cst).trunc(NVTBits), NVT); - Hi = DAG.getConstant(Cst.lshr(NVTBits).trunc(NVTBits), NVT); - break; - } - case ISD::ConstantFP: { - ConstantFPSDNode *CFP = cast(Node); - if (CFP->getValueType(0) == MVT::ppcf128) { - APInt api = CFP->getValueAPF().bitcastToAPInt(); - Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[1])), - MVT::f64); - Hi = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[0])), - MVT::f64); - break; - } - Lo = ExpandConstantFP(CFP, false, DAG, TLI); - if (getTypeAction(Lo.getValueType()) == Expand) - ExpandOp(Lo, Lo, Hi); - break; - } - case ISD::BUILD_PAIR: - // Return the operands. - Lo = Node->getOperand(0); - Hi = Node->getOperand(1); - break; - - case ISD::MERGE_VALUES: - if (Node->getNumValues() == 1) { - ExpandOp(Op.getOperand(0), Lo, Hi); - break; - } - // FIXME: For now only expand i64,chain = MERGE_VALUES (x, y) - assert(Op.getResNo() == 0 && Node->getNumValues() == 2 && - Op.getValue(1).getValueType() == MVT::Other && - "unhandled MERGE_VALUES"); - ExpandOp(Op.getOperand(0), Lo, Hi); - // Remember that we legalized the chain. - AddLegalizedOperand(Op.getValue(1), LegalizeOp(Op.getOperand(1))); - break; - - case ISD::SIGN_EXTEND_INREG: - ExpandOp(Node->getOperand(0), Lo, Hi); - // sext_inreg the low part if needed. - Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Lo, Node->getOperand(1)); - - // The high part gets the sign extension from the lo-part. This handles - // things like sextinreg V:i64 from i8. - Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, - DAG.getConstant(NVT.getSizeInBits()-1, - TLI.getShiftAmountTy())); - break; - - case ISD::BSWAP: { - ExpandOp(Node->getOperand(0), Lo, Hi); - SDValue TempLo = DAG.getNode(ISD::BSWAP, dl, NVT, Hi); - Hi = DAG.getNode(ISD::BSWAP, dl, NVT, Lo); - Lo = TempLo; - break; - } - - case ISD::CTPOP: - ExpandOp(Node->getOperand(0), Lo, Hi); - Lo = DAG.getNode(ISD::ADD, dl, NVT, // ctpop(HL) -> ctpop(H)+ctpop(L) - DAG.getNode(ISD::CTPOP, dl, NVT, Lo), - DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); - Hi = DAG.getConstant(0, NVT); - break; - - case ISD::CTLZ: { - // ctlz (HL) -> ctlz(H) != 32 ? ctlz(H) : (ctlz(L)+32) - ExpandOp(Node->getOperand(0), Lo, Hi); - SDValue BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT); - SDValue HLZ = DAG.getNode(ISD::CTLZ, dl, NVT, Hi); - SDValue TopNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), HLZ, - BitsC, ISD::SETNE); - SDValue LowPart = DAG.getNode(ISD::CTLZ, dl, NVT, Lo); - LowPart = DAG.getNode(ISD::ADD, dl, NVT, LowPart, BitsC); - - Lo = DAG.getNode(ISD::SELECT, dl, NVT, TopNotZero, HLZ, LowPart); - Hi = DAG.getConstant(0, NVT); - break; - } - - case ISD::CTTZ: { - // cttz (HL) -> cttz(L) != 32 ? cttz(L) : (cttz(H)+32) - ExpandOp(Node->getOperand(0), Lo, Hi); - SDValue BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT); - SDValue LTZ = DAG.getNode(ISD::CTTZ, dl, NVT, Lo); - SDValue BotNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), LTZ, - BitsC, ISD::SETNE); - SDValue HiPart = DAG.getNode(ISD::CTTZ, dl, NVT, Hi); - HiPart = DAG.getNode(ISD::ADD, dl, NVT, HiPart, BitsC); - - Lo = DAG.getNode(ISD::SELECT, dl, NVT, BotNotZero, LTZ, HiPart); - Hi = DAG.getConstant(0, NVT); - break; - } - - case ISD::VAARG: { - SDValue Ch = Node->getOperand(0); // Legalize the chain. - SDValue Ptr = Node->getOperand(1); // Legalize the pointer. - Lo = DAG.getVAArg(NVT, dl, Ch, Ptr, Node->getOperand(2)); - Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, Node->getOperand(2)); - - // Remember that we legalized the chain. - Hi = LegalizeOp(Hi); - AddLegalizedOperand(Op.getValue(1), Hi.getValue(1)); - if (TLI.isBigEndian()) - std::swap(Lo, Hi); - break; - } - - case ISD::LOAD: { - LoadSDNode *LD = cast(Node); - SDValue Ch = LD->getChain(); // Legalize the chain. - SDValue Ptr = LD->getBasePtr(); // Legalize the pointer. - ISD::LoadExtType ExtType = LD->getExtensionType(); - const Value *SV = LD->getSrcValue(); - int SVOffset = LD->getSrcValueOffset(); - unsigned Alignment = LD->getAlignment(); - bool isVolatile = LD->isVolatile(); - - if (ExtType == ISD::NON_EXTLOAD) { - Lo = DAG.getLoad(NVT, dl, Ch, Ptr, SV, SVOffset, - isVolatile, Alignment); - if (VT == MVT::f32 || VT == MVT::f64) { - // f32->i32 or f64->i64 one to one expansion. - // Remember that we legalized the chain. - AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Lo.getValue(1))); - // Recursively expand the new load. - if (getTypeAction(NVT) == Expand) - ExpandOp(Lo, Lo, Hi); - break; - } - - // Increment the pointer to the other half. - unsigned IncrementSize = Lo.getValueType().getSizeInBits()/8; - Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, - DAG.getIntPtrConstant(IncrementSize)); - SVOffset += IncrementSize; - Alignment = MinAlign(Alignment, IncrementSize); - Hi = DAG.getLoad(NVT, dl, Ch, Ptr, SV, SVOffset, - isVolatile, Alignment); - - // Build a factor node to remember that this load is independent of the - // other one. - SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), - Hi.getValue(1)); - - // Remember that we legalized the chain. - AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF)); - if (TLI.isBigEndian()) - std::swap(Lo, Hi); - } else { - MVT EVT = LD->getMemoryVT(); - - if ((VT == MVT::f64 && EVT == MVT::f32) || - (VT == MVT::ppcf128 && (EVT==MVT::f64 || EVT==MVT::f32))) { - // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND - SDValue Load = DAG.getLoad(EVT, dl, Ch, Ptr, SV, - SVOffset, isVolatile, Alignment); - // Remember that we legalized the chain. - AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Load.getValue(1))); - ExpandOp(DAG.getNode(ISD::FP_EXTEND, dl, VT, Load), Lo, Hi); - break; - } - - if (EVT == NVT) - Lo = DAG.getLoad(NVT, dl, Ch, Ptr, SV, - SVOffset, isVolatile, Alignment); - else - Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, SV, - SVOffset, EVT, isVolatile, - Alignment); - - // Remember that we legalized the chain. - AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Lo.getValue(1))); - - if (ExtType == ISD::SEXTLOAD) { - // The high part is obtained by SRA'ing all but one of the bits of the - // lo part. - unsigned LoSize = Lo.getValueType().getSizeInBits(); - Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, - DAG.getConstant(LoSize-1, TLI.getShiftAmountTy())); - } else if (ExtType == ISD::ZEXTLOAD) { - // The high part is just a zero. - Hi = DAG.getConstant(0, NVT); - } else /* if (ExtType == ISD::EXTLOAD) */ { - // The high part is undefined. - Hi = DAG.getUNDEF(NVT); - } - } - break; - } - case ISD::AND: - case ISD::OR: - case ISD::XOR: { // Simple logical operators -> two trivial pieces. - SDValue LL, LH, RL, RH; - ExpandOp(Node->getOperand(0), LL, LH); - ExpandOp(Node->getOperand(1), RL, RH); - Lo = DAG.getNode(Node->getOpcode(), dl, NVT, LL, RL); - Hi = DAG.getNode(Node->getOpcode(), dl, NVT, LH, RH); - break; - } - case ISD::SELECT: { - SDValue LL, LH, RL, RH; - ExpandOp(Node->getOperand(1), LL, LH); - ExpandOp(Node->getOperand(2), RL, RH); - if (getTypeAction(NVT) == Expand) - NVT = TLI.getTypeToExpandTo(NVT); - Lo = DAG.getNode(ISD::SELECT, dl, NVT, Node->getOperand(0), LL, RL); - if (VT != MVT::f32) - Hi = DAG.getNode(ISD::SELECT, dl, NVT, Node->getOperand(0), LH, RH); - break; - } - case ISD::SELECT_CC: { - SDValue TL, TH, FL, FH; - ExpandOp(Node->getOperand(2), TL, TH); - ExpandOp(Node->getOperand(3), FL, FH); - if (getTypeAction(NVT) == Expand) - NVT = TLI.getTypeToExpandTo(NVT); - Lo = DAG.getNode(ISD::SELECT_CC, dl, NVT, Node->getOperand(0), - Node->getOperand(1), TL, FL, Node->getOperand(4)); - if (VT != MVT::f32) - Hi = DAG.getNode(ISD::SELECT_CC, dl, NVT, Node->getOperand(0), - Node->getOperand(1), TH, FH, Node->getOperand(4)); - break; - } - case ISD::ANY_EXTEND: - // The low part is any extension of the input (which degenerates to a copy). - Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0)); - // The high part is undefined. - Hi = DAG.getUNDEF(NVT); - break; - case ISD::SIGN_EXTEND: { - // The low part is just a sign extension of the input (which degenerates to - // a copy). - Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, Node->getOperand(0)); - - // The high part is obtained by SRA'ing all but one of the bits of the lo - // part. - unsigned LoSize = Lo.getValueType().getSizeInBits(); - Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, - DAG.getConstant(LoSize-1, TLI.getShiftAmountTy())); - break; - } - case ISD::ZERO_EXTEND: - // The low part is just a zero extension of the input (which degenerates to - // a copy). - Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); - - // The high part is just a zero. - Hi = DAG.getConstant(0, NVT); - break; - - case ISD::TRUNCATE: { - // The input value must be larger than this value. Expand *it*. - SDValue NewLo; - ExpandOp(Node->getOperand(0), NewLo, Hi); - - // The low part is now either the right size, or it is closer. If not the - // right size, make an illegal truncate so we recursively expand it. - if (NewLo.getValueType() != Node->getValueType(0)) - NewLo = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), NewLo); - ExpandOp(NewLo, Lo, Hi); - break; - } - - case ISD::BIT_CONVERT: { - SDValue Tmp; - if (TLI.getOperationAction(ISD::BIT_CONVERT, VT) == TargetLowering::Custom){ - // If the target wants to, allow it to lower this itself. - switch (getTypeAction(Node->getOperand(0).getValueType())) { - case Expand: assert(0 && "cannot expand FP!"); - case Legal: Tmp = LegalizeOp(Node->getOperand(0)); break; - case Promote: Tmp = PromoteOp (Node->getOperand(0)); break; - } - Tmp = TLI.LowerOperation(DAG.getNode(ISD::BIT_CONVERT, dl, VT, Tmp), DAG); - } - - // f32 / f64 must be expanded to i32 / i64. - if (VT == MVT::f32 || VT == MVT::f64) { - Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Node->getOperand(0)); - if (getTypeAction(NVT) == Expand) - ExpandOp(Lo, Lo, Hi); - break; - } - - // If source operand will be expanded to the same type as VT, i.e. - // i64 <- f64, i32 <- f32, expand the source operand instead. - MVT VT0 = Node->getOperand(0).getValueType(); - if (getTypeAction(VT0) == Expand && TLI.getTypeToTransformTo(VT0) == VT) { - ExpandOp(Node->getOperand(0), Lo, Hi); - break; - } - - // Turn this into a load/store pair by default. - if (Tmp.getNode() == 0) - Tmp = EmitStackConvert(Node->getOperand(0), VT, VT, dl); - - ExpandOp(Tmp, Lo, Hi); - break; - } - - case ISD::READCYCLECOUNTER: { - assert(TLI.getOperationAction(ISD::READCYCLECOUNTER, VT) == - TargetLowering::Custom && - "Must custom expand ReadCycleCounter"); - SDValue Tmp = TLI.LowerOperation(Op, DAG); - assert(Tmp.getNode() && "Node must be custom expanded!"); - ExpandOp(Tmp.getValue(0), Lo, Hi); - AddLegalizedOperand(SDValue(Node, 1), // Remember we legalized the chain. - LegalizeOp(Tmp.getValue(1))); - break; - } - - case ISD::ATOMIC_CMP_SWAP: { - // This operation does not need a loop. - SDValue Tmp = TLI.LowerOperation(Op, DAG); - assert(Tmp.getNode() && "Node must be custom expanded!"); - ExpandOp(Tmp.getValue(0), Lo, Hi); - AddLegalizedOperand(SDValue(Node, 1), // Remember we legalized the chain. - LegalizeOp(Tmp.getValue(1))); - break; - } - - case ISD::ATOMIC_LOAD_ADD: - case ISD::ATOMIC_LOAD_SUB: - case ISD::ATOMIC_LOAD_AND: - case ISD::ATOMIC_LOAD_OR: - case ISD::ATOMIC_LOAD_XOR: - case ISD::ATOMIC_LOAD_NAND: - case ISD::ATOMIC_SWAP: { - // These operations require a loop to be generated. We can't do that yet, - // so substitute a target-dependent pseudo and expand that later. - SDValue In2Lo, In2Hi, In2; - ExpandOp(Op.getOperand(2), In2Lo, In2Hi); - In2 = DAG.getNode(ISD::BUILD_PAIR, dl, VT, In2Lo, In2Hi); - AtomicSDNode* Anode = cast(Node); - SDValue Replace = - DAG.getAtomic(Op.getOpcode(), dl, Anode->getMemoryVT(), - Op.getOperand(0), Op.getOperand(1), In2, - Anode->getSrcValue(), Anode->getAlignment()); - SDValue Result = TLI.LowerOperation(Replace, DAG); - ExpandOp(Result.getValue(0), Lo, Hi); - // Remember that we legalized the chain. - AddLegalizedOperand(SDValue(Node, 1), LegalizeOp(Result.getValue(1))); - break; - } - - // These operators cannot be expanded directly, emit them as calls to - // library functions. - case ISD::FP_TO_SINT: { - if (TLI.getOperationAction(ISD::FP_TO_SINT, VT) == TargetLowering::Custom) { - SDValue Op; - switch (getTypeAction(Node->getOperand(0).getValueType())) { - case Expand: assert(0 && "cannot expand FP!"); - case Legal: Op = LegalizeOp(Node->getOperand(0)); break; - case Promote: Op = PromoteOp (Node->getOperand(0)); break; - } - - Op = TLI.LowerOperation(DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op), DAG); - - // Now that the custom expander is done, expand the result, which is still - // VT. - if (Op.getNode()) { - ExpandOp(Op, Lo, Hi); - break; - } - } - - RTLIB::Libcall LC = RTLIB::getFPTOSINT(Node->getOperand(0).getValueType(), - VT); - assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected uint-to-fp conversion!"); - Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi); - break; - } - - case ISD::FP_TO_UINT: { - if (TLI.getOperationAction(ISD::FP_TO_UINT, VT) == TargetLowering::Custom) { - SDValue Op; - switch (getTypeAction(Node->getOperand(0).getValueType())) { - case Expand: assert(0 && "cannot expand FP!"); - case Legal: Op = LegalizeOp(Node->getOperand(0)); break; - case Promote: Op = PromoteOp (Node->getOperand(0)); break; - } - - Op = TLI.LowerOperation(DAG.getNode(ISD::FP_TO_UINT, dl, VT, Op), DAG); - - // Now that the custom expander is done, expand the result. - if (Op.getNode()) { - ExpandOp(Op, Lo, Hi); - break; - } - } - - RTLIB::Libcall LC = RTLIB::getFPTOUINT(Node->getOperand(0).getValueType(), - VT); - assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); - Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi); - break; - } - - case ISD::SHL: { - // If the target wants custom lowering, do so. - SDValue ShiftAmt = LegalizeOp(Node->getOperand(1)); - if (TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Custom) { - SDValue Op = DAG.getNode(ISD::SHL, dl, VT, Node->getOperand(0), ShiftAmt); - Op = TLI.LowerOperation(Op, DAG); - if (Op.getNode()) { - // Now that the custom expander is done, expand the result, which is - // still VT. - ExpandOp(Op, Lo, Hi); - break; - } - } - - // If ADDC/ADDE are supported and if the shift amount is a constant 1, emit - // this X << 1 as X+X. - if (ConstantSDNode *ShAmt = dyn_cast(ShiftAmt)) { - if (ShAmt->getAPIntValue() == 1 && - TLI.isOperationLegalOrCustom(ISD::ADDC, NVT) && - TLI.isOperationLegalOrCustom(ISD::ADDE, NVT)) { - SDValue LoOps[2], HiOps[3]; - ExpandOp(Node->getOperand(0), LoOps[0], HiOps[0]); - SDVTList VTList = DAG.getVTList(LoOps[0].getValueType(), MVT::Flag); - LoOps[1] = LoOps[0]; - Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); - - HiOps[1] = HiOps[0]; - HiOps[2] = Lo.getValue(1); - Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); - break; - } - } - - // If we can emit an efficient shift operation, do so now. - if (ExpandShift(ISD::SHL, Node->getOperand(0), ShiftAmt, Lo, Hi, dl)) - break; - - // If this target supports SHL_PARTS, use it. - TargetLowering::LegalizeAction Action = - TLI.getOperationAction(ISD::SHL_PARTS, NVT); - if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || - Action == TargetLowering::Custom) { - ExpandShiftParts(ISD::SHL_PARTS, Node->getOperand(0), - ShiftAmt, Lo, Hi, dl); - break; - } - - // Otherwise, emit a libcall. - Lo = ExpandLibCall(RTLIB::SHL_I64, Node, false/*left shift=unsigned*/, Hi); - break; - } - - case ISD::SRA: { - // If the target wants custom lowering, do so. - SDValue ShiftAmt = LegalizeOp(Node->getOperand(1)); - if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Custom) { - SDValue Op = DAG.getNode(ISD::SRA, dl, VT, Node->getOperand(0), ShiftAmt); - Op = TLI.LowerOperation(Op, DAG); - if (Op.getNode()) { - // Now that the custom expander is done, expand the result, which is - // still VT. - ExpandOp(Op, Lo, Hi); - break; - } - } - - // If we can emit an efficient shift operation, do so now. - if (ExpandShift(ISD::SRA, Node->getOperand(0), ShiftAmt, Lo, Hi, dl)) - break; - - // If this target supports SRA_PARTS, use it. - TargetLowering::LegalizeAction Action = - TLI.getOperationAction(ISD::SRA_PARTS, NVT); - if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || - Action == TargetLowering::Custom) { - ExpandShiftParts(ISD::SRA_PARTS, Node->getOperand(0), - ShiftAmt, Lo, Hi, dl); - break; - } - - // Otherwise, emit a libcall. - Lo = ExpandLibCall(RTLIB::SRA_I64, Node, true/*ashr is signed*/, Hi); - break; - } - - case ISD::SRL: { - // If the target wants custom lowering, do so. - SDValue ShiftAmt = LegalizeOp(Node->getOperand(1)); - if (TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Custom) { - SDValue Op = DAG.getNode(ISD::SRL, dl, VT, Node->getOperand(0), ShiftAmt); - Op = TLI.LowerOperation(Op, DAG); - if (Op.getNode()) { - // Now that the custom expander is done, expand the result, which is - // still VT. - ExpandOp(Op, Lo, Hi); - break; - } - } - - // If we can emit an efficient shift operation, do so now. - if (ExpandShift(ISD::SRL, Node->getOperand(0), ShiftAmt, Lo, Hi, dl)) - break; - - // If this target supports SRL_PARTS, use it. - TargetLowering::LegalizeAction Action = - TLI.getOperationAction(ISD::SRL_PARTS, NVT); - if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || - Action == TargetLowering::Custom) { - ExpandShiftParts(ISD::SRL_PARTS, - Node->getOperand(0), ShiftAmt, Lo, Hi, dl); - break; - } - - // Otherwise, emit a libcall. - Lo = ExpandLibCall(RTLIB::SRL_I64, Node, false/*lshr is unsigned*/, Hi); - break; - } - - case ISD::ADD: - case ISD::SUB: { - // If the target wants to custom expand this, let them. - if (TLI.getOperationAction(Node->getOpcode(), VT) == - TargetLowering::Custom) { - SDValue Result = TLI.LowerOperation(Op, DAG); - if (Result.getNode()) { - ExpandOp(Result, Lo, Hi); - break; - } - } - // Expand the subcomponents. - SDValue LHSL, LHSH, RHSL, RHSH; - ExpandOp(Node->getOperand(0), LHSL, LHSH); - ExpandOp(Node->getOperand(1), RHSL, RHSH); - SDValue LoOps[2], HiOps[3]; - LoOps[0] = LHSL; - LoOps[1] = RHSL; - HiOps[0] = LHSH; - HiOps[1] = RHSH; - - //cascaded check to see if any smaller size has a a carry flag. - unsigned OpV = Node->getOpcode() == ISD::ADD ? ISD::ADDC : ISD::SUBC; - bool hasCarry = false; - for (unsigned BitSize = NVT.getSizeInBits(); BitSize != 0; BitSize /= 2) { - MVT AVT = MVT::getIntegerVT(BitSize); - if (TLI.isOperationLegalOrCustom(OpV, AVT)) { - hasCarry = true; - break; - } - } - - if(hasCarry) { - SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); - if (Node->getOpcode() == ISD::ADD) { - Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); - HiOps[2] = Lo.getValue(1); - Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); - } else { - Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2); - HiOps[2] = Lo.getValue(1); - Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3); - } - break; - } else { - if (Node->getOpcode() == ISD::ADD) { - Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps, 2); - Hi = DAG.getNode(ISD::ADD, dl, NVT, HiOps, 2); - SDValue Cmp1 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), - Lo, LoOps[0], ISD::SETULT); - SDValue Carry1 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp1, - DAG.getConstant(1, NVT), - DAG.getConstant(0, NVT)); - SDValue Cmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), - Lo, LoOps[1], ISD::SETULT); - SDValue Carry2 = DAG.getNode(ISD::SELECT, dl, NVT, Cmp2, - DAG.getConstant(1, NVT), - Carry1); - Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); - } else { - Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps, 2); - Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps, 2); - SDValue Cmp = DAG.getSetCC(dl, NVT, LoOps[0], LoOps[1], ISD::SETULT); - SDValue Borrow = DAG.getNode(ISD::SELECT, dl, NVT, Cmp, - DAG.getConstant(1, NVT), - DAG.getConstant(0, NVT)); - Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); - } - break; - } - } - - case ISD::ADDC: - case ISD::SUBC: { - // Expand the subcomponents. - SDValue LHSL, LHSH, RHSL, RHSH; - ExpandOp(Node->getOperand(0), LHSL, LHSH); - ExpandOp(Node->getOperand(1), RHSL, RHSH); - SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); - SDValue LoOps[2] = { LHSL, RHSL }; - SDValue HiOps[3] = { LHSH, RHSH }; - - if (Node->getOpcode() == ISD::ADDC) { - Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps, 2); - HiOps[2] = Lo.getValue(1); - Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps, 3); - } else { - Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps, 2); - HiOps[2] = Lo.getValue(1); - Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps, 3); - } - // Remember that we legalized the flag. - AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1))); - break; - } - case ISD::ADDE: - case ISD::SUBE: { - // Expand the subcomponents. - SDValue LHSL, LHSH, RHSL, RHSH; - ExpandOp(Node->getOperand(0), LHSL, LHSH); - ExpandOp(Node->getOperand(1), RHSL, RHSH); - SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag); - SDValue LoOps[3] = { LHSL, RHSL, Node->getOperand(2) }; - SDValue HiOps[3] = { LHSH, RHSH }; - - Lo = DAG.getNode(Node->getOpcode(), dl, VTList, LoOps, 3); - HiOps[2] = Lo.getValue(1); - Hi = DAG.getNode(Node->getOpcode(), dl, VTList, HiOps, 3); - - // Remember that we legalized the flag. - AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1))); - break; - } - case ISD::MUL: { - // If the target wants to custom expand this, let them. - if (TLI.getOperationAction(ISD::MUL, VT) == TargetLowering::Custom) { - SDValue New = TLI.LowerOperation(Op, DAG); - if (New.getNode()) { - ExpandOp(New, Lo, Hi); - break; - } - } - - bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, NVT); - bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, NVT); - bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, NVT); - bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, NVT); - if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) { - SDValue LL, LH, RL, RH; - ExpandOp(Node->getOperand(0), LL, LH); - ExpandOp(Node->getOperand(1), RL, RH); - unsigned OuterBitSize = Op.getValueSizeInBits(); - unsigned InnerBitSize = RH.getValueSizeInBits(); - unsigned LHSSB = DAG.ComputeNumSignBits(Op.getOperand(0)); - unsigned RHSSB = DAG.ComputeNumSignBits(Op.getOperand(1)); - APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize); - if (DAG.MaskedValueIsZero(Node->getOperand(0), HighMask) && - DAG.MaskedValueIsZero(Node->getOperand(1), HighMask)) { - // The inputs are both zero-extended. - if (HasUMUL_LOHI) { - // We can emit a umul_lohi. - Lo = DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL); - Hi = SDValue(Lo.getNode(), 1); - break; - } - if (HasMULHU) { - // We can emit a mulhu+mul. - Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); - Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL); - break; - } - } - if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) { - // The input values are both sign-extended. - if (HasSMUL_LOHI) { - // We can emit a smul_lohi. - Lo = DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(NVT, NVT), LL, RL); - Hi = SDValue(Lo.getNode(), 1); - break; - } - if (HasMULHS) { - // We can emit a mulhs+mul. - Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); - Hi = DAG.getNode(ISD::MULHS, dl, NVT, LL, RL); - break; - } - } - if (HasUMUL_LOHI) { - // Lo,Hi = umul LHS, RHS. - SDValue UMulLOHI = DAG.getNode(ISD::UMUL_LOHI, dl, - DAG.getVTList(NVT, NVT), LL, RL); - Lo = UMulLOHI; - Hi = UMulLOHI.getValue(1); - RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH); - LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL); - Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH); - Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH); - break; - } - if (HasMULHU) { - Lo = DAG.getNode(ISD::MUL, dl, NVT, LL, RL); - Hi = DAG.getNode(ISD::MULHU, dl, NVT, LL, RL); - RH = DAG.getNode(ISD::MUL, dl, NVT, LL, RH); - LH = DAG.getNode(ISD::MUL, dl, NVT, LH, RL); - Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, RH); - Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, LH); - break; - } - } - - // If nothing else, we can make a libcall. - Lo = ExpandLibCall(RTLIB::MUL_I64, Node, false/*sign irrelevant*/, Hi); - break; - } - case ISD::SDIV: - Lo = ExpandLibCall(RTLIB::SDIV_I64, Node, true, Hi); - break; - case ISD::UDIV: - Lo = ExpandLibCall(RTLIB::UDIV_I64, Node, true, Hi); - break; - case ISD::SREM: - Lo = ExpandLibCall(RTLIB::SREM_I64, Node, true, Hi); - break; - case ISD::UREM: - Lo = ExpandLibCall(RTLIB::UREM_I64, Node, true, Hi); - break; - - case ISD::FADD: - Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::ADD_F32, - RTLIB::ADD_F64, - RTLIB::ADD_F80, - RTLIB::ADD_PPCF128), - Node, false, Hi); - break; - case ISD::FSUB: - Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::SUB_F32, - RTLIB::SUB_F64, - RTLIB::SUB_F80, - RTLIB::SUB_PPCF128), - Node, false, Hi); - break; - case ISD::FMUL: - Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::MUL_F32, - RTLIB::MUL_F64, - RTLIB::MUL_F80, - RTLIB::MUL_PPCF128), - Node, false, Hi); - break; - case ISD::FDIV: - Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::DIV_F32, - RTLIB::DIV_F64, - RTLIB::DIV_F80, - RTLIB::DIV_PPCF128), - Node, false, Hi); - break; - case ISD::FP_EXTEND: { - if (VT == MVT::ppcf128) { - assert(Node->getOperand(0).getValueType()==MVT::f32 || - Node->getOperand(0).getValueType()==MVT::f64); - const uint64_t zero = 0; - if (Node->getOperand(0).getValueType()==MVT::f32) - Hi = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Node->getOperand(0)); - else - Hi = Node->getOperand(0); - Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64); - break; - } - RTLIB::Libcall LC = RTLIB::getFPEXT(Node->getOperand(0).getValueType(), VT); - assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); - Lo = ExpandLibCall(LC, Node, true, Hi); - break; - } - case ISD::FP_ROUND: { - RTLIB::Libcall LC = RTLIB::getFPROUND(Node->getOperand(0).getValueType(), - VT); - assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!"); - Lo = ExpandLibCall(LC, Node, true, Hi); - break; - } - case ISD::FSQRT: - case ISD::FSIN: - case ISD::FCOS: - case ISD::FLOG: - case ISD::FLOG2: - case ISD::FLOG10: - case ISD::FEXP: - case ISD::FEXP2: - case ISD::FTRUNC: - case ISD::FFLOOR: - case ISD::FCEIL: - case ISD::FRINT: - case ISD::FNEARBYINT: - case ISD::FPOW: - case ISD::FPOWI: { - RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; - switch(Node->getOpcode()) { - case ISD::FSQRT: - LC = GetFPLibCall(VT, RTLIB::SQRT_F32, RTLIB::SQRT_F64, - RTLIB::SQRT_F80, RTLIB::SQRT_PPCF128); - break; - case ISD::FSIN: - LC = GetFPLibCall(VT, RTLIB::SIN_F32, RTLIB::SIN_F64, - RTLIB::SIN_F80, RTLIB::SIN_PPCF128); - break; - case ISD::FCOS: - LC = GetFPLibCall(VT, RTLIB::COS_F32, RTLIB::COS_F64, - RTLIB::COS_F80, RTLIB::COS_PPCF128); - break; - case ISD::FLOG: - LC = GetFPLibCall(VT, RTLIB::LOG_F32, RTLIB::LOG_F64, - RTLIB::LOG_F80, RTLIB::LOG_PPCF128); - break; - case ISD::FLOG2: - LC = GetFPLibCall(VT, RTLIB::LOG2_F32, RTLIB::LOG2_F64, - RTLIB::LOG2_F80, RTLIB::LOG2_PPCF128); - break; - case ISD::FLOG10: - LC = GetFPLibCall(VT, RTLIB::LOG10_F32, RTLIB::LOG10_F64, - RTLIB::LOG10_F80, RTLIB::LOG10_PPCF128); - break; - case ISD::FEXP: - LC = GetFPLibCall(VT, RTLIB::EXP_F32, RTLIB::EXP_F64, - RTLIB::EXP_F80, RTLIB::EXP_PPCF128); - break; - case ISD::FEXP2: - LC = GetFPLibCall(VT, RTLIB::EXP2_F32, RTLIB::EXP2_F64, - RTLIB::EXP2_F80, RTLIB::EXP2_PPCF128); - break; - case ISD::FTRUNC: - LC = GetFPLibCall(VT, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, - RTLIB::TRUNC_F80, RTLIB::TRUNC_PPCF128); - break; - case ISD::FFLOOR: - LC = GetFPLibCall(VT, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, - RTLIB::FLOOR_F80, RTLIB::FLOOR_PPCF128); - break; - case ISD::FCEIL: - LC = GetFPLibCall(VT, RTLIB::CEIL_F32, RTLIB::CEIL_F64, - RTLIB::CEIL_F80, RTLIB::CEIL_PPCF128); - break; - case ISD::FRINT: - LC = GetFPLibCall(VT, RTLIB::RINT_F32, RTLIB::RINT_F64, - RTLIB::RINT_F80, RTLIB::RINT_PPCF128); - break; - case ISD::FNEARBYINT: - LC = GetFPLibCall(VT, RTLIB::NEARBYINT_F32, RTLIB::NEARBYINT_F64, - RTLIB::NEARBYINT_F80, RTLIB::NEARBYINT_PPCF128); - break; - case ISD::FPOW: - LC = GetFPLibCall(VT, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80, - RTLIB::POW_PPCF128); - break; - case ISD::FPOWI: - LC = GetFPLibCall(VT, RTLIB::POWI_F32, RTLIB::POWI_F64, RTLIB::POWI_F80, - RTLIB::POWI_PPCF128); - break; - default: assert(0 && "Unreachable!"); - } - Lo = ExpandLibCall(LC, Node, false, Hi); - break; - } - case ISD::FABS: { - if (VT == MVT::ppcf128) { - SDValue Tmp; - ExpandOp(Node->getOperand(0), Lo, Tmp); - Hi = DAG.getNode(ISD::FABS, dl, NVT, Tmp); - // lo = hi==fabs(hi) ? lo : -lo; - Lo = DAG.getNode(ISD::SELECT_CC, dl, NVT, Hi, Tmp, - Lo, DAG.getNode(ISD::FNEG, dl, NVT, Lo), - DAG.getCondCode(ISD::SETEQ)); - break; - } - SDValue Mask = (VT == MVT::f64) - ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT) - : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT); - Mask = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Mask); - Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Node->getOperand(0)); - Lo = DAG.getNode(ISD::AND, dl, NVT, Lo, Mask); - if (getTypeAction(NVT) == Expand) - ExpandOp(Lo, Lo, Hi); - break; - } - case ISD::FNEG: { - if (VT == MVT::ppcf128) { - ExpandOp(Node->getOperand(0), Lo, Hi); - Lo = DAG.getNode(ISD::FNEG, dl, MVT::f64, Lo); - Hi = DAG.getNode(ISD::FNEG, dl, MVT::f64, Hi); - break; - } - SDValue Mask = (VT == MVT::f64) - ? DAG.getConstantFP(BitsToDouble(1ULL << 63), VT) - : DAG.getConstantFP(BitsToFloat(1U << 31), VT); - Mask = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Mask); - Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, Node->getOperand(0)); - Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Mask); - if (getTypeAction(NVT) == Expand) - ExpandOp(Lo, Lo, Hi); - break; - } - case ISD::FCOPYSIGN: { - Lo = ExpandFCOPYSIGNToBitwiseOps(Node, NVT, DAG, TLI); - if (getTypeAction(NVT) == Expand) - ExpandOp(Lo, Lo, Hi); - break; - } - case ISD::SINT_TO_FP: - case ISD::UINT_TO_FP: { - bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP; - MVT SrcVT = Node->getOperand(0).getValueType(); - - // Promote the operand if needed. Do this before checking for - // ppcf128 so conversions of i16 and i8 work. - if (getTypeAction(SrcVT) == Promote) { - SDValue Tmp = PromoteOp(Node->getOperand(0)); - Tmp = isSigned - ? DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Tmp.getValueType(), Tmp, - DAG.getValueType(SrcVT)) - : DAG.getZeroExtendInReg(Tmp, dl, SrcVT); - Node = DAG.UpdateNodeOperands(Op, Tmp).getNode(); - SrcVT = Node->getOperand(0).getValueType(); - } - - if (VT == MVT::ppcf128 && SrcVT == MVT::i32) { - static const uint64_t zero = 0; - if (isSigned) { - Hi = LegalizeOp(DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f64, - Node->getOperand(0))); - Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64); - } else { - static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 }; - Hi = LegalizeOp(DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f64, - Node->getOperand(0))); - Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64); - Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi); - // X>=0 ? {(f64)x, 0} : {(f64)x, 0} + 2^32 - ExpandOp(DAG.getNode(ISD::SELECT_CC, dl, - MVT::ppcf128, Node->getOperand(0), - DAG.getConstant(0, MVT::i32), - DAG.getNode(ISD::FADD, dl, MVT::ppcf128, Hi, - DAG.getConstantFP - (APFloat(APInt(128, 2, TwoE32)), - MVT::ppcf128)), - Hi, - DAG.getCondCode(ISD::SETLT)), - Lo, Hi); - } - break; - } - if (VT == MVT::ppcf128 && SrcVT == MVT::i64 && !isSigned) { - // si64->ppcf128 done by libcall, below - static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 }; - ExpandOp(DAG.getNode(ISD::SINT_TO_FP, dl, MVT::ppcf128, - Node->getOperand(0)), Lo, Hi); - Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi); - // x>=0 ? (ppcf128)(i64)x : (ppcf128)(i64)x + 2^64 - ExpandOp(DAG.getNode(ISD::SELECT_CC, dl, MVT::ppcf128, - Node->getOperand(0), - DAG.getConstant(0, MVT::i64), - DAG.getNode(ISD::FADD, dl, MVT::ppcf128, Hi, - DAG.getConstantFP - (APFloat(APInt(128, 2, TwoE64)), - MVT::ppcf128)), - Hi, - DAG.getCondCode(ISD::SETLT)), - Lo, Hi); - break; - } - - Lo = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, VT, - Node->getOperand(0), dl); - if (getTypeAction(Lo.getValueType()) == Expand) - // float to i32 etc. can be 'expanded' to a single node. - ExpandOp(Lo, Lo, Hi); - break; - } - } - - // Make sure the resultant values have been legalized themselves, unless this - // is a type that requires multi-step expansion. - if (getTypeAction(NVT) != Expand && NVT != MVT::isVoid) { - Lo = LegalizeOp(Lo); - if (Hi.getNode()) - // Don't legalize the high part if it is expanded to a single node. - Hi = LegalizeOp(Hi); - } - - // Remember in a map if the values will be reused later. - bool isNew = - ExpandedNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second; - assert(isNew && "Value already expanded?!?"); - isNew = isNew; -} - -/// SplitVectorOp - Given an operand of vector type, break it down into -/// two smaller values, still of vector type. -void SelectionDAGLegalize::SplitVectorOp(SDValue Op, SDValue &Lo, - SDValue &Hi) { - assert(0 && "This should be dead!"); - assert(Op.getValueType().isVector() && "Cannot split non-vector type!"); - SDNode *Node = Op.getNode(); - DebugLoc dl = Node->getDebugLoc(); - unsigned NumElements = Op.getValueType().getVectorNumElements(); - assert(NumElements > 1 && "Cannot split a single element vector!"); - - MVT NewEltVT = Op.getValueType().getVectorElementType(); - - unsigned NewNumElts_Lo = 1 << Log2_32(NumElements-1); - unsigned NewNumElts_Hi = NumElements - NewNumElts_Lo; - - MVT NewVT_Lo = MVT::getVectorVT(NewEltVT, NewNumElts_Lo); - MVT NewVT_Hi = MVT::getVectorVT(NewEltVT, NewNumElts_Hi); - - // See if we already split it. - std::map >::iterator I - = SplitNodes.find(Op); - if (I != SplitNodes.end()) { - Lo = I->second.first; - Hi = I->second.second; - return; - } - - switch (Node->getOpcode()) { - default: -#ifndef NDEBUG - Node->dump(&DAG); -#endif - assert(0 && "Unhandled operation in SplitVectorOp!"); - case ISD::UNDEF: - Lo = DAG.getUNDEF(NewVT_Lo); - Hi = DAG.getUNDEF(NewVT_Hi); - break; - case ISD::BUILD_PAIR: - Lo = Node->getOperand(0); - Hi = Node->getOperand(1); - break; - case ISD::INSERT_VECTOR_ELT: { - if (ConstantSDNode *Idx = dyn_cast(Node->getOperand(2))) { - SplitVectorOp(Node->getOperand(0), Lo, Hi); - unsigned Index = Idx->getZExtValue(); - SDValue ScalarOp = Node->getOperand(1); - if (Index < NewNumElts_Lo) - Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVT_Lo, Lo, ScalarOp, - DAG.getIntPtrConstant(Index)); - else - Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVT_Hi, Hi, ScalarOp, - DAG.getIntPtrConstant(Index - NewNumElts_Lo)); - break; - } - SDValue Tmp = PerformInsertVectorEltInMemory(Node->getOperand(0), - Node->getOperand(1), - Node->getOperand(2), dl); - SplitVectorOp(Tmp, Lo, Hi); - break; - } - case ISD::VECTOR_SHUFFLE: { - // Build the low part. - SDValue Mask = Node->getOperand(2); - SmallVector Ops; - MVT PtrVT = TLI.getPointerTy(); - - // Insert all of the elements from the input that are needed. We use - // buildvector of extractelement here because the input vectors will have - // to be legalized, so this makes the code simpler. - for (unsigned i = 0; i != NewNumElts_Lo; ++i) { - SDValue IdxNode = Mask.getOperand(i); - if (IdxNode.getOpcode() == ISD::UNDEF) { - Ops.push_back(DAG.getUNDEF(NewEltVT)); - continue; - } - unsigned Idx = cast(IdxNode)->getZExtValue(); - SDValue InVec = Node->getOperand(0); - if (Idx >= NumElements) { - InVec = Node->getOperand(1); - Idx -= NumElements; - } - Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, InVec, - DAG.getConstant(Idx, PtrVT))); - } - Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT_Lo, &Ops[0], Ops.size()); - Ops.clear(); - - for (unsigned i = NewNumElts_Lo; i != NumElements; ++i) { - SDValue IdxNode = Mask.getOperand(i); - if (IdxNode.getOpcode() == ISD::UNDEF) { - Ops.push_back(DAG.getUNDEF(NewEltVT)); - continue; - } - unsigned Idx = cast(IdxNode)->getZExtValue(); - SDValue InVec = Node->getOperand(0); - if (Idx >= NumElements) { - InVec = Node->getOperand(1); - Idx -= NumElements; - } - Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, InVec, - DAG.getConstant(Idx, PtrVT))); - } - Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT_Hi, &Ops[0], Ops.size()); - break; - } - case ISD::BUILD_VECTOR: { - SmallVector LoOps(Node->op_begin(), - Node->op_begin()+NewNumElts_Lo); - Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT_Lo, &LoOps[0], LoOps.size()); - - SmallVector HiOps(Node->op_begin()+NewNumElts_Lo, - Node->op_end()); - Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT_Hi, &HiOps[0], HiOps.size()); - break; - } - case ISD::CONCAT_VECTORS: { - // FIXME: Handle non-power-of-two vectors? - unsigned NewNumSubvectors = Node->getNumOperands() / 2; - if (NewNumSubvectors == 1) { - Lo = Node->getOperand(0); - Hi = Node->getOperand(1); - } else { - SmallVector LoOps(Node->op_begin(), - Node->op_begin()+NewNumSubvectors); - Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT_Lo, - &LoOps[0], LoOps.size()); - - SmallVector HiOps(Node->op_begin()+NewNumSubvectors, - Node->op_end()); - Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT_Hi, - &HiOps[0], HiOps.size()); - } - break; - } - case ISD::EXTRACT_SUBVECTOR: { - SDValue Vec = Op.getOperand(0); - SDValue Idx = Op.getOperand(1); - MVT IdxVT = Idx.getValueType(); - - Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT_Lo, Vec, Idx); - ConstantSDNode *CIdx = dyn_cast(Idx); - if (CIdx) { - Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT_Hi, Vec, - DAG.getConstant(CIdx->getZExtValue() + NewNumElts_Lo, - IdxVT)); - } else { - Idx = DAG.getNode(ISD::ADD, dl, IdxVT, Idx, - DAG.getConstant(NewNumElts_Lo, IdxVT)); - Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT_Hi, Vec, Idx); - } - break; - } - case ISD::SELECT: { - SDValue Cond = Node->getOperand(0); - - SDValue LL, LH, RL, RH; - SplitVectorOp(Node->getOperand(1), LL, LH); - SplitVectorOp(Node->getOperand(2), RL, RH); - - if (Cond.getValueType().isVector()) { - // Handle a vector merge. - SDValue CL, CH; - SplitVectorOp(Cond, CL, CH); - Lo = DAG.getNode(Node->getOpcode(), dl, NewVT_Lo, CL, LL, RL); - Hi = DAG.getNode(Node->getOpcode(), dl, NewVT_Hi, CH, LH, RH); - } else { - // Handle a simple select with vector operands. - Lo = DAG.getNode(Node->getOpcode(), dl, NewVT_Lo, Cond, LL, RL); - Hi = DAG.getNode(Node->getOpcode(), dl, NewVT_Hi, Cond, LH, RH); - } - break; - } - case ISD::SELECT_CC: { - SDValue CondLHS = Node->getOperand(0); - SDValue CondRHS = Node->getOperand(1); - SDValue CondCode = Node->getOperand(4); - - SDValue LL, LH, RL, RH; - SplitVectorOp(Node->getOperand(2), LL, LH); - SplitVectorOp(Node->getOperand(3), RL, RH); - - // Handle a simple select with vector operands. - Lo = DAG.getNode(ISD::SELECT_CC, dl, NewVT_Lo, CondLHS, CondRHS, - LL, RL, CondCode); - Hi = DAG.getNode(ISD::SELECT_CC, dl, NewVT_Hi, CondLHS, CondRHS, - LH, RH, CondCode); - break; - } - case ISD::VSETCC: { - SDValue LL, LH, RL, RH; - SplitVectorOp(Node->getOperand(0), LL, LH); - SplitVectorOp(Node->getOperand(1), RL, RH); - Lo = DAG.getNode(ISD::VSETCC, dl, NewVT_Lo, LL, RL, Node->getOperand(2)); - Hi = DAG.getNode(ISD::VSETCC, dl, NewVT_Hi, LH, RH, Node->getOperand(2)); - break; - } - case ISD::ADD: - case ISD::SUB: - case ISD::MUL: - case ISD::FADD: - case ISD::FSUB: - case ISD::FMUL: - case ISD::SDIV: - case ISD::UDIV: - case ISD::FDIV: - case ISD::FPOW: - case ISD::AND: - case ISD::OR: - case ISD::XOR: - case ISD::UREM: - case ISD::SREM: - case ISD::FREM: - case ISD::SHL: - case ISD::SRA: - case ISD::SRL: { - SDValue LL, LH, RL, RH; - SplitVectorOp(Node->getOperand(0), LL, LH); - SplitVectorOp(Node->getOperand(1), RL, RH); - - Lo = DAG.getNode(Node->getOpcode(), dl, NewVT_Lo, LL, RL); - Hi = DAG.getNode(Node->getOpcode(), dl, NewVT_Hi, LH, RH); - break; - } - case ISD::FP_ROUND: - case ISD::FPOWI: { - SDValue L, H; - SplitVectorOp(Node->getOperand(0), L, H); - - Lo = DAG.getNode(Node->getOpcode(), dl, NewVT_Lo, L, Node->getOperand(1)); - Hi = DAG.getNode(Node->getOpcode(), dl, NewVT_Hi, H, Node->getOperand(1)); - break; - } - case ISD::CTTZ: - case ISD::CTLZ: - case ISD::CTPOP: - case ISD::FNEG: - case ISD::FABS: - case ISD::FSQRT: - case ISD::FSIN: - case ISD::FCOS: - case ISD::FLOG: - case ISD::FLOG2: - case ISD::FLOG10: - case ISD::FEXP: - case ISD::FEXP2: - case ISD::FP_TO_SINT: - case ISD::FP_TO_UINT: - case ISD::SINT_TO_FP: - case ISD::UINT_TO_FP: - case ISD::TRUNCATE: - case ISD::ANY_EXTEND: - case ISD::SIGN_EXTEND: - case ISD::ZERO_EXTEND: - case ISD::FP_EXTEND: { - SDValue L, H; - SplitVectorOp(Node->getOperand(0), L, H); - - Lo = DAG.getNode(Node->getOpcode(), dl, NewVT_Lo, L); - Hi = DAG.getNode(Node->getOpcode(), dl, NewVT_Hi, H); - break; - } - case ISD::CONVERT_RNDSAT: { - ISD::CvtCode CvtCode = cast(Node)->getCvtCode(); - SDValue L, H; - SplitVectorOp(Node->getOperand(0), L, H); - SDValue DTyOpL = DAG.getValueType(NewVT_Lo); - SDValue DTyOpH = DAG.getValueType(NewVT_Hi); - SDValue STyOpL = DAG.getValueType(L.getValueType()); - SDValue STyOpH = DAG.getValueType(H.getValueType()); - - SDValue RndOp = Node->getOperand(3); - SDValue SatOp = Node->getOperand(4); - - Lo = DAG.getConvertRndSat(NewVT_Lo, dl, L, DTyOpL, STyOpL, - RndOp, SatOp, CvtCode); - Hi = DAG.getConvertRndSat(NewVT_Hi, dl, H, DTyOpH, STyOpH, - RndOp, SatOp, CvtCode); - break; - } - case ISD::LOAD: { - LoadSDNode *LD = cast(Node); - SDValue Ch = LD->getChain(); - SDValue Ptr = LD->getBasePtr(); - ISD::LoadExtType ExtType = LD->getExtensionType(); - const Value *SV = LD->getSrcValue(); - int SVOffset = LD->getSrcValueOffset(); - MVT MemoryVT = LD->getMemoryVT(); - unsigned Alignment = LD->getAlignment(); - bool isVolatile = LD->isVolatile(); - - assert(LD->isUnindexed() && "Indexed vector loads are not supported yet!"); - SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); - - MVT MemNewEltVT = MemoryVT.getVectorElementType(); - MVT MemNewVT_Lo = MVT::getVectorVT(MemNewEltVT, NewNumElts_Lo); - MVT MemNewVT_Hi = MVT::getVectorVT(MemNewEltVT, NewNumElts_Hi); - - Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, - NewVT_Lo, Ch, Ptr, Offset, - SV, SVOffset, MemNewVT_Lo, isVolatile, Alignment); - unsigned IncrementSize = NewNumElts_Lo * MemNewEltVT.getSizeInBits()/8; - Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, - DAG.getIntPtrConstant(IncrementSize)); - SVOffset += IncrementSize; - Alignment = MinAlign(Alignment, IncrementSize); - Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, - NewVT_Hi, Ch, Ptr, Offset, - SV, SVOffset, MemNewVT_Hi, isVolatile, Alignment); - - // Build a factor node to remember that this load is independent of the - // other one. - SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), - Hi.getValue(1)); - - // Remember that we legalized the chain. - AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF)); - break; - } - case ISD::BIT_CONVERT: { - // We know the result is a vector. The input may be either a vector or a - // scalar value. - SDValue InOp = Node->getOperand(0); - if (!InOp.getValueType().isVector() || - InOp.getValueType().getVectorNumElements() == 1) { - // The input is a scalar or single-element vector. - // Lower to a store/load so that it can be split. - // FIXME: this could be improved probably. - unsigned LdAlign = TLI.getTargetData()-> - getPrefTypeAlignment(Op.getValueType().getTypeForMVT()); - SDValue Ptr = DAG.CreateStackTemporary(InOp.getValueType(), LdAlign); - int FI = cast(Ptr.getNode())->getIndex(); - - SDValue St = DAG.getStore(DAG.getEntryNode(), dl, - InOp, Ptr, - PseudoSourceValue::getFixedStack(FI), 0); - InOp = DAG.getLoad(Op.getValueType(), dl, St, Ptr, - PseudoSourceValue::getFixedStack(FI), 0); - } - // Split the vector and convert each of the pieces now. - SplitVectorOp(InOp, Lo, Hi); - Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT_Lo, Lo); - Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT_Hi, Hi); - break; - } - } - - // Remember in a map if the values will be reused later. - bool isNew = - SplitNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second; - assert(isNew && "Value already split?!?"); - isNew = isNew; -} - - -/// ScalarizeVectorOp - Given an operand of single-element vector type -/// (e.g. v1f32), convert it into the equivalent operation that returns a -/// scalar (e.g. f32) value. -SDValue SelectionDAGLegalize::ScalarizeVectorOp(SDValue Op) { - assert(0 && "This should be dead!"); - assert(Op.getValueType().isVector() && "Bad ScalarizeVectorOp invocation!"); - SDNode *Node = Op.getNode(); - DebugLoc dl = Node->getDebugLoc(); - MVT NewVT = Op.getValueType().getVectorElementType(); - assert(Op.getValueType().getVectorNumElements() == 1); - - // See if we already scalarized it. - std::map::iterator I = ScalarizedNodes.find(Op); - if (I != ScalarizedNodes.end()) return I->second; - - SDValue Result; - switch (Node->getOpcode()) { - default: -#ifndef NDEBUG - Node->dump(&DAG); cerr << "\n"; -#endif - assert(0 && "Unknown vector operation in ScalarizeVectorOp!"); - case ISD::ADD: - case ISD::FADD: - case ISD::SUB: - case ISD::FSUB: - case ISD::MUL: - case ISD::FMUL: - case ISD::SDIV: - case ISD::UDIV: - case ISD::FDIV: - case ISD::SREM: - case ISD::UREM: - case ISD::FREM: - case ISD::FPOW: - case ISD::AND: - case ISD::OR: - case ISD::XOR: - Result = DAG.getNode(Node->getOpcode(), dl, - NewVT, - ScalarizeVectorOp(Node->getOperand(0)), - ScalarizeVectorOp(Node->getOperand(1))); - break; - case ISD::FNEG: - case ISD::FABS: - case ISD::FSQRT: - case ISD::FSIN: - case ISD::FCOS: - case ISD::FLOG: - case ISD::FLOG2: - case ISD::FLOG10: - case ISD::FEXP: - case ISD::FEXP2: - case ISD::FP_TO_SINT: - case ISD::FP_TO_UINT: - case ISD::SINT_TO_FP: - case ISD::UINT_TO_FP: - case ISD::SIGN_EXTEND: - case ISD::ZERO_EXTEND: - case ISD::ANY_EXTEND: - case ISD::TRUNCATE: - case ISD::FP_EXTEND: - Result = DAG.getNode(Node->getOpcode(), dl, - NewVT, - ScalarizeVectorOp(Node->getOperand(0))); - break; - case ISD::CONVERT_RNDSAT: { - SDValue Op0 = ScalarizeVectorOp(Node->getOperand(0)); - Result = DAG.getConvertRndSat(NewVT, dl, Op0, - DAG.getValueType(NewVT), - DAG.getValueType(Op0.getValueType()), - Node->getOperand(3), - Node->getOperand(4), - cast(Node)->getCvtCode()); - break; - } - case ISD::FPOWI: - case ISD::FP_ROUND: - Result = DAG.getNode(Node->getOpcode(), dl, - NewVT, - ScalarizeVectorOp(Node->getOperand(0)), - Node->getOperand(1)); - break; - case ISD::LOAD: { - LoadSDNode *LD = cast(Node); - SDValue Ch = LegalizeOp(LD->getChain()); // Legalize the chain. - SDValue Ptr = LegalizeOp(LD->getBasePtr()); // Legalize the pointer. - ISD::LoadExtType ExtType = LD->getExtensionType(); - const Value *SV = LD->getSrcValue(); - int SVOffset = LD->getSrcValueOffset(); - MVT MemoryVT = LD->getMemoryVT(); - unsigned Alignment = LD->getAlignment(); - bool isVolatile = LD->isVolatile(); - - assert(LD->isUnindexed() && "Indexed vector loads are not supported yet!"); - SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); - - Result = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, - NewVT, Ch, Ptr, Offset, SV, SVOffset, - MemoryVT.getVectorElementType(), - isVolatile, Alignment); - - // Remember that we legalized the chain. - AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1))); - break; - } - case ISD::BUILD_VECTOR: - Result = Node->getOperand(0); - break; - case ISD::INSERT_VECTOR_ELT: - // Returning the inserted scalar element. - Result = Node->getOperand(1); - break; - case ISD::CONCAT_VECTORS: - assert(Node->getOperand(0).getValueType() == NewVT && - "Concat of non-legal vectors not yet supported!"); - Result = Node->getOperand(0); - break; - case ISD::VECTOR_SHUFFLE: { - // Figure out if the scalar is the LHS or RHS and return it. - SDValue EltNum = Node->getOperand(2).getOperand(0); - if (cast(EltNum)->getZExtValue()) - Result = ScalarizeVectorOp(Node->getOperand(1)); - else - Result = ScalarizeVectorOp(Node->getOperand(0)); - break; - } - case ISD::EXTRACT_SUBVECTOR: - Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, - Node->getOperand(0), Node->getOperand(1)); - break; - case ISD::BIT_CONVERT: { - SDValue Op0 = Op.getOperand(0); - if (Op0.getValueType().isVector() && - Op0.getValueType().getVectorNumElements() == 1) - Op0 = ScalarizeVectorOp(Op0); - Result = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, Op0); - break; - } - case ISD::SELECT: - Result = DAG.getNode(ISD::SELECT, dl, NewVT, Op.getOperand(0), - ScalarizeVectorOp(Op.getOperand(1)), - ScalarizeVectorOp(Op.getOperand(2))); - break; - case ISD::SELECT_CC: - Result = DAG.getNode(ISD::SELECT_CC, dl, NewVT, Node->getOperand(0), - Node->getOperand(1), - ScalarizeVectorOp(Op.getOperand(2)), - ScalarizeVectorOp(Op.getOperand(3)), - Node->getOperand(4)); - break; - case ISD::VSETCC: { - SDValue Op0 = ScalarizeVectorOp(Op.getOperand(0)); - SDValue Op1 = ScalarizeVectorOp(Op.getOperand(1)); - Result = DAG.getNode(ISD::SETCC, dl, - TLI.getSetCCResultType(Op0.getValueType()), - Op0, Op1, Op.getOperand(2)); - Result = DAG.getNode(ISD::SELECT, dl, NewVT, Result, - DAG.getConstant(-1ULL, NewVT), - DAG.getConstant(0ULL, NewVT)); - break; - } - } - - if (TLI.isTypeLegal(NewVT)) - Result = LegalizeOp(Result); - bool isNew = ScalarizedNodes.insert(std::make_pair(Op, Result)).second; - assert(isNew && "Value already scalarized?"); - isNew = isNew; - return Result; -} - - -SDValue SelectionDAGLegalize::WidenVectorOp(SDValue Op, MVT WidenVT) { - assert(0 && "This should be dead!"); - std::map::iterator I = WidenNodes.find(Op); - if (I != WidenNodes.end()) return I->second; - - MVT VT = Op.getValueType(); - assert(VT.isVector() && "Cannot widen non-vector type!"); - - SDValue Result; - SDNode *Node = Op.getNode(); - DebugLoc dl = Node->getDebugLoc(); - MVT EVT = VT.getVectorElementType(); - - unsigned NumElts = VT.getVectorNumElements(); - unsigned NewNumElts = WidenVT.getVectorNumElements(); - assert(NewNumElts > NumElts && "Cannot widen to smaller type!"); - assert(NewNumElts < 17); - - // When widen is called, it is assumed that it is more efficient to use a - // wide type. The default action is to widen to operation to a wider legal - // vector type and then do the operation if it is legal by calling LegalizeOp - // again. If there is no vector equivalent, we will unroll the operation, do - // it, and rebuild the vector. If most of the operations are vectorizible to - // the legal type, the resulting code will be more efficient. If this is not - // the case, the resulting code will preform badly as we end up generating - // code to pack/unpack the results. It is the function that calls widen - // that is responsible for seeing this doesn't happen. - switch (Node->getOpcode()) { - default: -#ifndef NDEBUG - Node->dump(&DAG); -#endif - assert(0 && "Unexpected operation in WidenVectorOp!"); - break; - case ISD::CopyFromReg: - assert(0 && "CopyFromReg doesn't need widening!"); - case ISD::Constant: - case ISD::ConstantFP: - // To build a vector of these elements, clients should call BuildVector - // and with each element instead of creating a node with a vector type - assert(0 && "Unexpected operation in WidenVectorOp!"); - case ISD::VAARG: - // Variable Arguments with vector types doesn't make any sense to me - assert(0 && "Unexpected operation in WidenVectorOp!"); - break; - case ISD::UNDEF: - Result = DAG.getUNDEF(WidenVT); - break; - case ISD::BUILD_VECTOR: { - // Build a vector with undefined for the new nodes - SDValueVector NewOps(Node->op_begin(), Node->op_end()); - for (unsigned i = NumElts; i < NewNumElts; ++i) { - NewOps.push_back(DAG.getUNDEF(EVT)); - } - Result = DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, - &NewOps[0], NewOps.size()); - break; - } - case ISD::INSERT_VECTOR_ELT: { - SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); - Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WidenVT, Tmp1, - Node->getOperand(1), Node->getOperand(2)); - break; - } - case ISD::VECTOR_SHUFFLE: { - SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); - SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT); - ShuffleVectorSDNode *SVOp = cast(Node); - SmallVector NewMask; - for (unsigned i = 0; i < NumElts; ++i) { - int Idx = SVOp->getMaskElt(i); - if (Idx < (int)NumElts) - NewMask.push_back(Idx); - else - NewMask.push_back(Idx + NewNumElts - NumElts); - } - for (unsigned i = NumElts; i < NewNumElts; ++i) - NewMask.push_back(-1); - - Result = DAG.getVectorShuffle(WidenVT, dl, Tmp1, Tmp2, &NewMask[0]); - break; - } - case ISD::LOAD: { - // If the load widen returns true, we can use a single load for the - // vector. Otherwise, it is returning a token factor for multiple - // loads. - SDValue TFOp; - if (LoadWidenVectorOp(Result, TFOp, Op, WidenVT)) - AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(1))); - else - AddLegalizedOperand(Op.getValue(1), LegalizeOp(TFOp.getValue(0))); - break; - } - - case ISD::BIT_CONVERT: { - SDValue Tmp1 = Node->getOperand(0); - // Converts between two different types so we need to determine - // the correct widen type for the input operand. - MVT InVT = Tmp1.getValueType(); - unsigned WidenSize = WidenVT.getSizeInBits(); - if (InVT.isVector()) { - MVT InEltVT = InVT.getVectorElementType(); - unsigned InEltSize = InEltVT.getSizeInBits(); - assert(WidenSize % InEltSize == 0 && - "can not widen bit convert that are not multiple of element type"); - MVT NewInWidenVT = MVT::getVectorVT(InEltVT, WidenSize / InEltSize); - Tmp1 = WidenVectorOp(Tmp1, NewInWidenVT); - assert(Tmp1.getValueType().getSizeInBits() == WidenVT.getSizeInBits()); - Result = DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, Tmp1); - } else { - // If the result size is a multiple of the input size, widen the input - // and then convert. - unsigned InSize = InVT.getSizeInBits(); - assert(WidenSize % InSize == 0 && - "can not widen bit convert that are not multiple of element type"); - unsigned NewNumElts = WidenSize / InSize; - SmallVector Ops(NewNumElts); - SDValue UndefVal = DAG.getUNDEF(InVT); - Ops[0] = Tmp1; - for (unsigned i = 1; i < NewNumElts; ++i) - Ops[i] = UndefVal; - - MVT NewInVT = MVT::getVectorVT(InVT, NewNumElts); - Result = DAG.getNode(ISD::BUILD_VECTOR, dl, NewInVT, &Ops[0], NewNumElts); - Result = DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, Result); - } - break; - } - - case ISD::SINT_TO_FP: - case ISD::UINT_TO_FP: - case ISD::FP_TO_SINT: - case ISD::FP_TO_UINT: - case ISD::FP_ROUND: { - SDValue Tmp1 = Node->getOperand(0); - // Converts between two different types so we need to determine - // the correct widen type for the input operand. - MVT TVT = Tmp1.getValueType(); - assert(TVT.isVector() && "can not widen non vector type"); - MVT TEVT = TVT.getVectorElementType(); - MVT TWidenVT = MVT::getVectorVT(TEVT, NewNumElts); - Tmp1 = WidenVectorOp(Tmp1, TWidenVT); - assert(Tmp1.getValueType().getVectorNumElements() == NewNumElts); - Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Tmp1); - break; - } - - case ISD::FP_EXTEND: - assert(0 && "Case not implemented. Dynamically dead with 2 FP types!"); - case ISD::TRUNCATE: - case ISD::SIGN_EXTEND: - case ISD::ZERO_EXTEND: - case ISD::ANY_EXTEND: - case ISD::SIGN_EXTEND_INREG: - case ISD::FABS: - case ISD::FNEG: - case ISD::FSQRT: - case ISD::FSIN: - case ISD::FCOS: - case ISD::CTPOP: - case ISD::CTTZ: - case ISD::CTLZ: { - // Unary op widening - SDValue Tmp1; - Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); - assert(Tmp1.getValueType() == WidenVT); - Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Tmp1); - break; - } - case ISD::CONVERT_RNDSAT: { - SDValue RndOp = Node->getOperand(3); - SDValue SatOp = Node->getOperand(4); - SDValue SrcOp = Node->getOperand(0); - - // Converts between two different types so we need to determine - // the correct widen type for the input operand. - MVT SVT = SrcOp.getValueType(); - assert(SVT.isVector() && "can not widen non vector type"); - MVT SEVT = SVT.getVectorElementType(); - MVT SWidenVT = MVT::getVectorVT(SEVT, NewNumElts); - - SrcOp = WidenVectorOp(SrcOp, SWidenVT); - assert(SrcOp.getValueType() == WidenVT); - SDValue DTyOp = DAG.getValueType(WidenVT); - SDValue STyOp = DAG.getValueType(SrcOp.getValueType()); - ISD::CvtCode CvtCode = cast(Node)->getCvtCode(); - - Result = DAG.getConvertRndSat(WidenVT, dl, SrcOp, DTyOp, STyOp, - RndOp, SatOp, CvtCode); - break; - } - case ISD::FPOW: - case ISD::FPOWI: - case ISD::ADD: - case ISD::SUB: - case ISD::MUL: - case ISD::MULHS: - case ISD::MULHU: - case ISD::AND: - case ISD::OR: - case ISD::XOR: - case ISD::FADD: - case ISD::FSUB: - case ISD::FMUL: - case ISD::SDIV: - case ISD::SREM: - case ISD::FDIV: - case ISD::FREM: - case ISD::FCOPYSIGN: - case ISD::UDIV: - case ISD::UREM: - case ISD::BSWAP: { - // Binary op widening - SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); - SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), WidenVT); - assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT); - Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Tmp1, Tmp2); - break; - } - - case ISD::SHL: - case ISD::SRA: - case ISD::SRL: { - SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); - assert(Tmp1.getValueType() == WidenVT); - SDValue ShOp = Node->getOperand(1); - MVT ShVT = ShOp.getValueType(); - MVT NewShVT = MVT::getVectorVT(ShVT.getVectorElementType(), - WidenVT.getVectorNumElements()); - ShOp = WidenVectorOp(ShOp, NewShVT); - assert(ShOp.getValueType() == NewShVT); - Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Tmp1, ShOp); - break; - } - - case ISD::EXTRACT_VECTOR_ELT: { - SDValue Tmp1 = WidenVectorOp(Node->getOperand(0), WidenVT); - assert(Tmp1.getValueType() == WidenVT); - Result = DAG.getNode(Node->getOpcode(), dl, EVT, Tmp1, Node->getOperand(1)); - break; - } - case ISD::CONCAT_VECTORS: { - // We concurrently support only widen on a multiple of the incoming vector. - // We could widen on a multiple of the incoming operand if necessary. - unsigned NumConcat = NewNumElts / NumElts; - assert(NewNumElts % NumElts == 0 && "Can widen only a multiple of vector"); - SDValue UndefVal = DAG.getUNDEF(VT); - SmallVector MOps; - MOps.push_back(Op); - for (unsigned i = 1; i != NumConcat; ++i) { - MOps.push_back(UndefVal); - } - Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, - &MOps[0], MOps.size())); - break; - } - case ISD::EXTRACT_SUBVECTOR: { - SDValue Tmp1 = Node->getOperand(0); - SDValue Idx = Node->getOperand(1); - ConstantSDNode *CIdx = dyn_cast(Idx); - if (CIdx && CIdx->getZExtValue() == 0) { - // Since we are access the start of the vector, the incoming - // vector type might be the proper. - MVT Tmp1VT = Tmp1.getValueType(); - if (Tmp1VT == WidenVT) - return Tmp1; - else { - unsigned Tmp1VTNumElts = Tmp1VT.getVectorNumElements(); - if (Tmp1VTNumElts < NewNumElts) - Result = WidenVectorOp(Tmp1, WidenVT); - else - Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, Tmp1, Idx); - } - } else if (NewNumElts % NumElts == 0) { - // Widen the extracted subvector. - unsigned NumConcat = NewNumElts / NumElts; - SDValue UndefVal = DAG.getUNDEF(VT); - SmallVector MOps; - MOps.push_back(Op); - for (unsigned i = 1; i != NumConcat; ++i) { - MOps.push_back(UndefVal); - } - Result = LegalizeOp(DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, - &MOps[0], MOps.size())); - } else { - assert(0 && "can not widen extract subvector"); - // This could be implemented using insert and build vector but I would - // like to see when this happens. - } - break; - } - - case ISD::SELECT: { - // Determine new condition widen type and widen - SDValue Cond1 = Node->getOperand(0); - MVT CondVT = Cond1.getValueType(); - assert(CondVT.isVector() && "can not widen non vector type"); - MVT CondEVT = CondVT.getVectorElementType(); - MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts); - Cond1 = WidenVectorOp(Cond1, CondWidenVT); - assert(Cond1.getValueType() == CondWidenVT && "Condition not widen"); - - SDValue Tmp1 = WidenVectorOp(Node->getOperand(1), WidenVT); - SDValue Tmp2 = WidenVectorOp(Node->getOperand(2), WidenVT); - assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT); - Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Cond1, Tmp1, Tmp2); - break; - } - - case ISD::SELECT_CC: { - // Determine new condition widen type and widen - SDValue Cond1 = Node->getOperand(0); - SDValue Cond2 = Node->getOperand(1); - MVT CondVT = Cond1.getValueType(); - assert(CondVT.isVector() && "can not widen non vector type"); - assert(CondVT == Cond2.getValueType() && "mismatch lhs/rhs"); - MVT CondEVT = CondVT.getVectorElementType(); - MVT CondWidenVT = MVT::getVectorVT(CondEVT, NewNumElts); - Cond1 = WidenVectorOp(Cond1, CondWidenVT); - Cond2 = WidenVectorOp(Cond2, CondWidenVT); - assert(Cond1.getValueType() == CondWidenVT && - Cond2.getValueType() == CondWidenVT && "condition not widen"); - - SDValue Tmp1 = WidenVectorOp(Node->getOperand(2), WidenVT); - SDValue Tmp2 = WidenVectorOp(Node->getOperand(3), WidenVT); - assert(Tmp1.getValueType() == WidenVT && Tmp2.getValueType() == WidenVT && - "operands not widen"); - Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Cond1, Cond2, Tmp1, - Tmp2, Node->getOperand(4)); - break; - } - case ISD::VSETCC: { - // Determine widen for the operand - SDValue Tmp1 = Node->getOperand(0); - MVT TmpVT = Tmp1.getValueType(); - assert(TmpVT.isVector() && "can not widen non vector type"); - MVT TmpEVT = TmpVT.getVectorElementType(); - MVT TmpWidenVT = MVT::getVectorVT(TmpEVT, NewNumElts); - Tmp1 = WidenVectorOp(Tmp1, TmpWidenVT); - SDValue Tmp2 = WidenVectorOp(Node->getOperand(1), TmpWidenVT); - Result = DAG.getNode(Node->getOpcode(), dl, WidenVT, Tmp1, Tmp2, - Node->getOperand(2)); - break; - } - case ISD::ATOMIC_CMP_SWAP: - case ISD::ATOMIC_LOAD_ADD: - case ISD::ATOMIC_LOAD_SUB: - case ISD::ATOMIC_LOAD_AND: - case ISD::ATOMIC_LOAD_OR: - case ISD::ATOMIC_LOAD_XOR: - case ISD::ATOMIC_LOAD_NAND: - case ISD::ATOMIC_LOAD_MIN: - case ISD::ATOMIC_LOAD_MAX: - case ISD::ATOMIC_LOAD_UMIN: - case ISD::ATOMIC_LOAD_UMAX: - case ISD::ATOMIC_SWAP: { - // For now, we assume that using vectors for these operations don't make - // much sense so we just split it. We return an empty result - SDValue X, Y; - SplitVectorOp(Op, X, Y); - return Result; - break; - } - - } // end switch (Node->getOpcode()) - - assert(Result.getNode() && "Didn't set a result!"); - if (Result != Op) - Result = LegalizeOp(Result); - - AddWidenedOperand(Op, Result); - return Result; -} - -// Utility function to find a legal vector type and its associated element -// type from a preferred width and whose vector type must be the same size -// as the VVT. -// TLI: Target lowering used to determine legal types -// Width: Preferred width of element type -// VVT: Vector value type whose size we must match. -// Returns VecEVT and EVT - the vector type and its associated element type -static void FindWidenVecType(const TargetLowering &TLI, unsigned Width, MVT VVT, - MVT& EVT, MVT& VecEVT) { - // We start with the preferred width, make it a power of 2 and see if - // we can find a vector type of that width. If not, we reduce it by - // another power of 2. If we have widen the type, a vector of bytes should - // always be legal. - assert(TLI.isTypeLegal(VVT)); - unsigned EWidth = Width + 1; - do { - assert(EWidth > 0); - EWidth = (1 << Log2_32(EWidth-1)); - EVT = MVT::getIntegerVT(EWidth); - unsigned NumEVT = VVT.getSizeInBits()/EWidth; - VecEVT = MVT::getVectorVT(EVT, NumEVT); - } while (!TLI.isTypeLegal(VecEVT) || - VVT.getSizeInBits() != VecEVT.getSizeInBits()); -} - -SDValue SelectionDAGLegalize::genWidenVectorLoads(SDValueVector& LdChain, - SDValue Chain, - SDValue BasePtr, - const Value *SV, - int SVOffset, - unsigned Alignment, - bool isVolatile, - unsigned LdWidth, - MVT ResType, - DebugLoc dl) { - // We assume that we have good rules to handle loading power of two loads so - // we break down the operations to power of 2 loads. The strategy is to - // load the largest power of 2 that we can easily transform to a legal vector - // and then insert into that vector, and the cast the result into the legal - // vector that we want. This avoids unnecessary stack converts. - // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and - // the load is nonvolatile, we an use a wider load for the value. - // Find a vector length we can load a large chunk - MVT EVT, VecEVT; - unsigned EVTWidth; - FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT); - EVTWidth = EVT.getSizeInBits(); - - SDValue LdOp = DAG.getLoad(EVT, dl, Chain, BasePtr, SV, SVOffset, - isVolatile, Alignment); - SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecEVT, LdOp); - LdChain.push_back(LdOp.getValue(1)); - - // Check if we can load the element with one instruction - if (LdWidth == EVTWidth) { - return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); - } - - // The vector element order is endianness dependent. - unsigned Idx = 1; - LdWidth -= EVTWidth; - unsigned Offset = 0; - - while (LdWidth > 0) { - unsigned Increment = EVTWidth / 8; - Offset += Increment; - BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, - DAG.getIntPtrConstant(Increment)); - - if (LdWidth < EVTWidth) { - // Our current type we are using is too large, use a smaller size by - // using a smaller power of 2 - unsigned oEVTWidth = EVTWidth; - FindWidenVecType(TLI, LdWidth, ResType, EVT, VecEVT); - EVTWidth = EVT.getSizeInBits(); - // Readjust position and vector position based on new load type - Idx = Idx * (oEVTWidth/EVTWidth); - VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, VecEVT, VecOp); - } - - SDValue LdOp = DAG.getLoad(EVT, dl, Chain, BasePtr, SV, - SVOffset+Offset, isVolatile, - MinAlign(Alignment, Offset)); - LdChain.push_back(LdOp.getValue(1)); - VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecEVT, VecOp, LdOp, - DAG.getIntPtrConstant(Idx++)); - - LdWidth -= EVTWidth; - } - - return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp); -} - -bool SelectionDAGLegalize::LoadWidenVectorOp(SDValue& Result, - SDValue& TFOp, - SDValue Op, - MVT NVT) { - // TODO: Add support for ConcatVec and the ability to load many vector - // types (e.g., v4i8). This will not work when a vector register - // to memory mapping is strange (e.g., vector elements are not - // stored in some sequential order). - - // It must be true that the widen vector type is bigger than where - // we need to load from. - LoadSDNode *LD = cast(Op.getNode()); - MVT LdVT = LD->getMemoryVT(); - DebugLoc dl = LD->getDebugLoc(); - assert(LdVT.isVector() && NVT.isVector()); - assert(LdVT.getVectorElementType() == NVT.getVectorElementType()); - - // Load information - SDValue Chain = LD->getChain(); - SDValue BasePtr = LD->getBasePtr(); - int SVOffset = LD->getSrcValueOffset(); - unsigned Alignment = LD->getAlignment(); - bool isVolatile = LD->isVolatile(); - const Value *SV = LD->getSrcValue(); - unsigned int LdWidth = LdVT.getSizeInBits(); - - // Load value as a large register - SDValueVector LdChain; - Result = genWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset, - Alignment, isVolatile, LdWidth, NVT, dl); - - if (LdChain.size() == 1) { - TFOp = LdChain[0]; - return true; - } - else { - TFOp=DAG.getNode(ISD::TokenFactor, dl, MVT::Other, - &LdChain[0], LdChain.size()); - return false; - } -} - - -void SelectionDAGLegalize::genWidenVectorStores(SDValueVector& StChain, - SDValue Chain, - SDValue BasePtr, - const Value *SV, - int SVOffset, - unsigned Alignment, - bool isVolatile, - SDValue ValOp, - unsigned StWidth, - DebugLoc dl) { - // Breaks the stores into a series of power of 2 width stores. For any - // width, we convert the vector to the vector of element size that we - // want to store. This avoids requiring a stack convert. - - // Find a width of the element type we can store with - MVT VVT = ValOp.getValueType(); - MVT EVT, VecEVT; - unsigned EVTWidth; - FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT); - EVTWidth = EVT.getSizeInBits(); - - SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, VecEVT, ValOp); - SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EVT, VecOp, - DAG.getIntPtrConstant(0)); - SDValue StOp = DAG.getStore(Chain, dl, EOp, BasePtr, SV, SVOffset, - isVolatile, Alignment); - StChain.push_back(StOp); - - // Check if we are done - if (StWidth == EVTWidth) { - return; - } - - unsigned Idx = 1; - StWidth -= EVTWidth; - unsigned Offset = 0; - - while (StWidth > 0) { - unsigned Increment = EVTWidth / 8; - Offset += Increment; - BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, - DAG.getIntPtrConstant(Increment)); - - if (StWidth < EVTWidth) { - // Our current type we are using is too large, use a smaller size by - // using a smaller power of 2 - unsigned oEVTWidth = EVTWidth; - FindWidenVecType(TLI, StWidth, VVT, EVT, VecEVT); - EVTWidth = EVT.getSizeInBits(); - // Readjust position and vector position based on new load type - Idx = Idx * (oEVTWidth/EVTWidth); - VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, VecEVT, VecOp); - } - - EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EVT, VecOp, - DAG.getIntPtrConstant(Idx++)); - StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV, - SVOffset + Offset, isVolatile, - MinAlign(Alignment, Offset))); - StWidth -= EVTWidth; - } -} - - -SDValue SelectionDAGLegalize::StoreWidenVectorOp(StoreSDNode *ST, - SDValue Chain, - SDValue BasePtr) { - // TODO: It might be cleaner if we can use SplitVector and have more legal - // vector types that can be stored into memory (e.g., v4xi8 can - // be stored as a word). This will not work when a vector register - // to memory mapping is strange (e.g., vector elements are not - // stored in some sequential order). - - MVT StVT = ST->getMemoryVT(); - SDValue ValOp = ST->getValue(); - DebugLoc dl = ST->getDebugLoc(); - - // Check if we have widen this node with another value - std::map::iterator I = WidenNodes.find(ValOp); - if (I != WidenNodes.end()) - ValOp = I->second; - - MVT VVT = ValOp.getValueType(); - - // It must be true that we the widen vector type is bigger than where - // we need to store. - assert(StVT.isVector() && VVT.isVector()); - assert(StVT.bitsLT(VVT)); - assert(StVT.getVectorElementType() == VVT.getVectorElementType()); - - // Store value - SDValueVector StChain; - genWidenVectorStores(StChain, Chain, BasePtr, ST->getSrcValue(), - ST->getSrcValueOffset(), ST->getAlignment(), - ST->isVolatile(), ValOp, StVT.getSizeInBits(), dl); - if (StChain.size() == 1) - return StChain[0]; - else - return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, - &StChain[0], StChain.size()); -} - - // SelectionDAG::Legalize - This is the entry point for the file. // void SelectionDAG::Legalize(bool TypesNeedLegalizing,