diff --git a/lib/Transforms/InstCombine/InstCombine.h b/lib/Transforms/InstCombine/InstCombine.h index a5eddc20613..ca834c10940 100644 --- a/lib/Transforms/InstCombine/InstCombine.h +++ b/lib/Transforms/InstCombine/InstCombine.h @@ -116,7 +116,7 @@ public: Instruction *visitSub(BinaryOperator &I); Instruction *visitFSub(BinaryOperator &I); Instruction *visitMul(BinaryOperator &I); - Value *foldFMulConst(Instruction *FMulOrDiv, ConstantFP *C, + Value *foldFMulConst(Instruction *FMulOrDiv, Constant *C, Instruction *InsertBefore); Instruction *visitFMul(BinaryOperator &I); Instruction *visitURem(BinaryOperator &I); diff --git a/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp index 2fcd003bcbd..e6c04f51dad 100644 --- a/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp +++ b/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp @@ -313,16 +313,41 @@ static void detectLog2OfHalf(Value *&Op, Value *&Y, IntrinsicInst *&Log2) { if (I->getOpcode() != Instruction::FMul || !I->hasUnsafeAlgebra()) return; - ConstantFP *CFP = dyn_cast(I->getOperand(0)); - if (CFP && CFP->isExactlyValue(0.5)) { + if (match(I->getOperand(0), m_SpecificFP(0.5))) Y = I->getOperand(1); - return; - } - CFP = dyn_cast(I->getOperand(1)); - if (CFP && CFP->isExactlyValue(0.5)) + else if (match(I->getOperand(1), m_SpecificFP(0.5))) Y = I->getOperand(0); } +static bool isFiniteNonZeroFp(Constant *C) { + if (C->getType()->isVectorTy()) { + for (unsigned I = 0, E = C->getType()->getVectorNumElements(); I != E; + ++I) { + ConstantFP *CFP = dyn_cast(C->getAggregateElement(I)); + if (!CFP || !CFP->getValueAPF().isFiniteNonZero()) + return false; + } + return true; + } + + return isa(C) && + cast(C)->getValueAPF().isFiniteNonZero(); +} + +static bool isNormalFp(Constant *C) { + if (C->getType()->isVectorTy()) { + for (unsigned I = 0, E = C->getType()->getVectorNumElements(); I != E; + ++I) { + ConstantFP *CFP = dyn_cast(C->getAggregateElement(I)); + if (!CFP || !CFP->getValueAPF().isNormal()) + return false; + } + return true; + } + + return isa(C) && cast(C)->getValueAPF().isNormal(); +} + /// Helper function of InstCombiner::visitFMul(BinaryOperator(). It returns /// true iff the given value is FMul or FDiv with one and only one operand /// being a normal constant (i.e. not Zero/NaN/Infinity). @@ -332,19 +357,13 @@ static bool isFMulOrFDivWithConstant(Value *V) { I->getOpcode() != Instruction::FDiv)) return false; - ConstantFP *C0 = dyn_cast(I->getOperand(0)); - ConstantFP *C1 = dyn_cast(I->getOperand(1)); + Constant *C0 = dyn_cast(I->getOperand(0)); + Constant *C1 = dyn_cast(I->getOperand(1)); if (C0 && C1) return false; - return (C0 && C0->getValueAPF().isFiniteNonZero()) || - (C1 && C1->getValueAPF().isFiniteNonZero()); -} - -static bool isNormalFp(const ConstantFP *C) { - const APFloat &Flt = C->getValueAPF(); - return Flt.isNormal(); + return (C0 && isFiniteNonZeroFp(C0)) || (C1 && isFiniteNonZeroFp(C1)); } /// foldFMulConst() is a helper routine of InstCombiner::visitFMul(). @@ -354,41 +373,41 @@ static bool isNormalFp(const ConstantFP *C) { /// resulting expression. Note that this function could return NULL in /// case the constants cannot be folded into a normal floating-point. /// -Value *InstCombiner::foldFMulConst(Instruction *FMulOrDiv, ConstantFP *C, +Value *InstCombiner::foldFMulConst(Instruction *FMulOrDiv, Constant *C, Instruction *InsertBefore) { assert(isFMulOrFDivWithConstant(FMulOrDiv) && "V is invalid"); Value *Opnd0 = FMulOrDiv->getOperand(0); Value *Opnd1 = FMulOrDiv->getOperand(1); - ConstantFP *C0 = dyn_cast(Opnd0); - ConstantFP *C1 = dyn_cast(Opnd1); + Constant *C0 = dyn_cast(Opnd0); + Constant *C1 = dyn_cast(Opnd1); BinaryOperator *R = 0; // (X * C0) * C => X * (C0*C) if (FMulOrDiv->getOpcode() == Instruction::FMul) { Constant *F = ConstantExpr::getFMul(C1 ? C1 : C0, C); - if (isNormalFp(cast(F))) + if (isNormalFp(F)) R = BinaryOperator::CreateFMul(C1 ? Opnd0 : Opnd1, F); } else { if (C0) { // (C0 / X) * C => (C0 * C) / X if (FMulOrDiv->hasOneUse()) { // It would otherwise introduce another div. - ConstantFP *F = cast(ConstantExpr::getFMul(C0, C)); + Constant *F = ConstantExpr::getFMul(C0, C); if (isNormalFp(F)) R = BinaryOperator::CreateFDiv(F, Opnd1); } } else { // (X / C1) * C => X * (C/C1) if C/C1 is not a denormal - ConstantFP *F = cast(ConstantExpr::getFDiv(C, C1)); + Constant *F = ConstantExpr::getFDiv(C, C1); if (isNormalFp(F)) { R = BinaryOperator::CreateFMul(Opnd0, F); } else { // (X / C1) * C => X / (C1/C) Constant *F = ConstantExpr::getFDiv(C1, C); - if (isNormalFp(cast(F))) + if (isNormalFp(F)) R = BinaryOperator::CreateFDiv(Opnd0, F); } } @@ -433,17 +452,15 @@ Instruction *InstCombiner::visitFMul(BinaryOperator &I) { return RI; } - ConstantFP *C = dyn_cast(Op1); - if (C && AllowReassociate && C->getValueAPF().isFiniteNonZero()) { + Constant *C = cast(Op1); + if (AllowReassociate && isFiniteNonZeroFp(C)) { // Let MDC denote an expression in one of these forms: // X * C, C/X, X/C, where C is a constant. // // Try to simplify "MDC * Constant" - if (isFMulOrFDivWithConstant(Op0)) { - Value *V = foldFMulConst(cast(Op0), C, &I); - if (V) + if (isFMulOrFDivWithConstant(Op0)) + if (Value *V = foldFMulConst(cast(Op0), C, &I)) return ReplaceInstUsesWith(I, V); - } // (MDC +/- C1) * C => (MDC * C) +/- (C1 * C) Instruction *FAddSub = dyn_cast(Op0); @@ -452,8 +469,8 @@ Instruction *InstCombiner::visitFMul(BinaryOperator &I) { FAddSub->getOpcode() == Instruction::FSub)) { Value *Opnd0 = FAddSub->getOperand(0); Value *Opnd1 = FAddSub->getOperand(1); - ConstantFP *C0 = dyn_cast(Opnd0); - ConstantFP *C1 = dyn_cast(Opnd1); + Constant *C0 = dyn_cast(Opnd0); + Constant *C1 = dyn_cast(Opnd1); bool Swap = false; if (C0) { std::swap(C0, C1); @@ -461,10 +478,9 @@ Instruction *InstCombiner::visitFMul(BinaryOperator &I) { Swap = true; } - if (C1 && C1->getValueAPF().isFiniteNonZero() && - isFMulOrFDivWithConstant(Opnd0)) { + if (C1 && isFiniteNonZeroFp(C1) && isFMulOrFDivWithConstant(Opnd0)) { Value *M1 = ConstantExpr::getFMul(C1, C); - Value *M0 = isNormalFp(cast(M1)) ? + Value *M0 = isNormalFp(cast(M1)) ? foldFMulConst(cast(Opnd0), C, &I) : 0; if (M0 && M1) { @@ -575,7 +591,8 @@ Instruction *InstCombiner::visitFMul(BinaryOperator &I) { if (!match(RHS, m_UIToFP(m_Value(C)))) std::swap(LHS, RHS); - if (match(RHS, m_UIToFP(m_Value(C))) && C->getType()->isIntegerTy(1)) { + if (match(RHS, m_UIToFP(m_Value(C))) && + C->getType()->getScalarType()->isIntegerTy(1)) { B = LHS; Value *Zero = ConstantFP::getNegativeZero(B->getType()); return SelectInst::Create(C, B, Zero); @@ -590,7 +607,7 @@ Instruction *InstCombiner::visitFMul(BinaryOperator &I) { std::swap(LHS, RHS); if (match(RHS, m_FSub(m_FPOne(), m_UIToFP(m_Value(C)))) && - C->getType()->isIntegerTy(1)) { + C->getType()->getScalarType()->isIntegerTy(1)) { A = LHS; Value *Zero = ConstantFP::getNegativeZero(A->getType()); return SelectInst::Create(C, Zero, A); @@ -976,9 +993,12 @@ Instruction *InstCombiner::visitSDiv(BinaryOperator &I) { /// returned; otherwise, NULL is returned. /// static Instruction *CvtFDivConstToReciprocal(Value *Dividend, - ConstantFP *Divisor, + Constant *Divisor, bool AllowReciprocal) { - const APFloat &FpVal = Divisor->getValueAPF(); + if (!isa(Divisor)) // TODO: handle vectors. + return 0; + + const APFloat &FpVal = cast(Divisor)->getValueAPF(); APFloat Reciprocal(FpVal.getSemantics()); bool Cvt = FpVal.getExactInverse(&Reciprocal); @@ -1010,32 +1030,29 @@ Instruction *InstCombiner::visitFDiv(BinaryOperator &I) { bool AllowReassociate = I.hasUnsafeAlgebra(); bool AllowReciprocal = I.hasAllowReciprocal(); - if (ConstantFP *Op1C = dyn_cast(Op1)) { + if (Constant *Op1C = dyn_cast(Op1)) { if (SelectInst *SI = dyn_cast(Op0)) if (Instruction *R = FoldOpIntoSelect(I, SI)) return R; if (AllowReassociate) { - ConstantFP *C1 = 0; - ConstantFP *C2 = Op1C; + Constant *C1 = 0; + Constant *C2 = Op1C; Value *X; Instruction *Res = 0; - if (match(Op0, m_FMul(m_Value(X), m_ConstantFP(C1)))) { + if (match(Op0, m_FMul(m_Value(X), m_Constant(C1)))) { // (X*C1)/C2 => X * (C1/C2) // Constant *C = ConstantExpr::getFDiv(C1, C2); - const APFloat &F = cast(C)->getValueAPF(); - if (F.isNormal()) + if (isNormalFp(C)) Res = BinaryOperator::CreateFMul(X, C); - } else if (match(Op0, m_FDiv(m_Value(X), m_ConstantFP(C1)))) { + } else if (match(Op0, m_FDiv(m_Value(X), m_Constant(C1)))) { // (X/C1)/C2 => X /(C2*C1) [=> X * 1/(C2*C1) if reciprocal is allowed] // Constant *C = ConstantExpr::getFMul(C1, C2); - const APFloat &F = cast(C)->getValueAPF(); - if (F.isNormal()) { - Res = CvtFDivConstToReciprocal(X, cast(C), - AllowReciprocal); + if (isNormalFp(C)) { + Res = CvtFDivConstToReciprocal(X, C, AllowReciprocal); if (!Res) Res = BinaryOperator::CreateFDiv(X, C); } @@ -1056,33 +1073,29 @@ Instruction *InstCombiner::visitFDiv(BinaryOperator &I) { return 0; } - if (AllowReassociate && isa(Op0)) { - ConstantFP *C1 = cast(Op0), *C2; + if (AllowReassociate && isa(Op0)) { + Constant *C1 = cast(Op0), *C2; Constant *Fold = 0; Value *X; bool CreateDiv = true; // C1 / (X*C2) => (C1/C2) / X - if (match(Op1, m_FMul(m_Value(X), m_ConstantFP(C2)))) + if (match(Op1, m_FMul(m_Value(X), m_Constant(C2)))) Fold = ConstantExpr::getFDiv(C1, C2); - else if (match(Op1, m_FDiv(m_Value(X), m_ConstantFP(C2)))) { + else if (match(Op1, m_FDiv(m_Value(X), m_Constant(C2)))) { // C1 / (X/C2) => (C1*C2) / X Fold = ConstantExpr::getFMul(C1, C2); - } else if (match(Op1, m_FDiv(m_ConstantFP(C2), m_Value(X)))) { + } else if (match(Op1, m_FDiv(m_Constant(C2), m_Value(X)))) { // C1 / (C2/X) => (C1/C2) * X Fold = ConstantExpr::getFDiv(C1, C2); CreateDiv = false; } - if (Fold) { - const APFloat &FoldC = cast(Fold)->getValueAPF(); - if (FoldC.isNormal()) { - Instruction *R = CreateDiv ? - BinaryOperator::CreateFDiv(Fold, X) : - BinaryOperator::CreateFMul(X, Fold); - R->setFastMathFlags(I.getFastMathFlags()); - return R; - } + if (Fold && isNormalFp(Fold)) { + Instruction *R = CreateDiv ? BinaryOperator::CreateFDiv(Fold, X) + : BinaryOperator::CreateFMul(X, Fold); + R->setFastMathFlags(I.getFastMathFlags()); + return R; } return 0; } @@ -1095,14 +1108,14 @@ Instruction *InstCombiner::visitFDiv(BinaryOperator &I) { if (Op0->hasOneUse() && match(Op0, m_FDiv(m_Value(X), m_Value(Y)))) { // (X/Y) / Z => X / (Y*Z) // - if (!isa(Y) || !isa(Op1)) { + if (!isa(Y) || !isa(Op1)) { NewInst = Builder->CreateFMul(Y, Op1); SimpR = BinaryOperator::CreateFDiv(X, NewInst); } } else if (Op1->hasOneUse() && match(Op1, m_FDiv(m_Value(X), m_Value(Y)))) { // Z / (X/Y) => Z*Y / X // - if (!isa(Y) || !isa(Op0)) { + if (!isa(Y) || !isa(Op0)) { NewInst = Builder->CreateFMul(Op0, Y); SimpR = BinaryOperator::CreateFDiv(NewInst, X); } diff --git a/test/Transforms/InstCombine/add4.ll b/test/Transforms/InstCombine/add4.ll index 208c7f03200..f9b7e3b5a07 100644 --- a/test/Transforms/InstCombine/add4.ll +++ b/test/Transforms/InstCombine/add4.ll @@ -77,3 +77,26 @@ define float @test7(float %A, float %B, i32 %C) { ; CHECK: uitofp } +define <4 x float> @test8(<4 x float> %A, <4 x float> %B, <4 x i1> %C) { + ;; B*(uitofp i1 C) + A*(1 - uitofp i1 C) -> select C, A, B + %cf = uitofp <4 x i1> %C to <4 x float> + %mc = fsub fast <4 x float> , %cf + %p1 = fmul fast <4 x float> %A, %mc + %p2 = fmul fast <4 x float> %B, %cf + %s1 = fadd fast <4 x float> %p2, %p1 + ret <4 x float> %s1 +; CHECK-LABEL: @test8( +; CHECK: select <4 x i1> %C, <4 x float> %B, <4 x float> %A +} + +define <4 x float> @test9(<4 x float> %A, <4 x float> %B, <4 x i1> %C) { + ;; A*(1 - uitofp i1 C) + B*(uitofp i1 C) -> select C, A, B + %cf = uitofp <4 x i1> %C to <4 x float> + %mc = fsub fast <4 x float> , %cf + %p1 = fmul fast <4 x float> %A, %mc + %p2 = fmul fast <4 x float> %B, %cf + %s1 = fadd fast <4 x float> %p1, %p2 + ret <4 x float> %s1 +; CHECK-LABEL: @test9 +; CHECK: select <4 x i1> %C, <4 x float> %B, <4 x float> %A +} diff --git a/test/Transforms/InstCombine/fast-math.ll b/test/Transforms/InstCombine/fast-math.ll index de51c494685..2ee4b0f2c38 100644 --- a/test/Transforms/InstCombine/fast-math.ll +++ b/test/Transforms/InstCombine/fast-math.ll @@ -259,6 +259,14 @@ define float @fmul3(float %f1, float %f2) { ; CHECK: fmul fast float %f1, 3.000000e+00 } +define <4 x float> @fmul3_vec(<4 x float> %f1, <4 x float> %f2) { + %t1 = fdiv <4 x float> %f1, + %t3 = fmul fast <4 x float> %t1, + ret <4 x float> %t3 +; CHECK-LABEL: @fmul3_vec( +; CHECK: fmul fast <4 x float> %f1, +} + ; Rule "X/C1 * C2 => X * (C2/C1) is not applicable if C2/C1 is either a special ; value of a denormal. The 0x3810000000000000 here take value FLT_MIN ; @@ -345,6 +353,15 @@ define float @fdiv2(float %x) { ; CHECK: fmul fast float %x, 0x3FE0B21660000000 } +define <2 x float> @fdiv2_vec(<2 x float> %x) { + %mul = fmul <2 x float> %x, + %div1 = fdiv fast <2 x float> %mul, + ret <2 x float> %div1 + +; CHECK-LABEL: @fdiv2_vec( +; CHECK: fmul fast <2 x float> %x, +} + ; "X/C1 / C2 => X * (1/(C2*C1))" is disabled (for now) is C2/C1 is a denormal ; define float @fdiv3(float %x) {