diff --git a/lib/Transforms/Vectorize/BBVectorize.cpp b/lib/Transforms/Vectorize/BBVectorize.cpp index 32eec7974b0..28eb6345071 100644 --- a/lib/Transforms/Vectorize/BBVectorize.cpp +++ b/lib/Transforms/Vectorize/BBVectorize.cpp @@ -647,6 +647,20 @@ namespace { // FIXME: We may want to vectorize non-constant shuffles also. } + // The powi intrinsic is special because only the first argument is + // vectorized, the second arguments must be equal. + CallInst *CI = dyn_cast<CallInst>(I); + Function *FI; + if (CI && (FI = CI->getCalledFunction()) && + FI->getIntrinsicID() == Intrinsic::powi) { + + Value *A1I = CI->getArgOperand(1), + *A1J = cast<CallInst>(J)->getArgOperand(1); + const SCEV *A1ISCEV = SE->getSCEV(A1I), + *A1JSCEV = SE->getSCEV(A1J); + return (A1ISCEV == A1JSCEV); + } + return true; } @@ -1518,19 +1532,27 @@ namespace { ReplacedOperands[o] = getReplacementPointerInput(Context, I, J, o, FlipMemInputs); continue; - } else if (isa<CallInst>(I) && o == NumOperands-1) { + } else if (isa<CallInst>(I)) { Function *F = cast<CallInst>(I)->getCalledFunction(); unsigned IID = F->getIntrinsicID(); - BasicBlock &BB = *I->getParent(); - - Module *M = BB.getParent()->getParent(); - Type *ArgType = I->getType(); - Type *VArgType = getVecTypeForPair(ArgType); - - // FIXME: is it safe to do this here? - ReplacedOperands[o] = Intrinsic::getDeclaration(M, - (Intrinsic::ID) IID, VArgType); - continue; + if (o == NumOperands-1) { + BasicBlock &BB = *I->getParent(); + + Module *M = BB.getParent()->getParent(); + Type *ArgType = I->getType(); + Type *VArgType = getVecTypeForPair(ArgType); + + // FIXME: is it safe to do this here? + ReplacedOperands[o] = Intrinsic::getDeclaration(M, + (Intrinsic::ID) IID, VArgType); + continue; + } else if (IID == Intrinsic::powi && o == 1) { + // The second argument of powi is a single integer and we've already + // checked that both arguments are equal. As a result, we just keep + // I's second argument. + ReplacedOperands[o] = I->getOperand(o); + continue; + } } else if (isa<ShuffleVectorInst>(I) && o == NumOperands-1) { ReplacedOperands[o] = getReplacementShuffleMask(Context, I, J); continue; diff --git a/test/Transforms/BBVectorize/simple-int.ll b/test/Transforms/BBVectorize/simple-int.ll index b2ef27b2b8e..68449771436 100644 --- a/test/Transforms/BBVectorize/simple-int.ll +++ b/test/Transforms/BBVectorize/simple-int.ll @@ -3,6 +3,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 declare double @llvm.fma.f64(double, double, double) declare double @llvm.cos.f64(double) +declare double @llvm.powi.f64(double, i32) ; Basic depth-3 chain with fma define double @test1(double %A1, double %A2, double %B1, double %B2, double %C1, double %C2) { @@ -54,6 +55,49 @@ define double @test2(double %A1, double %A2, double %B1, double %B2) { ; CHECK: ret double %R } +; Basic depth-3 chain with powi +define double @test3(double %A1, double %A2, double %B1, double %B2, i32 %P) { + + %X1 = fsub double %A1, %B1 + %X2 = fsub double %A2, %B2 + %Y1 = call double @llvm.powi.f64(double %X1, i32 %P) + %Y2 = call double @llvm.powi.f64(double %X2, i32 %P) + %Z1 = fadd double %Y1, %B1 + %Z2 = fadd double %Y2, %B2 + %R = fmul double %Z1, %Z2 + ret double %R +; CHECK: @test3 +; CHECK: %X1.v.i1.1 = insertelement <2 x double> undef, double %B1, i32 0 +; CHECK: %X1.v.i0.1 = insertelement <2 x double> undef, double %A1, i32 0 +; CHECK: %X1.v.i1.2 = insertelement <2 x double> %X1.v.i1.1, double %B2, i32 1 +; CHECK: %X1.v.i0.2 = insertelement <2 x double> %X1.v.i0.1, double %A2, i32 1 +; CHECK: %X1 = fsub <2 x double> %X1.v.i0.2, %X1.v.i1.2 +; CHECK: %Y1 = call <2 x double> @llvm.powi.v2f64(<2 x double> %X1, i32 %P) +; CHECK: %Z1 = fadd <2 x double> %Y1, %X1.v.i1.2 +; CHECK: %Z1.v.r1 = extractelement <2 x double> %Z1, i32 0 +; CHECK: %Z1.v.r2 = extractelement <2 x double> %Z1, i32 1 +; CHECK: %R = fmul double %Z1.v.r1, %Z1.v.r2 +; CHECK: ret double %R +} + +; Basic depth-3 chain with powi (different powers: should not vectorize) +define double @test4(double %A1, double %A2, double %B1, double %B2, i32 %P) { + + %X1 = fsub double %A1, %B1 + %X2 = fsub double %A2, %B2 + %P2 = add i32 %P, 1 + %Y1 = call double @llvm.powi.f64(double %X1, i32 %P) + %Y2 = call double @llvm.powi.f64(double %X2, i32 %P2) + %Z1 = fadd double %Y1, %B1 + %Z2 = fadd double %Y2, %B2 + %R = fmul double %Z1, %Z2 + ret double %R +; CHECK: @test4 +; CHECK-NOT: <2 x double> +; CHECK: ret double %R +} + ; CHECK: declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>) nounwind readnone ; CHECK: declare <2 x double> @llvm.cos.v2f64(<2 x double>) nounwind readonly +; CHECK: declare <2 x double> @llvm.powi.v2f64(<2 x double>, i32) nounwind readonly