llvm-6502/lib/Transforms/Utils/IntegerDivision.cpp
Chandler Carruth d04a8d4b33 Use the new script to sort the includes of every file under lib.
Sooooo many of these had incorrect or strange main module includes.
I have manually inspected all of these, and fixed the main module
include to be the nearest plausible thing I could find. If you own or
care about any of these source files, I encourage you to take some time
and check that these edits were sensible. I can't have broken anything
(I strictly added headers, and reordered them, never removed), but they
may not be the headers you'd really like to identify as containing the
API being implemented.

Many forward declarations and missing includes were added to a header
files to allow them to parse cleanly when included first. The main
module rule does in fact have its merits. =]

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@169131 91177308-0d34-0410-b5e6-96231b3b80d8
2012-12-03 16:50:05 +00:00

421 lines
18 KiB
C++

//===-- IntegerDivision.cpp - Expand integer division ---------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file contains an implementation of 32bit scalar integer division for
// targets that don't have native support. It's largely derived from
// compiler-rt's implementation of __udivsi3, but hand-tuned to reduce the
// amount of control flow
//
//===----------------------------------------------------------------------===//
#define DEBUG_TYPE "integer-division"
#include "llvm/Transforms/Utils/IntegerDivision.h"
#include "llvm/Function.h"
#include "llvm/IRBuilder.h"
#include "llvm/Instructions.h"
#include "llvm/Intrinsics.h"
using namespace llvm;
/// Generate code to compute the remainder of two signed integers. Returns the
/// remainder, which will have the sign of the dividend. Builder's insert point
/// should be pointing where the caller wants code generated, e.g. at the srem
/// instruction. This will generate a urem in the process, and Builder's insert
/// point will be pointing at the uren (if present, i.e. not folded), ready to
/// be expanded if the user wishes
static Value *generateSignedRemainderCode(Value *Dividend, Value *Divisor,
IRBuilder<> &Builder) {
ConstantInt *ThirtyOne = Builder.getInt32(31);
// ; %dividend_sgn = ashr i32 %dividend, 31
// ; %divisor_sgn = ashr i32 %divisor, 31
// ; %dvd_xor = xor i32 %dividend, %dividend_sgn
// ; %dvs_xor = xor i32 %divisor, %divisor_sgn
// ; %u_dividend = sub i32 %dvd_xor, %dividend_sgn
// ; %u_divisor = sub i32 %dvs_xor, %divisor_sgn
// ; %urem = urem i32 %dividend, %divisor
// ; %xored = xor i32 %urem, %dividend_sgn
// ; %srem = sub i32 %xored, %dividend_sgn
Value *DividendSign = Builder.CreateAShr(Dividend, ThirtyOne);
Value *DivisorSign = Builder.CreateAShr(Divisor, ThirtyOne);
Value *DvdXor = Builder.CreateXor(Dividend, DividendSign);
Value *DvsXor = Builder.CreateXor(Divisor, DivisorSign);
Value *UDividend = Builder.CreateSub(DvdXor, DividendSign);
Value *UDivisor = Builder.CreateSub(DvsXor, DivisorSign);
Value *URem = Builder.CreateURem(UDividend, UDivisor);
Value *Xored = Builder.CreateXor(URem, DividendSign);
Value *SRem = Builder.CreateSub(Xored, DividendSign);
if (Instruction *URemInst = dyn_cast<Instruction>(URem))
Builder.SetInsertPoint(URemInst);
return SRem;
}
/// Generate code to compute the remainder of two unsigned integers. Returns the
/// remainder. Builder's insert point should be pointing where the caller wants
/// code generated, e.g. at the urem instruction. This will generate a udiv in
/// the process, and Builder's insert point will be pointing at the udiv (if
/// present, i.e. not folded), ready to be expanded if the user wishes
static Value *generatedUnsignedRemainderCode(Value *Dividend, Value *Divisor,
IRBuilder<> &Builder) {
// Remainder = Dividend - Quotient*Divisor
// ; %quotient = udiv i32 %dividend, %divisor
// ; %product = mul i32 %divisor, %quotient
// ; %remainder = sub i32 %dividend, %product
Value *Quotient = Builder.CreateUDiv(Dividend, Divisor);
Value *Product = Builder.CreateMul(Divisor, Quotient);
Value *Remainder = Builder.CreateSub(Dividend, Product);
if (Instruction *UDiv = dyn_cast<Instruction>(Quotient))
Builder.SetInsertPoint(UDiv);
return Remainder;
}
/// Generate code to divide two signed integers. Returns the quotient, rounded
/// towards 0. Builder's insert point should be pointing where the caller wants
/// code generated, e.g. at the sdiv instruction. This will generate a udiv in
/// the process, and Builder's insert point will be pointing at the udiv (if
/// present, i.e. not folded), ready to be expanded if the user wishes.
static Value *generateSignedDivisionCode(Value *Dividend, Value *Divisor,
IRBuilder<> &Builder) {
// Implementation taken from compiler-rt's __divsi3
ConstantInt *ThirtyOne = Builder.getInt32(31);
// ; %tmp = ashr i32 %dividend, 31
// ; %tmp1 = ashr i32 %divisor, 31
// ; %tmp2 = xor i32 %tmp, %dividend
// ; %u_dvnd = sub nsw i32 %tmp2, %tmp
// ; %tmp3 = xor i32 %tmp1, %divisor
// ; %u_dvsr = sub nsw i32 %tmp3, %tmp1
// ; %q_sgn = xor i32 %tmp1, %tmp
// ; %q_mag = udiv i32 %u_dvnd, %u_dvsr
// ; %tmp4 = xor i32 %q_mag, %q_sgn
// ; %q = sub i32 %tmp4, %q_sgn
Value *Tmp = Builder.CreateAShr(Dividend, ThirtyOne);
Value *Tmp1 = Builder.CreateAShr(Divisor, ThirtyOne);
Value *Tmp2 = Builder.CreateXor(Tmp, Dividend);
Value *U_Dvnd = Builder.CreateSub(Tmp2, Tmp);
Value *Tmp3 = Builder.CreateXor(Tmp1, Divisor);
Value *U_Dvsr = Builder.CreateSub(Tmp3, Tmp1);
Value *Q_Sgn = Builder.CreateXor(Tmp1, Tmp);
Value *Q_Mag = Builder.CreateUDiv(U_Dvnd, U_Dvsr);
Value *Tmp4 = Builder.CreateXor(Q_Mag, Q_Sgn);
Value *Q = Builder.CreateSub(Tmp4, Q_Sgn);
if (Instruction *UDiv = dyn_cast<Instruction>(Q_Mag))
Builder.SetInsertPoint(UDiv);
return Q;
}
/// Generates code to divide two unsigned scalar 32-bit integers. Returns the
/// quotient, rounded towards 0. Builder's insert point should be pointing where
/// the caller wants code generated, e.g. at the udiv instruction.
static Value *generateUnsignedDivisionCode(Value *Dividend, Value *Divisor,
IRBuilder<> &Builder) {
// The basic algorithm can be found in the compiler-rt project's
// implementation of __udivsi3.c. Here, we do a lower-level IR based approach
// that's been hand-tuned to lessen the amount of control flow involved.
// Some helper values
IntegerType *I32Ty = Builder.getInt32Ty();
ConstantInt *Zero = Builder.getInt32(0);
ConstantInt *One = Builder.getInt32(1);
ConstantInt *ThirtyOne = Builder.getInt32(31);
ConstantInt *NegOne = ConstantInt::getSigned(I32Ty, -1);
ConstantInt *True = Builder.getTrue();
BasicBlock *IBB = Builder.GetInsertBlock();
Function *F = IBB->getParent();
Function *CTLZi32 = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
I32Ty);
// Our CFG is going to look like:
// +---------------------+
// | special-cases |
// | ... |
// +---------------------+
// | |
// | +----------+
// | | bb1 |
// | | ... |
// | +----------+
// | | |
// | | +------------+
// | | | preheader |
// | | | ... |
// | | +------------+
// | | |
// | | | +---+
// | | | | |
// | | +------------+ |
// | | | do-while | |
// | | | ... | |
// | | +------------+ |
// | | | | |
// | +-----------+ +---+
// | | loop-exit |
// | | ... |
// | +-----------+
// | |
// +-------+
// | ... |
// | end |
// +-------+
BasicBlock *SpecialCases = Builder.GetInsertBlock();
SpecialCases->setName(Twine(SpecialCases->getName(), "_udiv-special-cases"));
BasicBlock *End = SpecialCases->splitBasicBlock(Builder.GetInsertPoint(),
"udiv-end");
BasicBlock *LoopExit = BasicBlock::Create(Builder.getContext(),
"udiv-loop-exit", F, End);
BasicBlock *DoWhile = BasicBlock::Create(Builder.getContext(),
"udiv-do-while", F, End);
BasicBlock *Preheader = BasicBlock::Create(Builder.getContext(),
"udiv-preheader", F, End);
BasicBlock *BB1 = BasicBlock::Create(Builder.getContext(),
"udiv-bb1", F, End);
// We'll be overwriting the terminator to insert our extra blocks
SpecialCases->getTerminator()->eraseFromParent();
// First off, check for special cases: dividend or divisor is zero, divisor
// is greater than dividend, and divisor is 1.
// ; special-cases:
// ; %ret0_1 = icmp eq i32 %divisor, 0
// ; %ret0_2 = icmp eq i32 %dividend, 0
// ; %ret0_3 = or i1 %ret0_1, %ret0_2
// ; %tmp0 = tail call i32 @llvm.ctlz.i32(i32 %divisor, i1 true)
// ; %tmp1 = tail call i32 @llvm.ctlz.i32(i32 %dividend, i1 true)
// ; %sr = sub nsw i32 %tmp0, %tmp1
// ; %ret0_4 = icmp ugt i32 %sr, 31
// ; %ret0 = or i1 %ret0_3, %ret0_4
// ; %retDividend = icmp eq i32 %sr, 31
// ; %retVal = select i1 %ret0, i32 0, i32 %dividend
// ; %earlyRet = or i1 %ret0, %retDividend
// ; br i1 %earlyRet, label %end, label %bb1
Builder.SetInsertPoint(SpecialCases);
Value *Ret0_1 = Builder.CreateICmpEQ(Divisor, Zero);
Value *Ret0_2 = Builder.CreateICmpEQ(Dividend, Zero);
Value *Ret0_3 = Builder.CreateOr(Ret0_1, Ret0_2);
Value *Tmp0 = Builder.CreateCall2(CTLZi32, Divisor, True);
Value *Tmp1 = Builder.CreateCall2(CTLZi32, Dividend, True);
Value *SR = Builder.CreateSub(Tmp0, Tmp1);
Value *Ret0_4 = Builder.CreateICmpUGT(SR, ThirtyOne);
Value *Ret0 = Builder.CreateOr(Ret0_3, Ret0_4);
Value *RetDividend = Builder.CreateICmpEQ(SR, ThirtyOne);
Value *RetVal = Builder.CreateSelect(Ret0, Zero, Dividend);
Value *EarlyRet = Builder.CreateOr(Ret0, RetDividend);
Builder.CreateCondBr(EarlyRet, End, BB1);
// ; bb1: ; preds = %special-cases
// ; %sr_1 = add i32 %sr, 1
// ; %tmp2 = sub i32 31, %sr
// ; %q = shl i32 %dividend, %tmp2
// ; %skipLoop = icmp eq i32 %sr_1, 0
// ; br i1 %skipLoop, label %loop-exit, label %preheader
Builder.SetInsertPoint(BB1);
Value *SR_1 = Builder.CreateAdd(SR, One);
Value *Tmp2 = Builder.CreateSub(ThirtyOne, SR);
Value *Q = Builder.CreateShl(Dividend, Tmp2);
Value *SkipLoop = Builder.CreateICmpEQ(SR_1, Zero);
Builder.CreateCondBr(SkipLoop, LoopExit, Preheader);
// ; preheader: ; preds = %bb1
// ; %tmp3 = lshr i32 %dividend, %sr_1
// ; %tmp4 = add i32 %divisor, -1
// ; br label %do-while
Builder.SetInsertPoint(Preheader);
Value *Tmp3 = Builder.CreateLShr(Dividend, SR_1);
Value *Tmp4 = Builder.CreateAdd(Divisor, NegOne);
Builder.CreateBr(DoWhile);
// ; do-while: ; preds = %do-while, %preheader
// ; %carry_1 = phi i32 [ 0, %preheader ], [ %carry, %do-while ]
// ; %sr_3 = phi i32 [ %sr_1, %preheader ], [ %sr_2, %do-while ]
// ; %r_1 = phi i32 [ %tmp3, %preheader ], [ %r, %do-while ]
// ; %q_2 = phi i32 [ %q, %preheader ], [ %q_1, %do-while ]
// ; %tmp5 = shl i32 %r_1, 1
// ; %tmp6 = lshr i32 %q_2, 31
// ; %tmp7 = or i32 %tmp5, %tmp6
// ; %tmp8 = shl i32 %q_2, 1
// ; %q_1 = or i32 %carry_1, %tmp8
// ; %tmp9 = sub i32 %tmp4, %tmp7
// ; %tmp10 = ashr i32 %tmp9, 31
// ; %carry = and i32 %tmp10, 1
// ; %tmp11 = and i32 %tmp10, %divisor
// ; %r = sub i32 %tmp7, %tmp11
// ; %sr_2 = add i32 %sr_3, -1
// ; %tmp12 = icmp eq i32 %sr_2, 0
// ; br i1 %tmp12, label %loop-exit, label %do-while
Builder.SetInsertPoint(DoWhile);
PHINode *Carry_1 = Builder.CreatePHI(I32Ty, 2);
PHINode *SR_3 = Builder.CreatePHI(I32Ty, 2);
PHINode *R_1 = Builder.CreatePHI(I32Ty, 2);
PHINode *Q_2 = Builder.CreatePHI(I32Ty, 2);
Value *Tmp5 = Builder.CreateShl(R_1, One);
Value *Tmp6 = Builder.CreateLShr(Q_2, ThirtyOne);
Value *Tmp7 = Builder.CreateOr(Tmp5, Tmp6);
Value *Tmp8 = Builder.CreateShl(Q_2, One);
Value *Q_1 = Builder.CreateOr(Carry_1, Tmp8);
Value *Tmp9 = Builder.CreateSub(Tmp4, Tmp7);
Value *Tmp10 = Builder.CreateAShr(Tmp9, 31);
Value *Carry = Builder.CreateAnd(Tmp10, One);
Value *Tmp11 = Builder.CreateAnd(Tmp10, Divisor);
Value *R = Builder.CreateSub(Tmp7, Tmp11);
Value *SR_2 = Builder.CreateAdd(SR_3, NegOne);
Value *Tmp12 = Builder.CreateICmpEQ(SR_2, Zero);
Builder.CreateCondBr(Tmp12, LoopExit, DoWhile);
// ; loop-exit: ; preds = %do-while, %bb1
// ; %carry_2 = phi i32 [ 0, %bb1 ], [ %carry, %do-while ]
// ; %q_3 = phi i32 [ %q, %bb1 ], [ %q_1, %do-while ]
// ; %tmp13 = shl i32 %q_3, 1
// ; %q_4 = or i32 %carry_2, %tmp13
// ; br label %end
Builder.SetInsertPoint(LoopExit);
PHINode *Carry_2 = Builder.CreatePHI(I32Ty, 2);
PHINode *Q_3 = Builder.CreatePHI(I32Ty, 2);
Value *Tmp13 = Builder.CreateShl(Q_3, One);
Value *Q_4 = Builder.CreateOr(Carry_2, Tmp13);
Builder.CreateBr(End);
// ; end: ; preds = %loop-exit, %special-cases
// ; %q_5 = phi i32 [ %q_4, %loop-exit ], [ %retVal, %special-cases ]
// ; ret i32 %q_5
Builder.SetInsertPoint(End, End->begin());
PHINode *Q_5 = Builder.CreatePHI(I32Ty, 2);
// Populate the Phis, since all values have now been created. Our Phis were:
// ; %carry_1 = phi i32 [ 0, %preheader ], [ %carry, %do-while ]
Carry_1->addIncoming(Zero, Preheader);
Carry_1->addIncoming(Carry, DoWhile);
// ; %sr_3 = phi i32 [ %sr_1, %preheader ], [ %sr_2, %do-while ]
SR_3->addIncoming(SR_1, Preheader);
SR_3->addIncoming(SR_2, DoWhile);
// ; %r_1 = phi i32 [ %tmp3, %preheader ], [ %r, %do-while ]
R_1->addIncoming(Tmp3, Preheader);
R_1->addIncoming(R, DoWhile);
// ; %q_2 = phi i32 [ %q, %preheader ], [ %q_1, %do-while ]
Q_2->addIncoming(Q, Preheader);
Q_2->addIncoming(Q_1, DoWhile);
// ; %carry_2 = phi i32 [ 0, %bb1 ], [ %carry, %do-while ]
Carry_2->addIncoming(Zero, BB1);
Carry_2->addIncoming(Carry, DoWhile);
// ; %q_3 = phi i32 [ %q, %bb1 ], [ %q_1, %do-while ]
Q_3->addIncoming(Q, BB1);
Q_3->addIncoming(Q_1, DoWhile);
// ; %q_5 = phi i32 [ %q_4, %loop-exit ], [ %retVal, %special-cases ]
Q_5->addIncoming(Q_4, LoopExit);
Q_5->addIncoming(RetVal, SpecialCases);
return Q_5;
}
/// Generate code to calculate the remainder of two integers, replacing Rem with
/// the generated code. This currently generates code using the udiv expansion,
/// but future work includes generating more specialized code, e.g. when more
/// information about the operands are known. Currently only implements 32bit
/// scalar division (due to udiv's limitation), but future work is removing this
/// limitation.
///
/// @brief Replace Rem with generated code.
bool llvm::expandRemainder(BinaryOperator *Rem) {
assert((Rem->getOpcode() == Instruction::SRem ||
Rem->getOpcode() == Instruction::URem) &&
"Trying to expand remainder from a non-remainder function");
IRBuilder<> Builder(Rem);
// First prepare the sign if it's a signed remainder
if (Rem->getOpcode() == Instruction::SRem) {
Value *Remainder = generateSignedRemainderCode(Rem->getOperand(0),
Rem->getOperand(1), Builder);
Rem->replaceAllUsesWith(Remainder);
Rem->dropAllReferences();
Rem->eraseFromParent();
// If we didn't actually generate a udiv instruction, we're done
BinaryOperator *BO = dyn_cast<BinaryOperator>(Builder.GetInsertPoint());
if (!BO || BO->getOpcode() != Instruction::URem)
return true;
Rem = BO;
}
Value *Remainder = generatedUnsignedRemainderCode(Rem->getOperand(0),
Rem->getOperand(1),
Builder);
Rem->replaceAllUsesWith(Remainder);
Rem->dropAllReferences();
Rem->eraseFromParent();
// Expand the udiv
if (BinaryOperator *UDiv = dyn_cast<BinaryOperator>(Builder.GetInsertPoint())) {
assert(UDiv->getOpcode() == Instruction::UDiv && "Non-udiv in expansion?");
expandDivision(UDiv);
}
return true;
}
/// Generate code to divide two integers, replacing Div with the generated
/// code. This currently generates code similarly to compiler-rt's
/// implementations, but future work includes generating more specialized code
/// when more information about the operands are known. Currently only
/// implements 32bit scalar division, but future work is removing this
/// limitation.
///
/// @brief Replace Div with generated code.
bool llvm::expandDivision(BinaryOperator *Div) {
assert((Div->getOpcode() == Instruction::SDiv ||
Div->getOpcode() == Instruction::UDiv) &&
"Trying to expand division from a non-division function");
IRBuilder<> Builder(Div);
if (Div->getType()->isVectorTy())
llvm_unreachable("Div over vectors not supported");
// First prepare the sign if it's a signed division
if (Div->getOpcode() == Instruction::SDiv) {
// Lower the code to unsigned division, and reset Div to point to the udiv.
Value *Quotient = generateSignedDivisionCode(Div->getOperand(0),
Div->getOperand(1), Builder);
Div->replaceAllUsesWith(Quotient);
Div->dropAllReferences();
Div->eraseFromParent();
// If we didn't actually generate a udiv instruction, we're done
BinaryOperator *BO = dyn_cast<BinaryOperator>(Builder.GetInsertPoint());
if (!BO || BO->getOpcode() != Instruction::UDiv)
return true;
Div = BO;
}
// Insert the unsigned division code
Value *Quotient = generateUnsignedDivisionCode(Div->getOperand(0),
Div->getOperand(1),
Builder);
Div->replaceAllUsesWith(Quotient);
Div->dropAllReferences();
Div->eraseFromParent();
return true;
}