diff --git a/lib/Target/PowerPC/CMakeLists.txt b/lib/Target/PowerPC/CMakeLists.txt index bcd8bd29162..7204926526b 100644 --- a/lib/Target/PowerPC/CMakeLists.txt +++ b/lib/Target/PowerPC/CMakeLists.txt @@ -14,6 +14,7 @@ add_llvm_target(PowerPCCodeGen PPCAsmPrinter.cpp PPCBranchSelector.cpp PPCCodeEmitter.cpp + PPCCTRLoops.cpp PPCHazardRecognizers.cpp PPCInstrInfo.cpp PPCISelDAGToDAG.cpp diff --git a/lib/Target/PowerPC/PPC.h b/lib/Target/PowerPC/PPC.h index e7e72da11fe..9103e123250 100644 --- a/lib/Target/PowerPC/PPC.h +++ b/lib/Target/PowerPC/PPC.h @@ -30,6 +30,7 @@ namespace llvm { class AsmPrinter; class MCInst; + FunctionPass *createPPCCTRLoops(); FunctionPass *createPPCBranchSelectionPass(); FunctionPass *createPPCISelDag(PPCTargetMachine &TM); FunctionPass *createPPCJITCodeEmitterPass(PPCTargetMachine &TM, diff --git a/lib/Target/PowerPC/PPCBranchSelector.cpp b/lib/Target/PowerPC/PPCBranchSelector.cpp index 5f775e16f1c..21a0fb200f2 100644 --- a/lib/Target/PowerPC/PPCBranchSelector.cpp +++ b/lib/Target/PowerPC/PPCBranchSelector.cpp @@ -135,21 +135,33 @@ bool PPCBSel::runOnMachineFunction(MachineFunction &Fn) { MBBStartOffset += 4; continue; } - + // Otherwise, we have to expand it to a long branch. - // The BCC operands are: - // 0. PPC branch predicate - // 1. CR register - // 2. Target MBB - PPC::Predicate Pred = (PPC::Predicate)I->getOperand(0).getImm(); - unsigned CRReg = I->getOperand(1).getReg(); - MachineInstr *OldBranch = I; DebugLoc dl = OldBranch->getDebugLoc(); - - // Jump over the uncond branch inst (i.e. $PC+8) on opposite condition. - BuildMI(MBB, I, dl, TII->get(PPC::BCC)) - .addImm(PPC::InvertPredicate(Pred)).addReg(CRReg).addImm(2); + + if (I->getOpcode() == PPC::BCC) { + // The BCC operands are: + // 0. PPC branch predicate + // 1. CR register + // 2. Target MBB + PPC::Predicate Pred = (PPC::Predicate)I->getOperand(0).getImm(); + unsigned CRReg = I->getOperand(1).getReg(); + + // Jump over the uncond branch inst (i.e. $PC+8) on opposite condition. + BuildMI(MBB, I, dl, TII->get(PPC::BCC)) + .addImm(PPC::InvertPredicate(Pred)).addReg(CRReg).addImm(2); + } else if (I->getOpcode() == PPC::BDNZ) { + BuildMI(MBB, I, dl, TII->get(PPC::BDZ)).addImm(2); + } else if (I->getOpcode() == PPC::BDNZ8) { + BuildMI(MBB, I, dl, TII->get(PPC::BDZ8)).addImm(2); + } else if (I->getOpcode() == PPC::BDZ) { + BuildMI(MBB, I, dl, TII->get(PPC::BDNZ)).addImm(2); + } else if (I->getOpcode() == PPC::BDZ8) { + BuildMI(MBB, I, dl, TII->get(PPC::BDNZ8)).addImm(2); + } else { + llvm_unreachable("Unhandled branch type!"); + } // Uncond branch to the real destination. I = BuildMI(MBB, I, dl, TII->get(PPC::B)).addMBB(Dest); diff --git a/lib/Target/PowerPC/PPCCTRLoops.cpp b/lib/Target/PowerPC/PPCCTRLoops.cpp new file mode 100644 index 00000000000..4671893c3e4 --- /dev/null +++ b/lib/Target/PowerPC/PPCCTRLoops.cpp @@ -0,0 +1,679 @@ +//===-- PPCCTRLoops.cpp - Identify and generate CTR loops -----------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This pass identifies loops where we can generate the PPC branch instructions +// that decrement and test the count register (CTR) (bdnz and friends). +// This pass is based on the HexagonHardwareLoops pass. +// +// The pattern that defines the induction variable can changed depending on +// prior optimizations. For example, the IndVarSimplify phase run by 'opt' +// normalizes induction variables, and the Loop Strength Reduction pass +// run by 'llc' may also make changes to the induction variable. +// The pattern detected by this phase is due to running Strength Reduction. +// +// Criteria for CTR loops: +// - Countable loops (w/ ind. var for a trip count) +// - Assumes loops are normalized by IndVarSimplify +// - Try inner-most loops first +// - No nested CTR loops. +// - No function calls in loops. +// +// Note: As with unconverted loops, PPCBranchSelector must be run after this +// pass in order to convert long-displacement jumps into jump pairs. +// +//===----------------------------------------------------------------------===// + +#define DEBUG_TYPE "ctrloops" +#include "PPC.h" +#include "PPCTargetMachine.h" +#include "llvm/Constants.h" +#include "llvm/PassSupport.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/RegisterScavenging.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetInstrInfo.h" +#include + +using namespace llvm; + +STATISTIC(NumCTRLoops, "Number of loops converted to CTR loops"); + +namespace { + class CountValue; + struct PPCCTRLoops : public MachineFunctionPass { + MachineLoopInfo *MLI; + MachineRegisterInfo *MRI; + const TargetInstrInfo *TII; + + public: + static char ID; // Pass identification, replacement for typeid + + PPCCTRLoops() : MachineFunctionPass(ID) {} + + virtual bool runOnMachineFunction(MachineFunction &MF); + + const char *getPassName() const { return "PPC CTR Loops"; } + + virtual void getAnalysisUsage(AnalysisUsage &AU) const { + AU.setPreservesCFG(); + AU.addRequired(); + AU.addPreserved(); + AU.addRequired(); + AU.addPreserved(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + private: + /// getCanonicalInductionVariable - Check to see if the loop has a canonical + /// induction variable. + /// Should be defined in MachineLoop. Based upon version in class Loop. + MachineInstr *getCanonicalInductionVariable(MachineLoop *L, + MachineInstr *&IOp) const; + + /// getTripCount - Return a loop-invariant LLVM register indicating the + /// number of times the loop will be executed. If the trip-count cannot + /// be determined, this return null. + CountValue *getTripCount(MachineLoop *L, bool &WordCmp, + SmallVector &OldInsts) const; + + /// isInductionOperation - Return true if the instruction matches the + /// pattern for an opertion that defines an induction variable. + bool isInductionOperation(const MachineInstr *MI, unsigned IVReg) const; + + /// isInvalidOperation - Return true if the instruction is not valid within + /// a CTR loop. + bool isInvalidLoopOperation(const MachineInstr *MI) const; + + /// containsInavlidInstruction - Return true if the loop contains an + /// instruction that inhibits using the CTR loop. + bool containsInvalidInstruction(MachineLoop *L) const; + + /// converToCTRLoop - Given a loop, check if we can convert it to a + /// CTR loop. If so, then perform the conversion and return true. + bool convertToCTRLoop(MachineLoop *L); + + /// isDead - Return true if the instruction is now dead. + bool isDead(const MachineInstr *MI, + SmallVector &DeadPhis) const; + + /// removeIfDead - Remove the instruction if it is now dead. + void removeIfDead(MachineInstr *MI); + }; + + char PPCCTRLoops::ID = 0; + + + // CountValue class - Abstraction for a trip count of a loop. A + // smaller vesrsion of the MachineOperand class without the concerns + // of changing the operand representation. + class CountValue { + public: + enum CountValueType { + CV_Register, + CV_Immediate + }; + private: + CountValueType Kind; + union Values { + unsigned RegNum; + int64_t ImmVal; + Values(unsigned r) : RegNum(r) {} + Values(int64_t i) : ImmVal(i) {} + } Contents; + bool isNegative; + + public: + CountValue(unsigned r, bool neg) : Kind(CV_Register), Contents(r), + isNegative(neg) {} + explicit CountValue(int64_t i) : Kind(CV_Immediate), Contents(i), + isNegative(i < 0) {} + CountValueType getType() const { return Kind; } + bool isReg() const { return Kind == CV_Register; } + bool isImm() const { return Kind == CV_Immediate; } + bool isNeg() const { return isNegative; } + + unsigned getReg() const { + assert(isReg() && "Wrong CountValue accessor"); + return Contents.RegNum; + } + void setReg(unsigned Val) { + Contents.RegNum = Val; + } + int64_t getImm() const { + assert(isImm() && "Wrong CountValue accessor"); + if (isNegative) { + return -Contents.ImmVal; + } + return Contents.ImmVal; + } + void setImm(int64_t Val) { + Contents.ImmVal = Val; + } + + void print(raw_ostream &OS, const TargetMachine *TM = 0) const { + if (isReg()) { OS << PrintReg(getReg()); } + if (isImm()) { OS << getImm(); } + } + }; +} // end anonymous namespace + + +/// isCompareEquals - Returns true if the instruction is a compare equals +/// instruction with an immediate operand. +static bool isCompareEqualsImm(const MachineInstr *MI, bool &WordCmp) { + if (MI->getOpcode() == PPC::CMPWI || MI->getOpcode() == PPC::CMPLWI) { + WordCmp = true; + return true; + } else if (MI->getOpcode() == PPC::CMPDI || MI->getOpcode() == PPC::CMPLDI) { + WordCmp = false; + return true; + } + + return false; +} + + +/// createPPCCTRLoops - Factory for creating +/// the CTR loop phase. +FunctionPass *llvm::createPPCCTRLoops() { + return new PPCCTRLoops(); +} + + +bool PPCCTRLoops::runOnMachineFunction(MachineFunction &MF) { + DEBUG(dbgs() << "********* PPC CTR Loops *********\n"); + + bool Changed = false; + + // get the loop information + MLI = &getAnalysis(); + // get the register information + MRI = &MF.getRegInfo(); + // the target specific instructio info. + TII = MF.getTarget().getInstrInfo(); + + for (MachineLoopInfo::iterator I = MLI->begin(), E = MLI->end(); + I != E; ++I) { + MachineLoop *L = *I; + if (!L->getParentLoop()) { + Changed |= convertToCTRLoop(L); + } + } + + return Changed; +} + +/// getCanonicalInductionVariable - Check to see if the loop has a canonical +/// induction variable. We check for a simple recurrence pattern - an +/// integer recurrence that decrements by one each time through the loop and +/// ends at zero. If so, return the phi node that corresponds to it. +/// +/// Based upon the similar code in LoopInfo except this code is specific to +/// the machine. +/// This method assumes that the IndVarSimplify pass has been run by 'opt'. +/// +MachineInstr +*PPCCTRLoops::getCanonicalInductionVariable(MachineLoop *L, + MachineInstr *&IOp) const { + MachineBasicBlock *TopMBB = L->getTopBlock(); + MachineBasicBlock::pred_iterator PI = TopMBB->pred_begin(); + assert(PI != TopMBB->pred_end() && + "Loop must have more than one incoming edge!"); + MachineBasicBlock *Backedge = *PI++; + if (PI == TopMBB->pred_end()) return 0; // dead loop + MachineBasicBlock *Incoming = *PI++; + if (PI != TopMBB->pred_end()) return 0; // multiple backedges? + + // make sure there is one incoming and one backedge and determine which + // is which. + if (L->contains(Incoming)) { + if (L->contains(Backedge)) + return 0; + std::swap(Incoming, Backedge); + } else if (!L->contains(Backedge)) + return 0; + + // Loop over all of the PHI nodes, looking for a canonical induction variable: + // - The PHI node is "reg1 = PHI reg2, BB1, reg3, BB2". + // - The recurrence comes from the backedge. + // - the definition is an induction operatio.n + for (MachineBasicBlock::iterator I = TopMBB->begin(), E = TopMBB->end(); + I != E && I->isPHI(); ++I) { + MachineInstr *MPhi = &*I; + unsigned DefReg = MPhi->getOperand(0).getReg(); + for (unsigned i = 1; i != MPhi->getNumOperands(); i += 2) { + // Check each operand for the value from the backedge. + MachineBasicBlock *MBB = MPhi->getOperand(i+1).getMBB(); + if (L->contains(MBB)) { // operands comes from the backedge + // Check if the definition is an induction operation. + MachineInstr *DI = MRI->getVRegDef(MPhi->getOperand(i).getReg()); + if (isInductionOperation(DI, DefReg)) { + IOp = DI; + return MPhi; + } + } + } + } + return 0; +} + +/// getTripCount - Return a loop-invariant LLVM value indicating the +/// number of times the loop will be executed. The trip count can +/// be either a register or a constant value. If the trip-count +/// cannot be determined, this returns null. +/// +/// We find the trip count from the phi instruction that defines the +/// induction variable. We follow the links to the CMP instruction +/// to get the trip count. +/// +/// Based upon getTripCount in LoopInfo. +/// +CountValue *PPCCTRLoops::getTripCount(MachineLoop *L, bool &WordCmp, + SmallVector &OldInsts) const { + // Check that the loop has a induction variable. + MachineInstr *IOp; + MachineInstr *IV_Inst = getCanonicalInductionVariable(L, IOp); + if (IV_Inst == 0) return 0; + + // Canonical loops will end with a 'cmpwi/cmpdi cr, IV, Imm', + // if Imm is 0, get the count from the PHI opnd + // if Imm is -M, than M is the count + // Otherwise, Imm is the count + MachineOperand *IV_Opnd; + const MachineOperand *InitialValue; + if (!L->contains(IV_Inst->getOperand(2).getMBB())) { + InitialValue = &IV_Inst->getOperand(1); + IV_Opnd = &IV_Inst->getOperand(3); + } else { + InitialValue = &IV_Inst->getOperand(3); + IV_Opnd = &IV_Inst->getOperand(1); + } + + // Look for the cmp instruction to determine if we + // can get a useful trip count. The trip count can + // be either a register or an immediate. The location + // of the value depends upon the type (reg or imm). + while ((IV_Opnd = IV_Opnd->getNextOperandForReg())) { + MachineInstr *MI = IV_Opnd->getParent(); + if (L->contains(MI) && isCompareEqualsImm(MI, WordCmp)) { + OldInsts.push_back(MI); + OldInsts.push_back(IOp); + + const MachineOperand &MO = MI->getOperand(2); + assert(MO.isImm() && "IV Cmp Operand should be an immediate"); + int64_t ImmVal = MO.getImm(); + + const MachineInstr *IV_DefInstr = MRI->getVRegDef(IV_Opnd->getReg()); + assert(L->contains(IV_DefInstr->getParent()) && + "IV definition should occurs in loop"); + int64_t iv_value = IV_DefInstr->getOperand(2).getImm(); + + if (ImmVal == 0) { + // Make sure the induction variable changes by one on each iteration. + if (iv_value != 1 && iv_value != -1) { + return 0; + } + return new CountValue(InitialValue->getReg(), iv_value > 0); + } else { + assert(InitialValue->isReg() && "Expecting register for init value"); + const MachineInstr *DefInstr = MRI->getVRegDef(InitialValue->getReg()); + + // Here we need to look for an immediate load (an li or lis/ori pair). + if (DefInstr && (DefInstr->getOpcode() == PPC::ORI8 || + DefInstr->getOpcode() == PPC::ORI)) { + int64_t start = DefInstr->getOperand(2).getImm(); + const MachineInstr *DefInstr2 = + MRI->getVRegDef(DefInstr->getOperand(0).getReg()); + if (DefInstr2 && (DefInstr2->getOpcode() == PPC::LIS8 || + DefInstr2->getOpcode() == PPC::LIS)) { + start |= DefInstr2->getOperand(1).getImm() << 16; + + int64_t count = ImmVal - start; + if ((count % iv_value) != 0) { + return 0; + } + return new CountValue(count/iv_value); + } + } else if (DefInstr && (DefInstr->getOpcode() == PPC::LI8 || + DefInstr->getOpcode() == PPC::LI)) { + int64_t count = ImmVal - DefInstr->getOperand(1).getImm(); + if ((count % iv_value) != 0) { + return 0; + } + return new CountValue(count/iv_value); + } + } + } + } + return 0; +} + +/// isInductionOperation - return true if the operation is matches the +/// pattern that defines an induction variable: +/// addi iv, c +/// +bool +PPCCTRLoops::isInductionOperation(const MachineInstr *MI, + unsigned IVReg) const { + return ((MI->getOpcode() == PPC::ADDI || MI->getOpcode() == PPC::ADDI8) && + MI->getOperand(1).getReg() == IVReg); +} + +/// isInvalidOperation - Return true if the operation is invalid within +/// CTR loop. +bool +PPCCTRLoops::isInvalidLoopOperation(const MachineInstr *MI) const { + + // call is not allowed because the callee may use a CTR loop + if (MI->getDesc().isCall()) { + return true; + } + // check if the instruction defines a CTR loop register + // (this will also catch nested CTR loops) + for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { + const MachineOperand &MO = MI->getOperand(i); + if (MO.isReg() && MO.isDef() && + (MO.getReg() == PPC::CTR || MO.getReg() == PPC::CTR8)) { + return true; + } + } + return false; +} + +/// containsInvalidInstruction - Return true if the loop contains +/// an instruction that inhibits the use of the CTR loop function. +/// +bool PPCCTRLoops::containsInvalidInstruction(MachineLoop *L) const { + const std::vector Blocks = L->getBlocks(); + for (unsigned i = 0, e = Blocks.size(); i != e; ++i) { + MachineBasicBlock *MBB = Blocks[i]; + for (MachineBasicBlock::iterator + MII = MBB->begin(), E = MBB->end(); MII != E; ++MII) { + const MachineInstr *MI = &*MII; + if (isInvalidLoopOperation(MI)) { + return true; + } + } + } + return false; +} + +/// isDead returns true if the instruction is dead +/// (this was essentially copied from DeadMachineInstructionElim::isDead, but +/// with special cases for inline asm, physical registers and instructions with +/// side effects removed) +bool PPCCTRLoops::isDead(const MachineInstr *MI, + SmallVector &DeadPhis) const { + // Examine each operand. + for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { + const MachineOperand &MO = MI->getOperand(i); + if (MO.isReg() && MO.isDef()) { + unsigned Reg = MO.getReg(); + if (!MRI->use_nodbg_empty(Reg)) { + // This instruction has users, but if the only user is the phi node for the + // parent block, and the only use of that phi node is this instruction, then + // this instruction is dead: both it (and the phi node) can be removed. + MachineRegisterInfo::use_iterator I = MRI->use_begin(Reg); + if (llvm::next(I) == MRI->use_end() && + I.getOperand().getParent()->isPHI()) { + MachineInstr *OnePhi = I.getOperand().getParent(); + + for (unsigned j = 0, f = OnePhi->getNumOperands(); j != f; ++j) { + const MachineOperand &OPO = OnePhi->getOperand(j); + if (OPO.isReg() && OPO.isDef()) { + unsigned OPReg = OPO.getReg(); + + MachineRegisterInfo::use_iterator nextJ; + for (MachineRegisterInfo::use_iterator J = MRI->use_begin(OPReg), + E = MRI->use_end(); J!=E; J=nextJ) { + nextJ = llvm::next(J); + MachineOperand& Use = J.getOperand(); + MachineInstr *UseMI = Use.getParent(); + + if (MI != UseMI) { + // The phi node has a user that is not MI, bail... + return false; + } + } + } + } + + DeadPhis.push_back(OnePhi); + } else { + // This def has a non-debug use. Don't delete the instruction! + return false; + } + } + } + } + + // If there are no defs with uses, the instruction is dead. + return true; +} + +void PPCCTRLoops::removeIfDead(MachineInstr *MI) { + // This procedure was essentially copied from DeadMachineInstructionElim + + SmallVector DeadPhis; + if (isDead(MI, DeadPhis)) { + DEBUG(dbgs() << "CTR looping will remove: " << *MI); + + // It is possible that some DBG_VALUE instructions refer to this + // instruction. Examine each def operand for such references; + // if found, mark the DBG_VALUE as undef (but don't delete it). + for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { + const MachineOperand &MO = MI->getOperand(i); + if (!MO.isReg() || !MO.isDef()) + continue; + unsigned Reg = MO.getReg(); + MachineRegisterInfo::use_iterator nextI; + for (MachineRegisterInfo::use_iterator I = MRI->use_begin(Reg), + E = MRI->use_end(); I!=E; I=nextI) { + nextI = llvm::next(I); // I is invalidated by the setReg + MachineOperand& Use = I.getOperand(); + MachineInstr *UseMI = Use.getParent(); + if (UseMI==MI) + continue; + if (Use.isDebug()) // this might also be a instr -> phi -> instr case + // which can also be removed. + UseMI->getOperand(0).setReg(0U); + } + } + + MI->eraseFromParent(); + for (unsigned i = 0; i < DeadPhis.size(); ++i) { + DeadPhis[i]->eraseFromParent(); + } + } +} + +/// converToCTRLoop - check if the loop is a candidate for +/// converting to a CTR loop. If so, then perform the +/// transformation. +/// +/// This function works on innermost loops first. A loop can +/// be converted if it is a counting loop; either a register +/// value or an immediate. +/// +/// The code makes several assumptions about the representation +/// of the loop in llvm. +bool PPCCTRLoops::convertToCTRLoop(MachineLoop *L) { + bool Changed = false; + // Process nested loops first. + for (MachineLoop::iterator I = L->begin(), E = L->end(); I != E; ++I) { + Changed |= convertToCTRLoop(*I); + } + // If a nested loop has been converted, then we can't convert this loop. + if (Changed) { + return Changed; + } + + bool WordCmp; + SmallVector OldInsts; + // Are we able to determine the trip count for the loop? + CountValue *TripCount = getTripCount(L, WordCmp, OldInsts); + if (TripCount == 0) { + DEBUG(dbgs() << "failed to get trip count!\n"); + return false; + } + // Does the loop contain any invalid instructions? + if (containsInvalidInstruction(L)) { + return false; + } + MachineBasicBlock *Preheader = L->getLoopPreheader(); + // No preheader means there's not place for the loop instr. + if (Preheader == 0) { + return false; + } + MachineBasicBlock::iterator InsertPos = Preheader->getFirstTerminator(); + + DebugLoc dl; + if (InsertPos != Preheader->end()) + dl = InsertPos->getDebugLoc(); + + MachineBasicBlock *LastMBB = L->getExitingBlock(); + // Don't generate CTR loop if the loop has more than one exit. + if (LastMBB == 0) { + return false; + } + MachineBasicBlock::iterator LastI = LastMBB->getFirstTerminator(); + + // Determine the loop start. + MachineBasicBlock *LoopStart = L->getTopBlock(); + if (L->getLoopLatch() != LastMBB) { + // When the exit and latch are not the same, use the latch block as the + // start. + // The loop start address is used only after the 1st iteration, and the loop + // latch may contains instrs. that need to be executed after the 1st iter. + LoopStart = L->getLoopLatch(); + // Make sure the latch is a successor of the exit, otherwise it won't work. + if (!LastMBB->isSuccessor(LoopStart)) { + return false; + } + } + + // Convert the loop to a CTR loop + DEBUG(dbgs() << "Change to CTR loop at "; L->dump()); + + MachineFunction *MF = LastMBB->getParent(); + const PPCSubtarget &Subtarget = MF->getTarget().getSubtarget(); + bool isPPC64 = Subtarget.isPPC64(); + + unsigned CountReg; + if (TripCount->isReg()) { + // Create a copy of the loop count register. + const TargetRegisterClass *RC = + MF->getRegInfo().getRegClass(TripCount->getReg()); + CountReg = MF->getRegInfo().createVirtualRegister(RC); + BuildMI(*Preheader, InsertPos, dl, + TII->get(TargetOpcode::COPY), CountReg).addReg(TripCount->getReg()); + if (TripCount->isNeg()) { + unsigned CountReg1 = CountReg; + CountReg = MF->getRegInfo().createVirtualRegister(RC); + BuildMI(*Preheader, InsertPos, dl, + TII->get(isPPC64 ? PPC::NEG8 : PPC::NEG), + CountReg).addReg(CountReg1); + } + + // On a 64-bit system, if the original comparison was only 32-bit, then + // mask out the higher-order part of the count. + if (isPPC64 && WordCmp) { + unsigned CountReg1 = CountReg; + CountReg = MF->getRegInfo().createVirtualRegister(RC); + BuildMI(*Preheader, InsertPos, dl, + TII->get(PPC::RLDICL), CountReg).addReg(CountReg1 + ).addImm(0).addImm(32); + } + } else { + assert(TripCount->isImm() && "Expecting immedate vaule for trip count"); + // Put the trip count in a register for transfer into the count register. + const TargetRegisterClass *GPRC = &PPC::GPRCRegClass; + const TargetRegisterClass *G8RC = &PPC::G8RCRegClass; + const TargetRegisterClass *RC = isPPC64 ? G8RC : GPRC; + + int64_t CountImm = TripCount->getImm(); + if (TripCount->isNeg()) + CountImm = -CountImm; + + CountReg = MF->getRegInfo().createVirtualRegister(RC); + if (CountImm > 0xFFFF) { + BuildMI(*Preheader, InsertPos, dl, + TII->get(isPPC64 ? PPC::LIS8 : PPC::LIS), + CountReg).addImm(CountImm >> 16); + unsigned CountReg1 = CountReg; + CountReg = MF->getRegInfo().createVirtualRegister(RC); + BuildMI(*Preheader, InsertPos, dl, + TII->get(isPPC64 ? PPC::ORI8 : PPC::ORI), + CountReg).addReg(CountReg1).addImm(CountImm & 0xFFFF); + } else { + BuildMI(*Preheader, InsertPos, dl, + TII->get(isPPC64 ? PPC::LI8 : PPC::LI), + CountReg).addImm(CountImm); + } + } + + // Add the mtctr instruction to the beginning of the loop. + BuildMI(*Preheader, InsertPos, dl, + TII->get(isPPC64 ? PPC::MTCTR8 : PPC::MTCTR)).addReg(CountReg, + TripCount->isImm() ? RegState::Kill : 0); + + // Make sure the loop start always has a reference in the CFG. We need to + // create a BlockAddress operand to get this mechanism to work both the + // MachineBasicBlock and BasicBlock objects need the flag set. + LoopStart->setHasAddressTaken(); + // This line is needed to set the hasAddressTaken flag on the BasicBlock + // object + BlockAddress::get(const_cast(LoopStart->getBasicBlock())); + + // Replace the loop branch with a bdnz instruction. + dl = LastI->getDebugLoc(); + const std::vector Blocks = L->getBlocks(); + for (unsigned i = 0, e = Blocks.size(); i != e; ++i) { + MachineBasicBlock *MBB = Blocks[i]; + if (MBB != Preheader) + MBB->addLiveIn(isPPC64 ? PPC::CTR8 : PPC::CTR); + } + + // The loop ends with either: + // - a conditional branch followed by an unconditional branch, or + // - a conditional branch to the loop start. + assert(LastI->getOpcode() == PPC::BCC && + "loop end must start with a BCC instruction"); + // Either the BCC branches to the beginning of the loop, or it + // branches out of the loop and there is an unconditional branch + // to the start of the loop. + MachineBasicBlock *BranchTarget = LastI->getOperand(2).getMBB(); + BuildMI(*LastMBB, LastI, dl, + TII->get((BranchTarget == LoopStart) ? + (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) : + (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(BranchTarget); + + // Conditional branch; just delete it. + LastMBB->erase(LastI); + + delete TripCount; + + // The induction operation (add) and the comparison (cmpwi) may now be + // unneeded. If these are unneeded, then remove them. + for (unsigned i = 0; i < OldInsts.size(); ++i) + removeIfDead(OldInsts[i]); + + ++NumCTRLoops; + return true; +} + diff --git a/lib/Target/PowerPC/PPCInstr64Bit.td b/lib/Target/PowerPC/PPCInstr64Bit.td index 5da8f48507b..30699d27243 100644 --- a/lib/Target/PowerPC/PPCInstr64Bit.td +++ b/lib/Target/PowerPC/PPCInstr64Bit.td @@ -229,6 +229,15 @@ def : Pat<(PPCtc_return (i64 texternalsym:$dst), imm:$imm), def : Pat<(PPCtc_return CTRRC8:$dst, imm:$imm), (TCRETURNri8 CTRRC8:$dst, imm:$imm)>; +let isBranch = 1, isTerminator = 1, hasCtrlDep = 1, PPC970_Unit = 7 in { + let Defs = [CTR8], Uses = [CTR8] in { + def BDZ8 : IForm_ext<16, 18, 0, 0, (outs), (ins condbrtarget:$dst), + "bdz $dst", BrB, []>; + def BDNZ8 : IForm_ext<16, 16, 0, 0, (outs), (ins condbrtarget:$dst), + "bdnz $dst", BrB, []>; + } +} + // 64-but CR instructions def MTCRF8 : XFXForm_5<31, 144, (outs crbitm:$FXM), (ins G8RC:$rS), "mtcrf $FXM, $rS", BrMCRX>, diff --git a/lib/Target/PowerPC/PPCInstrFormats.td b/lib/Target/PowerPC/PPCInstrFormats.td index d8e4b2bdf34..a41a0279d21 100644 --- a/lib/Target/PowerPC/PPCInstrFormats.td +++ b/lib/Target/PowerPC/PPCInstrFormats.td @@ -94,6 +94,12 @@ class IForm opcode, bit aa, bit lk, dag OOL, dag IOL, string asmstr, let Inst{31} = lk; } +class IForm_ext opcode, bits<5> bo, bit aa, bit lk, dag OOL, dag IOL, + string asmstr, InstrItinClass itin, list pattern> + : IForm { + let LI{0-4} = bo; +} + // 1.7.2 B-Form class BForm opcode, bit aa, bit lk, dag OOL, dag IOL, string asmstr> : I { diff --git a/lib/Target/PowerPC/PPCInstrInfo.cpp b/lib/Target/PowerPC/PPCInstrInfo.cpp index c0b9e92c2d0..d6522822432 100644 --- a/lib/Target/PowerPC/PPCInstrInfo.cpp +++ b/lib/Target/PowerPC/PPCInstrInfo.cpp @@ -186,10 +186,14 @@ void PPCInstrInfo::insertNoop(MachineBasicBlock &MBB, // Branch analysis. +// Note: If the condition register is set to CTR or CTR8 then this is a +// BDNZ (imm == 1) or BDZ (imm == 0) branch. bool PPCInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl &Cond, bool AllowModify) const { + bool isPPC64 = TM.getSubtargetImpl()->isPPC64(); + // If the block has no terminators, it just falls into the block after it. MachineBasicBlock::iterator I = MBB.end(); if (I == MBB.begin()) @@ -221,7 +225,26 @@ bool PPCInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,MachineBasicBlock *&TBB, Cond.push_back(LastInst->getOperand(0)); Cond.push_back(LastInst->getOperand(1)); return false; + } else if (LastInst->getOpcode() == PPC::BDNZ8 || + LastInst->getOpcode() == PPC::BDNZ) { + if (!LastInst->getOperand(0).isMBB()) + return true; + TBB = LastInst->getOperand(0).getMBB(); + Cond.push_back(MachineOperand::CreateImm(1)); + Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR, + true)); + return false; + } else if (LastInst->getOpcode() == PPC::BDZ8 || + LastInst->getOpcode() == PPC::BDZ) { + if (!LastInst->getOperand(0).isMBB()) + return true; + TBB = LastInst->getOperand(0).getMBB(); + Cond.push_back(MachineOperand::CreateImm(0)); + Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR, + true)); + return false; } + // Otherwise, don't know what this is. return true; } @@ -245,6 +268,30 @@ bool PPCInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,MachineBasicBlock *&TBB, Cond.push_back(SecondLastInst->getOperand(1)); FBB = LastInst->getOperand(0).getMBB(); return false; + } else if ((SecondLastInst->getOpcode() == PPC::BDNZ8 || + SecondLastInst->getOpcode() == PPC::BDNZ) && + LastInst->getOpcode() == PPC::B) { + if (!SecondLastInst->getOperand(0).isMBB() || + !LastInst->getOperand(0).isMBB()) + return true; + TBB = SecondLastInst->getOperand(0).getMBB(); + Cond.push_back(MachineOperand::CreateImm(1)); + Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR, + true)); + FBB = LastInst->getOperand(0).getMBB(); + return false; + } else if ((SecondLastInst->getOpcode() == PPC::BDZ8 || + SecondLastInst->getOpcode() == PPC::BDZ) && + LastInst->getOpcode() == PPC::B) { + if (!SecondLastInst->getOperand(0).isMBB() || + !LastInst->getOperand(0).isMBB()) + return true; + TBB = SecondLastInst->getOperand(0).getMBB(); + Cond.push_back(MachineOperand::CreateImm(0)); + Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR, + true)); + FBB = LastInst->getOperand(0).getMBB(); + return false; } // If the block ends with two PPC:Bs, handle it. The second one is not @@ -273,7 +320,9 @@ unsigned PPCInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const { return 0; --I; } - if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC) + if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC && + I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ && + I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ) return 0; // Remove the branch. @@ -283,7 +332,9 @@ unsigned PPCInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const { if (I == MBB.begin()) return 1; --I; - if (I->getOpcode() != PPC::BCC) + if (I->getOpcode() != PPC::BCC && + I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ && + I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ) return 1; // Remove the branch. @@ -301,10 +352,16 @@ PPCInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, assert((Cond.size() == 2 || Cond.size() == 0) && "PPC branch conditions have two components!"); + bool isPPC64 = TM.getSubtargetImpl()->isPPC64(); + // One-way branch. if (FBB == 0) { if (Cond.empty()) // Unconditional branch BuildMI(&MBB, DL, get(PPC::B)).addMBB(TBB); + else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8) + BuildMI(&MBB, DL, get(Cond[0].getImm() ? + (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) : + (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB); else // Conditional branch BuildMI(&MBB, DL, get(PPC::BCC)) .addImm(Cond[0].getImm()).addReg(Cond[1].getReg()).addMBB(TBB); @@ -312,8 +369,13 @@ PPCInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, } // Two-way Conditional Branch. - BuildMI(&MBB, DL, get(PPC::BCC)) - .addImm(Cond[0].getImm()).addReg(Cond[1].getReg()).addMBB(TBB); + if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8) + BuildMI(&MBB, DL, get(Cond[0].getImm() ? + (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) : + (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB); + else + BuildMI(&MBB, DL, get(PPC::BCC)) + .addImm(Cond[0].getImm()).addReg(Cond[1].getReg()).addMBB(TBB); BuildMI(&MBB, DL, get(PPC::B)).addMBB(FBB); return 2; } @@ -665,8 +727,11 @@ PPCInstrInfo::emitFrameIndexDebugValue(MachineFunction &MF, bool PPCInstrInfo:: ReverseBranchCondition(SmallVectorImpl &Cond) const { assert(Cond.size() == 2 && "Invalid PPC branch opcode!"); - // Leave the CR# the same, but invert the condition. - Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm())); + if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR) + Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0); + else + // Leave the CR# the same, but invert the condition. + Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm())); return false; } diff --git a/lib/Target/PowerPC/PPCInstrInfo.td b/lib/Target/PowerPC/PPCInstrInfo.td index d2512423604..f75f4e522eb 100644 --- a/lib/Target/PowerPC/PPCInstrInfo.td +++ b/lib/Target/PowerPC/PPCInstrInfo.td @@ -438,6 +438,13 @@ let isBranch = 1, isTerminator = 1, hasCtrlDep = 1, PPC970_Unit = 7 in { def BCC : BForm<16, 0, 0, (outs), (ins pred:$cond, condbrtarget:$dst), "b${cond:cc} ${cond:reg}, $dst" /*[(PPCcondbranch CRRC:$crS, imm:$opc, bb:$dst)]*/>; + + let Defs = [CTR], Uses = [CTR] in { + def BDZ : IForm_ext<16, 18, 0, 0, (outs), (ins condbrtarget:$dst), + "bdz $dst", BrB, []>; + def BDNZ : IForm_ext<16, 16, 0, 0, (outs), (ins condbrtarget:$dst), + "bdnz $dst", BrB, []>; + } } // Darwin ABI Calls. diff --git a/lib/Target/PowerPC/PPCTargetMachine.cpp b/lib/Target/PowerPC/PPCTargetMachine.cpp index 50f3db8b27f..6f7fc4bb391 100644 --- a/lib/Target/PowerPC/PPCTargetMachine.cpp +++ b/lib/Target/PowerPC/PPCTargetMachine.cpp @@ -17,10 +17,15 @@ #include "llvm/MC/MCStreamer.h" #include "llvm/CodeGen/Passes.h" #include "llvm/Target/TargetOptions.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/FormattedStream.h" #include "llvm/Support/TargetRegistry.h" using namespace llvm; +static cl:: +opt DisableCTRLoops("disable-ppc-ctrloops", cl::Hidden, + cl::desc("Disable CTR loops for PPC")); + extern "C" void LLVMInitializePowerPCTarget() { // Register the targets RegisterTargetMachine A(ThePPC32Target); @@ -81,6 +86,7 @@ public: return getTM(); } + virtual bool addPreRegAlloc(); virtual bool addInstSelector(); virtual bool addPreEmitPass(); }; @@ -96,6 +102,14 @@ TargetPassConfig *PPCTargetMachine::createPassConfig(PassManagerBase &PM) { return PassConfig; } +bool PPCPassConfig::addPreRegAlloc() { + if (!DisableCTRLoops && getOptLevel() != CodeGenOpt::None) { + PM->add(createPPCCTRLoops()); + } + + return false; +} + bool PPCPassConfig::addInstSelector() { // Install an instruction selector. PM->add(createPPCISelDag(getPPCTargetMachine())); diff --git a/test/CodeGen/PowerPC/ctrloops.ll b/test/CodeGen/PowerPC/ctrloops.ll new file mode 100644 index 00000000000..f7bd354ce8e --- /dev/null +++ b/test/CodeGen/PowerPC/ctrloops.ll @@ -0,0 +1,78 @@ +target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v128:128:128-n32:64" +target triple = "powerpc64-unknown-freebsd10.0" +; RUN: llc < %s -march=ppc64 | FileCheck %s + +@a = common global i32 0, align 4 + +define void @test1(i32 %c) nounwind { +entry: + br label %for.body + +for.body: ; preds = %for.body, %entry + %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.body ] + %0 = load volatile i32* @a, align 4, !tbaa !0 + %add = add nsw i32 %0, %c + store volatile i32 %add, i32* @a, align 4, !tbaa !0 + %inc = add nsw i32 %i.01, 1 + %exitcond = icmp eq i32 %inc, 2048 + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body + ret void +; CHECK: @test1 +; CHECK: mtctr +; CHECK-NOT: addi +; CHECK-NOT: cmplwi +; CHECK: bdnz +} + +define void @test2(i32 %c, i32 %d) nounwind { +entry: + %cmp1 = icmp sgt i32 %d, 0 + br i1 %cmp1, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %i.02 = phi i32 [ %inc, %for.body ], [ 0, %entry ] + %0 = load volatile i32* @a, align 4, !tbaa !0 + %add = add nsw i32 %0, %c + store volatile i32 %add, i32* @a, align 4, !tbaa !0 + %inc = add nsw i32 %i.02, 1 + %exitcond = icmp eq i32 %inc, %d + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +; CHECK: @test2 +; CHECK: mtctr +; CHECK-NOT: addi +; CHECK-NOT: cmplwi +; CHECK: bdnz +} + +define void @test3(i32 %c, i32 %d) nounwind { +entry: + %cmp1 = icmp sgt i32 %d, 0 + br i1 %cmp1, label %for.body, label %for.end + +for.body: ; preds = %entry, %for.body + %i.02 = phi i32 [ %inc, %for.body ], [ 0, %entry ] + %mul = mul nsw i32 %i.02, %c + %0 = load volatile i32* @a, align 4, !tbaa !0 + %add = add nsw i32 %0, %mul + store volatile i32 %add, i32* @a, align 4, !tbaa !0 + %inc = add nsw i32 %i.02, 1 + %exitcond = icmp eq i32 %inc, %d + br i1 %exitcond, label %for.end, label %for.body + +for.end: ; preds = %for.body, %entry + ret void +; CHECK: @test3 +; CHECK: mtctr +; CHECK-NOT: addi +; CHECK-NOT: cmplwi +; CHECK: bdnz +} + +!0 = metadata !{metadata !"int", metadata !1} +!1 = metadata !{metadata !"omnipotent char", metadata !2} +!2 = metadata !{metadata !"Simple C/C++ TBAA"}