diff --git a/include/llvm/CodeGen/ScheduleDAG.h b/include/llvm/CodeGen/ScheduleDAG.h index d82edae600c..ed7d3ed4022 100644 --- a/include/llvm/CodeGen/ScheduleDAG.h +++ b/include/llvm/CodeGen/ScheduleDAG.h @@ -418,6 +418,8 @@ namespace llvm { public: SelectionDAG *DAG; // DAG of the current basic block MachineBasicBlock *BB; // Current basic block + MachineBasicBlock::iterator Begin; // The beginning of the range to be scheduled. + MachineBasicBlock::iterator End; // The end of the range to be scheduled. const TargetMachine &TM; // Target processor const TargetInstrInfo *TII; // Target instruction information const TargetRegisterInfo *TRI; // Target processor register info @@ -440,7 +442,9 @@ namespace llvm { /// Run - perform scheduling. /// - void Run(SelectionDAG *DAG, MachineBasicBlock *MBB); + void Run(SelectionDAG *DAG, MachineBasicBlock *MBB, + MachineBasicBlock::iterator Begin, + MachineBasicBlock::iterator End); /// BuildSchedGraph - Build SUnits and set up their Preds and Succs /// to form the scheduling dependency graph. diff --git a/lib/CodeGen/PostRASchedulerList.cpp b/lib/CodeGen/PostRASchedulerList.cpp index eef20b9386f..7bf3b3c3423 100644 --- a/lib/CodeGen/PostRASchedulerList.cpp +++ b/lib/CodeGen/PostRASchedulerList.cpp @@ -187,9 +187,17 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) { // Loop over all of the basic blocks for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end(); MBB != MBBe; ++MBB) { + // Schedule each sequence of instructions not interrupted by a label + // or anything else that effectively needs to shut down scheduling. + MachineBasicBlock::iterator Current = MBB->begin(), End = MBB->end(); + for (MachineBasicBlock::iterator MI = Current; MI != End; ++MI) + if (MI->getDesc().isTerminator() || MI->isLabel()) { + Scheduler.Run(0, MBB, Current, MI); + Scheduler.EmitSchedule(); + Current = next(MI); + } - Scheduler.Run(0, MBB); - + Scheduler.Run(0, MBB, Current, End); Scheduler.EmitSchedule(); } diff --git a/lib/CodeGen/ScheduleDAG.cpp b/lib/CodeGen/ScheduleDAG.cpp index e309a4385e4..718f591902f 100644 --- a/lib/CodeGen/ScheduleDAG.cpp +++ b/lib/CodeGen/ScheduleDAG.cpp @@ -46,11 +46,18 @@ void ScheduleDAG::dumpSchedule() const { /// Run - perform scheduling. /// -void ScheduleDAG::Run(SelectionDAG *dag, MachineBasicBlock *bb) { +void ScheduleDAG::Run(SelectionDAG *dag, MachineBasicBlock *bb, + MachineBasicBlock::iterator begin, + MachineBasicBlock::iterator end) { + assert((!dag || begin == end) && + "An instruction range was given for SelectionDAG scheduling!"); + SUnits.clear(); Sequence.clear(); DAG = dag; BB = bb; + Begin = begin; + End = end; Schedule(); diff --git a/lib/CodeGen/ScheduleDAGEmit.cpp b/lib/CodeGen/ScheduleDAGEmit.cpp index d5bc67a14fc..0c8435da6cb 100644 --- a/lib/CodeGen/ScheduleDAGEmit.cpp +++ b/lib/CodeGen/ScheduleDAGEmit.cpp @@ -33,7 +33,7 @@ void ScheduleDAG::AddMemOperand(MachineInstr *MI, const MachineMemOperand &MO) { } void ScheduleDAG::EmitNoop() { - TII->insertNoop(*BB, BB->end()); + TII->insertNoop(*BB, End); } void ScheduleDAG::EmitPhysRegCopy(SUnit *SU, @@ -54,7 +54,7 @@ void ScheduleDAG::EmitPhysRegCopy(SUnit *SU, break; } } - TII->copyRegToReg(*BB, BB->end(), Reg, VRI->second, + TII->copyRegToReg(*BB, End, Reg, VRI->second, SU->CopyDstRC, SU->CopySrcRC); } else { // Copy from physical register. @@ -63,7 +63,7 @@ void ScheduleDAG::EmitPhysRegCopy(SUnit *SU, bool isNew = VRBaseMap.insert(std::make_pair(SU, VRBase)).second; isNew = isNew; // Silence compiler warning. assert(isNew && "Node emitted out of order - early"); - TII->copyRegToReg(*BB, BB->end(), VRBase, I->getReg(), + TII->copyRegToReg(*BB, End, VRBase, I->getReg(), SU->CopyDstRC, SU->CopySrcRC); } break; diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp index 7b5690c07b8..0ffdd05c2ee 100644 --- a/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -112,10 +112,16 @@ void ScheduleDAGInstrs::BuildSchedGraph() { std::map MemDefs; std::map > MemUses; - // Terminators can perform control transfers, we we need to make sure that - // all the work of the block is done before the terminator. + // If we have an SUnit which is representing a terminator instruction, we + // can use it as a place-holder successor for inter-block dependencies. SUnit *Terminator = 0; + // Terminators can perform control transfers, we we need to make sure that + // all the work of the block is done before the terminator. Labels can + // mark points of interest for various types of meta-data (eg. EH data), + // and we need to make sure nothing is scheduled around them. + SUnit *SchedulingBarrier = 0; + LoopDependencies LoopRegs(MLI, MDT); // Track which regs are live into a loop, to help guide back-edge-aware @@ -137,7 +143,7 @@ void ScheduleDAGInstrs::BuildSchedGraph() { unsigned SpecialAddressLatency = TM.getSubtarget().getSpecialAddressLatency(); - for (MachineBasicBlock::iterator MII = BB->end(), MIE = BB->begin(); + for (MachineBasicBlock::iterator MII = End, MIE = Begin; MII != MIE; --MII) { MachineInstr *MI = prior(MII); const TargetInstrDesc &TID = MI->getDesc(); @@ -368,11 +374,26 @@ void ScheduleDAGInstrs::BuildSchedGraph() { } } - // Add chain edges from the terminator to ensure that all the work of the - // block is completed before any control transfers. - if (Terminator && SU->Succs.empty()) - Terminator->addPred(SDep(SU, SDep::Order, SU->Latency)); + // Add chain edges from terminators and labels to ensure that no + // instructions are scheduled past them. + if (SchedulingBarrier && SU->Succs.empty()) + SchedulingBarrier->addPred(SDep(SU, SDep::Order, SU->Latency)); + // If we encounter a mid-block label, we need to go back and add + // dependencies on SUnits we've already processed to prevent the + // label from moving downward. + if (MI->isLabel()) + for (SUnit *I = SU; I != &SUnits[0]; --I) { + SUnit *SuccSU = SU-1; + SuccSU->addPred(SDep(SU, SDep::Order, SU->Latency)); + MachineInstr *SuccMI = SuccSU->getInstr(); + if (SuccMI->getDesc().isTerminator() || SuccMI->isLabel()) + break; + } + // If this instruction obstructs all scheduling, remember it. if (TID.isTerminator() || MI->isLabel()) + SchedulingBarrier = SU; + // If this instruction is a terminator, remember it. + if (TID.isTerminator()) Terminator = SU; } @@ -413,8 +434,11 @@ std::string ScheduleDAGInstrs::getGraphNodeLabel(const SUnit *SU) const { MachineBasicBlock *ScheduleDAGInstrs::EmitSchedule() { // For MachineInstr-based scheduling, we're rescheduling the instructions in // the block, so start by removing them from the block. - while (!BB->empty()) - BB->remove(BB->begin()); + while (Begin != End) { + MachineBasicBlock::iterator I = Begin; + ++Begin; + BB->remove(I); + } // Then re-insert them according to the given schedule. for (unsigned i = 0, e = Sequence.size(); i != e; i++) { @@ -425,7 +449,7 @@ MachineBasicBlock *ScheduleDAGInstrs::EmitSchedule() { continue; } - BB->push_back(SU->getInstr()); + BB->insert(End, SU->getInstr()); } return BB; diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodesEmit.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodesEmit.cpp index 7bb9bc20c01..2fa53e08e22 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodesEmit.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodesEmit.cpp @@ -127,7 +127,7 @@ void ScheduleDAGSDNodes::EmitCopyFromReg(SDNode *Node, unsigned ResNo, // Create the reg, emit the copy. VRBase = MRI.createVirtualRegister(DstRC); bool Emitted = - TII->copyRegToReg(*BB, BB->end(), VRBase, SrcReg, DstRC, SrcRC); + TII->copyRegToReg(*BB, End, VRBase, SrcReg, DstRC, SrcRC); Emitted = Emitted; // Silence compiler warning. assert(Emitted && "Unable to issue a copy instruction!"); } @@ -410,7 +410,7 @@ void ScheduleDAGSDNodes::EmitSubregNode(SDNode *Node, MI->addOperand(MachineOperand::CreateReg(VRBase, true)); AddOperand(MI, Node->getOperand(0), 0, 0, VRBaseMap); MI->addOperand(MachineOperand::CreateImm(SubIdx)); - BB->push_back(MI); + BB->insert(End, MI); } else if (Opc == TargetInstrInfo::INSERT_SUBREG || Opc == TargetInstrInfo::SUBREG_TO_REG) { SDValue N0 = Node->getOperand(0); @@ -445,7 +445,7 @@ void ScheduleDAGSDNodes::EmitSubregNode(SDNode *Node, // Add the subregster being inserted AddOperand(MI, N1, 0, 0, VRBaseMap); MI->addOperand(MachineOperand::CreateImm(SubIdx)); - BB->push_back(MI); + BB->insert(End, MI); } else assert(0 && "Node is not insert_subreg, extract_subreg, or subreg_to_reg"); @@ -505,12 +505,13 @@ void ScheduleDAGSDNodes::EmitNode(SDNode *Node, bool IsClone, bool IsCloned, for (unsigned i = NodeOperands; i != MemOperandsEnd; ++i) AddMemOperand(MI, cast(Node->getOperand(i))->MO); - if (II.usesCustomDAGSchedInsertionHook()) + if (II.usesCustomDAGSchedInsertionHook()) { // Insert this instruction into the basic block using a target // specific inserter which may returns a new basic block. BB = TLI->EmitInstrWithCustomInserter(MI, BB); - else - BB->push_back(MI); + Begin = End = BB->end(); + } else + BB->insert(End, MI); // Additional results must be an physical register def. if (HasPhysRegOuts) { @@ -559,7 +560,7 @@ void ScheduleDAGSDNodes::EmitNode(SDNode *Node, bool IsClone, bool IsCloned, else DstTRC = TRI->getPhysicalRegisterRegClass(DestReg, Node->getOperand(1).getValueType()); - TII->copyRegToReg(*BB, BB->end(), DestReg, SrcReg, DstTRC, SrcTRC); + TII->copyRegToReg(*BB, End, DestReg, SrcReg, DstTRC, SrcTRC); break; } case ISD::CopyFromReg: { @@ -614,7 +615,7 @@ void ScheduleDAGSDNodes::EmitNode(SDNode *Node, bool IsClone, bool IsCloned, break; } } - BB->push_back(MI); + BB->insert(End, MI); break; } } diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 3cae888b691..547cd8df33a 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -1074,7 +1074,7 @@ ScheduleDAG *SelectionDAGISel::Schedule() { } ScheduleDAG *Scheduler = Ctor(this, Fast); - Scheduler->Run(CurDAG, BB); + Scheduler->Run(CurDAG, BB, BB->end(), BB->end()); return Scheduler; }