From d454a973a5c4e7c09e84a235ba8f9c458bb67f79 Mon Sep 17 00:00:00 2001 From: Tanya Lattner Date: Fri, 17 Jun 2005 04:00:57 +0000 Subject: [PATCH] Numerous bug fixes and the completed modschedSB algorithm (minor bugs still exist for course). git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@22239 91177308-0d34-0410-b5e6-96231b3b80d8 --- .../ModuloScheduling/DependenceAnalyzer.cpp | 25 +- .../SparcV9/ModuloScheduling/MSSchedule.cpp | 215 +- .../SparcV9/ModuloScheduling/MSSchedule.h | 8 +- .../ModuloScheduling/ModuloScheduling.cpp | 174 +- .../ModuloScheduling/ModuloScheduling.h | 3 +- .../ModuloSchedulingSuperBlock.cpp | 2972 ++++++++++++++++- .../ModuloSchedulingSuperBlock.h | 119 +- 7 files changed, 3313 insertions(+), 203 deletions(-) diff --git a/lib/Target/SparcV9/ModuloScheduling/DependenceAnalyzer.cpp b/lib/Target/SparcV9/ModuloScheduling/DependenceAnalyzer.cpp index 25ad03ad7d9..d0b5db304a1 100644 --- a/lib/Target/SparcV9/ModuloScheduling/DependenceAnalyzer.cpp +++ b/lib/Target/SparcV9/ModuloScheduling/DependenceAnalyzer.cpp @@ -159,10 +159,10 @@ void DependenceAnalyzer::advancedDepAnalysis(GetElementPtrInst *gp1, SCEVHandle SV2 = SE->getSCEV(Gep2Idx); //Now handle special cases of dependence analysis - SV1->print(std::cerr); - std::cerr << "\n"; - SV2->print(std::cerr); - std::cerr << "\n"; + //SV1->print(std::cerr); + //std::cerr << "\n"; + //SV2->print(std::cerr); + //std::cerr << "\n"; //Check if we have an SCEVAddExpr, cause we can only handle those SCEVAddRecExpr *SVAdd1 = dyn_cast(SV1); @@ -217,7 +217,7 @@ void DependenceAnalyzer::advancedDepAnalysis(GetElementPtrInst *gp1, //Find constant index difference int diff = A1->getValue()->getRawValue() - A2->getValue()->getRawValue(); - std::cerr << diff << "\n"; + //std::cerr << diff << "\n"; if(diff > 5) diff = 2; @@ -240,14 +240,21 @@ void DependenceAnalyzer::createDep(std::vector &deps, //If load/store pair if(valLoad && !val2Load) { - //Anti Dep - deps.push_back(Dependence(diff, Dependence::AntiDep)); + if(srcBeforeDest) + //Anti Dep + deps.push_back(Dependence(diff, Dependence::AntiDep)); + else + deps.push_back(Dependence(diff, Dependence::TrueDep)); + ++NumDeps; } //If store/load pair else if(!valLoad && val2Load) { - //True Dep - deps.push_back(Dependence(diff, Dependence::TrueDep)); + if(srcBeforeDest) + //True Dep + deps.push_back(Dependence(diff, Dependence::TrueDep)); + else + deps.push_back(Dependence(diff, Dependence::AntiDep)); ++NumDeps; } //If store/store pair diff --git a/lib/Target/SparcV9/ModuloScheduling/MSSchedule.cpp b/lib/Target/SparcV9/ModuloScheduling/MSSchedule.cpp index d1aaa4f46b1..6e8a6db3473 100644 --- a/lib/Target/SparcV9/ModuloScheduling/MSSchedule.cpp +++ b/lib/Target/SparcV9/ModuloScheduling/MSSchedule.cpp @@ -16,18 +16,23 @@ #include "llvm/Support/Debug.h" #include "llvm/Target/TargetSchedInfo.h" #include "../SparcV9Internals.h" +#include "llvm/CodeGen/MachineInstr.h" using namespace llvm; +//Check if all resources are free +bool resourcesFree(MSchedGraphNode*, int, +std::map > &resourceNumPerCycle); + //Returns a boolean indicating if the start cycle needs to be increased/decreased -bool MSSchedule::insert(MSchedGraphNode *node, int cycle) { +bool MSSchedule::insert(MSchedGraphNode *node, int cycle, int II) { //First, check if the cycle has a spot free to start if(schedule.find(cycle) != schedule.end()) { //Check if we have a free issue slot at this cycle if (schedule[cycle].size() < numIssue) { //Now check if all the resources in their respective cycles are available - if(resourcesFree(node, cycle)) { + if(resourcesFree(node, cycle, II)) { //Insert to preserve dependencies addToSchedule(cycle,node); DEBUG(std::cerr << "Found spot in map, and there is an issue slot\n"); @@ -37,7 +42,7 @@ bool MSSchedule::insert(MSchedGraphNode *node, int cycle) { } //Not in the map yet so put it in else { - if(resourcesFree(node,cycle)) { + if(resourcesFree(node,cycle,II)) { std::vector nodes; nodes.push_back(node); schedule[cycle] = nodes; @@ -68,99 +73,113 @@ void MSSchedule::addToSchedule(int cycle, MSchedGraphNode *node) { schedule[cycle] = nodes; } +bool MSSchedule::resourceAvailable(int resourceNum, int cycle) { + bool isFree = true; -bool MSSchedule::resourcesFree(MSchedGraphNode *node, int cycle) { + //Get Map for this cycle + if(resourceNumPerCycle.count(cycle)) { + if(resourceNumPerCycle[cycle].count(resourceNum)) { + int maxRes = CPUResource::getCPUResource(resourceNum)->maxNumUsers; + if(resourceNumPerCycle[cycle][resourceNum] >= maxRes) + isFree = false; + } + } + + return isFree; +} + +void MSSchedule::useResource(int resourceNum, int cycle) { + + //Get Map for this cycle + if(resourceNumPerCycle.count(cycle)) { + if(resourceNumPerCycle[cycle].count(resourceNum)) { + resourceNumPerCycle[cycle][resourceNum]++; + } + else { + resourceNumPerCycle[cycle][resourceNum] = 1; + } + } + //If no map, create one! + else { + std::map resourceUse; + resourceUse[resourceNum] = 1; + resourceNumPerCycle[cycle] = resourceUse; + } + +} + +bool MSSchedule::resourcesFree(MSchedGraphNode *node, int cycle, int II) { //Get Resource usage for this instruction const TargetSchedInfo *msi = node->getParent()->getTarget()->getSchedInfo(); int currentCycle = cycle; bool success = true; - //Get resource usage for this instruction - InstrRUsage rUsage = msi->getInstrRUsage(node->getInst()->getOpcode()); - std::vector > resources = rUsage.resourcesByCycle; + //Create vector of starting cycles + std::vector cyclesMayConflict; + cyclesMayConflict.push_back(cycle); - //Loop over resources in each cycle and increments their usage count - for(unsigned i=0; i < resources.size(); ++i) { - for(unsigned j=0; j < resources[i].size(); ++j) { - - //Get Resource to check its availability - int resourceNum = resources[i][j]; - - DEBUG(std::cerr << "Attempting to schedule Resource Num: " << resourceNum << " in cycle: " << currentCycle << "\n"); - - //Check if this resource is available for this cycle - std::map >::iterator resourcesForCycle = resourceNumPerCycle.find(currentCycle); - - //First check if map exists for this cycle - if(resourcesForCycle != resourceNumPerCycle.end()) { - //A map exists for this cycle, so lets check for the resource - std::map::iterator resourceUse = resourcesForCycle->second.find(resourceNum); - if(resourceUse != resourcesForCycle->second.end()) { - //Check if there are enough of this resource and if so, increase count and move on - if(resourceUse->second < CPUResource::getCPUResource(resourceNum)->maxNumUsers) - ++resourceUse->second; - - else { - DEBUG(std::cerr << "No resource num " << resourceNum << " available for cycle " << currentCycle << "\n"); - success = false; - } - } - //Not in the map yet, so put it - else - resourcesForCycle->second[resourceNum] = 1; - - } - else { - //Create a new map and put in our resource - std::map resourceMap; - resourceMap[resourceNum] = 1; - resourceNumPerCycle[currentCycle] = resourceMap; - } - if(!success) - break; - } - if(!success) - break; - - - //Increase cycle - currentCycle++; + if(resourceNumPerCycle.size() > 0) { + for(int i = cycle-II; i >= (resourceNumPerCycle.begin()->first); i-=II) + cyclesMayConflict.push_back(i); + for(int i = cycle+II; i <= resourceNumPerCycle.end()->first; i+=II) + cyclesMayConflict.push_back(i); } - if(!success) { - int oldCycle = cycle; - DEBUG(std::cerr << "Backtrack\n"); + //Now check all cycles for conflicts + for(int index = 0; index < (int) cyclesMayConflict.size(); ++index) { + currentCycle = cyclesMayConflict[index]; + //Get resource usage for this instruction InstrRUsage rUsage = msi->getInstrRUsage(node->getInst()->getOpcode()); std::vector > resources = rUsage.resourcesByCycle; - + //Loop over resources in each cycle and increments their usage count for(unsigned i=0; i < resources.size(); ++i) { - if(oldCycle < currentCycle) { + for(unsigned j=0; j < resources[i].size(); ++j) { + + //Get Resource to check its availability + int resourceNum = resources[i][j]; + + DEBUG(std::cerr << "Attempting to schedule Resource Num: " << resourceNum << " in cycle: " << currentCycle << "\n"); + + success = resourceAvailable(resourceNum, currentCycle); + + if(!success) + break; - //Check if this resource is available for this cycle - std::map >::iterator resourcesForCycle = resourceNumPerCycle.find(oldCycle); - if(resourcesForCycle != resourceNumPerCycle.end()) { - for(unsigned j=0; j < resources[i].size(); ++j) { - int resourceNum = resources[i][j]; - //remove from map - std::map::iterator resourceUse = resourcesForCycle->second.find(resourceNum); - //assert if not in the map.. since it should be! - //assert(resourceUse != resourcesForCycle.end() && "Resource should be in map!"); - DEBUG(std::cerr << "Removing resource num " << resourceNum << " from cycle " << oldCycle << "\n"); - --resourceUse->second; - } - } } - else + + if(!success) break; - oldCycle++; + + //Increase cycle + currentCycle++; } - return false; - + + if(!success) + return false; } + //Actually put resources into the map + if(success) { + + int currentCycle = cycle; + //Get resource usage for this instruction + InstrRUsage rUsage = msi->getInstrRUsage(node->getInst()->getOpcode()); + std::vector > resources = rUsage.resourcesByCycle; + + //Loop over resources in each cycle and increments their usage count + for(unsigned i=0; i < resources.size(); ++i) { + for(unsigned j=0; j < resources[i].size(); ++j) { + int resourceNum = resources[i][j]; + useResource(resourceNum, currentCycle); + } + currentCycle++; + } + } + + return true; } @@ -174,12 +193,9 @@ bool MSSchedule::constructKernel(int II, std::vector &branches DEBUG(std::cerr << "Offset: " << offset << "\n"); - //Not sure what happens in this case, but assert if offset is > II - //assert(offset > -II && "Offset can not be more then II"); - + //Using the schedule, fold up into kernel and check resource conflicts as we go std::vector > tempKernel; - - + int stageNum = ((schedule.rbegin()->first-offset)+1)/ II; int maxSN = 0; @@ -192,21 +208,18 @@ bool MSSchedule::constructKernel(int II, std::vector &branches for(std::vector::iterator I = schedule[i].begin(), E = schedule[i].end(); I != E; ++I) { //Check if its a branch - if((*I)->isBranch()) { - assert(count == 0 && "Branch can not be from a previous iteration"); - tempKernel.push_back(std::make_pair(*I, count)); - } - else { - //FIXME: Check if the instructions in the earlier stage conflict - tempKernel.push_back(std::make_pair(*I, count)); - maxSN = std::max(maxSN, count); - } + assert(!(*I)->isBranch() && "Branch should not be schedule!"); + + tempKernel.push_back(std::make_pair(*I, count)); + maxSN = std::max(maxSN, count); + } } ++count; } } + //Add in induction var code for(std::vector >::iterator I = tempKernel.begin(), IE = tempKernel.end(); I != IE; ++I) { @@ -253,6 +266,30 @@ bool MSSchedule::constructKernel(int II, std::vector &branches return true; } +bool MSSchedule::defPreviousStage(Value *def, int stage) { + + //Loop over kernel and determine if value is being defined in previous stage + for(std::vector >::iterator P = kernel.begin(), PE = kernel.end(); P != PE; ++P) { + MachineInstr* inst = P->first; + + //Loop over Machine Operands + for(unsigned i=0; i < inst->getNumOperands(); ++i) { + //get machine operand + const MachineOperand &mOp = inst->getOperand(i); + if(mOp.getType() == MachineOperand::MO_VirtualRegister && mOp.isDef()) { + if(def == mOp.getVRegValue()) { + if(P->second >= stage) + return false; + else + return true; + } + } + } + } + + assert(0 && "We should always have found the def in our kernel\n"); +} + void MSSchedule::print(std::ostream &os) const { os << "Schedule:\n"; diff --git a/lib/Target/SparcV9/ModuloScheduling/MSSchedule.h b/lib/Target/SparcV9/ModuloScheduling/MSSchedule.h index d9f42a25b4a..92a942b43c1 100644 --- a/lib/Target/SparcV9/ModuloScheduling/MSSchedule.h +++ b/lib/Target/SparcV9/ModuloScheduling/MSSchedule.h @@ -28,7 +28,9 @@ namespace llvm { std::map > resourceNumPerCycle; //Check if all resources are free - bool resourcesFree(MSchedGraphNode*, int); + bool resourcesFree(MSchedGraphNode*, int, int II); + bool resourceAvailable(int resourceNum, int cycle); + void useResource(int resourceNum, int cycle); //Resulting kernel std::vector > kernel; @@ -42,13 +44,13 @@ namespace llvm { public: MSSchedule(int num) : numIssue(num) {} MSSchedule() : numIssue(4) {} - bool insert(MSchedGraphNode *node, int cycle); + bool insert(MSchedGraphNode *node, int cycle, int II); int getStartCycle(MSchedGraphNode *node); void clear() { schedule.clear(); resourceNumPerCycle.clear(); kernel.clear(); } std::vector >* getKernel() { return &kernel; } bool constructKernel(int II, std::vector &branches, std::map &indVar); int getMaxStage() { return maxStage; } - + bool defPreviousStage(Value *def, int stage); //iterators typedef std::map >::iterator schedule_iterator; diff --git a/lib/Target/SparcV9/ModuloScheduling/ModuloScheduling.cpp b/lib/Target/SparcV9/ModuloScheduling/ModuloScheduling.cpp index 4e57c65aae1..65008a67f84 100644 --- a/lib/Target/SparcV9/ModuloScheduling/ModuloScheduling.cpp +++ b/lib/Target/SparcV9/ModuloScheduling/ModuloScheduling.cpp @@ -178,7 +178,7 @@ bool ModuloSchedulingPass::runOnFunction(Function &F) { } else ++JumboBB; - std::cerr << "BB Size: " << BI->size() << "\n"; + } defaultInst = 0; @@ -230,7 +230,6 @@ bool ModuloSchedulingPass::runOnFunction(Function &F) { II = std::max(RecMII, ResMII); int mII = II; - IISum += mII; //Print out II, RecMII, and ResMII DEBUG(std::cerr << "II starts out as " << II << " ( RecMII=" << RecMII << " and ResMII=" << ResMII << ")\n"); @@ -285,12 +284,15 @@ bool ModuloSchedulingPass::runOnFunction(Function &F) { reconstructLoop(*BI); ++MSLoops; Changed = true; + FinalIISum += II; + IISum += mII; if(schedule.getMaxStage() == 0) ++SameStage; } - else + else { ++NoSched; + } //Clear out our maps for the next basic block that is processed nodeToAttributesMap.clear(); @@ -323,7 +325,9 @@ bool ModuloSchedulingPass::CreateDefMap(MachineBasicBlock *BI) { if(mOp.getType() == MachineOperand::MO_VirtualRegister && mOp.isDef()) { //assert if this is the second def we have seen //DEBUG(std::cerr << "Putting " << *(mOp.getVRegValue()) << " into map\n"); - assert(!defMap.count(mOp.getVRegValue()) && "Def already in the map"); + //assert(!defMap.count(mOp.getVRegValue()) && "Def already in the map"); + if(defMap.count(mOp.getVRegValue())) + return false; defMap[mOp.getVRegValue()] = &*I; } @@ -397,7 +401,7 @@ bool ModuloSchedulingPass::MachineBBisValid(const MachineBasicBlock *BI) { || OC == V9::MOVRGEZi || OC == V9::MOVLEr || OC == V9::MOVLEi || OC == V9::MOVLEUr || OC == V9::MOVLEUi || OC == V9::MOVFLEr || OC == V9::MOVFLEi || OC == V9::MOVNEr || OC == V9::MOVNEi || OC == V9::MOVNEGr || OC == V9::MOVNEGi - || OC == V9::MOVFNEr || OC == V9::MOVFNEi) { + || OC == V9::MOVFNEr || OC == V9::MOVFNEi || OC == V9::MOVGr || OC == V9::MOVGi) { ++LoopsWithCondMov; return false; } @@ -579,6 +583,8 @@ int ModuloSchedulingPass::calculateResMII(const MachineBasicBlock *BI) { //Divide the usage count by either the max number we can issue or the number of //resources (whichever is its upper bound) double finalUsageCount; + DEBUG(std::cerr << "Resource Num: " << RB->first << " Usage: " << usageCount << " TotalNum: " << resourceNum << "\n"); + if( resourceNum <= issueSlots) finalUsageCount = ceil(1.0 * usageCount / resourceNum); else @@ -1035,6 +1041,56 @@ void ModuloSchedulingPass::addRecc(std::vector &stack, std::ma } +void ModuloSchedulingPass::addSCC(std::vector &SCC, std::map &newNodes) { + + int totalDelay = 0; + int totalDistance = 0; + std::vector recc; + MSchedGraphNode *start = 0; + MSchedGraphNode *end = 0; + + //Loop over recurrence, get delay and distance + for(std::vector::iterator N = SCC.begin(), NE = SCC.end(); N != NE; ++N) { + DEBUG(std::cerr << **N << "\n"); + totalDelay += (*N)->getLatency(); + + for(unsigned i = 0; i < (*N)->succ_size(); ++i) { + MSchedGraphEdge *edge = (*N)->getSuccessor(i); + if(find(SCC.begin(), SCC.end(), edge->getDest()) != SCC.end()) { + totalDistance += edge->getIteDiff(); + if(edge->getIteDiff() > 0) + if(!start && !end) { + start = *N; + end = edge->getDest(); + } + + } + } + + + //Get the original node + recc.push_back(newNodes[*N]); + + + } + + DEBUG(std::cerr << "End Recc\n"); + CircCount++; + + assert( (start && end) && "Must have start and end node to ignore edge for SCC"); + + if(start && end) { + //Insert reccurrence into the list + DEBUG(std::cerr << "Ignore Edge from!!: " << *start << " to " << *end << "\n"); + edgesToIgnore.insert(std::make_pair(newNodes[start], (newNodes[end])->getInEdgeNum(newNodes[start]))); + } + + int lastII = totalDelay / totalDistance; + + + recurrenceList.insert(std::make_pair(lastII, recc)); + +} void ModuloSchedulingPass::findAllCircuits(MSchedGraph *g, int II) { @@ -1074,6 +1130,7 @@ void ModuloSchedulingPass::findAllCircuits(MSchedGraph *g, int II) { std::set Visited; std::vector Vk; MSchedGraphNode* s = 0; + int numEdges = 0; //Find scc with the least vertex for (MSchedGraph::iterator GI = MSG->begin(), E = MSG->end(); GI != E; ++GI) @@ -1085,7 +1142,19 @@ void ModuloSchedulingPass::findAllCircuits(MSchedGraph *g, int II) { if (Visited.insert(nextSCC[0]).second) { Visited.insert(nextSCC.begin()+1, nextSCC.end()); - DEBUG(std::cerr << "SCC size: " << nextSCC.size() << "\n"); + if(nextSCC.size() > 1) { + std::cerr << "SCC size: " << nextSCC.size() << "\n"; + + for(unsigned i = 0; i < nextSCC.size(); ++i) { + //Loop over successor and see if in scc, then count edge + MSchedGraphNode *node = nextSCC[i]; + for(MSchedGraphNode::succ_iterator S = node->succ_begin(), SE = node->succ_end(); S != SE; ++S) { + if(find(nextSCC.begin(), nextSCC.end(), *S) != nextSCC.end()) + numEdges++; + } + } + std::cerr << "Num Edges: " << numEdges << "\n"; + } //Ignore self loops if(nextSCC.size() > 1) { @@ -1120,7 +1189,10 @@ void ModuloSchedulingPass::findAllCircuits(MSchedGraph *g, int II) { B[*N].clear(); } if(Vk.size() > 1) { - circuit(s, stack, blocked, Vk, s, B, II, newNodes); + if(numEdges < 98) + circuit(s, stack, blocked, Vk, s, B, II, newNodes); + else + addSCC(Vk, newNodes); //Delete nodes from the graph //Find all nodes up to s and delete them @@ -1860,8 +1932,8 @@ bool ModuloSchedulingPass::computeSchedule(const MachineBasicBlock *BB, MSchedGr schedule.clear(); } DEBUG(std::cerr << "Final II: " << II << "\n"); - FinalIISum += II; } + if(II >= capII) { DEBUG(std::cerr << "Maximum II reached, giving up\n"); @@ -1900,7 +1972,7 @@ bool ModuloSchedulingPass::scheduleNode(MSchedGraphNode *node, increaseSC = false; - increaseSC = schedule.insert(node, cycle); + increaseSC = schedule.insert(node, cycle, II); if(!increaseSC) return true; @@ -2034,18 +2106,11 @@ void ModuloSchedulingPass::writePrologues(std::vector &prol } } - - /*for(std::vector::iterator BR = branches.begin(), BE = branches.end(); BR != BE; ++BR) { - - //Stick in branch at the end - machineBB->push_back((*BR)->getInst()->clone()); - - //Add nop - BuildMI(machineBB, V9::NOP, 0); - }*/ - - - (((MachineBasicBlock*)origBB)->getParent())->getBasicBlockList().push_back(machineBB); + MachineFunction *F = (((MachineBasicBlock*)origBB)->getParent()); + MachineFunction::BasicBlockListType &BL = F->getBasicBlockList(); + MachineFunction::BasicBlockListType::iterator BLI = origBB; + assert(BLI != BL.end() && "Must find original BB in machine function\n"); + BL.insert(BLI,machineBB); prologues.push_back(machineBB); llvm_prologues.push_back(llvmBB); } @@ -2143,12 +2208,16 @@ void ModuloSchedulingPass::writeEpilogues(std::vector &epil } } - (((MachineBasicBlock*)origBB)->getParent())->getBasicBlockList().push_back(machineBB); - epilogues.push_back(machineBB); - llvm_epilogues.push_back(llvmBB); - - DEBUG(std::cerr << "EPILOGUE #" << i << "\n"); - DEBUG(machineBB->print(std::cerr)); + MachineFunction *F = (((MachineBasicBlock*)origBB)->getParent()); + MachineFunction::BasicBlockListType &BL = F->getBasicBlockList(); + MachineFunction::BasicBlockListType::iterator BLI = (MachineBasicBlock*) origBB; + assert(BLI != BL.end() && "Must find original BB in machine function\n"); + BL.insert(BLI,machineBB); + epilogues.push_back(machineBB); + llvm_epilogues.push_back(llvmBB); + + DEBUG(std::cerr << "EPILOGUE #" << i << "\n"); + DEBUG(machineBB->print(std::cerr)); } } @@ -2170,14 +2239,6 @@ void ModuloSchedulingPass::writeKernel(BasicBlock *llvmBB, MachineBasicBlock *ma DEBUG(std::cerr << "Stage: " << I->second << " Inst: " << *(I->first) << "\n";); - /*if(I->first->isBranch()) { - //Clone instruction - const MachineInstr *inst = I->first->getInst(); - MachineInstr *instClone = inst->clone(); - branches.push_back(instClone); - continue; - }*/ - //Clone instruction const MachineInstr *inst = I->first; MachineInstr *instClone = inst->clone(); @@ -2208,6 +2269,8 @@ void ModuloSchedulingPass::writeKernel(BasicBlock *llvmBB, MachineBasicBlock *ma //Check if we already have a final PHI value for this if(!finalPHIValue.count(mOp.getVRegValue())) { + //Only create phi if the operand def is from a stage before this one + if(schedule.defPreviousStage(mOp.getVRegValue(), I->second)) { TmpInstruction *tmp = new TmpInstruction(mOp.getVRegValue()); //Get machine code for this instruction @@ -2220,6 +2283,7 @@ void ModuloSchedulingPass::writeKernel(BasicBlock *llvmBB, MachineBasicBlock *ma //save this as our final phi finalPHIValue[mOp.getVRegValue()] = tmp; newValLocation[tmp] = machineBB; + } } else { //Use the previous final phi value @@ -2538,6 +2602,7 @@ void ModuloSchedulingPass::reconstructLoop(MachineBasicBlock *BB) { //Keep track of instructions we have already seen and their stage because //we don't want to "save" values if they are used in the kernel immediately std::map lastInstrs; + std::map phiUses; //Loop over kernel and only look at instructions from a stage > 0 //Look at its operands and save values *'s that are read @@ -2557,7 +2622,7 @@ void ModuloSchedulingPass::reconstructLoop(MachineBasicBlock *BB) { //find the value in the map if (const Value* srcI = mOp.getVRegValue()) { - if(isa(srcI) || isa(srcI) || isa(srcI)) + if(isa(srcI) || isa(srcI)) continue; //Before we declare this Value* one that we should save @@ -2582,9 +2647,27 @@ void ModuloSchedulingPass::reconstructLoop(MachineBasicBlock *BB) { } } - if(save) + if(save) { + assert(!phiUses.count(srcI) && "Did not expect to see phi use twice"); + if(isa(srcI)) + phiUses[srcI] = I->second; + valuesToSave[srcI] = std::make_pair(I->first, i); - } + + } + } + } + else if(mOp.getType() == MachineOperand::MO_VirtualRegister && mOp.isDef()) { + if (const Value* destI = mOp.getVRegValue()) { + if(!isa(destI)) + continue; + if(phiUses.count(destI)) { + if(phiUses[destI] == I->second) { + //remove from save list + valuesToSave.erase(destI); + } + } + } } if(mOp.getType() != MachineOperand::MO_VirtualRegister && mOp.isUse()) { @@ -2623,7 +2706,14 @@ void ModuloSchedulingPass::reconstructLoop(MachineBasicBlock *BB) { BasicBlock *llvmKernelBB = new BasicBlock("Kernel", (Function*) (BB->getBasicBlock()->getParent())); MachineBasicBlock *machineKernelBB = new MachineBasicBlock(llvmKernelBB); - (((MachineBasicBlock*)BB)->getParent())->getBasicBlockList().push_back(machineKernelBB); + + MachineFunction *F = (((MachineBasicBlock*)BB)->getParent()); + MachineFunction::BasicBlockListType &BL = F->getBasicBlockList(); + MachineFunction::BasicBlockListType::iterator BLI = BB; + assert(BLI != BL.end() && "Must find original BB in machine function\n"); + BL.insert(BLI,machineKernelBB); + + //(((MachineBasicBlock*)BB)->getParent())->getBasicBlockList().push_back(machineKernelBB); writeKernel(llvmKernelBB, machineKernelBB, valuesToSave, newValues, newValLocation, kernelPHIs); @@ -2833,7 +2923,8 @@ void ModuloSchedulingPass::fixBranches(std::vector &prologu for(unsigned opNum = 0; opNum < temp->getNumOperands(); ++opNum) { MachineOperand &mOp = temp->getOperand(opNum); if (mOp.getType() == MachineOperand::MO_PCRelativeDisp) { - mOp.setValueReg(llvm_prologues[0]); + if(mOp.getVRegValue() == llvmBB) + mOp.setValueReg(llvm_prologues[0]); } } } @@ -2853,7 +2944,8 @@ void ModuloSchedulingPass::fixBranches(std::vector &prologu for(unsigned opNum = 0; opNum < temp->getNumOperands(); ++opNum) { MachineOperand &mOp = temp->getOperand(opNum); if (mOp.getType() == MachineOperand::MO_PCRelativeDisp) { - mOp.setValueReg(llvmKernelBB); + if(mOp.getVRegValue() == llvmBB) + mOp.setValueReg(llvmKernelBB); } } } diff --git a/lib/Target/SparcV9/ModuloScheduling/ModuloScheduling.h b/lib/Target/SparcV9/ModuloScheduling/ModuloScheduling.h index 263a6130de6..f47de9f357b 100644 --- a/lib/Target/SparcV9/ModuloScheduling/ModuloScheduling.h +++ b/lib/Target/SparcV9/ModuloScheduling/ModuloScheduling.h @@ -98,6 +98,7 @@ namespace llvm { void findAllReccurrences(MSchedGraphNode *node, std::vector &visitedNodes, int II); void addReccurrence(std::vector &recurrence, int II, MSchedGraphNode*, MSchedGraphNode*); + void addSCC(std::vector &SCC, std::map &newNodes); void findAllCircuits(MSchedGraph *MSG, int II); bool circuit(MSchedGraphNode *v, std::vector &stack, @@ -142,7 +143,7 @@ namespace llvm { void writeKernel(BasicBlock *llvmBB, MachineBasicBlock *machineBB, std::map > &valuesToSave, std::map > &newValues, std::map &newValLocation, std::map > &kernelPHIs); - void removePHIs(const MachineBasicBlock *origBB, std::vector &prologues, std::vector &epilogues, MachineBasicBlock *kernelBB, std::map &newValLocation); + void removePHIs(const MachineBasicBlock* SB, std::vector &prologues, std::vector &epilogues, MachineBasicBlock *kernelBB, std::map &newValLocation); void connectedComponentSet(MSchedGraphNode *node, std::set &ccSet, std::set &lastNodes); diff --git a/lib/Target/SparcV9/ModuloScheduling/ModuloSchedulingSuperBlock.cpp b/lib/Target/SparcV9/ModuloScheduling/ModuloSchedulingSuperBlock.cpp index bc18027ff4d..80594c34516 100644 --- a/lib/Target/SparcV9/ModuloScheduling/ModuloSchedulingSuperBlock.cpp +++ b/lib/Target/SparcV9/ModuloScheduling/ModuloSchedulingSuperBlock.cpp @@ -17,12 +17,16 @@ #include "DependenceAnalyzer.h" #include "ModuloSchedulingSuperBlock.h" +#include "llvm/Constants.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/Passes.h" #include "llvm/Support/CFG.h" #include "llvm/Support/Debug.h" #include "llvm/Support/GraphWriter.h" +#include "llvm/Support/Timer.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/SCCIterator.h" #include "llvm/Instructions.h" #include "../MachineCodeForInstruction.h" #include "../SparcV9RegisterInfo.h" @@ -30,6 +34,8 @@ #include "../SparcV9TmpInstr.h" #include #include +#include +#include using namespace llvm; /// Create ModuloSchedulingSBPass @@ -39,9 +45,18 @@ FunctionPass *llvm::createModuloSchedulingSBPass(TargetMachine & targ) { return new ModuloSchedulingSBPass(targ); } + +#if 1 +#define TIME_REGION(VARNAME, DESC) \ + NamedRegionTimer VARNAME(DESC) +#else +#define TIME_REGION(VARNAME, DESC) +#endif + + //Graph Traits for printing out the dependence graph template -static void WriteGraphToFile(std::ostream &O, const std::string &GraphName, +static void WriteGraphToFileSB(std::ostream &O, const std::string &GraphName, const GraphType >) { std::string Filename = GraphName + ".dot"; O << "Writing '" << Filename << "'..."; @@ -58,8 +73,82 @@ namespace llvm { Statistic<> NumLoops("moduloschedSB-numLoops", "Total Number of Loops"); Statistic<> NumSB("moduloschedSB-numSuperBlocks", "Total Number of SuperBlocks"); Statistic<> BBWithCalls("modulosched-BBCalls", "Basic Blocks rejected due to calls"); - Statistic<> BBWithCondMov("modulosched-loopCondMov", "Basic Blocks rejected due to conditional moves"); + Statistic<> BBWithCondMov("modulosched-loopCondMov", + "Basic Blocks rejected due to conditional moves"); + Statistic<> SBResourceConstraint("modulosched-resourceConstraint", + "Loops constrained by resources"); + Statistic<> SBRecurrenceConstraint("modulosched-recurrenceConstraint", + "Loops constrained by recurrences"); + Statistic<> SBFinalIISum("modulosched-finalIISum", "Sum of all final II"); + Statistic<> SBIISum("modulosched-IISum", "Sum of all theoretical II"); + Statistic<> SBMSLoops("modulosched-schedLoops", "Number of loops successfully modulo-scheduled"); + Statistic<> SBNoSched("modulosched-noSched", "No schedule"); + Statistic<> SBSameStage("modulosched-sameStage", "Max stage is 0"); + Statistic<> SBBLoops("modulosched-SBBLoops", "Number single basic block loops"); + Statistic<> SBInvalid("modulosched-SBInvalid", "Number invalid superblock loops"); + Statistic<> SBValid("modulosched-SBValid", "Number valid superblock loops"); + Statistic<> SBSize("modulosched-SBSize", "Total size of all valid superblocks"); + + template<> + struct DOTGraphTraits : public DefaultDOTGraphTraits { + static std::string getGraphName(MSchedGraphSB *F) { + return "Dependence Graph"; + } + + static std::string getNodeLabel(MSchedGraphSBNode *Node, MSchedGraphSB *Graph) { + if(!Node->isPredicate()) { + if (Node->getInst()) { + std::stringstream ss; + ss << *(Node->getInst()); + return ss.str(); //((MachineInstr*)Node->getInst()); + } + else + return "No Inst"; + } + else + return "Pred Node"; + } + static std::string getEdgeSourceLabel(MSchedGraphSBNode *Node, + MSchedGraphSBNode::succ_iterator I) { + //Label each edge with the type of dependence + std::string edgelabel = ""; + switch (I.getEdge().getDepOrderType()) { + + case MSchedGraphSBEdge::TrueDep: + edgelabel = "True"; + break; + + case MSchedGraphSBEdge::AntiDep: + edgelabel = "Anti"; + break; + + case MSchedGraphSBEdge::OutputDep: + edgelabel = "Output"; + break; + + case MSchedGraphSBEdge::NonDataDep: + edgelabel = "Pred"; + break; + + default: + edgelabel = "Unknown"; + break; + } + + //FIXME + int iteDiff = I.getEdge().getIteDiff(); + std::string intStr = "(IteDiff: "; + intStr += itostr(iteDiff); + + intStr += ")"; + edgelabel += intStr; + + return edgelabel; + } + }; + bool ModuloSchedulingSBPass::runOnFunction(Function &F) { + alarm(100); bool Changed = false; //Get MachineFunction @@ -91,14 +180,97 @@ namespace llvm { continue; } - MSchedGraph *MSG = new MSchedGraph(*SB, target, indVarInstrs[*SB], DA, + MSchedGraphSB *MSG = new MSchedGraphSB(*SB, target, indVarInstrs[*SB], DA, machineTollvm[*SB]); //Write Graph out to file - DEBUG(WriteGraphToFile(std::cerr, F.getName(), MSG)); - DEBUG(MSG->print(std::cerr)); + DEBUG(WriteGraphToFileSB(std::cerr, F.getName(), MSG)); + + //Calculate Resource II + int ResMII = calculateResMII(*SB); + //Calculate Recurrence II + int RecMII = calculateRecMII(MSG, ResMII); + + DEBUG(std::cerr << "Number of reccurrences found: " << recurrenceList.size() << "\n"); + + //Our starting initiation interval is the maximum of RecMII and ResMII + if(RecMII < ResMII) + ++SBRecurrenceConstraint; + else + ++SBResourceConstraint; + + II = std::max(RecMII, ResMII); + int mII = II; + + + //Print out II, RecMII, and ResMII + DEBUG(std::cerr << "II starts out as " << II << " ( RecMII=" << RecMII << " and ResMII=" << ResMII << ")\n"); + + //Calculate Node Properties + calculateNodeAttributes(MSG, ResMII); + + //Dump node properties if in debug mode + DEBUG(for(std::map::iterator I = nodeToAttributesMap.begin(), + E = nodeToAttributesMap.end(); I !=E; ++I) { + std::cerr << "Node: " << *(I->first) << " ASAP: " << I->second.ASAP << " ALAP: " + << I->second.ALAP << " MOB: " << I->second.MOB << " Depth: " << I->second.depth + << " Height: " << I->second.height << "\n"; + }); + + + //Put nodes in order to schedule them + computePartialOrder(); + + //Dump out partial order + DEBUG(for(std::vector >::iterator I = partialOrder.begin(), + E = partialOrder.end(); I !=E; ++I) { + std::cerr << "Start set in PO\n"; + for(std::set::iterator J = I->begin(), JE = I->end(); J != JE; ++J) + std::cerr << "PO:" << **J << "\n"; + }); + + //Place nodes in final order + orderNodes(); + + //Dump out order of nodes + DEBUG(for(std::vector::iterator I = FinalNodeOrder.begin(), E = FinalNodeOrder.end(); I != E; ++I) { + std::cerr << "FO:" << **I << "\n"; + }); + + + //Finally schedule nodes + bool haveSched = computeSchedule(*SB, MSG); + + //Print out final schedule + DEBUG(schedule.print(std::cerr)); + + //Final scheduling step is to reconstruct the loop only if we actual have + //stage > 0 + if(haveSched) { + //schedule.printSchedule(std::cerr); + reconstructLoop(*SB); + ++SBMSLoops; + //Changed = true; + SBIISum += mII; + SBFinalIISum += II; + + if(schedule.getMaxStage() == 0) + ++SBSameStage; + } + else + ++SBNoSched; + + //Clear out our maps for the next basic block that is processed + nodeToAttributesMap.clear(); + partialOrder.clear(); + recurrenceList.clear(); + FinalNodeOrder.clear(); + schedule.clear(); + defMap.clear(); + } + alarm(0); return Changed; } @@ -145,63 +317,211 @@ namespace llvm { MachineBasicBlock *currentMBB = bbMap[header]; bool done = false; bool success = true; + unsigned offset = 0; + std::map indexMap; while(!done) { - - if(MachineBBisValid(currentMBB)) { - - //Loop over successors of this BB, they should be in the loop block - //and be valid - BasicBlock *next = 0; - for(succ_iterator I = succ_begin(current), E = succ_end(current); - I != E; ++I) { - if(L->contains(*I)) { - if(!next) - next = *I; - else { - done = true; - success = false; - break; - } - } - } - if(success) { - superBlock.push_back(currentMBB); - if(next == header) - done = true; - else if(!next->getSinglePredecessor()) { + //Loop over successors of this BB, they should be in the + //loop block and be valid + BasicBlock *next = 0; + for(succ_iterator I = succ_begin(current), E = succ_end(current); + I != E; ++I) { + if(L->contains(*I)) { + if(!next) + next = *I; + else { done = true; success = false; - } - else { - //Check that the next BB only has one entry - current = next; - assert(bbMap.count(current) && "LLVM BB must have corresponding Machine BB"); - currentMBB = bbMap[current]; + break; } } } - else { - done = true; - success = false; + + if(success) { + superBlock.push_back(currentMBB); + if(next == header) + done = true; + else if(!next->getSinglePredecessor()) { + done = true; + success = false; + } + else { + //Check that the next BB only has one entry + current = next; + assert(bbMap.count(current) && "LLVM BB must have corresponding Machine BB"); + currentMBB = bbMap[current]; + } } } + + + + if(success) { ++NumSB; - Worklist.push_back(superBlock); + + //Loop over all the blocks in the superblock + for(std::vector::iterator currentMBB = superBlock.begin(), MBBEnd = superBlock.end(); currentMBB != MBBEnd; ++currentMBB) { + if(!MachineBBisValid(*currentMBB, indexMap, offset)) { + success = false; + break; + } + } + } + + if(success) { + if(getIndVar(superBlock, bbMap, indexMap)) { + ++SBValid; + Worklist.push_back(superBlock); + SBSize += superBlock.size(); + } + else + ++SBInvalid; } - } - } } + + bool ModuloSchedulingSBPass::getIndVar(std::vector &superBlock, std::map &bbMap, + std::map &indexMap) { + //See if we can get induction var instructions + std::set llvmSuperBlock; + + for(unsigned i =0; i < superBlock.size(); ++i) + llvmSuperBlock.insert(superBlock[i]->getBasicBlock()); + + //Get Target machine instruction info + const TargetInstrInfo *TMI = target.getInstrInfo(); + + //Get the loop back branch + BranchInst *b = dyn_cast(((BasicBlock*) (superBlock[superBlock.size()-1])->getBasicBlock())->getTerminator()); + std::set indVar; + + if(b->isConditional()) { + //Get the condition for the branch + Value *cond = b->getCondition(); + + DEBUG(std::cerr << "Condition: " << *cond << "\n"); + + //List of instructions associated with induction variable + std::vector stack; + + //Add branch + indVar.insert(b); + + if(Instruction *I = dyn_cast(cond)) + if(bbMap.count(I->getParent())) { + if (!assocIndVar(I, indVar, stack, bbMap, superBlock[(superBlock.size()-1)]->getBasicBlock(), llvmSuperBlock)) + return false; + } + else + return false; + else + return false; + } + else { + indVar.insert(b); + } + + //Dump out instructions associate with indvar for debug reasons + DEBUG(for(std::set::iterator N = indVar.begin(), NE = indVar.end(); + N != NE; ++N) { + std::cerr << **N << "\n"; + }); + + //Create map of machine instr to llvm instr + std::map mllvm; + for(std::vector::iterator MBB = superBlock.begin(), MBE = superBlock.end(); MBB != MBE; ++MBB) { + BasicBlock *BB = (BasicBlock*) (*MBB)->getBasicBlock(); + for(BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) { + MachineCodeForInstruction & tempMvec = MachineCodeForInstruction::get(I); + for (unsigned j = 0; j < tempMvec.size(); j++) { + mllvm[tempMvec[j]] = I; + } + } + } + + //Convert list of LLVM Instructions to list of Machine instructions + std::map mIndVar; + for(std::set::iterator N = indVar.begin(), + NE = indVar.end(); N != NE; ++N) { + + //If we have a load, we can't handle this loop because + //there is no way to preserve dependences between loads + //and stores + if(isa(*N)) + return false; + + MachineCodeForInstruction & tempMvec = MachineCodeForInstruction::get(*N); + for (unsigned j = 0; j < tempMvec.size(); j++) { + MachineOpCode OC = (tempMvec[j])->getOpcode(); + if(TMI->isNop(OC)) + continue; + if(!indexMap.count(tempMvec[j])) + continue; + mIndVar[(MachineInstr*) tempMvec[j]] = indexMap[(MachineInstr*) tempMvec[j]]; + DEBUG(std::cerr << *(tempMvec[j]) << " at index " << indexMap[(MachineInstr*) tempMvec[j]] << "\n"); + } + } + + //Put into a map for future access + indVarInstrs[superBlock] = mIndVar; + machineTollvm[superBlock] = mllvm; + + return true; + + } + + bool ModuloSchedulingSBPass::assocIndVar(Instruction *I, + std::set &indVar, + std::vector &stack, + std::map &bbMap, + const BasicBlock *last, std::set &llvmSuperBlock) { + + stack.push_back(I); + + //If this is a phi node, check if its the canonical indvar + if(PHINode *PN = dyn_cast(I)) { + if(llvmSuperBlock.count(PN->getParent())) { + if (Instruction *Inc = + dyn_cast(PN->getIncomingValueForBlock(last))) + if (Inc->getOpcode() == Instruction::Add && Inc->getOperand(0) == PN) + if (ConstantInt *CI = dyn_cast(Inc->getOperand(1))) + if (CI->equalsInt(1)) { + //We have found the indvar, so add the stack, and inc instruction to the set + indVar.insert(stack.begin(), stack.end()); + indVar.insert(Inc); + stack.pop_back(); + return true; + } + return false; + } + } + else { + //Loop over each of the instructions operands, check if they are an instruction and in this BB + for(unsigned i = 0; i < I->getNumOperands(); ++i) { + if(Instruction *N = dyn_cast(I->getOperand(i))) { + if(bbMap.count(N->getParent())) + if(!assocIndVar(N, indVar, stack, bbMap, last, llvmSuperBlock)) + return false; + } + } + } + + stack.pop_back(); + return true; + } + + /// This function checks if a Machine Basic Block is valid for modulo /// scheduling. This means that it has no control flow (if/else or /// calls) in the block. Currently ModuloScheduling only works on /// single basic block loops. - bool ModuloSchedulingSBPass::MachineBBisValid(const MachineBasicBlock *BI) { + bool ModuloSchedulingSBPass::MachineBBisValid(const MachineBasicBlock *BI, + std::map &indexMap, + unsigned &offset) { //Check size of our basic block.. make sure we have more then just the terminator in it if(BI->getBasicBlock()->size() == 1) @@ -210,9 +530,6 @@ namespace llvm { //Get Target machine instruction info const TargetInstrInfo *TMI = target.getInstrInfo(); - //Check each instruction and look for calls, keep map to get index later - std::map indexMap; - unsigned count = 0; for(MachineBasicBlock::const_iterator I = BI->begin(), E = BI->end(); I != E; ++I) { //Get opcode to check instruction type @@ -236,7 +553,7 @@ namespace llvm { return false; } - indexMap[I] = count; + indexMap[I] = count + offset; if(TMI->isNop(OC)) continue; @@ -244,6 +561,8 @@ namespace llvm { ++count; } + offset += count; + return true; } } @@ -258,10 +577,16 @@ bool ModuloSchedulingSBPass::CreateDefMap(std::vector for(unsigned opNum = 0; opNum < I->getNumOperands(); ++opNum) { const MachineOperand &mOp = I->getOperand(opNum); if(mOp.getType() == MachineOperand::MO_VirtualRegister && mOp.isDef()) { - + Value *V = mOp.getVRegValue(); //assert if this is the second def we have seen - assert(!defMap.count(mOp.getVRegValue()) && "Def already in the map"); - defMap[mOp.getVRegValue()] = (MachineInstr*) &*I; + if(defMap.count(V) && isa(V)) + DEBUG(std::cerr << "FIXME: Dup def for phi!\n"); + else { + //assert(!defMap.count(V) && "Def already in the map"); + if(defMap.count(V)) + return false; + defMap[V] = (MachineInstr*) &*I; + } } //See if we can use this Value* as our defaultInst @@ -280,3 +605,2554 @@ bool ModuloSchedulingSBPass::CreateDefMap(std::vector return true; } + + +//ResMII is calculated by determining the usage count for each resource +//and using the maximum. +//FIXME: In future there should be a way to get alternative resources +//for each instruction +int ModuloSchedulingSBPass::calculateResMII(std::vector &superBlock) { + + TIME_REGION(X, "calculateResMII"); + + const TargetInstrInfo *mii = target.getInstrInfo(); + const TargetSchedInfo *msi = target.getSchedInfo(); + + int ResMII = 0; + + //Map to keep track of usage count of each resource + std::map resourceUsageCount; + + for(std::vector::iterator BI = superBlock.begin(), BE = superBlock.end(); BI != BE; ++BI) { + for(MachineBasicBlock::const_iterator I = (*BI)->begin(), E = (*BI)->end(); I != E; ++I) { + + //Get resource usage for this instruction + InstrRUsage rUsage = msi->getInstrRUsage(I->getOpcode()); + std::vector > resources = rUsage.resourcesByCycle; + + //Loop over resources in each cycle and increments their usage count + for(unsigned i=0; i < resources.size(); ++i) + for(unsigned j=0; j < resources[i].size(); ++j) { + if(!resourceUsageCount.count(resources[i][j])) { + resourceUsageCount[resources[i][j]] = 1; + } + else { + resourceUsageCount[resources[i][j]] = resourceUsageCount[resources[i][j]] + 1; + } + } + } + } + + //Find maximum usage count + + //Get max number of instructions that can be issued at once. (FIXME) + int issueSlots = msi->maxNumIssueTotal; + + for(std::map::iterator RB = resourceUsageCount.begin(), RE = resourceUsageCount.end(); RB != RE; ++RB) { + + //Get the total number of the resources in our cpu + int resourceNum = CPUResource::getCPUResource(RB->first)->maxNumUsers; + + //Get total usage count for this resources + unsigned usageCount = RB->second; + + //Divide the usage count by either the max number we can issue or the number of + //resources (whichever is its upper bound) + double finalUsageCount; + DEBUG(std::cerr << "Resource Num: " << RB->first << " Usage: " << usageCount << " TotalNum: " << resourceNum << "\n"); + + if( resourceNum <= issueSlots) + finalUsageCount = ceil(1.0 * usageCount / resourceNum); + else + finalUsageCount = ceil(1.0 * usageCount / issueSlots); + + + //Only keep track of the max + ResMII = std::max( (int) finalUsageCount, ResMII); + + } + + return ResMII; + +} + +/// calculateRecMII - Calculates the value of the highest recurrence +/// By value we mean the total latency/distance +int ModuloSchedulingSBPass::calculateRecMII(MSchedGraphSB *graph, int MII) { + + TIME_REGION(X, "calculateRecMII"); + + findAllCircuits(graph, MII); + int RecMII = 0; + + for(std::set > >::iterator I = recurrenceList.begin(), E=recurrenceList.end(); I !=E; ++I) { + RecMII = std::max(RecMII, I->first); + } + + return MII; +} + +int CircCountSB; + +void ModuloSchedulingSBPass::unblock(MSchedGraphSBNode *u, std::set &blocked, + std::map > &B) { + + //Unblock u + DEBUG(std::cerr << "Unblocking: " << *u << "\n"); + blocked.erase(u); + + //std::set toErase; + while (!B[u].empty()) { + MSchedGraphSBNode *W = *B[u].begin(); + B[u].erase(W); + //toErase.insert(*W); + DEBUG(std::cerr << "Removed: " << *W << "from B-List\n"); + if(blocked.count(W)) + unblock(W, blocked, B); + } + +} + +void ModuloSchedulingSBPass::addSCC(std::vector &SCC, std::map &newNodes) { + + int totalDelay = 0; + int totalDistance = 0; + std::vector recc; + MSchedGraphSBNode *start = 0; + MSchedGraphSBNode *end = 0; + + //Loop over recurrence, get delay and distance + for(std::vector::iterator N = SCC.begin(), NE = SCC.end(); N != NE; ++N) { + DEBUG(std::cerr << **N << "\n"); + totalDelay += (*N)->getLatency(); + + for(unsigned i = 0; i < (*N)->succ_size(); ++i) { + MSchedGraphSBEdge *edge = (*N)->getSuccessor(i); + if(find(SCC.begin(), SCC.end(), edge->getDest()) != SCC.end()) { + totalDistance += edge->getIteDiff(); + if(edge->getIteDiff() > 0) + if(!start && !end) { + start = *N; + end = edge->getDest(); + } + + } + } + + + //Get the original node + recc.push_back(newNodes[*N]); + + + } + + DEBUG(std::cerr << "End Recc\n"); + + + assert( (start && end) && "Must have start and end node to ignore edge for SCC"); + + if(start && end) { + //Insert reccurrence into the list + DEBUG(std::cerr << "Ignore Edge from!!: " << *start << " to " << *end << "\n"); + edgesToIgnore.insert(std::make_pair(newNodes[start], (newNodes[end])->getInEdgeNum(newNodes[start]))); + } + + int lastII = totalDelay / totalDistance; + + + recurrenceList.insert(std::make_pair(lastII, recc)); + +} + +bool ModuloSchedulingSBPass::circuit(MSchedGraphSBNode *v, std::vector &stack, + std::set &blocked, std::vector &SCC, + MSchedGraphSBNode *s, std::map > &B, + int II, std::map &newNodes) { + bool f = false; + + DEBUG(std::cerr << "Finding Circuits Starting with: ( " << v << ")"<< *v << "\n"); + + //Push node onto the stack + stack.push_back(v); + + //block this node + blocked.insert(v); + + //Loop over all successors of node v that are in the scc, create Adjaceny list + std::set AkV; + for(MSchedGraphSBNode::succ_iterator I = v->succ_begin(), E = v->succ_end(); I != E; ++I) { + if((std::find(SCC.begin(), SCC.end(), *I) != SCC.end())) { + AkV.insert(*I); + } + } + + for(std::set::iterator I = AkV.begin(), E = AkV.end(); I != E; ++I) { + if(*I == s) { + //We have a circuit, so add it to our list + addRecc(stack, newNodes); + f = true; + } + else if(!blocked.count(*I)) { + if(circuit(*I, stack, blocked, SCC, s, B, II, newNodes)) + f = true; + } + else + DEBUG(std::cerr << "Blocked: " << **I << "\n"); + } + + + if(f) { + unblock(v, blocked, B); + } + else { + for(std::set::iterator I = AkV.begin(), E = AkV.end(); I != E; ++I) + B[*I].insert(v); + + } + + //Pop v + stack.pop_back(); + + return f; + +} + +void ModuloSchedulingSBPass::addRecc(std::vector &stack, std::map &newNodes) { + std::vector recc; + //Dump recurrence for now + DEBUG(std::cerr << "Starting Recc\n"); + + int totalDelay = 0; + int totalDistance = 0; + MSchedGraphSBNode *lastN = 0; + MSchedGraphSBNode *start = 0; + MSchedGraphSBNode *end = 0; + + //Loop over recurrence, get delay and distance + for(std::vector::iterator N = stack.begin(), NE = stack.end(); N != NE; ++N) { + DEBUG(std::cerr << **N << "\n"); + totalDelay += (*N)->getLatency(); + if(lastN) { + int iteDiff = (*N)->getInEdge(lastN).getIteDiff(); + totalDistance += iteDiff; + + if(iteDiff > 0) { + start = lastN; + end = *N; + } + } + //Get the original node + lastN = *N; + recc.push_back(newNodes[*N]); + + + } + + //Get the loop edge + totalDistance += lastN->getIteDiff(*stack.begin()); + + DEBUG(std::cerr << "End Recc\n"); + CircCountSB++; + + if(start && end) { + //Insert reccurrence into the list + DEBUG(std::cerr << "Ignore Edge from!!: " << *start << " to " << *end << "\n"); + edgesToIgnore.insert(std::make_pair(newNodes[start], (newNodes[end])->getInEdgeNum(newNodes[start]))); + } + else { + //Insert reccurrence into the list + DEBUG(std::cerr << "Ignore Edge from: " << *lastN << " to " << **stack.begin() << "\n"); + edgesToIgnore.insert(std::make_pair(newNodes[lastN], newNodes[(*stack.begin())]->getInEdgeNum(newNodes[lastN]))); + + } + //Adjust II until we get close to the inequality delay - II*distance <= 0 + int RecMII = II; //Starting value + int value = totalDelay-(RecMII * totalDistance); + int lastII = II; + while(value < 0) { + + lastII = RecMII; + RecMII--; + value = totalDelay-(RecMII * totalDistance); + } + + recurrenceList.insert(std::make_pair(lastII, recc)); + +} + + +void ModuloSchedulingSBPass::findAllCircuits(MSchedGraphSB *g, int II) { + + CircCountSB = 0; + + //Keep old to new node mapping information + std::map newNodes; + + //copy the graph + MSchedGraphSB *MSG = new MSchedGraphSB(*g, newNodes); + + DEBUG(std::cerr << "Finding All Circuits\n"); + + //Set of blocked nodes + std::set blocked; + + //Stack holding current circuit + std::vector stack; + + //Map for B Lists + std::map > B; + + //current node + MSchedGraphSBNode *s; + + + //Iterate over the graph until its down to one node or empty + while(MSG->size() > 1) { + + //Write Graph out to file + //WriteGraphToFile(std::cerr, "Graph" + utostr(MSG->size()), MSG); + + DEBUG(std::cerr << "Graph Size: " << MSG->size() << "\n"); + DEBUG(std::cerr << "Finding strong component Vk with least vertex\n"); + + //Iterate over all the SCCs in the graph + std::set Visited; + std::vector Vk; + MSchedGraphSBNode* s = 0; + int numEdges = 0; + + //Find scc with the least vertex + for (MSchedGraphSB::iterator GI = MSG->begin(), E = MSG->end(); GI != E; ++GI) + if (Visited.insert(GI->second).second) { + for (scc_iterator SCCI = scc_begin(GI->second), + E = scc_end(GI->second); SCCI != E; ++SCCI) { + std::vector &nextSCC = *SCCI; + + if (Visited.insert(nextSCC[0]).second) { + Visited.insert(nextSCC.begin()+1, nextSCC.end()); + + if(nextSCC.size() > 1) { + DEBUG(std::cerr << "SCC size: " << nextSCC.size() << "\n"); + + for(unsigned i = 0; i < nextSCC.size(); ++i) { + //Loop over successor and see if in scc, then count edge + MSchedGraphSBNode *node = nextSCC[i]; + for(MSchedGraphSBNode::succ_iterator S = node->succ_begin(), SE = node->succ_end(); S != SE; ++S) { + if(find(nextSCC.begin(), nextSCC.end(), *S) != nextSCC.end()) + numEdges++; + } + } + DEBUG(std::cerr << "Num Edges: " << numEdges << "\n"); + } + + //Ignore self loops + if(nextSCC.size() > 1) { + + //Get least vertex in Vk + if(!s) { + s = nextSCC[0]; + Vk = nextSCC; + } + + for(unsigned i = 0; i < nextSCC.size(); ++i) { + if(nextSCC[i] < s) { + s = nextSCC[i]; + Vk = nextSCC; + } + } + } + } + } + } + + + + //Process SCC + DEBUG(for(std::vector::iterator N = Vk.begin(), NE = Vk.end(); + N != NE; ++N) { std::cerr << *((*N)->getInst()); }); + + //Iterate over all nodes in this scc + for(std::vector::iterator N = Vk.begin(), NE = Vk.end(); + N != NE; ++N) { + blocked.erase(*N); + B[*N].clear(); + } + if(Vk.size() > 1) { + if(numEdges < 98) + circuit(s, stack, blocked, Vk, s, B, II, newNodes); + else + addSCC(Vk, newNodes); + + + //Delete nodes from the graph + //Find all nodes up to s and delete them + std::vector nodesToRemove; + nodesToRemove.push_back(s); + for(MSchedGraphSB::iterator N = MSG->begin(), NE = MSG->end(); N != NE; ++N) { + if(N->second < s ) + nodesToRemove.push_back(N->second); + } + for(std::vector::iterator N = nodesToRemove.begin(), NE = nodesToRemove.end(); N != NE; ++N) { + DEBUG(std::cerr << "Deleting Node: " << **N << "\n"); + MSG->deleteNode(*N); + } + } + else + break; + } + DEBUG(std::cerr << "Num Circuits found: " << CircCountSB << "\n"); +} +/// calculateNodeAttributes - The following properties are calculated for +/// each node in the dependence graph: ASAP, ALAP, Depth, Height, and +/// MOB. +void ModuloSchedulingSBPass::calculateNodeAttributes(MSchedGraphSB *graph, int MII) { + + TIME_REGION(X, "calculateNodeAttributes"); + + assert(nodeToAttributesMap.empty() && "Node attribute map was not cleared"); + + //Loop over the nodes and add them to the map + for(MSchedGraphSB::iterator I = graph->begin(), E = graph->end(); I != E; ++I) { + + DEBUG(std::cerr << "Inserting node into attribute map: " << *I->second << "\n"); + + //Assert if its already in the map + assert(nodeToAttributesMap.count(I->second) == 0 && + "Node attributes are already in the map"); + + //Put into the map with default attribute values + nodeToAttributesMap[I->second] = MSNodeSBAttributes(); + } + + //Create set to deal with reccurrences + std::set visitedNodes; + + //Now Loop over map and calculate the node attributes + for(std::map::iterator I = nodeToAttributesMap.begin(), E = nodeToAttributesMap.end(); I != E; ++I) { + calculateASAP(I->first, MII, (MSchedGraphSBNode*) 0); + visitedNodes.clear(); + } + + int maxASAP = findMaxASAP(); + //Calculate ALAP which depends on ASAP being totally calculated + for(std::map::iterator I = nodeToAttributesMap.begin(), E = nodeToAttributesMap.end(); I != E; ++I) { + calculateALAP(I->first, MII, maxASAP, (MSchedGraphSBNode*) 0); + visitedNodes.clear(); + } + + //Calculate MOB which depends on ASAP being totally calculated, also do depth and height + for(std::map::iterator I = nodeToAttributesMap.begin(), E = nodeToAttributesMap.end(); I != E; ++I) { + (I->second).MOB = std::max(0,(I->second).ALAP - (I->second).ASAP); + + DEBUG(std::cerr << "MOB: " << (I->second).MOB << " (" << *(I->first) << ")\n"); + calculateDepth(I->first, (MSchedGraphSBNode*) 0); + calculateHeight(I->first, (MSchedGraphSBNode*) 0); + } + + +} + +/// ignoreEdge - Checks to see if this edge of a recurrence should be ignored or not +bool ModuloSchedulingSBPass::ignoreEdge(MSchedGraphSBNode *srcNode, MSchedGraphSBNode *destNode) { + if(destNode == 0 || srcNode ==0) + return false; + + bool findEdge = edgesToIgnore.count(std::make_pair(srcNode, destNode->getInEdgeNum(srcNode))); + + DEBUG(std::cerr << "Ignoring edge? from: " << *srcNode << " to " << *destNode << "\n"); + + return findEdge; +} + + +/// calculateASAP - Calculates the +int ModuloSchedulingSBPass::calculateASAP(MSchedGraphSBNode *node, int MII, MSchedGraphSBNode *destNode) { + + DEBUG(std::cerr << "Calculating ASAP for " << *node << "\n"); + + //Get current node attributes + MSNodeSBAttributes &attributes = nodeToAttributesMap.find(node)->second; + + if(attributes.ASAP != -1) + return attributes.ASAP; + + int maxPredValue = 0; + + //Iterate over all of the predecessors and find max + for(MSchedGraphSBNode::pred_iterator P = node->pred_begin(), E = node->pred_end(); P != E; ++P) { + + //Only process if we are not ignoring the edge + if(!ignoreEdge(*P, node)) { + int predASAP = -1; + predASAP = calculateASAP(*P, MII, node); + + assert(predASAP != -1 && "ASAP has not been calculated"); + int iteDiff = node->getInEdge(*P).getIteDiff(); + + int currentPredValue = predASAP + (*P)->getLatency() - (iteDiff * MII); + DEBUG(std::cerr << "pred ASAP: " << predASAP << ", iteDiff: " << iteDiff << ", PredLatency: " << (*P)->getLatency() << ", Current ASAP pred: " << currentPredValue << "\n"); + maxPredValue = std::max(maxPredValue, currentPredValue); + } + } + + attributes.ASAP = maxPredValue; + + DEBUG(std::cerr << "ASAP: " << attributes.ASAP << " (" << *node << ")\n"); + + return maxPredValue; +} + + +int ModuloSchedulingSBPass::calculateALAP(MSchedGraphSBNode *node, int MII, + int maxASAP, MSchedGraphSBNode *srcNode) { + + DEBUG(std::cerr << "Calculating ALAP for " << *node << "\n"); + + MSNodeSBAttributes &attributes = nodeToAttributesMap.find(node)->second; + + if(attributes.ALAP != -1) + return attributes.ALAP; + + if(node->hasSuccessors()) { + + //Trying to deal with the issue where the node has successors, but + //we are ignoring all of the edges to them. So this is my hack for + //now.. there is probably a more elegant way of doing this (FIXME) + bool processedOneEdge = false; + + //FIXME, set to something high to start + int minSuccValue = 9999999; + + //Iterate over all of the predecessors and fine max + for(MSchedGraphSBNode::succ_iterator P = node->succ_begin(), + E = node->succ_end(); P != E; ++P) { + + //Only process if we are not ignoring the edge + if(!ignoreEdge(node, *P)) { + processedOneEdge = true; + int succALAP = -1; + succALAP = calculateALAP(*P, MII, maxASAP, node); + + assert(succALAP != -1 && "Successors ALAP should have been caclulated"); + + int iteDiff = P.getEdge().getIteDiff(); + + int currentSuccValue = succALAP - node->getLatency() + iteDiff * MII; + + DEBUG(std::cerr << "succ ALAP: " << succALAP << ", iteDiff: " << iteDiff << ", SuccLatency: " << (*P)->getLatency() << ", Current ALAP succ: " << currentSuccValue << "\n"); + + minSuccValue = std::min(minSuccValue, currentSuccValue); + } + } + + if(processedOneEdge) + attributes.ALAP = minSuccValue; + + else + attributes.ALAP = maxASAP; + } + else + attributes.ALAP = maxASAP; + + DEBUG(std::cerr << "ALAP: " << attributes.ALAP << " (" << *node << ")\n"); + + if(attributes.ALAP < 0) + attributes.ALAP = 0; + + return attributes.ALAP; +} + +int ModuloSchedulingSBPass::findMaxASAP() { + int maxASAP = 0; + + for(std::map::iterator I = nodeToAttributesMap.begin(), + E = nodeToAttributesMap.end(); I != E; ++I) + maxASAP = std::max(maxASAP, I->second.ASAP); + return maxASAP; +} + + +int ModuloSchedulingSBPass::calculateHeight(MSchedGraphSBNode *node,MSchedGraphSBNode *srcNode) { + + MSNodeSBAttributes &attributes = nodeToAttributesMap.find(node)->second; + + if(attributes.height != -1) + return attributes.height; + + int maxHeight = 0; + + //Iterate over all of the predecessors and find max + for(MSchedGraphSBNode::succ_iterator P = node->succ_begin(), + E = node->succ_end(); P != E; ++P) { + + + if(!ignoreEdge(node, *P)) { + int succHeight = calculateHeight(*P, node); + + assert(succHeight != -1 && "Successors Height should have been caclulated"); + + int currentHeight = succHeight + node->getLatency(); + maxHeight = std::max(maxHeight, currentHeight); + } + } + attributes.height = maxHeight; + DEBUG(std::cerr << "Height: " << attributes.height << " (" << *node << ")\n"); + return maxHeight; +} + + +int ModuloSchedulingSBPass::calculateDepth(MSchedGraphSBNode *node, + MSchedGraphSBNode *destNode) { + + MSNodeSBAttributes &attributes = nodeToAttributesMap.find(node)->second; + + if(attributes.depth != -1) + return attributes.depth; + + int maxDepth = 0; + + //Iterate over all of the predecessors and fine max + for(MSchedGraphSBNode::pred_iterator P = node->pred_begin(), E = node->pred_end(); P != E; ++P) { + + if(!ignoreEdge(*P, node)) { + int predDepth = -1; + predDepth = calculateDepth(*P, node); + + assert(predDepth != -1 && "Predecessors ASAP should have been caclulated"); + + int currentDepth = predDepth + (*P)->getLatency(); + maxDepth = std::max(maxDepth, currentDepth); + } + } + attributes.depth = maxDepth; + + DEBUG(std::cerr << "Depth: " << attributes.depth << " (" << *node << "*)\n"); + return maxDepth; +} + +void ModuloSchedulingSBPass::computePartialOrder() { + + TIME_REGION(X, "calculatePartialOrder"); + + DEBUG(std::cerr << "Computing Partial Order\n"); + + //Steps to add a recurrence to the partial order 1) Find reccurrence + //with the highest RecMII. Add it to the partial order. 2) For each + //recurrence with decreasing RecMII, add it to the partial order + //along with any nodes that connect this recurrence to recurrences + //already in the partial order + for(std::set > >::reverse_iterator + I = recurrenceList.rbegin(), E=recurrenceList.rend(); I !=E; ++I) { + + std::set new_recurrence; + + //Loop through recurrence and remove any nodes already in the partial order + for(std::vector::const_iterator N = I->second.begin(), + NE = I->second.end(); N != NE; ++N) { + + bool found = false; + for(std::vector >::iterator PO = partialOrder.begin(), + PE = partialOrder.end(); PO != PE; ++PO) { + if(PO->count(*N)) + found = true; + } + + //Check if its a branch, and remove to handle special + if(!found) { + new_recurrence.insert(*N); + } + + } + + + if(new_recurrence.size() > 0) { + + std::vector path; + std::set nodesToAdd; + + //Dump recc we are dealing with (minus nodes already in PO) + DEBUG(std::cerr << "Recc: "); + DEBUG(for(std::set::iterator R = new_recurrence.begin(), RE = new_recurrence.end(); R != RE; ++R) { std::cerr << **R ; }); + + //Add nodes that connect this recurrence to recurrences in the partial path + for(std::set::iterator N = new_recurrence.begin(), + NE = new_recurrence.end(); N != NE; ++N) + searchPath(*N, path, nodesToAdd, new_recurrence); + + //Add nodes to this recurrence if they are not already in the partial order + for(std::set::iterator N = nodesToAdd.begin(), NE = nodesToAdd.end(); + N != NE; ++N) { + bool found = false; + for(std::vector >::iterator PO = partialOrder.begin(), + PE = partialOrder.end(); PO != PE; ++PO) { + if(PO->count(*N)) + found = true; + } + if(!found) { + assert("FOUND CONNECTOR"); + new_recurrence.insert(*N); + } + } + + partialOrder.push_back(new_recurrence); + } + } + + //Add any nodes that are not already in the partial order + //Add them in a set, one set per connected component + std::set lastNodes; + std::set noPredNodes; + for(std::map::iterator I = nodeToAttributesMap.begin(), + E = nodeToAttributesMap.end(); I != E; ++I) { + + bool found = false; + + //Check if its already in our partial order, if not add it to the final vector + for(std::vector >::iterator PO = partialOrder.begin(), + PE = partialOrder.end(); PO != PE; ++PO) { + if(PO->count(I->first)) + found = true; + } + if(!found) + lastNodes.insert(I->first); + } + + //For each node w/out preds, see if there is a path to one of the + //recurrences, and if so add them to that current recc + /*for(std::set::iterator N = noPredNodes.begin(), NE = noPredNodes.end(); + N != NE; ++N) { + DEBUG(std::cerr << "No Pred Path from: " << **N << "\n"); + for(std::vector >::iterator PO = partialOrder.begin(), + PE = partialOrder.end(); PO != PE; ++PO) { + std::vector path; + pathToRecc(*N, path, *PO, lastNodes); + } + }*/ + + + //Break up remaining nodes that are not in the partial order + ///into their connected compoenents + while(lastNodes.size() > 0) { + std::set ccSet; + connectedComponentSet(*(lastNodes.begin()),ccSet, lastNodes); + if(ccSet.size() > 0) + partialOrder.push_back(ccSet); + } + +} + +void ModuloSchedulingSBPass::connectedComponentSet(MSchedGraphSBNode *node, std::set &ccSet, std::set &lastNodes) { + + //Add to final set + if( !ccSet.count(node) && lastNodes.count(node)) { + lastNodes.erase(node); + ccSet.insert(node); + } + else + return; + + //Loop over successors and recurse if we have not seen this node before + for(MSchedGraphSBNode::succ_iterator node_succ = node->succ_begin(), end=node->succ_end(); node_succ != end; ++node_succ) { + connectedComponentSet(*node_succ, ccSet, lastNodes); + } + +} + +void ModuloSchedulingSBPass::searchPath(MSchedGraphSBNode *node, + std::vector &path, + std::set &nodesToAdd, + std::set &new_reccurrence) { + //Push node onto the path + path.push_back(node); + + //Loop over all successors and see if there is a path from this node to + //a recurrence in the partial order, if so.. add all nodes to be added to recc + for(MSchedGraphSBNode::succ_iterator S = node->succ_begin(), SE = node->succ_end(); S != SE; + ++S) { + + //Check if we should ignore this edge first + if(ignoreEdge(node,*S)) + continue; + + //check if successor is in this recurrence, we will get to it eventually + if(new_reccurrence.count(*S)) + continue; + + //If this node exists in a recurrence already in the partial + //order, then add all nodes in the path to the set of nodes to add + //Check if its already in our partial order, if not add it to the + //final vector + bool found = false; + for(std::vector >::iterator PO = partialOrder.begin(), + PE = partialOrder.end(); PO != PE; ++PO) { + + if(PO->count(*S)) { + found = true; + break; + } + } + + if(!found) { + nodesToAdd.insert(*S); + searchPath(*S, path, nodesToAdd, new_reccurrence); + } + } + + //Pop Node off the path + path.pop_back(); +} + +void dumpIntersection(std::set &IntersectCurrent) { + std::cerr << "Intersection ("; + for(std::set::iterator I = IntersectCurrent.begin(), E = IntersectCurrent.end(); I != E; ++I) + std::cerr << **I << ", "; + std::cerr << ")\n"; +} + +void ModuloSchedulingSBPass::orderNodes() { + + TIME_REGION(X, "orderNodes"); + + int BOTTOM_UP = 0; + int TOP_DOWN = 1; + + //Set default order + int order = BOTTOM_UP; + + //Loop over and find all pred nodes and schedule them first + /*for(std::vector >::iterator CurrentSet = partialOrder.begin(), E= partialOrder.end(); CurrentSet != E; ++CurrentSet) { + for(std::set::iterator N = CurrentSet->begin(), NE = CurrentSet->end(); N != NE; ++N) + if((*N)->isPredicate()) { + FinalNodeOrder.push_back(*N); + CurrentSet->erase(*N); + } + }*/ + + + + //Loop over all the sets and place them in the final node order + for(std::vector >::iterator CurrentSet = partialOrder.begin(), E= partialOrder.end(); CurrentSet != E; ++CurrentSet) { + + DEBUG(std::cerr << "Processing set in S\n"); + DEBUG(dumpIntersection(*CurrentSet)); + + //Result of intersection + std::set IntersectCurrent; + + predIntersect(*CurrentSet, IntersectCurrent); + + //If the intersection of predecessor and current set is not empty + //sort nodes bottom up + if(IntersectCurrent.size() != 0) { + DEBUG(std::cerr << "Final Node Order Predecessors and Current Set interesection is NOT empty\n"); + order = BOTTOM_UP; + } + //If empty, use successors + else { + DEBUG(std::cerr << "Final Node Order Predecessors and Current Set interesection is empty\n"); + + succIntersect(*CurrentSet, IntersectCurrent); + + //sort top-down + if(IntersectCurrent.size() != 0) { + DEBUG(std::cerr << "Final Node Order Successors and Current Set interesection is NOT empty\n"); + order = TOP_DOWN; + } + else { + DEBUG(std::cerr << "Final Node Order Successors and Current Set interesection is empty\n"); + //Find node with max ASAP in current Set + MSchedGraphSBNode *node; + int maxASAP = 0; + DEBUG(std::cerr << "Using current set of size " << CurrentSet->size() << "to find max ASAP\n"); + for(std::set::iterator J = CurrentSet->begin(), JE = CurrentSet->end(); J != JE; ++J) { + //Get node attributes + MSNodeSBAttributes nodeAttr= nodeToAttributesMap.find(*J)->second; + //assert(nodeAttr != nodeToAttributesMap.end() && "Node not in attributes map!"); + + if(maxASAP <= nodeAttr.ASAP) { + maxASAP = nodeAttr.ASAP; + node = *J; + } + } + assert(node != 0 && "In node ordering node should not be null"); + IntersectCurrent.insert(node); + order = BOTTOM_UP; + } + } + + //Repeat until all nodes are put into the final order from current set + while(IntersectCurrent.size() > 0) { + + if(order == TOP_DOWN) { + DEBUG(std::cerr << "Order is TOP DOWN\n"); + + while(IntersectCurrent.size() > 0) { + DEBUG(std::cerr << "Intersection is not empty, so find heighest height\n"); + + int MOB = 0; + int height = 0; + MSchedGraphSBNode *highestHeightNode = *(IntersectCurrent.begin()); + + //Find node in intersection with highest heigh and lowest MOB + for(std::set::iterator I = IntersectCurrent.begin(), + E = IntersectCurrent.end(); I != E; ++I) { + + //Get current nodes properties + MSNodeSBAttributes nodeAttr= nodeToAttributesMap.find(*I)->second; + + if(height < nodeAttr.height) { + highestHeightNode = *I; + height = nodeAttr.height; + MOB = nodeAttr.MOB; + } + else if(height == nodeAttr.height) { + if(MOB > nodeAttr.height) { + highestHeightNode = *I; + height = nodeAttr.height; + MOB = nodeAttr.MOB; + } + } + } + + //Append our node with greatest height to the NodeOrder + if(std::find(FinalNodeOrder.begin(), FinalNodeOrder.end(), highestHeightNode) == FinalNodeOrder.end()) { + DEBUG(std::cerr << "Adding node to Final Order: " << *highestHeightNode << "\n"); + FinalNodeOrder.push_back(highestHeightNode); + } + + //Remove V from IntersectOrder + IntersectCurrent.erase(std::find(IntersectCurrent.begin(), + IntersectCurrent.end(), highestHeightNode)); + + + //Intersect V's successors with CurrentSet + for(MSchedGraphSBNode::succ_iterator P = highestHeightNode->succ_begin(), + E = highestHeightNode->succ_end(); P != E; ++P) { + //if(lower_bound(CurrentSet->begin(), + // CurrentSet->end(), *P) != CurrentSet->end()) { + if(std::find(CurrentSet->begin(), CurrentSet->end(), *P) != CurrentSet->end()) { + if(ignoreEdge(highestHeightNode, *P)) + continue; + //If not already in Intersect, add + if(!IntersectCurrent.count(*P)) + IntersectCurrent.insert(*P); + } + } + } //End while loop over Intersect Size + + //Change direction + order = BOTTOM_UP; + + //Reset Intersect to reflect changes in OrderNodes + IntersectCurrent.clear(); + predIntersect(*CurrentSet, IntersectCurrent); + + } //End If TOP_DOWN + + //Begin if BOTTOM_UP + else { + DEBUG(std::cerr << "Order is BOTTOM UP\n"); + while(IntersectCurrent.size() > 0) { + DEBUG(std::cerr << "Intersection of size " << IntersectCurrent.size() << ", finding highest depth\n"); + + //dump intersection + DEBUG(dumpIntersection(IntersectCurrent)); + //Get node with highest depth, if a tie, use one with lowest + //MOB + int MOB = 0; + int depth = 0; + MSchedGraphSBNode *highestDepthNode = *(IntersectCurrent.begin()); + + for(std::set::iterator I = IntersectCurrent.begin(), + E = IntersectCurrent.end(); I != E; ++I) { + //Find node attribute in graph + MSNodeSBAttributes nodeAttr= nodeToAttributesMap.find(*I)->second; + + if(depth < nodeAttr.depth) { + highestDepthNode = *I; + depth = nodeAttr.depth; + MOB = nodeAttr.MOB; + } + else if(depth == nodeAttr.depth) { + if(MOB > nodeAttr.MOB) { + highestDepthNode = *I; + depth = nodeAttr.depth; + MOB = nodeAttr.MOB; + } + } + } + + + + //Append highest depth node to the NodeOrder + if(std::find(FinalNodeOrder.begin(), FinalNodeOrder.end(), highestDepthNode) == FinalNodeOrder.end()) { + DEBUG(std::cerr << "Adding node to Final Order: " << *highestDepthNode << "\n"); + FinalNodeOrder.push_back(highestDepthNode); + } + //Remove heightestDepthNode from IntersectOrder + IntersectCurrent.erase(highestDepthNode); + + + //Intersect heightDepthNode's pred with CurrentSet + for(MSchedGraphSBNode::pred_iterator P = highestDepthNode->pred_begin(), + E = highestDepthNode->pred_end(); P != E; ++P) { + if(CurrentSet->count(*P)) { + if(ignoreEdge(*P, highestDepthNode)) + continue; + + //If not already in Intersect, add + if(!IntersectCurrent.count(*P)) + IntersectCurrent.insert(*P); + } + } + + } //End while loop over Intersect Size + + //Change order + order = TOP_DOWN; + + //Reset IntersectCurrent to reflect changes in OrderNodes + IntersectCurrent.clear(); + succIntersect(*CurrentSet, IntersectCurrent); + } //End if BOTTOM_DOWN + + DEBUG(std::cerr << "Current Intersection Size: " << IntersectCurrent.size() << "\n"); + } + //End Wrapping while loop + DEBUG(std::cerr << "Ending Size of Current Set: " << CurrentSet->size() << "\n"); + }//End for over all sets of nodes + + //FIXME: As the algorithm stands it will NEVER add an instruction such as ba (with no + //data dependencies) to the final order. We add this manually. It will always be + //in the last set of S since its not part of a recurrence + //Loop over all the sets and place them in the final node order + std::vector > ::reverse_iterator LastSet = partialOrder.rbegin(); + for(std::set::iterator CurrentNode = LastSet->begin(), LastNode = LastSet->end(); + CurrentNode != LastNode; ++CurrentNode) { + if((*CurrentNode)->getInst()->getOpcode() == V9::BA) + FinalNodeOrder.push_back(*CurrentNode); + } + //Return final Order + //return FinalNodeOrder; +} + + +void ModuloSchedulingSBPass::predIntersect(std::set &CurrentSet, std::set &IntersectResult) { + + for(unsigned j=0; j < FinalNodeOrder.size(); ++j) { + for(MSchedGraphSBNode::pred_iterator P = FinalNodeOrder[j]->pred_begin(), + E = FinalNodeOrder[j]->pred_end(); P != E; ++P) { + + //Check if we are supposed to ignore this edge or not + if(ignoreEdge(*P,FinalNodeOrder[j])) + continue; + + if(CurrentSet.count(*P)) + if(std::find(FinalNodeOrder.begin(), FinalNodeOrder.end(), *P) == FinalNodeOrder.end()) + IntersectResult.insert(*P); + } + } +} + +void ModuloSchedulingSBPass::succIntersect(std::set &CurrentSet, std::set &IntersectResult) { + + for(unsigned j=0; j < FinalNodeOrder.size(); ++j) { + for(MSchedGraphSBNode::succ_iterator P = FinalNodeOrder[j]->succ_begin(), + E = FinalNodeOrder[j]->succ_end(); P != E; ++P) { + + //Check if we are supposed to ignore this edge or not + if(ignoreEdge(FinalNodeOrder[j],*P)) + continue; + + if(CurrentSet.count(*P)) + if(std::find(FinalNodeOrder.begin(), FinalNodeOrder.end(), *P) == FinalNodeOrder.end()) + IntersectResult.insert(*P); + } + } +} + + + +bool ModuloSchedulingSBPass::computeSchedule(std::vector &SB, MSchedGraphSB *MSG) { + + TIME_REGION(X, "computeSchedule"); + + bool success = false; + + //FIXME: Should be set to max II of the original loop + //Cap II in order to prevent infinite loop + int capII = MSG->totalDelay(); + + while(!success) { + + //Keep track of branches, but do not insert into the schedule + std::vector branches; + + //Loop over the final node order and process each node + for(std::vector::iterator I = FinalNodeOrder.begin(), + E = FinalNodeOrder.end(); I != E; ++I) { + + //CalculateEarly and Late start + bool initialLSVal = false; + bool initialESVal = false; + int EarlyStart = 0; + int LateStart = 0; + bool hasSucc = false; + bool hasPred = false; + bool sched; + + if((*I)->isBranch()) + if((*I)->hasPredecessors()) + sched = true; + else + sched = false; + else + sched = true; + + if(sched) { + //Loop over nodes in the schedule and determine if they are predecessors + //or successors of the node we are trying to schedule + for(MSScheduleSB::schedule_iterator nodesByCycle = schedule.begin(), nodesByCycleEnd = schedule.end(); + nodesByCycle != nodesByCycleEnd; ++nodesByCycle) { + + //For this cycle, get the vector of nodes schedule and loop over it + for(std::vector::iterator schedNode = nodesByCycle->second.begin(), SNE = nodesByCycle->second.end(); schedNode != SNE; ++schedNode) { + + if((*I)->isPredecessor(*schedNode)) { + int diff = (*I)->getInEdge(*schedNode).getIteDiff(); + int ES_Temp = nodesByCycle->first + (*schedNode)->getLatency() - diff * II; + DEBUG(std::cerr << "Diff: " << diff << " Cycle: " << nodesByCycle->first << "\n"); + DEBUG(std::cerr << "Temp EarlyStart: " << ES_Temp << " Prev EarlyStart: " << EarlyStart << "\n"); + if(initialESVal) + EarlyStart = std::max(EarlyStart, ES_Temp); + else { + EarlyStart = ES_Temp; + initialESVal = true; + } + hasPred = true; + } + if((*I)->isSuccessor(*schedNode)) { + int diff = (*schedNode)->getInEdge(*I).getIteDiff(); + int LS_Temp = nodesByCycle->first - (*I)->getLatency() + diff * II; + DEBUG(std::cerr << "Diff: " << diff << " Cycle: " << nodesByCycle->first << "\n"); + DEBUG(std::cerr << "Temp LateStart: " << LS_Temp << " Prev LateStart: " << LateStart << "\n"); + if(initialLSVal) + LateStart = std::min(LateStart, LS_Temp); + else { + LateStart = LS_Temp; + initialLSVal = true; + } + hasSucc = true; + } + } + } + } + else { + branches.push_back(*I); + continue; + } + + //Check if the node has no pred or successors and set Early Start to its ASAP + if(!hasSucc && !hasPred) + EarlyStart = nodeToAttributesMap.find(*I)->second.ASAP; + + DEBUG(std::cerr << "Has Successors: " << hasSucc << ", Has Pred: " << hasPred << "\n"); + DEBUG(std::cerr << "EarlyStart: " << EarlyStart << ", LateStart: " << LateStart << "\n"); + + //Now, try to schedule this node depending upon its pred and successor in the schedule + //already + if(!hasSucc && hasPred) + success = scheduleNode(*I, EarlyStart, (EarlyStart + II -1)); + else if(!hasPred && hasSucc) + success = scheduleNode(*I, LateStart, (LateStart - II +1)); + else if(hasPred && hasSucc) { + if(EarlyStart > LateStart) { + success = false; + //LateStart = EarlyStart; + DEBUG(std::cerr << "Early Start can not be later then the late start cycle, schedule fails\n"); + } + else + success = scheduleNode(*I, EarlyStart, std::min(LateStart, (EarlyStart + II -1))); + } + else + success = scheduleNode(*I, EarlyStart, EarlyStart + II - 1); + + if(!success) { + ++II; + schedule.clear(); + break; + } + + } + + if(success) { + DEBUG(std::cerr << "Constructing Schedule Kernel\n"); + success = schedule.constructKernel(II, branches, indVarInstrs[SB]); + DEBUG(std::cerr << "Done Constructing Schedule Kernel\n"); + if(!success) { + ++II; + schedule.clear(); + } + DEBUG(std::cerr << "Final II: " << II << "\n"); + + } + + if(II >= capII) { + DEBUG(std::cerr << "Maximum II reached, giving up\n"); + return false; + } + + assert(II < capII && "The II should not exceed the original loop number of cycles"); + } + return true; +} + + +bool ModuloSchedulingSBPass::scheduleNode(MSchedGraphSBNode *node, + int start, int end) { + bool success = false; + + DEBUG(std::cerr << *node << " (Start Cycle: " << start << ", End Cycle: " << end << ")\n"); + + //Make sure start and end are not negative + //if(start < 0) { + //start = 0; + + //} + //if(end < 0) + //end = 0; + + bool forward = true; + if(start > end) + forward = false; + + bool increaseSC = true; + int cycle = start ; + + + while(increaseSC) { + + increaseSC = false; + + increaseSC = schedule.insert(node, cycle, II); + + if(!increaseSC) + return true; + + //Increment cycle to try again + if(forward) { + ++cycle; + DEBUG(std::cerr << "Increase cycle: " << cycle << "\n"); + if(cycle > end) + return false; + } + else { + --cycle; + DEBUG(std::cerr << "Decrease cycle: " << cycle << "\n"); + if(cycle < end) + return false; + } + } + + return success; +} + +void ModuloSchedulingSBPass::reconstructLoop(std::vector &SB) { + + TIME_REGION(X, "reconstructLoop"); + + + DEBUG(std::cerr << "Reconstructing Loop\n"); + + //First find the value *'s that we need to "save" + std::map > valuesToSave; + + //Keep track of instructions we have already seen and their stage because + //we don't want to "save" values if they are used in the kernel immediately + std::map lastInstrs; + + + std::set seenBranchesBB; + const TargetInstrInfo *MTI = target.getInstrInfo(); + std::map > > instrsMovedDown; + std::map branchStage; + + //Loop over kernel and only look at instructions from a stage > 0 + //Look at its operands and save values *'s that are read + for(MSScheduleSB::kernel_iterator I = schedule.kernel_begin(), E = schedule.kernel_end(); I != E; ++I) { + + if(I->second !=0) { + //For this instruction, get the Value*'s that it reads and put them into the set. + //Assert if there is an operand of another type that we need to save + const MachineInstr *inst = I->first; + lastInstrs[inst] = I->second; + + for(unsigned i=0; i < inst->getNumOperands(); ++i) { + //get machine operand + const MachineOperand &mOp = inst->getOperand(i); + + if(mOp.getType() == MachineOperand::MO_VirtualRegister && mOp.isUse()) { + //find the value in the map + if (const Value* srcI = mOp.getVRegValue()) { + + if(isa(srcI) || isa(srcI)) + continue; + + //Before we declare this Value* one that we should save + //make sure its def is not of the same stage as this instruction + //because it will be consumed before its used + Instruction *defInst = (Instruction*) srcI; + + //Should we save this value? + bool save = true; + + //Continue if not in the def map, loop invariant code does not need to be saved + if(!defMap.count(srcI)) + continue; + + MachineInstr *defInstr = defMap[srcI]; + + + if(lastInstrs.count(defInstr)) { + if(lastInstrs[defInstr] == I->second) { + save = false; + + } + } + + if(save) + valuesToSave[srcI] = std::make_pair(I->first, i); + } + } + + if(mOp.getType() != MachineOperand::MO_VirtualRegister && mOp.isUse()) { + assert("Our assumption is wrong. We have another type of register that needs to be saved\n"); + } + } + } + + + //Do a check to see if instruction was moved below its original branch + if(MTI->isBranch(I->first->getOpcode())) { + seenBranchesBB.insert(I->first->getParent()); + branchStage[I->first->getParent()] = I->second; + } + else { + instrsMovedDown[I->first->getParent()].push_back(std::make_pair(I->first, I->second)); + //assert(seenBranchesBB.count(I->first->getParent()) && "Instruction moved below branch\n"); + } + + } + + //The new loop will consist of one or more prologues, the kernel, and one or more epilogues. + + //Map to keep track of old to new values + std::map > newValues; + + //Map to keep track of old to new values in kernel + std::map > kernelPHIs; + + //Another map to keep track of what machine basic blocks these new value*s are in since + //they have no llvm instruction equivalent + std::map newValLocation; + + std::vector > prologues; + std::vector > llvm_prologues; + + //Map to keep track of where the inner branches go + std::map sideExits; + + + //Write prologue + if(schedule.getMaxStage() != 0) + writePrologues(prologues, SB, llvm_prologues, valuesToSave, newValues, newValLocation); + + std::vector llvmKernelBBs; + std::vector machineKernelBBs; + Function *parent = (Function*) SB[0]->getBasicBlock()->getParent(); + + for(unsigned i = 0; i < SB.size(); ++i) { + llvmKernelBBs.push_back(new BasicBlock("Kernel", parent)); + + machineKernelBBs.push_back(new MachineBasicBlock(llvmKernelBBs[i])); + (((MachineBasicBlock*)SB[0])->getParent())->getBasicBlockList().push_back(machineKernelBBs[i]); + } + + writeKernel(llvmKernelBBs, machineKernelBBs, valuesToSave, newValues, newValLocation, kernelPHIs); + + + std::vector > epilogues; + std::vector > llvm_epilogues; + + //Write epilogues + if(schedule.getMaxStage() != 0) + writeEpilogues(epilogues, SB, llvm_epilogues, valuesToSave, newValues, newValLocation, kernelPHIs); + + + //Fix our branches + fixBranches(prologues, llvm_prologues, machineKernelBBs, llvmKernelBBs, epilogues, llvm_epilogues, SB, sideExits); + + //Print out epilogues and prologue + DEBUG(for(std::vector >::iterator PI = prologues.begin(), PE = prologues.end(); + PI != PE; ++PI) { + std::cerr << "PROLOGUE\n"; + for(std::vector::iterator I = PI->begin(), E = PI->end(); I != E; ++I) + (*I)->print(std::cerr); + }); + + DEBUG(std::cerr << "KERNEL\n"); + DEBUG(for(std::vector::iterator I = machineKernelBBs.begin(), E = machineKernelBBs.end(); I != E; ++I) { (*I)->print(std::cerr);}); + + DEBUG(for(std::vector >::iterator EI = epilogues.begin(), EE = epilogues.end(); + EI != EE; ++EI) { + std::cerr << "EPILOGUE\n"; + for(std::vector::iterator I = EI->begin(), E = EI->end(); I != E; ++I) + (*I)->print(std::cerr); + }); + + + //Remove phis + removePHIs(SB, prologues, epilogues, machineKernelBBs, newValLocation); + + //Print out epilogues and prologue + DEBUG(for(std::vector >::iterator PI = prologues.begin(), PE = prologues.end(); + PI != PE; ++PI) { + std::cerr << "PROLOGUE\n"; + for(std::vector::iterator I = PI->begin(), E = PI->end(); I != E; ++I) + (*I)->print(std::cerr); + }); + + DEBUG(std::cerr << "KERNEL\n"); + DEBUG(for(std::vector::iterator I = machineKernelBBs.begin(), E = machineKernelBBs.end(); I != E; ++I) { (*I)->print(std::cerr);}); + + DEBUG(for(std::vector >::iterator EI = epilogues.begin(), EE = epilogues.end(); + EI != EE; ++EI) { + std::cerr << "EPILOGUE\n"; + for(std::vector::iterator I = EI->begin(), E = EI->end(); I != E; ++I) + (*I)->print(std::cerr); + }); + + writeSideExits(prologues, llvm_prologues, epilogues, llvm_epilogues, sideExits, instrsMovedDown, SB, machineKernelBBs, branchStage); + + + DEBUG(std::cerr << "New Machine Function" << "\n"); +} + + +void ModuloSchedulingSBPass::fixBranches(std::vector > &prologues, std::vector > &llvm_prologues, std::vector &machineKernelBB, std::vector &llvmKernelBB, std::vector > &epilogues, std::vector > &llvm_epilogues, std::vector &SB, std::map &sideExits) { + + const TargetInstrInfo *TMI = target.getInstrInfo(); + + //Get exit BB + BasicBlock *last = (BasicBlock*) SB[SB.size()-1]->getBasicBlock(); + BasicBlock *kernel_exit = 0; + bool sawFirst = false; + + for(succ_iterator I = succ_begin(last), + E = succ_end(last); I != E; ++I) { + if (*I != SB[0]->getBasicBlock()) { + kernel_exit = *I; + break; + } + else + sawFirst = true; + } + if(!kernel_exit && sawFirst) { + kernel_exit = (BasicBlock*) SB[0]->getBasicBlock(); + } + + assert(kernel_exit && "Kernel Exit can not be null"); + + if(schedule.getMaxStage() != 0) { + //Fix prologue branches + for(unsigned i = 0; i < prologues.size(); ++i) { + + for(unsigned j = 0; j < prologues[i].size(); ++j) { + + MachineBasicBlock *currentMBB = prologues[i][j]; + + //Find terminator since getFirstTerminator does not work! + for(MachineBasicBlock::reverse_iterator mInst = currentMBB->rbegin(), mInstEnd = currentMBB->rend(); mInst != mInstEnd; ++mInst) { + MachineOpCode OC = mInst->getOpcode(); + //If its a branch update its branchto + if(TMI->isBranch(OC)) { + for(unsigned opNum = 0; opNum < mInst->getNumOperands(); ++opNum) { + MachineOperand &mOp = mInst->getOperand(opNum); + if (mOp.getType() == MachineOperand::MO_PCRelativeDisp) { + //Check if we are branching to the kernel, if not branch to epilogue + if(mOp.getVRegValue() == SB[0]->getBasicBlock()) { + if(i >= prologues.size()-1) + mOp.setValueReg(llvmKernelBB[0]); + else + mOp.setValueReg(llvm_prologues[i+1][0]); + } + else if( (mOp.getVRegValue() == kernel_exit) && (j == prologues[i].size()-1)) { + mOp.setValueReg(llvm_epilogues[i][0]); + } + else if(mOp.getVRegValue() == SB[j+1]->getBasicBlock()) { + mOp.setValueReg(llvm_prologues[i][j+1]); + } + + } + } + + DEBUG(std::cerr << "New Prologue Branch: " << *mInst << "\n"); + } + } + + //Update llvm basic block with our new branch instr + DEBUG(std::cerr << SB[i]->getBasicBlock()->getTerminator() << "\n"); + + const BranchInst *branchVal = dyn_cast(SB[i]->getBasicBlock()->getTerminator()); + + //Check for inner branch + if(j < prologues[i].size()-1) { + //Find our side exit LLVM basic block + BasicBlock *sideExit = 0; + for(unsigned s = 0; s < branchVal->getNumSuccessors(); ++s) { + if(branchVal->getSuccessor(s) != SB[i+1]->getBasicBlock()) + sideExit = branchVal->getSuccessor(s); + } + assert(sideExit && "Must have side exit llvm basic block"); + TerminatorInst *newBranch = new BranchInst(sideExit, + llvm_prologues[i][j+1], + branchVal->getCondition(), + llvm_prologues[i][j]); + } + else { + //If last prologue + if(i == prologues.size()-1) { + TerminatorInst *newBranch = new BranchInst(llvmKernelBB[0], + llvm_epilogues[i][0], + branchVal->getCondition(), + llvm_prologues[i][j]); + } + else { + TerminatorInst *newBranch = new BranchInst(llvm_prologues[i+1][0], + llvm_epilogues[i][0], + branchVal->getCondition(), + llvm_prologues[i][j]); + } + } + } + } + } + + //Fix up kernel machine branches + for(unsigned i = 0; i < machineKernelBB.size(); ++i) { + MachineBasicBlock *currentMBB = machineKernelBB[i]; + + for(MachineBasicBlock::reverse_iterator mInst = currentMBB->rbegin(), mInstEnd = currentMBB->rend(); mInst != mInstEnd; ++mInst) { + MachineOpCode OC = mInst->getOpcode(); + if(TMI->isBranch(OC)) { + for(unsigned opNum = 0; opNum < mInst->getNumOperands(); ++opNum) { + MachineOperand &mOp = mInst->getOperand(opNum); + + if(mOp.getType() == MachineOperand::MO_PCRelativeDisp) { + //Deal with inner kernel branches + if(i < machineKernelBB.size()-1) { + if(mOp.getVRegValue() == SB[i+1]->getBasicBlock()) + mOp.setValueReg(llvmKernelBB[i+1]); + //Side exit! + else { + sideExits[SB[i]] = mOp.getVRegValue(); + } + } + else { + if(mOp.getVRegValue() == SB[0]->getBasicBlock()) + mOp.setValueReg(llvmKernelBB[0]); + else { + if(llvm_epilogues.size() > 0) + mOp.setValueReg(llvm_epilogues[0][0]); + } + } + } + } + } + } + + //Update kernelLLVM branches + const BranchInst *branchVal = dyn_cast(SB[0]->getBasicBlock()->getTerminator()); + + //deal with inner branch + if(i < machineKernelBB.size()-1) { + + //Find our side exit LLVM basic block + BasicBlock *sideExit = 0; + for(unsigned s = 0; s < branchVal->getNumSuccessors(); ++s) { + if(branchVal->getSuccessor(s) != SB[i+1]->getBasicBlock()) + sideExit = branchVal->getSuccessor(s); + } + assert(sideExit && "Must have side exit llvm basic block"); + TerminatorInst *newBranch = new BranchInst(sideExit, + llvmKernelBB[i+1], + branchVal->getCondition(), + llvmKernelBB[i]); + } + else { + //Deal with outter branches + if(epilogues.size() > 0) { + TerminatorInst *newBranch = new BranchInst(llvmKernelBB[0], + llvm_epilogues[0][0], + branchVal->getCondition(), + llvmKernelBB[i]); + } + else { + TerminatorInst *newBranch = new BranchInst(llvmKernelBB[0], + kernel_exit, + branchVal->getCondition(), + llvmKernelBB[i]); + } + } + } + + if(schedule.getMaxStage() != 0) { + + //Lastly add unconditional branches for the epilogues + for(unsigned i = 0; i < epilogues.size(); ++i) { + + for(unsigned j=0; j < epilogues[i].size(); ++j) { + //Now since we don't have fall throughs, add a unconditional + //branch to the next prologue + + //Before adding these, we need to check if the epilogue already has + //a branch in it + bool hasBranch = false; + /*if(j < epilogues[i].size()-1) { + MachineBasicBlock *currentMBB = epilogues[i][j]; + for(MachineBasicBlock::reverse_iterator mInst = currentMBB->rbegin(), mInstEnd = currentMBB->rend(); mInst != mInstEnd; ++mInst) { + + MachineOpCode OC = mInst->getOpcode(); + + //If its a branch update its branchto + if(TMI->isBranch(OC)) { + hasBranch = true; + for(unsigned opNum = 0; opNum < mInst->getNumOperands(); ++opNum) { + MachineOperand &mOp = mInst->getOperand(opNum); + if (mOp.getType() == MachineOperand::MO_PCRelativeDisp) { + + if(mOp.getVRegValue() != sideExits[SB[j]]) { + mOp.setValueReg(llvm_epilogues[i][j+1]); + } + + } + } + + + DEBUG(std::cerr << "New Epilogue Branch: " << *mInst << "\n"); + } + } + if(hasBranch) { + const BranchInst *branchVal = dyn_cast(SB[j]->getBasicBlock()->getTerminator()); + TerminatorInst *newBranch = new BranchInst((BasicBlock*)sideExits[SB[j]], + llvm_epilogues[i][j+1], + branchVal->getCondition(), + llvm_epilogues[i][j]); + } + }*/ + + if(!hasBranch) { + + //Handle inner branches + if(j < epilogues[i].size()-1) { + BuildMI(epilogues[i][j], V9::BA, 1).addPCDisp(llvm_epilogues[i][j+1]); + TerminatorInst *newBranch = new BranchInst(llvm_epilogues[i][j+1], + llvm_epilogues[i][j]); + } + else { + + //Check if this is the last epilogue + if(i != epilogues.size()-1) { + BuildMI(epilogues[i][j], V9::BA, 1).addPCDisp(llvm_epilogues[i+1][0]); + //Add unconditional branch to end of epilogue + TerminatorInst *newBranch = new BranchInst(llvm_epilogues[i+1][0], + llvm_epilogues[i][j]); + + } + else { + BuildMI(epilogues[i][j], V9::BA, 1).addPCDisp(kernel_exit); + TerminatorInst *newBranch = new BranchInst(kernel_exit, llvm_epilogues[i][j]); + } + } + + //Add one more nop! + BuildMI(epilogues[i][j], V9::NOP, 0); + + } + } + } + } + + //Find all llvm basic blocks that branch to the loop entry and + //change to our first prologue. + const BasicBlock *llvmBB = SB[0]->getBasicBlock(); + + std::vectorPreds (pred_begin(llvmBB), pred_end(llvmBB)); + + for(std::vector::iterator P = Preds.begin(), + PE = Preds.end(); P != PE; ++P) { + if(*P == SB[SB.size()-1]->getBasicBlock()) + continue; + else { + DEBUG(std::cerr << "Found our entry BB\n"); + DEBUG((*P)->print(std::cerr)); + //Get the Terminator instruction for this basic block and print it out + //DEBUG(std::cerr << *((*P)->getTerminator()) << "\n"); + + //Update the terminator + TerminatorInst *term = ((BasicBlock*)*P)->getTerminator(); + for(unsigned i=0; i < term->getNumSuccessors(); ++i) { + if(term->getSuccessor(i) == llvmBB) { + DEBUG(std::cerr << "Replacing successor bb\n"); + if(llvm_prologues.size() > 0) { + term->setSuccessor(i, llvm_prologues[0][0]); + + DEBUG(std::cerr << "New Term" << *((*P)->getTerminator()) << "\n"); + + //Also update its corresponding machine instruction + MachineCodeForInstruction & tempMvec = + MachineCodeForInstruction::get(term); + for (unsigned j = 0; j < tempMvec.size(); j++) { + MachineInstr *temp = tempMvec[j]; + MachineOpCode opc = temp->getOpcode(); + if(TMI->isBranch(opc)) { + DEBUG(std::cerr << *temp << "\n"); + //Update branch + for(unsigned opNum = 0; opNum < temp->getNumOperands(); ++opNum) { + MachineOperand &mOp = temp->getOperand(opNum); + if (mOp.getType() == MachineOperand::MO_PCRelativeDisp) { + if(mOp.getVRegValue() == llvmBB) + mOp.setValueReg(llvm_prologues[0][0]); + } + } + } + } + } + else { + term->setSuccessor(i, llvmKernelBB[0]); + + //Also update its corresponding machine instruction + MachineCodeForInstruction & tempMvec = + MachineCodeForInstruction::get(term); + for(unsigned j = 0; j < tempMvec.size(); j++) { + MachineInstr *temp = tempMvec[j]; + MachineOpCode opc = temp->getOpcode(); + if(TMI->isBranch(opc)) { + DEBUG(std::cerr << *temp << "\n"); + //Update branch + for(unsigned opNum = 0; opNum < temp->getNumOperands(); ++opNum) { + MachineOperand &mOp = temp->getOperand(opNum); + if(mOp.getType() == MachineOperand::MO_PCRelativeDisp) { + if(mOp.getVRegValue() == llvmBB) + mOp.setValueReg(llvmKernelBB[0]); + } + } + } + } + } + } + } + break; + } + } + +} + + +void ModuloSchedulingSBPass::writePrologues(std::vector > &prologues, std::vector &origSB, std::vector > &llvm_prologues, std::map > &valuesToSave, std::map > &newValues, std::map &newValLocation) { + + //Keep a map to easily know whats in the kernel + std::map > inKernel; + int maxStageCount = 0; + + //Keep a map of new values we consumed in case they need to be added back + std::map > consumedValues; + + DEBUG(schedule.print(std::cerr)); + + for(MSScheduleSB::kernel_iterator I = schedule.kernel_begin(), E = schedule.kernel_end(); I != E; ++I) { + maxStageCount = std::max(maxStageCount, I->second); + + //Put int the map so we know what instructions in each stage are in the kernel + DEBUG(std::cerr << "Inserting instruction " << *(I->first) << " into map at stage " << I->second << "\n"); + inKernel[I->second].insert(I->first); + } + + //Get target information to look at machine operands + const TargetInstrInfo *mii = target.getInstrInfo(); + + //Now write the prologues + for(int i = 0; i < maxStageCount; ++i) { + std::vector current_prologue; + std::vector current_llvm_prologue; + + for(std::vector::iterator MB = origSB.begin(), + MBE = origSB.end(); MB != MBE; ++MB) { + const MachineBasicBlock *MBB = *MB; + //Create new llvm and machine bb + BasicBlock *llvmBB = new BasicBlock("PROLOGUE", (Function*) (MBB->getBasicBlock()->getParent())); + MachineBasicBlock *machineBB = new MachineBasicBlock(llvmBB); + + DEBUG(std::cerr << "i=" << i << "\n"); + + for(int j = i; j >= 0; --j) { + //iterate over instructions in original bb + for(MachineBasicBlock::const_iterator MI = MBB->begin(), + ME = MBB->end(); ME != MI; ++MI) { + if(inKernel[j].count(&*MI)) { + MachineInstr *instClone = MI->clone(); + machineBB->push_back(instClone); + + //If its a branch, insert a nop + if(mii->isBranch(instClone->getOpcode())) + BuildMI(machineBB, V9::NOP, 0); + + + DEBUG(std::cerr << "Cloning: " << *MI << "\n"); + + //After cloning, we may need to save the value that this instruction defines + for(unsigned opNum=0; opNum < MI->getNumOperands(); ++opNum) { + Instruction *tmp; + + //get machine operand + MachineOperand &mOp = instClone->getOperand(opNum); + if(mOp.getType() == MachineOperand::MO_VirtualRegister + && mOp.isDef()) { + + //Check if this is a value we should save + if(valuesToSave.count(mOp.getVRegValue())) { + //Save copy in tmpInstruction + tmp = new TmpInstruction(mOp.getVRegValue()); + + //Add TmpInstruction to safe LLVM Instruction MCFI + MachineCodeForInstruction & tempMvec = MachineCodeForInstruction::get(defaultInst); + tempMvec.addTemp((Value*) tmp); + + DEBUG(std::cerr << "Value: " << *(mOp.getVRegValue()) + << " New Value: " << *tmp << " Stage: " << i << "\n"); + + newValues[mOp.getVRegValue()][i]= tmp; + newValLocation[tmp] = machineBB; + + DEBUG(std::cerr << "Machine Instr Operands: " + << *(mOp.getVRegValue()) << ", 0, " << *tmp << "\n"); + + //Create machine instruction and put int machineBB + MachineInstr *saveValue; + if(mOp.getVRegValue()->getType() == Type::FloatTy) + saveValue = BuildMI(machineBB, V9::FMOVS, 3).addReg(mOp.getVRegValue()).addRegDef(tmp); + else if(mOp.getVRegValue()->getType() == Type::DoubleTy) + saveValue = BuildMI(machineBB, V9::FMOVD, 3).addReg(mOp.getVRegValue()).addRegDef(tmp); + else + saveValue = BuildMI(machineBB, V9::ORr, 3).addReg(mOp.getVRegValue()).addImm(0).addRegDef(tmp); + + + DEBUG(std::cerr << "Created new machine instr: " << *saveValue << "\n"); + } + } + + //We may also need to update the value that we use if + //its from an earlier prologue + if(j != 0) { + if(mOp.getType() == MachineOperand::MO_VirtualRegister && mOp.isUse()) { + if(newValues.count(mOp.getVRegValue())) { + if(newValues[mOp.getVRegValue()].count(i-1)) { + Value *oldV = mOp.getVRegValue(); + DEBUG(std::cerr << "Replaced this value: " << mOp.getVRegValue() << " With:" << (newValues[mOp.getVRegValue()][i-1]) << "\n"); + //Update the operand with the right value + mOp.setValueReg(newValues[mOp.getVRegValue()][i-1]); + + //Remove this value since we have consumed it + //NOTE: Should this only be done if j != maxStage? + consumedValues[oldV][i-1] = (newValues[oldV][i-1]); + DEBUG(std::cerr << "Deleted value: " << consumedValues[oldV][i-1] << "\n"); + newValues[oldV].erase(i-1); + } + } + else + if(consumedValues.count(mOp.getVRegValue())) + assert(!consumedValues[mOp.getVRegValue()].count(i-1) && "Found a case where we need the value"); + } + } + } + } + } + } + (((MachineBasicBlock*)MBB)->getParent())->getBasicBlockList().push_back(machineBB); + current_prologue.push_back(machineBB); + current_llvm_prologue.push_back(llvmBB); + } + prologues.push_back(current_prologue); + llvm_prologues.push_back(current_llvm_prologue); + + } +} + + +void ModuloSchedulingSBPass::writeEpilogues(std::vector > &epilogues, std::vector &origSB, std::vector > &llvm_epilogues, std::map > &valuesToSave, std::map > &newValues,std::map &newValLocation, std::map > &kernelPHIs ) { + + std::map > inKernel; + const TargetInstrInfo *MTI = target.getInstrInfo(); + + for(MSScheduleSB::kernel_iterator I = schedule.kernel_begin(), E = schedule.kernel_end(); I != E; ++I) { + + //Put int the map so we know what instructions in each stage are in the kernel + inKernel[I->second].insert(I->first); + } + + std::map valPHIs; + + //some debug stuff, will remove later + DEBUG(for(std::map >::iterator V = newValues.begin(), E = newValues.end(); V !=E; ++V) { + std::cerr << "Old Value: " << *(V->first) << "\n"; + for(std::map::iterator I = V->second.begin(), IE = V->second.end(); I != IE; ++I) + std::cerr << "Stage: " << I->first << " Value: " << *(I->second) << "\n"; + }); + + + //Now write the epilogues + for(int i = schedule.getMaxStage()-1; i >= 0; --i) { + std::vector current_epilogue; + std::vector current_llvm_epilogue; + + for(std::vector::iterator MB = origSB.begin(), MBE = origSB.end(); MB != MBE; ++MB) { + const MachineBasicBlock *MBB = *MB; + + BasicBlock *llvmBB = new BasicBlock("EPILOGUE", (Function*) (MBB->getBasicBlock()->getParent())); + MachineBasicBlock *machineBB = new MachineBasicBlock(llvmBB); + + DEBUG(std::cerr << " Epilogue #: " << i << "\n"); + + std::map inEpilogue; + + for(MachineBasicBlock::const_iterator MI = MBB->begin(), ME = MBB->end(); ME != MI; ++MI) { + for(int j=schedule.getMaxStage(); j > i; --j) { + if(inKernel[j].count(&*MI)) { + DEBUG(std::cerr << "Cloning instruction " << *MI << "\n"); + MachineInstr *clone = MI->clone(); + + //Update operands that need to use the result from the phi + for(unsigned opNum=0; opNum < clone->getNumOperands(); ++opNum) { + //get machine operand + const MachineOperand &mOp = clone->getOperand(opNum); + + if((mOp.getType() == MachineOperand::MO_VirtualRegister && mOp.isUse())) { + + DEBUG(std::cerr << "Writing PHI for " << (mOp.getVRegValue()) << "\n"); + + //If this is the last instructions for the max iterations ago, don't update operands + if(inEpilogue.count(mOp.getVRegValue())) + if(inEpilogue[mOp.getVRegValue()] == i) + continue; + + //Quickly write appropriate phis for this operand + if(newValues.count(mOp.getVRegValue())) { + if(newValues[mOp.getVRegValue()].count(i)) { + Instruction *tmp = new TmpInstruction(newValues[mOp.getVRegValue()][i]); + + //Get machine code for this instruction + MachineCodeForInstruction & tempMvec = MachineCodeForInstruction::get(defaultInst); + tempMvec.addTemp((Value*) tmp); + + //assert of no kernelPHI for this value + assert(kernelPHIs[mOp.getVRegValue()][i] !=0 && "Must have final kernel phi to construct epilogue phi"); + + MachineInstr *saveValue = BuildMI(machineBB, V9::PHI, 3).addReg(newValues[mOp.getVRegValue()][i]).addReg(kernelPHIs[mOp.getVRegValue()][i]).addRegDef(tmp); + DEBUG(std::cerr << "Resulting PHI: " << *saveValue << "\n"); + valPHIs[mOp.getVRegValue()] = tmp; + } + } + + if(valPHIs.count(mOp.getVRegValue())) { + //Update the operand in the cloned instruction + clone->getOperand(opNum).setValueReg(valPHIs[mOp.getVRegValue()]); + } + } + else if((mOp.getType() == MachineOperand::MO_VirtualRegister && mOp.isDef())) { + inEpilogue[mOp.getVRegValue()] = i; + } + + } + machineBB->push_back(clone); + //if(MTI->isBranch(clone->getOpcode())) + //BuildMI(machineBB, V9::NOP, 0); + } + } + } + (((MachineBasicBlock*)MBB)->getParent())->getBasicBlockList().push_back(machineBB); + current_epilogue.push_back(machineBB); + current_llvm_epilogue.push_back(llvmBB); + } + + DEBUG(std::cerr << "EPILOGUE #" << i << "\n"); + DEBUG(for(std::vector::iterator B = current_epilogue.begin(), BE = current_epilogue.end(); B != BE; ++B) { + (*B)->print(std::cerr);}); + + epilogues.push_back(current_epilogue); + llvm_epilogues.push_back(current_llvm_epilogue); + } +} + +void ModuloSchedulingSBPass::writeKernel(std::vector &llvmBB, std::vector &machineBB, std::map > &valuesToSave, std::map > &newValues, std::map &newValLocation, std::map > &kernelPHIs) { + + //Keep track of operands that are read and saved from a previous iteration. The new clone + //instruction will use the result of the phi instead. + std::map finalPHIValue; + std::map kernelValue; + + //Branches are a special case + std::vector branches; + + //Get target information to look at machine operands + const TargetInstrInfo *mii = target.getInstrInfo(); + unsigned index = 0; + int numBr = 0; + bool seenBranch = false; + + //Create TmpInstructions for the final phis + for(MSScheduleSB::kernel_iterator I = schedule.kernel_begin(), E = schedule.kernel_end(); I != E; ++I) { + + DEBUG(std::cerr << "Stage: " << I->second << " Inst: " << *(I->first) << "\n";); + + //Clone instruction + const MachineInstr *inst = I->first; + MachineInstr *instClone = inst->clone(); + + if(seenBranch && !mii->isBranch(instClone->getOpcode())) { + index++; + seenBranch = false; + numBr = 0; + } + else if(seenBranch && (numBr == 2)) { + index++; + numBr = 0; + } + + //Insert into machine basic block + assert(index < machineBB.size() && "Must have a valid index into kernel MBBs"); + machineBB[index]->push_back(instClone); + + if(mii->isBranch(instClone->getOpcode())) { + BuildMI(machineBB[index], V9::NOP, 0); + + seenBranch = true; + numBr++; + } + + DEBUG(std::cerr << "Cloned Inst: " << *instClone << "\n"); + + //Loop over Machine Operands + for(unsigned i=0; i < inst->getNumOperands(); ++i) { + //get machine operand + const MachineOperand &mOp = inst->getOperand(i); + + if(I->second != 0) { + if(mOp.getType() == MachineOperand::MO_VirtualRegister && mOp.isUse()) { + + //Check to see where this operand is defined if this instruction is from max stage + if(I->second == schedule.getMaxStage()) { + DEBUG(std::cerr << "VREG: " << *(mOp.getVRegValue()) << "\n"); + } + + //If its in the value saved, we need to create a temp instruction and use that instead + if(valuesToSave.count(mOp.getVRegValue())) { + + //Check if we already have a final PHI value for this + if(!finalPHIValue.count(mOp.getVRegValue())) { + //Only create phi if the operand def is from a stage before this one + if(schedule.defPreviousStage(mOp.getVRegValue(), I->second)) { + TmpInstruction *tmp = new TmpInstruction(mOp.getVRegValue()); + + //Get machine code for this instruction + MachineCodeForInstruction & tempMvec = MachineCodeForInstruction::get(defaultInst); + tempMvec.addTemp((Value*) tmp); + + //Update the operand in the cloned instruction + instClone->getOperand(i).setValueReg(tmp); + + //save this as our final phi + finalPHIValue[mOp.getVRegValue()] = tmp; + newValLocation[tmp] = machineBB[index]; + } + } + else { + //Use the previous final phi value + instClone->getOperand(i).setValueReg(finalPHIValue[mOp.getVRegValue()]); + } + } + } + } + if(I->second != schedule.getMaxStage()) { + if(mOp.getType() == MachineOperand::MO_VirtualRegister && mOp.isDef()) { + if(valuesToSave.count(mOp.getVRegValue())) { + + TmpInstruction *tmp = new TmpInstruction(mOp.getVRegValue()); + + //Get machine code for this instruction + MachineCodeForInstruction & tempVec = MachineCodeForInstruction::get(defaultInst); + tempVec.addTemp((Value*) tmp); + + //Create new machine instr and put in MBB + MachineInstr *saveValue; + if(mOp.getVRegValue()->getType() == Type::FloatTy) + saveValue = BuildMI(machineBB[index], V9::FMOVS, 3).addReg(mOp.getVRegValue()).addRegDef(tmp); + else if(mOp.getVRegValue()->getType() == Type::DoubleTy) + saveValue = BuildMI(machineBB[index], V9::FMOVD, 3).addReg(mOp.getVRegValue()).addRegDef(tmp); + else + saveValue = BuildMI(machineBB[index], V9::ORr, 3).addReg(mOp.getVRegValue()).addImm(0).addRegDef(tmp); + + + //Save for future cleanup + kernelValue[mOp.getVRegValue()] = tmp; + newValLocation[tmp] = machineBB[index]; + kernelPHIs[mOp.getVRegValue()][schedule.getMaxStage()-1] = tmp; + } + } + } + } + + } + + //Loop over each value we need to generate phis for + for(std::map >::iterator V = newValues.begin(), + E = newValues.end(); V != E; ++V) { + + + DEBUG(std::cerr << "Writing phi for" << *(V->first)); + DEBUG(std::cerr << "\nMap of Value* for this phi\n"); + DEBUG(for(std::map::iterator I = V->second.begin(), + IE = V->second.end(); I != IE; ++I) { + std::cerr << "Stage: " << I->first; + std::cerr << " Value: " << *(I->second) << "\n"; + }); + + //If we only have one current iteration live, its safe to set + //lastPhi = to kernel value + if(V->second.size() == 1) { + assert(kernelValue[V->first] != 0 && "Kernel value* must exist to create phi"); + MachineInstr *saveValue = BuildMI(*machineBB[0], machineBB[0]->begin(),V9::PHI, 3).addReg(V->second.begin()->second).addReg(kernelValue[V->first]).addRegDef(finalPHIValue[V->first]); + DEBUG(std::cerr << "Resulting PHI (one live): " << *saveValue << "\n"); + kernelPHIs[V->first][V->second.begin()->first] = kernelValue[V->first]; + DEBUG(std::cerr << "Put kernel phi in at stage: " << schedule.getMaxStage()-1 << " (map stage = " << V->second.begin()->first << ")\n"); + } + else { + + //Keep track of last phi created. + Instruction *lastPhi = 0; + + unsigned count = 1; + //Loop over the the map backwards to generate phis + for(std::map::reverse_iterator I = V->second.rbegin(), IE = V->second.rend(); + I != IE; ++I) { + + if(count < (V->second).size()) { + if(lastPhi == 0) { + lastPhi = new TmpInstruction(I->second); + + //Get machine code for this instruction + MachineCodeForInstruction & tempMvec = MachineCodeForInstruction::get(defaultInst); + tempMvec.addTemp((Value*) lastPhi); + + MachineInstr *saveValue = BuildMI(*machineBB[0], machineBB[0]->begin(), V9::PHI, 3).addReg(kernelValue[V->first]).addReg(I->second).addRegDef(lastPhi); + DEBUG(std::cerr << "Resulting PHI: " << *saveValue << "\n"); + newValLocation[lastPhi] = machineBB[0]; + } + else { + Instruction *tmp = new TmpInstruction(I->second); + + //Get machine code for this instruction + MachineCodeForInstruction & tempMvec = MachineCodeForInstruction::get(defaultInst); + tempMvec.addTemp((Value*) tmp); + + + MachineInstr *saveValue = BuildMI(*machineBB[0], machineBB[0]->begin(), V9::PHI, 3).addReg(lastPhi).addReg(I->second).addRegDef(tmp); + DEBUG(std::cerr << "Resulting PHI: " << *saveValue << "\n"); + lastPhi = tmp; + kernelPHIs[V->first][I->first] = lastPhi; + newValLocation[lastPhi] = machineBB[0]; + } + } + //Final phi value + else { + //The resulting value must be the Value* we created earlier + assert(lastPhi != 0 && "Last phi is NULL!\n"); + MachineInstr *saveValue = BuildMI(*machineBB[0], machineBB[0]->begin(), V9::PHI, 3).addReg(lastPhi).addReg(I->second).addRegDef(finalPHIValue[V->first]); + DEBUG(std::cerr << "Resulting PHI: " << *saveValue << "\n"); + kernelPHIs[V->first][I->first] = finalPHIValue[V->first]; + } + + ++count; + } + + } + } +} + + +void ModuloSchedulingSBPass::removePHIs(std::vector &SB, std::vector > &prologues, std::vector > &epilogues, std::vector &kernelBB, std::map &newValLocation) { + + //Worklist to delete things + std::vector > worklist; + + //Worklist of TmpInstructions that need to be added to a MCFI + std::vector addToMCFI; + + //Worklist to add OR instructions to end of kernel so not to invalidate the iterator + //std::vector > newORs; + + const TargetInstrInfo *TMI = target.getInstrInfo(); + + //Start with the kernel and for each phi insert a copy for the phi + //def and for each arg + //phis are only in the first BB in the kernel + for(MachineBasicBlock::iterator I = kernelBB[0]->begin(), E = kernelBB[0]->end(); + I != E; ++I) { + + DEBUG(std::cerr << "Looking at Instr: " << *I << "\n"); + + //Get op code and check if its a phi + if(I->getOpcode() == V9::PHI) { + + DEBUG(std::cerr << "Replacing PHI: " << *I << "\n"); + Instruction *tmp = 0; + + for(unsigned i = 0; i < I->getNumOperands(); ++i) { + + //Get Operand + const MachineOperand &mOp = I->getOperand(i); + assert(mOp.getType() == MachineOperand::MO_VirtualRegister + && "Should be a Value*\n"); + + if(!tmp) { + tmp = new TmpInstruction(mOp.getVRegValue()); + addToMCFI.push_back(tmp); + } + + //Now for all our arguments we read, OR to the new + //TmpInstruction that we created + if(mOp.isUse()) { + DEBUG(std::cerr << "Use: " << mOp << "\n"); + //Place a copy at the end of its BB but before the branches + assert(newValLocation.count(mOp.getVRegValue()) && "We must know where this value is located\n"); + //Reverse iterate to find the branches, we can safely assume no instructions have been + //put in the nop positions + for(MachineBasicBlock::iterator inst = --(newValLocation[mOp.getVRegValue()])->end(), endBB = (newValLocation[mOp.getVRegValue()])->begin(); inst != endBB; --inst) { + MachineOpCode opc = inst->getOpcode(); + if(TMI->isBranch(opc) || TMI->isNop(opc)) + continue; + else { + if(mOp.getVRegValue()->getType() == Type::FloatTy) + BuildMI(*(newValLocation[mOp.getVRegValue()]), ++inst, V9::FMOVS, 3).addReg(mOp.getVRegValue()).addRegDef(tmp); + else if(mOp.getVRegValue()->getType() == Type::DoubleTy) + BuildMI(*(newValLocation[mOp.getVRegValue()]), ++inst, V9::FMOVD, 3).addReg(mOp.getVRegValue()).addRegDef(tmp); + else + BuildMI(*(newValLocation[mOp.getVRegValue()]), ++inst, V9::ORr, 3).addReg(mOp.getVRegValue()).addImm(0).addRegDef(tmp); + + break; + } + + } + + } + else { + //Remove the phi and replace it with an OR + DEBUG(std::cerr << "Def: " << mOp << "\n"); + //newORs.push_back(std::make_pair(tmp, mOp.getVRegValue())); + if(tmp->getType() == Type::FloatTy) + BuildMI(*kernelBB[0], I, V9::FMOVS, 3).addReg(tmp).addRegDef(mOp.getVRegValue()); + else if(tmp->getType() == Type::DoubleTy) + BuildMI(*kernelBB[0], I, V9::FMOVD, 3).addReg(tmp).addRegDef(mOp.getVRegValue()); + else + BuildMI(*kernelBB[0], I, V9::ORr, 3).addReg(tmp).addImm(0).addRegDef(mOp.getVRegValue()); + + + worklist.push_back(std::make_pair(kernelBB[0], I)); + } + + } + + } + + + } + + //Add TmpInstructions to some MCFI + if(addToMCFI.size() > 0) { + MachineCodeForInstruction & tempMvec = MachineCodeForInstruction::get(defaultInst); + for(unsigned x = 0; x < addToMCFI.size(); ++x) { + tempMvec.addTemp(addToMCFI[x]); + } + addToMCFI.clear(); + } + + + //Remove phis from epilogue + for(std::vector >::iterator MB = epilogues.begin(), + ME = epilogues.end(); MB != ME; ++MB) { + + for(std::vector::iterator currentMBB = MB->begin(), currentME = MB->end(); currentMBB != currentME; ++currentMBB) { + + for(MachineBasicBlock::iterator I = (*currentMBB)->begin(), + E = (*currentMBB)->end(); I != E; ++I) { + + DEBUG(std::cerr << "Looking at Instr: " << *I << "\n"); + //Get op code and check if its a phi + if(I->getOpcode() == V9::PHI) { + Instruction *tmp = 0; + + for(unsigned i = 0; i < I->getNumOperands(); ++i) { + //Get Operand + const MachineOperand &mOp = I->getOperand(i); + assert(mOp.getType() == MachineOperand::MO_VirtualRegister && "Should be a Value*\n"); + + if(!tmp) { + tmp = new TmpInstruction(mOp.getVRegValue()); + addToMCFI.push_back(tmp); + } + + //Now for all our arguments we read, OR to the new TmpInstruction that we created + if(mOp.isUse()) { + DEBUG(std::cerr << "Use: " << mOp << "\n"); + //Place a copy at the end of its BB but before the branches + assert(newValLocation.count(mOp.getVRegValue()) && "We must know where this value is located\n"); + //Reverse iterate to find the branches, we can safely assume no instructions have been + //put in the nop positions + for(MachineBasicBlock::iterator inst = --(newValLocation[mOp.getVRegValue()])->end(), endBB = (newValLocation[mOp.getVRegValue()])->begin(); inst != endBB; --inst) { + MachineOpCode opc = inst->getOpcode(); + if(TMI->isBranch(opc) || TMI->isNop(opc)) + continue; + else { + if(mOp.getVRegValue()->getType() == Type::FloatTy) + BuildMI(*(newValLocation[mOp.getVRegValue()]), ++inst, V9::FMOVS, 3).addReg(mOp.getVRegValue()).addRegDef(tmp); + else if(mOp.getVRegValue()->getType() == Type::DoubleTy) + BuildMI(*(newValLocation[mOp.getVRegValue()]), ++inst, V9::FMOVD, 3).addReg(mOp.getVRegValue()).addRegDef(tmp); + else + BuildMI(*(newValLocation[mOp.getVRegValue()]), ++inst, V9::ORr, 3).addReg(mOp.getVRegValue()).addImm(0).addRegDef(tmp); + + + break; + } + + } + + } + else { + //Remove the phi and replace it with an OR + DEBUG(std::cerr << "Def: " << mOp << "\n"); + if(tmp->getType() == Type::FloatTy) + BuildMI(**currentMBB, I, V9::FMOVS, 3).addReg(tmp).addRegDef(mOp.getVRegValue()); + else if(tmp->getType() == Type::DoubleTy) + BuildMI(**currentMBB, I, V9::FMOVD, 3).addReg(tmp).addRegDef(mOp.getVRegValue()); + else + BuildMI(**currentMBB, I, V9::ORr, 3).addReg(tmp).addImm(0).addRegDef(mOp.getVRegValue()); + + worklist.push_back(std::make_pair(*currentMBB,I)); + } + } + } + } + } + } + + + if(addToMCFI.size() > 0) { + MachineCodeForInstruction & tempMvec = MachineCodeForInstruction::get(defaultInst); + for(unsigned x = 0; x < addToMCFI.size(); ++x) { + tempMvec.addTemp(addToMCFI[x]); + } + addToMCFI.clear(); + } + + //Delete the phis + for(std::vector >::iterator I = worklist.begin(), E = worklist.end(); I != E; ++I) { + DEBUG(std::cerr << "Deleting PHI " << *I->second << "\n"); + I->first->erase(I->second); + + } + + + assert((addToMCFI.size() == 0) && "We should have added all TmpInstructions to some MachineCodeForInstruction"); +} + + + + +void ModuloSchedulingSBPass::writeSideExits(std::vector > &prologues, std::vector > &llvm_prologues, std::vector > &epilogues, std::vector > &llvm_epilogues, std::map &sideExits, std::map > > &instrsMovedDown, std::vector &SB, std::vector &kernelMBBs, std::map branchStage) { + + const TargetInstrInfo *TMI = target.getInstrInfo(); + + //Repeat for each side exit + for(unsigned sideExitNum = 0; sideExitNum < SB.size()-1; ++sideExitNum) { + + std::vector > side_llvm_epilogues; + std::vector > side_epilogues; + MachineBasicBlock* sideMBB; + BasicBlock* sideBB; + + //Create side exit blocks + //Get the LLVM basic block + BasicBlock *bb = (BasicBlock*) SB[sideExitNum]->getBasicBlock(); + MachineBasicBlock *mbb = (MachineBasicBlock*) SB[sideExitNum]; + + int stage = branchStage[mbb]; + + //Create new basic blocks for our side exit instructios that were moved below the branch + sideBB = new BasicBlock("SideExit", (Function*) bb->getParent()); + sideMBB = new MachineBasicBlock(sideBB); + (((MachineBasicBlock*)SB[0])->getParent())->getBasicBlockList().push_back(sideMBB); + + + if(instrsMovedDown.count(mbb)) { + for(std::vector >::iterator I = instrsMovedDown[mbb].begin(), E = instrsMovedDown[mbb].end(); I != E; ++I) { + if(branchStage[mbb] == I->second) + sideMBB->push_back((I->first)->clone()); + } + + //Add unconditional branches to original exits + BuildMI(sideMBB, V9::BA, 1).addPCDisp(sideExits[mbb]); + BuildMI(sideMBB, V9::NOP, 0); + + //Add unconditioal branch to llvm BB + BasicBlock *extBB = dyn_cast(sideExits[mbb]); + assert(extBB && "Side exit basicblock can not be null"); + TerminatorInst *newBranch = new BranchInst(extBB, sideBB); + } + + //Clone epilogues and update their branches, one cloned epilogue set per side exit + //only clone epilogues that are from a greater stage! + for(unsigned i = 0; i < epilogues.size()-stage; ++i) { + std::vector MB = epilogues[i]; + + std::vector newEp; + std::vector newLLVMEp; + + for(std::vector::iterator currentMBB = MB.begin(), + lastMBB = MB.end(); currentMBB != lastMBB; ++currentMBB) { + BasicBlock *tmpBB = new BasicBlock("SideEpilogue", (Function*) (*currentMBB)->getBasicBlock()->getParent()); + MachineBasicBlock *tmp = new MachineBasicBlock(tmpBB); + + //Clone instructions and insert into new MBB + for(MachineBasicBlock::iterator I = (*currentMBB)->begin(), + E = (*currentMBB)->end(); I != E; ++I) { + + MachineInstr *clone = I->clone(); + if(clone->getOpcode() == V9::BA && (currentMBB+1 == lastMBB)) { + //update branch to side exit + for(unsigned i = 0; i < clone->getNumOperands(); ++i) { + MachineOperand &mOp = clone->getOperand(i); + if (mOp.getType() == MachineOperand::MO_PCRelativeDisp) { + mOp.setValueReg(sideBB); + } + } + } + + tmp->push_back(clone); + + } + + //Add llvm branch + TerminatorInst *newBranch = new BranchInst(sideBB, tmpBB); + + newEp.push_back(tmp); + (((MachineBasicBlock*)SB[0])->getParent())->getBasicBlockList().push_back(tmp); + + newLLVMEp.push_back(tmpBB); + + } + side_llvm_epilogues.push_back(newLLVMEp); + side_epilogues.push_back(newEp); + } + + //Now stich up all the branches + + //Loop over prologues, and if its an inner branch and branches to our original side exit + //then have it branch to the appropriate epilogue first (if it exists) + for(unsigned P = 0; P < prologues.size(); ++P) { + + //Get BB side exit we are dealing with + MachineBasicBlock *currentMBB = prologues[P][sideExitNum]; + if(P >= (unsigned) stage) { + //Iterate backwards of machine instructions to find the branch we need to update + for(MachineBasicBlock::reverse_iterator mInst = currentMBB->rbegin(), mInstEnd = currentMBB->rend(); mInst != mInstEnd; ++mInst) { + MachineOpCode OC = mInst->getOpcode(); + + //If its a branch update its branchto + if(TMI->isBranch(OC)) { + for(unsigned opNum = 0; opNum < mInst->getNumOperands(); ++opNum) { + MachineOperand &mOp = mInst->getOperand(opNum); + if (mOp.getType() == MachineOperand::MO_PCRelativeDisp) { + //Check if we branch to side exit + if(mOp.getVRegValue() == sideExits[mbb]) { + mOp.setValueReg(side_llvm_epilogues[P][0]); + } + } + } + DEBUG(std::cerr << "New Prologue Branch: " << *mInst << "\n"); + } + } + + //Update llvm branch + TerminatorInst *branchVal = ((BasicBlock*) currentMBB->getBasicBlock())->getTerminator(); + DEBUG(std::cerr << *branchVal << "\n"); + + for(unsigned i=0; i < branchVal->getNumSuccessors(); ++i) { + if(branchVal->getSuccessor(i) == sideExits[mbb]) { + DEBUG(std::cerr << "Replacing successor bb\n"); + branchVal->setSuccessor(i, side_llvm_epilogues[P][0]); + } + } + } + else { + //must add BA branch because another prologue or kernel has the actual side exit branch + //Add unconditional branches to original exits + assert( (sideExitNum+1) < prologues[P].size() && "must have valid prologue to branch to"); + BuildMI(prologues[P][sideExitNum], V9::BA, 1).addPCDisp((BasicBlock*)(prologues[P][sideExitNum+1])->getBasicBlock()); + BuildMI(prologues[P][sideExitNum], V9::NOP, 0); + + TerminatorInst *newBranch = new BranchInst((BasicBlock*) (prologues[P][sideExitNum+1])->getBasicBlock(), (BasicBlock*) (prologues[P][sideExitNum])->getBasicBlock()); + + } + } + + + //Update side exits in kernel + MachineBasicBlock *currentMBB = kernelMBBs[sideExitNum]; + //Iterate backwards of machine instructions to find the branch we need to update + for(MachineBasicBlock::reverse_iterator mInst = currentMBB->rbegin(), mInstEnd = currentMBB->rend(); mInst != mInstEnd; ++mInst) { + MachineOpCode OC = mInst->getOpcode(); + + //If its a branch update its branchto + if(TMI->isBranch(OC)) { + for(unsigned opNum = 0; opNum < mInst->getNumOperands(); ++opNum) { + MachineOperand &mOp = mInst->getOperand(opNum); + if (mOp.getType() == MachineOperand::MO_PCRelativeDisp) { + //Check if we branch to side exit + if(mOp.getVRegValue() == sideExits[mbb]) { + if(side_llvm_epilogues.size() > 0) + mOp.setValueReg(side_llvm_epilogues[0][0]); + else + mOp.setValueReg(sideBB); + } + } + } + DEBUG(std::cerr << "New Prologue Branch: " << *mInst << "\n"); + } + } + + //Update llvm branch + //Update llvm branch + TerminatorInst *branchVal = ((BasicBlock*)currentMBB->getBasicBlock())->getTerminator(); + DEBUG(std::cerr << *branchVal << "\n"); + + for(unsigned i=0; i < branchVal->getNumSuccessors(); ++i) { + if(branchVal->getSuccessor(i) == sideExits[mbb]) { + DEBUG(std::cerr << "Replacing successor bb\n"); + if(side_llvm_epilogues.size() > 0) + branchVal->setSuccessor(i, side_llvm_epilogues[0][0]); + else + branchVal->setSuccessor(i, sideBB); + } + } + } +} + diff --git a/lib/Target/SparcV9/ModuloScheduling/ModuloSchedulingSuperBlock.h b/lib/Target/SparcV9/ModuloScheduling/ModuloSchedulingSuperBlock.h index 8d38784a240..df9e302e9aa 100644 --- a/lib/Target/SparcV9/ModuloScheduling/ModuloSchedulingSuperBlock.h +++ b/lib/Target/SparcV9/ModuloScheduling/ModuloSchedulingSuperBlock.h @@ -15,22 +15,24 @@ #define LLVM_MODULOSCHEDULINGSB_H #include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ScalarEvolution.h" #include "llvm/Function.h" #include "llvm/Pass.h" #include "llvm/CodeGen/MachineBasicBlock.h" -#include "MSSchedule.h" -#include "MSchedGraph.h" +#include "MSScheduleSB.h" +#include "MSchedGraphSB.h" + namespace llvm { //Struct to contain ModuloScheduling Specific Information for each node - struct MSNodeAttributes { + struct MSNodeSBAttributes { int ASAP; //Earliest time at which the opreation can be scheduled int ALAP; //Latest time at which the operation can be scheduled. int MOB; int depth; int height; - MSNodeAttributes(int asap=-1, int alap=-1, int mob=-1, + MSNodeSBAttributes(int asap=-1, int alap=-1, int mob=-1, int d=-1, int h=-1) : ASAP(asap), ALAP(alap), MOB(mob), depth(d), height(h) {} @@ -55,22 +57,22 @@ namespace llvm { Instruction *defaultInst; //Map that holds node to node attribute information - std::map nodeToAttributesMap; + std::map nodeToAttributesMap; //Map to hold all reccurrences - std::set > > recurrenceList; + std::set > > recurrenceList; //Set of edges to ignore, stored as src node and index into vector of successors - std::set > edgesToIgnore; + std::set > edgesToIgnore; //Vector containing the partial order - std::vector > partialOrder; + std::vector > partialOrder; //Vector containing the final node order - std::vector FinalNodeOrder; + std::vector FinalNodeOrder; //Schedule table, key is the cycle number and the vector is resource, node pairs - MSSchedule schedule; + MSScheduleSB schedule; //Current initiation interval int II; @@ -78,10 +80,99 @@ namespace llvm { //Internal Functions void FindSuperBlocks(Function &F, LoopInfo &LI, std::vector > &Worklist); - bool MachineBBisValid(const MachineBasicBlock *B); + bool MachineBBisValid(const MachineBasicBlock *B, + std::map &indexMap, + unsigned &offset); bool CreateDefMap(std::vector &SB); + bool getIndVar(std::vector &superBlock, + std::map &bbMap, + std::map &indexMap); + bool assocIndVar(Instruction *I, std::set &indVar, + std::vector &stack, + std::map &bbMap, + const BasicBlock *first, + std::set &llvmSuperBlock); + int calculateResMII(std::vector &superBlock); + int calculateRecMII(MSchedGraphSB *graph, int MII); + void findAllCircuits(MSchedGraphSB *g, int II); + void addRecc(std::vector &stack, + std::map &newNodes); + bool circuit(MSchedGraphSBNode *v, std::vector &stack, + std::set &blocked, std::vector &SCC, + MSchedGraphSBNode *s, std::map > &B, + int II, std::map &newNodes); + void unblock(MSchedGraphSBNode *u, std::set &blocked, + std::map > &B); + void addSCC(std::vector &SCC, std::map &newNodes); + void calculateNodeAttributes(MSchedGraphSB *graph, int MII); + bool ignoreEdge(MSchedGraphSBNode *srcNode, MSchedGraphSBNode *destNode); + int calculateASAP(MSchedGraphSBNode *node, int MII, MSchedGraphSBNode *destNode); + int calculateALAP(MSchedGraphSBNode *node, int MII, + int maxASAP, MSchedGraphSBNode *srcNode); + int findMaxASAP(); + int calculateHeight(MSchedGraphSBNode *node,MSchedGraphSBNode *srcNode); + int calculateDepth(MSchedGraphSBNode *node, MSchedGraphSBNode *destNode); + void computePartialOrder(); + void connectedComponentSet(MSchedGraphSBNode *node, std::set &ccSet, + std::set &lastNodes); + void searchPath(MSchedGraphSBNode *node, + std::vector &path, + std::set &nodesToAdd, + std::set &new_reccurrence); + void orderNodes(); + bool computeSchedule(std::vector &BB, MSchedGraphSB *MSG); + bool scheduleNode(MSchedGraphSBNode *node, int start, int end); + void predIntersect(std::set &CurrentSet, std::set &IntersectResult); + void succIntersect(std::set &CurrentSet, std::set &IntersectResult); + void reconstructLoop(std::vector &SB); + void fixBranches(std::vector > &prologues, + std::vector > &llvm_prologues, + std::vector &machineKernelBB, + std::vector &llvmKernelBB, + std::vector > &epilogues, + std::vector > &llvm_epilogues, + std::vector &SB, + std::map &sideExits); - public: + void writePrologues(std::vector > &prologues, + std::vector &origBB, + std::vector > &llvm_prologues, + std::map > &valuesToSave, + std::map > &newValues, + std::map &newValLocation); + + void writeKernel(std::vector &llvmBB, std::vector &machineBB, + std::map > &valuesToSave, + std::map > &newValues, + std::map &newValLocation, + std::map > &kernelPHIs); + + void removePHIs(std::vector &SB, + std::vector > &prologues, + std::vector > &epilogues, + std::vector &kernelBB, + std::map &newValLocation); + + void writeEpilogues(std::vector > &epilogues, + std::vector &origSB, + std::vector > &llvm_epilogues, + std::map > &valuesToSave, + std::map > &newValues, + std::map &newValLocation, + std::map > &kernelPHIs); + + void writeSideExits(std::vector > &prologues, + std::vector > &llvm_prologues, + std::vector > &epilogues, + std::vector > &llvm_epilogues, + std::map &sideExits, + std::map > > &instrsMovedDown, + std::vector &SB, + std::vector &kernelMBBs, + std::map branchStage); + + public: ModuloSchedulingSBPass(TargetMachine &targ) : target(targ) {} virtual bool runOnFunction(Function &F); virtual const char* getPassName() const { return "ModuloScheduling-SuperBlock"; } @@ -89,7 +180,11 @@ namespace llvm { // getAnalysisUsage virtual void getAnalysisUsage(AnalysisUsage &AU) const { + /// HACK: We don't actually need scev, but we have + /// to say we do so that the pass manager does not delete it + /// before we run. AU.addRequired(); + AU.addRequired(); AU.addRequired(); } };