2013-04-01 21:48:05 +00:00
|
|
|
//===-- R600ControlFlowFinalizer.cpp - Finalize Control Flow Inst----------===//
|
|
|
|
//
|
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
//
|
|
|
|
/// \file
|
|
|
|
/// This pass compute turns all control flow pseudo instructions into native one
|
|
|
|
/// computing their address on the fly ; it also sets STACK_SIZE info.
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2013-04-03 16:24:09 +00:00
|
|
|
#define DEBUG_TYPE "r600cf"
|
|
|
|
#include "llvm/Support/Debug.h"
|
2013-04-01 21:48:05 +00:00
|
|
|
#include "AMDGPU.h"
|
|
|
|
#include "R600Defines.h"
|
|
|
|
#include "R600InstrInfo.h"
|
|
|
|
#include "R600MachineFunctionInfo.h"
|
|
|
|
#include "R600RegisterInfo.h"
|
|
|
|
#include "llvm/CodeGen/MachineFunctionPass.h"
|
|
|
|
#include "llvm/CodeGen/MachineInstrBuilder.h"
|
|
|
|
#include "llvm/CodeGen/MachineRegisterInfo.h"
|
2013-05-23 17:10:37 +00:00
|
|
|
#include "llvm/Support/raw_ostream.h"
|
2013-04-01 21:48:05 +00:00
|
|
|
|
2013-05-23 17:10:37 +00:00
|
|
|
using namespace llvm;
|
|
|
|
|
|
|
|
namespace {
|
2013-04-01 21:48:05 +00:00
|
|
|
|
|
|
|
class R600ControlFlowFinalizer : public MachineFunctionPass {
|
|
|
|
|
|
|
|
private:
|
2013-04-30 00:13:53 +00:00
|
|
|
typedef std::pair<MachineInstr *, std::vector<MachineInstr *> > ClauseFile;
|
|
|
|
|
2013-04-08 13:05:49 +00:00
|
|
|
enum ControlFlowInstruction {
|
|
|
|
CF_TC,
|
2013-04-30 00:13:39 +00:00
|
|
|
CF_VC,
|
2013-04-08 13:05:49 +00:00
|
|
|
CF_CALL_FS,
|
|
|
|
CF_WHILE_LOOP,
|
|
|
|
CF_END_LOOP,
|
|
|
|
CF_LOOP_BREAK,
|
|
|
|
CF_LOOP_CONTINUE,
|
|
|
|
CF_JUMP,
|
|
|
|
CF_ELSE,
|
2013-04-23 17:34:00 +00:00
|
|
|
CF_POP,
|
|
|
|
CF_END
|
2013-04-08 13:05:49 +00:00
|
|
|
};
|
2013-04-11 04:16:22 +00:00
|
|
|
|
2013-04-01 21:48:05 +00:00
|
|
|
static char ID;
|
|
|
|
const R600InstrInfo *TII;
|
2013-06-07 20:28:55 +00:00
|
|
|
const R600RegisterInfo *TRI;
|
2013-04-01 21:48:05 +00:00
|
|
|
unsigned MaxFetchInst;
|
2013-04-08 13:05:49 +00:00
|
|
|
const AMDGPUSubtarget &ST;
|
2013-04-01 21:48:05 +00:00
|
|
|
|
|
|
|
bool IsTrivialInst(MachineInstr *MI) const {
|
|
|
|
switch (MI->getOpcode()) {
|
|
|
|
case AMDGPU::KILL:
|
|
|
|
case AMDGPU::RETURN:
|
|
|
|
return true;
|
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-04-08 13:05:49 +00:00
|
|
|
const MCInstrDesc &getHWInstrDesc(ControlFlowInstruction CFI) const {
|
2013-04-23 17:34:00 +00:00
|
|
|
unsigned Opcode = 0;
|
2013-06-07 20:37:48 +00:00
|
|
|
bool isEg = (ST.getGeneration() >= AMDGPUSubtarget::EVERGREEN);
|
2013-04-23 17:34:00 +00:00
|
|
|
switch (CFI) {
|
|
|
|
case CF_TC:
|
|
|
|
Opcode = isEg ? AMDGPU::CF_TC_EG : AMDGPU::CF_TC_R600;
|
|
|
|
break;
|
2013-04-30 00:13:39 +00:00
|
|
|
case CF_VC:
|
|
|
|
Opcode = isEg ? AMDGPU::CF_VC_EG : AMDGPU::CF_VC_R600;
|
|
|
|
break;
|
2013-04-23 17:34:00 +00:00
|
|
|
case CF_CALL_FS:
|
|
|
|
Opcode = isEg ? AMDGPU::CF_CALL_FS_EG : AMDGPU::CF_CALL_FS_R600;
|
|
|
|
break;
|
|
|
|
case CF_WHILE_LOOP:
|
|
|
|
Opcode = isEg ? AMDGPU::WHILE_LOOP_EG : AMDGPU::WHILE_LOOP_R600;
|
|
|
|
break;
|
|
|
|
case CF_END_LOOP:
|
|
|
|
Opcode = isEg ? AMDGPU::END_LOOP_EG : AMDGPU::END_LOOP_R600;
|
|
|
|
break;
|
|
|
|
case CF_LOOP_BREAK:
|
|
|
|
Opcode = isEg ? AMDGPU::LOOP_BREAK_EG : AMDGPU::LOOP_BREAK_R600;
|
|
|
|
break;
|
|
|
|
case CF_LOOP_CONTINUE:
|
|
|
|
Opcode = isEg ? AMDGPU::CF_CONTINUE_EG : AMDGPU::CF_CONTINUE_R600;
|
|
|
|
break;
|
|
|
|
case CF_JUMP:
|
|
|
|
Opcode = isEg ? AMDGPU::CF_JUMP_EG : AMDGPU::CF_JUMP_R600;
|
|
|
|
break;
|
|
|
|
case CF_ELSE:
|
|
|
|
Opcode = isEg ? AMDGPU::CF_ELSE_EG : AMDGPU::CF_ELSE_R600;
|
|
|
|
break;
|
|
|
|
case CF_POP:
|
|
|
|
Opcode = isEg ? AMDGPU::POP_EG : AMDGPU::POP_R600;
|
|
|
|
break;
|
|
|
|
case CF_END:
|
2013-06-07 20:37:48 +00:00
|
|
|
if (ST.hasCaymanISA()) {
|
2013-04-23 17:34:00 +00:00
|
|
|
Opcode = AMDGPU::CF_END_CM;
|
|
|
|
break;
|
2013-04-08 13:05:49 +00:00
|
|
|
}
|
2013-04-23 17:34:00 +00:00
|
|
|
Opcode = isEg ? AMDGPU::CF_END_EG : AMDGPU::CF_END_R600;
|
|
|
|
break;
|
2013-04-08 13:05:49 +00:00
|
|
|
}
|
2013-04-23 17:34:00 +00:00
|
|
|
assert (Opcode && "No opcode selected");
|
|
|
|
return TII->get(Opcode);
|
2013-04-08 13:05:49 +00:00
|
|
|
}
|
|
|
|
|
2013-04-30 00:14:00 +00:00
|
|
|
bool isCompatibleWithClause(const MachineInstr *MI,
|
2013-06-07 23:30:26 +00:00
|
|
|
std::set<unsigned> &DstRegs) const {
|
2013-04-30 00:14:00 +00:00
|
|
|
unsigned DstMI, SrcMI;
|
|
|
|
for (MachineInstr::const_mop_iterator I = MI->operands_begin(),
|
|
|
|
E = MI->operands_end(); I != E; ++I) {
|
|
|
|
const MachineOperand &MO = *I;
|
|
|
|
if (!MO.isReg())
|
|
|
|
continue;
|
2013-05-23 18:26:42 +00:00
|
|
|
if (MO.isDef()) {
|
|
|
|
unsigned Reg = MO.getReg();
|
|
|
|
if (AMDGPU::R600_Reg128RegClass.contains(Reg))
|
|
|
|
DstMI = Reg;
|
|
|
|
else
|
2013-06-07 20:28:55 +00:00
|
|
|
DstMI = TRI->getMatchingSuperReg(Reg,
|
|
|
|
TRI->getSubRegFromChannel(TRI->getHWRegChan(Reg)),
|
2013-05-23 18:26:42 +00:00
|
|
|
&AMDGPU::R600_Reg128RegClass);
|
|
|
|
}
|
2013-04-30 00:14:00 +00:00
|
|
|
if (MO.isUse()) {
|
|
|
|
unsigned Reg = MO.getReg();
|
|
|
|
if (AMDGPU::R600_Reg128RegClass.contains(Reg))
|
|
|
|
SrcMI = Reg;
|
|
|
|
else
|
2013-06-07 20:28:55 +00:00
|
|
|
SrcMI = TRI->getMatchingSuperReg(Reg,
|
|
|
|
TRI->getSubRegFromChannel(TRI->getHWRegChan(Reg)),
|
2013-04-30 00:14:00 +00:00
|
|
|
&AMDGPU::R600_Reg128RegClass);
|
|
|
|
}
|
|
|
|
}
|
2013-06-07 23:30:26 +00:00
|
|
|
if ((DstRegs.find(SrcMI) == DstRegs.end())) {
|
2013-04-30 00:14:00 +00:00
|
|
|
DstRegs.insert(DstMI);
|
|
|
|
return true;
|
|
|
|
} else
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2013-04-30 00:13:53 +00:00
|
|
|
ClauseFile
|
|
|
|
MakeFetchClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
|
|
|
|
const {
|
2013-04-01 21:48:05 +00:00
|
|
|
MachineBasicBlock::iterator ClauseHead = I;
|
2013-04-30 00:13:53 +00:00
|
|
|
std::vector<MachineInstr *> ClauseContent;
|
2013-04-01 21:48:05 +00:00
|
|
|
unsigned AluInstCount = 0;
|
2013-04-30 00:13:39 +00:00
|
|
|
bool IsTex = TII->usesTextureCache(ClauseHead);
|
2013-06-07 23:30:26 +00:00
|
|
|
std::set<unsigned> DstRegs;
|
2013-04-01 21:48:05 +00:00
|
|
|
for (MachineBasicBlock::iterator E = MBB.end(); I != E; ++I) {
|
|
|
|
if (IsTrivialInst(I))
|
|
|
|
continue;
|
2013-05-17 16:49:55 +00:00
|
|
|
if (AluInstCount >= MaxFetchInst)
|
2013-04-30 00:13:53 +00:00
|
|
|
break;
|
2013-04-30 00:13:39 +00:00
|
|
|
if ((IsTex && !TII->usesTextureCache(I)) ||
|
|
|
|
(!IsTex && !TII->usesVertexCache(I)))
|
2013-04-01 21:48:05 +00:00
|
|
|
break;
|
2013-06-07 23:30:26 +00:00
|
|
|
if (!isCompatibleWithClause(I, DstRegs))
|
2013-04-30 00:14:00 +00:00
|
|
|
break;
|
2013-04-01 21:48:05 +00:00
|
|
|
AluInstCount ++;
|
2013-04-30 00:13:53 +00:00
|
|
|
ClauseContent.push_back(I);
|
2013-04-01 21:48:05 +00:00
|
|
|
}
|
2013-04-30 00:13:53 +00:00
|
|
|
MachineInstr *MIb = BuildMI(MBB, ClauseHead, MBB.findDebugLoc(ClauseHead),
|
2013-04-30 00:13:39 +00:00
|
|
|
getHWInstrDesc(IsTex?CF_TC:CF_VC))
|
2013-04-30 00:13:53 +00:00
|
|
|
.addImm(0) // ADDR
|
|
|
|
.addImm(AluInstCount - 1); // COUNT
|
|
|
|
return ClauseFile(MIb, ClauseContent);
|
2013-04-01 21:48:05 +00:00
|
|
|
}
|
2013-04-30 00:13:53 +00:00
|
|
|
|
2013-05-02 21:53:03 +00:00
|
|
|
void getLiteral(MachineInstr *MI, std::vector<int64_t> &Lits) const {
|
2013-07-15 06:39:13 +00:00
|
|
|
static const unsigned LiteralRegs[] = {
|
2013-04-30 00:14:38 +00:00
|
|
|
AMDGPU::ALU_LITERAL_X,
|
|
|
|
AMDGPU::ALU_LITERAL_Y,
|
|
|
|
AMDGPU::ALU_LITERAL_Z,
|
|
|
|
AMDGPU::ALU_LITERAL_W
|
|
|
|
};
|
2013-05-17 16:50:02 +00:00
|
|
|
const SmallVector<std::pair<MachineOperand *, int64_t>, 3 > Srcs =
|
|
|
|
TII->getSrcs(MI);
|
|
|
|
for (unsigned i = 0, e = Srcs.size(); i < e; ++i) {
|
|
|
|
if (Srcs[i].first->getReg() != AMDGPU::ALU_LITERAL_X)
|
2013-04-30 00:14:38 +00:00
|
|
|
continue;
|
2013-05-17 16:50:02 +00:00
|
|
|
int64_t Imm = Srcs[i].second;
|
2013-05-02 21:53:03 +00:00
|
|
|
std::vector<int64_t>::iterator It =
|
2013-04-30 00:14:38 +00:00
|
|
|
std::find(Lits.begin(), Lits.end(), Imm);
|
|
|
|
if (It != Lits.end()) {
|
|
|
|
unsigned Index = It - Lits.begin();
|
2013-05-17 16:50:02 +00:00
|
|
|
Srcs[i].first->setReg(LiteralRegs[Index]);
|
2013-04-30 00:14:38 +00:00
|
|
|
} else {
|
|
|
|
assert(Lits.size() < 4 && "Too many literals in Instruction Group");
|
2013-05-17 16:50:02 +00:00
|
|
|
Srcs[i].first->setReg(LiteralRegs[Lits.size()]);
|
2013-04-30 00:14:38 +00:00
|
|
|
Lits.push_back(Imm);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
MachineBasicBlock::iterator insertLiterals(
|
|
|
|
MachineBasicBlock::iterator InsertPos,
|
|
|
|
const std::vector<unsigned> &Literals) const {
|
|
|
|
MachineBasicBlock *MBB = InsertPos->getParent();
|
|
|
|
for (unsigned i = 0, e = Literals.size(); i < e; i+=2) {
|
|
|
|
unsigned LiteralPair0 = Literals[i];
|
|
|
|
unsigned LiteralPair1 = (i + 1 < e)?Literals[i + 1]:0;
|
|
|
|
InsertPos = BuildMI(MBB, InsertPos->getDebugLoc(),
|
|
|
|
TII->get(AMDGPU::LITERALS))
|
|
|
|
.addImm(LiteralPair0)
|
|
|
|
.addImm(LiteralPair1);
|
|
|
|
}
|
|
|
|
return InsertPos;
|
|
|
|
}
|
|
|
|
|
|
|
|
ClauseFile
|
|
|
|
MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
|
|
|
|
const {
|
|
|
|
MachineBasicBlock::iterator ClauseHead = I;
|
|
|
|
std::vector<MachineInstr *> ClauseContent;
|
|
|
|
I++;
|
|
|
|
for (MachineBasicBlock::instr_iterator E = MBB.instr_end(); I != E;) {
|
|
|
|
if (IsTrivialInst(I)) {
|
|
|
|
++I;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (!I->isBundle() && !TII->isALUInstr(I->getOpcode()))
|
|
|
|
break;
|
2013-05-02 21:53:03 +00:00
|
|
|
std::vector<int64_t> Literals;
|
2013-04-30 00:14:38 +00:00
|
|
|
if (I->isBundle()) {
|
|
|
|
MachineInstr *DeleteMI = I;
|
|
|
|
MachineBasicBlock::instr_iterator BI = I.getInstrIterator();
|
|
|
|
while (++BI != E && BI->isBundledWithPred()) {
|
|
|
|
BI->unbundleFromPred();
|
|
|
|
for (unsigned i = 0, e = BI->getNumOperands(); i != e; ++i) {
|
|
|
|
MachineOperand &MO = BI->getOperand(i);
|
|
|
|
if (MO.isReg() && MO.isInternalRead())
|
|
|
|
MO.setIsInternalRead(false);
|
|
|
|
}
|
|
|
|
getLiteral(BI, Literals);
|
|
|
|
ClauseContent.push_back(BI);
|
|
|
|
}
|
|
|
|
I = BI;
|
|
|
|
DeleteMI->eraseFromParent();
|
|
|
|
} else {
|
|
|
|
getLiteral(I, Literals);
|
|
|
|
ClauseContent.push_back(I);
|
|
|
|
I++;
|
|
|
|
}
|
|
|
|
for (unsigned i = 0, e = Literals.size(); i < e; i+=2) {
|
|
|
|
unsigned literal0 = Literals[i];
|
|
|
|
unsigned literal2 = (i + 1 < e)?Literals[i + 1]:0;
|
|
|
|
MachineInstr *MILit = BuildMI(MBB, I, I->getDebugLoc(),
|
|
|
|
TII->get(AMDGPU::LITERALS))
|
|
|
|
.addImm(literal0)
|
|
|
|
.addImm(literal2);
|
|
|
|
ClauseContent.push_back(MILit);
|
|
|
|
}
|
|
|
|
}
|
2013-07-09 15:03:33 +00:00
|
|
|
assert(ClauseContent.size() < 128 && "ALU clause is too big");
|
2013-04-30 00:14:38 +00:00
|
|
|
ClauseHead->getOperand(7).setImm(ClauseContent.size() - 1);
|
|
|
|
return ClauseFile(ClauseHead, ClauseContent);
|
|
|
|
}
|
|
|
|
|
2013-04-30 00:13:53 +00:00
|
|
|
void
|
|
|
|
EmitFetchClause(MachineBasicBlock::iterator InsertPos, ClauseFile &Clause,
|
|
|
|
unsigned &CfCount) {
|
|
|
|
CounterPropagateAddr(Clause.first, CfCount);
|
|
|
|
MachineBasicBlock *BB = Clause.first->getParent();
|
|
|
|
BuildMI(BB, InsertPos->getDebugLoc(), TII->get(AMDGPU::FETCH_CLAUSE))
|
|
|
|
.addImm(CfCount);
|
|
|
|
for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
|
|
|
|
BB->splice(InsertPos, BB, Clause.second[i]);
|
|
|
|
}
|
|
|
|
CfCount += 2 * Clause.second.size();
|
|
|
|
}
|
|
|
|
|
2013-04-30 00:14:38 +00:00
|
|
|
void
|
|
|
|
EmitALUClause(MachineBasicBlock::iterator InsertPos, ClauseFile &Clause,
|
|
|
|
unsigned &CfCount) {
|
2013-07-09 15:03:33 +00:00
|
|
|
Clause.first->getOperand(0).setImm(0);
|
2013-04-30 00:14:38 +00:00
|
|
|
CounterPropagateAddr(Clause.first, CfCount);
|
|
|
|
MachineBasicBlock *BB = Clause.first->getParent();
|
|
|
|
BuildMI(BB, InsertPos->getDebugLoc(), TII->get(AMDGPU::ALU_CLAUSE))
|
|
|
|
.addImm(CfCount);
|
|
|
|
for (unsigned i = 0, e = Clause.second.size(); i < e; ++i) {
|
|
|
|
BB->splice(InsertPos, BB, Clause.second[i]);
|
|
|
|
}
|
|
|
|
CfCount += Clause.second.size();
|
|
|
|
}
|
|
|
|
|
2013-04-01 21:48:05 +00:00
|
|
|
void CounterPropagateAddr(MachineInstr *MI, unsigned Addr) const {
|
2013-04-03 16:24:09 +00:00
|
|
|
MI->getOperand(0).setImm(Addr + MI->getOperand(0).getImm());
|
2013-04-01 21:48:05 +00:00
|
|
|
}
|
|
|
|
void CounterPropagateAddr(std::set<MachineInstr *> MIs, unsigned Addr)
|
|
|
|
const {
|
|
|
|
for (std::set<MachineInstr *>::iterator It = MIs.begin(), E = MIs.end();
|
|
|
|
It != E; ++It) {
|
|
|
|
MachineInstr *MI = *It;
|
|
|
|
CounterPropagateAddr(MI, Addr);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-04-23 17:34:12 +00:00
|
|
|
unsigned getHWStackSize(unsigned StackSubEntry, bool hasPush) const {
|
2013-06-07 20:37:48 +00:00
|
|
|
switch (ST.getGeneration()) {
|
|
|
|
case AMDGPUSubtarget::R600:
|
|
|
|
case AMDGPUSubtarget::R700:
|
2013-04-23 17:34:12 +00:00
|
|
|
if (hasPush)
|
|
|
|
StackSubEntry += 2;
|
|
|
|
break;
|
2013-06-07 20:37:48 +00:00
|
|
|
case AMDGPUSubtarget::EVERGREEN:
|
2013-04-23 17:34:12 +00:00
|
|
|
if (hasPush)
|
|
|
|
StackSubEntry ++;
|
2013-06-07 20:37:48 +00:00
|
|
|
case AMDGPUSubtarget::NORTHERN_ISLANDS:
|
2013-04-23 17:34:12 +00:00
|
|
|
StackSubEntry += 2;
|
|
|
|
break;
|
2013-06-07 20:37:48 +00:00
|
|
|
default: llvm_unreachable("Not a VLIW4/VLIW5 GPU");
|
2013-04-23 17:34:12 +00:00
|
|
|
}
|
|
|
|
return (StackSubEntry + 3)/4; // Need ceil value of StackSubEntry/4
|
|
|
|
}
|
|
|
|
|
2013-04-01 21:48:05 +00:00
|
|
|
public:
|
|
|
|
R600ControlFlowFinalizer(TargetMachine &tm) : MachineFunctionPass(ID),
|
2013-06-07 20:28:55 +00:00
|
|
|
TII (0), TRI(0),
|
2013-04-08 13:05:49 +00:00
|
|
|
ST(tm.getSubtarget<AMDGPUSubtarget>()) {
|
2013-04-01 21:48:05 +00:00
|
|
|
const AMDGPUSubtarget &ST = tm.getSubtarget<AMDGPUSubtarget>();
|
2013-05-17 16:49:55 +00:00
|
|
|
MaxFetchInst = ST.getTexVTXClauseSize();
|
2013-04-01 21:48:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
virtual bool runOnMachineFunction(MachineFunction &MF) {
|
2013-06-07 20:28:55 +00:00
|
|
|
TII=static_cast<const R600InstrInfo *>(MF.getTarget().getInstrInfo());
|
|
|
|
TRI=static_cast<const R600RegisterInfo *>(MF.getTarget().getRegisterInfo());
|
|
|
|
|
2013-04-01 21:48:05 +00:00
|
|
|
unsigned MaxStack = 0;
|
|
|
|
unsigned CurrentStack = 0;
|
2013-05-23 14:55:00 +00:00
|
|
|
bool HasPush = false;
|
2013-04-01 21:48:05 +00:00
|
|
|
for (MachineFunction::iterator MB = MF.begin(), ME = MF.end(); MB != ME;
|
|
|
|
++MB) {
|
|
|
|
MachineBasicBlock &MBB = *MB;
|
|
|
|
unsigned CfCount = 0;
|
|
|
|
std::vector<std::pair<unsigned, std::set<MachineInstr *> > > LoopStack;
|
2013-04-03 16:24:09 +00:00
|
|
|
std::vector<MachineInstr * > IfThenElseStack;
|
2013-04-01 21:48:05 +00:00
|
|
|
R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
|
|
|
|
if (MFI->ShaderType == 1) {
|
|
|
|
BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()),
|
2013-04-08 13:05:49 +00:00
|
|
|
getHWInstrDesc(CF_CALL_FS));
|
2013-04-01 21:48:05 +00:00
|
|
|
CfCount++;
|
2013-06-03 15:44:42 +00:00
|
|
|
MaxStack = 1;
|
2013-04-01 21:48:05 +00:00
|
|
|
}
|
2013-04-30 00:14:38 +00:00
|
|
|
std::vector<ClauseFile> FetchClauses, AluClauses;
|
2013-07-19 21:45:15 +00:00
|
|
|
std::vector<MachineInstr *> LastAlu(1);
|
|
|
|
std::vector<MachineInstr *> ToPopAfter;
|
|
|
|
|
2013-04-01 21:48:05 +00:00
|
|
|
for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
|
|
|
|
I != E;) {
|
2013-04-30 00:13:39 +00:00
|
|
|
if (TII->usesTextureCache(I) || TII->usesVertexCache(I)) {
|
2013-04-03 16:24:09 +00:00
|
|
|
DEBUG(dbgs() << CfCount << ":"; I->dump(););
|
2013-04-30 00:13:53 +00:00
|
|
|
FetchClauses.push_back(MakeFetchClause(MBB, I));
|
2013-04-01 21:48:05 +00:00
|
|
|
CfCount++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
MachineBasicBlock::iterator MI = I;
|
2013-07-19 21:45:15 +00:00
|
|
|
if (MI->getOpcode() != AMDGPU::ENDIF)
|
|
|
|
LastAlu.back() = 0;
|
|
|
|
if (MI->getOpcode() == AMDGPU::CF_ALU)
|
|
|
|
LastAlu.back() = MI;
|
2013-04-01 21:48:05 +00:00
|
|
|
I++;
|
|
|
|
switch (MI->getOpcode()) {
|
|
|
|
case AMDGPU::CF_ALU_PUSH_BEFORE:
|
|
|
|
CurrentStack++;
|
|
|
|
MaxStack = std::max(MaxStack, CurrentStack);
|
2013-05-23 14:55:00 +00:00
|
|
|
HasPush = true;
|
2013-04-01 21:48:05 +00:00
|
|
|
case AMDGPU::CF_ALU:
|
2013-04-30 00:14:38 +00:00
|
|
|
I = MI;
|
|
|
|
AluClauses.push_back(MakeALUClause(MBB, I));
|
2013-04-04 13:59:59 +00:00
|
|
|
case AMDGPU::EG_ExportBuf:
|
|
|
|
case AMDGPU::EG_ExportSwz:
|
|
|
|
case AMDGPU::R600_ExportBuf:
|
|
|
|
case AMDGPU::R600_ExportSwz:
|
2013-04-10 13:29:20 +00:00
|
|
|
case AMDGPU::RAT_WRITE_CACHELESS_32_eg:
|
2013-08-01 15:23:42 +00:00
|
|
|
case AMDGPU::RAT_WRITE_CACHELESS_64_eg:
|
2013-04-10 13:29:20 +00:00
|
|
|
case AMDGPU::RAT_WRITE_CACHELESS_128_eg:
|
2013-08-01 15:23:42 +00:00
|
|
|
case AMDGPU::RAT_STORE_DWORD32_cm:
|
|
|
|
case AMDGPU::RAT_STORE_DWORD64_cm:
|
2013-04-03 16:24:09 +00:00
|
|
|
DEBUG(dbgs() << CfCount << ":"; MI->dump(););
|
2013-04-01 21:48:05 +00:00
|
|
|
CfCount++;
|
|
|
|
break;
|
|
|
|
case AMDGPU::WHILELOOP: {
|
2013-04-23 17:34:12 +00:00
|
|
|
CurrentStack+=4;
|
2013-04-01 21:48:05 +00:00
|
|
|
MaxStack = std::max(MaxStack, CurrentStack);
|
|
|
|
MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
|
2013-04-08 13:05:49 +00:00
|
|
|
getHWInstrDesc(CF_WHILE_LOOP))
|
2013-04-10 13:29:20 +00:00
|
|
|
.addImm(1);
|
2013-04-01 21:48:05 +00:00
|
|
|
std::pair<unsigned, std::set<MachineInstr *> > Pair(CfCount,
|
|
|
|
std::set<MachineInstr *>());
|
|
|
|
Pair.second.insert(MIb);
|
|
|
|
LoopStack.push_back(Pair);
|
|
|
|
MI->eraseFromParent();
|
|
|
|
CfCount++;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case AMDGPU::ENDLOOP: {
|
2013-04-23 17:34:12 +00:00
|
|
|
CurrentStack-=4;
|
2013-04-01 21:48:05 +00:00
|
|
|
std::pair<unsigned, std::set<MachineInstr *> > Pair =
|
|
|
|
LoopStack.back();
|
|
|
|
LoopStack.pop_back();
|
|
|
|
CounterPropagateAddr(Pair.second, CfCount);
|
2013-04-08 13:05:49 +00:00
|
|
|
BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END_LOOP))
|
2013-04-01 21:48:05 +00:00
|
|
|
.addImm(Pair.first + 1);
|
|
|
|
MI->eraseFromParent();
|
|
|
|
CfCount++;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case AMDGPU::IF_PREDICATE_SET: {
|
2013-07-19 21:45:15 +00:00
|
|
|
LastAlu.push_back(0);
|
2013-04-01 21:48:05 +00:00
|
|
|
MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
|
2013-04-08 13:05:49 +00:00
|
|
|
getHWInstrDesc(CF_JUMP))
|
2013-04-01 21:48:05 +00:00
|
|
|
.addImm(0)
|
|
|
|
.addImm(0);
|
2013-04-03 16:24:09 +00:00
|
|
|
IfThenElseStack.push_back(MIb);
|
|
|
|
DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
|
2013-04-01 21:48:05 +00:00
|
|
|
MI->eraseFromParent();
|
|
|
|
CfCount++;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case AMDGPU::ELSE: {
|
2013-04-03 16:24:09 +00:00
|
|
|
MachineInstr * JumpInst = IfThenElseStack.back();
|
2013-04-01 21:48:05 +00:00
|
|
|
IfThenElseStack.pop_back();
|
2013-04-03 16:24:09 +00:00
|
|
|
CounterPropagateAddr(JumpInst, CfCount);
|
2013-04-01 21:48:05 +00:00
|
|
|
MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
|
2013-04-08 13:05:49 +00:00
|
|
|
getHWInstrDesc(CF_ELSE))
|
2013-04-01 21:48:05 +00:00
|
|
|
.addImm(0)
|
2013-07-19 21:45:15 +00:00
|
|
|
.addImm(0);
|
2013-04-03 16:24:09 +00:00
|
|
|
DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
|
|
|
|
IfThenElseStack.push_back(MIb);
|
2013-04-01 21:48:05 +00:00
|
|
|
MI->eraseFromParent();
|
|
|
|
CfCount++;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case AMDGPU::ENDIF: {
|
|
|
|
CurrentStack--;
|
2013-07-19 21:45:15 +00:00
|
|
|
if (LastAlu.back()) {
|
|
|
|
ToPopAfter.push_back(LastAlu.back());
|
|
|
|
} else {
|
|
|
|
MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
|
|
|
|
getHWInstrDesc(CF_POP))
|
|
|
|
.addImm(CfCount + 1)
|
|
|
|
.addImm(1);
|
|
|
|
(void)MIb;
|
|
|
|
DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
|
|
|
|
CfCount++;
|
|
|
|
}
|
|
|
|
|
2013-04-03 16:24:09 +00:00
|
|
|
MachineInstr *IfOrElseInst = IfThenElseStack.back();
|
2013-04-01 21:48:05 +00:00
|
|
|
IfThenElseStack.pop_back();
|
2013-07-19 21:45:15 +00:00
|
|
|
CounterPropagateAddr(IfOrElseInst, CfCount);
|
|
|
|
IfOrElseInst->getOperand(1).setImm(1);
|
|
|
|
LastAlu.pop_back();
|
2013-04-01 21:48:05 +00:00
|
|
|
MI->eraseFromParent();
|
|
|
|
break;
|
|
|
|
}
|
2013-07-31 19:31:14 +00:00
|
|
|
case AMDGPU::BREAK: {
|
|
|
|
CfCount ++;
|
2013-04-01 21:48:05 +00:00
|
|
|
MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
|
2013-04-08 13:05:49 +00:00
|
|
|
getHWInstrDesc(CF_LOOP_BREAK))
|
2013-04-01 21:48:05 +00:00
|
|
|
.addImm(0);
|
|
|
|
LoopStack.back().second.insert(MIb);
|
|
|
|
MI->eraseFromParent();
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case AMDGPU::CONTINUE: {
|
|
|
|
MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
|
2013-04-08 13:05:49 +00:00
|
|
|
getHWInstrDesc(CF_LOOP_CONTINUE))
|
2013-04-03 16:24:09 +00:00
|
|
|
.addImm(0);
|
2013-04-01 21:48:05 +00:00
|
|
|
LoopStack.back().second.insert(MIb);
|
|
|
|
MI->eraseFromParent();
|
|
|
|
CfCount++;
|
|
|
|
break;
|
|
|
|
}
|
2013-04-23 17:34:00 +00:00
|
|
|
case AMDGPU::RETURN: {
|
|
|
|
BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END));
|
|
|
|
CfCount++;
|
|
|
|
MI->eraseFromParent();
|
|
|
|
if (CfCount % 2) {
|
|
|
|
BuildMI(MBB, I, MBB.findDebugLoc(MI), TII->get(AMDGPU::PAD));
|
|
|
|
CfCount++;
|
|
|
|
}
|
2013-04-30 00:13:53 +00:00
|
|
|
for (unsigned i = 0, e = FetchClauses.size(); i < e; i++)
|
|
|
|
EmitFetchClause(I, FetchClauses[i], CfCount);
|
2013-04-30 00:14:38 +00:00
|
|
|
for (unsigned i = 0, e = AluClauses.size(); i < e; i++)
|
|
|
|
EmitALUClause(I, AluClauses[i], CfCount);
|
2013-04-23 17:34:00 +00:00
|
|
|
}
|
2013-04-01 21:48:05 +00:00
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2013-07-19 21:45:15 +00:00
|
|
|
for (unsigned i = 0, e = ToPopAfter.size(); i < e; ++i) {
|
|
|
|
MachineInstr *Alu = ToPopAfter[i];
|
|
|
|
BuildMI(MBB, Alu, MBB.findDebugLoc((MachineBasicBlock::iterator)Alu),
|
|
|
|
TII->get(AMDGPU::CF_ALU_POP_AFTER))
|
|
|
|
.addImm(Alu->getOperand(0).getImm())
|
|
|
|
.addImm(Alu->getOperand(1).getImm())
|
|
|
|
.addImm(Alu->getOperand(2).getImm())
|
|
|
|
.addImm(Alu->getOperand(3).getImm())
|
|
|
|
.addImm(Alu->getOperand(4).getImm())
|
|
|
|
.addImm(Alu->getOperand(5).getImm())
|
|
|
|
.addImm(Alu->getOperand(6).getImm())
|
|
|
|
.addImm(Alu->getOperand(7).getImm())
|
|
|
|
.addImm(Alu->getOperand(8).getImm());
|
|
|
|
Alu->eraseFromParent();
|
|
|
|
}
|
2013-05-23 14:55:00 +00:00
|
|
|
MFI->StackSize = getHWStackSize(MaxStack, HasPush);
|
2013-04-01 21:48:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
const char *getPassName() const {
|
|
|
|
return "R600 Control Flow Finalizer Pass";
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
char R600ControlFlowFinalizer::ID = 0;
|
|
|
|
|
2013-05-23 17:10:37 +00:00
|
|
|
} // end anonymous namespace
|
2013-04-01 21:48:05 +00:00
|
|
|
|
|
|
|
|
|
|
|
llvm::FunctionPass *llvm::createR600ControlFlowFinalizer(TargetMachine &TM) {
|
|
|
|
return new R600ControlFlowFinalizer(TM);
|
|
|
|
}
|