|
|
|
@@ -14,7 +14,6 @@
|
|
|
|
|
#include "AArch64InstrInfo.h"
|
|
|
|
|
#include "AArch64Subtarget.h"
|
|
|
|
|
#include "MCTargetDesc/AArch64AddressingModes.h"
|
|
|
|
|
#include "AArch64MachineCombinerPattern.h"
|
|
|
|
|
#include "llvm/CodeGen/MachineFrameInfo.h"
|
|
|
|
|
#include "llvm/CodeGen/MachineInstrBuilder.h"
|
|
|
|
|
#include "llvm/CodeGen/MachineMemOperand.h"
|
|
|
|
@@ -698,12 +697,17 @@ static bool UpdateOperandRegClass(MachineInstr *Instr) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// convertFlagSettingOpcode - return opcode that does not
|
|
|
|
|
/// set flags when possible. The caller is responsible to do
|
|
|
|
|
/// the actual substitution and legality checking.
|
|
|
|
|
static unsigned convertFlagSettingOpcode(MachineInstr *MI) {
|
|
|
|
|
/// optimizeCompareInstr - Convert the instruction supplying the argument to the
|
|
|
|
|
/// comparison into one that sets the zero bit in the flags register.
|
|
|
|
|
bool AArch64InstrInfo::optimizeCompareInstr(
|
|
|
|
|
MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2, int CmpMask,
|
|
|
|
|
int CmpValue, const MachineRegisterInfo *MRI) const {
|
|
|
|
|
|
|
|
|
|
// Replace SUBSWrr with SUBWrr if NZCV is not used.
|
|
|
|
|
int Cmp_NZCV = CmpInstr->findRegisterDefOperandIdx(AArch64::NZCV, true);
|
|
|
|
|
if (Cmp_NZCV != -1) {
|
|
|
|
|
unsigned NewOpc;
|
|
|
|
|
switch (MI->getOpcode()) {
|
|
|
|
|
switch (CmpInstr->getOpcode()) {
|
|
|
|
|
default:
|
|
|
|
|
return false;
|
|
|
|
|
case AArch64::ADDSWrr: NewOpc = AArch64::ADDWrr; break;
|
|
|
|
@@ -723,22 +727,7 @@ static unsigned convertFlagSettingOpcode(MachineInstr *MI) {
|
|
|
|
|
case AArch64::SUBSXrs: NewOpc = AArch64::SUBXrs; break;
|
|
|
|
|
case AArch64::SUBSXrx: NewOpc = AArch64::SUBXrx; break;
|
|
|
|
|
}
|
|
|
|
|
return NewOpc;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// optimizeCompareInstr - Convert the instruction supplying the argument to the
|
|
|
|
|
/// comparison into one that sets the zero bit in the flags register.
|
|
|
|
|
bool AArch64InstrInfo::optimizeCompareInstr(
|
|
|
|
|
MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2, int CmpMask,
|
|
|
|
|
int CmpValue, const MachineRegisterInfo *MRI) const {
|
|
|
|
|
|
|
|
|
|
// Replace SUBSWrr with SUBWrr if NZCV is not used.
|
|
|
|
|
int Cmp_NZCV = CmpInstr->findRegisterDefOperandIdx(AArch64::NZCV, true);
|
|
|
|
|
if (Cmp_NZCV != -1) {
|
|
|
|
|
unsigned Opc = CmpInstr->getOpcode();
|
|
|
|
|
unsigned NewOpc = convertFlagSettingOpcode(CmpInstr);
|
|
|
|
|
if (NewOpc == Opc)
|
|
|
|
|
return false;
|
|
|
|
|
const MCInstrDesc &MCID = get(NewOpc);
|
|
|
|
|
CmpInstr->setDesc(MCID);
|
|
|
|
|
CmpInstr->RemoveOperand(Cmp_NZCV);
|
|
|
|
@@ -2196,448 +2185,3 @@ void AArch64InstrInfo::getNoopForMachoTarget(MCInst &NopInst) const {
|
|
|
|
|
NopInst.setOpcode(AArch64::HINT);
|
|
|
|
|
NopInst.addOperand(MCOperand::CreateImm(0));
|
|
|
|
|
}
|
|
|
|
|
/// useMachineCombiner - return true when a target supports MachineCombiner
|
|
|
|
|
bool AArch64InstrInfo::useMachineCombiner(void) const {
|
|
|
|
|
// AArch64 supports the combiner
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
//
|
|
|
|
|
// True when Opc sets flag
|
|
|
|
|
static bool isCombineInstrSettingFlag(unsigned Opc) {
|
|
|
|
|
switch (Opc) {
|
|
|
|
|
case AArch64::ADDSWrr:
|
|
|
|
|
case AArch64::ADDSWri:
|
|
|
|
|
case AArch64::ADDSXrr:
|
|
|
|
|
case AArch64::ADDSXri:
|
|
|
|
|
case AArch64::SUBSWrr:
|
|
|
|
|
case AArch64::SUBSXrr:
|
|
|
|
|
// Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
|
|
|
|
|
case AArch64::SUBSWri:
|
|
|
|
|
case AArch64::SUBSXri:
|
|
|
|
|
return true;
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
//
|
|
|
|
|
// 32b Opcodes that can be combined with a MUL
|
|
|
|
|
static bool isCombineInstrCandidate32(unsigned Opc) {
|
|
|
|
|
switch (Opc) {
|
|
|
|
|
case AArch64::ADDWrr:
|
|
|
|
|
case AArch64::ADDWri:
|
|
|
|
|
case AArch64::SUBWrr:
|
|
|
|
|
case AArch64::ADDSWrr:
|
|
|
|
|
case AArch64::ADDSWri:
|
|
|
|
|
case AArch64::SUBSWrr:
|
|
|
|
|
// Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
|
|
|
|
|
case AArch64::SUBWri:
|
|
|
|
|
case AArch64::SUBSWri:
|
|
|
|
|
return true;
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
//
|
|
|
|
|
// 64b Opcodes that can be combined with a MUL
|
|
|
|
|
static bool isCombineInstrCandidate64(unsigned Opc) {
|
|
|
|
|
switch (Opc) {
|
|
|
|
|
case AArch64::ADDXrr:
|
|
|
|
|
case AArch64::ADDXri:
|
|
|
|
|
case AArch64::SUBXrr:
|
|
|
|
|
case AArch64::ADDSXrr:
|
|
|
|
|
case AArch64::ADDSXri:
|
|
|
|
|
case AArch64::SUBSXrr:
|
|
|
|
|
// Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
|
|
|
|
|
case AArch64::SUBXri:
|
|
|
|
|
case AArch64::SUBSXri:
|
|
|
|
|
return true;
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
//
|
|
|
|
|
// Opcodes that can be combined with a MUL
|
|
|
|
|
static bool isCombineInstrCandidate(unsigned Opc) {
|
|
|
|
|
return (isCombineInstrCandidate32(Opc) || isCombineInstrCandidate64(Opc));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
|
|
|
|
|
unsigned MulOpc, unsigned ZeroReg) {
|
|
|
|
|
MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
|
|
|
|
|
MachineInstr *MI = nullptr;
|
|
|
|
|
// We need a virtual register definition.
|
|
|
|
|
if (MO.isReg() && TargetRegisterInfo::isVirtualRegister(MO.getReg()))
|
|
|
|
|
MI = MRI.getUniqueVRegDef(MO.getReg());
|
|
|
|
|
// And it needs to be in the trace (otherwise, it won't have a depth).
|
|
|
|
|
if (!MI || MI->getParent() != &MBB || (unsigned)MI->getOpcode() != MulOpc)
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
|
|
|
|
|
MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
|
|
|
|
|
MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
|
|
|
|
|
|
|
|
|
|
// The third input reg must be zero.
|
|
|
|
|
if (MI->getOperand(3).getReg() != ZeroReg)
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
// Must only used by the user we combine with.
|
|
|
|
|
if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// hasPattern - return true when there is potentially a faster code sequence
|
|
|
|
|
/// for an instruction chain ending in \p Root. All potential patterns are
|
|
|
|
|
/// listed
|
|
|
|
|
/// in the \p Pattern vector. Pattern should be sorted in priority order since
|
|
|
|
|
/// the pattern evaluator stops checking as soon as it finds a faster sequence.
|
|
|
|
|
|
|
|
|
|
bool AArch64InstrInfo::hasPattern(
|
|
|
|
|
MachineInstr &Root,
|
|
|
|
|
SmallVectorImpl<MachineCombinerPattern::MC_PATTERN> &Pattern) const {
|
|
|
|
|
unsigned Opc = Root.getOpcode();
|
|
|
|
|
MachineBasicBlock &MBB = *Root.getParent();
|
|
|
|
|
bool Found = false;
|
|
|
|
|
|
|
|
|
|
if (!isCombineInstrCandidate(Opc))
|
|
|
|
|
return 0;
|
|
|
|
|
if (isCombineInstrSettingFlag(Opc)) {
|
|
|
|
|
int Cmp_NZCV = Root.findRegisterDefOperandIdx(AArch64::NZCV, true);
|
|
|
|
|
// When NZCV is live bail out.
|
|
|
|
|
if (Cmp_NZCV == -1)
|
|
|
|
|
return 0;
|
|
|
|
|
unsigned NewOpc = convertFlagSettingOpcode(&Root);
|
|
|
|
|
// When opcode can't change bail out.
|
|
|
|
|
// CHECKME: do we miss any cases for opcode conversion?
|
|
|
|
|
if (NewOpc == Opc)
|
|
|
|
|
return 0;
|
|
|
|
|
Opc = NewOpc;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
switch (Opc) {
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
case AArch64::ADDWrr:
|
|
|
|
|
assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
|
|
|
|
|
"ADDWrr does not have register operands");
|
|
|
|
|
if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
|
|
|
|
|
AArch64::WZR)) {
|
|
|
|
|
Pattern.push_back(MachineCombinerPattern::MC_MULADDW_OP1);
|
|
|
|
|
Found = true;
|
|
|
|
|
}
|
|
|
|
|
if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDWrrr,
|
|
|
|
|
AArch64::WZR)) {
|
|
|
|
|
Pattern.push_back(MachineCombinerPattern::MC_MULADDW_OP2);
|
|
|
|
|
Found = true;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
case AArch64::ADDXrr:
|
|
|
|
|
if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
|
|
|
|
|
AArch64::XZR)) {
|
|
|
|
|
Pattern.push_back(MachineCombinerPattern::MC_MULADDX_OP1);
|
|
|
|
|
Found = true;
|
|
|
|
|
}
|
|
|
|
|
if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDXrrr,
|
|
|
|
|
AArch64::XZR)) {
|
|
|
|
|
Pattern.push_back(MachineCombinerPattern::MC_MULADDX_OP2);
|
|
|
|
|
Found = true;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
case AArch64::SUBWrr:
|
|
|
|
|
if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
|
|
|
|
|
AArch64::WZR)) {
|
|
|
|
|
Pattern.push_back(MachineCombinerPattern::MC_MULSUBW_OP1);
|
|
|
|
|
Found = true;
|
|
|
|
|
}
|
|
|
|
|
if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDWrrr,
|
|
|
|
|
AArch64::WZR)) {
|
|
|
|
|
Pattern.push_back(MachineCombinerPattern::MC_MULSUBW_OP2);
|
|
|
|
|
Found = true;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
case AArch64::SUBXrr:
|
|
|
|
|
if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
|
|
|
|
|
AArch64::XZR)) {
|
|
|
|
|
Pattern.push_back(MachineCombinerPattern::MC_MULSUBX_OP1);
|
|
|
|
|
Found = true;
|
|
|
|
|
}
|
|
|
|
|
if (canCombineWithMUL(MBB, Root.getOperand(2), AArch64::MADDXrrr,
|
|
|
|
|
AArch64::XZR)) {
|
|
|
|
|
Pattern.push_back(MachineCombinerPattern::MC_MULSUBX_OP2);
|
|
|
|
|
Found = true;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
case AArch64::ADDWri:
|
|
|
|
|
if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
|
|
|
|
|
AArch64::WZR)) {
|
|
|
|
|
Pattern.push_back(MachineCombinerPattern::MC_MULADDWI_OP1);
|
|
|
|
|
Found = true;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
case AArch64::ADDXri:
|
|
|
|
|
if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
|
|
|
|
|
AArch64::XZR)) {
|
|
|
|
|
Pattern.push_back(MachineCombinerPattern::MC_MULADDXI_OP1);
|
|
|
|
|
Found = true;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
case AArch64::SUBWri:
|
|
|
|
|
if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDWrrr,
|
|
|
|
|
AArch64::WZR)) {
|
|
|
|
|
Pattern.push_back(MachineCombinerPattern::MC_MULSUBWI_OP1);
|
|
|
|
|
Found = true;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
case AArch64::SUBXri:
|
|
|
|
|
if (canCombineWithMUL(MBB, Root.getOperand(1), AArch64::MADDXrrr,
|
|
|
|
|
AArch64::XZR)) {
|
|
|
|
|
Pattern.push_back(MachineCombinerPattern::MC_MULSUBXI_OP1);
|
|
|
|
|
Found = true;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
return Found;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// genMadd - Generate madd instruction and combine mul and add.
|
|
|
|
|
/// Example:
|
|
|
|
|
/// MUL I=A,B,0
|
|
|
|
|
/// ADD R,I,C
|
|
|
|
|
/// ==> MADD R,A,B,C
|
|
|
|
|
/// \param Root is the ADD instruction
|
|
|
|
|
/// \param [out] InsInstr is a vector of machine instructions and will
|
|
|
|
|
/// contain the generated madd instruction
|
|
|
|
|
/// \param IdxMulOpd is index of operand in Root that is the result of
|
|
|
|
|
/// the MUL. In the example above IdxMulOpd is 1.
|
|
|
|
|
/// \param MaddOpc the opcode fo the madd instruction
|
|
|
|
|
static MachineInstr *genMadd(MachineFunction &MF, MachineRegisterInfo &MRI,
|
|
|
|
|
const TargetInstrInfo *TII, MachineInstr &Root,
|
|
|
|
|
SmallVectorImpl<MachineInstr *> &InsInstrs,
|
|
|
|
|
unsigned IdxMulOpd, unsigned MaddOpc) {
|
|
|
|
|
assert(IdxMulOpd == 1 || IdxMulOpd == 2);
|
|
|
|
|
|
|
|
|
|
unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
|
|
|
|
|
MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
|
|
|
|
|
MachineOperand R = Root.getOperand(0);
|
|
|
|
|
MachineOperand A = MUL->getOperand(1);
|
|
|
|
|
MachineOperand B = MUL->getOperand(2);
|
|
|
|
|
MachineOperand C = Root.getOperand(IdxOtherOpd);
|
|
|
|
|
MachineInstrBuilder MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc))
|
|
|
|
|
.addOperand(R)
|
|
|
|
|
.addOperand(A)
|
|
|
|
|
.addOperand(B)
|
|
|
|
|
.addOperand(C);
|
|
|
|
|
// Insert the MADD
|
|
|
|
|
InsInstrs.push_back(MIB);
|
|
|
|
|
return MUL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// genMaddR - Generate madd instruction and combine mul and add using
|
|
|
|
|
/// an extra virtual register
|
|
|
|
|
/// Example - an ADD intermediate needs to be stored in a register:
|
|
|
|
|
/// MUL I=A,B,0
|
|
|
|
|
/// ADD R,I,Imm
|
|
|
|
|
/// ==> ORR V, ZR, Imm
|
|
|
|
|
/// ==> MADD R,A,B,V
|
|
|
|
|
/// \param Root is the ADD instruction
|
|
|
|
|
/// \param [out] InsInstr is a vector of machine instructions and will
|
|
|
|
|
/// contain the generated madd instruction
|
|
|
|
|
/// \param IdxMulOpd is index of operand in Root that is the result of
|
|
|
|
|
/// the MUL. In the example above IdxMulOpd is 1.
|
|
|
|
|
/// \param MaddOpc the opcode fo the madd instruction
|
|
|
|
|
/// \param VR is a virtual register that holds the value of an ADD operand
|
|
|
|
|
/// (V in the example above).
|
|
|
|
|
static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
|
|
|
|
|
const TargetInstrInfo *TII, MachineInstr &Root,
|
|
|
|
|
SmallVectorImpl<MachineInstr *> &InsInstrs,
|
|
|
|
|
unsigned IdxMulOpd, unsigned MaddOpc,
|
|
|
|
|
unsigned VR) {
|
|
|
|
|
assert(IdxMulOpd == 1 || IdxMulOpd == 2);
|
|
|
|
|
|
|
|
|
|
MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
|
|
|
|
|
MachineOperand R = Root.getOperand(0);
|
|
|
|
|
MachineOperand A = MUL->getOperand(1);
|
|
|
|
|
MachineOperand B = MUL->getOperand(2);
|
|
|
|
|
MachineInstrBuilder MIB = BuildMI(MF, Root.getDebugLoc(), TII->get(MaddOpc))
|
|
|
|
|
.addOperand(R)
|
|
|
|
|
.addOperand(A)
|
|
|
|
|
.addOperand(B)
|
|
|
|
|
.addReg(VR);
|
|
|
|
|
// Insert the MADD
|
|
|
|
|
InsInstrs.push_back(MIB);
|
|
|
|
|
return MUL;
|
|
|
|
|
}
|
|
|
|
|
/// genAlternativeCodeSequence - when hasPattern() finds a pattern
|
|
|
|
|
/// this function generates the instructions that could replace the
|
|
|
|
|
/// original code sequence
|
|
|
|
|
void AArch64InstrInfo::genAlternativeCodeSequence(
|
|
|
|
|
MachineInstr &Root, MachineCombinerPattern::MC_PATTERN Pattern,
|
|
|
|
|
SmallVectorImpl<MachineInstr *> &InsInstrs,
|
|
|
|
|
SmallVectorImpl<MachineInstr *> &DelInstrs,
|
|
|
|
|
DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
|
|
|
|
|
MachineBasicBlock &MBB = *Root.getParent();
|
|
|
|
|
MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
|
|
|
|
|
MachineFunction &MF = *MBB.getParent();
|
|
|
|
|
const TargetInstrInfo *TII = MF.getTarget().getSubtargetImpl()->getInstrInfo();
|
|
|
|
|
|
|
|
|
|
MachineInstr *MUL;
|
|
|
|
|
unsigned Opc;
|
|
|
|
|
switch (Pattern) {
|
|
|
|
|
default:
|
|
|
|
|
// signal error.
|
|
|
|
|
break;
|
|
|
|
|
case MachineCombinerPattern::MC_MULADDW_OP1:
|
|
|
|
|
case MachineCombinerPattern::MC_MULADDX_OP1:
|
|
|
|
|
// MUL I=A,B,0
|
|
|
|
|
// ADD R,I,C
|
|
|
|
|
// ==> MADD R,A,B,C
|
|
|
|
|
// --- Create(MADD);
|
|
|
|
|
Opc = Pattern == MachineCombinerPattern::MC_MULADDW_OP1 ? AArch64::MADDWrrr
|
|
|
|
|
: AArch64::MADDXrrr;
|
|
|
|
|
MUL = genMadd(MF, MRI, TII, Root, InsInstrs, 1, Opc);
|
|
|
|
|
break;
|
|
|
|
|
case MachineCombinerPattern::MC_MULADDW_OP2:
|
|
|
|
|
case MachineCombinerPattern::MC_MULADDX_OP2:
|
|
|
|
|
// MUL I=A,B,0
|
|
|
|
|
// ADD R,C,I
|
|
|
|
|
// ==> MADD R,A,B,C
|
|
|
|
|
// --- Create(MADD);
|
|
|
|
|
Opc = Pattern == MachineCombinerPattern::MC_MULADDW_OP2 ? AArch64::MADDWrrr
|
|
|
|
|
: AArch64::MADDXrrr;
|
|
|
|
|
MUL = genMadd(MF, MRI, TII, Root, InsInstrs, 2, Opc);
|
|
|
|
|
break;
|
|
|
|
|
case MachineCombinerPattern::MC_MULADDWI_OP1:
|
|
|
|
|
case MachineCombinerPattern::MC_MULADDXI_OP1:
|
|
|
|
|
// MUL I=A,B,0
|
|
|
|
|
// ADD R,I,Imm
|
|
|
|
|
// ==> ORR V, ZR, Imm
|
|
|
|
|
// ==> MADD R,A,B,V
|
|
|
|
|
// --- Create(MADD);
|
|
|
|
|
{
|
|
|
|
|
const TargetRegisterClass *RC =
|
|
|
|
|
MRI.getRegClass(Root.getOperand(1).getReg());
|
|
|
|
|
unsigned NewVR = MRI.createVirtualRegister(RC);
|
|
|
|
|
unsigned BitSize, OrrOpc, ZeroReg;
|
|
|
|
|
if (Pattern == MachineCombinerPattern::MC_MULADDWI_OP1) {
|
|
|
|
|
BitSize = 32;
|
|
|
|
|
OrrOpc = AArch64::ORRWri;
|
|
|
|
|
ZeroReg = AArch64::WZR;
|
|
|
|
|
Opc = AArch64::MADDWrrr;
|
|
|
|
|
} else {
|
|
|
|
|
OrrOpc = AArch64::ORRXri;
|
|
|
|
|
BitSize = 64;
|
|
|
|
|
ZeroReg = AArch64::XZR;
|
|
|
|
|
Opc = AArch64::MADDXrrr;
|
|
|
|
|
}
|
|
|
|
|
uint64_t Imm = Root.getOperand(2).getImm();
|
|
|
|
|
|
|
|
|
|
if (Root.getOperand(3).isImm()) {
|
|
|
|
|
unsigned val = Root.getOperand(3).getImm();
|
|
|
|
|
Imm = Imm << val;
|
|
|
|
|
}
|
|
|
|
|
uint64_t UImm = Imm << (64 - BitSize) >> (64 - BitSize);
|
|
|
|
|
uint64_t Encoding;
|
|
|
|
|
|
|
|
|
|
if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
|
|
|
|
|
MachineInstrBuilder MIB1 =
|
|
|
|
|
BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc))
|
|
|
|
|
.addOperand(MachineOperand::CreateReg(NewVR, RegState::Define))
|
|
|
|
|
.addReg(ZeroReg)
|
|
|
|
|
.addImm(Encoding);
|
|
|
|
|
InsInstrs.push_back(MIB1);
|
|
|
|
|
InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
|
|
|
|
|
MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
case MachineCombinerPattern::MC_MULSUBW_OP1:
|
|
|
|
|
case MachineCombinerPattern::MC_MULSUBX_OP1: {
|
|
|
|
|
// MUL I=A,B,0
|
|
|
|
|
// SUB R,I, C
|
|
|
|
|
// ==> SUB V, 0, C
|
|
|
|
|
// ==> MADD R,A,B,V // = -C + A*B
|
|
|
|
|
// --- Create(MADD);
|
|
|
|
|
const TargetRegisterClass *RC =
|
|
|
|
|
MRI.getRegClass(Root.getOperand(1).getReg());
|
|
|
|
|
unsigned NewVR = MRI.createVirtualRegister(RC);
|
|
|
|
|
unsigned SubOpc, ZeroReg;
|
|
|
|
|
if (Pattern == MachineCombinerPattern::MC_MULSUBW_OP1) {
|
|
|
|
|
SubOpc = AArch64::SUBWrr;
|
|
|
|
|
ZeroReg = AArch64::WZR;
|
|
|
|
|
Opc = AArch64::MADDWrrr;
|
|
|
|
|
} else {
|
|
|
|
|
SubOpc = AArch64::SUBXrr;
|
|
|
|
|
ZeroReg = AArch64::XZR;
|
|
|
|
|
Opc = AArch64::MADDXrrr;
|
|
|
|
|
}
|
|
|
|
|
// SUB NewVR, 0, C
|
|
|
|
|
MachineInstrBuilder MIB1 =
|
|
|
|
|
BuildMI(MF, Root.getDebugLoc(), TII->get(SubOpc))
|
|
|
|
|
.addOperand(MachineOperand::CreateReg(NewVR, RegState::Define))
|
|
|
|
|
.addReg(ZeroReg)
|
|
|
|
|
.addOperand(Root.getOperand(2));
|
|
|
|
|
InsInstrs.push_back(MIB1);
|
|
|
|
|
InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
|
|
|
|
|
MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR);
|
|
|
|
|
} break;
|
|
|
|
|
case MachineCombinerPattern::MC_MULSUBW_OP2:
|
|
|
|
|
case MachineCombinerPattern::MC_MULSUBX_OP2:
|
|
|
|
|
// MUL I=A,B,0
|
|
|
|
|
// SUB R,C,I
|
|
|
|
|
// ==> MSUB R,A,B,C (computes C - A*B)
|
|
|
|
|
// --- Create(MSUB);
|
|
|
|
|
Opc = Pattern == MachineCombinerPattern::MC_MULSUBW_OP2 ? AArch64::MSUBWrrr
|
|
|
|
|
: AArch64::MSUBXrrr;
|
|
|
|
|
MUL = genMadd(MF, MRI, TII, Root, InsInstrs, 2, Opc);
|
|
|
|
|
break;
|
|
|
|
|
case MachineCombinerPattern::MC_MULSUBWI_OP1:
|
|
|
|
|
case MachineCombinerPattern::MC_MULSUBXI_OP1: {
|
|
|
|
|
// MUL I=A,B,0
|
|
|
|
|
// SUB R,I, Imm
|
|
|
|
|
// ==> ORR V, ZR, -Imm
|
|
|
|
|
// ==> MADD R,A,B,V // = -Imm + A*B
|
|
|
|
|
// --- Create(MADD);
|
|
|
|
|
const TargetRegisterClass *RC =
|
|
|
|
|
MRI.getRegClass(Root.getOperand(1).getReg());
|
|
|
|
|
unsigned NewVR = MRI.createVirtualRegister(RC);
|
|
|
|
|
unsigned BitSize, OrrOpc, ZeroReg;
|
|
|
|
|
if (Pattern == MachineCombinerPattern::MC_MULSUBWI_OP1) {
|
|
|
|
|
BitSize = 32;
|
|
|
|
|
OrrOpc = AArch64::ORRWri;
|
|
|
|
|
ZeroReg = AArch64::WZR;
|
|
|
|
|
Opc = AArch64::MADDWrrr;
|
|
|
|
|
} else {
|
|
|
|
|
OrrOpc = AArch64::ORRXri;
|
|
|
|
|
BitSize = 64;
|
|
|
|
|
ZeroReg = AArch64::XZR;
|
|
|
|
|
Opc = AArch64::MADDXrrr;
|
|
|
|
|
}
|
|
|
|
|
int Imm = Root.getOperand(2).getImm();
|
|
|
|
|
if (Root.getOperand(3).isImm()) {
|
|
|
|
|
unsigned val = Root.getOperand(3).getImm();
|
|
|
|
|
Imm = Imm << val;
|
|
|
|
|
}
|
|
|
|
|
uint64_t UImm = -Imm << (64 - BitSize) >> (64 - BitSize);
|
|
|
|
|
uint64_t Encoding;
|
|
|
|
|
if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) {
|
|
|
|
|
MachineInstrBuilder MIB1 =
|
|
|
|
|
BuildMI(MF, Root.getDebugLoc(), TII->get(OrrOpc))
|
|
|
|
|
.addOperand(MachineOperand::CreateReg(NewVR, RegState::Define))
|
|
|
|
|
.addReg(ZeroReg)
|
|
|
|
|
.addImm(Encoding);
|
|
|
|
|
InsInstrs.push_back(MIB1);
|
|
|
|
|
InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
|
|
|
|
|
MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR);
|
|
|
|
|
}
|
|
|
|
|
} break;
|
|
|
|
|
}
|
|
|
|
|
// Record MUL and ADD/SUB for deletion
|
|
|
|
|
DelInstrs.push_back(MUL);
|
|
|
|
|
DelInstrs.push_back(&Root);
|
|
|
|
|
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|