diff --git a/include/llvm/CodeGen/FastISel.h b/include/llvm/CodeGen/FastISel.h index b1bb8eb1708..7d9e2078385 100644 --- a/include/llvm/CodeGen/FastISel.h +++ b/include/llvm/CodeGen/FastISel.h @@ -23,6 +23,7 @@ namespace llvm { class AllocaInst; class Constant; class ConstantFP; +class CallInst; class DataLayout; class FunctionLoweringInfo; class Instruction; @@ -411,6 +412,9 @@ private: /// beginning of the block. It helps to avoid spilling cached variables across /// heavy instructions like calls. void flushLocalValueMap(); + + bool addStackMapLiveVars(SmallVectorImpl &Ops, + const CallInst *CI, unsigned StartIdx); }; } diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp index 99931c14ebc..94d3be90cb9 100644 --- a/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -45,9 +45,11 @@ #include "llvm/Analysis/Loads.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/FunctionLoweringInfo.h" +#include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/StackMaps.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/Function.h" @@ -558,6 +560,36 @@ bool FastISel::SelectGetElementPtr(const User *I) { return true; } +/// \brief Add a stack map intrinsic call's live variable operands to a stackmap +/// or patchpoint machine instruction. +/// +bool FastISel::addStackMapLiveVars(SmallVectorImpl &Ops, + const CallInst *CI, unsigned StartIdx) { + for (unsigned i = StartIdx, e = CI->getNumArgOperands(); i != e; ++i) { + Value *Val = CI->getArgOperand(i); + if (auto *C = dyn_cast(Val)) { + Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp)); + Ops.push_back(MachineOperand::CreateImm(C->getSExtValue())); + } else if (isa(Val)) { + Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp)); + Ops.push_back(MachineOperand::CreateImm(0)); + } else if (auto *AI = dyn_cast(Val)) { + auto SI = FuncInfo.StaticAllocaMap.find(AI); + if (SI != FuncInfo.StaticAllocaMap.end()) + Ops.push_back(MachineOperand::CreateFI(SI->second)); + else + return false; + } else { + unsigned Reg = getRegForValue(Val); + if (Reg == 0) + return false; + Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false)); + } + } + + return true; +} + bool FastISel::SelectCall(const User *I) { const CallInst *Call = cast(I); @@ -713,6 +745,76 @@ bool FastISel::SelectCall(const User *I) { UpdateValueMap(Call, ResultReg); return true; } + case Intrinsic::experimental_stackmap: { + // void @llvm.experimental.stackmap(i64 , i32 , + // [live variables...]) + + assert(Call->getCalledFunction()->getReturnType()->isVoidTy() && + "Stackmap cannot return a value."); + + // The stackmap intrinsic only records the live variables (the arguments + // passed to it) and emits NOPS (if requested). Unlike the patchpoint + // intrinsic, this won't be lowered to a function call. This means we don't + // have to worry about calling conventions and target-specific lowering + // code. Instead we perform the call lowering right here. + // + // CALLSEQ_START(0) + // STACKMAP(id, nbytes, ...) + // CALLSEQ_END(0, 0) + // + + SmallVector Ops; + + // Add the and constants. + assert(isa(Call->getOperand(PatchPointOpers::IDPos)) && + "Expected a constant integer."); + auto IDVal = cast(Call->getOperand(PatchPointOpers::IDPos)); + Ops.push_back(MachineOperand::CreateImm(IDVal->getZExtValue())); + + assert(isa(Call->getOperand(PatchPointOpers::NBytesPos)) && + "Expected a constant integer."); + auto NBytesVal = + cast(Call->getOperand(PatchPointOpers::NBytesPos)); + Ops.push_back(MachineOperand::CreateImm(NBytesVal->getZExtValue())); + + // Push live variables for the stack map. + if (!addStackMapLiveVars(Ops, Call, 2)) + return false; + + // We are not adding any register mask info here, because the stackmap + // doesn't clobber anything. + + // Add scratch registers as implicit def and early clobber. + CallingConv::ID CC = Call->getCallingConv(); + const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC); + for (unsigned i = 0; ScratchRegs[i]; ++i) + Ops.push_back(MachineOperand::CreateReg( + ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false, + /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true)); + + // Issue CALLSEQ_START + unsigned AdjStackDown = TII.getCallFrameSetupOpcode(); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown)) + .addImm(0); + + // Issue STACKMAP. + MachineInstrBuilder MIB; + MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, + TII.get(TargetOpcode::STACKMAP)); + + for (auto const &MO : Ops) + MIB.addOperand(MO); + + // Issue CALLSEQ_END + unsigned AdjStackUp = TII.getCallFrameDestroyOpcode(); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp)) + .addImm(0).addImm(0); + + // Inform the Frame Information that we have a stackmap in this function. + FuncInfo.MF->getFrameInfo()->setHasStackMap(); + + return true; + } } // Usually, it does not make sense to initialize a value, diff --git a/test/CodeGen/X86/stackmap-fast-isel.ll b/test/CodeGen/X86/stackmap-fast-isel.ll new file mode 100644 index 00000000000..0b7e6dbdc7a --- /dev/null +++ b/test/CodeGen/X86/stackmap-fast-isel.ll @@ -0,0 +1,165 @@ +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7 -disable-fp-elim | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7 -disable-fp-elim -fast-isel -fast-isel-abort | FileCheck %s + +; CHECK-LABEL: .section __LLVM_STACKMAPS,__llvm_stackmaps +; CHECK-NEXT: __LLVM_StackMaps: +; Header +; CHECK-NEXT: .byte 1 +; CHECK-NEXT: .byte 0 +; CHECK-NEXT: .short 0 +; Num Functions +; CHECK-NEXT: .long 4 +; Num LargeConstants +; CHECK-NEXT: .long 3 +; Num Callsites +; CHECK-NEXT: .long 7 + +; Functions and stack size +; CHECK-NEXT: .quad _constantargs +; CHECK-NEXT: .quad 8 +; CHECK-NEXT: .quad _liveConstant +; CHECK-NEXT: .quad 8 +; CHECK-NEXT: .quad _directFrameIdx +; CHECK-NEXT: .quad 40 +; CHECK-NEXT: .quad _longid +; CHECK-NEXT: .quad 8 + +; Large Constants +; CHECK-NEXT: .quad 2147483648 +; CHECK-NEXT: .quad 4294967295 +; CHECK-NEXT: .quad 4294967296 + +; Callsites +; Constant arguments +; +; CHECK-NEXT: .quad 1 +; CHECK-NEXT: .long L{{.*}}-_constantargs +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .short 12 +; SmallConstant +; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long -1 +; SmallConstant +; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long -1 +; SmallConstant +; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 65536 +; SmallConstant +; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 2000000000 +; SmallConstant +; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 2147483647 +; SmallConstant +; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long -1 +; SmallConstant +; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long -1 +; SmallConstant +; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 0 +; LargeConstant at index 0 +; CHECK-NEXT: .byte 5 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 0 +; LargeConstant at index 1 +; CHECK-NEXT: .byte 5 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 1 +; LargeConstant at index 2 +; CHECK-NEXT: .byte 5 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 2 +; SmallConstant +; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long -1 + +define void @constantargs() { +entry: + tail call void (i64, i32, ...)* @llvm.experimental.stackmap(i64 1, i32 15, i16 65535, i16 -1, i32 65536, i32 2000000000, i32 2147483647, i32 -1, i32 4294967295, i32 4294967296, i64 2147483648, i64 4294967295, i64 4294967296, i64 -1) + ret void +} + +; Map a constant value. +; +; CHECK-LABEL: .long L{{.*}}-_liveConstant +; CHECK-NEXT: .short 0 +; 1 location +; CHECK-NEXT: .short 1 +; Loc 0: SmallConstant +; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .short 0 +; CHECK-NEXT: .long 33 + +define void @liveConstant() { + tail call void (i64, i32, ...)* @llvm.experimental.stackmap(i64 15, i32 5, i32 33) + ret void +} + +; Directly map an alloca's address. +; +; Callsite 16 +; CHECK-LABEL: .long L{{.*}}-_directFrameIdx +; CHECK-NEXT: .short 0 +; 1 location +; CHECK-NEXT: .short 1 +; Loc 0: Direct RBP - ofs +; CHECK-NEXT: .byte 2 +; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .short 6 +; CHECK-NEXT: .long + +define void @directFrameIdx() { +entry: + %metadata1 = alloca i64, i32 3, align 8 + store i64 11, i64* %metadata1 + store i64 12, i64* %metadata1 + store i64 13, i64* %metadata1 + call void (i64, i32, ...)* @llvm.experimental.stackmap(i64 16, i32 0, i64* %metadata1) + ret void +} + +; Test a 64-bit ID. +; +; CHECK: .quad 4294967295 +; CHECK-LABEL: .long L{{.*}}-_longid +; CHECK: .quad 4294967296 +; CHECK-LABEL: .long L{{.*}}-_longid +; CHECK: .quad 9223372036854775807 +; CHECK-LABEL: .long L{{.*}}-_longid +; CHECK: .quad -1 +; CHECK-LABEL: .long L{{.*}}-_longid +define void @longid() { +entry: + tail call void (i64, i32, ...)* @llvm.experimental.stackmap(i64 4294967295, i32 0) + tail call void (i64, i32, ...)* @llvm.experimental.stackmap(i64 4294967296, i32 0) + tail call void (i64, i32, ...)* @llvm.experimental.stackmap(i64 9223372036854775807, i32 0) + tail call void (i64, i32, ...)* @llvm.experimental.stackmap(i64 -1, i32 0) + ret void +} + +declare void @llvm.experimental.stackmap(i64, i32, ...)