X86 PIC JIT support fixes: encoding bugs, add lazy pointer stubs support.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@45575 91177308-0d34-0410-b5e6-96231b3b80d8

include/llvm/CodeGen/MachineRelocation.h

@@ -39,6 +39,7 @@ class MachineRelocation {
   enum AddressType {
     isResult,         // Relocation has be transformed into its result pointer.
     isGV,             // The Target.GV field is valid.
+    isGVLazyPtr,      // Relocation of a lazily resolved GV address.
     isBB,             // Relocation of BB address.
     isExtSym,         // The Target.ExtSym field is valid.
     isConstPool,      // Relocation of constant pool address.
@@ -55,7 +56,7 @@ class MachineRelocation {
 
   union {
     void *Result;           // If this has been resolved to a resolved pointer
-    GlobalValue *GV;        // If this is a pointer to an LLVM global
+    GlobalValue *GV;        // If this is a pointer to a GV or a GV lazy ptr
     MachineBasicBlock *MBB; // If this is a pointer to a LLVM BB
     const char *ExtSym;     // If this is a pointer to a named symbol
     unsigned Index;         // Constant pool / jump table index
@@ -93,6 +94,25 @@ public:
     return Result;
   }
 
+  /// MachineRelocation::getGVLazyPtr - Return a relocation entry for a
+  /// lazily resolved GlobalValue address.
+  static MachineRelocation getGVLazyPtr(intptr_t offset,
+                                 unsigned RelocationType, 
+                                 GlobalValue *GV, intptr_t cst = 0,
+                                 bool NeedStub = 0,
+                                 bool GOTrelative = 0) {
+    assert((RelocationType & ~63) == 0 && "Relocation type too large!");
+    MachineRelocation Result;
+    Result.Offset = offset;
+    Result.ConstantVal = cst;
+    Result.TargetReloType = RelocationType;
+    Result.AddrType = isGVLazyPtr;
+    Result.NeedStub = NeedStub;
+    Result.GOTRelative = GOTrelative;
+    Result.Target.GV = GV;
+    return Result;
+  }
+
   /// MachineRelocation::getBB - Return a relocation entry for a BB.
   ///
   static MachineRelocation getBB(intptr_t offset,unsigned RelocationType,
@@ -193,6 +213,12 @@ public:
     return AddrType == isGV;
   }
 
+  /// isGlobalValueVLazyPtr - Return true if this relocation is the address
+  /// of a lazily resolved GlobalValue.
+  bool isGlobalValueLazyPtr() const {
+    return AddrType == isGVLazyPtr;
+  }
+
   /// isBasicBlock - Return true if this relocation is a basic block reference.
   ///
   bool isBasicBlock() const {
@@ -234,7 +260,8 @@ public:
   /// getGlobalValue - If this is a global value reference, return the
   /// referenced global.
   GlobalValue *getGlobalValue() const {
-    assert(isGlobalValue() && "This is not a global value reference!");
+    assert((isGlobalValue() || isGlobalValueLazyPtr()) &&
+           "This is not a global value reference!");
     return Target.GV;
   }
 

include/llvm/Target/TargetJITInfo.h

@@ -40,6 +40,13 @@ namespace llvm {
     ///
     virtual void replaceMachineCodeForFunction(void *Old, void *New) = 0;
 
+    /// emitGlobalValueLazyPtr - Use the specified MachineCodeEmitter object to
+    /// emit a lazy pointer which contains the address of the specified GV.
+    virtual void *emitGlobalValueLazyPtr(void *GV, MachineCodeEmitter &MCE) {
+      assert(0 && "This target doesn't implement emitGlobalValueLazyPtr!");
+      return 0;
+    }
+
     /// emitFunctionStub - Use the specified MachineCodeEmitter object to emit a
     /// small native function that simply calls the function at the specified
     /// address.  Return the address of the resultant function.

lib/ExecutionEngine/JIT/JITEmitter.cpp

@@ -52,6 +52,10 @@ namespace {
     /// corresponds to.
     std::map<void*, Function*> StubToFunctionMap;
 
+    /// GlobalToLazyPtrMap - Keep track of the lazy pointer created for a
+    /// particular GlobalVariable so that we can reuse them if necessary.
+    std::map<GlobalValue*, void*> GlobalToLazyPtrMap;
+
   public:
     std::map<Function*, void*>& getFunctionToStubMap(const MutexGuard& locked) {
       assert(locked.holds(TheJIT->lock));
@@ -62,6 +66,12 @@ namespace {
       assert(locked.holds(TheJIT->lock));
       return StubToFunctionMap;
     }
+
+    std::map<GlobalValue*, void*>&
+    getGlobalToLazyPtrMap(const MutexGuard& locked) {
+      assert(locked.holds(TheJIT->lock));
+      return GlobalToLazyPtrMap;
+    }
   };
 
   /// JITResolver - Keep track of, and resolve, call sites for functions that
@@ -103,6 +113,10 @@ namespace {
     /// specified address, created lazily on demand.
     void *getExternalFunctionStub(void *FnAddr);
 
+    /// getGlobalValueLazyPtr - Return a lazy pointer containing the specified
+    /// GV address.
+    void *getGlobalValueLazyPtr(GlobalValue *V, void *GVAddress);
+
     /// AddCallbackAtLocation - If the target is capable of rewriting an
     /// instruction without the use of a stub, record the location of the use so
     /// we know which function is being used at the location.
@@ -181,6 +195,25 @@ void *JITResolver::getFunctionStub(Function *F) {
   return Stub;
 }
 
+/// getGlobalValueLazyPtr - Return a lazy pointer containing the specified
+/// GV address.
+void *JITResolver::getGlobalValueLazyPtr(GlobalValue *GV, void *GVAddress) {
+  MutexGuard locked(TheJIT->lock);
+
+  // If we already have a stub for this global variable, recycle it.
+  void *&LazyPtr = state.getGlobalToLazyPtrMap(locked)[GV];
+  if (LazyPtr) return LazyPtr;
+
+  // Otherwise, codegen a new lazy pointer.
+  LazyPtr = TheJIT->getJITInfo().emitGlobalValueLazyPtr(GVAddress,
+                                                    *TheJIT->getCodeEmitter());
+
+  DOUT << "JIT: Stub emitted at [" << LazyPtr << "] for GV '"
+       << GV->getName() << "'\n";
+
+  return LazyPtr;
+}
+
 /// getExternalFunctionStub - Return a stub for the function at the
 /// specified address, created lazily on demand.
 void *JITResolver::getExternalFunctionStub(void *FnAddr) {
@@ -361,6 +394,8 @@ namespace {
     }
   private:
     void *getPointerToGlobal(GlobalValue *GV, void *Reference, bool NoNeedStub);
+    void *getPointerToGVLazyPtr(GlobalValue *V, void *Reference,
+                                bool NoNeedStub);
   };
 }
 
@@ -396,6 +431,16 @@ void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference,
   return Resolver.getFunctionStub(F);
 }
 
+void *JITEmitter::getPointerToGVLazyPtr(GlobalValue *V, void *Reference,
+                                        bool DoesntNeedStub) {
+  // Make sure GV is emitted first.
+  // FIXME: For now, if the GV is an external function we force the JIT to
+  // compile it so the lazy pointer will contain the fully resolved address.
+  void *GVAddress = getPointerToGlobal(V, Reference, true);
+  return Resolver.getGlobalValueLazyPtr(V, GVAddress);
+}
+
+
 void JITEmitter::startFunction(MachineFunction &F) {
   uintptr_t ActualSize;
   BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(),
@@ -450,6 +495,10 @@ bool JITEmitter::finishFunction(MachineFunction &F) {
         ResultPtr = getPointerToGlobal(MR.getGlobalValue(),
                                        BufferBegin+MR.getMachineCodeOffset(),
                                        MR.doesntNeedStub());
+      } else if (MR.isGlobalValueLazyPtr()) {
+        ResultPtr = getPointerToGVLazyPtr(MR.getGlobalValue(),
+                                          BufferBegin+MR.getMachineCodeOffset(),
+                                          MR.doesntNeedStub());
       } else if (MR.isBasicBlock()) {
         ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock());
       } else if (MR.isConstantPoolIndex()) {

lib/Target/X86/X86CodeEmitter.cpp

@@ -40,20 +40,17 @@ namespace {
     intptr_t PICBase;
     bool Is64BitMode;
     bool IsPIC;
-    bool IsStatic;
   public:
     static char ID;
     explicit Emitter(TargetMachine &tm, MachineCodeEmitter &mce)
       : MachineFunctionPass((intptr_t)&ID), II(0), TD(0), TM(tm), 
       MCE(mce), PICBase(0), Is64BitMode(false),
-      IsPIC(TM.getRelocationModel() == Reloc::PIC_),
+      IsPIC(TM.getRelocationModel() == Reloc::PIC_) {}
-      IsStatic(TM.getRelocationModel() == Reloc::Static) {}
     Emitter(TargetMachine &tm, MachineCodeEmitter &mce,
             const X86InstrInfo &ii, const TargetData &td, bool is64)
       : MachineFunctionPass((intptr_t)&ID), II(&ii), TD(&td), TM(tm), 
       MCE(mce), PICBase(0), Is64BitMode(is64),
-      IsPIC(TM.getRelocationModel() == Reloc::PIC_),
+      IsPIC(TM.getRelocationModel() == Reloc::PIC_) {}
-      IsStatic(TM.getRelocationModel() == Reloc::Static) {}
 
     bool runOnMachineFunction(MachineFunction &MF);
 
@@ -67,7 +64,7 @@ namespace {
     void emitPCRelativeBlockAddress(MachineBasicBlock *MBB);
     void emitGlobalAddress(GlobalValue *GV, unsigned Reloc,
                            int Disp = 0, intptr_t PCAdj = 0,
-                           bool NeedStub = false);
+                           bool NeedStub = false, bool IsLazy = false);
     void emitExternalSymbolAddress(const char *ES, unsigned Reloc);
     void emitConstPoolAddress(unsigned CPI, unsigned Reloc, int Disp = 0,
                               intptr_t PCAdj = 0);
@@ -88,6 +85,8 @@ namespace {
     unsigned getX86RegNum(unsigned RegNo);
     bool isX86_64ExtendedReg(const MachineOperand &MO);
     unsigned determineREX(const MachineInstr &MI);
+
+    bool gvNeedsLazyPtr(const GlobalValue *GV);
   };
   char Emitter::ID = 0;
 }
@@ -103,10 +102,9 @@ bool Emitter::runOnMachineFunction(MachineFunction &MF) {
   assert((MF.getTarget().getRelocationModel() != Reloc::Default ||
           MF.getTarget().getRelocationModel() != Reloc::Static) &&
          "JIT relocation model must be set to static or default!");
-  II = ((X86TargetMachine&)MF.getTarget()).getInstrInfo();
+  II = ((X86TargetMachine&)TM).getInstrInfo();
-  TD = ((X86TargetMachine&)MF.getTarget()).getTargetData();
+  TD = ((X86TargetMachine&)TM).getTargetData();
-  Is64BitMode =
+  Is64BitMode = TM.getSubtarget<X86Subtarget>().is64Bit();
-    ((X86TargetMachine&)MF.getTarget()).getSubtarget<X86Subtarget>().is64Bit();
 
   do {
     MCE.startFunction(MF);
@@ -139,11 +137,19 @@ void Emitter::emitPCRelativeBlockAddress(MachineBasicBlock *MBB) {
 ///
 void Emitter::emitGlobalAddress(GlobalValue *GV, unsigned Reloc,
                                 int Disp /* = 0 */, intptr_t PCAdj /* = 0 */,
-                                bool NeedStub /* = false */) {
+                                bool NeedStub /* = false */,
+                                bool isLazy /* = false */) {
+  intptr_t RelocCST = 0;
   if (Reloc == X86::reloc_picrel_word)
-    PCAdj += PICBase;
+    RelocCST = PICBase;
-  MCE.addRelocation(MachineRelocation::getGV(MCE.getCurrentPCOffset(), Reloc,
+  else if (Reloc == X86::reloc_pcrel_word)
-                                             GV, PCAdj, NeedStub));
+    RelocCST = PCAdj;
+  MachineRelocation MR = isLazy 
+    ? MachineRelocation::getGVLazyPtr(MCE.getCurrentPCOffset(), Reloc,
+                                      GV, RelocCST, NeedStub)
+    : MachineRelocation::getGV(MCE.getCurrentPCOffset(), Reloc,
+                               GV, RelocCST, NeedStub);
+  MCE.addRelocation(MR);
   if (Reloc == X86::reloc_absolute_dword)
     MCE.emitWordLE(0);
   MCE.emitWordLE(Disp); // The relocated value will be added to the displacement
@@ -153,9 +159,9 @@ void Emitter::emitGlobalAddress(GlobalValue *GV, unsigned Reloc,
 /// be emitted to the current location in the function, and allow it to be PC
 /// relative.
 void Emitter::emitExternalSymbolAddress(const char *ES, unsigned Reloc) {
-  intptr_t PCAdj = (Reloc == X86::reloc_picrel_word) ? PICBase : 0;
+  intptr_t RelocCST = (Reloc == X86::reloc_picrel_word) ? PICBase : 0;
   MCE.addRelocation(MachineRelocation::getExtSym(MCE.getCurrentPCOffset(),
-                                                 Reloc, ES, PCAdj));
+                                                 Reloc, ES, RelocCST));
   if (Reloc == X86::reloc_absolute_dword)
     MCE.emitWordLE(0);
   MCE.emitWordLE(0);
@@ -167,10 +173,13 @@ void Emitter::emitExternalSymbolAddress(const char *ES, unsigned Reloc) {
 void Emitter::emitConstPoolAddress(unsigned CPI, unsigned Reloc,
                                    int Disp /* = 0 */,
                                    intptr_t PCAdj /* = 0 */) {
+  intptr_t RelocCST = 0;
   if (Reloc == X86::reloc_picrel_word)
-    PCAdj += PICBase;
+    RelocCST = PICBase;
+  else if (Reloc == X86::reloc_pcrel_word)
+    RelocCST = PCAdj;
   MCE.addRelocation(MachineRelocation::getConstPool(MCE.getCurrentPCOffset(),
-                                                    Reloc, CPI, PCAdj));
+                                                    Reloc, CPI, RelocCST));
   if (Reloc == X86::reloc_absolute_dword)
     MCE.emitWordLE(0);
   MCE.emitWordLE(Disp); // The relocated value will be added to the displacement
@@ -181,10 +190,13 @@ void Emitter::emitConstPoolAddress(unsigned CPI, unsigned Reloc,
 /// relative.
 void Emitter::emitJumpTableAddress(unsigned JTI, unsigned Reloc,
                                    intptr_t PCAdj /* = 0 */) {
+  intptr_t RelocCST = 0;
   if (Reloc == X86::reloc_picrel_word)
-    PCAdj += PICBase;
+    RelocCST = PICBase;
+  else if (Reloc == X86::reloc_pcrel_word)
+    RelocCST = PCAdj;
   MCE.addRelocation(MachineRelocation::getJumpTable(MCE.getCurrentPCOffset(),
-                                                    Reloc, JTI, PCAdj));
+                                                    Reloc, JTI, RelocCST));
   if (Reloc == X86::reloc_absolute_dword)
     MCE.emitWordLE(0);
   MCE.emitWordLE(0); // The relocated value will be added to the displacement
@@ -223,6 +235,11 @@ static bool isDisp8(int Value) {
   return Value == (signed char)Value;
 }
 
+bool Emitter::gvNeedsLazyPtr(const GlobalValue *GV) {
+  return !Is64BitMode && 
+    TM.getSubtarget<X86Subtarget>().GVRequiresExtraLoad(GV, TM, false);
+}
+
 void Emitter::emitDisplacementField(const MachineOperand *RelocOp,
                                     int DispVal, intptr_t PCAdj) {
   // If this is a simple integer displacement that doesn't require a relocation,
@@ -241,9 +258,10 @@ void Emitter::emitDisplacementField(const MachineOperand *RelocOp,
     //	89 04 25 00 00 00 00 	mov    %eax,0x0      # Absolute
     unsigned rt = Is64BitMode ? X86::reloc_pcrel_word
       : (IsPIC ? X86::reloc_picrel_word : X86::reloc_absolute_word);
-    bool NeedStub = !IsStatic || isa<Function>(RelocOp->getGlobal());
+    bool NeedStub = isa<Function>(RelocOp->getGlobal());
+    bool isLazy = gvNeedsLazyPtr(RelocOp->getGlobal());
     emitGlobalAddress(RelocOp->getGlobal(), rt, RelocOp->getOffset(),
-                      PCAdj, NeedStub);
+                      PCAdj, NeedStub, isLazy);
   } else if (RelocOp->isConstantPoolIndex()) {
     unsigned rt = Is64BitMode ? X86::reloc_pcrel_word : X86::reloc_picrel_word;
     emitConstPoolAddress(RelocOp->getIndex(), rt,
@@ -601,8 +619,7 @@ void Emitter::emitInstruction(const MachineInstr &MI) {
       if (MO.isMachineBasicBlock()) {
         emitPCRelativeBlockAddress(MO.getMBB());
       } else if (MO.isGlobalAddress()) {
-        bool NeedStub = !IsStatic ||
+        bool NeedStub = Is64BitMode && TM.getCodeModel() == CodeModel::Large;
-          (Is64BitMode && TM.getCodeModel() == CodeModel::Large);
         emitGlobalAddress(MO.getGlobal(), X86::reloc_pcrel_word,
                           0, 0, NeedStub);
       } else if (MO.isExternalSymbol()) {
@@ -634,8 +651,10 @@ void Emitter::emitInstruction(const MachineInstr &MI) {
         if (Opcode == X86::MOV64ri)
           rt = X86::reloc_absolute_dword;  // FIXME: add X86II flag?
         if (MO1.isGlobalAddress()) {
-          bool NeedStub = !IsStatic || isa<Function>(MO1.getGlobal());
+          bool NeedStub = isa<Function>(MO1.getGlobal());
-          emitGlobalAddress(MO1.getGlobal(), rt, MO1.getOffset(), 0, NeedStub);
+          bool isLazy = gvNeedsLazyPtr(MO1.getGlobal());
+          emitGlobalAddress(MO1.getGlobal(), rt, MO1.getOffset(), 0,
+                            NeedStub, isLazy);
         } else if (MO1.isExternalSymbol())
           emitExternalSymbolAddress(MO1.getSymbolName(), rt);
         else if (MO1.isConstantPoolIndex())
@@ -704,8 +723,10 @@ void Emitter::emitInstruction(const MachineInstr &MI) {
         if (Opcode == X86::MOV64ri32)
           rt = X86::reloc_absolute_word;  // FIXME: add X86II flag?
         if (MO1.isGlobalAddress()) {
-          bool NeedStub = !IsStatic || isa<Function>(MO1.getGlobal());
+          bool NeedStub = isa<Function>(MO1.getGlobal());
-          emitGlobalAddress(MO1.getGlobal(), rt, MO1.getOffset(), 0, NeedStub);
+          bool isLazy = gvNeedsLazyPtr(MO1.getGlobal());
+          emitGlobalAddress(MO1.getGlobal(), rt, MO1.getOffset(), 0,
+                            NeedStub, isLazy);
         } else if (MO1.isExternalSymbol())
           emitExternalSymbolAddress(MO1.getSymbolName(), rt);
         else if (MO1.isConstantPoolIndex())
@@ -739,8 +760,10 @@ void Emitter::emitInstruction(const MachineInstr &MI) {
         if (Opcode == X86::MOV64mi32)
           rt = X86::reloc_absolute_word;  // FIXME: add X86II flag?
         if (MO.isGlobalAddress()) {
-          bool NeedStub = !IsStatic || isa<Function>(MO.getGlobal());
+          bool NeedStub = isa<Function>(MO.getGlobal());
-          emitGlobalAddress(MO.getGlobal(), rt, MO.getOffset(), 0, NeedStub);
+          bool isLazy = gvNeedsLazyPtr(MO.getGlobal());
+          emitGlobalAddress(MO.getGlobal(), rt, MO.getOffset(), 0,
+                            NeedStub, isLazy);
         } else if (MO.isExternalSymbol())
           emitExternalSymbolAddress(MO.getSymbolName(), rt);
         else if (MO.isConstantPoolIndex())

lib/Target/X86/X86JITInfo.cpp

@@ -380,6 +380,18 @@ X86JITInfo::getLazyResolverFunction(JITCompilerFn F) {
   return X86CompilationCallback;
 }
 
+void *X86JITInfo::emitGlobalValueLazyPtr(void *GV, MachineCodeEmitter &MCE) {
+#ifdef __x86_64__
+  MCE.startFunctionStub(8, 8);
+  MCE.emitWordLE(((unsigned *)&GV)[0]);
+  MCE.emitWordLE(((unsigned *)&GV)[1]);
+#else
+  MCE.startFunctionStub(4, 4);
+  MCE.emitWordLE((unsigned)GV);
+#endif
+  return MCE.finishFunctionStub(0);
+}
+
 void *X86JITInfo::emitFunctionStub(void *Fn, MachineCodeEmitter &MCE) {
   // Note, we cast to intptr_t here to silence a -pedantic warning that 
   // complains about casting a function pointer to a normal pointer.

lib/Target/X86/X86JITInfo.h

@@ -31,6 +31,10 @@ namespace llvm {
     ///
     virtual void replaceMachineCodeForFunction(void *Old, void *New);
 
+    /// emitGlobalValueLazyPtr - Use the specified MachineCodeEmitter object to
+    /// emit a lazy pointer which contains the address of the specified GV.
+    virtual void *emitGlobalValueLazyPtr(void *GV, MachineCodeEmitter &MCE);
+
     /// emitFunctionStub - Use the specified MachineCodeEmitter object to emit a
     /// small native function that simply calls the function at the specified
     /// address.