- Refactor the code that resolve basic block references to a TargetJITInfo

method.
- Added synchronizeICache() to TargetJITInfo. It is called after each block
  of code is emitted to flush the icache. This ensures correct execution
  on targets that have separate dcache and icache.
- Added PPC / Mac OS X specific code to do icache flushing.


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@29276 91177308-0d34-0410-b5e6-96231b3b80d8

include/llvm/Target/TargetJITInfo.h

@@ -18,10 +18,12 @@
 #define LLVM_TARGET_TARGETJITINFO_H
 
 #include <cassert>
+#include <vector>
 
 namespace llvm {
   class Function;
   class FunctionPassManager;
+  class MachineBasicBlock;
   class MachineCodeEmitter;
   class MachineRelocation;
 
@@ -81,6 +83,20 @@ namespace llvm {
       assert(NumRelocs == 0 && "This target does not have relocations!");
     }
 
+    /// resolveBBRefs - Resolve branches to BasicBlocks for the JIT emitted
+    /// function.
+    virtual void resolveBBRefs(MachineCodeEmitter &MCE) {}
+
+    /// synchronizeICache - On some targets, the JIT emitted code must be
+    /// explicitly refetched to ensure correct execution.
+    virtual void synchronizeICache(const void *Addr, size_t len) {}
+
+    /// addBBRef - Add a BasicBlock reference to be resolved after the function
+    /// is emitted.
+    void addBBRef(MachineBasicBlock *BB, intptr_t PC) {
+      BBRefs.push_back(std::make_pair(BB, PC));
+    }
+
     /// needsGOT - Allows a target to specify that it would like the
     // JIT to manage a GOT for it.
     bool needsGOT() const { return useGOT; }
@@ -88,6 +104,9 @@ namespace llvm {
   protected:
     bool useGOT;
 
+    // Tracks which instruction references which BasicBlock
+    std::vector<std::pair<MachineBasicBlock*, intptr_t> > BBRefs;
+    
   };
 } // End llvm namespace
 

lib/ExecutionEngine/JIT/JITEmitter.cpp

@@ -542,6 +542,10 @@ void *JITResolver::getFunctionStub(Function *F) {
     TheJIT->updateGlobalMapping(F, Stub);
   }
 
+  // Invalidate the icache if necessary.
+  TheJIT->getJITInfo().
+    synchronizeICache(Stub, MCE.getCurrentPCValue()-(intptr_t)Stub);
+
   DEBUG(std::cerr << "JIT: Stub emitted at [" << Stub << "] for function '"
                   << F->getName() << "'\n");
 
@@ -559,6 +563,11 @@ void *JITResolver::getExternalFunctionStub(void *FnAddr) {
   if (Stub) return Stub;
 
   Stub = TheJIT->getJITInfo().emitFunctionStub(FnAddr, MCE);
+
+  // Invalidate the icache if necessary.
+  TheJIT->getJITInfo().
+    synchronizeICache(Stub, MCE.getCurrentPCValue()-(intptr_t)Stub);
+
   DEBUG(std::cerr << "JIT: Stub emitted at [" << Stub
         << "] for external function at '" << FnAddr << "'\n");
   return Stub;
@@ -747,7 +756,7 @@ void JITEmitter::startFunction(MachineFunction &F) {
   // About to start emitting the machine code for the function.
   emitAlignment(std::max(F.getFunction()->getAlignment(), 8U));
   TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr);
-  
+
   MBBLocations.clear();
 }
 
@@ -825,6 +834,12 @@ bool JITEmitter::finishFunction(MachineFunction &F) {
     }
   }
 
+  // Resolve BasicaBlock references.
+  TheJIT->getJITInfo().resolveBBRefs(*this);
+
+  // Invalidate the icache if necessary.
+  TheJIT->getJITInfo().synchronizeICache(FnStart, FnEnd-FnStart);
+
   DEBUG(std::cerr << "JIT: Finished CodeGen of [" << (void*)FnStart
                   << "] Function: " << F.getFunction()->getName()
                   << ": " << (FnEnd-FnStart) << " bytes of text, "

lib/Target/Alpha/Alpha.h

@@ -19,6 +19,7 @@
 
 namespace llvm {
 
+  class AlphaTargetMachine;
   class FunctionPass;
   class TargetMachine;
   class MachineCodeEmitter;
@@ -28,7 +29,8 @@ namespace llvm {
   FunctionPass *createAlphaCodePrinterPass(std::ostream &OS,
                                              TargetMachine &TM);
   FunctionPass *createAlphaPatternInstructionSelector(TargetMachine &TM);
-  FunctionPass *createAlphaCodeEmitterPass(MachineCodeEmitter &MCE);
+  FunctionPass *createAlphaCodeEmitterPass(AlphaTargetMachine &TM,
+                                           MachineCodeEmitter &MCE);
 } // end namespace llvm;
 
 // Defines symbolic names for Alpha registers.  This defines a mapping from

lib/Target/Alpha/AlphaCodeEmitter.cpp

@@ -34,17 +34,19 @@ namespace {
 namespace {
   class AlphaCodeEmitter : public MachineFunctionPass {
     const AlphaInstrInfo  *II;
+    TargetMachine &TM;
     MachineCodeEmitter  &MCE;
-    std::vector<std::pair<MachineBasicBlock *, unsigned*> > BBRefs;
 
     /// getMachineOpValue - evaluates the MachineOperand of a given MachineInstr
     ///
     int getMachineOpValue(MachineInstr &MI, MachineOperand &MO);
 
   public:
-    explicit AlphaCodeEmitter(MachineCodeEmitter &mce) : II(0), MCE(mce) {}
+    explicit AlphaCodeEmitter(TargetMachine &tm, MachineCodeEmitter &mce)
-    AlphaCodeEmitter(MachineCodeEmitter &mce, const AlphaInstrInfo& ii)
+      : II(0), TM(tm), MCE(mce) {}
-        : II(&ii), MCE(mce) {}
+    AlphaCodeEmitter(TargetMachine &tm, MachineCodeEmitter &mce,
+                     const AlphaInstrInfo& ii)
+      : II(&ii), TM(tm), MCE(mce) {}
 
     bool runOnMachineFunction(MachineFunction &MF);
 
@@ -68,34 +70,20 @@ namespace {
 
 /// createAlphaCodeEmitterPass - Return a pass that emits the collected Alpha code
 /// to the specified MCE object.
-FunctionPass *llvm::createAlphaCodeEmitterPass(MachineCodeEmitter &MCE) {
+FunctionPass *llvm::createAlphaCodeEmitterPass(AlphaTargetMachine &TM,
-  return new AlphaCodeEmitter(MCE);
+                                               MachineCodeEmitter &MCE) {
+  return new AlphaCodeEmitter(TM, MCE);
 }
 
 bool AlphaCodeEmitter::runOnMachineFunction(MachineFunction &MF) {
   II = ((AlphaTargetMachine&)MF.getTarget()).getInstrInfo();
 
   do {
-    BBRefs.clear();
-    
     MCE.startFunction(MF);
     for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I)
       emitBasicBlock(*I);
   } while (MCE.finishFunction(MF));
 
-  // Resolve all forward branches now...
-  for (unsigned i = 0, e = BBRefs.size(); i != e; ++i) {
-    unsigned* Location =
-      (unsigned*)MCE.getMachineBasicBlockAddress(BBRefs[i].first);
-    unsigned* Ref = (unsigned*)BBRefs[i].second;
-    intptr_t BranchTargetDisp = 
-      (((unsigned char*)Location  - (unsigned char*)Ref) >> 2) - 1;
-    DEBUG(std::cerr << "Fixup @ " << (void*)Ref << " to " << (void*)Location
-          << " Disp " << BranchTargetDisp 
-          << " using " <<  (BranchTargetDisp & ((1 << 22)-1)) << "\n");
-    *Ref |= (BranchTargetDisp & ((1 << 21)-1));
-  }
-  BBRefs.clear();
   return false;
 }
 
@@ -227,8 +215,8 @@ int AlphaCodeEmitter::getMachineOpValue(MachineInstr &MI, MachineOperand &MO) {
                                           Reloc, MO.getConstantPoolIndex(),
                                           Offset));
   } else if (MO.isMachineBasicBlock()) {
-    unsigned* CurrPC = (unsigned*)(intptr_t)MCE.getCurrentPCValue();
+    TM.getJITInfo()->addBBRef(MO.getMachineBasicBlock(),
-    BBRefs.push_back(std::make_pair(MO.getMachineBasicBlock(), CurrPC));
+                              MCE.getCurrentPCValue());
   }else {
     std::cerr << "ERROR: Unknown type of MachineOperand: " << MO << "\n";
     abort();

lib/Target/Alpha/AlphaJITInfo.cpp

@@ -304,3 +304,19 @@ void AlphaJITInfo::relocate(void *Function, MachineRelocation *MR,
     }
   }
 }
+
+void AlphaJITInfo::resolveBBRefs(MachineCodeEmitter &MCE) {
+  // Resolve all forward branches now...
+  for (unsigned i = 0, e = BBRefs.size(); i != e; ++i) {
+    unsigned* Location =
+      (unsigned*)MCE.getMachineBasicBlockAddress(BBRefs[i].first);
+    unsigned* Ref = (unsigned*)BBRefs[i].second;
+    intptr_t BranchTargetDisp = 
+      (((unsigned char*)Location  - (unsigned char*)Ref) >> 2) - 1;
+    DEBUG(std::cerr << "Fixup @ " << (void*)Ref << " to " << (void*)Location
+          << " Disp " << BranchTargetDisp 
+          << " using " <<  (BranchTargetDisp & ((1 << 22)-1)) << "\n");
+    *Ref |= (BranchTargetDisp & ((1 << 21)-1));
+  }
+  BBRefs.clear();
+}

lib/Target/Alpha/AlphaJITInfo.h

@@ -47,6 +47,7 @@ namespace llvm {
     ///
     virtual void replaceMachineCodeForFunction(void *Old, void *New);
 
+    virtual void resolveBBRefs(MachineCodeEmitter &MCE);
   private:
     static const unsigned GOToffset = 4096;
 

lib/Target/Alpha/AlphaTargetMachine.cpp

@@ -138,7 +138,7 @@ void AlphaJITInfo::addPassesToJITCompile(FunctionPassManager &PM) {
 
 bool AlphaTargetMachine::addPassesToEmitMachineCode(FunctionPassManager &PM,
                                                     MachineCodeEmitter &MCE) {
-  PM.add(createAlphaCodeEmitterPass(MCE));
+  PM.add(createAlphaCodeEmitterPass(*this, MCE));
   // Delete machine code for this function
   PM.add(createMachineCodeDeleter());
   return false;

lib/Target/PowerPC/PPCCodeEmitter.cpp

@@ -32,9 +32,6 @@ namespace {
     TargetMachine &TM;
     MachineCodeEmitter &MCE;
 
-    // Tracks which instruction references which BasicBlock
-    std::vector<std::pair<MachineBasicBlock*, unsigned*> > BBRefs;
-    
     /// getMachineOpValue - evaluates the MachineOperand of a given MachineInstr
     ///
     int getMachineOpValue(MachineInstr &MI, MachineOperand &MO);
@@ -80,39 +77,20 @@ bool PPCTargetMachine::addPassesToEmitMachineCode(FunctionPassManager &PM,
   return false;
 }
 
+#ifdef __APPLE__ 
+extern "C" void sys_icache_invalidate(const void *Addr, size_t len);
+#endif
+
 bool PPCCodeEmitter::runOnMachineFunction(MachineFunction &MF) {
   assert((MF.getTarget().getRelocationModel() != Reloc::Default ||
           MF.getTarget().getRelocationModel() != Reloc::Static) &&
          "JIT relocation model must be set to static or default!");
   do {
-    BBRefs.clear();
-
     MCE.startFunction(MF);
     for (MachineFunction::iterator BB = MF.begin(), E = MF.end(); BB != E; ++BB)
       emitBasicBlock(*BB);
   } while (MCE.finishFunction(MF));
 
-  // Resolve branches to BasicBlocks for the entire function
-  for (unsigned i = 0, e = BBRefs.size(); i != e; ++i) {
-    intptr_t Location = MCE.getMachineBasicBlockAddress(BBRefs[i].first);
-    unsigned *Ref = BBRefs[i].second;
-    DEBUG(std::cerr << "Fixup @ " << (void*)Ref << " to " << (void*)Location
-                    << "\n");
-    unsigned Instr = *Ref;
-    intptr_t BranchTargetDisp = (Location - (intptr_t)Ref) >> 2;
-
-    switch (Instr >> 26) {
-    default: assert(0 && "Unknown branch user!");
-    case 18:  // This is B or BL
-      *Ref |= (BranchTargetDisp & ((1 << 24)-1)) << 2;
-      break;
-    case 16:  // This is BLT,BLE,BEQ,BGE,BGT,BNE, or other bcx instruction
-      *Ref |= (BranchTargetDisp & ((1 << 14)-1)) << 2;
-      break;
-    }
-  }
-  BBRefs.clear();
-
   return false;
 }
 
@@ -203,7 +181,7 @@ int PPCCodeEmitter::getMachineOpValue(MachineInstr &MI, MachineOperand &MO) {
                                           Reloc, MO.getSymbolName(), 0));
   } else if (MO.isMachineBasicBlock()) {
     unsigned* CurrPC = (unsigned*)(intptr_t)MCE.getCurrentPCValue();
-    BBRefs.push_back(std::make_pair(MO.getMachineBasicBlock(), CurrPC));
+    TM.getJITInfo()->addBBRef(MO.getMachineBasicBlock(), (intptr_t)CurrPC);
   } else if (MO.isConstantPoolIndex() || MO.isJumpTableIndex()) {
     if (MO.isConstantPoolIndex())
       rv = MCE.getConstantPoolEntryAddress(MO.getConstantPoolIndex());

lib/Target/PowerPC/PPCJITInfo.cpp

@@ -16,7 +16,9 @@
 #include "PPCRelocations.h"
 #include "llvm/CodeGen/MachineCodeEmitter.h"
 #include "llvm/Config/alloca.h"
+#include "llvm/Support/Debug.h"
 #include <set>
+#include <iostream>
 using namespace llvm;
 
 static TargetJITInfo::JITCompilerFn JITCompilerFunction;
@@ -243,3 +245,36 @@ void PPCJITInfo::relocate(void *Function, MachineRelocation *MR,
 void PPCJITInfo::replaceMachineCodeForFunction(void *Old, void *New) {
   EmitBranchToAt(Old, New, false);
 }
+
+void PPCJITInfo::resolveBBRefs(MachineCodeEmitter &MCE) {
+  // Resolve branches to BasicBlocks for the entire function
+  for (unsigned i = 0, e = BBRefs.size(); i != e; ++i) {
+    intptr_t Location = MCE.getMachineBasicBlockAddress(BBRefs[i].first);
+    unsigned *Ref = (unsigned *)BBRefs[i].second;
+    DEBUG(std::cerr << "Fixup @ " << (void*)Ref << " to " << (void*)Location
+                    << "\n");
+    unsigned Instr = *Ref;
+    intptr_t BranchTargetDisp = (Location - (intptr_t)Ref) >> 2;
+
+    switch (Instr >> 26) {
+    default: assert(0 && "Unknown branch user!");
+    case 18:  // This is B or BL
+      *Ref |= (BranchTargetDisp & ((1 << 24)-1)) << 2;
+      break;
+    case 16:  // This is BLT,BLE,BEQ,BGE,BGT,BNE, or other bcx instruction
+      *Ref |= (BranchTargetDisp & ((1 << 14)-1)) << 2;
+      break;
+    }
+  }
+  BBRefs.clear();
+}
+
+#ifdef __APPLE__ 
+extern "C" void sys_icache_invalidate(const void *Addr, size_t len);
+#endif
+
+void PPCJITInfo::synchronizeICache(const void *Addr, size_t Len) {
+#ifdef __APPLE__
+  sys_icache_invalidate(Addr, Len);
+#endif
+}

lib/Target/PowerPC/PPCJITInfo.h

@@ -42,6 +42,9 @@ namespace llvm {
     /// code.
     ///
     virtual void replaceMachineCodeForFunction(void *Old, void *New);
+
+    virtual void resolveBBRefs(MachineCodeEmitter &MCE);
+    virtual void synchronizeICache(const void *Addr, size_t Len);
   };
 }
 

lib/Target/X86/X86.h

@@ -44,7 +44,8 @@ FunctionPass *createX86CodePrinterPass(std::ostream &o, X86TargetMachine &tm);
 
 /// createX86CodeEmitterPass - Return a pass that emits the collected X86 code
 /// to the specified MCE object.
-FunctionPass *createX86CodeEmitterPass(MachineCodeEmitter &MCE);
+FunctionPass *createX86CodeEmitterPass(X86TargetMachine &TM,
+                                       MachineCodeEmitter &MCE);
 
 /// addX86ELFObjectWriterPass - Add passes to the FPM that output the generated
 /// code as an ELF object file.

lib/Target/X86/X86CodeEmitter.cpp

@@ -35,12 +35,14 @@ namespace {
 namespace {
   class VISIBILITY_HIDDEN Emitter : public MachineFunctionPass {
     const X86InstrInfo  *II;
+    TargetMachine &TM;
     MachineCodeEmitter  &MCE;
-    std::vector<std::pair<MachineBasicBlock *, unsigned> > BBRefs;
   public:
-    explicit Emitter(MachineCodeEmitter &mce) : II(0), MCE(mce) {}
+    explicit Emitter(TargetMachine &tm, MachineCodeEmitter &mce)
-    Emitter(MachineCodeEmitter &mce, const X86InstrInfo& ii)
+      : II(0), TM(tm), MCE(mce) {}
-        : II(&ii), MCE(mce) {}
+    Emitter(TargetMachine &tm, MachineCodeEmitter &mce,
+            const X86InstrInfo& ii)
+      : II(&ii), TM(tm), MCE(mce) {}
 
     bool runOnMachineFunction(MachineFunction &MF);
 
@@ -71,8 +73,9 @@ namespace {
 
 /// createX86CodeEmitterPass - Return a pass that emits the collected X86 code
 /// to the specified MCE object.
-FunctionPass *llvm::createX86CodeEmitterPass(MachineCodeEmitter &MCE) {
+FunctionPass *llvm::createX86CodeEmitterPass(X86TargetMachine &TM,
-  return new Emitter(MCE);
+                                             MachineCodeEmitter &MCE) {
+  return new Emitter(TM, MCE);
 }
 
 bool Emitter::runOnMachineFunction(MachineFunction &MF) {
@@ -82,8 +85,6 @@ bool Emitter::runOnMachineFunction(MachineFunction &MF) {
   II = ((X86TargetMachine&)MF.getTarget()).getInstrInfo();
 
   do {
-    BBRefs.clear();
-
     MCE.startFunction(MF);
     for (MachineFunction::iterator MBB = MF.begin(), E = MF.end(); 
          MBB != E; ++MBB) {
@@ -94,13 +95,6 @@ bool Emitter::runOnMachineFunction(MachineFunction &MF) {
     }
   } while (MCE.finishFunction(MF));
 
-  // Resolve all forward branches now.
-  for (unsigned i = 0, e = BBRefs.size(); i != e; ++i) {
-    unsigned Location = MCE.getMachineBasicBlockAddress(BBRefs[i].first);
-    unsigned Ref = BBRefs[i].second;
-    *((unsigned*)(intptr_t)Ref) = Location-Ref-4;
-  }
-  BBRefs.clear();
   return false;
 }
 
@@ -117,7 +111,7 @@ void Emitter::emitPCRelativeValue(unsigned Address) {
 void Emitter::emitPCRelativeBlockAddress(MachineBasicBlock *MBB) {
   // Remember where this reference was and where it is to so we can
   // deal with it later.
-  BBRefs.push_back(std::make_pair(MBB, MCE.getCurrentPCValue()));
+  TM.getJITInfo()->addBBRef(MBB, MCE.getCurrentPCValue());
   MCE.emitWordLE(0);
 }
 

lib/Target/X86/X86ELFWriter.cpp

@@ -35,5 +35,5 @@ void llvm::addX86ELFObjectWriterPass(PassManager &FPM,
                                      std::ostream &O, X86TargetMachine &TM) {
   X86ELFWriter *EW = new X86ELFWriter(O, TM);
   FPM.add(EW);
-  FPM.add(createX86CodeEmitterPass(EW->getMachineCodeEmitter()));
+  FPM.add(createX86CodeEmitterPass(TM, EW->getMachineCodeEmitter()));
 }

lib/Target/X86/X86JITInfo.cpp

@@ -203,3 +203,13 @@ void X86JITInfo::relocate(void *Function, MachineRelocation *MR,
     }
   }
 }
+
+void X86JITInfo::resolveBBRefs(MachineCodeEmitter &MCE) {
+  // Resolve all forward branches now.
+  for (unsigned i = 0, e = BBRefs.size(); i != e; ++i) {
+    unsigned Location = MCE.getMachineBasicBlockAddress(BBRefs[i].first);
+    intptr_t Ref = BBRefs[i].second;
+    *((unsigned*)Ref) = Location-Ref-4;
+  }
+  BBRefs.clear();
+}

lib/Target/X86/X86JITInfo.h

@@ -51,6 +51,8 @@ namespace llvm {
     /// referenced global symbols.
     virtual void relocate(void *Function, MachineRelocation *MR,
                           unsigned NumRelocs, unsigned char* GOTBase);
+
+    virtual void resolveBBRefs(MachineCodeEmitter &MCE);
   };
 }
 

lib/Target/X86/X86TargetMachine.cpp

@@ -194,7 +194,7 @@ void X86JITInfo::addPassesToJITCompile(FunctionPassManager &PM) {
 
 bool X86TargetMachine::addPassesToEmitMachineCode(FunctionPassManager &PM,
                                                   MachineCodeEmitter &MCE) {
-  PM.add(createX86CodeEmitterPass(MCE));
+  PM.add(createX86CodeEmitterPass(*this, MCE));
   // Delete machine code for this function
   PM.add(createMachineCodeDeleter());
   return false;