diff --git a/include/llvm/System/Memory.h b/include/llvm/System/Memory.h index 305ec68cbb3..3a9d89f8030 100644 --- a/include/llvm/System/Memory.h +++ b/include/llvm/System/Memory.h @@ -43,9 +43,11 @@ namespace sys { /// This method allocates a block of Read/Write/Execute memory that is /// suitable for executing dynamically generated code (e.g. JIT). An /// attempt to allocate \p NumBytes bytes of virtual memory is made. + /// \p NearBlock may point to an existing allocation in which case + /// an attempt is made to allocate more memory near the existing block. /// @throws std::string if an error occurred. /// @brief Allocate Read/Write/Execute memory. - static MemoryBlock AllocateRWX(unsigned NumBytes); + static MemoryBlock AllocateRWX(unsigned NumBytes, const MemoryBlock* NearBlock); /// This method releases a block of Read/Write/Execute memory that was /// allocated with the AllocateRWX method. It should not be used to diff --git a/lib/ExecutionEngine/JIT/JITEmitter.cpp b/lib/ExecutionEngine/JIT/JITEmitter.cpp index 0d47a97e2fa..d82e54509f4 100644 --- a/lib/ExecutionEngine/JIT/JITEmitter.cpp +++ b/lib/ExecutionEngine/JIT/JITEmitter.cpp @@ -26,6 +26,8 @@ #include "llvm/Support/Debug.h" #include "llvm/ADT/Statistic.h" #include "llvm/System/Memory.h" +#include +#include using namespace llvm; namespace { @@ -47,12 +49,15 @@ namespace { /// are emitting is. This never bothers to release the memory, because when /// we are ready to destroy the JIT, the program exits. class JITMemoryManager { - sys::MemoryBlock MemBlock; // Virtual memory block allocated RWX - unsigned char *MemBase; // Base of block of memory, start of stub mem + std::list Blocks; // List of blocks allocated by the JIT unsigned char *FunctionBase; // Start of the function body area - unsigned char *ConstantPool; // Memory allocated for constant pools - unsigned char *CurStubPtr, *CurFunctionPtr, *CurConstantPtr; + unsigned char *GlobalBase; // Start of the Global area + unsigned char *ConstantBase; // Memory allocated for constant pools + unsigned char *CurStubPtr, *CurFunctionPtr, *CurConstantPtr, *CurGlobalPtr; unsigned char *GOTBase; //Target Specific reserved memory + + // centralize memory block allocation + sys::MemoryBlock getNewMemoryBlock(unsigned size); public: JITMemoryManager(bool useGOT); ~JITMemoryManager(); @@ -71,32 +76,45 @@ namespace { } JITMemoryManager::JITMemoryManager(bool useGOT) { - // Allocate a 16M block of memory... - MemBlock = sys::Memory::AllocateRWX((16 << 20)); - MemBase = reinterpret_cast(MemBlock.base()); - ConstantPool = MemBase; - GOTBase = ConstantPool + 512*1024; //512 for constants - //8k number of entries in the GOT - FunctionBase = GOTBase + 8192 * sizeof(void*) + 512*1024; // Use 512k for stubs + // Allocate a 16M block of memory for functions + sys::MemoryBlock FunBlock = getNewMemoryBlock(16 << 20); + // Allocate a 1M block of memory for Constants + sys::MemoryBlock ConstBlock = getNewMemoryBlock(1 << 20); + // Allocate a 1M Block of memory for Globals + sys::MemoryBlock GVBlock = getNewMemoryBlock(1 << 20); - //make it easier to tell if we are managing the GOT - if (!useGOT) - GOTBase = NULL; + Blocks.push_front(FunBlock); + Blocks.push_front(ConstBlock); + Blocks.push_front(GVBlock); - // Allocate stubs backwards from the function base, allocate functions forward - // from the function base. - CurStubPtr = CurFunctionPtr = FunctionBase; + FunctionBase = reinterpret_cast(FunBlock.base()); + ConstantBase = reinterpret_cast(ConstBlock.base()); + GlobalBase = reinterpret_cast(GVBlock.base()); - CurConstantPtr = ConstantPool + 512*1024; + //Allocate the GOT just like a global array + GOTBase = NULL; + if (useGOT) + GOTBase = allocateGlobal(sizeof(void*) * 8192, 8); + + // Allocate stubs backwards from the base, allocate functions forward + // from the base. + CurStubPtr = CurFunctionPtr = FunctionBase + 512*1024;// Use 512k for stubs + + CurConstantPtr = ConstantBase + ConstBlock.size(); + CurGlobalPtr = GlobalBase + GVBlock.size(); } JITMemoryManager::~JITMemoryManager() { - sys::Memory::ReleaseRWX(MemBlock); + for (std::list::iterator ib = Blocks.begin(), ie = Blocks.end(); + ib != ie; ++ib) + sys::Memory::ReleaseRWX(*ib); + Blocks.clear(); } unsigned char *JITMemoryManager::allocateStub(unsigned StubSize) { CurStubPtr -= StubSize; - if (CurStubPtr < MemBase) { + if (CurStubPtr < FunctionBase) { + //FIXME: allocate a new block std::cerr << "JIT ran out of memory for function stubs!\n"; abort(); } @@ -110,26 +128,31 @@ unsigned char *JITMemoryManager::allocateConstant(unsigned ConstantSize, CurConstantPtr = (unsigned char *)((intptr_t)CurConstantPtr & ~((intptr_t)Alignment - 1)); - if (CurConstantPtr < ConstantPool) { - std::cerr << "JIT ran out of memory for constant pools!\n"; - abort(); + if (CurConstantPtr < ConstantBase) { + //Either allocate another MB or 2xConstantSize + sys::MemoryBlock ConstBlock = getNewMemoryBlock(2 * ConstantSize); + ConstantBase = reinterpret_cast(ConstBlock.base()); + CurConstantPtr = ConstantBase + ConstBlock.size(); + return allocateConstant(ConstantSize, Alignment); } return CurConstantPtr; } unsigned char *JITMemoryManager::allocateGlobal(unsigned Size, unsigned Alignment) { - // For now, intersperse them with Constants - // Reserve space and align pointer. - CurConstantPtr -= Size; - CurConstantPtr = - (unsigned char *)((intptr_t)CurConstantPtr & ~((intptr_t)Alignment - 1)); + // Reserve space and align pointer. + CurGlobalPtr -= Size; + CurGlobalPtr = + (unsigned char *)((intptr_t)CurGlobalPtr & ~((intptr_t)Alignment - 1)); - if (CurConstantPtr < ConstantPool) { - std::cerr << "JIT ran out of memory for Globals!\n"; - abort(); + if (CurGlobalPtr < GlobalBase) { + //Either allocate another MB or 2xSize + sys::MemoryBlock GVBlock = getNewMemoryBlock(2 * Size); + GlobalBase = reinterpret_cast(GVBlock.base()); + CurGlobalPtr = GlobalBase + GVBlock.size(); + return allocateGlobal(Size, Alignment); } - return CurConstantPtr; + return CurGlobalPtr; } unsigned char *JITMemoryManager::startFunctionBody() { @@ -151,6 +174,23 @@ bool JITMemoryManager::isManagingGOT() const { return GOTBase != NULL; } +sys::MemoryBlock JITMemoryManager::getNewMemoryBlock(unsigned size) { + const sys::MemoryBlock* BOld = 0; + if (Blocks.size()) + BOld = &Blocks.front(); + //never allocate less than 1 MB + sys::MemoryBlock B; + try { + B = sys::Memory::AllocateRWX(std::max(((unsigned)1 << 20), size), BOld); + } catch (std::string& err) { + std::cerr << "Allocation failed when allocating new memory in the JIT\n"; + std::cerr << err << "\n"; + abort(); + } + Blocks.push_front(B); + return B; +} + //===----------------------------------------------------------------------===// // JIT lazy compilation code. // diff --git a/lib/System/Unix/Memory.inc b/lib/System/Unix/Memory.inc index a89fd22d2e9..4475960e117 100644 --- a/lib/System/Unix/Memory.inc +++ b/lib/System/Unix/Memory.inc @@ -25,7 +25,7 @@ namespace llvm { /// to emit code to the memory then jump to it. Getting this type of memory /// is very OS specific. /// -MemoryBlock Memory::AllocateRWX(unsigned NumBytes) { +MemoryBlock Memory::AllocateRWX(unsigned NumBytes, const MemoryBlock* NearBlock) { if (NumBytes == 0) return MemoryBlock(); long pageSize = Process::GetPageSize(); @@ -47,10 +47,16 @@ MemoryBlock Memory::AllocateRWX(unsigned NumBytes) { MAP_ANON #endif ; - void *pa = ::mmap(0, pageSize*NumPages, PROT_READ|PROT_WRITE|PROT_EXEC, + + void* start = NearBlock ? (unsigned char*) NearBlock->base() + NearBlock->size() : 0; + + void *pa = ::mmap(start, pageSize*NumPages, PROT_READ|PROT_WRITE|PROT_EXEC, flags, fd, 0); if (pa == MAP_FAILED) { - ThrowErrno("Can't allocate RWX Memory"); + if (NearBlock) //Try again without a near hint + return AllocateRWX(NumBytes, 0); + else + ThrowErrno("Can't allocate RWX Memory"); } MemoryBlock result; result.Address = pa; diff --git a/lib/System/Win32/Memory.inc b/lib/System/Win32/Memory.inc index 9ebef6a71a3..7e93dee24eb 100644 --- a/lib/System/Win32/Memory.inc +++ b/lib/System/Win32/Memory.inc @@ -23,12 +23,14 @@ using namespace sys; //=== and must not be UNIX code //===----------------------------------------------------------------------===// -MemoryBlock Memory::AllocateRWX(unsigned NumBytes) { +MemoryBlock Memory::AllocateRWX(unsigned NumBytes, const MemoryBlock* NearBlock) { if (NumBytes == 0) return MemoryBlock(); static const long pageSize = Process::GetPageSize(); unsigned NumPages = (NumBytes+pageSize-1)/pageSize; + //FIXME: support NearBlock if ever needed on Win64. + void *pa = VirtualAlloc(NULL, NumPages*pageSize, MEM_COMMIT, PAGE_EXECUTE_READWRITE); if (pa == NULL) {