mirror of
https://github.com/autc04/Retro68.git
synced 2024-12-11 03:52:59 +00:00
294 lines
9.2 KiB
Go
294 lines
9.2 KiB
Go
// Copyright 2016 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// The gccgo version of mem_*.go.
|
|
|
|
package runtime
|
|
|
|
import (
|
|
"runtime/internal/sys"
|
|
"unsafe"
|
|
)
|
|
|
|
// Functions called by C code.
|
|
//go:linkname sysAlloc runtime.sysAlloc
|
|
//go:linkname sysFree runtime.sysFree
|
|
|
|
//extern mmap
|
|
func sysMmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uintptr) unsafe.Pointer
|
|
|
|
//extern munmap
|
|
func munmap(addr unsafe.Pointer, length uintptr) int32
|
|
|
|
//extern mincore
|
|
func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32
|
|
|
|
//extern madvise
|
|
func madvise(addr unsafe.Pointer, n uintptr, flags int32) int32
|
|
|
|
var mmapFD = int32(-1)
|
|
|
|
var devZero = []byte("/dev/zero\x00")
|
|
|
|
func init() {
|
|
if _MAP_ANON == 0 {
|
|
mmapFD = open(&devZero[0], 0 /* O_RDONLY */, 0)
|
|
if mmapFD < 0 {
|
|
println("open /dev/zero: errno=", errno())
|
|
exit(2)
|
|
}
|
|
}
|
|
}
|
|
|
|
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uintptr) (unsafe.Pointer, int) {
|
|
p := sysMmap(addr, n, prot, flags, fd, off)
|
|
if uintptr(p) == _MAP_FAILED {
|
|
return nil, errno()
|
|
}
|
|
return p, 0
|
|
}
|
|
|
|
// NOTE: vec must be just 1 byte long here.
|
|
// Mincore returns ENOMEM if any of the pages are unmapped,
|
|
// but we want to know that all of the pages are unmapped.
|
|
// To make these the same, we can only ask about one page
|
|
// at a time. See golang.org/issue/7476.
|
|
var addrspace_vec [1]byte
|
|
|
|
func addrspace_free(v unsafe.Pointer, n uintptr) bool {
|
|
for off := uintptr(0); off < n; off += physPageSize {
|
|
// Use a length of 1 byte, which the kernel will round
|
|
// up to one physical page regardless of the true
|
|
// physical page size.
|
|
errval := 0
|
|
if mincore(unsafe.Pointer(uintptr(v)+off), 1, &addrspace_vec[0]) < 0 {
|
|
errval = errno()
|
|
}
|
|
if errval == _ENOSYS {
|
|
// mincore is not available on this system.
|
|
// Assume the address is available.
|
|
return true
|
|
}
|
|
if errval == _EINVAL {
|
|
// Address is not a multiple of the physical
|
|
// page size. Shouldn't happen, but just ignore it.
|
|
continue
|
|
}
|
|
// ENOMEM means unmapped, which is what we want.
|
|
// Anything else we assume means the pages are mapped.
|
|
if errval != _ENOMEM {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func mmap_fixed(v unsafe.Pointer, n uintptr, prot, flags, fd int32, offset uintptr) (unsafe.Pointer, int) {
|
|
p, err := mmap(v, n, prot, flags, fd, offset)
|
|
// On some systems, mmap ignores v without
|
|
// MAP_FIXED, so retry if the address space is free.
|
|
if p != v && addrspace_free(v, n) {
|
|
if err == 0 {
|
|
munmap(p, n)
|
|
}
|
|
p, err = mmap(v, n, prot, flags|_MAP_FIXED, fd, offset)
|
|
}
|
|
return p, err
|
|
}
|
|
|
|
// Don't split the stack as this method may be invoked without a valid G, which
|
|
// prevents us from allocating more stack.
|
|
//go:nosplit
|
|
func sysAlloc(n uintptr, sysStat *uint64) unsafe.Pointer {
|
|
p, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, mmapFD, 0)
|
|
if err != 0 {
|
|
if err == _EACCES {
|
|
print("runtime: mmap: access denied\n")
|
|
exit(2)
|
|
}
|
|
if err == _EAGAIN {
|
|
print("runtime: mmap: too much locked memory (check 'ulimit -l').\n")
|
|
exit(2)
|
|
}
|
|
return nil
|
|
}
|
|
mSysStatInc(sysStat, n)
|
|
return p
|
|
}
|
|
|
|
func sysUnused(v unsafe.Pointer, n uintptr) {
|
|
// By default, Linux's "transparent huge page" support will
|
|
// merge pages into a huge page if there's even a single
|
|
// present regular page, undoing the effects of the DONTNEED
|
|
// below. On amd64, that means khugepaged can turn a single
|
|
// 4KB page to 2MB, bloating the process's RSS by as much as
|
|
// 512X. (See issue #8832 and Linux kernel bug
|
|
// https://bugzilla.kernel.org/show_bug.cgi?id=93111)
|
|
//
|
|
// To work around this, we explicitly disable transparent huge
|
|
// pages when we release pages of the heap. However, we have
|
|
// to do this carefully because changing this flag tends to
|
|
// split the VMA (memory mapping) containing v in to three
|
|
// VMAs in order to track the different values of the
|
|
// MADV_NOHUGEPAGE flag in the different regions. There's a
|
|
// default limit of 65530 VMAs per address space (sysctl
|
|
// vm.max_map_count), so we must be careful not to create too
|
|
// many VMAs (see issue #12233).
|
|
//
|
|
// Since huge pages are huge, there's little use in adjusting
|
|
// the MADV_NOHUGEPAGE flag on a fine granularity, so we avoid
|
|
// exploding the number of VMAs by only adjusting the
|
|
// MADV_NOHUGEPAGE flag on a large granularity. This still
|
|
// gets most of the benefit of huge pages while keeping the
|
|
// number of VMAs under control. With hugePageSize = 2MB, even
|
|
// a pessimal heap can reach 128GB before running out of VMAs.
|
|
if sys.HugePageSize != 0 && _MADV_NOHUGEPAGE != 0 {
|
|
var s uintptr = sys.HugePageSize // division by constant 0 is a compile-time error :(
|
|
|
|
// If it's a large allocation, we want to leave huge
|
|
// pages enabled. Hence, we only adjust the huge page
|
|
// flag on the huge pages containing v and v+n-1, and
|
|
// only if those aren't aligned.
|
|
var head, tail uintptr
|
|
if uintptr(v)%s != 0 {
|
|
// Compute huge page containing v.
|
|
head = uintptr(v) &^ (s - 1)
|
|
}
|
|
if (uintptr(v)+n)%s != 0 {
|
|
// Compute huge page containing v+n-1.
|
|
tail = (uintptr(v) + n - 1) &^ (s - 1)
|
|
}
|
|
|
|
// Note that madvise will return EINVAL if the flag is
|
|
// already set, which is quite likely. We ignore
|
|
// errors.
|
|
if head != 0 && head+sys.HugePageSize == tail {
|
|
// head and tail are different but adjacent,
|
|
// so do this in one call.
|
|
madvise(unsafe.Pointer(head), 2*sys.HugePageSize, _MADV_NOHUGEPAGE)
|
|
} else {
|
|
// Advise the huge pages containing v and v+n-1.
|
|
if head != 0 {
|
|
madvise(unsafe.Pointer(head), sys.HugePageSize, _MADV_NOHUGEPAGE)
|
|
}
|
|
if tail != 0 && tail != head {
|
|
madvise(unsafe.Pointer(tail), sys.HugePageSize, _MADV_NOHUGEPAGE)
|
|
}
|
|
}
|
|
}
|
|
|
|
if uintptr(v)&(physPageSize-1) != 0 || n&(physPageSize-1) != 0 {
|
|
// madvise will round this to any physical page
|
|
// *covered* by this range, so an unaligned madvise
|
|
// will release more memory than intended.
|
|
throw("unaligned sysUnused")
|
|
}
|
|
|
|
if _MADV_DONTNEED != 0 {
|
|
madvise(v, n, _MADV_DONTNEED)
|
|
} else if _MADV_FREE != 0 {
|
|
madvise(v, n, _MADV_FREE)
|
|
}
|
|
}
|
|
|
|
func sysUsed(v unsafe.Pointer, n uintptr) {
|
|
if sys.HugePageSize != 0 && _MADV_HUGEPAGE != 0 {
|
|
// Partially undo the NOHUGEPAGE marks from sysUnused
|
|
// for whole huge pages between v and v+n. This may
|
|
// leave huge pages off at the end points v and v+n
|
|
// even though allocations may cover these entire huge
|
|
// pages. We could detect this and undo NOHUGEPAGE on
|
|
// the end points as well, but it's probably not worth
|
|
// the cost because when neighboring allocations are
|
|
// freed sysUnused will just set NOHUGEPAGE again.
|
|
var s uintptr = sys.HugePageSize
|
|
|
|
// Round v up to a huge page boundary.
|
|
beg := (uintptr(v) + (s - 1)) &^ (s - 1)
|
|
// Round v+n down to a huge page boundary.
|
|
end := (uintptr(v) + n) &^ (s - 1)
|
|
|
|
if beg < end {
|
|
madvise(unsafe.Pointer(beg), end-beg, _MADV_HUGEPAGE)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Don't split the stack as this function may be invoked without a valid G,
|
|
// which prevents us from allocating more stack.
|
|
//go:nosplit
|
|
func sysFree(v unsafe.Pointer, n uintptr, sysStat *uint64) {
|
|
mSysStatDec(sysStat, n)
|
|
munmap(v, n)
|
|
}
|
|
|
|
func sysFault(v unsafe.Pointer, n uintptr) {
|
|
mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, mmapFD, 0)
|
|
}
|
|
|
|
func sysReserve(v unsafe.Pointer, n uintptr, reserved *bool) unsafe.Pointer {
|
|
// On 64-bit, people with ulimit -v set complain if we reserve too
|
|
// much address space. Instead, assume that the reservation is okay
|
|
// if we can reserve at least 64K and check the assumption in SysMap.
|
|
// Only user-mode Linux (UML) rejects these requests.
|
|
if sys.PtrSize == 8 && uint64(n) > 1<<32 {
|
|
p, err := mmap_fixed(v, 64<<10, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, mmapFD, 0)
|
|
if p != v || err != 0 {
|
|
if err == 0 {
|
|
munmap(p, 64<<10)
|
|
}
|
|
return nil
|
|
}
|
|
munmap(p, 64<<10)
|
|
*reserved = false
|
|
return v
|
|
}
|
|
|
|
p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, mmapFD, 0)
|
|
if err != 0 {
|
|
return nil
|
|
}
|
|
*reserved = true
|
|
return p
|
|
}
|
|
|
|
func sysMap(v unsafe.Pointer, n uintptr, reserved bool, sysStat *uint64) {
|
|
mSysStatInc(sysStat, n)
|
|
|
|
// On 64-bit, we don't actually have v reserved, so tread carefully.
|
|
if !reserved {
|
|
flags := int32(_MAP_ANON | _MAP_PRIVATE)
|
|
if GOOS == "dragonfly" {
|
|
// TODO(jsing): For some reason DragonFly seems to return
|
|
// memory at a different address than we requested, even when
|
|
// there should be no reason for it to do so. This can be
|
|
// avoided by using MAP_FIXED, but I'm not sure we should need
|
|
// to do this - we do not on other platforms.
|
|
flags |= _MAP_FIXED
|
|
}
|
|
p, err := mmap_fixed(v, n, _PROT_READ|_PROT_WRITE, flags, mmapFD, 0)
|
|
if err == _ENOMEM {
|
|
throw("runtime: out of memory")
|
|
}
|
|
if p != v || err != 0 {
|
|
print("runtime: address space conflict: map(", v, ") = ", p, " (err ", err, ")\n")
|
|
throw("runtime: address space conflict")
|
|
}
|
|
return
|
|
}
|
|
|
|
if GOOS == "aix" {
|
|
// AIX does not allow mapping a range that is already mapped.
|
|
// So always unmap first even if it is already unmapped.
|
|
munmap(v, n)
|
|
}
|
|
p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, mmapFD, 0)
|
|
if err == _ENOMEM {
|
|
throw("runtime: out of memory")
|
|
}
|
|
if p != v || err != 0 {
|
|
throw("runtime: cannot map pages in arena address space")
|
|
}
|
|
}
|