mirror of
https://github.com/autc04/Retro68.git
synced 2024-09-08 05:54:40 +00:00
1069 lines
31 KiB
Go
1069 lines
31 KiB
Go
// Copyright 2013 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package runtime
|
|
|
|
import (
|
|
"runtime/internal/atomic"
|
|
"runtime/internal/sys"
|
|
"unsafe"
|
|
)
|
|
|
|
/*
|
|
Stack layout parameters.
|
|
Included both by runtime (compiled via 6c) and linkers (compiled via gcc).
|
|
|
|
The per-goroutine g->stackguard is set to point StackGuard bytes
|
|
above the bottom of the stack. Each function compares its stack
|
|
pointer against g->stackguard to check for overflow. To cut one
|
|
instruction from the check sequence for functions with tiny frames,
|
|
the stack is allowed to protrude StackSmall bytes below the stack
|
|
guard. Functions with large frames don't bother with the check and
|
|
always call morestack. The sequences are (for amd64, others are
|
|
similar):
|
|
|
|
guard = g->stackguard
|
|
frame = function's stack frame size
|
|
argsize = size of function arguments (call + return)
|
|
|
|
stack frame size <= StackSmall:
|
|
CMPQ guard, SP
|
|
JHI 3(PC)
|
|
MOVQ m->morearg, $(argsize << 32)
|
|
CALL morestack(SB)
|
|
|
|
stack frame size > StackSmall but < StackBig
|
|
LEAQ (frame-StackSmall)(SP), R0
|
|
CMPQ guard, R0
|
|
JHI 3(PC)
|
|
MOVQ m->morearg, $(argsize << 32)
|
|
CALL morestack(SB)
|
|
|
|
stack frame size >= StackBig:
|
|
MOVQ m->morearg, $((argsize << 32) | frame)
|
|
CALL morestack(SB)
|
|
|
|
The bottom StackGuard - StackSmall bytes are important: there has
|
|
to be enough room to execute functions that refuse to check for
|
|
stack overflow, either because they need to be adjacent to the
|
|
actual caller's frame (deferproc) or because they handle the imminent
|
|
stack overflow (morestack).
|
|
|
|
For example, deferproc might call malloc, which does one of the
|
|
above checks (without allocating a full frame), which might trigger
|
|
a call to morestack. This sequence needs to fit in the bottom
|
|
section of the stack. On amd64, morestack's frame is 40 bytes, and
|
|
deferproc's frame is 56 bytes. That fits well within the
|
|
StackGuard - StackSmall bytes at the bottom.
|
|
The linkers explore all possible call traces involving non-splitting
|
|
functions to make sure that this limit cannot be violated.
|
|
*/
|
|
|
|
const (
|
|
// StackSystem is a number of additional bytes to add
|
|
// to each stack below the usual guard area for OS-specific
|
|
// purposes like signal handling. Used on Windows, Plan 9,
|
|
// and Darwin/ARM because they do not use a separate stack.
|
|
_StackSystem = sys.GoosWindows*512*sys.PtrSize + sys.GoosPlan9*512 + sys.GoosDarwin*sys.GoarchArm*1024
|
|
|
|
// The minimum size of stack used by Go code
|
|
_StackMin = 2048
|
|
|
|
// The minimum stack size to allocate.
|
|
// The hackery here rounds FixedStack0 up to a power of 2.
|
|
_FixedStack0 = _StackMin + _StackSystem
|
|
_FixedStack1 = _FixedStack0 - 1
|
|
_FixedStack2 = _FixedStack1 | (_FixedStack1 >> 1)
|
|
_FixedStack3 = _FixedStack2 | (_FixedStack2 >> 2)
|
|
_FixedStack4 = _FixedStack3 | (_FixedStack3 >> 4)
|
|
_FixedStack5 = _FixedStack4 | (_FixedStack4 >> 8)
|
|
_FixedStack6 = _FixedStack5 | (_FixedStack5 >> 16)
|
|
_FixedStack = _FixedStack6 + 1
|
|
|
|
// Functions that need frames bigger than this use an extra
|
|
// instruction to do the stack split check, to avoid overflow
|
|
// in case SP - framesize wraps below zero.
|
|
// This value can be no bigger than the size of the unmapped
|
|
// space at zero.
|
|
_StackBig = 4096
|
|
|
|
// The stack guard is a pointer this many bytes above the
|
|
// bottom of the stack.
|
|
_StackGuard = 720*sys.StackGuardMultiplier + _StackSystem
|
|
|
|
// After a stack split check the SP is allowed to be this
|
|
// many bytes below the stack guard. This saves an instruction
|
|
// in the checking sequence for tiny frames.
|
|
_StackSmall = 128
|
|
|
|
// The maximum number of bytes that a chain of NOSPLIT
|
|
// functions can use.
|
|
_StackLimit = _StackGuard - _StackSystem - _StackSmall
|
|
)
|
|
|
|
// Goroutine preemption request.
|
|
// Stored into g->stackguard0 to cause split stack check failure.
|
|
// Must be greater than any real sp.
|
|
// 0xfffffade in hex.
|
|
const (
|
|
_StackPreempt = uintptrMask & -1314
|
|
_StackFork = uintptrMask & -1234
|
|
)
|
|
|
|
const (
|
|
// stackDebug == 0: no logging
|
|
// == 1: logging of per-stack operations
|
|
// == 2: logging of per-frame operations
|
|
// == 3: logging of per-word updates
|
|
// == 4: logging of per-word reads
|
|
stackDebug = 0
|
|
stackFromSystem = 0 // allocate stacks from system memory instead of the heap
|
|
stackFaultOnFree = 0 // old stacks are mapped noaccess to detect use after free
|
|
stackPoisonCopy = 0 // fill stack that should not be accessed with garbage, to detect bad dereferences during copy
|
|
|
|
stackCache = 1
|
|
)
|
|
|
|
const (
|
|
uintptrMask = 1<<(8*sys.PtrSize) - 1
|
|
poisonStack = uintptrMask & 0x6868686868686868
|
|
|
|
// Goroutine preemption request.
|
|
// Stored into g->stackguard0 to cause split stack check failure.
|
|
// Must be greater than any real sp.
|
|
// 0xfffffade in hex.
|
|
stackPreempt = uintptrMask & -1314
|
|
|
|
// Thread is forking.
|
|
// Stored into g->stackguard0 to cause split stack check failure.
|
|
// Must be greater than any real sp.
|
|
stackFork = uintptrMask & -1234
|
|
)
|
|
|
|
// Global pool of spans that have free stacks.
|
|
// Stacks are assigned an order according to size.
|
|
// order = log_2(size/FixedStack)
|
|
// There is a free list for each order.
|
|
// TODO: one lock per order?
|
|
var stackpool [_NumStackOrders]mSpanList
|
|
var stackpoolmu mutex
|
|
|
|
// Global pool of large stack spans.
|
|
var stackLarge struct {
|
|
lock mutex
|
|
free [_MHeapMap_Bits]mSpanList // free lists by log_2(s.npages)
|
|
}
|
|
|
|
// Cached value of haveexperiment("framepointer")
|
|
var framepointer_enabled bool
|
|
|
|
func stackinit() {
|
|
if _StackCacheSize&_PageMask != 0 {
|
|
throw("cache size must be a multiple of page size")
|
|
}
|
|
for i := range stackpool {
|
|
stackpool[i].init()
|
|
}
|
|
for i := range stackLarge.free {
|
|
stackLarge.free[i].init()
|
|
}
|
|
}
|
|
|
|
// stacklog2 returns ⌊log_2(n)⌋.
|
|
func stacklog2(n uintptr) int {
|
|
log2 := 0
|
|
for n > 1 {
|
|
n >>= 1
|
|
log2++
|
|
}
|
|
return log2
|
|
}
|
|
|
|
// Allocates a stack from the free pool. Must be called with
|
|
// stackpoolmu held.
|
|
func stackpoolalloc(order uint8) gclinkptr {
|
|
list := &stackpool[order]
|
|
s := list.first
|
|
if s == nil {
|
|
// no free stacks. Allocate another span worth.
|
|
s = mheap_.allocStack(_StackCacheSize >> _PageShift)
|
|
if s == nil {
|
|
throw("out of memory")
|
|
}
|
|
if s.ref != 0 {
|
|
throw("bad ref")
|
|
}
|
|
if s.freelist.ptr() != nil {
|
|
throw("bad freelist")
|
|
}
|
|
for i := uintptr(0); i < _StackCacheSize; i += _FixedStack << order {
|
|
x := gclinkptr(uintptr(s.start)<<_PageShift + i)
|
|
x.ptr().next = s.freelist
|
|
s.freelist = x
|
|
}
|
|
list.insert(s)
|
|
}
|
|
x := s.freelist
|
|
if x.ptr() == nil {
|
|
throw("span has no free stacks")
|
|
}
|
|
s.freelist = x.ptr().next
|
|
s.ref++
|
|
if s.freelist.ptr() == nil {
|
|
// all stacks in s are allocated.
|
|
list.remove(s)
|
|
}
|
|
return x
|
|
}
|
|
|
|
// Adds stack x to the free pool. Must be called with stackpoolmu held.
|
|
func stackpoolfree(x gclinkptr, order uint8) {
|
|
s := mheap_.lookup(unsafe.Pointer(x))
|
|
if s.state != _MSpanStack {
|
|
throw("freeing stack not in a stack span")
|
|
}
|
|
if s.freelist.ptr() == nil {
|
|
// s will now have a free stack
|
|
stackpool[order].insert(s)
|
|
}
|
|
x.ptr().next = s.freelist
|
|
s.freelist = x
|
|
s.ref--
|
|
if gcphase == _GCoff && s.ref == 0 {
|
|
// Span is completely free. Return it to the heap
|
|
// immediately if we're sweeping.
|
|
//
|
|
// If GC is active, we delay the free until the end of
|
|
// GC to avoid the following type of situation:
|
|
//
|
|
// 1) GC starts, scans a SudoG but does not yet mark the SudoG.elem pointer
|
|
// 2) The stack that pointer points to is copied
|
|
// 3) The old stack is freed
|
|
// 4) The containing span is marked free
|
|
// 5) GC attempts to mark the SudoG.elem pointer. The
|
|
// marking fails because the pointer looks like a
|
|
// pointer into a free span.
|
|
//
|
|
// By not freeing, we prevent step #4 until GC is done.
|
|
stackpool[order].remove(s)
|
|
s.freelist = 0
|
|
mheap_.freeStack(s)
|
|
}
|
|
}
|
|
|
|
// stackcacherefill/stackcacherelease implement a global pool of stack segments.
|
|
// The pool is required to prevent unlimited growth of per-thread caches.
|
|
func stackcacherefill(c *mcache, order uint8) {
|
|
if stackDebug >= 1 {
|
|
print("stackcacherefill order=", order, "\n")
|
|
}
|
|
|
|
// Grab some stacks from the global cache.
|
|
// Grab half of the allowed capacity (to prevent thrashing).
|
|
var list gclinkptr
|
|
var size uintptr
|
|
lock(&stackpoolmu)
|
|
for size < _StackCacheSize/2 {
|
|
x := stackpoolalloc(order)
|
|
x.ptr().next = list
|
|
list = x
|
|
size += _FixedStack << order
|
|
}
|
|
unlock(&stackpoolmu)
|
|
c.stackcache[order].list = list
|
|
c.stackcache[order].size = size
|
|
}
|
|
|
|
func stackcacherelease(c *mcache, order uint8) {
|
|
if stackDebug >= 1 {
|
|
print("stackcacherelease order=", order, "\n")
|
|
}
|
|
x := c.stackcache[order].list
|
|
size := c.stackcache[order].size
|
|
lock(&stackpoolmu)
|
|
for size > _StackCacheSize/2 {
|
|
y := x.ptr().next
|
|
stackpoolfree(x, order)
|
|
x = y
|
|
size -= _FixedStack << order
|
|
}
|
|
unlock(&stackpoolmu)
|
|
c.stackcache[order].list = x
|
|
c.stackcache[order].size = size
|
|
}
|
|
|
|
func stackcache_clear(c *mcache) {
|
|
if stackDebug >= 1 {
|
|
print("stackcache clear\n")
|
|
}
|
|
lock(&stackpoolmu)
|
|
for order := uint8(0); order < _NumStackOrders; order++ {
|
|
x := c.stackcache[order].list
|
|
for x.ptr() != nil {
|
|
y := x.ptr().next
|
|
stackpoolfree(x, order)
|
|
x = y
|
|
}
|
|
c.stackcache[order].list = 0
|
|
c.stackcache[order].size = 0
|
|
}
|
|
unlock(&stackpoolmu)
|
|
}
|
|
|
|
func stackalloc(n uint32) (stack, []stkbar) {
|
|
// Stackalloc must be called on scheduler stack, so that we
|
|
// never try to grow the stack during the code that stackalloc runs.
|
|
// Doing so would cause a deadlock (issue 1547).
|
|
thisg := getg()
|
|
if thisg != thisg.m.g0 {
|
|
throw("stackalloc not on scheduler stack")
|
|
}
|
|
if n&(n-1) != 0 {
|
|
throw("stack size not a power of 2")
|
|
}
|
|
if stackDebug >= 1 {
|
|
print("stackalloc ", n, "\n")
|
|
}
|
|
|
|
// Compute the size of stack barrier array.
|
|
maxstkbar := gcMaxStackBarriers(int(n))
|
|
nstkbar := unsafe.Sizeof(stkbar{}) * uintptr(maxstkbar)
|
|
|
|
if debug.efence != 0 || stackFromSystem != 0 {
|
|
v := sysAlloc(round(uintptr(n), _PageSize), &memstats.stacks_sys)
|
|
if v == nil {
|
|
throw("out of memory (stackalloc)")
|
|
}
|
|
top := uintptr(n) - nstkbar
|
|
stkbarSlice := slice{add(v, top), 0, maxstkbar}
|
|
return stack{uintptr(v), uintptr(v) + top}, *(*[]stkbar)(unsafe.Pointer(&stkbarSlice))
|
|
}
|
|
|
|
// Small stacks are allocated with a fixed-size free-list allocator.
|
|
// If we need a stack of a bigger size, we fall back on allocating
|
|
// a dedicated span.
|
|
var v unsafe.Pointer
|
|
if stackCache != 0 && n < _FixedStack<<_NumStackOrders && n < _StackCacheSize {
|
|
order := uint8(0)
|
|
n2 := n
|
|
for n2 > _FixedStack {
|
|
order++
|
|
n2 >>= 1
|
|
}
|
|
var x gclinkptr
|
|
c := thisg.m.mcache
|
|
if c == nil || thisg.m.preemptoff != "" || thisg.m.helpgc != 0 {
|
|
// c == nil can happen in the guts of exitsyscall or
|
|
// procresize. Just get a stack from the global pool.
|
|
// Also don't touch stackcache during gc
|
|
// as it's flushed concurrently.
|
|
lock(&stackpoolmu)
|
|
x = stackpoolalloc(order)
|
|
unlock(&stackpoolmu)
|
|
} else {
|
|
x = c.stackcache[order].list
|
|
if x.ptr() == nil {
|
|
stackcacherefill(c, order)
|
|
x = c.stackcache[order].list
|
|
}
|
|
c.stackcache[order].list = x.ptr().next
|
|
c.stackcache[order].size -= uintptr(n)
|
|
}
|
|
v = unsafe.Pointer(x)
|
|
} else {
|
|
var s *mspan
|
|
npage := uintptr(n) >> _PageShift
|
|
log2npage := stacklog2(npage)
|
|
|
|
// Try to get a stack from the large stack cache.
|
|
lock(&stackLarge.lock)
|
|
if !stackLarge.free[log2npage].isEmpty() {
|
|
s = stackLarge.free[log2npage].first
|
|
stackLarge.free[log2npage].remove(s)
|
|
}
|
|
unlock(&stackLarge.lock)
|
|
|
|
if s == nil {
|
|
// Allocate a new stack from the heap.
|
|
s = mheap_.allocStack(npage)
|
|
if s == nil {
|
|
throw("out of memory")
|
|
}
|
|
}
|
|
v = unsafe.Pointer(s.start << _PageShift)
|
|
}
|
|
|
|
if raceenabled {
|
|
racemalloc(v, uintptr(n))
|
|
}
|
|
if msanenabled {
|
|
msanmalloc(v, uintptr(n))
|
|
}
|
|
if stackDebug >= 1 {
|
|
print(" allocated ", v, "\n")
|
|
}
|
|
top := uintptr(n) - nstkbar
|
|
stkbarSlice := slice{add(v, top), 0, maxstkbar}
|
|
return stack{uintptr(v), uintptr(v) + top}, *(*[]stkbar)(unsafe.Pointer(&stkbarSlice))
|
|
}
|
|
|
|
func stackfree(stk stack, n uintptr) {
|
|
gp := getg()
|
|
v := unsafe.Pointer(stk.lo)
|
|
if n&(n-1) != 0 {
|
|
throw("stack not a power of 2")
|
|
}
|
|
if stk.lo+n < stk.hi {
|
|
throw("bad stack size")
|
|
}
|
|
if stackDebug >= 1 {
|
|
println("stackfree", v, n)
|
|
memclr(v, n) // for testing, clobber stack data
|
|
}
|
|
if debug.efence != 0 || stackFromSystem != 0 {
|
|
if debug.efence != 0 || stackFaultOnFree != 0 {
|
|
sysFault(v, n)
|
|
} else {
|
|
sysFree(v, n, &memstats.stacks_sys)
|
|
}
|
|
return
|
|
}
|
|
if msanenabled {
|
|
msanfree(v, n)
|
|
}
|
|
if stackCache != 0 && n < _FixedStack<<_NumStackOrders && n < _StackCacheSize {
|
|
order := uint8(0)
|
|
n2 := n
|
|
for n2 > _FixedStack {
|
|
order++
|
|
n2 >>= 1
|
|
}
|
|
x := gclinkptr(v)
|
|
c := gp.m.mcache
|
|
if c == nil || gp.m.preemptoff != "" || gp.m.helpgc != 0 {
|
|
lock(&stackpoolmu)
|
|
stackpoolfree(x, order)
|
|
unlock(&stackpoolmu)
|
|
} else {
|
|
if c.stackcache[order].size >= _StackCacheSize {
|
|
stackcacherelease(c, order)
|
|
}
|
|
x.ptr().next = c.stackcache[order].list
|
|
c.stackcache[order].list = x
|
|
c.stackcache[order].size += n
|
|
}
|
|
} else {
|
|
s := mheap_.lookup(v)
|
|
if s.state != _MSpanStack {
|
|
println(hex(s.start<<_PageShift), v)
|
|
throw("bad span state")
|
|
}
|
|
if gcphase == _GCoff {
|
|
// Free the stack immediately if we're
|
|
// sweeping.
|
|
mheap_.freeStack(s)
|
|
} else {
|
|
// If the GC is running, we can't return a
|
|
// stack span to the heap because it could be
|
|
// reused as a heap span, and this state
|
|
// change would race with GC. Add it to the
|
|
// large stack cache instead.
|
|
log2npage := stacklog2(s.npages)
|
|
lock(&stackLarge.lock)
|
|
stackLarge.free[log2npage].insert(s)
|
|
unlock(&stackLarge.lock)
|
|
}
|
|
}
|
|
}
|
|
|
|
var maxstacksize uintptr = 1 << 20 // enough until runtime.main sets it for real
|
|
|
|
var ptrnames = []string{
|
|
0: "scalar",
|
|
1: "ptr",
|
|
}
|
|
|
|
// Stack frame layout
|
|
//
|
|
// (x86)
|
|
// +------------------+
|
|
// | args from caller |
|
|
// +------------------+ <- frame->argp
|
|
// | return address |
|
|
// +------------------+
|
|
// | caller's BP (*) | (*) if framepointer_enabled && varp < sp
|
|
// +------------------+ <- frame->varp
|
|
// | locals |
|
|
// +------------------+
|
|
// | args to callee |
|
|
// +------------------+ <- frame->sp
|
|
//
|
|
// (arm)
|
|
// +------------------+
|
|
// | args from caller |
|
|
// +------------------+ <- frame->argp
|
|
// | caller's retaddr |
|
|
// +------------------+ <- frame->varp
|
|
// | locals |
|
|
// +------------------+
|
|
// | args to callee |
|
|
// +------------------+
|
|
// | return address |
|
|
// +------------------+ <- frame->sp
|
|
|
|
type adjustinfo struct {
|
|
old stack
|
|
delta uintptr // ptr distance from old to new stack (newbase - oldbase)
|
|
cache pcvalueCache
|
|
}
|
|
|
|
// Adjustpointer checks whether *vpp is in the old stack described by adjinfo.
|
|
// If so, it rewrites *vpp to point into the new stack.
|
|
func adjustpointer(adjinfo *adjustinfo, vpp unsafe.Pointer) {
|
|
pp := (*unsafe.Pointer)(vpp)
|
|
p := *pp
|
|
if stackDebug >= 4 {
|
|
print(" ", pp, ":", p, "\n")
|
|
}
|
|
if adjinfo.old.lo <= uintptr(p) && uintptr(p) < adjinfo.old.hi {
|
|
*pp = add(p, adjinfo.delta)
|
|
if stackDebug >= 3 {
|
|
print(" adjust ptr ", pp, ":", p, " -> ", *pp, "\n")
|
|
}
|
|
}
|
|
}
|
|
|
|
// Information from the compiler about the layout of stack frames.
|
|
type bitvector struct {
|
|
n int32 // # of bits
|
|
bytedata *uint8
|
|
}
|
|
|
|
type gobitvector struct {
|
|
n uintptr
|
|
bytedata []uint8
|
|
}
|
|
|
|
func gobv(bv bitvector) gobitvector {
|
|
return gobitvector{
|
|
uintptr(bv.n),
|
|
(*[1 << 30]byte)(unsafe.Pointer(bv.bytedata))[:(bv.n+7)/8],
|
|
}
|
|
}
|
|
|
|
func ptrbit(bv *gobitvector, i uintptr) uint8 {
|
|
return (bv.bytedata[i/8] >> (i % 8)) & 1
|
|
}
|
|
|
|
// bv describes the memory starting at address scanp.
|
|
// Adjust any pointers contained therein.
|
|
func adjustpointers(scanp unsafe.Pointer, cbv *bitvector, adjinfo *adjustinfo, f *_func) {
|
|
bv := gobv(*cbv)
|
|
minp := adjinfo.old.lo
|
|
maxp := adjinfo.old.hi
|
|
delta := adjinfo.delta
|
|
num := uintptr(bv.n)
|
|
for i := uintptr(0); i < num; i++ {
|
|
if stackDebug >= 4 {
|
|
print(" ", add(scanp, i*sys.PtrSize), ":", ptrnames[ptrbit(&bv, i)], ":", hex(*(*uintptr)(add(scanp, i*sys.PtrSize))), " # ", i, " ", bv.bytedata[i/8], "\n")
|
|
}
|
|
if ptrbit(&bv, i) == 1 {
|
|
pp := (*uintptr)(add(scanp, i*sys.PtrSize))
|
|
p := *pp
|
|
if f != nil && 0 < p && p < _PageSize && debug.invalidptr != 0 || p == poisonStack {
|
|
// Looks like a junk value in a pointer slot.
|
|
// Live analysis wrong?
|
|
getg().m.traceback = 2
|
|
print("runtime: bad pointer in frame ", funcname(f), " at ", pp, ": ", hex(p), "\n")
|
|
throw("invalid stack pointer")
|
|
}
|
|
if minp <= p && p < maxp {
|
|
if stackDebug >= 3 {
|
|
print("adjust ptr ", p, " ", funcname(f), "\n")
|
|
}
|
|
*pp = p + delta
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Note: the argument/return area is adjusted by the callee.
|
|
func adjustframe(frame *stkframe, arg unsafe.Pointer) bool {
|
|
adjinfo := (*adjustinfo)(arg)
|
|
targetpc := frame.continpc
|
|
if targetpc == 0 {
|
|
// Frame is dead.
|
|
return true
|
|
}
|
|
f := frame.fn
|
|
if stackDebug >= 2 {
|
|
print(" adjusting ", funcname(f), " frame=[", hex(frame.sp), ",", hex(frame.fp), "] pc=", hex(frame.pc), " continpc=", hex(frame.continpc), "\n")
|
|
}
|
|
if f.entry == systemstack_switchPC {
|
|
// A special routine at the bottom of stack of a goroutine that does an systemstack call.
|
|
// We will allow it to be copied even though we don't
|
|
// have full GC info for it (because it is written in asm).
|
|
return true
|
|
}
|
|
if targetpc != f.entry {
|
|
targetpc--
|
|
}
|
|
pcdata := pcdatavalue(f, _PCDATA_StackMapIndex, targetpc, &adjinfo.cache)
|
|
if pcdata == -1 {
|
|
pcdata = 0 // in prologue
|
|
}
|
|
|
|
// Adjust local variables if stack frame has been allocated.
|
|
size := frame.varp - frame.sp
|
|
var minsize uintptr
|
|
switch sys.TheChar {
|
|
case '7':
|
|
minsize = sys.SpAlign
|
|
default:
|
|
minsize = sys.MinFrameSize
|
|
}
|
|
if size > minsize {
|
|
var bv bitvector
|
|
stackmap := (*stackmap)(funcdata(f, _FUNCDATA_LocalsPointerMaps))
|
|
if stackmap == nil || stackmap.n <= 0 {
|
|
print("runtime: frame ", funcname(f), " untyped locals ", hex(frame.varp-size), "+", hex(size), "\n")
|
|
throw("missing stackmap")
|
|
}
|
|
// Locals bitmap information, scan just the pointers in locals.
|
|
if pcdata < 0 || pcdata >= stackmap.n {
|
|
// don't know where we are
|
|
print("runtime: pcdata is ", pcdata, " and ", stackmap.n, " locals stack map entries for ", funcname(f), " (targetpc=", targetpc, ")\n")
|
|
throw("bad symbol table")
|
|
}
|
|
bv = stackmapdata(stackmap, pcdata)
|
|
size = uintptr(bv.n) * sys.PtrSize
|
|
if stackDebug >= 3 {
|
|
print(" locals ", pcdata, "/", stackmap.n, " ", size/sys.PtrSize, " words ", bv.bytedata, "\n")
|
|
}
|
|
adjustpointers(unsafe.Pointer(frame.varp-size), &bv, adjinfo, f)
|
|
}
|
|
|
|
// Adjust saved base pointer if there is one.
|
|
if sys.TheChar == '6' && frame.argp-frame.varp == 2*sys.RegSize {
|
|
if !framepointer_enabled {
|
|
print("runtime: found space for saved base pointer, but no framepointer experiment\n")
|
|
print("argp=", hex(frame.argp), " varp=", hex(frame.varp), "\n")
|
|
throw("bad frame layout")
|
|
}
|
|
if stackDebug >= 3 {
|
|
print(" saved bp\n")
|
|
}
|
|
adjustpointer(adjinfo, unsafe.Pointer(frame.varp))
|
|
}
|
|
|
|
// Adjust arguments.
|
|
if frame.arglen > 0 {
|
|
var bv bitvector
|
|
if frame.argmap != nil {
|
|
bv = *frame.argmap
|
|
} else {
|
|
stackmap := (*stackmap)(funcdata(f, _FUNCDATA_ArgsPointerMaps))
|
|
if stackmap == nil || stackmap.n <= 0 {
|
|
print("runtime: frame ", funcname(f), " untyped args ", frame.argp, "+", uintptr(frame.arglen), "\n")
|
|
throw("missing stackmap")
|
|
}
|
|
if pcdata < 0 || pcdata >= stackmap.n {
|
|
// don't know where we are
|
|
print("runtime: pcdata is ", pcdata, " and ", stackmap.n, " args stack map entries for ", funcname(f), " (targetpc=", targetpc, ")\n")
|
|
throw("bad symbol table")
|
|
}
|
|
bv = stackmapdata(stackmap, pcdata)
|
|
}
|
|
if stackDebug >= 3 {
|
|
print(" args\n")
|
|
}
|
|
adjustpointers(unsafe.Pointer(frame.argp), &bv, adjinfo, nil)
|
|
}
|
|
return true
|
|
}
|
|
|
|
func adjustctxt(gp *g, adjinfo *adjustinfo) {
|
|
adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.ctxt))
|
|
}
|
|
|
|
func adjustdefers(gp *g, adjinfo *adjustinfo) {
|
|
// Adjust defer argument blocks the same way we adjust active stack frames.
|
|
tracebackdefers(gp, adjustframe, noescape(unsafe.Pointer(adjinfo)))
|
|
|
|
// Adjust pointers in the Defer structs.
|
|
// Defer structs themselves are never on the stack.
|
|
for d := gp._defer; d != nil; d = d.link {
|
|
adjustpointer(adjinfo, unsafe.Pointer(&d.fn))
|
|
adjustpointer(adjinfo, unsafe.Pointer(&d.sp))
|
|
adjustpointer(adjinfo, unsafe.Pointer(&d._panic))
|
|
}
|
|
}
|
|
|
|
func adjustpanics(gp *g, adjinfo *adjustinfo) {
|
|
// Panics are on stack and already adjusted.
|
|
// Update pointer to head of list in G.
|
|
adjustpointer(adjinfo, unsafe.Pointer(&gp._panic))
|
|
}
|
|
|
|
func adjustsudogs(gp *g, adjinfo *adjustinfo) {
|
|
// the data elements pointed to by a SudoG structure
|
|
// might be in the stack.
|
|
for s := gp.waiting; s != nil; s = s.waitlink {
|
|
adjustpointer(adjinfo, unsafe.Pointer(&s.elem))
|
|
adjustpointer(adjinfo, unsafe.Pointer(&s.selectdone))
|
|
}
|
|
}
|
|
|
|
func adjuststkbar(gp *g, adjinfo *adjustinfo) {
|
|
for i := int(gp.stkbarPos); i < len(gp.stkbar); i++ {
|
|
adjustpointer(adjinfo, unsafe.Pointer(&gp.stkbar[i].savedLRPtr))
|
|
}
|
|
}
|
|
|
|
func fillstack(stk stack, b byte) {
|
|
for p := stk.lo; p < stk.hi; p++ {
|
|
*(*byte)(unsafe.Pointer(p)) = b
|
|
}
|
|
}
|
|
|
|
// Copies gp's stack to a new stack of a different size.
|
|
// Caller must have changed gp status to Gcopystack.
|
|
func copystack(gp *g, newsize uintptr) {
|
|
if gp.syscallsp != 0 {
|
|
throw("stack growth not allowed in system call")
|
|
}
|
|
old := gp.stack
|
|
if old.lo == 0 {
|
|
throw("nil stackbase")
|
|
}
|
|
used := old.hi - gp.sched.sp
|
|
|
|
// allocate new stack
|
|
new, newstkbar := stackalloc(uint32(newsize))
|
|
if stackPoisonCopy != 0 {
|
|
fillstack(new, 0xfd)
|
|
}
|
|
if stackDebug >= 1 {
|
|
print("copystack gp=", gp, " [", hex(old.lo), " ", hex(old.hi-used), " ", hex(old.hi), "]/", gp.stackAlloc, " -> [", hex(new.lo), " ", hex(new.hi-used), " ", hex(new.hi), "]/", newsize, "\n")
|
|
}
|
|
|
|
// Disallow sigprof scans of this stack and block if there's
|
|
// one in progress.
|
|
gcLockStackBarriers(gp)
|
|
|
|
// adjust pointers in the to-be-copied frames
|
|
var adjinfo adjustinfo
|
|
adjinfo.old = old
|
|
adjinfo.delta = new.hi - old.hi
|
|
gentraceback(^uintptr(0), ^uintptr(0), 0, gp, 0, nil, 0x7fffffff, adjustframe, noescape(unsafe.Pointer(&adjinfo)), 0)
|
|
|
|
// adjust other miscellaneous things that have pointers into stacks.
|
|
adjustctxt(gp, &adjinfo)
|
|
adjustdefers(gp, &adjinfo)
|
|
adjustpanics(gp, &adjinfo)
|
|
adjustsudogs(gp, &adjinfo)
|
|
adjuststkbar(gp, &adjinfo)
|
|
|
|
// copy the stack to the new location
|
|
if stackPoisonCopy != 0 {
|
|
fillstack(new, 0xfb)
|
|
}
|
|
memmove(unsafe.Pointer(new.hi-used), unsafe.Pointer(old.hi-used), used)
|
|
|
|
// copy old stack barriers to new stack barrier array
|
|
newstkbar = newstkbar[:len(gp.stkbar)]
|
|
copy(newstkbar, gp.stkbar)
|
|
|
|
// Swap out old stack for new one
|
|
gp.stack = new
|
|
gp.stackguard0 = new.lo + _StackGuard // NOTE: might clobber a preempt request
|
|
gp.sched.sp = new.hi - used
|
|
oldsize := gp.stackAlloc
|
|
gp.stackAlloc = newsize
|
|
gp.stkbar = newstkbar
|
|
gp.stktopsp += adjinfo.delta
|
|
|
|
gcUnlockStackBarriers(gp)
|
|
|
|
// free old stack
|
|
if stackPoisonCopy != 0 {
|
|
fillstack(old, 0xfc)
|
|
}
|
|
stackfree(old, oldsize)
|
|
}
|
|
|
|
// round x up to a power of 2.
|
|
func round2(x int32) int32 {
|
|
s := uint(0)
|
|
for 1<<s < x {
|
|
s++
|
|
}
|
|
return 1 << s
|
|
}
|
|
|
|
// Called from runtime·morestack when more stack is needed.
|
|
// Allocate larger stack and relocate to new stack.
|
|
// Stack growth is multiplicative, for constant amortized cost.
|
|
//
|
|
// g->atomicstatus will be Grunning or Gscanrunning upon entry.
|
|
// If the GC is trying to stop this g then it will set preemptscan to true.
|
|
func newstack() {
|
|
thisg := getg()
|
|
// TODO: double check all gp. shouldn't be getg().
|
|
if thisg.m.morebuf.g.ptr().stackguard0 == stackFork {
|
|
throw("stack growth after fork")
|
|
}
|
|
if thisg.m.morebuf.g.ptr() != thisg.m.curg {
|
|
print("runtime: newstack called from g=", hex(thisg.m.morebuf.g), "\n"+"\tm=", thisg.m, " m->curg=", thisg.m.curg, " m->g0=", thisg.m.g0, " m->gsignal=", thisg.m.gsignal, "\n")
|
|
morebuf := thisg.m.morebuf
|
|
traceback(morebuf.pc, morebuf.sp, morebuf.lr, morebuf.g.ptr())
|
|
throw("runtime: wrong goroutine in newstack")
|
|
}
|
|
if thisg.m.curg.throwsplit {
|
|
gp := thisg.m.curg
|
|
// Update syscallsp, syscallpc in case traceback uses them.
|
|
morebuf := thisg.m.morebuf
|
|
gp.syscallsp = morebuf.sp
|
|
gp.syscallpc = morebuf.pc
|
|
print("runtime: newstack sp=", hex(gp.sched.sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n",
|
|
"\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n",
|
|
"\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n")
|
|
|
|
traceback(morebuf.pc, morebuf.sp, morebuf.lr, gp)
|
|
throw("runtime: stack split at bad time")
|
|
}
|
|
|
|
gp := thisg.m.curg
|
|
morebuf := thisg.m.morebuf
|
|
thisg.m.morebuf.pc = 0
|
|
thisg.m.morebuf.lr = 0
|
|
thisg.m.morebuf.sp = 0
|
|
thisg.m.morebuf.g = 0
|
|
rewindmorestack(&gp.sched)
|
|
|
|
// NOTE: stackguard0 may change underfoot, if another thread
|
|
// is about to try to preempt gp. Read it just once and use that same
|
|
// value now and below.
|
|
preempt := atomic.Loaduintptr(&gp.stackguard0) == stackPreempt
|
|
|
|
// Be conservative about where we preempt.
|
|
// We are interested in preempting user Go code, not runtime code.
|
|
// If we're holding locks, mallocing, or preemption is disabled, don't
|
|
// preempt.
|
|
// This check is very early in newstack so that even the status change
|
|
// from Grunning to Gwaiting and back doesn't happen in this case.
|
|
// That status change by itself can be viewed as a small preemption,
|
|
// because the GC might change Gwaiting to Gscanwaiting, and then
|
|
// this goroutine has to wait for the GC to finish before continuing.
|
|
// If the GC is in some way dependent on this goroutine (for example,
|
|
// it needs a lock held by the goroutine), that small preemption turns
|
|
// into a real deadlock.
|
|
if preempt {
|
|
if thisg.m.locks != 0 || thisg.m.mallocing != 0 || thisg.m.preemptoff != "" || thisg.m.p.ptr().status != _Prunning {
|
|
// Let the goroutine keep running for now.
|
|
// gp->preempt is set, so it will be preempted next time.
|
|
gp.stackguard0 = gp.stack.lo + _StackGuard
|
|
gogo(&gp.sched) // never return
|
|
}
|
|
}
|
|
|
|
// The goroutine must be executing in order to call newstack,
|
|
// so it must be Grunning (or Gscanrunning).
|
|
casgstatus(gp, _Grunning, _Gwaiting)
|
|
gp.waitreason = "stack growth"
|
|
|
|
if gp.stack.lo == 0 {
|
|
throw("missing stack in newstack")
|
|
}
|
|
sp := gp.sched.sp
|
|
if sys.TheChar == '6' || sys.TheChar == '8' {
|
|
// The call to morestack cost a word.
|
|
sp -= sys.PtrSize
|
|
}
|
|
if stackDebug >= 1 || sp < gp.stack.lo {
|
|
print("runtime: newstack sp=", hex(sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n",
|
|
"\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n",
|
|
"\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n")
|
|
}
|
|
if sp < gp.stack.lo {
|
|
print("runtime: gp=", gp, ", gp->status=", hex(readgstatus(gp)), "\n ")
|
|
print("runtime: split stack overflow: ", hex(sp), " < ", hex(gp.stack.lo), "\n")
|
|
throw("runtime: split stack overflow")
|
|
}
|
|
|
|
if gp.sched.ctxt != nil {
|
|
// morestack wrote sched.ctxt on its way in here,
|
|
// without a write barrier. Run the write barrier now.
|
|
// It is not possible to be preempted between then
|
|
// and now, so it's okay.
|
|
writebarrierptr_nostore((*uintptr)(unsafe.Pointer(&gp.sched.ctxt)), uintptr(gp.sched.ctxt))
|
|
}
|
|
|
|
if preempt {
|
|
if gp == thisg.m.g0 {
|
|
throw("runtime: preempt g0")
|
|
}
|
|
if thisg.m.p == 0 && thisg.m.locks == 0 {
|
|
throw("runtime: g is running but p is not")
|
|
}
|
|
if gp.preemptscan {
|
|
for !castogscanstatus(gp, _Gwaiting, _Gscanwaiting) {
|
|
// Likely to be racing with the GC as
|
|
// it sees a _Gwaiting and does the
|
|
// stack scan. If so, gcworkdone will
|
|
// be set and gcphasework will simply
|
|
// return.
|
|
}
|
|
if !gp.gcscandone {
|
|
scanstack(gp)
|
|
gp.gcscandone = true
|
|
}
|
|
gp.preemptscan = false
|
|
gp.preempt = false
|
|
casfrom_Gscanstatus(gp, _Gscanwaiting, _Gwaiting)
|
|
casgstatus(gp, _Gwaiting, _Grunning)
|
|
gp.stackguard0 = gp.stack.lo + _StackGuard
|
|
gogo(&gp.sched) // never return
|
|
}
|
|
|
|
// Act like goroutine called runtime.Gosched.
|
|
casgstatus(gp, _Gwaiting, _Grunning)
|
|
gopreempt_m(gp) // never return
|
|
}
|
|
|
|
// Allocate a bigger segment and move the stack.
|
|
oldsize := int(gp.stackAlloc)
|
|
newsize := oldsize * 2
|
|
if uintptr(newsize) > maxstacksize {
|
|
print("runtime: goroutine stack exceeds ", maxstacksize, "-byte limit\n")
|
|
throw("stack overflow")
|
|
}
|
|
|
|
casgstatus(gp, _Gwaiting, _Gcopystack)
|
|
|
|
// The concurrent GC will not scan the stack while we are doing the copy since
|
|
// the gp is in a Gcopystack status.
|
|
copystack(gp, uintptr(newsize))
|
|
if stackDebug >= 1 {
|
|
print("stack grow done\n")
|
|
}
|
|
casgstatus(gp, _Gcopystack, _Grunning)
|
|
gogo(&gp.sched)
|
|
}
|
|
|
|
//go:nosplit
|
|
func nilfunc() {
|
|
*(*uint8)(nil) = 0
|
|
}
|
|
|
|
// adjust Gobuf as if it executed a call to fn
|
|
// and then did an immediate gosave.
|
|
func gostartcallfn(gobuf *gobuf, fv *funcval) {
|
|
var fn unsafe.Pointer
|
|
if fv != nil {
|
|
fn = unsafe.Pointer(fv.fn)
|
|
} else {
|
|
fn = unsafe.Pointer(funcPC(nilfunc))
|
|
}
|
|
gostartcall(gobuf, fn, unsafe.Pointer(fv))
|
|
}
|
|
|
|
// Maybe shrink the stack being used by gp.
|
|
// Called at garbage collection time.
|
|
func shrinkstack(gp *g) {
|
|
if readgstatus(gp) == _Gdead {
|
|
if gp.stack.lo != 0 {
|
|
// Free whole stack - it will get reallocated
|
|
// if G is used again.
|
|
stackfree(gp.stack, gp.stackAlloc)
|
|
gp.stack.lo = 0
|
|
gp.stack.hi = 0
|
|
gp.stkbar = nil
|
|
gp.stkbarPos = 0
|
|
}
|
|
return
|
|
}
|
|
if gp.stack.lo == 0 {
|
|
throw("missing stack in shrinkstack")
|
|
}
|
|
|
|
if debug.gcshrinkstackoff > 0 {
|
|
return
|
|
}
|
|
|
|
oldsize := gp.stackAlloc
|
|
newsize := oldsize / 2
|
|
// Don't shrink the allocation below the minimum-sized stack
|
|
// allocation.
|
|
if newsize < _FixedStack {
|
|
return
|
|
}
|
|
// Compute how much of the stack is currently in use and only
|
|
// shrink the stack if gp is using less than a quarter of its
|
|
// current stack. The currently used stack includes everything
|
|
// down to the SP plus the stack guard space that ensures
|
|
// there's room for nosplit functions.
|
|
avail := gp.stack.hi - gp.stack.lo
|
|
if used := gp.stack.hi - gp.sched.sp + _StackLimit; used >= avail/4 {
|
|
return
|
|
}
|
|
|
|
// We can't copy the stack if we're in a syscall.
|
|
// The syscall might have pointers into the stack.
|
|
if gp.syscallsp != 0 {
|
|
return
|
|
}
|
|
if sys.GoosWindows != 0 && gp.m != nil && gp.m.libcallsp != 0 {
|
|
return
|
|
}
|
|
|
|
if stackDebug > 0 {
|
|
print("shrinking stack ", oldsize, "->", newsize, "\n")
|
|
}
|
|
|
|
oldstatus := casgcopystack(gp)
|
|
copystack(gp, newsize)
|
|
casgstatus(gp, _Gcopystack, oldstatus)
|
|
}
|
|
|
|
// freeStackSpans frees unused stack spans at the end of GC.
|
|
func freeStackSpans() {
|
|
lock(&stackpoolmu)
|
|
|
|
// Scan stack pools for empty stack spans.
|
|
for order := range stackpool {
|
|
list := &stackpool[order]
|
|
for s := list.first; s != nil; {
|
|
next := s.next
|
|
if s.ref == 0 {
|
|
list.remove(s)
|
|
s.freelist = 0
|
|
mheap_.freeStack(s)
|
|
}
|
|
s = next
|
|
}
|
|
}
|
|
|
|
unlock(&stackpoolmu)
|
|
|
|
// Free large stack spans.
|
|
lock(&stackLarge.lock)
|
|
for i := range stackLarge.free {
|
|
for s := stackLarge.free[i].first; s != nil; {
|
|
next := s.next
|
|
stackLarge.free[i].remove(s)
|
|
mheap_.freeStack(s)
|
|
s = next
|
|
}
|
|
}
|
|
unlock(&stackLarge.lock)
|
|
}
|
|
|
|
//go:nosplit
|
|
func morestackc() {
|
|
systemstack(func() {
|
|
throw("attempt to execute C code on Go stack")
|
|
})
|
|
}
|