2017-04-10 11:32:00 +00:00
// Copyright 2013 The Go Authors. All rights reserved.
2015-08-28 15:33:40 +00:00
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package runtime
2017-04-10 11:32:00 +00:00
import (
"runtime/internal/atomic"
"runtime/internal/sys"
"unsafe"
)
/ *
Stack layout parameters .
Included both by runtime ( compiled via 6 c ) and linkers ( compiled via gcc ) .
The per - goroutine g - > stackguard is set to point StackGuard bytes
above the bottom of the stack . Each function compares its stack
pointer against g - > stackguard to check for overflow . To cut one
instruction from the check sequence for functions with tiny frames ,
the stack is allowed to protrude StackSmall bytes below the stack
guard . Functions with large frames don ' t bother with the check and
always call morestack . The sequences are ( for amd64 , others are
similar ) :
guard = g - > stackguard
frame = function ' s stack frame size
argsize = size of function arguments ( call + return )
stack frame size <= StackSmall :
CMPQ guard , SP
JHI 3 ( PC )
MOVQ m - > morearg , $ ( argsize << 32 )
CALL morestack ( SB )
stack frame size > StackSmall but < StackBig
LEAQ ( frame - StackSmall ) ( SP ) , R0
CMPQ guard , R0
JHI 3 ( PC )
MOVQ m - > morearg , $ ( argsize << 32 )
CALL morestack ( SB )
stack frame size >= StackBig :
MOVQ m - > morearg , $ ( ( argsize << 32 ) | frame )
CALL morestack ( SB )
The bottom StackGuard - StackSmall bytes are important : there has
to be enough room to execute functions that refuse to check for
stack overflow , either because they need to be adjacent to the
actual caller ' s frame ( deferproc ) or because they handle the imminent
stack overflow ( morestack ) .
For example , deferproc might call malloc , which does one of the
above checks ( without allocating a full frame ) , which might trigger
a call to morestack . This sequence needs to fit in the bottom
section of the stack . On amd64 , morestack ' s frame is 40 bytes , and
deferproc ' s frame is 56 bytes . That fits well within the
StackGuard - StackSmall bytes at the bottom .
The linkers explore all possible call traces involving non - splitting
functions to make sure that this limit cannot be violated .
* /
2015-08-28 15:33:40 +00:00
const (
2017-04-10 11:32:00 +00:00
// StackSystem is a number of additional bytes to add
// to each stack below the usual guard area for OS-specific
// purposes like signal handling. Used on Windows, Plan 9,
// and Darwin/ARM because they do not use a separate stack.
_StackSystem = sys . GoosWindows * 512 * sys . PtrSize + sys . GoosPlan9 * 512 + sys . GoosDarwin * sys . GoarchArm * 1024
// The minimum size of stack used by Go code
_StackMin = 2048
// The minimum stack size to allocate.
// The hackery here rounds FixedStack0 up to a power of 2.
_FixedStack0 = _StackMin + _StackSystem
_FixedStack1 = _FixedStack0 - 1
_FixedStack2 = _FixedStack1 | ( _FixedStack1 >> 1 )
_FixedStack3 = _FixedStack2 | ( _FixedStack2 >> 2 )
_FixedStack4 = _FixedStack3 | ( _FixedStack3 >> 4 )
_FixedStack5 = _FixedStack4 | ( _FixedStack4 >> 8 )
_FixedStack6 = _FixedStack5 | ( _FixedStack5 >> 16 )
_FixedStack = _FixedStack6 + 1
// Functions that need frames bigger than this use an extra
// instruction to do the stack split check, to avoid overflow
// in case SP - framesize wraps below zero.
// This value can be no bigger than the size of the unmapped
// space at zero.
_StackBig = 4096
// The stack guard is a pointer this many bytes above the
// bottom of the stack.
_StackGuard = 720 * sys . StackGuardMultiplier + _StackSystem
// After a stack split check the SP is allowed to be this
// many bytes below the stack guard. This saves an instruction
// in the checking sequence for tiny frames.
_StackSmall = 128
// The maximum number of bytes that a chain of NOSPLIT
// functions can use.
_StackLimit = _StackGuard - _StackSystem - _StackSmall
)
// Goroutine preemption request.
// Stored into g->stackguard0 to cause split stack check failure.
// Must be greater than any real sp.
// 0xfffffade in hex.
const (
_StackPreempt = uintptrMask & - 1314
_StackFork = uintptrMask & - 1234
)
const (
// stackDebug == 0: no logging
// == 1: logging of per-stack operations
// == 2: logging of per-frame operations
// == 3: logging of per-word updates
// == 4: logging of per-word reads
stackDebug = 0
stackFromSystem = 0 // allocate stacks from system memory instead of the heap
stackFaultOnFree = 0 // old stacks are mapped noaccess to detect use after free
stackPoisonCopy = 0 // fill stack that should not be accessed with garbage, to detect bad dereferences during copy
stackCache = 1
)
const (
uintptrMask = 1 << ( 8 * sys . PtrSize ) - 1
poisonStack = uintptrMask & 0x6868686868686868
2015-08-28 15:33:40 +00:00
// Goroutine preemption request.
// Stored into g->stackguard0 to cause split stack check failure.
// Must be greater than any real sp.
// 0xfffffade in hex.
2017-04-10 11:32:00 +00:00
stackPreempt = uintptrMask & - 1314
// Thread is forking.
// Stored into g->stackguard0 to cause split stack check failure.
// Must be greater than any real sp.
stackFork = uintptrMask & - 1234
2015-08-28 15:33:40 +00:00
)
2017-04-10 11:32:00 +00:00
// Global pool of spans that have free stacks.
// Stacks are assigned an order according to size.
// order = log_2(size/FixedStack)
// There is a free list for each order.
// TODO: one lock per order?
var stackpool [ _NumStackOrders ] mSpanList
var stackpoolmu mutex
// Global pool of large stack spans.
var stackLarge struct {
lock mutex
free [ _MHeapMap_Bits ] mSpanList // free lists by log_2(s.npages)
}
// Cached value of haveexperiment("framepointer")
var framepointer_enabled bool
func stackinit ( ) {
if _StackCacheSize & _PageMask != 0 {
throw ( "cache size must be a multiple of page size" )
}
for i := range stackpool {
stackpool [ i ] . init ( )
}
for i := range stackLarge . free {
stackLarge . free [ i ] . init ( )
}
}
// stacklog2 returns ⌊log_2(n)⌋.
func stacklog2 ( n uintptr ) int {
log2 := 0
for n > 1 {
n >>= 1
log2 ++
}
return log2
}
// Allocates a stack from the free pool. Must be called with
// stackpoolmu held.
func stackpoolalloc ( order uint8 ) gclinkptr {
list := & stackpool [ order ]
s := list . first
if s == nil {
// no free stacks. Allocate another span worth.
s = mheap_ . allocStack ( _StackCacheSize >> _PageShift )
if s == nil {
throw ( "out of memory" )
}
if s . ref != 0 {
throw ( "bad ref" )
}
if s . freelist . ptr ( ) != nil {
throw ( "bad freelist" )
}
for i := uintptr ( 0 ) ; i < _StackCacheSize ; i += _FixedStack << order {
x := gclinkptr ( uintptr ( s . start ) << _PageShift + i )
x . ptr ( ) . next = s . freelist
s . freelist = x
}
list . insert ( s )
}
x := s . freelist
if x . ptr ( ) == nil {
throw ( "span has no free stacks" )
}
s . freelist = x . ptr ( ) . next
s . ref ++
if s . freelist . ptr ( ) == nil {
// all stacks in s are allocated.
list . remove ( s )
}
return x
}
// Adds stack x to the free pool. Must be called with stackpoolmu held.
func stackpoolfree ( x gclinkptr , order uint8 ) {
s := mheap_ . lookup ( unsafe . Pointer ( x ) )
if s . state != _MSpanStack {
throw ( "freeing stack not in a stack span" )
}
if s . freelist . ptr ( ) == nil {
// s will now have a free stack
stackpool [ order ] . insert ( s )
}
x . ptr ( ) . next = s . freelist
s . freelist = x
s . ref --
if gcphase == _GCoff && s . ref == 0 {
// Span is completely free. Return it to the heap
// immediately if we're sweeping.
//
// If GC is active, we delay the free until the end of
// GC to avoid the following type of situation:
//
// 1) GC starts, scans a SudoG but does not yet mark the SudoG.elem pointer
// 2) The stack that pointer points to is copied
// 3) The old stack is freed
// 4) The containing span is marked free
// 5) GC attempts to mark the SudoG.elem pointer. The
// marking fails because the pointer looks like a
// pointer into a free span.
//
// By not freeing, we prevent step #4 until GC is done.
stackpool [ order ] . remove ( s )
s . freelist = 0
mheap_ . freeStack ( s )
}
}
// stackcacherefill/stackcacherelease implement a global pool of stack segments.
// The pool is required to prevent unlimited growth of per-thread caches.
func stackcacherefill ( c * mcache , order uint8 ) {
if stackDebug >= 1 {
print ( "stackcacherefill order=" , order , "\n" )
}
// Grab some stacks from the global cache.
// Grab half of the allowed capacity (to prevent thrashing).
var list gclinkptr
var size uintptr
lock ( & stackpoolmu )
for size < _StackCacheSize / 2 {
x := stackpoolalloc ( order )
x . ptr ( ) . next = list
list = x
size += _FixedStack << order
}
unlock ( & stackpoolmu )
c . stackcache [ order ] . list = list
c . stackcache [ order ] . size = size
}
func stackcacherelease ( c * mcache , order uint8 ) {
if stackDebug >= 1 {
print ( "stackcacherelease order=" , order , "\n" )
}
x := c . stackcache [ order ] . list
size := c . stackcache [ order ] . size
lock ( & stackpoolmu )
for size > _StackCacheSize / 2 {
y := x . ptr ( ) . next
stackpoolfree ( x , order )
x = y
size -= _FixedStack << order
}
unlock ( & stackpoolmu )
c . stackcache [ order ] . list = x
c . stackcache [ order ] . size = size
}
func stackcache_clear ( c * mcache ) {
if stackDebug >= 1 {
print ( "stackcache clear\n" )
}
lock ( & stackpoolmu )
for order := uint8 ( 0 ) ; order < _NumStackOrders ; order ++ {
x := c . stackcache [ order ] . list
for x . ptr ( ) != nil {
y := x . ptr ( ) . next
stackpoolfree ( x , order )
x = y
}
c . stackcache [ order ] . list = 0
c . stackcache [ order ] . size = 0
}
unlock ( & stackpoolmu )
}
func stackalloc ( n uint32 ) ( stack , [ ] stkbar ) {
// Stackalloc must be called on scheduler stack, so that we
// never try to grow the stack during the code that stackalloc runs.
// Doing so would cause a deadlock (issue 1547).
thisg := getg ( )
if thisg != thisg . m . g0 {
throw ( "stackalloc not on scheduler stack" )
}
if n & ( n - 1 ) != 0 {
throw ( "stack size not a power of 2" )
}
if stackDebug >= 1 {
print ( "stackalloc " , n , "\n" )
}
// Compute the size of stack barrier array.
maxstkbar := gcMaxStackBarriers ( int ( n ) )
nstkbar := unsafe . Sizeof ( stkbar { } ) * uintptr ( maxstkbar )
if debug . efence != 0 || stackFromSystem != 0 {
v := sysAlloc ( round ( uintptr ( n ) , _PageSize ) , & memstats . stacks_sys )
if v == nil {
throw ( "out of memory (stackalloc)" )
}
top := uintptr ( n ) - nstkbar
stkbarSlice := slice { add ( v , top ) , 0 , maxstkbar }
return stack { uintptr ( v ) , uintptr ( v ) + top } , * ( * [ ] stkbar ) ( unsafe . Pointer ( & stkbarSlice ) )
}
// Small stacks are allocated with a fixed-size free-list allocator.
// If we need a stack of a bigger size, we fall back on allocating
// a dedicated span.
var v unsafe . Pointer
if stackCache != 0 && n < _FixedStack << _NumStackOrders && n < _StackCacheSize {
order := uint8 ( 0 )
n2 := n
for n2 > _FixedStack {
order ++
n2 >>= 1
}
var x gclinkptr
c := thisg . m . mcache
if c == nil || thisg . m . preemptoff != "" || thisg . m . helpgc != 0 {
// c == nil can happen in the guts of exitsyscall or
// procresize. Just get a stack from the global pool.
// Also don't touch stackcache during gc
// as it's flushed concurrently.
lock ( & stackpoolmu )
x = stackpoolalloc ( order )
unlock ( & stackpoolmu )
} else {
x = c . stackcache [ order ] . list
if x . ptr ( ) == nil {
stackcacherefill ( c , order )
x = c . stackcache [ order ] . list
}
c . stackcache [ order ] . list = x . ptr ( ) . next
c . stackcache [ order ] . size -= uintptr ( n )
}
v = unsafe . Pointer ( x )
} else {
var s * mspan
npage := uintptr ( n ) >> _PageShift
log2npage := stacklog2 ( npage )
// Try to get a stack from the large stack cache.
lock ( & stackLarge . lock )
if ! stackLarge . free [ log2npage ] . isEmpty ( ) {
s = stackLarge . free [ log2npage ] . first
stackLarge . free [ log2npage ] . remove ( s )
}
unlock ( & stackLarge . lock )
if s == nil {
// Allocate a new stack from the heap.
s = mheap_ . allocStack ( npage )
if s == nil {
throw ( "out of memory" )
}
}
v = unsafe . Pointer ( s . start << _PageShift )
}
if raceenabled {
racemalloc ( v , uintptr ( n ) )
}
if msanenabled {
msanmalloc ( v , uintptr ( n ) )
}
if stackDebug >= 1 {
print ( " allocated " , v , "\n" )
}
top := uintptr ( n ) - nstkbar
stkbarSlice := slice { add ( v , top ) , 0 , maxstkbar }
return stack { uintptr ( v ) , uintptr ( v ) + top } , * ( * [ ] stkbar ) ( unsafe . Pointer ( & stkbarSlice ) )
}
func stackfree ( stk stack , n uintptr ) {
gp := getg ( )
v := unsafe . Pointer ( stk . lo )
if n & ( n - 1 ) != 0 {
throw ( "stack not a power of 2" )
}
if stk . lo + n < stk . hi {
throw ( "bad stack size" )
}
if stackDebug >= 1 {
println ( "stackfree" , v , n )
memclr ( v , n ) // for testing, clobber stack data
}
if debug . efence != 0 || stackFromSystem != 0 {
if debug . efence != 0 || stackFaultOnFree != 0 {
sysFault ( v , n )
} else {
sysFree ( v , n , & memstats . stacks_sys )
}
return
}
if msanenabled {
msanfree ( v , n )
}
if stackCache != 0 && n < _FixedStack << _NumStackOrders && n < _StackCacheSize {
order := uint8 ( 0 )
n2 := n
for n2 > _FixedStack {
order ++
n2 >>= 1
}
x := gclinkptr ( v )
c := gp . m . mcache
if c == nil || gp . m . preemptoff != "" || gp . m . helpgc != 0 {
lock ( & stackpoolmu )
stackpoolfree ( x , order )
unlock ( & stackpoolmu )
} else {
if c . stackcache [ order ] . size >= _StackCacheSize {
stackcacherelease ( c , order )
}
x . ptr ( ) . next = c . stackcache [ order ] . list
c . stackcache [ order ] . list = x
c . stackcache [ order ] . size += n
}
} else {
s := mheap_ . lookup ( v )
if s . state != _MSpanStack {
println ( hex ( s . start << _PageShift ) , v )
throw ( "bad span state" )
}
if gcphase == _GCoff {
// Free the stack immediately if we're
// sweeping.
mheap_ . freeStack ( s )
} else {
// If the GC is running, we can't return a
// stack span to the heap because it could be
// reused as a heap span, and this state
// change would race with GC. Add it to the
// large stack cache instead.
log2npage := stacklog2 ( s . npages )
lock ( & stackLarge . lock )
stackLarge . free [ log2npage ] . insert ( s )
unlock ( & stackLarge . lock )
}
}
}
var maxstacksize uintptr = 1 << 20 // enough until runtime.main sets it for real
var ptrnames = [ ] string {
0 : "scalar" ,
1 : "ptr" ,
}
// Stack frame layout
//
// (x86)
// +------------------+
// | args from caller |
// +------------------+ <- frame->argp
// | return address |
// +------------------+
// | caller's BP (*) | (*) if framepointer_enabled && varp < sp
// +------------------+ <- frame->varp
// | locals |
// +------------------+
// | args to callee |
// +------------------+ <- frame->sp
//
// (arm)
// +------------------+
// | args from caller |
// +------------------+ <- frame->argp
// | caller's retaddr |
// +------------------+ <- frame->varp
// | locals |
// +------------------+
// | args to callee |
// +------------------+
// | return address |
// +------------------+ <- frame->sp
type adjustinfo struct {
old stack
delta uintptr // ptr distance from old to new stack (newbase - oldbase)
cache pcvalueCache
}
// Adjustpointer checks whether *vpp is in the old stack described by adjinfo.
// If so, it rewrites *vpp to point into the new stack.
func adjustpointer ( adjinfo * adjustinfo , vpp unsafe . Pointer ) {
pp := ( * unsafe . Pointer ) ( vpp )
p := * pp
if stackDebug >= 4 {
print ( " " , pp , ":" , p , "\n" )
}
if adjinfo . old . lo <= uintptr ( p ) && uintptr ( p ) < adjinfo . old . hi {
* pp = add ( p , adjinfo . delta )
if stackDebug >= 3 {
print ( " adjust ptr " , pp , ":" , p , " -> " , * pp , "\n" )
}
}
}
// Information from the compiler about the layout of stack frames.
type bitvector struct {
n int32 // # of bits
bytedata * uint8
}
type gobitvector struct {
n uintptr
bytedata [ ] uint8
}
func gobv ( bv bitvector ) gobitvector {
return gobitvector {
uintptr ( bv . n ) ,
( * [ 1 << 30 ] byte ) ( unsafe . Pointer ( bv . bytedata ) ) [ : ( bv . n + 7 ) / 8 ] ,
}
}
func ptrbit ( bv * gobitvector , i uintptr ) uint8 {
return ( bv . bytedata [ i / 8 ] >> ( i % 8 ) ) & 1
}
// bv describes the memory starting at address scanp.
// Adjust any pointers contained therein.
func adjustpointers ( scanp unsafe . Pointer , cbv * bitvector , adjinfo * adjustinfo , f * _func ) {
bv := gobv ( * cbv )
minp := adjinfo . old . lo
maxp := adjinfo . old . hi
delta := adjinfo . delta
num := uintptr ( bv . n )
for i := uintptr ( 0 ) ; i < num ; i ++ {
if stackDebug >= 4 {
print ( " " , add ( scanp , i * sys . PtrSize ) , ":" , ptrnames [ ptrbit ( & bv , i ) ] , ":" , hex ( * ( * uintptr ) ( add ( scanp , i * sys . PtrSize ) ) ) , " # " , i , " " , bv . bytedata [ i / 8 ] , "\n" )
}
if ptrbit ( & bv , i ) == 1 {
pp := ( * uintptr ) ( add ( scanp , i * sys . PtrSize ) )
p := * pp
if f != nil && 0 < p && p < _PageSize && debug . invalidptr != 0 || p == poisonStack {
// Looks like a junk value in a pointer slot.
// Live analysis wrong?
getg ( ) . m . traceback = 2
print ( "runtime: bad pointer in frame " , funcname ( f ) , " at " , pp , ": " , hex ( p ) , "\n" )
throw ( "invalid stack pointer" )
}
if minp <= p && p < maxp {
if stackDebug >= 3 {
print ( "adjust ptr " , p , " " , funcname ( f ) , "\n" )
}
* pp = p + delta
}
}
}
}
// Note: the argument/return area is adjusted by the callee.
func adjustframe ( frame * stkframe , arg unsafe . Pointer ) bool {
adjinfo := ( * adjustinfo ) ( arg )
targetpc := frame . continpc
if targetpc == 0 {
// Frame is dead.
return true
}
f := frame . fn
if stackDebug >= 2 {
print ( " adjusting " , funcname ( f ) , " frame=[" , hex ( frame . sp ) , "," , hex ( frame . fp ) , "] pc=" , hex ( frame . pc ) , " continpc=" , hex ( frame . continpc ) , "\n" )
}
if f . entry == systemstack_switchPC {
// A special routine at the bottom of stack of a goroutine that does an systemstack call.
// We will allow it to be copied even though we don't
// have full GC info for it (because it is written in asm).
return true
}
if targetpc != f . entry {
targetpc --
}
pcdata := pcdatavalue ( f , _PCDATA_StackMapIndex , targetpc , & adjinfo . cache )
if pcdata == - 1 {
pcdata = 0 // in prologue
}
// Adjust local variables if stack frame has been allocated.
size := frame . varp - frame . sp
var minsize uintptr
switch sys . TheChar {
case '7' :
minsize = sys . SpAlign
default :
minsize = sys . MinFrameSize
}
if size > minsize {
var bv bitvector
stackmap := ( * stackmap ) ( funcdata ( f , _FUNCDATA_LocalsPointerMaps ) )
if stackmap == nil || stackmap . n <= 0 {
print ( "runtime: frame " , funcname ( f ) , " untyped locals " , hex ( frame . varp - size ) , "+" , hex ( size ) , "\n" )
throw ( "missing stackmap" )
}
// Locals bitmap information, scan just the pointers in locals.
if pcdata < 0 || pcdata >= stackmap . n {
// don't know where we are
print ( "runtime: pcdata is " , pcdata , " and " , stackmap . n , " locals stack map entries for " , funcname ( f ) , " (targetpc=" , targetpc , ")\n" )
throw ( "bad symbol table" )
}
bv = stackmapdata ( stackmap , pcdata )
size = uintptr ( bv . n ) * sys . PtrSize
if stackDebug >= 3 {
print ( " locals " , pcdata , "/" , stackmap . n , " " , size / sys . PtrSize , " words " , bv . bytedata , "\n" )
}
adjustpointers ( unsafe . Pointer ( frame . varp - size ) , & bv , adjinfo , f )
}
// Adjust saved base pointer if there is one.
if sys . TheChar == '6' && frame . argp - frame . varp == 2 * sys . RegSize {
if ! framepointer_enabled {
print ( "runtime: found space for saved base pointer, but no framepointer experiment\n" )
print ( "argp=" , hex ( frame . argp ) , " varp=" , hex ( frame . varp ) , "\n" )
throw ( "bad frame layout" )
}
if stackDebug >= 3 {
print ( " saved bp\n" )
}
adjustpointer ( adjinfo , unsafe . Pointer ( frame . varp ) )
}
// Adjust arguments.
if frame . arglen > 0 {
var bv bitvector
if frame . argmap != nil {
bv = * frame . argmap
} else {
stackmap := ( * stackmap ) ( funcdata ( f , _FUNCDATA_ArgsPointerMaps ) )
if stackmap == nil || stackmap . n <= 0 {
print ( "runtime: frame " , funcname ( f ) , " untyped args " , frame . argp , "+" , uintptr ( frame . arglen ) , "\n" )
throw ( "missing stackmap" )
}
if pcdata < 0 || pcdata >= stackmap . n {
// don't know where we are
print ( "runtime: pcdata is " , pcdata , " and " , stackmap . n , " args stack map entries for " , funcname ( f ) , " (targetpc=" , targetpc , ")\n" )
throw ( "bad symbol table" )
}
bv = stackmapdata ( stackmap , pcdata )
}
if stackDebug >= 3 {
print ( " args\n" )
}
adjustpointers ( unsafe . Pointer ( frame . argp ) , & bv , adjinfo , nil )
}
return true
}
func adjustctxt ( gp * g , adjinfo * adjustinfo ) {
adjustpointer ( adjinfo , unsafe . Pointer ( & gp . sched . ctxt ) )
}
func adjustdefers ( gp * g , adjinfo * adjustinfo ) {
// Adjust defer argument blocks the same way we adjust active stack frames.
tracebackdefers ( gp , adjustframe , noescape ( unsafe . Pointer ( adjinfo ) ) )
// Adjust pointers in the Defer structs.
// Defer structs themselves are never on the stack.
for d := gp . _defer ; d != nil ; d = d . link {
adjustpointer ( adjinfo , unsafe . Pointer ( & d . fn ) )
adjustpointer ( adjinfo , unsafe . Pointer ( & d . sp ) )
adjustpointer ( adjinfo , unsafe . Pointer ( & d . _panic ) )
}
}
func adjustpanics ( gp * g , adjinfo * adjustinfo ) {
// Panics are on stack and already adjusted.
// Update pointer to head of list in G.
adjustpointer ( adjinfo , unsafe . Pointer ( & gp . _panic ) )
}
func adjustsudogs ( gp * g , adjinfo * adjustinfo ) {
// the data elements pointed to by a SudoG structure
// might be in the stack.
for s := gp . waiting ; s != nil ; s = s . waitlink {
adjustpointer ( adjinfo , unsafe . Pointer ( & s . elem ) )
adjustpointer ( adjinfo , unsafe . Pointer ( & s . selectdone ) )
}
}
func adjuststkbar ( gp * g , adjinfo * adjustinfo ) {
for i := int ( gp . stkbarPos ) ; i < len ( gp . stkbar ) ; i ++ {
adjustpointer ( adjinfo , unsafe . Pointer ( & gp . stkbar [ i ] . savedLRPtr ) )
}
}
func fillstack ( stk stack , b byte ) {
for p := stk . lo ; p < stk . hi ; p ++ {
* ( * byte ) ( unsafe . Pointer ( p ) ) = b
}
}
// Copies gp's stack to a new stack of a different size.
// Caller must have changed gp status to Gcopystack.
func copystack ( gp * g , newsize uintptr ) {
if gp . syscallsp != 0 {
throw ( "stack growth not allowed in system call" )
}
old := gp . stack
if old . lo == 0 {
throw ( "nil stackbase" )
}
used := old . hi - gp . sched . sp
// allocate new stack
new , newstkbar := stackalloc ( uint32 ( newsize ) )
if stackPoisonCopy != 0 {
fillstack ( new , 0xfd )
}
if stackDebug >= 1 {
print ( "copystack gp=" , gp , " [" , hex ( old . lo ) , " " , hex ( old . hi - used ) , " " , hex ( old . hi ) , "]/" , gp . stackAlloc , " -> [" , hex ( new . lo ) , " " , hex ( new . hi - used ) , " " , hex ( new . hi ) , "]/" , newsize , "\n" )
}
// Disallow sigprof scans of this stack and block if there's
// one in progress.
gcLockStackBarriers ( gp )
// adjust pointers in the to-be-copied frames
var adjinfo adjustinfo
adjinfo . old = old
adjinfo . delta = new . hi - old . hi
gentraceback ( ^ uintptr ( 0 ) , ^ uintptr ( 0 ) , 0 , gp , 0 , nil , 0x7fffffff , adjustframe , noescape ( unsafe . Pointer ( & adjinfo ) ) , 0 )
// adjust other miscellaneous things that have pointers into stacks.
adjustctxt ( gp , & adjinfo )
adjustdefers ( gp , & adjinfo )
adjustpanics ( gp , & adjinfo )
adjustsudogs ( gp , & adjinfo )
adjuststkbar ( gp , & adjinfo )
// copy the stack to the new location
if stackPoisonCopy != 0 {
fillstack ( new , 0xfb )
}
memmove ( unsafe . Pointer ( new . hi - used ) , unsafe . Pointer ( old . hi - used ) , used )
// copy old stack barriers to new stack barrier array
newstkbar = newstkbar [ : len ( gp . stkbar ) ]
copy ( newstkbar , gp . stkbar )
// Swap out old stack for new one
gp . stack = new
gp . stackguard0 = new . lo + _StackGuard // NOTE: might clobber a preempt request
gp . sched . sp = new . hi - used
oldsize := gp . stackAlloc
gp . stackAlloc = newsize
gp . stkbar = newstkbar
gp . stktopsp += adjinfo . delta
gcUnlockStackBarriers ( gp )
// free old stack
if stackPoisonCopy != 0 {
fillstack ( old , 0xfc )
}
stackfree ( old , oldsize )
}
// round x up to a power of 2.
func round2 ( x int32 ) int32 {
s := uint ( 0 )
for 1 << s < x {
s ++
}
return 1 << s
}
// Called from runtime·morestack when more stack is needed.
// Allocate larger stack and relocate to new stack.
// Stack growth is multiplicative, for constant amortized cost.
//
// g->atomicstatus will be Grunning or Gscanrunning upon entry.
// If the GC is trying to stop this g then it will set preemptscan to true.
func newstack ( ) {
thisg := getg ( )
// TODO: double check all gp. shouldn't be getg().
if thisg . m . morebuf . g . ptr ( ) . stackguard0 == stackFork {
throw ( "stack growth after fork" )
}
if thisg . m . morebuf . g . ptr ( ) != thisg . m . curg {
print ( "runtime: newstack called from g=" , hex ( thisg . m . morebuf . g ) , "\n" + "\tm=" , thisg . m , " m->curg=" , thisg . m . curg , " m->g0=" , thisg . m . g0 , " m->gsignal=" , thisg . m . gsignal , "\n" )
morebuf := thisg . m . morebuf
traceback ( morebuf . pc , morebuf . sp , morebuf . lr , morebuf . g . ptr ( ) )
throw ( "runtime: wrong goroutine in newstack" )
}
if thisg . m . curg . throwsplit {
gp := thisg . m . curg
// Update syscallsp, syscallpc in case traceback uses them.
morebuf := thisg . m . morebuf
gp . syscallsp = morebuf . sp
gp . syscallpc = morebuf . pc
print ( "runtime: newstack sp=" , hex ( gp . sched . sp ) , " stack=[" , hex ( gp . stack . lo ) , ", " , hex ( gp . stack . hi ) , "]\n" ,
"\tmorebuf={pc:" , hex ( morebuf . pc ) , " sp:" , hex ( morebuf . sp ) , " lr:" , hex ( morebuf . lr ) , "}\n" ,
"\tsched={pc:" , hex ( gp . sched . pc ) , " sp:" , hex ( gp . sched . sp ) , " lr:" , hex ( gp . sched . lr ) , " ctxt:" , gp . sched . ctxt , "}\n" )
traceback ( morebuf . pc , morebuf . sp , morebuf . lr , gp )
throw ( "runtime: stack split at bad time" )
}
gp := thisg . m . curg
morebuf := thisg . m . morebuf
thisg . m . morebuf . pc = 0
thisg . m . morebuf . lr = 0
thisg . m . morebuf . sp = 0
thisg . m . morebuf . g = 0
rewindmorestack ( & gp . sched )
// NOTE: stackguard0 may change underfoot, if another thread
// is about to try to preempt gp. Read it just once and use that same
// value now and below.
preempt := atomic . Loaduintptr ( & gp . stackguard0 ) == stackPreempt
// Be conservative about where we preempt.
// We are interested in preempting user Go code, not runtime code.
// If we're holding locks, mallocing, or preemption is disabled, don't
// preempt.
// This check is very early in newstack so that even the status change
// from Grunning to Gwaiting and back doesn't happen in this case.
// That status change by itself can be viewed as a small preemption,
// because the GC might change Gwaiting to Gscanwaiting, and then
// this goroutine has to wait for the GC to finish before continuing.
// If the GC is in some way dependent on this goroutine (for example,
// it needs a lock held by the goroutine), that small preemption turns
// into a real deadlock.
if preempt {
if thisg . m . locks != 0 || thisg . m . mallocing != 0 || thisg . m . preemptoff != "" || thisg . m . p . ptr ( ) . status != _Prunning {
// Let the goroutine keep running for now.
// gp->preempt is set, so it will be preempted next time.
gp . stackguard0 = gp . stack . lo + _StackGuard
gogo ( & gp . sched ) // never return
}
}
// The goroutine must be executing in order to call newstack,
// so it must be Grunning (or Gscanrunning).
casgstatus ( gp , _Grunning , _Gwaiting )
gp . waitreason = "stack growth"
if gp . stack . lo == 0 {
throw ( "missing stack in newstack" )
}
sp := gp . sched . sp
if sys . TheChar == '6' || sys . TheChar == '8' {
// The call to morestack cost a word.
sp -= sys . PtrSize
}
if stackDebug >= 1 || sp < gp . stack . lo {
print ( "runtime: newstack sp=" , hex ( sp ) , " stack=[" , hex ( gp . stack . lo ) , ", " , hex ( gp . stack . hi ) , "]\n" ,
"\tmorebuf={pc:" , hex ( morebuf . pc ) , " sp:" , hex ( morebuf . sp ) , " lr:" , hex ( morebuf . lr ) , "}\n" ,
"\tsched={pc:" , hex ( gp . sched . pc ) , " sp:" , hex ( gp . sched . sp ) , " lr:" , hex ( gp . sched . lr ) , " ctxt:" , gp . sched . ctxt , "}\n" )
}
if sp < gp . stack . lo {
print ( "runtime: gp=" , gp , ", gp->status=" , hex ( readgstatus ( gp ) ) , "\n " )
print ( "runtime: split stack overflow: " , hex ( sp ) , " < " , hex ( gp . stack . lo ) , "\n" )
throw ( "runtime: split stack overflow" )
}
if gp . sched . ctxt != nil {
// morestack wrote sched.ctxt on its way in here,
// without a write barrier. Run the write barrier now.
// It is not possible to be preempted between then
// and now, so it's okay.
writebarrierptr_nostore ( ( * uintptr ) ( unsafe . Pointer ( & gp . sched . ctxt ) ) , uintptr ( gp . sched . ctxt ) )
}
if preempt {
if gp == thisg . m . g0 {
throw ( "runtime: preempt g0" )
}
if thisg . m . p == 0 && thisg . m . locks == 0 {
throw ( "runtime: g is running but p is not" )
}
if gp . preemptscan {
for ! castogscanstatus ( gp , _Gwaiting , _Gscanwaiting ) {
// Likely to be racing with the GC as
// it sees a _Gwaiting and does the
// stack scan. If so, gcworkdone will
// be set and gcphasework will simply
// return.
}
if ! gp . gcscandone {
scanstack ( gp )
gp . gcscandone = true
}
gp . preemptscan = false
gp . preempt = false
casfrom_Gscanstatus ( gp , _Gscanwaiting , _Gwaiting )
casgstatus ( gp , _Gwaiting , _Grunning )
gp . stackguard0 = gp . stack . lo + _StackGuard
gogo ( & gp . sched ) // never return
}
// Act like goroutine called runtime.Gosched.
casgstatus ( gp , _Gwaiting , _Grunning )
gopreempt_m ( gp ) // never return
}
// Allocate a bigger segment and move the stack.
oldsize := int ( gp . stackAlloc )
newsize := oldsize * 2
if uintptr ( newsize ) > maxstacksize {
print ( "runtime: goroutine stack exceeds " , maxstacksize , "-byte limit\n" )
throw ( "stack overflow" )
}
casgstatus ( gp , _Gwaiting , _Gcopystack )
// The concurrent GC will not scan the stack while we are doing the copy since
// the gp is in a Gcopystack status.
copystack ( gp , uintptr ( newsize ) )
if stackDebug >= 1 {
print ( "stack grow done\n" )
}
casgstatus ( gp , _Gcopystack , _Grunning )
gogo ( & gp . sched )
}
//go:nosplit
func nilfunc ( ) {
* ( * uint8 ) ( nil ) = 0
}
// adjust Gobuf as if it executed a call to fn
// and then did an immediate gosave.
func gostartcallfn ( gobuf * gobuf , fv * funcval ) {
var fn unsafe . Pointer
if fv != nil {
fn = unsafe . Pointer ( fv . fn )
} else {
fn = unsafe . Pointer ( funcPC ( nilfunc ) )
}
gostartcall ( gobuf , fn , unsafe . Pointer ( fv ) )
}
// Maybe shrink the stack being used by gp.
// Called at garbage collection time.
func shrinkstack ( gp * g ) {
if readgstatus ( gp ) == _Gdead {
if gp . stack . lo != 0 {
// Free whole stack - it will get reallocated
// if G is used again.
stackfree ( gp . stack , gp . stackAlloc )
gp . stack . lo = 0
gp . stack . hi = 0
gp . stkbar = nil
gp . stkbarPos = 0
}
return
}
if gp . stack . lo == 0 {
throw ( "missing stack in shrinkstack" )
}
if debug . gcshrinkstackoff > 0 {
return
}
oldsize := gp . stackAlloc
newsize := oldsize / 2
// Don't shrink the allocation below the minimum-sized stack
// allocation.
if newsize < _FixedStack {
return
}
// Compute how much of the stack is currently in use and only
// shrink the stack if gp is using less than a quarter of its
// current stack. The currently used stack includes everything
// down to the SP plus the stack guard space that ensures
// there's room for nosplit functions.
avail := gp . stack . hi - gp . stack . lo
if used := gp . stack . hi - gp . sched . sp + _StackLimit ; used >= avail / 4 {
return
}
// We can't copy the stack if we're in a syscall.
// The syscall might have pointers into the stack.
if gp . syscallsp != 0 {
return
}
if sys . GoosWindows != 0 && gp . m != nil && gp . m . libcallsp != 0 {
return
}
if stackDebug > 0 {
print ( "shrinking stack " , oldsize , "->" , newsize , "\n" )
}
oldstatus := casgcopystack ( gp )
copystack ( gp , newsize )
casgstatus ( gp , _Gcopystack , oldstatus )
}
// freeStackSpans frees unused stack spans at the end of GC.
func freeStackSpans ( ) {
lock ( & stackpoolmu )
// Scan stack pools for empty stack spans.
for order := range stackpool {
list := & stackpool [ order ]
for s := list . first ; s != nil ; {
next := s . next
if s . ref == 0 {
list . remove ( s )
s . freelist = 0
mheap_ . freeStack ( s )
}
s = next
}
}
unlock ( & stackpoolmu )
// Free large stack spans.
lock ( & stackLarge . lock )
for i := range stackLarge . free {
for s := stackLarge . free [ i ] . first ; s != nil ; {
next := s . next
stackLarge . free [ i ] . remove ( s )
mheap_ . freeStack ( s )
s = next
}
}
unlock ( & stackLarge . lock )
}
//go:nosplit
func morestackc ( ) {
systemstack ( func ( ) {
throw ( "attempt to execute C code on Go stack" )
} )
}