ir: making sure all names are scoped properly. textelite now runs in vm

codeCore/src/prog8/code/SymbolTable.kt

@@ -221,12 +221,11 @@ class StMemorySlab(
     name: String,
     val size: UInt,
     val align: UInt,
-    val allocatedAddress: UInt? = null,     // this is used (for now) in the VM code generator. TODO remove this once no longer used
     position: Position
 ):
     StNode(name, StNodeType.MEMORYSLAB, position) {
     override fun printProperties() {
-        print("$name  size=$size  align=$align  address=$allocatedAddress")
+        print("$name  size=$size  align=$align")
     }
 }
 

codeGenCpu6502/src/prog8/codegen/cpu6502/AsmGen.kt

@@ -2900,7 +2900,7 @@ $repeatLabel    lda  $counterVar
 
     fun addMemorySlab(name: String, size: UInt, align: UInt, position: Position): String {
         val prefixedName = "prog8_memoryslab_$name"
-        symbolTable.add(StMemorySlab(prefixedName, size, align, null, position))
+        symbolTable.add(StMemorySlab(prefixedName, size, align, position))
         return prefixedName
     }
 }

codeGenIntermediate/src/prog8/codegen/intermediate/IRCodeGen.kt

@@ -27,7 +27,7 @@ class IRCodeGen(
         flattenLabelNames()
         flattenNestedSubroutines()
 
-        val irProg = IRProgram(program.name, symbolTable, options, program.encoding)
+        val irProg = IRProgram(program.name, IRSymbolTable(symbolTable), options, program.encoding)
 
         if(!options.dontReinitGlobals) {
             // collect global variables initializers
@@ -996,7 +996,8 @@ class IRCodeGen(
     private fun translate(parameters: List<PtSubroutineParameter>) =
         parameters.map {
             val flattenedName = (it.definingSub()!!.scopedName + it.name)
-            symbolTable.flat.getValue(flattenedName) as StStaticVariable
+            val orig = symbolTable.flat.getValue(flattenedName) as StStaticVariable
+            IRSubroutine.IRParam(flattenedName.joinToString("."), orig.dt, orig)
         }
 
     private fun translate(alignment: PtBlock.BlockAlignment): IRBlock.BlockAlignment {
@@ -1036,7 +1037,7 @@ class IRCodeGen(
 
     fun addMemorySlab(name: String, size: UInt, align: UInt, position: Position): String {
         val scopedName = "prog8_memoryslab_$name"
-        symbolTable.add(StMemorySlab(scopedName, size, align, null, position))
+        symbolTable.add(StMemorySlab(scopedName, size, align, position))
         return scopedName
     }
 }

codeGenIntermediate/test/TestIRPeepholeOpt.kt

@@ -1,6 +1,5 @@
 import io.kotest.core.spec.style.FunSpec
 import io.kotest.matchers.shouldBe
-import prog8.code.SymbolTable
 import prog8.code.core.*
 import prog8.code.target.VMTarget
 import prog8.codegen.intermediate.IRPeepholeOptimizer
@@ -15,7 +14,6 @@ class TestIRPeepholeOpt: FunSpec({
             chunk += line
         sub += chunk
         block += sub
-        val st = SymbolTable()
         val target = VMTarget()
         val options = CompilationOptions(
             OutputType.RAW,
@@ -27,7 +25,7 @@ class TestIRPeepholeOpt: FunSpec({
             compTarget = target,
             loadAddress = target.machine.PROGRAM_LOAD_ADDRESS
         )
-        val prog = IRProgram("test", st, options, target)
+        val prog = IRProgram("test", IRSymbolTable(null), options, target)
         prog.addBlock(block)
         return prog
     }

codeGenVirtual/src/prog8/codegen/virtual/VmAssemblyProgram.kt

@@ -20,7 +20,7 @@ internal class VmAssemblyProgram(override val name: String, private val irProgra
 
         outfile.bufferedWriter().use { out ->
             allocations.asVmMemory().forEach { (name, alloc) ->
-                out.write("var ${name.joinToString(".")} $alloc\n")
+                out.write("var ${name} $alloc\n")
             }
 
             out.write("------PROGRAM------\n")

codeGenVirtual/src/prog8/codegen/virtual/VmVariableAllocator.kt

@@ -1,12 +1,12 @@
 package prog8.codegen.virtual
 
-import prog8.code.SymbolTable
 import prog8.code.core.*
+import prog8.intermediate.IRSymbolTable
 import prog8.intermediate.getTypeString
 
-internal class VmVariableAllocator(val st: SymbolTable, val encoding: IStringEncoding, memsizer: IMemSizer) {
+internal class VmVariableAllocator(val st: IRSymbolTable, val encoding: IStringEncoding, memsizer: IMemSizer) {
 
-    internal val allocations = mutableMapOf<List<String>, Int>()
+    internal val allocations = mutableMapOf<String, Int>()
     private var freeMemoryStart: Int
 
     val freeMem: Int
@@ -15,7 +15,7 @@ internal class VmVariableAllocator(val st: SymbolTable, val encoding: IStringEnc
 
     init {
         var nextLocation = 0
-        for (variable in st.allVariables) {
+        for (variable in st.allVariables()) {
             val memsize =
                 when (variable.dt) {
                     DataType.STR -> variable.onetimeInitializationStringValue!!.first.length + 1  // include the zero byte
@@ -24,24 +24,24 @@ internal class VmVariableAllocator(val st: SymbolTable, val encoding: IStringEnc
                     else -> throw InternalCompilerException("weird dt")
                 }
 
-            allocations[variable.scopedName] = nextLocation
+            allocations[variable.name] = nextLocation
             nextLocation += memsize
         }
-        for(slab in st.allMemorySlabs) {
+        for(slab in st.allMemorySlabs()) {
             // we ignore the alignment for the VM.
-            allocations[slab.scopedName] = nextLocation
+            allocations[slab.name] = nextLocation
             nextLocation += slab.size.toInt()
         }
 
         freeMemoryStart = nextLocation
     }
 
-    fun asVmMemory(): List<Pair<List<String>, String>> {
+    fun asVmMemory(): List<Pair<String, String>> {
-        val mm = mutableListOf<Pair<List<String>, String>>()
+        val mm = mutableListOf<Pair<String, String>>()
 
         // normal variables
-        for (variable in st.allVariables) {
+        for (variable in st.allVariables()) {
-            val location = allocations.getValue(variable.scopedName)
+            val location = allocations.getValue(variable.name)
             val value = when(variable.dt) {
                 DataType.FLOAT -> (variable.onetimeInitializationNumericValue ?: 0.0).toString()
                 in NumericDatatypes -> (variable.onetimeInitializationNumericValue ?: 0).toHex()
@@ -70,11 +70,11 @@ internal class VmVariableAllocator(val st: SymbolTable, val encoding: IStringEnc
                 }
                 else -> throw InternalCompilerException("weird dt")
             }
-            mm.add(Pair(variable.scopedName, "@$location ${getTypeString(variable)} $value"))
+            mm.add(Pair(variable.name, "@$location ${getTypeString(variable)} $value"))
         }
 
         // memory mapped variables
-        for (variable in st.allMemMappedVariables) {
+        for (variable in st.allMemMappedVariables()) {
             val value = when(variable.dt) {
                 DataType.FLOAT -> "0.0"
                 in NumericDatatypes -> "0"
@@ -82,13 +82,13 @@ internal class VmVariableAllocator(val st: SymbolTable, val encoding: IStringEnc
                 in ArrayDatatypes -> (1..variable.length!!).joinToString(",") { "0" }
                 else -> throw InternalCompilerException("weird dt for mem mapped var")
             }
-            mm.add(Pair(variable.scopedName, "@${variable.address} ${getTypeString(variable)} $value"))
+            mm.add(Pair(variable.name, "@${variable.address} ${getTypeString(variable)} $value"))
         }
 
         // memory slabs.
-        for(slab in st.allMemorySlabs) {
+        for(slab in st.allMemorySlabs()) {
-            val address = allocations.getValue(slab.scopedName)
+            val address = allocations.getValue(slab.name)
-            mm.add(Pair(slab.scopedName, "@$address ubyte[${slab.size}] 0"))
+            mm.add(Pair(slab.name, "@$address ubyte[${slab.size}] 0"))
         }
 
         return mm

compiler/test/TestTypecasts.kt

@@ -14,7 +14,6 @@ import prog8.ast.statements.VarDecl
 import prog8.code.core.DataType
 import prog8.code.core.Position
 import prog8.code.target.C64Target
-import prog8.code.target.VMTarget
 import prog8tests.helpers.ErrorReporterForTests
 import prog8tests.helpers.compileText
 
@@ -31,7 +30,7 @@ class TestTypecasts: FunSpec({
                 }
             }"""
         val errors = ErrorReporterForTests()
-        val result = compileText(VMTarget(), false, text, writeAssembly = false, errors=errors)
+        val result = compileText(C64Target(), false, text, writeAssembly = false, errors=errors)
         result shouldBe null
         errors.errors.size shouldBe 1
         errors.errors[0] shouldContain "type mismatch, was: FLOAT expected one of: [UBYTE, BYTE, UWORD, WORD]"
@@ -627,9 +626,9 @@ main  {
                     ubyte @shared wordNr2 = (interlaced >= ${'$'}33) + (interlaced >= ${'$'}66) + (interlaced >= ${'$'}99) + (interlaced >= ${'$'}CC)
                 }
             }"""
-        val result = compileText(VMTarget(), false, text, writeAssembly = true)!!
+        val result = compileText(C64Target(), false, text, writeAssembly = true)!!
         val stmts = result.program.entrypoint.statements
-        stmts.size shouldBe 14
+        stmts.size shouldBeGreaterThan 10
     }
 
     test("word to byte casts") {

docs/source/todo.rst

@@ -3,7 +3,6 @@ TODO
 
 For next release
 ^^^^^^^^^^^^^^^^
-- fix vm assembler bug in TestCompilerVirtual when no IR code is written (IRWriter.writeVariableAllocations TODO)
 - fix examples/vm/textelite.p8 having wrong randomization? (starts with wrong planet)
 
 ...

intermediate/src/prog8/intermediate/IRFileReader.kt

@@ -32,7 +32,7 @@ class IRFileReader(outputDir: Path, programName: String) {
         val initGlobals = parseInitGlobals(lines)
         val blocks = parseBlocksUntilProgramEnd(lines, variables)
 
-        val st = SymbolTable()
+        val st = IRSymbolTable(null)
         variables.forEach { st.add(it) }
         memorymapped.forEach { st.add(it) }
         slabs.forEach { st.add(it) }
@@ -213,7 +213,7 @@ class IRFileReader(outputDir: Path, programName: String) {
             // example: "SLAB slabname 4096 0"
             val match = slabPattern.matchEntire(line) ?: throw IRParseException("invalid SLAB $line")
             val (name, size, align) = match.destructured
-            slabs.add(StMemorySlab(name, size.toUInt(), align.toUInt(), null, Position.DUMMY))
+            slabs.add(StMemorySlab(name, size.toUInt(), align.toUInt(), Position.DUMMY))
         }
         return slabs
     }
@@ -365,18 +365,19 @@ class IRFileReader(outputDir: Path, programName: String) {
         return sub
     }
 
-    private fun parseParameters(lines: Iterator<String>, variables: List<StStaticVariable>): List<StStaticVariable> {
+    private fun parseParameters(lines: Iterator<String>, variables: List<StStaticVariable>): List<IRSubroutine.IRParam> {
         var line = lines.next()
         if(line!="<PARAMS>")
             throw IRParseException("missing PARAMS")
-        val params = mutableListOf<StStaticVariable>()
+        val params = mutableListOf<IRSubroutine.IRParam>()
         while(true) {
             line = lines.next()
             if(line=="</PARAMS>")
                 return params
             val (datatype, name) = line.split(' ')
             val dt = parseDatatype(datatype, datatype.contains('['))
-            params.add(variables.single { it.dt==dt && it.name==name})
+            val orig = variables.single { it.dt==dt && it.name==name}
+            params.add(IRSubroutine.IRParam(name, dt, orig))
         }
     }
 

intermediate/src/prog8/intermediate/IRFileWriter.kt

@@ -38,7 +38,7 @@ class IRFileWriter(private val irProgram: IRProgram) {
             block.subroutines.forEach {
                 out.write("<SUB NAME=${it.name} RETURNTYPE=${it.returnType.toString().lowercase()} POS=${it.position}>\n")
                 out.write("<PARAMS>\n")
-                it.parameters.forEach { param -> out.write("${getTypeString(param)} ${param.scopedName.joinToString(".")}\n") }
+                it.parameters.forEach { param -> out.write("${getTypeString(param.dt)} ${param.name}\n") }
                 out.write("</PARAMS>\n")
                 it.chunks.forEach { chunk ->
                     if(chunk is IRInlineAsmChunk) {
@@ -100,7 +100,7 @@ class IRFileWriter(private val irProgram: IRProgram) {
 
     private fun writeVariableAllocations() {
         out.write("\n<VARIABLES>\n")
-        for (variable in irProgram.st.allVariables) {
+        for (variable in irProgram.st.allVariables()) {
             val typeStr = getTypeString(variable)
             val value: String = when(variable.dt) {
                 DataType.FLOAT -> (variable.onetimeInitializationNumericValue ?: 0.0).toString()
@@ -121,12 +121,8 @@ class IRFileWriter(private val irProgram: IRProgram) {
                         variable.onetimeInitializationArrayValue!!.joinToString(",") {
                             if(it.number!=null)
                                 it.number!!.toInt().toString()
-                            else {
+                            else
-                                // TODO : don't do a lookup; addressOf should be scoped properly already!
+                                "&${it.addressOf!!.joinToString(".")}"
-                                val target = variable.lookup(it.addressOf!!)
-                                    ?: throw InternalCompilerException("symbol not found: ${it.addressOf} in ${variable.scopedName}")
-                                "&${target.scopedName.joinToString(".")}"
-                            }
                         }
                     } else {
                         (1..variable.length!!).joinToString(",") { "0" }
@@ -135,19 +131,19 @@ class IRFileWriter(private val irProgram: IRProgram) {
                 else -> throw InternalCompilerException("weird dt")
             }
             // TODO have uninitialized variables and arrays? (BSS SECTION)
-            out.write("$typeStr ${variable.scopedName.joinToString(".")}=$value zp=${variable.zpwish}\n")
+            out.write("$typeStr ${variable.name}=$value zp=${variable.zpwish}\n")
         }
         out.write("</VARIABLES>\n")
 
         out.write("\n<MEMORYMAPPEDVARIABLES>\n")
-        for (variable in irProgram.st.allMemMappedVariables) {
+        for (variable in irProgram.st.allMemMappedVariables()) {
             val typeStr = getTypeString(variable)
-            out.write("&$typeStr ${variable.scopedName.joinToString(".")}=${variable.address}\n")
+            out.write("&$typeStr ${variable.name}=${variable.address}\n")
         }
         out.write("</MEMORYMAPPEDVARIABLES>\n")
 
         out.write("\n<MEMORYSLABS>\n")
-        irProgram.st.allMemorySlabs.forEach{ slab -> out.write("SLAB ${slab.name} ${slab.size} ${slab.align}\n") }
+        irProgram.st.allMemorySlabs().forEach{ slab -> out.write("SLAB ${slab.name} ${slab.size} ${slab.align}\n") }
         out.write("</MEMORYSLABS>\n")
     }
 

intermediate/src/prog8/intermediate/IRProgram.kt

@@ -1,7 +1,6 @@
 package prog8.intermediate
 
 import prog8.code.StStaticVariable
-import prog8.code.SymbolTable
 import prog8.code.core.*
 
 /*
@@ -47,7 +46,7 @@ PROGRAM:
 */
 
 class IRProgram(val name: String,
-                val st: SymbolTable,
+                val st: IRSymbolTable,
                 val options: CompilationOptions,
                 val encoding: IStringEncoding) {
 
@@ -86,10 +85,12 @@ class IRBlock(
 }
 
 class IRSubroutine(val name: String,
-                   val parameters: List<StStaticVariable>,  // NOTE: these are the same objects as their occurrences as variables in the symbol table
+                   val parameters: List<IRParam>,
                    val returnType: DataType?,
                    val position: Position) {
 
+    class IRParam(val name: String, val dt: DataType, val orig: StStaticVariable)
+
     val chunks = mutableListOf<IRCodeChunkBase>()
 
     init {
@@ -115,8 +116,6 @@ class IRAsmSubroutine(val name: String,
                       val parameters: List<Pair<DataType, RegisterOrStatusflag>>,
                       val returns: List<Pair<DataType, RegisterOrStatusflag>>,
                       val assembly: String) {
-    val lines = mutableListOf<IRCodeLine>()
-
     init {
         if(!name.contains('.'))
             throw IllegalArgumentException("subroutine name is not scoped: $name")

intermediate/src/prog8/intermediate/IRSymbolTable.kt

@@ -0,0 +1,111 @@
+package prog8.intermediate
+
+import prog8.code.*
+
+
+// In the Intermediate Representation, all nesting has been removed.
+// So the symbol table is just a big flat mapping of (scoped)name to node.
+
+class IRSymbolTable(sourceSt: SymbolTable?) {
+    private val table = mutableMapOf<String, StNode>()
+
+    init {
+        if(sourceSt!=null) {
+            sourceSt.allVariables.forEach {
+                add(it)
+            }
+            sourceSt.allMemMappedVariables.forEach {
+                add(it)
+            }
+            sourceSt.allMemorySlabs.forEach {
+                add(it)
+            }
+
+            require(table.all { it.key==it.value.name })
+
+            allVariables().forEach {variable ->
+                variable.onetimeInitializationArrayValue?.let {
+                    it.forEach { arrayElt ->
+                        if(arrayElt.addressOf!=null) {
+                            require(arrayElt.addressOf!!.size > 1) {
+                                "pointer var in array should be properly scoped: ${arrayElt.addressOf!!} in ${variable.name}"
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    fun allVariables(): Sequence<StStaticVariable> =
+        table.asSequence().map { it.value }.filterIsInstance<StStaticVariable>()
+
+    fun allMemMappedVariables(): Sequence<StMemVar> =
+        table.asSequence().map { it.value }.filterIsInstance<StMemVar>()
+
+    fun allMemorySlabs(): Sequence<StMemorySlab> =
+        table.asSequence().map { it.value }.filterIsInstance<StMemorySlab>()
+
+    fun lookup(name: String) = table[name]
+
+    fun add(variable: StStaticVariable) {
+        val scopedName: String
+        val varToadd: StStaticVariable
+        if('.' in variable.name) {
+            scopedName = variable.name
+            varToadd = variable
+        } else {
+            fun fixupAddressOfInArray(array: List<StArrayElement>?): List<StArrayElement>? {
+                if(array==null)
+                    return null
+                val newArray = mutableListOf<StArrayElement>()
+                array.forEach {
+                    if(it.addressOf!=null) {
+                        val target = variable.lookup(it.addressOf!!)!!
+                        newArray.add(StArrayElement(null, target.scopedName))
+                    } else {
+                        newArray.add(it)
+                    }
+                }
+                return newArray
+            }
+            scopedName = variable.scopedName.joinToString(".")
+            varToadd = StStaticVariable(scopedName, variable.dt,
+                variable.onetimeInitializationNumericValue,
+                variable.onetimeInitializationStringValue,
+                fixupAddressOfInArray(variable.onetimeInitializationArrayValue),
+                variable.length,
+                variable.zpwish,
+                variable.position
+            )
+        }
+        table[scopedName] = varToadd
+    }
+
+
+    fun add(variable: StMemVar) {
+        val scopedName: String
+        val varToadd: StMemVar
+        if('.' in variable.name) {
+            scopedName = variable.name
+            varToadd = variable
+        } else {
+            scopedName = variable.scopedName.joinToString(".")
+            varToadd = StMemVar(scopedName, variable.dt, variable.address, variable.length, variable.position)
+        }
+        table[scopedName] = varToadd
+    }
+
+    fun add(variable: StMemorySlab) {
+        val scopedName: String
+        val varToadd: StMemorySlab
+        if('.' in variable.name) {
+            scopedName = variable.name
+            varToadd = variable
+        } else {
+            scopedName = variable.scopedName.joinToString(".")
+            varToadd = StMemorySlab(scopedName, variable.size, variable.align, variable.position)
+        }
+        table[scopedName] = varToadd
+    }
+}

intermediate/src/prog8/intermediate/Utils.kt

@@ -6,7 +6,7 @@ import prog8.code.core.DataType
 import prog8.code.core.InternalCompilerException
 
 
-public fun getTypeString(dt : DataType): String {
+fun getTypeString(dt : DataType): String {
     return when(dt) {
         DataType.UBYTE -> "ubyte"
         DataType.BYTE -> "byte"
@@ -22,7 +22,7 @@ public fun getTypeString(dt : DataType): String {
     }
 }
 
-public fun getTypeString(memvar: StMemVar): String {
+fun getTypeString(memvar: StMemVar): String {
     return when(memvar.dt) {
         DataType.UBYTE -> "ubyte"
         DataType.BYTE -> "byte"
@@ -38,7 +38,7 @@ public fun getTypeString(memvar: StMemVar): String {
     }
 }
 
-public fun getTypeString(variable : StStaticVariable): String {
+fun getTypeString(variable : StStaticVariable): String {
     return when(variable.dt) {
         DataType.UBYTE -> "ubyte"
         DataType.BYTE -> "byte"

intermediate/test/TestIRFileInOut.kt

@@ -3,7 +3,6 @@ import io.kotest.matchers.ints.shouldBeGreaterThan
 import io.kotest.matchers.shouldBe
 import io.kotest.matchers.types.instanceOf
 import prog8.code.StStaticVariable
-import prog8.code.SymbolTable
 import prog8.code.core.CbmPrgLauncherType
 import prog8.code.core.CompilationOptions
 import prog8.code.core.OutputType
@@ -12,11 +11,11 @@ import prog8.code.target.Cx16Target
 import prog8.intermediate.IRFileReader
 import prog8.intermediate.IRFileWriter
 import prog8.intermediate.IRProgram
+import prog8.intermediate.IRSymbolTable
 import kotlin.io.path.*
 
 class TestIRFileInOut: FunSpec({
     test("test IR writer") {
-        val st = SymbolTable()
         val target = Cx16Target()
         val tempdir = Path(System.getProperty("java.io.tmpdir"))
         val options = CompilationOptions(
@@ -30,7 +29,7 @@ class TestIRFileInOut: FunSpec({
             loadAddress = target.machine.PROGRAM_LOAD_ADDRESS,
             outputDir = tempdir
         )
-        val program = IRProgram("unittest-irwriter", st, options, target)
+        val program = IRProgram("unittest-irwriter", IRSymbolTable(null), options, target)
         val writer = IRFileWriter(program)
         writer.writeFile()
         val generatedFile = tempdir.resolve("unittest-irwriter.p8ir")
@@ -96,7 +95,7 @@ return
 </BLOCK>
 </PROGRAM>
 """
-        val tempfile = kotlin.io.path.createTempFile(suffix = ".p8ir")
+        val tempfile = createTempFile(suffix = ".p8ir")
         tempfile.writeText(source)
         val filepart = tempfile.name.dropLast(5)
         val reader = IRFileReader(tempfile.parent, filepart)
@@ -104,6 +103,7 @@ return
         tempfile.deleteExisting()
         program.name shouldBe "test-ir-reader"
         program.blocks.size shouldBe 2
+        program.st.allVariables().count() shouldBe 1
         program.st.lookup("sys.wait.jiffies") shouldBe instanceOf<StStaticVariable>()
     }
 })