vm: get rid of .p8virt file and cruft

codeCore/src/prog8/code/target/virtual/VirtualMachineDefinition.kt

@@ -4,12 +4,11 @@ import prog8.code.core.CompilationOptions
 import prog8.code.core.CpuType
 import prog8.code.core.IMachineDefinition
 import prog8.code.core.Zeropage
-import java.io.File
 import java.nio.file.Path
+import kotlin.io.path.isReadable
 import kotlin.io.path.name
 import kotlin.io.path.readText
 
-
 class VirtualMachineDefinition: IMachineDefinition {
 
     override val cpu = CpuType.VIRTUAL
@@ -35,14 +34,15 @@ class VirtualMachineDefinition: IMachineDefinition {
         // to not have external module dependencies in our own module, we launch the virtual machine via reflection
         val vm = Class.forName("prog8.vm.VmRunner").getDeclaredConstructor().newInstance() as IVirtualMachineRunner
         val filename = programNameWithPath.name
-        if(filename.endsWith(".p8virt")) {
+        if(programNameWithPath.isReadable()) {
             vm.runProgram(programNameWithPath.readText())
-        } else if(File("$filename.p8virt").isFile) {
-            val source = File("$filename.p8virt").readText()
-            vm.runProgram(source)
+        } else {
+            val withExt = programNameWithPath.resolveSibling("$filename.p8ir")
+            if(withExt.isReadable())
+                vm.runProgram(withExt.readText())
+            else
+                throw NoSuchFileException(withExt.toFile(), reason="not a .p8ir file")
         }
-        else
-            throw IllegalArgumentException("vm can only run .p8virt or .p8ir files")
     }
 
     override fun isIOAddress(address: UInt): Boolean = false
@@ -53,5 +53,5 @@ class VirtualMachineDefinition: IMachineDefinition {
 }
 
 interface IVirtualMachineRunner {
-    fun runProgram(source: String)
+    fun runProgram(irSource: CharSequence)
 }

codeGenIntermediate/src/prog8/codegen/intermediate/IRCodeGen.kt

@@ -65,7 +65,7 @@ class IRCodeGen(
         irProg.blocks.asSequence().flatMap { it.subroutines }.flatMap { it.chunks }.forEach { chunk ->
             chunk.lines.withIndex().forEach {
                 (lineIndex, line)-> if(line is IRCodeInstruction) {
-                    val symbolExpr = line.ins.labelSymbol?.single()
+                    val symbolExpr = line.ins.labelSymbol
                     if(symbolExpr!=null) {
                         val symbol: String
                         val index: UInt

codeGenIntermediate/src/prog8/codegen/intermediate/IRPeepholeOptimizer.kt

@@ -84,7 +84,7 @@ internal class IRPeepholeOptimizer(private val irprog: IRProgram) {
                 // if jumping to label immediately following this
                 if(idx < chunk.lines.size-1) {
                     val label = chunk.lines[idx+1] as? IRCodeLabel
-                    if(labelSymbol.size==1 && label?.name == labelSymbol[0]) {
+                    if(label?.name == labelSymbol) {
                         chunk.lines.removeAt(idx)
                         changed = true
                     }

codeGenVirtual/src/prog8/codegen/virtual/VmAssemblyProgram.kt

@@ -1,125 +1,16 @@
 package prog8.codegen.virtual
 
-import prog8.code.core.AssemblyError
 import prog8.code.core.CompilationOptions
 import prog8.code.core.IAssemblyProgram
-import prog8.intermediate.*
-import prog8.vm.Syscall
-import java.io.BufferedWriter
-import kotlin.io.path.bufferedWriter
-import kotlin.io.path.div
+import prog8.intermediate.IRFileWriter
+import prog8.intermediate.IRProgram
+
 
 internal class VmAssemblyProgram(override val name: String, private val irProgram: IRProgram): IAssemblyProgram {
 
-    override fun assemble(dummyOptions: CompilationOptions): Boolean {
-        val outfile = irProgram.options.outputDir / ("$name.p8virt")
-        println("write code to $outfile")
-
-        // at last, allocate the variables in memory.
-        val allocations = VmVariableAllocator(irProgram.st, irProgram.encoding, irProgram.options.compTarget)
-
-        outfile.bufferedWriter().use { out ->
-            allocations.asVmMemory().forEach { (name, alloc) ->
-                out.write("var ${name} $alloc\n")
-            }
-
-            out.write("------PROGRAM------\n")
-
-            if(!irProgram.options.dontReinitGlobals) {
-                out.write("; global var inits\n")
-                irProgram.globalInits.forEach { out.writeLine(it) }
-            }
-            irProgram.blocks.firstOrNull()?.let {
-                if(it.subroutines.any { it.name=="main.start" }) {
-                    // there is a "main.start" entrypoint, jump to it
-                    out.writeLine(IRCodeInstruction(Opcode.JUMP, labelSymbol = "main.start"))
-                }
-            }
-
-            out.write("; actual program code\n")
-
-            irProgram.blocks.forEach { block ->
-                if(block.address!=null)
-                    TODO("blocks can't have a load address for vm")
-                out.write("; BLOCK ${block.name} ${block.position}\n")
-                block.inlineAssembly.forEach { asm ->
-                    out.write("; ASM ${asm.position}\n")
-                    out.write(asm.assembly)
-                    out.write("\n")
-                }
-                block.subroutines.forEach { sub ->
-                    out.write("; SUB ${sub.name} ${sub.position}\n")
-                    out.write("_${sub.name}:\n")
-                    sub.chunks.forEach { chunk ->
-                        if(chunk is IRInlineAsmChunk) {
-                            out.write("; ASM ${chunk.position}\n")
-                            out.write(processInlinedAsm(chunk.assembly, allocations))
-                            out.write("\n")
-                        } else {
-                            chunk.lines.forEach { out.writeLine(it) }
-                        }
-                    }
-                    out.write("; END SUB ${sub.name}\n")
-                }
-                block.asmSubroutines.forEach { sub ->
-                    out.write("; ASMSUB ${sub.name} ${sub.position}\n")
-                    out.write("_${sub.name}:\n")
-                    out.write(processInlinedAsm(sub.assembly, allocations))
-                    out.write("\n; END ASMSUB ${sub.name}\n")
-                }
-                out.write("; END BLOCK ${block.name}\n")
-            }
-        }
+    override fun assemble(options: CompilationOptions): Boolean {
+        val writtenFile = IRFileWriter(irProgram, null).write()
+        println("Wrote intermediate representation to $writtenFile")
         return true
     }
-
-    private fun processInlinedAsm(asm: String, allocations: VmVariableAllocator): String {
-        // TODO do we have to replace variable names by their allocated address???
-        return asm
-    }
-}
-
-private fun BufferedWriter.writeLine(line: IRCodeLine) {
-    when(line) {
-        is IRCodeComment -> {
-            write("; ${line.comment}\n")
-        }
-        is IRCodeInstruction -> {
-            if(line.ins.opcode==Opcode.SYSCALL) {
-                // convert IM Syscall to VM Syscall
-                val vmSyscall = when(line.ins.value!!) {
-                    IMSyscall.SORT_UBYTE.ordinal -> Syscall.SORT_UBYTE
-                    IMSyscall.SORT_BYTE.ordinal -> Syscall.SORT_BYTE
-                    IMSyscall.SORT_UWORD.ordinal -> Syscall.SORT_UWORD
-                    IMSyscall.SORT_WORD.ordinal -> Syscall.SORT_WORD
-                    IMSyscall.ANY_BYTE.ordinal -> Syscall.ANY_BYTE
-                    IMSyscall.ANY_WORD.ordinal -> Syscall.ANY_WORD
-                    IMSyscall.ANY_FLOAT.ordinal -> Syscall.ANY_FLOAT
-                    IMSyscall.ALL_BYTE.ordinal -> Syscall.ALL_BYTE
-                    IMSyscall.ALL_WORD.ordinal -> Syscall.ALL_WORD
-                    IMSyscall.ALL_FLOAT.ordinal -> Syscall.ALL_FLOAT
-                    IMSyscall.REVERSE_BYTES.ordinal -> Syscall.REVERSE_BYTES
-                    IMSyscall.REVERSE_WORDS.ordinal -> Syscall.REVERSE_WORDS
-                    IMSyscall.REVERSE_FLOATS.ordinal -> Syscall.REVERSE_FLOATS
-                    else -> throw IllegalArgumentException("invalid IM syscall number ${line.ins.value}")
-                }
-                val newIns = line.ins.copy(value = vmSyscall.ordinal)
-                write(newIns.toString() + "\n")
-            } else
-                write(line.ins.toString() + "\n")
-        }
-        is IRCodeInlineBinary -> {
-            write("!binary ")
-            line.data.withIndex().forEach {(index, byte) ->
-                write(byte.toString(16).padStart(2,'0'))
-                if(index and 63 == 63 && index<line.data.size-1)
-                    write("\n!binary ")
-            }
-            write("\n")
-        }
-        is IRCodeLabel -> {
-            write("_${line.name}:\n")
-        }
-        else -> throw AssemblyError("invalid IR code line")
-    }
 }

compiler/src/prog8/CompilerMain.kt

@@ -2,18 +2,21 @@ package prog8
 
 import kotlinx.cli.*
 import prog8.ast.base.AstException
-import prog8.code.core.*
+import prog8.code.core.CbmPrgLauncherType
+import prog8.code.core.toHex
 import prog8.code.target.*
 import prog8.code.target.virtual.VirtualMachineDefinition
-import prog8.codegen.virtual.VmCodeGen
 import prog8.compiler.CompilationResult
 import prog8.compiler.CompilerArguments
 import prog8.compiler.compileProgram
 import java.io.File
-import java.nio.file.*
+import java.nio.file.FileSystems
+import java.nio.file.Path
+import java.nio.file.StandardWatchEventKinds
+import java.nio.file.WatchKey
 import java.time.LocalDateTime
-import kotlin.system.exitProcess
 import kotlin.io.path.Path
+import kotlin.system.exitProcess
 
 
 fun main(args: Array<String>) {
@@ -48,7 +51,7 @@ private fun compileMain(args: Array<String>): Boolean {
     val slowCodegenWarnings by cli.option(ArgType.Boolean, fullName = "slowwarn", description="show debug warnings about slow/problematic assembly code generation")
     val sourceDirs by cli.option(ArgType.String, fullName="srcdirs", description = "list of extra paths, separated with ${File.pathSeparator}, to search in for imported modules").multiple().delimiter(File.pathSeparator)
     val compilationTarget by cli.option(ArgType.String, fullName = "target", description = "target output of the compiler (one of '${C64Target.NAME}', '${C128Target.NAME}', '${Cx16Target.NAME}', '${AtariTarget.NAME}', '${VMTarget.NAME}')").default(C64Target.NAME)
-    val startVm by cli.option(ArgType.Boolean, fullName = "vm", description = "load and run a p8-virt or p8-ir listing in the VM instead")
+    val startVm by cli.option(ArgType.Boolean, fullName = "vm", description = "load and run a .p8ir IR source file in the VM")
     val watchMode by cli.option(ArgType.Boolean, fullName = "watch", description = "continuous compilation mode (watch for file changes)")
     val moduleFiles by cli.argument(ArgType.String, fullName = "modules", description = "main module file(s) to compile").multiple(999)
 
@@ -247,26 +250,6 @@ private fun processSymbolDefs(symbolDefs: List<String>): Map<String, String>? {
 
 fun runVm(irFilename: String): Boolean {
     val irFile = Path(irFilename)
-    if(irFilename.endsWith(".p8ir")) {
-        val withoutSuffix = irFilename.substring(0, irFilename.length-5)
-        val compiled = VmCodeGen.compileIR(irFile)
-        if (!compiled.assemble(CompilationOptions(  // these are just dummy options, the actual options are inside the .p8ir file itself:
-                OutputType.PRG,
-                CbmPrgLauncherType.NONE,
-                ZeropageType.DONTUSE,
-                emptyList(),
-                floats = true,
-                noSysInit = true,
-                compTarget = VMTarget(),
-                loadAddress = VMTarget().machine.PROGRAM_LOAD_ADDRESS
-            ))
-        ) {
-            return false
-        }
-        val vmdef = VirtualMachineDefinition()
-        vmdef.launchEmulator(0, Paths.get(withoutSuffix))
-        return true
-    }
     val vmdef = VirtualMachineDefinition()
     vmdef.launchEmulator(0, irFile)
     return true

compiler/test/TestLaunchEmu.kt

@@ -10,8 +10,27 @@ class TestLaunchEmu: FunSpec({
 
     test("test launch virtualmachine via target") {
         val target = VMTarget()
-        val tmpfile = kotlin.io.path.createTempFile(suffix=".p8virt")
-        tmpfile.writeText(";comment\n------PROGRAM------\n;comment\n")
+        val tmpfile = kotlin.io.path.createTempFile(suffix=".p8ir")
+        tmpfile.writeText("""<PROGRAM NAME=test>
+<OPTIONS>
+</OPTIONS>
+
+<VARIABLES>
+</VARIABLES>
+
+<MEMORYMAPPEDVARIABLES>
+</MEMORYMAPPEDVARIABLES>
+
+<MEMORYSLABS>
+</MEMORYSLABS>
+
+<INITGLOBALS>
+</INITGLOBALS>
+
+<BLOCK NAME=main ADDRESS=null ALIGN=NONE POS=[unittest: line 42 col 1-9]>
+</BLOCK>
+</PROGRAM>
+""")
         target.machine.launchEmulator(0, tmpfile)
         tmpfile.deleteExisting()
     }

compiler/test/vm/TestCompilerVirtual.kt

@@ -26,10 +26,10 @@ main {
 }"""
         val target = VMTarget()
         val result = compileText(target, true, src, writeAssembly = true, keepIR=true)!!
-        val virtfile = result.compilationOptions.outputDir.resolve(result.program.name + ".p8virt")
+        val virtfile = result.compilationOptions.outputDir.resolve(result.program.name + ".p8ir")
         VmRunner().runProgram(virtfile.readText())
         val result2 = compileText(target, true, src, writeAssembly = true, keepIR=false)!!
-        val virtfile2 = result2.compilationOptions.outputDir.resolve(result2.program.name + ".p8virt")
+        val virtfile2 = result2.compilationOptions.outputDir.resolve(result2.program.name + ".p8ir")
         VmRunner().runProgram(virtfile2.readText())
     }
 
@@ -51,10 +51,10 @@ main {
         compileText(othertarget, true, src, writeAssembly = true, keepIR=true) shouldNotBe null
         val target = VMTarget()
         val result = compileText(target, true, src, writeAssembly = true, keepIR=true)!!
-        val virtfile = result.compilationOptions.outputDir.resolve(result.program.name + ".p8virt")
+        val virtfile = result.compilationOptions.outputDir.resolve(result.program.name + ".p8ir")
         VmRunner().runProgram(virtfile.readText())
         val result2 = compileText(target, true, src, writeAssembly = true, keepIR=false)!!
-        val virtfile2 = result2.compilationOptions.outputDir.resolve(result2.program.name + ".p8virt")
+        val virtfile2 = result2.compilationOptions.outputDir.resolve(result2.program.name + ".p8ir")
         VmRunner().runProgram(virtfile2.readText())
     }
 
@@ -72,10 +72,10 @@ main {
 }"""
         val target = VMTarget()
         val result = compileText(target, true, src, writeAssembly = true, keepIR=true)!!
-        val virtfile = result.compilationOptions.outputDir.resolve(result.program.name + ".p8virt")
+        val virtfile = result.compilationOptions.outputDir.resolve(result.program.name + ".p8ir")
         VmRunner().runProgram(virtfile.readText())
         val result2 = compileText(target, true, src, writeAssembly = true, keepIR=false)!!
-        val virtfile2 = result2.compilationOptions.outputDir.resolve(result2.program.name + ".p8virt")
+        val virtfile2 = result2.compilationOptions.outputDir.resolve(result2.program.name + ".p8ir")
         VmRunner().runProgram(virtfile2.readText())
     }
 
@@ -115,10 +115,10 @@ mylabel_inside:
 
         val target = VMTarget()
         val result = compileText(target, true, src, writeAssembly = true, keepIR=true)!!
-        val virtfile = result.compilationOptions.outputDir.resolve(result.program.name + ".p8virt")
+        val virtfile = result.compilationOptions.outputDir.resolve(result.program.name + ".p8ir")
         VmRunner().runProgram(virtfile.readText())
         val result2 = compileText(target, true, src, writeAssembly = true, keepIR=false)!!
-        val virtfile2 = result2.compilationOptions.outputDir.resolve(result2.program.name + ".p8virt")
+        val virtfile2 = result2.compilationOptions.outputDir.resolve(result2.program.name + ".p8ir")
         VmRunner().runProgram(virtfile2.readText())
     }
 
@@ -143,14 +143,14 @@ skipLABEL:
         compileText(othertarget, true, src, writeAssembly = true, keepIR=true) shouldNotBe null
         val target = VMTarget()
         val result = compileText(target, true, src, writeAssembly = true, keepIR=true)!!
-        val virtfile = result.compilationOptions.outputDir.resolve(result.program.name + ".p8virt")
+        val virtfile = result.compilationOptions.outputDir.resolve(result.program.name + ".p8ir")
         VmRunner().runAndTestProgram(virtfile.readText()) { vm ->
             vm.memory.getUB(0) shouldBe 42u
             vm.memory.getUB(3) shouldBe 66u
         }
 
         val result2 = compileText(target, true, src, writeAssembly = true, keepIR=false)!!
-        val virtfile2 = result2.compilationOptions.outputDir.resolve(result2.program.name + ".p8virt")
+        val virtfile2 = result2.compilationOptions.outputDir.resolve(result2.program.name + ".p8ir")
         VmRunner().runAndTestProgram(virtfile2.readText()) { vm ->
             vm.memory.getUB(0) shouldBe 42u
             vm.memory.getUB(3) shouldBe 66u

docs/source/todo.rst

@@ -3,7 +3,9 @@ TODO
 
 For next release
 ^^^^^^^^^^^^^^^^
-- vm: get rid of p8virt format and Assembler, run p8ir directly
+- vm: merge IRCodeInstruction into Instruction directly
+- vm: actually translate IRProgram to vm program list again.
+- docs: modify compiler arch picture once this is done.
 
 ...
 

intermediate/src/prog8/intermediate/IRProgram.kt

@@ -126,7 +126,7 @@ class IRAsmSubroutine(val name: String,
 
 sealed class IRCodeLine
 
-class IRCodeInstruction(
+class IRCodeInstruction(        // TODO join this into Instruction directly
     opcode: Opcode,
     type: VmDataType?=null,
     reg1: Int?=null,        // 0-$ffff
@@ -137,7 +137,7 @@ class IRCodeInstruction(
     fpValue: Float?=null,
     labelSymbol: String?=null    // alternative to value
 ): IRCodeLine() {
-    val ins = Instruction(opcode, type, reg1, reg2, fpReg1, fpReg2, value, fpValue,  if(labelSymbol==null) null else listOf(labelSymbol))
+    val ins = Instruction(opcode, type, reg1, reg2, fpReg1, fpReg2, value, fpValue, labelSymbol)
 
     init {
         if(reg1!=null && (reg1<0 || reg1>65536))

intermediate/src/prog8/intermediate/Instructions.kt

@@ -14,9 +14,10 @@ Value stack, max 128 entries of 1 byte each.
 Status flags: Carry, Zero, Negative.   NOTE: status flags are only affected by the CMP instruction or explicit CLC/SEC!!!
                                              logical or arithmetic operations DO NOT AFFECT THE STATUS FLAGS UNLESS EXPLICITLY NOTED!
 
-Most instructions have an associated data type 'b','w','f'. (omitting it means 'b'/byte).
+Instruction set is mostly a load/store architecture, there are few instructions operating on memory directly.
+Most instructions have an associated data type 'b','w','f'. (omitting it defaults to 'b' - byte).
 Currently NO support for 24 or 32 bits integers.
-Floating point operations are just 'f' typed regular instructions, and additionally there are a few fp conversion instructions
+Floating point operations are just 'f' typed regular instructions, however there are a few unique fp conversion instructions.
 
 
 LOAD/STORE
@@ -406,7 +407,7 @@ data class Instruction(
     val fpReg2: Int?=null,      // 0-$ffff
     val value: Int?=null,       // 0-$ffff
     val fpValue: Float?=null,
-    val labelSymbol: List<String>?=null,    // symbolic label name as alternative to value (so only for Branch/jump/call Instructions!)
+    val labelSymbol: String?=null,    // symbolic label name as alternative to value (so only for Branch/jump/call Instructions!)
     val binaryData: ByteArray?=null
 ) {
     // reg1 and fpreg1 can be IN/OUT/INOUT (all others are readonly INPUT)
@@ -494,10 +495,10 @@ data class Instruction(
             result.add(",")
         }
         labelSymbol?.let {
-            if(labelSymbol[0].startsWith('&'))
-                result.add(it.joinToString("."))    // address-of something
+            if(it.startsWith('&'))
+                result.add(it)    // address-of something
             else
-                result.add("_" + it.joinToString("."))
+                result.add("_$it")
         }
         if(result.last() == ",")
             result.removeLast()

intermediate/test/TestInstructions.kt

@@ -34,7 +34,7 @@ class TestInstructions: FunSpec({
     }
 
     test("with label") {
-        val ins = Instruction(Opcode.BZ, VmDataType.WORD, reg1=11, labelSymbol = listOf("a","b","c"))
+        val ins = Instruction(Opcode.BZ, VmDataType.WORD, reg1=11, labelSymbol = "a.b.c")
         ins.opcode shouldBe Opcode.BZ
         ins.type shouldBe VmDataType.WORD
         ins.reg1 shouldBe 11

scripts/clean.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env sh
 
-rm -f *.bin *.xex *.jar *.asm *.prg *.vm.txt *.vice-mon-list *.list *.p8virt a.out imgui.ini
+rm -f *.bin *.xex *.jar *.asm *.prg *.vm.txt *.vice-mon-list *.list *.p8ir a.out imgui.ini
 rm -rf build out
 rm -rf compiler/build codeGenCpu6502/build codeGenExperimental/build codeOptimizers/build compilerAst/build dbusCompilerService/build httpCompilerService/build parser/build parser/src/prog8/parser
 

virtualmachine/src/prog8/vm/Assembler.kt

@@ -1,349 +0,0 @@
-package prog8.vm
-
-import prog8.code.core.unescape
-import prog8.intermediate.*
-
-
-class Assembler {
-    private val symbolAddresses = mutableMapOf<String, Int>()
-    private val placeholders = mutableMapOf<Int, String>()
-    var cx16virtualregBaseAdress = 0xff02
-
-    init {
-        require(instructionFormats.size== Opcode.values().size) {
-            "missing " + (Opcode.values().toSet() - instructionFormats.keys)
-        }
-    }
-
-    fun initializeMemory(memsrc: String, memory: Memory) {
-        symbolAddresses.clear()
-        val arrayValuePlaceholders = mutableListOf<Pair<Int, String>>()
-
-        val instrPattern = Regex("""var (.+) @([0-9]+) ([a-z]+)(\[[0-9]+\])? (.+)""", RegexOption.IGNORE_CASE)
-        for(line in memsrc.lines()) {
-            if(line.isBlank() || line.startsWith(';'))
-                continue
-            val match = instrPattern.matchEntire(line.trim())
-            if(match==null)
-                throw IllegalArgumentException("invalid line $line")
-            else {
-                val (name, addrStr, datatype, arrayspec, values) = match.destructured
-                if(name=="cx16.r0") {
-                    cx16virtualregBaseAdress = addrStr.toInt()
-                }
-                val numArrayElts = if(arrayspec.isBlank()) 1 else arrayspec.substring(1, arrayspec.length-1).toInt()
-                var address = parseValue(Opcode.LOADCPU, addrStr, 0).toInt()
-                symbolAddresses[name] = address
-                when(datatype) {
-                    "str" -> {
-                        val string = values.trim('"').unescape()
-                        memory.setString(address, string, false)
-                    }
-                    "strz" -> {
-                        val string = values.trim('"').unescape()
-                        memory.setString(address, string, true)
-                    }
-                    "ubyte", "byte" -> {
-                        val array = values.split(',').map { parseValue(Opcode.LOADCPU, it.trim(), 0).toInt() }
-                        require(array.size==numArrayElts || array.size==1)
-                        if(numArrayElts > array.size) {
-                            val value = array.single().toUByte()
-                            repeat(numArrayElts) {
-                                memory.setUB(address, value)
-                                address++
-                            }
-                        } else {
-                            for (value in array) {
-                                memory.setUB(address, value.toUByte())
-                                address++
-                            }
-                        }
-                    }
-                    "uword", "word" -> {
-                        if(arrayspec.isBlank()) {
-                            // single value
-                            val value = parseValue(Opcode.LOADCPU, values.trim(), 0).toInt()
-                            memory.setUW(address, value.toUShort())
-                            address += 2
-                        } else {
-                            // array initializer
-                            val array = values.split(',').withIndex().map {(index, value) ->
-                                val tv = value.trim()
-                                if(tv.startsWith('&')) {
-                                    arrayValuePlaceholders += Pair(address+index*2, tv.drop(1))
-                                    9999    // will be replaced with correct value at the end.
-                                } else
-                                    parseValue(Opcode.LOADCPU, tv, 0).toInt()
-                            }
-                            require(array.size==numArrayElts || array.size==1)
-                            if(numArrayElts>array.size) {
-                                val value = array.single().toUShort()
-                                repeat(numArrayElts) {
-                                    memory.setUW(address, value)
-                                    address += 2
-                                }
-                            } else {
-                                for (value in array) {
-                                    memory.setUW(address, value.toUShort())
-                                    address += 2
-                                }
-                            }
-                        }
-                    }
-                    "float" -> {
-                        val array = values.split(',').map { it.toFloat() }
-                        require(array.size==numArrayElts || array.size==1)
-                        if(numArrayElts>array.size) {
-                            val value = array.single()
-                            repeat(numArrayElts) {
-                                memory.setFloat(address, value)
-                                address += 4    // 32-bits floats
-                            }
-                        } else {
-                            for (value in array) {
-                                memory.setFloat(address, value)
-                                address += 4    // 32-bits floats
-                            }
-                        }
-                    }
-                    else -> throw IllegalArgumentException("invalid datatype $datatype")
-                }
-            }
-        }
-
-        // correct the addres-of values in array initializers
-        arrayValuePlaceholders.forEach { (address, symbol) ->
-            val addr = this.symbolAddresses.getValue(symbol)
-            memory.setUW(address, addr.toUShort())
-        }
-    }
-
-    fun assembleProgram(source: CharSequence): List<Instruction> {
-        placeholders.clear()
-        val program = mutableListOf<Instruction>()
-        val instructionPattern = Regex("""([a-z]+)(\.b|\.w|\.f)?(.*)""", RegexOption.IGNORE_CASE)
-        val labelPattern = Regex("""_([a-zA-Z\d\._]+):""")
-        val binaryPattern = Regex("""!binary (.+)""")
-        for (line in source.lines()) {
-            if(line.isBlank() || line.startsWith(';'))
-                continue
-            val match = instructionPattern.matchEntire(line.trim())
-            if(match==null) {
-                val binarymatch = binaryPattern.matchEntire(line.trim())
-                if(binarymatch!=null) {
-                    val hex = binarymatch.groups[1]!!.value
-                    val binary = hex.windowed(size=2, step=2).map {
-                        it.toByte(16)
-                    }.toByteArray()
-                    program.add(Instruction(Opcode.BINARYDATA, binaryData = binary))
-                } else {
-                    val labelmatch = labelPattern.matchEntire(line.trim())
-                    if (labelmatch == null)
-                        throw IllegalArgumentException("invalid line $line at line ${program.size + 1}")
-                    else {
-                        val label = labelmatch.groupValues[1]
-                        if (label in symbolAddresses)
-                            throw IllegalArgumentException("label redefined $label")
-                        symbolAddresses[label] = program.size
-                    }
-                }
-            } else {
-                val (_, instr, typestr, rest) = match.groupValues
-                if(instr=="incbin") {
-                    println("warning: ignoring incbin command: $rest")
-                    continue
-                }
-                val opcode = try {
-                    Opcode.valueOf(instr.uppercase())
-                } catch (ax: IllegalArgumentException) {
-                    throw IllegalArgumentException("invalid vmasm instruction: $instr", ax)
-                }
-                var type: VmDataType? = convertType(typestr)
-                val formats = instructionFormats.getValue(opcode)
-                val format: InstructionFormat
-                if(type !in formats) {
-                    type = VmDataType.BYTE
-                    format = if(type !in formats)
-                        formats.getValue(null)
-                    else
-                        formats.getValue(type)
-                } else {
-                    format = formats.getValue(type)
-                }
-                // parse the operands
-                val operands = rest.lowercase().split(",").toMutableList()
-                var reg1: Int? = null
-                var reg2: Int? = null
-                var reg3: Int? = null
-                var fpReg1: Int? = null
-                var fpReg2: Int? = null
-                var fpReg3: Int? = null
-                var value: Float? = null
-                var operand: String?
-
-                fun parseValueOrPlaceholder(operand: String, pc: Int, rest: String, restIndex: Int, opcode: Opcode): Float {
-                    return if(operand.startsWith('_')) {
-                        placeholders[pc] = rest.split(",")[restIndex].trim().drop(1)
-                        0f
-                    } else if(operand[0].isLetter()) {
-                        placeholders[pc] = rest.split(",")[restIndex].trim()
-                        0f
-                    } else
-                        parseValue(opcode, operand, pc)
-                }
-
-                if(operands.isNotEmpty() && operands[0].isNotEmpty()) {
-                    operand = operands.removeFirst().trim()
-                    if(operand[0]=='r')
-                        reg1 = operand.substring(1).toInt()
-                    else if(operand[0]=='f' && operand[1]=='r')
-                        fpReg1 = operand.substring(2).toInt()
-                    else {
-                        value = parseValueOrPlaceholder(operand, program.size, rest, 0, opcode)
-                        operands.clear()
-                    }
-                    if(operands.isNotEmpty()) {
-                        operand = operands.removeFirst().trim()
-                        if(operand[0]=='r')
-                            reg2 = operand.substring(1).toInt()
-                        else if(operand[0]=='f' && operand[1]=='r')
-                            fpReg2 = operand.substring(2).toInt()
-                        else {
-                            value = parseValueOrPlaceholder(operand, program.size, rest, 1, opcode)
-                            operands.clear()
-                        }
-                        if(operands.isNotEmpty()) {
-                            operand = operands.removeFirst().trim()
-                            if(operand[0]=='r')
-                                reg3 = operand.substring(1).toInt()
-                            else if(operand[0]=='f' && operand[1]=='r')
-                                fpReg3 = operand.substring(2).toInt()
-                            else {
-                                value = parseValueOrPlaceholder(operand, program.size, rest, 2, opcode)
-                                operands.clear()
-                            }
-                            if(operands.isNotEmpty()) {
-                                TODO("placeholder symbol? $operands  rest=$rest'")
-                                operands.clear()
-                            }
-                        }
-                    }
-                }
-
-                // shift the operands back into place
-                while(reg1==null && reg2!=null) {
-                    reg1 = reg2
-                    reg2 = reg3
-                    reg3 = null
-                }
-                while(fpReg1==null && fpReg2!=null) {
-                    fpReg1 = fpReg2
-                    fpReg2 = fpReg3
-                    fpReg3 = null
-                }
-                if(reg3!=null)
-                    throw IllegalArgumentException("too many reg arguments $line")
-                if(fpReg3!=null)
-                    throw IllegalArgumentException("too many fpreg arguments $line")
-
-                if(type!=null && type !in formats)
-                    throw IllegalArgumentException("invalid type code for $line")
-                if(format.reg1 && reg1==null)
-                    throw IllegalArgumentException("needs reg1 for $line")
-                if(format.reg2 && reg2==null)
-                    throw IllegalArgumentException("needs reg2 for $line")
-                if(format.value && value==null)
-                    throw IllegalArgumentException("needs value for $line")
-                if(!format.reg1 && reg1!=null)
-                    throw IllegalArgumentException("invalid reg1 for $line")
-                if(!format.reg2 && reg2!=null)
-                    throw IllegalArgumentException("invalid reg2 for $line")
-                if(value!=null && opcode !in OpcodesWithAddress) {
-                    when (type) {
-                        VmDataType.BYTE -> {
-                            if (value < -128 || value > 255)
-                                throw IllegalArgumentException("value out of range for byte: $value")
-                        }
-                        VmDataType.WORD -> {
-                            if (value < -32768 || value > 65535)
-                                throw IllegalArgumentException("value out of range for word: $value")
-                        }
-                        VmDataType.FLOAT -> {}
-                        null -> {}
-                    }
-                }
-                var floatValue: Float? = null
-                var intValue: Int? = null
-
-                if(format.value)
-                    intValue = value!!.toInt()
-                if(format.fpValue)
-                    floatValue = value!!
-
-                if(opcode in OpcodesForCpuRegisters) {
-                    val regStr = rest.split(',').last().lowercase().trim()
-                    val reg = if(regStr.startsWith('_')) regStr.substring(1) else regStr
-                    if(reg !in setOf(
-                        "a", "x", "y",
-                        "ax", "ay", "xy",
-                        "r0", "r1", "r2", "r3",
-                        "r4", "r5", "r6", "r7",
-                        "r8", "r9", "r10","r11",
-                        "r12", "r13", "r14", "r15",
-                        "pc", "pz", "pv","pn"))
-                        throw IllegalArgumentException("invalid cpu reg: $reg")
-                    program.add(Instruction(opcode, type, reg1, labelSymbol = listOf(reg)))
-                } else {
-                    program.add(Instruction(opcode, type, reg1, reg2, fpReg1, fpReg2, intValue, floatValue))
-                }
-            }
-        }
-
-        pass2replaceLabels(program)
-        return program
-    }
-
-    private fun pass2replaceLabels(program: MutableList<Instruction>) {
-        for((line, label) in placeholders) {
-            val replacement = symbolAddresses[label]
-            if(replacement==null) {
-                // it could be an address + index:   symbol+42
-                if('+' in label) {
-                    val (symbol, indexStr) = label.split('+')
-                    val index = indexStr.toInt()
-                    val address = symbolAddresses.getValue(symbol) + index
-                    program[line] = program[line].copy(value = address)
-                } else {
-                    throw IllegalArgumentException("placeholder not found in labels: $label")
-                }
-            } else {
-                program[line] = program[line].copy(value = replacement)
-            }
-        }
-    }
-
-    private fun parseValue(opcode: Opcode, value: String, pc: Int): Float {
-        return if(value.startsWith("-"))
-            -parseValue(opcode, value.substring(1), pc)
-        else if(value.startsWith('$'))
-            value.substring(1).toInt(16).toFloat()
-        else if(value.startsWith('%'))
-            value.substring(1).toInt(2).toFloat()
-        else if(value.startsWith("0x"))
-            value.substring(2).toInt(16).toFloat()
-        else if(value.startsWith('_') || value[0].isLetter())
-            throw IllegalArgumentException("attempt to parse non-numeric value $value")
-        else
-            value.toFloat()
-    }
-
-    private fun convertType(typestr: String): VmDataType? {
-        return when(typestr.lowercase()) {
-            "" -> null
-            ".b" -> VmDataType.BYTE
-            ".w" -> VmDataType.WORD
-            ".f" -> VmDataType.FLOAT
-            else -> throw IllegalArgumentException("invalid type $typestr")
-        }
-    }
-}

virtualmachine/src/prog8/vm/Main.kt

@@ -1,43 +0,0 @@
-package prog8.vm
-
-fun main(args: Array<String>) {
-    val memsrc = """
-$4000 strz "Hello from program! "derp" bye.\n"
-$2000 ubyte 65,66,67,68,0
-$2100 uword $1111,$2222,$3333,$4444
-"""
-    val src = """
-; enable lores gfx screen
-load r0, 0
-syscall 8   
-load.w r10, 320
-load.w r11, 240
-load.b r12, 0
-
-_forever:
-load.w r1, 0
-_yloop:
-load.w r0, 0
-_xloop:
-mul.b r2,r0,r1
-add.b r2,r2,r12
-syscall 10 
-addi.w r0,r0,1
-blt.w r0, r10, _xloop
-addi.w r1,r1,1
-blt.w r1, r11,_yloop
-addi.b r12,r12,1
-jump _forever
-
-load.w r0, 2000
-syscall 7
-load.w r0,0
-return"""
-    val memory = Memory()
-    val assembler = Assembler()
-    assembler.initializeMemory(memsrc, memory)
-    val program = assembler.assembleProgram(src)
-
-    val vm = VirtualMachine(memory, program, assembler.cx16virtualregBaseAdress)
-    vm.run()
-}

virtualmachine/src/prog8/vm/VirtualMachine.kt

@@ -1,9 +1,7 @@
 package prog8.vm
 
 import prog8.code.target.virtual.IVirtualMachineRunner
-import prog8.intermediate.Instruction
-import prog8.intermediate.Opcode
-import prog8.intermediate.VmDataType
+import prog8.intermediate.*
 import java.awt.Color
 import java.awt.Toolkit
 import java.util.*
@@ -31,9 +29,10 @@ class BreakpointException(val pc: Int): Exception()
 
 
 @Suppress("FunctionName")
-class VirtualMachine(val memory: Memory, program: List<Instruction>, val cx16virtualregsBaseAddress: Int) {
+class VirtualMachine(irProgram: IRProgram) {
+    val memory = Memory()
+    val program: Array<Instruction> = VmProgramLoader().load(irProgram, memory)  // TODO convert irProgram
     val registers = Registers()
-    val program: Array<Instruction> = program.toTypedArray()
     val callStack = Stack<Int>()
     val valueStack = Stack<UByte>()       // max 128 entries
     var pc = 0
@@ -41,6 +40,7 @@ class VirtualMachine(val memory: Memory, program: List<Instruction>, val cx16vir
     var statusCarry = false
     var statusZero = false
     var statusNegative = false
+    val cx16virtualregsBaseAddress = 0xff00     // TODO obtain from irProgram
 
     init {
         if(program.size>65536)
@@ -305,7 +305,7 @@ class VirtualMachine(val memory: Memory, program: List<Instruction>, val cx16vir
     }
 
     private fun InsLOADCPU(i: Instruction) {
-        val reg = i.labelSymbol!!.single()
+        val reg = i.labelSymbol!!
         val value: UInt
         if(reg.startsWith('r')) {
             val regnum = reg.substring(1).toInt()
@@ -340,12 +340,12 @@ class VirtualMachine(val memory: Memory, program: List<Instruction>, val cx16vir
             VmDataType.WORD -> registers.getUW(i.reg1!!).toUInt()
             VmDataType.FLOAT -> throw IllegalArgumentException("there are no float cpu registers")
         }
-        StoreCPU(value, i.type!!, i.labelSymbol!!.single())
+        StoreCPU(value, i.type!!, i.labelSymbol!!)
         pc++
     }
 
     private fun InsSTOREZCPU(i: Instruction) {
-        StoreCPU(0u, i.type!!, i.labelSymbol!!.single())
+        StoreCPU(0u, i.type!!, i.labelSymbol!!)
         pc++
     }
 
@@ -2105,17 +2105,13 @@ class VirtualMachine(val memory: Memory, program: List<Instruction>, val cx16vir
 
 // probably called via reflection
 class VmRunner: IVirtualMachineRunner {
-    override fun runProgram(source: String) {
-        runAndTestProgram(source) { /* no tests */ }
+    override fun runProgram(irSource: CharSequence) {
+        runAndTestProgram(irSource) { /* no tests */ }
     }
 
-    fun runAndTestProgram(source: String, test: (VirtualMachine) -> Unit) {
-        val (memsrc, programsrc) = source.split("------PROGRAM------".toRegex(), 2)
-        val memory = Memory()
-        val assembler = Assembler()
-        assembler.initializeMemory(memsrc, memory)
-        val program = assembler.assembleProgram(programsrc)
-        val vm = VirtualMachine(memory, program, assembler.cx16virtualregBaseAdress)
+    fun runAndTestProgram(irSource: CharSequence, test: (VirtualMachine) -> Unit) {
+        val irProgram = IRFileReader().read(irSource)
+        val vm = VirtualMachine(irProgram)
         vm.run()
         test(vm)
     }

virtualmachine/src/prog8/vm/VmProgramLoader.kt

@@ -0,0 +1,56 @@
+package prog8.vm
+
+import prog8.intermediate.*
+import java.lang.IllegalArgumentException
+
+class VmProgramLoader {
+
+    fun load(irProgram: IRProgram, memory: Memory): Array<Instruction> {
+
+        // at long last, allocate the variables in memory.
+        val allocations = VmVariableAllocator(irProgram.st, irProgram.encoding, irProgram.options.compTarget)
+        val program = mutableListOf<Instruction>()
+
+        // TODO stuff the allocated variables into memory
+
+        if(!irProgram.options.dontReinitGlobals) {
+            irProgram.globalInits.forEach {
+                // TODO put global init line into program
+            }
+        }
+
+        // make sure that if there is a "main.start" entrypoint, we jump to it
+        irProgram.blocks.firstOrNull()?.let {
+            if(it.subroutines.any { sub -> sub.name=="main.start" }) {
+                program.add(Instruction(Opcode.JUMP, labelSymbol = "main.start"))
+            }
+        }
+
+        irProgram.blocks.forEach { block ->
+            if(block.address!=null)
+                throw IllegalArgumentException("blocks cannot have a load address for vm: ${block.name}")
+
+            block.inlineAssembly.forEach {
+
+                // TODO put it.assembly into program
+            }
+            block.subroutines.forEach {
+                // TODO subroutine label ?
+                it.chunks.forEach { chunk ->
+                    if(chunk is IRInlineAsmChunk) {
+                        // TODO put it.assembly into program
+                    } else {
+                        chunk.lines.forEach {
+                            // TODO put line into program
+                        }
+                    }
+                }
+            }
+            block.asmSubroutines.forEach {
+                // TODO add asmsub to program
+            }
+        }
+        return emptyArray()
+    }
+
+}

virtualmachine/src/prog8/vm/VmVariableAllocator.kt

@@ -1,4 +1,4 @@
-package prog8.codegen.virtual
+package prog8.vm
 
 import prog8.code.core.*
 import prog8.intermediate.IRSymbolTable

virtualmachine/test/TestVm.kt

@@ -1,17 +1,31 @@
 import io.kotest.core.spec.style.FunSpec
 import io.kotest.matchers.collections.shouldBeEmpty
 import io.kotest.matchers.shouldBe
-import prog8.intermediate.Instruction
-import prog8.intermediate.Opcode
-import prog8.intermediate.VmDataType
-import prog8.vm.Memory
+import prog8.code.core.*
+import prog8.code.target.VMTarget
+import prog8.intermediate.*
 import prog8.vm.VirtualMachine
 import prog8.vm.VmRunner
 
 class TestVm: FunSpec( {
+
+    fun getTestOptions(): CompilationOptions {
+        val target = VMTarget()
+        return CompilationOptions(
+            OutputType.RAW,
+            CbmPrgLauncherType.NONE,
+            ZeropageType.DONTUSE,
+            zpReserved = emptyList(),
+            floats = true,
+            noSysInit = false,
+            compTarget = target,
+            loadAddress = target.machine.PROGRAM_LOAD_ADDRESS
+        )
+    }
+
     test("vm execution: empty program") {
-        val memory = Memory()
-        val vm = VirtualMachine(memory, emptyList(), 0xff00)
+        val program = IRProgram("test", IRSymbolTable(null), getTestOptions(), VMTarget())
+        val vm = VirtualMachine(program)
         vm.callStack.shouldBeEmpty()
         vm.valueStack.shouldBeEmpty()
         vm.pc shouldBe 0
@@ -24,21 +38,25 @@ class TestVm: FunSpec( {
     }
 
     test("vm execution: modify memory") {
-        val memory = Memory()
-        val program = listOf(
-            Instruction(Opcode.LOAD, VmDataType.WORD, reg1=1, value=12345),
-            Instruction(Opcode.STOREM, VmDataType.WORD, reg1=1, value=1000),
-            Instruction(Opcode.RETURN)
-        )
-        val vm = VirtualMachine(memory, program, 0xff00)
+        val program = IRProgram("test", IRSymbolTable(null), getTestOptions(), VMTarget())
+        val block = IRBlock("main", null, IRBlock.BlockAlignment.NONE, Position.DUMMY)
+        val startSub = IRSubroutine("main.start2222", emptyList(), null, Position.DUMMY)        // TODO proper name main.start
+        val code = IRCodeChunk(Position.DUMMY)
+        code += IRCodeInstruction(Opcode.LOAD, VmDataType.WORD, reg1=1, value=12345)
+        code += IRCodeInstruction(Opcode.STOREM, VmDataType.WORD, reg1=1, value=1000)
+        code += IRCodeInstruction(Opcode.RETURN)
+        startSub += code
+        block += startSub
+        program.addBlock(block)
+        val vm = VirtualMachine(program)
 
-        memory.getUW(1000) shouldBe 0u
+        vm.memory.getUW(1000) shouldBe 0u
         vm.callStack.shouldBeEmpty()
         vm.valueStack.shouldBeEmpty()
         vm.pc shouldBe 0
         vm.stepCount shouldBe 0
         vm.run()
-        memory.getUW(1000) shouldBe 12345u
+        vm.memory.getUW(1000) shouldBe 12345u
         vm.callStack.shouldBeEmpty()
         vm.valueStack.shouldBeEmpty()
         vm.pc shouldBe 2
@@ -47,6 +65,26 @@ class TestVm: FunSpec( {
 
     test("vmrunner") {
         val runner = VmRunner()
-        runner.runProgram(";comment\n------PROGRAM------\n;comment\n")
+        val irSource="""<PROGRAM NAME=test>
+<OPTIONS>
+</OPTIONS>
+
+<VARIABLES>
+</VARIABLES>
+
+<MEMORYMAPPEDVARIABLES>
+</MEMORYMAPPEDVARIABLES>
+
+<MEMORYSLABS>
+</MEMORYSLABS>
+
+<INITGLOBALS>
+</INITGLOBALS>
+
+<BLOCK NAME=main ADDRESS=null ALIGN=NONE POS=[unittest: line 42 col 1-9]>
+</BLOCK>
+</PROGRAM>
+"""
+        runner.runProgram(irSource)
     }
 })