added a few more vm optimizations and unit tests

codeGenVirtual/build.gradle

@@ -24,9 +24,9 @@ compileTestKotlin {
 }
 
 dependencies {
-    implementation project(':virtualmachine')
     implementation project(':codeAst')
     implementation project(':codeCore')
+    implementation project(':virtualmachine')
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8"
     // implementation "org.jetbrains.kotlin:kotlin-reflect"
     implementation "com.michael-bull.kotlin-result:kotlin-result-jvm:1.1.16"

codeGenVirtual/src/prog8/codegen/virtual/AssemblyProgram.kt

@@ -13,8 +13,7 @@ import kotlin.io.path.bufferedWriter
 import kotlin.io.path.div
 
 
-internal class AssemblyProgram(override val name: String,
+class AssemblyProgram(override val name: String, private val allocations: VariableAllocator
-                               private val allocations: VariableAllocator
 ) : IAssemblyProgram {
 
     private val globalInits = mutableListOf<VmCodeLine>()
@@ -71,9 +70,9 @@ internal class AssemblyProgram(override val name: String,
     fun getBlocks(): List<VmCodeChunk> = blocks
 }
 
-internal sealed class VmCodeLine
+sealed class VmCodeLine
 
-internal class VmCodeInstruction(
+class VmCodeInstruction(
     opcode: Opcode,
     type: VmDataType?=null,
     reg1: Int?=null,        // 0-$ffff
@@ -112,10 +111,10 @@ internal class VmCodeInstruction(
     }
 }
 
-internal class VmCodeLabel(val name: List<String>): VmCodeLine()
+class VmCodeLabel(val name: List<String>): VmCodeLine()
 internal class VmCodeComment(val comment: String): VmCodeLine()
 
-internal class VmCodeChunk(initial: VmCodeLine? = null) {
+class VmCodeChunk(initial: VmCodeLine? = null) {
     val lines = mutableListOf<VmCodeLine>()
 
     init {

codeGenVirtual/src/prog8/codegen/virtual/CodeGen.kt

@@ -40,7 +40,7 @@ class CodeGen(internal val program: PtProgram,
               internal val errors: IErrorReporter
 ): IAssemblyGenerator {
 
-    internal val allocations = VariableAllocator(symbolTable, program, errors)
+    internal val allocations = VariableAllocator(symbolTable, program)
     private val expressionEval = ExpressionGen(this)
     private val builtinFuncGen = BuiltinFuncGen(this, expressionEval)
     private val assignmentGen = AssignmentGen(this, expressionEval)
@@ -72,7 +72,7 @@ class CodeGen(internal val program: PtProgram,
             optimizer.optimize()
         }
 
-        println("Vm codegen: amount of vm registers=${vmRegisters.peekNext()}")
+        println("Vm codegen: virtual registers=${vmRegisters.peekNext()} memory usage=${allocations.freeMem}")
 
         return vmprog
     }

codeGenVirtual/src/prog8/codegen/virtual/VariableAllocator.kt

@@ -4,7 +4,7 @@ import prog8.code.SymbolTable
 import prog8.code.ast.PtProgram
 import prog8.code.core.*
 
-class VariableAllocator(private val st: SymbolTable, private val program: PtProgram, errors: IErrorReporter) {
+class VariableAllocator(private val st: SymbolTable, private val program: PtProgram) {
 
     private val allocations = mutableMapOf<List<String>, Int>()
     private var freeMemoryStart: Int

codeGenVirtual/src/prog8/codegen/virtual/VmPeepholeOptimizer.kt

@@ -3,30 +3,138 @@ package prog8.codegen.virtual
 import prog8.vm.Instruction
 import prog8.vm.Opcode
 
-internal class VmPeepholeOptimizer(private val vmprog: AssemblyProgram, private val allocations: VariableAllocator) {
+internal class VmOptimizerException(msg: String): Exception(msg)
+
+
+class VmPeepholeOptimizer(private val vmprog: AssemblyProgram, private val allocations: VariableAllocator) {
     fun optimize() {
         vmprog.getBlocks().forEach { block ->
             do {
                 val indexedInstructions = block.lines.withIndex()
                     .filter { it.value is VmCodeInstruction }
-                    .map { IndexedValue(it.index, (it.value as VmCodeInstruction).ins)}
+                    .map { IndexedValue(it.index, (it.value as VmCodeInstruction).ins) }
-                val changed = optimizeRemoveNops(block, indexedInstructions)
+                val changed = removeNops(block, indexedInstructions)
-                        || optimizeDoubleLoadsAndStores(block, indexedInstructions)
+                        || removeDoubleLoadsAndStores(block, indexedInstructions)
-                        // TODO other optimizations:
+                        // || removeUselessArithmetic(block, indexedInstructions)   // TODO enable
-                        //  useless arithmethic (div/mul by 1, add/sub 0, ...)
+                        || removeWeirdBranches(block, indexedInstructions)
-                        //  useless logical (bitwise (x)or 0, bitwise and by ffff, shl followed by shr or vice versa (no carry)... )
+                        || removeDoubleSecClc(block, indexedInstructions)
-                        //  jump/branch to label immediately below
+                        || cleanupPushPop(block, indexedInstructions)
-                        //  branch instructions with reg1==reg2
+                // TODO other optimizations:
-                        //  conditional set instructions with reg1==reg2
+                //  other useless logical?
-                        //  push followed by pop to same target, or different target replace with load
+                //  conditional set instructions with reg1==reg2
-                        //  double sec, clc
+                //  move complex optimizations such as unused registers, ...
-                        //  sec+clc or clc+sec
-                        //  move complex optimizations such as unused registers, ...
             } while(changed)
         }
     }
 
-    private fun optimizeRemoveNops(block: VmCodeChunk, indexedInstructions: List<IndexedValue<Instruction>>): Boolean {
+    private fun cleanupPushPop(block: VmCodeChunk, indexedInstructions: List<IndexedValue<Instruction>>): Boolean {
+        //  push followed by pop to same target, or different target->replace with load
+        var changed = false
+        indexedInstructions.reversed().forEach { (idx, ins) ->
+            if(ins.opcode==Opcode.PUSH) {
+                if(idx < block.lines.size-1) {
+                    val insAfter = block.lines[idx+1] as? VmCodeInstruction
+                    if(insAfter!=null && insAfter.ins.opcode ==Opcode.POP) {
+                        if(ins.reg1==insAfter.ins.reg1) {
+                            block.lines.removeAt(idx)
+                            block.lines.removeAt(idx)
+                        } else {
+                            block.lines[idx] = VmCodeInstruction(Opcode.LOADR, ins.type, reg1=insAfter.ins.reg1, reg2=ins.reg1)
+                            block.lines.removeAt(idx+1)
+                        }
+                        changed = true
+                    }
+                }
+            }
+        }
+        return changed
+    }
+
+
+    private fun removeDoubleSecClc(block: VmCodeChunk, indexedInstructions: List<IndexedValue<Instruction>>): Boolean {
+        //  double sec, clc
+        //  sec+clc or clc+sec
+        var changed = false
+        indexedInstructions.reversed().forEach { (idx, ins) ->
+            if(ins.opcode==Opcode.SEC || ins.opcode==Opcode.CLC) {
+                if(idx < block.lines.size-1) {
+                    val insAfter = block.lines[idx+1] as? VmCodeInstruction
+                    if(insAfter?.ins?.opcode == ins.opcode) {
+                        block.lines.removeAt(idx)
+                        changed = true
+                    }
+                    else if(ins.opcode==Opcode.SEC && insAfter?.ins?.opcode==Opcode.CLC) {
+                        block.lines.removeAt(idx)
+                        changed = true
+                    }
+                    else if(ins.opcode==Opcode.CLC && insAfter?.ins?.opcode==Opcode.SEC) {
+                        block.lines.removeAt(idx)
+                        changed = true
+                    }
+                }
+            }
+        }
+        return changed
+    }
+
+    private fun removeWeirdBranches(block: VmCodeChunk, indexedInstructions: List<IndexedValue<Instruction>>): Boolean {
+        //  jump/branch to label immediately below
+        //  branch instructions with reg1==reg2
+        var changed = false
+        indexedInstructions.reversed().forEach { (idx, ins) ->
+            if(ins.opcode==Opcode.JUMP && ins.labelSymbol!=null) {
+                // if jumping to label immediately following this
+                if(idx < block.lines.size-1) {
+                    val label = block.lines[idx+1] as? VmCodeLabel
+                    if(label?.name == ins.labelSymbol) {
+                        block.lines.removeAt(idx)
+                        changed = true
+                    }
+                }
+            }
+/*
+beq         reg1, reg2,       location  - jump to location in program given by location, if reg1 == reg2
+bne         reg1, reg2,       location  - jump to location in program given by location, if reg1 != reg2
+blt         reg1, reg2,       location  - jump to location in program given by location, if reg1 < reg2 (unsigned)
+blts        reg1, reg2,       location  - jump to location in program given by location, if reg1 < reg2 (signed)
+ble         reg1, reg2,       location  - jump to location in program given by location, if reg1 <= reg2 (unsigned)
+bles        reg1, reg2,       location  - jump to location in program given by location, if reg1 <= reg2 (signed)
+bgt         reg1, reg2,       location  - jump to location in program given by location, if reg1 > reg2 (unsigned)
+bgts        reg1, reg2,       location  - jump to location in program given by location, if reg1 > reg2 (signed)
+bge         reg1, reg2,       location  - jump to location in program given by location, if reg1 >= reg2 (unsigned)
+bges        reg1, reg2,       location  - jump to location in program given by location, if reg1 >= reg2 (signed)
+ */
+        }
+        return changed
+    }
+
+    private fun removeUselessArithmetic(block: VmCodeChunk, indexedInstructions: List<IndexedValue<Instruction>>): Boolean {
+        // TODO this is hard to solve atm because the values are loaded into registers first
+        var changed = false
+        indexedInstructions.reversed().forEach { (idx, ins) ->
+            when (ins.opcode) {
+                Opcode.DIV, Opcode.DIVS, Opcode.MUL, Opcode.MOD -> {
+                    TODO("remove div/mul by 1")
+                }
+                Opcode.ADD, Opcode.SUB -> {
+                    TODO("remove add/sub by 1 -> inc/dec, by 0->remove")
+                }
+                Opcode.AND -> {
+                    TODO("and 0 -> 0, and ffff -> remove")
+                }
+                Opcode.OR -> {
+                    TODO("or 0 -> remove, of ffff -> ffff")
+                }
+                Opcode.XOR -> {
+                    TODO("xor 0 -> remove")
+                }
+                else -> {}
+            }
+        }
+        return changed
+    }
+
+    private fun removeNops(block: VmCodeChunk, indexedInstructions: List<IndexedValue<Instruction>>): Boolean {
         var changed = false
         indexedInstructions.reversed().forEach { (idx, ins) ->
             if (ins.opcode == Opcode.NOP) {
@@ -37,7 +145,7 @@ internal class VmPeepholeOptimizer(private val vmprog: AssemblyProgram, private
         return changed
     }
 
-    private fun optimizeDoubleLoadsAndStores(block: VmCodeChunk, indexedInstructions: List<IndexedValue<Instruction>>): Boolean {
+    private fun removeDoubleLoadsAndStores(block: VmCodeChunk, indexedInstructions: List<IndexedValue<Instruction>>): Boolean {
         var changed = false
         indexedInstructions.forEach { (idx, ins) ->
 

compiler/build.gradle

@@ -34,6 +34,7 @@ dependencies {
     implementation project(':codeGenCpu6502')
     implementation project(':codeGenVirtual')
     implementation project(':codeGenExperimental')
+    implementation project(':virtualmachine')
     implementation 'org.antlr:antlr4-runtime:4.10.1'
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8"
     // implementation "org.jetbrains.kotlin:kotlin-reflect"

compiler/compiler.iml

@@ -23,5 +23,6 @@
     <orderEntry type="module" module-name="codeGenCpu6502" />
     <orderEntry type="module" module-name="codeGenExperimental" />
     <orderEntry type="module" module-name="codeGenVirtual" />
+    <orderEntry type="module" module-name="virtualmachine" />
   </component>
 </module>

compiler/test/vm/TestVmPeepholeOpt.kt

@@ -0,0 +1,96 @@
+package prog8tests.vm
+
+import io.kotest.core.spec.style.FunSpec
+import io.kotest.matchers.shouldBe
+import prog8.code.SymbolTable
+import prog8.code.ast.PtProgram
+import prog8.codegen.virtual.*
+import prog8.vm.Opcode
+import prog8.vm.VmDataType
+import prog8tests.helpers.DummyMemsizer
+import prog8tests.helpers.DummyStringEncoder
+
+class TestVmPeepholeOpt: FunSpec({
+    fun makeVmProgram(lines: List<VmCodeLine>): Pair<AssemblyProgram, VariableAllocator> {
+        val st = SymbolTable()
+        val program = PtProgram("test", DummyMemsizer, DummyStringEncoder)
+        val allocations = VariableAllocator(st, program)
+        val asm = AssemblyProgram("test", allocations)
+        val block = VmCodeChunk()
+        for(line in lines)
+            block += line
+        asm.addBlock(block)
+        return Pair(asm, allocations)
+    }
+
+    fun AssemblyProgram.lines(): List<VmCodeLine> = this.getBlocks().flatMap { it.lines }
+
+    test("remove nops") {
+        val(asm, allocations) = makeVmProgram(listOf(
+            VmCodeInstruction(Opcode.JUMP, labelSymbol = listOf("dummy")),
+            VmCodeInstruction(Opcode.NOP),
+            VmCodeInstruction(Opcode.NOP)
+        ))
+        asm.lines().size shouldBe 3
+        val opt = VmPeepholeOptimizer(asm, allocations)
+        opt.optimize()
+        asm.lines().size shouldBe 1
+    }
+
+    test("remove jmp to label below") {
+        val(asm, allocations) = makeVmProgram(listOf(
+            VmCodeInstruction(Opcode.JUMP, labelSymbol = listOf("label")),  // removed
+            VmCodeLabel(listOf("label")),
+            VmCodeInstruction(Opcode.JUMP, labelSymbol = listOf("label2")), // removed
+            VmCodeInstruction(Opcode.NOP),  // removed
+            VmCodeLabel(listOf("label2")),
+            VmCodeInstruction(Opcode.JUMP, labelSymbol = listOf("label3")),
+            VmCodeInstruction(Opcode.INC, VmDataType.BYTE, reg1=1),
+            VmCodeLabel(listOf("label3"))
+        ))
+        asm.lines().size shouldBe 8
+        val opt = VmPeepholeOptimizer(asm, allocations)
+        opt.optimize()
+        val lines = asm.lines()
+        lines.size shouldBe 5
+        (lines[0] as VmCodeLabel).name shouldBe listOf("label")
+        (lines[1] as VmCodeLabel).name shouldBe listOf("label2")
+        (lines[2] as VmCodeInstruction).ins.opcode shouldBe Opcode.JUMP
+        (lines[3] as VmCodeInstruction).ins.opcode shouldBe Opcode.INC
+        (lines[4] as VmCodeLabel).name shouldBe listOf("label3")
+    }
+
+    test("remove double sec/clc") {
+        val(asm, allocations) = makeVmProgram(listOf(
+            VmCodeInstruction(Opcode.SEC),
+            VmCodeInstruction(Opcode.SEC),
+            VmCodeInstruction(Opcode.SEC),
+            VmCodeInstruction(Opcode.CLC),
+            VmCodeInstruction(Opcode.CLC),
+            VmCodeInstruction(Opcode.CLC)
+        ))
+        asm.lines().size shouldBe 6
+        val opt = VmPeepholeOptimizer(asm, allocations)
+        opt.optimize()
+        val lines = asm.lines()
+        lines.size shouldBe 1
+        (lines[0] as VmCodeInstruction).ins.opcode shouldBe Opcode.CLC
+    }
+
+    test("push followed by pop") {
+        val(asm, allocations) = makeVmProgram(listOf(
+            VmCodeInstruction(Opcode.PUSH, VmDataType.BYTE, reg1=42),
+            VmCodeInstruction(Opcode.POP, VmDataType.BYTE, reg1=42),
+            VmCodeInstruction(Opcode.PUSH, VmDataType.BYTE, reg1=99),
+            VmCodeInstruction(Opcode.POP, VmDataType.BYTE, reg1=222)
+        ))
+        asm.lines().size shouldBe 4
+        val opt = VmPeepholeOptimizer(asm, allocations)
+        opt.optimize()
+        val lines = asm.lines()
+        lines.size shouldBe 1
+        (lines[0] as VmCodeInstruction).ins.opcode shouldBe Opcode.LOADR
+        (lines[0] as VmCodeInstruction).ins.reg1 shouldBe 222
+        (lines[0] as VmCodeInstruction).ins.reg2 shouldBe 99
+    }
+})

docs/source/todo.rst

@@ -3,6 +3,7 @@ TODO
 
 For next release
 ^^^^^^^^^^^^^^^^
+- add VM instructions to add/sub/mul/div/mod by an immediate value
 ...
 
 
@@ -25,10 +26,12 @@ Compiler:
 - vm Instruction needs to know what the read-registers/memory are, and what the write-register/memory is.
   this info is needed for more advanced optimizations and later code generation steps.
 - vm: implement remaining sin/cos functions in math.p8
+- vm: find a solution for the cx16.r0..r15 that "overlap" (r0, r0L, r0H etc) but in the vm each get their own separate variable location now
 - vm: somehow deal with asmsubs otherwise the vm IR can't fully encode all of prog8
 - vm: don't store symbol names in instructions to make optimizing the IR easier? but what about jumps to labels. And it's no longer readable by humans.
 - vm: how to remove all unused subroutines? (in the 6502 assembly codegen, we let 64tass solve this for us)
 - vm: rather than being able to jump to any 'address' (IPTR), use 'blocks' that have entry and exit points -> even better dead code elimination possible too
+- move the vm unit tests to codeGenVirtual module and remove virtualmachine dependency in the compiler module
 - when the vm is stable and *if* its language can get promoted to prog8 IL, the variable allocation should be changed.
   It's now done before the vm code generation, but the IL should probably not depend on the allocations already performed.
   So the CodeGen doesn't do VariableAlloc *before* the codegen, but as a last step.

virtualmachine/src/prog8/vm/Instructions.kt

@@ -13,8 +13,7 @@ Status flags: Carry, Zero, Negative.   NOTE: status flags are only affected by t
 Program to execute is not stored in this memory, it's just a separate list of instructions.
 Most instructions have an associated data type 'b','w','f'. (omitting it means 'b'/byte).
 Currently NO support for 24 or 32 bits integers.
-Floating point operations are just 'f' typed regular instructions, and additionally there are
+Floating point operations are just 'f' typed regular instructions, and additionally there are a few fp conversion instructions
-a few fp conversion instructions to
 
 
 LOAD/STORE