first setup of LSP languageserver

working on multiplexer

.aiignore

@@ -0,0 +1,17 @@
+# An .aiignore file follows the same syntax as a .gitignore file.
+# .gitignore documentation: https://git-scm.com/docs/gitignore
+
+# you can ignore files
+.DS_Store
+*.log
+*.tmp
+*.bin
+*.prg
+
+# or folders
+dist/
+build/
+out/
+output/
+.gradle/
+.kotlin/

.idea/modules.xml

@@ -15,6 +15,7 @@
       <module fileurl="file://$PROJECT_DIR$/docs/docs.iml" filepath="$PROJECT_DIR$/docs/docs.iml" />
       <module fileurl="file://$PROJECT_DIR$/examples/examples.iml" filepath="$PROJECT_DIR$/examples/examples.iml" />
       <module fileurl="file://$PROJECT_DIR$/intermediate/intermediate.iml" filepath="$PROJECT_DIR$/intermediate/intermediate.iml" />
+      <module fileurl="file://$PROJECT_DIR$/languageServer/languageServer.iml" filepath="$PROJECT_DIR$/languageServer/languageServer.iml" />
       <module fileurl="file://$PROJECT_DIR$/parser/parser.iml" filepath="$PROJECT_DIR$/parser/parser.iml" />
       <module fileurl="file://$PROJECT_DIR$/.idea/modules/prog8.iml" filepath="$PROJECT_DIR$/.idea/modules/prog8.iml" />
       <module fileurl="file://$PROJECT_DIR$/simpleAst/simpleAst.iml" filepath="$PROJECT_DIR$/simpleAst/simpleAst.iml" />

codeGenCpu6502/src/prog8/codegen/cpu6502/AsmGen.kt

@@ -558,7 +558,7 @@ class AsmGen6502Internal (
                 } else {
                     return if (allocator.isZpVar((target as PtNamedNode).scopedName)) {
                         // pointervar is already in the zero page, no need to copy
-                        loadAFromZpPointerVar(sourceName, true)
+                        loadAFromZpPointerVar(sourceName)
                         sourceName
                     } else {
                         out("""
@@ -617,7 +617,7 @@ class AsmGen6502Internal (
         }
     }
 
-    internal fun loadAFromZpPointerVar(zpPointerVar: String, keepY: Boolean) {
+    internal fun loadAFromZpPointerVar(zpPointerVar: String, keepY: Boolean=false) {
         if (isTargetCpu(CpuType.CPU65C02))
             out("  lda  ($zpPointerVar)")
         else {
@@ -813,6 +813,9 @@ class AsmGen6502Internal (
     }
 
     internal fun assignExpressionTo(value: PtExpression, target: AsmAssignTarget) {
+
+        // this is basically the fallback assignment routine, it is RARELY called
+
         when {
             target.datatype.isByteOrBool -> {
                 if (value.asConstInteger()==0) {

codeGenCpu6502/src/prog8/codegen/cpu6502/AsmOptimizer.kt

@@ -463,17 +463,21 @@ private fun optimizeStoreLoadSame(
                 mods.add(Modification(lines[2].index, true, null))
         }
 
-        //  all 3 registers:  lda VALUE + sta SOMEWHERE + lda VALUE  -> last load can be eliminated
+        //  all 3 registers:  lda VALUE + sta SOMEWHERE + lda VALUE  -> last load can be eliminated IF NOT IO ADDRESS
         if (first.startsWith("lda ") && second.startsWith("sta ") && third.startsWith("lda ") ||
             first.startsWith("ldx ") && second.startsWith("stx ") && third.startsWith("ldx ") ||
             first.startsWith("ldy ") && second.startsWith("sty ") && third.startsWith("ldy ")
         ) {
             val firstVal = first.substring(4).trimStart()
             val thirdVal = third.substring(4).trimStart()
-            if (firstVal == thirdVal)
+            if (firstVal == thirdVal) {
+                val address = getAddressArg(third, symbolTable)
+                if (address != null && !machine.isIOAddress(address)) {
                     mods.add(Modification(lines[3].index, true, null))
                 }
             }
+        }
+    }
     return mods
 }
 

codeGenCpu6502/src/prog8/codegen/cpu6502/assignment/AsmAssignment.kt

@@ -10,7 +10,7 @@ internal enum class TargetStorageKind {
     ARRAY,
     MEMORY,
     REGISTER,
-    POINTER,        // wherever the pointer variable points to
+    POINTER,
     VOID              // assign nothing - used in multi-value assigns for void placeholders
 }
 

codeGenCpu6502/src/prog8/codegen/cpu6502/assignment/AssignmentAsmGen.kt

@@ -393,7 +393,7 @@ internal class AssignmentAsmGen(
 
         // fallback assignmen through temporary pointer var
         assignExpressionToVariable(address, "P8ZP_SCRATCH_W2", DataType.UWORD)
-        asmgen.loadAFromZpPointerVar("P8ZP_SCRATCH_W2", false)
+        asmgen.loadAFromZpPointerVar("P8ZP_SCRATCH_W2")
         assignRegisterByte(target, CpuRegister.A, false, true)
     }
 
@@ -2245,7 +2245,7 @@ $endLabel""")
 
                     fun assignViaExprEval(addressExpression: PtExpression) {
                         asmgen.assignExpressionToVariable(addressExpression, "P8ZP_SCRATCH_W2", DataType.UWORD)
-                        asmgen.loadAFromZpPointerVar("P8ZP_SCRATCH_W2", false)
+                        asmgen.loadAFromZpPointerVar("P8ZP_SCRATCH_W2")
                         asmgen.out("  ldy  #0")
                         assignRegisterpairWord(target, RegisterOrPair.AY)
                     }

codeGenCpu6502/src/prog8/codegen/cpu6502/assignment/PointerAssignmentsGen.kt

@@ -499,10 +499,9 @@ internal class PointerAssignmentsGen(private val asmgen: AsmGen6502Internal, pri
     }
 
     private fun addUnsignedByteOrWordToAY(value: PtExpression, scale: Int?) {
-        // TODO generate more optimal code from this, a bit too much stack and temporary variable juggling seems to be done in some easy cases...
 
         fun complicatedFallback() {
-            // slow fallback routine that can deal with any expression for value
+            // slow fallback routine that can deal with any expression for value (const number and variable have been dealt with already)
 
             fun restoreAYandAddAsmVariable(varname: String, isByte: Boolean) {
                 asmgen.restoreRegisterStack(CpuRegister.Y, false)       // msb

codeGenIntermediate/src/prog8/codegen/intermediate/AssignmentGen.kt

@@ -463,10 +463,6 @@ internal class AssignmentGen(private val codeGen: IRCodeGen, private val express
 
     private fun translateRegularAssign(assignment: PtAssignment): IRCodeChunks {
         // note: assigning array and string values is done via an explicit memcopy/stringcopy function call.
-        val targetIdent = assignment.target.identifier
-        val targetMemory = assignment.target.memory
-        val targetArray = assignment.target.array
-        val targetPointerDeref = assignment.target.pointerDeref
         val valueDt = irType(assignment.value.type)
         val targetDt = irType(assignment.target.type)
         val result = mutableListOf<IRCodeChunkBase>()
@@ -507,41 +503,32 @@ internal class AssignmentGen(private val codeGen: IRCodeGen, private val express
             }
         }
 
-        if(targetIdent!=null) {
+        with(assignment.target) {
+            when {
+                identifier != null -> {
                     val instruction = if(zero) {
-                IRInstruction(Opcode.STOREZM, targetDt, labelSymbol = targetIdent.name)
+                        IRInstruction(Opcode.STOREZM, targetDt, labelSymbol = identifier!!.name)
                     } else {
                         if (targetDt == IRDataType.FLOAT) {
                             require(valueFpRegister>=0)
-                    IRInstruction(Opcode.STOREM, targetDt, fpReg1 = valueFpRegister, labelSymbol = targetIdent.name)
+                            IRInstruction(Opcode.STOREM, targetDt, fpReg1 = valueFpRegister, labelSymbol = identifier!!.name)
                         }
                         else {
                             require(valueRegister>=0)
-                    IRInstruction(Opcode.STOREM, targetDt, reg1 = valueRegister, labelSymbol = targetIdent.name)
+                            IRInstruction(Opcode.STOREM, targetDt, reg1 = valueRegister, labelSymbol = identifier!!.name)
                         }
                     }
                     result += IRCodeChunk(null, null).also { it += instruction }
                     return result
                 }
-        else if(targetArray!=null) {
-            val eltSize = codeGen.program.memsizer.memorySize(targetArray.type, null)
-            val variable = targetArray.variable
-            if(variable==null)
-                translateRegularAssignPointerIndexed(result, targetArray.pointerderef!!, eltSize, targetArray, zero, targetDt, valueRegister, valueFpRegister)
-            else if(variable.type.isPointer)
-                assignToIndexedSimplePointer(result, variable, eltSize, targetArray, zero, targetDt, valueRegister, valueFpRegister)
-            else
-                translateRegularAssignArrayIndexed(result, variable.name, eltSize, targetArray, zero, targetDt, valueRegister, valueFpRegister)
-            return result
-        }
-        else if(targetMemory!=null) {
-            require(targetDt == IRDataType.BYTE) { "must be byte type ${targetMemory.position}"}
+                memory != null -> {
+                    require(targetDt == IRDataType.BYTE) { "must be byte type ${memory!!.position}"}
                     if(zero) {
-                if(targetMemory.address is PtNumber) {
-                    val chunk = IRCodeChunk(null, null).also { it += IRInstruction(Opcode.STOREZM, targetDt, address = (targetMemory.address as PtNumber).number.toInt()) }
+                        if(memory!!.address is PtNumber) {
+                            val chunk = IRCodeChunk(null, null).also { it += IRInstruction(Opcode.STOREZM, targetDt, address = (memory!!.address as PtNumber).number.toInt()) }
                             result += chunk
                         } else {
-                    val tr = expressionEval.translateExpression(targetMemory.address)
+                            val tr = expressionEval.translateExpression(memory!!.address)
                             val addressReg = tr.resultReg
                             addToResult(result, tr, tr.resultReg, -1)
                             result += IRCodeChunk(null, null).also {
@@ -549,12 +536,12 @@ internal class AssignmentGen(private val codeGen: IRCodeGen, private val express
                             }
                         }
                     } else {
-                val constAddress = targetMemory.address as? PtNumber
+                        val constAddress = memory!!.address as? PtNumber
                         if(constAddress!=null) {
                             addInstr(result, IRInstruction(Opcode.STOREM, targetDt, reg1=valueRegister, address=constAddress.number.toInt()), null)
                             return result
                         }
-                val ptrWithOffset = targetMemory.address as? PtBinaryExpression
+                        val ptrWithOffset = memory!!.address as? PtBinaryExpression
                         if(ptrWithOffset!=null) {
                             if(ptrWithOffset.operator=="+" && ptrWithOffset.left is PtIdentifier) {
                                 val constOffset = (ptrWithOffset.right as? PtNumber)?.number?.toInt()
@@ -582,7 +569,7 @@ internal class AssignmentGen(private val codeGen: IRCodeGen, private val express
                             }
                         }
 
-                val tr = expressionEval.translateExpression(targetMemory.address)
+                        val tr = expressionEval.translateExpression(memory!!.address)
                         val addressReg = tr.resultReg
                         addToResult(result, tr, tr.resultReg, -1)
                         addInstr(result, IRInstruction(Opcode.STOREI, targetDt, reg1=valueRegister, reg2=addressReg), null)
@@ -591,15 +578,30 @@ internal class AssignmentGen(private val codeGen: IRCodeGen, private val express
 
                     return result
                 }
-        else if(targetPointerDeref!=null) {
-            val (addressReg, offset) = codeGen.evaluatePointerAddressIntoReg(result, targetPointerDeref)
-            val actualValueReg = if(targetPointerDeref.type.isFloat) valueFpRegister else valueRegister
-            codeGen.storeValueAtPointersLocation(result, addressReg, offset, targetPointerDeref.type, zero, actualValueReg)
+                array != null -> {
+                    val eltSize = codeGen.program.memsizer.memorySize(array!!.type, null)
+                    val variable = array!!.variable
+                    if(variable==null)
+                        translateRegularAssignPointerIndexed(result, array!!.pointerderef!!, eltSize, array!!, zero, targetDt, valueRegister, valueFpRegister)
+                    else if(variable.type.isPointer)
+                        assignToIndexedSimplePointer(result, variable, eltSize, array!!, zero, targetDt, valueRegister, valueFpRegister)
+                    else
+                        translateRegularAssignArrayIndexed(result, variable.name, eltSize, array!!, zero, targetDt, valueRegister, valueFpRegister)
                     return result
                 }
-        else
+                pointerDeref != null -> {
+                    val (addressReg, offset) = codeGen.evaluatePointerAddressIntoReg(result, pointerDeref!!)
+                    val actualValueReg = if(pointerDeref!!.type.isFloat) valueFpRegister else valueRegister
+                    codeGen.storeValueAtPointersLocation(result, addressReg, offset, pointerDeref!!.type, zero, actualValueReg)
+                    return result
+
+                }
+                else -> {
                     throw AssemblyError("weird assigntarget")
                 }
+            }
+        }
+    }
 
     private fun assignToIndexedSimplePointer(
         result: MutableList<IRCodeChunkBase>,

codeOptimizers/src/prog8/optimizer/ConstantFoldingOptimizer.kt

@@ -85,6 +85,8 @@ class ConstantFoldingOptimizer(private val program: Program, private val errors:
      *        (X + c1) - c2  ->  X + (c1-c2)
      */
     override fun after(expr: BinaryExpression, parent: Node): Iterable<IAstModification> {
+        if(expr.operator==".")
+            return noModifications
         val modifications = mutableListOf<IAstModification>()
         val leftconst = expr.left.constValue(program)
         val rightconst = expr.right.constValue(program)

codeOptimizers/src/prog8/optimizer/ConstantIdentifierReplacer.kt

@@ -28,7 +28,6 @@ class VarConstantValueTypeAdjuster(
         if(decl.parent is AnonymousScope)
             throw FatalAstException("vardecl may no longer occur in anonymousscope ${decl.position}")
 
-        try {
         val declConstValue = decl.value?.constValue(program)
         if(declConstValue!=null && (decl.type== VarDeclType.VAR || decl.type==VarDeclType.CONST)
             && declConstValue.type != decl.datatype.base) {
@@ -37,9 +36,6 @@ class VarConstantValueTypeAdjuster(
                 errors.err("refused truncating of float to avoid loss of precision", decl.value!!.position)
             }
         }
-        } catch (x: UndefinedSymbolError) {
-            errors.err(x.message, x.position)
-        }
 
         // replace variables by constants, if possible
         if(options.optimize) {
@@ -313,7 +309,6 @@ internal class ConstantIdentifierReplacer(
         if(!dt.isKnown || !dt.isNumeric && !dt.isBool)
             return noModifications
 
-        try {
         val cval = identifier.constValue(program) ?: return noModifications
         val arrayIdx = identifier.parent as? ArrayIndexedExpression
         if(arrayIdx!=null && cval.type.isNumeric) {
@@ -345,10 +340,6 @@ internal class ConstantIdentifierReplacer(
             cval.type.isPassByRef -> throw InternalCompilerException("pass-by-reference type should not be considered a constant")
             else -> return noModifications
         }
-        } catch (x: UndefinedSymbolError) {
-            errors.err(x.message, x.position)
-            return noModifications
-        }
     }
 
     override fun after(decl: VarDecl, parent: Node): Iterable<IAstModification> {

codeOptimizers/src/prog8/optimizer/ExpressionSimplifier.kt

@@ -9,6 +9,7 @@ import prog8.ast.walk.AstWalker
 import prog8.ast.walk.IAstModification
 import prog8.code.core.*
 import kotlin.math.abs
+import kotlin.math.floor
 import kotlin.math.log2
 
 
@@ -90,6 +91,8 @@ class ExpressionSimplifier(private val program: Program, private val errors: IEr
     }
 
     override fun after(expr: BinaryExpression, parent: Node): Iterable<IAstModification> {
+        if(expr.operator==".")
+            return noModifications
         val newExpr = applyAbsorptionLaws(expr)
         if(newExpr!=null)
             return listOf(IAstModification.ReplaceNode(expr, newExpr, parent))
@@ -424,19 +427,92 @@ class ExpressionSimplifier(private val program: Program, private val errors: IEr
             }
         }
 
+//        fun isPowerOfTwo(number: Double): Boolean {
+//            val intValue = number.toInt()
+//            return intValue > 0 && (intValue and (intValue - 1)) == 0
+//        }
+
+        fun isFactorOf256(number: Double): Boolean {
+            val intValue = number.toInt()
+            return intValue >= 256 && (intValue and 0xFF) == 0
+        }
+
+        if (leftDt.isUnsignedWord && rightVal!=null) {
+            if (expr.operator == ">" && rightVal.number == 255.0 || expr.operator == ">=" && rightVal.number == 256.0) {
+                // uword > 255  -->  msb(value)!=0
+                // uword >= 256 -->  msb(value)!=0
+                expr.operator = "!="
+                expr.left = FunctionCallExpression(IdentifierReference(listOf("msb"), expr.left.position), mutableListOf(expr.left), expr.left.position)
+                expr.right = NumericLiteral(BaseDataType.UBYTE, 0.0, expr.right.position)
+                expr.linkParents(parent)
+            }
+            else if(expr.operator==">=" && isFactorOf256(rightVal.number)) {
+                // uword >= $xx00 --> msb(value)>=xx
+                val msbFraction = floor(rightVal.number / 256.0)
+                expr.operator = ">="
+                expr.left = FunctionCallExpression(IdentifierReference(listOf("msb"), expr.left.position), mutableListOf(expr.left), expr.left.position)
+                expr.right = NumericLiteral(BaseDataType.UBYTE, msbFraction, expr.right.position)
+                expr.linkParents(parent)
+            }
+            else if(expr.operator==">" && isFactorOf256(rightVal.number+1)) {
+                // uword > $xxFF --> msb(value)>xx
+                val msbFraction = floor((rightVal.number) / 256.0)
+                expr.operator = ">"
+                expr.left = FunctionCallExpression(IdentifierReference(listOf("msb"), expr.left.position), mutableListOf(expr.left), expr.left.position)
+                expr.right = NumericLiteral(BaseDataType.UBYTE, msbFraction, expr.right.position)
+                expr.linkParents(parent)
+            }
+            else if(expr.operator == "<" && rightVal.number == 256.0 || expr.operator == "<=" && rightVal.number == 255.0) {
+                // uword < 256  -->  msb(value)==0
+                // uword <= 255 -->  msb(value)==0
+                expr.operator = "=="
+                expr.left = FunctionCallExpression(IdentifierReference(listOf("msb"), expr.left.position), mutableListOf(expr.left), expr.left.position)
+                expr.right = NumericLiteral(BaseDataType.UBYTE, 0.0, expr.right.position)
+                expr.linkParents(parent)
+            }
+            else if(expr.operator == "<" && isFactorOf256(rightVal.number)) {
+                // uword < $xx00 --> msb(value)<xx
+                val msbFraction = floor(rightVal.number / 256.0)
+                expr.operator = "<"
+                expr.left = FunctionCallExpression(IdentifierReference(listOf("msb"), expr.left.position), mutableListOf(expr.left), expr.left.position)
+                expr.right = NumericLiteral(BaseDataType.UBYTE, msbFraction, expr.right.position)
+                expr.linkParents(parent)
+            }
+            else if(expr.operator=="<=" && isFactorOf256(rightVal.number+1)) {
+                // uword <= $xxFF --> msb(value)<=xx
+                val msbFraction = floor((rightVal.number) / 256.0)
+                expr.operator = "<="
+                expr.left = FunctionCallExpression(IdentifierReference(listOf("msb"), expr.left.position), mutableListOf(expr.left), expr.left.position)
+                expr.right = NumericLiteral(BaseDataType.UBYTE, msbFraction, expr.right.position)
+                expr.linkParents(parent)
+            }
+
+        }
+
         return noModifications
     }
 
     override fun after(arrayIndexedExpression: ArrayIndexedExpression, parent: Node): Iterable<IAstModification> {
         if(arrayIndexedExpression.indexer.constIndex()==0) {
             if(arrayIndexedExpression.plainarrayvar!=null) {
-                if((arrayIndexedExpression.parent as? BinaryExpression)?.operator !=".") {
+                val binexprParent = arrayIndexedExpression.parent as? BinaryExpression
+                if(binexprParent?.operator!=".") {
                     val dt = arrayIndexedExpression.plainarrayvar!!.inferType(program).getOrUndef()
                     if(dt.isPointer) {
                         // pointer[0]  -->   pointer^^
                         val deref = PtrDereference(arrayIndexedExpression.plainarrayvar!!.nameInSource, true, arrayIndexedExpression.plainarrayvar!!.position)
                         return listOf(IAstModification.ReplaceNode(arrayIndexedExpression,deref, parent))
                     }
+                } else if(arrayIndexedExpression.pointerderef==null) {
+                    // possibly     pointer[0].field   -->  pointer.field
+                    val target = arrayIndexedExpression.plainarrayvar!!.targetVarDecl()
+                    if(target?.datatype?.isPointer==true) {
+                        val field = (binexprParent.right as? IdentifierReference)?.nameInSource
+                        if(field!=null) {
+                            val deref = PtrDereference(arrayIndexedExpression.plainarrayvar!!.nameInSource + field, false, arrayIndexedExpression.plainarrayvar!!.position)
+                            return listOf(IAstModification.ReplaceNode(arrayIndexedExpression.parent, deref, arrayIndexedExpression.parent.parent))
+                        }
+                    }
                 }
             }
             val ptrDeref = arrayIndexedExpression.pointerderef

compiler/res/prog8lib/c128/syslib.p8

@@ -506,6 +506,7 @@ save_SCRATCH_ZPWORD2	.word  ?
 
 asmsub  set_irq(uword handler @AY) clobbers(A)  {
 	%asm {{
+	    php
 		sei
         sta  _vector
         sty  _vector+1
@@ -513,7 +514,7 @@ asmsub  set_irq(uword handler @AY) clobbers(A)  {
 		sta  cbm.CINV
 		lda  #>_irq_handler
 		sta  cbm.CINV+1
-		cli
+		plp
 		rts
 _irq_handler
         jsr  sys.save_prog8_internals
@@ -545,6 +546,7 @@ _vector	.word ?
 
 asmsub  restore_irq() clobbers(A) {
 	%asm {{
+	    php
 		sei
 		lda  #<cbm.IRQDFRT
 		sta  cbm.CINV
@@ -554,24 +556,34 @@ asmsub  restore_irq() clobbers(A) {
 		sta  c64.IREQMASK	; enable raster irq
 		lda  #%10000001
 		sta  c64.CIA1ICR	; restore CIA1 irq
-		cli
+		plp
 		rts
 	}}
 }
 
 asmsub  set_rasterirq(uword handler @AY, uword rasterpos @R0) clobbers(A) {
 	%asm {{
+	    php
         sei
-        sta  _vector
-        sty  _vector+1
+        sta  user_vector
+        sty  user_vector+1
+
+        lda  #%01111111
+        sta  c64.CIA1ICR    ; "switch off" interrupts signals from cia-1
+        sta  c64.CIA2ICR    ; "switch off" interrupts signals from cia-2
+        lda  c64.CIA1ICR    ; ack previous irq
+        lda  c64.CIA2ICR    ; ack previous irq
         lda  cx16.r0
         ldy  cx16.r0+1
-        jsr  _setup_raster_irq
+        jsr  sys.set_rasterline
+        lda  #%00000001
+        sta  c64.IREQMASK   ; enable raster interrupt signals from vic
+
         lda  #<_raster_irq_handler
         sta  cbm.CINV
         lda  #>_raster_irq_handler
         sta  cbm.CINV+1
-        cli
+        plp
         rts
 
 _raster_irq_handler
@@ -593,34 +605,46 @@ _raster_irq_handler
 		pla
 		rti
 _run_custom
-		jmp  (_vector)
+		jmp  (user_vector)
 		.section BSS
-_vector	.word ?
+user_vector	.word ?
 		.send BSS
 
-
-_setup_raster_irq
-		pha
-		lda  #%01111111
-		sta  c64.CIA1ICR    ; "switch off" interrupts signals from cia-1
-		sta  c64.CIA2ICR    ; "switch off" interrupts signals from cia-2
-		and  c64.SCROLY
-		sta  c64.SCROLY     ; clear most significant bit of raster position
-		lda  c64.CIA1ICR    ; ack previous irq
-		lda  c64.CIA2ICR    ; ack previous irq
-		pla
-		sta  c64.RASTER     ; set the raster line number where interrupt should occur
-		cpy  #0
-		beq  +
-		lda  c64.SCROLY
-		ora  #%10000000
-		sta  c64.SCROLY     ; set most significant bit of raster position
-+		lda  #%00000001
-		sta  c64.IREQMASK   ; enable raster interrupt signals from vic
-		rts
+        ; !notreached!
 	}}
 }
 
+    asmsub update_rasterirq(uword handler @AY, uword rasterpos @R0) clobbers(A) {
+        ; -- just update the IRQ handler and raster line position for the raster IRQ
+        ;    this is much more efficient than calling set_rasterirq() again every time.
+        ;    (but you have to call that one initially at least once to setup the prog8 handler itself)
+        %asm {{
+            php
+            sei
+            sta  sys.set_rasterirq.user_vector
+            sty  sys.set_rasterirq.user_vector+1
+            lda  cx16.r0L
+            ldy  cx16.r0H
+            jsr  sys.set_rasterline
+            plp
+            rts
+        }}
+    }
+
+    asmsub  set_rasterline(uword line @AY) {
+        ; -- only set a new raster line for the raster IRQ
+        %asm {{
+            sta  c64.RASTER     ; set the raster line number where interrupt should occur
+            lda  c64.SCROLY
+            and  #%01111111
+            cpy  #0
+            beq  +
+            ora  #%10000000
+    +       sta  c64.SCROLY     ; clear most significant bit of raster position
+            rts
+        }}
+    }
+
     asmsub  reset_system()  {
         ; Soft-reset the system back to initial power-on Basic prompt.
         %asm {{
@@ -680,6 +704,26 @@ _loop       lda  P8ZP_SCRATCH_W1
         }}
     }
 
+    asmsub waitrasterline(uword line @AY) {
+        ; -- CPU busy wait until the given raster line is reached
+        %asm {{
+            cpy  #0
+            bne  _larger
+-           cmp  c64.RASTER
+            bne  -
+            bit  c64.SCROLY
+            bmi  -
+            rts
+_larger
+            cmp  c64.RASTER
+            bne  _larger
+            bit  c64.SCROLY
+            bpl  _larger
+            rts
+        }}
+    }
+
+
     asmsub internal_stringcopy(str source @R0, str target @AY) clobbers (A,Y) {
         ; Called when the compiler wants to assign a string value to another string.
         %asm {{

compiler/res/prog8lib/c64/syslib.p8

@@ -313,12 +313,13 @@ asmsub banks(ubyte banks @A) {
     %asm {{
         and  #%00000111
         sta  P8ZP_SCRATCH_REG
+        php
         sei
         lda  $01
         and  #%11111000
         ora  P8ZP_SCRATCH_REG
         sta  $01
-        cli
+        plp
         rts
     }}
 }
@@ -511,6 +512,7 @@ save_SCRATCH_ZPWORD2	.word  ?
 
 asmsub  set_irq(uword handler @AY) clobbers(A)  {
 	%asm {{
+	    php
 	    sei
         sta  _vector
         sty  _vector+1
@@ -518,7 +520,7 @@ asmsub  set_irq(uword handler @AY) clobbers(A)  {
 		sta  cbm.CINV
 		lda  #>_irq_handler
 		sta  cbm.CINV+1
-		cli
+		plp
 		rts
 _irq_handler
         jsr  sys.save_prog8_internals
@@ -551,6 +553,7 @@ _vector	.word ?
 
 asmsub  restore_irq() clobbers(A) {
 	%asm {{
+	    php
 		sei
 		lda  #<cbm.IRQDFRT
 		sta  cbm.CINV
@@ -560,24 +563,34 @@ asmsub  restore_irq() clobbers(A) {
 		sta  c64.IREQMASK	; disable raster irq
 		lda  #%10000001
 		sta  c64.CIA1ICR	; restore CIA1 irq
-		cli
+		plp
 		rts
 	}}
 }
 
 asmsub  set_rasterirq(uword handler @AY, uword rasterpos @R0) clobbers(A) {
 	%asm {{
+	    php
 	    sei
-        sta  _vector
-        sty  _vector+1
+        sta  user_vector
+        sty  user_vector+1
+
+		lda  #%01111111
+		sta  c64.CIA1ICR    ; "switch off" interrupts signals from cia-1
+		sta  c64.CIA2ICR    ; "switch off" interrupts signals from cia-2
+		lda  c64.CIA1ICR    ; ack previous irq
+		lda  c64.CIA2ICR    ; ack previous irq
         lda  cx16.r0
         ldy  cx16.r0+1
-        jsr  _setup_raster_irq
+		jsr  sys.set_rasterline
+ 		lda  #%00000001
+		sta  c64.IREQMASK   ; enable raster interrupt signals from vic
+
         lda  #<_raster_irq_handler
         sta  cbm.CINV
         lda  #>_raster_irq_handler
         sta  cbm.CINV+1
-        cli
+        plp
         rts
 
 _raster_irq_handler
@@ -600,29 +613,42 @@ _raster_irq_handler
 		rti
 
 _run_custom
-		jmp  (_vector)
+		jmp  (user_vector)
 		.section BSS
-_vector	.word ?
+user_vector	.word ?
 		.send BSS
 
-_setup_raster_irq
-		pha
-		lda  #%01111111
-		sta  c64.CIA1ICR    ; "switch off" interrupts signals from cia-1
-		sta  c64.CIA2ICR    ; "switch off" interrupts signals from cia-2
-		and  c64.SCROLY
-		sta  c64.SCROLY     ; clear most significant bit of raster position
-		lda  c64.CIA1ICR    ; ack previous irq
-		lda  c64.CIA2ICR    ; ack previous irq
-		pla
+		; !notreached!
+	}}
+}
+
+    asmsub update_rasterirq(uword handler @AY, uword rasterpos @R0) clobbers(A) {
+        ; -- just update the IRQ handler and raster line position for the raster IRQ
+        ;    this is much more efficient than calling set_rasterirq() again every time.
+        ;    (but you have to call that one initially at least once to setup the prog8 handler itself)
+        %asm {{
+            php
+            sei
+            sta  sys.set_rasterirq.user_vector
+            sty  sys.set_rasterirq.user_vector+1
+            lda  cx16.r0L
+            ldy  cx16.r0H
+            jsr  sys.set_rasterline
+            plp
+            rts
+        }}
+    }
+
+asmsub  set_rasterline(uword line @AY) {
+    ; -- only set a new raster line for the raster IRQ
+    %asm {{
         sta  c64.RASTER     ; set the raster line number where interrupt should occur
+        lda  c64.SCROLY
+        and  #%01111111
         cpy  #0
         beq  +
-		lda  c64.SCROLY
         ora  #%10000000
-		sta  c64.SCROLY     ; set most significant bit of raster position
-+		lda  #%00000001
-		sta  c64.IREQMASK   ; enable raster interrupt signals from vic
++       sta  c64.SCROLY     ; clear most significant bit of raster position
         rts
     }}
 }
@@ -687,6 +713,26 @@ _loop       lda  P8ZP_SCRATCH_W1
         }}
     }
 
+    asmsub waitrasterline(uword line @AY) {
+        ; -- CPU busy wait until the given raster line is reached
+        %asm {{
+            cpy  #0
+            bne  _larger
+-           cmp  c64.RASTER
+            bne  -
+            bit  c64.SCROLY
+            bmi  -
+            rts
+_larger
+            cmp  c64.RASTER
+            bne  _larger
+            bit  c64.SCROLY
+            bpl  _larger
+            rts
+        }}
+    }
+
+
     asmsub internal_stringcopy(str source @R0, str target @AY) clobbers (A,Y) {
         ; Called when the compiler wants to assign a string value to another string.
         %asm {{

compiler/res/prog8lib/cx16/palette.p8

@@ -283,7 +283,7 @@ palette {
     ; NOTE: this routine requires rom version 49+
     alias set_default = cx16.set_default_palette
 
-    ; get the bank and address of the word-array containing the 256 default palette colors
+    ; get the bank and address (in A, and XY) of the word-array containing the 256 default palette colors
     ; NOTE: this routine requires rom version 49+
     alias get_default = cx16.get_default_palette
 }

compiler/res/prog8lib/cx16/sprites.p8

@@ -1,10 +1,13 @@
 ; Simple routines to control sprites.
 
 ; They're not written for high performance, but for simplicity.
-; That's why they control 1 sprite at a time. \
+; That's why they control 1 sprite at a time.
 ; The exceptions are pos_batch() and pos_batch_split(); these are quite efficient
 ; to update sprite positions of multiple sprites in one call.
 
+; HIGH PERFORMANCE sprite handling would probably have a copy of the sprite registers for each sprite instead,
+; and a unrolled loop that copies those into the VERA registers when needed.
+
 ; note: sprites z-order will be in front of all layers.
 ; note: collision mask is not supported here yet.
 ; note: "palette offset" is counted as 0-15  (vera multiplies the offset by 16 to get at the actual color index)

compiler/res/prog8lib/cx16/syslib.p8

@@ -1576,6 +1576,7 @@ sys {
 asmsub  set_irq(uword handler @AY) clobbers(A)  {
     ; Sets the handler for the VSYNC interrupt, and enable that interrupt.
 	%asm {{
+	    php
         sei
         sta  _vector
         sty  _vector+1
@@ -1585,7 +1586,7 @@ asmsub  set_irq(uword handler @AY) clobbers(A)  {
         sta  cbm.CINV+1
         lda  #1
         tsb  cx16.VERA_IEN      ; enable the vsync irq
-        cli
+        plp
         rts
 
 _irq_handler
@@ -1615,6 +1616,7 @@ _vector	.word ?
 
 asmsub  restore_irq() clobbers(A) {
 	%asm {{
+	    php
 	    sei
 	    lda  _orig_irqvec
 	    sta  cbm.CINV
@@ -1624,7 +1626,7 @@ asmsub  restore_irq() clobbers(A) {
 	    and  #%11110000     ; disable all Vera IRQs but the vsync
 	    ora  #%00000001
 	    sta  cx16.VERA_IEN
-	    cli
+	    plp
 	    rts
         .section BSS_NOCLEAR
 _orig_irqvec    .word  ?
@@ -1636,9 +1638,10 @@ _orig_irqvec    .word  ?
 asmsub  set_rasterirq(uword handler @AY, uword rasterpos @R0) clobbers(A) {
     ; Sets the handler for the LINE interrupt, and enable (only) that interrupt.
 	%asm {{
+	        php
             sei
-            sta  _vector
-            sty  _vector+1
+            sta  user_vector
+            sty  user_vector+1
             lda  cx16.r0
             ldy  cx16.r0+1
             lda  cx16.VERA_IEN
@@ -1652,7 +1655,7 @@ asmsub  set_rasterirq(uword handler @AY, uword rasterpos @R0) clobbers(A) {
             sta  cbm.CINV
             lda  #>_raster_irq_handler
             sta  cbm.CINV+1
-            cli
+            plp
             rts
 
 _raster_irq_handler
@@ -1668,15 +1671,32 @@ _raster_irq_handler
             pla
             rti
 _run_custom
-		    jmp  (_vector)
+		    jmp  (user_vector)
 		    .section BSS
-_vector	    .word ?
+user_vector	    .word ?
 		    .send BSS
 
 		    ; !notreached!
         }}
 }
 
+asmsub update_rasterirq(uword handler @AY, uword rasterpos @R0) clobbers(A) {
+    ; -- just update the IRQ handler and raster line position for the raster IRQ
+    ;    this is much more efficient than calling set_rasterirq() again every time.
+    ;    (but you have to call that one initially at least once to setup the prog8 handler itself)
+    %asm {{
+        php
+        sei
+        sta  sys.set_rasterirq.user_vector
+        sty  sys.set_rasterirq.user_vector+1
+        lda  cx16.r0L
+        ldy  cx16.r0H
+        jsr  set_rasterline
+        plp
+        rts
+    }}
+}
+
 asmsub  set_rasterline(uword line @AY) {
     %asm {{
         php
@@ -1732,13 +1752,15 @@ _loop       lda  P8ZP_SCRATCH_W1
             bne  +
             rts
 
-+           sei
++           php
+            sei
             jsr  cbm.RDTIM
-            cli
+            plp
             sta  P8ZP_SCRATCH_B1
--           sei
+-           php
+            sei
             jsr  cbm.RDTIM
-            cli
+            plp
             cmp  P8ZP_SCRATCH_B1
             beq  -
 
@@ -1759,6 +1781,26 @@ _loop       lda  P8ZP_SCRATCH_W1
         }}
     }
 
+    asmsub waitrasterline(uword line @AY) {
+        ; -- CPU busy wait until the given raster line is reached
+        %asm {{
+            cpy  #0
+            bne  _larger
+-           cmp  cx16.VERA_SCANLINE_L
+            bne  -
+            bit  cx16.VERA_IEN
+            bvs  -
+            rts
+_larger
+            cmp  cx16.VERA_SCANLINE_L
+            bne  _larger
+            bit  cx16.VERA_IEN
+            bvc  _larger
+            rts
+        }}
+    }
+
+
     asmsub internal_stringcopy(str source @R0, str target @AY) clobbers (A,Y) {
         ; Called when the compiler wants to assign a string value to another string.
         %asm {{

compiler/src/prog8/compiler/astprocessing/AstChecker.kt

@@ -1092,7 +1092,7 @@ internal class AstChecker(private val program: Program,
 
         if(decl.datatype.isStructInstance && decl.origin!=VarDeclOrigin.SUBROUTINEPARAM) {
             if(decl.type==VarDeclType.MEMORY)
-                errors.err("cannot declare memory mapped struct instances, use a pointer and memory allocation call or direct address assignment instead", decl.position)
+                errors.err("struct instances cannot be declared as memory-mapped currently, use a pointer and memory allocation call or direct address assignment instead", decl.position)
             else
                 errors.err("struct instances cannot be declared directly, use a pointer and memory allocation call or direct address assignment instead", decl.position)
         }
@@ -1952,7 +1952,10 @@ internal class AstChecker(private val program: Program,
                 errors.err("index out of bounds", arrayIndexedExpression.indexer.position)
             }
         } else if(target!=null) {
-            throw FatalAstException("target is not a variable")
+            if(target is Subroutine || target is Label )
+                errors.err("cannot array index a subroutine or label directly, use an intermediate uword pointer = &symbol instead", arrayIndexedExpression.position)
+            else
+                errors.err("indexed symbol is not a suitable variable", arrayIndexedExpression.position)
         }
 
         if(arrayIndexedExpression.pointerderef!=null) {
@@ -2157,6 +2160,8 @@ internal class AstChecker(private val program: Program,
             else
                 errors.err("unable to determine type of dereferenced pointer expression", deref.position)
         }
+
+        super.visit(deref)
     }
 
     private fun checkLongType(expression: Expression) {
@@ -2260,6 +2265,20 @@ internal class AstChecker(private val program: Program,
                 }
                 return err("invalid word array initialization value ${value.type}, expected $targetDt")
             }
+            targetDt.isPointerArray -> {
+                val arraySpecSize = arrayspec.constIndex()
+                val arraySize = value.value.size
+                val maxLength = if(targetDt.isSplitWordArray) 256 else 128
+                if(arraySpecSize!=null && arraySpecSize>0) {
+                    if(arraySpecSize>maxLength)
+                        return err("array length must be 1-$maxLength")
+                    val expectedSize = arrayspec.constIndex() ?: return err("array size specifier must be constant integer value")
+                    if (arraySize != expectedSize)
+                        return err("array size mismatch (expecting $expectedSize, got $arraySize)")
+                    return true
+                }
+                return err("invalid array size, must be 1-$maxLength")
+            }
             targetDt.isFloatArray -> {
                 // value may be either a single float, or a float arraysize
                 if(value.type istype targetDt) {
@@ -2492,10 +2511,10 @@ internal class AstChecker(private val program: Program,
     }
 
     override fun visit(deref: ArrayIndexedPtrDereference) {
-        if(deref.parent is AssignTarget)
-            errors.err("no support for assigning to a array indexed pointer target like this yet. Split the assignment statement by using an intermediate variable.", deref.position)
-        else
+        if(deref.parent !is AssignTarget)
             errors.err("no support for getting the target value of pointer array indexing like this yet. Split the expression by using an intermediate variable.", deref.position) // this may never occur anymore since more ArrayIndexedPtrDereference got rewritten
+
+        super.visit(deref)
     }
 
     override fun visit(initializer: StaticStructInitializer) {
@@ -2525,6 +2544,7 @@ internal class AstChecker(private val program: Program,
 
         super.visit(initializer)
     }
+
 }
 
 internal fun checkUnusedReturnValues(call: FunctionCallStatement, target: Statement, errors: IErrorReporter) {

compiler/src/prog8/compiler/astprocessing/AstIdentifiersChecker.kt

@@ -165,8 +165,15 @@ internal class AstIdentifiersChecker(private val errors: IErrorReporter,
         super.visit(string)
     }
 
-    override fun visit(functionCallExpr: FunctionCallExpression) =  visitFunctionCall(functionCallExpr)
-    override fun visit(functionCallStatement: FunctionCallStatement) =  visitFunctionCall(functionCallStatement)
+    override fun visit(functionCallExpr: FunctionCallExpression) {
+        visitFunctionCall(functionCallExpr)
+        super.visit(functionCallExpr)
+    }
+
+    override fun visit(functionCallStatement: FunctionCallStatement) {
+        visitFunctionCall(functionCallStatement)
+        super.visit(functionCallStatement)
+    }
 
     override fun visit(initializer: StaticStructInitializer) {
         val struct = initializer.structname.targetStructDecl()

compiler/src/prog8/compiler/astprocessing/AstPreprocessor.kt

@@ -174,6 +174,8 @@ class AstPreprocessor(val program: Program,
     }
 
     override fun after(expr: BinaryExpression, parent: Node): Iterable<IAstModification> {
+        if(expr.operator==".")
+            return noModifications
         if(expr.operator=="in") {
             val containment = ContainmentCheck(expr.left, expr.right, expr.position)
             return listOf(IAstModification.ReplaceNode(expr, containment, parent))

compiler/src/prog8/compiler/astprocessing/BeforeAsmAstChanger.kt

@@ -153,6 +153,8 @@ internal class BeforeAsmAstChanger(val program: Program, private val options: Co
     }
 
     override fun after(expr: BinaryExpression, parent: Node): Iterable<IAstModification> {
+        if(expr.operator==".")
+            return noModifications
         if (options.compTarget.name == VMTarget.NAME)
             return noModifications
 

compiler/src/prog8/compiler/astprocessing/BeforeAsmTypecastCleaner.kt

@@ -91,6 +91,8 @@ internal class BeforeAsmTypecastCleaner(val program: Program,
     }
 
     override fun after(expr: BinaryExpression, parent: Node): Iterable<IAstModification> {
+        if(expr.operator==".")
+            return noModifications
         if(expr.operator=="<<" || expr.operator==">>") {
             val shifts = expr.right.constValue(program)
             if(shifts!=null) {

compiler/src/prog8/compiler/astprocessing/CodeDesugarer.kt

@@ -662,6 +662,61 @@ _after:
     override fun after(deref: ArrayIndexedPtrDereference, parent: Node): Iterable<IAstModification> {
         // get rid of the ArrayIndexedPtrDereference AST node, replace it with other AST nodes that are equivalent
 
+        fun pokeFunc(dt: DataType): Pair<String, DataType?> {
+            return when {
+                dt.isBool -> "pokebool" to null
+                dt.isUnsignedByte -> "poke" to null
+                dt.isSignedByte -> "poke" to DataType.UBYTE
+                dt.isUnsignedWord -> "pokew" to null
+                dt.isSignedWord -> "pokew" to DataType.UWORD
+                dt.isLong -> "pokel" to null
+                dt.isFloat -> "pokef" to null
+                else -> throw FatalAstException("can only deref a numeric or boolean pointer here")
+            }
+        }
+
+        if(parent is AssignTarget) {
+            if(!deref.derefLast) {
+                val assignment = parent.parent as Assignment
+                val field = deref.chain.last()
+                val ptr = deref.chain.dropLast(1)
+                if(field.second==null && ptr.last().second!=null) {
+                    val ptrName = ptr.map { it.first }
+                    val ptrVar = deref.definingScope.lookup(ptrName) as? VarDecl
+                    if(ptrVar!=null && (ptrVar.datatype.isPointer || ptrVar.datatype.isPointerArray)) {
+                        val struct = ptrVar.datatype.subType!! as StructDecl
+                        val offsetNumber = NumericLiteral.optimalInteger(struct.offsetof(field.first, program.memsizer)!!.toInt(), deref.position)
+                        val pointerIdentifier = IdentifierReference(ptrName, deref.position)
+                        val address: Expression
+                        if(ptrVar.datatype.isPointer) {
+                            // pointer[idx].field = value       -->  pokeXXX(pointer as uword + idx*sizeof(Struct) + offsetof(Struct.field), value)
+                            val structSize = ptrVar.datatype.dereference().size(program.memsizer)
+                            val pointerAsUword = TypecastExpression(pointerIdentifier, DataType.UWORD, true, deref.position)
+                            val idx = ptr.last().second!!.indexExpr
+                            val scaledIndex = BinaryExpression(idx, "*", NumericLiteral(BaseDataType.UWORD, structSize.toDouble(), deref.position), deref.position)
+                            val structAddr = BinaryExpression(pointerAsUword, "+", scaledIndex, deref.position)
+                            address = BinaryExpression(structAddr, "+", offsetNumber, deref.position)
+                        }
+                        else {
+                            // pointerarray[idx].field = value  -->  pokeXXX(pointerarray[idx] as uword + offsetof(Struct.field), value)
+                            val index = ArrayIndexedExpression(pointerIdentifier, null, ptr.last().second!!, deref.position)
+                            val pointerAsUword = TypecastExpression(index, DataType.UWORD, true, deref.position)
+                            address = BinaryExpression(pointerAsUword, "+", offsetNumber, deref.position)
+                        }
+                        val (pokeFunc, valueCast) = pokeFunc(parent.inferType(program).getOrUndef())
+                        val value = if(valueCast==null) assignment.value else TypecastExpression(assignment.value, valueCast, true, assignment.value.position)
+                        val pokeCall = FunctionCallStatement(IdentifierReference(listOf(pokeFunc), assignment.position),
+                            mutableListOf(address, value), false, assignment.position)
+
+                        if(assignment.isAugmentable)
+                            errors.warn("in-place assignment of indexed pointer variable currently is very inefficient, maybe use a temporary pointer variable", assignment.position)
+                        return listOf(IAstModification.ReplaceNode(assignment, pokeCall, assignment.parent))
+                    }
+                }
+            }
+        }
+
+
         if(deref.chain.last().second!=null && deref.derefLast && deref.chain.dropLast(1).all { it.second==null } ) {
 
             // parent could be Assigment directly, or a binexpr chained pointer expression (with '.' operator)
@@ -714,15 +769,7 @@ _after:
                 val dt = deref.inferType(program).getOrUndef()
                 if(dt.isNumericOrBool) {
                     // if it's something else beside number (like, a struct instance) we don't support rewriting that...
-                    val (pokeFunc, cast) =
-                        if (dt.isBool) "pokebool" to null
-                        else if (dt.isUnsignedByte) "poke" to null
-                        else if (dt.isSignedByte) "poke" to DataType.UBYTE
-                        else if (dt.isUnsignedWord) "pokew" to null
-                        else if (dt.isSignedWord) "pokew" to DataType.UWORD
-                        else if (dt.isLong) "pokel" to null
-                        else if (dt.isFloat) "pokef" to null
-                        else throw FatalAstException("can only deref a numeric or boolean pointer here")
+                    val (pokeFunc, cast) = pokeFunc(dt)
                     val indexer = deref.chain.last().second!!
                     val identifier = IdentifierReference(deref.chain.map { it.first }, deref.position)
                     val indexed = ArrayIndexedExpression(identifier, null, indexer, deref.position)

compiler/src/prog8/compiler/astprocessing/SimplifiedAstMaker.kt

@@ -270,7 +270,7 @@ class SimplifiedAstMaker(private val program: Program, private val errors: IErro
             srcTarget.arrayindexed!=null -> target.add(transform(srcTarget.arrayindexed!!))
             srcTarget.memoryAddress!=null -> target.add(transform(srcTarget.memoryAddress!!))
             srcTarget.pointerDereference!=null -> target.add(transform(srcTarget.pointerDereference!!))
-            srcTarget.arrayIndexedDereference!=null -> TODO("this array indexed dereference should have been converted to some other ast nodes ${srcTarget.position}")
+            srcTarget.arrayIndexedDereference!=null -> throw FatalAstException("this should have been converted to some other ast nodes ${srcTarget.position}")
             !srcTarget.void -> throw FatalAstException("invalid AssignTarget")
         }
         return target

compiler/src/prog8/compiler/astprocessing/StatementReorderer.kt

@@ -384,4 +384,25 @@ internal class StatementReorderer(
         )
         return listOf(IAstModification.ReplaceNode(assign, strcopy, assign.parent))
     }
+
+    override fun after(deref: ArrayIndexedPtrDereference, parent: Node): Iterable<IAstModification> {
+        if(parent is AssignTarget) {
+            val zeroIndexer = deref.chain.firstOrNull { it.second?.constIndex()==0 }
+            if(zeroIndexer!=null) {
+                val target = deref.definingScope.lookup(listOf(zeroIndexer.first))
+                if(target is VarDecl && target.datatype.isPointer) {
+                    val position = deref.chain.indexOf(zeroIndexer)
+                    val rest = deref.chain.drop(position + 1)
+                    if (rest.size == 1 && rest[0].second == null) {
+                        // pointer[0]^^.field = xxx  -->  pointer^^.field = xxx
+                        val noindexer = zeroIndexer.first to null
+                        val newchain = deref.chain.take(position) + noindexer + rest
+                        val newDeref = PtrDereference(newchain.map { it.first }, false, deref.position)
+                        return listOf(IAstModification.ReplaceNode(deref, newDeref, parent))
+                    }
+                }
+            }
+        }
+        return noModifications
+    }
 }

compiler/src/prog8/compiler/astprocessing/TypecastsAdder.kt

@@ -79,6 +79,9 @@ class TypecastsAdder(val program: Program, val options: CompilationOptions, val
     }
 
     override fun after(expr: BinaryExpression, parent: Node): Iterable<IAstModification> {
+        if(expr.operator==".")
+            return noModifications
+
         val leftDt = expr.left.inferType(program)
         val rightDt = expr.right.inferType(program)
         val leftCv = expr.left.constValue(program)

compiler/test/TestCompilerOnExamples.kt

@@ -97,6 +97,7 @@ class TestCompilerOnExamplesC64: FunSpec({
             "library/main",
             "plasma",
             "rasterbars",
+            "simplemultiplexer",
             "sprites",
             "starfield",
             "tehtriz",
@@ -149,6 +150,7 @@ class TestCompilerOnExamplesCx16: FunSpec({
             "bdmusic",
             "bobs",
             "bubbleuniverse",
+            "charfade",
             "charsets",
             "circles",
             "cobramk3-gfx",

compiler/test/TestPointers.kt

@@ -3,7 +3,6 @@ package prog8tests.compiler
 import io.kotest.assertions.withClue
 import io.kotest.core.spec.style.FunSpec
 import io.kotest.engine.spec.tempdir
-import io.kotest.matchers.comparables.shouldBeGreaterThanOrEqualTo
 import io.kotest.matchers.shouldBe
 import io.kotest.matchers.shouldNotBe
 import io.kotest.matchers.string.shouldContain
@@ -1027,118 +1026,6 @@ main {
         st[8] shouldBe instanceOf<Assignment>()
     }
 
-    test("indexing pointers with index 0 is just a direct pointer dereference except when followed by a struct field lookup") {
-        val src="""
-%import floats
-main {
-    struct List {
-        ^^uword s
-        ubyte n
-    }
-    sub start() {
-        ^^List l1 = ^^List : []
-        ^^word @shared wptr
-        ^^float @shared fptr
-        float f1,f2
-
-        cx16.r0 = l1.s^^
-        cx16.r1 = l1^^.s^^
-        cx16.r2 = l1.s^^
-        cx16.r3 = l1.s[0]
-        cx16.r4 = l1^^.s[0]
-
-        l1.s^^ = 4242
-        l1^^.s^^ = 4242
-        l1.s^^ = 4242
-        l1.s[0] = 4242
-        ;; l1^^.s[0] = 4242        ; TODO fix parse syntax error
-
-        cx16.r0s = wptr[0]
-        cx16.r1s = wptr^^
-        wptr^^ = 4242
-        wptr[0] = 4242
-
-        f1 = fptr^^
-        f2 = fptr[0]
-        fptr^^ = 1.234
-        fptr[0] = 1.234
-        
-        ; not changed to dereference:
-        cx16.r0L = l1[0].n
-        cx16.r1L = l1[1].n
-    }
-}"""
-
-        val result = compileText(VMTarget(), true, src, outputDir, writeAssembly = false)!!
-        val st = result.compilerAst.entrypoint.statements
-        st.size shouldBe 30
-        val dr0 = (st[10] as Assignment).value as PtrDereference
-        val dr1 = (st[11] as Assignment).value as PtrDereference
-        val dr2 = (st[12] as Assignment).value as PtrDereference
-        val dr3 = (st[13] as Assignment).value as PtrDereference
-        val dr4 = (st[14] as Assignment).value as PtrDereference
-
-        val dr5 = (st[15] as Assignment).target.pointerDereference!!
-        val dr6 = (st[16] as Assignment).target.pointerDereference!!
-        val dr7 = (st[17] as Assignment).target.pointerDereference!!
-        val dr8 = (st[18] as Assignment).target.pointerDereference!!
-
-        val dr9 = (st[19] as Assignment).value as FunctionCallExpression
-        val dr10 = (st[20] as Assignment).value as PtrDereference
-        val dr11 = (st[21] as Assignment).target.pointerDereference!!
-        (st[22] as FunctionCallStatement).target.nameInSource shouldBe listOf("pokew")
-
-        val dr13 = (st[23] as Assignment).value as PtrDereference
-        ((st[24] as Assignment).value as FunctionCallExpression).target.nameInSource shouldBe listOf("peekf")
-        val dr15 = (st[25] as Assignment).target.pointerDereference!!
-        (st[26] as FunctionCallStatement).target.nameInSource shouldBe listOf("pokef")
-
-        dr0.chain shouldBe listOf("l1", "s")
-        dr0.derefLast shouldBe true
-        dr1.chain shouldBe listOf("l1", "s")
-        dr1.derefLast shouldBe true
-        dr2.chain shouldBe listOf("l1", "s")
-        dr2.derefLast shouldBe true
-        dr3.chain shouldBe listOf("l1", "s")
-        dr3.derefLast shouldBe true
-        dr4.chain shouldBe listOf("l1", "s")
-        dr4.derefLast shouldBe true
-
-        dr5.chain shouldBe listOf("l1", "s")
-        dr5.derefLast shouldBe true
-        dr6.chain shouldBe listOf("l1", "s")
-        dr6.derefLast shouldBe true
-        dr7.chain shouldBe listOf("l1", "s")
-        dr7.derefLast shouldBe true
-        dr8.chain shouldBe listOf("l1", "s")
-        dr8.derefLast shouldBe true
-
-        dr9.target.nameInSource shouldBe listOf("peekw")
-        dr10.chain shouldBe listOf("wptr")
-        dr10.derefLast shouldBe true
-        dr11.chain shouldBe listOf("wptr")
-        dr11.derefLast shouldBe true
-
-        dr13.chain shouldBe listOf("fptr")
-        dr13.derefLast shouldBe true
-        dr15.chain shouldBe listOf("fptr")
-        dr15.derefLast shouldBe true
-
-        val list0 = (st[27] as Assignment).value as BinaryExpression
-        val list1 = (st[28] as Assignment).value as BinaryExpression
-
-        list0.operator shouldBe "."
-        (list0.right as IdentifierReference).nameInSource shouldBe listOf("n")
-        val list0left = list0.left as ArrayIndexedExpression
-        list0left.plainarrayvar!!.nameInSource shouldBe listOf("l1")
-        list0left.indexer.constIndex() shouldBe 0
-        list1.operator shouldBe "."
-        (list1.right as IdentifierReference).nameInSource shouldBe listOf("n")
-        val list1left = list0.left as ArrayIndexedExpression
-        list1left.plainarrayvar!!.nameInSource shouldBe listOf("l1")
-        list1left.indexer.constIndex() shouldBe 0
-    }
-
     test("indexing pointers to structs") {
         val src="""
 %import floats
@@ -1458,33 +1345,32 @@ other {
         compileText(Cx16Target(), false, src, outputDir) shouldNotBe null
     }
 
-    test("a.b.c[i]^^.value = X where pointer is struct gives good error message") {
+    test("support for assigning to indexed pointers") {
         val src="""
 main {
+
     sub start() {
-        other.foo.listarray[3]^^.value = cx16.r0
-        other.foo()
-    }
-}
+        sprptr[2]^^.y = 99
+        pokew(sprptr as uword + (sizeof(Sprite) as uword)*2 + offsetof(Sprite.y), 99)
+        sprptr[cx16.r0L]^^.y = 99
+        pokew(sprptr as uword + (sizeof(Sprite) as uword)*cx16.r0L + offsetof(Sprite.y), 99)
 
-other {
-    sub foo() {
-        struct List {
-            bool b
-            uword value
+        sprites[2]^^.y = 99
+        pokew(sprites[2] as uword + offsetof(Sprite.y), 99)
+        sprites[cx16.r0L]^^.y = 99
+        pokew(sprites[cx16.r0L] as uword + offsetof(Sprite.y), 99)
     }
 
-        ^^List[10] listarray
-        listarray[3]^^.value = cx16.r0
-        listarray[3]^^ = 999        ; cannot assign word value to struct instance
+    struct Sprite {
+        ubyte x
+        uword y
     }
+
+    ^^Sprite[4] @shared sprites
+    ^^Sprite @shared sprptr
 }"""
-        val errors = ErrorReporterForTests()
-        compileText(VMTarget(), false, src, outputDir, errors=errors) shouldBe null
-        errors.errors.size shouldBeGreaterThanOrEqualTo 3
-        errors.errors[0] shouldContain "no support for"
-        errors.errors[1] shouldContain "no support for"
-        errors.errors[2] shouldContain "assigning this value to struct instance not supported"
+        compileText(VMTarget(), false, src, outputDir) shouldNotBe null
+        compileText(C64Target(), false, src, outputDir) shouldNotBe null
     }
 
     xtest("array indexed assignment parses with and without explicit dereference after struct pointer [IGNORED because it's a parser error right now]") {
@@ -2552,4 +2438,73 @@ structdefs {
         a5t.pointerDereference!!.chain shouldBe listOf("structdefs", "element", "value2")
         a6t.pointerDereference!!.chain shouldBe listOf("structdefs", "element", "value2")
     }
+
+    test("assigning field with same name should not confuse compiler") {
+        val src="""
+main {
+
+    sub start() {
+        ubyte @shared ok = sprites[2].y    ; this one is fine...
+        ubyte @shared y = sprites[2].y     ; this used to crash
+    }
+
+    struct Sprite {
+        uword x
+        ubyte y
+    }
+
+    ^^Sprite[4] @shared sprites
+}"""
+        compileText(VMTarget(), false, src, outputDir, writeAssembly = false) shouldNotBe null
+    }
+
+    test("0-indexed optimizations") {
+        val src="""
+main {
+    struct Sprite {
+        uword x
+        ubyte y
+    }
+
+    ^^Sprite[4] @shared sprites
+    ^^Sprite @shared sprptr
+
+    sub start() {
+        sprptr.y = 99
+        sprptr[0]^^.y = 99
+        ;; sprites[0]^^.y = 99     ; no change here.    TODO: this syntax doesn't compile yet...
+        cx16.r0 = &sprptr[0]
+
+        cx16.r2L = sprptr.y
+        cx16.r0L = sprptr[0].y
+        cx16.r1L = sprites[0].y     ; no change here, need first array element
+        cx16.r0 = sprites[0]        ; no change here, need first array element
+        cx16.r0 = sprites[0]        ; no change here, need first array element
+    }
+}"""
+        val result = compileText(VMTarget(), true, src, outputDir, writeAssembly = false)!!
+        val st = result.compilerAst.entrypoint.statements
+        st.size shouldBe 8
+        val a1 = st[0] as Assignment
+        val a2 = st[1] as Assignment
+        val a3 = st[2] as Assignment
+        val a4 = st[3] as Assignment
+        val a5 = st[4] as Assignment
+        val a6 = st[5] as Assignment
+        val a7 = st[6] as Assignment
+
+        a1.target.arrayIndexedDereference shouldBe null
+        a1.target.pointerDereference!!.chain shouldBe listOf("sprptr", "y")
+        a2.target.arrayIndexedDereference shouldBe null
+        a2.target.pointerDereference!!.chain shouldBe listOf("sprptr", "y")
+
+        (a3.value as? AddressOf)?.identifier?.nameInSource shouldBe listOf("sprptr")
+        (a4.value as? PtrDereference)?.chain shouldBe listOf("sprptr", "y")
+        (a5.value as? PtrDereference)?.chain shouldBe listOf("sprptr", "y")
+        val be6 = a6.value as BinaryExpression      // this one is an actual array and we need the first element so no change here
+        be6.operator shouldBe "."
+        be6.left shouldBe instanceOf<ArrayIndexedExpression>()
+        be6.right shouldBe instanceOf<IdentifierReference>()
+        (a7.value as? ArrayIndexedExpression)?.indexer?.constIndex() shouldBe 0
+    }
 })

compiler/test/TestSubroutines.kt

@@ -240,6 +240,33 @@ main {
         errors.errors[1] shouldContain "9:25: invalid number of arguments"
     }
 
+    test("invalid number of args check on normal subroutine (in expression)") {
+        val text="""
+            main {
+              sub thing(ubyte a1, ubyte a2) -> ubyte {
+                return 42
+              }
+            
+              sub wrapper(ubyte value) {
+                value++
+              }
+
+              sub start() {
+                  wrapper(thing(1))
+                  wrapper(thing(1,2))
+                  wrapper(thing(1,2,3))
+              }
+            }
+        """
+
+        val errors = ErrorReporterForTests()
+        compileText(C64Target(), false, text, outputDir, writeAssembly = false, errors=errors) shouldBe null
+        errors.errors.size shouldBe 2
+        errors.errors[0] shouldContain "12:33: invalid number of arguments"
+        errors.errors[1] shouldContain "14:33: invalid number of arguments"
+    }
+
+
     test("invalid number of args check on asm subroutine") {
         val text="""
             main {
@@ -261,6 +288,35 @@ main {
         errors.errors[1] shouldContain "9:25: invalid number of arguments"
     }
 
+    test("invalid number of args check on asm subroutine (in expression)") {
+        val text="""
+            main {
+              asmsub thing(ubyte a1 @A, ubyte a2 @Y) -> ubyte @A {
+                %asm {{
+                  rts
+                }}
+              }
+            
+              sub wrapper(ubyte value) {
+                value++
+              }
+
+              sub start() {
+                  wrapper(thing(1))
+                  wrapper(thing(1,2))
+                  wrapper(thing(1,2,3))
+              }
+            }
+        """
+
+        val errors = ErrorReporterForTests()
+        compileText(C64Target(), false, text, outputDir, writeAssembly = false, errors=errors) shouldBe null
+        errors.errors.size shouldBe 2
+        errors.errors[0] shouldContain "14:33: invalid number of arguments"
+        errors.errors[1] shouldContain "16:33: invalid number of arguments"
+    }
+
+
     test("invalid number of args check on call to label and builtin func") {
         val text="""
             main {

compiler/test/ast/TestVariousCompilerAst.kt

@@ -1173,5 +1173,30 @@ main {
         compileText(VMTarget(), optimize=false, src, outputDir, writeAssembly=false) shouldNotBe null
     }
 
+    test("breakpoint after expression") {
+        val src="""
+main {
+    ubyte a,b
+
+    sub start() {
+        func1()
+        func2()
+    }
+
+    sub func1() {
+        a = b
+        %breakpoint     ; parse error
+    }
+
+    sub func2() {
+        a = b
+        %breakpoint!     ; parse ok
+    }
+}"""
+        val errors = ErrorReporterForTests()
+        compileText(VMTarget(), optimize=false, src, outputDir, errors=errors) shouldBe null
+        errors.errors.size shouldBe 3
+        errors.errors[1] shouldContain "12:10: undefined symbol: breakpoint"
+    }
 })
 

compilerAst/src/prog8/ast/AstToSourceTextConverter.kt

@@ -571,7 +571,18 @@ class AstToSourceTextConverter(val output: (text: String) -> Unit, val program:
     }
 
     override fun visit(deref: ArrayIndexedPtrDereference) {
-        output("???? Array Indexed Ptr Dereference should have been converted to other AST nodes ????")
+        for(c in deref.chain) {
+            output(c.first)
+            c.second?.let {
+                output("[")
+                it.accept(this)
+                output("]")
+            }
+            if(c !== deref.chain.last())
+                output("^^.")
+        }
+        if(deref.derefLast)
+            output("^^")
     }
 
     override fun visit(initializer: StaticStructInitializer) {

compilerAst/src/prog8/ast/AstToplevel.kt

@@ -427,6 +427,7 @@ fun defaultZero(dt: BaseDataType, position: Position) = when(dt) {
     BaseDataType.BYTE -> NumericLiteral(BaseDataType.BYTE, 0.0,  position)
     BaseDataType.UWORD, BaseDataType.STR -> NumericLiteral(BaseDataType.UWORD, 0.0, position)
     BaseDataType.WORD -> NumericLiteral(BaseDataType.WORD, 0.0, position)
+    BaseDataType.LONG -> NumericLiteral(BaseDataType.LONG, 0.0, position)
     BaseDataType.FLOAT -> NumericLiteral(BaseDataType.FLOAT, 0.0, position)
     BaseDataType.POINTER -> NumericLiteral(BaseDataType.UWORD, 0.0, position)
     else -> throw FatalAstException("can only determine default zero value for a numeric type")

compilerAst/src/prog8/ast/Errors.kt

@@ -1,6 +1,5 @@
 package prog8.ast
 
-import prog8.ast.expressions.IdentifierReference
 import prog8.code.core.Position
 
 /**
@@ -19,6 +18,3 @@ open class SyntaxError(override var message: String, val position: Position) : A
 class ExpressionError(message: String, val position: Position) : AstException(message) {
     override fun toString() = "${position.toClickableStr()} Error: $message"
 }
-
-class UndefinedSymbolError(symbol: IdentifierReference)
-    : SyntaxError("undefined symbol: ${symbol.nameInSource.joinToString(".")}", symbol.position)

compilerAst/src/prog8/ast/expressions/AstExpressions.kt

@@ -179,7 +179,12 @@ class BinaryExpression(
     override fun accept(visitor: IAstVisitor) = visitor.visit(this)
     override fun accept(visitor: AstWalker, parent: Node)= visitor.visit(this, parent)
 
-    override fun referencesIdentifier(nameInSource: List<String>) = left.referencesIdentifier(nameInSource) || right.referencesIdentifier(nameInSource)
+    override fun referencesIdentifier(nameInSource: List<String>): Boolean {
+        return if(operator==".")
+            left.referencesIdentifier(nameInSource)
+        else
+            left.referencesIdentifier(nameInSource) || right.referencesIdentifier(nameInSource)
+    }
     override fun inferType(program: Program): InferredTypes.InferredType {
 
         val leftDt = left.inferType(program)
@@ -1804,6 +1809,35 @@ class ArrayIndexedPtrDereference(
             }
 
         }
+
+
+        if(parent !is BinaryExpression || (parent as BinaryExpression).operator!=".") {
+            val indexPosition = this.chain.indexOfFirst { it.second!=null }
+            val identifier = this.chain.take(indexPosition+1).map {it.first}
+            val target = this.definingScope.lookup(identifier)
+            if(target is VarDecl && (target.datatype.isPointer || target.datatype.isPointerArray)) {
+                val sub = target.datatype.subType
+                if(sub!=null) {
+                    val field = this.chain.drop(indexPosition+1).singleOrNull()
+                    if(field==null) {
+                        return InferredTypes.knownFor(DataType.structInstance(sub))
+                    } else {
+                        val fieldDt= sub.getFieldType(field.first)!!
+                        return if(derefLast)
+                            InferredTypes.knownFor(fieldDt.dereference())
+                        else
+                            InferredTypes.knownFor(fieldDt)
+                    }
+                } else {
+                    require(indexPosition==this.chain.size-1) {"when indexing primitive pointer type there cannot be a field after it"}
+                    return if(derefLast)
+                        InferredTypes.knownFor(target.datatype.dereference())
+                    else
+                        InferredTypes.knownFor(target.datatype)
+                }
+            }
+        }
+
         // too hard to determine the type....?
         return InferredTypes.unknown()
     }

docs/source/comparing.rst

@@ -80,9 +80,9 @@ Subroutines
 Pointers and Structs
 --------------------
 
-Legacy 'untyped' pointers:
+*Legacy 'untyped' pointers:*
 
-- In Prog8 versions before 12.0 there was no support for typed pointers, only 'untyped' ones:
+- In Prog8 versions **before 12.0** there was no support for typed pointers, only 'untyped' ones:
   Variables of the ``uword`` datatype can be used as a pointer to one of the possible 65536 memory locations,
   so the value it points to is always a single byte. This is similar to ``uint8_t*`` from C.
   You have to deal with the uword manually if the object it points to is something different.
@@ -92,9 +92,9 @@ Legacy 'untyped' pointers:
 - Pointers don't have to be a variable, you can immediately access the value of a given memory location using ``@($d020)`` for instance.
   Reading is done by assigning it to a variable, writing is done by just assigning the new value to it.
 
-Typed pointers and structs:
+*Typed pointers and structs:*
 
-- Since version 12, prog8 supports struct types and typed pointers.
+- Since **version 12.0**, prog8 supports struct types and typed pointers.
 - Structs are a grouping of one or more fields, that together make up the struct type.
 - Typed pointers are just that: a pointer to a specific type (which can be a simple type such as float, or a struct type.)
 

docs/source/programming.rst

@@ -340,10 +340,20 @@ Directives
         }
 
 
-.. data:: %breakpoint
+.. data:: %breakpoint!  or  %breakpoint
 
     Level: not at module scope.
     Defines a debugging breakpoint at this location. See :ref:`debugging`
+    The version with the explamation point '!' at the end can be used even
+    if the breakpoint follows an expression. If you don't use the '!' version in this case
+    the compiler may think it is just a term in the expression (modulo operator and breakpoint operand value),
+    instead of a breakpoint directive::
+
+        a = b
+        %breakpoint       ; parse error because it thinks it is part of the previous line
+
+        a = b
+        %breakpoint!      ; parsed correctly as directive
 
 
 .. data:: %encoding <encodingname>

docs/source/structpointers.rst

@@ -15,23 +15,24 @@ Structs and Pointers
     priority over other variables to be placed into zeropage.
 
 .. note::
-    Due to some limitations in the language parser, not all pointer related syntax is currently supported
-    if it is a pointer to a struct type.
+    Due to a few limitations in the language parser, some pointer related syntax is currently unsupported.
     The compiler tries its best to give a descriptive error message but sometimes there is still a
-    parser limitation that has to be worked around at the moment. For example, this pointer arithmetic
-    indexing syntax is not supported right now and will result in a parse error::
+    parser limitation that has to be worked around at the moment. For example, this assigment syntax doesn't parse correctly::
 
         ^^Node  np
-        np[2].field = 9999
+        np[2].field = 9999          ; cannot use this syntax as assignment target right now
+        ubyte value = np[2].field   ; note that using it as expression value works fine
 
-    To work around this (and similar) cases you'll have to break up the expression in multiple steps,
-    in this case something like::
+    To work around this you'll have to explicitly write the pointer dereferencing operator,
+    or break up the expression in multiple steps (which can be beneficial too when you are assigning multiple fields
+    because it will save a pointer calculation for every assignment)::
 
-        ^^Node thirdnode = &&np[2]
-        thirdnode.field = 9999
-
-    *Note: this example is not regular array indexing syntax, it's pointer arithmetic by indexing on the pointer itself. Regular array syntax is supported just fine for arrays containing pointers etc.*
+        ^^Node  np
+        np[2]^^.field = 9999
 
+        ; alternatively, split up:
+        ^^Node thenode = &&np[2]
+        thenode.field = 9999
 
 
 Legacy untyped pointers (uword)
@@ -132,7 +133,7 @@ dealing with all of them separately.  You first define the struct type like so::
 You can use boolean fields, numeric fields (byte, word, float), and pointer fields (including str, which is translated into ^^ubyte).
 You cannot nest struct types nor put arrays in them as a field.
 Fields in a struct are 'packed' (meaning the values are placed back-to-back in memory), and placed in memory in order of declaration. This guarantees exact size and place of the fields.
-``sizeof()`` knows how to calculate the size of a struct.
+``sizeof()`` knows how to calculate the combined size of a struct, and ``offsetof()`` can be used to get the byte offset of a given field in the struct.
 The size of a struct cannot exceed 1 memory page (256 bytes).
 
 You can copy the whole contents of a struct to another one by assigning the dereferenced pointers::
@@ -154,9 +155,11 @@ Because it implies pointer dereferencing you can usually omit the explicit `^^`,
 
 
 .. note::
-    Structs are only supported as a *reference type* (via a pointer). It is currently not possible to use them as a value type.
+    Structs are currently only supported as a *reference type* (they always have to be accessed through a pointer).
+    It is not yet possible to use them as a value type, or as memory-mapped types.
     This means you cannot create an array of structs either - only arrays of pointers to structs.
-    There are a couple of simple case where the compiler allows assignment of struct instances. You are allowed to write::
+    There are a couple of simple case where the compiler does allow assignment of struct instances though, and it will
+    automatically copy all the fields for you. You are allowed to write::
 
         ptr2^^ = ptr1^^
         ptr2^^ = ptr1[2]
@@ -166,14 +169,14 @@ Because it implies pointer dereferencing you can usually omit the explicit `^^`,
     In the future more cases may be supported.
 
 .. note::
-    Using structs instead of plain arrays may result in less efficent code being generated.
+    Using structs instead of plain arrays usually results in more and less efficent code being generated.
     This is because the 6502 CPU is not particularly well equipped to dealing with pointers and accessing struct fields via offsets,
     as compared to direct variable access or array indexing. The prog8 program code may be easier to work with though!
 
 .. note::
     Accessing the first field in a struct is more efficient than subsequent fields, because it
-    is at offset 0 so no additional addition has to be done on a pointer to reach the first field.
-    Try to put the most often accessed field as the first field to gain a rather substantial code size and efficiency boost.
+    is at offset 0 so no additional addition has to be computed on a pointer to reach the first field.
+    Try to put the most often accessed field as the first field to potentially gain a rather substantial boost in code efficiency.
 
 
 Static initialization of structs

docs/source/targetsystem.rst

@@ -196,6 +196,8 @@ These routines are::
 
     sys.set_irq(uword handler_address)
     sys.set_rasterirq(uword handler_address, uword rasterline)
+    sys.update_rasterirq(uword handler_address, uword rasterline)
+    sys.set_rasterline(uword rasterline)
     sys.restore_irq()     ; set everything back to the systems default irq handler
 
 The IRQ handler routine must return a boolean value (0 or 1) in the A register:

docs/source/todo.rst

@@ -1,6 +1,101 @@
 TODO
 ====
 
+
+STRUCTS and TYPED POINTERS
+--------------------------
+
+- implement the remaining TODO's in PointerAssignmentsGen.
+- optimize deref in PointerAssignmentsGen: optimize 'forceTemporary' to only use a temporary when the offset is >0
+- optimize the float copying in assignIndexedPointer() (also word?)
+- optimize augmented assignments to indexed pointer targets like sprptr[2]^^.y++  (these are now not performend in-place but as a regular assignment)
+- implement even more struct instance assignments (via memcopy) in CodeDesugarer (see the TODO) (add to documentation as well, paragraph 'Structs')
+- support @nosplit pointer arrays?
+- support pointer to pointer?
+- support for typed function pointers?  (&routine could be typed by default as well then)
+- really fixing the pointer dereferencing issues (cursed hybrid beween IdentifierReference, PtrDereferece and PtrIndexedDereference) may require getting rid of scoped identifiers altogether and treat '.' as a "scope or pointer following operator"
+- (later, nasty parser problem:) support chaining pointer dereference on function calls that return a pointer.  (type checking now fails on stuff like func().field and func().next.field)
+
+
+Future Things and Ideas
+^^^^^^^^^^^^^^^^^^^^^^^
+
+- improve ANTLR grammar with better error handling (as suggested by Qwen AI)
+- allow memory() to occur in array initializer
+- when a complete block is removed because unused, suppress all info messages about everything in the block being removed
+- fix the line, cols in Position, sometimes they count from 0 sometimes from 1
+- is "checkAssignmentCompatible" redundant (gets called just 1 time!) when we also have "checkValueTypeAndRange" ?
+- enums?
+- romable: should we have a way to explicitly set the memory address for the BSS area (add a -varsaddress and -slabsaddress options?)
+- romable: fix remaining codegens (some for loops, see ForLoopsAsmGen)
+- Kotlin: can we use inline value classes in certain spots? (domain types instead of primitives)
+- add float support to the configurable compiler targets
+- Improve the SublimeText syntax file for prog8, you can also install this for 'bat': https://github.com/sharkdp/bat?tab=readme-ov-file#adding-new-syntaxes--language-definitions
+- Change scoping rules for qualified symbols so that they don't always start from the root but behave like other programming languages (look in local scope first), maybe only when qualified symbol starts with '.' such as: .local.value = 33
+- something to reduce the need to use fully qualified names all the time. 'with' ?  Or 'using <prefix>'?
+- Improve register load order in subroutine call args assignments:
+  in certain situations (need examples!), the "wrong" order of evaluation of function call arguments is done which results
+  in overwriting registers that already got their value, which requires a lot of stack juggling (especially on plain 6502 cpu!)
+  Maybe this routine can be made more intelligent.  See usesOtherRegistersWhileEvaluating() and argumentsViaRegisters().
+- Does it make codegen easier if everything is an expression?  Start with the PtProgram ast classes, change statements to expressions that have (new) VOID data type
+- Can we support signed % (remainder) somehow?
+- Multidimensional arrays and chained indexing, purely as syntactic sugar over regular arrays. Probaby only useful once we have typed pointers. (addressed in 'struct' branch)
+- make a form of "manual generics" possible like: varsub routine(T arg)->T  where T is expanded to a specific type
+  (this is already done hardcoded for several of the builtin functions)
+- [much work:] more support for (64tass) SEGMENTS in the prog8 syntax itself?
+- ability to use a sub instead of only a var for @bank ? what for though? dynamic bank/overlay loading?
+- Zig-like try-based error handling where the V flag could indicate error condition? and/or BRK to jump into monitor on failure? (has to set BRK vector for that) But the V flag is also set on certain normal instructions
+
+
+IR/VM
+-----
+- make MSIG instruction set flags and skip cmp #0 afterwards   (if msb(x)>0)
+- is it possible to use LOADFIELD/STOREFIELD instructions even more?
+- make multiple classes of registers and maybe also categorize by life time , to prepare for better register allocation in the future
+    SYSCALL_ARGS,        // Reserved for syscall arguments (r99000-99099, r99100-99199)
+    FUNCTION_PARAMS,     // For passing function parameters
+    FUNCTION_RETURNS,    // For function return values
+    TEMPORARY,           // Short-lived temporary values
+    LOCAL_VARIABLES,     // Local variables within functions
+    GLOBAL_VARIABLES,    // Global/static variables
+    HARDWARE_MAPPED,     // Mapped to CPU hardware registers
+    LOOP_INDICES,        // Used as loop counters
+    ADDRESS_CALCULATION  // Used for pointer arithmetic
+  Categorizing registers by lifetime can significantly improve allocation:
+   - Short-lived: Temporary registers used in expressions
+   - Medium-lived: Local variables within a function
+  Registers could be categorized by how frequently they're accessed:
+   - Hot Registers: Frequently accessed (should be allocated to faster physical registers)
+   - Warm Registers: Moderately accessed
+   - Cold Registers: Rarely accessed (can be spilled to memory if needed)
+  We already have type-based pools
+    - byte, word, float registers
+
+- pointer dt's are all reduced to just an uword (in the irTypeString method) - is this okay or could it be beneficial to reintroduce the actual pointer type information? See commit 88b074c208450c58aa32469745afa03e4c5f564a
+- change the instruction format so an indirect register (a pointer) can be used more often, at least for the inplace assignment operators that operate on pointer
+- getting it in shape for code generation...: the IR file should be able to encode every detail about a prog8 program (the VM doesn't have to actually be able to run all of it though!)
+- fix call() return value handling (... what's wrong with it again?)
+- encode asmsub/extsub clobber info in the call , or maybe include these definitions in the p8ir file itself too.  (return registers are already encoded in the CALL instruction)
+- proper code gen for the CALLI instruction and that it (optionally) returns a word value that needs to be assigned to a reg
+- implement fast code paths for TODO("inplace split....
+- implement more TODOs in AssignmentGen
+- do something with the 'split' tag on split word arrays
+- add more optimizations in IRPeepholeOptimizer
+- apparently for SSA form, the IRCodeChunk is not a proper "basic block" yet because the last operation should be a branch or return, and no other branches
+- reduce register usage via linear-scan algorithm (based on live intervals) https://anoopsarkar.github.io/compilers-class/assets/lectures/opt3-regalloc-linearscan.pdf
+  don't forget to take into account the data type of the register when it's going to be reused!
+- idea: (but LLVM IR simply keeps the variables, so not a good idea then?...): replace all scalar variables by an allocated register. Keep a table of the variable to register mapping (including the datatype)
+  global initialization values are simply a list of LOAD instructions.
+  Variables replaced include all subroutine parameters!  So the only variables that remain as variables are arrays and strings.
+- the @split arrays are currently also split in _lsb/_msb arrays in the IR, and operations take multiple (byte) instructions that may lead to verbose and slow operation and machine code generation down the line.
+  maybe another representation is needed once actual codegeneration is done from the IR...?
+- ExpressionCodeResult:  get rid of the separation between single result register and multiple result registers? maybe not, this requires hundreds of lines to change
+- sometimes source lines end up missing in the output p8ir, for example the first assignment is gone in:
+     sub start() {
+     cx16.r0L = cx16.r1 as ubyte
+     cx16.r0sL = cx16.r1s as byte }
+     more detailed example:
+
 not all source lines are correctly reported in the IR file,
 for example the below subroutine only shows the sub() line::
 
@@ -55,115 +150,9 @@ and for example the below code omits line 5::
     }
 
 
-STRUCTS and TYPED POINTERS
---------------------------
-
-- the type of struct initializer arrays should not be uword[] but ^^struct[] ?
-- implement the remaining TODO's in PointerAssignmentsGen.
-- optimize deref in PointerAssignmentsGen: optimize 'forceTemporary' to only use a temporary when the offset is >0
-- optimize addUnsignedByteOrWordToAY in PointerAssignmentsGen a bit more
-- optimize the float copying in assignIndexedPointer() (also word?)
-- implement even more struct instance assignments (via memcopy) in CodeDesugarer (see the TODO) (add to documentation as well, paragraph 'Structs')
-- support @nosplit pointer arrays?
-- support pointer to pointer?
-- support for typed function pointers?  (&routine could be typed by default as well then)
-- really fixing the pointer dereferencing issues (cursed hybrid beween IdentifierReference, PtrDereferece and PtrIndexedDereference) may require getting rid of scoped identifiers altogether and treat '.' as a "scope or pointer following operator"
-- (later, nasty parser problem:) support chaining pointer dereference on function calls that return a pointer.  (type checking now fails on stuff like func().field and func().next.field)
-
-
-Future Things and Ideas
-^^^^^^^^^^^^^^^^^^^^^^^
-
-- optimization conflict detection:
-    1 fun applyModificationsSafely(modifications: List<IAstModification>) {
-    2     // Can check for conflicts before applying
-    3     modifications.groupBy { getTargetNode(it) }.forEach { target, mods ->
-    4         if (mods.size > 1) {
-    5             // Handle or report conflicting modifications
-    6         }
-    7     }
-    8     modifications.forEach { it.perform() }
-    9 }
-- improve ANTLR grammar with better error handling (according to Qwen AI)
-- allow memory() to occur in array initializer
-- %breakpoint after an assignment is parsed as part of the expression (x % breakpoint), that should not happen
-- when a complete block is removed because unused, suppress all info messages about everything in the block being removed
-- fix the line, cols in Position, sometimes they count from 0 sometimes from 1
-- is "checkAssignmentCompatible" redundant (gets called just 1 time!) when we also have "checkValueTypeAndRange" ?
-- enums?
-- romable: should we have a way to explicitly set the memory address for the BSS area (add a -varsaddress and -slabsaddress options?)
-- romable: fix remaining codegens (some for loops, see ForLoopsAsmGen)
-- Kotlin: can we use inline value classes in certain spots? (domain types instead of primitives)
-- add float support to the configurable compiler targets
-- Improve the SublimeText syntax file for prog8, you can also install this for 'bat': https://github.com/sharkdp/bat?tab=readme-ov-file#adding-new-syntaxes--language-definitions
-- Change scoping rules for qualified symbols so that they don't always start from the root but behave like other programming languages (look in local scope first), maybe only when qualified symbol starts with '.' such as: .local.value = 33
-- something to reduce the need to use fully qualified names all the time. 'with' ?  Or 'using <prefix>'?
-- Improve register load order in subroutine call args assignments:
-  in certain situations (need examples!), the "wrong" order of evaluation of function call arguments is done which results
-  in overwriting registers that already got their value, which requires a lot of stack juggling (especially on plain 6502 cpu!)
-  Maybe this routine can be made more intelligent.  See usesOtherRegistersWhileEvaluating() and argumentsViaRegisters().
-- Does it make codegen easier if everything is an expression?  Start with the PtProgram ast classes, change statements to expressions that have (new) VOID data type
-- Can we support signed % (remainder) somehow?
-- Multidimensional arrays and chained indexing, purely as syntactic sugar over regular arrays. Probaby only useful once we have typed pointers. (addressed in 'struct' branch)
-- make a form of "manual generics" possible like: varsub routine(T arg)->T  where T is expanded to a specific type
-  (this is already done hardcoded for several of the builtin functions)
-- [much work:] more support for (64tass) SEGMENTS in the prog8 syntax itself?
-- ability to use a sub instead of only a var for @bank ? what for though? dynamic bank/overlay loading?
-- Zig-like try-based error handling where the V flag could indicate error condition? and/or BRK to jump into monitor on failure? (has to set BRK vector for that) But the V flag is also set on certain normal instructions
-
-
-IR/VM
------
-- is it possible to use LOADFIELD/STOREFIELD instructions more?
-- make multiple classes of registers and maybe also categorize by life time , to prepare for better register allocation in the future
-    SYSCALL_ARGS,        // Reserved for syscall arguments (r99000-99099, r99100-99199)
-    FUNCTION_PARAMS,     // For passing function parameters
-    FUNCTION_RETURNS,    // For function return values
-    TEMPORARY,           // Short-lived temporary values
-    LOCAL_VARIABLES,     // Local variables within functions
-    GLOBAL_VARIABLES,    // Global/static variables
-    HARDWARE_MAPPED,     // Mapped to CPU hardware registers
-    LOOP_INDICES,        // Used as loop counters
-    ADDRESS_CALCULATION  // Used for pointer arithmetic
-  Categorizing registers by lifetime can significantly improve allocation:
-   - Short-lived: Temporary registers used in expressions
-   - Medium-lived: Local variables within a function
-  Registers could be categorized by how frequently they're accessed:
-   - Hot Registers: Frequently accessed (should be allocated to faster physical registers)
-   - Warm Registers: Moderately accessed
-   - Cold Registers: Rarely accessed (can be spilled to memory if needed)
-  We already have type-based pools
-    - byte, word, float registers
-
-- pointer dt's are all reduced to just an uword (in the irTypeString method) - is this okay or could it be beneficial to reintroduce the actual pointer type information? See commit 88b074c208450c58aa32469745afa03e4c5f564a
-- change the instruction format so an indirect register (a pointer) can be used more often, at least for the inplace assignment operators that operate on pointer
-- getting it in shape for code generation...: the IR file should be able to encode every detail about a prog8 program (the VM doesn't have to actually be able to run all of it though!)
-- fix call() return value handling (... what's wrong with it again?)
-- encode asmsub/extsub clobber info in the call , or maybe include these definitions in the p8ir file itself too.  (return registers are already encoded in the CALL instruction)
-- proper code gen for the CALLI instruction and that it (optionally) returns a word value that needs to be assigned to a reg
-- implement fast code paths for TODO("inplace split....
-- implement more TODOs in AssignmentGen
-- sometimes source lines end up missing in the output p8ir, for example the first assignment is gone in:
-     sub start() {
-     cx16.r0L = cx16.r1 as ubyte
-     cx16.r0sL = cx16.r1s as byte }
-- do something with the 'split' tag on split word arrays
-- add more optimizations in IRPeepholeOptimizer
-- apparently for SSA form, the IRCodeChunk is not a proper "basic block" yet because the last operation should be a branch or return, and no other branches
-- reduce register usage via linear-scan algorithm (based on live intervals) https://anoopsarkar.github.io/compilers-class/assets/lectures/opt3-regalloc-linearscan.pdf
-  don't forget to take into account the data type of the register when it's going to be reused!
-- idea: (but LLVM IR simply keeps the variables, so not a good idea then?...): replace all scalar variables by an allocated register. Keep a table of the variable to register mapping (including the datatype)
-  global initialization values are simply a list of LOAD instructions.
-  Variables replaced include all subroutine parameters!  So the only variables that remain as variables are arrays and strings.
-- the @split arrays are currently also split in _lsb/_msb arrays in the IR, and operations take multiple (byte) instructions that may lead to verbose and slow operation and machine code generation down the line.
-  maybe another representation is needed once actual codegeneration is done from the IR...?
-- ExpressionCodeResult:  get rid of the separation between single result register and multiple result registers? maybe not, this requires hundreds of lines to change
-
-
 Libraries
 ---------
 - Add split-word array sorting routines to sorting module?
-- See if the raster interrupt handler on the C64 can be tweaked to be a more stable raster irq
 - pet32 target: make syslib more complete (missing kernal routines)?
 - need help with: PET disk routines (OPEN, SETLFS etc are not exposed as kernal calls)
 - c128 target: make syslib more complete (missing kernal routines)?
@@ -172,12 +161,13 @@ Libraries
 Optimizations
 -------------
 
+- more optimized operator handling of different types, for example uword a ^ byte b now does a type cast of b to word first
 - Port benchmarks from https://thred.github.io/c-bench-64/  to prog8 and see how it stacks up.
-- Since fixing the missing zp-var initialization, programs grew in size again because STZ's reappered. Can we add more intelligent (and correct!) optimizations to remove those STZs that might be redundant again?
+- Since fixing the missing zp-var initialization, programs grew in size again because STZ's reappeared. Can we add more intelligent (and correct!) optimizations to remove those STZs that might be redundant again?
 - in Identifier: use typedarray of strings instead of listOf? Other places?
 - Compilation speed: try to join multiple modifications in 1 result in the AST processors instead of returning it straight away every time
-- Compare output of some Oscar64 samples to what prog8 does for the equivalent code (see https://github.com/drmortalwombat/OscarTutorials/tree/main and https://github.com/drmortalwombat/oscar64/tree/main/samples)
 - Optimize the IfExpression code generation to be more like regular if-else code.  (both 6502 and IR) search for "TODO don't store condition as expression"
 - VariableAllocator: can we think of a smarter strategy for allocating variables into zeropage, rather than first-come-first-served?
   for instance, vars used inside loops first, then loopvars, then uwords used as pointers (or these first??), then the rest
+  This will probably need the register categorization from the IR explained there, for the old 6502 codegen there is not enough information to act on
 - various optimizers skip stuff if compTarget.name==VMTarget.NAME.  Once 6502-codegen is done from IR code, those checks should probably be removed, or be made permanent

examples/c64/charset.p8

@@ -27,11 +27,9 @@ charset {
         ; copies the charset from ROM to RAM so we can modify it
 
         sys.set_irqd()
-        ubyte bank = @($0001)
-        @($0001) = bank & %11111011     ; enable CHAREN, so the character rom accessible at $d000
+        c64.banks(%011)  ; enable CHAREN, so the character rom accessible at $d000
         sys.memcopy($d000, CHARSET, 256*8*2)  ; copy the charset to RAM
-
-        @($0001) = bank         ; reset previous memory banking
+        c64.banks(%111)  ; enable I/O registers again
         sys.clear_irqd()
     }
 

examples/c64/multiplexer.p8

@@ -0,0 +1,197 @@
+; TODO in progress attempts to make a flexible sprite multiplexer
+; For an easy sprite multiplexer (actually, just a sprite duplicator) see the "simplemultiplexer" example.
+; inspiration: https://codebase64.net/doku.php?id=base:sprite_multiplexing
+
+%import syslib
+%import math
+%import textio
+%option no_sysinit
+
+main {
+    const ubyte NUM_VSPRITES = 16
+    const uword spritedata_base = $3000     ; NOTE: in VIC bank 0 (and 2), the Char Rom is at $1000 so we can't store sprite data there
+
+    sub start() {
+        setup_sprite_graphics()
+        c64.SPENA = 255    ; enable all sprites
+
+        method1_wait_raster_before_each_sprite()
+
+        ; Theoretically, I think all sprite Y values can be set in one go, and again after the last of the 8 hw sprites has started drawing, etc.
+        ; There's no strict need to set it right before the sprite is starting to draw. But it only saves a few cycles?
+        ; method2_after_old_sprite()
+
+    }
+
+    sub setup_sprite_graphics() {
+        ; prepare the sprite graphics (put a different letter in each of the balloons)
+        sys.set_irqd()
+        c64.banks(%011)  ; enable CHAREN, so the character rom accessible at $d000 (inverse at $d400)
+        ubyte i
+        for i in 0 to NUM_VSPRITES-1 {
+            uword sprdat = spritedata_base + $0040*i
+            sys.memcopy(balloonsprite, sprdat, 64)
+            ubyte cptr
+            for cptr in 0 to 7 {
+                sprdat[7+cptr*3] = @($d400+(i+sc:'a')*$0008 + cptr)
+            }
+            ^^Sprite sprite = sprites[i]
+            sprite.dataptr = lsb(spritedata_base/64) + i
+        }
+        c64.banks(%111)     ; enable IO registers again
+        ; sys.clear_irqd()  ; do not re-enable IRQ as this will glitch the display because it interferes with raster timing
+    }
+
+    sub method1_wait_raster_before_each_sprite() {
+        repeat {
+            animate_sprites()
+            ; wait for raster lines and update hardware sprites
+            ubyte vs_index
+            for vs_index in 0 to NUM_VSPRITES-1 {
+                ubyte virtual_sprite = sort_virtualsprite[vs_index]
+                ubyte hw_sprite = virtual_sprite & 7
+                ^^Sprite sprite = sprites[virtual_sprite]
+                sys.waitrasterline(sprite.y-3)      ; wait for a few lines above the sprite so we have time to set all attributes properly
+                c64.EXTCOL++
+                c64.SPXY[hw_sprite*2+1] = sprite.y
+                c64.SPXY[hw_sprite*2] = lsb(sprite.x)
+                if sprite.x >= 256
+                    c64.MSIGX |= msigx_setmask[hw_sprite]
+                else
+                    c64.MSIGX &= msigx_clearmask[hw_sprite]
+                c64.SPCOL[hw_sprite] = sprite.color
+                c64.SPRPTR[hw_sprite] = sprite.dataptr
+                c64.EXTCOL--
+            }
+        }
+    }
+
+    ubyte tt
+    sub animate_sprites() {
+        tt += 2
+        ubyte st = tt
+        ubyte virtual_sprite
+        for virtual_sprite in 0 to NUM_VSPRITES-1 {
+            ^^Sprite sprite = sprites[virtual_sprite]
+            sprite.x = $0028 + math.sin8u(st)
+            ; TODO  nice anim   sprite.y = $50 + math.cos8u(st*3)/2
+            st += 10
+            sort_ypositions[virtual_sprite] = sprite.y
+            sort_virtualsprite[virtual_sprite] = virtual_sprite
+            ; TODO keep working with the previously sorted result instead of rewriting the list every time, makes sorting faster if not much changes in the Y positions
+        }
+
+        ; TODO remove this simplistic anim but it's here to test the algorithms
+        sprite = sprites[0]
+        sprite.y++
+        sort_ypositions[0] = sprites[0].y
+        sprite = sprites[1]
+        sprite.y--
+        sort_ypositions[1] = sprites[1].y
+
+        ;c64.EXTCOL--
+        sort_virtual_sprites()
+        ;c64.EXTCOL++
+    }
+
+    ubyte[NUM_VSPRITES] sort_ypositions
+    ubyte[NUM_VSPRITES] sort_virtualsprite
+
+    ; TODO rewrite in assembly, sorting must be super fast
+    sub sort_virtual_sprites() {
+        ubyte @zp pos=1
+        while pos != NUM_VSPRITES {
+            if sort_ypositions[pos]>=sort_ypositions[pos-1]
+                pos++
+            else {
+                ; swap elements
+                cx16.r0L = sort_ypositions[pos-1]
+                sort_ypositions[pos-1] = sort_ypositions[pos]
+                sort_ypositions[pos] = cx16.r0L
+                ; swap virtual sprite indexes
+                cx16.r0L = sort_virtualsprite[pos-1]
+                sort_virtualsprite[pos-1] = sort_virtualsprite[pos]
+                sort_virtualsprite[pos] = cx16.r0L
+                pos--
+                if_z
+                    pos++
+            }
+        }
+    }
+
+
+    ubyte[8] msigx_setmask = [
+        %00000001,
+        %00000010,
+        %00000100,
+        %00001000,
+        %00010000,
+        %00100000,
+        %01000000,
+        %10000000
+    ]
+
+    ubyte[8] msigx_clearmask = [
+        %11111110,
+        %11111101,
+        %11111011,
+        %11110111,
+        %11101111,
+        %11011111,
+        %10111111,
+        %01111111
+    ]
+
+    struct Sprite {
+        ubyte color
+        ubyte dataptr
+        uword x
+        ubyte y
+    }
+
+
+    ^^Sprite[NUM_VSPRITES]  sprites = [
+        [0, $ff, 20, 60],
+        [1, $ff, 40, 70],
+        [2, $ff, 60, 80],
+        [3, $ff, 80, 90],
+        [4, $ff, 100, 100],
+        [5, $ff, 120, 110],
+        [7, $ff, 140, 120],       ; skip color 6 (blue on blue)
+        [8, $ff, 160, 130],
+        [9, $ff, 180, 140],
+        [10, $ff, 200, 150],
+        [11, $ff, 220, 160],
+        [12, $ff, 240, 170],
+        [13, $ff, 260, 180],
+        [14, $ff, 280, 190],
+        [15, $ff, 300, 200],
+        [0, $ff, 320, 210],
+    ]
+
+
+    ubyte[] balloonsprite = [
+        %00000000,%01111111,%00000000,
+        %00000001,%11111111,%11000000,
+        %00000011,%11111111,%11100000,
+        %00000011,%11100011,%11100000,
+        %00000111,%11011100,%11110000,
+        %00000111,%11011101,%11110000,
+        %00000111,%11011100,%11110000,
+        %00000011,%11100011,%11100000,
+        %00000011,%11111111,%11100000,
+        %00000011,%11111111,%11100000,
+        %00000010,%11111111,%10100000,
+        %00000001,%01111111,%01000000,
+        %00000001,%00111110,%01000000,
+        %00000000,%10011100,%10000000,
+        %00000000,%10011100,%10000000,
+        %00000000,%01001001,%00000000,
+        %00000000,%01001001,%00000000,
+        %00000000,%00111110,%00000000,
+        %00000000,%00111110,%00000000,
+        %00000000,%00111110,%00000000,
+        %00000000,%00011100,%00000000
+    ]
+}
+

examples/c64/simplemultiplexer.p8

@@ -0,0 +1,113 @@
+; Simple sprite multiplexer.
+; This is just a sprite duplicator routine that displays the 8 hardware sprites multiple times in rows,
+; it does not do any complicated sprite Y manipulation/multiplexing.
+; No assembly code is used.
+
+
+%import syslib
+%zeropage basicsafe
+%option no_sysinit
+
+main {
+
+    sub start() {
+        ubyte i
+        for i in 0 to 7 {
+            c64.set_sprite_ptr(i, &sprites.balloonsprite)           ; alternatively, set directly:  c64.SPRPTR[i] = $0a00 / 64
+        }
+        sprites.set_sprites_X(0)
+        sprites.set_sprites_Y(sprites.sprites_Y_start)
+        c64.SPCOL[5] = 8   ; change blue balloon to different color
+
+        c64.SPENA = 255       ; enable all sprites
+        irq.first_sprite_X = 0
+        irq.sprites_Y = sprites.sprites_Y_start
+        sys.set_rasterirq(&irq.multiplexer, irq.sprites_Y+1)
+
+        ; exit program back to basic, balloons will keep flying :-)
+    }
+
+}
+
+irq {
+    uword first_sprite_X
+    ubyte sprites_Y
+
+    ; Here is the actual multiplexing routine.
+    ; it's a raster irq just after the start of the sprite,
+    ; that updates the Y position of all the sprits,
+    ; and registers a new rater irq for that next row of sprites.
+    ; If the bottom of the screen is reached, it resets the X position of the sprites as well,
+    ; and moves the sprites back to the top of the screen.
+    sub multiplexer() -> bool {
+        c64.EXTCOL++
+        sprites_Y += 24
+        bool system_irq = false
+        if sprites_Y > (255-24-1) {
+            cx16.r2 = c64.RASTER + 23
+            sprites_Y = sprites.sprites_Y_start
+            first_sprite_X++
+            if first_sprite_X >= 340
+                first_sprite_X =0
+            sprites.set_sprites_Y(sprites_Y)
+            while c64.RASTER != cx16.r2 {
+                ; wait until raster line after sprite has been fully drawn (at least 24 lines down)
+            }
+            sprites.set_sprites_X(first_sprite_X)        ; we can now update the X positions without risk of sprite tearing
+            system_irq = true
+        } else {
+            sprites.set_sprites_Y(sprites_Y)
+        }
+
+        sys.set_rasterline(sprites_Y+1)
+        c64.EXTCOL--
+        return system_irq
+    }
+
+}
+
+sprites {
+    const ubyte sprites_Y_start = 55
+
+    sub set_sprites_Y(ubyte y) {
+        for cx16.r0L in 1 to 15 step 2 {
+            c64.SPXY[cx16.r0L] = y
+        }
+    }
+
+    sub set_sprites_X(uword x) {
+        for cx16.r0L in 0 to 14 step 2 {
+            x += 20
+            if x >= 340
+                x -= 340
+            c64.SPXY[cx16.r0L] = lsb(x)
+            cx16.r1L = msb(x)>>1
+            ror(c64.MSIGX)      ; rotate that MSB bit of the sprite X position into the MSB sprite X register for each sprite
+        }
+    }
+
+    ubyte[] @align64 balloonsprite = [
+        %00000000,%01111111,%00000000,
+        %00000001,%11111111,%11000000,
+        %00000011,%11111111,%11100000,
+        %00000011,%11100011,%11100000,
+        %00000111,%11011100,%11110000,
+        %00000111,%11011101,%11110000,
+        %00000111,%11011100,%11110000,
+        %00000011,%11100011,%11100000,
+        %00000011,%11111111,%11100000,
+        %00000011,%11111111,%11100000,
+        %00000010,%11111111,%10100000,
+        %00000001,%01111111,%01000000,
+        %00000001,%00111110,%01000000,
+        %00000000,%10011100,%10000000,
+        %00000000,%10011100,%10000000,
+        %00000000,%01001001,%00000000,
+        %00000000,%01001001,%00000000,
+        %00000000,%00111110,%00000000,
+        %00000000,%00111110,%00000000,
+        %00000000,%00111110,%00000000,
+        %00000000,%00011100,%00000000
+    ]
+}
+

examples/c64/sprites.p8

@@ -7,17 +7,17 @@ main {
 
     sub start() {
 
-        txt.print("balloon sprites!\n...we are all floating...\n")
+        txt.print("balloon sprites!\n...we are all floating...\nborders are open too\n")
 
         ubyte @zp i
         for i in 0 to 7 {
             c64.set_sprite_ptr(i, &spritedata.balloonsprite)           ; alternatively, set directly:  c64.SPRPTR[i] = $0a00 / 64
-            c64.SPXY[i*2] = 50+25*i
+            c64.SPXY[i*2] = 60+22*i
             c64.SPXY[i*2+1] = math.rnd()
         }
 
         c64.SPENA = 255       ; enable all sprites
-        sys.set_rasterirq(&irq.irqhandler, 255)     ; enable animation
+        sys.set_rasterirq(&irq.irqhandler, 248)         ; trigger irq just above bottom border line
     }
 }
 
@@ -25,6 +25,7 @@ main {
 irq {
 
     sub irqhandler() -> bool {
+        c64.SCROLY = 19             ; 24 row mode, preparing for border opening
         c64.EXTCOL--
 
         ; float up & wobble horizontally
@@ -39,6 +40,7 @@ irq {
         }
 
         c64.EXTCOL++
+        c64.SCROLY = 27            ; 25 row mode, border is open
         return true
     }
 

examples/customtarget/readme.txt

@@ -4,8 +4,8 @@ more example materials.
 
 Look in the Makefile to see how to build or run the various programs.
 
-The user 'adiee5' has been working on a Nintendo Entertainment System (NES) compilation target
+*adiee5* has been working on a Nintendo Entertainment System (NES) compilation target
 and example program, you can find those efforts here on GitHub: https://github.com/adiee5/prog8-nes-target
-Note that the NES is a very alien architecture for Prog8 still and the support is very limited
-(for example, prog8 is not aware that the program code usually is going to end up in a ROM cartridge,
-and currently still generates code that might not work in ROM.)
+
+*gillham* has been working on a few other compilation targets, such as VIC-20 (various editions), Foenix, and CX16OS.
+you can find them here on GitHub: https://github.com/gillham/prog8targets

examples/cx16/charfade.p8

@@ -0,0 +1,113 @@
+
+; color fader that doesn't do palette fade but color attribute fade.
+; NOTE: this needs ROM 49+
+
+%import textio
+%import math
+%zeropage basicsafe
+%option no_sysinit
+
+
+main {
+    ^^uword palette = memory("palette", 256*2, 0)
+    ^^ubyte closest = memory("closest", 4096, 0)
+
+    const ubyte WIDTH = 40
+    const ubyte HEIGHT = 30
+
+    sub start() {
+
+        fill_default_palette()
+
+        txt.print("\nprecalc color fade table, patience plz ")
+        precalc_fade_table()
+
+        cx16.set_screen_mode(128)
+        cx16.GRAPH_set_colors(0,0,0)
+        cx16.GRAPH_clear()
+        cx16.VERA_L1_CONFIG |= %1000    ; T256C on!
+        ubyte x,y
+        for y in 0 to HEIGHT-1 {
+            for x in 0 to WIDTH-1 {
+                txt.setcc2(x,y, math.rnd(), math.rnd())
+            }
+        }
+
+        sys.wait(60)
+
+        repeat  {
+            repeat 4 sys.waitvsync()
+
+            for y in 0 to HEIGHT-1 {
+                for x in 0 to WIDTH-1 {
+                    ubyte @zp currentcolor = txt.getclr(x,y)
+                    cx16.r0L = closest[darken(palette[currentcolor])]
+                    if cx16.r0L>0 and cx16.r0L == currentcolor
+                        cx16.r0L = 15       ; to avoid stuck colors, make it gray, this will be able to fade to black because the pallette contains all shades of gray.
+                    txt.setclr(x,y, cx16.r0L)
+                }
+
+                txt.setcc2(math.rnd() % WIDTH, math.rnd() % HEIGHT, math.rnd(), math.rnd())     ; add new chars
+            }
+        }
+    }
+
+    sub precalc_fade_table() {
+        uword colorindex
+        ubyte r,g,b
+        alias color = cx16.r2
+
+        for r in 0 to 15 {
+            txt.print_ub(16-r)
+            txt.spc()
+            for g in 0 to 15 {
+                for b in 0 to 15 {
+                    color = mkword(r, g<<4 | b)
+                    closest[colorindex] = find_closest(color)
+                    colorindex++
+                }
+            }
+        }
+    }
+
+    sub darken(uword color @R0) -> uword {
+        if cx16.r0H!=0
+            cx16.r0H--
+        if cx16.r0L & $0f != 0
+            cx16.r0L--
+        if cx16.r0L & $f0 != 0
+            cx16.r0L -= $10
+        return cx16.r0
+    }
+
+    sub find_closest(uword rgb @R0) -> ubyte {
+        ubyte distance = 255
+        ubyte current = 0
+        alias index = cx16.r1L
+
+        for index in 0 to 255 {
+            uword @zp pc = palette[index]
+            ubyte @zp d2 = abs(msb(pc) as byte - msb(rgb) as byte)                ; RED distance
+            d2 += abs((lsb(pc) & $f0) as byte - (lsb(rgb) & $f0) as byte) >> 4    ; GREEN distance
+            d2 += abs((lsb(pc) & $0f) as byte - (lsb(rgb) & $0f) as byte)         ; BLUE distance
+            if d2==0
+                return index
+            if d2 < distance {
+                distance = d2
+                current = index
+            }
+        }
+
+        return current
+    }
+
+    sub fill_default_palette() {
+        ubyte pal_bank
+        uword pal_addr
+
+        pal_bank, pal_addr = cx16.get_default_palette()      ; needs ROM 49+
+        cx16.push_rombank(pal_bank)
+        sys.memcopy(pal_addr, palette, 256*2)
+        cx16.pop_rombank()
+    }
+}

examples/test.p8

@@ -1,34 +1,36 @@
+%option enable_floats
+
 main {
+    struct Node {
+        ubyte id
+        str name
+        uword array
+        bool flag
+        float perc
+    }
+    struct Foobar {
+        bool thing
+    }
+
     sub start() {
-        ^^uword @shared ptr
+        ^^Node test = []
 
-        add1()
-        add2()
-        sub1()
-        sub2()
-
-        sub add1() {
-            ptr += 5
-            cx16.r0 = ptr + 5
-            cx16.r0 = peekw(ptr + 5)
-        }
-
-        sub add2() {
-            ptr += cx16.r0L
-            cx16.r0 = ptr + cx16.r0L
-            cx16.r0 = peekw(ptr + cx16.r0L)
-        }
-
-        sub sub1() {
-            ptr -= 5
-            cx16.r0 = ptr - 5
-            cx16.r0 = peekw(ptr - 5)
-        }
-
-        sub sub2() {
-            ptr -= cx16.r0L
-            cx16.r0 = ptr - cx16.r0L
-            cx16.r0 = peekw(ptr - cx16.r0L)
-        }
+        test.id ++
+        test.array += 1000
+        test.id <<= 2
+        test.id <<= cx16.r0L
+        test.id >>= 3
+        test.id >>= cx16.r0L
+        test.id &= 1
+        test.id *= 5        ; TODO implement this
+        test.id /= 5        ; TODO implement this
+        test.array ^= 1000
+        test.array |= 1000
+        test.array &= 1000
+        test.array >>= 3
+        test.array >>= cx16.r0L
+        test.array <<= 2
+        test.array <<= cx16.r0L
+        test.array *= 5
     }
 }

gradle.properties

@@ -3,4 +3,4 @@ org.gradle.console=rich
 org.gradle.parallel=true
 org.gradle.daemon=true
 kotlin.code.style=official
-version=12.0-BETA2
+version=12.0-BETA3

intermediate/src/prog8/intermediate/IRInstructions.kt

@@ -973,7 +973,7 @@ data class IRInstruction(
         when (this.reg2direction) {
             OperandDirection.UNUSED -> {}
             OperandDirection.READ -> {
-                writeRegsCounts[this.reg2!!] = writeRegsCounts.getValue(this.reg2)+1
+                readRegsCounts[this.reg2!!] = readRegsCounts.getValue(this.reg2)+1
                 val actualtype = determineReg2Type()
                 if(actualtype!=null) {
                     val existingType = regsTypes[reg2]
@@ -989,7 +989,7 @@ data class IRInstruction(
         when (this.reg3direction) {
             OperandDirection.UNUSED -> {}
             OperandDirection.READ -> {
-                writeRegsCounts[this.reg3!!] = writeRegsCounts.getValue(this.reg3)+1
+                readRegsCounts[this.reg3!!] = readRegsCounts.getValue(this.reg3)+1
                 val actualtype = determineReg3Type()
                 if(actualtype!=null) {
                     val existingType = regsTypes[reg3]

languageServer/build.gradle.kts

@@ -0,0 +1,105 @@
+plugins {
+    kotlin("jvm")
+    id("application")
+}
+
+val debugPort = 8000
+val debugArgs = "-agentlib:jdwp=transport=dt_socket,server=y,address=8000,suspend=n,quiet=y"
+
+val serverMainClassName = "prog8lsp.MainKt"
+val applicationName = "prog8-language-server"
+
+application {
+    mainClass.set(serverMainClassName)
+    description = "Code completions, diagnostics and more for Prog8"
+    // applicationDefaultJvmArgs = listOf("-DkotlinLanguageServer.version=$version")
+    applicationDistribution.into("bin") {
+        filePermissions {
+            user {
+                read=true
+                execute=true
+                write=true
+            }
+            other.execute = true
+            group.execute = true
+        }
+    }
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation("org.eclipse.lsp4j:org.eclipse.lsp4j:0.24.0")
+    implementation("org.eclipse.lsp4j:org.eclipse.lsp4j.jsonrpc:0.24.0")
+    
+    // For JSON processing if needed
+    //implementation("com.google.code.gson:gson:2.10.1")
+    
+    // For more advanced text processing
+    //implementation("org.apache.commons:commons-lang3:3.12.0")
+
+    implementation(project(":compiler"))
+
+}
+
+configurations.forEach { config ->
+    config.resolutionStrategy {
+        preferProjectModules()
+    }
+}
+
+sourceSets.main {
+    java.srcDir("src")
+    resources.srcDir("resources")
+}
+
+tasks.startScripts {
+    applicationName = "prog8-language-server"
+}
+
+tasks.register<Exec>("fixFilePermissions") {
+    // When running on macOS or Linux the start script
+    // needs executable permissions to run.
+
+    onlyIf { !System.getProperty("os.name").lowercase().contains("windows") }
+    commandLine("chmod", "+x", "${tasks.installDist.get().destinationDir}/bin/prog8-language-server")
+}
+
+tasks.register<JavaExec>("debugRun") {
+    mainClass.set(serverMainClassName)
+    classpath(sourceSets.main.get().runtimeClasspath)
+    standardInput = System.`in`
+
+    jvmArgs(debugArgs)
+    doLast {
+        println("Using debug port $debugPort")
+    }
+}
+
+tasks.register<CreateStartScripts>("debugStartScripts") {
+    applicationName = "prog8-language-server"
+    mainClass.set(serverMainClassName)
+    outputDir = tasks.installDist.get().destinationDir.toPath().resolve("bin").toFile()
+    classpath = tasks.startScripts.get().classpath
+    defaultJvmOpts = listOf(debugArgs)
+}
+
+tasks.register<Sync>("installDebugDist") {
+    dependsOn("installDist")
+    finalizedBy("debugStartScripts")
+}
+
+tasks.withType<Test> {
+    // Disable tests for now since we don't have any
+    enabled = false
+}
+
+tasks.installDist {
+    finalizedBy("fixFilePermissions")
+}
+
+tasks.build {
+    finalizedBy("installDist")
+}

languageServer/languageServer.iml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="eclipse.lsp4j" level="project" />
+    <orderEntry type="library" name="KotlinJavaRuntime" level="project" />
+    <orderEntry type="module" module-name="compiler" />
+  </component>
+</module>

languageServer/src/prog8lsp/AsyncExecutor.kt

@@ -0,0 +1,34 @@
+package prog8lsp
+
+import java.util.concurrent.CompletableFuture
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+import java.util.function.Supplier
+
+private var threadCount = 0
+
+class AsyncExecutor {
+    private val workerThread = Executors.newSingleThreadExecutor { Thread(it, "async${threadCount++}") }
+
+    fun execute(task: () -> Unit) =
+        CompletableFuture.runAsync(Runnable(task), workerThread)
+
+    fun <R> compute(task: () -> R) =
+        CompletableFuture.supplyAsync(Supplier(task), workerThread)
+
+    fun <R> computeOr(defaultValue: R, task: () -> R?) =
+        CompletableFuture.supplyAsync(Supplier {
+            try {
+                task() ?: defaultValue
+            } catch (e: Exception) {
+                defaultValue
+            }
+        }, workerThread)
+
+    fun shutdown(awaitTermination: Boolean) {
+        workerThread.shutdown()
+        if (awaitTermination) {
+            workerThread.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS)
+        }
+    }
+}

languageServer/src/prog8lsp/Main.kt

@@ -0,0 +1,22 @@
+package prog8lsp
+
+import org.eclipse.lsp4j.launch.LSPLauncher
+import prog8.buildversion.VERSION
+import java.util.concurrent.Executors
+import java.util.logging.Level
+import java.util.logging.Logger
+
+fun main(args: Array<String>) {
+    Logger.getLogger("").level = Level.INFO
+    
+    val inStream = System.`in`
+    val outStream = System.out
+    val server = Prog8LanguageServer()
+    val threads = Executors.newCachedThreadPool { Thread(it, "client") }
+    val launcher = LSPLauncher.createServerLauncher(server, inStream, outStream, threads) { it }
+
+    server.connect(launcher.remoteProxy)
+    launcher.startListening()
+    
+    println("Prog8 Language Server started. Prog8 version: ${VERSION}")
+}

languageServer/src/prog8lsp/Prog8LanguageServer.kt

@@ -0,0 +1,85 @@
+package prog8lsp
+
+import org.eclipse.lsp4j.*
+import org.eclipse.lsp4j.jsonrpc.messages.Either
+import org.eclipse.lsp4j.services.*
+import java.io.Closeable
+import java.util.concurrent.CompletableFuture
+import java.util.concurrent.CompletableFuture.completedFuture
+import java.util.logging.Logger
+
+class Prog8LanguageServer: LanguageServer, LanguageClientAware, Closeable {
+    private lateinit var client: LanguageClient
+    private val textDocuments = Prog8TextDocumentService()
+    private val workspaces = Prog8WorkspaceService()
+    private val async = AsyncExecutor()
+    private val logger = Logger.getLogger(Prog8LanguageServer::class.simpleName)
+
+    override fun initialize(params: InitializeParams): CompletableFuture<InitializeResult> = async.compute {
+        logger.info("Initializing LanguageServer")
+
+        val result = InitializeResult()
+        val capabilities = ServerCapabilities()
+        
+        // Text document synchronization
+        capabilities.textDocumentSync = Either.forLeft(TextDocumentSyncKind.Full)
+        
+        // Completion support
+        val completionOptions = CompletionOptions()
+        completionOptions.resolveProvider = true
+        completionOptions.triggerCharacters = listOf(".", ":")
+        capabilities.completionProvider = completionOptions
+        
+        // Document symbol support
+        capabilities.documentSymbolProvider = Either.forLeft(true)
+        
+        // Hover support
+        capabilities.hoverProvider = Either.forLeft(true)
+        
+        // Definition support
+        capabilities.definitionProvider = Either.forLeft(true)
+        
+        // Code action support
+        val codeActionOptions = CodeActionOptions()
+        codeActionOptions.codeActionKinds = listOf(CodeActionKind.QuickFix)
+        capabilities.codeActionProvider = Either.forRight(codeActionOptions)
+        
+        // Document formatting support
+        capabilities.documentFormattingProvider = Either.forLeft(true)
+        capabilities.documentRangeFormattingProvider = Either.forLeft(true)
+        
+        // Rename support
+        val renameOptions = RenameOptions()
+        renameOptions.prepareProvider = true
+        capabilities.renameProvider = Either.forRight(renameOptions)
+        
+        // Workspace symbol support
+        capabilities.workspaceSymbolProvider = Either.forLeft(true)
+        
+        result.capabilities = capabilities
+        result
+    }
+
+    override fun shutdown(): CompletableFuture<Any> {
+        close()
+        return completedFuture(null)
+    }
+
+    override fun exit() { }
+
+    override fun getTextDocumentService(): TextDocumentService = textDocuments
+
+    override fun getWorkspaceService(): WorkspaceService = workspaces
+
+    override fun connect(client: LanguageClient) {
+        logger.info("connecting to language client")
+        this.client = client
+        workspaces.connect(client)
+        textDocuments.connect(client)
+    }
+
+    override fun close() {
+        logger.info("closing down")
+        async.shutdown(awaitTermination = true)
+    }
+}

languageServer/src/prog8lsp/Prog8TextDocumentService.kt

@@ -0,0 +1,389 @@
+package prog8lsp
+
+import org.eclipse.lsp4j.*
+import org.eclipse.lsp4j.jsonrpc.messages.Either
+import org.eclipse.lsp4j.services.LanguageClient
+import org.eclipse.lsp4j.services.TextDocumentService
+import java.util.concurrent.CompletableFuture
+import java.util.logging.Logger
+import kotlin.system.measureTimeMillis
+
+// Document model to maintain in memory
+data class Prog8Document(
+    val uri: String,
+    var text: String,
+    var version: Int
+)
+
+class Prog8TextDocumentService: TextDocumentService {
+    private var client: LanguageClient? = null
+    private val async = AsyncExecutor()
+    private val logger = Logger.getLogger(Prog8TextDocumentService::class.simpleName)
+    
+    // In-memory document store
+    private val documents = mutableMapOf<String, Prog8Document>()
+
+    fun connect(client: LanguageClient) {
+        this.client = client
+    }
+
+    override fun didOpen(params: DidOpenTextDocumentParams) {
+        logger.info("didOpen: ${params.textDocument.uri}")
+        
+        // Create and store document model
+        val document = Prog8Document(
+            uri = params.textDocument.uri,
+            text = params.textDocument.text,
+            version = params.textDocument.version
+        )
+        documents[params.textDocument.uri] = document
+        
+        // Trigger diagnostics when a document is opened
+        validateDocument(document)
+    }
+
+    override fun didChange(params: DidChangeTextDocumentParams) {
+        logger.info("didChange: ${params.textDocument.uri}")
+        
+        // Get the document from our store
+        val document = documents[params.textDocument.uri]
+        if (document != null) {
+            // Update document version
+            document.version = params.textDocument.version
+            
+            // Apply changes to the document text
+            // For simplicity, we're assuming full document sync (TextDocumentSyncKind.Full)
+            // In a real implementation, you might need to handle incremental changes
+            val text = params.contentChanges.firstOrNull()?.text
+            if (text != null) {
+                document.text = text
+            }
+            
+            // Trigger diagnostics when a document changes
+            validateDocument(document)
+        }
+    }
+
+    override fun didClose(params: DidCloseTextDocumentParams) {
+        logger.info("didClose: ${params.textDocument.uri}")
+        
+        // Remove document from our store
+        documents.remove(params.textDocument.uri)
+        
+        // Clear diagnostics when a document is closed
+        client?.publishDiagnostics(PublishDiagnosticsParams(params.textDocument.uri, listOf()))
+    }
+
+    override fun didSave(params: DidSaveTextDocumentParams) {
+        logger.info("didSave: ${params.textDocument.uri}")
+        // Handle save events if needed
+    }
+
+    override fun documentSymbol(params: DocumentSymbolParams): CompletableFuture<MutableList<Either<SymbolInformation, DocumentSymbol>>> = async.compute {
+        logger.info("Find symbols in ${params.textDocument.uri}")
+        val result: MutableList<Either<SymbolInformation, DocumentSymbol>>
+        val time = measureTimeMillis {
+            result = mutableListOf()
+            
+            // Get document from our store
+            val document = documents[params.textDocument.uri]
+            if (document != null) {
+                // Parse document and extract symbols
+                // This is just a placeholder implementation
+                val range = Range(Position(0, 0), Position(0, 10))
+                val selectionRange = Range(Position(0, 0), Position(0, 10))
+                val symbol = DocumentSymbol("exampleSymbol", SymbolKind.Function, range, selectionRange)
+                result.add(Either.forRight(symbol))
+            }
+        }
+        logger.info("Finished in $time ms")
+        result
+    }
+
+    override fun completion(params: CompletionParams): CompletableFuture<Either<MutableList<CompletionItem>, CompletionList>> = async.compute {
+        logger.info("Completion for ${params.textDocument.uri} at ${params.position}")
+        val result: Either<MutableList<CompletionItem>, CompletionList>
+        val time = measureTimeMillis {
+            val items = mutableListOf<CompletionItem>()
+            
+            // Get document from our store
+            val document = documents[params.textDocument.uri]
+            if (document != null) {
+                // Implement actual completion logic based on context
+                // This is just a placeholder implementation
+                val printItem = CompletionItem("print")
+                printItem.kind = CompletionItemKind.Function
+                printItem.detail = "Print text to console"
+                printItem.documentation = Either.forLeft("Outputs the given text to the console")
+                
+                val forItem = CompletionItem("for")
+                forItem.kind = CompletionItemKind.Keyword
+                forItem.detail = "For loop"
+                forItem.documentation = Either.forLeft("Iterates over a range or collection")
+                
+                val ifItem = CompletionItem("if")
+                ifItem.kind = CompletionItemKind.Keyword
+                ifItem.detail = "Conditional statement"
+                ifItem.documentation = Either.forLeft("Executes code based on a condition")
+                
+                items.add(printItem)
+                items.add(forItem)
+                items.add(ifItem)
+            }
+            
+            val list = CompletionList(false, items)
+            result = Either.forRight(list)
+        }
+        logger.info("Finished in $time ms")
+        result
+    }
+
+    override fun hover(params: HoverParams): CompletableFuture<Hover?> = async.compute {
+        logger.info("Hover for ${params.textDocument.uri} at ${params.position}")
+        
+        // Get document from our store
+        val document = documents[params.textDocument.uri]
+        if (document != null) {
+            // Simple implementation that checks for keywords at the position
+            val keyword = getWordAtPosition(document, params.position)
+            
+            when (keyword) {
+                "print" -> {
+                    val hover = Hover()
+                    hover.contents = Either.forLeft(listOf(Either.forLeft("**print** - Outputs text to the console\n\n```prog8\nprint \"Hello, World!\"\n```")))
+                    return@compute hover
+                }
+                "for" -> {
+                    val hover = Hover()
+                    hover.contents = Either.forLeft(listOf(Either.forLeft("**for** - Loop construct\n\n```prog8\nfor i in 0..10 {\n    print i\n}\n```")))
+                    return@compute hover
+                }
+                "if" -> {
+                    val hover = Hover()
+                    hover.contents = Either.forLeft(listOf(Either.forLeft("**if** - Conditional statement\n\n```prog8\nif x > 5 {\n    print \"x is greater than 5\"\n}\n```")))
+                    return@compute hover
+                }
+                "sub" -> {
+                    val hover = Hover()
+                    hover.contents = Either.forLeft(listOf(Either.forLeft("**sub** - Defines a subroutine\n\n```prog8\nsub myFunction() {\n    print \"Hello from function\"\n}\n```")))
+                    return@compute hover
+                }
+                else -> {
+                    // Return null for unknown symbols
+                    return@compute null
+                }
+            }
+        }
+        
+        // Return null if document not found
+        null
+    }
+
+    override fun definition(params: DefinitionParams): CompletableFuture<Either<MutableList<out Location>, MutableList<out LocationLink>>> = async.compute {
+        logger.info("Definition request for ${params.textDocument.uri} at ${params.position}")
+        
+        // Get document from our store
+        val document = documents[params.textDocument.uri]
+        if (document != null) {
+            // Implement actual definition lookup
+            // This would involve parsing the document, finding the symbol at the position,
+            // and then finding where that symbol is defined
+            val locations = mutableListOf<Location>()
+            
+            // Placeholder implementation
+            // locations.add(Location("file:///path/to/definition.p8", Range(Position(0, 0), Position(0, 10))))
+            
+            return@compute Either.forLeft(locations)
+        }
+        
+        Either.forLeft(mutableListOf<Location>())
+    }
+
+    override fun formatting(params: DocumentFormattingParams): CompletableFuture<MutableList<out TextEdit>> = async.compute {
+        logger.info("Formatting document ${params.textDocument.uri}")
+        
+        // Get document from our store
+        val document = documents[params.textDocument.uri]
+        if (document != null) {
+            // Implement actual code formatting
+            // This is just a placeholder implementation
+            val edits = mutableListOf<TextEdit>()
+            
+            // Example of how you might implement formatting:
+            // 1. Parse the document
+            // 2. Apply formatting rules (indentation, spacing, etc.)
+            // 3. Generate TextEdit objects for the changes
+            
+            return@compute edits
+        }
+        
+        mutableListOf<TextEdit>()
+    }
+
+    override fun rangeFormatting(params: DocumentRangeFormattingParams): CompletableFuture<MutableList<out TextEdit>> = async.compute {
+        logger.info("Range formatting document ${params.textDocument.uri}")
+        
+        // Get document from our store
+        val document = documents[params.textDocument.uri]
+        if (document != null) {
+            // Implement actual code formatting for range
+            // This is just a placeholder implementation
+            val edits = mutableListOf<TextEdit>()
+            
+            // Example of how you might implement range formatting:
+            // 1. Parse the document range
+            // 2. Apply formatting rules to the selected range
+            // 3. Generate TextEdit objects for the changes
+            
+            return@compute edits
+        }
+        
+        mutableListOf<TextEdit>()
+    }
+
+    override fun rename(params: RenameParams): CompletableFuture<WorkspaceEdit> = async.compute {
+        logger.info("Rename symbol in ${params.textDocument.uri} at ${params.position}")
+        
+        // Get document from our store
+        val document = documents[params.textDocument.uri]
+        if (document != null) {
+            // Implement actual rename functionality
+            // This would involve:
+            // 1. Finding all references to the symbol at the given position
+            // 2. Creating TextEdit objects to rename each reference
+            // 3. Adding the edits to a WorkspaceEdit
+            
+            return@compute WorkspaceEdit()
+        }
+        
+        WorkspaceEdit()
+    }
+
+    override fun codeAction(params: CodeActionParams): CompletableFuture<MutableList<Either<Command, CodeAction>>> = async.compute {
+        logger.info("Code actions for ${params.textDocument.uri}")
+        
+        // Get document from our store
+        val document = documents[params.textDocument.uri]
+        if (document != null) {
+            val actions = mutableListOf<Either<Command, CodeAction>>()
+            
+            // Check diagnostics to provide quick fixes
+            for (diagnostic in params.context.diagnostics) {
+                when (diagnostic.code?.left) {
+                    "UnmatchedQuotes" -> {
+                        val action = CodeAction()
+                        action.title = "Add closing quote"
+                        action.kind = CodeActionKind.QuickFix
+                        action.diagnostics = listOf(diagnostic)
+                        action.isPreferred = true
+                        // TODO: Add actual TextEdit to fix the issue
+                        actions.add(Either.forRight(action))
+                    }
+                    "InvalidCharacter" -> {
+                        val action = CodeAction()
+                        action.title = "Remove invalid characters"
+                        action.kind = CodeActionKind.QuickFix
+                        action.diagnostics = listOf(diagnostic)
+                        // TODO: Add actual TextEdit to fix the issue
+                        actions.add(Either.forRight(action))
+                    }
+                }
+            }
+            
+            // Add some general code actions
+            val organizeImportsAction = CodeAction()
+            organizeImportsAction.title = "Organize imports"
+            organizeImportsAction.kind = CodeActionKind.SourceOrganizeImports
+            actions.add(Either.forRight(organizeImportsAction))
+            
+            return@compute actions
+        }
+        
+        mutableListOf<Either<Command, CodeAction>>()
+    }
+
+    private fun getWordAtPosition(document: Prog8Document, position: Position): String {
+        // Extract the word at the given position from the document text
+        val lines = document.text.lines()
+        if (position.line < lines.size) {
+            val line = lines[position.line]
+            // Simple word extraction - in a real implementation, you'd want a more robust solution
+            val words = line.split(Regex("\\s+|[^a-zA-Z0-9_]"))
+            var charIndex = 0
+            for (word in words) {
+                if (position.character >= charIndex && position.character <= charIndex + word.length) {
+                    return word
+                }
+                charIndex += word.length + 1 // +1 for the separator
+            }
+        }
+        return "" // Default to empty string
+    }
+
+    private fun validateDocument(document: Prog8Document) {
+        logger.info("Validating document: ${document.uri}")
+        val diagnostics = mutableListOf<Diagnostic>()
+        
+        // Split text into lines for easier processing
+        val lines = document.text.lines()
+        
+        // Check for syntax errors
+        for ((lineNumber, line) in lines.withIndex()) {
+            // Check for unmatched quotes
+            val quoteCount = line.count { it == '"' }
+            if (quoteCount % 2 != 0) {
+                val range = Range(Position(lineNumber, 0), Position(lineNumber, line.length))
+                val diagnostic = Diagnostic(
+                    range,
+                    "Unmatched quotes",
+                    DiagnosticSeverity.Error,
+                    "prog8-lsp",
+                    "UnmatchedQuotes"
+                )
+                diagnostics.add(diagnostic)
+            }
+            
+            // Check for invalid characters
+            if (line.contains(Regex("[^\\u0000-\\u007F]"))) {
+                val range = Range(Position(lineNumber, 0), Position(lineNumber, line.length))
+                val diagnostic = Diagnostic(
+                    range,
+                    "Invalid character found",
+                    DiagnosticSeverity.Error,
+                    "prog8-lsp",
+                    "InvalidCharacter"
+                )
+                diagnostics.add(diagnostic)
+            }
+            
+            // Check for common Prog8 syntax issues
+            // For example, check if a line starts with a keyword but doesn't follow proper syntax
+            if (line.trim().startsWith("sub ") && !line.contains("(")) {
+                val range = Range(Position(lineNumber, 0), Position(lineNumber, line.length))
+                val diagnostic = Diagnostic(
+                    range,
+                    "Subroutine declaration missing parentheses",
+                    DiagnosticSeverity.Error,
+                    "prog8-lsp",
+                    "InvalidSubroutine"
+                )
+                diagnostics.add(diagnostic)
+            }
+        }
+        
+        // Check for other issues
+        if (document.text.contains("error")) {
+            val range = Range(Position(0, 0), Position(0, 5))
+            val diagnostic = Diagnostic(
+                range,
+                "This is a sample diagnostic",
+                DiagnosticSeverity.Warning,
+                "prog8-lsp",
+                "SampleDiagnostic"
+            )
+            diagnostics.add(diagnostic)
+        }
+        
+        client?.publishDiagnostics(PublishDiagnosticsParams(document.uri, diagnostics))
+    }
+}

languageServer/src/prog8lsp/Prog8WorkspaceService.kt

@@ -0,0 +1,89 @@
+package prog8lsp
+
+import org.eclipse.lsp4j.*
+import org.eclipse.lsp4j.jsonrpc.messages.Either
+import org.eclipse.lsp4j.services.LanguageClient
+import org.eclipse.lsp4j.services.WorkspaceService
+import java.util.concurrent.CompletableFuture
+import java.util.logging.Logger
+
+class Prog8WorkspaceService: WorkspaceService {
+    private var client: LanguageClient? = null
+    private val logger = Logger.getLogger(Prog8WorkspaceService::class.simpleName)
+
+    fun connect(client: LanguageClient) {
+        this.client = client
+    }
+
+    override fun executeCommand(params: ExecuteCommandParams): CompletableFuture<Any> {
+        logger.info("executeCommand $params")
+        return super.executeCommand(params)
+    }
+
+    override fun symbol(params: WorkspaceSymbolParams): CompletableFuture<Either<MutableList<out SymbolInformation>, MutableList<out WorkspaceSymbol>>> {
+        logger.info("symbol $params")
+        // TODO: Implement workspace symbol search
+        // This is just a placeholder implementation
+        val symbols = mutableListOf<WorkspaceSymbol>()
+        val symbol = WorkspaceSymbol(
+            "workspaceSymbol",
+            SymbolKind.Function,
+            Either.forLeft(Location("file:///example.p8", Range(Position(0, 0), Position(0, 10))))
+        )
+        symbols.add(symbol)
+        return CompletableFuture.completedFuture(Either.forRight(symbols))
+    }
+
+    override fun resolveWorkspaceSymbol(workspaceSymbol: WorkspaceSymbol): CompletableFuture<WorkspaceSymbol> {
+        logger.info("resolveWorkspaceSymbol $workspaceSymbol")
+        return CompletableFuture.completedFuture(workspaceSymbol)
+    }
+
+    override fun didChangeConfiguration(params: DidChangeConfigurationParams) {
+        logger.info("didChangeConfiguration: $params")
+    }
+
+    override fun didChangeWatchedFiles(params: DidChangeWatchedFilesParams) {
+        logger.info("didChangeWatchedFiles: $params")
+    }
+
+    override fun didChangeWorkspaceFolders(params: DidChangeWorkspaceFoldersParams) {
+        logger.info("didChangeWorkspaceFolders $params")
+        super.didChangeWorkspaceFolders(params)
+    }
+
+    override fun willCreateFiles(params: CreateFilesParams): CompletableFuture<WorkspaceEdit> {
+        logger.info("willCreateFiles $params")
+        return super.willCreateFiles(params)
+    }
+
+    override fun didCreateFiles(params: CreateFilesParams) {
+        logger.info("didCreateFiles $params")
+        super.didCreateFiles(params)
+    }
+
+    override fun willRenameFiles(params: RenameFilesParams): CompletableFuture<WorkspaceEdit> {
+        logger.info("willRenameFiles $params")
+        return super.willRenameFiles(params)
+    }
+
+    override fun didRenameFiles(params: RenameFilesParams) {
+        logger.info("didRenameFiles $params")
+        super.didRenameFiles(params)
+    }
+
+    override fun willDeleteFiles(params: DeleteFilesParams): CompletableFuture<WorkspaceEdit> {
+        logger.info("willDeleteFiles $params")
+        return super.willDeleteFiles(params)
+    }
+
+    override fun didDeleteFiles(params: DeleteFilesParams) {
+        logger.info("didDeleteFiles $params")
+        super.didDeleteFiles(params)
+    }
+
+    override fun diagnostic(params: WorkspaceDiagnosticParams): CompletableFuture<WorkspaceDiagnosticReport> {
+        logger.info("diagnostic $params")
+        return super.diagnostic(params)
+    }
+}

parser/src/main/antlr/Prog8ANTLR.g4

@@ -151,7 +151,7 @@ labeldef :  identifier ':'  ;
 
 unconditionaljump :  GOTO  expression ;
 
-directive : directivename (directivenamelist | (directivearg? | directivearg (',' directivearg)*))   ;
+directive : directivename '!'? (directivenamelist | (directivearg? | directivearg (',' directivearg)*))   ;
 
 directivename: '%' UNICODEDNAME;
 

settings.gradle

@@ -10,5 +10,6 @@ include(
     ':codeGenCpu6502',
     ':codeGenExperimental',
     ':compiler',
-    ':beanshell'
+    ':beanshell',
+    ':languageServer'
 )