cbm charset codecs, name checking

il65/examples/imported2.ill

@@ -13,6 +13,12 @@
 
 	X = 42
 	return 44
+
+	    sub foo() -> () {
+          A=99
+          return
+        }
+
 }
 
 

il65/examples/test.ill

@@ -84,7 +84,6 @@ cool:
 
   }
 
-
 some_label_def:   A=44
   return 1+999
 	%breakpoint

il65/src/il65/Main.kt

@@ -18,12 +18,14 @@ fun main(args: Array<String>) {
         val moduleAst = importModule(filepath)
         moduleAst.linkParents()
         val globalNamespace = moduleAst.namespace()
-        // globalNamespace.debugPrint()
+        //globalNamespace.debugPrint()
 
+        moduleAst.checkIdentifiers(globalNamespace)
         moduleAst.optimizeExpressions(globalNamespace)
         moduleAst.optimizeStatements(globalNamespace)
         val globalNamespaceAfterOptimize = moduleAst.namespace()  // it could have changed in the meantime
         moduleAst.checkValid(globalNamespaceAfterOptimize)        // check if final tree is valid
+        val allScopedSymbolDefinitions = moduleAst.checkIdentifiers(globalNamespace)
 
         // determine special compiler options
         val options = moduleAst.statements.filter { it is Directive && it.directive=="%option" }.flatMap { (it as Directive).args }.toSet()

il65/src/il65/ast/AST.kt

@@ -38,13 +38,20 @@ class FatalAstException (override var message: String) : Exception(message)
 
 open class AstException (override var message: String) : Exception(message)
 
-open class SyntaxError(override var message: String, val position: Position?) : AstException(message) {
+class SyntaxError(override var message: String, val position: Position?) : AstException(message) {
     override fun toString(): String {
         val location = position?.toString() ?: ""
         return "$location Syntax error: $message"
     }
 }
 
+class NameError(override var message: String, val position: Position?) : AstException(message) {
+    override fun toString(): String {
+        val location = position?.toString() ?: ""
+        return "$location Name error: $message"
+    }
+}
+
 class ExpressionException(message: String, val position: Position?) : AstException(message) {
     override fun toString(): String {
         val location = position?.toString() ?: ""
@@ -95,7 +102,7 @@ interface IAstProcessor {
         functionCall.arglist = functionCall.arglist.map { it.process(this) }
         return functionCall
     }
-    fun process(identifier: Identifier): IExpression {
+    fun process(identifier: IdentifierReference): IExpression {
         return identifier
     }
     fun process(jump: Jump): IStatement {
@@ -114,6 +121,10 @@ interface IAstProcessor {
         range.to = range.to.process(this)
         return range
     }
+
+    fun process(label: Label): IStatement {
+        return label
+    }
 }
 
 
@@ -126,11 +137,23 @@ interface Node {
 
 interface IStatement : Node {
     fun process(processor: IAstProcessor) : IStatement
+    fun scopedName(name: String): String {
+        val scope = mutableListOf<String>()
+        var statementScope = this.parent
+        while(statementScope!=null && statementScope !is Module) {
+            if(statementScope is INameScope) {
+                scope.add(0, statementScope.name)
+            }
+            statementScope = statementScope.parent
+        }
+        scope.add(name)
+        return scope.joinToString(".")
+    }
 }
 
 
 interface IFunctionCall {
-    var target: Identifier
+    var target: IdentifierReference
     var arglist: List<IExpression>
 }
 
@@ -289,7 +312,7 @@ data class Label(val name: String) : IStatement {
         this.parent = parent
     }
 
-    override fun process(processor: IAstProcessor) = this
+    override fun process(processor: IAstProcessor) = processor.process(this)
 }
 
 
@@ -377,7 +400,7 @@ data class Assignment(var target: AssignTarget, val aug_op : String?, var value:
     }
 }
 
-data class AssignTarget(val register: Register?, val identifier: Identifier?) : Node {
+data class AssignTarget(val register: Register?, val identifier: IdentifierReference?) : Node {
     override var position: Position? = null
     override var parent: Node? = null
 
@@ -510,7 +533,7 @@ data class RegisterExpr(val register: Register) : IExpression {
 }
 
 
-data class Identifier(val scopedName: List<String>) : IExpression {
+data class IdentifierReference(val scopedName: List<String>) : IExpression {
     override var position: Position? = null
     override var parent: Node? = null
 
@@ -552,7 +575,7 @@ data class PostIncrDecr(var target: AssignTarget, val operator: String) : IState
 }
 
 
-data class Jump(val address: Int?, val identifier: Identifier?) : IStatement {
+data class Jump(val address: Int?, val identifier: IdentifierReference?) : IStatement {
     override var position: Position? = null
     override var parent: Node? = null
 
@@ -565,7 +588,7 @@ data class Jump(val address: Int?, val identifier: Identifier?) : IStatement {
 }
 
 
-data class FunctionCall(override var target: Identifier, override var arglist: List<IExpression>) : IExpression, IFunctionCall {
+data class FunctionCall(override var target: IdentifierReference, override var arglist: List<IExpression>) : IExpression, IFunctionCall {
     override var position: Position? = null
     override var parent: Node? = null
 
@@ -603,7 +626,7 @@ data class FunctionCall(override var target: Identifier, override var arglist: L
 }
 
 
-data class FunctionCallStatement(override var target: Identifier, override var arglist: List<IExpression>) : IStatement, IFunctionCall {
+data class FunctionCallStatement(override var target: IdentifierReference, override var arglist: List<IExpression>) : IStatement, IFunctionCall {
     override var position: Position? = null
     override var parent: Node? = null
 
@@ -1029,15 +1052,15 @@ private fun il65Parser.ExpressionContext.toAst(withPosition: Boolean) : IExpress
 private fun il65Parser.Expression_listContext.toAst(withPosition: Boolean) = expression().map{ it.toAst(withPosition) }
 
 
-private fun il65Parser.IdentifierContext.toAst(withPosition: Boolean) : Identifier {
-    val ident = Identifier(listOf(text))
+private fun il65Parser.IdentifierContext.toAst(withPosition: Boolean) : IdentifierReference {
+    val ident = IdentifierReference(listOf(text))
     ident.position = toPosition(withPosition)
     return ident
 }
 
 
-private fun il65Parser.Scoped_identifierContext.toAst(withPosition: Boolean) : Identifier {
-    val ident = Identifier(NAME().map { it.text })
+private fun il65Parser.Scoped_identifierContext.toAst(withPosition: Boolean) : IdentifierReference {
+    val ident = IdentifierReference(NAME().map { it.text })
     ident.position = toPosition(withPosition)
     return ident
 }

il65/src/il65/ast/AstChecker.kt

@@ -27,7 +27,6 @@ fun Module.checkValid(globalNamespace: INameScope) {
 
 class AstChecker(private val globalNamespace: INameScope) : IAstProcessor {
     private val checkResult: MutableList<SyntaxError> = mutableListOf()
-    private val blockNames: HashMap<String, Position?> = hashMapOf()
 
     fun result(): List<SyntaxError> {
         return checkResult
@@ -36,10 +35,10 @@ class AstChecker(private val globalNamespace: INameScope) : IAstProcessor {
     override fun process(module: Module) {
         super.process(module)
         val directives = module.statements.filter { it is Directive }.groupBy { (it as Directive).directive }
-        directives.filter { it.value.size > 1 }.forEach{
-            when(it.key) {
+        directives.filter { it.value.size > 1 }.forEach{ entry ->
+            when(entry.key) {
                 "%output", "%launcher", "%zeropage", "%address" ->
-                    it.value.mapTo(checkResult) { SyntaxError("directive can just occur once", it.position) }
+                    entry.value.mapTo(checkResult) { SyntaxError("directive can just occur once", it.position) }
             }
         }
     }
@@ -55,51 +54,7 @@ class AstChecker(private val globalNamespace: INameScope) : IAstProcessor {
         if(block.address!=null && (block.address<0 || block.address>65535)) {
             checkResult.add(SyntaxError("block memory address must be valid integer 0..\$ffff", block.position))
         }
-        val existing = blockNames[block.name]
-        if(existing!=null) {
-            checkResult.add(SyntaxError("block name conflict, first defined in ${existing.file} line ${existing.line}", block.position))
-        } else {
-            blockNames[block.name] = block.position
-        }
-
         super.process(block)
-
-        // check if labels are unique
-        val labels = block.statements.filter { it is Label }.map { it as Label }
-        val labelnames = mutableMapOf<String, Position?>()
-        labels.forEach {
-            val existing = labelnames[it.name]
-            if(existing!=null) {
-                checkResult.add(SyntaxError("label name conflict, first defined on line ${existing.line}", it.position))
-            } else {
-                labelnames[it.name] = it.position
-            }
-        }
-
-        // check if var names are unique
-        val variables = block.statements.filter { it is VarDecl }.map{ it as VarDecl }
-        val varnames= mutableMapOf<String, Position?>()
-        variables.forEach {
-            val existing = varnames[it.name]
-            if(existing!=null) {
-                checkResult.add(SyntaxError("variable name conflict, first defined on line ${existing.line}", it.position))
-            } else {
-                varnames[it.name] = it.position
-            }
-        }
-
-        // check if subroutine names are unique
-        val subroutines = block.statements.filter { it is Subroutine }.map{ it as Subroutine }
-        val subnames = mutableMapOf<String, Position?>()
-        subroutines.forEach {
-            val existing = subnames[it.name]
-            if(existing!=null) {
-                checkResult.add(SyntaxError("subroutine name conflict, first defined on line ${existing.line}", it.position))
-            } else {
-                subnames[it.name] = it.position
-            }
-        }
-
         return block
     }
 
@@ -130,30 +85,6 @@ class AstChecker(private val globalNamespace: INameScope) : IAstProcessor {
 
         super.process(subroutine)
 
-        // check if labels are unique
-        val labels = subroutine.statements.filter { it is Label }.map { it as Label }
-        val labelnames = mutableMapOf<String, Position?>()
-        labels.forEach {
-            val existing = labelnames[it.name]
-            if(existing!=null) {
-                checkResult.add(SyntaxError("label name conflict, first defined on line ${existing.line}", it.position))
-            } else {
-                labelnames[it.name] = it.position
-            }
-        }
-
-        // check if var names are unique
-        val variables = subroutine.statements.filter { it is VarDecl }.map{ it as VarDecl }
-        val varnames= mutableMapOf<String, Position?>()
-        variables.forEach {
-            val existing = varnames[it.name]
-            if(existing!=null) {
-                checkResult.add(SyntaxError("variable name conflict, first defined on line ${existing.line}", it.position))
-            } else {
-                varnames[it.name] = it.position
-            }
-        }
-
         // subroutine must contain at least one 'return' or 'goto'
         // (or if it has an asm block, that must contain a 'rts' or 'jmp')
         if(subroutine.statements.count { it is Return || it is Jump } == 0) {
@@ -165,7 +96,7 @@ class AstChecker(private val globalNamespace: INameScope) : IAstProcessor {
                              it.contains(" jmp") || it.contains("\tjmp")}
             if(amount==0 )
                 err("subroutine must have at least one 'return' or 'goto' in it (or 'rts' / 'jmp' in case of %asm)")
-                }
+            }
         }
 
         return subroutine
@@ -377,7 +308,7 @@ class AstChecker(private val globalNamespace: INameScope) : IAstProcessor {
         return super.process(functionCall)
     }
 
-    private fun checkFunctionExists(target: Identifier, statement: IStatement) {
+    private fun checkFunctionExists(target: IdentifierReference, statement: IStatement) {
         if(globalNamespace.lookup(target.scopedName, statement)==null)
             checkResult.add(SyntaxError("undefined function or subroutine: ${target.scopedName.joinToString(".")}", statement.position))
     }

il65/src/il65/ast/AstIdentifiersChecker.kt

@@ -0,0 +1,84 @@
+package il65.ast
+
+import il65.parser.ParsingFailedError
+
+/**
+ * Checks the validity of all identifiers (no conflicts)
+ * Also builds a list of all (scoped) symbol definitions
+ */
+
+fun Module.checkIdentifiers(globalNamespace: INameScope): MutableMap<String, IStatement> {
+    val checker = AstIdentifiersChecker(globalNamespace)
+    this.process(checker)
+    val checkResult = checker.result()
+    checkResult.forEach {
+        System.err.println(it)
+    }
+    if(checkResult.isNotEmpty())
+        throw ParsingFailedError("There are ${checkResult.size} errors in module '$name'.")
+    return checker.symbols
+}
+
+
+class AstIdentifiersChecker(private val globalNamespace: INameScope) : IAstProcessor {
+    private val checkResult: MutableList<AstException> = mutableListOf()
+
+    var symbols: MutableMap<String, IStatement> = mutableMapOf()
+        private set
+
+    fun result(): List<AstException> {
+        return checkResult
+    }
+
+    private fun nameError(name: String, position: Position?, existing: IStatement) {
+        checkResult.add(NameError("name conflict '$name', first defined in ${existing.position?.file} line ${existing.position?.line}", position))
+    }
+
+    override fun process(block: Block): IStatement {
+        val scopedName = block.scopedName(block.name)
+        val existing = symbols[scopedName]
+        if(existing!=null) {
+            nameError(block.name, block.position, existing)
+        } else {
+            symbols[scopedName] = block
+        }
+        super.process(block)
+        return block
+    }
+
+    override fun process(decl: VarDecl): IStatement {
+        val scopedName = decl.scopedName(decl.name)
+        val existing = symbols[scopedName]
+        if(existing!=null) {
+            nameError(decl.name, decl.position, existing)
+        } else {
+            symbols[scopedName] = decl
+        }
+        super.process(decl)
+        return decl
+    }
+
+    override fun process(subroutine: Subroutine): IStatement {
+        val scopedName = subroutine.scopedName(subroutine.name)
+        val existing = symbols[scopedName]
+        if(existing!=null) {
+            nameError(subroutine.name, subroutine.position, existing)
+        } else {
+            symbols[scopedName] = subroutine
+        }
+        super.process(subroutine)
+        return subroutine
+    }
+
+    override fun process(label: Label): IStatement {
+        val scopedName = label.scopedName(label.name)
+        val existing = symbols[scopedName]
+        if(existing!=null) {
+            nameError(label.name, label.position, existing)
+        } else {
+            symbols[scopedName] = label
+        }
+        super.process(label)
+        return label
+    }
+}

il65/src/il65/optimizing/ExpressionOptimizer.kt

@@ -70,7 +70,7 @@ class ExpressionOptimizer(private val globalNamespace: INameScope) : IAstProcess
     /**
      * replace identifiers that refer to const value, with the value itself
      */
-    override fun process(identifier: Identifier): IExpression {
+    override fun process(identifier: IdentifierReference): IExpression {
         return try {
             identifier.constValue(globalNamespace) ?: identifier
         } catch (ax: AstException) {

il65/src/il65/optimizing/StatementsOptimizer.kt

@@ -18,7 +18,7 @@ fun Module.optimizeStatements(globalNamespace: INameScope) {
 }
 
 /*
-    todo remove unused blocks and unused subroutines
+    todo remove unused blocks, subroutines and variable decls (replace with empty AnonymousStatementList)
     todo statement optimization: create augmented assignment from assignment that only refers to its lvalue (A=A+10, A=4*A, ...)
     todo statement optimization: X+=1, X-=1  --> X++/X--  ,
     todo remove statements that have no effect  X=X , X+=0, X-=0, X*=1, X/=1, X//=1, A |= 0, A ^= 0, A<<=0, etc etc

il65/test/UnitTests.kt

@@ -202,3 +202,64 @@ class TestZeropage {
         assert(zp.available()==0)
     }
 }
+
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+class TestPetscii {
+
+    @Test
+    fun testLowercase() {
+        assertThat(Petscii.encodePetscii("hello WORLD 123 @!£", true), equalTo(
+                shortArrayOf(72, 69, 76, 76, 79, 32, 0xd7, 0xcf, 0xd2, 0xcc, 0xc4, 32, 49, 50, 51, 32, 64, 33, 0x5c)))
+        assertThat(Petscii.encodePetscii("\uf11a", true), equalTo(shortArrayOf(0x12)))   // reverse vid
+        assertThat(Petscii.encodePetscii("✓", true), equalTo(shortArrayOf(0xfa)))
+        assertFailsWith<CompilerException> { Petscii.encodePetscii("π", true) }
+        assertFailsWith<CompilerException> { Petscii.encodePetscii("♥", true) }
+
+        assertThat(Petscii.decodePetscii(shortArrayOf(72, 0xd7, 0x5c, 0xfa, 0x12), true), equalTo("hW£✓\uF11A"))
+        assertFailsWith<ArrayIndexOutOfBoundsException> { Petscii.decodePetscii(shortArrayOf(-1), true) }
+        assertFailsWith<ArrayIndexOutOfBoundsException> { Petscii.decodePetscii(shortArrayOf(256), true) }
+    }
+
+    @Test
+    fun testUppercase() {
+        assertThat(Petscii.encodePetscii("HELLO 123 @!£"), equalTo(
+                shortArrayOf(72, 69, 76, 76, 79, 32, 49, 50, 51, 32, 64, 33, 0x5c)))
+        assertThat(Petscii.encodePetscii("\uf11a"), equalTo(shortArrayOf(0x12)))   // reverse vid
+        assertThat(Petscii.encodePetscii("♥"), equalTo(shortArrayOf(0xd3)))
+        assertThat(Petscii.encodePetscii("π"), equalTo(shortArrayOf(0xff)))
+        assertFailsWith<CompilerException> { Petscii.encodePetscii("✓") }
+
+        assertThat(Petscii.decodePetscii(shortArrayOf(72, 0x5c, 0xd3, 0xff)), equalTo("H£♥π"))
+        assertFailsWith<ArrayIndexOutOfBoundsException> { Petscii.decodePetscii(shortArrayOf(-1)) }
+        assertFailsWith<ArrayIndexOutOfBoundsException> { Petscii.decodePetscii(shortArrayOf(256)) }
+    }
+
+    @Test
+    fun testScreencodeLowercase() {
+        assertThat(Petscii.encodeScreencode("hello WORLD 123 @!£", true), equalTo(
+                shortArrayOf(0x08, 0x05, 0x0c, 0x0c, 0x0f, 0x20, 0x57, 0x4f, 0x52, 0x4c, 0x44, 0x20, 0x31, 0x32, 0x33, 0x20, 0x00, 0x21, 0x1c)
+        ))
+        assertThat(Petscii.encodeScreencode("✓", true), equalTo(shortArrayOf(0x7a)))
+        assertFailsWith<CompilerException> { Petscii.encodeScreencode("♥", true) }
+        assertFailsWith<CompilerException> { Petscii.encodeScreencode("π", true) }
+
+        assertThat(Petscii.decodeScreencode(shortArrayOf(0x08, 0x57, 0x1c, 0x7a), true), equalTo("hW£✓"))
+        assertFailsWith<ArrayIndexOutOfBoundsException> { Petscii.decodeScreencode(shortArrayOf(-1), true) }
+        assertFailsWith<ArrayIndexOutOfBoundsException> { Petscii.decodeScreencode(shortArrayOf(256), true) }
+    }
+
+    @Test
+    fun testScreencodeUppercase() {
+        assertThat(Petscii.encodeScreencode("WORLD 123 @!£"), equalTo(
+                shortArrayOf(0x17, 0x0f, 0x12, 0x0c, 0x04, 0x20, 0x31, 0x32, 0x33, 0x20, 0x00, 0x21, 0x1c)))
+        assertThat(Petscii.encodeScreencode("♥"), equalTo(shortArrayOf(0x53)))
+        assertThat(Petscii.encodeScreencode("π"), equalTo(shortArrayOf(0x5e)))
+        assertFailsWith<CompilerException> { Petscii.encodeScreencode("✓") }
+        assertFailsWith<CompilerException> { Petscii.encodeScreencode("hello") }
+
+        assertThat(Petscii.decodeScreencode(shortArrayOf(0x17, 0x1c, 0x53, 0x5e)), equalTo("W£♥π"))
+        assertFailsWith<ArrayIndexOutOfBoundsException> { Petscii.decodeScreencode(shortArrayOf(-1)) }
+        assertFailsWith<ArrayIndexOutOfBoundsException> { Petscii.decodeScreencode(shortArrayOf(256)) }
+    }
+}