generalize string encoding flag into enum

codeGeneration/src/prog8/codegen/target/C128Target.kt

@@ -9,21 +9,31 @@ import prog8.codegen.target.c128.C128MachineDefinition
 import prog8.codegen.target.cbm.Petscii
 import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsEvalOrder
 import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsHaveRegisterClobberRisk
+import prog8.compilerinterface.Encoding
 import prog8.compilerinterface.ICompilationTarget
 
 
 object C128Target: ICompilationTarget {
     override val name = "c128"
     override val machine = C128MachineDefinition()
-    override fun encodeString(str: String, altEncoding: Boolean): List<UByte> {
+    override fun encodeString(str: String, encoding: Encoding): List<UByte> {
-        val coded = if (altEncoding) Petscii.encodeScreencode(str, true) else Petscii.encodePetscii(str, true)
+        val coded = when(encoding) {
+            Encoding.PETSCII -> Petscii.encodePetscii(str, true)
+            Encoding.SCREENCODES -> Petscii.encodeScreencode(str, true)
+            else -> throw FatalAstException("unsupported encoding $encoding")
+        }
         return coded.fold(
             failure = { throw it },
             success = { it }
         )
     }
-    override fun decodeString(bytes: List<UByte>, altEncoding: Boolean) =
+    override fun decodeString(bytes: List<UByte>, encoding: Encoding): String {
-        if (altEncoding) Petscii.decodeScreencode(bytes, true) else Petscii.decodePetscii(bytes, true)
+        return when(encoding) {
+            Encoding.PETSCII -> Petscii.decodePetscii(bytes, true)
+            Encoding.SCREENCODES -> Petscii.decodeScreencode(bytes, true)
+            else -> throw FatalAstException("unsupported encoding $encoding")
+        }
+    }
 
     override fun asmsubArgsEvalOrder(sub: Subroutine): List<Int> =
         asmsub6502ArgsEvalOrder(sub)

codeGeneration/src/prog8/codegen/target/C64Target.kt

@@ -9,21 +9,32 @@ import prog8.codegen.target.c64.C64MachineDefinition
 import prog8.codegen.target.cbm.Petscii
 import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsEvalOrder
 import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsHaveRegisterClobberRisk
+import prog8.compilerinterface.Encoding
 import prog8.compilerinterface.ICompilationTarget
 
 
 object C64Target: ICompilationTarget {
     override val name = "c64"
     override val machine = C64MachineDefinition()
-    override fun encodeString(str: String, altEncoding: Boolean): List<UByte> {
+    override fun encodeString(str: String, encoding: Encoding): List<UByte> {
-        val coded = if (altEncoding) Petscii.encodeScreencode(str, true) else Petscii.encodePetscii(str, true)
+        val coded = when(encoding) {
+            Encoding.PETSCII -> Petscii.encodePetscii(str, true)
+            Encoding.SCREENCODES -> Petscii.encodeScreencode(str, true)
+            else -> throw FatalAstException("unsupported encoding $encoding")
+        }
         return coded.fold(
             failure = { throw it },
             success = { it }
         )
     }
-    override fun decodeString(bytes: List<UByte>, altEncoding: Boolean) =
+    override fun decodeString(bytes: List<UByte>, encoding: Encoding): String {
-        if (altEncoding) Petscii.decodeScreencode(bytes, true) else Petscii.decodePetscii(bytes, true)
+        return when(encoding) {
+            Encoding.PETSCII -> Petscii.decodePetscii(bytes, true)
+            Encoding.SCREENCODES -> Petscii.decodeScreencode(bytes, true)
+            else -> throw FatalAstException("unsupported encoding $encoding")
+        }
+    }
+
 
     override fun asmsubArgsEvalOrder(sub: Subroutine): List<Int> =
         asmsub6502ArgsEvalOrder(sub)

codeGeneration/src/prog8/codegen/target/Cx16Target.kt

@@ -1,10 +1,7 @@
 package prog8.codegen.target
 
 import com.github.michaelbull.result.fold
-import prog8.ast.base.ByteDatatypes
+import prog8.ast.base.*
-import prog8.ast.base.DataType
-import prog8.ast.base.PassByReferenceDatatypes
-import prog8.ast.base.WordDatatypes
 import prog8.ast.expressions.Expression
 import prog8.ast.statements.RegisterOrStatusflag
 import prog8.ast.statements.Subroutine
@@ -12,21 +9,34 @@ import prog8.codegen.target.cbm.Petscii
 import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsEvalOrder
 import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsHaveRegisterClobberRisk
 import prog8.codegen.target.cx16.CX16MachineDefinition
+import prog8.compilerinterface.Encoding
 import prog8.compilerinterface.ICompilationTarget
 
 
 object Cx16Target: ICompilationTarget {
     override val name = "cx16"
     override val machine = CX16MachineDefinition()
-    override fun encodeString(str: String, altEncoding: Boolean): List<UByte> {
+    override fun encodeString(str: String, encoding: Encoding): List<UByte> {
-        val coded = if (altEncoding) Petscii.encodeScreencode(str, true) else Petscii.encodePetscii(str, true)
+        val coded = when(encoding) {
+            Encoding.PETSCII -> Petscii.encodePetscii(str, true)
+            Encoding.SCREENCODES -> Petscii.encodeScreencode(str, true)
+            Encoding.ISO -> TODO("cx16 iso-encoding")
+            else -> throw FatalAstException("unsupported encoding $encoding")
+        }
         return coded.fold(
             failure = { throw it },
             success = { it }
         )
     }
-    override fun decodeString(bytes: List<UByte>, altEncoding: Boolean) =
+    override fun decodeString(bytes: List<UByte>, encoding: Encoding): String {
-        if (altEncoding) Petscii.decodeScreencode(bytes, true) else Petscii.decodePetscii(bytes, true)
+        return when(encoding) {
+            Encoding.PETSCII -> Petscii.decodePetscii(bytes, true)
+            Encoding.SCREENCODES -> Petscii.decodeScreencode(bytes, true)
+            Encoding.ISO -> TODO("cx16 iso-encoding")
+            else -> throw FatalAstException("unsupported encoding $encoding")
+        }
+    }
+
 
     override fun asmsubArgsEvalOrder(sub: Subroutine): List<Int> =
         asmsub6502ArgsEvalOrder(sub)

codeGeneration/src/prog8/codegen/target/cpu6502/codegen/AsmGen.kt

@@ -525,9 +525,8 @@ class AsmGen(private val program: Program,
     private fun outputStringvar(strdecl: VarDecl, nameOverride: String?=null) {
         val varname = nameOverride ?: strdecl.name
         val sv = strdecl.value as StringLiteralValue
-        val altEncoding = if(sv.altEncoding) "@" else ""
+        out("$varname\t; ${strdecl.datatype} ${sv.encoding}:\"${escape(sv.value).replace("\u0000", "<NULL>")}\"")
-        out("$varname\t; ${strdecl.datatype} $altEncoding\"${escape(sv.value).replace("\u0000", "<NULL>")}\"")
+        val bytes = compTarget.encodeString(sv.value, sv.encoding).plus(0.toUByte())
-        val bytes = compTarget.encodeString(sv.value, sv.altEncoding).plus(0.toUByte())
         val outputBytes = bytes.map { "$" + it.toString(16).padStart(2, '0') }
         for (chunk in outputBytes.chunked(16))
             out("  .byte  " + chunk.joinToString())

codeGeneration/src/prog8/codegen/target/cpu6502/codegen/assignment/AssignmentAsmGen.kt

@@ -361,7 +361,7 @@ internal class AssignmentAsmGen(private val program: Program, private val asmgen
                     DataType.STR -> {
                         require(elementDt.isBytes)
                         val stringVal = variable.value as StringLiteralValue
-                        val encoded = program.encoding.encodeString(stringVal.value, stringVal.altEncoding)
+                        val encoded = program.encoding.encodeString(stringVal.value, stringVal.encoding)
                         return containmentCheckIntoA(containment.element, elementDt.getOr(DataType.UNDEFINED), encoded.map { it.toInt() })
                     }
                     DataType.ARRAY_F -> {
@@ -410,7 +410,7 @@ internal class AssignmentAsmGen(private val program: Program, private val asmgen
         val stringVal = containment.iterable as? StringLiteralValue
         if(stringVal!=null) {
             require(elementDt.isBytes)
-            val encoded = program.encoding.encodeString(stringVal.value, stringVal.altEncoding)
+            val encoded = program.encoding.encodeString(stringVal.value, stringVal.encoding)
             return containmentCheckIntoA(containment.element, elementDt.getOr(DataType.UNDEFINED), encoded.map { it.toInt() })
         }
         val arrayVal = containment.iterable as? ArrayLiteralValue

codeOptimizers/src/prog8/optimizer/StatementOptimizer.kt

@@ -77,7 +77,7 @@ class StatementOptimizer(private val program: Program,
                 if(string!=null) {
                     val pos = functionCallStatement.position
                     if (string.value.length == 1) {
-                        val firstCharEncoded = compTarget.encodeString(string.value, string.altEncoding)[0]
+                        val firstCharEncoded = compTarget.encodeString(string.value, string.encoding)[0]
                         val chrout = FunctionCallStatement(
                             IdentifierReference(listOf("txt", "chrout"), pos),
                             mutableListOf(NumericLiteralValue(DataType.UBYTE, firstCharEncoded.toDouble(), pos)),
@@ -85,7 +85,7 @@ class StatementOptimizer(private val program: Program,
                         )
                         return listOf(IAstModification.ReplaceNode(functionCallStatement, chrout, parent))
                     } else if (string.value.length == 2) {
-                        val firstTwoCharsEncoded = compTarget.encodeString(string.value.take(2), string.altEncoding)
+                        val firstTwoCharsEncoded = compTarget.encodeString(string.value.take(2), string.encoding)
                         val chrout1 = FunctionCallStatement(
                             IdentifierReference(listOf("txt", "chrout"), pos),
                             mutableListOf(NumericLiteralValue(DataType.UBYTE, firstTwoCharsEncoded[0].toDouble(), pos)),
@@ -195,7 +195,7 @@ class StatementOptimizer(private val program: Program,
                 val size = sv.value.length
                 if(size==1) {
                     // loop over string of length 1 -> just assign the single character
-                    val character = compTarget.encodeString(sv.value, sv.altEncoding)[0]
+                    val character = compTarget.encodeString(sv.value, sv.encoding)[0]
                     val byte = NumericLiteralValue(DataType.UBYTE, character.toDouble(), iterable.position)
                     val scope = AnonymousScope(mutableListOf(), forLoop.position)
                     scope.statements.add(Assignment(AssignTarget(forLoop.loopVar, null, null, forLoop.position), byte, AssignmentOrigin.OPTIMIZER, forLoop.position))

compiler/src/prog8/compiler/astprocessing/AstChecker.kt

@@ -779,7 +779,7 @@ internal class AstChecker(private val program: Program,
 
     override fun visit(char: CharLiteral) {
         try {  // just *try* if it can be encoded, don't actually do it
-            compilerOptions.compTarget.encodeString(char.value.toString(), char.altEncoding)
+            compilerOptions.compTarget.encodeString(char.value.toString(), char.encoding)
         } catch (cx: CharConversionException) {
             errors.err(cx.message ?: "can't encode character", char.position)
         }
@@ -791,7 +791,7 @@ internal class AstChecker(private val program: Program,
         checkValueTypeAndRangeString(DataType.STR, string)
 
         try {  // just *try* if it can be encoded, don't actually do it
-            val bytes = compilerOptions.compTarget.encodeString(string.value, string.altEncoding)
+            val bytes = compilerOptions.compTarget.encodeString(string.value, string.encoding)
             if(0u in bytes)
                 errors.warn("a character in the string encodes into the 0-byte, which will terminate the string prematurely", string.position)
         } catch (cx: CharConversionException) {

compiler/src/prog8/compiler/astprocessing/AstExtensions.kt

@@ -51,7 +51,7 @@ internal fun Program.charLiteralsToUByteLiterals(enc: IStringEncoding) {
         override fun after(char: CharLiteral, parent: Node): Iterable<IAstModification> {
             return listOf(IAstModification.ReplaceNode(
                 char,
-                NumericLiteralValue(DataType.UBYTE, enc.encodeString(char.value.toString(), char.altEncoding)[0].toDouble(), char.position),
+                NumericLiteralValue(DataType.UBYTE, enc.encodeString(char.value.toString(), char.encoding)[0].toDouble(), char.position),
                 parent
             ))
         }

compiler/src/prog8/compiler/astprocessing/AstIdentifiersChecker.kt

@@ -9,6 +9,7 @@ import prog8.ast.expressions.StringLiteralValue
 import prog8.ast.statements.*
 import prog8.ast.walk.IAstVisitor
 import prog8.compilerinterface.BuiltinFunctions
+import prog8.compilerinterface.Encoding
 import prog8.compilerinterface.ICompilationTarget
 import prog8.compilerinterface.IErrorReporter
 
@@ -154,7 +155,7 @@ internal class AstIdentifiersChecker(private val errors: IErrorReporter,
                             else
                                 '_'
                         }.joinToString("")
-                        call.args[0] = StringLiteralValue(processed, false, name.position)
+                        call.args[0] = StringLiteralValue(processed, Encoding.PETSCII, name.position)
                         call.args[0].linkParents(call as Node)
                     }
                 }

compiler/src/prog8/compiler/astprocessing/AstVariousTransforms.kt

@@ -49,7 +49,7 @@ internal class AstVariousTransforms(private val program: Program) : AstWalker()
                 val amount = expr.right.constValue(program)
                 if(amount!=null) {
                     val string = leftStr.value.repeat(amount.number.toInt())
-                    val strval = StringLiteralValue(string, leftStr.altEncoding, expr.position)
+                    val strval = StringLiteralValue(string, leftStr.encoding, expr.position)
                     return listOf(IAstModification.ReplaceNode(expr, strval, parent))
                 }
             }
@@ -57,7 +57,7 @@ internal class AstVariousTransforms(private val program: Program) : AstWalker()
                 val amount = expr.right.constValue(program)
                 if(amount!=null) {
                     val string = rightStr.value.repeat(amount.number.toInt())
-                    val strval = StringLiteralValue(string, rightStr.altEncoding, expr.position)
+                    val strval = StringLiteralValue(string, rightStr.encoding, expr.position)
                     return listOf(IAstModification.ReplaceNode(expr, strval, parent))
                 }
             }
@@ -80,17 +80,17 @@ internal class AstVariousTransforms(private val program: Program) : AstWalker()
                 if(subStrVal==null)
                     null
                 else
-                    StringLiteralValue("${subStrVal.value}${rightStrval.value}", subStrVal.altEncoding, rightStrval.position)
+                    StringLiteralValue("${subStrVal.value}${rightStrval.value}", subStrVal.encoding, rightStrval.position)
             }
             expr.right is BinaryExpression && leftStrval!=null -> {
                 val subStrVal = concatString(expr.right as BinaryExpression)
                 if(subStrVal==null)
                     null
                 else
-                    StringLiteralValue("${leftStrval.value}${subStrVal.value}", subStrVal.altEncoding, leftStrval.position)
+                    StringLiteralValue("${leftStrval.value}${subStrVal.value}", subStrVal.encoding, leftStrval.position)
             }
             leftStrval!=null && rightStrval!=null -> {
-                StringLiteralValue("${leftStrval.value}${rightStrval.value}", leftStrval.altEncoding, leftStrval.position)
+                StringLiteralValue("${leftStrval.value}${rightStrval.value}", leftStrval.encoding, leftStrval.position)
             }
             else -> null
         }

compiler/src/prog8/compiler/astprocessing/VariousCleanups.kt

@@ -158,7 +158,7 @@ internal class VariousCleanups(val program: Program, val errors: IErrorReporter,
             if(stringVal.value.isEmpty())
                 return replaceWithFalse()
             if(stringVal.value.length==1) {
-                val string = program.encoding.encodeString(stringVal.value, stringVal.altEncoding)
+                val string = program.encoding.encodeString(stringVal.value, stringVal.encoding)
                 return replaceWithEquals(NumericLiteralValue(DataType.UBYTE, string[0].toDouble(), stringVal.position))
             }
             return noModifications

compiler/test/TestCompilerOnCharLit.kt

@@ -11,6 +11,7 @@ import prog8.ast.base.VarDeclType
 import prog8.ast.expressions.IdentifierReference
 import prog8.ast.expressions.NumericLiteralValue
 import prog8.codegen.target.Cx16Target
+import prog8.compilerinterface.Encoding
 import prog8tests.helpers.assertSuccess
 import prog8tests.helpers.compileText
 
@@ -42,7 +43,7 @@ class TestCompilerOnCharLit: FunSpec({
         }
         val arg = funCall.args[0] as NumericLiteralValue
         arg.type shouldBe DataType.UBYTE
-        arg.number shouldBe platform.encodeString("\n", false)[0].toDouble()
+        arg.number shouldBe platform.encodeString("\n", Encoding.PETSCII)[0].toDouble()
     }
 
     test("testCharVarAsRomsubArg") {
@@ -82,7 +83,7 @@ class TestCompilerOnCharLit: FunSpec({
         }
         val initializerValue = assignInitialValue.value as NumericLiteralValue
         initializerValue.type shouldBe DataType.UBYTE
-        initializerValue.number shouldBe platform.encodeString("\n", false)[0].toDouble()
+        initializerValue.number shouldBe platform.encodeString("\n", Encoding.PETSCII)[0].toDouble()
     }
 
     test("testCharConstAsRomsubArg") {
@@ -107,10 +108,10 @@ class TestCompilerOnCharLit: FunSpec({
                 val decl = arg.targetVarDecl(program)!!
                 decl.type shouldBe VarDeclType.CONST
                 decl.datatype shouldBe DataType.UBYTE
-                (decl.value as NumericLiteralValue).number shouldBe platform.encodeString("\n", false)[0]
+                (decl.value as NumericLiteralValue).number shouldBe platform.encodeString("\n", Encoding.PETSCII)[0]
             }
             is NumericLiteralValue -> {
-                arg.number shouldBe platform.encodeString("\n", false)[0].toDouble()
+                arg.number shouldBe platform.encodeString("\n", Encoding.PETSCII)[0].toDouble()
             }
             else -> fail("invalid arg type") // funCall.args[0] shouldBe instanceOf<IdentifierReference>() // make test fail
         }

compiler/test/TestCompilerOnRanges.kt

@@ -12,6 +12,7 @@ import prog8.ast.statements.ForLoop
 import prog8.ast.statements.VarDecl
 import prog8.codegen.target.C64Target
 import prog8.codegen.target.Cx16Target
+import prog8.compilerinterface.Encoding
 import prog8tests.helpers.*
 import prog8tests.helpers.ErrorReporterForTests
 import prog8tests.helpers.assertFailure
@@ -44,8 +45,8 @@ class TestCompilerOnRanges: FunSpec({
         val rhsValues = (decl.value as ArrayLiteralValue)
             .value // Array<Expression>
             .map { (it as NumericLiteralValue).number.toInt() }
-        val expectedStart = platform.encodeString("a", true)[0].toInt()
+        val expectedStart = platform.encodeString("a", Encoding.SCREENCODES)[0].toInt()
-        val expectedEnd = platform.encodeString("z", false)[0].toInt()
+        val expectedEnd = platform.encodeString("z", Encoding.PETSCII)[0].toInt()
         val expectedStr = "$expectedStart .. $expectedEnd"
 
         val actualStr = "${rhsValues.first()} .. ${rhsValues.last()}"
@@ -76,8 +77,8 @@ class TestCompilerOnRanges: FunSpec({
         val rhsValues = (decl.value as ArrayLiteralValue)
             .value // Array<Expression>
             .map { (it as NumericLiteralValue).number.toInt() }
-        val expectedStart = platform.encodeString("a", false)[0].toInt()
+        val expectedStart = platform.encodeString("a", Encoding.PETSCII)[0].toInt()
-        val expectedEnd = platform.encodeString("z", false)[0].toInt()
+        val expectedEnd = platform.encodeString("z", Encoding.PETSCII)[0].toInt()
         val expectedStr = "$expectedStart .. $expectedEnd"
 
         val actualStr = "${rhsValues.first()} .. ${rhsValues.last()}"
@@ -147,8 +148,8 @@ class TestCompilerOnRanges: FunSpec({
             .map { it.iterable }[0]
         val rangeExpr = iterable as RangeExpression
 
-        val expectedStart = platform.encodeString("a", true)[0].toInt()
+        val expectedStart = platform.encodeString("a", Encoding.SCREENCODES)[0].toInt()
-        val expectedEnd = platform.encodeString("f", false)[0].toInt()
+        val expectedEnd = platform.encodeString("f", Encoding.PETSCII)[0].toInt()
         val expectedStr = "$expectedStart .. $expectedEnd"
 
         val intProgression = rangeExpr.toConstantIntegerRange()

compiler/test/TestNumericLiteralValue.kt

@@ -12,6 +12,7 @@ import prog8.ast.expressions.ArrayLiteralValue
 import prog8.ast.expressions.InferredTypes
 import prog8.ast.expressions.NumericLiteralValue
 import prog8.ast.expressions.StringLiteralValue
+import prog8.compilerinterface.Encoding
 
 class TestNumericLiteralValue: FunSpec({
 
@@ -94,11 +95,11 @@ class TestNumericLiteralValue: FunSpec({
     }
 
     test("testEqualsRef") {
-        (StringLiteralValue("hello", false, dummyPos) == StringLiteralValue("hello", false, dummyPos)) shouldBe true
+        (StringLiteralValue("hello", Encoding.PETSCII, dummyPos) == StringLiteralValue("hello", Encoding.PETSCII, dummyPos)) shouldBe true
-        (StringLiteralValue("hello", false, dummyPos) != StringLiteralValue("bye", false, dummyPos)) shouldBe true
+        (StringLiteralValue("hello", Encoding.PETSCII, dummyPos) != StringLiteralValue("bye", Encoding.PETSCII, dummyPos)) shouldBe true
-        (StringLiteralValue("hello", true, dummyPos) == StringLiteralValue("hello", true, dummyPos)) shouldBe true
+        (StringLiteralValue("hello", Encoding.SCREENCODES, dummyPos) == StringLiteralValue("hello", Encoding.SCREENCODES, dummyPos)) shouldBe true
-        (StringLiteralValue("hello", true, dummyPos) != StringLiteralValue("bye", true, dummyPos)) shouldBe true
+        (StringLiteralValue("hello", Encoding.SCREENCODES, dummyPos) != StringLiteralValue("bye", Encoding.SCREENCODES, dummyPos)) shouldBe true
-        (StringLiteralValue("hello", true, dummyPos) != StringLiteralValue("hello", false, dummyPos)) shouldBe true
+        (StringLiteralValue("hello", Encoding.SCREENCODES, dummyPos) != StringLiteralValue("hello", Encoding.PETSCII, dummyPos)) shouldBe true
 
         val lvOne = NumericLiteralValue(DataType.UBYTE, 1.0, dummyPos)
         val lvTwo = NumericLiteralValue(DataType.UBYTE, 2.0, dummyPos)

compiler/test/ZeropageTests.kt

@@ -13,48 +13,18 @@ import io.kotest.matchers.comparables.shouldBeGreaterThan
 import io.kotest.matchers.shouldBe
 import io.kotest.matchers.shouldNotBe
 import prog8.ast.base.DataType
-import prog8.ast.expressions.Expression
-import prog8.ast.statements.RegisterOrStatusflag
-import prog8.ast.statements.Subroutine
 import prog8.codegen.target.C64Target
 import prog8.codegen.target.Cx16Target
 import prog8.codegen.target.c64.C64Zeropage
 import prog8.codegen.target.cx16.CX16Zeropage
 import prog8.compilerinterface.*
+import prog8tests.helpers.DummyCompilationTarget
 import prog8tests.helpers.ErrorReporterForTests
 import java.lang.IllegalArgumentException
 
 
 class TestAbstractZeropage: FunSpec({
 
-    class DummyCompilationTarget: ICompilationTarget {
-        override val name: String = "dummy"
-        override val machine: IMachineDefinition
-            get() = throw NotImplementedError("dummy")
-
-        override fun encodeString(str: String, altEncoding: Boolean): List<UByte> {
-            throw NotImplementedError("dummy")
-        }
-
-        override fun decodeString(bytes: List<UByte>, altEncoding: Boolean): String {
-            throw NotImplementedError("dummy")
-        }
-
-        override fun asmsubArgsEvalOrder(sub: Subroutine): List<Int> {
-            throw NotImplementedError("dummy")
-        }
-
-        override fun asmsubArgsHaveRegisterClobberRisk(args: List<Expression>,
-                                                       paramRegisters: List<RegisterOrStatusflag>): Boolean {
-            throw NotImplementedError("dummy")
-        }
-
-        override fun memorySize(dt: DataType): Int {
-            throw NotImplementedError("dummy")
-        }
-
-    }
-
     class DummyZeropage(options: CompilationOptions) : Zeropage(options) {
         override val SCRATCH_B1 = 0x10u
         override val SCRATCH_REG = 0x11u
@@ -70,7 +40,6 @@ class TestAbstractZeropage: FunSpec({
 
 
     test("testAbstractZeropage") {
-        val compTarget = DummyCompilationTarget()
         val zp = DummyZeropage(
             CompilationOptions(
                 OutputType.RAW,
@@ -79,7 +48,7 @@ class TestAbstractZeropage: FunSpec({
                 listOf((0x50u..0x5fu)),
                 false,
                 false,
-                compTarget
+                DummyCompilationTarget
             )
         )
         zp.free.size shouldBe 256-6-16

compiler/test/ast/TestProg8Parser.kt

@@ -21,6 +21,7 @@ import prog8.ast.expressions.*
 import prog8.ast.statements.*
 import prog8.codegen.target.C64Target
 import prog8.codegen.target.cbm.Petscii
+import prog8.compilerinterface.Encoding
 import prog8.parser.ParseError
 import prog8.parser.Prog8Parser.parseModule
 import prog8.parser.SourceCode
@@ -413,7 +414,7 @@ class TestProg8Parser: FunSpec( {
             char.value shouldBe '\n'
         }
 
-        test("on rhs of block-level var decl, no AltEnc") {
+        test("on rhs of block-level var decl, default encoding") {
             val src = SourceCode.Text("""
                 main {
                     ubyte c = 'x'
@@ -425,11 +426,11 @@ class TestProg8Parser: FunSpec( {
                 .statements.filterIsInstance<VarDecl>()[0]
 
             val rhs = decl.value as CharLiteral
+            rhs.encoding shouldBe Encoding.PETSCII
             rhs.value shouldBe 'x'
-            rhs.altEncoding shouldBe false
         }
 
-        test("on rhs of block-level const decl, with AltEnc") {
+        test("on rhs of block-level const decl, with screencode enc (old syntax)") {
             val src = SourceCode.Text("""
                 main {
                     const ubyte c = @'x'
@@ -441,11 +442,44 @@ class TestProg8Parser: FunSpec( {
                 .statements.filterIsInstance<VarDecl>()[0]
 
             val rhs = decl.value as CharLiteral
+            rhs.encoding shouldBe Encoding.SCREENCODES
             rhs.value shouldBe 'x'
-            rhs.altEncoding shouldBe true
         }
 
-        test("on rhs of subroutine-level var decl, no AltEnc") {
+        xtest("on rhs of block-level const decl, with screencode enc (new syntax)") {
+            val src = SourceCode.Text("""
+                main {
+                    const ubyte c = sc:'x'
+                }
+            """)
+            val module = parseModule(src)
+            val decl = module
+                .statements.filterIsInstance<Block>()[0]
+                .statements.filterIsInstance<VarDecl>()[0]
+
+            val rhs = decl.value as CharLiteral
+            rhs.encoding shouldBe Encoding.SCREENCODES
+            rhs.value shouldBe 'x'
+        }
+
+        xtest("on rhs of block-level const decl, with iso encoding") {
+            val src = SourceCode.Text("""
+                main {
+                    const ubyte c = iso:'_'
+                }
+            """)
+            val module = parseModule(src)
+            val decl = module
+                .statements.filterIsInstance<Block>()[0]
+                .statements.filterIsInstance<VarDecl>()[0]
+
+            val rhs = decl.value as CharLiteral
+            rhs.encoding shouldBe Encoding.ISO
+            rhs.value shouldBe '_'
+        }
+
+
+        test("on rhs of subroutine-level var decl, default encoding") {
             val src = SourceCode.Text("""
                 main {
                     sub start() {
@@ -461,10 +495,10 @@ class TestProg8Parser: FunSpec( {
 
             val rhs = decl.value as CharLiteral
             rhs.value shouldBe 'x'
-            rhs.altEncoding shouldBe false
+            rhs.encoding shouldBe Encoding.PETSCII
         }
 
-        test("on rhs of subroutine-level const decl, with AltEnc") {
+        test("on rhs of subroutine-level const decl, screencode (old syntax)") {
             val src = SourceCode.Text("""
                 main {
                     sub start() {
@@ -479,9 +513,54 @@ class TestProg8Parser: FunSpec( {
                 .statements.filterIsInstance<VarDecl>()[0]
 
             val rhs = decl.value as CharLiteral
+            rhs.encoding shouldBe Encoding.SCREENCODES
             rhs.value shouldBe 'x'
-            rhs.altEncoding shouldBe true
         }
+
+        xtest("on rhs of subroutine-level const decl, screencode (new syntax)") {
+            val src = SourceCode.Text("""
+                main {
+                    sub start() {
+                        const ubyte c = sc:'x'
+                    }
+                }
+            """)
+            val module = parseModule(src)
+            val decl = module
+                .statements.filterIsInstance<Block>()[0]
+                .statements.filterIsInstance<Subroutine>()[0]
+                .statements.filterIsInstance<VarDecl>()[0]
+
+            val rhs = decl.value as CharLiteral
+            rhs.encoding shouldBe Encoding.SCREENCODES
+            rhs.value shouldBe 'x'
+        }
+
+        xtest("on rhs of subroutine-level const decl, iso encoding") {
+            val src = SourceCode.Text("""
+                main {
+                    sub start() {
+                        const ubyte c = iso:'_'
+                    }
+                }
+            """)
+            val module = parseModule(src)
+            val decl = module
+                .statements.filterIsInstance<Block>()[0]
+                .statements.filterIsInstance<Subroutine>()[0]
+                .statements.filterIsInstance<VarDecl>()[0]
+
+            val rhs = decl.value as CharLiteral
+            rhs.encoding shouldBe Encoding.ISO
+            rhs.value shouldBe '_'
+        }
+    }
+
+    context("Strings") {
+
+        // TODO test encoding in all available encodings
+        // check that '~' cant be encoded in petscii and screencode
+        // check that '~' CAN be encoded correctly in iso
     }
 
     context("Ranges") {
@@ -534,12 +613,14 @@ class TestProg8Parser: FunSpec( {
     }
 
     test("testCharLiteralConstValue") {
-        val char1 = CharLiteral('A', false, Position.DUMMY)
+        val char1 = CharLiteral('A', Encoding.PETSCII, Position.DUMMY)
-        val char2 = CharLiteral('z', true, Position.DUMMY)
+        val char2 = CharLiteral('z', Encoding.SCREENCODES, Position.DUMMY)
+        val char3 = CharLiteral('_', Encoding.ISO, Position.DUMMY)
 
         val program = Program("test", DummyFunctions, DummyMemsizer, AsciiStringEncoder)
         char1.constValue(program).number.toInt() shouldBe 65
         char2.constValue(program).number.toInt() shouldBe 122
+        char3.constValue(program).number.toInt() shouldBe 95
     }
 
     test("testLiteralValueComparisons") {
@@ -560,8 +641,8 @@ class TestProg8Parser: FunSpec( {
         (ten <= ten) shouldBe true
         (ten < ten) shouldBe false
 
-        val abc = StringLiteralValue("abc", false, Position.DUMMY)
+        val abc = StringLiteralValue("abc", Encoding.PETSCII, Position.DUMMY)
-        val abd = StringLiteralValue("abd", false, Position.DUMMY)
+        val abd = StringLiteralValue("abd", Encoding.PETSCII, Position.DUMMY)
         abc shouldBe abc
         (abc!=abd) shouldBe true
         (abc!=abc) shouldBe false
@@ -726,7 +807,6 @@ class TestProg8Parser: FunSpec( {
         ubexpr.inferType(program).getOrElse { fail("dt") } shouldBe DataType.UBYTE
     }
 
-
     test("assignment isAugmented correctness") {
         val src = SourceCode.Text("""
             main {

compiler/test/helpers/Dummies.kt

@@ -6,8 +6,9 @@ import prog8.ast.base.Position
 import prog8.ast.expressions.Expression
 import prog8.ast.expressions.InferredTypes
 import prog8.ast.expressions.NumericLiteralValue
-import prog8.compilerinterface.IMemSizer
+import prog8.ast.statements.RegisterOrStatusflag
-import prog8.compilerinterface.IStringEncoding
+import prog8.ast.statements.Subroutine
+import prog8.compilerinterface.*
 
 internal val DummyFunctions = object : IBuiltinFunctions {
     override val names: Set<String> = emptySet()
@@ -26,19 +27,46 @@ internal val DummyMemsizer = object : IMemSizer {
 }
 
 internal val DummyStringEncoder = object : IStringEncoding {
-    override fun encodeString(str: String, altEncoding: Boolean): List<UByte> {
+    override fun encodeString(str: String, encoding: Encoding): List<UByte> {
         return emptyList()
     }
 
-    override fun decodeString(bytes: List<UByte>, altEncoding: Boolean): String {
+    override fun decodeString(bytes: List<UByte>, encoding: Encoding): String {
         return ""
     }
 }
 
 internal val AsciiStringEncoder = object : IStringEncoding {
-    override fun encodeString(str: String, altEncoding: Boolean): List<UByte> = str.map { it.code.toUByte() }
+    override fun encodeString(str: String, encoding: Encoding): List<UByte> = str.map { it.code.toUByte() }
 
-    override fun decodeString(bytes: List<UByte>, altEncoding: Boolean): String {
+    override fun decodeString(bytes: List<UByte>, encoding: Encoding): String {
         return bytes.joinToString()
     }
 }
+
+internal val DummyCompilationTarget = object : ICompilationTarget {
+    override val name: String = "dummy"
+    override val machine: IMachineDefinition
+        get() = throw NotImplementedError("dummy")
+
+    override fun encodeString(str: String, encoding: Encoding): List<UByte> {
+        throw NotImplementedError("dummy")
+    }
+
+    override fun decodeString(bytes: List<UByte>, encoding: Encoding): String {
+        throw NotImplementedError("dummy")
+    }
+
+    override fun asmsubArgsEvalOrder(sub: Subroutine): List<Int> {
+        throw NotImplementedError("dummy")
+    }
+
+    override fun asmsubArgsHaveRegisterClobberRisk(args: List<Expression>,
+                                                   paramRegisters: List<RegisterOrStatusflag>): Boolean {
+        throw NotImplementedError("dummy")
+    }
+
+    override fun memorySize(dt: DataType): Int {
+        throw NotImplementedError("dummy")
+    }
+}

compilerAst/src/prog8/ast/AstToSourceTextConverter.kt

@@ -2,11 +2,13 @@ package prog8.ast
 
 import prog8.ast.antlr.escape
 import prog8.ast.base.DataType
+import prog8.ast.base.FatalAstException
 import prog8.ast.base.NumericDatatypes
 import prog8.ast.base.VarDeclType
 import prog8.ast.expressions.*
 import prog8.ast.statements.*
 import prog8.ast.walk.IAstVisitor
+import prog8.compilerinterface.Encoding
 
 
 /**
@@ -286,15 +288,11 @@ class AstToSourceTextConverter(val output: (text: String) -> Unit, val program:
     }
 
     override fun visit(char: CharLiteral) {
-        if (char.altEncoding)
+        output("${char.encoding.prefix}:'${escape(char.value.toString())}'")
-            output("@")
-        output("'${escape(char.value.toString())}'")
     }
 
     override fun visit(string: StringLiteralValue) {
-        if (string.altEncoding)
+        output("${string.encoding.prefix}:\"${escape(string.value)}\"")
-            output("@")
-        output("\"${escape(string.value)}\"")
     }
 
     override fun visit(array: ArrayLiteralValue) {

compilerAst/src/prog8/ast/Program.kt

@@ -7,6 +7,7 @@ import prog8.ast.base.VarDeclType
 import prog8.ast.expressions.ContainmentCheck
 import prog8.ast.expressions.StringLiteralValue
 import prog8.ast.statements.*
+import prog8.compilerinterface.Encoding
 import prog8.compilerinterface.IMemSizer
 import prog8.compilerinterface.IStringEncoding
 import prog8.parser.SourceCode
@@ -74,7 +75,7 @@ class Program(val name: String,
         get() = toplevelModule.loadAddress
 
     var actualLoadAddress = 0u
-    private val internedStringsUnique = mutableMapOf<Pair<String, Boolean>, List<String>>()
+    private val internedStringsUnique = mutableMapOf<Pair<String, Encoding>, List<String>>()
 
     fun internString(string: StringLiteralValue): List<String> {
         // Move a string literal into the internal, deduplicated, string pool
@@ -99,7 +100,7 @@ class Program(val name: String,
             return listOf(internedStringsModuleName, decl.name)
         }
 
-        val key = Pair(string.value, string.altEncoding)
+        val key = Pair(string.value, string.encoding)
         val existing = internedStringsUnique[key]
         if (existing != null)
             return existing

compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt

@@ -5,6 +5,7 @@ import org.antlr.v4.runtime.tree.TerminalNode
 import prog8.ast.base.*
 import prog8.ast.expressions.*
 import prog8.ast.statements.*
+import prog8.compilerinterface.Encoding
 import prog8.parser.Prog8ANTLRParser
 import prog8.parser.SourceCode
 import java.nio.file.Path
@@ -450,12 +451,16 @@ private fun Prog8ANTLRParser.ExpressionContext.toAst() : Expression {
 
 private fun Prog8ANTLRParser.CharliteralContext.toAst(): CharLiteral {
     val text = this.SINGLECHAR().text
-    return CharLiteral(unescape(text.substring(1, text.length-1), toPosition())[0], this.ALT_STRING_ENCODING() != null, toPosition())
+    // TODO ISO-encoding, alternative encoding syntax
+    val encoding = if(ALT_STRING_ENCODING()==null) Encoding.PETSCII else Encoding.SCREENCODES
+    return CharLiteral(unescape(text.substring(1, text.length-1), toPosition())[0], encoding, toPosition())
 }
 
 private fun Prog8ANTLRParser.StringliteralContext.toAst(): StringLiteralValue {
     val text=this.STRING().text
-    return StringLiteralValue(unescape(text.substring(1, text.length-1), toPosition()), ALT_STRING_ENCODING() != null, toPosition())
+    // TODO ISO-encoding, alternative encoding syntax
+    val encoding = if(ALT_STRING_ENCODING()==null) Encoding.PETSCII else Encoding.SCREENCODES
+    return StringLiteralValue(unescape(text.substring(1, text.length-1), toPosition()), encoding, toPosition())
 }
 
 private fun Prog8ANTLRParser.ArrayindexedContext.toAst(): ArrayIndexedExpression {

compilerAst/src/prog8/ast/expressions/AstExpressions.kt

@@ -6,6 +6,7 @@ import prog8.ast.base.*
 import prog8.ast.statements.*
 import prog8.ast.walk.AstWalker
 import prog8.ast.walk.IAstVisitor
+import prog8.compilerinterface.Encoding
 import java.util.*
 import kotlin.math.abs
 import kotlin.math.round
@@ -589,7 +590,7 @@ class NumericLiteralValue(val type: DataType,    // only numerical types allowed
 }
 
 class CharLiteral(val value: Char,
-                 val altEncoding: Boolean,          // such as: screencodes instead of Petscii for the C64
+                  val encoding: Encoding,
                   override val position: Position) : Expression() {
     override lateinit var parent: Node
 
@@ -603,10 +604,10 @@ class CharLiteral(val value: Char,
         throw FatalAstException("can't replace here")
     }
 
-    override fun copy() = CharLiteral(value, altEncoding, position)
+    override fun copy() = CharLiteral(value, encoding, position)
     override fun referencesIdentifier(nameInSource: List<String>) = false
     override fun constValue(program: Program): NumericLiteralValue {
-        val bytevalue = program.encoding.encodeString(value.toString(), altEncoding).single()
+        val bytevalue = program.encoding.encodeString(value.toString(), encoding).single()
         return NumericLiteralValue(DataType.UBYTE, bytevalue.toDouble(), position)
     }
     override fun accept(visitor: IAstVisitor) = visitor.visit(this)
@@ -615,16 +616,16 @@ class CharLiteral(val value: Char,
     override fun toString(): String = "'${escape(value.toString())}'"
     override fun inferType(program: Program) = InferredTypes.knownFor(DataType.UBYTE)
     operator fun compareTo(other: CharLiteral): Int = value.compareTo(other.value)
-    override fun hashCode(): Int = Objects.hash(value, altEncoding)
+    override fun hashCode(): Int = Objects.hash(value, encoding)
     override fun equals(other: Any?): Boolean {
         if (other == null || other !is CharLiteral)
             return false
-        return value == other.value && altEncoding == other.altEncoding
+        return value == other.value && encoding == other.encoding
     }
 }
 
 class StringLiteralValue(val value: String,
-                         val altEncoding: Boolean,          // such as: screencodes instead of Petscii for the C64
+                         val encoding: Encoding,
                          override val position: Position) : Expression() {
     override lateinit var parent: Node
 
@@ -633,7 +634,7 @@ class StringLiteralValue(val value: String,
     }
 
     override val isSimple = true
-    override fun copy() = StringLiteralValue(value, altEncoding, position)
+    override fun copy() = StringLiteralValue(value, encoding, position)
 
     override fun replaceChildNode(node: Node, replacement: Node) {
         throw FatalAstException("can't replace here")
@@ -647,11 +648,11 @@ class StringLiteralValue(val value: String,
     override fun toString(): String = "'${escape(value)}'"
     override fun inferType(program: Program) = InferredTypes.knownFor(DataType.STR)
     operator fun compareTo(other: StringLiteralValue): Int = value.compareTo(other.value)
-    override fun hashCode(): Int = Objects.hash(value, altEncoding)
+    override fun hashCode(): Int = Objects.hash(value, encoding)
     override fun equals(other: Any?): Boolean {
         if(other==null || other !is StringLiteralValue)
             return false
-        return value==other.value && altEncoding == other.altEncoding
+        return value==other.value && encoding == other.encoding
     }
 }
 
@@ -1025,7 +1026,7 @@ class ContainmentCheck(var element: Expression,
                 is StringLiteralValue -> {
                     if(elementConst.type in ByteDatatypes) {
                         val stringval = iterable as StringLiteralValue
-                        val exists = program.encoding.encodeString(stringval.value, stringval.altEncoding).contains(elementConst.number.toInt().toUByte() )
+                        val exists = program.encoding.encodeString(stringval.value, stringval.encoding).contains(elementConst.number.toInt().toUByte() )
                         return NumericLiteralValue.fromBoolean(exists, position)
                     }
                 }

compilerAst/src/prog8/compilerinterface/IStringEncoding.kt

@@ -1,6 +1,12 @@
 package prog8.compilerinterface
 
-interface IStringEncoding {
+enum class Encoding(val prefix: String) {
-    fun encodeString(str: String, altEncoding: Boolean): List<UByte>
+    PETSCII("petscii"),           // c64/c128/cx16
-    fun decodeString(bytes: List<UByte>, altEncoding: Boolean): String
+    SCREENCODES("sc"),            // c64/c128/cx16
+    ISO("iso")                    // cx16
+}
+
+interface IStringEncoding {
+    fun encodeString(str: String, encoding: Encoding): List<UByte>
+    fun decodeString(bytes: List<UByte>, encoding: Encoding): String
 }

compilerInterfaces/src/prog8/compilerinterface/ICompilationTarget.kt

@@ -4,11 +4,14 @@ import prog8.ast.expressions.Expression
 import prog8.ast.statements.RegisterOrStatusflag
 import prog8.ast.statements.Subroutine
 
+
+// TODO list of supported string encodings
+
 interface ICompilationTarget: IStringEncoding, IMemSizer {
     val name: String
     val machine: IMachineDefinition
-    override fun encodeString(str: String, altEncoding: Boolean): List<UByte>
+    override fun encodeString(str: String, encoding: Encoding): List<UByte>
-    override fun decodeString(bytes: List<UByte>, altEncoding: Boolean): String
+    override fun decodeString(bytes: List<UByte>, encoding: Encoding): String
 
     fun asmsubArgsEvalOrder(sub: Subroutine): List<Int>
     fun asmsubArgsHaveRegisterClobberRisk(args: List<Expression>,

docs/source/todo.rst

@@ -3,7 +3,9 @@ TODO
 
 For next compiler release (7.7)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-...
+- implement iso encoding and alternate encoding syntax
+- document new encoding syntax
+
 
 
 Need help with

examples/test.p8

@@ -1,21 +1,41 @@
 %import textio
+%zeropage basicsafe
 
 main {
-    str myBar = "main.bar"
+    str s1 = "Irmen_"
-
+    str s2 = @"IRMEN_"
-foo_bar:
+    ;str s3 = iso:"Irmen_~"
-    %asm {{
-        nop
-    }}
 
     sub start() {
-	    txt.print(myBar)
+        txt.lowercase()
+        txt.nl()
+        txt.nl()
+        txt.nl()
+        txt.nl()
+        txt.nl()
+        txt.print(s1)
+        txt.nl()
+        txt.print(s2)
+        txt.nl()
+;        txt.print(s3)
+;        txt.nl()
 
-	    %breakpoint
+        sc(1, s1)
+        sc(2, s2)
+        ; sc(3, s3)
+    }
 
-	    txt.print_uwhex(&foo_bar, true)
+    sub sc(ubyte row, str text) {
-
+        uword addr = 1024+row*40
-	    %breakpoint
+        ubyte ix = 0
+        ubyte ss
+        repeat {
+            ss = text[ix]
+            if not ss
                 return
+            @(addr) = ss
+            addr++
+            ix++
+        }
     }
 }