Apple-2-SW/prog8
1
0
Fork 0
mirror of https://github.com/irmen/prog8.git synced 2024-07-05 06:29:02 +00:00
Code Issues Projects Releases Wiki Activity

generalize string encoding flag into enum

Browse Source
This commit is contained in:
Irmen de Jong 2022-01-18 21:21:49 +01:00
parent 72640ae058
commit 3b59592110
25 changed files with 289 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
codeGeneration/src/prog8/codegen/target/C128Target.kt
View File

@ -9,21 +9,31 @@ import prog8.codegen.target.c128.C128MachineDefinition
import prog8.codegen.target.cbm.Petscii
import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsEvalOrder
import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsHaveRegisterClobberRisk
import prog8.compilerinterface.Encoding
import prog8.compilerinterface.ICompilationTarget
object C128Target: ICompilationTarget {
override val name = "c128"
override val machine = C128MachineDefinition()
override fun encodeString(str: String, altEncoding: Boolean): List<UByte> {
val coded = if (altEncoding) Petscii.encodeScreencode(str, true) else Petscii.encodePetscii(str, true)
override fun encodeString(str: String, encoding: Encoding): List<UByte> {
val coded = when(encoding) {
Encoding.PETSCII -> Petscii.encodePetscii(str, true)
Encoding.SCREENCODES -> Petscii.encodeScreencode(str, true)
else -> throw FatalAstException("unsupported encoding $encoding")
}
return coded.fold(
failure = { throw it },
success = { it }
)
}
override fun decodeString(bytes: List<UByte>, altEncoding: Boolean) =
if (altEncoding) Petscii.decodeScreencode(bytes, true) else Petscii.decodePetscii(bytes, true)
override fun decodeString(bytes: List<UByte>, encoding: Encoding): String {
return when(encoding) {
Encoding.PETSCII -> Petscii.decodePetscii(bytes, true)
Encoding.SCREENCODES -> Petscii.decodeScreencode(bytes, true)
else -> throw FatalAstException("unsupported encoding $encoding")
}
}
override fun asmsubArgsEvalOrder(sub: Subroutine): List<Int> =
asmsub6502ArgsEvalOrder(sub)

19
codeGeneration/src/prog8/codegen/target/C64Target.kt
View File

@ -9,21 +9,32 @@ import prog8.codegen.target.c64.C64MachineDefinition
import prog8.codegen.target.cbm.Petscii
import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsEvalOrder
import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsHaveRegisterClobberRisk
import prog8.compilerinterface.Encoding
import prog8.compilerinterface.ICompilationTarget
object C64Target: ICompilationTarget {
override val name = "c64"
override val machine = C64MachineDefinition()
override fun encodeString(str: String, altEncoding: Boolean): List<UByte> {
val coded = if (altEncoding) Petscii.encodeScreencode(str, true) else Petscii.encodePetscii(str, true)
override fun encodeString(str: String, encoding: Encoding): List<UByte> {
val coded = when(encoding) {
Encoding.PETSCII -> Petscii.encodePetscii(str, true)
Encoding.SCREENCODES -> Petscii.encodeScreencode(str, true)
else -> throw FatalAstException("unsupported encoding $encoding")
}
return coded.fold(
failure = { throw it },
success = { it }
)
}
override fun decodeString(bytes: List<UByte>, altEncoding: Boolean) =
if (altEncoding) Petscii.decodeScreencode(bytes, true) else Petscii.decodePetscii(bytes, true)
override fun decodeString(bytes: List<UByte>, encoding: Encoding): String {
return when(encoding) {
Encoding.PETSCII -> Petscii.decodePetscii(bytes, true)
Encoding.SCREENCODES -> Petscii.decodeScreencode(bytes, true)
else -> throw FatalAstException("unsupported encoding $encoding")
}
}
override fun asmsubArgsEvalOrder(sub: Subroutine): List<Int> =
asmsub6502ArgsEvalOrder(sub)

26
codeGeneration/src/prog8/codegen/target/Cx16Target.kt
View File

@ -1,10 +1,7 @@
package prog8.codegen.target
import com.github.michaelbull.result.fold
import prog8.ast.base.ByteDatatypes
import prog8.ast.base.DataType
import prog8.ast.base.PassByReferenceDatatypes
import prog8.ast.base.WordDatatypes
import prog8.ast.base.*
import prog8.ast.expressions.Expression
import prog8.ast.statements.RegisterOrStatusflag
import prog8.ast.statements.Subroutine
@ -12,21 +9,34 @@ import prog8.codegen.target.cbm.Petscii
import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsEvalOrder
import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsHaveRegisterClobberRisk
import prog8.codegen.target.cx16.CX16MachineDefinition
import prog8.compilerinterface.Encoding
import prog8.compilerinterface.ICompilationTarget
object Cx16Target: ICompilationTarget {
override val name = "cx16"
override val machine = CX16MachineDefinition()
override fun encodeString(str: String, altEncoding: Boolean): List<UByte> {
val coded = if (altEncoding) Petscii.encodeScreencode(str, true) else Petscii.encodePetscii(str, true)
override fun encodeString(str: String, encoding: Encoding): List<UByte> {
val coded = when(encoding) {
Encoding.PETSCII -> Petscii.encodePetscii(str, true)
Encoding.SCREENCODES -> Petscii.encodeScreencode(str, true)
Encoding.ISO -> TODO("cx16 iso-encoding")
else -> throw FatalAstException("unsupported encoding $encoding")
}
return coded.fold(
failure = { throw it },
success = { it }
)
}
override fun decodeString(bytes: List<UByte>, altEncoding: Boolean) =
if (altEncoding) Petscii.decodeScreencode(bytes, true) else Petscii.decodePetscii(bytes, true)
override fun decodeString(bytes: List<UByte>, encoding: Encoding): String {
return when(encoding) {
Encoding.PETSCII -> Petscii.decodePetscii(bytes, true)
Encoding.SCREENCODES -> Petscii.decodeScreencode(bytes, true)
Encoding.ISO -> TODO("cx16 iso-encoding")
else -> throw FatalAstException("unsupported encoding $encoding")
}
}
override fun asmsubArgsEvalOrder(sub: Subroutine): List<Int> =
asmsub6502ArgsEvalOrder(sub)

5
codeGeneration/src/prog8/codegen/target/cpu6502/codegen/AsmGen.kt
View File

@ -525,9 +525,8 @@ class AsmGen(private val program: Program,
private fun outputStringvar(strdecl: VarDecl, nameOverride: String?=null) {
val varname = nameOverride ?: strdecl.name
val sv = strdecl.value as StringLiteralValue
val altEncoding = if(sv.altEncoding) "@" else ""
out("$varname\t; ${strdecl.datatype} $altEncoding\"${escape(sv.value).replace("\u0000", "<NULL>")}\"")
val bytes = compTarget.encodeString(sv.value, sv.altEncoding).plus(0.toUByte())
out("$varname\t; ${strdecl.datatype} ${sv.encoding}:\"${escape(sv.value).replace("\u0000", "<NULL>")}\"")
val bytes = compTarget.encodeString(sv.value, sv.encoding).plus(0.toUByte())
val outputBytes = bytes.map { "$" + it.toString(16).padStart(2, '0') }
for (chunk in outputBytes.chunked(16))
out(" .byte " + chunk.joinToString())

4
codeGeneration/src/prog8/codegen/target/cpu6502/codegen/assignment/AssignmentAsmGen.kt
View File

@ -361,7 +361,7 @@ internal class AssignmentAsmGen(private val program: Program, private val asmgen
DataType.STR -> {
require(elementDt.isBytes)
val stringVal = variable.value as StringLiteralValue
val encoded = program.encoding.encodeString(stringVal.value, stringVal.altEncoding)
val encoded = program.encoding.encodeString(stringVal.value, stringVal.encoding)
return containmentCheckIntoA(containment.element, elementDt.getOr(DataType.UNDEFINED), encoded.map { it.toInt() })
}
DataType.ARRAY_F -> {
@ -410,7 +410,7 @@ internal class AssignmentAsmGen(private val program: Program, private val asmgen
val stringVal = containment.iterable as? StringLiteralValue
if(stringVal!=null) {
require(elementDt.isBytes)
val encoded = program.encoding.encodeString(stringVal.value, stringVal.altEncoding)
val encoded = program.encoding.encodeString(stringVal.value, stringVal.encoding)
return containmentCheckIntoA(containment.element, elementDt.getOr(DataType.UNDEFINED), encoded.map { it.toInt() })
}
val arrayVal = containment.iterable as? ArrayLiteralValue

6
codeOptimizers/src/prog8/optimizer/StatementOptimizer.kt
View File

@ -77,7 +77,7 @@ class StatementOptimizer(private val program: Program,
if(string!=null) {
val pos = functionCallStatement.position
if (string.value.length == 1) {
val firstCharEncoded = compTarget.encodeString(string.value, string.altEncoding)[0]
val firstCharEncoded = compTarget.encodeString(string.value, string.encoding)[0]
val chrout = FunctionCallStatement(
IdentifierReference(listOf("txt", "chrout"), pos),
mutableListOf(NumericLiteralValue(DataType.UBYTE, firstCharEncoded.toDouble(), pos)),
@ -85,7 +85,7 @@ class StatementOptimizer(private val program: Program,
)
return listOf(IAstModification.ReplaceNode(functionCallStatement, chrout, parent))
} else if (string.value.length == 2) {
val firstTwoCharsEncoded = compTarget.encodeString(string.value.take(2), string.altEncoding)
val firstTwoCharsEncoded = compTarget.encodeString(string.value.take(2), string.encoding)
val chrout1 = FunctionCallStatement(
IdentifierReference(listOf("txt", "chrout"), pos),
mutableListOf(NumericLiteralValue(DataType.UBYTE, firstTwoCharsEncoded[0].toDouble(), pos)),
@ -195,7 +195,7 @@ class StatementOptimizer(private val program: Program,
val size = sv.value.length
if(size==1) {
// loop over string of length 1 -> just assign the single character
val character = compTarget.encodeString(sv.value, sv.altEncoding)[0]
val character = compTarget.encodeString(sv.value, sv.encoding)[0]
val byte = NumericLiteralValue(DataType.UBYTE, character.toDouble(), iterable.position)
val scope = AnonymousScope(mutableListOf(), forLoop.position)
scope.statements.add(Assignment(AssignTarget(forLoop.loopVar, null, null, forLoop.position), byte, AssignmentOrigin.OPTIMIZER, forLoop.position))

4
compiler/src/prog8/compiler/astprocessing/AstChecker.kt
View File

@ -779,7 +779,7 @@ internal class AstChecker(private val program: Program,
override fun visit(char: CharLiteral) {
try { // just *try* if it can be encoded, don't actually do it
compilerOptions.compTarget.encodeString(char.value.toString(), char.altEncoding)
compilerOptions.compTarget.encodeString(char.value.toString(), char.encoding)
} catch (cx: CharConversionException) {
errors.err(cx.message ?: "can't encode character", char.position)
}
@ -791,7 +791,7 @@ internal class AstChecker(private val program: Program,
checkValueTypeAndRangeString(DataType.STR, string)
try { // just *try* if it can be encoded, don't actually do it
val bytes = compilerOptions.compTarget.encodeString(string.value, string.altEncoding)
val bytes = compilerOptions.compTarget.encodeString(string.value, string.encoding)
if(0u in bytes)
errors.warn("a character in the string encodes into the 0-byte, which will terminate the string prematurely", string.position)
} catch (cx: CharConversionException) {

2
compiler/src/prog8/compiler/astprocessing/AstExtensions.kt
View File

@ -51,7 +51,7 @@ internal fun Program.charLiteralsToUByteLiterals(enc: IStringEncoding) {
override fun after(char: CharLiteral, parent: Node): Iterable<IAstModification> {
return listOf(IAstModification.ReplaceNode(
char,
NumericLiteralValue(DataType.UBYTE, enc.encodeString(char.value.toString(), char.altEncoding)[0].toDouble(), char.position),
NumericLiteralValue(DataType.UBYTE, enc.encodeString(char.value.toString(), char.encoding)[0].toDouble(), char.position),
parent
))
}

3
compiler/src/prog8/compiler/astprocessing/AstIdentifiersChecker.kt
View File

@ -9,6 +9,7 @@ import prog8.ast.expressions.StringLiteralValue
import prog8.ast.statements.*
import prog8.ast.walk.IAstVisitor
import prog8.compilerinterface.BuiltinFunctions
import prog8.compilerinterface.Encoding
import prog8.compilerinterface.ICompilationTarget
import prog8.compilerinterface.IErrorReporter
@ -154,7 +155,7 @@ internal class AstIdentifiersChecker(private val errors: IErrorReporter,
else
'_'
}.joinToString("")
call.args[0] = StringLiteralValue(processed, false, name.position)
call.args[0] = StringLiteralValue(processed, Encoding.PETSCII, name.position)
call.args[0].linkParents(call as Node)
}
}

10
compiler/src/prog8/compiler/astprocessing/AstVariousTransforms.kt
View File

@ -49,7 +49,7 @@ internal class AstVariousTransforms(private val program: Program) : AstWalker()
val amount = expr.right.constValue(program)
if(amount!=null) {
val string = leftStr.value.repeat(amount.number.toInt())
val strval = StringLiteralValue(string, leftStr.altEncoding, expr.position)
val strval = StringLiteralValue(string, leftStr.encoding, expr.position)
return listOf(IAstModification.ReplaceNode(expr, strval, parent))
}
}
@ -57,7 +57,7 @@ internal class AstVariousTransforms(private val program: Program) : AstWalker()
val amount = expr.right.constValue(program)
if(amount!=null) {
val string = rightStr.value.repeat(amount.number.toInt())
val strval = StringLiteralValue(string, rightStr.altEncoding, expr.position)
val strval = StringLiteralValue(string, rightStr.encoding, expr.position)
return listOf(IAstModification.ReplaceNode(expr, strval, parent))
}
}
@ -80,17 +80,17 @@ internal class AstVariousTransforms(private val program: Program) : AstWalker()
if(subStrVal==null)
null
else
StringLiteralValue("${subStrVal.value}${rightStrval.value}", subStrVal.altEncoding, rightStrval.position)
StringLiteralValue("${subStrVal.value}${rightStrval.value}", subStrVal.encoding, rightStrval.position)
}
expr.right is BinaryExpression && leftStrval!=null -> {
val subStrVal = concatString(expr.right as BinaryExpression)
if(subStrVal==null)
null
else
StringLiteralValue("${leftStrval.value}${subStrVal.value}", subStrVal.altEncoding, leftStrval.position)
StringLiteralValue("${leftStrval.value}${subStrVal.value}", subStrVal.encoding, leftStrval.position)
}
leftStrval!=null && rightStrval!=null -> {
StringLiteralValue("${leftStrval.value}${rightStrval.value}", leftStrval.altEncoding, leftStrval.position)
StringLiteralValue("${leftStrval.value}${rightStrval.value}", leftStrval.encoding, leftStrval.position)
}
else -> null
}

2
compiler/src/prog8/compiler/astprocessing/VariousCleanups.kt
View File

@ -158,7 +158,7 @@ internal class VariousCleanups(val program: Program, val errors: IErrorReporter,
if(stringVal.value.isEmpty())
return replaceWithFalse()
if(stringVal.value.length==1) {
val string = program.encoding.encodeString(stringVal.value, stringVal.altEncoding)
val string = program.encoding.encodeString(stringVal.value, stringVal.encoding)
return replaceWithEquals(NumericLiteralValue(DataType.UBYTE, string[0].toDouble(), stringVal.position))
}
return noModifications

9
compiler/test/TestCompilerOnCharLit.kt
View File

@ -11,6 +11,7 @@ import prog8.ast.base.VarDeclType
import prog8.ast.expressions.IdentifierReference
import prog8.ast.expressions.NumericLiteralValue
import prog8.codegen.target.Cx16Target
import prog8.compilerinterface.Encoding
import prog8tests.helpers.assertSuccess
import prog8tests.helpers.compileText
@ -42,7 +43,7 @@ class TestCompilerOnCharLit: FunSpec({
}
val arg = funCall.args[0] as NumericLiteralValue
arg.type shouldBe DataType.UBYTE
arg.number shouldBe platform.encodeString("\n", false)[0].toDouble()
arg.number shouldBe platform.encodeString("\n", Encoding.PETSCII)[0].toDouble()
}
test("testCharVarAsRomsubArg") {
@ -82,7 +83,7 @@ class TestCompilerOnCharLit: FunSpec({
}
val initializerValue = assignInitialValue.value as NumericLiteralValue
initializerValue.type shouldBe DataType.UBYTE
initializerValue.number shouldBe platform.encodeString("\n", false)[0].toDouble()
initializerValue.number shouldBe platform.encodeString("\n", Encoding.PETSCII)[0].toDouble()
}
test("testCharConstAsRomsubArg") {
@ -107,10 +108,10 @@ class TestCompilerOnCharLit: FunSpec({
val decl = arg.targetVarDecl(program)!!
decl.type shouldBe VarDeclType.CONST
decl.datatype shouldBe DataType.UBYTE
(decl.value as NumericLiteralValue).number shouldBe platform.encodeString("\n", false)[0]
(decl.value as NumericLiteralValue).number shouldBe platform.encodeString("\n", Encoding.PETSCII)[0]
}
is NumericLiteralValue -> {
arg.number shouldBe platform.encodeString("\n", false)[0].toDouble()
arg.number shouldBe platform.encodeString("\n", Encoding.PETSCII)[0].toDouble()
}
else -> fail("invalid arg type") // funCall.args[0] shouldBe instanceOf<IdentifierReference>() // make test fail
}

13
compiler/test/TestCompilerOnRanges.kt
View File

@ -12,6 +12,7 @@ import prog8.ast.statements.ForLoop
import prog8.ast.statements.VarDecl
import prog8.codegen.target.C64Target
import prog8.codegen.target.Cx16Target
import prog8.compilerinterface.Encoding
import prog8tests.helpers.*
import prog8tests.helpers.ErrorReporterForTests
import prog8tests.helpers.assertFailure
@ -44,8 +45,8 @@ class TestCompilerOnRanges: FunSpec({
val rhsValues = (decl.value as ArrayLiteralValue)
.value // Array<Expression>
.map { (it as NumericLiteralValue).number.toInt() }
val expectedStart = platform.encodeString("a", true)[0].toInt()
val expectedEnd = platform.encodeString("z", false)[0].toInt()
val expectedStart = platform.encodeString("a", Encoding.SCREENCODES)[0].toInt()
val expectedEnd = platform.encodeString("z", Encoding.PETSCII)[0].toInt()
val expectedStr = "$expectedStart .. $expectedEnd"
val actualStr = "${rhsValues.first()} .. ${rhsValues.last()}"
@ -76,8 +77,8 @@ class TestCompilerOnRanges: FunSpec({
val rhsValues = (decl.value as ArrayLiteralValue)
.value // Array<Expression>
.map { (it as NumericLiteralValue).number.toInt() }
val expectedStart = platform.encodeString("a", false)[0].toInt()
val expectedEnd = platform.encodeString("z", false)[0].toInt()
val expectedStart = platform.encodeString("a", Encoding.PETSCII)[0].toInt()
val expectedEnd = platform.encodeString("z", Encoding.PETSCII)[0].toInt()
val expectedStr = "$expectedStart .. $expectedEnd"
val actualStr = "${rhsValues.first()} .. ${rhsValues.last()}"
@ -147,8 +148,8 @@ class TestCompilerOnRanges: FunSpec({
.map { it.iterable }[0]
val rangeExpr = iterable as RangeExpression
val expectedStart = platform.encodeString("a", true)[0].toInt()
val expectedEnd = platform.encodeString("f", false)[0].toInt()
val expectedStart = platform.encodeString("a", Encoding.SCREENCODES)[0].toInt()
val expectedEnd = platform.encodeString("f", Encoding.PETSCII)[0].toInt()
val expectedStr = "$expectedStart .. $expectedEnd"
val intProgression = rangeExpr.toConstantIntegerRange()

11
compiler/test/TestNumericLiteralValue.kt
View File

@ -12,6 +12,7 @@ import prog8.ast.expressions.ArrayLiteralValue
import prog8.ast.expressions.InferredTypes
import prog8.ast.expressions.NumericLiteralValue
import prog8.ast.expressions.StringLiteralValue
import prog8.compilerinterface.Encoding
class TestNumericLiteralValue: FunSpec({
@ -94,11 +95,11 @@ class TestNumericLiteralValue: FunSpec({
}
test("testEqualsRef") {
(StringLiteralValue("hello", false, dummyPos) == StringLiteralValue("hello", false, dummyPos)) shouldBe true
(StringLiteralValue("hello", false, dummyPos) != StringLiteralValue("bye", false, dummyPos)) shouldBe true
(StringLiteralValue("hello", true, dummyPos) == StringLiteralValue("hello", true, dummyPos)) shouldBe true
(StringLiteralValue("hello", true, dummyPos) != StringLiteralValue("bye", true, dummyPos)) shouldBe true
(StringLiteralValue("hello", true, dummyPos) != StringLiteralValue("hello", false, dummyPos)) shouldBe true
(StringLiteralValue("hello", Encoding.PETSCII, dummyPos) == StringLiteralValue("hello", Encoding.PETSCII, dummyPos)) shouldBe true
(StringLiteralValue("hello", Encoding.PETSCII, dummyPos) != StringLiteralValue("bye", Encoding.PETSCII, dummyPos)) shouldBe true
(StringLiteralValue("hello", Encoding.SCREENCODES, dummyPos) == StringLiteralValue("hello", Encoding.SCREENCODES, dummyPos)) shouldBe true
(StringLiteralValue("hello", Encoding.SCREENCODES, dummyPos) != StringLiteralValue("bye", Encoding.SCREENCODES, dummyPos)) shouldBe true
(StringLiteralValue("hello", Encoding.SCREENCODES, dummyPos) != StringLiteralValue("hello", Encoding.PETSCII, dummyPos)) shouldBe true
val lvOne = NumericLiteralValue(DataType.UBYTE, 1.0, dummyPos)
val lvTwo = NumericLiteralValue(DataType.UBYTE, 2.0, dummyPos)

35
compiler/test/ZeropageTests.kt
View File

@ -13,48 +13,18 @@ import io.kotest.matchers.comparables.shouldBeGreaterThan
import io.kotest.matchers.shouldBe
import io.kotest.matchers.shouldNotBe
import prog8.ast.base.DataType
import prog8.ast.expressions.Expression
import prog8.ast.statements.RegisterOrStatusflag
import prog8.ast.statements.Subroutine
import prog8.codegen.target.C64Target
import prog8.codegen.target.Cx16Target
import prog8.codegen.target.c64.C64Zeropage
import prog8.codegen.target.cx16.CX16Zeropage
import prog8.compilerinterface.*
import prog8tests.helpers.DummyCompilationTarget
import prog8tests.helpers.ErrorReporterForTests
import java.lang.IllegalArgumentException
class TestAbstractZeropage: FunSpec({
class DummyCompilationTarget: ICompilationTarget {
override val name: String = "dummy"
override val machine: IMachineDefinition
get() = throw NotImplementedError("dummy")
override fun encodeString(str: String, altEncoding: Boolean): List<UByte> {
throw NotImplementedError("dummy")
}
override fun decodeString(bytes: List<UByte>, altEncoding: Boolean): String {
throw NotImplementedError("dummy")
}
override fun asmsubArgsEvalOrder(sub: Subroutine): List<Int> {
throw NotImplementedError("dummy")
}
override fun asmsubArgsHaveRegisterClobberRisk(args: List<Expression>,
paramRegisters: List<RegisterOrStatusflag>): Boolean {
throw NotImplementedError("dummy")
}
override fun memorySize(dt: DataType): Int {
throw NotImplementedError("dummy")
}
}
class DummyZeropage(options: CompilationOptions) : Zeropage(options) {
override val SCRATCH_B1 = 0x10u
override val SCRATCH_REG = 0x11u
@ -70,7 +40,6 @@ class TestAbstractZeropage: FunSpec({
test("testAbstractZeropage") {
val compTarget = DummyCompilationTarget()
val zp = DummyZeropage(
CompilationOptions(
OutputType.RAW,
@ -79,7 +48,7 @@ class TestAbstractZeropage: FunSpec({
listOf((0x50u..0x5fu)),
false,
false,
compTarget
DummyCompilationTarget
)
)
zp.free.size shouldBe 256-6-16

106
compiler/test/ast/TestProg8Parser.kt
View File

@ -21,6 +21,7 @@ import prog8.ast.expressions.*
import prog8.ast.statements.*
import prog8.codegen.target.C64Target
import prog8.codegen.target.cbm.Petscii
import prog8.compilerinterface.Encoding
import prog8.parser.ParseError
import prog8.parser.Prog8Parser.parseModule
import prog8.parser.SourceCode
@ -413,7 +414,7 @@ class TestProg8Parser: FunSpec( {
char.value shouldBe '\n'
}
test("on rhs of block-level var decl, no AltEnc") {
test("on rhs of block-level var decl, default encoding") {
val src = SourceCode.Text("""
main {
ubyte c = 'x'
@ -425,11 +426,11 @@ class TestProg8Parser: FunSpec( {
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as CharLiteral
rhs.encoding shouldBe Encoding.PETSCII
rhs.value shouldBe 'x'
rhs.altEncoding shouldBe false
}
test("on rhs of block-level const decl, with AltEnc") {
test("on rhs of block-level const decl, with screencode enc (old syntax)") {
val src = SourceCode.Text("""
main {
const ubyte c = @'x'
@ -441,11 +442,44 @@ class TestProg8Parser: FunSpec( {
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as CharLiteral
rhs.encoding shouldBe Encoding.SCREENCODES
rhs.value shouldBe 'x'
rhs.altEncoding shouldBe true
}
test("on rhs of subroutine-level var decl, no AltEnc") {
xtest("on rhs of block-level const decl, with screencode enc (new syntax)") {
val src = SourceCode.Text("""
main {
const ubyte c = sc:'x'
}
""")
val module = parseModule(src)
val decl = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as CharLiteral
rhs.encoding shouldBe Encoding.SCREENCODES
rhs.value shouldBe 'x'
}
xtest("on rhs of block-level const decl, with iso encoding") {
val src = SourceCode.Text("""
main {
const ubyte c = iso:'_'
}
""")
val module = parseModule(src)
val decl = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as CharLiteral
rhs.encoding shouldBe Encoding.ISO
rhs.value shouldBe '_'
}
test("on rhs of subroutine-level var decl, default encoding") {
val src = SourceCode.Text("""
main {
sub start() {
@ -461,10 +495,10 @@ class TestProg8Parser: FunSpec( {
val rhs = decl.value as CharLiteral
rhs.value shouldBe 'x'
rhs.altEncoding shouldBe false
rhs.encoding shouldBe Encoding.PETSCII
}
test("on rhs of subroutine-level const decl, with AltEnc") {
test("on rhs of subroutine-level const decl, screencode (old syntax)") {
val src = SourceCode.Text("""
main {
sub start() {
@ -479,9 +513,54 @@ class TestProg8Parser: FunSpec( {
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as CharLiteral
rhs.encoding shouldBe Encoding.SCREENCODES
rhs.value shouldBe 'x'
rhs.altEncoding shouldBe true
}
xtest("on rhs of subroutine-level const decl, screencode (new syntax)") {
val src = SourceCode.Text("""
main {
sub start() {
const ubyte c = sc:'x'
}
}
""")
val module = parseModule(src)
val decl = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<Subroutine>()[0]
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as CharLiteral
rhs.encoding shouldBe Encoding.SCREENCODES
rhs.value shouldBe 'x'
}
xtest("on rhs of subroutine-level const decl, iso encoding") {
val src = SourceCode.Text("""
main {
sub start() {
const ubyte c = iso:'_'
}
}
""")
val module = parseModule(src)
val decl = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<Subroutine>()[0]
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as CharLiteral
rhs.encoding shouldBe Encoding.ISO
rhs.value shouldBe '_'
}
}
context("Strings") {
// TODO test encoding in all available encodings
// check that '~' cant be encoded in petscii and screencode
// check that '~' CAN be encoded correctly in iso
}
context("Ranges") {
@ -534,12 +613,14 @@ class TestProg8Parser: FunSpec( {
}
test("testCharLiteralConstValue") {
val char1 = CharLiteral('A', false, Position.DUMMY)
val char2 = CharLiteral('z', true, Position.DUMMY)
val char1 = CharLiteral('A', Encoding.PETSCII, Position.DUMMY)
val char2 = CharLiteral('z', Encoding.SCREENCODES, Position.DUMMY)
val char3 = CharLiteral('_', Encoding.ISO, Position.DUMMY)
val program = Program("test", DummyFunctions, DummyMemsizer, AsciiStringEncoder)
char1.constValue(program).number.toInt() shouldBe 65
char2.constValue(program).number.toInt() shouldBe 122
char3.constValue(program).number.toInt() shouldBe 95
}
test("testLiteralValueComparisons") {
@ -560,8 +641,8 @@ class TestProg8Parser: FunSpec( {
(ten <= ten) shouldBe true
(ten < ten) shouldBe false
val abc = StringLiteralValue("abc", false, Position.DUMMY)
val abd = StringLiteralValue("abd", false, Position.DUMMY)
val abc = StringLiteralValue("abc", Encoding.PETSCII, Position.DUMMY)
val abd = StringLiteralValue("abd", Encoding.PETSCII, Position.DUMMY)
abc shouldBe abc
(abc!=abd) shouldBe true
(abc!=abc) shouldBe false
@ -726,7 +807,6 @@ class TestProg8Parser: FunSpec( {
ubexpr.inferType(program).getOrElse { fail("dt") } shouldBe DataType.UBYTE
}
test("assignment isAugmented correctness") {
val src = SourceCode.Text("""
main {

40
compiler/test/helpers/Dummies.kt
View File

@ -6,8 +6,9 @@ import prog8.ast.base.Position
import prog8.ast.expressions.Expression
import prog8.ast.expressions.InferredTypes
import prog8.ast.expressions.NumericLiteralValue
import prog8.compilerinterface.IMemSizer
import prog8.compilerinterface.IStringEncoding
import prog8.ast.statements.RegisterOrStatusflag
import prog8.ast.statements.Subroutine
import prog8.compilerinterface.*
internal val DummyFunctions = object : IBuiltinFunctions {
override val names: Set<String> = emptySet()
@ -26,19 +27,46 @@ internal val DummyMemsizer = object : IMemSizer {
}
internal val DummyStringEncoder = object : IStringEncoding {
override fun encodeString(str: String, altEncoding: Boolean): List<UByte> {
override fun encodeString(str: String, encoding: Encoding): List<UByte> {
return emptyList()
}
override fun decodeString(bytes: List<UByte>, altEncoding: Boolean): String {
override fun decodeString(bytes: List<UByte>, encoding: Encoding): String {
return ""
}
}
internal val AsciiStringEncoder = object : IStringEncoding {
override fun encodeString(str: String, altEncoding: Boolean): List<UByte> = str.map { it.code.toUByte() }
override fun encodeString(str: String, encoding: Encoding): List<UByte> = str.map { it.code.toUByte() }
override fun decodeString(bytes: List<UByte>, altEncoding: Boolean): String {
override fun decodeString(bytes: List<UByte>, encoding: Encoding): String {
return bytes.joinToString()
}
}
internal val DummyCompilationTarget = object : ICompilationTarget {
override val name: String = "dummy"
override val machine: IMachineDefinition
get() = throw NotImplementedError("dummy")
override fun encodeString(str: String, encoding: Encoding): List<UByte> {
throw NotImplementedError("dummy")
}
override fun decodeString(bytes: List<UByte>, encoding: Encoding): String {
throw NotImplementedError("dummy")
}
override fun asmsubArgsEvalOrder(sub: Subroutine): List<Int> {
throw NotImplementedError("dummy")
}
override fun asmsubArgsHaveRegisterClobberRisk(args: List<Expression>,
paramRegisters: List<RegisterOrStatusflag>): Boolean {
throw NotImplementedError("dummy")
}
override fun memorySize(dt: DataType): Int {
throw NotImplementedError("dummy")
}
}

10
compilerAst/src/prog8/ast/AstToSourceTextConverter.kt
View File

@ -2,11 +2,13 @@ package prog8.ast
import prog8.ast.antlr.escape
import prog8.ast.base.DataType
import prog8.ast.base.FatalAstException
import prog8.ast.base.NumericDatatypes
import prog8.ast.base.VarDeclType
import prog8.ast.expressions.*
import prog8.ast.statements.*
import prog8.ast.walk.IAstVisitor
import prog8.compilerinterface.Encoding
/**
@ -286,15 +288,11 @@ class AstToSourceTextConverter(val output: (text: String) -> Unit, val program:
}
override fun visit(char: CharLiteral) {
if (char.altEncoding)
output("@")
output("'${escape(char.value.toString())}'")
output("${char.encoding.prefix}:'${escape(char.value.toString())}'")
}
override fun visit(string: StringLiteralValue) {
if (string.altEncoding)
output("@")
output("\"${escape(string.value)}\"")
output("${string.encoding.prefix}:\"${escape(string.value)}\"")
}
override fun visit(array: ArrayLiteralValue) {

5
compilerAst/src/prog8/ast/Program.kt
View File

@ -7,6 +7,7 @@ import prog8.ast.base.VarDeclType
import prog8.ast.expressions.ContainmentCheck
import prog8.ast.expressions.StringLiteralValue
import prog8.ast.statements.*
import prog8.compilerinterface.Encoding
import prog8.compilerinterface.IMemSizer
import prog8.compilerinterface.IStringEncoding
import prog8.parser.SourceCode
@ -74,7 +75,7 @@ class Program(val name: String,
get() = toplevelModule.loadAddress
var actualLoadAddress = 0u
private val internedStringsUnique = mutableMapOf<Pair<String, Boolean>, List<String>>()
private val internedStringsUnique = mutableMapOf<Pair<String, Encoding>, List<String>>()
fun internString(string: StringLiteralValue): List<String> {
// Move a string literal into the internal, deduplicated, string pool
@ -99,7 +100,7 @@ class Program(val name: String,
return listOf(internedStringsModuleName, decl.name)
}
val key = Pair(string.value, string.altEncoding)
val key = Pair(string.value, string.encoding)
val existing = internedStringsUnique[key]
if (existing != null)
return existing

9
compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt
View File

@ -5,6 +5,7 @@ import org.antlr.v4.runtime.tree.TerminalNode
import prog8.ast.base.*
import prog8.ast.expressions.*
import prog8.ast.statements.*
import prog8.compilerinterface.Encoding
import prog8.parser.Prog8ANTLRParser
import prog8.parser.SourceCode
import java.nio.file.Path
@ -450,12 +451,16 @@ private fun Prog8ANTLRParser.ExpressionContext.toAst() : Expression {
private fun Prog8ANTLRParser.CharliteralContext.toAst(): CharLiteral {
val text = this.SINGLECHAR().text
return CharLiteral(unescape(text.substring(1, text.length-1), toPosition())[0], this.ALT_STRING_ENCODING() != null, toPosition())
// TODO ISO-encoding, alternative encoding syntax
val encoding = if(ALT_STRING_ENCODING()==null) Encoding.PETSCII else Encoding.SCREENCODES
return CharLiteral(unescape(text.substring(1, text.length-1), toPosition())[0], encoding, toPosition())
}
private fun Prog8ANTLRParser.StringliteralContext.toAst(): StringLiteralValue {
val text=this.STRING().text
return StringLiteralValue(unescape(text.substring(1, text.length-1), toPosition()), ALT_STRING_ENCODING() != null, toPosition())
// TODO ISO-encoding, alternative encoding syntax
val encoding = if(ALT_STRING_ENCODING()==null) Encoding.PETSCII else Encoding.SCREENCODES
return StringLiteralValue(unescape(text.substring(1, text.length-1), toPosition()), encoding, toPosition())
}
private fun Prog8ANTLRParser.ArrayindexedContext.toAst(): ArrayIndexedExpression {

23
compilerAst/src/prog8/ast/expressions/AstExpressions.kt
View File

@ -6,6 +6,7 @@ import prog8.ast.base.*
import prog8.ast.statements.*
import prog8.ast.walk.AstWalker
import prog8.ast.walk.IAstVisitor
import prog8.compilerinterface.Encoding
import java.util.*
import kotlin.math.abs
import kotlin.math.round
@ -589,8 +590,8 @@ class NumericLiteralValue(val type: DataType, // only numerical types allowed
}
class CharLiteral(val value: Char,
val altEncoding: Boolean, // such as: screencodes instead of Petscii for the C64
override val position: Position) : Expression() {
val encoding: Encoding,
override val position: Position) : Expression() {
override lateinit var parent: Node
override fun linkParents(parent: Node) {
@ -603,10 +604,10 @@ class CharLiteral(val value: Char,
throw FatalAstException("can't replace here")
}
override fun copy() = CharLiteral(value, altEncoding, position)
override fun copy() = CharLiteral(value, encoding, position)
override fun referencesIdentifier(nameInSource: List<String>) = false
override fun constValue(program: Program): NumericLiteralValue {
val bytevalue = program.encoding.encodeString(value.toString(), altEncoding).single()
val bytevalue = program.encoding.encodeString(value.toString(), encoding).single()
return NumericLiteralValue(DataType.UBYTE, bytevalue.toDouble(), position)
}
override fun accept(visitor: IAstVisitor) = visitor.visit(this)
@ -615,16 +616,16 @@ class CharLiteral(val value: Char,
override fun toString(): String = "'${escape(value.toString())}'"
override fun inferType(program: Program) = InferredTypes.knownFor(DataType.UBYTE)
operator fun compareTo(other: CharLiteral): Int = value.compareTo(other.value)
override fun hashCode(): Int = Objects.hash(value, altEncoding)
override fun hashCode(): Int = Objects.hash(value, encoding)
override fun equals(other: Any?): Boolean {
if (other == null || other !is CharLiteral)
return false
return value == other.value && altEncoding == other.altEncoding
return value == other.value && encoding == other.encoding
}
}
class StringLiteralValue(val value: String,
val altEncoding: Boolean, // such as: screencodes instead of Petscii for the C64
val encoding: Encoding,
override val position: Position) : Expression() {
override lateinit var parent: Node
@ -633,7 +634,7 @@ class StringLiteralValue(val value: String,
}
override val isSimple = true
override fun copy() = StringLiteralValue(value, altEncoding, position)
override fun copy() = StringLiteralValue(value, encoding, position)
override fun replaceChildNode(node: Node, replacement: Node) {
throw FatalAstException("can't replace here")
@ -647,11 +648,11 @@ class StringLiteralValue(val value: String,
override fun toString(): String = "'${escape(value)}'"
override fun inferType(program: Program) = InferredTypes.knownFor(DataType.STR)
operator fun compareTo(other: StringLiteralValue): Int = value.compareTo(other.value)
override fun hashCode(): Int = Objects.hash(value, altEncoding)
override fun hashCode(): Int = Objects.hash(value, encoding)
override fun equals(other: Any?): Boolean {
if(other==null || other !is StringLiteralValue)
return false
return value==other.value && altEncoding == other.altEncoding
return value==other.value && encoding == other.encoding
}
}
@ -1025,7 +1026,7 @@ class ContainmentCheck(var element: Expression,
is StringLiteralValue -> {
if(elementConst.type in ByteDatatypes) {
val stringval = iterable as StringLiteralValue
val exists = program.encoding.encodeString(stringval.value, stringval.altEncoding).contains(elementConst.number.toInt().toUByte() )
val exists = program.encoding.encodeString(stringval.value, stringval.encoding).contains(elementConst.number.toInt().toUByte() )
return NumericLiteralValue.fromBoolean(exists, position)
}
}

12
compilerAst/src/prog8/compilerinterface/IStringEncoding.kt
View File

@ -1,6 +1,12 @@
package prog8.compilerinterface
interface IStringEncoding {
fun encodeString(str: String, altEncoding: Boolean): List<UByte>
fun decodeString(bytes: List<UByte>, altEncoding: Boolean): String
enum class Encoding(val prefix: String) {
PETSCII("petscii"), // c64/c128/cx16
SCREENCODES("sc"), // c64/c128/cx16
ISO("iso") // cx16
}
interface IStringEncoding {
fun encodeString(str: String, encoding: Encoding): List<UByte>
fun decodeString(bytes: List<UByte>, encoding: Encoding): String
}

7
compilerInterfaces/src/prog8/compilerinterface/ICompilationTarget.kt
View File

@ -4,11 +4,14 @@ import prog8.ast.expressions.Expression
import prog8.ast.statements.RegisterOrStatusflag
import prog8.ast.statements.Subroutine
// TODO list of supported string encodings
interface ICompilationTarget: IStringEncoding, IMemSizer {
val name: String
val machine: IMachineDefinition
override fun encodeString(str: String, altEncoding: Boolean): List<UByte>
override fun decodeString(bytes: List<UByte>, altEncoding: Boolean): String
override fun encodeString(str: String, encoding: Encoding): List<UByte>
override fun decodeString(bytes: List<UByte>, encoding: Encoding): String
fun asmsubArgsEvalOrder(sub: Subroutine): List<Int>
fun asmsubArgsHaveRegisterClobberRisk(args: List<Expression>,

4
docs/source/todo.rst
View File

@ -3,7 +3,9 @@ TODO
For next compiler release (7.7)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
...
- implement iso encoding and alternate encoding syntax
- document new encoding syntax
Need help with

48
examples/test.p8
View File

@ -1,21 +1,41 @@
%import textio
%zeropage basicsafe
main {
str myBar = "main.bar"
str s1 = "Irmen_"
str s2 = @"IRMEN_"
;str s3 = iso:"Irmen_~"
foo_bar:
%asm {{
nop
}}
sub start() {
txt.lowercase()
txt.nl()
txt.nl()
txt.nl()
txt.nl()
txt.nl()
txt.print(s1)
txt.nl()
txt.print(s2)
txt.nl()
; txt.print(s3)
; txt.nl()
sub start() {
txt.print(myBar)
sc(1, s1)
sc(2, s2)
; sc(3, s3)
}
%breakpoint
txt.print_uwhex(&foo_bar, true)
%breakpoint
return
}
sub sc(ubyte row, str text) {
uword addr = 1024+row*40
ubyte ix = 0
ubyte ss
repeat {
ss = text[ix]
if not ss
return
@(addr) = ss
addr++
ix++
}
}
}