diff --git a/codeGeneration/src/prog8/codegen/target/Cx16Target.kt b/codeGeneration/src/prog8/codegen/target/Cx16Target.kt index cb103abb0..eacfc743d 100644 --- a/codeGeneration/src/prog8/codegen/target/Cx16Target.kt +++ b/codeGeneration/src/prog8/codegen/target/Cx16Target.kt @@ -5,6 +5,7 @@ import prog8.ast.base.* import prog8.ast.expressions.Expression import prog8.ast.statements.RegisterOrStatusflag import prog8.ast.statements.Subroutine +import prog8.codegen.target.cbm.IsoEncoding import prog8.codegen.target.cbm.Petscii import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsEvalOrder import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsHaveRegisterClobberRisk @@ -20,7 +21,7 @@ object Cx16Target: ICompilationTarget { val coded = when(encoding) { Encoding.PETSCII -> Petscii.encodePetscii(str, true) Encoding.SCREENCODES -> Petscii.encodeScreencode(str, true) - Encoding.ISO -> TODO("cx16 iso-encoding") + Encoding.ISO -> IsoEncoding.encode(str) else -> throw FatalAstException("unsupported encoding $encoding") } return coded.fold( @@ -32,7 +33,7 @@ object Cx16Target: ICompilationTarget { return when(encoding) { Encoding.PETSCII -> Petscii.decodePetscii(bytes, true) Encoding.SCREENCODES -> Petscii.decodeScreencode(bytes, true) - Encoding.ISO -> TODO("cx16 iso-encoding") + Encoding.ISO -> IsoEncoding.decode(bytes) else -> throw FatalAstException("unsupported encoding $encoding") } } diff --git a/codeGeneration/src/prog8/codegen/target/cbm/IsoEncoding.kt b/codeGeneration/src/prog8/codegen/target/cbm/IsoEncoding.kt new file mode 100644 index 000000000..cfd34de89 --- /dev/null +++ b/codeGeneration/src/prog8/codegen/target/cbm/IsoEncoding.kt @@ -0,0 +1,21 @@ +package prog8.codegen.target.cbm + +import com.github.michaelbull.result.Result +import com.github.michaelbull.result.Ok +import com.github.michaelbull.result.Err +import java.io.CharConversionException + +object IsoEncoding { + fun encode(str: String): Result, CharConversionException> { + return try { + Ok(str.toByteArray(Charsets.ISO_8859_1).map { it.toUByte() }) + } catch (ce: CharConversionException) { + Err(ce) + } + } + + fun decode(bytes: List): String { + // TODO use Result + return String(bytes.map { it.toByte() }.toByteArray(), Charsets.ISO_8859_1) + } +} diff --git a/codeGeneration/src/prog8/codegen/target/cbm/Petscii.kt b/codeGeneration/src/prog8/codegen/target/cbm/Petscii.kt index c07474198..ef8a41eff 100644 --- a/codeGeneration/src/prog8/codegen/target/cbm/Petscii.kt +++ b/codeGeneration/src/prog8/codegen/target/cbm/Petscii.kt @@ -1099,7 +1099,7 @@ object Petscii { return petscii.map { val code = it.toInt() if(code<0 || code>= decodingPetsciiLowercase.size) - throw CharConversionException("petscii $code out of range 0..${decodingPetsciiLowercase.size-1}") + throw CharConversionException("petscii $code out of range 0..${decodingPetsciiLowercase.size-1}") // TODO don't throw, use Result if(lowercase) decodingPetsciiLowercase[code] else decodingPetsciiUppercase[code] }.joinToString("") } @@ -1138,7 +1138,7 @@ object Petscii { return screencode.map { val code = it.toInt() if(code<0 || code>= decodingScreencodeLowercase.size) - throw CharConversionException("screencode $code out of range 0..${decodingScreencodeLowercase.size-1}") + throw CharConversionException("screencode $code out of range 0..${decodingScreencodeLowercase.size-1}") // TODO don't throw, use Result if (lowercase) decodingScreencodeLowercase[code] else decodingScreencodeUppercase[code] }.joinToString("") } diff --git a/compiler/res/prog8lib/cx16/textio.p8 b/compiler/res/prog8lib/cx16/textio.p8 index 7a35eb6fa..0690df3e6 100644 --- a/compiler/res/prog8lib/cx16/textio.p8 +++ b/compiler/res/prog8lib/cx16/textio.p8 @@ -169,15 +169,23 @@ sub color2 (ubyte txtcol, ubyte bgcol) { } sub lowercase() { - cx16.screen_set_charset(3, 0) ; lowercase petscii charset + c64.CHROUT($0e) + ; this is not 100% compatible: cx16.screen_set_charset(3, 0) ; lowercase petscii charset } sub uppercase() { - cx16.screen_set_charset(2, 0) ; uppercase petscii charset + c64.CHROUT($8e) + ; this is not 100% compatible: cx16.screen_set_charset(2, 0) ; uppercase petscii charset } sub iso() { - cx16.screen_set_charset(1, 0) ; iso charset + c64.CHROUT($0f) + ; This doesn't enable it completely: cx16.screen_set_charset(1, 0) ; iso charset +} + +sub iso_off() { + ; -- you have to call this first when switching back from iso charset to regular charset. + c64.CHROUT($8f) } diff --git a/compiler/test/TestPetscii.kt b/compiler/test/TestPetscii.kt index 6279d3a1f..8aab2453c 100644 --- a/compiler/test/TestPetscii.kt +++ b/compiler/test/TestPetscii.kt @@ -6,170 +6,186 @@ import com.github.michaelbull.result.getOrElse import io.kotest.assertions.withClue import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.shouldBe +import prog8.codegen.target.cbm.IsoEncoding import prog8.codegen.target.cbm.Petscii -class TestPetscii: FunSpec({ +class TestStringEncodings: FunSpec({ - test("testZero") { - Petscii.encodePetscii("\u0000", true) shouldBe Ok(listOf(0u)) - Petscii.encodePetscii("\u0000", false) shouldBe Ok(listOf(0u)) - Petscii.decodePetscii(listOf(0u), true) shouldBe "\u0000" - Petscii.decodePetscii(listOf(0u), false) shouldBe "\u0000" - } - - test("testLowercase") { - Petscii.encodePetscii("hello WORLD 123 @!£", true) shouldBe - Ok(listOf(72u, 69u, 76u, 76u, 79u, 32u, 0xd7u, 0xcfu, 0xd2u, 0xccu, 0xc4u, 32u, 49u, 50u, 51u, 32u, 64u, 33u, 0x5cu)) - Petscii.encodePetscii("\uf11a", true) shouldBe Ok(listOf(0x12u)) // reverse vid - Petscii.encodePetscii("✓", true) shouldBe Ok(listOf(0xfau)) - withClue("expect lowercase error fallback") { - Petscii.encodePetscii("π", true) shouldBe Ok(listOf(255u)) - Petscii.encodePetscii("♥", true) shouldBe Ok(listOf(0xd3u)) + context("petscii") { + test("testZero") { + Petscii.encodePetscii("\u0000", true) shouldBe Ok(listOf(0u)) + Petscii.encodePetscii("\u0000", false) shouldBe Ok(listOf(0u)) + Petscii.decodePetscii(listOf(0u), true) shouldBe "\u0000" + Petscii.decodePetscii(listOf(0u), false) shouldBe "\u0000" } - Petscii.decodePetscii(listOf(72u, 0xd7u, 0x5cu, 0xfau, 0x12u), true) shouldBe "hW£✓\uF11A" - } + test("testLowercase") { + Petscii.encodePetscii("hello WORLD 123 @!£", true) shouldBe + Ok(listOf(72u, 69u, 76u, 76u, 79u, 32u, 0xd7u, 0xcfu, 0xd2u, 0xccu, 0xc4u, 32u, 49u, 50u, 51u, 32u, 64u, 33u, 0x5cu)) + Petscii.encodePetscii("\uf11a", true) shouldBe Ok(listOf(0x12u)) // reverse vid + Petscii.encodePetscii("✓", true) shouldBe Ok(listOf(0xfau)) + withClue("expect lowercase error fallback") { + Petscii.encodePetscii("π", true) shouldBe Ok(listOf(255u)) + Petscii.encodePetscii("♥", true) shouldBe Ok(listOf(0xd3u)) + } - test("testUppercase") { - Petscii.encodePetscii("HELLO 123 @!£") shouldBe - Ok(listOf(72u, 69u, 76u, 76u, 79u, 32u, 49u, 50u, 51u, 32u, 64u, 33u, 0x5cu)) - Petscii.encodePetscii("\uf11a") shouldBe Ok(listOf(0x12u)) // reverse vid - Petscii.encodePetscii("♥") shouldBe Ok(listOf(0xd3u)) - Petscii.encodePetscii("π") shouldBe Ok(listOf(0xffu)) - withClue("expecting fallback") { - Petscii.encodePetscii("✓") shouldBe Ok(listOf(250u)) + Petscii.decodePetscii(listOf(72u, 0xd7u, 0x5cu, 0xfau, 0x12u), true) shouldBe "hW£✓\uF11A" } - Petscii.decodePetscii(listOf(72u, 0x5cu, 0xd3u, 0xffu)) shouldBe "H£♥π" - } + test("testUppercase") { + Petscii.encodePetscii("HELLO 123 @!£") shouldBe + Ok(listOf(72u, 69u, 76u, 76u, 79u, 32u, 49u, 50u, 51u, 32u, 64u, 33u, 0x5cu)) + Petscii.encodePetscii("\uf11a") shouldBe Ok(listOf(0x12u)) // reverse vid + Petscii.encodePetscii("♥") shouldBe Ok(listOf(0xd3u)) + Petscii.encodePetscii("π") shouldBe Ok(listOf(0xffu)) + withClue("expecting fallback") { + Petscii.encodePetscii("✓") shouldBe Ok(listOf(250u)) + } - test("testScreencodeLowercase") { - Petscii.encodeScreencode("hello WORLD 123 @!£", true) shouldBe - Ok(listOf(0x08u, 0x05u, 0x0cu, 0x0cu, 0x0fu, 0x20u, 0x57u, 0x4fu, 0x52u, 0x4cu, 0x44u, 0x20u, 0x31u, 0x32u, 0x33u, 0x20u, 0x00u, 0x21u, 0x1cu)) - Petscii.encodeScreencode("✓", true) shouldBe Ok(listOf(0x7au)) - withClue("expect fallback") { - Petscii.encodeScreencode("♥", true) shouldBe Ok(listOf(83u)) - Petscii.encodeScreencode("π", true) shouldBe Ok(listOf(94u)) + Petscii.decodePetscii(listOf(72u, 0x5cu, 0xd3u, 0xffu)) shouldBe "H£♥π" } - Petscii.decodeScreencode(listOf(0x08u, 0x57u, 0x1cu, 0x7au), true) shouldBe "hW£✓" - } + test("testScreencodeLowercase") { + Petscii.encodeScreencode("hello WORLD 123 @!£", true) shouldBe + Ok(listOf(0x08u, 0x05u, 0x0cu, 0x0cu, 0x0fu, 0x20u, 0x57u, 0x4fu, 0x52u, 0x4cu, 0x44u, 0x20u, 0x31u, 0x32u, 0x33u, 0x20u, 0x00u, 0x21u, 0x1cu)) + Petscii.encodeScreencode("✓", true) shouldBe Ok(listOf(0x7au)) + withClue("expect fallback") { + Petscii.encodeScreencode("♥", true) shouldBe Ok(listOf(83u)) + Petscii.encodeScreencode("π", true) shouldBe Ok(listOf(94u)) + } - test("testScreencodeUppercase") { - Petscii.encodeScreencode("WORLD 123 @!£") shouldBe - Ok(listOf(0x17u, 0x0fu, 0x12u, 0x0cu, 0x04u, 0x20u, 0x31u, 0x32u, 0x33u, 0x20u, 0x00u, 0x21u, 0x1cu)) - Petscii.encodeScreencode("♥") shouldBe Ok(listOf(0x53u)) - Petscii.encodeScreencode("π") shouldBe Ok(listOf(0x5eu)) - Petscii.encodeScreencode("HELLO") shouldBe Ok(listOf(8u, 5u, 12u, 12u, 15u)) - withClue("expecting fallback") { - Petscii.encodeScreencode("hello") shouldBe Ok(listOf(8u, 5u, 12u, 12u, 15u)) - Petscii.encodeScreencode("✓") shouldBe Ok(listOf(122u)) + Petscii.decodeScreencode(listOf(0x08u, 0x57u, 0x1cu, 0x7au), true) shouldBe "hW£✓" } - Petscii.decodeScreencode(listOf(0x17u, 0x1cu, 0x53u, 0x5eu)) shouldBe "W£♥π" + test("testScreencodeUppercase") { + Petscii.encodeScreencode("WORLD 123 @!£") shouldBe + Ok(listOf(0x17u, 0x0fu, 0x12u, 0x0cu, 0x04u, 0x20u, 0x31u, 0x32u, 0x33u, 0x20u, 0x00u, 0x21u, 0x1cu)) + Petscii.encodeScreencode("♥") shouldBe Ok(listOf(0x53u)) + Petscii.encodeScreencode("π") shouldBe Ok(listOf(0x5eu)) + Petscii.encodeScreencode("HELLO") shouldBe Ok(listOf(8u, 5u, 12u, 12u, 15u)) + withClue("expecting fallback") { + Petscii.encodeScreencode("hello") shouldBe Ok(listOf(8u, 5u, 12u, 12u, 15u)) + Petscii.encodeScreencode("✓") shouldBe Ok(listOf(122u)) + } + + Petscii.decodeScreencode(listOf(0x17u, 0x1cu, 0x53u, 0x5eu)) shouldBe "W£♥π" + } + + test("testErrorCases") { + Petscii.encodePetscii("~", true).expectError { "shouldn't be able to encode tilde" } + Petscii.encodePetscii("~", false).expectError { "shouldn't be able to encode tilde" } + Petscii.encodeScreencode("~", true).expectError { "shouldn't be able to encode tilde" } + Petscii.encodeScreencode("~", false).expectError { "shouldn't be able to encode tilde" } + } + + test("testSpecialReplacements") { + fun encodeP(c: Char, lower: Boolean) = Petscii.encodePetscii(c.toString(), lower).getOrElse { throw it }.single() + fun encodeS(c: Char, lower: Boolean) = Petscii.encodeScreencode(c.toString(), lower).getOrElse { throw it }.single() + + Petscii.encodePetscii("`", false).expectError { "shouldn't have translation for backtick" } + Petscii.encodePetscii("`", true).expectError { "shouldn't have translation for backtick" } + Petscii.encodePetscii("~", false).expectError { "shouldn't have translation for tilde" } + Petscii.encodePetscii("~", true).expectError { "shouldn't have translation for tilde" } + + encodeP('^', false) shouldBe 94u + encodeP('^', true) shouldBe 94u + encodeS('^', false) shouldBe 30u + encodeS('^', true) shouldBe 30u + encodeP('_', false) shouldBe 228u + encodeP('_', true) shouldBe 228u + encodeS('_', false) shouldBe 100u + encodeS('_', true) shouldBe 100u + encodeP('{', false) shouldBe 243u + encodeP('{', true) shouldBe 243u + encodeS('{', false) shouldBe 115u + encodeS('{', true) shouldBe 115u + encodeP('}', false) shouldBe 235u + encodeP('}', true) shouldBe 235u + encodeS('}', false) shouldBe 107u + encodeS('}', true) shouldBe 107u + encodeP('|', false) shouldBe 221u + encodeP('|', true) shouldBe 221u + encodeS('|', false) shouldBe 93u + encodeS('|', true) shouldBe 93u + encodeP('\\', false) shouldBe 205u + encodeP('\\', true) shouldBe 205u + encodeS('\\', false) shouldBe 77u + encodeS('\\', true) shouldBe 77u + } + + test("testBoxDrawingCharsEncoding") { + fun encodeP(c: Char, lower: Boolean) = Petscii.encodePetscii(c.toString(), lower).getOrElse { throw it }.single() + fun encodeS(c: Char, lower: Boolean) = Petscii.encodeScreencode(c.toString(), lower).getOrElse { throw it }.single() + + // pipe char + encodeP('|', false) shouldBe 221u + encodeP('|', true) shouldBe 221u + encodeS('|', false) shouldBe 93u + encodeS('|', true) shouldBe 93u + // ... same as '│', 0x7D -> BOX DRAWINGS LIGHT VERTICAL + encodeP('│', false) shouldBe 221u + encodeP('│', true) shouldBe 221u + encodeS('│', false) shouldBe 93u + encodeS('│', true) shouldBe 93u + + // underscore + encodeP('_', false) shouldBe 228u + encodeP('_', true) shouldBe 228u + encodeS('_', false) shouldBe 100u + encodeS('_', true) shouldBe 100u + // ... same as '▁', 0xE4 LOWER ONE EIGHTH BLOCK + encodeP('▁', false) shouldBe 228u + encodeP('▁', true) shouldBe 228u + encodeS('▁', false) shouldBe 100u + encodeS('▁', true) shouldBe 100u + + // ─ 0xC0 -> BOX DRAWINGS LIGHT HORIZONTAL + encodeP('─', false) shouldBe 192u + encodeP('─', true) shouldBe 192u + encodeS('─', false) shouldBe 64u + encodeS('─', true) shouldBe 64u + // │ 0x62 -> BOX DRAWINGS LIGHT VERTICAL + encodeP('│', false) shouldBe 221u + encodeP('│', true) shouldBe 221u + encodeS('│', false) shouldBe 93u + encodeS('│', true) shouldBe 93u + } + + test("testBoxDrawingCharsDecoding") { + // ─ 0xC0 -> BOX DRAWINGS LIGHT HORIZONTAL + Petscii.decodePetscii(listOf(195u), false).single() shouldBe '\uf13b' //"BOX DRAWINGS LIGHT HORIZONTAL ONE EIGHTH UP (CUS)" + Petscii.decodePetscii(listOf(195u), true).single() shouldBe 'C' + Petscii.decodePetscii(listOf(192u), false).single() shouldBe '─' + Petscii.decodePetscii(listOf(192u), true).single() shouldBe '─' + Petscii.decodeScreencode(listOf(67u), false).single() shouldBe '\uf13b' //"BOX DRAWINGS LIGHT HORIZONTAL ONE EIGHTH UP (CUS)" + Petscii.decodeScreencode(listOf(67u), true).single() shouldBe 'C' + Petscii.decodeScreencode(listOf(64u), false).single() shouldBe '─' + Petscii.decodeScreencode(listOf(64u), true).single() shouldBe '─' + + // │ 0x62 -> BOX DRAWINGS LIGHT VERTICAL + Petscii.decodePetscii(listOf(125u), false).single() shouldBe '│' + Petscii.decodePetscii(listOf(125u), true).single() shouldBe '│' + Petscii.decodePetscii(listOf(221u), false).single() shouldBe '│' + Petscii.decodePetscii(listOf(221u), true).single() shouldBe '│' + Petscii.decodeScreencode(listOf(93u), false).single() shouldBe '│' + Petscii.decodeScreencode(listOf(93u), true).single() shouldBe '│' + Petscii.decodeScreencode(listOf(66u), false).single() shouldBe '\uf13c' // "BOX DRAWINGS LIGHT VERTICAL ONE EIGHTH LEFT (CUS)" + Petscii.decodeScreencode(listOf(66u), true).single() shouldBe 'B' + } } - test("testErrorCases") { - Petscii.encodePetscii("~", true).expectError { "shouldn't be able to encode tilde" } - Petscii.encodePetscii("~", false).expectError { "shouldn't be able to encode tilde" } - Petscii.encodeScreencode("~", true).expectError { "shouldn't be able to encode tilde" } - Petscii.encodeScreencode("~", false).expectError { "shouldn't be able to encode tilde" } + context("iso") { + test("iso accepts iso-characters") { + val result = IsoEncoding.encode("a_~ëç") + result.getOrElse { throw it }.map {it.toInt()} shouldBe listOf(97, 95, 126, 235, 231) + } + + test("non-iso doesn't accept iso-characters") { + var result = Petscii.encodePetscii("a_~ë") + result.expectError { "should not encode" } + result = Petscii.encodeScreencode("a_~ë") + result.expectError { "should not encode" } + } } - - test("testSpecialReplacements") { - fun encodeP(c: Char, lower: Boolean) = Petscii.encodePetscii(c.toString(), lower).getOrElse { throw it }.single() - fun encodeS(c: Char, lower: Boolean) = Petscii.encodeScreencode(c.toString(), lower).getOrElse { throw it }.single() - - Petscii.encodePetscii("`", false).expectError { "shouldn't have translation for backtick" } - Petscii.encodePetscii("`", true).expectError { "shouldn't have translation for backtick" } - Petscii.encodePetscii("~", false).expectError { "shouldn't have translation for tilde" } - Petscii.encodePetscii("~", true).expectError { "shouldn't have translation for tilde" } - - encodeP('^', false) shouldBe 94u - encodeP('^', true) shouldBe 94u - encodeS('^', false) shouldBe 30u - encodeS('^', true) shouldBe 30u - encodeP('_', false) shouldBe 228u - encodeP('_', true) shouldBe 228u - encodeS('_', false) shouldBe 100u - encodeS('_', true) shouldBe 100u - encodeP('{', false) shouldBe 243u - encodeP('{', true) shouldBe 243u - encodeS('{', false) shouldBe 115u - encodeS('{', true) shouldBe 115u - encodeP('}', false) shouldBe 235u - encodeP('}', true) shouldBe 235u - encodeS('}', false) shouldBe 107u - encodeS('}', true) shouldBe 107u - encodeP('|', false) shouldBe 221u - encodeP('|', true) shouldBe 221u - encodeS('|', false) shouldBe 93u - encodeS('|', true) shouldBe 93u - encodeP('\\', false) shouldBe 205u - encodeP('\\', true) shouldBe 205u - encodeS('\\', false) shouldBe 77u - encodeS('\\', true) shouldBe 77u - } - - test("testBoxDrawingCharsEncoding") { - fun encodeP(c: Char, lower: Boolean) = Petscii.encodePetscii(c.toString(), lower).getOrElse { throw it }.single() - fun encodeS(c: Char, lower: Boolean) = Petscii.encodeScreencode(c.toString(), lower).getOrElse { throw it }.single() - - // pipe char - encodeP('|', false) shouldBe 221u - encodeP('|', true) shouldBe 221u - encodeS('|', false) shouldBe 93u - encodeS('|', true) shouldBe 93u - // ... same as '│', 0x7D -> BOX DRAWINGS LIGHT VERTICAL - encodeP('│', false) shouldBe 221u - encodeP('│', true) shouldBe 221u - encodeS('│', false) shouldBe 93u - encodeS('│', true) shouldBe 93u - - // underscore - encodeP('_', false) shouldBe 228u - encodeP('_', true) shouldBe 228u - encodeS('_', false) shouldBe 100u - encodeS('_', true) shouldBe 100u - // ... same as '▁', 0xE4 LOWER ONE EIGHTH BLOCK - encodeP('▁', false) shouldBe 228u - encodeP('▁', true) shouldBe 228u - encodeS('▁', false) shouldBe 100u - encodeS('▁', true) shouldBe 100u - - // ─ 0xC0 -> BOX DRAWINGS LIGHT HORIZONTAL - encodeP('─', false) shouldBe 192u - encodeP('─', true) shouldBe 192u - encodeS('─', false) shouldBe 64u - encodeS('─', true) shouldBe 64u - // │ 0x62 -> BOX DRAWINGS LIGHT VERTICAL - encodeP('│', false) shouldBe 221u - encodeP('│', true) shouldBe 221u - encodeS('│', false) shouldBe 93u - encodeS('│', true) shouldBe 93u - } - - test("testBoxDrawingCharsDecoding") { - // ─ 0xC0 -> BOX DRAWINGS LIGHT HORIZONTAL - Petscii.decodePetscii(listOf(195u), false).single() shouldBe '\uf13b' //"BOX DRAWINGS LIGHT HORIZONTAL ONE EIGHTH UP (CUS)" - Petscii.decodePetscii(listOf(195u), true).single() shouldBe 'C' - Petscii.decodePetscii(listOf(192u), false).single() shouldBe '─' - Petscii.decodePetscii(listOf(192u), true).single() shouldBe '─' - Petscii.decodeScreencode(listOf(67u), false).single() shouldBe '\uf13b' //"BOX DRAWINGS LIGHT HORIZONTAL ONE EIGHTH UP (CUS)" - Petscii.decodeScreencode(listOf(67u), true).single() shouldBe 'C' - Petscii.decodeScreencode(listOf(64u), false).single() shouldBe '─' - Petscii.decodeScreencode(listOf(64u), true).single() shouldBe '─' - - // │ 0x62 -> BOX DRAWINGS LIGHT VERTICAL - Petscii.decodePetscii(listOf(125u), false).single() shouldBe '│' - Petscii.decodePetscii(listOf(125u), true).single() shouldBe '│' - Petscii.decodePetscii(listOf(221u), false).single() shouldBe '│' - Petscii.decodePetscii(listOf(221u), true).single() shouldBe '│' - Petscii.decodeScreencode(listOf(93u), false).single() shouldBe '│' - Petscii.decodeScreencode(listOf(93u), true).single() shouldBe '│' - Petscii.decodeScreencode(listOf(66u), false).single() shouldBe '\uf13c' // "BOX DRAWINGS LIGHT VERTICAL ONE EIGHTH LEFT (CUS)" - Petscii.decodeScreencode(listOf(66u), true).single() shouldBe 'B' - } - }) diff --git a/compiler/test/ast/TestAstToSourceText.kt b/compiler/test/ast/TestAstToSourceText.kt index 1f71ad1ab..5883bf32c 100644 --- a/compiler/test/ast/TestAstToSourceText.kt +++ b/compiler/test/ast/TestAstToSourceText.kt @@ -61,47 +61,80 @@ class TestAstToSourceText: AnnotationSpec() { @Test - fun testStringLiteral_noAlt() { + fun testStringLiteral_DefaultEnc() { val orig = SourceCode.Text(""" main { str s = "fooBar\n" } """) val (txt, _) = roundTrip(parseModule(orig)) - txt shouldContain Regex("str +s += +\"fooBar\\\\n\"") + txt shouldContain Regex("str +s += +petscii:\"fooBar\\\\n\"") } @Test - fun testStringLiteral_withAlt() { + fun testStringLiteral_withSc() { + val orig = SourceCode.Text(""" + main { + str sAlt = sc:"fooBar\n" + } + """) + val (txt, _) = roundTrip(parseModule(orig)) + txt shouldContain Regex("str +sAlt += +sc:\"fooBar\\\\n\"") + } + + @Test + fun testStringLiteral_withOldSc() { val orig = SourceCode.Text(""" main { str sAlt = @"fooBar\n" } """) val (txt, _) = roundTrip(parseModule(orig)) - txt shouldContain Regex("str +sAlt += +@\"fooBar\\\\n\"") + txt shouldContain Regex("str +sAlt += +sc:\"fooBar\\\\n\"") } @Test - fun testCharLiteral_noAlt() { + fun testStringLiteral_withIso() { + val orig = SourceCode.Text(""" + main { + str sAlt = iso:"fooBar\n" + } + """) + val (txt, _) = roundTrip(parseModule(orig)) + txt shouldContain Regex("str +sAlt += +iso:\"fooBar\\\\n\"") + } + + @Test + fun testCharLiteral_defaultEnc() { val orig = SourceCode.Text(""" main { ubyte c = 'x' } """) val (txt, _) = roundTrip(parseModule(orig)) - txt shouldContain Regex("ubyte +c += +'x'") + txt shouldContain Regex("ubyte +c += +petscii:'x'") } @Test - fun testCharLiteral_withAlt() { + fun testCharLiteral_OldSc() { val orig = SourceCode.Text(""" main { ubyte cAlt = @'x' } """) val (txt, _) = roundTrip(parseModule(orig)) - txt shouldContain Regex("ubyte +cAlt += +@'x'") + txt shouldContain Regex("ubyte +cAlt += +sc:'x'") + } + + @Test + fun testCharLiteral_Sc() { + val orig = SourceCode.Text(""" + main { + ubyte cAlt = sc:'x' + } + """) + val (txt, _) = roundTrip(parseModule(orig)) + txt shouldContain Regex("ubyte +cAlt += +sc:'x'") } @Test diff --git a/compiler/test/ast/TestProg8Parser.kt b/compiler/test/ast/TestProg8Parser.kt index 6b6619359..56f7ab522 100644 --- a/compiler/test/ast/TestProg8Parser.kt +++ b/compiler/test/ast/TestProg8Parser.kt @@ -446,7 +446,7 @@ class TestProg8Parser: FunSpec( { rhs.value shouldBe 'x' } - xtest("on rhs of block-level const decl, with screencode enc (new syntax)") { + test("on rhs of block-level const decl, with screencode enc (new syntax)") { val src = SourceCode.Text(""" main { const ubyte c = sc:'x' @@ -462,7 +462,7 @@ class TestProg8Parser: FunSpec( { rhs.value shouldBe 'x' } - xtest("on rhs of block-level const decl, with iso encoding") { + test("on rhs of block-level const decl, with iso encoding") { val src = SourceCode.Text(""" main { const ubyte c = iso:'_' @@ -517,7 +517,7 @@ class TestProg8Parser: FunSpec( { rhs.value shouldBe 'x' } - xtest("on rhs of subroutine-level const decl, screencode (new syntax)") { + test("on rhs of subroutine-level const decl, screencode (new syntax)") { val src = SourceCode.Text(""" main { sub start() { @@ -536,7 +536,7 @@ class TestProg8Parser: FunSpec( { rhs.value shouldBe 'x' } - xtest("on rhs of subroutine-level const decl, iso encoding") { + test("on rhs of subroutine-level const decl, iso encoding") { val src = SourceCode.Text(""" main { sub start() { @@ -558,9 +558,55 @@ class TestProg8Parser: FunSpec( { context("Strings") { - // TODO test encoding in all available encodings - // check that '~' cant be encoded in petscii and screencode - // check that '~' CAN be encoded correctly in iso + test("default encoding") { + val source = """ + main { + str name = "name" + }""" + val module = parseModule(SourceCode.Text(source)) + val decl = module + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] + val rhs = decl.value as StringLiteralValue + rhs.encoding shouldBe Encoding.PETSCII + rhs.value shouldBe "name" + } + + test("old syntax alt encoding") { + val source = """ + main { + str name = @"name" + }""" + val module = parseModule(SourceCode.Text(source)) + val decl = module + .statements.filterIsInstance()[0] + .statements.filterIsInstance()[0] + val rhs = decl.value as StringLiteralValue + rhs.encoding shouldBe Encoding.SCREENCODES + rhs.value shouldBe "name" + } + + test("new syntax encodings") { + val source = """ + main { + str name1 = petscii:"Name" + str name2 = sc:"Name" + str name3 = iso:"Name" + }""" + val module = parseModule(SourceCode.Text(source)) + val (decl1, decl2, decl3) = module + .statements.filterIsInstance()[0] + .statements.filterIsInstance() + val rhs1 = decl1.value as StringLiteralValue + val rhs2 = decl2.value as StringLiteralValue + val rhs3 = decl3.value as StringLiteralValue + rhs1.encoding shouldBe Encoding.PETSCII + rhs1.value shouldBe "Name" + rhs2.encoding shouldBe Encoding.SCREENCODES + rhs2.value shouldBe "Name" + rhs3.encoding shouldBe Encoding.ISO + rhs3.value shouldBe "Name" + } } context("Ranges") { diff --git a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt index d7b851f32..9a0a82158 100644 --- a/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt +++ b/compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt @@ -321,8 +321,10 @@ internal fun Prog8ANTLRParser.DirectiveContext.toAst() : Directive = private fun Prog8ANTLRParser.DirectiveargContext.toAst() : DirectiveArg { val str = stringliteral() - if(str?.ALT_STRING_ENCODING() != null) - throw SyntaxError("can't use alternate string s for directive arguments", toPosition()) + val oldenc = str?.old_alt_encoding?.text + val enc = str?.encoding?.text + if(oldenc!=null || enc !=null) + throw SyntaxError("can't use alternate string encoding for directive arguments", toPosition()) return DirectiveArg(str?.text?.substring(1, text.length-1), identifier()?.text, integerliteral()?.toAst()?.number?.toUInt(), toPosition()) } @@ -451,15 +453,29 @@ private fun Prog8ANTLRParser.ExpressionContext.toAst() : Expression { private fun Prog8ANTLRParser.CharliteralContext.toAst(): CharLiteral { val text = this.SINGLECHAR().text - // TODO ISO-encoding, alternative encoding syntax - val encoding = if(ALT_STRING_ENCODING()==null) Encoding.PETSCII else Encoding.SCREENCODES + val enc = this.encoding?.text + val encoding = + if(old_alt_encoding!=null) + Encoding.SCREENCODES + else if(enc!=null) + Encoding.values().singleOrNull { it.prefix == enc } + ?: throw SyntaxError("invalid encoding", toPosition()) + else + Encoding.PETSCII return CharLiteral(unescape(text.substring(1, text.length-1), toPosition())[0], encoding, toPosition()) } private fun Prog8ANTLRParser.StringliteralContext.toAst(): StringLiteralValue { val text=this.STRING().text - // TODO ISO-encoding, alternative encoding syntax - val encoding = if(ALT_STRING_ENCODING()==null) Encoding.PETSCII else Encoding.SCREENCODES + val enc = encoding?.text + val encoding = + if(old_alt_encoding!=null) + Encoding.SCREENCODES + else if(enc!=null) + Encoding.values().singleOrNull { it.prefix == enc } + ?: throw SyntaxError("invalid encoding", toPosition()) + else + Encoding.PETSCII return StringLiteralValue(unescape(text.substring(1, text.length-1), toPosition()), encoding, toPosition()) } diff --git a/docs/source/todo.rst b/docs/source/todo.rst index 9039357e3..3518f8be0 100644 --- a/docs/source/todo.rst +++ b/docs/source/todo.rst @@ -3,8 +3,8 @@ TODO For next compiler release (7.7) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- implement iso encoding and alternate encoding syntax -- document new encoding syntax +- document new string encoding syntax +- fix petscii decode methods that still throw an exception rather than Result diff --git a/parser/antlr/Prog8ANTLR.g4 b/parser/antlr/Prog8ANTLR.g4 index 016d3f263..0272a1cc9 100644 --- a/parser/antlr/Prog8ANTLR.g4 +++ b/parser/antlr/Prog8ANTLR.g4 @@ -28,7 +28,6 @@ DEC_INTEGER : ('0'..'9') | (('1'..'9')('0'..'9')+); HEX_INTEGER : '$' (('a'..'f') | ('A'..'F') | ('0'..'9'))+ ; BIN_INTEGER : '%' ('0' | '1')+ ; ADDRESS_OF: '&'; -ALT_STRING_ENCODING: '@'; FLOAT_NUMBER : FNUMBER (('E'|'e') ('+' | '-')? FNUMBER)? ; // sign comes later from unary expression fragment FNUMBER : ('0' .. '9') + ('.' ('0' .. '9') +)? ; @@ -221,9 +220,9 @@ booleanliteral : 'true' | 'false' ; arrayliteral : '[' EOL? expression (',' EOL? expression)* EOL? ']' ; // you can split the values over several lines -stringliteral : ALT_STRING_ENCODING? STRING ; +stringliteral : (old_alt_encoding='@' | encoding=NAME ':')? STRING ; -charliteral : ALT_STRING_ENCODING? SINGLECHAR ; +charliteral : (old_alt_encoding='@' | encoding=NAME ':')? SINGLECHAR ; floatliteral : FLOAT_NUMBER ;