Apple-2-SW/prog8
1
0
Fork 0
mirror of https://github.com/irmen/prog8.git synced 2024-09-30 00:55:52 +00:00
Code Issues Projects Releases Wiki Activity

implement iso encoding and new string encoding syntax, fixes #38

Browse Source
This commit is contained in:
Irmen de Jong 2022-01-19 01:08:24 +01:00
parent 5237e55326
commit 6b02f2eea0
10 changed files with 321 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
codeGeneration/src/prog8/codegen/target/Cx16Target.kt
View File

@ -5,6 +5,7 @@ import prog8.ast.base.*
import prog8.ast.expressions.Expression
import prog8.ast.statements.RegisterOrStatusflag
import prog8.ast.statements.Subroutine
import prog8.codegen.target.cbm.IsoEncoding
import prog8.codegen.target.cbm.Petscii
import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsEvalOrder
import prog8.codegen.target.cpu6502.codegen.asmsub6502ArgsHaveRegisterClobberRisk
@ -20,7 +21,7 @@ object Cx16Target: ICompilationTarget {
val coded = when(encoding) {
Encoding.PETSCII -> Petscii.encodePetscii(str, true)
Encoding.SCREENCODES -> Petscii.encodeScreencode(str, true)
Encoding.ISO -> TODO("cx16 iso-encoding")
Encoding.ISO -> IsoEncoding.encode(str)
else -> throw FatalAstException("unsupported encoding $encoding")
}
return coded.fold(
@ -32,7 +33,7 @@ object Cx16Target: ICompilationTarget {
return when(encoding) {
Encoding.PETSCII -> Petscii.decodePetscii(bytes, true)
Encoding.SCREENCODES -> Petscii.decodeScreencode(bytes, true)
Encoding.ISO -> TODO("cx16 iso-encoding")
Encoding.ISO -> IsoEncoding.decode(bytes)
else -> throw FatalAstException("unsupported encoding $encoding")
}
}

21
codeGeneration/src/prog8/codegen/target/cbm/IsoEncoding.kt Normal file
View File

@ -0,0 +1,21 @@
package prog8.codegen.target.cbm
import com.github.michaelbull.result.Result
import com.github.michaelbull.result.Ok
import com.github.michaelbull.result.Err
import java.io.CharConversionException
object IsoEncoding {
fun encode(str: String): Result<List<UByte>, CharConversionException> {
return try {
Ok(str.toByteArray(Charsets.ISO_8859_1).map { it.toUByte() })
} catch (ce: CharConversionException) {
Err(ce)
}
}
fun decode(bytes: List<UByte>): String {
// TODO use Result
return String(bytes.map { it.toByte() }.toByteArray(), Charsets.ISO_8859_1)
}
}

4
codeGeneration/src/prog8/codegen/target/cbm/Petscii.kt
View File

@ -1099,7 +1099,7 @@ object Petscii {
return petscii.map {
val code = it.toInt()
if(code<0 || code>= decodingPetsciiLowercase.size)
throw CharConversionException("petscii $code out of range 0..${decodingPetsciiLowercase.size-1}")
throw CharConversionException("petscii $code out of range 0..${decodingPetsciiLowercase.size-1}") // TODO don't throw, use Result
if(lowercase) decodingPetsciiLowercase[code] else decodingPetsciiUppercase[code]
}.joinToString("")
}
@ -1138,7 +1138,7 @@ object Petscii {
return screencode.map {
val code = it.toInt()
if(code<0 || code>= decodingScreencodeLowercase.size)
throw CharConversionException("screencode $code out of range 0..${decodingScreencodeLowercase.size-1}")
throw CharConversionException("screencode $code out of range 0..${decodingScreencodeLowercase.size-1}") // TODO don't throw, use Result
if (lowercase) decodingScreencodeLowercase[code] else decodingScreencodeUppercase[code]
}.joinToString("")
}

14
compiler/res/prog8lib/cx16/textio.p8
View File

@ -169,15 +169,23 @@ sub color2 (ubyte txtcol, ubyte bgcol) {
}
sub lowercase() {
cx16.screen_set_charset(3, 0) ; lowercase petscii charset
c64.CHROUT($0e)
; this is not 100% compatible: cx16.screen_set_charset(3, 0) ; lowercase petscii charset
}
sub uppercase() {
cx16.screen_set_charset(2, 0) ; uppercase petscii charset
c64.CHROUT($8e)
; this is not 100% compatible: cx16.screen_set_charset(2, 0) ; uppercase petscii charset
}
sub iso() {
cx16.screen_set_charset(1, 0) ; iso charset
c64.CHROUT($0f)
; This doesn't enable it completely: cx16.screen_set_charset(1, 0) ; iso charset
}
sub iso_off() {
; -- you have to call this first when switching back from iso charset to regular charset.
c64.CHROUT($8f)
}

312
compiler/test/TestPetscii.kt
View File

@ -6,170 +6,186 @@ import com.github.michaelbull.result.getOrElse
import io.kotest.assertions.withClue
import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.shouldBe
import prog8.codegen.target.cbm.IsoEncoding
import prog8.codegen.target.cbm.Petscii
class TestPetscii: FunSpec({
class TestStringEncodings: FunSpec({
test("testZero") {
Petscii.encodePetscii("\u0000", true) shouldBe Ok(listOf<UByte>(0u))
Petscii.encodePetscii("\u0000", false) shouldBe Ok(listOf<UByte>(0u))
Petscii.decodePetscii(listOf(0u), true) shouldBe "\u0000"
Petscii.decodePetscii(listOf(0u), false) shouldBe "\u0000"
}
test("testLowercase") {
Petscii.encodePetscii("hello WORLD 123 @!£", true) shouldBe
Ok(listOf<UByte>(72u, 69u, 76u, 76u, 79u, 32u, 0xd7u, 0xcfu, 0xd2u, 0xccu, 0xc4u, 32u, 49u, 50u, 51u, 32u, 64u, 33u, 0x5cu))
Petscii.encodePetscii("\uf11a", true) shouldBe Ok(listOf<UByte>(0x12u)) // reverse vid
Petscii.encodePetscii("✓", true) shouldBe Ok(listOf<UByte>(0xfau))
withClue("expect lowercase error fallback") {
Petscii.encodePetscii("π", true) shouldBe Ok(listOf<UByte>(255u))
Petscii.encodePetscii("♥", true) shouldBe Ok(listOf<UByte>(0xd3u))
context("petscii") {
test("testZero") {
Petscii.encodePetscii("\u0000", true) shouldBe Ok(listOf<UByte>(0u))
Petscii.encodePetscii("\u0000", false) shouldBe Ok(listOf<UByte>(0u))
Petscii.decodePetscii(listOf(0u), true) shouldBe "\u0000"
Petscii.decodePetscii(listOf(0u), false) shouldBe "\u0000"
}
Petscii.decodePetscii(listOf(72u, 0xd7u, 0x5cu, 0xfau, 0x12u), true) shouldBe "hW£✓\uF11A"
}
test("testLowercase") {
Petscii.encodePetscii("hello WORLD 123 @!£", true) shouldBe
Ok(listOf<UByte>(72u, 69u, 76u, 76u, 79u, 32u, 0xd7u, 0xcfu, 0xd2u, 0xccu, 0xc4u, 32u, 49u, 50u, 51u, 32u, 64u, 33u, 0x5cu))
Petscii.encodePetscii("\uf11a", true) shouldBe Ok(listOf<UByte>(0x12u)) // reverse vid
Petscii.encodePetscii("✓", true) shouldBe Ok(listOf<UByte>(0xfau))
withClue("expect lowercase error fallback") {
Petscii.encodePetscii("π", true) shouldBe Ok(listOf<UByte>(255u))
Petscii.encodePetscii("♥", true) shouldBe Ok(listOf<UByte>(0xd3u))
}
test("testUppercase") {
Petscii.encodePetscii("HELLO 123 @!£") shouldBe
Ok(listOf<UByte>(72u, 69u, 76u, 76u, 79u, 32u, 49u, 50u, 51u, 32u, 64u, 33u, 0x5cu))
Petscii.encodePetscii("\uf11a") shouldBe Ok(listOf<UByte>(0x12u)) // reverse vid
Petscii.encodePetscii("♥") shouldBe Ok(listOf<UByte>(0xd3u))
Petscii.encodePetscii("π") shouldBe Ok(listOf<UByte>(0xffu))
withClue("expecting fallback") {
Petscii.encodePetscii("✓") shouldBe Ok(listOf<UByte>(250u))
Petscii.decodePetscii(listOf(72u, 0xd7u, 0x5cu, 0xfau, 0x12u), true) shouldBe "hW£✓\uF11A"
}
Petscii.decodePetscii(listOf(72u, 0x5cu, 0xd3u, 0xffu)) shouldBe "H£♥π"
}
test("testUppercase") {
Petscii.encodePetscii("HELLO 123 @!£") shouldBe
Ok(listOf<UByte>(72u, 69u, 76u, 76u, 79u, 32u, 49u, 50u, 51u, 32u, 64u, 33u, 0x5cu))
Petscii.encodePetscii("\uf11a") shouldBe Ok(listOf<UByte>(0x12u)) // reverse vid
Petscii.encodePetscii("♥") shouldBe Ok(listOf<UByte>(0xd3u))
Petscii.encodePetscii("π") shouldBe Ok(listOf<UByte>(0xffu))
withClue("expecting fallback") {
Petscii.encodePetscii("✓") shouldBe Ok(listOf<UByte>(250u))
}
test("testScreencodeLowercase") {
Petscii.encodeScreencode("hello WORLD 123 @!£", true) shouldBe
Ok(listOf<UByte>(0x08u, 0x05u, 0x0cu, 0x0cu, 0x0fu, 0x20u, 0x57u, 0x4fu, 0x52u, 0x4cu, 0x44u, 0x20u, 0x31u, 0x32u, 0x33u, 0x20u, 0x00u, 0x21u, 0x1cu))
Petscii.encodeScreencode("✓", true) shouldBe Ok(listOf<UByte>(0x7au))
withClue("expect fallback") {
Petscii.encodeScreencode("♥", true) shouldBe Ok(listOf<UByte>(83u))
Petscii.encodeScreencode("π", true) shouldBe Ok(listOf<UByte>(94u))
Petscii.decodePetscii(listOf(72u, 0x5cu, 0xd3u, 0xffu)) shouldBe "H£♥π"
}
Petscii.decodeScreencode(listOf(0x08u, 0x57u, 0x1cu, 0x7au), true) shouldBe "hW£✓"
}
test("testScreencodeLowercase") {
Petscii.encodeScreencode("hello WORLD 123 @!£", true) shouldBe
Ok(listOf<UByte>(0x08u, 0x05u, 0x0cu, 0x0cu, 0x0fu, 0x20u, 0x57u, 0x4fu, 0x52u, 0x4cu, 0x44u, 0x20u, 0x31u, 0x32u, 0x33u, 0x20u, 0x00u, 0x21u, 0x1cu))
Petscii.encodeScreencode("✓", true) shouldBe Ok(listOf<UByte>(0x7au))
withClue("expect fallback") {
Petscii.encodeScreencode("♥", true) shouldBe Ok(listOf<UByte>(83u))
Petscii.encodeScreencode("π", true) shouldBe Ok(listOf<UByte>(94u))
}
test("testScreencodeUppercase") {
Petscii.encodeScreencode("WORLD 123 @!£") shouldBe
Ok(listOf<UByte>(0x17u, 0x0fu, 0x12u, 0x0cu, 0x04u, 0x20u, 0x31u, 0x32u, 0x33u, 0x20u, 0x00u, 0x21u, 0x1cu))
Petscii.encodeScreencode("♥") shouldBe Ok(listOf<UByte>(0x53u))
Petscii.encodeScreencode("π") shouldBe Ok(listOf<UByte>(0x5eu))
Petscii.encodeScreencode("HELLO") shouldBe Ok(listOf<UByte>(8u, 5u, 12u, 12u, 15u))
withClue("expecting fallback") {
Petscii.encodeScreencode("hello") shouldBe Ok(listOf<UByte>(8u, 5u, 12u, 12u, 15u))
Petscii.encodeScreencode("✓") shouldBe Ok(listOf<UByte>(122u))
Petscii.decodeScreencode(listOf(0x08u, 0x57u, 0x1cu, 0x7au), true) shouldBe "hW£✓"
}
Petscii.decodeScreencode(listOf(0x17u, 0x1cu, 0x53u, 0x5eu)) shouldBe "W£♥π"
test("testScreencodeUppercase") {
Petscii.encodeScreencode("WORLD 123 @!£") shouldBe
Ok(listOf<UByte>(0x17u, 0x0fu, 0x12u, 0x0cu, 0x04u, 0x20u, 0x31u, 0x32u, 0x33u, 0x20u, 0x00u, 0x21u, 0x1cu))
Petscii.encodeScreencode("♥") shouldBe Ok(listOf<UByte>(0x53u))
Petscii.encodeScreencode("π") shouldBe Ok(listOf<UByte>(0x5eu))
Petscii.encodeScreencode("HELLO") shouldBe Ok(listOf<UByte>(8u, 5u, 12u, 12u, 15u))
withClue("expecting fallback") {
Petscii.encodeScreencode("hello") shouldBe Ok(listOf<UByte>(8u, 5u, 12u, 12u, 15u))
Petscii.encodeScreencode("✓") shouldBe Ok(listOf<UByte>(122u))
}
Petscii.decodeScreencode(listOf(0x17u, 0x1cu, 0x53u, 0x5eu)) shouldBe "W£♥π"
}
test("testErrorCases") {
Petscii.encodePetscii("~", true).expectError { "shouldn't be able to encode tilde" }
Petscii.encodePetscii("~", false).expectError { "shouldn't be able to encode tilde" }
Petscii.encodeScreencode("~", true).expectError { "shouldn't be able to encode tilde" }
Petscii.encodeScreencode("~", false).expectError { "shouldn't be able to encode tilde" }
}
test("testSpecialReplacements") {
fun encodeP(c: Char, lower: Boolean) = Petscii.encodePetscii(c.toString(), lower).getOrElse { throw it }.single()
fun encodeS(c: Char, lower: Boolean) = Petscii.encodeScreencode(c.toString(), lower).getOrElse { throw it }.single()
Petscii.encodePetscii("`", false).expectError { "shouldn't have translation for backtick" }
Petscii.encodePetscii("`", true).expectError { "shouldn't have translation for backtick" }
Petscii.encodePetscii("~", false).expectError { "shouldn't have translation for tilde" }
Petscii.encodePetscii("~", true).expectError { "shouldn't have translation for tilde" }
encodeP('^', false) shouldBe 94u
encodeP('^', true) shouldBe 94u
encodeS('^', false) shouldBe 30u
encodeS('^', true) shouldBe 30u
encodeP('_', false) shouldBe 228u
encodeP('_', true) shouldBe 228u
encodeS('_', false) shouldBe 100u
encodeS('_', true) shouldBe 100u
encodeP('{', false) shouldBe 243u
encodeP('{', true) shouldBe 243u
encodeS('{', false) shouldBe 115u
encodeS('{', true) shouldBe 115u
encodeP('}', false) shouldBe 235u
encodeP('}', true) shouldBe 235u
encodeS('}', false) shouldBe 107u
encodeS('}', true) shouldBe 107u
encodeP('|', false) shouldBe 221u
encodeP('|', true) shouldBe 221u
encodeS('|', false) shouldBe 93u
encodeS('|', true) shouldBe 93u
encodeP('\\', false) shouldBe 205u
encodeP('\\', true) shouldBe 205u
encodeS('\\', false) shouldBe 77u
encodeS('\\', true) shouldBe 77u
}
test("testBoxDrawingCharsEncoding") {
fun encodeP(c: Char, lower: Boolean) = Petscii.encodePetscii(c.toString(), lower).getOrElse { throw it }.single()
fun encodeS(c: Char, lower: Boolean) = Petscii.encodeScreencode(c.toString(), lower).getOrElse { throw it }.single()
// pipe char
encodeP('|', false) shouldBe 221u
encodeP('|', true) shouldBe 221u
encodeS('|', false) shouldBe 93u
encodeS('|', true) shouldBe 93u
// ... same as '│', 0x7D -> BOX DRAWINGS LIGHT VERTICAL
encodeP('│', false) shouldBe 221u
encodeP('│', true) shouldBe 221u
encodeS('│', false) shouldBe 93u
encodeS('│', true) shouldBe 93u
// underscore
encodeP('_', false) shouldBe 228u
encodeP('_', true) shouldBe 228u
encodeS('_', false) shouldBe 100u
encodeS('_', true) shouldBe 100u
// ... same as '▁', 0xE4 LOWER ONE EIGHTH BLOCK
encodeP('▁', false) shouldBe 228u
encodeP('▁', true) shouldBe 228u
encodeS('▁', false) shouldBe 100u
encodeS('▁', true) shouldBe 100u
// ─ 0xC0 -> BOX DRAWINGS LIGHT HORIZONTAL
encodeP('─', false) shouldBe 192u
encodeP('─', true) shouldBe 192u
encodeS('─', false) shouldBe 64u
encodeS('─', true) shouldBe 64u
// │ 0x62 -> BOX DRAWINGS LIGHT VERTICAL
encodeP('│', false) shouldBe 221u
encodeP('│', true) shouldBe 221u
encodeS('│', false) shouldBe 93u
encodeS('│', true) shouldBe 93u
}
test("testBoxDrawingCharsDecoding") {
// ─ 0xC0 -> BOX DRAWINGS LIGHT HORIZONTAL
Petscii.decodePetscii(listOf(195u), false).single() shouldBe '\uf13b' //"BOX DRAWINGS LIGHT HORIZONTAL ONE EIGHTH UP (CUS)"
Petscii.decodePetscii(listOf(195u), true).single() shouldBe 'C'
Petscii.decodePetscii(listOf(192u), false).single() shouldBe '─'
Petscii.decodePetscii(listOf(192u), true).single() shouldBe '─'
Petscii.decodeScreencode(listOf(67u), false).single() shouldBe '\uf13b' //"BOX DRAWINGS LIGHT HORIZONTAL ONE EIGHTH UP (CUS)"
Petscii.decodeScreencode(listOf(67u), true).single() shouldBe 'C'
Petscii.decodeScreencode(listOf(64u), false).single() shouldBe '─'
Petscii.decodeScreencode(listOf(64u), true).single() shouldBe '─'
// │ 0x62 -> BOX DRAWINGS LIGHT VERTICAL
Petscii.decodePetscii(listOf(125u), false).single() shouldBe '│'
Petscii.decodePetscii(listOf(125u), true).single() shouldBe '│'
Petscii.decodePetscii(listOf(221u), false).single() shouldBe '│'
Petscii.decodePetscii(listOf(221u), true).single() shouldBe '│'
Petscii.decodeScreencode(listOf(93u), false).single() shouldBe '│'
Petscii.decodeScreencode(listOf(93u), true).single() shouldBe '│'
Petscii.decodeScreencode(listOf(66u), false).single() shouldBe '\uf13c' // "BOX DRAWINGS LIGHT VERTICAL ONE EIGHTH LEFT (CUS)"
Petscii.decodeScreencode(listOf(66u), true).single() shouldBe 'B'
}
}
test("testErrorCases") {
Petscii.encodePetscii("~", true).expectError { "shouldn't be able to encode tilde" }
Petscii.encodePetscii("~", false).expectError { "shouldn't be able to encode tilde" }
Petscii.encodeScreencode("~", true).expectError { "shouldn't be able to encode tilde" }
Petscii.encodeScreencode("~", false).expectError { "shouldn't be able to encode tilde" }
context("iso") {
test("iso accepts iso-characters") {
val result = IsoEncoding.encode("a_~ëç")
result.getOrElse { throw it }.map {it.toInt()} shouldBe listOf(97, 95, 126, 235, 231)
}
test("non-iso doesn't accept iso-characters") {
var result = Petscii.encodePetscii("a_~ë")
result.expectError { "should not encode" }
result = Petscii.encodeScreencode("a_~ë")
result.expectError { "should not encode" }
}
}
test("testSpecialReplacements") {
fun encodeP(c: Char, lower: Boolean) = Petscii.encodePetscii(c.toString(), lower).getOrElse { throw it }.single()
fun encodeS(c: Char, lower: Boolean) = Petscii.encodeScreencode(c.toString(), lower).getOrElse { throw it }.single()
Petscii.encodePetscii("`", false).expectError { "shouldn't have translation for backtick" }
Petscii.encodePetscii("`", true).expectError { "shouldn't have translation for backtick" }
Petscii.encodePetscii("~", false).expectError { "shouldn't have translation for tilde" }
Petscii.encodePetscii("~", true).expectError { "shouldn't have translation for tilde" }
encodeP('^', false) shouldBe 94u
encodeP('^', true) shouldBe 94u
encodeS('^', false) shouldBe 30u
encodeS('^', true) shouldBe 30u
encodeP('_', false) shouldBe 228u
encodeP('_', true) shouldBe 228u
encodeS('_', false) shouldBe 100u
encodeS('_', true) shouldBe 100u
encodeP('{', false) shouldBe 243u
encodeP('{', true) shouldBe 243u
encodeS('{', false) shouldBe 115u
encodeS('{', true) shouldBe 115u
encodeP('}', false) shouldBe 235u
encodeP('}', true) shouldBe 235u
encodeS('}', false) shouldBe 107u
encodeS('}', true) shouldBe 107u
encodeP('|', false) shouldBe 221u
encodeP('|', true) shouldBe 221u
encodeS('|', false) shouldBe 93u
encodeS('|', true) shouldBe 93u
encodeP('\\', false) shouldBe 205u
encodeP('\\', true) shouldBe 205u
encodeS('\\', false) shouldBe 77u
encodeS('\\', true) shouldBe 77u
}
test("testBoxDrawingCharsEncoding") {
fun encodeP(c: Char, lower: Boolean) = Petscii.encodePetscii(c.toString(), lower).getOrElse { throw it }.single()
fun encodeS(c: Char, lower: Boolean) = Petscii.encodeScreencode(c.toString(), lower).getOrElse { throw it }.single()
// pipe char
encodeP('|', false) shouldBe 221u
encodeP('|', true) shouldBe 221u
encodeS('|', false) shouldBe 93u
encodeS('|', true) shouldBe 93u
// ... same as '│', 0x7D -> BOX DRAWINGS LIGHT VERTICAL
encodeP('│', false) shouldBe 221u
encodeP('│', true) shouldBe 221u
encodeS('│', false) shouldBe 93u
encodeS('│', true) shouldBe 93u
// underscore
encodeP('_', false) shouldBe 228u
encodeP('_', true) shouldBe 228u
encodeS('_', false) shouldBe 100u
encodeS('_', true) shouldBe 100u
// ... same as '▁', 0xE4 LOWER ONE EIGHTH BLOCK
encodeP('▁', false) shouldBe 228u
encodeP('▁', true) shouldBe 228u
encodeS('▁', false) shouldBe 100u
encodeS('▁', true) shouldBe 100u
// ─ 0xC0 -> BOX DRAWINGS LIGHT HORIZONTAL
encodeP('─', false) shouldBe 192u
encodeP('─', true) shouldBe 192u
encodeS('─', false) shouldBe 64u
encodeS('─', true) shouldBe 64u
// │ 0x62 -> BOX DRAWINGS LIGHT VERTICAL
encodeP('│', false) shouldBe 221u
encodeP('│', true) shouldBe 221u
encodeS('│', false) shouldBe 93u
encodeS('│', true) shouldBe 93u
}
test("testBoxDrawingCharsDecoding") {
// ─ 0xC0 -> BOX DRAWINGS LIGHT HORIZONTAL
Petscii.decodePetscii(listOf(195u), false).single() shouldBe '\uf13b' //"BOX DRAWINGS LIGHT HORIZONTAL ONE EIGHTH UP (CUS)"
Petscii.decodePetscii(listOf(195u), true).single() shouldBe 'C'
Petscii.decodePetscii(listOf(192u), false).single() shouldBe '─'
Petscii.decodePetscii(listOf(192u), true).single() shouldBe '─'
Petscii.decodeScreencode(listOf(67u), false).single() shouldBe '\uf13b' //"BOX DRAWINGS LIGHT HORIZONTAL ONE EIGHTH UP (CUS)"
Petscii.decodeScreencode(listOf(67u), true).single() shouldBe 'C'
Petscii.decodeScreencode(listOf(64u), false).single() shouldBe '─'
Petscii.decodeScreencode(listOf(64u), true).single() shouldBe '─'
// │ 0x62 -> BOX DRAWINGS LIGHT VERTICAL
Petscii.decodePetscii(listOf(125u), false).single() shouldBe '│'
Petscii.decodePetscii(listOf(125u), true).single() shouldBe '│'
Petscii.decodePetscii(listOf(221u), false).single() shouldBe '│'
Petscii.decodePetscii(listOf(221u), true).single() shouldBe '│'
Petscii.decodeScreencode(listOf(93u), false).single() shouldBe '│'
Petscii.decodeScreencode(listOf(93u), true).single() shouldBe '│'
Petscii.decodeScreencode(listOf(66u), false).single() shouldBe '\uf13c' // "BOX DRAWINGS LIGHT VERTICAL ONE EIGHTH LEFT (CUS)"
Petscii.decodeScreencode(listOf(66u), true).single() shouldBe 'B'
}
})

49
compiler/test/ast/TestAstToSourceText.kt
View File

@ -61,47 +61,80 @@ class TestAstToSourceText: AnnotationSpec() {
@Test
fun testStringLiteral_noAlt() {
fun testStringLiteral_DefaultEnc() {
val orig = SourceCode.Text("""
main {
str s = "fooBar\n"
}
""")
val (txt, _) = roundTrip(parseModule(orig))
txt shouldContain Regex("str +s += +\"fooBar\\\\n\"")
txt shouldContain Regex("str +s += +petscii:\"fooBar\\\\n\"")
}
@Test
fun testStringLiteral_withAlt() {
fun testStringLiteral_withSc() {
val orig = SourceCode.Text("""
main {
str sAlt = sc:"fooBar\n"
}
""")
val (txt, _) = roundTrip(parseModule(orig))
txt shouldContain Regex("str +sAlt += +sc:\"fooBar\\\\n\"")
}
@Test
fun testStringLiteral_withOldSc() {
val orig = SourceCode.Text("""
main {
str sAlt = @"fooBar\n"
}
""")
val (txt, _) = roundTrip(parseModule(orig))
txt shouldContain Regex("str +sAlt += +@\"fooBar\\\\n\"")
txt shouldContain Regex("str +sAlt += +sc:\"fooBar\\\\n\"")
}
@Test
fun testCharLiteral_noAlt() {
fun testStringLiteral_withIso() {
val orig = SourceCode.Text("""
main {
str sAlt = iso:"fooBar\n"
}
""")
val (txt, _) = roundTrip(parseModule(orig))
txt shouldContain Regex("str +sAlt += +iso:\"fooBar\\\\n\"")
}
@Test
fun testCharLiteral_defaultEnc() {
val orig = SourceCode.Text("""
main {
ubyte c = 'x'
}
""")
val (txt, _) = roundTrip(parseModule(orig))
txt shouldContain Regex("ubyte +c += +'x'")
txt shouldContain Regex("ubyte +c += +petscii:'x'")
}
@Test
fun testCharLiteral_withAlt() {
fun testCharLiteral_OldSc() {
val orig = SourceCode.Text("""
main {
ubyte cAlt = @'x'
}
""")
val (txt, _) = roundTrip(parseModule(orig))
txt shouldContain Regex("ubyte +cAlt += +@'x'")
txt shouldContain Regex("ubyte +cAlt += +sc:'x'")
}
@Test
fun testCharLiteral_Sc() {
val orig = SourceCode.Text("""
main {
ubyte cAlt = sc:'x'
}
""")
val (txt, _) = roundTrip(parseModule(orig))
txt shouldContain Regex("ubyte +cAlt += +sc:'x'")
}
@Test

60
compiler/test/ast/TestProg8Parser.kt
View File

@ -446,7 +446,7 @@ class TestProg8Parser: FunSpec( {
rhs.value shouldBe 'x'
}
xtest("on rhs of block-level const decl, with screencode enc (new syntax)") {
test("on rhs of block-level const decl, with screencode enc (new syntax)") {
val src = SourceCode.Text("""
main {
const ubyte c = sc:'x'
@ -462,7 +462,7 @@ class TestProg8Parser: FunSpec( {
rhs.value shouldBe 'x'
}
xtest("on rhs of block-level const decl, with iso encoding") {
test("on rhs of block-level const decl, with iso encoding") {
val src = SourceCode.Text("""
main {
const ubyte c = iso:'_'
@ -517,7 +517,7 @@ class TestProg8Parser: FunSpec( {
rhs.value shouldBe 'x'
}
xtest("on rhs of subroutine-level const decl, screencode (new syntax)") {
test("on rhs of subroutine-level const decl, screencode (new syntax)") {
val src = SourceCode.Text("""
main {
sub start() {
@ -536,7 +536,7 @@ class TestProg8Parser: FunSpec( {
rhs.value shouldBe 'x'
}
xtest("on rhs of subroutine-level const decl, iso encoding") {
test("on rhs of subroutine-level const decl, iso encoding") {
val src = SourceCode.Text("""
main {
sub start() {
@ -558,9 +558,55 @@ class TestProg8Parser: FunSpec( {
context("Strings") {
// TODO test encoding in all available encodings
// check that '~' cant be encoded in petscii and screencode
// check that '~' CAN be encoded correctly in iso
test("default encoding") {
val source = """
main {
str name = "name"
}"""
val module = parseModule(SourceCode.Text(source))
val decl = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as StringLiteralValue
rhs.encoding shouldBe Encoding.PETSCII
rhs.value shouldBe "name"
}
test("old syntax alt encoding") {
val source = """
main {
str name = @"name"
}"""
val module = parseModule(SourceCode.Text(source))
val decl = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<VarDecl>()[0]
val rhs = decl.value as StringLiteralValue
rhs.encoding shouldBe Encoding.SCREENCODES
rhs.value shouldBe "name"
}
test("new syntax encodings") {
val source = """
main {
str name1 = petscii:"Name"
str name2 = sc:"Name"
str name3 = iso:"Name"
}"""
val module = parseModule(SourceCode.Text(source))
val (decl1, decl2, decl3) = module
.statements.filterIsInstance<Block>()[0]
.statements.filterIsInstance<VarDecl>()
val rhs1 = decl1.value as StringLiteralValue
val rhs2 = decl2.value as StringLiteralValue
val rhs3 = decl3.value as StringLiteralValue
rhs1.encoding shouldBe Encoding.PETSCII
rhs1.value shouldBe "Name"
rhs2.encoding shouldBe Encoding.SCREENCODES
rhs2.value shouldBe "Name"
rhs3.encoding shouldBe Encoding.ISO
rhs3.value shouldBe "Name"
}
}
context("Ranges") {

28
compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt
View File

@ -321,8 +321,10 @@ internal fun Prog8ANTLRParser.DirectiveContext.toAst() : Directive =
private fun Prog8ANTLRParser.DirectiveargContext.toAst() : DirectiveArg {
val str = stringliteral()
if(str?.ALT_STRING_ENCODING() != null)
throw SyntaxError("can't use alternate string s for directive arguments", toPosition())
val oldenc = str?.old_alt_encoding?.text
val enc = str?.encoding?.text
if(oldenc!=null || enc !=null)
throw SyntaxError("can't use alternate string encoding for directive arguments", toPosition())
return DirectiveArg(str?.text?.substring(1, text.length-1), identifier()?.text, integerliteral()?.toAst()?.number?.toUInt(), toPosition())
}
@ -451,15 +453,29 @@ private fun Prog8ANTLRParser.ExpressionContext.toAst() : Expression {
private fun Prog8ANTLRParser.CharliteralContext.toAst(): CharLiteral {
val text = this.SINGLECHAR().text
// TODO ISO-encoding, alternative encoding syntax
val encoding = if(ALT_STRING_ENCODING()==null) Encoding.PETSCII else Encoding.SCREENCODES
val enc = this.encoding?.text
val encoding =
if(old_alt_encoding!=null)
Encoding.SCREENCODES
else if(enc!=null)
Encoding.values().singleOrNull { it.prefix == enc }
?: throw SyntaxError("invalid encoding", toPosition())
else
Encoding.PETSCII
return CharLiteral(unescape(text.substring(1, text.length-1), toPosition())[0], encoding, toPosition())
}
private fun Prog8ANTLRParser.StringliteralContext.toAst(): StringLiteralValue {
val text=this.STRING().text
// TODO ISO-encoding, alternative encoding syntax
val encoding = if(ALT_STRING_ENCODING()==null) Encoding.PETSCII else Encoding.SCREENCODES
val enc = encoding?.text
val encoding =
if(old_alt_encoding!=null)
Encoding.SCREENCODES
else if(enc!=null)
Encoding.values().singleOrNull { it.prefix == enc }
?: throw SyntaxError("invalid encoding", toPosition())
else
Encoding.PETSCII
return StringLiteralValue(unescape(text.substring(1, text.length-1), toPosition()), encoding, toPosition())
}

4
docs/source/todo.rst
View File

@ -3,8 +3,8 @@ TODO
For next compiler release (7.7)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
- implement iso encoding and alternate encoding syntax
- document new encoding syntax
- document new string encoding syntax
- fix petscii decode methods that still throw an exception rather than Result

5
parser/antlr/Prog8ANTLR.g4
View File

@ -28,7 +28,6 @@ DEC_INTEGER : ('0'..'9') | (('1'..'9')('0'..'9')+);
HEX_INTEGER : '$' (('a'..'f') | ('A'..'F') | ('0'..'9'))+ ;
BIN_INTEGER : '%' ('0' | '1')+ ;
ADDRESS_OF: '&';
ALT_STRING_ENCODING: '@';
FLOAT_NUMBER : FNUMBER (('E'|'e') ('+' | '-')? FNUMBER)? ; // sign comes later from unary expression
fragment FNUMBER : ('0' .. '9') + ('.' ('0' .. '9') +)? ;
@ -221,9 +220,9 @@ booleanliteral : 'true' | 'false' ;
arrayliteral : '[' EOL? expression (',' EOL? expression)* EOL? ']' ; // you can split the values over several lines
stringliteral : ALT_STRING_ENCODING? STRING ;
stringliteral : (old_alt_encoding='@' | encoding=NAME ':')? STRING ;
charliteral : ALT_STRING_ENCODING? SINGLECHAR ;
charliteral : (old_alt_encoding='@' | encoding=NAME ':')? SINGLECHAR ;
floatliteral : FLOAT_NUMBER ;