From 24eac6708b062f37afd1ac43727253cf280a71ae Mon Sep 17 00:00:00 2001 From: Karol Stasiak Date: Sat, 13 Mar 2021 21:40:18 +0100 Subject: [PATCH] Fix escape sequences in many encodings --- include/encoding/apple2.tbl | 4 +-- include/encoding/apple2c.tbl | 1 + include/encoding/apple2e.tbl | 4 +-- include/encoding/apple2gs.tbl | 2 +- include/encoding/cp1250.tbl | 1 + include/encoding/cp1251.tbl | 1 + include/encoding/cp1252.tbl | 1 + include/encoding/cp437.tbl | 1 + include/encoding/cp850.tbl | 1 + include/encoding/cp851.tbl | 1 + include/encoding/cp858.tbl | 1 + include/encoding/cpc_da.tbl | 3 ++ include/encoding/cpc_en.tbl | 3 ++ include/encoding/cpc_es.tbl | 2 ++ include/encoding/cpc_fr.tbl | 3 ++ include/encoding/ebcdic.tbl | 1 + include/encoding/iso8859_3.tbl | 1 + include/encoding/kamenicky.tbl | 1 + include/encoding/koi8f.tbl | 1 + include/encoding/koi8r.tbl | 1 + include/encoding/koi8ru.tbl | 1 + include/encoding/koi8t.tbl | 1 + include/encoding/koi8u.tbl | 1 + include/encoding/macroman.tbl | 1 + include/encoding/mazovia.tbl | 1 + include/encoding/pcw.tbl | 5 +++ include/encoding/pokemon1en.tbl | 1 + include/encoding/pokemon1es.tbl | 1 + include/encoding/pokemon1fr.tbl | 2 ++ .../test/auxilary/EncodingSanitySuite.scala | 31 +++++++++++++++++-- 30 files changed, 72 insertions(+), 7 deletions(-) diff --git a/include/encoding/apple2.tbl b/include/encoding/apple2.tbl index b4c5173a..30260eeb 100644 --- a/include/encoding/apple2.tbl +++ b/include/encoding/apple2.tbl @@ -7,7 +7,7 @@ C0-DF=@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_ a-z=C1 -{q}=02 -{apos}=07 +{q}=A2 +{apos}=A7 {nbsp}=40 {n}=8D diff --git a/include/encoding/apple2c.tbl b/include/encoding/apple2c.tbl index 5bfb0798..143ae8fa 100644 --- a/include/encoding/apple2c.tbl +++ b/include/encoding/apple2c.tbl @@ -13,3 +13,4 @@ EOT=00 {apos}=27 {lbrace}=7b {rbrace}=7d +{pound}=5c diff --git a/include/encoding/apple2e.tbl b/include/encoding/apple2e.tbl index 99baaf18..2d0f9296 100644 --- a/include/encoding/apple2e.tbl +++ b/include/encoding/apple2e.tbl @@ -7,8 +7,8 @@ EOT=7F 20-3f=@abcdefghijklmnopqrstuvwxyz[\]^_ 60-7e=πABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~ -{q}=22 -{apos}=27 +{q}=02 +{apos}=07 {lbrace}=7b {rbrace}=7d {pi}=60 diff --git a/include/encoding/apple2gs.tbl b/include/encoding/apple2gs.tbl index edf0c332..c9191a8b 100644 --- a/include/encoding/apple2gs.tbl +++ b/include/encoding/apple2gs.tbl @@ -9,4 +9,4 @@ E0-FE=`abcdefghijklmnopqrstuvwxyz{\}~ {q}=A2 {apos}=A7 {lbrace}=FB -{rbrace}=FC +{rbrace}=FD diff --git a/include/encoding/cp1250.tbl b/include/encoding/cp1250.tbl index e38a0af0..059aae3c 100644 --- a/include/encoding/cp1250.tbl +++ b/include/encoding/cp1250.tbl @@ -26,6 +26,7 @@ f0-ff=đńňóôőö÷řůúűüýţ˙ {apos}=27 {lbrace}=7b {rbrace}=7d +{euro}=80 {copy}=a9 {ss}=df {nbsp}=A0 diff --git a/include/encoding/cp1251.tbl b/include/encoding/cp1251.tbl index 3bf77d00..6aa375cc 100644 --- a/include/encoding/cp1251.tbl +++ b/include/encoding/cp1251.tbl @@ -23,6 +23,7 @@ f0-ff=рстуфхцчшщъыьэюя {apos}=27 {lbrace}=7b {rbrace}=7d +{euro}=88 {copy}=a9 {nbsp}=A0 {shy}=AD diff --git a/include/encoding/cp1252.tbl b/include/encoding/cp1252.tbl index 30484ec3..8e8fb153 100644 --- a/include/encoding/cp1252.tbl +++ b/include/encoding/cp1252.tbl @@ -25,6 +25,7 @@ f0-ff=ðñòóôõö÷øùúûüýþÿ {apos}=27 {lbrace}=7b {rbrace}=7d +{euro}=80 {cent}=a2 {pound}=a3 {yen}=a5 diff --git a/include/encoding/cp437.tbl b/include/encoding/cp437.tbl index 432abf7e..763f914d 100644 --- a/include/encoding/cp437.tbl +++ b/include/encoding/cp437.tbl @@ -27,4 +27,5 @@ F0-FE=≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ {pound}=9c {yen}=9d {ss}=e1 +{pi}=e3 {nbsp}=FF diff --git a/include/encoding/cp850.tbl b/include/encoding/cp850.tbl index 12493d40..c1bb142f 100644 --- a/include/encoding/cp850.tbl +++ b/include/encoding/cp850.tbl @@ -23,6 +23,7 @@ F1-FE=±‗¾¶§÷¸°¨·¹³²■ {apos}=27 {lbrace}=7b {rbrace}=7d +{copy}=b8 {cent}=BD {pound}=9c {yen}=BE diff --git a/include/encoding/cp851.tbl b/include/encoding/cp851.tbl index e54ccd92..0f81ee6f 100644 --- a/include/encoding/cp851.tbl +++ b/include/encoding/cp851.tbl @@ -24,6 +24,7 @@ F1-FE=±υφχ§ψ¸°¨ωϋΰώ■ {apos}=27 {lbrace}=7b {rbrace}=7d +{pound}=9c {ss}=D7 {nbsp}=FF {shy}=F0 diff --git a/include/encoding/cp858.tbl b/include/encoding/cp858.tbl index f1051e47..45c64de2 100644 --- a/include/encoding/cp858.tbl +++ b/include/encoding/cp858.tbl @@ -25,6 +25,7 @@ F1-FE=±‗¾¶§÷¸°¨·¹³²■ {rbrace}=7d {cent}=BD {pound}=9c +{copy}=b8 {yen}=BE {euro}=D5 {ss}=e1 diff --git a/include/encoding/cpc_da.tbl b/include/encoding/cpc_da.tbl index c9b5f10e..1c916da4 100644 --- a/include/encoding/cpc_da.tbl +++ b/include/encoding/cpc_da.tbl @@ -19,6 +19,9 @@ fe-ff=↕↔ {apos}=27 {lbrace}=7b {rbrace}=7d +{pound}=a3 +{copy}=a4 +{pi}=b8 {AE}=5b {OE}=5c {AA}=5d diff --git a/include/encoding/cpc_en.tbl b/include/encoding/cpc_en.tbl index 809eda7b..496e1967 100644 --- a/include/encoding/cpc_en.tbl +++ b/include/encoding/cpc_en.tbl @@ -19,3 +19,6 @@ fe-ff=↕↔ {apos}=27 {lbrace}=7b {rbrace}=7d +{pound}=a3 +{copy}=a4 +{pi}=b8 diff --git a/include/encoding/cpc_es.tbl b/include/encoding/cpc_es.tbl index c161176c..9d9768e7 100644 --- a/include/encoding/cpc_es.tbl +++ b/include/encoding/cpc_es.tbl @@ -19,3 +19,5 @@ fe-ff=↕↔ {apos}=27 {lbrace}=7b {rbrace}=7d +{copy}=a4 +{pi}=b8 diff --git a/include/encoding/cpc_fr.tbl b/include/encoding/cpc_fr.tbl index c1c4012c..f9255d5a 100644 --- a/include/encoding/cpc_fr.tbl +++ b/include/encoding/cpc_fr.tbl @@ -20,3 +20,6 @@ fe-ff=↕↔ {apos}=27 {lbrace}=7b {rbrace}=7d +{pound}=a3 +{copy}=a4 +{pi}=b8 diff --git a/include/encoding/ebcdic.tbl b/include/encoding/ebcdic.tbl index 187a66a8..13e5b70a 100644 --- a/include/encoding/ebcdic.tbl +++ b/include/encoding/ebcdic.tbl @@ -24,3 +24,4 @@ F0-F9=0123456789 {q}=7F {lbrace}=C0 {rbrace}=D0 +{cent}=4A diff --git a/include/encoding/iso8859_3.tbl b/include/encoding/iso8859_3.tbl index 3d109f69..03b14d59 100644 --- a/include/encoding/iso8859_3.tbl +++ b/include/encoding/iso8859_3.tbl @@ -27,3 +27,4 @@ f1-ff=ñòóôġö÷ĝùúûüŭŝ˙ {ss}=df {nbsp}=A0 {shy}=AD +{pound}=A3 diff --git a/include/encoding/kamenicky.tbl b/include/encoding/kamenicky.tbl index bf94ff56..b69e221b 100644 --- a/include/encoding/kamenicky.tbl +++ b/include/encoding/kamenicky.tbl @@ -24,4 +24,5 @@ F0-FE=≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ {lbrace}=7b {rbrace}=7d {ss}=e1 +{pi}=e3 {nbsp}=FF diff --git a/include/encoding/koi8f.tbl b/include/encoding/koi8f.tbl index c415d701..de58ef68 100644 --- a/include/encoding/koi8f.tbl +++ b/include/encoding/koi8f.tbl @@ -14,6 +14,7 @@ EOT=00 {lbrace}=7b {rbrace}=7d {nbsp}=a0 +{copy}=98 •=95 b0=№ diff --git a/include/encoding/koi8r.tbl b/include/encoding/koi8r.tbl index bd79dcda..1f501297 100644 --- a/include/encoding/koi8r.tbl +++ b/include/encoding/koi8r.tbl @@ -14,6 +14,7 @@ EOT=00 {lbrace}=7b {rbrace}=7d {nbsp}=9a +{copy}=bf 80-8f=─│┌┐└┘├┤┬┴┼▀▄█▌▐ 90-99=░▒▓⌠■∙√≈≤≥ diff --git a/include/encoding/koi8ru.tbl b/include/encoding/koi8ru.tbl index 7bf7013a..2d84de91 100644 --- a/include/encoding/koi8ru.tbl +++ b/include/encoding/koi8ru.tbl @@ -14,6 +14,7 @@ EOT=00 {lbrace}=7b {rbrace}=7d {nbsp}=9a +{copy}=bf 80-8f=─│┌┐└┘├┤┬┴┼▀▄█▌▐ 90-99=░▒▓“■∙”—№™ diff --git a/include/encoding/koi8t.tbl b/include/encoding/koi8t.tbl index 1bbf4489..fcb38323 100644 --- a/include/encoding/koi8t.tbl +++ b/include/encoding/koi8t.tbl @@ -14,6 +14,7 @@ EOT=00 {lbrace}=7b {rbrace}=7d {shy}=8d +{copy}=bf diff --git a/include/encoding/koi8u.tbl b/include/encoding/koi8u.tbl index 4060b1ea..4c14610a 100644 --- a/include/encoding/koi8u.tbl +++ b/include/encoding/koi8u.tbl @@ -14,6 +14,7 @@ EOT=00 {lbrace}=7b {rbrace}=7d {nbsp}=9a +{copy}=bf 80-8f=─│┌┐└┘├┤┬┴┼▀▄█▌▐ 90-99=░▒▓⌠■∙√≈≤≥ diff --git a/include/encoding/macroman.tbl b/include/encoding/macroman.tbl index 4d0ccb6b..180d7508 100644 --- a/include/encoding/macroman.tbl +++ b/include/encoding/macroman.tbl @@ -29,5 +29,6 @@ f1-ff=ÒÚÛÙıˆ˜¯˘˙˚¸˝˛ˇ {ss}=a7 {nbsp}=CA {euro}=DB +{pi}=b9 €=DB U+F8FF=F0 diff --git a/include/encoding/mazovia.tbl b/include/encoding/mazovia.tbl index eac2430b..7542b493 100644 --- a/include/encoding/mazovia.tbl +++ b/include/encoding/mazovia.tbl @@ -26,4 +26,5 @@ F0-FE=≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ {cent}=9b {yen}=9d {ss}=e1 +{pi}=e3 {nbsp}=FF diff --git a/include/encoding/pcw.tbl b/include/encoding/pcw.tbl index f2c7e6c0..bb9bed3b 100644 --- a/include/encoding/pcw.tbl +++ b/include/encoding/pcw.tbl @@ -55,3 +55,8 @@ f0-ff=äëïöüçæåøñãõ⇒⇐⇔≡ {lbrace}=7b {rbrace}=7d {pi}=1b18 +{pound}=a3 +{copy}=a4 +{yen}=bd +{cent}=b1 +{ss}=ba diff --git a/include/encoding/pokemon1en.tbl b/include/encoding/pokemon1en.tbl index 1712d20e..8d103ae9 100644 --- a/include/encoding/pokemon1en.tbl +++ b/include/encoding/pokemon1en.tbl @@ -23,6 +23,7 @@ F0-FF=₽×./,♀0123456789 {ot}=BE {'v}=BF {hav}=BF +{apos}=E0 {'r}=E4 {ar}=E4 {'m}=E5 diff --git a/include/encoding/pokemon1es.tbl b/include/encoding/pokemon1es.tbl index 209ff539..8de1dd24 100644 --- a/include/encoding/pokemon1es.tbl +++ b/include/encoding/pokemon1es.tbl @@ -22,6 +22,7 @@ F0-FF=₽×./,♀0123456789 {'t}=DD {'v}=DE +{apos}=E0 {PK}=E1 {pk}=E1 {MN}=E2 diff --git a/include/encoding/pokemon1fr.tbl b/include/encoding/pokemon1fr.tbl index 22de6e11..557ac33a 100644 --- a/include/encoding/pokemon1fr.tbl +++ b/include/encoding/pokemon1fr.tbl @@ -13,6 +13,7 @@ E6-E8=?!. EC-EF=▷▶▼♂ F0-FF=₽×./,♀0123456789 +{ss}=be {c'}=d4 {ce}=d4 {d'}=d5 @@ -37,6 +38,7 @@ F0-FF=₽×./,♀0123456789 {u'}=de {ue}=de {y'}=df +{apos}=e0 {PK}=E1 {pk}=E1 diff --git a/src/test/scala/millfork/test/auxilary/EncodingSanitySuite.scala b/src/test/scala/millfork/test/auxilary/EncodingSanitySuite.scala index 93308ce4..50c639dc 100644 --- a/src/test/scala/millfork/test/auxilary/EncodingSanitySuite.scala +++ b/src/test/scala/millfork/test/auxilary/EncodingSanitySuite.scala @@ -18,7 +18,7 @@ class EncodingSanitySuite extends FunSuite with Matchers { test("Encoding sanity test") { val log = new ConsoleLogger() val repo = new TextCodecRepository(List("include")) - val options = CompilationOptions(EmuPlatform.get(Cpu.Intel8080), Map(), None, 0, Map(), new TextCodecRepository(List("D:/dokumenty/millfork/include/encoding")), JobContext(log, new LabelGenerator)) + val options = CompilationOptions(EmuPlatform.get(Cpu.Intel8080), Map(), None, 0, Map(), new TextCodecRepository(List("include")), JobContext(log, new LabelGenerator)) def roundtrip(codec: TableTextCodec, str1: String, str2: String): Unit = { val l1 = codec.encode(log, None, str1.toCharArray.map(_.toInt).toList, options, lenient = false) @@ -27,8 +27,18 @@ class EncodingSanitySuite extends FunSuite with Matchers { fail(s"Strings $str1 and $str2 encoded to $l1 and $l2 in encoding ${codec.name}") } } + def getSingleByte(codec: TableTextCodec, str1: String): Int = { + val l1 = codec.encode(log, None, str1.toCharArray.map(_.toInt).toList, options, lenient = false) + if (l1.size != 1) { + fail(s"String $str1 encoded to $l1 in encoding ${codec.name}") + } + l1.head + } - for (encoding <- new File("include/encoding").list()) { + val allEncodingFiles = new File("./include/encoding").list() + log.info(s"Testing ${allEncodingFiles.size} encodings") + for (encoding <- allEncodingFiles) { + log.info(s"Testing $encoding") repo.forName(encoding.stripSuffix(".tbl"), None, log).codec match { case codec:TableTextCodec => codec.escapeSequences.foreach { @@ -38,6 +48,23 @@ class EncodingSanitySuite extends FunSuite with Matchers { case ("yen", _) => roundtrip(codec, "{yen}", "¥") case ("pound", _) => roundtrip(codec, "{pound}", "£") case ("cent", _) => roundtrip(codec, "{cent}", "¢") + case ("euro", _) => roundtrip(codec, "{euro}", "€") + case ("apos", _) => roundtrip(codec, "{apos}", "'") + case ("q", _) => roundtrip(codec, "{q}", "\"") + case _ => + } + codec.map.foreach { + case '©' => roundtrip(codec, "{copy}", "©") + case 'ß' => roundtrip(codec, "{ss}", "ß") + case 'π' => roundtrip(codec, "{pi}", "π") + case '¥' => roundtrip(codec, "{yen}", "¥") + case '£' => roundtrip(codec, "{pound}", "£") + case '¢' => roundtrip(codec, "{cent}", "¢") + case '€' => roundtrip(codec, "{euro}", "€") + case '\'' => roundtrip(codec, "{apos}", "'") + case '"' => roundtrip(codec, "{q}", "\"") + case '{' => getSingleByte(codec, "{lbrace}") should equal (codec.map.indexOf('{')) + case '}' => getSingleByte(codec, "{rbrace}") should equal (codec.map.indexOf('}')) case _ => } case _ =>