Underscores in numeric literals. Fix parsing of Intel hex constants starting with 0B.

docs/lang/literals.md

@@ -14,7 +14,14 @@ Octal: `0o172`
 
 Hexadecimal: `$D323`, `0x2a2`
 
-When using Intel syntax for inline assembly, another hexadecimal syntax is available: `0D323H`, `2a2h`.
+Digits can be separated by underscores for readability. Underscores are also allowed between the radix prefix and the digits:
+
+    123_456
+    0x01_ff
+    0b_0101_1111__1100_0001
+    $___________FF
+
+When using Intel syntax for inline assembly, another hexadecimal syntax is available: `0D323H`, `2a2h`, `3e_h`.
 It is not allowed in any other places.
 
 The type of a literal is the smallest type of undefined signedness

src/main/scala/millfork/parser/MfParser.scala

@@ -3,15 +3,16 @@ package millfork.parser
 import java.lang.Long.parseLong
 import java.nio.file.{Files, Paths}
 import java.util
-
 import fastparse.all._
 import fastparse.core.Parsed.Failure
+import fastparse.parsers.Intrinsics
 import millfork.assembly.Elidability
 import millfork.env._
 import millfork.error.{ConsoleLogger, Logger}
 import millfork.node._
 import millfork.output.{DivisibleAlignment, MemoryAlignment, NoAlignment}
 import millfork.{CompilationFlag, CompilationOptions, Confusables, SeparatedList}
+import org.apache.commons.lang3.StringUtils
 
 import scala.collection.immutable.BitSet
 
@@ -192,7 +193,7 @@ abstract class MfParser[T](fileId: String, input: String, currentDirectory: Stri
 
   val literalAtom: P[LiteralExpression] = binaryAtom | hexAtom | octalAtom | quaternaryAtom | decimalAtom | charAtom
 
-  val literalAtomWithIntel: P[LiteralExpression] = binaryAtom | hexAtom | octalAtom | quaternaryAtom | intelHexAtom | decimalAtom | charAtom
+  val literalAtomWithIntel: P[LiteralExpression] = hexAtom | octalAtom | quaternaryAtom | intelHexAtom | binaryAtom | decimalAtom | charAtom
 
   val atom: P[Expression] = P(position("atom") ~ (variableAtom | localLabelAtom | literalAtom | textLiteralAtom)).map{case (p,a) => a.pos(p)}
 
@@ -894,8 +895,22 @@ object MfParser {
     Character.SURROGATE,
     Character.UNASSIGNED)
 
-  val decimalAtom: P[LiteralExpression] =
-    ("-".!.? ~ CharsWhileIn("0123456789", min = 1).!.opaque("<decimal digits>") ~ !(CharIn("xXbBoOqQ".toSeq))).map{
+  private val decimalDigit: P[String] = CharIn("0123456789").!
+  private val decimalDigits: P[String] = CharsWhileIn("0123456789", min = 1).!
+  private val binaryDigits: P[String] = CharsWhileIn("01", min = 1).!
+  private val hexDigits: P[String] = CharsWhileIn("1234567890abcdefABCDEF", min = 1).!
+  private val octalDigits: P[String] = CharsWhileIn("01234567", min = 1).!
+  private val quaternaryDigits: P[String] = CharsWhileIn("0123", min = 1).!
+
+  private def underscores(digits: P[String], initialDigit: P[String] = null): P[String] =
+    (if (initialDigit eq null) {
+      ("_".rep(min = 0) ~ digits).rep(min = 1)
+    } else {
+      (initialDigit ~ ("_".rep(min = 0) ~ digits).rep(min = 0))
+    }).!.map(s => StringUtils.remove(s, '_'))
+
+    val decimalAtom: P[LiteralExpression] =
+      ("-".!.? ~ underscores(decimalDigits, initialDigit = decimalDigit).opaque("<decimal digits>") ~ !(CharIn("xXbBoOqQhH".toSeq))).map{
       case (minus, s) =>
         val abs = parseLong(s, 10)
         val value = sign(abs, minus.isDefined)
@@ -903,7 +918,7 @@ object MfParser {
     }
 
   val binaryAtom: P[LiteralExpression] =
-    ("-".!.? ~ ("0b" | "0B" | "%") ~/ CharsWhileIn("01", min = 1).!.opaque("<binary digits>")).map{
+    ("-".!.? ~ ("0b" | "0B" | "%") ~/ underscores(binaryDigits).opaque("<binary digits>")).map{
       case (minus, s) =>
         val abs = parseLong(s, 2)
         val value = sign(abs, minus.isDefined)
@@ -911,7 +926,7 @@ object MfParser {
     }
 
   val hexAtom: P[LiteralExpression] =
-    ("-".!.? ~ ("0x" | "0X" | "$") ~/ CharsWhileIn("1234567890abcdefABCDEF", min = 1).!.opaque("<hex digits>")).map{
+    ("-".!.? ~ ("0x" | "0X" | "$") ~/ underscores(hexDigits).opaque("<hex digits>")).map{
       case (minus, s) =>
         val abs = parseLong(s, 16)
         val value = sign(abs, minus.isDefined)
@@ -919,16 +934,16 @@ object MfParser {
     }
 
   val intelHexAtom: P[LiteralExpression] =
-    ("-".!.? ~ CharIn("0123456789").! ~ CharsWhileIn("1234567890abcdefABCDEF", min = 1).!.opaque("<hex digits>") ~ P("h" | "H") ~/ Pass).map{
-      case (minus, head, tail) =>
-        val s = if (head == "0" && tail.nonEmpty && tail.head >'9') tail else head + tail
+    ("-".!.? ~ underscores(hexDigits, initialDigit = decimalDigit).opaque("<hex digits>") ~ "_".rep(min = 0) ~ P("h" | "H") ~/ Pass).map{
+      case (minus, digits) =>
+        val s = if (digits.startsWith("0") && digits.length > 1 && digits(1) > '9') digits.tail else digits
         val abs = parseLong(s, 16)
         val value = sign(abs, minus.isDefined)
         LiteralExpression(value, size(value, s.length > 2, s.length > 4, s.length > 6, s.length > 8, s.length > 10, s.length > 12, s.length > 14))
     }
 
   val octalAtom: P[LiteralExpression] =
-    ("-".!.? ~ ("0o" | "0O") ~/ CharsWhileIn("01234567", min = 1).!.opaque("<octal digits>")).map{
+    ("-".!.? ~ ("0o" | "0O") ~/ underscores(octalDigits).opaque("<octal digits>")).map{
       case (minus, s) =>
         val abs = parseLong(s, 8)
         val value = sign(abs, minus.isDefined)
@@ -936,7 +951,7 @@ object MfParser {
     }
 
   val quaternaryAtom: P[LiteralExpression] =
-    ("-".!.? ~ ("0q" | "0Q") ~/ CharsWhileIn("0123", min = 1).!.opaque("<quaternary digits>")).map{
+    ("-".!.? ~ ("0q" | "0Q") ~/ underscores(quaternaryDigits).opaque("<quaternary digits>")).map{
       case (minus, s) =>
       val abs = parseLong(s, 4)
       val value = sign(abs, minus.isDefined)

src/test/scala/millfork/test/BasicSymonTest.scala

@@ -332,14 +332,17 @@ class BasicSymonTest extends FunSuite with Matchers {
   test("Numeric literals") {
     EmuUnoptimizedRun(
       """
+        |#if 1_1 == 0x_0_b
+        |#endif
         |array a @$c000 = [
         | %0,  %1,  %001,  %000001,  %11100,
-        |0b0, 0b1, 0b001, 0b000001, 0b11100,
-        |0q12, 0q1, 0q0, 0q1230,
-        |0o001, 0o0, 0o6, 0o52,
+        |0b0, 0b1, 0b001, 0b000001, 0b11100, 0b___00_1,
+        |0q12, 0q1, 0q0, 0q1230, 0q_00_1,
+        |0o001, 0o0, 0o6, 0o52, 0o_00_1,
         |0x23, 0xdd, 0x55, 0x0, 0x1, 0xf,
-        | $23,  $dd,  $55,  $0,  $1,  $f,
-        |000
+        | $23,  $dd,  $55,  $0,  $1,  $f, $___1,
+        | 0x1_1, 0x_1___1,
+        |000, 0_0, 0, 1
         |]
         |
         |void main() {

src/test/scala/millfork/test/Z80AssemblySuite.scala

@@ -275,6 +275,27 @@ class Z80AssemblySuite extends FunSuite with Matchers {
         | }
       """.stripMargin)
   }
+
+  test("Intel hex syntax disambiguation") {
+    EmuUnoptimizedCrossPlatformRun(Cpu.Intel8080)(
+      """
+        | #pragma intel_syntax
+        | asm void main () {
+        |   lxi h, 0c000h
+        |   mvi m, 0b1h
+        |   mvi m, 0b1_h
+        |   mvi m, 0b_1_h
+        |   inx h
+        |   mvi m, 0b1
+        |   mvi m, 0b_1
+        |   ret
+        |   }
+        |   """.stripMargin){m =>
+      m.readByte(0xc000) should equal(0xb1)
+      m.readByte(0xc001) should equal(1)
+    }
+  }
+
   test("Common I80 instructions (without RST, Intel syntax)") {
     EmuUnoptimizedCrossPlatformRun(Cpu.Intel8080, Cpu.Intel8086)(
       """
@@ -288,17 +309,17 @@ class Z80AssemblySuite extends FunSuite with Matchers {
         |   inx b
         |   inr b
         |   dcr b
-        |   mvi b,6
+        |   mvi b,6h
         |   rlc
         |   dad b
         |   ldax b
         |   dcx b
         |   inr c
         |   dcr c
-        |   mvi c,0eh
+        |   mvi c,0e_h
         |   rrc
         |
-        |   lxi d,1111h
+        |   lxi d,11_11h
         |   stax d
         |   inx d
         |   inr d
@@ -491,7 +512,7 @@ class Z80AssemblySuite extends FunSuite with Matchers {
         |   jmp main
         |   cnz main
         |   push b
-        |   adi 1
+        |   adi 1_h
         |
         |   rz
         |   ret