allow underscores for numerical grouping

compiler/test/ast/TestProg8Parser.kt

@@ -1025,4 +1025,25 @@ main {
         compileText(VMTarget(),  false, src, writeAssembly = false) shouldNotBe null
     }
 
+    test("underscores for numeric groupings") {
+        val src="""
+%option enable_floats
+main {
+    sub start() {
+        uword w1 = 000_1234_5__
+        uword w2 = ${'$'}ff_ee
+        uword w3 = %11_0000_111111__0000
+        float fl = 3_000_001.141_592_654
+    }
+}"""
+        val result = compileText(VMTarget(),  false, src, writeAssembly = false)!!
+        val st = result.compilerAst.entrypoint.statements
+        st.size shouldBe 8
+        val assigns = st.filterIsInstance<Assignment>()
+        (assigns[0].value as NumericLiteral).number shouldBe 12345
+        (assigns[1].value as NumericLiteral).number shouldBe 0xffee
+        (assigns[2].value as NumericLiteral).number shouldBe 0b1100001111110000
+        (assigns[3].value as NumericLiteral).number shouldBe 3000001.141592654
+    }
+
 })

compilerAst/src/prog8/ast/antlr/Antlr2Kotlin.kt

@@ -371,13 +371,14 @@ private fun DirectiveargContext.toAst() : DirectiveArg {
 }
 
 private fun IntegerliteralContext.toAst(): NumericLiteralNode {
-    fun makeLiteral(text: String, radix: Int): NumericLiteralNode {
+    fun makeLiteral(literalTextWithGrouping: String, radix: Int): NumericLiteralNode {
+        val literalText = literalTextWithGrouping.replace("_", "")
         val integer: Int
         var datatype = DataType.UBYTE
         when (radix) {
             10 -> {
                 integer = try {
-                    text.toInt()
+                    literalText.toInt()
                 } catch(x: NumberFormatException) {
                     throw SyntaxError("invalid decimal literal ${x.message}", toPosition())
                 }
@@ -391,19 +392,19 @@ private fun IntegerliteralContext.toAst(): NumericLiteralNode {
                 }
             }
             2 -> {
-                if(text.length>8)
+                if(literalText.length>8)
                     datatype = DataType.UWORD
                 try {
-                    integer = text.toInt(2)
+                    integer = literalText.toInt(2)
                 } catch(x: NumberFormatException) {
                     throw SyntaxError("invalid binary literal ${x.message}", toPosition())
                 }
             }
             16 -> {
-                if(text.length>2)
+                if(literalText.length>2)
                     datatype = DataType.UWORD
                 try {
-                    integer = text.toInt(16)
+                    integer = literalText.toInt(16)
                 } catch(x: NumberFormatException) {
                     throw SyntaxError("invalid hexadecimal literal ${x.message}", toPosition())
                 }
@@ -542,7 +543,7 @@ private fun Expression_listContext.toAst() = expression().map{ it.toAst() }
 private fun Scoped_identifierContext.toAst() : IdentifierReference =
         IdentifierReference(NAME().map { it.text }, toPosition())
 
-private fun FloatliteralContext.toAst() = text.toDouble()
+private fun FloatliteralContext.toAst() = text.replace("_","").toDouble()
 
 private fun BooleanliteralContext.toAst() = when(text) {
     "true" -> true

docs/source/todo.rst

@@ -2,6 +2,8 @@
 TODO
 ====
 
+- document underscores in numeric literals for grouping
+
 - [on branch: shortcircuit] investigate McCarthy evaluation again? this may also reduce code size perhaps for things like if a>4 or a<2 ....
 
 ...
@@ -78,7 +80,6 @@ What if we were to re-introduce Structs in prog8? Some thoughts:
 Other language/syntax features to think about
 ---------------------------------------------
 
-- underscores in numeric literals for grouping
 - chained comparisons   `10<x<20` ,   `x==y==z`   (desugars to  `10<x and x<20`,   `x==y and y==z`) BUT this changes the semantics of what it is right now ! (x==(y==z) 0> x==true)
 - postincrdecr as expression, preincrdecr expression   (`y = x++`,   `y = ++x`)    .... is this even possible, expression with side effects like this?
 - negative array index to refer to an element from the end of the array.  Python `[-1]` or Raku syntax `[\*-1]`  , `[\*/2]` .... \*=size of the array

examples/test.p8

@@ -1,25 +1,9 @@
-%import textio
-%zeropage basicsafe
-
+%option enable_floats
 main {
-    const ubyte VAL = 11
     sub start() {
-        uword w
-
-        for w in 0 to 20 {
-            ubyte x,y,z=13
-
-            txt.print_ub(x)
-            txt.spc()
-            txt.print_ub(y)
-            txt.spc()
-            txt.print_ub(z)
-            txt.spc()
-            txt.print_uw(w)
-            txt.nl()
-            x++
-            y++
-            z++
-        }
+        uword w1 = 000_1234_5__
+        uword w2 = $ff_ee
+        uword w3 = %11_0000_111111__0000
+        float fl = 3_000_001.141_592_654
     }
 }

parser/antlr/Prog8ANTLR.g4

@@ -25,16 +25,20 @@ WS :  [ \t] -> skip ;
 // WS2 : '\\' EOL -> skip;
 VOID: 'void';
 NAME :  [\p{Letter}][\p{Letter}\p{Mark}\p{Digit}_]* ;           // match unicode properties
-DEC_INTEGER :  ('0'..'9') | (('1'..'9')('0'..'9')+);
-HEX_INTEGER :  '$' (('a'..'f') | ('A'..'F') | ('0'..'9'))+ ;
-BIN_INTEGER :  '%' ('0' | '1')+ ;
+DEC_INTEGER :  DEC_DIGIT (DEC_DIGIT | '_')* ;
+HEX_INTEGER :  '$' HEX_DIGIT (HEX_DIGIT | '_')* ;
+BIN_INTEGER :  '%' BIN_DIGIT (BIN_DIGIT | '_')* ;
 ADDRESS_OF: '&' ;
 INVALID_AND_COMPOSITE: '&&' ;
 
+fragment HEX_DIGIT: ('a'..'f') | ('A'..'F') | ('0'..'9') ;
+fragment BIN_DIGIT: ('0' | '1') ;
+fragment DEC_DIGIT: ('0'..'9') ;
+
 FLOAT_NUMBER :  FNUMBER (('E'|'e') ('+' | '-')? DEC_INTEGER)? ;	// sign comes later from unary expression
 FNUMBER : FDOTNUMBER |  FNUMDOTNUMBER ;
-FDOTNUMBER : '.' ('0'..'9')+ ;
-FNUMDOTNUMBER : ('0'..'9')+ ('.' ('0'..'9')+ )? ;
+FDOTNUMBER : '.' (DEC_DIGIT | '_')+ ;
+FNUMDOTNUMBER : DEC_DIGIT (DEC_DIGIT | '_')* FDOTNUMBER? ;
 
 STRING_ESCAPE_SEQ :  '\\' . | '\\x' . . | '\\u' . . . .;
 STRING :