Adding number preservation tests which revealed classic tokenizer was not generating the correct tokens for '&', '+', '-', '*', '/', '^', '<', '=', and '>'.

api/src/main/java/org/applecommander/bastools/api/ClassicTokenReader.java

@@ -48,6 +48,8 @@ public class ClassicTokenReader {
     }
 
     static class LinePopulator {
+        /** These are the alternate tokens that do not start with an alphabetic character. */
+        private static final Set<Character> ALT_TOKENS = Set.of('&','+','-','*','/','^','<','=','>');
         private final int lineNo;
         private final LinkedList<Token> tokens;
         private boolean dataFlag = false;
@@ -115,7 +117,8 @@ public class ClassicTokenReader {
                 }
 
                 // Keyword handling
-                if (Character.isLetter(ch)) {
+                // Additional: &+-*/^<=>
+                if (Character.isLetter(ch) || ALT_TOKENS.contains(ch)) {
                     int n = handleKeyword(i, line);
                     if (n == -1) {
                         // No keyword found, must be identifier
@@ -144,10 +147,6 @@ public class ClassicTokenReader {
                 else if (ch == '.') {
                     emitNumber(ch);
                 }
-                // Cases of missed tokens(?)
-                else if (ch == '=') {
-                    emitKeyword(ApplesoftKeyword.eq);
-                }
                 // A "$" _might_ be a directive, figure that out
                 else if (ch == '$') {
                     int n = handleDirective(i, line);

tools/tests/src/test/resources/bt-config.yml

@@ -13,6 +13,19 @@ files:
       30 PRINT "HELLO, WORLD"
       40 END
     suffix: .bas
+  numberPreservation:    # See https://github.com/AppleCommander/bastools/issues/49
+    type: text
+    content: |
+      10 PRINT "MATHING"
+      30 A = .4
+      40 B = 0.6000
+      50 C = -.250
+      60 D = -0.70
+      70 PRINT "A=";A
+      80 PRINT "B=";B
+      90 PRINT "C=";C
+      95 PRINT "D=";D
+    suffix: .bas
 
 tests:
   - name: Forms of help
@@ -64,3 +77,37 @@ tests:
           0811: 2c 38 3a ac 00 00 00 1e 08 0a 00 89 00 24 08 14  ,8:..........$..
           0821: 00 97 00 38 08 1e 00 ba 22 48 45 4c 4c 4f 2c 20  ...8...."HELLO,
           0831: 57 4f 52 4c 44 22 00 3e 08 28 00 80 00 00 00 ..  WORLD".>.(..... 
+  - name: check number preservation
+    steps:
+      # This keeps numbers as-is
+      - command: bt --preserve --list $numberPreservation
+        criteria:
+          whitespace: trim
+        stdout: |
+          10  PRINT "MATHING"
+          30 A = .4
+          40 B = 0.6000
+          50 C =  - .250
+          60 D =  - 0.70
+          70  PRINT "A=";A
+          80  PRINT "B=";B
+          90  PRINT "C=";C
+          95  PRINT "D=";D
+  - name: check numbers not being preserved
+    variables:
+      switch: ["--modern", "--classic"]
+    steps:
+      # These rewrite numbers
+      - command: bt $switch --list $numberPreservation
+        criteria:
+          whitespace: trim
+        stdout: |
+          10  PRINT "MATHING"
+          30 A = 0.4
+          40 B = 0.6
+          50 C =  - 0.25
+          60 D =  - 0.7
+          70  PRINT "A=";A
+          80  PRINT "B=";B
+          90  PRINT "C=";C
+          95  PRINT "D=";D