ecs: token.eol; fixed bugs and tests

src/common/ecs/compiler.ts

@@ -41,6 +41,7 @@ export class ECSCompiler extends Tokenizer {
             { type: TokenType.Ident, regex: /[A-Za-z_][A-Za-z0-9_]*/ },
             { type: TokenType.Ignore, regex: /\/\/.*?[\n\r]/ },
             { type: TokenType.Ignore, regex: /\/\*.*?\*\// },
+            { type: TokenType.EOL, regex: /[\n\r]+/ },
             { type: TokenType.Ignore, regex: /\s+/ },
         ]);
         this.errorOnCatchAll = true;

src/common/ecs/ecs.ts

@@ -1548,9 +1548,9 @@ export class EntityScope implements SourceLocated {
                 }
                 let subcall = this.dialect.call(stats.labels[0]);
                 for (let label of stats.labels) {
-                    // TODO: use dialect
+                    // TODO: need to trim()?
-                    let startdelim = this.dialect.comment(`start action ${label}`);
+                    let startdelim = this.dialect.comment(`start action ${label}`).trim();
-                    let enddelim = this.dialect.comment(`end action ${label}`);
+                    let enddelim = this.dialect.comment(`end action ${label}`).trim();
                     let istart = code.indexOf(startdelim);
                     let iend = code.indexOf(enddelim, istart);
                     if (istart >= 0 && iend > istart) {

src/common/ecs/main.ts

@@ -4,7 +4,7 @@ import { Dialect_CA65, EntityManager, SourceFileExport } from "./ecs";
 
 class ECSMain {
 
-    compiler = new ECSCompiler(new EntityManager(new Dialect_CA65())); // TODO
+    compiler = new ECSCompiler(new EntityManager(new Dialect_CA65()), true); // TODO
 
     constructor(readonly args: string[]) {
     }

src/common/tokenizer.ts

@@ -32,6 +32,7 @@ export enum TokenType {
 export class Token implements SourceLocated {
     str: string;
     type: string;
+    eol: boolean;   // end of line?
     $loc: SourceLocation;
 }
 
@@ -92,7 +93,7 @@ export class Tokenizer {
             this.lineindex.push(m.index);
         }
         this._tokenize(contents);
-        this.eof = { type: TokenType.EOF, str: "", $loc: { path: this.path, line: this.lineno } };
+        this.eof = { type: TokenType.EOF, str: "", eol: true, $loc: { path: this.path, line: this.lineno } };
         this.pushToken(this.eof);
     }
     _tokenize(text: string): void {
@@ -121,7 +122,12 @@ export class Tokenizer {
                                 this.compileError(`I didn't expect the character "${m[0]}" here.`, loc);
                             }
                         default:
-                            this.pushToken({ str: s, type: rule.type, $loc: loc });
+                            this.pushToken({ str: s, type: rule.type, $loc: loc, eol: false });
+                            break;
+                        case TokenType.EOL:
+                            // set EOL for last token
+                            if (this.tokens.length)
+                                this.tokens[this.tokens.length-1].eol = true;
                         case TokenType.Comment:
                         case TokenType.Ignore:
                             break;

src/test/testutil.ts

@@ -143,6 +143,7 @@ describe('Tokenizer', function () {
       { type: 'delim', regex: /[\(\)\{\}\[\]]/ },
       { type: 'qstring', regex: /".*?"/ },
       { type: 'integer', regex: /[-]?\d+/ },
+      { type: 'eol', regex: /\n+/ },
       { type: 'ignore', regex: /\s+/ },
     ]);
     t.tokenizeFile("a\n{\"key\" value\n \"number\" 531\n\n \"f\" (fn [x] (+ x 2))}\n", "test.file");
@@ -150,6 +151,8 @@ describe('Tokenizer', function () {
       'ident delim qstring ident qstring integer qstring delim ident delim ident delim delim catch-all ident integer delim delim delim eof');
     assert.strictEqual(t.tokens.map(t => t.str).join(' '),
       'a { "key" value "number" 531 "f" ( fn [ x ] ( + x 2 ) ) } ');
+    assert.strictEqual(t.tokens.filter(t => t.eol).map(t => t.str).join(' '),
+      'a value 531 } ');
     assert.strictEqual(t.tokens.map(t => t.$loc.line).join(' '),
       '1 2 2 2 3 3 5 5 5 5 5 5 5 5 5 5 5 5 5 6');
     assert.strictEqual(20, t.tokens.length);

test/ecs/errors1.ecs

@@ -28,4 +28,4 @@ demo Main
     const lines = 2
     const color = $16
   end
-end
+end demo

test/ecs/score.ecs

@@ -293,4 +293,5 @@ demo Main
     {{!AddBCD4 $0210}}
 ---
   end
-end
+end demo
+

test/ecs/score.txt

@@ -1,20 +1,13 @@
-EVENT__main_init = 1
 EVENT__start = 1
-EVENT__nextframe = 1
-EVENT__resetswitch = 1
-EVENT__postframe = 1
-EVENT__SetHorizPos = 1
 EVENT__preframe = 1
 EVENT__kernel = 1
-EVENT__kerneldraw = 1
-EVENT__kernelsetup = 1
-EVENT__joybutton = 1
 EVENT__FontTable = 1
 EVENT__compute2digit = 1
 EVENT__fetchdigit = 1
 EVENT__FontTablePF = 1
-EVENT__FontTablePFFancy = 1
+EVENT__postframe = 1
 EVENT__AddBCD4 = 1
+EVENT__joybutton = 1
 .scope Main
 .zeropage
 BCDScore6_digits_b0:

test/ecs/sprites.ecs

@@ -436,5 +436,6 @@ demo Main
     	init sprite = #Sprite3
     end
 
-end
+end demo
+
 

test/ecs/sprites.txt

@@ -1,19 +1,13 @@
-EVENT__main_init = 1
 EVENT__start = 1
-EVENT__nextframe = 1
-EVENT__resetswitch = 1
-EVENT__postframe = 1
-EVENT__SetHorizPos = 1
 EVENT__preframe = 1
 EVENT__kernel = 1
-EVENT__kerneldraw = 1
-EVENT__kernelsetup = 1
-EVENT__joybutton = 1
 EVENT__scanline = 1
+EVENT__postframe = 1
 EVENT__joyleft = 1
 EVENT__joyright = 1
 EVENT__joyup = 1
 EVENT__joydown = 1
+EVENT__SetHorizPos = 1
 .scope Main
 .zeropage
 HasBitmap_bitmap_b0:

test/ecs/sprites1.ecs

@@ -396,5 +396,6 @@ demo Main
     	init sprite = #Sprite3
     end
 
-end
+end demo
+
 

test/ecs/sprites1.txt

@@ -1,20 +1,13 @@
-EVENT__main_init = 1
 EVENT__start = 1
-EVENT__nextframe = 1
-EVENT__resetswitch = 1
-EVENT__postframe = 1
-EVENT__SetHorizPos = 1
 EVENT__preframe = 1
 EVENT__kernel = 1
-EVENT__kerneldraw = 1
-EVENT__kernelsetup = 1
-EVENT__joybutton = 1
 EVENT__scanline1 = 1
-EVENT__scanline2 = 1
+EVENT__postframe = 1
 EVENT__joyleft = 1
 EVENT__joyright = 1
 EVENT__joyup = 1
 EVENT__joydown = 1
+EVENT__SetHorizPos = 1
 .scope Main
 .zeropage
 HasBitmap_bitmap_b0:

test/ecs/superman.txt

@@ -1,23 +1,17 @@
-EVENT__main_init = 1
 EVENT__start = 1
-EVENT__nextframe = 1
-EVENT__resetswitch = 1
-EVENT__postframe = 1
-EVENT__SetHorizPos = 1
 EVENT__preframe = 1
 EVENT__kernel = 1
-EVENT__kerneldraw = 1
-EVENT__kernelsetup = 1
-EVENT__joybutton = 1
 EVENT__scanline = 1
+EVENT__postframe = 1
 EVENT__joyleft = 1
 EVENT__joyright = 1
-EVENT__joyup = 1
-EVENT__joydown = 1
 EVENT__gowest = 1
 EVENT__goeast = 1
 EVENT__gonorth = 1
 EVENT__gosouth = 1
+EVENT__joyup = 1
+EVENT__joydown = 1
+EVENT__SetHorizPos = 1
 .scope Main
 .zeropage
 Location_room_b0:

test/ecs/titles.ecs

@@ -256,4 +256,5 @@ end
     ---
   end
 
-end
+end demo
+

test/ecs/titles.txt

@@ -1,13 +1,9 @@
-EVENT__main_init = 1
 EVENT__start = 1
-EVENT__nextframe = 1
+EVENT__kernelsetup = 1
-EVENT__resetswitch = 1
+EVENT__kerneldraw = 1
-EVENT__postframe = 1
-EVENT__SetHorizPos = 1
 EVENT__preframe = 1
 EVENT__kernel = 1
-EVENT__kerneldraw = 1
+EVENT__postframe = 1
-EVENT__kernelsetup = 1
 EVENT__joybutton = 1
 .scope TitleDemo
 .zeropage

test/ecs/vcs1.ecs

@@ -231,4 +231,5 @@ demo Main
     inc {{set Trees.pfcolor}}
     ---
   end
-end
+end demo
+

test/ecs/vcs1.txt

@@ -1,12 +1,10 @@
-EVENT__main_init = 1
 EVENT__start = 1
 EVENT__nextframe = 1
 EVENT__resetswitch = 1
-EVENT__postframe = 1
-EVENT__SetHorizPos = 1
 EVENT__preframe = 1
 EVENT__kernel = 1
 EVENT__kernelsetup = 1
+EVENT__postframe = 1
 EVENT__joybutton = 1
 .scope Main
 .zeropage

test/ecs/vcslib.ecs

@@ -236,4 +236,5 @@ demo Main
     inc {{set Trees.pfcolor}}
     ---
   end
-end
+end demo
+

test/ecs/vcslib.txt

@@ -1,13 +1,11 @@
-EVENT__main_init = 1
 EVENT__start = 1
 EVENT__nextframe = 1
 EVENT__resetswitch = 1
-EVENT__postframe = 1
-EVENT__SetHorizPos = 1
 EVENT__preframe = 1
 EVENT__kernel = 1
 EVENT__kerneldraw = 1
 EVENT__kernelsetup = 1
+EVENT__postframe = 1
 EVENT__joybutton = 1
 .scope Main
 .zeropage