use {{ }} delimiters, tokenizer, testecs, compiler

This commit is contained in:
Steven Hugg 2022-01-28 11:22:59 -06:00
parent 46f6063929
commit d2caee8cc3
8 changed files with 1026 additions and 426 deletions

9
package-lock.json generated
View File

@ -21,7 +21,6 @@
"fast-png": "^5.0.4",
"file-saver": "^2.0.5",
"jquery": "^3.6.0",
"js-yaml": "^4.1.0",
"jszip": "^3.7.0",
"localforage": "^1.9.0",
"mousetrap": "^1.6.5",
@ -746,7 +745,8 @@
"node_modules/argparse": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
"dev": true
},
"node_modules/array-equal": {
"version": "1.0.0",
@ -3599,6 +3599,7 @@
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
"integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
"dev": true,
"dependencies": {
"argparse": "^2.0.1"
},
@ -8029,7 +8030,8 @@
"argparse": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
"dev": true
},
"array-equal": {
"version": "1.0.0",
@ -10205,6 +10207,7 @@
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
"integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
"dev": true,
"requires": {
"argparse": "^2.0.1"
}

View File

@ -22,7 +22,6 @@
"fast-png": "^5.0.4",
"file-saver": "^2.0.5",
"jquery": "^3.6.0",
"js-yaml": "^4.1.0",
"jszip": "^3.7.0",
"localforage": "^1.9.0",
"mousetrap": "^1.6.5",

212
src/common/ecs/compiler.ts Normal file
View File

@ -0,0 +1,212 @@
import { Tokenizer, TokenType } from "../tokenizer";
import { Action, ArrayType, ComponentType, DataField, DataType, DataValue, Dialect_CA65, Entity, EntityArchetype, EntityManager, EntityScope, IntType, Query, RefType, SelectType, SourceFileExport, System } from "./ecs";
export enum ECSTokenType {
Ellipsis = 'ellipsis',
Operator = 'delimiter',
QuotedString = 'quoted-string',
Integer = 'integer',
}
export class ECSCompiler extends Tokenizer {
em = new EntityManager(new Dialect_CA65()); // TODO
currentScope: EntityScope | null = null;
constructor() {
super();
//this.includeEOL = true;
this.setTokenRules([
{ type: TokenType.Ident, regex: /[$A-Za-z_][A-Za-z0-9_-]*/ },
{ type: TokenType.CodeFragment, regex: /---/ },
{ type: ECSTokenType.Ellipsis, regex: /\.\./ },
{ type: ECSTokenType.Operator, regex: /[=,:(){}\[\]]/ },
{ type: ECSTokenType.QuotedString, regex: /".*?"/ },
{ type: ECSTokenType.Integer, regex: /[-]?\d+/ },
{ type: TokenType.Ignore, regex: /\s+/ },
]);
this.errorOnCatchAll = true;
}
parseFile(text: string, path: string) {
this.tokenizeFile(text, path);
while (!this.isEOF()) {
this.parseTopLevel();
}
}
parseTopLevel() {
//this.skipBlankLines();
let tok = this.expectTokens(['component', 'system', 'scope', 'comment']);
if (tok.str == 'component') {
return this.em.defineComponent(this.parseComponentDefinition());
}
if (tok.str == 'system') {
return this.em.defineSystem(this.parseSystem());
}
if (tok.str == 'scope') {
return this.parseScope();
}
if (tok.str == 'comment') {
this.expectTokenTypes([TokenType.CodeFragment]);
return;
}
this.compileError(`Unexpected top-level keyword: ${tok.str}`);
}
parseComponentDefinition(): ComponentType {
let name = this.expectIdent().str;
let fields = [];
while (this.peekToken().str != 'end') {
fields.push(this.parseComponentField());
}
this.expectToken('end');
return { name, fields };
}
parseComponentField(): DataField {
let name = this.expectIdent();
this.expectToken(':');
let type = this.parseDataType();
return { name: name.str, ...type };
}
parseDataType(): DataType {
if (this.peekToken().type == 'integer') {
let lo = this.expectInteger();
this.expectToken('..');
let hi = this.expectInteger();
return { dtype: 'int', lo, hi } as IntType;
}
if (this.peekToken().str == '[') {
return { dtype: 'ref', query: this.parseQuery() } as RefType;
}
if (this.peekToken().str == 'array') {
this.expectToken('array');
this.expectToken('of');
return { dtype: 'array', elem: this.parseDataType() } as ArrayType;
}
this.compileError(`Unknown data type`); // TODO
}
parseDataValue() : DataValue {
if (this.peekToken().type == 'integer') {
return this.expectInteger();
}
this.compileError(`Unknown data value`); // TODO
}
expectInteger(): number {
let i = parseInt(this.consumeToken().str);
if (isNaN(i)) this.compileError('There should be an integer here.');
return i;
}
parseSystem(): System {
let name = this.expectIdent().str;
let actions: Action[] = [];
let system: System = { name, actions };
let cmd;
while ((cmd = this.consumeToken().str) != 'end') {
if (cmd == 'on') {
actions.push(this.parseAction());
} else if (cmd == 'locals') {
system.tempbytes = this.expectInteger();
} else {
this.compileError(`Unexpected system keyword: ${cmd}`);
}
}
return system;
}
parseAction(): Action {
let event = this.expectIdent().str;
this.expectToken('do');
let select = this.expectTokens(['once', 'each']).str as SelectType; // TODO: type check?
let query = this.parseQuery();
let text = this.parseCode();
return { text, event, query, select };
}
parseQuery() {
let q: Query = { include: [] };
this.expectToken('[');
q.include = this.parseList(this.parseComponentRef, ',').map(c => c.name);
// TODO: other params
this.expectToken(']');
return q;
}
parseEvent() {
return this.expectIdent().str;
}
parseCode(): string {
return this.expectTokenTypes([TokenType.CodeFragment]).str;
}
parseScope() : EntityScope {
let name = this.expectIdent().str;
let scope = this.em.newScope(name, this.currentScope);
this.currentScope = scope;
let cmd;
while ((cmd = this.consumeToken().str) != 'end') {
if (cmd == 'entity') {
this.parseEntity();
} else {
this.compileError(`Unexpected scope keyword: ${cmd}`);
}
}
this.currentScope = scope.parent;
return scope;
}
parseEntity() : Entity {
let name = '';
if (this.peekToken().type == TokenType.Ident) {
name = this.expectIdent().str;
}
let etype = this.parseEntityArchetype();
let e = this.currentScope.newEntity(etype);
e.name = name;
let cmd;
while ((cmd = this.consumeToken().str) != 'end') {
// TODO: check data types
if (cmd == 'const') {
let name = this.expectIdent().str;
this.expectToken('=');
e.consts[name] = this.parseDataValue();
} else if (cmd == 'init') {
let name = this.expectIdent().str;
this.expectToken('=');
e.inits[name] = this.parseDataValue();
} else {
this.compileError(`Unexpected scope keyword: ${cmd}`);
}
}
return e;
}
parseEntityArchetype() : EntityArchetype {
this.expectToken('[');
let components = this.parseList(this.parseComponentRef, ',');
this.expectToken(']');
return {components};
}
parseComponentRef() : ComponentType {
let name = this.expectIdent().str;
let cref = this.em.getComponentByName(name);
if (!cref) this.compileError(`I couldn't find a component named "${name}".`)
return cref;
}
exportToFile(src: SourceFileExport) {
for (let scope of Object.values(this.em.scopes)) {
scope.analyzeEntities();
scope.generateCode();
scope.dump(src);
}
}
}

View File

@ -1,6 +1,4 @@
import * as YAML from "js-yaml";
// entity scopes contain entities, and are nested
// also contain segments (code, bss, rodata)
// components and systems are global
@ -31,6 +29,8 @@ import * as YAML from "js-yaml";
// - converting assets to native formats?
// - removing unused data
import { Token } from "../tokenizer";
function mksymbol(c: ComponentType, fieldName: string) {
return c.name + '_' + fieldName;
}
@ -40,6 +40,7 @@ function mkscopesymbol(s: EntityScope, c: ComponentType, fieldName: string) {
export interface Entity {
id: number;
name?: string;
etype: EntityArchetype;
consts: { [component_field: string]: DataValue };
inits: { [scope_component_field: string]: DataValue };
@ -62,7 +63,7 @@ export interface ComponentType {
}
export interface Query {
include: string[];
include: string[]; // TODO: make ComponentType
listen?: string[];
exclude?: string[];
updates?: string[];
@ -79,9 +80,11 @@ export interface Action {
text: string;
event: string;
query: Query;
select: 'once' | 'each' | 'source'
select: SelectType
}
export type SelectType = 'once' | 'each' | 'source';
export type DataValue = number | boolean | Uint8Array | Uint16Array;
export type DataField = { name: string } & DataType;
@ -129,22 +132,21 @@ interface ArchetypeMatch {
cmatch: ComponentType[];
}
class Dialect_CA65 {
export class Dialect_CA65 {
readonly ASM_ITERATE_EACH = `
ldx #0
%{@__each}:
%{code}
{{@__each}}:
{{code}}
inx
cpx #%{ecount}
bne %{@__each}
cpx #{{ecount}}
bne {{@__each}}
`;
readonly INIT_FROM_ARRAY = `
ldy #%{nbytes}
:
lda %{src}-1,y
sta %{dest}-1,y
ldy #{{nbytes}}
: lda {{src}}-1,y
sta {{dest}}-1,y
dey
bne -
bne :-
`
readonly HEADER = `
.include "vcs-ca65.h"
@ -159,9 +161,19 @@ VecBRK: .word Start
Start:
CLEAN_START
`
comment(s: string) {
return `\n;;; ${s}\n`;
}
absolute(ident: string) {
return ident;
}
indexed_x(ident: string) {
return ident + ',x';
}
}
class SourceFileExport {
export class SourceFileExport {
lines: string[] = [];
comment(text: string) {
@ -254,7 +266,7 @@ class Segment {
}
getOriginSymbol() {
let a = this.ofs2sym.get(0);
if (!a) throw new Error('getOriginSymbol');
if (!a) throw new Error('getOriginSymbol(): no symbol at offset 0'); // TODO
return a[0];
}
}
@ -334,7 +346,7 @@ export class EntityScope {
}
allocateSegment(segment: Segment, readonly: boolean) {
let fields = Object.values(segment.fieldranges);
fields.sort((a, b) => (a.ehi - a.elo + 1) * getPackedFieldSize(a.field));
// TODO: fields.sort((a, b) => (a.ehi - a.elo + 1) * getPackedFieldSize(a.field));
let f;
while (f = fields.pop()) {
let name = mksymbol(f.component, f.field.name);
@ -413,9 +425,9 @@ export class EntityScope {
let bufofs = this.rodata.allocateInitData(bufsym, initbytes);
let code = this.dialect.INIT_FROM_ARRAY;
//TODO: function to repalce from dict?
code = code.replace('%{nbytes}', initbytes.length.toString())
code = code.replace('%{src}', bufsym);
code = code.replace('%{dest}', segment.getOriginSymbol());
code = code.replace('{{nbytes}}', initbytes.length.toString())
code = code.replace('{{src}}', bufsym);
code = code.replace('{{dest}}', segment.getOriginSymbol());
return code;
}
setConstValue(e: Entity, component: ComponentType, fieldName: string, value: DataValue) {
@ -454,9 +466,9 @@ export class EntityScope {
for (let action of sys.actions) {
if (action.event == event) {
let code = this.replaceCode(action.text, sys, action);
s += `\n; <action ${sys.name}:${event}>\n`;
s += this.dialect.comment(`<action ${sys.name}:${event}>`);
s += code;
s += `\n; </action ${sys.name}:${event}>\n`;
s += this.dialect.comment(`</action ${sys.name}:${event}>`);
// TODO: check that this happens once?
}
}
@ -468,12 +480,13 @@ export class EntityScope {
this.maxTempBytes = Math.max(this.tempOffset, this.maxTempBytes);
}
replaceCode(code: string, sys: System, action: Action): string {
const re = /\%\{(.+?)\}/g;
const re = /\{\{(.+?)\}\}/g;
let label = sys.name + '_' + action.event;
let atypes = this.em.archetypesMatching(action.query);
let entities = this.entitiesMatching(atypes);
// TODO: detect cycles
// TODO: "source"?
// TODO: what if only 1 item?
if (action.select == 'each') {
code = this.wrapCodeInLoop(code, sys, action, entities);
//console.log(sys.name, action.event, ents);
@ -513,10 +526,10 @@ export class EntityScope {
// TODO: check ents
// TODO: check segment bounds
let s = this.dialect.ASM_ITERATE_EACH;
s = s.replace('%{elo}', ents[0].id.toString());
s = s.replace('%{ehi}', ents[ents.length - 1].id.toString());
s = s.replace('%{ecount}', ents.length.toString());
s = s.replace('%{code}', code);
s = s.replace('{{elo}}', ents[0].id.toString());
s = s.replace('{{ehi}}', ents[ents.length - 1].id.toString());
s = s.replace('{{ecount}}', ents.length.toString());
s = s.replace('{{code}}', code);
return s;
}
generateCodeForField(sys: System, action: Action,
@ -547,12 +560,13 @@ export class EntityScope {
// TODO: offset > 0?
//let range = this.bss.getFieldRange(component, fieldName);
// TODO: dialect
let ident = `${component.name}_${fieldName}_b${bitofs}`;
if (action.select == 'once') {
if (entities.length != 1)
throw new Error(`can't choose multiple entities for ${fieldName} with select=once`);
return `${component.name}_${fieldName}_b${bitofs}` // TODO? check there's only 1 entity?
return this.dialect.absolute(ident) // TODO? check there's only 1 entity?
} else {
return `${component.name}_${fieldName}_b${bitofs},x` // TODO? ,x?
return this.dialect.indexed_x(ident)
}
}
entitiesMatching(atypes: ArchetypeMatch[]) {
@ -625,12 +639,13 @@ export class EntityScope {
}
export class EntityManager {
dialect = new Dialect_CA65();
archtypes = new Set<EntityArchetype>();
components: { [name: string]: ComponentType } = {};
systems: { [name: string]: System } = {};
scopes: { [name: string]: EntityScope } = {};
constructor(public readonly dialect: Dialect_CA65) {
}
newScope(name: string, parent?: EntityScope) {
let scope = new EntityScope(this, this.dialect, name, parent);
if (this.scopes[name]) throw new Error(`scope ${name} already defined`);
@ -643,7 +658,7 @@ export class EntityManager {
}
defineSystem(system: System) {
if (this.systems[system.name]) throw new Error(`system ${system.name} already defined`);
this.systems[system.name] = system;
return this.systems[system.name] = system;
}
componentsMatching(q: Query, etype: EntityArchetype) {
let list = [];
@ -680,395 +695,13 @@ export class EntityManager {
}
}
}
toYAML() {
return YAML.dump({
getComponentByName(name: string): ComponentType {
return this.components[name];
}
toJSON() {
return JSON.stringify({
components: this.components,
systems: this.systems,
systems: this.systems
})
}
}
///
const TEMPLATE1 = `
%{@NextFrame}:
FRAME_START
%{!preframe}
KERNEL_START
%{!kernel}
KERNEL_END
%{!postframe}
FRAME_END
lsr SWCHB ; test Game Reset switch
bcs @NoStart
jmp Start
@NoStart:
jmp %{@NextFrame}
`;
// TODO: two sticks?
const TEMPLATE2_a = `
lda SWCHA
sta %{$0}
`
const TEMPLATE2_b = `
asl %{$0}
bcs %{@SkipMoveRight}
%{!joyright}
%{@SkipMoveRight}:
asl %{$0}
bcs %{@SkipMoveLeft}
%{!joyleft}
%{@SkipMoveLeft}:
asl %{$0}
bcs %{@SkipMoveDown}
%{!joydown}
%{@SkipMoveDown}:
asl %{$0}
bcs %{@SkipMoveUp}
%{!joyup}
%{@SkipMoveUp}:
`;
const TEMPLATE3_L = `
lda %{<xpos}
sec
sbc #1
bcc %{@nomove}
sta %{<xpos}
%{@nomove}:
`;
const TEMPLATE3_R = `
lda %{<xpos}
clc
adc #1
cmp #150
bcs %{@nomove}
sta %{<xpos}
%{@nomove}:
`;
const TEMPLATE3_U = `
lda %{<ypos}
sec
sbc #1
bcc %{@nomove}
sta %{<ypos}
%{@nomove}:
`;
const TEMPLATE3_D = `
lda %{<ypos}
clc
adc #1
cmp #150
bcs %{@nomove}
sta %{<ypos}
%{@nomove}:
`;
const TEMPLATE4_S1 = `
.macro %{@KernelSetup} ent,ofs
lda #192 ; TODO: numlines
sec
sbc ypos_ypos_b0+ent
sta %{$5}+ofs
ldy hasbitmap_bitmap_b0+ent
lda bitmap_bitmapdata_b0,y
sec
sbc %{$5}+ofs
sta %{$0}+ofs
lda bitmap_bitmapdata_b8,y
sbc #0
sta %{$1}+ofs
ldy hascolormap_colormap_b0+ent
lda colormap_colormapdata_b0,y
sec
sbc %{$5}+ofs
sta %{$2}+ofs
lda colormap_colormapdata_b8,y
sbc #0
sta %{$3}+ofs
lda sprite_height_b0+ent
sta %{$4}+ofs
lda ypos_ypos_b0+ent
sta %{$5}+ofs
.endmacro
`
const TEMPLATE4_S2 = `
%{@KernelSetup} 0,0
%{@KernelSetup} 1,6
`
// https://atariage.com/forums/topic/75982-skipdraw-and-graphics/?tab=comments#comment-928232
// https://atariage.com/forums/topic/129683-advice-on-a-masking-kernel/
// https://atariage.com/forums/topic/128147-having-trouble-with-2-free-floating-player-graphics/?tab=comments#comment-1547059
const TEMPLATE4_K = `
lda %{<bgcolor}
sta COLUBK
ldy %{<lines}
@LVScan:
lda %{$4} ; height
dcp %{$5}
bcs @DoDraw1
lda #0
.byte $2C
@DoDraw1:
lda (%{$0}),y
sta WSYNC
sta GRP0
lda (%{$2}),y
sta COLUP0
lda %{$10} ; height
dcp %{$11}
bcs @DoDraw2
lda #0
.byte $2C
@DoDraw2:
lda (%{$6}),y
sta GRP1
lda (%{$8}),y
sta COLUP1
dey ; decrement
bne @LVScan ; repeat until 192 lines
`;
const SET_XPOS = `
lda %{<xpos}
ldy %{<plyrindex}
sta HMCLR
jsr %{^SetHorizPos}
`
const SETHORIZPOS = `
; SetHorizPos routine
; A = X coordinate
; Y = player number (0 or 1)
SetHorizPos:
sta WSYNC ; start a new line
sec ; set carry flag
nop
@DivideLoop:
sbc #15 ; subtract 15
bcs @DivideLoop ; branch until negative
eor #7 ; calculate fine offset
asl
asl
asl
asl
sta RESP0,y ; fix coarse position
sta HMP0,y ; set fine offset
sta WSYNC
sta HMOVE
rts ; return to caller
`
const INITFROMSPARSE = `
MemSrc equ $80
MemDest equ $82
InitMemory:
ldy #0
lda (MemSrc),y
beq .done
tax
iny
lda (MemSrc),y
sta MemDest
iny
lda (MemSrc),y
sta MemDest+1
.loop
iny
lda (MemSrc),y
sta (MemDest),y
dex
bne .loop
.done rts
`
function test() {
let em = new EntityManager();
let c_kernel = em.defineComponent({
name: 'kernel', fields: [
{ name: 'lines', dtype: 'int', lo: 0, hi: 255 },
{ name: 'bgcolor', dtype: 'int', lo: 0, hi: 255 },
]
})
let c_sprite = em.defineComponent({
name: 'sprite', fields: [
{ name: 'height', dtype: 'int', lo: 0, hi: 255 },
{ name: 'plyrindex', dtype: 'int', lo: 0, hi: 1 },
]
})
let c_plyrflags = em.defineComponent({
name: 'nusizable', fields: [
{ name: 'plyrflags', dtype: 'int', lo: 0, hi: 63 },
]
})
let c_player = em.defineComponent({
name: 'player', fields: [
//TODO: optional?
]
})
let c_hasbitmap = em.defineComponent({
name: 'hasbitmap', fields: [
{ name: 'bitmap', dtype: 'ref', query: { include: ['bitmap'] } },
]
})
let c_hascolormap = em.defineComponent({
name: 'hascolormap', fields: [
{ name: 'colormap', dtype: 'ref', query: { include: ['colormap'] } },
]
})
let c_bitmap = em.defineComponent({
name: 'bitmap', fields: [
{ name: 'bitmapdata', dtype: 'array', elem: { dtype: 'int', lo: 0, hi: 255 } }
]
})
let c_colormap = em.defineComponent({
name: 'colormap', fields: [
{ name: 'colormapdata', dtype: 'array', elem: { dtype: 'int', lo: 0, hi: 255 } }
]
})
let c_xpos = em.defineComponent({
name: 'xpos', fields: [
{ name: 'xpos', dtype: 'int', lo: 0, hi: 255 }
]
})
let c_ypos = em.defineComponent({
name: 'ypos', fields: [
{ name: 'ypos', dtype: 'int', lo: 0, hi: 255 }
]
})
let c_xyvel = em.defineComponent({
name: 'xyvel', fields: [
{ name: 'xvel', dtype: 'int', lo: -8, hi: 7 },
{ name: 'yvel', dtype: 'int', lo: -8, hi: 7 }
]
})
// init -> [start] -> frameloop
// frameloop -> [preframe] [kernel] [postframe]
// temp between preframe + frame?
// TODO: check names for identifierness
em.defineSystem({
name: 'kernel_simple',
tempbytes: 8,
actions: [
{
text: TEMPLATE4_S1, event: 'preframe', select: 'once', query: {
include: ['kernel']
}
},
{
// TODO: should include kernel for numlines
text: TEMPLATE4_S2, event: 'preframe', select: 'once', query: {
include: ['sprite', 'hasbitmap', 'hascolormap', 'ypos'],
},
},
{
text: TEMPLATE4_K, event: 'kernel', select: 'once', query: {
include: ['kernel']
}
},
]
})
em.defineSystem({
name: 'set_xpos',
actions: [
{
text: SET_XPOS, event: 'preframe', select: 'each', query: {
include: ['sprite', 'xpos']
},
},
//{ text:SETHORIZPOS },
]
})
// https://docs.unity3d.com/Packages/com.unity.entities@0.17/manual/ecs_systems.html
em.defineSystem({
name: 'frameloop',
emits: ['preframe', 'kernel', 'postframe'],
actions: [
{ text: TEMPLATE1, event: 'start', select: 'once', query: { include: ['kernel'] } }
]
})
em.defineSystem({
name: 'joyread',
tempbytes: 1,
emits: ['joyup', 'joydown', 'joyleft', 'joyright', 'joybutton'],
actions: [
{ text: TEMPLATE2_a, event: 'postframe', select: 'once', query: { include: ['player'] } },
{ text: TEMPLATE2_b, event: 'postframe', select: 'each', query: { include: ['player'] } }
]
});
em.defineSystem({
name: 'move_x',
actions: [
{ text: TEMPLATE3_L, event: 'joyleft', select: 'source', query: { include: ['player', 'xpos'] }, },
{ text: TEMPLATE3_R, event: 'joyright', select: 'source', query: { include: ['player', 'xpos'] }, },
]
});
em.defineSystem({
name: 'move_y',
actions: [
{ text: TEMPLATE3_U, event: 'joyup', select: 'source', query: { include: ['player', 'ypos'] } },
{ text: TEMPLATE3_D, event: 'joydown', select: 'source', query: { include: ['player', 'ypos'] } },
]
});
em.defineSystem({
name: 'SetHorizPos',
actions: [
{ text: SETHORIZPOS, event: 'SetHorizPos', select: 'once', query: { include: ['xpos'] } },
]
});
let root = em.newScope("Root");
let scene = em.newScope("Scene", root);
let e_ekernel = root.newEntity({ components: [c_kernel] });
root.setConstValue(e_ekernel, c_kernel, 'lines', 192);
//root.setConstValue(e_ekernel, c_kernel, 'bgcolor', 0x92);
root.setInitValue(e_ekernel, c_kernel, 'bgcolor', 0x92);
let e_bitmap0 = root.newEntity({ components: [c_bitmap] });
// TODO: store array sizes?
root.setConstValue(e_bitmap0, c_bitmap, 'bitmapdata', new Uint8Array([1, 1, 3, 7, 15, 31, 63, 127]));
let e_colormap0 = root.newEntity({ components: [c_colormap] });
root.setConstValue(e_colormap0, c_colormap, 'colormapdata', new Uint8Array([6, 3, 6, 9, 12, 14, 31, 63]));
let ea_playerSprite = { components: [c_sprite, c_hasbitmap, c_hascolormap, c_xpos, c_ypos, c_player] };
let e_player0 = root.newEntity(ea_playerSprite);
root.setConstValue(e_player0, c_sprite, 'plyrindex', 0);
root.setInitValue(e_player0, c_sprite, 'height', 8);
root.setInitValue(e_player0, c_xpos, 'xpos', 50);
root.setInitValue(e_player0, c_ypos, 'ypos', 50);
let e_player1 = root.newEntity(ea_playerSprite);
root.setConstValue(e_player1, c_sprite, 'plyrindex', 1);
root.setInitValue(e_player1, c_sprite, 'height', 8);
root.setInitValue(e_player1, c_xpos, 'xpos', 100);
root.setInitValue(e_player1, c_ypos, 'ypos', 60);
//console.log(em.archetypesMatching({ include:['xpos','ypos']})[0])
let src = new SourceFileExport();
root.analyzeEntities();
root.generateCode();
root.dump(src);
console.log(src.toString());
//console.log(em.toYAML());
}
// TODO: files in markdown?
// TODO: jsr OperModeExecutionTree?
test();

30
src/common/ecs/main.ts Normal file
View File

@ -0,0 +1,30 @@
import { readFileSync } from "fs";
import { ECSCompiler } from "./compiler";
import { SourceFileExport } from "./ecs";
class ECSMain {
compiler = new ECSCompiler();
constructor(readonly args: string[]) {
}
run() {
for (let path of this.args) {
let text = readFileSync(path, 'utf-8');
try {
this.compiler.parseFile(text, path);
} catch (e) {
console.log(e);
for (let err of this.compiler.errors) {
console.log(`${err.path}:${err.line}:${err.start}: ${err.msg}`);
}
}
}
let file = new SourceFileExport();
this.compiler.exportToFile(file);
console.log(file.toString());
}
}
new ECSMain(process.argv.slice(2)).run();

239
src/common/tokenizer.ts Normal file
View File

@ -0,0 +1,239 @@
import type { SourceLocation, SourceLine, WorkerError } from "./workertypes";
// objects that have source code position info
export interface SourceLocated {
$loc?: SourceLocation;
}
// statements also have the 'offset' (pc) field from SourceLine
export interface SourceLineLocated {
$loc?: SourceLine;
}
export class CompileError extends Error {
$loc: SourceLocation;
constructor(msg: string, loc: SourceLocation) {
super(msg);
Object.setPrototypeOf(this, CompileError.prototype);
this.$loc = loc;
}
}
export function mergeLocs(a: SourceLocation, b: SourceLocation): SourceLocation {
return {
line: Math.min(a.line, b.line),
start: Math.min(a.start, b.start),
end: Math.max(a.end, b.end),
label: a.label || b.label,
path: a.path || b.path,
}
}
export enum TokenType {
EOF = 'eof',
EOL = 'eol',
Ident = 'ident',
Comment = 'comment',
Ignore = 'ignore',
CodeFragment = 'code-fragment',
CatchAll = 'catch-all',
}
export class Token implements SourceLocated {
str: string;
type: string;
$loc: SourceLocation;
}
export class TokenRule {
type: string;
regex: RegExp;
}
const CATCH_ALL_RULES: TokenRule[] = [
{ type: TokenType.CatchAll, regex: /.+?/ }
]
function re_escape(rule: TokenRule): string {
return `(${rule.regex.source})`;
}
export class Tokenizer {
rules: TokenRule[];
regex: RegExp;
path: string;
lineno: number;
tokens: Token[];
lasttoken: Token;
errors: WorkerError[];
curlabel: string;
eol: Token;
includeEOL = false;
errorOnCatchAll = false;
codeFragment : string | null = null;
constructor() {
this.lineno = 0;
this.errors = [];
}
setTokenRules(rules: TokenRule[]) {
this.rules = rules.concat(CATCH_ALL_RULES);
var pattern = this.rules.map(re_escape).join('|');
this.regex = new RegExp(pattern, "g");
}
tokenizeFile(contents: string, path: string) {
this.path = path;
this.tokens = []; // can't have errors until this is set
let txtlines = contents.split(/\n|\r\n?/);
txtlines.forEach((line) => this._tokenize(line));
this._pushToken({ type: TokenType.EOF, str: "", $loc: { path: this.path, line: this.lineno } });
}
tokenizeLine(line: string) : void {
this.lineno++;
this._tokenize(line);
}
_tokenize(line: string): void {
this.lineno++;
this.eol = { type: TokenType.EOL, str: "", $loc: { path: this.path, line: this.lineno, start: line.length } };
// iterate over each token via re_toks regex
let m: RegExpMatchArray;
while (m = this.regex.exec(line)) {
let found = false;
// find out which capture group was matched, and thus token type
for (let i = 0; i < this.rules.length; i++) {
let s: string = m[i + 1];
if (s != null) {
found = true;
let loc = { path: this.path, line: this.lineno, start: m.index, end: m.index + s.length };
let rule = this.rules[i];
// add token to list
switch (rule.type) {
case TokenType.CodeFragment:
if (this.codeFragment) {
this._pushToken({ str: this.codeFragment, type: rule.type, $loc: loc }); //TODO: merge start/end
this.codeFragment = null;
} else {
this.codeFragment = '';
return; // don't add any more tokens (TODO: check for trash?)
}
break;
case TokenType.CatchAll:
if (this.errorOnCatchAll && this.codeFragment == null) {
this.compileError(`I didn't expect the character "${m[0]}" here.`);
}
default:
if (this.codeFragment == null) {
this._pushToken({ str: s, type: rule.type, $loc: loc });
}
case TokenType.Ignore:
break;
}
break;
}
}
if (!found) {
this.compileError(`Could not parse token: <<${m[0]}>>`)
}
}
if (this.includeEOL) {
this._pushToken(this.eol);
}
if (this.codeFragment != null) {
this.codeFragment += line + '\n';
}
}
_pushToken(token: Token) {
this.tokens.push(token);
}
addError(msg: string, loc?: SourceLocation) {
let tok = this.lasttoken || this.peekToken();
if (!loc) loc = tok.$loc;
this.errors.push({ path: loc.path, line: loc.line, label: this.curlabel, start: loc.start, end: loc.end, msg: msg });
}
internalError() {
this.compileError("Internal error.");
}
notImplementedError() {
this.compileError("Not yet implemented.");
}
compileError(msg: string, loc?: SourceLocation, loc2?: SourceLocation) {
this.addError(msg, loc);
//if (loc2 != null) this.addError(`...`, loc2);
throw new CompileError(msg, loc);
}
peekToken(lookahead?: number): Token {
let tok = this.tokens[lookahead || 0];
return tok ? tok : this.eol;
}
consumeToken(): Token {
let tok = this.lasttoken = (this.tokens.shift() || this.eol);
return tok;
}
expectToken(str: string, msg?: string): Token {
let tok = this.consumeToken();
let tokstr = tok.str;
if (str != tokstr) {
this.compileError(msg || `There should be a "${str}" here.`);
}
return tok;
}
expectTokens(strlist: string[], msg?: string): Token {
let tok = this.consumeToken();
let tokstr = tok.str;
if (strlist.indexOf(tokstr) < 0) {
this.compileError(msg || `There should be a ${strlist.map((s) => `"${s}"`).join(' or ')} here, not "${tokstr}.`);
}
return tok;
}
parseModifiers(modifiers: string[]): { [modifier: string]: boolean } {
let result = {};
do {
var tok = this.peekToken();
if (modifiers.indexOf(tok.str) < 0)
return result;
this.consumeToken();
result[tok.str] = true;
} while (tok != null);
}
expectIdent(msg?: string): Token {
let tok = this.consumeToken();
if (tok.type != TokenType.Ident)
this.compileError(msg || `There should be an identifier here.`);
return tok;
}
pushbackToken(tok: Token) {
this.tokens.unshift(tok);
}
isEOF() {
return this.tokens.length == 0 || this.peekToken().type == 'eof'; // TODO?
}
expectEOL(msg?: string) {
let tok = this.consumeToken();
if (tok.type != TokenType.EOL)
this.compileError(msg || `There's too much stuff on this line.`);
}
skipBlankLines() {
this.skipTokenTypes(['eol']);
}
skipTokenTypes(types: string[]) {
while (types.includes(this.peekToken().type))
this.consumeToken();
}
expectTokenTypes(types: string[], msg?: string) {
let tok = this.consumeToken();
if (!types.includes(tok.type))
this.compileError(msg || `There should be a ${types.map((s) => `"${s}"`).join(' or ')} here. not a "${tok.type}".`);
return tok;
}
parseList<T>(parseFunc:()=>T, delim:string): T[] {
var sep;
var list = [];
do {
var el = parseFunc.bind(this)(); // call parse function
if (el != null) list.push(el); // add parsed element to list
sep = this.consumeToken(); // consume seperator token
} while (sep.str == delim);
this.pushbackToken(sep);
return list;
}
}

449
src/test/testecs.ts Normal file
View File

@ -0,0 +1,449 @@
import { describe } from "mocha";
import { ECSCompiler } from "../common/ecs/compiler";
import { Dialect_CA65, EntityManager, SourceFileExport } from "../common/ecs/ecs";
const TEMPLATE1 = `
{{@NextFrame}}:
FRAME_START
{{!preframe}}
KERNEL_START
{{!kernel}}
KERNEL_END
{{!postframe}}
FRAME_END
lsr SWCHB ; test Game Reset switch
bcs @NoStart
jmp Start
@NoStart:
jmp {{@NextFrame}}
`;
// TODO: two sticks?
const TEMPLATE2_a = `
lda SWCHA
sta {{$0}}
`
const TEMPLATE2_b = `
asl {{$0}}
bcs {{@SkipMoveRight}}
{{!joyright}}
{{@SkipMoveRight}}:
asl {{$0}}
bcs {{@SkipMoveLeft}}
{{!joyleft}}
{{@SkipMoveLeft}}:
asl {{$0}}
bcs {{@SkipMoveDown}}
{{!joydown}}
{{@SkipMoveDown}}:
asl {{$0}}
bcs {{@SkipMoveUp}}
{{!joyup}}
{{@SkipMoveUp}}:
`;
const TEMPLATE3_L = `
lda {{<xpos}}
sec
sbc #1
bcc {{@nomove}}
sta {{<xpos}}
{{@nomove}}:
`;
const TEMPLATE3_R = `
lda {{<xpos}}
clc
adc #1
cmp #150
bcs {{@nomove}}
sta {{<xpos}}
{{@nomove}}:
`;
const TEMPLATE3_U = `
lda {{<ypos}}
sec
sbc #1
bcc {{@nomove}}
sta {{<ypos}}
{{@nomove}}:
`;
const TEMPLATE3_D = `
lda {{<ypos}}
clc
adc #1
cmp #150
bcs {{@nomove}}
sta {{<ypos}}
{{@nomove}}:
`;
const TEMPLATE4_S1 = `
.macro {{@KernelSetup}} ent,ofs
lda #192 ; TODO: numlines
sec
sbc ypos_ypos_b0+ent
sta {{$5}}+ofs
ldy hasbitmap_bitmap_b0+ent
lda bitmap_bitmapdata_b0,y
sec
sbc {{$5}}+ofs
sta {{$0}}+ofs
lda bitmap_bitmapdata_b8,y
sbc #0
sta {{$1}}+ofs
ldy hascolormap_colormap_b0+ent
lda colormap_colormapdata_b0,y
sec
sbc {{$5}}+ofs
sta {{$2}}+ofs
lda colormap_colormapdata_b8,y
sbc #0
sta {{$3}}+ofs
lda sprite_height_b0+ent
sta {{$4}}+ofs
lda ypos_ypos_b0+ent
sta {{$5}}+ofs
.endmacro
`
const TEMPLATE4_S2 = `
{{@KernelSetup}} 0,0
{{@KernelSetup}} 1,6
`
// https://atariage.com/forums/topic/75982-skipdraw-and-graphics/?tab=comments#comment-928232
// https://atariage.com/forums/topic/129683-advice-on-a-masking-kernel/
// https://atariage.com/forums/topic/128147-having-trouble-with-2-free-floating-player-graphics/?tab=comments#comment-1547059
const TEMPLATE4_K = `
lda {{<bgcolor}}
sta COLUBK
ldy {{<lines}}
@LVScan:
lda {{$4}} ; height
dcp {{$5}}
bcs @DoDraw1
lda #0
.byte $2C
@DoDraw1:
lda ({{$0}}),y
sta WSYNC
sta GRP0
lda ({{$2}}),y
sta COLUP0
lda {{$10}} ; height
dcp {{$11}}
bcs @DoDraw2
lda #0
.byte $2C
@DoDraw2:
lda ({{$6}}),y
sta GRP1
lda ({{$8}}),y
sta COLUP1
dey ; decrement
bne @LVScan ; repeat until 192 lines
`;
const SET_XPOS = `
lda {{<xpos}}
ldy {{<plyrindex}}
sta HMCLR
jsr {{^SetHorizPos}}
`
const SETHORIZPOS = `
; SetHorizPos routine
; A = X coordinate
; Y = player number (0 or 1)
SetHorizPos:
sta WSYNC ; start a new line
sec ; set carry flag
nop
@DivideLoop:
sbc #15 ; subtract 15
bcs @DivideLoop ; branch until negative
eor #7 ; calculate fine offset
asl
asl
asl
asl
sta RESP0,y ; fix coarse position
sta HMP0,y ; set fine offset
sta WSYNC
sta HMOVE
rts ; return to caller
`
const INITFROMSPARSE = `
MemSrc equ $80
MemDest equ $82
InitMemory:
ldy #0
lda (MemSrc),y
beq .done
tax
iny
lda (MemSrc),y
sta MemDest
iny
lda (MemSrc),y
sta MemDest+1
.loop
iny
lda (MemSrc),y
sta (MemDest),y
dex
bne .loop
.done rts
`
function testECS() {
let em = new EntityManager(new Dialect_CA65());
let c_kernel = em.defineComponent({
name: 'kernel', fields: [
{ name: 'lines', dtype: 'int', lo: 0, hi: 255 },
{ name: 'bgcolor', dtype: 'int', lo: 0, hi: 255 },
]
})
let c_sprite = em.defineComponent({
name: 'sprite', fields: [
{ name: 'height', dtype: 'int', lo: 0, hi: 255 },
{ name: 'plyrindex', dtype: 'int', lo: 0, hi: 1 },
]
})
let c_plyrflags = em.defineComponent({
name: 'nusizable', fields: [
{ name: 'plyrflags', dtype: 'int', lo: 0, hi: 63 },
]
})
let c_player = em.defineComponent({
name: 'player', fields: [
//TODO: optional?
]
})
let c_hasbitmap = em.defineComponent({
name: 'hasbitmap', fields: [
{ name: 'bitmap', dtype: 'ref', query: { include: ['bitmap'] } },
]
})
let c_hascolormap = em.defineComponent({
name: 'hascolormap', fields: [
{ name: 'colormap', dtype: 'ref', query: { include: ['colormap'] } },
]
})
let c_bitmap = em.defineComponent({
name: 'bitmap', fields: [
{ name: 'bitmapdata', dtype: 'array', elem: { dtype: 'int', lo: 0, hi: 255 } }
]
})
let c_colormap = em.defineComponent({
name: 'colormap', fields: [
{ name: 'colormapdata', dtype: 'array', elem: { dtype: 'int', lo: 0, hi: 255 } }
]
})
let c_xpos = em.defineComponent({
name: 'xpos', fields: [
{ name: 'xpos', dtype: 'int', lo: 0, hi: 255 }
]
})
let c_ypos = em.defineComponent({
name: 'ypos', fields: [
{ name: 'ypos', dtype: 'int', lo: 0, hi: 255 }
]
})
let c_xyvel = em.defineComponent({
name: 'xyvel', fields: [
{ name: 'xvel', dtype: 'int', lo: -8, hi: 7 },
{ name: 'yvel', dtype: 'int', lo: -8, hi: 7 }
]
})
// init -> [start] -> frameloop
// frameloop -> [preframe] [kernel] [postframe]
// temp between preframe + frame?
// TODO: check names for identifierness
em.defineSystem({
name: 'kernel_simple',
tempbytes: 8,
actions: [
{
text: TEMPLATE4_S1, event: 'preframe', select: 'once', query: {
include: ['kernel']
}
},
{
// TODO: should include kernel for numlines
text: TEMPLATE4_S2, event: 'preframe', select: 'once', query: {
include: ['sprite', 'hasbitmap', 'hascolormap', 'ypos'],
},
},
{
text: TEMPLATE4_K, event: 'kernel', select: 'once', query: {
include: ['kernel']
}
},
]
})
em.defineSystem({
name: 'set_xpos',
actions: [
{
text: SET_XPOS, event: 'preframe', select: 'each', query: {
include: ['sprite', 'xpos']
},
},
//{ text:SETHORIZPOS },
]
})
// https://docs.unity3d.com/Packages/com.unity.entities@0.17/manual/ecs_systems.html
em.defineSystem({
name: 'frameloop',
emits: ['preframe', 'kernel', 'postframe'],
actions: [
{ text: TEMPLATE1, event: 'start', select: 'once', query: { include: ['kernel'] } }
]
})
em.defineSystem({
name: 'joyread',
tempbytes: 1,
emits: ['joyup', 'joydown', 'joyleft', 'joyright', 'joybutton'],
actions: [
{ text: TEMPLATE2_a, event: 'postframe', select: 'once', query: { include: ['player'] } },
{ text: TEMPLATE2_b, event: 'postframe', select: 'each', query: { include: ['player'] } }
]
});
em.defineSystem({
name: 'move_x',
actions: [
{ text: TEMPLATE3_L, event: 'joyleft', select: 'source', query: { include: ['player', 'xpos'] }, },
{ text: TEMPLATE3_R, event: 'joyright', select: 'source', query: { include: ['player', 'xpos'] }, },
]
});
em.defineSystem({
name: 'move_y',
actions: [
{ text: TEMPLATE3_U, event: 'joyup', select: 'source', query: { include: ['player', 'ypos'] } },
{ text: TEMPLATE3_D, event: 'joydown', select: 'source', query: { include: ['player', 'ypos'] } },
]
});
em.defineSystem({
name: 'SetHorizPos',
actions: [
{ text: SETHORIZPOS, event: 'SetHorizPos', select: 'once', query: { include: ['xpos'] } },
]
});
let root = em.newScope("Root");
let scene = em.newScope("Scene", root);
let e_ekernel = root.newEntity({ components: [c_kernel] });
root.setConstValue(e_ekernel, c_kernel, 'lines', 192);
//root.setConstValue(e_ekernel, c_kernel, 'bgcolor', 0x92);
root.setInitValue(e_ekernel, c_kernel, 'bgcolor', 0x92);
let e_bitmap0 = root.newEntity({ components: [c_bitmap] });
// TODO: store array sizes?
root.setConstValue(e_bitmap0, c_bitmap, 'bitmapdata', new Uint8Array([1, 1, 3, 7, 15, 31, 63, 127]));
let e_colormap0 = root.newEntity({ components: [c_colormap] });
root.setConstValue(e_colormap0, c_colormap, 'colormapdata', new Uint8Array([6, 3, 6, 9, 12, 14, 31, 63]));
let ea_playerSprite = { components: [c_sprite, c_hasbitmap, c_hascolormap, c_xpos, c_ypos, c_player] };
let e_player0 = root.newEntity(ea_playerSprite);
root.setConstValue(e_player0, c_sprite, 'plyrindex', 0);
root.setInitValue(e_player0, c_sprite, 'height', 8);
root.setInitValue(e_player0, c_xpos, 'xpos', 50);
root.setInitValue(e_player0, c_ypos, 'ypos', 50);
let e_player1 = root.newEntity(ea_playerSprite);
root.setConstValue(e_player1, c_sprite, 'plyrindex', 1);
root.setInitValue(e_player1, c_sprite, 'height', 8);
root.setInitValue(e_player1, c_xpos, 'xpos', 100);
root.setInitValue(e_player1, c_ypos, 'ypos', 60);
//console.log(em.archetypesMatching({ include:['xpos','ypos']})[0])
root.analyzeEntities();
root.generateCode();
let src = new SourceFileExport();
root.dump(src);
//console.log(src.toString());
//console.log(em.toYAML());
}
function testCompiler() {
let c = new ECSCompiler();
try {
c.parseFile(`
component Kernel
lines: 0..255
bgcolor: 0..255
end
component Bitmap
data: array of 0..255
end
component HasBitmap
bitmap: [Bitmap]
end
system SimpleKernel
locals 8
on preframe do once [Kernel] --- JUNK_AT_END
lda #5
sta #6
Label:
---
end
scope Root
entity kernel [Kernel]
const lines = 100
end
entity player1 [HasBitmap]
const plyrindex = 0
init height = 8
init xpos = 100
init ypos = 100
end
end
`, 'foo.txt');
console.log('json', c.em.toJSON());
let src = new SourceFileExport();
c.exportToFile(src);
// TODO: test?
//console.log(src.toString());
} catch (e) {
console.log(e);
for (let err of c.errors) {
console.log(err);
}
console.log(c.tokens);
}
}
// TODO: files in markdown?
// TODO: jsr OperModeExecutionTree?
describe('Tokenizer', function() {
it('Should use API', function() {
testECS();
});
it('Should use Compiler', function() {
testCompiler();
});
});

View File

@ -1,8 +1,9 @@
import assert from "assert";
import { describe } from "mocha";
import { EmuHalt } from "../../src/common/emu"
import { lzgmini, isProbablyBinary } from "../../src/common/util";
import { EmuHalt } from "../common/emu"
import { lzgmini, isProbablyBinary } from "../common/util";
import { Tokenizer, TokenType } from "../common/tokenizer";
var NES_CONIO_ROM_LZG = [
76,90,71,0,0,160,16,0,0,11,158,107,131,223,83,1,9,17,21,22,78,69,83,26,2,1,3,0,22,6,120,216,
@ -133,4 +134,38 @@ var NES_CONIO_ROM_LZG = [
assert.ok(e.hasOwnProperty('$loc'));
});
});
describe('Tokenizer', function() {
it('Should tokenize', function() {
var t = new Tokenizer();
t.setTokenRules([
{ type: 'ident', regex: /[$A-Za-z_][A-Za-z0-9_-]*/ },
{ type: 'delim', regex: /[\(\)\{\}\[\]]/ },
{ type: 'qstring', regex: /".*?"/ },
{ type: 'integer', regex: /[-]?\d+/ },
{ type: 'ignore', regex: /\s+/ },
{ type: TokenType.CodeFragment, regex: /---/ },
]);
t.tokenizeFile("\n{\"key\" value\n \"number\" 531\n \"f\" (fn [x] (+ x 2))}\n", "test.file");
assert.strictEqual(t.tokens.map(t => t.type).join(' '),
'delim qstring ident qstring integer qstring delim ident delim ident delim delim catch-all ident integer delim delim delim eof');
assert.strictEqual(t.tokens.map(t => t.str).join(' '),
'{ "key" value "number" 531 "f" ( fn [ x ] ( + x 2 ) ) } ');
assert.strictEqual(19, t.tokens.length);
assert.strictEqual('{', t.peekToken().str);
assert.strictEqual('{', t.expectToken('{').str);
t.pushbackToken(t.consumeToken());
assert.strictEqual('"key"', t.consumeToken().str);
assert.deepStrictEqual({'value':true}, t.parseModifiers(['foo','value','bar']));
assert.deepStrictEqual([], t.errors);
t.includeEOL = true;
t.tokenizeFile("\n{\"key\" value\n \"number\" 531\n \"f\" (fn [x] (+ x 2))}\n", "test.file");
assert.strictEqual(24, t.tokens.length);
assert.strictEqual(t.tokens.map(t => t.type).join(' '),
'eol delim qstring ident eol qstring integer eol qstring delim ident delim ident delim delim catch-all ident integer delim delim delim eol eol eof');
t.includeEOL = false;
t.tokenizeFile("key value ---\nthis is\na fragment\n--- foo", "test.file");
assert.strictEqual(t.tokens.map(t => t.type).join(' '),
'ident ident code-fragment ident eof');
});
});