diff --git a/package-lock.json b/package-lock.json
index 7d53b8fa..1c715b31 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -21,7 +21,6 @@
"fast-png": "^5.0.4",
"file-saver": "^2.0.5",
"jquery": "^3.6.0",
- "js-yaml": "^4.1.0",
"jszip": "^3.7.0",
"localforage": "^1.9.0",
"mousetrap": "^1.6.5",
@@ -746,7 +745,8 @@
"node_modules/argparse": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
- "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
+ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
+ "dev": true
},
"node_modules/array-equal": {
"version": "1.0.0",
@@ -3599,6 +3599,7 @@
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
"integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+ "dev": true,
"dependencies": {
"argparse": "^2.0.1"
},
@@ -8029,7 +8030,8 @@
"argparse": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
- "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
+ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
+ "dev": true
},
"array-equal": {
"version": "1.0.0",
@@ -10205,6 +10207,7 @@
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
"integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
+ "dev": true,
"requires": {
"argparse": "^2.0.1"
}
diff --git a/package.json b/package.json
index 5cb29d87..adc8c9db 100644
--- a/package.json
+++ b/package.json
@@ -22,7 +22,6 @@
"fast-png": "^5.0.4",
"file-saver": "^2.0.5",
"jquery": "^3.6.0",
- "js-yaml": "^4.1.0",
"jszip": "^3.7.0",
"localforage": "^1.9.0",
"mousetrap": "^1.6.5",
diff --git a/src/common/ecs/compiler.ts b/src/common/ecs/compiler.ts
new file mode 100644
index 00000000..4954125f
--- /dev/null
+++ b/src/common/ecs/compiler.ts
@@ -0,0 +1,212 @@
+
+import { Tokenizer, TokenType } from "../tokenizer";
+import { Action, ArrayType, ComponentType, DataField, DataType, DataValue, Dialect_CA65, Entity, EntityArchetype, EntityManager, EntityScope, IntType, Query, RefType, SelectType, SourceFileExport, System } from "./ecs";
+
+export enum ECSTokenType {
+ Ellipsis = 'ellipsis',
+ Operator = 'delimiter',
+ QuotedString = 'quoted-string',
+ Integer = 'integer',
+}
+
+export class ECSCompiler extends Tokenizer {
+
+ em = new EntityManager(new Dialect_CA65()); // TODO
+ currentScope: EntityScope | null = null;
+
+ constructor() {
+ super();
+ //this.includeEOL = true;
+ this.setTokenRules([
+ { type: TokenType.Ident, regex: /[$A-Za-z_][A-Za-z0-9_-]*/ },
+ { type: TokenType.CodeFragment, regex: /---/ },
+ { type: ECSTokenType.Ellipsis, regex: /\.\./ },
+ { type: ECSTokenType.Operator, regex: /[=,:(){}\[\]]/ },
+ { type: ECSTokenType.QuotedString, regex: /".*?"/ },
+ { type: ECSTokenType.Integer, regex: /[-]?\d+/ },
+ { type: TokenType.Ignore, regex: /\s+/ },
+ ]);
+ this.errorOnCatchAll = true;
+ }
+
+ parseFile(text: string, path: string) {
+ this.tokenizeFile(text, path);
+ while (!this.isEOF()) {
+ this.parseTopLevel();
+ }
+ }
+
+ parseTopLevel() {
+ //this.skipBlankLines();
+ let tok = this.expectTokens(['component', 'system', 'scope', 'comment']);
+ if (tok.str == 'component') {
+ return this.em.defineComponent(this.parseComponentDefinition());
+ }
+ if (tok.str == 'system') {
+ return this.em.defineSystem(this.parseSystem());
+ }
+ if (tok.str == 'scope') {
+ return this.parseScope();
+ }
+ if (tok.str == 'comment') {
+ this.expectTokenTypes([TokenType.CodeFragment]);
+ return;
+ }
+ this.compileError(`Unexpected top-level keyword: ${tok.str}`);
+ }
+
+ parseComponentDefinition(): ComponentType {
+ let name = this.expectIdent().str;
+ let fields = [];
+ while (this.peekToken().str != 'end') {
+ fields.push(this.parseComponentField());
+ }
+ this.expectToken('end');
+ return { name, fields };
+ }
+
+ parseComponentField(): DataField {
+ let name = this.expectIdent();
+ this.expectToken(':');
+ let type = this.parseDataType();
+ return { name: name.str, ...type };
+ }
+
+ parseDataType(): DataType {
+ if (this.peekToken().type == 'integer') {
+ let lo = this.expectInteger();
+ this.expectToken('..');
+ let hi = this.expectInteger();
+ return { dtype: 'int', lo, hi } as IntType;
+ }
+ if (this.peekToken().str == '[') {
+ return { dtype: 'ref', query: this.parseQuery() } as RefType;
+ }
+ if (this.peekToken().str == 'array') {
+ this.expectToken('array');
+ this.expectToken('of');
+ return { dtype: 'array', elem: this.parseDataType() } as ArrayType;
+ }
+ this.compileError(`Unknown data type`); // TODO
+ }
+
+ parseDataValue() : DataValue {
+ if (this.peekToken().type == 'integer') {
+ return this.expectInteger();
+ }
+ this.compileError(`Unknown data value`); // TODO
+ }
+
+ expectInteger(): number {
+ let i = parseInt(this.consumeToken().str);
+ if (isNaN(i)) this.compileError('There should be an integer here.');
+ return i;
+ }
+
+ parseSystem(): System {
+ let name = this.expectIdent().str;
+ let actions: Action[] = [];
+ let system: System = { name, actions };
+ let cmd;
+ while ((cmd = this.consumeToken().str) != 'end') {
+ if (cmd == 'on') {
+ actions.push(this.parseAction());
+ } else if (cmd == 'locals') {
+ system.tempbytes = this.expectInteger();
+ } else {
+ this.compileError(`Unexpected system keyword: ${cmd}`);
+ }
+ }
+ return system;
+ }
+
+ parseAction(): Action {
+ let event = this.expectIdent().str;
+ this.expectToken('do');
+ let select = this.expectTokens(['once', 'each']).str as SelectType; // TODO: type check?
+ let query = this.parseQuery();
+ let text = this.parseCode();
+ return { text, event, query, select };
+ }
+
+ parseQuery() {
+ let q: Query = { include: [] };
+ this.expectToken('[');
+ q.include = this.parseList(this.parseComponentRef, ',').map(c => c.name);
+ // TODO: other params
+ this.expectToken(']');
+ return q;
+ }
+
+ parseEvent() {
+ return this.expectIdent().str;
+ }
+
+ parseCode(): string {
+ return this.expectTokenTypes([TokenType.CodeFragment]).str;
+ }
+
+ parseScope() : EntityScope {
+ let name = this.expectIdent().str;
+ let scope = this.em.newScope(name, this.currentScope);
+ this.currentScope = scope;
+ let cmd;
+ while ((cmd = this.consumeToken().str) != 'end') {
+ if (cmd == 'entity') {
+ this.parseEntity();
+ } else {
+ this.compileError(`Unexpected scope keyword: ${cmd}`);
+ }
+ }
+ this.currentScope = scope.parent;
+ return scope;
+ }
+
+ parseEntity() : Entity {
+ let name = '';
+ if (this.peekToken().type == TokenType.Ident) {
+ name = this.expectIdent().str;
+ }
+ let etype = this.parseEntityArchetype();
+ let e = this.currentScope.newEntity(etype);
+ e.name = name;
+ let cmd;
+ while ((cmd = this.consumeToken().str) != 'end') {
+ // TODO: check data types
+ if (cmd == 'const') {
+ let name = this.expectIdent().str;
+ this.expectToken('=');
+ e.consts[name] = this.parseDataValue();
+ } else if (cmd == 'init') {
+ let name = this.expectIdent().str;
+ this.expectToken('=');
+ e.inits[name] = this.parseDataValue();
+ } else {
+ this.compileError(`Unexpected scope keyword: ${cmd}`);
+ }
+ }
+ return e;
+ }
+
+ parseEntityArchetype() : EntityArchetype {
+ this.expectToken('[');
+ let components = this.parseList(this.parseComponentRef, ',');
+ this.expectToken(']');
+ return {components};
+ }
+
+ parseComponentRef() : ComponentType {
+ let name = this.expectIdent().str;
+ let cref = this.em.getComponentByName(name);
+ if (!cref) this.compileError(`I couldn't find a component named "${name}".`)
+ return cref;
+ }
+
+ exportToFile(src: SourceFileExport) {
+ for (let scope of Object.values(this.em.scopes)) {
+ scope.analyzeEntities();
+ scope.generateCode();
+ scope.dump(src);
+ }
+ }
+}
diff --git a/src/common/ecs/ecs.ts b/src/common/ecs/ecs.ts
index 5c29d247..8f822dc4 100644
--- a/src/common/ecs/ecs.ts
+++ b/src/common/ecs/ecs.ts
@@ -1,6 +1,4 @@
-import * as YAML from "js-yaml";
-
// entity scopes contain entities, and are nested
// also contain segments (code, bss, rodata)
// components and systems are global
@@ -31,6 +29,8 @@ import * as YAML from "js-yaml";
// - converting assets to native formats?
// - removing unused data
+import { Token } from "../tokenizer";
+
function mksymbol(c: ComponentType, fieldName: string) {
return c.name + '_' + fieldName;
}
@@ -40,6 +40,7 @@ function mkscopesymbol(s: EntityScope, c: ComponentType, fieldName: string) {
export interface Entity {
id: number;
+ name?: string;
etype: EntityArchetype;
consts: { [component_field: string]: DataValue };
inits: { [scope_component_field: string]: DataValue };
@@ -62,7 +63,7 @@ export interface ComponentType {
}
export interface Query {
- include: string[];
+ include: string[]; // TODO: make ComponentType
listen?: string[];
exclude?: string[];
updates?: string[];
@@ -79,9 +80,11 @@ export interface Action {
text: string;
event: string;
query: Query;
- select: 'once' | 'each' | 'source'
+ select: SelectType
}
+export type SelectType = 'once' | 'each' | 'source';
+
export type DataValue = number | boolean | Uint8Array | Uint16Array;
export type DataField = { name: string } & DataType;
@@ -129,22 +132,21 @@ interface ArchetypeMatch {
cmatch: ComponentType[];
}
-class Dialect_CA65 {
+export class Dialect_CA65 {
readonly ASM_ITERATE_EACH = `
ldx #0
-%{@__each}:
- %{code}
+{{@__each}}:
+ {{code}}
inx
- cpx #%{ecount}
- bne %{@__each}
+ cpx #{{ecount}}
+ bne {{@__each}}
`;
readonly INIT_FROM_ARRAY = `
- ldy #%{nbytes}
-:
- lda %{src}-1,y
- sta %{dest}-1,y
+ ldy #{{nbytes}}
+: lda {{src}}-1,y
+ sta {{dest}}-1,y
dey
- bne -
+ bne :-
`
readonly HEADER = `
.include "vcs-ca65.h"
@@ -159,9 +161,19 @@ VecBRK: .word Start
Start:
CLEAN_START
`
+
+ comment(s: string) {
+ return `\n;;; ${s}\n`;
+ }
+ absolute(ident: string) {
+ return ident;
+ }
+ indexed_x(ident: string) {
+ return ident + ',x';
+ }
}
-class SourceFileExport {
+export class SourceFileExport {
lines: string[] = [];
comment(text: string) {
@@ -254,7 +266,7 @@ class Segment {
}
getOriginSymbol() {
let a = this.ofs2sym.get(0);
- if (!a) throw new Error('getOriginSymbol');
+ if (!a) throw new Error('getOriginSymbol(): no symbol at offset 0'); // TODO
return a[0];
}
}
@@ -334,7 +346,7 @@ export class EntityScope {
}
allocateSegment(segment: Segment, readonly: boolean) {
let fields = Object.values(segment.fieldranges);
- fields.sort((a, b) => (a.ehi - a.elo + 1) * getPackedFieldSize(a.field));
+ // TODO: fields.sort((a, b) => (a.ehi - a.elo + 1) * getPackedFieldSize(a.field));
let f;
while (f = fields.pop()) {
let name = mksymbol(f.component, f.field.name);
@@ -413,9 +425,9 @@ export class EntityScope {
let bufofs = this.rodata.allocateInitData(bufsym, initbytes);
let code = this.dialect.INIT_FROM_ARRAY;
//TODO: function to repalce from dict?
- code = code.replace('%{nbytes}', initbytes.length.toString())
- code = code.replace('%{src}', bufsym);
- code = code.replace('%{dest}', segment.getOriginSymbol());
+ code = code.replace('{{nbytes}}', initbytes.length.toString())
+ code = code.replace('{{src}}', bufsym);
+ code = code.replace('{{dest}}', segment.getOriginSymbol());
return code;
}
setConstValue(e: Entity, component: ComponentType, fieldName: string, value: DataValue) {
@@ -454,9 +466,9 @@ export class EntityScope {
for (let action of sys.actions) {
if (action.event == event) {
let code = this.replaceCode(action.text, sys, action);
- s += `\n; \n`;
+ s += this.dialect.comment(``);
s += code;
- s += `\n; \n`;
+ s += this.dialect.comment(``);
// TODO: check that this happens once?
}
}
@@ -468,12 +480,13 @@ export class EntityScope {
this.maxTempBytes = Math.max(this.tempOffset, this.maxTempBytes);
}
replaceCode(code: string, sys: System, action: Action): string {
- const re = /\%\{(.+?)\}/g;
+ const re = /\{\{(.+?)\}\}/g;
let label = sys.name + '_' + action.event;
let atypes = this.em.archetypesMatching(action.query);
let entities = this.entitiesMatching(atypes);
// TODO: detect cycles
// TODO: "source"?
+ // TODO: what if only 1 item?
if (action.select == 'each') {
code = this.wrapCodeInLoop(code, sys, action, entities);
//console.log(sys.name, action.event, ents);
@@ -513,10 +526,10 @@ export class EntityScope {
// TODO: check ents
// TODO: check segment bounds
let s = this.dialect.ASM_ITERATE_EACH;
- s = s.replace('%{elo}', ents[0].id.toString());
- s = s.replace('%{ehi}', ents[ents.length - 1].id.toString());
- s = s.replace('%{ecount}', ents.length.toString());
- s = s.replace('%{code}', code);
+ s = s.replace('{{elo}}', ents[0].id.toString());
+ s = s.replace('{{ehi}}', ents[ents.length - 1].id.toString());
+ s = s.replace('{{ecount}}', ents.length.toString());
+ s = s.replace('{{code}}', code);
return s;
}
generateCodeForField(sys: System, action: Action,
@@ -547,12 +560,13 @@ export class EntityScope {
// TODO: offset > 0?
//let range = this.bss.getFieldRange(component, fieldName);
// TODO: dialect
+ let ident = `${component.name}_${fieldName}_b${bitofs}`;
if (action.select == 'once') {
if (entities.length != 1)
throw new Error(`can't choose multiple entities for ${fieldName} with select=once`);
- return `${component.name}_${fieldName}_b${bitofs}` // TODO? check there's only 1 entity?
+ return this.dialect.absolute(ident) // TODO? check there's only 1 entity?
} else {
- return `${component.name}_${fieldName}_b${bitofs},x` // TODO? ,x?
+ return this.dialect.indexed_x(ident)
}
}
entitiesMatching(atypes: ArchetypeMatch[]) {
@@ -625,12 +639,13 @@ export class EntityScope {
}
export class EntityManager {
- dialect = new Dialect_CA65();
archtypes = new Set();
components: { [name: string]: ComponentType } = {};
systems: { [name: string]: System } = {};
scopes: { [name: string]: EntityScope } = {};
+ constructor(public readonly dialect: Dialect_CA65) {
+ }
newScope(name: string, parent?: EntityScope) {
let scope = new EntityScope(this, this.dialect, name, parent);
if (this.scopes[name]) throw new Error(`scope ${name} already defined`);
@@ -643,7 +658,7 @@ export class EntityManager {
}
defineSystem(system: System) {
if (this.systems[system.name]) throw new Error(`system ${system.name} already defined`);
- this.systems[system.name] = system;
+ return this.systems[system.name] = system;
}
componentsMatching(q: Query, etype: EntityArchetype) {
let list = [];
@@ -680,395 +695,13 @@ export class EntityManager {
}
}
}
- toYAML() {
- return YAML.dump({
+ getComponentByName(name: string): ComponentType {
+ return this.components[name];
+ }
+ toJSON() {
+ return JSON.stringify({
components: this.components,
- systems: this.systems,
+ systems: this.systems
})
}
}
-
-///
-
-const TEMPLATE1 = `
-%{@NextFrame}:
- FRAME_START
- %{!preframe}
- KERNEL_START
- %{!kernel}
- KERNEL_END
- %{!postframe}
- FRAME_END
- lsr SWCHB ; test Game Reset switch
- bcs @NoStart
- jmp Start
-@NoStart:
- jmp %{@NextFrame}
-`;
-
-// TODO: two sticks?
-const TEMPLATE2_a = `
- lda SWCHA
- sta %{$0}
-`
-const TEMPLATE2_b = `
- asl %{$0}
- bcs %{@SkipMoveRight}
- %{!joyright}
-%{@SkipMoveRight}:
- asl %{$0}
- bcs %{@SkipMoveLeft}
- %{!joyleft}
-%{@SkipMoveLeft}:
- asl %{$0}
- bcs %{@SkipMoveDown}
- %{!joydown}
-%{@SkipMoveDown}:
- asl %{$0}
- bcs %{@SkipMoveUp}
- %{!joyup}
-%{@SkipMoveUp}:
-`;
-
-const TEMPLATE3_L = `
- lda %{ [start] -> frameloop
- // frameloop -> [preframe] [kernel] [postframe]
-
- // temp between preframe + frame?
- // TODO: check names for identifierness
- em.defineSystem({
- name: 'kernel_simple',
- tempbytes: 8,
- actions: [
- {
- text: TEMPLATE4_S1, event: 'preframe', select: 'once', query: {
- include: ['kernel']
- }
- },
- {
- // TODO: should include kernel for numlines
- text: TEMPLATE4_S2, event: 'preframe', select: 'once', query: {
- include: ['sprite', 'hasbitmap', 'hascolormap', 'ypos'],
- },
- },
- {
- text: TEMPLATE4_K, event: 'kernel', select: 'once', query: {
- include: ['kernel']
- }
- },
- ]
- })
- em.defineSystem({
- name: 'set_xpos',
- actions: [
- {
- text: SET_XPOS, event: 'preframe', select: 'each', query: {
- include: ['sprite', 'xpos']
- },
- },
- //{ text:SETHORIZPOS },
- ]
- })
- // https://docs.unity3d.com/Packages/com.unity.entities@0.17/manual/ecs_systems.html
- em.defineSystem({
- name: 'frameloop',
- emits: ['preframe', 'kernel', 'postframe'],
- actions: [
- { text: TEMPLATE1, event: 'start', select: 'once', query: { include: ['kernel'] } }
- ]
- })
- em.defineSystem({
- name: 'joyread',
- tempbytes: 1,
- emits: ['joyup', 'joydown', 'joyleft', 'joyright', 'joybutton'],
- actions: [
- { text: TEMPLATE2_a, event: 'postframe', select: 'once', query: { include: ['player'] } },
- { text: TEMPLATE2_b, event: 'postframe', select: 'each', query: { include: ['player'] } }
- ]
- });
- em.defineSystem({
- name: 'move_x',
- actions: [
- { text: TEMPLATE3_L, event: 'joyleft', select: 'source', query: { include: ['player', 'xpos'] }, },
- { text: TEMPLATE3_R, event: 'joyright', select: 'source', query: { include: ['player', 'xpos'] }, },
- ]
- });
- em.defineSystem({
- name: 'move_y',
- actions: [
- { text: TEMPLATE3_U, event: 'joyup', select: 'source', query: { include: ['player', 'ypos'] } },
- { text: TEMPLATE3_D, event: 'joydown', select: 'source', query: { include: ['player', 'ypos'] } },
- ]
- });
- em.defineSystem({
- name: 'SetHorizPos',
- actions: [
- { text: SETHORIZPOS, event: 'SetHorizPos', select: 'once', query: { include: ['xpos'] } },
- ]
- });
-
- let root = em.newScope("Root");
- let scene = em.newScope("Scene", root);
- let e_ekernel = root.newEntity({ components: [c_kernel] });
- root.setConstValue(e_ekernel, c_kernel, 'lines', 192);
- //root.setConstValue(e_ekernel, c_kernel, 'bgcolor', 0x92);
- root.setInitValue(e_ekernel, c_kernel, 'bgcolor', 0x92);
-
- let e_bitmap0 = root.newEntity({ components: [c_bitmap] });
- // TODO: store array sizes?
- root.setConstValue(e_bitmap0, c_bitmap, 'bitmapdata', new Uint8Array([1, 1, 3, 7, 15, 31, 63, 127]));
-
- let e_colormap0 = root.newEntity({ components: [c_colormap] });
- root.setConstValue(e_colormap0, c_colormap, 'colormapdata', new Uint8Array([6, 3, 6, 9, 12, 14, 31, 63]));
-
- let ea_playerSprite = { components: [c_sprite, c_hasbitmap, c_hascolormap, c_xpos, c_ypos, c_player] };
- let e_player0 = root.newEntity(ea_playerSprite);
- root.setConstValue(e_player0, c_sprite, 'plyrindex', 0);
- root.setInitValue(e_player0, c_sprite, 'height', 8);
- root.setInitValue(e_player0, c_xpos, 'xpos', 50);
- root.setInitValue(e_player0, c_ypos, 'ypos', 50);
- let e_player1 = root.newEntity(ea_playerSprite);
- root.setConstValue(e_player1, c_sprite, 'plyrindex', 1);
- root.setInitValue(e_player1, c_sprite, 'height', 8);
- root.setInitValue(e_player1, c_xpos, 'xpos', 100);
- root.setInitValue(e_player1, c_ypos, 'ypos', 60);
-
- //console.log(em.archetypesMatching({ include:['xpos','ypos']})[0])
-
- let src = new SourceFileExport();
- root.analyzeEntities();
- root.generateCode();
- root.dump(src);
- console.log(src.toString());
- //console.log(em.toYAML());
-}
-
-// TODO: files in markdown?
-// TODO: jsr OperModeExecutionTree?
-
-test();
-
diff --git a/src/common/ecs/main.ts b/src/common/ecs/main.ts
new file mode 100644
index 00000000..e035ed53
--- /dev/null
+++ b/src/common/ecs/main.ts
@@ -0,0 +1,30 @@
+import { readFileSync } from "fs";
+import { ECSCompiler } from "./compiler";
+import { SourceFileExport } from "./ecs";
+
+class ECSMain {
+
+ compiler = new ECSCompiler();
+
+ constructor(readonly args: string[]) {
+ }
+
+ run() {
+ for (let path of this.args) {
+ let text = readFileSync(path, 'utf-8');
+ try {
+ this.compiler.parseFile(text, path);
+ } catch (e) {
+ console.log(e);
+ for (let err of this.compiler.errors) {
+ console.log(`${err.path}:${err.line}:${err.start}: ${err.msg}`);
+ }
+ }
+ }
+ let file = new SourceFileExport();
+ this.compiler.exportToFile(file);
+ console.log(file.toString());
+ }
+}
+
+new ECSMain(process.argv.slice(2)).run();
diff --git a/src/common/tokenizer.ts b/src/common/tokenizer.ts
new file mode 100644
index 00000000..c603ae7e
--- /dev/null
+++ b/src/common/tokenizer.ts
@@ -0,0 +1,239 @@
+
+import type { SourceLocation, SourceLine, WorkerError } from "./workertypes";
+
+// objects that have source code position info
+export interface SourceLocated {
+ $loc?: SourceLocation;
+}
+// statements also have the 'offset' (pc) field from SourceLine
+export interface SourceLineLocated {
+ $loc?: SourceLine;
+}
+
+export class CompileError extends Error {
+ $loc: SourceLocation;
+ constructor(msg: string, loc: SourceLocation) {
+ super(msg);
+ Object.setPrototypeOf(this, CompileError.prototype);
+ this.$loc = loc;
+ }
+}
+
+export function mergeLocs(a: SourceLocation, b: SourceLocation): SourceLocation {
+ return {
+ line: Math.min(a.line, b.line),
+ start: Math.min(a.start, b.start),
+ end: Math.max(a.end, b.end),
+ label: a.label || b.label,
+ path: a.path || b.path,
+ }
+}
+
+export enum TokenType {
+ EOF = 'eof',
+ EOL = 'eol',
+ Ident = 'ident',
+ Comment = 'comment',
+ Ignore = 'ignore',
+ CodeFragment = 'code-fragment',
+ CatchAll = 'catch-all',
+}
+
+export class Token implements SourceLocated {
+ str: string;
+ type: string;
+ $loc: SourceLocation;
+}
+
+export class TokenRule {
+ type: string;
+ regex: RegExp;
+}
+
+const CATCH_ALL_RULES: TokenRule[] = [
+ { type: TokenType.CatchAll, regex: /.+?/ }
+]
+
+function re_escape(rule: TokenRule): string {
+ return `(${rule.regex.source})`;
+}
+
+export class Tokenizer {
+ rules: TokenRule[];
+ regex: RegExp;
+ path: string;
+ lineno: number;
+ tokens: Token[];
+ lasttoken: Token;
+ errors: WorkerError[];
+ curlabel: string;
+ eol: Token;
+ includeEOL = false;
+ errorOnCatchAll = false;
+ codeFragment : string | null = null;
+
+ constructor() {
+ this.lineno = 0;
+ this.errors = [];
+ }
+ setTokenRules(rules: TokenRule[]) {
+ this.rules = rules.concat(CATCH_ALL_RULES);
+ var pattern = this.rules.map(re_escape).join('|');
+ this.regex = new RegExp(pattern, "g");
+ }
+ tokenizeFile(contents: string, path: string) {
+ this.path = path;
+ this.tokens = []; // can't have errors until this is set
+ let txtlines = contents.split(/\n|\r\n?/);
+ txtlines.forEach((line) => this._tokenize(line));
+ this._pushToken({ type: TokenType.EOF, str: "", $loc: { path: this.path, line: this.lineno } });
+ }
+ tokenizeLine(line: string) : void {
+ this.lineno++;
+ this._tokenize(line);
+ }
+ _tokenize(line: string): void {
+ this.lineno++;
+ this.eol = { type: TokenType.EOL, str: "", $loc: { path: this.path, line: this.lineno, start: line.length } };
+ // iterate over each token via re_toks regex
+ let m: RegExpMatchArray;
+ while (m = this.regex.exec(line)) {
+ let found = false;
+ // find out which capture group was matched, and thus token type
+ for (let i = 0; i < this.rules.length; i++) {
+ let s: string = m[i + 1];
+ if (s != null) {
+ found = true;
+ let loc = { path: this.path, line: this.lineno, start: m.index, end: m.index + s.length };
+ let rule = this.rules[i];
+ // add token to list
+ switch (rule.type) {
+ case TokenType.CodeFragment:
+ if (this.codeFragment) {
+ this._pushToken({ str: this.codeFragment, type: rule.type, $loc: loc }); //TODO: merge start/end
+ this.codeFragment = null;
+ } else {
+ this.codeFragment = '';
+ return; // don't add any more tokens (TODO: check for trash?)
+ }
+ break;
+ case TokenType.CatchAll:
+ if (this.errorOnCatchAll && this.codeFragment == null) {
+ this.compileError(`I didn't expect the character "${m[0]}" here.`);
+ }
+ default:
+ if (this.codeFragment == null) {
+ this._pushToken({ str: s, type: rule.type, $loc: loc });
+ }
+ case TokenType.Ignore:
+ break;
+ }
+ break;
+ }
+ }
+ if (!found) {
+ this.compileError(`Could not parse token: <<${m[0]}>>`)
+ }
+ }
+ if (this.includeEOL) {
+ this._pushToken(this.eol);
+ }
+ if (this.codeFragment != null) {
+ this.codeFragment += line + '\n';
+ }
+ }
+ _pushToken(token: Token) {
+ this.tokens.push(token);
+ }
+ addError(msg: string, loc?: SourceLocation) {
+ let tok = this.lasttoken || this.peekToken();
+ if (!loc) loc = tok.$loc;
+ this.errors.push({ path: loc.path, line: loc.line, label: this.curlabel, start: loc.start, end: loc.end, msg: msg });
+ }
+ internalError() {
+ this.compileError("Internal error.");
+ }
+ notImplementedError() {
+ this.compileError("Not yet implemented.");
+ }
+ compileError(msg: string, loc?: SourceLocation, loc2?: SourceLocation) {
+ this.addError(msg, loc);
+ //if (loc2 != null) this.addError(`...`, loc2);
+ throw new CompileError(msg, loc);
+ }
+ peekToken(lookahead?: number): Token {
+ let tok = this.tokens[lookahead || 0];
+ return tok ? tok : this.eol;
+ }
+ consumeToken(): Token {
+ let tok = this.lasttoken = (this.tokens.shift() || this.eol);
+ return tok;
+ }
+ expectToken(str: string, msg?: string): Token {
+ let tok = this.consumeToken();
+ let tokstr = tok.str;
+ if (str != tokstr) {
+ this.compileError(msg || `There should be a "${str}" here.`);
+ }
+ return tok;
+ }
+ expectTokens(strlist: string[], msg?: string): Token {
+ let tok = this.consumeToken();
+ let tokstr = tok.str;
+ if (strlist.indexOf(tokstr) < 0) {
+ this.compileError(msg || `There should be a ${strlist.map((s) => `"${s}"`).join(' or ')} here, not "${tokstr}.`);
+ }
+ return tok;
+ }
+ parseModifiers(modifiers: string[]): { [modifier: string]: boolean } {
+ let result = {};
+ do {
+ var tok = this.peekToken();
+ if (modifiers.indexOf(tok.str) < 0)
+ return result;
+ this.consumeToken();
+ result[tok.str] = true;
+ } while (tok != null);
+ }
+ expectIdent(msg?: string): Token {
+ let tok = this.consumeToken();
+ if (tok.type != TokenType.Ident)
+ this.compileError(msg || `There should be an identifier here.`);
+ return tok;
+ }
+ pushbackToken(tok: Token) {
+ this.tokens.unshift(tok);
+ }
+ isEOF() {
+ return this.tokens.length == 0 || this.peekToken().type == 'eof'; // TODO?
+ }
+ expectEOL(msg?: string) {
+ let tok = this.consumeToken();
+ if (tok.type != TokenType.EOL)
+ this.compileError(msg || `There's too much stuff on this line.`);
+ }
+ skipBlankLines() {
+ this.skipTokenTypes(['eol']);
+ }
+ skipTokenTypes(types: string[]) {
+ while (types.includes(this.peekToken().type))
+ this.consumeToken();
+ }
+ expectTokenTypes(types: string[], msg?: string) {
+ let tok = this.consumeToken();
+ if (!types.includes(tok.type))
+ this.compileError(msg || `There should be a ${types.map((s) => `"${s}"`).join(' or ')} here. not a "${tok.type}".`);
+ return tok;
+ }
+ parseList(parseFunc:()=>T, delim:string): T[] {
+ var sep;
+ var list = [];
+ do {
+ var el = parseFunc.bind(this)(); // call parse function
+ if (el != null) list.push(el); // add parsed element to list
+ sep = this.consumeToken(); // consume seperator token
+ } while (sep.str == delim);
+ this.pushbackToken(sep);
+ return list;
+ }
+}
diff --git a/src/test/testecs.ts b/src/test/testecs.ts
new file mode 100644
index 00000000..6709a9a1
--- /dev/null
+++ b/src/test/testecs.ts
@@ -0,0 +1,449 @@
+import { describe } from "mocha";
+import { ECSCompiler } from "../common/ecs/compiler";
+import { Dialect_CA65, EntityManager, SourceFileExport } from "../common/ecs/ecs";
+
+const TEMPLATE1 = `
+{{@NextFrame}}:
+ FRAME_START
+ {{!preframe}}
+ KERNEL_START
+ {{!kernel}}
+ KERNEL_END
+ {{!postframe}}
+ FRAME_END
+ lsr SWCHB ; test Game Reset switch
+ bcs @NoStart
+ jmp Start
+@NoStart:
+ jmp {{@NextFrame}}
+`;
+
+// TODO: two sticks?
+const TEMPLATE2_a = `
+ lda SWCHA
+ sta {{$0}}
+`
+const TEMPLATE2_b = `
+ asl {{$0}}
+ bcs {{@SkipMoveRight}}
+ {{!joyright}}
+{{@SkipMoveRight}}:
+ asl {{$0}}
+ bcs {{@SkipMoveLeft}}
+ {{!joyleft}}
+{{@SkipMoveLeft}}:
+ asl {{$0}}
+ bcs {{@SkipMoveDown}}
+ {{!joydown}}
+{{@SkipMoveDown}}:
+ asl {{$0}}
+ bcs {{@SkipMoveUp}}
+ {{!joyup}}
+{{@SkipMoveUp}}:
+`;
+
+const TEMPLATE3_L = `
+ lda {{ [start] -> frameloop
+ // frameloop -> [preframe] [kernel] [postframe]
+
+ // temp between preframe + frame?
+ // TODO: check names for identifierness
+ em.defineSystem({
+ name: 'kernel_simple',
+ tempbytes: 8,
+ actions: [
+ {
+ text: TEMPLATE4_S1, event: 'preframe', select: 'once', query: {
+ include: ['kernel']
+ }
+ },
+ {
+ // TODO: should include kernel for numlines
+ text: TEMPLATE4_S2, event: 'preframe', select: 'once', query: {
+ include: ['sprite', 'hasbitmap', 'hascolormap', 'ypos'],
+ },
+ },
+ {
+ text: TEMPLATE4_K, event: 'kernel', select: 'once', query: {
+ include: ['kernel']
+ }
+ },
+ ]
+ })
+ em.defineSystem({
+ name: 'set_xpos',
+ actions: [
+ {
+ text: SET_XPOS, event: 'preframe', select: 'each', query: {
+ include: ['sprite', 'xpos']
+ },
+ },
+ //{ text:SETHORIZPOS },
+ ]
+ })
+ // https://docs.unity3d.com/Packages/com.unity.entities@0.17/manual/ecs_systems.html
+ em.defineSystem({
+ name: 'frameloop',
+ emits: ['preframe', 'kernel', 'postframe'],
+ actions: [
+ { text: TEMPLATE1, event: 'start', select: 'once', query: { include: ['kernel'] } }
+ ]
+ })
+ em.defineSystem({
+ name: 'joyread',
+ tempbytes: 1,
+ emits: ['joyup', 'joydown', 'joyleft', 'joyright', 'joybutton'],
+ actions: [
+ { text: TEMPLATE2_a, event: 'postframe', select: 'once', query: { include: ['player'] } },
+ { text: TEMPLATE2_b, event: 'postframe', select: 'each', query: { include: ['player'] } }
+ ]
+ });
+ em.defineSystem({
+ name: 'move_x',
+ actions: [
+ { text: TEMPLATE3_L, event: 'joyleft', select: 'source', query: { include: ['player', 'xpos'] }, },
+ { text: TEMPLATE3_R, event: 'joyright', select: 'source', query: { include: ['player', 'xpos'] }, },
+ ]
+ });
+ em.defineSystem({
+ name: 'move_y',
+ actions: [
+ { text: TEMPLATE3_U, event: 'joyup', select: 'source', query: { include: ['player', 'ypos'] } },
+ { text: TEMPLATE3_D, event: 'joydown', select: 'source', query: { include: ['player', 'ypos'] } },
+ ]
+ });
+ em.defineSystem({
+ name: 'SetHorizPos',
+ actions: [
+ { text: SETHORIZPOS, event: 'SetHorizPos', select: 'once', query: { include: ['xpos'] } },
+ ]
+ });
+
+ let root = em.newScope("Root");
+ let scene = em.newScope("Scene", root);
+ let e_ekernel = root.newEntity({ components: [c_kernel] });
+ root.setConstValue(e_ekernel, c_kernel, 'lines', 192);
+ //root.setConstValue(e_ekernel, c_kernel, 'bgcolor', 0x92);
+ root.setInitValue(e_ekernel, c_kernel, 'bgcolor', 0x92);
+
+ let e_bitmap0 = root.newEntity({ components: [c_bitmap] });
+ // TODO: store array sizes?
+ root.setConstValue(e_bitmap0, c_bitmap, 'bitmapdata', new Uint8Array([1, 1, 3, 7, 15, 31, 63, 127]));
+
+ let e_colormap0 = root.newEntity({ components: [c_colormap] });
+ root.setConstValue(e_colormap0, c_colormap, 'colormapdata', new Uint8Array([6, 3, 6, 9, 12, 14, 31, 63]));
+
+ let ea_playerSprite = { components: [c_sprite, c_hasbitmap, c_hascolormap, c_xpos, c_ypos, c_player] };
+ let e_player0 = root.newEntity(ea_playerSprite);
+ root.setConstValue(e_player0, c_sprite, 'plyrindex', 0);
+ root.setInitValue(e_player0, c_sprite, 'height', 8);
+ root.setInitValue(e_player0, c_xpos, 'xpos', 50);
+ root.setInitValue(e_player0, c_ypos, 'ypos', 50);
+ let e_player1 = root.newEntity(ea_playerSprite);
+ root.setConstValue(e_player1, c_sprite, 'plyrindex', 1);
+ root.setInitValue(e_player1, c_sprite, 'height', 8);
+ root.setInitValue(e_player1, c_xpos, 'xpos', 100);
+ root.setInitValue(e_player1, c_ypos, 'ypos', 60);
+
+ //console.log(em.archetypesMatching({ include:['xpos','ypos']})[0])
+
+ root.analyzeEntities();
+ root.generateCode();
+ let src = new SourceFileExport();
+ root.dump(src);
+ //console.log(src.toString());
+ //console.log(em.toYAML());
+}
+
+function testCompiler() {
+ let c = new ECSCompiler();
+ try {
+ c.parseFile(`
+
+component Kernel
+ lines: 0..255
+ bgcolor: 0..255
+end
+
+component Bitmap
+ data: array of 0..255
+end
+
+component HasBitmap
+ bitmap: [Bitmap]
+end
+
+system SimpleKernel
+locals 8
+on preframe do once [Kernel] --- JUNK_AT_END
+ lda #5
+ sta #6
+Label:
+---
+end
+
+scope Root
+
+ entity kernel [Kernel]
+ const lines = 100
+ end
+
+ entity player1 [HasBitmap]
+ const plyrindex = 0
+ init height = 8
+ init xpos = 100
+ init ypos = 100
+ end
+
+end
+
+`, 'foo.txt');
+ console.log('json', c.em.toJSON());
+ let src = new SourceFileExport();
+ c.exportToFile(src);
+ // TODO: test?
+ //console.log(src.toString());
+ } catch (e) {
+ console.log(e);
+ for (let err of c.errors) {
+ console.log(err);
+ }
+ console.log(c.tokens);
+ }
+}
+
+// TODO: files in markdown?
+// TODO: jsr OperModeExecutionTree?
+
+describe('Tokenizer', function() {
+ it('Should use API', function() {
+ testECS();
+ });
+ it('Should use Compiler', function() {
+ testCompiler();
+ });
+});
diff --git a/src/test/testutil.ts b/src/test/testutil.ts
index b95210ec..9e340a69 100644
--- a/src/test/testutil.ts
+++ b/src/test/testutil.ts
@@ -1,8 +1,9 @@
import assert from "assert";
import { describe } from "mocha";
-import { EmuHalt } from "../../src/common/emu"
-import { lzgmini, isProbablyBinary } from "../../src/common/util";
+import { EmuHalt } from "../common/emu"
+import { lzgmini, isProbablyBinary } from "../common/util";
+import { Tokenizer, TokenType } from "../common/tokenizer";
var NES_CONIO_ROM_LZG = [
76,90,71,0,0,160,16,0,0,11,158,107,131,223,83,1,9,17,21,22,78,69,83,26,2,1,3,0,22,6,120,216,
@@ -133,4 +134,38 @@ var NES_CONIO_ROM_LZG = [
assert.ok(e.hasOwnProperty('$loc'));
});
});
-
\ No newline at end of file
+
+describe('Tokenizer', function() {
+ it('Should tokenize', function() {
+ var t = new Tokenizer();
+ t.setTokenRules([
+ { type: 'ident', regex: /[$A-Za-z_][A-Za-z0-9_-]*/ },
+ { type: 'delim', regex: /[\(\)\{\}\[\]]/ },
+ { type: 'qstring', regex: /".*?"/ },
+ { type: 'integer', regex: /[-]?\d+/ },
+ { type: 'ignore', regex: /\s+/ },
+ { type: TokenType.CodeFragment, regex: /---/ },
+ ]);
+ t.tokenizeFile("\n{\"key\" value\n \"number\" 531\n \"f\" (fn [x] (+ x 2))}\n", "test.file");
+ assert.strictEqual(t.tokens.map(t => t.type).join(' '),
+ 'delim qstring ident qstring integer qstring delim ident delim ident delim delim catch-all ident integer delim delim delim eof');
+ assert.strictEqual(t.tokens.map(t => t.str).join(' '),
+ '{ "key" value "number" 531 "f" ( fn [ x ] ( + x 2 ) ) } ');
+ assert.strictEqual(19, t.tokens.length);
+ assert.strictEqual('{', t.peekToken().str);
+ assert.strictEqual('{', t.expectToken('{').str);
+ t.pushbackToken(t.consumeToken());
+ assert.strictEqual('"key"', t.consumeToken().str);
+ assert.deepStrictEqual({'value':true}, t.parseModifiers(['foo','value','bar']));
+ assert.deepStrictEqual([], t.errors);
+ t.includeEOL = true;
+ t.tokenizeFile("\n{\"key\" value\n \"number\" 531\n \"f\" (fn [x] (+ x 2))}\n", "test.file");
+ assert.strictEqual(24, t.tokens.length);
+ assert.strictEqual(t.tokens.map(t => t.type).join(' '),
+ 'eol delim qstring ident eol qstring integer eol qstring delim ident delim ident delim delim catch-all ident integer delim delim delim eol eol eof');
+ t.includeEOL = false;
+ t.tokenizeFile("key value ---\nthis is\na fragment\n--- foo", "test.file");
+ assert.strictEqual(t.tokens.map(t => t.type).join(' '),
+ 'ident ident code-fragment ident eof');
+ });
+});