diff --git a/package-lock.json b/package-lock.json index 7d53b8fa..1c715b31 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,6 @@ "fast-png": "^5.0.4", "file-saver": "^2.0.5", "jquery": "^3.6.0", - "js-yaml": "^4.1.0", "jszip": "^3.7.0", "localforage": "^1.9.0", "mousetrap": "^1.6.5", @@ -746,7 +745,8 @@ "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true }, "node_modules/array-equal": { "version": "1.0.0", @@ -3599,6 +3599,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, "dependencies": { "argparse": "^2.0.1" }, @@ -8029,7 +8030,8 @@ "argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true }, "array-equal": { "version": "1.0.0", @@ -10205,6 +10207,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, "requires": { "argparse": "^2.0.1" } diff --git a/package.json b/package.json index 5cb29d87..adc8c9db 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,6 @@ "fast-png": "^5.0.4", "file-saver": "^2.0.5", "jquery": "^3.6.0", - "js-yaml": "^4.1.0", "jszip": "^3.7.0", "localforage": "^1.9.0", "mousetrap": "^1.6.5", diff --git a/src/common/ecs/compiler.ts b/src/common/ecs/compiler.ts new file mode 100644 index 00000000..4954125f --- /dev/null +++ b/src/common/ecs/compiler.ts @@ -0,0 +1,212 @@ + +import { Tokenizer, TokenType } from "../tokenizer"; +import { Action, ArrayType, ComponentType, DataField, DataType, DataValue, Dialect_CA65, Entity, EntityArchetype, EntityManager, EntityScope, IntType, Query, RefType, SelectType, SourceFileExport, System } from "./ecs"; + +export enum ECSTokenType { + Ellipsis = 'ellipsis', + Operator = 'delimiter', + QuotedString = 'quoted-string', + Integer = 'integer', +} + +export class ECSCompiler extends Tokenizer { + + em = new EntityManager(new Dialect_CA65()); // TODO + currentScope: EntityScope | null = null; + + constructor() { + super(); + //this.includeEOL = true; + this.setTokenRules([ + { type: TokenType.Ident, regex: /[$A-Za-z_][A-Za-z0-9_-]*/ }, + { type: TokenType.CodeFragment, regex: /---/ }, + { type: ECSTokenType.Ellipsis, regex: /\.\./ }, + { type: ECSTokenType.Operator, regex: /[=,:(){}\[\]]/ }, + { type: ECSTokenType.QuotedString, regex: /".*?"/ }, + { type: ECSTokenType.Integer, regex: /[-]?\d+/ }, + { type: TokenType.Ignore, regex: /\s+/ }, + ]); + this.errorOnCatchAll = true; + } + + parseFile(text: string, path: string) { + this.tokenizeFile(text, path); + while (!this.isEOF()) { + this.parseTopLevel(); + } + } + + parseTopLevel() { + //this.skipBlankLines(); + let tok = this.expectTokens(['component', 'system', 'scope', 'comment']); + if (tok.str == 'component') { + return this.em.defineComponent(this.parseComponentDefinition()); + } + if (tok.str == 'system') { + return this.em.defineSystem(this.parseSystem()); + } + if (tok.str == 'scope') { + return this.parseScope(); + } + if (tok.str == 'comment') { + this.expectTokenTypes([TokenType.CodeFragment]); + return; + } + this.compileError(`Unexpected top-level keyword: ${tok.str}`); + } + + parseComponentDefinition(): ComponentType { + let name = this.expectIdent().str; + let fields = []; + while (this.peekToken().str != 'end') { + fields.push(this.parseComponentField()); + } + this.expectToken('end'); + return { name, fields }; + } + + parseComponentField(): DataField { + let name = this.expectIdent(); + this.expectToken(':'); + let type = this.parseDataType(); + return { name: name.str, ...type }; + } + + parseDataType(): DataType { + if (this.peekToken().type == 'integer') { + let lo = this.expectInteger(); + this.expectToken('..'); + let hi = this.expectInteger(); + return { dtype: 'int', lo, hi } as IntType; + } + if (this.peekToken().str == '[') { + return { dtype: 'ref', query: this.parseQuery() } as RefType; + } + if (this.peekToken().str == 'array') { + this.expectToken('array'); + this.expectToken('of'); + return { dtype: 'array', elem: this.parseDataType() } as ArrayType; + } + this.compileError(`Unknown data type`); // TODO + } + + parseDataValue() : DataValue { + if (this.peekToken().type == 'integer') { + return this.expectInteger(); + } + this.compileError(`Unknown data value`); // TODO + } + + expectInteger(): number { + let i = parseInt(this.consumeToken().str); + if (isNaN(i)) this.compileError('There should be an integer here.'); + return i; + } + + parseSystem(): System { + let name = this.expectIdent().str; + let actions: Action[] = []; + let system: System = { name, actions }; + let cmd; + while ((cmd = this.consumeToken().str) != 'end') { + if (cmd == 'on') { + actions.push(this.parseAction()); + } else if (cmd == 'locals') { + system.tempbytes = this.expectInteger(); + } else { + this.compileError(`Unexpected system keyword: ${cmd}`); + } + } + return system; + } + + parseAction(): Action { + let event = this.expectIdent().str; + this.expectToken('do'); + let select = this.expectTokens(['once', 'each']).str as SelectType; // TODO: type check? + let query = this.parseQuery(); + let text = this.parseCode(); + return { text, event, query, select }; + } + + parseQuery() { + let q: Query = { include: [] }; + this.expectToken('['); + q.include = this.parseList(this.parseComponentRef, ',').map(c => c.name); + // TODO: other params + this.expectToken(']'); + return q; + } + + parseEvent() { + return this.expectIdent().str; + } + + parseCode(): string { + return this.expectTokenTypes([TokenType.CodeFragment]).str; + } + + parseScope() : EntityScope { + let name = this.expectIdent().str; + let scope = this.em.newScope(name, this.currentScope); + this.currentScope = scope; + let cmd; + while ((cmd = this.consumeToken().str) != 'end') { + if (cmd == 'entity') { + this.parseEntity(); + } else { + this.compileError(`Unexpected scope keyword: ${cmd}`); + } + } + this.currentScope = scope.parent; + return scope; + } + + parseEntity() : Entity { + let name = ''; + if (this.peekToken().type == TokenType.Ident) { + name = this.expectIdent().str; + } + let etype = this.parseEntityArchetype(); + let e = this.currentScope.newEntity(etype); + e.name = name; + let cmd; + while ((cmd = this.consumeToken().str) != 'end') { + // TODO: check data types + if (cmd == 'const') { + let name = this.expectIdent().str; + this.expectToken('='); + e.consts[name] = this.parseDataValue(); + } else if (cmd == 'init') { + let name = this.expectIdent().str; + this.expectToken('='); + e.inits[name] = this.parseDataValue(); + } else { + this.compileError(`Unexpected scope keyword: ${cmd}`); + } + } + return e; + } + + parseEntityArchetype() : EntityArchetype { + this.expectToken('['); + let components = this.parseList(this.parseComponentRef, ','); + this.expectToken(']'); + return {components}; + } + + parseComponentRef() : ComponentType { + let name = this.expectIdent().str; + let cref = this.em.getComponentByName(name); + if (!cref) this.compileError(`I couldn't find a component named "${name}".`) + return cref; + } + + exportToFile(src: SourceFileExport) { + for (let scope of Object.values(this.em.scopes)) { + scope.analyzeEntities(); + scope.generateCode(); + scope.dump(src); + } + } +} diff --git a/src/common/ecs/ecs.ts b/src/common/ecs/ecs.ts index 5c29d247..8f822dc4 100644 --- a/src/common/ecs/ecs.ts +++ b/src/common/ecs/ecs.ts @@ -1,6 +1,4 @@ -import * as YAML from "js-yaml"; - // entity scopes contain entities, and are nested // also contain segments (code, bss, rodata) // components and systems are global @@ -31,6 +29,8 @@ import * as YAML from "js-yaml"; // - converting assets to native formats? // - removing unused data +import { Token } from "../tokenizer"; + function mksymbol(c: ComponentType, fieldName: string) { return c.name + '_' + fieldName; } @@ -40,6 +40,7 @@ function mkscopesymbol(s: EntityScope, c: ComponentType, fieldName: string) { export interface Entity { id: number; + name?: string; etype: EntityArchetype; consts: { [component_field: string]: DataValue }; inits: { [scope_component_field: string]: DataValue }; @@ -62,7 +63,7 @@ export interface ComponentType { } export interface Query { - include: string[]; + include: string[]; // TODO: make ComponentType listen?: string[]; exclude?: string[]; updates?: string[]; @@ -79,9 +80,11 @@ export interface Action { text: string; event: string; query: Query; - select: 'once' | 'each' | 'source' + select: SelectType } +export type SelectType = 'once' | 'each' | 'source'; + export type DataValue = number | boolean | Uint8Array | Uint16Array; export type DataField = { name: string } & DataType; @@ -129,22 +132,21 @@ interface ArchetypeMatch { cmatch: ComponentType[]; } -class Dialect_CA65 { +export class Dialect_CA65 { readonly ASM_ITERATE_EACH = ` ldx #0 -%{@__each}: - %{code} +{{@__each}}: + {{code}} inx - cpx #%{ecount} - bne %{@__each} + cpx #{{ecount}} + bne {{@__each}} `; readonly INIT_FROM_ARRAY = ` - ldy #%{nbytes} -: - lda %{src}-1,y - sta %{dest}-1,y + ldy #{{nbytes}} +: lda {{src}}-1,y + sta {{dest}}-1,y dey - bne - + bne :- ` readonly HEADER = ` .include "vcs-ca65.h" @@ -159,9 +161,19 @@ VecBRK: .word Start Start: CLEAN_START ` + + comment(s: string) { + return `\n;;; ${s}\n`; + } + absolute(ident: string) { + return ident; + } + indexed_x(ident: string) { + return ident + ',x'; + } } -class SourceFileExport { +export class SourceFileExport { lines: string[] = []; comment(text: string) { @@ -254,7 +266,7 @@ class Segment { } getOriginSymbol() { let a = this.ofs2sym.get(0); - if (!a) throw new Error('getOriginSymbol'); + if (!a) throw new Error('getOriginSymbol(): no symbol at offset 0'); // TODO return a[0]; } } @@ -334,7 +346,7 @@ export class EntityScope { } allocateSegment(segment: Segment, readonly: boolean) { let fields = Object.values(segment.fieldranges); - fields.sort((a, b) => (a.ehi - a.elo + 1) * getPackedFieldSize(a.field)); + // TODO: fields.sort((a, b) => (a.ehi - a.elo + 1) * getPackedFieldSize(a.field)); let f; while (f = fields.pop()) { let name = mksymbol(f.component, f.field.name); @@ -413,9 +425,9 @@ export class EntityScope { let bufofs = this.rodata.allocateInitData(bufsym, initbytes); let code = this.dialect.INIT_FROM_ARRAY; //TODO: function to repalce from dict? - code = code.replace('%{nbytes}', initbytes.length.toString()) - code = code.replace('%{src}', bufsym); - code = code.replace('%{dest}', segment.getOriginSymbol()); + code = code.replace('{{nbytes}}', initbytes.length.toString()) + code = code.replace('{{src}}', bufsym); + code = code.replace('{{dest}}', segment.getOriginSymbol()); return code; } setConstValue(e: Entity, component: ComponentType, fieldName: string, value: DataValue) { @@ -454,9 +466,9 @@ export class EntityScope { for (let action of sys.actions) { if (action.event == event) { let code = this.replaceCode(action.text, sys, action); - s += `\n; \n`; + s += this.dialect.comment(``); s += code; - s += `\n; \n`; + s += this.dialect.comment(``); // TODO: check that this happens once? } } @@ -468,12 +480,13 @@ export class EntityScope { this.maxTempBytes = Math.max(this.tempOffset, this.maxTempBytes); } replaceCode(code: string, sys: System, action: Action): string { - const re = /\%\{(.+?)\}/g; + const re = /\{\{(.+?)\}\}/g; let label = sys.name + '_' + action.event; let atypes = this.em.archetypesMatching(action.query); let entities = this.entitiesMatching(atypes); // TODO: detect cycles // TODO: "source"? + // TODO: what if only 1 item? if (action.select == 'each') { code = this.wrapCodeInLoop(code, sys, action, entities); //console.log(sys.name, action.event, ents); @@ -513,10 +526,10 @@ export class EntityScope { // TODO: check ents // TODO: check segment bounds let s = this.dialect.ASM_ITERATE_EACH; - s = s.replace('%{elo}', ents[0].id.toString()); - s = s.replace('%{ehi}', ents[ents.length - 1].id.toString()); - s = s.replace('%{ecount}', ents.length.toString()); - s = s.replace('%{code}', code); + s = s.replace('{{elo}}', ents[0].id.toString()); + s = s.replace('{{ehi}}', ents[ents.length - 1].id.toString()); + s = s.replace('{{ecount}}', ents.length.toString()); + s = s.replace('{{code}}', code); return s; } generateCodeForField(sys: System, action: Action, @@ -547,12 +560,13 @@ export class EntityScope { // TODO: offset > 0? //let range = this.bss.getFieldRange(component, fieldName); // TODO: dialect + let ident = `${component.name}_${fieldName}_b${bitofs}`; if (action.select == 'once') { if (entities.length != 1) throw new Error(`can't choose multiple entities for ${fieldName} with select=once`); - return `${component.name}_${fieldName}_b${bitofs}` // TODO? check there's only 1 entity? + return this.dialect.absolute(ident) // TODO? check there's only 1 entity? } else { - return `${component.name}_${fieldName}_b${bitofs},x` // TODO? ,x? + return this.dialect.indexed_x(ident) } } entitiesMatching(atypes: ArchetypeMatch[]) { @@ -625,12 +639,13 @@ export class EntityScope { } export class EntityManager { - dialect = new Dialect_CA65(); archtypes = new Set(); components: { [name: string]: ComponentType } = {}; systems: { [name: string]: System } = {}; scopes: { [name: string]: EntityScope } = {}; + constructor(public readonly dialect: Dialect_CA65) { + } newScope(name: string, parent?: EntityScope) { let scope = new EntityScope(this, this.dialect, name, parent); if (this.scopes[name]) throw new Error(`scope ${name} already defined`); @@ -643,7 +658,7 @@ export class EntityManager { } defineSystem(system: System) { if (this.systems[system.name]) throw new Error(`system ${system.name} already defined`); - this.systems[system.name] = system; + return this.systems[system.name] = system; } componentsMatching(q: Query, etype: EntityArchetype) { let list = []; @@ -680,395 +695,13 @@ export class EntityManager { } } } - toYAML() { - return YAML.dump({ + getComponentByName(name: string): ComponentType { + return this.components[name]; + } + toJSON() { + return JSON.stringify({ components: this.components, - systems: this.systems, + systems: this.systems }) } } - -/// - -const TEMPLATE1 = ` -%{@NextFrame}: - FRAME_START - %{!preframe} - KERNEL_START - %{!kernel} - KERNEL_END - %{!postframe} - FRAME_END - lsr SWCHB ; test Game Reset switch - bcs @NoStart - jmp Start -@NoStart: - jmp %{@NextFrame} -`; - -// TODO: two sticks? -const TEMPLATE2_a = ` - lda SWCHA - sta %{$0} -` -const TEMPLATE2_b = ` - asl %{$0} - bcs %{@SkipMoveRight} - %{!joyright} -%{@SkipMoveRight}: - asl %{$0} - bcs %{@SkipMoveLeft} - %{!joyleft} -%{@SkipMoveLeft}: - asl %{$0} - bcs %{@SkipMoveDown} - %{!joydown} -%{@SkipMoveDown}: - asl %{$0} - bcs %{@SkipMoveUp} - %{!joyup} -%{@SkipMoveUp}: -`; - -const TEMPLATE3_L = ` - lda %{ [start] -> frameloop - // frameloop -> [preframe] [kernel] [postframe] - - // temp between preframe + frame? - // TODO: check names for identifierness - em.defineSystem({ - name: 'kernel_simple', - tempbytes: 8, - actions: [ - { - text: TEMPLATE4_S1, event: 'preframe', select: 'once', query: { - include: ['kernel'] - } - }, - { - // TODO: should include kernel for numlines - text: TEMPLATE4_S2, event: 'preframe', select: 'once', query: { - include: ['sprite', 'hasbitmap', 'hascolormap', 'ypos'], - }, - }, - { - text: TEMPLATE4_K, event: 'kernel', select: 'once', query: { - include: ['kernel'] - } - }, - ] - }) - em.defineSystem({ - name: 'set_xpos', - actions: [ - { - text: SET_XPOS, event: 'preframe', select: 'each', query: { - include: ['sprite', 'xpos'] - }, - }, - //{ text:SETHORIZPOS }, - ] - }) - // https://docs.unity3d.com/Packages/com.unity.entities@0.17/manual/ecs_systems.html - em.defineSystem({ - name: 'frameloop', - emits: ['preframe', 'kernel', 'postframe'], - actions: [ - { text: TEMPLATE1, event: 'start', select: 'once', query: { include: ['kernel'] } } - ] - }) - em.defineSystem({ - name: 'joyread', - tempbytes: 1, - emits: ['joyup', 'joydown', 'joyleft', 'joyright', 'joybutton'], - actions: [ - { text: TEMPLATE2_a, event: 'postframe', select: 'once', query: { include: ['player'] } }, - { text: TEMPLATE2_b, event: 'postframe', select: 'each', query: { include: ['player'] } } - ] - }); - em.defineSystem({ - name: 'move_x', - actions: [ - { text: TEMPLATE3_L, event: 'joyleft', select: 'source', query: { include: ['player', 'xpos'] }, }, - { text: TEMPLATE3_R, event: 'joyright', select: 'source', query: { include: ['player', 'xpos'] }, }, - ] - }); - em.defineSystem({ - name: 'move_y', - actions: [ - { text: TEMPLATE3_U, event: 'joyup', select: 'source', query: { include: ['player', 'ypos'] } }, - { text: TEMPLATE3_D, event: 'joydown', select: 'source', query: { include: ['player', 'ypos'] } }, - ] - }); - em.defineSystem({ - name: 'SetHorizPos', - actions: [ - { text: SETHORIZPOS, event: 'SetHorizPos', select: 'once', query: { include: ['xpos'] } }, - ] - }); - - let root = em.newScope("Root"); - let scene = em.newScope("Scene", root); - let e_ekernel = root.newEntity({ components: [c_kernel] }); - root.setConstValue(e_ekernel, c_kernel, 'lines', 192); - //root.setConstValue(e_ekernel, c_kernel, 'bgcolor', 0x92); - root.setInitValue(e_ekernel, c_kernel, 'bgcolor', 0x92); - - let e_bitmap0 = root.newEntity({ components: [c_bitmap] }); - // TODO: store array sizes? - root.setConstValue(e_bitmap0, c_bitmap, 'bitmapdata', new Uint8Array([1, 1, 3, 7, 15, 31, 63, 127])); - - let e_colormap0 = root.newEntity({ components: [c_colormap] }); - root.setConstValue(e_colormap0, c_colormap, 'colormapdata', new Uint8Array([6, 3, 6, 9, 12, 14, 31, 63])); - - let ea_playerSprite = { components: [c_sprite, c_hasbitmap, c_hascolormap, c_xpos, c_ypos, c_player] }; - let e_player0 = root.newEntity(ea_playerSprite); - root.setConstValue(e_player0, c_sprite, 'plyrindex', 0); - root.setInitValue(e_player0, c_sprite, 'height', 8); - root.setInitValue(e_player0, c_xpos, 'xpos', 50); - root.setInitValue(e_player0, c_ypos, 'ypos', 50); - let e_player1 = root.newEntity(ea_playerSprite); - root.setConstValue(e_player1, c_sprite, 'plyrindex', 1); - root.setInitValue(e_player1, c_sprite, 'height', 8); - root.setInitValue(e_player1, c_xpos, 'xpos', 100); - root.setInitValue(e_player1, c_ypos, 'ypos', 60); - - //console.log(em.archetypesMatching({ include:['xpos','ypos']})[0]) - - let src = new SourceFileExport(); - root.analyzeEntities(); - root.generateCode(); - root.dump(src); - console.log(src.toString()); - //console.log(em.toYAML()); -} - -// TODO: files in markdown? -// TODO: jsr OperModeExecutionTree? - -test(); - diff --git a/src/common/ecs/main.ts b/src/common/ecs/main.ts new file mode 100644 index 00000000..e035ed53 --- /dev/null +++ b/src/common/ecs/main.ts @@ -0,0 +1,30 @@ +import { readFileSync } from "fs"; +import { ECSCompiler } from "./compiler"; +import { SourceFileExport } from "./ecs"; + +class ECSMain { + + compiler = new ECSCompiler(); + + constructor(readonly args: string[]) { + } + + run() { + for (let path of this.args) { + let text = readFileSync(path, 'utf-8'); + try { + this.compiler.parseFile(text, path); + } catch (e) { + console.log(e); + for (let err of this.compiler.errors) { + console.log(`${err.path}:${err.line}:${err.start}: ${err.msg}`); + } + } + } + let file = new SourceFileExport(); + this.compiler.exportToFile(file); + console.log(file.toString()); + } +} + +new ECSMain(process.argv.slice(2)).run(); diff --git a/src/common/tokenizer.ts b/src/common/tokenizer.ts new file mode 100644 index 00000000..c603ae7e --- /dev/null +++ b/src/common/tokenizer.ts @@ -0,0 +1,239 @@ + +import type { SourceLocation, SourceLine, WorkerError } from "./workertypes"; + +// objects that have source code position info +export interface SourceLocated { + $loc?: SourceLocation; +} +// statements also have the 'offset' (pc) field from SourceLine +export interface SourceLineLocated { + $loc?: SourceLine; +} + +export class CompileError extends Error { + $loc: SourceLocation; + constructor(msg: string, loc: SourceLocation) { + super(msg); + Object.setPrototypeOf(this, CompileError.prototype); + this.$loc = loc; + } +} + +export function mergeLocs(a: SourceLocation, b: SourceLocation): SourceLocation { + return { + line: Math.min(a.line, b.line), + start: Math.min(a.start, b.start), + end: Math.max(a.end, b.end), + label: a.label || b.label, + path: a.path || b.path, + } +} + +export enum TokenType { + EOF = 'eof', + EOL = 'eol', + Ident = 'ident', + Comment = 'comment', + Ignore = 'ignore', + CodeFragment = 'code-fragment', + CatchAll = 'catch-all', +} + +export class Token implements SourceLocated { + str: string; + type: string; + $loc: SourceLocation; +} + +export class TokenRule { + type: string; + regex: RegExp; +} + +const CATCH_ALL_RULES: TokenRule[] = [ + { type: TokenType.CatchAll, regex: /.+?/ } +] + +function re_escape(rule: TokenRule): string { + return `(${rule.regex.source})`; +} + +export class Tokenizer { + rules: TokenRule[]; + regex: RegExp; + path: string; + lineno: number; + tokens: Token[]; + lasttoken: Token; + errors: WorkerError[]; + curlabel: string; + eol: Token; + includeEOL = false; + errorOnCatchAll = false; + codeFragment : string | null = null; + + constructor() { + this.lineno = 0; + this.errors = []; + } + setTokenRules(rules: TokenRule[]) { + this.rules = rules.concat(CATCH_ALL_RULES); + var pattern = this.rules.map(re_escape).join('|'); + this.regex = new RegExp(pattern, "g"); + } + tokenizeFile(contents: string, path: string) { + this.path = path; + this.tokens = []; // can't have errors until this is set + let txtlines = contents.split(/\n|\r\n?/); + txtlines.forEach((line) => this._tokenize(line)); + this._pushToken({ type: TokenType.EOF, str: "", $loc: { path: this.path, line: this.lineno } }); + } + tokenizeLine(line: string) : void { + this.lineno++; + this._tokenize(line); + } + _tokenize(line: string): void { + this.lineno++; + this.eol = { type: TokenType.EOL, str: "", $loc: { path: this.path, line: this.lineno, start: line.length } }; + // iterate over each token via re_toks regex + let m: RegExpMatchArray; + while (m = this.regex.exec(line)) { + let found = false; + // find out which capture group was matched, and thus token type + for (let i = 0; i < this.rules.length; i++) { + let s: string = m[i + 1]; + if (s != null) { + found = true; + let loc = { path: this.path, line: this.lineno, start: m.index, end: m.index + s.length }; + let rule = this.rules[i]; + // add token to list + switch (rule.type) { + case TokenType.CodeFragment: + if (this.codeFragment) { + this._pushToken({ str: this.codeFragment, type: rule.type, $loc: loc }); //TODO: merge start/end + this.codeFragment = null; + } else { + this.codeFragment = ''; + return; // don't add any more tokens (TODO: check for trash?) + } + break; + case TokenType.CatchAll: + if (this.errorOnCatchAll && this.codeFragment == null) { + this.compileError(`I didn't expect the character "${m[0]}" here.`); + } + default: + if (this.codeFragment == null) { + this._pushToken({ str: s, type: rule.type, $loc: loc }); + } + case TokenType.Ignore: + break; + } + break; + } + } + if (!found) { + this.compileError(`Could not parse token: <<${m[0]}>>`) + } + } + if (this.includeEOL) { + this._pushToken(this.eol); + } + if (this.codeFragment != null) { + this.codeFragment += line + '\n'; + } + } + _pushToken(token: Token) { + this.tokens.push(token); + } + addError(msg: string, loc?: SourceLocation) { + let tok = this.lasttoken || this.peekToken(); + if (!loc) loc = tok.$loc; + this.errors.push({ path: loc.path, line: loc.line, label: this.curlabel, start: loc.start, end: loc.end, msg: msg }); + } + internalError() { + this.compileError("Internal error."); + } + notImplementedError() { + this.compileError("Not yet implemented."); + } + compileError(msg: string, loc?: SourceLocation, loc2?: SourceLocation) { + this.addError(msg, loc); + //if (loc2 != null) this.addError(`...`, loc2); + throw new CompileError(msg, loc); + } + peekToken(lookahead?: number): Token { + let tok = this.tokens[lookahead || 0]; + return tok ? tok : this.eol; + } + consumeToken(): Token { + let tok = this.lasttoken = (this.tokens.shift() || this.eol); + return tok; + } + expectToken(str: string, msg?: string): Token { + let tok = this.consumeToken(); + let tokstr = tok.str; + if (str != tokstr) { + this.compileError(msg || `There should be a "${str}" here.`); + } + return tok; + } + expectTokens(strlist: string[], msg?: string): Token { + let tok = this.consumeToken(); + let tokstr = tok.str; + if (strlist.indexOf(tokstr) < 0) { + this.compileError(msg || `There should be a ${strlist.map((s) => `"${s}"`).join(' or ')} here, not "${tokstr}.`); + } + return tok; + } + parseModifiers(modifiers: string[]): { [modifier: string]: boolean } { + let result = {}; + do { + var tok = this.peekToken(); + if (modifiers.indexOf(tok.str) < 0) + return result; + this.consumeToken(); + result[tok.str] = true; + } while (tok != null); + } + expectIdent(msg?: string): Token { + let tok = this.consumeToken(); + if (tok.type != TokenType.Ident) + this.compileError(msg || `There should be an identifier here.`); + return tok; + } + pushbackToken(tok: Token) { + this.tokens.unshift(tok); + } + isEOF() { + return this.tokens.length == 0 || this.peekToken().type == 'eof'; // TODO? + } + expectEOL(msg?: string) { + let tok = this.consumeToken(); + if (tok.type != TokenType.EOL) + this.compileError(msg || `There's too much stuff on this line.`); + } + skipBlankLines() { + this.skipTokenTypes(['eol']); + } + skipTokenTypes(types: string[]) { + while (types.includes(this.peekToken().type)) + this.consumeToken(); + } + expectTokenTypes(types: string[], msg?: string) { + let tok = this.consumeToken(); + if (!types.includes(tok.type)) + this.compileError(msg || `There should be a ${types.map((s) => `"${s}"`).join(' or ')} here. not a "${tok.type}".`); + return tok; + } + parseList(parseFunc:()=>T, delim:string): T[] { + var sep; + var list = []; + do { + var el = parseFunc.bind(this)(); // call parse function + if (el != null) list.push(el); // add parsed element to list + sep = this.consumeToken(); // consume seperator token + } while (sep.str == delim); + this.pushbackToken(sep); + return list; + } +} diff --git a/src/test/testecs.ts b/src/test/testecs.ts new file mode 100644 index 00000000..6709a9a1 --- /dev/null +++ b/src/test/testecs.ts @@ -0,0 +1,449 @@ +import { describe } from "mocha"; +import { ECSCompiler } from "../common/ecs/compiler"; +import { Dialect_CA65, EntityManager, SourceFileExport } from "../common/ecs/ecs"; + +const TEMPLATE1 = ` +{{@NextFrame}}: + FRAME_START + {{!preframe}} + KERNEL_START + {{!kernel}} + KERNEL_END + {{!postframe}} + FRAME_END + lsr SWCHB ; test Game Reset switch + bcs @NoStart + jmp Start +@NoStart: + jmp {{@NextFrame}} +`; + +// TODO: two sticks? +const TEMPLATE2_a = ` + lda SWCHA + sta {{$0}} +` +const TEMPLATE2_b = ` + asl {{$0}} + bcs {{@SkipMoveRight}} + {{!joyright}} +{{@SkipMoveRight}}: + asl {{$0}} + bcs {{@SkipMoveLeft}} + {{!joyleft}} +{{@SkipMoveLeft}}: + asl {{$0}} + bcs {{@SkipMoveDown}} + {{!joydown}} +{{@SkipMoveDown}}: + asl {{$0}} + bcs {{@SkipMoveUp}} + {{!joyup}} +{{@SkipMoveUp}}: +`; + +const TEMPLATE3_L = ` + lda {{ [start] -> frameloop + // frameloop -> [preframe] [kernel] [postframe] + + // temp between preframe + frame? + // TODO: check names for identifierness + em.defineSystem({ + name: 'kernel_simple', + tempbytes: 8, + actions: [ + { + text: TEMPLATE4_S1, event: 'preframe', select: 'once', query: { + include: ['kernel'] + } + }, + { + // TODO: should include kernel for numlines + text: TEMPLATE4_S2, event: 'preframe', select: 'once', query: { + include: ['sprite', 'hasbitmap', 'hascolormap', 'ypos'], + }, + }, + { + text: TEMPLATE4_K, event: 'kernel', select: 'once', query: { + include: ['kernel'] + } + }, + ] + }) + em.defineSystem({ + name: 'set_xpos', + actions: [ + { + text: SET_XPOS, event: 'preframe', select: 'each', query: { + include: ['sprite', 'xpos'] + }, + }, + //{ text:SETHORIZPOS }, + ] + }) + // https://docs.unity3d.com/Packages/com.unity.entities@0.17/manual/ecs_systems.html + em.defineSystem({ + name: 'frameloop', + emits: ['preframe', 'kernel', 'postframe'], + actions: [ + { text: TEMPLATE1, event: 'start', select: 'once', query: { include: ['kernel'] } } + ] + }) + em.defineSystem({ + name: 'joyread', + tempbytes: 1, + emits: ['joyup', 'joydown', 'joyleft', 'joyright', 'joybutton'], + actions: [ + { text: TEMPLATE2_a, event: 'postframe', select: 'once', query: { include: ['player'] } }, + { text: TEMPLATE2_b, event: 'postframe', select: 'each', query: { include: ['player'] } } + ] + }); + em.defineSystem({ + name: 'move_x', + actions: [ + { text: TEMPLATE3_L, event: 'joyleft', select: 'source', query: { include: ['player', 'xpos'] }, }, + { text: TEMPLATE3_R, event: 'joyright', select: 'source', query: { include: ['player', 'xpos'] }, }, + ] + }); + em.defineSystem({ + name: 'move_y', + actions: [ + { text: TEMPLATE3_U, event: 'joyup', select: 'source', query: { include: ['player', 'ypos'] } }, + { text: TEMPLATE3_D, event: 'joydown', select: 'source', query: { include: ['player', 'ypos'] } }, + ] + }); + em.defineSystem({ + name: 'SetHorizPos', + actions: [ + { text: SETHORIZPOS, event: 'SetHorizPos', select: 'once', query: { include: ['xpos'] } }, + ] + }); + + let root = em.newScope("Root"); + let scene = em.newScope("Scene", root); + let e_ekernel = root.newEntity({ components: [c_kernel] }); + root.setConstValue(e_ekernel, c_kernel, 'lines', 192); + //root.setConstValue(e_ekernel, c_kernel, 'bgcolor', 0x92); + root.setInitValue(e_ekernel, c_kernel, 'bgcolor', 0x92); + + let e_bitmap0 = root.newEntity({ components: [c_bitmap] }); + // TODO: store array sizes? + root.setConstValue(e_bitmap0, c_bitmap, 'bitmapdata', new Uint8Array([1, 1, 3, 7, 15, 31, 63, 127])); + + let e_colormap0 = root.newEntity({ components: [c_colormap] }); + root.setConstValue(e_colormap0, c_colormap, 'colormapdata', new Uint8Array([6, 3, 6, 9, 12, 14, 31, 63])); + + let ea_playerSprite = { components: [c_sprite, c_hasbitmap, c_hascolormap, c_xpos, c_ypos, c_player] }; + let e_player0 = root.newEntity(ea_playerSprite); + root.setConstValue(e_player0, c_sprite, 'plyrindex', 0); + root.setInitValue(e_player0, c_sprite, 'height', 8); + root.setInitValue(e_player0, c_xpos, 'xpos', 50); + root.setInitValue(e_player0, c_ypos, 'ypos', 50); + let e_player1 = root.newEntity(ea_playerSprite); + root.setConstValue(e_player1, c_sprite, 'plyrindex', 1); + root.setInitValue(e_player1, c_sprite, 'height', 8); + root.setInitValue(e_player1, c_xpos, 'xpos', 100); + root.setInitValue(e_player1, c_ypos, 'ypos', 60); + + //console.log(em.archetypesMatching({ include:['xpos','ypos']})[0]) + + root.analyzeEntities(); + root.generateCode(); + let src = new SourceFileExport(); + root.dump(src); + //console.log(src.toString()); + //console.log(em.toYAML()); +} + +function testCompiler() { + let c = new ECSCompiler(); + try { + c.parseFile(` + +component Kernel + lines: 0..255 + bgcolor: 0..255 +end + +component Bitmap + data: array of 0..255 +end + +component HasBitmap + bitmap: [Bitmap] +end + +system SimpleKernel +locals 8 +on preframe do once [Kernel] --- JUNK_AT_END + lda #5 + sta #6 +Label: +--- +end + +scope Root + + entity kernel [Kernel] + const lines = 100 + end + + entity player1 [HasBitmap] + const plyrindex = 0 + init height = 8 + init xpos = 100 + init ypos = 100 + end + +end + +`, 'foo.txt'); + console.log('json', c.em.toJSON()); + let src = new SourceFileExport(); + c.exportToFile(src); + // TODO: test? + //console.log(src.toString()); + } catch (e) { + console.log(e); + for (let err of c.errors) { + console.log(err); + } + console.log(c.tokens); + } +} + +// TODO: files in markdown? +// TODO: jsr OperModeExecutionTree? + +describe('Tokenizer', function() { + it('Should use API', function() { + testECS(); + }); + it('Should use Compiler', function() { + testCompiler(); + }); +}); diff --git a/src/test/testutil.ts b/src/test/testutil.ts index b95210ec..9e340a69 100644 --- a/src/test/testutil.ts +++ b/src/test/testutil.ts @@ -1,8 +1,9 @@ import assert from "assert"; import { describe } from "mocha"; -import { EmuHalt } from "../../src/common/emu" -import { lzgmini, isProbablyBinary } from "../../src/common/util"; +import { EmuHalt } from "../common/emu" +import { lzgmini, isProbablyBinary } from "../common/util"; +import { Tokenizer, TokenType } from "../common/tokenizer"; var NES_CONIO_ROM_LZG = [ 76,90,71,0,0,160,16,0,0,11,158,107,131,223,83,1,9,17,21,22,78,69,83,26,2,1,3,0,22,6,120,216, @@ -133,4 +134,38 @@ var NES_CONIO_ROM_LZG = [ assert.ok(e.hasOwnProperty('$loc')); }); }); - \ No newline at end of file + +describe('Tokenizer', function() { + it('Should tokenize', function() { + var t = new Tokenizer(); + t.setTokenRules([ + { type: 'ident', regex: /[$A-Za-z_][A-Za-z0-9_-]*/ }, + { type: 'delim', regex: /[\(\)\{\}\[\]]/ }, + { type: 'qstring', regex: /".*?"/ }, + { type: 'integer', regex: /[-]?\d+/ }, + { type: 'ignore', regex: /\s+/ }, + { type: TokenType.CodeFragment, regex: /---/ }, + ]); + t.tokenizeFile("\n{\"key\" value\n \"number\" 531\n \"f\" (fn [x] (+ x 2))}\n", "test.file"); + assert.strictEqual(t.tokens.map(t => t.type).join(' '), + 'delim qstring ident qstring integer qstring delim ident delim ident delim delim catch-all ident integer delim delim delim eof'); + assert.strictEqual(t.tokens.map(t => t.str).join(' '), + '{ "key" value "number" 531 "f" ( fn [ x ] ( + x 2 ) ) } '); + assert.strictEqual(19, t.tokens.length); + assert.strictEqual('{', t.peekToken().str); + assert.strictEqual('{', t.expectToken('{').str); + t.pushbackToken(t.consumeToken()); + assert.strictEqual('"key"', t.consumeToken().str); + assert.deepStrictEqual({'value':true}, t.parseModifiers(['foo','value','bar'])); + assert.deepStrictEqual([], t.errors); + t.includeEOL = true; + t.tokenizeFile("\n{\"key\" value\n \"number\" 531\n \"f\" (fn [x] (+ x 2))}\n", "test.file"); + assert.strictEqual(24, t.tokens.length); + assert.strictEqual(t.tokens.map(t => t.type).join(' '), + 'eol delim qstring ident eol qstring integer eol qstring delim ident delim ident delim delim catch-all ident integer delim delim delim eol eol eof'); + t.includeEOL = false; + t.tokenizeFile("key value ---\nthis is\na fragment\n--- foo", "test.file"); + assert.strictEqual(t.tokens.map(t => t.type).join(' '), + 'ident ident code-fragment ident eof'); + }); +});