diff --git a/extension/src/buildcore.ts b/extension/src/buildcore.ts index d9eeedb3..5ac19842 100644 --- a/extension/src/buildcore.ts +++ b/extension/src/buildcore.ts @@ -42,6 +42,12 @@ export interface BuildOutcome { segments?: Segment[]; /** every project path the build read */ paths: string[]; + /** + * The files the build read besides the main file, keyed by both worker + * filename and project path. The emulator reads some at load time + * (verilog's $readmem). + */ + files?: { [path: string]: FileData }; /** true when no inputs changed since the previous build */ unchanged?: boolean; } @@ -84,6 +90,8 @@ export class Builder { getToolForFilename: getTool, }, deps); var paths = [req.mainPath].concat(deps.map(d => d.path)); + var files: { [path: string]: FileData } = {}; + for (var d of deps) files[d.filename] = files[d.path] = d.data; var key = `${req.platform}/${tool}/${req.mainPath}`; var result: WorkerResult; try { @@ -102,7 +110,7 @@ export class Builder { // Keep the last success even after a failed build: reverting to the // source that built it makes the worker say unchanged again, and that // output is what Run (and the next unchanged build) must use. - return { ...this.last.get(key), success: true, tool, paths, diagnostics: [], unchanged: true }; + return { ...this.last.get(key), success: true, tool, paths, files, diagnostics: [], unchanged: true }; } if ('errors' in result && result.errors && result.errors.length) { var toPath = (err: WorkerError) => (err.path && filename2path[err.path]) || err.path || req.mainPath; @@ -116,7 +124,7 @@ export class Builder { // the receiver runs projectcore.processListings on them var r = result as any; var outcome: BuildOutcome = { - success: true, tool, paths, diagnostics: [], + success: true, tool, paths, files, diagnostics: [], output: r.output, listings: r.listings, symbolmap: r.symbolmap, segments: r.segments, }; this.last.set(key, outcome); diff --git a/extension/src/emuworker.ts b/extension/src/emuworker.ts index 10de6bb9..b81ffa93 100644 --- a/extension/src/emuworker.ts +++ b/extension/src/emuworker.ts @@ -11,6 +11,7 @@ import { clearLastKeycodeMap, describeControls, getLastKeycodeMap, setHaltHandle import { ControlHint, PLATFORM_CONTROLS } from '../../src/common/controls'; import { AudioStream, setAudioStreamFactory } from '../../src/common/audio'; import { getRootBasePlatform } from '../../src/common/util'; +import type { FileData } from '../../src/common/workertypes'; /** Audio is resampled to this rate in the worker; the webview plays it back. */ const STREAM_RATE = 48000; @@ -76,7 +77,8 @@ class RpcAudioStream implements AudioStream { } const rpc: Rpc = new Rpc(parentPort, { - async start(platform: string, rom: any) { + /** `files` are the project files the program reads at load time (BuildOutcome.files) */ + async start(platform: string, rom: any, files?: { [path: string]: FileData }) { // platform modules keep global state (Javatari deletes its own start()), // so a worker runs one emulator; the host starts a new worker per run if (started) throw new Error('This emulator worker already ran a platform; start a new worker.'); @@ -87,14 +89,16 @@ const rpc: Rpc = new Rpc(parentPort, { await target.start(); // machines build their keyboard handler as they start controls = PLATFORM_CONTROLS[getRootBasePlatform(platform)] || describeControls(getLastKeycodeMap()); - target.loadROM(rom); + target.setFileData(files || {}); + await target.loadROM(rom); resume(); rpc.emit('audioReset', null); return status(); }, - loadROM(rom: any) { + async loadROM(rom: any, files?: { [path: string]: FileData }) { if (!target) throw new Error('emulator not started'); - target.loadROM(rom); + target.setFileData(files || {}); + await target.loadROM(rom); resume(); rpc.emit('audioReset', null); return status(); diff --git a/extension/src/extension.ts b/extension/src/extension.ts index 5692845a..bd91c28c 100644 --- a/extension/src/extension.ts +++ b/extension/src/extension.ts @@ -46,8 +46,8 @@ interface Target { /** What the emulator runs, so a rebuild of the same thing reloads it. */ let running: Target | undefined; -/** The ROM it runs, so an identical rebuild (a comment edit) doesn't restart it. */ -let runningRom: Uint8Array | undefined; +/** The build it runs, so an identical rebuild (a comment edit) doesn't restart it. */ +let runningBuild: BuildOutcome | undefined; /** The target auto-build rebuilds: the last one changed or saved. */ let autoTarget: Target | undefined; /** Diagnostics from a failed type-build, shown after a pause in typing. */ @@ -360,7 +360,7 @@ async function buildAndMaybeRun(target: Target, run: boolean, reason: BuildReaso return; } if (run) { - if (result.output) await startEmulator(target, result.output); + if (result.output) await startEmulator(target, result); else { output.appendLine('8bitworkshop: the build produced no ROM to run; see the output for the last build.'); output.show(true); @@ -407,7 +407,7 @@ async function followActiveEditor() { await scope.setTargetChoice(project, 'follow'); } -async function startEmulator(target: Target, rom: any) { +async function startEmulator(target: Target, build: BuildOutcome) { // each run gets a fresh worker: platforms keep global state (see emuworker) emu?.dispose(); emu = undefined; @@ -436,11 +436,11 @@ async function startEmulator(target: Target, rom: any) { panel.setTitle(`${title} (${target.platform})`); panel.setMuted(muted); try { - emuStatus = await worker.call('start', target.platform, rom); + emuStatus = await worker.call('start', target.platform, build.output, build.files); worker.call('setMuted', muted); panel.showStatus(emuStatus); running = target; - runningRom = rom; + runningBuild = build; output.appendLine(`Running ${title} on ${target.platform}`); } catch (e) { output.appendLine(`Emulator failed to start: ${e && e.stack || e}`); @@ -455,10 +455,15 @@ async function reloadEmulator(target: Target, reason: BuildReason, result: Build var mode = config(target.main).get('reloadOnBuild', 'always'); if (mode === 'never' || (mode === 'onSave' && reason === 'type')) return; if (emuStatus.platform !== target.platform) { - await startEmulator(target, result.output); - } else if (!sameBytes(runningRom, result.output)) { - emuStatus = await (await getEmu(target.platform)).call('loadROM', result.output); - runningRom = result.output; + await startEmulator(target, result); + } else if (!sameBuild(runningBuild, result)) { + runningBuild = result; + try { + emuStatus = await (await getEmu(target.platform)).call('loadROM', result.output, result.files); + } catch (e) { + output.appendLine(`Emulator failed to load: ${e && e.message || e}`); + output.show(true); + } } } @@ -471,10 +476,27 @@ function setMuted(m: boolean) { vscode.commands.executeCommand('setContext', '8bitworkshop.muted', m); } -function sameBytes(a: Uint8Array | undefined, b: Uint8Array | undefined): boolean { - if (!a || !b || a.length !== b.length) return false; - for (var i = 0; i < a.length; i++) if (a[i] !== b[i]) return false; - return true; +/** Same output and same load-time files (see BuildOutcome.files). */ +function sameBuild(a: BuildOutcome | undefined, b: BuildOutcome): boolean { + if (!a || !sameData(a.output, b.output)) return false; + var af = a.files || {}, bf = b.files || {}; + var keys = Object.keys(bf); + return keys.length === Object.keys(af).length && keys.every(k => sameData(af[k], bf[k])); +} + +/** Compares ROM bytes, file text, or (verilog) the compiled unit. */ +function sameData(a: any, b: any): boolean { + if (a instanceof Uint8Array && b instanceof Uint8Array) { + if (a.length !== b.length) return false; + for (var i = 0; i < a.length; i++) if (a[i] !== b[i]) return false; + return true; + } + if (typeof a === 'string' || typeof b === 'string') return a === b; + try { + return !!a && !!b && JSON.stringify(a) === JSON.stringify(b); + } catch (e) { + return false; // not JSON (cycles): assume it changed + } } /** diff --git a/extension/test/emuworker.test.ts b/extension/test/emuworker.test.ts index 2506c635..5aa94c45 100644 --- a/extension/test/emuworker.test.ts +++ b/extension/test/emuworker.test.ts @@ -5,6 +5,7 @@ import * as path from 'path'; import { Worker } from 'worker_threads'; import { Rpc } from '../src/rpc'; import { findRootDir } from '../src/projectinfo'; +import { Builder, ProjectFileProvider } from '../src/buildcore'; import type { AudioChunk, EmuStatus, FrameEvent } from '../src/emuworker'; // Drives out/emuworker.js in a worker thread, the way the extension host does. @@ -133,4 +134,53 @@ describe('extension emuworker', function () { await rpc.call('setMuted', false); await waitForAudio(1); }); + + // $readmem reads project files at load time, so they travel with the ROM + describe('verilog $readmem', function () { + const MAIN = [ + 'module top(clk, reset, hsync, vsync, rgb);', + ' input clk, reset;', + ' output hsync, vsync;', + ' output [3:0] rgb;', + ' reg [7:0] rom[0:3];', + ' initial $readmemh("rom_data.hex", rom);', + ' assign hsync = 0;', + ' assign vsync = 0;', + ' assign rgb = rom[1][3:0];', + 'endmodule', + '', + ].join('\n'); + var built: Awaited>; + + before(async function () { + var enc = new TextEncoder(); + var files: { [path: string]: string } = { 'main.v': MAIN, 'rom_data.hex': '00\n0f\n00\n00\n' }; + var read = async (rel: string) => rel in files ? enc.encode(files[rel]) : null; + built = await new Builder(ROOT).build({ + platform: 'verilog', mainPath: 'main.v', mainText: MAIN, + files: new ProjectFileProvider(read, ROOT, 'verilog'), + }); + assert.deepEqual(built.diagnostics, []); + }); + + it('builds with the data file attached', function () { + assert.equal(built.files['rom_data.hex'], '00\n0f\n00\n00\n'); + }); + + it('runs with the data file', async function () { + var s = await rpc.call('start', 'verilog', built.output, built.files); + assert.equal(s.state, 'running'); + await waitForFrames(3); + assert.ok(!statuses.some(s => s && s.state === 'halted'), JSON.stringify(statuses)); + }); + + it('reports a missing data file without killing the worker', async function () { + var exited = false; + worker.on('exit', () => { exited = true; }); + await assert.rejects(rpc.call('start', 'verilog', built.output, {}), /no file "rom_data.hex"/); + await new Promise(r => setTimeout(r, 200)); + assert.ok(!exited, 'worker exited'); + await rpc.call('status'); + }); + }); }); diff --git a/src/common/emucore.ts b/src/common/emucore.ts index eb21cb19..5a8648dd 100644 --- a/src/common/emucore.ts +++ b/src/common/emucore.ts @@ -20,6 +20,7 @@ import { CpuState, DisasmLine, EmuState, Machine, Platform, hasProbe, isDebuggable, } from "./baseplatform"; import { ProbeAll, SampledAudioParams, TrapCondition } from "./devices"; +import { FileData } from "./workertypes"; import { disassemble6502 } from "./cpu/disasm6502"; import { disassembleZ80 } from "./cpu/disasmz80"; import { disassembleSM83 } from "./cpu/disasmSM83"; @@ -107,11 +108,14 @@ function installHeadlessVideo() { const datau8 = new Uint8Array(buffer); const datau32 = new Uint32Array(buffer); // the first one is the screen; later ones are debug views (nes nametables) - if (!pixels) { + const isScreen = !pixels; + if (isScreen) { params = { width, height, rotate: options?.rotate, aspect: options?.aspect }; pixels = datau32; } this.create = function () { this.width = width; this.height = height; }; + // verilog rotates at reset, when the design asks for it + this.setRotate = function (rotate: number) { if (isScreen) params.rotate = rotate || undefined; }; this.setKeyboardEvents = setKeyboardEvents; this.getFrameData = function () { return datau32; }; this.getImageData = function () { return { data: datau8, width, height }; }; @@ -201,7 +205,20 @@ export class EmuCore { } } reset() { this.platform.reset(); } - loadROM(data: Uint8Array, title = 'ROM') { this.platform.loadROM(title, data); } + /** + * `data` is a ROM image, or whatever else the build produced (verilog's + * compiled unit). Some platforms (verilog) load asynchronously; await this + * to see their errors. + */ + async loadROM(data: Uint8Array | object, title = 'ROM') { await this.platform.loadROM(title, data); } + + /** + * Project files the program reads at load time (verilog's $readmem), keyed + * by the name the program uses. The IDE gets these from the open project. + */ + setFileData(files: { [path: string]: FileData }) { + this.platform.sourceFileFetch = (path) => files[path]; + } loadBIOS(data: Uint8Array, title = 'BIOS'): boolean { if (!this.platform.loadBIOS) return false; diff --git a/src/tools/8bws.ts b/src/tools/8bws.ts index 558b5d47..963a1726 100644 --- a/src/tools/8bws.ts +++ b/src/tools/8bws.ts @@ -110,7 +110,8 @@ function buildOverrides(args: Args) { // build interface Build { - rom: Uint8Array; + /** null when the output isn't a ROM image (verilog's compiled unit) */ + rom: Uint8Array | null; symbolmap: { [name: string]: number }; tool: string; platform: string; @@ -151,11 +152,14 @@ async function doBuild(args: Args, positional: string[]): Promise { const built = await compileSource(args, source, platform); const outputFile = str(args, 'output'); - if (outputFile && !checkOnly) fs.writeFileSync(outputFile, Buffer.from(built.rom)); + if (outputFile && !checkOnly) { + if (!built.rom) fail('build', `${built.tool} produces no ROM image to write`); + fs.writeFileSync(outputFile, Buffer.from(built.rom)); + } const data: any = { tool: built.tool, platform, source, - outputSize: built.rom.length, + outputSize: built.rom ? built.rom.length : null, outputFile: outputFile || null, }; if (args['symbols']) { @@ -181,11 +185,12 @@ async function saveBuildFiles(source: string, data: any): Promise { data.saveDir = saveDir; } -function romBytes(result: CompileResult): Uint8Array { +function romBytes(result: CompileResult): Uint8Array | null { const out = result.output?.code ?? result.output; if (out instanceof Uint8Array) return out; if (typeof out === 'string') return new TextEncoder().encode(out); - throw new Error('compiler produced no ROM image'); + if (out == null) throw new Error('compiler produced no output'); + return null; } //////////////////////////////////////////////////////////////////////// @@ -241,17 +246,22 @@ async function doRun(args: Args, positional: string[]): Promise { } // A source file is built first; a ROM is loaded as-is. - let romFile = input; + let romFile: string | null = input; let symbols: { [name: string]: number } = {}; let built: Awaited> | undefined; let platformId = platformArg || resolvedPlatform || ROM_PLATFORMS[path.extname(input).toLowerCase()]; if (!looksLikeROM(input)) { if (!platformId) platformId = await inferPlatform('run', input); built = await compileSource(args, input, platformId); - romFile = path.join(os.tmpdir(), '8bws-' + path.basename(input).replace(/\.\w+$/, '') + '.rom'); - fs.writeFileSync(romFile, Buffer.from(built.rom)); symbols = built.symbolmap; - note(`built ${input} with ${built.tool} -> ${romFile} (${built.rom.length} bytes)`); + if (built.rom) { + romFile = path.join(os.tmpdir(), '8bws-' + path.basename(input).replace(/\.\w+$/, '') + '.rom'); + fs.writeFileSync(romFile, Buffer.from(built.rom)); + note(`built ${input} with ${built.tool} -> ${romFile} (${built.rom.length} bytes)`); + } else { + romFile = null; // loaded straight from the build + note(`built ${input} with ${built.tool}`); + } } else if (!platformId) { fail('run', `Cannot infer a platform from '${path.basename(input)}': pass --platform`); } @@ -259,11 +269,16 @@ async function doRun(args: Args, positional: string[]): Promise { const target = await openTarget(args, platformId); // The temp file is always written as .rom, but some platforms use the title // to pick a load format -- an Atari XEX named .rom would load as a cartridge. - let romTitle = path.basename(romFile); - if (built && target.platform.getROMExtension) { - romTitle = path.basename(romFile, '.rom') + target.platform.getROMExtension(new Uint8Array(built.rom)); + if (built) target.setFileData(built.result.files || {}); + if (romFile) { + let romTitle = path.basename(romFile); + if (built && target.platform.getROMExtension) { + romTitle = path.basename(romFile, '.rom') + target.platform.getROMExtension(new Uint8Array(built.rom)); + } + await target.loadROM(new Uint8Array(fs.readFileSync(romFile)), romTitle); + } else { + await target.loadROM(built.result.output, path.basename(input)); } - target.loadROM(new Uint8Array(fs.readFileSync(romFile)), romTitle); const script = new RunScript(target); script.addSymbols(symbols); diff --git a/src/tools/testlib.ts b/src/tools/testlib.ts index b610c110..e241ca61 100644 --- a/src/tools/testlib.ts +++ b/src/tools/testlib.ts @@ -38,6 +38,8 @@ export interface CompileResult { segments?: any; params?: any; unchanged?: boolean; + /** every file the build read, by worker filename (verilog's $readmem reads them at load time) */ + files?: { [path: string]: FileData }; } let initialized = false; @@ -160,7 +162,10 @@ export async function compileSourceFile(tool: string, platform: string, filePath await initialize(); var msg = await buildSourceFileMessage(tool, platform, filePath, buildAs, opts); await handleMessage({ reset: true } as any); - return workerResultToCompileResult(await handleMessage(msg)); + var result = workerResultToCompileResult(await handleMessage(msg)); + result.files = {}; + for (var u of msg.updates) result.files[u.path] = u.data; + return result; } /** diff --git a/src/worker/workerlib.ts b/src/worker/workerlib.ts index cbdb45d8..d3cb6c61 100644 --- a/src/worker/workerlib.ts +++ b/src/worker/workerlib.ts @@ -39,6 +39,35 @@ class Blob { } } +/** Scripts loaded through the importScripts shim (Emscripten glue) */ +const importedScripts = new Set(); +const GLUE_EVENTS = ['uncaughtException', 'unhandledRejection']; +let glueHandlersIgnored = false; + +/** + * Emscripten glue running under Node adds uncaughtException and + * unhandledRejection handlers every time a tool module is instantiated. They + * rethrow or abort, which kills a long-lived host (the VS Code extension), and + * they pile up, one pair per tool per build. Drop the ones the glue adds; the + * build catches tool errors itself. + */ +function ignoreGlueProcessHandlers() { + if (glueHandlersIgnored) return; + glueHandlersIgnored = true; + const fromGlue = () => { + const stack = new Error().stack || ''; + for (const script of importedScripts) if (stack.includes(script)) return true; + return false; + }; + for (const method of ['on', 'addListener', 'prependListener'] as const) { + const original = process[method]; + (process as any)[method] = function (event: string | symbol, listener: (...args: any[]) => void) { + if (GLUE_EVENTS.includes(event as string) && fromGlue()) return this; + return original.call(this, event, listener); + }; + } +} + /** * Set up the Node.js environment to provide XMLHttpRequest, fetch, and other * browser globals that the worker build system expects. @@ -115,8 +144,10 @@ export function setupNodeEnvironment(rootDir: string = process.cwd()) { var resolved = scriptPath.replace(/^\.\.\/\.\.\//, ''); var fullPath = path.resolve(rootDir, resolved); var code = fs.readFileSync(fullPath, 'utf-8'); + importedScripts.add(fullPath); vm.runInThisContext(code, fullPath); }; + ignoreGlueProcessHandlers(); // Suppress onmessage/postMessage (not used in Node mode) emglobal.onmessage = null; diff --git a/test/cli/testcli.js b/test/cli/testcli.js index 964397b5..28bfee5f 100644 --- a/test/cli/testcli.js +++ b/test/cli/testcli.js @@ -185,6 +185,40 @@ describe('8bws CLI', function () { }); }); + // verilog builds a compiled unit, not a ROM image, and $readmem reads + // project files when it loads + describe('run --platform verilog', function () { + const MAIN = [ + 'module top(clk, reset, hsync, vsync, rgb);', + ' input clk, reset;', + ' output hsync, vsync;', + ' output [3:0] rgb;', + ' reg [7:0] rom[0:3];', + ' initial $readmemh("rom_data.hex", rom);', + ' assign hsync = 0;', + ' assign vsync = 0;', + ' assign rgb = rom[1][3:0];', + 'endmodule', + '', + ].join('\n'); + function project(withData) { + var dir = fs.mkdtempSync(path.join(os.tmpdir(), '8bws-verilog-')); + fs.writeFileSync(path.join(dir, 'main.v'), MAIN); + if (withData) fs.writeFileSync(path.join(dir, 'rom_data.hex'), '00\n0f\n00\n00\n'); + return path.join(dir, 'main.v'); + } + it('should build and run a source file that reads data with $readmem', function () { + var r = cliJSON('run', '--platform', 'verilog', '--frames', '5', project(true)); + assert.ok(r.success, r.error); + assert.equal(r.data.frames, 5); + assert.strictEqual(r.data.rom, null); + }); + it('should report a missing $readmem file', function () { + var e = cliFails('run', '--platform', 'verilog', '--frames', '5', project(false)); + assert.ok(/no file "rom_data.hex"/.test(e.stdout + e.stderr), e.stdout + e.stderr); + }); + }); + describe('run: emulator control', function () { // these all reach through the Platform to its Machine (common/devices.ts) it('should reject an unknown platform', function () { diff --git a/test/unit/testwasmlisteners.ts b/test/unit/testwasmlisteners.ts new file mode 100644 index 00000000..306c9a79 --- /dev/null +++ b/test/unit/testwasmlisteners.ts @@ -0,0 +1,39 @@ +import assert from "assert"; +import { describe, it } from "mocha"; +import { compile, initialize, preload } from "../../src/tools/testlib"; + +// Emscripten glue adds process-wide uncaughtException/unhandledRejection +// handlers each time a tool module is instantiated. Those handlers rethrow or +// abort, which kills a long-lived host (the VS Code extension), and they pile up. +describe("emscripten process listeners", function () { + this.timeout(60000); + + const counts = () => ({ + uncaught: process.listenerCount('uncaughtException'), + rejection: process.listenerCount('unhandledRejection'), + }); + + it("stay flat across builds", async function () { + await preload('ca65', 'nes'); + const before = counts(); + for (let i = 0; i < 12; i++) { + const result = await compile({ + tool: 'ca65', platform: 'nes', path: 'main.s', + code: `\t.export _main\n\t.segment "CODE"\n_main:\tlda #${i}\n\trts\n`, + }); + assert.ok(result.errors === undefined || result.errors.length === 0, JSON.stringify(result.errors)); + } + assert.deepStrictEqual(counts(), before); + }); + + it("keeps listeners the host adds itself", async function () { + await initialize(); + const mine = () => { }; + process.on('unhandledRejection', mine); + try { + assert.ok(process.listeners('unhandledRejection').includes(mine)); + } finally { + process.off('unhandledRejection', mine); + } + }); +});