vscode: fixed verilog $readmem, fixed worker listener leak, verilog in CLI

extension/src/buildcore.ts

@@ -42,6 +42,12 @@ export interface BuildOutcome {
   segments?: Segment[];
   /** every project path the build read */
   paths: string[];
+  /**
+   * The files the build read besides the main file, keyed by both worker
+   * filename and project path. The emulator reads some at load time
+   * (verilog's $readmem).
+   */
+  files?: { [path: string]: FileData };
   /** true when no inputs changed since the previous build */
   unchanged?: boolean;
 }
@@ -84,6 +90,8 @@ export class Builder {
       getToolForFilename: getTool,
     }, deps);
     var paths = [req.mainPath].concat(deps.map(d => d.path));
+    var files: { [path: string]: FileData } = {};
+    for (var d of deps) files[d.filename] = files[d.path] = d.data;
     var key = `${req.platform}/${tool}/${req.mainPath}`;
     var result: WorkerResult;
     try {
@@ -102,7 +110,7 @@ export class Builder {
       // Keep the last success even after a failed build: reverting to the
       // source that built it makes the worker say unchanged again, and that
       // output is what Run (and the next unchanged build) must use.
-      return { ...this.last.get(key), success: true, tool, paths, diagnostics: [], unchanged: true };
+      return { ...this.last.get(key), success: true, tool, paths, files, diagnostics: [], unchanged: true };
     }
     if ('errors' in result && result.errors && result.errors.length) {
       var toPath = (err: WorkerError) => (err.path && filename2path[err.path]) || err.path || req.mainPath;
@@ -116,7 +124,7 @@ export class Builder {
       // the receiver runs projectcore.processListings on them
       var r = result as any;
       var outcome: BuildOutcome = {
-        success: true, tool, paths, diagnostics: [],
+        success: true, tool, paths, files, diagnostics: [],
         output: r.output, listings: r.listings, symbolmap: r.symbolmap, segments: r.segments,
       };
       this.last.set(key, outcome);

extension/src/emuworker.ts

@@ -11,6 +11,7 @@ import { clearLastKeycodeMap, describeControls, getLastKeycodeMap, setHaltHandle
 import { ControlHint, PLATFORM_CONTROLS } from '../../src/common/controls';
 import { AudioStream, setAudioStreamFactory } from '../../src/common/audio';
 import { getRootBasePlatform } from '../../src/common/util';
+import type { FileData } from '../../src/common/workertypes';
 
 /** Audio is resampled to this rate in the worker; the webview plays it back. */
 const STREAM_RATE = 48000;
@@ -76,7 +77,8 @@ class RpcAudioStream implements AudioStream {
 }
 
 const rpc: Rpc = new Rpc(parentPort, {
-  async start(platform: string, rom: any) {
+  /** `files` are the project files the program reads at load time (BuildOutcome.files) */
+  async start(platform: string, rom: any, files?: { [path: string]: FileData }) {
     // platform modules keep global state (Javatari deletes its own start()),
     // so a worker runs one emulator; the host starts a new worker per run
     if (started) throw new Error('This emulator worker already ran a platform; start a new worker.');
@@ -87,14 +89,16 @@ const rpc: Rpc = new Rpc(parentPort, {
     await target.start();
     // machines build their keyboard handler as they start
     controls = PLATFORM_CONTROLS[getRootBasePlatform(platform)] || describeControls(getLastKeycodeMap());
-    target.loadROM(rom);
+    target.setFileData(files || {});
+    await target.loadROM(rom);
     resume();
     rpc.emit('audioReset', null);
     return status();
   },
-  loadROM(rom: any) {
+  async loadROM(rom: any, files?: { [path: string]: FileData }) {
     if (!target) throw new Error('emulator not started');
-    target.loadROM(rom);
+    target.setFileData(files || {});
+    await target.loadROM(rom);
     resume();
     rpc.emit('audioReset', null);
     return status();

extension/src/extension.ts

@@ -46,8 +46,8 @@ interface Target {
 
 /** What the emulator runs, so a rebuild of the same thing reloads it. */
 let running: Target | undefined;
-/** The ROM it runs, so an identical rebuild (a comment edit) doesn't restart it. */
-let runningRom: Uint8Array | undefined;
+/** The build it runs, so an identical rebuild (a comment edit) doesn't restart it. */
+let runningBuild: BuildOutcome | undefined;
 /** The target auto-build rebuilds: the last one changed or saved. */
 let autoTarget: Target | undefined;
 /** Diagnostics from a failed type-build, shown after a pause in typing. */
@@ -360,7 +360,7 @@ async function buildAndMaybeRun(target: Target, run: boolean, reason: BuildReaso
     return;
   }
   if (run) {
-    if (result.output) await startEmulator(target, result.output);
+    if (result.output) await startEmulator(target, result);
     else {
       output.appendLine('8bitworkshop: the build produced no ROM to run; see the output for the last build.');
       output.show(true);
@@ -407,7 +407,7 @@ async function followActiveEditor() {
   await scope.setTargetChoice(project, 'follow');
 }
 
-async function startEmulator(target: Target, rom: any) {
+async function startEmulator(target: Target, build: BuildOutcome) {
   // each run gets a fresh worker: platforms keep global state (see emuworker)
   emu?.dispose();
   emu = undefined;
@@ -436,11 +436,11 @@ async function startEmulator(target: Target, rom: any) {
   panel.setTitle(`${title} (${target.platform})`);
   panel.setMuted(muted);
   try {
-    emuStatus = await worker.call<EmuStatus>('start', target.platform, rom);
+    emuStatus = await worker.call<EmuStatus>('start', target.platform, build.output, build.files);
     worker.call('setMuted', muted);
     panel.showStatus(emuStatus);
     running = target;
-    runningRom = rom;
+    runningBuild = build;
     output.appendLine(`Running ${title} on ${target.platform}`);
   } catch (e) {
     output.appendLine(`Emulator failed to start: ${e && e.stack || e}`);
@@ -455,10 +455,15 @@ async function reloadEmulator(target: Target, reason: BuildReason, result: Build
   var mode = config(target.main).get<string>('reloadOnBuild', 'always');
   if (mode === 'never' || (mode === 'onSave' && reason === 'type')) return;
   if (emuStatus.platform !== target.platform) {
-    await startEmulator(target, result.output);
-  } else if (!sameBytes(runningRom, result.output)) {
-    emuStatus = await (await getEmu(target.platform)).call<EmuStatus>('loadROM', result.output);
-    runningRom = result.output;
+    await startEmulator(target, result);
+  } else if (!sameBuild(runningBuild, result)) {
+    runningBuild = result;
+    try {
+      emuStatus = await (await getEmu(target.platform)).call<EmuStatus>('loadROM', result.output, result.files);
+    } catch (e) {
+      output.appendLine(`Emulator failed to load: ${e && e.message || e}`);
+      output.show(true);
+    }
   }
 }
 
@@ -471,10 +476,27 @@ function setMuted(m: boolean) {
   vscode.commands.executeCommand('setContext', '8bitworkshop.muted', m);
 }
 
-function sameBytes(a: Uint8Array | undefined, b: Uint8Array | undefined): boolean {
-  if (!a || !b || a.length !== b.length) return false;
-  for (var i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
-  return true;
+/** Same output and same load-time files (see BuildOutcome.files). */
+function sameBuild(a: BuildOutcome | undefined, b: BuildOutcome): boolean {
+  if (!a || !sameData(a.output, b.output)) return false;
+  var af = a.files || {}, bf = b.files || {};
+  var keys = Object.keys(bf);
+  return keys.length === Object.keys(af).length && keys.every(k => sameData(af[k], bf[k]));
+}
+
+/** Compares ROM bytes, file text, or (verilog) the compiled unit. */
+function sameData(a: any, b: any): boolean {
+  if (a instanceof Uint8Array && b instanceof Uint8Array) {
+    if (a.length !== b.length) return false;
+    for (var i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
+    return true;
+  }
+  if (typeof a === 'string' || typeof b === 'string') return a === b;
+  try {
+    return !!a && !!b && JSON.stringify(a) === JSON.stringify(b);
+  } catch (e) {
+    return false;  // not JSON (cycles): assume it changed
+  }
 }
 
 /**

extension/test/emuworker.test.ts

@@ -5,6 +5,7 @@ import * as path from 'path';
 import { Worker } from 'worker_threads';
 import { Rpc } from '../src/rpc';
 import { findRootDir } from '../src/projectinfo';
+import { Builder, ProjectFileProvider } from '../src/buildcore';
 import type { AudioChunk, EmuStatus, FrameEvent } from '../src/emuworker';
 
 // Drives out/emuworker.js in a worker thread, the way the extension host does.
@@ -133,4 +134,53 @@ describe('extension emuworker', function () {
     await rpc.call('setMuted', false);
     await waitForAudio(1);
   });
+
+  // $readmem reads project files at load time, so they travel with the ROM
+  describe('verilog $readmem', function () {
+    const MAIN = [
+      'module top(clk, reset, hsync, vsync, rgb);',
+      '  input clk, reset;',
+      '  output hsync, vsync;',
+      '  output [3:0] rgb;',
+      '  reg [7:0] rom[0:3];',
+      '  initial $readmemh("rom_data.hex", rom);',
+      '  assign hsync = 0;',
+      '  assign vsync = 0;',
+      '  assign rgb = rom[1][3:0];',
+      'endmodule',
+      '',
+    ].join('\n');
+    var built: Awaited<ReturnType<Builder['build']>>;
+
+    before(async function () {
+      var enc = new TextEncoder();
+      var files: { [path: string]: string } = { 'main.v': MAIN, 'rom_data.hex': '00\n0f\n00\n00\n' };
+      var read = async (rel: string) => rel in files ? enc.encode(files[rel]) : null;
+      built = await new Builder(ROOT).build({
+        platform: 'verilog', mainPath: 'main.v', mainText: MAIN,
+        files: new ProjectFileProvider(read, ROOT, 'verilog'),
+      });
+      assert.deepEqual(built.diagnostics, []);
+    });
+
+    it('builds with the data file attached', function () {
+      assert.equal(built.files['rom_data.hex'], '00\n0f\n00\n00\n');
+    });
+
+    it('runs with the data file', async function () {
+      var s = await rpc.call<EmuStatus>('start', 'verilog', built.output, built.files);
+      assert.equal(s.state, 'running');
+      await waitForFrames(3);
+      assert.ok(!statuses.some(s => s && s.state === 'halted'), JSON.stringify(statuses));
+    });
+
+    it('reports a missing data file without killing the worker', async function () {
+      var exited = false;
+      worker.on('exit', () => { exited = true; });
+      await assert.rejects(rpc.call('start', 'verilog', built.output, {}), /no file "rom_data.hex"/);
+      await new Promise(r => setTimeout(r, 200));
+      assert.ok(!exited, 'worker exited');
+      await rpc.call('status');
+    });
+  });
 });

src/common/emucore.ts

@@ -20,6 +20,7 @@ import {
   CpuState, DisasmLine, EmuState, Machine, Platform, hasProbe, isDebuggable,
 } from "./baseplatform";
 import { ProbeAll, SampledAudioParams, TrapCondition } from "./devices";
+import { FileData } from "./workertypes";
 import { disassemble6502 } from "./cpu/disasm6502";
 import { disassembleZ80 } from "./cpu/disasmz80";
 import { disassembleSM83 } from "./cpu/disasmSM83";
@@ -107,11 +108,14 @@ function installHeadlessVideo() {
     const datau8 = new Uint8Array(buffer);
     const datau32 = new Uint32Array(buffer);
     // the first one is the screen; later ones are debug views (nes nametables)
-    if (!pixels) {
+    const isScreen = !pixels;
+    if (isScreen) {
       params = { width, height, rotate: options?.rotate, aspect: options?.aspect };
       pixels = datau32;
     }
     this.create = function () { this.width = width; this.height = height; };
+    // verilog rotates at reset, when the design asks for it
+    this.setRotate = function (rotate: number) { if (isScreen) params.rotate = rotate || undefined; };
     this.setKeyboardEvents = setKeyboardEvents;
     this.getFrameData = function () { return datau32; };
     this.getImageData = function () { return { data: datau8, width, height }; };
@@ -201,7 +205,20 @@ export class EmuCore {
     }
   }
   reset() { this.platform.reset(); }
-  loadROM(data: Uint8Array, title = 'ROM') { this.platform.loadROM(title, data); }
+  /**
+   * `data` is a ROM image, or whatever else the build produced (verilog's
+   * compiled unit). Some platforms (verilog) load asynchronously; await this
+   * to see their errors.
+   */
+  async loadROM(data: Uint8Array | object, title = 'ROM') { await this.platform.loadROM(title, data); }
+
+  /**
+   * Project files the program reads at load time (verilog's $readmem), keyed
+   * by the name the program uses. The IDE gets these from the open project.
+   */
+  setFileData(files: { [path: string]: FileData }) {
+    this.platform.sourceFileFetch = (path) => files[path];
+  }
 
   loadBIOS(data: Uint8Array, title = 'BIOS'): boolean {
     if (!this.platform.loadBIOS) return false;

src/tools/8bws.ts

@@ -110,7 +110,8 @@ function buildOverrides(args: Args) {
 // build
 
 interface Build {
-  rom: Uint8Array;
+  /** null when the output isn't a ROM image (verilog's compiled unit) */
+  rom: Uint8Array | null;
   symbolmap: { [name: string]: number };
   tool: string;
   platform: string;
@@ -151,11 +152,14 @@ async function doBuild(args: Args, positional: string[]): Promise<void> {
   const built = await compileSource(args, source, platform);
 
   const outputFile = str(args, 'output');
-  if (outputFile && !checkOnly) fs.writeFileSync(outputFile, Buffer.from(built.rom));
+  if (outputFile && !checkOnly) {
+    if (!built.rom) fail('build', `${built.tool} produces no ROM image to write`);
+    fs.writeFileSync(outputFile, Buffer.from(built.rom));
+  }
 
   const data: any = {
     tool: built.tool, platform, source,
-    outputSize: built.rom.length,
+    outputSize: built.rom ? built.rom.length : null,
     outputFile: outputFile || null,
   };
   if (args['symbols']) {
@@ -181,11 +185,12 @@ async function saveBuildFiles(source: string, data: any): Promise<void> {
   data.saveDir = saveDir;
 }
 
-function romBytes(result: CompileResult): Uint8Array {
+function romBytes(result: CompileResult): Uint8Array | null {
   const out = result.output?.code ?? result.output;
   if (out instanceof Uint8Array) return out;
   if (typeof out === 'string') return new TextEncoder().encode(out);
-  throw new Error('compiler produced no ROM image');
+  if (out == null) throw new Error('compiler produced no output');
+  return null;
 }
 
 ////////////////////////////////////////////////////////////////////////
@@ -241,17 +246,22 @@ async function doRun(args: Args, positional: string[]): Promise<void> {
   }
 
   // A source file is built first; a ROM is loaded as-is.
-  let romFile = input;
+  let romFile: string | null = input;
   let symbols: { [name: string]: number } = {};
   let built: Awaited<ReturnType<typeof compileSource>> | undefined;
   let platformId = platformArg || resolvedPlatform || ROM_PLATFORMS[path.extname(input).toLowerCase()];
   if (!looksLikeROM(input)) {
     if (!platformId) platformId = await inferPlatform('run', input);
     built = await compileSource(args, input, platformId);
-    romFile = path.join(os.tmpdir(), '8bws-' + path.basename(input).replace(/\.\w+$/, '') + '.rom');
-    fs.writeFileSync(romFile, Buffer.from(built.rom));
     symbols = built.symbolmap;
-    note(`built ${input} with ${built.tool} -> ${romFile} (${built.rom.length} bytes)`);
+    if (built.rom) {
+      romFile = path.join(os.tmpdir(), '8bws-' + path.basename(input).replace(/\.\w+$/, '') + '.rom');
+      fs.writeFileSync(romFile, Buffer.from(built.rom));
+      note(`built ${input} with ${built.tool} -> ${romFile} (${built.rom.length} bytes)`);
+    } else {
+      romFile = null;  // loaded straight from the build
+      note(`built ${input} with ${built.tool}`);
+    }
   } else if (!platformId) {
     fail('run', `Cannot infer a platform from '${path.basename(input)}': pass --platform`);
   }
@@ -259,11 +269,16 @@ async function doRun(args: Args, positional: string[]): Promise<void> {
   const target = await openTarget(args, platformId);
   // The temp file is always written as .rom, but some platforms use the title
   // to pick a load format -- an Atari XEX named .rom would load as a cartridge.
-  let romTitle = path.basename(romFile);
-  if (built && target.platform.getROMExtension) {
-    romTitle = path.basename(romFile, '.rom') + target.platform.getROMExtension(new Uint8Array(built.rom));
+  if (built) target.setFileData(built.result.files || {});
+  if (romFile) {
+    let romTitle = path.basename(romFile);
+    if (built && target.platform.getROMExtension) {
+      romTitle = path.basename(romFile, '.rom') + target.platform.getROMExtension(new Uint8Array(built.rom));
+    }
+    await target.loadROM(new Uint8Array(fs.readFileSync(romFile)), romTitle);
+  } else {
+    await target.loadROM(built.result.output, path.basename(input));
   }
-  target.loadROM(new Uint8Array(fs.readFileSync(romFile)), romTitle);
 
   const script = new RunScript(target);
   script.addSymbols(symbols);

src/tools/testlib.ts

@@ -38,6 +38,8 @@ export interface CompileResult {
   segments?: any;
   params?: any;
   unchanged?: boolean;
+  /** every file the build read, by worker filename (verilog's $readmem reads them at load time) */
+  files?: { [path: string]: FileData };
 }
 
 let initialized = false;
@@ -160,7 +162,10 @@ export async function compileSourceFile(tool: string, platform: string, filePath
   await initialize();
   var msg = await buildSourceFileMessage(tool, platform, filePath, buildAs, opts);
   await handleMessage({ reset: true } as any);
-  return workerResultToCompileResult(await handleMessage(msg));
+  var result = workerResultToCompileResult(await handleMessage(msg));
+  result.files = {};
+  for (var u of msg.updates) result.files[u.path] = u.data;
+  return result;
 }
 
 /**

src/worker/workerlib.ts

@@ -39,6 +39,35 @@ class Blob {
   }
 }
 
+/** Scripts loaded through the importScripts shim (Emscripten glue) */
+const importedScripts = new Set<string>();
+const GLUE_EVENTS = ['uncaughtException', 'unhandledRejection'];
+let glueHandlersIgnored = false;
+
+/**
+ * Emscripten glue running under Node adds uncaughtException and
+ * unhandledRejection handlers every time a tool module is instantiated. They
+ * rethrow or abort, which kills a long-lived host (the VS Code extension), and
+ * they pile up, one pair per tool per build. Drop the ones the glue adds; the
+ * build catches tool errors itself.
+ */
+function ignoreGlueProcessHandlers() {
+  if (glueHandlersIgnored) return;
+  glueHandlersIgnored = true;
+  const fromGlue = () => {
+    const stack = new Error().stack || '';
+    for (const script of importedScripts) if (stack.includes(script)) return true;
+    return false;
+  };
+  for (const method of ['on', 'addListener', 'prependListener'] as const) {
+    const original = process[method];
+    (process as any)[method] = function (event: string | symbol, listener: (...args: any[]) => void) {
+      if (GLUE_EVENTS.includes(event as string) && fromGlue()) return this;
+      return original.call(this, event, listener);
+    };
+  }
+}
+
 /**
  * Set up the Node.js environment to provide XMLHttpRequest, fetch, and other
  * browser globals that the worker build system expects.
@@ -115,8 +144,10 @@ export function setupNodeEnvironment(rootDir: string = process.cwd()) {
     var resolved = scriptPath.replace(/^\.\.\/\.\.\//, '');
     var fullPath = path.resolve(rootDir, resolved);
     var code = fs.readFileSync(fullPath, 'utf-8');
+    importedScripts.add(fullPath);
     vm.runInThisContext(code, fullPath);
   };
+  ignoreGlueProcessHandlers();
 
   // Suppress onmessage/postMessage (not used in Node mode)
   emglobal.onmessage = null;

test/cli/testcli.js

@@ -185,6 +185,40 @@ describe('8bws CLI', function () {
         });
     });
 
+    // verilog builds a compiled unit, not a ROM image, and $readmem reads
+    // project files when it loads
+    describe('run --platform verilog', function () {
+        const MAIN = [
+            'module top(clk, reset, hsync, vsync, rgb);',
+            '  input clk, reset;',
+            '  output hsync, vsync;',
+            '  output [3:0] rgb;',
+            '  reg [7:0] rom[0:3];',
+            '  initial $readmemh("rom_data.hex", rom);',
+            '  assign hsync = 0;',
+            '  assign vsync = 0;',
+            '  assign rgb = rom[1][3:0];',
+            'endmodule',
+            '',
+        ].join('\n');
+        function project(withData) {
+            var dir = fs.mkdtempSync(path.join(os.tmpdir(), '8bws-verilog-'));
+            fs.writeFileSync(path.join(dir, 'main.v'), MAIN);
+            if (withData) fs.writeFileSync(path.join(dir, 'rom_data.hex'), '00\n0f\n00\n00\n');
+            return path.join(dir, 'main.v');
+        }
+        it('should build and run a source file that reads data with $readmem', function () {
+            var r = cliJSON('run', '--platform', 'verilog', '--frames', '5', project(true));
+            assert.ok(r.success, r.error);
+            assert.equal(r.data.frames, 5);
+            assert.strictEqual(r.data.rom, null);
+        });
+        it('should report a missing $readmem file', function () {
+            var e = cliFails('run', '--platform', 'verilog', '--frames', '5', project(false));
+            assert.ok(/no file "rom_data.hex"/.test(e.stdout + e.stderr), e.stdout + e.stderr);
+        });
+    });
+
     describe('run: emulator control', function () {
         // these all reach through the Platform to its Machine (common/devices.ts)
         it('should reject an unknown platform', function () {

test/unit/testwasmlisteners.ts

@@ -0,0 +1,39 @@
+import assert from "assert";
+import { describe, it } from "mocha";
+import { compile, initialize, preload } from "../../src/tools/testlib";
+
+// Emscripten glue adds process-wide uncaughtException/unhandledRejection
+// handlers each time a tool module is instantiated. Those handlers rethrow or
+// abort, which kills a long-lived host (the VS Code extension), and they pile up.
+describe("emscripten process listeners", function () {
+  this.timeout(60000);
+
+  const counts = () => ({
+    uncaught: process.listenerCount('uncaughtException'),
+    rejection: process.listenerCount('unhandledRejection'),
+  });
+
+  it("stay flat across builds", async function () {
+    await preload('ca65', 'nes');
+    const before = counts();
+    for (let i = 0; i < 12; i++) {
+      const result = await compile({
+        tool: 'ca65', platform: 'nes', path: 'main.s',
+        code: `\t.export _main\n\t.segment "CODE"\n_main:\tlda #${i}\n\trts\n`,
+      });
+      assert.ok(result.errors === undefined || result.errors.length === 0, JSON.stringify(result.errors));
+    }
+    assert.deepStrictEqual(counts(), before);
+  });
+
+  it("keeps listeners the host adds itself", async function () {
+    await initialize();
+    const mine = () => { };
+    process.on('unhandledRejection', mine);
+    try {
+      assert.ok(process.listeners('unhandledRejection').includes(mine));
+    } finally {
+      process.off('unhandledRejection', mine);
+    }
+  });
+});