vscode: extension bootstrap with in-process builds and diagnostics

extension/.gitignore

@@ -0,0 +1,3 @@
+out/
+node_modules/
+*.vsix

extension/.vscode/launch.json

@@ -0,0 +1,14 @@
+{
+  // Open the extension/ folder in VS Code and press F5.
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Run Extension",
+      "type": "extensionHost",
+      "request": "launch",
+      "args": ["--extensionDevelopmentPath=${workspaceFolder}", "${workspaceFolder}/../presets/nes"],
+      "outFiles": ["${workspaceFolder}/out/**/*.js"],
+      "preLaunchTask": "build"
+    }
+  ]
+}

extension/.vscode/tasks.json

@@ -0,0 +1,12 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "build",
+      "type": "npm",
+      "script": "build",
+      "group": "build",
+      "problemMatcher": []
+    }
+  ]
+}

extension/package-lock.json

@@ -0,0 +1,26 @@
+{
+  "name": "8bitworkshop",
+  "version": "0.1.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "8bitworkshop",
+      "version": "0.1.0",
+      "license": "GPL-3.0",
+      "devDependencies": {
+        "@types/vscode": "~1.90.0"
+      },
+      "engines": {
+        "vscode": "^1.90.0"
+      }
+    },
+    "node_modules/@types/vscode": {
+      "version": "1.90.0",
+      "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.90.0.tgz",
+      "integrity": "sha512-oT+ZJL7qHS9Z8bs0+WKf/kQ27qWYR3trsXpq46YDjFqBsMLG4ygGGjPaJ2tyrH0wJzjOEmDyg9PDJBBhWg9pkQ==",
+      "dev": true,
+      "license": "MIT"
+    }
+  }
+}

extension/package.json

@@ -0,0 +1,76 @@
+{
+  "name": "8bitworkshop",
+  "displayName": "8bitworkshop",
+  "description": "Build and run retro 8-bit programs with the 8bitworkshop toolchains and emulators",
+  "version": "0.1.0",
+  "publisher": "8bitworkshop",
+  "license": "GPL-3.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/sehugg/8bitworkshop.git"
+  },
+  "engines": {
+    "vscode": "^1.90.0"
+  },
+  "categories": [
+    "Programming Languages",
+    "Debuggers"
+  ],
+  "main": "./out/extension.js",
+  "activationEvents": [
+    "onStartupFinished"
+  ],
+  "contributes": {
+    "commands": [
+      {
+        "command": "8bitworkshop.build",
+        "title": "Build",
+        "category": "8bitworkshop"
+      },
+      {
+        "command": "8bitworkshop.selectPlatform",
+        "title": "Select Platform",
+        "category": "8bitworkshop"
+      }
+    ],
+    "configuration": {
+      "title": "8bitworkshop",
+      "properties": {
+        "8bitworkshop.platform": {
+          "type": "string",
+          "default": "",
+          "description": "Target platform id (e.g. c64, nes, vcs, gb)."
+        },
+        "8bitworkshop.mainFile": {
+          "type": "string",
+          "default": "",
+          "description": "Main source file, relative to the workspace folder. Defaults to the active editor's file."
+        },
+        "8bitworkshop.tool": {
+          "type": "string",
+          "default": "",
+          "description": "Tool for the main file. Defaults to the platform's tool for the file extension."
+        },
+        "8bitworkshop.buildOnSave": {
+          "type": "boolean",
+          "default": true,
+          "description": "Build when a source file or one of its dependencies is saved."
+        },
+        "8bitworkshop.toolchainPath": {
+          "type": "string",
+          "default": "",
+          "description": "Directory containing src/worker (toolchain assets). Defaults to the extension's own copy."
+        }
+      }
+    }
+  },
+  "scripts": {
+    "build": "node scripts/build.mjs",
+    "watch": "node scripts/build.mjs --watch",
+    "typecheck": "tsc --noEmit -p .",
+    "test": "node scripts/build.mjs && mocha --timeout 60000 out/test/*.test.js"
+  },
+  "devDependencies": {
+    "@types/vscode": "~1.90.0"
+  }
+}

extension/scripts/build.mjs

@@ -0,0 +1,40 @@
+// Bundles the extension with esbuild (from the repo's node_modules).
+//
+//   node scripts/build.mjs           # one build
+//   node scripts/build.mjs --watch   # rebuild on change
+//
+// extension.js stays small; it loads buildcore.js on the first build.
+import esbuild from 'esbuild';
+import { readdirSync } from 'fs';
+import { fileURLToPath } from 'url';
+import path from 'path';
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const watch = process.argv.includes('--watch');
+const tests = readdirSync(path.join(root, 'test'))
+  .filter((f) => f.endsWith('.test.ts'))
+  .map((f) => [`test/${f.slice(0, -3)}`, `test/${f}`]);
+
+const ctx = await esbuild.context({
+  absWorkingDir: root,
+  entryPoints: {
+    extension: 'src/extension.ts',
+    buildcore: 'src/buildcore.ts',
+    ...Object.fromEntries(tests),
+  },
+  outdir: 'out',
+  bundle: true,
+  platform: 'node',
+  format: 'cjs',
+  target: 'node20',
+  sourcemap: true,
+  external: ['vscode'],
+  logLevel: 'warning',
+});
+if (watch) {
+  await ctx.watch();
+  console.error('[esbuild] watching extension');
+} else {
+  await ctx.rebuild();
+  await ctx.dispose();
+}

extension/src/buildcore.ts

@@ -0,0 +1,167 @@
+
+// buildcore - compiles a project in-process with the worker build system.
+// No vscode imports, so tests can run it under plain Node.
+
+import * as fs from 'fs';
+import * as path from 'path';
+import type { CodeListingMap, FileData, Segment, WorkerError, WorkerResult } from "../../src/common/workertypes";
+import { getBasePlatform, isProbablyBinary } from "../../src/common/util";
+import { getToolForPlatform } from "../../src/common/toolselect";
+import { TOOL_META } from "../../src/common/toolmeta";
+import { FileProvider, buildWorkerMessage, processListings, resolveDependencies } from "../../src/common/projectcore";
+import { setupNodeEnvironment, handleMessage } from "../../src/worker/workerlib";
+import { PLATFORM_PARAMS } from "../../src/worker/platforms";
+
+export type { FileProvider };
+
+export interface BuildRequest {
+  platform: string;
+  /** main file, relative to the project root (posix separators) */
+  mainPath: string;
+  mainText: string;
+  files: FileProvider;
+  /** overrides the tool for the main file */
+  tool?: string;
+}
+
+export interface BuildDiagnostic {
+  /** project-relative path */
+  path: string;
+  /** 1-based line number (0 = no line) */
+  line: number;
+  msg: string;
+}
+
+export interface BuildOutcome {
+  success: boolean;
+  tool: string;
+  output?: Uint8Array;
+  diagnostics: BuildDiagnostic[];
+  listings?: CodeListingMap;
+  symbolmap?: { [sym: string]: number };
+  segments?: Segment[];
+  /** every project path the build read */
+  paths: string[];
+  /** true when no inputs changed since the previous build */
+  unchanged?: boolean;
+}
+
+/**
+ * Runs builds against the toolchain assets under `rootDir/src/worker`.
+ * The worker store is global, so builds run one at a time.
+ */
+export class Builder {
+  private queue: Promise<any> = Promise.resolve();
+  private initialized = false;
+  private preloaded = new Set<string>();
+
+  constructor(readonly rootDir: string) { }
+
+  build(req: BuildRequest): Promise<BuildOutcome> {
+    var next = this.queue.then(() => this.doBuild(req));
+    this.queue = next.catch(() => { });
+    return next;
+  }
+
+  private async doBuild(req: BuildRequest): Promise<BuildOutcome> {
+    if (!this.initialized) {
+      setupNodeEnvironment(this.rootDir);
+      this.initialized = true;
+    }
+    var getTool = (fn: string) => (req.tool && fn === req.mainPath) ? req.tool : getToolForPlatform(req.platform, fn);
+    var tool = getTool(req.mainPath);
+    var deps = await resolveDependencies(req.files, req.mainPath, req.mainText, req.platform, getTool);
+    var { msg, filename2path, preloads } = buildWorkerMessage({
+      mainPath: req.mainPath,
+      mainData: req.mainText,
+      platformId: req.platform,
+      getToolForFilename: getTool,
+    }, deps);
+    var paths = [req.mainPath].concat(deps.map(d => d.path));
+    var result: WorkerResult;
+    try {
+      for (var t of preloads) {
+        if (!this.preloaded.has(t + "/" + req.platform)) {
+          await handleMessage({ preload: t, platform: req.platform } as any);
+          this.preloaded.add(t + "/" + req.platform);
+        }
+      }
+      result = await handleMessage(msg);
+    } catch (e) {
+      return { success: false, tool, paths, diagnostics: [{ path: req.mainPath, line: 0, msg: String(e && e.message || e) }] };
+    }
+    if (!result || ('unchanged' in result && result.unchanged)) {
+      return { success: true, tool, paths, diagnostics: [], unchanged: true };
+    }
+    if ('errors' in result && result.errors && result.errors.length) {
+      var toPath = (err: WorkerError) => (err.path && filename2path[err.path]) || err.path || req.mainPath;
+      return {
+        success: false, tool, paths,
+        diagnostics: result.errors.map(err => ({ path: toPath(err), line: err.line || 0, msg: err.msg })),
+      };
+    }
+    if ('output' in result) {
+      var r = result as any;
+      if (r.listings) processListings(r.listings);
+      return {
+        success: true, tool, paths, diagnostics: [],
+        output: r.output, listings: r.listings, symbolmap: r.symbolmap, segments: r.segments,
+      };
+    }
+    return { success: false, tool, paths, diagnostics: [{ path: req.mainPath, line: 0, msg: 'Unknown build result' }] };
+  }
+}
+
+/**
+ * Reads project files through `read`, then falls back to the bundled
+ * presets/<base platform> directory (for shared headers and libraries).
+ */
+export class ProjectFileProvider implements FileProvider {
+  constructor(
+    readonly read: (relpath: string) => Promise<Uint8Array | null>,
+    readonly rootDir: string,
+    readonly platform: string) { }
+
+  async readFile(relpath: string): Promise<FileData | null> {
+    var data = await this.read(relpath);
+    if (data == null) {
+      var p = path.resolve(this.rootDir, 'presets', getBasePlatform(this.platform), relpath);
+      try {
+        if (fs.statSync(p).isFile()) data = new Uint8Array(fs.readFileSync(p));
+      } catch (e) {
+        return null;
+      }
+    }
+    if (data == null) return null;
+    return isProbablyBinary(relpath) ? data : new TextDecoder().decode(data);
+  }
+}
+
+export function listPlatforms(): string[] {
+  return Object.keys(PLATFORM_PARAMS).sort();
+}
+
+var sourceExtensions: Set<string>;
+
+/** True if some tool consumes files with this name's extension. */
+export function isSourceFile(fn: string): boolean {
+  if (!sourceExtensions) {
+    sourceExtensions = new Set(['.asm', '.a', '.inc', '.h', '.s', '.c', '.bas']);
+    for (var id in TOOL_META)
+      for (var ext of TOOL_META[id].extensions || [])
+        sourceExtensions.add(ext.toLowerCase());
+  }
+  return sourceExtensions.has(path.extname(fn).toLowerCase());
+}
+
+/** The asset root: a directory with src/worker, searched upward from `start`. */
+export function findRootDir(start: string): string | null {
+  var dir = path.resolve(start);
+  for (var i = 0; i < 4; i++) {
+    if (fs.existsSync(path.join(dir, 'src', 'worker', 'wasm'))) return dir;
+    var parent = path.dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  return null;
+}

extension/src/extension.ts

@@ -0,0 +1,192 @@
+
+// 8bitworkshop VS Code extension entry point.
+// Keep this module light: the build system loads on the first build.
+
+import * as vscode from 'vscode';
+import * as path from 'path';
+import type * as BuildCore from './buildcore';
+
+const CONFIG = '8bitworkshop';
+
+let buildcore: typeof BuildCore;
+let builder: BuildCore.Builder;
+let output: vscode.OutputChannel;
+let diagnostics: vscode.DiagnosticCollection;
+let status: vscode.StatusBarItem;
+let lastMain: vscode.Uri | undefined;
+let lastPaths = new Set<string>();   // fsPaths the last build read
+let lastBuild: BuildCore.BuildOutcome | undefined;
+let pending: NodeJS.Timeout | undefined;
+
+export function activate(context: vscode.ExtensionContext) {
+  output = vscode.window.createOutputChannel('8bitworkshop');
+  diagnostics = vscode.languages.createDiagnosticCollection('8bitworkshop');
+  status = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Left);
+  status.command = '8bitworkshop.selectPlatform';
+  context.subscriptions.push(output, diagnostics, status);
+  updateStatus();
+
+  context.subscriptions.push(
+    vscode.commands.registerCommand('8bitworkshop.build', () => buildCommand(context)),
+    vscode.commands.registerCommand('8bitworkshop.selectPlatform', () => selectPlatform(context)),
+    vscode.workspace.onDidSaveTextDocument(doc => onSave(context, doc)),
+    vscode.workspace.onDidChangeConfiguration(e => {
+      if (e.affectsConfiguration(CONFIG)) updateStatus();
+    }),
+  );
+}
+
+export function deactivate() {
+  if (pending) clearTimeout(pending);
+}
+
+function config() {
+  return vscode.workspace.getConfiguration(CONFIG);
+}
+
+function updateStatus() {
+  var platform = config().get<string>('platform');
+  status.text = '$(chip) ' + (platform || 'No platform');
+  status.tooltip = 'Select 8bitworkshop platform';
+  status.show();
+}
+
+function loadBuildCore(context: vscode.ExtensionContext): typeof BuildCore {
+  if (!buildcore) {
+    // a computed path keeps esbuild from inlining it
+    buildcore = require(path.join(__dirname, 'buildcore'));
+    var configured = config().get<string>('toolchainPath');
+    var root = configured || buildcore.findRootDir(context.extensionPath);
+    if (!root) throw new Error('Cannot find toolchain assets (src/worker). Set 8bitworkshop.toolchainPath.');
+    output.appendLine(`Toolchain root: ${root}`);
+    builder = new buildcore.Builder(root);
+  }
+  return buildcore;
+}
+
+async function selectPlatform(context: vscode.ExtensionContext): Promise<string | undefined> {
+  var core = loadBuildCore(context);
+  var picked = await vscode.window.showQuickPick(core.listPlatforms(), { placeHolder: 'Target platform' });
+  if (!picked) return undefined;
+  var target = vscode.workspace.workspaceFolders ? vscode.ConfigurationTarget.Workspace : vscode.ConfigurationTarget.Global;
+  await config().update('platform', picked, target);
+  return picked;
+}
+
+async function buildCommand(context: vscode.ExtensionContext) {
+  var platform = config().get<string>('platform') || await selectPlatform(context);
+  if (!platform) return;
+  var main = findMainFile(context);
+  if (!main) {
+    vscode.window.showWarningMessage('8bitworkshop: open a source file or set 8bitworkshop.mainFile.');
+    return;
+  }
+  var result = await runBuild(context, platform, main);
+  if (result && !result.success) output.show(true);
+}
+
+function onSave(context: vscode.ExtensionContext, doc: vscode.TextDocument) {
+  var platform = config().get<string>('platform');
+  if (!platform || !config().get<boolean>('buildOnSave')) return;
+  if (doc.uri.scheme === 'untitled') return;
+  var isDep = lastPaths.has(doc.uri.fsPath);
+  if (!isDep && !loadBuildCore(context).isSourceFile(doc.fileName)) return;
+  if (pending) clearTimeout(pending);
+  pending = setTimeout(() => {
+    pending = undefined;
+    // a saved dependency rebuilds the last main file
+    var main = (isDep && lastMain) || findMainFile(context);
+    if (main) runBuild(context, platform!, main);
+  }, 300);
+}
+
+/** The setting, else the active editor's file, else the last main file. */
+function findMainFile(context: vscode.ExtensionContext): vscode.Uri | undefined {
+  var setting = config().get<string>('mainFile');
+  var folder = vscode.workspace.workspaceFolders?.[0];
+  if (setting) {
+    return path.isAbsolute(setting) || !folder ? vscode.Uri.file(setting) : vscode.Uri.joinPath(folder.uri, setting);
+  }
+  var editor = vscode.window.activeTextEditor;
+  if (editor && loadBuildCore(context).isSourceFile(editor.document.fileName)) {
+    return editor.document.uri;
+  }
+  return lastMain;
+}
+
+async function runBuild(context: vscode.ExtensionContext, platform: string, main: vscode.Uri) {
+  var core = loadBuildCore(context);
+  // paths are relative to the main file's directory, like the IDE's project root
+  var rootUri = vscode.Uri.joinPath(main, '..');
+  var toUri = (rel: string) => vscode.Uri.joinPath(rootUri, rel);
+  var read = async (rel: string): Promise<Uint8Array | null> => {
+    var uri = toUri(rel);
+    // unsaved editor contents win over disk
+    var doc = vscode.workspace.textDocuments.find(d => d.uri.toString() === uri.toString());
+    if (doc) return new TextEncoder().encode(doc.getText());
+    try {
+      return await vscode.workspace.fs.readFile(uri);
+    } catch (e) {
+      return null;
+    }
+  };
+  var mainPath = path.posix.basename(main.path);
+  var mainData = await read(mainPath);
+  if (mainData == null) {
+    vscode.window.showErrorMessage(`8bitworkshop: cannot read ${main.fsPath}`);
+    return;
+  }
+  var t0 = Date.now();
+  status.text = '$(sync~spin) ' + platform;
+  var result: BuildCore.BuildOutcome;
+  try {
+    result = await builder.build({
+      platform,
+      mainPath,
+      mainText: new TextDecoder().decode(mainData),
+      files: new core.ProjectFileProvider(read, builder.rootDir, platform),
+      tool: config().get<string>('tool') || undefined,
+    });
+  } catch (e) {
+    output.appendLine(`Build crashed: ${e && e.stack || e}`);
+    output.show(true);
+    return;
+  } finally {
+    updateStatus();
+  }
+  lastMain = main;
+  lastPaths = new Set(result.paths.map(p => toUri(p).fsPath));
+  if (result.unchanged) return result;
+  lastBuild = result;
+  showDiagnostics(result, toUri);
+  var elapsed = Date.now() - t0;
+  if (result.success) {
+    var size = result.output?.length ?? 0;
+    output.appendLine(`${mainPath}: built with ${result.tool} for ${platform}, ${size} bytes (${elapsed} ms)`);
+  } else {
+    output.appendLine(`${mainPath}: ${result.diagnostics.length} error(s) from ${result.tool} (${elapsed} ms)`);
+    for (var d of result.diagnostics) output.appendLine(`  ${d.path}:${d.line}: ${d.msg}`);
+  }
+  return result;
+}
+
+function showDiagnostics(result: BuildCore.BuildOutcome, toUri: (rel: string) => vscode.Uri) {
+  diagnostics.clear();
+  var byFile = new Map<string, { uri: vscode.Uri, diags: vscode.Diagnostic[] }>();
+  for (var d of result.diagnostics) {
+    var uri = toUri(d.path);
+    var key = uri.toString();
+    if (!byFile.has(key)) byFile.set(key, { uri, diags: [] });
+    var line = Math.max(0, d.line - 1);
+    var range = new vscode.Range(line, 0, line, Number.MAX_SAFE_INTEGER);
+    var diag = new vscode.Diagnostic(range, d.msg, vscode.DiagnosticSeverity.Error);
+    diag.source = result.tool;
+    byFile.get(key)!.diags.push(diag);
+  }
+  for (var { uri, diags } of byFile.values()) diagnostics.set(uri, diags);
+}
+
+/** The most recent successful or failed build, for the emulator (milestone 2). */
+export function getLastBuild() {
+  return lastBuild;
+}

extension/test/buildcore.test.ts

@@ -0,0 +1,73 @@
+
+import * as assert from 'assert';
+import * as path from 'path';
+import { Builder, ProjectFileProvider, findRootDir, isSourceFile } from '../src/buildcore';
+
+// Mocha runs from extension/, not the repo root, so this also checks that
+// the toolchain assets load from the configured root rather than the cwd.
+const ROOT = findRootDir(__dirname);
+
+function project(files: { [path: string]: string }) {
+  var enc = new TextEncoder();
+  return async (rel: string) => rel in files ? enc.encode(files[rel]) : null;
+}
+
+describe('extension buildcore', function () {
+  var builder = new Builder(ROOT);
+
+  it('finds the repo as the asset root', function () {
+    assert.equal(ROOT, path.resolve(__dirname, '../../..'));
+  });
+
+  it('recognizes source files', function () {
+    assert.ok(isSourceFile('main.c'));
+    assert.ok(isSourceFile('game.dasm'));
+    assert.ok(!isSourceFile('settings.json'));
+  });
+
+  it('builds a C file with a local header and a preset header', async function () {
+    var files = project({
+      'main.c': '#include "neslib.h"\n#include "util.h"\nvoid main(void) { ppu_on_all(); while (1) { x = 1; } }\n',
+      'util.h': 'static int x;\n',
+    });
+    var mainText = new TextDecoder().decode(await files('main.c'));
+    var r = await builder.build({
+      platform: 'nes', mainPath: 'main.c', mainText,
+      files: new ProjectFileProvider(files, ROOT, 'nes'),
+    });
+    assert.deepEqual(r.diagnostics, []);
+    assert.ok(r.success);
+    assert.equal(r.tool, 'cc65');
+    assert.ok(r.output.length > 0);
+    assert.ok(r.paths.indexOf('util.h') >= 0);
+    assert.ok(r.listings);
+  });
+
+  it('reports errors against the file that has them', async function () {
+    var files = project({
+      'main.c': '#include "util.h"\nvoid main(void) { }\n',
+      'util.h': 'int x;\nthis is not C;\n',
+    });
+    var mainText = new TextDecoder().decode(await files('main.c'));
+    var r = await builder.build({
+      platform: 'nes', mainPath: 'main.c', mainText,
+      files: new ProjectFileProvider(files, ROOT, 'nes'),
+    });
+    assert.ok(!r.success);
+    assert.ok(r.diagnostics.length > 0);
+    assert.ok(r.diagnostics.every(d => d.path === 'util.h'), JSON.stringify(r.diagnostics));
+    assert.equal(r.diagnostics[0].line, 2);
+  });
+
+  it('reports assembler errors in the main file', async function () {
+    var src = '\tprocessor 6502\n\torg $f000\nStart\n\tlda #1\n\tbogus\n';
+    var r = await builder.build({
+      platform: 'vcs', mainPath: 'game.a', mainText: src,
+      files: new ProjectFileProvider(project({ 'game.a': src }), ROOT, 'vcs'),
+    });
+    assert.ok(!r.success);
+    assert.equal(r.tool, 'dasm');
+    assert.equal(r.diagnostics[0].path, 'game.a');
+    assert.equal(r.diagnostics[0].line, 5);
+  });
+});

extension/tsconfig.json

@@ -0,0 +1,14 @@
+{
+    // Type checking only; scripts/build.mjs bundles with esbuild.
+    "extends": "../tsconfig.json",
+    "include": [
+        "src/**/*.ts",
+        "test/**/*.ts"
+    ],
+    "compilerOptions": {
+        "noEmit": true,
+        "rootDir": "..",
+        "types": ["node", "mocha", "vscode"],
+        "typeRoots": ["./node_modules/@types", "../node_modules/@types"]
+    }
+}