npm: tweaks for npm release, github workflow, exclude vic20

.github/workflows/release-cli.yml

@@ -0,0 +1,109 @@
+# Publishes the 8bws command line to npm. Push a tag cli-v<version> (the
+# extension/package.json version, which the npm package shares) to run it:
+#
+#   1. build the packs and stage the npm package
+#   2. upload the packs and assets.json to the GitHub Release for the tag
+#   3. install the packed tarball in an empty dir and build from the Release
+#   4. npm publish --provenance
+#
+# Upload comes before publish, or a published version would point at 404s.
+# The packs are built once per tag (their hashes change with every build).
+# TODO: pin softprops/action-gh-release and download-artifact to commit SHAs
+# like node.js.yml does.
+# Needs the `npm` environment with secret NPM_TOKEN, and its required reviewers.
+name: Release 8bws CLI
+
+on:
+  push:
+    tags: ['cli-v*']
+
+jobs:
+  package:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+    steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+    - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+      with:
+        node-version: 24.x
+        cache: 'npm'
+    - name: Check the tag matches package.json
+      id: version
+      run: |
+        version=$(node -p 'require("./extension/package.json").version')
+        if [ "refs/tags/cli-v$version" != "$GITHUB_REF" ]; then
+          echo "tag $GITHUB_REF does not match extension/package.json version $version" >&2
+          exit 1
+        fi
+        echo "version=$version" >> "$GITHUB_OUTPUT"
+    - run: git submodule init
+    - run: git submodule update
+    - run: npm ci --no-audit --legacy-peer-deps
+    - run: npm run build --if-present
+    - run: npm ci --no-audit --legacy-peer-deps --prefix extension
+    - run: npm run typecheck --prefix extension
+    - run: npm test --prefix extension
+    # quality 9 packs; 'npm-package' also builds out/8bws.js and out/assets.json
+    - run: npm run npm-package --prefix extension
+    - name: Pack the tarball
+      run: npm pack --pack-destination "$RUNNER_TEMP/npm" ./extension/out/npm
+    - name: Upload packs to the GitHub Release
+      uses: softprops/action-gh-release@v2
+      with:
+        files: |
+          extension/out/assets/*.tar.br
+          extension/out/assets.json
+    - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      with:
+        name: npm-package
+        path: ${{ runner.temp }}/npm/*.tgz
+        if-no-files-found: error
+
+  # Install the tarball as a user would, with only the Release to fetch packs from.
+  smoke-test:
+    needs: package
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+      with:
+        node-version: 20.x
+    - uses: actions/download-artifact@v4
+      with:
+        name: npm-package
+        path: pkg
+    - name: Build and run from the Release
+      run: |
+        mkdir smoke && cd smoke
+        npm init -y >/dev/null
+        npm install ../pkg/*.tgz
+        cat > hello.c <<'EOC'
+        #include <stdint.h>
+        void main(void) { while (1) ; }
+        EOC
+        npx 8bws build --check -p nes hello.c
+        npx 8bws run -p nes hello.c --frames 20
+      env:
+        XDG_CACHE_HOME: ${{ runner.temp }}/cache
+
+  publish:
+    needs: [package, smoke-test]
+    runs-on: ubuntu-latest
+    environment: npm
+    permissions:
+      contents: read
+      id-token: write   # provenance
+    steps:
+    - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+      with:
+        node-version: 24.x
+        registry-url: https://registry.npmjs.org
+    - uses: actions/download-artifact@v4
+      with:
+        name: npm-package
+        path: pkg
+    - run: npm publish pkg/*.tgz --access public --provenance
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

CHANGELOG.md

@@ -20,6 +20,7 @@ See [extension/CHANGELOG.md](extension/CHANGELOG.md).
 ### `8bws` CLI
 
 - New `8bws` command-line tool (`npm run cli`) to build and run programs headlessly: `build`, `run`, `verify-replay`, `profile`, `detect`, `list-platforms`, `list-tools`; scripted emulator control; `--png`, `--json`, `--symbols`, `--memdump`, `--info`; platform and main file detected from a project directory.
+- `8bws` is packaged for npm (`npm run npm-package` in `extension/`; named `8bws` until `8bitworkshop` is available). It downloads toolchain packs on first use, from the version's GitHub Release first; pushing a `cli-v<version>` tag publishes it; its version follows the extension's (`.github/workflows/release-cli.yml`).
 - `8bws run --frames-dir <dir>` records an animation for ffmpeg, writing every frame as `DIR/frame_NNNNN.png` (the run script adds `capture N DIR` and `png FILE`); `--fps` sets the nominal frame rate and the result includes the matching `ffmpeg` command.
 - `8bws run` script `paddle X Y [BUTTONS]` moves the paddle on platforms and Verilog designs that have one.
 - `8bws run` script `vcd FILE [MAXMB]` / `vcd off` records a Verilog design's signal changes to a VCD file as it runs, streamed so the recording's length isn't limited by memory (a `.gz` name compresses it; about 3 MB per frame of a design with video, 4x less gzipped), stopping at 1 GB unless told otherwise.

extension/npm/README.md

@@ -6,10 +6,10 @@ many more, with the same compilers, assemblers and emulators as
 [8bitworkshop.com](https://8bitworkshop.com).
 
 ```sh
-npx 8bitworkshop build --check hello.c        # compile only
-npx 8bitworkshop build hello.c -o hello.nes   # platform is detected
-npx 8bitworkshop run hello.c --frames 60 --png screen.png
-npx 8bitworkshop help                          # every command and option
+npx {{package}} build --check hello.c        # compile only
+npx {{package}} build hello.c -o hello.nes   # platform is detected
+npx {{package}} run hello.c --frames 60 --png screen.png
+npx {{package}} help                          # every command and option
 ```
 
 Add `--json` for machine-readable output. Requires Node 20 or later.

extension/scripts/assetpack.ts

@@ -57,7 +57,8 @@ function expandLink(root: string, rel: string, tracked: string[]): string[] {
 async function main() {
   var lists = listPackFiles(rootDir);
   var ideVersion = JSON.parse(fs.readFileSync(path.join(rootDir, 'package.json'), 'utf-8')).version;
-  var manifest: AssetManifest = { ideVersion, packs: {} };
+  var version = JSON.parse(fs.readFileSync(path.join(__dirname, '../package.json'), 'utf-8')).version;
+  var manifest: AssetManifest = { ideVersion, version, packs: {} };
   fs.mkdirSync(outDir, { recursive: true });
   // compress the packs in parallel (zlib runs on the thread pool)
   await Promise.all(PACKS.map(async pack => {

extension/scripts/stage-npm.mjs

@@ -1,10 +1,13 @@
 // Stages the npm package for the 8bws command line in out/npm (or --out DIR):
 // the CLI bundle with a shebang, assets.json, license files and a package.json
-// whose version is the repo's. The packs themselves are not included; the CLI
+// whose version is the extension's. The packs themselves are not included; the CLI
 // downloads them on first use (see src/cli.ts). Run `npm run build` and
 // `npm run assets` first so out/8bws.js and out/assets.json are current:
 //
-//   node scripts/stage-npm.mjs [--out DIR]
+//   node scripts/stage-npm.mjs [--out DIR] [--name PACKAGE]
+//
+// The package is named `8bws` until the `8bitworkshop` name is ours; pass
+// --name 8bitworkshop (or set NPM_PACKAGE_NAME) to publish under that.
 //   cd out/npm && npm pack
 import { chmodSync, copyFileSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs';
 import { fileURLToPath } from 'url';
@@ -13,10 +16,13 @@ import path from 'path';
 const ext = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 const outIdx = process.argv.indexOf('--out');
 const out = path.resolve(outIdx > 0 ? process.argv[outIdx + 1] : path.join(ext, 'out', 'npm'));
+const nameIdx = process.argv.indexOf('--name');
+const name = nameIdx > 0 ? process.argv[nameIdx + 1] : process.env.NPM_PACKAGE_NAME || '8bws';
 const rootPkg = JSON.parse(readFileSync(path.join(ext, '..', 'package.json'), 'utf-8'));
 const manifest = JSON.parse(readFileSync(path.join(ext, 'out', 'assets.json'), 'utf-8'));
-if (manifest.ideVersion !== rootPkg.version) {
-  throw new Error(`out/assets.json is for ${manifest.ideVersion}, not ${rootPkg.version}: run npm run assets`);
+const extPkg = JSON.parse(readFileSync(path.join(ext, 'package.json'), 'utf-8'));
+if (manifest.version !== extPkg.version) {
+  throw new Error(`out/assets.json is for ${manifest.version}, not ${extPkg.version}: run npm run assets`);
 }
 
 rmSync(out, { recursive: true, force: true });
@@ -28,11 +34,11 @@ chmodSync(path.join(out, '8bws.js'), 0o755);
 copyFileSync(path.join(ext, 'out', 'assets.json'), path.join(out, 'assets.json'));
 copyFileSync(path.join(ext, 'LICENSE'), path.join(out, 'LICENSE'));
 copyFileSync(path.join(ext, 'THIRD-PARTY-NOTICES.md'), path.join(out, 'THIRD-PARTY-NOTICES.md'));
-copyFileSync(path.join(ext, 'npm', 'README.md'), path.join(out, 'README.md'));
+writeFileSync(path.join(out, 'README.md'), readFileSync(path.join(ext, 'npm', 'README.md'), 'utf-8').replaceAll('{{package}}', name));
 
 writeFileSync(path.join(out, 'package.json'), JSON.stringify({
-  name: '8bitworkshop',
-  version: rootPkg.version,
+  name,
+  version: extPkg.version,
   description: 'Build and run retro-computer and arcade programs from the command line: the 8bitworkshop compilers and emulators, headless',
   keywords: ['retro', '8-bit', '6502', 'z80', 'emulator', 'assembler', 'compiler', 'nes', 'c64', 'atari'],
   bin: { '8bws': '8bws.js' },
@@ -40,8 +46,8 @@ writeFileSync(path.join(out, 'package.json'), JSON.stringify({
   engines: { node: '>=20' },
   license: 'GPL-3.0-only',
   author: rootPkg.author,
-  repository: { type: 'git', url: 'git+https://github.com/sehugg/8bitworkshop.git' },
-  bugs: { url: 'https://github.com/sehugg/8bitworkshop/issues' },
+  repository: { type: 'git', url: 'git+https://github.com/8bitworkshop/8bitworkshop.git' },
+  bugs: { url: 'https://github.com/8bitworkshop/8bitworkshop/issues' },
   homepage: 'https://8bitworkshop.com',
 }, null, 2) + '\n');
-console.log(`staged ${rootPkg.name} ${rootPkg.version} in ${out}`);
+console.log(`staged ${name} ${extPkg.version} in ${out}`);

extension/src/cli.ts

@@ -7,7 +7,7 @@
 
 import * as fs from 'fs';
 import * as path from 'path';
-import { ASSET_URLS, AssetStore, defaultCacheDir } from '../../src/tools/assets';
+import { assetServers, AssetStore, defaultCacheDir } from '../../src/tools/assets';
 import { AssetManifest, isPackable, packsForPlatform } from '../../src/tools/assetpacks';
 import { UNREVIEWED_TOOLS, UNSUPPORTED_PLATFORMS } from '../../src/tools/exclusions';
 import { setToolchainHost } from '../../src/tools/toolroot';
@@ -16,7 +16,7 @@ import { getRootBasePlatform } from '../../src/common/util';
 /** The asset store: the packs next to this file (the extension), then the servers. */
 function openStore(): AssetStore {
   var manifest: AssetManifest = JSON.parse(fs.readFileSync(path.join(__dirname, 'assets.json'), 'utf-8'));
-  var urls = ASSET_URLS.slice();
+  var urls = assetServers(manifest);
   var bundled = path.join(__dirname, 'assets');
   if (fs.existsSync(bundled)) urls.unshift(bundled);
   // a mirror, or a directory of packs for offline use

extension/src/extension.ts

@@ -23,7 +23,7 @@ import { Detection, classifyFinding, isBuildableSource } from '../../src/common/
 import { buildProducts } from '../../src/common/workertypes';
 import { TOOL_META } from '../../src/common/toolmeta';
 import { getToolForPlatform } from '../../src/common/toolselect';
-import { ASSET_URLS, AssetStore } from '../../src/tools/assets';
+import { assetServers, AssetStore } from '../../src/tools/assets';
 import { projectReadme, writeLauncher } from './terminalcli';
 import type { PlatformInfo } from './presettypes';
 import { registerTools } from './lmtools';
@@ -247,7 +247,7 @@ function getAssets(): AssetStore {
     // 8bitworkshop servers (for packs not bundled).
     var urls: string[] = [];
     if (Object.keys(manifest.packs).some(p => hasBundledPack(manifest, p))) urls.push(bundledAssetsDir());
-    urls.push(...ASSET_URLS);
+    urls.push(...assetServers(manifest));
     assets = new AssetStore(toolchainCacheDir(), manifest,
       urls, msg => output.appendLine(msg));
   }

extension/test/assets.test.ts

@@ -7,7 +7,7 @@ import { createHash } from 'crypto';
 import { Worker } from 'worker_threads';
 import * as zlib from 'zlib';
 import { execFileSync } from 'child_process';
-import { AssetStore, defaultCacheDir } from '../../src/tools/assets';
+import { AssetStore, assetServers, defaultCacheDir } from '../../src/tools/assets';
 import { EXCLUDED_PLATFORM_MODULES, UNSUPPORTED_PLATFORMS } from '../../src/tools/exclusions';
 import { needsSdcc4, AssetManifest, PackInfo, isUnreviewedFile, makePack, packForFile, packsForPlatform, readPack } from '../../src/tools/assetpacks';
 import { listPackFiles } from '../scripts/assetpack';
@@ -170,7 +170,7 @@ describe('extension AssetStore', function () {
   before(async function () {
     var [base, baseZip] = await pack('base', { 'src/worker/wasm/a.wasm': 'AAA', 'presets/nes/hello.c': 'main' });
     var [verilog, verilogZip] = await pack('verilog', { 'src/worker/wasm/verilator_bin.wasm': 'VVV' });
-    manifest = { ideVersion: 'test', packs: { base, verilog } };
+    manifest = { ideVersion: 'test', version: 'test', packs: { base, verilog } };
     served = { [base.file]: baseZip, [verilog.file]: verilogZip };
     server = http.createServer((req, res) => {
       var name = path.posix.basename(req.url || '');
@@ -378,7 +378,7 @@ describe('8bws bundle with an empty cache', function () {
   it('lists only the tools and platforms the packs can build', function () {
     var tools: string[] = JSON.parse(run('list-tools', '--json')).data.tools;
     assert.ok(tools.includes('cc65') && tools.includes('dasm'));
-    for (var t of ['dialog', 'armtcc', 'inform', 'nesasm', 'merlin32', 'xa', 'xasm6809']) assert.ok(!tools.includes(t), `${t} listed`);
+    for (var t of ['dialog', 'armtcc', 'inform', 'nesasm', 'merlin32', 'xasm6809']) assert.ok(!tools.includes(t), `${t} listed`);
     var platforms = Object.keys(JSON.parse(run('list-platforms', '--json')).data.platforms);
     assert.ok(platforms.includes('nes'));
     for (var p of UNSUPPORTED_PLATFORMS) assert.ok(!platforms.includes(p), `${p} listed`);
@@ -398,6 +398,14 @@ describe('8bws bundle with an empty cache', function () {
   });
 });
 
+describe('asset servers', function () {
+  it('try the version\'s GitHub Release first, then the Pages fallbacks', function () {
+    var urls = assetServers({ ideVersion: '4.0.0', version: '0.1.2', packs: {} });
+    assert.equal(urls[0], 'https://github.com/8bitworkshop/8bitworkshop/releases/download/cli-v0.1.2/');
+    assert.ok(urls.length > 1 && urls.slice(1).every(u => !u.includes('/releases/')));
+  });
+});
+
 describe('8bws npm package', function () {
   this.timeout(120000);
   var dir: string;
@@ -417,13 +425,24 @@ describe('8bws npm package', function () {
     for (var d of [dir && path.dirname(dir), home, work]) if (d) fs.rmSync(d, { recursive: true, force: true });
   });
 
-  it('has a package.json that matches the repo version, and a bin with a shebang', function () {
+  it('has a package.json that matches the extension version, and a bin with a shebang', function () {
     var pkg = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf-8'));
-    assert.equal(pkg.version, JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf-8')).version);
+    assert.equal(pkg.version, JSON.parse(fs.readFileSync(path.join(OUT, '..', 'package.json'), 'utf-8')).version);
     assert.deepEqual(pkg.bin, { '8bws': '8bws.js' });
     assert.ok(fs.readFileSync(path.join(dir, '8bws.js'), 'utf-8').startsWith('#!/usr/bin/env node\n'));
   });
 
+  it('is named 8bws by default and takes --name', function () {
+    var pkg = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf-8'));
+    assert.equal(pkg.name, '8bws');
+    var other = path.join(tmpdir(), 'named');
+    execFileSync(process.execPath, [path.join(OUT, '..', 'scripts', 'stage-npm.mjs'), '--out', other, '--name', '8bitworkshop'], { stdio: 'pipe' });
+    assert.equal(JSON.parse(fs.readFileSync(path.join(other, 'package.json'), 'utf-8')).name, '8bitworkshop');
+    assert.ok(fs.readFileSync(path.join(other, 'README.md'), 'utf-8').includes('npx 8bitworkshop build'));
+    assert.ok(fs.readFileSync(path.join(dir, 'README.md'), 'utf-8').includes('npx 8bws build'));
+    fs.rmSync(path.dirname(other), { recursive: true, force: true });
+  });
+
   it('stages the CLI, manifest and licenses, and lists exactly those to publish', function () {
     var expected = ['8bws.js', 'LICENSE', 'README.md', 'THIRD-PARTY-NOTICES.md', 'assets.json', 'package.json'];
     assert.deepEqual(fs.readdirSync(dir).sort(), expected);

res/README.md

@@ -20,8 +20,8 @@ res/common/                  files shared by platforms
 | `common/` | `src/platform/basic.ts` |
 
 The browser IDE serves this directory as static files; the VS Code extension
-ships it in the base asset pack. Everything except the Altirra debug listings
-and the x86 images is included (see `extension/src/assetpacks.ts`).
+ships it in the base asset pack. Everything except the Altirra debug listings,
+the x86 images and the CPC and VIC-20 files is included (see `src/tools/exclusions.ts`).
 
 ## Provenance
 

src/tools/assetpacks.ts

@@ -115,6 +115,8 @@ export interface PackInfo {
 export interface AssetManifest {
   /** the IDE version the assets come from (the repo's package.json) */
   ideVersion: string;
+  /** the extension/npm package version that published these packs; names the GitHub Release */
+  version: string;
   packs: { [pack: string]: PackInfo };
 }
 

src/tools/assets.ts

@@ -13,6 +13,21 @@ import { AssetManifest, PackInfo, readPack } from './assetpacks';
 /** The 8bitworkshop asset servers, tried in order. */
 export const ASSET_URLS = ['https://8bitworkshop.github.io/8bitworkshop/vscode/', 'https://8bitworkshop.com/vscode/'];
 
+/**
+ * The GitHub Release that holds the packs for a package `version` (the
+ * extension's, which the npm package shares; tag cli-v<version>).
+ * Releases are permanent, so a pinned npm version keeps working after the
+ * Pages deploy replaces its files.
+ */
+export function releaseAssetUrl(version: string): string {
+  return `https://github.com/8bitworkshop/8bitworkshop/releases/download/cli-v${version}/`;
+}
+
+/** Asset servers for a manifest: its Release first, then the Pages fallbacks. */
+export function assetServers(manifest: AssetManifest): string[] {
+  return [releaseAssetUrl(manifest.version), ...ASSET_URLS];
+}
+
 /**
  * Where the 8bws command line keeps unpacked toolchains when
  * $EIGHTBITWORKSHOP_TOOLCHAINS isn't set: the user's cache directory.

src/tools/exclusions.ts

@@ -12,8 +12,8 @@
 // ---- 1. PACKS ----
 
 /**
- * Tracked files the extension never loads. res/: Altirra debug listings, and
- * the x86 BIOSes (x86 isn't offered in the extension). Everything else under
+ * Tracked files the extension never loads. res/: Altirra debug listings, the
+ * x86 BIOSes (x86 isn't offered in the extension), CPC and VIC-20 (not offered). Everything else under
  * res/ (kernels, BIOSes, wasm cores) goes in the base pack. src/worker/: the
  * Dialog, Inform 6, armips, YASM, arm-tcc and smlrc toolchains, whose
  * platforms (Z-machine, MIPS, x86, ARM) the extension doesn't offer.
@@ -22,9 +22,12 @@ export const ASSET_EXCLUDE = new RegExp('^(' + [
   'res/atari8/altirra/.*\\.(lab|lst)',
   'res/x86/.*',
   'res/cpc/.*',
+  'res/vic20/.*',
+  'presets/vic20/.*',
   'src/worker/wasm/(dialogc|armips|inform|yasm|arm-tcc|smlrc)\\.(js|wasm)',
   'src/worker/lib/arm32/.*',
   'src/worker/lib/cpc/.*',
+  'src/worker/fs/cc65-fs-vic20\\.zip',
   'src/worker/fs/(dialog-fs\\.zip|arm32-fs\\.zip|fsinform\\.|fssmlrc\\.)[^/]*',
 ].join('|') + ')$');
 
@@ -68,11 +71,12 @@ export const EXCLUDED_PLATFORM_MODULES = ['vectrex'];
 
 /**
  * Platforms the packaged CLI can't build or run: x86 and ARM have no
- * toolchain in the packs, and CPC has no library or core (see ASSET_EXCLUDE);
+ * toolchain in the packs, CPC has no library or core, and VIC-20 has no BIOS or
+ * core (the open-roms BIOS isn't tracked; see ASSET_EXCLUDE);
  * the modules in EXCLUDED_PLATFORM_MODULES have no emulator. The CLI reports them as
  * unavailable instead of failing partway.
  */
-export const UNSUPPORTED_PLATFORMS = ['x86', 'arm32', 'cpc', ...EXCLUDED_PLATFORM_MODULES];
+export const UNSUPPORTED_PLATFORMS = ['x86', 'arm32', 'cpc', 'vic20', ...EXCLUDED_PLATFORM_MODULES];
 
 // ---- 4. EXTENSION_UI ----