cache WASI shared FS so we can chase #includes for those tools

src/common/toolmeta.ts

@@ -22,6 +22,8 @@
  * (unless noWorkerBuild) and vice versa.
  */
 
+import { getRootBasePlatform } from "./util";
+
 export type ToolKind =
   | 'compiler'      // source -> object/output (cc65, sdcc, cmoc, oscar64, ...)
   | 'assembler'     // asm -> object (dasm, acme, ca65, ...)
@@ -94,11 +96,15 @@ export interface ToolMeta {
   includePatterns?: (RegExp | ToolIncludePattern)[];
   /** regexes matching link directives, used for dependency parsing */
   linkPatterns?: (RegExp | ToolIncludePattern)[];
-  /** directories inside the preload FS containing searchable shared code
+  /** directories inside the shared filesystem containing searchable shared code
    *  (headers, asm includes). Paths are relative to the FS root as they appear
-   *  in the filesystem package metadata, e.g. ['/include', '/asminc'].
-   *  Only meaningful when paired with platforms[].preloadFS. */
+   *  in the filesystem metadata, e.g. ['/include', '/asminc', '/headers'].
+   *  Used with platforms[].preloadFS or wasiFSZip. */
   includeDirs?: string[];
+  /** name of a WASI filesystem zip (in src/worker/fs/) that contains the
+   *  toolchain's headers, for tools that don't use an emscripten preloadFS
+   *  package (e.g. cc2600/cc7800). Referenced as 'wasi:<name>' in worker msgs. */
+  wasiFSZip?: string;
 
   // ---- per-platform metadata (was TOOL_PRELOADFS / PLATFORM_PARAMS) ----
 
@@ -374,7 +380,8 @@ export const TOOL_META: { [id: string]: ToolMeta } = {
   cmoc: {
     id: 'cmoc', name: 'CMOC', kind: 'compiler', arch: '6809',
     extensions: ['.c', '.h'],
-    includeDirs: ['/include'],
+    // NOTE: no bundled filesystem -- the compiler passes -I/share/include but
+    // never mounts one, so system headers can't be linked in the UI
     editorStyle: 'text/x-csrc',
     helpURL: 'http://perso.b2b2c.ca/~sarrazip/dev/cmoc.html',
     wasmModule: 'cmoc',
@@ -420,6 +427,7 @@ export const TOOL_META: { [id: string]: ToolMeta } = {
     id: 'armtcc', name: 'TCC (ARM)', kind: 'compiler', arch: 'arm32',
     extensions: ['.c', '.s'],
     includeDirs: ['/include'],
+    wasiFSZip: 'arm32-fs.zip',
     editorStyle: 'text/x-csrc',
     wasmModule: 'arm-tcc',
     includePatterns: SHARED_INCLUDE_PATTERNS,
@@ -437,7 +445,7 @@ export const TOOL_META: { [id: string]: ToolMeta } = {
   smlrc: {
     id: 'smlrc', name: 'SmallerC', kind: 'compiler', arch: 'x86',
     extensions: ['.c'],
-    includeDirs: ['/include'],
+    // NOTE: no bundled filesystem and no -I arg -- no headers to link in the UI
     editorStyle: 'text/x-csrc',
     wasmModule: 'smlrc',
     includePatterns: SHARED_INCLUDE_PATTERNS,
@@ -459,6 +467,7 @@ export const TOOL_META: { [id: string]: ToolMeta } = {
     id: 'oscar64', name: 'Oscar64', kind: 'compiler', arch: '6502',
     extensions: ['.c', '.cpp', '.cc', '.o64'],
     includeDirs: ['/include'],
+    wasiFSZip: 'oscar64-fs.zip',
     editorStyle: 'text/x-csrc',
     helpURL: 'https://github.com/drmortalwombat/oscar64/blob/main/oscar64.md',
     wasmModule: 'oscar64',
@@ -494,6 +503,8 @@ export const TOOL_META: { [id: string]: ToolMeta } = {
     extensions: ['.cc2600'],
     editorStyle: 'text/x-csrc',
     wasmModule: 'cc2600',
+    wasiFSZip: 'cc2600-fs.zip',
+    includeDirs: ['/headers'],
     includePatterns: SHARED_INCLUDE_PATTERNS,
     linkPatterns: SHARED_LINK_PATTERNS,
   },
@@ -503,6 +514,8 @@ export const TOOL_META: { [id: string]: ToolMeta } = {
     extensions: ['.cc7800', '.c78'],
     editorStyle: 'text/x-csrc',
     wasmModule: 'cc7800',
+    wasiFSZip: 'cc7800-fs.zip',
+    includeDirs: ['/headers'],
     includePatterns: SHARED_INCLUDE_PATTERNS,
     linkPatterns: SHARED_LINK_PATTERNS,
   },
@@ -518,7 +531,7 @@ export const TOOL_META: { [id: string]: ToolMeta } = {
   wiz: {
     id: 'wiz', name: 'wiz', kind: 'compiler',
     extensions: ['.wiz'],
-    includeDirs: ['/common'],
+    includeDirs: ['/common/$platform'],
     editorStyle: 'text/x-wiz',
     helpURL: 'https://github.com/wiz-lang/wiz/blob/master/readme.md#wiz',
     wasmModule: 'wiz',
@@ -552,6 +565,8 @@ export const TOOL_META: { [id: string]: ToolMeta } = {
     editorStyle: 'dialog',
     helpURL: 'https://linusakesson.net/dialog/docs/',
     wasmModule: 'dialogc',
+    wasiFSZip: 'dialog-fs.zip',
+    includeDirs: ['/'],
     includePatterns: DIALOG_INCLUDE_PATTERNS,
   },
 
@@ -672,7 +687,7 @@ export function getSkeletonName(tool: string): string {
  * worked before this registry. A tool that really has no include directives
  * says so with an explicit empty list.
  */
-export function getIncludePatterns(tool?: string, platform?: string): (RegExp | ToolIncludePattern)[] {
+export function getIncludePatterns(tool: string, platform?: string): (RegExp | ToolIncludePattern)[] {
   let meta = tool && getToolMeta(tool);
   if (meta && meta.includePatterns) return meta.includePatterns;
   if (platform && platform.startsWith('verilog')) return VERILOG_INCLUDE_PATTERNS;
@@ -680,7 +695,7 @@ export function getIncludePatterns(tool?: string, platform?: string): (RegExp |
 }
 
 /** Link patterns ("//#link") to use when scanning a source file. */
-export function getLinkPatterns(tool?: string, platform?: string): (RegExp | ToolIncludePattern)[] {
+export function getLinkPatterns(tool: string, platform?: string): (RegExp | ToolIncludePattern)[] {
   let meta = tool && getToolMeta(tool);
   if (meta && meta.linkPatterns) return meta.linkPatterns;
   if (platform && platform.startsWith('verilog')) return [];
@@ -688,12 +703,28 @@ export function getLinkPatterns(tool?: string, platform?: string): (RegExp | Too
 }
 
 /**
- * Directories containing shared code (headers etc.) for this tool's preload
- * filesystem, e.g. ['/include','/asminc'] -- empty if none/unknown.
+ * Directories containing shared code (headers etc.) for this tool's shared
+ * filesystem, e.g. ['/include','/asminc','/headers'] -- empty if none/unknown.
  */
-export function getIncludeDirs(tool?: string): string[] {
+export function getIncludeDirs(tool: string, platform_id: string): string[] {
+  platform_id = platform_id && getRootBasePlatform(platform_id);
   let meta = tool && getToolMeta(tool);
-  return (meta && meta.includeDirs) || [];
+  let out = (meta && meta.includeDirs) || [];
+  out = out.map(s => s.replace('$platform', platform_id));
+  return out;
+}
+
+/**
+ * Name of the shared filesystem to query for this tool's bundled files:
+ * either the emscripten preloadFS package name (e.g. '65-nes') or, for tools
+ * with a WASI filesystem zip (e.g. cc2600), 'wasi:<zipname>'.
+ * Undefined if the tool has no bundled filesystem.
+ */
+export function getSharedFileSystemName(tool: string, platform?: string): string | undefined {
+  var fsName = getPreloadFSName(tool, platform);
+  if (fsName) return fsName;
+  let meta = tool && getToolMeta(tool);
+  return (meta && meta.wasiFSZip) ? 'wasi:' + meta.wasiFSZip : undefined;
 }
 
 /**

src/ide/ui.ts

@@ -289,10 +289,10 @@ var openHeaderViews: string[] = [];
 export function openHeaderFile(fn: string) {
   if (!openHeaderViews.includes(fn)) {
     openHeaderViews.push(fn);
-    projectWindows.setCreateFunc('#headerview/' + encodeURIComponent(fn), () => new HeaderView(fn));
+    projectWindows.setCreateFunc('#headerview/' + fn, () => new HeaderView(fn));
     refreshWindowList();
   }
-  var hash = '#headerview/' + encodeURIComponent(fn);
+  var hash = '#headerview/' + fn;
   if (window.location.hash !== hash) {
     window.location.hash = hash.substring(1);
   }
@@ -369,7 +369,7 @@ function refreshWindowList() {
 
   // add opened toolchain headers (from include badges)
   for (let hdrfn of openHeaderViews) {
-    let hdrid = '#headerview/' + encodeURIComponent(hdrfn);
+    let hdrid = '#headerview/' + hdrfn;
     projectWindows.setCreateFunc(hdrid, () => new HeaderView(hdrfn));
     addWindowItem(hdrid, getFilenameForPath(hdrfn), () => {
       return new HeaderView(hdrfn);

src/ide/views/editors.ts

@@ -8,7 +8,7 @@ import { crosshairCursor, drawSelection, dropCursor, EditorView, highlightActive
 import { CodeAnalyzer } from "../../common/analysis";
 import { ProbeFlags, ProbeRecorder } from "../../common/probe";
 import { getFilenameForPath, getFolderForPath, hex, rpad } from "../../common/util";
-import { getIncludeDirs, getIncludePatterns, getLinkPatterns, getPreloadFSName, getSystemIncludePatterns } from "../../common/toolmeta";
+import { getIncludeDirs, getIncludePatterns, getLinkPatterns, getSharedFileSystemName, getSystemIncludePatterns } from "../../common/toolmeta";
 import { WorkerMessage } from "../../common/workertypes";
 import { SourceFile, SourceLocation, WorkerError } from "../../common/workertypes";
 import { asm6502 } from "../../parser/lang-6502";
@@ -965,14 +965,18 @@ const sharedFileCache = new Map<string, Promise<string | null>>();
 
 export function lookupSharedFileText(fn: string): Promise<string | null> {
   const tool = current_project.getToolForFilename(current_project.mainPath);
-  const fsName = getPreloadFSName(tool, current_project.platform_id);
-  const dirs = getIncludeDirs(tool);
+  const fsName = getSharedFileSystemName(tool, current_project.platform_id);
+  const dirs = getIncludeDirs(tool, current_project.platform_id);
   if (!fsName || !dirs.length) return Promise.resolve(null);
   const key = fsName + ':' + fn;
   if (!sharedFileCache.has(key)) {
     sharedFileCache.set(key, (async () => {
-      for (var dir of dirs) {
-        var msg = { preload_fs: fsName, readshared: dir + '/' + fn, updates: [], buildsteps: [] } as WorkerMessage;
+      // try the include under each known dir (e.g. /headers/vcs.h for "vcs.h"),
+      // then the filename as-is (some includes already carry the dir,
+      // e.g. "headers/vcs.h")
+      var candidates = [...dirs.map(dir => dir + '/' + fn), fn];
+      for (var path of candidates) {
+        var msg = { preload_fs: fsName, readshared: path, updates: [], buildsteps: [] } as WorkerMessage;
         var result = await current_project.queryWorker(msg);
         var output = result && (result as any).output;
         if (output instanceof Uint8Array && output.length > 0) {

src/ide/views/includedecorations.ts

@@ -37,8 +37,11 @@ function buildIncludeLinkDecorations(view: EditorView, patterns: (RegExp | ToolI
       p.re.lastIndex = 0; // patterns are shared and global, so rewind first
       let m;
       while ((m = p.re.exec(text))) {
-        const filename = m[p.group != null ? p.group : m.length - 1];
-        if (!filename) continue;
+        const matched = m[p.group != null ? p.group : m.length - 1];
+        if (!matched) continue;
+        // apply any implicit extension (e.g. wiz `import "nes"` -> "nes.wiz"),
+        // same as matchDependencyPatterns() does for build deps
+        const filename = p.suffix ? matched + p.suffix : matched;
         // place widget at end of the line containing the match
         const matchEnd = Math.min(from + m.index + m[0].length, doc.length);
         const line = doc.lineAt(matchEnd);

src/worker/wasmutils.ts

@@ -218,6 +218,58 @@ export function listSharedFiles(name: string, dir: string): string[] {
   return result.sort();
 }
 
+/// shared code in WASI filesystem zips (e.g. cc2600/cc7800 headers)
+// referenced by worker msgs as 'wasi:<zipname>'
+
+import { loadWASIFilesystemZip } from './wasiutils';
+import type { WASIMemoryFilesystem } from '../common/wasi/wasishim';
+
+var wasiFilesystems: { [zipname: string]: WASIMemoryFilesystem } = {};
+
+export async function ensureWasiFilesystem(zipname: string): Promise<WASIMemoryFilesystem | null> {
+  try {
+    if (!wasiFilesystems[zipname]) {
+      wasiFilesystems[zipname] = await loadWASIFilesystemZip(zipname);
+    }
+    return wasiFilesystems[zipname];
+  } catch (e) {
+    console.log('Error loading WASI filesystem', zipname, e);
+    return null;
+  }
+}
+
+// zip entries are stored under '<path>' (rootPath './' adds a './' prefix);
+// strip it plus any leading '/' so '/headers/vcs.h' matches 'headers/vcs.h'
+function normalizeWasiPath(path: string): string {
+  path = path.startsWith('./') ? path.substring(2) : path;
+  return path.startsWith('/') ? path.substring(1) : path;
+}
+
+/** Read a file from a WASI filesystem zip. Returns null if not found. */
+export async function readWasiSharedFile(zipname: string, path: string): Promise<Uint8Array | null> {
+  let fs = await ensureWasiFilesystem(zipname);
+  if (!fs) return null;
+  let norm = normalizeWasiPath(path);
+  // zip entries are stored under './<path>', so try both key forms
+  let fd = fs.getFile(norm) || fs.getFile('./' + norm);
+  if (!fd || !fd.size) return null;
+  let data = new Uint8Array(fd.size);
+  fd.offset = 0;
+  fd.read(data);
+  return data;
+}
+
+/** List files under a directory prefix (e.g. '/headers') of a WASI filesystem zip. */
+export async function listWasiSharedFiles(zipname: string, dir: string): Promise<string[]> {
+  let fs = await ensureWasiFilesystem(zipname);
+  if (!fs) return [];
+  let prefix = normalizeWasiPath(dir) + '/';
+  return fs.getFiles()
+    .map(f => normalizeWasiPath(f.name))
+    .filter(name => name.startsWith(prefix))
+    .sort();
+}
+
 /** Read a file from a filesystem package blob. Returns null if not found.
  *  Async because Blob reading is async; call from worker message handlers. */
 export async function readSharedFile(name: string, path: string): Promise<Uint8Array | null> {

src/worker/workerlib.ts

@@ -9,7 +9,12 @@ import type { WorkerResult, WorkerMessage } from "../common/workertypes";
 import { getBasePlatform, getRootBasePlatform } from "../common/util";
 import { getPreloadFSName } from "../common/toolmeta";
 import { store, builder } from "./builder";
-import { emglobal, fsMeta, loadFilesystem, listSharedFiles, readSharedFile, ensureFilesystem } from "./wasmutils";
+import { emglobal, fsMeta, loadFilesystem, listSharedFiles, readSharedFile, ensureFilesystem, readWasiSharedFile, listWasiSharedFiles, ensureWasiFilesystem } from "./wasmutils";
+
+// shared FS names starting with 'wasi:' refer to a WASI filesystem zip
+function splitWasiFSName(fsName: string): { wasi: boolean, name: string } {
+  return fsName.startsWith('wasi:') ? { wasi: true, name: fsName.substring(5) } : { wasi: false, name: fsName };
+}
 import { setupRequireFunction } from "./workermain";
 
 export { store, builder };
@@ -139,19 +144,23 @@ export async function handleMessage(data: WorkerMessage): Promise<WorkerResult>
   }
   // read a file from a filesystem package (shared code)
   if (data.readshared) {
-    ensureFilesystem(data.preload_fs);
-    var contents = await readSharedFile(data.preload_fs, data.readshared);
+    var fs1 = splitWasiFSName(data.preload_fs);
+    var contents = fs1.wasi ? await readWasiSharedFile(fs1.name, data.readshared)
+      : (ensureFilesystem(fs1.name), await readSharedFile(fs1.name, data.readshared));
     return { output: contents, qid: data.qid } as WorkerResult;
   }
   // list files in a filesystem package directory (shared code)
   if (data.listshared != null) {
-    ensureFilesystem(data.preload_fs);
-    var files = listSharedFiles(data.preload_fs, data.listshared);
+    var fs2 = splitWasiFSName(data.preload_fs);
+    var files = fs2.wasi ? await listWasiSharedFiles(fs2.name, data.listshared)
+      : (ensureFilesystem(fs2.name), listSharedFiles(fs2.name, data.listshared));
     return { output: files, qid: data.qid } as WorkerResult;
   }
   // preload a filesystem package directly by name
   if (data.preload_fs) {
-    ensureFilesystem(data.preload_fs);
+    var fs3 = splitWasiFSName(data.preload_fs);
+    if (fs3.wasi) await ensureWasiFilesystem(fs3.name);
+    else ensureFilesystem(fs3.name);
     return;
   }
   // clear filesystem?

src/worker/workermain.ts

@@ -3,7 +3,12 @@ import type { WorkerResult, WorkerMessage, WorkerError, SourceLine } from "../co
 import { getBasePlatform, getRootBasePlatform } from "../common/util";
 import { getPreloadFSName } from "../common/toolmeta";
 import { store, builder, errorResult, getWorkFileAsString } from "./builder";
-import { emglobal, fsMeta, loadFilesystem, listSharedFiles, readSharedFile, ensureFilesystem } from "./wasmutils";
+import { emglobal, fsMeta, loadFilesystem, listSharedFiles, readSharedFile, ensureFilesystem, readWasiSharedFile, listWasiSharedFiles, ensureWasiFilesystem } from "./wasmutils";
+
+// shared FS names starting with 'wasi:' refer to a WASI filesystem zip
+function splitWasiFSName(fsName: string): { wasi: boolean, name: string } {
+  return fsName.startsWith('wasi:') ? { wasi: true, name: fsName.substring(5) } : { wasi: false, name: fsName };
+}
 
 declare function importScripts(path: string);
 declare function postMessage(msg);
@@ -40,19 +45,23 @@ async function handleMessage(data: WorkerMessage): Promise<WorkerResult> {
   }
   // read a file from a filesystem package (shared code)
   if (data.readshared) {
-    ensureFilesystem(data.preload_fs);
-    var contents = await readSharedFile(data.preload_fs, data.readshared);
+    var fs1 = splitWasiFSName(data.preload_fs);
+    var contents = fs1.wasi ? await readWasiSharedFile(fs1.name, data.readshared)
+      : (ensureFilesystem(fs1.name), await readSharedFile(fs1.name, data.readshared));
     return { output: contents, qid: data.qid } as WorkerResult;
   }
   // list files in a filesystem package directory (shared code)
   if (data.listshared != null) {
-    ensureFilesystem(data.preload_fs);
-    var files = listSharedFiles(data.preload_fs, data.listshared);
+    var fs2 = splitWasiFSName(data.preload_fs);
+    var files = fs2.wasi ? await listWasiSharedFiles(fs2.name, data.listshared)
+      : (ensureFilesystem(fs2.name), listSharedFiles(fs2.name, data.listshared));
     return { output: files, qid: data.qid } as WorkerResult;
   }
   // preload a filesystem package directly by name
   if (data.preload_fs) {
-    ensureFilesystem(data.preload_fs);
+    var fs3 = splitWasiFSName(data.preload_fs);
+    if (fs3.wasi) await ensureWasiFilesystem(fs3.name);
+    else ensureFilesystem(fs3.name);
     return;
   }
   // clear filesystem? (TODO: buildkey)

test/cli/testworker.js

@@ -115,6 +115,24 @@ describe('Worker', function() {
     assert.equal(msg.output, null);
     assert.equal(msg.qid, 125);
   });
+  it('should list shared files in WASI filesystem zip', async function() {
+    var msg = await queryWorker({preload_fs:'wasi:cc7800-fs.zip', listshared:'/headers', updates:[], buildsteps:[], qid:126});
+    assert.ok(Array.isArray(msg.output));
+    assert.ok(msg.output.length > 0);
+    assert.ok(msg.output.includes('headers/prosystem.h'));
+    assert.equal(msg.qid, 126);
+  });
+  it('should read a shared header from WASI filesystem zip', async function() {
+    var msg = await queryWorker({preload_fs:'wasi:cc2600-fs.zip', readshared:'/headers/vcs.h', updates:[], buildsteps:[], qid:127});
+    assert.ok(msg.output instanceof Uint8Array);
+    assert.ok(msg.output.length > 100);
+    assert.equal(msg.qid, 127);
+  });
+  it('should return null for missing file in WASI filesystem zip', async function() {
+    var msg = await queryWorker({preload_fs:'wasi:cc2600-fs.zip', readshared:'/headers/nosuchfile.h', updates:[], buildsteps:[], qid:128});
+    assert.equal(msg.output, null);
+    assert.equal(msg.qid, 128);
+  });
   it('should assemble DASM', function(done) {
     compile('dasm', '\tprocessor 6502\n\torg $f000\n MAC mack\n lda #0\n ENDM\nfoo: mack\n mack\n', 'vcs.mame', done, 4, 4);
   });

test/unit/testtoolmeta.ts

@@ -3,7 +3,7 @@ import { describe, it } from "mocha";
 import {
   TOOL_META, getToolMeta, getToolMetaForFilename, getPreloadFSName, getSkeletonName,
   getIncludePatterns, getLinkPatterns, matchDependencyPatterns,
-  getSystemIncludePatterns, SYSTEM_INCLUDE_PATTERNS
+  getSystemIncludePatterns, SYSTEM_INCLUDE_PATTERNS, getSharedFileSystemName
 } from "../../src/common/toolmeta";
 // Node-friendly worker entry point (re-exports TOOLS without Worker wiring)
 import { TOOLS } from "../../src/worker/workerlib";
@@ -119,6 +119,24 @@ describe('Tool metadata registry', function () {
     assert.deepStrictEqual(deps('#include "foo.h"\n', 'cc65'), []);
   });
 
+  it('resolves shared filesystem names for all tools with includeDirs', function () {
+    // preloadFS-backed tools
+    assert.equal(getSharedFileSystemName('cc65', 'nes'), '65-nes');
+    assert.equal(getSharedFileSystemName('sdcc', 'coleco'), 'sdcc');
+    assert.equal(getSharedFileSystemName('bataribasic', 'vcs'), '2600basic');
+    assert.equal(getSharedFileSystemName('wiz'), 'wiz');
+    // WASI zip-backed tools (cc2600/cc7800/armtcc/oscar64/dialog)
+    assert.equal(getSharedFileSystemName('cc2600'), 'wasi:cc2600-fs.zip');
+    assert.equal(getSharedFileSystemName('cc7800'), 'wasi:cc7800-fs.zip');
+    assert.equal(getSharedFileSystemName('armtcc'), 'wasi:arm32-fs.zip');
+    assert.equal(getSharedFileSystemName('oscar64'), 'wasi:oscar64-fs.zip');
+    assert.equal(getSharedFileSystemName('dialog'), 'wasi:dialog-fs.zip');
+    // tools without a bundled filesystem
+    assert.equal(getSharedFileSystemName('cmoc'), undefined);
+    assert.equal(getSharedFileSystemName('smlrc'), undefined);
+    assert.equal(getSharedFileSystemName('dasm'), undefined);
+  });
+
   it('matches link directives only for tools that link', function () {
     let deps = (text: string, tool?: string, platform?: string) =>
       matchDependencyPatterns(text, getLinkPatterns(tool, platform));