diff --git a/src/common/toolmeta.ts b/src/common/toolmeta.ts index 94eb3834..cfb7dcac 100644 --- a/src/common/toolmeta.ts +++ b/src/common/toolmeta.ts @@ -22,6 +22,8 @@ * (unless noWorkerBuild) and vice versa. */ +import { getRootBasePlatform } from "./util"; + export type ToolKind = | 'compiler' // source -> object/output (cc65, sdcc, cmoc, oscar64, ...) | 'assembler' // asm -> object (dasm, acme, ca65, ...) @@ -94,11 +96,15 @@ export interface ToolMeta { includePatterns?: (RegExp | ToolIncludePattern)[]; /** regexes matching link directives, used for dependency parsing */ linkPatterns?: (RegExp | ToolIncludePattern)[]; - /** directories inside the preload FS containing searchable shared code + /** directories inside the shared filesystem containing searchable shared code * (headers, asm includes). Paths are relative to the FS root as they appear - * in the filesystem package metadata, e.g. ['/include', '/asminc']. - * Only meaningful when paired with platforms[].preloadFS. */ + * in the filesystem metadata, e.g. ['/include', '/asminc', '/headers']. + * Used with platforms[].preloadFS or wasiFSZip. */ includeDirs?: string[]; + /** name of a WASI filesystem zip (in src/worker/fs/) that contains the + * toolchain's headers, for tools that don't use an emscripten preloadFS + * package (e.g. cc2600/cc7800). Referenced as 'wasi:' in worker msgs. */ + wasiFSZip?: string; // ---- per-platform metadata (was TOOL_PRELOADFS / PLATFORM_PARAMS) ---- @@ -374,7 +380,8 @@ export const TOOL_META: { [id: string]: ToolMeta } = { cmoc: { id: 'cmoc', name: 'CMOC', kind: 'compiler', arch: '6809', extensions: ['.c', '.h'], - includeDirs: ['/include'], + // NOTE: no bundled filesystem -- the compiler passes -I/share/include but + // never mounts one, so system headers can't be linked in the UI editorStyle: 'text/x-csrc', helpURL: 'http://perso.b2b2c.ca/~sarrazip/dev/cmoc.html', wasmModule: 'cmoc', @@ -420,6 +427,7 @@ export const TOOL_META: { [id: string]: ToolMeta } = { id: 'armtcc', name: 'TCC (ARM)', kind: 'compiler', arch: 'arm32', extensions: ['.c', '.s'], includeDirs: ['/include'], + wasiFSZip: 'arm32-fs.zip', editorStyle: 'text/x-csrc', wasmModule: 'arm-tcc', includePatterns: SHARED_INCLUDE_PATTERNS, @@ -437,7 +445,7 @@ export const TOOL_META: { [id: string]: ToolMeta } = { smlrc: { id: 'smlrc', name: 'SmallerC', kind: 'compiler', arch: 'x86', extensions: ['.c'], - includeDirs: ['/include'], + // NOTE: no bundled filesystem and no -I arg -- no headers to link in the UI editorStyle: 'text/x-csrc', wasmModule: 'smlrc', includePatterns: SHARED_INCLUDE_PATTERNS, @@ -459,6 +467,7 @@ export const TOOL_META: { [id: string]: ToolMeta } = { id: 'oscar64', name: 'Oscar64', kind: 'compiler', arch: '6502', extensions: ['.c', '.cpp', '.cc', '.o64'], includeDirs: ['/include'], + wasiFSZip: 'oscar64-fs.zip', editorStyle: 'text/x-csrc', helpURL: 'https://github.com/drmortalwombat/oscar64/blob/main/oscar64.md', wasmModule: 'oscar64', @@ -494,6 +503,8 @@ export const TOOL_META: { [id: string]: ToolMeta } = { extensions: ['.cc2600'], editorStyle: 'text/x-csrc', wasmModule: 'cc2600', + wasiFSZip: 'cc2600-fs.zip', + includeDirs: ['/headers'], includePatterns: SHARED_INCLUDE_PATTERNS, linkPatterns: SHARED_LINK_PATTERNS, }, @@ -503,6 +514,8 @@ export const TOOL_META: { [id: string]: ToolMeta } = { extensions: ['.cc7800', '.c78'], editorStyle: 'text/x-csrc', wasmModule: 'cc7800', + wasiFSZip: 'cc7800-fs.zip', + includeDirs: ['/headers'], includePatterns: SHARED_INCLUDE_PATTERNS, linkPatterns: SHARED_LINK_PATTERNS, }, @@ -518,7 +531,7 @@ export const TOOL_META: { [id: string]: ToolMeta } = { wiz: { id: 'wiz', name: 'wiz', kind: 'compiler', extensions: ['.wiz'], - includeDirs: ['/common'], + includeDirs: ['/common/$platform'], editorStyle: 'text/x-wiz', helpURL: 'https://github.com/wiz-lang/wiz/blob/master/readme.md#wiz', wasmModule: 'wiz', @@ -552,6 +565,8 @@ export const TOOL_META: { [id: string]: ToolMeta } = { editorStyle: 'dialog', helpURL: 'https://linusakesson.net/dialog/docs/', wasmModule: 'dialogc', + wasiFSZip: 'dialog-fs.zip', + includeDirs: ['/'], includePatterns: DIALOG_INCLUDE_PATTERNS, }, @@ -672,7 +687,7 @@ export function getSkeletonName(tool: string): string { * worked before this registry. A tool that really has no include directives * says so with an explicit empty list. */ -export function getIncludePatterns(tool?: string, platform?: string): (RegExp | ToolIncludePattern)[] { +export function getIncludePatterns(tool: string, platform?: string): (RegExp | ToolIncludePattern)[] { let meta = tool && getToolMeta(tool); if (meta && meta.includePatterns) return meta.includePatterns; if (platform && platform.startsWith('verilog')) return VERILOG_INCLUDE_PATTERNS; @@ -680,7 +695,7 @@ export function getIncludePatterns(tool?: string, platform?: string): (RegExp | } /** Link patterns ("//#link") to use when scanning a source file. */ -export function getLinkPatterns(tool?: string, platform?: string): (RegExp | ToolIncludePattern)[] { +export function getLinkPatterns(tool: string, platform?: string): (RegExp | ToolIncludePattern)[] { let meta = tool && getToolMeta(tool); if (meta && meta.linkPatterns) return meta.linkPatterns; if (platform && platform.startsWith('verilog')) return []; @@ -688,12 +703,28 @@ export function getLinkPatterns(tool?: string, platform?: string): (RegExp | Too } /** - * Directories containing shared code (headers etc.) for this tool's preload - * filesystem, e.g. ['/include','/asminc'] -- empty if none/unknown. + * Directories containing shared code (headers etc.) for this tool's shared + * filesystem, e.g. ['/include','/asminc','/headers'] -- empty if none/unknown. */ -export function getIncludeDirs(tool?: string): string[] { +export function getIncludeDirs(tool: string, platform_id: string): string[] { + platform_id = platform_id && getRootBasePlatform(platform_id); let meta = tool && getToolMeta(tool); - return (meta && meta.includeDirs) || []; + let out = (meta && meta.includeDirs) || []; + out = out.map(s => s.replace('$platform', platform_id)); + return out; +} + +/** + * Name of the shared filesystem to query for this tool's bundled files: + * either the emscripten preloadFS package name (e.g. '65-nes') or, for tools + * with a WASI filesystem zip (e.g. cc2600), 'wasi:'. + * Undefined if the tool has no bundled filesystem. + */ +export function getSharedFileSystemName(tool: string, platform?: string): string | undefined { + var fsName = getPreloadFSName(tool, platform); + if (fsName) return fsName; + let meta = tool && getToolMeta(tool); + return (meta && meta.wasiFSZip) ? 'wasi:' + meta.wasiFSZip : undefined; } /** diff --git a/src/ide/ui.ts b/src/ide/ui.ts index fc73cb5e..042f3a3c 100644 --- a/src/ide/ui.ts +++ b/src/ide/ui.ts @@ -289,10 +289,10 @@ var openHeaderViews: string[] = []; export function openHeaderFile(fn: string) { if (!openHeaderViews.includes(fn)) { openHeaderViews.push(fn); - projectWindows.setCreateFunc('#headerview/' + encodeURIComponent(fn), () => new HeaderView(fn)); + projectWindows.setCreateFunc('#headerview/' + fn, () => new HeaderView(fn)); refreshWindowList(); } - var hash = '#headerview/' + encodeURIComponent(fn); + var hash = '#headerview/' + fn; if (window.location.hash !== hash) { window.location.hash = hash.substring(1); } @@ -369,7 +369,7 @@ function refreshWindowList() { // add opened toolchain headers (from include badges) for (let hdrfn of openHeaderViews) { - let hdrid = '#headerview/' + encodeURIComponent(hdrfn); + let hdrid = '#headerview/' + hdrfn; projectWindows.setCreateFunc(hdrid, () => new HeaderView(hdrfn)); addWindowItem(hdrid, getFilenameForPath(hdrfn), () => { return new HeaderView(hdrfn); diff --git a/src/ide/views/editors.ts b/src/ide/views/editors.ts index ca4c5e08..532058e5 100644 --- a/src/ide/views/editors.ts +++ b/src/ide/views/editors.ts @@ -8,7 +8,7 @@ import { crosshairCursor, drawSelection, dropCursor, EditorView, highlightActive import { CodeAnalyzer } from "../../common/analysis"; import { ProbeFlags, ProbeRecorder } from "../../common/probe"; import { getFilenameForPath, getFolderForPath, hex, rpad } from "../../common/util"; -import { getIncludeDirs, getIncludePatterns, getLinkPatterns, getPreloadFSName, getSystemIncludePatterns } from "../../common/toolmeta"; +import { getIncludeDirs, getIncludePatterns, getLinkPatterns, getSharedFileSystemName, getSystemIncludePatterns } from "../../common/toolmeta"; import { WorkerMessage } from "../../common/workertypes"; import { SourceFile, SourceLocation, WorkerError } from "../../common/workertypes"; import { asm6502 } from "../../parser/lang-6502"; @@ -965,14 +965,18 @@ const sharedFileCache = new Map>(); export function lookupSharedFileText(fn: string): Promise { const tool = current_project.getToolForFilename(current_project.mainPath); - const fsName = getPreloadFSName(tool, current_project.platform_id); - const dirs = getIncludeDirs(tool); + const fsName = getSharedFileSystemName(tool, current_project.platform_id); + const dirs = getIncludeDirs(tool, current_project.platform_id); if (!fsName || !dirs.length) return Promise.resolve(null); const key = fsName + ':' + fn; if (!sharedFileCache.has(key)) { sharedFileCache.set(key, (async () => { - for (var dir of dirs) { - var msg = { preload_fs: fsName, readshared: dir + '/' + fn, updates: [], buildsteps: [] } as WorkerMessage; + // try the include under each known dir (e.g. /headers/vcs.h for "vcs.h"), + // then the filename as-is (some includes already carry the dir, + // e.g. "headers/vcs.h") + var candidates = [...dirs.map(dir => dir + '/' + fn), fn]; + for (var path of candidates) { + var msg = { preload_fs: fsName, readshared: path, updates: [], buildsteps: [] } as WorkerMessage; var result = await current_project.queryWorker(msg); var output = result && (result as any).output; if (output instanceof Uint8Array && output.length > 0) { diff --git a/src/ide/views/includedecorations.ts b/src/ide/views/includedecorations.ts index fac1f6c4..ff068cc5 100644 --- a/src/ide/views/includedecorations.ts +++ b/src/ide/views/includedecorations.ts @@ -37,8 +37,11 @@ function buildIncludeLinkDecorations(view: EditorView, patterns: (RegExp | ToolI p.re.lastIndex = 0; // patterns are shared and global, so rewind first let m; while ((m = p.re.exec(text))) { - const filename = m[p.group != null ? p.group : m.length - 1]; - if (!filename) continue; + const matched = m[p.group != null ? p.group : m.length - 1]; + if (!matched) continue; + // apply any implicit extension (e.g. wiz `import "nes"` -> "nes.wiz"), + // same as matchDependencyPatterns() does for build deps + const filename = p.suffix ? matched + p.suffix : matched; // place widget at end of the line containing the match const matchEnd = Math.min(from + m.index + m[0].length, doc.length); const line = doc.lineAt(matchEnd); diff --git a/src/worker/wasmutils.ts b/src/worker/wasmutils.ts index 9863044b..3b8513b9 100644 --- a/src/worker/wasmutils.ts +++ b/src/worker/wasmutils.ts @@ -218,6 +218,58 @@ export function listSharedFiles(name: string, dir: string): string[] { return result.sort(); } +/// shared code in WASI filesystem zips (e.g. cc2600/cc7800 headers) +// referenced by worker msgs as 'wasi:' + +import { loadWASIFilesystemZip } from './wasiutils'; +import type { WASIMemoryFilesystem } from '../common/wasi/wasishim'; + +var wasiFilesystems: { [zipname: string]: WASIMemoryFilesystem } = {}; + +export async function ensureWasiFilesystem(zipname: string): Promise { + try { + if (!wasiFilesystems[zipname]) { + wasiFilesystems[zipname] = await loadWASIFilesystemZip(zipname); + } + return wasiFilesystems[zipname]; + } catch (e) { + console.log('Error loading WASI filesystem', zipname, e); + return null; + } +} + +// zip entries are stored under '' (rootPath './' adds a './' prefix); +// strip it plus any leading '/' so '/headers/vcs.h' matches 'headers/vcs.h' +function normalizeWasiPath(path: string): string { + path = path.startsWith('./') ? path.substring(2) : path; + return path.startsWith('/') ? path.substring(1) : path; +} + +/** Read a file from a WASI filesystem zip. Returns null if not found. */ +export async function readWasiSharedFile(zipname: string, path: string): Promise { + let fs = await ensureWasiFilesystem(zipname); + if (!fs) return null; + let norm = normalizeWasiPath(path); + // zip entries are stored under './', so try both key forms + let fd = fs.getFile(norm) || fs.getFile('./' + norm); + if (!fd || !fd.size) return null; + let data = new Uint8Array(fd.size); + fd.offset = 0; + fd.read(data); + return data; +} + +/** List files under a directory prefix (e.g. '/headers') of a WASI filesystem zip. */ +export async function listWasiSharedFiles(zipname: string, dir: string): Promise { + let fs = await ensureWasiFilesystem(zipname); + if (!fs) return []; + let prefix = normalizeWasiPath(dir) + '/'; + return fs.getFiles() + .map(f => normalizeWasiPath(f.name)) + .filter(name => name.startsWith(prefix)) + .sort(); +} + /** Read a file from a filesystem package blob. Returns null if not found. * Async because Blob reading is async; call from worker message handlers. */ export async function readSharedFile(name: string, path: string): Promise { diff --git a/src/worker/workerlib.ts b/src/worker/workerlib.ts index 1cec4ed4..3e27eb89 100644 --- a/src/worker/workerlib.ts +++ b/src/worker/workerlib.ts @@ -9,7 +9,12 @@ import type { WorkerResult, WorkerMessage } from "../common/workertypes"; import { getBasePlatform, getRootBasePlatform } from "../common/util"; import { getPreloadFSName } from "../common/toolmeta"; import { store, builder } from "./builder"; -import { emglobal, fsMeta, loadFilesystem, listSharedFiles, readSharedFile, ensureFilesystem } from "./wasmutils"; +import { emglobal, fsMeta, loadFilesystem, listSharedFiles, readSharedFile, ensureFilesystem, readWasiSharedFile, listWasiSharedFiles, ensureWasiFilesystem } from "./wasmutils"; + +// shared FS names starting with 'wasi:' refer to a WASI filesystem zip +function splitWasiFSName(fsName: string): { wasi: boolean, name: string } { + return fsName.startsWith('wasi:') ? { wasi: true, name: fsName.substring(5) } : { wasi: false, name: fsName }; +} import { setupRequireFunction } from "./workermain"; export { store, builder }; @@ -139,19 +144,23 @@ export async function handleMessage(data: WorkerMessage): Promise } // read a file from a filesystem package (shared code) if (data.readshared) { - ensureFilesystem(data.preload_fs); - var contents = await readSharedFile(data.preload_fs, data.readshared); + var fs1 = splitWasiFSName(data.preload_fs); + var contents = fs1.wasi ? await readWasiSharedFile(fs1.name, data.readshared) + : (ensureFilesystem(fs1.name), await readSharedFile(fs1.name, data.readshared)); return { output: contents, qid: data.qid } as WorkerResult; } // list files in a filesystem package directory (shared code) if (data.listshared != null) { - ensureFilesystem(data.preload_fs); - var files = listSharedFiles(data.preload_fs, data.listshared); + var fs2 = splitWasiFSName(data.preload_fs); + var files = fs2.wasi ? await listWasiSharedFiles(fs2.name, data.listshared) + : (ensureFilesystem(fs2.name), listSharedFiles(fs2.name, data.listshared)); return { output: files, qid: data.qid } as WorkerResult; } // preload a filesystem package directly by name if (data.preload_fs) { - ensureFilesystem(data.preload_fs); + var fs3 = splitWasiFSName(data.preload_fs); + if (fs3.wasi) await ensureWasiFilesystem(fs3.name); + else ensureFilesystem(fs3.name); return; } // clear filesystem? diff --git a/src/worker/workermain.ts b/src/worker/workermain.ts index 8a4ff4b6..f28c751f 100644 --- a/src/worker/workermain.ts +++ b/src/worker/workermain.ts @@ -3,7 +3,12 @@ import type { WorkerResult, WorkerMessage, WorkerError, SourceLine } from "../co import { getBasePlatform, getRootBasePlatform } from "../common/util"; import { getPreloadFSName } from "../common/toolmeta"; import { store, builder, errorResult, getWorkFileAsString } from "./builder"; -import { emglobal, fsMeta, loadFilesystem, listSharedFiles, readSharedFile, ensureFilesystem } from "./wasmutils"; +import { emglobal, fsMeta, loadFilesystem, listSharedFiles, readSharedFile, ensureFilesystem, readWasiSharedFile, listWasiSharedFiles, ensureWasiFilesystem } from "./wasmutils"; + +// shared FS names starting with 'wasi:' refer to a WASI filesystem zip +function splitWasiFSName(fsName: string): { wasi: boolean, name: string } { + return fsName.startsWith('wasi:') ? { wasi: true, name: fsName.substring(5) } : { wasi: false, name: fsName }; +} declare function importScripts(path: string); declare function postMessage(msg); @@ -40,19 +45,23 @@ async function handleMessage(data: WorkerMessage): Promise { } // read a file from a filesystem package (shared code) if (data.readshared) { - ensureFilesystem(data.preload_fs); - var contents = await readSharedFile(data.preload_fs, data.readshared); + var fs1 = splitWasiFSName(data.preload_fs); + var contents = fs1.wasi ? await readWasiSharedFile(fs1.name, data.readshared) + : (ensureFilesystem(fs1.name), await readSharedFile(fs1.name, data.readshared)); return { output: contents, qid: data.qid } as WorkerResult; } // list files in a filesystem package directory (shared code) if (data.listshared != null) { - ensureFilesystem(data.preload_fs); - var files = listSharedFiles(data.preload_fs, data.listshared); + var fs2 = splitWasiFSName(data.preload_fs); + var files = fs2.wasi ? await listWasiSharedFiles(fs2.name, data.listshared) + : (ensureFilesystem(fs2.name), listSharedFiles(fs2.name, data.listshared)); return { output: files, qid: data.qid } as WorkerResult; } // preload a filesystem package directly by name if (data.preload_fs) { - ensureFilesystem(data.preload_fs); + var fs3 = splitWasiFSName(data.preload_fs); + if (fs3.wasi) await ensureWasiFilesystem(fs3.name); + else ensureFilesystem(fs3.name); return; } // clear filesystem? (TODO: buildkey) diff --git a/test/cli/testworker.js b/test/cli/testworker.js index 6bb6c461..99dd69b6 100644 --- a/test/cli/testworker.js +++ b/test/cli/testworker.js @@ -115,6 +115,24 @@ describe('Worker', function() { assert.equal(msg.output, null); assert.equal(msg.qid, 125); }); + it('should list shared files in WASI filesystem zip', async function() { + var msg = await queryWorker({preload_fs:'wasi:cc7800-fs.zip', listshared:'/headers', updates:[], buildsteps:[], qid:126}); + assert.ok(Array.isArray(msg.output)); + assert.ok(msg.output.length > 0); + assert.ok(msg.output.includes('headers/prosystem.h')); + assert.equal(msg.qid, 126); + }); + it('should read a shared header from WASI filesystem zip', async function() { + var msg = await queryWorker({preload_fs:'wasi:cc2600-fs.zip', readshared:'/headers/vcs.h', updates:[], buildsteps:[], qid:127}); + assert.ok(msg.output instanceof Uint8Array); + assert.ok(msg.output.length > 100); + assert.equal(msg.qid, 127); + }); + it('should return null for missing file in WASI filesystem zip', async function() { + var msg = await queryWorker({preload_fs:'wasi:cc2600-fs.zip', readshared:'/headers/nosuchfile.h', updates:[], buildsteps:[], qid:128}); + assert.equal(msg.output, null); + assert.equal(msg.qid, 128); + }); it('should assemble DASM', function(done) { compile('dasm', '\tprocessor 6502\n\torg $f000\n MAC mack\n lda #0\n ENDM\nfoo: mack\n mack\n', 'vcs.mame', done, 4, 4); }); diff --git a/test/unit/testtoolmeta.ts b/test/unit/testtoolmeta.ts index 0d4d64a8..5c109408 100644 --- a/test/unit/testtoolmeta.ts +++ b/test/unit/testtoolmeta.ts @@ -3,7 +3,7 @@ import { describe, it } from "mocha"; import { TOOL_META, getToolMeta, getToolMetaForFilename, getPreloadFSName, getSkeletonName, getIncludePatterns, getLinkPatterns, matchDependencyPatterns, - getSystemIncludePatterns, SYSTEM_INCLUDE_PATTERNS + getSystemIncludePatterns, SYSTEM_INCLUDE_PATTERNS, getSharedFileSystemName } from "../../src/common/toolmeta"; // Node-friendly worker entry point (re-exports TOOLS without Worker wiring) import { TOOLS } from "../../src/worker/workerlib"; @@ -119,6 +119,24 @@ describe('Tool metadata registry', function () { assert.deepStrictEqual(deps('#include "foo.h"\n', 'cc65'), []); }); + it('resolves shared filesystem names for all tools with includeDirs', function () { + // preloadFS-backed tools + assert.equal(getSharedFileSystemName('cc65', 'nes'), '65-nes'); + assert.equal(getSharedFileSystemName('sdcc', 'coleco'), 'sdcc'); + assert.equal(getSharedFileSystemName('bataribasic', 'vcs'), '2600basic'); + assert.equal(getSharedFileSystemName('wiz'), 'wiz'); + // WASI zip-backed tools (cc2600/cc7800/armtcc/oscar64/dialog) + assert.equal(getSharedFileSystemName('cc2600'), 'wasi:cc2600-fs.zip'); + assert.equal(getSharedFileSystemName('cc7800'), 'wasi:cc7800-fs.zip'); + assert.equal(getSharedFileSystemName('armtcc'), 'wasi:arm32-fs.zip'); + assert.equal(getSharedFileSystemName('oscar64'), 'wasi:oscar64-fs.zip'); + assert.equal(getSharedFileSystemName('dialog'), 'wasi:dialog-fs.zip'); + // tools without a bundled filesystem + assert.equal(getSharedFileSystemName('cmoc'), undefined); + assert.equal(getSharedFileSystemName('smlrc'), undefined); + assert.equal(getSharedFileSystemName('dasm'), undefined); + }); + it('matches link directives only for tools that link', function () { let deps = (text: string, tool?: string, platform?: string) => matchDependencyPatterns(text, getLinkPatterns(tool, platform));