cli: better but still needs work

src/tools/8bws.ts

@@ -3,7 +3,7 @@
 // 8bws - 8bitworkshop CLI tool for compilation, ROM execution, and platform info
 
 import * as fs from 'fs';
-import { initialize, compile, compileSourceFile, preload, listTools, listPlatforms, PLATFORM_PARAMS, TOOLS, TOOL_PRELOADFS } from './testlib';
+import { initialize, compile, compileSourceFile, preload, listTools, listPlatforms, getToolForFilename, PLATFORM_PARAMS, TOOLS, TOOL_PRELOADFS } from './testlib';
 
 interface CLIResult {
   success: boolean;
@@ -22,8 +22,8 @@ function usage(): void {
     command: 'help',
     data: {
       commands: {
-        'compile': 'compile --tool <tool> --platform <platform> [--output <file>] <source>',
-        'check': 'check --tool <tool> --platform <platform> <source>',
+        'compile': 'compile --platform <platform> [--tool <tool>] [--output <file>] <source>',
+        'check': 'check --platform <platform> [--tool <tool>] <source>',
         'run': 'run --platform <platform> [--frames N] <rom>',
         'list-tools': 'list-tools',
         'list-platforms': 'list-platforms',
@@ -61,15 +61,20 @@ async function doCompile(args: { [key: string]: string }, positional: string[],
   var outputFile = args['output'];
   var sourceFile = positional[0];
 
-  if (!tool || !platform || !sourceFile) {
+  if (!platform || !sourceFile) {
     outputJSON({
       success: false,
       command: checkOnly ? 'check' : 'compile',
-      error: 'Required: --tool <tool> --platform <platform> <source>'
+      error: 'Required: --platform <platform> <source> [--tool <tool>]'
     });
     process.exit(1);
   }
 
+  // Auto-detect tool from filename if not specified
+  if (!tool) {
+    tool = getToolForFilename(sourceFile, platform);
+  }
+
   if (!TOOLS[tool]) {
     outputJSON({
       success: false,

src/tools/testlib.ts

@@ -1,9 +1,13 @@
 
 // testlib - Clean async API for compiling and testing 8bitworkshop projects
 // Wraps the worker build system for use in tests and CLI tools
+// FOR TESTING ONLY
 
 import * as fs from 'fs';
-import type { WorkerResult, WorkerMessage, WorkerErrorResult, WorkerOutputResult } from "../common/workertypes";
+import * as path from 'path';
+import type { WorkerResult, WorkerMessage, WorkerErrorResult, WorkerOutputResult, Dependency } from "../common/workertypes";
+import { getFolderForPath, isProbablyBinary, getBasePlatform } from "../common/util";
+import { getToolForFilename_z80, getToolForFilename_6502, getToolForFilename_6809 } from "../common/baseplatform";
 import { setupNodeEnvironment, handleMessage, store, TOOL_PRELOADFS } from "../worker/workerlib";
 import { PLATFORM_PARAMS } from "../worker/platforms";
 import { TOOLS } from "../worker/workertools";
@@ -103,19 +107,258 @@ export async function compileFile(tool: string, platform: string, presetPath: st
   });
 }
 
+/**
+ * Parse include and link dependencies from source text.
+ * Extracted from CodeProject.parseIncludeDependencies / parseLinkDependencies.
+ * TODO: project.ts should be refactored so we don't have to duplicate the logic
+ */
+function parseIncludeDependencies(text: string, platformId: string, mainPath: string): string[] {
+  let files: string[] = [];
+  let m;
+  var dir = getFolderForPath(mainPath);
+
+  function pushFile(fn: string) {
+    files.push(fn);
+    if (dir.length > 0 && dir != 'local')
+      files.push(dir + '/' + fn);
+  }
+
+  if (platformId.startsWith('verilog')) {
+    let re1 = /^\s*(`include|[.]include)\s+"(.+?)"/gmi;
+    while (m = re1.exec(text)) { pushFile(m[2]); }
+    let re1a = /^\s*\$(include|\$dofile|\$write_image_in_table)\('(.+?)'/gmi;
+    while (m = re1a.exec(text)) { pushFile(m[2]); }
+    let re2 = /^\s*([.]arch)\s+(\w+)/gmi;
+    while (m = re2.exec(text)) { pushFile(m[2] + ".json"); }
+    let re3 = /\$readmem[bh]\("(.+?)"/gmi;
+    while (m = re3.exec(text)) { pushFile(m[1]); }
+  } else {
+    let re2 = /^\s*[.#%]?(include|incbin|embed)\s+"(.+?)"/gmi;
+    while (m = re2.exec(text)) { pushFile(m[2]); }
+    let re3 = /^\s*([;']|[/][/])#(resource)\s+"(.+?)"/gm;
+    while (m = re3.exec(text)) { pushFile(m[3]); }
+    let re4 = /^\s+(USE|ASM)\s+(\S+[.]\S+)/gm;
+    while (m = re4.exec(text)) { pushFile(m[2]); }
+    let re5 = /^\s*(import|embed)\s*"(.+?)";/gmi;
+    while (m = re5.exec(text)) {
+      if (m[1] == 'import') pushFile(m[2] + ".wiz");
+      else pushFile(m[2]);
+    }
+    let re6 = /^\s*(import)\s*"(.+?)"/gmi;
+    while (m = re6.exec(text)) { pushFile(m[2]); }
+    let re7 = /^[!]src\s+"(.+?)"/gmi;
+    while (m = re7.exec(text)) { pushFile(m[1]); }
+  }
+  return files;
+}
+
+function parseLinkDependencies(text: string, platformId: string, mainPath: string): string[] {
+  let files: string[] = [];
+  let m;
+  var dir = getFolderForPath(mainPath);
+
+  function pushFile(fn: string) {
+    files.push(fn);
+    if (dir.length > 0 && dir != 'local')
+      files.push(dir + '/' + fn);
+  }
+
+  if (!platformId.startsWith('verilog')) {
+    let re = /^\s*([;]|[/][/])#link\s+"(.+?)"/gm;
+    while (m = re.exec(text)) { pushFile(m[2]); }
+  }
+  return files;
+}
+
+type FileData = string | Uint8Array;
+
+interface ResolvedFile {
+  path: string;       // path as referenced (may include folder prefix)
+  filename: string;   // stripped filename for the worker
+  data: FileData;
+  link: boolean;
+}
+
+/**
+ * Try to resolve a file path by searching the source directory,
+ * the presets directory for the platform, and the current working directory.
+ */
+function resolveFileData(filePath: string, sourceDir: string, platform: string): FileData | null {
+  var searchPaths = [];
+
+  // Try relative to source file directory
+  if (sourceDir) {
+    searchPaths.push(path.resolve(sourceDir, filePath));
+  }
+
+  // Try presets directory
+  var basePlatform = getBasePlatform(platform);
+  searchPaths.push(path.resolve('presets', basePlatform, filePath));
+
+  // Try current working directory
+  searchPaths.push(path.resolve(filePath));
+
+  for (var p of searchPaths) {
+    try {
+      if (fs.existsSync(p)) {
+        if (isProbablyBinary(filePath)) {
+          return new Uint8Array(fs.readFileSync(p));
+        } else {
+          return fs.readFileSync(p, 'utf-8');
+        }
+      }
+    } catch (e) {
+      // continue searching
+    }
+  }
+  return null;
+}
+
+/**
+ * Strips the main file's folder prefix from a path (matching CodeProject.stripLocalPath).
+ */
+function stripLocalPath(filePath: string, mainPath: string): string {
+  var folder = getFolderForPath(mainPath);
+  if (folder != '' && filePath.startsWith(folder + '/')) {
+    filePath = filePath.substring(folder.length + 1);
+  }
+  return filePath;
+}
+
+/**
+ * Recursively resolve all file dependencies for a source file.
+ */
+function resolveAllDependencies(
+  mainText: string, mainPath: string, platform: string, sourceDir: string
+): ResolvedFile[] {
+  var resolved: ResolvedFile[] = [];
+  var seen = new Set<string>();
+
+  function resolve(text: string, currentPath: string) {
+    var includes = parseIncludeDependencies(text, platform, currentPath);
+    var links = parseLinkDependencies(text, platform, currentPath);
+    var allPaths = includes.concat(links);
+    var linkSet = new Set(links);
+
+    for (var depPath of allPaths) {
+      var filename = stripLocalPath(depPath, mainPath);
+      if (seen.has(filename)) continue;
+      seen.add(filename);
+
+      var data = resolveFileData(depPath, sourceDir, platform);
+      if (data != null) {
+        resolved.push({
+          path: depPath,
+          filename: filename,
+          data: data,
+          link: linkSet.has(depPath),
+        });
+        // Recursively parse text files for their own dependencies
+        if (typeof data === 'string') {
+          resolve(data, depPath);
+        }
+      }
+    }
+  }
+
+  resolve(mainText, mainPath);
+  return resolved;
+}
+
+// TODO: refactor dependency parsing and tool selection into a common library
+// shared between CodeProject (src/ide/project.ts) and testlib
+
+/**
+ * Select the appropriate tool for a filename based on platform architecture.
+ */
+export function getToolForFilename(fn: string, platform: string): string {
+  var params = PLATFORM_PARAMS[getBasePlatform(platform)];
+  var arch = params && params.arch;
+  switch (arch) {
+    case 'z80':
+    case 'gbz80':
+      return getToolForFilename_z80(fn);
+    case '6502':
+      return getToolForFilename_6502(fn);
+    case '6809':
+      return getToolForFilename_6809(fn);
+    default:
+      return getToolForFilename_z80(fn); // fallback
+  }
+}
+
 /**
  * Compile an arbitrary source file path.
+ * Parses include/link/resource directives and loads dependent files.
  */
 export async function compileSourceFile(tool: string, platform: string, filePath: string): Promise<CompileResult> {
   await initialize();
 
   var code = fs.readFileSync(filePath, 'utf-8');
   var basename = filePath.split('/').pop();
+  var sourceDir = path.dirname(path.resolve(filePath));
+
+  // Auto-detect tool from filename if not specified
+  if (!tool) {
+    tool = getToolForFilename(basename, platform);
+  }
+
+  // Parse and resolve all dependencies
+  var deps = resolveAllDependencies(code, basename, platform, sourceDir);
+
+  if (deps.length === 0) {
+    // No dependencies found, use simple single-file path
+    return compile({
+      tool: tool,
+      platform: platform,
+      code: code,
+      path: basename,
+    });
+  }
+
+  // Build multi-file message with updates and buildsteps
+  var files: { path: string; data: string | Uint8Array }[] = [];
+  var depFilenames: string[] = [];
+
+  // Main file first
+  files.push({ path: basename, data: code });
+
+  // Include files (non-link dependencies)
+  for (var dep of deps) {
+    if (!dep.link) {
+      files.push({ path: dep.filename, data: dep.data });
+      depFilenames.push(dep.filename);
+    }
+  }
+
+  // Build steps: main file first
+  var buildsteps: any[] = [];
+  buildsteps.push({
+    path: basename,
+    files: [basename].concat(depFilenames),
+    platform: platform,
+    tool: tool,
+    mainfile: true,
+  });
+
+  // Link dependencies get their own build steps, with tool selected by extension
+  for (var dep of deps) {
+    if (dep.link && dep.data) {
+      files.push({ path: dep.filename, data: dep.data });
+      buildsteps.push({
+        path: dep.filename,
+        files: [dep.filename].concat(depFilenames),
+        platform: platform,
+        tool: getToolForFilename(dep.filename, platform),
+      });
+    }
+  }
+
   return compile({
     tool: tool,
     platform: platform,
-    code: code,
-    path: basename,
+    files: files,
+    buildsteps: buildsteps,
   });
 }
 

src/worker/workerlib.ts

@@ -1,6 +1,7 @@
 
 // workerlib.ts - Node.js-friendly entry point for the worker build system
 // Re-exports core worker functionality without Web Worker onmessage/postMessage wiring
+// FOR TESTING ONLY
 
 import * as fs from 'fs';
 import * as path from 'path';
@@ -46,6 +47,14 @@ class Blob {
  */
 export function setupNodeEnvironment() {
   // Basic globals expected by various parts of the worker system
+  // Some Emscripten-generated WASM modules check for __filename/__dirname
+  if (typeof globalThis.__filename === 'undefined') {
+    (globalThis as any).__filename = __filename;
+  }
+  if (typeof globalThis.__dirname === 'undefined') {
+    (globalThis as any).__dirname = __dirname;
+    // TODO: support require('path').dirname
+  }
   emglobal.window = emglobal;
   emglobal.exports = {};
   emglobal.self = emglobal;