oscar64: parse listings/symbols/map

src/common/wasi/wasishim.ts

@@ -622,7 +622,7 @@ export class WASIRunner {
         const filename = this.peekUTF8(path_ptr, path_len);
         const path = filename.startsWith('/') ? filename : dir.name + '/' + filename; // TODO?
         const fd = this.fs.getFile(path);
-        console.log("path_filestat_get", dir+"", filename, path, filestat_ptr, '->', fd+"");
+        debug("path_filestat_get", dir+"", filename, path, filestat_ptr, '->', fd+"");
         if (!fd) return WASIErrors.NOENT;
         this.poke_filestat(filestat_ptr, fd);
         return WASIErrors.SUCCESS;

src/worker/server/buildenv.ts

@@ -5,6 +5,7 @@ import { spawn } from 'child_process';
 import { CodeListing, CodeListingMap, Segment, WorkerBuildStep, WorkerErrorResult, WorkerFileUpdate, WorkerResult, isOutputResult } from '../../common/workertypes';
 import { getFilenamePrefix, getRootBasePlatform, replaceAll } from '../../common/util';
 import { parseObjDump } from './clang';
+import { parseOscar64Listing, parseOscar64Lbl, parseOscar64Map } from '../tools/oscar64parse';
 import { BuildStep } from '../builder';
 import { makeErrorMatcher } from '../listingutils';
 
@@ -119,72 +120,30 @@ async function oscar64ProcessOutput(step: WorkerBuildStep, outpath: string): Pro
     let symbolmap: { [sym: string]: number } = {};
     let debuginfo = {};
     let segments: Segment[] = [];
-    // read segments
+    // parse .map file for segments and symbols
     {
         let txt = await fs.promises.readFile(prefix_path + '.map', { encoding: 'utf-8' });
-        for (let line of txt.split("\n")) {
-            // 0880 - 0887 : DATA, code
-            const m1 = line.match(/([0-9a-f]+) - ([0-9a-f]+) : ([A-Z_]+), (.+)/);
-            if (m1) {
-                const name = m1[4];
-                const start = parseInt(m1[1], 16);
-                const end = parseInt(m1[2], 16);
-                segments.push({
-                    name, start, size: end - start,
-                });
-            }
-            // 0801 (0062) : startup, NATIVE_CODE:startup
-            const m2 = line.match(/([0-9a-f]+) \(([0-9a-f]+)\) : ([^,]+), (.+)/);
-            if (m2) {
-                const addr = parseInt(m2[1], 16);
-                const name = m2[3];
-                symbolmap[name] = addr;
-            }
-        }
+        let parsed = parseOscar64Map(txt);
+        segments = parsed.segments;
+        symbolmap = parsed.symbolmap;
     }
-    // read listings
+    // parse .lbl file for any extra symbols
+    {
+        try {
+            let txt = await fs.promises.readFile(prefix_path + '.lbl', { encoding: 'utf-8' });
+            symbolmap = Object.assign(parseOscar64Lbl(txt), symbolmap);
+        } catch (e) { /* .lbl file not present */ }
+    }
+    // parse .asm listing
     {
         let txt = await fs.promises.readFile(prefix_path + '.asm', { encoding: 'utf-8' });
-        let lst : CodeListing = { lines: [], text: txt };
-        let asm_lineno = 0;
-        let c_lineno = 0;
-        let c_path = '';
-        const path = step.path;
-        for (let line of txt.split("\n")) {
-            asm_lineno++;
-            //;   4, "/Users/sehugg/PuzzlingPlans/8bitworkshop/server-root/oscar64/main.c"
-            let m2 = line.match(/;\s*(\d+), "(.+?)"/);
-            if (m2) {
-                c_lineno = parseInt(m2[1]);
-                c_path = m2[2].split('/').pop(); // TODO
-            }
-            //0807 : 30 36 __ BMI $083f ; (startup + 62)
-            let m = line.match(/([0-9a-f]+) : ([0-9a-f _]{8}) (.+)/);
-            if (m) {
-                let offset = parseInt(m[1], 16);
-                let hex = m[2];
-                let asm = m[3];
-                if (c_path) {
-                    lst.lines.push({
-                        line: c_lineno,
-                        path: c_path,
-                        offset,
-                        iscode: true
-                    });
-                    c_path = '';
-                    c_lineno = 0;
-                }
-                /*
-                lst.asmlines.push({
-                    line: asm_lineno,
-                    path,
-                    offset,
-                    insns: hex + ' ' + asm,
-                    iscode: true });
-                */
-            }
-        }
-        listings[getFilenamePrefix(step.path) + '.lst'] = lst;
+        let { srclines, asmlines } = parseOscar64Listing(txt, step.path);
+        let lstfn = getFilenamePrefix(step.path) + '.lst';
+        listings[lstfn] = {
+            lines: srclines,
+            asmlines: asmlines,
+            text: txt,
+        };
     }
     return { output, listings, symbolmap, segments, debuginfo };
 }

src/worker/tools/oscar64.ts

@@ -1,12 +1,25 @@
 import { WASIFilesystem, WASIRunner } from "../../common/wasi/wasishim";
 import { BuildStep, BuildStepResult, gatherFiles, staleFiles, store, putWorkFile } from "../builder";
 import { makeErrorMatcher } from "../listingutils";
+import { parseOscar64Listing, parseOscar64Lbl, parseOscar64Map } from "./oscar64parse";
 import { loadWASIFilesystemZip } from "../wasiutils";
 import { loadWASMBinary } from "../wasmutils";
 
 let oscar64_fs: WASIFilesystem | null = null;
 let wasiModule: WebAssembly.Module | null = null;
 
+// find a file in the WASI fs whose name ends with the given suffix.
+// oscar64 writes files as "./path/name.ext" (or "././name.ext" when the
+// output was specified with a "./" prefix), so match on suffix only.
+function getWasiFileAsString(wasi: WASIRunner, suffix: string): string | null {
+    for (const fd of wasi.fs.getFiles()) {
+        if (fd.name.endsWith(suffix)) {
+            return fd.getBytesAsString();
+        }
+    }
+    return null;
+}
+
 export async function compileOscar64(step: BuildStep): Promise<BuildStepResult> {
     const errors = [];
     gatherFiles(step, { mainFilePath: "main.c" });
@@ -48,11 +61,41 @@ export async function compileOscar64(step: BuildStep): Promise<BuildStepResult>
         }
         const output = wasi.fs.getFile("./" + destpath).getBytes();
         putWorkFile(destpath, output);
+        // read and parse oscar64 auxiliary output files (.map, .lbl, .asm)
+        const prefix = destpath.replace(/\.prg$/, '');
+        let mapout = getWasiFileAsString(wasi, prefix + ".map") || getWasiFileAsString(wasi, ".map");
+        let lblout = getWasiFileAsString(wasi, prefix + ".lbl") || getWasiFileAsString(wasi, ".lbl");
+        let asmout = getWasiFileAsString(wasi, prefix + ".asm") || getWasiFileAsString(wasi, ".asm");
+        let segments = [];
+        let symbolmap = {};
+        if (mapout) {
+            let parsed = parseOscar64Map(mapout);
+            segments = parsed.segments;
+            symbolmap = parsed.symbolmap;
+            putWorkFile(prefix + ".map", mapout);
+        }
+        if (lblout) {
+            // merge any extra symbols from the .lbl file
+            symbolmap = Object.assign(parseOscar64Lbl(lblout), symbolmap);
+            putWorkFile(prefix + ".lbl", lblout);
+        }
+        let listings = {};
+        if (asmout) {
+            let { srclines, asmlines } = parseOscar64Listing(asmout, step.path);
+            let lstpath = prefix.replace(/^\.\//, '') + '.lst';
+            putWorkFile(prefix + ".asm", asmout);
+            listings[lstpath] = {
+                lines: srclines,
+                asmlines: asmlines,
+                text: asmout,
+            };
+        }
         return {
             output,
             errors,
-            //listings,
-            //symbolmap
+            listings,
+            symbolmap,
+            segments,
         };
     }
 }

src/worker/tools/oscar64parse.ts

@@ -0,0 +1,139 @@
+// Parser for the oscar64 compiler's auxiliary output files.
+// oscar64 writes a .map (memory map), .lbl (label file) and .asm
+// (assembly listing with C source line markers) next to the output.
+// This module is shared by the browser/WASM tool (src/worker/tools/oscar64.ts)
+// and the native server tool (src/worker/server/buildenv.ts).
+
+import { CodeListingMap, Segment, SourceLine } from "../../common/workertypes";
+
+export interface Oscar64ParseResult {
+    segments: Segment[];
+    symbolmap: { [sym: string]: number };
+    listings: CodeListingMap;
+}
+
+/**
+ * Parse an oscar64 .map file.
+ * Returns segments (from the 'sections' section) and a symbol map
+ * (from the 'objects' section).
+ *
+ * Sample:
+ *   sections
+ *   0880 - 0889 : DATA, code
+ *   0889 - 088a : DATA, data
+ *   ...
+ *   objects
+ *   0801 - 0853 : startup, NATIVE_CODE:startup
+ *   0880 - 0889 : main, NATIVE_CODE:code
+ */
+export function parseOscar64Map(mapout: string) {
+    let segments: Segment[] = [];
+    let symbolmap: { [sym: string]: number } = {};
+    let section = '';
+    for (let line of mapout.split('\n')) {
+        line = line.trim();
+        if (line === 'sections' || line === 'regions' || line === 'objects' || line === 'objects by size') {
+            section = line;
+            continue;
+        }
+        // "0880 - 0889 : DATA, code"  or  "0801 - 0853 : startup, NATIVE_CODE:startup"
+        let m = /^([0-9a-f]+) - ([0-9a-f]+) : ([^,]+), (.+)$/.exec(line);
+        if (m) {
+            const start = parseInt(m[1], 16);
+            const end = parseInt(m[2], 16);
+            const name = m[4].trim();
+            if (section === 'sections') {
+                // derive rom/ram classification from the segment name
+                let type = 'ram';
+                if (/code/.test(m[4]) || m[4] === 'startup' || /rom/.test(m[4])) {
+                    type = 'rom';
+                }
+                segments.push({ name: m[4], start, size: end - start, type });
+            } else if (section === 'objects') {
+                if (m[3] !== '*') symbolmap[m[3]] = start;
+            }
+        }
+    }
+    return { segments, symbolmap };
+}
+
+/**
+ * Parse an oscar64 .lbl file (VICE label format), e.g. "al 0880 .main".
+ * Used as a fallback when the .map has no "objects" section.
+ */
+export function parseOscar64Lbl(lblout: string) {
+    let symbolmap: { [sym: string]: number } = {};
+    for (let line of lblout.split('\n')) {
+        let toks = line.trim().split(/\s+/);
+        if (toks[0] == 'al' && toks.length >= 3) {
+            const ofs = parseInt(toks[1], 16);
+            const name = toks[2];
+            // prefer the un-dotted form used by the assembler listing
+            const clean = name.replace(/^\./, '');
+            if (!symbolmap[clean]) symbolmap[clean] = ofs;
+        }
+    }
+    return symbolmap;
+}
+
+/**
+ * Parse an oscar64 .asm listing.
+ * Returns source lines (each instruction offset mapped to its C source line)
+ * and assembly lines (one entry per assembly line in the listing).
+ *
+ * Sample:
+ *   ; Compiled with 1.32.266
+ *   --------------------------------------------------------------------
+ *   startup: ; startup
+ *   0801 : 0b __ __ INV
+ *   ...
+ *   main: ; main()->i16
+ *   ;   2, "/test.c"
+ *   .s4:
+ *   ;   2, "/test.c"
+ *   0880 : a9 04 __ LDA #$04
+ */
+export function parseOscar64Listing(asmout: string, asmfn: string) {
+    let srclines: SourceLine[] = [];
+    let asmlines: SourceLine[] = [];
+    let c_lineno = 0;
+    let c_path = '';
+    let asm_lineno = 0;
+    // ;   2, "/test.c"
+    let re_src = /^;\s*(\d+), "(.+?)"/;
+    // 0801 : 0b __ __ INV
+    let re_insn = /^([0-9a-f]+) : ([0-9a-f _]{8}) (.*)/;
+    for (let line of asmout.split('\n')) {
+        asm_lineno++;
+        let m2 = re_src.exec(line);
+        if (m2) {
+            c_lineno = parseInt(m2[1]);
+            // path may be absolute; use basename for source line mapping
+            c_path = m2[2].split('/').pop();
+        }
+        let m = re_insn.exec(line);
+        if (m) {
+            let offset = parseInt(m[1], 16);
+            let hex = m[2];
+            let asm = m[3];
+            let insns = (hex + ' ' + asm).trim();
+            asmlines.push({
+                line: asm_lineno,
+                path: asmfn,
+                offset,
+                insns,
+                iscode: true,
+            });
+            if (c_path) {
+                srclines.push({
+                    line: c_lineno,
+                    path: c_path,
+                    offset,
+                    iscode: true,
+                });
+                c_path = '';
+            }
+        }
+    }
+    return { srclines, asmlines };
+}

src/worker/wasiutils.ts

@@ -9,7 +9,12 @@ export function loadBlobSync(path: string) {
     return xhr.response;
 }
 
-export async function unzipWASIFilesystem(zipdata: ArrayBuffer, rootPath: string = "./") {
+export async function unzipWASIFilesystem(zipdata: any, rootPath: string = "./") {
+    // In the Node.js test env, the XMLHttpRequest shim returns a custom Blob
+    // that JSZip does not recognize. Convert it to an ArrayBuffer first.
+    if (zipdata && typeof zipdata.asArrayBuffer === 'function') {
+        zipdata = zipdata.asArrayBuffer();
+    }
     const jszip = new JSZip();
     await jszip.loadAsync(zipdata);
     let fs = new WASIMemoryFilesystem();
@@ -33,6 +38,5 @@ export async function loadWASIFilesystemZip(zippath: string, rootPath: string =
     const jszip = new JSZip();
     const path = '../../src/worker/fs/' + zippath;
     const zipdata = loadBlobSync(path);
-    console.log(zippath, zipdata);
     return unzipWASIFilesystem(zipdata, rootPath);
 }

test/cli/testworker.js

@@ -120,6 +120,28 @@ describe('Worker', function() {
   it('should compile SDCC w/ include', function(done) {
     compile('sdcc', '#include <string.h>\nvoid main() {\nstrlen(0);\n}\n', 'mw8080bw', done, 8192, 2, 0);
   });
+  it('should compile oscar64 and return listings/symbols/segments', async function() {
+    var msgs = [{code:'#include <stdio.h>\nint main() { printf("FOO"); return 0; }', platform:'c64', tool:'oscar64', path:'main.c', mainfile:true}];
+    var result = await new Promise(function(resolve, reject) {
+      global.postMessage = function(msg) {
+        if (!msg.unchanged) {
+          assert.ok(!msg.errors || msg.errors.length === 0, JSON.stringify(msg.errors));
+          resolve(msg);
+        }
+      };
+      (async function() {
+        await global.onmessage({data:{reset:true}});
+        await global.onmessage({data:msgs[0]});
+      })().catch(reject);
+    });
+    assert.ok(result.listings && Object.keys(result.listings).length > 0, 'no listings');
+    var lst = result.listings[Object.keys(result.listings)[0]];
+    assert.ok(lst.lines.length > 0, 'no source lines');
+    assert.ok(lst.asmlines.length > 0, 'no asm lines');
+    assert.ok(result.symbolmap && result.symbolmap['main'] > 0, 'no main symbol');
+    assert.ok(result.segments && result.segments.length > 0, 'no segments');
+  });
+
   it('should compile mw8080 skeleton', function(done) {
     var csource = ab2str(fs.readFileSync('presets/mw8080bw/skeleton.sdcc'));
     compile('sdcc', csource, 'mw8080bw', done, 8192, 84, 0);

test/unit/testwasishim.ts

@@ -2,6 +2,7 @@ import assert from "assert";
 import { WASIRunner } from "../../src/common/wasi/wasishim";
 import * as fs from "fs";
 import { loadWASIFilesystemZip, unzipWASIFilesystem } from "../../src/worker/wasiutils";
+import { parseOscar64Listing, parseOscar64Lbl, parseOscar64Map } from "../../src/worker/tools/oscar64parse";
 
 async function loadWASM(filename: string) {
     const wasmdata = fs.readFileSync(`./src/worker/wasm/${filename}.wasm`);
@@ -121,26 +122,40 @@ describe('test WASI dialogc', function () {
     });
 });
 
-/*
 describe('test WASI oscar64', function () {
-    it('oscar64 compile', async function () {
+    it('oscar64 compile + parse output', async function () {
         let shim = await loadOscar64();
         const zipdata = fs.readFileSync(`./src/worker/fs/oscar64-fs.zip`);
-        shim.fs = await unzipWASIFilesystem(zipdata, "/root/");
-        // https://github.com/WebAssembly/wasi-filesystem/issues/24
-        // https://github.com/WebAssembly/wasi-libc/pull/214
-        shim.addPreopenDirectory("/root");
-        shim.fs.putSymbolicLink("/proc/self/exe", "/root/bin/oscar64");
-        shim.fs.putFile("/root/main.c", `#include <stdio.h>\nint main() { printf("FOO"); return 0; }`);
-        shim.setArgs(["oscar64", '-v', '-g', '-O', '-o=foo.prg', 'main.c']);
+        let oscar64_fs = await unzipWASIFilesystem(zipdata, "./");
+        shim.fs.setParent(oscar64_fs);
+        shim.fs.putFile("./main.c", `#include <stdio.h>\nint main() { printf("FOO"); return 0; }`);
+        shim.addPreopenDirectory("include");
+        shim.addPreopenDirectory(".");
+        shim.setArgs(["oscar64", '-v', '-gp', '-ii=include', '-o=./foo.prg', 'main.c']);
         let errno = shim.run();
         const stdout = shim.fds[1].getBytesAsString();
-        console.log(stdout);
         const stderr = shim.fds[2].getBytesAsString();
-        console.log(stderr);
-        assert.strictEqual(errno, 0);
-        assert.ok(stdout.indexOf('Starting oscar64') >= 0);
-        console.log(shim.fs.getFile("/root/foo.asm").getBytesAsString());
+        assert.strictEqual(errno, 0, stdout + '\n' + stderr);
+        assert.ok(stdout.indexOf('Starting oscar64') >= 0, stdout);
+        // oscar64 should have written a .map, .lbl and .asm file
+        assert.ok(shim.fs.getFile("././foo.prg"), "foo.prg not written");
+        assert.ok(shim.fs.getFile("././foo.map"), "foo.map not written");
+        assert.ok(shim.fs.getFile("././foo.lbl"), "foo.lbl not written");
+        assert.ok(shim.fs.getFile("././foo.asm"), "foo.asm not written");
+        // parse the map file for segments and symbols
+        let mapout = shim.fs.getFile("././foo.map").getBytesAsString();
+        let parsed = parseOscar64Map(mapout);
+        assert.ok(parsed.segments.length > 0);
+        assert.ok(parsed.symbolmap['main'] > 0);
+        // parse the lbl file
+        let lblout = shim.fs.getFile("././foo.lbl").getBytesAsString();
+        let lbl = parseOscar64Lbl(lblout);
+        assert.ok(lbl['main'] === parsed.symbolmap['main']);
+        // parse the asm listing
+        let asmout = shim.fs.getFile("././foo.asm").getBytesAsString();
+        let listing = parseOscar64Listing(asmout, 'main.c');
+        assert.ok(listing.asmlines.length > 0);
+        assert.ok(listing.srclines.length > 0);
+        assert.ok(listing.srclines[0].offset > 0);
     });
 });
-*/