417 lines
12 KiB
TypeScript
417 lines
12 KiB
TypeScript
import { getBasePlatform } from "../common/util";
|
|
import { WorkerBuildStep, WorkerError, WorkerErrorResult, WorkerMessage, WorkerResult, WorkingStore } from "../common/workertypes";
|
|
import { PLATFORM_PARAMS } from "./platforms";
|
|
import { TOOLS } from "./workertools";
|
|
|
|
/// working file store and build steps
|
|
|
|
const PSRC = "../../src/";
|
|
export const PWORKER = PSRC + "worker/";
|
|
|
|
export type FileData = string | Uint8Array;
|
|
|
|
export type FileEntry = {
|
|
path: string
|
|
encoding: string
|
|
data: FileData
|
|
ts: number
|
|
};
|
|
|
|
export type BuildOptions = {
|
|
mainFilePath: string,
|
|
processFn?: (s: string, d: FileData) => FileData
|
|
};
|
|
|
|
// TODO
|
|
export type BuildStepResult = WorkerResult | WorkerNextToolResult;
|
|
|
|
export interface WorkerNextToolResult {
|
|
nexttool?: string
|
|
linktool?: string
|
|
path?: string
|
|
args: string[]
|
|
files: string[]
|
|
bblines?: boolean
|
|
}
|
|
|
|
export interface BuildStep extends WorkerBuildStep {
|
|
files?: string[]
|
|
args?: string[]
|
|
nextstep?: BuildStep
|
|
linkstep?: BuildStep
|
|
params?
|
|
result?: BuildStepResult
|
|
code?
|
|
prefix?
|
|
maxts?
|
|
debuginfo?
|
|
};
|
|
|
|
///
|
|
|
|
export class FileWorkingStore implements WorkingStore {
|
|
workfs: { [path: string]: FileEntry } = {};
|
|
workerseq: number = 0;
|
|
items: {};
|
|
|
|
constructor() {
|
|
this.reset();
|
|
}
|
|
reset() {
|
|
this.workfs = {};
|
|
this.newVersion();
|
|
}
|
|
currentVersion() {
|
|
return this.workerseq;
|
|
}
|
|
newVersion() {
|
|
let ts = new Date().getTime();
|
|
if (ts <= this.workerseq)
|
|
ts = ++this.workerseq;
|
|
return ts;
|
|
}
|
|
putFile(path: string, data: FileData): FileEntry {
|
|
var encoding = (typeof data === 'string') ? 'utf8' : 'binary';
|
|
var entry = this.workfs[path];
|
|
if (!entry || !compareData(entry.data, data) || entry.encoding != encoding) {
|
|
this.workfs[path] = entry = { path: path, data: data, encoding: encoding, ts: this.newVersion() };
|
|
console.log('+++', entry.path, entry.encoding, entry.data.length, entry.ts);
|
|
}
|
|
return entry;
|
|
}
|
|
hasFile(path: string) {
|
|
return this.workfs[path] != null;
|
|
}
|
|
getFileData(path: string): FileData {
|
|
return this.workfs[path] && this.workfs[path].data;
|
|
}
|
|
getFileAsString(path: string): string {
|
|
let data = this.getFileData(path);
|
|
if (data != null && typeof data !== 'string')
|
|
throw new Error(`${path}: expected string`)
|
|
return data as string; // TODO
|
|
}
|
|
getFileEntry(path: string): FileEntry {
|
|
return this.workfs[path];
|
|
}
|
|
setItem(key: string, value: object) {
|
|
this.items[key] = value;
|
|
}
|
|
}
|
|
|
|
export var store = new FileWorkingStore();
|
|
|
|
///
|
|
|
|
export function errorResult(msg: string): WorkerErrorResult {
|
|
return { errors: [{ line: 0, msg: msg }] };
|
|
}
|
|
|
|
export class Builder {
|
|
steps: BuildStep[] = [];
|
|
startseq: number = 0;
|
|
|
|
// returns true if file changed during this build step
|
|
wasChanged(entry: FileEntry): boolean {
|
|
return entry.ts > this.startseq;
|
|
}
|
|
async executeBuildSteps(): Promise<WorkerResult> {
|
|
this.startseq = store.currentVersion();
|
|
var linkstep: BuildStep = null;
|
|
while (this.steps.length) {
|
|
var step = this.steps.shift(); // get top of array
|
|
var platform = step.platform;
|
|
var [tool, remoteTool] = step.tool.split(':', 2);
|
|
var toolfn = TOOLS[tool];
|
|
if (!toolfn) {
|
|
throw Error(`no tool named "${tool}"`);
|
|
}
|
|
if (remoteTool) {
|
|
step.tool = remoteTool;
|
|
}
|
|
step.params = PLATFORM_PARAMS[getBasePlatform(platform)];
|
|
try {
|
|
step.result = await toolfn(step);
|
|
} catch (e) {
|
|
console.log("EXCEPTION", e, e.stack);
|
|
return errorResult(e + ""); // TODO: catch errors already generated?
|
|
}
|
|
if (step.result) {
|
|
(step.result as any).params = step.params; // TODO: type check
|
|
if (step.debuginfo) {
|
|
let r = step.result as any; // TODO
|
|
if (!r.debuginfo) r.debuginfo = {};
|
|
Object.assign(r.debuginfo, step.debuginfo);
|
|
}
|
|
// errors? return them
|
|
if ('errors' in step.result && step.result.errors.length) {
|
|
applyDefaultErrorPath(step.result.errors, step.path);
|
|
return step.result;
|
|
}
|
|
// if we got some output, return it immediately
|
|
if ('output' in step.result && step.result.output) {
|
|
return step.result;
|
|
}
|
|
// combine files with a link tool?
|
|
if ('linktool' in step.result) {
|
|
// add to existing link step
|
|
if (linkstep) {
|
|
linkstep.files = linkstep.files.concat(step.result.files);
|
|
linkstep.args = linkstep.args.concat(step.result.args);
|
|
} else {
|
|
linkstep = {
|
|
tool: step.result.linktool,
|
|
platform: platform,
|
|
files: step.result.files,
|
|
args: step.result.args
|
|
};
|
|
}
|
|
linkstep.debuginfo = step.debuginfo; // TODO: multiple debuginfos
|
|
}
|
|
// process with another tool?
|
|
if ('nexttool' in step.result) {
|
|
var asmstep: BuildStep = {
|
|
tool: step.result.nexttool,
|
|
platform: platform,
|
|
...step.result
|
|
}
|
|
this.steps.push(asmstep);
|
|
}
|
|
// process final step?
|
|
if (this.steps.length == 0 && linkstep) {
|
|
this.steps.push(linkstep);
|
|
linkstep = null;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
async handleMessage(data: WorkerMessage): Promise<WorkerResult> {
|
|
this.steps = [];
|
|
// file updates
|
|
if (data.updates) {
|
|
data.updates.forEach((u) => store.putFile(u.path, u.data));
|
|
}
|
|
// object update
|
|
if (data.setitems) {
|
|
data.setitems.forEach((i) => store.setItem(i.key, i.value));
|
|
}
|
|
// build steps
|
|
if (data.buildsteps) {
|
|
this.steps.push.apply(this.steps, data.buildsteps);
|
|
}
|
|
// single-file
|
|
if (data.code) {
|
|
this.steps.push(data as BuildStep); // TODO: remove cast
|
|
}
|
|
// execute build steps
|
|
if (this.steps.length) {
|
|
var result = await this.executeBuildSteps();
|
|
return result ? result : { unchanged: true };
|
|
}
|
|
// TODO: cache results
|
|
// message not recognized
|
|
console.log("Unknown message", data);
|
|
}
|
|
}
|
|
|
|
function applyDefaultErrorPath(errors: WorkerError[], path: string) {
|
|
if (!path) return;
|
|
for (var i = 0; i < errors.length; i++) {
|
|
var err = errors[i];
|
|
if (!err.path && err.line) err.path = path;
|
|
}
|
|
}
|
|
|
|
function compareData(a: FileData, b: FileData): boolean {
|
|
if (a.length != b.length) return false;
|
|
if (typeof a === 'string' && typeof b === 'string') {
|
|
return a == b;
|
|
} else {
|
|
for (var i = 0; i < a.length; i++) {
|
|
//if (a[i] != b[i]) console.log('differ at byte',i,a[i],b[i]);
|
|
if (a[i] != b[i]) return false;
|
|
}
|
|
return true;
|
|
}
|
|
}
|
|
|
|
export const builder = new Builder();
|
|
|
|
var _t1;
|
|
export function starttime() { _t1 = new Date(); }
|
|
export function endtime(msg) { var _t2 = new Date(); console.log(msg, _t2.getTime() - _t1.getTime(), "ms"); }
|
|
|
|
///
|
|
|
|
export function putWorkFile(path: string, data: FileData) {
|
|
return store.putFile(path, data);
|
|
}
|
|
|
|
export function getWorkFileAsString(path: string): string {
|
|
return store.getFileAsString(path);
|
|
}
|
|
|
|
export function populateEntry(fs, path: string, entry: FileEntry, options: BuildOptions) {
|
|
var data = entry.data;
|
|
if (options && options.processFn) {
|
|
data = options.processFn(path, data);
|
|
}
|
|
// create subfolders
|
|
var toks = path.split('/');
|
|
if (toks.length > 1) {
|
|
for (var i = 0; i < toks.length - 1; i++)
|
|
try {
|
|
fs.mkdir(toks[i]);
|
|
} catch (e) { }
|
|
}
|
|
// write file
|
|
fs.writeFile(path, data, { encoding: entry.encoding });
|
|
var time = new Date(entry.ts);
|
|
fs.utime(path, time, time);
|
|
console.log("<<<", path, entry.data.length);
|
|
}
|
|
|
|
// can call multiple times (from populateFiles)
|
|
export function gatherFiles(step: BuildStep, options?: BuildOptions): number {
|
|
var maxts = 0;
|
|
if (step.files) {
|
|
for (var i = 0; i < step.files.length; i++) {
|
|
var path = step.files[i];
|
|
var entry = store.workfs[path];
|
|
if (!entry) {
|
|
throw new Error("No entry for path '" + path + "'");
|
|
} else {
|
|
maxts = Math.max(maxts, entry.ts);
|
|
}
|
|
}
|
|
}
|
|
else if (step.code) {
|
|
var path = step.path ? step.path : options.mainFilePath; // TODO: what if options null
|
|
if (!path) throw Error("need path or mainFilePath");
|
|
var code = step.code;
|
|
var entry = putWorkFile(path, code);
|
|
step.path = path;
|
|
step.files = [path];
|
|
maxts = entry.ts;
|
|
}
|
|
else if (step.path) {
|
|
var path = step.path;
|
|
var entry = store.workfs[path];
|
|
maxts = entry.ts;
|
|
step.files = [path];
|
|
}
|
|
if (step.path && !step.prefix) {
|
|
step.prefix = getPrefix(step.path);
|
|
}
|
|
step.maxts = maxts;
|
|
return maxts;
|
|
}
|
|
|
|
export function getPrefix(s: string): string {
|
|
var pos = s.lastIndexOf('.');
|
|
return (pos > 0) ? s.substring(0, pos) : s;
|
|
}
|
|
|
|
export function populateFiles(step: BuildStep, fs, options?: BuildOptions) {
|
|
gatherFiles(step, options);
|
|
if (!step.files) throw Error("call gatherFiles() first");
|
|
for (var i = 0; i < step.files.length; i++) {
|
|
var path = step.files[i];
|
|
populateEntry(fs, path, store.workfs[path], options);
|
|
}
|
|
}
|
|
|
|
export function populateExtraFiles(step: BuildStep, fs, extrafiles) {
|
|
if (extrafiles) {
|
|
for (var i = 0; i < extrafiles.length; i++) {
|
|
var xfn = extrafiles[i];
|
|
// is this file cached?
|
|
if (store.workfs[xfn]) {
|
|
fs.writeFile(xfn, store.workfs[xfn].data, { encoding: 'binary' });
|
|
continue;
|
|
}
|
|
// fetch from network
|
|
var xpath = "lib/" + getBasePlatform(step.platform) + "/" + xfn;
|
|
var xhr = new XMLHttpRequest();
|
|
xhr.responseType = 'arraybuffer';
|
|
xhr.open("GET", PWORKER + xpath, false); // synchronous request
|
|
xhr.send(null);
|
|
if (xhr.response && xhr.status == 200) {
|
|
var data = new Uint8Array(xhr.response);
|
|
fs.writeFile(xfn, data, { encoding: 'binary' });
|
|
putWorkFile(xfn, data);
|
|
console.log(":::", xfn, data.length);
|
|
} else {
|
|
throw Error("Could not load extra file " + xpath);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
export function staleFiles(step: BuildStep, targets: string[]) {
|
|
if (!step.maxts) throw Error("call populateFiles() first");
|
|
// see if any target files are more recent than inputs
|
|
for (var i = 0; i < targets.length; i++) {
|
|
var entry = store.workfs[targets[i]];
|
|
if (!entry || step.maxts > entry.ts)
|
|
return true;
|
|
}
|
|
console.log("unchanged", step.maxts, targets);
|
|
return false;
|
|
}
|
|
|
|
export function anyTargetChanged(step: BuildStep, targets: string[]) {
|
|
if (!step.maxts) throw Error("call populateFiles() first");
|
|
// see if any target files are more recent than inputs
|
|
for (var i = 0; i < targets.length; i++) {
|
|
var entry = store.workfs[targets[i]];
|
|
if (!entry || entry.ts > step.maxts)
|
|
return true;
|
|
}
|
|
console.log("unchanged", step.maxts, targets);
|
|
return false;
|
|
}
|
|
|
|
export function fixParamsWithDefines(path: string, params) {
|
|
var libargs = params.libargs;
|
|
if (path && libargs) {
|
|
var code = getWorkFileAsString(path);
|
|
if (code) {
|
|
var oldcfgfile = params.cfgfile;
|
|
var ident2index = {};
|
|
// find all lib args "IDENT=VALUE"
|
|
for (var i = 0; i < libargs.length; i++) {
|
|
var toks = libargs[i].split('=');
|
|
if (toks.length == 2) {
|
|
ident2index[toks[0]] = i;
|
|
}
|
|
}
|
|
// find #defines and replace them
|
|
var re = /^[;]?#define\s+(\w+)\s+(\S+)/gmi; // TODO: empty string?
|
|
var m;
|
|
while (m = re.exec(code)) {
|
|
var ident = m[1];
|
|
var value = m[2];
|
|
var index = ident2index[ident];
|
|
if (index >= 0) {
|
|
libargs[index] = ident + "=" + value;
|
|
console.log('Using libargs', index, libargs[index]);
|
|
// TODO: MMC3 mapper switch
|
|
if (ident == 'NES_MAPPER' && value == '4') {
|
|
params.cfgfile = 'nesbanked.cfg';
|
|
console.log("using config file", params.cfgfile);
|
|
}
|
|
} else if (ident == 'CFGFILE' && value) {
|
|
params.cfgfile = value;
|
|
} else if (ident == 'LIBARGS' && value) {
|
|
params.libargs = value.split(',').filter((s) => { return s != ''; });
|
|
console.log('Using libargs', params.libargs);
|
|
} else if (ident == 'CC65_FLAGS' && value) {
|
|
params.extra_compiler_args = value.split(',').filter((s) => { return s != ''; });
|
|
console.log('Using compiler flags', params.extra_compiler_args);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|