started on server

package.json

@@ -30,6 +30,8 @@
     "octokat": "^0.10.0",
     "preact": "^10.5.14",
     "split.js": "^1.6.2",
+    "atob": "^2.1.x",
+    "btoa": "^1.2.x",
     "yufka": "^2.0.1"
   },
   "devDependencies": {
@@ -40,11 +42,11 @@
     "@types/jquery": "^3.5.5",
     "@types/mocha": "^9.1.0",
     "@types/node": "^14.14.20",
-    "atob": "^2.1.x",
     "bootstrap": "^3.4.1",
     "bootstrap-tourist": "^0.2.1",
-    "btoa": "^1.2.x",
     "command-exists": "^1.2.9",
+    "cors": "^2.8.5",
+    "express": "^4.18.2",
     "esbuild": "^0.12.29",
     "jsdom": "^21.1.0",
     "lzg": "^1.0.x",
@@ -68,8 +70,10 @@
     "tsbuild": "tsc --build tsconfig.json",
     "esbuild": "npm run esbuild-worker && npm run esbuild-ui",
     "esbuild-clean": "rm -f ./gen/*.*",
+    "esbuild-server": "esbuild src/worker/server/server.ts --platform=node --bundle --sourcemap --target=es2020 --outfile=./gen/server/server.js",
     "esbuild-worker": "esbuild src/worker/workermain.ts --bundle --sourcemap --target=es2017 --outfile=./gen/worker/bundle.js",
     "esbuild-ui": "esbuild src/ide/ui.ts src/ide/embedui.ts --splitting --format=esm --bundle --minify --sourcemap --target=es2017 --outdir=./gen/ --external:path --external:fs",
+    "server": "npm run esbuild-server && rm -fr ./server-root/sessions && node gen/server/server.js",
     "test-one": "NODE_PATH=$(pwd) mocha --recursive --timeout 60000",
     "test-node": "NODE_PATH=$(pwd) mocha --recursive --timeout 60000 test/cli gen/test",
     "test-profile": "NODE_PATH=$(pwd) mocha --recursive --timeout 60000 --prof test/cli gen/test",
@@ -97,7 +101,7 @@
   "build": {
     "appId": "com.8bitworkshop.ide",
     "icon": "meta/icons/8bitworkshop-icon-1024.png",
-    "copyright": "Copyright (c) 2021 Puzzling Plans LLC",
+    "copyright": "Copyright (c) 2023 Puzzling Plans LLC",
     "linux": {
       "category": "Development"
     }

src/ide/project.ts

@@ -101,6 +101,7 @@ export class CodeProject {
   filename2path = {}; // map stripped paths to full paths
   filesystem : ProjectFilesystem;
   dataItems : WorkerItemUpdate[];
+  remoteTool? : string;
 
   callbackBuildResult : BuildResultCallback;
   callbackBuildStatus : BuildStatusCallback;
@@ -135,8 +136,16 @@ export class CodeProject {
     this.callbackBuildResult(data);
   }
 
+  getToolForFilename(path) {
+    if (this.remoteTool) {
+      return "remote:" + this.remoteTool;
+    } else {
+      return this.platform.getToolForFilename(path);
+    }
+  }
+
   preloadWorker(path:string) {
-    var tool = this.platform.getToolForFilename(path);
+    var tool = this.getToolForFilename(path);
     if (tool && !this.tools_preloaded[tool]) {
       this.worker.postMessage({preload:tool, platform:this.platform_id});
       this.tools_preloaded[tool] = true;
@@ -277,7 +286,7 @@ export class CodeProject {
       path:mainfilename,
       files:[mainfilename].concat(depfiles),
       platform:this.platform_id,
-      tool:this.platform.getToolForFilename(this.mainPath),
+      tool:this.getToolForFilename(this.mainPath),
       mainfile:true});
     for (var dep of depends) {
       if (dep.data && dep.link) {
@@ -287,7 +296,7 @@ export class CodeProject {
           path:dep.filename,
           files:[dep.filename].concat(depfiles),
           platform:this.platform_id,
-          tool:this.platform.getToolForFilename(dep.path)});
+          tool:this.getToolForFilename(dep.path)});
       }
     }
     if (this.dataItems) msg.setitems = this.dataItems;

src/ide/ui.ts

@@ -46,6 +46,7 @@ interface UIQueryString {
   file0_name? : string;
   file0_data? : string;
   file0_type? : string;
+  tool?: string;
 }
 
 export var qs : UIQueryString = decodeQueryString(window.location.search||'?') as UIQueryString;
@@ -252,6 +253,7 @@ async function newFilesystem() {
 async function initProject() {
   var filesystem = await newFilesystem();
   current_project = new CodeProject(newWorker(), platform_id, platform, filesystem);
+  current_project.remoteTool = qs.tool || null;
   projectWindows = new ProjectWindows($("#workspace")[0] as HTMLElement, current_project);
   current_project.callbackBuildResult = (result:WorkerResult) => {
     setCompileOutput(result);

src/worker/server/buildenv.ts

@@ -0,0 +1,163 @@
+
+import fs from 'fs';
+import path from 'path';
+import { spawn } from 'child_process';
+import { WorkerBuildStep, WorkerError, WorkerFileUpdate, WorkerResult } from '../../common/workertypes';
+import { getBasePlatform, getRootBasePlatform } from '../../common/util';
+import { BuildStep, makeErrorMatcher } from '../workermain';
+
+
+const LLVM_MOS_TOOL: ServerBuildTool = {
+    name: 'llvm-mos',
+    version: '',
+    extensions: ['.c'],
+    archs: ['6502'],
+    platforms: ['atari8', 'c64', 'nes'],
+    platform_configs: {
+        c64: {
+            binpath: 'llvm-mos/bin',
+            command: 'mos-c64-clang',
+            args: ['-Os', '-g', '-o', '$OUTFILE', '$INFILES']
+        },
+        debug: { // TODO
+            binpath: 'llvm-mos/bin',
+            command: 'llvm-objdump',
+            args: ['-l', '-t', '$ELFFILE']
+        }
+    }
+}
+
+export function findBestTool(step: BuildStep) {
+    if (!step?.tool) throw new Error('No tool specified');
+    const [name, version] = step.tool.split('@');
+    for (let tool of TOOLS) {
+        if (tool.name === name && (!version || version === 'latest' || tool.version === version)) {
+            return tool;
+        }
+    }
+    throw new Error(`Tool not found: ${step.tool}`);
+}
+
+export const TOOLS: ServerBuildTool[] = [
+    Object.assign({}, LLVM_MOS_TOOL, { version: '0.13.2' }),
+];
+
+interface ServerBuildTool {
+    name: string;
+    version: string;
+    extensions: string[];
+    archs: string[];
+    platforms: string[];
+    platform_configs: { [platform: string]: ServerBuildToolPlatformConfig };
+}
+
+interface ServerBuildToolPlatformConfig {
+    binpath: string;
+    command: string;
+    args: string[];
+}
+
+
+export class ServerBuildEnv {
+
+    rootdir: string;
+    sessionID: string;
+    tool: ServerBuildTool;
+    sessionDir: string;
+    errors: WorkerError[] = [];
+
+    constructor(rootdir: string, sessionID: string, tool: ServerBuildTool) {
+        this.rootdir = rootdir;
+        this.sessionID = sessionID;
+        this.tool = tool;
+        // make sure sessionID is well-formed
+        if (!sessionID.match(/^[a-zA-Z0-9_-]+$/)) {
+            throw new Error(`Invalid sessionID: ${sessionID}`);
+        }
+        // create sessionID directory if it doesn't exist
+        this.sessionDir = path.join(rootdir, 'sessions', sessionID);
+        if (!fs.existsSync(this.sessionDir)) {
+            fs.mkdirSync(this.sessionDir);
+        }
+    }
+
+    async addFileUpdate(file: WorkerFileUpdate) {
+        // make sure file.path contains no path components
+        if (file.path.match(/[\\\/]/)) {
+            throw new Error(`Invalid file path: ${file.path}`);
+        }
+        await fs.promises.writeFile(path.join(this.sessionDir, file.path), file.data);
+    }
+
+    async build(step: WorkerBuildStep): Promise<void> {
+        let platformID = getRootBasePlatform(step.platform);
+        let config = this.tool.platform_configs[platformID];
+        if (!config) {
+            throw new Error(`No config for platform ${platformID}`);
+        }
+        let args = config.args.slice(0); //copy array
+        let command = config.command;
+        // replace $OUTFILE
+        let outfile = path.join(this.sessionDir, 'a.out');
+        for (let i = 0; i < args.length; i++) {
+            if (args[i] === '$OUTFILE') {
+                args[i] = outfile;
+            }
+        }
+        // replace $INFILES with the list of input files
+        // TODO
+        let infiles = [];
+        for (let i = 0; i < step.files.length; i++) {
+            let f = step.files[i];
+            if (f.endsWith(this.tool.extensions[0])) {
+                infiles.push(path.join(this.sessionDir, f));
+            }
+        }
+        for (let i = 0; i < args.length; i++) {
+            if (args[i] === '$INFILES') {
+                args = args.slice(0, i).concat(infiles).concat(args.slice(i + 1));
+                break;
+            }
+        }
+        console.log(`Running: ${command} ${args.join(' ')}`);
+        // spawn after setting PATH env var
+        // TODO
+        let childProcess = spawn(command, args, { shell: true, env: { PATH: path.join(this.rootdir, config.binpath) } });
+        let outputData = '';
+        let errorData = '';
+        let errors = [];
+        let errorMatcher = makeErrorMatcher(errors, /([^:/]+):(\d+):(\d+):\s*(.+)/, 2, 4, step.path, 1);
+        childProcess.stdout.on('data', (data) => {
+            outputData += data.toString();
+        });
+        childProcess.stderr.on('data', (data) => {
+            errorData += data.toString();
+        });
+        await new Promise((resolve, reject) => {
+            childProcess.on('close', (code) => {
+                // split errorData into lines
+                for (let line of errorData.split('\n')) {
+                    errorMatcher(line);
+                }
+                if (code === 0) {
+                    this.errors = [];
+                    resolve(0);
+                } else {
+                    this.errors = errors;
+                    resolve(0);
+                }
+            });
+        });
+    }
+
+    async processResult(): Promise<WorkerResult> {
+        if (this.errors.length) {
+            return { errors: this.errors };
+        } else {
+            let outfile = path.join(this.sessionDir, 'a.out');
+            let output = await fs.promises.readFile(outfile, { encoding: 'base64' });
+            return { output };
+        }
+    }
+}
+

src/worker/server/server.ts

@@ -0,0 +1,79 @@
+
+import fs from 'fs';
+import path from 'path';
+import express, { Request, Response } from 'express';
+import cors from 'cors';
+import { WorkerBuildStep, WorkerFileUpdate } from '../../common/workertypes';
+import { ServerBuildEnv, TOOLS, findBestTool } from './buildenv';
+
+////////////////////
+
+const SERVER_ROOT = './server-root';
+const SESSION_ROOT = path.join(SERVER_ROOT, 'sessions');
+
+if (!fs.existsSync(SESSION_ROOT)) {
+    fs.mkdirSync(SESSION_ROOT);
+}
+
+const app = express();
+
+app.use(cors());
+
+app.use(express.json());
+
+app.get('/info', (req: Request, res: Response) => {
+    // send a list of supported tools
+    res.json({ tools: TOOLS });
+});
+
+app.get('/test', async (req: Request, res: Response, next) => {
+    // quick test of the build
+    try {
+        const updates: WorkerFileUpdate[] = [{ path: 'test.c', data: 'int main() { return 0; }' }];
+        const buildStep: WorkerBuildStep = { tool: 'llvm-mos', platform: 'c64', files: ['test.c'] };
+        const env = new ServerBuildEnv(SERVER_ROOT, 'test', TOOLS[0]);
+        for (let file of updates) {
+            await env.addFileUpdate(file);
+        }
+        await env.build(buildStep);
+        const result = await env.processResult();
+        res.json(result);
+    } catch (err) {
+        return next(err);
+    }
+});
+
+app.post('/build', async (req: Request, res: Response, next) => {
+    try {
+        const updates: WorkerFileUpdate[] = req.body.updates;
+        const buildStep: WorkerBuildStep = req.body.buildStep;
+        const sessionID = req.body.sessionID;
+        const bestTool = findBestTool(buildStep);
+        const env = new ServerBuildEnv(SERVER_ROOT, sessionID, bestTool);
+        for (let file of updates) {
+            await env.addFileUpdate(file);
+        }
+        await env.build(buildStep);
+        const result = await env.processResult();
+        res.json(result);
+    } catch (err) {
+        return next(err);
+    }
+});
+
+// Catch errors
+app.use((err: Error, req: Request, res: Response, next: Function) => {
+    console.error(err.stack);
+    res.status(500).send('Something broke!');
+});
+
+// Start the server
+const port = 3009;
+
+/*{
+    origin: [`http://localhost:${port}`, 'http://localhost:8000']
+}));*/
+
+app.listen(port, () => {
+    console.log(`Server is listening on port ${port}`);
+});

src/worker/workermain.ts

@@ -516,8 +516,14 @@ class Builder {
     while (this.steps.length) {
       var step = this.steps.shift(); // get top of array
       var platform = step.platform;
-      var toolfn = TOOLS[step.tool];
-      if (!toolfn) throw Error("no tool named " + step.tool);
+      var [tool, remoteTool] = step.tool.split(':', 2);
+      var toolfn = TOOLS[tool];
+      if (!toolfn) {
+        throw Error(`no tool named "${tool}"`);
+      }
+      if (remoteTool) {
+        step.tool = remoteTool;
+      }
       step.params = PLATFORM_PARAMS[getBasePlatform(platform)];
       try {
         step.result = await toolfn(step);
@@ -1121,6 +1127,7 @@ import * as x86 from './tools/x86'
 import * as arm from './tools/arm'
 import * as script from './tools/script'
 import * as ecs from './tools/ecs'
+import * as remote from './tools/remote'
 
 var TOOLS = {
   'dasm': dasm.assembleDASM,
@@ -1158,6 +1165,7 @@ var TOOLS = {
   'vasmarm': arm.assembleVASMARM,
   //'js': script.runJavascript,
   'ecs': ecs.assembleECS,
+  'remote': remote.buildRemote
 }
 
 var TOOL_PRELOADFS = {