verilog: use standard worker message formats

doc/notes.txt

@@ -84,7 +84,7 @@ TODO:
 - wasm dynamic linking of emulators (https://github.com/WebAssembly/tool-conventions/blob/master/DynamicLinking.md)
 - use alternate confirm/prompt dialogs
 - https://github.com/jvilk/BrowserFS
-- verilog: in .asm file editing .v files doesnt update
+- what if error in include file you can't edit b/c it never appears?
 
 
 WEB WORKER FORMAT

src/project.ts

@@ -264,19 +264,8 @@ export class CodeProject {
         console.log(err); // TODO?
       }
       if (!depends) depends = [];
-      if (this.platform_id.startsWith('verilog')) {
+      var workermsg = this.buildWorkerMessage(depends);
-        // TODO: should get rid of this msg format
+      this.worker.postMessage(workermsg);
-        this.worker.postMessage({
-          code:text,
-          path:getFilenameForPath(this.mainpath),
-          dependencies:depends,
-          platform:this.platform_id,
-          tool:this.platform.getToolForFilename(this.mainpath)
-        });
-      } else {
-        var workermsg = this.buildWorkerMessage(depends);
-        this.worker.postMessage(workermsg);
-      }
       this.isCompiling = true;
     });
   }

src/worker/workermain.ts

@@ -215,7 +215,8 @@ type FileEntry = {
 };
 
 type BuildOptions = {
-  mainFilePath : string
+  mainFilePath : string,
+  processFn?: (FileData) => FileData
 };
 
 // TODO
@@ -230,7 +231,6 @@ interface BuildStep extends WorkerBuildStep {
   generated?
   prefix?
   maxts?
-  dependencies?
 };
 
 var buildsteps : BuildStep[] = [];
@@ -270,8 +270,11 @@ function getWorkFileAsString(path:string) : string {
   return workfs[path] && workfs[path].data as string; // TODO
 }
 
-function populateEntry(fs, path:string, entry:FileEntry) {
+function populateEntry(fs, path:string, entry:FileEntry, options:BuildOptions) {
-  fs.writeFile(path, entry.data, {encoding:entry.encoding});
+  var data = entry.data;
+  if (options && options.processFn)
+    data = options.processFn(data);
+  fs.writeFile(path, data, {encoding:entry.encoding});
   fs.utime(path, entry.ts, entry.ts);
   console.log("<<<", path, entry.data.length);
 }
@@ -283,7 +286,11 @@ function gatherFiles(step:BuildStep, options?:BuildOptions) {
     for (var i=0; i<step.files.length; i++) {
       var path = step.files[i];
       var entry = workfs[path];
-      maxts = Math.max(maxts, entry.ts);
+      if (!entry) {
+        throw new Error("No entry for path '" + path + "'");
+      } else {
+        maxts = Math.max(maxts, entry.ts);
+      }
     }
   }
   else if (step.code) {
@@ -313,7 +320,7 @@ function populateFiles(step:BuildStep, fs, options?:BuildOptions) {
   if (!step.files) throw "call gatherFiles() first";
   for (var i=0; i<step.files.length; i++) {
     var path = step.files[i];
-    populateEntry(fs, path, workfs[path]);
+    populateEntry(fs, path, workfs[path], options);
   }
 }
 
@@ -485,14 +492,14 @@ function msvcErrorMatcher(errors:WorkerError[]) {
   }
 }
 
-function makeErrorMatcher(errors:WorkerError[], regex, iline:number, imsg:number, path:string) {
+function makeErrorMatcher(errors:WorkerError[], regex, iline:number, imsg:number, mainpath:string, ifilename?:number) {
   return function(s) {
     var matches = regex.exec(s);
     if (matches) {
       errors.push({
         line:parseInt(matches[iline]) || 1,
         msg:matches[imsg],
-        path:path
+        path:ifilename ? matches[ifilename] : mainpath
       });
     } else {
       console.log("??? "+s);
@@ -1268,19 +1275,7 @@ function detectTopModuleName(code:string) {
   return topmod;
 }
 
-function writeDependencies(depends:Dependency[], FS, errors:WorkerError[], callback?) {
+// cached stuff (TODO)
-  if (depends) {
-    for (var i=0; i<depends.length; i++) {
-      var d = depends[i];
-      var text = d.data;
-      if (callback)
-        text = callback(d, text);
-      if (text && FS)
-        FS.writeFile(d.filename, text, {encoding:'utf8'});
-    }
-  }
-}
-
 var jsasm_module_top;
 var jsasm_module_output;
 var jsasm_module_key;
@@ -1290,12 +1285,7 @@ function compileJSASM(asmcode:string, platform, options, is_inline) {
   var asm = new emglobal.exports.Assembler();
   var includes = [];
   asm.loadJSON = (filename:string) => {
-    var jsontext : string;
+    var jsontext = getWorkFileAsString(filename);
-    for (var dep of options.dependencies) {
-      if (dep.filename == filename)
-        jsontext = dep.data as string;
-    }
-    // TODO: var jsontext = getWorkFileAsString(filename) || getWorkFileAsString("local/"+filename);
     if (!jsontext) throw "could not load " + filename;
     return JSON.parse(jsontext);
   };
@@ -1307,24 +1297,26 @@ function compileJSASM(asmcode:string, platform, options, is_inline) {
   };
   var loaded_module = false;
   asm.loadModule = function(top_module) {
-    // TODO: cache module
     // compile last file in list
     loaded_module = true;
     var key = top_module + '/' + includes;
-    if (key != jsasm_module_key) {
+    if (jsasm_module_key != key) {
       jsasm_module_key = key;
-      jsasm_module_top = top_module;
+      jsasm_module_output = null;
-      var main_filename = includes[includes.length-1];
-      var code = '`include "' + main_filename + '"\n';
-      code += "/*\nmodule " + top_module + "\n*/\n";
-      var voutput = compileVerilator({code:code, platform:platform, dependencies:options.dependencies, path:options.path, tool:'verilator'}); // TODO
-      if (voutput.errors.length)
-        return voutput.errors[0].msg;
-      jsasm_module_output = voutput;
     }
+    jsasm_module_top = top_module;
+    var main_filename = includes[includes.length-1];
+    // TODO: take out .asm dependency
+    var voutput = compileVerilator({platform:platform, files:includes, path:main_filename, tool:'verilator'});
+    if (voutput)
+      jsasm_module_output = voutput;
   }
   var result = asm.assembleFile(asmcode);
   if (loaded_module && jsasm_module_output) {
+    // errors? return them
+    if (jsasm_module_output.errors && jsasm_module_output.errors.length)
+      return jsasm_module_output;
+    // return program ROM array
     var asmout = result.output;
     // TODO: unify
     result.output = jsasm_module_output.output;
@@ -1338,10 +1330,10 @@ function compileJSASM(asmcode:string, platform, options, is_inline) {
 }
 
 function compileJSASMStep(step:BuildStep) {
-  // TODO
+  gatherFiles(step);
-  var code = step.code;
+  var code = getWorkFileAsString(step.path);
   var platform = step.platform || 'verilog';
-  return compileJSASM(code, platform, step, false); // TODO
+  return compileJSASM(code, platform, step, false);
 }
 
 function compileInlineASM(code:string, platform, options, errors, asmlines) {
@@ -1374,58 +1366,62 @@ function compileInlineASM(code:string, platform, options, errors, asmlines) {
   return code;
 }
 
-// TODO: make compliant with standard msg format
 function compileVerilator(step:BuildStep) {
   loadNative("verilator_bin");
   loadGen("worker/verilator2js");
   var platform = step.platform || 'verilog';
   var errors = [];
   var asmlines = [];
-  // TODO? gatherFiles(step);
+  gatherFiles(step);
-  step.code = compileInlineASM(step.code, platform, step, errors, asmlines);
+  // compile verilog if files are stale
-  if (errors.length) {
+  var outjs = "main.js";
-    return {errors:errors};
+  if (staleFiles(step, [outjs])) {
-  }
+    var match_fn = makeErrorMatcher(errors, /%(.+?): (.+?):(\d+)?[:]?\s*(.+)/i, 3, 4, step.path, 2);
-  var code = step.code;
+    var verilator_mod = emglobal.verilator_bin({
-  var match_fn = makeErrorMatcher(errors, /%(.+?): (.+?:)?(\d+)?[:]?\s*(.+)/i, 3, 4, step.path);
+      instantiateWasm: moduleInstFn('verilator_bin'),
-  var verilator_mod = emglobal.verilator_bin({
+      noInitialRun:true,
-    instantiateWasm: moduleInstFn('verilator_bin'),
+      print:print_fn,
-    noInitialRun:true,
+      printErr:match_fn,
-    print:print_fn,
+      TOTAL_MEMORY:256*1024*1024,
-    printErr:match_fn,
+    });
-    TOTAL_MEMORY:256*1024*1024,
+    var code = getWorkFileAsString(step.path);
-  });
+    var topmod = detectTopModuleName(code);
-  var topmod = detectTopModuleName(code);
+    var FS = verilator_mod['FS'];
-  var FS = verilator_mod['FS'];
+    populateFiles(step, FS, {
-  populateFiles(step, FS, {mainFilePath:step.path});
+      mainFilePath:step.path,
-  writeDependencies(step.dependencies, FS, errors, function(d, code) {
+      processFn:(code) => {
-    return compileInlineASM(code, platform, step, errors, null);
+        return compileInlineASM(code, platform, step, errors, asmlines);
-  });
+      }
-  starttime();
+    });
-  try {
+    starttime();
-    var args = ["--cc", "-O3", "-DEXT_INLINE_ASM", "-DTOPMOD__"+topmod,
+    try {
-      "-Wall", "-Wno-DECLFILENAME", "-Wno-UNUSED", '--report-unoptflat',
+      var args = ["--cc", "-O3", "-DEXT_INLINE_ASM", "-DTOPMOD__"+topmod,
-      "--x-assign", "fast", "--noassert", "--pins-bv", "33",
+        "-Wall", "-Wno-DECLFILENAME", "-Wno-UNUSED", '--report-unoptflat',
-      "--top-module", topmod, step.path]
+        "--x-assign", "fast", "--noassert", "--pins-bv", "33",
-    verilator_mod.callMain(args);
+        "--top-module", topmod, step.path]
-  } catch (e) {
+      verilator_mod.callMain(args);
-    console.log(e);
+    } catch (e) {
-    errors.push({line:0,msg:"Compiler internal error: " + e});
+      console.log(e);
-  }
+      errors.push({line:0,msg:"Compiler internal error: " + e});
-  endtime("compile");
+    }
-  // remove boring errors
+    endtime("compile");
-  errors = errors.filter(function(e) { return !/Exiting due to \d+/.exec(e.msg); }, errors);
+    // remove boring errors
-  errors = errors.filter(function(e) { return !/Use ["][/][*]/.exec(e.msg); }, errors);
+    errors = errors.filter(function(e) { return !/Exiting due to \d+/.exec(e.msg); }, errors);
-  if (errors.length) {
+    errors = errors.filter(function(e) { return !/Use ["][/][*]/.exec(e.msg); }, errors);
-    return {errors:errors};
+    if (errors.length) {
-  }
+      return {errors:errors};
-  try {
+    }
-    var h_file = FS.readFile("obj_dir/V"+topmod+".h", {encoding:'utf8'});
+    try {
-    var cpp_file = FS.readFile("obj_dir/V"+topmod+".cpp", {encoding:'utf8'});
+      var h_file = FS.readFile("obj_dir/V"+topmod+".h", {encoding:'utf8'});
-    var rtn = translateVerilatorOutputToJS(h_file, cpp_file);
+      var cpp_file = FS.readFile("obj_dir/V"+topmod+".cpp", {encoding:'utf8'});
-    putWorkFile("main.js", rtn.output.code);
+      var rtn = translateVerilatorOutputToJS(h_file, cpp_file);
-    if (!anyTargetChanged(step, ["main.js"]))
+      putWorkFile(outjs, rtn.output.code);
-      return;
+      if (!anyTargetChanged(step, [outjs]))
+        return;
+    } catch(e) {
+      console.log(e);
+      return {errors:errors};
+    }
     //rtn.intermediate = {listing:h_file + cpp_file}; // TODO
     var listings = {};
     // TODO: what if found in non-top-module?
@@ -1436,9 +1432,6 @@ function compileVerilator(step:BuildStep) {
       errors: errors,
       listings: listings,
     };
-  } catch(e) {
-    console.log(e);
-    return {errors:errors};
   }
 }
 
@@ -1459,7 +1452,6 @@ function compileYosys(step:BuildStep) {
   var topmod = detectTopModuleName(code);
   var FS = yosys_mod['FS'];
   FS.writeFile(topmod+".v", code);
-  writeDependencies(step.dependencies, FS, errors);
   starttime();
   try {
     yosys_mod.callMain(["-q", "-o", topmod+".json", "-S", topmod+".v"]);
@@ -1726,7 +1718,6 @@ function executeBuildSteps() {
   buildstartseq = workerseq;
   while (buildsteps.length) {
     var step = buildsteps.shift(); // get top of array
-    var code = step.code;
     var platform = step.platform;
     var toolfn = TOOLS[step.tool];
     if (!toolfn) throw "no tool named " + step.tool;
@@ -1797,7 +1788,6 @@ function handleMessage(data : WorkerMessage) : WorkerResult {
     workfs = {};
     return;
   }
-  // (code,platform,tool,dependencies)
   buildsteps = [];
   // file updates
   if (data.updates) {

test/cli/testworker.js

@@ -173,7 +173,7 @@ describe('Worker', function() {
   */
   it('should compile verilog example', function(done) {
     var csource = ab2str(fs.readFileSync('presets/verilog/lfsr.v'));
-    var msgs = [{code:csource, platform:"verilog", tool:"verilator", dependencies:[], path:'main.v'}];
+    var msgs = [{code:csource, platform:"verilog", tool:"verilator", path:'main.v'}];
     var done2 = function(err, msg) {
       var jscode = msg.output.code;
       var fn = new Function(jscode);
@@ -184,18 +184,20 @@ describe('Worker', function() {
   });
   it('should NOT compile verilog example', function(done) {
     var csource = "foobar";
-    var msgs = [{code:csource, platform:"verilog", tool:"verilator", dependencies:[], path:'foomain.v'}];
+    var msgs = [{code:csource, platform:"verilog", tool:"verilator", path:'foomain.v'}];
     doBuild(msgs, done, 0, 0, 1);
   });
   it('should compile verilog inline assembler (JSASM)', function(done) {
-    var csource = ab2str(fs.readFileSync('presets/verilog/racing_game_cpu.v'));
+    var dependfiles = ["racing_game_cpu.v", "hvsync_generator.v", "sprite_bitmap.v", "sprite_renderer.v", "cpu8.v", "femto8.json"];
-    var dependfiles = ["hvsync_generator.v", "sprite_bitmap.v", "sprite_renderer.v", "cpu8.v", "femto8.json"];
     var depends = [];
     for (var dfile of dependfiles) {
       var code = ab2str(fs.readFileSync('presets/verilog/' + dfile));
-      depends.push({filename:dfile, data:code, prefix:"verilog"});
+      depends.push({path:dfile, data:code});
     }
-    var msgs = [{code:csource, platform:"verilog", tool:"verilator", dependencies:depends, path:'racing_game_cpu.v'}];
+    var msgs = [{
+      updates:depends,
+      buildsteps:[{platform:"verilog", tool:"verilator", path:'racing_game_cpu.v', files:dependfiles}]
+    }];
     var done2 = function(err, msg) {
       var jscode = msg.output.code;
       var fn = new Function(jscode);
@@ -205,14 +207,16 @@ describe('Worker', function() {
     doBuild(msgs, done2, 51459, 0, 0);
   });
   it('should compile verilog assembler file (JSASM)', function(done) {
-    var csource = ab2str(fs.readFileSync('presets/verilog/test2.asm'));
+    var dependfiles = ["test2.asm", "hvsync_generator.v", "font_cp437_8x8.v", "ram.v", "tile_renderer.v", "sprite_scanline_renderer.v", "lfsr.v", "sound_generator.v", "cpu16.v", "cpu_platform.v", "femto16.json"];
-    var dependfiles = ["hvsync_generator.v", "font_cp437_8x8.v", "ram.v", "tile_renderer.v", "sprite_scanline_renderer.v", "lfsr.v", "sound_generator.v", "cpu16.v", "cpu_platform.v", "femto16.json"];
     var depends = [];
     for (var dfile of dependfiles) {
       var code = ab2str(fs.readFileSync('presets/verilog/' + dfile));
-      depends.push({filename:dfile, data:code, prefix:"verilog"});
+      depends.push({path:dfile, data:code});
     }
-    var msgs = [{code:csource, platform:"verilog", tool:"jsasm", dependencies:depends, path:'main.asm'}];
+    var msgs = [{
+      updates:depends,
+      buildsteps:[{platform:"verilog", tool:"jsasm", path:'test2.asm', files:dependfiles}]
+    }];
     var done2 = function(err, msg) {
       var jscode = msg.output.code;
       var fn = new Function(jscode);