ProjectFilesystem class

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "8bitworkshop",
-  "version": "3.7.0",
+  "version": "3.7.1",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

src/ide/project.ts

@@ -1,12 +1,72 @@
 
 import { FileData, Dependency, SourceLine, SourceFile, CodeListing, CodeListingMap, WorkerError, Segment, WorkerResult } from "../common/workertypes";
-import { getFilenamePrefix, getFolderForPath, isProbablyBinary, getBasePlatform } from "../common/util";
+import { getFilenamePrefix, getFolderForPath, isProbablyBinary, getBasePlatform, getWithBinary } from "../common/util";
 import { Platform } from "../common/baseplatform";
 
+export interface ProjectFilesystem {
+  getFileData(path: string) : Promise<FileData>;
+  setFileData(path: string, data: FileData) : Promise<void>;
+}
+
+export class WebPresetsFileSystem implements ProjectFilesystem {
+  preset_id : string;
+  constructor(platform_id: string) {
+    // found on remote fetch?
+    this.preset_id = getBasePlatform(platform_id); // remove .suffix from preset name
+  }
+  async getRemoteFile(path: string): Promise<FileData> {
+    return new Promise( (yes,no)=> {
+      return getWithBinary(path, yes, isProbablyBinary(path) ? 'arraybuffer' : 'text');
+    });
+  }
+  async getFileData(path: string) : Promise<FileData> {
+    var webpath = "presets/" + this.preset_id + "/" + path;
+    var data = await this.getRemoteFile(webpath);
+    if (data) console.log("read",webpath,data.length,'bytes');
+    return data;
+  }
+  async setFileData(path: string, data: FileData) : Promise<void> {
+    // not implemented
+  }
+}
+
+export class NullFilesystem implements ProjectFilesystem {
+  gets = [];
+  sets = [];
+  getFileData(path: string): Promise<FileData> {
+    this.gets.push(path);
+    return null;
+  }
+  setFileData(path: string, data: FileData): Promise<void> {
+    this.sets.push(path);
+    return;
+  }
+  
+}
+
+export class LocalForageFilesystem implements ProjectFilesystem {
+  basefs: ProjectFilesystem;
+  store: any;
+  constructor(store: any, basefs: ProjectFilesystem) {
+    this.store = store;
+    this.basefs = basefs;
+  }
+  async getFileData(path: string): Promise<FileData> {
+    var data = await this.store.getItem(path);
+    if (data == null) {
+      data = await this.basefs.getFileData(path);
+    }
+    return data;
+  }
+  async setFileData(path: string, data: FileData): Promise<void> {
+    await this.store.setItem(path, data);
+    await this.basefs.setFileData(path, data);
+  }
+}
+
 type BuildResultCallback = (result:WorkerResult) => void;
 type BuildStatusCallback = (busy:boolean) => void;
 type IterateFilesCallback = (path:string, data:FileData) => void;
-type GetRemoteCallback = (path:string, callback:(data:FileData) => void, datatype:'text'|'arraybuffer') => any;
 
 function isEmptyString(text : FileData) {
   return typeof text == 'string' && text.trim && text.trim().length == 0;
@@ -22,21 +82,18 @@ export class CodeProject {
   worker : Worker;
   platform_id : string;
   platform : Platform;
-  store : any;
   isCompiling : boolean = false;
   filename2path = {}; // map stripped paths to full paths
-  persistent : boolean = true; // set to true and won't modify store
+  filesystem : ProjectFilesystem;
 
   callbackBuildResult : BuildResultCallback;
   callbackBuildStatus : BuildStatusCallback;
-  callbackGetRemote : GetRemoteCallback;
-  callbackStoreFile : IterateFilesCallback;
 
-  constructor(worker, platform_id:string, platform, store) {
+  constructor(worker, platform_id:string, platform, filesystem: ProjectFilesystem) {
     this.worker = worker;
     this.platform_id = platform_id;
     this.platform = platform;
-    this.store = store;
+    this.filesystem = filesystem;
 
     worker.onmessage = (e) => {
       this.receiveWorkerMessage(e.data);
@@ -163,13 +220,7 @@ export class CodeProject {
   }
 
   updateFileInStore(path:string, text:FileData) {
-    // protect against accidential whole-file deletion
+    this.filesystem.setFileData(path, text);
-    if (this.persistent && !isEmptyString(text)) {
-      this.store.setItem(path, text);
-    }
-    if (this.callbackStoreFile != null) {
-      this.callbackStoreFile(path, text);
-    }
   }
 
   // TODO: test duplicate files, local paths mixed with presets
@@ -210,10 +261,9 @@ export class CodeProject {
   }
 
   // TODO: get local file as well as presets?
-  loadFiles(paths:string[]) : Promise<Dependency[]> {
+  async loadFiles(paths:string[]) : Promise<Dependency[]> {
-   return new Promise( (yes,no) => {
     var result : Dependency[] = [];
-    var addResult = (path, data) => {
+    var addResult = (path:string, data:FileData) => {
       result.push({
         path:path,
         filename:this.stripLocalPath(path),
@@ -221,53 +271,24 @@ export class CodeProject {
         data:data
       });
     }
-    var loadNext = () => {
+    for (var path of paths) {
-      var path = paths.shift();
+      // look in cache
-      if (!path) {
+      if (path in this.filedata) { // found in cache?
-        // finished loading all files; return result
+        var data = this.filedata[path];
-        yes(result);
+        if (data) {
+          addResult(path, data);
+        }
       } else {
-        // look in cache
+        var data = await this.filesystem.getFileData(path);
-        if (path in this.filedata) { // found in cache?
+        if (data) {
-          var data = this.filedata[path];
+          this.filedata[path] = data; // do not update store, just cache
-          if (data)
+          addResult(path, data);
-            addResult(path, data);
-          loadNext();
         } else {
-          // look in store
+          this.filedata[path] = null; // mark entry as invalid
-          this.store.getItem(path, (err, value) => {
-            if (err) { // err fetching from store
-              no(err);
-            } else if (value) { // found in store?
-              this.filedata[path] = value; // do not update store, just cache
-              addResult(path, value);
-              loadNext();
-            } else {
-              // found on remote fetch?
-              var preset_id = this.platform_id;
-              preset_id = getBasePlatform(preset_id); // remove .suffix from preset name
-              var webpath = "presets/" + preset_id + "/" + path;
-              // try to GET file, use file ext to determine text/binary
-              this.callbackGetRemote( webpath, (data:FileData) => {
-                if (data == null) {
-                  console.log("Could not load preset file", path);
-                  this.filedata[path] = null; // mark cache entry as invalid
-                } else {
-                  if (data instanceof ArrayBuffer)
-                    data = new Uint8Array(data); // convert to typed array
-                  console.log("read",webpath,data.length,'bytes');
-                  this.filedata[path] = data; // do not update store, just cache
-                  addResult(path, data);
-                }
-                loadNext();
-              }, isProbablyBinary(path) ? 'arraybuffer' : 'text');
-            }
-          });
         }
       }
     }
-    loadNext(); // load first file
+    return result;
-   });
   }
 
   getFile(path:string):FileData {

src/ide/ui.ts

@@ -3,7 +3,7 @@
 
 import $ = require("jquery");
 import * as bootstrap from "bootstrap";
-import { CodeProject } from "./project";
+import { CodeProject, LocalForageFilesystem, ProjectFilesystem, WebPresetsFileSystem } from "./project";
 import { WorkerResult, WorkerOutput, VerilogOutput, SourceFile, WorkerError, FileData } from "../common/workertypes";
 import { ProjectWindows } from "./windows";
 import { Platform, Preset, DebugSymbols, DebugEvalCondition, isDebuggable, EmuState } from "../common/baseplatform";
@@ -170,14 +170,16 @@ function unsetLastPreset() {
 }
 
 function initProject() {
-  current_project = new CodeProject(newWorker(), platform_id, platform, store);
+  var filesystem = new LocalForageFilesystem(store, new WebPresetsFileSystem(platform_id));
+  current_project = new CodeProject(newWorker(), platform_id, platform, filesystem);
   projectWindows = new ProjectWindows($("#workspace")[0] as HTMLElement, current_project);
   if (isElectronWorkspace) {
+    // TODO
+    /*
     current_project.persistent = false;
     current_project.callbackGetRemote = getElectronFile;
     current_project.callbackStoreFile = putWorkspaceFile;
-  } else {
+    */
-    current_project.callbackGetRemote = getWithBinary;
   }
   current_project.callbackBuildResult = (result:WorkerResult) => {
     setCompileOutput(result);

test/cli/testgithub.js

@@ -17,7 +17,9 @@ var test_platform_id = "_TEST";
 function newGH(store, platform_id) {
   localStorage.clear();
   // pzpinfo user
-  var project = new prj.CodeProject({}, platform_id||test_platform_id, null, store);
+  var pid = platform_id||test_platform_id;
+  var fs = new prj.LocalForageFilesystem(store, new prj.NullFilesystem());
+  var project = new prj.CodeProject({}, pid, null, fs);
   project.mainPath = 'local/main.asm';
   project.updateFileInStore(project.mainPath, '\torg $0 ; test\n');
   return new serv.GithubService(Octokat, process.env.TEST8BIT_GITHUB_TOKEN, store, project);
@@ -25,7 +27,7 @@ function newGH(store, platform_id) {
 
 const t0 = new Date().getTime();
 
-describe('Store', function() {
+describe('Github', function() {
 
   it('Should import from Github (check README)', function(done) {
     var store = mstore.createNewPersistentStore('vcs', function(store) {

test/cli/teststore.js

@@ -12,6 +12,10 @@ var prj = require("gen/ide/project.js");
 
 var test_platform_id = "_TEST";
 
+function newFilesystem(store, platform_id) {
+  return new prj.LocalForageFilesystem(store, new prj.NullFilesystem());
+}
+
 describe('Store', function () {
 
   it('Should load local project', function (done) {
@@ -19,14 +23,9 @@ describe('Store', function () {
     store.setItem('local/test', 'a');
     var worker = {};
     var platform = {};
-    var project = new prj.CodeProject(worker, test_platform_id, platform, store);
+    var project = new prj.CodeProject(worker, test_platform_id, platform, newFilesystem(store, test_platform_id));
-    var remote = [];
-    project.callbackGetRemote = function (path, success, datatype) {
-      remote.push(path);
-      success();
-    };
     project.loadFiles(['local/test', 'test']).then((result) => {
-      assert.deepEqual(["presets/_TEST/test"], remote);
+      assert.deepEqual(["test"], project.filesystem.basefs.gets);
       assert.deepEqual([{ path: 'local/test', filename: 'local/test', data: 'a', link: true }], result);
       done();
     });
@@ -53,7 +52,7 @@ describe('Store', function () {
     var platform = {
       getToolForFilename: function (fn) { return 'dasm'; },
     };
-    var project = new prj.CodeProject(worker, test_platform_id, platform, store);
+    var project = new prj.CodeProject(worker, test_platform_id, platform, newFilesystem(store, test_platform_id));
     project.callbackBuildStatus = function (b) { msgs.push(b) };
     project.callbackBuildResult = function (b) { msgs.push(1) };
     project.updateFile('test.a', ' lda #0');
@@ -78,7 +77,7 @@ describe('Store', function () {
     };
     var platform = {
     };
-    var project = new prj.CodeProject(worker, test_platform_id, platform, store);
+    var project = new prj.CodeProject(worker, test_platform_id, platform, newFilesystem(store, test_platform_id));
     project.callbackBuildStatus = function (b) { msgs.push(b) };
     project.callbackBuildResult = function (b) { msgs.push(1) };
     var buildresult = {