better caching when reading directories

Makefile

@@ -3,7 +3,8 @@ LDFLAGS=-L/usr/local/lib
 SDLLIBS=-lSDL2 -lpthread
 FBLIBS=-lpthread
 
-CXXFLAGS=-Wall -I/usr/include/SDL2 -I .. -I . -I apple -I nix -I sdl -I/usr/local/include/SDL2 -g -O3 -DSUPPRESSREALTIME -DSTATICALLOC
+CFLAGS=-Wall -I/usr/include/SDL2 -I .. -I . -I apple -I nix -I sdl -I/usr/local/include/SDL2 -g -DSUPPRESSREALTIME -DSTATICALLOC
+CXXFLAGS=-Wall -I/usr/include/SDL2 -I .. -I . -I apple -I nix -I sdl -I/usr/local/include/SDL2 -g -DSUPPRESSREALTIME -DSTATICALLOC
 
 TSRC=cpu.cpp util/testharness.cpp
 

apple/woz.cpp

@@ -3,9 +3,7 @@
 #include "crc32.h"
 #include "nibutil.h"
 #include "version.h"
-#ifdef TEENSYDUINO
 #include "fscompat.h"
-#endif
 
 extern    uint32_t FreeRamEstimate();
 

bios.cpp

@@ -13,6 +13,21 @@
 extern Bounce resetButtonDebouncer;
 #endif
 
+// Experimenting with using EXTMEM to cache all the filenames in a directory
+#ifndef TEENSYDUINO
+#define EXTMEM
+#endif
+
+struct _cacheEntry {
+  char fn[BIOS_MAXPATH];
+};
+#define BIOSCACHESIZE 1024 // hope that's enough files?
+EXTMEM char cachedPath[BIOS_MAXPATH] = {0};
+EXTMEM char cachedFilter[BIOS_MAXPATH] = {0};
+EXTMEM struct _cacheEntry biosCache[BIOSCACHESIZE];
+uint16_t numCacheEntries = 0;
+
+
 enum {
   ACT_EXIT = 1,
   ACT_RESET = 2,
@@ -92,12 +107,12 @@ void BIOS::DrawMenuBar()
 
   for (int i=0; i<NUM_TITLES; i++) {
     for (int x=0; x<titleWidths[i] + 2*XPADDING; x++) {
-      g_display->drawUIPixel(xpos+x, 0, 0xFFFF);
+      g_display->drawPixel(xpos+x, 0, 0xFFFF);
-      g_display->drawUIPixel(xpos+x, 16, 0xFFFF);
+      g_display->drawPixel(xpos+x, 16, 0xFFFF);
     }
     for (int y=0; y<=16; y++) {
-      g_display->drawUIPixel(xpos, y, 0xFFFF);
+      g_display->drawPixel(xpos, y, 0xFFFF);
-      g_display->drawUIPixel(xpos + titleWidths[i] + 2*XPADDING, y, 0xFFFF);
+      g_display->drawPixel(xpos + titleWidths[i] + 2*XPADDING, y, 0xFFFF);
     }
 
     xpos += XPADDING;
@@ -111,6 +126,10 @@ void BIOS::DrawMenuBar()
 
 bool BIOS::runUntilDone()
 {
+  // Reset the cache
+  cachedPath[0] = 0;
+  numCacheEntries = 0;
+
   g_filemanager->getRootPath(rootPath, sizeof(rootPath));
 
   // FIXME: abstract these constant speeds
@@ -566,7 +585,7 @@ void BIOS::DrawHardwareMenu()
   uint16_t volCutoff = 300.0 * (float)((float) g_volume / 15.0);
   for (uint8_t y=234; y<=235; y++) {
     for (uint16_t x = 0; x< 300; x++) {
-      g_display->drawUIPixel( x, y, x <= volCutoff ? 0xFFFF : 0x0010 );
+      g_display->drawPixel( x, y, x <= volCutoff ? 0xFFFF : 0x0010 );
     }
   }
 }
@@ -704,9 +723,9 @@ bool BIOS::SelectDiskImage(const char *filter)
 {
   int8_t sel = 0;
   int8_t page = 0;
-
+  uint16_t fileCount = 0;
   while (1) {
-    DrawDiskNames(page, sel, filter);
+    fileCount = DrawDiskNames(page, sel, filter);
 
     while (!g_keyboard->kbhit())
       ;
@@ -729,31 +748,31 @@ bool BIOS::SelectDiskImage(const char *filter)
 	//	else sel = BIOS_MAXFILES + 1;
       }
       else if (sel == BIOS_MAXFILES+1) {
-	page++;
+	if (fileCount == BIOS_MAXFILES) { // don't let them select 'Next' if there were no files in the list or if the list isn't full
-	//sel = 0;
+	  page++;
-      } else {
-	if (strcmp(fileDirectory[sel-1], "../") == 0) {
-	  // Go up a directory (strip a directory name from rootPath)
-	  stripDirectory();
-	  page = 0;
 	  //sel = 0;
-	  continue;
-	} else if (fileDirectory[sel-1][strlen(fileDirectory[sel-1])-1] == '/') {
-	  // Descend in to the directory. FIXME: file path length?
-	  strcat(rootPath, fileDirectory[sel-1]);
-	  sel = 0;
-	  page = 0;
-	  continue;
-	} else {
-	  selectedFile = sel - 1;
-	  g_display->flush();
-	  return true;
 	}
+      } else if (strcmp(fileDirectory[sel-1], "../") == 0) {
+	// Go up a directory (strip a directory name from rootPath)
+	stripDirectory();
+	page = 0;
+	//sel = 0;
+	continue;
+      } else if (fileDirectory[sel-1][strlen(fileDirectory[sel-1])-1] == '/') {
+	// Descend in to the directory. FIXME: file path length?
+	strcat(rootPath, fileDirectory[sel-1]);
+	sel = 0;
+	page = 0;
+	continue;
+      } else {
+	selectedFile = sel - 1;
+	g_display->flush();
+	return true;
       }
       break;
     }
   }
-  g_display->flush();
+  /* NOTREACHED */
 }
 
 void BIOS::stripDirectory()
@@ -772,9 +791,9 @@ void BIOS::stripDirectory()
   }
 }
 
-void BIOS::DrawDiskNames(uint8_t page, int8_t selection, const char *filter)
+uint16_t BIOS::DrawDiskNames(uint8_t page, int8_t selection, const char *filter)
 {
-  uint8_t fileCount = GatherFilenames(page, filter);
+  uint16_t fileCount = GatherFilenames(page, filter);
   g_display->clrScr();
   g_display->drawString(M_NORMAL, 0, 12, "BIOS Configuration - pick disk");
 
@@ -795,37 +814,78 @@ void BIOS::DrawDiskNames(uint8_t page, int8_t selection, const char *filter)
   }
 
   // FIXME: this doesn't accurately say whether or not there *are* more.
-  if (fileCount == BIOS_MAXFILES || fileCount == 0) {
+  if (fileCount < BIOS_MAXFILES) {
     g_display->drawString((i+1 == selection) ? M_SELECTDISABLED : M_DISABLED, 10, 50 + 14 * (i+1), "<Next>");
   } else {
     g_display->drawString(i+1 == selection ? M_SELECTED : M_NORMAL, 10, 50 + 14 * (i+1), "<Next>");
   }
 
   g_display->flush();
+  return fileCount;
 }
 
+uint16_t BIOS::cacheAllEntries(const char *filter)
+{
+  // If we've already cached this directory, then just return it
+  if (numCacheEntries && !strcmp(cachedPath, rootPath) && !strcmp(cachedFilter, filter))
+    return numCacheEntries;
 
-uint8_t BIOS::GatherFilenames(uint8_t pageOffset, const char *filter)
+  // Otherwise flush the cache and start over
+  numCacheEntries = 0;
+  strcpy(cachedPath, rootPath);
+  strcpy(cachedFilter, filter);
+  
+  // This could be a lengthy process, so...
+  g_display->clrScr();
+  g_display->drawString(M_SELECTED,
+                        0,
+                        0,
+                        "Loading...");
+  g_display->flush();
+  
+  // read all the entries we can find
+  int16_t idx = 0;
+  while (1) {
+    struct _cacheEntry *ce = &biosCache[numCacheEntries];
+    idx = g_filemanager->readDir(rootPath, filter, ce->fn, idx, BIOS_MAXPATH);
+    if (idx == -1) {
+      return numCacheEntries;
+    }
+    idx++;
+    numCacheEntries++;
+    if (numCacheEntries >= BIOSCACHESIZE-1) {
+      return numCacheEntries;
+    }
+  }
+  /* NOTREACHED */
+}
+
+uint16_t BIOS::GatherFilenames(uint8_t pageOffset, const char *filter)
 {
   uint8_t startNum = 10 * pageOffset;
   uint8_t count = 0; // number we're including in our listing
 
+  uint16_t numEntriesTotal = cacheAllEntries(filter);
+  if (numEntriesTotal > BIOSCACHESIZE) {
+    // ... umm, this is a problem. FIXME?
+  }
+  struct _cacheEntry *nextEntry = biosCache;
+  while (startNum) {
+    nextEntry++;
+    startNum--;
+  }
+
   while (1) {
-    char fn[BIOS_MAXPATH];
+    if (nextEntry->fn[0] == 0)
-    int8_t idx = g_filemanager->readDir(rootPath, filter, fn, startNum + count, BIOS_MAXPATH);
-
-    if (idx == -1) {
       return count;
-    }
 
-    idx++;
+    strncpy(fileDirectory[count], nextEntry->fn, BIOS_MAXPATH);
-
-    strncpy(fileDirectory[count], fn, BIOS_MAXPATH);
     count++;
 
     if (count >= BIOS_MAXFILES) {
       return count;
     }
+    nextEntry++;
   }
 }
 
@@ -840,7 +900,7 @@ void BIOS::showAbout()
 
   g_display->drawString(M_NORMAL, 
 			15, 20,
-			"(c) 2017 Jorj Bauer");
+			"(c) 2017-2020 Jorj Bauer");
 
   g_display->drawString(M_NORMAL,
 			15, 38,

bios.h

@@ -36,8 +36,8 @@ class BIOS {
   void ColdReboot();
 
   bool SelectDiskImage(const char *filter);
-  void DrawDiskNames(uint8_t page, int8_t selection, const char *filter);
+  uint16_t DrawDiskNames(uint8_t page, int8_t selection, const char *filter);
-  uint8_t GatherFilenames(uint8_t pageOffset, const char *filter);
+  uint16_t GatherFilenames(uint8_t pageOffset, const char *filter);
 
   void ConfigurePaddles();
 
@@ -45,6 +45,8 @@ class BIOS {
 
   void showAbout();
 
+  uint16_t cacheAllEntries(const char *filter);
+
  private:
   int8_t selectedFile;
   char fileDirectory[BIOS_MAXFILES][BIOS_MAXPATH+1];

filemanager.h

@@ -126,7 +126,8 @@ class FileManager {
 
   virtual const char *fileName(int8_t fd) = 0;
 
-  virtual int8_t readDir(const char *where, const char *suffix, char *outputFN, int8_t startIdx, uint16_t maxlen) = 0;
+  virtual int16_t readDir(const char *where, const char *suffix, char *outputFN, int16_t startIdx, uint16_t maxlen) = 0;
+  virtual void closeDir() = 0;
 
   virtual void getRootPath(char *toWhere, int8_t maxLen) = 0;
 

nix/nix-filemanager.cpp

@@ -65,27 +65,38 @@ const char *NixFileManager::fileName(int8_t fd)
   return cachedNames[fd];
 }
 
-int8_t NixFileManager::readDir(const char *where, const char *suffix, char *outputFN, int8_t startIdx, uint16_t maxlen)
+// FIXME make these member vars instead of globals
+static DIR *dirp = NULL;
+
+void NixFileManager::closeDir()
 {
-  int idx = 1;
+  if (dirp) {
-  if (strcmp(where, ROOTDIR)) {
+    closedir(dirp);
-    // First entry is always "../"
+    dirp = NULL;
-    if (startIdx == 0) {
+  }
+}
+
+int16_t NixFileManager::readDir(const char *where, const char *suffix, char *outputFN, int16_t startIdx, uint16_t maxlen)
+{
+  if (startIdx == 0 || !dirp) {
+    // This is an openDir() -- so reset state, open the directory, etc.
+    closeDir();
+    dirp = opendir(where);
+    if (!dirp)
+      return -1;
+  }
+  
+  if (startIdx == 0) {
+    if (strcmp(where, ROOTDIR)) {
+      // As long as we're not at the root, we start with "../"
       strcpy(outputFN, "../");
       return 0;
     }
-  } else {
-    idx = 0; // we skipped ROOTDIR
   }
 
-  DIR *dirp = opendir(where);
-  if (!dirp)
-    return -1;
-
-  struct dirent *dp;
-
   outputFN[0] = '\0';
 
+  struct dirent *dp;
   while ((dp = readdir(dirp)) != NULL) {
     if (dp->d_name[0] == '.') {
       // Skip any dot files (and dot directories)
@@ -128,30 +139,20 @@ int8_t NixFileManager::readDir(const char *where, const char *suffix, char *outp
       }
     }
     // If we get here, it's something we want to show.
-    if (idx == startIdx) {
+    strncpy(outputFN, dp->d_name, maxlen-1);
-      // Fill in the reply
+    
-      strncpy(outputFN, dp->d_name, maxlen-1);
+    if (dp->d_type & DT_DIR) {
-
+      // suffix
-      if (dp->d_type & DT_DIR) {
+      strcat(outputFN, "/");
-	// suffix
-	strcat(outputFN, "/");
-      }
-      break;
     }
-
+    
-    // Next!
+    return startIdx;
-    idx++;
   }
 
   // Exited the loop - all done.
-  closedir(dirp);
+  // didn't find any more
-
+  closeDir();
-  if (!outputFN[0]) {
+  return -1;
-    // didn't find any more
-    return -1;
-  }
-
-  return idx;
 }
 
 void NixFileManager::getRootPath(char *toWhere, int8_t maxLen)

nix/nix-filemanager.h

@@ -14,7 +14,8 @@ class NixFileManager : public FileManager {
 
   virtual const char *fileName(int8_t fd);
   
-  virtual int8_t readDir(const char *where, const char *suffix, char *outputFN, int8_t startIdx, uint16_t maxlen);
+  virtual int16_t readDir(const char *where, const char *suffix, char *outputFN, int16_t startIdx, uint16_t maxlen);
+  virtual void closeDir();
 
   void getRootPath(char *toWhere, int8_t maxLen);
 

sdl/fscompat.h

@@ -0,0 +1,18 @@
+/* Filesystem compatability layer.
+ *
+ * Right now I've got a hokey (but well abstracted) g_filemanager object
+ * that has a bunch of file operation primitives which don't conform to 
+ * POSIX at all. The woz code comes from my Wozzle utility, and I'd rather
+ * not butcher the heck out of it for the sake of Aiie... so this is 
+ * bridging that gap, for now.
+ */
+
+#include "globals.h"
+
+#define open(path, flags, perms) g_filemanager->openFile(path)
+#define close(filedes) g_filemanager->closeFile(filedes)
+#define write(filedes,buf,nbyte) g_filemanager->write(filedes,buf,nbyte)
+#define read(filedes,buf,nbyte) g_filemanager->read(filedes,buf,nbyte)
+#define lseek(filedes,offset,whence) g_filemanager->lseek(filedes,offset,whence)
+
+  

sdl/sdl-display.h

@@ -10,7 +10,7 @@
 
 // scale can be 1,2,4. '1' is half-width at the highest resolution
 // (80-col mode). '2' is full width. '4' is double full width.
-#define SDLDISPLAY_SCALE 1
+#define SDLDISPLAY_SCALE 2
 #define SDLDISPLAY_WIDTH (320*SDLDISPLAY_SCALE)
 #define SDLDISPLAY_HEIGHT (240*SDLDISPLAY_SCALE)
 

teensy/fscompat.h

@@ -0,0 +1,25 @@
+/* Filesystem compatability layer.
+ *
+ * Right now I've got a hokey (but well abstracted) g_filemanager object
+ * that has a bunch of file operation primitives which don't conform to 
+ * POSIX at all. The woz code comes from my Wozzle utility, and I'd rather
+ * not butcher the heck out of it for the sake of Aiie... so this is 
+ * bridging that gap, for now.
+ */
+
+#include "globals.h"
+#include <Arduino.h>
+#include <TeensyThreads.h>
+
+#define open(path, flags, perms) g_filemanager->openFile(path)
+#define close(filedes) g_filemanager->closeFile(filedes)
+#define write(filedes,buf,nbyte) g_filemanager->write(filedes,buf,nbyte)
+#define read(filedes,buf,nbyte) g_filemanager->read(filedes,buf,nbyte)
+#define lseek(filedes,offset,whence) g_filemanager->lseek(filedes,offset,whence)
+
+Threads::Mutex serlock;
+static char fsbuf[200];
+#define printf(x, ...) {sprintf(fsbuf, x, ##__VA_ARGS__); serlock.lock(); Serial.println(fsbuf); Serial.flush(); Serial.send_now(); serlock.unlock();}
+#define fprintf(f, x, ...) {sprintf(fsbuf, x, ##__VA_ARGS__); serlock.lock(); Serial.println(fsbuf); Serial.flush(); Serial.send_now(); serlock.lock();}
+#define perror(x) {serlock.lock();Serial.println(x);Serial.flush(); Serial.send_now(); serlock.unlock();}
+  

teensy/teensy-filemanager.cpp

@@ -78,28 +78,42 @@ const char *TeensyFileManager::fileName(int8_t fd)
   return cachedNames[fd];
 }
 
+File outerDir;
+
+void TeensyFileManager::closeDir()
+{
+  // FIXME: this should Threads::Scope lock, but it's being called
+  // from readDir, so that would block
+  if (outerDir) {
+    outerDir.close();
+  }
+}
+
 // suffix may be comma-separated
-int8_t TeensyFileManager::readDir(const char *where, const char *suffix, char *outputFN, int8_t startIdx, uint16_t maxlen)
+int16_t TeensyFileManager::readDir(const char *where, const char *suffix, char *outputFN, int16_t startIdx, uint16_t maxlen)
 {
   Threads::Scope locker(fslock);
-  //  ... open, read, save next name, close, return name. Horribly
-  //  inefficient but hopefully won't break the sd layer. And if it
-  //  works then we can make this more efficient later.
 
-  // First entry is always "../"
+  // First entry is always "../" if we're in a subdir of the root
-  if (startIdx == 0) {
+  if (startIdx == 0 || !outerDir) {
+    if (outerDir)
+      closeDir();
+    outerDir = SD.open(where, FILE_READ);
+    if (!outerDir)
+      return -1;
+    if (strcmp(where, "/")) { // FIXME: is this correct for the root?
       strcpy(outputFN, "../");
       return 0;
+    }
   }
 
-  int8_t idxCount = 1;
+  outputFN[0] = '\0';
-  File f = SD.open(where, FILE_READ);
 
   while (1) {
-    File e = f.openNextFile();
+    File e = outerDir.openNextFile();
     if (!e) {
       // No more - all done.
-      f.close();
+      closeDir();
       return -1;
     }
 
@@ -139,16 +153,12 @@ int8_t TeensyFileManager::readDir(const char *where, const char *suffix, char *o
       }
     }
 
-    if (idxCount == startIdx) {
+    if (e.isDirectory()) {
-      if (e.isDirectory()) {
+      strcat(outputFN, "/");
-      	strcat(outputFN, "/");
-      }
-      e.close();
-      f.close();
-      return idxCount;
     }
-
+    e.close();
-    idxCount++;
+    
+    return startIdx;
   }
 
   /* NOTREACHED */

teensy/teensy-filemanager.h

@@ -16,8 +16,9 @@ class TeensyFileManager : public FileManager {
 
   virtual const char *fileName(int8_t fd);
 
-  virtual int8_t readDir(const char *where, const char *suffix, char *outputFN, int8_t startIdx, uint16_t maxlen);
+  virtual int16_t readDir(const char *where, const char *suffix, char *outputFN, int16_t startIdx, uint16_t maxlen);
-
+  virtual void closeDir();
+  
   virtual void getRootPath(char *toWhere, int8_t maxLen);
 
   virtual bool setSeekPosition(int8_t fd, uint32_t pos);

teensy/teensy-println.cpp

@@ -0,0 +1,15 @@
+#include "teensy-println.h"
+
+namespace arduino_preprocessor_is_buggy {
+  
+  bool serialavailable() {
+    Threads::Scope locker(getSerialLock());
+    return Serial.available();
+  }
+  
+  char serialgetch() {
+    Threads::Scope locker(getSerialLock());
+    return Serial.read();
+  }
+
+};

teensy/teensy-println.h

@@ -0,0 +1,48 @@
+#include <Arduino.h>
+#include <utility>
+#include <TeensyThreads.h>
+
+// cf. https://forum.pjrc.com/threads/41504-Teensy-3-x-multithreading-library-first-release/page6
+
+// println implementation - can be placed in a header file
+namespace arduino_preprocessor_is_buggy {
+    // helper for parameter pack expansion, 
+    // due to GCC bug 51253 we use an array
+    using expand = int[];
+    // used for suppressing compiler warnings
+    template<class T> void silence(T&&) {};
+
+    inline Threads::Mutex& getSerialLock() {
+        static Threads::Mutex serial_lock;
+        return serial_lock;
+    }
+
+  bool serialavailable();
+  char serialgetch();
+
+    template<class T> void print_fwd(const T& arg) {
+        Serial.print(arg);
+    }
+
+    template<class T1, class T2> void print_fwd(const std::pair<T1, T2>& arg) {
+        Serial.print(arg.first, arg.second);
+    }
+
+    template<class... args_t> void print(args_t... params) {
+        Threads::Scope locker(getSerialLock());
+        silence(expand{ (print_fwd(params), 42)... });
+    }
+  
+    template<class... args_t> void println(args_t... params) {
+        Threads::Scope locker(getSerialLock());
+        silence(expand{ (print_fwd(params), 42)... });
+        Serial.println();
+    }
+}
+
+using arduino_preprocessor_is_buggy::print;
+using arduino_preprocessor_is_buggy::println;
+using arduino_preprocessor_is_buggy::serialavailable;
+using arduino_preprocessor_is_buggy::serialgetch;
+using std::make_pair;
+// end println implementation