From 5c8e1de195d33112e0d379c4d5601f161ea76ac4 Mon Sep 17 00:00:00 2001 From: Jorj Bauer Date: Sat, 11 Jul 2020 07:39:56 -0400 Subject: [PATCH] better caching when reading directories --- Makefile | 3 +- apple/woz.cpp | 2 - bios.cpp | 138 ++++++++++++++++++++++++---------- bios.h | 6 +- filemanager.h | 3 +- nix/nix-filemanager.cpp | 67 +++++++++-------- nix/nix-filemanager.h | 3 +- sdl/fscompat.h | 18 +++++ sdl/sdl-display.h | 2 +- teensy/fscompat.h | 25 ++++++ teensy/teensy-filemanager.cpp | 48 +++++++----- teensy/teensy-filemanager.h | 5 +- teensy/teensy-println.cpp | 15 ++++ teensy/teensy-println.h | 48 ++++++++++++ 14 files changed, 282 insertions(+), 101 deletions(-) create mode 100644 sdl/fscompat.h create mode 100644 teensy/fscompat.h create mode 100644 teensy/teensy-println.cpp create mode 100644 teensy/teensy-println.h diff --git a/Makefile b/Makefile index 4b80375..dcf8e35 100755 --- a/Makefile +++ b/Makefile @@ -3,7 +3,8 @@ LDFLAGS=-L/usr/local/lib SDLLIBS=-lSDL2 -lpthread FBLIBS=-lpthread -CXXFLAGS=-Wall -I/usr/include/SDL2 -I .. -I . -I apple -I nix -I sdl -I/usr/local/include/SDL2 -g -O3 -DSUPPRESSREALTIME -DSTATICALLOC +CFLAGS=-Wall -I/usr/include/SDL2 -I .. -I . -I apple -I nix -I sdl -I/usr/local/include/SDL2 -g -DSUPPRESSREALTIME -DSTATICALLOC +CXXFLAGS=-Wall -I/usr/include/SDL2 -I .. -I . -I apple -I nix -I sdl -I/usr/local/include/SDL2 -g -DSUPPRESSREALTIME -DSTATICALLOC TSRC=cpu.cpp util/testharness.cpp diff --git a/apple/woz.cpp b/apple/woz.cpp index 479e075..9fa62bf 100644 --- a/apple/woz.cpp +++ b/apple/woz.cpp @@ -3,9 +3,7 @@ #include "crc32.h" #include "nibutil.h" #include "version.h" -#ifdef TEENSYDUINO #include "fscompat.h" -#endif extern uint32_t FreeRamEstimate(); diff --git a/bios.cpp b/bios.cpp index 07a9e32..95cd08e 100644 --- a/bios.cpp +++ b/bios.cpp @@ -13,6 +13,21 @@ extern Bounce resetButtonDebouncer; #endif +// Experimenting with using EXTMEM to cache all the filenames in a directory +#ifndef TEENSYDUINO +#define EXTMEM +#endif + +struct _cacheEntry { + char fn[BIOS_MAXPATH]; +}; +#define BIOSCACHESIZE 1024 // hope that's enough files? +EXTMEM char cachedPath[BIOS_MAXPATH] = {0}; +EXTMEM char cachedFilter[BIOS_MAXPATH] = {0}; +EXTMEM struct _cacheEntry biosCache[BIOSCACHESIZE]; +uint16_t numCacheEntries = 0; + + enum { ACT_EXIT = 1, ACT_RESET = 2, @@ -92,12 +107,12 @@ void BIOS::DrawMenuBar() for (int i=0; idrawUIPixel(xpos+x, 0, 0xFFFF); - g_display->drawUIPixel(xpos+x, 16, 0xFFFF); + g_display->drawPixel(xpos+x, 0, 0xFFFF); + g_display->drawPixel(xpos+x, 16, 0xFFFF); } for (int y=0; y<=16; y++) { - g_display->drawUIPixel(xpos, y, 0xFFFF); - g_display->drawUIPixel(xpos + titleWidths[i] + 2*XPADDING, y, 0xFFFF); + g_display->drawPixel(xpos, y, 0xFFFF); + g_display->drawPixel(xpos + titleWidths[i] + 2*XPADDING, y, 0xFFFF); } xpos += XPADDING; @@ -111,6 +126,10 @@ void BIOS::DrawMenuBar() bool BIOS::runUntilDone() { + // Reset the cache + cachedPath[0] = 0; + numCacheEntries = 0; + g_filemanager->getRootPath(rootPath, sizeof(rootPath)); // FIXME: abstract these constant speeds @@ -566,7 +585,7 @@ void BIOS::DrawHardwareMenu() uint16_t volCutoff = 300.0 * (float)((float) g_volume / 15.0); for (uint8_t y=234; y<=235; y++) { for (uint16_t x = 0; x< 300; x++) { - g_display->drawUIPixel( x, y, x <= volCutoff ? 0xFFFF : 0x0010 ); + g_display->drawPixel( x, y, x <= volCutoff ? 0xFFFF : 0x0010 ); } } } @@ -704,9 +723,9 @@ bool BIOS::SelectDiskImage(const char *filter) { int8_t sel = 0; int8_t page = 0; - + uint16_t fileCount = 0; while (1) { - DrawDiskNames(page, sel, filter); + fileCount = DrawDiskNames(page, sel, filter); while (!g_keyboard->kbhit()) ; @@ -729,31 +748,31 @@ bool BIOS::SelectDiskImage(const char *filter) // else sel = BIOS_MAXFILES + 1; } else if (sel == BIOS_MAXFILES+1) { - page++; - //sel = 0; - } else { - if (strcmp(fileDirectory[sel-1], "../") == 0) { - // Go up a directory (strip a directory name from rootPath) - stripDirectory(); - page = 0; + if (fileCount == BIOS_MAXFILES) { // don't let them select 'Next' if there were no files in the list or if the list isn't full + page++; //sel = 0; - continue; - } else if (fileDirectory[sel-1][strlen(fileDirectory[sel-1])-1] == '/') { - // Descend in to the directory. FIXME: file path length? - strcat(rootPath, fileDirectory[sel-1]); - sel = 0; - page = 0; - continue; - } else { - selectedFile = sel - 1; - g_display->flush(); - return true; } + } else if (strcmp(fileDirectory[sel-1], "../") == 0) { + // Go up a directory (strip a directory name from rootPath) + stripDirectory(); + page = 0; + //sel = 0; + continue; + } else if (fileDirectory[sel-1][strlen(fileDirectory[sel-1])-1] == '/') { + // Descend in to the directory. FIXME: file path length? + strcat(rootPath, fileDirectory[sel-1]); + sel = 0; + page = 0; + continue; + } else { + selectedFile = sel - 1; + g_display->flush(); + return true; } break; } } - g_display->flush(); + /* NOTREACHED */ } void BIOS::stripDirectory() @@ -772,9 +791,9 @@ void BIOS::stripDirectory() } } -void BIOS::DrawDiskNames(uint8_t page, int8_t selection, const char *filter) +uint16_t BIOS::DrawDiskNames(uint8_t page, int8_t selection, const char *filter) { - uint8_t fileCount = GatherFilenames(page, filter); + uint16_t fileCount = GatherFilenames(page, filter); g_display->clrScr(); g_display->drawString(M_NORMAL, 0, 12, "BIOS Configuration - pick disk"); @@ -795,37 +814,78 @@ void BIOS::DrawDiskNames(uint8_t page, int8_t selection, const char *filter) } // FIXME: this doesn't accurately say whether or not there *are* more. - if (fileCount == BIOS_MAXFILES || fileCount == 0) { + if (fileCount < BIOS_MAXFILES) { g_display->drawString((i+1 == selection) ? M_SELECTDISABLED : M_DISABLED, 10, 50 + 14 * (i+1), ""); } else { g_display->drawString(i+1 == selection ? M_SELECTED : M_NORMAL, 10, 50 + 14 * (i+1), ""); } g_display->flush(); + return fileCount; } +uint16_t BIOS::cacheAllEntries(const char *filter) +{ + // If we've already cached this directory, then just return it + if (numCacheEntries && !strcmp(cachedPath, rootPath) && !strcmp(cachedFilter, filter)) + return numCacheEntries; -uint8_t BIOS::GatherFilenames(uint8_t pageOffset, const char *filter) + // Otherwise flush the cache and start over + numCacheEntries = 0; + strcpy(cachedPath, rootPath); + strcpy(cachedFilter, filter); + + // This could be a lengthy process, so... + g_display->clrScr(); + g_display->drawString(M_SELECTED, + 0, + 0, + "Loading..."); + g_display->flush(); + + // read all the entries we can find + int16_t idx = 0; + while (1) { + struct _cacheEntry *ce = &biosCache[numCacheEntries]; + idx = g_filemanager->readDir(rootPath, filter, ce->fn, idx, BIOS_MAXPATH); + if (idx == -1) { + return numCacheEntries; + } + idx++; + numCacheEntries++; + if (numCacheEntries >= BIOSCACHESIZE-1) { + return numCacheEntries; + } + } + /* NOTREACHED */ +} + +uint16_t BIOS::GatherFilenames(uint8_t pageOffset, const char *filter) { uint8_t startNum = 10 * pageOffset; uint8_t count = 0; // number we're including in our listing + uint16_t numEntriesTotal = cacheAllEntries(filter); + if (numEntriesTotal > BIOSCACHESIZE) { + // ... umm, this is a problem. FIXME? + } + struct _cacheEntry *nextEntry = biosCache; + while (startNum) { + nextEntry++; + startNum--; + } + while (1) { - char fn[BIOS_MAXPATH]; - int8_t idx = g_filemanager->readDir(rootPath, filter, fn, startNum + count, BIOS_MAXPATH); - - if (idx == -1) { + if (nextEntry->fn[0] == 0) return count; - } - idx++; - - strncpy(fileDirectory[count], fn, BIOS_MAXPATH); + strncpy(fileDirectory[count], nextEntry->fn, BIOS_MAXPATH); count++; if (count >= BIOS_MAXFILES) { return count; } + nextEntry++; } } @@ -840,7 +900,7 @@ void BIOS::showAbout() g_display->drawString(M_NORMAL, 15, 20, - "(c) 2017 Jorj Bauer"); + "(c) 2017-2020 Jorj Bauer"); g_display->drawString(M_NORMAL, 15, 38, diff --git a/bios.h b/bios.h index 2d55759..5109ba5 100644 --- a/bios.h +++ b/bios.h @@ -36,8 +36,8 @@ class BIOS { void ColdReboot(); bool SelectDiskImage(const char *filter); - void DrawDiskNames(uint8_t page, int8_t selection, const char *filter); - uint8_t GatherFilenames(uint8_t pageOffset, const char *filter); + uint16_t DrawDiskNames(uint8_t page, int8_t selection, const char *filter); + uint16_t GatherFilenames(uint8_t pageOffset, const char *filter); void ConfigurePaddles(); @@ -45,6 +45,8 @@ class BIOS { void showAbout(); + uint16_t cacheAllEntries(const char *filter); + private: int8_t selectedFile; char fileDirectory[BIOS_MAXFILES][BIOS_MAXPATH+1]; diff --git a/filemanager.h b/filemanager.h index 3ad9caa..ed1bd75 100644 --- a/filemanager.h +++ b/filemanager.h @@ -126,7 +126,8 @@ class FileManager { virtual const char *fileName(int8_t fd) = 0; - virtual int8_t readDir(const char *where, const char *suffix, char *outputFN, int8_t startIdx, uint16_t maxlen) = 0; + virtual int16_t readDir(const char *where, const char *suffix, char *outputFN, int16_t startIdx, uint16_t maxlen) = 0; + virtual void closeDir() = 0; virtual void getRootPath(char *toWhere, int8_t maxLen) = 0; diff --git a/nix/nix-filemanager.cpp b/nix/nix-filemanager.cpp index 7d55603..83f2489 100644 --- a/nix/nix-filemanager.cpp +++ b/nix/nix-filemanager.cpp @@ -65,27 +65,38 @@ const char *NixFileManager::fileName(int8_t fd) return cachedNames[fd]; } -int8_t NixFileManager::readDir(const char *where, const char *suffix, char *outputFN, int8_t startIdx, uint16_t maxlen) +// FIXME make these member vars instead of globals +static DIR *dirp = NULL; + +void NixFileManager::closeDir() { - int idx = 1; - if (strcmp(where, ROOTDIR)) { - // First entry is always "../" - if (startIdx == 0) { + if (dirp) { + closedir(dirp); + dirp = NULL; + } +} + +int16_t NixFileManager::readDir(const char *where, const char *suffix, char *outputFN, int16_t startIdx, uint16_t maxlen) +{ + if (startIdx == 0 || !dirp) { + // This is an openDir() -- so reset state, open the directory, etc. + closeDir(); + dirp = opendir(where); + if (!dirp) + return -1; + } + + if (startIdx == 0) { + if (strcmp(where, ROOTDIR)) { + // As long as we're not at the root, we start with "../" strcpy(outputFN, "../"); return 0; } - } else { - idx = 0; // we skipped ROOTDIR } - DIR *dirp = opendir(where); - if (!dirp) - return -1; - - struct dirent *dp; - outputFN[0] = '\0'; + struct dirent *dp; while ((dp = readdir(dirp)) != NULL) { if (dp->d_name[0] == '.') { // Skip any dot files (and dot directories) @@ -128,30 +139,20 @@ int8_t NixFileManager::readDir(const char *where, const char *suffix, char *outp } } // If we get here, it's something we want to show. - if (idx == startIdx) { - // Fill in the reply - strncpy(outputFN, dp->d_name, maxlen-1); - - if (dp->d_type & DT_DIR) { - // suffix - strcat(outputFN, "/"); - } - break; + strncpy(outputFN, dp->d_name, maxlen-1); + + if (dp->d_type & DT_DIR) { + // suffix + strcat(outputFN, "/"); } - - // Next! - idx++; + + return startIdx; } // Exited the loop - all done. - closedir(dirp); - - if (!outputFN[0]) { - // didn't find any more - return -1; - } - - return idx; + // didn't find any more + closeDir(); + return -1; } void NixFileManager::getRootPath(char *toWhere, int8_t maxLen) diff --git a/nix/nix-filemanager.h b/nix/nix-filemanager.h index 6cd8ed7..6300d60 100644 --- a/nix/nix-filemanager.h +++ b/nix/nix-filemanager.h @@ -14,7 +14,8 @@ class NixFileManager : public FileManager { virtual const char *fileName(int8_t fd); - virtual int8_t readDir(const char *where, const char *suffix, char *outputFN, int8_t startIdx, uint16_t maxlen); + virtual int16_t readDir(const char *where, const char *suffix, char *outputFN, int16_t startIdx, uint16_t maxlen); + virtual void closeDir(); void getRootPath(char *toWhere, int8_t maxLen); diff --git a/sdl/fscompat.h b/sdl/fscompat.h new file mode 100644 index 0000000..d94adfd --- /dev/null +++ b/sdl/fscompat.h @@ -0,0 +1,18 @@ +/* Filesystem compatability layer. + * + * Right now I've got a hokey (but well abstracted) g_filemanager object + * that has a bunch of file operation primitives which don't conform to + * POSIX at all. The woz code comes from my Wozzle utility, and I'd rather + * not butcher the heck out of it for the sake of Aiie... so this is + * bridging that gap, for now. + */ + +#include "globals.h" + +#define open(path, flags, perms) g_filemanager->openFile(path) +#define close(filedes) g_filemanager->closeFile(filedes) +#define write(filedes,buf,nbyte) g_filemanager->write(filedes,buf,nbyte) +#define read(filedes,buf,nbyte) g_filemanager->read(filedes,buf,nbyte) +#define lseek(filedes,offset,whence) g_filemanager->lseek(filedes,offset,whence) + + diff --git a/sdl/sdl-display.h b/sdl/sdl-display.h index 475c6db..8d3439c 100644 --- a/sdl/sdl-display.h +++ b/sdl/sdl-display.h @@ -10,7 +10,7 @@ // scale can be 1,2,4. '1' is half-width at the highest resolution // (80-col mode). '2' is full width. '4' is double full width. -#define SDLDISPLAY_SCALE 1 +#define SDLDISPLAY_SCALE 2 #define SDLDISPLAY_WIDTH (320*SDLDISPLAY_SCALE) #define SDLDISPLAY_HEIGHT (240*SDLDISPLAY_SCALE) diff --git a/teensy/fscompat.h b/teensy/fscompat.h new file mode 100644 index 0000000..9b944f5 --- /dev/null +++ b/teensy/fscompat.h @@ -0,0 +1,25 @@ +/* Filesystem compatability layer. + * + * Right now I've got a hokey (but well abstracted) g_filemanager object + * that has a bunch of file operation primitives which don't conform to + * POSIX at all. The woz code comes from my Wozzle utility, and I'd rather + * not butcher the heck out of it for the sake of Aiie... so this is + * bridging that gap, for now. + */ + +#include "globals.h" +#include +#include + +#define open(path, flags, perms) g_filemanager->openFile(path) +#define close(filedes) g_filemanager->closeFile(filedes) +#define write(filedes,buf,nbyte) g_filemanager->write(filedes,buf,nbyte) +#define read(filedes,buf,nbyte) g_filemanager->read(filedes,buf,nbyte) +#define lseek(filedes,offset,whence) g_filemanager->lseek(filedes,offset,whence) + +Threads::Mutex serlock; +static char fsbuf[200]; +#define printf(x, ...) {sprintf(fsbuf, x, ##__VA_ARGS__); serlock.lock(); Serial.println(fsbuf); Serial.flush(); Serial.send_now(); serlock.unlock();} +#define fprintf(f, x, ...) {sprintf(fsbuf, x, ##__VA_ARGS__); serlock.lock(); Serial.println(fsbuf); Serial.flush(); Serial.send_now(); serlock.lock();} +#define perror(x) {serlock.lock();Serial.println(x);Serial.flush(); Serial.send_now(); serlock.unlock();} + diff --git a/teensy/teensy-filemanager.cpp b/teensy/teensy-filemanager.cpp index 6c9cae4..8871bed 100644 --- a/teensy/teensy-filemanager.cpp +++ b/teensy/teensy-filemanager.cpp @@ -78,28 +78,42 @@ const char *TeensyFileManager::fileName(int8_t fd) return cachedNames[fd]; } +File outerDir; + +void TeensyFileManager::closeDir() +{ + // FIXME: this should Threads::Scope lock, but it's being called + // from readDir, so that would block + if (outerDir) { + outerDir.close(); + } +} + // suffix may be comma-separated -int8_t TeensyFileManager::readDir(const char *where, const char *suffix, char *outputFN, int8_t startIdx, uint16_t maxlen) +int16_t TeensyFileManager::readDir(const char *where, const char *suffix, char *outputFN, int16_t startIdx, uint16_t maxlen) { Threads::Scope locker(fslock); - // ... open, read, save next name, close, return name. Horribly - // inefficient but hopefully won't break the sd layer. And if it - // works then we can make this more efficient later. - // First entry is always "../" - if (startIdx == 0) { + // First entry is always "../" if we're in a subdir of the root + if (startIdx == 0 || !outerDir) { + if (outerDir) + closeDir(); + outerDir = SD.open(where, FILE_READ); + if (!outerDir) + return -1; + if (strcmp(where, "/")) { // FIXME: is this correct for the root? strcpy(outputFN, "../"); return 0; + } } - int8_t idxCount = 1; - File f = SD.open(where, FILE_READ); + outputFN[0] = '\0'; while (1) { - File e = f.openNextFile(); + File e = outerDir.openNextFile(); if (!e) { // No more - all done. - f.close(); + closeDir(); return -1; } @@ -139,16 +153,12 @@ int8_t TeensyFileManager::readDir(const char *where, const char *suffix, char *o } } - if (idxCount == startIdx) { - if (e.isDirectory()) { - strcat(outputFN, "/"); - } - e.close(); - f.close(); - return idxCount; + if (e.isDirectory()) { + strcat(outputFN, "/"); } - - idxCount++; + e.close(); + + return startIdx; } /* NOTREACHED */ diff --git a/teensy/teensy-filemanager.h b/teensy/teensy-filemanager.h index 2f580af..17558e6 100644 --- a/teensy/teensy-filemanager.h +++ b/teensy/teensy-filemanager.h @@ -16,8 +16,9 @@ class TeensyFileManager : public FileManager { virtual const char *fileName(int8_t fd); - virtual int8_t readDir(const char *where, const char *suffix, char *outputFN, int8_t startIdx, uint16_t maxlen); - + virtual int16_t readDir(const char *where, const char *suffix, char *outputFN, int16_t startIdx, uint16_t maxlen); + virtual void closeDir(); + virtual void getRootPath(char *toWhere, int8_t maxLen); virtual bool setSeekPosition(int8_t fd, uint32_t pos); diff --git a/teensy/teensy-println.cpp b/teensy/teensy-println.cpp new file mode 100644 index 0000000..a073ce4 --- /dev/null +++ b/teensy/teensy-println.cpp @@ -0,0 +1,15 @@ +#include "teensy-println.h" + +namespace arduino_preprocessor_is_buggy { + + bool serialavailable() { + Threads::Scope locker(getSerialLock()); + return Serial.available(); + } + + char serialgetch() { + Threads::Scope locker(getSerialLock()); + return Serial.read(); + } + +}; diff --git a/teensy/teensy-println.h b/teensy/teensy-println.h new file mode 100644 index 0000000..65c5174 --- /dev/null +++ b/teensy/teensy-println.h @@ -0,0 +1,48 @@ +#include +#include +#include + +// cf. https://forum.pjrc.com/threads/41504-Teensy-3-x-multithreading-library-first-release/page6 + +// println implementation - can be placed in a header file +namespace arduino_preprocessor_is_buggy { + // helper for parameter pack expansion, + // due to GCC bug 51253 we use an array + using expand = int[]; + // used for suppressing compiler warnings + template void silence(T&&) {}; + + inline Threads::Mutex& getSerialLock() { + static Threads::Mutex serial_lock; + return serial_lock; + } + + bool serialavailable(); + char serialgetch(); + + template void print_fwd(const T& arg) { + Serial.print(arg); + } + + template void print_fwd(const std::pair& arg) { + Serial.print(arg.first, arg.second); + } + + template void print(args_t... params) { + Threads::Scope locker(getSerialLock()); + silence(expand{ (print_fwd(params), 42)... }); + } + + template void println(args_t... params) { + Threads::Scope locker(getSerialLock()); + silence(expand{ (print_fwd(params), 42)... }); + Serial.println(); + } +} + +using arduino_preprocessor_is_buggy::print; +using arduino_preprocessor_is_buggy::println; +using arduino_preprocessor_is_buggy::serialavailable; +using arduino_preprocessor_is_buggy::serialgetch; +using std::make_pair; +// end println implementation