asset editor: scan presets to validate assets, computeRequiredWords() fix

presets/c64/joymove.c

@@ -1,7 +1,7 @@
 
 #include "common.h"
 
-/*{w:24,h:21,bpp:1,brev:1,count:1}*/
+/*{w:24,h:21,bpp:1,brev:1,wpimg:64,count:1}*/
 const char SPRITE_DATA[64] = {
  0x0f,0xff,0x80,0x17,0xff,0x40,0x2b,0xfe,
  0xa0,0x7f,0xff,0xf0,0xff,0xc0,0x3f,0xe0,

presets/nes/tileset1.c

@@ -8,7 +8,7 @@ const unsigned char palSprites[16]={/*{pal:"nes",n:4,sets:4}*/
 };
 
 const unsigned char TILESET[8*256] = {
-/*{w:8,h:8,bpp:1,count:256,brev:1,np:2,pofs:8,remap:[0,1,2,4,5,6,7,8,9,10,11,12]}*/
+/*{w:8,h:8,bpp:1,count:128,brev:1,np:2,pofs:8,remap:[0,1,2,4,5,6,7,8,9,10,11,12]}*/
 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x38,0x7c,0x7c,0x7c,0x38,0x00,0x38,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x6c,0x6c,0x48,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x6c,0xfe,0x6c,0xfe,0x6c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x10,0xfe,0xd0,0xfe,0x16,0xfe,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xce,0xdc,0x38,0x76,0xe6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x38,0x6c,0x7c,0xec,0xee,0x7e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x38,0x38,0x30,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x38,0x70,0x70,0x70,0x70,0x70,0x38,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x70,0x38,0x38,0x38,0x38,0x38,0x30,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x6c,0x38,0x6c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x38,0x38,0xfe,0x38,0x38,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x30,0x30,0x60,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x7c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x60,0x60,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x0e,0x1e,0x3c,0x78,0xf0,0xe0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 0x00,0x7c,0xee,0xee,0xee,0xee,0x7c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x38,0x78,0x38,0x38,0x38,0x7c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x7c,0x0e,0x7c,0xe0,0xee,0xfe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfc,0x0e,0x3c,0x0e,0x0e,0xfc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3e,0x7e,0xee,0xee,0xfe,0x0e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfc,0xe0,0xfc,0x0e,0xee,0x7c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x7c,0xe0,0xfc,0xee,0xee,0x7c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfe,0xee,0x1c,0x1c,0x38,0x38,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x7c,0xee,0x7c,0xee,0xee,0x7c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x7c,0xee,0xee,0x7e,0x0e,0x3c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x60,0x60,0x00,0x00,0x60,0x60,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x60,0x60,0x00,0x00,0x60,0x60,0xc0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x1c,0x38,0x70,0x70,0x38,0x1c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x7c,0x00,0x00,0x7c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x70,0x38,0x1c,0x1c,0x38,0x70,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x7c,0xee,0x1c,0x38,0x00,0x38,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
 0x00,0x7c,0xee,0xee,0xee,0xe0,0x7c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x7c,0xee,0xee,0xee,0xfe,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfc,0xee,0xfc,0xee,0xee,0xfc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x7c,0xee,0xe0,0xe0,0xee,0x7c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xf8,0xec,0xee,0xee,0xee,0xfc,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfe,0xe0,0xf0,0xe0,0xe0,0xfe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfe,0xe0,0xf8,0xe0,0xe0,0xe0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x7c,0xe0,0xee,0xee,0xee,0x7e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xee,0xee,0xfe,0xee,0xee,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x7c,0x38,0x38,0x38,0x38,0x7c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x0e,0x0e,0x0e,0x0e,0xee,0x7c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xee,0xfc,0xf8,0xec,0xee,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xe0,0xe0,0xe0,0xe0,0xee,0xfe,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xc6,0xee,0xfe,0xfe,0xee,0xee,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xce,0xee,0xfe,0xfe,0xee,0xe6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x7c,0xee,0xee,0xee,0xee,0x7c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,

presets/vcs/examples/retrigger.a

@@ -392,6 +392,7 @@ Frame0
         .byte #%01111110;$FA
         .byte #%00111110;$FA
         .byte #%01000001;$FA
+;;
 ;;{pal:"vcs"};;
 ColorFrame0
 	.byte #10	; height

src/ide/pixeleditor.ts

@@ -291,7 +291,9 @@ export function convertWordsToImages(words:UintArray, fmt:PixelEditorImageFormat
 }
 
 function computeRequiredWords(fmt): number {
-  // same variables as convertWordsToImages
+  // mirrors the addressing performed by convertWordsToImages: the data must cover
+  // every word the converter can read, and must also span the full per-image block
+  // extent (wpimg * count), e.g. hardware sprite strides larger than the visible rows.
   var count = fmt.count || 1;
   var bpp = fmt.bpp || 1;
   var nplanes = fmt.np || 1;
@@ -300,29 +302,29 @@ function computeRequiredWords(fmt): number {
   var pofs = fmt.pofs || wordsperline * fmt.h * count;
   var skip = fmt.skip || 0;
   var wpimg = fmt.wpimg || (fmt.map === 'nesnt' ? 1024 : wordsperline * fmt.h);
+  var rowstride = wordsperline;
+  if (fmt.il) { wpimg = wordsperline; rowstride = wordsperline * count; }
 
-  var maxOffset = 0;
-  for (var n = 0; n < count; n++) {
-    for (var i = 0; i < wpimg; i++) {
-      var offset0 = wpimg * n + i;
-      var offset;
-      if (fmt.reindex) {
-        var maxReindexOfs = 0;
-        for (var x = 0; x < fmt.w; x++) {
-          maxReindexOfs = Math.max(maxReindexOfs, reindexMask(x, fmt.reindex)[0]);
-        }
-        offset = offset0 + maxReindexOfs;
-      } else {
-        offset = remapBits(offset0, fmt.remap);
-      }
-      maxOffset = Math.max(maxOffset, offset);
+  var maxReindexOfs = 0;
+  if (fmt.reindex) {
+    for (var x = 0; x < fmt.w; x++) {
+      maxReindexOfs = Math.max(maxReindexOfs, reindexMask(x, fmt.reindex)[0]);
     }
   }
-  // Planar formats (e.g. NES pofs>=wpimg) store extra planes past each image block.
-  // Interleaved formats (GB/SMS pofs < wpimg) already include plane bytes in maxOffset.
-  var planeExtent = (nplanes > 1) ? (nplanes - 1) * pofs : 0;
-  if (planeExtent < wpimg) planeExtent = 0;
-  return maxOffset + planeExtent + 1 + skip;
+  var maxAddr = 0;
+  for (var n = 0; n < count; n++) {
+    for (var y = 0; y < fmt.h; y++) {
+      var yp = fmt.flip ? fmt.h - 1 - y : y;
+      var ofs0 = wpimg * n + yp * rowstride;
+      var steps = Math.ceil(fmt.w * bpp / bitsperword);
+      for (var s = 0; s < steps; s++) {
+        var ofs = fmt.reindex ? (ofs0 + maxReindexOfs) : remapBits(ofs0, fmt.remap);
+        maxAddr = Math.max(maxAddr, ofs + (nplanes - 1) * pofs + skip);
+        ofs0++;
+      }
+    }
+  }
+  return Math.max(wpimg * count, maxAddr + 1);
 }
 
 export function validateAssetData(datastr: string, fmt): string | null {

test/unit/testpresets.ts

@@ -0,0 +1,111 @@
+
+import assert from "assert";
+import { describe, it } from "mocha";
+import fs from "fs";
+import path from "path";
+import {
+  scanTextForAssetFragments, resolveEmbedPath,
+  validateAssetData, validateAssetByteLength, AssetFragment
+} from "../../src/ide/pixeleditor";
+
+// Scans every file under presets/ for asset editor blocks (/*{...}*/ and ;;{...};;)
+// and validates them the same way the AssetEditorView does:
+//  - headers must parse (valid lenient JSON, closed by ';', ';;' or 'end')
+//  - inline data blocks must contain exactly the number of words required by fmt
+//  - #embed'd binary files must exist and have exactly the required byte length
+
+const PRESETS_DIR = path.join(process.cwd(), 'presets');
+
+const TEXT_EXTENSIONS = new Set(['.c', '.h', '.a', '.s', '.asm', '.dasm', '.v', '.inc', '.cfg', '.mac']);
+
+function isVerilogFile(relpath: string): boolean {
+  return relpath.includes('verilog') || relpath.endsWith('.v');
+}
+
+function* walkFiles(dir: string): Generator<string> {
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const p = path.join(dir, entry.name);
+    if (entry.isDirectory()) yield* walkFiles(p);
+    else if (entry.isFile()) yield p;
+  }
+}
+
+function isTextPresetFile(relpath: string): boolean {
+  if (relpath.endsWith('~')) return false; // editor backups
+  const ext = path.extname(relpath);
+  return ext !== '' && TEXT_EXTENSIONS.has(ext);
+}
+
+type ScannedFragment = { relpath: string; frag: AssetFragment };
+
+function scanAllPresets(): ScannedFragment[] {
+  var result: ScannedFragment[] = [];
+  for (const filepath of walkFiles(PRESETS_DIR)) {
+    const relpath = path.relative(PRESETS_DIR, filepath).split(path.sep).join('/');
+    if (!isTextPresetFile(relpath)) continue;
+    let data: string;
+    try {
+      data = fs.readFileSync(filepath, 'utf8');
+    } catch (e) {
+      continue;
+    }
+    // skip binary files that slipped through the extension filter
+    if (/\0/.test(data.slice(0, 2000))) continue;
+    for (const frag of scanTextForAssetFragments(data, isVerilogFile(relpath))) {
+      result.push({ relpath, frag });
+    }
+  }
+  return result;
+}
+
+describe('Presets (asset editor blocks)', function () {
+
+  var scanned: ScannedFragment[];
+
+  before(function () {
+    assert.ok(fs.existsSync(PRESETS_DIR), `presets directory not found at ${PRESETS_DIR}`);
+    scanned = scanAllPresets();
+  });
+
+  it('should find a reasonable number of asset declarations', function () {
+    assert.ok(scanned.length > 100, `only found ${scanned.length} asset declarations in presets/`);
+  });
+
+  it('should have no parse errors in asset headers', function () {
+    var errors = scanned.filter(({ relpath, frag }) => frag.error)
+      .map(({ relpath, frag }) => `${relpath}:${frag.startline} ${frag.header} -> ${frag.error}`);
+    assert.deepEqual(errors, []);
+  });
+
+  it('should have valid data sizes for inline data blocks', function () {
+    var errors: string[] = [];
+    for (var { relpath, frag } of scanned) {
+      if (frag.error || !frag.fmt) continue;
+      if (frag.embedFile) continue; // checked below
+      if (!(frag.fmt.w > 0 && frag.fmt.h > 0) && !frag.fmt.pal) continue;
+      const data = fs.readFileSync(path.join(PRESETS_DIR, relpath), 'utf8');
+      const err = validateAssetData(data.substring(frag.start!, frag.end!), frag.fmt);
+      if (err) errors.push(`${relpath}:${frag.startline} ${frag.header} -> ${err}`);
+    }
+    assert.deepEqual(errors, []);
+  });
+
+  it('should resolve #embed files and match their byte length to the format', function () {
+    var errors: string[] = [];
+    // (presets may not use #embed yet, but validate any that do)
+    for (var { relpath, frag } of scanned) {
+      if (frag.error || !frag.fmt || !frag.embedFile) continue;
+      const resolved = resolveEmbedPath(relpath, frag.embedFile,
+        (p) => fs.existsSync(path.join(PRESETS_DIR, p)));
+      if (!resolved) {
+        errors.push(`${relpath}:${frag.startline} #embed file not found: "${frag.embedFile}"`);
+        continue;
+      }
+      const bytelen = fs.statSync(path.join(PRESETS_DIR, resolved)).size;
+      const err = validateAssetByteLength(bytelen, frag.fmt);
+      if (err) errors.push(`${relpath}:${frag.startline} ${frag.header} (${resolved}) -> ${err}`);
+    }
+    assert.deepEqual(errors, []);
+  });
+
+});