test/cli/testdeterminism.js finding bugs in platforms

src/common/history.ts

@@ -110,6 +110,11 @@ export class History<S = any> {
     this.onChange?.();
   }
 
+  /** Round a target to what the core can actually stop at. */
+  clamp(t: Timestamp): Timestamp {
+    return this.core.clamp ? this.core.clamp(t) : t;
+  }
+
   /** Oldest point still reachable. Earlier checkpoints have been dropped. */
   first(): Timestamp { return this.checkpoints[0].at; }
   /** The present: the furthest point recording has reached. */
@@ -162,6 +167,7 @@ export class History<S = any> {
    * costs the same as seeking forwards -- that's the whole trick.
    */
   seek(t: Timestamp, trap?: TrapCondition | null): RunResult {
+    t = this.clamp(t);
     if (compareTimestamps(t, this.head) > 0) t = this.head;
     const cp = this.checkpointAtOrBefore(t);
     if (!cp) {
@@ -181,6 +187,7 @@ export class History<S = any> {
    * hook lands in the right place; the core handles everything within a frame.
    */
   private runTo(t: Timestamp, trap?: TrapCondition | null): RunResult {
+    t = this.clamp(t);
     while (compareTimestamps(this.core.now(), t) < 0) {
       const at = this.core.now();
       if (at.step === 0) this.input?.replay(at.frame);
@@ -188,6 +195,8 @@ export class History<S = any> {
       const stop = compareTimestamps(frameEnd, t) < 0 ? frameEnd : t;
       const r = this.core.runUntil(stop, trap);
       if (r.trapped) return r;
+      // a core that reports no progress can't reach t; stop rather than spin
+      if (compareTimestamps(r.at, at) <= 0) break;
     }
     return { at: this.core.now(), trapped: false };
   }
@@ -197,8 +206,8 @@ export class History<S = any> {
    * the hit, or at `to` if there wasn't one.
    */
   findNext(cond: SearchCondition, from?: Timestamp, to?: Timestamp): Timestamp | null {
-    from = from ?? this.core.now();
-    to = to ?? this.head;
+    from = this.clamp(from ?? this.core.now());
+    to = this.clamp(to ?? this.head);
     if (compareTimestamps(from, to) >= 0) return null;
     return this.withProbe(cond, () => {
       this.seek(from);
@@ -232,8 +241,8 @@ export class History<S = any> {
    * far back the answer is. Leaves the core parked on the hit.
    */
   findLast(cond: SearchCondition, from?: Timestamp, to?: Timestamp): Timestamp | null {
-    from = from ?? this.first();
-    to = to ?? this.core.now();
+    from = this.clamp(from ?? this.first());
+    to = this.clamp(to ?? this.core.now());
     if (compareTimestamps(from, to) >= 0) return null;
     return this.withProbe(cond, () => this.findLastUnprobed(cond, from, to));
   }

src/common/platformcore.ts

@@ -0,0 +1,176 @@
+// Bridges the existing Platform objects to the deterministic timeline.
+//
+// Most platforms are built on a Machine and get MachineCore, which can stop
+// between instructions. The ones that aren't -- vcs (Javatari), verilog, x86 --
+// only know how to run a whole frame, so they get FramePlatformCore, which says
+// so through its granularity instead of pretending to sub-frame accuracy.
+
+import { Platform } from "./baseplatform";
+import { getNoiseSeed, setNoiseSeed } from "./emu";
+import { FrameInputSource } from "./history";
+import {
+  DeterministicCore,
+  formatTimestamp,
+  Granularity,
+  MachineCore,
+  RunResult,
+  Timestamp,
+  timestamp,
+} from "./timeline";
+import { TrapCondition } from "./devices";
+
+export interface RecordedKey {
+  key: number;
+  code: number;
+  flags: number;
+}
+
+export interface PlatformInputOptions {
+  /** Which frame an incoming key event belongs to. */
+  currentFrame?: () => number;
+  /** How to re-deliver a recorded key event during a replay. */
+  dispatchKey?: (key: number, code: number, flags: number) => void;
+}
+
+/**
+ * Captures and replays what varies between otherwise identical frames: the
+ * controls, the seed of the shared PRNG, and the key events themselves.
+ *
+ * Snapshotting the controls is not enough on its own. Some platforms *act* on a
+ * key event rather than only latching it -- atari8 raises a POKEY interrupt on
+ * key-down (atari8.ts, getKeyboardFunction) -- and that effect lands in POKEY's
+ * registers, which saveControlsState() does not cover. A replay that only
+ * restored controls would silently lose the interrupt.
+ *
+ * So when a dispatcher is supplied, the recorded events are the source of truth
+ * and the controls snapshot is not restored at all. The two must not be
+ * combined: a checkpoint already contains the effect of every earlier event, so
+ * redelivering on top of a restore applies them twice -- on c64 the key goes
+ * into a queue inside the WASM state, and the doubled press diverges
+ * immediately. Replaying just the events that came after the checkpoint is both
+ * necessary and sufficient. Without a dispatcher there is nothing to replay, so
+ * the controls snapshot is used instead.
+ *
+ * Frames are a fine granularity for this because the browser cannot deliver a
+ * key event in the middle of advanceFrame() -- JS is single-threaded, so events
+ * queue until the frame returns.
+ */
+export class PlatformFrameInput implements FrameInputSource {
+  private frames = new Map<number, { controls: any, seed: number }>();
+  private keys = new Map<number, RecordedKey[]>();
+  private oldest = 0;
+
+  constructor(readonly platform: Platform, readonly opts: PlatformInputOptions = {}) { }
+
+  /** Note a key event as it is delivered, so the replay can redeliver it. */
+  recordKey(key: number, code: number, flags: number): void {
+    const frame = this.opts.currentFrame ? this.opts.currentFrame() : 0;
+    var evs = this.keys.get(frame);
+    if (!evs) this.keys.set(frame, evs = []);
+    evs.push({ key, code, flags });
+  }
+
+  capture(frame: number): void {
+    this.frames.set(frame, {
+      controls: this.platform.saveControlsState ? this.platform.saveControlsState() : undefined,
+      seed: getNoiseSeed(),
+    });
+  }
+
+  replay(frame: number): void {
+    const rec = this.frames.get(frame);
+    if (rec) {
+      if (!this.opts.dispatchKey && rec.controls !== undefined && this.platform.loadControlsState) {
+        this.platform.loadControlsState(rec.controls);
+      }
+      setNoiseSeed(rec.seed);
+    }
+    const evs = this.opts.dispatchKey && this.keys.get(frame);
+    if (evs) {
+      for (const e of evs) this.opts.dispatchKey(e.key, e.code, e.flags);
+    }
+  }
+
+  trim(firstFrame: number): void {
+    while (this.oldest < firstFrame) {
+      this.frames.delete(this.oldest);
+      this.keys.delete(this.oldest);
+      this.oldest++;
+    }
+  }
+
+  get size(): number { return this.frames.size; }
+  get keyEventCount(): number {
+    var n = 0;
+    this.keys.forEach((v) => n += v.length);
+    return n;
+  }
+}
+
+/**
+ * A core for platforms with no Machine behind them. Frames are the only thing
+ * it can count, so every target is rounded down to a frame boundary and
+ * clamp() tells the history layer as much -- without that, a caller asking for
+ * a sub-frame position would spin waiting for progress that can never happen.
+ */
+export class FramePlatformCore implements DeterministicCore {
+
+  readonly granularity: Granularity = 'frame';
+  private frame: number = 0;
+
+  constructor(readonly platform: Platform) { }
+
+  now(): Timestamp { return timestamp(this.frame, 0); }
+
+  clamp(t: Timestamp): Timestamp {
+    return t.step === 0 ? t : timestamp(t.frame, 0);
+  }
+
+  snapshot() { return this.platform.saveState(); }
+
+  restore(state: any, at: Timestamp): void {
+    if (at.step !== 0) {
+      throw new Error(`can only restore at a frame boundary, not ${formatTimestamp(at)}`);
+    }
+    this.platform.loadState(state);
+    this.frame = at.frame;
+  }
+
+  runUntil(target: Timestamp, trap?: TrapCondition | null): RunResult {
+    while (this.frame < target.frame) {
+      if (trap && trap()) return { at: this.now(), trapped: true };
+      this.advanceFrame();
+      this.frame++;
+    }
+    return { at: this.now(), trapped: false };
+  }
+
+  private advanceFrame(): void {
+    const p = this.platform as any;
+    // advance() is the bare frame; nextFrame() would also poll controls and
+    // drive the old recorder, which would fight with this one
+    if (p.advance) p.advance(true);
+    else if (p.nextFrame) p.nextFrame(true);
+    else throw new Error('platform cannot advance a frame');
+  }
+}
+
+/** True if this platform can be rewound at all. */
+export function isRewindable(platform: Platform): boolean {
+  return !!(platform && platform.saveState && platform.loadState &&
+    ((platform as any).machine || (platform as any).advance));
+}
+
+/**
+ * Pick the most precise core the platform supports. WASM machines trap per
+ * clock tick, JS machines per instruction; both are exposed as "steps" but the
+ * granularity is reported so the UI can label the sub-frame axis honestly.
+ */
+export function createCore(platform: Platform): DeterministicCore {
+  const machine = (platform as any).machine;
+  if (machine && typeof machine.advanceFrame === 'function' && typeof machine.saveState === 'function') {
+    const granularity: Granularity = typeof machine.advanceFrameClock === 'function' ? 'clock' : 'insn';
+    return new MachineCore(machine, granularity);
+  }
+  return new FramePlatformCore(platform);
+}

src/common/statehash.ts

@@ -0,0 +1,69 @@
+// A stable hash over an emulator state, for detecting divergence between a
+// live run and a replay of it. Save states are a grab bag of plain objects,
+// typed arrays, numbers and (on vcs) base64 strings, so the walk has to cope
+// with all of them and, above all, be order-independent for object keys --
+// two states that differ only in key insertion order are the same state.
+
+function fnv(h: number, byte: number): number {
+  return Math.imul(h ^ (byte & 0xff), 0x01000193) >>> 0;
+}
+
+const scratch = new DataView(new ArrayBuffer(8));
+
+function hashNumber(h: number, v: number): number {
+  scratch.setFloat64(0, v);
+  for (var i = 0; i < 8; i++) h = fnv(h, scratch.getUint8(i));
+  return h;
+}
+
+function hashString(h: number, s: string): number {
+  for (var i = 0; i < s.length; i++) {
+    h = fnv(h, s.charCodeAt(i));
+    h = fnv(h, s.charCodeAt(i) >> 8);
+  }
+  return h;
+}
+
+function hashValue(h: number, v: any, path: Set<object>): number {
+  if (v == null) return fnv(h, v === null ? 1 : 2);
+  switch (typeof v) {
+    case 'number': return hashNumber(fnv(h, 3), v);
+    case 'boolean': return fnv(fnv(h, 4), v ? 1 : 0);
+    case 'string': return hashString(fnv(h, 5), v);
+    case 'function': return fnv(h, 6);           // not state
+  }
+  if (ArrayBuffer.isView(v)) {
+    const bytes = new Uint8Array((v as any).buffer, (v as any).byteOffset, (v as any).byteLength);
+    h = fnv(h, 7);
+    h = hashNumber(h, bytes.length);
+    for (var i = 0; i < bytes.length; i++) h = fnv(h, bytes[i]);
+    return h;
+  }
+  // Emulator states are graphs, not trees -- jsnes's mapper points back at the
+  // machine. Only cycles are collapsed, by tracking the current path; a shared
+  // subtree reachable twice is hashed twice. Recording object *identity*
+  // instead would make two states with identical contents hash differently
+  // just because loadState() rebuilt an array that used to be shared, which is
+  // divergence that isn't there. A depth cutoff is worse still: where it bites
+  // depends on where the walk started.
+  if (path.has(v)) return fnv(h, 10);
+  path.add(v);
+  if (Array.isArray(v)) {
+    h = fnv(h, 8);
+    h = hashNumber(h, v.length);
+    for (var i = 0; i < v.length; i++) h = hashValue(h, v[i], path);
+  } else {
+    h = fnv(h, 9);
+    for (const k of Object.keys(v).sort()) {
+      h = hashString(h, k);
+      h = hashValue(h, v[k], path);
+    }
+  }
+  path.delete(v);
+  return h;
+}
+
+/** FNV-1a over the whole state graph. Equal hashes mean equal states. */
+export function hashState(state: any): number {
+  return hashValue(0x811c9dc5, state, new Set());
+}

src/common/timeline.ts

@@ -87,6 +87,13 @@ export interface DeterministicCore<S = any> {
    */
   runUntil(target: Timestamp, trap?: TrapCondition | null): RunResult;
 
+  /**
+   * Round a target to something this core can actually stop at. A frame-only
+   * core rounds sub-frame targets down; without this a caller could ask for a
+   * position the core can never reach and wait forever for progress.
+   */
+  clamp?(t: Timestamp): Timestamp;
+
   /** Connect a probe, if this core supports probing. */
   connectProbe?(probe: ProbeAll): void;
 }

src/tools/8bws.ts

@@ -13,6 +13,11 @@
 import * as fs from 'fs';
 import * as os from 'os';
 import * as path from 'path';
+import { KeyFlags } from '../common/emu';
+import { History } from '../common/history';
+import { createCore, isRewindable, PlatformFrameInput } from '../common/platformcore';
+import { hashState } from '../common/statehash';
+import { formatTimestamp, timestamp } from '../common/timeline';
 import { isProbablyBinary } from '../common/util';
 import { fail, hasOutput, note, output, setJsonMode } from './cliformat';
 import { EmuTarget, loadPlatform } from './emutarget';
@@ -41,12 +46,14 @@ const SHORT_FLAGS: { [short: string]: string } = {
 const BOOLEAN_FLAGS: { [command: string]: string[] } = {
   build: ['check', 'symbols', 'save'],
   run: ['info'],
+  'verify-replay': ['verbose'],
 };
 
 const ALIASES: { [alias: string]: string } = {
   compile: 'build',
   check: 'build',
   compilerun: 'run',
+  verify: 'verify-replay',
 };
 
 function parseArgs(argv: string[]): { command: string; args: Args; positional: string[] } {
@@ -249,6 +256,150 @@ async function writeScreenshot(video: ReturnType<EmuTarget['getVideo']>, pngFile
   }
 }
 
+////////////////////////////////////////////////////////////////////////
+// determinism
+
+/**
+ * Keys worth mashing: the directions and the fire/start buttons, which is what
+ * a joystick-driven platform reads. Pressing arbitrary keyboard keys would be
+ * noisier without testing anything more.
+ */
+const EXERCISE_KEYS = [37, 38, 39, 40, 32, 13];   // left up right down space enter
+
+/**
+ * A seeded button masher. Verification is only meaningful if the recording has
+ * input in it -- without input the controls path is never exercised, and a
+ * platform whose loadControlsState() is not the inverse of saveControlsState()
+ * passes for the wrong reason. Seeded so a failure is reproducible.
+ */
+class InputExerciser {
+  private state = 1;
+  private held = new Map<number, boolean>();
+  /** Where delivered events are logged so the replay can redeliver them. */
+  log: PlatformFrameInput | null = null;
+
+  constructor(readonly target: EmuTarget, seed: number) {
+    this.state = seed | 0 || 1;
+  }
+
+  private next(): number {
+    // xorshift32, same shape as the emulator's own noise()
+    let x = this.state;
+    x ^= x << 13; x ^= x >>> 17; x ^= x << 5;
+    return (this.state = x) >>> 0;
+  }
+
+  /** Press or release a key or two, as if a player were at the controls. */
+  frame(): void {
+    for (const key of EXERCISE_KEYS) {
+      if ((this.next() & 7) !== 0) continue;         // ~1 change per key per 8 frames
+      const down = !this.held.get(key);
+      this.held.set(key, down);
+      const flags = down ? KeyFlags.KeyDown : KeyFlags.KeyUp;
+      this.target.setKeyInput(key, key, flags);
+      this.log?.recordKey(key, key, flags);
+    }
+  }
+}
+
+/**
+ * Record a run, then replay it from the first checkpoint and check that every
+ * frame comes back bit for bit. This is what keeps the deterministic debugger
+ * honest: rewinding is only meaningful on platforms that reproduce, and a
+ * platform that quietly picks up entropy (an unseeded Math.random, a clock)
+ * shows up here as a diverging frame rather than as a mystery months later.
+ */
+async function doVerifyReplay(args: Args, positional: string[]): Promise<void> {
+  const input = positional[0];
+  const platformId = str(args, 'platform') || (input ? ROM_PLATFORMS[path.extname(input).toLowerCase()] : null);
+  if (!platformId) fail('verify-replay', 'Required: verify-replay --platform <id> [rom]');
+
+  const target = await openTarget(args, platformId);
+  if (input) {
+    if (!fs.existsSync(input)) fail('verify-replay', `No such file: ${input}`);
+    target.loadROM(new Uint8Array(fs.readFileSync(input)), path.basename(input));
+  }
+  const platform = target.platform;
+  if (!isRewindable(platform)) {
+    fail('verify-replay', `'${platformId}' cannot save and restore state`);
+  }
+
+  const frames = parseInt(str(args, 'frames') ?? '') || 60;
+  const wantInput = (str(args, 'input') ?? 'keys') !== 'none';
+  const seed = parseInt(str(args, 'seed') ?? '') || 12345;
+  let exerciser: InputExerciser | null = null;
+  let inputMode = 'none';
+  if (wantInput) {
+    exerciser = new InputExerciser(target, seed);
+    try {
+      exerciser.frame();          // fails fast if the target takes no key input
+      inputMode = 'keys';
+    } catch (e) {
+      exerciser = null;
+      inputMode = 'unsupported';
+    }
+  }
+  const core = createCore(platform);
+  const input_ = new PlatformFrameInput(platform, {
+    currentFrame: () => core.now().frame,
+    dispatchKey: (key, code, flags) => target.setKeyInput(key, code, flags),
+  });
+  if (exerciser) exerciser.log = input_;
+  // one checkpoint only, so the replay re-runs the whole span rather than
+  // being handed a fresh state part way through
+  const hist = new History(core, { checkpointInterval: frames + 1, input: input_ });
+
+  // record, hashing the state at every frame boundary
+  const recorded: number[] = [];
+  // distinct control states seen; 1 means the input never reached the machine,
+  // which would make a passing verification meaningless
+  const controlStates = new Set<number>();
+  for (let i = 0; i < frames; i++) {
+    // input goes in before recordFrame(), so the frame's captured controls
+    // include it -- that is what the replay has to reproduce
+    exerciser?.frame();
+    if (platform.saveControlsState) controlStates.add(hashState(platform.saveControlsState()));
+    hist.recordFrame();
+    recorded.push(hashState(core.snapshot()));
+  }
+
+  // replay the same span and compare
+  const diverged: { frame: number }[] = [];
+  hist.seek(hist.first());
+  for (let i = 0; i < frames; i++) {
+    hist.seek(timestamp(i + 1, 0));
+    const h = hashState(core.snapshot());
+    if (h !== recorded[i]) {
+      diverged.push({ frame: i + 1 });
+      if (diverged.length >= 5) break;
+    }
+  }
+
+  const ok = diverged.length === 0;
+  if (!ok) {
+    note(`first divergence at frame ${diverged[0].frame} of ${frames}`);
+  }
+  output({
+    success: ok,
+    command: 'verify-replay',
+    error: ok ? undefined : `replay diverged at frame ${diverged[0].frame}`,
+    data: {
+      platform: platformId,
+      rom: input ?? null,
+      frames,
+      input: inputMode === 'keys' && controlStates.size < 2 ? 'keys (no effect)' : inputMode,
+      controlStates: controlStates.size,
+      keyEvents: input_.keyEventCount,
+      seed: inputMode === 'keys' ? seed : null,
+      granularity: core.granularity,
+      deterministic: ok,
+      firstDivergentFrame: ok ? null : diverged[0].frame,
+      divergentFrames: diverged.map((d) => d.frame),
+      recordedTo: formatTimestamp(hist.last()),
+    }
+  });
+}
+
 ////////////////////////////////////////////////////////////////////////
 // listings & help
 
@@ -322,6 +473,7 @@ async function main() {
     switch (command) {
       case 'build': await doBuild(args, positional); break;
       case 'run': await doRun(args, positional); break;
+      case 'verify-replay': await doVerifyReplay(args, positional); break;
       case 'list-tools':
       case 'list-platforms': await doList(command); break;
       case 'help': usage(); break;

test/cli/testdeterminism.js

@@ -0,0 +1,111 @@
+"use strict";
+
+// Determinism regression tests: record a run with input, replay it from the
+// first checkpoint, and require every frame to come back bit for bit.
+//
+// This is the guard that keeps rewinding meaningful. Without it, a field that
+// is written but never saved (ANTIC's left/right, the WASM machines' joymask)
+// leaks across a rewind and nothing notices until a user is staring at a
+// debugger that shows the wrong past.
+
+var assert = require('assert');
+var fs = require('fs');
+var path = require('path');
+
+var emutarget = require('../../gen/tools/emutarget.js');
+var platformcore = require('../../gen/common/platformcore.js');
+var history = require('../../gen/common/history.js');
+var statehash = require('../../gen/common/statehash.js');
+var emu = require('../../gen/common/emu.js');
+
+var KEYS = [37, 38, 39, 40, 32, 13];   // left up right down space enter
+
+/** Seeded button masher, so a failure is reproducible. */
+function makeExerciser(target, log, seed) {
+  var state = seed | 0 || 1;
+  var held = {};
+  function next() {
+    var x = state;
+    x ^= x << 13; x ^= x >>> 17; x ^= x << 5;
+    return (state = x) >>> 0;
+  }
+  return function () {
+    for (var i = 0; i < KEYS.length; i++) {
+      var key = KEYS[i];
+      if ((next() & 7) !== 0) continue;
+      var down = !held[key];
+      held[key] = down;
+      var flags = down ? emu.KeyFlags.KeyDown : emu.KeyFlags.KeyUp;
+      target.setKeyInput(key, key, flags);
+      log.recordKey(key, key, flags);
+    }
+  };
+}
+
+function firstROM(platid) {
+  var dir = path.join('test', 'roms', platid);
+  return path.join(dir, fs.readdirSync(dir)[0]);
+}
+
+/** Record `frames` frames with input, then replay and compare every frame. */
+async function checkDeterministic(platid, frames) {
+  var target = await emutarget.loadPlatform(platid);
+  await target.start();
+  target.loadROM(new Uint8Array(fs.readFileSync(firstROM(platid))));
+
+  var core = platformcore.createCore(target.platform);
+  var input = new platformcore.PlatformFrameInput(target.platform, {
+    currentFrame: function () { return core.now().frame; },
+    dispatchKey: function (key, code, flags) { target.setKeyInput(key, code, flags); },
+  });
+  // one checkpoint, so the replay re-runs the whole span
+  var hist = new history.History(core, { checkpointInterval: frames + 1, input: input });
+  var exercise = makeExerciser(target, input, 12345);
+
+  var recorded = [];
+  var controlStates = {};
+  for (var i = 0; i < frames; i++) {
+    exercise();
+    if (target.platform.saveControlsState) {
+      controlStates[statehash.hashState(target.platform.saveControlsState())] = 1;
+    }
+    hist.recordFrame();
+    recorded.push(statehash.hashState(core.snapshot()));
+  }
+  // the input has to actually reach the machine, or a pass means nothing
+  assert.ok(Object.keys(controlStates).length > 1,
+    platid + ': the exerciser never changed the controls, so nothing was tested');
+
+  hist.seek(hist.first());
+  for (var i = 0; i < frames; i++) {
+    hist.seek({ frame: i + 1, step: 0 });
+    assert.strictEqual(statehash.hashState(core.snapshot()), recorded[i],
+      platid + ': replay diverged at frame ' + (i + 1) + ' of ' + frames);
+  }
+  return core.granularity;
+}
+
+describe('Deterministic replay', function () {
+  this.timeout(120000);
+
+  // a JS scanline machine, the two that had state-restore bugs, and a WASM
+  // machine of each kind
+  it('should replay mw8080bw exactly', async function () {
+    assert.strictEqual(await checkDeterministic('mw8080bw', 20), 'insn');
+  });
+  it('should replay apple2 exactly', async function () {
+    await checkDeterministic('apple2', 20);
+  });
+  it('should replay atari8-800 exactly (ANTIC left/right must round-trip)', async function () {
+    await checkDeterministic('atari8-800', 20);
+  });
+  it('should replay atari8-5200 exactly (key-down IRQ must be redelivered)', async function () {
+    await checkDeterministic('atari8-5200', 20);
+  });
+  it('should replay c64 exactly (joymask must round-trip)', async function () {
+    assert.strictEqual(await checkDeterministic('c64', 20), 'clock');
+  });
+  it('should replay zx exactly (joymask must round-trip)', async function () {
+    await checkDeterministic('zx', 20);
+  });
+});