ui: "click screen" hit before audio unmuted

CHANGELOG.md

@@ -44,7 +44,7 @@ See [extension/CHANGELOG.md](extension/CHANGELOG.md).
 - "Share Playable Link" packs the game into the URL fragment instead of the query string, so it no longer hits the web server's ~8 KB request-line limit, and encodes it as URL-safe Base64 to avoid percent-encoding. Old `?r=` links still play.
 - Removed the Markdown/Showdown platform, the old `script` platform, Google Analytics, Sentry (replaced with a new error-reporting endpoint) and the HTTPS redirect.
 - "Highlight Executed Lines" draws a hot/cold heat map: lines run since the last update are colored by how often they have run (blue = cold, red = hot) and disappear once they stop executing.
-- Audio reliably unlocks under the browser autoplay policy: the emulator resumes its `AudioContext` on the first click/keypress anywhere on the page, not only when the emulator canvas gets focus.
+- Audio reliably unlocks under the browser autoplay policy: the emulator resumes its `AudioContext` on the first click/keypress anywhere on the page, not only when the emulator canvas gets focus. A hint under the emulator screen says so until it is unlocked.
 - Memory Map: a Modules column shows each object file's slice of every SDCC or cc65 segment (`Segment.modules`; SDCC from the `.rel` sizes in link order, with the remainder as `(libraries)`; cc65 from the ld65 map's module list, library members by name).
 - Memory Browser: a symbol label covers only its reported size, and an unsized symbol stops at its segment boundary instead of running into the next segment.
 

css/ui.css

@@ -1029,6 +1029,12 @@ div.asset_toolbar {
   margin-top: auto;
   margin-bottom: 2em;
 }
+.audio-hint {
+  color: #ccc;
+  margin-top: 0.5em;
+  margin-left: 1em;
+  margin-right: 1em;
+}
 .control-key {
   border: 2px solid rgba(0,0,0,0.2);
   border-radius: 0.6em;

index.html

@@ -298,6 +298,7 @@ body {
       <div id="javatari-console-panel" style="margin: 0 auto; box-shadow: 2px 2px 10px rgb(60, 60, 60);"></div>
     </div>
     <!-- control instructions: src/common/controls.ts -->
+    <div id="emuaudiohint" class="text-center small audio-hint" style="display:none">Click screen to use controls and unmute audio</div>
     <div id="emucontrols" class="text-center small control-insns" style="display:none"></div>
     <!-- -->
     <div id="emuoverlay" class="emuoverlay" style="display:none">

src/common/audio.ts

@@ -398,17 +398,88 @@ export const AUDIO_CHUNK_SAMPLES = 2048;
 // must be called synchronously inside the handler for Safari/iOS to allow it.
 // Called on every start(), since stop() suspends the context.
 const audioUnlockEvents = ['pointerdown', 'touchend', 'mousedown', 'keydown', 'click'];
-const pendingUnlocks = new WeakSet<AudioContext>();
+const pendingUnlocks = new WeakSet<AudioContext>();   // waiting for the first user gesture
+const unlockListeners = new WeakSet<AudioContext>();  // document gesture handlers installed
+
+// Every AudioContext the IDE starts is registered here so the UI can tell
+// whether Web Audio is still waiting for the user gesture that owns the page.
+// Browsers keep one suspended until then, and resume() resolves asynchronously,
+// so notify listeners on registration, state changes and resume completion.
+const audioContexts = new Set<AudioContext>();
+const audioStateListeners = new Set<() => void>();
+
+function notifyAudioState() {
+  for (const listener of Array.from(audioStateListeners)) listener();
+}
+
+function markAudioUnlocked(ctx: AudioContext) {
+  pendingUnlocks.delete(ctx);
+  notifyAudioState();
+}
+
+function trackAudioContext(ctx: AudioContext) {
+  if (audioContexts.has(ctx)) return;
+  audioContexts.add(ctx);
+  if (typeof ctx.addEventListener === 'function') {
+    ctx.addEventListener('statechange', () => {
+      if (ctx.state === 'running') {
+        pendingUnlocks.delete(ctx);
+      } else if (ctx.state === 'closed') {
+        // a closed context can never be resumed, so stop counting it as blocked
+        audioContexts.delete(ctx);
+        pendingUnlocks.delete(ctx);
+      }
+      notifyAudioState();
+    });
+  }
+  notifyAudioState();
+}
+
+/**
+ * True while an AudioContext is waiting for the user gesture that unlocks
+ * Web Audio. A context the IDE suspends itself (pause, tab blur) is not blocked;
+ * it comes back without a gesture.
+ */
+export function isAudioBlocked(): boolean {
+  for (const ctx of audioContexts) {
+    if (ctx.state !== 'running' && pendingUnlocks.has(ctx)) return true;
+  }
+  return false;
+}
+
+/** Subscribe to AudioContext creation and state changes. Returns an unsubscribe fn. */
+export function onAudioStateChange(listener: () => void): () => void {
+  audioStateListeners.add(listener);
+  return () => { audioStateListeners.delete(listener); };
+}
+
+function resumeTrackedContext(ctx: AudioContext) {
+  const resumed = ctx.resume();
+  if (resumed && typeof resumed.then === 'function') {
+    resumed.then(() => {
+      if (ctx.state === 'running') markAudioUnlocked(ctx);
+      else notifyAudioState();
+    }, () => {});
+  }
+}
+
 export function resumeAudioContext(ctx: AudioContext) {
-  if (!ctx || ctx.state === 'running') return;
-  ctx.resume().catch(() => {});
-  if (typeof document === 'undefined' || pendingUnlocks.has(ctx)) return;
+  if (!ctx || ctx.state === 'closed') return;
+  trackAudioContext(ctx);
+  if (ctx.state === 'running') {
+    pendingUnlocks.delete(ctx);
+    return;
+  }
   pendingUnlocks.add(ctx);
+  notifyAudioState(); // show the click hint even if resume() stays suspended
+  resumeTrackedContext(ctx);
+  if (typeof document === 'undefined' || unlockListeners.has(ctx)) return;
+  unlockListeners.add(ctx);
   const onGesture = () => {
-    ctx.resume().catch(() => {});
+    resumeTrackedContext(ctx);
     if (ctx.state === 'running') {
       for (const ev of audioUnlockEvents) document.removeEventListener(ev, onGesture, true);
-      pendingUnlocks.delete(ctx);
+      unlockListeners.delete(ctx);
     }
   };
   for (const ev of audioUnlockEvents) document.addEventListener(ev, onGesture, true);

src/ide/ui.ts

@@ -6,6 +6,7 @@ import { BaseDebugPlatform, DebugEvalCondition, DebugSymbols, EmuState, isDebugg
 import { EmuHalt, PLATFORMS, setHaltHandler } from "../common/emu";
 import { BOOKS, bookFor } from "../common/books";
 import { PLATFORM_CONTROLS } from "../common/controls";
+import { isAudioBlocked, onAudioStateChange } from "../common/audio";
 import { installHDLHost } from "./hdlhost";
 import { StateRecorderImpl } from "../common/recorder";
 import {
@@ -2323,7 +2324,27 @@ function renderControls(div: JQuery) {
   return div.children().length > 0;
 }
 
+function showAudioHint() {
+  const hint = $("#emuaudiohint");
+  if (!hint.length) return;
+  let wasBlocked = false;
+  const update = () => {
+    if (isAudioBlocked()) {
+      wasBlocked = true;
+      hint.show();
+    } else {
+      hint.hide();
+      if (wasBlocked) unsubscribe();
+    }
+  };
+  // AudioContext.resume() is async, so wait for the context to actually start
+  // instead of hiding on the click that asked it to.
+  const unsubscribe = onAudioStateChange(update);
+  update();
+}
+
 function showInstructions() {
+  showAudioHint();
   var div = $("#emucontrols");
   if (!renderControls(div)) return;
   if (platform_id.endsWith(".mame")) div.show(); // TODO: MAME seems to eat the focus() event

test/unit/testaudio.ts

@@ -1,6 +1,6 @@
 import assert from "assert";
 import { describe, it, afterEach } from "mocha";
-import { AUDIO_CHUNK_SAMPLES, AudioStream, SampledAudio, setAudioStreamFactory } from "../../src/common/audio";
+import { AUDIO_CHUNK_SAMPLES, AudioStream, SampledAudio, isAudioBlocked, onAudioStateChange, resumeAudioContext, setAudioStreamFactory } from "../../src/common/audio";
 
 // A stand-in for the extension's worker-side stream (see extension/emuworker.ts).
 class TestStream implements AudioStream {
@@ -100,6 +100,28 @@ class SuspendedAudioContext extends FakeAudioContext {
   resume() { this.resumes++; this.state = 'running'; return Promise.resolve(); }
 }
 
+// Like a real context: resume() takes a moment, then fires statechange. Used to
+// check that the UI hint waits for Web Audio to actually start.
+class GestureAudioContext extends FakeAudioContext {
+  state = 'suspended';
+  stateListeners: (() => void)[] = [];
+  addEventListener(type: string, listener: () => void) {
+    if (type === 'statechange') this.stateListeners.push(listener);
+  }
+  resume() {
+    return Promise.resolve().then(() => {
+      this.state = 'running';
+      for (const listener of this.stateListeners) listener();
+    });
+  }
+  suspend() {
+    return Promise.resolve().then(() => {
+      this.state = 'suspended';
+      for (const listener of this.stateListeners) listener();
+    });
+  }
+}
+
 const RING_BUFFER_SAMPLES = 2048;
 
 function withFakeAudio(fn: () => void | Promise<void>) {
@@ -146,6 +168,28 @@ describe('SampleAudio autoplay unlock', function () {
       (global as any).window = prev;
     }
   });
+
+  it('reports a context as blocked until its resume() resolves', async function () {
+    const ctx = new GestureAudioContext();
+    let changes = 0;
+    const unsubscribe = onAudioStateChange(() => changes++);
+    resumeAudioContext(ctx as any);
+    assert.equal(isAudioBlocked(), true);
+    await new Promise((r) => setTimeout(r, 0));
+    assert.equal(ctx.state, 'running');
+    assert.equal(isAudioBlocked(), false);
+    assert.ok(changes >= 2, 'expected registration and unlock notifications');
+    unsubscribe();
+  });
+
+  it('does not treat a context the IDE suspends as blocked', async function () {
+    const ctx = new GestureAudioContext();
+    resumeAudioContext(ctx as any);
+    await new Promise((r) => setTimeout(r, 0));
+    assert.equal(isAudioBlocked(), false);
+    await ctx.suspend();
+    assert.equal(isAudioBlocked(), false);
+  });
 });
 
 describe('SampleAudio ring depth', function () {