search: toolchainsource just does a header search on the worker

src/ide/search/projectsource.ts

@@ -6,14 +6,22 @@ import { extractSymbols } from "../../common/symbols";
 // Avoid a hard dependency on ../ui (which pulls in the whole IDE and its
 // language grammar imports that crash under plain Node). ui.ts registers a
 // getter here during startup instead.
-type ProjectProvider = () => { mainPath: string; iterateFiles(cb: (path: string, data: any) => void): void } | null;
+export interface SearchableProject {
+  mainPath: string;
+  platform_id?: string;
+  getToolForFilename?(path: string): string;
+  iterateFiles(cb: (path: string, data: any) => void): void;
+  queryWorker?(msg: any): Promise<any | null>;
+}
+
+type ProjectProvider = () => SearchableProject | null;
 let projectProvider: ProjectProvider = () => null;
 
 export function setProjectProvider(fn: ProjectProvider) {
   projectProvider = fn;
 }
 
-function getProject() {
+export function getProject(): SearchableProject | null {
   return projectProvider();
 }
 

src/ide/search/toolchainsource.ts

@@ -1,68 +1,191 @@
-import { SymbolIndex } from "./symbolindex";
-import { SymbolSourceKind, PrebuiltIndex, SYMIDX_VERSION } from "../../common/searchtypes";
 import { SearchHit, SearchSource } from "./types";
-import { current_project } from "../ui";
-import { getSharedFileSystemName } from "../../common/toolmeta";
+import { SymbolSourceKind } from "../../common/searchtypes";
+import { getProject } from "./projectsource";
+import { getSharedFileSystemName, getIncludeDirs } from "../../common/toolmeta";
+import { WorkerMessage } from "../../common/workertypes";
+
+/**
+ * ToolchainSource - dumb full-text search over the toolchain's bundled
+ * header files, served by the worker's preload filesystem (same path the
+ * HeaderView uses via readshared/listshared messages).
+ *
+ * Instead of a prebuilt symbol index (gen/symidx/*.json never shipped), we
+ * lazily list files under the tool's include dirs and read their text from
+ * the worker in small batches as the user types. Results are low-priority
+ * text hits, ranked below project symbols in the unified service.
+ */
+
+// File extensions that plausibly contain text to search.
+const SEARCHABLE_EXT = new Set(['h', 'hh', 'hpp', 'inc', 'asm', 'a65', 's', 'mac', 'defs', 'def', 'equ', 'i', 'm', 'c', 'bas', 'txt']);
+
+// How many unread header files to fetch from the worker per query.
+const BATCH_SIZE = 20;
 
 export class ToolchainSource implements SearchSource {
   id: string = 'toolchain';
   kind: SymbolSourceKind = 'toolchain';
-  private indices: Map<string, SymbolIndex> = new Map();
-  private corpusReady: Map<string, boolean> = new Map();
 
-  constructor() {}
+  private fsName: string | null = null;
+  private files: string[] = [];              // all searchable files (sorted)
+  private loaded: Map<string, string> = new Map(); // path -> decoded text
+  private readyDone: boolean = false;
+  private loadIdx: number = 0;               // next unread file index
+  batchSize: number = BATCH_SIZE;            // files fetched per query (overridable in tests)
 
   async ready() {
-    const project = current_project;
+    if (this.readyDone) return;
+    this.readyDone = true;
+
+    const project = getProject();
     if (!project) return;
 
-    const tool = project.getToolForFilename(project.mainPath);
-    const corpus = getSharedFileSystemName(tool, project.platform_id);
-    if (corpus) {
-      await this.loadCorpus(corpus);
-    }
-  }
-
-  async loadCorpus(corpus: string) {
-    if (this.corpusReady.get(corpus)) return;
-
+    let tool: string;
     try {
-      const url = `gen/symidx/${corpus}.json`;
-      const response = await fetch(url);
-      if (!response.ok) {
-        this.corpusReady.set(corpus, true);
-        return;
-      }
-      const json = await response.json();
-      if (json.version !== SYMIDX_VERSION) {
-        this.corpusReady.set(corpus, true);
-        return;
-      }
-      const indexData = json as PrebuiltIndex;
-      const idx = new SymbolIndex(indexData.records);
-      await idx.init();
-      if (indexData.ms) {
-        await idx.setMiniSearch(indexData.ms);
-      }
-      this.indices.set(corpus, idx);
-      this.corpusReady.set(corpus, true);
+      tool = project.getToolForFilename ? project.getToolForFilename(project.mainPath) : '';
     } catch (e) {
-      console.debug('Failed to load toolchain index:', e);
-      this.corpusReady.set(corpus, true);
+      tool = '';
+    }
+
+    const fsName = getSharedFileSystemName(tool, project.platform_id);
+    const dirs = getIncludeDirs(tool, project.platform_id);
+    if (!fsName || !dirs.length) return;
+    this.fsName = fsName;
+
+    // List files under each include dir, e.g. /include, /asminc, /headers.
+    const seen = new Set<string>();
+    for (const dir of dirs) {
+      const files = await this.listFiles(dir);
+      for (const f of files) {
+        if (this.isSearchable(f) && !seen.has(f)) {
+          seen.add(f);
+          this.files.push(f);
+        }
+      }
+    }
+    this.files.sort();
+  }
+
+  isValid(): boolean {
+    return this.fsName != null && this.files.length > 0;
+  }
+
+  /** Ask the worker to list files under a directory of the preload fs. */
+  private async listFiles(dir: string): Promise<string[]> {
+    const project = getProject();
+    if (!project || !this.fsName || !project.queryWorker) return [];
+    try {
+      const msg = { preload_fs: this.fsName, listshared: dir, updates: [], buildsteps: [] } as WorkerMessage;
+      const result = await project.queryWorker(msg);
+      const files = result && result.output;
+      return Array.isArray(files) ? (files as string[]) : [];
+    } catch (e) {
+      console.debug('Toolchain list error:', e);
+      return [];
     }
   }
 
-  query(needle: string, limit: number): SearchHit[] {
-    const hits: SearchHit[] = [];
-    for (const idx of this.indices.values()) {
-      const result = idx.query(needle, limit);
-      hits.push(...result);
+  /** Read one file from the worker's preload fs, decoded as text. */
+  private async readFile(path: string): Promise<string | null> {
+    const project = getProject();
+    if (!project || !this.fsName || !project.queryWorker) return null;
+    try {
+      const msg = { preload_fs: this.fsName, readshared: path, updates: [], buildsteps: [] } as WorkerMessage;
+      const result = await project.queryWorker(msg);
+      const output = result && result.output;
+      if (output instanceof Uint8Array && output.length > 0) {
+        return new TextDecoder().decode(output);
+      }
+      return null;
+    } catch (e) {
+      console.debug('Toolchain read error:', e);
+      return null;
     }
+  }
+
+  private isSearchable(path: string): boolean {
+    const base = path.split('/').pop() || path;
+    const dot = base.lastIndexOf('.');
+    if (dot < 0) return true; // no extension: assume text
+    return SEARCHABLE_EXT.has(base.substring(dot + 1).toLowerCase());
+  }
+
+  /**
+   * Load the next batch of unread files (or finish a partially-loaded one).
+   * Called per query so keystrokes incrementally widen the searchable corpus.
+   */
+  private async ensureBatch() {
+    const project = getProject();
+    if (!project || !project.queryWorker) return;
+    while (this.loadIdx < this.files.length) {
+      const batch = this.files.slice(this.loadIdx, this.loadIdx + this.batchSize);
+      this.loadIdx += batch.length;
+      const entries = await Promise.all(batch.map(async (path) => {
+        const text = await this.readFile(path);
+        return [path, text] as const;
+      }));
+      for (const [path, text] of entries) {
+        if (text != null) this.loaded.set(path, text);
+      }
+      // Keep loading until we have at least one text file in the batch
+      // (binary/skipped files shouldn't stall the palette), but never more
+      // than once per query -- the next query pulls the next batch.
+      if (this.loaded.size > 0 || this.loadIdx >= this.files.length) break;
+    }
+  }
+
+  async query(needle: string, limit: number): Promise<SearchHit[]> {
+    await this.ready();
+    if (!this.isValid() || needle.length < 2) return [];
+    await this.ensureBatch();
+    return this.textQuery(needle, limit);
+  }
+
+  /** Low-priority dumb text search over the loaded header files. */
+  private textQuery(needle: string, limit: number): SearchHit[] {
+    if (needle.length < 2) return [];
+
+    const needleLower = needle.toLowerCase();
+    const hits: SearchHit[] = [];
+    const MAX_PER_FILE = Math.max(limit, 5);
+
+    for (const [path, text] of this.loaded) {
+      const lines = text.split('\n');
+      let fileCount = 0;
+      for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const lineLower = line.toLowerCase();
+        const col = lineLower.indexOf(needleLower);
+        if (col >= 0) {
+          const trimmed = line.trim();
+          if (!trimmed) continue;
+          hits.push({
+            record: {
+              id: `toolchain:${path}:${i + 1}`,
+              name: trimmed.length > 80 ? trimmed.slice(0, 80) + '…' : trimmed,
+              kind: 'text',
+              source: 'toolchain',
+              file: path,
+              line: i + 1,
+            },
+            // Same scoring shape as project fulltext: well below symbols,
+            // slight bonus for match near line start.
+            score: 200 + Math.max(0, 100 - col * 2),
+            ranges: [col, col + needle.length],
+          });
+          if (++fileCount >= MAX_PER_FILE) break;
+        }
+      }
+    }
+
+    hits.sort((a, b) => b.score - a.score);
     return hits.slice(0, limit);
   }
 
   invalidate() {
-    this.indices.clear();
-    this.corpusReady.clear();
+    this.fsName = null;
+    this.files = [];
+    this.loaded.clear();
+    this.readyDone = false;
+    this.loadIdx = 0;
   }
 }

src/ide/search/types.ts

@@ -10,7 +10,7 @@ export interface SearchSource {
   id: string;
   kind: SymbolSourceKind;
   ready(): Promise<void>;
-  query(needle: string, limit: number): SearchHit[];
+  query(needle: string, limit: number): SearchHit[] | Promise<SearchHit[]>;
 }
 
 export interface SearchQuery {

test/unit/testsearchtoolchain.ts

@@ -0,0 +1,145 @@
+/**
+ * ToolchainSource tests - dumb full-text search over the worker's bundled
+ * header files. The worker is mocked by providing a queryWorker on the
+ * stubbed project (same provider injection as ProjectSource).
+ */
+
+import * as assert from 'assert';
+import { ToolchainSource } from '../../src/ide/search/toolchainsource';
+import { setProjectProvider } from '../../src/ide/search/projectsource';
+
+/** Build a fake "worker" backed by a hash of paths -> text. */
+function makeFs(files: { [path: string]: string }) {
+  const paths = Object.keys(files).sort();
+  return {
+    mainPath: 'main.c',
+    platform_id: 'nes',
+    iterateFiles: (cb: any) => {},
+    getToolForFilename: () => 'cc65',
+    queryWorker: async (msg: any) => {
+      if (msg.listshared != null) {
+        const prefix = msg.listshared + '/';
+        return { output: paths.filter(p => p.startsWith(prefix)) };
+      }
+      if (msg.readshared != null) {
+        const text = files[msg.readshared];
+        if (text == null) return { output: null };
+        return { output: new TextEncoder().encode(text) };
+      }
+      return null;
+    }
+  };
+}
+
+const NES_H = `#ifndef NES_H
+#define NES_H
+/* NES registers */
+#define PPU_CTRL  0x2000
+#define PPU_MASK  0x2001
+/* Turn off the screen */
+#define VBLANK    0x00
+#define PPU_STATUS 0x2002
+static char PPU_READ(unsigned char a) {
+  return a;
+}
+#endif
+`;
+
+const APPLE_H = `#ifndef APPLE_H
+#define APPLE_H
+/* Apple II registers */
+#define KBD 0xC000
+/* Read keyboard */
+#define KBD_READ 0xC000
+#endif
+`;
+
+describe('ToolchainSource', function () {
+  after(function () {
+    setProjectProvider(() => null);
+  });
+
+  it('should list and lazily search bundled header files', async function () {
+    setProjectProvider(() => makeFs({ '/include/nes.h': NES_H, '/include/apple.h': APPLE_H }));
+    const ts = new ToolchainSource();
+
+    // First query triggers ready() (lists files) + ensureBatch() (reads them)
+    const hits = await ts.query('VBLANK', 10);
+
+    assert.ok(hits.length >= 1, 'should find VBLANK in nes.h');
+    const rec = hits[0].record;
+    assert.strictEqual(rec.kind, 'text');
+    assert.strictEqual(rec.source, 'toolchain');
+    assert.strictEqual(rec.file, '/include/nes.h');
+    assert.ok(rec.line && rec.line > 0);
+    // The two hash-flavored hits should both appear (each file is searched)
+    assert.ok(hits.some(h => h.record.file === '/include/nes.h' && h.record.name.includes('VBLANK')));
+  });
+
+  it('should be case-insensitive', async function () {
+    setProjectProvider(() => makeFs({ '/include/nes.h': NES_H }));
+    const ts = new ToolchainSource();
+    const hits = await ts.query('vblank', 10);
+    assert.ok(hits.length >= 1);
+  });
+
+  it('should require at least 2 chars', async function () {
+    setProjectProvider(() => makeFs({ '/include/nes.h': NES_H }));
+    const ts = new ToolchainSource();
+    const hits = await ts.query('x', 10);
+    assert.strictEqual(hits.length, 0);
+  });
+
+  it('should load files in batches across queries', async function () {
+    const fs = makeFs({ '/include/nes.h': NES_H, '/include/apple.h': APPLE_H });
+    setProjectProvider(() => fs);
+    const ts = new ToolchainSource();
+
+    // Shrink batch so a query only loads a subset
+    (ts as any).files = ['/include/nes.h', '/include/apple.h'];
+    (ts as any).loadIdx = 0;
+    (ts as any).fsName = '65-nes';
+    (ts as any).readyDone = true;
+    (ts as any).batchSize = 1;
+
+    // First query: only nes.h loaded
+    let hits = await ts.query('VBLANK', 10);
+    assert.ok(hits.length >= 1, 'nes.h loaded in first batch');
+    assert.strictEqual(ts['loaded'].size, 1);
+
+    // Second query: apple.h loaded too
+    hits = await ts.query('VBLANK', 10);
+    assert.strictEqual(ts['loaded'].size, 2);
+  });
+
+  it('should rank matches near the start of the line higher', async function () {
+    setProjectProvider(() => makeFs({ '/include/nes.h': NES_H }));
+    const ts = new ToolchainSource();
+    const hits = await ts.query('PPU', 10);
+    assert.ok(hits.length >= 2, 'PPU matches several lines');
+    // Earliest-column hit first (PPU_CTRL at col 17 is #define PPU_CTRL); the
+    // column-based score should order them without any flakiness
+    const cols = hits.map(h => h.ranges ? h.ranges[0] : 0);
+    for (let i = 1; i < cols.length; i++) {
+      assert.ok(cols[i - 1] <= cols[i], 'hits sorted by column');
+    }
+  });
+
+  it('should skip non-searchable file types', async function () {
+    const ts = new ToolchainSource();
+    assert.strictEqual((ts as any).isSearchable('/include/nes.h'), true);
+    assert.strictEqual((ts as any).isSearchable('/include/nes.o'), false);
+    assert.strictEqual((ts as any).isSearchable('/lib/binary.bin'), false);
+    assert.strictEqual((ts as any).isSearchable('/scripts/makefile'), true); // no ext
+  });
+
+  it('should handle missing worker gracefully', async function () {
+    setProjectProvider(() => ({
+      mainPath: 'main.c',
+      iterateFiles: (cb: any) => {},
+    }));
+    const ts = new ToolchainSource();
+    const hits = await ts.query('anything', 10);
+    assert.deepStrictEqual(hits, []);
+  });
+});