From 8f6c7c2fe2007339fc2d7549c47d9c8b88f27698 Mon Sep 17 00:00:00 2001 From: Steven Hugg Date: Sat, 29 Aug 2026 17:42:19 +0200 Subject: [PATCH] search: toolchainsource just does a header search on the worker --- src/ide/search/projectsource.ts | 12 +- src/ide/search/toolchainsource.ts | 215 +++++++++++++++++++++++------- src/ide/search/types.ts | 2 +- test/unit/testsearchtoolchain.ts | 145 ++++++++++++++++++++ 4 files changed, 325 insertions(+), 49 deletions(-) create mode 100644 test/unit/testsearchtoolchain.ts diff --git a/src/ide/search/projectsource.ts b/src/ide/search/projectsource.ts index fc5f603a..9b894ac3 100644 --- a/src/ide/search/projectsource.ts +++ b/src/ide/search/projectsource.ts @@ -6,14 +6,22 @@ import { extractSymbols } from "../../common/symbols"; // Avoid a hard dependency on ../ui (which pulls in the whole IDE and its // language grammar imports that crash under plain Node). ui.ts registers a // getter here during startup instead. -type ProjectProvider = () => { mainPath: string; iterateFiles(cb: (path: string, data: any) => void): void } | null; +export interface SearchableProject { + mainPath: string; + platform_id?: string; + getToolForFilename?(path: string): string; + iterateFiles(cb: (path: string, data: any) => void): void; + queryWorker?(msg: any): Promise; +} + +type ProjectProvider = () => SearchableProject | null; let projectProvider: ProjectProvider = () => null; export function setProjectProvider(fn: ProjectProvider) { projectProvider = fn; } -function getProject() { +export function getProject(): SearchableProject | null { return projectProvider(); } diff --git a/src/ide/search/toolchainsource.ts b/src/ide/search/toolchainsource.ts index 2a0f4852..03fce160 100644 --- a/src/ide/search/toolchainsource.ts +++ b/src/ide/search/toolchainsource.ts @@ -1,68 +1,191 @@ -import { SymbolIndex } from "./symbolindex"; -import { SymbolSourceKind, PrebuiltIndex, SYMIDX_VERSION } from "../../common/searchtypes"; import { SearchHit, SearchSource } from "./types"; -import { current_project } from "../ui"; -import { getSharedFileSystemName } from "../../common/toolmeta"; +import { SymbolSourceKind } from "../../common/searchtypes"; +import { getProject } from "./projectsource"; +import { getSharedFileSystemName, getIncludeDirs } from "../../common/toolmeta"; +import { WorkerMessage } from "../../common/workertypes"; + +/** + * ToolchainSource - dumb full-text search over the toolchain's bundled + * header files, served by the worker's preload filesystem (same path the + * HeaderView uses via readshared/listshared messages). + * + * Instead of a prebuilt symbol index (gen/symidx/*.json never shipped), we + * lazily list files under the tool's include dirs and read their text from + * the worker in small batches as the user types. Results are low-priority + * text hits, ranked below project symbols in the unified service. + */ + +// File extensions that plausibly contain text to search. +const SEARCHABLE_EXT = new Set(['h', 'hh', 'hpp', 'inc', 'asm', 'a65', 's', 'mac', 'defs', 'def', 'equ', 'i', 'm', 'c', 'bas', 'txt']); + +// How many unread header files to fetch from the worker per query. +const BATCH_SIZE = 20; export class ToolchainSource implements SearchSource { id: string = 'toolchain'; kind: SymbolSourceKind = 'toolchain'; - private indices: Map = new Map(); - private corpusReady: Map = new Map(); - constructor() {} + private fsName: string | null = null; + private files: string[] = []; // all searchable files (sorted) + private loaded: Map = new Map(); // path -> decoded text + private readyDone: boolean = false; + private loadIdx: number = 0; // next unread file index + batchSize: number = BATCH_SIZE; // files fetched per query (overridable in tests) async ready() { - const project = current_project; + if (this.readyDone) return; + this.readyDone = true; + + const project = getProject(); if (!project) return; - const tool = project.getToolForFilename(project.mainPath); - const corpus = getSharedFileSystemName(tool, project.platform_id); - if (corpus) { - await this.loadCorpus(corpus); - } - } - - async loadCorpus(corpus: string) { - if (this.corpusReady.get(corpus)) return; - + let tool: string; try { - const url = `gen/symidx/${corpus}.json`; - const response = await fetch(url); - if (!response.ok) { - this.corpusReady.set(corpus, true); - return; - } - const json = await response.json(); - if (json.version !== SYMIDX_VERSION) { - this.corpusReady.set(corpus, true); - return; - } - const indexData = json as PrebuiltIndex; - const idx = new SymbolIndex(indexData.records); - await idx.init(); - if (indexData.ms) { - await idx.setMiniSearch(indexData.ms); - } - this.indices.set(corpus, idx); - this.corpusReady.set(corpus, true); + tool = project.getToolForFilename ? project.getToolForFilename(project.mainPath) : ''; } catch (e) { - console.debug('Failed to load toolchain index:', e); - this.corpusReady.set(corpus, true); + tool = ''; + } + + const fsName = getSharedFileSystemName(tool, project.platform_id); + const dirs = getIncludeDirs(tool, project.platform_id); + if (!fsName || !dirs.length) return; + this.fsName = fsName; + + // List files under each include dir, e.g. /include, /asminc, /headers. + const seen = new Set(); + for (const dir of dirs) { + const files = await this.listFiles(dir); + for (const f of files) { + if (this.isSearchable(f) && !seen.has(f)) { + seen.add(f); + this.files.push(f); + } + } + } + this.files.sort(); + } + + isValid(): boolean { + return this.fsName != null && this.files.length > 0; + } + + /** Ask the worker to list files under a directory of the preload fs. */ + private async listFiles(dir: string): Promise { + const project = getProject(); + if (!project || !this.fsName || !project.queryWorker) return []; + try { + const msg = { preload_fs: this.fsName, listshared: dir, updates: [], buildsteps: [] } as WorkerMessage; + const result = await project.queryWorker(msg); + const files = result && result.output; + return Array.isArray(files) ? (files as string[]) : []; + } catch (e) { + console.debug('Toolchain list error:', e); + return []; } } - query(needle: string, limit: number): SearchHit[] { - const hits: SearchHit[] = []; - for (const idx of this.indices.values()) { - const result = idx.query(needle, limit); - hits.push(...result); + /** Read one file from the worker's preload fs, decoded as text. */ + private async readFile(path: string): Promise { + const project = getProject(); + if (!project || !this.fsName || !project.queryWorker) return null; + try { + const msg = { preload_fs: this.fsName, readshared: path, updates: [], buildsteps: [] } as WorkerMessage; + const result = await project.queryWorker(msg); + const output = result && result.output; + if (output instanceof Uint8Array && output.length > 0) { + return new TextDecoder().decode(output); + } + return null; + } catch (e) { + console.debug('Toolchain read error:', e); + return null; } + } + + private isSearchable(path: string): boolean { + const base = path.split('/').pop() || path; + const dot = base.lastIndexOf('.'); + if (dot < 0) return true; // no extension: assume text + return SEARCHABLE_EXT.has(base.substring(dot + 1).toLowerCase()); + } + + /** + * Load the next batch of unread files (or finish a partially-loaded one). + * Called per query so keystrokes incrementally widen the searchable corpus. + */ + private async ensureBatch() { + const project = getProject(); + if (!project || !project.queryWorker) return; + while (this.loadIdx < this.files.length) { + const batch = this.files.slice(this.loadIdx, this.loadIdx + this.batchSize); + this.loadIdx += batch.length; + const entries = await Promise.all(batch.map(async (path) => { + const text = await this.readFile(path); + return [path, text] as const; + })); + for (const [path, text] of entries) { + if (text != null) this.loaded.set(path, text); + } + // Keep loading until we have at least one text file in the batch + // (binary/skipped files shouldn't stall the palette), but never more + // than once per query -- the next query pulls the next batch. + if (this.loaded.size > 0 || this.loadIdx >= this.files.length) break; + } + } + + async query(needle: string, limit: number): Promise { + await this.ready(); + if (!this.isValid() || needle.length < 2) return []; + await this.ensureBatch(); + return this.textQuery(needle, limit); + } + + /** Low-priority dumb text search over the loaded header files. */ + private textQuery(needle: string, limit: number): SearchHit[] { + if (needle.length < 2) return []; + + const needleLower = needle.toLowerCase(); + const hits: SearchHit[] = []; + const MAX_PER_FILE = Math.max(limit, 5); + + for (const [path, text] of this.loaded) { + const lines = text.split('\n'); + let fileCount = 0; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const lineLower = line.toLowerCase(); + const col = lineLower.indexOf(needleLower); + if (col >= 0) { + const trimmed = line.trim(); + if (!trimmed) continue; + hits.push({ + record: { + id: `toolchain:${path}:${i + 1}`, + name: trimmed.length > 80 ? trimmed.slice(0, 80) + '…' : trimmed, + kind: 'text', + source: 'toolchain', + file: path, + line: i + 1, + }, + // Same scoring shape as project fulltext: well below symbols, + // slight bonus for match near line start. + score: 200 + Math.max(0, 100 - col * 2), + ranges: [col, col + needle.length], + }); + if (++fileCount >= MAX_PER_FILE) break; + } + } + } + + hits.sort((a, b) => b.score - a.score); return hits.slice(0, limit); } invalidate() { - this.indices.clear(); - this.corpusReady.clear(); + this.fsName = null; + this.files = []; + this.loaded.clear(); + this.readyDone = false; + this.loadIdx = 0; } } \ No newline at end of file diff --git a/src/ide/search/types.ts b/src/ide/search/types.ts index a91f47f7..9a545570 100644 --- a/src/ide/search/types.ts +++ b/src/ide/search/types.ts @@ -10,7 +10,7 @@ export interface SearchSource { id: string; kind: SymbolSourceKind; ready(): Promise; - query(needle: string, limit: number): SearchHit[]; + query(needle: string, limit: number): SearchHit[] | Promise; } export interface SearchQuery { diff --git a/test/unit/testsearchtoolchain.ts b/test/unit/testsearchtoolchain.ts new file mode 100644 index 00000000..793a4f33 --- /dev/null +++ b/test/unit/testsearchtoolchain.ts @@ -0,0 +1,145 @@ +/** + * ToolchainSource tests - dumb full-text search over the worker's bundled + * header files. The worker is mocked by providing a queryWorker on the + * stubbed project (same provider injection as ProjectSource). + */ + +import * as assert from 'assert'; +import { ToolchainSource } from '../../src/ide/search/toolchainsource'; +import { setProjectProvider } from '../../src/ide/search/projectsource'; + +/** Build a fake "worker" backed by a hash of paths -> text. */ +function makeFs(files: { [path: string]: string }) { + const paths = Object.keys(files).sort(); + return { + mainPath: 'main.c', + platform_id: 'nes', + iterateFiles: (cb: any) => {}, + getToolForFilename: () => 'cc65', + queryWorker: async (msg: any) => { + if (msg.listshared != null) { + const prefix = msg.listshared + '/'; + return { output: paths.filter(p => p.startsWith(prefix)) }; + } + if (msg.readshared != null) { + const text = files[msg.readshared]; + if (text == null) return { output: null }; + return { output: new TextEncoder().encode(text) }; + } + return null; + } + }; +} + +const NES_H = `#ifndef NES_H +#define NES_H +/* NES registers */ +#define PPU_CTRL 0x2000 +#define PPU_MASK 0x2001 +/* Turn off the screen */ +#define VBLANK 0x00 +#define PPU_STATUS 0x2002 +static char PPU_READ(unsigned char a) { + return a; +} +#endif +`; + +const APPLE_H = `#ifndef APPLE_H +#define APPLE_H +/* Apple II registers */ +#define KBD 0xC000 +/* Read keyboard */ +#define KBD_READ 0xC000 +#endif +`; + +describe('ToolchainSource', function () { + after(function () { + setProjectProvider(() => null); + }); + + it('should list and lazily search bundled header files', async function () { + setProjectProvider(() => makeFs({ '/include/nes.h': NES_H, '/include/apple.h': APPLE_H })); + const ts = new ToolchainSource(); + + // First query triggers ready() (lists files) + ensureBatch() (reads them) + const hits = await ts.query('VBLANK', 10); + + assert.ok(hits.length >= 1, 'should find VBLANK in nes.h'); + const rec = hits[0].record; + assert.strictEqual(rec.kind, 'text'); + assert.strictEqual(rec.source, 'toolchain'); + assert.strictEqual(rec.file, '/include/nes.h'); + assert.ok(rec.line && rec.line > 0); + // The two hash-flavored hits should both appear (each file is searched) + assert.ok(hits.some(h => h.record.file === '/include/nes.h' && h.record.name.includes('VBLANK'))); + }); + + it('should be case-insensitive', async function () { + setProjectProvider(() => makeFs({ '/include/nes.h': NES_H })); + const ts = new ToolchainSource(); + const hits = await ts.query('vblank', 10); + assert.ok(hits.length >= 1); + }); + + it('should require at least 2 chars', async function () { + setProjectProvider(() => makeFs({ '/include/nes.h': NES_H })); + const ts = new ToolchainSource(); + const hits = await ts.query('x', 10); + assert.strictEqual(hits.length, 0); + }); + + it('should load files in batches across queries', async function () { + const fs = makeFs({ '/include/nes.h': NES_H, '/include/apple.h': APPLE_H }); + setProjectProvider(() => fs); + const ts = new ToolchainSource(); + + // Shrink batch so a query only loads a subset + (ts as any).files = ['/include/nes.h', '/include/apple.h']; + (ts as any).loadIdx = 0; + (ts as any).fsName = '65-nes'; + (ts as any).readyDone = true; + (ts as any).batchSize = 1; + + // First query: only nes.h loaded + let hits = await ts.query('VBLANK', 10); + assert.ok(hits.length >= 1, 'nes.h loaded in first batch'); + assert.strictEqual(ts['loaded'].size, 1); + + // Second query: apple.h loaded too + hits = await ts.query('VBLANK', 10); + assert.strictEqual(ts['loaded'].size, 2); + }); + + it('should rank matches near the start of the line higher', async function () { + setProjectProvider(() => makeFs({ '/include/nes.h': NES_H })); + const ts = new ToolchainSource(); + const hits = await ts.query('PPU', 10); + assert.ok(hits.length >= 2, 'PPU matches several lines'); + // Earliest-column hit first (PPU_CTRL at col 17 is #define PPU_CTRL); the + // column-based score should order them without any flakiness + const cols = hits.map(h => h.ranges ? h.ranges[0] : 0); + for (let i = 1; i < cols.length; i++) { + assert.ok(cols[i - 1] <= cols[i], 'hits sorted by column'); + } + }); + + it('should skip non-searchable file types', async function () { + const ts = new ToolchainSource(); + assert.strictEqual((ts as any).isSearchable('/include/nes.h'), true); + assert.strictEqual((ts as any).isSearchable('/include/nes.o'), false); + assert.strictEqual((ts as any).isSearchable('/lib/binary.bin'), false); + assert.strictEqual((ts as any).isSearchable('/scripts/makefile'), true); // no ext + }); + + it('should handle missing worker gracefully', async function () { + setProjectProvider(() => ({ + mainPath: 'main.c', + iterateFiles: (cb: any) => {}, + })); + const ts = new ToolchainSource(); + const hits = await ts.query('anything', 10); + assert.deepStrictEqual(hits, []); + }); +}); \ No newline at end of file