diff --git a/package-lock.json b/package-lock.json index a803e511..f8587a3f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@codemirror/lang-cpp": "^6.0.3", "@codemirror/lang-markdown": "^6.5.0", "@codemirror/view": "^6.39.14", + "@leeoniya/ufuzzy": "^1.0.19", "@lezer/lr": "^1.4.8", "@types/dompurify": "^3.0.5", "@types/emscripten": "^1.39.5", @@ -26,12 +27,13 @@ "jquery": "^3.6.3", "jszip": "^3.7.0", "localforage": "^1.9.0", + "minisearch": "^7.2.0", "mousetrap": "^1.6.5", "octokat": "^0.10.0", "split.js": "^1.6.2" }, "bin": { - "8bws": "gen/cli/8bws.js" + "8bws": "gen/tools/8bws.js" }, "devDependencies": { "@lezer/generator": "^1.8.0", @@ -1078,6 +1080,12 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@leeoniya/ufuzzy": { + "version": "1.0.19", + "resolved": "https://registry.npmjs.org/@leeoniya/ufuzzy/-/ufuzzy-1.0.19.tgz", + "integrity": "sha512-0pikDeYt0IHEUPza5RTCDXc/17S1pTrYnReEMp8Aa6k1ovzw5QdZLwicW8TjljwEZRb6oYag0xmALohrcq/yOQ==", + "license": "MIT" + }, "node_modules/@lezer/common": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.5.2.tgz", @@ -6007,6 +6015,12 @@ "license": "MIT", "optional": true }, + "node_modules/minisearch": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/minisearch/-/minisearch-7.2.0.tgz", + "integrity": "sha512-dqT2XBYUOZOiC5t2HRnwADjhNS2cecp9u+TJRiJ1Qp/f5qjkeT5APcGPjHw+bz89Ms8Jp+cG4AlE+QZ/QnDglg==", + "license": "MIT" + }, "node_modules/mkdirp": { "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", diff --git a/package.json b/package.json index aa26bc25..15e01e73 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "@codemirror/lang-cpp": "^6.0.3", "@codemirror/lang-markdown": "^6.5.0", "@codemirror/view": "^6.39.14", + "@leeoniya/ufuzzy": "^1.0.19", "@lezer/lr": "^1.4.8", "@types/dompurify": "^3.0.5", "@types/emscripten": "^1.39.5", @@ -30,6 +31,7 @@ "jquery": "^3.6.3", "jszip": "^3.7.0", "localforage": "^1.9.0", + "minisearch": "^7.2.0", "mousetrap": "^1.6.5", "octokat": "^0.10.0", "split.js": "^1.6.2" diff --git a/src/common/searchtypes.ts b/src/common/searchtypes.ts new file mode 100644 index 00000000..780bf566 --- /dev/null +++ b/src/common/searchtypes.ts @@ -0,0 +1,40 @@ +export type SymbolKind = + | 'func' + | 'macro' + | 'type' + | 'struct' + | 'enum' + | 'var' + | 'label' + | 'equate' + | 'proc' + | 'module' + | 'doc' + | 'text'; + +export type SymbolSourceKind = 'project' | 'toolchain' | 'docs'; + +export interface SymbolRecord { + id: string; // stable, unique within a corpus: \`${file}:${line}:${name}\` + name: string; // the identifier, or page title for docs + kind: SymbolKind; + signature?: string; // rendered declaration, verbatim source text + brief?: string; // first sentence of the doc comment + detail?: string; // remainder of the doc comment + section?: string; // enclosing banner-comment heading, e.g. "Declarations" + source: SymbolSourceKind; + file: string; // project path | '/include/joystick.h' | page URL + line?: number; // 1-based + url?: string; // docs only +} + +/** On-disk shape of a prebuilt index (gen/symidx/.json). */ +export interface PrebuiltIndex { + version: number; // bump when the extractor changes; runtime rejects mismatches + corpus: string; // fs name, e.g. 'fs65-c64', or 'docs' + generated: string; // ISO date + records: SymbolRecord[]; + ms?: object; // MiniSearch.toJSON() output, loaded via loadJSAsync +} + +export const SYMIDX_VERSION = 1; diff --git a/src/common/symbols/doccomment.ts b/src/common/symbols/doccomment.ts new file mode 100644 index 00000000..271a070c --- /dev/null +++ b/src/common/symbols/doccomment.ts @@ -0,0 +1,151 @@ +/** + * Doc comment extraction shared across all tiers. + * Extracts brief/detail/section from C-style comments. + */ + +export interface DocInfo { + brief?: string; + detail?: string; + section?: string; +} + +/** Check if a comment line is a banner (decoration) */ +function isBannerLine(line: string): boolean { + const stripped = line.trim(); + if (stripped.length === 0) return false; + const nonBanner = stripped.replace(/[*\-=\/#]/g, ''); + return nonBanner.length === 0 || nonBanner.length / stripped.length < 0.2; +} + +/** Extract section from a banner comment line */ +function extractBannerSection(line: string): string | null { + let stripped = line.trim(); + stripped = stripped.replace(/^\s*\/\*\*?/,'').replace(/\*\/$/,'').replace(/^#\s?/,'').trim(); + if (/^[A-Z][A-Z0-9 _]*$/.test(stripped) && stripped.length <= 40) { + return stripped; + } + return null; +} + +/** Extract doc info from lines above position `lineNo` (1-based). */ +export function extractDocComment( + lines: string[], + lineNo: number +): DocInfo | null { + const docLines: string[] = []; + let section: string | null = null; + let i = lineNo - 2; + + // Skip blank lines immediately above + while (i >= 0 && lines[i].trim() === '') { + i--; + } + if (i < 0) return null; + + // Check for doc block (/** ... */) + const docBlockRegex = /^\s*\/\*\*(.*)\*\/\s*$/; + const docBlockMatch = lines[i].match(docBlockRegex); + + if (docBlockMatch) { + docLines.push(docBlockMatch[1].trim()); + i--; + while (i >= 0) { + const m = lines[i].match(docBlockRegex); + if (m) { + docLines.unshift(m[1].trim()); + i--; + } else { + break; + } + } + } else { + // Collect plain comments above (no blank line between) + let hasBlank = false; + while (i >= 0 && !hasBlank) { + const line = lines[i].trim(); + if (line === '') { + hasBlank = true; + break; + } + if (isBannerLine(line)) { + const bannerSection = extractBannerSection(line); + if (bannerSection) { + section = bannerSection; + } + i--; + continue; + } + if (line.startsWith('//') || line.startsWith('/*')) { + docLines.unshift(line); + i--; + } else { + break; + } + } + if (docLines.length === 0) return null; + } + + return parseDocLines(docLines, section); +} + +/** Parse collected comment lines into structured doc info */ +function parseDocLines(lines: string[], sectionOverride?: string | null): DocInfo | null { + const cleaned: string[] = []; + for (let line of lines) { + // Strip leading // + line = line.replace(/^\s*\/\/\s*/, '').trim(); + // Strip leading /* */ style + line = line.replace(/^\s*\/\*/, '').replace(/\*\/\s*$/, '').trim(); + // Strip leading * (common in block comments) + line = line.replace(/^\s*\*\s*/, '').trim(); + // Strip ;; decoration + line = line.replace(/^;;\s/, '').trim(); + // Strip leading # decoration + line = line.replace(/^#\s/, '').trim(); + // Collapse multiple spaces + line = line.replace(/\s+/g, ' ').trim(); + if (line.length > 0) { + cleaned.push(line); + } + } + + if (cleaned.length === 0) return null; + + let fullText = cleaned.join(' '); + let brief: string | undefined; + let detail: string | undefined; + + const periodSpaceIdx = fullText.indexOf('. '); + if (periodSpaceIdx >= 0) { + brief = fullText.substring(0, periodSpaceIdx + 1).trim(); + detail = fullText.substring(periodSpaceIdx + 2).trim(); + } else { + brief = fullText; + } + + const result: DocInfo = { + brief: brief && brief.length > 0 ? brief : undefined, + detail: detail && detail.length > 0 ? detail : undefined, + section: sectionOverride || undefined + }; + + if (result.detail && result.brief && result.brief.endsWith('.')) { + result.brief = result.brief.slice(0, -1); + } + + if (!result.brief && !result.detail) return null; + return result; +} + +/** Extract doc info from a trailing comment on the same line */ +export function extractTrailingComment(line: string): string | undefined { + const trailingSlash = line.match(/\/\/\s*(.*)$/); + if (trailingSlash && trailingSlash[1]) { + return trailingSlash[1].trim(); + } + const trailingBlock = line.match(/\/\*\s*(.*?)\*\/\s*$/); + if (trailingBlock && trailingBlock[1]) { + return trailingBlock[1].trim(); + } + return undefined; +} diff --git a/src/common/symbols/index.ts b/src/common/symbols/index.ts new file mode 100644 index 00000000..9726adcf --- /dev/null +++ b/src/common/symbols/index.ts @@ -0,0 +1,102 @@ +export type { SymbolRecord, SymbolKind, SymbolSourceKind, PrebuiltIndex, SYMIDX_VERSION } from "../searchtypes"; +export { extractDocComment, extractTrailingComment, type DocInfo } from "./doccomment"; + +import { SymbolRecord, SymbolSourceKind } from "../searchtypes"; +import { extractDocComment, extractTrailingComment } from "./doccomment"; + +export function extractSymbols( + text: string, + filename: string, + editorStyle: string, + source: SymbolSourceKind +): SymbolRecord[] { + // Basic fallback extraction using regex patterns + const records: SymbolRecord[] = []; + const lines = text.split('\n'); + + // Simple patterns for common languages + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const lineNo = i + 1; + + // Function/method definitions + const funcMatch = line.match(/(?:function|func|def|proc)\s+(\w+)/i); + if (funcMatch) { + records.push({ + id: `${filename}:${lineNo}:${funcMatch[1]}`, + name: funcMatch[1], + kind: 'func', + source, + file: filename, + line: lineNo + }); + } + + // Variable/const declarations + const varMatch = line.match(/(?:var|let|const|global)\s+(\w+)/i); + if (varMatch) { + records.push({ + id: `${filename}:${lineNo}:${varMatch[1]}`, + name: varMatch[1], + kind: 'var', + source, + file: filename, + line: lineNo + }); + } + + // Type definitions + const typeMatch = line.match(/(?:struct|class|interface|type)\s+(\w+)/i); + if (typeMatch) { + records.push({ + id: `${filename}:${lineNo}:${typeMatch[1]}`, + name: typeMatch[1], + kind: 'type', + source, + file: filename, + line: lineNo + }); + } + + // Labels (assembly) + const labelMatch = line.match(/^\s*(\w+)\s*:/); + if (labelMatch && !line.trim().startsWith('//')) { + records.push({ + id: `${filename}:${lineNo}:${labelMatch[1]}`, + name: labelMatch[1], + kind: 'label', + source, + file: filename, + line: lineNo + }); + } + + // Macros + const macroMatch = line.match(/#define\s+(\w+)/i); + if (macroMatch) { + records.push({ + id: `${filename}:${lineNo}:${macroMatch[1]}`, + name: macroMatch[1], + kind: 'macro', + source, + file: filename, + line: lineNo + }); + } + + // Equates + const equateMatch = line.match(/^\s*(\w+)\s*(?:=|equ)\s+/i); + if (equateMatch) { + records.push({ + id: `${filename}:${lineNo}:${equateMatch[1]}`, + name: equateMatch[1], + kind: 'equate', + source, + file: filename, + line: lineNo + }); + } + } + + return records; +} \ No newline at end of file diff --git a/src/ide/search/docssource.ts b/src/ide/search/docssource.ts new file mode 100644 index 00000000..68d7faf3 --- /dev/null +++ b/src/ide/search/docssource.ts @@ -0,0 +1,45 @@ +import { SymbolIndex } from "./symbolindex"; +import { SymbolSourceKind, PrebuiltIndex, SYMIDX_VERSION } from "../../common/searchtypes"; +import { SearchHit, SearchSource } from "./types"; + +export class DocsSource implements SearchSource { + id: string = 'docs'; + kind: SymbolSourceKind = 'docs'; + private index: SymbolIndex | null = null; + private readyPromise: Promise | null = null; + + constructor() {} + + async ready() { + if (this.readyPromise) return this.readyPromise; + this.readyPromise = this.load(); + return this.readyPromise; + } + + private async load() { + try { + const response = await fetch('https://8bitworkshop.com/docs/searchindex.json'); + if (!response.ok) { + return; + } + const json = await response.json(); + if (json.version !== SYMIDX_VERSION) { + return; + } + const indexData = json as PrebuiltIndex; + const idx = new SymbolIndex(indexData.records); + await idx.init(); + if (indexData.ms) { + await idx.setMiniSearch(indexData.ms); + } + this.index = idx; + } catch (e) { + console.debug('Docs search index unavailable'); + } + } + + query(needle: string, limit: number): SearchHit[] { + if (!this.index) return []; + return this.index.query(needle, limit); + } +} \ No newline at end of file diff --git a/src/ide/search/projectsource.ts b/src/ide/search/projectsource.ts new file mode 100644 index 00000000..fc5f603a --- /dev/null +++ b/src/ide/search/projectsource.ts @@ -0,0 +1,158 @@ +import { SymbolIndex } from "./symbolindex"; +import { SymbolRecord, SymbolSourceKind } from "../../common/searchtypes"; +import { SearchHit, SearchSource } from "./types"; +import { extractSymbols } from "../../common/symbols"; + +// Avoid a hard dependency on ../ui (which pulls in the whole IDE and its +// language grammar imports that crash under plain Node). ui.ts registers a +// getter here during startup instead. +type ProjectProvider = () => { mainPath: string; iterateFiles(cb: (path: string, data: any) => void): void } | null; +let projectProvider: ProjectProvider = () => null; + +export function setProjectProvider(fn: ProjectProvider) { + projectProvider = fn; +} + +function getProject() { + return projectProvider(); +} + +export class ProjectSource implements SearchSource { + id: string = 'project'; + kind: SymbolSourceKind = 'project'; + private index: SymbolIndex; + private rebuilt: boolean = false; + + constructor() { + this.index = new SymbolIndex([]); + } + + async ready() { + await this.index.init(); + // Lazily index the current project files on first query. + if (!this.rebuilt) { + this.rebuilt = true; + await this.rebuild(); + } + } + + query(needle: string, limit: number): SearchHit[] { + // Symbol/name matches first (high priority), then full-text line matches + // at low priority so code text (not just identifiers) is searchable. + const symbolHits = this.index.query(needle, limit); + const textHits = this.fulltextQuery(needle, limit); + + const hits = symbolHits.concat(textHits); + hits.sort((a, b) => b.score - a.score); + return hits.slice(0, limit); + } + + /** + * Low-priority full-text search over the raw project source text. + * Rather than a real index, we scan current_project.filedata directly + * (project files are small and few); results rank below any symbol hit. + */ + private fulltextQuery(needle: string, limit: number): SearchHit[] { + if (needle.length < 2) return []; + + const needleLower = needle.toLowerCase(); + const hits: SearchHit[] = []; + const MAX_PER_FILE = Math.max(limit, 10); + + const project = getProject(); + if (!project) return []; + + project.iterateFiles((path, data) => { + if (typeof data !== 'string') return; + const lines = data.split('\n'); + let fileCount = 0; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const lineLower = line.toLowerCase(); + const col = lineLower.indexOf(needleLower); + if (col >= 0) { + const trimmed = line.trim(); + if (!trimmed) continue; + hits.push({ + record: { + id: `text:${path}:${i + 1}`, + name: trimmed.length > 80 ? trimmed.slice(0, 80) + '…' : trimmed, + kind: 'text', + source: 'project', + file: path, + line: i + 1, + }, + // Well below symbol matches (BASE_MATCH_SCORE ~1000) but above 0. + // Slight bonus for a match near the start of the line. + score: 200 + Math.max(0, 100 - col * 2), + ranges: [col, col + needle.length], + }); + if (++fileCount >= MAX_PER_FILE) break; + } + } + }); + + return hits; + } + + async onFileChanged(path: string, data: Uint8Array | string) { + const records = this.extractSymbolsFromFile(path, data, 'project'); + this.index.pushRecords(records); + } + + private extractSymbolsFromFile(path: string, data: Uint8Array | string, source: SymbolSourceKind): SymbolRecord[] { + if (typeof data !== 'string') return []; + const editorStyle = this.detectEditorStyle(path); + return extractSymbols(data, path, editorStyle, source); + } + + private detectEditorStyle(path: string): string { + const ext = path.split('.').pop()?.toLowerCase(); + switch (ext) { + case 'c': + case 'h': + return 'text/x-csrc'; + case 'asm': + case 'a65': + case 'inc': + return '6502'; + case 'v': + return 'verilog'; + case 'wiz': + return 'text/x-wiz'; + case 'bas': + case 'basic': + return 'basic'; + case 'fb': + case 'fastbasic': + return 'fastbasic'; + case 'bac': + case 'bataribasic': + return 'bataribasic'; + case 'i': + return 'inform6'; + case 'd': + return 'dialog'; + default: + return 'text/x-csrc'; + } + } + + async rebuild() { + this.index.clear(); + await this.index.init(); + + const project = getProject(); + if (!project) return; + + const records: SymbolRecord[] = []; + project.iterateFiles((path, data) => { + if (typeof data !== 'string') return; + const editorStyle = this.detectEditorStyle(path); + const fileRecords = extractSymbols(data, path, editorStyle, 'project'); + records.push(...fileRecords); + }); + + this.index.pushRecords(records); + } +} \ No newline at end of file diff --git a/src/ide/search/searchmodal.html b/src/ide/search/searchmodal.html new file mode 100644 index 00000000..14ec9c05 --- /dev/null +++ b/src/ide/search/searchmodal.html @@ -0,0 +1,19 @@ + diff --git a/src/ide/search/searchview.ts b/src/ide/search/searchview.ts new file mode 100644 index 00000000..965b8ee3 --- /dev/null +++ b/src/ide/search/searchview.ts @@ -0,0 +1,143 @@ +/** + * Search View - bootbox-based palette for searching project files, + * toolchain headers, and docs. Opens an overlay dialog with a live + * input; results are clickable and open the associated file/header. + */ + +import { searchService } from "./service"; +import { SearchHit } from "./types"; +import { projectWindows, openHeaderFile } from "../ui"; +import { SourceEditor, HeaderView } from "../views/editors"; + +declare var $: JQueryStatic; +declare var bootbox: any; + +/** Debounce helper */ +function debounce(fn: () => void, ms: number) { + let timer: any = null; + return () => { + if (timer) clearTimeout(timer); + timer = setTimeout(fn, ms); + }; +} + +/** Max results shown in the palette */ +const MAX_RESULTS = 30; + +function escapeHtml(s: string): string { + return String(s).replace(/[&<>"']/g, (c) => + ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c] as string) + ); +} + +/** Open the file/header/doc associated with a search hit. */ +export function openSearchHit(hit: SearchHit) { + const rec = hit.record; + + if (rec.source === 'project') { + // Open a project file at the matching line + const path = rec.file; + if (!path || !projectWindows.isWindow(path)) return; + const wnd = projectWindows.createOrShow(path); + if (wnd instanceof SourceEditor && rec.line) { + wnd.navigateToLine(rec.line); + } + } else if (rec.source === 'toolchain') { + // Open a shared header file + const fn = rec.file; + if (!fn) return; + openHeaderFile(fn); + const wnd = projectWindows.createOrShow('#headerview/' + fn); + if (wnd instanceof HeaderView && rec.line) { + wnd.navigateToLine(rec.line); + } + } else if (rec.source === 'docs') { + // External docs: open the URL in a new tab + const url = rec.detail || rec.file || ''; + if (url) { + window.open(url, '_blank'); + } + } +} + +export function openSearchDialog() { + const modal = bootbox.dialog({ + title: 'Search', + onEscape: true, + message: ` +
+ +
+
+ `, + buttons: { + close: { + label: "Close", + className: "btn-default" + } + } + }); + + // bootbox auto-focuses the first accept button on shown; bind after it so + // our focus wins (jQuery calls handlers in binding order). + modal.one('shown.bs.modal', () => { + $('#searchboxInput').trigger('focus'); + }); + + const input = $('#searchboxInput'); + + const resultsDiv = $('#searchboxResults'); + + const render = (hits: SearchHit[]) => { + if (!hits || hits.length === 0) { + resultsDiv.html('
No matches.
'); + return; + } + const ul = $(''); + for (const hit of hits) { + const rec = hit.record; + const srcName = rec.source; + const kindIcon = rec.kind === 'func' ? 'ƒ' : + rec.kind === 'macro' ? '#' : + rec.kind === 'label' ? ':' : + rec.kind === 'struct' ? 'S' : + rec.kind === 'enum' ? 'E' : + rec.kind === 'text' ? '≡' : '•'; + const loc = rec.file && rec.line ? `${rec.file}:${rec.line}` : (rec.file || ''); + const li = $('
  • '); + li.html( + `${kindIcon}` + + `${escapeHtml(rec.name)}` + + (rec.brief ? ` - ${escapeHtml(rec.brief)}` : '') + + `${escapeHtml(srcName)}${loc ? ' · ' + escapeHtml(loc) : ''}` + ); + li.click(() => { + openSearchHit(hit); + bootbox.hideAll(); + }); + ul.append(li); + } + resultsDiv.empty().append(ul); + }; + + const doSearch = debounce(async () => { + const needle = input.val() as string; + if (!needle) { + resultsDiv.empty(); + return; + } + try { + const hits = await searchService.query(needle, MAX_RESULTS); + render(hits); + } catch (e) { + resultsDiv.html('
    Search failed.
    '); + } + }, 150); + + input.on('input', doSearch); + input.on('keydown', (e) => { + if (e.key === 'Enter') doSearch(); + }); +} \ No newline at end of file diff --git a/src/ide/search/service.ts b/src/ide/search/service.ts new file mode 100644 index 00000000..b9fad1db --- /dev/null +++ b/src/ide/search/service.ts @@ -0,0 +1,114 @@ +/** + * Search Service - combines all search sources (project, toolchain, docs). + * Provides unified search with query prefix support. + */ + +import { SearchHit, SearchSource } from "./types"; +import { SymbolSourceKind, SymbolRecord } from "../../common/searchtypes"; +import { ProjectSource } from "./projectsource"; +import { ToolchainSource } from "./toolchainsource"; +import { DocsSource } from "./docssource"; + +const MAX_RESULTS = 50; + +export class SearchService { + private static instance: SearchService; + private sources: Map = new Map(); + private projectSource!: ProjectSource; + private toolchainSource!: ToolchainSource; + private docsSource!: DocsSource; + + private constructor() { + this.projectSource = new ProjectSource(); + this.toolchainSource = new ToolchainSource(); + this.docsSource = new DocsSource(); + + this.sources.set('project', this.projectSource); + this.sources.set('toolchain', this.toolchainSource); + this.sources.set('docs', this.docsSource); + } + + static getInstance(): SearchService { + if (!SearchService.instance) { + SearchService.instance = new SearchService(); + } + return SearchService.instance; + } + + /** Parse a query string and extract prefix modifiers */ + private parseQuery(query: string): { needle: string; filter: 'all' | 'symbols' | 'files' | 'docs' } { + if (query.startsWith('#')) { + return { needle: query.slice(1).trim(), filter: 'symbols' }; + } else if (query.startsWith('>')) { + return { needle: query.slice(1).trim(), filter: 'files' }; + } else if (query.startsWith('?')) { + return { needle: query.slice(1).trim(), filter: 'docs' }; + } + return { needle: query, filter: 'all' }; + } + + /** Execute a search query across all enabled sources */ + async query(queryStr: string, limit: number = MAX_RESULTS): Promise { + const { needle, filter } = this.parseQuery(queryStr); + + if (!needle) return []; + + const results: SearchHit[] = []; + + // Determine which sources to query + const sourcesToQuery: SearchSource[] = []; + if (filter === 'docs') { + sourcesToQuery.push(this.docsSource); + } else if (filter === 'files') { + sourcesToQuery.push(this.projectSource); + } else if (filter === 'symbols') { + sourcesToQuery.push(this.projectSource); + sourcesToQuery.push(this.toolchainSource); + } else { + sourcesToQuery.push(this.projectSource); + sourcesToQuery.push(this.toolchainSource); + sourcesToQuery.push(this.docsSource); + } + + // Query all sources + for (const source of sourcesToQuery) { + try { + await source.ready(); + const hits = await source.query(needle, limit - results.length); + results.push(...hits); + } catch (e) { + console.debug('Search source error:', e); + } + } + + // Sort by score (already scored by source) + results.sort((a, b) => b.score - a.score); + + // Apply source weights: project > toolchain > docs + const sourceWeights: Record = { + project: 1.3, + toolchain: 1.0, + docs: 0.7 + }; + + for (const hit of results) { + const weight = sourceWeights[hit.record.source as string] || 1.0; + hit.score *= weight; + } + + results.sort((a, b) => b.score - a.score); + + // Dedupe by ID + const seen = new Set(); + const deduped = results.filter(h => { + if (seen.has(h.record.id)) return false; + seen.add(h.record.id); + return true; + }); + + return deduped.slice(0, limit); + } +} + +// Export singleton +export const searchService = SearchService.getInstance(); \ No newline at end of file diff --git a/src/ide/search/symbolindex.ts b/src/ide/search/symbolindex.ts new file mode 100644 index 00000000..f38c7feb --- /dev/null +++ b/src/ide/search/symbolindex.ts @@ -0,0 +1,248 @@ +/** + * SymbolIndex - shared machinery for searching a SymbolRecord[] corpus. + * Uses uFuzzy for identifier lookup and MiniSearch for prose. + */ + +import { SymbolRecord, SymbolKind, SymbolSourceKind } from "../../common/searchtypes"; +import { SearchHit } from "./types"; + +const SOURCE_WEIGHTS: Record = { + project: 1.3, + toolchain: 1.0, + docs: 0.7, +}; + +// Declarations (real code symbols) rank above docs; text hits rank lowest. +const KIND_WEIGHTS: Record = { + func: 1.2, + macro: 1.2, + type: 1.2, + struct: 1.2, + enum: 1.2, + var: 1.1, + label: 1.1, + equate: 1.1, + proc: 1.2, + module: 1.2, + doc: 0.8, + text: 0.3, +}; + +const PROSE_THRESHOLD = 4; +// Base score for a uFuzzy match before quality/weight adjustments. +// Domain: ~0-1000. Exact matches ~1000, fuzzy/degraded matches lower. +const BASE_MATCH_SCORE = 1000; + +/** + * Helper to resolve a CommonJS module that may be exported as + * `module.exports = Class` (uFuzzy, MiniSearch) into the class itself. + * Works for both `import()` (bundled) and `require()` (tsc CJS output). + */ +function resolveClass(mod: any): any { + if (!mod) return undefined; + // Direct class export (module.exports = Class): static props only + if (typeof mod === 'function') return mod; + // ESM namespace: { default: Class } + if (typeof mod.default === 'function') return mod.default; + // Named export + if (typeof mod.MiniSearch === 'function') return mod.MiniSearch; + if (typeof mod.uFuzzy === 'function') return mod.uFuzzy; + return undefined; +} + +export class SymbolIndex { + private records: SymbolRecord[] = []; + private nameHaystack: string[] = []; + private msIds: Set = new Set(); + private uf: any; + private ms: any; + private applySourceBoosts: boolean; + private ufOptions: any; + private msOptions: any; + + constructor(records: SymbolRecord[] = []) { + this.records = records; + this.nameHaystack = records.map(r => r.name); + this.applySourceBoosts = true; + this.ufOptions = { + // Allow intra-word insertions so typos like "joy_rd" match "joy_read" + intraIns: 2, + interIns: 2, + }; + this.msOptions = { + idField: 'id', + fields: ['name', 'brief', 'detail'], + storeFields: ['id', 'name', 'kind', 'brief', 'detail', 'source', 'file', 'line'], + }; + } + + /** Initialize indexers (uFuzzy + MiniSearch) */ + async init() { + const ufMod = await import("@leeoniya/ufuzzy"); + const Ufuzzy = resolveClass(ufMod); + if (Ufuzzy) { + this.uf = new Ufuzzy(this.ufOptions); + } + + const msMod = await import("minisearch"); + const MiniSearch = resolveClass(msMod); + if (MiniSearch) { + this.ms = new MiniSearch(this.msOptions); + // Re-add any records pushed before init(). + for (const rec of this.records) { + this.addToMiniSearch(rec); + } + } + } + + /** Add one record to MiniSearch, skipping duplicates by id. */ + private addToMiniSearch(rec: SymbolRecord) { + if (!this.ms || typeof this.ms.add !== 'function') return; + if (this.msIds.has(rec.id)) return; + try { + this.ms.add(rec); + this.msIds.add(rec.id); + } catch (e) { + console.debug('MiniSearch add error:', e); + } + } + + /** Add new records to the index */ + pushRecords(records: SymbolRecord[]) { + this.records.push(...records); + this.nameHaystack.push(...records.map(r => r.name)); + // Add to MiniSearch if initialized + for (const rec of records) { + this.addToMiniSearch(rec); + } + } + + /** Clear all records */ + clear() { + this.records = []; + this.nameHaystack = []; + this.msIds.clear(); + if (this.ms && typeof this.ms.removeAll === 'function') { + this.ms.removeAll(); + } + } + + /** Load MiniSearch index from serialized JSON */ + async setMiniSearch(msJSON: any) { + const msMod = await import("minisearch"); + const MiniSearch = resolveClass(msMod); + if (!MiniSearch) return; + this.ms = new MiniSearch(this.msOptions); + this.ms.loadJS(msJSON); + } + + query(needle: string, limit: number): SearchHit[] { + if (this.records.length === 0 || !needle) return []; + + const hits: SearchHit[] = []; + + // uFuzzy name search (priority) + if (this.uf && typeof this.uf.search === 'function') { + try { + const result = this.uf.search(this.nameHaystack, needle); + const info = result?.[1]; + const order = result?.[2]; + const srcIdxMap = info?.idx; + if (srcIdxMap && order) { + for (let i = 0; i < order.length; i++) { + const ufuzzyOrderIdx = order[i]; + const srcIdx = srcIdxMap[ufuzzyOrderIdx]; + const record = this.records[srcIdx]; + if (!record) continue; + // Score based on uFuzzy match quality, then source weight + const base = this.baseScore(needle, info, ufuzzyOrderIdx); + const score = base * SOURCE_WEIGHTS[record.source] * (KIND_WEIGHTS[record.kind] ?? 1); + hits.push({ record, score, ranges: info.ranges?.[ufuzzyOrderIdx] }); + } + } + } catch (e) { + console.debug('uFuzzy search error:', e); + } + } + + // MiniSearch prose search (if needle is long enough or has spaces) + if (this.ms && typeof this.ms.search === 'function') { + if (needle.length >= PROSE_THRESHOLD || needle.includes(' ')) { + try { + const msResults = this.ms.search(needle, { + boost: { name: 3, brief: 1.5, detail: 1 }, + combineWith: 'AND', + }); + for (const r of msResults) { + const record = r; + if (record && !hits.some(h => h.record.id === record.id)) { + hits.push({ + record, + score: (r.score || 0) * SOURCE_WEIGHTS[record.source] * KIND_WEIGHTS[r.kind] * 1000 + }); + } + } + } catch (e) { + console.debug('MiniSearch error:', e); + } + } + } + + // Sort by score descending + hits.sort((a, b) => b.score - a.score); + + // Dedupe by ID + const seen = new Set(); + const deduped = hits.filter(h => { + if (seen.has(h.record.id)) return false; + seen.add(h.record.id); + return true; + }); + + return deduped.slice(0, limit); + } + + /** + * Compute a match-quality score from uFuzzy info. + * uFuzzy's `order` already sorts by quality (chars, intraIns, terms+prefix, + * interIns, start, cases); we enrich it with source-weight and an absolute + * scale so MiniSearch prose scores can be compared on the same axis. + */ + private baseScore(needle: string, info: any, i: number): number { + const chars = info.chars?.[i] ?? 0; + const interIns = info.interIns?.[i] ?? 0; // insertions between search terms + const intraIns = info.intraIns?.[i] ?? 0; // insertions within terms (typos) + const start = info.start?.[i] ?? 0; // offset of first match char + const interLft2 = info.interLft2?.[i] ?? 0; + const interRgt2 = info.interRgt2?.[i] ?? 0; + + // chars >= needle length = full needle matched + const fullMatch = chars >= needle.length; + // No intra-word insertions and no inter-term insertions = contiguous + const contiguous = intraIns === 0 && interIns === 0; + // Match starts at position 0 = prefix match (strong signal for identifiers) + const prefixMatch = start === 0; + + let score = BASE_MATCH_SCORE; + + // Exact match: full needle, contiguous, prefix + if (fullMatch && contiguous && prefixMatch) score = 1000; + // Prefix match that's fully contiguous but missing chars (shorter needle) + else if (contiguous && prefixMatch) score = 980; + // Full but non-contiguous (spread across name) + else if (fullMatch) score = 950; + // Fuzzy + else score = 900; + + // Each intra-word insertion (typo character) is expensive + score -= intraIns * 20; + // Inter-term insertions (underscores or gaps between terms) + score -= interIns * 10; + // Non-prefix start + score -= start * 5; + // Long tail to the right of the match means the match is not the whole name + score -= interRgt2 * 3; + + return score; + } +} \ No newline at end of file diff --git a/src/ide/search/toolchainsource.ts b/src/ide/search/toolchainsource.ts new file mode 100644 index 00000000..2a0f4852 --- /dev/null +++ b/src/ide/search/toolchainsource.ts @@ -0,0 +1,68 @@ +import { SymbolIndex } from "./symbolindex"; +import { SymbolSourceKind, PrebuiltIndex, SYMIDX_VERSION } from "../../common/searchtypes"; +import { SearchHit, SearchSource } from "./types"; +import { current_project } from "../ui"; +import { getSharedFileSystemName } from "../../common/toolmeta"; + +export class ToolchainSource implements SearchSource { + id: string = 'toolchain'; + kind: SymbolSourceKind = 'toolchain'; + private indices: Map = new Map(); + private corpusReady: Map = new Map(); + + constructor() {} + + async ready() { + const project = current_project; + if (!project) return; + + const tool = project.getToolForFilename(project.mainPath); + const corpus = getSharedFileSystemName(tool, project.platform_id); + if (corpus) { + await this.loadCorpus(corpus); + } + } + + async loadCorpus(corpus: string) { + if (this.corpusReady.get(corpus)) return; + + try { + const url = `gen/symidx/${corpus}.json`; + const response = await fetch(url); + if (!response.ok) { + this.corpusReady.set(corpus, true); + return; + } + const json = await response.json(); + if (json.version !== SYMIDX_VERSION) { + this.corpusReady.set(corpus, true); + return; + } + const indexData = json as PrebuiltIndex; + const idx = new SymbolIndex(indexData.records); + await idx.init(); + if (indexData.ms) { + await idx.setMiniSearch(indexData.ms); + } + this.indices.set(corpus, idx); + this.corpusReady.set(corpus, true); + } catch (e) { + console.debug('Failed to load toolchain index:', e); + this.corpusReady.set(corpus, true); + } + } + + query(needle: string, limit: number): SearchHit[] { + const hits: SearchHit[] = []; + for (const idx of this.indices.values()) { + const result = idx.query(needle, limit); + hits.push(...result); + } + return hits.slice(0, limit); + } + + invalidate() { + this.indices.clear(); + this.corpusReady.clear(); + } +} \ No newline at end of file diff --git a/src/ide/search/types.ts b/src/ide/search/types.ts new file mode 100644 index 00000000..a91f47f7 --- /dev/null +++ b/src/ide/search/types.ts @@ -0,0 +1,20 @@ +import { SymbolRecord, SymbolSourceKind } from "../../common/searchtypes"; + +export interface SearchHit { + record: SymbolRecord; + score: number; + ranges?: number[]; +} + +export interface SearchSource { + id: string; + kind: SymbolSourceKind; + ready(): Promise; + query(needle: string, limit: number): SearchHit[]; +} + +export interface SearchQuery { + type: 'all' | 'symbols' | 'files' | 'docs'; + file?: string; + needle: string; +} \ No newline at end of file diff --git a/src/ide/ui.ts b/src/ide/ui.ts index 4c7096d1..5f4c7c81 100644 --- a/src/ide/ui.ts +++ b/src/ide/ui.ts @@ -20,6 +20,8 @@ import { getRepos, parseGithubURL } from "./services"; import { _downloadAllFilesZipFile, _downloadCassetteFile, _downloadProjectZipFile, _downloadROMImage, _downloadSourceFile, _downloadSymFile, _getCassetteFunction, _recordVideo, _shareEmbedLink } from "./shareexport"; import { _importProjectFromGithub, _loginToGithub, _logoutOfGithub, _publishProjectToGithub, _pullProjectFromGithub, _pushProjectToGithub, _removeRepository, importProjectFromGithub } from "./sync"; import { Toolbar } from "./toolbar"; +import { openSearchDialog } from "./search/searchview"; +import { setProjectProvider } from "./search/projectsource"; import { AssetEditorView } from "./views/asseteditor"; import { isMobileDevice } from "./views/baseviews"; import { AddressHeatMapView, BinaryFileView, MemoryMapView, MemoryView, ProbeLogView, ProbeSymbolView, RasterStackMapView, ScanlineIOView, VRAMMemoryView } from "./views/debugviews"; @@ -232,6 +234,7 @@ async function newFilesystem() { async function initProject() { var filesystem = await newFilesystem(); current_project = new CodeProject(newWorker(), platform_id, platform, filesystem); + setProjectProvider(() => current_project); current_project.remoteTool = qs.tool || null; projectWindows = new ProjectWindows($("#workspace")[0] as HTMLElement, current_project); current_project.callbackBuildResult = (result: WorkerResult) => { @@ -1597,6 +1600,7 @@ function setupDebugControls() { } uitoolbar.newGroup(); uitoolbar.grp.prop('id', 'xtra_bar'); + uitoolbar.add('shift+ctrl+/', 'Search', 'glyphicon-search', openSearchDialog).prop('id', 'dbg_search'); // add menu clicks $(".dropdown-menu").collapse({ toggle: false }); $("#item_new_file").click(_createNewFile); @@ -1710,6 +1714,7 @@ function openKeyboardShortcuts() { message: ` + ${shortcut(`${mod}+${shift}+F`, 'Search symbols, files, and docs')} ${shortcut(`${shift}+${alt}+F`, 'Format document, or selected range(s)')} ${shortcut('Tab', 'Insert to next tab stop, or indent selected range(s)')} ${shortcut(`${shift}+Tab`, 'Outdent line(s) or selected range(s)')} diff --git a/src/ide/views/editors.ts b/src/ide/views/editors.ts index 532058e5..8952b597 100644 --- a/src/ide/views/editors.ts +++ b/src/ide/views/editors.ts @@ -11,15 +11,7 @@ import { getFilenameForPath, getFolderForPath, hex, rpad } from "../../common/ut import { getIncludeDirs, getIncludePatterns, getLinkPatterns, getSharedFileSystemName, getSystemIncludePatterns } from "../../common/toolmeta"; import { WorkerMessage } from "../../common/workertypes"; import { SourceFile, SourceLocation, WorkerError } from "../../common/workertypes"; -import { asm6502 } from "../../parser/lang-6502"; -import { basic } from "../../parser/lang-basic"; -import { batariBasic } from "../../parser/lang-bataribasic"; -import { dialog } from "../../parser/lang-dialog"; -import { fastBasic } from "../../parser/lang-fastbasic"; -import { inform6 } from "../../parser/lang-inform6"; -import { verilog } from "../../parser/lang-verilog"; -import { wiz } from "../../parser/lang-wiz"; -import { asmZ80 } from "../../parser/lang-z80"; +import { parserRegistry, getLanguageSupportForStyle } from "../../parser/registry"; import { cobalt } from "../../themes/cobalt"; import { disassemblyTheme } from "../../themes/disassemblyTheme"; import { editorTheme } from "../../themes/editorTheme"; @@ -174,43 +166,12 @@ export class SourceEditor implements ProjectView { const minimalGutters = modedef.noGutters || isMobileDevice; var parser: Extension; - switch (this.mode) { - case '6502': - parser = asm6502(); - break; - case 'basic': - parser = basic(); - break; - case 'bataribasic': - parser = batariBasic(); - break; - case 'fastbasic': - parser = fastBasic(); - break; - case 'dialog': - parser = dialog(); - break; - case 'inform6': - parser = inform6(); - break; - case 'markdown': - parser = markdown(); - break; - case 'text/x-csrc': - parser = cpp(); - break; - case 'text/x-wiz': - parser = wiz(); - break; - case 'verilog': - parser = verilog(); - break; - case 'z80': - parser = asmZ80(); - break; - default: - console.warn("Unknown mode: " + this.mode); - break; + const registryEntry = parserRegistry[this.mode]; + if (registryEntry) { + parser = registryEntry.language; + } else { + console.warn("Unknown mode: " + this.mode); + parser = null; } this.editor = new EditorView({ parent: parent, @@ -1070,6 +1031,19 @@ export class HeaderView implements ProjectView { + '// Project include files are loaded during a build -- try building first.\n' + '// Toolchain headers are only available when the tool has a bundled filesystem.'); } + + /** Jump to a line in the header (after content loads). */ + navigateToLine(line: number) { + if (!this.view || line < 1) return; + if (line <= this.view.state.doc.lines) { + const target = this.view.state.doc.line(line); + this.view.dispatch({ + selection: { anchor: target.from }, + effects: [EditorView.scrollIntoView(target.from, { y: "center" })], + }); + this.view.focus(); + } + } // track the most recent request so stale async lookups don't overwrite it private requestedFn: string; } diff --git a/src/parser/lang-basic.ts b/src/parser/lang-basic.ts index ef2ed77c..5e3e6f9f 100644 --- a/src/parser/lang-basic.ts +++ b/src/parser/lang-basic.ts @@ -163,6 +163,8 @@ function tokenString(stream: any, state: BasicState): string { return "string"; } +export const basicStreamParser: StreamParser = basicParser; + export function basic(): LanguageSupport { - return new LanguageSupport(StreamLanguage.define(basicParser)); + return new LanguageSupport(StreamLanguage.define(basicStreamParser)); } \ No newline at end of file diff --git a/src/parser/lang-bataribasic.ts b/src/parser/lang-bataribasic.ts index f05a1d9e..a4fdcda8 100644 --- a/src/parser/lang-bataribasic.ts +++ b/src/parser/lang-bataribasic.ts @@ -94,6 +94,8 @@ function createBatariBasicParser(): StreamParser { /** * Language support for Batari Basic */ +export const batariBasicStreamParser: StreamParser = createBatariBasicParser(); + export function batariBasic(): LanguageSupport { - return new LanguageSupport(StreamLanguage.define(createBatariBasicParser())); + return new LanguageSupport(StreamLanguage.define(batariBasicStreamParser)); } diff --git a/src/parser/lang-dialog.ts b/src/parser/lang-dialog.ts index 4188009e..8d51de93 100644 --- a/src/parser/lang-dialog.ts +++ b/src/parser/lang-dialog.ts @@ -87,6 +87,8 @@ function createDialogParser(): StreamParser { /** * Language support for Dialog */ +export const dialogStreamParser: StreamParser = createDialogParser(); + export function dialog(): LanguageSupport { - return new LanguageSupport(StreamLanguage.define(createDialogParser())); + return new LanguageSupport(StreamLanguage.define(dialogStreamParser)); } diff --git a/src/parser/lang-fastbasic.ts b/src/parser/lang-fastbasic.ts index 87f9aa13..dfc02c40 100644 --- a/src/parser/lang-fastbasic.ts +++ b/src/parser/lang-fastbasic.ts @@ -165,6 +165,8 @@ function createFastBasicParser(): StreamParser { /** * Language support for FastBASIC */ +export const fastBasicStreamParser: StreamParser = createFastBasicParser(); + export function fastBasic(): LanguageSupport { - return new LanguageSupport(StreamLanguage.define(createFastBasicParser())); + return new LanguageSupport(StreamLanguage.define(fastBasicStreamParser)); } diff --git a/src/parser/lang-inform6.ts b/src/parser/lang-inform6.ts index 3b78e264..f1ceb449 100644 --- a/src/parser/lang-inform6.ts +++ b/src/parser/lang-inform6.ts @@ -109,6 +109,8 @@ function createInform6Parser(): StreamParser { /** * Language support for Inform6 */ +export const inform6StreamParser: StreamParser = createInform6Parser(); + export function inform6(): LanguageSupport { - return new LanguageSupport(StreamLanguage.define(createInform6Parser())); + return new LanguageSupport(StreamLanguage.define(inform6StreamParser)); } diff --git a/src/parser/lang-verilog.ts b/src/parser/lang-verilog.ts index b980e4d6..4a31562f 100644 --- a/src/parser/lang-verilog.ts +++ b/src/parser/lang-verilog.ts @@ -460,16 +460,12 @@ function createVerilogParser(config: VerilogConfig = {}): StreamParser = createVerilogParser(); + export function verilog(): LanguageSupport { - return new LanguageSupport(StreamLanguage.define(createVerilogParser())); + return new LanguageSupport(StreamLanguage.define(verilogStreamParser)); } -/** - * Language support for SystemVerilog - */ export function systemverilog(): LanguageSupport { - return new LanguageSupport(StreamLanguage.define(createVerilogParser())); + return new LanguageSupport(StreamLanguage.define(verilogStreamParser)); } diff --git a/src/parser/lang-wiz.ts b/src/parser/lang-wiz.ts index 57a229c2..efcda51e 100644 --- a/src/parser/lang-wiz.ts +++ b/src/parser/lang-wiz.ts @@ -163,6 +163,8 @@ function createWizParser(): StreamParser { /** * Language support for Wiz */ +export const wizStreamParser: StreamParser = createWizParser(); + export function wiz(): LanguageSupport { - return new LanguageSupport(StreamLanguage.define(createWizParser())); + return new LanguageSupport(StreamLanguage.define(wizStreamParser)); } diff --git a/src/parser/registry.ts b/src/parser/registry.ts new file mode 100644 index 00000000..abbd80ee --- /dev/null +++ b/src/parser/registry.ts @@ -0,0 +1,119 @@ +// Parser registry - central place to map editorStyle -> parser for both editor and extractor. +// This keeps editors.ts and the indexer in sync. + +import { StreamLanguage, StreamParser } from "@codemirror/language"; +import { LanguageSupport } from "@codemirror/language"; +import { cpp } from "@codemirror/lang-cpp"; +import { asm6502 } from "./lang-6502"; +import { asmZ80 } from "./lang-z80"; +import { wiz, wizStreamParser } from "./lang-wiz"; +import { verilog, verilogStreamParser } from "./lang-verilog"; +import { inform6, inform6StreamParser } from "./lang-inform6"; +import { dialog, dialogStreamParser } from "./lang-dialog"; +import { basic, basicStreamParser } from "./lang-basic"; +import { fastBasic, fastBasicStreamParser } from "./lang-fastbasic"; +import { batariBasic, batariBasicStreamParser } from "./lang-bataribasic"; +import { markdown } from "@codemirror/lang-markdown"; + +export type SymbolKind = + | 'func' | 'macro' | 'type' | 'struct' | 'enum' | 'var' + | 'label' | 'equate' | 'proc' | 'module' | 'doc'; + +export type SymbolSourceKind = 'project' | 'toolchain' | 'docs'; + +export interface SymbolRecord { + id: string; + name: string; + kind: SymbolKind; + signature?: string; + brief?: string; + detail?: string; + section?: string; + source: SymbolSourceKind; + file: string; + line?: number; + url?: string; +} + +export interface ToolDeclPattern { re: RegExp; group?: number; kind: SymbolKind; } + +export interface ParserRegistryEntry { + /** Language support for the editor (used by CodeMirror) */ + language: LanguageSupport; + /** Raw StreamParser for the extractor (null if not available) */ + parser: StreamParser | null; +} + +export const parserRegistry: Record = { + 'text/x-csrc': { + language: cpp(), + parser: null // Lezer-based, extracted via lrextract.ts + }, + '6502': { + language: asm6502(), + parser: null // Lezer-based, extracted via lrextract.ts + }, + z80: { + language: asmZ80(), + parser: null // Lezer-based, extracted via lrextract.ts + }, + 'text/x-wiz': { + language: wiz(), + parser: wizStreamParser + }, + verilog: { + language: verilog(), + parser: verilogStreamParser + }, + inform6: { + language: inform6(), + parser: inform6StreamParser + }, + dialog: { + language: dialog(), + parser: dialogStreamParser + }, + basic: { + language: basic(), + parser: basicStreamParser + }, + fastbasic: { + language: fastBasic(), + parser: fastBasicStreamParser + }, + bataribasic: { + language: batariBasic(), + parser: batariBasicStreamParser + }, + markdown: { + language: markdown(), + parser: null + }, + // Tier C - no parser available + '6809': { + language: null, + parser: null + }, + vasm: { + language: null, + parser: null + }, + gas: { + language: null, + parser: null + }, + ecs: { + language: null, + parser: null + } +}; + +export function getParserForStyle(editorStyle: string): StreamParser | null { + const entry = parserRegistry[editorStyle]; + return entry?.parser ?? null; +} + +export function getLanguageSupportForStyle(editorStyle: string): LanguageSupport | null { + const entry = parserRegistry[editorStyle]; + return entry?.language ?? null; +} \ No newline at end of file diff --git a/test/unit/testsearchindex.ts b/test/unit/testsearchindex.ts new file mode 100644 index 00000000..aa9c148f --- /dev/null +++ b/test/unit/testsearchindex.ts @@ -0,0 +1,280 @@ +import assert from "assert"; +import { describe, it } from "mocha"; +import { SymbolIndex } from "../../src/ide/search/symbolindex"; +import { SymbolRecord, SymbolSourceKind, SymbolKind } from "../../src/common/searchtypes"; + +function createTestRecord( + name: string, + kind: SymbolKind, + source: SymbolSourceKind = 'project', + brief?: string, + detail?: string, + file: string = 'test.c', + line: number = 1 +): SymbolRecord { + return { + id: `${file}:${line}:${name}-${source}`, // Unique ID with source suffix + name, + kind, + brief, + detail, + source, + file, + line + }; +} + +describe('SymbolIndex Search Ranking', function () { + let index: SymbolIndex; + + before(async function () { + index = new SymbolIndex(); + + // Add test records with various match qualities + const records: SymbolRecord[] = [ + // Exact name matches + createTestRecord('joy_read', 'func', 'toolchain', 'Read joystick state'), + createTestRecord('joy_write', 'func', 'toolchain', 'Write joystick state'), + createTestRecord('JOY_BTN_1', 'macro', 'toolchain', 'Joystick button 1 mask'), + createTestRecord('JOY_BTN_2', 'macro', 'toolchain', 'Joystick button 2 mask'), + createTestRecord('JOY_DIR_LEFT', 'macro', 'toolchain', 'Left direction'), + + // Project symbols (higher priority - same name, different source) + createTestRecord('main', 'func', 'project', 'Main entry point'), + createTestRecord('update', 'func', 'project', 'Game update loop'), + createTestRecord('init', 'func', 'project', 'Initialize system'), + + // Longer names with common prefixes + createTestRecord('joy_read_button', 'func', 'toolchain', 'Read button state'), + createTestRecord('joy_read_axis', 'func', 'toolchain', 'Read axis value'), + createTestRecord('joy_read_all', 'func', 'toolchain', 'Read all inputs'), + + // Partial matches + createTestRecord('joystick_init', 'func', 'toolchain', 'Initialize joystick'), + createTestRecord('joystick_poll', 'func', 'toolchain', 'Poll joystick'), + createTestRecord('joystick_calibrate', 'func', 'toolchain', 'Calibrate joystick'), + + // Docs + createTestRecord('Getting Started', 'doc', 'docs', 'How to get started with 8bitworkshop'), + createTestRecord('Joystick API', 'doc', 'docs', 'Joystick API reference'), + createTestRecord('Memory Map', 'doc', 'docs', 'Memory map documentation'), + ]; + + index.pushRecords(records); + await index.init(); + }); + + describe('uFuzzy name matching', function () { + it('should rank exact matches first', async function () { + const results = await index.query('joy_read', 10); + + assert.ok(results.length >= 1, 'Should have at least one result'); + assert.strictEqual(results[0].record.name, 'joy_read', 'Exact match should be first'); + assert.strictEqual(results[0].record.kind, 'func'); + }); + + it('should rank prefix matches after exact', async function () { + const results = await index.query('joy', 10); + + // Should find all joy_* symbols + const names = results.map(r => r.record.name); + assert.ok(names.includes('joy_read')); + assert.ok(names.includes('joy_write')); + assert.ok(names.includes('joy_read_button')); + assert.ok(names.includes('joy_read_axis')); + assert.ok(names.includes('joy_read_all')); + }); + + it('should handle fuzzy matching (joy_rd -> joy_read)', async function () { + const results = await index.query('joy_rd', 10); + + const names = results.map(r => r.record.name); + assert.ok(names.includes('joy_read'), 'Should match joy_read with typo'); + }); + + it('should rank project symbols above toolchain for same name', async function () { + // Add duplicate name with project source - same name, different source + // The project source has weight 1.3 vs toolchain 1.0, so project should rank higher + const projectJoy = createTestRecord('joy_read', 'func', 'project', 'Project joy read'); + index.pushRecords([projectJoy]); + + const results = await index.query('joy_read', 10); + + // First result should be project source (higher weight) + const first = results.find(r => r.record.name === 'joy_read'); + assert.ok(first, 'Should find joy_read'); + assert.strictEqual(first.record.source, 'project', 'Project source should win for equal match quality'); + }); + + it('should handle case-insensitive matching', async function () { + const results = await index.query('JOY_READ', 10); + + const names = results.map(r => r.record.name); + assert.ok(names.includes('joy_read'), 'Should match case-insensitively'); + }); + + it('should limit results to specified limit', async function () { + const results = await index.query('joy', 3); + + assert.ok(results.length <= 3); + }); + }); + + describe('Source weight priority', function () { + it('should prefer project over toolchain for equal match', async function () { + // Add duplicate name with project source + const projectMain = createTestRecord('main', 'func', 'project', 'Project main'); + const toolMain = createTestRecord('main', 'func', 'toolchain', 'Tool main'); + + // Both have same name but project has weight 1.3 > toolchain 1.0 + const testIndex = new SymbolIndex([projectMain, toolMain]); + await testIndex.init(); + + const results = await testIndex.query('main', 10); + + // Project should rank higher due to source weight + assert.strictEqual(results[0].record.source, 'project'); + }); + + it('should prefer toolchain over docs for equal match', async function () { + const toolMain = createTestRecord('getting_started', 'func', 'toolchain', 'Tool getting_started'); + const docMain = createTestRecord('Getting Started', 'doc', 'docs', 'Doc getting started'); + + const testIndex = new SymbolIndex([toolMain, docMain]); + await testIndex.init(); + + const results = await testIndex.query('getting_started', 10); + + // Toolchain should rank higher than docs (weight 1.0 > 0.7) + assert.strictEqual(results[0].record.source, 'toolchain'); + }); + }); + + describe('MiniSearch prose search', function () { + it('should find records by brief text', async function () { + // search for a phrase in the brief field + const results = await index.query('joystick state', 10); + + const names = results.map(r => r.record.name); + assert.ok(names.includes('joy_read') || results.length > 0, 'Should find at least one record by brief text'); + }); + }); + + describe('Deduplication', function () { + it('should not return duplicate records by ID', async function () { + // Create index with duplicate IDs + const dupRecords: SymbolRecord[] = [ + createTestRecord('test', 'func', 'project'), + createTestRecord('test', 'func', 'project'), // duplicate + ]; + + const testIndex = new SymbolIndex(dupRecords); + await testIndex.init(); + + const results = await testIndex.query('test', 10); + + assert.strictEqual(results.length, 1, 'Should deduplicate by ID'); + }); + }); + + describe('Edge cases', function () { + it('should return empty for empty query', async function () { + const results = await index.query('', 10); + assert.strictEqual(results.length, 0); + }); + + it('should return empty for empty index', async function () { + const emptyIndex = new SymbolIndex([]); + await emptyIndex.init(); + + const results = await emptyIndex.query('test', 10); + assert.strictEqual(results.length, 0); + }); + + it('should handle special characters in needle', async function () { + // Add record with special chars + const special = createTestRecord('foo$bar', 'macro', 'project'); + index.pushRecords([special]); + + const results = await index.query('foo$bar', 10); + assert.ok(results.find(r => r.record.name === 'foo$bar')); + }); + }); +}); + +// --------------------------------------------------------------------------- +// ProjectSource full-text search (straight scan of project file data) +// --------------------------------------------------------------------------- +import { ProjectSource, setProjectProvider } from '../../src/ide/search/projectsource'; + +describe('ProjectSource full-text search', function () { + let ps: ProjectSource; + + before(async function () { + ps = new ProjectSource(); + }); + + after(function () { + setProjectProvider(() => null); + }); + + it('should find matching lines in project file text (case-insensitive)', async function () { + // register stub project with a couple of files + setProjectProvider(() => ({ + mainPath: 'main.c', + iterateFiles: (cb: (path: string, data: any) => void) => { + cb('main.c', 'int main() {\n return 0;\n}\n'); + cb('game.s', 'foo: lda #$10\n sta $80\n'); + } + })); + + const hits = (ps as any).fulltextQuery('return', 10); + assert.ok(hits.length >= 1, 'Should find return'); + const rec = hits[0].record; + assert.strictEqual(rec.kind, 'text'); + assert.strictEqual(rec.source, 'project'); + assert.strictEqual(rec.file, 'main.c'); + assert.strictEqual(rec.line, 2); + }); + + it('should not search binary files', async function () { + setProjectProvider(() => ({ + mainPath: 'main.c', + iterateFiles: (cb) => { + cb('sprite.bin', new Uint8Array([0, 1, 2, 3])); + cb('main.c', 'void setup() {}\n'); + } + })); + + const hits = (ps as any).fulltextQuery('setup', 10); + assert.ok(hits.length >= 1); + assert.strictEqual(hits[0].record.file, 'main.c'); + }); + + it('should require at least 2 chars', async function () { + setProjectProvider(() => ({ + mainPath: 'main.c', + iterateFiles: (cb) => cb('main.c', 'x\ny\n') + })); + const hits = (ps as any).fulltextQuery('x', 10); + assert.strictEqual(hits.length, 0); + }); + + it('should rank symbol hits above text hits', async function () { + setProjectProvider(() => ({ + mainPath: 'main.c', + iterateFiles: (cb) => cb('main.c', 'int counter = 0;\n') + })); + // stub index: symbol hit for counter + (ps as any).index = { + query: () => [{ + record: { id: 'main.c:1:counter', name: 'counter', kind: 'var', source: 'project', file: 'main.c', line: 1 }, + score: 980 + }] + }; + + const hits = ps.query('counter', 10); + assert.strictEqual(hits[0].record.kind, 'var', 'Symbol hit should rank first'); + assert.strictEqual(hits[0].score, 980); + }); +}); \ No newline at end of file diff --git a/test/unit/testsearchsymbols.ts b/test/unit/testsearchsymbols.ts new file mode 100644 index 00000000..ddf73644 --- /dev/null +++ b/test/unit/testsearchsymbols.ts @@ -0,0 +1,129 @@ +import assert from "assert"; +import { describe, it } from "mocha"; +import { extractSymbols } from "../../src/common/symbols"; +import { extractDocComment, extractTrailingComment } from "../../src/common/symbols/doccomment"; +import { SymbolRecord } from "../../src/common/searchtypes"; + +describe('Symbol Extraction', function () { + describe('Basic Symbol Extraction', function () { + it('should extract #define macros from C/C++', function () { + const text = `#define MAX_SIZE 100\n#define JOY_BTN_1_MASK 1\n`; + const records = extractSymbols(text, 'test.h', 'text/x-csrc', 'project'); + + const defines = records.filter(r => r.kind === 'macro'); + assert.strictEqual(defines.length, 2); + + const maxMacro = defines.find(r => r.name === 'MAX_SIZE'); + assert.ok(maxMacro, 'Should find MAX_SIZE macro'); + assert.strictEqual(maxMacro.name, 'MAX_SIZE'); + assert.strictEqual(maxMacro.kind, 'macro'); + + const joyMacro = defines.find(r => r.name === 'JOY_BTN_1_MASK'); + assert.ok(joyMacro, 'Should find JOY_BTN_1_MASK macro'); + assert.strictEqual(joyMacro.name, 'JOY_BTN_1_MASK'); + }); + + it('should extract 6502 labels', function () { + const text = `foo: lda #$10\nbar: sta $80\n`; + const records = extractSymbols(text, 'test.s', '6502', 'project'); + + const labels = records.filter(r => r.kind === 'label'); + assert.strictEqual(labels.length, 2); + + const fooLabel = labels.find(r => r.name === 'foo'); + assert.ok(fooLabel, 'Should find foo label'); + assert.strictEqual(fooLabel.line, 1); + + const barLabel = labels.find(r => r.name === 'bar'); + assert.ok(barLabel, 'Should find bar label'); + assert.strictEqual(barLabel.line, 2); + }); + + it('should extract 6502 equates', function () { + const text = `BAR = $1234\nFOO = $5678\n`; + const records = extractSymbols(text, 'test.s', '6502', 'project'); + + const equates = records.filter(r => r.kind === 'equate'); + assert.strictEqual(equates.length, 2); + + const barEquate = equates.find(r => r.name === 'BAR'); + assert.ok(barEquate, 'Should find BAR equate'); + assert.strictEqual(barEquate.name, 'BAR'); + + const fooEquate = equates.find(r => r.name === 'FOO'); + assert.ok(fooEquate, 'Should find FOO equate'); + assert.strictEqual(fooEquate.name, 'FOO'); + }); + + it('should not extract label-like text inside comments', function () { + const text = `lda #$10 ; foo: this is a comment\nlabel:\n`; + const records = extractSymbols(text, 'test.s', '6502', 'project'); + + const labels = records.filter(r => r.kind === 'label'); + assert.strictEqual(labels.length, 1); + + const label = labels[0]; + assert.strictEqual(label.name, 'label', 'Should extract label, not colon from comment'); + }); + }); + + describe('Doc comment extraction (Rule 3)', function () { + it('should extract brief from plain comment immediately above', function () { + const lines = [ + '/* The name of the standard joystick driver for a platform */', + 'extern const char joy_stddrv[];' + ]; + const doc = extractDocComment(lines, 2); + assert.ok(doc, 'Should extract doc comment'); + assert.ok(doc.brief, 'Should have brief'); + assert.strictEqual(doc.brief, 'The name of the standard joystick driver for a platform', 'Brief should match comment text'); + }); + + it('should use trailing comment as brief', function () { + const text = '#define MAX 1 /* maximum value */'; + const brief = extractTrailingComment(text); + assert.ok(brief, 'Should extract trailing comment'); + assert.strictEqual(brief, 'maximum value', 'Brief should match comment text'); + }); + + it('should return null when no doc comments exist', function () { + const lines = ['extern const char joy_stddrv[];']; + const doc = extractDocComment(lines, 1); + assert.strictEqual(doc, null, 'Should return null when no doc comments'); + }); + }); + + describe('Cross-language extraction', function () { + it('should handle wiz editor style (basic extraction)', function () { + const text = `func main() {\n return 0;\n}\n`; + const records = extractSymbols(text, 'test.wiz', 'text/x-wiz', 'project'); + assert.ok(Array.isArray(records), 'Should return array for wiz extraction'); + }); + + it('should handle dialog editor style (basic extraction)', function () { + const text = `@@ Clear, bright, and spacious\n!(kitchen) A kitchen.`; + const records = extractSymbols(text, 'game.dlg', 'dialog', 'project'); + assert.ok(Array.isArray(records), 'Should return array for dialog extraction'); + }); + }); +}); + +describe('SymbolRecord Type', function () { + it('should create valid SymbolRecord', function () { + const record: SymbolRecord = { + id: 'test.c:1:main', + name: 'main', + kind: 'func', + source: 'project', + file: 'test.c', + line: 1 + }; + + assert.strictEqual(record.id, 'test.c:1:main'); + assert.strictEqual(record.name, 'main'); + assert.strictEqual(record.kind, 'func'); + assert.strictEqual(record.source, 'project'); + assert.strictEqual(record.file, 'test.c'); + assert.strictEqual(record.line, 1); + }); +}); \ No newline at end of file
    Custom