vscode: tweaks to detection for directories with lots of projects, detect on open file

extension/docs/detection.md

@@ -12,12 +12,21 @@ it always shows you the clue it used.
 - **Opening a folder** that has no 8bitworkshop settings. A notification
   appears only when the clues are strong and point at one platform:
   "This looks like an NES project (...). [Use It] [Choose...] [Not Now]
-  [Don't Ask]". Nothing changes until you choose **Use It**.
+  [Don't Ask]". Nothing changes until you choose **Use It**. A large
+  folder with projects scattered through it, such as a big codebase
+  with a few examples inside, brings up no notification.
+- **Opening a source file** in a folder with no project. If its folder
+  has strong clues, you get the same notification. After **Not Now**,
+  8bitworkshop doesn't ask again until you reopen the window. Weaker
+  clues show only in the status bar, such as **NES?**; click it to
+  choose a platform.
 - **8bitworkshop: Detect Projects** scans the folder again, even after
-  **Don't Ask**.
+  **Don't Ask**. It also lists folders with weaker clues as potential
+  projects, unchecked.
 
 If a folder holds several projects, such as `nes/` and `c64/`
-subfolders, the notification says how many it found. Click **Review**
+subfolders or a folder of cloned repos, the notification says how many
+it found. A project's own subfolders count as part of it. Click **Review**
 and check the ones to use; each gets its own entry in
 `8bitworkshop.folders`.
 
@@ -36,7 +45,10 @@ Strongest first:
 | A folder named after a platform | `nes/`, `c64/` |
 
 Plain C or assembly with none of these never brings up a notification,
-so other projects you open aren't interrupted.
+so other projects you open aren't interrupted. Neither do ROM files,
+linker configs, headers, or folder names alone: a notification needs a
+clue in a program's own source. A C program needs a `main()`, and
+startup code such as `crt0.s` is never the main file.
 
 Hardware registers and function names in a header only declare a
 library, so a folder of nothing but headers and library sources isn't

extension/notes/PLAN.md

@@ -785,9 +785,13 @@ soon as the folder opens.
    a platform header or hardware register in a source file, a cc65
    target. `.c` and `.s` files alone never do, and neither do headers,
    linker configs, ROM files, or folder names without code. Startup
-   code (`crt0.*`) is never a main file. Unasked, more than 5 projects,
-   or a scan that hit its file limit, means a big repo that isn't ours:
-   no notification.
+   code (`crt0.*`) and C files without `main()` are never main files.
+   A finding inside a project with a main file is part of it, not its
+   own project. Unasked, prompt only if the folder is made of projects:
+   at most 5, or projects in at least half its top-level folders (a
+   folder of cloned repos). Projects scattered through a big repo, or a
+   scan that hit its file limit, mean no notification; opening a file
+   there still detects (below).
    - *One project*: "This looks like an NES project (`#include
      "neslib.h"` in main.c). [Use It] [Choose...] [Not Now]". Unlike case
      2, the user didn't ask, so nothing applies until they click Use It,
@@ -805,6 +809,13 @@ soon as the folder opens.
    - *Dismissed*: closing the notification without choosing is "not
      now"; "Don't ask for this folder" is remembered. Either way the
      Projects view keeps the findings.
+   - *Opening a file*: a buildable source file in a directory with no
+     project detects that directory, once a session, after the folder
+     scan. A strong, clear detection gets the one-project prompt; any
+     answer but Use It quiets file prompts in that workspace folder for
+     the session. A weaker one only shows in the status bar (`$(chip)
+     NES?`), whose menu leads to Set as Main File. Directories the
+     folder prompt already covered don't prompt again.
 5. **Other build systems.** Makefiles, cc65 `.cfg` files, and linker
    scripts are evidence for detection only. We build with our own
    toolchains and never run `make`. If the repo needs include paths

extension/src/detection.ts

@@ -4,7 +4,7 @@
 
 import * as vscode from 'vscode';
 import * as path from 'path';
-import { Detection, detectProject, detectionSummary, describeDetection, describeFinding, isBuildableSource, classifyFinding, isClearWinner, isFolderOfPrograms, toolForDialect } from '../../src/common/detect';
+import { Detection, DirFinding, detectProject, detectionSummary, describeDetection, describeFinding, isBuildableSource, classifyFinding, dropNestedFindings, isClearWinner, isFolderOfPrograms, isMadeOfProjects, toolForDialect } from '../../src/common/detect';
 export { detectionSummary, describeDetection, describeFinding, isFolderOfPrograms };
 import { PLATFORM_PARAMS } from '../../src/worker/platforms';
 import { getToolForPlatform } from '../../src/common/toolselect';
@@ -61,6 +61,16 @@ export async function detectDirectory(templates: Templates, dir: vscode.Uri, fil
   });
 }
 
+/** Detect for a directory on disk, from the files directly in it. */
+export async function detectDirectoryAt(templates: Templates, dir: vscode.Uri): Promise<Detection[]> {
+  try {
+    var files = (await vscode.workspace.fs.readDirectory(dir)).filter(([, t]) => t === vscode.FileType.File).map(([n]) => n);
+    return await detectDirectory(templates, dir, files);
+  } catch (e) {
+    return [];
+  }
+}
+
 export interface FileChoice {
   platform: string;
   tool?: string;
@@ -153,6 +163,8 @@ export interface FolderScan {
   findings: FolderFinding[];
   /** the folder has more files than the scan looks at */
   truncated: boolean;
+  /** the projects make up the folder, rather than being scattered through it */
+  madeOfProjects: boolean;
 }
 
 /**
@@ -169,23 +181,21 @@ export async function scanFolder(templates: Templates, folder: vscode.WorkspaceF
     if (!byDir.has(dir)) byDir.set(dir, []);
     byDir.get(dir).push(path.basename(uri.fsPath));
   }
-  var findings: FolderFinding[] = [];
+  var rel = (dir: string) => path.relative(folder.uri.fsPath, dir).split(path.sep).join('/');
+  var found: (DirFinding & { uri: vscode.Uri })[] = [];
   for (var [dir, files] of byDir) {
     if (!files.some(f => isBuildableSource(f) || /\.(nes|a26|a78|gb|col|sms)$/i.test(f))) continue;
-    var found = await detectDirectory(templates, vscode.Uri.file(dir), files);
-    var kind = classifyFinding(found);
-    if (kind) findings.push({ dir: vscode.Uri.file(dir), detection: found[0], potential: kind === 'potential' });
+    var detections = await detectDirectory(templates, vscode.Uri.file(dir), files);
+    var kind = classifyFinding(detections);
+    if (kind) found.push({ dir: rel(dir), uri: vscode.Uri.file(dir), detection: detections[0], potential: kind === 'potential' });
   }
-  // a folder of programs already covers a nested library folder inside it;
-  // a nested project with its own main file stays
-  findings = findings.filter(f => !findings.some(p =>
-    p !== f && isFolderOfPrograms(p.detection) && !f.detection.mainFile && isInsideDir(p.dir.fsPath, f.dir.fsPath)));
-  return { findings, truncated: uris.length >= MAX_SCAN_FILES };
-}
-
-function isInsideDir(parent: string, child: string): boolean {
-  var rel = path.relative(parent, child);
-  return rel !== '' && !rel.startsWith('..') && !path.isAbsolute(rel);
+  found = dropNestedFindings(found);
+  var projectDirs = found.filter(f => !f.potential).map(f => f.dir);
+  return {
+    findings: found.map(f => ({ dir: f.uri, detection: f.detection, potential: f.potential })),
+    truncated: uris.length >= MAX_SCAN_FILES,
+    madeOfProjects: isMadeOfProjects(projectDirs, [...byDir.keys()].map(rel)),
+  };
 }
 
 export function dontAskKey(folder: vscode.WorkspaceFolder) {

extension/src/extension.ts

@@ -15,7 +15,8 @@ import { Project, findRootDir, isHeaderFile, isInside, isOwnExtension, isSourceF
 import { CONFIG, ProjectScope } from './projectscope';
 import { BuildReason, BuildScheduler } from './autobuild';
 import { PRESET_SCHEME, PresetFileSystem, Templates } from './templates';
-import { chooseForFile, detectionSummary, describeFinding, dontAskKey, FolderFinding, platformName, scanFolder } from './detection';
+import { chooseForFile, detectDirectoryAt, detectionSummary, describeDetection, describeFinding, dontAskKey, FolderFinding, platformName, scanFolder } from './detection';
+import { Detection, classifyFinding, isBuildableSource } from '../../src/common/detect';
 import { TOOL_META } from '../../src/common/toolmeta';
 import { AssetStore } from './assets';
 import { AssetManifest, packsForPlatform } from './assetpacks';
@@ -112,12 +113,13 @@ export function activate(ctx: vscode.ExtensionContext) {
   ctx.subscriptions.push(
     vscode.workspace.onDidSaveTextDocument(doc => onSave(doc)),
     vscode.workspace.onDidChangeTextDocument(e => onChange(e)),
-    vscode.window.onDidChangeActiveTextEditor(() => updateStatus()),
+    vscode.window.onDidChangeActiveTextEditor(() => { updateStatus(); detectForActiveFile(); }),
     scope.onDidChange(() => updateStatus()),
     vscode.workspace.onDidChangeWorkspaceFolders(() => detectProjects(false)),
     { dispose: () => { builds?.dispose(); emu?.dispose(); } },
   );
-  scope.loadReadmes().then(() => detectProjects(false));
+  folderScans = scope.loadReadmes().then(() => detectProjects(false));
+  detectForActiveFile();
   updateStatus();
 }
 
@@ -276,6 +278,11 @@ function updateStatus() {
     tip.appendMarkdown('Click for project options.');
     status.tooltip = tip;
     status.show();
+  } else if (uri && uri.scheme === 'file' && dirGuesses.get(path.dirname(uri.fsPath))) {
+    var guess = dirGuesses.get(path.dirname(uri.fsPath))!;
+    status.text = `$(chip) ${platformName(templates, guess.platform)}?`;
+    status.tooltip = `Looks like ${platformName(templates, guess.platform)}: ${describeDetection(guess)}. Click to choose a platform.`;
+    status.show();
   } else if (uri && uri.scheme === 'file' && isOwnExtension(uri.fsPath)) {
     status.text = '$(chip) No platform';
     status.tooltip = 'Choose a platform for this file';
@@ -797,13 +804,27 @@ async function closeEditor(uri: vscode.Uri) {
 
 ////// detection on open (case 3)
 
-// unasked, prompt only for a folder that is plainly one or a few projects;
-// findings scattered across a big repo mean it isn't ours
-const MAX_UNASKED_PROJECTS = 5;
+interface FolderOffer {
+  folder: vscode.WorkspaceFolder;
+  strong: FolderFinding[];
+  potential: FolderFinding[];
+}
 
-async function detectProjects(asked: boolean) {
-  // "Don't Ask" only makes sense on a prompt the user didn't ask for
-  var dontAsk = asked ? [] : ["Don't Ask"];
+// resolves when the latest folder scan is done (not its prompts)
+let folderScans: Promise<void> = Promise.resolve();
+
+/**
+ * Scan the workspace folders and offer what they hold. Resolves once the
+ * scans are done; the prompts carry on.
+ */
+function detectProjects(asked: boolean): Promise<void> {
+  var scans = scanForOffers(asked);
+  scans.then(offers => offerFindings(offers, asked)).catch(e => output.appendLine(`detect: ${e}`));
+  return folderScans = scans.then(() => undefined, () => undefined);
+}
+
+async function scanForOffers(asked: boolean): Promise<FolderOffer[]> {
+  var offers: FolderOffer[] = [];
   for (var folder of vscode.workspace.workspaceFolders || []) {
     if (folder.uri.scheme !== 'file') continue;
     if (!asked && (scope.folderHasProject(folder) || context.workspaceState.get(dontAskKey(folder)))) continue;
@@ -813,36 +834,93 @@ async function detectProjects(asked: boolean) {
     var strong = findings.filter(f => !f.potential);
     // weak evidence is offered only when the user asks
     var potential = asked ? findings.filter(f => f.potential) : [];
-    if (!asked && (scan.truncated || strong.length > MAX_UNASKED_PROJECTS)) continue;
+    // unasked, a big repo with projects scattered through it isn't ours;
+    // opening a file in one of them still asks
+    if (!asked && (scan.truncated || !scan.madeOfProjects)) continue;
     if (!strong.length && !potential.length) {
       if (asked) vscode.window.showInformationMessage(`No 8bitworkshop projects found in ${folder.name}.`);
       continue;
     }
+    // this prompt speaks for the folder; opening a file in it won't ask again
+    for (var f of findings) dirGuesses.set(f.dir.fsPath, f.detection);
+    quietFolders.add(folder.uri.toString());
+    offers.push({ folder, strong, potential });
+  }
+  return offers;
+}
+
+async function offerFindings(offers: FolderOffer[], asked: boolean) {
+  for (var { folder, strong, potential } of offers) {
     if (strong.length === 1 && !potential.length) {
-      var f = strong[0], d = f.detection;
-      var where = path.relative(folder.uri.fsPath, f.dir.fsPath);
-      // a sure thing needs no clue; just say what it found
-      var why = d.score >= 1 ? detectionSummary(d) : describeFinding(d);
-      var answer = await vscode.window.showInformationMessage(
-        `This looks like a${/^[aeiou]/i.test(platformName(templates, d.platform)) ? 'n' : ''} ${platformName(templates, d.platform)} project${where ? ' in ' + where : ''}${why ? ` (${why})` : ''}.`,
-        'Use It', 'Choose...', 'Not Now', ...dontAsk);
-      if (answer === 'Use It') await acceptFinding(f.dir.fsPath, d.platform, d.mainFile, d.tool);
-      else if (answer === 'Choose...') {
-        var picked = await templates.pickPlatform('Platform for this folder', d.platform);
-        if (picked) await acceptFinding(f.dir.fsPath, picked.id, d.mainFile, undefined);
-      } else if (answer === "Don't Ask") await context.workspaceState.update(dontAskKey(folder), true);
+      await offerProject(folder, strong[0].dir, strong[0].detection, asked);
       continue;
     }
     var plural = (n: number) => n === 1 ? '' : 's';
     var message = strong.length
       ? `Found ${strong.length} 8bitworkshop project${plural(strong.length)}${potential.length ? ` and ${potential.length} potential` : ''} in ${folder.name}.`
       : `Found ${potential.length} potential 8bitworkshop project${plural(potential.length)} in ${folder.name}.`;
-    var review = await vscode.window.showInformationMessage(message, 'Review', 'Not Now', ...dontAsk);
+    // "Don't Ask" only makes sense on a prompt the user didn't ask for
+    var review = await vscode.window.showInformationMessage(message, 'Review', 'Not Now', ...(asked ? [] : ["Don't Ask"]));
     if (review === "Don't Ask") await context.workspaceState.update(dontAskKey(folder), true);
     if (review === 'Review') await reviewFindings(folder, [...strong, ...potential]);
   }
 }
 
+/** Offer one directory as a project. Returns true if the user took it. */
+async function offerProject(folder: vscode.WorkspaceFolder | undefined, dir: vscode.Uri, d: Detection, asked: boolean): Promise<boolean> {
+  var where = folder ? path.relative(folder.uri.fsPath, dir.fsPath) : '';
+  var name = platformName(templates, d.platform);
+  // a sure thing needs no clue; just say what it found
+  var why = d.score >= 1 ? detectionSummary(d) : describeFinding(d);
+  var answer = await vscode.window.showInformationMessage(
+    `This looks like a${/^[aeiou]/i.test(name) ? 'n' : ''} ${name} project${where ? ' in ' + where : ''}${why ? ` (${why})` : ''}.`,
+    'Use It', 'Choose...', 'Not Now', ...(asked || !folder ? [] : ["Don't Ask"]));
+  if (answer === 'Use It') {
+    await acceptFinding(dir.fsPath, d.platform, d.mainFile, d.tool);
+    return true;
+  }
+  if (answer === 'Choose...') {
+    var picked = await templates.pickPlatform('Platform for this folder', d.platform);
+    if (picked) await acceptFinding(dir.fsPath, picked.id, d.mainFile, undefined);
+    return !!picked;
+  }
+  if (answer === "Don't Ask" && folder) await context.workspaceState.update(dontAskKey(folder), true);
+  return false;
+}
+
+////// detection on opening a file
+
+// directories detected this session, with the best guess if any
+const dirGuesses = new Map<string, Detection | undefined>();
+// workspace folders not to prompt in again this session
+const quietFolders = new Set<string>();
+
+/**
+ * Opening a source file in a directory with no project detects that
+ * directory, once a session: a strong guess prompts, a weaker one only
+ * shows in the status bar.
+ */
+async function detectForActiveFile() {
+  var uri = activeUri();
+  if (!uri || uri.scheme !== 'file' || currentTarget() || !isBuildableSource(uri.fsPath)) return;
+  await folderScans;
+  var dir = path.dirname(uri.fsPath);
+  if (dirGuesses.has(dir) || currentTarget()) return;
+  dirGuesses.set(dir, undefined);
+  var found = await detectDirectoryAt(templates, vscode.Uri.file(dir));
+  var kind = classifyFinding(found);
+  if (!kind) return;
+  dirGuesses.set(dir, found[0]);
+  updateStatus();
+  if (kind !== 'project') return;
+  var folder = vscode.workspace.getWorkspaceFolder(uri);
+  var key = folder?.uri.toString() || dir;
+  if (quietFolders.has(key) || (folder && context.workspaceState.get(dontAskKey(folder)))) return;
+  // one prompt at a time; turning it down quiets the folder for the session
+  quietFolders.add(key);
+  if (await offerProject(folder, vscode.Uri.file(dir), found[0], false)) quietFolders.delete(key);
+}
+
 /** Let the user pick which findings to keep as projects. */
 async function reviewFindings(folder: vscode.WorkspaceFolder, findings: FolderFinding[]) {
   type Item = vscode.QuickPickItem & { f: FolderFinding };

src/common/detect.ts

@@ -319,6 +319,45 @@ export function classifyFinding(detections: Detection[]): 'project' | 'potential
   return undefined;
 }
 
+/** A directory's finding in a folder scan; `dir` is relative, with posix separators, '' for the folder itself. */
+export interface DirFinding {
+  dir: string;
+  detection: Detection;
+  potential?: boolean;
+}
+
+function isInsideDir(parent: string, child: string): boolean {
+  return parent !== child && (parent === '' || child.startsWith(parent + '/'));
+}
+
+/**
+ * Drop findings that belong to another: anything inside a project with a
+ * main file (its code and data folders), and a library folder inside a
+ * folder of programs. A nested project with its own main file stays.
+ */
+export function dropNestedFindings<T extends DirFinding>(findings: T[]): T[] {
+  return findings.filter(f => !findings.some(p => isInsideDir(p.dir, f.dir) && (
+    (!p.potential && !!p.detection.mainFile) ||
+    (isFolderOfPrograms(p.detection) && !f.detection.mainFile))));
+}
+
+// unasked, this many projects anywhere in a folder is still worth a prompt
+const FEW_PROJECTS = 5;
+
+/**
+ * True if a folder is made of projects, so a prompt nobody asked for is
+ * welcome: a few projects, or projects in at least half its top-level
+ * folders (a folder of cloned repos). Projects scattered through a big
+ * codebase don't count. `allDirs` is every directory the scan saw.
+ */
+export function isMadeOfProjects(projectDirs: string[], allDirs: string[]): boolean {
+  if (projectDirs.length <= FEW_PROJECTS) return true;
+  var top = (d: string) => d.split('/')[0];
+  var tops = new Set(allDirs.filter(d => d).map(top));
+  var covered = new Set(projectDirs.filter(d => d).map(top));
+  return covered.size * 2 >= tops.size;
+}
+
 /** True for a header or include file: a declaration, not a program. */
 export function isHeaderFile(fn: string): boolean {
   return HEADER_EXTS.includes(extname(fn));

test/unit/testdetect.ts

@@ -2,7 +2,7 @@ import assert from "assert";
 import { describe, it } from "mocha";
 import * as fs from "fs";
 import * as path from "path";
-import { detectProject, detectDialect, detectionSummary, describeFinding, findMainCandidates, headersFromPresets, isClearWinner, isFolderOfPrograms, isHeaderFile, classifyFinding, mainEvidence, parseReadmeBadge, toolForDialect } from "../../src/common/detect";
+import { detectProject, detectDialect, detectionSummary, describeFinding, findMainCandidates, headersFromPresets, isClearWinner, isFolderOfPrograms, isHeaderFile, classifyFinding, dropNestedFindings, isMadeOfProjects, mainEvidence, parseReadmeBadge, toolForDialect } from "../../src/common/detect";
 import { PLATFORM_PARAMS } from "../../src/worker/platforms";
 
 const PRESETS = 'presets';
@@ -114,6 +114,32 @@ describe('detect', () => {
     assert.strictEqual(classifyFinding(d), 'potential');
   });
 
+  it('keeps subfolders part of their project', () => {
+    var main = { platform: 'vcs', score: 1, evidence: [], mainFile: 'main.asm' };
+    var part = { platform: 'vcs', score: 0.5, evidence: [], mainFile: 'kernel.asm' };
+    var programs = { platform: 'nes', score: 1, evidence: [], mainCandidates: ['a.c', 'b.c'] };
+    var kept = dropNestedFindings([
+      { dir: 'nyancat', detection: main },
+      { dir: 'nyancat/bank1/code', detection: part },
+      { dir: 'nyancat2', detection: part },
+      { dir: 'presets/nes', detection: programs },
+      { dir: 'presets/nes/chase', detection: part },
+      { dir: 'presets/nes/lib', detection: { ...part, mainFile: undefined } },
+    ]).map(f => f.dir);
+    assert.deepStrictEqual(kept, ['nyancat', 'nyancat2', 'presets/nes', 'presets/nes/chase']);
+  });
+
+  it('prompts for a folder made of projects, not a big repo', () => {
+    // ~/emu/workspace: a folder of cloned project repos
+    var repos = ['2048', 'chase', 'falling', 'genemedic', 'guntner', 'mango', 'pacman'];
+    assert.ok(isMadeOfProjects(repos, [...repos, 'guntner/exports', 'scripts']));
+    // the 8bitworkshop repo: projects under a few of many top-level folders
+    var projects = ['src/worker/lib/verilog', 'test/ecs', 'tmp', 'presets/nes', 'presets/c64', 'presets/vcs', 'presets/gb', 'presets/apple2', 'presets/zx'];
+    var dirs = [...projects, 'src/common', 'css', 'res', 'gen', 'doc', 'scripts', 'extension', 'images', 'web', 'meta'];
+    assert.ok(!isMadeOfProjects(projects, dirs));
+    assert.ok(isMadeOfProjects(['presets/nes', 'presets/c64'], dirs), 'a few projects anywhere still count');
+  });
+
   it('lists every program in a folder of programs', async () => {
     var programs = ['chase.c', 'climber.c', 'testphys.c'];
     var map: { [fn: string]: string } = {};