check / scan: project walks skip dot folders, worktrees and git-ignored output, no stat per entry (244 s -> 46 s on a big game repo)

CHANGELOG.md

@@ -3,6 +3,7 @@
 
 ## Unreleased
 
+- `check` and `scan` are fast on big repos: project walks skip dot folders (tool state, worktree copies), linked git worktrees and the folders git ignores (Pods, Carthage, SourcePackages and ios / android stay in), and read entry types from the listing instead of a stat per file. A game repo with worktree copies went from 244 s to 46 s for `check`.
 - `storebound mcp --http`: one Streamable HTTP MCP server that every agent session of a machine shares (no process per session). Each tool gets a `project` argument (absolute path inside `--roots`); auth is a local bearer token (`~/.storebound/mcp-token`, made on first start) or, for a server behind TLS, OAuth bearer tokens checked by introspection (RFC 7662) with protected resource metadata (RFC 9728). docs/mcp.md.
 
 ## 0.8.57 · 2026-10-27

cli/lib/node-packages.ts

@@ -3,8 +3,9 @@
 // folders (package.json "workspaces", npm / yarn / pnpm-style array or { packages }, as Bun reads them) and local
 // SwiftPM packages (`.package(path:)` in a Package.swift, e.g. Capacitor 8's CapApp-SPM). Used by `privacy-manifest`
 // and `scan`. Read-only; never throws.
-import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs';
+import { type Dirent, existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs';
 import { dirname, join, resolve } from 'node:path';
+import { skipDir } from './walk-skip';
 
 interface PkgJson { name?: string; dependencies?: Record<string, string>; workspaces?: string[] | { packages?: string[] } }
 const readJson = (p: string): PkgJson | null => { try { return JSON.parse(readFileSync(p, 'utf8')) as PkgJson; } catch { return null; } };
@@ -67,9 +68,10 @@ export function localSwiftPackages(dir: string): string[] {
   const skip = new Set(['Pods', 'build', 'DerivedData', 'node_modules', '.build', 'SourcePackages', '.git', 'xcuserdata']);
   const walk = (d: string, depth: number) => {
     if (depth > 4) return;
-    let names: string[] = [];
-    try { names = readdirSync(d); } catch { return; }
-    for (const n of names) {
+    let ents: Dirent[] = [];
+    try { ents = readdirSync(d, { withFileTypes: true }); } catch { return; }
+    for (const e of ents) {
+      const n = e.name;
       if (skip.has(n)) continue;
       const p = join(d, n);
       if (n === 'Package.swift') {
@@ -79,7 +81,7 @@ export function localSwiftPackages(dir: string): string[] {
           const target = resolve(d, m[1]!);
           if (isDir(target)) out.push(target);
         }
-      } else if (isDir(p) && !/\.(xc)?framework$|\.xcassets$/.test(n)) walk(p, depth + 1);
+      } else if ((e.isDirectory() || (e.isSymbolicLink() && isDir(p))) && !/\.(xc)?framework$|\.xcassets$/.test(n) && !skipDir(dir, p, n)) walk(p, depth + 1);
     }
   };
   walk(dir, 0);

cli/lib/privacy-files.ts

@@ -2,9 +2,10 @@
 // .xcframework, SwiftPM checkouts, Capacitor / Cordova / React Native plugins in node_modules, a built .app's
 // Frameworks and bundles) and every PrivacyInfo.xcprivacy inside an .ipa or .app (XML or binary). Read-only; the
 // directory walk is bounded.
-import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { type Dirent, readdirSync, readFileSync, statSync } from 'node:fs';
 import { basename, join, relative, sep } from 'node:path';
 import { nativeIosPlugins } from './node-packages';
+import { skipDir } from './walk-skip';
 import { parseBinaryPlist } from './plist';
 import { type Manifest, manifestFromPlist, parseManifest, trackingIssue } from './privacy-manifest';
 import { zipEntries, zipRead } from './zip';
@@ -63,15 +64,16 @@ export function bundleTrackingIssues(list: BundleManifest[]): string[] {
 
 function walk(dir: string, visit: (p: string, isDir: boolean) => boolean | void, depth = 0, budget = { n: 60_000 }): void {
   if (depth > 16 || budget.n <= 0) return;
-  let names: string[] = [];
-  try { names = readdirSync(dir).sort(); } catch { return; }
-  for (const n of names) {
+  let ents: Dirent[] = [];
+  try { ents = readdirSync(dir, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); } catch { return; }
+  for (const e of ents) {
     if (--budget.n <= 0) return;
-    const p = join(dir, n);
-    let st;
-    try { st = statSync(p); } catch { continue; }
-    const descend = visit(p, st.isDirectory());
-    if (st.isDirectory() && descend !== false) walk(p, visit, depth + 1, budget);
+    const p = join(dir, e.name);
+    // the entry type comes with the listing (no stat per file: a stat each made a big tree take minutes on Windows)
+    let isDir = e.isDirectory();
+    if (e.isSymbolicLink()) { try { isDir = statSync(p).isDirectory(); } catch { continue; } }
+    const descend = visit(p, isDir);
+    if (isDir && descend !== false) walk(p, visit, depth + 1, budget);
   }
 }
 
@@ -102,7 +104,7 @@ export function projectSdkManifests(root: string): SdkManifest[] {
     try { out.push({ sdk, manifest: manifestFromBytes(relative(root, abs).split(sep).join('/'), readFileSync(abs)) }); } catch { /* unreadable */ }
   };
   walk(root, (p, isDir) => {
-    if (isDir) return !SKIP.has(basename(p));
+    if (isDir) return !SKIP.has(basename(p)) && !skipDir(root, p, basename(p));
     if (basename(p) !== 'PrivacyInfo.xcprivacy') return;
     const sdk = sdkOfProjectPath(relative(root, p).split(sep).join('/'));
     if (sdk) add(p, sdk);

cli/lib/scan/project.ts

@@ -3,9 +3,10 @@
 // Xcode INFOPLIST_KEY_* build settings, Expo app.json, Unity ProjectSettings and Godot export presets), entitlements,
 // privacy manifests, the Android manifest (permissions, services, target SDK) and the app's own source files.
 // Tolerant on purpose: an unreadable or odd file is skipped, never an exception.
-import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { type Dirent, readdirSync, readFileSync, statSync } from 'node:fs';
 import { basename, dirname, extname, join, relative, sep } from 'node:path';
 import { resolvePackage } from '../node-packages';
+import { skipDir } from '../walk-skip';
 import { isBuildCopyPath, type Manifest, parseManifest } from '../privacy-manifest';
 import { type DepRef, type Eco, matchesSdk, SDKS, type SdkDef } from './sdks';
 import { exportedIpas, ipaInfo } from '../xcode';
@@ -104,33 +105,32 @@ const SRC_EXT = new Set(['.swift', '.m', '.mm', '.h', '.kt', '.java', '.dart', '
 const MAX_FILES = 25_000;
 const MAX_TEXT = 1_500_000;
 
-/** A linked git worktree: its .git is a file pointing into <repo>/.git/worktrees/ (a submodule's points into modules/). */
-function isWorktree(dir: string): boolean {
-  try { return /^gitdir:.*[\\/]worktrees[\\/]/m.test(readFileSync(join(dir, '.git'), 'utf8')); } catch { return false; }
-}
-
 /** Every file under root (relative paths), skipping build output, dependency folders and VCS data. Native libraries
  *  (.so) of any size also go into `bins` when given (they are not read, only listed). */
 export function listFiles(root: string, bins?: string[]): string[] {
   const out: string[] = [];
   const walk = (dir: string, depth: number) => {
     if (depth > 14 || out.length >= MAX_FILES) return;
-    let names: string[];
-    try { names = readdirSync(dir); } catch { return; }
-    for (const n of names.sort()) {
+    let ents: Dirent[];
+    try { ents = readdirSync(dir, { withFileTypes: true }); } catch { return; }
+    for (const e of ents.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) {
+      const n = e.name;
       if (SKIP_DIRS.has(n) || n === '.DS_Store') continue;
       const p = join(dir, n);
-      let st;
-      try { st = statSync(p); } catch { continue; }
-      if (st.isDirectory()) {
+      // the entry type comes with the listing; only files (for their size) and links get a stat
+      let isDir = e.isDirectory();
+      if (e.isSymbolicLink()) { try { isDir = statSync(p).isDirectory(); } catch { continue; } }
+      if (isDir) {
         // compiled frameworks are SDK binaries, not app code
         if (/\.(xc)?framework$|\.xcassets$|\.bundle$|\.app$|\.dSYM$/.test(n)) continue;
-        // a git worktree inside the project (another checkout of the same repo; submodules stay in)
-        if (isWorktree(p)) continue;
+        // dot folders, git worktrees (other checkouts of the same repo; submodules stay in), git-ignored output
+        if (skipDir(root, p, n)) continue;
         walk(p, depth + 1);
       } else {
         if (bins && n.endsWith('.so')) bins.push(relative(root, p).split(sep).join('/'));
-        if (st.size <= MAX_TEXT) out.push(relative(root, p).split(sep).join('/'));
+        let size = 0;
+        try { size = statSync(p).size; } catch { continue; }
+        if (size <= MAX_TEXT) out.push(relative(root, p).split(sep).join('/'));
       }
       if (out.length >= MAX_FILES) return;
     }

cli/lib/walk-skip.ts

@@ -0,0 +1,42 @@
+// Which folders a project walk (scan, check's privacy manifests) leaves out, beyond each walker's own name list:
+//  - dot folders below the root (.harness-worktrees, .wt-head, .ota-out, .native-build, ...): tool state, caches and
+//    whole copies of the repo, never app sources;
+//  - linked git worktrees (another checkout of the same repo);
+//  - folders git ignores at the root (release-out/, artifacts/, ...), asked once per root with
+//    `git ls-files --others --ignored --exclude-standard --directory`; SDK folders the privacy checks read (Pods,
+//    Carthage, SourcePackages) and generated native projects (ios, android) stay in even when ignored.
+// Seen live: a game repo with several worktrees and release folders made `check` walk ~60 000 files for 2.5 minutes.
+import { readFileSync } from 'node:fs';
+import { join, relative, resolve, sep } from 'node:path';
+import * as rt from './rt';
+
+// kept even when git ignores them: SDK folders the privacy checks read, and generated native projects (Expo prebuild)
+const KEEP = new Set(['Pods', 'Carthage', 'SourcePackages', 'ios', 'android', 'macos']);
+const cache = new Map<string, Set<string>>();
+
+/** A linked git worktree: its .git is a file pointing into <repo>/.git/worktrees/ (a submodule's points into modules/). */
+export function isWorktree(dir: string): boolean {
+  try { return /^gitdir:.*[\\/]worktrees[\\/]/m.test(readFileSync(join(dir, '.git'), 'utf8')); } catch { return false; }
+}
+
+/** Root-relative folders (with "/" separators, no trailing slash) git ignores in this repo; empty outside git. */
+export function gitIgnoredDirs(root: string): Set<string> {
+  const key = resolve(root);
+  const hit = cache.get(key);
+  if (hit) return hit;
+  const out = new Set<string>();
+  try {
+    const r = rt.spawnSync(['git', 'ls-files', '--others', '--ignored', '--exclude-standard', '--directory', '-z'], { cwd: key, timeout: 15_000 });
+    if (r.success) for (const p of r.stdout.toString('utf8').split('\0')) if (p.endsWith('/')) out.add(p.slice(0, -1));
+  } catch { /* no git */ }
+  cache.set(key, out);
+  return out;
+}
+
+/** Leave this folder out of a walk that started at root? */
+export function skipDir(root: string, abs: string, name: string): boolean {
+  if (KEEP.has(name)) return false;
+  if (name.startsWith('.') && resolve(abs) !== resolve(root)) return true;
+  if (gitIgnoredDirs(root).has(relative(root, abs).split(sep).join('/'))) return true;
+  return isWorktree(abs);
+}

tests/walk-skip.test.ts

@@ -0,0 +1,29 @@
+import { afterAll, expect, test } from 'bun:test';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { projectSdkManifests } from '../cli/lib/privacy-files';
+import { listFiles } from '../cli/lib/scan/project';
+import { skipDir } from '../cli/lib/walk-skip';
+
+const root = mkdtempSync(join(tmpdir(), 'walk-'));
+afterAll(() => rmSync(root, { recursive: true, force: true }));
+const put = (rel: string, text = 'x') => { mkdirSync(join(root, rel, '..'), { recursive: true }); writeFileSync(join(root, rel), text); };
+
+test('dot folders, linked worktrees and SDK folders', () => {
+  put('.harness-worktrees/a/src/App.swift');
+  put('wt/.git', 'gitdir: /repo/.git/worktrees/wt\n');
+  put('wt/src/App.swift');
+  put('src/App.swift');
+  put('ios/Pods/Some/PrivacyInfo.xcprivacy', '<plist><dict/></plist>');
+  expect(skipDir(root, join(root, '.harness-worktrees'), '.harness-worktrees')).toBe(true);
+  expect(skipDir(root, join(root, 'wt'), 'wt')).toBe(true);
+  expect(skipDir(root, join(root, 'src'), 'src')).toBe(false);
+  expect(skipDir(root, join(root, 'ios', 'Pods'), 'Pods')).toBe(false);
+  const files = listFiles(root);
+  expect(files).toContain('src/App.swift');
+  expect(files.some(f => f.startsWith('.harness-worktrees/') || f.startsWith('wt/'))).toBe(false);
+  // worktree copies would add their SDK manifests a second time
+  put('.wt-head/ios/Pods/Some/PrivacyInfo.xcprivacy', '<plist><dict/></plist>');
+  expect(projectSdkManifests(root).map(m => m.manifest.path)).toEqual(['ios/Pods/Some/PrivacyInfo.xcprivacy']);
+});