diff --git a/CHANGELOG.md b/CHANGELOG.md index c0fea25..cd423a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased +- `check` and `scan` are fast on big repos: project walks skip dot folders (tool state, worktree copies), linked git worktrees and the folders git ignores (Pods, Carthage, SourcePackages and ios / android stay in), and read entry types from the listing instead of a stat per file. A game repo with worktree copies went from 244 s to 46 s for `check`. - `storebound mcp --http`: one Streamable HTTP MCP server that every agent session of a machine shares (no process per session). Each tool gets a `project` argument (absolute path inside `--roots`); auth is a local bearer token (`~/.storebound/mcp-token`, made on first start) or, for a server behind TLS, OAuth bearer tokens checked by introspection (RFC 7662) with protected resource metadata (RFC 9728). docs/mcp.md. ## 0.8.57 ยท 2026-10-27 diff --git a/cli/lib/node-packages.ts b/cli/lib/node-packages.ts index ac44c79..ba58bb9 100644 --- a/cli/lib/node-packages.ts +++ b/cli/lib/node-packages.ts @@ -3,8 +3,9 @@ // folders (package.json "workspaces", npm / yarn / pnpm-style array or { packages }, as Bun reads them) and local // SwiftPM packages (`.package(path:)` in a Package.swift, e.g. Capacitor 8's CapApp-SPM). Used by `privacy-manifest` // and `scan`. Read-only; never throws. -import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs'; +import { type Dirent, existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; +import { skipDir } from './walk-skip'; interface PkgJson { name?: string; dependencies?: Record; workspaces?: string[] | { packages?: string[] } } const readJson = (p: string): PkgJson | null => { try { return JSON.parse(readFileSync(p, 'utf8')) as PkgJson; } catch { return null; } }; @@ -67,9 +68,10 @@ export function localSwiftPackages(dir: string): string[] { const skip = new Set(['Pods', 'build', 'DerivedData', 'node_modules', '.build', 'SourcePackages', '.git', 'xcuserdata']); const walk = (d: string, depth: number) => { if (depth > 4) return; - let names: string[] = []; - try { names = readdirSync(d); } catch { return; } - for (const n of names) { + let ents: Dirent[] = []; + try { ents = readdirSync(d, { withFileTypes: true }); } catch { return; } + for (const e of ents) { + const n = e.name; if (skip.has(n)) continue; const p = join(d, n); if (n === 'Package.swift') { @@ -79,7 +81,7 @@ export function localSwiftPackages(dir: string): string[] { const target = resolve(d, m[1]!); if (isDir(target)) out.push(target); } - } else if (isDir(p) && !/\.(xc)?framework$|\.xcassets$/.test(n)) walk(p, depth + 1); + } else if ((e.isDirectory() || (e.isSymbolicLink() && isDir(p))) && !/\.(xc)?framework$|\.xcassets$/.test(n) && !skipDir(dir, p, n)) walk(p, depth + 1); } }; walk(dir, 0); diff --git a/cli/lib/privacy-files.ts b/cli/lib/privacy-files.ts index 779b942..94a3450 100644 --- a/cli/lib/privacy-files.ts +++ b/cli/lib/privacy-files.ts @@ -2,9 +2,10 @@ // .xcframework, SwiftPM checkouts, Capacitor / Cordova / React Native plugins in node_modules, a built .app's // Frameworks and bundles) and every PrivacyInfo.xcprivacy inside an .ipa or .app (XML or binary). Read-only; the // directory walk is bounded. -import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { type Dirent, readdirSync, readFileSync, statSync } from 'node:fs'; import { basename, join, relative, sep } from 'node:path'; import { nativeIosPlugins } from './node-packages'; +import { skipDir } from './walk-skip'; import { parseBinaryPlist } from './plist'; import { type Manifest, manifestFromPlist, parseManifest, trackingIssue } from './privacy-manifest'; import { zipEntries, zipRead } from './zip'; @@ -63,15 +64,16 @@ export function bundleTrackingIssues(list: BundleManifest[]): string[] { function walk(dir: string, visit: (p: string, isDir: boolean) => boolean | void, depth = 0, budget = { n: 60_000 }): void { if (depth > 16 || budget.n <= 0) return; - let names: string[] = []; - try { names = readdirSync(dir).sort(); } catch { return; } - for (const n of names) { + let ents: Dirent[] = []; + try { ents = readdirSync(dir, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); } catch { return; } + for (const e of ents) { if (--budget.n <= 0) return; - const p = join(dir, n); - let st; - try { st = statSync(p); } catch { continue; } - const descend = visit(p, st.isDirectory()); - if (st.isDirectory() && descend !== false) walk(p, visit, depth + 1, budget); + const p = join(dir, e.name); + // the entry type comes with the listing (no stat per file: a stat each made a big tree take minutes on Windows) + let isDir = e.isDirectory(); + if (e.isSymbolicLink()) { try { isDir = statSync(p).isDirectory(); } catch { continue; } } + const descend = visit(p, isDir); + if (isDir && descend !== false) walk(p, visit, depth + 1, budget); } } @@ -102,7 +104,7 @@ export function projectSdkManifests(root: string): SdkManifest[] { try { out.push({ sdk, manifest: manifestFromBytes(relative(root, abs).split(sep).join('/'), readFileSync(abs)) }); } catch { /* unreadable */ } }; walk(root, (p, isDir) => { - if (isDir) return !SKIP.has(basename(p)); + if (isDir) return !SKIP.has(basename(p)) && !skipDir(root, p, basename(p)); if (basename(p) !== 'PrivacyInfo.xcprivacy') return; const sdk = sdkOfProjectPath(relative(root, p).split(sep).join('/')); if (sdk) add(p, sdk); diff --git a/cli/lib/scan/project.ts b/cli/lib/scan/project.ts index f0b8f8b..01dc57a 100644 --- a/cli/lib/scan/project.ts +++ b/cli/lib/scan/project.ts @@ -3,9 +3,10 @@ // Xcode INFOPLIST_KEY_* build settings, Expo app.json, Unity ProjectSettings and Godot export presets), entitlements, // privacy manifests, the Android manifest (permissions, services, target SDK) and the app's own source files. // Tolerant on purpose: an unreadable or odd file is skipped, never an exception. -import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { type Dirent, readdirSync, readFileSync, statSync } from 'node:fs'; import { basename, dirname, extname, join, relative, sep } from 'node:path'; import { resolvePackage } from '../node-packages'; +import { skipDir } from '../walk-skip'; import { isBuildCopyPath, type Manifest, parseManifest } from '../privacy-manifest'; import { type DepRef, type Eco, matchesSdk, SDKS, type SdkDef } from './sdks'; import { exportedIpas, ipaInfo } from '../xcode'; @@ -104,33 +105,32 @@ const SRC_EXT = new Set(['.swift', '.m', '.mm', '.h', '.kt', '.java', '.dart', ' const MAX_FILES = 25_000; const MAX_TEXT = 1_500_000; -/** A linked git worktree: its .git is a file pointing into /.git/worktrees/ (a submodule's points into modules/). */ -function isWorktree(dir: string): boolean { - try { return /^gitdir:.*[\\/]worktrees[\\/]/m.test(readFileSync(join(dir, '.git'), 'utf8')); } catch { return false; } -} - /** Every file under root (relative paths), skipping build output, dependency folders and VCS data. Native libraries * (.so) of any size also go into `bins` when given (they are not read, only listed). */ export function listFiles(root: string, bins?: string[]): string[] { const out: string[] = []; const walk = (dir: string, depth: number) => { if (depth > 14 || out.length >= MAX_FILES) return; - let names: string[]; - try { names = readdirSync(dir); } catch { return; } - for (const n of names.sort()) { + let ents: Dirent[]; + try { ents = readdirSync(dir, { withFileTypes: true }); } catch { return; } + for (const e of ents.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) { + const n = e.name; if (SKIP_DIRS.has(n) || n === '.DS_Store') continue; const p = join(dir, n); - let st; - try { st = statSync(p); } catch { continue; } - if (st.isDirectory()) { + // the entry type comes with the listing; only files (for their size) and links get a stat + let isDir = e.isDirectory(); + if (e.isSymbolicLink()) { try { isDir = statSync(p).isDirectory(); } catch { continue; } } + if (isDir) { // compiled frameworks are SDK binaries, not app code if (/\.(xc)?framework$|\.xcassets$|\.bundle$|\.app$|\.dSYM$/.test(n)) continue; - // a git worktree inside the project (another checkout of the same repo; submodules stay in) - if (isWorktree(p)) continue; + // dot folders, git worktrees (other checkouts of the same repo; submodules stay in), git-ignored output + if (skipDir(root, p, n)) continue; walk(p, depth + 1); } else { if (bins && n.endsWith('.so')) bins.push(relative(root, p).split(sep).join('/')); - if (st.size <= MAX_TEXT) out.push(relative(root, p).split(sep).join('/')); + let size = 0; + try { size = statSync(p).size; } catch { continue; } + if (size <= MAX_TEXT) out.push(relative(root, p).split(sep).join('/')); } if (out.length >= MAX_FILES) return; } diff --git a/cli/lib/walk-skip.ts b/cli/lib/walk-skip.ts new file mode 100644 index 0000000..2015fb9 --- /dev/null +++ b/cli/lib/walk-skip.ts @@ -0,0 +1,42 @@ +// Which folders a project walk (scan, check's privacy manifests) leaves out, beyond each walker's own name list: +// - dot folders below the root (.harness-worktrees, .wt-head, .ota-out, .native-build, ...): tool state, caches and +// whole copies of the repo, never app sources; +// - linked git worktrees (another checkout of the same repo); +// - folders git ignores at the root (release-out/, artifacts/, ...), asked once per root with +// `git ls-files --others --ignored --exclude-standard --directory`; SDK folders the privacy checks read (Pods, +// Carthage, SourcePackages) and generated native projects (ios, android) stay in even when ignored. +// Seen live: a game repo with several worktrees and release folders made `check` walk ~60 000 files for 2.5 minutes. +import { readFileSync } from 'node:fs'; +import { join, relative, resolve, sep } from 'node:path'; +import * as rt from './rt'; + +// kept even when git ignores them: SDK folders the privacy checks read, and generated native projects (Expo prebuild) +const KEEP = new Set(['Pods', 'Carthage', 'SourcePackages', 'ios', 'android', 'macos']); +const cache = new Map>(); + +/** A linked git worktree: its .git is a file pointing into /.git/worktrees/ (a submodule's points into modules/). */ +export function isWorktree(dir: string): boolean { + try { return /^gitdir:.*[\\/]worktrees[\\/]/m.test(readFileSync(join(dir, '.git'), 'utf8')); } catch { return false; } +} + +/** Root-relative folders (with "/" separators, no trailing slash) git ignores in this repo; empty outside git. */ +export function gitIgnoredDirs(root: string): Set { + const key = resolve(root); + const hit = cache.get(key); + if (hit) return hit; + const out = new Set(); + try { + const r = rt.spawnSync(['git', 'ls-files', '--others', '--ignored', '--exclude-standard', '--directory', '-z'], { cwd: key, timeout: 15_000 }); + if (r.success) for (const p of r.stdout.toString('utf8').split('\0')) if (p.endsWith('/')) out.add(p.slice(0, -1)); + } catch { /* no git */ } + cache.set(key, out); + return out; +} + +/** Leave this folder out of a walk that started at root? */ +export function skipDir(root: string, abs: string, name: string): boolean { + if (KEEP.has(name)) return false; + if (name.startsWith('.') && resolve(abs) !== resolve(root)) return true; + if (gitIgnoredDirs(root).has(relative(root, abs).split(sep).join('/'))) return true; + return isWorktree(abs); +} diff --git a/tests/walk-skip.test.ts b/tests/walk-skip.test.ts new file mode 100644 index 0000000..0dea615 --- /dev/null +++ b/tests/walk-skip.test.ts @@ -0,0 +1,29 @@ +import { afterAll, expect, test } from 'bun:test'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { projectSdkManifests } from '../cli/lib/privacy-files'; +import { listFiles } from '../cli/lib/scan/project'; +import { skipDir } from '../cli/lib/walk-skip'; + +const root = mkdtempSync(join(tmpdir(), 'walk-')); +afterAll(() => rmSync(root, { recursive: true, force: true })); +const put = (rel: string, text = 'x') => { mkdirSync(join(root, rel, '..'), { recursive: true }); writeFileSync(join(root, rel), text); }; + +test('dot folders, linked worktrees and SDK folders', () => { + put('.harness-worktrees/a/src/App.swift'); + put('wt/.git', 'gitdir: /repo/.git/worktrees/wt\n'); + put('wt/src/App.swift'); + put('src/App.swift'); + put('ios/Pods/Some/PrivacyInfo.xcprivacy', ''); + expect(skipDir(root, join(root, '.harness-worktrees'), '.harness-worktrees')).toBe(true); + expect(skipDir(root, join(root, 'wt'), 'wt')).toBe(true); + expect(skipDir(root, join(root, 'src'), 'src')).toBe(false); + expect(skipDir(root, join(root, 'ios', 'Pods'), 'Pods')).toBe(false); + const files = listFiles(root); + expect(files).toContain('src/App.swift'); + expect(files.some(f => f.startsWith('.harness-worktrees/') || f.startsWith('wt/'))).toBe(false); + // worktree copies would add their SDK manifests a second time + put('.wt-head/ios/Pods/Some/PrivacyInfo.xcprivacy', ''); + expect(projectSdkManifests(root).map(m => m.manifest.path)).toEqual(['ios/Pods/Some/PrivacyInfo.xcprivacy']); +});