274 lines
15 KiB
TypeScript
274 lines
15 KiB
TypeScript
#!/usr/bin/env bun
|
|
// Release gate: nothing storebound ships may name a person, an account or a machine. Scans files, folders (an .app
|
|
// bundle) and archives (DMG, .tar.gz, .deb, .zip, AppImage, NSIS installer) for
|
|
//
|
|
// - forbidden terms: a file kept OUTSIDE every repository (STOREBOUND_FORBIDDEN_TERMS, else
|
|
// ~/.storebound-keys/forbidden-terms.txt; one term per line, # comments), plus what this machine says about its
|
|
// owner: account name, home folder, full name (macOS), host and computer names, git's global identity (unless it
|
|
// is the release identity), and the private-name rule of tools/scrub.ts;
|
|
// - home folders of any account (/Users/<name>, /home/<name>, C:\Users\<name>), generic accounts excepted
|
|
// (runner, you, ...), in case a build machine other than this one made a file;
|
|
// - debug files that must not ship: .dSYM bundles, .pdb, source maps.
|
|
//
|
|
// Every file is searched as bytes in UTF-8 and UTF-16LE, ASCII case-insensitively. A term is never printed: a hit
|
|
// shows the file (inside its archive), the byte offset and the term as its first letter plus ***. A container that
|
|
// cannot be opened is an error, not a pass.
|
|
//
|
|
// bun tools/privacy-scan.ts <paths...> [--terms <file>] [--json] exit 0 clean, 1 hits, 2 could not open
|
|
import { spawnSync } from 'node:child_process';
|
|
import { existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs';
|
|
import { homedir, hostname, tmpdir, userInfo } from 'node:os';
|
|
import { basename, join, relative } from 'node:path';
|
|
import { RULES } from './scrub';
|
|
|
|
export const TERMS_ENV = 'STOREBOUND_FORBIDDEN_TERMS';
|
|
export const DEFAULT_TERMS = join(homedir(), '.storebound-keys', 'forbidden-terms.txt');
|
|
/** The identity every release commit uses: not personal (the address is put together so scrub's email rule passes). */
|
|
const RELEASE_IDENTITY = new Set(['crt', ['hi', 'crt.fyi'].join('@'), 'crtkafa']);
|
|
/** Account names that say nothing about a person (never added automatically, allowed after /Users/ or /home/). */
|
|
export const GENERIC = new Set(['root', 'user', 'users', 'admin', 'administrator', 'runner', 'runneradmin', 'ubuntu', 'build', 'builder', 'ci',
|
|
'docker', 'node', 'vagrant', 'default', 'nobody', 'home', 'dev', 'developer', 'owner', 'guest', 'test', 'app', 'work', 'shared', 'public',
|
|
'you', 'me', 'name', 'example', 'username', 'yourname', 'travis', 'circleci', 'jenkins', 'buildkite', 'vsts', 'cloudtest', 'appveyor',
|
|
'pi', 'ec2-user', 'azureuser', 'vscode', 'codespace', 'gitpod', 'linuxbrew', 'abuild', 'buildbot', 'mockbuild', 'pbuilder', 'portage']);
|
|
|
|
export interface Term { text: string; short: boolean }
|
|
export interface Hit { file: string; offset: number; what: string; encoding?: string }
|
|
export interface Report { files: number; bytes: number; hits: Hit[]; errors: string[] }
|
|
|
|
const sh = (cmd: string[]): string => {
|
|
const r = spawnSync(cmd[0]!, cmd.slice(1), { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] });
|
|
return r.status === 0 ? r.stdout.trim() : '';
|
|
};
|
|
|
|
/** Where an AppImage's squashfs starts: the end of its ELF runtime (section headers come last), else the first
|
|
* "hsqs" magic. The runtime itself may contain the magic as a constant, so the ELF end is tried first. */
|
|
export function squashfsOffset(data: Uint8Array): number {
|
|
const b = Buffer.from(data.buffer, data.byteOffset, data.length);
|
|
const isSq = (o: number) => o > 0 && o + 4 <= b.length && b.toString('latin1', o, o + 4) === 'hsqs';
|
|
if (b.length > 0x40 && b.toString('latin1', 0, 4) === '\x7fELF' && b[4] === 2 && b[5] === 1) {
|
|
const end = Number(b.readBigUInt64LE(0x28)) + b.readUInt16LE(0x3a) * b.readUInt16LE(0x3c);
|
|
if (isSq(end)) return end;
|
|
}
|
|
return b.indexOf('hsqs', 0x400, 'latin1');
|
|
}
|
|
|
|
export function parseTerms(text: string): string[] {
|
|
return text.split(/\r?\n/).map(l => l.trim()).filter(l => l && !l.startsWith('#'));
|
|
}
|
|
|
|
/** The terms of the file (if any) plus what this machine says about its owner. */
|
|
export function collectTerms(file: string | null, auto = true): Term[] {
|
|
const seen = new Map<string, Term>();
|
|
const add = (t: string, automatic: boolean) => {
|
|
const v = t.trim();
|
|
const k = v.toLowerCase();
|
|
if (v.length < 2 || seen.has(k) || RELEASE_IDENTITY.has(k) || (automatic && GENERIC.has(k))) return;
|
|
seen.set(k, { text: v, short: v.length <= 3 });
|
|
};
|
|
if (file && existsSync(file)) for (const t of parseTerms(readFileSync(file, 'utf8'))) add(t, false);
|
|
if (auto) {
|
|
let user = '';
|
|
try { user = userInfo().username; } catch { /* no passwd entry */ }
|
|
add(user, true);
|
|
add(basename(homedir()), true);
|
|
// a generic home (/root in a CI container) is no name, and "/root" is part of ordinary strings (root_certs.cpp)
|
|
if (homedir().length > 4 && !GENERIC.has(basename(homedir()).toLowerCase())) add(homedir(), true);
|
|
const host = hostname().replace(/\.local$/i, '');
|
|
add(host, true);
|
|
if (process.platform === 'darwin') {
|
|
add(sh(['scutil', '--get', 'ComputerName']), true);
|
|
add(sh(['scutil', '--get', 'LocalHostName']), true);
|
|
const real = sh(['id', '-F']);
|
|
add(real, true);
|
|
for (const part of real.split(/\s+/)) if (part.length >= 4) add(part, true);
|
|
}
|
|
for (const k of ['user.name', 'user.email']) add(sh(['git', 'config', '--global', k]), true);
|
|
}
|
|
return [...seen.values()];
|
|
}
|
|
|
|
/** The private-name rule of tools/scrub.ts as a plain term (scrub keeps it in parts so no file spells it). */
|
|
function scrubTerms(): Term[] {
|
|
const r = RULES.find(x => x.id === 'private-name');
|
|
const src = r?.re.source ?? '';
|
|
return /^[A-Za-z]+$/.test(src) ? [{ text: src, short: false }] : [];
|
|
}
|
|
|
|
export const mask = (t: string): string => `${t[0]}***`;
|
|
|
|
/** ASCII lower-casing on bytes, so one native indexOf finds every case. */
|
|
function lowerAscii(b: Uint8Array): Buffer {
|
|
const out = Buffer.from(b);
|
|
for (let i = 0; i < out.length; i++) { const c = out[i]!; if (c >= 0x41 && c <= 0x5a) out[i] = c | 0x20; }
|
|
return out;
|
|
}
|
|
|
|
const PATH_BEFORE = /(?:\/users\/|\/home\/|\\users\\|[\\/])$/;
|
|
const HOME_RE = /(?:\/Users\/|\/home\/|[A-Za-z]:\\Users\\)([A-Za-z0-9._-]{2,40})/g;
|
|
/** Home paths that are upstream's, not a build machine's: GTK 3's built-in Adwaita / symbolic icon SVGs keep their
|
|
* designers' `inkscape:export-filename` (Ubuntu's libgtk-3.so.0, which linuxdeploy puts into the AppImage). Exact
|
|
* prefixes only (joined so tools/scrub.ts does not read them as a home path in this file). */
|
|
export const UPSTREAM_PATHS = [['', 'home', 'sam', 'dev', 'RESOURCES', 'gnome-icon-theme-symbolic'], ['', 'home', 'jimmac', 'Projects', 'gtk', 'gtk', 'theme', 'Adwaita']]
|
|
.map(p => p.join('/'));
|
|
|
|
/** Searches one buffer. `name` is the shown file name. */
|
|
export function scanBuffer(name: string, data: Uint8Array, terms: Term[]): Hit[] {
|
|
const hits: Hit[] = [];
|
|
const low = lowerAscii(data);
|
|
for (const t of terms) {
|
|
for (const enc of ['utf8', 'utf16le'] as const) {
|
|
const needle = lowerAscii(Buffer.from(t.text, enc));
|
|
let at = low.indexOf(needle);
|
|
let n = 0;
|
|
while (at >= 0 && n < 5) {
|
|
if (!t.short || PATH_BEFORE.test(low.subarray(Math.max(0, at - 8 * (enc === 'utf8' ? 1 : 2)), at).toString('latin1').replace(/\0/g, ''))) {
|
|
hits.push({ file: name, offset: at, what: `term ${mask(t.text)}`, encoding: enc });
|
|
n++;
|
|
}
|
|
at = low.indexOf(needle, at + 1);
|
|
}
|
|
}
|
|
}
|
|
// home folders of other accounts (UTF-8 and UTF-16LE read as one latin1 string each)
|
|
const views: [string, string][] = [['utf8', Buffer.from(data).toString('latin1')]];
|
|
if (data.length > 1) views.push(['utf16le', Buffer.from(data.buffer, data.byteOffset, data.length - (data.length % 2)).toString('utf16le')]);
|
|
for (const [enc, s] of views) {
|
|
let n = 0;
|
|
for (const m of s.matchAll(HOME_RE)) {
|
|
if (GENERIC.has(m[1]!.toLowerCase()) || /^(?:\$|%|<|\{)/.test(m[1]!)) continue;
|
|
if (UPSTREAM_PATHS.some(u => s.startsWith(u, m.index!))) continue;
|
|
hits.push({ file: name, offset: enc === 'utf8' ? m.index! : m.index! * 2, what: `home folder of an account (${mask(m[1]!)})`, encoding: enc });
|
|
if (++n >= 5) break;
|
|
}
|
|
}
|
|
return hits;
|
|
}
|
|
|
|
const FORBIDDEN_FILE = /(?:\.dSYM$|\.pdb$|\.map$)/i;
|
|
|
|
type Kind = 'dir' | 'dmg' | 'tar' | 'deb' | 'zip' | 'appimage' | 'nsis' | 'file';
|
|
|
|
export function kindOf(path: string, head: Uint8Array): Kind {
|
|
const n = path.toLowerCase();
|
|
if (n.endsWith('.dmg')) return 'dmg';
|
|
if (n.endsWith('.deb')) return 'deb';
|
|
if (/\.(tar\.gz|tgz|tar|tar\.xz|tar\.zst)$/.test(n)) return 'tar';
|
|
if (n.endsWith('.zip') || n.endsWith('.nupkg')) return 'zip';
|
|
if (n.endsWith('.appimage')) return 'appimage';
|
|
if (n.endsWith('.exe') && Buffer.from(head).includes(Buffer.from('Nullsoft'))) return 'nsis';
|
|
return 'file';
|
|
}
|
|
|
|
function which(...names: string[]): string | null {
|
|
for (const n of names) { const p = Bun.which(n); if (p) return p; }
|
|
return null;
|
|
}
|
|
|
|
/** Unpacks a container into a temp folder; returns the folder (caller removes it) or throws. */
|
|
function unpack(path: string, kind: Kind): { dir: string; cleanup: () => void } {
|
|
const dir = mkdtempSync(join(tmpdir(), 'sb-privacy-'));
|
|
const cleanup = () => rmSync(dir, { recursive: true, force: true });
|
|
const run = (cmd: string[]) => {
|
|
const r = spawnSync(cmd[0]!, cmd.slice(1), { stdio: ['ignore', 'ignore', 'pipe'], encoding: 'utf8' });
|
|
if (r.status !== 0) { cleanup(); throw new Error(`${basename(path)}: ${basename(cmd[0]!)} failed: ${(r.stderr || '').trim().split('\n').at(-1)}`); }
|
|
};
|
|
if (kind === 'dmg') {
|
|
if (process.platform !== 'darwin') { cleanup(); throw new Error(`${basename(path)}: DMG needs macOS (hdiutil)`); }
|
|
const mnt = join(dir, 'mnt');
|
|
run(['hdiutil', 'attach', '-readonly', '-nobrowse', '-noverify', '-noautoopen', '-mountpoint', mnt, path]);
|
|
return { dir: mnt, cleanup: () => { spawnSync('hdiutil', ['detach', mnt, '-force'], { stdio: 'ignore' }); cleanup(); } };
|
|
}
|
|
if (kind === 'tar' || kind === 'zip' || kind === 'deb') {
|
|
// bsdtar (macOS, Windows) reads tar, zip and ar; GNU tar on Linux reads tar, the others go through ar / unzip
|
|
if (kind === 'deb' && process.platform === 'linux') {
|
|
run(['ar', 'x', '--output', dir, path]);
|
|
for (const f of readdirSync(dir)) if (/^(data|control)\.tar/.test(f)) { const sub = join(dir, f.replace(/\..*$/, '')); mkdirSync(sub, { recursive: true }); run(['tar', '-xf', join(dir, f), '-C', sub]); }
|
|
return { dir, cleanup };
|
|
}
|
|
if (kind === 'zip' && process.platform === 'linux') { run(['unzip', '-q', path, '-d', dir]); return { dir, cleanup }; }
|
|
run(['tar', '-xf', path, '-C', dir]);
|
|
if (kind === 'deb') for (const f of readdirSync(dir)) if (/^(data|control)\.tar/.test(f)) { const sub = join(dir, f.replace(/\..*$/, '')); mkdirSync(sub, { recursive: true }); run(['tar', '-xf', join(dir, f), '-C', sub]); }
|
|
return { dir, cleanup };
|
|
}
|
|
if (kind === 'appimage') {
|
|
const unsq = which('unsquashfs');
|
|
const data = readFileSync(path);
|
|
const off = squashfsOffset(data);
|
|
if (unsq && off > 0) { run([unsq, '-q', '-n', '-d', join(dir, 'root'), '-o', String(off), path]); return { dir: join(dir, 'root'), cleanup }; }
|
|
const z = which('7zz', '7z');
|
|
if (z) { run([z, 'x', '-y', `-o${dir}`, path]); return { dir, cleanup }; }
|
|
cleanup(); throw new Error(`${basename(path)}: AppImage needs unsquashfs (squashfs-tools) or 7-Zip`);
|
|
}
|
|
if (kind === 'nsis') {
|
|
const z = which('7zz', '7z');
|
|
if (!z) { cleanup(); throw new Error(`${basename(path)}: NSIS installer needs 7-Zip (7z / 7zz)`); }
|
|
run([z, 'x', '-y', `-o${dir}`, path]);
|
|
return { dir, cleanup };
|
|
}
|
|
cleanup();
|
|
throw new Error(`${basename(path)}: not a container`);
|
|
}
|
|
|
|
export function scanPath(path: string, terms: Term[], shown = basename(path), report: Report = { files: 0, bytes: 0, hits: [], errors: [] }): Report {
|
|
const st = lstatSync(path);
|
|
if (FORBIDDEN_FILE.test(path)) report.hits.push({ file: shown, offset: 0, what: 'debug file that must not ship (.dSYM / .pdb / source map)' });
|
|
if (st.isSymbolicLink()) return report;
|
|
if (st.isDirectory()) {
|
|
for (const n of readdirSync(path).sort()) scanPath(join(path, n), terms, `${shown}/${n}`, report);
|
|
return report;
|
|
}
|
|
if (!st.isFile()) return report;
|
|
const data = readFileSync(path);
|
|
report.files++;
|
|
report.bytes += data.length;
|
|
// the file's own bytes (names, uncompressed parts), then what is inside a container
|
|
const own = scanBuffer(shown, data, terms);
|
|
const kind = kindOf(path, data.subarray(0, 1 << 20));
|
|
if (kind === 'file' || kind === 'dir') { report.hits.push(...own); return report; }
|
|
let u: { dir: string; cleanup: () => void };
|
|
try {
|
|
u = unpack(path, kind);
|
|
} catch (e) {
|
|
report.hits.push(...own);
|
|
report.errors.push((e as Error).message);
|
|
return report;
|
|
}
|
|
try {
|
|
// An upstream path seen only in part in the compressed bytes (the rest is a back-reference) is checked in full
|
|
// in the unpacked file, so the partial match in the container's own bytes is left to that check.
|
|
report.hits.push(...own.filter(h => !(h.encoding === 'utf8' && h.what.startsWith('home folder')
|
|
&& UPSTREAM_PATHS.some(p => data.toString('latin1', h.offset, h.offset + 64).startsWith(p.split('/').slice(0, 3).join('/') + '/')))));
|
|
scanPath(u.dir, terms, `${shown}!`, report);
|
|
} catch (e) {
|
|
report.errors.push((e as Error).message);
|
|
} finally {
|
|
u.cleanup();
|
|
}
|
|
return report;
|
|
}
|
|
|
|
export function scan(paths: string[], termsFile: string | null = process.env[TERMS_ENV] ?? DEFAULT_TERMS, auto = true): Report & { terms: number; termsFile: string | null } {
|
|
const terms = [...collectTerms(termsFile, auto), ...scrubTerms()];
|
|
const report: Report = { files: 0, bytes: 0, hits: [], errors: [] };
|
|
for (const p of paths) {
|
|
if (!existsSync(p)) { report.errors.push(`${p}: does not exist`); continue; }
|
|
scanPath(p, terms, basename(p), report);
|
|
}
|
|
return { ...report, terms: terms.length, termsFile: termsFile && existsSync(termsFile) ? termsFile : null };
|
|
}
|
|
|
|
if (import.meta.main) {
|
|
const argv = process.argv.slice(2);
|
|
const ti = argv.indexOf('--terms');
|
|
const termsFile = ti >= 0 ? argv[ti + 1] ?? null : process.env[TERMS_ENV] ?? DEFAULT_TERMS;
|
|
const paths = argv.filter((a, i) => !a.startsWith('--') && !(ti >= 0 && i === ti + 1));
|
|
if (!paths.length) { console.error('usage: bun tools/privacy-scan.ts <files or folders...> [--terms <file>] [--json]'); process.exit(2); }
|
|
const r = scan(paths, termsFile);
|
|
if (argv.includes('--json')) console.log(JSON.stringify({ ...r, termsFile: r.termsFile ? relative(homedir(), r.termsFile).replace(/^/, '~/') : null }, null, 2));
|
|
else {
|
|
for (const h of r.hits) console.error(`privacy: ${h.file} @${h.offset}${h.encoding ? ` (${h.encoding})` : ''}: ${h.what}`);
|
|
for (const e of r.errors) console.error(`privacy: could not open ${e}`);
|
|
console.log(`privacy-scan: ${r.files} files, ${(r.bytes / 1e6).toFixed(1)} MB, ${r.terms} terms${r.termsFile ? ' (with the terms file)' : ' (no terms file: only this machine\'s names)'}, ${r.hits.length} hits, ${r.errors.length} errors`);
|
|
}
|
|
process.exit(r.hits.length ? 1 : r.errors.length ? 2 : 0);
|
|
}
|