177 lines
11 KiB
TypeScript
177 lines
11 KiB
TypeScript
#!/usr/bin/env bun
|
|
// Everything CI runs, as a small task graph: independent steps run at the same time, output is streamed with a
|
|
// [step] prefix, the first failure stops everything, and a summary prints the time per step.
|
|
// bun run ci full gate before pushing (every check, every test)
|
|
// bun run ci:quick inner loop: typecheck + the tests related to the change (git diff against origin/main,
|
|
// mapped through imports) + scrub
|
|
// The tests run as shards: N = CPU cores - 1 (max 8; STOREBOUND_CI_SHARDS=n overrides) parallel `bun test <files>`
|
|
// processes, balanced by the durations recorded after earlier runs (node_modules/.cache/storebound-ci/, smoothed by
|
|
// mergeTimings so one run on a loaded machine does not skew the split; falling back to file size). Browser suites
|
|
// (tools/ci-plan.ts HEAVY) each get a shard of their own and run their opted-in tests BROWSER_CONCURRENCY at a time.
|
|
// `bun test` covers the CLI and the desktop webview (apps/desktop/tests), the site overflow check and the
|
|
// no-hand-written-SVG guard; the desktop boot check (tools/desktop-boot-check.ts: the built window in headless
|
|
// Chromium, needs playwright's browser or Google Chrome) runs after the desktop build.
|
|
// The desktop Rust host (cargo test, clippy with warnings as errors) runs when cargo is installed and
|
|
// apps/desktop/src-tauri or the sidecar build changed against origin/main, or always with CI=full;
|
|
// STOREBOUND_CI_RUST=0 skips it on a machine without Rust.
|
|
import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
import { homedir, tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { hostTarget, OUT_DIR, outName } from './build-sidecars';
|
|
import { BROWSER_CONCURRENCY, changedFiles, HEAVY, junitTimes, listTests, mergeTimings, planShards, readTimings, relatedTests, ROOT, rustNeeded, writeTimings } from './ci-plan';
|
|
|
|
interface Task { name: string; cmd: string[]; cwd?: string; env?: Record<string, string>; after?: string[]; junit?: string }
|
|
|
|
const quick = process.argv.includes('--quick');
|
|
const tauri = join(ROOT, 'apps/desktop/src-tauri');
|
|
const cache = join(ROOT, 'node_modules/.cache/storebound-ci');
|
|
const changed = changedFiles();
|
|
const tests = listTests();
|
|
const touches = (re: RegExp) => changed === null || changed.some(f => re.test(f));
|
|
const tasks: Task[] = [];
|
|
const add = (t: Task) => tasks.push(t);
|
|
const tsc = (name: string, project: string) => add({ name, cmd: ['bunx', 'tsc', '--noEmit', '--incremental', '-p', project,
|
|
'--tsBuildInfoFile', join(cache, `${name.replace(/\W+/g, '-')}.tsbuildinfo`)] });
|
|
const junitDir = mkdtempSync(join(tmpdir(), 'sb-ci-'));
|
|
|
|
function addTests(files: string[]) {
|
|
planShards(files).forEach((shard, i, all) => {
|
|
const junit = join(junitDir, `shard-${i + 1}.xml`);
|
|
// the built-bundle check in apps/desktop/tests/webview.test.ts runs as "desktop bundle check" once the build is
|
|
// done; here it would read dist/ while vite rewrites it
|
|
// browser suites share the machine with the other steps: a browser start or close can pass bun's 5 s default
|
|
// browser tests that opt in (testScope().concurrent) run BROWSER_CONCURRENCY at a time within their file
|
|
const timeout = shard.some(f => HEAVY.includes(f)) ? ['--timeout=30000', `--max-concurrency=${BROWSER_CONCURRENCY}`] : [];
|
|
add({ name: `tests ${i + 1}/${all.length}`, cmd: ['bun', 'test', ...timeout, '--reporter=junit', `--reporter-outfile=${junit}`, ...shard.map(f => `./${f}`)],
|
|
env: { STOREBOUND_SKIP_DIST_CHECK: '1' }, junit });
|
|
});
|
|
}
|
|
|
|
if (quick) {
|
|
const related = changed === null ? tests : relatedTests(changed, tests, ROOT, HEAVY);
|
|
const far = changed === null ? [] : relatedTests(changed, tests).filter(t => !related.includes(t));
|
|
console.log(`ci:quick: ${changed === null ? 'no origin/main to diff against: every test' : `${changed.length} changed file(s), ${related.length} related test file(s)`}`);
|
|
if (far.length) console.log(`ci:quick: left out ${far.length} browser suite(s) reached only through shared modules (bun run ci runs them): ${far.join(' ')}`);
|
|
add({ name: 'adapters', cmd: ['bun', 'tools/gen-adapters.ts', '--check'] });
|
|
tsc('typecheck', '.');
|
|
if (touches(/^apps\/desktop\//)) tsc('desktop typecheck', 'apps/desktop');
|
|
if (related.length) addTests(related);
|
|
add({ name: 'scrub', cmd: ['bun', 'tools/scrub.ts'] });
|
|
} else {
|
|
const cargo = process.env.STOREBOUND_CI_RUST === '0' ? null : Bun.which('cargo');
|
|
const rust = cargo && rustNeeded(changed);
|
|
if (!cargo) console.log('ci: cargo not found (or STOREBOUND_CI_RUST=0): desktop Rust tests and clippy skipped');
|
|
else if (!rust) console.log('ci: apps/desktop/src-tauri and the sidecar build unchanged against origin/main: desktop Rust tests and clippy skipped (CI=full runs them)');
|
|
add({ name: 'adapters', cmd: ['bun', 'tools/gen-adapters.ts', '--check'] });
|
|
// the App Store Connect / Google Play operation tables, the wrapped-by scan and docs/api-coverage.md match specs/ and cli/
|
|
add({ name: 'api tables', cmd: ['bun', 'tools/gen-api.ts', '--check'] });
|
|
// one desktop version: package.json desktopVersion = apps/desktop/package.json = tauri.conf.json = Cargo.toml / .lock
|
|
add({ name: 'desktop version', cmd: ['bun', 'tools/desktop-version.ts'] });
|
|
add({ name: 'desktop i18n', cmd: ['bun', 'tools/i18n-check.ts'] });
|
|
tsc('typecheck', '.');
|
|
tsc('desktop typecheck', 'apps/desktop');
|
|
add({ name: 'scrub', cmd: ['bun', 'tools/scrub.ts'] });
|
|
addTests(tests);
|
|
// `vite build` only: the desktop package's own build script runs tsc first, which "desktop typecheck" already does
|
|
add({ name: 'desktop build', cmd: ['bunx', 'vite', 'build', '--logLevel', 'warn'], cwd: join(ROOT, 'apps/desktop') });
|
|
// the release bundle must not carry the dev fixture mode
|
|
add({ name: 'desktop bundle check', cmd: ['bun', 'test', './apps/desktop/tests/webview.test.ts'], after: ['desktop build'] });
|
|
// the built window boots (release CSP, Tauri's frozen Object.prototype, mocked host + real sidecar) on an empty, a
|
|
// D1 (folders gone) and a full D3 saved state, through every screen: no uncaught error, never an empty window
|
|
add({ name: 'desktop boot check', cmd: ['bun', 'tools/desktop-boot-check.ts'], after: ['desktop build'] });
|
|
// the CLI compiled into the desktop sidecar (bun build --compile, scanned for this machine's paths): rebuilt when
|
|
// its inputs changed, with CI=full, or when missing; tauri's build script needs it for cargo test / clippy
|
|
const hostBin = join(OUT_DIR, outName(hostTarget()));
|
|
const sidecarBuild = process.env.CI === 'full' || !existsSync(hostBin) || touches(/^(cli\/|tools\/build-sidecars\.ts$|package\.json$|bun\.lock$)/);
|
|
if (sidecarBuild) add({ name: 'desktop sidecar (host)', cmd: ['bun', 'tools/build-sidecars.ts', '--host'] });
|
|
if (cargo && rust) {
|
|
const after = sidecarBuild ? ['desktop sidecar (host)'] : [];
|
|
// cargo locks the target dir, so test and clippy run one after the other
|
|
add({ name: 'desktop rust tests', cmd: [cargo, 'test', '--quiet'], cwd: tauri, after });
|
|
add({ name: 'desktop clippy', cmd: [cargo, 'clippy', '--all-targets', '--quiet', '--', '-D', 'warnings'], cwd: tauri, after: ['desktop rust tests'] });
|
|
}
|
|
}
|
|
|
|
// The browser suites fall back to Google Chrome without playwright's own headless Chromium; Chrome's shutdown takes
|
|
// seconds to half a minute per browser, which makes those shards several times slower.
|
|
if (tasks.some(t => t.name === 'desktop boot check' || t.cmd.some(a => HEAVY.some(h => a.endsWith(h))))) try {
|
|
const { registry } = await import('playwright-core/lib/server/registry/index' as string);
|
|
const exe = registry.findExecutable('chromium-headless-shell')?.executablePath('javascript') as string | undefined;
|
|
if (exe && !existsSync(exe)) console.log(`ci: playwright's headless Chromium is not installed (${exe.replace(homedir(), '~')}): the browser tests use Google Chrome, which is much slower to close. Install it once: bunx playwright install chromium-headless-shell`);
|
|
} catch { /* playwright internals moved: no hint */ }
|
|
|
|
// ---- runner ----
|
|
|
|
interface Run { task: Task; start: number; end?: number; code?: number; proc?: ReturnType<typeof Bun.spawn> }
|
|
const runs = new Map<string, Run>();
|
|
const t0 = Date.now();
|
|
const width = Math.max(...tasks.map(t => t.name.length));
|
|
const tag = (n: string) => `[${n.padEnd(width)}]`;
|
|
let failed: string | null = null;
|
|
|
|
async function pump(stream: ReadableStream<Uint8Array>, name: string, out: NodeJS.WriteStream) {
|
|
const dec = new TextDecoder();
|
|
let buf = '';
|
|
const reader = stream.getReader();
|
|
for (;;) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
if (failed && failed !== name) continue; // a stopped step's last words are noise
|
|
buf += dec.decode(value, { stream: true });
|
|
const lines = buf.split('\n');
|
|
buf = lines.pop()!;
|
|
if (lines.length) out.write(lines.map(l => `${tag(name)} ${l}\n`).join(''));
|
|
}
|
|
if (buf && !(failed && failed !== name)) out.write(`${tag(name)} ${buf}\n`);
|
|
}
|
|
|
|
async function start(task: Task): Promise<void> {
|
|
const run: Run = { task, start: Date.now() };
|
|
runs.set(task.name, run);
|
|
console.log(`${tag(task.name)} $ ${task.cmd.join(' ').replace(junitDir, '$TMP')}`);
|
|
const proc = Bun.spawn(task.cmd, { cwd: task.cwd ?? ROOT, stdout: 'pipe', stderr: 'pipe', env: { ...process.env, FORCE_COLOR: '0', ...task.env } });
|
|
run.proc = proc;
|
|
await Promise.all([pump(proc.stdout, task.name, process.stdout), pump(proc.stderr, task.name, process.stderr), proc.exited]);
|
|
run.code = proc.exitCode ?? 1;
|
|
run.end = Date.now();
|
|
if (failed) return;
|
|
console.log(`${tag(task.name)} ${run.code === 0 ? 'ok' : `FAILED (exit ${run.code})`} in ${((run.end - run.start) / 1000).toFixed(1)}s`);
|
|
if (run.code !== 0) {
|
|
failed = task.name;
|
|
for (const r of runs.values()) if (r.end === undefined) r.proc?.kill('SIGKILL'); // bun test ignores SIGTERM until its files finish
|
|
}
|
|
}
|
|
|
|
const pending = new Set(tasks);
|
|
const active = new Set<Promise<void>>();
|
|
while ((pending.size || active.size) && !failed) {
|
|
for (const t of [...pending]) {
|
|
if ((t.after ?? []).every(a => runs.get(a)?.code === 0)) {
|
|
pending.delete(t);
|
|
const p = start(t).finally(() => active.delete(p));
|
|
active.add(p);
|
|
}
|
|
}
|
|
if (!active.size) { failed = 'scheduler (unmet dependency)'; break; }
|
|
await Promise.race(active);
|
|
}
|
|
if (failed) await Promise.allSettled([...active]);
|
|
|
|
// record per-file durations for the next split (only from shards that finished)
|
|
const fresh: Record<string, number> = {};
|
|
for (const r of runs.values()) {
|
|
if (!r.task.junit || r.end === undefined) continue;
|
|
try { Object.assign(fresh, junitTimes(readFileSync(r.task.junit, 'utf8'))); } catch { /* no report */ }
|
|
}
|
|
writeTimings(mergeTimings(readTimings(), fresh));
|
|
rmSync(junitDir, { recursive: true, force: true });
|
|
|
|
console.log(`\n${quick ? 'ci:quick' : 'ci'} summary (wall ${((Date.now() - t0) / 1000).toFixed(1)}s)`);
|
|
for (const r of [...runs.values()].sort((a, b) => a.start - b.start)) {
|
|
const s = r.end === undefined ? 'killed' : r.code === 0 ? 'ok' : r.task.name === failed ? 'FAILED' : 'stopped';
|
|
const secs = (((r.end ?? Date.now()) - r.start) / 1000).toFixed(1).padStart(6);
|
|
console.log(` ${r.task.name.padEnd(width)} ${secs}s @${((r.start - t0) / 1000).toFixed(1).padStart(5)}s ${s}`);
|
|
}
|
|
for (const t of pending) console.log(` ${t.name.padEnd(width)} - not started`);
|
|
if (failed) { console.error(`\nci: ${failed} failed`); process.exit(1); }
|
|
console.log(`\n${quick ? 'ci:quick' : 'ci'}: all green`);
|