storebound/tools/ci.ts

#!/usr/bin/env bun
// Everything CI runs, as a small task graph: independent steps run at the same time, output is streamed with a
// [step] prefix, the first failure stops everything, and a summary prints the time per step.
//   bun run ci          full gate before pushing (every check, every test)
//   bun run ci:quick    inner loop: typecheck + the tests related to the change (git diff against origin/main,
//                       mapped through imports) + scrub
// The tests run as shards: N = CPU cores - 1 (max 8; STOREBOUND_CI_SHARDS=n overrides) parallel `bun test <files>`
// processes, balanced by the durations recorded after earlier runs (node_modules/.cache/storebound-ci/, smoothed by
// mergeTimings so one run on a loaded machine does not skew the split; falling back to file size). The browser suites
// (tools/ci-plan.ts HEAVY) run in one dedicated shard, started first, their opted-in tests BROWSER_CONCURRENCY at a
// time. The machine is shared: their timeouts scale with load ÷ cores (tests/helpers/load.ts, capped at 5x, the factor
// is logged), and a browser test that fails by timing out under load (factor > 2) runs once more; the summary lists
// those as "retried under load". An assertion failure is never retried.
// `bun test` covers the CLI and the desktop webview (apps/desktop/tests), the site overflow check and the
// no-hand-written-SVG guard; the desktop boot check (tools/desktop-boot-check.ts: the built window in headless
// Chromium, needs playwright's browser or Google Chrome) runs after the desktop build.
// The desktop Rust host (cargo test, clippy with warnings as errors) runs when cargo is installed and
// apps/desktop/src-tauri or the sidecar build changed against origin/main, or always with CI=full;
// STOREBOUND_CI_RUST=0 skips it on a machine without Rust.
import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
import { cpus, homedir, loadavg, tmpdir } from 'node:os';
import { join } from 'node:path';
import { hostTarget, OUT_DIR, outName } from './build-sidecars';
import { loadFactor, loadNote, RETRY_MARK, scaled } from '../tests/helpers/load';
import { BROWSER_CONCURRENCY, changedFiles, deferBrowser, HEAVY, junitTimes, listTests, mergeTimings, planShards, readTimings, relatedTests, ROOT, rustNeeded, writeTimings } from './ci-plan';

interface Task { name: string; cmd: string[]; cwd?: string; env?: Record<string, string>; after?: string[]; junit?: string }

const quick = process.argv.includes('--quick');
const tauri = join(ROOT, 'apps/desktop/src-tauri');
const cache = join(ROOT, 'node_modules/.cache/storebound-ci');
const changed = changedFiles();
const deferred = deferBrowser();
const allTests = listTests();
const tests = deferred ? allTests.filter(f => !HEAVY.includes(f)) : allTests;
if (deferred) console.log(`ci: browser suites deferred to Forgejo CI (load ${loadavg()[0]!.toFixed(0)} > 2 x ${cpus().length} cores): ${allTests.filter(f => HEAVY.includes(f)).join(' ')} and the desktop boot check. CI=full runs them here.`);
const touches = (re: RegExp) => changed === null || changed.some(f => re.test(f));
const tasks: Task[] = [];
const add = (t: Task) => tasks.push(t);
const tsc = (name: string, project: string) => add({ name, cmd: ['bunx', 'tsc', '--noEmit', '--incremental', '-p', project,
  '--tsBuildInfoFile', join(cache, `${name.replace(/\W+/g, '-')}.tsbuildinfo`)] });
const retried: string[] = [];
console.log(`ci: ${loadNote(loadFactor())}`);
const junitDir = mkdtempSync(join(tmpdir(), 'sb-ci-'));
const node = Bun.which('node');

/** The npm bundle (dist/storebound.mjs, Node 20+) and its smoke test with the real `node` (tools/node-smoke.ts). */
function addNode() {
  add({ name: 'node bundle', cmd: ['bun', 'tools/build-node.ts'] });
  if (node) add({ name: 'node smoke', cmd: ['bun', 'tools/node-smoke.ts', '--node', node], after: ['node bundle'] });
  else console.log('ci: no `node` on PATH: the Node smoke test of the npm bundle is skipped');
}

function addTests(files: string[]) {
  planShards(files).forEach((shard, i, all) => {
    const junit = join(junitDir, `shard-${i + 1}.xml`);
    // the built-bundle check in apps/desktop/tests/webview.test.ts runs as "desktop bundle check" once the build is
    // done; here it would read dist/ while vite rewrites it
    // browser suites share the machine with the other steps: a browser start or close can pass bun's 5 s default
    // (scaled by the load now); browser tests that opt in (testScope().concurrent) run BROWSER_CONCURRENCY at a time
    // every other shard: bun's 5 s default (tests and hooks without their own timeout), scaled the same way
    const f = loadFactor();
    const timeout = shard.some(x => HEAVY.includes(x)) ? [`--timeout=${scaled(30_000, f)}`, `--max-concurrency=${BROWSER_CONCURRENCY}`] : [`--timeout=${scaled(5_000, f)}`];
    add({ name: `tests ${i + 1}/${all.length}`, cmd: ['bun', 'test', ...timeout, '--reporter=junit', `--reporter-outfile=${junit}`, ...shard.map(f => `./${f}`)],
      env: { STOREBOUND_SKIP_DIST_CHECK: '1' }, junit });
  });
}

if (quick) {
  const related = changed === null ? tests : relatedTests(changed, tests, ROOT, HEAVY);
  const far = changed === null ? [] : relatedTests(changed, tests).filter(t => !related.includes(t));
  console.log(`ci:quick: ${changed === null ? 'no origin/main to diff against: every test' : `${changed.length} changed file(s), ${related.length} related test file(s)`}`);
  if (far.length) console.log(`ci:quick: left out ${far.length} browser suite(s) reached only through shared modules (bun run ci runs them): ${far.join(' ')}`);
  add({ name: 'adapters', cmd: ['bun', 'tools/gen-adapters.ts', '--check'] });
  add({ name: 'embedded', cmd: ['bun', 'tools/gen-embedded.ts', '--check'] });
  tsc('typecheck', '.');
  if (touches(/^(cli\/|templates\/|tools\/(build-node|node-smoke|gen-embedded)\.ts$|package\.json$)/)) addNode();
  if (touches(/^apps\/desktop\//)) tsc('desktop typecheck', 'apps/desktop');
  if (related.length) addTests(related);
  add({ name: 'scrub', cmd: ['bun', 'tools/scrub.ts'] });
} else {
  const cargo = process.env.STOREBOUND_CI_RUST === '0' ? null : Bun.which('cargo');
  const rust = cargo && rustNeeded(changed);
  if (!cargo) console.log('ci: cargo not found (or STOREBOUND_CI_RUST=0): desktop Rust tests and clippy skipped');
  else if (!rust) console.log('ci: apps/desktop/src-tauri and the sidecar build unchanged against origin/main: desktop Rust tests and clippy skipped (CI=full runs them)');
  add({ name: 'adapters', cmd: ['bun', 'tools/gen-adapters.ts', '--check'] });
  // templates/, cli/shots/*.html and Play's form export, embedded in cli/lib/embedded.ts (every runtime and build)
  add({ name: 'embedded', cmd: ['bun', 'tools/gen-embedded.ts', '--check'] });
  addNode();
  // the App Store Connect / Google Play operation tables, the wrapped-by scan and docs/api-coverage.md match specs/ and cli/
  add({ name: 'api tables', cmd: ['bun', 'tools/gen-api.ts', '--check'] });
  // one desktop version: package.json desktopVersion = apps/desktop/package.json = tauri.conf.json = Cargo.toml / .lock
  add({ name: 'desktop version', cmd: ['bun', 'tools/desktop-version.ts'] });
  add({ name: 'desktop i18n', cmd: ['bun', 'tools/i18n-check.ts'] });
  tsc('typecheck', '.');
  tsc('desktop typecheck', 'apps/desktop');
  add({ name: 'scrub', cmd: ['bun', 'tools/scrub.ts'] });
  addTests(tests);
  // `vite build` only: the desktop package's own build script runs tsc first, which "desktop typecheck" already does
  add({ name: 'desktop build', cmd: ['bunx', 'vite', 'build', '--logLevel', 'warn'], cwd: join(ROOT, 'apps/desktop') });
  // the release bundle must not carry the dev fixture mode
  add({ name: 'desktop bundle check', cmd: ['bun', 'test', './apps/desktop/tests/webview.test.ts'], after: ['desktop build'] });
  // the built window boots (release CSP, Tauri's frozen Object.prototype, mocked host + real sidecar) on an empty, a
  // D1 (folders gone) and a full D3 saved state, through every screen: no uncaught error, never an empty window
  if (!deferred) add({ name: 'desktop boot check', cmd: ['bun', 'tools/desktop-boot-check.ts'], after: ['desktop build'] });
  // the CLI compiled into the desktop sidecar (bun build --compile, scanned for this machine's paths): rebuilt when
  // its inputs changed, with CI=full, or when missing; tauri's build script needs it for cargo test / clippy
  const hostBin = join(OUT_DIR, outName(hostTarget()));
  const sidecarBuild = process.env.CI === 'full' || !existsSync(hostBin) || touches(/^(cli\/|tools\/build-sidecars\.ts$|package\.json$|bun\.lock$)/);
  if (sidecarBuild) add({ name: 'desktop sidecar (host)', cmd: ['bun', 'tools/build-sidecars.ts', '--host'] });
  if (cargo && rust) {
    const after = sidecarBuild ? ['desktop sidecar (host)'] : [];
    // cargo locks the target dir, so test and clippy run one after the other
    add({ name: 'desktop rust tests', cmd: [cargo, 'test', '--quiet'], cwd: tauri, after });
    add({ name: 'desktop clippy', cmd: [cargo, 'clippy', '--all-targets', '--quiet', '--', '-D', 'warnings'], cwd: tauri, after: ['desktop rust tests'] });
  }
}

// The browser suites fall back to Google Chrome without playwright's own headless Chromium; Chrome's shutdown takes
// seconds to half a minute per browser, which makes those shards several times slower.
if (tasks.some(t => t.name === 'desktop boot check' || t.cmd.some(a => HEAVY.some(h => a.endsWith(h))))) try {
  const { registry } = await import('playwright-core/lib/server/registry/index' as string);
  const exe = registry.findExecutable('chromium-headless-shell')?.executablePath('javascript') as string | undefined;
  if (exe && !existsSync(exe)) console.log(`ci: playwright's headless Chromium is not installed (${exe.replace(homedir(), '~')}): the browser tests use Google Chrome, which is much slower to close. Install it once: bunx playwright install chromium-headless-shell`);
} catch { /* playwright internals moved: no hint */ }

// ---- runner ----

interface Run { task: Task; start: number; end?: number; code?: number; proc?: ReturnType<typeof Bun.spawn> }
const runs = new Map<string, Run>();
const t0 = Date.now();
const width = Math.max(...tasks.map(t => t.name.length));
const tag = (n: string) => `[${n.padEnd(width)}]`;
let failed: string | null = null;

async function pump(stream: ReadableStream<Uint8Array>, name: string, out: NodeJS.WriteStream) {
  const dec = new TextDecoder();
  let buf = '';
  const reader = stream.getReader();
  for (;;) {
    const { done, value } = await reader.read();
    if (done) break;
    if (failed && failed !== name) continue; // a stopped step's last words are noise
    buf += dec.decode(value, { stream: true });
    const lines = buf.split('\n');
    buf = lines.pop()!;
    for (const l of lines) if (l.includes(RETRY_MARK)) retried.push(`${name}: ${l.slice(l.indexOf(RETRY_MARK) + RETRY_MARK.length).trim()}`);
    if (lines.length) out.write(lines.map(l => `${tag(name)} ${l}\n`).join(''));
  }
  if (buf && !(failed && failed !== name)) out.write(`${tag(name)} ${buf}\n`);
}

async function start(task: Task): Promise<void> {
  const run: Run = { task, start: Date.now() };
  runs.set(task.name, run);
  console.log(`${tag(task.name)} $ ${task.cmd.join(' ').replace(junitDir, '$TMP')}`);
  const proc = Bun.spawn(task.cmd, { cwd: task.cwd ?? ROOT, stdout: 'pipe', stderr: 'pipe', env: { ...process.env, FORCE_COLOR: '0', ...task.env } });
  run.proc = proc;
  await Promise.all([pump(proc.stdout, task.name, process.stdout), pump(proc.stderr, task.name, process.stderr), proc.exited]);
  run.code = proc.exitCode ?? 1;
  run.end = Date.now();
  if (failed) return;
  console.log(`${tag(task.name)} ${run.code === 0 ? 'ok' : `FAILED (exit ${run.code})`} in ${((run.end - run.start) / 1000).toFixed(1)}s`);
  if (run.code !== 0) {
    failed = task.name;
    for (const r of runs.values()) if (r.end === undefined) r.proc?.kill('SIGKILL'); // bun test ignores SIGTERM until its files finish
  }
}

const pending = new Set(tasks);
const active = new Set<Promise<void>>();
while ((pending.size || active.size) && !failed) {
  for (const t of [...pending]) {
    if ((t.after ?? []).every(a => runs.get(a)?.code === 0)) {
      pending.delete(t);
      const p = start(t).finally(() => active.delete(p));
      active.add(p);
    }
  }
  if (!active.size) { failed = 'scheduler (unmet dependency)'; break; }
  await Promise.race(active);
}
if (failed) await Promise.allSettled([...active]);

// record per-file durations for the next split (only from shards that finished)
const fresh: Record<string, number> = {};
for (const r of runs.values()) {
  if (!r.task.junit || r.end === undefined) continue;
  try { Object.assign(fresh, junitTimes(readFileSync(r.task.junit, 'utf8'))); } catch { /* no report */ }
}
writeTimings(mergeTimings(readTimings(), fresh));
rmSync(junitDir, { recursive: true, force: true });

console.log(`\n${quick ? 'ci:quick' : 'ci'} summary (wall ${((Date.now() - t0) / 1000).toFixed(1)}s)`);
for (const r of [...runs.values()].sort((a, b) => a.start - b.start)) {
  const s = r.end === undefined ? 'killed' : r.code === 0 ? 'ok' : r.task.name === failed ? 'FAILED' : 'stopped';
  const secs = (((r.end ?? Date.now()) - r.start) / 1000).toFixed(1).padStart(6);
  console.log(`  ${r.task.name.padEnd(width)} ${secs}s  @${((r.start - t0) / 1000).toFixed(1).padStart(5)}s  ${s}`);
}
for (const t of pending) console.log(`  ${t.name.padEnd(width)}      -   not started`);
if (retried.length) console.log(`\nretried under load (${retried.length}; each timed out once while load ÷ cores was above 2, then ran again):\n${retried.map(r => `  ${r}`).join('\n')}`);
if (deferred) console.log(`\nci: browser suites deferred to Forgejo CI (load ${loadavg()[0]!.toFixed(0)}); check its run on main`);
if (failed) { console.error(`\nci: ${failed} failed`); process.exit(1); }
console.log(`\n${quick ? 'ci:quick' : 'ci'}: all green`);