CI green on a loaded shared machine: browser suites (now also glyph-ligatures and feature-fit, through the shared launch helper) in one dedicated shard started first at concurrency 2; timeouts scaled by load ÷ cores (tests/helpers/load.ts, max(1-, 5-min average), capped at 5x, logged) for browser tests, waits, launch/close, the other shards' default and the desktop boot check; one retry for a browser test that timed out while the factor was above 2, listed as 'retried under load' in the CI summary, never for an assertion failure; the six-type App Privacy walk no longer dumps every step (the --debug-dump check walks two types), wait timeouts print per-line times; play-edit-lock checks reads in flight instead of wall time; tests/load-retry.test.ts

tests/helpers/browser.ts

@@ -15,11 +15,14 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { test } from 'bun:test';
+import { LOAD, loadFactor, loadNote, RETRY_LOAD, RETRY_MARK, scaled } from './load';
+
+export { LOAD, scaled } from './load';
 
 /** Launching Chromium: well above a cold start on a loaded machine (seconds), far below "hung". */
-export const LAUNCH_TIMEOUT_MS = 60_000;
+export const LAUNCH_TIMEOUT_MS = scaled(60_000);
 /** One context close, or an engine run ending after its window closed; longer is abandoned (and the browser killed). */
-export const CLOSE_TIMEOUT_MS = 15_000;
+export const CLOSE_TIMEOUT_MS = scaled(15_000);
 /**
  * Hook timeouts. afterEach: a test's contexts close in parallel, each capped at CLOSE_TIMEOUT_MS, then the engine runs
  * it started, each capped at CLOSE_TIMEOUT_MS. afterAll: disconnect (capped) and kill the browser server.
@@ -59,6 +62,7 @@ export interface TestBrowser {
  */
 export async function launchTestBrowser(pkg: 'playwright' | 'playwright-core'): Promise<TestBrowser> {
   const none = (e: unknown): TestBrowser => ({ browser: null, skipWhy: firstLine(e), close: async () => {} });
+  console.log(`browser tests: ${loadNote()}`);
   let chromium: any;
   try { chromium = (await import(pkg)).chromium; } catch (e) { return none(e); }
   const opts = { headless: true, timeout: LAUNCH_TIMEOUT_MS };
@@ -140,30 +144,64 @@ export function testScope(browser: () => any) {
     track: <T>(ctx: T): T => cur().track(ctx),
     run: <T>(p: Promise<T>): Promise<T> => cur().run(p),
     settle: (): Promise<void> => shared.settle(),
+    /** `ms`: the test's budget, already scaled by the load (scaled(), tests/helpers/load.ts). A run that fails by timing
+     *  out (a wait, a Playwright timeout or the budget itself) while the load factor is above RETRY_LOAD runs once
+     *  more in a fresh scope and prints RETRY_MARK; an assertion failure is never retried. */
     concurrent(name: string, fn: () => Promise<void>, ms: number): void {
-      test.concurrent(name, () => {
+      const attempt = () => {
         const s = plainScope(browser);
-        return own.run(s, async () => { try { await fn(); } finally { await s.settle(); } });
-      }, ms + AFTER_EACH_MS);
+        return own.run(s, () => withBudget(fn, ms, () => s.settle()));
+      };
+      test.concurrent(name, () => retryUnderLoad(name, attempt), 2 * (ms + AFTER_EACH_MS));
     },
   };
 }
 
+/** A wait that ran out of time (logLines().wait, a test's budget): the one kind of failure a loaded machine may cause. */
+export class WaitTimeout extends Error { override name = 'WaitTimeout'; }
+/** A failure by timing out (ours or Playwright's), never an assertion. */
+export const isTimeout = (e: unknown): boolean => e instanceof WaitTimeout || (e as Error | null)?.name === 'TimeoutError';
+
+/** Runs `fn` within `ms` (else fails with a WaitTimeout), then `after` (the test's settle) either way. */
+export async function withBudget(fn: () => Promise<void>, ms: number, after: () => Promise<void> = async () => {}): Promise<void> {
+  let t: ReturnType<typeof setTimeout> | undefined;
+  const over = new Promise<never>((_, reject) => { t = setTimeout(() => reject(new WaitTimeout(`test budget of ${ms} ms used up`)), ms); });
+  try { await Promise.race([fn(), over]); } finally { clearTimeout(t); await after(); }
+}
+
+/** Runs `attempt`; when it fails by timing out while the load factor is above RETRY_LOAD, says RETRY_MARK (tools/ci.ts
+ *  lists those in its summary) and runs it once more. Any other failure, or a timeout on a quiet machine, is final. */
+export async function retryUnderLoad(name: string, attempt: () => Promise<void>, factor: () => number = loadFactor, say: (m: string) => void = m => console.error(m)): Promise<void> {
+  try { await attempt(); } catch (e) {
+    const f = factor();
+    if (!isTimeout(e) || f <= RETRY_LOAD) throw e;
+    // the whole error under the mark line: a wait's timeout lists every log line with its time (where it went)
+    const rest = (e as Error).message.split('\n').slice(1).map(l => `  ${l}`).join('\n');
+    say(`${RETRY_MARK} ${name} (load x${f}): ${firstLine(e)}${rest ? `\n${rest}` : ''}`);
+    await attempt();
+  }
+}
+
 /**
  * The log lines of an engine run (`say`), with event-driven waits: `wait` resolves on the line that makes `pred`
  * true (checked on every new line, no polling) and fails after `ms` with every line seen so far.
  */
 export function logLines() {
   const lines: string[] = [];
+  const at: number[] = [];
+  const t0 = Date.now();
   const waiters = new Set<() => void>();
+  /** The lines with the seconds since the log started (a timeout dump shows where the time went). */
+  const timed = () => lines.map((l, i) => `+${((at[i]! - t0) / 1000).toFixed(1)}s ${l}`).join('\n');
   return {
     lines,
-    say: (m: string) => { lines.push(m); for (const w of [...waiters]) w(); },
-    wait(what: string, pred: (lines: string[]) => boolean, ms = 30_000): Promise<void> {
+    say: (m: string) => { lines.push(m); at.push(Date.now()); for (const w of [...waiters]) w(); },
+    /** `ms`: already scaled by the caller (scaled()); the default is 30 s scaled by the load. */
+    wait(what: string, pred: (lines: string[]) => boolean, ms = scaled(30_000)): Promise<void> {
       if (pred(lines)) return Promise.resolve();
       return new Promise<void>((resolve, reject) => {
         const check = () => { if (pred(lines)) { done(); resolve(); } };
-        const t = setTimeout(() => { done(); reject(new Error(`timed out after ${ms} ms waiting for: ${what}\n${lines.join('\n')}`)); }, ms);
+        const t = setTimeout(() => { done(); reject(new WaitTimeout(`timed out after ${ms} ms waiting for: ${what}\n${timed()}`)); }, ms);
         const done = () => { clearTimeout(t); waiters.delete(check); };
         waiters.add(check);
       });

tests/helpers/load.ts

@@ -0,0 +1,26 @@
+// Load-aware timeouts for the browser tests and the CI runner. The machine is shared: other sessions' builds push the
+// load average to 100-500 on 10 cores, and a browser walk that takes 5 s idle then takes 30-60 s. Fixed wall-clock
+// timeouts turned that into red CI runs that only passed on a quiet machine. Timeouts scale by load ÷ cores (at least
+// 1, at most LOAD_CAP), so they still catch a hang; a timeout under heavy load gets ONE retry (testScope().concurrent),
+// an assertion failure never does.
+import { cpus, loadavg } from 'node:os';
+
+export const LOAD_CAP = 5;
+/** Above this factor a test that failed by timing out is retried once ("retried under load" in the CI summary). */
+export const RETRY_LOAD = 2;
+/** The line a retried test prints; tools/ci.ts collects these for its summary. */
+export const RETRY_MARK = 'RETRIED UNDER LOAD:';
+
+/** max(1-min, 5-min load average) ÷ cores, between 1 and cap. STOREBOUND_LOAD_FACTOR=n overrides (tests, debugging). */
+export function loadFactor(env = process.env, load = loadavg(), cores = cpus().length, cap = LOAD_CAP): number {
+  const forced = Number(env.STOREBOUND_LOAD_FACTOR);
+  if (Number.isFinite(forced) && forced > 0) return Math.min(cap, Math.max(1, forced));
+  const per = Math.max(load[0] ?? 0, load[1] ?? 0) / Math.max(1, cores);
+  return Math.min(cap, Math.max(1, Math.round(per * 10) / 10));
+}
+
+/** The factor of this test process, sampled once when the first browser test file loads, so a test's waits and its
+ *  own timeout scale by the same number. */
+export const LOAD = loadFactor();
+export const scaled = (ms: number, f = LOAD): number => Math.round(ms * f);
+export const loadNote = (f = LOAD): string => `load ${loadavg().map(n => n.toFixed(0)).join('/')} on ${cpus().length} cores: timeouts x${f}`;