CI green on a loaded shared machine: browser suites (now also glyph-ligatures and feature-fit, through the shared launch helper) in one dedicated shard started first at concurrency 2; timeouts scaled by load ÷ cores (tests/helpers/load.ts, max(1-, 5-min average), capped at 5x, logged) for browser tests, waits, launch/close, the other shards' default and the desktop boot check; one retry for a browser test that timed out while the factor was above 2, listed as 'retried under load' in the CI summary, never for an assertion failure; the six-type App Privacy walk no longer dumps every step (the --debug-dump check walks two types), wait timeouts print per-line times; play-edit-lock checks reads in flight instead of wall time; tests/load-retry.test.ts

tests/helpers/browser.ts

@@ -15,11 +15,14 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { test } from 'bun:test';
+import { LOAD, loadFactor, loadNote, RETRY_LOAD, RETRY_MARK, scaled } from './load';
+
+export { LOAD, scaled } from './load';
 
 /** Launching Chromium: well above a cold start on a loaded machine (seconds), far below "hung". */
-export const LAUNCH_TIMEOUT_MS = 60_000;
+export const LAUNCH_TIMEOUT_MS = scaled(60_000);
 /** One context close, or an engine run ending after its window closed; longer is abandoned (and the browser killed). */
-export const CLOSE_TIMEOUT_MS = 15_000;
+export const CLOSE_TIMEOUT_MS = scaled(15_000);
 /**
  * Hook timeouts. afterEach: a test's contexts close in parallel, each capped at CLOSE_TIMEOUT_MS, then the engine runs
  * it started, each capped at CLOSE_TIMEOUT_MS. afterAll: disconnect (capped) and kill the browser server.
@@ -59,6 +62,7 @@ export interface TestBrowser {
  */
 export async function launchTestBrowser(pkg: 'playwright' | 'playwright-core'): Promise<TestBrowser> {
   const none = (e: unknown): TestBrowser => ({ browser: null, skipWhy: firstLine(e), close: async () => {} });
+  console.log(`browser tests: ${loadNote()}`);
   let chromium: any;
   try { chromium = (await import(pkg)).chromium; } catch (e) { return none(e); }
   const opts = { headless: true, timeout: LAUNCH_TIMEOUT_MS };
@@ -140,30 +144,64 @@ export function testScope(browser: () => any) {
     track: <T>(ctx: T): T => cur().track(ctx),
     run: <T>(p: Promise<T>): Promise<T> => cur().run(p),
     settle: (): Promise<void> => shared.settle(),
+    /** `ms`: the test's budget, already scaled by the load (scaled(), tests/helpers/load.ts). A run that fails by timing
+     *  out (a wait, a Playwright timeout or the budget itself) while the load factor is above RETRY_LOAD runs once
+     *  more in a fresh scope and prints RETRY_MARK; an assertion failure is never retried. */
     concurrent(name: string, fn: () => Promise<void>, ms: number): void {
-      test.concurrent(name, () => {
+      const attempt = () => {
         const s = plainScope(browser);
-        return own.run(s, async () => { try { await fn(); } finally { await s.settle(); } });
-      }, ms + AFTER_EACH_MS);
+        return own.run(s, () => withBudget(fn, ms, () => s.settle()));
+      };
+      test.concurrent(name, () => retryUnderLoad(name, attempt), 2 * (ms + AFTER_EACH_MS));
     },
   };
 }
 
+/** A wait that ran out of time (logLines().wait, a test's budget): the one kind of failure a loaded machine may cause. */
+export class WaitTimeout extends Error { override name = 'WaitTimeout'; }
+/** A failure by timing out (ours or Playwright's), never an assertion. */
+export const isTimeout = (e: unknown): boolean => e instanceof WaitTimeout || (e as Error | null)?.name === 'TimeoutError';
+
+/** Runs `fn` within `ms` (else fails with a WaitTimeout), then `after` (the test's settle) either way. */
+export async function withBudget(fn: () => Promise<void>, ms: number, after: () => Promise<void> = async () => {}): Promise<void> {
+  let t: ReturnType<typeof setTimeout> | undefined;
+  const over = new Promise<never>((_, reject) => { t = setTimeout(() => reject(new WaitTimeout(`test budget of ${ms} ms used up`)), ms); });
+  try { await Promise.race([fn(), over]); } finally { clearTimeout(t); await after(); }
+}
+
+/** Runs `attempt`; when it fails by timing out while the load factor is above RETRY_LOAD, says RETRY_MARK (tools/ci.ts
+ *  lists those in its summary) and runs it once more. Any other failure, or a timeout on a quiet machine, is final. */
+export async function retryUnderLoad(name: string, attempt: () => Promise<void>, factor: () => number = loadFactor, say: (m: string) => void = m => console.error(m)): Promise<void> {
+  try { await attempt(); } catch (e) {
+    const f = factor();
+    if (!isTimeout(e) || f <= RETRY_LOAD) throw e;
+    // the whole error under the mark line: a wait's timeout lists every log line with its time (where it went)
+    const rest = (e as Error).message.split('\n').slice(1).map(l => `  ${l}`).join('\n');
+    say(`${RETRY_MARK} ${name} (load x${f}): ${firstLine(e)}${rest ? `\n${rest}` : ''}`);
+    await attempt();
+  }
+}
+
 /**
  * The log lines of an engine run (`say`), with event-driven waits: `wait` resolves on the line that makes `pred`
  * true (checked on every new line, no polling) and fails after `ms` with every line seen so far.
  */
 export function logLines() {
   const lines: string[] = [];
+  const at: number[] = [];
+  const t0 = Date.now();
   const waiters = new Set<() => void>();
+  /** The lines with the seconds since the log started (a timeout dump shows where the time went). */
+  const timed = () => lines.map((l, i) => `+${((at[i]! - t0) / 1000).toFixed(1)}s ${l}`).join('\n');
   return {
     lines,
-    say: (m: string) => { lines.push(m); for (const w of [...waiters]) w(); },
-    wait(what: string, pred: (lines: string[]) => boolean, ms = 30_000): Promise<void> {
+    say: (m: string) => { lines.push(m); at.push(Date.now()); for (const w of [...waiters]) w(); },
+    /** `ms`: already scaled by the caller (scaled()); the default is 30 s scaled by the load. */
+    wait(what: string, pred: (lines: string[]) => boolean, ms = scaled(30_000)): Promise<void> {
       if (pred(lines)) return Promise.resolve();
       return new Promise<void>((resolve, reject) => {
         const check = () => { if (pred(lines)) { done(); resolve(); } };
-        const t = setTimeout(() => { done(); reject(new Error(`timed out after ${ms} ms waiting for: ${what}\n${lines.join('\n')}`)); }, ms);
+        const t = setTimeout(() => { done(); reject(new WaitTimeout(`timed out after ${ms} ms waiting for: ${what}\n${timed()}`)); }, ms);
         const done = () => { clearTimeout(t); waiters.delete(check); };
         waiters.add(check);
       });