CI green on a loaded shared machine: browser suites (now also glyph-ligatures and feature-fit, through the shared launch helper) in one dedicated shard started first at concurrency 2; timeouts scaled by load ÷ cores (tests/helpers/load.ts, max(1-, 5-min average), capped at 5x, logged) for browser tests, waits, launch/close, the other shards' default and the desktop boot check; one retry for a browser test that timed out while the factor was above 2, listed as 'retried under load' in the CI summary, never for an assertion failure; the six-type App Privacy walk no longer dumps every step (the --debug-dump check walks two types), wait timeouts print per-line times; play-edit-lock checks reads in flight instead of wall time; tests/load-retry.test.ts
This commit is contained in:
parent
21fa91afd1
commit
918146d4a6
13 changed files with 261 additions and 96 deletions
26
tests/helpers/load.ts
Normal file
26
tests/helpers/load.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
// Load-aware timeouts for the browser tests and the CI runner. The machine is shared: other sessions' builds push the
|
||||
// load average to 100-500 on 10 cores, and a browser walk that takes 5 s idle then takes 30-60 s. Fixed wall-clock
|
||||
// timeouts turned that into red CI runs that only passed on a quiet machine. Timeouts scale by load ÷ cores (at least
|
||||
// 1, at most LOAD_CAP), so they still catch a hang; a timeout under heavy load gets ONE retry (testScope().concurrent),
|
||||
// an assertion failure never does.
|
||||
import { cpus, loadavg } from 'node:os';
|
||||
|
||||
export const LOAD_CAP = 5;
|
||||
/** Above this factor a test that failed by timing out is retried once ("retried under load" in the CI summary). */
|
||||
export const RETRY_LOAD = 2;
|
||||
/** The line a retried test prints; tools/ci.ts collects these for its summary. */
|
||||
export const RETRY_MARK = 'RETRIED UNDER LOAD:';
|
||||
|
||||
/** max(1-min, 5-min load average) ÷ cores, between 1 and cap. STOREBOUND_LOAD_FACTOR=n overrides (tests, debugging). */
|
||||
export function loadFactor(env = process.env, load = loadavg(), cores = cpus().length, cap = LOAD_CAP): number {
|
||||
const forced = Number(env.STOREBOUND_LOAD_FACTOR);
|
||||
if (Number.isFinite(forced) && forced > 0) return Math.min(cap, Math.max(1, forced));
|
||||
const per = Math.max(load[0] ?? 0, load[1] ?? 0) / Math.max(1, cores);
|
||||
return Math.min(cap, Math.max(1, Math.round(per * 10) / 10));
|
||||
}
|
||||
|
||||
/** The factor of this test process, sampled once when the first browser test file loads, so a test's waits and its
|
||||
* own timeout scale by the same number. */
|
||||
export const LOAD = loadFactor();
|
||||
export const scaled = (ms: number, f = LOAD): number => Math.round(ms * f);
|
||||
export const loadNote = (f = LOAD): string => `load ${loadavg().map(n => n.toFixed(0)).join('/')} on ${cpus().length} cores: timeouts x${f}`;
|
||||
Loading…
Add table
Add a link
Reference in a new issue