26 lines
1.7 KiB
TypeScript
26 lines
1.7 KiB
TypeScript
// Load-aware timeouts for the browser tests and the CI runner. The machine is shared: other sessions' builds push the
|
|
// load average to 100-500 on 10 cores, and a browser walk that takes 5 s idle then takes 30-60 s. Fixed wall-clock
|
|
// timeouts turned that into red CI runs that only passed on a quiet machine. Timeouts scale by load ÷ cores (at least
|
|
// 1, at most LOAD_CAP), so they still catch a hang; a timeout under heavy load gets ONE retry (testScope().concurrent),
|
|
// an assertion failure never does.
|
|
import { cpus, loadavg } from 'node:os';
|
|
|
|
export const LOAD_CAP = 5;
|
|
/** Above this factor a test that failed by timing out is retried once ("retried under load" in the CI summary). */
|
|
export const RETRY_LOAD = 2;
|
|
/** The line a retried test prints; tools/ci.ts collects these for its summary. */
|
|
export const RETRY_MARK = 'RETRIED UNDER LOAD:';
|
|
|
|
/** max(1-min, 5-min load average) ÷ cores, between 1 and cap. STOREBOUND_LOAD_FACTOR=n overrides (tests, debugging). */
|
|
export function loadFactor(env = process.env, load = loadavg(), cores = cpus().length, cap = LOAD_CAP): number {
|
|
const forced = Number(env.STOREBOUND_LOAD_FACTOR);
|
|
if (Number.isFinite(forced) && forced > 0) return Math.min(cap, Math.max(1, forced));
|
|
const per = Math.max(load[0] ?? 0, load[1] ?? 0) / Math.max(1, cores);
|
|
return Math.min(cap, Math.max(1, Math.round(per * 10) / 10));
|
|
}
|
|
|
|
/** The factor of this test process, sampled once when the first browser test file loads, so a test's waits and its
|
|
* own timeout scale by the same number. */
|
|
export const LOAD = loadFactor();
|
|
export const scaled = (ms: number, f = LOAD): number => Math.round(ms * f);
|
|
export const loadNote = (f = LOAD): string => `load ${loadavg().map(n => n.toFixed(0)).join('/')} on ${cpus().length} cores: timeouts x${f}`;
|