diff --git a/tools/acceptance.ts b/tools/acceptance.ts index 4c8d540..8a899b6 100644 --- a/tools/acceptance.ts +++ b/tools/acceptance.ts @@ -22,6 +22,7 @@ import { makeAab } from '../tests/build-fixtures'; import { ALPHA3 } from '../cli/lib/money'; import { CURRENCY } from '../cli/lib/ppp'; import { cliKey } from './cli-key'; +import { loadFactor, RETRY_MARK, scaled } from '../tests/helpers/load'; const ROOT = resolve(import.meta.dir, '..'); const PKG = JSON.parse(readFileSync(join(ROOT, 'package.json'), 'utf8')) as { version: string }; @@ -409,7 +410,10 @@ async function runCell(cell: Cell, rt: Runtime, mock: string, tpl: { app: string if (cell.stdin) sin.write(cell.stdin); await sin.end(); let timedOut = false; - const timer = setTimeout(() => { timedOut = true; killTree(proc.pid); }, cell.timeoutMs ?? 120_000); + // the limit grows with the machine's load like tools/ci.ts (a release once failed on two cells that take 3 s alone, + // at load 95 on 10 cores) + const limit = scaled(cell.timeoutMs ?? 120_000, loadFactor()); + const timer = setTimeout(() => { timedOut = true; killTree(proc.pid); }, limit); const [out, err, code] = await Promise.all([new Response(proc.stdout as ReadableStream).text(), new Response(proc.stderr as ReadableStream).text(), proc.exited]); clearTimeout(timer); running.delete(cell.id); @@ -418,7 +422,7 @@ async function runCell(cell: Cell, rt: Runtime, mock: string, tpl: { app: string const st = stores.get(slug); let error: string | undefined; const crash = all.split('\n').find(l => CRASH.test(l) && !/^\s*"/.test(l)); - if (timedOut) error = `timed out after ${(cell.timeoutMs ?? 120_000) / 1000} s; last output: ${firstLine(all.split('\n').filter(Boolean).slice(-3).join('\n'))}`; + if (timedOut) error = `timed out after ${Math.round(limit / 1000)} s; last output: ${firstLine(all.split('\n').filter(Boolean).slice(-3).join('\n'))}`; else if (crash) error = `crash text: ${crash.trim().slice(0, 200)}`; else if (!want.includes(code)) error = `exit ${code}, want ${want.join('|')}: ${firstLine(err.match(/^error: .*$/m)?.[0] ?? '') || firstLine(err) || firstLine(out)}`; else if (cell.expect && !cell.expect.test(all)) error = `output does not match ${cell.expect}: ${firstLine(all).slice(0, 160)}`; @@ -480,7 +484,13 @@ async function run(argv: string[]): Promise { await Promise.all(Array.from({ length: jobs }, async () => { for (let c = queue.shift(); c; c = queue.shift()) { if (stopped) { results.push({ id: c.id, group: c.group, cmd: `storebound ${c.args.join(' ')}`, platform, runtime: rt.name, ok: false, exit: null, want: String(c.exit), ms: 0, error: 'not run: the run budget was used up' }); continue; } - const r = await runCell(c, rt, mock.base, tpl, home, platform, running); + let r = await runCell(c, rt, mock.base, tpl, home, platform, running); + // a cell that only timed out runs once more, marked: a real hang times out twice, a starved machine does not + if (!r.ok && r.error?.startsWith('timed out')) { + console.log(`retry ${r.id}: ${r.error}`); + const again = await runCell(c, rt, mock.base, tpl, home, platform, running); + r = again.ok ? { ...again, note: [`${RETRY_MARK} ${r.error.slice(0, 80)}`, again.note].filter(Boolean).join('; ') } : again; + } console.log(`${r.ok ? 'ok ' : 'FAIL'} ${r.id} (${r.ms} ms)${r.error ? `: ${r.error}` : ''}${r.note ? ` [${r.note}]` : ''}`); results.push(r); }