From 9ba847100204af54426dcc5421653011faa2d748 Mon Sep 17 00:00:00 2001 From: CRT Date: Tue, 13 Oct 2026 02:20:03 +0800 Subject: [PATCH] ci speed: desktop boot check runs its cases in parallel (one browser, killed at the end; a page per case; condition waits instead of fixed sleeps: element, text or a quiet window of host calls and DOM); shots capture / render close the browser with a 3 s cap, then kill it; console engine waits injectable for tests (WAITS / engineWaits: poll rounds, a missing step ends once its loaded page stays quiet, team switch settle, field-miss grace), fillPage looks up each data type's dialog once per round and skips dialog sections when no dialog is open; browser tests run 4 at a time per file (testScope().concurrent with a scope per test, --max-concurrency in ci), 20 ms rounds; capture-web counts boots per test and runs its browser tests concurrently; console-session / -persist Chrome tests concurrent; recorded test timings smoothed (mergeTimings) so one loaded run does not skew the shard split --- cli/console/accounts.ts | 12 +-- cli/console/fill.ts | 103 ++++++++++++++++------ cli/shots/browser.ts | 26 ++++++ cli/shots/capture.ts | 4 +- cli/shots/render.ts | 4 +- docs/desktop.md | 4 +- tests/capture-web.test.ts | 38 ++++---- tests/ci-plan.test.ts | 7 +- tests/console-fill.test.ts | 87 ++++++++++--------- tests/console-flow.test.ts | 42 +++++---- tests/console-persist.test.ts | 2 +- tests/console-session.test.ts | 4 +- tests/helpers/browser.ts | 46 ++++++++-- tools/ci-plan.ts | 16 ++++ tools/ci.ts | 16 ++-- tools/desktop-boot-check.ts | 157 ++++++++++++++++++++++++---------- 16 files changed, 390 insertions(+), 178 deletions(-) diff --git a/cli/console/accounts.ts b/cli/console/accounts.ts index b9cba98..c557509 100644 --- a/cli/console/accounts.ts +++ b/cli/console/accounts.ts @@ -25,7 +25,7 @@ import { dirname, join } from 'node:path'; import type { ShipConfig } from '../lib/config'; import type { FormId } from '../lib/console-answers'; import { hasToken, type Name, NAV_BASES, type NavBases, type NavStep, navPlan, playAppIdOf, playDeveloperId, switchAllowed } from '../lib/console-forms'; -import { ACTION_MS, accName, findAny, type Locator, navClick, navigate, type NavResult, type Page, poll, realRole, safeClick, signInPage, SignInSeen, type Stop, vis } from './fill'; +import { ACTION_MS, accName, findAny, type Locator, navClick, navigate, WAITS, type NavResult, type Page, poll, realRole, safeClick, signInPage, SignInSeen, type Stop, vis } from './fill'; // ---- the cache ---- @@ -141,7 +141,7 @@ export async function readChooser(page: Page): Promise { */ export async function playAccounts(page: Page, playBase: string, ms: number, stop?: Stop): Promise { await page.goto(`${playBase.replace(/\/$/, '')}/u/0/developers`, { waitUntil: 'domcontentloaded' }).catch(() => { /* read what is there */ }); - const found = await poll(ms, async () => { const r = await readChooser(page); return r.length ? r : null; }, stop); + const found = await poll(ms, async () => { const r = await readChooser(page); return r.length ? r : null; }, stop, page); if (found) return found; const here = playDeveloperId(page.url()); return here ? [{ id: here, name: '' }] : []; @@ -185,7 +185,7 @@ async function accountId(page: Page, a: PlayAccount, listed: string[], ms: numbe const role = await realRole(row).catch(() => ''); if (!switchAllowed(a.name, ['link', 'menuitem', 'menuitemradio', 'option', 'radio', 'row', 'button'].includes(role) ? role : 'row', listed)) return null; await row.click({ timeout: ACTION_MS }); - return poll(ms, async () => playDeveloperId(page.url()), stop); + return poll(ms, async () => playDeveloperId(page.url()), stop, page); } // ---- App Store Connect: teams ---- @@ -197,7 +197,7 @@ const CANCEL: Name[] = ['Cancel', 'İptal', 'Vazgeç']; interface TeamItem { name: string; loc: Locator; role: string; current: boolean } async function teamMenu(page: Page, ms: number, stop?: Stop): Promise { - const t = await poll(ms, async () => { const f = await findAny(page, ['button'], ACCOUNT_MENU); return f.state === 'ok' ? f.loc : null; }, stop); + const t = await poll(ms, async () => { const f = await findAny(page, ['button'], ACCOUNT_MENU); return f.state === 'ok' ? f.loc : null; }, stop, page); if (!t || !(await navClick(t)).clicked) return null; return poll(Math.min(ms, 3000), async () => { const menus = vis(page.getByRole('menu')); @@ -213,7 +213,7 @@ async function teamMenu(page: Page, ms: number, stop?: Stop): Promise { await page.keyboard.press('Escape').catch(() => { /* gone */ }); }; @@ -235,7 +235,7 @@ export async function switchTeam(page: Page, name: string, listed: string[], ms: if (!switchAllowed(real, role, listed)) { await closeMenu(page); return false; } await it.loc.click({ timeout: ACTION_MS }); await page.waitForLoadState('domcontentloaded', { timeout: ms }).catch(() => { /* same page, or slow: read what is there */ }); - await Bun.sleep(300); + await Bun.sleep(WAITS.settleMs); // the switched team's page settles return true; } diff --git a/cli/console/fill.ts b/cli/console/fill.ts index 721fa10..2b77325 100644 --- a/cli/console/fill.ts +++ b/cli/console/fill.ts @@ -33,7 +33,7 @@ import type { Ctx } from '../lib/ctx'; import { type Answer, type ConsoleResult, consoleResult, DISCLAIMER, FORM_IDS, type FormId, fillValue, formMeta, isForm, valueText } from '../lib/console-answers'; import type { ShipConfig } from '../lib/config'; import { APPLE_PRIVACY_TYPES } from '../lib/app-privacy'; -import { ALL_APPS, APP_PRIVACY_HEADING, consoleReached, CREATE_FORM_FIELD, createAppAllowed, DATA_TYPES_HEADING, type FieldDef, FLOW_NAMES, flowAllowed, flowClass, type FlowWhere, type FormDef, formDef, FORM_OPENERS, forbiddenControl, quoteTolerant, forbiddenTick, hasToken, isAppleSignInFrame, isSignInText, isSignInUrl, type Name, navAllowed, type NavBases, type NavStep, navPlan, setUpAllowed, setUpTexts, typeLabels } from '../lib/console-forms'; +import { ALL_APPS, APP_PRIVACY_HEADING, consoleReached, CREATE_FORM_FIELD, createAppAllowed, DATA_TYPES_HEADING, type FieldDef, FLOW_NAMES, flowAllowed, flowClass, type FlowWhere, type FormDef, formDef, FORM_OPENERS, forbiddenControl, quoteTolerant, forbiddenTick, hasToken, isAppleSignInFrame, isSignInText, isSignInUrl, type Name, navAllowed, type NavBases, type NavStep, navPlan, type SectionDef, setUpAllowed, setUpTexts, typeLabels } from '../lib/console-forms'; import { log, printJson, warn } from '../lib/log'; import { openAcross, withAccountIds } from './accounts'; import { appleAppId, withAppleAppId } from './app-id'; @@ -269,6 +269,25 @@ export interface FillStep { pending?: Locator[][]; } +/** The dialog of a section: by its names, else the one visible dialog with such a heading, else (the data type + * storebound just opened) the one visible dialog. */ +async function sectionDialog(page: Page, sec: SectionDef, current?: string): Promise { + const names = sec.dialog ?? []; + let d = await find(page, 'dialog', names); + // a dialog titled otherwise (one dialog per step, the title elsewhere): the one visible dialog with such a heading + if (d.state === 'none') for (const n of names) { + const l = vis(page.getByRole('dialog').filter({ has: page.getByRole('heading', nameOpts(n)) })); + const c = await l.count(); + if (c) { d = c === 1 ? { state: 'ok', loc: l } : { state: 'ambiguous' }; break; } + } + // the data type storebound just opened (App Privacy): its dialog may be titled otherwise; the one visible dialog + if (d.state === 'none' && sec.type && sec.type === current) { + const l = vis(page.getByRole('dialog')); + if (await l.count() === 1) d = { state: 'ok', loc: l }; + } + return d; +} + /** * Fills the first section of the form that is on the page and not done yet: locate every field first; any missing or * ambiguous field -> fallback (nothing of that section is touched). Certain answers are set, the rest is highlighted @@ -276,22 +295,19 @@ export interface FillStep { */ export async function fillPage(page: Page, def: FormDef, answers: Answer[], done: Set = new Set(), partial = false, current?: string): Promise { const byId = new Map(answers.map(a => [a.id, a])); + // the sections of one data type share their dialog names (one array): the dialog is looked up once per call, not + // once per section (App Privacy: 3 sections x 6 types, each lookup several page queries) + const dialogs = new Map(); + // no visible dialog at all (the page between two data types): every dialog section is skipped with one query + let anyDialog: boolean | undefined; for (const [i, sec] of def.sections.entries()) { if (done.has(i)) continue; let root: Locator = page; if (sec.dialog) { - let d = await find(page, 'dialog', sec.dialog); - // a dialog titled otherwise (one dialog per step, the title elsewhere): the one visible dialog with such a heading - if (d.state === 'none') for (const n of sec.dialog) { - const l = vis(page.getByRole('dialog').filter({ has: page.getByRole('heading', nameOpts(n)) })); - const c = await l.count(); - if (c) { d = c === 1 ? { state: 'ok', loc: l } : { state: 'ambiguous' }; break; } - } - // the data type storebound just opened (App Privacy): its dialog may be titled otherwise; the one visible dialog - if (d.state === 'none' && sec.type && sec.type === current) { - const l = vis(page.getByRole('dialog')); - if (await l.count() === 1) d = { state: 'ok', loc: l }; - } + anyDialog ??= await vis(page.getByRole('dialog')).count() > 0; + if (!anyDialog) continue; + let d = dialogs.get(sec.dialog); + if (!d) { d = await sectionDialog(page, sec, current); dialogs.set(sec.dialog, d); } if (d.state === 'none') continue; if (d.state !== 'ok') return { section: i, title: sec.title, filled: [], highlighted: [], missing: [`dialog ${sec.title}`], fallback: true }; root = d.loc; @@ -624,16 +640,53 @@ export class SignInSeen extends Error {} export type Stop = () => Promise; /** - * Polls `fn` until it returns a value or the time is up (errors while the page navigates count as "not yet"). `stop` - * is checked first on every round: a sign-in page ends the wait at once (SignInSeen). + * The engine's internal waits that tests shorten (engineWaits); production keeps these defaults. + * pollMs the pause between two poll rounds + * quietMs 0 (production): a poll waits its whole time for something that is not there. Above 0 (tests, on local + * fixture pages): a poll also ends, as "not found", once the page has finished loading and its DOM has not + * changed for quietMs (a MutationObserver the poll puts into the page). A missing step then costs about + * quietMs instead of the whole step timeout, and a step that is there is still waited for as long as needed. + * settleMs the pause after a click that loads another page in place (switching the App Store Connect team) + * missMs fillSession: how long a field must stay missing (and at least 3 rounds) before it counts as a UI change */ -export async function poll(ms: number, fn: () => Promise, stop?: Stop): Promise { +export const WAITS = { pollMs: 200, quietMs: 0, settleMs: 300, missMs: 1500 }; +/** Tests: sets engine waits (WAITS); returns the function that restores the previous ones. */ +export function engineWaits(w: Partial): () => void { + const before = { ...WAITS }; + Object.assign(WAITS, w); + return () => { Object.assign(WAITS, before); }; +} + +/** How long the page has been loaded and unchanged (ms; 0 while it loads or is leaving for another page). Puts the + * observer in on the first call. */ +async function quietFor(page: Page): Promise { + return page.evaluate(() => { + const w = window as any; + if (!w.__storeboundQuiet) { + const q = { at: performance.now(), leaving: false }; + w.__storeboundQuiet = q; + new MutationObserver(() => { q.at = performance.now(); }).observe(document, { subtree: true, childList: true, attributes: true, characterData: true }); + // a click or script that starts a navigation: the old page stays "complete" until the new one commits + addEventListener('beforeunload', () => { q.leaving = true; }); + } + const q = w.__storeboundQuiet; + return document.readyState === 'complete' && !q.leaving ? performance.now() - q.at : 0; + }).catch(() => 0); +} + +/** + * Polls `fn` until it returns a value or the time is up (errors while the page navigates count as "not yet"). `stop` + * is checked first on every round: a sign-in page ends the wait at once (SignInSeen). With `page` and WAITS.quietMs + * (tests), a loaded page that stays unchanged for quietMs ends it early too (not found). + */ +export async function poll(ms: number, fn: () => Promise, stop?: Stop, page?: Page): Promise { const end = Date.now() + ms; for (;;) { if (stop && await stop().catch(() => false)) throw new SignInSeen(); try { const r = await fn(); if (r !== null) return r; } catch { /* navigating */ } if (Date.now() >= end) return null; - await Bun.sleep(200); + if (page && WAITS.quietMs > 0 && await quietFor(page) >= WAITS.quietMs) return null; + await Bun.sleep(WAITS.pollMs); } } @@ -648,7 +701,7 @@ const opener = new RegExp(`^(?:${FORM_OPENERS.join('|')})$`); export async function pickApp(page: Page, s: Extract, ms: number, stop?: Stop): Promise<{ loc?: Locator; candidates?: string[] } | null> { if (s.search && s.tokens[0]) { - const box = await poll(Math.min(ms, 3000), async () => { const f = await findAny(page, ['searchbox', 'textbox', 'combobox'], s.search!); return f.state === 'ok' ? f.loc : null; }, stop); + const box = await poll(Math.min(ms, 3000), async () => { const f = await findAny(page, ['searchbox', 'textbox', 'combobox'], s.search!); return f.state === 'ok' ? f.loc : null; }, stop, page); if (box) { await box.fill(s.tokens[0]); } } return poll(ms, async () => { @@ -677,7 +730,7 @@ export async function pickApp(page: Page, s: Extract 1) return { candidates: hits.map(h => h.name) }; } return null; - }, stop); + }, stop, page); } async function formLink(page: Page, entry: Name[], ms: number, stop?: Stop): Promise<{ loc?: Locator; ambiguous?: boolean } | null> { @@ -704,7 +757,7 @@ async function formLink(page: Page, entry: Name[], ms: number, stop?: Stop): Pro } if (found.length === 1) return { loc: found[0]!.loc }; return found.length > 1 ? { ambiguous: true } : null; - }, stop); + }, stop, page); } /** The wait target is on the page (a heading of `level` when given). */ @@ -750,7 +803,7 @@ export async function navigate(page: Page, plan: NavStep[], opts: NavOpts = {}): try { await page.goto(s.url, { waitUntil: 'domcontentloaded' }); } catch (e) { return fail(s, `could not load ${s.url} (${(e as Error).message.split('\n')[0]})`); } } else if (s.kind === 'click') { if (s.optional && s.unless && (await findAny(page, s.unless.roles, s.unless.names)).state !== 'none') { say(' (not needed)'); continue; } - const f = await poll(s.optional ? Math.min(ms, 2000) : ms, async () => { const r = await findAny(page, s.roles, s.names); return r.state === 'none' ? null : r; }, stop); + const f = await poll(s.optional ? Math.min(ms, 2000) : ms, async () => { const r = await findAny(page, s.roles, s.names); return r.state === 'none' ? null : r; }, stop, page); if (!f) { if (s.optional) { say(' (not on this page, skipped)'); continue; } return fail(s, 'not found'); } if (f.state !== 'ok') return fail(s, 'more than one match'); const c = await navClick(f.loc, !!s.createApp); @@ -769,7 +822,7 @@ export async function navigate(page: Page, plan: NavStep[], opts: NavOpts = {}): const c = await navClick(r.loc); if (!c.clicked) return fail(s, `refused to click "${c.name}"`); } else { - const f = await poll(ms, async () => ((await present(page, s)) ? true : null), stop); + const f = await poll(ms, async () => ((await present(page, s)) ? true : null), stop, page); if (!f) return fail(s, 'did not appear'); } } catch (e) { @@ -1277,8 +1330,8 @@ export async function fillSession(b: ConsoleBrowser, o: FillSessionOpts): Promis let s: FillStep; try { s = await fillPage(p, def, pageAnswers, done, false, currentType); } catch { await Bun.sleep(tick); continue; } // navigating if (s.section !== null) { - // a page still loading can miss a field for a moment: only a miss that lasts (3 rounds, at least 1.5 s) is a UI change - if (s.fallback && (++misses < 3 || Date.now() - firstMiss < 1500)) { if (misses === 1) firstMiss = Date.now(); await Bun.sleep(tick); continue; } + // a page still loading can miss a field for a moment: only a miss that lasts (3 rounds, at least WAITS.missMs) is a UI change + if (s.fallback && (++misses < 3 || Date.now() - firstMiss < WAITS.missMs)) { if (misses === 1) firstMiss = Date.now(); await Bun.sleep(tick); continue; } misses = 0; // the miss lasts: fill every field that was found, leave the missing ones to the owner (only all-missing is a UI change) if (s.fallback) { try { s = await fillPage(p, def, pageAnswers, done, true, currentType); } catch { await Bun.sleep(tick); continue; } } @@ -1297,7 +1350,7 @@ export async function fillSession(b: ConsoleBrowser, o: FillSessionOpts): Promis if (form === 'play-app-record' && s.filled.includes('packageName')) { const check = await controlWhere(p, FLOW_NAMES.checkAvailability, () => true, dataTypes).catch(() => null); if (check && await flowClick(check, form, dataTypes, say).catch(() => false)) { - const ok = await poll(5000, async () => ((await vis(p.getByText(/package name (is )?available/i)).count()) > 0 ? true : null)).catch(() => null); + const ok = await poll(5000, async () => ((await vis(p.getByText(/package name (is )?available/i)).count()) > 0 ? true : null), undefined, p).catch(() => null); say(ok ? 'App package name: "Package name available"' : 'App package name: Play Console did not say "Package name available": check the message next to the field'); } } diff --git a/cli/shots/browser.ts b/cli/shots/browser.ts index 930e32d..8d46c76 100644 --- a/cli/shots/browser.ts +++ b/cli/shots/browser.ts @@ -175,6 +175,32 @@ export async function launch(root: string, opts: Record, deps: return withCandidates(kind, deps.installed ?? installedBrowsers(c), o => c.launch({ ...opts, ...o })); } +/** How long a launched (throwaway profile) browser gets to close gracefully before it is killed. */ +export const CLOSE_GRACE_MS = 3000; + +/** + * Ends a browser from `launch` once the work is done: a graceful close capped at `ms`, then SIGKILL of its process. + * Google Chrome often hangs 10-30 s in its own shutdown after http pages (tests/helpers/browser.ts); a throwaway + * profile has nothing to flush, so waiting for it only costs time. The pid is read over CDP before the close. Never + * throws. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export async function closeBrowser(browser: any, ms = CLOSE_GRACE_MS): Promise { + let pid: number | undefined; + try { + const s = await browser.newBrowserCDPSession(); + const info = await s.send('SystemInfo.getProcessInfo') as { processInfo: { type: string; id: number }[] }; + pid = info.processInfo.find(p => p.type === 'browser')?.id; + } catch { /* not Chromium, or already gone */ } + let t: ReturnType | undefined; + const closed = await Promise.race([ + Promise.resolve().then(() => browser.close()).then(() => true, () => true), + new Promise(r => { t = setTimeout(() => r(false), ms); }), + ]); + clearTimeout(t); + if (!closed && pid) try { process.kill(pid, 'SIGKILL'); } catch { /* exited meanwhile */ } +} + /** * A persistent browser profile (cookies of a console sign-in the owner did by hand stay there between runs; storebound * never reads them). Used by `storebound console fill`: always a real Chrome / Chromium (kind "visible", also when a diff --git a/cli/shots/capture.ts b/cli/shots/capture.ts index 71e517a..d07087a 100644 --- a/cli/shots/capture.ts +++ b/cli/shots/capture.ts @@ -29,7 +29,7 @@ import type { ShipConfig, ShotsCapture } from '../lib/config'; import type { Ctx } from '../lib/ctx'; import { need } from '../lib/ctx'; import { done, log, step, warn } from '../lib/log'; -import { launch } from './browser'; +import { closeBrowser, launch } from './browser'; import { checkFrameFolder } from './check'; import { adbAppLocaleArgs, adbDemoExitArgs, adbDemoModeArgs, adbForceStopArgs, adbInstallArgs, adbScreencapArgs, adbStartArgs, adbStateArgs, adbTarget, cropArgs, cropTopFor, fill, pickSimulator, shotsFor, simBootArgs, simBootStatusArgs, simInstallArgs, simLaunchArgs, simOpenUrlArgs, @@ -138,7 +138,7 @@ export async function run(ctx: Ctx): Promise { if (todo.length) { const browser = await launch(root, { headless: true, args: ['--mute-audio', '--ignore-gpu-blocklist', '--enable-webgl', '--enable-gpu'] }); let report; - try { report = await captureWeb(browser, cap, todo, { concurrency, dims }); } finally { await browser.close(); } + try { report = await captureWeb(browser, cap, todo, { concurrency, dims }); } finally { await closeBrowser(browser); } // capped close, then kill: Chrome can hang 10-30 s in its shutdown report.skipped = skipped.map(j => j.frame); writeCaptureState(root, report, cache, todo); for (const l of summaryLines(report)) log('shots', l); diff --git a/cli/shots/render.ts b/cli/shots/render.ts index 78a9545..a49c69b 100644 --- a/cli/shots/render.ts +++ b/cli/shots/render.ts @@ -21,7 +21,7 @@ import { join, relative } from 'node:path'; import type { Ctx } from '../lib/ctx'; import { need } from '../lib/ctx'; import { done, log, warn } from '../lib/log'; -import { launch } from './browser'; +import { closeBrowser, launch } from './browser'; import { defaultConcurrency, pool, progressLine } from './capture-web'; import { shotsStyle } from '../lib/config'; import { type CopyFile, SIZES, copyText, defaultSizes, frameDims, mergeCopy, palette, renderDir, scriptFont, sheetLayout, validateCopy, variantScenes } from './lib'; @@ -239,7 +239,7 @@ export async function run(ctx: Ctx): Promise { }); } finally { for (const p of pages) await p?.page.context().close().catch(() => {}); - await browser.close(); + await closeBrowser(browser); server.stop(true); if (Object.keys(report).length) await Bun.write(join(outDir, 'report.json'), JSON.stringify(mergeReport(readReport(outDir), report), null, 1) + '\n'); } diff --git a/docs/desktop.md b/docs/desktop.md index d6bb0b5..fcc5fb9 100644 --- a/docs/desktop.md +++ b/docs/desktop.md @@ -379,7 +379,9 @@ Download sources (one click on the Dependencies screen or in the first-run guide fixtures (`apps/desktop/tests/fixtures/state`: empty, D1 with folders gone, D3 full; the host answers are written by the Rust migration itself, `src-tauri/tests/state.rs`) and the real sidecar for reads. It opens every screen and app and fails on any uncaught or console error, an error panel or an empty `#root`; a last case removes the bundle - and expects the boot.js panel. + and expects the boot.js panel. One browser (killed at the end), a page per case, the cases in parallel against the + one build, and no fixed sleeps: each step waits for its element, its text or a quiet window (no host call in + flight, no DOM change). ## Develop diff --git a/tests/capture-web.test.ts b/tests/capture-web.test.ts index 048d7cf..cb617c8 100644 --- a/tests/capture-web.test.ts +++ b/tests/capture-web.test.ts @@ -53,14 +53,17 @@ describe('capture helpers', () => { }); }); -// ---- a local fixture web app: /scene/?lang=xx, a SPA (popstate) that counts its boots ---- -let boots = 0; +// ---- a local fixture web app: //scene/?lang=xx, a SPA (popstate) that counts its boots per key: each +// browser test has its own key, so they run at the same time (test.concurrent) ---- +const boots = new Map(); +const booted = (key: string) => boots.get(key) ?? 0; const server = Bun.serve({ port: 0, hostname: '127.0.0.1', fetch(req) { const u = new URL(req.url); if (u.pathname === '/app.js') { - boots++; + const key = u.searchParams.get('k') ?? ''; + boots.set(key, booted(key) + 1); return new Response(` const COLORS = { en: '#1e66f5', de: '#40a02b', tr: '#d20f39' }; function show() { @@ -74,7 +77,8 @@ const server = Bun.serve({ addEventListener('popstate', show); show();`, { headers: { 'content-type': 'text/javascript' } }); } - return new Response('', { headers: { 'content-type': 'text/html' } }); + const key = encodeURIComponent(u.pathname.split('/')[1] ?? ''); + return new Response(``, { headers: { 'content-type': 'text/html' } }); }, }); afterAll(() => server.stop(true)); @@ -99,15 +103,16 @@ function project(cfg: ShipConfig, files: Record): { root: string } const shot = (name: string, extra: object = {}) => ({ path: `/scene/${name}`, waitFor: `window.scene === '${name}'`, delayMs: 20, ...extra }); -function config(capture: Partial = {}): ShipConfig { +/** `key`: the fixture app's boot counter this config's runs count on (one per browser test). */ +function config(capture: Partial = {}, key = 'app'): ShipConfig { return { ...BASE, shots: { copy: 'store/shots/copy.json', frames: 'store/shots/frames', out: 'store/renders', scenes: [{ id: 'hero', palette: 'fire', frames: [{ src: 'hero.png' }] }], phone: { w: 180, h: 240, dpr: 1 }, - capture: { source: 'web', baseUrl: `http://127.0.0.1:${server.port}`, query: 'lang={lang}', build: 'src', shots: { hero: shot('hero'), battle: shot('battle'), king: shot('king') }, ...capture } } }; + capture: { source: 'web', baseUrl: `http://127.0.0.1:${server.port}/${key}`, query: 'lang={lang}', build: 'src', shots: { hero: shot('hero'), battle: shot('battle'), king: shot('king') }, ...capture } } }; } const files = { 'store/shots/copy.json': '{}', 'src/main.js': 'v1' }; const frames = (root: string) => ['en', 'de', 'tr'].flatMap(l => ['hero', 'battle', 'king'].map(n => join(root, 'store/shots/frames', l, 'phone', `${n}.png`))); describe.skipIf(!canLaunch)('shots capture (web) with a fixture app', () => { - test('concurrency 3 gives the same files as serial', async () => { + test.concurrent('concurrency 3 gives the same files as serial', async () => { const p = project(config(), files); try { await p.cli(['--device', 'phone', '--concurrency', '1', '--force']); @@ -120,34 +125,31 @@ describe.skipIf(!canLaunch)('shots capture (web) with a fixture app', () => { } finally { rmSync(p.root, { recursive: true, force: true }); } }, 120_000); - test('per-language session boots the app once per language; per-shot once per frame', async () => { - const p = project(config({ session: 'per-language' }), files); + test.concurrent('per-language session boots the app once per language; per-shot once per frame', async () => { + const p = project(config({ session: 'per-language' }, 'lang'), files); try { - boots = 0; const r = await p.cli(['--device', 'phone', '--concurrency', '2', '--force']); - expect(boots).toBe(3); + expect(booted('lang')).toBe(3); expect(r.out).toContain('(reused page)'); const report = JSON.parse(readFileSync(join(p.root, '.storebound/capture-report.json'), 'utf8')) as CaptureReport; expect(report.frames.filter(f => f.reused).length).toBe(6); expect(frames(p.root).every(f => existsSync(f))).toBe(true); // a shot whose precondition fails in the reused page (it needs a freshly booted page) falls back to a fresh page - const bad = project(config({ session: 'per-language', reuseTimeoutMs: 300, shots: { hero: shot('hero'), fresh: { path: '/scene/fresh', waitFor: "window.scene === 'fresh' && window.shows === 1", delayMs: 20 } } }), files); + const bad = project(config({ session: 'per-language', reuseTimeoutMs: 300, shots: { hero: shot('hero'), fresh: { path: '/scene/fresh', waitFor: "window.scene === 'fresh' && window.shows === 1", delayMs: 20 } } }, 'fresh'), files); try { - boots = 0; const r2 = await bad.cli(['--device', 'phone', '--lang', 'en', '--force']); expect(r2.err).toContain('en/phone/fresh: reused page failed'); expect(r2.out).toContain('capture done: 2 frames'); - expect(boots).toBe(2); + expect(booted('fresh')).toBe(2); } finally { rmSync(bad.root, { recursive: true, force: true }); } - boots = 0; - const q = project(config(), files); + const q = project(config({}, 'shot'), files); await q.cli(['--device', 'phone', '--lang', 'en', '--force']); rmSync(q.root, { recursive: true, force: true }); - expect(boots).toBe(3); + expect(booted('shot')).toBe(3); } finally { rmSync(p.root, { recursive: true, force: true }); } }, 180_000); - test('cache skips unchanged frames, re-runs changed ones; --only-missing; timing breakdown', async () => { + test.concurrent('cache skips unchanged frames, re-runs changed ones; --only-missing; timing breakdown', async () => { const p = project(config(), files); try { let r = await p.cli(['--device', 'phone']); diff --git a/tests/ci-plan.test.ts b/tests/ci-plan.test.ts index f34396d..9e80c8a 100644 --- a/tests/ci-plan.test.ts +++ b/tests/ci-plan.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'bun:test'; -import { balance, depsOf, GLOBAL, HEAVY, junitTimes, listTests, relatedTests, rustNeeded, shardCount } from '../tools/ci-plan'; +import { balance, depsOf, GLOBAL, HEAVY, junitTimes, listTests, mergeTimings, relatedTests, rustNeeded, shardCount } from '../tools/ci-plan'; describe('tools/ci-plan: test shards', () => { test('shard count: cores - 1, between 1 and 8, env override', () => { @@ -27,6 +27,11 @@ describe('tools/ci-plan: test shards', () => { `; expect(junitTimes(xml)).toEqual({ 'tests/a.test.ts': 1.5, 'tests/b.test.ts': 0.25 }); }); + test('recorded timings: a new file takes its time; a known one moves halfway, never up past the new time', () => { + const t = mergeTimings({ 'a.test.ts': 10, 'b.test.ts': 40, 'keep.test.ts': 3 }, { 'a.test.ts': 30, 'b.test.ts': 20, 'new.test.ts': 5 }); + expect(t).toEqual({ 'a.test.ts': 20, 'b.test.ts': 20, 'keep.test.ts': 3, 'new.test.ts': 5 }); + }); + test('every heavy suite exists and every test file is found', () => { const all = listTests(); for (const h of HEAVY) expect(all).toContain(h); diff --git a/tests/console-fill.test.ts b/tests/console-fill.test.ts index 45345a1..19b545b 100644 --- a/tests/console-fill.test.ts +++ b/tests/console-fill.test.ts @@ -3,21 +3,22 @@ // the public help pages, not recorded) served on 127.0.0.1. No network. Browser tests skip when no Chromium is // installed (a launch that times out fails instead). Load-proofing (tests/helpers/browser.ts): one launch with a long // launch timeout, every context closed right after its test, event-driven waits for log lines, explicit test timeouts. -// Slow on purpose (marked SLOW): the sign-in / watching tests and the account tests wait out real "not here" timeouts -// of the engine (navTimeoutMs / navMs, scaled down through the options). The relaunch of storebound's own profile +// Speed: the browser tests run BROWSER_CONCURRENCY at a time (scope.concurrent, each with its own contexts) and the +// engine's waits are the test ones (TEST_WAITS / TEST_TICK_MS: 20 ms rounds, a missing step ends once its page is +// loaded and quiet). SLOW tests still wait out some "not here" timeouts of the engine (navMs, scaled down). The relaunch of storebound's own profile // (session cookies kept) needs a real Chrome: tests/console-persist.test.ts. import { afterAll, afterEach, beforeAll, describe, expect, test } from 'bun:test'; import { mkdtempSync, readdirSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { matchAccount, openAcross, readAccounts, rememberApple, rememberPlay, withAccountIds } from '../cli/console/accounts'; -import { createAppHere, fillPage, fillSession, navClick, navigate, resumeAt, run, safeClick, signInPage, signInReason } from '../cli/console/fill'; +import { createAppHere, engineWaits, fillPage, fillSession, navClick, navigate, resumeAt, run, safeClick, signInPage, signInReason } from '../cli/console/fill'; import type { ConsoleBrowser } from '../cli/console/session'; import { parseArgs } from '../cli/lib/args'; import { type ShipConfig, validateExtras } from '../cli/lib/config'; import { AGE_RATING_API, consoleAnswers, consoleResult, fillValue, FORM_IDS, type FormId, formMeta } from '../cli/lib/console-answers'; import { consoleReached, CREATE_APP_OPENERS, createAppAllowed, formDef, FORM_OPENERS, forbiddenControl, hasToken, isSignInText, isSignInUrl, navAllowed, navPlan, playDeveloperId, switchAllowed } from '../cli/lib/console-forms'; -import { AFTER_ALL_MS, AFTER_EACH_MS, launchTestBrowser, logLines, testScope } from './helpers/browser'; +import { AFTER_ALL_MS, AFTER_EACH_MS, launchTestBrowser, logLines, TEST_TICK_MS, TEST_WAITS, testScope } from './helpers/browser'; const config: ShipConfig = { app: { name: 'Example Game', profile: 'game', privacyPolicyUrl: 'https://example.com/privacy', supportUrl: 'https://example.com/support', website: 'https://example.com', contactEmail: 'support@example.com' }, @@ -208,7 +209,9 @@ beforeAll(() => { }); // bounded: every context is closed per test (afterEach) and each close is capped (tests/helpers/browser.ts) afterEach(() => scope.settle(), AFTER_EACH_MS); -afterAll(async () => { await shared.close(); server?.stop(true); }, AFTER_ALL_MS); +let restoreWaits = () => {}; +beforeAll(() => { restoreWaits = engineWaits(TEST_WAITS); }); +afterAll(async () => { restoreWaits(); await shared.close(); server?.stop(true); }, AFTER_ALL_MS); async function open(file: string) { const page = await scope.newPage(); @@ -221,7 +224,7 @@ const flagged = (page: any) => page.evaluate(() => document.querySelectorAll('[d const clicks = (page: any) => page.evaluate(() => (window as any).__clicks); describe.skipIf(!browser)('engine (headless, modeled pages)', () => { - test('app record: certain fields filled, uncertain empty + highlighted, Create never clicked', async () => { + scope.concurrent('app record: certain fields filled, uncertain empty + highlighted, Create never clicked', async () => { const page = await open('apple-app-record.html'); const def = formDef('apple-app-record', pageAnswers('apple-app-record')); const s = await fillPage(page, def, pageAnswers('apple-app-record')); @@ -242,7 +245,7 @@ describe.skipIf(!browser)('engine (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('User Access under a plain heading (real App Store Connect): the radios are found in the dialog', async () => { + scope.concurrent('User Access under a plain heading (real App Store Connect): the radios are found in the dialog', async () => { const page = await open('apple-app-record-heading.html'); const s = await fillPage(page, formDef('apple-app-record', pageAnswers('apple-app-record')), pageAnswers('apple-app-record')); expect(s.fallback).toBe(false); @@ -252,7 +255,7 @@ describe.skipIf(!browser)('engine (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('a lasting partial miss fills every field that was found and leaves only the missing one', async () => { + scope.concurrent('a lasting partial miss fills every field that was found and leaves only the missing one', async () => { const page = await open('apple-app-record-renamed.html'); const def = formDef('apple-app-record', pageAnswers('apple-app-record')); const s = await fillPage(page, def, pageAnswers('apple-app-record'), new Set(), true); @@ -265,7 +268,7 @@ describe.skipIf(!browser)('engine (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('a renamed label stops filling and falls back to guidance', async () => { + scope.concurrent('a renamed label stops filling and falls back to guidance', async () => { const page = await open('apple-app-record-renamed.html'); const s = await fillPage(page, formDef('apple-app-record', pageAnswers('apple-app-record')), pageAnswers('apple-app-record')); expect(s.fallback).toBe(true); @@ -277,7 +280,7 @@ describe.skipIf(!browser)('engine (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('app privacy (fillPage alone): section by section as the page changes; fillPage itself never clicks Next / Save / Publish', async () => { + scope.concurrent('app privacy (fillPage alone): section by section as the page changes; fillPage itself never clicks Next / Save / Publish', async () => { const page = await open('apple-app-privacy.html'); const ans = pageAnswers('apple-app-privacy'); const def = formDef('apple-app-privacy', ans); @@ -311,7 +314,7 @@ describe.skipIf(!browser)('engine (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('content rating: full email in the page, Game, purchases yes, the rest empty + highlighted', async () => { + scope.concurrent('content rating: full email in the page, Game, purchases yes, the rest empty + highlighted', async () => { const page = await open('play-content-rating.html'); const ans = pageAnswers('play-content-rating'); const def = formDef('play-content-rating', ans); @@ -332,7 +335,7 @@ describe.skipIf(!browser)('engine (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('target audience: ages, appeal highlighted, ads yes', async () => { + scope.concurrent('target audience: ages, appeal highlighted, ads yes', async () => { const page = await open('play-target-audience.html'); const ans = pageAnswers('play-target-audience'); const def = formDef('play-target-audience', ans); @@ -349,7 +352,7 @@ describe.skipIf(!browser)('engine (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('sign-in detector and safeClick', async () => { + scope.concurrent('sign-in detector and safeClick', async () => { const signin = await open('signin.html'); expect(await signInPage(signin)).toBe(true); await signin.close(); @@ -419,7 +422,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => { const noneDenied = (names: string[]) => { for (const n of names) expect(navAllowed(n, 'link')).toBe(true); }; const fresh = async () => { const p = await scope.newPage(); await p.goto(base + 'blank.html'); await p.evaluate(() => sessionStorage.clear()); return p; }; - test('App Store Connect (EN): + > New App opens the dialog and it is filled; Create and Submit for Review never clicked', async () => { + scope.concurrent('App Store Connect (EN): + > New App opens the dialog and it is filled; Create and Submit for Review never clicked', async () => { const page = await fresh(); const r = await navigate(page, navPlan('apple-app-record', config, bases()), quiet); expect(r).toMatchObject({ ok: true, steps: ['open Apps', 'the add button (+)', 'New App in the menu', 'the New App dialog'] }); @@ -431,7 +434,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('App Store Connect (TR): Uygulama Ekle > Yeni Uygulama; Oluştur and İncelemeye Gönder never clicked', async () => { + scope.concurrent('App Store Connect (TR): Uygulama Ekle > Yeni Uygulama; Oluştur and İncelemeye Gönder never clicked', async () => { const page = await fresh(); const r = await navigate(page, navPlan('apple-app-record', config, bases('asc-tr')), quiet); expect(r.ok).toBe(true); @@ -440,7 +443,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('a missing step stops navigation, names the step and clicks nothing', async () => { + scope.concurrent('a missing step stops navigation, names the step and clicks nothing', async () => { const page = await fresh(); const said: string[] = []; const r = await navigate(page, navPlan('apple-app-record', config, bases('asc-noplus')), { timeoutMs: 800, say: m => said.push(m) }); @@ -451,7 +454,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('App Privacy: appId deep link, or app list > the app > App Privacy; two apps of one name -> stop and list', async () => { + scope.concurrent('App Privacy: appId deep link, or app list > the app > App Privacy; two apps of one name -> stop and list', async () => { let page = await fresh(); let r = await navigate(page, navPlan('apple-app-privacy', { ...config, apple: { ...config.apple!, appId: '123' } } as ShipConfig, bases()), quiet); expect(r).toMatchObject({ ok: true, steps: ['open App Privacy of the app (apple.appId)', 'the App Privacy page'] }); @@ -469,7 +472,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('App Privacy: an app card that names the app twice (icon aria-label + title link, same /apps/) is one app', async () => { + scope.concurrent('App Privacy: an app card that names the app twice (icon aria-label + title link, same /apps/) is one app', async () => { const page = await fresh(); const r = await navigate(page, navPlan('apple-app-privacy', config, bases('asc-card')), quiet); expect(r).toMatchObject({ ok: true }); @@ -479,7 +482,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('Play: the app by package name > Policy and programs > App content > Start / Manage; nothing deny-listed clicked', async () => { + scope.concurrent('Play: the app by package name > Policy and programs > App content > Start / Manage; nothing deny-listed clicked', async () => { for (const [form, link, heading] of [['play-content-rating', 'Start', 'Content rating'], ['play-target-audience', 'Manage', 'Target audience and content']] as const) { const page = await fresh(); const r = await navigate(page, navPlan(form, config, bases()), quiet); @@ -492,7 +495,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => { } }, BROWSER_TEST_MS); - test('navClick refuses a Start button, Start rollout, Save and Publish even when pointed at them', async () => { + scope.concurrent('navClick refuses a Start button, Start rollout, Save and Publish even when pointed at them', async () => { const page = await open('play-app-content.html'); expect((await navClick(page.getByRole('button', { name: 'Start' }))).clicked).toBe(false); expect((await navClick(page.getByRole('button', { name: 'Save' }))).clicked).toBe(false); @@ -504,7 +507,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('already on the form: nothing is clicked', async () => { + scope.concurrent('already on the form: nothing is clicked', async () => { const page = await open('play-content-rating.html'); const r = await navigate(page, navPlan('play-content-rating', config, bases()), quiet); expect(r).toMatchObject({ ok: true, steps: ['already open'] }); @@ -535,14 +538,14 @@ afterAll(() => { for (const d of tmpDirs) rmSync(d, { recursive: true, force: tr const tmpAccounts = () => { const d = mkdtempSync(join(tmpdir(), 'storebound-accounts-')); tmpDirs.push(d); return join(d, 'console-accounts.json'); }; /** * The engine's timings scaled down for tests (production: tick 1 s, 20 s per navigation step, 10 min for a sign-in). - * navTimeoutMs is both how long a step that IS there may take to show up (headroom for a loaded machine) and what a - * step that is NOT there costs (the watching tests wait that out once), so it stays a few seconds, not FOUND_MS. + * navTimeoutMs is how long a step that IS there may take to show up (headroom for a loaded machine); a step that is + * NOT there ends earlier, once its loaded page stays unchanged for TEST_WAITS.quietMs (tests/helpers/browser.ts). */ -const fast = (log: ReturnType) => ({ say: log.say, tickMs: 100, navTimeoutMs: 5000, signInWaitMs: 30_000, accountsFile: tmpAccounts(), env: {} }); +const fast = (log: ReturnType) => ({ say: log.say, tickMs: TEST_TICK_MS, navTimeoutMs: 5000, signInWaitMs: 30_000, accountsFile: tmpAccounts(), env: {} }); const FILLED = (lines: string[]) => lines.some(m => /^New App: filled \d+/.test(m)); describe.skipIf(!browser)('sign-in and watching (headless, modeled pages on the real console hosts, routed locally)', () => { - test('ASC /login with an idmsa frame and no password field: detected mid-navigation, never touched; after sign-in navigation restarts at step 1', async () => { + scope.concurrent('ASC /login with an idmsa frame and no password field: detected mid-navigation, never touched; after sign-in navigation restarts at step 1', async () => { const b = await consoleBrowser((u, cookie) => u.hostname === 'idmsa.apple.com' ? 'idmsa-signin.html' : u.hostname !== 'appstoreconnect.apple.com' ? null : u.pathname === '/login' ? 'asc-login.html' : u.pathname === '/apps' ? (cookie.includes('myacinfo=') ? 'asc-apps.html' : 'asc-shell.html') : null); @@ -565,7 +568,7 @@ describe.skipIf(!browser)('sign-in and watching (headless, modeled pages on the expect((await run).filled).toBeGreaterThan(0); }, SLOW_TEST_MS); - test('after the printed steps storebound keeps watching: the owner opens Apps by hand, navigation resumes at step 2 and fills', async () => { + scope.concurrent('after the printed steps storebound keeps watching: the owner opens Apps by hand, navigation resumes at step 2 and fills', async () => { const b = await consoleBrowser(u => u.hostname === 'appstoreconnect.apple.com' && u.pathname === '/apps' ? (u.search.includes('owner') ? 'asc-apps.html' : 'asc-apps-noplus.html') : null); const log = logLines(), said = log.lines; const run = scope.run(fillSession(b, { form: 'apple-app-record', config, ...fast(log) })); @@ -580,7 +583,7 @@ describe.skipIf(!browser)('sign-in and watching (headless, modeled pages on the await run; }, SLOW_TEST_MS); - test('after the printed steps: the owner opens the dialog by hand, storebound sees it and fills it', async () => { + scope.concurrent('after the printed steps: the owner opens the dialog by hand, storebound sees it and fills it', async () => { const b = await consoleBrowser(u => u.hostname === 'appstoreconnect.apple.com' && u.pathname === '/apps' ? (u.search.includes('dialog') ? 'apple-app-record.html' : 'asc-apps-noplus.html') : null); const log = logLines(); const run = scope.run(fillSession(b, { form: 'apple-app-record', config, ...fast(log) })); @@ -592,7 +595,7 @@ describe.skipIf(!browser)('sign-in and watching (headless, modeled pages on the await run; }, SLOW_TEST_MS); - test('resumeAt: the furthest step visible now, never the initial page load', async () => { + scope.concurrent('resumeAt: the furthest step visible now, never the initial page load', async () => { const page = await open('asc-apps.html'); const plan = navPlan('apple-app-record', config, bases()); expect(await resumeAt(page, plan)).toBe(1); @@ -657,7 +660,7 @@ describe.skipIf(!browser)('accounts (headless, modeled pages)', () => { const fresh = async () => { const p = await scope.newPage(); await p.goto(base + 'blank.html'); await p.evaluate(() => sessionStorage.clear()); return p; }; const DENIED = ['Sign Out', 'Edit Profile', 'Create', 'Submit for Review', 'Create app', 'Create developer account', 'Save', 'Publish']; - test('App Store Connect, 3 teams, the bundle id in the 2nd: switched through the account menu, remembered, then picked first', async () => { + scope.concurrent('App Store Connect, 3 teams, the bundle id in the 2nd: switched through the account menu, remembered, then picked first', async () => { const file = tmpAccounts(), said: string[] = []; let page = await fresh(); await page.goto(base + 'teams/apps'); @@ -684,7 +687,7 @@ describe.skipIf(!browser)('accounts (headless, modeled pages)', () => { await page.close(); }, SLOW_TEST_MS); - test('Play, one login in 5 developer accounts, the app in the 4th: found by package name, remembered, then opened directly', async () => { + scope.concurrent('Play, one login in 5 developer accounts, the app in the 4th: found by package name, remembered, then opened directly', async () => { const file = tmpAccounts(), said: string[] = []; const pb = { asc: base + 'asc', play: base + 'playm' }; let page = await fresh(); @@ -711,7 +714,7 @@ describe.skipIf(!browser)('accounts (headless, modeled pages)', () => { await page.close(); }, SLOW_TEST_MS); - test('Play: a package in none of the developer accounts is said plainly; nothing deny-listed clicked', async () => { + scope.concurrent('Play: a package in none of the developer accounts is said plainly; nothing deny-listed clicked', async () => { const page = await fresh(); const absent = { ...config, play: { packageName: 'com.example.absent' } } as ShipConfig; const r = await openAcross(page, { form: 'play-content-rating', config: absent, say: () => {}, bases: { asc: base + 'asc', play: base + 'playm' }, navMs: 800, file: tmpAccounts(), env: {} }); @@ -828,7 +831,7 @@ describe.skipIf(!browser)('play-app-record (headless, modeled pages)', () => { }, BROWSER_TEST_MS); } - test('an agreement answer that says yes is still never ticked', async () => { + scope.concurrent('an agreement answer that says yes is still never ticked', async () => { const page = await open('play-create-app.html'); const yes = ans().map(a => (a.agreement ? { ...a, value: true, certain: true } : a)); const s = await fillPage(page, def(), yes); @@ -838,7 +841,7 @@ describe.skipIf(!browser)('play-app-record (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('a renamed field: partial fill, the missing one left to the owner; nothing ticked or submitted', async () => { + scope.concurrent('a renamed field: partial fill, the missing one left to the owner; nothing ticked or submitted', async () => { const page = await open('play-create-app-renamed.html'); expect((await fillPage(page, def(), ans())).fallback).toBe(true); // first rounds: wait (the page may be loading) const s = await fillPage(page, def(), ans(), new Set(), true); @@ -851,7 +854,7 @@ describe.skipIf(!browser)('play-app-record (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('the opener inside a dialog, or on a page that is not the app list, is refused', async () => { + scope.concurrent('the opener inside a dialog, or on a page that is not the app list, is refused', async () => { const page = await open('blank.html'); await page.setContent('

All apps

'); expect((await navClick(page.getByRole('button', { name: 'Create app' }), true)).clicked).toBe(false); @@ -862,7 +865,7 @@ describe.skipIf(!browser)('play-app-record (headless, modeled pages)', () => { await page.close(); }, BROWSER_TEST_MS); - test('accounts: several and none picked -> stop and list; by play.developerAccount -> that one, remembered; by play.developerId -> directly', async () => { + scope.concurrent('accounts: several and none picked -> stop and list; by play.developerAccount -> that one, remembered; by play.developerId -> directly', async () => { const go = async (c: ShipConfig, file: string, said: string[], env: Record = {}) => { const page = await fresh(); const r = await openAcross(page, { form: 'play-app-record', config: c, say: m => { said.push(m); }, bases: pb('playn'), navMs: FOUND_MS, file, env }); @@ -904,7 +907,7 @@ describe.skipIf(!browser)('play-app-record (headless, modeled pages)', () => { await page.close(); }, SLOW_TEST_MS); - test('the chooser as it really looks (nested spans, icon fonts, role=button rows with data ids, an avatar row without an id): exact, case pair, prefix, not found', async () => { + scope.concurrent('the chooser as it really looks (nested spans, icon fonts, role=button rows with data ids, an avatar row without an id): exact, case pair, prefix, not found', async () => { const go = async (account: string | undefined, said: string[], play = 'playr') => { const page = await fresh(); const c = { ...config, play: { ...config.play!, ...(account ? { developerAccount: account } : {}) } } as ShipConfig; @@ -955,7 +958,7 @@ describe.skipIf(!browser)('play-app-record (headless, modeled pages)', () => { await page.close(); }, SLOW_TEST_MS); - test('fillSession: notes logged, form filled, the run ends without any submit', async () => { + scope.concurrent('fillSession: notes logged, form filled, the run ends without any submit', async () => { const b = await consoleBrowser(u => (u.hostname !== 'play.google.com' ? null : u.pathname === '/console' || u.pathname.endsWith('/app-list') ? 'play-home.html' : u.pathname === '/play-create-app.html' || u.pathname.endsWith('/create-new-app') ? 'play-create-app.html' : null)); const log = logLines(), said = log.lines; const c = { ...config, play: { ...config.play!, developerId: '4444' } } as ShipConfig; @@ -1007,7 +1010,7 @@ async function until(pred: () => boolean): Promise { } describe.skipIf(!browser)('sign-in detection: only the top-level URL, the main frame\'s form or an Apple sign-in frame', () => { - test('the Play account chooser with the account widget (accounts.google.com frames, avatar and email header) is NOT a sign-in page', async () => { + scope.concurrent('the Play account chooser with the account widget (accounts.google.com frames, avatar and email header) is NOT a sign-in page', async () => { const b = await consoleBrowser(playOgb); await b.page.goto('https://play.google.com/console/u/0/developers'); await until(() => frameOn(b.page, 'accounts.google.com') && frameOn(b.page, 'ogs.google.com')); @@ -1016,7 +1019,7 @@ describe.skipIf(!browser)('sign-in detection: only the top-level URL, the main f await b.context.close(); }, BROWSER_TEST_MS); - test('Play home with the widget frame is NOT a sign-in page', async () => { + scope.concurrent('Play home with the widget frame is NOT a sign-in page', async () => { const b = await consoleBrowser(playOgb); await b.page.goto('https://play.google.com/console/u/0/developers/7000000001/app-list'); await until(() => frameOn(b.page, 'accounts.google.com') && frameOn(b.page, 'ogs.google.com')); @@ -1024,7 +1027,7 @@ describe.skipIf(!browser)('sign-in detection: only the top-level URL, the main f await b.context.close(); }, BROWSER_TEST_MS); - test('a top-level accounts.google.com page IS a sign-in page, and the rule says why (URL without its query)', async () => { + scope.concurrent('a top-level accounts.google.com page IS a sign-in page, and the rule says why (URL without its query)', async () => { const b = await consoleBrowser(playOgb); await b.page.goto('https://accounts.google.com/v3/signin/identifier?continue=https%3A%2F%2Fplay.google.com%2Fconsole'); expect(await signInReason(b.page)).toEqual({ rule: 'the top-level URL is a sign-in page', url: 'https://accounts.google.com/v3/signin/identifier' }); @@ -1035,7 +1038,7 @@ describe.skipIf(!browser)('sign-in detection: only the top-level URL, the main f await page.close(); }, BROWSER_TEST_MS); - test('fillSession play-app-record: the chooser with the widget frames is passed, the form filled, no sign-in said', async () => { + scope.concurrent('fillSession play-app-record: the chooser with the widget frames is passed, the form filled, no sign-in said', async () => { const b = await consoleBrowser(playOgb); const log = logLines(), said = log.lines; const done = scope.run(fillSession(b, { form: 'play-app-record', config, ...fast(log) })); @@ -1047,7 +1050,7 @@ describe.skipIf(!browser)('sign-in detection: only the top-level URL, the main f await done; }, SLOW_TEST_MS); - test('a copied Chrome session that lands on accounts.google.com: said plainly with the rule; --debug-dump dumps the sign-in wait (at most 3), the watchdog once', async () => { + scope.concurrent('a copied Chrome session that lands on accounts.google.com: said plainly with the rule; --debug-dump dumps the sign-in wait (at most 3), the watchdog once', async () => { const dir = mkdtempSync(join(tmpdir(), 'sb-signin-debug-')); tmpDirs.push(dir); const b: ConsoleBrowser = { ...await consoleBrowser(u => (u.hostname === 'play.google.com' ? 'play-signed-out.html' : playOgb(u))), mode: 'copy' }; diff --git a/tests/console-flow.test.ts b/tests/console-flow.test.ts index 59a25b1..0c58354 100644 --- a/tests/console-flow.test.ts +++ b/tests/console-flow.test.ts @@ -7,13 +7,13 @@ import { afterAll, afterEach, beforeAll, describe, expect, test } from 'bun:test import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { fillSession } from '../cli/console/fill'; +import { engineWaits, fillSession } from '../cli/console/fill'; import type { ConsoleBrowser } from '../cli/console/session'; import { APPLE_DATA_TYPE_IDS, type AppleDataTypeId, type AppleTypeAnswer } from '../cli/lib/app-privacy'; import type { ShipConfig } from '../cli/lib/config'; import { type FlowWhere, flowAllowed, flowClass, FORM_BUTTONS, labelNames, setUpAllowed, setUpTexts, TYPE_ALIASES } from '../cli/lib/console-forms'; import { APPLE_PRIVACY_TYPES } from '../cli/lib/app-privacy'; -import { AFTER_ALL_MS, AFTER_EACH_MS, launchTestBrowser, logLines, testScope } from './helpers/browser'; +import { AFTER_ALL_MS, AFTER_EACH_MS, launchTestBrowser, logLines, TEST_TICK_MS, TEST_WAITS, testScope } from './helpers/browser'; const FIX = join(import.meta.dir, 'fixtures', 'console'); const base0: ShipConfig = { @@ -52,7 +52,9 @@ beforeAll(() => { base = `http://127.0.0.1:${server.port}/`; }); afterEach(() => scope.settle(), AFTER_EACH_MS); -afterAll(async () => { await shared.close(); server?.stop(true); }, AFTER_ALL_MS); +let restoreWaits = () => {}; +beforeAll(() => { restoreWaits = engineWaits(TEST_WAITS); }); +afterAll(async () => { restoreWaits(); await shared.close(); server?.stop(true); }, AFTER_ALL_MS); async function local(): Promise { const ctx = scope.track(await browser.newContext()); @@ -61,7 +63,7 @@ async function local(): Promise { const done = new Promise(r => ctx.on('close', () => { closed = true; r(); })); return { mode: 'profile', context: ctx, page, pages: () => ctx.pages(), closed: done, isClosed: () => closed, close: () => ctx.close(), cleanup: () => {} }; } -const fast = (log: ReturnType) => ({ say: log.say, tickMs: 100, navTimeoutMs: 5000, signInWaitMs: 30_000, env: {} }); +const fast = (log: ReturnType) => ({ say: log.say, tickMs: TEST_TICK_MS, navTimeoutMs: 5000, signInWaitMs: 30_000, env: {} }); const clicks = (b: ConsoleBrowser): Promise => b.page.evaluate(() => (window as any).__clicks); const navClicks = (b: ConsoleBrowser): Promise => b.page.evaluate(() => (window as any).__navClicks()); /** Guardrail: a line that tells the owner to click a navigation control themselves. */ @@ -139,7 +141,7 @@ describe('button classes (FORM_BUTTONS, flowClass, flowAllowed)', () => { }); describe.skipIf(!browser)('App Privacy: the whole questionnaire (fixture apple-app-privacy-flow.html)', () => { - test('first time: Get Started > yes > Next > data types > Save > each type (purposes > Next > linked > Next > tracking > Save); Publish never clicked', async () => { + scope.concurrent('first time: Get Started > yes > Next > data types > Save > each type (purposes > Next > linked > Next > tracking > Save); Publish never clicked', async () => { const b = await local(); const log = logLines(); const run = scope.run(fillSession(b, { form: 'apple-app-privacy', config: privacy(), startUrl: base + 'apple-app-privacy-flow.html', ...fast(log) })); @@ -159,7 +161,7 @@ describe.skipIf(!browser)('App Privacy: the whole questionnaire (fixture apple-a expect((await run).fallback).toBe(false); }, TEST_MS); - test('answers exist and the list differs from the config: Edit next to Data Types (never the Privacy Policy Edit), then every type', async () => { + scope.concurrent('answers exist and the list differs from the config: Edit next to Data Types (never the Privacy Policy Edit), then every type', async () => { const b = await local(); const log = logLines(); const config = privacy({ COARSE_LOCATION: { collected: true, purposes: ['ANALYTICS'], linked: false, tracking: false } }); @@ -174,7 +176,7 @@ describe.skipIf(!browser)('App Privacy: the whole questionnaire (fixture apple-a await run; }, TEST_MS); - test('an uncertain answer stops the walk on its page until the owner sets it, then it goes on', async () => { + scope.concurrent('an uncertain answer stops the walk on its page until the owner sets it, then it goes on', async () => { const b = await local(); const log = logLines(); const config = privacy({ DEVICE_ID: { collected: true, purposes: ['ANALYTICS'], tracking: false } }); // linked: not in the config @@ -194,7 +196,7 @@ describe.skipIf(!browser)('App Privacy: the whole questionnaire (fixture apple-a await run; }, TEST_MS); - test('no data collected: "No" and the dialog\'s Save, then the summary', async () => { + scope.concurrent('no data collected: "No" and the dialog\'s Save, then the summary', async () => { const b = await local(); const log = logLines(); const types = Object.fromEntries(APPLE_DATA_TYPE_IDS.map(id => [id, { collected: false }])); @@ -224,7 +226,7 @@ const SETUP = 'apple-app-privacy-setup.html'; const answersOf = (b: ConsoleBrowser) => b.page.evaluate(() => (window as any).__answers); describe.skipIf(!browser)('App Privacy: each data type set up from the real page ("Set Up " as plain text; fixture apple-app-privacy-setup.html)', () => { - test('every collected type is set up through its dialog (purposes > Next > linked > Next > two info steps > Next > Next > tracking > Save; purpose checkboxes without an accessible name); "?", Publish and the Edits never clicked; every step dumped', async () => { + scope.concurrent('every collected type is set up through its dialog (purposes > Next > linked > Next > two info steps > Next > Next > tracking > Save; purpose checkboxes without an accessible name); "?", Publish and the Edits never clicked; every step dumped', async () => { const dir = mkdtempSync(join(tmpdir(), 'sb-console-debug-')); try { const b = await local(); @@ -258,7 +260,7 @@ describe.skipIf(!browser)('App Privacy: each data type set up from the real page } finally { rmSync(dir, { recursive: true, force: true }); } }, TEST_MS); - test('an uncertain answer ("you decide") pauses on its step, names it, and goes on once the owner set it', async () => { + scope.concurrent('an uncertain answer ("you decide") pauses on its step, names it, and goes on once the owner set it', async () => { const b = await local(); const log = logLines(); const config = six({ PURCHASE_HISTORY: { collected: true, purposes: ['APP_FUNCTIONALITY'], tracking: false } }); // linked: the owner's @@ -279,7 +281,7 @@ describe.skipIf(!browser)('App Privacy: each data type set up from the real page await run; }, TEST_MS); - test('an unknown dialog step: a debug dump, a pause that names the data type, within the step timeout; nothing clicked past it', async () => { + scope.concurrent('an unknown dialog step: a debug dump, a pause that names the data type, within the step timeout; nothing clicked past it', async () => { const dir = mkdtempSync(join(tmpdir(), 'sb-console-debug-')); try { const b = await local(); @@ -294,7 +296,8 @@ describe.skipIf(!browser)('App Privacy: each data type set up from the real page const said = log.lines.join('\n'); expect(said).toMatch(/the "Device ID" dialog step after "Next" is not a page storebound knows .*Buttons in the dialog: "Back", "Cancel", "Next"/); expect(said).toContain(`Debug saved to ${dir}`); - const sub = readdirSync(dir).find(n => /unknown/.test(n))!; + // the Device ID pause's dump (on a loaded machine a slow earlier step can get a dump of its own first) + const sub = readdirSync(dir).find(n => /unknown/.test(n) && /device-id/.test(n))!; const pj = JSON.parse(readFileSync(join(dir, sub, 'page.json'), 'utf8')); expect(pj.dialogs).toContain('Device ID'); expect(pj.controls).toContainEqual({ role: 'switch', name: 'Account identity', checked: false }); @@ -308,7 +311,7 @@ describe.skipIf(!browser)('App Privacy: each data type set up from the real page } finally { rmSync(dir, { recursive: true, force: true }); } }, TEST_MS); - test('types set up before: the same purposes are left as they are, other purposes are opened with their Edit', async () => { + scope.concurrent('types set up before: the same purposes are left as they are, other purposes are opened with their Edit', async () => { const b = await local(); const log = logLines(); const url = `${base}${SETUP}?preset=${encodeURIComponent('Crash Data:App Functionality')}&preset=${encodeURIComponent('Device ID:Analytics')}`; @@ -327,7 +330,7 @@ describe.skipIf(!browser)('App Privacy: each data type set up from the real page }); describe.skipIf(!browser)('the other forms: navigation by itself, final buttons and draft saves left to the owner', () => { - test('content rating: Save draft and Next by itself on each page, a stop at the uncertain questions until answered, the Summary; its Save never clicked', async () => { + scope.concurrent('content rating: Save draft and Next by itself on each page, a stop at the uncertain questions until answered, the Summary; its Save never clicked', async () => { const b = await local(); const log = logLines(); const run = scope.run(fillSession(b, { form: 'play-content-rating', config: base0, startUrl: base + 'play-content-rating.html', ...fast(log) })); @@ -348,7 +351,7 @@ describe.skipIf(!browser)('the other forms: navigation by itself, final buttons await run; }, TEST_MS); - test('target audience: Next by itself, a stop at the uncertain appeal question until answered, then Next; Save stays the owner\'s', async () => { + scope.concurrent('target audience: Next by itself, a stop at the uncertain appeal question until answered, then Next; Save stays the owner\'s', async () => { const b = await local(); const log = logLines(); const run = scope.run(fillSession(b, { form: 'play-target-audience', config: base0, startUrl: base + 'play-target-audience.html', ...fast(log) })); @@ -362,7 +365,7 @@ describe.skipIf(!browser)('the other forms: navigation by itself, final buttons await run; }, TEST_MS); - test('New App (App Store) and Create app (Play): filled, the final button is the owner\'s, no navigation asked of the owner', async () => { + scope.concurrent('New App (App Store) and Create app (Play): filled, the final button is the owner\'s, no navigation asked of the owner', async () => { for (const [form, file, final] of [['apple-app-record', 'apple-app-record.html', 'Create'], ['play-app-record', 'play-create-app.html', 'Create app']] as const) { const b = await local(); const log = logLines(); @@ -378,7 +381,7 @@ describe.skipIf(!browser)('the other forms: navigation by itself, final buttons }); describe.skipIf(!browser)('watchdog: an unknown page after a click never means a silent wait', () => { - test('Next leads to a page storebound does not know: heartbeat, a sanitized debug dump, the manual steps, a bounded stop', async () => { + scope.concurrent('Next leads to a page storebound does not know: heartbeat, a sanitized debug dump, the manual steps, a bounded stop', async () => { const dir = mkdtempSync(join(tmpdir(), 'sb-console-debug-')); try { const b = await local(); @@ -399,7 +402,8 @@ describe.skipIf(!browser)('watchdog: an unknown page after a click never means a expect(said).toContain(`Debug saved to ${dir}`); expect(said).toContain('this part is left to the calling agent with its browser tool (`storebound console plan apple-app-privacy`'); expect(said).toContain('it stops here'); - const [sub] = readdirSync(dir); + // the dump of the unknown page (on a loaded machine a slow earlier step can get a dump of its own first) + const sub = readdirSync(dir).find(n => /unknown-the-page-after-next/.test(n)); const files = readdirSync(join(dir, sub!)).sort(); expect(files).toEqual(['page.html', 'page.json', 'screen.jpg']); const pj = JSON.parse(readFileSync(join(dir, sub!, 'page.json'), 'utf8')); @@ -412,7 +416,7 @@ describe.skipIf(!browser)('watchdog: an unknown page after a click never means a } finally { rmSync(dir, { recursive: true, force: true }); } }, TEST_MS); - test('--debug-dump: a dump at the start, after each click and after each filled page', async () => { + scope.concurrent('--debug-dump: a dump at the start, after each click and after each filled page', async () => { const dir = mkdtempSync(join(tmpdir(), 'sb-console-debug-')); try { const b = await local(); diff --git a/tests/console-persist.test.ts b/tests/console-persist.test.ts index 30ad021..3bcb698 100644 --- a/tests/console-persist.test.ts +++ b/tests/console-persist.test.ts @@ -31,7 +31,7 @@ afterAll(() => { server?.stop(true); }); describe.skipIf(!real.label)(`storebound's own profile (${real.label ?? 'no real Chrome'})`, () => { for (const store of ['apple', 'play'] as const) { - test(`${store}: a session cookie survives closing and relaunching storebound's own profile`, async () => { + test.concurrent(`${store}: a session cookie survives closing and relaunching storebound's own profile`, async () => { const dir = mkdtempSync(join(tmpdir(), `storebound-profile-${store}-`)); const root = join(import.meta.dir, '..'); const { chromium } = await import('playwright'); diff --git a/tests/console-session.test.ts b/tests/console-session.test.ts index faaa061..340dc54 100644 --- a/tests/console-session.test.ts +++ b/tests/console-session.test.ts @@ -193,7 +193,7 @@ const hasChrome = process.platform === 'darwin' && existsSync(CHROME); if (!hasChrome) console.warn('console-session integration test skipped: Google Chrome is not installed (or not macOS)'); describe.skipIf(!hasChrome)('copy mode with real Chrome (throwaway profile, local site)', () => { - test('a cookie set in the source profile is there in the copy; another host\'s is filtered out; the copy is deleted on close', async () => { + test.concurrent('a cookie set in the source profile is there in the copy; another host\'s is filtered out; the copy is deleted on close', async () => { let chromium: any; try { chromium = (await import('playwright')).chromium; } catch { console.warn('skipped: playwright not installed'); return; } const server = Bun.serve({ hostname: '127.0.0.1', port: 0, fetch: req => { @@ -238,7 +238,7 @@ describe.skipIf(!hasChrome)('copy mode with real Chrome (throwaway profile, loca }); describe.skipIf(!hasChrome)('the console window has no automation markers (real Chrome, throwaway profile, local page)', () => { - test('navigator.webdriver is false and the real command line has no --enable-automation', async () => { + test.concurrent('navigator.webdriver is false and the real command line has no --enable-automation', async () => { const server = Bun.serve({ hostname: '127.0.0.1', port: 0, fetch: () => new Response( '

', { headers: { 'content-type': 'text/html' } }) }); diff --git a/tests/helpers/browser.ts b/tests/helpers/browser.ts index 89d4879..fc14023 100644 --- a/tests/helpers/browser.ts +++ b/tests/helpers/browser.ts @@ -13,6 +13,8 @@ // - polling sleeps in the waits for log lines: event-driven now (resolved on each new line) // Hook timeouts are bounded by the numbers below (each close is capped), never open-ended. /* eslint-disable @typescript-eslint/no-explicit-any */ +import { AsyncLocalStorage } from 'node:async_hooks'; +import { test } from 'bun:test'; /** Launching Chromium: well above a cold start on a loaded machine (seconds), far below "hung". */ export const LAUNCH_TIMEOUT_MS = 60_000; @@ -25,6 +27,16 @@ export const CLOSE_TIMEOUT_MS = 15_000; export const AFTER_EACH_MS = 2 * CLOSE_TIMEOUT_MS + 5_000; export const AFTER_ALL_MS = CLOSE_TIMEOUT_MS + 5_000; +/** + * The console engine's timings for tests on local fixture pages (production: a 1 s watch round, 200 ms poll rounds, + * a missing step waited out for the whole step timeout). A round of 20 ms; a poll ends as "not found" once the page + * is loaded and unchanged for 1.5 s (well above the one timer a fixture page has, asc-shell.html's 700 ms redirect), + * so a step that is not there no longer costs the whole navTimeoutMs, while a step that is there is still waited for + * as long as the machine needs. Set with engineWaits (cli/console/fill.ts) in beforeAll, restored in afterAll. + */ +export const TEST_TICK_MS = 20; +export const TEST_WAITS = { pollMs: 50, quietMs: 1500, settleMs: 50, missMs: 300 }; + /** Waits for `p` (its error ignored) or `ms`, whichever is first: true when `p` settled in time. Never rejects. */ export async function capped(p: Promise, ms: number): Promise { let t: ReturnType | undefined; @@ -88,11 +100,8 @@ export async function persistentBrowser(): Promise<{ label: string | null; skipW : { label: null, skipWhy: 'neither Google Chrome nor Playwright\'s full Chromium is installed (the headless shell drops session cookies on relaunch); install Google Chrome or run `bunx playwright install chromium`' }; } -/** - * The contexts and engine runs of the current test. `newPage` opens a page in a fresh context (as browser.newPage - * does) and tracks it; `track` adds one opened elsewhere; `settle` (afterEach) closes them all and waits for the runs. - */ -export function testScope(browser: () => any) { +/** The contexts and engine runs of one test (see testScope). */ +function plainScope(browser: () => any) { const contexts = new Set(); const runs = new Set>(); return { @@ -113,6 +122,33 @@ export function testScope(browser: () => any) { }; } +/** + * The contexts and engine runs of the current test. `newPage` opens a page in a fresh context (as browser.newPage + * does) and tracks it; `track` adds one opened elsewhere; `settle` (afterEach) closes them all and waits for the runs. + * `concurrent` declares a browser test that runs at the same time as the file's other concurrent tests + * (test.concurrent, at most --max-concurrency at once: tools/ci.ts passes BROWSER_CONCURRENCY from tools/ci-plan.ts): it gets a scope of + * its own (AsyncLocalStorage, so newPage / track / run inside it reach that scope), settled when it ends, inside its + * timeout. The tests share the one browser; each has its own contexts, pages and log. + */ +export function testScope(browser: () => any) { + type Scope = ReturnType; + const shared = plainScope(browser); + const own = new AsyncLocalStorage(); + const cur = (): Scope => own.getStore() ?? shared; + return { + newPage: (opts?: Record): Promise => cur().newPage(opts), + track: (ctx: T): T => cur().track(ctx), + run: (p: Promise): Promise => cur().run(p), + settle: (): Promise => shared.settle(), + concurrent(name: string, fn: () => Promise, ms: number): void { + test.concurrent(name, () => { + const s = plainScope(browser); + return own.run(s, async () => { try { await fn(); } finally { await s.settle(); } }); + }, ms + AFTER_EACH_MS); + }, + }; +} + /** * The log lines of an engine run (`say`), with event-driven waits: `wait` resolves on the line that makes `pred` * true (checked on every new line, no polling) and fails after `ms` with every line seen so far. diff --git a/tools/ci-plan.ts b/tools/ci-plan.ts index 3b47e77..5654f13 100644 --- a/tools/ci-plan.ts +++ b/tools/ci-plan.ts @@ -16,6 +16,11 @@ export const HEAVY = [ 'tests/capture-web.test.ts', 'tests/console-session.test.ts', 'tests/console-persist.test.ts', 'tests/console-flow.test.ts', ]; +/** Browser tests of one file at the same time (bun test --max-concurrency for the shards with a HEAVY file; the + * tests opt in with testScope().concurrent, tests/helpers/browser.ts): enough to overlap their waits, few enough that + * a loaded machine does not stretch each one towards its timeout. */ +export const BROWSER_CONCURRENCY = 4; + /** Changes that can affect every test: run the whole suite. */ export const GLOBAL = /^(package\.json|bun\.lock|bunfig\.toml|tsconfig\.json|tests\/helpers\/preload\.ts)$/; @@ -50,6 +55,17 @@ export function writeTimings(t: Record, file = TIMINGS): void { try { mkdirSync(dirname(file), { recursive: true }); writeFileSync(file, `${JSON.stringify(t, null, 1)}\n`); } catch { /* cache only */ } } +/** + * The recorded time of each file after a run: a new file takes its measurement; a known one moves halfway towards it + * and never above it. The machine is shared, so one run under heavy load (other sessions' CI, cargo) must not make a + * file look several times slower for the next split, while a file that really got faster or slower still converges. + */ +export function mergeTimings(old: Record, fresh: Record): Record { + const out = { ...old }; + for (const [f, t] of Object.entries(fresh)) out[f] = old[f] === undefined ? t : Math.min(t, (old[f]! + t) / 2); + return out; +} + /** Per-file seconds from bun's JUnit report (the top-level of each file). */ export function junitTimes(xml: string): Record { const out: Record = {}; diff --git a/tools/ci.ts b/tools/ci.ts index 4f13c35..9d986e1 100644 --- a/tools/ci.ts +++ b/tools/ci.ts @@ -5,8 +5,9 @@ // bun run ci:quick inner loop: typecheck + the tests related to the change (git diff against origin/main, // mapped through imports) + scrub // The tests run as shards: N = CPU cores - 1 (max 8; STOREBOUND_CI_SHARDS=n overrides) parallel `bun test ` -// processes, balanced by the durations recorded after the last run (node_modules/.cache/storebound-ci/, falling back -// to file size). Browser suites (tools/ci-plan.ts HEAVY) each get a shard of their own. +// processes, balanced by the durations recorded after earlier runs (node_modules/.cache/storebound-ci/, smoothed by +// mergeTimings so one run on a loaded machine does not skew the split; falling back to file size). Browser suites +// (tools/ci-plan.ts HEAVY) each get a shard of their own and run their opted-in tests BROWSER_CONCURRENCY at a time. // `bun test` covers the CLI and the desktop webview (apps/desktop/tests), the site overflow check and the // no-hand-written-SVG guard; the desktop boot check (tools/desktop-boot-check.ts: the built window in headless // Chromium, needs playwright's browser or Google Chrome) runs after the desktop build. @@ -17,7 +18,7 @@ import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { homedir, tmpdir } from 'node:os'; import { join } from 'node:path'; import { hostTarget, OUT_DIR, outName } from './build-sidecars'; -import { changedFiles, HEAVY, junitTimes, listTests, planShards, readTimings, relatedTests, ROOT, rustNeeded, writeTimings } from './ci-plan'; +import { BROWSER_CONCURRENCY, changedFiles, HEAVY, junitTimes, listTests, mergeTimings, planShards, readTimings, relatedTests, ROOT, rustNeeded, writeTimings } from './ci-plan'; interface Task { name: string; cmd: string[]; cwd?: string; env?: Record; after?: string[]; junit?: string } @@ -39,7 +40,8 @@ function addTests(files: string[]) { // the built-bundle check in apps/desktop/tests/webview.test.ts runs as "desktop bundle check" once the build is // done; here it would read dist/ while vite rewrites it // browser suites share the machine with the other steps: a browser start or close can pass bun's 5 s default - const timeout = shard.some(f => HEAVY.includes(f)) ? ['--timeout=30000'] : []; + // browser tests that opt in (testScope().concurrent) run BROWSER_CONCURRENCY at a time within their file + const timeout = shard.some(f => HEAVY.includes(f)) ? ['--timeout=30000', `--max-concurrency=${BROWSER_CONCURRENCY}`] : []; add({ name: `tests ${i + 1}/${all.length}`, cmd: ['bun', 'test', ...timeout, '--reporter=junit', `--reporter-outfile=${junit}`, ...shard.map(f => `./${f}`)], env: { STOREBOUND_SKIP_DIST_CHECK: '1' }, junit }); }); @@ -156,12 +158,12 @@ while ((pending.size || active.size) && !failed) { if (failed) await Promise.allSettled([...active]); // record per-file durations for the next split (only from shards that finished) -const timings = readTimings(); +const fresh: Record = {}; for (const r of runs.values()) { if (!r.task.junit || r.end === undefined) continue; - try { Object.assign(timings, junitTimes(readFileSync(r.task.junit, 'utf8'))); } catch { /* no report */ } + try { Object.assign(fresh, junitTimes(readFileSync(r.task.junit, 'utf8'))); } catch { /* no report */ } } -writeTimings(timings); +writeTimings(mergeTimings(readTimings(), fresh)); rmSync(junitDir, { recursive: true, force: true }); console.log(`\n${quick ? 'ci:quick' : 'ci'} summary (wall ${((Date.now() - t0) / 1000).toFixed(1)}s)`); diff --git a/tools/desktop-boot-check.ts b/tools/desktop-boot-check.ts index 9e970ca..dd50f83 100644 --- a/tools/desktop-boot-check.ts +++ b/tools/desktop-boot-check.ts @@ -13,8 +13,11 @@ // build picker, the plan panel and a console plan (the real sidecar) by DOM text, then every tab of the app and the // Dependencies screen. // bun tools/desktop-boot-check.ts [--dist apps/desktop/dist] [--only d1-missing-folders] [--json] -// Uses playwright: Google Chrome when installed (channel "chrome"), else playwright's Chromium -// (bunx playwright install chromium). Part of `bun run ci` and of `tools/desktop-release.ts mac`. +// Speed: one browser (killed at the end, never closed gracefully), one page per case, the cases in parallel +// (STOREBOUND_BOOT_CHECK_JOBS, default 8) against the one build; no fixed sleeps: after each action it waits for a +// condition (the element, the text, or settle(): no host call in flight and no DOM change for QUIET_MS). +// Uses playwright: its headless shell (bunx playwright install chromium-headless-shell), else Google Chrome +// (channel "chrome"), else its full Chromium. Part of `bun run ci` and of `tools/desktop-release.ts mac`. import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'; import { extname, join, resolve } from 'node:path'; import type { Browser, Page } from 'playwright'; @@ -173,12 +176,21 @@ const initScript = (freeze: boolean) => `(() => { unregisterCallback(id) { delete cbs[id]; }, convertFileSrc(p) { return p; }, async invoke(cmd, args) { - const r = await window.__bootCheckHost(cmd, args || {}); - if ('err' in r) throw r.err; - return r.ok; + window.__bootInflight = (window.__bootInflight || 0) + 1; + window.__bootChanged = performance.now(); + try { + const r = await window.__bootCheckHost(cmd, args || {}); + if ('err' in r) throw r.err; + return r.ok; + } finally { window.__bootInflight--; window.__bootChanged = performance.now(); } }, }; window.__TAURI_EVENT_PLUGIN_INTERNALS__ = { unregisterListener() {} }; + // settle(): the last host call or DOM change, so the check waits for a quiet window instead of a fixed time + window.__bootInflight = 0; + window.__bootChanged = performance.now(); + new MutationObserver(() => { window.__bootChanged = performance.now(); }) + .observe(document, { childList: true, subtree: true, attributes: true, characterData: true }); })();`; // --------------------------------------------------------------------------------------------------------------------- @@ -211,6 +223,32 @@ async function state(page: Page) { })); } +/** A quiet window: no host call in flight and no DOM change for this long means the last action is done. */ +const QUIET_MS = 150; +/** Upper bound of one wait for a condition (a loaded machine); a wait that runs out is a failure or a check as-is. */ +const WAIT_MS = 15_000; + +/** + * Waits until the window is quiet (initScript's counters: no host call in flight, no DOM change for QUIET_MS): what a + * click or the boot started has drawn. Condition-based, no fixed sleep; a window that never gets quiet (an animation + * written by script) is checked as it is after `ms`. + */ +async function settle(page: Page, ms = WAIT_MS): Promise { + await page.waitForFunction((q: number) => { + const w = window as unknown as { __bootInflight: number; __bootChanged: number }; + return w.__bootInflight === 0 && performance.now() - w.__bootChanged >= q; + }, QUIET_MS, { timeout: ms, polling: 50 }).catch(() => {}); +} + +/** Waits until `sel` is gone (a dialog or panel closed); false when it is still there after `ms`. */ +const gone = (page: Page, sel: string, ms = WAIT_MS): Promise => + page.waitForSelector(sel, { state: 'detached', timeout: ms }).then(() => true, () => false); + +/** Waits until the first `sel` contains `text` (whitespace folded); false after `ms`. */ +const shows = (page: Page, sel: string, text: string, ms = WAIT_MS): Promise => + page.waitForFunction(([s, t]) => (document.querySelector(s!) as HTMLElement | null)?.innerText.replace(/\s+/g, ' ').includes(t!) ?? false, + [sel, text] as const, { timeout: ms, polling: 50 }).then(() => true, () => false); + async function runCase(browser: Browser, name: string, sidecar: Sidecar, o: { breakBundle?: boolean } = {}): Promise { const freeze = conf.app.security.freezePrototype === true; const server = serve(DIST, cspHeader(conf.app.security.csp), !!o.breakBundle); @@ -232,13 +270,13 @@ async function runCase(browser: Browser, name: string, sidecar: Sidecar, o: { br try { await page.goto(`http://127.0.0.1:${server.port}/`, { waitUntil: 'load' }); if (o.breakBundle) { - await page.waitForSelector('#boot-fallback', { timeout: 8000 }).catch(() => {}); + await page.waitForSelector('#boot-fallback', { timeout: WAIT_MS }).catch(() => {}); const s = await state(page); const ok = s.crash.some(c => c.startsWith('boot')); return { name, ok, problems: ok ? [] : ['the boot.js error panel did not appear for a missing bundle'], visited: 0, notes: [`boot.js error panel shown: ${ok}`, ...logged.map(l => `webview.log: ${l}`)] }; } - await page.waitForFunction(() => document.documentElement.dataset.booted === '1', null, { timeout: 15000 }).catch(() => problems.push('React never drew the window (no data-booted)')); - await page.waitForTimeout(1500); + await page.waitForFunction(() => document.documentElement.dataset.booted === '1', null, { timeout: WAIT_MS }).catch(() => problems.push('React never drew the window (no data-booted)')); + await settle(page); const check = async (where: string) => { const s = await state(page); if (freeze && !s.frozen) problems.push('Object.prototype is not frozen in the page (the check would not see the override mistake)'); @@ -251,7 +289,7 @@ async function runCase(browser: Browser, name: string, sidecar: Sidecar, o: { br if (await wizard.count()) { notes.push('welcome guide shown'); await wizard.press('Escape').catch(() => {}); - await page.waitForTimeout(300); + await gone(page, '.wizard[role="dialog"]'); if (await wizard.count()) problems.push('the welcome guide does not close with Escape'); } // every sidebar entry: the portfolio, each app (folders that are gone included), each workspace screen @@ -259,8 +297,8 @@ async function runCase(browser: Browser, name: string, sidecar: Sidecar, o: { br for (let i = 0; i < count; i++) { const b = page.locator('nav.side button.nav').nth(i); const label = ((await b.innerText().catch(() => '')) || `entry ${i}`).split('\n')[0]!.trim(); - await b.click({ timeout: 5000 }).catch(e => problems.push(`could not open ${label}: ${String(e).split('\n')[0]}`)); - await page.waitForTimeout(700); + await b.click({ timeout: WAIT_MS }).catch(e => problems.push(`could not open ${label}: ${String(e).split('\n')[0]}`)); + await settle(page); await check(label); visited++; } @@ -268,8 +306,9 @@ async function runCase(browser: Browser, name: string, sidecar: Sidecar, o: { br for (const p of fx.projects_list.filter(x => !isDir(x.path))) { const item = page.locator('nav.side .nav-app', { hasText: p.name }).first(); if (!(await item.count())) { problems.push(`${p.name}: not in the sidebar`); continue; } - await item.click(); - await page.waitForTimeout(500); + await item.click({ timeout: WAIT_MS }); + await shows(page, '#content', 'Folder not found'); + await settle(page); const text = await page.locator('#content').innerText(); if (!text.includes('Folder not found') || !text.includes('Locate…') || !text.includes('Remove from list')) problems.push(`${p.name}: no "Folder not found" panel with Locate… / Remove`); else notes.push(`${p.name}: Folder not found, Locate… / Remove`); @@ -321,18 +360,18 @@ async function runSubmitCase(browser: Browser, sidecar: Sidecar, base: 'd3-full' const crashCheck = async (where: string) => { for (const c of (await state(page)).crash) problems.push(`${where}: error panel ${c}`); }; try { await page.goto(`http://127.0.0.1:${server.port}/`, { waitUntil: 'load' }); - await page.waitForFunction(() => document.documentElement.dataset.booted === '1', null, { timeout: 15000 }).catch(() => problems.push('React never drew the window (no data-booted)')); - await page.waitForTimeout(1200); + await page.waitForFunction(() => document.documentElement.dataset.booted === '1', null, { timeout: WAIT_MS }).catch(() => problems.push('React never drew the window (no data-booted)')); + await settle(page); const wizard = page.locator('.wizard[role="dialog"]'); - if (await wizard.count()) { await wizard.press('Escape').catch(() => {}); await page.waitForTimeout(300); } + if (await wizard.count()) { await wizard.press('Escape').catch(() => {}); await gone(page, '.wizard[role="dialog"]'); } const app = fx.projects_list.find(p => isDir(p.path))!; // the sidebar names the app by its config name once the offline portfolio is read const item = page.locator('nav.side .nav-app').filter({ hasText: base === 'd3-full' ? /Prism Puddle|demo-game/ : new RegExp(app.name) }).first(); - await item.click({ timeout: 8000 }).catch(e => problems.push(`could not open the app: ${String(e).split('\n')[0]}`)); - await page.waitForTimeout(800); - await page.locator('[role="tab"]#tab-releases').click({ timeout: 5000 }).catch(() => {}); - await page.waitForSelector('[data-section="submit"]', { timeout: 8000 }).catch(() => problems.push('Releases: no send-for-review section')); - await page.waitForTimeout(600); + await item.click({ timeout: WAIT_MS }).catch(e => problems.push(`could not open the app: ${String(e).split('\n')[0]}`)); + await settle(page); + await page.locator('[role="tab"]#tab-releases').click({ timeout: WAIT_MS }).catch(() => {}); + await page.waitForSelector('[data-section="submit"]', { timeout: WAIT_MS }).catch(() => problems.push('Releases: no send-for-review section')); + await settle(page); const rel = await text('[data-section="submit"]'); for (const want of SUBMIT_EXPECT[variant]) if (!rel.includes(want)) problems.push(`Releases (${variant}): missing "${want}"`); if (rel.includes('storebound never submits for review')) problems.push('Releases: the old "submit stays in the console" note is still there'); @@ -346,15 +385,16 @@ async function runSubmitCase(browser: Browser, sidecar: Sidecar, base: 'd3-full' const action = page.locator('[data-section="submit"] button', { hasText: variant === 'ready' ? 'Submit for review' : 'Resubmit' }).first(); if (!(await action.isDisabled())) problems.push('the submit button is enabled before a build is picked'); await page.locator('[data-section="submit"] select').first().selectOption('52'); - await action.click({ timeout: 5000 }); - await page.waitForSelector('.oa-panel', { timeout: 8000 }).catch(() => problems.push('no plan panel after the submit button')); + await action.click({ timeout: WAIT_MS }); + await page.waitForSelector('.oa-panel', { timeout: WAIT_MS }).catch(() => problems.push('no plan panel after the submit button')); + await settle(page); const panel = await text('.oa-panel'); for (const want of ['Plan (dry run, nothing sent yet)', 'STEP', 'attach build 52', 'a system dialog shows this plan again']) if (!panel.includes(want)) problems.push(`plan panel: missing "${want}"`); - await page.locator('.oa-panel .lb-foot button', { hasText: 'Continue' }).click({ timeout: 5000 }).catch(e => problems.push(`Continue: ${String(e).split('\n')[0]}`)); - await page.waitForTimeout(500); + await page.locator('.oa-panel .lb-foot button', { hasText: 'Continue' }).click({ timeout: WAIT_MS }).catch(e => problems.push(`Continue: ${String(e).split('\n')[0]}`)); + await shows(page, '.oa-panel', 'nothing was sent'); if (!(await text('.oa-panel')).includes('nothing was sent')) problems.push('plan panel: no "nothing was sent" after the dialog said Cancel'); await page.keyboard.press('Escape'); - await page.waitForTimeout(300); + await gone(page, '.oa-panel'); if (await page.locator('.oa-panel').count()) problems.push('plan panel does not close with Escape'); const want = variant === 'ready' ? 'owner.apple_submit' : 'owner.apple_resubmit'; if (!live.calls.some(c => c.startsWith(`dry ${want} {"build":"52","version":"2.4.0"}`))) problems.push(`no dry run of ${want} with build 52 and version 2.4.0: ${live.calls.join(' | ')}`); @@ -364,27 +404,29 @@ async function runSubmitCase(browser: Browser, sidecar: Sidecar, base: 'd3-full' const codeOpts = await codes.locator('option').evaluateAll(os => os.map(o => `${(o as HTMLOptionElement).value}${(o as HTMLOptionElement).disabled ? '!' : ''}`)); if (JSON.stringify(codeOpts) !== JSON.stringify(['', '24', '23!', '22'])) problems.push(`versionCode picker: ${JSON.stringify(codeOpts)}`); await codes.selectOption('24'); - await page.locator('[data-section="submit"] button', { hasText: 'Send to production' }).first().click({ timeout: 5000 }); - await page.waitForSelector('.oa-panel', { timeout: 8000 }).catch(() => problems.push('no plan panel after Send to production')); + await page.locator('[data-section="submit"] button', { hasText: 'Send to production' }).first().click({ timeout: WAIT_MS }); + await page.waitForSelector('.oa-panel', { timeout: WAIT_MS }).catch(() => problems.push('no plan panel after Send to production')); + await settle(page); const pp = await text('.oa-panel'); for (const w of ['--version-code 24 --production completed', 'managed publishing is off']) if (!pp.includes(w)) problems.push(`production plan: missing "${w}"`); await page.keyboard.press('Escape'); - await page.waitForTimeout(300); + await gone(page, '.oa-panel'); if (!live.calls.some(c => c.startsWith('dry owner.play_production {"version_code":"24","status":"completed"}'))) problems.push(`no dry run of owner.play_production 24: ${live.calls.join(' | ')}`); // a console plan from the real sidecar (offline): Play's Send changes for review - await page.locator('[data-section="submit"] button', { hasText: 'Send changes for review' }).first().click({ timeout: 5000 }); - await page.waitForSelector('.cp-panel', { timeout: 10000 }).catch(() => problems.push('no console plan panel')); + await page.locator('[data-section="submit"] button', { hasText: 'Send changes for review' }).first().click({ timeout: WAIT_MS }); + await page.waitForSelector('.cp-panel', { timeout: WAIT_MS }).catch(() => problems.push('no console plan panel')); + await settle(page); const cp = await text('.cp-panel'); for (const w of ['send the first production release for review', 'Send changes for review', 'Check the result']) if (!cp.includes(w)) problems.push(`console plan: missing "${w}"`); await page.keyboard.press('Escape'); - await page.waitForTimeout(300); + await gone(page, '.cp-panel'); await crashCheck('Releases'); visited++; // every tab of the app, then App info's console steps and the Dependencies screen const tabs = await page.locator('[role="tab"]').evaluateAll(ts => ts.map(x => x.id)); for (const id of tabs) { - await page.locator(`[role="tab"]#${id}`).click({ timeout: 5000 }).catch(e => problems.push(`tab ${id}: ${String(e).split('\n')[0]}`)); - await page.waitForTimeout(500); + await page.locator(`[role="tab"]#${id}`).click({ timeout: WAIT_MS }).catch(e => problems.push(`tab ${id}: ${String(e).split('\n')[0]}`)); + await settle(page); await crashCheck(id); visited++; if (id === 'tab-appinfo') { @@ -395,8 +437,9 @@ async function runSubmitCase(browser: Browser, sidecar: Sidecar, base: 'd3-full' } notes.push(`tabs: ${tabs.join(', ')}`); const deps = page.locator('nav.side button.nav', { hasText: 'Dependencies' }).first(); - await deps.click({ timeout: 5000 }).catch(() => {}); - await page.waitForTimeout(800); + await deps.click({ timeout: WAIT_MS }).catch(() => {}); + await shows(page, '#content', 'Android SDK'); + await settle(page); if (!(await text()).includes('Android SDK')) problems.push('Dependencies: no Android SDK card'); await crashCheck('Dependencies'); visited++; @@ -412,27 +455,47 @@ async function runSubmitCase(browser: Browser, sidecar: Sidecar, base: 'd3-full' return { name, ok: problems.length === 0, problems: [...new Set(problems)], visited, notes }; } -async function launch(): Promise { +/** + * One headless browser for every case, as a browser server that is KILLED at the end (a throwaway profile, nothing to + * flush): Google Chrome's graceful close takes 10-30 s after http pages (tests/helpers/browser.ts). Playwright's + * headless shell first (fast to start), else Google Chrome (channel "chrome"), else playwright's full Chromium. + */ +async function launch(): Promise<{ browser: Browser; kill: () => Promise }> { const { chromium } = await import('playwright'); - try { return await chromium.launch({ channel: 'chrome', headless: true }); } catch { /* no Google Chrome */ } - return chromium.launch({ headless: true }); + const opts = { headless: true, timeout: 60_000 }; + let server: Awaited> | undefined, first: unknown; + for (const o of [{}, { channel: 'chrome' }, { channel: 'chromium' }]) { + try { server = await chromium.launchServer({ ...opts, ...o }); break; } catch (e) { first ??= e; } + } + if (!server) throw first; + const s = server; + const browser = await chromium.connect(s.wsEndpoint(), { timeout: 60_000 }).catch(async (e: unknown) => { await s.kill(); throw e; }); + return { browser, kill: async () => { await Promise.race([browser.close().catch(() => {}), Bun.sleep(5000)]); await s.kill(); } }; } +/** Cases at once (a page each, one browser); STOREBOUND_BOOT_CHECK_JOBS overrides. */ +const JOBS = Math.max(1, Number(process.env.STOREBOUND_BOOT_CHECK_JOBS) || 8); + export async function bootCheck(): Promise { if (!existsSync(join(DIST, 'index.html'))) throw new Error(`no built frontend at ${DIST} (bun run --cwd apps/desktop build)`); const names = readdirSync(FIXTURES).filter(f => f.endsWith('.host.json')).map(f => f.replace(/\.host\.json$/, '')).sort(); if (names.length < 3) throw new Error(`expected the state fixtures in ${FIXTURES}`); const sidecar = new Sidecar(); - await sidecar.start(); - const browser = await launch(); - const results: CaseResult[] = []; + const [, { browser, kill }] = await Promise.all([sidecar.start(), launch()]); + // every case in its own context and page, JOBS at a time, against the one build; results keep the case order + const cases: (() => Promise)[] = []; + for (const n of names) if (!ONLY.length || ONLY.includes(n)) cases.push(() => runCase(browser, n, sidecar)); + if (!ONLY.length || ONLY.includes('missing-bundle')) cases.push(() => runCase(browser, 'missing-bundle', sidecar, { breakBundle: true })); + for (const base of ['d3-full', 'd1-missing-folders'] as const) for (const v of ['ready', 'unresolved'] as const) + if (!ONLY.length || ONLY.includes('submit') || ONLY.includes(`submit-${v}`)) cases.push(() => runSubmitCase(browser, sidecar, base, v)); + const results: CaseResult[] = new Array(cases.length); + let next = 0; try { - for (const n of names) if (!ONLY.length || ONLY.includes(n)) results.push(await runCase(browser, n, sidecar)); - if (!ONLY.length || ONLY.includes('missing-bundle')) results.push(await runCase(browser, 'missing-bundle', sidecar, { breakBundle: true })); - for (const base of ['d3-full', 'd1-missing-folders'] as const) for (const v of ['ready', 'unresolved'] as const) - if (!ONLY.length || ONLY.includes('submit') || ONLY.includes(`submit-${v}`)) results.push(await runSubmitCase(browser, sidecar, base, v)); + await Promise.all(Array.from({ length: Math.min(JOBS, cases.length) }, async () => { + while (next < cases.length) { const i = next++; results[i] = await cases[i]!(); } + })); } finally { - await browser.close(); + await kill(); sidecar.stop(); } return results;