CRT/storebound
1
0
Fork 0
Code Issues Pull requests Releases 1 Packages 1 Activity Actions

ci speed: desktop boot check runs its cases in parallel (one browser, killed at the end; a page per case; condition waits instead of fixed sleeps: element, text or a quiet window of host calls and DOM); shots capture / render close the browser with a 3 s cap, then kill it; console engine waits injectable for tests (WAITS / engineWaits: poll rounds, a missing step ends once its loaded page stays quiet, team switch settle, field-miss grace), fillPage looks up each data type's dialog once per round and skips dialog sections when no dialog is open; browser tests run 4 at a time per file (testScope().concurrent with a scope per test, --max-concurrency in ci), 20 ms rounds; capture-web counts boots per test and runs its browser tests concurrently; console-session / -persist Chrome tests concurrent; recorded test timings smoothed (mergeTimings) so one loaded run does not skew the shard split

Browse source
This commit is contained in:
CRTkafa 2026-10-13 02:20:03 +08:00
parent f2d7062bcb
commit 9ba8471002
16 changed files with 390 additions and 178 deletions
Download patch file Download diff file Expand all files Collapse all files

12
cli/console/accounts.ts
View file

@ -25,7 +25,7 @@ import { dirname, join } from 'node:path';
import type { ShipConfig } from '../lib/config';
import type { FormId } from '../lib/console-answers';
import { hasToken, type Name, NAV_BASES, type NavBases, type NavStep, navPlan, playAppIdOf, playDeveloperId, switchAllowed } from '../lib/console-forms';
import { ACTION_MS, accName, findAny, type Locator, navClick, navigate, type NavResult, type Page, poll, realRole, safeClick, signInPage, SignInSeen, type Stop, vis } from './fill';
import { ACTION_MS, accName, findAny, type Locator, navClick, navigate, WAITS, type NavResult, type Page, poll, realRole, safeClick, signInPage, SignInSeen, type Stop, vis } from './fill';
// ---- the cache ----
@ -141,7 +141,7 @@ export async function readChooser(page: Page): Promise<PlayAccount[]> {
*/
export async function playAccounts(page: Page, playBase: string, ms: number, stop?: Stop): Promise<PlayAccount[]> {
await page.goto(`${playBase.replace(/\/$/, '')}/u/0/developers`, { waitUntil: 'domcontentloaded' }).catch(() => { /* read what is there */ });
const found = await poll(ms, async () => { const r = await readChooser(page); return r.length ? r : null; }, stop);
const found = await poll(ms, async () => { const r = await readChooser(page); return r.length ? r : null; }, stop, page);
if (found) return found;
const here = playDeveloperId(page.url());
return here ? [{ id: here, name: '' }] : [];
@ -185,7 +185,7 @@ async function accountId(page: Page, a: PlayAccount, listed: string[], ms: numbe
const role = await realRole(row).catch(() => '');
if (!switchAllowed(a.name, ['link', 'menuitem', 'menuitemradio', 'option', 'radio', 'row', 'button'].includes(role) ? role : 'row', listed)) return null;
await row.click({ timeout: ACTION_MS });
return poll(ms, async () => playDeveloperId(page.url()), stop);
return poll(ms, async () => playDeveloperId(page.url()), stop, page);
}
// ---- App Store Connect: teams ----
@ -197,7 +197,7 @@ const CANCEL: Name[] = ['Cancel', 'İptal', 'Vazgeç'];
interface TeamItem { name: string; loc: Locator; role: string; current: boolean }
async function teamMenu(page: Page, ms: number, stop?: Stop): Promise<TeamItem[] | null> {
const t = await poll(ms, async () => { const f = await findAny(page, ['button'], ACCOUNT_MENU); return f.state === 'ok' ? f.loc : null; }, stop);
const t = await poll(ms, async () => { const f = await findAny(page, ['button'], ACCOUNT_MENU); return f.state === 'ok' ? f.loc : null; }, stop, page);
if (!t || !(await navClick(t)).clicked) return null;
return poll(Math.min(ms, 3000), async () => {
const menus = vis(page.getByRole('menu'));
@ -213,7 +213,7 @@ async function teamMenu(page: Page, ms: number, stop?: Stop): Promise<TeamItem[]
}
}
return out.length ? out : null;
}, stop);
}, stop, page);
}
const closeMenu = async (page: Page) => { await page.keyboard.press('Escape').catch(() => { /* gone */ }); };
@ -235,7 +235,7 @@ export async function switchTeam(page: Page, name: string, listed: string[], ms:
if (!switchAllowed(real, role, listed)) { await closeMenu(page); return false; }
await it.loc.click({ timeout: ACTION_MS });
await page.waitForLoadState('domcontentloaded', { timeout: ms }).catch(() => { /* same page, or slow: read what is there */ });
await Bun.sleep(300);
await Bun.sleep(WAITS.settleMs); // the switched team's page settles
return true;
}

103
cli/console/fill.ts
View file

@ -33,7 +33,7 @@ import type { Ctx } from '../lib/ctx';
import { type Answer, type ConsoleResult, consoleResult, DISCLAIMER, FORM_IDS, type FormId, fillValue, formMeta, isForm, valueText } from '../lib/console-answers';
import type { ShipConfig } from '../lib/config';
import { APPLE_PRIVACY_TYPES } from '../lib/app-privacy';
import { ALL_APPS, APP_PRIVACY_HEADING, consoleReached, CREATE_FORM_FIELD, createAppAllowed, DATA_TYPES_HEADING, type FieldDef, FLOW_NAMES, flowAllowed, flowClass, type FlowWhere, type FormDef, formDef, FORM_OPENERS, forbiddenControl, quoteTolerant, forbiddenTick, hasToken, isAppleSignInFrame, isSignInText, isSignInUrl, type Name, navAllowed, type NavBases, type NavStep, navPlan, setUpAllowed, setUpTexts, typeLabels } from '../lib/console-forms';
import { ALL_APPS, APP_PRIVACY_HEADING, consoleReached, CREATE_FORM_FIELD, createAppAllowed, DATA_TYPES_HEADING, type FieldDef, FLOW_NAMES, flowAllowed, flowClass, type FlowWhere, type FormDef, formDef, FORM_OPENERS, forbiddenControl, quoteTolerant, forbiddenTick, hasToken, isAppleSignInFrame, isSignInText, isSignInUrl, type Name, navAllowed, type NavBases, type NavStep, navPlan, type SectionDef, setUpAllowed, setUpTexts, typeLabels } from '../lib/console-forms';
import { log, printJson, warn } from '../lib/log';
import { openAcross, withAccountIds } from './accounts';
import { appleAppId, withAppleAppId } from './app-id';
@ -269,6 +269,25 @@ export interface FillStep {
pending?: Locator[][];
}
/** The dialog of a section: by its names, else the one visible dialog with such a heading, else (the data type
* storebound just opened) the one visible dialog. */
async function sectionDialog(page: Page, sec: SectionDef, current?: string): Promise<Found> {
const names = sec.dialog ?? [];
let d = await find(page, 'dialog', names);
// a dialog titled otherwise (one dialog per step, the title elsewhere): the one visible dialog with such a heading
if (d.state === 'none') for (const n of names) {
const l = vis(page.getByRole('dialog').filter({ has: page.getByRole('heading', nameOpts(n)) }));
const c = await l.count();
if (c) { d = c === 1 ? { state: 'ok', loc: l } : { state: 'ambiguous' }; break; }
}
// the data type storebound just opened (App Privacy): its dialog may be titled otherwise; the one visible dialog
if (d.state === 'none' && sec.type && sec.type === current) {
const l = vis(page.getByRole('dialog'));
if (await l.count() === 1) d = { state: 'ok', loc: l };
}
return d;
}
/**
* Fills the first section of the form that is on the page and not done yet: locate every field first; any missing or
* ambiguous field -> fallback (nothing of that section is touched). Certain answers are set, the rest is highlighted
@ -276,22 +295,19 @@ export interface FillStep {
*/
export async function fillPage(page: Page, def: FormDef, answers: Answer[], done: Set<number> = new Set(), partial = false, current?: string): Promise<FillStep> {
const byId = new Map(answers.map(a => [a.id, a]));
// the sections of one data type share their dialog names (one array): the dialog is looked up once per call, not
// once per section (App Privacy: 3 sections x 6 types, each lookup several page queries)
const dialogs = new Map<Name[], Found>();
// no visible dialog at all (the page between two data types): every dialog section is skipped with one query
let anyDialog: boolean | undefined;
for (const [i, sec] of def.sections.entries()) {
if (done.has(i)) continue;
let root: Locator = page;
if (sec.dialog) {
let d = await find(page, 'dialog', sec.dialog);
// a dialog titled otherwise (one dialog per step, the title elsewhere): the one visible dialog with such a heading
if (d.state === 'none') for (const n of sec.dialog) {
const l = vis(page.getByRole('dialog').filter({ has: page.getByRole('heading', nameOpts(n)) }));
const c = await l.count();
if (c) { d = c === 1 ? { state: 'ok', loc: l } : { state: 'ambiguous' }; break; }
}
// the data type storebound just opened (App Privacy): its dialog may be titled otherwise; the one visible dialog
if (d.state === 'none' && sec.type && sec.type === current) {
const l = vis(page.getByRole('dialog'));
if (await l.count() === 1) d = { state: 'ok', loc: l };
}
anyDialog ??= await vis(page.getByRole('dialog')).count() > 0;
if (!anyDialog) continue;
let d = dialogs.get(sec.dialog);
if (!d) { d = await sectionDialog(page, sec, current); dialogs.set(sec.dialog, d); }
if (d.state === 'none') continue;
if (d.state !== 'ok') return { section: i, title: sec.title, filled: [], highlighted: [], missing: [`dialog ${sec.title}`], fallback: true };
root = d.loc;
@ -624,16 +640,53 @@ export class SignInSeen extends Error {}
export type Stop = () => Promise<boolean>;
/**
* Polls `fn` until it returns a value or the time is up (errors while the page navigates count as "not yet"). `stop`
* is checked first on every round: a sign-in page ends the wait at once (SignInSeen).
* The engine's internal waits that tests shorten (engineWaits); production keeps these defaults.
* pollMs the pause between two poll rounds
* quietMs 0 (production): a poll waits its whole time for something that is not there. Above 0 (tests, on local
* fixture pages): a poll also ends, as "not found", once the page has finished loading and its DOM has not
* changed for quietMs (a MutationObserver the poll puts into the page). A missing step then costs about
* quietMs instead of the whole step timeout, and a step that is there is still waited for as long as needed.
* settleMs the pause after a click that loads another page in place (switching the App Store Connect team)
* missMs fillSession: how long a field must stay missing (and at least 3 rounds) before it counts as a UI change
*/
export async function poll<T>(ms: number, fn: () => Promise<T | null>, stop?: Stop): Promise<T | null> {
export const WAITS = { pollMs: 200, quietMs: 0, settleMs: 300, missMs: 1500 };
/** Tests: sets engine waits (WAITS); returns the function that restores the previous ones. */
export function engineWaits(w: Partial<typeof WAITS>): () => void {
const before = { ...WAITS };
Object.assign(WAITS, w);
return () => { Object.assign(WAITS, before); };
}
/** How long the page has been loaded and unchanged (ms; 0 while it loads or is leaving for another page). Puts the
* observer in on the first call. */
async function quietFor(page: Page): Promise<number> {
return page.evaluate(() => {
const w = window as any;
if (!w.__storeboundQuiet) {
const q = { at: performance.now(), leaving: false };
w.__storeboundQuiet = q;
new MutationObserver(() => { q.at = performance.now(); }).observe(document, { subtree: true, childList: true, attributes: true, characterData: true });
// a click or script that starts a navigation: the old page stays "complete" until the new one commits
addEventListener('beforeunload', () => { q.leaving = true; });
}
const q = w.__storeboundQuiet;
return document.readyState === 'complete' && !q.leaving ? performance.now() - q.at : 0;
}).catch(() => 0);
}
/**
* Polls `fn` until it returns a value or the time is up (errors while the page navigates count as "not yet"). `stop`
* is checked first on every round: a sign-in page ends the wait at once (SignInSeen). With `page` and WAITS.quietMs
* (tests), a loaded page that stays unchanged for quietMs ends it early too (not found).
*/
export async function poll<T>(ms: number, fn: () => Promise<T | null>, stop?: Stop, page?: Page): Promise<T | null> {
const end = Date.now() + ms;
for (;;) {
if (stop && await stop().catch(() => false)) throw new SignInSeen();
try { const r = await fn(); if (r !== null) return r; } catch { /* navigating */ }
if (Date.now() >= end) return null;
await Bun.sleep(200);
if (page && WAITS.quietMs > 0 && await quietFor(page) >= WAITS.quietMs) return null;
await Bun.sleep(WAITS.pollMs);
}
}
@ -648,7 +701,7 @@ const opener = new RegExp(`^(?:${FORM_OPENERS.join('|')})$`);
export async function pickApp(page: Page, s: Extract<NavStep, { kind: 'pickApp' }>, ms: number, stop?: Stop): Promise<{ loc?: Locator; candidates?: string[] } | null> {
if (s.search && s.tokens[0]) {
const box = await poll(Math.min(ms, 3000), async () => { const f = await findAny(page, ['searchbox', 'textbox', 'combobox'], s.search!); return f.state === 'ok' ? f.loc : null; }, stop);
const box = await poll(Math.min(ms, 3000), async () => { const f = await findAny(page, ['searchbox', 'textbox', 'combobox'], s.search!); return f.state === 'ok' ? f.loc : null; }, stop, page);
if (box) { await box.fill(s.tokens[0]); }
}
return poll(ms, async () => {
@ -677,7 +730,7 @@ export async function pickApp(page: Page, s: Extract<NavStep, { kind: 'pickApp'
if (hits.length > 1) return { candidates: hits.map(h => h.name) };
}
return null;
}, stop);
}, stop, page);
}
async function formLink(page: Page, entry: Name[], ms: number, stop?: Stop): Promise<{ loc?: Locator; ambiguous?: boolean } | null> {
@ -704,7 +757,7 @@ async function formLink(page: Page, entry: Name[], ms: number, stop?: Stop): Pro
}
if (found.length === 1) return { loc: found[0]!.loc };
return found.length > 1 ? { ambiguous: true } : null;
}, stop);
}, stop, page);
}
/** The wait target is on the page (a heading of `level` when given). */
@ -750,7 +803,7 @@ export async function navigate(page: Page, plan: NavStep[], opts: NavOpts = {}):
try { await page.goto(s.url, { waitUntil: 'domcontentloaded' }); } catch (e) { return fail(s, `could not load ${s.url} (${(e as Error).message.split('\n')[0]})`); }
} else if (s.kind === 'click') {
if (s.optional && s.unless && (await findAny(page, s.unless.roles, s.unless.names)).state !== 'none') { say(' (not needed)'); continue; }
const f = await poll(s.optional ? Math.min(ms, 2000) : ms, async () => { const r = await findAny(page, s.roles, s.names); return r.state === 'none' ? null : r; }, stop);
const f = await poll(s.optional ? Math.min(ms, 2000) : ms, async () => { const r = await findAny(page, s.roles, s.names); return r.state === 'none' ? null : r; }, stop, page);
if (!f) { if (s.optional) { say(' (not on this page, skipped)'); continue; } return fail(s, 'not found'); }
if (f.state !== 'ok') return fail(s, 'more than one match');
const c = await navClick(f.loc, !!s.createApp);
@ -769,7 +822,7 @@ export async function navigate(page: Page, plan: NavStep[], opts: NavOpts = {}):
const c = await navClick(r.loc);
if (!c.clicked) return fail(s, `refused to click "${c.name}"`);
} else {
const f = await poll(ms, async () => ((await present(page, s)) ? true : null), stop);
const f = await poll(ms, async () => ((await present(page, s)) ? true : null), stop, page);
if (!f) return fail(s, 'did not appear');
}
} catch (e) {
@ -1277,8 +1330,8 @@ export async function fillSession(b: ConsoleBrowser, o: FillSessionOpts): Promis
let s: FillStep;
try { s = await fillPage(p, def, pageAnswers, done, false, currentType); } catch { await Bun.sleep(tick); continue; } // navigating
if (s.section !== null) {
// a page still loading can miss a field for a moment: only a miss that lasts (3 rounds, at least 1.5 s) is a UI change
if (s.fallback && (++misses < 3 || Date.now() - firstMiss < 1500)) { if (misses === 1) firstMiss = Date.now(); await Bun.sleep(tick); continue; }
// a page still loading can miss a field for a moment: only a miss that lasts (3 rounds, at least WAITS.missMs) is a UI change
if (s.fallback && (++misses < 3 || Date.now() - firstMiss < WAITS.missMs)) { if (misses === 1) firstMiss = Date.now(); await Bun.sleep(tick); continue; }
misses = 0;
// the miss lasts: fill every field that was found, leave the missing ones to the owner (only all-missing is a UI change)
if (s.fallback) { try { s = await fillPage(p, def, pageAnswers, done, true, currentType); } catch { await Bun.sleep(tick); continue; } }
@ -1297,7 +1350,7 @@ export async function fillSession(b: ConsoleBrowser, o: FillSessionOpts): Promis
if (form === 'play-app-record' && s.filled.includes('packageName')) {
const check = await controlWhere(p, FLOW_NAMES.checkAvailability, () => true, dataTypes).catch(() => null);
if (check && await flowClick(check, form, dataTypes, say).catch(() => false)) {
const ok = await poll(5000, async () => ((await vis(p.getByText(/package name (is )?available/i)).count()) > 0 ? true : null)).catch(() => null);
const ok = await poll(5000, async () => ((await vis(p.getByText(/package name (is )?available/i)).count()) > 0 ? true : null), undefined, p).catch(() => null);
say(ok ? 'App package name: "Package name available"' : 'App package name: Play Console did not say "Package name available": check the message next to the field');
}
}

26
cli/shots/browser.ts
View file

@ -175,6 +175,32 @@ export async function launch(root: string, opts: Record<string, unknown>, deps:
return withCandidates(kind, deps.installed ?? installedBrowsers(c), o => c.launch({ ...opts, ...o }));
}
/** How long a launched (throwaway profile) browser gets to close gracefully before it is killed. */
export const CLOSE_GRACE_MS = 3000;
/**
* Ends a browser from `launch` once the work is done: a graceful close capped at `ms`, then SIGKILL of its process.
* Google Chrome often hangs 10-30 s in its own shutdown after http pages (tests/helpers/browser.ts); a throwaway
* profile has nothing to flush, so waiting for it only costs time. The pid is read over CDP before the close. Never
* throws.
*/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export async function closeBrowser(browser: any, ms = CLOSE_GRACE_MS): Promise<void> {
let pid: number | undefined;
try {
const s = await browser.newBrowserCDPSession();
const info = await s.send('SystemInfo.getProcessInfo') as { processInfo: { type: string; id: number }[] };
pid = info.processInfo.find(p => p.type === 'browser')?.id;
} catch { /* not Chromium, or already gone */ }
let t: ReturnType<typeof setTimeout> | undefined;
const closed = await Promise.race([
Promise.resolve().then(() => browser.close()).then(() => true, () => true),
new Promise<boolean>(r => { t = setTimeout(() => r(false), ms); }),
]);
clearTimeout(t);
if (!closed && pid) try { process.kill(pid, 'SIGKILL'); } catch { /* exited meanwhile */ }
}
/**
* A persistent browser profile (cookies of a console sign-in the owner did by hand stay there between runs; storebound
* never reads them). Used by `storebound console fill`: always a real Chrome / Chromium (kind "visible", also when a

4
cli/shots/capture.ts
View file

@ -29,7 +29,7 @@ import type { ShipConfig, ShotsCapture } from '../lib/config';
import type { Ctx } from '../lib/ctx';
import { need } from '../lib/ctx';
import { done, log, step, warn } from '../lib/log';
import { launch } from './browser';
import { closeBrowser, launch } from './browser';
import { checkFrameFolder } from './check';
import { adbAppLocaleArgs, adbDemoExitArgs, adbDemoModeArgs, adbForceStopArgs, adbInstallArgs, adbScreencapArgs, adbStartArgs, adbStateArgs, adbTarget,
cropArgs, cropTopFor, fill, pickSimulator, shotsFor, simBootArgs, simBootStatusArgs, simInstallArgs, simLaunchArgs, simOpenUrlArgs,
@ -138,7 +138,7 @@ export async function run(ctx: Ctx): Promise<void> {
if (todo.length) {
const browser = await launch(root, { headless: true, args: ['--mute-audio', '--ignore-gpu-blocklist', '--enable-webgl', '--enable-gpu'] });
let report;
try { report = await captureWeb(browser, cap, todo, { concurrency, dims }); } finally { await browser.close(); }
try { report = await captureWeb(browser, cap, todo, { concurrency, dims }); } finally { await closeBrowser(browser); } // capped close, then kill: Chrome can hang 10-30 s in its shutdown
report.skipped = skipped.map(j => j.frame);
writeCaptureState(root, report, cache, todo);
for (const l of summaryLines(report)) log('shots', l);

4
cli/shots/render.ts
View file

@ -21,7 +21,7 @@ import { join, relative } from 'node:path';
import type { Ctx } from '../lib/ctx';
import { need } from '../lib/ctx';
import { done, log, warn } from '../lib/log';
import { launch } from './browser';
import { closeBrowser, launch } from './browser';
import { defaultConcurrency, pool, progressLine } from './capture-web';
import { shotsStyle } from '../lib/config';
import { type CopyFile, SIZES, copyText, defaultSizes, frameDims, mergeCopy, palette, renderDir, scriptFont, sheetLayout, validateCopy, variantScenes } from './lib';
@ -239,7 +239,7 @@ export async function run(ctx: Ctx): Promise<void> {
});
} finally {
for (const p of pages) await p?.page.context().close().catch(() => {});
await browser.close();
await closeBrowser(browser);
server.stop(true);
if (Object.keys(report).length) await Bun.write(join(outDir, 'report.json'), JSON.stringify(mergeReport(readReport(outDir), report), null, 1) + '\n');
}

4
docs/desktop.md
View file

@ -379,7 +379,9 @@ Download sources (one click on the Dependencies screen or in the first-run guide
fixtures (`apps/desktop/tests/fixtures/state`: empty, D1 with folders gone, D3 full; the host answers are written by
the Rust migration itself, `src-tauri/tests/state.rs`) and the real sidecar for reads. It opens every screen and
app and fails on any uncaught or console error, an error panel or an empty `#root`; a last case removes the bundle
and expects the boot.js panel.
and expects the boot.js panel. One browser (killed at the end), a page per case, the cases in parallel against the
one build, and no fixed sleeps: each step waits for its element, its text or a quiet window (no host call in
flight, no DOM change).
## Develop

38
tests/capture-web.test.ts
View file

@ -53,14 +53,17 @@ describe('capture helpers', () => {
});
});
// ---- a local fixture web app: /scene/<name>?lang=xx, a SPA (popstate) that counts its boots ----
let boots = 0;
// ---- a local fixture web app: /<key>/scene/<name>?lang=xx, a SPA (popstate) that counts its boots per key: each
// browser test has its own key, so they run at the same time (test.concurrent) ----
const boots = new Map<string, number>();
const booted = (key: string) => boots.get(key) ?? 0;
const server = Bun.serve({
port: 0, hostname: '127.0.0.1',
fetch(req) {
const u = new URL(req.url);
if (u.pathname === '/app.js') {
boots++;
const key = u.searchParams.get('k') ?? '';
boots.set(key, booted(key) + 1);
return new Response(`
const COLORS = { en: '#1e66f5', de: '#40a02b', tr: '#d20f39' };
function show() {
@ -74,7 +77,8 @@ const server = Bun.serve({
addEventListener('popstate', show);
show();`, { headers: { 'content-type': 'text/javascript' } });
}
return new Response('<!doctype html><html><head><meta charset="utf-8"><script src="/app.js" defer></script></head><body></body></html>', { headers: { 'content-type': 'text/html' } });
const key = encodeURIComponent(u.pathname.split('/')[1] ?? '');
return new Response(`<!doctype html><html><head><meta charset="utf-8"><script src="/app.js?k=${key}" defer></script></head><body></body></html>`, { headers: { 'content-type': 'text/html' } });
},
});
afterAll(() => server.stop(true));
@ -99,15 +103,16 @@ function project(cfg: ShipConfig, files: Record<string, string>): { root: string
}
const shot = (name: string, extra: object = {}) => ({ path: `/scene/${name}`, waitFor: `window.scene === '${name}'`, delayMs: 20, ...extra });
function config(capture: Partial<ShotsCapture> = {}): ShipConfig {
/** `key`: the fixture app's boot counter this config's runs count on (one per browser test). */
function config(capture: Partial<ShotsCapture> = {}, key = 'app'): ShipConfig {
return { ...BASE, shots: { copy: 'store/shots/copy.json', frames: 'store/shots/frames', out: 'store/renders', scenes: [{ id: 'hero', palette: 'fire', frames: [{ src: 'hero.png' }] }], phone: { w: 180, h: 240, dpr: 1 },
capture: { source: 'web', baseUrl: `http://127.0.0.1:${server.port}`, query: 'lang={lang}', build: 'src', shots: { hero: shot('hero'), battle: shot('battle'), king: shot('king') }, ...capture } } };
capture: { source: 'web', baseUrl: `http://127.0.0.1:${server.port}/${key}`, query: 'lang={lang}', build: 'src', shots: { hero: shot('hero'), battle: shot('battle'), king: shot('king') }, ...capture } } };
}
const files = { 'store/shots/copy.json': '{}', 'src/main.js': 'v1' };
const frames = (root: string) => ['en', 'de', 'tr'].flatMap(l => ['hero', 'battle', 'king'].map(n => join(root, 'store/shots/frames', l, 'phone', `${n}.png`)));
describe.skipIf(!canLaunch)('shots capture (web) with a fixture app', () => {
test('concurrency 3 gives the same files as serial', async () => {
test.concurrent('concurrency 3 gives the same files as serial', async () => {
const p = project(config(), files);
try {
await p.cli(['--device', 'phone', '--concurrency', '1', '--force']);
@ -120,34 +125,31 @@ describe.skipIf(!canLaunch)('shots capture (web) with a fixture app', () => {
} finally { rmSync(p.root, { recursive: true, force: true }); }
}, 120_000);
test('per-language session boots the app once per language; per-shot once per frame', async () => {
const p = project(config({ session: 'per-language' }), files);
test.concurrent('per-language session boots the app once per language; per-shot once per frame', async () => {
const p = project(config({ session: 'per-language' }, 'lang'), files);
try {
boots = 0;
const r = await p.cli(['--device', 'phone', '--concurrency', '2', '--force']);
expect(boots).toBe(3);
expect(booted('lang')).toBe(3);
expect(r.out).toContain('(reused page)');
const report = JSON.parse(readFileSync(join(p.root, '.storebound/capture-report.json'), 'utf8')) as CaptureReport;
expect(report.frames.filter(f => f.reused).length).toBe(6);
expect(frames(p.root).every(f => existsSync(f))).toBe(true);
// a shot whose precondition fails in the reused page (it needs a freshly booted page) falls back to a fresh page
const bad = project(config({ session: 'per-language', reuseTimeoutMs: 300, shots: { hero: shot('hero'), fresh: { path: '/scene/fresh', waitFor: "window.scene === 'fresh' && window.shows === 1", delayMs: 20 } } }), files);
const bad = project(config({ session: 'per-language', reuseTimeoutMs: 300, shots: { hero: shot('hero'), fresh: { path: '/scene/fresh', waitFor: "window.scene === 'fresh' && window.shows === 1", delayMs: 20 } } }, 'fresh'), files);
try {
boots = 0;
const r2 = await bad.cli(['--device', 'phone', '--lang', 'en', '--force']);
expect(r2.err).toContain('en/phone/fresh: reused page failed');
expect(r2.out).toContain('capture done: 2 frames');
expect(boots).toBe(2);
expect(booted('fresh')).toBe(2);
} finally { rmSync(bad.root, { recursive: true, force: true }); }
boots = 0;
const q = project(config(), files);
const q = project(config({}, 'shot'), files);
await q.cli(['--device', 'phone', '--lang', 'en', '--force']);
rmSync(q.root, { recursive: true, force: true });
expect(boots).toBe(3);
expect(booted('shot')).toBe(3);
} finally { rmSync(p.root, { recursive: true, force: true }); }
}, 180_000);
test('cache skips unchanged frames, re-runs changed ones; --only-missing; timing breakdown', async () => {
test.concurrent('cache skips unchanged frames, re-runs changed ones; --only-missing; timing breakdown', async () => {
const p = project(config(), files);
try {
let r = await p.cli(['--device', 'phone']);

7
tests/ci-plan.test.ts
View file

@ -1,5 +1,5 @@
import { describe, expect, test } from 'bun:test';
import { balance, depsOf, GLOBAL, HEAVY, junitTimes, listTests, relatedTests, rustNeeded, shardCount } from '../tools/ci-plan';
import { balance, depsOf, GLOBAL, HEAVY, junitTimes, listTests, mergeTimings, relatedTests, rustNeeded, shardCount } from '../tools/ci-plan';
describe('tools/ci-plan: test shards', () => {
test('shard count: cores - 1, between 1 and 8, env override', () => {
@ -27,6 +27,11 @@ describe('tools/ci-plan: test shards', () => {
<testsuite name="tests/b.test.ts" file="tests/b.test.ts" time="0.25"></testsuite></testsuites>`;
expect(junitTimes(xml)).toEqual({ 'tests/a.test.ts': 1.5, 'tests/b.test.ts': 0.25 });
});
test('recorded timings: a new file takes its time; a known one moves halfway, never up past the new time', () => {
const t = mergeTimings({ 'a.test.ts': 10, 'b.test.ts': 40, 'keep.test.ts': 3 }, { 'a.test.ts': 30, 'b.test.ts': 20, 'new.test.ts': 5 });
expect(t).toEqual({ 'a.test.ts': 20, 'b.test.ts': 20, 'keep.test.ts': 3, 'new.test.ts': 5 });
});
test('every heavy suite exists and every test file is found', () => {
const all = listTests();
for (const h of HEAVY) expect(all).toContain(h);

87
tests/console-fill.test.ts
View file

@ -3,21 +3,22 @@
// the public help pages, not recorded) served on 127.0.0.1. No network. Browser tests skip when no Chromium is
// installed (a launch that times out fails instead). Load-proofing (tests/helpers/browser.ts): one launch with a long
// launch timeout, every context closed right after its test, event-driven waits for log lines, explicit test timeouts.
// Slow on purpose (marked SLOW): the sign-in / watching tests and the account tests wait out real "not here" timeouts
// of the engine (navTimeoutMs / navMs, scaled down through the options). The relaunch of storebound's own profile
// Speed: the browser tests run BROWSER_CONCURRENCY at a time (scope.concurrent, each with its own contexts) and the
// engine's waits are the test ones (TEST_WAITS / TEST_TICK_MS: 20 ms rounds, a missing step ends once its page is
// loaded and quiet). SLOW tests still wait out some "not here" timeouts of the engine (navMs, scaled down). The relaunch of storebound's own profile
// (session cookies kept) needs a real Chrome: tests/console-persist.test.ts.
import { afterAll, afterEach, beforeAll, describe, expect, test } from 'bun:test';
import { mkdtempSync, readdirSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { matchAccount, openAcross, readAccounts, rememberApple, rememberPlay, withAccountIds } from '../cli/console/accounts';
import { createAppHere, fillPage, fillSession, navClick, navigate, resumeAt, run, safeClick, signInPage, signInReason } from '../cli/console/fill';
import { createAppHere, engineWaits, fillPage, fillSession, navClick, navigate, resumeAt, run, safeClick, signInPage, signInReason } from '../cli/console/fill';
import type { ConsoleBrowser } from '../cli/console/session';
import { parseArgs } from '../cli/lib/args';
import { type ShipConfig, validateExtras } from '../cli/lib/config';
import { AGE_RATING_API, consoleAnswers, consoleResult, fillValue, FORM_IDS, type FormId, formMeta } from '../cli/lib/console-answers';
import { consoleReached, CREATE_APP_OPENERS, createAppAllowed, formDef, FORM_OPENERS, forbiddenControl, hasToken, isSignInText, isSignInUrl, navAllowed, navPlan, playDeveloperId, switchAllowed } from '../cli/lib/console-forms';
import { AFTER_ALL_MS, AFTER_EACH_MS, launchTestBrowser, logLines, testScope } from './helpers/browser';
import { AFTER_ALL_MS, AFTER_EACH_MS, launchTestBrowser, logLines, TEST_TICK_MS, TEST_WAITS, testScope } from './helpers/browser';
const config: ShipConfig = {
app: { name: 'Example Game', profile: 'game', privacyPolicyUrl: 'https://example.com/privacy', supportUrl: 'https://example.com/support', website: 'https://example.com', contactEmail: 'support@example.com' },
@ -208,7 +209,9 @@ beforeAll(() => {
});
// bounded: every context is closed per test (afterEach) and each close is capped (tests/helpers/browser.ts)
afterEach(() => scope.settle(), AFTER_EACH_MS);
afterAll(async () => { await shared.close(); server?.stop(true); }, AFTER_ALL_MS);
let restoreWaits = () => {};
beforeAll(() => { restoreWaits = engineWaits(TEST_WAITS); });
afterAll(async () => { restoreWaits(); await shared.close(); server?.stop(true); }, AFTER_ALL_MS);
async function open(file: string) {
const page = await scope.newPage();
@ -221,7 +224,7 @@ const flagged = (page: any) => page.evaluate(() => document.querySelectorAll('[d
const clicks = (page: any) => page.evaluate(() => (window as any).__clicks);
describe.skipIf(!browser)('engine (headless, modeled pages)', () => {
test('app record: certain fields filled, uncertain empty + highlighted, Create never clicked', async () => {
scope.concurrent('app record: certain fields filled, uncertain empty + highlighted, Create never clicked', async () => {
const page = await open('apple-app-record.html');
const def = formDef('apple-app-record', pageAnswers('apple-app-record'));
const s = await fillPage(page, def, pageAnswers('apple-app-record'));
@ -242,7 +245,7 @@ describe.skipIf(!browser)('engine (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('User Access under a plain heading (real App Store Connect): the radios are found in the dialog', async () => {
scope.concurrent('User Access under a plain heading (real App Store Connect): the radios are found in the dialog', async () => {
const page = await open('apple-app-record-heading.html');
const s = await fillPage(page, formDef('apple-app-record', pageAnswers('apple-app-record')), pageAnswers('apple-app-record'));
expect(s.fallback).toBe(false);
@ -252,7 +255,7 @@ describe.skipIf(!browser)('engine (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('a lasting partial miss fills every field that was found and leaves only the missing one', async () => {
scope.concurrent('a lasting partial miss fills every field that was found and leaves only the missing one', async () => {
const page = await open('apple-app-record-renamed.html');
const def = formDef('apple-app-record', pageAnswers('apple-app-record'));
const s = await fillPage(page, def, pageAnswers('apple-app-record'), new Set(), true);
@ -265,7 +268,7 @@ describe.skipIf(!browser)('engine (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('a renamed label stops filling and falls back to guidance', async () => {
scope.concurrent('a renamed label stops filling and falls back to guidance', async () => {
const page = await open('apple-app-record-renamed.html');
const s = await fillPage(page, formDef('apple-app-record', pageAnswers('apple-app-record')), pageAnswers('apple-app-record'));
expect(s.fallback).toBe(true);
@ -277,7 +280,7 @@ describe.skipIf(!browser)('engine (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('app privacy (fillPage alone): section by section as the page changes; fillPage itself never clicks Next / Save / Publish', async () => {
scope.concurrent('app privacy (fillPage alone): section by section as the page changes; fillPage itself never clicks Next / Save / Publish', async () => {
const page = await open('apple-app-privacy.html');
const ans = pageAnswers('apple-app-privacy');
const def = formDef('apple-app-privacy', ans);
@ -311,7 +314,7 @@ describe.skipIf(!browser)('engine (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('content rating: full email in the page, Game, purchases yes, the rest empty + highlighted', async () => {
scope.concurrent('content rating: full email in the page, Game, purchases yes, the rest empty + highlighted', async () => {
const page = await open('play-content-rating.html');
const ans = pageAnswers('play-content-rating');
const def = formDef('play-content-rating', ans);
@ -332,7 +335,7 @@ describe.skipIf(!browser)('engine (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('target audience: ages, appeal highlighted, ads yes', async () => {
scope.concurrent('target audience: ages, appeal highlighted, ads yes', async () => {
const page = await open('play-target-audience.html');
const ans = pageAnswers('play-target-audience');
const def = formDef('play-target-audience', ans);
@ -349,7 +352,7 @@ describe.skipIf(!browser)('engine (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('sign-in detector and safeClick', async () => {
scope.concurrent('sign-in detector and safeClick', async () => {
const signin = await open('signin.html');
expect(await signInPage(signin)).toBe(true);
await signin.close();
@ -419,7 +422,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => {
const noneDenied = (names: string[]) => { for (const n of names) expect(navAllowed(n, 'link')).toBe(true); };
const fresh = async () => { const p = await scope.newPage(); await p.goto(base + 'blank.html'); await p.evaluate(() => sessionStorage.clear()); return p; };
test('App Store Connect (EN): + > New App opens the dialog and it is filled; Create and Submit for Review never clicked', async () => {
scope.concurrent('App Store Connect (EN): + > New App opens the dialog and it is filled; Create and Submit for Review never clicked', async () => {
const page = await fresh();
const r = await navigate(page, navPlan('apple-app-record', config, bases()), quiet);
expect(r).toMatchObject({ ok: true, steps: ['open Apps', 'the add button (+)', 'New App in the menu', 'the New App dialog'] });
@ -431,7 +434,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('App Store Connect (TR): Uygulama Ekle > Yeni Uygulama; Oluştur and İncelemeye Gönder never clicked', async () => {
scope.concurrent('App Store Connect (TR): Uygulama Ekle > Yeni Uygulama; Oluştur and İncelemeye Gönder never clicked', async () => {
const page = await fresh();
const r = await navigate(page, navPlan('apple-app-record', config, bases('asc-tr')), quiet);
expect(r.ok).toBe(true);
@ -440,7 +443,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('a missing step stops navigation, names the step and clicks nothing', async () => {
scope.concurrent('a missing step stops navigation, names the step and clicks nothing', async () => {
const page = await fresh();
const said: string[] = [];
const r = await navigate(page, navPlan('apple-app-record', config, bases('asc-noplus')), { timeoutMs: 800, say: m => said.push(m) });
@ -451,7 +454,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('App Privacy: appId deep link, or app list > the app > App Privacy; two apps of one name -> stop and list', async () => {
scope.concurrent('App Privacy: appId deep link, or app list > the app > App Privacy; two apps of one name -> stop and list', async () => {
let page = await fresh();
let r = await navigate(page, navPlan('apple-app-privacy', { ...config, apple: { ...config.apple!, appId: '123' } } as ShipConfig, bases()), quiet);
expect(r).toMatchObject({ ok: true, steps: ['open App Privacy of the app (apple.appId)', 'the App Privacy page'] });
@ -469,7 +472,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('App Privacy: an app card that names the app twice (icon aria-label + title link, same /apps/<id>) is one app', async () => {
scope.concurrent('App Privacy: an app card that names the app twice (icon aria-label + title link, same /apps/<id>) is one app', async () => {
const page = await fresh();
const r = await navigate(page, navPlan('apple-app-privacy', config, bases('asc-card')), quiet);
expect(r).toMatchObject({ ok: true });
@ -479,7 +482,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('Play: the app by package name > Policy and programs > App content > Start / Manage; nothing deny-listed clicked', async () => {
scope.concurrent('Play: the app by package name > Policy and programs > App content > Start / Manage; nothing deny-listed clicked', async () => {
for (const [form, link, heading] of [['play-content-rating', 'Start', 'Content rating'], ['play-target-audience', 'Manage', 'Target audience and content']] as const) {
const page = await fresh();
const r = await navigate(page, navPlan(form, config, bases()), quiet);
@ -492,7 +495,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => {
}
}, BROWSER_TEST_MS);
test('navClick refuses a Start button, Start rollout, Save and Publish even when pointed at them', async () => {
scope.concurrent('navClick refuses a Start button, Start rollout, Save and Publish even when pointed at them', async () => {
const page = await open('play-app-content.html');
expect((await navClick(page.getByRole('button', { name: 'Start' }))).clicked).toBe(false);
expect((await navClick(page.getByRole('button', { name: 'Save' }))).clicked).toBe(false);
@ -504,7 +507,7 @@ describe.skipIf(!browser)('navigation (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('already on the form: nothing is clicked', async () => {
scope.concurrent('already on the form: nothing is clicked', async () => {
const page = await open('play-content-rating.html');
const r = await navigate(page, navPlan('play-content-rating', config, bases()), quiet);
expect(r).toMatchObject({ ok: true, steps: ['already open'] });
@ -535,14 +538,14 @@ afterAll(() => { for (const d of tmpDirs) rmSync(d, { recursive: true, force: tr
const tmpAccounts = () => { const d = mkdtempSync(join(tmpdir(), 'storebound-accounts-')); tmpDirs.push(d); return join(d, 'console-accounts.json'); };
/**
* The engine's timings scaled down for tests (production: tick 1 s, 20 s per navigation step, 10 min for a sign-in).
* navTimeoutMs is both how long a step that IS there may take to show up (headroom for a loaded machine) and what a
* step that is NOT there costs (the watching tests wait that out once), so it stays a few seconds, not FOUND_MS.
* navTimeoutMs is how long a step that IS there may take to show up (headroom for a loaded machine); a step that is
* NOT there ends earlier, once its loaded page stays unchanged for TEST_WAITS.quietMs (tests/helpers/browser.ts).
*/
const fast = (log: ReturnType<typeof logLines>) => ({ say: log.say, tickMs: 100, navTimeoutMs: 5000, signInWaitMs: 30_000, accountsFile: tmpAccounts(), env: {} });
const fast = (log: ReturnType<typeof logLines>) => ({ say: log.say, tickMs: TEST_TICK_MS, navTimeoutMs: 5000, signInWaitMs: 30_000, accountsFile: tmpAccounts(), env: {} });
const FILLED = (lines: string[]) => lines.some(m => /^New App: filled \d+/.test(m));
describe.skipIf(!browser)('sign-in and watching (headless, modeled pages on the real console hosts, routed locally)', () => {
test('ASC /login with an idmsa frame and no password field: detected mid-navigation, never touched; after sign-in navigation restarts at step 1', async () => {
scope.concurrent('ASC /login with an idmsa frame and no password field: detected mid-navigation, never touched; after sign-in navigation restarts at step 1', async () => {
const b = await consoleBrowser((u, cookie) => u.hostname === 'idmsa.apple.com' ? 'idmsa-signin.html'
: u.hostname !== 'appstoreconnect.apple.com' ? null
: u.pathname === '/login' ? 'asc-login.html' : u.pathname === '/apps' ? (cookie.includes('myacinfo=') ? 'asc-apps.html' : 'asc-shell.html') : null);
@ -565,7 +568,7 @@ describe.skipIf(!browser)('sign-in and watching (headless, modeled pages on the
expect((await run).filled).toBeGreaterThan(0);
}, SLOW_TEST_MS);
test('after the printed steps storebound keeps watching: the owner opens Apps by hand, navigation resumes at step 2 and fills', async () => {
scope.concurrent('after the printed steps storebound keeps watching: the owner opens Apps by hand, navigation resumes at step 2 and fills', async () => {
const b = await consoleBrowser(u => u.hostname === 'appstoreconnect.apple.com' && u.pathname === '/apps' ? (u.search.includes('owner') ? 'asc-apps.html' : 'asc-apps-noplus.html') : null);
const log = logLines(), said = log.lines;
const run = scope.run(fillSession(b, { form: 'apple-app-record', config, ...fast(log) }));
@ -580,7 +583,7 @@ describe.skipIf(!browser)('sign-in and watching (headless, modeled pages on the
await run;
}, SLOW_TEST_MS);
test('after the printed steps: the owner opens the dialog by hand, storebound sees it and fills it', async () => {
scope.concurrent('after the printed steps: the owner opens the dialog by hand, storebound sees it and fills it', async () => {
const b = await consoleBrowser(u => u.hostname === 'appstoreconnect.apple.com' && u.pathname === '/apps' ? (u.search.includes('dialog') ? 'apple-app-record.html' : 'asc-apps-noplus.html') : null);
const log = logLines();
const run = scope.run(fillSession(b, { form: 'apple-app-record', config, ...fast(log) }));
@ -592,7 +595,7 @@ describe.skipIf(!browser)('sign-in and watching (headless, modeled pages on the
await run;
}, SLOW_TEST_MS);
test('resumeAt: the furthest step visible now, never the initial page load', async () => {
scope.concurrent('resumeAt: the furthest step visible now, never the initial page load', async () => {
const page = await open('asc-apps.html');
const plan = navPlan('apple-app-record', config, bases());
expect(await resumeAt(page, plan)).toBe(1);
@ -657,7 +660,7 @@ describe.skipIf(!browser)('accounts (headless, modeled pages)', () => {
const fresh = async () => { const p = await scope.newPage(); await p.goto(base + 'blank.html'); await p.evaluate(() => sessionStorage.clear()); return p; };
const DENIED = ['Sign Out', 'Edit Profile', 'Create', 'Submit for Review', 'Create app', 'Create developer account', 'Save', 'Publish'];
test('App Store Connect, 3 teams, the bundle id in the 2nd: switched through the account menu, remembered, then picked first', async () => {
scope.concurrent('App Store Connect, 3 teams, the bundle id in the 2nd: switched through the account menu, remembered, then picked first', async () => {
const file = tmpAccounts(), said: string[] = [];
let page = await fresh();
await page.goto(base + 'teams/apps');
@ -684,7 +687,7 @@ describe.skipIf(!browser)('accounts (headless, modeled pages)', () => {
await page.close();
}, SLOW_TEST_MS);
test('Play, one login in 5 developer accounts, the app in the 4th: found by package name, remembered, then opened directly', async () => {
scope.concurrent('Play, one login in 5 developer accounts, the app in the 4th: found by package name, remembered, then opened directly', async () => {
const file = tmpAccounts(), said: string[] = [];
const pb = { asc: base + 'asc', play: base + 'playm' };
let page = await fresh();
@ -711,7 +714,7 @@ describe.skipIf(!browser)('accounts (headless, modeled pages)', () => {
await page.close();
}, SLOW_TEST_MS);
test('Play: a package in none of the developer accounts is said plainly; nothing deny-listed clicked', async () => {
scope.concurrent('Play: a package in none of the developer accounts is said plainly; nothing deny-listed clicked', async () => {
const page = await fresh();
const absent = { ...config, play: { packageName: 'com.example.absent' } } as ShipConfig;
const r = await openAcross(page, { form: 'play-content-rating', config: absent, say: () => {}, bases: { asc: base + 'asc', play: base + 'playm' }, navMs: 800, file: tmpAccounts(), env: {} });
@ -828,7 +831,7 @@ describe.skipIf(!browser)('play-app-record (headless, modeled pages)', () => {
}, BROWSER_TEST_MS);
}
test('an agreement answer that says yes is still never ticked', async () => {
scope.concurrent('an agreement answer that says yes is still never ticked', async () => {
const page = await open('play-create-app.html');
const yes = ans().map(a => (a.agreement ? { ...a, value: true, certain: true } : a));
const s = await fillPage(page, def(), yes);
@ -838,7 +841,7 @@ describe.skipIf(!browser)('play-app-record (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('a renamed field: partial fill, the missing one left to the owner; nothing ticked or submitted', async () => {
scope.concurrent('a renamed field: partial fill, the missing one left to the owner; nothing ticked or submitted', async () => {
const page = await open('play-create-app-renamed.html');
expect((await fillPage(page, def(), ans())).fallback).toBe(true); // first rounds: wait (the page may be loading)
const s = await fillPage(page, def(), ans(), new Set(), true);
@ -851,7 +854,7 @@ describe.skipIf(!browser)('play-app-record (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('the opener inside a dialog, or on a page that is not the app list, is refused', async () => {
scope.concurrent('the opener inside a dialog, or on a page that is not the app list, is refused', async () => {
const page = await open('blank.html');
await page.setContent('<h1>All apps</h1><div role="dialog" aria-label="x"><button type="button">Create app</button></div>');
expect((await navClick(page.getByRole('button', { name: 'Create app' }), true)).clicked).toBe(false);
@ -862,7 +865,7 @@ describe.skipIf(!browser)('play-app-record (headless, modeled pages)', () => {
await page.close();
}, BROWSER_TEST_MS);
test('accounts: several and none picked -> stop and list; by play.developerAccount -> that one, remembered; by play.developerId -> directly', async () => {
scope.concurrent('accounts: several and none picked -> stop and list; by play.developerAccount -> that one, remembered; by play.developerId -> directly', async () => {
const go = async (c: ShipConfig, file: string, said: string[], env: Record<string, string> = {}) => {
const page = await fresh();
const r = await openAcross(page, { form: 'play-app-record', config: c, say: m => { said.push(m); }, bases: pb('playn'), navMs: FOUND_MS, file, env });
@ -904,7 +907,7 @@ describe.skipIf(!browser)('play-app-record (headless, modeled pages)', () => {
await page.close();
}, SLOW_TEST_MS);
test('the chooser as it really looks (nested spans, icon fonts, role=button rows with data ids, an avatar row without an id): exact, case pair, prefix, not found', async () => {
scope.concurrent('the chooser as it really looks (nested spans, icon fonts, role=button rows with data ids, an avatar row without an id): exact, case pair, prefix, not found', async () => {
const go = async (account: string | undefined, said: string[], play = 'playr') => {
const page = await fresh();
const c = { ...config, play: { ...config.play!, ...(account ? { developerAccount: account } : {}) } } as ShipConfig;
@ -955,7 +958,7 @@ describe.skipIf(!browser)('play-app-record (headless, modeled pages)', () => {
await page.close();
}, SLOW_TEST_MS);
test('fillSession: notes logged, form filled, the run ends without any submit', async () => {
scope.concurrent('fillSession: notes logged, form filled, the run ends without any submit', async () => {
const b = await consoleBrowser(u => (u.hostname !== 'play.google.com' ? null : u.pathname === '/console' || u.pathname.endsWith('/app-list') ? 'play-home.html' : u.pathname === '/play-create-app.html' || u.pathname.endsWith('/create-new-app') ? 'play-create-app.html' : null));
const log = logLines(), said = log.lines;
const c = { ...config, play: { ...config.play!, developerId: '4444' } } as ShipConfig;
@ -1007,7 +1010,7 @@ async function until(pred: () => boolean): Promise<void> {
}
describe.skipIf(!browser)('sign-in detection: only the top-level URL, the main frame\'s form or an Apple sign-in frame', () => {
test('the Play account chooser with the account widget (accounts.google.com frames, avatar and email header) is NOT a sign-in page', async () => {
scope.concurrent('the Play account chooser with the account widget (accounts.google.com frames, avatar and email header) is NOT a sign-in page', async () => {
const b = await consoleBrowser(playOgb);
await b.page.goto('https://play.google.com/console/u/0/developers');
await until(() => frameOn(b.page, 'accounts.google.com') && frameOn(b.page, 'ogs.google.com'));
@ -1016,7 +1019,7 @@ describe.skipIf(!browser)('sign-in detection: only the top-level URL, the main f
await b.context.close();
}, BROWSER_TEST_MS);
test('Play home with the widget frame is NOT a sign-in page', async () => {
scope.concurrent('Play home with the widget frame is NOT a sign-in page', async () => {
const b = await consoleBrowser(playOgb);
await b.page.goto('https://play.google.com/console/u/0/developers/7000000001/app-list');
await until(() => frameOn(b.page, 'accounts.google.com') && frameOn(b.page, 'ogs.google.com'));
@ -1024,7 +1027,7 @@ describe.skipIf(!browser)('sign-in detection: only the top-level URL, the main f
await b.context.close();
}, BROWSER_TEST_MS);
test('a top-level accounts.google.com page IS a sign-in page, and the rule says why (URL without its query)', async () => {
scope.concurrent('a top-level accounts.google.com page IS a sign-in page, and the rule says why (URL without its query)', async () => {
const b = await consoleBrowser(playOgb);
await b.page.goto('https://accounts.google.com/v3/signin/identifier?continue=https%3A%2F%2Fplay.google.com%2Fconsole');
expect(await signInReason(b.page)).toEqual({ rule: 'the top-level URL is a sign-in page', url: 'https://accounts.google.com/v3/signin/identifier' });
@ -1035,7 +1038,7 @@ describe.skipIf(!browser)('sign-in detection: only the top-level URL, the main f
await page.close();
}, BROWSER_TEST_MS);
test('fillSession play-app-record: the chooser with the widget frames is passed, the form filled, no sign-in said', async () => {
scope.concurrent('fillSession play-app-record: the chooser with the widget frames is passed, the form filled, no sign-in said', async () => {
const b = await consoleBrowser(playOgb);
const log = logLines(), said = log.lines;
const done = scope.run(fillSession(b, { form: 'play-app-record', config, ...fast(log) }));
@ -1047,7 +1050,7 @@ describe.skipIf(!browser)('sign-in detection: only the top-level URL, the main f
await done;
}, SLOW_TEST_MS);
test('a copied Chrome session that lands on accounts.google.com: said plainly with the rule; --debug-dump dumps the sign-in wait (at most 3), the watchdog once', async () => {
scope.concurrent('a copied Chrome session that lands on accounts.google.com: said plainly with the rule; --debug-dump dumps the sign-in wait (at most 3), the watchdog once', async () => {
const dir = mkdtempSync(join(tmpdir(), 'sb-signin-debug-'));
tmpDirs.push(dir);
const b: ConsoleBrowser = { ...await consoleBrowser(u => (u.hostname === 'play.google.com' ? 'play-signed-out.html' : playOgb(u))), mode: 'copy' };

42
tests/console-flow.test.ts
View file

@ -7,13 +7,13 @@ import { afterAll, afterEach, beforeAll, describe, expect, test } from 'bun:test
import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { fillSession } from '../cli/console/fill';
import { engineWaits, fillSession } from '../cli/console/fill';
import type { ConsoleBrowser } from '../cli/console/session';
import { APPLE_DATA_TYPE_IDS, type AppleDataTypeId, type AppleTypeAnswer } from '../cli/lib/app-privacy';
import type { ShipConfig } from '../cli/lib/config';
import { type FlowWhere, flowAllowed, flowClass, FORM_BUTTONS, labelNames, setUpAllowed, setUpTexts, TYPE_ALIASES } from '../cli/lib/console-forms';
import { APPLE_PRIVACY_TYPES } from '../cli/lib/app-privacy';
import { AFTER_ALL_MS, AFTER_EACH_MS, launchTestBrowser, logLines, testScope } from './helpers/browser';
import { AFTER_ALL_MS, AFTER_EACH_MS, launchTestBrowser, logLines, TEST_TICK_MS, TEST_WAITS, testScope } from './helpers/browser';
const FIX = join(import.meta.dir, 'fixtures', 'console');
const base0: ShipConfig = {
@ -52,7 +52,9 @@ beforeAll(() => {
base = `http://127.0.0.1:${server.port}/`;
});
afterEach(() => scope.settle(), AFTER_EACH_MS);
afterAll(async () => { await shared.close(); server?.stop(true); }, AFTER_ALL_MS);
let restoreWaits = () => {};
beforeAll(() => { restoreWaits = engineWaits(TEST_WAITS); });
afterAll(async () => { restoreWaits(); await shared.close(); server?.stop(true); }, AFTER_ALL_MS);
async function local(): Promise<ConsoleBrowser> {
const ctx = scope.track(await browser.newContext());
@ -61,7 +63,7 @@ async function local(): Promise<ConsoleBrowser> {
const done = new Promise<void>(r => ctx.on('close', () => { closed = true; r(); }));
return { mode: 'profile', context: ctx, page, pages: () => ctx.pages(), closed: done, isClosed: () => closed, close: () => ctx.close(), cleanup: () => {} };
}
const fast = (log: ReturnType<typeof logLines>) => ({ say: log.say, tickMs: 100, navTimeoutMs: 5000, signInWaitMs: 30_000, env: {} });
const fast = (log: ReturnType<typeof logLines>) => ({ say: log.say, tickMs: TEST_TICK_MS, navTimeoutMs: 5000, signInWaitMs: 30_000, env: {} });
const clicks = (b: ConsoleBrowser): Promise<string[]> => b.page.evaluate(() => (window as any).__clicks);
const navClicks = (b: ConsoleBrowser): Promise<string[]> => b.page.evaluate(() => (window as any).__navClicks());
/** Guardrail: a line that tells the owner to click a navigation control themselves. */
@ -139,7 +141,7 @@ describe('button classes (FORM_BUTTONS, flowClass, flowAllowed)', () => {
});
describe.skipIf(!browser)('App Privacy: the whole questionnaire (fixture apple-app-privacy-flow.html)', () => {
test('first time: Get Started > yes > Next > data types > Save > each type (purposes > Next > linked > Next > tracking > Save); Publish never clicked', async () => {
scope.concurrent('first time: Get Started > yes > Next > data types > Save > each type (purposes > Next > linked > Next > tracking > Save); Publish never clicked', async () => {
const b = await local();
const log = logLines();
const run = scope.run(fillSession(b, { form: 'apple-app-privacy', config: privacy(), startUrl: base + 'apple-app-privacy-flow.html', ...fast(log) }));
@ -159,7 +161,7 @@ describe.skipIf(!browser)('App Privacy: the whole questionnaire (fixture apple-a
expect((await run).fallback).toBe(false);
}, TEST_MS);
test('answers exist and the list differs from the config: Edit next to Data Types (never the Privacy Policy Edit), then every type', async () => {
scope.concurrent('answers exist and the list differs from the config: Edit next to Data Types (never the Privacy Policy Edit), then every type', async () => {
const b = await local();
const log = logLines();
const config = privacy({ COARSE_LOCATION: { collected: true, purposes: ['ANALYTICS'], linked: false, tracking: false } });
@ -174,7 +176,7 @@ describe.skipIf(!browser)('App Privacy: the whole questionnaire (fixture apple-a
await run;
}, TEST_MS);
test('an uncertain answer stops the walk on its page until the owner sets it, then it goes on', async () => {
scope.concurrent('an uncertain answer stops the walk on its page until the owner sets it, then it goes on', async () => {
const b = await local();
const log = logLines();
const config = privacy({ DEVICE_ID: { collected: true, purposes: ['ANALYTICS'], tracking: false } }); // linked: not in the config
@ -194,7 +196,7 @@ describe.skipIf(!browser)('App Privacy: the whole questionnaire (fixture apple-a
await run;
}, TEST_MS);
test('no data collected: "No" and the dialog\'s Save, then the summary', async () => {
scope.concurrent('no data collected: "No" and the dialog\'s Save, then the summary', async () => {
const b = await local();
const log = logLines();
const types = Object.fromEntries(APPLE_DATA_TYPE_IDS.map(id => [id, { collected: false }]));
@ -224,7 +226,7 @@ const SETUP = 'apple-app-privacy-setup.html';
const answersOf = (b: ConsoleBrowser) => b.page.evaluate(() => (window as any).__answers);
describe.skipIf(!browser)('App Privacy: each data type set up from the real page ("Set Up <type>" as plain text; fixture apple-app-privacy-setup.html)', () => {
test('every collected type is set up through its dialog (purposes > Next > linked > Next > two info steps > Next > Next > tracking > Save; purpose checkboxes without an accessible name); "?", Publish and the Edits never clicked; every step dumped', async () => {
scope.concurrent('every collected type is set up through its dialog (purposes > Next > linked > Next > two info steps > Next > Next > tracking > Save; purpose checkboxes without an accessible name); "?", Publish and the Edits never clicked; every step dumped', async () => {
const dir = mkdtempSync(join(tmpdir(), 'sb-console-debug-'));
try {
const b = await local();
@ -258,7 +260,7 @@ describe.skipIf(!browser)('App Privacy: each data type set up from the real page
} finally { rmSync(dir, { recursive: true, force: true }); }
}, TEST_MS);
test('an uncertain answer ("you decide") pauses on its step, names it, and goes on once the owner set it', async () => {
scope.concurrent('an uncertain answer ("you decide") pauses on its step, names it, and goes on once the owner set it', async () => {
const b = await local();
const log = logLines();
const config = six({ PURCHASE_HISTORY: { collected: true, purposes: ['APP_FUNCTIONALITY'], tracking: false } }); // linked: the owner's
@ -279,7 +281,7 @@ describe.skipIf(!browser)('App Privacy: each data type set up from the real page
await run;
}, TEST_MS);
test('an unknown dialog step: a debug dump, a pause that names the data type, within the step timeout; nothing clicked past it', async () => {
scope.concurrent('an unknown dialog step: a debug dump, a pause that names the data type, within the step timeout; nothing clicked past it', async () => {
const dir = mkdtempSync(join(tmpdir(), 'sb-console-debug-'));
try {
const b = await local();
@ -294,7 +296,8 @@ describe.skipIf(!browser)('App Privacy: each data type set up from the real page
const said = log.lines.join('\n');
expect(said).toMatch(/the "Device ID" dialog step after "Next" is not a page storebound knows .*Buttons in the dialog: "Back", "Cancel", "Next"/);
expect(said).toContain(`Debug saved to ${dir}`);
const sub = readdirSync(dir).find(n => /unknown/.test(n))!;
// the Device ID pause's dump (on a loaded machine a slow earlier step can get a dump of its own first)
const sub = readdirSync(dir).find(n => /unknown/.test(n) && /device-id/.test(n))!;
const pj = JSON.parse(readFileSync(join(dir, sub, 'page.json'), 'utf8'));
expect(pj.dialogs).toContain('Device ID');
expect(pj.controls).toContainEqual({ role: 'switch', name: 'Account identity', checked: false });
@ -308,7 +311,7 @@ describe.skipIf(!browser)('App Privacy: each data type set up from the real page
} finally { rmSync(dir, { recursive: true, force: true }); }
}, TEST_MS);
test('types set up before: the same purposes are left as they are, other purposes are opened with their Edit', async () => {
scope.concurrent('types set up before: the same purposes are left as they are, other purposes are opened with their Edit', async () => {
const b = await local();
const log = logLines();
const url = `${base}${SETUP}?preset=${encodeURIComponent('Crash Data:App Functionality')}&preset=${encodeURIComponent('Device ID:Analytics')}`;
@ -327,7 +330,7 @@ describe.skipIf(!browser)('App Privacy: each data type set up from the real page
});
describe.skipIf(!browser)('the other forms: navigation by itself, final buttons and draft saves left to the owner', () => {
test('content rating: Save draft and Next by itself on each page, a stop at the uncertain questions until answered, the Summary; its Save never clicked', async () => {
scope.concurrent('content rating: Save draft and Next by itself on each page, a stop at the uncertain questions until answered, the Summary; its Save never clicked', async () => {
const b = await local();
const log = logLines();
const run = scope.run(fillSession(b, { form: 'play-content-rating', config: base0, startUrl: base + 'play-content-rating.html', ...fast(log) }));
@ -348,7 +351,7 @@ describe.skipIf(!browser)('the other forms: navigation by itself, final buttons
await run;
}, TEST_MS);
test('target audience: Next by itself, a stop at the uncertain appeal question until answered, then Next; Save stays the owner\'s', async () => {
scope.concurrent('target audience: Next by itself, a stop at the uncertain appeal question until answered, then Next; Save stays the owner\'s', async () => {
const b = await local();
const log = logLines();
const run = scope.run(fillSession(b, { form: 'play-target-audience', config: base0, startUrl: base + 'play-target-audience.html', ...fast(log) }));
@ -362,7 +365,7 @@ describe.skipIf(!browser)('the other forms: navigation by itself, final buttons
await run;
}, TEST_MS);
test('New App (App Store) and Create app (Play): filled, the final button is the owner\'s, no navigation asked of the owner', async () => {
scope.concurrent('New App (App Store) and Create app (Play): filled, the final button is the owner\'s, no navigation asked of the owner', async () => {
for (const [form, file, final] of [['apple-app-record', 'apple-app-record.html', 'Create'], ['play-app-record', 'play-create-app.html', 'Create app']] as const) {
const b = await local();
const log = logLines();
@ -378,7 +381,7 @@ describe.skipIf(!browser)('the other forms: navigation by itself, final buttons
});
describe.skipIf(!browser)('watchdog: an unknown page after a click never means a silent wait', () => {
test('Next leads to a page storebound does not know: heartbeat, a sanitized debug dump, the manual steps, a bounded stop', async () => {
scope.concurrent('Next leads to a page storebound does not know: heartbeat, a sanitized debug dump, the manual steps, a bounded stop', async () => {
const dir = mkdtempSync(join(tmpdir(), 'sb-console-debug-'));
try {
const b = await local();
@ -399,7 +402,8 @@ describe.skipIf(!browser)('watchdog: an unknown page after a click never means a
expect(said).toContain(`Debug saved to ${dir}`);
expect(said).toContain('this part is left to the calling agent with its browser tool (`storebound console plan apple-app-privacy`');
expect(said).toContain('it stops here');
const [sub] = readdirSync(dir);
// the dump of the unknown page (on a loaded machine a slow earlier step can get a dump of its own first)
const sub = readdirSync(dir).find(n => /unknown-the-page-after-next/.test(n));
const files = readdirSync(join(dir, sub!)).sort();
expect(files).toEqual(['page.html', 'page.json', 'screen.jpg']);
const pj = JSON.parse(readFileSync(join(dir, sub!, 'page.json'), 'utf8'));
@ -412,7 +416,7 @@ describe.skipIf(!browser)('watchdog: an unknown page after a click never means a
} finally { rmSync(dir, { recursive: true, force: true }); }
}, TEST_MS);
test('--debug-dump: a dump at the start, after each click and after each filled page', async () => {
scope.concurrent('--debug-dump: a dump at the start, after each click and after each filled page', async () => {
const dir = mkdtempSync(join(tmpdir(), 'sb-console-debug-'));
try {
const b = await local();

2
tests/console-persist.test.ts
View file

@ -31,7 +31,7 @@ afterAll(() => { server?.stop(true); });
describe.skipIf(!real.label)(`storebound's own profile (${real.label ?? 'no real Chrome'})`, () => {
for (const store of ['apple', 'play'] as const) {
test(`${store}: a session cookie survives closing and relaunching storebound's own profile`, async () => {
test.concurrent(`${store}: a session cookie survives closing and relaunching storebound's own profile`, async () => {
const dir = mkdtempSync(join(tmpdir(), `storebound-profile-${store}-`));
const root = join(import.meta.dir, '..');
const { chromium } = await import('playwright');

4
tests/console-session.test.ts
View file

@ -193,7 +193,7 @@ const hasChrome = process.platform === 'darwin' && existsSync(CHROME);
if (!hasChrome) console.warn('console-session integration test skipped: Google Chrome is not installed (or not macOS)');
describe.skipIf(!hasChrome)('copy mode with real Chrome (throwaway profile, local site)', () => {
test('a cookie set in the source profile is there in the copy; another host\'s is filtered out; the copy is deleted on close', async () => {
test.concurrent('a cookie set in the source profile is there in the copy; another host\'s is filtered out; the copy is deleted on close', async () => {
let chromium: any;
try { chromium = (await import('playwright')).chromium; } catch { console.warn('skipped: playwright not installed'); return; }
const server = Bun.serve({ hostname: '127.0.0.1', port: 0, fetch: req => {
@ -238,7 +238,7 @@ describe.skipIf(!hasChrome)('copy mode with real Chrome (throwaway profile, loca
});
describe.skipIf(!hasChrome)('the console window has no automation markers (real Chrome, throwaway profile, local page)', () => {
test('navigator.webdriver is false and the real command line has no --enable-automation', async () => {
test.concurrent('navigator.webdriver is false and the real command line has no --enable-automation', async () => {
const server = Bun.serve({ hostname: '127.0.0.1', port: 0, fetch: () => new Response(
'<p id="r"></p><script>document.getElementById("r").textContent = JSON.stringify({ webdriver: navigator.webdriver, ua: navigator.userAgent })</script>',
{ headers: { 'content-type': 'text/html' } }) });

46
tests/helpers/browser.ts
View file

@ -13,6 +13,8 @@
// - polling sleeps in the waits for log lines: event-driven now (resolved on each new line)
// Hook timeouts are bounded by the numbers below (each close is capped), never open-ended.
/* eslint-disable @typescript-eslint/no-explicit-any */
import { AsyncLocalStorage } from 'node:async_hooks';
import { test } from 'bun:test';
/** Launching Chromium: well above a cold start on a loaded machine (seconds), far below "hung". */
export const LAUNCH_TIMEOUT_MS = 60_000;
@ -25,6 +27,16 @@ export const CLOSE_TIMEOUT_MS = 15_000;
export const AFTER_EACH_MS = 2 * CLOSE_TIMEOUT_MS + 5_000;
export const AFTER_ALL_MS = CLOSE_TIMEOUT_MS + 5_000;
/**
* The console engine's timings for tests on local fixture pages (production: a 1 s watch round, 200 ms poll rounds,
* a missing step waited out for the whole step timeout). A round of 20 ms; a poll ends as "not found" once the page
* is loaded and unchanged for 1.5 s (well above the one timer a fixture page has, asc-shell.html's 700 ms redirect),
* so a step that is not there no longer costs the whole navTimeoutMs, while a step that is there is still waited for
* as long as the machine needs. Set with engineWaits (cli/console/fill.ts) in beforeAll, restored in afterAll.
*/
export const TEST_TICK_MS = 20;
export const TEST_WAITS = { pollMs: 50, quietMs: 1500, settleMs: 50, missMs: 300 };
/** Waits for `p` (its error ignored) or `ms`, whichever is first: true when `p` settled in time. Never rejects. */
export async function capped(p: Promise<unknown>, ms: number): Promise<boolean> {
let t: ReturnType<typeof setTimeout> | undefined;
@ -88,11 +100,8 @@ export async function persistentBrowser(): Promise<{ label: string | null; skipW
: { label: null, skipWhy: 'neither Google Chrome nor Playwright\'s full Chromium is installed (the headless shell drops session cookies on relaunch); install Google Chrome or run `bunx playwright install chromium`' };
}
/**
* The contexts and engine runs of the current test. `newPage` opens a page in a fresh context (as browser.newPage
* does) and tracks it; `track` adds one opened elsewhere; `settle` (afterEach) closes them all and waits for the runs.
*/
export function testScope(browser: () => any) {
/** The contexts and engine runs of one test (see testScope). */
function plainScope(browser: () => any) {
const contexts = new Set<any>();
const runs = new Set<Promise<unknown>>();
return {
@ -113,6 +122,33 @@ export function testScope(browser: () => any) {
};
}
/**
* The contexts and engine runs of the current test. `newPage` opens a page in a fresh context (as browser.newPage
* does) and tracks it; `track` adds one opened elsewhere; `settle` (afterEach) closes them all and waits for the runs.
* `concurrent` declares a browser test that runs at the same time as the file's other concurrent tests
* (test.concurrent, at most --max-concurrency at once: tools/ci.ts passes BROWSER_CONCURRENCY from tools/ci-plan.ts): it gets a scope of
* its own (AsyncLocalStorage, so newPage / track / run inside it reach that scope), settled when it ends, inside its
* timeout. The tests share the one browser; each has its own contexts, pages and log.
*/
export function testScope(browser: () => any) {
type Scope = ReturnType<typeof plainScope>;
const shared = plainScope(browser);
const own = new AsyncLocalStorage<Scope>();
const cur = (): Scope => own.getStore() ?? shared;
return {
newPage: (opts?: Record<string, unknown>): Promise<any> => cur().newPage(opts),
track: <T>(ctx: T): T => cur().track(ctx),
run: <T>(p: Promise<T>): Promise<T> => cur().run(p),
settle: (): Promise<void> => shared.settle(),
concurrent(name: string, fn: () => Promise<void>, ms: number): void {
test.concurrent(name, () => {
const s = plainScope(browser);
return own.run(s, async () => { try { await fn(); } finally { await s.settle(); } });
}, ms + AFTER_EACH_MS);
},
};
}
/**
* The log lines of an engine run (`say`), with event-driven waits: `wait` resolves on the line that makes `pred`
* true (checked on every new line, no polling) and fails after `ms` with every line seen so far.

16
tools/ci-plan.ts
View file

@ -16,6 +16,11 @@ export const HEAVY = [
'tests/capture-web.test.ts', 'tests/console-session.test.ts', 'tests/console-persist.test.ts', 'tests/console-flow.test.ts',
];
/** Browser tests of one file at the same time (bun test --max-concurrency for the shards with a HEAVY file; the
* tests opt in with testScope().concurrent, tests/helpers/browser.ts): enough to overlap their waits, few enough that
* a loaded machine does not stretch each one towards its timeout. */
export const BROWSER_CONCURRENCY = 4;
/** Changes that can affect every test: run the whole suite. */
export const GLOBAL = /^(package\.json|bun\.lock|bunfig\.toml|tsconfig\.json|tests\/helpers\/preload\.ts)$/;
@ -50,6 +55,17 @@ export function writeTimings(t: Record<string, number>, file = TIMINGS): void {
try { mkdirSync(dirname(file), { recursive: true }); writeFileSync(file, `${JSON.stringify(t, null, 1)}\n`); } catch { /* cache only */ }
}
/**
* The recorded time of each file after a run: a new file takes its measurement; a known one moves halfway towards it
* and never above it. The machine is shared, so one run under heavy load (other sessions' CI, cargo) must not make a
* file look several times slower for the next split, while a file that really got faster or slower still converges.
*/
export function mergeTimings(old: Record<string, number>, fresh: Record<string, number>): Record<string, number> {
const out = { ...old };
for (const [f, t] of Object.entries(fresh)) out[f] = old[f] === undefined ? t : Math.min(t, (old[f]! + t) / 2);
return out;
}
/** Per-file seconds from bun's JUnit report (the top-level <testsuite> of each file). */
export function junitTimes(xml: string): Record<string, number> {
const out: Record<string, number> = {};

16
tools/ci.ts
View file

@ -5,8 +5,9 @@
// bun run ci:quick inner loop: typecheck + the tests related to the change (git diff against origin/main,
// mapped through imports) + scrub
// The tests run as shards: N = CPU cores - 1 (max 8; STOREBOUND_CI_SHARDS=n overrides) parallel `bun test <files>`
// processes, balanced by the durations recorded after the last run (node_modules/.cache/storebound-ci/, falling back
// to file size). Browser suites (tools/ci-plan.ts HEAVY) each get a shard of their own.
// processes, balanced by the durations recorded after earlier runs (node_modules/.cache/storebound-ci/, smoothed by
// mergeTimings so one run on a loaded machine does not skew the split; falling back to file size). Browser suites
// (tools/ci-plan.ts HEAVY) each get a shard of their own and run their opted-in tests BROWSER_CONCURRENCY at a time.
// `bun test` covers the CLI and the desktop webview (apps/desktop/tests), the site overflow check and the
// no-hand-written-SVG guard; the desktop boot check (tools/desktop-boot-check.ts: the built window in headless
// Chromium, needs playwright's browser or Google Chrome) runs after the desktop build.
@ -17,7 +18,7 @@ import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
import { homedir, tmpdir } from 'node:os';
import { join } from 'node:path';
import { hostTarget, OUT_DIR, outName } from './build-sidecars';
import { changedFiles, HEAVY, junitTimes, listTests, planShards, readTimings, relatedTests, ROOT, rustNeeded, writeTimings } from './ci-plan';
import { BROWSER_CONCURRENCY, changedFiles, HEAVY, junitTimes, listTests, mergeTimings, planShards, readTimings, relatedTests, ROOT, rustNeeded, writeTimings } from './ci-plan';
interface Task { name: string; cmd: string[]; cwd?: string; env?: Record<string, string>; after?: string[]; junit?: string }
@ -39,7 +40,8 @@ function addTests(files: string[]) {
// the built-bundle check in apps/desktop/tests/webview.test.ts runs as "desktop bundle check" once the build is
// done; here it would read dist/ while vite rewrites it
// browser suites share the machine with the other steps: a browser start or close can pass bun's 5 s default
const timeout = shard.some(f => HEAVY.includes(f)) ? ['--timeout=30000'] : [];
// browser tests that opt in (testScope().concurrent) run BROWSER_CONCURRENCY at a time within their file
const timeout = shard.some(f => HEAVY.includes(f)) ? ['--timeout=30000', `--max-concurrency=${BROWSER_CONCURRENCY}`] : [];
add({ name: `tests ${i + 1}/${all.length}`, cmd: ['bun', 'test', ...timeout, '--reporter=junit', `--reporter-outfile=${junit}`, ...shard.map(f => `./${f}`)],
env: { STOREBOUND_SKIP_DIST_CHECK: '1' }, junit });
});
@ -156,12 +158,12 @@ while ((pending.size || active.size) && !failed) {
if (failed) await Promise.allSettled([...active]);
// record per-file durations for the next split (only from shards that finished)
const timings = readTimings();
const fresh: Record<string, number> = {};
for (const r of runs.values()) {
if (!r.task.junit || r.end === undefined) continue;
try { Object.assign(timings, junitTimes(readFileSync(r.task.junit, 'utf8'))); } catch { /* no report */ }
try { Object.assign(fresh, junitTimes(readFileSync(r.task.junit, 'utf8'))); } catch { /* no report */ }
}
writeTimings(timings);
writeTimings(mergeTimings(readTimings(), fresh));
rmSync(junitDir, { recursive: true, force: true });
console.log(`\n${quick ? 'ci:quick' : 'ci'} summary (wall ${((Date.now() - t0) / 1000).toFixed(1)}s)`);

157
tools/desktop-boot-check.ts
View file

@ -13,8 +13,11 @@
// build picker, the plan panel and a console plan (the real sidecar) by DOM text, then every tab of the app and the
// Dependencies screen.
// bun tools/desktop-boot-check.ts [--dist apps/desktop/dist] [--only d1-missing-folders] [--json]
// Uses playwright: Google Chrome when installed (channel "chrome"), else playwright's Chromium
// (bunx playwright install chromium). Part of `bun run ci` and of `tools/desktop-release.ts mac`.
// Speed: one browser (killed at the end, never closed gracefully), one page per case, the cases in parallel
// (STOREBOUND_BOOT_CHECK_JOBS, default 8) against the one build; no fixed sleeps: after each action it waits for a
// condition (the element, the text, or settle(): no host call in flight and no DOM change for QUIET_MS).
// Uses playwright: its headless shell (bunx playwright install chromium-headless-shell), else Google Chrome
// (channel "chrome"), else its full Chromium. Part of `bun run ci` and of `tools/desktop-release.ts mac`.
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
import { extname, join, resolve } from 'node:path';
import type { Browser, Page } from 'playwright';
@ -173,12 +176,21 @@ const initScript = (freeze: boolean) => `(() => {
unregisterCallback(id) { delete cbs[id]; },
convertFileSrc(p) { return p; },
async invoke(cmd, args) {
const r = await window.__bootCheckHost(cmd, args || {});
if ('err' in r) throw r.err;
return r.ok;
window.__bootInflight = (window.__bootInflight || 0) + 1;
window.__bootChanged = performance.now();
try {
const r = await window.__bootCheckHost(cmd, args || {});
if ('err' in r) throw r.err;
return r.ok;
} finally { window.__bootInflight--; window.__bootChanged = performance.now(); }
},
};
window.__TAURI_EVENT_PLUGIN_INTERNALS__ = { unregisterListener() {} };
// settle(): the last host call or DOM change, so the check waits for a quiet window instead of a fixed time
window.__bootInflight = 0;
window.__bootChanged = performance.now();
new MutationObserver(() => { window.__bootChanged = performance.now(); })
.observe(document, { childList: true, subtree: true, attributes: true, characterData: true });
})();`;
// ---------------------------------------------------------------------------------------------------------------------
@ -211,6 +223,32 @@ async function state(page: Page) {
}));
}
/** A quiet window: no host call in flight and no DOM change for this long means the last action is done. */
const QUIET_MS = 150;
/** Upper bound of one wait for a condition (a loaded machine); a wait that runs out is a failure or a check as-is. */
const WAIT_MS = 15_000;
/**
* Waits until the window is quiet (initScript's counters: no host call in flight, no DOM change for QUIET_MS): what a
* click or the boot started has drawn. Condition-based, no fixed sleep; a window that never gets quiet (an animation
* written by script) is checked as it is after `ms`.
*/
async function settle(page: Page, ms = WAIT_MS): Promise<void> {
await page.waitForFunction((q: number) => {
const w = window as unknown as { __bootInflight: number; __bootChanged: number };
return w.__bootInflight === 0 && performance.now() - w.__bootChanged >= q;
}, QUIET_MS, { timeout: ms, polling: 50 }).catch(() => {});
}
/** Waits until `sel` is gone (a dialog or panel closed); false when it is still there after `ms`. */
const gone = (page: Page, sel: string, ms = WAIT_MS): Promise<boolean> =>
page.waitForSelector(sel, { state: 'detached', timeout: ms }).then(() => true, () => false);
/** Waits until the first `sel` contains `text` (whitespace folded); false after `ms`. */
const shows = (page: Page, sel: string, text: string, ms = WAIT_MS): Promise<boolean> =>
page.waitForFunction(([s, t]) => (document.querySelector(s!) as HTMLElement | null)?.innerText.replace(/\s+/g, ' ').includes(t!) ?? false,
[sel, text] as const, { timeout: ms, polling: 50 }).then(() => true, () => false);
async function runCase(browser: Browser, name: string, sidecar: Sidecar, o: { breakBundle?: boolean } = {}): Promise<CaseResult> {
const freeze = conf.app.security.freezePrototype === true;
const server = serve(DIST, cspHeader(conf.app.security.csp), !!o.breakBundle);
@ -232,13 +270,13 @@ async function runCase(browser: Browser, name: string, sidecar: Sidecar, o: { br
try {
await page.goto(`http://127.0.0.1:${server.port}/`, { waitUntil: 'load' });
if (o.breakBundle) {
await page.waitForSelector('#boot-fallback', { timeout: 8000 }).catch(() => {});
await page.waitForSelector('#boot-fallback', { timeout: WAIT_MS }).catch(() => {});
const s = await state(page);
const ok = s.crash.some(c => c.startsWith('boot'));
return { name, ok, problems: ok ? [] : ['the boot.js error panel did not appear for a missing bundle'], visited: 0, notes: [`boot.js error panel shown: ${ok}`, ...logged.map(l => `webview.log: ${l}`)] };
}
await page.waitForFunction(() => document.documentElement.dataset.booted === '1', null, { timeout: 15000 }).catch(() => problems.push('React never drew the window (no data-booted)'));
await page.waitForTimeout(1500);
await page.waitForFunction(() => document.documentElement.dataset.booted === '1', null, { timeout: WAIT_MS }).catch(() => problems.push('React never drew the window (no data-booted)'));
await settle(page);
const check = async (where: string) => {
const s = await state(page);
if (freeze && !s.frozen) problems.push('Object.prototype is not frozen in the page (the check would not see the override mistake)');
@ -251,7 +289,7 @@ async function runCase(browser: Browser, name: string, sidecar: Sidecar, o: { br
if (await wizard.count()) {
notes.push('welcome guide shown');
await wizard.press('Escape').catch(() => {});
await page.waitForTimeout(300);
await gone(page, '.wizard[role="dialog"]');
if (await wizard.count()) problems.push('the welcome guide does not close with Escape');
}
// every sidebar entry: the portfolio, each app (folders that are gone included), each workspace screen
@ -259,8 +297,8 @@ async function runCase(browser: Browser, name: string, sidecar: Sidecar, o: { br
for (let i = 0; i < count; i++) {
const b = page.locator('nav.side button.nav').nth(i);
const label = ((await b.innerText().catch(() => '')) || `entry ${i}`).split('\n')[0]!.trim();
await b.click({ timeout: 5000 }).catch(e => problems.push(`could not open ${label}: ${String(e).split('\n')[0]}`));
await page.waitForTimeout(700);
await b.click({ timeout: WAIT_MS }).catch(e => problems.push(`could not open ${label}: ${String(e).split('\n')[0]}`));
await settle(page);
await check(label);
visited++;
}
@ -268,8 +306,9 @@ async function runCase(browser: Browser, name: string, sidecar: Sidecar, o: { br
for (const p of fx.projects_list.filter(x => !isDir(x.path))) {
const item = page.locator('nav.side .nav-app', { hasText: p.name }).first();
if (!(await item.count())) { problems.push(`${p.name}: not in the sidebar`); continue; }
await item.click();
await page.waitForTimeout(500);
await item.click({ timeout: WAIT_MS });
await shows(page, '#content', 'Folder not found');
await settle(page);
const text = await page.locator('#content').innerText();
if (!text.includes('Folder not found') || !text.includes('Locate…') || !text.includes('Remove from list')) problems.push(`${p.name}: no "Folder not found" panel with Locate… / Remove`);
else notes.push(`${p.name}: Folder not found, Locate… / Remove`);
@ -321,18 +360,18 @@ async function runSubmitCase(browser: Browser, sidecar: Sidecar, base: 'd3-full'
const crashCheck = async (where: string) => { for (const c of (await state(page)).crash) problems.push(`${where}: error panel ${c}`); };
try {
await page.goto(`http://127.0.0.1:${server.port}/`, { waitUntil: 'load' });
await page.waitForFunction(() => document.documentElement.dataset.booted === '1', null, { timeout: 15000 }).catch(() => problems.push('React never drew the window (no data-booted)'));
await page.waitForTimeout(1200);
await page.waitForFunction(() => document.documentElement.dataset.booted === '1', null, { timeout: WAIT_MS }).catch(() => problems.push('React never drew the window (no data-booted)'));
await settle(page);
const wizard = page.locator('.wizard[role="dialog"]');
if (await wizard.count()) { await wizard.press('Escape').catch(() => {}); await page.waitForTimeout(300); }
if (await wizard.count()) { await wizard.press('Escape').catch(() => {}); await gone(page, '.wizard[role="dialog"]'); }
const app = fx.projects_list.find(p => isDir(p.path))!;
// the sidebar names the app by its config name once the offline portfolio is read
const item = page.locator('nav.side .nav-app').filter({ hasText: base === 'd3-full' ? /Prism Puddle|demo-game/ : new RegExp(app.name) }).first();
await item.click({ timeout: 8000 }).catch(e => problems.push(`could not open the app: ${String(e).split('\n')[0]}`));
await page.waitForTimeout(800);
await page.locator('[role="tab"]#tab-releases').click({ timeout: 5000 }).catch(() => {});
await page.waitForSelector('[data-section="submit"]', { timeout: 8000 }).catch(() => problems.push('Releases: no send-for-review section'));
await page.waitForTimeout(600);
await item.click({ timeout: WAIT_MS }).catch(e => problems.push(`could not open the app: ${String(e).split('\n')[0]}`));
await settle(page);
await page.locator('[role="tab"]#tab-releases').click({ timeout: WAIT_MS }).catch(() => {});
await page.waitForSelector('[data-section="submit"]', { timeout: WAIT_MS }).catch(() => problems.push('Releases: no send-for-review section'));
await settle(page);
const rel = await text('[data-section="submit"]');
for (const want of SUBMIT_EXPECT[variant]) if (!rel.includes(want)) problems.push(`Releases (${variant}): missing "${want}"`);
if (rel.includes('storebound never submits for review')) problems.push('Releases: the old "submit stays in the console" note is still there');
@ -346,15 +385,16 @@ async function runSubmitCase(browser: Browser, sidecar: Sidecar, base: 'd3-full'
const action = page.locator('[data-section="submit"] button', { hasText: variant === 'ready' ? 'Submit for review' : 'Resubmit' }).first();
if (!(await action.isDisabled())) problems.push('the submit button is enabled before a build is picked');
await page.locator('[data-section="submit"] select').first().selectOption('52');
await action.click({ timeout: 5000 });
await page.waitForSelector('.oa-panel', { timeout: 8000 }).catch(() => problems.push('no plan panel after the submit button'));
await action.click({ timeout: WAIT_MS });
await page.waitForSelector('.oa-panel', { timeout: WAIT_MS }).catch(() => problems.push('no plan panel after the submit button'));
await settle(page);
const panel = await text('.oa-panel');
for (const want of ['Plan (dry run, nothing sent yet)', 'STEP', 'attach build 52', 'a system dialog shows this plan again']) if (!panel.includes(want)) problems.push(`plan panel: missing "${want}"`);
await page.locator('.oa-panel .lb-foot button', { hasText: 'Continue' }).click({ timeout: 5000 }).catch(e => problems.push(`Continue: ${String(e).split('\n')[0]}`));
await page.waitForTimeout(500);
await page.locator('.oa-panel .lb-foot button', { hasText: 'Continue' }).click({ timeout: WAIT_MS }).catch(e => problems.push(`Continue: ${String(e).split('\n')[0]}`));
await shows(page, '.oa-panel', 'nothing was sent');
if (!(await text('.oa-panel')).includes('nothing was sent')) problems.push('plan panel: no "nothing was sent" after the dialog said Cancel');
await page.keyboard.press('Escape');
await page.waitForTimeout(300);
await gone(page, '.oa-panel');
if (await page.locator('.oa-panel').count()) problems.push('plan panel does not close with Escape');
const want = variant === 'ready' ? 'owner.apple_submit' : 'owner.apple_resubmit';
if (!live.calls.some(c => c.startsWith(`dry ${want} {"build":"52","version":"2.4.0"}`))) problems.push(`no dry run of ${want} with build 52 and version 2.4.0: ${live.calls.join(' | ')}`);
@ -364,27 +404,29 @@ async function runSubmitCase(browser: Browser, sidecar: Sidecar, base: 'd3-full'
const codeOpts = await codes.locator('option').evaluateAll(os => os.map(o => `${(o as HTMLOptionElement).value}${(o as HTMLOptionElement).disabled ? '!' : ''}`));
if (JSON.stringify(codeOpts) !== JSON.stringify(['', '24', '23!', '22'])) problems.push(`versionCode picker: ${JSON.stringify(codeOpts)}`);
await codes.selectOption('24');
await page.locator('[data-section="submit"] button', { hasText: 'Send to production' }).first().click({ timeout: 5000 });
await page.waitForSelector('.oa-panel', { timeout: 8000 }).catch(() => problems.push('no plan panel after Send to production'));
await page.locator('[data-section="submit"] button', { hasText: 'Send to production' }).first().click({ timeout: WAIT_MS });
await page.waitForSelector('.oa-panel', { timeout: WAIT_MS }).catch(() => problems.push('no plan panel after Send to production'));
await settle(page);
const pp = await text('.oa-panel');
for (const w of ['--version-code 24 --production completed', 'managed publishing is off']) if (!pp.includes(w)) problems.push(`production plan: missing "${w}"`);
await page.keyboard.press('Escape');
await page.waitForTimeout(300);
await gone(page, '.oa-panel');
if (!live.calls.some(c => c.startsWith('dry owner.play_production {"version_code":"24","status":"completed"}'))) problems.push(`no dry run of owner.play_production 24: ${live.calls.join(' | ')}`);
// a console plan from the real sidecar (offline): Play's Send changes for review
await page.locator('[data-section="submit"] button', { hasText: 'Send changes for review' }).first().click({ timeout: 5000 });
await page.waitForSelector('.cp-panel', { timeout: 10000 }).catch(() => problems.push('no console plan panel'));
await page.locator('[data-section="submit"] button', { hasText: 'Send changes for review' }).first().click({ timeout: WAIT_MS });
await page.waitForSelector('.cp-panel', { timeout: WAIT_MS }).catch(() => problems.push('no console plan panel'));
await settle(page);
const cp = await text('.cp-panel');
for (const w of ['send the first production release for review', 'Send changes for review', 'Check the result']) if (!cp.includes(w)) problems.push(`console plan: missing "${w}"`);
await page.keyboard.press('Escape');
await page.waitForTimeout(300);
await gone(page, '.cp-panel');
await crashCheck('Releases');
visited++;
// every tab of the app, then App info's console steps and the Dependencies screen
const tabs = await page.locator('[role="tab"]').evaluateAll(ts => ts.map(x => x.id));
for (const id of tabs) {
await page.locator(`[role="tab"]#${id}`).click({ timeout: 5000 }).catch(e => problems.push(`tab ${id}: ${String(e).split('\n')[0]}`));
await page.waitForTimeout(500);
await page.locator(`[role="tab"]#${id}`).click({ timeout: WAIT_MS }).catch(e => problems.push(`tab ${id}: ${String(e).split('\n')[0]}`));
await settle(page);
await crashCheck(id);
visited++;
if (id === 'tab-appinfo') {
@ -395,8 +437,9 @@ async function runSubmitCase(browser: Browser, sidecar: Sidecar, base: 'd3-full'
}
notes.push(`tabs: ${tabs.join(', ')}`);
const deps = page.locator('nav.side button.nav', { hasText: 'Dependencies' }).first();
await deps.click({ timeout: 5000 }).catch(() => {});
await page.waitForTimeout(800);
await deps.click({ timeout: WAIT_MS }).catch(() => {});
await shows(page, '#content', 'Android SDK');
await settle(page);
if (!(await text()).includes('Android SDK')) problems.push('Dependencies: no Android SDK card');
await crashCheck('Dependencies');
visited++;
@ -412,27 +455,47 @@ async function runSubmitCase(browser: Browser, sidecar: Sidecar, base: 'd3-full'
return { name, ok: problems.length === 0, problems: [...new Set(problems)], visited, notes };
}
async function launch(): Promise<Browser> {
/**
* One headless browser for every case, as a browser server that is KILLED at the end (a throwaway profile, nothing to
* flush): Google Chrome's graceful close takes 10-30 s after http pages (tests/helpers/browser.ts). Playwright's
* headless shell first (fast to start), else Google Chrome (channel "chrome"), else playwright's full Chromium.
*/
async function launch(): Promise<{ browser: Browser; kill: () => Promise<void> }> {
const { chromium } = await import('playwright');
try { return await chromium.launch({ channel: 'chrome', headless: true }); } catch { /* no Google Chrome */ }
return chromium.launch({ headless: true });
const opts = { headless: true, timeout: 60_000 };
let server: Awaited<ReturnType<typeof chromium.launchServer>> | undefined, first: unknown;
for (const o of [{}, { channel: 'chrome' }, { channel: 'chromium' }]) {
try { server = await chromium.launchServer({ ...opts, ...o }); break; } catch (e) { first ??= e; }
}
if (!server) throw first;
const s = server;
const browser = await chromium.connect(s.wsEndpoint(), { timeout: 60_000 }).catch(async (e: unknown) => { await s.kill(); throw e; });
return { browser, kill: async () => { await Promise.race([browser.close().catch(() => {}), Bun.sleep(5000)]); await s.kill(); } };
}
/** Cases at once (a page each, one browser); STOREBOUND_BOOT_CHECK_JOBS overrides. */
const JOBS = Math.max(1, Number(process.env.STOREBOUND_BOOT_CHECK_JOBS) || 8);
export async function bootCheck(): Promise<CaseResult[]> {
if (!existsSync(join(DIST, 'index.html'))) throw new Error(`no built frontend at ${DIST} (bun run --cwd apps/desktop build)`);
const names = readdirSync(FIXTURES).filter(f => f.endsWith('.host.json')).map(f => f.replace(/\.host\.json$/, '')).sort();
if (names.length < 3) throw new Error(`expected the state fixtures in ${FIXTURES}`);
const sidecar = new Sidecar();
await sidecar.start();
const browser = await launch();
const results: CaseResult[] = [];
const [, { browser, kill }] = await Promise.all([sidecar.start(), launch()]);
// every case in its own context and page, JOBS at a time, against the one build; results keep the case order
const cases: (() => Promise<CaseResult>)[] = [];
for (const n of names) if (!ONLY.length || ONLY.includes(n)) cases.push(() => runCase(browser, n, sidecar));
if (!ONLY.length || ONLY.includes('missing-bundle')) cases.push(() => runCase(browser, 'missing-bundle', sidecar, { breakBundle: true }));
for (const base of ['d3-full', 'd1-missing-folders'] as const) for (const v of ['ready', 'unresolved'] as const)
if (!ONLY.length || ONLY.includes('submit') || ONLY.includes(`submit-${v}`)) cases.push(() => runSubmitCase(browser, sidecar, base, v));
const results: CaseResult[] = new Array(cases.length);
let next = 0;
try {
for (const n of names) if (!ONLY.length || ONLY.includes(n)) results.push(await runCase(browser, n, sidecar));
if (!ONLY.length || ONLY.includes('missing-bundle')) results.push(await runCase(browser, 'missing-bundle', sidecar, { breakBundle: true }));
for (const base of ['d3-full', 'd1-missing-folders'] as const) for (const v of ['ready', 'unresolved'] as const)
if (!ONLY.length || ONLY.includes('submit') || ONLY.includes(`submit-${v}`)) results.push(await runSubmitCase(browser, sidecar, base, v));
await Promise.all(Array.from({ length: Math.min(JOBS, cases.length) }, async () => {
while (next < cases.length) { const i = next++; results[i] = await cases[i]!(); }
}));
} finally {
await browser.close();
await kill();
sidecar.stop();
}
return results;