282 lines
19 KiB
TypeScript
282 lines
19 KiB
TypeScript
// Web capture (shots.capture.source "web"), parallel and incremental:
|
|
// --concurrency N browser contexts in one headless Chromium (default shots.capture.concurrency, else from the CPU
|
|
// count: a WebGL game is GPU-bound, so capture defaults lower than render). Files do not depend on
|
|
// the order: every frame has its own path.
|
|
// session shots.capture.session "per-shot" (default: a new context + page and a full boot per frame) or
|
|
// "per-language": one page per language and device, booted once; the next shots switch scenes in
|
|
// that page (navigate "spa": history.pushState + popstate to the shot's path, then waitFor / eval;
|
|
// navigate "reload": page.goto in the same page). A precondition that fails there (waitFor times out
|
|
// after reuseTimeoutMs) falls back to a fresh page.
|
|
// cache .storebound/capture-cache.json: per frame a hash of its inputs (shot entry, URL + query, frame
|
|
// size, session, the copy text of its language, the app build (shots.capture.build files, else
|
|
// the HTML and scripts at baseUrl), fonts.css). Unchanged frames that exist are skipped;
|
|
// --only-missing only captures frames that do not exist; --force captures everything.
|
|
// auth shots.capture.auth (auth.ts): every new context is signed in before its first shot (request
|
|
// auth through the context's request API, or a storageState file); a shot that lands on a login
|
|
// page warns; resolved secrets and cookie values are masked in every log line and the report.
|
|
// timing per frame: goto, waitFor, eval, delay, screenshot, total, in the progress line and in
|
|
// .storebound/capture-report.json, with the slowest frames and steps at the end.
|
|
import { createHash } from 'node:crypto';
|
|
import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
import { availableParallelism, cpus } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import type { ShotSpec, ShotsCapture } from '../lib/config';
|
|
import { log, warn } from '../lib/log';
|
|
import { type AuthPlan, expiredHint, landedOnLogin, maskSecrets, safeUrl } from './auth';
|
|
import { type CaptureDevice, shotsFor } from './devices';
|
|
import { applySelectors, candidates, probe } from './selectors';
|
|
import { DEFAULT_LANDSCAPE, type FrameDims } from './lib';
|
|
|
|
/** Viewport per device class; landscape defaults to shots.landscape.size's default when a caller passes only phone / tablet. */
|
|
export type CaptureDims = Record<'phone' | 'tablet', FrameDims> & { landscape?: FrameDims };
|
|
const dimsOf = (dims: CaptureDims, device: Device): FrameDims => dims[device] ?? DEFAULT_LANDSCAPE;
|
|
|
|
type Device = CaptureDevice;
|
|
export const STEPS = ['goto', 'waitFor', 'eval', 'delay', 'screenshot'] as const;
|
|
export type Step = typeof STEPS[number];
|
|
export type Timing = Record<Step | 'total', number>;
|
|
|
|
/** frame = path relative to shots.frames (cache and report key), out = absolute file. */
|
|
export interface WebJob { lang: string; device: Device; name: string; spec: ShotSpec; url: string; path: string; out: string; frame: string; hash: string }
|
|
|
|
/** CPU cores (availableParallelism where Bun has it). */
|
|
export const coreCount = (): number => { try { return availableParallelism(); } catch { return cpus().length || 2; } };
|
|
|
|
/** Default parallel workers: render min(4, cores / 2); capture (GPU-heavy WebGL / game boots) min(3, cores / 4). */
|
|
export function defaultConcurrency(kind: 'render' | 'capture', cores = coreCount()): number {
|
|
return Math.max(1, kind === 'render' ? Math.min(4, Math.floor(cores / 2)) : Math.min(3, Math.floor(cores / 4)));
|
|
}
|
|
|
|
/** Runs fn over items with at most n at a time; results keep the input order. */
|
|
export async function pool<T, R>(items: T[], n: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]> {
|
|
const out = new Array<R>(items.length);
|
|
let next = 0;
|
|
const worker = async (): Promise<void> => { while (next < items.length) { const i = next++; out[i] = await fn(items[i]!, i); } };
|
|
await Promise.all(Array.from({ length: Math.max(1, Math.min(n, items.length)) }, worker));
|
|
return out;
|
|
}
|
|
|
|
const fmtS = (ms: number): string => (ms >= 60_000 ? `${Math.floor(ms / 60_000)}m${String(Math.round((ms % 60_000) / 1000)).padStart(2, '0')}s` : `${(ms / 1000).toFixed(1)}s`);
|
|
|
|
/** "n/total, ETA" progress for parallel work: ETA = remaining items x average wall time per finished item. */
|
|
export function progressLine(done: number, total: number, startedMs: number, nowMs = Date.now()): string {
|
|
const w = String(total).length;
|
|
const eta = done ? ((nowMs - startedMs) / done) * (total - done) : 0;
|
|
return `${String(done).padStart(w)}/${total}${done < total && done ? ` ETA ${fmtS(eta)}` : ''}`;
|
|
}
|
|
|
|
export const timingText = (t: Timing): string => `goto ${fmtS(t.goto)} wait ${fmtS(t.waitFor)} eval ${fmtS(t.eval)} delay ${fmtS(t.delay)} shot ${fmtS(t.screenshot)} = ${fmtS(t.total)}`;
|
|
|
|
/** Slowest frames and the time per step over all frames (for the end of the log and the report). */
|
|
export function timingSummary(frames: { frame: string; timing: Timing }[], top = 5): { slowest: { frame: string; total: number; step: Step }[]; steps: Record<Step, number>; total: number } {
|
|
const steps = Object.fromEntries(STEPS.map(s => [s, 0])) as Record<Step, number>;
|
|
for (const f of frames) for (const s of STEPS) steps[s] += f.timing[s];
|
|
const slowest = [...frames].sort((a, b) => b.timing.total - a.timing.total).slice(0, top)
|
|
.map(f => ({ frame: f.frame, total: f.timing.total, step: STEPS.reduce((m, s) => (f.timing[s] > f.timing[m] ? s : m), STEPS[0] as Step) }));
|
|
return { slowest, steps, total: frames.reduce((t, f) => t + f.timing.total, 0) };
|
|
}
|
|
|
|
const sha = (s: string | Uint8Array): string => createHash('sha256').update(s).digest('hex').slice(0, 16);
|
|
|
|
/** Content hash of build files (a file or folder; node_modules / dot folders skipped): paths, sizes and mtimes. */
|
|
export function hashPaths(paths: string[]): string {
|
|
const h = createHash('sha256');
|
|
const walk = (p: string): void => {
|
|
if (!existsSync(p)) { h.update(`missing:${p}`); return; }
|
|
const st = statSync(p);
|
|
if (st.isDirectory()) { for (const e of readdirSync(p).sort()) if (e !== 'node_modules' && !e.startsWith('.')) walk(join(p, e)); return; }
|
|
h.update(`${p}:${st.size}:${st.mtimeMs}\n`);
|
|
};
|
|
for (const p of paths) walk(p);
|
|
return h.digest('hex').slice(0, 16);
|
|
}
|
|
|
|
/** App build fingerprint from the running server: the HTML at baseUrl and the scripts it loads (dev servers too). */
|
|
export async function hashServed(baseUrl: string): Promise<string> {
|
|
try {
|
|
const html = await (await fetch(baseUrl, { signal: AbortSignal.timeout(10_000) })).text();
|
|
const h = createHash('sha256').update(html);
|
|
for (const m of html.matchAll(/<script[^>]+src=["']([^"']+)["']/g)) {
|
|
try { h.update(await (await fetch(new URL(m[1]!, baseUrl), { signal: AbortSignal.timeout(10_000) })).text()); } catch { h.update(`unreachable:${m[1]}`); }
|
|
}
|
|
return h.digest('hex').slice(0, 16);
|
|
} catch { return 'unreachable'; }
|
|
}
|
|
|
|
/** Every web frame to capture, in a fixed order (lang, device, shot), with its URL and its input hash. */
|
|
export function planWeb(cap: ShotsCapture, o: { langs: string[]; devices: Device[]; only: string[]; dims: CaptureDims; framesDir: string;
|
|
build: string; copyText?: (lang: string) => string; fonts?: string; allLandscape?: boolean }): WebJob[] {
|
|
const jobs: WebJob[] = [];
|
|
for (const lang of o.langs) for (const device of o.devices) for (const [name, spec] of shotsFor(cap, device, o.only, o.allLandscape)) {
|
|
const q = (cap.query ?? '').replaceAll('{lang}', lang);
|
|
const path = (spec.path ?? '/').replaceAll('{lang}', lang) + (q ? ((spec.path ?? '/').includes('?') ? '&' : '?') + q : '');
|
|
const url = (cap.baseUrl ?? '').replace(/\/$/, '') + path;
|
|
const hash = sha(JSON.stringify({ spec, url, dims: dimsOf(o.dims, device), session: cap.session ?? 'per-shot', build: o.build, copy: o.copyText?.(lang) ?? '', fonts: o.fonts ?? '' }));
|
|
const frame = `${lang}/${device}/${name}.png`;
|
|
jobs.push({ lang, device, name, spec, url, path, out: join(o.framesDir, frame), frame, hash });
|
|
}
|
|
return jobs;
|
|
}
|
|
|
|
export type Cache = Record<string, string>;
|
|
export const cachePath = (root: string): string => join(root, '.storebound', 'capture-cache.json');
|
|
export function readCache(root: string): Cache { try { return JSON.parse(readFileSync(cachePath(root), 'utf8')) as Cache; } catch { return {}; } }
|
|
|
|
/** Which jobs to run: --force all; --only-missing the ones whose file is missing; else missing or changed inputs. */
|
|
export function selectJobs(jobs: WebJob[], cache: Cache, mode: 'force' | 'only-missing' | 'changed', exists: (file: string) => boolean = existsSync): { run: WebJob[]; skipped: WebJob[] } {
|
|
const run: WebJob[] = [], skipped: WebJob[] = [];
|
|
for (const j of jobs) {
|
|
const keep = mode === 'force' ? false : mode === 'only-missing' ? exists(j.out) : exists(j.out) && cache[j.frame] === j.hash;
|
|
(keep ? skipped : run).push(j);
|
|
}
|
|
return { run, skipped };
|
|
}
|
|
|
|
export interface CaptureReport {
|
|
at: string; concurrency: number; session: string; wallMs: number;
|
|
frames: { frame: string; ok: boolean; reused: boolean; timing: Timing; error?: string }[];
|
|
skipped: string[];
|
|
summary: ReturnType<typeof timingSummary>;
|
|
}
|
|
|
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
/** Captures the jobs in a browser: per-shot contexts, or one page per language + device in per-language sessions. */
|
|
export async function captureWeb(browser: any, cap: ShotsCapture, jobs: WebJob[], o: { concurrency: number; dims: CaptureDims; auth?: AuthPlan }): Promise<CaptureReport> {
|
|
const auth = o.auth;
|
|
const mask = (t: string): string => (auth ? maskSecrets(t, auth.secrets) : t);
|
|
const expired = new Set<string>();
|
|
const started = Date.now();
|
|
const perLang = cap.session === 'per-language';
|
|
const report: CaptureReport['frames'] = [];
|
|
let finished = 0;
|
|
const total = jobs.length;
|
|
const newPage = async (j: WebJob): Promise<{ c: any; page: any; errs: string[] }> => {
|
|
const d = dimsOf(o.dims, j.device);
|
|
const c = await browser.newContext({ viewport: { width: d.w, height: d.h }, deviceScaleFactor: d.dpr, hasTouch: j.device !== 'landscape', isMobile: j.device !== 'landscape', serviceWorkers: 'block', locale: j.lang, ...(auth?.contextOptions ?? {}) }); // landscape = a desktop window: no touch, no mobile emulation
|
|
// sign in once per context, cookies kept in it (request auth); a failed sign-in stops the whole capture
|
|
if (auth) try { await auth.prepare(c); } catch (e) { await c.close(); throw Object.assign(new Error(mask((e as Error).message)), { authFailed: true }); }
|
|
const page = await c.newPage();
|
|
const errs: string[] = [];
|
|
page.on('pageerror', (e: Error) => errs.push(e.message));
|
|
return { c, page, errs };
|
|
};
|
|
const shoot = async (s: { page: any; errs: string[] }, j: WebJob, how: 'goto' | 'spa', waitTimeout: number): Promise<Timing> => {
|
|
const t: Timing = { goto: 0, waitFor: 0, eval: 0, delay: 0, screenshot: 0, total: 0 };
|
|
let t0 = Date.now();
|
|
if (how === 'goto') await s.page.goto(j.url, { waitUntil: 'domcontentloaded', timeout: 90000 });
|
|
else await s.page.evaluate(`(p => { if (location.pathname + location.search !== p) { history.pushState(null, '', p); dispatchEvent(new PopStateEvent('popstate', { state: null })); } })(${JSON.stringify(j.path)})`);
|
|
t.goto = Date.now() - t0; t0 = Date.now();
|
|
if (auth && landedOnLogin(j.url, s.page.url(), auth.loginUrl)) {
|
|
// a sign-in page is never a store frame: warn once per login URL, fail this shot now (no waitFor timeout)
|
|
if (!expired.has(safeUrl(s.page.url()))) { expired.add(safeUrl(s.page.url())); warn('shots', `${j.lang}/${j.device}/${j.name}: ${mask(expiredHint(s.page.url()))}`); }
|
|
throw new Error(`landed on the sign-in page ${safeUrl(s.page.url())}`);
|
|
}
|
|
await applySelectors(s.page, j.spec, { device: j.device, w: dimsOf(o.dims, j.device).w, h: dimsOf(o.dims, j.device).h }, waitTimeout); // waitSelector, click: a miss names the selector and the size
|
|
if (j.spec.waitFor) await s.page.waitForFunction(j.spec.waitFor, null, { timeout: waitTimeout, polling: 50 });
|
|
t.waitFor = Date.now() - t0; t0 = Date.now();
|
|
for (const js of j.spec.eval ?? []) await s.page.evaluate(js);
|
|
t.eval = Date.now() - t0; t0 = Date.now();
|
|
await s.page.waitForTimeout(j.spec.delayMs ?? 1000);
|
|
t.delay = Date.now() - t0; t0 = Date.now();
|
|
mkdirSync(join(j.out, '..'), { recursive: true });
|
|
await s.page.screenshot({ path: j.out });
|
|
t.screenshot = Date.now() - t0;
|
|
t.total = t.goto + t.waitFor + t.eval + t.delay + t.screenshot;
|
|
return t;
|
|
};
|
|
const record = (j: WebJob, r: { ok: boolean; reused: boolean; timing: Timing; error?: string; errs?: string[] }): void => {
|
|
finished++;
|
|
if (r.error) r.error = mask(r.error);
|
|
if (r.errs) r.errs = r.errs.map(mask);
|
|
report.push({ frame: j.frame, ok: r.ok, reused: r.reused, timing: r.timing, ...(r.error ? { error: r.error } : {}) });
|
|
const head = `${progressLine(finished, total, started)} ${j.lang}/${j.device}/${j.name}`;
|
|
if (r.ok) log('shots', `${head}.png ${r.reused ? '(reused page) ' : ''}${timingText(r.timing)}${r.errs?.length ? ' (page errors: ' + r.errs.join(' | ').slice(0, 200) + ')' : ''}`);
|
|
else warn('shots', `${head}: ${r.error}`);
|
|
};
|
|
const zero = (): Timing => ({ goto: 0, waitFor: 0, eval: 0, delay: 0, screenshot: 0, total: 0 });
|
|
|
|
if (!perLang) {
|
|
await pool(jobs, o.concurrency, async j => {
|
|
const s = await newPage(j);
|
|
try { const timing = await shoot(s, j, 'goto', 120000); record(j, { ok: true, reused: false, timing, errs: s.errs }); }
|
|
catch (e) { record(j, { ok: false, reused: false, timing: zero(), error: (e as Error).message.split('\n')[0]!.slice(0, 300) }); }
|
|
await s.c.close();
|
|
});
|
|
} else {
|
|
// one group per language + device, its shots in order on one booted page
|
|
const groups = new Map<string, WebJob[]>();
|
|
for (const j of jobs) { const k = `${j.lang}/${j.device}`; groups.set(k, [...(groups.get(k) ?? []), j]); }
|
|
await pool([...groups.values()], o.concurrency, async group => {
|
|
let s: Awaited<ReturnType<typeof newPage>> | null = null;
|
|
for (const j of group) {
|
|
const nav = j.spec.navigate ?? 'spa';
|
|
if (s) {
|
|
try {
|
|
const timing = await shoot(s, j, nav === 'reload' ? 'goto' : 'spa', cap.reuseTimeoutMs ?? 15000);
|
|
record(j, { ok: true, reused: true, timing, errs: s.errs });
|
|
continue;
|
|
} catch (e) {
|
|
warn('shots', `${j.lang}/${j.device}/${j.name}: reused page failed (${(e as Error).message.split('\n')[0]!.slice(0, 120)}), fresh page`);
|
|
await s.c.close(); s = null;
|
|
}
|
|
}
|
|
s = await newPage(j);
|
|
try { const timing = await shoot(s, j, 'goto', 120000); record(j, { ok: true, reused: false, timing, errs: s.errs }); }
|
|
catch (e) { record(j, { ok: false, reused: false, timing: zero(), error: (e as Error).message.split('\n')[0]!.slice(0, 300) }); await s.c.close(); s = null; }
|
|
}
|
|
if (s) await s.c.close();
|
|
});
|
|
}
|
|
const order = new Map(jobs.map((j, i) => [j.frame, i] as const));
|
|
report.sort((a, b) => order.get(a.frame)! - order.get(b.frame)!);
|
|
const ok = report.filter(f => f.ok);
|
|
return { at: new Date().toISOString(), concurrency: o.concurrency, session: cap.session ?? 'per-shot', wallMs: Date.now() - started, frames: report, skipped: [],
|
|
summary: timingSummary(ok.map(f => ({ frame: f.frame, timing: f.timing }))) };
|
|
}
|
|
/* eslint-enable @typescript-eslint/no-explicit-any */
|
|
|
|
/** Log lines for the end of a capture: wall time, sum of frame time, slowest frames and steps. */
|
|
export function summaryLines(r: CaptureReport): string[] {
|
|
const s = r.summary;
|
|
const lines = [`${r.frames.filter(f => f.ok).length} frames in ${fmtS(r.wallMs)} wall (${fmtS(s.total)} of frame time, ${r.concurrency} at a time, session ${r.session})`];
|
|
if (s.total) lines.push('time per step: ' + STEPS.map(st => `${st} ${fmtS(s.steps[st])} (${Math.round((100 * s.steps[st]) / s.total)} %)`).join(', '));
|
|
for (const f of s.slowest) lines.push(`slowest: ${f.frame} ${fmtS(f.total)} (mostly ${f.step})`);
|
|
return lines;
|
|
}
|
|
|
|
export function writeCaptureState(root: string, report: CaptureReport, cache: Cache, captured: WebJob[]): void {
|
|
mkdirSync(join(root, '.storebound'), { recursive: true });
|
|
const okFrames = new Set(report.frames.filter(f => f.ok).map(f => f.frame));
|
|
for (const j of captured) if (okFrames.has(j.frame)) cache[j.frame] = j.hash;
|
|
writeFileSync(cachePath(root), JSON.stringify(Object.fromEntries(Object.entries(cache).sort()), null, 1) + '\n');
|
|
writeFileSync(join(root, '.storebound', 'capture-report.json'), JSON.stringify(report, null, 1) + '\n');
|
|
}
|
|
|
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
/** `shots capture --dry --probe`: opens each page (no screenshot), runs the shot's waitFor, and reports per size which
|
|
* selector candidates match. Returns the log lines; `missed` counts shots where no candidate of a list matched. */
|
|
export async function probeSelectors(browser: any, jobs: WebJob[], o: { dims: CaptureDims; auth?: AuthPlan; timeoutMs?: number }): Promise<{ lines: string[]; missed: number }> {
|
|
const lines: string[] = []; let missed = 0;
|
|
for (const j of jobs) {
|
|
const lists = [['waitSelector', candidates(j.spec.waitSelector, j.device)], ...(Array.isArray(j.spec.click) && j.spec.click.some(c => typeof c !== 'string') ? j.spec.click : j.spec.click === undefined ? [] : [j.spec.click])
|
|
.map((c, i) => [`click[${i}]`, candidates(c as any, j.device)] as const)].filter(([, l]) => l.length) as [string, string[]][];
|
|
if (!lists.length) continue;
|
|
const d = dimsOf(o.dims, j.device);
|
|
const c = await browser.newContext({ viewport: { width: d.w, height: d.h }, deviceScaleFactor: 1, hasTouch: j.device !== 'landscape', isMobile: j.device !== 'landscape', serviceWorkers: 'block', locale: j.lang, ...(o.auth?.contextOptions ?? {}) });
|
|
try {
|
|
if (o.auth) await o.auth.prepare(c);
|
|
const page = await c.newPage();
|
|
await page.goto(j.url, { waitUntil: 'domcontentloaded', timeout: 90000 });
|
|
if (j.spec.waitFor) await page.waitForFunction(j.spec.waitFor, null, { timeout: o.timeoutMs ?? 20000, polling: 50 }).catch(() => lines.push(`${j.frame} ${d.w}x${d.h}: waitFor did not become true in ${(o.timeoutMs ?? 20000) / 1000}s`));
|
|
for (const [what, list] of lists) {
|
|
await page.waitForTimeout(500);
|
|
const res = await probe(page, list);
|
|
const hit = res.find(r => r.matches > 0);
|
|
if (!hit) missed++;
|
|
lines.push(`${j.frame} ${j.device} ${d.w}x${d.h} ${what}: ${hit ? `ok via ${hit.selector}` : 'MISS'} (${res.map(r => `${r.selector} -> ${r.matches < 0 ? 'invalid' : r.matches}`).join('; ')})`);
|
|
}
|
|
} catch (e) { missed++; lines.push(`${j.frame}: probe failed: ${(e as Error).message.split('\n')[0]!.slice(0, 200)}`); } finally { await c.close(); }
|
|
}
|
|
return { lines, missed };
|
|
}
|
|
/* eslint-enable @typescript-eslint/no-explicit-any */
|