// Web capture (shots.capture.source "web"), parallel and incremental: // --concurrency N browser contexts in one headless Chromium (default shots.capture.concurrency, else from the CPU // count: a WebGL game is GPU-bound, so capture defaults lower than render). Files do not depend on // the order: every frame has its own path. // session shots.capture.session "per-shot" (default: a new context + page and a full boot per frame) or // "per-language": one page per language and device, booted once; the next shots switch scenes in // that page (navigate "spa": history.pushState + popstate to the shot's path, then waitFor / eval; // navigate "reload": page.goto in the same page). A precondition that fails there (waitFor times out // after reuseTimeoutMs) falls back to a fresh page. // cache .storebound/capture-cache.json: per frame a hash of its inputs (shot entry, URL + query, frame // size, session, the copy text of its language, the app build (shots.capture.build files, else // the HTML and scripts at baseUrl), fonts.css). Unchanged frames that exist are skipped; // --only-missing only captures frames that do not exist; --force captures everything. // timing per frame: goto, waitFor, eval, delay, screenshot, total, in the progress line and in // .storebound/capture-report.json, with the slowest frames and steps at the end. import { createHash } from 'node:crypto'; import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; import { availableParallelism, cpus } from 'node:os'; import { join } from 'node:path'; import type { ShotSpec, ShotsCapture } from '../lib/config'; import { log, warn } from '../lib/log'; import { shotsFor } from './devices'; import type { FrameDims } from './lib'; type Device = 'phone' | 'tablet'; export const STEPS = ['goto', 'waitFor', 'eval', 'delay', 'screenshot'] as const; export type Step = typeof STEPS[number]; export type Timing = Record; /** frame = path relative to shots.frames (cache and report key), out = absolute file. */ export interface WebJob { lang: string; device: Device; name: string; spec: ShotSpec; url: string; path: string; out: string; frame: string; hash: string } /** CPU cores (availableParallelism where Bun has it). */ export const coreCount = (): number => { try { return availableParallelism(); } catch { return cpus().length || 2; } }; /** Default parallel workers: render min(4, cores / 2); capture (GPU-heavy WebGL / game boots) min(3, cores / 4). */ export function defaultConcurrency(kind: 'render' | 'capture', cores = coreCount()): number { return Math.max(1, kind === 'render' ? Math.min(4, Math.floor(cores / 2)) : Math.min(3, Math.floor(cores / 4))); } /** Runs fn over items with at most n at a time; results keep the input order. */ export async function pool(items: T[], n: number, fn: (item: T, index: number) => Promise): Promise { const out = new Array(items.length); let next = 0; const worker = async (): Promise => { while (next < items.length) { const i = next++; out[i] = await fn(items[i]!, i); } }; await Promise.all(Array.from({ length: Math.max(1, Math.min(n, items.length)) }, worker)); return out; } const fmtS = (ms: number): string => (ms >= 60_000 ? `${Math.floor(ms / 60_000)}m${String(Math.round((ms % 60_000) / 1000)).padStart(2, '0')}s` : `${(ms / 1000).toFixed(1)}s`); /** "n/total, ETA" progress for parallel work: ETA = remaining items x average wall time per finished item. */ export function progressLine(done: number, total: number, startedMs: number, nowMs = Date.now()): string { const w = String(total).length; const eta = done ? ((nowMs - startedMs) / done) * (total - done) : 0; return `${String(done).padStart(w)}/${total}${done < total && done ? ` ETA ${fmtS(eta)}` : ''}`; } export const timingText = (t: Timing): string => `goto ${fmtS(t.goto)} wait ${fmtS(t.waitFor)} eval ${fmtS(t.eval)} delay ${fmtS(t.delay)} shot ${fmtS(t.screenshot)} = ${fmtS(t.total)}`; /** Slowest frames and the time per step over all frames (for the end of the log and the report). */ export function timingSummary(frames: { frame: string; timing: Timing }[], top = 5): { slowest: { frame: string; total: number; step: Step }[]; steps: Record; total: number } { const steps = Object.fromEntries(STEPS.map(s => [s, 0])) as Record; for (const f of frames) for (const s of STEPS) steps[s] += f.timing[s]; const slowest = [...frames].sort((a, b) => b.timing.total - a.timing.total).slice(0, top) .map(f => ({ frame: f.frame, total: f.timing.total, step: STEPS.reduce((m, s) => (f.timing[s] > f.timing[m] ? s : m), STEPS[0] as Step) })); return { slowest, steps, total: frames.reduce((t, f) => t + f.timing.total, 0) }; } const sha = (s: string | Uint8Array): string => createHash('sha256').update(s).digest('hex').slice(0, 16); /** Content hash of build files (a file or folder; node_modules / dot folders skipped): paths, sizes and mtimes. */ export function hashPaths(paths: string[]): string { const h = createHash('sha256'); const walk = (p: string): void => { if (!existsSync(p)) { h.update(`missing:${p}`); return; } const st = statSync(p); if (st.isDirectory()) { for (const e of readdirSync(p).sort()) if (e !== 'node_modules' && !e.startsWith('.')) walk(join(p, e)); return; } h.update(`${p}:${st.size}:${st.mtimeMs}\n`); }; for (const p of paths) walk(p); return h.digest('hex').slice(0, 16); } /** App build fingerprint from the running server: the HTML at baseUrl and the scripts it loads (dev servers too). */ export async function hashServed(baseUrl: string): Promise { try { const html = await (await fetch(baseUrl, { signal: AbortSignal.timeout(10_000) })).text(); const h = createHash('sha256').update(html); for (const m of html.matchAll(/]+src=["']([^"']+)["']/g)) { try { h.update(await (await fetch(new URL(m[1]!, baseUrl), { signal: AbortSignal.timeout(10_000) })).text()); } catch { h.update(`unreachable:${m[1]}`); } } return h.digest('hex').slice(0, 16); } catch { return 'unreachable'; } } /** Every web frame to capture, in a fixed order (lang, device, shot), with its URL and its input hash. */ export function planWeb(cap: ShotsCapture, o: { langs: string[]; devices: Device[]; only: string[]; dims: Record; framesDir: string; build: string; copyText?: (lang: string) => string; fonts?: string }): WebJob[] { const jobs: WebJob[] = []; for (const lang of o.langs) for (const device of o.devices) for (const [name, spec] of shotsFor(cap, device, o.only)) { const q = (cap.query ?? '').replaceAll('{lang}', lang); const path = (spec.path ?? '/').replaceAll('{lang}', lang) + (q ? ((spec.path ?? '/').includes('?') ? '&' : '?') + q : ''); const url = (cap.baseUrl ?? '').replace(/\/$/, '') + path; const hash = sha(JSON.stringify({ spec, url, dims: o.dims[device], session: cap.session ?? 'per-shot', build: o.build, copy: o.copyText?.(lang) ?? '', fonts: o.fonts ?? '' })); const frame = `${lang}/${device}/${name}.png`; jobs.push({ lang, device, name, spec, url, path, out: join(o.framesDir, frame), frame, hash }); } return jobs; } export type Cache = Record; export const cachePath = (root: string): string => join(root, '.storebound', 'capture-cache.json'); export function readCache(root: string): Cache { try { return JSON.parse(readFileSync(cachePath(root), 'utf8')) as Cache; } catch { return {}; } } /** Which jobs to run: --force all; --only-missing the ones whose file is missing; else missing or changed inputs. */ export function selectJobs(jobs: WebJob[], cache: Cache, mode: 'force' | 'only-missing' | 'changed', exists: (file: string) => boolean = existsSync): { run: WebJob[]; skipped: WebJob[] } { const run: WebJob[] = [], skipped: WebJob[] = []; for (const j of jobs) { const keep = mode === 'force' ? false : mode === 'only-missing' ? exists(j.out) : exists(j.out) && cache[j.frame] === j.hash; (keep ? skipped : run).push(j); } return { run, skipped }; } export interface CaptureReport { at: string; concurrency: number; session: string; wallMs: number; frames: { frame: string; ok: boolean; reused: boolean; timing: Timing; error?: string }[]; skipped: string[]; summary: ReturnType; } /* eslint-disable @typescript-eslint/no-explicit-any */ /** Captures the jobs in a browser: per-shot contexts, or one page per language + device in per-language sessions. */ export async function captureWeb(browser: any, cap: ShotsCapture, jobs: WebJob[], o: { concurrency: number; dims: Record }): Promise { const started = Date.now(); const perLang = cap.session === 'per-language'; const report: CaptureReport['frames'] = []; let finished = 0; const total = jobs.length; const newPage = async (j: WebJob): Promise<{ c: any; page: any; errs: string[] }> => { const d = o.dims[j.device]; const c = await browser.newContext({ viewport: { width: d.w, height: d.h }, deviceScaleFactor: d.dpr, hasTouch: true, isMobile: true, serviceWorkers: 'block', locale: j.lang }); const page = await c.newPage(); const errs: string[] = []; page.on('pageerror', (e: Error) => errs.push(e.message)); return { c, page, errs }; }; const shoot = async (s: { page: any; errs: string[] }, j: WebJob, how: 'goto' | 'spa', waitTimeout: number): Promise => { const t: Timing = { goto: 0, waitFor: 0, eval: 0, delay: 0, screenshot: 0, total: 0 }; let t0 = Date.now(); if (how === 'goto') await s.page.goto(j.url, { waitUntil: 'domcontentloaded', timeout: 90000 }); else await s.page.evaluate(`(p => { if (location.pathname + location.search !== p) { history.pushState(null, '', p); dispatchEvent(new PopStateEvent('popstate', { state: null })); } })(${JSON.stringify(j.path)})`); t.goto = Date.now() - t0; t0 = Date.now(); if (j.spec.waitFor) await s.page.waitForFunction(j.spec.waitFor, null, { timeout: waitTimeout, polling: 50 }); t.waitFor = Date.now() - t0; t0 = Date.now(); for (const js of j.spec.eval ?? []) await s.page.evaluate(js); t.eval = Date.now() - t0; t0 = Date.now(); await s.page.waitForTimeout(j.spec.delayMs ?? 1000); t.delay = Date.now() - t0; t0 = Date.now(); mkdirSync(join(j.out, '..'), { recursive: true }); await s.page.screenshot({ path: j.out }); t.screenshot = Date.now() - t0; t.total = t.goto + t.waitFor + t.eval + t.delay + t.screenshot; return t; }; const record = (j: WebJob, r: { ok: boolean; reused: boolean; timing: Timing; error?: string; errs?: string[] }): void => { finished++; report.push({ frame: j.frame, ok: r.ok, reused: r.reused, timing: r.timing, ...(r.error ? { error: r.error } : {}) }); const head = `${progressLine(finished, total, started)} ${j.lang}/${j.device}/${j.name}`; if (r.ok) log('shots', `${head}.png ${r.reused ? '(reused page) ' : ''}${timingText(r.timing)}${r.errs?.length ? ' (page errors: ' + r.errs.join(' | ').slice(0, 200) + ')' : ''}`); else warn('shots', `${head}: ${r.error}`); }; const zero = (): Timing => ({ goto: 0, waitFor: 0, eval: 0, delay: 0, screenshot: 0, total: 0 }); if (!perLang) { await pool(jobs, o.concurrency, async j => { const s = await newPage(j); try { const timing = await shoot(s, j, 'goto', 120000); record(j, { ok: true, reused: false, timing, errs: s.errs }); } catch (e) { record(j, { ok: false, reused: false, timing: zero(), error: (e as Error).message.split('\n')[0]!.slice(0, 300) }); } await s.c.close(); }); } else { // one group per language + device, its shots in order on one booted page const groups = new Map(); for (const j of jobs) { const k = `${j.lang}/${j.device}`; groups.set(k, [...(groups.get(k) ?? []), j]); } await pool([...groups.values()], o.concurrency, async group => { let s: Awaited> | null = null; for (const j of group) { const nav = j.spec.navigate ?? 'spa'; if (s) { try { const timing = await shoot(s, j, nav === 'reload' ? 'goto' : 'spa', cap.reuseTimeoutMs ?? 15000); record(j, { ok: true, reused: true, timing, errs: s.errs }); continue; } catch (e) { warn('shots', `${j.lang}/${j.device}/${j.name}: reused page failed (${(e as Error).message.split('\n')[0]!.slice(0, 120)}), fresh page`); await s.c.close(); s = null; } } s = await newPage(j); try { const timing = await shoot(s, j, 'goto', 120000); record(j, { ok: true, reused: false, timing, errs: s.errs }); } catch (e) { record(j, { ok: false, reused: false, timing: zero(), error: (e as Error).message.split('\n')[0]!.slice(0, 300) }); await s.c.close(); s = null; } } if (s) await s.c.close(); }); } const order = new Map(jobs.map((j, i) => [j.frame, i] as const)); report.sort((a, b) => order.get(a.frame)! - order.get(b.frame)!); const ok = report.filter(f => f.ok); return { at: new Date().toISOString(), concurrency: o.concurrency, session: cap.session ?? 'per-shot', wallMs: Date.now() - started, frames: report, skipped: [], summary: timingSummary(ok.map(f => ({ frame: f.frame, timing: f.timing }))) }; } /* eslint-enable @typescript-eslint/no-explicit-any */ /** Log lines for the end of a capture: wall time, sum of frame time, slowest frames and steps. */ export function summaryLines(r: CaptureReport): string[] { const s = r.summary; const lines = [`${r.frames.filter(f => f.ok).length} frames in ${fmtS(r.wallMs)} wall (${fmtS(s.total)} of frame time, ${r.concurrency} at a time, session ${r.session})`]; if (s.total) lines.push('time per step: ' + STEPS.map(st => `${st} ${fmtS(s.steps[st])} (${Math.round((100 * s.steps[st]) / s.total)} %)`).join(', ')); for (const f of s.slowest) lines.push(`slowest: ${f.frame} ${fmtS(f.total)} (mostly ${f.step})`); return lines; } export function writeCaptureState(root: string, report: CaptureReport, cache: Cache, captured: WebJob[]): void { mkdirSync(join(root, '.storebound'), { recursive: true }); const okFrames = new Set(report.frames.filter(f => f.ok).map(f => f.frame)); for (const j of captured) if (okFrames.has(j.frame)) cache[j.frame] = j.hash; writeFileSync(cachePath(root), JSON.stringify(Object.fromEntries(Object.entries(cache).sort()), null, 1) + '\n'); writeFileSync(join(root, '.storebound', 'capture-report.json'), JSON.stringify(report, null, 1) + '\n'); }