CRT/storebound
1
0
Fork 0
Code Issues Pull requests Releases 1 Packages 1 Activity Actions

shots record (web): frames at device pixels through Page.captureScreenshot (clip scale = dpr, long side capped at 1920, 3 requests in flight, midpoint timestamps into the ffmpeg concat); Chromium's screencast stayed at CSS size. Warnings for frames smaller than asked and under 20 frames/s; record-web browser test, pitfall 85

Browse source
This commit is contained in:
CRTkafa 2026-10-21 03:59:39 +07:00
parent 2d7cf6ec2b
commit dda21d2570
10 changed files with 144 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cli/mcp-tools.ts
View file

@ -239,7 +239,7 @@ export const TOOLS: ToolDef[] = [
description: 'Write <renders>/index.html: one self-contained page with every language, size, scene, variant and app preview, store check issues highlighted. Open it in a browser to review before uploading.',
props: { lang }, args: i => opt('lang', i.lang) },
{ name: 'shots_record', title: 'Record app or gameplay clips', command: ['shots', 'record'], write: true, network: false, idempotent: true,
description: 'App or gameplay clips for app previews from shots.video.record: web build (Chromium screencast), iOS Simulator (simctl recordVideo), Android emulator / device (screenrecord, never booted) or a folder listing. Local files only.',
description: 'App or gameplay clips for app previews from shots.video.record: web build (Chromium frames at device pixels), iOS Simulator (simctl recordVideo), Android emulator / device (screenrecord, never booted) or a folder listing. Local files only.',
props: { lang, only: { type: 'string', description: 'Comma-separated clip names' }, source },
args: i => [...opt('lang', i.lang), ...opt('only', i.only), ...opt('source', i.source)] },
{ name: 'shots_video', title: 'Cut app previews', command: ['shots', 'video'], write: true, network: false, idempotent: true,

26
cli/shots/devices.ts
View file

@ -207,7 +207,7 @@ export const trimArgs = (inp: string, out: string, start: number, seconds: numbe
export interface ScreencastFrame { file: string; t: number }
/** ffmpeg concat-demuxer list for screencast frames (Chromium sends a frame only when the page changes): each frame lasts
/** ffmpeg concat-demuxer list for timestamped frames (`t` in seconds, frames come at an uneven rate): each frame lasts
* until the next one; the last one fills up to `seconds`. Pure. */
export function concatList(frames: ScreencastFrame[], seconds: number): string {
const t0 = frames[0]?.t ?? 0;
@ -226,11 +226,25 @@ export const concatArgs = (list: string, out: string, seconds: number, fps = 30)
['-v', 'error', '-y', '-f', 'concat', '-safe', '0', '-i', list, '-t', seconds.toFixed(3),
'-vf', `fps=${fps},scale=trunc(iw/2)*2:trunc(ih/2)*2,format=yuv420p`, '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '16', '-an', '-movflags', '+faststart', out];
/** Screencast size for a frame: the frame at 2x (capped at 1600 px on the long side), even numbers. */
export function recordSize(d: { w: number; h: number }): { width: number; height: number } {
const k = Math.min(2, 1600 / Math.max(d.w, d.h));
const even = (n: number) => Math.max(2, Math.round(n * k / 2) * 2);
return { width: even(d.w), height: even(d.h) };
/** Long side cap of a web recording: the tallest store video (886 x 1920 iPhone preview, 1080 x 1920 Play promo), so
* `shots video` only scales down; a 3x phone (2706 px) or a 13" iPad (2752 px) would cost capture speed for pixels
* every output throws away. */
export const RECORD_MAX_SIDE = 1920;
/** Web capture for a frame: CDP `Page.captureScreenshot` with clip = the viewport in CSS px and `scale` = the frame's
* dpr (capped at RECORD_MAX_SIDE on the long side). Chromium returns CSS size x clip.scale whatever the context's
* deviceScaleFactor (checked on Chromium 153); its screencast stays at CSS size (maxWidth only shrinks), which is why
* the recorder does not use it. width / height = the expected image size. Pure. */
export function recordSize(d: { w: number; h: number; dpr?: number }): { width: number; height: number; scale: number } {
const scale = Math.min(d.dpr ?? 2, RECORD_MAX_SIDE / Math.max(d.w, d.h));
return { width: Math.round(d.w * scale), height: Math.round(d.h * scale), scale };
}
/** Guardrail: a first frame smaller than requested means the browser ignored the capture scale (the clip would be an
* upscale in `shots video`). Message or null. Pure. */
export function captureShortfall(got: { w: number; h: number }, want: { width: number; height: number }): string | null {
if (got.w >= want.width - 2 && got.h >= want.height - 2) return null;
return `frames are ${got.w}x${got.h}, asked for ${want.width}x${want.height}: this browser ignores the capture scale, so the clip is upscaled later (set STOREBOUND_CHROME to a current Chromium or record with --source folder)`;
}
// ---- frames ----

48
cli/shots/record.ts
View file

@ -3,8 +3,10 @@
// Config: shots.video.record { source?, device?, clips: { <name>: { seconds, path | url | args | extras, waitFor?, delayMs? } } };
// the URL / simulator / emulator settings are the ones of shots.capture. Out: <shots.video.clips>/<lang>/NN-<name>.mp4
// (NN = config order, so `shots video` joins them in that order): H.264, 30 fps, no audio.
// web Chromium screencast of the page through Playwright (JPEG frames at up to 2x the frame size, joined by
// your ffmpeg; no Playwright ffmpeg download needed). Filming starts when waitFor turns true (+ delayMs).
// web JPEG frames of the page in Chromium through Playwright (CDP Page.captureScreenshot at the frame's device
// pixels, long side capped at 1920; 3 requests in flight, ~45-55 frames/s on an M1 Pro), each stamped
// with the middle of its request and joined by your ffmpeg at their real durations (no Playwright ffmpeg
// download needed). Not the screencast: it stays at CSS size. Filming starts when waitFor turns true (+ delayMs).
// ios-sim `xcrun simctl io <udid> recordVideo` while the app runs (launch args with -AppleLanguages, deep link),
// stopped with SIGINT after `seconds`. Boots the simulator if needed (shut down again afterwards unless
// --keep-booted), status bar 9:41, one warm-up launch after a cold boot (shots.capture.ios.warmupMs).
@ -13,7 +15,8 @@
// folder your own clips (screen recorder, engine capture, OBS): lists <clips>/<lang>/ or <clips>/ with durations.
// Make the app or game run hands-free for this (an autoplay / attract mode switch) or record yourself playing with
// --source folder. Apple: previews must show footage captured from the app itself.
import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { spawnSync } from 'node:child_process';
import type { ClipSpec, VideoRecord } from '../lib/config';
@ -23,9 +26,10 @@ import { done, log, step, warn } from '../lib/log';
import { launch } from './browser';
import { adbPath, localeOf, requireTool, sh } from './capture';
import { ADB_REMOTE_CLIP, adbAppLocaleArgs, adbDemoExitArgs, adbDemoModeArgs, adbForceStopArgs, adbPullArgs, adbRecordArgs, adbRmArgs, adbStartArgs, adbStateArgs,
adbTarget, clipName, fill, pickSimulator, recordSize, simBootArgs, simBootStatusArgs, simLaunchArgs, simOpenUrlArgs, simRecordArgs, simShutdownArgs,
adbTarget, captureShortfall, clipName, fill, pickSimulator, recordSize, simBootArgs, simBootStatusArgs, simLaunchArgs, simOpenUrlArgs, simRecordArgs, simShutdownArgs,
simStatusBarArgs, simStatusBarClearArgs, simTarget, SOURCES, type Source, trimArgs, validateRecord, concatArgs, concatList, type ScreencastFrame } from './devices';
import { frameDims } from './lib';
import { imageInfo } from './spec';
import { CLIP_RE, clipFiles } from './video';
import * as rt from '../lib/rt';
@ -34,6 +38,9 @@ export function clipsToRecord(r: VideoRecord, only: string[] = []): [number, str
return Object.entries(r.clips ?? {}).map(([n, c], i) => [i, n, c] as [number, string, ClipSpec]).filter(([, n]) => !only.length || only.includes(n));
}
/** Capture requests kept in flight for a web clip. */
const IN_FLIGHT = 3;
const probeDuration = (file: string): number => {
const r = spawnSync('ffprobe', ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=nw=1:nk=1', file]);
return Number(r.stdout?.toString().trim()) || 0;
@ -109,27 +116,36 @@ export async function run(ctx: Ctx): Promise<void> {
if (cl.waitFor) await page.waitForFunction(cl.waitFor, null, { timeout: 120000, polling: 50 });
for (const js of cl.eval ?? []) await page.evaluate(js);
await page.waitForTimeout(cl.delayMs ?? 1000);
// Chromium's screencast (what Playwright's own recorder uses), straight to JPEG frames + system ffmpeg
// captureScreenshot, not the screencast (its frames stay at CSS size); a few requests in flight keep the
// rate near the page's own (one at a time reaches ~20 frames/s)
const cdp = await context.newCDPSession(page);
const clip = { x: 0, y: 0, width: d.w, height: d.h, scale: size.scale };
const frames: ScreencastFrame[] = [];
let n = 0;
cdp.on('Page.screencastFrame', (f: { data: string; sessionId: number; metadata: { timestamp: number } }) => {
const file = join(dir, `f${String(++n).padStart(5, '0')}.jpg`);
writeFileSync(file, Buffer.from(f.data, 'base64'));
frames.push({ file, t: f.metadata.timestamp });
cdp.send('Page.screencastFrameAck', { sessionId: f.sessionId }).catch(() => {});
});
await cdp.send('Page.startScreencast', { format: 'jpeg', quality: 92, maxWidth: size.width, maxHeight: size.height, everyNthFrame: 1 });
await page.waitForTimeout(cl.seconds * 1000 + 300);
await cdp.send('Page.stopScreencast');
const end = performance.now() + cl.seconds * 1000 + 300;
const grab = async () => {
while (performance.now() < end) {
const t0 = performance.now();
const r = await cdp.send('Page.captureScreenshot', { format: 'jpeg', quality: 92, optimizeForSpeed: true, clip }) as { data: string };
const file = join(dir, `f${String(++n).padStart(5, '0')}.jpg`);
frames.push({ file, t: (t0 + performance.now()) / 2000 });
await writeFile(file, Buffer.from(r.data, 'base64'));
}
};
await Promise.all(Array.from({ length: IN_FLIGHT }, grab));
await context.close();
if (frames.length < 2) throw new Error(`the screencast delivered ${frames.length} frames (is the page animating?)`);
if (frames.length < 2) throw new Error(`the capture delivered ${frames.length} frames`);
frames.sort((x, y) => x.t - y.t);
const short = captureShortfall(imageInfo(readFileSync(frames[0]!.file)), size);
if (short) warn('shots', `${lang}/${name}: ${short}`);
const rate = frames.length / (frames.at(-1)!.t - frames[0]!.t || 1);
if (rate < 20) warn('shots', `${lang}/${name}: only ${rate.toFixed(0)} frames/s captured (the clip will stutter): close heavy apps, or record with --source folder`);
const list = join(dir, 'frames.txt');
writeFileSync(list, concatList(frames, cl.seconds));
sh('ffmpeg', concatArgs(list, outFor(lang, idx, name), cl.seconds), { timeout: 600_000 });
rmSync(dir, { recursive: true, force: true });
made++;
progress(`${lang}/${clipName(idx, name)} (${cl.seconds} s, ${frames.length} frames)`);
progress(`${lang}/${clipName(idx, name)} (${cl.seconds} s, ${size.width}x${size.height}, ${frames.length} frames, ${rate.toFixed(0)}/s)`);
} catch (e) { failed++; i++; warn('shots', `${lang}/${name}: ${(e as Error).message.slice(0, 300)}`); await context.close().catch(() => {}); }
}
} finally { await browser.close(); }