CRT
b4d924b132
A live run where only the token exchange, edit insert / delete and Reporting :query were let through: - health / validate: errorIssues:search answered 400 "Unsupported timezone" for America/Los_Angeles (it takes hour-aligned UTC only); the failure dropped the crash and ANR rates and the reason was cut off in the query string. Issues are asked in UTC; a failing issues search is a vitals-issues warning; Reporting errors leave out the query. - play listing --dry / play images --dry wrote into the edit (PUT details and listings, image clears and uploads, :validate) before discarding it. Now one throwaway edit is read (details, listings, image sha256) and compared with the local copy, then deleted. - play.read(): one helper for read-only edits (status, next-version-code), always deleted, warns if the delete fails; next-version-code counts distinct codes. - tests on sanitized recorded answers (tests/play-post-reads.test.ts)
109 lines
9.1 KiB
TypeScript
109 lines
9.1 KiB
TypeScript
import { describe, expect, test } from 'bun:test';
|
|
import { mkdtempSync, writeFileSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { previousSnapshot } from '../cli/health';
|
|
import {
|
|
ANR_METRICS, betaCrashCounts, CRASH_METRICS, deviceBreakdown, healthFindings, healthMarkdown, type HealthSnapshot, healthTable, issuesQuery,
|
|
lastValue, latestDailyEnd, metricQuery, parseIssues, parseMetricRows, parsePerfPower, PLAY_THRESHOLDS, weightedAverage, ymd,
|
|
} from '../cli/lib/health';
|
|
import { TOOLS, toArgv } from '../cli/mcp-tools';
|
|
|
|
const row = (day: number, values: Record<string, string>, dims: Record<string, string> = {}) => ({
|
|
startTime: { year: 2026, month: 10, day, timeZone: { id: 'America/Los_Angeles' } },
|
|
dimensions: Object.entries(dims).map(([dimension, stringValue]) => ({ dimension, stringValue })),
|
|
metrics: Object.entries(values).map(([metric, value]) => ({ metric, decimalValue: { value } })),
|
|
});
|
|
|
|
describe('reporting api requests', () => {
|
|
test('DAILY timeline in America/Los_Angeles, exclusive end, the documented metric names', () => {
|
|
const q = metricQuery(CRASH_METRICS, { year: 2026, month: 10, day: 8 }, 7) as { timelineSpec: Record<string, unknown>; metrics: string[]; dimensions?: string[] };
|
|
expect(q.timelineSpec).toEqual({ aggregationPeriod: 'DAILY', startTime: { year: 2026, month: 10, day: 1, timeZone: { id: 'America/Los_Angeles' } }, endTime: { year: 2026, month: 10, day: 8, timeZone: { id: 'America/Los_Angeles' } } });
|
|
expect(q.metrics).toEqual(['userPerceivedCrashRate', 'userPerceivedCrashRate28dUserWeighted', 'crashRate', 'distinctUsers']);
|
|
expect(ANR_METRICS).toContain('userPerceivedAnrRate28dUserWeighted');
|
|
expect(q.dimensions).toBeUndefined();
|
|
expect((metricQuery(['x'], { year: 2026, month: 3, day: 1 }, 1, ['deviceModel']) as { dimensions: string[]; timelineSpec: { startTime: unknown } }).timelineSpec.startTime).toMatchObject({ month: 2, day: 28 });
|
|
expect(ymd(new Date('2026-01-01T10:00:00Z'), 1)).toEqual({ year: 2025, month: 12, day: 31 });
|
|
});
|
|
test('freshness and issue search parameters', () => {
|
|
expect(latestDailyEnd({ freshnessInfo: { freshnesses: [{ aggregationPeriod: 'HOURLY', latestEndTime: { year: 2026, month: 10, day: 9 } }, { aggregationPeriod: 'DAILY', latestEndTime: { year: 2026, month: 10, day: 8 } }] } })).toEqual({ year: 2026, month: 10, day: 8 });
|
|
expect(latestDailyEnd({})).toBeNull();
|
|
const p = new URLSearchParams(issuesQuery({ year: 2026, month: 10, day: 8 }, 7, { type: 'CRASH' }));
|
|
expect(p.get('interval.startTime.day')).toBe('1');
|
|
expect(p.get('interval.endTime.timeZone.id')).toBe('UTC'); // errorIssues:search takes hour-aligned UTC only (400 for LA)
|
|
expect(p.get('filter')).toBe('errorIssueType = CRASH');
|
|
expect(p.get('orderBy')).toBe('errorReportCount desc');
|
|
});
|
|
});
|
|
|
|
describe('parsing', () => {
|
|
const crash = parseMetricRows({ rows: [row(2, { userPerceivedCrashRate: '0.004', distinctUsers: '1000', userPerceivedCrashRate28dUserWeighted: '0.006' }), row(1, { userPerceivedCrashRate: '0.010', distinctUsers: '3000' })] });
|
|
test('rows sorted by date, decimals as numbers, weighted averages', () => {
|
|
expect(crash.map(r => r.date)).toEqual(['2026-10-01', '2026-10-02']);
|
|
expect(lastValue(crash, 'userPerceivedCrashRate28dUserWeighted')).toBe(0.006);
|
|
expect(lastValue(crash, 'nope')).toBeNull();
|
|
expect(weightedAverage(crash, 'userPerceivedCrashRate')).toBeCloseTo((0.01 * 3000 + 0.004 * 1000) / 4000, 10);
|
|
});
|
|
test('devices: weighted per model, small samples dropped, worst first', () => {
|
|
const c = parseMetricRows({ rows: [row(1, { userPerceivedCrashRate: '0.09', distinctUsers: '600' }, { deviceModel: 'acme/old' }), row(1, { userPerceivedCrashRate: '0.5', distinctUsers: '10' }, { deviceModel: 'acme/rare' }),
|
|
row(1, { userPerceivedCrashRate: '0.01', distinctUsers: '5000' }, { deviceModel: 'acme/new' })] });
|
|
const d = deviceBreakdown(c, []);
|
|
expect(d.map(x => x.model)).toEqual(['acme/old', 'acme/new']);
|
|
expect(d[0]!.crash).toBeCloseTo(0.09, 10);
|
|
});
|
|
test('issues, Apple metrics, TestFlight crashes', () => {
|
|
expect(parseIssues({ errorIssues: [{ type: 'APPLICATION_NOT_RESPONDING', cause: 'Input dispatching timed out', location: 'com.example.Main', errorReportCount: '12', distinctUsers: '9', lastAppVersion: { versionCode: '42' } }] }))
|
|
.toEqual([{ type: 'ANR', cause: 'Input dispatching timed out', location: 'com.example.Main', reports: 12, users: 9, lastVersion: '42', uri: undefined }]);
|
|
const perf = parsePerfPower({ productData: [{ metricCategories: [{ identifier: 'HANG', metrics: [{ identifier: 'hangRate', unit: { displayName: 's/hr' },
|
|
datasets: [{ filterCriteria: { device: 'all_iphones', percentile: 'percentile.fifty' }, points: [{ version: '1.0', value: 0.5, goal: 'good' }, { version: '1.1', value: 4.2, goal: 'poor' }] }] }] }] }] });
|
|
expect(perf).toEqual([{ category: 'HANG', metric: 'hangRate', unit: 's/hr', version: '1.1', value: 4.2, goal: 'poor' }]);
|
|
const subs = [{ attributes: { createdDate: '2026-10-05T10:00:00Z' }, relationships: { build: { data: { id: 'b1' } } } }, { attributes: { createdDate: '2026-10-06T10:00:00Z' }, relationships: { build: { data: { id: 'b1' } } } },
|
|
{ attributes: { createdDate: '2026-09-01T10:00:00Z' }, relationships: { build: { data: { id: 'b1' } } } }];
|
|
expect(betaCrashCounts(subs, new Map([['b1', '42']]), '2026-10-01')).toEqual([{ build: '42', count: 2 }]);
|
|
});
|
|
});
|
|
|
|
describe('findings and report', () => {
|
|
const play = (crash: string, anr: string) => ({ window: { start: '2026-10-01', end: '2026-10-08', days: 7 },
|
|
crash: parseMetricRows({ rows: [row(7, { userPerceivedCrashRate: crash, userPerceivedCrashRate28dUserWeighted: crash, distinctUsers: '1000' })] }),
|
|
anr: parseMetricRows({ rows: [row(7, { userPerceivedAnrRate: anr, userPerceivedAnrRate28dUserWeighted: anr, distinctUsers: '1000' })] }), devices: [], issues: [] });
|
|
test('thresholds: over = blocker, from 80 % = warn, under = ok', () => {
|
|
expect(PLAY_THRESHOLDS).toEqual({ crash: 0.0109, anr: 0.0047, perDevice: 0.08 });
|
|
const levels = (h: HealthSnapshot) => Object.fromEntries(healthFindings(h).map(f => [f.code, f.level]));
|
|
expect(levels({ date: 'd', play: play('0.0120', '0.0010') })).toEqual({ 'vitals-crash': 'blocker', 'vitals-anr': 'ok' });
|
|
expect(levels({ date: 'd', play: play('0.0090', '0.0040') })).toEqual({ 'vitals-crash': 'warn', 'vitals-anr': 'warn' });
|
|
const dev = healthFindings({ date: 'd', play: { ...play('0.001', '0.001'), devices: [{ model: 'acme/old', crash: 0.09, users: 600 }] } });
|
|
expect(dev.find(f => f.code === 'vitals-device')).toMatchObject({ level: 'blocker' });
|
|
expect(healthFindings({ date: 'd', play: { error: '403 forbidden' } })[0]).toMatchObject({ level: 'warn', code: 'vitals' });
|
|
expect(healthFindings({ date: 'd', play: { error: '403' } }, { unreadable: 'ok' })[0]!.level).toBe('ok');
|
|
expect(healthFindings({ date: 'd', play: { ...play('0', '0'), crash: [], anr: [] } })[0]!.msg).toContain('no vitals data yet');
|
|
const apple = healthFindings({ date: 'd', apple: { metrics: [{ category: 'HANG', metric: 'hangRate', unit: 's/hr', version: '1.1', value: 4, goal: 'poor' }], signatures: [{ build: '42', type: 'HANGS', signature: 'main thread', weight: 0.4 }], betaCrashes: [] } });
|
|
expect(apple.map(f => f.level)).toEqual(['warn', 'warn']);
|
|
});
|
|
test('table and markdown with week-over-week change', () => {
|
|
const h: HealthSnapshot = { date: '2026-10-09', play: { ...play('0.0120', '0.0010'), issues: [{ type: 'CRASH', cause: 'NullPointerException', location: 'Main.kt', reports: 5, users: 4, lastVersion: '42' }] },
|
|
apple: { metrics: [], signatures: [], betaCrashes: [{ build: '42', count: 2 }] } };
|
|
const t = healthTable(h);
|
|
expect(t).toContain('1.20%');
|
|
expect(t).toContain('TestFlight crash reports');
|
|
const md = healthMarkdown(h, { appName: 'Example Game', previous: { date: '2026-10-02', play: play('0.0100', '0.0010') } });
|
|
expect(md).toContain('# Health report · Example Game · 2026-10-09');
|
|
expect(md).toContain('OVER THRESHOLD');
|
|
expect(md).toContain('+0.20 pt');
|
|
expect(md).toContain('| CRASH | NullPointerException | Main.kt | 5 | 4 | 42 |');
|
|
});
|
|
test('previous snapshot is the newest one before today', () => {
|
|
const dir = mkdtempSync(join(tmpdir(), 'storebound-health-'));
|
|
writeFileSync(join(dir, '2026-09-25.json'), JSON.stringify({ date: '2026-09-25' }));
|
|
writeFileSync(join(dir, '2026-10-02.json'), JSON.stringify({ date: '2026-10-02' }));
|
|
writeFileSync(join(dir, '2026-10-09.json'), JSON.stringify({ date: '2026-10-09' }));
|
|
expect(previousSnapshot(dir, '2026-10-09')!.date).toBe('2026-10-02');
|
|
expect(previousSnapshot(join(dir, 'none'), '2026-10-09')).toBeNull();
|
|
});
|
|
test('mcp: health is read-only, the report is a dry-by-default local write', () => {
|
|
const t = (n: string) => TOOLS.find(x => x.name === n)!;
|
|
expect(t('health')).toMatchObject({ write: false, network: true });
|
|
expect(toArgv(t('health'), { store: 'play', days: 28 }).argv).toEqual(['health', '--play', '--days', '28']);
|
|
expect(toArgv(t('health_report'), {}).argv).toEqual(['health', '--weekly', '--dry']);
|
|
});
|
|
});
|