162 lines
11 KiB
TypeScript
162 lines
11 KiB
TypeScript
// Store analytics snapshots next to the listing changes.
|
|
// storebound analytics setup [--dry] App Store: create the ONGOING analytics report request (Admin key; once)
|
|
// storebound analytics snapshot [--days 28] [--store apple|play] [--dry] -> .storebound/analytics/<date>.json
|
|
// storebound analytics compare [--days 7] [--paths store,storebound.config.ts] [--md]
|
|
// App Store: Analytics Reports API (Discovery and Engagement: impressions, product page views; Downloads: first-time
|
|
// downloads and redownloads), every segment checked against its md5 checksum. Google Play has no statistics API:
|
|
// the monthly CSV exports in your Cloud Storage bucket (play.reportsBucket = "pubsite_prod_rev_<id>", Play Console >
|
|
// Download reports > Statistics > Copy Cloud Storage URI) are read with the service account (store listing visitors,
|
|
// acquisitions, daily user installs). compare lines the snapshots up with the git history of the store files (listing
|
|
// text, screenshots, config): the days before vs after each change. Read-only on both stores except `setup`.
|
|
import { consoleLink } from './lib/console-links';
|
|
import { storeReadError } from './lib/google-errors';
|
|
import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
import { join } from 'node:path';
|
|
import { createOnce } from './lib/upsert';
|
|
import { ascClient, type Asc, type J } from './apple/asc';
|
|
import type { Ctx } from './lib/ctx';
|
|
import {
|
|
type AppleDay, APPLE_REPORTS, appleDaily, appleTotals, bucketName, checksumOk, compareAroundEvents, decodeText, effectMarkdown, effectTable, inWindow,
|
|
latestPerDate, mergeSeries, monthsBetween, snapshotProblem, storeErrors, parseGitLog, parseReport, pickReport, type PlayDay, playDaily, playObjects, playTotals, type Snapshot,
|
|
} from './lib/analytics';
|
|
import { STORAGE_READ_SCOPE } from './lib/google-auth';
|
|
import { done, log, printJson, warn } from './lib/log';
|
|
import { pct, textTable } from './lib/table';
|
|
import { googleClient } from './play/google';
|
|
import * as rt from './lib/rt';
|
|
|
|
const TAG = 'analytics';
|
|
const day = (d: Date) => d.toISOString().slice(0, 10);
|
|
|
|
async function ongoingRequest(c: Asc, appId: string): Promise<J | null> {
|
|
const reqs = await c.all(`/v1/apps/${appId}/analyticsReportRequests?filter[accessType]=ONGOING&limit=50`);
|
|
return reqs.find(r => !r.attributes?.stoppedDueToInactivity) ?? null;
|
|
}
|
|
|
|
async function reportRows(c: Asc, report: J, start: string, tag: string): Promise<Record<string, string>[]> {
|
|
const instances = (await c.all(`/v1/analyticsReports/${report.id}/instances?filter[granularity]=DAILY&limit=200`))
|
|
.filter(i => (i.attributes?.processingDate ?? '') >= start).sort((a, b) => a.attributes.processingDate.localeCompare(b.attributes.processingDate));
|
|
log(TAG, `apple: ${tag}: ${instances.length} daily instance(s) since ${start}`);
|
|
const parts: { processingDate: string; rows: Record<string, string>[] }[] = [];
|
|
for (const inst of instances) {
|
|
const rows: Record<string, string>[] = [];
|
|
for (const seg of await c.all(`/v1/analyticsReportInstances/${inst.id}/segments?limit=200`)) {
|
|
// Segment URLs are pre-signed (valid 5 minutes): fetched without the API token.
|
|
const r = await fetch(seg.attributes.url);
|
|
if (!r.ok) throw new Error(`segment ${seg.id}: HTTP ${r.status}`);
|
|
const gz = new Uint8Array(await r.arrayBuffer());
|
|
const raw = gz[0] === 0x1f && gz[1] === 0x8b ? rt.gunzipSync(gz) : gz;
|
|
// Apple documents an md5 checksum per segment without saying of which bytes: accept the file or its content.
|
|
if (seg.attributes.checksum && !checksumOk(gz, seg.attributes.checksum) && !checksumOk(raw, seg.attributes.checksum)) throw new Error(`segment ${seg.id}: md5 checksum mismatch`);
|
|
rows.push(...parseReport(decodeText(raw)));
|
|
}
|
|
parts.push({ processingDate: inst.attributes.processingDate, rows });
|
|
}
|
|
return latestPerDate(parts);
|
|
}
|
|
|
|
async function appleSnapshot(ctx: Ctx, start: string, end: string): Promise<{ daily: AppleDay[] }> {
|
|
const { config, root } = ctx.loaded;
|
|
const c = await ascClient(root, config.apple!.bundleId, config.apple?.appId);
|
|
const appId = await c.app();
|
|
const req = await ongoingRequest(c, appId);
|
|
if (!req) throw new Error('no active ONGOING analytics report request: run storebound analytics setup (Admin key), data starts 1-2 days later');
|
|
const reports = await c.all(`/v1/analyticsReportRequests/${req.id}/reports?limit=200`);
|
|
const eng = pickReport(reports.filter(r => r.attributes?.category === APPLE_REPORTS.engagement.category), APPLE_REPORTS.engagement.match);
|
|
const dl = pickReport(reports.filter(r => r.attributes?.category === APPLE_REPORTS.downloads.category), APPLE_REPORTS.downloads.match);
|
|
if (!eng || !dl) throw new Error(`reports not found yet (${reports.length} listed): wait a day after setup`);
|
|
const daily = appleDaily(await reportRows(c, eng, start, eng.attributes.name), await reportRows(c, dl, start, dl.attributes.name));
|
|
return { daily: inWindow(daily, start, end) };
|
|
}
|
|
|
|
async function playSnapshot(ctx: Ctx, start: string, end: string): Promise<{ daily: PlayDay[] }> {
|
|
const { config, root } = ctx.loaded;
|
|
const bucket = config.play?.reportsBucket;
|
|
if (!bucket) throw new Error(`play.reportsBucket is not set (${consoleLink('play.reports-statistics', '> Copy Cloud Storage URI')})`);
|
|
const g = googleClient(root, STORAGE_READ_SCOPE);
|
|
const sp: Record<string, string>[] = [], inst: Record<string, string>[] = [];
|
|
for (const month of monthsBetween(start, end)) {
|
|
const o = playObjects(config.play!.packageName, month);
|
|
for (const [name, into] of [[o.storePerformance, sp], [o.installs, inst]] as const) {
|
|
const url = `https://storage.googleapis.com/storage/v1/b/${encodeURIComponent(bucketName(bucket))}/o/${encodeURIComponent(name)}?alt=media`;
|
|
try { into.push(...parseReport(decodeText(await g.bytes(url)))); log(TAG, `play: ${name}`); }
|
|
catch (e) { if (/-> 404/.test(String(e))) warn(TAG, `play: ${name} not there (yet)`); else throw e; }
|
|
}
|
|
}
|
|
return { daily: inWindow(playDaily(sp, inst), start, end) };
|
|
}
|
|
|
|
function readSnapshots(dir: string): Snapshot[] {
|
|
if (!existsSync(dir)) return [];
|
|
return readdirSync(dir).filter(n => /^\d{4}-\d{2}-\d{2}\.json$/.test(n)).sort().map(n => JSON.parse(readFileSync(join(dir, n), 'utf8')) as Snapshot);
|
|
}
|
|
|
|
export async function run(ctx: Ctx): Promise<void> {
|
|
const { config, root, resolve } = ctx.loaded;
|
|
const a = ctx.args;
|
|
const sub = a._[0];
|
|
const dir = resolve(a.get('dir') ?? '.storebound/analytics');
|
|
const dry = a.has('dry');
|
|
|
|
if (sub === 'setup') {
|
|
if (!config.apple) throw new Error('no apple section in the config (Google Play needs no setup: set play.reportsBucket)');
|
|
const c = await ascClient(root, config.apple.bundleId, config.apple.appId);
|
|
const appId = await c.app();
|
|
const cur = await ongoingRequest(c, appId);
|
|
if (cur) { done(TAG, `apple: ONGOING report request exists (${cur.id})`); return; }
|
|
const body = { data: { type: 'analyticsReportRequests', attributes: { accessType: 'ONGOING' }, relationships: { app: { data: { type: 'apps', id: appId } } } } };
|
|
if (dry) { log(TAG, 'would POST /v1/analyticsReportRequests {accessType=ONGOING} (dry)'); return; }
|
|
// one ONGOING request per app: a 409 or an unclear answer is checked by reading it back, never re-sent blindly
|
|
await createOnce({ what: 'ONGOING analytics report request', tag: TAG, unique: true, find: () => ongoingRequest(c, appId),
|
|
create: async () => (await c.asc('POST', '/v1/analyticsReportRequests', body)).data });
|
|
done(TAG, 'apple: ONGOING report request created; the first reports appear after 1-2 days, then daily');
|
|
return;
|
|
}
|
|
|
|
if (sub === 'snapshot') {
|
|
const days = a.num('days', 28);
|
|
if (!(days >= 1 && days <= 120)) throw new Error('--days: 1-120');
|
|
const store = a.get('store');
|
|
const now = new Date();
|
|
const end = day(new Date(now.getTime() - 86_400_000)), start = day(new Date(now.getTime() - days * 86_400_000));
|
|
const snap: Snapshot = { date: day(now), window: { start, end, days } };
|
|
if ((!store || store === 'apple') && config.apple) {
|
|
try { const r = await appleSnapshot(ctx, start, end); snap.apple = { daily: r.daily, totals: appleTotals(r.daily) }; }
|
|
catch (e) { snap.apple = { error: storeReadError((e as Error).message).error.slice(0, 400) }; warn(TAG, `apple: ${snap.apple.error}`); }
|
|
}
|
|
if ((!store || store === 'play') && config.play) {
|
|
try { const r = await playSnapshot(ctx, start, end); snap.play = { daily: r.daily, totals: playTotals(r.daily) }; }
|
|
catch (e) { snap.play = { error: storeReadError((e as Error).message, { app: config.play.packageName }).error.slice(0, 400) }; warn(TAG, `play: ${snap.play.error}`); }
|
|
}
|
|
const rows: (string | number)[][] = [];
|
|
if (snap.apple && !('error' in snap.apple)) { const t = snap.apple.totals; rows.push(['apple', `${t.impressions} impressions`, `${t.pageViews} page views`, `${t.downloads} downloads`, pct(t.conversion ?? null)]); }
|
|
if (snap.play && !('error' in snap.play)) { const t = snap.play.totals; rows.push(['play', `${t.visitors} visitors`, `${t.acquisitions} acquisitions`, `${t.installs} installs`, pct(t.conversion ?? null)]); }
|
|
if (rows.length) console.log(textTable(['store', 'reach', 'interest', 'installs', 'conversion'], rows));
|
|
const file = join(dir, `${snap.date}.json`);
|
|
const problem = snapshotProblem(snap);
|
|
if (problem) throw new Error(`no store data, nothing written (${problem})`);
|
|
if (dry) { done(TAG, `dry run: would write ${file} (${start} to ${end})`); return; }
|
|
mkdirSync(dir, { recursive: true });
|
|
writeFileSync(file, JSON.stringify(snap, null, 2) + '\n');
|
|
done(TAG, `snapshot ${file} (${start} to ${end})`);
|
|
return;
|
|
}
|
|
|
|
if (sub === 'compare') {
|
|
const snaps = readSnapshots(dir);
|
|
if (!snaps.length) throw new Error(`no snapshots in ${dir} (storebound analytics snapshot)`);
|
|
const series = mergeSeries(snaps);
|
|
const first = [series.apple[0]?.date, series.play[0]?.date].filter(Boolean).sort()[0] ?? snaps[0]!.window.start;
|
|
const paths = a.list('paths').length ? a.list('paths') : ['store', 'storebound.config.ts', ...(config.shots?.copy ? [config.shots.copy] : [])];
|
|
const g = rt.spawnSync(['git', 'log', `--since=${first}`, '--date=short', '--format=%x1e%h%x1f%ad%x1f%s', '--name-only', '--', ...paths], { cwd: root, stdout: 'pipe', stderr: 'pipe' });
|
|
if (g.exitCode !== 0) throw new Error(`git log: ${g.stderr.toString().trim()}`);
|
|
const events = parseGitLog(g.stdout.toString()).reverse();
|
|
log(TAG, `${snaps.length} snapshot(s), ${series.apple.length} App Store day(s), ${series.play.length} Play day(s), ${events.length} store change(s) since ${first}`);
|
|
const fx = compareAroundEvents(series, events, a.num('days', 7));
|
|
if (a.has('json')) { printJson({ snapshots: snaps.length, since: first, series, effects: fx, errors: storeErrors(snaps) }); return; }
|
|
console.log(a.has('md') ? effectMarkdown(fx) : effectTable(fx));
|
|
log(TAG, 'reading: a change needs several days on both sides and enough traffic; seasonality, featuring and ads move these numbers too');
|
|
return;
|
|
}
|
|
throw new Error('usage: storebound analytics setup | snapshot | compare');
|
|
}
|