210 lines
13 KiB
TypeScript
210 lines
13 KiB
TypeScript
// Store analytics snapshots (impressions, product page views, downloads / installs, conversion) and their comparison
|
|
// with listing changes, as pure parts. `storebound analytics` downloads the data (read-only) and writes the snapshots.
|
|
// Sources (checked October 2026):
|
|
// App Store Connect Analytics Reports API: an ONGOING analyticsReportRequest (Admin creates it once; first data after
|
|
// 1-2 days, then daily; instances kept 35 days; a request stops after long inactivity: stoppedDueToInactivity),
|
|
// reports -> instances (DAILY) -> segments (url valid 5 minutes, md5 checksum, gzip text files; every segment of
|
|
// an instance is needed). A later processingDate replaces rows of the same Date: never sum one Date across instances.
|
|
// https://developer.apple.com/documentation/appstoreconnectapi/downloading-analytics-reports
|
|
// https://developer.apple.com/documentation/analytics-reports/app-store-discovery-and-engagement
|
|
// https://developer.apple.com/documentation/analytics-reports/app-download
|
|
// Google Play: no API for store listing statistics (the Reporting API has vitals only). Monthly CSV exports in the
|
|
// developer's Cloud Storage bucket gs://pubsite_prod_rev_<id> (Play Console > Download reports > Statistics, "Copy
|
|
// Cloud Storage URI"), UTF-16, read with the service account (Play Console permission "View app information and
|
|
// download bulk reports", scope devstorage.read_only):
|
|
// stats/store_performance/store_performance_<package>_<yyyyMM>_country.csv (Store listing acquisitions / visitors)
|
|
// stats/installs/installs_<package>_<yyyyMM>_country.csv (Daily User Installs, ...)
|
|
// https://support.google.com/googleplay/android-developer/answer/6135870
|
|
import { createHash } from 'node:crypto';
|
|
import { parseCsvRecords } from './csv';
|
|
import { mdTable, pct, textTable } from './table';
|
|
|
|
export const APPLE_REPORTS = {
|
|
engagement: { category: 'APP_STORE_ENGAGEMENT', match: /discovery and engagement/i },
|
|
downloads: { category: 'COMMERCE', match: /download/i },
|
|
} as const;
|
|
|
|
/** Standard over Detailed (Detailed adds noise for privacy and splits by campaign). */
|
|
export function pickReport<T extends { attributes?: { name?: string } }>(reports: T[], match: RegExp): T | null {
|
|
const hits = reports.filter(r => match.test(r.attributes?.name ?? ''));
|
|
return hits.find(r => /standard/i.test(r.attributes?.name ?? '')) ?? hits[0] ?? null;
|
|
}
|
|
|
|
/** Bytes of a downloaded segment vs its md5 checksum. */
|
|
export const checksumOk = (b: Uint8Array, checksum: string): boolean => createHash('md5').update(b).digest('hex').toLowerCase() === checksum.toLowerCase();
|
|
|
|
/** Text of a report file: UTF-16 with a BOM (Play exports), else UTF-8. */
|
|
export function decodeText(b: Uint8Array): string {
|
|
if (b[0] === 0xff && b[1] === 0xfe) return new TextDecoder('utf-16le').decode(b.subarray(2));
|
|
if (b[0] === 0xfe && b[1] === 0xff) return new TextDecoder('utf-16be').decode(b.subarray(2));
|
|
return new TextDecoder('utf-8').decode(b);
|
|
}
|
|
|
|
/** Tab- or comma-separated report text -> records keyed by lower-case column name (by name: Apple may reorder). */
|
|
export function parseReport(text: string): Record<string, string>[] {
|
|
const first = text.replace(/^/, '').split(/\r?\n/, 1)[0] ?? '';
|
|
return parseCsvRecords(text, first.includes('\t') ? '\t' : ',');
|
|
}
|
|
|
|
/** Rows of several instances: per Date only the rows of the newest processingDate (later instances restate days). */
|
|
export function latestPerDate(instances: { processingDate: string; rows: Record<string, string>[] }[]): Record<string, string>[] {
|
|
const best = new Map<string, string>();
|
|
for (const i of instances) for (const r of i.rows) { const d = r.date ?? ''; if (!best.has(d) || best.get(d)! < i.processingDate) best.set(d, i.processingDate); }
|
|
return instances.flatMap(i => i.rows.filter(r => best.get(r.date ?? '') === i.processingDate));
|
|
}
|
|
|
|
export interface AppleDay { date: string; impressions: number; uniqueImpressions: number; pageViews: number; uniquePageViews: number; downloads: number; firstTimeDownloads: number }
|
|
export interface PlayDay { date: string; visitors: number; acquisitions: number; installs: number }
|
|
|
|
const num = (v: string | undefined) => { const n = Number((v ?? '').replace(/,/g, '')); return Number.isFinite(n) ? n : 0; };
|
|
|
|
/** Discovery and Engagement + Downloads rows -> one line per day (App Store, every territory and source summed). */
|
|
export function appleDaily(engagement: Record<string, string>[], downloads: Record<string, string>[]): AppleDay[] {
|
|
const days = new Map<string, AppleDay>();
|
|
const day = (d: string) => days.get(d) ?? days.set(d, { date: d, impressions: 0, uniqueImpressions: 0, pageViews: 0, uniquePageViews: 0, downloads: 0, firstTimeDownloads: 0 }).get(d)!;
|
|
for (const r of engagement) {
|
|
if (!r.date) continue;
|
|
const ev = (r.event ?? '').toLowerCase();
|
|
const x = day(r.date);
|
|
if (ev === 'impression') { x.impressions += num(r.counts); x.uniqueImpressions += num(r.unique_counts); }
|
|
else if (ev === 'page view') { x.pageViews += num(r.counts); x.uniquePageViews += num(r.unique_counts); }
|
|
}
|
|
for (const r of downloads) {
|
|
if (!r.date) continue;
|
|
const t = (r.download_type ?? '').toLowerCase();
|
|
const x = day(r.date);
|
|
if (t === 'first-time download') { x.firstTimeDownloads += num(r.counts); x.downloads += num(r.counts); }
|
|
else if (t === 'redownload') x.downloads += num(r.counts);
|
|
}
|
|
return [...days.values()].sort((a, b) => a.date.localeCompare(b.date));
|
|
}
|
|
|
|
/** Play store performance + installs CSV rows (country files) -> one line per day. Dates are YYYY-MM-DD. */
|
|
export function playDaily(storePerformance: Record<string, string>[], installs: Record<string, string>[]): PlayDay[] {
|
|
const days = new Map<string, PlayDay>();
|
|
const day = (d: string) => days.get(d) ?? days.set(d, { date: d, visitors: 0, acquisitions: 0, installs: 0 }).get(d)!;
|
|
for (const r of storePerformance) if (r.date) { const x = day(r.date); x.visitors += num(r.store_listing_visitors); x.acquisitions += num(r.store_listing_acquisitions); }
|
|
for (const r of installs) if (r.date) day(r.date).installs += num(r.daily_user_installs);
|
|
return [...days.values()].sort((a, b) => a.date.localeCompare(b.date));
|
|
}
|
|
|
|
/** Months (yyyyMM) a date window touches, for the Play file names. */
|
|
export function monthsBetween(start: string, end: string): string[] {
|
|
const out: string[] = [];
|
|
let y = Number(start.slice(0, 4)), m = Number(start.slice(5, 7));
|
|
const ey = Number(end.slice(0, 4)), em = Number(end.slice(5, 7));
|
|
while (y < ey || (y === ey && m <= em)) { out.push(`${y}${String(m).padStart(2, '0')}`); if (++m > 12) { m = 1; y++; } }
|
|
return out;
|
|
}
|
|
|
|
export const playObjects = (pkg: string, month: string) => ({
|
|
storePerformance: `stats/store_performance/store_performance_${pkg}_${month}_country.csv`,
|
|
installs: `stats/installs/installs_${pkg}_${month}_country.csv`,
|
|
});
|
|
|
|
/** "gs://pubsite_prod_rev_123/" or "pubsite_prod_rev_123" -> bucket name. */
|
|
export const bucketName = (b: string): string => b.replace(/^gs:\/\//, '').replace(/\/.*$/, '');
|
|
|
|
export interface Totals { impressions?: number; pageViews?: number; downloads?: number; conversion?: number | null; visitors?: number; acquisitions?: number; installs?: number }
|
|
|
|
export function appleTotals(days: AppleDay[]): Totals {
|
|
const s = (k: keyof AppleDay) => days.reduce((n, d) => n + (d[k] as number), 0);
|
|
const uniq = s('uniqueImpressions');
|
|
return { impressions: s('impressions'), pageViews: s('pageViews'), downloads: s('downloads'), conversion: uniq ? s('downloads') / uniq : null };
|
|
}
|
|
export function playTotals(days: PlayDay[]): Totals {
|
|
const s = (k: keyof PlayDay) => days.reduce((n, d) => n + (d[k] as number), 0);
|
|
const v = s('visitors');
|
|
return { visitors: v, acquisitions: s('acquisitions'), installs: s('installs'), conversion: v ? s('acquisitions') / v : null };
|
|
}
|
|
|
|
export interface Snapshot {
|
|
date: string;
|
|
window: { start: string; end: string; days: number };
|
|
apple?: { daily: AppleDay[]; totals: Totals } | { error: string };
|
|
play?: { daily: PlayDay[]; totals: Totals } | { error: string };
|
|
}
|
|
|
|
/**
|
|
* Why a fresh snapshot is not worth writing: every store it read failed (no ONGOING report request yet, no reports
|
|
* bucket, no access). Null when at least one store has days. A snapshot of errors only would make `snapshot` look done
|
|
* while `compare` and the desktop chart stay empty.
|
|
*/
|
|
export function snapshotProblem(s: Snapshot): string | null {
|
|
const parts = (['apple', 'play'] as const).filter(k => s[k]);
|
|
if (!parts.length) return 'no store to read (no apple or play section in the config, or --store names none)';
|
|
if (parts.some(k => !('error' in s[k]!))) return null;
|
|
return parts.map(k => `${k}: ${(s[k] as { error: string }).error}`).join('; ');
|
|
}
|
|
|
|
/** The newest error per store when that store has no days in any snapshot (shown instead of an empty chart). */
|
|
export function storeErrors(snaps: Snapshot[]): { apple?: string; play?: string } {
|
|
const out: { apple?: string; play?: string } = {};
|
|
const series = mergeSeries(snaps);
|
|
for (const k of ['apple', 'play'] as const) {
|
|
if (series[k].length) continue;
|
|
const last = [...snaps].sort((x, y) => x.date.localeCompare(y.date)).reverse().find(s => s[k] && 'error' in s[k]!);
|
|
if (last) out[k] = (last[k] as { error: string }).error;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
export const inWindow = <T extends { date: string }>(rows: T[], start: string, end: string): T[] => rows.filter(r => r.date >= start && r.date <= end);
|
|
|
|
// ---- compare with listing changes ----
|
|
|
|
export interface Event { date: string; commit: string; subject: string; files: string[] }
|
|
|
|
/** `git log --date=short --format=%x1e%h%x1f%ad%x1f%s --name-only -- <paths>` -> events. */
|
|
export function parseGitLog(out: string): Event[] {
|
|
return out.split('\x1e').map(s => s.trim()).filter(Boolean).map(block => {
|
|
const [head, ...files] = block.split('\n');
|
|
const [commit = '', date = '', subject = ''] = (head ?? '').split('\x1f');
|
|
return { date, commit, subject, files: files.map(f => f.trim()).filter(Boolean) };
|
|
}).filter(e => /^\d{4}-\d{2}-\d{2}$/.test(e.date));
|
|
}
|
|
|
|
/** Snapshots -> one series per store (newest snapshot wins for a date). */
|
|
export function mergeSeries(snaps: Snapshot[]): { apple: AppleDay[]; play: PlayDay[] } {
|
|
const a = new Map<string, AppleDay>(), p = new Map<string, PlayDay>();
|
|
for (const s of [...snaps].sort((x, y) => x.date.localeCompare(y.date))) {
|
|
if (s.apple && !('error' in s.apple)) for (const d of s.apple.daily) a.set(d.date, d);
|
|
if (s.play && !('error' in s.play)) for (const d of s.play.daily) p.set(d.date, d);
|
|
}
|
|
return { apple: [...a.values()].sort((x, y) => x.date.localeCompare(y.date)), play: [...p.values()].sort((x, y) => x.date.localeCompare(y.date)) };
|
|
}
|
|
|
|
const addDays = (d: string, n: number) => new Date(Date.parse(d + 'T00:00:00Z') + n * 86_400_000).toISOString().slice(0, 10);
|
|
|
|
export interface EventEffect { event: Event; store: 'apple' | 'play'; before: Totals; after: Totals; daysBefore: number; daysAfter: number }
|
|
|
|
/** For every event: the `days` before it (excluding the event day) vs the `days` from it, per store, where data exists. */
|
|
export function compareAroundEvents(series: { apple: AppleDay[]; play: PlayDay[] }, events: Event[], days = 7): EventEffect[] {
|
|
const out: EventEffect[] = [];
|
|
for (const e of events) {
|
|
const bs = addDays(e.date, -days), be = addDays(e.date, -1), as = e.date, ae = addDays(e.date, days - 1);
|
|
const ab = inWindow(series.apple, bs, be), aa = inWindow(series.apple, as, ae);
|
|
if (ab.length || aa.length) out.push({ event: e, store: 'apple', before: appleTotals(ab), after: appleTotals(aa), daysBefore: ab.length, daysAfter: aa.length });
|
|
const pb = inWindow(series.play, bs, be), pa = inWindow(series.play, as, ae);
|
|
if (pb.length || pa.length) out.push({ event: e, store: 'play', before: playTotals(pb), after: playTotals(pa), daysBefore: pb.length, daysAfter: pa.length });
|
|
}
|
|
return out;
|
|
}
|
|
|
|
const perDay = (v: number | undefined, n: number) => (v === undefined || !n ? null : v / n);
|
|
const fmtN = (v: number | null) => (v === null ? 'n/a' : v >= 100 ? Math.round(v).toString() : v.toFixed(1));
|
|
const delta = (a: number | null | undefined, b: number | null | undefined) => (a == null || b == null || a === 0 ? '' : `${b >= a ? '+' : ''}${(((b - a) / a) * 100).toFixed(0)}%`);
|
|
|
|
/** Rows for the comparison table: per event and store, daily averages before / after and the conversion change. */
|
|
export function effectRows(fx: EventEffect[]): (string | number)[][] {
|
|
return fx.map(f => {
|
|
const top = f.store === 'apple' ? 'impressions' : 'visitors';
|
|
const b = perDay(f.before[top], f.daysBefore), a = perDay(f.after[top], f.daysAfter);
|
|
return [f.event.date, f.store, f.event.subject.slice(0, 48), `${fmtN(b)} -> ${fmtN(a)} ${delta(b, a)}`.trim(),
|
|
`${pct(f.before.conversion ?? null)} -> ${pct(f.after.conversion ?? null)}`, `${f.daysBefore}/${f.daysAfter}`];
|
|
});
|
|
}
|
|
|
|
export const EFFECT_HEAD = ['date', 'store', 'change', 'impressions or visitors / day', 'conversion', 'days before/after'];
|
|
export const effectTable = (fx: EventEffect[]): string => (fx.length ? textTable(EFFECT_HEAD, effectRows(fx)) : 'no listing changes inside the snapshot data yet');
|
|
export const effectMarkdown = (fx: EventEffect[]): string => mdTable(EFFECT_HEAD, effectRows(fx));
|