storebound/cli/lib/analytics.ts

// Store analytics snapshots (impressions, product page views, downloads / installs, conversion) and their comparison
// with listing changes, as pure parts. `storebound analytics` downloads the data (read-only) and writes the snapshots.
// Sources (checked October 2026):
//   App Store Connect Analytics Reports API: an ONGOING analyticsReportRequest (Admin creates it once; first data after
//   1-2 days, then daily; instances kept 35 days; a request stops after long inactivity: stoppedDueToInactivity),
//   reports -> instances (DAILY) -> segments (url valid 5 minutes, md5 checksum, gzip text files; every segment of
//   an instance is needed). A later processingDate replaces rows of the same Date: never sum one Date across instances.
//     https://developer.apple.com/documentation/appstoreconnectapi/downloading-analytics-reports
//     https://developer.apple.com/documentation/analytics-reports/app-store-discovery-and-engagement
//     https://developer.apple.com/documentation/analytics-reports/app-download
//   Google Play: no API for store listing statistics (the Reporting API has vitals only). Monthly CSV exports in the
//   developer's Cloud Storage bucket gs://pubsite_prod_rev_<id> (Play Console > Download reports > Statistics, "Copy
//   Cloud Storage URI"), UTF-16, read with the service account (Play Console permission "View app information and
//   download bulk reports", scope devstorage.read_only):
//     stats/store_performance/store_performance_<package>_<yyyyMM>_country.csv (Store listing acquisitions / visitors)
//     stats/installs/installs_<package>_<yyyyMM>_country.csv (Daily User Installs, ...)
//     https://support.google.com/googleplay/android-developer/answer/6135870
import { plainError } from './google-errors';
import { createHash } from 'node:crypto';
import { parseCsvRecords } from './csv';
import { mdTable, pct, textTable } from './table';

export const APPLE_REPORTS = {
  engagement: { category: 'APP_STORE_ENGAGEMENT', match: /discovery and engagement/i },
  downloads: { category: 'COMMERCE', match: /download/i },
  // App Usage (Analytics Reports API category APP_USAGE): App Sessions (Sessions, Total Session Duration, Unique Devices) and
  // App Crashes (Crashes, Unique Devices), per day, version, device, platform version and territory
  sessions: { category: 'APP_USAGE', match: /app sessions/i },
  crashes: { category: 'APP_USAGE', match: /app crashes/i },
} as const;

/** Standard over Detailed (Detailed adds noise for privacy and splits by campaign). */
export function pickReport<T extends { attributes?: { name?: string } }>(reports: T[], match: RegExp): T | null {
  const hits = reports.filter(r => match.test(r.attributes?.name ?? ''));
  return hits.find(r => /standard/i.test(r.attributes?.name ?? '')) ?? hits[0] ?? null;
}

/** Bytes of a downloaded segment vs its md5 checksum. */
export const checksumOk = (b: Uint8Array, checksum: string): boolean => createHash('md5').update(b).digest('hex').toLowerCase() === checksum.toLowerCase();

/** Text of a report file: UTF-16 with a BOM (Play exports), else UTF-8. */
export function decodeText(b: Uint8Array): string {
  if (b[0] === 0xff && b[1] === 0xfe) return new TextDecoder('utf-16le').decode(b.subarray(2));
  if (b[0] === 0xfe && b[1] === 0xff) return new TextDecoder('utf-16be').decode(b.subarray(2));
  return new TextDecoder('utf-8').decode(b);
}

/** Tab- or comma-separated report text -> records keyed by lower-case column name (by name: Apple may reorder). */
export function parseReport(text: string): Record<string, string>[] {
  const first = text.replace(/^/, '').split(/\r?\n/, 1)[0] ?? '';
  return parseCsvRecords(text, first.includes('\t') ? '\t' : ',');
}

/** Rows of several instances: per Date only the rows of the newest processingDate (later instances restate days). */
export function latestPerDate(instances: { processingDate: string; rows: Record<string, string>[] }[]): Record<string, string>[] {
  const best = new Map<string, string>();
  for (const i of instances) for (const r of i.rows) { const d = r.date ?? ''; if (!best.has(d) || best.get(d)! < i.processingDate) best.set(d, i.processingDate); }
  return instances.flatMap(i => i.rows.filter(r => best.get(r.date ?? '') === i.processingDate));
}

export interface AppleDay {
  date: string; impressions: number; uniqueImpressions: number; pageViews: number; uniquePageViews: number; downloads: number; firstTimeDownloads: number;
  /** App Usage (only when the App Sessions / App Crashes reports have the day): app sessions, active devices (sum of the
   *  rows' Unique Devices: a device seen in two rows, e.g. two app versions, counts twice), total session time in
   *  seconds, crashes. */
  sessions?: number; activeDevices?: number; sessionSeconds?: number; crashes?: number;
}
/** installs = Daily User Installs. The two stock numbers are Play's own columns: devices with the app installed that
 *  were active in the last 30 days (Active Device Installs) and all users who ever installed it (Total User Installs). */
export interface PlayDay { date: string; visitors: number; acquisitions: number; installs: number; activeDeviceInstalls?: number; totalUserInstalls?: number }

const num = (v: string | undefined) => { const n = Number((v ?? '').replace(/,/g, '')); return Number.isFinite(n) ? n : 0; };

/** Discovery and Engagement + Downloads rows -> one line per day (App Store, every territory and source summed). */
export function appleDaily(engagement: Record<string, string>[], downloads: Record<string, string>[]): AppleDay[] {
  const days = new Map<string, AppleDay>();
  const day = (d: string) => days.get(d) ?? days.set(d, { date: d, impressions: 0, uniqueImpressions: 0, pageViews: 0, uniquePageViews: 0, downloads: 0, firstTimeDownloads: 0 }).get(d)!;
  for (const r of engagement) {
    if (!r.date) continue;
    const ev = (r.event ?? '').toLowerCase();
    const x = day(r.date);
    if (ev === 'impression') { x.impressions += num(r.counts); x.uniqueImpressions += num(r.unique_counts); }
    else if (ev === 'page view') { x.pageViews += num(r.counts); x.uniquePageViews += num(r.unique_counts); }
  }
  for (const r of downloads) {
    if (!r.date) continue;
    const t = (r.download_type ?? '').toLowerCase();
    const x = day(r.date);
    if (t === 'first-time download') { x.firstTimeDownloads += num(r.counts); x.downloads += num(r.counts); }
    else if (t === 'redownload') x.downloads += num(r.counts);
  }
  return [...days.values()].sort((a, b) => a.date.localeCompare(b.date));
}

/** App Sessions + App Crashes rows -> App Usage fields per day, merged into the days (a day only in the usage reports
 *  becomes a day with zero impressions and downloads). Sessions' Unique Devices are the active devices of the day. */
export function mergeAppleUsage(daily: AppleDay[], sessions: Record<string, string>[], crashes: Record<string, string>[]): AppleDay[] {
  const days = new Map(daily.map(d => [d.date, { ...d }]));
  const day = (d: string) => days.get(d) ?? days.set(d, { date: d, impressions: 0, uniqueImpressions: 0, pageViews: 0, uniquePageViews: 0, downloads: 0, firstTimeDownloads: 0 }).get(d)!;
  for (const r of sessions) {
    if (!r.date) continue;
    const x = day(r.date);
    x.sessions = (x.sessions ?? 0) + num(r.sessions);
    x.activeDevices = (x.activeDevices ?? 0) + num(r.unique_devices);
    x.sessionSeconds = (x.sessionSeconds ?? 0) + num(r.total_session_duration);
  }
  for (const r of crashes) {
    if (!r.date) continue;
    const x = day(r.date);
    x.crashes = (x.crashes ?? 0) + num(r.crashes ?? r.counts);
  }
  return [...days.values()].sort((a, b) => a.date.localeCompare(b.date));
}

/** Play store performance + installs CSV rows (country files) -> one line per day. Dates are YYYY-MM-DD. The stock
 *  columns (Active Device Installs, Total User Installs) come from the overview rows when given (all countries in one
 *  row per day), else from the country rows summed. */
export function playDaily(storePerformance: Record<string, string>[], installs: Record<string, string>[], overview: Record<string, string>[] = []): PlayDay[] {
  const days = new Map<string, PlayDay>();
  const day = (d: string) => days.get(d) ?? days.set(d, { date: d, visitors: 0, acquisitions: 0, installs: 0 }).get(d)!;
  for (const r of storePerformance) if (r.date) { const x = day(r.date); x.visitors += num(r.store_listing_visitors); x.acquisitions += num(r.store_listing_acquisitions); }
  for (const r of installs) if (r.date) day(r.date).installs += num(r.daily_user_installs);
  const stock = (rows: Record<string, string>[]) => {
    const m = new Map<string, { active?: number; total?: number }>();
    for (const r of rows) {
      if (!r.date) continue;
      const x = m.get(r.date) ?? m.set(r.date, {}).get(r.date)!;
      if (r.active_device_installs !== undefined && r.active_device_installs !== '') x.active = (x.active ?? 0) + num(r.active_device_installs);
      if (r.total_user_installs !== undefined && r.total_user_installs !== '') x.total = (x.total ?? 0) + num(r.total_user_installs);
    }
    return m;
  };
  const fromCountries = stock(installs), fromOverview = stock(overview);
  for (const [date, x] of (fromOverview.size ? fromOverview : fromCountries)) {
    const d = day(date);
    if (x.active !== undefined) d.activeDeviceInstalls = x.active;
    if (x.total !== undefined) d.totalUserInstalls = x.total;
  }
  return [...days.values()].sort((a, b) => a.date.localeCompare(b.date));
}

/** Months (yyyyMM) a date window touches, for the Play file names. */
export function monthsBetween(start: string, end: string): string[] {
  const out: string[] = [];
  let y = Number(start.slice(0, 4)), m = Number(start.slice(5, 7));
  const ey = Number(end.slice(0, 4)), em = Number(end.slice(5, 7));
  while (y < ey || (y === ey && m <= em)) { out.push(`${y}${String(m).padStart(2, '0')}`); if (++m > 12) { m = 1; y++; } }
  return out;
}

export const playObjects = (pkg: string, month: string) => ({
  storePerformance: `stats/store_performance/store_performance_${pkg}_${month}_country.csv`,
  installs: `stats/installs/installs_${pkg}_${month}_country.csv`,
  installsOverview: `stats/installs/installs_${pkg}_${month}_overview.csv`,
});

/** "gs://pubsite_prod_rev_123/" or "pubsite_prod_rev_123" -> bucket name. */
export const bucketName = (b: string): string => b.replace(/^gs:\/\//, '').replace(/\/.*$/, '');

export interface Totals {
  impressions?: number; pageViews?: number; downloads?: number; conversion?: number | null; visitors?: number; acquisitions?: number; installs?: number;
  /** App Usage: sessions and crashes summed, active devices as the mean of the days that have usage data. */
  sessions?: number; crashes?: number; activeDevicesAvg?: number;
  /** Play stock numbers: the value of the newest day that has one (never summed). */
  activeDeviceInstalls?: number; totalUserInstalls?: number;
}

export function appleTotals(days: AppleDay[]): Totals {
  const s = (k: keyof AppleDay) => days.reduce((n, d) => n + (d[k] as number), 0);
  const uniq = s('uniqueImpressions');
  const t: Totals = { impressions: s('impressions'), pageViews: s('pageViews'), downloads: s('downloads'), conversion: uniq ? s('downloads') / uniq : null };
  const used = days.filter(d => d.sessions !== undefined || d.crashes !== undefined);
  if (used.length) {
    t.sessions = used.reduce((n, d) => n + (d.sessions ?? 0), 0);
    t.crashes = used.reduce((n, d) => n + (d.crashes ?? 0), 0);
    const act = used.filter(d => d.activeDevices !== undefined);
    if (act.length) t.activeDevicesAvg = Math.round(act.reduce((n, d) => n + d.activeDevices!, 0) / act.length);
  }
  return t;
}
export function playTotals(days: PlayDay[]): Totals {
  const s = (k: keyof PlayDay) => days.reduce((n, d) => n + (d[k] as number), 0);
  const v = s('visitors');
  const t: Totals = { visitors: v, acquisitions: s('acquisitions'), installs: s('installs'), conversion: v ? s('acquisitions') / v : null };
  const last = (k: 'activeDeviceInstalls' | 'totalUserInstalls') => [...days].reverse().find(d => d[k] !== undefined)?.[k];
  const a = last('activeDeviceInstalls'), tot = last('totalUserInstalls');
  if (a !== undefined) t.activeDeviceInstalls = a;
  if (tot !== undefined) t.totalUserInstalls = tot;
  return t;
}

export interface Snapshot {
  date: string;
  window: { start: string; end: string; days: number };
  /** notes: what is missing but not fatal (e.g. the App Usage reports are not listed yet). */
  apple?: { daily: AppleDay[]; totals: Totals; notes?: string[] } | { error: string };
  play?: { daily: PlayDay[]; totals: Totals; notes?: string[] } | { error: string };
}

/**
 * Why a fresh snapshot is not worth writing: every store it read failed (no ONGOING report request yet, no reports
 * bucket, no access). Null when at least one store has days. A snapshot of errors only would make `snapshot` look done
 * while `compare` and the desktop chart stay empty.
 */
export function snapshotProblem(s: Snapshot): string | null {
  const parts = (['apple', 'play'] as const).filter(k => s[k]);
  if (!parts.length) return 'no store to read (no apple or play section in the config, or --store names none)';
  if (parts.some(k => !('error' in s[k]!))) return null;
  return parts.map(k => `${k}: ${(s[k] as { error: string }).error}`).join('; ');
}

/** The newest error per store when that store has no days in any snapshot (shown instead of an empty chart). */
export function storeErrors(snaps: Snapshot[]): { apple?: string; play?: string } {
  const out: { apple?: string; play?: string } = {};
  const series = mergeSeries(snaps);
  for (const k of ['apple', 'play'] as const) {
    if (series[k].length) continue;
    const last = [...snaps].sort((x, y) => x.date.localeCompare(y.date)).reverse().find(s => s[k] && 'error' in s[k]!);
    if (last) out[k] = plainError((last[k] as { error: string }).error, { withUrl: true });
  }
  return out;
}

export const inWindow = <T extends { date: string }>(rows: T[], start: string, end: string): T[] => rows.filter(r => r.date >= start && r.date <= end);

// ---- compare with listing changes ----

export interface Event { date: string; commit: string; subject: string; files: string[] }

/** `git log --date=short --format=%x1e%h%x1f%ad%x1f%s --name-only -- <paths>` -> events. */
export function parseGitLog(out: string): Event[] {
  return out.split('\x1e').map(s => s.trim()).filter(Boolean).map(block => {
    const [head, ...files] = block.split('\n');
    const [commit = '', date = '', subject = ''] = (head ?? '').split('\x1f');
    return { date, commit, subject, files: files.map(f => f.trim()).filter(Boolean) };
  }).filter(e => /^\d{4}-\d{2}-\d{2}$/.test(e.date));
}

/** Snapshots -> one series per store (newest snapshot wins for a date). */
export function mergeSeries(snaps: Snapshot[]): { apple: AppleDay[]; play: PlayDay[] } {
  const a = new Map<string, AppleDay>(), p = new Map<string, PlayDay>();
  for (const s of [...snaps].sort((x, y) => x.date.localeCompare(y.date))) {
    if (s.apple && !('error' in s.apple)) for (const d of s.apple.daily) a.set(d.date, d);
    if (s.play && !('error' in s.play)) for (const d of s.play.daily) p.set(d.date, d);
  }
  return { apple: [...a.values()].sort((x, y) => x.date.localeCompare(y.date)), play: [...p.values()].sort((x, y) => x.date.localeCompare(y.date)) };
}

const addDays = (d: string, n: number) => new Date(Date.parse(d + 'T00:00:00Z') + n * 86_400_000).toISOString().slice(0, 10);

export interface EventEffect { event: Event; store: 'apple' | 'play'; before: Totals; after: Totals; daysBefore: number; daysAfter: number }

/** For every event: the `days` before it (excluding the event day) vs the `days` from it, per store, where data exists. */
export function compareAroundEvents(series: { apple: AppleDay[]; play: PlayDay[] }, events: Event[], days = 7): EventEffect[] {
  const out: EventEffect[] = [];
  for (const e of events) {
    const bs = addDays(e.date, -days), be = addDays(e.date, -1), as = e.date, ae = addDays(e.date, days - 1);
    const ab = inWindow(series.apple, bs, be), aa = inWindow(series.apple, as, ae);
    if (ab.length || aa.length) out.push({ event: e, store: 'apple', before: appleTotals(ab), after: appleTotals(aa), daysBefore: ab.length, daysAfter: aa.length });
    const pb = inWindow(series.play, bs, be), pa = inWindow(series.play, as, ae);
    if (pb.length || pa.length) out.push({ event: e, store: 'play', before: playTotals(pb), after: playTotals(pa), daysBefore: pb.length, daysAfter: pa.length });
  }
  return out;
}

const perDay = (v: number | undefined, n: number) => (v === undefined || !n ? null : v / n);
const fmtN = (v: number | null) => (v === null ? 'n/a' : v >= 100 ? Math.round(v).toString() : v.toFixed(1));
const delta = (a: number | null | undefined, b: number | null | undefined) => (a == null || b == null || a === 0 ? '' : `${b >= a ? '+' : ''}${(((b - a) / a) * 100).toFixed(0)}%`);

/** Rows for the comparison table: per event and store, daily averages before / after and the conversion change. */
export function effectRows(fx: EventEffect[]): (string | number)[][] {
  return fx.map(f => {
    const top = f.store === 'apple' ? 'impressions' : 'visitors';
    const b = perDay(f.before[top], f.daysBefore), a = perDay(f.after[top], f.daysAfter);
    return [f.event.date, f.store, f.event.subject.slice(0, 48), `${fmtN(b)} -> ${fmtN(a)} ${delta(b, a)}`.trim(),
      `${pct(f.before.conversion ?? null)} -> ${pct(f.after.conversion ?? null)}`, `${f.daysBefore}/${f.daysAfter}`];
  });
}

export const EFFECT_HEAD = ['date', 'store', 'change', 'impressions or visitors / day', 'conversion', 'days before/after'];
export const effectTable = (fx: EventEffect[]): string => (fx.length ? textTable(EFFECT_HEAD, effectRows(fx)) : 'no listing changes inside the snapshot data yet');
export const effectMarkdown = (fx: EventEffect[]): string => mdTable(EFFECT_HEAD, effectRows(fx));