// The background read scheduler: every store read of the window goes through one queue, so reading "everything for // every app" stays far inside the stores' limits. Pure logic with an injectable clock (tests/scheduler.test.ts). // // - Lanes: 'apple' (App Store Connect), 'play' (Google Play Developer + Reporting API), 'local' (offline project // reads). A task takes one slot in each of its lanes; each lane has its own concurrency limit (default 4). // - Credentials: starts on one key are spaced (default 250 ms apart). App Store Connect sends X-Rate-Limit // (user-hour-lim / user-hour-rem, about 3600 per key per hour): below 20 % left, the spacing spreads the rest over the // hour; at the reserve (default 100, kept for the owner's own commands) the key pauses for 10 minutes. // - 429 / RESOURCE_EXHAUSTED: the task waits and runs again with exponential backoff and jitter (and its key waits too), // at most `maxAttempts` times. // - Priority: 0 the visible app and tab, 1 portfolio status, 2 the rest; FIFO within a priority. // - Dedupe: one in-flight (or queued) read per key; enqueueing it again returns the same promise (and may raise its // priority). export type Lane = 'apple' | 'play' | 'local'; export type Priority = 0 | 1 | 2; export interface Clock { now(): number; setTimeout(fn: () => void, ms: number): unknown; clearTimeout(h: unknown): void; random(): number; } export const realClock: Clock = { now: () => Date.now(), setTimeout: (fn, ms) => setTimeout(fn, ms), clearTimeout: h => clearTimeout(h as ReturnType), random: () => Math.random(), }; export interface QuotaSeen { limit?: number; remaining?: number; throttled?: boolean } /** What one finished run says about the store's limits. */ export interface Outcome { /** 429 / RESOURCE_EXHAUSTED: run it again later. */ throttled?: boolean; /** Quota per credential (App Store Connect X-Rate-Limit). */ quota?: { credential: string; limit: number; remaining: number }; } export interface ReadTask { key: string; lanes: Lane[]; /** Keys of the store accounts it uses ("apple:", "play:"). */ accounts: string[]; priority: Priority; run: () => Promise; } export interface SchedulerOptions { limits?: Partial>; /** Minimum gap between two starts on one credential. */ spacingMs?: number; /** Requests per hour kept free on an App Store Connect key for the owner's own commands. */ reserve?: number; /** Below this share of the hourly limit left, starts are spread over the rest of the hour. */ slowBelow?: number; /** How long a key pauses once it is at the reserve. */ pauseMs?: number; backoffBaseMs?: number; backoffMaxMs?: number; maxAttempts?: number; clock?: Clock; /** Reads a result (or the error) for throttling and quota. */ classify?: (result: unknown, error: unknown, task: ReadTask) => Outcome; } export interface SchedulerStats { queued: number; running: number; /** Finished in this round (since the queue was last empty). */ done: number; /** done + running + queued of this round. */ total: number; failed: number; /** The lowest App Store Connect quota seen per credential this hour. */ quota: Record; /** Credentials waiting on a backoff or quota pause, with the time they may start again. */ paused: Record; } interface Item { task: ReadTask; seq: number; attempt: number; notBefore: number; resolve: (r: R) => void; reject: (e: unknown) => void; promise: Promise; } const HOUR = 3_600_000; export class ReadScheduler { private o: Required> & { limits: Record }; private clock: Clock; private classify: (result: unknown, error: unknown, task: ReadTask) => Outcome; private queue: Item[] = []; private running = new Map>(); private busy: Record = { apple: 0, play: 0, local: 0 }; private nextAt = new Map(); private quotas = new Map(); private seq = 0; private done = 0; private failed = 0; private timer: unknown = null; private timerAt = 0; private listeners = new Set<() => void>(); constructor(o: SchedulerOptions = {}) { this.o = { limits: { apple: 4, play: 4, local: 2, ...o.limits }, spacingMs: o.spacingMs ?? 250, reserve: o.reserve ?? 100, slowBelow: o.slowBelow ?? 0.2, pauseMs: o.pauseMs ?? 10 * 60_000, backoffBaseMs: o.backoffBaseMs ?? 5_000, backoffMaxMs: o.backoffMaxMs ?? 5 * 60_000, maxAttempts: o.maxAttempts ?? 4, }; this.clock = o.clock ?? realClock; this.classify = o.classify ?? (() => ({})); } setLimits(limits: Partial>): void { this.o.limits = { ...this.o.limits, ...limits }; this.pump(); } subscribe(fn: () => void): () => void { this.listeners.add(fn); return () => { this.listeners.delete(fn); }; } has(key: string): boolean { return this.running.has(key) || this.queue.some(i => i.task.key === key); } enqueue(task: ReadTask): Promise { const run = this.running.get(task.key); if (run) return run.promise as Promise; const queued = this.queue.find(i => i.task.key === task.key); if (queued) { if (task.priority < queued.task.priority) { queued.task = { ...queued.task, priority: task.priority }; this.pump(); } return queued.promise as Promise; } let resolve!: (r: R) => void, reject!: (e: unknown) => void; const promise = new Promise((a, b) => { resolve = a; reject = b; }); this.queue.push({ task, seq: ++this.seq, attempt: 0, notBefore: 0, resolve, reject, promise } as unknown as Item); this.pump(); this.emit(); return promise; } /** Drops queued tasks (not the running ones); their promises reject with "cancelled". */ clear(filter: (key: string) => boolean = () => true): void { const drop = this.queue.filter(i => filter(i.task.key)); this.queue = this.queue.filter(i => !filter(i.task.key)); for (const i of drop) i.reject(new Error('cancelled')); this.pump(); this.emit(); } stats(): SchedulerStats { const now = this.clock.now(); const paused: Record = {}; for (const [c, at] of this.nextAt) if (at - now > this.o.spacingMs) paused[c] = at; const quota: SchedulerStats['quota'] = {}; for (const [c, q] of this.quotas) if (now - q.at < HOUR) quota[c] = q; return { queued: this.queue.length, running: this.running.size, done: this.done, total: this.done + this.running.size + this.queue.length, failed: this.failed, quota, paused }; } /** Earliest time a credential may start again. */ private gap(cred: string, now: number): number { const q = this.quotas.get(cred); if (!q || now - q.at >= HOUR) return this.o.spacingMs; if (q.remaining <= this.o.reserve) return this.o.pauseMs; if (q.remaining < q.limit * this.o.slowBelow) return Math.max(this.o.spacingMs, Math.ceil(HOUR / Math.max(1, q.remaining - this.o.reserve))); return this.o.spacingMs; } private ready(i: Item, now: number): number { // 0 = can start now; otherwise the time it can (Infinity: waits on a lane slot) let at = i.notBefore; for (const c of i.task.accounts) at = Math.max(at, this.nextAt.get(c) ?? 0); if (i.task.lanes.some(l => this.busy[l] >= Math.max(1, this.o.limits[l]))) return Infinity; return at <= now ? 0 : at; } private pump(): void { const now = this.clock.now(); let wake = Infinity; for (;;) { const order = [...this.queue].sort((a, b) => a.task.priority - b.task.priority || a.seq - b.seq); let started = false; for (const i of order) { const r = this.ready(i, now); if (r === 0) { this.start(i, now); started = true; break; } if (r !== Infinity) wake = Math.min(wake, r); } if (!started) break; } if (this.timer !== null && (wake === Infinity || wake !== this.timerAt)) { this.clock.clearTimeout(this.timer); this.timer = null; } if (wake !== Infinity && this.timer === null) { this.timerAt = wake; this.timer = this.clock.setTimeout(() => { this.timer = null; this.pump(); }, Math.max(0, wake - now)); } } private start(i: Item, now: number): void { this.queue = this.queue.filter(x => x !== i); this.running.set(i.task.key, i); for (const l of i.task.lanes) this.busy[l]++; for (const c of i.task.accounts) this.nextAt.set(c, now + this.gap(c, now)); i.attempt++; this.emit(); i.task.run().then(r => this.finish(i, r, undefined, true), e => this.finish(i, undefined, e, false)); } private finish(i: Item, result: unknown, error: unknown, ok: boolean): void { const now = this.clock.now(); this.running.delete(i.task.key); for (const l of i.task.lanes) this.busy[l]--; let out: Outcome = {}; try { out = this.classify(result, error, i.task); } catch { /* a classifier bug never stalls the queue */ } if (out.quota) { const prev = this.quotas.get(out.quota.credential); // the hour is the key's: keep the lowest count seen in it const q = prev && now - prev.at < HOUR && prev.remaining < out.quota.remaining ? prev : { limit: out.quota.limit, remaining: out.quota.remaining, at: now }; this.quotas.set(out.quota.credential, q); this.nextAt.set(out.quota.credential, Math.max(this.nextAt.get(out.quota.credential) ?? 0, now + this.gap(out.quota.credential, now))); } if (out.throttled && i.attempt < this.o.maxAttempts) { const exp = Math.min(this.o.backoffMaxMs, this.o.backoffBaseMs * 2 ** (i.attempt - 1)); const delay = Math.round(exp / 2 + this.clock.random() * exp / 2); // "equal jitter" i.notBefore = now + delay; for (const c of i.task.accounts) this.nextAt.set(c, Math.max(this.nextAt.get(c) ?? 0, now + delay)); this.queue.push(i); } else { this.done++; if (!ok || out.throttled) this.failed++; if (ok) i.resolve(result); else i.reject(error); } if (!this.queue.length && !this.running.size) { this.done = 0; this.failed = 0; } this.pump(); this.emit(); } private emit(): void { for (const f of this.listeners) { try { f(); } catch { /* listeners never break the queue */ } } } }