storebound/apps/desktop/src/lib/scheduler.ts

243 lines
10 KiB
TypeScript

// The background read scheduler: every store read of the window goes through one queue, so reading "everything for
// every app" stays far inside the stores' limits. Pure logic with an injectable clock (tests/scheduler.test.ts).
//
// - Lanes: 'apple' (App Store Connect), 'play' (Google Play Developer + Reporting API), 'local' (offline project
// reads). A task takes one slot in each of its lanes; each lane has its own concurrency limit (default 4).
// - Credentials: starts on one key are spaced (default 250 ms apart). App Store Connect sends X-Rate-Limit
// (user-hour-lim / user-hour-rem, about 3600 per key per hour): below 20 % left, the spacing spreads the rest over the
// hour; at the reserve (default 100, kept for the owner's own commands) the key pauses for 10 minutes.
// - 429 / RESOURCE_EXHAUSTED: the task waits and runs again with exponential backoff and jitter (and its key waits too),
// at most `maxAttempts` times.
// - Priority: 0 the visible app and tab, 1 portfolio status, 2 the rest; FIFO within a priority.
// - Dedupe: one in-flight (or queued) read per key; enqueueing it again returns the same promise (and may raise its
// priority).
export type Lane = 'apple' | 'play' | 'local';
export type Priority = 0 | 1 | 2;
export interface Clock {
now(): number;
setTimeout(fn: () => void, ms: number): unknown;
clearTimeout(h: unknown): void;
random(): number;
}
export const realClock: Clock = {
now: () => Date.now(),
setTimeout: (fn, ms) => setTimeout(fn, ms),
clearTimeout: h => clearTimeout(h as ReturnType<typeof setTimeout>),
random: () => Math.random(),
};
export interface QuotaSeen { limit?: number; remaining?: number; throttled?: boolean }
/** What one finished run says about the store's limits. */
export interface Outcome {
/** 429 / RESOURCE_EXHAUSTED: run it again later. */
throttled?: boolean;
/** Quota per credential (App Store Connect X-Rate-Limit). */
quota?: { credential: string; limit: number; remaining: number };
}
export interface ReadTask<R> {
key: string;
lanes: Lane[];
/** Keys of the store accounts it uses ("apple:<account>", "play:<account>"). */
accounts: string[];
priority: Priority;
run: () => Promise<R>;
}
export interface SchedulerOptions {
limits?: Partial<Record<Lane, number>>;
/** Minimum gap between two starts on one credential. */
spacingMs?: number;
/** Requests per hour kept free on an App Store Connect key for the owner's own commands. */
reserve?: number;
/** Below this share of the hourly limit left, starts are spread over the rest of the hour. */
slowBelow?: number;
/** How long a key pauses once it is at the reserve. */
pauseMs?: number;
backoffBaseMs?: number;
backoffMaxMs?: number;
maxAttempts?: number;
clock?: Clock;
/** Reads a result (or the error) for throttling and quota. */
classify?: (result: unknown, error: unknown, task: ReadTask<unknown>) => Outcome;
}
export interface SchedulerStats {
queued: number;
running: number;
/** Finished in this round (since the queue was last empty). */
done: number;
/** done + running + queued of this round. */
total: number;
failed: number;
/** The lowest App Store Connect quota seen per credential this hour. */
quota: Record<string, { limit: number; remaining: number; at: number }>;
/** Credentials waiting on a backoff or quota pause, with the time they may start again. */
paused: Record<string, number>;
}
interface Item<R> {
task: ReadTask<R>;
seq: number;
attempt: number;
notBefore: number;
resolve: (r: R) => void;
reject: (e: unknown) => void;
promise: Promise<R>;
}
const HOUR = 3_600_000;
export class ReadScheduler {
private o: Required<Omit<SchedulerOptions, 'limits' | 'clock' | 'classify'>> & { limits: Record<Lane, number> };
private clock: Clock;
private classify: (result: unknown, error: unknown, task: ReadTask<unknown>) => Outcome;
private queue: Item<unknown>[] = [];
private running = new Map<string, Item<unknown>>();
private busy: Record<Lane, number> = { apple: 0, play: 0, local: 0 };
private nextAt = new Map<string, number>();
private quotas = new Map<string, { limit: number; remaining: number; at: number }>();
private seq = 0;
private done = 0;
private failed = 0;
private timer: unknown = null;
private timerAt = 0;
private listeners = new Set<() => void>();
constructor(o: SchedulerOptions = {}) {
this.o = {
limits: { apple: 4, play: 4, local: 2, ...o.limits },
spacingMs: o.spacingMs ?? 250,
reserve: o.reserve ?? 100,
slowBelow: o.slowBelow ?? 0.2,
pauseMs: o.pauseMs ?? 10 * 60_000,
backoffBaseMs: o.backoffBaseMs ?? 5_000,
backoffMaxMs: o.backoffMaxMs ?? 5 * 60_000,
maxAttempts: o.maxAttempts ?? 4,
};
this.clock = o.clock ?? realClock;
this.classify = o.classify ?? (() => ({}));
}
setLimits(limits: Partial<Record<Lane, number>>): void { this.o.limits = { ...this.o.limits, ...limits }; this.pump(); }
subscribe(fn: () => void): () => void { this.listeners.add(fn); return () => { this.listeners.delete(fn); }; }
has(key: string): boolean { return this.running.has(key) || this.queue.some(i => i.task.key === key); }
enqueue<R>(task: ReadTask<R>): Promise<R> {
const run = this.running.get(task.key);
if (run) return run.promise as Promise<R>;
const queued = this.queue.find(i => i.task.key === task.key);
if (queued) {
if (task.priority < queued.task.priority) { queued.task = { ...queued.task, priority: task.priority }; this.pump(); }
return queued.promise as Promise<R>;
}
let resolve!: (r: R) => void, reject!: (e: unknown) => void;
const promise = new Promise<R>((a, b) => { resolve = a; reject = b; });
this.queue.push({ task, seq: ++this.seq, attempt: 0, notBefore: 0, resolve, reject, promise } as unknown as Item<unknown>);
this.pump();
this.emit();
return promise;
}
/** Drops queued tasks (not the running ones); their promises reject with "cancelled". */
clear(filter: (key: string) => boolean = () => true): void {
const drop = this.queue.filter(i => filter(i.task.key));
this.queue = this.queue.filter(i => !filter(i.task.key));
for (const i of drop) i.reject(new Error('cancelled'));
this.pump();
this.emit();
}
stats(): SchedulerStats {
const now = this.clock.now();
const paused: Record<string, number> = {};
for (const [c, at] of this.nextAt) if (at - now > this.o.spacingMs) paused[c] = at;
const quota: SchedulerStats['quota'] = {};
for (const [c, q] of this.quotas) if (now - q.at < HOUR) quota[c] = q;
return { queued: this.queue.length, running: this.running.size, done: this.done, total: this.done + this.running.size + this.queue.length, failed: this.failed, quota, paused };
}
/** Earliest time a credential may start again. */
private gap(cred: string, now: number): number {
const q = this.quotas.get(cred);
if (!q || now - q.at >= HOUR) return this.o.spacingMs;
if (q.remaining <= this.o.reserve) return this.o.pauseMs;
if (q.remaining < q.limit * this.o.slowBelow) return Math.max(this.o.spacingMs, Math.ceil(HOUR / Math.max(1, q.remaining - this.o.reserve)));
return this.o.spacingMs;
}
private ready(i: Item<unknown>, now: number): number {
// 0 = can start now; otherwise the time it can (Infinity: waits on a lane slot)
let at = i.notBefore;
for (const c of i.task.accounts) at = Math.max(at, this.nextAt.get(c) ?? 0);
if (i.task.lanes.some(l => this.busy[l] >= Math.max(1, this.o.limits[l]))) return Infinity;
return at <= now ? 0 : at;
}
private pump(): void {
const now = this.clock.now();
let wake = Infinity;
for (;;) {
const order = [...this.queue].sort((a, b) => a.task.priority - b.task.priority || a.seq - b.seq);
let started = false;
for (const i of order) {
const r = this.ready(i, now);
if (r === 0) { this.start(i, now); started = true; break; }
if (r !== Infinity) wake = Math.min(wake, r);
}
if (!started) break;
}
if (this.timer !== null && (wake === Infinity || wake !== this.timerAt)) { this.clock.clearTimeout(this.timer); this.timer = null; }
if (wake !== Infinity && this.timer === null) {
this.timerAt = wake;
this.timer = this.clock.setTimeout(() => { this.timer = null; this.pump(); }, Math.max(0, wake - now));
}
}
private start(i: Item<unknown>, now: number): void {
this.queue = this.queue.filter(x => x !== i);
this.running.set(i.task.key, i);
for (const l of i.task.lanes) this.busy[l]++;
for (const c of i.task.accounts) this.nextAt.set(c, now + this.gap(c, now));
i.attempt++;
this.emit();
i.task.run().then(r => this.finish(i, r, undefined, true), e => this.finish(i, undefined, e, false));
}
private finish(i: Item<unknown>, result: unknown, error: unknown, ok: boolean): void {
const now = this.clock.now();
this.running.delete(i.task.key);
for (const l of i.task.lanes) this.busy[l]--;
let out: Outcome = {};
try { out = this.classify(result, error, i.task); } catch { /* a classifier bug never stalls the queue */ }
if (out.quota) {
const prev = this.quotas.get(out.quota.credential);
// the hour is the key's: keep the lowest count seen in it
const q = prev && now - prev.at < HOUR && prev.remaining < out.quota.remaining ? prev : { limit: out.quota.limit, remaining: out.quota.remaining, at: now };
this.quotas.set(out.quota.credential, q);
this.nextAt.set(out.quota.credential, Math.max(this.nextAt.get(out.quota.credential) ?? 0, now + this.gap(out.quota.credential, now)));
}
if (out.throttled && i.attempt < this.o.maxAttempts) {
const exp = Math.min(this.o.backoffMaxMs, this.o.backoffBaseMs * 2 ** (i.attempt - 1));
const delay = Math.round(exp / 2 + this.clock.random() * exp / 2); // "equal jitter"
i.notBefore = now + delay;
for (const c of i.task.accounts) this.nextAt.set(c, Math.max(this.nextAt.get(c) ?? 0, now + delay));
this.queue.push(i);
} else {
this.done++;
if (!ok || out.throttled) this.failed++;
if (ok) i.resolve(result); else i.reject(error);
}
if (!this.queue.length && !this.running.size) { this.done = 0; this.failed = 0; }
this.pump();
this.emit();
}
private emit(): void { for (const f of this.listeners) { try { f(); } catch { /* listeners never break the queue */ } } }
}