desktop reads everything by itself: one background read scheduler (4 per store, starts spaced per account, App Store Connect X-Rate-Limit slows it below 20 % and pauses at 100 left, backoff with jitter on 429 / RESOURCE_EXHAUSTED, visible app first, one read per key), per-method TTLs over the saved reads, Settings auto-read (on) and interval (15 min, paused while hidden), Refresh all, a refresh icon and 'read 3 min ago' / 'couldn't refresh' per card, writes invalidate, status line with progress and quota; quota reaches the window through ToolResult.quota (cli/lib/quota.ts); Health top issues open with versions, sample stack (health --issue / --signature, read health_issue), Open in Play Console, Copy for AI and likely-cause hints (cli/lib/crash-hints.ts); tests/scheduler.test.ts, tests/quota-crash-hints.test.ts, boot check auto-read case
This commit is contained in:
parent
89256f1e80
commit
0fb4ce0001
46 changed files with 1531 additions and 141 deletions
243
apps/desktop/src/lib/scheduler.ts
Normal file
243
apps/desktop/src/lib/scheduler.ts
Normal file
|
|
@ -0,0 +1,243 @@
|
|||
// The background read scheduler: every store read of the window goes through one queue, so reading "everything for
|
||||
// every app" stays far inside the stores' limits. Pure logic with an injectable clock (tests/scheduler.test.ts).
|
||||
//
|
||||
// - Lanes: 'apple' (App Store Connect), 'play' (Google Play Developer + Reporting API), 'local' (offline project
|
||||
// reads). A task takes one slot in each of its lanes; each lane has its own concurrency limit (default 4).
|
||||
// - Credentials: starts on one key are spaced (default 250 ms apart). App Store Connect sends X-Rate-Limit
|
||||
// (user-hour-lim / user-hour-rem, about 3600 per key per hour): below 20 % left, the spacing spreads the rest over the
|
||||
// hour; at the reserve (default 100, kept for the owner's own commands) the key pauses for 10 minutes.
|
||||
// - 429 / RESOURCE_EXHAUSTED: the task waits and runs again with exponential backoff and jitter (and its key waits too),
|
||||
// at most `maxAttempts` times.
|
||||
// - Priority: 0 the visible app and tab, 1 portfolio status, 2 the rest; FIFO within a priority.
|
||||
// - Dedupe: one in-flight (or queued) read per key; enqueueing it again returns the same promise (and may raise its
|
||||
// priority).
|
||||
|
||||
export type Lane = 'apple' | 'play' | 'local';
|
||||
export type Priority = 0 | 1 | 2;
|
||||
|
||||
export interface Clock {
|
||||
now(): number;
|
||||
setTimeout(fn: () => void, ms: number): unknown;
|
||||
clearTimeout(h: unknown): void;
|
||||
random(): number;
|
||||
}
|
||||
|
||||
export const realClock: Clock = {
|
||||
now: () => Date.now(),
|
||||
setTimeout: (fn, ms) => setTimeout(fn, ms),
|
||||
clearTimeout: h => clearTimeout(h as ReturnType<typeof setTimeout>),
|
||||
random: () => Math.random(),
|
||||
};
|
||||
|
||||
export interface QuotaSeen { limit?: number; remaining?: number; throttled?: boolean }
|
||||
|
||||
/** What one finished run says about the store's limits. */
|
||||
export interface Outcome {
|
||||
/** 429 / RESOURCE_EXHAUSTED: run it again later. */
|
||||
throttled?: boolean;
|
||||
/** Quota per credential (App Store Connect X-Rate-Limit). */
|
||||
quota?: { credential: string; limit: number; remaining: number };
|
||||
}
|
||||
|
||||
export interface ReadTask<R> {
|
||||
key: string;
|
||||
lanes: Lane[];
|
||||
/** Keys of the store credentials it uses ("apple:<account>", "play:<account>"). */
|
||||
credentials: string[];
|
||||
priority: Priority;
|
||||
run: () => Promise<R>;
|
||||
}
|
||||
|
||||
export interface SchedulerOptions {
|
||||
limits?: Partial<Record<Lane, number>>;
|
||||
/** Minimum gap between two starts on one credential. */
|
||||
spacingMs?: number;
|
||||
/** Requests per hour kept free on an App Store Connect key for the owner's own commands. */
|
||||
reserve?: number;
|
||||
/** Below this share of the hourly limit left, starts are spread over the rest of the hour. */
|
||||
slowBelow?: number;
|
||||
/** How long a key pauses once it is at the reserve. */
|
||||
pauseMs?: number;
|
||||
backoffBaseMs?: number;
|
||||
backoffMaxMs?: number;
|
||||
maxAttempts?: number;
|
||||
clock?: Clock;
|
||||
/** Reads a result (or the error) for throttling and quota. */
|
||||
classify?: (result: unknown, error: unknown, task: ReadTask<unknown>) => Outcome;
|
||||
}
|
||||
|
||||
export interface SchedulerStats {
|
||||
queued: number;
|
||||
running: number;
|
||||
/** Finished in this round (since the queue was last empty). */
|
||||
done: number;
|
||||
/** done + running + queued of this round. */
|
||||
total: number;
|
||||
failed: number;
|
||||
/** The lowest App Store Connect quota seen per credential this hour. */
|
||||
quota: Record<string, { limit: number; remaining: number; at: number }>;
|
||||
/** Credentials waiting on a backoff or quota pause, with the time they may start again. */
|
||||
paused: Record<string, number>;
|
||||
}
|
||||
|
||||
interface Item<R> {
|
||||
task: ReadTask<R>;
|
||||
seq: number;
|
||||
attempt: number;
|
||||
notBefore: number;
|
||||
resolve: (r: R) => void;
|
||||
reject: (e: unknown) => void;
|
||||
promise: Promise<R>;
|
||||
}
|
||||
|
||||
const HOUR = 3_600_000;
|
||||
|
||||
export class ReadScheduler {
|
||||
private o: Required<Omit<SchedulerOptions, 'limits' | 'clock' | 'classify'>> & { limits: Record<Lane, number> };
|
||||
private clock: Clock;
|
||||
private classify: (result: unknown, error: unknown, task: ReadTask<unknown>) => Outcome;
|
||||
private queue: Item<unknown>[] = [];
|
||||
private running = new Map<string, Item<unknown>>();
|
||||
private busy: Record<Lane, number> = { apple: 0, play: 0, local: 0 };
|
||||
private nextAt = new Map<string, number>();
|
||||
private quotas = new Map<string, { limit: number; remaining: number; at: number }>();
|
||||
private seq = 0;
|
||||
private done = 0;
|
||||
private failed = 0;
|
||||
private timer: unknown = null;
|
||||
private timerAt = 0;
|
||||
private listeners = new Set<() => void>();
|
||||
|
||||
constructor(o: SchedulerOptions = {}) {
|
||||
this.o = {
|
||||
limits: { apple: 4, play: 4, local: 2, ...o.limits },
|
||||
spacingMs: o.spacingMs ?? 250,
|
||||
reserve: o.reserve ?? 100,
|
||||
slowBelow: o.slowBelow ?? 0.2,
|
||||
pauseMs: o.pauseMs ?? 10 * 60_000,
|
||||
backoffBaseMs: o.backoffBaseMs ?? 5_000,
|
||||
backoffMaxMs: o.backoffMaxMs ?? 5 * 60_000,
|
||||
maxAttempts: o.maxAttempts ?? 4,
|
||||
};
|
||||
this.clock = o.clock ?? realClock;
|
||||
this.classify = o.classify ?? (() => ({}));
|
||||
}
|
||||
|
||||
setLimits(limits: Partial<Record<Lane, number>>): void { this.o.limits = { ...this.o.limits, ...limits }; this.pump(); }
|
||||
|
||||
subscribe(fn: () => void): () => void { this.listeners.add(fn); return () => { this.listeners.delete(fn); }; }
|
||||
|
||||
has(key: string): boolean { return this.running.has(key) || this.queue.some(i => i.task.key === key); }
|
||||
|
||||
enqueue<R>(task: ReadTask<R>): Promise<R> {
|
||||
const run = this.running.get(task.key);
|
||||
if (run) return run.promise as Promise<R>;
|
||||
const queued = this.queue.find(i => i.task.key === task.key);
|
||||
if (queued) {
|
||||
if (task.priority < queued.task.priority) { queued.task = { ...queued.task, priority: task.priority }; this.pump(); }
|
||||
return queued.promise as Promise<R>;
|
||||
}
|
||||
let resolve!: (r: R) => void, reject!: (e: unknown) => void;
|
||||
const promise = new Promise<R>((a, b) => { resolve = a; reject = b; });
|
||||
this.queue.push({ task, seq: ++this.seq, attempt: 0, notBefore: 0, resolve, reject, promise } as unknown as Item<unknown>);
|
||||
this.pump();
|
||||
this.emit();
|
||||
return promise;
|
||||
}
|
||||
|
||||
/** Drops queued tasks (not the running ones); their promises reject with "cancelled". */
|
||||
clear(filter: (key: string) => boolean = () => true): void {
|
||||
const drop = this.queue.filter(i => filter(i.task.key));
|
||||
this.queue = this.queue.filter(i => !filter(i.task.key));
|
||||
for (const i of drop) i.reject(new Error('cancelled'));
|
||||
this.pump();
|
||||
this.emit();
|
||||
}
|
||||
|
||||
stats(): SchedulerStats {
|
||||
const now = this.clock.now();
|
||||
const paused: Record<string, number> = {};
|
||||
for (const [c, at] of this.nextAt) if (at - now > this.o.spacingMs) paused[c] = at;
|
||||
const quota: SchedulerStats['quota'] = {};
|
||||
for (const [c, q] of this.quotas) if (now - q.at < HOUR) quota[c] = q;
|
||||
return { queued: this.queue.length, running: this.running.size, done: this.done, total: this.done + this.running.size + this.queue.length, failed: this.failed, quota, paused };
|
||||
}
|
||||
|
||||
/** Earliest time a credential may start again. */
|
||||
private gap(cred: string, now: number): number {
|
||||
const q = this.quotas.get(cred);
|
||||
if (!q || now - q.at >= HOUR) return this.o.spacingMs;
|
||||
if (q.remaining <= this.o.reserve) return this.o.pauseMs;
|
||||
if (q.remaining < q.limit * this.o.slowBelow) return Math.max(this.o.spacingMs, Math.ceil(HOUR / Math.max(1, q.remaining - this.o.reserve)));
|
||||
return this.o.spacingMs;
|
||||
}
|
||||
|
||||
private ready(i: Item<unknown>, now: number): number {
|
||||
// 0 = can start now; otherwise the time it can (Infinity: waits on a lane slot)
|
||||
let at = i.notBefore;
|
||||
for (const c of i.task.credentials) at = Math.max(at, this.nextAt.get(c) ?? 0);
|
||||
if (i.task.lanes.some(l => this.busy[l] >= Math.max(1, this.o.limits[l]))) return Infinity;
|
||||
return at <= now ? 0 : at;
|
||||
}
|
||||
|
||||
private pump(): void {
|
||||
const now = this.clock.now();
|
||||
let wake = Infinity;
|
||||
for (;;) {
|
||||
const order = [...this.queue].sort((a, b) => a.task.priority - b.task.priority || a.seq - b.seq);
|
||||
let started = false;
|
||||
for (const i of order) {
|
||||
const r = this.ready(i, now);
|
||||
if (r === 0) { this.start(i, now); started = true; break; }
|
||||
if (r !== Infinity) wake = Math.min(wake, r);
|
||||
}
|
||||
if (!started) break;
|
||||
}
|
||||
if (this.timer !== null && (wake === Infinity || wake !== this.timerAt)) { this.clock.clearTimeout(this.timer); this.timer = null; }
|
||||
if (wake !== Infinity && this.timer === null) {
|
||||
this.timerAt = wake;
|
||||
this.timer = this.clock.setTimeout(() => { this.timer = null; this.pump(); }, Math.max(0, wake - now));
|
||||
}
|
||||
}
|
||||
|
||||
private start(i: Item<unknown>, now: number): void {
|
||||
this.queue = this.queue.filter(x => x !== i);
|
||||
this.running.set(i.task.key, i);
|
||||
for (const l of i.task.lanes) this.busy[l]++;
|
||||
for (const c of i.task.credentials) this.nextAt.set(c, now + this.gap(c, now));
|
||||
i.attempt++;
|
||||
this.emit();
|
||||
i.task.run().then(r => this.finish(i, r, undefined, true), e => this.finish(i, undefined, e, false));
|
||||
}
|
||||
|
||||
private finish(i: Item<unknown>, result: unknown, error: unknown, ok: boolean): void {
|
||||
const now = this.clock.now();
|
||||
this.running.delete(i.task.key);
|
||||
for (const l of i.task.lanes) this.busy[l]--;
|
||||
let out: Outcome = {};
|
||||
try { out = this.classify(result, error, i.task); } catch { /* a classifier bug never stalls the queue */ }
|
||||
if (out.quota) {
|
||||
const prev = this.quotas.get(out.quota.credential);
|
||||
// the hour is the key's: keep the lowest count seen in it
|
||||
const q = prev && now - prev.at < HOUR && prev.remaining < out.quota.remaining ? prev : { limit: out.quota.limit, remaining: out.quota.remaining, at: now };
|
||||
this.quotas.set(out.quota.credential, q);
|
||||
this.nextAt.set(out.quota.credential, Math.max(this.nextAt.get(out.quota.credential) ?? 0, now + this.gap(out.quota.credential, now)));
|
||||
}
|
||||
if (out.throttled && i.attempt < this.o.maxAttempts) {
|
||||
const exp = Math.min(this.o.backoffMaxMs, this.o.backoffBaseMs * 2 ** (i.attempt - 1));
|
||||
const delay = Math.round(exp / 2 + this.clock.random() * exp / 2); // "equal jitter"
|
||||
i.notBefore = now + delay;
|
||||
for (const c of i.task.credentials) this.nextAt.set(c, Math.max(this.nextAt.get(c) ?? 0, now + delay));
|
||||
this.queue.push(i);
|
||||
} else {
|
||||
this.done++;
|
||||
if (!ok || out.throttled) this.failed++;
|
||||
if (ok) i.resolve(result); else i.reject(error);
|
||||
}
|
||||
if (!this.queue.length && !this.running.size) { this.done = 0; this.failed = 0; }
|
||||
this.pump();
|
||||
this.emit();
|
||||
}
|
||||
|
||||
private emit(): void { for (const f of this.listeners) { try { f(); } catch { /* listeners never break the queue */ } } }
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue