quotacap 0.0.44 → 0.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/index.d.ts +1 -0
- package/dist/adapters/index.js +11 -6
- package/dist/runtime/poll.d.ts +13 -0
- package/dist/runtime/poll.js +124 -6
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/adapters/index.d.ts
CHANGED
|
@@ -25,6 +25,7 @@ export interface PollAllOptions {
|
|
|
25
25
|
export declare function computeRetryDelayMs(failedAttempt: number, baseMs?: number, capMs?: number, rng?: () => number): number;
|
|
26
26
|
/** Order-preserving concurrency-limited map. */
|
|
27
27
|
export declare function mapWithConcurrency<T, R>(items: T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]>;
|
|
28
|
+
export declare const RETRYABLE_DIAGNOSTIC_CODES: ReadonlySet<string>;
|
|
28
29
|
export declare function isRetryableChildFailure(reason: unknown): boolean;
|
|
29
30
|
export declare function pollAll(enabled: string[], opts?: PollAllOptions): Promise<({
|
|
30
31
|
provider: string;
|
package/dist/adapters/index.js
CHANGED
|
@@ -56,13 +56,18 @@ export async function mapWithConcurrency(items, limit, fn) {
|
|
|
56
56
|
await Promise.all(workers);
|
|
57
57
|
return results;
|
|
58
58
|
}
|
|
59
|
-
// A child that ran and produced a diagnosable output problem is worth
|
|
60
|
-
//
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
|
|
59
|
+
// A child that ran and produced a diagnosable output problem is worth more
|
|
60
|
+
// attempts; a login wall, missing binary, trust prompt, or rate limit will
|
|
61
|
+
// not clear on retry, and a mid-flight child death (killAll, crash) must
|
|
62
|
+
// never respawn — especially during shutdown. Shared with the coordinator,
|
|
63
|
+
// which consults it for both in-generation retries and recovery polls.
|
|
64
|
+
export const RETRYABLE_DIAGNOSTIC_CODES = new Set([
|
|
65
|
+
"parse_error",
|
|
66
|
+
"network",
|
|
67
|
+
"service_unavailable",
|
|
68
|
+
]);
|
|
64
69
|
export function isRetryableChildFailure(reason) {
|
|
65
|
-
return
|
|
70
|
+
return RETRYABLE_DIAGNOSTIC_CODES.has(classifyFailure("Provider", reason).diagnosticCode);
|
|
66
71
|
}
|
|
67
72
|
async function pollOnceWithGate(id, adapter, timeout) {
|
|
68
73
|
// One controller per attempt: a timeout aborts only that attempt's
|
package/dist/runtime/poll.d.ts
CHANGED
|
@@ -34,6 +34,15 @@ export interface CoordinatorState {
|
|
|
34
34
|
lastCompletedPollAt: string | null;
|
|
35
35
|
lastResult: RefreshResult | null;
|
|
36
36
|
}
|
|
37
|
+
export interface RecoveryOptions {
|
|
38
|
+
/** First recovery delay. Default 60000. Doubles per consecutive failure. */
|
|
39
|
+
baseDelayMs?: number;
|
|
40
|
+
/** Recovery delay ceiling. Default 600000. */
|
|
41
|
+
maxDelayMs?: number;
|
|
42
|
+
/** Stop auto-recovery after this many consecutive failures per provider.
|
|
43
|
+
* Scheduled ticks keep trying; a success resets the count. Default 5. */
|
|
44
|
+
maxFailures?: number;
|
|
45
|
+
}
|
|
37
46
|
export interface CoordinatorOptions {
|
|
38
47
|
db: any;
|
|
39
48
|
enabledProviders: string[];
|
|
@@ -48,6 +57,10 @@ export interface CoordinatorOptions {
|
|
|
48
57
|
dataDir?: string;
|
|
49
58
|
/** Opt-in forensics: write a redacted evidence bundle per poll failure. */
|
|
50
59
|
debugFailureBundles?: boolean;
|
|
60
|
+
/** Auto-recovery: re-poll providers whose window expired or went stale
|
|
61
|
+
* instead of waiting for the next scheduled tick. Enabled by default;
|
|
62
|
+
* pass false to disable. */
|
|
63
|
+
recovery?: false | RecoveryOptions;
|
|
51
64
|
}
|
|
52
65
|
export interface Coordinator {
|
|
53
66
|
refresh(): Promise<RefreshResult>;
|
package/dist/runtime/poll.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// Poll coordinator: scheduled and manual refresh share one in-flight poll
|
|
2
2
|
// with a completion-measured cooldown (decisions D3/D4).
|
|
3
|
-
import { pollAll, ADAPTER_TIMEOUTS } from "../adapters/index.js";
|
|
4
|
-
import { getAllLatest, upsertQuota } from "../store/quotas.js";
|
|
3
|
+
import { pollAll, ADAPTER_TIMEOUTS, RETRYABLE_DIAGNOSTIC_CODES } from "../adapters/index.js";
|
|
4
|
+
import { getAllLatest, getLatestByProvider, upsertQuota } from "../store/quotas.js";
|
|
5
|
+
import { STALE_MS } from "../advisory/snapshot.js";
|
|
5
6
|
import { readServiceMetadata } from "../config.js";
|
|
6
7
|
import { writeFailureBundle, providerBin, getCliVersion } from "../diagnostics/bundle.js";
|
|
7
8
|
import { VERSION } from "../version.js";
|
|
@@ -56,6 +57,11 @@ export function createCoordinator(opts) {
|
|
|
56
57
|
const onSettled = opts.onSettled;
|
|
57
58
|
const dataDir = opts.dataDir;
|
|
58
59
|
const debugFailureBundles = opts.debugFailureBundles === true;
|
|
60
|
+
const recoveryEnabled = opts.recovery !== false;
|
|
61
|
+
const recoveryOpts = typeof opts.recovery === "object" ? opts.recovery : {};
|
|
62
|
+
const recoveryBaseMs = recoveryOpts.baseDelayMs ?? 60000;
|
|
63
|
+
const recoveryMaxMs = recoveryOpts.maxDelayMs ?? 600000;
|
|
64
|
+
const recoveryMaxFailures = recoveryOpts.maxFailures ?? 5;
|
|
59
65
|
let inFlight = null;
|
|
60
66
|
let lastResult = null;
|
|
61
67
|
let lastCompletedPollAt = null;
|
|
@@ -64,6 +70,9 @@ export function createCoordinator(opts) {
|
|
|
64
70
|
let started = false;
|
|
65
71
|
let intervalMs = 0;
|
|
66
72
|
let timer = null;
|
|
73
|
+
let recoveryTimer = null;
|
|
74
|
+
const recoveryFailures = new Map();
|
|
75
|
+
const recoveryExhausted = new Set();
|
|
67
76
|
function arm() {
|
|
68
77
|
if (timer)
|
|
69
78
|
clearTimeout(timer);
|
|
@@ -81,9 +90,9 @@ export function createCoordinator(opts) {
|
|
|
81
90
|
enabledProviders,
|
|
82
91
|
}));
|
|
83
92
|
}
|
|
84
|
-
async function runGeneration() {
|
|
93
|
+
async function runGeneration(ids = enabledProviders) {
|
|
85
94
|
const attemptedAt = new Date(now()).toISOString();
|
|
86
|
-
const rows = await pollFn(
|
|
95
|
+
const rows = await pollFn(ids);
|
|
87
96
|
const completedAt = new Date(now()).toISOString();
|
|
88
97
|
// Self-fence before any poll write (decision D1).
|
|
89
98
|
if (ownershipVerify && !ownershipVerify()) {
|
|
@@ -202,12 +211,117 @@ export function createCoordinator(opts) {
|
|
|
202
211
|
}
|
|
203
212
|
}
|
|
204
213
|
}
|
|
214
|
+
// Partial (recovery) generations merge into the previous full result so
|
|
215
|
+
// a refresh response never drops providers that were not re-polled.
|
|
216
|
+
if (lastResult && ids.length < enabledProviders.length) {
|
|
217
|
+
const idsSet = new Set(ids);
|
|
218
|
+
result.fulfilled = [
|
|
219
|
+
...lastResult.fulfilled.filter((f) => !idsSet.has(f?.provider)),
|
|
220
|
+
...result.fulfilled,
|
|
221
|
+
];
|
|
222
|
+
result.rejected = [
|
|
223
|
+
...lastResult.rejected.filter((r) => !idsSet.has(r.provider)),
|
|
224
|
+
...result.rejected,
|
|
225
|
+
];
|
|
226
|
+
result.results = [
|
|
227
|
+
...lastResult.results.filter((r) => !idsSet.has(r.provider)),
|
|
228
|
+
...result.results,
|
|
229
|
+
];
|
|
230
|
+
result.degraded = result.rejected.length > 0;
|
|
231
|
+
}
|
|
205
232
|
lastResult = result;
|
|
206
233
|
lastCompletedPollAt = completedAt;
|
|
207
234
|
completedAtMs = new Date(completedAt).getTime();
|
|
235
|
+
// Recovery bookkeeping before re-arming: an attempt is an attempt,
|
|
236
|
+
// whether it came from the schedule, a manual refresh, or recovery.
|
|
237
|
+
for (const id of ids) {
|
|
238
|
+
if (needsRecovery(id)) {
|
|
239
|
+
recoveryFailures.set(id, (recoveryFailures.get(id) ?? 0) + 1);
|
|
240
|
+
}
|
|
241
|
+
else {
|
|
242
|
+
recoveryFailures.delete(id);
|
|
243
|
+
recoveryExhausted.delete(id);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
evaluateRecovery();
|
|
208
247
|
return result;
|
|
209
248
|
}
|
|
210
|
-
|
|
249
|
+
/** A provider needs recovery when its row is expired but a fresh poll
|
|
250
|
+
* could plausibly fix it: reset passed or stale, and the last failure —
|
|
251
|
+
* if any — is a retryable or timeout cause, never auth or a hard wall. */
|
|
252
|
+
function needsRecovery(id) {
|
|
253
|
+
if (id === "manual")
|
|
254
|
+
return false;
|
|
255
|
+
const quota = getLatestByProvider(db, id);
|
|
256
|
+
if (!quota)
|
|
257
|
+
return false;
|
|
258
|
+
const nowMs = now();
|
|
259
|
+
const resetsMs = new Date(quota.resetsAt).getTime();
|
|
260
|
+
const fetchedMs = new Date(quota.fetchedAt).getTime();
|
|
261
|
+
const resetPassed = !Number.isNaN(resetsMs) && resetsMs <= nowMs;
|
|
262
|
+
const stale = !Number.isNaN(fetchedMs) && nowMs - fetchedMs > STALE_MS;
|
|
263
|
+
if (!resetPassed && !stale)
|
|
264
|
+
return false;
|
|
265
|
+
const attempt = getAttempt(db, id);
|
|
266
|
+
if (attempt && !attempt.success && attempt.diagnosticCode) {
|
|
267
|
+
const code = attempt.diagnosticCode;
|
|
268
|
+
if (code !== "timeout" && !RETRYABLE_DIAGNOSTIC_CODES.has(code))
|
|
269
|
+
return false;
|
|
270
|
+
}
|
|
271
|
+
return true;
|
|
272
|
+
}
|
|
273
|
+
function evaluateRecovery() {
|
|
274
|
+
if (recoveryTimer) {
|
|
275
|
+
clearTimeout(recoveryTimer);
|
|
276
|
+
recoveryTimer = null;
|
|
277
|
+
}
|
|
278
|
+
if (closing || !recoveryEnabled)
|
|
279
|
+
return;
|
|
280
|
+
const needy = enabledProviders.filter((id) => needsRecovery(id) && (recoveryFailures.get(id) ?? 0) < recoveryMaxFailures);
|
|
281
|
+
for (const id of enabledProviders) {
|
|
282
|
+
if (needsRecovery(id) && (recoveryFailures.get(id) ?? 0) >= recoveryMaxFailures) {
|
|
283
|
+
if (!recoveryExhausted.has(id)) {
|
|
284
|
+
recoveryExhausted.add(id);
|
|
285
|
+
console.log(`[quotacap] [${id}] recovery exhausted; waiting for the next scheduled poll`);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
if (needy.length === 0)
|
|
290
|
+
return;
|
|
291
|
+
const worst = Math.max(...needy.map((id) => recoveryFailures.get(id) ?? 0));
|
|
292
|
+
const delay = Math.min(recoveryMaxMs, recoveryBaseMs * 2 ** worst);
|
|
293
|
+
console.log(`[quotacap] [${needy.join(", ")}] scheduling recovery poll in ${Math.round(delay / 1000)}s`);
|
|
294
|
+
recoveryTimer = setTimeout(() => {
|
|
295
|
+
recoveryTimer = null;
|
|
296
|
+
void fireRecovery(needy);
|
|
297
|
+
}, delay);
|
|
298
|
+
}
|
|
299
|
+
async function fireRecovery(ids) {
|
|
300
|
+
if (closing)
|
|
301
|
+
return;
|
|
302
|
+
// Self-fence on wake, like scheduledTick: do not spawn doomed children.
|
|
303
|
+
if (ownershipVerify && !ownershipVerify()) {
|
|
304
|
+
closing = true;
|
|
305
|
+
onOwnershipLost();
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
const targets = ids.filter((id) => needsRecovery(id));
|
|
309
|
+
if (targets.length === 0) {
|
|
310
|
+
evaluateRecovery();
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
try {
|
|
314
|
+
await (inFlight ?? launch(targets));
|
|
315
|
+
}
|
|
316
|
+
catch (e) {
|
|
317
|
+
const detail = classifyFailure("all", e).errorDetail;
|
|
318
|
+
console.warn("[quotacap] recovery poll failed", detail);
|
|
319
|
+
}
|
|
320
|
+
finally {
|
|
321
|
+
evaluateRecovery();
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
function launch(ids = enabledProviders) {
|
|
211
325
|
// Gate first: inFlight is set before pollFn runs, so getState()
|
|
212
326
|
// reports in-progress from inside the poll itself.
|
|
213
327
|
let resolve;
|
|
@@ -222,7 +336,7 @@ export function createCoordinator(opts) {
|
|
|
222
336
|
inFlight = null;
|
|
223
337
|
};
|
|
224
338
|
gate.then(clear, clear);
|
|
225
|
-
runGeneration().then(resolve, reject);
|
|
339
|
+
runGeneration(ids).then(resolve, reject);
|
|
226
340
|
return gate;
|
|
227
341
|
}
|
|
228
342
|
function refresh() {
|
|
@@ -281,6 +395,10 @@ export function createCoordinator(opts) {
|
|
|
281
395
|
clearTimeout(timer);
|
|
282
396
|
timer = null;
|
|
283
397
|
}
|
|
398
|
+
if (recoveryTimer) {
|
|
399
|
+
clearTimeout(recoveryTimer);
|
|
400
|
+
recoveryTimer = null;
|
|
401
|
+
}
|
|
284
402
|
}
|
|
285
403
|
function getState() {
|
|
286
404
|
const polling = inFlight !== null
|
package/dist/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
// generated by scripts/build-embed.mjs — do not edit
|
|
2
|
-
export const VERSION = "0.0.
|
|
2
|
+
export const VERSION = "0.0.45";
|