teamclaude-cloud 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +320 -0
- package/package.json +38 -0
- package/src/account-manager.js +1271 -0
- package/src/cloud.js +618 -0
- package/src/config.js +148 -0
- package/src/index.js +1726 -0
- package/src/oauth.js +347 -0
- package/src/server.js +1267 -0
- package/src/tui.js +843 -0
|
@@ -0,0 +1,1271 @@
|
|
|
1
|
+
import { refreshAccessToken, isTokenExpiringSoon } from './oauth.js';
|
|
2
|
+
|
|
3
|
+
/** Coerce a per-account / global concurrency cap to a positive integer, else fallback. */
|
|
4
|
+
function coerceMaxConcurrent(value, fallback) {
|
|
5
|
+
return Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
function emptyQuota() {
|
|
9
|
+
return {
|
|
10
|
+
// Standard API rate limits (API key accounts)
|
|
11
|
+
tokensLimit: null,
|
|
12
|
+
tokensRemaining: null,
|
|
13
|
+
requestsLimit: null,
|
|
14
|
+
requestsRemaining: null,
|
|
15
|
+
// Unified rate limits (Claude Max accounts)
|
|
16
|
+
unified5h: null, // utilization 0-1
|
|
17
|
+
unified7d: null, // utilization 0-1
|
|
18
|
+
unified5hReset: null, // ms timestamp
|
|
19
|
+
unified7dReset: null, // ms timestamp
|
|
20
|
+
unifiedStatus: null, // allowed | allowed_warning | rejected
|
|
21
|
+
// Model-scoped weekly windows, keyed by header window label — e.g. `7d_oi`,
|
|
22
|
+
// the separate weekly limit for the top model tier shown as "Fable" in
|
|
23
|
+
// Claude's usage UI. Parsed generically from
|
|
24
|
+
// anthropic-ratelimit-unified-<window>-* so a renamed/added window keeps
|
|
25
|
+
// being tracked without a code change. Display-only: it never feeds
|
|
26
|
+
// availability, because an account over its Fable weekly limit still
|
|
27
|
+
// serves every other model.
|
|
28
|
+
modelWeekly: {}, // { '7d_oi': { utilization: 0-1, reset: msTimestamp } }
|
|
29
|
+
resetsAt: null, // soonest standard reset (session-order fallback)
|
|
30
|
+
// Token and request windows can reset at DIFFERENT times; tracked separately
|
|
31
|
+
// so retry-after can wait for whichever over-threshold window frees last
|
|
32
|
+
// (resetsAt alone collapses them, preferring the sooner token reset).
|
|
33
|
+
tokensReset: null, // standard token-window reset (date string)
|
|
34
|
+
requestsReset: null, // standard request-window reset (date string)
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export class AccountManager {
|
|
39
|
+
constructor(accounts, switchThreshold = 0.98, reevalIntervalMs = 5 * 60 * 1000, maxConcurrentDefault = 3, overflowQueueMaxDepth = 256) {
|
|
40
|
+
this.maxConcurrentDefault = coerceMaxConcurrent(maxConcurrentDefault, 3);
|
|
41
|
+
// Hard cap on the overflow queue so a flood of concurrent requests can't grow
|
|
42
|
+
// it (and the buffered bodies / sockets / timers it pins) without bound. Past
|
|
43
|
+
// this depth acquireAccount rejects immediately (→ 429) instead of queuing.
|
|
44
|
+
this.maxQueueDepth = Number.isFinite(overflowQueueMaxDepth) && overflowQueueMaxDepth >= 0
|
|
45
|
+
? Math.floor(overflowQueueMaxDepth) : 256;
|
|
46
|
+
this.accounts = accounts.map((acct, index) => ({
|
|
47
|
+
index,
|
|
48
|
+
name: acct.name,
|
|
49
|
+
type: acct.type,
|
|
50
|
+
accountUuid: acct.accountUuid || null,
|
|
51
|
+
credential: acct.accessToken || acct.apiKey,
|
|
52
|
+
refreshToken: acct.refreshToken || null,
|
|
53
|
+
expiresAt: acct.expiresAt || null,
|
|
54
|
+
status: 'active',
|
|
55
|
+
// Manual on/off switch. A disabled account is excluded from ALL rotation
|
|
56
|
+
// (warm-up, use-or-lose selection, recover, acquire) via _isAvailable —
|
|
57
|
+
// in-flight requests still drain, but no new request is routed to it.
|
|
58
|
+
// Defaults to enabled; only an explicit `enabled: false` disables it.
|
|
59
|
+
enabled: acct.enabled !== false,
|
|
60
|
+
// Explicit selection priority: lower number = preferred first. Null/unset
|
|
61
|
+
// means "no preference" — selection then falls back to use-or-lose. So a
|
|
62
|
+
// config with no priorities behaves exactly as before.
|
|
63
|
+
priority: Number.isFinite(acct.priority) ? Math.floor(acct.priority) : null,
|
|
64
|
+
quota: emptyQuota(),
|
|
65
|
+
usage: {
|
|
66
|
+
totalInputTokens: 0,
|
|
67
|
+
totalOutputTokens: 0,
|
|
68
|
+
totalRequests: 0,
|
|
69
|
+
lastUsed: null,
|
|
70
|
+
},
|
|
71
|
+
rateLimitedUntil: null,
|
|
72
|
+
// Concurrency: how many requests are in flight through this account right
|
|
73
|
+
// now, and the per-account cap above which the selector treats it as
|
|
74
|
+
// momentarily full (so concurrent load spreads to other accounts).
|
|
75
|
+
inflight: 0,
|
|
76
|
+
maxConcurrent: coerceMaxConcurrent(acct.maxConcurrent, this.maxConcurrentDefault),
|
|
77
|
+
}));
|
|
78
|
+
this.currentIndex = 0;
|
|
79
|
+
this.switchThreshold = switchThreshold;
|
|
80
|
+
this.reevalIntervalMs = reevalIntervalMs;
|
|
81
|
+
this.lastEvalAt = 0; // 0 forces a priority pick on the first request
|
|
82
|
+
this.maxWarmupTries = 3; // give up warming an account after this many unmeasured attempts
|
|
83
|
+
// Slow retry backstop for the active warm-up convergence cap: a capped
|
|
84
|
+
// account (deterministic fruitless probes) is retried once per this window,
|
|
85
|
+
// so an upstream that only LOOKED deterministic (e.g. temporarily
|
|
86
|
+
// header-less) recovers without a restart, while a truly pathological one
|
|
87
|
+
// costs at most one probe per window.
|
|
88
|
+
this.probeRetryAfterMs = 15 * 60 * 1000;
|
|
89
|
+
this._warmupCursor = 0; // round-robin pointer used during warm-up
|
|
90
|
+
this._waiters = []; // overflow queue: requests waiting for a free slot
|
|
91
|
+
// Soft connection→account affinity (keyed by the client socket). Keeps one
|
|
92
|
+
// keep-alive connection's *sequential* requests on the same account so
|
|
93
|
+
// Anthropic's per-account prompt cache stays warm. A WeakMap so an entry is
|
|
94
|
+
// GC'd when the socket is collected (connection closed) — no manual cleanup,
|
|
95
|
+
// no leak. The stored value is the account *object* (not its index, which
|
|
96
|
+
// shifts on removeAccount); a stale entry is detected and ignored.
|
|
97
|
+
this._affinity = new WeakMap();
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Get the account to use for the next request.
|
|
102
|
+
*
|
|
103
|
+
* Policy:
|
|
104
|
+
* - Cold-start warm-up: while any available account is still unmeasured,
|
|
105
|
+
* route to it so its quota (usage % / reset) gets populated before any
|
|
106
|
+
* priority decision is made on incomplete data.
|
|
107
|
+
* - If the current account is unavailable (near quota / throttled / error),
|
|
108
|
+
* switch immediately to the highest-priority account. This is the old
|
|
109
|
+
* "switch at threshold" trigger — but it now picks by priority rather
|
|
110
|
+
* than round-robin to the next index.
|
|
111
|
+
* - Otherwise re-evaluate priority at most once per `reevalIntervalMs`
|
|
112
|
+
* (default 5 min) and switch if a higher-priority account exists. Set
|
|
113
|
+
* `reevalIntervalMs <= 0` (config `reevalIntervalMs: 0`) to disable this
|
|
114
|
+
* timer entirely — the account then only changes when it becomes
|
|
115
|
+
* unavailable or via per-request 429 failover.
|
|
116
|
+
* - Between re-evaluations the current account is sticky, so a request
|
|
117
|
+
* stream stays on one account and keeps Anthropic's per-account prompt
|
|
118
|
+
* cache warm.
|
|
119
|
+
*
|
|
120
|
+
* Priority is "use-or-lose": soonest WEEKLY (7d) reset first, then soonest
|
|
121
|
+
* session reset, then lowest session usage — so quota about to reset (and
|
|
122
|
+
* otherwise be wasted) is consumed first, starting with the scarcer weekly
|
|
123
|
+
* window. Returns null if every account is exhausted.
|
|
124
|
+
*/
|
|
125
|
+
getActiveAccount(exclude = null) {
|
|
126
|
+
const now = Date.now();
|
|
127
|
+
|
|
128
|
+
// Per-request failover: a prior account already returned a non-quota 429
|
|
129
|
+
// for THIS request (those accounts are in `exclude`, a Set of objects). Pick another available
|
|
130
|
+
// account by priority WITHOUT touching the sticky primary or warm-up state
|
|
131
|
+
// — this diverts only the overflow of one request; steady-state selection
|
|
132
|
+
// still prefers the use-or-lose primary, keeping its prompt cache warm.
|
|
133
|
+
// Returns null once every available account has been tried this request.
|
|
134
|
+
if (exclude && exclude.size) return this._selectBest(exclude);
|
|
135
|
+
|
|
136
|
+
const current = this.accounts[this.currentIndex];
|
|
137
|
+
|
|
138
|
+
// Cold-start warm-up: until every available account has been measured at
|
|
139
|
+
// least once, round-robin across the unmeasured accounts so their quota
|
|
140
|
+
// (usage % / reset) gets populated. Round-robin (not "first unmeasured")
|
|
141
|
+
// means a concurrent startup burst of any size spreads evenly instead of
|
|
142
|
+
// hammering one unknown-quota account. Only once all are measured does the
|
|
143
|
+
// use-or-lose priority below take over — with complete data.
|
|
144
|
+
const warmup = this._nextWarmup();
|
|
145
|
+
if (warmup) {
|
|
146
|
+
if (warmup.index !== this.currentIndex) {
|
|
147
|
+
console.log(`[TeamClaude] Warm-up: measuring account "${warmup.name}"`);
|
|
148
|
+
this.currentIndex = warmup.index;
|
|
149
|
+
}
|
|
150
|
+
return warmup;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
if (!this._isAvailable(current)) {
|
|
154
|
+
const best = this._selectBest();
|
|
155
|
+
if (best) {
|
|
156
|
+
if (best.index !== this.currentIndex) {
|
|
157
|
+
console.log(`[TeamClaude] Switched to account "${best.name}" (current unavailable)`);
|
|
158
|
+
}
|
|
159
|
+
this.currentIndex = best.index;
|
|
160
|
+
this.lastEvalAt = now;
|
|
161
|
+
}
|
|
162
|
+
return best;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// Periodic re-prioritization. Disabled when reevalIntervalMs <= 0: the
|
|
166
|
+
// current account then stays sticky and only changes when it becomes
|
|
167
|
+
// unavailable (exhausted / throttled / error) or via per-request 429
|
|
168
|
+
// failover — no timer-driven switching.
|
|
169
|
+
if (this.reevalIntervalMs > 0 && now - this.lastEvalAt >= this.reevalIntervalMs) {
|
|
170
|
+
this.lastEvalAt = now;
|
|
171
|
+
const best = this._selectBest();
|
|
172
|
+
if (best && best.index !== this.currentIndex) {
|
|
173
|
+
console.log(`[TeamClaude] Re-prioritized to account "${best.name}" (weekly reset soonest)`);
|
|
174
|
+
this.currentIndex = best.index;
|
|
175
|
+
return best;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// While the current account is still unmeasured, keep load-balancing via
|
|
180
|
+
// _selectBest (which rotates among equal-rank accounts) instead of sticking
|
|
181
|
+
// to an unknown-quota account — so a cold-start burst stays spread even
|
|
182
|
+
// after per-account warm-up attempts are exhausted.
|
|
183
|
+
if (!this._isMeasured(current)) {
|
|
184
|
+
const best = this._selectBest();
|
|
185
|
+
if (best) {
|
|
186
|
+
this.currentIndex = best.index;
|
|
187
|
+
return best;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
return current;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// ── Concurrency layer: per-account in-flight cap + overflow queue ──────────
|
|
195
|
+
//
|
|
196
|
+
// getActiveAccount() above picks ONE account (sticky, use-or-lose). On its own
|
|
197
|
+
// that funnels every concurrent terminal onto the same account, which then hits
|
|
198
|
+
// Anthropic's per-account rate / concurrency limit (429) while other accounts
|
|
199
|
+
// sit idle with quota to spare. The layer below fixes that PROACTIVELY: each
|
|
200
|
+
// account carries an `inflight` counter and a `maxConcurrent` cap, and
|
|
201
|
+
// acquireAccount() treats a capped account as momentarily unavailable (folds it
|
|
202
|
+
// into the exclude set). The existing priority logic then naturally spreads
|
|
203
|
+
// load to the next account — filling A up to its cap, then B, then C, by
|
|
204
|
+
// use-or-lose priority. When every available account is at its cap the request
|
|
205
|
+
// waits briefly for a slot to free (overflow queue) instead of 429-storming.
|
|
206
|
+
|
|
207
|
+
/** Has this account a free concurrency slot? */
|
|
208
|
+
_hasCapacity(account) {
|
|
209
|
+
return account.inflight < account.maxConcurrent;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Resolve an account handle to the live account object. Accepts the object
|
|
214
|
+
* itself (reindex-safe — what server.js passes) or a numeric index (legacy /
|
|
215
|
+
* tests). All public per-account methods route their first arg through this so
|
|
216
|
+
* a stale index captured before a removeAccount() can't hit the wrong account.
|
|
217
|
+
*/
|
|
218
|
+
_resolve(accountOrIndex) {
|
|
219
|
+
return typeof accountOrIndex === 'number' ? this.accounts[accountOrIndex] : accountOrIndex;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Available accounts currently at their concurrency cap, as a Set of account
|
|
224
|
+
* OBJECTS (not indexes). Object identity is stable across a removeAccount()
|
|
225
|
+
* re-index, so an exclude/capped set captured before the request awaits
|
|
226
|
+
* upstream can't later point at the wrong account.
|
|
227
|
+
*/
|
|
228
|
+
_cappedSet(exclude = null) {
|
|
229
|
+
const capped = new Set();
|
|
230
|
+
for (const a of this.accounts) {
|
|
231
|
+
if (exclude && exclude.has(a)) continue;
|
|
232
|
+
if (this._isAvailable(a) && !this._hasCapacity(a)) capped.add(a);
|
|
233
|
+
}
|
|
234
|
+
return capped;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** Is there an available account with a free slot (not excluded)? Non-mutating. (`exclude` = Set of account objects.) */
|
|
238
|
+
anyUsable(exclude = null) {
|
|
239
|
+
return this.accounts.some(a =>
|
|
240
|
+
this._isAvailable(a) && this._hasCapacity(a) && !(exclude && exclude.has(a)));
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Is there an available-but-capped account (not excluded)? A freed slot could serve it. (`exclude` = Set of account objects.) */
|
|
244
|
+
anyCapped(exclude = null) {
|
|
245
|
+
return this.accounts.some(a =>
|
|
246
|
+
this._isAvailable(a) && !this._hasCapacity(a) && !(exclude && exclude.has(a)));
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Synchronously pick + reserve the best account that is available AND has a
|
|
251
|
+
* free concurrency slot, honoring `exclude`. Capped accounts are folded into
|
|
252
|
+
* the exclusion so the existing getActiveAccount / _selectBest priority logic
|
|
253
|
+
* (warm-up, use-or-lose, recover) only ever chooses an account that can take
|
|
254
|
+
* the request. Increments the chosen account's inflight. Returns null when
|
|
255
|
+
* nothing is currently acquirable (all exhausted, excluded, or capped).
|
|
256
|
+
*
|
|
257
|
+
* Single-threaded JS keeps this race-free: there is no await between selecting
|
|
258
|
+
* the account and the inflight++ that reserves its slot.
|
|
259
|
+
*/
|
|
260
|
+
_tryAcquire(exclude = null, affinityKey = null) {
|
|
261
|
+
// Only an object/function is a valid WeakMap key. Ignore anything else (a
|
|
262
|
+
// primitive key from an external caller would otherwise throw on get/set).
|
|
263
|
+
const affOk = affinityKey != null
|
|
264
|
+
&& (typeof affinityKey === 'object' || typeof affinityKey === 'function');
|
|
265
|
+
|
|
266
|
+
// Connection affinity (cache locality): prefer the account this connection
|
|
267
|
+
// already used — but only as a *soft* hint, and DEFER to cold-start warm-up.
|
|
268
|
+
// While any account still needs measuring, skip affinity so it can't pin all
|
|
269
|
+
// of a connection's traffic to one account and starve the others of quota
|
|
270
|
+
// data (warm-up round-robins the unmeasured accounts instead). Once measured,
|
|
271
|
+
// affinity is honored only when that account is still available, has a free
|
|
272
|
+
// slot, and isn't excluded for this request; otherwise it falls through to
|
|
273
|
+
// normal selection. So it never exceeds a cap, revives an exhausted account,
|
|
274
|
+
// or disturbs use-or-lose for new connections. (`accounts[idx] === a` rejects
|
|
275
|
+
// a stale entry left by a removeAccount that re-indexed the array.)
|
|
276
|
+
if (affOk && !this.accounts.some(acc => this._isWarmupTarget(acc))) {
|
|
277
|
+
const a = this._affinity.get(affinityKey);
|
|
278
|
+
// Require the home to be MEASURED — not just past its warm-up tries. A
|
|
279
|
+
// headerless account stays unmeasured forever; pinning a connection to it
|
|
280
|
+
// would bypass getActiveAccount's unmeasured-rebalance (which keeps
|
|
281
|
+
// spreading to gather quota data / let tokens refresh on use). Once an
|
|
282
|
+
// account returns rate-limit headers (every real Anthropic response does),
|
|
283
|
+
// affinity engages normally.
|
|
284
|
+
if (a && this.accounts[a.index] === a && this._isMeasured(a) && this._isAvailable(a)
|
|
285
|
+
&& this._hasCapacity(a) && !(exclude && exclude.has(a))) {
|
|
286
|
+
a.inflight++;
|
|
287
|
+
return a;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
const capped = this._cappedSet(exclude);
|
|
292
|
+
const eff = ((exclude && exclude.size) || capped.size)
|
|
293
|
+
? new Set([...(exclude || []), ...capped])
|
|
294
|
+
: null;
|
|
295
|
+
// eff === null → full sticky / warm-up path (cold start, nothing capped).
|
|
296
|
+
// eff set → getActiveAccount routes to _selectBest(eff), which already skips
|
|
297
|
+
// every excluded + capped account.
|
|
298
|
+
const account = eff ? this.getActiveAccount(eff) : this.getActiveAccount();
|
|
299
|
+
if (account && this._isAvailable(account) && this._hasCapacity(account)
|
|
300
|
+
&& !(eff && eff.has(account))) {
|
|
301
|
+
account.inflight++;
|
|
302
|
+
// (Re)write affinity ONLY when the connection has no still-usable home.
|
|
303
|
+
// Reaching this fall-through path means we left the home account — but that
|
|
304
|
+
// can be merely transient: the home may be momentarily capped (overflow
|
|
305
|
+
// spill) or failover-excluded for THIS request, yet still perfectly
|
|
306
|
+
// available. Overwriting it then would let one blip permanently evict the
|
|
307
|
+
// connection from its cache-warm account. So keep an available home (even
|
|
308
|
+
// capped/excluded right now); replace it only when it's genuinely gone
|
|
309
|
+
// (removed, unavailable, or exhausted — `_isAvailable` is false).
|
|
310
|
+
if (affOk) {
|
|
311
|
+
const home = this._affinity.get(affinityKey);
|
|
312
|
+
const homeUsable = home && this.accounts[home.index] === home && this._isAvailable(home);
|
|
313
|
+
if (!homeUsable) this._affinity.set(affinityKey, account);
|
|
314
|
+
}
|
|
315
|
+
return account;
|
|
316
|
+
}
|
|
317
|
+
return null;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* Acquire an account for a request, reserving one of its concurrency slots.
|
|
322
|
+
* If none is immediately acquirable but an available account is merely at its
|
|
323
|
+
* cap (overflow), wait up to `timeoutMs` for a slot to free — a releaseAccount
|
|
324
|
+
* elsewhere wakes the waiter. Returns null when every account is genuinely
|
|
325
|
+
* unavailable (quota-exhausted / auth-error / excluded) or the wait times out,
|
|
326
|
+
* so the caller surfaces a 429 for the client to back off on.
|
|
327
|
+
*
|
|
328
|
+
* The caller MUST releaseAccount(account) exactly once when the request
|
|
329
|
+
* (including any streamed body) finishes — pass the returned account OBJECT,
|
|
330
|
+
* not its index, so a concurrent removeAccount() can't misattribute the slot.
|
|
331
|
+
* `exclude` is a Set of account OBJECTS (per-request failover).
|
|
332
|
+
*/
|
|
333
|
+
async acquireAccount(exclude = null, timeoutMs = 0, signal = null, affinityKey = null) {
|
|
334
|
+
if (signal?.aborted) return null;
|
|
335
|
+
const account = this._tryAcquire(exclude, affinityKey);
|
|
336
|
+
if (account) return account;
|
|
337
|
+
// Queue only when the blockage is cap-saturation (a slot WILL free as
|
|
338
|
+
// in-flight requests finish) AND the queue isn't already full. If no
|
|
339
|
+
// available account exists at all, or the queue is at its depth cap, return
|
|
340
|
+
// null and let the caller 429 — never grow the backlog without bound.
|
|
341
|
+
if (timeoutMs <= 0 || !this.anyCapped(exclude) || this.isQueueFull()) return null;
|
|
342
|
+
return this._enqueue(exclude, timeoutMs, signal, affinityKey);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/** Is the overflow queue at its depth cap? */
|
|
346
|
+
isQueueFull() {
|
|
347
|
+
return this._waiters.length >= this.maxQueueDepth;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* Upper bound on concurrent in-flight requests the proxy may admit (server.js
|
|
352
|
+
* caps `inFlightProxied` to this to bound buffered memory): each ENABLED
|
|
353
|
+
* account contributes its full cap (capacity it can still take), each DISABLED
|
|
354
|
+
* account contributes only its *current* in-flight (requests still draining —
|
|
355
|
+
* it accepts no new ones), plus the queue depth.
|
|
356
|
+
*
|
|
357
|
+
* This is the tightest bound that's still safe: it covers the draining requests
|
|
358
|
+
* on a just-disabled account (so they can't push inFlightProxied over the
|
|
359
|
+
* ceiling and 429 traffic the enabled accounts could serve), without admitting
|
|
360
|
+
* fresh requests against a disabled account's dead future capacity (which could
|
|
361
|
+
* only be buffered and then 429'd at acquire). As those draining requests
|
|
362
|
+
* finish, the disabled account's contribution falls to zero.
|
|
363
|
+
*/
|
|
364
|
+
totalCapacity() {
|
|
365
|
+
const caps = this.accounts.reduce(
|
|
366
|
+
(sum, a) => sum + (a.enabled === false ? a.inflight : a.maxConcurrent), 0);
|
|
367
|
+
return caps + this.maxQueueDepth;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
_enqueue(exclude, timeoutMs, signal = null, affinityKey = null) {
|
|
371
|
+
return new Promise(resolve => {
|
|
372
|
+
const waiter = { exclude, resolve, done: false, timer: null, signal, onAbort: null, affinityKey };
|
|
373
|
+
waiter.timer = setTimeout(() => this._settleWaiter(waiter, null), timeoutMs);
|
|
374
|
+
// Cancel the wait if the client disconnects — otherwise an aborted request
|
|
375
|
+
// would still acquire a slot later and be dispatched upstream, burning quota.
|
|
376
|
+
if (signal) {
|
|
377
|
+
waiter.onAbort = () => this._settleWaiter(waiter, null);
|
|
378
|
+
signal.addEventListener('abort', waiter.onAbort, { once: true });
|
|
379
|
+
}
|
|
380
|
+
this._waiters.push(waiter);
|
|
381
|
+
});
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/** Resolve a queued waiter exactly once, cleaning up its timer/abort listener. */
|
|
385
|
+
_settleWaiter(waiter, value) {
|
|
386
|
+
if (waiter.done) return false;
|
|
387
|
+
waiter.done = true;
|
|
388
|
+
clearTimeout(waiter.timer);
|
|
389
|
+
if (waiter.signal && waiter.onAbort) waiter.signal.removeEventListener('abort', waiter.onAbort);
|
|
390
|
+
const i = this._waiters.indexOf(waiter);
|
|
391
|
+
if (i >= 0) this._waiters.splice(i, 1);
|
|
392
|
+
waiter.resolve(value);
|
|
393
|
+
return true;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* Release a concurrency slot held by a request and hand any freed capacity to
|
|
398
|
+
* the longest-waiting overflow request that can use it (FIFO, but a waiter
|
|
399
|
+
* whose exclude set can't currently be satisfied is skipped rather than
|
|
400
|
+
* head-of-line blocking a later waiter that can run).
|
|
401
|
+
*/
|
|
402
|
+
releaseAccount(accountOrIndex) {
|
|
403
|
+
// Resolve to the account OBJECT (what the server holds — reindex-safe across a
|
|
404
|
+
// removeAccount) so a release decrements the slot of the *account that was
|
|
405
|
+
// acquired*, never whatever happens to sit at that index now. A numeric index
|
|
406
|
+
// is still accepted for convenience/tests.
|
|
407
|
+
const account = this._resolve(accountOrIndex);
|
|
408
|
+
if (account && account.inflight > 0) account.inflight--;
|
|
409
|
+
this._drainWaiters();
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
_drainWaiters() {
|
|
413
|
+
for (let i = 0; i < this._waiters.length;) {
|
|
414
|
+
const waiter = this._waiters[i];
|
|
415
|
+
const account = this._tryAcquire(waiter.exclude, waiter.affinityKey);
|
|
416
|
+
if (account) {
|
|
417
|
+
// _settleWaiter splices the waiter out, so don't advance i. If it was
|
|
418
|
+
// already settled (shouldn't happen — settled waiters aren't in the list),
|
|
419
|
+
// give the slot back instead of leaking it.
|
|
420
|
+
if (!this._settleWaiter(waiter, account)) { account.inflight--; i++; }
|
|
421
|
+
continue;
|
|
422
|
+
}
|
|
423
|
+
// No slot right now. If no account this waiter could use is even
|
|
424
|
+
// available-but-capped, nothing will ever free for it (e.g. the account it
|
|
425
|
+
// was queued for just got disabled or exhausted) — settle it null so it
|
|
426
|
+
// releases its finite queue slot instead of blocking later, satisfiable
|
|
427
|
+
// overflow requests until its timeout. A waiter that still has a cappable
|
|
428
|
+
// account to hope for is left in place.
|
|
429
|
+
if (!this.anyCapped(waiter.exclude)) { this._settleWaiter(waiter, null); continue; }
|
|
430
|
+
i++;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
/**
|
|
435
|
+
* Highest-priority available account by use-or-lose ordering: soonest WEEKLY
|
|
436
|
+
* (7d) reset first — weekly quota is the scarce resource, so an account whose
|
|
437
|
+
* week is about to renew (and whose unspent quota would be wasted) is drained
|
|
438
|
+
* first — then soonest session reset, then lowest session utilization. Falls
|
|
439
|
+
* back to the soonest-resetting account when none are currently available.
|
|
440
|
+
*
|
|
441
|
+
* `exclude` (a Set of account objects) is used for per-request failover: those
|
|
442
|
+
* accounts are skipped, and when nothing else is eligible this returns null
|
|
443
|
+
* (instead of recovering one) so the caller can pass the 429 through.
|
|
444
|
+
*/
|
|
445
|
+
_selectBest(exclude = null) {
|
|
446
|
+
const has = a => (exclude ? exclude.has(a) : false);
|
|
447
|
+
const eligible = this.accounts.filter(a => this._isAvailable(a) && !has(a));
|
|
448
|
+
if (eligible.length === 0) return exclude ? null : this._recoverSoonest();
|
|
449
|
+
|
|
450
|
+
eligible.sort((a, b) => {
|
|
451
|
+
const pa = this._priority(a);
|
|
452
|
+
const pb = this._priority(b);
|
|
453
|
+
if (pa !== pb) return pa - pb; // explicit priority first (lower = preferred)
|
|
454
|
+
return this.autoCompare(a, b); // then the automatic use-or-lose order
|
|
455
|
+
});
|
|
456
|
+
|
|
457
|
+
// Accounts tied for the best rank (notably all-unknown at cold start, or all
|
|
458
|
+
// sharing one priority) are load-balanced round-robin instead of always
|
|
459
|
+
// pinning to the lowest index, so a startup burst can't pile onto one account
|
|
460
|
+
// before quotas are known.
|
|
461
|
+
const p0 = this._priority(eligible[0]);
|
|
462
|
+
const w0 = this._weeklyResetTime(eligible[0]);
|
|
463
|
+
const r0 = this._sessionResetTime(eligible[0]);
|
|
464
|
+
const u0 = this._sessionUtilization(eligible[0]);
|
|
465
|
+
const tied = eligible
|
|
466
|
+
.filter(a => this._priority(a) === p0
|
|
467
|
+
&& this._weeklyResetTime(a) === w0
|
|
468
|
+
&& this._sessionResetTime(a) === r0
|
|
469
|
+
&& this._sessionUtilization(a) === u0)
|
|
470
|
+
.sort((a, b) => a.index - b.index);
|
|
471
|
+
if (tied.length <= 1) return eligible[0];
|
|
472
|
+
return tied.find(a => a.index > this.currentIndex) || tied[0];
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* Explicit selection priority: lower = preferred. Unset (null) sorts last
|
|
477
|
+
* (Infinity) so an account WITH any finite priority — however large — is chosen
|
|
478
|
+
* ahead of those without. When no account sets a priority, every account ties
|
|
479
|
+
* here (Infinity === Infinity) and the sort falls through to use-or-lose, i.e.
|
|
480
|
+
* the original behavior unchanged. The callers compare with a `pa !== pb` guard
|
|
481
|
+
* before any subtraction, so Infinity never produces a NaN sort key.
|
|
482
|
+
*/
|
|
483
|
+
_priority(account) {
|
|
484
|
+
return Number.isFinite(account.priority) ? account.priority : Infinity;
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/**
|
|
488
|
+
* The automatic ("auto") use-or-lose comparator, shared by selection and the
|
|
489
|
+
* TUI display order: soonest WEEKLY reset (drain what renews first) → soonest
|
|
490
|
+
* session reset → lowest session utilization. Returns 0 on a full tie, so a
|
|
491
|
+
* stable sort keeps ties in array order (the pre-weekly behavior for API-key
|
|
492
|
+
* fleets and unmeasured accounts).
|
|
493
|
+
*/
|
|
494
|
+
autoCompare(a, b) {
|
|
495
|
+
const wa = this._weeklyResetTime(a);
|
|
496
|
+
const wb = this._weeklyResetTime(b);
|
|
497
|
+
if (wa !== wb) return wa - wb;
|
|
498
|
+
const ra = this._sessionResetTime(a);
|
|
499
|
+
const rb = this._sessionResetTime(b);
|
|
500
|
+
if (ra !== rb) return ra - rb;
|
|
501
|
+
return this._sessionUtilization(a) - this._sessionUtilization(b);
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
/**
|
|
505
|
+
* Weekly reset timestamp (ms): unified 7d (Max) → Infinity. API-key accounts
|
|
506
|
+
* have no weekly window, so they tie at Infinity and the session tiebreak
|
|
507
|
+
* decides — exactly the pre-weekly-ordering behavior. The window counts only
|
|
508
|
+
* when BOTH utilization and reset are present: a partial/garbled header pair
|
|
509
|
+
* (reset without utilization) must not outrank accounts with no 7d data,
|
|
510
|
+
* matching the documented "no weekly data ranks at Infinity" semantics.
|
|
511
|
+
*
|
|
512
|
+
* A timestamp that has PASSED ranks at Infinity too: the moment a window
|
|
513
|
+
* rolls over, the account's old "resets soonest" claim is void (its fresh
|
|
514
|
+
* window is unknown until re-measured) — without this, the past timestamp
|
|
515
|
+
* (smallest value) would pin the account at the top of the order until a
|
|
516
|
+
* request-path sweep happened to clear it, so the order would NOT follow
|
|
517
|
+
* reset rollovers. The lazy sweep in _isNearQuota still clears the fields;
|
|
518
|
+
* this just makes ORDERING (selection and the TUI display, which has no
|
|
519
|
+
* sweep) reflect the rollover instantly.
|
|
520
|
+
*/
|
|
521
|
+
_weeklyResetTime(account) {
|
|
522
|
+
const q = account.quota;
|
|
523
|
+
const r = (q.unified7d != null && q.unified7dReset) ? q.unified7dReset : Infinity;
|
|
524
|
+
return r > Date.now() ? r : Infinity;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/**
|
|
528
|
+
* Session reset timestamp (ms): unified 5h (Max) → standard reset → Infinity.
|
|
529
|
+
* Expired timestamps rank at Infinity for the same rollover reason as
|
|
530
|
+
* _weeklyResetTime above.
|
|
531
|
+
*/
|
|
532
|
+
_sessionResetTime(account) {
|
|
533
|
+
const q = account.quota;
|
|
534
|
+
const r = q.unified5hReset
|
|
535
|
+
|| (q.resetsAt ? new Date(q.resetsAt).getTime() : Infinity);
|
|
536
|
+
return r > Date.now() ? r : Infinity;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/** Session utilization 0–1: unified 5h (Max) → standard token/request usage → 0. */
|
|
540
|
+
_sessionUtilization(account) {
|
|
541
|
+
const q = account.quota;
|
|
542
|
+
if (q.unified5h != null) return q.unified5h;
|
|
543
|
+
if (q.tokensLimit != null && q.tokensRemaining != null) {
|
|
544
|
+
return 1 - q.tokensRemaining / q.tokensLimit;
|
|
545
|
+
}
|
|
546
|
+
if (q.requestsLimit != null && q.requestsRemaining != null) {
|
|
547
|
+
return 1 - q.requestsRemaining / q.requestsLimit;
|
|
548
|
+
}
|
|
549
|
+
return 0;
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
/**
|
|
553
|
+
* Clear every account's expired quota windows NOW. The lazy sweep inside
|
|
554
|
+
* _isNearQuota only runs on selection paths (i.e. when a request flows), so
|
|
555
|
+
* on an idle proxy a rolled-over window would keep its stale values — and
|
|
556
|
+
* stay "measured", which prevents the periodic active warm-up from
|
|
557
|
+
* re-probing it. The server's warm-up timer calls this first, closing the
|
|
558
|
+
* loop: rollover → sweep → unmeasured → probe → fresh data → order updates.
|
|
559
|
+
* Idempotent and cheap (pure field clears).
|
|
560
|
+
*/
|
|
561
|
+
sweepExpired() {
|
|
562
|
+
for (const a of this.accounts) this._isNearQuota(a);
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/** True once we have any quota data for this account (rate-limit headers seen). */
|
|
566
|
+
_isMeasured(account) {
|
|
567
|
+
const q = account.quota;
|
|
568
|
+
return q.unified5h != null || q.unified7d != null
|
|
569
|
+
|| q.tokensLimit != null || q.requestsLimit != null;
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
/**
|
|
573
|
+
* Fully measured, for ACTIVE warm-up candidacy: an OAuth (Max) response
|
|
574
|
+
* always carries both the 5h and the 7d window, so a missing half — e.g. a
|
|
575
|
+
* weekly rollover swept `unified7d` while the session window survived —
|
|
576
|
+
* means the account needs a re-probe or its weekly quota/ordering stays
|
|
577
|
+
* unknown until real traffic reaches it. One probe repopulates both windows,
|
|
578
|
+
* so candidacy converges. API-key accounts keep the any-data semantics.
|
|
579
|
+
*/
|
|
580
|
+
_fullyMeasured(account) {
|
|
581
|
+
if (account.type === 'oauth') {
|
|
582
|
+
// A window counts only when COMPLETE (utilization AND reset) — the same
|
|
583
|
+
// semantics the ordering helpers use. Utilization without its reset
|
|
584
|
+
// timestamp gives use-or-lose nothing to sort on, so such an account
|
|
585
|
+
// still needs a re-probe.
|
|
586
|
+
const q = account.quota;
|
|
587
|
+
return q.unified5h != null && q.unified5hReset != null
|
|
588
|
+
&& q.unified7d != null && q.unified7dReset != null;
|
|
589
|
+
}
|
|
590
|
+
return this._isMeasured(account);
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
/**
|
|
594
|
+
* A fully-measured OAuth account whose model-scoped weekly window (the Fable
|
|
595
|
+
* `7d_oi` limit) is still absent. Such a window only appears on responses to
|
|
596
|
+
* Fable-tier requests, so an account that was measured by lower-tier traffic
|
|
597
|
+
* or a lower-tier probe keeps its `Fbl` bar blank — and because it IS fully
|
|
598
|
+
* measured for 5h/7d, ordinary warm-up (which only targets unmeasured
|
|
599
|
+
* accounts) never re-probes it. This flags it for a bounded model-weekly
|
|
600
|
+
* top-up probe, run ONLY when the committed probe template can actually
|
|
601
|
+
* elicit the window (see server.js). The `_mwProbes` cap stops an account
|
|
602
|
+
* whose upstream genuinely never reports the window from being probed every
|
|
603
|
+
* interval forever; it resets when the window populates or a quota window is
|
|
604
|
+
* swept (a fresh week is a fresh reason to look).
|
|
605
|
+
*/
|
|
606
|
+
needsModelWeekly(account) {
|
|
607
|
+
return account.type === 'oauth'
|
|
608
|
+
&& this._fullyMeasured(account)
|
|
609
|
+
&& Object.keys(account.quota.modelWeekly).length === 0
|
|
610
|
+
&& (account._mwProbes || 0) < this.maxWarmupTries;
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
/**
|
|
614
|
+
* An account still needing warm-up: available, not yet MEASURED, under the
|
|
615
|
+
* per-account attempt cap.
|
|
616
|
+
*
|
|
617
|
+
* Keying on `!_isMeasured` (not on "has it made a request") is deliberate: a
|
|
618
|
+
* request can return *no* rate-limit headers — a `HEAD /` health check, a
|
|
619
|
+
* 404, an auth failure — which would leave the account unmeasured. Gating
|
|
620
|
+
* warm-up on `totalRequests === 0` used to permanently disqualify such an
|
|
621
|
+
* account after that single header-less request, trapping it as "unmeasured"
|
|
622
|
+
* forever: it then sorts to the bottom of use-or-lose priority (no reset
|
|
623
|
+
* data) and the unmeasured-rebalance bounces any switch away from it, so it
|
|
624
|
+
* never gets used again — and its token never gets refreshed, so it expires.
|
|
625
|
+
*
|
|
626
|
+
* maxWarmupTries provides the loop-safety instead: a genuinely dead account
|
|
627
|
+
* (always header-less / 401) is abandoned after a few attempts rather than
|
|
628
|
+
* looping forever. (An expired-token account is resolved on its first warm-up
|
|
629
|
+
* routing anyway — ensureTokenFresh either refreshes it into a measurable
|
|
630
|
+
* state or marks it `error`, which makes it unavailable here.)
|
|
631
|
+
*/
|
|
632
|
+
_isWarmupTarget(account) {
|
|
633
|
+
return this._isAvailable(account)
|
|
634
|
+
&& !this._isMeasured(account)
|
|
635
|
+
&& (account._warmupTries || 0) < this.maxWarmupTries;
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
/**
|
|
639
|
+
* Next account to warm up, round-robin across the warm-up targets so a burst
|
|
640
|
+
* spreads evenly. Advances the cursor and bumps the chosen account's attempt
|
|
641
|
+
* counter synchronously, so concurrent calls pick different accounts even
|
|
642
|
+
* before any response arrives. Returns null when no target remains.
|
|
643
|
+
*/
|
|
644
|
+
_nextWarmup() {
|
|
645
|
+
const n = this.accounts.length;
|
|
646
|
+
for (let i = 0; i < n; i++) {
|
|
647
|
+
const idx = (this._warmupCursor + i) % n;
|
|
648
|
+
const a = this.accounts[idx];
|
|
649
|
+
if (this._isWarmupTarget(a)) {
|
|
650
|
+
this._warmupCursor = idx + 1;
|
|
651
|
+
a._warmupTries = (a._warmupTries || 0) + 1;
|
|
652
|
+
return a;
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
return null;
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
/**
|
|
659
|
+
* Accounts eligible for an *active* warm-up probe: available (enabled, not
|
|
660
|
+
* throttled / exhausted / error), with no quota data yet, AND not already
|
|
661
|
+
* handling a request. The server sends each one a minimal upstream request to
|
|
662
|
+
* populate its quota so the dashboard reflects the whole fleet shortly after a
|
|
663
|
+
* (re)start, instead of waiting for client traffic to organically reach every
|
|
664
|
+
* account.
|
|
665
|
+
*
|
|
666
|
+
* `inflight === 0` matters: a request already in flight will itself populate
|
|
667
|
+
* the account's quota (updateQuota runs on its response headers), so probing it
|
|
668
|
+
* would just race that request and waste an upstream call. Cold start's very
|
|
669
|
+
* first request holds its (still-unmeasured) account here, so the startup
|
|
670
|
+
* fan-out probes only the genuinely idle rest of the fleet — never the account
|
|
671
|
+
* that request is already measuring. (An unmeasured account can't be near-quota,
|
|
672
|
+
* so no extra status carve-outs are needed beyond _isAvailable.)
|
|
673
|
+
*/
|
|
674
|
+
warmupCandidates() {
|
|
675
|
+
return this.accounts.filter(a =>
|
|
676
|
+
this._isAvailable(a) && !this._fullyMeasured(a) && a.inflight === 0
|
|
677
|
+
// Convergence cap: against the real upstream one probe fully measures an
|
|
678
|
+
// account (responses always carry both window families), but a
|
|
679
|
+
// pathological upstream — a 2xx missing a family, header-less responses,
|
|
680
|
+
// or a deterministic 4xx — must not be probed every interval forever. A
|
|
681
|
+
// probe that completes with such a deterministic fruitless outcome
|
|
682
|
+
// increments _partialProbes (transient 5xx/429/network failures don't
|
|
683
|
+
// count — see warmupAccount); after maxWarmupTries fruitless probes the
|
|
684
|
+
// account stops being a candidate. The counter resets when a window is
|
|
685
|
+
// swept (a fresh rollover is a fresh reason to probe) or when the
|
|
686
|
+
// account becomes fully measured — and, because a fully UNMEASURED
|
|
687
|
+
// account has no reset timestamp for the sweep to fire on, a capped
|
|
688
|
+
// account is retried once per probeRetryAfterMs as a slow backstop, so a
|
|
689
|
+
// "deterministic-looking" outage (e.g. headers temporarily missing) still
|
|
690
|
+
// recovers instead of suppressing active warm-up until a restart.
|
|
691
|
+
&& ((a._partialProbes || 0) < this.maxWarmupTries
|
|
692
|
+
|| Date.now() - (a._lastFruitlessProbeAt || 0) >= this.probeRetryAfterMs));
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
_isAvailable(account) {
|
|
696
|
+
if (!account) return false;
|
|
697
|
+
|
|
698
|
+
// Manually disabled accounts are out of rotation entirely. This single gate
|
|
699
|
+
// covers every selection path (warm-up target, _selectBest, _cappedSet,
|
|
700
|
+
// anyUsable/anyCapped, the sticky-current check) so a disabled account is
|
|
701
|
+
// never chosen for a new request. _recoverSoonest iterates accounts directly
|
|
702
|
+
// (not via this), so it skips disabled accounts itself.
|
|
703
|
+
if (account.enabled === false) return false;
|
|
704
|
+
|
|
705
|
+
// Check rate limit expiry
|
|
706
|
+
if (account.status === 'throttled' && account.rateLimitedUntil) {
|
|
707
|
+
if (Date.now() < account.rateLimitedUntil) return false;
|
|
708
|
+
account.status = 'active';
|
|
709
|
+
account.rateLimitedUntil = null;
|
|
710
|
+
console.log(`[TeamClaude] Account "${account.name}" rate limit expired, marking active`);
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
if (account.status === 'exhausted' || account.status === 'error') return false;
|
|
714
|
+
if (this._isNearQuota(account)) return false;
|
|
715
|
+
|
|
716
|
+
return true;
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
_isNearQuota(account) {
|
|
720
|
+
const q = account.quota;
|
|
721
|
+
const now = Date.now();
|
|
722
|
+
|
|
723
|
+
// Clear expired unified quotas. The reset timestamp is cleared even when the
|
|
724
|
+
// matching utilization was never set (a partial/garbled header pair) — a
|
|
725
|
+
// stale past timestamp would otherwise survive forever and, since selection
|
|
726
|
+
// sorts by reset time, permanently bias the ordering toward that account.
|
|
727
|
+
if (q.unified5hReset && now >= q.unified5hReset) {
|
|
728
|
+
if (q.unified5h != null) console.log(`[TeamClaude] Account "${account.name}" session quota reset`);
|
|
729
|
+
q.unified5h = null;
|
|
730
|
+
q.unified5hReset = null;
|
|
731
|
+
// Fresh rollover → BOTH warm-up budgets renew: active probes
|
|
732
|
+
// (_partialProbes) and the passive request-routing warm-up
|
|
733
|
+
// (_warmupTries) get a fresh reason to re-measure this window.
|
|
734
|
+
account._partialProbes = 0;
|
|
735
|
+
account._warmupTries = 0;
|
|
736
|
+
}
|
|
737
|
+
if (q.unified7dReset && now >= q.unified7dReset) {
|
|
738
|
+
if (q.unified7d != null) console.log(`[TeamClaude] Account "${account.name}" weekly quota reset`);
|
|
739
|
+
q.unified7d = null;
|
|
740
|
+
q.unified7dReset = null;
|
|
741
|
+
q.unifiedStatus = null;
|
|
742
|
+
account._partialProbes = 0; // fresh rollover → re-probes allowed again
|
|
743
|
+
account._warmupTries = 0;
|
|
744
|
+
}
|
|
745
|
+
// Clear expired model-scoped weekly windows (display-only, but a stale
|
|
746
|
+
// "94% Fable" bar after the window reset would mislead). A cleared window is
|
|
747
|
+
// a fresh reason to top it up again, so renew the model-weekly probe budget.
|
|
748
|
+
for (const [label, win] of Object.entries(q.modelWeekly)) {
|
|
749
|
+
if (win.reset && now >= win.reset) {
|
|
750
|
+
console.log(`[TeamClaude] Account "${account.name}" ${label} quota reset`);
|
|
751
|
+
delete q.modelWeekly[label];
|
|
752
|
+
account._mwProbes = 0;
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
// Clear expired standard quotas — each window INDEPENDENTLY. The token and
|
|
757
|
+
// request windows reset at different times; sweeping both on the collapsed
|
|
758
|
+
// resetsAt (token-first) made a still-blocked request window vanish the
|
|
759
|
+
// moment the sooner token window reset, freeing the account ~an hour early.
|
|
760
|
+
// A window without its own reset falls back to resetsAt (old snapshots /
|
|
761
|
+
// upstreams that only send one reset header) — same sweep as before.
|
|
762
|
+
const tokensResetAt = q.tokensReset || q.resetsAt;
|
|
763
|
+
if (tokensResetAt && now >= new Date(tokensResetAt).getTime()) {
|
|
764
|
+
q.tokensRemaining = null;
|
|
765
|
+
q.tokensLimit = null;
|
|
766
|
+
q.tokensReset = null;
|
|
767
|
+
}
|
|
768
|
+
const requestsResetAt = q.requestsReset || q.resetsAt;
|
|
769
|
+
if (requestsResetAt && now >= new Date(requestsResetAt).getTime()) {
|
|
770
|
+
q.requestsRemaining = null;
|
|
771
|
+
q.requestsLimit = null;
|
|
772
|
+
q.requestsReset = null;
|
|
773
|
+
}
|
|
774
|
+
// Advance the collapsed resetsAt (session-ordering fallback) to the soonest
|
|
775
|
+
// still-future window reset, or clear it once no window remains.
|
|
776
|
+
if (q.resetsAt && now >= new Date(q.resetsAt).getTime()) {
|
|
777
|
+
const future = [q.tokensReset, q.requestsReset]
|
|
778
|
+
.map(r => (r ? new Date(r).getTime() : null))
|
|
779
|
+
.filter(t => t && t > now);
|
|
780
|
+
q.resetsAt = future.length ? new Date(Math.min(...future)).toISOString() : null;
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
// Unified quotas (Claude Max) — utilization is already 0-1
|
|
784
|
+
if (q.unified5h != null && q.unified5h >= this.switchThreshold) return true;
|
|
785
|
+
if (q.unified7d != null && q.unified7d >= this.switchThreshold) return true;
|
|
786
|
+
|
|
787
|
+
// Standard quotas (API key accounts)
|
|
788
|
+
if (q.tokensLimit != null && q.tokensRemaining != null) {
|
|
789
|
+
const used = 1 - (q.tokensRemaining / q.tokensLimit);
|
|
790
|
+
if (used >= this.switchThreshold) return true;
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
if (q.requestsLimit != null && q.requestsRemaining != null) {
|
|
794
|
+
const used = 1 - (q.requestsRemaining / q.requestsLimit);
|
|
795
|
+
if (used >= this.switchThreshold) return true;
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
return false;
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
/** When all accounts are unavailable, return the soonest to reset (if it has already reset). */
|
|
802
|
+
_recoverSoonest() {
|
|
803
|
+
let soonestAccount = null;
|
|
804
|
+
let soonestTime = Infinity;
|
|
805
|
+
|
|
806
|
+
for (const account of this.accounts) {
|
|
807
|
+
// Never recover a manually-disabled account into rotation.
|
|
808
|
+
if (account.enabled === false) continue;
|
|
809
|
+
const resetTime = account.rateLimitedUntil
|
|
810
|
+
|| account.quota.unified5hReset
|
|
811
|
+
|| account.quota.unified7dReset
|
|
812
|
+
|| (account.quota.resetsAt ? new Date(account.quota.resetsAt).getTime() : null);
|
|
813
|
+
|
|
814
|
+
if (resetTime && resetTime < soonestTime) {
|
|
815
|
+
soonestTime = resetTime;
|
|
816
|
+
soonestAccount = account;
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
if (soonestAccount && soonestTime <= Date.now()) {
|
|
821
|
+
soonestAccount.status = 'active';
|
|
822
|
+
soonestAccount.rateLimitedUntil = null;
|
|
823
|
+
this.currentIndex = soonestAccount.index;
|
|
824
|
+
console.log(`[TeamClaude] Account "${soonestAccount.name}" reset, switching to it`);
|
|
825
|
+
return soonestAccount;
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
return null;
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
/**
|
|
832
|
+
* Update an account's quota tracking from upstream response headers.
|
|
833
|
+
*/
|
|
834
|
+
updateQuota(accountIndex, headers) {
|
|
835
|
+
const account = this._resolve(accountIndex);
|
|
836
|
+
if (!account) return;
|
|
837
|
+
|
|
838
|
+
// Unified rate limits (Claude Max)
|
|
839
|
+
const u5h = parseFloat(headers['anthropic-ratelimit-unified-5h-utilization']);
|
|
840
|
+
const u7d = parseFloat(headers['anthropic-ratelimit-unified-7d-utilization']);
|
|
841
|
+
if (!isNaN(u5h)) account.quota.unified5h = u5h;
|
|
842
|
+
if (!isNaN(u7d)) account.quota.unified7d = u7d;
|
|
843
|
+
|
|
844
|
+
const r5h = headers['anthropic-ratelimit-unified-5h-reset'];
|
|
845
|
+
const r7d = headers['anthropic-ratelimit-unified-7d-reset'];
|
|
846
|
+
if (r5h) account.quota.unified5hReset = parseInt(r5h, 10) * 1000;
|
|
847
|
+
if (r7d) account.quota.unified7dReset = parseInt(r7d, 10) * 1000;
|
|
848
|
+
|
|
849
|
+
const uStatus = headers['anthropic-ratelimit-unified-status'];
|
|
850
|
+
if (uStatus) account.quota.unifiedStatus = uStatus;
|
|
851
|
+
|
|
852
|
+
// Model-scoped weekly windows (7d_<label>), e.g. `7d_oi` — the weekly limit
|
|
853
|
+
// for the top model tier ("Fable" in Claude's usage UI). These headers only
|
|
854
|
+
// appear on responses to requests for that model tier, so the value sticks
|
|
855
|
+
// around from the last such request. Matched generically so a renamed or
|
|
856
|
+
// newly added window is picked up as-is.
|
|
857
|
+
for (const [key, value] of Object.entries(headers)) {
|
|
858
|
+
const m = /^anthropic-ratelimit-unified-(7d_[a-z0-9_]+)-(utilization|reset)$/.exec(key);
|
|
859
|
+
if (!m) continue;
|
|
860
|
+
const win = account.quota.modelWeekly[m[1]]
|
|
861
|
+
|| (account.quota.modelWeekly[m[1]] = { utilization: null, reset: null });
|
|
862
|
+
if (m[2] === 'utilization') {
|
|
863
|
+
const u = parseFloat(value);
|
|
864
|
+
if (!isNaN(u)) win.utilization = u;
|
|
865
|
+
} else {
|
|
866
|
+
const r = parseInt(value, 10);
|
|
867
|
+
if (!isNaN(r)) win.reset = r * 1000;
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
// Standard rate limits (API key accounts)
|
|
872
|
+
const tokensLimit = parseInt(headers['anthropic-ratelimit-tokens-limit'], 10);
|
|
873
|
+
const tokensRemaining = parseInt(headers['anthropic-ratelimit-tokens-remaining'], 10);
|
|
874
|
+
const tokensReset = headers['anthropic-ratelimit-tokens-reset'];
|
|
875
|
+
const requestsLimit = parseInt(headers['anthropic-ratelimit-requests-limit'], 10);
|
|
876
|
+
const requestsRemaining = parseInt(headers['anthropic-ratelimit-requests-remaining'], 10);
|
|
877
|
+
const requestsReset = headers['anthropic-ratelimit-requests-reset'];
|
|
878
|
+
|
|
879
|
+
if (!isNaN(tokensLimit)) account.quota.tokensLimit = tokensLimit;
|
|
880
|
+
if (!isNaN(tokensRemaining)) account.quota.tokensRemaining = tokensRemaining;
|
|
881
|
+
if (!isNaN(requestsLimit)) account.quota.requestsLimit = requestsLimit;
|
|
882
|
+
if (!isNaN(requestsRemaining)) account.quota.requestsRemaining = requestsRemaining;
|
|
883
|
+
|
|
884
|
+
if (tokensReset) account.quota.tokensReset = tokensReset;
|
|
885
|
+
if (requestsReset) account.quota.requestsReset = requestsReset;
|
|
886
|
+
if (tokensReset) account.quota.resetsAt = tokensReset;
|
|
887
|
+
else if (requestsReset) account.quota.resetsAt = requestsReset;
|
|
888
|
+
|
|
889
|
+
account.usage.totalRequests++;
|
|
890
|
+
account.usage.lastUsed = new Date().toISOString();
|
|
891
|
+
|
|
892
|
+
// Log when approaching quota
|
|
893
|
+
if (this._isNearQuota(account)) {
|
|
894
|
+
const pct = account.quota.unified7d != null
|
|
895
|
+
? (account.quota.unified7d * 100).toFixed(1)
|
|
896
|
+
: account.quota.tokensLimit
|
|
897
|
+
? ((1 - account.quota.tokensRemaining / account.quota.tokensLimit) * 100).toFixed(1)
|
|
898
|
+
: '?';
|
|
899
|
+
console.log(`[TeamClaude] Account "${account.name}" at ${pct}% usage — will switch on next request`);
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
/**
|
|
904
|
+
* Update cumulative token usage from response body data.
|
|
905
|
+
*/
|
|
906
|
+
updateUsage(accountIndex, inputTokens, outputTokens) {
|
|
907
|
+
const account = this._resolve(accountIndex);
|
|
908
|
+
if (!account) return;
|
|
909
|
+
if (inputTokens) account.usage.totalInputTokens += inputTokens;
|
|
910
|
+
if (outputTokens) account.usage.totalOutputTokens += outputTokens;
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
/**
|
|
914
|
+
* Does a 429 from this account indicate genuine *account-level quota
|
|
915
|
+
* exhaustion* (vs a transient / global / IP / request-level 429)?
|
|
916
|
+
*
|
|
917
|
+
* Only exhaustion 429s should throttle the account and trigger a switch to
|
|
918
|
+
* another account. A non-exhaustion 429 must NOT be replayed across the
|
|
919
|
+
* fleet — otherwise a single request whose 429 is request-global (e.g. a
|
|
920
|
+
* malformed request, an org/IP limit, or a momentary upstream blip) would
|
|
921
|
+
* poison every account and make unrelated requests fail too.
|
|
922
|
+
*
|
|
923
|
+
* Call this *after* updateQuota() has folded the 429's rate-limit headers
|
|
924
|
+
* into the account's quota state.
|
|
925
|
+
*
|
|
926
|
+
* Model-scoped windows (quota.modelWeekly, e.g. the Fable 7d_oi limit) are
|
|
927
|
+
* deliberately NOT consulted here. On a real upstream 429 for that model tier
|
|
928
|
+
* the top-level `unified-status` is `rejected` too (the binding claim is
|
|
929
|
+
* reflected there — verified against live traffic), so the exhaustion IS
|
|
930
|
+
* detected; folding 7d_oi in additionally would change nothing on real
|
|
931
|
+
* headers, and reacting to it alone would globally throttle an account that
|
|
932
|
+
* still serves every other model. Per-model routing (skip Fable-exhausted
|
|
933
|
+
* accounts only for Fable requests, without the 5-min global throttle) would
|
|
934
|
+
* need the request's model plumbed into selection — a separate feature.
|
|
935
|
+
*/
|
|
936
|
+
isExhausted(accountIndex) {
|
|
937
|
+
const account = this._resolve(accountIndex);
|
|
938
|
+
if (!account) return false;
|
|
939
|
+
// Claude Max: upstream explicitly rejects when over the unified limit.
|
|
940
|
+
if (account.quota.unifiedStatus === 'rejected') return true;
|
|
941
|
+
// Otherwise rely on measured utilization (unified or standard headers).
|
|
942
|
+
return this._isNearQuota(account);
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
/**
|
|
946
|
+
* Mark an account as rate-limited for a given duration.
|
|
947
|
+
*/
|
|
948
|
+
markRateLimited(accountIndex, retryAfterSeconds) {
|
|
949
|
+
const account = this._resolve(accountIndex);
|
|
950
|
+
if (!account) return;
|
|
951
|
+
account.status = 'throttled';
|
|
952
|
+
account.rateLimitedUntil = Date.now() + (retryAfterSeconds * 1000);
|
|
953
|
+
console.log(`[TeamClaude] Account "${account.name}" rate limited for ${retryAfterSeconds}s`);
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
/**
|
|
957
|
+
* Ensure an OAuth account's token is fresh, refreshing if needed.
|
|
958
|
+
* Pass force=true to refresh regardless of expiry (e.g. after a 401).
|
|
959
|
+
* Concurrent calls for the same account coalesce into a single refresh.
|
|
960
|
+
*/
|
|
961
|
+
async ensureTokenFresh(accountIndex, force = false) {
|
|
962
|
+
const account = this._resolve(accountIndex);
|
|
963
|
+
if (!account || account.type !== 'oauth' || !account.refreshToken) return;
|
|
964
|
+
|
|
965
|
+
if (!force && !isTokenExpiringSoon(account.expiresAt)) return;
|
|
966
|
+
|
|
967
|
+
// Coalesce concurrent refreshes
|
|
968
|
+
if (account._refreshPromise) return account._refreshPromise;
|
|
969
|
+
|
|
970
|
+
account._refreshPromise = (async () => {
|
|
971
|
+
console.log(`[TeamClaude] Refreshing token for account "${account.name}"...`);
|
|
972
|
+
try {
|
|
973
|
+
// Refresh via an injected handler when set (e.g. cloud-centralized refresh
|
|
974
|
+
// so multiple PCs don't each rotate the same account's refresh token); it
|
|
975
|
+
// falls back to a local refresh internally. Default: local OAuth refresh.
|
|
976
|
+
const newTokens = this._refreshHandler
|
|
977
|
+
? await this._refreshHandler(account)
|
|
978
|
+
: await refreshAccessToken(account.refreshToken);
|
|
979
|
+
account.credential = newTokens.accessToken;
|
|
980
|
+
account.refreshToken = newTokens.refreshToken;
|
|
981
|
+
account.expiresAt = newTokens.expiresAt;
|
|
982
|
+
console.log(`[TeamClaude] Token refreshed for account "${account.name}"`);
|
|
983
|
+
// Only persist if the account is still live at its claimed index. If it was
|
|
984
|
+
// removed during the (awaited) network refresh, its `.index` is stale and
|
|
985
|
+
// would misattribute the write to the survivor that shifted into that slot
|
|
986
|
+
// — and a deleted account's tokens don't need persisting anyway.
|
|
987
|
+
if (this.accounts[account.index] === account) this._onTokenRefresh?.(account.index, newTokens);
|
|
988
|
+
} catch (err) {
|
|
989
|
+
console.error(`[TeamClaude] Token refresh failed for "${account.name}": ${err.message}`);
|
|
990
|
+
// Only mark as error if the access token is actually expired;
|
|
991
|
+
// a failed proactive refresh shouldn't kill a still-valid token
|
|
992
|
+
if (!account.expiresAt || Date.now() >= account.expiresAt) {
|
|
993
|
+
account.status = 'error';
|
|
994
|
+
}
|
|
995
|
+
} finally {
|
|
996
|
+
account._refreshPromise = null;
|
|
997
|
+
}
|
|
998
|
+
})();
|
|
999
|
+
|
|
1000
|
+
return account._refreshPromise;
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
/**
|
|
1004
|
+
* Set a callback to persist refreshed tokens to config.
|
|
1005
|
+
*/
|
|
1006
|
+
onTokenRefresh(callback) {
|
|
1007
|
+
this._onTokenRefresh = callback;
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
/**
|
|
1011
|
+
* Inject the token-refresh strategy. `handler(account)` must return
|
|
1012
|
+
* { accessToken, refreshToken, expiresAt } (or throw). Used to route refreshes
|
|
1013
|
+
* through the cloud (single-authority rotation) instead of refreshing locally.
|
|
1014
|
+
* Unset → local OAuth refresh (the default).
|
|
1015
|
+
*/
|
|
1016
|
+
setRefreshHandler(handler) {
|
|
1017
|
+
this._refreshHandler = handler;
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
/**
|
|
1021
|
+
* Update a specific account's OAuth tokens (e.g. after intercepting a token refresh).
|
|
1022
|
+
*/
|
|
1023
|
+
updateAccountTokens(accountIndex, { accessToken, refreshToken, expiresAt }) {
|
|
1024
|
+
const account = this._resolve(accountIndex);
|
|
1025
|
+
if (!account || account.type !== 'oauth') return;
|
|
1026
|
+
|
|
1027
|
+
account.credential = accessToken;
|
|
1028
|
+
if (refreshToken) account.refreshToken = refreshToken;
|
|
1029
|
+
account.expiresAt = expiresAt;
|
|
1030
|
+
if (account.status === 'error') account.status = 'active';
|
|
1031
|
+
console.log(`[TeamClaude] Updated tokens for account "${account.name}"`);
|
|
1032
|
+
// Same liveness guard as ensureTokenFresh: never emit a stale index for a
|
|
1033
|
+
// removed account (here the path is synchronous, but keep the invariant uniform).
|
|
1034
|
+
if (this.accounts[account.index] === account) this._onTokenRefresh?.(account.index, {
|
|
1035
|
+
accessToken,
|
|
1036
|
+
refreshToken: account.refreshToken,
|
|
1037
|
+
expiresAt: account.expiresAt,
|
|
1038
|
+
});
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
/**
|
|
1042
|
+
* Add a new account at runtime.
|
|
1043
|
+
*/
|
|
1044
|
+
addAccount(acctData) {
|
|
1045
|
+
const index = this.accounts.length;
|
|
1046
|
+
this.accounts.push({
|
|
1047
|
+
index,
|
|
1048
|
+
name: acctData.name,
|
|
1049
|
+
type: acctData.type,
|
|
1050
|
+
accountUuid: acctData.accountUuid || null,
|
|
1051
|
+
credential: acctData.accessToken || acctData.apiKey,
|
|
1052
|
+
refreshToken: acctData.refreshToken || null,
|
|
1053
|
+
expiresAt: acctData.expiresAt || null,
|
|
1054
|
+
status: 'active',
|
|
1055
|
+
enabled: acctData.enabled !== false,
|
|
1056
|
+
priority: Number.isFinite(acctData.priority) ? Math.floor(acctData.priority) : null,
|
|
1057
|
+
quota: emptyQuota(),
|
|
1058
|
+
usage: { totalInputTokens: 0, totalOutputTokens: 0, totalRequests: 0, lastUsed: null },
|
|
1059
|
+
rateLimitedUntil: null,
|
|
1060
|
+
inflight: 0,
|
|
1061
|
+
maxConcurrent: coerceMaxConcurrent(acctData.maxConcurrent, this.maxConcurrentDefault),
|
|
1062
|
+
});
|
|
1063
|
+
// The new account has free capacity — hand it to any request waiting in the
|
|
1064
|
+
// overflow queue instead of letting it time out to a 429 while a usable
|
|
1065
|
+
// account sits idle.
|
|
1066
|
+
this._drainWaiters();
|
|
1067
|
+
return index;
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
/**
|
|
1071
|
+
* Remove an account by index.
|
|
1072
|
+
*/
|
|
1073
|
+
removeAccount(index) {
|
|
1074
|
+
if (index < 0 || index >= this.accounts.length) return;
|
|
1075
|
+
this.accounts.splice(index, 1);
|
|
1076
|
+
this.accounts.forEach((a, i) => a.index = i);
|
|
1077
|
+
if (this.currentIndex >= this.accounts.length) {
|
|
1078
|
+
this.currentIndex = Math.max(0, this.accounts.length - 1);
|
|
1079
|
+
} else if (this.currentIndex > index) {
|
|
1080
|
+
this.currentIndex--;
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
/**
|
|
1085
|
+
* Resolve a caller-facing account reference — an account object or a name
|
|
1086
|
+
* string — to the live account object (or null). Used by the public
|
|
1087
|
+
* setEnabled/setPriority.
|
|
1088
|
+
*
|
|
1089
|
+
* A bare numeric index is intentionally NOT accepted here (unlike the internal
|
|
1090
|
+
* `_resolve`): a setter is a mutation, and an index captured before a
|
|
1091
|
+
* removeAccount() re-index would silently disable/reprioritize whatever account
|
|
1092
|
+
* shifted into that slot. Callers pass the account object (TUI / sync) or its
|
|
1093
|
+
* name (CLI) — both survive a re-index.
|
|
1094
|
+
*/
|
|
1095
|
+
_resolveRef(ref) {
|
|
1096
|
+
if (typeof ref === 'string') return this.accounts.find(a => a.name === ref) || null;
|
|
1097
|
+
if (ref && typeof ref === 'object') return this.accounts.includes(ref) ? ref : null;
|
|
1098
|
+
return null;
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
/**
|
|
1102
|
+
* Enable or disable an account at runtime. A disabled account is excluded from
|
|
1103
|
+
* rotation (via _isAvailable) but keeps any in-flight requests until they
|
|
1104
|
+
* finish. Re-enabling hands its now-free capacity to any queued waiters.
|
|
1105
|
+
* Returns the affected account, or null if `ref` matched nothing.
|
|
1106
|
+
*/
|
|
1107
|
+
setEnabled(ref, enabled) {
|
|
1108
|
+
const account = this._resolveRef(ref);
|
|
1109
|
+
if (!account) return null;
|
|
1110
|
+
account.enabled = enabled !== false;
|
|
1111
|
+
// Re-evaluate the overflow queue either way: re-enabling hands the account's
|
|
1112
|
+
// free slots to waiters; disabling may leave a waiter that could *only* be
|
|
1113
|
+
// served by this account with no hope — _drainWaiters settles those now (so
|
|
1114
|
+
// they release their finite queue slot) instead of stranding them to timeout.
|
|
1115
|
+
this._drainWaiters();
|
|
1116
|
+
this._reprioritize();
|
|
1117
|
+
return account;
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
/**
|
|
1121
|
+
* Set (or clear) an account's explicit selection priority. Lower number =
|
|
1122
|
+
* preferred first; pass null/undefined/NaN to clear it (back to use-or-lose).
|
|
1123
|
+
* Returns the affected account, or null if `ref` matched nothing.
|
|
1124
|
+
*/
|
|
1125
|
+
setPriority(ref, priority) {
|
|
1126
|
+
const account = this._resolveRef(ref);
|
|
1127
|
+
if (!account) return null;
|
|
1128
|
+
account.priority = Number.isFinite(priority) ? Math.floor(priority) : null;
|
|
1129
|
+
this._reprioritize();
|
|
1130
|
+
return account;
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
/**
|
|
1134
|
+
* A preference change (enable/disable/priority) should take effect promptly,
|
|
1135
|
+
* not wait out the sticky `reevalIntervalMs` window (and not at all when the
|
|
1136
|
+
* timer is off). Re-pick the active account *directly* here — in either mode —
|
|
1137
|
+
* but ONLY when it actually matters: the current account is no longer usable
|
|
1138
|
+
* (e.g. just disabled), or a strictly higher-priority account is available.
|
|
1139
|
+
*
|
|
1140
|
+
* A no-op change (or one that doesn't dethrone the current account) leaves the
|
|
1141
|
+
* sticky primary untouched, so it can't churn cache locality. We deliberately
|
|
1142
|
+
* do NOT reset `lastEvalAt` to 0 — that would wake the periodic timer re-eval,
|
|
1143
|
+
* whose tie round-robin would switch the primary even when nothing changed.
|
|
1144
|
+
*/
|
|
1145
|
+
_reprioritize() {
|
|
1146
|
+
const current = this.accounts[this.currentIndex];
|
|
1147
|
+
const best = this._selectBest();
|
|
1148
|
+
if (!best || best.index === this.currentIndex) return;
|
|
1149
|
+
// Switch only when `best` is *strictly* preferred over the current account by
|
|
1150
|
+
// the full selection order (priority → soonest reset → least used), or the
|
|
1151
|
+
// current account is unusable. Comparing the full order — not priority alone —
|
|
1152
|
+
// means clearing a priority correctly restores use-or-lose routing, while a
|
|
1153
|
+
// true tie (best ranks equal to current) still leaves the sticky primary put
|
|
1154
|
+
// so there's no cache-churn.
|
|
1155
|
+
if (this._isAvailable(current) && !this._strictlyPrefer(best, current)) return;
|
|
1156
|
+
this.currentIndex = best.index;
|
|
1157
|
+
this.lastEvalAt = Date.now(); // just evaluated — don't also trigger a timer re-eval
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
/**
|
|
1161
|
+
* Is account `a` strictly preferred over `b` by the same lexicographic order
|
|
1162
|
+
* `_selectBest` sorts on: explicit priority (lower first), then soonest weekly
|
|
1163
|
+
* reset, then soonest session reset, then lowest utilization. Returns false
|
|
1164
|
+
* when they rank equal (a tie).
|
|
1165
|
+
*/
|
|
1166
|
+
_strictlyPrefer(a, b) {
|
|
1167
|
+
const pa = this._priority(a), pb = this._priority(b);
|
|
1168
|
+
if (pa !== pb) return pa < pb;
|
|
1169
|
+
const wa = this._weeklyResetTime(a), wb = this._weeklyResetTime(b);
|
|
1170
|
+
if (wa !== wb) return wa < wb;
|
|
1171
|
+
const ra = this._sessionResetTime(a), rb = this._sessionResetTime(b);
|
|
1172
|
+
if (ra !== rb) return ra < rb;
|
|
1173
|
+
return this._sessionUtilization(a) < this._sessionUtilization(b);
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
/**
|
|
1177
|
+
* Snapshot of per-account quota state for persistence across restarts
|
|
1178
|
+
* (credential-free). Quota lives only in memory otherwise, so a restart used
|
|
1179
|
+
* to blank the whole dashboard (and blind use-or-lose ordering) until traffic
|
|
1180
|
+
* organically re-measured every account.
|
|
1181
|
+
*/
|
|
1182
|
+
exportQuotaState() {
|
|
1183
|
+
return this.accounts.map(a => ({
|
|
1184
|
+
accountUuid: a.accountUuid || null,
|
|
1185
|
+
name: a.name,
|
|
1186
|
+
quota: {
|
|
1187
|
+
...a.quota,
|
|
1188
|
+
modelWeekly: Object.fromEntries(
|
|
1189
|
+
Object.entries(a.quota.modelWeekly).map(([k, w]) => [k, { ...w }])),
|
|
1190
|
+
},
|
|
1191
|
+
rateLimitedUntil: a.rateLimitedUntil,
|
|
1192
|
+
usage: { ...a.usage },
|
|
1193
|
+
}));
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
/**
|
|
1197
|
+
* Restore a quota snapshot from a previous run. A snapshot entry WITH an
|
|
1198
|
+
* accountUuid is matched by uuid ONLY — a same-name account with a different
|
|
1199
|
+
* uuid is a *replaced* account (a different underlying identity), and
|
|
1200
|
+
* restoring the old quota/throttle onto it would falsely mark a fresh
|
|
1201
|
+
* account near-quota or throttled. Name matching is the fallback solely for
|
|
1202
|
+
* entries without a uuid (API-key accounts, whose identity key is the name).
|
|
1203
|
+
* Unknown entries are skipped (→ unmeasured, exactly the pre-restore state).
|
|
1204
|
+
* Values may be slightly stale, but the proxy takes no traffic while it's
|
|
1205
|
+
* down, and expired windows are lazily swept by _isNearQuota on first use —
|
|
1206
|
+
* so a restore is strictly better than starting blind. A still-future
|
|
1207
|
+
* rateLimitedUntil re-throttles the account; error/exhausted statuses are
|
|
1208
|
+
* deliberately NOT restored (a bad token may have been fixed since).
|
|
1209
|
+
*/
|
|
1210
|
+
importQuotaState(saved) {
|
|
1211
|
+
for (const s of Array.isArray(saved) ? saved : []) {
|
|
1212
|
+
if (!s || typeof s !== 'object') continue;
|
|
1213
|
+
const a = s.accountUuid
|
|
1214
|
+
? this.accounts.find(x => x.accountUuid === s.accountUuid)
|
|
1215
|
+
: this.accounts.find(x => x.name === s.name);
|
|
1216
|
+
if (!a) continue;
|
|
1217
|
+
if (s.quota && typeof s.quota === 'object') {
|
|
1218
|
+
// Merge over emptyQuota so a cache written by an older version (missing
|
|
1219
|
+
// newer fields like modelWeekly) still yields a complete quota object.
|
|
1220
|
+
a.quota = {
|
|
1221
|
+
...emptyQuota(),
|
|
1222
|
+
...s.quota,
|
|
1223
|
+
// unifiedStatus is a PER-RESPONSE signal — isExhausted() treats a
|
|
1224
|
+
// 'rejected' here as "this 429 is account exhaustion". Restoring a
|
|
1225
|
+
// stale one would misclassify a later transient/headerless 429 as
|
|
1226
|
+
// exhaustion and wrongly throttle the account. Only a live response
|
|
1227
|
+
// (updateQuota) may set it.
|
|
1228
|
+
unifiedStatus: null,
|
|
1229
|
+
modelWeekly: Object.fromEntries(
|
|
1230
|
+
Object.entries(s.quota.modelWeekly && typeof s.quota.modelWeekly === 'object' ? s.quota.modelWeekly : {})
|
|
1231
|
+
.map(([k, w]) => [k, { ...w }])),
|
|
1232
|
+
};
|
|
1233
|
+
}
|
|
1234
|
+
if (s.usage && typeof s.usage === 'object') a.usage = { ...a.usage, ...s.usage };
|
|
1235
|
+
if (Number.isFinite(s.rateLimitedUntil) && s.rateLimitedUntil > Date.now()) {
|
|
1236
|
+
a.rateLimitedUntil = s.rateLimitedUntil;
|
|
1237
|
+
a.status = 'throttled';
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
/**
|
|
1243
|
+
* Return a status summary of all accounts (safe to expose, no credentials).
|
|
1244
|
+
*/
|
|
1245
|
+
getStatus() {
|
|
1246
|
+
return {
|
|
1247
|
+
currentAccount: this.accounts[this.currentIndex]?.name,
|
|
1248
|
+
switchThreshold: this.switchThreshold,
|
|
1249
|
+
accounts: this.accounts.map(a => ({
|
|
1250
|
+
name: a.name,
|
|
1251
|
+
type: a.type,
|
|
1252
|
+
status: a.status,
|
|
1253
|
+
enabled: a.enabled !== false,
|
|
1254
|
+
priority: a.priority ?? null,
|
|
1255
|
+
// Deep-copy the nested modelWeekly map — the shallow quota spread would
|
|
1256
|
+
// otherwise hand callers a live reference into account state.
|
|
1257
|
+
quota: {
|
|
1258
|
+
...a.quota,
|
|
1259
|
+
modelWeekly: Object.fromEntries(
|
|
1260
|
+
Object.entries(a.quota.modelWeekly).map(([k, w]) => [k, { ...w }])),
|
|
1261
|
+
},
|
|
1262
|
+
usage: { ...a.usage },
|
|
1263
|
+
inflight: a.inflight,
|
|
1264
|
+
maxConcurrent: a.maxConcurrent,
|
|
1265
|
+
rateLimitedUntil: a.rateLimitedUntil
|
|
1266
|
+
? new Date(a.rateLimitedUntil).toISOString()
|
|
1267
|
+
: null,
|
|
1268
|
+
})),
|
|
1269
|
+
};
|
|
1270
|
+
}
|
|
1271
|
+
}
|