teamclaude-cloud 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1271 @@
1
+ import { refreshAccessToken, isTokenExpiringSoon } from './oauth.js';
2
+
3
+ /** Coerce a per-account / global concurrency cap to a positive integer, else fallback. */
4
+ function coerceMaxConcurrent(value, fallback) {
5
+ return Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback;
6
+ }
7
+
8
+ function emptyQuota() {
9
+ return {
10
+ // Standard API rate limits (API key accounts)
11
+ tokensLimit: null,
12
+ tokensRemaining: null,
13
+ requestsLimit: null,
14
+ requestsRemaining: null,
15
+ // Unified rate limits (Claude Max accounts)
16
+ unified5h: null, // utilization 0-1
17
+ unified7d: null, // utilization 0-1
18
+ unified5hReset: null, // ms timestamp
19
+ unified7dReset: null, // ms timestamp
20
+ unifiedStatus: null, // allowed | allowed_warning | rejected
21
+ // Model-scoped weekly windows, keyed by header window label — e.g. `7d_oi`,
22
+ // the separate weekly limit for the top model tier shown as "Fable" in
23
+ // Claude's usage UI. Parsed generically from
24
+ // anthropic-ratelimit-unified-<window>-* so a renamed/added window keeps
25
+ // being tracked without a code change. Display-only: it never feeds
26
+ // availability, because an account over its Fable weekly limit still
27
+ // serves every other model.
28
+ modelWeekly: {}, // { '7d_oi': { utilization: 0-1, reset: msTimestamp } }
29
+ resetsAt: null, // soonest standard reset (session-order fallback)
30
+ // Token and request windows can reset at DIFFERENT times; tracked separately
31
+ // so retry-after can wait for whichever over-threshold window frees last
32
+ // (resetsAt alone collapses them, preferring the sooner token reset).
33
+ tokensReset: null, // standard token-window reset (date string)
34
+ requestsReset: null, // standard request-window reset (date string)
35
+ };
36
+ }
37
+
38
+ export class AccountManager {
39
+ constructor(accounts, switchThreshold = 0.98, reevalIntervalMs = 5 * 60 * 1000, maxConcurrentDefault = 3, overflowQueueMaxDepth = 256) {
40
+ this.maxConcurrentDefault = coerceMaxConcurrent(maxConcurrentDefault, 3);
41
+ // Hard cap on the overflow queue so a flood of concurrent requests can't grow
42
+ // it (and the buffered bodies / sockets / timers it pins) without bound. Past
43
+ // this depth acquireAccount rejects immediately (→ 429) instead of queuing.
44
+ this.maxQueueDepth = Number.isFinite(overflowQueueMaxDepth) && overflowQueueMaxDepth >= 0
45
+ ? Math.floor(overflowQueueMaxDepth) : 256;
46
+ this.accounts = accounts.map((acct, index) => ({
47
+ index,
48
+ name: acct.name,
49
+ type: acct.type,
50
+ accountUuid: acct.accountUuid || null,
51
+ credential: acct.accessToken || acct.apiKey,
52
+ refreshToken: acct.refreshToken || null,
53
+ expiresAt: acct.expiresAt || null,
54
+ status: 'active',
55
+ // Manual on/off switch. A disabled account is excluded from ALL rotation
56
+ // (warm-up, use-or-lose selection, recover, acquire) via _isAvailable —
57
+ // in-flight requests still drain, but no new request is routed to it.
58
+ // Defaults to enabled; only an explicit `enabled: false` disables it.
59
+ enabled: acct.enabled !== false,
60
+ // Explicit selection priority: lower number = preferred first. Null/unset
61
+ // means "no preference" — selection then falls back to use-or-lose. So a
62
+ // config with no priorities behaves exactly as before.
63
+ priority: Number.isFinite(acct.priority) ? Math.floor(acct.priority) : null,
64
+ quota: emptyQuota(),
65
+ usage: {
66
+ totalInputTokens: 0,
67
+ totalOutputTokens: 0,
68
+ totalRequests: 0,
69
+ lastUsed: null,
70
+ },
71
+ rateLimitedUntil: null,
72
+ // Concurrency: how many requests are in flight through this account right
73
+ // now, and the per-account cap above which the selector treats it as
74
+ // momentarily full (so concurrent load spreads to other accounts).
75
+ inflight: 0,
76
+ maxConcurrent: coerceMaxConcurrent(acct.maxConcurrent, this.maxConcurrentDefault),
77
+ }));
78
+ this.currentIndex = 0;
79
+ this.switchThreshold = switchThreshold;
80
+ this.reevalIntervalMs = reevalIntervalMs;
81
+ this.lastEvalAt = 0; // 0 forces a priority pick on the first request
82
+ this.maxWarmupTries = 3; // give up warming an account after this many unmeasured attempts
83
+ // Slow retry backstop for the active warm-up convergence cap: a capped
84
+ // account (deterministic fruitless probes) is retried once per this window,
85
+ // so an upstream that only LOOKED deterministic (e.g. temporarily
86
+ // header-less) recovers without a restart, while a truly pathological one
87
+ // costs at most one probe per window.
88
+ this.probeRetryAfterMs = 15 * 60 * 1000;
89
+ this._warmupCursor = 0; // round-robin pointer used during warm-up
90
+ this._waiters = []; // overflow queue: requests waiting for a free slot
91
+ // Soft connection→account affinity (keyed by the client socket). Keeps one
92
+ // keep-alive connection's *sequential* requests on the same account so
93
+ // Anthropic's per-account prompt cache stays warm. A WeakMap so an entry is
94
+ // GC'd when the socket is collected (connection closed) — no manual cleanup,
95
+ // no leak. The stored value is the account *object* (not its index, which
96
+ // shifts on removeAccount); a stale entry is detected and ignored.
97
+ this._affinity = new WeakMap();
98
+ }
99
+
100
+ /**
101
+ * Get the account to use for the next request.
102
+ *
103
+ * Policy:
104
+ * - Cold-start warm-up: while any available account is still unmeasured,
105
+ * route to it so its quota (usage % / reset) gets populated before any
106
+ * priority decision is made on incomplete data.
107
+ * - If the current account is unavailable (near quota / throttled / error),
108
+ * switch immediately to the highest-priority account. This is the old
109
+ * "switch at threshold" trigger — but it now picks by priority rather
110
+ * than round-robin to the next index.
111
+ * - Otherwise re-evaluate priority at most once per `reevalIntervalMs`
112
+ * (default 5 min) and switch if a higher-priority account exists. Set
113
+ * `reevalIntervalMs <= 0` (config `reevalIntervalMs: 0`) to disable this
114
+ * timer entirely — the account then only changes when it becomes
115
+ * unavailable or via per-request 429 failover.
116
+ * - Between re-evaluations the current account is sticky, so a request
117
+ * stream stays on one account and keeps Anthropic's per-account prompt
118
+ * cache warm.
119
+ *
120
+ * Priority is "use-or-lose": soonest WEEKLY (7d) reset first, then soonest
121
+ * session reset, then lowest session usage — so quota about to reset (and
122
+ * otherwise be wasted) is consumed first, starting with the scarcer weekly
123
+ * window. Returns null if every account is exhausted.
124
+ */
125
+ getActiveAccount(exclude = null) {
126
+ const now = Date.now();
127
+
128
+ // Per-request failover: a prior account already returned a non-quota 429
129
+ // for THIS request (those accounts are in `exclude`, a Set of objects). Pick another available
130
+ // account by priority WITHOUT touching the sticky primary or warm-up state
131
+ // — this diverts only the overflow of one request; steady-state selection
132
+ // still prefers the use-or-lose primary, keeping its prompt cache warm.
133
+ // Returns null once every available account has been tried this request.
134
+ if (exclude && exclude.size) return this._selectBest(exclude);
135
+
136
+ const current = this.accounts[this.currentIndex];
137
+
138
+ // Cold-start warm-up: until every available account has been measured at
139
+ // least once, round-robin across the unmeasured accounts so their quota
140
+ // (usage % / reset) gets populated. Round-robin (not "first unmeasured")
141
+ // means a concurrent startup burst of any size spreads evenly instead of
142
+ // hammering one unknown-quota account. Only once all are measured does the
143
+ // use-or-lose priority below take over — with complete data.
144
+ const warmup = this._nextWarmup();
145
+ if (warmup) {
146
+ if (warmup.index !== this.currentIndex) {
147
+ console.log(`[TeamClaude] Warm-up: measuring account "${warmup.name}"`);
148
+ this.currentIndex = warmup.index;
149
+ }
150
+ return warmup;
151
+ }
152
+
153
+ if (!this._isAvailable(current)) {
154
+ const best = this._selectBest();
155
+ if (best) {
156
+ if (best.index !== this.currentIndex) {
157
+ console.log(`[TeamClaude] Switched to account "${best.name}" (current unavailable)`);
158
+ }
159
+ this.currentIndex = best.index;
160
+ this.lastEvalAt = now;
161
+ }
162
+ return best;
163
+ }
164
+
165
+ // Periodic re-prioritization. Disabled when reevalIntervalMs <= 0: the
166
+ // current account then stays sticky and only changes when it becomes
167
+ // unavailable (exhausted / throttled / error) or via per-request 429
168
+ // failover — no timer-driven switching.
169
+ if (this.reevalIntervalMs > 0 && now - this.lastEvalAt >= this.reevalIntervalMs) {
170
+ this.lastEvalAt = now;
171
+ const best = this._selectBest();
172
+ if (best && best.index !== this.currentIndex) {
173
+ console.log(`[TeamClaude] Re-prioritized to account "${best.name}" (weekly reset soonest)`);
174
+ this.currentIndex = best.index;
175
+ return best;
176
+ }
177
+ }
178
+
179
+ // While the current account is still unmeasured, keep load-balancing via
180
+ // _selectBest (which rotates among equal-rank accounts) instead of sticking
181
+ // to an unknown-quota account — so a cold-start burst stays spread even
182
+ // after per-account warm-up attempts are exhausted.
183
+ if (!this._isMeasured(current)) {
184
+ const best = this._selectBest();
185
+ if (best) {
186
+ this.currentIndex = best.index;
187
+ return best;
188
+ }
189
+ }
190
+
191
+ return current;
192
+ }
193
+
194
+ // ── Concurrency layer: per-account in-flight cap + overflow queue ──────────
195
+ //
196
+ // getActiveAccount() above picks ONE account (sticky, use-or-lose). On its own
197
+ // that funnels every concurrent terminal onto the same account, which then hits
198
+ // Anthropic's per-account rate / concurrency limit (429) while other accounts
199
+ // sit idle with quota to spare. The layer below fixes that PROACTIVELY: each
200
+ // account carries an `inflight` counter and a `maxConcurrent` cap, and
201
+ // acquireAccount() treats a capped account as momentarily unavailable (folds it
202
+ // into the exclude set). The existing priority logic then naturally spreads
203
+ // load to the next account — filling A up to its cap, then B, then C, by
204
+ // use-or-lose priority. When every available account is at its cap the request
205
+ // waits briefly for a slot to free (overflow queue) instead of 429-storming.
206
+
207
+ /** Has this account a free concurrency slot? */
208
+ _hasCapacity(account) {
209
+ return account.inflight < account.maxConcurrent;
210
+ }
211
+
212
+ /**
213
+ * Resolve an account handle to the live account object. Accepts the object
214
+ * itself (reindex-safe — what server.js passes) or a numeric index (legacy /
215
+ * tests). All public per-account methods route their first arg through this so
216
+ * a stale index captured before a removeAccount() can't hit the wrong account.
217
+ */
218
+ _resolve(accountOrIndex) {
219
+ return typeof accountOrIndex === 'number' ? this.accounts[accountOrIndex] : accountOrIndex;
220
+ }
221
+
222
+ /**
223
+ * Available accounts currently at their concurrency cap, as a Set of account
224
+ * OBJECTS (not indexes). Object identity is stable across a removeAccount()
225
+ * re-index, so an exclude/capped set captured before the request awaits
226
+ * upstream can't later point at the wrong account.
227
+ */
228
+ _cappedSet(exclude = null) {
229
+ const capped = new Set();
230
+ for (const a of this.accounts) {
231
+ if (exclude && exclude.has(a)) continue;
232
+ if (this._isAvailable(a) && !this._hasCapacity(a)) capped.add(a);
233
+ }
234
+ return capped;
235
+ }
236
+
237
+ /** Is there an available account with a free slot (not excluded)? Non-mutating. (`exclude` = Set of account objects.) */
238
+ anyUsable(exclude = null) {
239
+ return this.accounts.some(a =>
240
+ this._isAvailable(a) && this._hasCapacity(a) && !(exclude && exclude.has(a)));
241
+ }
242
+
243
+ /** Is there an available-but-capped account (not excluded)? A freed slot could serve it. (`exclude` = Set of account objects.) */
244
+ anyCapped(exclude = null) {
245
+ return this.accounts.some(a =>
246
+ this._isAvailable(a) && !this._hasCapacity(a) && !(exclude && exclude.has(a)));
247
+ }
248
+
249
+ /**
250
+ * Synchronously pick + reserve the best account that is available AND has a
251
+ * free concurrency slot, honoring `exclude`. Capped accounts are folded into
252
+ * the exclusion so the existing getActiveAccount / _selectBest priority logic
253
+ * (warm-up, use-or-lose, recover) only ever chooses an account that can take
254
+ * the request. Increments the chosen account's inflight. Returns null when
255
+ * nothing is currently acquirable (all exhausted, excluded, or capped).
256
+ *
257
+ * Single-threaded JS keeps this race-free: there is no await between selecting
258
+ * the account and the inflight++ that reserves its slot.
259
+ */
260
+ _tryAcquire(exclude = null, affinityKey = null) {
261
+ // Only an object/function is a valid WeakMap key. Ignore anything else (a
262
+ // primitive key from an external caller would otherwise throw on get/set).
263
+ const affOk = affinityKey != null
264
+ && (typeof affinityKey === 'object' || typeof affinityKey === 'function');
265
+
266
+ // Connection affinity (cache locality): prefer the account this connection
267
+ // already used — but only as a *soft* hint, and DEFER to cold-start warm-up.
268
+ // While any account still needs measuring, skip affinity so it can't pin all
269
+ // of a connection's traffic to one account and starve the others of quota
270
+ // data (warm-up round-robins the unmeasured accounts instead). Once measured,
271
+ // affinity is honored only when that account is still available, has a free
272
+ // slot, and isn't excluded for this request; otherwise it falls through to
273
+ // normal selection. So it never exceeds a cap, revives an exhausted account,
274
+ // or disturbs use-or-lose for new connections. (`accounts[idx] === a` rejects
275
+ // a stale entry left by a removeAccount that re-indexed the array.)
276
+ if (affOk && !this.accounts.some(acc => this._isWarmupTarget(acc))) {
277
+ const a = this._affinity.get(affinityKey);
278
+ // Require the home to be MEASURED — not just past its warm-up tries. A
279
+ // headerless account stays unmeasured forever; pinning a connection to it
280
+ // would bypass getActiveAccount's unmeasured-rebalance (which keeps
281
+ // spreading to gather quota data / let tokens refresh on use). Once an
282
+ // account returns rate-limit headers (every real Anthropic response does),
283
+ // affinity engages normally.
284
+ if (a && this.accounts[a.index] === a && this._isMeasured(a) && this._isAvailable(a)
285
+ && this._hasCapacity(a) && !(exclude && exclude.has(a))) {
286
+ a.inflight++;
287
+ return a;
288
+ }
289
+ }
290
+
291
+ const capped = this._cappedSet(exclude);
292
+ const eff = ((exclude && exclude.size) || capped.size)
293
+ ? new Set([...(exclude || []), ...capped])
294
+ : null;
295
+ // eff === null → full sticky / warm-up path (cold start, nothing capped).
296
+ // eff set → getActiveAccount routes to _selectBest(eff), which already skips
297
+ // every excluded + capped account.
298
+ const account = eff ? this.getActiveAccount(eff) : this.getActiveAccount();
299
+ if (account && this._isAvailable(account) && this._hasCapacity(account)
300
+ && !(eff && eff.has(account))) {
301
+ account.inflight++;
302
+ // (Re)write affinity ONLY when the connection has no still-usable home.
303
+ // Reaching this fall-through path means we left the home account — but that
304
+ // can be merely transient: the home may be momentarily capped (overflow
305
+ // spill) or failover-excluded for THIS request, yet still perfectly
306
+ // available. Overwriting it then would let one blip permanently evict the
307
+ // connection from its cache-warm account. So keep an available home (even
308
+ // capped/excluded right now); replace it only when it's genuinely gone
309
+ // (removed, unavailable, or exhausted — `_isAvailable` is false).
310
+ if (affOk) {
311
+ const home = this._affinity.get(affinityKey);
312
+ const homeUsable = home && this.accounts[home.index] === home && this._isAvailable(home);
313
+ if (!homeUsable) this._affinity.set(affinityKey, account);
314
+ }
315
+ return account;
316
+ }
317
+ return null;
318
+ }
319
+
320
+ /**
321
+ * Acquire an account for a request, reserving one of its concurrency slots.
322
+ * If none is immediately acquirable but an available account is merely at its
323
+ * cap (overflow), wait up to `timeoutMs` for a slot to free — a releaseAccount
324
+ * elsewhere wakes the waiter. Returns null when every account is genuinely
325
+ * unavailable (quota-exhausted / auth-error / excluded) or the wait times out,
326
+ * so the caller surfaces a 429 for the client to back off on.
327
+ *
328
+ * The caller MUST releaseAccount(account) exactly once when the request
329
+ * (including any streamed body) finishes — pass the returned account OBJECT,
330
+ * not its index, so a concurrent removeAccount() can't misattribute the slot.
331
+ * `exclude` is a Set of account OBJECTS (per-request failover).
332
+ */
333
+ async acquireAccount(exclude = null, timeoutMs = 0, signal = null, affinityKey = null) {
334
+ if (signal?.aborted) return null;
335
+ const account = this._tryAcquire(exclude, affinityKey);
336
+ if (account) return account;
337
+ // Queue only when the blockage is cap-saturation (a slot WILL free as
338
+ // in-flight requests finish) AND the queue isn't already full. If no
339
+ // available account exists at all, or the queue is at its depth cap, return
340
+ // null and let the caller 429 — never grow the backlog without bound.
341
+ if (timeoutMs <= 0 || !this.anyCapped(exclude) || this.isQueueFull()) return null;
342
+ return this._enqueue(exclude, timeoutMs, signal, affinityKey);
343
+ }
344
+
345
+ /** Is the overflow queue at its depth cap? */
346
+ isQueueFull() {
347
+ return this._waiters.length >= this.maxQueueDepth;
348
+ }
349
+
350
+ /**
351
+ * Upper bound on concurrent in-flight requests the proxy may admit (server.js
352
+ * caps `inFlightProxied` to this to bound buffered memory): each ENABLED
353
+ * account contributes its full cap (capacity it can still take), each DISABLED
354
+ * account contributes only its *current* in-flight (requests still draining —
355
+ * it accepts no new ones), plus the queue depth.
356
+ *
357
+ * This is the tightest bound that's still safe: it covers the draining requests
358
+ * on a just-disabled account (so they can't push inFlightProxied over the
359
+ * ceiling and 429 traffic the enabled accounts could serve), without admitting
360
+ * fresh requests against a disabled account's dead future capacity (which could
361
+ * only be buffered and then 429'd at acquire). As those draining requests
362
+ * finish, the disabled account's contribution falls to zero.
363
+ */
364
+ totalCapacity() {
365
+ const caps = this.accounts.reduce(
366
+ (sum, a) => sum + (a.enabled === false ? a.inflight : a.maxConcurrent), 0);
367
+ return caps + this.maxQueueDepth;
368
+ }
369
+
370
+ _enqueue(exclude, timeoutMs, signal = null, affinityKey = null) {
371
+ return new Promise(resolve => {
372
+ const waiter = { exclude, resolve, done: false, timer: null, signal, onAbort: null, affinityKey };
373
+ waiter.timer = setTimeout(() => this._settleWaiter(waiter, null), timeoutMs);
374
+ // Cancel the wait if the client disconnects — otherwise an aborted request
375
+ // would still acquire a slot later and be dispatched upstream, burning quota.
376
+ if (signal) {
377
+ waiter.onAbort = () => this._settleWaiter(waiter, null);
378
+ signal.addEventListener('abort', waiter.onAbort, { once: true });
379
+ }
380
+ this._waiters.push(waiter);
381
+ });
382
+ }
383
+
384
+ /** Resolve a queued waiter exactly once, cleaning up its timer/abort listener. */
385
+ _settleWaiter(waiter, value) {
386
+ if (waiter.done) return false;
387
+ waiter.done = true;
388
+ clearTimeout(waiter.timer);
389
+ if (waiter.signal && waiter.onAbort) waiter.signal.removeEventListener('abort', waiter.onAbort);
390
+ const i = this._waiters.indexOf(waiter);
391
+ if (i >= 0) this._waiters.splice(i, 1);
392
+ waiter.resolve(value);
393
+ return true;
394
+ }
395
+
396
+ /**
397
+ * Release a concurrency slot held by a request and hand any freed capacity to
398
+ * the longest-waiting overflow request that can use it (FIFO, but a waiter
399
+ * whose exclude set can't currently be satisfied is skipped rather than
400
+ * head-of-line blocking a later waiter that can run).
401
+ */
402
+ releaseAccount(accountOrIndex) {
403
+ // Resolve to the account OBJECT (what the server holds — reindex-safe across a
404
+ // removeAccount) so a release decrements the slot of the *account that was
405
+ // acquired*, never whatever happens to sit at that index now. A numeric index
406
+ // is still accepted for convenience/tests.
407
+ const account = this._resolve(accountOrIndex);
408
+ if (account && account.inflight > 0) account.inflight--;
409
+ this._drainWaiters();
410
+ }
411
+
412
+ _drainWaiters() {
413
+ for (let i = 0; i < this._waiters.length;) {
414
+ const waiter = this._waiters[i];
415
+ const account = this._tryAcquire(waiter.exclude, waiter.affinityKey);
416
+ if (account) {
417
+ // _settleWaiter splices the waiter out, so don't advance i. If it was
418
+ // already settled (shouldn't happen — settled waiters aren't in the list),
419
+ // give the slot back instead of leaking it.
420
+ if (!this._settleWaiter(waiter, account)) { account.inflight--; i++; }
421
+ continue;
422
+ }
423
+ // No slot right now. If no account this waiter could use is even
424
+ // available-but-capped, nothing will ever free for it (e.g. the account it
425
+ // was queued for just got disabled or exhausted) — settle it null so it
426
+ // releases its finite queue slot instead of blocking later, satisfiable
427
+ // overflow requests until its timeout. A waiter that still has a cappable
428
+ // account to hope for is left in place.
429
+ if (!this.anyCapped(waiter.exclude)) { this._settleWaiter(waiter, null); continue; }
430
+ i++;
431
+ }
432
+ }
433
+
434
+ /**
435
+ * Highest-priority available account by use-or-lose ordering: soonest WEEKLY
436
+ * (7d) reset first — weekly quota is the scarce resource, so an account whose
437
+ * week is about to renew (and whose unspent quota would be wasted) is drained
438
+ * first — then soonest session reset, then lowest session utilization. Falls
439
+ * back to the soonest-resetting account when none are currently available.
440
+ *
441
+ * `exclude` (a Set of account objects) is used for per-request failover: those
442
+ * accounts are skipped, and when nothing else is eligible this returns null
443
+ * (instead of recovering one) so the caller can pass the 429 through.
444
+ */
445
+ _selectBest(exclude = null) {
446
+ const has = a => (exclude ? exclude.has(a) : false);
447
+ const eligible = this.accounts.filter(a => this._isAvailable(a) && !has(a));
448
+ if (eligible.length === 0) return exclude ? null : this._recoverSoonest();
449
+
450
+ eligible.sort((a, b) => {
451
+ const pa = this._priority(a);
452
+ const pb = this._priority(b);
453
+ if (pa !== pb) return pa - pb; // explicit priority first (lower = preferred)
454
+ return this.autoCompare(a, b); // then the automatic use-or-lose order
455
+ });
456
+
457
+ // Accounts tied for the best rank (notably all-unknown at cold start, or all
458
+ // sharing one priority) are load-balanced round-robin instead of always
459
+ // pinning to the lowest index, so a startup burst can't pile onto one account
460
+ // before quotas are known.
461
+ const p0 = this._priority(eligible[0]);
462
+ const w0 = this._weeklyResetTime(eligible[0]);
463
+ const r0 = this._sessionResetTime(eligible[0]);
464
+ const u0 = this._sessionUtilization(eligible[0]);
465
+ const tied = eligible
466
+ .filter(a => this._priority(a) === p0
467
+ && this._weeklyResetTime(a) === w0
468
+ && this._sessionResetTime(a) === r0
469
+ && this._sessionUtilization(a) === u0)
470
+ .sort((a, b) => a.index - b.index);
471
+ if (tied.length <= 1) return eligible[0];
472
+ return tied.find(a => a.index > this.currentIndex) || tied[0];
473
+ }
474
+
475
+ /**
476
+ * Explicit selection priority: lower = preferred. Unset (null) sorts last
477
+ * (Infinity) so an account WITH any finite priority — however large — is chosen
478
+ * ahead of those without. When no account sets a priority, every account ties
479
+ * here (Infinity === Infinity) and the sort falls through to use-or-lose, i.e.
480
+ * the original behavior unchanged. The callers compare with a `pa !== pb` guard
481
+ * before any subtraction, so Infinity never produces a NaN sort key.
482
+ */
483
+ _priority(account) {
484
+ return Number.isFinite(account.priority) ? account.priority : Infinity;
485
+ }
486
+
487
+ /**
488
+ * The automatic ("auto") use-or-lose comparator, shared by selection and the
489
+ * TUI display order: soonest WEEKLY reset (drain what renews first) → soonest
490
+ * session reset → lowest session utilization. Returns 0 on a full tie, so a
491
+ * stable sort keeps ties in array order (the pre-weekly behavior for API-key
492
+ * fleets and unmeasured accounts).
493
+ */
494
+ autoCompare(a, b) {
495
+ const wa = this._weeklyResetTime(a);
496
+ const wb = this._weeklyResetTime(b);
497
+ if (wa !== wb) return wa - wb;
498
+ const ra = this._sessionResetTime(a);
499
+ const rb = this._sessionResetTime(b);
500
+ if (ra !== rb) return ra - rb;
501
+ return this._sessionUtilization(a) - this._sessionUtilization(b);
502
+ }
503
+
504
+ /**
505
+ * Weekly reset timestamp (ms): unified 7d (Max) → Infinity. API-key accounts
506
+ * have no weekly window, so they tie at Infinity and the session tiebreak
507
+ * decides — exactly the pre-weekly-ordering behavior. The window counts only
508
+ * when BOTH utilization and reset are present: a partial/garbled header pair
509
+ * (reset without utilization) must not outrank accounts with no 7d data,
510
+ * matching the documented "no weekly data ranks at Infinity" semantics.
511
+ *
512
+ * A timestamp that has PASSED ranks at Infinity too: the moment a window
513
+ * rolls over, the account's old "resets soonest" claim is void (its fresh
514
+ * window is unknown until re-measured) — without this, the past timestamp
515
+ * (smallest value) would pin the account at the top of the order until a
516
+ * request-path sweep happened to clear it, so the order would NOT follow
517
+ * reset rollovers. The lazy sweep in _isNearQuota still clears the fields;
518
+ * this just makes ORDERING (selection and the TUI display, which has no
519
+ * sweep) reflect the rollover instantly.
520
+ */
521
+ _weeklyResetTime(account) {
522
+ const q = account.quota;
523
+ const r = (q.unified7d != null && q.unified7dReset) ? q.unified7dReset : Infinity;
524
+ return r > Date.now() ? r : Infinity;
525
+ }
526
+
527
+ /**
528
+ * Session reset timestamp (ms): unified 5h (Max) → standard reset → Infinity.
529
+ * Expired timestamps rank at Infinity for the same rollover reason as
530
+ * _weeklyResetTime above.
531
+ */
532
+ _sessionResetTime(account) {
533
+ const q = account.quota;
534
+ const r = q.unified5hReset
535
+ || (q.resetsAt ? new Date(q.resetsAt).getTime() : Infinity);
536
+ return r > Date.now() ? r : Infinity;
537
+ }
538
+
539
+ /** Session utilization 0–1: unified 5h (Max) → standard token/request usage → 0. */
540
+ _sessionUtilization(account) {
541
+ const q = account.quota;
542
+ if (q.unified5h != null) return q.unified5h;
543
+ if (q.tokensLimit != null && q.tokensRemaining != null) {
544
+ return 1 - q.tokensRemaining / q.tokensLimit;
545
+ }
546
+ if (q.requestsLimit != null && q.requestsRemaining != null) {
547
+ return 1 - q.requestsRemaining / q.requestsLimit;
548
+ }
549
+ return 0;
550
+ }
551
+
552
+ /**
553
+ * Clear every account's expired quota windows NOW. The lazy sweep inside
554
+ * _isNearQuota only runs on selection paths (i.e. when a request flows), so
555
+ * on an idle proxy a rolled-over window would keep its stale values — and
556
+ * stay "measured", which prevents the periodic active warm-up from
557
+ * re-probing it. The server's warm-up timer calls this first, closing the
558
+ * loop: rollover → sweep → unmeasured → probe → fresh data → order updates.
559
+ * Idempotent and cheap (pure field clears).
560
+ */
561
+ sweepExpired() {
562
+ for (const a of this.accounts) this._isNearQuota(a);
563
+ }
564
+
565
+ /** True once we have any quota data for this account (rate-limit headers seen). */
566
+ _isMeasured(account) {
567
+ const q = account.quota;
568
+ return q.unified5h != null || q.unified7d != null
569
+ || q.tokensLimit != null || q.requestsLimit != null;
570
+ }
571
+
572
+ /**
573
+ * Fully measured, for ACTIVE warm-up candidacy: an OAuth (Max) response
574
+ * always carries both the 5h and the 7d window, so a missing half — e.g. a
575
+ * weekly rollover swept `unified7d` while the session window survived —
576
+ * means the account needs a re-probe or its weekly quota/ordering stays
577
+ * unknown until real traffic reaches it. One probe repopulates both windows,
578
+ * so candidacy converges. API-key accounts keep the any-data semantics.
579
+ */
580
+ _fullyMeasured(account) {
581
+ if (account.type === 'oauth') {
582
+ // A window counts only when COMPLETE (utilization AND reset) — the same
583
+ // semantics the ordering helpers use. Utilization without its reset
584
+ // timestamp gives use-or-lose nothing to sort on, so such an account
585
+ // still needs a re-probe.
586
+ const q = account.quota;
587
+ return q.unified5h != null && q.unified5hReset != null
588
+ && q.unified7d != null && q.unified7dReset != null;
589
+ }
590
+ return this._isMeasured(account);
591
+ }
592
+
593
+ /**
594
+ * A fully-measured OAuth account whose model-scoped weekly window (the Fable
595
+ * `7d_oi` limit) is still absent. Such a window only appears on responses to
596
+ * Fable-tier requests, so an account that was measured by lower-tier traffic
597
+ * or a lower-tier probe keeps its `Fbl` bar blank — and because it IS fully
598
+ * measured for 5h/7d, ordinary warm-up (which only targets unmeasured
599
+ * accounts) never re-probes it. This flags it for a bounded model-weekly
600
+ * top-up probe, run ONLY when the committed probe template can actually
601
+ * elicit the window (see server.js). The `_mwProbes` cap stops an account
602
+ * whose upstream genuinely never reports the window from being probed every
603
+ * interval forever; it resets when the window populates or a quota window is
604
+ * swept (a fresh week is a fresh reason to look).
605
+ */
606
+ needsModelWeekly(account) {
607
+ return account.type === 'oauth'
608
+ && this._fullyMeasured(account)
609
+ && Object.keys(account.quota.modelWeekly).length === 0
610
+ && (account._mwProbes || 0) < this.maxWarmupTries;
611
+ }
612
+
613
+ /**
614
+ * An account still needing warm-up: available, not yet MEASURED, under the
615
+ * per-account attempt cap.
616
+ *
617
+ * Keying on `!_isMeasured` (not on "has it made a request") is deliberate: a
618
+ * request can return *no* rate-limit headers — a `HEAD /` health check, a
619
+ * 404, an auth failure — which would leave the account unmeasured. Gating
620
+ * warm-up on `totalRequests === 0` used to permanently disqualify such an
621
+ * account after that single header-less request, trapping it as "unmeasured"
622
+ * forever: it then sorts to the bottom of use-or-lose priority (no reset
623
+ * data) and the unmeasured-rebalance bounces any switch away from it, so it
624
+ * never gets used again — and its token never gets refreshed, so it expires.
625
+ *
626
+ * maxWarmupTries provides the loop-safety instead: a genuinely dead account
627
+ * (always header-less / 401) is abandoned after a few attempts rather than
628
+ * looping forever. (An expired-token account is resolved on its first warm-up
629
+ * routing anyway — ensureTokenFresh either refreshes it into a measurable
630
+ * state or marks it `error`, which makes it unavailable here.)
631
+ */
632
+ _isWarmupTarget(account) {
633
+ return this._isAvailable(account)
634
+ && !this._isMeasured(account)
635
+ && (account._warmupTries || 0) < this.maxWarmupTries;
636
+ }
637
+
638
+ /**
639
+ * Next account to warm up, round-robin across the warm-up targets so a burst
640
+ * spreads evenly. Advances the cursor and bumps the chosen account's attempt
641
+ * counter synchronously, so concurrent calls pick different accounts even
642
+ * before any response arrives. Returns null when no target remains.
643
+ */
644
+ _nextWarmup() {
645
+ const n = this.accounts.length;
646
+ for (let i = 0; i < n; i++) {
647
+ const idx = (this._warmupCursor + i) % n;
648
+ const a = this.accounts[idx];
649
+ if (this._isWarmupTarget(a)) {
650
+ this._warmupCursor = idx + 1;
651
+ a._warmupTries = (a._warmupTries || 0) + 1;
652
+ return a;
653
+ }
654
+ }
655
+ return null;
656
+ }
657
+
658
+ /**
659
+ * Accounts eligible for an *active* warm-up probe: available (enabled, not
660
+ * throttled / exhausted / error), with no quota data yet, AND not already
661
+ * handling a request. The server sends each one a minimal upstream request to
662
+ * populate its quota so the dashboard reflects the whole fleet shortly after a
663
+ * (re)start, instead of waiting for client traffic to organically reach every
664
+ * account.
665
+ *
666
+ * `inflight === 0` matters: a request already in flight will itself populate
667
+ * the account's quota (updateQuota runs on its response headers), so probing it
668
+ * would just race that request and waste an upstream call. Cold start's very
669
+ * first request holds its (still-unmeasured) account here, so the startup
670
+ * fan-out probes only the genuinely idle rest of the fleet — never the account
671
+ * that request is already measuring. (An unmeasured account can't be near-quota,
672
+ * so no extra status carve-outs are needed beyond _isAvailable.)
673
+ */
674
+ warmupCandidates() {
675
+ return this.accounts.filter(a =>
676
+ this._isAvailable(a) && !this._fullyMeasured(a) && a.inflight === 0
677
+ // Convergence cap: against the real upstream one probe fully measures an
678
+ // account (responses always carry both window families), but a
679
+ // pathological upstream — a 2xx missing a family, header-less responses,
680
+ // or a deterministic 4xx — must not be probed every interval forever. A
681
+ // probe that completes with such a deterministic fruitless outcome
682
+ // increments _partialProbes (transient 5xx/429/network failures don't
683
+ // count — see warmupAccount); after maxWarmupTries fruitless probes the
684
+ // account stops being a candidate. The counter resets when a window is
685
+ // swept (a fresh rollover is a fresh reason to probe) or when the
686
+ // account becomes fully measured — and, because a fully UNMEASURED
687
+ // account has no reset timestamp for the sweep to fire on, a capped
688
+ // account is retried once per probeRetryAfterMs as a slow backstop, so a
689
+ // "deterministic-looking" outage (e.g. headers temporarily missing) still
690
+ // recovers instead of suppressing active warm-up until a restart.
691
+ && ((a._partialProbes || 0) < this.maxWarmupTries
692
+ || Date.now() - (a._lastFruitlessProbeAt || 0) >= this.probeRetryAfterMs));
693
+ }
694
+
695
+ _isAvailable(account) {
696
+ if (!account) return false;
697
+
698
+ // Manually disabled accounts are out of rotation entirely. This single gate
699
+ // covers every selection path (warm-up target, _selectBest, _cappedSet,
700
+ // anyUsable/anyCapped, the sticky-current check) so a disabled account is
701
+ // never chosen for a new request. _recoverSoonest iterates accounts directly
702
+ // (not via this), so it skips disabled accounts itself.
703
+ if (account.enabled === false) return false;
704
+
705
+ // Check rate limit expiry
706
+ if (account.status === 'throttled' && account.rateLimitedUntil) {
707
+ if (Date.now() < account.rateLimitedUntil) return false;
708
+ account.status = 'active';
709
+ account.rateLimitedUntil = null;
710
+ console.log(`[TeamClaude] Account "${account.name}" rate limit expired, marking active`);
711
+ }
712
+
713
+ if (account.status === 'exhausted' || account.status === 'error') return false;
714
+ if (this._isNearQuota(account)) return false;
715
+
716
+ return true;
717
+ }
718
+
719
+ _isNearQuota(account) {
720
+ const q = account.quota;
721
+ const now = Date.now();
722
+
723
+ // Clear expired unified quotas. The reset timestamp is cleared even when the
724
+ // matching utilization was never set (a partial/garbled header pair) — a
725
+ // stale past timestamp would otherwise survive forever and, since selection
726
+ // sorts by reset time, permanently bias the ordering toward that account.
727
+ if (q.unified5hReset && now >= q.unified5hReset) {
728
+ if (q.unified5h != null) console.log(`[TeamClaude] Account "${account.name}" session quota reset`);
729
+ q.unified5h = null;
730
+ q.unified5hReset = null;
731
+ // Fresh rollover → BOTH warm-up budgets renew: active probes
732
+ // (_partialProbes) and the passive request-routing warm-up
733
+ // (_warmupTries) get a fresh reason to re-measure this window.
734
+ account._partialProbes = 0;
735
+ account._warmupTries = 0;
736
+ }
737
+ if (q.unified7dReset && now >= q.unified7dReset) {
738
+ if (q.unified7d != null) console.log(`[TeamClaude] Account "${account.name}" weekly quota reset`);
739
+ q.unified7d = null;
740
+ q.unified7dReset = null;
741
+ q.unifiedStatus = null;
742
+ account._partialProbes = 0; // fresh rollover → re-probes allowed again
743
+ account._warmupTries = 0;
744
+ }
745
+ // Clear expired model-scoped weekly windows (display-only, but a stale
746
+ // "94% Fable" bar after the window reset would mislead). A cleared window is
747
+ // a fresh reason to top it up again, so renew the model-weekly probe budget.
748
+ for (const [label, win] of Object.entries(q.modelWeekly)) {
749
+ if (win.reset && now >= win.reset) {
750
+ console.log(`[TeamClaude] Account "${account.name}" ${label} quota reset`);
751
+ delete q.modelWeekly[label];
752
+ account._mwProbes = 0;
753
+ }
754
+ }
755
+
756
+ // Clear expired standard quotas — each window INDEPENDENTLY. The token and
757
+ // request windows reset at different times; sweeping both on the collapsed
758
+ // resetsAt (token-first) made a still-blocked request window vanish the
759
+ // moment the sooner token window reset, freeing the account ~an hour early.
760
+ // A window without its own reset falls back to resetsAt (old snapshots /
761
+ // upstreams that only send one reset header) — same sweep as before.
762
+ const tokensResetAt = q.tokensReset || q.resetsAt;
763
+ if (tokensResetAt && now >= new Date(tokensResetAt).getTime()) {
764
+ q.tokensRemaining = null;
765
+ q.tokensLimit = null;
766
+ q.tokensReset = null;
767
+ }
768
+ const requestsResetAt = q.requestsReset || q.resetsAt;
769
+ if (requestsResetAt && now >= new Date(requestsResetAt).getTime()) {
770
+ q.requestsRemaining = null;
771
+ q.requestsLimit = null;
772
+ q.requestsReset = null;
773
+ }
774
+ // Advance the collapsed resetsAt (session-ordering fallback) to the soonest
775
+ // still-future window reset, or clear it once no window remains.
776
+ if (q.resetsAt && now >= new Date(q.resetsAt).getTime()) {
777
+ const future = [q.tokensReset, q.requestsReset]
778
+ .map(r => (r ? new Date(r).getTime() : null))
779
+ .filter(t => t && t > now);
780
+ q.resetsAt = future.length ? new Date(Math.min(...future)).toISOString() : null;
781
+ }
782
+
783
+ // Unified quotas (Claude Max) — utilization is already 0-1
784
+ if (q.unified5h != null && q.unified5h >= this.switchThreshold) return true;
785
+ if (q.unified7d != null && q.unified7d >= this.switchThreshold) return true;
786
+
787
+ // Standard quotas (API key accounts)
788
+ if (q.tokensLimit != null && q.tokensRemaining != null) {
789
+ const used = 1 - (q.tokensRemaining / q.tokensLimit);
790
+ if (used >= this.switchThreshold) return true;
791
+ }
792
+
793
+ if (q.requestsLimit != null && q.requestsRemaining != null) {
794
+ const used = 1 - (q.requestsRemaining / q.requestsLimit);
795
+ if (used >= this.switchThreshold) return true;
796
+ }
797
+
798
+ return false;
799
+ }
800
+
801
+ /** When all accounts are unavailable, return the soonest to reset (if it has already reset). */
802
+ _recoverSoonest() {
803
+ let soonestAccount = null;
804
+ let soonestTime = Infinity;
805
+
806
+ for (const account of this.accounts) {
807
+ // Never recover a manually-disabled account into rotation.
808
+ if (account.enabled === false) continue;
809
+ const resetTime = account.rateLimitedUntil
810
+ || account.quota.unified5hReset
811
+ || account.quota.unified7dReset
812
+ || (account.quota.resetsAt ? new Date(account.quota.resetsAt).getTime() : null);
813
+
814
+ if (resetTime && resetTime < soonestTime) {
815
+ soonestTime = resetTime;
816
+ soonestAccount = account;
817
+ }
818
+ }
819
+
820
+ if (soonestAccount && soonestTime <= Date.now()) {
821
+ soonestAccount.status = 'active';
822
+ soonestAccount.rateLimitedUntil = null;
823
+ this.currentIndex = soonestAccount.index;
824
+ console.log(`[TeamClaude] Account "${soonestAccount.name}" reset, switching to it`);
825
+ return soonestAccount;
826
+ }
827
+
828
+ return null;
829
+ }
830
+
831
+ /**
832
+ * Update an account's quota tracking from upstream response headers.
833
+ */
834
+ updateQuota(accountIndex, headers) {
835
+ const account = this._resolve(accountIndex);
836
+ if (!account) return;
837
+
838
+ // Unified rate limits (Claude Max)
839
+ const u5h = parseFloat(headers['anthropic-ratelimit-unified-5h-utilization']);
840
+ const u7d = parseFloat(headers['anthropic-ratelimit-unified-7d-utilization']);
841
+ if (!isNaN(u5h)) account.quota.unified5h = u5h;
842
+ if (!isNaN(u7d)) account.quota.unified7d = u7d;
843
+
844
+ const r5h = headers['anthropic-ratelimit-unified-5h-reset'];
845
+ const r7d = headers['anthropic-ratelimit-unified-7d-reset'];
846
+ if (r5h) account.quota.unified5hReset = parseInt(r5h, 10) * 1000;
847
+ if (r7d) account.quota.unified7dReset = parseInt(r7d, 10) * 1000;
848
+
849
+ const uStatus = headers['anthropic-ratelimit-unified-status'];
850
+ if (uStatus) account.quota.unifiedStatus = uStatus;
851
+
852
+ // Model-scoped weekly windows (7d_<label>), e.g. `7d_oi` — the weekly limit
853
+ // for the top model tier ("Fable" in Claude's usage UI). These headers only
854
+ // appear on responses to requests for that model tier, so the value sticks
855
+ // around from the last such request. Matched generically so a renamed or
856
+ // newly added window is picked up as-is.
857
+ for (const [key, value] of Object.entries(headers)) {
858
+ const m = /^anthropic-ratelimit-unified-(7d_[a-z0-9_]+)-(utilization|reset)$/.exec(key);
859
+ if (!m) continue;
860
+ const win = account.quota.modelWeekly[m[1]]
861
+ || (account.quota.modelWeekly[m[1]] = { utilization: null, reset: null });
862
+ if (m[2] === 'utilization') {
863
+ const u = parseFloat(value);
864
+ if (!isNaN(u)) win.utilization = u;
865
+ } else {
866
+ const r = parseInt(value, 10);
867
+ if (!isNaN(r)) win.reset = r * 1000;
868
+ }
869
+ }
870
+
871
+ // Standard rate limits (API key accounts)
872
+ const tokensLimit = parseInt(headers['anthropic-ratelimit-tokens-limit'], 10);
873
+ const tokensRemaining = parseInt(headers['anthropic-ratelimit-tokens-remaining'], 10);
874
+ const tokensReset = headers['anthropic-ratelimit-tokens-reset'];
875
+ const requestsLimit = parseInt(headers['anthropic-ratelimit-requests-limit'], 10);
876
+ const requestsRemaining = parseInt(headers['anthropic-ratelimit-requests-remaining'], 10);
877
+ const requestsReset = headers['anthropic-ratelimit-requests-reset'];
878
+
879
+ if (!isNaN(tokensLimit)) account.quota.tokensLimit = tokensLimit;
880
+ if (!isNaN(tokensRemaining)) account.quota.tokensRemaining = tokensRemaining;
881
+ if (!isNaN(requestsLimit)) account.quota.requestsLimit = requestsLimit;
882
+ if (!isNaN(requestsRemaining)) account.quota.requestsRemaining = requestsRemaining;
883
+
884
+ if (tokensReset) account.quota.tokensReset = tokensReset;
885
+ if (requestsReset) account.quota.requestsReset = requestsReset;
886
+ if (tokensReset) account.quota.resetsAt = tokensReset;
887
+ else if (requestsReset) account.quota.resetsAt = requestsReset;
888
+
889
+ account.usage.totalRequests++;
890
+ account.usage.lastUsed = new Date().toISOString();
891
+
892
+ // Log when approaching quota
893
+ if (this._isNearQuota(account)) {
894
+ const pct = account.quota.unified7d != null
895
+ ? (account.quota.unified7d * 100).toFixed(1)
896
+ : account.quota.tokensLimit
897
+ ? ((1 - account.quota.tokensRemaining / account.quota.tokensLimit) * 100).toFixed(1)
898
+ : '?';
899
+ console.log(`[TeamClaude] Account "${account.name}" at ${pct}% usage — will switch on next request`);
900
+ }
901
+ }
902
+
903
+ /**
904
+ * Update cumulative token usage from response body data.
905
+ */
906
+ updateUsage(accountIndex, inputTokens, outputTokens) {
907
+ const account = this._resolve(accountIndex);
908
+ if (!account) return;
909
+ if (inputTokens) account.usage.totalInputTokens += inputTokens;
910
+ if (outputTokens) account.usage.totalOutputTokens += outputTokens;
911
+ }
912
+
913
+ /**
914
+ * Does a 429 from this account indicate genuine *account-level quota
915
+ * exhaustion* (vs a transient / global / IP / request-level 429)?
916
+ *
917
+ * Only exhaustion 429s should throttle the account and trigger a switch to
918
+ * another account. A non-exhaustion 429 must NOT be replayed across the
919
+ * fleet — otherwise a single request whose 429 is request-global (e.g. a
920
+ * malformed request, an org/IP limit, or a momentary upstream blip) would
921
+ * poison every account and make unrelated requests fail too.
922
+ *
923
+ * Call this *after* updateQuota() has folded the 429's rate-limit headers
924
+ * into the account's quota state.
925
+ *
926
+ * Model-scoped windows (quota.modelWeekly, e.g. the Fable 7d_oi limit) are
927
+ * deliberately NOT consulted here. On a real upstream 429 for that model tier
928
+ * the top-level `unified-status` is `rejected` too (the binding claim is
929
+ * reflected there — verified against live traffic), so the exhaustion IS
930
+ * detected; folding 7d_oi in additionally would change nothing on real
931
+ * headers, and reacting to it alone would globally throttle an account that
932
+ * still serves every other model. Per-model routing (skip Fable-exhausted
933
+ * accounts only for Fable requests, without the 5-min global throttle) would
934
+ * need the request's model plumbed into selection — a separate feature.
935
+ */
936
+ isExhausted(accountIndex) {
937
+ const account = this._resolve(accountIndex);
938
+ if (!account) return false;
939
+ // Claude Max: upstream explicitly rejects when over the unified limit.
940
+ if (account.quota.unifiedStatus === 'rejected') return true;
941
+ // Otherwise rely on measured utilization (unified or standard headers).
942
+ return this._isNearQuota(account);
943
+ }
944
+
945
+ /**
946
+ * Mark an account as rate-limited for a given duration.
947
+ */
948
+ markRateLimited(accountIndex, retryAfterSeconds) {
949
+ const account = this._resolve(accountIndex);
950
+ if (!account) return;
951
+ account.status = 'throttled';
952
+ account.rateLimitedUntil = Date.now() + (retryAfterSeconds * 1000);
953
+ console.log(`[TeamClaude] Account "${account.name}" rate limited for ${retryAfterSeconds}s`);
954
+ }
955
+
956
+ /**
957
+ * Ensure an OAuth account's token is fresh, refreshing if needed.
958
+ * Pass force=true to refresh regardless of expiry (e.g. after a 401).
959
+ * Concurrent calls for the same account coalesce into a single refresh.
960
+ */
961
+ async ensureTokenFresh(accountIndex, force = false) {
962
+ const account = this._resolve(accountIndex);
963
+ if (!account || account.type !== 'oauth' || !account.refreshToken) return;
964
+
965
+ if (!force && !isTokenExpiringSoon(account.expiresAt)) return;
966
+
967
+ // Coalesce concurrent refreshes
968
+ if (account._refreshPromise) return account._refreshPromise;
969
+
970
+ account._refreshPromise = (async () => {
971
+ console.log(`[TeamClaude] Refreshing token for account "${account.name}"...`);
972
+ try {
973
+ // Refresh via an injected handler when set (e.g. cloud-centralized refresh
974
+ // so multiple PCs don't each rotate the same account's refresh token); it
975
+ // falls back to a local refresh internally. Default: local OAuth refresh.
976
+ const newTokens = this._refreshHandler
977
+ ? await this._refreshHandler(account)
978
+ : await refreshAccessToken(account.refreshToken);
979
+ account.credential = newTokens.accessToken;
980
+ account.refreshToken = newTokens.refreshToken;
981
+ account.expiresAt = newTokens.expiresAt;
982
+ console.log(`[TeamClaude] Token refreshed for account "${account.name}"`);
983
+ // Only persist if the account is still live at its claimed index. If it was
984
+ // removed during the (awaited) network refresh, its `.index` is stale and
985
+ // would misattribute the write to the survivor that shifted into that slot
986
+ // — and a deleted account's tokens don't need persisting anyway.
987
+ if (this.accounts[account.index] === account) this._onTokenRefresh?.(account.index, newTokens);
988
+ } catch (err) {
989
+ console.error(`[TeamClaude] Token refresh failed for "${account.name}": ${err.message}`);
990
+ // Only mark as error if the access token is actually expired;
991
+ // a failed proactive refresh shouldn't kill a still-valid token
992
+ if (!account.expiresAt || Date.now() >= account.expiresAt) {
993
+ account.status = 'error';
994
+ }
995
+ } finally {
996
+ account._refreshPromise = null;
997
+ }
998
+ })();
999
+
1000
+ return account._refreshPromise;
1001
+ }
1002
+
1003
+ /**
1004
+ * Set a callback to persist refreshed tokens to config.
1005
+ */
1006
+ onTokenRefresh(callback) {
1007
+ this._onTokenRefresh = callback;
1008
+ }
1009
+
1010
+ /**
1011
+ * Inject the token-refresh strategy. `handler(account)` must return
1012
+ * { accessToken, refreshToken, expiresAt } (or throw). Used to route refreshes
1013
+ * through the cloud (single-authority rotation) instead of refreshing locally.
1014
+ * Unset → local OAuth refresh (the default).
1015
+ */
1016
+ setRefreshHandler(handler) {
1017
+ this._refreshHandler = handler;
1018
+ }
1019
+
1020
+ /**
1021
+ * Update a specific account's OAuth tokens (e.g. after intercepting a token refresh).
1022
+ */
1023
+ updateAccountTokens(accountIndex, { accessToken, refreshToken, expiresAt }) {
1024
+ const account = this._resolve(accountIndex);
1025
+ if (!account || account.type !== 'oauth') return;
1026
+
1027
+ account.credential = accessToken;
1028
+ if (refreshToken) account.refreshToken = refreshToken;
1029
+ account.expiresAt = expiresAt;
1030
+ if (account.status === 'error') account.status = 'active';
1031
+ console.log(`[TeamClaude] Updated tokens for account "${account.name}"`);
1032
+ // Same liveness guard as ensureTokenFresh: never emit a stale index for a
1033
+ // removed account (here the path is synchronous, but keep the invariant uniform).
1034
+ if (this.accounts[account.index] === account) this._onTokenRefresh?.(account.index, {
1035
+ accessToken,
1036
+ refreshToken: account.refreshToken,
1037
+ expiresAt: account.expiresAt,
1038
+ });
1039
+ }
1040
+
1041
+ /**
1042
+ * Add a new account at runtime.
1043
+ */
1044
+ addAccount(acctData) {
1045
+ const index = this.accounts.length;
1046
+ this.accounts.push({
1047
+ index,
1048
+ name: acctData.name,
1049
+ type: acctData.type,
1050
+ accountUuid: acctData.accountUuid || null,
1051
+ credential: acctData.accessToken || acctData.apiKey,
1052
+ refreshToken: acctData.refreshToken || null,
1053
+ expiresAt: acctData.expiresAt || null,
1054
+ status: 'active',
1055
+ enabled: acctData.enabled !== false,
1056
+ priority: Number.isFinite(acctData.priority) ? Math.floor(acctData.priority) : null,
1057
+ quota: emptyQuota(),
1058
+ usage: { totalInputTokens: 0, totalOutputTokens: 0, totalRequests: 0, lastUsed: null },
1059
+ rateLimitedUntil: null,
1060
+ inflight: 0,
1061
+ maxConcurrent: coerceMaxConcurrent(acctData.maxConcurrent, this.maxConcurrentDefault),
1062
+ });
1063
+ // The new account has free capacity — hand it to any request waiting in the
1064
+ // overflow queue instead of letting it time out to a 429 while a usable
1065
+ // account sits idle.
1066
+ this._drainWaiters();
1067
+ return index;
1068
+ }
1069
+
1070
+ /**
1071
+ * Remove an account by index.
1072
+ */
1073
+ removeAccount(index) {
1074
+ if (index < 0 || index >= this.accounts.length) return;
1075
+ this.accounts.splice(index, 1);
1076
+ this.accounts.forEach((a, i) => a.index = i);
1077
+ if (this.currentIndex >= this.accounts.length) {
1078
+ this.currentIndex = Math.max(0, this.accounts.length - 1);
1079
+ } else if (this.currentIndex > index) {
1080
+ this.currentIndex--;
1081
+ }
1082
+ }
1083
+
1084
+ /**
1085
+ * Resolve a caller-facing account reference — an account object or a name
1086
+ * string — to the live account object (or null). Used by the public
1087
+ * setEnabled/setPriority.
1088
+ *
1089
+ * A bare numeric index is intentionally NOT accepted here (unlike the internal
1090
+ * `_resolve`): a setter is a mutation, and an index captured before a
1091
+ * removeAccount() re-index would silently disable/reprioritize whatever account
1092
+ * shifted into that slot. Callers pass the account object (TUI / sync) or its
1093
+ * name (CLI) — both survive a re-index.
1094
+ */
1095
+ _resolveRef(ref) {
1096
+ if (typeof ref === 'string') return this.accounts.find(a => a.name === ref) || null;
1097
+ if (ref && typeof ref === 'object') return this.accounts.includes(ref) ? ref : null;
1098
+ return null;
1099
+ }
1100
+
1101
+ /**
1102
+ * Enable or disable an account at runtime. A disabled account is excluded from
1103
+ * rotation (via _isAvailable) but keeps any in-flight requests until they
1104
+ * finish. Re-enabling hands its now-free capacity to any queued waiters.
1105
+ * Returns the affected account, or null if `ref` matched nothing.
1106
+ */
1107
+ setEnabled(ref, enabled) {
1108
+ const account = this._resolveRef(ref);
1109
+ if (!account) return null;
1110
+ account.enabled = enabled !== false;
1111
+ // Re-evaluate the overflow queue either way: re-enabling hands the account's
1112
+ // free slots to waiters; disabling may leave a waiter that could *only* be
1113
+ // served by this account with no hope — _drainWaiters settles those now (so
1114
+ // they release their finite queue slot) instead of stranding them to timeout.
1115
+ this._drainWaiters();
1116
+ this._reprioritize();
1117
+ return account;
1118
+ }
1119
+
1120
+ /**
1121
+ * Set (or clear) an account's explicit selection priority. Lower number =
1122
+ * preferred first; pass null/undefined/NaN to clear it (back to use-or-lose).
1123
+ * Returns the affected account, or null if `ref` matched nothing.
1124
+ */
1125
+ setPriority(ref, priority) {
1126
+ const account = this._resolveRef(ref);
1127
+ if (!account) return null;
1128
+ account.priority = Number.isFinite(priority) ? Math.floor(priority) : null;
1129
+ this._reprioritize();
1130
+ return account;
1131
+ }
1132
+
1133
+ /**
1134
+ * A preference change (enable/disable/priority) should take effect promptly,
1135
+ * not wait out the sticky `reevalIntervalMs` window (and not at all when the
1136
+ * timer is off). Re-pick the active account *directly* here — in either mode —
1137
+ * but ONLY when it actually matters: the current account is no longer usable
1138
+ * (e.g. just disabled), or a strictly higher-priority account is available.
1139
+ *
1140
+ * A no-op change (or one that doesn't dethrone the current account) leaves the
1141
+ * sticky primary untouched, so it can't churn cache locality. We deliberately
1142
+ * do NOT reset `lastEvalAt` to 0 — that would wake the periodic timer re-eval,
1143
+ * whose tie round-robin would switch the primary even when nothing changed.
1144
+ */
1145
+ _reprioritize() {
1146
+ const current = this.accounts[this.currentIndex];
1147
+ const best = this._selectBest();
1148
+ if (!best || best.index === this.currentIndex) return;
1149
+ // Switch only when `best` is *strictly* preferred over the current account by
1150
+ // the full selection order (priority → soonest reset → least used), or the
1151
+ // current account is unusable. Comparing the full order — not priority alone —
1152
+ // means clearing a priority correctly restores use-or-lose routing, while a
1153
+ // true tie (best ranks equal to current) still leaves the sticky primary put
1154
+ // so there's no cache-churn.
1155
+ if (this._isAvailable(current) && !this._strictlyPrefer(best, current)) return;
1156
+ this.currentIndex = best.index;
1157
+ this.lastEvalAt = Date.now(); // just evaluated — don't also trigger a timer re-eval
1158
+ }
1159
+
1160
+ /**
1161
+ * Is account `a` strictly preferred over `b` by the same lexicographic order
1162
+ * `_selectBest` sorts on: explicit priority (lower first), then soonest weekly
1163
+ * reset, then soonest session reset, then lowest utilization. Returns false
1164
+ * when they rank equal (a tie).
1165
+ */
1166
+ _strictlyPrefer(a, b) {
1167
+ const pa = this._priority(a), pb = this._priority(b);
1168
+ if (pa !== pb) return pa < pb;
1169
+ const wa = this._weeklyResetTime(a), wb = this._weeklyResetTime(b);
1170
+ if (wa !== wb) return wa < wb;
1171
+ const ra = this._sessionResetTime(a), rb = this._sessionResetTime(b);
1172
+ if (ra !== rb) return ra < rb;
1173
+ return this._sessionUtilization(a) < this._sessionUtilization(b);
1174
+ }
1175
+
1176
+ /**
1177
+ * Snapshot of per-account quota state for persistence across restarts
1178
+ * (credential-free). Quota lives only in memory otherwise, so a restart used
1179
+ * to blank the whole dashboard (and blind use-or-lose ordering) until traffic
1180
+ * organically re-measured every account.
1181
+ */
1182
+ exportQuotaState() {
1183
+ return this.accounts.map(a => ({
1184
+ accountUuid: a.accountUuid || null,
1185
+ name: a.name,
1186
+ quota: {
1187
+ ...a.quota,
1188
+ modelWeekly: Object.fromEntries(
1189
+ Object.entries(a.quota.modelWeekly).map(([k, w]) => [k, { ...w }])),
1190
+ },
1191
+ rateLimitedUntil: a.rateLimitedUntil,
1192
+ usage: { ...a.usage },
1193
+ }));
1194
+ }
1195
+
1196
+ /**
1197
+ * Restore a quota snapshot from a previous run. A snapshot entry WITH an
1198
+ * accountUuid is matched by uuid ONLY — a same-name account with a different
1199
+ * uuid is a *replaced* account (a different underlying identity), and
1200
+ * restoring the old quota/throttle onto it would falsely mark a fresh
1201
+ * account near-quota or throttled. Name matching is the fallback solely for
1202
+ * entries without a uuid (API-key accounts, whose identity key is the name).
1203
+ * Unknown entries are skipped (→ unmeasured, exactly the pre-restore state).
1204
+ * Values may be slightly stale, but the proxy takes no traffic while it's
1205
+ * down, and expired windows are lazily swept by _isNearQuota on first use —
1206
+ * so a restore is strictly better than starting blind. A still-future
1207
+ * rateLimitedUntil re-throttles the account; error/exhausted statuses are
1208
+ * deliberately NOT restored (a bad token may have been fixed since).
1209
+ */
1210
+ importQuotaState(saved) {
1211
+ for (const s of Array.isArray(saved) ? saved : []) {
1212
+ if (!s || typeof s !== 'object') continue;
1213
+ const a = s.accountUuid
1214
+ ? this.accounts.find(x => x.accountUuid === s.accountUuid)
1215
+ : this.accounts.find(x => x.name === s.name);
1216
+ if (!a) continue;
1217
+ if (s.quota && typeof s.quota === 'object') {
1218
+ // Merge over emptyQuota so a cache written by an older version (missing
1219
+ // newer fields like modelWeekly) still yields a complete quota object.
1220
+ a.quota = {
1221
+ ...emptyQuota(),
1222
+ ...s.quota,
1223
+ // unifiedStatus is a PER-RESPONSE signal — isExhausted() treats a
1224
+ // 'rejected' here as "this 429 is account exhaustion". Restoring a
1225
+ // stale one would misclassify a later transient/headerless 429 as
1226
+ // exhaustion and wrongly throttle the account. Only a live response
1227
+ // (updateQuota) may set it.
1228
+ unifiedStatus: null,
1229
+ modelWeekly: Object.fromEntries(
1230
+ Object.entries(s.quota.modelWeekly && typeof s.quota.modelWeekly === 'object' ? s.quota.modelWeekly : {})
1231
+ .map(([k, w]) => [k, { ...w }])),
1232
+ };
1233
+ }
1234
+ if (s.usage && typeof s.usage === 'object') a.usage = { ...a.usage, ...s.usage };
1235
+ if (Number.isFinite(s.rateLimitedUntil) && s.rateLimitedUntil > Date.now()) {
1236
+ a.rateLimitedUntil = s.rateLimitedUntil;
1237
+ a.status = 'throttled';
1238
+ }
1239
+ }
1240
+ }
1241
+
1242
+ /**
1243
+ * Return a status summary of all accounts (safe to expose, no credentials).
1244
+ */
1245
+ getStatus() {
1246
+ return {
1247
+ currentAccount: this.accounts[this.currentIndex]?.name,
1248
+ switchThreshold: this.switchThreshold,
1249
+ accounts: this.accounts.map(a => ({
1250
+ name: a.name,
1251
+ type: a.type,
1252
+ status: a.status,
1253
+ enabled: a.enabled !== false,
1254
+ priority: a.priority ?? null,
1255
+ // Deep-copy the nested modelWeekly map — the shallow quota spread would
1256
+ // otherwise hand callers a live reference into account state.
1257
+ quota: {
1258
+ ...a.quota,
1259
+ modelWeekly: Object.fromEntries(
1260
+ Object.entries(a.quota.modelWeekly).map(([k, w]) => [k, { ...w }])),
1261
+ },
1262
+ usage: { ...a.usage },
1263
+ inflight: a.inflight,
1264
+ maxConcurrent: a.maxConcurrent,
1265
+ rateLimitedUntil: a.rateLimitedUntil
1266
+ ? new Date(a.rateLimitedUntil).toISOString()
1267
+ : null,
1268
+ })),
1269
+ };
1270
+ }
1271
+ }