switchroom 0.14.84 → 0.14.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-broker/index.js +9 -9
- package/dist/cli/autoaccept-poll.js +1 -1
- package/dist/cli/switchroom.js +2 -2
- package/package.json +1 -1
- package/telegram-plugin/credits-watch.ts +50 -10
- package/telegram-plugin/dist/gateway/gateway.js +34 -13
- package/telegram-plugin/gateway/exhaust-until.ts +45 -0
- package/telegram-plugin/gateway/gateway.ts +22 -10
- package/telegram-plugin/tests/credits-watch.test.ts +65 -1
- package/telegram-plugin/tests/exhaust-until.test.ts +86 -0
|
@@ -13941,24 +13941,24 @@ class AuthBroker {
|
|
|
13941
13941
|
}
|
|
13942
13942
|
servingAccount(identity2) {
|
|
13943
13943
|
const account = this.callerAccount(identity2);
|
|
13944
|
-
if (identity2.kind
|
|
13944
|
+
if (identity2.kind === "operator")
|
|
13945
13945
|
return account;
|
|
13946
|
-
return this.
|
|
13946
|
+
return this.accountWithFailover(account);
|
|
13947
13947
|
}
|
|
13948
13948
|
isAccountExhausted(account) {
|
|
13949
13949
|
const q = this.quota[account];
|
|
13950
13950
|
return q !== undefined && q.exhausted_until > this.now();
|
|
13951
13951
|
}
|
|
13952
|
-
|
|
13953
|
-
if (!
|
|
13954
|
-
return
|
|
13952
|
+
accountWithFailover(account) {
|
|
13953
|
+
if (!account || !this.isAccountExhausted(account))
|
|
13954
|
+
return account ?? null;
|
|
13955
13955
|
for (const cand of this.config.auth?.fallback_order ?? []) {
|
|
13956
|
-
if (cand ===
|
|
13956
|
+
if (cand === account || this.isAccountExhausted(cand))
|
|
13957
13957
|
continue;
|
|
13958
13958
|
if (readAccountCredentials(cand, this.home))
|
|
13959
13959
|
return cand;
|
|
13960
13960
|
}
|
|
13961
|
-
return
|
|
13961
|
+
return account;
|
|
13962
13962
|
}
|
|
13963
13963
|
async opGetCredentials(socket, id, identity2) {
|
|
13964
13964
|
const account = this.servingAccount(identity2);
|
|
@@ -14600,7 +14600,7 @@ class AuthBroker {
|
|
|
14600
14600
|
const auth = this.config.auth ?? {};
|
|
14601
14601
|
const fanned = [];
|
|
14602
14602
|
for (const [name, agent] of Object.entries(this.config.agents ?? {})) {
|
|
14603
|
-
const effective = agent.auth?.override ?? auth.active;
|
|
14603
|
+
const effective = this.accountWithFailover(agent.auth?.override ?? auth.active);
|
|
14604
14604
|
if (effective === label) {
|
|
14605
14605
|
if (this.fanoutForAgent(name))
|
|
14606
14606
|
fanned.push(name);
|
|
@@ -14645,7 +14645,7 @@ class AuthBroker {
|
|
|
14645
14645
|
const agent = (this.config.agents ?? {})[name];
|
|
14646
14646
|
if (!agent)
|
|
14647
14647
|
return false;
|
|
14648
|
-
const effective = agent.auth?.override ?? auth.active;
|
|
14648
|
+
const effective = this.accountWithFailover(agent.auth?.override ?? auth.active);
|
|
14649
14649
|
if (!effective)
|
|
14650
14650
|
return false;
|
|
14651
14651
|
return this.mirrorAccountToAgent(effective, name);
|
|
@@ -176,7 +176,7 @@ function sendKeys2(agentName, keys) {
|
|
|
176
176
|
|
|
177
177
|
// src/agents/wedge-watchdog.ts
|
|
178
178
|
var WEDGE_FOOTER_SIGNATURE = /(?=[\s\S]*[Ee]sc(?:ape)?[^\n]*cancel)(?=[\s\S]*(?:to select|to navigate|\u2191\/\u2193))/;
|
|
179
|
-
var RATE_LIMIT_MENU_SIGNATURE = /(?=[\s\S]
|
|
179
|
+
var RATE_LIMIT_MENU_SIGNATURE = /(?=[\s\S]*Stop and wait for)(?=[\s\S]*(?:usage credits|Upgrade your plan|\/rate-limit-options))/;
|
|
180
180
|
var MONTHS = {
|
|
181
181
|
jan: 0,
|
|
182
182
|
feb: 1,
|
package/dist/cli/switchroom.js
CHANGED
|
@@ -49815,8 +49815,8 @@ var {
|
|
|
49815
49815
|
} = import__.default;
|
|
49816
49816
|
|
|
49817
49817
|
// src/build-info.ts
|
|
49818
|
-
var VERSION = "0.14.
|
|
49819
|
-
var COMMIT_SHA = "
|
|
49818
|
+
var VERSION = "0.14.86";
|
|
49819
|
+
var COMMIT_SHA = "8c518120";
|
|
49820
49820
|
|
|
49821
49821
|
// src/cli/agent.ts
|
|
49822
49822
|
init_source();
|
package/package.json
CHANGED
|
@@ -27,20 +27,53 @@ import { join } from "path";
|
|
|
27
27
|
const STATE_FILE = "credits-watch.json";
|
|
28
28
|
|
|
29
29
|
/**
|
|
30
|
-
*
|
|
31
|
-
* that warrant a user-facing notification. Other values (null,
|
|
32
|
-
* undefined, transient unknowns) are treated as "no notification
|
|
33
|
-
* needed".
|
|
30
|
+
* Which `cachedExtraUsageDisabledReason` values warrant a user-facing alarm.
|
|
34
31
|
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
32
|
+
* IMPORTANT — empty by default (subscription-only model). The flag name says it
|
|
33
|
+
* all: `cachedExtraUsageDisabledReason` is *always* "why **extra usage** (the
|
|
34
|
+
* optional pay-as-you-go layer beyond the Pro/Max plan) is disabled." switchroom
|
|
35
|
+
* is subscription-only by design (compliance pillar 3 — "the plan is the
|
|
36
|
+
* ceiling, no API/pay-as-you-go"), so extra usage being OFF is the **expected,
|
|
37
|
+
* desired** state. Every value (`out_of_credits`, `extra_usage_disabled`,
|
|
38
|
+
* `credits_exhausted`, even `org_level_disabled` = "org admin disabled extra
|
|
39
|
+
* usage") describes that benign state — none indicates a real switchroom
|
|
40
|
+
* failure. Firing a "cron + replies will fail, buy credits" card on it is a
|
|
41
|
+
* false alarm AND advises something that contradicts the subscription-honest
|
|
42
|
+
* model.
|
|
43
|
+
*
|
|
44
|
+
* The genuine failure modes are covered elsewhere:
|
|
45
|
+
* - an account's subscription window exhausts → stderr → fleet auto-fallback
|
|
46
|
+
* (model-unavailable.ts → fireFleetAutoFallback) rolls to a healthy account;
|
|
47
|
+
* - the WHOLE fleet exhausts → the quota-watch all-exhausted operator alert.
|
|
48
|
+
*
|
|
49
|
+
* So credits-watch defaults to silent. An operator who actually runs on
|
|
50
|
+
* pay-as-you-go and wants the alarm can opt specific reasons back in via
|
|
51
|
+
* `SWITCHROOM_CREDITS_WATCH_FATAL_REASONS` (comma-separated). See
|
|
52
|
+
* `resolveCreditWatchFatalReasons`.
|
|
37
53
|
*/
|
|
38
|
-
const
|
|
54
|
+
export const DEFAULT_CREDIT_FATAL_REASONS = new Set<string>();
|
|
55
|
+
|
|
56
|
+
/** All reason strings recognized for opt-in via the env var. */
|
|
57
|
+
const KNOWN_CREDIT_REASONS = [
|
|
39
58
|
"out_of_credits",
|
|
40
59
|
"org_level_disabled",
|
|
41
60
|
"credits_exhausted",
|
|
42
61
|
"extra_usage_disabled",
|
|
43
|
-
]
|
|
62
|
+
] as const;
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Resolve the active fatal-reason set. Empty by default (subscription-only);
|
|
66
|
+
* if `SWITCHROOM_CREDITS_WATCH_FATAL_REASONS` is set, parse it as a
|
|
67
|
+
* comma-separated list (tokens kept verbatim — an unknown token is harmless,
|
|
68
|
+
* it simply never matches a real reason value). `*` opts in all known reasons.
|
|
69
|
+
*/
|
|
70
|
+
export function resolveCreditWatchFatalReasons(env: NodeJS.ProcessEnv): Set<string> {
|
|
71
|
+
const raw = env.SWITCHROOM_CREDITS_WATCH_FATAL_REASONS;
|
|
72
|
+
if (!raw || raw.trim().length === 0) return new Set(DEFAULT_CREDIT_FATAL_REASONS);
|
|
73
|
+
if (raw.trim() === "*") return new Set(KNOWN_CREDIT_REASONS);
|
|
74
|
+
const wanted = new Set(raw.split(",").map((s) => s.trim()).filter((s) => s.length > 0));
|
|
75
|
+
return wanted;
|
|
76
|
+
}
|
|
44
77
|
|
|
45
78
|
export interface CreditState {
|
|
46
79
|
/** Last reason we notified the user about. null when healthy / never notified. */
|
|
@@ -103,15 +136,22 @@ export function evaluateCreditState(args: {
|
|
|
103
136
|
currentReason: string | null;
|
|
104
137
|
prev: CreditState;
|
|
105
138
|
now: number;
|
|
139
|
+
/**
|
|
140
|
+
* Which reasons are treated as fatal (worth alarming). Defaults to
|
|
141
|
+
* `DEFAULT_CREDIT_FATAL_REASONS` (empty — subscription-only: extra-usage-off
|
|
142
|
+
* is never a real failure). The gateway passes the env-resolved set.
|
|
143
|
+
*/
|
|
144
|
+
fatalReasons?: Set<string>;
|
|
106
145
|
}): CreditDecision {
|
|
107
146
|
const { agentName, currentReason, prev, now } = args;
|
|
147
|
+
const fatalReasons = args.fatalReasons ?? DEFAULT_CREDIT_FATAL_REASONS;
|
|
108
148
|
|
|
109
149
|
// Non-fatal current state (null, or some unknown reason) — no
|
|
110
150
|
// notification regardless of prev (we already notified on entry to
|
|
111
151
|
// fatal; recovery from a known-fatal state below is the only path
|
|
112
152
|
// that fires when current is null).
|
|
113
|
-
const currentIsFatal = currentReason != null &&
|
|
114
|
-
const prevIsFatal = prev.lastNotifiedReason != null &&
|
|
153
|
+
const currentIsFatal = currentReason != null && fatalReasons.has(currentReason);
|
|
154
|
+
const prevIsFatal = prev.lastNotifiedReason != null && fatalReasons.has(prev.lastNotifiedReason);
|
|
115
155
|
|
|
116
156
|
// Recovery path: last-notified was fatal, current is null/non-fatal.
|
|
117
157
|
if (!currentIsFatal && prevIsFatal) {
|
|
@@ -41381,6 +41381,15 @@ function createFleetFallbackGate(opts) {
|
|
|
41381
41381
|
};
|
|
41382
41382
|
}
|
|
41383
41383
|
|
|
41384
|
+
// gateway/exhaust-until.ts
|
|
41385
|
+
var EXHAUST_WEEKLY_FLOOR_MS = 7 * 24 * 60 * 60 * 1000;
|
|
41386
|
+
function resolveExhaustUntil(resetAtMs, now = Date.now()) {
|
|
41387
|
+
if (typeof resetAtMs === "number" && Number.isFinite(resetAtMs) && resetAtMs > now) {
|
|
41388
|
+
return resetAtMs;
|
|
41389
|
+
}
|
|
41390
|
+
return now + EXHAUST_WEEKLY_FLOOR_MS;
|
|
41391
|
+
}
|
|
41392
|
+
|
|
41384
41393
|
// gateway/auth-add-flow.ts
|
|
41385
41394
|
import { spawn } from "node:child_process";
|
|
41386
41395
|
import { existsSync as existsSync13, mkdirSync as mkdirSync10, readFileSync as readFileSync9, rmSync as rmSync3 } from "node:fs";
|
|
@@ -52507,12 +52516,22 @@ function extractFlowItems(line) {
|
|
|
52507
52516
|
import { readFileSync as readFileSync32, writeFileSync as writeFileSync21, existsSync as existsSync33, mkdirSync as mkdirSync21 } from "fs";
|
|
52508
52517
|
import { join as join30 } from "path";
|
|
52509
52518
|
var STATE_FILE = "credits-watch.json";
|
|
52510
|
-
var
|
|
52519
|
+
var DEFAULT_CREDIT_FATAL_REASONS = new Set;
|
|
52520
|
+
var KNOWN_CREDIT_REASONS = [
|
|
52511
52521
|
"out_of_credits",
|
|
52512
52522
|
"org_level_disabled",
|
|
52513
52523
|
"credits_exhausted",
|
|
52514
52524
|
"extra_usage_disabled"
|
|
52515
|
-
]
|
|
52525
|
+
];
|
|
52526
|
+
function resolveCreditWatchFatalReasons(env) {
|
|
52527
|
+
const raw = env.SWITCHROOM_CREDITS_WATCH_FATAL_REASONS;
|
|
52528
|
+
if (!raw || raw.trim().length === 0)
|
|
52529
|
+
return new Set(DEFAULT_CREDIT_FATAL_REASONS);
|
|
52530
|
+
if (raw.trim() === "*")
|
|
52531
|
+
return new Set(KNOWN_CREDIT_REASONS);
|
|
52532
|
+
const wanted = new Set(raw.split(",").map((s) => s.trim()).filter((s) => s.length > 0));
|
|
52533
|
+
return wanted;
|
|
52534
|
+
}
|
|
52516
52535
|
function emptyCreditState() {
|
|
52517
52536
|
return { lastNotifiedReason: null, lastNotifiedAt: 0 };
|
|
52518
52537
|
}
|
|
@@ -52541,8 +52560,9 @@ function readClaudeJsonOverage(claudeConfigDir) {
|
|
|
52541
52560
|
}
|
|
52542
52561
|
function evaluateCreditState(args) {
|
|
52543
52562
|
const { agentName: agentName3, currentReason, prev, now } = args;
|
|
52544
|
-
const
|
|
52545
|
-
const
|
|
52563
|
+
const fatalReasons = args.fatalReasons ?? DEFAULT_CREDIT_FATAL_REASONS;
|
|
52564
|
+
const currentIsFatal = currentReason != null && fatalReasons.has(currentReason);
|
|
52565
|
+
const prevIsFatal = prev.lastNotifiedReason != null && fatalReasons.has(prev.lastNotifiedReason);
|
|
52546
52566
|
if (!currentIsFatal && prevIsFatal) {
|
|
52547
52567
|
return {
|
|
52548
52568
|
kind: "notify",
|
|
@@ -52869,11 +52889,11 @@ function sweepStaleTurnActiveMarker(stateDir, opts) {
|
|
|
52869
52889
|
}
|
|
52870
52890
|
|
|
52871
52891
|
// ../src/build-info.ts
|
|
52872
|
-
var VERSION = "0.14.
|
|
52873
|
-
var COMMIT_SHA = "
|
|
52874
|
-
var COMMIT_DATE = "2026-06-
|
|
52875
|
-
var LATEST_PR =
|
|
52876
|
-
var COMMITS_AHEAD_OF_TAG =
|
|
52892
|
+
var VERSION = "0.14.86";
|
|
52893
|
+
var COMMIT_SHA = "8c518120";
|
|
52894
|
+
var COMMIT_DATE = "2026-06-07T05:43:21Z";
|
|
52895
|
+
var LATEST_PR = 2223;
|
|
52896
|
+
var COMMITS_AHEAD_OF_TAG = 0;
|
|
52877
52897
|
|
|
52878
52898
|
// gateway/boot-version.ts
|
|
52879
52899
|
function formatRelativeAgo(iso) {
|
|
@@ -55234,7 +55254,8 @@ function emitGatewayOperatorEvent(event) {
|
|
|
55234
55254
|
});
|
|
55235
55255
|
renderedKeyboard = undefined;
|
|
55236
55256
|
if (willActuallyFire) {
|
|
55237
|
-
|
|
55257
|
+
const untilMs = resolveExhaustUntil(modelUnavailable.resetAt?.getTime());
|
|
55258
|
+
fireFleetAutoFallback(agent, untilMs);
|
|
55238
55259
|
}
|
|
55239
55260
|
} else {
|
|
55240
55261
|
try {
|
|
@@ -56171,8 +56192,7 @@ var ipcServer = createIpcServer({
|
|
|
56171
56192
|
}
|
|
56172
56193
|
},
|
|
56173
56194
|
onQuotaWallDetected(_client, msg) {
|
|
56174
|
-
const
|
|
56175
|
-
const untilMs = typeof msg.resetAt === "number" && Number.isFinite(msg.resetAt) && msg.resetAt > Date.now() ? msg.resetAt : Date.now() + WEEKLY_MS;
|
|
56195
|
+
const untilMs = resolveExhaustUntil(msg.resetAt);
|
|
56176
56196
|
process.stderr.write(`telegram gateway: quota_wall_detected agent=${msg.agentName} until=${new Date(untilMs).toISOString()}` + (msg.resetAt == null ? " (reset unparsed \u2192 +7d default)" : "") + ` \u2014 triggering fleet auto-fallback
|
|
56177
56197
|
`);
|
|
56178
56198
|
fireFleetAutoFallback(msg.agentName, untilMs);
|
|
@@ -61040,7 +61060,8 @@ async function runCreditWatch() {
|
|
|
61040
61060
|
agentName: agentName3,
|
|
61041
61061
|
currentReason: reason,
|
|
61042
61062
|
prev,
|
|
61043
|
-
now: Date.now()
|
|
61063
|
+
now: Date.now(),
|
|
61064
|
+
fatalReasons: resolveCreditWatchFatalReasons(process.env)
|
|
61044
61065
|
});
|
|
61045
61066
|
if (decision.kind === "skip") {
|
|
61046
61067
|
return;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* exhaust-until.ts — the single decision for the `until` (epoch-ms) handed to
|
|
3
|
+
* the broker's mark-exhausted, shared by BOTH quota-wall paths in the gateway:
|
|
4
|
+
* - the /rate-limit-options menu signal (onQuotaWallDetected), and
|
|
5
|
+
* - the inference-path 429 (the model-unavailable operator-event path).
|
|
6
|
+
*
|
|
7
|
+
* Factored out of gateway.ts so the decision is unit-testable without importing
|
|
8
|
+
* the gateway module (heavy import-time side effects). Pure: no IPC, no clock
|
|
9
|
+
* except the injectable `now`.
|
|
10
|
+
*
|
|
11
|
+
* The ONE invariant both paths depend on: NEVER return undefined. A weekly wall
|
|
12
|
+
* reaches the gateway as a reset hint the parser can't always read:
|
|
13
|
+
* - Menu path: the sidecar's parseWeeklyReset handles "resets Jun 9, 5am (tz)"
|
|
14
|
+
* → a finite future epoch flows straight through.
|
|
15
|
+
* - 429 path: model-unavailable.ts's parseResetTime handles the ROLLING
|
|
16
|
+
* wordings ("resets in 2h", "retry after 60s") → a finite ~hours epoch. But
|
|
17
|
+
* the WEEKLY wording reaches it as "resets <Mon> <D>, <H>am", which
|
|
18
|
+
* parseResetTime CANNOT parse (`new Date('Jun 9, 5am 2026')` → Invalid Date)
|
|
19
|
+
* → resetAtMs is undefined here EXACTLY when the wall is weekly.
|
|
20
|
+
*
|
|
21
|
+
* undefined (or a past / NaN reset) is the dangerous case: markExhausted's ~5h
|
|
22
|
+
* default would un-exhaust a weekly-walled account after 5h and let the broker
|
|
23
|
+
* re-mirror it onto the fleet (the #2218 rollback). So undefined → the +7d
|
|
24
|
+
* floor. A too-long `until` only delays the broker's re-probe of the account;
|
|
25
|
+
* it never serves an exhausted account, so erring long is the compliance-safe
|
|
26
|
+
* direction (vision.md pillar 3).
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
/** Weekly-quota window — the safety floor for an exhaustion `until`. */
|
|
30
|
+
export const EXHAUST_WEEKLY_FLOOR_MS = 7 * 24 * 60 * 60 * 1000
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Resolve the exhaustion `until` (epoch-ms) from a best-effort reset timestamp.
|
|
34
|
+
* Returns `resetAtMs` when it is a finite epoch in the future; otherwise the
|
|
35
|
+
* +7d floor anchored at `now`. NEVER returns undefined.
|
|
36
|
+
*/
|
|
37
|
+
export function resolveExhaustUntil(
|
|
38
|
+
resetAtMs: number | undefined,
|
|
39
|
+
now: number = Date.now(),
|
|
40
|
+
): number {
|
|
41
|
+
if (typeof resetAtMs === 'number' && Number.isFinite(resetAtMs) && resetAtMs > now) {
|
|
42
|
+
return resetAtMs
|
|
43
|
+
}
|
|
44
|
+
return now + EXHAUST_WEEKLY_FLOOR_MS
|
|
45
|
+
}
|
|
@@ -125,6 +125,7 @@ import {
|
|
|
125
125
|
} from './microsoft-connect-flow.js'
|
|
126
126
|
import { resolveAuthBrokerSocketPath } from '../../src/auth/broker/client.js'
|
|
127
127
|
import { createFleetFallbackGate } from '../fleet-fallback-gate.js'
|
|
128
|
+
import { resolveExhaustUntil } from './exhaust-until.js'
|
|
128
129
|
import {
|
|
129
130
|
pendingAuthAddFlows,
|
|
130
131
|
startAccountAuthSession,
|
|
@@ -408,6 +409,7 @@ import { synthesizeAllowRuleDiff, extractAddedAllowRule } from '../permission-di
|
|
|
408
409
|
import {
|
|
409
410
|
readClaudeJsonOverage,
|
|
410
411
|
evaluateCreditState,
|
|
412
|
+
resolveCreditWatchFatalReasons,
|
|
411
413
|
loadCreditState,
|
|
412
414
|
saveCreditState,
|
|
413
415
|
} from '../credits-watch.js'
|
|
@@ -4399,8 +4401,17 @@ function emitGatewayOperatorEvent(event: OperatorEvent): void {
|
|
|
4399
4401
|
// separately with the causal-shape headline ("5-hour limit on
|
|
4400
4402
|
// ken" instead of generic "quota exhausted") — see
|
|
4401
4403
|
// auth-snapshot-format.ts → renderFallbackAnnouncement.
|
|
4404
|
+
//
|
|
4405
|
+
// Thread the parsed reset as markExhausted's `until` via the shared
|
|
4406
|
+
// resolveExhaustUntil floor. parseResetTime gives a finite epoch for
|
|
4407
|
+
// ROLLING wordings ("resets in 2h", "retry after 60s"); a WEEKLY wall
|
|
4408
|
+
// ("resets Jun 9, 5am") is UNPARSEABLE here → undefined → +7d floor.
|
|
4409
|
+
// Pre-fix this called fireFleetAutoFallback(agent) with no until, so a
|
|
4410
|
+
// weekly wall that surfaced as a 429 got markExhausted's ~5h default and
|
|
4411
|
+
// the broker re-mirrored the still-walled account onto the fleet after 5h.
|
|
4402
4412
|
if (willActuallyFire) {
|
|
4403
|
-
|
|
4413
|
+
const untilMs = resolveExhaustUntil(modelUnavailable.resetAt?.getTime())
|
|
4414
|
+
void fireFleetAutoFallback(agent, untilMs)
|
|
4404
4415
|
}
|
|
4405
4416
|
} else {
|
|
4406
4417
|
try {
|
|
@@ -6270,11 +6281,7 @@ const ipcServer: IpcServer = createIpcServer({
|
|
|
6270
6281
|
// existing chain handles the rest (roll to a fallback subscription account,
|
|
6271
6282
|
// or the all-exhausted operator alert when none has quota). Fire-and-forget.
|
|
6272
6283
|
onQuotaWallDetected(_client: IpcClient, msg: QuotaWallDetectedMessage) {
|
|
6273
|
-
const
|
|
6274
|
-
const untilMs =
|
|
6275
|
-
typeof msg.resetAt === 'number' && Number.isFinite(msg.resetAt) && msg.resetAt > Date.now()
|
|
6276
|
-
? msg.resetAt
|
|
6277
|
-
: Date.now() + WEEKLY_MS
|
|
6284
|
+
const untilMs = resolveExhaustUntil(msg.resetAt)
|
|
6278
6285
|
process.stderr.write(
|
|
6279
6286
|
`telegram gateway: quota_wall_detected agent=${msg.agentName} ` +
|
|
6280
6287
|
`until=${new Date(untilMs).toISOString()}` +
|
|
@@ -14680,10 +14687,12 @@ async function doFireFleetAutoFallback(triggerAgent: string, untilMs?: number):
|
|
|
14680
14687
|
// operator is explicitly choosing, and is admin); only this automatic
|
|
14681
14688
|
// path moves to the non-admin verb.
|
|
14682
14689
|
failover: async () => {
|
|
14683
|
-
//
|
|
14684
|
-
// rate-limit-MENU path
|
|
14685
|
-
// reset,
|
|
14686
|
-
// the
|
|
14690
|
+
// BOTH paths now pass a resolved `untilMs` via resolveExhaustUntil
|
|
14691
|
+
// (never undefined): the rate-limit-MENU path threads the parsed weekly
|
|
14692
|
+
// reset, and the 429 inference path threads parseResetTime's value or
|
|
14693
|
+
// the +7d floor when it's a weekly wall it can't parse. So a
|
|
14694
|
+
// weekly-capped account is never re-probed (and re-wedged) within the
|
|
14695
|
+
// broker's ~5h default.
|
|
14687
14696
|
const r = await client.markExhausted(untilMs)
|
|
14688
14697
|
return { rolledTo: r.rolledTo ?? null, rolled: r.rolled }
|
|
14689
14698
|
},
|
|
@@ -14742,6 +14751,9 @@ async function runCreditWatch(): Promise<void> {
|
|
|
14742
14751
|
currentReason: reason,
|
|
14743
14752
|
prev,
|
|
14744
14753
|
now: Date.now(),
|
|
14754
|
+
// Subscription-only: empty by default → extra-usage-off never alarms.
|
|
14755
|
+
// Opt back in via SWITCHROOM_CREDITS_WATCH_FATAL_REASONS. See credits-watch.ts.
|
|
14756
|
+
fatalReasons: resolveCreditWatchFatalReasons(process.env),
|
|
14745
14757
|
})
|
|
14746
14758
|
if (decision.kind === 'skip') {
|
|
14747
14759
|
return
|
|
@@ -13,6 +13,7 @@ import { join } from "path";
|
|
|
13
13
|
import {
|
|
14
14
|
readClaudeJsonOverage,
|
|
15
15
|
evaluateCreditState,
|
|
16
|
+
resolveCreditWatchFatalReasons,
|
|
16
17
|
loadCreditState,
|
|
17
18
|
saveCreditState,
|
|
18
19
|
emptyCreditState,
|
|
@@ -81,10 +82,14 @@ describe("readClaudeJsonOverage", () => {
|
|
|
81
82
|
});
|
|
82
83
|
});
|
|
83
84
|
|
|
84
|
-
describe("evaluateCreditState — transition decisions", () => {
|
|
85
|
+
describe("evaluateCreditState — transition decisions (machinery, explicit fatal set)", () => {
|
|
85
86
|
const NOW = 1_780_000_000_000;
|
|
86
87
|
const HEALTHY = emptyCreditState();
|
|
87
88
|
const FATAL_OUT = { lastNotifiedReason: "out_of_credits", lastNotifiedAt: NOW - 1000 };
|
|
89
|
+
// The transition machinery is policy-agnostic — pass an explicit fatal set so
|
|
90
|
+
// these tests pin entered/changed/exited behaviour independent of the (now
|
|
91
|
+
// empty) subscription-only default.
|
|
92
|
+
const FATAL = new Set(["out_of_credits", "org_level_disabled", "credits_exhausted", "extra_usage_disabled"]);
|
|
88
93
|
|
|
89
94
|
it("entry: healthy → fatal triggers a notify", () => {
|
|
90
95
|
const d = evaluateCreditState({
|
|
@@ -92,6 +97,7 @@ describe("evaluateCreditState — transition decisions", () => {
|
|
|
92
97
|
currentReason: "out_of_credits",
|
|
93
98
|
prev: HEALTHY,
|
|
94
99
|
now: NOW,
|
|
100
|
+
fatalReasons: FATAL,
|
|
95
101
|
});
|
|
96
102
|
expect(d.kind).toBe("notify");
|
|
97
103
|
if (d.kind !== "notify") return;
|
|
@@ -108,6 +114,7 @@ describe("evaluateCreditState — transition decisions", () => {
|
|
|
108
114
|
currentReason: "out_of_credits",
|
|
109
115
|
prev: FATAL_OUT,
|
|
110
116
|
now: NOW,
|
|
117
|
+
fatalReasons: FATAL,
|
|
111
118
|
});
|
|
112
119
|
expect(d.kind).toBe("skip");
|
|
113
120
|
if (d.kind !== "skip") return;
|
|
@@ -120,6 +127,7 @@ describe("evaluateCreditState — transition decisions", () => {
|
|
|
120
127
|
currentReason: "org_level_disabled",
|
|
121
128
|
prev: FATAL_OUT,
|
|
122
129
|
now: NOW,
|
|
130
|
+
fatalReasons: FATAL,
|
|
123
131
|
});
|
|
124
132
|
expect(d.kind).toBe("notify");
|
|
125
133
|
if (d.kind !== "notify") return;
|
|
@@ -134,6 +142,7 @@ describe("evaluateCreditState — transition decisions", () => {
|
|
|
134
142
|
currentReason: null,
|
|
135
143
|
prev: FATAL_OUT,
|
|
136
144
|
now: NOW,
|
|
145
|
+
fatalReasons: FATAL,
|
|
137
146
|
});
|
|
138
147
|
expect(d.kind).toBe("notify");
|
|
139
148
|
if (d.kind !== "notify") return;
|
|
@@ -148,6 +157,7 @@ describe("evaluateCreditState — transition decisions", () => {
|
|
|
148
157
|
currentReason: "some_unknown_transient_reason",
|
|
149
158
|
prev: HEALTHY,
|
|
150
159
|
now: NOW,
|
|
160
|
+
fatalReasons: FATAL,
|
|
151
161
|
});
|
|
152
162
|
expect(d.kind).toBe("skip");
|
|
153
163
|
if (d.kind !== "skip") return;
|
|
@@ -160,6 +170,7 @@ describe("evaluateCreditState — transition decisions", () => {
|
|
|
160
170
|
currentReason: null,
|
|
161
171
|
prev: HEALTHY,
|
|
162
172
|
now: NOW,
|
|
173
|
+
fatalReasons: FATAL,
|
|
163
174
|
});
|
|
164
175
|
expect(d.kind).toBe("skip");
|
|
165
176
|
});
|
|
@@ -170,6 +181,7 @@ describe("evaluateCreditState — transition decisions", () => {
|
|
|
170
181
|
currentReason: "out_of_credits",
|
|
171
182
|
prev: HEALTHY,
|
|
172
183
|
now: NOW,
|
|
184
|
+
fatalReasons: FATAL,
|
|
173
185
|
});
|
|
174
186
|
expect(d.kind).toBe("notify");
|
|
175
187
|
if (d.kind !== "notify") return;
|
|
@@ -178,6 +190,58 @@ describe("evaluateCreditState — transition decisions", () => {
|
|
|
178
190
|
});
|
|
179
191
|
});
|
|
180
192
|
|
|
193
|
+
describe("evaluateCreditState — subscription-only default (the fix)", () => {
|
|
194
|
+
const NOW = 1_780_000_000_000;
|
|
195
|
+
const HEALTHY = emptyCreditState();
|
|
196
|
+
|
|
197
|
+
// With the default (empty) fatal set, NONE of the extra-usage reasons alarm —
|
|
198
|
+
// because for subscription-only switchroom, extra-usage-off is the expected
|
|
199
|
+
// state and real exhaustion is handled by failover. This is the bug fix.
|
|
200
|
+
for (const reason of ["out_of_credits", "extra_usage_disabled", "credits_exhausted", "org_level_disabled"]) {
|
|
201
|
+
it(`default: '${reason}' does NOT alarm (no false 'out of credits' card)`, () => {
|
|
202
|
+
const d = evaluateCreditState({
|
|
203
|
+
agentName: "clerk",
|
|
204
|
+
currentReason: reason,
|
|
205
|
+
prev: HEALTHY,
|
|
206
|
+
now: NOW,
|
|
207
|
+
// fatalReasons omitted → DEFAULT_CREDIT_FATAL_REASONS (empty)
|
|
208
|
+
});
|
|
209
|
+
expect(d.kind).toBe("skip");
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
it("opt-in via explicit set restores the alarm (operator on pay-as-you-go)", () => {
|
|
214
|
+
const d = evaluateCreditState({
|
|
215
|
+
agentName: "clerk",
|
|
216
|
+
currentReason: "out_of_credits",
|
|
217
|
+
prev: HEALTHY,
|
|
218
|
+
now: NOW,
|
|
219
|
+
fatalReasons: new Set(["out_of_credits"]),
|
|
220
|
+
});
|
|
221
|
+
expect(d.kind).toBe("notify");
|
|
222
|
+
});
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
describe("resolveCreditWatchFatalReasons", () => {
|
|
226
|
+
it("defaults to EMPTY (subscription-only)", () => {
|
|
227
|
+
expect(resolveCreditWatchFatalReasons({}).size).toBe(0);
|
|
228
|
+
});
|
|
229
|
+
it("parses a comma-separated opt-in list", () => {
|
|
230
|
+
const s = resolveCreditWatchFatalReasons({ SWITCHROOM_CREDITS_WATCH_FATAL_REASONS: "out_of_credits, org_level_disabled" });
|
|
231
|
+
expect(s.has("out_of_credits")).toBe(true);
|
|
232
|
+
expect(s.has("org_level_disabled")).toBe(true);
|
|
233
|
+
expect(s.size).toBe(2);
|
|
234
|
+
});
|
|
235
|
+
it("'*' opts in all known reasons", () => {
|
|
236
|
+
const s = resolveCreditWatchFatalReasons({ SWITCHROOM_CREDITS_WATCH_FATAL_REASONS: "*" });
|
|
237
|
+
expect(s.has("out_of_credits")).toBe(true);
|
|
238
|
+
expect(s.size).toBeGreaterThanOrEqual(4);
|
|
239
|
+
});
|
|
240
|
+
it("blank/whitespace → empty", () => {
|
|
241
|
+
expect(resolveCreditWatchFatalReasons({ SWITCHROOM_CREDITS_WATCH_FATAL_REASONS: " " }).size).toBe(0);
|
|
242
|
+
});
|
|
243
|
+
});
|
|
244
|
+
|
|
181
245
|
describe("loadCreditState / saveCreditState — round-trip", () => {
|
|
182
246
|
let tmp: string;
|
|
183
247
|
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unit tests for the shared exhaustion-`until` decision (exhaust-until.ts) and
|
|
3
|
+
* the 429-path composition it sits in.
|
|
4
|
+
*
|
|
5
|
+
* The load-bearing invariant (vision.md pillar 3 + the #2218 weekly-wall
|
|
6
|
+
* rollback): the value handed to markExhausted is NEVER undefined, so a weekly
|
|
7
|
+
* wall can never fall back to the broker's ~5h default and un-exhaust early.
|
|
8
|
+
*
|
|
9
|
+
* The 429-path composition test runs the REAL detectModelUnavailable
|
|
10
|
+
* (model-unavailable.ts) into the REAL resolveExhaustUntil — proving the gateway
|
|
11
|
+
* 429 branch (`resolveExhaustUntil(modelUnavailable.resetAt?.getTime())`) threads
|
|
12
|
+
* a finite `until` for both the rolling and the unparseable-weekly wordings.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { describe, it, expect } from 'vitest'
|
|
16
|
+
import { resolveExhaustUntil, EXHAUST_WEEKLY_FLOOR_MS } from '../gateway/exhaust-until.js'
|
|
17
|
+
import { detectModelUnavailable } from '../model-unavailable.js'
|
|
18
|
+
|
|
19
|
+
const NOW = Date.UTC(2026, 5, 7, 0, 0, 0) // 2026-06-07
|
|
20
|
+
|
|
21
|
+
describe('resolveExhaustUntil', () => {
|
|
22
|
+
it('passes through a finite reset in the future', () => {
|
|
23
|
+
const reset = NOW + 2 * 60 * 60 * 1000 // +2h (a rolling window)
|
|
24
|
+
expect(resolveExhaustUntil(reset, NOW)).toBe(reset)
|
|
25
|
+
})
|
|
26
|
+
|
|
27
|
+
it('undefined → +7d floor (the WEEKLY-wall case), never undefined', () => {
|
|
28
|
+
const until = resolveExhaustUntil(undefined, NOW)
|
|
29
|
+
expect(until).toBe(NOW + EXHAUST_WEEKLY_FLOOR_MS)
|
|
30
|
+
expect(until).toBeGreaterThan(NOW + 5 * 60 * 60 * 1000) // past the ~5h default
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
it('a past / now / NaN / non-finite reset → +7d floor (treated as unparsed)', () => {
|
|
34
|
+
expect(resolveExhaustUntil(NOW - 1, NOW)).toBe(NOW + EXHAUST_WEEKLY_FLOOR_MS)
|
|
35
|
+
expect(resolveExhaustUntil(NOW, NOW)).toBe(NOW + EXHAUST_WEEKLY_FLOOR_MS)
|
|
36
|
+
expect(resolveExhaustUntil(Number.NaN, NOW)).toBe(NOW + EXHAUST_WEEKLY_FLOOR_MS)
|
|
37
|
+
expect(resolveExhaustUntil(Number.POSITIVE_INFINITY, NOW)).toBe(NOW + EXHAUST_WEEKLY_FLOOR_MS)
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
it('the floor is comfortably longer than the broker ~5h mark-exhausted default', () => {
|
|
41
|
+
expect(EXHAUST_WEEKLY_FLOOR_MS).toBeGreaterThan(5 * 60 * 60 * 1000)
|
|
42
|
+
expect(EXHAUST_WEEKLY_FLOOR_MS).toBe(7 * 24 * 60 * 60 * 1000)
|
|
43
|
+
})
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
describe('429-path composition: detectModelUnavailable → resolveExhaustUntil', () => {
|
|
47
|
+
// This mirrors gateway.ts's 429 branch:
|
|
48
|
+
// const untilMs = resolveExhaustUntil(modelUnavailable.resetAt?.getTime())
|
|
49
|
+
// NB: parseResetTime anchors RELATIVE wordings ("in 2h", "retry after Ns") at
|
|
50
|
+
// the real Date.now() — not injectable through detectModelUnavailable — so the
|
|
51
|
+
// rolling-window assertions check the SHAPE (finite, future, well below the
|
|
52
|
+
// +7d floor, no longer than the stated window), never an exact epoch.
|
|
53
|
+
function untilFor(stderr: string): { u: number; before: number } {
|
|
54
|
+
const before = Date.now()
|
|
55
|
+
const d = detectModelUnavailable(stderr)
|
|
56
|
+
expect(d).not.toBeNull()
|
|
57
|
+
return { u: resolveExhaustUntil(d?.resetAt?.getTime(), before), before }
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
it('ROLLING wording ("resets in 2h 15m") → a finite ~2h until, NOT the +7d floor', () => {
|
|
61
|
+
const { u, before } = untilFor('usage limit reached · resets in 2h 15m')
|
|
62
|
+
expect(Number.isFinite(u)).toBe(true)
|
|
63
|
+
expect(u).toBeGreaterThan(before)
|
|
64
|
+
const window = (2 * 60 + 15) * 60 * 1000
|
|
65
|
+
expect(u).toBeLessThanOrEqual(before + window + 2000) // ~2h15m, allow detect/resolve skew
|
|
66
|
+
expect(u).toBeLessThan(before + EXHAUST_WEEKLY_FLOOR_MS / 2) // unmistakably NOT the floor
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
it('ROLLING wording ("retry after 60 seconds") → a finite ~60s until, NOT the +7d floor', () => {
|
|
70
|
+
const { u, before } = untilFor('rate_limit_error: retry after 60 seconds — usage limit')
|
|
71
|
+
expect(u).toBeGreaterThan(before)
|
|
72
|
+
expect(u).toBeLessThanOrEqual(before + 60 * 1000 + 2000)
|
|
73
|
+
expect(u).toBeLessThan(before + EXHAUST_WEEKLY_FLOOR_MS / 2)
|
|
74
|
+
})
|
|
75
|
+
|
|
76
|
+
it("WEEKLY wording (\"resets Jun 9, 5am (tz)\") is UNPARSEABLE on the 429 path → +7d floor, NOT undefined/5h", () => {
|
|
77
|
+
// parseResetTime can't read the calendar weekly wording — Invalid Date →
|
|
78
|
+
// resetAt undefined. resolveExhaustUntil MUST floor it to +7d, otherwise
|
|
79
|
+
// markExhausted's ~5h default re-wedges the weekly-walled account.
|
|
80
|
+
const d = detectModelUnavailable("You've hit your limit · resets Jun 9, 5am (Australia/Melbourne)")
|
|
81
|
+
expect(d?.kind).toBe('quota_exhausted')
|
|
82
|
+
expect(d?.resetAt).toBeUndefined() // pins the parser-blind-spot premise
|
|
83
|
+
const u = resolveExhaustUntil(d?.resetAt?.getTime(), NOW)
|
|
84
|
+
expect(u).toBe(NOW + EXHAUST_WEEKLY_FLOOR_MS)
|
|
85
|
+
})
|
|
86
|
+
})
|