switchroom 0.15.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +22 -1
- package/dist/auth-broker/index.js +36 -1
- package/dist/cli/drive-write-pretool.mjs +23 -2
- package/dist/cli/switchroom.js +39 -6
- package/package.json +1 -1
- package/telegram-plugin/auth-snapshot-format.ts +9 -0
- package/telegram-plugin/auto-fallback-fleet.ts +59 -0
- package/telegram-plugin/dist/gateway/gateway.js +224 -19
- package/telegram-plugin/gateway/auth-broker-client.ts +2 -0
- package/telegram-plugin/gateway/auth-command.ts +35 -2
- package/telegram-plugin/gateway/gateway.ts +203 -18
- package/telegram-plugin/quota-watch.ts +141 -3
- package/telegram-plugin/tests/auth-quota-util-cell.test.ts +23 -0
- package/telegram-plugin/tests/auto-fallback-fleet.test.ts +71 -0
- package/telegram-plugin/tests/quota-watch.test.ts +266 -0
|
@@ -16160,7 +16160,7 @@ function decodeResponse(line) {
|
|
|
16160
16160
|
}
|
|
16161
16161
|
return ResponseSchema.parse(parsed);
|
|
16162
16162
|
}
|
|
16163
|
-
var MAX_FRAME_BYTES, PROTOCOL_VERSION = 1, ProviderNameSchema, GetCredentialsRequestSchema, ListStateRequestSchema, SetActiveRequestSchema, MarkExhaustedRequestSchema, RefreshAccountRequestSchema, AnthropicCredentialsSchema, GoogleCredentialsSchema, MicrosoftCredentialsSchema, ProviderCredentialsSchema, AddAccountRequestSchema, RmAccountRequestSchema, SetOverrideRequestSchema, ListGoogleAccountsRequestSchema, ListMicrosoftAccountsRequestSchema, ProbeQuotaRequestSchema, RequestSchema, GetCredentialsDataSchema, AccountStateSchema, AgentStateSchema, ConsumerStateSchema, ListStateDataSchema, SetActiveDataSchema, MarkExhaustedDataSchema, RefreshAccountDataSchema, AddAccountDataSchema, RmAccountDataSchema, SetOverrideDataSchema, GoogleAccountStateSchema, ListGoogleAccountsDataSchema, MicrosoftAccountStateSchema, ListMicrosoftAccountsDataSchema, ErrorBodySchema, SuccessResponseSchema, ErrorResponseSchema, ResponseSchema;
|
|
16163
|
+
var MAX_FRAME_BYTES, PROTOCOL_VERSION = 1, ProviderNameSchema, GetCredentialsRequestSchema, ListStateRequestSchema, SetActiveRequestSchema, MarkExhaustedRequestSchema, RefreshAccountRequestSchema, AnthropicCredentialsSchema, GoogleCredentialsSchema, MicrosoftCredentialsSchema, ProviderCredentialsSchema, AddAccountRequestSchema, RmAccountRequestSchema, SetOverrideRequestSchema, ListGoogleAccountsRequestSchema, ListMicrosoftAccountsRequestSchema, ProbeQuotaRequestSchema, ClaimNotificationRequestSchema, RequestSchema, GetCredentialsDataSchema, AccountStateSchema, AgentStateSchema, ConsumerStateSchema, ListStateDataSchema, SetActiveDataSchema, MarkExhaustedDataSchema, RefreshAccountDataSchema, AddAccountDataSchema, RmAccountDataSchema, SetOverrideDataSchema, ClaimNotificationDataSchema, GoogleAccountStateSchema, ListGoogleAccountsDataSchema, MicrosoftAccountStateSchema, ListMicrosoftAccountsDataSchema, ErrorBodySchema, SuccessResponseSchema, ErrorResponseSchema, ResponseSchema;
|
|
16164
16164
|
var init_protocol = __esm(() => {
|
|
16165
16165
|
init_zod();
|
|
16166
16166
|
MAX_FRAME_BYTES = 64 * 1024;
|
|
@@ -16276,6 +16276,13 @@ var init_protocol = __esm(() => {
|
|
|
16276
16276
|
accounts: exports_external.array(exports_external.string().min(1)).min(1).max(32),
|
|
16277
16277
|
timeoutMs: exports_external.number().int().positive().max(60000).optional()
|
|
16278
16278
|
});
|
|
16279
|
+
ClaimNotificationRequestSchema = exports_external.object({
|
|
16280
|
+
v: exports_external.literal(PROTOCOL_VERSION),
|
|
16281
|
+
op: exports_external.literal("claim-notification"),
|
|
16282
|
+
id: exports_external.string().min(1),
|
|
16283
|
+
key: exports_external.string().min(1).max(512),
|
|
16284
|
+
windowMs: exports_external.number().int().positive().max(86400000)
|
|
16285
|
+
});
|
|
16279
16286
|
RequestSchema = exports_external.discriminatedUnion("op", [
|
|
16280
16287
|
GetCredentialsRequestSchema,
|
|
16281
16288
|
ListStateRequestSchema,
|
|
@@ -16287,7 +16294,8 @@ var init_protocol = __esm(() => {
|
|
|
16287
16294
|
SetOverrideRequestSchema,
|
|
16288
16295
|
ListGoogleAccountsRequestSchema,
|
|
16289
16296
|
ListMicrosoftAccountsRequestSchema,
|
|
16290
|
-
ProbeQuotaRequestSchema
|
|
16297
|
+
ProbeQuotaRequestSchema,
|
|
16298
|
+
ClaimNotificationRequestSchema
|
|
16291
16299
|
]);
|
|
16292
16300
|
GetCredentialsDataSchema = exports_external.object({
|
|
16293
16301
|
account: exports_external.string(),
|
|
@@ -16343,6 +16351,9 @@ var init_protocol = __esm(() => {
|
|
|
16343
16351
|
agent: exports_external.string(),
|
|
16344
16352
|
account: exports_external.string().nullable()
|
|
16345
16353
|
});
|
|
16354
|
+
ClaimNotificationDataSchema = exports_external.object({
|
|
16355
|
+
granted: exports_external.boolean()
|
|
16356
|
+
});
|
|
16346
16357
|
GoogleAccountStateSchema = exports_external.object({
|
|
16347
16358
|
account: exports_external.string(),
|
|
16348
16359
|
expiresAt: exports_external.number(),
|
|
@@ -16524,6 +16535,16 @@ class AuthBrokerClient {
|
|
|
16524
16535
|
const data = await this.send(req);
|
|
16525
16536
|
return data;
|
|
16526
16537
|
}
|
|
16538
|
+
async claimNotification(key, windowMs) {
|
|
16539
|
+
const data = await this.send({
|
|
16540
|
+
v: PROTOCOL_VERSION,
|
|
16541
|
+
id: randomUUID3(),
|
|
16542
|
+
op: "claim-notification",
|
|
16543
|
+
key,
|
|
16544
|
+
windowMs
|
|
16545
|
+
});
|
|
16546
|
+
return data;
|
|
16547
|
+
}
|
|
16527
16548
|
async refreshAccount(account) {
|
|
16528
16549
|
const data = await this.send({
|
|
16529
16550
|
v: PROTOCOL_VERSION,
|
|
@@ -30611,7 +30632,8 @@ function buildSnapshotsFromCachedState(state4) {
|
|
|
30611
30632
|
isActive: acc.label === state4.active,
|
|
30612
30633
|
quota: reviveLastQuota(lq),
|
|
30613
30634
|
quotaError: lq ? undefined : "no cached quota (no probe since broker start)",
|
|
30614
|
-
expiresAtMs: acc.expiresAt
|
|
30635
|
+
expiresAtMs: acc.expiresAt,
|
|
30636
|
+
capturedAtMs: lq?.capturedAt
|
|
30615
30637
|
};
|
|
30616
30638
|
});
|
|
30617
30639
|
}
|
|
@@ -40687,17 +40709,31 @@ function renderShowText(state3, now = Date.now(), opts = {}) {
|
|
|
40687
40709
|
`);
|
|
40688
40710
|
}
|
|
40689
40711
|
function formatAccountsTable(state3, now) {
|
|
40690
|
-
const rows = [["ACCOUNT", "STATUS", "EXPIRES", "QUOTA-RESET"]];
|
|
40712
|
+
const rows = [["ACCOUNT", "STATUS", "EXPIRES", "QUOTA 5h\u00b77d", "QUOTA-RESET"]];
|
|
40691
40713
|
for (const acc of state3.accounts) {
|
|
40692
40714
|
const isActive = acc.label === state3.active;
|
|
40693
40715
|
const marker = isActive ? "\u25cf" : acc.exhausted ? "!" : "\u2713";
|
|
40694
40716
|
const status = isActive ? "active" : acc.exhausted ? "exhausted" : "available";
|
|
40695
40717
|
const expires = acc.expiresAt != null ? formatRelativeMs(acc.expiresAt - now) : "\u2014";
|
|
40696
40718
|
const quotaReset = acc.exhausted && acc.exhausted_until != null && acc.exhausted_until > now ? formatRelativeMs(acc.exhausted_until - now) : "\u2014";
|
|
40697
|
-
rows.push([
|
|
40719
|
+
rows.push([
|
|
40720
|
+
`${marker} ${escapeHtml3(acc.label)}`,
|
|
40721
|
+
status,
|
|
40722
|
+
expires,
|
|
40723
|
+
formatQuotaUtilCell(acc, now),
|
|
40724
|
+
quotaReset
|
|
40725
|
+
]);
|
|
40698
40726
|
}
|
|
40699
40727
|
return alignTable(rows);
|
|
40700
40728
|
}
|
|
40729
|
+
function formatQuotaUtilCell(acc, now) {
|
|
40730
|
+
const lq = acc.last_quota;
|
|
40731
|
+
if (!lq)
|
|
40732
|
+
return "no data";
|
|
40733
|
+
const age = now - lq.capturedAt;
|
|
40734
|
+
const ageStr = age > 0 ? formatRelativeMs(age) : "0s";
|
|
40735
|
+
return `${Math.round(lq.fiveHourUtilizationPct)}%\u00b7${Math.round(lq.sevenDayUtilizationPct)}% (${ageStr} ago)`;
|
|
40736
|
+
}
|
|
40701
40737
|
function formatAgentsTable(state3) {
|
|
40702
40738
|
const rows = [["AGENT", "ACTIVE", "SOURCE"]];
|
|
40703
40739
|
for (const a of state3.agents) {
|
|
@@ -40801,7 +40837,8 @@ function createAuthBrokerClient() {
|
|
|
40801
40837
|
rmAccount: (label) => broker.rmAccount(label),
|
|
40802
40838
|
refreshAccount: (label) => broker.refreshAccount(label),
|
|
40803
40839
|
setOverride: (agent, account) => broker.setOverride(agent, account),
|
|
40804
|
-
probeQuota: (accounts, timeoutMs) => broker.probeQuota(accounts, timeoutMs)
|
|
40840
|
+
probeQuota: (accounts, timeoutMs) => broker.probeQuota(accounts, timeoutMs),
|
|
40841
|
+
claimNotification: (key, windowMs) => broker.claimNotification(key, windowMs)
|
|
40805
40842
|
};
|
|
40806
40843
|
return { client: client3, close: () => broker.close() };
|
|
40807
40844
|
}
|
|
@@ -41215,6 +41252,16 @@ class AuthBrokerClient2 {
|
|
|
41215
41252
|
const data = await this.send(req);
|
|
41216
41253
|
return data;
|
|
41217
41254
|
}
|
|
41255
|
+
async claimNotification(key, windowMs) {
|
|
41256
|
+
const data = await this.send({
|
|
41257
|
+
v: PROTOCOL_VERSION,
|
|
41258
|
+
id: randomUUID4(),
|
|
41259
|
+
op: "claim-notification",
|
|
41260
|
+
key,
|
|
41261
|
+
windowMs
|
|
41262
|
+
});
|
|
41263
|
+
return data;
|
|
41264
|
+
}
|
|
41218
41265
|
async refreshAccount(account) {
|
|
41219
41266
|
const data = await this.send({
|
|
41220
41267
|
v: PROTOCOL_VERSION,
|
|
@@ -42085,6 +42132,22 @@ function escHtml2(text) {
|
|
|
42085
42132
|
}
|
|
42086
42133
|
|
|
42087
42134
|
// auto-fallback-fleet.ts
|
|
42135
|
+
function renderFallbackFailureNotice(triggerAgent, reason) {
|
|
42136
|
+
return `\u26a0\ufe0f <b>Auto-failover could not run</b> (trigger: <b>${escFailureHtml(triggerAgent)}</b>)
|
|
42137
|
+
` + `${escFailureHtml(reason)}
|
|
42138
|
+
|
|
42139
|
+
` + `<i>Switch manually with <code>/auth use <label></code>, or <code>/auth</code> for fleet status.</i>`;
|
|
42140
|
+
}
|
|
42141
|
+
function escFailureHtml(s) {
|
|
42142
|
+
return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
42143
|
+
}
|
|
42144
|
+
var FALLBACK_FAILURE_NOTICE_COOLDOWN_MS = 30 * 60000;
|
|
42145
|
+
function evaluateFallbackFailureNotice(prev, now, cooldownMs = FALLBACK_FAILURE_NOTICE_COOLDOWN_MS) {
|
|
42146
|
+
if (now - prev.lastSentAtMs >= cooldownMs) {
|
|
42147
|
+
return { send: true, next: { lastSentAtMs: now } };
|
|
42148
|
+
}
|
|
42149
|
+
return { send: false, next: prev };
|
|
42150
|
+
}
|
|
42088
42151
|
async function runFleetAutoFallback(deps) {
|
|
42089
42152
|
const now = deps.now ?? new Date;
|
|
42090
42153
|
const tz = deps.tz ?? "UTC";
|
|
@@ -52727,9 +52790,35 @@ function emptyQuotaWatchState() {
|
|
|
52727
52790
|
function emptyAccountState() {
|
|
52728
52791
|
return { lastNotifiedHealth: null, lastNotifiedAt: 0 };
|
|
52729
52792
|
}
|
|
52793
|
+
var DEFAULT_QUOTA_WATCH_MAX_STALE_MS = 60 * 60000;
|
|
52794
|
+
var DEFAULT_QUOTA_WATCH_LATE_RECOVERY_MS = 6 * 60 * 60000;
|
|
52795
|
+
var QUOTA_WATCH_CLAIM_WINDOW_MS = 30 * 60000;
|
|
52796
|
+
function resolveQuotaWatchTuning(env) {
|
|
52797
|
+
const num = (raw, fallback) => {
|
|
52798
|
+
if (raw === undefined || raw === "")
|
|
52799
|
+
return fallback;
|
|
52800
|
+
const n = Number(raw);
|
|
52801
|
+
return Number.isFinite(n) && n >= 0 ? n : fallback;
|
|
52802
|
+
};
|
|
52803
|
+
return {
|
|
52804
|
+
maxStaleMs: num(env.SWITCHROOM_QUOTA_WATCH_MAX_STALE_MS, DEFAULT_QUOTA_WATCH_MAX_STALE_MS),
|
|
52805
|
+
lateRecoveryMs: num(env.SWITCHROOM_QUOTA_WATCH_LATE_RECOVERY_MS, DEFAULT_QUOTA_WATCH_LATE_RECOVERY_MS),
|
|
52806
|
+
fleetDedup: env.SWITCHROOM_QUOTA_WATCH_FLEET_DEDUP !== "0",
|
|
52807
|
+
sendOnProbeFail: env.SWITCHROOM_QUOTA_WATCH_SEND_ON_PROBE_FAIL === "1"
|
|
52808
|
+
};
|
|
52809
|
+
}
|
|
52810
|
+
function buildQuotaClaimKey(accountLabel, transition2, chatId) {
|
|
52811
|
+
return `quota-watch:${accountLabel}:${transition2}:${chatId}`;
|
|
52812
|
+
}
|
|
52730
52813
|
function evaluateQuotaWatchAccount(args) {
|
|
52731
52814
|
const { agentName: agentName3, snap, prev, now } = args;
|
|
52815
|
+
const bootTick = args.bootTick ?? false;
|
|
52816
|
+
const maxStaleMs = args.tuning?.maxStaleMs ?? 0;
|
|
52817
|
+
const lateRecoveryMs = args.tuning?.lateRecoveryMs ?? 0;
|
|
52732
52818
|
const label = snap.label;
|
|
52819
|
+
if (maxStaleMs > 0 && snap.capturedAtMs !== undefined && now - snap.capturedAtMs > maxStaleMs) {
|
|
52820
|
+
return { kind: "skip", accountLabel: label, reason: "stale-snapshot" };
|
|
52821
|
+
}
|
|
52733
52822
|
const currentHealth = classifyHealth(snap);
|
|
52734
52823
|
if (currentHealth === "unknown" || currentHealth === "blocked") {
|
|
52735
52824
|
return { kind: "skip", accountLabel: label, reason: `${currentHealth}-not-our-domain` };
|
|
@@ -52756,6 +52845,24 @@ function evaluateQuotaWatchAccount(args) {
|
|
|
52756
52845
|
lastNotifiedHealth: "healthy",
|
|
52757
52846
|
lastNotifiedAt: now
|
|
52758
52847
|
};
|
|
52848
|
+
if (bootTick) {
|
|
52849
|
+
return {
|
|
52850
|
+
kind: "reconcile",
|
|
52851
|
+
accountLabel: label,
|
|
52852
|
+
newAccountState: newState,
|
|
52853
|
+
transition: "recovered-to-healthy",
|
|
52854
|
+
reason: "boot-tick-recovery"
|
|
52855
|
+
};
|
|
52856
|
+
}
|
|
52857
|
+
if (lateRecoveryMs > 0 && now - prev.lastNotifiedAt > lateRecoveryMs) {
|
|
52858
|
+
return {
|
|
52859
|
+
kind: "reconcile",
|
|
52860
|
+
accountLabel: label,
|
|
52861
|
+
newAccountState: newState,
|
|
52862
|
+
transition: "recovered-to-healthy",
|
|
52863
|
+
reason: "late-recovery"
|
|
52864
|
+
};
|
|
52865
|
+
}
|
|
52759
52866
|
return {
|
|
52760
52867
|
kind: "notify",
|
|
52761
52868
|
accountLabel: label,
|
|
@@ -52971,10 +53078,10 @@ function readTurnActiveMarkerAgeMs(stateDir, now) {
|
|
|
52971
53078
|
}
|
|
52972
53079
|
|
|
52973
53080
|
// ../src/build-info.ts
|
|
52974
|
-
var VERSION = "0.
|
|
52975
|
-
var COMMIT_SHA = "
|
|
52976
|
-
var COMMIT_DATE = "2026-06-
|
|
52977
|
-
var LATEST_PR =
|
|
53081
|
+
var VERSION = "0.14.72";
|
|
53082
|
+
var COMMIT_SHA = "0e840d59";
|
|
53083
|
+
var COMMIT_DATE = "2026-06-06T00:39:32Z";
|
|
53084
|
+
var LATEST_PR = 2183;
|
|
52978
53085
|
var COMMITS_AHEAD_OF_TAG = 0;
|
|
52979
53086
|
|
|
52980
53087
|
// gateway/boot-version.ts
|
|
@@ -61156,12 +61263,32 @@ async function fireFleetAutoFallback(triggerAgent, untilMs) {
|
|
|
61156
61263
|
`);
|
|
61157
61264
|
});
|
|
61158
61265
|
}
|
|
61266
|
+
var fallbackFailureNoticeState = { lastSentAtMs: 0 };
|
|
61267
|
+
function broadcastFleetFallbackFailure(triggerAgent, reason) {
|
|
61268
|
+
if (process.env.SWITCHROOM_FLEET_FALLBACK_FAILURE_NOTICE === "0")
|
|
61269
|
+
return;
|
|
61270
|
+
const verdict = evaluateFallbackFailureNotice(fallbackFailureNoticeState, Date.now());
|
|
61271
|
+
if (!verdict.send) {
|
|
61272
|
+
process.stderr.write(`telegram gateway: [fleet-fallback] failure notice suppressed (cooldown) agent=${triggerAgent}: ${reason}
|
|
61273
|
+
`);
|
|
61274
|
+
return;
|
|
61275
|
+
}
|
|
61276
|
+
fallbackFailureNoticeState = verdict.next;
|
|
61277
|
+
const access = loadAccess();
|
|
61278
|
+
if (access.allowFrom.length === 0)
|
|
61279
|
+
return;
|
|
61280
|
+
const html = renderFallbackFailureNotice(triggerAgent, reason);
|
|
61281
|
+
for (const chat_id of access.allowFrom) {
|
|
61282
|
+
swallowingApiCall(() => bot.api.sendMessage(chat_id, html, { parse_mode: "HTML" }), { chat_id, verb: "fleet-fallback:failure-notify" });
|
|
61283
|
+
}
|
|
61284
|
+
}
|
|
61159
61285
|
async function doFireFleetAutoFallback(triggerAgent, untilMs) {
|
|
61160
61286
|
try {
|
|
61161
61287
|
const client3 = await getAuthBrokerClient(triggerAgent);
|
|
61162
61288
|
if (!client3) {
|
|
61163
61289
|
process.stderr.write(`telegram gateway: [fleet-fallback] skipped agent=${triggerAgent} reason=no-broker-client
|
|
61164
61290
|
`);
|
|
61291
|
+
broadcastFleetFallbackFailure(triggerAgent, "auth-broker unreachable (no client).");
|
|
61165
61292
|
return false;
|
|
61166
61293
|
}
|
|
61167
61294
|
const state4 = await client3.listState();
|
|
@@ -61194,6 +61321,7 @@ async function doFireFleetAutoFallback(triggerAgent, untilMs) {
|
|
|
61194
61321
|
} catch (err) {
|
|
61195
61322
|
process.stderr.write(`telegram gateway: [fleet-fallback] error agent=${triggerAgent}: ${err?.message ?? err}
|
|
61196
61323
|
`);
|
|
61324
|
+
broadcastFleetFallbackFailure(triggerAgent, err?.message ?? String(err));
|
|
61197
61325
|
return false;
|
|
61198
61326
|
}
|
|
61199
61327
|
}
|
|
@@ -61230,9 +61358,21 @@ async function runCreditWatch() {
|
|
|
61230
61358
|
`);
|
|
61231
61359
|
}
|
|
61232
61360
|
}
|
|
61233
|
-
async function
|
|
61361
|
+
async function claimQuotaNotification(brokerClient, key) {
|
|
61362
|
+
try {
|
|
61363
|
+
const res = await brokerClient.claimNotification(key, QUOTA_WATCH_CLAIM_WINDOW_MS);
|
|
61364
|
+
return res.granted;
|
|
61365
|
+
} catch (err) {
|
|
61366
|
+
process.stderr.write(`telegram gateway: quota-watch: claim failed (fail-open): ${err}
|
|
61367
|
+
`);
|
|
61368
|
+
return true;
|
|
61369
|
+
}
|
|
61370
|
+
}
|
|
61371
|
+
async function runQuotaWatch(opts = {}) {
|
|
61234
61372
|
const agentName3 = getMyAgentName();
|
|
61235
61373
|
const stateDir = STATE_DIR;
|
|
61374
|
+
const bootTick = opts.bootTick ?? false;
|
|
61375
|
+
const tuning = resolveQuotaWatchTuning(process.env);
|
|
61236
61376
|
const brokerClient = await getAuthBrokerClient(agentName3);
|
|
61237
61377
|
if (!brokerClient) {
|
|
61238
61378
|
process.stderr.write(`telegram gateway: quota-watch: broker client unavailable \u2014 skipping
|
|
@@ -61263,6 +61403,14 @@ async function runQuotaWatch() {
|
|
|
61263
61403
|
});
|
|
61264
61404
|
if (fleetDecision.kind === "notify") {
|
|
61265
61405
|
for (const chat_id of access.allowFrom) {
|
|
61406
|
+
if (tuning.fleetDedup) {
|
|
61407
|
+
const granted = await claimQuotaNotification(brokerClient, buildQuotaClaimKey(FLEET_ALL_EXHAUSTED_KEY, fleetDecision.transition, chat_id));
|
|
61408
|
+
if (!granted) {
|
|
61409
|
+
process.stderr.write(`telegram gateway: quota-watch: fleet-all-exhausted claim denied chat=${chat_id} \u2014 another agent notified
|
|
61410
|
+
`);
|
|
61411
|
+
continue;
|
|
61412
|
+
}
|
|
61413
|
+
}
|
|
61266
61414
|
await swallowingApiCall(() => bot.api.sendMessage(chat_id, fleetDecision.message, {
|
|
61267
61415
|
parse_mode: "HTML",
|
|
61268
61416
|
link_preview_options: { is_disabled: true }
|
|
@@ -61281,10 +61429,17 @@ async function runQuotaWatch() {
|
|
|
61281
61429
|
}
|
|
61282
61430
|
const pendingTransitions = [];
|
|
61283
61431
|
const labelToSnapIndex = new Map(snapshots.map((s, i) => [s.label, i]));
|
|
61432
|
+
let reconciledCount = 0;
|
|
61433
|
+
let mutatedState = watchState;
|
|
61284
61434
|
for (const snap of snapshots) {
|
|
61285
61435
|
const prev = watchState[snap.label] ?? emptyAccountState();
|
|
61286
|
-
const decision = evaluateQuotaWatchAccount({ agentName: agentName3, snap, prev, now });
|
|
61287
|
-
if (decision.kind
|
|
61436
|
+
const decision = evaluateQuotaWatchAccount({ agentName: agentName3, snap, prev, now, bootTick, tuning });
|
|
61437
|
+
if (decision.kind === "reconcile") {
|
|
61438
|
+
mutatedState = patchQuotaWatchState(mutatedState, decision.accountLabel, decision.newAccountState);
|
|
61439
|
+
reconciledCount++;
|
|
61440
|
+
process.stderr.write(`telegram gateway: quota-watch: reconciled ${decision.transition} for account=${decision.accountLabel} (${decision.reason}) \u2014 no notification
|
|
61441
|
+
`);
|
|
61442
|
+
} else if (decision.kind !== "skip") {
|
|
61288
61443
|
pendingTransitions.push({
|
|
61289
61444
|
accountLabel: snap.label,
|
|
61290
61445
|
snapIndex: labelToSnapIndex.get(snap.label) ?? -1,
|
|
@@ -61293,6 +61448,14 @@ async function runQuotaWatch() {
|
|
|
61293
61448
|
}
|
|
61294
61449
|
}
|
|
61295
61450
|
if (pendingTransitions.length === 0) {
|
|
61451
|
+
if (reconciledCount > 0) {
|
|
61452
|
+
try {
|
|
61453
|
+
saveQuotaWatchState(stateDir, mutatedState);
|
|
61454
|
+
} catch (err) {
|
|
61455
|
+
process.stderr.write(`telegram gateway: quota-watch state persist failed: ${err}
|
|
61456
|
+
`);
|
|
61457
|
+
}
|
|
61458
|
+
}
|
|
61296
61459
|
return;
|
|
61297
61460
|
}
|
|
61298
61461
|
const crossingLabels = pendingTransitions.map((t) => t.accountLabel);
|
|
@@ -61305,8 +61468,20 @@ async function runQuotaWatch() {
|
|
|
61305
61468
|
} catch (err) {
|
|
61306
61469
|
process.stderr.write(`telegram gateway: quota-watch: probe for crossing accounts failed: ${err}
|
|
61307
61470
|
`);
|
|
61471
|
+
if (!tuning.sendOnProbeFail) {
|
|
61472
|
+
if (reconciledCount > 0) {
|
|
61473
|
+
try {
|
|
61474
|
+
saveQuotaWatchState(stateDir, mutatedState);
|
|
61475
|
+
} catch (saveErr) {
|
|
61476
|
+
process.stderr.write(`telegram gateway: quota-watch state persist failed: ${saveErr}
|
|
61477
|
+
`);
|
|
61478
|
+
}
|
|
61479
|
+
}
|
|
61480
|
+
process.stderr.write(`telegram gateway: quota-watch: deferring ${pendingTransitions.length} notification(s) until probe succeeds
|
|
61481
|
+
`);
|
|
61482
|
+
return;
|
|
61483
|
+
}
|
|
61308
61484
|
}
|
|
61309
|
-
let mutatedState = watchState;
|
|
61310
61485
|
const notifications = [];
|
|
61311
61486
|
for (const { accountLabel, snapIndex, decision } of pendingTransitions) {
|
|
61312
61487
|
const freshResult = freshProbeMap.get(accountLabel);
|
|
@@ -61314,25 +61489,55 @@ async function runQuotaWatch() {
|
|
|
61314
61489
|
if (decision.kind !== "notify")
|
|
61315
61490
|
continue;
|
|
61316
61491
|
if (freshResult && freshResult.ok && snapIndex >= 0) {
|
|
61317
|
-
const enrichedSnap = { ...snapshots[snapIndex], quota: freshResult.data };
|
|
61492
|
+
const enrichedSnap = { ...snapshots[snapIndex], quota: freshResult.data, capturedAtMs: undefined };
|
|
61318
61493
|
const prev = watchState[accountLabel] ?? emptyAccountState();
|
|
61319
|
-
const re = evaluateQuotaWatchAccount({ agentName: agentName3, snap: enrichedSnap, prev, now });
|
|
61494
|
+
const re = evaluateQuotaWatchAccount({ agentName: agentName3, snap: enrichedSnap, prev, now, bootTick, tuning });
|
|
61320
61495
|
if (re.kind === "notify" && re.transition === decision.transition) {
|
|
61321
61496
|
enrichedDecision = re;
|
|
61497
|
+
} else if (re.kind === "reconcile") {
|
|
61498
|
+
mutatedState = patchQuotaWatchState(mutatedState, accountLabel, re.newAccountState);
|
|
61499
|
+
reconciledCount++;
|
|
61500
|
+
process.stderr.write(`telegram gateway: quota-watch: reconciled ${re.transition} for account=${accountLabel} (${re.reason}) \u2014 no notification
|
|
61501
|
+
`);
|
|
61502
|
+
continue;
|
|
61322
61503
|
} else if (re.kind === "skip") {
|
|
61323
61504
|
continue;
|
|
61324
61505
|
}
|
|
61506
|
+
} else if (!tuning.sendOnProbeFail) {
|
|
61507
|
+
process.stderr.write(`telegram gateway: quota-watch: probe unavailable for account=${accountLabel} \u2014 deferring notification
|
|
61508
|
+
`);
|
|
61509
|
+
continue;
|
|
61325
61510
|
}
|
|
61326
61511
|
if (enrichedDecision.kind !== "notify")
|
|
61327
61512
|
continue;
|
|
61328
|
-
notifications.push({
|
|
61513
|
+
notifications.push({
|
|
61514
|
+
message: enrichedDecision.message,
|
|
61515
|
+
accountLabel,
|
|
61516
|
+
transition: enrichedDecision.transition
|
|
61517
|
+
});
|
|
61329
61518
|
mutatedState = patchQuotaWatchState(mutatedState, accountLabel, enrichedDecision.newAccountState);
|
|
61330
61519
|
}
|
|
61331
61520
|
if (notifications.length === 0) {
|
|
61521
|
+
if (reconciledCount > 0) {
|
|
61522
|
+
try {
|
|
61523
|
+
saveQuotaWatchState(stateDir, mutatedState);
|
|
61524
|
+
} catch (err) {
|
|
61525
|
+
process.stderr.write(`telegram gateway: quota-watch state persist failed: ${err}
|
|
61526
|
+
`);
|
|
61527
|
+
}
|
|
61528
|
+
}
|
|
61332
61529
|
return;
|
|
61333
61530
|
}
|
|
61334
|
-
for (const { message, accountLabel } of notifications) {
|
|
61531
|
+
for (const { message, accountLabel, transition: transition2 } of notifications) {
|
|
61335
61532
|
for (const chat_id of access.allowFrom) {
|
|
61533
|
+
if (tuning.fleetDedup) {
|
|
61534
|
+
const granted = await claimQuotaNotification(brokerClient, buildQuotaClaimKey(accountLabel, transition2, chat_id));
|
|
61535
|
+
if (!granted) {
|
|
61536
|
+
process.stderr.write(`telegram gateway: quota-watch: claim denied account=${accountLabel} chat=${chat_id} \u2014 another agent notified
|
|
61537
|
+
`);
|
|
61538
|
+
continue;
|
|
61539
|
+
}
|
|
61540
|
+
}
|
|
61336
61541
|
await swallowingApiCall(() => bot.api.sendMessage(chat_id, message, {
|
|
61337
61542
|
parse_mode: "HTML",
|
|
61338
61543
|
link_preview_options: { is_disabled: true }
|
|
@@ -64661,7 +64866,7 @@ var didOneTimeSetup = false;
|
|
|
64661
64866
|
const QUOTA_WATCH_POLL_MS = Number(process.env.SWITCHROOM_QUOTA_WATCH_POLL_MS ?? 900000);
|
|
64662
64867
|
if (QUOTA_WATCH_POLL_MS > 0) {
|
|
64663
64868
|
setTimeout(() => {
|
|
64664
|
-
runQuotaWatch().catch((err) => {
|
|
64869
|
+
runQuotaWatch({ bootTick: true }).catch((err) => {
|
|
64665
64870
|
process.stderr.write(`telegram gateway: quota-watch initial run failed: ${err}
|
|
64666
64871
|
`);
|
|
64667
64872
|
});
|
|
@@ -34,6 +34,8 @@ export function createAuthBrokerClient(): {
|
|
|
34
34
|
broker.setOverride(agent, account),
|
|
35
35
|
probeQuota: (accounts: readonly string[], timeoutMs?: number) =>
|
|
36
36
|
broker.probeQuota(accounts, timeoutMs),
|
|
37
|
+
claimNotification: (key: string, windowMs: number) =>
|
|
38
|
+
broker.claimNotification(key, windowMs),
|
|
37
39
|
}
|
|
38
40
|
return { client, close: () => broker.close() }
|
|
39
41
|
}
|
|
@@ -238,6 +238,13 @@ export interface AuthBrokerClient {
|
|
|
238
238
|
accounts: readonly string[],
|
|
239
239
|
timeoutMs?: number,
|
|
240
240
|
): Promise<{ results: Array<{ label: string; result: import('../quota-check.js').QuotaResult }> }>
|
|
241
|
+
/**
|
|
242
|
+
* Fleet notification-dedup claim (quota-watch). First caller of `key`
|
|
243
|
+
* inside `windowMs` gets `granted: true` and sends; everyone else
|
|
244
|
+
* stays silent. Callers fail OPEN on error (skewed-rollout brokers
|
|
245
|
+
* reject the op at the protocol layer).
|
|
246
|
+
*/
|
|
247
|
+
claimNotification(key: string, windowMs: number): Promise<{ granted: boolean }>
|
|
241
248
|
}
|
|
242
249
|
|
|
243
250
|
export interface AuthCommandContext {
|
|
@@ -778,7 +785,7 @@ export function renderShowText(
|
|
|
778
785
|
}
|
|
779
786
|
|
|
780
787
|
function formatAccountsTable(state: ListStateData, now: number): string {
|
|
781
|
-
const rows: string[][] = [['ACCOUNT', 'STATUS', 'EXPIRES', 'QUOTA-RESET']]
|
|
788
|
+
const rows: string[][] = [['ACCOUNT', 'STATUS', 'EXPIRES', 'QUOTA 5h·7d', 'QUOTA-RESET']]
|
|
782
789
|
for (const acc of state.accounts) {
|
|
783
790
|
const isActive = acc.label === state.active
|
|
784
791
|
const marker = isActive
|
|
@@ -792,11 +799,37 @@ function formatAccountsTable(state: ListStateData, now: number): string {
|
|
|
792
799
|
acc.exhausted && acc.exhausted_until != null && acc.exhausted_until > now
|
|
793
800
|
? formatRelativeMs(acc.exhausted_until - now)
|
|
794
801
|
: '—'
|
|
795
|
-
rows.push([
|
|
802
|
+
rows.push([
|
|
803
|
+
`${marker} ${escapeHtml(acc.label)}`,
|
|
804
|
+
status,
|
|
805
|
+
expires,
|
|
806
|
+
formatQuotaUtilCell(acc, now),
|
|
807
|
+
quotaReset,
|
|
808
|
+
])
|
|
796
809
|
}
|
|
797
810
|
return alignTable(rows)
|
|
798
811
|
}
|
|
799
812
|
|
|
813
|
+
/**
|
|
814
|
+
* Cached-utilization cell for the legacy accounts table. Honesty fix
|
|
815
|
+
* (2026-06-09 incident follow-up): the table previously rendered ONLY
|
|
816
|
+
* the broker exhausted flag, so a 99%-utilized account read
|
|
817
|
+
* "available —" — and an agent parroting the table under-reported. Now
|
|
818
|
+
* shows the broker's cached 5h·7d utilization with its age, and
|
|
819
|
+
* distinguishes "no data" (never probed) from "not exhausted".
|
|
820
|
+
* Cached, not live: the (age) suffix is the staleness disclosure.
|
|
821
|
+
*/
|
|
822
|
+
export function formatQuotaUtilCell(
|
|
823
|
+
acc: { last_quota?: { fiveHourUtilizationPct: number; sevenDayUtilizationPct: number; capturedAt: number } | null },
|
|
824
|
+
now: number,
|
|
825
|
+
): string {
|
|
826
|
+
const lq = acc.last_quota
|
|
827
|
+
if (!lq) return 'no data'
|
|
828
|
+
const age = now - lq.capturedAt
|
|
829
|
+
const ageStr = age > 0 ? formatRelativeMs(age) : '0s'
|
|
830
|
+
return `${Math.round(lq.fiveHourUtilizationPct)}%·${Math.round(lq.sevenDayUtilizationPct)}% (${ageStr} ago)`
|
|
831
|
+
}
|
|
832
|
+
|
|
800
833
|
function formatAgentsTable(state: ListStateData): string {
|
|
801
834
|
const rows: string[][] = [['AGENT', 'ACTIVE', 'SOURCE']]
|
|
802
835
|
for (const a of state.agents) {
|