switchroom 0.18.13 → 0.18.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +49 -9
- package/dist/auth-broker/index.js +111 -7
- package/dist/cli/autoaccept-poll.js +23 -0
- package/dist/cli/drive-write-pretool.mjs +24 -1
- package/dist/cli/foreground-hog-pretool.mjs +264 -0
- package/dist/cli/notion-write-pretool.mjs +0 -1
- package/dist/cli/switchroom.js +35 -6
- package/dist/host-control/main.js +1 -2
- package/dist/vault/approvals/kernel-server.js +0 -1
- package/dist/vault/broker/server.js +0 -1
- package/package.json +1 -1
- package/profiles/coding/CLAUDE.md.hbs +2 -0
- package/profiles/default/CLAUDE.md.hbs +2 -0
- package/skills/switchroom-architecture/telegram.md +0 -1
- package/telegram-plugin/auth-snapshot-format.ts +37 -5
- package/telegram-plugin/auto-fallback-fleet.ts +29 -1
- package/telegram-plugin/bridge/bridge.ts +2 -0
- package/telegram-plugin/dist/bridge/bridge.js +2 -0
- package/telegram-plugin/dist/gateway/gateway.js +620 -67
- package/telegram-plugin/dist/server.js +2 -0
- package/telegram-plugin/gateway/auth-broker-client.ts +1 -0
- package/telegram-plugin/gateway/auth-command.ts +14 -0
- package/telegram-plugin/gateway/forward-origin.ts +235 -0
- package/telegram-plugin/gateway/gateway.ts +224 -10
- package/telegram-plugin/gateway/throttle-tier-wiring.ts +268 -0
- package/telegram-plugin/history.ts +55 -6
- package/telegram-plugin/model-unavailable.ts +20 -2
- package/telegram-plugin/render/rich-render.ts +40 -32
- package/telegram-plugin/stream-controller.ts +3 -2
- package/telegram-plugin/tests/auto-fallback-fleet.test.ts +72 -0
- package/telegram-plugin/tests/forward-origin.test.ts +309 -0
- package/telegram-plugin/tests/history.test.ts +157 -0
- package/telegram-plugin/tests/render/render-outbound-chunks.test.ts +6 -4
- package/telegram-plugin/tests/render/rich-render.test.ts +41 -22
- package/telegram-plugin/tests/single-mode-stream-reply.test.ts +5 -3
- package/telegram-plugin/tests/status-accent.test.ts +5 -3
- package/telegram-plugin/tests/stream-controller-chunk-cap.test.ts +20 -20
- package/telegram-plugin/tests/stream-reply-handler.test.ts +5 -2
- package/telegram-plugin/tests/throttle-tier-wiring.test.ts +290 -0
- package/telegram-plugin/tests/throttle-tier.test.ts +278 -0
- package/telegram-plugin/throttle-tier.ts +226 -0
- package/telegram-plugin/uat/scenarios/jtbd-rich-formatting-render-dm.test.ts +8 -7
|
@@ -11187,7 +11187,6 @@ var TelegramChannelSchema = exports_external.object({
|
|
|
11187
11187
|
enabled: exports_external.boolean().default(true).describe("Master switch for the per-agent Telegram gateway sidecar. " + "When false, start.sh skips the gateway supervise loop and the " + "agent boots without bot-token requirements (smoke-test + " + "offline-dev use case)."),
|
|
11188
11188
|
plugin: exports_external.enum(["switchroom", "official"]).optional().describe("Which Telegram MCP plugin to load. Default is 'switchroom' — the " + "enhanced fork with streaming edits, reactions, history, and " + "access control. Set to 'official' for the upstream marketplace " + "plugin (basic send/receive only)."),
|
|
11189
11189
|
format: exports_external.enum(["html", "markdownv2", "text"]).optional().describe("Default reply format passed to the plugin"),
|
|
11190
|
-
rate_limit_ms: exports_external.number().optional().describe("Minimum delay between outgoing messages in ms"),
|
|
11191
11190
|
stream_mode: exports_external.enum(["pty", "checklist"]).optional().describe("How live progress is streamed to Telegram during a turn. " + "'pty' (default) surfaces text snapshots of Claude Code's TUI — " + "compatible but can flicker as Ink re-renders. 'checklist' drives " + "a structured progress card from session-tail events — stable " + "order, per-tool status emojis, fires only on semantic transitions."),
|
|
11192
11191
|
stream_throttle_ms: exports_external.number().int().nonnegative().optional().describe("Throttle window in ms between successive in-place stream edits " + "during a turn. Lower = more responsive stream, higher = fewer API " + "calls. Floored at 250 by draft-stream itself. Default 400 ms for DMs " + "and 1000 ms for groups/forums (respects Telegram's ~1 edit/sec/message " + "practical ceiling). Override per-agent if a particular agent needs " + "snappier or quieter streaming."),
|
|
11193
11192
|
clear_status_on_completion: exports_external.boolean().optional().describe("When true, the live activity/status feed (the in-place 'what it's " + "doing' message — Reading X, Searching the web for Y, …) is DELETED " + "when the turn's final answer lands, so only the reply remains. " + "Default false: the status message is left in the chat as a record " + "(its last step marked done) — no post-then-delete. Per-agent " + "override; cascades defaults → profile → agent (per-key)."),
|
|
@@ -13636,19 +13635,30 @@ class JsonlAuditSink {
|
|
|
13636
13635
|
}
|
|
13637
13636
|
|
|
13638
13637
|
// src/scheduler/quota-preflight.ts
|
|
13639
|
-
function decideQuotaPreflight(state) {
|
|
13638
|
+
function decideQuotaPreflight(state, opts = {}) {
|
|
13639
|
+
const now = opts.now ?? Date.now();
|
|
13640
13640
|
const accounts = state.accounts ?? [];
|
|
13641
13641
|
if (accounts.length === 0) {
|
|
13642
13642
|
return { defer: false, reason: "no accounts in broker state" };
|
|
13643
13643
|
}
|
|
13644
13644
|
const healthy = accounts.filter((a) => !a.exhausted);
|
|
13645
|
-
if (healthy.length
|
|
13645
|
+
if (healthy.length === 0) {
|
|
13646
|
+
return { defer: true, reason: `all ${accounts.length} account(s) exhausted` };
|
|
13647
|
+
}
|
|
13648
|
+
const effectiveLabel = (opts.agent !== undefined ? (state.agents ?? []).find((a) => a.name === opts.agent)?.account : undefined) ?? state.active;
|
|
13649
|
+
const effective = accounts.find((a) => a.label === effectiveLabel);
|
|
13650
|
+
if (effective !== undefined && !effective.exhausted && effective.throttled_until !== undefined && effective.throttled_until > now) {
|
|
13651
|
+
const inS = Math.ceil((effective.throttled_until - now) / 1000);
|
|
13646
13652
|
return {
|
|
13647
|
-
defer:
|
|
13648
|
-
reason:
|
|
13653
|
+
defer: true,
|
|
13654
|
+
reason: `account '${effective.label}' rate-throttled (clears in ${inS}s)`,
|
|
13655
|
+
retryAtMs: effective.throttled_until
|
|
13649
13656
|
};
|
|
13650
13657
|
}
|
|
13651
|
-
return {
|
|
13658
|
+
return {
|
|
13659
|
+
defer: false,
|
|
13660
|
+
reason: `${healthy.length}/${accounts.length} account(s) healthy`
|
|
13661
|
+
};
|
|
13652
13662
|
}
|
|
13653
13663
|
|
|
13654
13664
|
// src/auth/broker/client.ts
|
|
@@ -13685,6 +13695,12 @@ var MarkExhaustedRequestSchema = exports_external.object({
|
|
|
13685
13695
|
id: exports_external.string().min(1),
|
|
13686
13696
|
until: exports_external.number().int().positive().optional()
|
|
13687
13697
|
});
|
|
13698
|
+
var MarkThrottledRequestSchema = exports_external.object({
|
|
13699
|
+
v: exports_external.literal(PROTOCOL_VERSION),
|
|
13700
|
+
op: exports_external.literal("mark-throttled"),
|
|
13701
|
+
id: exports_external.string().min(1),
|
|
13702
|
+
until: exports_external.number().int().positive()
|
|
13703
|
+
});
|
|
13688
13704
|
var RefreshAccountRequestSchema = exports_external.object({
|
|
13689
13705
|
v: exports_external.literal(PROTOCOL_VERSION),
|
|
13690
13706
|
op: exports_external.literal("refresh-account"),
|
|
@@ -13785,6 +13801,7 @@ var RequestSchema2 = exports_external.discriminatedUnion("op", [
|
|
|
13785
13801
|
ListStateRequestSchema,
|
|
13786
13802
|
SetActiveRequestSchema,
|
|
13787
13803
|
MarkExhaustedRequestSchema,
|
|
13804
|
+
MarkThrottledRequestSchema,
|
|
13788
13805
|
RefreshAccountRequestSchema,
|
|
13789
13806
|
AddAccountRequestSchema,
|
|
13790
13807
|
RmAccountRequestSchema,
|
|
@@ -13804,6 +13821,7 @@ var AccountStateSchema = exports_external.object({
|
|
|
13804
13821
|
expiresAt: exports_external.number().optional(),
|
|
13805
13822
|
exhausted: exports_external.boolean(),
|
|
13806
13823
|
exhausted_until: exports_external.number().optional(),
|
|
13824
|
+
throttled_until: exports_external.number().optional(),
|
|
13807
13825
|
threshold_violations: exports_external.number().int().nonnegative().optional(),
|
|
13808
13826
|
last_refreshed_at: exports_external.number().optional()
|
|
13809
13827
|
});
|
|
@@ -13834,6 +13852,12 @@ var MarkExhaustedDataSchema = exports_external.object({
|
|
|
13834
13852
|
rolled: exports_external.array(exports_external.string()),
|
|
13835
13853
|
rolledTo: exports_external.string().nullable().optional()
|
|
13836
13854
|
});
|
|
13855
|
+
var MarkThrottledDataSchema = exports_external.object({
|
|
13856
|
+
account: exports_external.string(),
|
|
13857
|
+
throttled_until: exports_external.number(),
|
|
13858
|
+
escalated: exports_external.boolean(),
|
|
13859
|
+
rolledTo: exports_external.string().nullable().optional()
|
|
13860
|
+
});
|
|
13837
13861
|
var RefreshAccountDataSchema = exports_external.object({
|
|
13838
13862
|
account: exports_external.string(),
|
|
13839
13863
|
expiresAt: exports_external.number().optional()
|
|
@@ -14059,6 +14083,15 @@ class AuthBrokerClient {
|
|
|
14059
14083
|
const data = await this.send(req);
|
|
14060
14084
|
return data;
|
|
14061
14085
|
}
|
|
14086
|
+
async markThrottled(until) {
|
|
14087
|
+
const data = await this.send({
|
|
14088
|
+
v: PROTOCOL_VERSION,
|
|
14089
|
+
id: randomUUID(),
|
|
14090
|
+
op: "mark-throttled",
|
|
14091
|
+
until
|
|
14092
|
+
});
|
|
14093
|
+
return data;
|
|
14094
|
+
}
|
|
14062
14095
|
async claimNotification(key, windowMs) {
|
|
14063
14096
|
const data = await this.send({
|
|
14064
14097
|
v: PROTOCOL_VERSION,
|
|
@@ -14729,6 +14762,12 @@ var DEFAULT_MAX_QUOTA_DEFER_ATTEMPTS = 3;
|
|
|
14729
14762
|
function defaultQuotaDeferBackoffMs(attempt) {
|
|
14730
14763
|
return [60000, 180000, 300000][attempt] ?? 300000;
|
|
14731
14764
|
}
|
|
14765
|
+
function resolveQuotaDeferDelayMs(decision, fallbackMs, nowMs) {
|
|
14766
|
+
if (decision.retryAtMs === undefined)
|
|
14767
|
+
return fallbackMs;
|
|
14768
|
+
const target = decision.retryAtMs - nowMs + 2000;
|
|
14769
|
+
return Math.min(Math.max(target, 5000), 10 * 60000);
|
|
14770
|
+
}
|
|
14732
14771
|
function registerAgentSchedule(opts) {
|
|
14733
14772
|
const tasks = [];
|
|
14734
14773
|
const now = opts.now ?? Date.now;
|
|
@@ -14769,7 +14808,7 @@ function registerAgentSchedule(opts) {
|
|
|
14769
14808
|
handle = scheduleRetry(() => {
|
|
14770
14809
|
pendingRetries.delete(handle);
|
|
14771
14810
|
attemptFire(attempt + 1);
|
|
14772
|
-
}, backoff(attempt));
|
|
14811
|
+
}, resolveQuotaDeferDelayMs(decision, backoff(attempt), now()));
|
|
14773
14812
|
pendingRetries.add(handle);
|
|
14774
14813
|
}
|
|
14775
14814
|
return;
|
|
@@ -15124,10 +15163,10 @@ Briefly and plainly tell the user these scheduled runs did not ` + "happen so th
|
|
|
15124
15163
|
}
|
|
15125
15164
|
const cronLib = __require("node-cron");
|
|
15126
15165
|
const quotaPreflightEnabled = process.env.SWITCHROOM_DISABLE_CRON_QUOTA_PREFLIGHT !== "1";
|
|
15127
|
-
const quotaGate = quotaPreflightEnabled ? async () => {
|
|
15166
|
+
const quotaGate = quotaPreflightEnabled ? async (agent) => {
|
|
15128
15167
|
const client = new AuthBrokerClient;
|
|
15129
15168
|
try {
|
|
15130
|
-
return decideQuotaPreflight(await client.listState());
|
|
15169
|
+
return decideQuotaPreflight(await client.listState(), { agent });
|
|
15131
15170
|
} finally {
|
|
15132
15171
|
await client.close().catch(() => {});
|
|
15133
15172
|
}
|
|
@@ -15206,6 +15245,7 @@ if (import.meta.url === `file://${process.argv[1]}` && /(?:^|[/\\])agent-schedul
|
|
|
15206
15245
|
export {
|
|
15207
15246
|
scheduleSignature,
|
|
15208
15247
|
resolveReloadPollMs,
|
|
15248
|
+
resolveQuotaDeferDelayMs,
|
|
15209
15249
|
resolveEntryThreadId,
|
|
15210
15250
|
resolveChannelTarget,
|
|
15211
15251
|
registerAgentSchedule,
|
|
@@ -16816,7 +16816,6 @@ var TelegramChannelSchema = exports_external.object({
|
|
|
16816
16816
|
enabled: exports_external.boolean().default(true).describe("Master switch for the per-agent Telegram gateway sidecar. " + "When false, start.sh skips the gateway supervise loop and the " + "agent boots without bot-token requirements (smoke-test + " + "offline-dev use case)."),
|
|
16817
16817
|
plugin: exports_external.enum(["switchroom", "official"]).optional().describe("Which Telegram MCP plugin to load. Default is 'switchroom' — the " + "enhanced fork with streaming edits, reactions, history, and " + "access control. Set to 'official' for the upstream marketplace " + "plugin (basic send/receive only)."),
|
|
16818
16818
|
format: exports_external.enum(["html", "markdownv2", "text"]).optional().describe("Default reply format passed to the plugin"),
|
|
16819
|
-
rate_limit_ms: exports_external.number().optional().describe("Minimum delay between outgoing messages in ms"),
|
|
16820
16819
|
stream_mode: exports_external.enum(["pty", "checklist"]).optional().describe("How live progress is streamed to Telegram during a turn. " + "'pty' (default) surfaces text snapshots of Claude Code's TUI — " + "compatible but can flicker as Ink re-renders. 'checklist' drives " + "a structured progress card from session-tail events — stable " + "order, per-tool status emojis, fires only on semantic transitions."),
|
|
16821
16820
|
stream_throttle_ms: exports_external.number().int().nonnegative().optional().describe("Throttle window in ms between successive in-place stream edits " + "during a turn. Lower = more responsive stream, higher = fewer API " + "calls. Floored at 250 by draft-stream itself. Default 400 ms for DMs " + "and 1000 ms for groups/forums (respects Telegram's ~1 edit/sec/message " + "practical ceiling). Override per-agent if a particular agent needs " + "snappier or quieter streaming."),
|
|
16822
16821
|
clear_status_on_completion: exports_external.boolean().optional().describe("When true, the live activity/status feed (the in-place 'what it's " + "doing' message — Reading X, Searching the web for Y, …) is DELETED " + "when the turn's final answer lands, so only the reply remains. " + "Default false: the status message is left in the chat as a record " + "(its last step marked done) — no post-then-delete. Per-agent " + "override; cascades defaults → profile → agent (per-key)."),
|
|
@@ -19765,6 +19764,12 @@ var MarkExhaustedRequestSchema = exports_external.object({
|
|
|
19765
19764
|
id: exports_external.string().min(1),
|
|
19766
19765
|
until: exports_external.number().int().positive().optional()
|
|
19767
19766
|
});
|
|
19767
|
+
var MarkThrottledRequestSchema = exports_external.object({
|
|
19768
|
+
v: exports_external.literal(PROTOCOL_VERSION),
|
|
19769
|
+
op: exports_external.literal("mark-throttled"),
|
|
19770
|
+
id: exports_external.string().min(1),
|
|
19771
|
+
until: exports_external.number().int().positive()
|
|
19772
|
+
});
|
|
19768
19773
|
var RefreshAccountRequestSchema = exports_external.object({
|
|
19769
19774
|
v: exports_external.literal(PROTOCOL_VERSION),
|
|
19770
19775
|
op: exports_external.literal("refresh-account"),
|
|
@@ -19865,6 +19870,7 @@ var RequestSchema2 = exports_external.discriminatedUnion("op", [
|
|
|
19865
19870
|
ListStateRequestSchema,
|
|
19866
19871
|
SetActiveRequestSchema,
|
|
19867
19872
|
MarkExhaustedRequestSchema,
|
|
19873
|
+
MarkThrottledRequestSchema,
|
|
19868
19874
|
RefreshAccountRequestSchema,
|
|
19869
19875
|
AddAccountRequestSchema,
|
|
19870
19876
|
RmAccountRequestSchema,
|
|
@@ -19884,6 +19890,7 @@ var AccountStateSchema = exports_external.object({
|
|
|
19884
19890
|
expiresAt: exports_external.number().optional(),
|
|
19885
19891
|
exhausted: exports_external.boolean(),
|
|
19886
19892
|
exhausted_until: exports_external.number().optional(),
|
|
19893
|
+
throttled_until: exports_external.number().optional(),
|
|
19887
19894
|
threshold_violations: exports_external.number().int().nonnegative().optional(),
|
|
19888
19895
|
last_refreshed_at: exports_external.number().optional()
|
|
19889
19896
|
});
|
|
@@ -19914,6 +19921,12 @@ var MarkExhaustedDataSchema = exports_external.object({
|
|
|
19914
19921
|
rolled: exports_external.array(exports_external.string()),
|
|
19915
19922
|
rolledTo: exports_external.string().nullable().optional()
|
|
19916
19923
|
});
|
|
19924
|
+
var MarkThrottledDataSchema = exports_external.object({
|
|
19925
|
+
account: exports_external.string(),
|
|
19926
|
+
throttled_until: exports_external.number(),
|
|
19927
|
+
escalated: exports_external.boolean(),
|
|
19928
|
+
rolledTo: exports_external.string().nullable().optional()
|
|
19929
|
+
});
|
|
19917
19930
|
var RefreshAccountDataSchema = exports_external.object({
|
|
19918
19931
|
account: exports_external.string(),
|
|
19919
19932
|
expiresAt: exports_external.number().optional()
|
|
@@ -20015,6 +20028,10 @@ function encodeError(id, code, message) {
|
|
|
20015
20028
|
var AUTH_BROKER_ROOT = "/run/switchroom/auth-broker";
|
|
20016
20029
|
var REFRESH_TICK_INTERVAL_MS = 60 * 1000;
|
|
20017
20030
|
var MARK_EXHAUSTED_DEFAULT_MS = 5 * 60 * 60 * 1000;
|
|
20031
|
+
var MARK_THROTTLED_MAX_MS = 30 * 60 * 1000;
|
|
20032
|
+
var THROTTLE_ESCALATION_HITS = 3;
|
|
20033
|
+
var THROTTLE_ESCALATION_WINDOW_MS = 10 * 60 * 1000;
|
|
20034
|
+
var MARK_THROTTLED_MIN_INTERVAL_MS = 5 * 1000;
|
|
20018
20035
|
var AUDIT_ROTATE_BYTES = 10 * 1024 * 1024;
|
|
20019
20036
|
var AUDIT_KEEP = 5;
|
|
20020
20037
|
var AUDIT_LINE_MAX = 4000;
|
|
@@ -20430,6 +20447,9 @@ class AuthBroker {
|
|
|
20430
20447
|
case "mark-exhausted":
|
|
20431
20448
|
await this.opMarkExhausted(socket, reqId, identity, req.until);
|
|
20432
20449
|
break;
|
|
20450
|
+
case "mark-throttled":
|
|
20451
|
+
await this.opMarkThrottled(socket, reqId, identity, req.until);
|
|
20452
|
+
break;
|
|
20433
20453
|
case "refresh-account": {
|
|
20434
20454
|
const provider = req.provider ?? "anthropic";
|
|
20435
20455
|
if (!this.providers.has(provider)) {
|
|
@@ -20550,9 +20570,15 @@ class AuthBroker {
|
|
|
20550
20570
|
isOverageAllowed(account) {
|
|
20551
20571
|
return (this.config.auth?.allow_overage_accounts ?? []).includes(account);
|
|
20552
20572
|
}
|
|
20573
|
+
exhaustionMarkOf(account) {
|
|
20574
|
+
const q = this.quota[account];
|
|
20575
|
+
if (!q || q.exhausted_until === undefined)
|
|
20576
|
+
return;
|
|
20577
|
+
return { exhausted_until: q.exhausted_until, marked_at: q.marked_at };
|
|
20578
|
+
}
|
|
20553
20579
|
isAccountExhausted(account) {
|
|
20554
20580
|
return isAccountBlocked({
|
|
20555
|
-
mark: this.
|
|
20581
|
+
mark: this.exhaustionMarkOf(account),
|
|
20556
20582
|
snapshot: this.lastQuotaCache[account],
|
|
20557
20583
|
now: this.now(),
|
|
20558
20584
|
allowOverage: this.isOverageAllowed(account)
|
|
@@ -20605,7 +20631,7 @@ class AuthBroker {
|
|
|
20605
20631
|
if (!overageLiftsWall(snapshot, true))
|
|
20606
20632
|
return false;
|
|
20607
20633
|
return accountEligibility({
|
|
20608
|
-
mark: this.
|
|
20634
|
+
mark: this.exhaustionMarkOf(account),
|
|
20609
20635
|
snapshot,
|
|
20610
20636
|
now,
|
|
20611
20637
|
allowOverage: true
|
|
@@ -20615,7 +20641,7 @@ class AuthBroker {
|
|
|
20615
20641
|
const snapshot = this.lastQuotaCache[account];
|
|
20616
20642
|
const allowOverage = this.isOverageAllowed(account);
|
|
20617
20643
|
const verdict = accountEligibility({
|
|
20618
|
-
mark: this.
|
|
20644
|
+
mark: this.exhaustionMarkOf(account),
|
|
20619
20645
|
snapshot,
|
|
20620
20646
|
now: this.now(),
|
|
20621
20647
|
allowOverage
|
|
@@ -20733,6 +20759,7 @@ class AuthBroker {
|
|
|
20733
20759
|
expiresAt: creds?.claudeAiOauth?.expiresAt,
|
|
20734
20760
|
exhausted,
|
|
20735
20761
|
exhausted_until: q?.exhausted_until,
|
|
20762
|
+
throttled_until: q?.throttled_until,
|
|
20736
20763
|
threshold_violations: this.thresholdViolations[label] ?? 0,
|
|
20737
20764
|
last_refreshed_at: meta?.lastRefreshedAt,
|
|
20738
20765
|
last_quota: lq ?? null
|
|
@@ -20843,7 +20870,7 @@ class AuthBroker {
|
|
|
20843
20870
|
};
|
|
20844
20871
|
this.lastQuotaCache[label] = snapshot;
|
|
20845
20872
|
this.persistLastQuotaCache();
|
|
20846
|
-
if (snapshotShouldClearMark(snapshot, this.
|
|
20873
|
+
if (snapshotShouldClearMark(snapshot, this.exhaustionMarkOf(label), this.now())) {
|
|
20847
20874
|
delete this.quota[label];
|
|
20848
20875
|
this.persistQuota();
|
|
20849
20876
|
process.stdout.write(`auth-broker: live probe shows ${label} healthy (5h=${snapshot.fiveHourUtilizationPct}% 7d=${snapshot.sevenDayUtilizationPct}%) — cleared stale exhaustion mark
|
|
@@ -20961,7 +20988,7 @@ class AuthBroker {
|
|
|
20961
20988
|
const existing = this.quota[label]?.exhausted_until;
|
|
20962
20989
|
if (existing !== undefined && existing >= exhaustedUntil)
|
|
20963
20990
|
continue;
|
|
20964
|
-
this.quota[label] = { exhausted_until: exhaustedUntil, marked_at: now };
|
|
20991
|
+
this.quota[label] = { ...this.quota[label], exhausted_until: exhaustedUntil, marked_at: now };
|
|
20965
20992
|
this.persistQuota();
|
|
20966
20993
|
this.audit({ op: "mark-exhausted", identity: { kind: "operator" }, account: label, accountKind: "claude", ok: true });
|
|
20967
20994
|
process.stdout.write(`auth-broker: consumer-quota-sensor marked ${label} exhausted until ${new Date(exhaustedUntil).toISOString()} — consumer(s) fail over
|
|
@@ -21002,6 +21029,83 @@ class AuthBroker {
|
|
|
21002
21029
|
this.audit({ op: "mark-exhausted", identity, account, accountKind: "claude", ok: true });
|
|
21003
21030
|
socket.write(encodeSuccess(id, { account, rolled, rolledTo }));
|
|
21004
21031
|
}
|
|
21032
|
+
async opMarkThrottled(socket, id, identity, until) {
|
|
21033
|
+
const account = this.callerAccount(identity);
|
|
21034
|
+
if (!account) {
|
|
21035
|
+
this.audit({ op: "mark-throttled", identity, accountKind: "claude", ok: false, error: "no-active-account" });
|
|
21036
|
+
socket.write(encodeError(id, "ACCOUNT_NOT_FOUND", "no active account configured"));
|
|
21037
|
+
return;
|
|
21038
|
+
}
|
|
21039
|
+
const now = this.now();
|
|
21040
|
+
const throttledUntil = Math.min(Math.max(until, now + 1000), now + MARK_THROTTLED_MAX_MS);
|
|
21041
|
+
const entry = this.quota[account] ?? {};
|
|
21042
|
+
const priorHits = entry.throttle_hits ?? [];
|
|
21043
|
+
const lastHit = priorHits.length > 0 ? priorHits[priorHits.length - 1] : undefined;
|
|
21044
|
+
if (lastHit !== undefined && now - lastHit < MARK_THROTTLED_MIN_INTERVAL_MS) {
|
|
21045
|
+
const refreshed = Math.max(entry.throttled_until ?? 0, throttledUntil);
|
|
21046
|
+
this.quota[account] = { ...entry, throttled_until: refreshed };
|
|
21047
|
+
this.persistQuota();
|
|
21048
|
+
this.audit({ op: "mark-throttled", identity, account, accountKind: "claude", ok: true });
|
|
21049
|
+
process.stdout.write(`auth-broker: mark-throttled ${account} deduped (re-mark within ${MARK_THROTTLED_MIN_INTERVAL_MS / 1000}s) — expiry refreshed, no new hit
|
|
21050
|
+
`);
|
|
21051
|
+
socket.write(encodeSuccess(id, {
|
|
21052
|
+
account,
|
|
21053
|
+
throttled_until: refreshed,
|
|
21054
|
+
escalated: false,
|
|
21055
|
+
rolledTo: null
|
|
21056
|
+
}));
|
|
21057
|
+
return;
|
|
21058
|
+
}
|
|
21059
|
+
const hits = priorHits.filter((t) => now - t < THROTTLE_ESCALATION_WINDOW_MS);
|
|
21060
|
+
hits.push(now);
|
|
21061
|
+
const escalate = hits.length >= THROTTLE_ESCALATION_HITS;
|
|
21062
|
+
this.quota[account] = {
|
|
21063
|
+
...entry,
|
|
21064
|
+
throttled_until: throttledUntil,
|
|
21065
|
+
throttle_hits: escalate ? [] : hits
|
|
21066
|
+
};
|
|
21067
|
+
this.persistQuota();
|
|
21068
|
+
this.audit({ op: "mark-throttled", identity, account, accountKind: "claude", ok: true });
|
|
21069
|
+
process.stdout.write(`auth-broker: mark-throttled ${account} until ${new Date(throttledUntil).toISOString()} ` + `(hit ${hits.length}/${THROTTLE_ESCALATION_HITS} in window)
|
|
21070
|
+
`);
|
|
21071
|
+
let escalated = false;
|
|
21072
|
+
let rolledTo = null;
|
|
21073
|
+
if (escalate) {
|
|
21074
|
+
const probe = await this.probeThrottleEscalation(account);
|
|
21075
|
+
if (probe.exhausted) {
|
|
21076
|
+
escalated = true;
|
|
21077
|
+
process.stdout.write(`auth-broker: throttle-escalation probe corroborates wall on ${account} — mark-exhausted + roll
|
|
21078
|
+
`);
|
|
21079
|
+
this.audit({ op: "mark-exhausted", identity, account, accountKind: "claude", ok: true, reason: "throttle-escalation" });
|
|
21080
|
+
const roll = await this.markExhaustedAndRoll(account, probe.until ?? undefined, identity);
|
|
21081
|
+
rolledTo = roll.rolledTo;
|
|
21082
|
+
} else {
|
|
21083
|
+
process.stdout.write(`auth-broker: throttle-escalation probe on ${account} did NOT corroborate a wall — staying throttled
|
|
21084
|
+
`);
|
|
21085
|
+
}
|
|
21086
|
+
}
|
|
21087
|
+
socket.write(encodeSuccess(id, {
|
|
21088
|
+
account,
|
|
21089
|
+
throttled_until: throttledUntil,
|
|
21090
|
+
escalated,
|
|
21091
|
+
rolledTo: escalated ? rolledTo : null
|
|
21092
|
+
}));
|
|
21093
|
+
}
|
|
21094
|
+
async probeThrottleEscalation(account) {
|
|
21095
|
+
const creds = readAccountCredentials(account, this.home);
|
|
21096
|
+
const token = creds?.claudeAiOauth?.accessToken;
|
|
21097
|
+
if (!token)
|
|
21098
|
+
return { exhausted: false, until: null };
|
|
21099
|
+
let result;
|
|
21100
|
+
try {
|
|
21101
|
+
result = await this.fetchQuotaImpl({ accessToken: token });
|
|
21102
|
+
} catch (err) {
|
|
21103
|
+
this.logErr(`throttle-escalation probe ${account}: ${err.message}`);
|
|
21104
|
+
return { exhausted: false, until: null };
|
|
21105
|
+
}
|
|
21106
|
+
this.cacheQuotaSnapshot(account, result);
|
|
21107
|
+
return quotaIndicatesExhaustion(result, this.isOverageAllowed(account));
|
|
21108
|
+
}
|
|
21005
21109
|
async markExhaustedAndRoll(account, until, identity) {
|
|
21006
21110
|
const now = this.now();
|
|
21007
21111
|
const exhaustedUntil = clampMarkExpiry({
|
|
@@ -21010,7 +21114,7 @@ class AuthBroker {
|
|
|
21010
21114
|
shortMs: MARK_EXHAUSTED_DEFAULT_MS,
|
|
21011
21115
|
snapshot: this.lastQuotaCache[account]
|
|
21012
21116
|
});
|
|
21013
|
-
this.quota[account] = { exhausted_until: exhaustedUntil, marked_at: now };
|
|
21117
|
+
this.quota[account] = { ...this.quota[account], exhausted_until: exhaustedUntil, marked_at: now };
|
|
21014
21118
|
this.persistQuota();
|
|
21015
21119
|
const rolledTo = await this.nextHealthyAccountLive(account, this.config.auth?.fallback_order ?? []);
|
|
21016
21120
|
const rolled = this.fanoutFailoverTo(account, rolledTo);
|
|
@@ -4741,6 +4741,12 @@ var MarkExhaustedRequestSchema = exports_external.object({
|
|
|
4741
4741
|
id: exports_external.string().min(1),
|
|
4742
4742
|
until: exports_external.number().int().positive().optional()
|
|
4743
4743
|
});
|
|
4744
|
+
var MarkThrottledRequestSchema = exports_external.object({
|
|
4745
|
+
v: exports_external.literal(PROTOCOL_VERSION),
|
|
4746
|
+
op: exports_external.literal("mark-throttled"),
|
|
4747
|
+
id: exports_external.string().min(1),
|
|
4748
|
+
until: exports_external.number().int().positive()
|
|
4749
|
+
});
|
|
4744
4750
|
var RefreshAccountRequestSchema = exports_external.object({
|
|
4745
4751
|
v: exports_external.literal(PROTOCOL_VERSION),
|
|
4746
4752
|
op: exports_external.literal("refresh-account"),
|
|
@@ -4841,6 +4847,7 @@ var RequestSchema = exports_external.discriminatedUnion("op", [
|
|
|
4841
4847
|
ListStateRequestSchema,
|
|
4842
4848
|
SetActiveRequestSchema,
|
|
4843
4849
|
MarkExhaustedRequestSchema,
|
|
4850
|
+
MarkThrottledRequestSchema,
|
|
4844
4851
|
RefreshAccountRequestSchema,
|
|
4845
4852
|
AddAccountRequestSchema,
|
|
4846
4853
|
RmAccountRequestSchema,
|
|
@@ -4860,6 +4867,7 @@ var AccountStateSchema = exports_external.object({
|
|
|
4860
4867
|
expiresAt: exports_external.number().optional(),
|
|
4861
4868
|
exhausted: exports_external.boolean(),
|
|
4862
4869
|
exhausted_until: exports_external.number().optional(),
|
|
4870
|
+
throttled_until: exports_external.number().optional(),
|
|
4863
4871
|
threshold_violations: exports_external.number().int().nonnegative().optional(),
|
|
4864
4872
|
last_refreshed_at: exports_external.number().optional()
|
|
4865
4873
|
});
|
|
@@ -4890,6 +4898,12 @@ var MarkExhaustedDataSchema = exports_external.object({
|
|
|
4890
4898
|
rolled: exports_external.array(exports_external.string()),
|
|
4891
4899
|
rolledTo: exports_external.string().nullable().optional()
|
|
4892
4900
|
});
|
|
4901
|
+
var MarkThrottledDataSchema = exports_external.object({
|
|
4902
|
+
account: exports_external.string(),
|
|
4903
|
+
throttled_until: exports_external.number(),
|
|
4904
|
+
escalated: exports_external.boolean(),
|
|
4905
|
+
rolledTo: exports_external.string().nullable().optional()
|
|
4906
|
+
});
|
|
4893
4907
|
var RefreshAccountDataSchema = exports_external.object({
|
|
4894
4908
|
account: exports_external.string(),
|
|
4895
4909
|
expiresAt: exports_external.number().optional()
|
|
@@ -5115,6 +5129,15 @@ class AuthBrokerClient {
|
|
|
5115
5129
|
const data = await this.send(req);
|
|
5116
5130
|
return data;
|
|
5117
5131
|
}
|
|
5132
|
+
async markThrottled(until) {
|
|
5133
|
+
const data = await this.send({
|
|
5134
|
+
v: PROTOCOL_VERSION,
|
|
5135
|
+
id: randomUUID2(),
|
|
5136
|
+
op: "mark-throttled",
|
|
5137
|
+
until
|
|
5138
|
+
});
|
|
5139
|
+
return data;
|
|
5140
|
+
}
|
|
5118
5141
|
async claimNotification(key, windowMs) {
|
|
5119
5142
|
const data = await this.send({
|
|
5120
5143
|
v: PROTOCOL_VERSION,
|
|
@@ -4000,7 +4000,7 @@ function decodeResponse(line) {
|
|
|
4000
4000
|
}
|
|
4001
4001
|
return ResponseSchema.parse(parsed);
|
|
4002
4002
|
}
|
|
4003
|
-
var MAX_FRAME_BYTES, PROTOCOL_VERSION = 1, ProviderNameSchema, GetCredentialsRequestSchema, ListStateRequestSchema, SetActiveRequestSchema, MarkExhaustedRequestSchema, RefreshAccountRequestSchema, AnthropicCredentialsSchema, GoogleCredentialsSchema, MicrosoftCredentialsSchema, ProviderCredentialsSchema, AddAccountRequestSchema, RmAccountRequestSchema, SetOverrideRequestSchema, ListGoogleAccountsRequestSchema, ListMicrosoftAccountsRequestSchema, ProbeQuotaRequestSchema, ClaimNotificationRequestSchema, RequestSchema, GetCredentialsDataSchema, AccountStateSchema, AgentStateSchema, ConsumerStateSchema, ListStateDataSchema, SetActiveDataSchema, MarkExhaustedDataSchema, RefreshAccountDataSchema, AddAccountDataSchema, RmAccountDataSchema, SetOverrideDataSchema, ClaimNotificationDataSchema, GoogleAccountStateSchema, ListGoogleAccountsDataSchema, MicrosoftAccountStateSchema, ListMicrosoftAccountsDataSchema, ErrorBodySchema, SuccessResponseSchema, ErrorResponseSchema, ResponseSchema;
|
|
4003
|
+
var MAX_FRAME_BYTES, PROTOCOL_VERSION = 1, ProviderNameSchema, GetCredentialsRequestSchema, ListStateRequestSchema, SetActiveRequestSchema, MarkExhaustedRequestSchema, MarkThrottledRequestSchema, RefreshAccountRequestSchema, AnthropicCredentialsSchema, GoogleCredentialsSchema, MicrosoftCredentialsSchema, ProviderCredentialsSchema, AddAccountRequestSchema, RmAccountRequestSchema, SetOverrideRequestSchema, ListGoogleAccountsRequestSchema, ListMicrosoftAccountsRequestSchema, ProbeQuotaRequestSchema, ClaimNotificationRequestSchema, RequestSchema, GetCredentialsDataSchema, AccountStateSchema, AgentStateSchema, ConsumerStateSchema, ListStateDataSchema, SetActiveDataSchema, MarkExhaustedDataSchema, MarkThrottledDataSchema, RefreshAccountDataSchema, AddAccountDataSchema, RmAccountDataSchema, SetOverrideDataSchema, ClaimNotificationDataSchema, GoogleAccountStateSchema, ListGoogleAccountsDataSchema, MicrosoftAccountStateSchema, ListMicrosoftAccountsDataSchema, ErrorBodySchema, SuccessResponseSchema, ErrorResponseSchema, ResponseSchema;
|
|
4004
4004
|
var init_protocol = __esm(() => {
|
|
4005
4005
|
init_zod();
|
|
4006
4006
|
MAX_FRAME_BYTES = 64 * 1024;
|
|
@@ -4029,6 +4029,12 @@ var init_protocol = __esm(() => {
|
|
|
4029
4029
|
id: exports_external.string().min(1),
|
|
4030
4030
|
until: exports_external.number().int().positive().optional()
|
|
4031
4031
|
});
|
|
4032
|
+
MarkThrottledRequestSchema = exports_external.object({
|
|
4033
|
+
v: exports_external.literal(PROTOCOL_VERSION),
|
|
4034
|
+
op: exports_external.literal("mark-throttled"),
|
|
4035
|
+
id: exports_external.string().min(1),
|
|
4036
|
+
until: exports_external.number().int().positive()
|
|
4037
|
+
});
|
|
4032
4038
|
RefreshAccountRequestSchema = exports_external.object({
|
|
4033
4039
|
v: exports_external.literal(PROTOCOL_VERSION),
|
|
4034
4040
|
op: exports_external.literal("refresh-account"),
|
|
@@ -4129,6 +4135,7 @@ var init_protocol = __esm(() => {
|
|
|
4129
4135
|
ListStateRequestSchema,
|
|
4130
4136
|
SetActiveRequestSchema,
|
|
4131
4137
|
MarkExhaustedRequestSchema,
|
|
4138
|
+
MarkThrottledRequestSchema,
|
|
4132
4139
|
RefreshAccountRequestSchema,
|
|
4133
4140
|
AddAccountRequestSchema,
|
|
4134
4141
|
RmAccountRequestSchema,
|
|
@@ -4148,6 +4155,7 @@ var init_protocol = __esm(() => {
|
|
|
4148
4155
|
expiresAt: exports_external.number().optional(),
|
|
4149
4156
|
exhausted: exports_external.boolean(),
|
|
4150
4157
|
exhausted_until: exports_external.number().optional(),
|
|
4158
|
+
throttled_until: exports_external.number().optional(),
|
|
4151
4159
|
threshold_violations: exports_external.number().int().nonnegative().optional(),
|
|
4152
4160
|
last_refreshed_at: exports_external.number().optional()
|
|
4153
4161
|
});
|
|
@@ -4178,6 +4186,12 @@ var init_protocol = __esm(() => {
|
|
|
4178
4186
|
rolled: exports_external.array(exports_external.string()),
|
|
4179
4187
|
rolledTo: exports_external.string().nullable().optional()
|
|
4180
4188
|
});
|
|
4189
|
+
MarkThrottledDataSchema = exports_external.object({
|
|
4190
|
+
account: exports_external.string(),
|
|
4191
|
+
throttled_until: exports_external.number(),
|
|
4192
|
+
escalated: exports_external.boolean(),
|
|
4193
|
+
rolledTo: exports_external.string().nullable().optional()
|
|
4194
|
+
});
|
|
4181
4195
|
RefreshAccountDataSchema = exports_external.object({
|
|
4182
4196
|
account: exports_external.string(),
|
|
4183
4197
|
expiresAt: exports_external.number().optional()
|
|
@@ -4376,6 +4390,15 @@ class AuthBrokerClient {
|
|
|
4376
4390
|
const data = await this.send(req);
|
|
4377
4391
|
return data;
|
|
4378
4392
|
}
|
|
4393
|
+
async markThrottled(until) {
|
|
4394
|
+
const data = await this.send({
|
|
4395
|
+
v: PROTOCOL_VERSION,
|
|
4396
|
+
id: randomUUID(),
|
|
4397
|
+
op: "mark-throttled",
|
|
4398
|
+
until
|
|
4399
|
+
});
|
|
4400
|
+
return data;
|
|
4401
|
+
}
|
|
4379
4402
|
async claimNotification(key, windowMs) {
|
|
4380
4403
|
const data = await this.send({
|
|
4381
4404
|
v: PROTOCOL_VERSION,
|