switchroom 0.18.15 → 0.18.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +16 -0
- package/dist/auth-broker/index.js +445 -10
- package/dist/cli/notion-write-pretool.mjs +16 -0
- package/dist/cli/switchroom.js +654 -479
- package/dist/host-control/main.js +20 -1
- package/dist/vault/approvals/kernel-server.js +16 -0
- package/dist/vault/broker/server.js +16 -0
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +81 -139
- package/telegram-plugin/bridge/bridge.ts +7 -1
- package/telegram-plugin/dist/bridge/bridge.js +26 -1
- package/telegram-plugin/dist/gateway/gateway.js +1758 -661
- package/telegram-plugin/dist/server.js +26 -1
- package/telegram-plugin/draft-stream.ts +78 -3
- package/telegram-plugin/fleet-fallback-resume.ts +26 -3
- package/telegram-plugin/gateway/approval-hold.ts +49 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +64 -22
- package/telegram-plugin/gateway/effort-command.ts +9 -7
- package/telegram-plugin/gateway/gateway.ts +627 -291
- package/telegram-plugin/gateway/linear-activity.ts +20 -4
- package/telegram-plugin/gateway/litellm-local-notice-wiring.ts +200 -0
- package/telegram-plugin/gateway/model-command.ts +96 -18
- package/telegram-plugin/gateway/pending-session-command.ts +10 -8
- package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
- package/telegram-plugin/gateway/session-model-file.ts +141 -172
- package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
- package/telegram-plugin/litellm-local-notice.ts +189 -0
- package/telegram-plugin/llm-error-present.ts +436 -0
- package/telegram-plugin/operator-events.ts +7 -1
- package/telegram-plugin/permission-title.ts +172 -10
- package/telegram-plugin/premium-recovery.ts +101 -0
- package/telegram-plugin/quota-watch.ts +16 -4
- package/telegram-plugin/raw-error-scrub.ts +73 -0
- package/telegram-plugin/retry-api-call.ts +8 -2
- package/telegram-plugin/runtime-metrics.ts +16 -0
- package/telegram-plugin/send-gate-degraded.test.ts +161 -8
- package/telegram-plugin/send-gate-observability.test.ts +140 -0
- package/telegram-plugin/send-gate-observability.ts +65 -20
- package/telegram-plugin/send-gate.test.ts +143 -1
- package/telegram-plugin/send-gate.ts +246 -23
- package/telegram-plugin/session-tail.ts +16 -0
- package/telegram-plugin/shared/local-time.ts +69 -0
- package/telegram-plugin/stream-controller.ts +143 -20
- package/telegram-plugin/stream-reply-handler.ts +12 -2
- package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
- package/telegram-plugin/tests/bot-api.harness.ts +7 -2
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
- package/telegram-plugin/tests/draft-stream.test.ts +110 -1
- package/telegram-plugin/tests/effort-command.test.ts +4 -4
- package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +5 -4
- package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +33 -19
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +47 -127
- package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
- package/telegram-plugin/tests/litellm-local-notice.test.ts +417 -0
- package/telegram-plugin/tests/llm-error-present.test.ts +380 -0
- package/telegram-plugin/tests/model-command.test.ts +84 -1
- package/telegram-plugin/tests/permission-title.test.ts +167 -4
- package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
- package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
- package/telegram-plugin/tests/quota-watch.test.ts +21 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +8 -3
- package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
- package/telegram-plugin/tests/session-model-file.test.ts +7 -155
- package/telegram-plugin/tests/stream-controller-send-gate.test.ts +521 -0
- package/telegram-plugin/tests/stream-reply-handler.test.ts +44 -0
- package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
- package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
- package/telegram-plugin/tests/worker-activity-feed.test.ts +212 -2
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
- package/telegram-plugin/tier-downgrade.ts +198 -0
- package/telegram-plugin/tool-activity-summary.ts +99 -0
- package/telegram-plugin/worker-activity-feed.ts +543 -368
|
@@ -11199,6 +11199,19 @@ var TelegramChannelSchema = exports_external.object({
|
|
|
11199
11199
|
approval_timeout_minutes: exports_external.number().int().nonnegative().optional().describe("Operator approval-card lifetime (minutes) for the tool-use 'Allow " + "once' card and the vault grant decision wait. After this long with " + "no operator tap, the card auto-denies (a TIMEOUT, not a denial — the " + "agent is told not to retry). Default 60. hostd-gated verbs " + "(mcp__hostd__*) keep their own longer window; the hostd " + "config-propose card is not governed by this key."),
|
|
11200
11200
|
sub_agent_tick_interval_ms: exports_external.number().int().nonnegative().optional().describe("Heartbeat tick interval (ms) for sub-agent rendering. Forces a " + "re-render of the elapsed-time counter while sub-agents are running, " + "even during silent stretches between tool calls. Default 10000 (10 s). " + "Set to 0 to disable the elapsed-ticker path."),
|
|
11201
11201
|
edit_budget_threshold: exports_external.number().int().nonnegative().optional().describe("Telegram API edit budget per minute before the progress-card driver " + "falls back to a slower coalesce window. When a chat accumulates more " + "than this many card edits in the trailing 60 s, the driver switches " + "to a wider coalesce interval until the rate drops back. Default 18. " + "Increase if your gateway frequently bumps the Telegram edit-rate ceiling " + "with many parallel sub-agents; decrease for a more conservative buffer."),
|
|
11202
|
+
send_gate: exports_external.object({
|
|
11203
|
+
enabled: exports_external.boolean().optional().describe("Master switch for the deterministic outbound send gate " + "(telegram-plugin/send-gate.ts) — the token-bucket scheduler every " + "Bot API call transits so per-surface throttles can't add up past a " + "flood ceiling. ON by default. Precedence: the operator break-glass " + "env var SWITCHROOM_TELEGRAM_SEND_GATE (0/false/off/no) ALWAYS wins " + "when explicitly set; this key only decides when that env var is " + "unset. Omit to keep the gate on."),
|
|
11204
|
+
global_per_sec: exports_external.number().positive().optional().describe("Global bucket sustained rate (Bot API calls/sec across ALL chats). " + "Default 25 (headroom under Telegram's ~30/s). Must be > 0 — a zero " + "or negative rate would wedge all outbound sends. Omit to keep 25."),
|
|
11205
|
+
global_burst: exports_external.number().int().positive().describe("Global bucket burst capacity. Default 4; worst-case 1s window " + "admits global_burst + global_per_sec = 29 < 30, a real margin under " + "the ceiling. Must be an integer >= 1 (a 0 capacity never admits a " + "token and wedges sends). Omit to keep 4.").optional(),
|
|
11206
|
+
per_chat_per_sec: exports_external.number().positive().optional().describe("Per-chat sustained rate (calls/sec to a single chat). Default 1. " + "Must be > 0 (zero/negative wedges that chat). Omit to keep 1."),
|
|
11207
|
+
per_chat_burst: exports_external.number().int().positive().optional().describe("Per-chat burst capacity. Default 3. Must be an integer >= 1 " + "(0 wedges the chat's bucket). Omit to keep 3."),
|
|
11208
|
+
per_group_per_min: exports_external.number().positive().optional().describe("Per-group sustained rate (calls/min to a single group/supergroup). " + "Default 18 (headroom under Telegram's ~20/min group ceiling). Must " + "be > 0. Omit to keep 18."),
|
|
11209
|
+
per_group_burst: exports_external.number().int().positive().optional().describe("Per-group burst capacity. Default 2. Must be an integer >= 1 " + "(0 wedges the group's bucket). Omit to keep 2."),
|
|
11210
|
+
edit_floor_ms: exports_external.number().int().nonnegative().optional().describe("Minimum ms between successive edits of the SAME message_id " + "(last-write-wins coalescing enforces this floor). Default 1500 " + "(Telegram's ~1 edit/sec/message practical ceiling). 0 disables the " + "floor. Must be an integer >= 0. Omit to keep 1500.")
|
|
11211
|
+
}).optional().describe("Tunable rate limits for the deterministic outbound send gate " + "(telegram-plugin/send-gate.ts). Every key is optional and defaults to " + "the send gate's built-in value, so omitting the whole block reproduces " + "today's exact behaviour — this is pure operator tuning, no default is " + "changed. Cascades from defaults.channels.telegram.send_gate."),
|
|
11212
|
+
worker_feed: exports_external.object({
|
|
11213
|
+
max_rows: exports_external.number().int().positive().optional().describe("Max live-worker rows rendered in the COMBINED worker-activity feed " + "(2+ background workers in one chat/thread coalesce into ONE message; " + "telegram-plugin/worker-activity-feed.ts) before a compact " + "'+M more working…' spill line — keeps the coalesced body compact and " + "legible (and under the rich-message wire ceiling). Default 8. A " + "single-worker chat renders the full \uD83D\uDEE0 Worker card and ignores this. " + "Must be an integer >= 1. Omit to keep 8.")
|
|
11214
|
+
}).optional().describe("Tuning for the coalesced worker-activity feed " + "(telegram-plugin/worker-activity-feed.ts). Cascades from " + "defaults.channels.telegram.worker_feed."),
|
|
11202
11215
|
stickers: exports_external.record(exports_external.string(), exports_external.string()).optional().describe("Sticker aliases for the `send_sticker` MCP tool (#576). Maps a " + "short alias name (e.g. 'happy', 'thinking') to a Telegram file_id. " + "Operator-curated — capture file_ids from inbound stickers the user " + "sends and add them here. The agent calls send_sticker(chat_id, " + "alias='happy') and the gateway resolves to the file_id at send " + "time. Aliases enable persona-flavored expressiveness without " + "exposing raw file_ids in the agent prompt. Personal-assistant / " + "health-coach personas benefit; coding agents typically don't " + "configure any."),
|
|
11203
11216
|
voice_in: exports_external.object({
|
|
11204
11217
|
enabled: exports_external.boolean().optional().describe("Master switch for voice-message transcription."),
|
|
@@ -11224,6 +11237,9 @@ var TelegramChannelSchema = exports_external.object({
|
|
|
11224
11237
|
window_ms: exports_external.number().int().nonnegative().optional().describe("Sliding-window (ms) for merging consecutive inbound messages from " + "the same sender+topic into ONE Claude turn. Each new message resets " + "the timer; the turn starts once the sender pauses for this long. " + "Catches forwarded bursts, pasted text the Telegram client split " + "into several messages, and mixed text+media forwards. Default 500. " + "Set 0 to disable (every message becomes its own turn). Raise for " + "users who think in multiple short messages; the trade-off is the " + "single-message turn start is delayed by this much (the \uD83D\uDC40 ack still " + "fires immediately, so perceived latency is unchanged)."),
|
|
11225
11238
|
max_attachments: exports_external.number().int().positive().optional().describe("Maximum number of media attachments carried into ONE coalesced " + "Claude turn. Default 10 — a full Telegram album (media_group caps " + "at 10) or a text+multi-image forwarded burst arrives as a single " + "turn; the agent sees numbered attachment fields (image_path, " + "image_path_2, …). Set 1 to restore the historical " + "single-attachment-per-turn behaviour. Excess attachments beyond " + "the cap spill into the next turn. Each attachment is downloaded, " + "so a high cap on a slow link delays turn start.")
|
|
11226
11239
|
}).optional().describe("Inbound coalescing — how the gateway groups rapid consecutive messages " + "into a single turn so a forwarded album or split paste doesn't fan out " + "into N separate turns. Cascades from defaults.channels.telegram.coalesce."),
|
|
11240
|
+
litellm_notice: exports_external.object({
|
|
11241
|
+
window_ms: exports_external.number().int().positive().optional().describe("Per-agent cooldown window (ms) for the litellm-local 429 notice. " + "When the agent trips the LiteLLM proxy's OWN tpm_limit/rpm_limit " + "cap (a `litellm-local` classified 429 — see docs/auth.md § " + "LiteLLM-proxy-local 429s), the gateway posts ONE calm notice " + "naming the fleet token limiter, then counts further hits silently " + "for this long; the first notice after the window expires says " + "how many were absorbed. Default 900000 (15 min). Invalid values " + "fall back to the default.")
|
|
11242
|
+
}).optional().describe("Debounce tuning for the litellm-local throttle notice — the calm " + "'fleet token limiter engaged' message posted when the LiteLLM " + "proxy's own rate cap trips (never an Anthropic account limit). " + "Cascades from defaults.channels.telegram.litellm_notice."),
|
|
11227
11243
|
interrupt: exports_external.object({
|
|
11228
11244
|
safe_boundary: exports_external.boolean().optional().describe("When true (the default), a `!`-prefix interrupt that arrives while " + "the agent is mid-tool-call is DEFERRED: the SIGINT and the " + "replacement turn wait until the in-flight tool call finishes (a " + "clean boundary) instead of C-c'ing the agent mid-write/mid-bash. If " + "no tool is in flight the interrupt still fires immediately. Bounded " + "by max_wait_ms so a long tool never strands the user. Set false to " + "fire synchronously the moment `!` is received (historical " + "behaviour). Rapid repeated `!` while one is pending coalesce into a " + "single deferred interrupt carrying the latest body."),
|
|
11229
11245
|
max_wait_ms: exports_external.number().int().positive().optional().describe("Upper bound (ms) the gateway waits for a safe boundary before firing " + "a deferred `!` interrupt anyway. Only consulted when safe_boundary is " + "true. Default 8000. Keep it short — the user explicitly asked to " + "interrupt, so a long in-flight tool shouldn't ghost them; the cap " + "trades a tiny risk of a mid-tool C-c for a guaranteed response.")
|
|
@@ -16828,6 +16828,19 @@ var TelegramChannelSchema = exports_external.object({
|
|
|
16828
16828
|
approval_timeout_minutes: exports_external.number().int().nonnegative().optional().describe("Operator approval-card lifetime (minutes) for the tool-use 'Allow " + "once' card and the vault grant decision wait. After this long with " + "no operator tap, the card auto-denies (a TIMEOUT, not a denial — the " + "agent is told not to retry). Default 60. hostd-gated verbs " + "(mcp__hostd__*) keep their own longer window; the hostd " + "config-propose card is not governed by this key."),
|
|
16829
16829
|
sub_agent_tick_interval_ms: exports_external.number().int().nonnegative().optional().describe("Heartbeat tick interval (ms) for sub-agent rendering. Forces a " + "re-render of the elapsed-time counter while sub-agents are running, " + "even during silent stretches between tool calls. Default 10000 (10 s). " + "Set to 0 to disable the elapsed-ticker path."),
|
|
16830
16830
|
edit_budget_threshold: exports_external.number().int().nonnegative().optional().describe("Telegram API edit budget per minute before the progress-card driver " + "falls back to a slower coalesce window. When a chat accumulates more " + "than this many card edits in the trailing 60 s, the driver switches " + "to a wider coalesce interval until the rate drops back. Default 18. " + "Increase if your gateway frequently bumps the Telegram edit-rate ceiling " + "with many parallel sub-agents; decrease for a more conservative buffer."),
|
|
16831
|
+
send_gate: exports_external.object({
|
|
16832
|
+
enabled: exports_external.boolean().optional().describe("Master switch for the deterministic outbound send gate " + "(telegram-plugin/send-gate.ts) — the token-bucket scheduler every " + "Bot API call transits so per-surface throttles can't add up past a " + "flood ceiling. ON by default. Precedence: the operator break-glass " + "env var SWITCHROOM_TELEGRAM_SEND_GATE (0/false/off/no) ALWAYS wins " + "when explicitly set; this key only decides when that env var is " + "unset. Omit to keep the gate on."),
|
|
16833
|
+
global_per_sec: exports_external.number().positive().optional().describe("Global bucket sustained rate (Bot API calls/sec across ALL chats). " + "Default 25 (headroom under Telegram's ~30/s). Must be > 0 — a zero " + "or negative rate would wedge all outbound sends. Omit to keep 25."),
|
|
16834
|
+
global_burst: exports_external.number().int().positive().describe("Global bucket burst capacity. Default 4; worst-case 1s window " + "admits global_burst + global_per_sec = 29 < 30, a real margin under " + "the ceiling. Must be an integer >= 1 (a 0 capacity never admits a " + "token and wedges sends). Omit to keep 4.").optional(),
|
|
16835
|
+
per_chat_per_sec: exports_external.number().positive().optional().describe("Per-chat sustained rate (calls/sec to a single chat). Default 1. " + "Must be > 0 (zero/negative wedges that chat). Omit to keep 1."),
|
|
16836
|
+
per_chat_burst: exports_external.number().int().positive().optional().describe("Per-chat burst capacity. Default 3. Must be an integer >= 1 " + "(0 wedges the chat's bucket). Omit to keep 3."),
|
|
16837
|
+
per_group_per_min: exports_external.number().positive().optional().describe("Per-group sustained rate (calls/min to a single group/supergroup). " + "Default 18 (headroom under Telegram's ~20/min group ceiling). Must " + "be > 0. Omit to keep 18."),
|
|
16838
|
+
per_group_burst: exports_external.number().int().positive().optional().describe("Per-group burst capacity. Default 2. Must be an integer >= 1 " + "(0 wedges the group's bucket). Omit to keep 2."),
|
|
16839
|
+
edit_floor_ms: exports_external.number().int().nonnegative().optional().describe("Minimum ms between successive edits of the SAME message_id " + "(last-write-wins coalescing enforces this floor). Default 1500 " + "(Telegram's ~1 edit/sec/message practical ceiling). 0 disables the " + "floor. Must be an integer >= 0. Omit to keep 1500.")
|
|
16840
|
+
}).optional().describe("Tunable rate limits for the deterministic outbound send gate " + "(telegram-plugin/send-gate.ts). Every key is optional and defaults to " + "the send gate's built-in value, so omitting the whole block reproduces " + "today's exact behaviour — this is pure operator tuning, no default is " + "changed. Cascades from defaults.channels.telegram.send_gate."),
|
|
16841
|
+
worker_feed: exports_external.object({
|
|
16842
|
+
max_rows: exports_external.number().int().positive().optional().describe("Max live-worker rows rendered in the COMBINED worker-activity feed " + "(2+ background workers in one chat/thread coalesce into ONE message; " + "telegram-plugin/worker-activity-feed.ts) before a compact " + "'+M more working…' spill line — keeps the coalesced body compact and " + "legible (and under the rich-message wire ceiling). Default 8. A " + "single-worker chat renders the full \uD83D\uDEE0 Worker card and ignores this. " + "Must be an integer >= 1. Omit to keep 8.")
|
|
16843
|
+
}).optional().describe("Tuning for the coalesced worker-activity feed " + "(telegram-plugin/worker-activity-feed.ts). Cascades from " + "defaults.channels.telegram.worker_feed."),
|
|
16831
16844
|
stickers: exports_external.record(exports_external.string(), exports_external.string()).optional().describe("Sticker aliases for the `send_sticker` MCP tool (#576). Maps a " + "short alias name (e.g. 'happy', 'thinking') to a Telegram file_id. " + "Operator-curated — capture file_ids from inbound stickers the user " + "sends and add them here. The agent calls send_sticker(chat_id, " + "alias='happy') and the gateway resolves to the file_id at send " + "time. Aliases enable persona-flavored expressiveness without " + "exposing raw file_ids in the agent prompt. Personal-assistant / " + "health-coach personas benefit; coding agents typically don't " + "configure any."),
|
|
16832
16845
|
voice_in: exports_external.object({
|
|
16833
16846
|
enabled: exports_external.boolean().optional().describe("Master switch for voice-message transcription."),
|
|
@@ -16853,6 +16866,9 @@ var TelegramChannelSchema = exports_external.object({
|
|
|
16853
16866
|
window_ms: exports_external.number().int().nonnegative().optional().describe("Sliding-window (ms) for merging consecutive inbound messages from " + "the same sender+topic into ONE Claude turn. Each new message resets " + "the timer; the turn starts once the sender pauses for this long. " + "Catches forwarded bursts, pasted text the Telegram client split " + "into several messages, and mixed text+media forwards. Default 500. " + "Set 0 to disable (every message becomes its own turn). Raise for " + "users who think in multiple short messages; the trade-off is the " + "single-message turn start is delayed by this much (the \uD83D\uDC40 ack still " + "fires immediately, so perceived latency is unchanged)."),
|
|
16854
16867
|
max_attachments: exports_external.number().int().positive().optional().describe("Maximum number of media attachments carried into ONE coalesced " + "Claude turn. Default 10 — a full Telegram album (media_group caps " + "at 10) or a text+multi-image forwarded burst arrives as a single " + "turn; the agent sees numbered attachment fields (image_path, " + "image_path_2, …). Set 1 to restore the historical " + "single-attachment-per-turn behaviour. Excess attachments beyond " + "the cap spill into the next turn. Each attachment is downloaded, " + "so a high cap on a slow link delays turn start.")
|
|
16855
16868
|
}).optional().describe("Inbound coalescing — how the gateway groups rapid consecutive messages " + "into a single turn so a forwarded album or split paste doesn't fan out " + "into N separate turns. Cascades from defaults.channels.telegram.coalesce."),
|
|
16869
|
+
litellm_notice: exports_external.object({
|
|
16870
|
+
window_ms: exports_external.number().int().positive().optional().describe("Per-agent cooldown window (ms) for the litellm-local 429 notice. " + "When the agent trips the LiteLLM proxy's OWN tpm_limit/rpm_limit " + "cap (a `litellm-local` classified 429 — see docs/auth.md § " + "LiteLLM-proxy-local 429s), the gateway posts ONE calm notice " + "naming the fleet token limiter, then counts further hits silently " + "for this long; the first notice after the window expires says " + "how many were absorbed. Default 900000 (15 min). Invalid values " + "fall back to the default.")
|
|
16871
|
+
}).optional().describe("Debounce tuning for the litellm-local throttle notice — the calm " + "'fleet token limiter engaged' message posted when the LiteLLM " + "proxy's own rate cap trips (never an Anthropic account limit). " + "Cascades from defaults.channels.telegram.litellm_notice."),
|
|
16856
16872
|
interrupt: exports_external.object({
|
|
16857
16873
|
safe_boundary: exports_external.boolean().optional().describe("When true (the default), a `!`-prefix interrupt that arrives while " + "the agent is mid-tool-call is DEFERRED: the SIGINT and the " + "replacement turn wait until the in-flight tool call finishes (a " + "clean boundary) instead of C-c'ing the agent mid-write/mid-bash. If " + "no tool is in flight the interrupt still fires immediately. Bounded " + "by max_wait_ms so a long tool never strands the user. Set false to " + "fire synchronously the moment `!` is received (historical " + "behaviour). Rapid repeated `!` while one is pending coalesce into a " + "single deferred interrupt carrying the latest body."),
|
|
16858
16874
|
max_wait_ms: exports_external.number().int().positive().optional().describe("Upper bound (ms) the gateway waits for a safe boundary before firing " + "a deferred `!` interrupt anyway. Only consulted when safe_boundary is " + "true. Default 8000. Keep it short — the user explicitly asked to " + "interrupt, so a long in-flight tool shouldn't ghost them; the cap " + "trades a tiny risk of a mid-tool C-c for a guaranteed response.")
|
|
@@ -18653,6 +18669,14 @@ var BIND_MOUNT_EXACT_SOURCE_DENY = new Set(["/var/run/docker.sock"]);
|
|
|
18653
18669
|
var OAUTH_BETA = "oauth-2025-04-20";
|
|
18654
18670
|
var DEFAULT_USER_AGENT = "claude-cli/1.0.0 (external, cli)";
|
|
18655
18671
|
var DEFAULT_PROBE_MODEL = "claude-haiku-4-5-20251001";
|
|
18672
|
+
var DEFAULT_MODEL_TIER_PROBE_MODEL = "claude-fable-5";
|
|
18673
|
+
function resolveModelTierProbeModel(env = process.env) {
|
|
18674
|
+
const raw = env.SWITCHROOM_MODEL_TIER_PROBE_MODEL;
|
|
18675
|
+
if (typeof raw === "string" && raw.trim().length > 0)
|
|
18676
|
+
return raw.trim();
|
|
18677
|
+
return DEFAULT_MODEL_TIER_PROBE_MODEL;
|
|
18678
|
+
}
|
|
18679
|
+
var MODEL_TIER_REPRESENTATIVE_CLAIM = "seven_day_overage_included";
|
|
18656
18680
|
function parseFloatHeader(headers, name) {
|
|
18657
18681
|
const v = headers.get(name);
|
|
18658
18682
|
if (v == null || v.trim().length === 0)
|
|
@@ -18669,15 +18693,37 @@ function parseEpochHeader(headers, name) {
|
|
|
18669
18693
|
return null;
|
|
18670
18694
|
return new Date(n * 1000);
|
|
18671
18695
|
}
|
|
18672
|
-
function
|
|
18696
|
+
function parseTierReset(headers, now) {
|
|
18697
|
+
const oi = parseEpochHeader(headers, "anthropic-ratelimit-unified-7d_oi-reset");
|
|
18698
|
+
if (oi != null)
|
|
18699
|
+
return oi;
|
|
18700
|
+
const unified = parseEpochHeader(headers, "anthropic-ratelimit-unified-reset");
|
|
18701
|
+
if (unified != null)
|
|
18702
|
+
return unified;
|
|
18703
|
+
const ra = headers.get("retry-after");
|
|
18704
|
+
if (ra != null) {
|
|
18705
|
+
const secs = Number(ra);
|
|
18706
|
+
if (Number.isFinite(secs) && secs > 0 && secs < 30 * 24 * 3600) {
|
|
18707
|
+
return new Date(now.getTime() + secs * 1000);
|
|
18708
|
+
}
|
|
18709
|
+
}
|
|
18710
|
+
return null;
|
|
18711
|
+
}
|
|
18712
|
+
function parseQuotaHeaders(headers, now = new Date) {
|
|
18673
18713
|
const fiveHour = parseFloatHeader(headers, "anthropic-ratelimit-unified-5h-utilization");
|
|
18674
18714
|
const sevenDay = parseFloatHeader(headers, "anthropic-ratelimit-unified-7d-utilization");
|
|
18675
|
-
|
|
18715
|
+
const unifiedStatus = headers.get("anthropic-ratelimit-unified-status");
|
|
18716
|
+
const sevenDayOiStatus = headers.get("anthropic-ratelimit-unified-7d_oi-status");
|
|
18717
|
+
const sevenDayOiUtil = parseFloatHeader(headers, "anthropic-ratelimit-unified-7d_oi-utilization");
|
|
18718
|
+
const representativeClaim = headers.get("anthropic-ratelimit-unified-representative-claim");
|
|
18719
|
+
const hasTierSignal = unifiedStatus != null || sevenDayOiStatus != null || sevenDayOiUtil != null;
|
|
18720
|
+
if (fiveHour == null && sevenDay == null && !hasTierSignal) {
|
|
18676
18721
|
return {
|
|
18677
18722
|
ok: false,
|
|
18678
18723
|
reason: "no unified rate-limit headers in response (API token, not OAuth?)"
|
|
18679
18724
|
};
|
|
18680
18725
|
}
|
|
18726
|
+
const oiPresent = sevenDayOiStatus != null || sevenDayOiUtil != null;
|
|
18681
18727
|
return {
|
|
18682
18728
|
ok: true,
|
|
18683
18729
|
data: {
|
|
@@ -18687,9 +18733,14 @@ function parseQuotaHeaders(headers) {
|
|
|
18687
18733
|
sevenDayUtilPresent: sevenDay != null,
|
|
18688
18734
|
fiveHourResetAt: parseEpochHeader(headers, "anthropic-ratelimit-unified-5h-reset"),
|
|
18689
18735
|
sevenDayResetAt: parseEpochHeader(headers, "anthropic-ratelimit-unified-7d-reset"),
|
|
18690
|
-
representativeClaim
|
|
18736
|
+
representativeClaim,
|
|
18691
18737
|
overageStatus: headers.get("anthropic-ratelimit-unified-overage-status"),
|
|
18692
|
-
overageDisabledReason: headers.get("anthropic-ratelimit-unified-overage-disabled-reason")
|
|
18738
|
+
overageDisabledReason: headers.get("anthropic-ratelimit-unified-overage-disabled-reason"),
|
|
18739
|
+
unifiedStatus,
|
|
18740
|
+
sevenDayOiStatus,
|
|
18741
|
+
sevenDayOiUtilizationPct: sevenDayOiUtil != null ? sevenDayOiUtil * 100 : null,
|
|
18742
|
+
sevenDayOiResetAt: parseTierReset(headers, now),
|
|
18743
|
+
sevenDayOiPresent: oiPresent
|
|
18693
18744
|
}
|
|
18694
18745
|
};
|
|
18695
18746
|
}
|
|
@@ -18894,6 +18945,175 @@ function resolveConsumerProbeIntervalMs(env) {
|
|
|
18894
18945
|
return DEFAULT_CONSUMER_PROBE_INTERVAL_MS;
|
|
18895
18946
|
}
|
|
18896
18947
|
|
|
18948
|
+
// src/auth/broker/model-tier-quota.ts
|
|
18949
|
+
var MODEL_TIER_WALL_PCT = 99.5;
|
|
18950
|
+
function quotaIndicatesModelTierWall(result) {
|
|
18951
|
+
const none = { walled: false, until: null, bucket: null };
|
|
18952
|
+
if (!result.ok)
|
|
18953
|
+
return none;
|
|
18954
|
+
const d = result.data;
|
|
18955
|
+
const statusRejected = d.sevenDayOiStatus === "rejected";
|
|
18956
|
+
const claimRejected = d.unifiedStatus === "rejected" && d.representativeClaim === MODEL_TIER_REPRESENTATIVE_CLAIM;
|
|
18957
|
+
const utilWalled = d.sevenDayOiUtilizationPct != null && d.sevenDayOiUtilizationPct >= MODEL_TIER_WALL_PCT;
|
|
18958
|
+
if (!statusRejected && !claimRejected && !utilWalled)
|
|
18959
|
+
return none;
|
|
18960
|
+
return {
|
|
18961
|
+
walled: true,
|
|
18962
|
+
until: d.sevenDayOiResetAt?.getTime() ?? null,
|
|
18963
|
+
bucket: MODEL_TIER_REPRESENTATIVE_CLAIM
|
|
18964
|
+
};
|
|
18965
|
+
}
|
|
18966
|
+
function quotaTierAllowed(result) {
|
|
18967
|
+
if (!result.ok)
|
|
18968
|
+
return false;
|
|
18969
|
+
if (quotaIndicatesModelTierWall(result).walled)
|
|
18970
|
+
return false;
|
|
18971
|
+
const d = result.data;
|
|
18972
|
+
return d.unifiedStatus === "allowed" || d.sevenDayOiStatus === "allowed";
|
|
18973
|
+
}
|
|
18974
|
+
function isModelTierWalled(opts) {
|
|
18975
|
+
const { mark, now, snapshotTierAllowed } = opts;
|
|
18976
|
+
if (snapshotTierAllowed === true)
|
|
18977
|
+
return false;
|
|
18978
|
+
if (mark !== undefined && mark.premium_walled_until > now)
|
|
18979
|
+
return true;
|
|
18980
|
+
return false;
|
|
18981
|
+
}
|
|
18982
|
+
var MODEL_TIER_WALL_DEFAULT_MS = 5 * 60 * 60 * 1000;
|
|
18983
|
+
|
|
18984
|
+
// src/auth/broker/usage-ledger.ts
|
|
18985
|
+
var STANDARD_WALL_PCT = 99.5;
|
|
18986
|
+
var USAGE_RING_CAP_DEFAULT = 288;
|
|
18987
|
+
function resolveUsageRingCap(raw) {
|
|
18988
|
+
if (raw == null || raw.trim().length === 0)
|
|
18989
|
+
return USAGE_RING_CAP_DEFAULT;
|
|
18990
|
+
const n = Number(raw);
|
|
18991
|
+
return Number.isFinite(n) && n >= 1 ? Math.floor(n) : USAGE_RING_CAP_DEFAULT;
|
|
18992
|
+
}
|
|
18993
|
+
function standardSampleFromResult(result, now) {
|
|
18994
|
+
if (!result.ok)
|
|
18995
|
+
return null;
|
|
18996
|
+
const d = result.data;
|
|
18997
|
+
const fivePresent = d.fiveHourUtilPresent !== false;
|
|
18998
|
+
const sevenPresent = d.sevenDayUtilPresent !== false;
|
|
18999
|
+
if (!fivePresent && !sevenPresent)
|
|
19000
|
+
return null;
|
|
19001
|
+
const wins = [];
|
|
19002
|
+
if (fivePresent) {
|
|
19003
|
+
wins.push({
|
|
19004
|
+
util: d.fiveHourUtilizationPct,
|
|
19005
|
+
reset: d.fiveHourResetAt?.getTime() ?? null
|
|
19006
|
+
});
|
|
19007
|
+
}
|
|
19008
|
+
if (sevenPresent) {
|
|
19009
|
+
wins.push({
|
|
19010
|
+
util: d.sevenDayUtilizationPct,
|
|
19011
|
+
reset: d.sevenDayResetAt?.getTime() ?? null
|
|
19012
|
+
});
|
|
19013
|
+
}
|
|
19014
|
+
const binding = wins.reduce((a, b) => b.util > a.util ? b : a);
|
|
19015
|
+
return {
|
|
19016
|
+
at: now,
|
|
19017
|
+
utilizationPct: binding.util,
|
|
19018
|
+
resetAt: binding.reset,
|
|
19019
|
+
status: d.unifiedStatus ?? null,
|
|
19020
|
+
walled: binding.util >= STANDARD_WALL_PCT
|
|
19021
|
+
};
|
|
19022
|
+
}
|
|
19023
|
+
function premiumSampleFromResult(result, now) {
|
|
19024
|
+
if (!result.ok)
|
|
19025
|
+
return null;
|
|
19026
|
+
const d = result.data;
|
|
19027
|
+
const hasSignal = d.unifiedStatus != null || d.sevenDayOiStatus != null || d.sevenDayOiUtilizationPct != null;
|
|
19028
|
+
if (!hasSignal)
|
|
19029
|
+
return null;
|
|
19030
|
+
const util3 = d.sevenDayOiUtilizationPct ?? null;
|
|
19031
|
+
const status = d.sevenDayOiStatus ?? d.unifiedStatus ?? null;
|
|
19032
|
+
const walled = d.sevenDayOiStatus === "rejected" || util3 != null && util3 >= MODEL_TIER_WALL_PCT;
|
|
19033
|
+
return {
|
|
19034
|
+
at: now,
|
|
19035
|
+
utilizationPct: util3,
|
|
19036
|
+
resetAt: d.sevenDayOiResetAt?.getTime() ?? null,
|
|
19037
|
+
status,
|
|
19038
|
+
walled
|
|
19039
|
+
};
|
|
19040
|
+
}
|
|
19041
|
+
function recordUsageSample(ledger, account, tier, sample, ringCap = USAGE_RING_CAP_DEFAULT) {
|
|
19042
|
+
if (sample == null)
|
|
19043
|
+
return ledger;
|
|
19044
|
+
const cap = ringCap >= 1 ? Math.floor(ringCap) : USAGE_RING_CAP_DEFAULT;
|
|
19045
|
+
const acct = ledger[account] ??= {};
|
|
19046
|
+
let tl = acct[tier];
|
|
19047
|
+
if (tl == null) {
|
|
19048
|
+
tl = {
|
|
19049
|
+
tier,
|
|
19050
|
+
samples: [],
|
|
19051
|
+
observations: 0,
|
|
19052
|
+
firstObservedAt: sample.at,
|
|
19053
|
+
lastObservedAt: sample.at
|
|
19054
|
+
};
|
|
19055
|
+
acct[tier] = tl;
|
|
19056
|
+
}
|
|
19057
|
+
tl.samples.push(sample);
|
|
19058
|
+
if (tl.samples.length > cap) {
|
|
19059
|
+
tl.samples.splice(0, tl.samples.length - cap);
|
|
19060
|
+
}
|
|
19061
|
+
tl.observations += 1;
|
|
19062
|
+
tl.lastObservedAt = sample.at;
|
|
19063
|
+
if (sample.at < tl.firstObservedAt)
|
|
19064
|
+
tl.firstObservedAt = sample.at;
|
|
19065
|
+
return ledger;
|
|
19066
|
+
}
|
|
19067
|
+
function summarizeTier(tl, now) {
|
|
19068
|
+
if (tl == null || tl.samples.length === 0)
|
|
19069
|
+
return null;
|
|
19070
|
+
const latest = tl.samples[tl.samples.length - 1];
|
|
19071
|
+
const refilled = latest.resetAt != null && latest.resetAt <= now;
|
|
19072
|
+
const rawUtil = latest.utilizationPct;
|
|
19073
|
+
const currentUtilizationPct = rawUtil == null ? null : refilled ? 0 : rawUtil;
|
|
19074
|
+
const headroomPct = currentUtilizationPct == null ? null : Math.max(0, 100 - currentUtilizationPct);
|
|
19075
|
+
const utilSamples = tl.samples.map((s) => s.utilizationPct).filter((u) => u != null);
|
|
19076
|
+
const peakUtilizationPct = utilSamples.length > 0 ? Math.max(...utilSamples) : null;
|
|
19077
|
+
return {
|
|
19078
|
+
tier: tl.tier,
|
|
19079
|
+
observations: tl.observations,
|
|
19080
|
+
retained: tl.samples.length,
|
|
19081
|
+
currentUtilizationPct,
|
|
19082
|
+
headroomPct,
|
|
19083
|
+
resetAt: latest.resetAt,
|
|
19084
|
+
status: latest.status,
|
|
19085
|
+
walled: latest.walled && !refilled,
|
|
19086
|
+
refilled,
|
|
19087
|
+
lastObservedAt: tl.lastObservedAt,
|
|
19088
|
+
peakUtilizationPct
|
|
19089
|
+
};
|
|
19090
|
+
}
|
|
19091
|
+
function summarizeAccountUsage(ledger, account, now) {
|
|
19092
|
+
const acct = ledger[account];
|
|
19093
|
+
return {
|
|
19094
|
+
account,
|
|
19095
|
+
standard: summarizeTier(acct?.standard, now),
|
|
19096
|
+
premium: summarizeTier(acct?.premium, now)
|
|
19097
|
+
};
|
|
19098
|
+
}
|
|
19099
|
+
function premiumHeadroomPct(ledger, account, now) {
|
|
19100
|
+
return summarizeAccountUsage(ledger, account, now).premium?.headroomPct ?? null;
|
|
19101
|
+
}
|
|
19102
|
+
function bestPremiumHeadroomAccount(ledger, candidates, now) {
|
|
19103
|
+
let best = null;
|
|
19104
|
+
let bestHeadroom = -Infinity;
|
|
19105
|
+
for (const cand of candidates) {
|
|
19106
|
+
const h = premiumHeadroomPct(ledger, cand, now);
|
|
19107
|
+
if (h == null)
|
|
19108
|
+
continue;
|
|
19109
|
+
if (h > bestHeadroom) {
|
|
19110
|
+
bestHeadroom = h;
|
|
19111
|
+
best = cand;
|
|
19112
|
+
}
|
|
19113
|
+
}
|
|
19114
|
+
return best;
|
|
19115
|
+
}
|
|
19116
|
+
|
|
18897
19117
|
// src/auth/broker/server.ts
|
|
18898
19118
|
init_atomic();
|
|
18899
19119
|
|
|
@@ -20058,7 +20278,11 @@ function cachedSnapshotToResult(s) {
|
|
|
20058
20278
|
overageStatus: s.overageStatus,
|
|
20059
20279
|
overageDisabledReason: s.overageDisabledReason,
|
|
20060
20280
|
fiveHourUtilPresent: s.fiveHourUtilPresent,
|
|
20061
|
-
sevenDayUtilPresent: s.sevenDayUtilPresent
|
|
20281
|
+
sevenDayUtilPresent: s.sevenDayUtilPresent,
|
|
20282
|
+
unifiedStatus: null,
|
|
20283
|
+
sevenDayOiStatus: null,
|
|
20284
|
+
sevenDayOiUtilizationPct: null,
|
|
20285
|
+
sevenDayOiResetAt: null
|
|
20062
20286
|
}
|
|
20063
20287
|
};
|
|
20064
20288
|
}
|
|
@@ -20121,6 +20345,10 @@ class AuthBroker {
|
|
|
20121
20345
|
thresholdViolations = {};
|
|
20122
20346
|
notificationClaims = {};
|
|
20123
20347
|
lastFleetRoll = null;
|
|
20348
|
+
lastTierQuotaCache = {};
|
|
20349
|
+
modelTierProbeModel = resolveModelTierProbeModel();
|
|
20350
|
+
usageLedger = {};
|
|
20351
|
+
usageRingCap = resolveUsageRingCap(process.env.SWITCHROOM_USAGE_LEDGER_RING_CAP);
|
|
20124
20352
|
lastWrittenExpiresAt = new Map;
|
|
20125
20353
|
refreshInFlight = new Set;
|
|
20126
20354
|
consumerLastSeen = {};
|
|
@@ -20586,6 +20814,112 @@ class AuthBroker {
|
|
|
20586
20814
|
allowOverage: this.isOverageAllowed(account)
|
|
20587
20815
|
});
|
|
20588
20816
|
}
|
|
20817
|
+
premiumWallMarkOf(account) {
|
|
20818
|
+
const q = this.quota[account];
|
|
20819
|
+
if (!q || q.premium_walled_until === undefined)
|
|
20820
|
+
return;
|
|
20821
|
+
return {
|
|
20822
|
+
premium_walled_until: q.premium_walled_until,
|
|
20823
|
+
premium_wall_bucket: q.premium_wall_bucket,
|
|
20824
|
+
premium_wall_marked_at: q.premium_wall_marked_at
|
|
20825
|
+
};
|
|
20826
|
+
}
|
|
20827
|
+
isAccountPremiumWalled(account) {
|
|
20828
|
+
const mark = this.premiumWallMarkOf(account);
|
|
20829
|
+
const tier = this.lastTierQuotaCache[account];
|
|
20830
|
+
const tierAllowed = tier != null && tier.capturedAt >= (mark?.premium_wall_marked_at ?? 0) && this.now() - tier.capturedAt <= SNAPSHOT_STALE_AGE_MS && (tier.unifiedStatus === "allowed" || tier.sevenDayOiStatus === "allowed") && !(tier.sevenDayOiStatus === "rejected");
|
|
20831
|
+
return isModelTierWalled({
|
|
20832
|
+
mark,
|
|
20833
|
+
now: this.now(),
|
|
20834
|
+
snapshotTierAllowed: tierAllowed ? true : undefined
|
|
20835
|
+
});
|
|
20836
|
+
}
|
|
20837
|
+
recordTierProbe(label, result) {
|
|
20838
|
+
if (result.ok) {
|
|
20839
|
+
const d = result.data;
|
|
20840
|
+
this.lastTierQuotaCache[label] = {
|
|
20841
|
+
unifiedStatus: d.unifiedStatus ?? null,
|
|
20842
|
+
sevenDayOiStatus: d.sevenDayOiStatus ?? null,
|
|
20843
|
+
sevenDayOiUtilizationPct: d.sevenDayOiUtilizationPct ?? null,
|
|
20844
|
+
sevenDayOiResetAt: d.sevenDayOiResetAt?.toISOString() ?? null,
|
|
20845
|
+
capturedAt: this.now()
|
|
20846
|
+
};
|
|
20847
|
+
this.persistLastTierQuotaCache();
|
|
20848
|
+
this.recordUsage(label, "premium", premiumSampleFromResult(result, this.now()));
|
|
20849
|
+
}
|
|
20850
|
+
const entry = this.quota[label];
|
|
20851
|
+
if (quotaTierAllowed(result) && entry?.premium_walled_until !== undefined) {
|
|
20852
|
+
const { premium_walled_until, premium_wall_bucket, premium_wall_marked_at, ...rest } = entry;
|
|
20853
|
+
this.quota[label] = rest;
|
|
20854
|
+
this.persistQuota();
|
|
20855
|
+
process.stdout.write(`auth-broker: premium canary shows ${label} flagship-tier allowed — cleared model-tier wall mark
|
|
20856
|
+
`);
|
|
20857
|
+
this.fanoutToAffectedConsumers(label);
|
|
20858
|
+
}
|
|
20859
|
+
if (quotaTierAllowed(result))
|
|
20860
|
+
this.clearPremiumAllWalled();
|
|
20861
|
+
}
|
|
20862
|
+
nextPremiumEligibleAccount(from) {
|
|
20863
|
+
const order = this.config.auth?.fallback_order ?? [];
|
|
20864
|
+
const start = order.indexOf(from);
|
|
20865
|
+
const ring = start === -1 ? [...order] : order.map((_, i) => order[(start + 1 + i) % order.length]);
|
|
20866
|
+
const eligible = [];
|
|
20867
|
+
for (const cand of ring) {
|
|
20868
|
+
if (!cand || cand === from)
|
|
20869
|
+
continue;
|
|
20870
|
+
if (this.isAccountExhausted(cand))
|
|
20871
|
+
continue;
|
|
20872
|
+
if (this.isAccountPremiumWalled(cand))
|
|
20873
|
+
continue;
|
|
20874
|
+
if (!accountExists(cand, this.home))
|
|
20875
|
+
continue;
|
|
20876
|
+
if (!readAccountCredentials(cand, this.home))
|
|
20877
|
+
continue;
|
|
20878
|
+
eligible.push(cand);
|
|
20879
|
+
}
|
|
20880
|
+
if (eligible.length === 0)
|
|
20881
|
+
return null;
|
|
20882
|
+
return bestPremiumHeadroomAccount(this.usageLedger, eligible, this.now()) ?? eligible[0];
|
|
20883
|
+
}
|
|
20884
|
+
earliestPremiumWallReset() {
|
|
20885
|
+
let earliest = null;
|
|
20886
|
+
for (const label of listAccounts(this.home)) {
|
|
20887
|
+
const until = this.quota[label]?.premium_walled_until;
|
|
20888
|
+
if (until != null && (earliest == null || until < earliest))
|
|
20889
|
+
earliest = until;
|
|
20890
|
+
}
|
|
20891
|
+
return earliest;
|
|
20892
|
+
}
|
|
20893
|
+
markPremiumWalledAndRoll(account, until, bucket) {
|
|
20894
|
+
const now = this.now();
|
|
20895
|
+
const premiumWalledUntil = until ?? now + MODEL_TIER_WALL_DEFAULT_MS;
|
|
20896
|
+
this.quota[account] = {
|
|
20897
|
+
...this.quota[account],
|
|
20898
|
+
premium_walled_until: premiumWalledUntil,
|
|
20899
|
+
premium_wall_bucket: bucket ?? undefined,
|
|
20900
|
+
premium_wall_marked_at: now
|
|
20901
|
+
};
|
|
20902
|
+
this.persistQuota();
|
|
20903
|
+
const target = this.nextPremiumEligibleAccount(account);
|
|
20904
|
+
if (!target) {
|
|
20905
|
+
return { rolledTo: null, allWalled: true, earliestReset: this.earliestPremiumWallReset() };
|
|
20906
|
+
}
|
|
20907
|
+
this.fanoutFailoverTo(account, target);
|
|
20908
|
+
this.fanoutToAffectedConsumers(account);
|
|
20909
|
+
this.clearPremiumAllWalled();
|
|
20910
|
+
if (this.config.auth?.active === account) {
|
|
20911
|
+
this.config = {
|
|
20912
|
+
...this.config,
|
|
20913
|
+
auth: { ...this.config.auth ?? {}, active: target }
|
|
20914
|
+
};
|
|
20915
|
+
this.persistActiveOverride(target);
|
|
20916
|
+
this.fanoutAllConsumers();
|
|
20917
|
+
this.audit({ op: "auto-promote-active", identity: { kind: "operator" }, account: target, accountKind: "claude", ok: true, reason: "model-tier-wall" });
|
|
20918
|
+
process.stdout.write(`auth-broker: auto-promoted auth.active ${account} → ${target} ` + `(${account} flagship-tier walled until ${new Date(premiumWalledUntil).toISOString()}, bucket=${bucket ?? "?"}) — persisted
|
|
20919
|
+
`);
|
|
20920
|
+
}
|
|
20921
|
+
return { rolledTo: target, allWalled: false, earliestReset: null };
|
|
20922
|
+
}
|
|
20589
20923
|
softAvoidState = {};
|
|
20590
20924
|
softAvoidRolledTo = {};
|
|
20591
20925
|
softAvoidSnapshotOf(account) {
|
|
@@ -20764,7 +21098,12 @@ class AuthBroker {
|
|
|
20764
21098
|
throttled_until: q?.throttled_until,
|
|
20765
21099
|
threshold_violations: this.thresholdViolations[label] ?? 0,
|
|
20766
21100
|
last_refreshed_at: meta?.lastRefreshedAt,
|
|
20767
|
-
last_quota: lq ?? null
|
|
21101
|
+
last_quota: lq ?? null,
|
|
21102
|
+
premium_walled: this.isAccountPremiumWalled(label),
|
|
21103
|
+
premium_walled_until: q?.premium_walled_until,
|
|
21104
|
+
premium_wall_bucket: q?.premium_wall_bucket,
|
|
21105
|
+
last_tier_quota: this.lastTierQuotaCache[label] ?? null,
|
|
21106
|
+
usage_ledger: summarizeAccountUsage(this.usageLedger, label, this.now())
|
|
20768
21107
|
};
|
|
20769
21108
|
});
|
|
20770
21109
|
const agents = Object.entries(this.config.agents ?? {}).map(([name, agent]) => {
|
|
@@ -20786,7 +21125,8 @@ class AuthBroker {
|
|
|
20786
21125
|
agents,
|
|
20787
21126
|
consumers,
|
|
20788
21127
|
active_overage_serving,
|
|
20789
|
-
last_fleet_roll: this.lastFleetRoll
|
|
21128
|
+
last_fleet_roll: this.lastFleetRoll,
|
|
21129
|
+
premium_tier_all_walled: this.premiumTierAllWalled
|
|
20790
21130
|
}));
|
|
20791
21131
|
}
|
|
20792
21132
|
async opListGoogleAccounts(socket, id, identity) {
|
|
@@ -20872,6 +21212,7 @@ class AuthBroker {
|
|
|
20872
21212
|
};
|
|
20873
21213
|
this.lastQuotaCache[label] = snapshot;
|
|
20874
21214
|
this.persistLastQuotaCache();
|
|
21215
|
+
this.recordUsage(label, "standard", standardSampleFromResult(result, this.now()));
|
|
20875
21216
|
if (snapshotShouldClearMark(snapshot, this.exhaustionMarkOf(label), this.now())) {
|
|
20876
21217
|
delete this.quota[label];
|
|
20877
21218
|
this.persistQuota();
|
|
@@ -20882,6 +21223,9 @@ class AuthBroker {
|
|
|
20882
21223
|
}
|
|
20883
21224
|
async fleetQuotaProbeTick() {
|
|
20884
21225
|
const probed = [];
|
|
21226
|
+
const tierWalled = new Map;
|
|
21227
|
+
const canarySet = this.premiumCanarySet();
|
|
21228
|
+
const canaryEnabled = process.env.SWITCHROOM_DISABLE_MODEL_TIER_PROBE !== "1";
|
|
20885
21229
|
for (const label of listAccounts(this.home)) {
|
|
20886
21230
|
const creds = readAccountCredentials(label, this.home);
|
|
20887
21231
|
const token = creds?.claudeAiOauth?.accessToken;
|
|
@@ -20896,12 +21240,49 @@ class AuthBroker {
|
|
|
20896
21240
|
}
|
|
20897
21241
|
this.cacheQuotaSnapshot(label, result);
|
|
20898
21242
|
probed.push({ label, result });
|
|
21243
|
+
if (canaryEnabled && canarySet.has(label)) {
|
|
21244
|
+
let tierResult;
|
|
21245
|
+
try {
|
|
21246
|
+
tierResult = await this.fetchQuotaImpl({ accessToken: token, model: this.modelTierProbeModel });
|
|
21247
|
+
} catch (err) {
|
|
21248
|
+
this.logErr(`fleet-tier-probe ${label}: ${err.message}`);
|
|
21249
|
+
continue;
|
|
21250
|
+
}
|
|
21251
|
+
this.recordTierProbe(label, tierResult);
|
|
21252
|
+
const wall = quotaIndicatesModelTierWall(tierResult);
|
|
21253
|
+
if (wall.walled)
|
|
21254
|
+
tierWalled.set(label, { until: wall.until, bucket: wall.bucket });
|
|
21255
|
+
}
|
|
21256
|
+
}
|
|
21257
|
+
for (const [label, wall] of tierWalled) {
|
|
21258
|
+
const now = this.now();
|
|
21259
|
+
this.quota[label] = {
|
|
21260
|
+
...this.quota[label],
|
|
21261
|
+
premium_walled_until: wall.until ?? now + MODEL_TIER_WALL_DEFAULT_MS,
|
|
21262
|
+
premium_wall_bucket: wall.bucket ?? undefined,
|
|
21263
|
+
premium_wall_marked_at: now
|
|
21264
|
+
};
|
|
20899
21265
|
}
|
|
21266
|
+
if (tierWalled.size > 0)
|
|
21267
|
+
this.persistQuota();
|
|
20900
21268
|
for (const { label, result } of probed) {
|
|
20901
21269
|
const active = this.config.auth?.active;
|
|
20902
21270
|
const isPinnedAgentAccount = this.pinnedAgentAccounts().has(label);
|
|
20903
21271
|
if (label !== active && !isPinnedAgentAccount)
|
|
20904
21272
|
continue;
|
|
21273
|
+
const tw = tierWalled.get(label);
|
|
21274
|
+
if (tw) {
|
|
21275
|
+
process.stdout.write(`auth-broker: fleet-tier-probe shows ${label === active ? "ACTIVE" : "PINNED"} ${label} ` + `flagship-tier walled (bucket=${tw.bucket ?? "?"}) — model-tier failover
|
|
21276
|
+
`);
|
|
21277
|
+
this.audit({ op: "mark-exhausted", identity: { kind: "operator" }, account: label, accountKind: "claude", ok: true, reason: "model-tier-wall" });
|
|
21278
|
+
const { rolledTo, allWalled, earliestReset } = this.markPremiumWalledAndRoll(label, tw.until, tw.bucket);
|
|
21279
|
+
if (rolledTo && label === active) {
|
|
21280
|
+
this.recordFleetRoll(label, rolledTo, "model-tier-wall", tw.until ?? undefined, tw.bucket);
|
|
21281
|
+
} else if (allWalled && label === active) {
|
|
21282
|
+
this.recordPremiumAllWalled(tw.bucket, earliestReset);
|
|
21283
|
+
}
|
|
21284
|
+
continue;
|
|
21285
|
+
}
|
|
20905
21286
|
const decision = quotaIndicatesExhaustion(result, this.isOverageAllowed(label));
|
|
20906
21287
|
if (decision.exhausted) {
|
|
20907
21288
|
process.stdout.write(`auth-broker: fleet-quota-probe shows ${label === active ? "ACTIVE" : "PINNED"} ${label} exhausted — proactive failover
|
|
@@ -20916,21 +21297,52 @@ class AuthBroker {
|
|
|
20916
21297
|
this.softAvoidProbeRoll(label);
|
|
20917
21298
|
}
|
|
20918
21299
|
}
|
|
20919
|
-
recordFleetRoll(from, to, reason, exhaustedUntil) {
|
|
21300
|
+
recordFleetRoll(from, to, reason, exhaustedUntil, bucket) {
|
|
20920
21301
|
const s = this.lastQuotaCache[from];
|
|
20921
21302
|
this.lastFleetRoll = {
|
|
20922
21303
|
from,
|
|
20923
21304
|
to,
|
|
20924
21305
|
at: this.now(),
|
|
20925
21306
|
reason,
|
|
21307
|
+
...bucket ? { bucket } : {},
|
|
20926
21308
|
...exhaustedUntil != null ? { exhausted_until: exhaustedUntil } : {},
|
|
20927
|
-
...s ? {
|
|
21309
|
+
...s && reason !== "model-tier-wall" ? {
|
|
20928
21310
|
window: s.fiveHourUtilizationPct >= s.sevenDayUtilizationPct ? "5h" : "7d",
|
|
20929
21311
|
pct: Math.max(s.fiveHourUtilizationPct, s.sevenDayUtilizationPct)
|
|
20930
21312
|
} : {}
|
|
20931
21313
|
};
|
|
20932
21314
|
this.persistLastFleetRoll();
|
|
20933
21315
|
}
|
|
21316
|
+
premiumTierAllWalled = null;
|
|
21317
|
+
recordPremiumAllWalled(bucket, earliestReset) {
|
|
21318
|
+
this.premiumTierAllWalled = {
|
|
21319
|
+
bucket,
|
|
21320
|
+
...earliestReset != null ? { earliest_reset: earliestReset } : {},
|
|
21321
|
+
at: this.now()
|
|
21322
|
+
};
|
|
21323
|
+
atomicWriteJsonSync(join8(this.stateDir, "premium-tier-all-walled.json"), this.premiumTierAllWalled, 384);
|
|
21324
|
+
process.stdout.write(`auth-broker: ALL accounts flagship-tier walled (bucket=${bucket ?? "?"}` + `${earliestReset != null ? `, earliest reset ${new Date(earliestReset).toISOString()}` : ""}) ` + `— no premium-eligible failover target
|
|
21325
|
+
`);
|
|
21326
|
+
}
|
|
21327
|
+
clearPremiumAllWalled() {
|
|
21328
|
+
if (this.premiumTierAllWalled == null)
|
|
21329
|
+
return;
|
|
21330
|
+
this.premiumTierAllWalled = null;
|
|
21331
|
+
try {
|
|
21332
|
+
unlinkSync(join8(this.stateDir, "premium-tier-all-walled.json"));
|
|
21333
|
+
} catch {}
|
|
21334
|
+
}
|
|
21335
|
+
premiumCanarySet() {
|
|
21336
|
+
const set = new Set;
|
|
21337
|
+
const active = this.config.auth?.active;
|
|
21338
|
+
if (active)
|
|
21339
|
+
set.add(active);
|
|
21340
|
+
for (const cand of this.config.auth?.fallback_order ?? [])
|
|
21341
|
+
set.add(cand);
|
|
21342
|
+
for (const acct of this.pinnedAgentAccounts())
|
|
21343
|
+
set.add(acct);
|
|
21344
|
+
return set;
|
|
21345
|
+
}
|
|
20934
21346
|
pinnedAgentAccounts() {
|
|
20935
21347
|
return new Set(Object.values(this.config.agents ?? {}).map((a) => a.auth?.override).filter((x) => typeof x === "string" && x.length > 0));
|
|
20936
21348
|
}
|
|
@@ -21840,6 +22252,9 @@ class AuthBroker {
|
|
|
21840
22252
|
this.notificationClaims = this.readJson("notification-claims.json") ?? {};
|
|
21841
22253
|
this.lastFleetRoll = this.readJson("last-fleet-roll.json") ?? null;
|
|
21842
22254
|
this.lastQuotaCache = this.readJson("last-quota.json") ?? {};
|
|
22255
|
+
this.lastTierQuotaCache = this.readJson("last-tier-quota.json") ?? {};
|
|
22256
|
+
this.premiumTierAllWalled = this.readJson("premium-tier-all-walled.json") ?? null;
|
|
22257
|
+
this.usageLedger = this.readJson("usage-ledger.json") ?? {};
|
|
21843
22258
|
this.applyActiveOverride();
|
|
21844
22259
|
}
|
|
21845
22260
|
applyActiveOverride() {
|
|
@@ -21893,6 +22308,22 @@ class AuthBroker {
|
|
|
21893
22308
|
persistLastQuotaCache() {
|
|
21894
22309
|
atomicWriteJsonSync(join8(this.stateDir, "last-quota.json"), this.lastQuotaCache, 384);
|
|
21895
22310
|
}
|
|
22311
|
+
persistLastTierQuotaCache() {
|
|
22312
|
+
atomicWriteJsonSync(join8(this.stateDir, "last-tier-quota.json"), this.lastTierQuotaCache, 384);
|
|
22313
|
+
}
|
|
22314
|
+
persistUsageLedger() {
|
|
22315
|
+
atomicWriteJsonSync(join8(this.stateDir, "usage-ledger.json"), this.usageLedger, 384);
|
|
22316
|
+
}
|
|
22317
|
+
recordUsage(label, tier, sample) {
|
|
22318
|
+
if (sample == null)
|
|
22319
|
+
return;
|
|
22320
|
+
try {
|
|
22321
|
+
recordUsageSample(this.usageLedger, label, tier, sample, this.usageRingCap);
|
|
22322
|
+
this.persistUsageLedger();
|
|
22323
|
+
} catch (err) {
|
|
22324
|
+
this.logErr(`usage-ledger ${label}/${tier}: ${err.message}`);
|
|
22325
|
+
}
|
|
22326
|
+
}
|
|
21896
22327
|
persistNotificationClaims() {
|
|
21897
22328
|
atomicWriteJsonSync(join8(this.stateDir, "notification-claims.json"), this.notificationClaims, 384);
|
|
21898
22329
|
}
|
|
@@ -22026,7 +22457,11 @@ class AuthBroker {
|
|
|
22026
22457
|
shaIndex: { ...this.shaIndex },
|
|
22027
22458
|
thresholdViolations: { ...this.thresholdViolations },
|
|
22028
22459
|
listeners: [...this.listeners.keys()],
|
|
22029
|
-
lastQuotaCache: structuredClone(this.lastQuotaCache)
|
|
22460
|
+
lastQuotaCache: structuredClone(this.lastQuotaCache),
|
|
22461
|
+
lastTierQuotaCache: structuredClone(this.lastTierQuotaCache),
|
|
22462
|
+
usageLedger: structuredClone(this.usageLedger),
|
|
22463
|
+
premiumTierAllWalled: this.premiumTierAllWalled ? { ...this.premiumTierAllWalled } : null,
|
|
22464
|
+
activeAccount: this.config.auth?.active
|
|
22030
22465
|
};
|
|
22031
22466
|
}
|
|
22032
22467
|
_fanoutAll() {
|