switchroom 0.18.14 → 0.18.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/dist/agent-scheduler/index.js +3 -0
  2. package/dist/auth-broker/index.js +473 -49
  3. package/dist/cli/notion-write-pretool.mjs +3 -0
  4. package/dist/cli/switchroom.js +1200 -1067
  5. package/dist/host-control/main.js +56 -51
  6. package/dist/vault/approvals/kernel-server.js +19 -12
  7. package/dist/vault/broker/server.js +675 -668
  8. package/package.json +1 -1
  9. package/profiles/_base/start.sh.hbs +81 -139
  10. package/telegram-plugin/dist/bridge/bridge.js +21 -0
  11. package/telegram-plugin/dist/gateway/gateway.js +531 -259
  12. package/telegram-plugin/dist/server.js +22 -1
  13. package/telegram-plugin/draft-stream.ts +78 -3
  14. package/telegram-plugin/gateway/bridge-dead-watchdog.ts +3 -4
  15. package/telegram-plugin/gateway/effort-command.ts +9 -7
  16. package/telegram-plugin/gateway/gateway.ts +310 -219
  17. package/telegram-plugin/gateway/litellm-local-notice-wiring.ts +200 -0
  18. package/telegram-plugin/gateway/model-command.ts +96 -18
  19. package/telegram-plugin/gateway/pending-session-command.ts +10 -8
  20. package/telegram-plugin/gateway/session-model-file.ts +38 -172
  21. package/telegram-plugin/litellm-local-notice.ts +189 -0
  22. package/telegram-plugin/model-unavailable.ts +214 -0
  23. package/telegram-plugin/quota-watch.ts +16 -4
  24. package/telegram-plugin/runtime-metrics.ts +47 -0
  25. package/telegram-plugin/send-gate-degraded.test.ts +9 -7
  26. package/telegram-plugin/send-gate.ts +34 -4
  27. package/telegram-plugin/session-tail.ts +14 -2
  28. package/telegram-plugin/stream-controller.ts +143 -20
  29. package/telegram-plugin/stream-reply-handler.ts +12 -2
  30. package/telegram-plugin/tests/bot-api.harness.ts +7 -2
  31. package/telegram-plugin/tests/draft-stream.test.ts +110 -1
  32. package/telegram-plugin/tests/effort-command.test.ts +4 -4
  33. package/telegram-plugin/tests/flood-windows-persistence.test.ts +2 -2
  34. package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +33 -19
  35. package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +47 -127
  36. package/telegram-plugin/tests/litellm-local-notice.test.ts +417 -0
  37. package/telegram-plugin/tests/model-command.test.ts +84 -1
  38. package/telegram-plugin/tests/model-unavailable.test.ts +187 -0
  39. package/telegram-plugin/tests/operator-events-session-tail.test.ts +55 -0
  40. package/telegram-plugin/tests/quota-watch.test.ts +21 -0
  41. package/telegram-plugin/tests/reaction-gate-routing.test.ts +2 -2
  42. package/telegram-plugin/tests/runtime-metrics.test.ts +24 -0
  43. package/telegram-plugin/tests/session-model-file.test.ts +7 -155
  44. package/telegram-plugin/tests/stream-controller-send-gate.test.ts +521 -0
  45. package/telegram-plugin/tests/stream-reply-handler.test.ts +44 -0
  46. package/telegram-plugin/tests/throttle-tier.test.ts +176 -0
  47. package/telegram-plugin/tests/worker-activity-feed.test.ts +207 -0
  48. package/telegram-plugin/throttle-tier.ts +98 -1
  49. package/telegram-plugin/worker-activity-feed.ts +83 -8
@@ -16853,6 +16853,9 @@ var TelegramChannelSchema = exports_external.object({
16853
16853
  window_ms: exports_external.number().int().nonnegative().optional().describe("Sliding-window (ms) for merging consecutive inbound messages from " + "the same sender+topic into ONE Claude turn. Each new message resets " + "the timer; the turn starts once the sender pauses for this long. " + "Catches forwarded bursts, pasted text the Telegram client split " + "into several messages, and mixed text+media forwards. Default 500. " + "Set 0 to disable (every message becomes its own turn). Raise for " + "users who think in multiple short messages; the trade-off is the " + "single-message turn start is delayed by this much (the \uD83D\uDC40 ack still " + "fires immediately, so perceived latency is unchanged)."),
16854
16854
  max_attachments: exports_external.number().int().positive().optional().describe("Maximum number of media attachments carried into ONE coalesced " + "Claude turn. Default 10 — a full Telegram album (media_group caps " + "at 10) or a text+multi-image forwarded burst arrives as a single " + "turn; the agent sees numbered attachment fields (image_path, " + "image_path_2, …). Set 1 to restore the historical " + "single-attachment-per-turn behaviour. Excess attachments beyond " + "the cap spill into the next turn. Each attachment is downloaded, " + "so a high cap on a slow link delays turn start.")
16855
16855
  }).optional().describe("Inbound coalescing — how the gateway groups rapid consecutive messages " + "into a single turn so a forwarded album or split paste doesn't fan out " + "into N separate turns. Cascades from defaults.channels.telegram.coalesce."),
16856
+ litellm_notice: exports_external.object({
16857
+ window_ms: exports_external.number().int().positive().optional().describe("Per-agent cooldown window (ms) for the litellm-local 429 notice. " + "When the agent trips the LiteLLM proxy's OWN tpm_limit/rpm_limit " + "cap (a `litellm-local` classified 429 — see docs/auth.md § " + "LiteLLM-proxy-local 429s), the gateway posts ONE calm notice " + "naming the fleet token limiter, then counts further hits silently " + "for this long; the first notice after the window expires says " + "how many were absorbed. Default 900000 (15 min). Invalid values " + "fall back to the default.")
16858
+ }).optional().describe("Debounce tuning for the litellm-local throttle notice — the calm " + "'fleet token limiter engaged' message posted when the LiteLLM " + "proxy's own rate cap trips (never an Anthropic account limit). " + "Cascades from defaults.channels.telegram.litellm_notice."),
16856
16859
  interrupt: exports_external.object({
16857
16860
  safe_boundary: exports_external.boolean().optional().describe("When true (the default), a `!`-prefix interrupt that arrives while " + "the agent is mid-tool-call is DEFERRED: the SIGINT and the " + "replacement turn wait until the in-flight tool call finishes (a " + "clean boundary) instead of C-c'ing the agent mid-write/mid-bash. If " + "no tool is in flight the interrupt still fires immediately. Bounded " + "by max_wait_ms so a long tool never strands the user. Set false to " + "fire synchronously the moment `!` is received (historical " + "behaviour). Rapid repeated `!` while one is pending coalesce into a " + "single deferred interrupt carrying the latest body."),
16858
16861
  max_wait_ms: exports_external.number().int().positive().optional().describe("Upper bound (ms) the gateway waits for a safe boundary before firing " + "a deferred `!` interrupt anyway. Only consulted when safe_boundary is " + "true. Default 8000. Keep it short — the user explicitly asked to " + "interrupt, so a long in-flight tool shouldn't ghost them; the cap " + "trades a tiny risk of a mid-tool C-c for a guaranteed response.")
@@ -18118,13 +18121,51 @@ import * as constants2 from "node:constants";
18118
18121
  import { createHash as createHash2 } from "node:crypto";
18119
18122
  import { dirname as dirname3, join as join8, resolve as resolve9 } from "node:path";
18120
18123
 
18121
- // src/agents/compose.ts
18122
- import { createHash } from "node:crypto";
18123
-
18124
18124
  // src/agents/scaffold.ts
18125
18125
  import { join as join4, resolve as resolve5 } from "node:path";
18126
18126
  init_atomic();
18127
18127
 
18128
+ // src/agents/agent-uid.ts
18129
+ import { createHash } from "node:crypto";
18130
+
18131
+ // src/broker-common/peercred-path.ts
18132
+ function matchSocketName(socketPath, patterns, maxNameLen) {
18133
+ if (typeof socketPath !== "string" || socketPath.length === 0)
18134
+ return null;
18135
+ let name = null;
18136
+ for (const re of patterns) {
18137
+ const m = socketPath.match(re);
18138
+ if (m && typeof m[1] === "string") {
18139
+ name = m[1];
18140
+ break;
18141
+ }
18142
+ }
18143
+ if (name === null || name.length === 0)
18144
+ return null;
18145
+ if (maxNameLen !== undefined && name.length > maxNameLen)
18146
+ return null;
18147
+ return name;
18148
+ }
18149
+
18150
+ // src/vault/broker/peercred.ts
18151
+ var RESERVED_AGENT_NAMES = new Set(["operator", "hostd"]);
18152
+ function isReservedAgentName(name) {
18153
+ return RESERVED_AGENT_NAMES.has(name);
18154
+ }
18155
+
18156
+ // src/agents/agent-uid.ts
18157
+ var AGENT_UID_MIN = 10001;
18158
+ var AGENT_UID_MAX = 10999;
18159
+ function allocateAgentUid(name) {
18160
+ if (isReservedAgentName(name)) {
18161
+ throw new Error(`agent name '${name}' is reserved by switchroom for another identity kind ` + `(see vault/broker/peercred.ts:RESERVED_AGENT_NAMES). Pick a different name.`);
18162
+ }
18163
+ const hash = createHash("sha256").update(name).digest();
18164
+ const u32 = hash.readUInt32BE(0);
18165
+ const range = AGENT_UID_MAX - AGENT_UID_MIN + 1;
18166
+ return AGENT_UID_MIN + u32 % range;
18167
+ }
18168
+
18128
18169
  // src/config/timezone.ts
18129
18170
  var CONTAINER_DEFAULT_UTC_ZONES = new Set([
18130
18171
  "UTC",
@@ -18268,31 +18309,6 @@ var SCRYPT_MAXMEM = 128 * 1024 * 1024;
18268
18309
  import { homedir as homedir3 } from "node:os";
18269
18310
  import { join as join3 } from "node:path";
18270
18311
 
18271
- // src/broker-common/peercred-path.ts
18272
- function matchSocketName(socketPath, patterns, maxNameLen) {
18273
- if (typeof socketPath !== "string" || socketPath.length === 0)
18274
- return null;
18275
- let name = null;
18276
- for (const re of patterns) {
18277
- const m = socketPath.match(re);
18278
- if (m && typeof m[1] === "string") {
18279
- name = m[1];
18280
- break;
18281
- }
18282
- }
18283
- if (name === null || name.length === 0)
18284
- return null;
18285
- if (maxNameLen !== undefined && name.length > maxNameLen)
18286
- return null;
18287
- return name;
18288
- }
18289
-
18290
- // src/vault/broker/peercred.ts
18291
- var RESERVED_AGENT_NAMES = new Set(["operator", "hostd"]);
18292
- function isReservedAgentName(name) {
18293
- return RESERVED_AGENT_NAMES.has(name);
18294
- }
18295
-
18296
18312
  // src/vault/broker/protocol.ts
18297
18313
  var MAX_FRAME_BYTES = 64 * 1024;
18298
18314
  var AgentNameSchema = exports_external.string().min(1).max(64, "agent name max 64 chars").regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/, "agent name must be kebab-case ASCII (alnum + _- only, first char alnum)").refine((s) => !isReservedAgentName(s), {
@@ -18634,23 +18650,20 @@ var DOCKER_TELEGRAM_PLUGIN_PATH = "/opt/switchroom/telegram-plugin";
18634
18650
  var DOCKER_HOOKS_PATH = `${DOCKER_TELEGRAM_PLUGIN_PATH}/hooks`;
18635
18651
 
18636
18652
  // src/agents/compose.ts
18637
- var AGENT_UID_MIN = 10001;
18638
- var AGENT_UID_MAX = 10999;
18639
- function allocateAgentUid(name) {
18640
- if (isReservedAgentName(name)) {
18641
- throw new Error(`agent name '${name}' is reserved by switchroom for another identity kind ` + `(see vault/broker/peercred.ts:RESERVED_AGENT_NAMES). Pick a different name.`);
18642
- }
18643
- const hash = createHash("sha256").update(name).digest();
18644
- const u32 = hash.readUInt32BE(0);
18645
- const range = AGENT_UID_MAX - AGENT_UID_MIN + 1;
18646
- return AGENT_UID_MIN + u32 % range;
18647
- }
18648
18653
  var BIND_MOUNT_EXACT_SOURCE_DENY = new Set(["/var/run/docker.sock"]);
18649
18654
 
18650
18655
  // src/auth/quota.ts
18651
18656
  var OAUTH_BETA = "oauth-2025-04-20";
18652
18657
  var DEFAULT_USER_AGENT = "claude-cli/1.0.0 (external, cli)";
18653
18658
  var DEFAULT_PROBE_MODEL = "claude-haiku-4-5-20251001";
18659
+ var DEFAULT_MODEL_TIER_PROBE_MODEL = "claude-fable-5";
18660
+ function resolveModelTierProbeModel(env = process.env) {
18661
+ const raw = env.SWITCHROOM_MODEL_TIER_PROBE_MODEL;
18662
+ if (typeof raw === "string" && raw.trim().length > 0)
18663
+ return raw.trim();
18664
+ return DEFAULT_MODEL_TIER_PROBE_MODEL;
18665
+ }
18666
+ var MODEL_TIER_REPRESENTATIVE_CLAIM = "seven_day_overage_included";
18654
18667
  function parseFloatHeader(headers, name) {
18655
18668
  const v = headers.get(name);
18656
18669
  if (v == null || v.trim().length === 0)
@@ -18667,15 +18680,37 @@ function parseEpochHeader(headers, name) {
18667
18680
  return null;
18668
18681
  return new Date(n * 1000);
18669
18682
  }
18670
- function parseQuotaHeaders(headers) {
18683
+ function parseTierReset(headers, now) {
18684
+ const oi = parseEpochHeader(headers, "anthropic-ratelimit-unified-7d_oi-reset");
18685
+ if (oi != null)
18686
+ return oi;
18687
+ const unified = parseEpochHeader(headers, "anthropic-ratelimit-unified-reset");
18688
+ if (unified != null)
18689
+ return unified;
18690
+ const ra = headers.get("retry-after");
18691
+ if (ra != null) {
18692
+ const secs = Number(ra);
18693
+ if (Number.isFinite(secs) && secs > 0 && secs < 30 * 24 * 3600) {
18694
+ return new Date(now.getTime() + secs * 1000);
18695
+ }
18696
+ }
18697
+ return null;
18698
+ }
18699
+ function parseQuotaHeaders(headers, now = new Date) {
18671
18700
  const fiveHour = parseFloatHeader(headers, "anthropic-ratelimit-unified-5h-utilization");
18672
18701
  const sevenDay = parseFloatHeader(headers, "anthropic-ratelimit-unified-7d-utilization");
18673
- if (fiveHour == null && sevenDay == null) {
18702
+ const unifiedStatus = headers.get("anthropic-ratelimit-unified-status");
18703
+ const sevenDayOiStatus = headers.get("anthropic-ratelimit-unified-7d_oi-status");
18704
+ const sevenDayOiUtil = parseFloatHeader(headers, "anthropic-ratelimit-unified-7d_oi-utilization");
18705
+ const representativeClaim = headers.get("anthropic-ratelimit-unified-representative-claim");
18706
+ const hasTierSignal = unifiedStatus != null || sevenDayOiStatus != null || sevenDayOiUtil != null;
18707
+ if (fiveHour == null && sevenDay == null && !hasTierSignal) {
18674
18708
  return {
18675
18709
  ok: false,
18676
18710
  reason: "no unified rate-limit headers in response (API token, not OAuth?)"
18677
18711
  };
18678
18712
  }
18713
+ const oiPresent = sevenDayOiStatus != null || sevenDayOiUtil != null;
18679
18714
  return {
18680
18715
  ok: true,
18681
18716
  data: {
@@ -18685,9 +18720,14 @@ function parseQuotaHeaders(headers) {
18685
18720
  sevenDayUtilPresent: sevenDay != null,
18686
18721
  fiveHourResetAt: parseEpochHeader(headers, "anthropic-ratelimit-unified-5h-reset"),
18687
18722
  sevenDayResetAt: parseEpochHeader(headers, "anthropic-ratelimit-unified-7d-reset"),
18688
- representativeClaim: headers.get("anthropic-ratelimit-unified-representative-claim"),
18723
+ representativeClaim,
18689
18724
  overageStatus: headers.get("anthropic-ratelimit-unified-overage-status"),
18690
- overageDisabledReason: headers.get("anthropic-ratelimit-unified-overage-disabled-reason")
18725
+ overageDisabledReason: headers.get("anthropic-ratelimit-unified-overage-disabled-reason"),
18726
+ unifiedStatus,
18727
+ sevenDayOiStatus,
18728
+ sevenDayOiUtilizationPct: sevenDayOiUtil != null ? sevenDayOiUtil * 100 : null,
18729
+ sevenDayOiResetAt: parseTierReset(headers, now),
18730
+ sevenDayOiPresent: oiPresent
18691
18731
  }
18692
18732
  };
18693
18733
  }
@@ -18892,6 +18932,175 @@ function resolveConsumerProbeIntervalMs(env) {
18892
18932
  return DEFAULT_CONSUMER_PROBE_INTERVAL_MS;
18893
18933
  }
18894
18934
 
18935
+ // src/auth/broker/model-tier-quota.ts
18936
+ var MODEL_TIER_WALL_PCT = 99.5;
18937
+ function quotaIndicatesModelTierWall(result) {
18938
+ const none = { walled: false, until: null, bucket: null };
18939
+ if (!result.ok)
18940
+ return none;
18941
+ const d = result.data;
18942
+ const statusRejected = d.sevenDayOiStatus === "rejected";
18943
+ const claimRejected = d.unifiedStatus === "rejected" && d.representativeClaim === MODEL_TIER_REPRESENTATIVE_CLAIM;
18944
+ const utilWalled = d.sevenDayOiUtilizationPct != null && d.sevenDayOiUtilizationPct >= MODEL_TIER_WALL_PCT;
18945
+ if (!statusRejected && !claimRejected && !utilWalled)
18946
+ return none;
18947
+ return {
18948
+ walled: true,
18949
+ until: d.sevenDayOiResetAt?.getTime() ?? null,
18950
+ bucket: MODEL_TIER_REPRESENTATIVE_CLAIM
18951
+ };
18952
+ }
18953
+ function quotaTierAllowed(result) {
18954
+ if (!result.ok)
18955
+ return false;
18956
+ if (quotaIndicatesModelTierWall(result).walled)
18957
+ return false;
18958
+ const d = result.data;
18959
+ return d.unifiedStatus === "allowed" || d.sevenDayOiStatus === "allowed";
18960
+ }
18961
+ function isModelTierWalled(opts) {
18962
+ const { mark, now, snapshotTierAllowed } = opts;
18963
+ if (snapshotTierAllowed === true)
18964
+ return false;
18965
+ if (mark !== undefined && mark.premium_walled_until > now)
18966
+ return true;
18967
+ return false;
18968
+ }
18969
+ var MODEL_TIER_WALL_DEFAULT_MS = 5 * 60 * 60 * 1000;
18970
+
18971
+ // src/auth/broker/usage-ledger.ts
18972
+ var STANDARD_WALL_PCT = 99.5;
18973
+ var USAGE_RING_CAP_DEFAULT = 288;
18974
+ function resolveUsageRingCap(raw) {
18975
+ if (raw == null || raw.trim().length === 0)
18976
+ return USAGE_RING_CAP_DEFAULT;
18977
+ const n = Number(raw);
18978
+ return Number.isFinite(n) && n >= 1 ? Math.floor(n) : USAGE_RING_CAP_DEFAULT;
18979
+ }
18980
+ function standardSampleFromResult(result, now) {
18981
+ if (!result.ok)
18982
+ return null;
18983
+ const d = result.data;
18984
+ const fivePresent = d.fiveHourUtilPresent !== false;
18985
+ const sevenPresent = d.sevenDayUtilPresent !== false;
18986
+ if (!fivePresent && !sevenPresent)
18987
+ return null;
18988
+ const wins = [];
18989
+ if (fivePresent) {
18990
+ wins.push({
18991
+ util: d.fiveHourUtilizationPct,
18992
+ reset: d.fiveHourResetAt?.getTime() ?? null
18993
+ });
18994
+ }
18995
+ if (sevenPresent) {
18996
+ wins.push({
18997
+ util: d.sevenDayUtilizationPct,
18998
+ reset: d.sevenDayResetAt?.getTime() ?? null
18999
+ });
19000
+ }
19001
+ const binding = wins.reduce((a, b) => b.util > a.util ? b : a);
19002
+ return {
19003
+ at: now,
19004
+ utilizationPct: binding.util,
19005
+ resetAt: binding.reset,
19006
+ status: d.unifiedStatus ?? null,
19007
+ walled: binding.util >= STANDARD_WALL_PCT
19008
+ };
19009
+ }
19010
+ function premiumSampleFromResult(result, now) {
19011
+ if (!result.ok)
19012
+ return null;
19013
+ const d = result.data;
19014
+ const hasSignal = d.unifiedStatus != null || d.sevenDayOiStatus != null || d.sevenDayOiUtilizationPct != null;
19015
+ if (!hasSignal)
19016
+ return null;
19017
+ const util3 = d.sevenDayOiUtilizationPct ?? null;
19018
+ const status = d.sevenDayOiStatus ?? d.unifiedStatus ?? null;
19019
+ const walled = d.sevenDayOiStatus === "rejected" || util3 != null && util3 >= MODEL_TIER_WALL_PCT;
19020
+ return {
19021
+ at: now,
19022
+ utilizationPct: util3,
19023
+ resetAt: d.sevenDayOiResetAt?.getTime() ?? null,
19024
+ status,
19025
+ walled
19026
+ };
19027
+ }
19028
+ function recordUsageSample(ledger, account, tier, sample, ringCap = USAGE_RING_CAP_DEFAULT) {
19029
+ if (sample == null)
19030
+ return ledger;
19031
+ const cap = ringCap >= 1 ? Math.floor(ringCap) : USAGE_RING_CAP_DEFAULT;
19032
+ const acct = ledger[account] ??= {};
19033
+ let tl = acct[tier];
19034
+ if (tl == null) {
19035
+ tl = {
19036
+ tier,
19037
+ samples: [],
19038
+ observations: 0,
19039
+ firstObservedAt: sample.at,
19040
+ lastObservedAt: sample.at
19041
+ };
19042
+ acct[tier] = tl;
19043
+ }
19044
+ tl.samples.push(sample);
19045
+ if (tl.samples.length > cap) {
19046
+ tl.samples.splice(0, tl.samples.length - cap);
19047
+ }
19048
+ tl.observations += 1;
19049
+ tl.lastObservedAt = sample.at;
19050
+ if (sample.at < tl.firstObservedAt)
19051
+ tl.firstObservedAt = sample.at;
19052
+ return ledger;
19053
+ }
19054
+ function summarizeTier(tl, now) {
19055
+ if (tl == null || tl.samples.length === 0)
19056
+ return null;
19057
+ const latest = tl.samples[tl.samples.length - 1];
19058
+ const refilled = latest.resetAt != null && latest.resetAt <= now;
19059
+ const rawUtil = latest.utilizationPct;
19060
+ const currentUtilizationPct = rawUtil == null ? null : refilled ? 0 : rawUtil;
19061
+ const headroomPct = currentUtilizationPct == null ? null : Math.max(0, 100 - currentUtilizationPct);
19062
+ const utilSamples = tl.samples.map((s) => s.utilizationPct).filter((u) => u != null);
19063
+ const peakUtilizationPct = utilSamples.length > 0 ? Math.max(...utilSamples) : null;
19064
+ return {
19065
+ tier: tl.tier,
19066
+ observations: tl.observations,
19067
+ retained: tl.samples.length,
19068
+ currentUtilizationPct,
19069
+ headroomPct,
19070
+ resetAt: latest.resetAt,
19071
+ status: latest.status,
19072
+ walled: latest.walled && !refilled,
19073
+ refilled,
19074
+ lastObservedAt: tl.lastObservedAt,
19075
+ peakUtilizationPct
19076
+ };
19077
+ }
19078
+ function summarizeAccountUsage(ledger, account, now) {
19079
+ const acct = ledger[account];
19080
+ return {
19081
+ account,
19082
+ standard: summarizeTier(acct?.standard, now),
19083
+ premium: summarizeTier(acct?.premium, now)
19084
+ };
19085
+ }
19086
+ function premiumHeadroomPct(ledger, account, now) {
19087
+ return summarizeAccountUsage(ledger, account, now).premium?.headroomPct ?? null;
19088
+ }
19089
+ function bestPremiumHeadroomAccount(ledger, candidates, now) {
19090
+ let best = null;
19091
+ let bestHeadroom = -Infinity;
19092
+ for (const cand of candidates) {
19093
+ const h = premiumHeadroomPct(ledger, cand, now);
19094
+ if (h == null)
19095
+ continue;
19096
+ if (h > bestHeadroom) {
19097
+ bestHeadroom = h;
19098
+ best = cand;
19099
+ }
19100
+ }
19101
+ return best;
19102
+ }
19103
+
18895
19104
  // src/auth/broker/server.ts
18896
19105
  init_atomic();
18897
19106
 
@@ -20056,7 +20265,11 @@ function cachedSnapshotToResult(s) {
20056
20265
  overageStatus: s.overageStatus,
20057
20266
  overageDisabledReason: s.overageDisabledReason,
20058
20267
  fiveHourUtilPresent: s.fiveHourUtilPresent,
20059
- sevenDayUtilPresent: s.sevenDayUtilPresent
20268
+ sevenDayUtilPresent: s.sevenDayUtilPresent,
20269
+ unifiedStatus: null,
20270
+ sevenDayOiStatus: null,
20271
+ sevenDayOiUtilizationPct: null,
20272
+ sevenDayOiResetAt: null
20060
20273
  }
20061
20274
  };
20062
20275
  }
@@ -20119,6 +20332,10 @@ class AuthBroker {
20119
20332
  thresholdViolations = {};
20120
20333
  notificationClaims = {};
20121
20334
  lastFleetRoll = null;
20335
+ lastTierQuotaCache = {};
20336
+ modelTierProbeModel = resolveModelTierProbeModel();
20337
+ usageLedger = {};
20338
+ usageRingCap = resolveUsageRingCap(process.env.SWITCHROOM_USAGE_LEDGER_RING_CAP);
20122
20339
  lastWrittenExpiresAt = new Map;
20123
20340
  refreshInFlight = new Set;
20124
20341
  consumerLastSeen = {};
@@ -20584,6 +20801,112 @@ class AuthBroker {
20584
20801
  allowOverage: this.isOverageAllowed(account)
20585
20802
  });
20586
20803
  }
20804
+ premiumWallMarkOf(account) {
20805
+ const q = this.quota[account];
20806
+ if (!q || q.premium_walled_until === undefined)
20807
+ return;
20808
+ return {
20809
+ premium_walled_until: q.premium_walled_until,
20810
+ premium_wall_bucket: q.premium_wall_bucket,
20811
+ premium_wall_marked_at: q.premium_wall_marked_at
20812
+ };
20813
+ }
20814
+ isAccountPremiumWalled(account) {
20815
+ const mark = this.premiumWallMarkOf(account);
20816
+ const tier = this.lastTierQuotaCache[account];
20817
+ const tierAllowed = tier != null && tier.capturedAt >= (mark?.premium_wall_marked_at ?? 0) && this.now() - tier.capturedAt <= SNAPSHOT_STALE_AGE_MS && (tier.unifiedStatus === "allowed" || tier.sevenDayOiStatus === "allowed") && !(tier.sevenDayOiStatus === "rejected");
20818
+ return isModelTierWalled({
20819
+ mark,
20820
+ now: this.now(),
20821
+ snapshotTierAllowed: tierAllowed ? true : undefined
20822
+ });
20823
+ }
20824
+ recordTierProbe(label, result) {
20825
+ if (result.ok) {
20826
+ const d = result.data;
20827
+ this.lastTierQuotaCache[label] = {
20828
+ unifiedStatus: d.unifiedStatus ?? null,
20829
+ sevenDayOiStatus: d.sevenDayOiStatus ?? null,
20830
+ sevenDayOiUtilizationPct: d.sevenDayOiUtilizationPct ?? null,
20831
+ sevenDayOiResetAt: d.sevenDayOiResetAt?.toISOString() ?? null,
20832
+ capturedAt: this.now()
20833
+ };
20834
+ this.persistLastTierQuotaCache();
20835
+ this.recordUsage(label, "premium", premiumSampleFromResult(result, this.now()));
20836
+ }
20837
+ const entry = this.quota[label];
20838
+ if (quotaTierAllowed(result) && entry?.premium_walled_until !== undefined) {
20839
+ const { premium_walled_until, premium_wall_bucket, premium_wall_marked_at, ...rest } = entry;
20840
+ this.quota[label] = rest;
20841
+ this.persistQuota();
20842
+ process.stdout.write(`auth-broker: premium canary shows ${label} flagship-tier allowed — cleared model-tier wall mark
20843
+ `);
20844
+ this.fanoutToAffectedConsumers(label);
20845
+ }
20846
+ if (quotaTierAllowed(result))
20847
+ this.clearPremiumAllWalled();
20848
+ }
20849
+ nextPremiumEligibleAccount(from) {
20850
+ const order = this.config.auth?.fallback_order ?? [];
20851
+ const start = order.indexOf(from);
20852
+ const ring = start === -1 ? [...order] : order.map((_, i) => order[(start + 1 + i) % order.length]);
20853
+ const eligible = [];
20854
+ for (const cand of ring) {
20855
+ if (!cand || cand === from)
20856
+ continue;
20857
+ if (this.isAccountExhausted(cand))
20858
+ continue;
20859
+ if (this.isAccountPremiumWalled(cand))
20860
+ continue;
20861
+ if (!accountExists(cand, this.home))
20862
+ continue;
20863
+ if (!readAccountCredentials(cand, this.home))
20864
+ continue;
20865
+ eligible.push(cand);
20866
+ }
20867
+ if (eligible.length === 0)
20868
+ return null;
20869
+ return bestPremiumHeadroomAccount(this.usageLedger, eligible, this.now()) ?? eligible[0];
20870
+ }
20871
+ earliestPremiumWallReset() {
20872
+ let earliest = null;
20873
+ for (const label of listAccounts(this.home)) {
20874
+ const until = this.quota[label]?.premium_walled_until;
20875
+ if (until != null && (earliest == null || until < earliest))
20876
+ earliest = until;
20877
+ }
20878
+ return earliest;
20879
+ }
20880
+ markPremiumWalledAndRoll(account, until, bucket) {
20881
+ const now = this.now();
20882
+ const premiumWalledUntil = until ?? now + MODEL_TIER_WALL_DEFAULT_MS;
20883
+ this.quota[account] = {
20884
+ ...this.quota[account],
20885
+ premium_walled_until: premiumWalledUntil,
20886
+ premium_wall_bucket: bucket ?? undefined,
20887
+ premium_wall_marked_at: now
20888
+ };
20889
+ this.persistQuota();
20890
+ const target = this.nextPremiumEligibleAccount(account);
20891
+ if (!target) {
20892
+ return { rolledTo: null, allWalled: true, earliestReset: this.earliestPremiumWallReset() };
20893
+ }
20894
+ this.fanoutFailoverTo(account, target);
20895
+ this.fanoutToAffectedConsumers(account);
20896
+ this.clearPremiumAllWalled();
20897
+ if (this.config.auth?.active === account) {
20898
+ this.config = {
20899
+ ...this.config,
20900
+ auth: { ...this.config.auth ?? {}, active: target }
20901
+ };
20902
+ this.persistActiveOverride(target);
20903
+ this.fanoutAllConsumers();
20904
+ this.audit({ op: "auto-promote-active", identity: { kind: "operator" }, account: target, accountKind: "claude", ok: true, reason: "model-tier-wall" });
20905
+ process.stdout.write(`auth-broker: auto-promoted auth.active ${account} → ${target} ` + `(${account} flagship-tier walled until ${new Date(premiumWalledUntil).toISOString()}, bucket=${bucket ?? "?"}) — persisted
20906
+ `);
20907
+ }
20908
+ return { rolledTo: target, allWalled: false, earliestReset: null };
20909
+ }
20587
20910
  softAvoidState = {};
20588
20911
  softAvoidRolledTo = {};
20589
20912
  softAvoidSnapshotOf(account) {
@@ -20762,7 +21085,12 @@ class AuthBroker {
20762
21085
  throttled_until: q?.throttled_until,
20763
21086
  threshold_violations: this.thresholdViolations[label] ?? 0,
20764
21087
  last_refreshed_at: meta?.lastRefreshedAt,
20765
- last_quota: lq ?? null
21088
+ last_quota: lq ?? null,
21089
+ premium_walled: this.isAccountPremiumWalled(label),
21090
+ premium_walled_until: q?.premium_walled_until,
21091
+ premium_wall_bucket: q?.premium_wall_bucket,
21092
+ last_tier_quota: this.lastTierQuotaCache[label] ?? null,
21093
+ usage_ledger: summarizeAccountUsage(this.usageLedger, label, this.now())
20766
21094
  };
20767
21095
  });
20768
21096
  const agents = Object.entries(this.config.agents ?? {}).map(([name, agent]) => {
@@ -20784,7 +21112,8 @@ class AuthBroker {
20784
21112
  agents,
20785
21113
  consumers,
20786
21114
  active_overage_serving,
20787
- last_fleet_roll: this.lastFleetRoll
21115
+ last_fleet_roll: this.lastFleetRoll,
21116
+ premium_tier_all_walled: this.premiumTierAllWalled
20788
21117
  }));
20789
21118
  }
20790
21119
  async opListGoogleAccounts(socket, id, identity) {
@@ -20870,6 +21199,7 @@ class AuthBroker {
20870
21199
  };
20871
21200
  this.lastQuotaCache[label] = snapshot;
20872
21201
  this.persistLastQuotaCache();
21202
+ this.recordUsage(label, "standard", standardSampleFromResult(result, this.now()));
20873
21203
  if (snapshotShouldClearMark(snapshot, this.exhaustionMarkOf(label), this.now())) {
20874
21204
  delete this.quota[label];
20875
21205
  this.persistQuota();
@@ -20880,6 +21210,9 @@ class AuthBroker {
20880
21210
  }
20881
21211
  async fleetQuotaProbeTick() {
20882
21212
  const probed = [];
21213
+ const tierWalled = new Map;
21214
+ const canarySet = this.premiumCanarySet();
21215
+ const canaryEnabled = process.env.SWITCHROOM_DISABLE_MODEL_TIER_PROBE !== "1";
20883
21216
  for (const label of listAccounts(this.home)) {
20884
21217
  const creds = readAccountCredentials(label, this.home);
20885
21218
  const token = creds?.claudeAiOauth?.accessToken;
@@ -20894,12 +21227,49 @@ class AuthBroker {
20894
21227
  }
20895
21228
  this.cacheQuotaSnapshot(label, result);
20896
21229
  probed.push({ label, result });
21230
+ if (canaryEnabled && canarySet.has(label)) {
21231
+ let tierResult;
21232
+ try {
21233
+ tierResult = await this.fetchQuotaImpl({ accessToken: token, model: this.modelTierProbeModel });
21234
+ } catch (err) {
21235
+ this.logErr(`fleet-tier-probe ${label}: ${err.message}`);
21236
+ continue;
21237
+ }
21238
+ this.recordTierProbe(label, tierResult);
21239
+ const wall = quotaIndicatesModelTierWall(tierResult);
21240
+ if (wall.walled)
21241
+ tierWalled.set(label, { until: wall.until, bucket: wall.bucket });
21242
+ }
20897
21243
  }
21244
+ for (const [label, wall] of tierWalled) {
21245
+ const now = this.now();
21246
+ this.quota[label] = {
21247
+ ...this.quota[label],
21248
+ premium_walled_until: wall.until ?? now + MODEL_TIER_WALL_DEFAULT_MS,
21249
+ premium_wall_bucket: wall.bucket ?? undefined,
21250
+ premium_wall_marked_at: now
21251
+ };
21252
+ }
21253
+ if (tierWalled.size > 0)
21254
+ this.persistQuota();
20898
21255
  for (const { label, result } of probed) {
20899
21256
  const active = this.config.auth?.active;
20900
21257
  const isPinnedAgentAccount = this.pinnedAgentAccounts().has(label);
20901
21258
  if (label !== active && !isPinnedAgentAccount)
20902
21259
  continue;
21260
+ const tw = tierWalled.get(label);
21261
+ if (tw) {
21262
+ process.stdout.write(`auth-broker: fleet-tier-probe shows ${label === active ? "ACTIVE" : "PINNED"} ${label} ` + `flagship-tier walled (bucket=${tw.bucket ?? "?"}) — model-tier failover
21263
+ `);
21264
+ this.audit({ op: "mark-exhausted", identity: { kind: "operator" }, account: label, accountKind: "claude", ok: true, reason: "model-tier-wall" });
21265
+ const { rolledTo, allWalled, earliestReset } = this.markPremiumWalledAndRoll(label, tw.until, tw.bucket);
21266
+ if (rolledTo && label === active) {
21267
+ this.recordFleetRoll(label, rolledTo, "model-tier-wall", tw.until ?? undefined, tw.bucket);
21268
+ } else if (allWalled && label === active) {
21269
+ this.recordPremiumAllWalled(tw.bucket, earliestReset);
21270
+ }
21271
+ continue;
21272
+ }
20903
21273
  const decision = quotaIndicatesExhaustion(result, this.isOverageAllowed(label));
20904
21274
  if (decision.exhausted) {
20905
21275
  process.stdout.write(`auth-broker: fleet-quota-probe shows ${label === active ? "ACTIVE" : "PINNED"} ${label} exhausted — proactive failover
@@ -20914,21 +21284,52 @@ class AuthBroker {
20914
21284
  this.softAvoidProbeRoll(label);
20915
21285
  }
20916
21286
  }
20917
- recordFleetRoll(from, to, reason, exhaustedUntil) {
21287
+ recordFleetRoll(from, to, reason, exhaustedUntil, bucket) {
20918
21288
  const s = this.lastQuotaCache[from];
20919
21289
  this.lastFleetRoll = {
20920
21290
  from,
20921
21291
  to,
20922
21292
  at: this.now(),
20923
21293
  reason,
21294
+ ...bucket ? { bucket } : {},
20924
21295
  ...exhaustedUntil != null ? { exhausted_until: exhaustedUntil } : {},
20925
- ...s ? {
21296
+ ...s && reason !== "model-tier-wall" ? {
20926
21297
  window: s.fiveHourUtilizationPct >= s.sevenDayUtilizationPct ? "5h" : "7d",
20927
21298
  pct: Math.max(s.fiveHourUtilizationPct, s.sevenDayUtilizationPct)
20928
21299
  } : {}
20929
21300
  };
20930
21301
  this.persistLastFleetRoll();
20931
21302
  }
21303
+ premiumTierAllWalled = null;
21304
+ recordPremiumAllWalled(bucket, earliestReset) {
21305
+ this.premiumTierAllWalled = {
21306
+ bucket,
21307
+ ...earliestReset != null ? { earliest_reset: earliestReset } : {},
21308
+ at: this.now()
21309
+ };
21310
+ atomicWriteJsonSync(join8(this.stateDir, "premium-tier-all-walled.json"), this.premiumTierAllWalled, 384);
21311
+ process.stdout.write(`auth-broker: ALL accounts flagship-tier walled (bucket=${bucket ?? "?"}` + `${earliestReset != null ? `, earliest reset ${new Date(earliestReset).toISOString()}` : ""}) ` + `— no premium-eligible failover target
21312
+ `);
21313
+ }
21314
+ clearPremiumAllWalled() {
21315
+ if (this.premiumTierAllWalled == null)
21316
+ return;
21317
+ this.premiumTierAllWalled = null;
21318
+ try {
21319
+ unlinkSync(join8(this.stateDir, "premium-tier-all-walled.json"));
21320
+ } catch {}
21321
+ }
21322
+ premiumCanarySet() {
21323
+ const set = new Set;
21324
+ const active = this.config.auth?.active;
21325
+ if (active)
21326
+ set.add(active);
21327
+ for (const cand of this.config.auth?.fallback_order ?? [])
21328
+ set.add(cand);
21329
+ for (const acct of this.pinnedAgentAccounts())
21330
+ set.add(acct);
21331
+ return set;
21332
+ }
20932
21333
  pinnedAgentAccounts() {
20933
21334
  return new Set(Object.values(this.config.agents ?? {}).map((a) => a.auth?.override).filter((x) => typeof x === "string" && x.length > 0));
20934
21335
  }
@@ -21838,6 +22239,9 @@ class AuthBroker {
21838
22239
  this.notificationClaims = this.readJson("notification-claims.json") ?? {};
21839
22240
  this.lastFleetRoll = this.readJson("last-fleet-roll.json") ?? null;
21840
22241
  this.lastQuotaCache = this.readJson("last-quota.json") ?? {};
22242
+ this.lastTierQuotaCache = this.readJson("last-tier-quota.json") ?? {};
22243
+ this.premiumTierAllWalled = this.readJson("premium-tier-all-walled.json") ?? null;
22244
+ this.usageLedger = this.readJson("usage-ledger.json") ?? {};
21841
22245
  this.applyActiveOverride();
21842
22246
  }
21843
22247
  applyActiveOverride() {
@@ -21891,6 +22295,22 @@ class AuthBroker {
21891
22295
  persistLastQuotaCache() {
21892
22296
  atomicWriteJsonSync(join8(this.stateDir, "last-quota.json"), this.lastQuotaCache, 384);
21893
22297
  }
22298
+ persistLastTierQuotaCache() {
22299
+ atomicWriteJsonSync(join8(this.stateDir, "last-tier-quota.json"), this.lastTierQuotaCache, 384);
22300
+ }
22301
+ persistUsageLedger() {
22302
+ atomicWriteJsonSync(join8(this.stateDir, "usage-ledger.json"), this.usageLedger, 384);
22303
+ }
22304
+ recordUsage(label, tier, sample) {
22305
+ if (sample == null)
22306
+ return;
22307
+ try {
22308
+ recordUsageSample(this.usageLedger, label, tier, sample, this.usageRingCap);
22309
+ this.persistUsageLedger();
22310
+ } catch (err) {
22311
+ this.logErr(`usage-ledger ${label}/${tier}: ${err.message}`);
22312
+ }
22313
+ }
21894
22314
  persistNotificationClaims() {
21895
22315
  atomicWriteJsonSync(join8(this.stateDir, "notification-claims.json"), this.notificationClaims, 384);
21896
22316
  }
@@ -22024,7 +22444,11 @@ class AuthBroker {
22024
22444
  shaIndex: { ...this.shaIndex },
22025
22445
  thresholdViolations: { ...this.thresholdViolations },
22026
22446
  listeners: [...this.listeners.keys()],
22027
- lastQuotaCache: structuredClone(this.lastQuotaCache)
22447
+ lastQuotaCache: structuredClone(this.lastQuotaCache),
22448
+ lastTierQuotaCache: structuredClone(this.lastTierQuotaCache),
22449
+ usageLedger: structuredClone(this.usageLedger),
22450
+ premiumTierAllWalled: this.premiumTierAllWalled ? { ...this.premiumTierAllWalled } : null,
22451
+ activeAccount: this.config.auth?.active
22028
22452
  };
22029
22453
  }
22030
22454
  _fanoutAll() {