switchroom 0.18.15 → 0.18.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/dist/agent-scheduler/index.js +3 -0
  2. package/dist/auth-broker/index.js +432 -10
  3. package/dist/cli/notion-write-pretool.mjs +3 -0
  4. package/dist/cli/switchroom.js +50 -1
  5. package/dist/host-control/main.js +4 -1
  6. package/dist/vault/approvals/kernel-server.js +3 -0
  7. package/dist/vault/broker/server.js +3 -0
  8. package/package.json +1 -1
  9. package/profiles/_base/start.sh.hbs +81 -139
  10. package/telegram-plugin/dist/gateway/gateway.js +386 -259
  11. package/telegram-plugin/draft-stream.ts +78 -3
  12. package/telegram-plugin/gateway/bridge-dead-watchdog.ts +3 -4
  13. package/telegram-plugin/gateway/effort-command.ts +9 -7
  14. package/telegram-plugin/gateway/gateway.ts +265 -220
  15. package/telegram-plugin/gateway/litellm-local-notice-wiring.ts +200 -0
  16. package/telegram-plugin/gateway/model-command.ts +96 -18
  17. package/telegram-plugin/gateway/pending-session-command.ts +10 -8
  18. package/telegram-plugin/gateway/session-model-file.ts +38 -172
  19. package/telegram-plugin/litellm-local-notice.ts +189 -0
  20. package/telegram-plugin/quota-watch.ts +16 -4
  21. package/telegram-plugin/runtime-metrics.ts +16 -0
  22. package/telegram-plugin/send-gate-degraded.test.ts +9 -7
  23. package/telegram-plugin/send-gate.ts +34 -4
  24. package/telegram-plugin/stream-controller.ts +143 -20
  25. package/telegram-plugin/stream-reply-handler.ts +12 -2
  26. package/telegram-plugin/tests/bot-api.harness.ts +7 -2
  27. package/telegram-plugin/tests/draft-stream.test.ts +110 -1
  28. package/telegram-plugin/tests/effort-command.test.ts +4 -4
  29. package/telegram-plugin/tests/flood-windows-persistence.test.ts +2 -2
  30. package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +33 -19
  31. package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +47 -127
  32. package/telegram-plugin/tests/litellm-local-notice.test.ts +417 -0
  33. package/telegram-plugin/tests/model-command.test.ts +84 -1
  34. package/telegram-plugin/tests/quota-watch.test.ts +21 -0
  35. package/telegram-plugin/tests/reaction-gate-routing.test.ts +2 -2
  36. package/telegram-plugin/tests/session-model-file.test.ts +7 -155
  37. package/telegram-plugin/tests/stream-controller-send-gate.test.ts +521 -0
  38. package/telegram-plugin/tests/stream-reply-handler.test.ts +44 -0
  39. package/telegram-plugin/tests/worker-activity-feed.test.ts +207 -0
  40. package/telegram-plugin/worker-activity-feed.ts +83 -8
@@ -11224,6 +11224,9 @@ var TelegramChannelSchema = exports_external.object({
11224
11224
  window_ms: exports_external.number().int().nonnegative().optional().describe("Sliding-window (ms) for merging consecutive inbound messages from " + "the same sender+topic into ONE Claude turn. Each new message resets " + "the timer; the turn starts once the sender pauses for this long. " + "Catches forwarded bursts, pasted text the Telegram client split " + "into several messages, and mixed text+media forwards. Default 500. " + "Set 0 to disable (every message becomes its own turn). Raise for " + "users who think in multiple short messages; the trade-off is the " + "single-message turn start is delayed by this much (the \uD83D\uDC40 ack still " + "fires immediately, so perceived latency is unchanged)."),
11225
11225
  max_attachments: exports_external.number().int().positive().optional().describe("Maximum number of media attachments carried into ONE coalesced " + "Claude turn. Default 10 — a full Telegram album (media_group caps " + "at 10) or a text+multi-image forwarded burst arrives as a single " + "turn; the agent sees numbered attachment fields (image_path, " + "image_path_2, …). Set 1 to restore the historical " + "single-attachment-per-turn behaviour. Excess attachments beyond " + "the cap spill into the next turn. Each attachment is downloaded, " + "so a high cap on a slow link delays turn start.")
11226
11226
  }).optional().describe("Inbound coalescing — how the gateway groups rapid consecutive messages " + "into a single turn so a forwarded album or split paste doesn't fan out " + "into N separate turns. Cascades from defaults.channels.telegram.coalesce."),
11227
+ litellm_notice: exports_external.object({
11228
+ window_ms: exports_external.number().int().positive().optional().describe("Per-agent cooldown window (ms) for the litellm-local 429 notice. " + "When the agent trips the LiteLLM proxy's OWN tpm_limit/rpm_limit " + "cap (a `litellm-local` classified 429 — see docs/auth.md § " + "LiteLLM-proxy-local 429s), the gateway posts ONE calm notice " + "naming the fleet token limiter, then counts further hits silently " + "for this long; the first notice after the window expires says " + "how many were absorbed. Default 900000 (15 min). Invalid values " + "fall back to the default.")
11229
+ }).optional().describe("Debounce tuning for the litellm-local throttle notice — the calm " + "'fleet token limiter engaged' message posted when the LiteLLM " + "proxy's own rate cap trips (never an Anthropic account limit). " + "Cascades from defaults.channels.telegram.litellm_notice."),
11227
11230
  interrupt: exports_external.object({
11228
11231
  safe_boundary: exports_external.boolean().optional().describe("When true (the default), a `!`-prefix interrupt that arrives while " + "the agent is mid-tool-call is DEFERRED: the SIGINT and the " + "replacement turn wait until the in-flight tool call finishes (a " + "clean boundary) instead of C-c'ing the agent mid-write/mid-bash. If " + "no tool is in flight the interrupt still fires immediately. Bounded " + "by max_wait_ms so a long tool never strands the user. Set false to " + "fire synchronously the moment `!` is received (historical " + "behaviour). Rapid repeated `!` while one is pending coalesce into a " + "single deferred interrupt carrying the latest body."),
11229
11232
  max_wait_ms: exports_external.number().int().positive().optional().describe("Upper bound (ms) the gateway waits for a safe boundary before firing " + "a deferred `!` interrupt anyway. Only consulted when safe_boundary is " + "true. Default 8000. Keep it short — the user explicitly asked to " + "interrupt, so a long in-flight tool shouldn't ghost them; the cap " + "trades a tiny risk of a mid-tool C-c for a guaranteed response.")
@@ -16853,6 +16853,9 @@ var TelegramChannelSchema = exports_external.object({
16853
16853
  window_ms: exports_external.number().int().nonnegative().optional().describe("Sliding-window (ms) for merging consecutive inbound messages from " + "the same sender+topic into ONE Claude turn. Each new message resets " + "the timer; the turn starts once the sender pauses for this long. " + "Catches forwarded bursts, pasted text the Telegram client split " + "into several messages, and mixed text+media forwards. Default 500. " + "Set 0 to disable (every message becomes its own turn). Raise for " + "users who think in multiple short messages; the trade-off is the " + "single-message turn start is delayed by this much (the \uD83D\uDC40 ack still " + "fires immediately, so perceived latency is unchanged)."),
16854
16854
  max_attachments: exports_external.number().int().positive().optional().describe("Maximum number of media attachments carried into ONE coalesced " + "Claude turn. Default 10 — a full Telegram album (media_group caps " + "at 10) or a text+multi-image forwarded burst arrives as a single " + "turn; the agent sees numbered attachment fields (image_path, " + "image_path_2, …). Set 1 to restore the historical " + "single-attachment-per-turn behaviour. Excess attachments beyond " + "the cap spill into the next turn. Each attachment is downloaded, " + "so a high cap on a slow link delays turn start.")
16855
16855
  }).optional().describe("Inbound coalescing — how the gateway groups rapid consecutive messages " + "into a single turn so a forwarded album or split paste doesn't fan out " + "into N separate turns. Cascades from defaults.channels.telegram.coalesce."),
16856
+ litellm_notice: exports_external.object({
16857
+ window_ms: exports_external.number().int().positive().optional().describe("Per-agent cooldown window (ms) for the litellm-local 429 notice. " + "When the agent trips the LiteLLM proxy's OWN tpm_limit/rpm_limit " + "cap (a `litellm-local` classified 429 — see docs/auth.md § " + "LiteLLM-proxy-local 429s), the gateway posts ONE calm notice " + "naming the fleet token limiter, then counts further hits silently " + "for this long; the first notice after the window expires says " + "how many were absorbed. Default 900000 (15 min). Invalid values " + "fall back to the default.")
16858
+ }).optional().describe("Debounce tuning for the litellm-local throttle notice — the calm " + "'fleet token limiter engaged' message posted when the LiteLLM " + "proxy's own rate cap trips (never an Anthropic account limit). " + "Cascades from defaults.channels.telegram.litellm_notice."),
16856
16859
  interrupt: exports_external.object({
16857
16860
  safe_boundary: exports_external.boolean().optional().describe("When true (the default), a `!`-prefix interrupt that arrives while " + "the agent is mid-tool-call is DEFERRED: the SIGINT and the " + "replacement turn wait until the in-flight tool call finishes (a " + "clean boundary) instead of C-c'ing the agent mid-write/mid-bash. If " + "no tool is in flight the interrupt still fires immediately. Bounded " + "by max_wait_ms so a long tool never strands the user. Set false to " + "fire synchronously the moment `!` is received (historical " + "behaviour). Rapid repeated `!` while one is pending coalesce into a " + "single deferred interrupt carrying the latest body."),
16858
16861
  max_wait_ms: exports_external.number().int().positive().optional().describe("Upper bound (ms) the gateway waits for a safe boundary before firing " + "a deferred `!` interrupt anyway. Only consulted when safe_boundary is " + "true. Default 8000. Keep it short — the user explicitly asked to " + "interrupt, so a long in-flight tool shouldn't ghost them; the cap " + "trades a tiny risk of a mid-tool C-c for a guaranteed response.")
@@ -18653,6 +18656,14 @@ var BIND_MOUNT_EXACT_SOURCE_DENY = new Set(["/var/run/docker.sock"]);
18653
18656
  var OAUTH_BETA = "oauth-2025-04-20";
18654
18657
  var DEFAULT_USER_AGENT = "claude-cli/1.0.0 (external, cli)";
18655
18658
  var DEFAULT_PROBE_MODEL = "claude-haiku-4-5-20251001";
18659
+ var DEFAULT_MODEL_TIER_PROBE_MODEL = "claude-fable-5";
18660
+ function resolveModelTierProbeModel(env = process.env) {
18661
+ const raw = env.SWITCHROOM_MODEL_TIER_PROBE_MODEL;
18662
+ if (typeof raw === "string" && raw.trim().length > 0)
18663
+ return raw.trim();
18664
+ return DEFAULT_MODEL_TIER_PROBE_MODEL;
18665
+ }
18666
+ var MODEL_TIER_REPRESENTATIVE_CLAIM = "seven_day_overage_included";
18656
18667
  function parseFloatHeader(headers, name) {
18657
18668
  const v = headers.get(name);
18658
18669
  if (v == null || v.trim().length === 0)
@@ -18669,15 +18680,37 @@ function parseEpochHeader(headers, name) {
18669
18680
  return null;
18670
18681
  return new Date(n * 1000);
18671
18682
  }
18672
- function parseQuotaHeaders(headers) {
18683
+ function parseTierReset(headers, now) {
18684
+ const oi = parseEpochHeader(headers, "anthropic-ratelimit-unified-7d_oi-reset");
18685
+ if (oi != null)
18686
+ return oi;
18687
+ const unified = parseEpochHeader(headers, "anthropic-ratelimit-unified-reset");
18688
+ if (unified != null)
18689
+ return unified;
18690
+ const ra = headers.get("retry-after");
18691
+ if (ra != null) {
18692
+ const secs = Number(ra);
18693
+ if (Number.isFinite(secs) && secs > 0 && secs < 30 * 24 * 3600) {
18694
+ return new Date(now.getTime() + secs * 1000);
18695
+ }
18696
+ }
18697
+ return null;
18698
+ }
18699
+ function parseQuotaHeaders(headers, now = new Date) {
18673
18700
  const fiveHour = parseFloatHeader(headers, "anthropic-ratelimit-unified-5h-utilization");
18674
18701
  const sevenDay = parseFloatHeader(headers, "anthropic-ratelimit-unified-7d-utilization");
18675
- if (fiveHour == null && sevenDay == null) {
18702
+ const unifiedStatus = headers.get("anthropic-ratelimit-unified-status");
18703
+ const sevenDayOiStatus = headers.get("anthropic-ratelimit-unified-7d_oi-status");
18704
+ const sevenDayOiUtil = parseFloatHeader(headers, "anthropic-ratelimit-unified-7d_oi-utilization");
18705
+ const representativeClaim = headers.get("anthropic-ratelimit-unified-representative-claim");
18706
+ const hasTierSignal = unifiedStatus != null || sevenDayOiStatus != null || sevenDayOiUtil != null;
18707
+ if (fiveHour == null && sevenDay == null && !hasTierSignal) {
18676
18708
  return {
18677
18709
  ok: false,
18678
18710
  reason: "no unified rate-limit headers in response (API token, not OAuth?)"
18679
18711
  };
18680
18712
  }
18713
+ const oiPresent = sevenDayOiStatus != null || sevenDayOiUtil != null;
18681
18714
  return {
18682
18715
  ok: true,
18683
18716
  data: {
@@ -18687,9 +18720,14 @@ function parseQuotaHeaders(headers) {
18687
18720
  sevenDayUtilPresent: sevenDay != null,
18688
18721
  fiveHourResetAt: parseEpochHeader(headers, "anthropic-ratelimit-unified-5h-reset"),
18689
18722
  sevenDayResetAt: parseEpochHeader(headers, "anthropic-ratelimit-unified-7d-reset"),
18690
- representativeClaim: headers.get("anthropic-ratelimit-unified-representative-claim"),
18723
+ representativeClaim,
18691
18724
  overageStatus: headers.get("anthropic-ratelimit-unified-overage-status"),
18692
- overageDisabledReason: headers.get("anthropic-ratelimit-unified-overage-disabled-reason")
18725
+ overageDisabledReason: headers.get("anthropic-ratelimit-unified-overage-disabled-reason"),
18726
+ unifiedStatus,
18727
+ sevenDayOiStatus,
18728
+ sevenDayOiUtilizationPct: sevenDayOiUtil != null ? sevenDayOiUtil * 100 : null,
18729
+ sevenDayOiResetAt: parseTierReset(headers, now),
18730
+ sevenDayOiPresent: oiPresent
18693
18731
  }
18694
18732
  };
18695
18733
  }
@@ -18894,6 +18932,175 @@ function resolveConsumerProbeIntervalMs(env) {
18894
18932
  return DEFAULT_CONSUMER_PROBE_INTERVAL_MS;
18895
18933
  }
18896
18934
 
18935
+ // src/auth/broker/model-tier-quota.ts
18936
+ var MODEL_TIER_WALL_PCT = 99.5;
18937
+ function quotaIndicatesModelTierWall(result) {
18938
+ const none = { walled: false, until: null, bucket: null };
18939
+ if (!result.ok)
18940
+ return none;
18941
+ const d = result.data;
18942
+ const statusRejected = d.sevenDayOiStatus === "rejected";
18943
+ const claimRejected = d.unifiedStatus === "rejected" && d.representativeClaim === MODEL_TIER_REPRESENTATIVE_CLAIM;
18944
+ const utilWalled = d.sevenDayOiUtilizationPct != null && d.sevenDayOiUtilizationPct >= MODEL_TIER_WALL_PCT;
18945
+ if (!statusRejected && !claimRejected && !utilWalled)
18946
+ return none;
18947
+ return {
18948
+ walled: true,
18949
+ until: d.sevenDayOiResetAt?.getTime() ?? null,
18950
+ bucket: MODEL_TIER_REPRESENTATIVE_CLAIM
18951
+ };
18952
+ }
18953
+ function quotaTierAllowed(result) {
18954
+ if (!result.ok)
18955
+ return false;
18956
+ if (quotaIndicatesModelTierWall(result).walled)
18957
+ return false;
18958
+ const d = result.data;
18959
+ return d.unifiedStatus === "allowed" || d.sevenDayOiStatus === "allowed";
18960
+ }
18961
+ function isModelTierWalled(opts) {
18962
+ const { mark, now, snapshotTierAllowed } = opts;
18963
+ if (snapshotTierAllowed === true)
18964
+ return false;
18965
+ if (mark !== undefined && mark.premium_walled_until > now)
18966
+ return true;
18967
+ return false;
18968
+ }
18969
+ var MODEL_TIER_WALL_DEFAULT_MS = 5 * 60 * 60 * 1000;
18970
+
18971
+ // src/auth/broker/usage-ledger.ts
18972
+ var STANDARD_WALL_PCT = 99.5;
18973
+ var USAGE_RING_CAP_DEFAULT = 288;
18974
+ function resolveUsageRingCap(raw) {
18975
+ if (raw == null || raw.trim().length === 0)
18976
+ return USAGE_RING_CAP_DEFAULT;
18977
+ const n = Number(raw);
18978
+ return Number.isFinite(n) && n >= 1 ? Math.floor(n) : USAGE_RING_CAP_DEFAULT;
18979
+ }
18980
+ function standardSampleFromResult(result, now) {
18981
+ if (!result.ok)
18982
+ return null;
18983
+ const d = result.data;
18984
+ const fivePresent = d.fiveHourUtilPresent !== false;
18985
+ const sevenPresent = d.sevenDayUtilPresent !== false;
18986
+ if (!fivePresent && !sevenPresent)
18987
+ return null;
18988
+ const wins = [];
18989
+ if (fivePresent) {
18990
+ wins.push({
18991
+ util: d.fiveHourUtilizationPct,
18992
+ reset: d.fiveHourResetAt?.getTime() ?? null
18993
+ });
18994
+ }
18995
+ if (sevenPresent) {
18996
+ wins.push({
18997
+ util: d.sevenDayUtilizationPct,
18998
+ reset: d.sevenDayResetAt?.getTime() ?? null
18999
+ });
19000
+ }
19001
+ const binding = wins.reduce((a, b) => b.util > a.util ? b : a);
19002
+ return {
19003
+ at: now,
19004
+ utilizationPct: binding.util,
19005
+ resetAt: binding.reset,
19006
+ status: d.unifiedStatus ?? null,
19007
+ walled: binding.util >= STANDARD_WALL_PCT
19008
+ };
19009
+ }
19010
+ function premiumSampleFromResult(result, now) {
19011
+ if (!result.ok)
19012
+ return null;
19013
+ const d = result.data;
19014
+ const hasSignal = d.unifiedStatus != null || d.sevenDayOiStatus != null || d.sevenDayOiUtilizationPct != null;
19015
+ if (!hasSignal)
19016
+ return null;
19017
+ const util3 = d.sevenDayOiUtilizationPct ?? null;
19018
+ const status = d.sevenDayOiStatus ?? d.unifiedStatus ?? null;
19019
+ const walled = d.sevenDayOiStatus === "rejected" || util3 != null && util3 >= MODEL_TIER_WALL_PCT;
19020
+ return {
19021
+ at: now,
19022
+ utilizationPct: util3,
19023
+ resetAt: d.sevenDayOiResetAt?.getTime() ?? null,
19024
+ status,
19025
+ walled
19026
+ };
19027
+ }
19028
+ function recordUsageSample(ledger, account, tier, sample, ringCap = USAGE_RING_CAP_DEFAULT) {
19029
+ if (sample == null)
19030
+ return ledger;
19031
+ const cap = ringCap >= 1 ? Math.floor(ringCap) : USAGE_RING_CAP_DEFAULT;
19032
+ const acct = ledger[account] ??= {};
19033
+ let tl = acct[tier];
19034
+ if (tl == null) {
19035
+ tl = {
19036
+ tier,
19037
+ samples: [],
19038
+ observations: 0,
19039
+ firstObservedAt: sample.at,
19040
+ lastObservedAt: sample.at
19041
+ };
19042
+ acct[tier] = tl;
19043
+ }
19044
+ tl.samples.push(sample);
19045
+ if (tl.samples.length > cap) {
19046
+ tl.samples.splice(0, tl.samples.length - cap);
19047
+ }
19048
+ tl.observations += 1;
19049
+ tl.lastObservedAt = sample.at;
19050
+ if (sample.at < tl.firstObservedAt)
19051
+ tl.firstObservedAt = sample.at;
19052
+ return ledger;
19053
+ }
19054
+ function summarizeTier(tl, now) {
19055
+ if (tl == null || tl.samples.length === 0)
19056
+ return null;
19057
+ const latest = tl.samples[tl.samples.length - 1];
19058
+ const refilled = latest.resetAt != null && latest.resetAt <= now;
19059
+ const rawUtil = latest.utilizationPct;
19060
+ const currentUtilizationPct = rawUtil == null ? null : refilled ? 0 : rawUtil;
19061
+ const headroomPct = currentUtilizationPct == null ? null : Math.max(0, 100 - currentUtilizationPct);
19062
+ const utilSamples = tl.samples.map((s) => s.utilizationPct).filter((u) => u != null);
19063
+ const peakUtilizationPct = utilSamples.length > 0 ? Math.max(...utilSamples) : null;
19064
+ return {
19065
+ tier: tl.tier,
19066
+ observations: tl.observations,
19067
+ retained: tl.samples.length,
19068
+ currentUtilizationPct,
19069
+ headroomPct,
19070
+ resetAt: latest.resetAt,
19071
+ status: latest.status,
19072
+ walled: latest.walled && !refilled,
19073
+ refilled,
19074
+ lastObservedAt: tl.lastObservedAt,
19075
+ peakUtilizationPct
19076
+ };
19077
+ }
19078
+ function summarizeAccountUsage(ledger, account, now) {
19079
+ const acct = ledger[account];
19080
+ return {
19081
+ account,
19082
+ standard: summarizeTier(acct?.standard, now),
19083
+ premium: summarizeTier(acct?.premium, now)
19084
+ };
19085
+ }
19086
+ function premiumHeadroomPct(ledger, account, now) {
19087
+ return summarizeAccountUsage(ledger, account, now).premium?.headroomPct ?? null;
19088
+ }
19089
+ function bestPremiumHeadroomAccount(ledger, candidates, now) {
19090
+ let best = null;
19091
+ let bestHeadroom = -Infinity;
19092
+ for (const cand of candidates) {
19093
+ const h = premiumHeadroomPct(ledger, cand, now);
19094
+ if (h == null)
19095
+ continue;
19096
+ if (h > bestHeadroom) {
19097
+ bestHeadroom = h;
19098
+ best = cand;
19099
+ }
19100
+ }
19101
+ return best;
19102
+ }
19103
+
18897
19104
  // src/auth/broker/server.ts
18898
19105
  init_atomic();
18899
19106
 
@@ -20058,7 +20265,11 @@ function cachedSnapshotToResult(s) {
20058
20265
  overageStatus: s.overageStatus,
20059
20266
  overageDisabledReason: s.overageDisabledReason,
20060
20267
  fiveHourUtilPresent: s.fiveHourUtilPresent,
20061
- sevenDayUtilPresent: s.sevenDayUtilPresent
20268
+ sevenDayUtilPresent: s.sevenDayUtilPresent,
20269
+ unifiedStatus: null,
20270
+ sevenDayOiStatus: null,
20271
+ sevenDayOiUtilizationPct: null,
20272
+ sevenDayOiResetAt: null
20062
20273
  }
20063
20274
  };
20064
20275
  }
@@ -20121,6 +20332,10 @@ class AuthBroker {
20121
20332
  thresholdViolations = {};
20122
20333
  notificationClaims = {};
20123
20334
  lastFleetRoll = null;
20335
+ lastTierQuotaCache = {};
20336
+ modelTierProbeModel = resolveModelTierProbeModel();
20337
+ usageLedger = {};
20338
+ usageRingCap = resolveUsageRingCap(process.env.SWITCHROOM_USAGE_LEDGER_RING_CAP);
20124
20339
  lastWrittenExpiresAt = new Map;
20125
20340
  refreshInFlight = new Set;
20126
20341
  consumerLastSeen = {};
@@ -20586,6 +20801,112 @@ class AuthBroker {
20586
20801
  allowOverage: this.isOverageAllowed(account)
20587
20802
  });
20588
20803
  }
20804
+ premiumWallMarkOf(account) {
20805
+ const q = this.quota[account];
20806
+ if (!q || q.premium_walled_until === undefined)
20807
+ return;
20808
+ return {
20809
+ premium_walled_until: q.premium_walled_until,
20810
+ premium_wall_bucket: q.premium_wall_bucket,
20811
+ premium_wall_marked_at: q.premium_wall_marked_at
20812
+ };
20813
+ }
20814
+ isAccountPremiumWalled(account) {
20815
+ const mark = this.premiumWallMarkOf(account);
20816
+ const tier = this.lastTierQuotaCache[account];
20817
+ const tierAllowed = tier != null && tier.capturedAt >= (mark?.premium_wall_marked_at ?? 0) && this.now() - tier.capturedAt <= SNAPSHOT_STALE_AGE_MS && (tier.unifiedStatus === "allowed" || tier.sevenDayOiStatus === "allowed") && !(tier.sevenDayOiStatus === "rejected");
20818
+ return isModelTierWalled({
20819
+ mark,
20820
+ now: this.now(),
20821
+ snapshotTierAllowed: tierAllowed ? true : undefined
20822
+ });
20823
+ }
20824
+ recordTierProbe(label, result) {
20825
+ if (result.ok) {
20826
+ const d = result.data;
20827
+ this.lastTierQuotaCache[label] = {
20828
+ unifiedStatus: d.unifiedStatus ?? null,
20829
+ sevenDayOiStatus: d.sevenDayOiStatus ?? null,
20830
+ sevenDayOiUtilizationPct: d.sevenDayOiUtilizationPct ?? null,
20831
+ sevenDayOiResetAt: d.sevenDayOiResetAt?.toISOString() ?? null,
20832
+ capturedAt: this.now()
20833
+ };
20834
+ this.persistLastTierQuotaCache();
20835
+ this.recordUsage(label, "premium", premiumSampleFromResult(result, this.now()));
20836
+ }
20837
+ const entry = this.quota[label];
20838
+ if (quotaTierAllowed(result) && entry?.premium_walled_until !== undefined) {
20839
+ const { premium_walled_until, premium_wall_bucket, premium_wall_marked_at, ...rest } = entry;
20840
+ this.quota[label] = rest;
20841
+ this.persistQuota();
20842
+ process.stdout.write(`auth-broker: premium canary shows ${label} flagship-tier allowed — cleared model-tier wall mark
20843
+ `);
20844
+ this.fanoutToAffectedConsumers(label);
20845
+ }
20846
+ if (quotaTierAllowed(result))
20847
+ this.clearPremiumAllWalled();
20848
+ }
20849
+ nextPremiumEligibleAccount(from) {
20850
+ const order = this.config.auth?.fallback_order ?? [];
20851
+ const start = order.indexOf(from);
20852
+ const ring = start === -1 ? [...order] : order.map((_, i) => order[(start + 1 + i) % order.length]);
20853
+ const eligible = [];
20854
+ for (const cand of ring) {
20855
+ if (!cand || cand === from)
20856
+ continue;
20857
+ if (this.isAccountExhausted(cand))
20858
+ continue;
20859
+ if (this.isAccountPremiumWalled(cand))
20860
+ continue;
20861
+ if (!accountExists(cand, this.home))
20862
+ continue;
20863
+ if (!readAccountCredentials(cand, this.home))
20864
+ continue;
20865
+ eligible.push(cand);
20866
+ }
20867
+ if (eligible.length === 0)
20868
+ return null;
20869
+ return bestPremiumHeadroomAccount(this.usageLedger, eligible, this.now()) ?? eligible[0];
20870
+ }
20871
+ earliestPremiumWallReset() {
20872
+ let earliest = null;
20873
+ for (const label of listAccounts(this.home)) {
20874
+ const until = this.quota[label]?.premium_walled_until;
20875
+ if (until != null && (earliest == null || until < earliest))
20876
+ earliest = until;
20877
+ }
20878
+ return earliest;
20879
+ }
20880
+ markPremiumWalledAndRoll(account, until, bucket) {
20881
+ const now = this.now();
20882
+ const premiumWalledUntil = until ?? now + MODEL_TIER_WALL_DEFAULT_MS;
20883
+ this.quota[account] = {
20884
+ ...this.quota[account],
20885
+ premium_walled_until: premiumWalledUntil,
20886
+ premium_wall_bucket: bucket ?? undefined,
20887
+ premium_wall_marked_at: now
20888
+ };
20889
+ this.persistQuota();
20890
+ const target = this.nextPremiumEligibleAccount(account);
20891
+ if (!target) {
20892
+ return { rolledTo: null, allWalled: true, earliestReset: this.earliestPremiumWallReset() };
20893
+ }
20894
+ this.fanoutFailoverTo(account, target);
20895
+ this.fanoutToAffectedConsumers(account);
20896
+ this.clearPremiumAllWalled();
20897
+ if (this.config.auth?.active === account) {
20898
+ this.config = {
20899
+ ...this.config,
20900
+ auth: { ...this.config.auth ?? {}, active: target }
20901
+ };
20902
+ this.persistActiveOverride(target);
20903
+ this.fanoutAllConsumers();
20904
+ this.audit({ op: "auto-promote-active", identity: { kind: "operator" }, account: target, accountKind: "claude", ok: true, reason: "model-tier-wall" });
20905
+ process.stdout.write(`auth-broker: auto-promoted auth.active ${account} → ${target} ` + `(${account} flagship-tier walled until ${new Date(premiumWalledUntil).toISOString()}, bucket=${bucket ?? "?"}) — persisted
20906
+ `);
20907
+ }
20908
+ return { rolledTo: target, allWalled: false, earliestReset: null };
20909
+ }
20589
20910
  softAvoidState = {};
20590
20911
  softAvoidRolledTo = {};
20591
20912
  softAvoidSnapshotOf(account) {
@@ -20764,7 +21085,12 @@ class AuthBroker {
20764
21085
  throttled_until: q?.throttled_until,
20765
21086
  threshold_violations: this.thresholdViolations[label] ?? 0,
20766
21087
  last_refreshed_at: meta?.lastRefreshedAt,
20767
- last_quota: lq ?? null
21088
+ last_quota: lq ?? null,
21089
+ premium_walled: this.isAccountPremiumWalled(label),
21090
+ premium_walled_until: q?.premium_walled_until,
21091
+ premium_wall_bucket: q?.premium_wall_bucket,
21092
+ last_tier_quota: this.lastTierQuotaCache[label] ?? null,
21093
+ usage_ledger: summarizeAccountUsage(this.usageLedger, label, this.now())
20768
21094
  };
20769
21095
  });
20770
21096
  const agents = Object.entries(this.config.agents ?? {}).map(([name, agent]) => {
@@ -20786,7 +21112,8 @@ class AuthBroker {
20786
21112
  agents,
20787
21113
  consumers,
20788
21114
  active_overage_serving,
20789
- last_fleet_roll: this.lastFleetRoll
21115
+ last_fleet_roll: this.lastFleetRoll,
21116
+ premium_tier_all_walled: this.premiumTierAllWalled
20790
21117
  }));
20791
21118
  }
20792
21119
  async opListGoogleAccounts(socket, id, identity) {
@@ -20872,6 +21199,7 @@ class AuthBroker {
20872
21199
  };
20873
21200
  this.lastQuotaCache[label] = snapshot;
20874
21201
  this.persistLastQuotaCache();
21202
+ this.recordUsage(label, "standard", standardSampleFromResult(result, this.now()));
20875
21203
  if (snapshotShouldClearMark(snapshot, this.exhaustionMarkOf(label), this.now())) {
20876
21204
  delete this.quota[label];
20877
21205
  this.persistQuota();
@@ -20882,6 +21210,9 @@ class AuthBroker {
20882
21210
  }
20883
21211
  async fleetQuotaProbeTick() {
20884
21212
  const probed = [];
21213
+ const tierWalled = new Map;
21214
+ const canarySet = this.premiumCanarySet();
21215
+ const canaryEnabled = process.env.SWITCHROOM_DISABLE_MODEL_TIER_PROBE !== "1";
20885
21216
  for (const label of listAccounts(this.home)) {
20886
21217
  const creds = readAccountCredentials(label, this.home);
20887
21218
  const token = creds?.claudeAiOauth?.accessToken;
@@ -20896,12 +21227,49 @@ class AuthBroker {
20896
21227
  }
20897
21228
  this.cacheQuotaSnapshot(label, result);
20898
21229
  probed.push({ label, result });
21230
+ if (canaryEnabled && canarySet.has(label)) {
21231
+ let tierResult;
21232
+ try {
21233
+ tierResult = await this.fetchQuotaImpl({ accessToken: token, model: this.modelTierProbeModel });
21234
+ } catch (err) {
21235
+ this.logErr(`fleet-tier-probe ${label}: ${err.message}`);
21236
+ continue;
21237
+ }
21238
+ this.recordTierProbe(label, tierResult);
21239
+ const wall = quotaIndicatesModelTierWall(tierResult);
21240
+ if (wall.walled)
21241
+ tierWalled.set(label, { until: wall.until, bucket: wall.bucket });
21242
+ }
21243
+ }
21244
+ for (const [label, wall] of tierWalled) {
21245
+ const now = this.now();
21246
+ this.quota[label] = {
21247
+ ...this.quota[label],
21248
+ premium_walled_until: wall.until ?? now + MODEL_TIER_WALL_DEFAULT_MS,
21249
+ premium_wall_bucket: wall.bucket ?? undefined,
21250
+ premium_wall_marked_at: now
21251
+ };
20899
21252
  }
21253
+ if (tierWalled.size > 0)
21254
+ this.persistQuota();
20900
21255
  for (const { label, result } of probed) {
20901
21256
  const active = this.config.auth?.active;
20902
21257
  const isPinnedAgentAccount = this.pinnedAgentAccounts().has(label);
20903
21258
  if (label !== active && !isPinnedAgentAccount)
20904
21259
  continue;
21260
+ const tw = tierWalled.get(label);
21261
+ if (tw) {
21262
+ process.stdout.write(`auth-broker: fleet-tier-probe shows ${label === active ? "ACTIVE" : "PINNED"} ${label} ` + `flagship-tier walled (bucket=${tw.bucket ?? "?"}) — model-tier failover
21263
+ `);
21264
+ this.audit({ op: "mark-exhausted", identity: { kind: "operator" }, account: label, accountKind: "claude", ok: true, reason: "model-tier-wall" });
21265
+ const { rolledTo, allWalled, earliestReset } = this.markPremiumWalledAndRoll(label, tw.until, tw.bucket);
21266
+ if (rolledTo && label === active) {
21267
+ this.recordFleetRoll(label, rolledTo, "model-tier-wall", tw.until ?? undefined, tw.bucket);
21268
+ } else if (allWalled && label === active) {
21269
+ this.recordPremiumAllWalled(tw.bucket, earliestReset);
21270
+ }
21271
+ continue;
21272
+ }
20905
21273
  const decision = quotaIndicatesExhaustion(result, this.isOverageAllowed(label));
20906
21274
  if (decision.exhausted) {
20907
21275
  process.stdout.write(`auth-broker: fleet-quota-probe shows ${label === active ? "ACTIVE" : "PINNED"} ${label} exhausted — proactive failover
@@ -20916,21 +21284,52 @@ class AuthBroker {
20916
21284
  this.softAvoidProbeRoll(label);
20917
21285
  }
20918
21286
  }
20919
- recordFleetRoll(from, to, reason, exhaustedUntil) {
21287
+ recordFleetRoll(from, to, reason, exhaustedUntil, bucket) {
20920
21288
  const s = this.lastQuotaCache[from];
20921
21289
  this.lastFleetRoll = {
20922
21290
  from,
20923
21291
  to,
20924
21292
  at: this.now(),
20925
21293
  reason,
21294
+ ...bucket ? { bucket } : {},
20926
21295
  ...exhaustedUntil != null ? { exhausted_until: exhaustedUntil } : {},
20927
- ...s ? {
21296
+ ...s && reason !== "model-tier-wall" ? {
20928
21297
  window: s.fiveHourUtilizationPct >= s.sevenDayUtilizationPct ? "5h" : "7d",
20929
21298
  pct: Math.max(s.fiveHourUtilizationPct, s.sevenDayUtilizationPct)
20930
21299
  } : {}
20931
21300
  };
20932
21301
  this.persistLastFleetRoll();
20933
21302
  }
21303
+ premiumTierAllWalled = null;
21304
+ recordPremiumAllWalled(bucket, earliestReset) {
21305
+ this.premiumTierAllWalled = {
21306
+ bucket,
21307
+ ...earliestReset != null ? { earliest_reset: earliestReset } : {},
21308
+ at: this.now()
21309
+ };
21310
+ atomicWriteJsonSync(join8(this.stateDir, "premium-tier-all-walled.json"), this.premiumTierAllWalled, 384);
21311
+ process.stdout.write(`auth-broker: ALL accounts flagship-tier walled (bucket=${bucket ?? "?"}` + `${earliestReset != null ? `, earliest reset ${new Date(earliestReset).toISOString()}` : ""}) ` + `— no premium-eligible failover target
21312
+ `);
21313
+ }
21314
+ clearPremiumAllWalled() {
21315
+ if (this.premiumTierAllWalled == null)
21316
+ return;
21317
+ this.premiumTierAllWalled = null;
21318
+ try {
21319
+ unlinkSync(join8(this.stateDir, "premium-tier-all-walled.json"));
21320
+ } catch {}
21321
+ }
21322
+ premiumCanarySet() {
21323
+ const set = new Set;
21324
+ const active = this.config.auth?.active;
21325
+ if (active)
21326
+ set.add(active);
21327
+ for (const cand of this.config.auth?.fallback_order ?? [])
21328
+ set.add(cand);
21329
+ for (const acct of this.pinnedAgentAccounts())
21330
+ set.add(acct);
21331
+ return set;
21332
+ }
20934
21333
  pinnedAgentAccounts() {
20935
21334
  return new Set(Object.values(this.config.agents ?? {}).map((a) => a.auth?.override).filter((x) => typeof x === "string" && x.length > 0));
20936
21335
  }
@@ -21840,6 +22239,9 @@ class AuthBroker {
21840
22239
  this.notificationClaims = this.readJson("notification-claims.json") ?? {};
21841
22240
  this.lastFleetRoll = this.readJson("last-fleet-roll.json") ?? null;
21842
22241
  this.lastQuotaCache = this.readJson("last-quota.json") ?? {};
22242
+ this.lastTierQuotaCache = this.readJson("last-tier-quota.json") ?? {};
22243
+ this.premiumTierAllWalled = this.readJson("premium-tier-all-walled.json") ?? null;
22244
+ this.usageLedger = this.readJson("usage-ledger.json") ?? {};
21843
22245
  this.applyActiveOverride();
21844
22246
  }
21845
22247
  applyActiveOverride() {
@@ -21893,6 +22295,22 @@ class AuthBroker {
21893
22295
  persistLastQuotaCache() {
21894
22296
  atomicWriteJsonSync(join8(this.stateDir, "last-quota.json"), this.lastQuotaCache, 384);
21895
22297
  }
22298
+ persistLastTierQuotaCache() {
22299
+ atomicWriteJsonSync(join8(this.stateDir, "last-tier-quota.json"), this.lastTierQuotaCache, 384);
22300
+ }
22301
+ persistUsageLedger() {
22302
+ atomicWriteJsonSync(join8(this.stateDir, "usage-ledger.json"), this.usageLedger, 384);
22303
+ }
22304
+ recordUsage(label, tier, sample) {
22305
+ if (sample == null)
22306
+ return;
22307
+ try {
22308
+ recordUsageSample(this.usageLedger, label, tier, sample, this.usageRingCap);
22309
+ this.persistUsageLedger();
22310
+ } catch (err) {
22311
+ this.logErr(`usage-ledger ${label}/${tier}: ${err.message}`);
22312
+ }
22313
+ }
21896
22314
  persistNotificationClaims() {
21897
22315
  atomicWriteJsonSync(join8(this.stateDir, "notification-claims.json"), this.notificationClaims, 384);
21898
22316
  }
@@ -22026,7 +22444,11 @@ class AuthBroker {
22026
22444
  shaIndex: { ...this.shaIndex },
22027
22445
  thresholdViolations: { ...this.thresholdViolations },
22028
22446
  listeners: [...this.listeners.keys()],
22029
- lastQuotaCache: structuredClone(this.lastQuotaCache)
22447
+ lastQuotaCache: structuredClone(this.lastQuotaCache),
22448
+ lastTierQuotaCache: structuredClone(this.lastTierQuotaCache),
22449
+ usageLedger: structuredClone(this.usageLedger),
22450
+ premiumTierAllWalled: this.premiumTierAllWalled ? { ...this.premiumTierAllWalled } : null,
22451
+ activeAccount: this.config.auth?.active
22030
22452
  };
22031
22453
  }
22032
22454
  _fanoutAll() {
@@ -11982,6 +11982,9 @@ var TelegramChannelSchema = exports_external.object({
11982
11982
  window_ms: exports_external.number().int().nonnegative().optional().describe("Sliding-window (ms) for merging consecutive inbound messages from " + "the same sender+topic into ONE Claude turn. Each new message resets " + "the timer; the turn starts once the sender pauses for this long. " + "Catches forwarded bursts, pasted text the Telegram client split " + "into several messages, and mixed text+media forwards. Default 500. " + "Set 0 to disable (every message becomes its own turn). Raise for " + "users who think in multiple short messages; the trade-off is the " + "single-message turn start is delayed by this much (the \uD83D\uDC40 ack still " + "fires immediately, so perceived latency is unchanged)."),
11983
11983
  max_attachments: exports_external.number().int().positive().optional().describe("Maximum number of media attachments carried into ONE coalesced " + "Claude turn. Default 10 \u2014 a full Telegram album (media_group caps " + "at 10) or a text+multi-image forwarded burst arrives as a single " + "turn; the agent sees numbered attachment fields (image_path, " + "image_path_2, \u2026). Set 1 to restore the historical " + "single-attachment-per-turn behaviour. Excess attachments beyond " + "the cap spill into the next turn. Each attachment is downloaded, " + "so a high cap on a slow link delays turn start.")
11984
11984
  }).optional().describe("Inbound coalescing \u2014 how the gateway groups rapid consecutive messages " + "into a single turn so a forwarded album or split paste doesn't fan out " + "into N separate turns. Cascades from defaults.channels.telegram.coalesce."),
11985
+ litellm_notice: exports_external.object({
11986
+ window_ms: exports_external.number().int().positive().optional().describe("Per-agent cooldown window (ms) for the litellm-local 429 notice. " + "When the agent trips the LiteLLM proxy's OWN tpm_limit/rpm_limit " + "cap (a `litellm-local` classified 429 \u2014 see docs/auth.md \u00a7 " + "LiteLLM-proxy-local 429s), the gateway posts ONE calm notice " + "naming the fleet token limiter, then counts further hits silently " + "for this long; the first notice after the window expires says " + "how many were absorbed. Default 900000 (15 min). Invalid values " + "fall back to the default.")
11987
+ }).optional().describe("Debounce tuning for the litellm-local throttle notice \u2014 the calm " + "'fleet token limiter engaged' message posted when the LiteLLM " + "proxy's own rate cap trips (never an Anthropic account limit). " + "Cascades from defaults.channels.telegram.litellm_notice."),
11985
11988
  interrupt: exports_external.object({
11986
11989
  safe_boundary: exports_external.boolean().optional().describe("When true (the default), a `!`-prefix interrupt that arrives while " + "the agent is mid-tool-call is DEFERRED: the SIGINT and the " + "replacement turn wait until the in-flight tool call finishes (a " + "clean boundary) instead of C-c'ing the agent mid-write/mid-bash. If " + "no tool is in flight the interrupt still fires immediately. Bounded " + "by max_wait_ms so a long tool never strands the user. Set false to " + "fire synchronously the moment `!` is received (historical " + "behaviour). Rapid repeated `!` while one is pending coalesce into a " + "single deferred interrupt carrying the latest body."),
11987
11990
  max_wait_ms: exports_external.number().int().positive().optional().describe("Upper bound (ms) the gateway waits for a safe boundary before firing " + "a deferred `!` interrupt anyway. Only consulted when safe_boundary is " + "true. Default 8000. Keep it short \u2014 the user explicitly asked to " + "interrupt, so a long in-flight tool shouldn't ghost them; the cap " + "trades a tiny risk of a mid-tool C-c for a guaranteed response.")