switchroom 0.18.15 → 0.18.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/dist/agent-scheduler/index.js +16 -0
  2. package/dist/auth-broker/index.js +445 -10
  3. package/dist/cli/notion-write-pretool.mjs +16 -0
  4. package/dist/cli/switchroom.js +654 -479
  5. package/dist/host-control/main.js +20 -1
  6. package/dist/vault/approvals/kernel-server.js +16 -0
  7. package/dist/vault/broker/server.js +16 -0
  8. package/package.json +1 -1
  9. package/profiles/_base/start.sh.hbs +81 -139
  10. package/telegram-plugin/bridge/bridge.ts +7 -1
  11. package/telegram-plugin/dist/bridge/bridge.js +26 -1
  12. package/telegram-plugin/dist/gateway/gateway.js +1758 -661
  13. package/telegram-plugin/dist/server.js +26 -1
  14. package/telegram-plugin/draft-stream.ts +78 -3
  15. package/telegram-plugin/fleet-fallback-resume.ts +26 -3
  16. package/telegram-plugin/gateway/approval-hold.ts +49 -0
  17. package/telegram-plugin/gateway/bridge-dead-watchdog.ts +64 -22
  18. package/telegram-plugin/gateway/effort-command.ts +9 -7
  19. package/telegram-plugin/gateway/gateway.ts +627 -291
  20. package/telegram-plugin/gateway/linear-activity.ts +20 -4
  21. package/telegram-plugin/gateway/litellm-local-notice-wiring.ts +200 -0
  22. package/telegram-plugin/gateway/model-command.ts +96 -18
  23. package/telegram-plugin/gateway/pending-session-command.ts +10 -8
  24. package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
  25. package/telegram-plugin/gateway/session-model-file.ts +141 -172
  26. package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
  27. package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
  28. package/telegram-plugin/litellm-local-notice.ts +189 -0
  29. package/telegram-plugin/llm-error-present.ts +436 -0
  30. package/telegram-plugin/operator-events.ts +7 -1
  31. package/telegram-plugin/permission-title.ts +172 -10
  32. package/telegram-plugin/premium-recovery.ts +101 -0
  33. package/telegram-plugin/quota-watch.ts +16 -4
  34. package/telegram-plugin/raw-error-scrub.ts +73 -0
  35. package/telegram-plugin/retry-api-call.ts +8 -2
  36. package/telegram-plugin/runtime-metrics.ts +16 -0
  37. package/telegram-plugin/send-gate-degraded.test.ts +161 -8
  38. package/telegram-plugin/send-gate-observability.test.ts +140 -0
  39. package/telegram-plugin/send-gate-observability.ts +65 -20
  40. package/telegram-plugin/send-gate.test.ts +143 -1
  41. package/telegram-plugin/send-gate.ts +246 -23
  42. package/telegram-plugin/session-tail.ts +16 -0
  43. package/telegram-plugin/shared/local-time.ts +69 -0
  44. package/telegram-plugin/stream-controller.ts +143 -20
  45. package/telegram-plugin/stream-reply-handler.ts +12 -2
  46. package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
  47. package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
  48. package/telegram-plugin/tests/bot-api.harness.ts +7 -2
  49. package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
  50. package/telegram-plugin/tests/draft-stream.test.ts +110 -1
  51. package/telegram-plugin/tests/effort-command.test.ts +4 -4
  52. package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
  53. package/telegram-plugin/tests/flood-windows-persistence.test.ts +5 -4
  54. package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +33 -19
  55. package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +47 -127
  56. package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
  57. package/telegram-plugin/tests/litellm-local-notice.test.ts +417 -0
  58. package/telegram-plugin/tests/llm-error-present.test.ts +380 -0
  59. package/telegram-plugin/tests/model-command.test.ts +84 -1
  60. package/telegram-plugin/tests/permission-title.test.ts +167 -4
  61. package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
  62. package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
  63. package/telegram-plugin/tests/quota-watch.test.ts +21 -0
  64. package/telegram-plugin/tests/reaction-gate-routing.test.ts +8 -3
  65. package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
  66. package/telegram-plugin/tests/session-model-file.test.ts +7 -155
  67. package/telegram-plugin/tests/stream-controller-send-gate.test.ts +521 -0
  68. package/telegram-plugin/tests/stream-reply-handler.test.ts +44 -0
  69. package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
  70. package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
  71. package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
  72. package/telegram-plugin/tests/worker-activity-feed.test.ts +212 -2
  73. package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
  74. package/telegram-plugin/tier-downgrade.ts +198 -0
  75. package/telegram-plugin/tool-activity-summary.ts +99 -0
  76. package/telegram-plugin/worker-activity-feed.ts +543 -368
@@ -184,6 +184,7 @@ import {
184
184
  selectHeldForRedelivery,
185
185
  holdReasonFor,
186
186
  heldRetryBackoffMs,
187
+ applyDeliveredHoldReset,
187
188
  type UndeliverableMark,
188
189
  } from './approval-hold.js'
189
190
  import { isTelegramReplyTool, isTelegramSurfaceTool } from '../tool-names.js'
@@ -207,7 +208,7 @@ import {
207
208
  isPhotoDimensionRejectError,
208
209
  isFloodWaitActiveError,
209
210
  } from '../retry-api-call.js'
210
- import { createSendGate, sendGateEnabledFromEnv } from '../send-gate.js'
211
+ import { createSendGate, sendGateConfigFromEnv } from '../send-gate.js'
211
212
  import { createStatsLogger, createFloodWindowObserver } from '../send-gate-observability.js'
212
213
  import { classifyPhotoFile, rerouteResultSuffix } from '../photo-precheck.js'
213
214
  import { installTgPostLogger, withTgPostTags } from '../shared/bot-runtime.js'
@@ -314,6 +315,11 @@ import {
314
315
  type OperatorEventKind,
315
316
  } from '../operator-events.js'
316
317
  import { recordOperatorEvent } from '../operator-events-history.js'
318
+ import {
319
+ parseLlmError,
320
+ renderLlmError,
321
+ decideErrorSurface,
322
+ } from '../llm-error-present.js'
317
323
  import {
318
324
  formatModelUnavailableCard,
319
325
  resolveModelUnavailableFromOperatorEvent,
@@ -326,6 +332,8 @@ import {
326
332
  throttleRetryInPlaceMaxMs,
327
333
  } from '../throttle-tier.js'
328
334
  import { createThrottleTierRunner } from './throttle-tier-wiring.js'
335
+ import { parseLitellmNoticeWindowMs } from '../litellm-local-notice.js'
336
+ import { createLitellmLocalNoticeRunner, decideRateLimitedSurface } from './litellm-local-notice-wiring.js'
329
337
  import { runFleetAutoFallback, renderFallbackFailureNotice, evaluateFallbackFailureNotice, evaluateAllBlockedNotice, type FallbackFailureNoticeState, type FallbackAllBlockedNoticeState } from '../auto-fallback-fleet.js'
330
338
  import { startRestartWatchdog } from './restart-watchdog.js'
331
339
  import { validateStringArray } from './access-validator.js'
@@ -436,6 +444,8 @@ import { injectSlashCommand as injectSlashCommandImpl } from '../../src/agents/i
436
444
  import { handleInjectCommand, type InjectDeps } from './inject-handler.js'
437
445
  import {
438
446
  parseModelCommand,
447
+ planModelCommand,
448
+ modelCommandReceiptLine,
439
449
  handleModelCommand,
440
450
  buildModelMenu,
441
451
  handleModelMenuCallback,
@@ -461,19 +471,16 @@ import {
461
471
  readSessionModelFileRaw,
462
472
  restoreSessionModelFileRaw,
463
473
  clearSessionModelFile,
464
- clearSessionModelBootAttempts,
465
474
  readConfiguredDefaultModel,
466
- writeRelaunchModelIntent,
467
- clearRelaunchModelIntent,
468
- intentForRestartReason,
469
- readSessionModelFile,
470
- RELAUNCH_MODEL_INTENT_FILE,
471
- GATEWAY_SHUTDOWN_INTENT_REASON_PREFIX,
472
- clearStaleGatewayShutdownIntent,
473
475
  writeSessionEffortFile,
474
476
  clearSessionEffortFile,
475
- readSessionEffortFile,
477
+ writePremiumRecoveryFile,
478
+ readPremiumRecoveryFile,
479
+ clearPremiumRecoveryFile,
476
480
  } from './session-model-file.js'
481
+ import { runTierDowngrade } from './tier-downgrade-wiring.js'
482
+ import { runPremiumRecoveryPing } from './premium-recovery-wiring.js'
483
+ import { decidePremiumRecovery } from '../premium-recovery.js'
477
484
  import { discoverModels, selectModel } from '../../src/agents/model-picker.js'
478
485
  import { resolveMainModel, SWITCHROOM_DEFAULT_THINKING_EFFORT } from '../../src/agents/scaffold.js'
479
486
  import {
@@ -1097,16 +1104,10 @@ function triggerSelfRestart(
1097
1104
  )
1098
1105
  return false
1099
1106
  }
1100
- // Session-model stickiness (reference/rfcs/session-model-stickiness.md):
1101
- // boot default is REVERT, so every switchroom-managed bounce must stamp
1102
- // its intent BEFORE the SIGTERM is even scheduled (write-before-kill
1103
- // invariant pinned by gateway-session-model-relaunch.test.ts). The
1104
- // per-reason table classifies recovery/model-switch bounces as "keep"
1105
- // and the deliberate inline restart button as "revert".
1106
- {
1107
- const smDir = resolveAgentDirFromEnv()
1108
- if (smDir) writeRelaunchModelIntent(smDir, intentForRestartReason(reason), reason)
1109
- }
1107
+ // Session-scoped /model (reference/rfcs/session-model-stickiness.md §0.1):
1108
+ // a `.session-model` carrier is consume-once start.sh applies it on the
1109
+ // apply-relaunch and deletes it, so no boot needs a keep/revert intent. A
1110
+ // switchroom-managed bounce simply reverts to the configured default.
1110
1111
  process.stderr.write(
1111
1112
  `telegram gateway: restart-via-SIGTERM-PID1 agent=${targetAgent} reason=${reason} (docker)\n`,
1112
1113
  )
@@ -1118,10 +1119,6 @@ function triggerSelfRestart(
1118
1119
  return true
1119
1120
  }
1120
1121
  // Legacy systemd path.
1121
- if (targetAgent === selfAgent) {
1122
- const smDir = resolveAgentDirFromEnv()
1123
- if (smDir) writeRelaunchModelIntent(smDir, intentForRestartReason(reason), reason)
1124
- }
1125
1122
  process.stderr.write(
1126
1123
  `telegram gateway: restart-via-systemctl agent=${targetAgent} reason=${reason}\n`,
1127
1124
  )
@@ -1141,24 +1138,6 @@ function triggerSelfRestart(
1141
1138
  }
1142
1139
  }
1143
1140
 
1144
- // #3018 finding 4: a gateway-only bounce (supervisor relaunch, bare gateway
1145
- // unit restart) leaves the shutdown handler's deploy-survival keep-intent
1146
- // stamp on disk UNCONSUMED — start.sh only runs on a container-level boot.
1147
- // If this gateway boot still sees a gateway-shutdown-stamped intent, the
1148
- // preceding bounce was gateway-only: clear it so a genuine crash inside the
1149
- // 10-min freshness window can't be converted into a "keep" (crash-reverts
1150
- // policy intact). A real container stop/deploy consumes the file in start.sh
1151
- // before any gateway boots, so a legitimate deploy stamp is never touched;
1152
- // triggerSelfRestart / user-slash stamps use un-prefixed reasons.
1153
- {
1154
- const bootSmDir = resolveAgentDirFromEnv()
1155
- if (bootSmDir != null && clearStaleGatewayShutdownIntent(bootSmDir)) {
1156
- process.stderr.write(
1157
- 'telegram gateway: cleared stale gateway-shutdown relaunch-model intent (previous bounce was gateway-only — container never restarted)\n',
1158
- )
1159
- }
1160
- }
1161
-
1162
1141
  // Cached lazily — the claude CLI binary doesn't change inside a running
1163
1142
  // gateway process; on `switchroom update` the gateway restarts, refreshing this.
1164
1143
  let cachedClaudeCliVersion: string | null | undefined = undefined
@@ -1410,6 +1389,11 @@ type Access = {
1410
1389
  parseMode?: 'html' | 'markdownv2' | 'text'
1411
1390
  disableLinkPreview?: boolean
1412
1391
  coalescingGapMs?: number
1392
+ /** Cooldown window (ms) for the litellm-local 429 notice — the debounced
1393
+ * "fleet token limiter engaged" message (litellm-local-notice.ts). Default
1394
+ * 15 min when unset/invalid (parseLitellmNoticeWindowMs). Projected from
1395
+ * channels.telegram.litellm_notice.window_ms by scaffold. */
1396
+ litellmNoticeWindowMs?: number
1413
1397
  /** A2: max media attachments folded into one coalesced turn. Default 10
1414
1398
  * (a full Telegram album / forwarded burst arrives as one turn). Set 1 to
1415
1399
  * restore single-attachment behaviour. Projected from
@@ -1564,6 +1548,7 @@ function readAccessFile(): Access {
1564
1548
  parseMode: parsed.parseMode,
1565
1549
  disableLinkPreview: parsed.disableLinkPreview,
1566
1550
  coalescingGapMs: parsed.coalescingGapMs,
1551
+ litellmNoticeWindowMs: parsed.litellmNoticeWindowMs,
1567
1552
  coalesceMaxAttachments: parsed.coalesceMaxAttachments,
1568
1553
  interruptSafeBoundary: parsed.interruptSafeBoundary,
1569
1554
  interruptMaxWaitMs: parsed.interruptMaxWaitMs,
@@ -5495,8 +5480,14 @@ const recordFloodWindow = makeFloodWindowRecorder(FLOOD_WINDOWS_PATH)
5495
5480
  // ban never resends into an open window. onWindowOpen write-throughs every
5496
5481
  // runtime-opened window to FLOOD_WINDOWS_PATH; bootRamp starts the global
5497
5482
  // bucket at half capacity for 10s to absorb the boot-card burst.
5483
+ // Resolve enabled + the tunable rate limits (channels.telegram.send_gate.* →
5484
+ // SWITCHROOM_TG_SEND_GATE_* env) ONCE at boot. Unset knobs are absent, so
5485
+ // createSendGate applies SEND_GATE_DEFAULTS — omitting config = today's exact
5486
+ // behaviour. The SWITCHROOM_TELEGRAM_SEND_GATE break-glass valve still wins on
5487
+ // `enabled` when explicitly set (see sendGateConfigFromEnv precedence).
5488
+ const sendGateConfig = sendGateConfigFromEnv()
5498
5489
  const sendGate = createSendGate({
5499
- enabled: sendGateEnabledFromEnv(),
5490
+ ...sendGateConfig,
5500
5491
  initialWindows: loadInitialFloodWindows(FLOOD_STATE_PATH, FLOOD_WINDOWS_PATH, Date.now()),
5501
5492
  bootRamp: {},
5502
5493
  onWindowOpen: (scopeKey, untilTs) => recordFloodWindow(scopeKey, untilTs),
@@ -5509,15 +5500,20 @@ const sendGate = createSendGate({
5509
5500
  const probeFloodWaitRemainingMs = makeFloodWaitProbe(FLOOD_STATE_PATH)
5510
5501
  const rawRobustApiCall = createRetryApiCall({
5511
5502
  log: (line) => process.stderr.write(line),
5512
- onFloodWait: (retryAfterSec) => {
5503
+ onFloodWait: (retryAfterSec, opts) => {
5513
5504
  // #2923/#3094 — persist the single-object global window (probe reads this).
5514
5505
  makeFloodWaitRecorder(FLOOD_STATE_PATH)(retryAfterSec)
5515
- // #3084 PR 2 — also open a GLOBAL send-gate window so cosmetic traffic sheds
5516
- // for the ban's duration even on a SHORT (slept-and-retried) 429 that never
5517
- // throws FLOOD_WAIT_ACTIVE. Scope-precise windows are opened by the gate's
5518
- // own FLOOD_WAIT_ACTIVE catch (which has the call's opts).
5506
+ // #3084 PR 2 / #3111 — also open SCOPE-PRECISE send-gate window(s) so cosmetic
5507
+ // traffic sheds for the ban's duration even on a SHORT (slept-and-retried)
5508
+ // 429 that never throws FLOOD_WAIT_ACTIVE. The retry policy now passes the
5509
+ // call's `opts` (#3111) so this opens the FINEST scope the 429 implies —
5510
+ // `chat:`/`group:`/`msg-edit:` for a chat-bound call, `global` only when the
5511
+ // call carries no chat scope (genuinely global) — instead of a blanket
5512
+ // `global` window that would suppress unrelated chats. This mirrors the
5513
+ // gate's own FLOOD_WAIT_ACTIVE catch, which uses the same scope-precise
5514
+ // opener for LONG bans.
5519
5515
  try {
5520
- sendGate.openFloodWindow('global', Date.now() + Math.max(0, retryAfterSec) * 1000)
5516
+ sendGate.openScopedFloodWindows(opts, Date.now() + Math.max(0, retryAfterSec) * 1000)
5521
5517
  } catch {
5522
5518
  /* best-effort — never let the window hook break the retry path */
5523
5519
  }
@@ -5661,6 +5657,9 @@ const sendGateStatsLogger = createStatsLogger({
5661
5657
  const floodWindowObserver = createFloodWindowObserver({
5662
5658
  clock: { now: () => Date.now(), sleep: (ms) => new Promise((r) => setTimeout(r, ms)) },
5663
5659
  log: (line) => process.stderr.write(line),
5660
+ // Render the operator-facing flood alerts in the configured local timezone
5661
+ // (same env the config cascade bakes into every agent — see timezone.ts).
5662
+ tz: process.env.SWITCHROOM_TIMEZONE ?? process.env.TZ ?? 'UTC',
5664
5663
  stats: () => sendGate.stats(),
5665
5664
  readWindows: (now) => readFloodWindows(FLOOD_WINDOWS_PATH, now),
5666
5665
  markAlerted: (scopeKey, alertedAt) =>
@@ -5681,7 +5680,7 @@ const floodWindowObserver = createFloodWindowObserver({
5681
5680
  )
5682
5681
  },
5683
5682
  })
5684
- if (sendGateEnabledFromEnv()) {
5683
+ if (sendGateConfig.enabled) {
5685
5684
  const observeTimer = setInterval(() => {
5686
5685
  try {
5687
5686
  sendGateStatsLogger.tick()
@@ -6753,10 +6752,19 @@ function isAutoFallbackCooldownActive(_agentName: string, now: number): boolean
6753
6752
 
6754
6753
  async function editCardExpired(chatId: string, messageId: number | undefined, body: string): Promise<void> {
6755
6754
  if (messageId == null) return
6756
- await lockedBot.api
6757
- // allow-raw-bot-api: message-id-targeted edit (no thread to lose); best-effort card-expiry strip from the reaper (no grammy ctx). Dropping reply_markup strips the stale keyboard atomically with the text edit.
6758
- .editMessageText(chatId, messageId, richMessage(body), { reply_markup: { inline_keyboard: [] } })
6759
- .catch(() => {})
6755
+ // #3084 bypass audit: this best-effort card-expiry strip (reaper + lazy
6756
+ // sweeps, no grammy ctx) previously fired a RAW `lockedBot.api.editMessageText`
6757
+ // OUTSIDE the send gate a residual flood vector (a 429 here never reached
6758
+ // `onFloodWait`, so the breaker stayed blind). Routed through `robustApiCall`
6759
+ // (chat-lock → send-gate → retry/breaker) as a `cosmetic` edit carrying
6760
+ // messageId+editPayload so it paces/sheds under pressure and its 429 records
6761
+ // the flood window. Dropping reply_markup strips the stale keyboard atomically
6762
+ // with the text edit; message-id-targeted so no thread to lose.
6763
+ const text = richMessage(body)
6764
+ await robustApiCall(
6765
+ () => lockedBot.api.editMessageText(chatId, messageId, text, { reply_markup: { inline_keyboard: [] } }),
6766
+ { chat_id: chatId, verb: 'card-expired.strip', priorityClass: 'cosmetic', messageId, editPayload: body },
6767
+ ).catch(() => {})
6760
6768
  }
6761
6769
 
6762
6770
  function recordMissedApproval(opts: {
@@ -7100,20 +7108,13 @@ function postPermissionCard(
7100
7108
  const landedThreadId = sent.message_thread_id ?? undefined
7101
7109
  live.cards.push({ chatId, messageId: sent.message_id, threadId: landedThreadId })
7102
7110
  // The card LANDED — the operator can see and tap it, so the block is over.
7103
- // Drop the hold mark and reconcile the off-Telegram surface. (PR 3 also
7104
- // resets `startedAt` here: the TTL measures how long the operator had to
7105
- // answer, and until this moment they had nothing to answer.)
7106
- if (live.undeliverable != null) {
7107
- live.undeliverable = null
7108
- live.redeliveryFailures = 0
7109
- // RESET THE TTL CLOCK. Load-bearing, not cosmetic. `startedAt` is when
7110
- // the agent asked; the TTL measures how long the operator had to
7111
- // answer. Until this instant they had NOTHING to answer — the card did
7112
- // not exist in any chat. Without the reset, a card held through a 4.6h
7113
- // ban lands already-expired against a 60-min TTL and the very next
7114
- // reaper tick auto-denies it: we would have MOVED the silent denial,
7115
- // not removed it.
7116
- live.startedAt = Date.now()
7111
+ // Drop the hold mark, reset the TTL clock, and reconcile the off-Telegram
7112
+ // surface as ONE shared decision (`applyDeliveredHoldReset`, #3128) that
7113
+ // the outcome test's harness ALSO drives, so deleting the `startedAt` reset
7114
+ // turns the behavioural `(d2)` test red instead of only a fragile grep.
7115
+ // PR 3: the reset is load-bearing — the TTL measures how long the operator
7116
+ // had to answer, and until this moment they had nothing to answer.
7117
+ if (applyDeliveredHoldReset(live, Date.now())) {
7117
7118
  reconcileBlockedApprovals()
7118
7119
  process.stderr.write(
7119
7120
  `telegram gateway: permission-card RE-DELIVERED request=${requestId} ` +
@@ -7713,11 +7714,14 @@ function emitGatewayOperatorEvent(event: OperatorEvent): void {
7713
7714
  // account-scoped 429s with fleet TPM even when the card is suppressed.
7714
7715
  let throttleEscalation: ModelUnavailableDetection | null = null
7715
7716
  let escalationFired = false
7717
+ // True when decideRateLimitedSurface already consulted (and armed) the
7718
+ // shared per-kind card cooldown for this event — the gate below must not
7719
+ // re-consult it, or the arm from the first consult would self-suppress.
7720
+ let rateLimitedCooldownConsulted = false
7716
7721
  const rateLimit429Classification =
7717
7722
  kind === 'rate-limited' ? classify429Detail(event.detail) : null
7718
7723
  if (rateLimit429Classification != null && rateLimit429Classification !== 'account-scoped') {
7719
- // litellm-local / generic-transient: the existing calm rate-limited
7720
- // path (fall through to renderOperatorEvent below). NO broker
7724
+ // litellm-local / generic-transient: the calm path. NO broker
7721
7725
  // mark-throttled, NO throttle-tier runner, NO failover — for a
7722
7726
  // proxy-local cap trip those would bench an account that was never
7723
7727
  // touched.
@@ -7730,12 +7734,49 @@ function emitGatewayOperatorEvent(event: OperatorEvent): void {
7730
7734
  now: Date.now(),
7731
7735
  }),
7732
7736
  )
7733
- if (rateLimit429Classification === 'litellm-local') {
7737
+ // Surface decision extracted (decideRateLimitedSurface,
7738
+ // litellm-local-notice-wiring.ts) so the ordering contract with the
7739
+ // shared per-kind card cooldown is pinnable by tests: litellm-local
7740
+ // resolves BEFORE the gate (never arms `${agent}:rate-limited`, never
7741
+ // suppressed by a cooldown a recent 529/generic card armed);
7742
+ // generic-transient consults the gate exactly once HERE.
7743
+ const surface = decideRateLimitedSurface({
7744
+ classification: rateLimit429Classification,
7745
+ agent,
7746
+ shouldEmitCard: (a) => shouldEmitOperatorEvent(a, 'rate-limited'),
7747
+ })
7748
+ if (surface === 'litellm-local-notice') {
7734
7749
  process.stderr.write(
7735
7750
  `telegram gateway: 429 classified litellm-proxy-local agent=${agent} — ` +
7736
7751
  `calm path, no account attribution, no failover\n`,
7737
7752
  )
7753
+ // The dedicated debounced notice REPLACES the generic "🚦 Rate limited"
7754
+ // card for this classification only: one calm message naming the fleet
7755
+ // token limiter (LiteLLM tpm_limit/rpm_limit) instead of a card that
7756
+ // reads like an Anthropic account problem. Classification, quota-ledger,
7757
+ // and failover behavior are untouched (nothing fired above on this
7758
+ // branch). Record into history ONLY when a notice actually posted — a
7759
+ // suppressed low-stakes proxy throttle must not overwrite a more
7760
+ // important most-recent event (e.g. credentials-expired) in the
7761
+ // /status enrichment (operator-events-history keeps the most recent
7762
+ // event per agent).
7763
+ const outcome = litellmLocalNoticeRunner.onRateLimited('litellm-local', agent)
7764
+ if (outcome === 'sent') {
7765
+ try {
7766
+ recordOperatorEvent(event)
7767
+ } catch { /* history is best-effort */ }
7768
+ }
7769
+ return
7770
+ }
7771
+ if (surface === 'cooldown-suppressed') {
7772
+ process.stderr.write(
7773
+ `telegram gateway: operator-event suppressed (cooldown) agent=${agent} kind=${kind}\n`,
7774
+ )
7775
+ return
7738
7776
  }
7777
+ // 'generic-card' — the gate passed (and armed) above; fall through to
7778
+ // the existing calm rate-limited card without re-consulting it.
7779
+ rateLimitedCooldownConsulted = true
7739
7780
  }
7740
7781
  if (rateLimit429Classification === 'account-scoped') {
7741
7782
  const throttleDecision = decideThrottleTier({
@@ -7800,7 +7841,7 @@ function emitGatewayOperatorEvent(event: OperatorEvent): void {
7800
7841
  }
7801
7842
  }
7802
7843
 
7803
- if (!shouldEmitOperatorEvent(agent, kind)) {
7844
+ if (!rateLimitedCooldownConsulted && !shouldEmitOperatorEvent(agent, kind)) {
7804
7845
  process.stderr.write(
7805
7846
  `telegram gateway: operator-event suppressed (cooldown) agent=${agent} kind=${kind}\n`,
7806
7847
  )
@@ -7897,6 +7938,28 @@ function emitGatewayOperatorEvent(event: OperatorEvent): void {
7897
7938
  // the old account fails — honest reset messaging on the enriched card.
7898
7939
  void fireFleetAutoFallback(agent, untilMs, modelUnavailable.resetAt)
7899
7940
  }
7941
+ } else if (kind === 'rate-limited' || kind === 'unknown-5xx') {
7942
+ // #llm-error-surfacing — surface #2 (the "🚦 Rate limited" operator card).
7943
+ // The transient rate-limit / overload family used to render the raw
7944
+ // synthetic-error `detail` (bytes and all) via renderOperatorEvent. Route
7945
+ // it through the humanized card instead: JSON-stripped coreText + reset in
7946
+ // LOCAL time. The ErrorPresenceGate is the cross-surface dedup authority —
7947
+ // the reply/done-card surfaces are already suppressed at the transcript
7948
+ // source (session-tail), so the operator card is the sole renderer and
7949
+ // wins the claim; a redundant burst within the collapse window is
7950
+ // suppressed here in addition to the existing 5-min per-kind cooldown.
7951
+ const parsed = parseLlmError(event.detail)
7952
+ const now = Date.now()
7953
+ if (decideErrorSurface(parsed, agent, { claim: true, now }) === 'suppress') {
7954
+ process.stderr.write(
7955
+ `telegram gateway: operator-event collapsed (error-presence-gate) agent=${agent} kind=${kind}\n`,
7956
+ )
7957
+ return
7958
+ }
7959
+ const tz = process.env.SWITCHROOM_TIMEZONE ?? process.env.TZ ?? 'UTC'
7960
+ const r = renderLlmError(parsed, agent, tz, new Date(now))
7961
+ renderedText = r.text
7962
+ renderedKeyboard = r.keyboard
7900
7963
  } else {
7901
7964
  try {
7902
7965
  const r = renderOperatorEvent(event)
@@ -8565,6 +8628,17 @@ async function runMidSessionCardReaper(): Promise<void> {
8565
8628
  const reaps = decideWorkerPinReaps({
8566
8629
  pins: candidates,
8567
8630
  statusOf: (agentId) => {
8631
+ // #3207: coalesced feed pins are GROUP-level (`wk:group:<feedKey>`),
8632
+ // so `workerAgentIdOfPinKey` yields `group:<feedKey>`, not a real
8633
+ // jsonl agent id. Vouch for these off the live feed instead of the
8634
+ // registry: a group the feed still tracks is 'running' (exempt from
8635
+ // the TTL); a lingering group pin the feed no longer knows about is a
8636
+ // missed unpin → 'terminal' (reap now). The feed's own group-empty
8637
+ // unpin is the primary path; this is the missed-unpin backstop.
8638
+ if (agentId.startsWith('group:')) {
8639
+ const feedKey = agentId.slice('group:'.length)
8640
+ return workerActivityFeed?.hasRunningInFeed(feedKey) ? 'running' : 'terminal'
8641
+ }
8568
8642
  if (turnsDb == null) return 'unknown'
8569
8643
  try {
8570
8644
  const row = getSubagentByJsonlId(turnsDb, agentId)
@@ -8723,34 +8797,14 @@ async function reconcileStatusPinInner(
8723
8797
  }
8724
8798
  }
8725
8799
 
8726
- /**
8727
- * Background-worker desired-pin, driven off the live `🛠 Worker` message.
8728
- * Reads the worker feed's current message_id (the EXISTING message we pin
8729
- * what the feed already rendered, never a new send) and reconciles a silent
8730
- * pin while it's running / an unpin on completion. No-op until the feed has
8731
- * actually painted a message for this worker (trivial sub-second workers stay
8732
- * silent and are never pinned). Keyed `wk:<agentId>`.
8733
- */
8734
- function reconcileWorkerPin(
8735
- agentId: string,
8736
- chatId: string | null,
8737
- running: boolean,
8738
- ): void {
8739
- if (!PIN_STATUS_WHILE_WORKING) return
8740
- const key = `wk:${agentId}`
8741
- if (!running) {
8742
- // Unpin: recover the chat we pinned in (caller may not have it at
8743
- // completion). No-op when nothing was pinned for this worker.
8744
- const unpinChat = chatId ?? statusPinChatIds.get(key)
8745
- if (unpinChat == null) return
8746
- void reconcileStatusPin(key, unpinChat, { pinned: false })
8747
- return
8748
- }
8749
- if (chatId == null) return
8750
- const messageId = workerActivityFeed?.messageIdOf(agentId) ?? null
8751
- if (messageId == null) return // no message painted yet — nothing to pin
8752
- void reconcileStatusPin(key, chatId, { pinned: true, messageId })
8753
- }
8800
+ // #3207: the per-worker `reconcileWorkerPin(agentId, …)` (keyed `wk:<agentId>`)
8801
+ // was removed. Now that background workers COALESCE into one shared message per
8802
+ // chat/thread, the pin is driven at the GROUP level by the feed itself
8803
+ // (`reconcilePin` `wk:group:<feedKey>`, wired at createWorkerActivityFeed):
8804
+ // it pins when the group's first worker paints and unpins only when the group
8805
+ // empties. A per-worker unpin used to physically unpin a message a sibling
8806
+ // still needed, after which the survivor's re-pin NO-OP'd (its claim still
8807
+ // named that id) — leaving live work unpinned (review blocker).
8754
8808
 
8755
8809
  /** Unpin every owned status pin — used by the pre-restart sweep so a
8756
8810
  * crash / interrupt never leaves a permanent pin behind. Best-effort;
@@ -10334,6 +10388,13 @@ const bridgeDeadWatchdog = createBridgeDeadWatchdog({
10334
10388
  // consecutive-escalation count. At the cap, arm() stands down loudly
10335
10389
  // instead of restart-looping a deterministically-failing bridge.
10336
10390
  priorStreak: bridgeDeadPriorStreak,
10391
+ // #3086 — only THIS gateway's own primary bridge (registering under
10392
+ // $SWITCHROOM_AGENT_NAME) drives the watchdog. A secondary/relay client
10393
+ // (e.g. `overlord-relay`) registering a different name into this socket
10394
+ // is named + non-cron but must NOT mark the (still-alive) primary bridge
10395
+ // dead when it disconnects. Empty string ⟹ fall back to the pre-#3086
10396
+ // "any named non-cron client" test inside the watchdog.
10397
+ selfAgentName: process.env.SWITCHROOM_AGENT_NAME ?? '',
10337
10398
  })
10338
10399
  if (BRIDGE_DEAD_ESCALATION_ENABLED) {
10339
10400
  bridgeDeadWatchdog.arm()
@@ -10373,21 +10434,12 @@ const ipcServer: IpcServer = createIpcServer({
10373
10434
  : []
10374
10435
  // #3038 — a REAL (named, non-cron) bridge registered: stand the
10375
10436
  // bridge-dead watchdog down. Anonymous clients (recall.py, mcp
10376
- // handshakes) and cron-session bridges must NOT satisfy it — the
10377
- // watchdog gates on the identity INTERNALLY (isRealBridgeIdentity), so
10378
- // this call is safe wherever it sits relative to the cron early-return
10379
- // above (#3038 review finding 5).
10437
+ // handshakes), cron-session bridges, and secondary/relay clients
10438
+ // registering a name other than this gateway's own agent (#3086) must
10439
+ // NOT satisfy it the watchdog gates on the identity INTERNALLY
10440
+ // (isRealBridgeIdentity vs selfAgentName), so this call is safe wherever
10441
+ // it sits relative to the cron early-return above (#3038 review finding 5).
10380
10442
  bridgeDeadWatchdog.noteBridgeRegistered(client.agentName)
10381
- // #3043 item 2: a REAL bridge registering is proof the boot came all the
10382
- // way up healthy — clear start.sh's crashloop boot-attempts counter so only
10383
- // boots that genuinely fail BEFORE the bridge registers accumulate toward
10384
- // the 3-strike override clear. Without this, three quick operator
10385
- // hand-bounces of a healthy agent (each <150s apart) spuriously wipe a
10386
- // working model override. Best-effort; no-op when the file is absent.
10387
- if (client.agentName != null) {
10388
- const smBootDir = resolveAgentDirFromEnv()
10389
- if (smBootDir != null) clearSessionModelBootAttempts(smBootDir)
10390
- }
10391
10443
  client.send({ type: 'status', status: 'agent_connected' })
10392
10444
 
10393
10445
  // Phase 2b PR 3a — bridgeUp cutover. The state machine's `bridgeUp`
@@ -10580,8 +10632,10 @@ const ipcServer: IpcServer = createIpcServer({
10580
10632
  // #3038 — the real bridge went away mid-life. Re-arm the grace
10581
10633
  // window: a normal claude restart re-registers within seconds and
10582
10634
  // stands it down; a bridge that died for good escalates once (the
10583
- // once-per-boot fuse inside the watchdog caps it). Cron/anonymous
10584
- // identities are ignored inside the watchdog itself (finding 5).
10635
+ // once-per-boot fuse inside the watchdog caps it). Cron/anonymous and
10636
+ // secondary/relay identities (a name other than this gateway's own
10637
+ // agent, #3086) are ignored inside the watchdog itself (finding 5) —
10638
+ // so a transient relay disconnect never bounces a healthy container.
10585
10639
  if (BRIDGE_DEAD_ESCALATION_ENABLED) bridgeDeadWatchdog.noteBridgeDisconnected(client.agentName)
10586
10640
  }
10587
10641
 
@@ -21821,15 +21875,10 @@ function buildModelDeps(restartCtx?: ModelDepsRestartContext): ModelMenuDeps & M
21821
21875
  })
21822
21876
  }
21823
21877
  stampUserRestartReason(reason)
21824
- // Model-switch restarts are switchroom-managed relaunches: the session
21825
- // override (written by the caller before this dispatch) must survive
21826
- // the bounce, so stamp keep-intent BEFORE dispatch (boot default is
21827
- // revert). hostd shells through `switchroom agent restart`, which
21828
- // deliberately writes no intent of its own.
21829
- {
21830
- const smDir = resolveAgentDirFromEnv()
21831
- if (smDir) writeRelaunchModelIntent(smDir, 'keep', reason)
21832
- }
21878
+ // Model-switch restarts are the APPLY-relaunch for a `.session-model`
21879
+ // carrier the caller wrote immediately above: start.sh consumes it on
21880
+ // the very next boot (this dispatch's boot), then reverts thereafter.
21881
+ // No keep/revert intent is needed the carrier is consume-once.
21833
21882
  await sweepBeforeSelfRestart()
21834
21883
  const hostdResp = await tryHostdDispatch(name, {
21835
21884
  v: 1,
@@ -21850,14 +21899,11 @@ function buildModelDeps(restartCtx?: ModelDepsRestartContext): ModelMenuDeps & M
21850
21899
  return
21851
21900
  }
21852
21901
  // hostd is configured but returned an error/denied result. No restart
21853
- // is coming, so the keep-intent stamped above must not linger a
21854
- // crash within its 10-min freshness window would wrongly KEEP.
21902
+ // is coming, so the carrier the caller wrote must not linger to be
21903
+ // consumed by an unrelated later boot scheduleModelRelaunch's catch
21904
+ // rolls it back on this throw.
21855
21905
  if (hostdResp.result !== 'started' && hostdResp.result !== 'completed') {
21856
21906
  clearRestartMarker()
21857
- {
21858
- const smDir = resolveAgentDirFromEnv()
21859
- if (smDir) clearRelaunchModelIntent(smDir)
21860
- }
21861
21907
  throw new Error(
21862
21908
  `hostd restart failed (result=${hostdResp.result}): ${hostdResp.error ?? '(no details)'}`,
21863
21909
  )
@@ -21866,11 +21912,11 @@ function buildModelDeps(restartCtx?: ModelDepsRestartContext): ModelMenuDeps & M
21866
21912
  /**
21867
21913
  * Switch TO a model that needs a relaunch (sr-* LiteLLM/OpenRouter ids,
21868
21914
  * which claude's native `/model` picker rejects, and the sr-to-claude
21869
- * direction). Write the DURABLE `.session-model` override (start.sh
21870
- * applies it on every keep-relaunch boot and launches `claude --model
21871
- * <token>`), set the in-memory session-model so /status stays honest
21872
- * across the restart window, then run the SAME restart dispatch as
21873
- * scheduleRestart above which stamps the keep-intent this boot needs.
21915
+ * direction). Write the CONSUME-ONCE `.session-model` carrier (start.sh
21916
+ * applies it on the very next boot this dispatch's apply-relaunch — and
21917
+ * deletes it, so it reverts on any subsequent restart), set the in-memory
21918
+ * session-model so /status stays honest across the restart window, then
21919
+ * run the SAME restart dispatch as scheduleRestart above.
21874
21920
  */
21875
21921
  scheduleModelRelaunch: async (model: string, reason: string) => {
21876
21922
  const agentDir = resolveAgentDirFromEnv()
@@ -21887,20 +21933,15 @@ function buildModelDeps(restartCtx?: ModelDepsRestartContext): ModelMenuDeps & M
21887
21933
  try {
21888
21934
  await deps.scheduleRestart(reason)
21889
21935
  } catch (err) {
21890
- // A restart already in flight OWNS the override we just wrote — its
21891
- // boot (stamped keep by the in-flight path's own intent, last-writer-
21892
- // wins) will apply our token, so the switch is queued, not lost: keep
21893
- // the file + override and let the caller tell the operator "~15s".
21894
- // Any OTHER dispatch failure means no restart is coming, so roll BOTH
21895
- // back a lingering file/override would lie to /status and
21896
- // mis-launch the NEXT relaunch. The keep-intent goes with them
21897
- // (belt-and-braces: scheduleRestart's failure branch clears it too):
21898
- // a fresh keep on disk with no restart coming would wrongly KEEP
21899
- // across a crash inside its 10-min window.
21936
+ // A restart already in flight OWNS the carrier we just wrote — its
21937
+ // boot will consume+apply our token, so the switch is queued, not
21938
+ // lost: keep the file + override and let the caller tell the operator
21939
+ // "~15s". Any OTHER dispatch failure means no restart is coming, so
21940
+ // roll BOTH back a lingering carrier would lie to /status and be
21941
+ // consumed (mis-applied) by an unrelated later boot.
21900
21942
  if ((err as { code?: string })?.code !== 'restart_in_flight') {
21901
21943
  restoreSessionModelFileRaw(agentDir, prevFileRaw)
21902
21944
  sessionModelSource.setOverride(prevOverride)
21903
- clearRelaunchModelIntent(agentDir)
21904
21945
  }
21905
21946
  throw err
21906
21947
  }
@@ -21921,19 +21962,24 @@ function modelMenuReplyMarkup(reply: ModelMenuReply): InlineKeyboard | undefined
21921
21962
 
21922
21963
  /**
21923
21964
  * Record a POSITIVELY-CONFIRMED typed `/model` switch: set the in-memory
21924
- * override so `/status` reflects the live model and persist the sticky
21925
- * `.session-model` carrier. Shared by the live `bot.command('model')` handler
21926
- * and the deferred (queued mid-turn) apply so both record identically. Returns
21927
- * a persist-warning suffix to append to the reply body (empty when clean).
21965
+ * override so `/status` reflects the live model. Shared by the live
21966
+ * `bot.command('model')` handler and the deferred (queued mid-turn) apply so
21967
+ * both record identically. Returns a warning suffix to append to the reply
21968
+ * body (currently always empty kept for a stable signature).
21928
21969
  *
21929
- * The `/status` honesty invariant lives here: only `reply.selectedModel`
21930
- * (present only on a confirmed switch) records; an unverified inject records
21931
- * nothing. `/model default` clears the carrier idempotently.
21970
+ * Session-scoped (rev 4): a live Claude `/model` switch writes NO
21971
+ * `.session-model` carrier it applies in-session and the explicit
21972
+ * `claude --model <configured>` flag reverts it on the next boot, so it lasts
21973
+ * exactly until the next restart with no durable state. (sr-* switches never
21974
+ * reach here — they go through scheduleModelRelaunch, which owns the
21975
+ * consume-once carrier.) The `/status` honesty invariant lives here: only
21976
+ * `reply.selectedModel` records; an unverified inject records nothing.
21977
+ * `/model default` clears any in-memory override and any leftover carrier.
21932
21978
  */
21933
21979
  function recordTypedModelSwitch(
21934
21980
  reply: { text: string; selectedModel?: string },
21935
21981
  requestedModelArg: string | null,
21936
- deps: ModelCommandDeps,
21982
+ _deps: ModelCommandDeps,
21937
21983
  ): string {
21938
21984
  const requested = requestedModelArg != null ? expandSrAlias(requestedModelArg) : null
21939
21985
  if (requested?.toLowerCase() === 'default') {
@@ -21944,28 +21990,18 @@ function recordTypedModelSwitch(
21944
21990
  }
21945
21991
  if (!reply.selectedModel) return ''
21946
21992
  sessionModelSource.setOverride(reply.selectedModel)
21947
- const smDir = resolveAgentDirFromEnv()
21948
- if (smDir && requested && isValidModelArg(requested) && !isSrModel(requested)) {
21949
- try {
21950
- writeSessionModelFile(
21951
- smDir,
21952
- requested,
21953
- readConfiguredDefaultModel(smDir) ??
21954
- resolveMainModel(deps.getConfiguredModel() ?? undefined),
21955
- )
21956
- } catch (err) {
21957
- process.stderr.write(
21958
- `telegram gateway: session-model persist failed (typed /model): ${(err as Error)?.message ?? String(err)}\n`,
21959
- )
21960
- return '\n⚠️ Couldn’t persist the sticky override — the switch is live now but won’t survive a relaunch.'
21961
- }
21962
- }
21993
+ // A manual /model apply to the dropped premium clears any pending recovery
21994
+ // marker so the "available again" ping can't still fire after the user has
21995
+ // already switched back themselves.
21996
+ clearPremiumRecoveryOnManualSwitch(reply.selectedModel)
21963
21997
  return ''
21964
21998
  }
21965
21999
 
21966
22000
  /**
21967
- * Record a model-MENU callback outcome (persist/clear sticky override) and
21968
- * drive an sr-*→Claude graceful restart when the tap crosses that boundary.
22001
+ * Record a model-MENU callback outcome (set the live in-memory override, clear
22002
+ * a leftover carrier on a Default tap) and drive an sr-*→Claude graceful
22003
+ * restart when the tap crosses that boundary — the only menu path that writes a
22004
+ * consume-once `.session-model` carrier (a live Claude tap writes none, rev 4).
21969
22005
  * Extracted from the live `mdl:*` dispatcher so the deferred (queued mid-turn)
21970
22006
  * apply records + restarts identically. Does NOT edit any Telegram message —
21971
22007
  * callers own the card edit. Returns a restart notice when a session restart
@@ -21979,29 +22015,18 @@ function recordModelMenuSideEffects(
21979
22015
  prevSessionModel: string | null,
21980
22016
  ): { restartNotice?: string } {
21981
22017
  // Record a successful session switch so /status reflects what's actually
21982
- // running, and persist the STICKY override
21983
- // (reference/rfcs/session-model-stickiness.md): the canonical token (never
21984
- // the display label) goes to the durable `.session-model`; a confirmed
21985
- // "Default (recommended)" selection clears it instead.
22018
+ // running. Session-scoped (rev 4): a live Claude menu tap writes NO
22019
+ // `.session-model` carrier it applies in-session (native picker) and
22020
+ // reverts on the next boot. Only the sr→Claude transition below (which
22021
+ // relaunches) writes the consume-once carrier. A confirmed "Default
22022
+ // (recommended)" selection clears any leftover carrier.
21986
22023
  if (outcome.selectedModel) {
21987
22024
  sessionModelSource.setOverride(outcome.selectedModel)
21988
- const smDir = resolveAgentDirFromEnv()
21989
- if (smDir && outcome.selectedModelToken) {
21990
- try {
21991
- writeSessionModelFile(
21992
- smDir,
21993
- outcome.selectedModelToken,
21994
- readConfiguredDefaultModel(smDir) ??
21995
- resolveMainModel(modelDeps.getConfiguredModel() ?? undefined),
21996
- )
21997
- } catch (err) {
21998
- outcome.reply.text +=
21999
- '\n⚠️ Couldn’t persist the sticky override — the switch is live now but won’t survive a relaunch.'
22000
- process.stderr.write(
22001
- `telegram gateway: session-model persist failed (menu): ${(err as Error)?.message ?? String(err)}\n`,
22002
- )
22003
- }
22004
- }
22025
+ // Clear a pending premium-recovery marker when the tap re-selects the
22026
+ // dropped premium (menu OR the recovery ping's own switch-back button):
22027
+ // no stale "available again" ping once we're back on it. Idempotent — the
22028
+ // ping-send path already consumed the marker, so this is a no-op there.
22029
+ clearPremiumRecoveryOnManualSwitch(outcome.selectedModel)
22005
22030
  }
22006
22031
  if (outcome.clearedDefault) {
22007
22032
  const smDir = resolveAgentDirFromEnv()
@@ -22013,9 +22038,11 @@ function recordModelMenuSideEffects(
22013
22038
  // torn down — a graceful restart (same mechanism as /restart) is required.
22014
22039
  if (outcome.selectedModel && isSrToClaudeTransition(prevSessionModel, outcome.selectedModel)) {
22015
22040
  const agentName = getMyAgentName()
22016
- // Carry the requested Claude model across the restart via the SAME durable
22017
- // `.session-model` override a Claude → sr-* switch uses — otherwise boot
22018
- // launches the CONFIGURED default and the tapped model is silently dropped.
22041
+ // Carry the requested Claude model across the restart via the SAME
22042
+ // consume-once `.session-model` carrier a Claude → sr-* switch uses —
22043
+ // otherwise this transition's apply-relaunch boots the CONFIGURED default
22044
+ // and the tapped model is silently dropped. Applied on that one boot,
22045
+ // then reverts on the next restart (rev 4).
22019
22046
  const agentDir = resolveAgentDirFromEnv()
22020
22047
  const token = outcome.selectedModelToken
22021
22048
  if (agentDir && token) {
@@ -22178,20 +22205,19 @@ function persistQueuedCommandForRestart(action: ShutdownResolutionAction): strin
22178
22205
  switch (action.persist) {
22179
22206
  case 'model': {
22180
22207
  // #3042 blocker 2a: this token was QUEUED, never confirmed by claude.
22181
- // Under the keep-by-default boot a garbage-but-shape-valid token
22182
- // persisted here would crashloop `claude --model <garbage>` with the
22183
- // gateway dead. Only offline-trustable tokens (static Claude aliases,
22184
- // curated sr-* alias targets) may be persisted unconfirmed; anything
22185
- // else gets the honest "couldn't verify — re-issue" card instead.
22208
+ // The carrier is written immediately before the bounce, so the next
22209
+ // boot IS its apply-relaunch: consume-once means a garbage token can
22210
+ // crash at most one boot before it reverts, but we still gate on
22211
+ // offline-trustable tokens (static Claude aliases, curated sr-* alias
22212
+ // targets) to avoid even that one crash-boot; anything else gets the
22213
+ // honest "couldn't verify — re-issue" card instead.
22186
22214
  if (!isOfflineTrustedModelToken(action.arg)) {
22187
22215
  return `↩️ Couldn’t verify \`${escapeHtmlForTg(action.cmd.targetLabel || action.arg)}\` as a known model without the live session — it was NOT saved. Re-issue \`/model ${escapeHtmlForTg(action.arg)}\` once the agent is back.`
22188
22216
  }
22189
22217
  const configured =
22190
22218
  readConfiguredDefaultModel(agentDir) ?? resolveMainModel(undefined)
22219
+ // Consume-once carrier: applied by the next boot, then reverts.
22191
22220
  writeSessionModelFile(agentDir, expandSrAlias(action.arg), configured)
22192
- // Boot default is keep (#3039), but stamp explicit keep-intent for
22193
- // reason-honesty in the boot notice.
22194
- writeRelaunchModelIntent(agentDir, 'keep', 'queued /model carried across restart')
22195
22221
  break
22196
22222
  }
22197
22223
  case 'clear-model':
@@ -22281,17 +22307,50 @@ bot.command('model', async ctx => {
22281
22307
  const parsed = parseModelCommand(text) ?? { kind: 'show' as const }
22282
22308
  const chatId = String(ctx.chat!.id)
22283
22309
  const threadId = resolveThreadId(chatId, ctx.message?.message_thread_id)
22310
+ // #3177 — durable receipt FIRST. A typed /model must NEVER be invisible: even
22311
+ // if every downstream reply is shed/dropped, or the session is in a
22312
+ // phantom-idle window (turn atom cleared while claude is still busy), the
22313
+ // command leaves a greppable log line + a history row before any branch. This
22314
+ // is the fix for the finn 2026-07-12 zero-trace swallow (no log, no reply, no
22315
+ // ack, no deferred apply). `busyNow` folds BOTH busy signals (turn atom AND
22316
+ // the authoritative delivery-machine/approval gate).
22317
+ const busyNow = currentTurn !== null || turnInFlightForGate()
22318
+ process.stderr.write(modelCommandReceiptLine(getMyAgentName(), parsed, busyNow) + '\n')
22319
+ if (HISTORY_ENABLED && ctx.message?.message_id != null) {
22320
+ try {
22321
+ recordInbound({
22322
+ chat_id: chatId,
22323
+ thread_id: threadId ?? null,
22324
+ message_id: ctx.message.message_id,
22325
+ user: ctx.from?.username ?? (ctx.from?.id != null ? String(ctx.from.id) : null),
22326
+ user_id: ctx.from?.id != null ? String(ctx.from.id) : null,
22327
+ ts: ctx.message.date ?? Math.floor(Date.now() / 1000),
22328
+ text,
22329
+ })
22330
+ } catch (err) {
22331
+ process.stderr.write(`telegram gateway: /model recordInbound failed: ${(err as Error)?.message ?? String(err)}\n`)
22332
+ }
22333
+ }
22284
22334
  const deps = buildModelDeps({ chatId, threadId })
22285
- if (parsed.kind === 'show' && process.env.SWITCHROOM_MODEL_MENU !== '0') {
22335
+ // Route on a pure disposition (#3177) that folds BOTH busy signals so a
22336
+ // session busy by EITHER measure ack+queues instead of silently injecting
22337
+ // into a busy pane. Every branch below produces a visible action.
22338
+ const disposition = planModelCommand(parsed, {
22339
+ currentTurnActive: currentTurn !== null,
22340
+ turnInFlight: turnInFlightForGate(),
22341
+ menuEnabled: process.env.SWITCHROOM_MODEL_MENU !== '0',
22342
+ })
22343
+ if (disposition.kind === 'menu') {
22286
22344
  const menu = await buildModelMenu(deps)
22287
22345
  await switchroomReply(ctx, menu.text, { html: true, reply_markup: modelMenuReplyMarkup(menu) })
22288
22346
  return
22289
22347
  }
22290
- // Mid-turn: instead of dead-ending ("Try again in a moment"), ACK + QUEUE +
22291
- // apply-on-idle + confirm (#3017). The typed set path either injects into
22292
- // claude's input box or triggers a carrier restart both unsafe mid-turn.
22293
- if (parsed.kind === 'set' && deps.isBusy()) {
22294
- const target = expandSrAlias(parsed.model)
22348
+ // Mid-turn (by either busy signal): instead of dead-ending ("Try again in a
22349
+ // moment") or silently injecting into a busy pane, ACK + QUEUE + apply-on-idle
22350
+ // + confirm (#3017/#3177). The typed set path either injects into claude's
22351
+ // input box or triggers a carrier restart — both unsafe while busy.
22352
+ if (disposition.kind === 'queue') {
22353
+ const target = disposition.target
22295
22354
  const sent = await ctx.replyWithRichMessage(
22296
22355
  richMessage(hardenCardBreaks(pendingCmdAckText('model', target, escapeHtmlForTg))),
22297
22356
  threadId != null ? { message_thread_id: threadId } : {},
@@ -22331,39 +22390,38 @@ bot.command('model', async ctx => {
22331
22390
  // is blocklisted for `/effort` since #2471), session-scoped — boot re-pins
22332
22391
  // the configured default via start.sh's `--effort`. Implementation in
22333
22392
  // effort-command.ts so it's unit-testable without booting the bot.
22393
+
22394
+ // The live session-effort override, in memory only (#3186, session-scoped
22395
+ // like /model rev 4). A confirmed live apply records here — NOT to the
22396
+ // `.session-effort` carrier — so it lasts exactly until the next restart
22397
+ // (start.sh's explicit `--effort <configured>` reverts it for free). Seeded
22398
+ // at boot from `.active-session-effort` (the effort sibling of
22399
+ // `.active-session-model`) so a queued-carrier apply-boot still shows the
22400
+ // honest live level on the /effort menu.
22401
+ let sessionEffortOverride: string | null = null
22402
+
22334
22403
  function buildEffortDeps(): EffortCommandDeps {
22335
22404
  return {
22336
- // #3039: single persistence choke point EVERY positively-confirmed
22337
- // effort apply (typed, menu tap, queued drain) durably records the level
22338
- // to `.session-effort`, which start.sh resolves into `--effort` on every
22339
- // boot. `/effort default` clears it via clearSessionEffort (the handler
22340
- // clears AFTER its restore-apply, so the wrapper's write is undone).
22405
+ // Session-scoped (#3186): a positively-confirmed live apply records the
22406
+ // level IN MEMORY only no durable carrier. The `.session-effort`
22407
+ // carrier is written solely by persistQueuedCommandForRestart (a queued
22408
+ // mid-turn /effort carried across the bounce) and is consume-once at
22409
+ // boot. `/effort default` clears via clearSessionEffort below.
22341
22410
  applyEffort: async (agent, level) => {
22342
22411
  const result = await applyEffort(agent, level)
22343
- if (result.ok) {
22344
- const agentDir = resolveAgentDirFromEnv()
22345
- if (agentDir) {
22346
- try {
22347
- writeSessionEffortFile(agentDir, level, getConfiguredEffortForPersist())
22348
- } catch (err) {
22349
- process.stderr.write(
22350
- `telegram gateway: session-effort persist failed level=${level}: ${(err as Error)?.message ?? String(err)}\n`,
22351
- )
22352
- }
22353
- }
22354
- }
22412
+ if (result.ok) sessionEffortOverride = level
22355
22413
  return result
22356
22414
  },
22357
22415
  getAgentName: getMyAgentName,
22358
22416
  getConfiguredEffort: () => getConfiguredEffortForPersist(),
22359
22417
  clearSessionEffort: () => {
22418
+ sessionEffortOverride = null
22419
+ // Also drop any leftover queued-command carrier so the next boot can't
22420
+ // consume a stale level the user just cleared.
22360
22421
  const agentDir = resolveAgentDirFromEnv()
22361
22422
  if (agentDir) clearSessionEffortFile(agentDir)
22362
22423
  },
22363
- getSessionEffort: () => {
22364
- const agentDir = resolveAgentDirFromEnv()
22365
- return agentDir ? (readSessionEffortFile(agentDir)?.level ?? null) : null
22366
- },
22424
+ getSessionEffort: () => sessionEffortOverride,
22367
22425
  escapeHtml: escapeHtmlForTg,
22368
22426
  }
22369
22427
  }
@@ -22508,14 +22566,10 @@ bot.command('restart', async ctx => {
22508
22566
  // greeting card shows "Restarted user: /restart from chat" instead
22509
22567
  // of whatever reason the downstream CLI would default to.
22510
22568
  stampUserRestartReason('user: /restart from chat')
22511
- // #3039: /restart is "bounce the session", NOT "clear my model" — the
22512
- // durable override survives every restart and is cleared only by
22513
- // `/model default`. Stamp keep for reason-honesty in the boot notice
22514
- // (absence of intent keeps anyway under the keep-by-default boot).
22515
- {
22516
- const smDir = resolveAgentDirFromEnv()
22517
- if (smDir) writeRelaunchModelIntent(smDir, 'keep', 'user: /restart from chat')
22518
- }
22569
+ // Session-scoped (rev 4): /restart reverts any live /model override to the
22570
+ // configured default. A consume-once `.session-model` carrier (if one was
22571
+ // in flight) was already consumed by its own apply-relaunch, so nothing to
22572
+ // do here start.sh boots the configured default.
22519
22573
  await sweepBeforeSelfRestart()
22520
22574
  const hostdResp = await tryHostdDispatch(getMyAgentName(), {
22521
22575
  v: 1,
@@ -22674,12 +22728,8 @@ async function handleNewCommand(ctx: Context): Promise<void> {
22674
22728
  // Stamp user attribution so the next greeting shows "Restarted user:
22675
22729
  // /new" / "user: /reset" rather than the downstream CLI default.
22676
22730
  stampUserRestartReason(`user: /${kind} from chat`)
22677
- // /new and /reset start a fresh CONVERSATION, not a fresh model choice:
22678
- // the sticky session-model override KEEPS across them (contract row 7).
22679
- // Boot default is revert, so the keep-intent must land before dispatch.
22680
- if (agentDir != null) {
22681
- writeRelaunchModelIntent(agentDir, 'keep', `user: /${kind} from chat`)
22682
- }
22731
+ // Session-scoped (rev 4): /new and /reset are restarts, so they revert any
22732
+ // live /model override to the configured default no carrier to preserve.
22683
22733
  await sweepBeforeSelfRestart()
22684
22734
  const hostdResp = await tryHostdDispatch(getMyAgentName(), {
22685
22735
  v: 1,
@@ -23459,6 +23509,46 @@ const throttleTierRunner = createThrottleTierRunner({
23459
23509
  log: (m) => process.stderr.write(`telegram gateway: ${m}\n`),
23460
23510
  })
23461
23511
 
23512
+ // ─── litellm-local 429 notice — side-effect wiring ──────────────────────────
23513
+ // State machine + text + config parsing live in litellm-local-notice.ts
23514
+ // (pure); the sequencing (classification guard → per-agent cooldown →
23515
+ // broadcast + metric) lives in litellm-local-notice-wiring.ts so it is
23516
+ // unit-testable with injected deps. This block only binds the real gateway
23517
+ // dependencies. Deliberately NO broker surface: the litellm-local calm
23518
+ // path's invariant is that account state is never touched.
23519
+ const litellmLocalNoticeRunner = createLitellmLocalNoticeRunner({
23520
+ listNoticeChats: () => loadAccess().allowFrom,
23521
+ sendNotice: (chat_id, markdown) => {
23522
+ // Topic routing — this notice REPLACES the generic operator-event card
23523
+ // for the litellm-local classification, so it must land where that card
23524
+ // would have: supergroup-mode agents route system notifications into the
23525
+ // alerts/admin alias topic ('compact-watchdog' kind, same resolution as
23526
+ // the emitGatewayOperatorEvent broadcast loop), while DM recipients get
23527
+ // a thread-less send (topicForRecipient guards the #2096 "message
23528
+ // thread not found" misrouting class).
23529
+ const noticeTopic = resolveAgentOutboundTopic({ kind: 'compact-watchdog' })
23530
+ const noticeSupergroup = resolveAgentSupergroupChatId()
23531
+ const noticeThread = topicForRecipient({
23532
+ recipientChatId: chat_id,
23533
+ resolvedTopic: noticeTopic,
23534
+ supergroupChatId: noticeSupergroup,
23535
+ })
23536
+ // Status notice, not the user's answer — silence the ping (same posture
23537
+ // as the throttle-tier / fleet-fallback announcements).
23538
+ void swallowingApiCall(
23539
+ // allow-raw-bot-api: wrapped in swallowingApiCall (retry policy)
23540
+ () => bot.api.sendRichMessage(chat_id, richMessage(markdown), {
23541
+ disable_notification: true,
23542
+ ...(noticeThread != null ? { message_thread_id: noticeThread } : {}),
23543
+ }),
23544
+ { chat_id: String(chat_id), verb: 'litellm-local-notice:notify' },
23545
+ )
23546
+ },
23547
+ windowMs: () => parseLitellmNoticeWindowMs(loadAccess().litellmNoticeWindowMs),
23548
+ emitMetric: (event) => emitRuntimeMetric(event),
23549
+ log: (m) => process.stderr.write(`telegram gateway: ${m}\n`),
23550
+ })
23551
+
23462
23552
  /**
23463
23553
  * Broadcast a fleet-fallback FAILURE notice to every authorized chat.
23464
23554
  *
@@ -23516,6 +23606,173 @@ function broadcastFleetFallbackFailure(triggerAgent: string, reason: string): vo
23516
23606
  }
23517
23607
  }
23518
23608
 
23609
+ /**
23610
+ * Broadcast a status notice to every authorized chat (system notice posture:
23611
+ * silenced ping, wrapped send). Shared by the tier-downgrade notices below.
23612
+ */
23613
+ function broadcastTierNotice(markdown: string): void {
23614
+ const access = loadAccess()
23615
+ if (access.allowFrom.length === 0) return
23616
+ for (const chat_id of access.allowFrom) {
23617
+ void swallowingApiCall(
23618
+ // allow-raw-bot-api: wrapped in swallowingApiCall (retry policy)
23619
+ () => bot.api.sendRichMessage(chat_id, richMessage(markdown), { disable_notification: true }),
23620
+ { chat_id: String(chat_id), verb: 'tier-downgrade:notify' },
23621
+ )
23622
+ }
23623
+ }
23624
+
23625
+ /**
23626
+ * MODEL-TIER downgrade failover (second recovery tier). Consulted ONLY from the
23627
+ * `all-blocked` branch of doFireFleetAutoFallback — i.e. AFTER account-swap has
23628
+ * been tried and found no account still serving the walled premium model
23629
+ * (precedence A: account-swap first). When a PREMIUM /model override is active
23630
+ * (distinct from the configured default), downgrade to the configured default
23631
+ * and resume the dead turn via a self-restart, rather than let the turn stall.
23632
+ *
23633
+ * - NO automatic return to the premium model. The /model override is
23634
+ * session-scoped and in-memory only (recordTypedModelSwitch writes no
23635
+ * carrier), so it dies on the downgrade SIGTERM. The consume-once
23636
+ * `.session-model` carrier written here targets the CONFIGURED DEFAULT:
23637
+ * start.sh applies+deletes it on the resume boot, and every later restart
23638
+ * also boots the default. The premium tier is never restored on its own —
23639
+ * the user must re-issue `/model <premium>`, which the notice says plainly.
23640
+ * - Effort is NATIVE: no `.session-effort` carrier is written, so the restart
23641
+ * sheds any live /effort override and the downgraded default boots at the
23642
+ * configured `thinking_effort` (the fleet `low` pin, #1978).
23643
+ * - Loop-bounded by the NATURAL on-default guard: after the downgrade boot the
23644
+ * session runs the configured default (override gone), so a re-entry returns
23645
+ * `skip` ('on-default') and never re-downgrades — even if the default is
23646
+ * itself walled (then the normal all-blocked card fires, no loop). Pacing
23647
+ * reuses the fleetFallbackResumeGate single-flight + 3h staleness.
23648
+ *
23649
+ * Returns:
23650
+ * 'downgraded' — carrier written, latch armed, restart fired; caller must
23651
+ * NOT also emit the all-blocked give-up card (we are
23652
+ * recovering, not giving up).
23653
+ * 'restart-pending' — a resume restart was ALREADY armed in this process (a
23654
+ * concurrent turn's downgrade or account-swap); that
23655
+ * restart replays the interrupted turn, so caller must NOT
23656
+ * emit a give-up card (avoids the contradictory "could not
23657
+ * be recovered" race).
23658
+ * 'skip' — not applicable (on the configured default, unresolved
23659
+ * default, or the turn is too stale to resume); caller
23660
+ * falls through to the all-blocked card.
23661
+ */
23662
+ function maybeTierDowngrade(triggerAgent: string): 'downgraded' | 'restart-pending' | 'skip' {
23663
+ // Thin adapter: the order-sensitive glue lives in runTierDowngrade
23664
+ // (tier-downgrade-wiring.ts) behind injected deps so it is unit-testable
23665
+ // without importing the gateway. This wires the real seams.
23666
+ return runTierDowngrade(triggerAgent, {
23667
+ getAgentDir: () => resolveAgentDirFromEnv() ?? null,
23668
+ getConfiguredDefault: () => {
23669
+ const dir = resolveAgentDirFromEnv()
23670
+ if (!dir) return null
23671
+ return resolveMainModel(readConfiguredDefaultModel(dir) ?? undefined)
23672
+ },
23673
+ getSessionOverride: () => sessionModelSource.getOverride(),
23674
+ resolve: (t) => resolveMainModel(t),
23675
+ // PEEK the resume gate WITHOUT arming (single-flight within this process +
23676
+ // 3h staleness), the same gate the account-swap resume path uses.
23677
+ peekResumeGate: () => fleetFallbackResumeGate.peek(newestActiveTurnStartedAtMs()),
23678
+ writeCarrier: (dir, toModel, cfg) => writeSessionModelFile(dir, toModel, cfg),
23679
+ armResumeGate: () => fleetFallbackResumeGate.arm(),
23680
+ // Records the DROPPED premium token + the notice's allowFrom chats; start.sh
23681
+ // never consumes `.premium-recovery`, so it survives the self-restart.
23682
+ writeRecoveryMarker: (dir, premiumModel) => {
23683
+ const chats = loadAccess().allowFrom.map((c) => String(c))
23684
+ if (chats.length > 0) writePremiumRecoveryFile(dir, premiumModel, chats)
23685
+ },
23686
+ broadcastNotice: (md) => broadcastTierNotice(md),
23687
+ selfRestart: (agent) => triggerSelfRestart(agent, 'tier-downgrade-resume'),
23688
+ selfAgent: (t) => process.env.SWITCHROOM_AGENT_NAME ?? t,
23689
+ log: (msg) => process.stderr.write(`telegram gateway: ${msg}\n`),
23690
+ })
23691
+ }
23692
+
23693
+ /**
23694
+ * "Premium model recovered" ping (tier-downgrade companion). Consulted from the
23695
+ * gateway's existing `runQuotaWatch` tick (every 15 min, on the cheap
23696
+ * `list-state` IPC read it already does — NO new poller, NO broker change).
23697
+ * When a `.premium-recovery` marker is pending (a downgrade dropped a premium
23698
+ * `/model` selection fleet-wide) AND the broker's live-authoritative per-account
23699
+ * eligibility shows the premium tier servable again — at least one account
23700
+ * neither `exhausted` nor `premium_walled` — fire EXACTLY ONE ping to the
23701
+ * recorded chats with a one-tap "switch back" button, then consume the marker.
23702
+ *
23703
+ * DETERMINISTIC: `exhausted` / `premium_walled` are the broker's own verdicts
23704
+ * (`isAccountExhausted` / `isAccountPremiumWalled` → `isModelTierWalled`, a pure
23705
+ * timestamp compare). No model judgement, no clock of our own.
23706
+ *
23707
+ * AT-MOST-ONCE: a fleet-wide `claim-notification` gates against a bounce or a
23708
+ * concurrent tick, and the marker is cleared BEFORE the send (never-storm). The
23709
+ * button routes through the SAME session-scoped `/model` apply path as the
23710
+ * model menu (`mdl:alias:<premium>` → handleModelMenuCallback +
23711
+ * recordModelMenuSideEffects) — it does not bypass it.
23712
+ */
23713
+ async function maybePremiumRecoveryPing(
23714
+ brokerClient: NonNullable<Awaited<ReturnType<typeof getAuthBrokerClient>>>,
23715
+ accounts: ReadonlyArray<{ exhausted: boolean; premium_walled?: boolean }>,
23716
+ ): Promise<void> {
23717
+ // Thin adapter: the order-sensitive never-storm glue lives in
23718
+ // runPremiumRecoveryPing (premium-recovery-wiring.ts) behind injected deps so
23719
+ // it is unit-testable without importing the gateway. This wires the real seams
23720
+ // (marker FS, fleet claim, send) — behaviour is preserved exactly.
23721
+ return runPremiumRecoveryPing({
23722
+ getAgentDir: () => resolveAgentDirFromEnv() ?? null,
23723
+ readMarker: (dir) => readPremiumRecoveryFile(dir),
23724
+ clearMarker: (dir) => clearPremiumRecoveryFile(dir),
23725
+ getAgent: () => getMyAgentName(),
23726
+ // The pure recovery predicate, bound to THIS tick's live per-account
23727
+ // eligibility (the broker's own verdicts; `premium_walled` may be absent).
23728
+ decide: () =>
23729
+ decidePremiumRecovery({
23730
+ hasMarker: true,
23731
+ accounts: accounts.map((a) => ({
23732
+ exhausted: a.exhausted,
23733
+ premiumWalled: a.premium_walled === true,
23734
+ })),
23735
+ }).fire,
23736
+ // Fail-open (claimQuotaNotification returns true on any broker error) — a
23737
+ // convenience ping degrades to at-least-once, never lost.
23738
+ claimNotification: (key) => claimQuotaNotification(brokerClient, key),
23739
+ fallbackChats: () => loadAccess().allowFrom.map((c) => String(c)),
23740
+ sendToChat: (chat_id, ping, keyboard) => {
23741
+ void swallowingApiCall(
23742
+ // allow-raw-bot-api: wrapped in swallowingApiCall (retry policy)
23743
+ () =>
23744
+ bot.api.sendRichMessage(chat_id, richMessage(ping.text), {
23745
+ disable_notification: true,
23746
+ reply_markup: keyboard,
23747
+ }),
23748
+ { chat_id: String(chat_id), verb: 'premium-recovery:notify' },
23749
+ )
23750
+ },
23751
+ log: (msg) => process.stderr.write(`telegram gateway: ${msg}\n`),
23752
+ })
23753
+ }
23754
+
23755
+ /**
23756
+ * Clear a pending premium-recovery marker when the user MANUALLY re-selects the
23757
+ * dropped premium model before the recovery ping fires — no stale "available
23758
+ * again" ping after they've already switched back. Called from every `/model`
23759
+ * apply chokepoint (typed + menu/callback). Matches on the canonicalized token
23760
+ * so an alias vs resolved-id spelling of the same model still clears it.
23761
+ */
23762
+ function clearPremiumRecoveryOnManualSwitch(appliedToken: string | null | undefined): void {
23763
+ if (appliedToken == null || appliedToken.length === 0) return
23764
+ const agentDir = resolveAgentDirFromEnv()
23765
+ if (!agentDir) return
23766
+ const marker = readPremiumRecoveryFile(agentDir)
23767
+ if (marker == null) return
23768
+ if (resolveMainModel(appliedToken) === resolveMainModel(marker.premiumModel)) {
23769
+ clearPremiumRecoveryFile(agentDir)
23770
+ process.stderr.write(
23771
+ `telegram gateway: [premium-recovery] marker cleared — user manually re-issued /model ${marker.premiumModel}\n`,
23772
+ )
23773
+ }
23774
+ }
23775
+
23519
23776
  /** Returns true iff the dispatcher actually performed a swap (and the
23520
23777
  * user-visible announcement was broadcast). False on no-op /
23521
23778
  * error / idempotent-skip — caller uses this to decide whether to
@@ -23598,6 +23855,21 @@ async function doFireFleetAutoFallback(
23598
23855
  if (outcome.kind === 'switched') {
23599
23856
  fallbackAllBlockedNoticeState = { lastSentAtMs: 0 }
23600
23857
  } else if (outcome.kind === 'all-blocked') {
23858
+ // ── Second recovery tier: MODEL-TIER downgrade (precedence A) ──────────
23859
+ // Account-swap just came back all-blocked (no account still serves the
23860
+ // walled model). Before giving up, if a PREMIUM /model override is active,
23861
+ // downgrade to the configured default and resume the dead turn rather than
23862
+ // stall. Only reachable here — a `switched` outcome never enters this
23863
+ // branch, so a model throttled on one account is always recovered by
23864
+ // account-swap first, never by a downgrade. Loop-bounded by the natural
23865
+ // on-default guard; effort revert is native (see maybeTierDowngrade).
23866
+ const tier = maybeTierDowngrade(triggerAgent)
23867
+ if (tier === 'downgraded' || tier === 'restart-pending') {
23868
+ // Either this turn armed a downgrade restart, or a concurrent turn
23869
+ // already armed a resume restart. Do NOT emit the all-blocked give-up
23870
+ // card — a restart is coming that replays the interrupted turn.
23871
+ return false
23872
+ }
23601
23873
  const verdict = evaluateAllBlockedNotice(fallbackAllBlockedNoticeState, Date.now())
23602
23874
  if (!verdict.send) {
23603
23875
  process.stderr.write(
@@ -23830,6 +24102,16 @@ async function runQuotaWatch(opts: { bootTick?: boolean } = {}): Promise<void> {
23830
24102
  return // No accounts — nothing to watch.
23831
24103
  }
23832
24104
 
24105
+ // Premium-model recovery ping (tier-downgrade companion). Reuses THIS tick's
24106
+ // list-state read (no extra IPC): if a downgrade left a `.premium-recovery`
24107
+ // marker and the broker now shows the premium tier servable again, ping once
24108
+ // with a one-tap switch-back button. At-most-once + fleet-claim-deduped.
24109
+ await maybePremiumRecoveryPing(brokerClient, listStateData.accounts).catch((err) => {
24110
+ process.stderr.write(
24111
+ `telegram gateway: [premium-recovery] ping check failed (non-fatal): ${(err as Error)?.message ?? err}\n`,
24112
+ )
24113
+ })
24114
+
23833
24115
  // Build AccountSnapshot[] from cached broker state only — no live probe.
23834
24116
  // Accounts with null last_quota produce quota=null snapshots; classifyHealth
23835
24117
  // returns 'unknown'; evaluateQuotaWatchAccount skips — no false alarms.
@@ -25597,7 +25879,7 @@ bot.on('callback_query:data', async ctx => {
25597
25879
  // sr-* TARGET tap: switch TO a non-Claude (LiteLLM/OpenRouter) model.
25598
25880
  // Parity with the text `/model sr-*` path — claude's native picker rejects
25599
25881
  // unknown sr-* ids, so an in-place inject can't set them. Carry the token
25600
- // across a graceful restart (the durable `.session-model` override) and
25882
+ // across a graceful restart (the consume-once `.session-model` carrier) and
25601
25883
  // relaunch `claude --model sr-*`. Session-only; reverts to the configured
25602
25884
  // default on the next restart. The sr-* → Claude direction is handled below
25603
25885
  // via the SELECT/alias outcome + isSrToClaudeTransition.
@@ -28006,46 +28288,36 @@ async function shutdown(signal: string): Promise<void> {
28006
28288
  } catch (err) {
28007
28289
  process.stderr.write(`telegram gateway: shutdown.clean_marker_write_failed err=${(err as Error).message}\n`)
28008
28290
  }
28009
- // #3017 persist a Telegram-set model across a GRACEFUL deploy/restart.
28010
- // Boot default is REVERT, and an EXTERNAL deploy (SIGTERM to PID 1 from
28011
- // `switchroom apply` / `docker compose up`) never routes through
28012
- // triggerSelfRestart, so it stamps no `.relaunch-model-intent` and start.sh
28013
- // drops the user's `/model` choice (the overlord `fable`→`opus` revert on
28014
- // the v0.18.9 roll). A graceful OS-signal shutdown IS a clean, planned
28015
- // bounce — stamp keep-intent for an active `.session-model` override so the
28016
- // chosen model survives and start.sh re-confirms it via `.session-model-alert`
28017
- // on boot. Respect an intent an initiator already stamped (a /restart stamps
28018
- // 'revert' before SIGTERM): only stamp when none exists. A crash routes
28019
- // through the non-OS-signal branch and still reverts (safe side preserved).
28020
- try {
28021
- const smDir = resolveAgentDirFromEnv()
28022
- if (smDir != null) {
28023
- const hasOverride = readSessionModelFile(smDir) != null
28024
- const intentAlreadyStamped = existsSync(join(smDir, RELAUNCH_MODEL_INTENT_FILE))
28025
- if (hasOverride && !intentAlreadyStamped) {
28026
- // The GATEWAY_SHUTDOWN_INTENT_REASON_PREFIX makes this stamp
28027
- // recognisable at the next GATEWAY boot: a gateway-only bounce never
28028
- // runs start.sh, so a leftover stamp with this prefix is cleared at
28029
- // boot (clearStaleGatewayShutdownIntent) instead of lingering to
28030
- // convert a later genuine crash into a "keep" (#3018 finding 4).
28031
- writeRelaunchModelIntent(smDir, 'keep', `${GATEWAY_SHUTDOWN_INTENT_REASON_PREFIX} graceful ${signal} shutdown (deploy/rolling restart) — preserving user-chosen session model`)
28032
- process.stderr.write(`telegram gateway: shutdown.session_model_keep_stamped signal=${signal}\n`)
28033
- }
28034
- }
28035
- } catch (err) {
28036
- process.stderr.write(`telegram gateway: shutdown.session_model_keep_stamp_failed err=${(err as Error).message}\n`)
28037
- }
28291
+ // Session-scoped (rev 4): a graceful deploy/restart REVERTS a live /model
28292
+ // override to the configured default there is no keep-intent to stamp.
28293
+ // (A live Claude override never had a `.session-model` carrier; an in-flight
28294
+ // sr-* carrier was already consumed by its own apply-relaunch.) A queued —
28295
+ // not-yet-applied /model IS still persisted just below so it applies as
28296
+ // the agent boots (its apply-relaunch), then reverts on the next restart.
28038
28297
  } else {
28039
28298
  process.stderr.write(`telegram gateway: shutdown.clean_marker_skipped signal=${signal} (crash path — banner will fire on next boot)\n`)
28040
28299
  }
28041
28300
 
28042
28301
  // #3018 finding 3 + #3039: resolve any queued /model|/effort ack cards. The
28043
28302
  // gateway (and with it the in-memory queue) is going away — persist each
28044
- // typed choice to the durable boot carriers (`.session-model` /
28045
- // `.session-effort`) so it still deterministically applies as the agent
28046
- // boots, and edit the ack card to say so. Only an unresolvable menu-tag
28047
- // selection falls back to a re-issue note. Best-effort and time-bounded so
28048
- // a wedged Telegram API can't block shutdown.
28303
+ // typed choice to the consume-once boot carriers (`.session-model` /
28304
+ // `.session-effort`, #3184/#3186) so it still deterministically applies as
28305
+ // the agent boots (that boot consumes the carrier; later restarts revert),
28306
+ // and edit the ack card to say so. Only an unresolvable menu-tag selection
28307
+ // falls back to a re-issue note. Best-effort and time-bounded so a wedged
28308
+ // Telegram API can't block shutdown.
28309
+ //
28310
+ // DELIBERATE (rev 4, #3184 review LOW-3 — applies to BOTH carriers): this
28311
+ // runs on EVERY shutdown path, including crashes (uncaughtException/
28312
+ // unhandledRejection route here), not just the isOsSignal branch above. So
28313
+ // a mid-turn queued /model or /effort + crash can apply on the
28314
+ // crash-recovery boot — technically at odds with a literal "crash reverts"
28315
+ // reading of the session-scoped contract. Intended: it preserves #3178's
28316
+ // "a queued command never silently vanishes" guarantee (the ack card
28317
+ // promised the switch), the model side is gated to offline-trusted tokens
28318
+ // (the effort side is allowlist-gated at write, so a garbage level can't
28319
+ // even cost one crash-boot), and the consume-once carriers bound it to
28320
+ // exactly that one recovery boot — the following restart reverts to config.
28049
28321
  const orphanedCmdActions = pendingCmdShutdownResolutionActions(pendingSessionCommand, escapeHtmlForTg)
28050
28322
  const orphanedCmdEdits = orphanedCmdActions.map(a => ({
28051
28323
  chatId: a.cmd.ackChatId,
@@ -28890,6 +29162,23 @@ void (async () => {
28890
29162
  } catch { /* leave override as-is on a bad read */ }
28891
29163
  }
28892
29164
 
29165
+ // Effort sibling (#3186): start.sh records the EFFECTIVE launched
29166
+ // effort to `.active-session-effort` every boot. Re-hydrate the
29167
+ // in-memory session-effort override so the /effort menu highlight
29168
+ // stays honest after a queued-carrier apply-boot. Only an effort
29169
+ // differing from the configured default counts as an override.
29170
+ const activeEffortPath = join(smAgentDir, '.active-session-effort')
29171
+ if (existsSync(activeEffortPath)) {
29172
+ try {
29173
+ const launchedEffort = readFileSync(activeEffortPath, 'utf8').trim()
29174
+ const configuredEffort = getConfiguredEffortForPersist()
29175
+ sessionEffortOverride =
29176
+ launchedEffort.length > 0 && launchedEffort !== configuredEffort
29177
+ ? launchedEffort
29178
+ : null
29179
+ } catch { /* leave override as-is on a bad read */ }
29180
+ }
29181
+
28893
29182
  const alertPath = join(smAgentDir, '.session-model-alert')
28894
29183
  if (existsSync(alertPath)) {
28895
29184
  let alertText: string | null = null
@@ -29063,6 +29352,12 @@ void (async () => {
29063
29352
  // supersedes the coarse 5-min bucket relay below to avoid
29064
29353
  // double-surfacing the same progress beat.
29065
29354
  const workerFeedEnabled = isWorkerActivityFeedEnabled(process.env.SWITCHROOM_WORKER_ACTIVITY_FEED)
29355
+ // Combined-feed row cap (channels.telegram.worker_feed.max_rows).
29356
+ // Unset / non-positive → the feed's built-in default (8).
29357
+ const workerFeedMaxRows = (() => {
29358
+ const raw = Number(process.env.SWITCHROOM_TG_WORKER_FEED_MAX_ROWS)
29359
+ return Number.isInteger(raw) && raw > 0 ? raw : undefined
29360
+ })()
29066
29361
  // Model A — foreground sub-agent nesting in the parent's live
29067
29362
  // activity draft. ON by default; this edits the SAME activity-
29068
29363
  // summary message the tool_label feed already owns (not the
@@ -29112,6 +29407,40 @@ void (async () => {
29112
29407
  },
29113
29408
  ),
29114
29409
  },
29410
+ // #3084 follow-up: the feed's send/edit adapters transit the send
29411
+ // gate, which SHEDS (resolves undefined) any call made during an
29412
+ // open flood window. Give the feed the SAME on-disk window probe
29413
+ // robustApiCall + the held-card sweep read, so a running/first-
29414
+ // paint tick parks in cooldown instead of re-firing a shed send
29415
+ // every ~6s for the whole ban (the worker-feed shed-contract bug:
29416
+ // 565 `sent.message_id` crashes in one 6h ban).
29417
+ floodWaitRemainingMs: probeFloodWaitRemainingMs,
29418
+ // #3084 follow-up: 2+ background workers in one chat/thread coalesce
29419
+ // into ONE combined feed message. `maxRows` caps the visible rows
29420
+ // (compact `+M more working…` spill) so the body stays under the
29421
+ // rich-message wire ceiling (STATUS_CARD_CHAR_BUDGET). Sourced from
29422
+ // channels.telegram.worker_feed.max_rows via the config cascade
29423
+ // (scaffold emits SWITCHROOM_TG_WORKER_FEED_MAX_ROWS); unset → 8.
29424
+ maxRows: workerFeedMaxRows,
29425
+ // #3207 review: GROUP-level status pin. Workers now coalesce into
29426
+ // ONE shared message, so the pin must follow the GROUP lifecycle,
29427
+ // not a single worker's — otherwise a sibling's finish unpins a
29428
+ // message the survivors still need and the survivor's re-pin NOOPs
29429
+ // (its claim still names that id), leaving live work unpinned. The
29430
+ // feed drives this: pin `wk:group:<feedKey>` when the group first
29431
+ // paints, unpin only when the group empties (messageId === null).
29432
+ reconcilePin: ({ feedKey, chatId, messageId }) => {
29433
+ if (!PIN_STATUS_WHILE_WORKING) return
29434
+ const key = `wk:group:${feedKey}`
29435
+ if (messageId != null) {
29436
+ void reconcileStatusPin(key, chatId, { pinned: true, messageId })
29437
+ } else {
29438
+ const unpinChat = chatId || statusPinChatIds.get(key)
29439
+ if (unpinChat != null && unpinChat.length > 0) {
29440
+ void reconcileStatusPin(key, unpinChat, { pinned: false })
29441
+ }
29442
+ }
29443
+ },
29115
29444
  log: (msg) => process.stderr.write(`telegram gateway: ${msg}\n`),
29116
29445
  })
29117
29446
  subagentWatcher = startSubagentWatcher({
@@ -29228,9 +29557,9 @@ void (async () => {
29228
29557
  // anything. The actual repaint + re-pin happen DOWNSTREAM on the
29229
29558
  // next replayed `running` cue: the watcher's re-registration
29230
29559
  // replays `onProgress`, which calls `workerActivityFeed.update()`
29231
- // (now un-gated) to first-paint a FRESH `🛠 Worker` message, and
29232
- // its `.then(reconcileWorkerPin(agentId, wkChat, true))` pins that
29233
- // new message via the `wk:<agentId>` status-pin. Clearing the gate
29560
+ // (now un-gated) to first-paint a FRESH message, and the feed's
29561
+ // own `reconcilePin` (#3207) re-pins that message at the GROUP
29562
+ // level (`wk:group:<feedKey>`). Clearing the gate
29234
29563
  // here FIRST is the ordering requirement — without it those first
29235
29564
  // replayed ticks would be swallowed by the finalized gate and no
29236
29565
  // new card would ever paint. Net effect restores the operator
@@ -29336,7 +29665,7 @@ void (async () => {
29336
29665
  // card keeps the model tag even with no live entry.
29337
29666
  model: dispatch.feedModel ?? undefined,
29338
29667
  })
29339
- reconcileWorkerPin(agentId, null, false)
29668
+ // #3207: group-level pin dropped by the feed on group-empty.
29340
29669
  }
29341
29670
  return
29342
29671
  }
@@ -29414,8 +29743,9 @@ void (async () => {
29414
29743
  state: outcome === 'failed' ? 'failed' : 'done',
29415
29744
  model: dispatch.feedModel ?? undefined,
29416
29745
  })
29417
- // Status-pin: worker done drop its pin.
29418
- reconcileWorkerPin(agentId, null, false)
29746
+ // #3207: the group-level pin is dropped by the feed itself
29747
+ // when this group's LAST worker finishes (reconcilePin
29748
+ // `wk:group:<feedKey>`); no per-worker unpin here.
29419
29749
  }
29420
29750
  return
29421
29751
  }
@@ -29434,8 +29764,8 @@ void (async () => {
29434
29764
  state: outcome === 'failed' ? 'failed' : 'done',
29435
29765
  model: dispatch.feedModel ?? undefined,
29436
29766
  })
29437
- // Status-pin: worker done drop its pin.
29438
- reconcileWorkerPin(agentId, null, false)
29767
+ // #3207: the group-level pin is dropped by the feed itself
29768
+ // when this group's LAST worker finishes; no per-worker unpin.
29439
29769
  }
29440
29770
 
29441
29771
  const handbackOrigin = resolveSubagentOriginChat(agentId)
@@ -29602,7 +29932,10 @@ void (async () => {
29602
29932
  model: feedModel,
29603
29933
  },
29604
29934
  wk.threadId,
29605
- )?.then(() => reconcileWorkerPin(agentId, wkChat, true))
29935
+ )
29936
+ // #3207: the feed pins the group's shared message itself when
29937
+ // it first paints (reconcilePin → `wk:group:<feedKey>`); no
29938
+ // per-worker pin chained here.
29606
29939
  return
29607
29940
  }
29608
29941
  if (surface !== 'nest') return // 'skip' — orphan-status off
@@ -29744,7 +30077,10 @@ void (async () => {
29744
30077
  model: feedModel,
29745
30078
  },
29746
30079
  wk.threadId,
29747
- )?.then(() => reconcileWorkerPin(agentId, wkChat, true))
30080
+ )
30081
+ // #3207: the feed pins the group's shared message itself when
30082
+ // it first paints (reconcilePin → `wk:group:<feedKey>`); no
30083
+ // per-worker pin chained here.
29748
30084
  return
29749
30085
  }
29750
30086