switchroom 0.18.17 → 0.18.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/agent-scheduler/index.js +13 -0
  2. package/dist/auth-broker/index.js +13 -0
  3. package/dist/cli/notion-write-pretool.mjs +13 -0
  4. package/dist/cli/switchroom.js +605 -479
  5. package/dist/host-control/main.js +17 -1
  6. package/dist/vault/approvals/kernel-server.js +13 -0
  7. package/dist/vault/broker/server.js +13 -0
  8. package/package.json +1 -1
  9. package/telegram-plugin/bridge/bridge.ts +7 -1
  10. package/telegram-plugin/dist/bridge/bridge.js +26 -1
  11. package/telegram-plugin/dist/gateway/gateway.js +1401 -431
  12. package/telegram-plugin/dist/server.js +26 -1
  13. package/telegram-plugin/fleet-fallback-resume.ts +26 -3
  14. package/telegram-plugin/gateway/approval-hold.ts +49 -0
  15. package/telegram-plugin/gateway/bridge-dead-watchdog.ts +61 -18
  16. package/telegram-plugin/gateway/gateway.ts +362 -71
  17. package/telegram-plugin/gateway/linear-activity.ts +20 -4
  18. package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
  19. package/telegram-plugin/gateway/session-model-file.ts +103 -0
  20. package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
  21. package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
  22. package/telegram-plugin/llm-error-present.ts +436 -0
  23. package/telegram-plugin/operator-events.ts +7 -1
  24. package/telegram-plugin/permission-title.ts +172 -10
  25. package/telegram-plugin/premium-recovery.ts +101 -0
  26. package/telegram-plugin/raw-error-scrub.ts +73 -0
  27. package/telegram-plugin/retry-api-call.ts +8 -2
  28. package/telegram-plugin/send-gate-degraded.test.ts +152 -1
  29. package/telegram-plugin/send-gate-observability.test.ts +140 -0
  30. package/telegram-plugin/send-gate-observability.ts +65 -20
  31. package/telegram-plugin/send-gate.test.ts +143 -1
  32. package/telegram-plugin/send-gate.ts +212 -19
  33. package/telegram-plugin/session-tail.ts +16 -0
  34. package/telegram-plugin/shared/local-time.ts +69 -0
  35. package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
  36. package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
  37. package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
  38. package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
  39. package/telegram-plugin/tests/flood-windows-persistence.test.ts +3 -2
  40. package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
  41. package/telegram-plugin/tests/llm-error-present.test.ts +380 -0
  42. package/telegram-plugin/tests/permission-title.test.ts +167 -4
  43. package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
  44. package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
  45. package/telegram-plugin/tests/reaction-gate-routing.test.ts +6 -1
  46. package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
  47. package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
  48. package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
  49. package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
  50. package/telegram-plugin/tests/worker-activity-feed.test.ts +5 -2
  51. package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
  52. package/telegram-plugin/tier-downgrade.ts +198 -0
  53. package/telegram-plugin/tool-activity-summary.ts +99 -0
  54. package/telegram-plugin/worker-activity-feed.ts +509 -409
@@ -184,6 +184,7 @@ import {
184
184
  selectHeldForRedelivery,
185
185
  holdReasonFor,
186
186
  heldRetryBackoffMs,
187
+ applyDeliveredHoldReset,
187
188
  type UndeliverableMark,
188
189
  } from './approval-hold.js'
189
190
  import { isTelegramReplyTool, isTelegramSurfaceTool } from '../tool-names.js'
@@ -207,7 +208,7 @@ import {
207
208
  isPhotoDimensionRejectError,
208
209
  isFloodWaitActiveError,
209
210
  } from '../retry-api-call.js'
210
- import { createSendGate, sendGateEnabledFromEnv } from '../send-gate.js'
211
+ import { createSendGate, sendGateConfigFromEnv } from '../send-gate.js'
211
212
  import { createStatsLogger, createFloodWindowObserver } from '../send-gate-observability.js'
212
213
  import { classifyPhotoFile, rerouteResultSuffix } from '../photo-precheck.js'
213
214
  import { installTgPostLogger, withTgPostTags } from '../shared/bot-runtime.js'
@@ -314,6 +315,11 @@ import {
314
315
  type OperatorEventKind,
315
316
  } from '../operator-events.js'
316
317
  import { recordOperatorEvent } from '../operator-events-history.js'
318
+ import {
319
+ parseLlmError,
320
+ renderLlmError,
321
+ decideErrorSurface,
322
+ } from '../llm-error-present.js'
317
323
  import {
318
324
  formatModelUnavailableCard,
319
325
  resolveModelUnavailableFromOperatorEvent,
@@ -468,7 +474,13 @@ import {
468
474
  readConfiguredDefaultModel,
469
475
  writeSessionEffortFile,
470
476
  clearSessionEffortFile,
477
+ writePremiumRecoveryFile,
478
+ readPremiumRecoveryFile,
479
+ clearPremiumRecoveryFile,
471
480
  } from './session-model-file.js'
481
+ import { runTierDowngrade } from './tier-downgrade-wiring.js'
482
+ import { runPremiumRecoveryPing } from './premium-recovery-wiring.js'
483
+ import { decidePremiumRecovery } from '../premium-recovery.js'
472
484
  import { discoverModels, selectModel } from '../../src/agents/model-picker.js'
473
485
  import { resolveMainModel, SWITCHROOM_DEFAULT_THINKING_EFFORT } from '../../src/agents/scaffold.js'
474
486
  import {
@@ -5468,8 +5480,14 @@ const recordFloodWindow = makeFloodWindowRecorder(FLOOD_WINDOWS_PATH)
5468
5480
  // ban never resends into an open window. onWindowOpen write-throughs every
5469
5481
  // runtime-opened window to FLOOD_WINDOWS_PATH; bootRamp starts the global
5470
5482
  // bucket at half capacity for 10s to absorb the boot-card burst.
5483
+ // Resolve enabled + the tunable rate limits (channels.telegram.send_gate.* →
5484
+ // SWITCHROOM_TG_SEND_GATE_* env) ONCE at boot. Unset knobs are absent, so
5485
+ // createSendGate applies SEND_GATE_DEFAULTS — omitting config = today's exact
5486
+ // behaviour. The SWITCHROOM_TELEGRAM_SEND_GATE break-glass valve still wins on
5487
+ // `enabled` when explicitly set (see sendGateConfigFromEnv precedence).
5488
+ const sendGateConfig = sendGateConfigFromEnv()
5471
5489
  const sendGate = createSendGate({
5472
- enabled: sendGateEnabledFromEnv(),
5490
+ ...sendGateConfig,
5473
5491
  initialWindows: loadInitialFloodWindows(FLOOD_STATE_PATH, FLOOD_WINDOWS_PATH, Date.now()),
5474
5492
  bootRamp: {},
5475
5493
  onWindowOpen: (scopeKey, untilTs) => recordFloodWindow(scopeKey, untilTs),
@@ -5482,15 +5500,20 @@ const sendGate = createSendGate({
5482
5500
  const probeFloodWaitRemainingMs = makeFloodWaitProbe(FLOOD_STATE_PATH)
5483
5501
  const rawRobustApiCall = createRetryApiCall({
5484
5502
  log: (line) => process.stderr.write(line),
5485
- onFloodWait: (retryAfterSec) => {
5503
+ onFloodWait: (retryAfterSec, opts) => {
5486
5504
  // #2923/#3094 — persist the single-object global window (probe reads this).
5487
5505
  makeFloodWaitRecorder(FLOOD_STATE_PATH)(retryAfterSec)
5488
- // #3084 PR 2 — also open a GLOBAL send-gate window so cosmetic traffic sheds
5489
- // for the ban's duration even on a SHORT (slept-and-retried) 429 that never
5490
- // throws FLOOD_WAIT_ACTIVE. Scope-precise windows are opened by the gate's
5491
- // own FLOOD_WAIT_ACTIVE catch (which has the call's opts).
5506
+ // #3084 PR 2 / #3111 — also open SCOPE-PRECISE send-gate window(s) so cosmetic
5507
+ // traffic sheds for the ban's duration even on a SHORT (slept-and-retried)
5508
+ // 429 that never throws FLOOD_WAIT_ACTIVE. The retry policy now passes the
5509
+ // call's `opts` (#3111) so this opens the FINEST scope the 429 implies —
5510
+ // `chat:`/`group:`/`msg-edit:` for a chat-bound call, `global` only when the
5511
+ // call carries no chat scope (genuinely global) — instead of a blanket
5512
+ // `global` window that would suppress unrelated chats. This mirrors the
5513
+ // gate's own FLOOD_WAIT_ACTIVE catch, which uses the same scope-precise
5514
+ // opener for LONG bans.
5492
5515
  try {
5493
- sendGate.openFloodWindow('global', Date.now() + Math.max(0, retryAfterSec) * 1000)
5516
+ sendGate.openScopedFloodWindows(opts, Date.now() + Math.max(0, retryAfterSec) * 1000)
5494
5517
  } catch {
5495
5518
  /* best-effort — never let the window hook break the retry path */
5496
5519
  }
@@ -5634,6 +5657,9 @@ const sendGateStatsLogger = createStatsLogger({
5634
5657
  const floodWindowObserver = createFloodWindowObserver({
5635
5658
  clock: { now: () => Date.now(), sleep: (ms) => new Promise((r) => setTimeout(r, ms)) },
5636
5659
  log: (line) => process.stderr.write(line),
5660
+ // Render the operator-facing flood alerts in the configured local timezone
5661
+ // (same env the config cascade bakes into every agent — see timezone.ts).
5662
+ tz: process.env.SWITCHROOM_TIMEZONE ?? process.env.TZ ?? 'UTC',
5637
5663
  stats: () => sendGate.stats(),
5638
5664
  readWindows: (now) => readFloodWindows(FLOOD_WINDOWS_PATH, now),
5639
5665
  markAlerted: (scopeKey, alertedAt) =>
@@ -5654,7 +5680,7 @@ const floodWindowObserver = createFloodWindowObserver({
5654
5680
  )
5655
5681
  },
5656
5682
  })
5657
- if (sendGateEnabledFromEnv()) {
5683
+ if (sendGateConfig.enabled) {
5658
5684
  const observeTimer = setInterval(() => {
5659
5685
  try {
5660
5686
  sendGateStatsLogger.tick()
@@ -6726,10 +6752,19 @@ function isAutoFallbackCooldownActive(_agentName: string, now: number): boolean
6726
6752
 
6727
6753
  async function editCardExpired(chatId: string, messageId: number | undefined, body: string): Promise<void> {
6728
6754
  if (messageId == null) return
6729
- await lockedBot.api
6730
- // allow-raw-bot-api: message-id-targeted edit (no thread to lose); best-effort card-expiry strip from the reaper (no grammy ctx). Dropping reply_markup strips the stale keyboard atomically with the text edit.
6731
- .editMessageText(chatId, messageId, richMessage(body), { reply_markup: { inline_keyboard: [] } })
6732
- .catch(() => {})
6755
+ // #3084 bypass audit: this best-effort card-expiry strip (reaper + lazy
6756
+ // sweeps, no grammy ctx) previously fired a RAW `lockedBot.api.editMessageText`
6757
+ // OUTSIDE the send gate a residual flood vector (a 429 here never reached
6758
+ // `onFloodWait`, so the breaker stayed blind). Routed through `robustApiCall`
6759
+ // (chat-lock → send-gate → retry/breaker) as a `cosmetic` edit carrying
6760
+ // messageId+editPayload so it paces/sheds under pressure and its 429 records
6761
+ // the flood window. Dropping reply_markup strips the stale keyboard atomically
6762
+ // with the text edit; message-id-targeted so no thread to lose.
6763
+ const text = richMessage(body)
6764
+ await robustApiCall(
6765
+ () => lockedBot.api.editMessageText(chatId, messageId, text, { reply_markup: { inline_keyboard: [] } }),
6766
+ { chat_id: chatId, verb: 'card-expired.strip', priorityClass: 'cosmetic', messageId, editPayload: body },
6767
+ ).catch(() => {})
6733
6768
  }
6734
6769
 
6735
6770
  function recordMissedApproval(opts: {
@@ -7073,20 +7108,13 @@ function postPermissionCard(
7073
7108
  const landedThreadId = sent.message_thread_id ?? undefined
7074
7109
  live.cards.push({ chatId, messageId: sent.message_id, threadId: landedThreadId })
7075
7110
  // The card LANDED — the operator can see and tap it, so the block is over.
7076
- // Drop the hold mark and reconcile the off-Telegram surface. (PR 3 also
7077
- // resets `startedAt` here: the TTL measures how long the operator had to
7078
- // answer, and until this moment they had nothing to answer.)
7079
- if (live.undeliverable != null) {
7080
- live.undeliverable = null
7081
- live.redeliveryFailures = 0
7082
- // RESET THE TTL CLOCK. Load-bearing, not cosmetic. `startedAt` is when
7083
- // the agent asked; the TTL measures how long the operator had to
7084
- // answer. Until this instant they had NOTHING to answer — the card did
7085
- // not exist in any chat. Without the reset, a card held through a 4.6h
7086
- // ban lands already-expired against a 60-min TTL and the very next
7087
- // reaper tick auto-denies it: we would have MOVED the silent denial,
7088
- // not removed it.
7089
- live.startedAt = Date.now()
7111
+ // Drop the hold mark, reset the TTL clock, and reconcile the off-Telegram
7112
+ // surface as ONE shared decision (`applyDeliveredHoldReset`, #3128) that
7113
+ // the outcome test's harness ALSO drives, so deleting the `startedAt` reset
7114
+ // turns the behavioural `(d2)` test red instead of only a fragile grep.
7115
+ // PR 3: the reset is load-bearing — the TTL measures how long the operator
7116
+ // had to answer, and until this moment they had nothing to answer.
7117
+ if (applyDeliveredHoldReset(live, Date.now())) {
7090
7118
  reconcileBlockedApprovals()
7091
7119
  process.stderr.write(
7092
7120
  `telegram gateway: permission-card RE-DELIVERED request=${requestId} ` +
@@ -7910,6 +7938,28 @@ function emitGatewayOperatorEvent(event: OperatorEvent): void {
7910
7938
  // the old account fails — honest reset messaging on the enriched card.
7911
7939
  void fireFleetAutoFallback(agent, untilMs, modelUnavailable.resetAt)
7912
7940
  }
7941
+ } else if (kind === 'rate-limited' || kind === 'unknown-5xx') {
7942
+ // #llm-error-surfacing — surface #2 (the "🚦 Rate limited" operator card).
7943
+ // The transient rate-limit / overload family used to render the raw
7944
+ // synthetic-error `detail` (bytes and all) via renderOperatorEvent. Route
7945
+ // it through the humanized card instead: JSON-stripped coreText + reset in
7946
+ // LOCAL time. The ErrorPresenceGate is the cross-surface dedup authority —
7947
+ // the reply/done-card surfaces are already suppressed at the transcript
7948
+ // source (session-tail), so the operator card is the sole renderer and
7949
+ // wins the claim; a redundant burst within the collapse window is
7950
+ // suppressed here in addition to the existing 5-min per-kind cooldown.
7951
+ const parsed = parseLlmError(event.detail)
7952
+ const now = Date.now()
7953
+ if (decideErrorSurface(parsed, agent, { claim: true, now }) === 'suppress') {
7954
+ process.stderr.write(
7955
+ `telegram gateway: operator-event collapsed (error-presence-gate) agent=${agent} kind=${kind}\n`,
7956
+ )
7957
+ return
7958
+ }
7959
+ const tz = process.env.SWITCHROOM_TIMEZONE ?? process.env.TZ ?? 'UTC'
7960
+ const r = renderLlmError(parsed, agent, tz, new Date(now))
7961
+ renderedText = r.text
7962
+ renderedKeyboard = r.keyboard
7913
7963
  } else {
7914
7964
  try {
7915
7965
  const r = renderOperatorEvent(event)
@@ -8578,6 +8628,17 @@ async function runMidSessionCardReaper(): Promise<void> {
8578
8628
  const reaps = decideWorkerPinReaps({
8579
8629
  pins: candidates,
8580
8630
  statusOf: (agentId) => {
8631
+ // #3207: coalesced feed pins are GROUP-level (`wk:group:<feedKey>`),
8632
+ // so `workerAgentIdOfPinKey` yields `group:<feedKey>`, not a real
8633
+ // jsonl agent id. Vouch for these off the live feed instead of the
8634
+ // registry: a group the feed still tracks is 'running' (exempt from
8635
+ // the TTL); a lingering group pin the feed no longer knows about is a
8636
+ // missed unpin → 'terminal' (reap now). The feed's own group-empty
8637
+ // unpin is the primary path; this is the missed-unpin backstop.
8638
+ if (agentId.startsWith('group:')) {
8639
+ const feedKey = agentId.slice('group:'.length)
8640
+ return workerActivityFeed?.hasRunningInFeed(feedKey) ? 'running' : 'terminal'
8641
+ }
8581
8642
  if (turnsDb == null) return 'unknown'
8582
8643
  try {
8583
8644
  const row = getSubagentByJsonlId(turnsDb, agentId)
@@ -8736,34 +8797,14 @@ async function reconcileStatusPinInner(
8736
8797
  }
8737
8798
  }
8738
8799
 
8739
- /**
8740
- * Background-worker desired-pin, driven off the live `🛠 Worker` message.
8741
- * Reads the worker feed's current message_id (the EXISTING message we pin
8742
- * what the feed already rendered, never a new send) and reconciles a silent
8743
- * pin while it's running / an unpin on completion. No-op until the feed has
8744
- * actually painted a message for this worker (trivial sub-second workers stay
8745
- * silent and are never pinned). Keyed `wk:<agentId>`.
8746
- */
8747
- function reconcileWorkerPin(
8748
- agentId: string,
8749
- chatId: string | null,
8750
- running: boolean,
8751
- ): void {
8752
- if (!PIN_STATUS_WHILE_WORKING) return
8753
- const key = `wk:${agentId}`
8754
- if (!running) {
8755
- // Unpin: recover the chat we pinned in (caller may not have it at
8756
- // completion). No-op when nothing was pinned for this worker.
8757
- const unpinChat = chatId ?? statusPinChatIds.get(key)
8758
- if (unpinChat == null) return
8759
- void reconcileStatusPin(key, unpinChat, { pinned: false })
8760
- return
8761
- }
8762
- if (chatId == null) return
8763
- const messageId = workerActivityFeed?.messageIdOf(agentId) ?? null
8764
- if (messageId == null) return // no message painted yet — nothing to pin
8765
- void reconcileStatusPin(key, chatId, { pinned: true, messageId })
8766
- }
8800
+ // #3207: the per-worker `reconcileWorkerPin(agentId, …)` (keyed `wk:<agentId>`)
8801
+ // was removed. Now that background workers COALESCE into one shared message per
8802
+ // chat/thread, the pin is driven at the GROUP level by the feed itself
8803
+ // (`reconcilePin` `wk:group:<feedKey>`, wired at createWorkerActivityFeed):
8804
+ // it pins when the group's first worker paints and unpins only when the group
8805
+ // empties. A per-worker unpin used to physically unpin a message a sibling
8806
+ // still needed, after which the survivor's re-pin NO-OP'd (its claim still
8807
+ // named that id) — leaving live work unpinned (review blocker).
8767
8808
 
8768
8809
  /** Unpin every owned status pin — used by the pre-restart sweep so a
8769
8810
  * crash / interrupt never leaves a permanent pin behind. Best-effort;
@@ -10347,6 +10388,13 @@ const bridgeDeadWatchdog = createBridgeDeadWatchdog({
10347
10388
  // consecutive-escalation count. At the cap, arm() stands down loudly
10348
10389
  // instead of restart-looping a deterministically-failing bridge.
10349
10390
  priorStreak: bridgeDeadPriorStreak,
10391
+ // #3086 — only THIS gateway's own primary bridge (registering under
10392
+ // $SWITCHROOM_AGENT_NAME) drives the watchdog. A secondary/relay client
10393
+ // (e.g. `overlord-relay`) registering a different name into this socket
10394
+ // is named + non-cron but must NOT mark the (still-alive) primary bridge
10395
+ // dead when it disconnects. Empty string ⟹ fall back to the pre-#3086
10396
+ // "any named non-cron client" test inside the watchdog.
10397
+ selfAgentName: process.env.SWITCHROOM_AGENT_NAME ?? '',
10350
10398
  })
10351
10399
  if (BRIDGE_DEAD_ESCALATION_ENABLED) {
10352
10400
  bridgeDeadWatchdog.arm()
@@ -10386,10 +10434,11 @@ const ipcServer: IpcServer = createIpcServer({
10386
10434
  : []
10387
10435
  // #3038 — a REAL (named, non-cron) bridge registered: stand the
10388
10436
  // bridge-dead watchdog down. Anonymous clients (recall.py, mcp
10389
- // handshakes) and cron-session bridges must NOT satisfy it — the
10390
- // watchdog gates on the identity INTERNALLY (isRealBridgeIdentity), so
10391
- // this call is safe wherever it sits relative to the cron early-return
10392
- // above (#3038 review finding 5).
10437
+ // handshakes), cron-session bridges, and secondary/relay clients
10438
+ // registering a name other than this gateway's own agent (#3086) must
10439
+ // NOT satisfy it the watchdog gates on the identity INTERNALLY
10440
+ // (isRealBridgeIdentity vs selfAgentName), so this call is safe wherever
10441
+ // it sits relative to the cron early-return above (#3038 review finding 5).
10393
10442
  bridgeDeadWatchdog.noteBridgeRegistered(client.agentName)
10394
10443
  client.send({ type: 'status', status: 'agent_connected' })
10395
10444
 
@@ -10583,8 +10632,10 @@ const ipcServer: IpcServer = createIpcServer({
10583
10632
  // #3038 — the real bridge went away mid-life. Re-arm the grace
10584
10633
  // window: a normal claude restart re-registers within seconds and
10585
10634
  // stands it down; a bridge that died for good escalates once (the
10586
- // once-per-boot fuse inside the watchdog caps it). Cron/anonymous
10587
- // identities are ignored inside the watchdog itself (finding 5).
10635
+ // once-per-boot fuse inside the watchdog caps it). Cron/anonymous and
10636
+ // secondary/relay identities (a name other than this gateway's own
10637
+ // agent, #3086) are ignored inside the watchdog itself (finding 5) —
10638
+ // so a transient relay disconnect never bounces a healthy container.
10588
10639
  if (BRIDGE_DEAD_ESCALATION_ENABLED) bridgeDeadWatchdog.noteBridgeDisconnected(client.agentName)
10589
10640
  }
10590
10641
 
@@ -21939,6 +21990,10 @@ function recordTypedModelSwitch(
21939
21990
  }
21940
21991
  if (!reply.selectedModel) return ''
21941
21992
  sessionModelSource.setOverride(reply.selectedModel)
21993
+ // A manual /model apply to the dropped premium clears any pending recovery
21994
+ // marker so the "available again" ping can't still fire after the user has
21995
+ // already switched back themselves.
21996
+ clearPremiumRecoveryOnManualSwitch(reply.selectedModel)
21942
21997
  return ''
21943
21998
  }
21944
21999
 
@@ -21967,6 +22022,11 @@ function recordModelMenuSideEffects(
21967
22022
  // (recommended)" selection clears any leftover carrier.
21968
22023
  if (outcome.selectedModel) {
21969
22024
  sessionModelSource.setOverride(outcome.selectedModel)
22025
+ // Clear a pending premium-recovery marker when the tap re-selects the
22026
+ // dropped premium (menu OR the recovery ping's own switch-back button):
22027
+ // no stale "available again" ping once we're back on it. Idempotent — the
22028
+ // ping-send path already consumed the marker, so this is a no-op there.
22029
+ clearPremiumRecoveryOnManualSwitch(outcome.selectedModel)
21970
22030
  }
21971
22031
  if (outcome.clearedDefault) {
21972
22032
  const smDir = resolveAgentDirFromEnv()
@@ -23546,6 +23606,173 @@ function broadcastFleetFallbackFailure(triggerAgent: string, reason: string): vo
23546
23606
  }
23547
23607
  }
23548
23608
 
23609
+ /**
23610
+ * Broadcast a status notice to every authorized chat (system notice posture:
23611
+ * silenced ping, wrapped send). Shared by the tier-downgrade notices below.
23612
+ */
23613
+ function broadcastTierNotice(markdown: string): void {
23614
+ const access = loadAccess()
23615
+ if (access.allowFrom.length === 0) return
23616
+ for (const chat_id of access.allowFrom) {
23617
+ void swallowingApiCall(
23618
+ // allow-raw-bot-api: wrapped in swallowingApiCall (retry policy)
23619
+ () => bot.api.sendRichMessage(chat_id, richMessage(markdown), { disable_notification: true }),
23620
+ { chat_id: String(chat_id), verb: 'tier-downgrade:notify' },
23621
+ )
23622
+ }
23623
+ }
23624
+
23625
+ /**
23626
+ * MODEL-TIER downgrade failover (second recovery tier). Consulted ONLY from the
23627
+ * `all-blocked` branch of doFireFleetAutoFallback — i.e. AFTER account-swap has
23628
+ * been tried and found no account still serving the walled premium model
23629
+ * (precedence A: account-swap first). When a PREMIUM /model override is active
23630
+ * (distinct from the configured default), downgrade to the configured default
23631
+ * and resume the dead turn via a self-restart, rather than let the turn stall.
23632
+ *
23633
+ * - NO automatic return to the premium model. The /model override is
23634
+ * session-scoped and in-memory only (recordTypedModelSwitch writes no
23635
+ * carrier), so it dies on the downgrade SIGTERM. The consume-once
23636
+ * `.session-model` carrier written here targets the CONFIGURED DEFAULT:
23637
+ * start.sh applies+deletes it on the resume boot, and every later restart
23638
+ * also boots the default. The premium tier is never restored on its own —
23639
+ * the user must re-issue `/model <premium>`, which the notice says plainly.
23640
+ * - Effort is NATIVE: no `.session-effort` carrier is written, so the restart
23641
+ * sheds any live /effort override and the downgraded default boots at the
23642
+ * configured `thinking_effort` (the fleet `low` pin, #1978).
23643
+ * - Loop-bounded by the NATURAL on-default guard: after the downgrade boot the
23644
+ * session runs the configured default (override gone), so a re-entry returns
23645
+ * `skip` ('on-default') and never re-downgrades — even if the default is
23646
+ * itself walled (then the normal all-blocked card fires, no loop). Pacing
23647
+ * reuses the fleetFallbackResumeGate single-flight + 3h staleness.
23648
+ *
23649
+ * Returns:
23650
+ * 'downgraded' — carrier written, latch armed, restart fired; caller must
23651
+ * NOT also emit the all-blocked give-up card (we are
23652
+ * recovering, not giving up).
23653
+ * 'restart-pending' — a resume restart was ALREADY armed in this process (a
23654
+ * concurrent turn's downgrade or account-swap); that
23655
+ * restart replays the interrupted turn, so caller must NOT
23656
+ * emit a give-up card (avoids the contradictory "could not
23657
+ * be recovered" race).
23658
+ * 'skip' — not applicable (on the configured default, unresolved
23659
+ * default, or the turn is too stale to resume); caller
23660
+ * falls through to the all-blocked card.
23661
+ */
23662
+ function maybeTierDowngrade(triggerAgent: string): 'downgraded' | 'restart-pending' | 'skip' {
23663
+ // Thin adapter: the order-sensitive glue lives in runTierDowngrade
23664
+ // (tier-downgrade-wiring.ts) behind injected deps so it is unit-testable
23665
+ // without importing the gateway. This wires the real seams.
23666
+ return runTierDowngrade(triggerAgent, {
23667
+ getAgentDir: () => resolveAgentDirFromEnv() ?? null,
23668
+ getConfiguredDefault: () => {
23669
+ const dir = resolveAgentDirFromEnv()
23670
+ if (!dir) return null
23671
+ return resolveMainModel(readConfiguredDefaultModel(dir) ?? undefined)
23672
+ },
23673
+ getSessionOverride: () => sessionModelSource.getOverride(),
23674
+ resolve: (t) => resolveMainModel(t),
23675
+ // PEEK the resume gate WITHOUT arming (single-flight within this process +
23676
+ // 3h staleness), the same gate the account-swap resume path uses.
23677
+ peekResumeGate: () => fleetFallbackResumeGate.peek(newestActiveTurnStartedAtMs()),
23678
+ writeCarrier: (dir, toModel, cfg) => writeSessionModelFile(dir, toModel, cfg),
23679
+ armResumeGate: () => fleetFallbackResumeGate.arm(),
23680
+ // Records the DROPPED premium token + the notice's allowFrom chats; start.sh
23681
+ // never consumes `.premium-recovery`, so it survives the self-restart.
23682
+ writeRecoveryMarker: (dir, premiumModel) => {
23683
+ const chats = loadAccess().allowFrom.map((c) => String(c))
23684
+ if (chats.length > 0) writePremiumRecoveryFile(dir, premiumModel, chats)
23685
+ },
23686
+ broadcastNotice: (md) => broadcastTierNotice(md),
23687
+ selfRestart: (agent) => triggerSelfRestart(agent, 'tier-downgrade-resume'),
23688
+ selfAgent: (t) => process.env.SWITCHROOM_AGENT_NAME ?? t,
23689
+ log: (msg) => process.stderr.write(`telegram gateway: ${msg}\n`),
23690
+ })
23691
+ }
23692
+
23693
+ /**
23694
+ * "Premium model recovered" ping (tier-downgrade companion). Consulted from the
23695
+ * gateway's existing `runQuotaWatch` tick (every 15 min, on the cheap
23696
+ * `list-state` IPC read it already does — NO new poller, NO broker change).
23697
+ * When a `.premium-recovery` marker is pending (a downgrade dropped a premium
23698
+ * `/model` selection fleet-wide) AND the broker's live-authoritative per-account
23699
+ * eligibility shows the premium tier servable again — at least one account
23700
+ * neither `exhausted` nor `premium_walled` — fire EXACTLY ONE ping to the
23701
+ * recorded chats with a one-tap "switch back" button, then consume the marker.
23702
+ *
23703
+ * DETERMINISTIC: `exhausted` / `premium_walled` are the broker's own verdicts
23704
+ * (`isAccountExhausted` / `isAccountPremiumWalled` → `isModelTierWalled`, a pure
23705
+ * timestamp compare). No model judgement, no clock of our own.
23706
+ *
23707
+ * AT-MOST-ONCE: a fleet-wide `claim-notification` gates against a bounce or a
23708
+ * concurrent tick, and the marker is cleared BEFORE the send (never-storm). The
23709
+ * button routes through the SAME session-scoped `/model` apply path as the
23710
+ * model menu (`mdl:alias:<premium>` → handleModelMenuCallback +
23711
+ * recordModelMenuSideEffects) — it does not bypass it.
23712
+ */
23713
+ async function maybePremiumRecoveryPing(
23714
+ brokerClient: NonNullable<Awaited<ReturnType<typeof getAuthBrokerClient>>>,
23715
+ accounts: ReadonlyArray<{ exhausted: boolean; premium_walled?: boolean }>,
23716
+ ): Promise<void> {
23717
+ // Thin adapter: the order-sensitive never-storm glue lives in
23718
+ // runPremiumRecoveryPing (premium-recovery-wiring.ts) behind injected deps so
23719
+ // it is unit-testable without importing the gateway. This wires the real seams
23720
+ // (marker FS, fleet claim, send) — behaviour is preserved exactly.
23721
+ return runPremiumRecoveryPing({
23722
+ getAgentDir: () => resolveAgentDirFromEnv() ?? null,
23723
+ readMarker: (dir) => readPremiumRecoveryFile(dir),
23724
+ clearMarker: (dir) => clearPremiumRecoveryFile(dir),
23725
+ getAgent: () => getMyAgentName(),
23726
+ // The pure recovery predicate, bound to THIS tick's live per-account
23727
+ // eligibility (the broker's own verdicts; `premium_walled` may be absent).
23728
+ decide: () =>
23729
+ decidePremiumRecovery({
23730
+ hasMarker: true,
23731
+ accounts: accounts.map((a) => ({
23732
+ exhausted: a.exhausted,
23733
+ premiumWalled: a.premium_walled === true,
23734
+ })),
23735
+ }).fire,
23736
+ // Fail-open (claimQuotaNotification returns true on any broker error) — a
23737
+ // convenience ping degrades to at-least-once, never lost.
23738
+ claimNotification: (key) => claimQuotaNotification(brokerClient, key),
23739
+ fallbackChats: () => loadAccess().allowFrom.map((c) => String(c)),
23740
+ sendToChat: (chat_id, ping, keyboard) => {
23741
+ void swallowingApiCall(
23742
+ // allow-raw-bot-api: wrapped in swallowingApiCall (retry policy)
23743
+ () =>
23744
+ bot.api.sendRichMessage(chat_id, richMessage(ping.text), {
23745
+ disable_notification: true,
23746
+ reply_markup: keyboard,
23747
+ }),
23748
+ { chat_id: String(chat_id), verb: 'premium-recovery:notify' },
23749
+ )
23750
+ },
23751
+ log: (msg) => process.stderr.write(`telegram gateway: ${msg}\n`),
23752
+ })
23753
+ }
23754
+
23755
+ /**
23756
+ * Clear a pending premium-recovery marker when the user MANUALLY re-selects the
23757
+ * dropped premium model before the recovery ping fires — no stale "available
23758
+ * again" ping after they've already switched back. Called from every `/model`
23759
+ * apply chokepoint (typed + menu/callback). Matches on the canonicalized token
23760
+ * so an alias vs resolved-id spelling of the same model still clears it.
23761
+ */
23762
+ function clearPremiumRecoveryOnManualSwitch(appliedToken: string | null | undefined): void {
23763
+ if (appliedToken == null || appliedToken.length === 0) return
23764
+ const agentDir = resolveAgentDirFromEnv()
23765
+ if (!agentDir) return
23766
+ const marker = readPremiumRecoveryFile(agentDir)
23767
+ if (marker == null) return
23768
+ if (resolveMainModel(appliedToken) === resolveMainModel(marker.premiumModel)) {
23769
+ clearPremiumRecoveryFile(agentDir)
23770
+ process.stderr.write(
23771
+ `telegram gateway: [premium-recovery] marker cleared — user manually re-issued /model ${marker.premiumModel}\n`,
23772
+ )
23773
+ }
23774
+ }
23775
+
23549
23776
  /** Returns true iff the dispatcher actually performed a swap (and the
23550
23777
  * user-visible announcement was broadcast). False on no-op /
23551
23778
  * error / idempotent-skip — caller uses this to decide whether to
@@ -23628,6 +23855,21 @@ async function doFireFleetAutoFallback(
23628
23855
  if (outcome.kind === 'switched') {
23629
23856
  fallbackAllBlockedNoticeState = { lastSentAtMs: 0 }
23630
23857
  } else if (outcome.kind === 'all-blocked') {
23858
+ // ── Second recovery tier: MODEL-TIER downgrade (precedence A) ──────────
23859
+ // Account-swap just came back all-blocked (no account still serves the
23860
+ // walled model). Before giving up, if a PREMIUM /model override is active,
23861
+ // downgrade to the configured default and resume the dead turn rather than
23862
+ // stall. Only reachable here — a `switched` outcome never enters this
23863
+ // branch, so a model throttled on one account is always recovered by
23864
+ // account-swap first, never by a downgrade. Loop-bounded by the natural
23865
+ // on-default guard; effort revert is native (see maybeTierDowngrade).
23866
+ const tier = maybeTierDowngrade(triggerAgent)
23867
+ if (tier === 'downgraded' || tier === 'restart-pending') {
23868
+ // Either this turn armed a downgrade restart, or a concurrent turn
23869
+ // already armed a resume restart. Do NOT emit the all-blocked give-up
23870
+ // card — a restart is coming that replays the interrupted turn.
23871
+ return false
23872
+ }
23631
23873
  const verdict = evaluateAllBlockedNotice(fallbackAllBlockedNoticeState, Date.now())
23632
23874
  if (!verdict.send) {
23633
23875
  process.stderr.write(
@@ -23860,6 +24102,16 @@ async function runQuotaWatch(opts: { bootTick?: boolean } = {}): Promise<void> {
23860
24102
  return // No accounts — nothing to watch.
23861
24103
  }
23862
24104
 
24105
+ // Premium-model recovery ping (tier-downgrade companion). Reuses THIS tick's
24106
+ // list-state read (no extra IPC): if a downgrade left a `.premium-recovery`
24107
+ // marker and the broker now shows the premium tier servable again, ping once
24108
+ // with a one-tap switch-back button. At-most-once + fleet-claim-deduped.
24109
+ await maybePremiumRecoveryPing(brokerClient, listStateData.accounts).catch((err) => {
24110
+ process.stderr.write(
24111
+ `telegram gateway: [premium-recovery] ping check failed (non-fatal): ${(err as Error)?.message ?? err}\n`,
24112
+ )
24113
+ })
24114
+
23863
24115
  // Build AccountSnapshot[] from cached broker state only — no live probe.
23864
24116
  // Accounts with null last_quota produce quota=null snapshots; classifyHealth
23865
24117
  // returns 'unknown'; evaluateQuotaWatchAccount skips — no false alarms.
@@ -29100,6 +29352,12 @@ void (async () => {
29100
29352
  // supersedes the coarse 5-min bucket relay below to avoid
29101
29353
  // double-surfacing the same progress beat.
29102
29354
  const workerFeedEnabled = isWorkerActivityFeedEnabled(process.env.SWITCHROOM_WORKER_ACTIVITY_FEED)
29355
+ // Combined-feed row cap (channels.telegram.worker_feed.max_rows).
29356
+ // Unset / non-positive → the feed's built-in default (8).
29357
+ const workerFeedMaxRows = (() => {
29358
+ const raw = Number(process.env.SWITCHROOM_TG_WORKER_FEED_MAX_ROWS)
29359
+ return Number.isInteger(raw) && raw > 0 ? raw : undefined
29360
+ })()
29103
29361
  // Model A — foreground sub-agent nesting in the parent's live
29104
29362
  // activity draft. ON by default; this edits the SAME activity-
29105
29363
  // summary message the tool_label feed already owns (not the
@@ -29157,6 +29415,32 @@ void (async () => {
29157
29415
  // every ~6s for the whole ban (the worker-feed shed-contract bug:
29158
29416
  // 565 `sent.message_id` crashes in one 6h ban).
29159
29417
  floodWaitRemainingMs: probeFloodWaitRemainingMs,
29418
+ // #3084 follow-up: 2+ background workers in one chat/thread coalesce
29419
+ // into ONE combined feed message. `maxRows` caps the visible rows
29420
+ // (compact `+M more working…` spill) so the body stays under the
29421
+ // rich-message wire ceiling (STATUS_CARD_CHAR_BUDGET). Sourced from
29422
+ // channels.telegram.worker_feed.max_rows via the config cascade
29423
+ // (scaffold emits SWITCHROOM_TG_WORKER_FEED_MAX_ROWS); unset → 8.
29424
+ maxRows: workerFeedMaxRows,
29425
+ // #3207 review: GROUP-level status pin. Workers now coalesce into
29426
+ // ONE shared message, so the pin must follow the GROUP lifecycle,
29427
+ // not a single worker's — otherwise a sibling's finish unpins a
29428
+ // message the survivors still need and the survivor's re-pin NOOPs
29429
+ // (its claim still names that id), leaving live work unpinned. The
29430
+ // feed drives this: pin `wk:group:<feedKey>` when the group first
29431
+ // paints, unpin only when the group empties (messageId === null).
29432
+ reconcilePin: ({ feedKey, chatId, messageId }) => {
29433
+ if (!PIN_STATUS_WHILE_WORKING) return
29434
+ const key = `wk:group:${feedKey}`
29435
+ if (messageId != null) {
29436
+ void reconcileStatusPin(key, chatId, { pinned: true, messageId })
29437
+ } else {
29438
+ const unpinChat = chatId || statusPinChatIds.get(key)
29439
+ if (unpinChat != null && unpinChat.length > 0) {
29440
+ void reconcileStatusPin(key, unpinChat, { pinned: false })
29441
+ }
29442
+ }
29443
+ },
29160
29444
  log: (msg) => process.stderr.write(`telegram gateway: ${msg}\n`),
29161
29445
  })
29162
29446
  subagentWatcher = startSubagentWatcher({
@@ -29273,9 +29557,9 @@ void (async () => {
29273
29557
  // anything. The actual repaint + re-pin happen DOWNSTREAM on the
29274
29558
  // next replayed `running` cue: the watcher's re-registration
29275
29559
  // replays `onProgress`, which calls `workerActivityFeed.update()`
29276
- // (now un-gated) to first-paint a FRESH `🛠 Worker` message, and
29277
- // its `.then(reconcileWorkerPin(agentId, wkChat, true))` pins that
29278
- // new message via the `wk:<agentId>` status-pin. Clearing the gate
29560
+ // (now un-gated) to first-paint a FRESH message, and the feed's
29561
+ // own `reconcilePin` (#3207) re-pins that message at the GROUP
29562
+ // level (`wk:group:<feedKey>`). Clearing the gate
29279
29563
  // here FIRST is the ordering requirement — without it those first
29280
29564
  // replayed ticks would be swallowed by the finalized gate and no
29281
29565
  // new card would ever paint. Net effect restores the operator
@@ -29381,7 +29665,7 @@ void (async () => {
29381
29665
  // card keeps the model tag even with no live entry.
29382
29666
  model: dispatch.feedModel ?? undefined,
29383
29667
  })
29384
- reconcileWorkerPin(agentId, null, false)
29668
+ // #3207: group-level pin dropped by the feed on group-empty.
29385
29669
  }
29386
29670
  return
29387
29671
  }
@@ -29459,8 +29743,9 @@ void (async () => {
29459
29743
  state: outcome === 'failed' ? 'failed' : 'done',
29460
29744
  model: dispatch.feedModel ?? undefined,
29461
29745
  })
29462
- // Status-pin: worker done drop its pin.
29463
- reconcileWorkerPin(agentId, null, false)
29746
+ // #3207: the group-level pin is dropped by the feed itself
29747
+ // when this group's LAST worker finishes (reconcilePin
29748
+ // `wk:group:<feedKey>`); no per-worker unpin here.
29464
29749
  }
29465
29750
  return
29466
29751
  }
@@ -29479,8 +29764,8 @@ void (async () => {
29479
29764
  state: outcome === 'failed' ? 'failed' : 'done',
29480
29765
  model: dispatch.feedModel ?? undefined,
29481
29766
  })
29482
- // Status-pin: worker done drop its pin.
29483
- reconcileWorkerPin(agentId, null, false)
29767
+ // #3207: the group-level pin is dropped by the feed itself
29768
+ // when this group's LAST worker finishes; no per-worker unpin.
29484
29769
  }
29485
29770
 
29486
29771
  const handbackOrigin = resolveSubagentOriginChat(agentId)
@@ -29647,7 +29932,10 @@ void (async () => {
29647
29932
  model: feedModel,
29648
29933
  },
29649
29934
  wk.threadId,
29650
- )?.then(() => reconcileWorkerPin(agentId, wkChat, true))
29935
+ )
29936
+ // #3207: the feed pins the group's shared message itself when
29937
+ // it first paints (reconcilePin → `wk:group:<feedKey>`); no
29938
+ // per-worker pin chained here.
29651
29939
  return
29652
29940
  }
29653
29941
  if (surface !== 'nest') return // 'skip' — orphan-status off
@@ -29789,7 +30077,10 @@ void (async () => {
29789
30077
  model: feedModel,
29790
30078
  },
29791
30079
  wk.threadId,
29792
- )?.then(() => reconcileWorkerPin(agentId, wkChat, true))
30080
+ )
30081
+ // #3207: the feed pins the group's shared message itself when
30082
+ // it first paints (reconcilePin → `wk:group:<feedKey>`); no
30083
+ // per-worker pin chained here.
29793
30084
  return
29794
30085
  }
29795
30086