switchroom 0.18.8 → 0.18.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/README.md +2 -2
  2. package/dist/agent-scheduler/index.js +1 -0
  3. package/dist/auth-broker/index.js +198 -13
  4. package/dist/cli/notion-write-pretool.mjs +1 -0
  5. package/dist/cli/switchroom.js +28 -4
  6. package/dist/host-control/main.js +3 -2
  7. package/dist/vault/approvals/kernel-server.js +2 -1
  8. package/dist/vault/broker/server.js +2 -1
  9. package/package.json +1 -1
  10. package/profiles/_base/start.sh.hbs +119 -37
  11. package/profiles/_shared/dev-protocol.md.hbs +42 -0
  12. package/skills/dev-protocol/SKILL.md +131 -0
  13. package/telegram-plugin/README.md +2 -1
  14. package/telegram-plugin/admin-commands/dispatch.test.ts +40 -2
  15. package/telegram-plugin/admin-commands/index.ts +6 -1
  16. package/telegram-plugin/bridge/bridge.ts +23 -1
  17. package/telegram-plugin/bridge/crash-breadcrumb.ts +42 -0
  18. package/telegram-plugin/chat-lock.ts +13 -0
  19. package/telegram-plugin/dist/bridge/bridge.js +24 -1
  20. package/telegram-plugin/dist/gateway/gateway.js +80236 -77465
  21. package/telegram-plugin/dist/server.js +29 -2
  22. package/telegram-plugin/fallback-card-collapse.ts +131 -0
  23. package/telegram-plugin/gateway/approval-card-stores.ts +99 -0
  24. package/telegram-plugin/gateway/bot-commands-ops-info.ts +194 -0
  25. package/telegram-plugin/gateway/bridge-dead-watchdog.ts +546 -0
  26. package/telegram-plugin/gateway/callback-query-handlers.ts +2660 -0
  27. package/telegram-plugin/gateway/effort-command.ts +47 -3
  28. package/telegram-plugin/gateway/gateway.ts +2051 -3180
  29. package/telegram-plugin/gateway/inbound-delivery-machine-dispatch.ts +181 -23
  30. package/telegram-plugin/gateway/inbound-delivery-machine.ts +8 -0
  31. package/telegram-plugin/gateway/model-command.ts +94 -8
  32. package/telegram-plugin/gateway/outbound-send-path.ts +375 -0
  33. package/telegram-plugin/gateway/pending-session-command.ts +365 -0
  34. package/telegram-plugin/gateway/pending-state-stores.ts +106 -0
  35. package/telegram-plugin/gateway/permission-timeout.ts +25 -0
  36. package/telegram-plugin/gateway/register-bot-commands.ts +30 -0
  37. package/telegram-plugin/gateway/resume-inbound-builder.ts +23 -3
  38. package/telegram-plugin/gateway/session-model-file.ts +166 -23
  39. package/telegram-plugin/gateway/stop-command.ts +56 -0
  40. package/telegram-plugin/photo-precheck.ts +201 -0
  41. package/telegram-plugin/quota-watch.ts +141 -2
  42. package/telegram-plugin/registry/subagents-schema.ts +26 -3
  43. package/telegram-plugin/registry/subagents.test.ts +67 -0
  44. package/telegram-plugin/retry-api-call.ts +31 -0
  45. package/telegram-plugin/subagent-watcher.ts +392 -1
  46. package/telegram-plugin/tests/approval-card-stores.test.ts +124 -0
  47. package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +576 -0
  48. package/telegram-plugin/tests/buffer-gate-broadened.test.ts +11 -5
  49. package/telegram-plugin/tests/callback-query-handlers.test.ts +701 -0
  50. package/telegram-plugin/tests/chat-lock-unhandled-rejection.test.ts +101 -0
  51. package/telegram-plugin/tests/crash-breadcrumb.test.ts +57 -0
  52. package/telegram-plugin/tests/effort-command.test.ts +59 -2
  53. package/telegram-plugin/tests/emission-determinism-wiring.test.ts +11 -4
  54. package/telegram-plugin/tests/fallback-card-collapse.test.ts +104 -0
  55. package/telegram-plugin/tests/fixtures/cutover-killswitch-probe.ts +75 -0
  56. package/telegram-plugin/tests/gateway-outbound-redact.test.ts +5 -1
  57. package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +124 -0
  58. package/telegram-plugin/tests/gateway-secret-detect.test.ts +7 -1
  59. package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +19 -11
  60. package/telegram-plugin/tests/inbound-delivery-cutover-flip.test.ts +418 -0
  61. package/telegram-plugin/tests/inbound-delivery-dispatch-equivalence.test.ts +348 -0
  62. package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +141 -52
  63. package/telegram-plugin/tests/mental-model-propose-callback-gate.test.ts +8 -1
  64. package/telegram-plugin/tests/model-command.test.ts +46 -3
  65. package/telegram-plugin/tests/outbound-send-chunks.test.ts +304 -0
  66. package/telegram-plugin/tests/outbound-send-path.test.ts +222 -0
  67. package/telegram-plugin/tests/pending-card-durability-wiring.test.ts +34 -15
  68. package/telegram-plugin/tests/pending-session-command.test.ts +322 -0
  69. package/telegram-plugin/tests/pending-state-stores.test.ts +235 -0
  70. package/telegram-plugin/tests/permission-timeout.test.ts +26 -0
  71. package/telegram-plugin/tests/permission-verdict-resume-guard.test.ts +16 -0
  72. package/telegram-plugin/tests/photo-dimension-fallback.test.ts +129 -0
  73. package/telegram-plugin/tests/photo-precheck.test.ts +240 -0
  74. package/telegram-plugin/tests/photo-reroute-wiring.test.ts +85 -0
  75. package/telegram-plugin/tests/quota-watch.test.ts +225 -0
  76. package/telegram-plugin/tests/session-model-file.test.ts +101 -2
  77. package/telegram-plugin/tests/stop-command.test.ts +234 -0
  78. package/telegram-plugin/tests/subagent-watcher-env-thresholds.test.ts +27 -9
  79. package/telegram-plugin/tests/subagent-watcher-resurrection.test.ts +398 -0
  80. package/telegram-plugin/tests/subagent-watcher-stall-terminal.test.ts +172 -0
  81. package/telegram-plugin/tests/turn-flush-safety.test.ts +18 -4
  82. package/telegram-plugin/tests/vault-approval-posture.test.ts +15 -7
  83. package/telegram-plugin/tests/vault-grant-auto-resume.test.ts +8 -4
  84. package/telegram-plugin/tests/vault-grant-union.test.ts +8 -4
  85. package/telegram-plugin/tests/vault-grant-wizard.test.ts +8 -1
  86. package/telegram-plugin/tests/vault-grants-revoke.test.ts +8 -1
  87. package/telegram-plugin/tests/vault-key-regex-allows-slash.test.ts +8 -4
  88. package/telegram-plugin/tests/vault-request-access-tool.test.ts +8 -4
  89. package/telegram-plugin/tests/vault-request-access-unlock-resume.test.ts +8 -4
  90. package/telegram-plugin/tests/worker-activity-feed.test.ts +37 -0
  91. package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +18 -4
  92. package/telegram-plugin/welcome-text.ts +4 -3
  93. package/telegram-plugin/worker-activity-feed.ts +27 -0
@@ -0,0 +1,546 @@
1
+ /**
2
+ * bridge-dead-watchdog.ts — gateway-side escalation when the MCP bridge
3
+ * never (re)registers (#3038, follow-up to #3033 / PR #3037).
4
+ *
5
+ * Failure mode: the gateway crashes and its supervisor relaunches it in
6
+ * ~1s, but the bridge MCP-server process inside the RUNNING claude
7
+ * session dies with it. Claude Code never respawns a dead MCP server, so
8
+ * the session stays alive but toolless and mute ("No such tool
9
+ * available: mcp__switchroom-telegram__reply") while the gateway sits at
10
+ * `bridge_dead` buffering inbounds — in the 2026-07-11 clerk incident,
11
+ * 7 minutes of a mute agent until a manual container restart.
12
+ *
13
+ * PR #3037 hardened the bridge against dying (uncaughtException handler
14
+ * + crash breadcrumbs to STATE_DIR/bridge-crash.log). This module is the
15
+ * structural belt-and-braces: after gateway boot (or after a main-bridge
16
+ * disconnect), if no main-agent bridge registers within a grace window
17
+ * while the claude session process is demonstrably alive, escalate by
18
+ * bouncing the whole container via the existing self-restart machinery
19
+ * (`triggerSelfRestart`) with the distinct reason
20
+ * `bridge-dead-resume`. The container restart respawns claude, which
21
+ * respawns the MCP bridge — the only recovery path that exists.
22
+ *
23
+ * Guard rails:
24
+ * - at most ONE escalation per gateway boot (in-memory fuse), AND at
25
+ * most MAX_CONSECUTIVE_ESCALATIONS across boots (the escalation
26
+ * marker carries a consecutive-fire count — see below): a bridge
27
+ * that dies deterministically at startup must not restart-loop the
28
+ * agent forever;
29
+ * - skipped while the gateway is mid-shutdown;
30
+ * - skipped when the claude session process is NOT alive (that is a
31
+ * different failure class — container boot still in progress, or
32
+ * claude itself down — where a bounce is either premature or
33
+ * futile); the timer re-arms so a slow-booting claude gets the full
34
+ * grace window again once it appears;
35
+ * - cron-session bridges (`<agent>-cron`) and anonymous IPC clients
36
+ * (recall.py one-shots, pre-handshake connects) neither satisfy nor
37
+ * re-arm the watchdog — the gating lives INSIDE noteBridge* so a
38
+ * gateway handler refactor can't silently reintroduce the
39
+ * bounce-after-cron-fire false positive;
40
+ * - loud, structured escalation log line including a fresh
41
+ * bridge-crash.log tail (PR #3037 breadcrumbs) so the operator sees
42
+ * WHY from the supervisor log alone.
43
+ *
44
+ * Honest resume: before firing the restart, the watchdog persists an
45
+ * escalation marker (`STATE_DIR/bridge-dead-escalation.json`). The next
46
+ * boot consumes it and (a) appends the real cause to the synthetic boot
47
+ * resume/report inbound when work was in flight, or (b) synthesizes a
48
+ * minimal idle notice (buildBridgeDeadIdleNoticeInbound) when nothing
49
+ * was — either way the agent tells the user "my chat bridge died and the
50
+ * framework restarted me" instead of implying a watchdog timeout or an
51
+ * operator restart, and the chat is never left wondering why the
52
+ * container bounced.
53
+ *
54
+ * Pure-ish module: all IO (fs, timers, process liveness, restart) is
55
+ * injected so the decision logic and the controller are unit-testable
56
+ * without booting a gateway.
57
+ */
58
+
59
+ import { readFileSync, writeFileSync, renameSync, unlinkSync } from 'node:fs'
60
+ import { isCronIdentity } from './cron-session.js'
61
+ import type { InboundMessage } from './ipc-protocol.js'
62
+
63
+ /** The distinct triggerSelfRestart reason for this escalation. Classified
64
+ * as intent 'keep' by intentForRestartReason (recovery bounce — session
65
+ * model stickiness preserved), like every other switchroom-managed
66
+ * relaunch. */
67
+ export const BRIDGE_DEAD_RESTART_REASON = 'bridge-dead-resume'
68
+
69
+ /** Default grace window before a missing bridge is treated as dead.
70
+ * Override with SWITCHROOM_BRIDGE_DEAD_GRACE_MS. Chosen well above the
71
+ * normal boot register time (bridge registers within ~1-5s of claude
72
+ * start) but well below the 7-minute mute window of the incident. */
73
+ export const DEFAULT_BRIDGE_DEAD_GRACE_MS = 90_000
74
+
75
+ /** Escalation marker freshness window at next boot. A marker older than
76
+ * this is stale debris (the escalated restart happened long ago or never
77
+ * completed) — cleared without surfacing, and the consecutive-escalation
78
+ * streak resets with it. */
79
+ export const ESCALATION_MARKER_MAX_AGE_MS = 10 * 60_000
80
+
81
+ /** Cross-boot restart-loop damper: the once-per-boot fuse is in-memory
82
+ * and a container bounce RESETS it, so a bridge that dies
83
+ * deterministically at startup (corrupt plugin bundle, bad .mcp.json)
84
+ * would otherwise bounce the container every grace window forever,
85
+ * spamming boot cards. The escalation marker carries a consecutive-fire
86
+ * `count`; a boot that consumed a marker with count >= this cap stands
87
+ * the watchdog down (loud audit line, operator investigates) instead of
88
+ * firing again. A successful bridge registration at any boot breaks the
89
+ * streak. With the default cap of 2 the framework attempts the
90
+ * self-heal twice, then stops. */
91
+ export const MAX_CONSECUTIVE_ESCALATIONS = 2
92
+
93
+ /** Crash-breadcrumb entries older than this are not "fresh" — they refer
94
+ * to some earlier incident and would mislead the escalation log line. */
95
+ export const CRASH_LOG_FRESH_WINDOW_MS = 15 * 60_000
96
+
97
+ /** Max breadcrumb lines quoted into the escalation log / marker. */
98
+ const CRASH_LOG_TAIL_LINES = 3
99
+
100
+ // ─── Pure decision ───────────────────────────────────────────────────────────
101
+
102
+ export interface BridgeDeadSnapshot {
103
+ /** A real (named, non-cron) bridge is currently registered. */
104
+ bridgeRegistered: boolean
105
+ /** A real bridge registered at some point THIS boot (breaks the
106
+ * cross-boot escalation streak even if it later disconnected). */
107
+ bridgeEverRegistered: boolean
108
+ /** The claude session process exists in the container. */
109
+ sessionAlive: boolean
110
+ /** The gateway is mid-shutdown (SIGTERM/SIGINT handler ran). */
111
+ shuttingDown: boolean
112
+ /** This gateway boot already fired the one allowed escalation. */
113
+ alreadyEscalated: boolean
114
+ /** Consecutive escalations by PRIOR boots (from the consumed marker;
115
+ * 0 when no fresh marker was found). */
116
+ priorStreak: number
117
+ /** The cross-boot cap (MAX_CONSECUTIVE_ESCALATIONS unless a test
118
+ * overrides). */
119
+ maxConsecutive: number
120
+ }
121
+
122
+ export type BridgeDeadDecision =
123
+ | { action: 'escalate' }
124
+ | {
125
+ action: 'skip'
126
+ why: 'bridge-registered' | 'shutting-down' | 'already-escalated' | 'streak-capped'
127
+ }
128
+ | { action: 'retry'; why: 'session-not-alive' }
129
+
130
+ /**
131
+ * Decide what the watchdog should do when the grace window expires.
132
+ * Precedence, highest first: a live bridge always wins; a shutdown in
133
+ * progress must never be raced by a restart; the once-per-boot fuse and
134
+ * the cross-boot streak cap are checked before anything fires; a missing
135
+ * claude process re-arms rather than escalating (bouncing a container
136
+ * whose claude never came up would loop without fixing anything — and a
137
+ * container still booting deserves the full grace window once claude
138
+ * appears).
139
+ */
140
+ export function decideBridgeDeadEscalation(s: BridgeDeadSnapshot): BridgeDeadDecision {
141
+ if (s.bridgeRegistered) return { action: 'skip', why: 'bridge-registered' }
142
+ if (s.shuttingDown) return { action: 'skip', why: 'shutting-down' }
143
+ if (s.alreadyEscalated) return { action: 'skip', why: 'already-escalated' }
144
+ // Cross-boot damper: prior boots already escalated `priorStreak` times
145
+ // in a row and the bridge STILL hasn't registered this boot — the bounce
146
+ // is not fixing it. Stand down. (A registration this boot breaks the
147
+ // streak: the current outage is then a NEW incident, not the same one.)
148
+ if (!s.bridgeEverRegistered && s.priorStreak >= s.maxConsecutive) {
149
+ return { action: 'skip', why: 'streak-capped' }
150
+ }
151
+ if (!s.sessionAlive) return { action: 'retry', why: 'session-not-alive' }
152
+ return { action: 'escalate' }
153
+ }
154
+
155
+ // ─── Crash-breadcrumb tail (PR #3037 integration) ────────────────────────────
156
+
157
+ /**
158
+ * Read the fresh tail of STATE_DIR/bridge-crash.log. Returns the last few
159
+ * breadcrumb lines whose leading ISO timestamp is within
160
+ * `freshWindowMs` of `nowMs`, joined with " || " (single log line —
161
+ * greppable, bounded). Returns null when the file is missing, unreadable,
162
+ * or has no fresh entries. Never throws.
163
+ */
164
+ export function readFreshCrashLogTail(
165
+ path: string,
166
+ opts: {
167
+ nowMs?: number
168
+ freshWindowMs?: number
169
+ maxLines?: number
170
+ readFile?: (p: string) => string
171
+ } = {},
172
+ ): string | null {
173
+ const nowMs = opts.nowMs ?? Date.now()
174
+ const freshWindowMs = opts.freshWindowMs ?? CRASH_LOG_FRESH_WINDOW_MS
175
+ const maxLines = opts.maxLines ?? CRASH_LOG_TAIL_LINES
176
+ const readFile = opts.readFile ?? ((p: string) => readFileSync(p, 'utf8'))
177
+ let raw: string
178
+ try {
179
+ raw = readFile(path)
180
+ } catch {
181
+ return null
182
+ }
183
+ const fresh = raw
184
+ .split('\n')
185
+ .filter((line) => {
186
+ const m = line.match(/^(\d{4}-\d{2}-\d{2}T[0-9:.]+Z)\s/)
187
+ if (!m) return false
188
+ const t = Date.parse(m[1])
189
+ return Number.isFinite(t) && nowMs - t >= 0 && nowMs - t <= freshWindowMs
190
+ })
191
+ .slice(-maxLines)
192
+ if (fresh.length === 0) return null
193
+ // Bound each quoted line — breadcrumbs are already capped at 4000 chars
194
+ // by the bridge, but one escalation log line quoting 3×4000 is noise.
195
+ return fresh.map((l) => l.slice(0, 500)).join(' || ')
196
+ }
197
+
198
+ // ─── Escalation marker (honest boot-resume cause + cross-boot damper) ────────
199
+
200
+ export interface BridgeDeadEscalationMarker {
201
+ /** Wall-clock ms when the escalation fired. */
202
+ ts: number
203
+ /** Always BRIDGE_DEAD_RESTART_REASON — kept in the file for grep-ability. */
204
+ reason: string
205
+ /** Consecutive escalations INCLUDING this one (1 = first fire of a
206
+ * streak). The next boot reads this as its `priorStreak` and stands
207
+ * down at MAX_CONSECUTIVE_ESCALATIONS. Absent in pre-damper markers —
208
+ * treated as 1. */
209
+ count?: number
210
+ /** Fresh crash-breadcrumb tail at escalation time, if any. */
211
+ crashTail?: string
212
+ }
213
+
214
+ export function writeBridgeDeadEscalationMarker(
215
+ path: string,
216
+ marker: BridgeDeadEscalationMarker,
217
+ ): void {
218
+ // Atomic tmp+rename so a partial write can't be read back as malformed
219
+ // JSON by the next boot (same discipline as clean-shutdown-marker.ts).
220
+ const tmp = `${path}.tmp-${process.pid}-${Date.now()}`
221
+ writeFileSync(tmp, JSON.stringify(marker), 'utf8')
222
+ renameSync(tmp, path)
223
+ }
224
+
225
+ /**
226
+ * Read + consume the escalation marker at boot. Returns the marker only
227
+ * when it is fresh (< maxAgeMs); a stale or malformed marker is cleared
228
+ * and ignored. The file is ALWAYS removed — the cause note must surface
229
+ * on exactly the boot that follows the escalation, never a later one.
230
+ *
231
+ * Known race window (accepted, documented per review): the marker is
232
+ * written by gateway boot N and consumed by whichever gateway boots NEXT.
233
+ * Normally that is the post-container-restart gateway (the SIGTERM to
234
+ * PID 1 fires ~1.5s after the write and takes the whole container down).
235
+ * But if the escalating gateway PROCESS dies and its supervisor relaunches
236
+ * a new gateway inside the same container before the SIGTERM lands, that
237
+ * interim gateway consumes the marker instead — the cause note is then
238
+ * surfaced (or dropped with the interim process) one boot early, and the
239
+ * post-restart boot sees no marker. Consequences are bounded and safe:
240
+ * the honesty note may be lost for one incident (the loud supervisor-log
241
+ * audit line survives regardless), and the cross-boot streak damper
242
+ * under-counts by one (strictly MORE willing to self-heal, never a
243
+ * tighter loop, since the in-memory once-per-boot fuse still caps each
244
+ * process at one fire). Not worth a consume-side handshake.
245
+ */
246
+ export function consumeBridgeDeadEscalationMarker(
247
+ path: string,
248
+ nowMs: number = Date.now(),
249
+ maxAgeMs: number = ESCALATION_MARKER_MAX_AGE_MS,
250
+ ): BridgeDeadEscalationMarker | null {
251
+ let marker: BridgeDeadEscalationMarker | null = null
252
+ try {
253
+ const parsed = JSON.parse(readFileSync(path, 'utf8')) as Partial<BridgeDeadEscalationMarker>
254
+ if (
255
+ typeof parsed.ts === 'number' &&
256
+ Number.isFinite(parsed.ts) &&
257
+ typeof parsed.reason === 'string'
258
+ ) {
259
+ const age = nowMs - parsed.ts
260
+ if (age >= 0 && age < maxAgeMs) {
261
+ marker = { ts: parsed.ts, reason: parsed.reason }
262
+ marker.count =
263
+ typeof parsed.count === 'number' && Number.isFinite(parsed.count) && parsed.count >= 1
264
+ ? Math.floor(parsed.count)
265
+ : 1
266
+ if (typeof parsed.crashTail === 'string') marker.crashTail = parsed.crashTail
267
+ }
268
+ }
269
+ } catch {
270
+ /* missing or malformed — nothing to surface */
271
+ }
272
+ try {
273
+ unlinkSync(path)
274
+ } catch {
275
+ /* best effort */
276
+ }
277
+ return marker
278
+ }
279
+
280
+ // ─── Idle notice (honest cause when NO turn was in flight) ───────────────────
281
+
282
+ /**
283
+ * Build the minimal synthetic inbound surfaced when the previous boot was
284
+ * a bridge-dead escalation but NO turn was interrupted (idle agent, dead
285
+ * bridge). Without this the cause reaches only stderr — the agent and the
286
+ * user never learn why the container bounced (review finding 2 on #3038).
287
+ * Mirrors the resume-builder shapes: meta.source is what Claude Code
288
+ * renders as `<channel source="…">`; meta.chat_id/message_id let the
289
+ * gateway's enqueue path build a currentTurn + ack the synthetic.
290
+ */
291
+ export function buildBridgeDeadIdleNoticeInbound(args: {
292
+ /** Chat to surface the notice in (the agent's default/owner chat). */
293
+ chatId: string
294
+ /** The consumed escalation marker. */
295
+ marker: BridgeDeadEscalationMarker
296
+ /** Wall-clock ms; defaults to Date.now(). */
297
+ nowMs?: number
298
+ }): InboundMessage {
299
+ const ts = args.nowMs ?? Date.now()
300
+ return {
301
+ type: 'inbound',
302
+ chatId: args.chatId,
303
+ messageId: ts,
304
+ user: 'switchroom',
305
+ userId: 0,
306
+ ts,
307
+ text:
308
+ `You just restarted. The framework itself triggered this restart: your Telegram ` +
309
+ `MCP bridge process had died (chat tools were unavailable — you could not send ` +
310
+ `replies), so the container was bounced to restore the chat surface. No work was ` +
311
+ `in flight when it happened. Briefly let the user know the messaging bridge died ` +
312
+ `and the framework restarted you to fix it — one short message, no drama. This was ` +
313
+ `NOT an operator-initiated restart and NOT a hang-watchdog kill; report only that ` +
314
+ `honest cause.`,
315
+ meta: {
316
+ source: 'bridge_dead_restart',
317
+ chat_id: args.chatId,
318
+ message_id: String(ts),
319
+ restart_cause: args.marker.reason,
320
+ },
321
+ }
322
+ }
323
+
324
+ // ─── Watchdog controller ─────────────────────────────────────────────────────
325
+
326
+ export interface BridgeDeadWatchdogOpts {
327
+ /** Grace window before a missing bridge is treated as dead. Doubled
328
+ * when the consumed marker shows a prior escalation streak (the retry
329
+ * after a failed self-heal deserves more patience). */
330
+ graceMs: number
331
+ /** Is the claude session process alive in this container? Callers
332
+ * should require a confident match (comm === 'claude'), NOT the
333
+ * heaviest-node fallback — an orphaned node process must not flip a
334
+ * retry into an escalation (review finding 4). */
335
+ isSessionAlive: () => boolean
336
+ /** Is the gateway mid-shutdown? */
337
+ isShuttingDown: () => boolean
338
+ /** Fire the container bounce (triggerSelfRestart wrapper). Returns
339
+ * whether the restart was actually dispatched. */
340
+ escalate: (reason: string) => boolean
341
+ /** STATE_DIR/bridge-crash.log — PR #3037 breadcrumbs. */
342
+ crashLogPath: string
343
+ /** STATE_DIR/bridge-dead-escalation.json — honest-resume marker. */
344
+ markerPath: string
345
+ /** Structured log sink (stderr). */
346
+ log: (line: string) => void
347
+ /** Consecutive escalations by prior boots (the consumed marker's
348
+ * `count`; 0 when no fresh marker). Drives the cross-boot damper. */
349
+ priorStreak?: number
350
+ /** Cross-boot cap override (tests). */
351
+ maxConsecutive?: number
352
+ /** Injectable timer pair for tests. Defaults to global setTimeout with
353
+ * unref (the watchdog must never keep the gateway process alive). */
354
+ setTimer?: (fn: () => void, ms: number) => unknown
355
+ clearTimer?: (handle: unknown) => void
356
+ /** Injectable clock (marker ts + crash-log freshness). */
357
+ nowMs?: () => number
358
+ /** Injectable marker writer (tests avoid real fs). */
359
+ writeMarker?: (path: string, marker: BridgeDeadEscalationMarker) => void
360
+ /** Injectable crash-log reader (tests avoid real fs). */
361
+ readCrashTail?: (path: string, nowMs: number) => string | null
362
+ }
363
+
364
+ export interface BridgeDeadWatchdog {
365
+ /** Arm the grace timer (gateway boot). Idempotent while armed. No-op
366
+ * when the cross-boot streak cap is already reached (stands down with
367
+ * an audit line instead). */
368
+ arm: () => void
369
+ /** A bridge client registered. Only a REAL bridge (named, non-cron)
370
+ * satisfies the watchdog — cron sessions (`<agent>-cron`) and
371
+ * anonymous clients (agentName null) are ignored HERE so a gateway
372
+ * handler refactor can't reorder the gating away (review finding 5). */
373
+ noteBridgeRegistered: (agentName: string | null | undefined) => void
374
+ /** A bridge client disconnected. Re-arms the grace window only for the
375
+ * real bridge (same internal gating), so a bridge that dies AFTER boot
376
+ * and never reconnects also escalates. Still capped by the
377
+ * once-per-boot fuse. */
378
+ noteBridgeDisconnected: (agentName: string | null | undefined) => void
379
+ /** Evaluate now (the timer body — exposed for tests). Returns the
380
+ * decision taken. */
381
+ check: () => BridgeDeadDecision
382
+ /** Cancel any pending timer (gateway shutdown). */
383
+ stop: () => void
384
+ /** Test/introspection: has this boot escalated already? */
385
+ hasEscalated: () => boolean
386
+ }
387
+
388
+ /** Is this client identity the REAL main-agent bridge (named, non-cron)? */
389
+ function isRealBridgeIdentity(agentName: string | null | undefined): boolean {
390
+ return agentName != null && agentName.length > 0 && !isCronIdentity(agentName)
391
+ }
392
+
393
+ export function createBridgeDeadWatchdog(opts: BridgeDeadWatchdogOpts): BridgeDeadWatchdog {
394
+ const setTimer =
395
+ opts.setTimer ??
396
+ ((fn: () => void, ms: number) => {
397
+ const t = setTimeout(fn, ms)
398
+ t.unref?.()
399
+ return t
400
+ })
401
+ const clearTimer = opts.clearTimer ?? ((h: unknown) => clearTimeout(h as NodeJS.Timeout))
402
+ const nowMs = opts.nowMs ?? (() => Date.now())
403
+ const writeMarker = opts.writeMarker ?? writeBridgeDeadEscalationMarker
404
+ const readCrashTail =
405
+ opts.readCrashTail ?? ((p: string, t: number) => readFreshCrashLogTail(p, { nowMs: t }))
406
+ const priorStreak = opts.priorStreak ?? 0
407
+ const maxConsecutive = opts.maxConsecutive ?? MAX_CONSECUTIVE_ESCALATIONS
408
+
409
+ let timer: unknown = null
410
+ let bridgeRegistered = false
411
+ let bridgeEverRegistered = false
412
+ let escalated = false
413
+
414
+ const cancel = (): void => {
415
+ if (timer != null) {
416
+ clearTimer(timer)
417
+ timer = null
418
+ }
419
+ }
420
+
421
+ /** The retry after a failed self-heal gets a doubled window: if one
422
+ * bounce didn't bring the bridge back inside graceMs, a second fire on
423
+ * the same clock mostly races container boot noise. */
424
+ const effectiveGraceMs = (): number =>
425
+ !bridgeEverRegistered && priorStreak >= 1 ? opts.graceMs * 2 : opts.graceMs
426
+
427
+ const armInternal = (): void => {
428
+ if (escalated) return // once-per-boot fuse — never re-arm after firing
429
+ if (!bridgeEverRegistered && priorStreak >= maxConsecutive) {
430
+ // Cross-boot damper: prior boots already bounced the container
431
+ // `priorStreak` times in a row for this same condition and the
432
+ // bridge still hasn't come back — a third bounce won't either.
433
+ // Stand down LOUDLY: the operator must investigate (corrupt plugin
434
+ // bundle / bad .mcp.json / broken IPC socket path).
435
+ opts.log(
436
+ `telegram gateway: [bridge-dead-watchdog] STANDING DOWN — ${priorStreak} consecutive ` +
437
+ `bridge-dead escalations already restarted this container without the bridge coming ` +
438
+ `back (cap=${maxConsecutive}). The bridge is failing deterministically; a further ` +
439
+ `restart will not fix it. Investigate the bridge (plugin bundle, .mcp.json, ` +
440
+ `STATE_DIR/bridge-crash.log). Manual recovery: fix the cause, then restart the ` +
441
+ `container. SWITCHROOM_BRIDGE_DEAD_ESCALATION=0 disables this watchdog entirely.`,
442
+ )
443
+ return
444
+ }
445
+ cancel()
446
+ timer = setTimer(() => {
447
+ timer = null
448
+ check()
449
+ }, effectiveGraceMs())
450
+ }
451
+
452
+ const check = (): BridgeDeadDecision => {
453
+ const decision = decideBridgeDeadEscalation({
454
+ bridgeRegistered,
455
+ bridgeEverRegistered,
456
+ sessionAlive: opts.isSessionAlive(),
457
+ shuttingDown: opts.isShuttingDown(),
458
+ alreadyEscalated: escalated,
459
+ priorStreak,
460
+ maxConsecutive,
461
+ })
462
+ if (decision.action === 'retry') {
463
+ // claude not up yet (slow container boot) — give it another full
464
+ // grace window rather than escalating a bounce that can't help.
465
+ opts.log(
466
+ `telegram gateway: [bridge-dead-watchdog] no bridge registered after ${effectiveGraceMs()}ms ` +
467
+ `but claude session not found — re-arming (no escalation)`,
468
+ )
469
+ armInternal()
470
+ return decision
471
+ }
472
+ if (decision.action === 'skip') {
473
+ if (decision.why !== 'bridge-registered') {
474
+ opts.log(
475
+ `telegram gateway: [bridge-dead-watchdog] bridge missing but skipping escalation (${decision.why})`,
476
+ )
477
+ }
478
+ return decision
479
+ }
480
+ // Escalate: bridge dead, session alive, not shutting down, first time
481
+ // this boot, streak under the cap. Set the fuse BEFORE any side effect
482
+ // so a throwing sink can't produce a second fire.
483
+ escalated = true
484
+ const t = nowMs()
485
+ const crashTail = (() => {
486
+ try {
487
+ return readCrashTail(opts.crashLogPath, t)
488
+ } catch {
489
+ return null
490
+ }
491
+ })()
492
+ // Streak accounting: a registration THIS boot breaks the prior streak
493
+ // (this escalation starts a new one at 1); otherwise it extends it.
494
+ const streakCount = bridgeEverRegistered ? 1 : priorStreak + 1
495
+ try {
496
+ const marker: BridgeDeadEscalationMarker = {
497
+ ts: t,
498
+ reason: BRIDGE_DEAD_RESTART_REASON,
499
+ count: streakCount,
500
+ }
501
+ if (crashTail != null) marker.crashTail = crashTail
502
+ writeMarker(opts.markerPath, marker)
503
+ } catch (err) {
504
+ opts.log(
505
+ `telegram gateway: [bridge-dead-watchdog] escalation marker write failed: ${(err as Error).message}`,
506
+ )
507
+ }
508
+ // The audit line: loud, structured, greppable — the operator's answer
509
+ // to "why did this container bounce itself".
510
+ opts.log(
511
+ `telegram gateway: [bridge-dead-watchdog] ESCALATING reason=${BRIDGE_DEAD_RESTART_REASON} ` +
512
+ `consecutive=${streakCount}/${maxConsecutive} — ` +
513
+ `no MCP bridge registered within ${effectiveGraceMs()}ms grace window while the claude session is alive ` +
514
+ `(Claude Code never respawns a dead MCP server; bouncing the container to restore the chat surface). ` +
515
+ (crashTail != null
516
+ ? `bridge-crash.log tail: ${crashTail}`
517
+ : `no fresh bridge-crash.log entries`),
518
+ )
519
+ const fired = opts.escalate(BRIDGE_DEAD_RESTART_REASON)
520
+ if (!fired) {
521
+ opts.log(
522
+ `telegram gateway: [bridge-dead-watchdog] escalate() reported failure — ` +
523
+ `restart not dispatched (fuse stays blown; no retry this boot)`,
524
+ )
525
+ }
526
+ return decision
527
+ }
528
+
529
+ return {
530
+ arm: armInternal,
531
+ noteBridgeRegistered: (agentName) => {
532
+ if (!isRealBridgeIdentity(agentName)) return
533
+ bridgeRegistered = true
534
+ bridgeEverRegistered = true
535
+ cancel()
536
+ },
537
+ noteBridgeDisconnected: (agentName) => {
538
+ if (!isRealBridgeIdentity(agentName)) return
539
+ bridgeRegistered = false
540
+ armInternal()
541
+ },
542
+ check,
543
+ stop: cancel,
544
+ hasEscalated: () => escalated,
545
+ }
546
+ }