switchroom 0.18.8 → 0.18.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/agent-scheduler/index.js +1 -0
- package/dist/auth-broker/index.js +198 -13
- package/dist/cli/notion-write-pretool.mjs +1 -0
- package/dist/cli/switchroom.js +28 -4
- package/dist/host-control/main.js +3 -2
- package/dist/vault/approvals/kernel-server.js +2 -1
- package/dist/vault/broker/server.js +2 -1
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +119 -37
- package/profiles/_shared/dev-protocol.md.hbs +42 -0
- package/skills/dev-protocol/SKILL.md +131 -0
- package/telegram-plugin/README.md +2 -1
- package/telegram-plugin/admin-commands/dispatch.test.ts +40 -2
- package/telegram-plugin/admin-commands/index.ts +6 -1
- package/telegram-plugin/bridge/bridge.ts +23 -1
- package/telegram-plugin/bridge/crash-breadcrumb.ts +42 -0
- package/telegram-plugin/chat-lock.ts +13 -0
- package/telegram-plugin/dist/bridge/bridge.js +24 -1
- package/telegram-plugin/dist/gateway/gateway.js +80236 -77465
- package/telegram-plugin/dist/server.js +29 -2
- package/telegram-plugin/fallback-card-collapse.ts +131 -0
- package/telegram-plugin/gateway/approval-card-stores.ts +99 -0
- package/telegram-plugin/gateway/bot-commands-ops-info.ts +194 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +546 -0
- package/telegram-plugin/gateway/callback-query-handlers.ts +2660 -0
- package/telegram-plugin/gateway/effort-command.ts +47 -3
- package/telegram-plugin/gateway/gateway.ts +2051 -3180
- package/telegram-plugin/gateway/inbound-delivery-machine-dispatch.ts +181 -23
- package/telegram-plugin/gateway/inbound-delivery-machine.ts +8 -0
- package/telegram-plugin/gateway/model-command.ts +94 -8
- package/telegram-plugin/gateway/outbound-send-path.ts +375 -0
- package/telegram-plugin/gateway/pending-session-command.ts +365 -0
- package/telegram-plugin/gateway/pending-state-stores.ts +106 -0
- package/telegram-plugin/gateway/permission-timeout.ts +25 -0
- package/telegram-plugin/gateway/register-bot-commands.ts +30 -0
- package/telegram-plugin/gateway/resume-inbound-builder.ts +23 -3
- package/telegram-plugin/gateway/session-model-file.ts +166 -23
- package/telegram-plugin/gateway/stop-command.ts +56 -0
- package/telegram-plugin/photo-precheck.ts +201 -0
- package/telegram-plugin/quota-watch.ts +141 -2
- package/telegram-plugin/registry/subagents-schema.ts +26 -3
- package/telegram-plugin/registry/subagents.test.ts +67 -0
- package/telegram-plugin/retry-api-call.ts +31 -0
- package/telegram-plugin/subagent-watcher.ts +392 -1
- package/telegram-plugin/tests/approval-card-stores.test.ts +124 -0
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +576 -0
- package/telegram-plugin/tests/buffer-gate-broadened.test.ts +11 -5
- package/telegram-plugin/tests/callback-query-handlers.test.ts +701 -0
- package/telegram-plugin/tests/chat-lock-unhandled-rejection.test.ts +101 -0
- package/telegram-plugin/tests/crash-breadcrumb.test.ts +57 -0
- package/telegram-plugin/tests/effort-command.test.ts +59 -2
- package/telegram-plugin/tests/emission-determinism-wiring.test.ts +11 -4
- package/telegram-plugin/tests/fallback-card-collapse.test.ts +104 -0
- package/telegram-plugin/tests/fixtures/cutover-killswitch-probe.ts +75 -0
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +5 -1
- package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +124 -0
- package/telegram-plugin/tests/gateway-secret-detect.test.ts +7 -1
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +19 -11
- package/telegram-plugin/tests/inbound-delivery-cutover-flip.test.ts +418 -0
- package/telegram-plugin/tests/inbound-delivery-dispatch-equivalence.test.ts +348 -0
- package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +141 -52
- package/telegram-plugin/tests/mental-model-propose-callback-gate.test.ts +8 -1
- package/telegram-plugin/tests/model-command.test.ts +46 -3
- package/telegram-plugin/tests/outbound-send-chunks.test.ts +304 -0
- package/telegram-plugin/tests/outbound-send-path.test.ts +222 -0
- package/telegram-plugin/tests/pending-card-durability-wiring.test.ts +34 -15
- package/telegram-plugin/tests/pending-session-command.test.ts +322 -0
- package/telegram-plugin/tests/pending-state-stores.test.ts +235 -0
- package/telegram-plugin/tests/permission-timeout.test.ts +26 -0
- package/telegram-plugin/tests/permission-verdict-resume-guard.test.ts +16 -0
- package/telegram-plugin/tests/photo-dimension-fallback.test.ts +129 -0
- package/telegram-plugin/tests/photo-precheck.test.ts +240 -0
- package/telegram-plugin/tests/photo-reroute-wiring.test.ts +85 -0
- package/telegram-plugin/tests/quota-watch.test.ts +225 -0
- package/telegram-plugin/tests/session-model-file.test.ts +101 -2
- package/telegram-plugin/tests/stop-command.test.ts +234 -0
- package/telegram-plugin/tests/subagent-watcher-env-thresholds.test.ts +27 -9
- package/telegram-plugin/tests/subagent-watcher-resurrection.test.ts +398 -0
- package/telegram-plugin/tests/subagent-watcher-stall-terminal.test.ts +172 -0
- package/telegram-plugin/tests/turn-flush-safety.test.ts +18 -4
- package/telegram-plugin/tests/vault-approval-posture.test.ts +15 -7
- package/telegram-plugin/tests/vault-grant-auto-resume.test.ts +8 -4
- package/telegram-plugin/tests/vault-grant-union.test.ts +8 -4
- package/telegram-plugin/tests/vault-grant-wizard.test.ts +8 -1
- package/telegram-plugin/tests/vault-grants-revoke.test.ts +8 -1
- package/telegram-plugin/tests/vault-key-regex-allows-slash.test.ts +8 -4
- package/telegram-plugin/tests/vault-request-access-tool.test.ts +8 -4
- package/telegram-plugin/tests/vault-request-access-unlock-resume.test.ts +8 -4
- package/telegram-plugin/tests/worker-activity-feed.test.ts +37 -0
- package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +18 -4
- package/telegram-plugin/welcome-text.ts +4 -3
- package/telegram-plugin/worker-activity-feed.ts +27 -0
|
@@ -0,0 +1,546 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* bridge-dead-watchdog.ts — gateway-side escalation when the MCP bridge
|
|
3
|
+
* never (re)registers (#3038, follow-up to #3033 / PR #3037).
|
|
4
|
+
*
|
|
5
|
+
* Failure mode: the gateway crashes and its supervisor relaunches it in
|
|
6
|
+
* ~1s, but the bridge MCP-server process inside the RUNNING claude
|
|
7
|
+
* session dies with it. Claude Code never respawns a dead MCP server, so
|
|
8
|
+
* the session stays alive but toolless and mute ("No such tool
|
|
9
|
+
* available: mcp__switchroom-telegram__reply") while the gateway sits at
|
|
10
|
+
* `bridge_dead` buffering inbounds — in the 2026-07-11 clerk incident,
|
|
11
|
+
* 7 minutes of a mute agent until a manual container restart.
|
|
12
|
+
*
|
|
13
|
+
* PR #3037 hardened the bridge against dying (uncaughtException handler
|
|
14
|
+
* + crash breadcrumbs to STATE_DIR/bridge-crash.log). This module is the
|
|
15
|
+
* structural belt-and-braces: after gateway boot (or after a main-bridge
|
|
16
|
+
* disconnect), if no main-agent bridge registers within a grace window
|
|
17
|
+
* while the claude session process is demonstrably alive, escalate by
|
|
18
|
+
* bouncing the whole container via the existing self-restart machinery
|
|
19
|
+
* (`triggerSelfRestart`) with the distinct reason
|
|
20
|
+
* `bridge-dead-resume`. The container restart respawns claude, which
|
|
21
|
+
* respawns the MCP bridge — the only recovery path that exists.
|
|
22
|
+
*
|
|
23
|
+
* Guard rails:
|
|
24
|
+
* - at most ONE escalation per gateway boot (in-memory fuse), AND at
|
|
25
|
+
* most MAX_CONSECUTIVE_ESCALATIONS across boots (the escalation
|
|
26
|
+
* marker carries a consecutive-fire count — see below): a bridge
|
|
27
|
+
* that dies deterministically at startup must not restart-loop the
|
|
28
|
+
* agent forever;
|
|
29
|
+
* - skipped while the gateway is mid-shutdown;
|
|
30
|
+
* - skipped when the claude session process is NOT alive (that is a
|
|
31
|
+
* different failure class — container boot still in progress, or
|
|
32
|
+
* claude itself down — where a bounce is either premature or
|
|
33
|
+
* futile); the timer re-arms so a slow-booting claude gets the full
|
|
34
|
+
* grace window again once it appears;
|
|
35
|
+
* - cron-session bridges (`<agent>-cron`) and anonymous IPC clients
|
|
36
|
+
* (recall.py one-shots, pre-handshake connects) neither satisfy nor
|
|
37
|
+
* re-arm the watchdog — the gating lives INSIDE noteBridge* so a
|
|
38
|
+
* gateway handler refactor can't silently reintroduce the
|
|
39
|
+
* bounce-after-cron-fire false positive;
|
|
40
|
+
* - loud, structured escalation log line including a fresh
|
|
41
|
+
* bridge-crash.log tail (PR #3037 breadcrumbs) so the operator sees
|
|
42
|
+
* WHY from the supervisor log alone.
|
|
43
|
+
*
|
|
44
|
+
* Honest resume: before firing the restart, the watchdog persists an
|
|
45
|
+
* escalation marker (`STATE_DIR/bridge-dead-escalation.json`). The next
|
|
46
|
+
* boot consumes it and (a) appends the real cause to the synthetic boot
|
|
47
|
+
* resume/report inbound when work was in flight, or (b) synthesizes a
|
|
48
|
+
* minimal idle notice (buildBridgeDeadIdleNoticeInbound) when nothing
|
|
49
|
+
* was — either way the agent tells the user "my chat bridge died and the
|
|
50
|
+
* framework restarted me" instead of implying a watchdog timeout or an
|
|
51
|
+
* operator restart, and the chat is never left wondering why the
|
|
52
|
+
* container bounced.
|
|
53
|
+
*
|
|
54
|
+
* Pure-ish module: all IO (fs, timers, process liveness, restart) is
|
|
55
|
+
* injected so the decision logic and the controller are unit-testable
|
|
56
|
+
* without booting a gateway.
|
|
57
|
+
*/
|
|
58
|
+
|
|
59
|
+
import { readFileSync, writeFileSync, renameSync, unlinkSync } from 'node:fs'
|
|
60
|
+
import { isCronIdentity } from './cron-session.js'
|
|
61
|
+
import type { InboundMessage } from './ipc-protocol.js'
|
|
62
|
+
|
|
63
|
+
/** The distinct triggerSelfRestart reason for this escalation. Classified
|
|
64
|
+
* as intent 'keep' by intentForRestartReason (recovery bounce — session
|
|
65
|
+
* model stickiness preserved), like every other switchroom-managed
|
|
66
|
+
* relaunch. */
|
|
67
|
+
export const BRIDGE_DEAD_RESTART_REASON = 'bridge-dead-resume'
|
|
68
|
+
|
|
69
|
+
/** Default grace window before a missing bridge is treated as dead.
|
|
70
|
+
* Override with SWITCHROOM_BRIDGE_DEAD_GRACE_MS. Chosen well above the
|
|
71
|
+
* normal boot register time (bridge registers within ~1-5s of claude
|
|
72
|
+
* start) but well below the 7-minute mute window of the incident. */
|
|
73
|
+
export const DEFAULT_BRIDGE_DEAD_GRACE_MS = 90_000
|
|
74
|
+
|
|
75
|
+
/** Escalation marker freshness window at next boot. A marker older than
|
|
76
|
+
* this is stale debris (the escalated restart happened long ago or never
|
|
77
|
+
* completed) — cleared without surfacing, and the consecutive-escalation
|
|
78
|
+
* streak resets with it. */
|
|
79
|
+
export const ESCALATION_MARKER_MAX_AGE_MS = 10 * 60_000
|
|
80
|
+
|
|
81
|
+
/** Cross-boot restart-loop damper: the once-per-boot fuse is in-memory
|
|
82
|
+
* and a container bounce RESETS it, so a bridge that dies
|
|
83
|
+
* deterministically at startup (corrupt plugin bundle, bad .mcp.json)
|
|
84
|
+
* would otherwise bounce the container every grace window forever,
|
|
85
|
+
* spamming boot cards. The escalation marker carries a consecutive-fire
|
|
86
|
+
* `count`; a boot that consumed a marker with count >= this cap stands
|
|
87
|
+
* the watchdog down (loud audit line, operator investigates) instead of
|
|
88
|
+
* firing again. A successful bridge registration at any boot breaks the
|
|
89
|
+
* streak. With the default cap of 2 the framework attempts the
|
|
90
|
+
* self-heal twice, then stops. */
|
|
91
|
+
export const MAX_CONSECUTIVE_ESCALATIONS = 2
|
|
92
|
+
|
|
93
|
+
/** Crash-breadcrumb entries older than this are not "fresh" — they refer
|
|
94
|
+
* to some earlier incident and would mislead the escalation log line. */
|
|
95
|
+
export const CRASH_LOG_FRESH_WINDOW_MS = 15 * 60_000
|
|
96
|
+
|
|
97
|
+
/** Max breadcrumb lines quoted into the escalation log / marker. */
|
|
98
|
+
const CRASH_LOG_TAIL_LINES = 3
|
|
99
|
+
|
|
100
|
+
// ─── Pure decision ───────────────────────────────────────────────────────────
|
|
101
|
+
|
|
102
|
+
export interface BridgeDeadSnapshot {
|
|
103
|
+
/** A real (named, non-cron) bridge is currently registered. */
|
|
104
|
+
bridgeRegistered: boolean
|
|
105
|
+
/** A real bridge registered at some point THIS boot (breaks the
|
|
106
|
+
* cross-boot escalation streak even if it later disconnected). */
|
|
107
|
+
bridgeEverRegistered: boolean
|
|
108
|
+
/** The claude session process exists in the container. */
|
|
109
|
+
sessionAlive: boolean
|
|
110
|
+
/** The gateway is mid-shutdown (SIGTERM/SIGINT handler ran). */
|
|
111
|
+
shuttingDown: boolean
|
|
112
|
+
/** This gateway boot already fired the one allowed escalation. */
|
|
113
|
+
alreadyEscalated: boolean
|
|
114
|
+
/** Consecutive escalations by PRIOR boots (from the consumed marker;
|
|
115
|
+
* 0 when no fresh marker was found). */
|
|
116
|
+
priorStreak: number
|
|
117
|
+
/** The cross-boot cap (MAX_CONSECUTIVE_ESCALATIONS unless a test
|
|
118
|
+
* overrides). */
|
|
119
|
+
maxConsecutive: number
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export type BridgeDeadDecision =
|
|
123
|
+
| { action: 'escalate' }
|
|
124
|
+
| {
|
|
125
|
+
action: 'skip'
|
|
126
|
+
why: 'bridge-registered' | 'shutting-down' | 'already-escalated' | 'streak-capped'
|
|
127
|
+
}
|
|
128
|
+
| { action: 'retry'; why: 'session-not-alive' }
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Decide what the watchdog should do when the grace window expires.
|
|
132
|
+
* Precedence, highest first: a live bridge always wins; a shutdown in
|
|
133
|
+
* progress must never be raced by a restart; the once-per-boot fuse and
|
|
134
|
+
* the cross-boot streak cap are checked before anything fires; a missing
|
|
135
|
+
* claude process re-arms rather than escalating (bouncing a container
|
|
136
|
+
* whose claude never came up would loop without fixing anything — and a
|
|
137
|
+
* container still booting deserves the full grace window once claude
|
|
138
|
+
* appears).
|
|
139
|
+
*/
|
|
140
|
+
export function decideBridgeDeadEscalation(s: BridgeDeadSnapshot): BridgeDeadDecision {
|
|
141
|
+
if (s.bridgeRegistered) return { action: 'skip', why: 'bridge-registered' }
|
|
142
|
+
if (s.shuttingDown) return { action: 'skip', why: 'shutting-down' }
|
|
143
|
+
if (s.alreadyEscalated) return { action: 'skip', why: 'already-escalated' }
|
|
144
|
+
// Cross-boot damper: prior boots already escalated `priorStreak` times
|
|
145
|
+
// in a row and the bridge STILL hasn't registered this boot — the bounce
|
|
146
|
+
// is not fixing it. Stand down. (A registration this boot breaks the
|
|
147
|
+
// streak: the current outage is then a NEW incident, not the same one.)
|
|
148
|
+
if (!s.bridgeEverRegistered && s.priorStreak >= s.maxConsecutive) {
|
|
149
|
+
return { action: 'skip', why: 'streak-capped' }
|
|
150
|
+
}
|
|
151
|
+
if (!s.sessionAlive) return { action: 'retry', why: 'session-not-alive' }
|
|
152
|
+
return { action: 'escalate' }
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// ─── Crash-breadcrumb tail (PR #3037 integration) ────────────────────────────
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Read the fresh tail of STATE_DIR/bridge-crash.log. Returns the last few
|
|
159
|
+
* breadcrumb lines whose leading ISO timestamp is within
|
|
160
|
+
* `freshWindowMs` of `nowMs`, joined with " || " (single log line —
|
|
161
|
+
* greppable, bounded). Returns null when the file is missing, unreadable,
|
|
162
|
+
* or has no fresh entries. Never throws.
|
|
163
|
+
*/
|
|
164
|
+
export function readFreshCrashLogTail(
|
|
165
|
+
path: string,
|
|
166
|
+
opts: {
|
|
167
|
+
nowMs?: number
|
|
168
|
+
freshWindowMs?: number
|
|
169
|
+
maxLines?: number
|
|
170
|
+
readFile?: (p: string) => string
|
|
171
|
+
} = {},
|
|
172
|
+
): string | null {
|
|
173
|
+
const nowMs = opts.nowMs ?? Date.now()
|
|
174
|
+
const freshWindowMs = opts.freshWindowMs ?? CRASH_LOG_FRESH_WINDOW_MS
|
|
175
|
+
const maxLines = opts.maxLines ?? CRASH_LOG_TAIL_LINES
|
|
176
|
+
const readFile = opts.readFile ?? ((p: string) => readFileSync(p, 'utf8'))
|
|
177
|
+
let raw: string
|
|
178
|
+
try {
|
|
179
|
+
raw = readFile(path)
|
|
180
|
+
} catch {
|
|
181
|
+
return null
|
|
182
|
+
}
|
|
183
|
+
const fresh = raw
|
|
184
|
+
.split('\n')
|
|
185
|
+
.filter((line) => {
|
|
186
|
+
const m = line.match(/^(\d{4}-\d{2}-\d{2}T[0-9:.]+Z)\s/)
|
|
187
|
+
if (!m) return false
|
|
188
|
+
const t = Date.parse(m[1])
|
|
189
|
+
return Number.isFinite(t) && nowMs - t >= 0 && nowMs - t <= freshWindowMs
|
|
190
|
+
})
|
|
191
|
+
.slice(-maxLines)
|
|
192
|
+
if (fresh.length === 0) return null
|
|
193
|
+
// Bound each quoted line — breadcrumbs are already capped at 4000 chars
|
|
194
|
+
// by the bridge, but one escalation log line quoting 3×4000 is noise.
|
|
195
|
+
return fresh.map((l) => l.slice(0, 500)).join(' || ')
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// ─── Escalation marker (honest boot-resume cause + cross-boot damper) ────────
|
|
199
|
+
|
|
200
|
+
export interface BridgeDeadEscalationMarker {
|
|
201
|
+
/** Wall-clock ms when the escalation fired. */
|
|
202
|
+
ts: number
|
|
203
|
+
/** Always BRIDGE_DEAD_RESTART_REASON — kept in the file for grep-ability. */
|
|
204
|
+
reason: string
|
|
205
|
+
/** Consecutive escalations INCLUDING this one (1 = first fire of a
|
|
206
|
+
* streak). The next boot reads this as its `priorStreak` and stands
|
|
207
|
+
* down at MAX_CONSECUTIVE_ESCALATIONS. Absent in pre-damper markers —
|
|
208
|
+
* treated as 1. */
|
|
209
|
+
count?: number
|
|
210
|
+
/** Fresh crash-breadcrumb tail at escalation time, if any. */
|
|
211
|
+
crashTail?: string
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
export function writeBridgeDeadEscalationMarker(
|
|
215
|
+
path: string,
|
|
216
|
+
marker: BridgeDeadEscalationMarker,
|
|
217
|
+
): void {
|
|
218
|
+
// Atomic tmp+rename so a partial write can't be read back as malformed
|
|
219
|
+
// JSON by the next boot (same discipline as clean-shutdown-marker.ts).
|
|
220
|
+
const tmp = `${path}.tmp-${process.pid}-${Date.now()}`
|
|
221
|
+
writeFileSync(tmp, JSON.stringify(marker), 'utf8')
|
|
222
|
+
renameSync(tmp, path)
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Read + consume the escalation marker at boot. Returns the marker only
|
|
227
|
+
* when it is fresh (< maxAgeMs); a stale or malformed marker is cleared
|
|
228
|
+
* and ignored. The file is ALWAYS removed — the cause note must surface
|
|
229
|
+
* on exactly the boot that follows the escalation, never a later one.
|
|
230
|
+
*
|
|
231
|
+
* Known race window (accepted, documented per review): the marker is
|
|
232
|
+
* written by gateway boot N and consumed by whichever gateway boots NEXT.
|
|
233
|
+
* Normally that is the post-container-restart gateway (the SIGTERM to
|
|
234
|
+
* PID 1 fires ~1.5s after the write and takes the whole container down).
|
|
235
|
+
* But if the escalating gateway PROCESS dies and its supervisor relaunches
|
|
236
|
+
* a new gateway inside the same container before the SIGTERM lands, that
|
|
237
|
+
* interim gateway consumes the marker instead — the cause note is then
|
|
238
|
+
* surfaced (or dropped with the interim process) one boot early, and the
|
|
239
|
+
* post-restart boot sees no marker. Consequences are bounded and safe:
|
|
240
|
+
* the honesty note may be lost for one incident (the loud supervisor-log
|
|
241
|
+
* audit line survives regardless), and the cross-boot streak damper
|
|
242
|
+
* under-counts by one (strictly MORE willing to self-heal, never a
|
|
243
|
+
* tighter loop, since the in-memory once-per-boot fuse still caps each
|
|
244
|
+
* process at one fire). Not worth a consume-side handshake.
|
|
245
|
+
*/
|
|
246
|
+
export function consumeBridgeDeadEscalationMarker(
|
|
247
|
+
path: string,
|
|
248
|
+
nowMs: number = Date.now(),
|
|
249
|
+
maxAgeMs: number = ESCALATION_MARKER_MAX_AGE_MS,
|
|
250
|
+
): BridgeDeadEscalationMarker | null {
|
|
251
|
+
let marker: BridgeDeadEscalationMarker | null = null
|
|
252
|
+
try {
|
|
253
|
+
const parsed = JSON.parse(readFileSync(path, 'utf8')) as Partial<BridgeDeadEscalationMarker>
|
|
254
|
+
if (
|
|
255
|
+
typeof parsed.ts === 'number' &&
|
|
256
|
+
Number.isFinite(parsed.ts) &&
|
|
257
|
+
typeof parsed.reason === 'string'
|
|
258
|
+
) {
|
|
259
|
+
const age = nowMs - parsed.ts
|
|
260
|
+
if (age >= 0 && age < maxAgeMs) {
|
|
261
|
+
marker = { ts: parsed.ts, reason: parsed.reason }
|
|
262
|
+
marker.count =
|
|
263
|
+
typeof parsed.count === 'number' && Number.isFinite(parsed.count) && parsed.count >= 1
|
|
264
|
+
? Math.floor(parsed.count)
|
|
265
|
+
: 1
|
|
266
|
+
if (typeof parsed.crashTail === 'string') marker.crashTail = parsed.crashTail
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
} catch {
|
|
270
|
+
/* missing or malformed — nothing to surface */
|
|
271
|
+
}
|
|
272
|
+
try {
|
|
273
|
+
unlinkSync(path)
|
|
274
|
+
} catch {
|
|
275
|
+
/* best effort */
|
|
276
|
+
}
|
|
277
|
+
return marker
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
// ─── Idle notice (honest cause when NO turn was in flight) ───────────────────
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Build the minimal synthetic inbound surfaced when the previous boot was
|
|
284
|
+
* a bridge-dead escalation but NO turn was interrupted (idle agent, dead
|
|
285
|
+
* bridge). Without this the cause reaches only stderr — the agent and the
|
|
286
|
+
* user never learn why the container bounced (review finding 2 on #3038).
|
|
287
|
+
* Mirrors the resume-builder shapes: meta.source is what Claude Code
|
|
288
|
+
* renders as `<channel source="…">`; meta.chat_id/message_id let the
|
|
289
|
+
* gateway's enqueue path build a currentTurn + ack the synthetic.
|
|
290
|
+
*/
|
|
291
|
+
export function buildBridgeDeadIdleNoticeInbound(args: {
|
|
292
|
+
/** Chat to surface the notice in (the agent's default/owner chat). */
|
|
293
|
+
chatId: string
|
|
294
|
+
/** The consumed escalation marker. */
|
|
295
|
+
marker: BridgeDeadEscalationMarker
|
|
296
|
+
/** Wall-clock ms; defaults to Date.now(). */
|
|
297
|
+
nowMs?: number
|
|
298
|
+
}): InboundMessage {
|
|
299
|
+
const ts = args.nowMs ?? Date.now()
|
|
300
|
+
return {
|
|
301
|
+
type: 'inbound',
|
|
302
|
+
chatId: args.chatId,
|
|
303
|
+
messageId: ts,
|
|
304
|
+
user: 'switchroom',
|
|
305
|
+
userId: 0,
|
|
306
|
+
ts,
|
|
307
|
+
text:
|
|
308
|
+
`You just restarted. The framework itself triggered this restart: your Telegram ` +
|
|
309
|
+
`MCP bridge process had died (chat tools were unavailable — you could not send ` +
|
|
310
|
+
`replies), so the container was bounced to restore the chat surface. No work was ` +
|
|
311
|
+
`in flight when it happened. Briefly let the user know the messaging bridge died ` +
|
|
312
|
+
`and the framework restarted you to fix it — one short message, no drama. This was ` +
|
|
313
|
+
`NOT an operator-initiated restart and NOT a hang-watchdog kill; report only that ` +
|
|
314
|
+
`honest cause.`,
|
|
315
|
+
meta: {
|
|
316
|
+
source: 'bridge_dead_restart',
|
|
317
|
+
chat_id: args.chatId,
|
|
318
|
+
message_id: String(ts),
|
|
319
|
+
restart_cause: args.marker.reason,
|
|
320
|
+
},
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// ─── Watchdog controller ─────────────────────────────────────────────────────
|
|
325
|
+
|
|
326
|
+
export interface BridgeDeadWatchdogOpts {
|
|
327
|
+
/** Grace window before a missing bridge is treated as dead. Doubled
|
|
328
|
+
* when the consumed marker shows a prior escalation streak (the retry
|
|
329
|
+
* after a failed self-heal deserves more patience). */
|
|
330
|
+
graceMs: number
|
|
331
|
+
/** Is the claude session process alive in this container? Callers
|
|
332
|
+
* should require a confident match (comm === 'claude'), NOT the
|
|
333
|
+
* heaviest-node fallback — an orphaned node process must not flip a
|
|
334
|
+
* retry into an escalation (review finding 4). */
|
|
335
|
+
isSessionAlive: () => boolean
|
|
336
|
+
/** Is the gateway mid-shutdown? */
|
|
337
|
+
isShuttingDown: () => boolean
|
|
338
|
+
/** Fire the container bounce (triggerSelfRestart wrapper). Returns
|
|
339
|
+
* whether the restart was actually dispatched. */
|
|
340
|
+
escalate: (reason: string) => boolean
|
|
341
|
+
/** STATE_DIR/bridge-crash.log — PR #3037 breadcrumbs. */
|
|
342
|
+
crashLogPath: string
|
|
343
|
+
/** STATE_DIR/bridge-dead-escalation.json — honest-resume marker. */
|
|
344
|
+
markerPath: string
|
|
345
|
+
/** Structured log sink (stderr). */
|
|
346
|
+
log: (line: string) => void
|
|
347
|
+
/** Consecutive escalations by prior boots (the consumed marker's
|
|
348
|
+
* `count`; 0 when no fresh marker). Drives the cross-boot damper. */
|
|
349
|
+
priorStreak?: number
|
|
350
|
+
/** Cross-boot cap override (tests). */
|
|
351
|
+
maxConsecutive?: number
|
|
352
|
+
/** Injectable timer pair for tests. Defaults to global setTimeout with
|
|
353
|
+
* unref (the watchdog must never keep the gateway process alive). */
|
|
354
|
+
setTimer?: (fn: () => void, ms: number) => unknown
|
|
355
|
+
clearTimer?: (handle: unknown) => void
|
|
356
|
+
/** Injectable clock (marker ts + crash-log freshness). */
|
|
357
|
+
nowMs?: () => number
|
|
358
|
+
/** Injectable marker writer (tests avoid real fs). */
|
|
359
|
+
writeMarker?: (path: string, marker: BridgeDeadEscalationMarker) => void
|
|
360
|
+
/** Injectable crash-log reader (tests avoid real fs). */
|
|
361
|
+
readCrashTail?: (path: string, nowMs: number) => string | null
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
export interface BridgeDeadWatchdog {
|
|
365
|
+
/** Arm the grace timer (gateway boot). Idempotent while armed. No-op
|
|
366
|
+
* when the cross-boot streak cap is already reached (stands down with
|
|
367
|
+
* an audit line instead). */
|
|
368
|
+
arm: () => void
|
|
369
|
+
/** A bridge client registered. Only a REAL bridge (named, non-cron)
|
|
370
|
+
* satisfies the watchdog — cron sessions (`<agent>-cron`) and
|
|
371
|
+
* anonymous clients (agentName null) are ignored HERE so a gateway
|
|
372
|
+
* handler refactor can't reorder the gating away (review finding 5). */
|
|
373
|
+
noteBridgeRegistered: (agentName: string | null | undefined) => void
|
|
374
|
+
/** A bridge client disconnected. Re-arms the grace window only for the
|
|
375
|
+
* real bridge (same internal gating), so a bridge that dies AFTER boot
|
|
376
|
+
* and never reconnects also escalates. Still capped by the
|
|
377
|
+
* once-per-boot fuse. */
|
|
378
|
+
noteBridgeDisconnected: (agentName: string | null | undefined) => void
|
|
379
|
+
/** Evaluate now (the timer body — exposed for tests). Returns the
|
|
380
|
+
* decision taken. */
|
|
381
|
+
check: () => BridgeDeadDecision
|
|
382
|
+
/** Cancel any pending timer (gateway shutdown). */
|
|
383
|
+
stop: () => void
|
|
384
|
+
/** Test/introspection: has this boot escalated already? */
|
|
385
|
+
hasEscalated: () => boolean
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/** Is this client identity the REAL main-agent bridge (named, non-cron)? */
|
|
389
|
+
function isRealBridgeIdentity(agentName: string | null | undefined): boolean {
|
|
390
|
+
return agentName != null && agentName.length > 0 && !isCronIdentity(agentName)
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
export function createBridgeDeadWatchdog(opts: BridgeDeadWatchdogOpts): BridgeDeadWatchdog {
|
|
394
|
+
const setTimer =
|
|
395
|
+
opts.setTimer ??
|
|
396
|
+
((fn: () => void, ms: number) => {
|
|
397
|
+
const t = setTimeout(fn, ms)
|
|
398
|
+
t.unref?.()
|
|
399
|
+
return t
|
|
400
|
+
})
|
|
401
|
+
const clearTimer = opts.clearTimer ?? ((h: unknown) => clearTimeout(h as NodeJS.Timeout))
|
|
402
|
+
const nowMs = opts.nowMs ?? (() => Date.now())
|
|
403
|
+
const writeMarker = opts.writeMarker ?? writeBridgeDeadEscalationMarker
|
|
404
|
+
const readCrashTail =
|
|
405
|
+
opts.readCrashTail ?? ((p: string, t: number) => readFreshCrashLogTail(p, { nowMs: t }))
|
|
406
|
+
const priorStreak = opts.priorStreak ?? 0
|
|
407
|
+
const maxConsecutive = opts.maxConsecutive ?? MAX_CONSECUTIVE_ESCALATIONS
|
|
408
|
+
|
|
409
|
+
let timer: unknown = null
|
|
410
|
+
let bridgeRegistered = false
|
|
411
|
+
let bridgeEverRegistered = false
|
|
412
|
+
let escalated = false
|
|
413
|
+
|
|
414
|
+
const cancel = (): void => {
|
|
415
|
+
if (timer != null) {
|
|
416
|
+
clearTimer(timer)
|
|
417
|
+
timer = null
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
/** The retry after a failed self-heal gets a doubled window: if one
|
|
422
|
+
* bounce didn't bring the bridge back inside graceMs, a second fire on
|
|
423
|
+
* the same clock mostly races container boot noise. */
|
|
424
|
+
const effectiveGraceMs = (): number =>
|
|
425
|
+
!bridgeEverRegistered && priorStreak >= 1 ? opts.graceMs * 2 : opts.graceMs
|
|
426
|
+
|
|
427
|
+
const armInternal = (): void => {
|
|
428
|
+
if (escalated) return // once-per-boot fuse — never re-arm after firing
|
|
429
|
+
if (!bridgeEverRegistered && priorStreak >= maxConsecutive) {
|
|
430
|
+
// Cross-boot damper: prior boots already bounced the container
|
|
431
|
+
// `priorStreak` times in a row for this same condition and the
|
|
432
|
+
// bridge still hasn't come back — a third bounce won't either.
|
|
433
|
+
// Stand down LOUDLY: the operator must investigate (corrupt plugin
|
|
434
|
+
// bundle / bad .mcp.json / broken IPC socket path).
|
|
435
|
+
opts.log(
|
|
436
|
+
`telegram gateway: [bridge-dead-watchdog] STANDING DOWN — ${priorStreak} consecutive ` +
|
|
437
|
+
`bridge-dead escalations already restarted this container without the bridge coming ` +
|
|
438
|
+
`back (cap=${maxConsecutive}). The bridge is failing deterministically; a further ` +
|
|
439
|
+
`restart will not fix it. Investigate the bridge (plugin bundle, .mcp.json, ` +
|
|
440
|
+
`STATE_DIR/bridge-crash.log). Manual recovery: fix the cause, then restart the ` +
|
|
441
|
+
`container. SWITCHROOM_BRIDGE_DEAD_ESCALATION=0 disables this watchdog entirely.`,
|
|
442
|
+
)
|
|
443
|
+
return
|
|
444
|
+
}
|
|
445
|
+
cancel()
|
|
446
|
+
timer = setTimer(() => {
|
|
447
|
+
timer = null
|
|
448
|
+
check()
|
|
449
|
+
}, effectiveGraceMs())
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
const check = (): BridgeDeadDecision => {
|
|
453
|
+
const decision = decideBridgeDeadEscalation({
|
|
454
|
+
bridgeRegistered,
|
|
455
|
+
bridgeEverRegistered,
|
|
456
|
+
sessionAlive: opts.isSessionAlive(),
|
|
457
|
+
shuttingDown: opts.isShuttingDown(),
|
|
458
|
+
alreadyEscalated: escalated,
|
|
459
|
+
priorStreak,
|
|
460
|
+
maxConsecutive,
|
|
461
|
+
})
|
|
462
|
+
if (decision.action === 'retry') {
|
|
463
|
+
// claude not up yet (slow container boot) — give it another full
|
|
464
|
+
// grace window rather than escalating a bounce that can't help.
|
|
465
|
+
opts.log(
|
|
466
|
+
`telegram gateway: [bridge-dead-watchdog] no bridge registered after ${effectiveGraceMs()}ms ` +
|
|
467
|
+
`but claude session not found — re-arming (no escalation)`,
|
|
468
|
+
)
|
|
469
|
+
armInternal()
|
|
470
|
+
return decision
|
|
471
|
+
}
|
|
472
|
+
if (decision.action === 'skip') {
|
|
473
|
+
if (decision.why !== 'bridge-registered') {
|
|
474
|
+
opts.log(
|
|
475
|
+
`telegram gateway: [bridge-dead-watchdog] bridge missing but skipping escalation (${decision.why})`,
|
|
476
|
+
)
|
|
477
|
+
}
|
|
478
|
+
return decision
|
|
479
|
+
}
|
|
480
|
+
// Escalate: bridge dead, session alive, not shutting down, first time
|
|
481
|
+
// this boot, streak under the cap. Set the fuse BEFORE any side effect
|
|
482
|
+
// so a throwing sink can't produce a second fire.
|
|
483
|
+
escalated = true
|
|
484
|
+
const t = nowMs()
|
|
485
|
+
const crashTail = (() => {
|
|
486
|
+
try {
|
|
487
|
+
return readCrashTail(opts.crashLogPath, t)
|
|
488
|
+
} catch {
|
|
489
|
+
return null
|
|
490
|
+
}
|
|
491
|
+
})()
|
|
492
|
+
// Streak accounting: a registration THIS boot breaks the prior streak
|
|
493
|
+
// (this escalation starts a new one at 1); otherwise it extends it.
|
|
494
|
+
const streakCount = bridgeEverRegistered ? 1 : priorStreak + 1
|
|
495
|
+
try {
|
|
496
|
+
const marker: BridgeDeadEscalationMarker = {
|
|
497
|
+
ts: t,
|
|
498
|
+
reason: BRIDGE_DEAD_RESTART_REASON,
|
|
499
|
+
count: streakCount,
|
|
500
|
+
}
|
|
501
|
+
if (crashTail != null) marker.crashTail = crashTail
|
|
502
|
+
writeMarker(opts.markerPath, marker)
|
|
503
|
+
} catch (err) {
|
|
504
|
+
opts.log(
|
|
505
|
+
`telegram gateway: [bridge-dead-watchdog] escalation marker write failed: ${(err as Error).message}`,
|
|
506
|
+
)
|
|
507
|
+
}
|
|
508
|
+
// The audit line: loud, structured, greppable — the operator's answer
|
|
509
|
+
// to "why did this container bounce itself".
|
|
510
|
+
opts.log(
|
|
511
|
+
`telegram gateway: [bridge-dead-watchdog] ESCALATING reason=${BRIDGE_DEAD_RESTART_REASON} ` +
|
|
512
|
+
`consecutive=${streakCount}/${maxConsecutive} — ` +
|
|
513
|
+
`no MCP bridge registered within ${effectiveGraceMs()}ms grace window while the claude session is alive ` +
|
|
514
|
+
`(Claude Code never respawns a dead MCP server; bouncing the container to restore the chat surface). ` +
|
|
515
|
+
(crashTail != null
|
|
516
|
+
? `bridge-crash.log tail: ${crashTail}`
|
|
517
|
+
: `no fresh bridge-crash.log entries`),
|
|
518
|
+
)
|
|
519
|
+
const fired = opts.escalate(BRIDGE_DEAD_RESTART_REASON)
|
|
520
|
+
if (!fired) {
|
|
521
|
+
opts.log(
|
|
522
|
+
`telegram gateway: [bridge-dead-watchdog] escalate() reported failure — ` +
|
|
523
|
+
`restart not dispatched (fuse stays blown; no retry this boot)`,
|
|
524
|
+
)
|
|
525
|
+
}
|
|
526
|
+
return decision
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
return {
|
|
530
|
+
arm: armInternal,
|
|
531
|
+
noteBridgeRegistered: (agentName) => {
|
|
532
|
+
if (!isRealBridgeIdentity(agentName)) return
|
|
533
|
+
bridgeRegistered = true
|
|
534
|
+
bridgeEverRegistered = true
|
|
535
|
+
cancel()
|
|
536
|
+
},
|
|
537
|
+
noteBridgeDisconnected: (agentName) => {
|
|
538
|
+
if (!isRealBridgeIdentity(agentName)) return
|
|
539
|
+
bridgeRegistered = false
|
|
540
|
+
armInternal()
|
|
541
|
+
},
|
|
542
|
+
check,
|
|
543
|
+
stop: cancel,
|
|
544
|
+
hasEscalated: () => escalated,
|
|
545
|
+
}
|
|
546
|
+
}
|