switchroom 0.18.9 → 0.18.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +1 -0
- package/dist/auth-broker/index.js +198 -13
- package/dist/cli/notion-write-pretool.mjs +1 -0
- package/dist/cli/switchroom.js +214 -83
- package/dist/cli/ui/apple-touch-icon.png +0 -0
- package/dist/cli/ui/favicon-32.png +0 -0
- package/dist/cli/ui/favicon.ico +0 -0
- package/dist/host-control/main.js +760 -257
- package/dist/vault/approvals/kernel-server.js +3 -1
- package/dist/vault/broker/server.js +3 -1
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +119 -37
- package/profiles/_shared/dev-protocol.md.hbs +42 -0
- package/skills/dev-protocol/SKILL.md +131 -0
- package/telegram-plugin/README.md +2 -1
- package/telegram-plugin/admin-commands/dispatch.test.ts +40 -2
- package/telegram-plugin/admin-commands/index.ts +6 -1
- package/telegram-plugin/bridge/bridge.ts +23 -1
- package/telegram-plugin/bridge/crash-breadcrumb.ts +42 -0
- package/telegram-plugin/chat-lock.ts +13 -0
- package/telegram-plugin/dist/bridge/bridge.js +24 -1
- package/telegram-plugin/dist/gateway/gateway.js +1832 -263
- package/telegram-plugin/dist/server.js +29 -2
- package/telegram-plugin/fallback-card-collapse.ts +131 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +546 -0
- package/telegram-plugin/gateway/effort-command.ts +47 -3
- package/telegram-plugin/gateway/gateway.ts +1435 -211
- package/telegram-plugin/gateway/model-command.ts +94 -8
- package/telegram-plugin/gateway/pending-session-command.ts +365 -0
- package/telegram-plugin/gateway/permission-timeout.ts +25 -0
- package/telegram-plugin/gateway/resume-inbound-builder.ts +23 -3
- package/telegram-plugin/gateway/session-model-file.ts +166 -23
- package/telegram-plugin/gateway/stop-command.ts +56 -0
- package/telegram-plugin/photo-precheck.ts +201 -0
- package/telegram-plugin/quota-watch.ts +141 -2
- package/telegram-plugin/registry/subagents-schema.ts +26 -3
- package/telegram-plugin/registry/subagents.test.ts +67 -0
- package/telegram-plugin/retry-api-call.ts +31 -0
- package/telegram-plugin/subagent-watcher.ts +392 -1
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +576 -0
- package/telegram-plugin/tests/buffer-gate-broadened.test.ts +11 -5
- package/telegram-plugin/tests/chat-lock-unhandled-rejection.test.ts +101 -0
- package/telegram-plugin/tests/crash-breadcrumb.test.ts +57 -0
- package/telegram-plugin/tests/effort-command.test.ts +59 -2
- package/telegram-plugin/tests/fallback-card-collapse.test.ts +104 -0
- package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +124 -0
- package/telegram-plugin/tests/gateway-secret-detect.test.ts +7 -1
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +19 -11
- package/telegram-plugin/tests/model-command.test.ts +46 -3
- package/telegram-plugin/tests/pending-session-command.test.ts +322 -0
- package/telegram-plugin/tests/permission-timeout.test.ts +26 -0
- package/telegram-plugin/tests/permission-verdict-resume-guard.test.ts +16 -0
- package/telegram-plugin/tests/photo-dimension-fallback.test.ts +129 -0
- package/telegram-plugin/tests/photo-precheck.test.ts +240 -0
- package/telegram-plugin/tests/photo-reroute-wiring.test.ts +85 -0
- package/telegram-plugin/tests/quota-watch.test.ts +225 -0
- package/telegram-plugin/tests/session-model-file.test.ts +101 -2
- package/telegram-plugin/tests/stop-command.test.ts +234 -0
- package/telegram-plugin/tests/subagent-watcher-env-thresholds.test.ts +27 -9
- package/telegram-plugin/tests/subagent-watcher-resurrection.test.ts +398 -0
- package/telegram-plugin/tests/subagent-watcher-stall-terminal.test.ts +172 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +37 -0
- package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +18 -4
- package/telegram-plugin/welcome-text.ts +4 -3
- package/telegram-plugin/worker-activity-feed.ts +27 -0
|
@@ -0,0 +1,576 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unit tests for telegram-plugin/gateway/bridge-dead-watchdog.ts (#3038).
|
|
3
|
+
*
|
|
4
|
+
* The contract under test (vitest — pure module, injected timers/fs/clock):
|
|
5
|
+
*
|
|
6
|
+
* - bridge re-registers inside the grace window → timer cancelled, no
|
|
7
|
+
* escalation;
|
|
8
|
+
* - no re-register + live claude session → exactly ONE escalation with
|
|
9
|
+
* reason 'bridge-dead-resume', marker written, crash-log tail quoted
|
|
10
|
+
* in the audit line;
|
|
11
|
+
* - a second miss in the same gateway boot (disconnect after the fuse
|
|
12
|
+
* blew) → no second escalation;
|
|
13
|
+
* - mid-shutdown → skip, no escalation;
|
|
14
|
+
* - claude session not alive → retry (re-arm), no escalation;
|
|
15
|
+
* - cross-boot damper: marker carries a consecutive count; at the cap
|
|
16
|
+
* the watchdog stands down (no restart loop for a deterministically
|
|
17
|
+
* failing bridge); a registration this boot breaks the streak; the
|
|
18
|
+
* retry boot gets a doubled grace window;
|
|
19
|
+
* - cron / anonymous identities neither satisfy nor re-arm the watchdog
|
|
20
|
+
* (gating INSIDE noteBridge*, refactor-proof);
|
|
21
|
+
* - readFreshCrashLogTail freshness filtering;
|
|
22
|
+
* - consumeBridgeDeadEscalationMarker fresh/stale/always-clears + count
|
|
23
|
+
* round-trip;
|
|
24
|
+
* - resume-inbound builders surface restartCause honestly (text note +
|
|
25
|
+
* meta.restart_cause) without changing meta.source;
|
|
26
|
+
* - idle notice inbound (no turn in flight) carries the honest cause.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { describe, it, expect } from 'vitest'
|
|
30
|
+
import { mkdtempSync, writeFileSync, existsSync, readFileSync } from 'node:fs'
|
|
31
|
+
import { tmpdir } from 'node:os'
|
|
32
|
+
import { join } from 'node:path'
|
|
33
|
+
import {
|
|
34
|
+
createBridgeDeadWatchdog,
|
|
35
|
+
decideBridgeDeadEscalation,
|
|
36
|
+
readFreshCrashLogTail,
|
|
37
|
+
writeBridgeDeadEscalationMarker,
|
|
38
|
+
consumeBridgeDeadEscalationMarker,
|
|
39
|
+
buildBridgeDeadIdleNoticeInbound,
|
|
40
|
+
BRIDGE_DEAD_RESTART_REASON,
|
|
41
|
+
MAX_CONSECUTIVE_ESCALATIONS,
|
|
42
|
+
type BridgeDeadWatchdogOpts,
|
|
43
|
+
type BridgeDeadEscalationMarker,
|
|
44
|
+
} from '../gateway/bridge-dead-watchdog.js'
|
|
45
|
+
import {
|
|
46
|
+
buildResumeInterruptedInbound,
|
|
47
|
+
buildResumeWatchdogReportInbound,
|
|
48
|
+
buildResumeDeferredReportInbound,
|
|
49
|
+
} from '../gateway/resume-inbound-builder.js'
|
|
50
|
+
import type { Turn } from '../registry/turns-schema.js'
|
|
51
|
+
|
|
52
|
+
const AGENT = 'clerk'
|
|
53
|
+
|
|
54
|
+
// ─── Manual timer harness ────────────────────────────────────────────────────
|
|
55
|
+
|
|
56
|
+
interface FakeTimer {
|
|
57
|
+
fn: () => void
|
|
58
|
+
ms: number
|
|
59
|
+
cleared: boolean
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function makeTimerHarness() {
|
|
63
|
+
const timers: FakeTimer[] = []
|
|
64
|
+
return {
|
|
65
|
+
timers,
|
|
66
|
+
setTimer: (fn: () => void, ms: number): unknown => {
|
|
67
|
+
const t: FakeTimer = { fn, ms, cleared: false }
|
|
68
|
+
timers.push(t)
|
|
69
|
+
return t
|
|
70
|
+
},
|
|
71
|
+
clearTimer: (h: unknown): void => {
|
|
72
|
+
;(h as FakeTimer).cleared = true
|
|
73
|
+
},
|
|
74
|
+
/** Fire the most recent un-cleared timer (the armed grace window). */
|
|
75
|
+
fireLatest(): void {
|
|
76
|
+
const live = timers.filter((t) => !t.cleared)
|
|
77
|
+
const t = live[live.length - 1]
|
|
78
|
+
if (!t) throw new Error('no live timer to fire')
|
|
79
|
+
t.cleared = true
|
|
80
|
+
t.fn()
|
|
81
|
+
},
|
|
82
|
+
liveCount(): number {
|
|
83
|
+
return timers.filter((t) => !t.cleared).length
|
|
84
|
+
},
|
|
85
|
+
latestMs(): number {
|
|
86
|
+
const live = timers.filter((t) => !t.cleared)
|
|
87
|
+
const t = live[live.length - 1]
|
|
88
|
+
if (!t) throw new Error('no live timer')
|
|
89
|
+
return t.ms
|
|
90
|
+
},
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function makeWatchdog(overrides: Partial<BridgeDeadWatchdogOpts> = {}) {
|
|
95
|
+
const harness = makeTimerHarness()
|
|
96
|
+
const escalations: string[] = []
|
|
97
|
+
const logs: string[] = []
|
|
98
|
+
const markers: Array<{ path: string; marker: BridgeDeadEscalationMarker }> = []
|
|
99
|
+
const state = { sessionAlive: true, shuttingDown: false, escalateResult: true }
|
|
100
|
+
const wd = createBridgeDeadWatchdog({
|
|
101
|
+
graceMs: 90_000,
|
|
102
|
+
isSessionAlive: () => state.sessionAlive,
|
|
103
|
+
isShuttingDown: () => state.shuttingDown,
|
|
104
|
+
escalate: (reason) => {
|
|
105
|
+
escalations.push(reason)
|
|
106
|
+
return state.escalateResult
|
|
107
|
+
},
|
|
108
|
+
crashLogPath: '/nonexistent/bridge-crash.log',
|
|
109
|
+
markerPath: '/nonexistent/bridge-dead-escalation.json',
|
|
110
|
+
log: (l) => logs.push(l),
|
|
111
|
+
setTimer: harness.setTimer,
|
|
112
|
+
clearTimer: harness.clearTimer,
|
|
113
|
+
nowMs: () => 1_000_000,
|
|
114
|
+
writeMarker: (path, marker) => markers.push({ path, marker }),
|
|
115
|
+
readCrashTail: () => null,
|
|
116
|
+
...overrides,
|
|
117
|
+
})
|
|
118
|
+
return { wd, harness, escalations, logs, markers, state }
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// ─── Pure decision table ─────────────────────────────────────────────────────
|
|
122
|
+
|
|
123
|
+
describe('decideBridgeDeadEscalation', () => {
|
|
124
|
+
const base = {
|
|
125
|
+
bridgeRegistered: false,
|
|
126
|
+
bridgeEverRegistered: false,
|
|
127
|
+
sessionAlive: true,
|
|
128
|
+
shuttingDown: false,
|
|
129
|
+
alreadyEscalated: false,
|
|
130
|
+
priorStreak: 0,
|
|
131
|
+
maxConsecutive: MAX_CONSECUTIVE_ESCALATIONS,
|
|
132
|
+
}
|
|
133
|
+
it('escalates when bridge missing, session alive, not shutting down, first time', () => {
|
|
134
|
+
expect(decideBridgeDeadEscalation(base)).toEqual({ action: 'escalate' })
|
|
135
|
+
})
|
|
136
|
+
it('a registered bridge always wins', () => {
|
|
137
|
+
expect(decideBridgeDeadEscalation({ ...base, bridgeRegistered: true })).toEqual({
|
|
138
|
+
action: 'skip',
|
|
139
|
+
why: 'bridge-registered',
|
|
140
|
+
})
|
|
141
|
+
})
|
|
142
|
+
it('mid-shutdown skips even with a dead bridge', () => {
|
|
143
|
+
expect(decideBridgeDeadEscalation({ ...base, shuttingDown: true })).toEqual({
|
|
144
|
+
action: 'skip',
|
|
145
|
+
why: 'shutting-down',
|
|
146
|
+
})
|
|
147
|
+
})
|
|
148
|
+
it('once-per-boot fuse skips a second fire', () => {
|
|
149
|
+
expect(decideBridgeDeadEscalation({ ...base, alreadyEscalated: true })).toEqual({
|
|
150
|
+
action: 'skip',
|
|
151
|
+
why: 'already-escalated',
|
|
152
|
+
})
|
|
153
|
+
})
|
|
154
|
+
it('claude not alive → retry, never escalate', () => {
|
|
155
|
+
expect(decideBridgeDeadEscalation({ ...base, sessionAlive: false })).toEqual({
|
|
156
|
+
action: 'retry',
|
|
157
|
+
why: 'session-not-alive',
|
|
158
|
+
})
|
|
159
|
+
})
|
|
160
|
+
it('cross-boot streak at the cap → streak-capped skip', () => {
|
|
161
|
+
expect(
|
|
162
|
+
decideBridgeDeadEscalation({ ...base, priorStreak: MAX_CONSECUTIVE_ESCALATIONS }),
|
|
163
|
+
).toEqual({ action: 'skip', why: 'streak-capped' })
|
|
164
|
+
})
|
|
165
|
+
it('streak below the cap still escalates', () => {
|
|
166
|
+
expect(decideBridgeDeadEscalation({ ...base, priorStreak: 1 })).toEqual({
|
|
167
|
+
action: 'escalate',
|
|
168
|
+
})
|
|
169
|
+
})
|
|
170
|
+
it('a registration THIS boot breaks the streak (new incident)', () => {
|
|
171
|
+
expect(
|
|
172
|
+
decideBridgeDeadEscalation({
|
|
173
|
+
...base,
|
|
174
|
+
bridgeEverRegistered: true,
|
|
175
|
+
priorStreak: MAX_CONSECUTIVE_ESCALATIONS + 3,
|
|
176
|
+
}),
|
|
177
|
+
).toEqual({ action: 'escalate' })
|
|
178
|
+
})
|
|
179
|
+
})
|
|
180
|
+
|
|
181
|
+
// ─── Watchdog controller ─────────────────────────────────────────────────────
|
|
182
|
+
|
|
183
|
+
describe('createBridgeDeadWatchdog', () => {
|
|
184
|
+
it('bridge re-registers in time → timer cancelled, no escalation', () => {
|
|
185
|
+
const { wd, harness, escalations } = makeWatchdog()
|
|
186
|
+
wd.arm()
|
|
187
|
+
expect(harness.liveCount()).toBe(1)
|
|
188
|
+
wd.noteBridgeRegistered(AGENT)
|
|
189
|
+
expect(harness.liveCount()).toBe(0) // grace timer cancelled
|
|
190
|
+
expect(escalations).toEqual([])
|
|
191
|
+
expect(wd.hasEscalated()).toBe(false)
|
|
192
|
+
})
|
|
193
|
+
|
|
194
|
+
it('registered bridge at expiry → skip (defensive: timer fired anyway)', () => {
|
|
195
|
+
const { wd, escalations } = makeWatchdog()
|
|
196
|
+
wd.arm()
|
|
197
|
+
wd.noteBridgeRegistered(AGENT)
|
|
198
|
+
// Even if check() runs somehow, the registered flag wins.
|
|
199
|
+
expect(wd.check()).toEqual({ action: 'skip', why: 'bridge-registered' })
|
|
200
|
+
expect(escalations).toEqual([])
|
|
201
|
+
})
|
|
202
|
+
|
|
203
|
+
it('no re-register + live session → exactly one escalation with the distinct reason', () => {
|
|
204
|
+
const { wd, harness, escalations, logs, markers } = makeWatchdog()
|
|
205
|
+
wd.arm()
|
|
206
|
+
harness.fireLatest()
|
|
207
|
+
expect(escalations).toEqual([BRIDGE_DEAD_RESTART_REASON])
|
|
208
|
+
expect(wd.hasEscalated()).toBe(true)
|
|
209
|
+
// Marker persisted for the honest resume + streak damper at next boot.
|
|
210
|
+
expect(markers).toHaveLength(1)
|
|
211
|
+
expect(markers[0].marker.reason).toBe(BRIDGE_DEAD_RESTART_REASON)
|
|
212
|
+
expect(markers[0].marker.ts).toBe(1_000_000)
|
|
213
|
+
expect(markers[0].marker.count).toBe(1) // first fire of a streak
|
|
214
|
+
// Audit line is loud and names the reason.
|
|
215
|
+
const audit = logs.find((l) => l.includes('ESCALATING'))
|
|
216
|
+
expect(audit).toBeTruthy()
|
|
217
|
+
expect(audit).toContain(`reason=${BRIDGE_DEAD_RESTART_REASON}`)
|
|
218
|
+
expect(audit).toContain('no fresh bridge-crash.log entries')
|
|
219
|
+
})
|
|
220
|
+
|
|
221
|
+
it('includes the fresh crash-log tail in the audit line and the marker', () => {
|
|
222
|
+
const { wd, harness, logs, markers } = makeWatchdog({
|
|
223
|
+
readCrashTail: () => '2026-07-11T01:41:11.000Z uncaughtException pid=42 boom',
|
|
224
|
+
})
|
|
225
|
+
wd.arm()
|
|
226
|
+
harness.fireLatest()
|
|
227
|
+
const audit = logs.find((l) => l.includes('ESCALATING'))
|
|
228
|
+
expect(audit).toContain('bridge-crash.log tail: 2026-07-11T01:41:11.000Z uncaughtException pid=42 boom')
|
|
229
|
+
expect(markers[0].marker.crashTail).toContain('uncaughtException pid=42 boom')
|
|
230
|
+
})
|
|
231
|
+
|
|
232
|
+
it('second miss in the same boot → no second escalation (fuse)', () => {
|
|
233
|
+
const { wd, harness, escalations } = makeWatchdog()
|
|
234
|
+
wd.arm()
|
|
235
|
+
harness.fireLatest()
|
|
236
|
+
expect(escalations).toHaveLength(1)
|
|
237
|
+
// The bridge briefly reconnects then dies again — disconnect re-arm is
|
|
238
|
+
// a no-op after the fuse blew, and a direct check() skips.
|
|
239
|
+
wd.noteBridgeDisconnected(AGENT)
|
|
240
|
+
expect(harness.liveCount()).toBe(0) // fuse prevents re-arm
|
|
241
|
+
expect(wd.check()).toEqual({ action: 'skip', why: 'already-escalated' })
|
|
242
|
+
expect(escalations).toHaveLength(1)
|
|
243
|
+
})
|
|
244
|
+
|
|
245
|
+
it('mid-shutdown → skip, no escalation, no marker', () => {
|
|
246
|
+
const { wd, harness, escalations, markers, state } = makeWatchdog()
|
|
247
|
+
wd.arm()
|
|
248
|
+
state.shuttingDown = true
|
|
249
|
+
harness.fireLatest()
|
|
250
|
+
expect(escalations).toEqual([])
|
|
251
|
+
expect(markers).toEqual([])
|
|
252
|
+
expect(wd.hasEscalated()).toBe(false)
|
|
253
|
+
})
|
|
254
|
+
|
|
255
|
+
it('claude session not alive → retry: re-arms, never escalates', () => {
|
|
256
|
+
const { wd, harness, escalations, state } = makeWatchdog()
|
|
257
|
+
state.sessionAlive = false
|
|
258
|
+
wd.arm()
|
|
259
|
+
harness.fireLatest()
|
|
260
|
+
expect(escalations).toEqual([])
|
|
261
|
+
expect(harness.liveCount()).toBe(1) // re-armed for another grace window
|
|
262
|
+
// claude comes up but the bridge never registers → NOW it escalates.
|
|
263
|
+
state.sessionAlive = true
|
|
264
|
+
harness.fireLatest()
|
|
265
|
+
expect(escalations).toEqual([BRIDGE_DEAD_RESTART_REASON])
|
|
266
|
+
})
|
|
267
|
+
|
|
268
|
+
it('bridge disconnect mid-life re-arms; a prompt reconnect stands it down', () => {
|
|
269
|
+
const { wd, harness, escalations } = makeWatchdog()
|
|
270
|
+
wd.arm()
|
|
271
|
+
wd.noteBridgeRegistered(AGENT)
|
|
272
|
+
expect(harness.liveCount()).toBe(0)
|
|
273
|
+
wd.noteBridgeDisconnected(AGENT)
|
|
274
|
+
expect(harness.liveCount()).toBe(1)
|
|
275
|
+
wd.noteBridgeRegistered(AGENT)
|
|
276
|
+
expect(harness.liveCount()).toBe(0)
|
|
277
|
+
expect(escalations).toEqual([])
|
|
278
|
+
})
|
|
279
|
+
|
|
280
|
+
it('bridge disconnect mid-life with no reconnect → one escalation', () => {
|
|
281
|
+
const { wd, harness, escalations } = makeWatchdog()
|
|
282
|
+
wd.arm()
|
|
283
|
+
wd.noteBridgeRegistered(AGENT)
|
|
284
|
+
wd.noteBridgeDisconnected(AGENT)
|
|
285
|
+
harness.fireLatest()
|
|
286
|
+
expect(escalations).toEqual([BRIDGE_DEAD_RESTART_REASON])
|
|
287
|
+
})
|
|
288
|
+
|
|
289
|
+
it('a throwing marker writer does not block the escalation itself', () => {
|
|
290
|
+
const { wd, harness, escalations, logs } = makeWatchdog({
|
|
291
|
+
writeMarker: () => {
|
|
292
|
+
throw new Error('disk full')
|
|
293
|
+
},
|
|
294
|
+
})
|
|
295
|
+
wd.arm()
|
|
296
|
+
harness.fireLatest()
|
|
297
|
+
expect(escalations).toEqual([BRIDGE_DEAD_RESTART_REASON])
|
|
298
|
+
expect(logs.some((l) => l.includes('escalation marker write failed'))).toBe(true)
|
|
299
|
+
})
|
|
300
|
+
|
|
301
|
+
it('stop() cancels the pending timer', () => {
|
|
302
|
+
const { wd, harness, escalations } = makeWatchdog()
|
|
303
|
+
wd.arm()
|
|
304
|
+
wd.stop()
|
|
305
|
+
expect(harness.liveCount()).toBe(0)
|
|
306
|
+
expect(escalations).toEqual([])
|
|
307
|
+
})
|
|
308
|
+
})
|
|
309
|
+
|
|
310
|
+
// ─── Cross-boot restart-loop damper (#3038 review finding 1) ─────────────────
|
|
311
|
+
|
|
312
|
+
describe('cross-boot escalation streak damper', () => {
|
|
313
|
+
it('priorStreak below the cap: escalates with an incremented count and doubled grace', () => {
|
|
314
|
+
const { wd, harness, escalations, markers } = makeWatchdog({ priorStreak: 1 })
|
|
315
|
+
wd.arm()
|
|
316
|
+
expect(harness.latestMs()).toBe(180_000) // doubled window on the retry boot
|
|
317
|
+
harness.fireLatest()
|
|
318
|
+
expect(escalations).toEqual([BRIDGE_DEAD_RESTART_REASON])
|
|
319
|
+
expect(markers[0].marker.count).toBe(2) // streak extended
|
|
320
|
+
})
|
|
321
|
+
|
|
322
|
+
it('priorStreak at the cap: arm() stands down loudly, no timer, no escalation', () => {
|
|
323
|
+
const { wd, harness, escalations, logs } = makeWatchdog({
|
|
324
|
+
priorStreak: MAX_CONSECUTIVE_ESCALATIONS,
|
|
325
|
+
})
|
|
326
|
+
wd.arm()
|
|
327
|
+
expect(harness.liveCount()).toBe(0) // never armed — no restart loop
|
|
328
|
+
expect(escalations).toEqual([])
|
|
329
|
+
const standDown = logs.find((l) => l.includes('STANDING DOWN'))
|
|
330
|
+
expect(standDown).toBeTruthy()
|
|
331
|
+
expect(standDown).toContain('SWITCHROOM_BRIDGE_DEAD_ESCALATION=0') // kill-switch reminder
|
|
332
|
+
// Even a direct check() cannot fire past the cap.
|
|
333
|
+
expect(wd.check()).toEqual({ action: 'skip', why: 'streak-capped' })
|
|
334
|
+
expect(escalations).toEqual([])
|
|
335
|
+
})
|
|
336
|
+
|
|
337
|
+
it('a successful registration this boot breaks the streak: escalation restarts at count=1', () => {
|
|
338
|
+
const { wd, harness, escalations, markers } = makeWatchdog({
|
|
339
|
+
priorStreak: MAX_CONSECUTIVE_ESCALATIONS,
|
|
340
|
+
})
|
|
341
|
+
// Bridge DID come back this boot — the old streak is over.
|
|
342
|
+
wd.noteBridgeRegistered(AGENT)
|
|
343
|
+
// …then it dies mid-life: this is a NEW incident and may escalate.
|
|
344
|
+
wd.noteBridgeDisconnected(AGENT)
|
|
345
|
+
expect(harness.liveCount()).toBe(1)
|
|
346
|
+
expect(harness.latestMs()).toBe(90_000) // normal grace — streak broken
|
|
347
|
+
harness.fireLatest()
|
|
348
|
+
expect(escalations).toEqual([BRIDGE_DEAD_RESTART_REASON])
|
|
349
|
+
expect(markers[0].marker.count).toBe(1)
|
|
350
|
+
})
|
|
351
|
+
})
|
|
352
|
+
|
|
353
|
+
// ─── Identity gating inside the watchdog (#3038 review finding 5) ────────────
|
|
354
|
+
|
|
355
|
+
describe('cron / anonymous identity gating', () => {
|
|
356
|
+
it('a cron-session bridge registering does NOT satisfy the watchdog', () => {
|
|
357
|
+
const { wd, harness, escalations } = makeWatchdog()
|
|
358
|
+
wd.arm()
|
|
359
|
+
wd.noteBridgeRegistered(`${AGENT}-cron`)
|
|
360
|
+
expect(harness.liveCount()).toBe(1) // still armed — cron is not the real bridge
|
|
361
|
+
harness.fireLatest()
|
|
362
|
+
expect(escalations).toEqual([BRIDGE_DEAD_RESTART_REASON]) // real bridge still dead
|
|
363
|
+
})
|
|
364
|
+
|
|
365
|
+
it('a cron-session bridge disconnecting does NOT re-arm (no bounce-after-cron-fire)', () => {
|
|
366
|
+
const { wd, harness, escalations } = makeWatchdog()
|
|
367
|
+
wd.arm()
|
|
368
|
+
wd.noteBridgeRegistered(AGENT) // real bridge healthy
|
|
369
|
+
expect(harness.liveCount()).toBe(0)
|
|
370
|
+
wd.noteBridgeDisconnected(`${AGENT}-cron`) // cron teardown — irrelevant
|
|
371
|
+
expect(harness.liveCount()).toBe(0)
|
|
372
|
+
expect(escalations).toEqual([])
|
|
373
|
+
})
|
|
374
|
+
|
|
375
|
+
it('anonymous clients (agentName null/empty) are ignored on both paths', () => {
|
|
376
|
+
const { wd, harness } = makeWatchdog()
|
|
377
|
+
wd.arm()
|
|
378
|
+
wd.noteBridgeRegistered(null)
|
|
379
|
+
wd.noteBridgeRegistered(undefined)
|
|
380
|
+
wd.noteBridgeRegistered('')
|
|
381
|
+
expect(harness.liveCount()).toBe(1) // none satisfied it
|
|
382
|
+
wd.noteBridgeRegistered(AGENT)
|
|
383
|
+
expect(harness.liveCount()).toBe(0)
|
|
384
|
+
wd.noteBridgeDisconnected(null) // recall.py one-shot closing
|
|
385
|
+
expect(harness.liveCount()).toBe(0) // did not re-arm
|
|
386
|
+
})
|
|
387
|
+
})
|
|
388
|
+
|
|
389
|
+
// ─── Crash-log tail ──────────────────────────────────────────────────────────
|
|
390
|
+
|
|
391
|
+
describe('readFreshCrashLogTail', () => {
|
|
392
|
+
const now = Date.parse('2026-07-11T02:00:00.000Z')
|
|
393
|
+
it('returns only fresh entries, last N, joined', () => {
|
|
394
|
+
const raw = [
|
|
395
|
+
'2026-07-11T00:00:00.000Z uncaughtException pid=1 old entry',
|
|
396
|
+
'2026-07-11T01:55:00.000Z uncaughtException pid=2 fresh one',
|
|
397
|
+
'2026-07-11T01:59:00.000Z unhandledRejection pid=2 fresh two',
|
|
398
|
+
'garbage line without timestamp',
|
|
399
|
+
'',
|
|
400
|
+
].join('\n')
|
|
401
|
+
const tail = readFreshCrashLogTail('/x', { nowMs: now, readFile: () => raw })
|
|
402
|
+
expect(tail).toBe(
|
|
403
|
+
'2026-07-11T01:55:00.000Z uncaughtException pid=2 fresh one || ' +
|
|
404
|
+
'2026-07-11T01:59:00.000Z unhandledRejection pid=2 fresh two',
|
|
405
|
+
)
|
|
406
|
+
})
|
|
407
|
+
it('null when file missing', () => {
|
|
408
|
+
expect(
|
|
409
|
+
readFreshCrashLogTail('/x', {
|
|
410
|
+
nowMs: now,
|
|
411
|
+
readFile: () => {
|
|
412
|
+
throw new Error('ENOENT')
|
|
413
|
+
},
|
|
414
|
+
}),
|
|
415
|
+
).toBeNull()
|
|
416
|
+
})
|
|
417
|
+
it('null when all entries are stale', () => {
|
|
418
|
+
const raw = '2026-07-10T00:00:00.000Z uncaughtException pid=1 ancient\n'
|
|
419
|
+
expect(readFreshCrashLogTail('/x', { nowMs: now, readFile: () => raw })).toBeNull()
|
|
420
|
+
})
|
|
421
|
+
it('caps to maxLines (most recent)', () => {
|
|
422
|
+
const raw = [1, 2, 3, 4, 5]
|
|
423
|
+
.map((i) => `2026-07-11T01:59:0${i}.000Z uncaughtException pid=${i} e${i}`)
|
|
424
|
+
.join('\n')
|
|
425
|
+
const tail = readFreshCrashLogTail('/x', {
|
|
426
|
+
nowMs: now,
|
|
427
|
+
readFile: () => raw,
|
|
428
|
+
maxLines: 2,
|
|
429
|
+
})
|
|
430
|
+
expect(tail).toContain('e4')
|
|
431
|
+
expect(tail).toContain('e5')
|
|
432
|
+
expect(tail).not.toContain('e3')
|
|
433
|
+
})
|
|
434
|
+
})
|
|
435
|
+
|
|
436
|
+
// ─── Escalation marker round-trip ────────────────────────────────────────────
|
|
437
|
+
|
|
438
|
+
describe('bridge-dead escalation marker', () => {
|
|
439
|
+
it('fresh marker round-trips (incl. count) and is consumed (file removed)', () => {
|
|
440
|
+
const dir = mkdtempSync(join(tmpdir(), 'bridge-dead-marker-'))
|
|
441
|
+
const p = join(dir, 'bridge-dead-escalation.json')
|
|
442
|
+
writeBridgeDeadEscalationMarker(p, {
|
|
443
|
+
ts: 5_000,
|
|
444
|
+
reason: BRIDGE_DEAD_RESTART_REASON,
|
|
445
|
+
count: 2,
|
|
446
|
+
crashTail: 'tail here',
|
|
447
|
+
})
|
|
448
|
+
const m = consumeBridgeDeadEscalationMarker(p, 6_000)
|
|
449
|
+
expect(m).toEqual({
|
|
450
|
+
ts: 5_000,
|
|
451
|
+
reason: BRIDGE_DEAD_RESTART_REASON,
|
|
452
|
+
count: 2,
|
|
453
|
+
crashTail: 'tail here',
|
|
454
|
+
})
|
|
455
|
+
expect(existsSync(p)).toBe(false)
|
|
456
|
+
})
|
|
457
|
+
it('legacy marker without count is read as count=1', () => {
|
|
458
|
+
const dir = mkdtempSync(join(tmpdir(), 'bridge-dead-marker-'))
|
|
459
|
+
const p = join(dir, 'bridge-dead-escalation.json')
|
|
460
|
+
writeFileSync(p, JSON.stringify({ ts: 5_000, reason: BRIDGE_DEAD_RESTART_REASON }), 'utf8')
|
|
461
|
+
const m = consumeBridgeDeadEscalationMarker(p, 6_000)
|
|
462
|
+
expect(m?.count).toBe(1)
|
|
463
|
+
})
|
|
464
|
+
it('stale marker is cleared and NOT surfaced', () => {
|
|
465
|
+
const dir = mkdtempSync(join(tmpdir(), 'bridge-dead-marker-'))
|
|
466
|
+
const p = join(dir, 'bridge-dead-escalation.json')
|
|
467
|
+
writeBridgeDeadEscalationMarker(p, { ts: 0, reason: BRIDGE_DEAD_RESTART_REASON })
|
|
468
|
+
const m = consumeBridgeDeadEscalationMarker(p, 60 * 60_000)
|
|
469
|
+
expect(m).toBeNull()
|
|
470
|
+
expect(existsSync(p)).toBe(false)
|
|
471
|
+
})
|
|
472
|
+
it('malformed marker is cleared without throwing', () => {
|
|
473
|
+
const dir = mkdtempSync(join(tmpdir(), 'bridge-dead-marker-'))
|
|
474
|
+
const p = join(dir, 'bridge-dead-escalation.json')
|
|
475
|
+
writeFileSync(p, 'not json', 'utf8')
|
|
476
|
+
expect(consumeBridgeDeadEscalationMarker(p, 1)).toBeNull()
|
|
477
|
+
expect(existsSync(p)).toBe(false)
|
|
478
|
+
})
|
|
479
|
+
it('missing marker → null, no throw', () => {
|
|
480
|
+
expect(consumeBridgeDeadEscalationMarker('/nonexistent/marker.json', 1)).toBeNull()
|
|
481
|
+
})
|
|
482
|
+
it('marker write is atomic-shaped (valid JSON on disk)', () => {
|
|
483
|
+
const dir = mkdtempSync(join(tmpdir(), 'bridge-dead-marker-'))
|
|
484
|
+
const p = join(dir, 'm.json')
|
|
485
|
+
writeBridgeDeadEscalationMarker(p, { ts: 1, reason: 'r', count: 1 })
|
|
486
|
+
expect(JSON.parse(readFileSync(p, 'utf8'))).toEqual({ ts: 1, reason: 'r', count: 1 })
|
|
487
|
+
})
|
|
488
|
+
})
|
|
489
|
+
|
|
490
|
+
// ─── Honest resume surfacing ─────────────────────────────────────────────────
|
|
491
|
+
|
|
492
|
+
function makeTurn(overrides: Partial<Turn> = {}): Turn {
|
|
493
|
+
return {
|
|
494
|
+
turn_key: '100001:11',
|
|
495
|
+
chat_id: '100001',
|
|
496
|
+
thread_id: null,
|
|
497
|
+
started_at: 1_000_000,
|
|
498
|
+
ended_at: null,
|
|
499
|
+
ended_via: 'restart',
|
|
500
|
+
last_assistant_msg_id: null,
|
|
501
|
+
last_assistant_done: null,
|
|
502
|
+
last_user_msg_id: 11,
|
|
503
|
+
user_prompt_preview: 'do the thing',
|
|
504
|
+
tool_call_count: 2,
|
|
505
|
+
interrupt_reason: null,
|
|
506
|
+
resumed_at: null,
|
|
507
|
+
...overrides,
|
|
508
|
+
} as Turn
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
describe('resume inbound restartCause surfacing (#3038)', () => {
|
|
512
|
+
const cause = {
|
|
513
|
+
reason: BRIDGE_DEAD_RESTART_REASON,
|
|
514
|
+
note: 'The framework itself triggered this restart: your Telegram MCP bridge process had died.',
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
it('resume_interrupted carries meta.restart_cause + the honest note, source unchanged', () => {
|
|
518
|
+
const msg = buildResumeInterruptedInbound({ turn: makeTurn(), nowMs: 2_000_000, restartCause: cause })
|
|
519
|
+
expect(msg.meta?.source).toBe('resume_interrupted') // never masquerades as watchdog
|
|
520
|
+
expect(msg.meta?.restart_cause).toBe(BRIDGE_DEAD_RESTART_REASON)
|
|
521
|
+
expect(msg.text).toContain('Why this restart happened:')
|
|
522
|
+
expect(msg.text).toContain('Telegram MCP bridge process had died')
|
|
523
|
+
})
|
|
524
|
+
|
|
525
|
+
it('watchdog report also carries the cause when one was recorded', () => {
|
|
526
|
+
const msg = buildResumeWatchdogReportInbound({
|
|
527
|
+
turn: makeTurn({ ended_via: 'timeout' }),
|
|
528
|
+
idleMs: 300_000,
|
|
529
|
+
nowMs: 2_000_000,
|
|
530
|
+
restartCause: cause,
|
|
531
|
+
})
|
|
532
|
+
expect(msg.meta?.source).toBe('resume_watchdog_timeout')
|
|
533
|
+
expect(msg.meta?.restart_cause).toBe(BRIDGE_DEAD_RESTART_REASON)
|
|
534
|
+
expect(msg.text).toContain('Why this restart happened:')
|
|
535
|
+
})
|
|
536
|
+
|
|
537
|
+
it('deferred report also carries the cause', () => {
|
|
538
|
+
const msg = buildResumeDeferredReportInbound({
|
|
539
|
+
turn: makeTurn(),
|
|
540
|
+
reason: 'loop-guard',
|
|
541
|
+
nowMs: 2_000_000,
|
|
542
|
+
restartCause: cause,
|
|
543
|
+
})
|
|
544
|
+
expect(msg.meta?.source).toBe('resume_deferred')
|
|
545
|
+
expect(msg.meta?.restart_cause).toBe(BRIDGE_DEAD_RESTART_REASON)
|
|
546
|
+
expect(msg.text).toContain('Why this restart happened:')
|
|
547
|
+
})
|
|
548
|
+
|
|
549
|
+
it('no restartCause → inbound unchanged (no meta key, no note)', () => {
|
|
550
|
+
const msg = buildResumeInterruptedInbound({ turn: makeTurn(), nowMs: 2_000_000 })
|
|
551
|
+
expect(msg.meta?.restart_cause).toBeUndefined()
|
|
552
|
+
expect(msg.text).not.toContain('Why this restart happened:')
|
|
553
|
+
})
|
|
554
|
+
})
|
|
555
|
+
|
|
556
|
+
// ─── Idle notice (#3038 review finding 2) ────────────────────────────────────
|
|
557
|
+
|
|
558
|
+
describe('buildBridgeDeadIdleNoticeInbound', () => {
|
|
559
|
+
it('carries the honest cause to the default chat when no turn was in flight', () => {
|
|
560
|
+
const msg = buildBridgeDeadIdleNoticeInbound({
|
|
561
|
+
chatId: '100001',
|
|
562
|
+
marker: { ts: 5_000, reason: BRIDGE_DEAD_RESTART_REASON, count: 1 },
|
|
563
|
+
nowMs: 2_000_000,
|
|
564
|
+
})
|
|
565
|
+
expect(msg.type).toBe('inbound')
|
|
566
|
+
expect(msg.chatId).toBe('100001')
|
|
567
|
+
expect(msg.meta?.source).toBe('bridge_dead_restart')
|
|
568
|
+
expect(msg.meta?.restart_cause).toBe(BRIDGE_DEAD_RESTART_REASON)
|
|
569
|
+
expect(msg.meta?.chat_id).toBe('100001')
|
|
570
|
+
expect(msg.text).toContain('MCP bridge process had died')
|
|
571
|
+
expect(msg.text).toContain('No work was in flight')
|
|
572
|
+
// Honesty rails: names what it was NOT.
|
|
573
|
+
expect(msg.text).toContain('NOT an operator-initiated restart')
|
|
574
|
+
expect(msg.text).toContain('NOT a hang-watchdog kill')
|
|
575
|
+
})
|
|
576
|
+
})
|
|
@@ -126,16 +126,22 @@ describe('buffer-gate release decoupled from final-answer classification', () =>
|
|
|
126
126
|
expect(releaseIdx).toBeGreaterThan(gateBlockClose)
|
|
127
127
|
})
|
|
128
128
|
|
|
129
|
-
it('the helper is invoked from
|
|
129
|
+
it('the helper is invoked from turn-terminal paths only — reply-finalize and halt', () => {
|
|
130
130
|
// Sanity: nothing else should call releaseTurnBufferGate. The
|
|
131
131
|
// helper is narrow on purpose. If future code adds new
|
|
132
|
-
// callsites that aren't
|
|
132
|
+
// callsites that aren't turn-terminal, the steer-vs-queue
|
|
133
133
|
// semantics could drift.
|
|
134
134
|
const callMatches = gatewaySrc.match(/releaseTurnBufferGate\(/g) ?? []
|
|
135
|
-
// Definition +
|
|
136
|
-
//
|
|
135
|
+
// Definition + 2 callsites = 3:
|
|
136
|
+
// - executeReply's post-send block (reply-finalize, the original).
|
|
137
|
+
// - executeHaltNow (#3020): an interrupt-cancelled turn never reaches
|
|
138
|
+
// reply-finalize (the C-c killed it and no replacement inbound
|
|
139
|
+
// follows), so the halt IS that turn's terminal — releasing there is
|
|
140
|
+
// the deterministic sibling of the reply-path release, not a
|
|
141
|
+
// mid-turn drift. (The retired stream_reply callsite was removed
|
|
142
|
+
// with executeStreamReply.)
|
|
137
143
|
// If this count grows the test catches it; reviewer must justify
|
|
138
144
|
// any new callsite.
|
|
139
|
-
expect(callMatches.length).toBe(
|
|
145
|
+
expect(callMatches.length).toBe(3)
|
|
140
146
|
})
|
|
141
147
|
})
|