switchroom 0.21.7 → 0.21.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/bin/tmp-reaper.sh +234 -0
  2. package/dist/agent-scheduler/index.js +1 -1
  3. package/dist/auth-broker/index.js +2 -2
  4. package/dist/cli/notion-write-pretool.mjs +1 -1
  5. package/dist/cli/switchroom.js +3421 -2744
  6. package/dist/host-control/main.js +177 -13
  7. package/dist/vault/approvals/kernel-server.js +2 -2
  8. package/dist/vault/broker/server.js +2 -2
  9. package/package.json +5 -4
  10. package/profiles/_base/start.sh.hbs +115 -0
  11. package/profiles/_shared/local-time.md.hbs +6 -0
  12. package/profiles/default/CLAUDE.md.hbs +0 -12
  13. package/telegram-plugin/dist/gateway/gateway.js +1017 -465
  14. package/telegram-plugin/gateway/agent-process-liveness.ts +558 -0
  15. package/telegram-plugin/gateway/approval-hold.ts +32 -1
  16. package/telegram-plugin/gateway/approval-outcome-sources.ts +274 -0
  17. package/telegram-plugin/gateway/bridge-dead-watchdog.ts +21 -9
  18. package/telegram-plugin/gateway/callback-query-handlers.ts +87 -15
  19. package/telegram-plugin/gateway/eval-case-proposal-inbound-builders.ts +197 -0
  20. package/telegram-plugin/gateway/gateway.ts +12 -10
  21. package/telegram-plugin/gateway/pending-inbound-buffer.ts +167 -11
  22. package/telegram-plugin/gateway/self-improve-proposal-wiring.test.ts +333 -0
  23. package/telegram-plugin/gateway/self-improve-proposal-wiring.ts +152 -3
  24. package/telegram-plugin/gateway/subagent-handback-marker.ts +19 -0
  25. package/telegram-plugin/tests/agent-process-liveness.test.ts +406 -0
  26. package/telegram-plugin/tests/approval-hold-record.test.ts +21 -8
  27. package/telegram-plugin/tests/boot-resume-gateway-only-respawn.test.ts +752 -0
  28. package/telegram-plugin/tests/boot-resume-guard-wiring.test.ts +203 -0
  29. package/telegram-plugin/tests/callback-query-handlers.test.ts +143 -1
  30. package/telegram-plugin/tests/eval-case-proposal-inbound-builders.test.ts +144 -0
  31. package/telegram-plugin/tests/hermes-messages-paging.test.ts +149 -0
  32. package/telegram-plugin/tests/hermes-session-search.test.ts +146 -0
  33. package/telegram-plugin/tests/pending-inbound-buffer.test.ts +443 -2
  34. package/telegram-plugin/tests/subagent-handback-marker.test.ts +14 -0
@@ -0,0 +1,752 @@
1
+ /**
2
+ * switchroom#4641 — OUTCOME test for the gateway-only-respawn guard.
3
+ *
4
+ * The bug: the gateway and the `claude` agent are separate supervised
5
+ * processes. When the gateway alone crashed and respawned (three Bun SIGBUS
6
+ * crashes on `overlord`, 2026-08-12), its boot block stamped the STILL-RUNNING
7
+ * turn `ended_via='restart'`, spooled a `resume_interrupted` synthetic telling
8
+ * the live session "You just restarted. Your previous turn was interrupted…",
9
+ * and listed its still-running sub-agents as "killed by the restart".
10
+ *
11
+ * What this test asserts is the OUTCOME the issue names, end to end over the
12
+ * REAL collaborators — a real bun:sqlite registry, the real boot reaper, the
13
+ * real interrupted-turn finder, the real inbound builders and the real durable
14
+ * inbound spool — with only the ORDERING supplied by `runBootSequence` below.
15
+ *
16
+ * ORDERING FIDELITY (the point an earlier revision of this file got wrong):
17
+ * gateway.ts's `bootResumeInit` block does NOT spool the resume. It builds
18
+ * `bootResumeInbound` in MEMORY; the durable `inboundSpool.put` lives ~8k lines
19
+ * later, and the generation-token stamp lives immediately after THAT. A
20
+ * harness that puts inside its model of the block collapses the whole window
21
+ * between "block done" and "resume durable" — the one window a crash can
22
+ * actually land in — and so cannot see a stamp placed too early. This harness
23
+ * keeps the three stages distinct and lets `crashAt` land in each of them.
24
+ * `stampAt: 'block-end'` reproduces the defective placement on purpose.
25
+ * (`boot-resume-guard-wiring.test.ts` pins that gateway.ts really does stamp
26
+ * after the put, since this file cannot import the block.)
27
+ *
28
+ * The second fidelity trap, same shape: an in-block failure is NOT a process
29
+ * death. gateway.ts swallows it (`catch { log; turnsDb = null }`) and module
30
+ * init runs on to the stamp. `throwAt` models that swallow — distinct from
31
+ * `crashAt`, which kills the sequence — and it is why the stamp is gated on
32
+ * `bootResumeThrew`, not merely placed late.
33
+ *
34
+ * - gateway killed mid-turn while the claude process LIVES
35
+ * → ZERO `resume_interrupted` entries in the on-disk spool,
36
+ * the in-flight turn row is left OPEN (no `ended_via` lie), and
37
+ * no `.pending-turn.env` is written.
38
+ * - a gateway that CRASHES DURING ITS OWN BOOT, before stamping the
39
+ * per-container-boot generation token — at each of the three reachable
40
+ * points (after the guard, after the block but before the durable spool
41
+ * put, after the put but before the stamp)
42
+ * → the successor gateway does the full boot resume; the interrupted
43
+ * turn is not lost. (This is the case the earlier starttime-ordering
44
+ * guard got wrong: it saw a live claude that predated it and
45
+ * suppressed forever.)
46
+ * - the same crash with the token stamped at the END OF THE BLOCK instead
47
+ * → the resume is DROPPED forever. Asserted explicitly as a defect
48
+ * reproduction, so the correct placement is pinned by outcome.
49
+ * - a suppressed respawn and the #3038 bridge-dead escalation marker
50
+ * → the marker is NOT re-consumed, because the gateway that stamped
51
+ * the token already consumed it this generation.
52
+ * - genuine agent restart (recorded process really is dead)
53
+ * → the resume still fires, exactly as before.
54
+ * - watchdog-timeout classification and first boot (no record file)
55
+ * → unchanged.
56
+ *
57
+ * Every "alive" claim is anchored on a process this test actually spawned, and
58
+ * every "dead" claim on one it actually killed — so a probe hard-wired to
59
+ * either answer fails here. The generation token is never hand-placed for a
60
+ * suppression case without a boot that legitimately stamped it, or an
61
+ * explicitly-stated stale-token scenario.
62
+ *
63
+ * Uses bun:sqlite (openTurnsDb) — excluded from vitest.config.ts, run by
64
+ * telegram-plugin/scripts/bun-test-ci.sh (which runs `tests/`).
65
+ */
66
+
67
+ import { describe, it, expect, beforeEach, afterEach } from 'bun:test'
68
+ import { spawn, spawnSync } from 'node:child_process'
69
+ import {
70
+ mkdtempSync,
71
+ rmSync,
72
+ existsSync,
73
+ readFileSync,
74
+ writeFileSync,
75
+ appendFileSync,
76
+ renameSync,
77
+ statSync,
78
+ mkdirSync,
79
+ } from 'node:fs'
80
+ import { tmpdir } from 'node:os'
81
+ import { join } from 'node:path'
82
+
83
+ import {
84
+ openTurnsDb,
85
+ recordTurnStart,
86
+ markOrphanedWithTimeoutClassification,
87
+ findLatestTurnIfInterrupted,
88
+ getTurnByKey,
89
+ markTurnResumed,
90
+ } from '../registry/turns-schema.js'
91
+ import {
92
+ decideBootResumeKind,
93
+ buildResumeInterruptedInbound,
94
+ buildResumeWatchdogReportInbound,
95
+ } from '../gateway/resume-inbound-builder.js'
96
+ import { writePendingTurnEnv } from '../gateway/pending-turn-env.js'
97
+ import { createInboundSpool } from '../gateway/inbound-spool.js'
98
+ import type { InboundMessage } from '../gateway/ipc-protocol.js'
99
+ import {
100
+ readProcIdentity,
101
+ shouldSkipBootResumeForGatewayOnlyRespawn,
102
+ markBootResumeComplete,
103
+ bootResumeDonePath,
104
+ AGENT_PROCESS_RECORD_FILE,
105
+ } from '../gateway/agent-process-liveness.js'
106
+ import { consumeBridgeDeadEscalationMarker } from '../gateway/bridge-dead-watchdog.js'
107
+
108
+ // ---------------------------------------------------------------------------
109
+ // Fixture
110
+ // ---------------------------------------------------------------------------
111
+
112
+ let agentDir: string
113
+ let stateDir: string
114
+ /** Long-lived stand-in for the `claude` process. */
115
+ let agentProc: ReturnType<typeof spawn>
116
+
117
+ function sleepMs(ms: number): void {
118
+ spawnSync('sleep', [String(ms / 1000)])
119
+ }
120
+
121
+ function starttimeOf(pid: number): string {
122
+ const id = readProcIdentity(pid)
123
+ if (id == null) throw new Error(`pid ${pid} unexpectedly not live`)
124
+ return id.starttime
125
+ }
126
+
127
+ /** Write the record start.sh writes immediately before `exec claude`. */
128
+ function writeAgentProcessRecord(pid: number, starttime: string): void {
129
+ writeFileSync(
130
+ join(stateDir, AGENT_PROCESS_RECORD_FILE),
131
+ JSON.stringify({ pid, starttime, boot_at: Date.now() }),
132
+ )
133
+ }
134
+
135
+ const SPOOL_FS = {
136
+ appendFileSync: (p: string, d: string) => appendFileSync(p, d),
137
+ readFileSync: (p: string) => readFileSync(p, 'utf8'),
138
+ writeFileSync: (p: string, d: string) => writeFileSync(p, d),
139
+ renameSync: (a: string, b: string) => renameSync(a, b),
140
+ existsSync: (p: string) => existsSync(p),
141
+ statSizeSync: (p: string) => statSync(p).size,
142
+ fsyncFileSync: () => {},
143
+ fsyncDirSync: () => {},
144
+ }
145
+
146
+ interface BootResult {
147
+ /** Every `source` value that reached the durable on-disk spool. */
148
+ spooledSources: string[]
149
+ /** Raw spool file contents (empty string when never created). */
150
+ spoolRaw: string
151
+ reaped: number
152
+ pendingEnvWritten: boolean
153
+ /** The #3038 bridge-dead marker this boot consumed, if any. */
154
+ bridgeDeadStreak: number | null
155
+ /** Did this boot stamp the per-container-boot generation token? */
156
+ tokenStamped: boolean
157
+ }
158
+
159
+ /**
160
+ * Where a gateway dies during its own boot (the Bun 1.3.13 crash-at-boot
161
+ * pattern that motivated the fix). Each value names the point the process
162
+ * vanishes; everything before it has already happened.
163
+ *
164
+ * 'after-guard' — before any side effect at all.
165
+ * 'after-block-before-put' — the WHOLE bootResumeInit block ran (turn reaped
166
+ * and durably stamped `ended_via='restart'`, env
167
+ * written, inbound built IN MEMORY) but the
168
+ * process dies before the durable spool put. This
169
+ * is the window a too-early token stamp loses
170
+ * work in, and the one the pre-fix harness could
171
+ * not express.
172
+ * 'after-put-before-stamp' — the resume IS durable; only the token is
173
+ * missing. Successor repeats, spool dedups.
174
+ */
175
+ type CrashPoint = 'after-guard' | 'after-block-before-put' | 'after-put-before-stamp'
176
+
177
+ /** Sentinel used to unwind out of the modelled boot at `crashAt`. */
178
+ const BOOT_CRASH = Symbol('boot-crash')
179
+ /**
180
+ * A throw INSIDE the boot block — swallowed, not fatal. gateway.ts's block
181
+ * ends in `catch (err) { log; turnsDb = null }` and module init CONTINUES, so
182
+ * unlike BOOT_CRASH the later stages still run. Modelling every in-block
183
+ * failure as a process death is what hid the swallow-then-stamp defect.
184
+ */
185
+ const BLOCK_THROW = Symbol('block-throw')
186
+
187
+ /**
188
+ * The gateway's boot sequence in gateway.ts's real order, over the real
189
+ * collaborators, in THREE distinct stages:
190
+ *
191
+ * 1. `bootResumeInit` — guard, reaper, marker, builders. In-memory only.
192
+ * 2. the durable `inboundSpool.put` (gateway.ts ~8k lines later).
193
+ * 3. `markBootResumeComplete` — unconditional, top level.
194
+ *
195
+ * `guarded: false` reproduces the pre-#4641 gateway. `stampAt: 'block-end'`
196
+ * reproduces this PR's first (defective) placement.
197
+ */
198
+ function runBootSequence(opts: {
199
+ guarded?: boolean
200
+ markerTurnKey?: string | null
201
+ markerAgeMs?: number | null
202
+ crashAt?: CrashPoint
203
+ /** Where the generation token is stamped. Default = production. */
204
+ stampAt?: 'block-end' | 'after-durable-put'
205
+ /**
206
+ * Throw inside the block and let gateway.ts's `catch` swallow it. Unlike
207
+ * `crashAt` the process survives, so stages 2 and 3 still run — which is the
208
+ * whole point: the reaper has already durably ended the turn.
209
+ */
210
+ throwAt?: 'after-reaper'
211
+ /**
212
+ * Production (post-fix) gates the stamp on the block not having thrown
213
+ * (`bootResumeThrew`). Set false to reproduce the ungated defect.
214
+ */
215
+ gateStampOnBlockThrow?: boolean
216
+ }): BootResult {
217
+ const spoolPath = join(stateDir, 'inbound-spool.jsonl')
218
+ const db = openTurnsDb(agentDir)
219
+ const spool = createInboundSpool({ path: spoolPath, fs: SPOOL_FS, log: () => {} })
220
+ const stampAt = opts.stampAt ?? 'after-durable-put'
221
+ let reaped = 0
222
+ let bridgeDeadStreak: number | null = null
223
+ let tokenStamped = false
224
+ const stamp = (): void => {
225
+ markBootResumeComplete(stateDir)
226
+ tokenStamped = true
227
+ }
228
+ try {
229
+ // ---- stage 1: the bootResumeInit block (IN MEMORY ONLY) ----------------
230
+ // gateway.ts's block-level `catch`: it logs, nulls turnsDb and lets module
231
+ // init CONTINUE. `bootResumeThrew` is the flag that catch sets.
232
+ let bootResumeThrew = false
233
+ // gateway.ts:1648 — `bootResumeInbound`, stashed and pushed to the spool
234
+ // only much later. Nothing on disk yet.
235
+ let bootResumeInbound: InboundMessage | null = null
236
+ try {
237
+ // The guard, gateway.ts: immediately after applySubagentsSchema.
238
+ const skip =
239
+ (opts.guarded ?? true) &&
240
+ shouldSkipBootResumeForGatewayOnlyRespawn(stateDir, { log: () => {} })
241
+ if (!skip) {
242
+ if (opts.crashAt === 'after-guard') throw BOOT_CRASH
243
+ const res = markOrphanedWithTimeoutClassification(db, {
244
+ markerTurnKey: opts.markerTurnKey ?? null,
245
+ markerAgeMs: opts.markerAgeMs ?? null,
246
+ hangThresholdMs: 300_000,
247
+ })
248
+ reaped = res.reaped
249
+ // The turn is NOW durably `ended_via='restart'`. Everything from here to
250
+ // the first `bootResumeInbound =` — findLatestTurnIfInterrupted, the
251
+ // clean-shutdown-marker read, listNonTerminalSubagentsForTurn, the
252
+ // builders — is ~156 lines of DB and fs I/O that can throw.
253
+ if (opts.throwAt === 'after-reaper') throw BLOCK_THROW
254
+ // gateway.ts:~1721 — consumed (always cleared) whether or not a turn was
255
+ // in flight. The `break` skips this too, which is only correct because
256
+ // the token proves a gateway already consumed it this generation.
257
+ const marker = consumeBridgeDeadEscalationMarker(
258
+ join(stateDir, 'bridge-dead-escalation.json'),
259
+ )
260
+ if (marker != null) bridgeDeadStreak = marker.count ?? 1
261
+ const pending = findLatestTurnIfInterrupted(db)
262
+ if (pending != null) {
263
+ const kind = decideBootResumeKind({
264
+ pending,
265
+ suppressed: false,
266
+ ageMs: Math.max(0, Date.now() - pending.started_at),
267
+ maxAgeMs: 10_800_000,
268
+ })
269
+ if (kind === 'resume') {
270
+ bootResumeInbound = buildResumeInterruptedInbound({ turn: pending, subagents: [] })
271
+ } else if (kind === 'report') {
272
+ bootResumeInbound = buildResumeWatchdogReportInbound({
273
+ turn: pending,
274
+ idleMs: 600_000,
275
+ subagents: [],
276
+ })
277
+ }
278
+ }
279
+ writePendingTurnEnv(agentDir, pending, () => {})
280
+ // THE DEFECT, reproducible on demand: stamping here marks the generation
281
+ // done while the resume exists only in this process's heap.
282
+ if (stampAt === 'block-end') stamp()
283
+ if (opts.crashAt === 'after-block-before-put') throw BOOT_CRASH
284
+ }
285
+ } catch (err) {
286
+ if (err === BOOT_CRASH) throw err // process death: nothing below runs
287
+ if (err !== BLOCK_THROW) throw err
288
+ bootResumeThrew = true // swallowed — module init continues below
289
+ }
290
+ // ---- stage 2: gateway.ts ~:10148, the DURABLE put ----------------------
291
+ if (bootResumeInbound != null) {
292
+ spool.put('tester', bootResumeInbound)
293
+ // …and the at-most-once ledger, stamped only AFTER the put — the same
294
+ // ordering rule the generation token now obeys (turns-schema.ts).
295
+ const rk = bootResumeInbound.meta?.resume_turn_key
296
+ if (typeof rk === 'string' && rk.length > 0) markTurnResumed(db, rk)
297
+ }
298
+ if (opts.crashAt === 'after-put-before-stamp') throw BOOT_CRASH
299
+ // ---- stage 3: the token (gateway.ts, top level) ------------------------
300
+ // Unconditional on there being anything to resume; NOT unconditional on
301
+ // the block having completed.
302
+ const gated = (opts.gateStampOnBlockThrow ?? true) && bootResumeThrew
303
+ if (stampAt === 'after-durable-put' && !gated) stamp()
304
+ } catch (err) {
305
+ if (err !== BOOT_CRASH) throw err
306
+ } finally {
307
+ db.close()
308
+ }
309
+ const spoolRaw = existsSync(spoolPath) ? readFileSync(spoolPath, 'utf8') : ''
310
+ const spooledSources = spoolRaw
311
+ .split('\n')
312
+ .filter((l) => l.trim().length > 0)
313
+ .map((l) => JSON.parse(l) as { t: string; msg?: { meta?: { source?: string } } })
314
+ .filter((r) => r.t === 'put')
315
+ .map((r) => r.msg?.meta?.source ?? '(none)')
316
+ return {
317
+ spooledSources,
318
+ spoolRaw,
319
+ reaped,
320
+ pendingEnvWritten: existsSync(join(agentDir, '.pending-turn.env')),
321
+ bridgeDeadStreak,
322
+ tokenStamped,
323
+ }
324
+ }
325
+
326
+ /** Seed an in-flight turn: started, never ended — the live turn's row. */
327
+ function seedInFlightTurn(turnKey = 'chat:1786528508921'): string {
328
+ const db = openTurnsDb(agentDir)
329
+ try {
330
+ recordTurnStart(db, {
331
+ turnKey,
332
+ chatId: '12345',
333
+ userPromptPreview: 'dispatch the worker',
334
+ })
335
+ } finally {
336
+ db.close()
337
+ }
338
+ return turnKey
339
+ }
340
+
341
+ beforeEach(() => {
342
+ agentDir = mkdtempSync(join(tmpdir(), 'boot-respawn-'))
343
+ stateDir = join(agentDir, 'telegram')
344
+ mkdirSync(stateDir, { recursive: true })
345
+ // start.sh's outer pass: a fresh container boot clears the generation token.
346
+ // Every case that wants a gateway-only respawn must first run a boot that
347
+ // stamps it, exactly as production does.
348
+ expect(existsSync(bootResumeDonePath(stateDir))).toBe(false)
349
+ agentProc = spawn('sleep', ['120'], { stdio: 'ignore' })
350
+ sleepMs(20)
351
+ })
352
+
353
+ afterEach(() => {
354
+ try { agentProc.kill('SIGKILL') } catch { /* already gone */ }
355
+ rmSync(agentDir, { recursive: true, force: true })
356
+ })
357
+
358
+ /** Write a fresh #3038 bridge-dead escalation marker. */
359
+ function writeBridgeDeadMarker(count = 2): string {
360
+ const p = join(stateDir, 'bridge-dead-escalation.json')
361
+ writeFileSync(p, JSON.stringify({ ts: Date.now(), reason: 'mcp-bridge-dead', count }))
362
+ return p
363
+ }
364
+
365
+ // ---------------------------------------------------------------------------
366
+ // The bug
367
+ // ---------------------------------------------------------------------------
368
+
369
+
370
+ /** A completed first boot of this container generation: stamps the token. */
371
+ function firstBootOfThisContainer(): BootResult {
372
+ return runBootSequence({})
373
+ }
374
+
375
+ describe('gateway-only respawn (claude still running)', () => {
376
+ it('spools ZERO resume_interrupted entries and leaves the live turn open', () => {
377
+ // Full production sequence. Boot #1 of this container: no turn in flight,
378
+ // nothing reaped, generation token stamped. start.sh then publishes the
379
+ // agent record and execs claude, which takes a message…
380
+ const first = firstBootOfThisContainer()
381
+ expect(first.tokenStamped).toBe(true)
382
+ expect(first.spooledSources).toEqual([])
383
+ writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
384
+ const turnKey = seedInFlightTurn()
385
+
386
+ // …and THEN the gateway alone crashes and respawns.
387
+ const out = runBootSequence({})
388
+
389
+ // The outcome the issue names.
390
+ expect(out.spooledSources).toEqual([])
391
+ expect(out.spoolRaw).not.toContain('resume_interrupted')
392
+ // …and no collateral: the live turn keeps its open row (no `restart` lie),
393
+ // and the one-shot wake-audit env is not written by the respawn either.
394
+ expect(out.reaped).toBe(0)
395
+ expect(out.pendingEnvWritten).toBe(false)
396
+ const db = openTurnsDb(agentDir)
397
+ try {
398
+ const turn = getTurnByKey(db, turnKey)!
399
+ expect(turn.ended_at).toBeNull()
400
+ expect(turn.ended_via).toBeNull()
401
+ } finally {
402
+ db.close()
403
+ }
404
+ })
405
+
406
+ it('WOULD fire the false resume without the guard (bug reproduction)', () => {
407
+ const turnKey = seedInFlightTurn()
408
+ writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
409
+ markBootResumeComplete(stateDir)
410
+
411
+ // Same fixture, guard bypassed — the pre-#4641 gateway.
412
+ const out = runBootSequence({ guarded: false })
413
+
414
+ expect(out.spooledSources).toEqual(['resume_interrupted'])
415
+ expect(out.reaped).toBe(1)
416
+ const db = openTurnsDb(agentDir)
417
+ try {
418
+ expect(getTurnByKey(db, turnKey)!.ended_via).toBe('restart')
419
+ } finally {
420
+ db.close()
421
+ }
422
+ })
423
+
424
+ it('stays silent across repeated gateway crashes', () => {
425
+ seedInFlightTurn()
426
+ writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
427
+ markBootResumeComplete(stateDir)
428
+ for (let i = 0; i < 3; i++) {
429
+ expect(runBootSequence({}).spooledSources).toEqual([])
430
+ }
431
+ })
432
+ })
433
+
434
+ // ---------------------------------------------------------------------------
435
+ // A gateway that crashes DURING its own boot (reviewer MAJOR 2)
436
+ // ---------------------------------------------------------------------------
437
+
438
+ describe('gateway crash during its own boot', () => {
439
+ it('still resumes the interrupted turn — the token was never stamped', () => {
440
+ // The container restarts with a genuinely interrupted turn. start.sh forks
441
+ // gateway #1 and (before the exec) publishes a record naming a LIVE agent
442
+ // process that predates gateway #2. Gateway #1 then dies before reaching
443
+ // the boot block, so it never stamps the token.
444
+ //
445
+ // The replacement gateway MUST do the full boot resume. The previous
446
+ // starttime-ordering guard suppressed exactly here — it saw a live claude
447
+ // older than itself — and the interrupted turn was never reaped or
448
+ // resumed at all.
449
+ const turnKey = seedInFlightTurn()
450
+ writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
451
+
452
+ const crashed = runBootSequence({ crashAt: 'after-guard' })
453
+ expect(crashed.tokenStamped).toBe(false)
454
+ expect(crashed.spooledSources).toEqual([])
455
+ expect(existsSync(bootResumeDonePath(stateDir))).toBe(false)
456
+
457
+ const out = runBootSequence({})
458
+
459
+ expect(out.spooledSources).toEqual(['resume_interrupted'])
460
+ expect(out.reaped).toBe(1)
461
+ expect(out.pendingEnvWritten).toBe(true)
462
+ expect(out.tokenStamped).toBe(true)
463
+ const db = openTurnsDb(agentDir)
464
+ try {
465
+ expect(getTurnByKey(db, turnKey)!.ended_via).toBe('restart')
466
+ } finally {
467
+ db.close()
468
+ }
469
+ })
470
+
471
+ it('still consumes the bridge-dead escalation marker', () => {
472
+ // #3038: the marker is consumed (always cleared) whether or not a turn was
473
+ // in flight, and its `count` seeds the cross-boot escalation damper. A
474
+ // gateway that crashed before consuming it must not strand it on disk —
475
+ // the successor's boot block has to reach it.
476
+ const markerPath = writeBridgeDeadMarker(3)
477
+ writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
478
+
479
+ const crashed = runBootSequence({ crashAt: 'after-guard' })
480
+ expect(crashed.bridgeDeadStreak).toBeNull()
481
+ expect(existsSync(markerPath)).toBe(true)
482
+
483
+ const out = runBootSequence({})
484
+ expect(out.bridgeDeadStreak).toBe(3)
485
+ expect(existsSync(markerPath)).toBe(false)
486
+ })
487
+
488
+ it('a crash after the DURABLE PUT but before the stamp does not suppress either', () => {
489
+ // The resume is already on disk; only the token is missing. The successor
490
+ // re-runs the block, and the spool's `s:resume:<turnKey>` dedup makes the
491
+ // repeat a no-op rather than a double-send.
492
+ seedInFlightTurn()
493
+ writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
494
+
495
+ const crashed = runBootSequence({ crashAt: 'after-put-before-stamp' })
496
+ expect(crashed.spooledSources).toEqual(['resume_interrupted'])
497
+ expect(crashed.tokenStamped).toBe(false)
498
+
499
+ const out = runBootSequence({})
500
+ expect(out.tokenStamped).toBe(true) // it did NOT suppress
501
+ // …and exactly ONE resume reached the spool across both boots.
502
+ expect(out.spooledSources).toEqual(['resume_interrupted'])
503
+ })
504
+
505
+ it('a crash AFTER the block but BEFORE the durable put still resumes (the reachable window)', () => {
506
+ // THE regression this placement fix exists for. The block ran to
507
+ // completion: the reaper has DURABLY stamped the turn `ended_via='restart'`
508
+ // and the resume inbound exists only in the dying process's heap. Nothing
509
+ // about the resume is on disk.
510
+ //
511
+ // With the token stamped after the durable put (production), the successor
512
+ // finds no token and re-mints the resume — the work survives.
513
+ const turnKey = seedInFlightTurn()
514
+ // Note: start.sh has NOT yet published agent-process.json at this point —
515
+ // the gateway is forked long before `exec claude`. That is precisely why a
516
+ // premature token is fatal here: the successor's only corroborating
517
+ // evidence is absent, so it returns `gateway-only-respawn-no-record`.
518
+ expect(existsSync(join(stateDir, AGENT_PROCESS_RECORD_FILE))).toBe(false)
519
+
520
+ const crashed = runBootSequence({ crashAt: 'after-block-before-put' })
521
+ expect(crashed.spooledSources).toEqual([]) // nothing durable yet
522
+ expect(crashed.reaped).toBe(1) // …but the turn IS durably stamped
523
+ expect(crashed.tokenStamped).toBe(false)
524
+ expect(existsSync(bootResumeDonePath(stateDir))).toBe(false)
525
+
526
+ const out = runBootSequence({})
527
+
528
+ expect(out.spooledSources).toEqual(['resume_interrupted'])
529
+ expect(out.tokenStamped).toBe(true)
530
+ const db = openTurnsDb(agentDir)
531
+ try {
532
+ expect(getTurnByKey(db, turnKey)!.ended_via).toBe('restart')
533
+ } finally {
534
+ db.close()
535
+ }
536
+ })
537
+
538
+ it('WOULD lose the turn forever if the token were stamped at the end of the block', () => {
539
+ // Defect reproduction for the placement, mirroring the `guarded: false`
540
+ // bug-reproduction above. Same crash point as the test directly above;
541
+ // only the stamp moves. The turn is durably `ended_via='restart'`, no
542
+ // resume ever reaches the spool, and the successor suppresses on the token
543
+ // alone — silent, permanent work loss, and a NEW failure mode versus the
544
+ // pre-#4641 gateway, which simply re-ran the block.
545
+ const turnKey = seedInFlightTurn()
546
+
547
+ const crashed = runBootSequence({ crashAt: 'after-block-before-put', stampAt: 'block-end' })
548
+ expect(crashed.tokenStamped).toBe(true) // stamped while the resume is in RAM
549
+ expect(crashed.spooledSources).toEqual([])
550
+ expect(existsSync(bootResumeDonePath(stateDir))).toBe(true)
551
+
552
+ const out = runBootSequence({})
553
+
554
+ expect(out.spooledSources, 'the resume is dropped, not retried').toEqual([])
555
+ expect(out.reaped).toBe(0)
556
+ const db = openTurnsDb(agentDir)
557
+ try {
558
+ const turn = getTurnByKey(db, turnKey)!
559
+ expect(turn.ended_via).toBe('restart') // durably ended…
560
+ expect(turn.resumed_at).toBeNull() // …and never resumed
561
+ } finally {
562
+ db.close()
563
+ }
564
+ })
565
+
566
+ it('a SWALLOWED throw inside the block leaves no token, so the successor still resumes', () => {
567
+ // gateway.ts's block ends in `catch { log; turnsDb = null }` — a swallow,
568
+ // not a death: module init continues and reaches the stamp with
569
+ // `bootResumeInbound` still null. By then the reaper has ALREADY durably
570
+ // written `ended_via='restart'`. Gating the stamp on that catch is what
571
+ // keeps the work recoverable.
572
+ const turnKey = seedInFlightTurn()
573
+
574
+ const threw = runBootSequence({ throwAt: 'after-reaper' })
575
+ expect(threw.reaped, 'the reaper ran and durably ended the turn').toBe(1)
576
+ expect(threw.spooledSources, 'nothing was spooled — the block never built it').toEqual([])
577
+ expect(threw.tokenStamped, 'a swallowed throw must not stamp the generation').toBe(false)
578
+ expect(existsSync(bootResumeDonePath(stateDir))).toBe(false)
579
+
580
+ // The gateway respawns (the Bun-crash pattern #4641 exists for).
581
+ const out = runBootSequence({})
582
+ expect(out.spooledSources, 'the successor re-mints the resume').toEqual(['resume_interrupted'])
583
+ const db = openTurnsDb(agentDir)
584
+ try {
585
+ expect(getTurnByKey(db, turnKey)!.resumed_at, 'and stamps it resumed').not.toBeNull()
586
+ } finally {
587
+ db.close()
588
+ }
589
+ })
590
+
591
+ it('WOULD lose the turn forever if a swallowed throw still stamped the token', () => {
592
+ // Defect reproduction for the ungated stamp: same swallow, stamp not gated
593
+ // on `bootResumeThrew`. The turn is durably ended, nothing is spooled, and
594
+ // the successor suppresses on the token alone.
595
+ const turnKey = seedInFlightTurn()
596
+
597
+ const threw = runBootSequence({ throwAt: 'after-reaper', gateStampOnBlockThrow: false })
598
+ expect(threw.reaped).toBe(1)
599
+ expect(threw.spooledSources).toEqual([])
600
+ expect(threw.tokenStamped, 'stamped with nothing spooled anywhere').toBe(true)
601
+
602
+ const out = runBootSequence({})
603
+ expect(out.spooledSources, 'the resume is dropped, not retried').toEqual([])
604
+ expect(out.reaped).toBe(0)
605
+ const db = openTurnsDb(agentDir)
606
+ try {
607
+ const turn = getTurnByKey(db, turnKey)!
608
+ expect(turn.ended_via).toBe('restart') // durably ended…
609
+ expect(turn.resumed_at).toBeNull() // …and never resumed
610
+ } finally {
611
+ db.close()
612
+ }
613
+ })
614
+ })
615
+
616
+ // ---------------------------------------------------------------------------
617
+ // What the `break` skips, beyond the resume synthetic (reviewer MEDIUM)
618
+ // ---------------------------------------------------------------------------
619
+
620
+ describe('a suppressed respawn and the bridge-dead escalation marker', () => {
621
+ it('leaves an already-consumed generation alone: no marker, no streak reset', () => {
622
+ // Boot #1 of this container consumes the marker (count=3 → the damper) and
623
+ // stamps the token. The respawn must NOT re-run that consumption: there is
624
+ // nothing left to consume, and a fresh marker written later in the SAME
625
+ // generation belongs to the live watchdog, not to a boot classification.
626
+ writeBridgeDeadMarker(3)
627
+ writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
628
+ const first = runBootSequence({})
629
+ expect(first.bridgeDeadStreak).toBe(3)
630
+ expect(first.tokenStamped).toBe(true)
631
+
632
+ const markerPath = writeBridgeDeadMarker(5)
633
+ const out = runBootSequence({})
634
+
635
+ expect(out.bridgeDeadStreak).toBeNull() // the break skipped the consumption
636
+ expect(existsSync(markerPath)).toBe(true) // …so the marker survives on disk
637
+ })
638
+ })
639
+
640
+ // ---------------------------------------------------------------------------
641
+ // The cases that must NOT change
642
+ // ---------------------------------------------------------------------------
643
+
644
+ describe('genuine agent restart', () => {
645
+ it('still fires resume_interrupted when the recorded process is really dead', () => {
646
+ const turnKey = seedInFlightTurn()
647
+ const doomed = spawn('sleep', ['120'], { stdio: 'ignore' })
648
+ sleepMs(20)
649
+ writeAgentProcessRecord(doomed.pid!, starttimeOf(doomed.pid!))
650
+ doomed.kill('SIGKILL')
651
+ sleepMs(200)
652
+ // Even with a token from the previous generation left behind (a stale
653
+ // volume, an unclean boot), a dead agent VETOES the suppression.
654
+ markBootResumeComplete(stateDir)
655
+
656
+ const out = runBootSequence({})
657
+
658
+ expect(out.spooledSources).toEqual(['resume_interrupted'])
659
+ expect(out.spoolRaw).toContain('resume_interrupted')
660
+ expect(out.reaped).toBe(1)
661
+ expect(out.pendingEnvWritten).toBe(true)
662
+ const db = openTurnsDb(agentDir)
663
+ try {
664
+ expect(getTurnByKey(db, turnKey)!.ended_via).toBe('restart')
665
+ } finally {
666
+ db.close()
667
+ }
668
+ })
669
+
670
+ it('still fires on a fresh container boot (token cleared by start.sh)', () => {
671
+ seedInFlightTurn()
672
+ // The fresh-boot shape the ordering arm used to guard: start.sh has
673
+ // already published a record for a LIVE agent process. Without the token
674
+ // there is no suppression, whatever /proc says.
675
+ writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
676
+ expect(existsSync(bootResumeDonePath(stateDir))).toBe(false)
677
+
678
+ const out = runBootSequence({})
679
+
680
+ expect(out.spooledSources).toEqual(['resume_interrupted'])
681
+ expect(out.pendingEnvWritten).toBe(true)
682
+ })
683
+
684
+ it('still fires on first boot, when no process record exists at all', () => {
685
+ seedInFlightTurn()
686
+ expect(existsSync(join(stateDir, AGENT_PROCESS_RECORD_FILE))).toBe(false)
687
+
688
+ const out = runBootSequence({})
689
+
690
+ expect(out.spooledSources).toEqual(['resume_interrupted'])
691
+ expect(out.pendingEnvWritten).toBe(true)
692
+ })
693
+
694
+ it('still fires when a recycled PID matches but the starttime does not', () => {
695
+ seedInFlightTurn()
696
+ // A live pid whose recorded starttime disagrees — the post-container-
697
+ // restart shape, where the old record's pid names a different process.
698
+ writeAgentProcessRecord(
699
+ agentProc.pid!,
700
+ String(BigInt(starttimeOf(agentProc.pid!)) - 1n),
701
+ )
702
+ markBootResumeComplete(stateDir)
703
+
704
+ expect(runBootSequence({}).spooledSources).toEqual(['resume_interrupted'])
705
+ })
706
+ })
707
+
708
+ describe('watchdog-timeout classification', () => {
709
+ it('still reports resume_watchdog_timeout for a dead agent that stalled', () => {
710
+ const turnKey = seedInFlightTurn()
711
+ const doomed = spawn('sleep', ['120'], { stdio: 'ignore' })
712
+ sleepMs(20)
713
+ writeAgentProcessRecord(doomed.pid!, starttimeOf(doomed.pid!))
714
+ doomed.kill('SIGKILL')
715
+ sleepMs(200)
716
+
717
+ const out = runBootSequence({
718
+ markerTurnKey: turnKey,
719
+ markerAgeMs: 600_000, // past the 300s hang threshold
720
+ })
721
+
722
+ expect(out.spooledSources).toEqual(['resume_watchdog_timeout'])
723
+ const db = openTurnsDb(agentDir)
724
+ try {
725
+ expect(getTurnByKey(db, turnKey)!.ended_via).toBe('timeout')
726
+ } finally {
727
+ db.close()
728
+ }
729
+ })
730
+
731
+ it('does NOT report a timeout when the agent is alive and only the gateway died', () => {
732
+ // The nastiest false positive: a long tool call means no marker touch for
733
+ // >5 min, so the pre-fix gateway classified a HEALTHY long-running turn as
734
+ // a hang and asked the user whether to retry it.
735
+ const turnKey = seedInFlightTurn()
736
+ writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
737
+ markBootResumeComplete(stateDir)
738
+
739
+ const out = runBootSequence({
740
+ markerTurnKey: turnKey,
741
+ markerAgeMs: 600_000,
742
+ })
743
+
744
+ expect(out.spooledSources).toEqual([])
745
+ const db = openTurnsDb(agentDir)
746
+ try {
747
+ expect(getTurnByKey(db, turnKey)!.ended_via).toBeNull()
748
+ } finally {
749
+ db.close()
750
+ }
751
+ })
752
+ })