switchroom 0.21.7 → 0.21.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/tmp-reaper.sh +234 -0
- package/dist/agent-scheduler/index.js +1 -1
- package/dist/auth-broker/index.js +2 -2
- package/dist/cli/notion-write-pretool.mjs +1 -1
- package/dist/cli/switchroom.js +3421 -2744
- package/dist/host-control/main.js +177 -13
- package/dist/vault/approvals/kernel-server.js +2 -2
- package/dist/vault/broker/server.js +2 -2
- package/package.json +5 -4
- package/profiles/_base/start.sh.hbs +115 -0
- package/profiles/_shared/local-time.md.hbs +6 -0
- package/profiles/default/CLAUDE.md.hbs +0 -12
- package/telegram-plugin/dist/gateway/gateway.js +1017 -465
- package/telegram-plugin/gateway/agent-process-liveness.ts +558 -0
- package/telegram-plugin/gateway/approval-hold.ts +32 -1
- package/telegram-plugin/gateway/approval-outcome-sources.ts +274 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +21 -9
- package/telegram-plugin/gateway/callback-query-handlers.ts +87 -15
- package/telegram-plugin/gateway/eval-case-proposal-inbound-builders.ts +197 -0
- package/telegram-plugin/gateway/gateway.ts +12 -10
- package/telegram-plugin/gateway/pending-inbound-buffer.ts +167 -11
- package/telegram-plugin/gateway/self-improve-proposal-wiring.test.ts +333 -0
- package/telegram-plugin/gateway/self-improve-proposal-wiring.ts +152 -3
- package/telegram-plugin/gateway/subagent-handback-marker.ts +19 -0
- package/telegram-plugin/tests/agent-process-liveness.test.ts +406 -0
- package/telegram-plugin/tests/approval-hold-record.test.ts +21 -8
- package/telegram-plugin/tests/boot-resume-gateway-only-respawn.test.ts +752 -0
- package/telegram-plugin/tests/boot-resume-guard-wiring.test.ts +203 -0
- package/telegram-plugin/tests/callback-query-handlers.test.ts +143 -1
- package/telegram-plugin/tests/eval-case-proposal-inbound-builders.test.ts +144 -0
- package/telegram-plugin/tests/hermes-messages-paging.test.ts +149 -0
- package/telegram-plugin/tests/hermes-session-search.test.ts +146 -0
- package/telegram-plugin/tests/pending-inbound-buffer.test.ts +443 -2
- package/telegram-plugin/tests/subagent-handback-marker.test.ts +14 -0
|
@@ -0,0 +1,752 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* switchroom#4641 — OUTCOME test for the gateway-only-respawn guard.
|
|
3
|
+
*
|
|
4
|
+
* The bug: the gateway and the `claude` agent are separate supervised
|
|
5
|
+
* processes. When the gateway alone crashed and respawned (three Bun SIGBUS
|
|
6
|
+
* crashes on `overlord`, 2026-08-12), its boot block stamped the STILL-RUNNING
|
|
7
|
+
* turn `ended_via='restart'`, spooled a `resume_interrupted` synthetic telling
|
|
8
|
+
* the live session "You just restarted. Your previous turn was interrupted…",
|
|
9
|
+
* and listed its still-running sub-agents as "killed by the restart".
|
|
10
|
+
*
|
|
11
|
+
* What this test asserts is the OUTCOME the issue names, end to end over the
|
|
12
|
+
* REAL collaborators — a real bun:sqlite registry, the real boot reaper, the
|
|
13
|
+
* real interrupted-turn finder, the real inbound builders and the real durable
|
|
14
|
+
* inbound spool — with only the ORDERING supplied by `runBootSequence` below.
|
|
15
|
+
*
|
|
16
|
+
* ORDERING FIDELITY (the point an earlier revision of this file got wrong):
|
|
17
|
+
* gateway.ts's `bootResumeInit` block does NOT spool the resume. It builds
|
|
18
|
+
* `bootResumeInbound` in MEMORY; the durable `inboundSpool.put` lives ~8k lines
|
|
19
|
+
* later, and the generation-token stamp lives immediately after THAT. A
|
|
20
|
+
* harness that puts inside its model of the block collapses the whole window
|
|
21
|
+
* between "block done" and "resume durable" — the one window a crash can
|
|
22
|
+
* actually land in — and so cannot see a stamp placed too early. This harness
|
|
23
|
+
* keeps the three stages distinct and lets `crashAt` land in each of them.
|
|
24
|
+
* `stampAt: 'block-end'` reproduces the defective placement on purpose.
|
|
25
|
+
* (`boot-resume-guard-wiring.test.ts` pins that gateway.ts really does stamp
|
|
26
|
+
* after the put, since this file cannot import the block.)
|
|
27
|
+
*
|
|
28
|
+
* The second fidelity trap, same shape: an in-block failure is NOT a process
|
|
29
|
+
* death. gateway.ts swallows it (`catch { log; turnsDb = null }`) and module
|
|
30
|
+
* init runs on to the stamp. `throwAt` models that swallow — distinct from
|
|
31
|
+
* `crashAt`, which kills the sequence — and it is why the stamp is gated on
|
|
32
|
+
* `bootResumeThrew`, not merely placed late.
|
|
33
|
+
*
|
|
34
|
+
* - gateway killed mid-turn while the claude process LIVES
|
|
35
|
+
* → ZERO `resume_interrupted` entries in the on-disk spool,
|
|
36
|
+
* the in-flight turn row is left OPEN (no `ended_via` lie), and
|
|
37
|
+
* no `.pending-turn.env` is written.
|
|
38
|
+
* - a gateway that CRASHES DURING ITS OWN BOOT, before stamping the
|
|
39
|
+
* per-container-boot generation token — at each of the three reachable
|
|
40
|
+
* points (after the guard, after the block but before the durable spool
|
|
41
|
+
* put, after the put but before the stamp)
|
|
42
|
+
* → the successor gateway does the full boot resume; the interrupted
|
|
43
|
+
* turn is not lost. (This is the case the earlier starttime-ordering
|
|
44
|
+
* guard got wrong: it saw a live claude that predated it and
|
|
45
|
+
* suppressed forever.)
|
|
46
|
+
* - the same crash with the token stamped at the END OF THE BLOCK instead
|
|
47
|
+
* → the resume is DROPPED forever. Asserted explicitly as a defect
|
|
48
|
+
* reproduction, so the correct placement is pinned by outcome.
|
|
49
|
+
* - a suppressed respawn and the #3038 bridge-dead escalation marker
|
|
50
|
+
* → the marker is NOT re-consumed, because the gateway that stamped
|
|
51
|
+
* the token already consumed it this generation.
|
|
52
|
+
* - genuine agent restart (recorded process really is dead)
|
|
53
|
+
* → the resume still fires, exactly as before.
|
|
54
|
+
* - watchdog-timeout classification and first boot (no record file)
|
|
55
|
+
* → unchanged.
|
|
56
|
+
*
|
|
57
|
+
* Every "alive" claim is anchored on a process this test actually spawned, and
|
|
58
|
+
* every "dead" claim on one it actually killed — so a probe hard-wired to
|
|
59
|
+
* either answer fails here. The generation token is never hand-placed for a
|
|
60
|
+
* suppression case without a boot that legitimately stamped it, or an
|
|
61
|
+
* explicitly-stated stale-token scenario.
|
|
62
|
+
*
|
|
63
|
+
* Uses bun:sqlite (openTurnsDb) — excluded from vitest.config.ts, run by
|
|
64
|
+
* telegram-plugin/scripts/bun-test-ci.sh (which runs `tests/`).
|
|
65
|
+
*/
|
|
66
|
+
|
|
67
|
+
import { describe, it, expect, beforeEach, afterEach } from 'bun:test'
|
|
68
|
+
import { spawn, spawnSync } from 'node:child_process'
|
|
69
|
+
import {
|
|
70
|
+
mkdtempSync,
|
|
71
|
+
rmSync,
|
|
72
|
+
existsSync,
|
|
73
|
+
readFileSync,
|
|
74
|
+
writeFileSync,
|
|
75
|
+
appendFileSync,
|
|
76
|
+
renameSync,
|
|
77
|
+
statSync,
|
|
78
|
+
mkdirSync,
|
|
79
|
+
} from 'node:fs'
|
|
80
|
+
import { tmpdir } from 'node:os'
|
|
81
|
+
import { join } from 'node:path'
|
|
82
|
+
|
|
83
|
+
import {
|
|
84
|
+
openTurnsDb,
|
|
85
|
+
recordTurnStart,
|
|
86
|
+
markOrphanedWithTimeoutClassification,
|
|
87
|
+
findLatestTurnIfInterrupted,
|
|
88
|
+
getTurnByKey,
|
|
89
|
+
markTurnResumed,
|
|
90
|
+
} from '../registry/turns-schema.js'
|
|
91
|
+
import {
|
|
92
|
+
decideBootResumeKind,
|
|
93
|
+
buildResumeInterruptedInbound,
|
|
94
|
+
buildResumeWatchdogReportInbound,
|
|
95
|
+
} from '../gateway/resume-inbound-builder.js'
|
|
96
|
+
import { writePendingTurnEnv } from '../gateway/pending-turn-env.js'
|
|
97
|
+
import { createInboundSpool } from '../gateway/inbound-spool.js'
|
|
98
|
+
import type { InboundMessage } from '../gateway/ipc-protocol.js'
|
|
99
|
+
import {
|
|
100
|
+
readProcIdentity,
|
|
101
|
+
shouldSkipBootResumeForGatewayOnlyRespawn,
|
|
102
|
+
markBootResumeComplete,
|
|
103
|
+
bootResumeDonePath,
|
|
104
|
+
AGENT_PROCESS_RECORD_FILE,
|
|
105
|
+
} from '../gateway/agent-process-liveness.js'
|
|
106
|
+
import { consumeBridgeDeadEscalationMarker } from '../gateway/bridge-dead-watchdog.js'
|
|
107
|
+
|
|
108
|
+
// ---------------------------------------------------------------------------
|
|
109
|
+
// Fixture
|
|
110
|
+
// ---------------------------------------------------------------------------
|
|
111
|
+
|
|
112
|
+
let agentDir: string
|
|
113
|
+
let stateDir: string
|
|
114
|
+
/** Long-lived stand-in for the `claude` process. */
|
|
115
|
+
let agentProc: ReturnType<typeof spawn>
|
|
116
|
+
|
|
117
|
+
function sleepMs(ms: number): void {
|
|
118
|
+
spawnSync('sleep', [String(ms / 1000)])
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function starttimeOf(pid: number): string {
|
|
122
|
+
const id = readProcIdentity(pid)
|
|
123
|
+
if (id == null) throw new Error(`pid ${pid} unexpectedly not live`)
|
|
124
|
+
return id.starttime
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Write the record start.sh writes immediately before `exec claude`. */
|
|
128
|
+
function writeAgentProcessRecord(pid: number, starttime: string): void {
|
|
129
|
+
writeFileSync(
|
|
130
|
+
join(stateDir, AGENT_PROCESS_RECORD_FILE),
|
|
131
|
+
JSON.stringify({ pid, starttime, boot_at: Date.now() }),
|
|
132
|
+
)
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const SPOOL_FS = {
|
|
136
|
+
appendFileSync: (p: string, d: string) => appendFileSync(p, d),
|
|
137
|
+
readFileSync: (p: string) => readFileSync(p, 'utf8'),
|
|
138
|
+
writeFileSync: (p: string, d: string) => writeFileSync(p, d),
|
|
139
|
+
renameSync: (a: string, b: string) => renameSync(a, b),
|
|
140
|
+
existsSync: (p: string) => existsSync(p),
|
|
141
|
+
statSizeSync: (p: string) => statSync(p).size,
|
|
142
|
+
fsyncFileSync: () => {},
|
|
143
|
+
fsyncDirSync: () => {},
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
interface BootResult {
|
|
147
|
+
/** Every `source` value that reached the durable on-disk spool. */
|
|
148
|
+
spooledSources: string[]
|
|
149
|
+
/** Raw spool file contents (empty string when never created). */
|
|
150
|
+
spoolRaw: string
|
|
151
|
+
reaped: number
|
|
152
|
+
pendingEnvWritten: boolean
|
|
153
|
+
/** The #3038 bridge-dead marker this boot consumed, if any. */
|
|
154
|
+
bridgeDeadStreak: number | null
|
|
155
|
+
/** Did this boot stamp the per-container-boot generation token? */
|
|
156
|
+
tokenStamped: boolean
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Where a gateway dies during its own boot (the Bun 1.3.13 crash-at-boot
|
|
161
|
+
* pattern that motivated the fix). Each value names the point the process
|
|
162
|
+
* vanishes; everything before it has already happened.
|
|
163
|
+
*
|
|
164
|
+
* 'after-guard' — before any side effect at all.
|
|
165
|
+
* 'after-block-before-put' — the WHOLE bootResumeInit block ran (turn reaped
|
|
166
|
+
* and durably stamped `ended_via='restart'`, env
|
|
167
|
+
* written, inbound built IN MEMORY) but the
|
|
168
|
+
* process dies before the durable spool put. This
|
|
169
|
+
* is the window a too-early token stamp loses
|
|
170
|
+
* work in, and the one the pre-fix harness could
|
|
171
|
+
* not express.
|
|
172
|
+
* 'after-put-before-stamp' — the resume IS durable; only the token is
|
|
173
|
+
* missing. Successor repeats, spool dedups.
|
|
174
|
+
*/
|
|
175
|
+
type CrashPoint = 'after-guard' | 'after-block-before-put' | 'after-put-before-stamp'
|
|
176
|
+
|
|
177
|
+
/** Sentinel used to unwind out of the modelled boot at `crashAt`. */
|
|
178
|
+
const BOOT_CRASH = Symbol('boot-crash')
|
|
179
|
+
/**
|
|
180
|
+
* A throw INSIDE the boot block — swallowed, not fatal. gateway.ts's block
|
|
181
|
+
* ends in `catch (err) { log; turnsDb = null }` and module init CONTINUES, so
|
|
182
|
+
* unlike BOOT_CRASH the later stages still run. Modelling every in-block
|
|
183
|
+
* failure as a process death is what hid the swallow-then-stamp defect.
|
|
184
|
+
*/
|
|
185
|
+
const BLOCK_THROW = Symbol('block-throw')
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* The gateway's boot sequence in gateway.ts's real order, over the real
|
|
189
|
+
* collaborators, in THREE distinct stages:
|
|
190
|
+
*
|
|
191
|
+
* 1. `bootResumeInit` — guard, reaper, marker, builders. In-memory only.
|
|
192
|
+
* 2. the durable `inboundSpool.put` (gateway.ts ~8k lines later).
|
|
193
|
+
* 3. `markBootResumeComplete` — unconditional, top level.
|
|
194
|
+
*
|
|
195
|
+
* `guarded: false` reproduces the pre-#4641 gateway. `stampAt: 'block-end'`
|
|
196
|
+
* reproduces this PR's first (defective) placement.
|
|
197
|
+
*/
|
|
198
|
+
function runBootSequence(opts: {
|
|
199
|
+
guarded?: boolean
|
|
200
|
+
markerTurnKey?: string | null
|
|
201
|
+
markerAgeMs?: number | null
|
|
202
|
+
crashAt?: CrashPoint
|
|
203
|
+
/** Where the generation token is stamped. Default = production. */
|
|
204
|
+
stampAt?: 'block-end' | 'after-durable-put'
|
|
205
|
+
/**
|
|
206
|
+
* Throw inside the block and let gateway.ts's `catch` swallow it. Unlike
|
|
207
|
+
* `crashAt` the process survives, so stages 2 and 3 still run — which is the
|
|
208
|
+
* whole point: the reaper has already durably ended the turn.
|
|
209
|
+
*/
|
|
210
|
+
throwAt?: 'after-reaper'
|
|
211
|
+
/**
|
|
212
|
+
* Production (post-fix) gates the stamp on the block not having thrown
|
|
213
|
+
* (`bootResumeThrew`). Set false to reproduce the ungated defect.
|
|
214
|
+
*/
|
|
215
|
+
gateStampOnBlockThrow?: boolean
|
|
216
|
+
}): BootResult {
|
|
217
|
+
const spoolPath = join(stateDir, 'inbound-spool.jsonl')
|
|
218
|
+
const db = openTurnsDb(agentDir)
|
|
219
|
+
const spool = createInboundSpool({ path: spoolPath, fs: SPOOL_FS, log: () => {} })
|
|
220
|
+
const stampAt = opts.stampAt ?? 'after-durable-put'
|
|
221
|
+
let reaped = 0
|
|
222
|
+
let bridgeDeadStreak: number | null = null
|
|
223
|
+
let tokenStamped = false
|
|
224
|
+
const stamp = (): void => {
|
|
225
|
+
markBootResumeComplete(stateDir)
|
|
226
|
+
tokenStamped = true
|
|
227
|
+
}
|
|
228
|
+
try {
|
|
229
|
+
// ---- stage 1: the bootResumeInit block (IN MEMORY ONLY) ----------------
|
|
230
|
+
// gateway.ts's block-level `catch`: it logs, nulls turnsDb and lets module
|
|
231
|
+
// init CONTINUE. `bootResumeThrew` is the flag that catch sets.
|
|
232
|
+
let bootResumeThrew = false
|
|
233
|
+
// gateway.ts:1648 — `bootResumeInbound`, stashed and pushed to the spool
|
|
234
|
+
// only much later. Nothing on disk yet.
|
|
235
|
+
let bootResumeInbound: InboundMessage | null = null
|
|
236
|
+
try {
|
|
237
|
+
// The guard, gateway.ts: immediately after applySubagentsSchema.
|
|
238
|
+
const skip =
|
|
239
|
+
(opts.guarded ?? true) &&
|
|
240
|
+
shouldSkipBootResumeForGatewayOnlyRespawn(stateDir, { log: () => {} })
|
|
241
|
+
if (!skip) {
|
|
242
|
+
if (opts.crashAt === 'after-guard') throw BOOT_CRASH
|
|
243
|
+
const res = markOrphanedWithTimeoutClassification(db, {
|
|
244
|
+
markerTurnKey: opts.markerTurnKey ?? null,
|
|
245
|
+
markerAgeMs: opts.markerAgeMs ?? null,
|
|
246
|
+
hangThresholdMs: 300_000,
|
|
247
|
+
})
|
|
248
|
+
reaped = res.reaped
|
|
249
|
+
// The turn is NOW durably `ended_via='restart'`. Everything from here to
|
|
250
|
+
// the first `bootResumeInbound =` — findLatestTurnIfInterrupted, the
|
|
251
|
+
// clean-shutdown-marker read, listNonTerminalSubagentsForTurn, the
|
|
252
|
+
// builders — is ~156 lines of DB and fs I/O that can throw.
|
|
253
|
+
if (opts.throwAt === 'after-reaper') throw BLOCK_THROW
|
|
254
|
+
// gateway.ts:~1721 — consumed (always cleared) whether or not a turn was
|
|
255
|
+
// in flight. The `break` skips this too, which is only correct because
|
|
256
|
+
// the token proves a gateway already consumed it this generation.
|
|
257
|
+
const marker = consumeBridgeDeadEscalationMarker(
|
|
258
|
+
join(stateDir, 'bridge-dead-escalation.json'),
|
|
259
|
+
)
|
|
260
|
+
if (marker != null) bridgeDeadStreak = marker.count ?? 1
|
|
261
|
+
const pending = findLatestTurnIfInterrupted(db)
|
|
262
|
+
if (pending != null) {
|
|
263
|
+
const kind = decideBootResumeKind({
|
|
264
|
+
pending,
|
|
265
|
+
suppressed: false,
|
|
266
|
+
ageMs: Math.max(0, Date.now() - pending.started_at),
|
|
267
|
+
maxAgeMs: 10_800_000,
|
|
268
|
+
})
|
|
269
|
+
if (kind === 'resume') {
|
|
270
|
+
bootResumeInbound = buildResumeInterruptedInbound({ turn: pending, subagents: [] })
|
|
271
|
+
} else if (kind === 'report') {
|
|
272
|
+
bootResumeInbound = buildResumeWatchdogReportInbound({
|
|
273
|
+
turn: pending,
|
|
274
|
+
idleMs: 600_000,
|
|
275
|
+
subagents: [],
|
|
276
|
+
})
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
writePendingTurnEnv(agentDir, pending, () => {})
|
|
280
|
+
// THE DEFECT, reproducible on demand: stamping here marks the generation
|
|
281
|
+
// done while the resume exists only in this process's heap.
|
|
282
|
+
if (stampAt === 'block-end') stamp()
|
|
283
|
+
if (opts.crashAt === 'after-block-before-put') throw BOOT_CRASH
|
|
284
|
+
}
|
|
285
|
+
} catch (err) {
|
|
286
|
+
if (err === BOOT_CRASH) throw err // process death: nothing below runs
|
|
287
|
+
if (err !== BLOCK_THROW) throw err
|
|
288
|
+
bootResumeThrew = true // swallowed — module init continues below
|
|
289
|
+
}
|
|
290
|
+
// ---- stage 2: gateway.ts ~:10148, the DURABLE put ----------------------
|
|
291
|
+
if (bootResumeInbound != null) {
|
|
292
|
+
spool.put('tester', bootResumeInbound)
|
|
293
|
+
// …and the at-most-once ledger, stamped only AFTER the put — the same
|
|
294
|
+
// ordering rule the generation token now obeys (turns-schema.ts).
|
|
295
|
+
const rk = bootResumeInbound.meta?.resume_turn_key
|
|
296
|
+
if (typeof rk === 'string' && rk.length > 0) markTurnResumed(db, rk)
|
|
297
|
+
}
|
|
298
|
+
if (opts.crashAt === 'after-put-before-stamp') throw BOOT_CRASH
|
|
299
|
+
// ---- stage 3: the token (gateway.ts, top level) ------------------------
|
|
300
|
+
// Unconditional on there being anything to resume; NOT unconditional on
|
|
301
|
+
// the block having completed.
|
|
302
|
+
const gated = (opts.gateStampOnBlockThrow ?? true) && bootResumeThrew
|
|
303
|
+
if (stampAt === 'after-durable-put' && !gated) stamp()
|
|
304
|
+
} catch (err) {
|
|
305
|
+
if (err !== BOOT_CRASH) throw err
|
|
306
|
+
} finally {
|
|
307
|
+
db.close()
|
|
308
|
+
}
|
|
309
|
+
const spoolRaw = existsSync(spoolPath) ? readFileSync(spoolPath, 'utf8') : ''
|
|
310
|
+
const spooledSources = spoolRaw
|
|
311
|
+
.split('\n')
|
|
312
|
+
.filter((l) => l.trim().length > 0)
|
|
313
|
+
.map((l) => JSON.parse(l) as { t: string; msg?: { meta?: { source?: string } } })
|
|
314
|
+
.filter((r) => r.t === 'put')
|
|
315
|
+
.map((r) => r.msg?.meta?.source ?? '(none)')
|
|
316
|
+
return {
|
|
317
|
+
spooledSources,
|
|
318
|
+
spoolRaw,
|
|
319
|
+
reaped,
|
|
320
|
+
pendingEnvWritten: existsSync(join(agentDir, '.pending-turn.env')),
|
|
321
|
+
bridgeDeadStreak,
|
|
322
|
+
tokenStamped,
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/** Seed an in-flight turn: started, never ended — the live turn's row. */
|
|
327
|
+
function seedInFlightTurn(turnKey = 'chat:1786528508921'): string {
|
|
328
|
+
const db = openTurnsDb(agentDir)
|
|
329
|
+
try {
|
|
330
|
+
recordTurnStart(db, {
|
|
331
|
+
turnKey,
|
|
332
|
+
chatId: '12345',
|
|
333
|
+
userPromptPreview: 'dispatch the worker',
|
|
334
|
+
})
|
|
335
|
+
} finally {
|
|
336
|
+
db.close()
|
|
337
|
+
}
|
|
338
|
+
return turnKey
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
beforeEach(() => {
|
|
342
|
+
agentDir = mkdtempSync(join(tmpdir(), 'boot-respawn-'))
|
|
343
|
+
stateDir = join(agentDir, 'telegram')
|
|
344
|
+
mkdirSync(stateDir, { recursive: true })
|
|
345
|
+
// start.sh's outer pass: a fresh container boot clears the generation token.
|
|
346
|
+
// Every case that wants a gateway-only respawn must first run a boot that
|
|
347
|
+
// stamps it, exactly as production does.
|
|
348
|
+
expect(existsSync(bootResumeDonePath(stateDir))).toBe(false)
|
|
349
|
+
agentProc = spawn('sleep', ['120'], { stdio: 'ignore' })
|
|
350
|
+
sleepMs(20)
|
|
351
|
+
})
|
|
352
|
+
|
|
353
|
+
afterEach(() => {
|
|
354
|
+
try { agentProc.kill('SIGKILL') } catch { /* already gone */ }
|
|
355
|
+
rmSync(agentDir, { recursive: true, force: true })
|
|
356
|
+
})
|
|
357
|
+
|
|
358
|
+
/** Write a fresh #3038 bridge-dead escalation marker. */
|
|
359
|
+
function writeBridgeDeadMarker(count = 2): string {
|
|
360
|
+
const p = join(stateDir, 'bridge-dead-escalation.json')
|
|
361
|
+
writeFileSync(p, JSON.stringify({ ts: Date.now(), reason: 'mcp-bridge-dead', count }))
|
|
362
|
+
return p
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
// ---------------------------------------------------------------------------
|
|
366
|
+
// The bug
|
|
367
|
+
// ---------------------------------------------------------------------------
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
/** A completed first boot of this container generation: stamps the token. */
|
|
371
|
+
function firstBootOfThisContainer(): BootResult {
|
|
372
|
+
return runBootSequence({})
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
describe('gateway-only respawn (claude still running)', () => {
|
|
376
|
+
it('spools ZERO resume_interrupted entries and leaves the live turn open', () => {
|
|
377
|
+
// Full production sequence. Boot #1 of this container: no turn in flight,
|
|
378
|
+
// nothing reaped, generation token stamped. start.sh then publishes the
|
|
379
|
+
// agent record and execs claude, which takes a message…
|
|
380
|
+
const first = firstBootOfThisContainer()
|
|
381
|
+
expect(first.tokenStamped).toBe(true)
|
|
382
|
+
expect(first.spooledSources).toEqual([])
|
|
383
|
+
writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
|
|
384
|
+
const turnKey = seedInFlightTurn()
|
|
385
|
+
|
|
386
|
+
// …and THEN the gateway alone crashes and respawns.
|
|
387
|
+
const out = runBootSequence({})
|
|
388
|
+
|
|
389
|
+
// The outcome the issue names.
|
|
390
|
+
expect(out.spooledSources).toEqual([])
|
|
391
|
+
expect(out.spoolRaw).not.toContain('resume_interrupted')
|
|
392
|
+
// …and no collateral: the live turn keeps its open row (no `restart` lie),
|
|
393
|
+
// and the one-shot wake-audit env is not written by the respawn either.
|
|
394
|
+
expect(out.reaped).toBe(0)
|
|
395
|
+
expect(out.pendingEnvWritten).toBe(false)
|
|
396
|
+
const db = openTurnsDb(agentDir)
|
|
397
|
+
try {
|
|
398
|
+
const turn = getTurnByKey(db, turnKey)!
|
|
399
|
+
expect(turn.ended_at).toBeNull()
|
|
400
|
+
expect(turn.ended_via).toBeNull()
|
|
401
|
+
} finally {
|
|
402
|
+
db.close()
|
|
403
|
+
}
|
|
404
|
+
})
|
|
405
|
+
|
|
406
|
+
it('WOULD fire the false resume without the guard (bug reproduction)', () => {
|
|
407
|
+
const turnKey = seedInFlightTurn()
|
|
408
|
+
writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
|
|
409
|
+
markBootResumeComplete(stateDir)
|
|
410
|
+
|
|
411
|
+
// Same fixture, guard bypassed — the pre-#4641 gateway.
|
|
412
|
+
const out = runBootSequence({ guarded: false })
|
|
413
|
+
|
|
414
|
+
expect(out.spooledSources).toEqual(['resume_interrupted'])
|
|
415
|
+
expect(out.reaped).toBe(1)
|
|
416
|
+
const db = openTurnsDb(agentDir)
|
|
417
|
+
try {
|
|
418
|
+
expect(getTurnByKey(db, turnKey)!.ended_via).toBe('restart')
|
|
419
|
+
} finally {
|
|
420
|
+
db.close()
|
|
421
|
+
}
|
|
422
|
+
})
|
|
423
|
+
|
|
424
|
+
it('stays silent across repeated gateway crashes', () => {
|
|
425
|
+
seedInFlightTurn()
|
|
426
|
+
writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
|
|
427
|
+
markBootResumeComplete(stateDir)
|
|
428
|
+
for (let i = 0; i < 3; i++) {
|
|
429
|
+
expect(runBootSequence({}).spooledSources).toEqual([])
|
|
430
|
+
}
|
|
431
|
+
})
|
|
432
|
+
})
|
|
433
|
+
|
|
434
|
+
// ---------------------------------------------------------------------------
|
|
435
|
+
// A gateway that crashes DURING its own boot (reviewer MAJOR 2)
|
|
436
|
+
// ---------------------------------------------------------------------------
|
|
437
|
+
|
|
438
|
+
describe('gateway crash during its own boot', () => {
|
|
439
|
+
it('still resumes the interrupted turn — the token was never stamped', () => {
|
|
440
|
+
// The container restarts with a genuinely interrupted turn. start.sh forks
|
|
441
|
+
// gateway #1 and (before the exec) publishes a record naming a LIVE agent
|
|
442
|
+
// process that predates gateway #2. Gateway #1 then dies before reaching
|
|
443
|
+
// the boot block, so it never stamps the token.
|
|
444
|
+
//
|
|
445
|
+
// The replacement gateway MUST do the full boot resume. The previous
|
|
446
|
+
// starttime-ordering guard suppressed exactly here — it saw a live claude
|
|
447
|
+
// older than itself — and the interrupted turn was never reaped or
|
|
448
|
+
// resumed at all.
|
|
449
|
+
const turnKey = seedInFlightTurn()
|
|
450
|
+
writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
|
|
451
|
+
|
|
452
|
+
const crashed = runBootSequence({ crashAt: 'after-guard' })
|
|
453
|
+
expect(crashed.tokenStamped).toBe(false)
|
|
454
|
+
expect(crashed.spooledSources).toEqual([])
|
|
455
|
+
expect(existsSync(bootResumeDonePath(stateDir))).toBe(false)
|
|
456
|
+
|
|
457
|
+
const out = runBootSequence({})
|
|
458
|
+
|
|
459
|
+
expect(out.spooledSources).toEqual(['resume_interrupted'])
|
|
460
|
+
expect(out.reaped).toBe(1)
|
|
461
|
+
expect(out.pendingEnvWritten).toBe(true)
|
|
462
|
+
expect(out.tokenStamped).toBe(true)
|
|
463
|
+
const db = openTurnsDb(agentDir)
|
|
464
|
+
try {
|
|
465
|
+
expect(getTurnByKey(db, turnKey)!.ended_via).toBe('restart')
|
|
466
|
+
} finally {
|
|
467
|
+
db.close()
|
|
468
|
+
}
|
|
469
|
+
})
|
|
470
|
+
|
|
471
|
+
it('still consumes the bridge-dead escalation marker', () => {
|
|
472
|
+
// #3038: the marker is consumed (always cleared) whether or not a turn was
|
|
473
|
+
// in flight, and its `count` seeds the cross-boot escalation damper. A
|
|
474
|
+
// gateway that crashed before consuming it must not strand it on disk —
|
|
475
|
+
// the successor's boot block has to reach it.
|
|
476
|
+
const markerPath = writeBridgeDeadMarker(3)
|
|
477
|
+
writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
|
|
478
|
+
|
|
479
|
+
const crashed = runBootSequence({ crashAt: 'after-guard' })
|
|
480
|
+
expect(crashed.bridgeDeadStreak).toBeNull()
|
|
481
|
+
expect(existsSync(markerPath)).toBe(true)
|
|
482
|
+
|
|
483
|
+
const out = runBootSequence({})
|
|
484
|
+
expect(out.bridgeDeadStreak).toBe(3)
|
|
485
|
+
expect(existsSync(markerPath)).toBe(false)
|
|
486
|
+
})
|
|
487
|
+
|
|
488
|
+
it('a crash after the DURABLE PUT but before the stamp does not suppress either', () => {
|
|
489
|
+
// The resume is already on disk; only the token is missing. The successor
|
|
490
|
+
// re-runs the block, and the spool's `s:resume:<turnKey>` dedup makes the
|
|
491
|
+
// repeat a no-op rather than a double-send.
|
|
492
|
+
seedInFlightTurn()
|
|
493
|
+
writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
|
|
494
|
+
|
|
495
|
+
const crashed = runBootSequence({ crashAt: 'after-put-before-stamp' })
|
|
496
|
+
expect(crashed.spooledSources).toEqual(['resume_interrupted'])
|
|
497
|
+
expect(crashed.tokenStamped).toBe(false)
|
|
498
|
+
|
|
499
|
+
const out = runBootSequence({})
|
|
500
|
+
expect(out.tokenStamped).toBe(true) // it did NOT suppress
|
|
501
|
+
// …and exactly ONE resume reached the spool across both boots.
|
|
502
|
+
expect(out.spooledSources).toEqual(['resume_interrupted'])
|
|
503
|
+
})
|
|
504
|
+
|
|
505
|
+
it('a crash AFTER the block but BEFORE the durable put still resumes (the reachable window)', () => {
|
|
506
|
+
// THE regression this placement fix exists for. The block ran to
|
|
507
|
+
// completion: the reaper has DURABLY stamped the turn `ended_via='restart'`
|
|
508
|
+
// and the resume inbound exists only in the dying process's heap. Nothing
|
|
509
|
+
// about the resume is on disk.
|
|
510
|
+
//
|
|
511
|
+
// With the token stamped after the durable put (production), the successor
|
|
512
|
+
// finds no token and re-mints the resume — the work survives.
|
|
513
|
+
const turnKey = seedInFlightTurn()
|
|
514
|
+
// Note: start.sh has NOT yet published agent-process.json at this point —
|
|
515
|
+
// the gateway is forked long before `exec claude`. That is precisely why a
|
|
516
|
+
// premature token is fatal here: the successor's only corroborating
|
|
517
|
+
// evidence is absent, so it returns `gateway-only-respawn-no-record`.
|
|
518
|
+
expect(existsSync(join(stateDir, AGENT_PROCESS_RECORD_FILE))).toBe(false)
|
|
519
|
+
|
|
520
|
+
const crashed = runBootSequence({ crashAt: 'after-block-before-put' })
|
|
521
|
+
expect(crashed.spooledSources).toEqual([]) // nothing durable yet
|
|
522
|
+
expect(crashed.reaped).toBe(1) // …but the turn IS durably stamped
|
|
523
|
+
expect(crashed.tokenStamped).toBe(false)
|
|
524
|
+
expect(existsSync(bootResumeDonePath(stateDir))).toBe(false)
|
|
525
|
+
|
|
526
|
+
const out = runBootSequence({})
|
|
527
|
+
|
|
528
|
+
expect(out.spooledSources).toEqual(['resume_interrupted'])
|
|
529
|
+
expect(out.tokenStamped).toBe(true)
|
|
530
|
+
const db = openTurnsDb(agentDir)
|
|
531
|
+
try {
|
|
532
|
+
expect(getTurnByKey(db, turnKey)!.ended_via).toBe('restart')
|
|
533
|
+
} finally {
|
|
534
|
+
db.close()
|
|
535
|
+
}
|
|
536
|
+
})
|
|
537
|
+
|
|
538
|
+
it('WOULD lose the turn forever if the token were stamped at the end of the block', () => {
|
|
539
|
+
// Defect reproduction for the placement, mirroring the `guarded: false`
|
|
540
|
+
// bug-reproduction above. Same crash point as the test directly above;
|
|
541
|
+
// only the stamp moves. The turn is durably `ended_via='restart'`, no
|
|
542
|
+
// resume ever reaches the spool, and the successor suppresses on the token
|
|
543
|
+
// alone — silent, permanent work loss, and a NEW failure mode versus the
|
|
544
|
+
// pre-#4641 gateway, which simply re-ran the block.
|
|
545
|
+
const turnKey = seedInFlightTurn()
|
|
546
|
+
|
|
547
|
+
const crashed = runBootSequence({ crashAt: 'after-block-before-put', stampAt: 'block-end' })
|
|
548
|
+
expect(crashed.tokenStamped).toBe(true) // stamped while the resume is in RAM
|
|
549
|
+
expect(crashed.spooledSources).toEqual([])
|
|
550
|
+
expect(existsSync(bootResumeDonePath(stateDir))).toBe(true)
|
|
551
|
+
|
|
552
|
+
const out = runBootSequence({})
|
|
553
|
+
|
|
554
|
+
expect(out.spooledSources, 'the resume is dropped, not retried').toEqual([])
|
|
555
|
+
expect(out.reaped).toBe(0)
|
|
556
|
+
const db = openTurnsDb(agentDir)
|
|
557
|
+
try {
|
|
558
|
+
const turn = getTurnByKey(db, turnKey)!
|
|
559
|
+
expect(turn.ended_via).toBe('restart') // durably ended…
|
|
560
|
+
expect(turn.resumed_at).toBeNull() // …and never resumed
|
|
561
|
+
} finally {
|
|
562
|
+
db.close()
|
|
563
|
+
}
|
|
564
|
+
})
|
|
565
|
+
|
|
566
|
+
it('a SWALLOWED throw inside the block leaves no token, so the successor still resumes', () => {
|
|
567
|
+
// gateway.ts's block ends in `catch { log; turnsDb = null }` — a swallow,
|
|
568
|
+
// not a death: module init continues and reaches the stamp with
|
|
569
|
+
// `bootResumeInbound` still null. By then the reaper has ALREADY durably
|
|
570
|
+
// written `ended_via='restart'`. Gating the stamp on that catch is what
|
|
571
|
+
// keeps the work recoverable.
|
|
572
|
+
const turnKey = seedInFlightTurn()
|
|
573
|
+
|
|
574
|
+
const threw = runBootSequence({ throwAt: 'after-reaper' })
|
|
575
|
+
expect(threw.reaped, 'the reaper ran and durably ended the turn').toBe(1)
|
|
576
|
+
expect(threw.spooledSources, 'nothing was spooled — the block never built it').toEqual([])
|
|
577
|
+
expect(threw.tokenStamped, 'a swallowed throw must not stamp the generation').toBe(false)
|
|
578
|
+
expect(existsSync(bootResumeDonePath(stateDir))).toBe(false)
|
|
579
|
+
|
|
580
|
+
// The gateway respawns (the Bun-crash pattern #4641 exists for).
|
|
581
|
+
const out = runBootSequence({})
|
|
582
|
+
expect(out.spooledSources, 'the successor re-mints the resume').toEqual(['resume_interrupted'])
|
|
583
|
+
const db = openTurnsDb(agentDir)
|
|
584
|
+
try {
|
|
585
|
+
expect(getTurnByKey(db, turnKey)!.resumed_at, 'and stamps it resumed').not.toBeNull()
|
|
586
|
+
} finally {
|
|
587
|
+
db.close()
|
|
588
|
+
}
|
|
589
|
+
})
|
|
590
|
+
|
|
591
|
+
it('WOULD lose the turn forever if a swallowed throw still stamped the token', () => {
|
|
592
|
+
// Defect reproduction for the ungated stamp: same swallow, stamp not gated
|
|
593
|
+
// on `bootResumeThrew`. The turn is durably ended, nothing is spooled, and
|
|
594
|
+
// the successor suppresses on the token alone.
|
|
595
|
+
const turnKey = seedInFlightTurn()
|
|
596
|
+
|
|
597
|
+
const threw = runBootSequence({ throwAt: 'after-reaper', gateStampOnBlockThrow: false })
|
|
598
|
+
expect(threw.reaped).toBe(1)
|
|
599
|
+
expect(threw.spooledSources).toEqual([])
|
|
600
|
+
expect(threw.tokenStamped, 'stamped with nothing spooled anywhere').toBe(true)
|
|
601
|
+
|
|
602
|
+
const out = runBootSequence({})
|
|
603
|
+
expect(out.spooledSources, 'the resume is dropped, not retried').toEqual([])
|
|
604
|
+
expect(out.reaped).toBe(0)
|
|
605
|
+
const db = openTurnsDb(agentDir)
|
|
606
|
+
try {
|
|
607
|
+
const turn = getTurnByKey(db, turnKey)!
|
|
608
|
+
expect(turn.ended_via).toBe('restart') // durably ended…
|
|
609
|
+
expect(turn.resumed_at).toBeNull() // …and never resumed
|
|
610
|
+
} finally {
|
|
611
|
+
db.close()
|
|
612
|
+
}
|
|
613
|
+
})
|
|
614
|
+
})
|
|
615
|
+
|
|
616
|
+
// ---------------------------------------------------------------------------
|
|
617
|
+
// What the `break` skips, beyond the resume synthetic (reviewer MEDIUM)
|
|
618
|
+
// ---------------------------------------------------------------------------
|
|
619
|
+
|
|
620
|
+
describe('a suppressed respawn and the bridge-dead escalation marker', () => {
|
|
621
|
+
it('leaves an already-consumed generation alone: no marker, no streak reset', () => {
|
|
622
|
+
// Boot #1 of this container consumes the marker (count=3 → the damper) and
|
|
623
|
+
// stamps the token. The respawn must NOT re-run that consumption: there is
|
|
624
|
+
// nothing left to consume, and a fresh marker written later in the SAME
|
|
625
|
+
// generation belongs to the live watchdog, not to a boot classification.
|
|
626
|
+
writeBridgeDeadMarker(3)
|
|
627
|
+
writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
|
|
628
|
+
const first = runBootSequence({})
|
|
629
|
+
expect(first.bridgeDeadStreak).toBe(3)
|
|
630
|
+
expect(first.tokenStamped).toBe(true)
|
|
631
|
+
|
|
632
|
+
const markerPath = writeBridgeDeadMarker(5)
|
|
633
|
+
const out = runBootSequence({})
|
|
634
|
+
|
|
635
|
+
expect(out.bridgeDeadStreak).toBeNull() // the break skipped the consumption
|
|
636
|
+
expect(existsSync(markerPath)).toBe(true) // …so the marker survives on disk
|
|
637
|
+
})
|
|
638
|
+
})
|
|
639
|
+
|
|
640
|
+
// ---------------------------------------------------------------------------
|
|
641
|
+
// The cases that must NOT change
|
|
642
|
+
// ---------------------------------------------------------------------------
|
|
643
|
+
|
|
644
|
+
describe('genuine agent restart', () => {
|
|
645
|
+
it('still fires resume_interrupted when the recorded process is really dead', () => {
|
|
646
|
+
const turnKey = seedInFlightTurn()
|
|
647
|
+
const doomed = spawn('sleep', ['120'], { stdio: 'ignore' })
|
|
648
|
+
sleepMs(20)
|
|
649
|
+
writeAgentProcessRecord(doomed.pid!, starttimeOf(doomed.pid!))
|
|
650
|
+
doomed.kill('SIGKILL')
|
|
651
|
+
sleepMs(200)
|
|
652
|
+
// Even with a token from the previous generation left behind (a stale
|
|
653
|
+
// volume, an unclean boot), a dead agent VETOES the suppression.
|
|
654
|
+
markBootResumeComplete(stateDir)
|
|
655
|
+
|
|
656
|
+
const out = runBootSequence({})
|
|
657
|
+
|
|
658
|
+
expect(out.spooledSources).toEqual(['resume_interrupted'])
|
|
659
|
+
expect(out.spoolRaw).toContain('resume_interrupted')
|
|
660
|
+
expect(out.reaped).toBe(1)
|
|
661
|
+
expect(out.pendingEnvWritten).toBe(true)
|
|
662
|
+
const db = openTurnsDb(agentDir)
|
|
663
|
+
try {
|
|
664
|
+
expect(getTurnByKey(db, turnKey)!.ended_via).toBe('restart')
|
|
665
|
+
} finally {
|
|
666
|
+
db.close()
|
|
667
|
+
}
|
|
668
|
+
})
|
|
669
|
+
|
|
670
|
+
it('still fires on a fresh container boot (token cleared by start.sh)', () => {
|
|
671
|
+
seedInFlightTurn()
|
|
672
|
+
// The fresh-boot shape the ordering arm used to guard: start.sh has
|
|
673
|
+
// already published a record for a LIVE agent process. Without the token
|
|
674
|
+
// there is no suppression, whatever /proc says.
|
|
675
|
+
writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
|
|
676
|
+
expect(existsSync(bootResumeDonePath(stateDir))).toBe(false)
|
|
677
|
+
|
|
678
|
+
const out = runBootSequence({})
|
|
679
|
+
|
|
680
|
+
expect(out.spooledSources).toEqual(['resume_interrupted'])
|
|
681
|
+
expect(out.pendingEnvWritten).toBe(true)
|
|
682
|
+
})
|
|
683
|
+
|
|
684
|
+
it('still fires on first boot, when no process record exists at all', () => {
|
|
685
|
+
seedInFlightTurn()
|
|
686
|
+
expect(existsSync(join(stateDir, AGENT_PROCESS_RECORD_FILE))).toBe(false)
|
|
687
|
+
|
|
688
|
+
const out = runBootSequence({})
|
|
689
|
+
|
|
690
|
+
expect(out.spooledSources).toEqual(['resume_interrupted'])
|
|
691
|
+
expect(out.pendingEnvWritten).toBe(true)
|
|
692
|
+
})
|
|
693
|
+
|
|
694
|
+
it('still fires when a recycled PID matches but the starttime does not', () => {
|
|
695
|
+
seedInFlightTurn()
|
|
696
|
+
// A live pid whose recorded starttime disagrees — the post-container-
|
|
697
|
+
// restart shape, where the old record's pid names a different process.
|
|
698
|
+
writeAgentProcessRecord(
|
|
699
|
+
agentProc.pid!,
|
|
700
|
+
String(BigInt(starttimeOf(agentProc.pid!)) - 1n),
|
|
701
|
+
)
|
|
702
|
+
markBootResumeComplete(stateDir)
|
|
703
|
+
|
|
704
|
+
expect(runBootSequence({}).spooledSources).toEqual(['resume_interrupted'])
|
|
705
|
+
})
|
|
706
|
+
})
|
|
707
|
+
|
|
708
|
+
describe('watchdog-timeout classification', () => {
|
|
709
|
+
it('still reports resume_watchdog_timeout for a dead agent that stalled', () => {
|
|
710
|
+
const turnKey = seedInFlightTurn()
|
|
711
|
+
const doomed = spawn('sleep', ['120'], { stdio: 'ignore' })
|
|
712
|
+
sleepMs(20)
|
|
713
|
+
writeAgentProcessRecord(doomed.pid!, starttimeOf(doomed.pid!))
|
|
714
|
+
doomed.kill('SIGKILL')
|
|
715
|
+
sleepMs(200)
|
|
716
|
+
|
|
717
|
+
const out = runBootSequence({
|
|
718
|
+
markerTurnKey: turnKey,
|
|
719
|
+
markerAgeMs: 600_000, // past the 300s hang threshold
|
|
720
|
+
})
|
|
721
|
+
|
|
722
|
+
expect(out.spooledSources).toEqual(['resume_watchdog_timeout'])
|
|
723
|
+
const db = openTurnsDb(agentDir)
|
|
724
|
+
try {
|
|
725
|
+
expect(getTurnByKey(db, turnKey)!.ended_via).toBe('timeout')
|
|
726
|
+
} finally {
|
|
727
|
+
db.close()
|
|
728
|
+
}
|
|
729
|
+
})
|
|
730
|
+
|
|
731
|
+
it('does NOT report a timeout when the agent is alive and only the gateway died', () => {
|
|
732
|
+
// The nastiest false positive: a long tool call means no marker touch for
|
|
733
|
+
// >5 min, so the pre-fix gateway classified a HEALTHY long-running turn as
|
|
734
|
+
// a hang and asked the user whether to retry it.
|
|
735
|
+
const turnKey = seedInFlightTurn()
|
|
736
|
+
writeAgentProcessRecord(agentProc.pid!, starttimeOf(agentProc.pid!))
|
|
737
|
+
markBootResumeComplete(stateDir)
|
|
738
|
+
|
|
739
|
+
const out = runBootSequence({
|
|
740
|
+
markerTurnKey: turnKey,
|
|
741
|
+
markerAgeMs: 600_000,
|
|
742
|
+
})
|
|
743
|
+
|
|
744
|
+
expect(out.spooledSources).toEqual([])
|
|
745
|
+
const db = openTurnsDb(agentDir)
|
|
746
|
+
try {
|
|
747
|
+
expect(getTurnByKey(db, turnKey)!.ended_via).toBeNull()
|
|
748
|
+
} finally {
|
|
749
|
+
db.close()
|
|
750
|
+
}
|
|
751
|
+
})
|
|
752
|
+
})
|