switchroom 0.18.8 → 0.18.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/agent-scheduler/index.js +1 -0
- package/dist/auth-broker/index.js +198 -13
- package/dist/cli/notion-write-pretool.mjs +1 -0
- package/dist/cli/switchroom.js +28 -4
- package/dist/host-control/main.js +3 -2
- package/dist/vault/approvals/kernel-server.js +2 -1
- package/dist/vault/broker/server.js +2 -1
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +119 -37
- package/profiles/_shared/dev-protocol.md.hbs +42 -0
- package/skills/dev-protocol/SKILL.md +131 -0
- package/telegram-plugin/README.md +2 -1
- package/telegram-plugin/admin-commands/dispatch.test.ts +40 -2
- package/telegram-plugin/admin-commands/index.ts +6 -1
- package/telegram-plugin/bridge/bridge.ts +23 -1
- package/telegram-plugin/bridge/crash-breadcrumb.ts +42 -0
- package/telegram-plugin/chat-lock.ts +13 -0
- package/telegram-plugin/dist/bridge/bridge.js +24 -1
- package/telegram-plugin/dist/gateway/gateway.js +80236 -77465
- package/telegram-plugin/dist/server.js +29 -2
- package/telegram-plugin/fallback-card-collapse.ts +131 -0
- package/telegram-plugin/gateway/approval-card-stores.ts +99 -0
- package/telegram-plugin/gateway/bot-commands-ops-info.ts +194 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +546 -0
- package/telegram-plugin/gateway/callback-query-handlers.ts +2660 -0
- package/telegram-plugin/gateway/effort-command.ts +47 -3
- package/telegram-plugin/gateway/gateway.ts +2051 -3180
- package/telegram-plugin/gateway/inbound-delivery-machine-dispatch.ts +181 -23
- package/telegram-plugin/gateway/inbound-delivery-machine.ts +8 -0
- package/telegram-plugin/gateway/model-command.ts +94 -8
- package/telegram-plugin/gateway/outbound-send-path.ts +375 -0
- package/telegram-plugin/gateway/pending-session-command.ts +365 -0
- package/telegram-plugin/gateway/pending-state-stores.ts +106 -0
- package/telegram-plugin/gateway/permission-timeout.ts +25 -0
- package/telegram-plugin/gateway/register-bot-commands.ts +30 -0
- package/telegram-plugin/gateway/resume-inbound-builder.ts +23 -3
- package/telegram-plugin/gateway/session-model-file.ts +166 -23
- package/telegram-plugin/gateway/stop-command.ts +56 -0
- package/telegram-plugin/photo-precheck.ts +201 -0
- package/telegram-plugin/quota-watch.ts +141 -2
- package/telegram-plugin/registry/subagents-schema.ts +26 -3
- package/telegram-plugin/registry/subagents.test.ts +67 -0
- package/telegram-plugin/retry-api-call.ts +31 -0
- package/telegram-plugin/subagent-watcher.ts +392 -1
- package/telegram-plugin/tests/approval-card-stores.test.ts +124 -0
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +576 -0
- package/telegram-plugin/tests/buffer-gate-broadened.test.ts +11 -5
- package/telegram-plugin/tests/callback-query-handlers.test.ts +701 -0
- package/telegram-plugin/tests/chat-lock-unhandled-rejection.test.ts +101 -0
- package/telegram-plugin/tests/crash-breadcrumb.test.ts +57 -0
- package/telegram-plugin/tests/effort-command.test.ts +59 -2
- package/telegram-plugin/tests/emission-determinism-wiring.test.ts +11 -4
- package/telegram-plugin/tests/fallback-card-collapse.test.ts +104 -0
- package/telegram-plugin/tests/fixtures/cutover-killswitch-probe.ts +75 -0
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +5 -1
- package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +124 -0
- package/telegram-plugin/tests/gateway-secret-detect.test.ts +7 -1
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +19 -11
- package/telegram-plugin/tests/inbound-delivery-cutover-flip.test.ts +418 -0
- package/telegram-plugin/tests/inbound-delivery-dispatch-equivalence.test.ts +348 -0
- package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +141 -52
- package/telegram-plugin/tests/mental-model-propose-callback-gate.test.ts +8 -1
- package/telegram-plugin/tests/model-command.test.ts +46 -3
- package/telegram-plugin/tests/outbound-send-chunks.test.ts +304 -0
- package/telegram-plugin/tests/outbound-send-path.test.ts +222 -0
- package/telegram-plugin/tests/pending-card-durability-wiring.test.ts +34 -15
- package/telegram-plugin/tests/pending-session-command.test.ts +322 -0
- package/telegram-plugin/tests/pending-state-stores.test.ts +235 -0
- package/telegram-plugin/tests/permission-timeout.test.ts +26 -0
- package/telegram-plugin/tests/permission-verdict-resume-guard.test.ts +16 -0
- package/telegram-plugin/tests/photo-dimension-fallback.test.ts +129 -0
- package/telegram-plugin/tests/photo-precheck.test.ts +240 -0
- package/telegram-plugin/tests/photo-reroute-wiring.test.ts +85 -0
- package/telegram-plugin/tests/quota-watch.test.ts +225 -0
- package/telegram-plugin/tests/session-model-file.test.ts +101 -2
- package/telegram-plugin/tests/stop-command.test.ts +234 -0
- package/telegram-plugin/tests/subagent-watcher-env-thresholds.test.ts +27 -9
- package/telegram-plugin/tests/subagent-watcher-resurrection.test.ts +398 -0
- package/telegram-plugin/tests/subagent-watcher-stall-terminal.test.ts +172 -0
- package/telegram-plugin/tests/turn-flush-safety.test.ts +18 -4
- package/telegram-plugin/tests/vault-approval-posture.test.ts +15 -7
- package/telegram-plugin/tests/vault-grant-auto-resume.test.ts +8 -4
- package/telegram-plugin/tests/vault-grant-union.test.ts +8 -4
- package/telegram-plugin/tests/vault-grant-wizard.test.ts +8 -1
- package/telegram-plugin/tests/vault-grants-revoke.test.ts +8 -1
- package/telegram-plugin/tests/vault-key-regex-allows-slash.test.ts +8 -4
- package/telegram-plugin/tests/vault-request-access-tool.test.ts +8 -4
- package/telegram-plugin/tests/vault-request-access-unlock-resume.test.ts +8 -4
- package/telegram-plugin/tests/worker-activity-feed.test.ts +37 -0
- package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +18 -4
- package/telegram-plugin/welcome-text.ts +4 -3
- package/telegram-plugin/worker-activity-feed.ts +27 -0
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for SubagentWatcher card resurrection (issue #3023).
|
|
3
|
+
*
|
|
4
|
+
* PR #3019 makes false terminal synthesis far less likely (in-flight tool
|
|
5
|
+
* gate + 45min cap), but can't eliminate it. This suite locks the second
|
|
6
|
+
* half of the operator invariant — "active work must always be visible":
|
|
7
|
+
*
|
|
8
|
+
* (a) When a worker's card is FALSELY finalised (silent-stall synthesis
|
|
9
|
+
* fires) and its JSONL later resumes growing, the watcher resurrects it
|
|
10
|
+
* exactly once — fires `onResurrect`, re-registers it LIVE, emits a
|
|
11
|
+
* resurrection log line. A further growth tick does NOT re-fire.
|
|
12
|
+
* (b) A worker resurrected once and then falsely finalised AGAIN is
|
|
13
|
+
* named-as-lost (`onWorkerLost`), NOT resurrected a second time — the
|
|
14
|
+
* resurrection chain is bounded.
|
|
15
|
+
* (c) A genuine boot-time historical rediscovery of an old completed worker
|
|
16
|
+
* stays historical/suppressed — it never records a false finish, so it
|
|
17
|
+
* can never be resurrected even if its file were to grow.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { describe, it, expect, vi } from 'vitest'
|
|
21
|
+
import { startSubagentWatcher } from '../subagent-watcher.js'
|
|
22
|
+
import * as fs from 'fs'
|
|
23
|
+
|
|
24
|
+
function buildJSONL(...lines: object[]): string {
|
|
25
|
+
return lines.map((l) => JSON.stringify(l)).join('\n') + '\n'
|
|
26
|
+
}
|
|
27
|
+
function subAgentUserMsg(promptText: string) {
|
|
28
|
+
return { type: 'user', message: { content: [{ type: 'text', text: promptText }] } }
|
|
29
|
+
}
|
|
30
|
+
function subAgentTurnEnd() {
|
|
31
|
+
return { type: 'system', subtype: 'turn_duration', duration_ms: 1234 }
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
interface Harness {
|
|
35
|
+
resurrectCalls: Array<{ agentId: string; description: string }>
|
|
36
|
+
lostCalls: Array<{ agentId: string; description: string }>
|
|
37
|
+
stallTerminalCalls: Array<{ agentId: string }>
|
|
38
|
+
finishCalls: Array<{ agentId: string; outcome: string }>
|
|
39
|
+
logs: string[]
|
|
40
|
+
advance: (ms: number) => void
|
|
41
|
+
watcher: ReturnType<typeof startSubagentWatcher>
|
|
42
|
+
fileContents: Map<string, Buffer>
|
|
43
|
+
jsonlPath: string
|
|
44
|
+
appendActivity: () => void
|
|
45
|
+
appendTurnEnd: () => void
|
|
46
|
+
presentAtBoot: (content: string) => void
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function makeHarness(opts: {
|
|
50
|
+
agentId?: string
|
|
51
|
+
stallThresholdMs?: number
|
|
52
|
+
silentStallTerminalMs?: number
|
|
53
|
+
rescanMs?: number
|
|
54
|
+
bootContent?: string
|
|
55
|
+
} = {}): Harness {
|
|
56
|
+
const {
|
|
57
|
+
agentId = 'resurrect-agent',
|
|
58
|
+
stallThresholdMs = 60_000,
|
|
59
|
+
silentStallTerminalMs = 300_000,
|
|
60
|
+
rescanMs = 500,
|
|
61
|
+
bootContent,
|
|
62
|
+
} = opts
|
|
63
|
+
|
|
64
|
+
let currentTime = 1000
|
|
65
|
+
const resurrectCalls: Array<{ agentId: string; description: string }> = []
|
|
66
|
+
const lostCalls: Array<{ agentId: string; description: string }> = []
|
|
67
|
+
const stallTerminalCalls: Array<{ agentId: string }> = []
|
|
68
|
+
const finishCalls: Array<{ agentId: string; outcome: string }> = []
|
|
69
|
+
const logs: string[] = []
|
|
70
|
+
|
|
71
|
+
const agentDir = '/home/user/.switchroom/agents/myagent'
|
|
72
|
+
const sessionId = 'mock-session'
|
|
73
|
+
const projectsRoot = `${agentDir}/.claude/projects`
|
|
74
|
+
const projectDir = `${projectsRoot}/mock-cwd`
|
|
75
|
+
const sessionDir = `${projectDir}/${sessionId}`
|
|
76
|
+
const subagentsDir = `${sessionDir}/subagents`
|
|
77
|
+
const jsonlPath = `${subagentsDir}/agent-${agentId}.jsonl`
|
|
78
|
+
|
|
79
|
+
const fileContents = new Map<string, Buffer>()
|
|
80
|
+
fileContents.set(
|
|
81
|
+
jsonlPath,
|
|
82
|
+
Buffer.from(bootContent ?? buildJSONL(subAgentUserMsg('bg task')), 'utf-8'),
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
let lastOpenedPath: string | null = null
|
|
86
|
+
const mockFs = {
|
|
87
|
+
existsSync: ((p: fs.PathLike) => {
|
|
88
|
+
const ps = String(p)
|
|
89
|
+
if (ps === projectsRoot || ps === projectDir || ps === sessionDir || ps === subagentsDir) return true
|
|
90
|
+
if (fileContents.has(ps)) return true
|
|
91
|
+
return false
|
|
92
|
+
}) as typeof fs.existsSync,
|
|
93
|
+
readdirSync: ((p: fs.PathLike) => {
|
|
94
|
+
const ps = String(p)
|
|
95
|
+
if (ps === projectsRoot) return ['mock-cwd']
|
|
96
|
+
if (ps === projectDir) return [sessionId]
|
|
97
|
+
if (ps === sessionDir) return ['subagents']
|
|
98
|
+
if (ps === subagentsDir) return [`agent-${agentId}.jsonl`]
|
|
99
|
+
return []
|
|
100
|
+
}) as unknown as typeof fs.readdirSync,
|
|
101
|
+
statSync: ((p: fs.PathLike) =>
|
|
102
|
+
({ size: fileContents.get(String(p))?.length ?? 0, mtimeMs: currentTime }) as fs.Stats) as typeof fs.statSync,
|
|
103
|
+
openSync: ((p: fs.PathLike) => {
|
|
104
|
+
lastOpenedPath = String(p)
|
|
105
|
+
return 42
|
|
106
|
+
}) as unknown as typeof fs.openSync,
|
|
107
|
+
closeSync: (() => { lastOpenedPath = null }) as typeof fs.closeSync,
|
|
108
|
+
readSync: ((
|
|
109
|
+
_fd: number,
|
|
110
|
+
buf: NodeJS.ArrayBufferView,
|
|
111
|
+
offset: number,
|
|
112
|
+
length: number,
|
|
113
|
+
position: number | null,
|
|
114
|
+
): number => {
|
|
115
|
+
const content = lastOpenedPath != null ? fileContents.get(lastOpenedPath) : undefined
|
|
116
|
+
if (!content) return 0
|
|
117
|
+
const pos = position ?? 0
|
|
118
|
+
const src = content.slice(pos, pos + length)
|
|
119
|
+
;(src as Buffer).copy(buf as Buffer, offset)
|
|
120
|
+
return src.length
|
|
121
|
+
}) as unknown as typeof fs.readSync,
|
|
122
|
+
watch: (() => ({ close: vi.fn() }) as unknown as fs.FSWatcher) as unknown as typeof fs.watch,
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const intervals: Array<{ fn: () => void; ms: number; ref: number; fireAt: number }> = []
|
|
126
|
+
const timeouts: Array<{ fn: () => void; ref: number; fireAt: number }> = []
|
|
127
|
+
let nextRef = 1
|
|
128
|
+
|
|
129
|
+
const watcher = startSubagentWatcher({
|
|
130
|
+
agentDir,
|
|
131
|
+
stallThresholdMs,
|
|
132
|
+
silentSynthesisStallThresholdMs: stallThresholdMs,
|
|
133
|
+
silentStallTerminalMs,
|
|
134
|
+
rescanMs,
|
|
135
|
+
onStallTerminal: (id) => stallTerminalCalls.push({ agentId: id }),
|
|
136
|
+
onResurrect: (id, desc) => resurrectCalls.push({ agentId: id, description: desc }),
|
|
137
|
+
onWorkerLost: (id, desc) => lostCalls.push({ agentId: id, description: desc }),
|
|
138
|
+
onFinish: ({ agentId: id, outcome }) => finishCalls.push({ agentId: id, outcome }),
|
|
139
|
+
now: () => currentTime,
|
|
140
|
+
setInterval: (fn, ms) => {
|
|
141
|
+
const ref = nextRef++
|
|
142
|
+
intervals.push({ fn, ms, ref, fireAt: currentTime + ms })
|
|
143
|
+
return { ref }
|
|
144
|
+
},
|
|
145
|
+
clearInterval: (handle) => {
|
|
146
|
+
const { ref } = handle as { ref: number }
|
|
147
|
+
const idx = intervals.findIndex((i) => i.ref === ref)
|
|
148
|
+
if (idx !== -1) intervals.splice(idx, 1)
|
|
149
|
+
},
|
|
150
|
+
setTimeout: (fn, ms) => {
|
|
151
|
+
const ref = nextRef++
|
|
152
|
+
timeouts.push({ fn, ref, fireAt: currentTime + ms })
|
|
153
|
+
return { ref }
|
|
154
|
+
},
|
|
155
|
+
clearTimeout: (handle) => {
|
|
156
|
+
const { ref } = handle as { ref: number }
|
|
157
|
+
const idx = timeouts.findIndex((t) => t.ref === ref)
|
|
158
|
+
if (idx !== -1) timeouts.splice(idx, 1)
|
|
159
|
+
},
|
|
160
|
+
fs: mockFs,
|
|
161
|
+
log: (msg) => logs.push(msg),
|
|
162
|
+
})
|
|
163
|
+
|
|
164
|
+
const advance = (ms: number): void => {
|
|
165
|
+
currentTime += ms
|
|
166
|
+
for (;;) {
|
|
167
|
+
intervals.sort((a, b) => a.fireAt - b.fireAt)
|
|
168
|
+
timeouts.sort((a, b) => a.fireAt - b.fireAt)
|
|
169
|
+
const nextI = intervals[0]
|
|
170
|
+
const nextT = timeouts[0]
|
|
171
|
+
const iReady = nextI && nextI.fireAt <= currentTime
|
|
172
|
+
const tReady = nextT && nextT.fireAt <= currentTime
|
|
173
|
+
if (!iReady && !tReady) break
|
|
174
|
+
if (tReady && (!iReady || nextT!.fireAt <= nextI!.fireAt)) {
|
|
175
|
+
timeouts.shift()
|
|
176
|
+
nextT!.fn()
|
|
177
|
+
} else {
|
|
178
|
+
nextI!.fireAt += nextI!.ms
|
|
179
|
+
nextI!.fn()
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
const appendActivity = (): void => {
|
|
185
|
+
const cur = fileContents.get(jsonlPath) ?? Buffer.alloc(0)
|
|
186
|
+
const more = buildJSONL({
|
|
187
|
+
type: 'assistant',
|
|
188
|
+
message: { content: [{ type: 'text', text: 'still working' }] },
|
|
189
|
+
})
|
|
190
|
+
fileContents.set(jsonlPath, Buffer.concat([cur, Buffer.from(more, 'utf-8')]))
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
const appendTurnEnd = (): void => {
|
|
194
|
+
const cur = fileContents.get(jsonlPath) ?? Buffer.alloc(0)
|
|
195
|
+
const more = buildJSONL(subAgentTurnEnd())
|
|
196
|
+
fileContents.set(jsonlPath, Buffer.concat([cur, Buffer.from(more, 'utf-8')]))
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
const presentAtBoot = (content: string): void => {
|
|
200
|
+
fileContents.set(jsonlPath, Buffer.from(content, 'utf-8'))
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
return {
|
|
204
|
+
resurrectCalls,
|
|
205
|
+
lostCalls,
|
|
206
|
+
stallTerminalCalls,
|
|
207
|
+
finishCalls,
|
|
208
|
+
logs,
|
|
209
|
+
advance,
|
|
210
|
+
watcher,
|
|
211
|
+
fileContents,
|
|
212
|
+
jsonlPath,
|
|
213
|
+
appendActivity,
|
|
214
|
+
appendTurnEnd,
|
|
215
|
+
presentAtBoot,
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function unmarkHistorical(harness: Harness, agentId: string): void {
|
|
220
|
+
const entry = harness.watcher.getRegistry().get(agentId)
|
|
221
|
+
if (entry) entry.historical = false
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** Drive a running worker through stall → silent-stall terminal synthesis. */
|
|
225
|
+
function driveToFalseFinish(h: Harness, agentId: string): void {
|
|
226
|
+
h.advance(62_000) // stall notification fires
|
|
227
|
+
h.advance(302_000) // past silentStallTerminalMs — synthesis fires
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
describe('subagent-watcher card resurrection (issue #3023)', () => {
|
|
231
|
+
it('(a) resurrects a falsely-finalised worker exactly once when its JSONL resumes', () => {
|
|
232
|
+
const agentId = 'resurrect-once'
|
|
233
|
+
const h = makeHarness({ agentId })
|
|
234
|
+
|
|
235
|
+
h.advance(500) // register
|
|
236
|
+
unmarkHistorical(h, agentId)
|
|
237
|
+
|
|
238
|
+
driveToFalseFinish(h, agentId)
|
|
239
|
+
expect(h.stallTerminalCalls).toHaveLength(1)
|
|
240
|
+
expect(h.finishCalls).toHaveLength(1)
|
|
241
|
+
expect(h.resurrectCalls).toHaveLength(0)
|
|
242
|
+
|
|
243
|
+
// The worker was NOT actually dead — its JSONL resumes growing.
|
|
244
|
+
h.appendActivity()
|
|
245
|
+
h.advance(500) // poll → checkResurrections detects the growth
|
|
246
|
+
|
|
247
|
+
expect(h.resurrectCalls).toHaveLength(1)
|
|
248
|
+
expect(h.resurrectCalls[0].agentId).toBe(agentId)
|
|
249
|
+
expect(h.lostCalls).toHaveLength(0)
|
|
250
|
+
expect(h.logs.some((l) => l.includes('RESURRECTING worker') && l.includes(agentId))).toBe(true)
|
|
251
|
+
|
|
252
|
+
// The revived entry is LIVE again (running, non-historical).
|
|
253
|
+
const entry = h.watcher.getRegistry().get(agentId)
|
|
254
|
+
expect(entry).toBeDefined()
|
|
255
|
+
expect(entry!.state).toBe('running')
|
|
256
|
+
expect(entry!.historical).toBe(false)
|
|
257
|
+
|
|
258
|
+
// A further growth tick for the SAME false finish does NOT re-resurrect.
|
|
259
|
+
h.appendActivity()
|
|
260
|
+
h.advance(500)
|
|
261
|
+
expect(h.resurrectCalls).toHaveLength(1)
|
|
262
|
+
})
|
|
263
|
+
|
|
264
|
+
it('(b) names a twice-falsely-finalised worker LOST instead of resurrecting again', () => {
|
|
265
|
+
const agentId = 'resurrect-bound'
|
|
266
|
+
const h = makeHarness({ agentId })
|
|
267
|
+
|
|
268
|
+
h.advance(500)
|
|
269
|
+
unmarkHistorical(h, agentId)
|
|
270
|
+
|
|
271
|
+
// First false finish + resurrection.
|
|
272
|
+
driveToFalseFinish(h, agentId)
|
|
273
|
+
h.appendActivity()
|
|
274
|
+
h.advance(500) // poll → checkResurrections resurrects the entry
|
|
275
|
+
h.advance(500) // poll → the revived entry reads its resumed JSONL (settles)
|
|
276
|
+
expect(h.resurrectCalls).toHaveLength(1)
|
|
277
|
+
expect(h.lostCalls).toHaveLength(0)
|
|
278
|
+
|
|
279
|
+
// The revived worker goes silent AGAIN and is falsely finalised a second
|
|
280
|
+
// time (no fresh activity → stall → synthesis).
|
|
281
|
+
driveToFalseFinish(h, agentId)
|
|
282
|
+
expect(h.stallTerminalCalls.length).toBeGreaterThanOrEqual(2)
|
|
283
|
+
|
|
284
|
+
// Its JSONL resumes once more — but the chain bound is spent.
|
|
285
|
+
h.appendActivity()
|
|
286
|
+
h.advance(500)
|
|
287
|
+
|
|
288
|
+
expect(h.resurrectCalls).toHaveLength(1) // NOT resurrected a second time
|
|
289
|
+
expect(h.lostCalls).toHaveLength(1)
|
|
290
|
+
expect(h.lostCalls[0].agentId).toBe(agentId)
|
|
291
|
+
expect(h.logs.some((l) => l.includes('NAMED AS LOST') && l.includes(agentId))).toBe(true)
|
|
292
|
+
|
|
293
|
+
// Further growth stays lost — no more resurrections, no more lost fires.
|
|
294
|
+
h.appendActivity()
|
|
295
|
+
h.advance(500)
|
|
296
|
+
expect(h.resurrectCalls).toHaveLength(1)
|
|
297
|
+
expect(h.lostCalls).toHaveLength(1)
|
|
298
|
+
})
|
|
299
|
+
|
|
300
|
+
it('(c) a genuine boot-time historical completed worker is never resurrected', () => {
|
|
301
|
+
const agentId = 'old-completed'
|
|
302
|
+
// Present at boot, already finished (turn_end in the file). This is the
|
|
303
|
+
// "old completed worker rediscovered at boot" population — it must stay
|
|
304
|
+
// historical/suppressed and never enter the resurrection path.
|
|
305
|
+
const h = makeHarness({
|
|
306
|
+
agentId,
|
|
307
|
+
bootContent: buildJSONL(subAgentUserMsg('done long ago'), subAgentTurnEnd()),
|
|
308
|
+
})
|
|
309
|
+
|
|
310
|
+
h.advance(500) // boot scan already ran at construction; register historical
|
|
311
|
+
|
|
312
|
+
const entry = h.watcher.getRegistry().get(agentId)
|
|
313
|
+
// Historical done-at-boot entries are short-circuited (completionNotified).
|
|
314
|
+
if (entry) expect(entry.historical).toBe(true)
|
|
315
|
+
|
|
316
|
+
// Even if the file grows post-boot, there is no false-finish record for it,
|
|
317
|
+
// so it can never be resurrected.
|
|
318
|
+
h.appendActivity()
|
|
319
|
+
h.advance(500)
|
|
320
|
+
h.advance(500)
|
|
321
|
+
|
|
322
|
+
expect(h.resurrectCalls).toHaveLength(0)
|
|
323
|
+
expect(h.lostCalls).toHaveLength(0)
|
|
324
|
+
expect(h.stallTerminalCalls).toHaveLength(0)
|
|
325
|
+
})
|
|
326
|
+
|
|
327
|
+
it('(d) late trailing turn_end flush → bounded resurrect→immediate-refinish flicker (N1)', () => {
|
|
328
|
+
// Accepted false positive: the silent-stall synthesis fires, then Claude
|
|
329
|
+
// Code finally flushes the worker's REAL buffered `turn_end` line. The
|
|
330
|
+
// byte growth trips resurrection (we can't distinguish it from live
|
|
331
|
+
// resumption without fragile tail parsing), the revived entry immediately
|
|
332
|
+
// reads the terminal line and re-finishes, and the chain stays bounded —
|
|
333
|
+
// no loop, no further resurrections.
|
|
334
|
+
const agentId = 'late-flush'
|
|
335
|
+
const h = makeHarness({ agentId })
|
|
336
|
+
|
|
337
|
+
h.advance(500)
|
|
338
|
+
unmarkHistorical(h, agentId)
|
|
339
|
+
|
|
340
|
+
driveToFalseFinish(h, agentId)
|
|
341
|
+
expect(h.finishCalls).toHaveLength(1) // the synthesized false finish
|
|
342
|
+
|
|
343
|
+
// The worker was genuinely done — its buffered turn_end flushes late.
|
|
344
|
+
h.appendTurnEnd()
|
|
345
|
+
h.advance(500) // poll → growth trips the (false-positive) resurrection
|
|
346
|
+
|
|
347
|
+
expect(h.resurrectCalls).toHaveLength(1) // budget burned, by design
|
|
348
|
+
h.advance(500) // revived entry reads the terminal line → immediate re-finish
|
|
349
|
+
|
|
350
|
+
expect(h.finishCalls).toHaveLength(2) // corrected real finish delivered
|
|
351
|
+
expect(h.finishCalls[1].agentId).toBe(agentId)
|
|
352
|
+
const entry = h.watcher.getRegistry().get(agentId)
|
|
353
|
+
expect(entry?.state).toBe('done')
|
|
354
|
+
|
|
355
|
+
// Chain stays bounded: nothing further fires without new growth, and a
|
|
356
|
+
// genuine turn_end never records a new false finish, so no loop.
|
|
357
|
+
h.advance(500)
|
|
358
|
+
h.advance(500)
|
|
359
|
+
expect(h.resurrectCalls).toHaveLength(1)
|
|
360
|
+
expect(h.finishCalls).toHaveLength(2)
|
|
361
|
+
expect(h.lostCalls).toHaveLength(0)
|
|
362
|
+
})
|
|
363
|
+
|
|
364
|
+
it('(e) a resurrected worker\'s genuine turn_end re-fires onFinish — corrected result supersedes the false one (N2)', () => {
|
|
365
|
+
// Intent (documented at the completionNotified reset in resurrectAgent):
|
|
366
|
+
// the false synthesized finish already delivered a possibly-wrong handback;
|
|
367
|
+
// when the resurrected worker later REALLY completes, onFinish MUST fire a
|
|
368
|
+
// second time so the corrected real result reaches the parent. The gateway
|
|
369
|
+
// spool dedups only concurrently-live handback envelopes, so this second
|
|
370
|
+
// delivery lands as a fresh turn.
|
|
371
|
+
const agentId = 'refire-finish'
|
|
372
|
+
const h = makeHarness({ agentId })
|
|
373
|
+
|
|
374
|
+
h.advance(500)
|
|
375
|
+
unmarkHistorical(h, agentId)
|
|
376
|
+
|
|
377
|
+
driveToFalseFinish(h, agentId)
|
|
378
|
+
expect(h.finishCalls).toHaveLength(1) // synthesized (false) finish
|
|
379
|
+
|
|
380
|
+
// Worker was alive: real activity resumes → resurrection.
|
|
381
|
+
h.appendActivity()
|
|
382
|
+
h.advance(500)
|
|
383
|
+
expect(h.resurrectCalls).toHaveLength(1)
|
|
384
|
+
h.advance(500) // revived entry settles reading the resumed JSONL
|
|
385
|
+
|
|
386
|
+
// Now the worker GENUINELY completes.
|
|
387
|
+
h.appendTurnEnd()
|
|
388
|
+
h.advance(500)
|
|
389
|
+
|
|
390
|
+
expect(h.finishCalls).toHaveLength(2) // real finish fired again — intended
|
|
391
|
+
expect(h.finishCalls[1].agentId).toBe(agentId)
|
|
392
|
+
expect(h.finishCalls[1].outcome).toBe('completed')
|
|
393
|
+
// A genuine turn_end never records a false finish → no further resurrection.
|
|
394
|
+
h.appendActivity()
|
|
395
|
+
h.advance(500)
|
|
396
|
+
expect(h.resurrectCalls).toHaveLength(1)
|
|
397
|
+
})
|
|
398
|
+
})
|
|
@@ -31,6 +31,25 @@ function subAgentUserMsg(promptText: string) {
|
|
|
31
31
|
function subAgentTurnEnd() {
|
|
32
32
|
return { type: 'system', subtype: 'turn_duration', duration_ms: 1234 }
|
|
33
33
|
}
|
|
34
|
+
// An assistant message that STARTS a tool call (e.g. a long-running Bash
|
|
35
|
+
// loop). A tool-using turn's stop_reason is 'tool_use', never 'end_turn',
|
|
36
|
+
// so this does NOT terminalise the worker.
|
|
37
|
+
function subAgentToolUse(toolUseId: string, name = 'Bash') {
|
|
38
|
+
return {
|
|
39
|
+
type: 'assistant',
|
|
40
|
+
message: {
|
|
41
|
+
stop_reason: 'tool_use',
|
|
42
|
+
content: [{ type: 'tool_use', id: toolUseId, name, input: { command: 'sleep 600' } }],
|
|
43
|
+
},
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
// A user message carrying the matching tool_result — the tool call is done.
|
|
47
|
+
function subAgentToolResult(toolUseId: string) {
|
|
48
|
+
return {
|
|
49
|
+
type: 'user',
|
|
50
|
+
message: { content: [{ type: 'tool_result', tool_use_id: toolUseId, content: 'ok' }] },
|
|
51
|
+
}
|
|
52
|
+
}
|
|
34
53
|
|
|
35
54
|
interface Harness {
|
|
36
55
|
stallCalls: Array<{ agentId: string; idleMs: number }>
|
|
@@ -43,18 +62,21 @@ interface Harness {
|
|
|
43
62
|
fileContents: Map<string, Buffer>
|
|
44
63
|
jsonlPath: string
|
|
45
64
|
appendActivity: () => void
|
|
65
|
+
appendLines: (...lines: object[]) => void
|
|
46
66
|
}
|
|
47
67
|
|
|
48
68
|
function makeHarness(opts: {
|
|
49
69
|
agentId?: string
|
|
50
70
|
stallThresholdMs?: number
|
|
51
71
|
silentStallTerminalMs?: number
|
|
72
|
+
inflightTerminalCapMs?: number
|
|
52
73
|
rescanMs?: number
|
|
53
74
|
} = {}): Harness {
|
|
54
75
|
const {
|
|
55
76
|
agentId = 'bug6-agent',
|
|
56
77
|
stallThresholdMs = 60_000,
|
|
57
78
|
silentStallTerminalMs = 300_000,
|
|
79
|
+
inflightTerminalCapMs,
|
|
58
80
|
rescanMs = 500,
|
|
59
81
|
} = opts
|
|
60
82
|
|
|
@@ -126,6 +148,7 @@ function makeHarness(opts: {
|
|
|
126
148
|
stallThresholdMs,
|
|
127
149
|
silentSynthesisStallThresholdMs: stallThresholdMs,
|
|
128
150
|
silentStallTerminalMs,
|
|
151
|
+
inflightTerminalCapMs,
|
|
129
152
|
rescanMs,
|
|
130
153
|
onStall: (id, idleMs) => stallCalls.push({ agentId: id, idleMs }),
|
|
131
154
|
onUnstall: (id) => unstallCalls.push({ agentId: id }),
|
|
@@ -168,6 +191,12 @@ function makeHarness(opts: {
|
|
|
168
191
|
fileContents.set(jsonlPath, Buffer.concat([cur, Buffer.from(more, 'utf-8')]))
|
|
169
192
|
}
|
|
170
193
|
|
|
194
|
+
const appendLines = (...lines: object[]): void => {
|
|
195
|
+
const cur = fileContents.get(jsonlPath) ?? Buffer.alloc(0)
|
|
196
|
+
const more = buildJSONL(...lines)
|
|
197
|
+
fileContents.set(jsonlPath, Buffer.concat([cur, Buffer.from(more, 'utf-8')]))
|
|
198
|
+
}
|
|
199
|
+
|
|
171
200
|
return {
|
|
172
201
|
stallCalls,
|
|
173
202
|
stallTerminalCalls,
|
|
@@ -179,6 +208,7 @@ function makeHarness(opts: {
|
|
|
179
208
|
fileContents,
|
|
180
209
|
jsonlPath,
|
|
181
210
|
appendActivity,
|
|
211
|
+
appendLines,
|
|
182
212
|
}
|
|
183
213
|
}
|
|
184
214
|
|
|
@@ -313,3 +343,145 @@ describe('subagent-watcher post-stall terminal synthesis (RFC §Bug 6)', () => {
|
|
|
313
343
|
expect(h.finishCalls).toHaveLength(1) // unchanged
|
|
314
344
|
})
|
|
315
345
|
})
|
|
346
|
+
|
|
347
|
+
describe('subagent-watcher in-flight tool-call gate (incident 2026-07-10)', () => {
|
|
348
|
+
it('does NOT synthesise terminal while a tool call is still in flight, even past silentStallTerminalMs', () => {
|
|
349
|
+
const agentId = 'inflight-no-synth'
|
|
350
|
+
const h = makeHarness({
|
|
351
|
+
agentId,
|
|
352
|
+
stallThresholdMs: 60_000,
|
|
353
|
+
silentStallTerminalMs: 300_000,
|
|
354
|
+
rescanMs: 500,
|
|
355
|
+
})
|
|
356
|
+
|
|
357
|
+
h.advance(500) // register
|
|
358
|
+
unmarkHistorical(h, agentId)
|
|
359
|
+
|
|
360
|
+
// Worker starts a long-running Bash tool call (a frame-capture loop).
|
|
361
|
+
// The transcript ends in a tool_use with NO matching tool_result — the
|
|
362
|
+
// JSONL then goes silent for the entire duration of the tool call.
|
|
363
|
+
h.appendLines(subAgentToolUse('toolu_longbash'))
|
|
364
|
+
h.advance(1_000) // poll reads the tool_use → in-flight set has 1 id
|
|
365
|
+
|
|
366
|
+
// Idle far past the stall threshold: the stall badge still fires (the
|
|
367
|
+
// worker IS quiet), but the entry is inside a legal long tool call.
|
|
368
|
+
h.advance(62_000)
|
|
369
|
+
expect(h.stallCalls).toHaveLength(1)
|
|
370
|
+
|
|
371
|
+
// Idle well past silentStallTerminalMs — with the OLD code this would
|
|
372
|
+
// finalise the live worker's card. The in-flight gate must suppress it.
|
|
373
|
+
h.advance(400_000)
|
|
374
|
+
expect(h.stallTerminalCalls).toHaveLength(0)
|
|
375
|
+
expect(h.finishCalls).toHaveLength(0)
|
|
376
|
+
expect(h.logs.some((l) => l.includes('terminal synthesis deferred'))).toBe(true)
|
|
377
|
+
})
|
|
378
|
+
|
|
379
|
+
it('synthesises terminal once the tool_result lands and idle persists', () => {
|
|
380
|
+
const agentId = 'inflight-then-synth'
|
|
381
|
+
const h = makeHarness({
|
|
382
|
+
agentId,
|
|
383
|
+
stallThresholdMs: 60_000,
|
|
384
|
+
silentStallTerminalMs: 300_000,
|
|
385
|
+
rescanMs: 500,
|
|
386
|
+
})
|
|
387
|
+
|
|
388
|
+
h.advance(500)
|
|
389
|
+
unmarkHistorical(h, agentId)
|
|
390
|
+
|
|
391
|
+
// In-flight tool call, then a long silent window — no synthesis.
|
|
392
|
+
h.appendLines(subAgentToolUse('toolu_longbash'))
|
|
393
|
+
h.advance(1_000)
|
|
394
|
+
h.advance(62_000) // stall fires
|
|
395
|
+
h.advance(400_000) // gated — no synthesis
|
|
396
|
+
expect(h.stallTerminalCalls).toHaveLength(0)
|
|
397
|
+
|
|
398
|
+
// The tool call finally completes: the matching tool_result lands.
|
|
399
|
+
// This is JSONL growth, so the un-stall path fires and the in-flight
|
|
400
|
+
// set drains — the gate re-opens.
|
|
401
|
+
h.appendLines(subAgentToolResult('toolu_longbash'))
|
|
402
|
+
h.advance(1_000)
|
|
403
|
+
expect(h.unstallCalls).toHaveLength(1)
|
|
404
|
+
expect(h.stallTerminalCalls).toHaveLength(0)
|
|
405
|
+
|
|
406
|
+
// Now the worker goes genuinely silent with nothing in flight: re-stall,
|
|
407
|
+
// then cross the terminal window — synthesis fires exactly as designed.
|
|
408
|
+
h.advance(62_000) // re-stall
|
|
409
|
+
h.advance(302_000) // past silentStallTerminalMs from the new stall
|
|
410
|
+
expect(h.stallTerminalCalls).toHaveLength(1)
|
|
411
|
+
expect(h.stallTerminalCalls[0].agentId).toBe(agentId)
|
|
412
|
+
expect(h.finishCalls).toHaveLength(1)
|
|
413
|
+
expect(h.finishCalls[0].outcome).toBe('completed')
|
|
414
|
+
})
|
|
415
|
+
|
|
416
|
+
it('releases terminal synthesis at inflightTerminalCapMs even with the tool call still unresolved (died-mid-tool)', () => {
|
|
417
|
+
// A worker killed mid-tool never writes the tool_result: without the
|
|
418
|
+
// cap it would defer forever AND the reaper's isLive cross-check would
|
|
419
|
+
// shield its DB row — wedged for the life of the gateway. The cap is
|
|
420
|
+
// the reconciliation with the #2777/#2782 completion-gate contract.
|
|
421
|
+
const agentId = 'inflight-cap-release'
|
|
422
|
+
const h = makeHarness({
|
|
423
|
+
agentId,
|
|
424
|
+
stallThresholdMs: 60_000,
|
|
425
|
+
silentStallTerminalMs: 300_000,
|
|
426
|
+
inflightTerminalCapMs: 600_000, // compressed 10-min cap
|
|
427
|
+
rescanMs: 500,
|
|
428
|
+
})
|
|
429
|
+
|
|
430
|
+
h.advance(500)
|
|
431
|
+
unmarkHistorical(h, agentId)
|
|
432
|
+
h.appendLines(subAgentToolUse('toolu_killed_midtool'))
|
|
433
|
+
h.advance(1_000) // in-flight set has 1 id
|
|
434
|
+
h.advance(62_000) // stall fires
|
|
435
|
+
expect(h.stallCalls).toHaveLength(1)
|
|
436
|
+
|
|
437
|
+
// Past silentStallTerminalMs but under the cap → deferred.
|
|
438
|
+
h.advance(400_000) // ~463s total idle < 600s cap
|
|
439
|
+
expect(h.stallTerminalCalls).toHaveLength(0)
|
|
440
|
+
|
|
441
|
+
// Cross the cap with the tool_result still missing → released.
|
|
442
|
+
h.advance(150_000) // ~613s total idle >= 600s cap
|
|
443
|
+
expect(h.stallTerminalCalls).toHaveLength(1)
|
|
444
|
+
expect(h.finishCalls).toHaveLength(1)
|
|
445
|
+
expect(h.logs.some((l) => l.includes('in-flight deferral cap reached'))).toBe(true)
|
|
446
|
+
})
|
|
447
|
+
|
|
448
|
+
it('gates on a nested Agent/Task dispatch the same as a plain tool call', () => {
|
|
449
|
+
// A FOREGROUND nested Agent/Task dispatch blocks this worker with the
|
|
450
|
+
// same frozen-JSONL shape as a long Bash — its tool_use routes to
|
|
451
|
+
// sub_agent_nested_spawn (not sub_agent_tool_use), so it needs its own
|
|
452
|
+
// add into the in-flight set. Cleared by the same tool_result handler.
|
|
453
|
+
const agentId = 'inflight-nested-spawn'
|
|
454
|
+
const h = makeHarness({
|
|
455
|
+
agentId,
|
|
456
|
+
stallThresholdMs: 60_000,
|
|
457
|
+
silentStallTerminalMs: 300_000,
|
|
458
|
+
rescanMs: 500,
|
|
459
|
+
})
|
|
460
|
+
|
|
461
|
+
h.advance(500)
|
|
462
|
+
unmarkHistorical(h, agentId)
|
|
463
|
+
h.appendLines({
|
|
464
|
+
type: 'assistant',
|
|
465
|
+
message: {
|
|
466
|
+
stop_reason: 'tool_use',
|
|
467
|
+
content: [{ type: 'tool_use', id: 'toolu_nested', name: 'Agent', input: { description: 'child task' } }],
|
|
468
|
+
},
|
|
469
|
+
})
|
|
470
|
+
h.advance(1_000)
|
|
471
|
+
h.advance(62_000) // stall fires (nested child is quiet in OUR jsonl)
|
|
472
|
+
expect(h.stallCalls).toHaveLength(1)
|
|
473
|
+
|
|
474
|
+
// Past silentStallTerminalMs — nested dispatch in flight → deferred.
|
|
475
|
+
h.advance(400_000)
|
|
476
|
+
expect(h.stallTerminalCalls).toHaveLength(0)
|
|
477
|
+
|
|
478
|
+
// Child returns: tool_result clears the id; after a fresh stall +
|
|
479
|
+
// terminal window with nothing in flight, synthesis proceeds.
|
|
480
|
+
h.appendLines(subAgentToolResult('toolu_nested'))
|
|
481
|
+
h.advance(1_000)
|
|
482
|
+
h.advance(62_000) // re-stall
|
|
483
|
+
h.advance(302_000)
|
|
484
|
+
expect(h.stallTerminalCalls).toHaveLength(1)
|
|
485
|
+
expect(h.finishCalls).toHaveLength(1)
|
|
486
|
+
})
|
|
487
|
+
})
|
|
@@ -261,10 +261,24 @@ describe('#2798 turn-flush punctuation/bold parity with reply', () => {
|
|
|
261
261
|
)
|
|
262
262
|
|
|
263
263
|
it('reply path: normalizes AFTER redact and BEFORE the voice scrub', () => {
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
264
|
+
// #2996: the reply-path entry pipeline moved into outbound-send-path.ts
|
|
265
|
+
// (`normalizeOutboundBody`). The reply path delegates to it via
|
|
266
|
+
// `normalizeOutboundBody(rawText, 'reply', redactOutboundText)`; the
|
|
267
|
+
// redact→normalize→scrub ordering is now pinned in the module source.
|
|
268
|
+
const replyDelegates = gatewaySrc.indexOf(
|
|
269
|
+
`normalizeOutboundBody(rawText, 'reply', redactOutboundText)`,
|
|
270
|
+
gatewaySrc.indexOf('async function executeReply('),
|
|
271
|
+
)
|
|
272
|
+
expect(replyDelegates).toBeGreaterThan(0)
|
|
273
|
+
|
|
274
|
+
const moduleSrc = readFileSync(
|
|
275
|
+
new URL('../gateway/outbound-send-path.ts', import.meta.url),
|
|
276
|
+
'utf8',
|
|
277
|
+
)
|
|
278
|
+
const start = moduleSrc.indexOf('export function normalizeOutboundBody(')
|
|
279
|
+
const redactIdx = moduleSrc.indexOf('redact(text, site)', start)
|
|
280
|
+
const normIdx = moduleSrc.indexOf('stripExcessBold(normalizePunctuation(text))', start)
|
|
281
|
+
const scrubIdx = moduleSrc.indexOf('scrubVoice(text)', start)
|
|
268
282
|
expect(start).toBeGreaterThan(0)
|
|
269
283
|
expect(redactIdx).toBeGreaterThan(start)
|
|
270
284
|
expect(normIdx).toBeGreaterThan(redactIdx) // normalize AFTER the reply redact
|