switchroom 0.21.7 → 0.21.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/tmp-reaper.sh +234 -0
- package/dist/agent-scheduler/index.js +1 -1
- package/dist/auth-broker/index.js +2 -2
- package/dist/cli/notion-write-pretool.mjs +1 -1
- package/dist/cli/switchroom.js +3421 -2744
- package/dist/host-control/main.js +177 -13
- package/dist/vault/approvals/kernel-server.js +2 -2
- package/dist/vault/broker/server.js +2 -2
- package/package.json +5 -4
- package/profiles/_base/start.sh.hbs +115 -0
- package/profiles/_shared/local-time.md.hbs +6 -0
- package/profiles/default/CLAUDE.md.hbs +0 -12
- package/telegram-plugin/dist/gateway/gateway.js +1017 -465
- package/telegram-plugin/gateway/agent-process-liveness.ts +558 -0
- package/telegram-plugin/gateway/approval-hold.ts +32 -1
- package/telegram-plugin/gateway/approval-outcome-sources.ts +274 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +21 -9
- package/telegram-plugin/gateway/callback-query-handlers.ts +87 -15
- package/telegram-plugin/gateway/eval-case-proposal-inbound-builders.ts +197 -0
- package/telegram-plugin/gateway/gateway.ts +12 -10
- package/telegram-plugin/gateway/pending-inbound-buffer.ts +167 -11
- package/telegram-plugin/gateway/self-improve-proposal-wiring.test.ts +333 -0
- package/telegram-plugin/gateway/self-improve-proposal-wiring.ts +152 -3
- package/telegram-plugin/gateway/subagent-handback-marker.ts +19 -0
- package/telegram-plugin/tests/agent-process-liveness.test.ts +406 -0
- package/telegram-plugin/tests/approval-hold-record.test.ts +21 -8
- package/telegram-plugin/tests/boot-resume-gateway-only-respawn.test.ts +752 -0
- package/telegram-plugin/tests/boot-resume-guard-wiring.test.ts +203 -0
- package/telegram-plugin/tests/callback-query-handlers.test.ts +143 -1
- package/telegram-plugin/tests/eval-case-proposal-inbound-builders.test.ts +144 -0
- package/telegram-plugin/tests/hermes-messages-paging.test.ts +149 -0
- package/telegram-plugin/tests/hermes-session-search.test.ts +146 -0
- package/telegram-plugin/tests/pending-inbound-buffer.test.ts +443 -2
- package/telegram-plugin/tests/subagent-handback-marker.test.ts +14 -0
|
@@ -21,20 +21,86 @@
|
|
|
21
21
|
|
|
22
22
|
import type { Bot, Context } from 'grammy'
|
|
23
23
|
import type { RetryCallOpts } from '../retry-api-call.js'
|
|
24
|
-
import type { PostSkillProposalMessage, PostEvalCaseProposalMessage } from './ipc-protocol.js'
|
|
24
|
+
import type { PostSkillProposalMessage, PostEvalCaseProposalMessage, InboundMessage } from './ipc-protocol.js'
|
|
25
25
|
import { renderSkillProposalCard, skillProposalKeyboard } from './skill-proposal-card.js'
|
|
26
26
|
import { renderEvalCaseProposalCard, evalCaseProposalKeyboard } from './eval-case-proposal-card.js'
|
|
27
27
|
import {
|
|
28
28
|
enqueueProposal as enqueueSkillProposal,
|
|
29
29
|
isSuppressed as isSkillProposalSuppressed,
|
|
30
|
+
REJECTION_TTL_MS,
|
|
30
31
|
} from '../../src/self-improve/skill-proposals.js'
|
|
31
|
-
import {
|
|
32
|
+
import {
|
|
33
|
+
enqueueEvalCaseProposal,
|
|
34
|
+
readEvalCaseProposals,
|
|
35
|
+
} from '../../src/self-improve/eval-case-proposals.js'
|
|
32
36
|
|
|
33
37
|
/** Collaborators the gateway injects into each handler. */
|
|
34
38
|
export interface ProposalWiringDeps {
|
|
35
39
|
bot: Bot<Context>
|
|
36
40
|
assertAllowedChat: (chatId: string) => void
|
|
37
41
|
swallowingApiCall: <T>(fn: () => Promise<T>, opts?: RetryCallOpts) => Promise<T | undefined>
|
|
42
|
+
/**
|
|
43
|
+
* Wake the proposing agent with a synthetic inbound (gateway.ts's
|
|
44
|
+
* `deliverResumeSyntheticOrBuffer`). REQUIRED, not optional: the suppressed
|
|
45
|
+
* branch below is a silent exit, and a call site that forgot to wire this
|
|
46
|
+
* would reinstate exactly the wait-forever bug it exists to prevent — so the
|
|
47
|
+
* type system forces every caller to supply it.
|
|
48
|
+
*/
|
|
49
|
+
deliverResumeSyntheticOrBuffer: (agent: string, inbound: InboundMessage) => boolean
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Build the synthetic inbound that tells a proposing agent its eval case was
|
|
54
|
+
* SUPPRESSED as a duplicate of one the operator already dismissed.
|
|
55
|
+
*
|
|
56
|
+
* WHY THIS EXISTS (cross-PR, #4662 + #4664). #4662 makes the agent's contract
|
|
57
|
+
* "fire `add-eval-case`, end your turn, wait for the outcome inbound". #4664
|
|
58
|
+
* adds a branch that posts NO card. The propose CLI is fire-and-forget — it
|
|
59
|
+
* prints `ok:true` for the IPC SEND, never for a card being posted
|
|
60
|
+
* (src/cli/self-improve-eval-case.ts) — so without this the agent would end its
|
|
61
|
+
* turn and wait for a tap that can never come: the exact silent block #4662
|
|
62
|
+
* exists to eliminate, re-entering through a new door.
|
|
63
|
+
*
|
|
64
|
+
* The tone is deliberate. Suppression is the system working as designed, not a
|
|
65
|
+
* failure, so the text says so plainly — an agent told only "no card" would
|
|
66
|
+
* reasonably retry, which is the nagging loop #4664 is removing.
|
|
67
|
+
*
|
|
68
|
+
* Co-located here rather than in #4662's `eval-case-proposal-inbound-builders.ts`
|
|
69
|
+
* because that module does not exist on this branch; consolidating the four
|
|
70
|
+
* builders once both PRs land is a tracked follow-up.
|
|
71
|
+
*/
|
|
72
|
+
export function buildEvalCaseSuppressedInbound(opts: {
|
|
73
|
+
agent: string
|
|
74
|
+
chatId: string
|
|
75
|
+
threadId?: number
|
|
76
|
+
skillSlug: string
|
|
77
|
+
fingerprint: string
|
|
78
|
+
nowMs?: number
|
|
79
|
+
}): InboundMessage {
|
|
80
|
+
const ts = opts.nowMs ?? Date.now()
|
|
81
|
+
return {
|
|
82
|
+
type: 'inbound',
|
|
83
|
+
chatId: opts.chatId,
|
|
84
|
+
...(opts.threadId != null ? { threadId: opts.threadId } : {}),
|
|
85
|
+
messageId: ts, // synthetic — no Telegram message id exists
|
|
86
|
+
user: 'self-improve',
|
|
87
|
+
userId: 0,
|
|
88
|
+
ts,
|
|
89
|
+
text:
|
|
90
|
+
`ℹ️ Your proposed eval case for \`${opts.skillSlug}\` was NOT posted as a ` +
|
|
91
|
+
`card: the operator already dismissed this exact case, and the dismissal ` +
|
|
92
|
+
`is still in effect. This is EXPECTED and is not an error or a failure on ` +
|
|
93
|
+
`your part — nothing was written and no card is pending, so do NOT wait ` +
|
|
94
|
+
`for an approval tap. Do NOT re-propose this case. Carry on with the ` +
|
|
95
|
+
`original task without it.`,
|
|
96
|
+
meta: {
|
|
97
|
+
source: 'eval_case_suppressed',
|
|
98
|
+
agent: opts.agent,
|
|
99
|
+
...(opts.threadId != null ? { message_thread_id: String(opts.threadId) } : {}),
|
|
100
|
+
skill_slug: opts.skillSlug,
|
|
101
|
+
fingerprint: opts.fingerprint,
|
|
102
|
+
},
|
|
103
|
+
}
|
|
38
104
|
}
|
|
39
105
|
|
|
40
106
|
/**
|
|
@@ -113,6 +179,61 @@ export function handlePostSkillProposal(
|
|
|
113
179
|
)
|
|
114
180
|
}
|
|
115
181
|
|
|
182
|
+
/**
|
|
183
|
+
* True iff the operator already DISMISSED this exact eval case within the
|
|
184
|
+
* rejection TTL — the eval-case twin of `isSuppressed` in skill-proposals.ts.
|
|
185
|
+
*
|
|
186
|
+
* Matching is deliberately TIGHTER than the skill path's. A skill proposal is
|
|
187
|
+
* free prose, so that check has to fuzzy-match (jaccard over content words,
|
|
188
|
+
* with the slug only relaxing the threshold). An eval case carries a
|
|
189
|
+
* precomputed `fingerprint` — `caseFingerprint(prompt)`, a hash of the
|
|
190
|
+
* normalized prompt — and the CLI's own dedup against a skill's applied
|
|
191
|
+
* evals.json compares those fingerprints EXACTLY
|
|
192
|
+
* (src/cli/self-improve-eval-case.ts). So this requires exact fingerprint
|
|
193
|
+
* equality AND same-slug, matching the CLI's per-skill dedup scope: no fuzzy
|
|
194
|
+
* bar to tune, and a different PROMPT can never be swallowed.
|
|
195
|
+
*
|
|
196
|
+
* KNOWN CONSEQUENCE — identity is the PROMPT, not the whole case.
|
|
197
|
+
* `caseFingerprint` hashes the normalized prompt ONLY; an `EvalCase` also
|
|
198
|
+
* carries `expected_output` and `expectations` (src/self-improve/eval-cases.ts),
|
|
199
|
+
* and neither is in the hash. So re-proposing the SAME prompt with a CORRECTED
|
|
200
|
+
* expected_output/expectations fingerprints identically to the dismissed one and
|
|
201
|
+
* stays suppressed for the full TTL. This is deliberate, not an oversight: it is
|
|
202
|
+
* the same identity the CLI's own evals.json dedup uses, so a looser rule here
|
|
203
|
+
* would disagree with the applier and re-admit cases the CLI would then reject.
|
|
204
|
+
* To land a revised assertion for an already-dismissed prompt, reword the prompt
|
|
205
|
+
* (a genuinely different input) or wait out REJECTION_TTL_MS.
|
|
206
|
+
*
|
|
207
|
+
* The TTL semantics ARE mirrored (a dismissal shouldn't suppress forever), on
|
|
208
|
+
* the same REJECTION_TTL_MS window. The eval-case store records no
|
|
209
|
+
* `rejected_at`, so age is measured from `created_at`, which is always <= the
|
|
210
|
+
* rejection time — the window therefore expires no LATER than a true
|
|
211
|
+
* rejected_at basis would, erring towards re-surfacing rather than
|
|
212
|
+
* over-suppressing. A record with an unparseable timestamp is treated as
|
|
213
|
+
* expired, same as the skill path.
|
|
214
|
+
*/
|
|
215
|
+
export function isEvalCaseProposalSuppressed(
|
|
216
|
+
stateDir: string,
|
|
217
|
+
candidate: { skillSlug: string; fingerprint: string },
|
|
218
|
+
opts: { now?: () => number; ttlMs?: number } = {},
|
|
219
|
+
): boolean {
|
|
220
|
+
// A missing/empty fingerprint carries no identity — never suppress on it.
|
|
221
|
+
if (typeof candidate.fingerprint !== 'string' || candidate.fingerprint.length === 0) {
|
|
222
|
+
return false
|
|
223
|
+
}
|
|
224
|
+
const now = (opts.now ?? Date.now)()
|
|
225
|
+
const ttl = opts.ttlMs ?? REJECTION_TTL_MS
|
|
226
|
+
for (const p of readEvalCaseProposals(stateDir)) {
|
|
227
|
+
if (p.status !== 'rejected') continue
|
|
228
|
+
if (p.skill_slug !== candidate.skillSlug) continue
|
|
229
|
+
if (p.fingerprint !== candidate.fingerprint) continue
|
|
230
|
+
const age = now - new Date(p.created_at).getTime()
|
|
231
|
+
if (!Number.isFinite(age) || age > ttl) continue // expired
|
|
232
|
+
return true
|
|
233
|
+
}
|
|
234
|
+
return false
|
|
235
|
+
}
|
|
236
|
+
|
|
116
237
|
/**
|
|
117
238
|
* RFC amendment §"corrections as eval cases" — persist an eval-case proposal
|
|
118
239
|
* and post its Approve/Dismiss card. On Approve the callback runs the
|
|
@@ -124,7 +245,7 @@ export function handlePostEvalCaseProposal(
|
|
|
124
245
|
msg: PostEvalCaseProposalMessage,
|
|
125
246
|
deps: ProposalWiringDeps,
|
|
126
247
|
): void {
|
|
127
|
-
const { bot, assertAllowedChat, swallowingApiCall } = deps
|
|
248
|
+
const { bot, assertAllowedChat, swallowingApiCall, deliverResumeSyntheticOrBuffer } = deps
|
|
128
249
|
const self = process.env.SWITCHROOM_AGENT_NAME
|
|
129
250
|
if (self && msg.agentName !== self) {
|
|
130
251
|
process.stderr.write(
|
|
@@ -145,6 +266,34 @@ export function handlePostEvalCaseProposal(
|
|
|
145
266
|
process.stderr.write(`telegram gateway: post_eval_case_proposal: TELEGRAM_STATE_DIR unset, skipping\n`)
|
|
146
267
|
return
|
|
147
268
|
}
|
|
269
|
+
// Dedup against still-live dismissals — never re-surface an eval case the
|
|
270
|
+
// operator already tapped Dismiss on (the skill path's guard, which the
|
|
271
|
+
// eval-case path was missing: a dismissed case re-posted the same card on
|
|
272
|
+
// every subsequent correction, since the CLI only dedups against evals.json
|
|
273
|
+
// entries that were already APPLIED).
|
|
274
|
+
if (isEvalCaseProposalSuppressed(stateDir, {
|
|
275
|
+
skillSlug: msg.skillSlug,
|
|
276
|
+
fingerprint: msg.fingerprint,
|
|
277
|
+
})) {
|
|
278
|
+
// Tell the agent, or this becomes a silent exit it waits on forever (see
|
|
279
|
+
// buildEvalCaseSuppressedInbound). No card is posted and nothing is written
|
|
280
|
+
// — the operator's dismissal already decided this case.
|
|
281
|
+
const delivered = deliverResumeSyntheticOrBuffer(
|
|
282
|
+
msg.agentName,
|
|
283
|
+
buildEvalCaseSuppressedInbound({
|
|
284
|
+
agent: msg.agentName,
|
|
285
|
+
chatId: msg.chatId,
|
|
286
|
+
...(msg.threadId != null ? { threadId: msg.threadId } : {}),
|
|
287
|
+
skillSlug: msg.skillSlug,
|
|
288
|
+
fingerprint: msg.fingerprint,
|
|
289
|
+
}),
|
|
290
|
+
)
|
|
291
|
+
process.stderr.write(
|
|
292
|
+
`telegram gateway: post_eval_case_proposal suppressed (dismissed before) ` +
|
|
293
|
+
`slug=${msg.skillSlug} fp=${msg.fingerprint} delivered=${delivered}\n`,
|
|
294
|
+
)
|
|
295
|
+
return
|
|
296
|
+
}
|
|
148
297
|
const proposal = enqueueEvalCaseProposal(stateDir, {
|
|
149
298
|
skill_slug: msg.skillSlug,
|
|
150
299
|
skill_dir: msg.skillDir,
|
|
@@ -176,6 +176,25 @@ export const INBOUND_SOURCE_CLASSIFICATION: Record<string, { decoupledCompletion
|
|
|
176
176
|
mental_model_proposal_failed: { decoupledCompletion: false },
|
|
177
177
|
webhook: { decoupledCompletion: false },
|
|
178
178
|
linear: { decoupledCompletion: false },
|
|
179
|
+
// Eval-case proposal outcomes (#4662) — the operator's Approve/Dismiss tap on
|
|
180
|
+
// an eval-case card wakes the PROPOSING agent with one of these, built in
|
|
181
|
+
// `eval-case-proposal-inbound-builders.ts` and injected via
|
|
182
|
+
// `deliverResumeSyntheticOrBuffer`. Exactly the `skill_proposal_apply` shape
|
|
183
|
+
// above: each lands as its OWN live inbound turn, so its reply resolves the
|
|
184
|
+
// live tier for its own turnId and structurally cannot supersede a different
|
|
185
|
+
// ended turn — it must NOT stamp. (Left unclassified, the fail-safe default
|
|
186
|
+
// stamps, holding the content gate chat-wide for 60 s after every eval-case
|
|
187
|
+
// decision and re-opening the reworded-own-answer visible dup in that window.)
|
|
188
|
+
eval_case_applied: { decoupledCompletion: false },
|
|
189
|
+
eval_case_rejected: { decoupledCompletion: false },
|
|
190
|
+
eval_case_apply_failed: { decoupledCompletion: false },
|
|
191
|
+
// Eval-case proposal SUPPRESSED (#4664): the gateway declined to post a card
|
|
192
|
+
// because the operator already dismissed this exact case, and tells the
|
|
193
|
+
// proposing agent so instead of exiting silently
|
|
194
|
+
// (buildEvalCaseSuppressedInbound, self-improve-proposal-wiring.ts). Delivered
|
|
195
|
+
// via `deliverResumeSyntheticOrBuffer` as its OWN live inbound turn, same as
|
|
196
|
+
// skill_proposal_apply above — it must NOT stamp.
|
|
197
|
+
eval_case_suppressed: { decoupledCompletion: false },
|
|
179
198
|
// Buzz co-channel (Phase 1): a Nostr kind:9 group message the buzz sidecar
|
|
180
199
|
// injects onto the gateway IPC queue as its OWN live inbound turn (anonymous
|
|
181
200
|
// inject, `meta.source="buzz"`) — never a decoupled completion resolving a
|
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* switchroom#4641 — the boot-resume generation guard.
|
|
3
|
+
*
|
|
4
|
+
* Two mechanisms, tested for what each is actually responsible for:
|
|
5
|
+
*
|
|
6
|
+
* - the per-container-boot GENERATION TOKEN (`.boot-resume-done`) is the
|
|
7
|
+
* only thing that can say "suppress". start.sh deletes it once per
|
|
8
|
+
* container boot before forking the gateway; the gateway stamps it after
|
|
9
|
+
* completing its boot-resume block.
|
|
10
|
+
* - the `/proc` AGENT RECORD is a VETO only: it can re-enable a boot resume
|
|
11
|
+
* when the recorded agent is provably gone, never suppress one.
|
|
12
|
+
*
|
|
13
|
+
* Liveness assertions use REAL processes and the REAL `/proc`, not a mocked
|
|
14
|
+
* fs: a fixture that hand-writes both answers would pass against a probe that
|
|
15
|
+
* always says "alive". Every "alive" claim is anchored on a process this test
|
|
16
|
+
* spawned; every "dead" claim on one it killed.
|
|
17
|
+
*
|
|
18
|
+
* Deliberately NOT tested here, because it no longer exists: any comparison of
|
|
19
|
+
* the agent's `/proc` starttime against the gateway's. The previous revision
|
|
20
|
+
* suppressed on "the agent predates me", which was correct only by accident of
|
|
21
|
+
* the docker tmux re-exec (measured margin on a live container: one clock
|
|
22
|
+
* tick) and which broke outright when a gateway crashed during its own boot.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'
|
|
26
|
+
import { spawn, type ChildProcess } from 'node:child_process'
|
|
27
|
+
import { mkdtempSync, rmSync, writeFileSync, readFileSync, existsSync, unlinkSync } from 'node:fs'
|
|
28
|
+
import { tmpdir } from 'node:os'
|
|
29
|
+
import { join } from 'node:path'
|
|
30
|
+
import {
|
|
31
|
+
parseProcStat,
|
|
32
|
+
readProcIdentity,
|
|
33
|
+
readAgentProcessRecord,
|
|
34
|
+
decideGatewayOnlyRespawn,
|
|
35
|
+
detectGatewayOnlyRespawn,
|
|
36
|
+
shouldSkipBootResumeForGatewayOnlyRespawn,
|
|
37
|
+
markBootResumeComplete,
|
|
38
|
+
bootResumeDonePath,
|
|
39
|
+
readBootResumeSentinel,
|
|
40
|
+
containerBootIdentity,
|
|
41
|
+
AGENT_PROCESS_RECORD_FILE,
|
|
42
|
+
BOOT_RESUME_DONE_FILE,
|
|
43
|
+
} from '../gateway/agent-process-liveness.js'
|
|
44
|
+
|
|
45
|
+
function starttimeOf(pid: number): string {
|
|
46
|
+
const id = readProcIdentity(pid)
|
|
47
|
+
expect(id, `pid ${pid} should be live`).not.toBeNull()
|
|
48
|
+
return id!.starttime
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function spawnSleeper(): ChildProcess {
|
|
52
|
+
return spawn('sleep', ['120'], { stdio: 'ignore' })
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function waitForExit(child: ChildProcess): Promise<void> {
|
|
56
|
+
if (child.exitCode != null || child.signalCode != null) return
|
|
57
|
+
await new Promise<void>((res) => child.once('exit', () => res()))
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
let dir: string
|
|
61
|
+
let agentProc: ChildProcess
|
|
62
|
+
|
|
63
|
+
beforeAll(async () => {
|
|
64
|
+
dir = mkdtempSync(join(tmpdir(), 'agent-liveness-'))
|
|
65
|
+
agentProc = spawnSleeper()
|
|
66
|
+
await new Promise((r) => setTimeout(r, 20))
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
afterAll(() => {
|
|
70
|
+
try { agentProc.kill('SIGKILL') } catch { /* already gone */ }
|
|
71
|
+
rmSync(dir, { recursive: true, force: true })
|
|
72
|
+
})
|
|
73
|
+
|
|
74
|
+
beforeEach(() => {
|
|
75
|
+
// Every case states its own generation-token state explicitly.
|
|
76
|
+
try { unlinkSync(bootResumeDonePath(dir)) } catch { /* absent */ }
|
|
77
|
+
try { unlinkSync(join(dir, AGENT_PROCESS_RECORD_FILE)) } catch { /* absent */ }
|
|
78
|
+
})
|
|
79
|
+
|
|
80
|
+
/** start.sh's outer-pass "open a new generation" step. */
|
|
81
|
+
function clearToken(): void {
|
|
82
|
+
try { unlinkSync(bootResumeDonePath(dir)) } catch { /* absent */ }
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function writeRecord(rec: unknown, name = AGENT_PROCESS_RECORD_FILE): string {
|
|
86
|
+
const p = join(dir, name)
|
|
87
|
+
writeFileSync(p, typeof rec === 'string' ? rec : JSON.stringify(rec))
|
|
88
|
+
return p
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
describe('parseProcStat', () => {
|
|
92
|
+
it('reads state + starttime past a comm containing spaces and parens', () => {
|
|
93
|
+
// A real-shaped line with a hostile comm — the naive `awk $22` split and a
|
|
94
|
+
// first-`)` split both return the wrong field here.
|
|
95
|
+
const raw =
|
|
96
|
+
'4242 (weird ) name) S 1 4242 4242 0 -1 4194304 167 0 0 0 0 0 0 0 20 0 1 0 ' +
|
|
97
|
+
'987654321 4165632 702 18446744073709551615 0 0 0 0 0 0 2 4 65536 1 0 0 17 0 0 0 0 0 0'
|
|
98
|
+
expect(parseProcStat(raw)).toEqual({
|
|
99
|
+
comm: 'weird ) name',
|
|
100
|
+
state: 'S',
|
|
101
|
+
starttime: '987654321',
|
|
102
|
+
})
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
it('agrees with awk on this process\'s own /proc entry', () => {
|
|
106
|
+
const raw = readFileSync(`/proc/${process.pid}/stat`, 'utf8')
|
|
107
|
+
const fields = raw.slice(raw.lastIndexOf(') ') + 2).trim().split(/\s+/)
|
|
108
|
+
expect(parseProcStat(raw)!.starttime).toBe(fields[19])
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
it('treats a zombie as dead', () => {
|
|
112
|
+
const raw = '4242 (claude) Z 1 4242 4242 0 -1 0 0 0 0 0 0 0 0 20 0 1 0 5 ' +
|
|
113
|
+
'0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0'
|
|
114
|
+
expect(parseProcStat(raw)!.state).toBe('Z')
|
|
115
|
+
})
|
|
116
|
+
})
|
|
117
|
+
|
|
118
|
+
describe('the generation token decides', () => {
|
|
119
|
+
it('suppresses ONLY after a gateway stamped the token this generation', () => {
|
|
120
|
+
writeRecord({ pid: agentProc.pid, starttime: starttimeOf(agentProc.pid!) })
|
|
121
|
+
|
|
122
|
+
// Fresh container boot: start.sh cleared the token, no gateway has
|
|
123
|
+
// finished its boot resume yet. The live agent record must NOT suppress.
|
|
124
|
+
expect(detectGatewayOnlyRespawn({ stateDir: dir })).toEqual({
|
|
125
|
+
gatewayOnly: false,
|
|
126
|
+
reason: 'no-boot-resume-sentinel',
|
|
127
|
+
pid: agentProc.pid,
|
|
128
|
+
})
|
|
129
|
+
|
|
130
|
+
// …the first gateway completes its boot-resume block…
|
|
131
|
+
markBootResumeComplete(dir)
|
|
132
|
+
expect(existsSync(bootResumeDonePath(dir))).toBe(true)
|
|
133
|
+
|
|
134
|
+
// …and a respawned gateway now sees the generation already handled.
|
|
135
|
+
expect(detectGatewayOnlyRespawn({ stateDir: dir })).toEqual({
|
|
136
|
+
gatewayOnly: true,
|
|
137
|
+
reason: 'gateway-only-respawn',
|
|
138
|
+
pid: agentProc.pid,
|
|
139
|
+
})
|
|
140
|
+
})
|
|
141
|
+
|
|
142
|
+
it('does NOT suppress when the gateway crashed during its own boot', () => {
|
|
143
|
+
// The #4641-motivating Bun crash-at-boot pattern: gateway #1 died before
|
|
144
|
+
// finishing the boot-resume block, so it never stamped the token — even
|
|
145
|
+
// though start.sh had already published a live agent record. Gateway #2
|
|
146
|
+
// MUST do the boot resume; the previous starttime-ordering guard
|
|
147
|
+
// suppressed here and lost the interrupted turn permanently.
|
|
148
|
+
writeRecord({ pid: agentProc.pid, starttime: starttimeOf(agentProc.pid!) })
|
|
149
|
+
const decision = detectGatewayOnlyRespawn({ stateDir: dir })
|
|
150
|
+
expect(decision.gatewayOnly).toBe(false)
|
|
151
|
+
expect(decision.reason).toBe('no-boot-resume-sentinel')
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
it('suppresses on a token with no record yet (gateway respawn pre-exec)', () => {
|
|
155
|
+
// The gateway boots long before start.sh reaches `exec claude`. A gateway
|
|
156
|
+
// that crashes in that window must still not repeat this generation's
|
|
157
|
+
// boot resume — repeating it would spool the resume synthetic twice.
|
|
158
|
+
markBootResumeComplete(dir)
|
|
159
|
+
expect(detectGatewayOnlyRespawn({ stateDir: dir })).toEqual({
|
|
160
|
+
gatewayOnly: true,
|
|
161
|
+
reason: 'gateway-only-respawn-no-record',
|
|
162
|
+
pid: null,
|
|
163
|
+
})
|
|
164
|
+
})
|
|
165
|
+
|
|
166
|
+
it('a container restart re-opens the generation (start.sh clears the token)', () => {
|
|
167
|
+
writeRecord({ pid: agentProc.pid, starttime: starttimeOf(agentProc.pid!) })
|
|
168
|
+
markBootResumeComplete(dir)
|
|
169
|
+
expect(detectGatewayOnlyRespawn({ stateDir: dir }).gatewayOnly).toBe(true)
|
|
170
|
+
clearToken()
|
|
171
|
+
expect(detectGatewayOnlyRespawn({ stateDir: dir }).gatewayOnly).toBe(false)
|
|
172
|
+
})
|
|
173
|
+
})
|
|
174
|
+
|
|
175
|
+
describe('the /proc record can only VETO a suppression', () => {
|
|
176
|
+
it('vetoes when the recorded agent process is really dead', async () => {
|
|
177
|
+
const doomed = spawnSleeper()
|
|
178
|
+
await new Promise((r) => setTimeout(r, 20))
|
|
179
|
+
writeRecord({ pid: doomed.pid, starttime: starttimeOf(doomed.pid!) })
|
|
180
|
+
doomed.kill('SIGKILL')
|
|
181
|
+
await waitForExit(doomed)
|
|
182
|
+
markBootResumeComplete(dir)
|
|
183
|
+
|
|
184
|
+
const decision = detectGatewayOnlyRespawn({ stateDir: dir })
|
|
185
|
+
expect(decision.gatewayOnly).toBe(false)
|
|
186
|
+
expect(decision.reason).toBe('agent-process-dead')
|
|
187
|
+
})
|
|
188
|
+
|
|
189
|
+
it('vetoes a recycled PID: same pid, different starttime', () => {
|
|
190
|
+
const live = starttimeOf(agentProc.pid!)
|
|
191
|
+
writeRecord({ pid: agentProc.pid, starttime: String(BigInt(live) - 1n) })
|
|
192
|
+
markBootResumeComplete(dir)
|
|
193
|
+
expect(detectGatewayOnlyRespawn({ stateDir: dir })).toEqual({
|
|
194
|
+
gatewayOnly: false,
|
|
195
|
+
reason: 'starttime-mismatch',
|
|
196
|
+
pid: agentProc.pid,
|
|
197
|
+
})
|
|
198
|
+
})
|
|
199
|
+
|
|
200
|
+
it('cannot manufacture a suppression on its own', () => {
|
|
201
|
+
// A live, matching, older record with NO token still runs the boot resume.
|
|
202
|
+
writeRecord({ pid: agentProc.pid, starttime: starttimeOf(agentProc.pid!) })
|
|
203
|
+
expect(detectGatewayOnlyRespawn({ stateDir: dir }).gatewayOnly).toBe(false)
|
|
204
|
+
})
|
|
205
|
+
|
|
206
|
+
it('fails open on a torn record even with the token present', () => {
|
|
207
|
+
markBootResumeComplete(dir)
|
|
208
|
+
writeRecord('{"pid": 12')
|
|
209
|
+
// Torn record is indistinguishable from "no record" — and with the token
|
|
210
|
+
// present that is a suppression, which is the safe answer here: the block
|
|
211
|
+
// demonstrably already ran this generation.
|
|
212
|
+
expect(detectGatewayOnlyRespawn({ stateDir: dir }).reason)
|
|
213
|
+
.toBe('gateway-only-respawn-no-record')
|
|
214
|
+
})
|
|
215
|
+
|
|
216
|
+
it('honours the SWITCHROOM_GATEWAY_RESPAWN_GUARD=0 escape hatch', () => {
|
|
217
|
+
writeRecord({ pid: agentProc.pid, starttime: starttimeOf(agentProc.pid!) })
|
|
218
|
+
markBootResumeComplete(dir)
|
|
219
|
+
const lines: string[] = []
|
|
220
|
+
const prev = process.env.SWITCHROOM_GATEWAY_RESPAWN_GUARD
|
|
221
|
+
process.env.SWITCHROOM_GATEWAY_RESPAWN_GUARD = '0'
|
|
222
|
+
try {
|
|
223
|
+
expect(
|
|
224
|
+
shouldSkipBootResumeForGatewayOnlyRespawn(dir, { log: (s) => lines.push(s) }),
|
|
225
|
+
).toBe(false)
|
|
226
|
+
} finally {
|
|
227
|
+
if (prev == null) delete process.env.SWITCHROOM_GATEWAY_RESPAWN_GUARD
|
|
228
|
+
else process.env.SWITCHROOM_GATEWAY_RESPAWN_GUARD = prev
|
|
229
|
+
}
|
|
230
|
+
expect(lines.join('')).toContain('guard-disabled')
|
|
231
|
+
})
|
|
232
|
+
|
|
233
|
+
it('skips the boot-resume path (and says so) for a live agent + token', () => {
|
|
234
|
+
writeRecord({ pid: agentProc.pid, starttime: starttimeOf(agentProc.pid!) })
|
|
235
|
+
markBootResumeComplete(dir)
|
|
236
|
+
const lines: string[] = []
|
|
237
|
+
expect(shouldSkipBootResumeForGatewayOnlyRespawn(dir, { log: (s) => lines.push(s) })).toBe(true)
|
|
238
|
+
const out = lines.join('')
|
|
239
|
+
expect(out).toContain('GATEWAY-ONLY respawn')
|
|
240
|
+
// The log must name the side effects the break skips (the MEDIUM finding:
|
|
241
|
+
// it skips more than the resume synthetic).
|
|
242
|
+
expect(out).toContain('bridge-dead marker')
|
|
243
|
+
expect(out).toContain('crash-redelivery')
|
|
244
|
+
})
|
|
245
|
+
})
|
|
246
|
+
|
|
247
|
+
describe('markBootResumeComplete', () => {
|
|
248
|
+
it('writes the token atomically and leaves no tmp file behind', () => {
|
|
249
|
+
markBootResumeComplete(dir)
|
|
250
|
+
const p = bootResumeDonePath(dir)
|
|
251
|
+
expect(p.endsWith(BOOT_RESUME_DONE_FILE)).toBe(true)
|
|
252
|
+
expect(JSON.parse(readFileSync(p, 'utf8'))).toMatchObject({ pid: process.pid })
|
|
253
|
+
expect(existsSync(`${p}.tmp.${process.pid}`)).toBe(false)
|
|
254
|
+
})
|
|
255
|
+
|
|
256
|
+
it('never throws when the state dir is unwritable — it logs and continues', () => {
|
|
257
|
+
const lines: string[] = []
|
|
258
|
+
expect(() =>
|
|
259
|
+
markBootResumeComplete(join(dir, 'no', 'such', 'dir'), { log: (s) => lines.push(s) }),
|
|
260
|
+
).not.toThrow()
|
|
261
|
+
expect(lines.join('')).toContain(BOOT_RESUME_DONE_FILE)
|
|
262
|
+
})
|
|
263
|
+
})
|
|
264
|
+
|
|
265
|
+
describe('decideGatewayOnlyRespawn (pure)', () => {
|
|
266
|
+
const record = { pid: 42, starttime: '1000' }
|
|
267
|
+
const live = { starttime: '1000', comm: 'claude', state: 'S' }
|
|
268
|
+
|
|
269
|
+
it('rejects a comm mismatch when the record carries one', () => {
|
|
270
|
+
expect(
|
|
271
|
+
decideGatewayOnlyRespawn({
|
|
272
|
+
sentinelPresent: true,
|
|
273
|
+
record: { ...record, comm: 'claude' },
|
|
274
|
+
live: { ...live, comm: 'imposter' },
|
|
275
|
+
}),
|
|
276
|
+
).toEqual({ gatewayOnly: false, reason: 'comm-mismatch', pid: 42 })
|
|
277
|
+
})
|
|
278
|
+
|
|
279
|
+
it('never suppresses without the generation token, whatever /proc says', () => {
|
|
280
|
+
for (const l of [live, null]) {
|
|
281
|
+
expect(decideGatewayOnlyRespawn({ sentinelPresent: false, record, live: l }).gatewayOnly)
|
|
282
|
+
.toBe(false)
|
|
283
|
+
}
|
|
284
|
+
})
|
|
285
|
+
})
|
|
286
|
+
|
|
287
|
+
describe('readAgentProcessRecord', () => {
|
|
288
|
+
it('accepts the exact shape start.sh writes', () => {
|
|
289
|
+
const p = writeRecord({ pid: 7, starttime: '213153204', boot_at: 1786572493000 })
|
|
290
|
+
expect(readAgentProcessRecord(p)).toEqual({
|
|
291
|
+
pid: 7,
|
|
292
|
+
starttime: '213153204',
|
|
293
|
+
boot_at: 1786572493000,
|
|
294
|
+
})
|
|
295
|
+
})
|
|
296
|
+
|
|
297
|
+
it('rejects a nonsense pid', () => {
|
|
298
|
+
expect(readAgentProcessRecord(writeRecord({ pid: 0, starttime: '5' }))).toBeNull()
|
|
299
|
+
expect(readAgentProcessRecord(writeRecord({ pid: -1, starttime: '5' }))).toBeNull()
|
|
300
|
+
})
|
|
301
|
+
})
|
|
302
|
+
|
|
303
|
+
// ---------------------------------------------------------------------------
|
|
304
|
+
// #4648 LOW-2: the token carries the container-boot identity
|
|
305
|
+
// ---------------------------------------------------------------------------
|
|
306
|
+
|
|
307
|
+
describe('generation-token boot identity', () => {
|
|
308
|
+
it('stamps the live container-boot identity (PID 1 starttime) into the token', () => {
|
|
309
|
+
markBootResumeComplete(dir)
|
|
310
|
+
const body = JSON.parse(readFileSync(bootResumeDonePath(dir), 'utf8')) as { boot?: string }
|
|
311
|
+
expect(body.boot).toBe(containerBootIdentity()!)
|
|
312
|
+
})
|
|
313
|
+
|
|
314
|
+
it('treats a token from a DIFFERENT container boot as stale, not as a suppressor', () => {
|
|
315
|
+
// The failure this closes: start.sh's `rm -f … 2>/dev/null || true` swallows
|
|
316
|
+
// a per-file unlink failure, so a token can outlive its generation and
|
|
317
|
+
// suppress EVERY later resume — permanently and silently, since
|
|
318
|
+
// `gateway-only-respawn-no-record` needs no corroborating evidence.
|
|
319
|
+
writeFileSync(
|
|
320
|
+
bootResumeDonePath(dir),
|
|
321
|
+
JSON.stringify({ pid: 4242, at: Date.now(), boot: '1' }) + '\n',
|
|
322
|
+
)
|
|
323
|
+
const decision = detectGatewayOnlyRespawn({ stateDir: dir })
|
|
324
|
+
expect(decision.gatewayOnly).toBe(false)
|
|
325
|
+
expect(decision.reason).toBe('stale-boot-token')
|
|
326
|
+
})
|
|
327
|
+
|
|
328
|
+
it('still suppresses for a token stamped in THIS boot, with no record present', () => {
|
|
329
|
+
// The `gateway-only-respawn-no-record` path must survive the hardening:
|
|
330
|
+
// the gateway boots long before start.sh `exec`s claude.
|
|
331
|
+
markBootResumeComplete(dir)
|
|
332
|
+
expect(existsSync(join(dir, AGENT_PROCESS_RECORD_FILE))).toBe(false)
|
|
333
|
+
const decision = detectGatewayOnlyRespawn({ stateDir: dir })
|
|
334
|
+
expect(decision.gatewayOnly).toBe(true)
|
|
335
|
+
expect(decision.reason).toBe('gateway-only-respawn-no-record')
|
|
336
|
+
})
|
|
337
|
+
|
|
338
|
+
it('keeps the pre-existing behaviour when the token carries NO identity', () => {
|
|
339
|
+
// One-directional by design: only a positive MISMATCH is evidence. A
|
|
340
|
+
// legacy/identity-less token must not become a fail-open path, or the
|
|
341
|
+
// hardening would itself re-open #4641.
|
|
342
|
+
writeFileSync(bootResumeDonePath(dir), JSON.stringify({ pid: 4242, at: Date.now() }) + '\n')
|
|
343
|
+
expect(readBootResumeSentinel(bootResumeDonePath(dir))).toEqual({ present: true, stale: false })
|
|
344
|
+
expect(detectGatewayOnlyRespawn({ stateDir: dir }).gatewayOnly).toBe(true)
|
|
345
|
+
})
|
|
346
|
+
|
|
347
|
+
it('keeps the pre-existing behaviour when the token body is unparseable', () => {
|
|
348
|
+
writeFileSync(bootResumeDonePath(dir), 'not json at all')
|
|
349
|
+
expect(readBootResumeSentinel(bootResumeDonePath(dir))).toEqual({ present: true, stale: false })
|
|
350
|
+
})
|
|
351
|
+
|
|
352
|
+
it('keeps the pre-existing behaviour when /proc/1 is unreadable', () => {
|
|
353
|
+
// No live identity to compare against → no evidence → not stale.
|
|
354
|
+
writeFileSync(
|
|
355
|
+
bootResumeDonePath(dir),
|
|
356
|
+
JSON.stringify({ pid: 4242, at: Date.now(), boot: '1' }) + '\n',
|
|
357
|
+
)
|
|
358
|
+
const emptyProc = mkdtempSync(join(tmpdir(), 'noproc-'))
|
|
359
|
+
try {
|
|
360
|
+
expect(containerBootIdentity(emptyProc)).toBeNull()
|
|
361
|
+
expect(readBootResumeSentinel(bootResumeDonePath(dir), { procRoot: emptyProc }))
|
|
362
|
+
.toEqual({ present: true, stale: false })
|
|
363
|
+
} finally {
|
|
364
|
+
rmSync(emptyProc, { recursive: true, force: true })
|
|
365
|
+
}
|
|
366
|
+
})
|
|
367
|
+
|
|
368
|
+
it('reports absent when there is no token at all', () => {
|
|
369
|
+
clearToken()
|
|
370
|
+
expect(readBootResumeSentinel(bootResumeDonePath(dir))).toEqual({ present: false, stale: false })
|
|
371
|
+
})
|
|
372
|
+
})
|
|
373
|
+
|
|
374
|
+
// ---------------------------------------------------------------------------
|
|
375
|
+
// #4648 LOW-1: no env var may redirect the token/record paths
|
|
376
|
+
// ---------------------------------------------------------------------------
|
|
377
|
+
|
|
378
|
+
describe('path overrides are test-injection only (fail-open invariant)', () => {
|
|
379
|
+
it('ignores SWITCHROOM_BOOT_RESUME_DONE_FILE / SWITCHROOM_AGENT_PROCESS_FILE', () => {
|
|
380
|
+
// If these were honoured, start.sh (which hard-codes
|
|
381
|
+
// "$TELEGRAM_STATE_DIR/.boot-resume-done") would clear one path while the
|
|
382
|
+
// gateway read another: the token would never be cleared and EVERY boot
|
|
383
|
+
// would suppress its resume forever — the module's one fail-CLOSED path.
|
|
384
|
+
const decoy = mkdtempSync(join(tmpdir(), 'decoy-'))
|
|
385
|
+
const prevToken = process.env.SWITCHROOM_BOOT_RESUME_DONE_FILE
|
|
386
|
+
const prevRecord = process.env.SWITCHROOM_AGENT_PROCESS_FILE
|
|
387
|
+
try {
|
|
388
|
+
process.env.SWITCHROOM_BOOT_RESUME_DONE_FILE = join(decoy, 'token')
|
|
389
|
+
process.env.SWITCHROOM_AGENT_PROCESS_FILE = join(decoy, 'record')
|
|
390
|
+
clearToken()
|
|
391
|
+
// A token written into the decoy path must NOT be seen…
|
|
392
|
+
writeFileSync(join(decoy, 'token'), JSON.stringify({ pid: 1, at: Date.now() }) + '\n')
|
|
393
|
+
expect(shouldSkipBootResumeForGatewayOnlyRespawn(dir, { log: () => {} })).toBe(false)
|
|
394
|
+
// …and the stamp must land in stateDir, not the decoy.
|
|
395
|
+
markBootResumeComplete(dir)
|
|
396
|
+
expect(existsSync(bootResumeDonePath(dir))).toBe(true)
|
|
397
|
+
expect(shouldSkipBootResumeForGatewayOnlyRespawn(dir, { log: () => {} })).toBe(true)
|
|
398
|
+
} finally {
|
|
399
|
+
if (prevToken == null) delete process.env.SWITCHROOM_BOOT_RESUME_DONE_FILE
|
|
400
|
+
else process.env.SWITCHROOM_BOOT_RESUME_DONE_FILE = prevToken
|
|
401
|
+
if (prevRecord == null) delete process.env.SWITCHROOM_AGENT_PROCESS_FILE
|
|
402
|
+
else process.env.SWITCHROOM_AGENT_PROCESS_FILE = prevRecord
|
|
403
|
+
rmSync(decoy, { recursive: true, force: true })
|
|
404
|
+
}
|
|
405
|
+
})
|
|
406
|
+
})
|
|
@@ -128,17 +128,30 @@ describe('blocked-approval record — the off-Telegram surface', () => {
|
|
|
128
128
|
// meant the whole feature was a silent no-op in production.
|
|
129
129
|
it('falls back to the agent\'s own state dir when the shared dir is not writable', () => {
|
|
130
130
|
// Simulate the production failure: Docker auto-created the bind source as
|
|
131
|
-
// root:root, so this agent's uid
|
|
132
|
-
//
|
|
133
|
-
//
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
131
|
+
// root:root, so this agent's uid cannot write inside it.
|
|
132
|
+
//
|
|
133
|
+
// This used to be modelled as a read-only PARENT (`chmod 0555`). uid 0
|
|
134
|
+
// IGNORES file modes, so under root the shared write SUCCEEDED, the fallback
|
|
135
|
+
// branch was never taken, and this test went red for the wrong reason (the
|
|
136
|
+
// suite runs as root in the fleet's debug container). Inject the EACCES
|
|
137
|
+
// instead — deterministic for every uid, and every branch of the real
|
|
138
|
+
// `write()` still runs.
|
|
139
|
+
const unwritable = join(root, 'root-owned', 'blocked-approvals')
|
|
138
140
|
const ownDir = join(root, 'agent-state')
|
|
139
141
|
mkdirSync(ownDir, { recursive: true })
|
|
140
142
|
|
|
141
|
-
const store = createBlockedApprovalStore(unwritable, 'overlord', ownDir
|
|
143
|
+
const store = createBlockedApprovalStore(unwritable, 'overlord', ownDir, {
|
|
144
|
+
writeFileSync: ((file, data, opts) => {
|
|
145
|
+
if (String(file).startsWith(unwritable)) {
|
|
146
|
+
const e: NodeJS.ErrnoException = new Error(
|
|
147
|
+
`EACCES: permission denied, open '${String(file)}'`,
|
|
148
|
+
)
|
|
149
|
+
e.code = 'EACCES'
|
|
150
|
+
throw e
|
|
151
|
+
}
|
|
152
|
+
return writeFileSync(file, data, opts)
|
|
153
|
+
}) as typeof writeFileSync,
|
|
154
|
+
})
|
|
142
155
|
store.write(REC)
|
|
143
156
|
|
|
144
157
|
// The record MUST exist somewhere — never silently lost.
|