switchroom 0.21.7 → 0.21.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/tmp-reaper.sh +234 -0
- package/dist/agent-scheduler/index.js +1 -1
- package/dist/auth-broker/index.js +2 -2
- package/dist/cli/notion-write-pretool.mjs +1 -1
- package/dist/cli/switchroom.js +3421 -2744
- package/dist/host-control/main.js +177 -13
- package/dist/vault/approvals/kernel-server.js +2 -2
- package/dist/vault/broker/server.js +2 -2
- package/package.json +5 -4
- package/profiles/_base/start.sh.hbs +115 -0
- package/profiles/_shared/local-time.md.hbs +6 -0
- package/profiles/default/CLAUDE.md.hbs +0 -12
- package/telegram-plugin/dist/gateway/gateway.js +1017 -465
- package/telegram-plugin/gateway/agent-process-liveness.ts +558 -0
- package/telegram-plugin/gateway/approval-hold.ts +32 -1
- package/telegram-plugin/gateway/approval-outcome-sources.ts +274 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +21 -9
- package/telegram-plugin/gateway/callback-query-handlers.ts +87 -15
- package/telegram-plugin/gateway/eval-case-proposal-inbound-builders.ts +197 -0
- package/telegram-plugin/gateway/gateway.ts +12 -10
- package/telegram-plugin/gateway/pending-inbound-buffer.ts +167 -11
- package/telegram-plugin/gateway/self-improve-proposal-wiring.test.ts +333 -0
- package/telegram-plugin/gateway/self-improve-proposal-wiring.ts +152 -3
- package/telegram-plugin/gateway/subagent-handback-marker.ts +19 -0
- package/telegram-plugin/tests/agent-process-liveness.test.ts +406 -0
- package/telegram-plugin/tests/approval-hold-record.test.ts +21 -8
- package/telegram-plugin/tests/boot-resume-gateway-only-respawn.test.ts +752 -0
- package/telegram-plugin/tests/boot-resume-guard-wiring.test.ts +203 -0
- package/telegram-plugin/tests/callback-query-handlers.test.ts +143 -1
- package/telegram-plugin/tests/eval-case-proposal-inbound-builders.test.ts +144 -0
- package/telegram-plugin/tests/hermes-messages-paging.test.ts +149 -0
- package/telegram-plugin/tests/hermes-session-search.test.ts +146 -0
- package/telegram-plugin/tests/pending-inbound-buffer.test.ts +443 -2
- package/telegram-plugin/tests/subagent-handback-marker.test.ts +14 -0
|
@@ -0,0 +1,558 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Boot-resume generation guard — "did the AGENT restart, or only the gateway?"
|
|
3
|
+
* (switchroom#4641)
|
|
4
|
+
*
|
|
5
|
+
* The gateway and the `claude` agent session are SEPARATE supervised
|
|
6
|
+
* processes: `profiles/_base/start.sh.hbs` launches the gateway as a
|
|
7
|
+
* supervised sidecar and then `exec claude`. When the gateway crashes (e.g.
|
|
8
|
+
* the Bun 1.3.13 SIGBUS that motivated #4641) the supervisor respawns ONLY the
|
|
9
|
+
* gateway — the claude session, its context, its in-flight turn, and its
|
|
10
|
+
* sub-agents all keep running, untouched.
|
|
11
|
+
*
|
|
12
|
+
* The gateway's boot-resume block reads gateway-local state only, so it read
|
|
13
|
+
* that respawn as "the agent restarted": it stamped the still-executing turn
|
|
14
|
+
* `ended_via='restart'`, queued a `resume_interrupted` synthetic telling the
|
|
15
|
+
* live session "You just restarted. Your previous turn was interrupted...",
|
|
16
|
+
* and listed the still-running sub-agents as "killed by the restart".
|
|
17
|
+
*
|
|
18
|
+
* ## The mechanism: a per-container-boot generation token
|
|
19
|
+
*
|
|
20
|
+
* The boot-resume block is a ONCE-PER-CONTAINER-BOOT action. So make that
|
|
21
|
+
* literal, with a sentinel file whose lifetime IS the container generation:
|
|
22
|
+
*
|
|
23
|
+
* 1. `start.sh`, in the OUTER docker pass and BEFORE it forks the gateway,
|
|
24
|
+
* deletes `<stateDir>/.boot-resume-done` (and any stale
|
|
25
|
+
* `agent-process.json`). That deletion happens exactly once per container
|
|
26
|
+
* boot, and the gateway supervisor cannot re-run it. NOTE: that clear
|
|
27
|
+
* lives inside start.sh's `[ "$SWITCHROOM_RUNTIME" = "docker" ]` guard, so
|
|
28
|
+
* it is DOCKER-ONLY. Under the legacy systemd runtime nothing clears the
|
|
29
|
+
* token, and the guard's fail-open rests entirely on the two vetoes below
|
|
30
|
+
* (the boot-identity check and the `/proc` record). The fleet is all
|
|
31
|
+
* docker; this is stated so a systemd revival does not inherit it silently.
|
|
32
|
+
* 2. The gateway runs its boot-resume block and, once the resume inbound is
|
|
33
|
+
* DURABLY spooled, writes the sentinel (`markBootResumeComplete`). The
|
|
34
|
+
* token embeds the container-boot identity it was written under.
|
|
35
|
+
* 3. A gateway that boots and FINDS the sentinel — carrying THIS container
|
|
36
|
+
* boot's identity — knows some gateway in this same container generation
|
|
37
|
+
* already did the boot resume, so this boot is a gateway-only respawn and
|
|
38
|
+
* must skip the whole block.
|
|
39
|
+
*
|
|
40
|
+
* ## WHERE the token is stamped, and why it is NOT the tail of the block
|
|
41
|
+
*
|
|
42
|
+
* The stamp must follow the point at which the resume becomes CRASH-SURVIVABLE,
|
|
43
|
+
* not the point at which the block stops running. Those are ~8k lines apart in
|
|
44
|
+
* gateway.ts: the `bootResumeInit` block only builds `bootResumeInbound` in
|
|
45
|
+
* MEMORY; the inbound becomes durable much later, at
|
|
46
|
+
* `inboundSpool.put(bootResumeInbound.agent, bootResumeInbound.msg)`. So
|
|
47
|
+
* `markBootResumeComplete` is called immediately AFTER that put (and after
|
|
48
|
+
* `markTurnResumed`, which obeys the identical rule — see turns-schema.ts:
|
|
49
|
+
* "the caller must stamp only AFTER the resume inbound is durably spooled").
|
|
50
|
+
*
|
|
51
|
+
* Stamping at the tail of the block instead would create a NEW, silent
|
|
52
|
+
* work-loss mode, worse than the bug this module fixes. On a genuine restart
|
|
53
|
+
* the reaper durably stamps the interrupted turn `ended_via='restart'`; if the
|
|
54
|
+
* gateway then dies before the durable put — the Bun-crash-at-boot pattern
|
|
55
|
+
* #4641 exists for, and also the `acquireStartupLock` → `process.exit(1)`
|
|
56
|
+
* path, both of which sit inside that window — the successor would find a
|
|
57
|
+
* token, find no `agent-process.json` (start.sh has not reached `exec claude`;
|
|
58
|
+
* its inner pass waits on the LiteLLM probe first), return
|
|
59
|
+
* `gateway-only-respawn-no-record` and suppress. The turn stays
|
|
60
|
+
* `ended_via='restart'` with `resumed_at` NULL and is never resumed.
|
|
61
|
+
*
|
|
62
|
+
* Stamping after the put moves that window the other way, which is safe BY
|
|
63
|
+
* CONSTRUCTION: a successor that re-runs the block re-mints the same resume,
|
|
64
|
+
* `resumed_at` is still NULL so the turn is still findable, and the spool
|
|
65
|
+
* dedups on `s:resume:<resume_turn_key>` — so the retry is idempotent, not a
|
|
66
|
+
* double-send. Lose the token, repeat the work; never lose the work.
|
|
67
|
+
*
|
|
68
|
+
* The stamp does not depend on there being anything to resume: a boot that
|
|
69
|
+
* found nothing still completed this generation's boot resume, and a later
|
|
70
|
+
* gateway-only respawn must still be suppressed — that respawn running the
|
|
71
|
+
* reaper against a meanwhile-started live turn IS bug #4641.
|
|
72
|
+
*
|
|
73
|
+
* It DOES depend on the block not having thrown. gateway.ts's boot block ends
|
|
74
|
+
* in a `catch` that logs, sets `turnsDb = null` and lets module init continue
|
|
75
|
+
* — a swallow, not a death. Without a guard that path reaches the stamp with
|
|
76
|
+
* `bootResumeInbound` still null, and it is reachable from ~156 lines of DB
|
|
77
|
+
* and fs I/O between the reaper and the first assignment
|
|
78
|
+
* (`findLatestTurnIfInterrupted`, the clean-shutdown-marker read,
|
|
79
|
+
* `listNonTerminalSubagentsForTurn`, the synthetic builders). The reaper has
|
|
80
|
+
* ALREADY durably written `ended_via='restart'` by then, so a token stamped on
|
|
81
|
+
* that path is exactly the tail-of-block failure above reached through a
|
|
82
|
+
* different door: gateway respawns, successor sees a this-generation token and
|
|
83
|
+
* no `agent-process.json`, suppresses, and the turn is never resumed. So
|
|
84
|
+
* gateway.ts carries a `bootResumeThrew` flag set in that `catch` and the
|
|
85
|
+
* stamp is gated on it — a swallowed throw leaves NO token, and the next
|
|
86
|
+
* gateway retries the boot resume. Fail open, as everywhere else here.
|
|
87
|
+
*
|
|
88
|
+
* (On `origin/main` a throw in that window also loses the boot's resume; what
|
|
89
|
+
* the guard preserves is the accidental recovery-on-respawn that the token
|
|
90
|
+
* would otherwise remove.)
|
|
91
|
+
*
|
|
92
|
+
* This is deliberately NOT timing-derived. An earlier revision of this module
|
|
93
|
+
* compared `/proc` starttimes ("the agent must predate me") and justified it
|
|
94
|
+
* with the claim that start.sh writes the record after forking the gateway, so
|
|
95
|
+
* the recorded process necessarily starts later. That reasoning was FALSE: the
|
|
96
|
+
* recorded pid is `$$`, and the shell that forks the gateway
|
|
97
|
+
* (`start.sh.hbs`, outer pass) obviously predates it. What actually made the
|
|
98
|
+
* ordering hold was an incidental, undocumented detail — the docker tmux
|
|
99
|
+
* re-exec (`start.sh.hbs`: `exec tmux ... bash -l "$0"`), which runs the inner
|
|
100
|
+
* pass in a FRESH shell forked by the tmux server. Measured on a live
|
|
101
|
+
* container the margin was one clock tick (gateway starttime 209640551, claude
|
|
102
|
+
* 209640552 — 10ms). Correct by accident, with nothing pinning the accident.
|
|
103
|
+
* The sentinel replaces that with a fact the boot sequence establishes.
|
|
104
|
+
*
|
|
105
|
+
* It also fixes a hole the starttime comparison could not see: a gateway that
|
|
106
|
+
* CRASHES DURING ITS OWN BOOT (exactly the Bun-crash-at-boot pattern this fix
|
|
107
|
+
* exists for) left a genuinely-interrupted turn unreaped forever, because the
|
|
108
|
+
* respawned gateway saw a live claude that predated it and suppressed. With
|
|
109
|
+
* the sentinel, gateway #1 never got far enough to write it, so gateway #2
|
|
110
|
+
* runs the full boot resume — the resume survives the crash.
|
|
111
|
+
*
|
|
112
|
+
* ## The `/proc` record is a VETO, not the decision
|
|
113
|
+
*
|
|
114
|
+
* `start.sh` still publishes `<stateDir>/agent-process.json` — the agent
|
|
115
|
+
* process's pid plus `/proc/<pid>/stat` field 22 (starttime) — immediately
|
|
116
|
+
* before `exec claude` (`exec` replaces the shell without forking, so both
|
|
117
|
+
* values survive into claude unchanged). It is no longer what decides
|
|
118
|
+
* suppression; it can only VETO one. If the sentinel says "already done" but
|
|
119
|
+
* the recorded agent process is provably gone, we run the boot resume anyway.
|
|
120
|
+
* That can only ever re-enable a resume, never suppress a legitimate one.
|
|
121
|
+
*
|
|
122
|
+
* Identity is the (pid, starttime) PAIR, never pid alone: PIDs are reused and
|
|
123
|
+
* a container restart resets the PID namespace, so a stale record's pid very
|
|
124
|
+
* plausibly names a live-but-different process. A zombie counts as dead.
|
|
125
|
+
*
|
|
126
|
+
* ## FAIL-OPEN is the invariant
|
|
127
|
+
*
|
|
128
|
+
* No sentinel → run the boot resume (the pre-#4641 behaviour). Sentinel from a
|
|
129
|
+
* DIFFERENT container boot → run it. Recorded process dead or mismatched → run
|
|
130
|
+
* it. Probe throws → run it. Plus a `SWITCHROOM_GATEWAY_RESPAWN_GUARD=0` ops
|
|
131
|
+
* escape hatch. Suppressing a real resume loses work; an extra resume on an
|
|
132
|
+
* already-restarted agent is merely the status quo.
|
|
133
|
+
*
|
|
134
|
+
* The invariant admits no path-override env vars. An earlier revision let
|
|
135
|
+
* `SWITCHROOM_BOOT_RESUME_DONE_FILE` / `SWITCHROOM_AGENT_PROCESS_FILE`
|
|
136
|
+
* redirect the token and record paths from the live process env, while
|
|
137
|
+
* start.sh hard-codes `"$TELEGRAM_STATE_DIR/.boot-resume-done"`. Setting
|
|
138
|
+
* either in a container would have made start.sh clear one path and the
|
|
139
|
+
* gateway read another — the token then never cleared, and EVERY boot
|
|
140
|
+
* suppressing its resume forever. That is the one fail-CLOSED direction the
|
|
141
|
+
* module can take, so the overrides are confined to `DetectOpts` (test
|
|
142
|
+
* injection). Verified: nothing in the repo ever set them.
|
|
143
|
+
*
|
|
144
|
+
* ## Why the token carries the container-boot identity
|
|
145
|
+
*
|
|
146
|
+
* `gateway-only-respawn-no-record` is the only branch that suppresses on the
|
|
147
|
+
* token ALONE, with no corroborating `/proc` evidence — and start.sh's clear is
|
|
148
|
+
* `rm -f … 2>/dev/null || true`, which swallows a per-file failure (EROFS,
|
|
149
|
+
* EACCES, an immutable attr) while the sibling `agent-process.json` unlink
|
|
150
|
+
* succeeds. A token surviving into the next container generation would then
|
|
151
|
+
* suppress a genuine restart's resume, permanently and silently.
|
|
152
|
+
*
|
|
153
|
+
* So the token records the container-boot identity it was written under: PID
|
|
154
|
+
* 1's `/proc` starttime. PID 1 is the container's entry process — it lives for
|
|
155
|
+
* exactly one container generation and a restart replaces it, which is the
|
|
156
|
+
* property we need. (`/proc/sys/kernel/random/boot_id` is NOT usable here: it
|
|
157
|
+
* identifies the HOST kernel boot, which containers share and which survives a
|
|
158
|
+
* container restart.) A token naming a DIFFERENT boot is self-evidently stale
|
|
159
|
+
* and is ignored.
|
|
160
|
+
*
|
|
161
|
+
* The check is deliberately one-directional — only a MISMATCH is evidence.
|
|
162
|
+
* A token with no recorded identity, or an unreadable `/proc/1`, keeps the
|
|
163
|
+
* pre-existing "present ⇒ suppress" behaviour, so the hardening can never
|
|
164
|
+
* itself re-open #4641.
|
|
165
|
+
*
|
|
166
|
+
* ## What `break bootResumeInit` skips, and why each is safe
|
|
167
|
+
*
|
|
168
|
+
* The guard in gateway.ts breaks out of the whole labelled boot block, which
|
|
169
|
+
* skips MORE than the resume synthetic. Because the sentinel means "a gateway
|
|
170
|
+
* in THIS container generation already ran this block to completion", every
|
|
171
|
+
* one of these was already done exactly once this generation:
|
|
172
|
+
*
|
|
173
|
+
* - the orphan-turn reaper (`markOrphanedWithTimeoutClassification`) — the
|
|
174
|
+
* thing that stamped the live turn `ended_via='restart'`;
|
|
175
|
+
* - `consumeBridgeDeadEscalationMarker` — its comment states the marker is
|
|
176
|
+
* "consumed (always cleared) whether or not a turn was in flight", and the
|
|
177
|
+
* gateway that wrote the sentinel is the one that consumed it. Re-running
|
|
178
|
+
* it here would find nothing; skipping it preserves the #3038 cross-boot
|
|
179
|
+
* escalation damper instead of double-counting or resetting it. If that
|
|
180
|
+
* gateway died BEFORE consuming, the sentinel is absent and we do not take
|
|
181
|
+
* this path at all;
|
|
182
|
+
* - the bridge-dead IDLE notice — same generation, same marker, already
|
|
183
|
+
* surfaced (or already declined for want of an allowFrom chat);
|
|
184
|
+
* - `bridgeDeadPriorStreak` — the #3038 cross-boot damper seed derived from
|
|
185
|
+
* that marker. Left at 0 on a respawn: the streak belongs to the boot that
|
|
186
|
+
* consumed the marker, and re-seeding it here would double-count;
|
|
187
|
+
* - the `pendingRedelivery` capture — a crash-redelivery candidate for the
|
|
188
|
+
* interrupted turn. Skipped deliberately: on a gateway-only respawn the
|
|
189
|
+
* turn is STILL RUNNING and will emit its own answer, so re-sending a
|
|
190
|
+
* "Recovered from an interrupted turn:" draft would double-send;
|
|
191
|
+
* - `writePendingTurnEnv` — passive wake-audit context for a restart that
|
|
192
|
+
* did not happen.
|
|
193
|
+
*
|
|
194
|
+
* (The `.wake-audit-pending` sentinel needs no handling: start.sh drops it
|
|
195
|
+
* once per CONTAINER boot and start.sh does not re-run on a gateway respawn.)
|
|
196
|
+
*/
|
|
197
|
+
|
|
198
|
+
import { existsSync, readFileSync, writeFileSync, renameSync, unlinkSync, readdirSync } from 'node:fs'
|
|
199
|
+
import { join, dirname, basename } from 'node:path'
|
|
200
|
+
|
|
201
|
+
/** Default basename of the record start.sh writes into the telegram state dir. */
|
|
202
|
+
export const AGENT_PROCESS_RECORD_FILE = 'agent-process.json'
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* Per-container-boot generation token. Deleted by start.sh in the outer docker
|
|
206
|
+
* pass BEFORE the gateway is forked; written by the gateway once its
|
|
207
|
+
* boot-resume block completes. Its presence means "this container generation's
|
|
208
|
+
* boot resume already happened".
|
|
209
|
+
*/
|
|
210
|
+
export const BOOT_RESUME_DONE_FILE = '.boot-resume-done'
|
|
211
|
+
|
|
212
|
+
/** The agent-process identity recorded by start.sh before `exec claude`. */
|
|
213
|
+
export interface AgentProcessRecord {
|
|
214
|
+
/** PID of the shell that `exec`s claude — i.e. the claude process's own pid. */
|
|
215
|
+
pid: number
|
|
216
|
+
/** `/proc/<pid>/stat` field 22 (starttime, clock ticks since host boot). */
|
|
217
|
+
starttime: string
|
|
218
|
+
/**
|
|
219
|
+
* `/proc/<pid>/stat` field 2 (comm) at record time. Optional extra veto.
|
|
220
|
+
* start.sh does NOT write it (it is "bash" pre-exec and claude's after), and
|
|
221
|
+
* with the generation token it buys nothing — but a mismatch, if a future
|
|
222
|
+
* writer does record it post-exec, still fails open.
|
|
223
|
+
*/
|
|
224
|
+
comm?: string
|
|
225
|
+
/** Wall-clock ms when the record was written. Diagnostic only. */
|
|
226
|
+
boot_at?: number
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/** A live process's identity as read back from `/proc`. */
|
|
230
|
+
export interface ProcIdentity {
|
|
231
|
+
starttime: string
|
|
232
|
+
comm: string
|
|
233
|
+
/** `/proc/<pid>/stat` field 3. `Z` is an unreaped corpse, not a live agent. */
|
|
234
|
+
state: string
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
export type RespawnReason =
|
|
238
|
+
| 'gateway-only-respawn'
|
|
239
|
+
| 'gateway-only-respawn-no-record'
|
|
240
|
+
| 'guard-disabled'
|
|
241
|
+
| 'no-boot-resume-sentinel'
|
|
242
|
+
| 'stale-boot-token'
|
|
243
|
+
| 'agent-process-dead'
|
|
244
|
+
| 'starttime-mismatch'
|
|
245
|
+
| 'comm-mismatch'
|
|
246
|
+
|
|
247
|
+
export interface RespawnDecision {
|
|
248
|
+
/**
|
|
249
|
+
* True ONLY when this container generation's boot resume is already done and
|
|
250
|
+
* nothing contradicts it — i.e. only the gateway respawned. The boot-resume
|
|
251
|
+
* block must then do nothing at all.
|
|
252
|
+
*/
|
|
253
|
+
gatewayOnly: boolean
|
|
254
|
+
reason: RespawnReason
|
|
255
|
+
/** Recorded pid, when a record was readable. Diagnostic. */
|
|
256
|
+
pid: number | null
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Parse `/proc/<pid>/stat` for (comm, state, starttime).
|
|
261
|
+
*
|
|
262
|
+
* `comm` is parenthesised and may itself contain spaces and `)`, so the field
|
|
263
|
+
* split is anchored on the LAST `) ` — the documented way to parse this file.
|
|
264
|
+
* Field 3 (state) is the first field after comm; field 22 (starttime) the 20th.
|
|
265
|
+
*/
|
|
266
|
+
export function parseProcStat(raw: string): ProcIdentity | null {
|
|
267
|
+
const close = raw.lastIndexOf(') ')
|
|
268
|
+
const open = raw.indexOf(' (')
|
|
269
|
+
if (close < 0 || open < 0 || close < open) return null
|
|
270
|
+
const comm = raw.slice(open + 2, close)
|
|
271
|
+
const rest = raw.slice(close + 2).trim().split(/\s+/)
|
|
272
|
+
const state = rest[0]
|
|
273
|
+
const starttime = rest[19]
|
|
274
|
+
if (state == null || state.length === 0) return null
|
|
275
|
+
if (starttime == null || !/^\d+$/.test(starttime)) return null
|
|
276
|
+
return { comm, state, starttime }
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Read a live process's identity, or null when it is gone.
|
|
281
|
+
*
|
|
282
|
+
* A zombie (`state === 'Z'`) counts as GONE: the agent has exited and only an
|
|
283
|
+
* unreaped exit status remains, so its turn really was interrupted.
|
|
284
|
+
*/
|
|
285
|
+
export function readProcIdentity(pid: number, procRoot = '/proc'): ProcIdentity | null {
|
|
286
|
+
if (!Number.isInteger(pid) || pid <= 0) return null
|
|
287
|
+
let identity: ProcIdentity | null
|
|
288
|
+
try {
|
|
289
|
+
identity = parseProcStat(readFileSync(join(procRoot, String(pid), 'stat'), 'utf8'))
|
|
290
|
+
} catch {
|
|
291
|
+
return null // process gone, or /proc unreadable — treat as dead (fail-open)
|
|
292
|
+
}
|
|
293
|
+
if (identity != null && identity.state === 'Z') return null
|
|
294
|
+
return identity
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** Read + validate the record start.sh wrote. Null on absent/torn/invalid. */
|
|
298
|
+
export function readAgentProcessRecord(path: string): AgentProcessRecord | null {
|
|
299
|
+
let parsed: unknown
|
|
300
|
+
try {
|
|
301
|
+
parsed = JSON.parse(readFileSync(path, 'utf8'))
|
|
302
|
+
} catch {
|
|
303
|
+
return null
|
|
304
|
+
}
|
|
305
|
+
if (parsed == null || typeof parsed !== 'object') return null
|
|
306
|
+
const o = parsed as Record<string, unknown>
|
|
307
|
+
const pid = typeof o.pid === 'number' ? o.pid : Number(o.pid)
|
|
308
|
+
const starttime = typeof o.starttime === 'string' ? o.starttime : String(o.starttime ?? '')
|
|
309
|
+
if (!Number.isInteger(pid) || pid <= 0) return null
|
|
310
|
+
if (!/^\d+$/.test(starttime)) return null
|
|
311
|
+
const rec: AgentProcessRecord = { pid, starttime }
|
|
312
|
+
if (typeof o.comm === 'string' && o.comm.length > 0) rec.comm = o.comm
|
|
313
|
+
if (typeof o.boot_at === 'number' && Number.isFinite(o.boot_at)) rec.boot_at = o.boot_at
|
|
314
|
+
return rec
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/** Resolve the generation-token path for a gateway STATE_DIR. */
|
|
318
|
+
export function bootResumeDonePath(stateDir: string): string {
|
|
319
|
+
return join(stateDir, BOOT_RESUME_DONE_FILE)
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/** On-disk shape of the generation token. All fields are best-effort. */
|
|
323
|
+
export interface BootResumeToken {
|
|
324
|
+
/** PID of the gateway that stamped it. Diagnostic. */
|
|
325
|
+
pid?: number
|
|
326
|
+
/** Wall-clock ms at stamp time. Diagnostic. */
|
|
327
|
+
at?: number
|
|
328
|
+
/** Container-boot identity (PID 1's starttime) at stamp time. */
|
|
329
|
+
boot?: string
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* This container generation's identity: PID 1's `/proc` starttime.
|
|
334
|
+
*
|
|
335
|
+
* PID 1 is the container's entry process — one per container generation,
|
|
336
|
+
* replaced on restart. Null when `/proc/1` is unreadable, which callers must
|
|
337
|
+
* treat as "no evidence", never as a mismatch.
|
|
338
|
+
*/
|
|
339
|
+
export function containerBootIdentity(procRoot = '/proc'): string | null {
|
|
340
|
+
return readProcIdentity(1, procRoot)?.starttime ?? null
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
export interface SentinelState {
|
|
344
|
+
/** The token file exists (whatever its contents). */
|
|
345
|
+
present: boolean
|
|
346
|
+
/** Present AND provably written under a DIFFERENT container boot. */
|
|
347
|
+
stale: boolean
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* Read the generation token and judge whether it belongs to THIS container
|
|
352
|
+
* boot. Never throws.
|
|
353
|
+
*
|
|
354
|
+
* Only a positive mismatch marks the token stale. An absent/blank/legacy
|
|
355
|
+
* `boot` field, an unparseable body, or an unreadable `/proc/1` all leave
|
|
356
|
+
* `stale: false` — so the hardening can only ever ADD a fail-open path, never
|
|
357
|
+
* suppress where the previous revision would not have.
|
|
358
|
+
*/
|
|
359
|
+
export function readBootResumeSentinel(
|
|
360
|
+
path: string,
|
|
361
|
+
opts: { procRoot?: string } = {},
|
|
362
|
+
): SentinelState {
|
|
363
|
+
let raw: string
|
|
364
|
+
try {
|
|
365
|
+
if (!existsSync(path)) return { present: false, stale: false }
|
|
366
|
+
raw = readFileSync(path, 'utf8')
|
|
367
|
+
} catch {
|
|
368
|
+
return { present: false, stale: false } // unreadable → fail open
|
|
369
|
+
}
|
|
370
|
+
let stamped: string | null = null
|
|
371
|
+
try {
|
|
372
|
+
const parsed = JSON.parse(raw) as BootResumeToken | null
|
|
373
|
+
if (parsed != null && typeof parsed === 'object' && typeof parsed.boot === 'string' && parsed.boot.length > 0) {
|
|
374
|
+
stamped = parsed.boot
|
|
375
|
+
}
|
|
376
|
+
} catch { /* legacy/foreign body — no identity evidence, treat as present */ }
|
|
377
|
+
const live = containerBootIdentity(opts.procRoot)
|
|
378
|
+
if (stamped != null && live != null && stamped !== live) return { present: true, stale: true }
|
|
379
|
+
return { present: true, stale: false }
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* Delete `<file>.tmp.<pid>` siblings left behind by a writer that died between
|
|
384
|
+
* `writeFileSync` and `renameSync`.
|
|
385
|
+
*
|
|
386
|
+
* The tmp name is pid-qualified (two writers must not share one tmp path), so
|
|
387
|
+
* a crashed write leaves a stray that nothing else removes: start.sh's cleanup
|
|
388
|
+
* (`start.sh.hbs`) `rm -f`s the two exact filenames with no glob, so strays
|
|
389
|
+
* would accumulate in STATE_DIR one per crashed generation, forever.
|
|
390
|
+
*
|
|
391
|
+
* Called only after our own successful rename, so it never races our own tmp.
|
|
392
|
+
* Best-effort and never throws — a stray costs bytes, not correctness.
|
|
393
|
+
*/
|
|
394
|
+
function sweepStrayTmps(path: string): void {
|
|
395
|
+
const prefix = `${basename(path)}.tmp.`
|
|
396
|
+
try {
|
|
397
|
+
for (const name of readdirSync(dirname(path))) {
|
|
398
|
+
if (!name.startsWith(prefix)) continue
|
|
399
|
+
try { unlinkSync(join(dirname(path), name)) } catch { /* best effort */ }
|
|
400
|
+
}
|
|
401
|
+
} catch { /* unreadable dir — nothing to sweep */ }
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/**
|
|
405
|
+
* Stamp the generation token: THIS container generation's boot-resume block
|
|
406
|
+
* ran to completion, so a later gateway respawn must not re-run it.
|
|
407
|
+
*
|
|
408
|
+
* Called from gateway.ts immediately AFTER the boot-resume inbound is durably
|
|
409
|
+
* spooled — NOT at the end of the boot block; see "WHERE the token is stamped"
|
|
410
|
+
* above for why those are different places and why the difference is a
|
|
411
|
+
* work-loss bug. Atomic tmp+rename, and never throws: failing to stamp only
|
|
412
|
+
* costs a duplicate boot-resume on a respawn, which is the pre-#4641
|
|
413
|
+
* behaviour.
|
|
414
|
+
*
|
|
415
|
+
* The body records the container-boot identity so a token that outlives its
|
|
416
|
+
* generation (start.sh's `rm -f … || true` swallowing a per-file failure) is
|
|
417
|
+
* self-evidently stale rather than a permanent resume suppressor.
|
|
418
|
+
*/
|
|
419
|
+
export function markBootResumeComplete(
|
|
420
|
+
stateDir: string,
|
|
421
|
+
opts: { path?: string; now?: number; procRoot?: string; log?: (s: string) => void } = {},
|
|
422
|
+
): void {
|
|
423
|
+
const path = opts.path ?? bootResumeDonePath(stateDir)
|
|
424
|
+
const tmp = `${path}.tmp.${process.pid}`
|
|
425
|
+
const boot = containerBootIdentity(opts.procRoot)
|
|
426
|
+
try {
|
|
427
|
+
const token: BootResumeToken = { pid: process.pid, at: opts.now ?? Date.now() }
|
|
428
|
+
if (boot != null) token.boot = boot
|
|
429
|
+
writeFileSync(tmp, JSON.stringify(token) + '\n')
|
|
430
|
+
renameSync(tmp, path)
|
|
431
|
+
sweepStrayTmps(path)
|
|
432
|
+
} catch (err) {
|
|
433
|
+
try { unlinkSync(tmp) } catch { /* best effort */ }
|
|
434
|
+
const log = opts.log ?? ((s: string) => process.stderr.write(s))
|
|
435
|
+
log(
|
|
436
|
+
`telegram gateway: boot: could not stamp ${BOOT_RESUME_DONE_FILE} ` +
|
|
437
|
+
`(${(err as Error).message}) — a gateway respawn may repeat the boot resume (#4641)\n`,
|
|
438
|
+
)
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
/**
|
|
443
|
+
* The decision, pure.
|
|
444
|
+
*
|
|
445
|
+
* `sentinelPresent` is the generation token — the ONLY thing that can say
|
|
446
|
+
* "suppress". `sentinelStale` and `live` (the `/proc` identity read back for
|
|
447
|
+
* `record.pid`, null when that pid is gone) can only VETO a suppression.
|
|
448
|
+
*/
|
|
449
|
+
export function decideGatewayOnlyRespawn(input: {
|
|
450
|
+
sentinelPresent: boolean
|
|
451
|
+
/** Token exists but names a different container boot. Defaults false. */
|
|
452
|
+
sentinelStale?: boolean
|
|
453
|
+
record: AgentProcessRecord | null
|
|
454
|
+
live: ProcIdentity | null
|
|
455
|
+
}): RespawnDecision {
|
|
456
|
+
const { sentinelPresent, sentinelStale, record, live } = input
|
|
457
|
+
// No token → no gateway in this container generation has completed the boot
|
|
458
|
+
// resume yet. That covers a genuine container restart, a first boot, AND a
|
|
459
|
+
// gateway that crashed during its own boot before finishing the block.
|
|
460
|
+
if (!sentinelPresent) {
|
|
461
|
+
return { gatewayOnly: false, reason: 'no-boot-resume-sentinel', pid: record?.pid ?? null }
|
|
462
|
+
}
|
|
463
|
+
// A token start.sh's `rm -f` failed to clear: it names a PREVIOUS container
|
|
464
|
+
// boot, so it proves nothing about this generation. Run the boot resume.
|
|
465
|
+
if (sentinelStale === true) {
|
|
466
|
+
return { gatewayOnly: false, reason: 'stale-boot-token', pid: record?.pid ?? null }
|
|
467
|
+
}
|
|
468
|
+
// Token present but start.sh has not yet published the agent record (the
|
|
469
|
+
// gateway boots long before `exec claude`). The token alone is proof enough:
|
|
470
|
+
// re-running the block would duplicate this generation's resume.
|
|
471
|
+
if (record == null) {
|
|
472
|
+
return { gatewayOnly: true, reason: 'gateway-only-respawn-no-record', pid: null }
|
|
473
|
+
}
|
|
474
|
+
if (live == null) return { gatewayOnly: false, reason: 'agent-process-dead', pid: record.pid }
|
|
475
|
+
// PID reuse guard: same pid, different process.
|
|
476
|
+
if (live.starttime !== record.starttime) {
|
|
477
|
+
return { gatewayOnly: false, reason: 'starttime-mismatch', pid: record.pid }
|
|
478
|
+
}
|
|
479
|
+
if (record.comm != null && record.comm !== live.comm) {
|
|
480
|
+
return { gatewayOnly: false, reason: 'comm-mismatch', pid: record.pid }
|
|
481
|
+
}
|
|
482
|
+
return { gatewayOnly: true, reason: 'gateway-only-respawn', pid: record.pid }
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
export interface DetectOpts {
|
|
486
|
+
/** Gateway STATE_DIR (`<agentDir>/telegram`). The record + token live here. */
|
|
487
|
+
stateDir: string
|
|
488
|
+
/**
|
|
489
|
+
* Override the record path. TEST INJECTION ONLY — deliberately not readable
|
|
490
|
+
* from the process env: start.sh hard-codes the production paths, so an env
|
|
491
|
+
* override could only ever desynchronise the writer from the reader. See
|
|
492
|
+
* "FAIL-OPEN is the invariant".
|
|
493
|
+
*/
|
|
494
|
+
recordPath?: string
|
|
495
|
+
/** Override the generation-token path. Test injection only (see above). */
|
|
496
|
+
sentinelPath?: string
|
|
497
|
+
/** `/proc` root; injectable for tests. */
|
|
498
|
+
procRoot?: string
|
|
499
|
+
/** Set false to force the pre-#4641 behaviour (ops escape hatch). */
|
|
500
|
+
enabled?: boolean
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/** Read the token, the record and `/proc`, and decide. Never throws. */
|
|
504
|
+
export function detectGatewayOnlyRespawn(opts: DetectOpts): RespawnDecision {
|
|
505
|
+
if (opts.enabled === false) return { gatewayOnly: false, reason: 'guard-disabled', pid: null }
|
|
506
|
+
const procRoot = opts.procRoot ?? '/proc'
|
|
507
|
+
const sentinel = readBootResumeSentinel(
|
|
508
|
+
opts.sentinelPath ?? bootResumeDonePath(opts.stateDir),
|
|
509
|
+
{ procRoot },
|
|
510
|
+
)
|
|
511
|
+
const recordPath = opts.recordPath ?? join(opts.stateDir, AGENT_PROCESS_RECORD_FILE)
|
|
512
|
+
const record = readAgentProcessRecord(recordPath)
|
|
513
|
+
const live = record == null ? null : readProcIdentity(record.pid, procRoot)
|
|
514
|
+
return decideGatewayOnlyRespawn({
|
|
515
|
+
sentinelPresent: sentinel.present,
|
|
516
|
+
sentinelStale: sentinel.stale,
|
|
517
|
+
record,
|
|
518
|
+
live,
|
|
519
|
+
})
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
/**
|
|
523
|
+
* Gateway boot entry point: decide, log the decision, and answer the one
|
|
524
|
+
* question the boot block asks — "should I skip the whole boot-resume path?".
|
|
525
|
+
*
|
|
526
|
+
* Called from gateway.ts BEFORE the orphan-turn reaper, so a gateway-only
|
|
527
|
+
* respawn touches nothing. See "What `break bootResumeInit` skips" above for
|
|
528
|
+
* the full accounting of the skipped side effects.
|
|
529
|
+
*/
|
|
530
|
+
export function shouldSkipBootResumeForGatewayOnlyRespawn(
|
|
531
|
+
stateDir: string,
|
|
532
|
+
opts: Omit<DetectOpts, 'stateDir'> & { log?: (s: string) => void } = {},
|
|
533
|
+
): boolean {
|
|
534
|
+
const log = opts.log ?? ((s: string) => process.stderr.write(s))
|
|
535
|
+
let decision: RespawnDecision
|
|
536
|
+
try {
|
|
537
|
+
decision = detectGatewayOnlyRespawn({
|
|
538
|
+
...opts,
|
|
539
|
+
stateDir,
|
|
540
|
+
enabled: opts.enabled ?? process.env.SWITCHROOM_GATEWAY_RESPAWN_GUARD !== '0',
|
|
541
|
+
})
|
|
542
|
+
} catch (err) {
|
|
543
|
+
// Fail-open: any surprise keeps the pre-#4641 behaviour.
|
|
544
|
+
log(`telegram gateway: boot: agent-liveness probe failed (${(err as Error).message}) — assuming agent restart\n`)
|
|
545
|
+
return false
|
|
546
|
+
}
|
|
547
|
+
if (decision.gatewayOnly) {
|
|
548
|
+
log(
|
|
549
|
+
`telegram gateway: boot: GATEWAY-ONLY respawn — this container generation's boot resume is already ` +
|
|
550
|
+
`done (${BOOT_RESUME_DONE_FILE} present, reason=${decision.reason}, agent pid=${decision.pid ?? 'unrecorded'}). ` +
|
|
551
|
+
`Skipping the orphan-turn reaper, the bridge-dead marker, the resume synthetic, crash-redelivery capture ` +
|
|
552
|
+
`and .pending-turn.env (#4641): the claude session, its in-flight turn and its sub-agents were never interrupted.\n`,
|
|
553
|
+
)
|
|
554
|
+
return true
|
|
555
|
+
}
|
|
556
|
+
log(`telegram gateway: boot: agent-liveness probe → treating as agent restart (reason=${decision.reason})\n`)
|
|
557
|
+
return false
|
|
558
|
+
}
|
|
@@ -233,6 +233,34 @@ export function safeActionForRecord(
|
|
|
233
233
|
: naturalAction(toolName, undefined)
|
|
234
234
|
}
|
|
235
235
|
|
|
236
|
+
/**
|
|
237
|
+
* The one fs call whose FAILURE selects the fallback branch, injectable.
|
|
238
|
+
*
|
|
239
|
+
* The production trigger is "the shared dir is not writable by this uid" — and
|
|
240
|
+
* that is exactly the condition a test cannot create for itself, because a test
|
|
241
|
+
* run as **root ignores file modes**. The suite runs as uid 0 in the fleet's
|
|
242
|
+
* debug container (and in any `docker run` without `--user`), where a
|
|
243
|
+
* `chmod 0500` on the parent is a no-op: the shared write SUCCEEDS, the fallback
|
|
244
|
+
* branch is never taken, and every assertion about the fallback record silently
|
|
245
|
+
* describes the SHARED record instead. Two tests passed that way for real
|
|
246
|
+
* (`is written world-readable`, `clears both locations`) — vacuous guards that
|
|
247
|
+
* report coverage they do not provide.
|
|
248
|
+
*
|
|
249
|
+
* A filesystem artifact can't fix that either: anything planted in the shared
|
|
250
|
+
* dir to make the write fail (an `<agent>.json` DIRECTORY → EISDIR, a symlink →
|
|
251
|
+
* the lstat guard below) is also visible to the READER, which then counts it as
|
|
252
|
+
* an unreadable record — changing the very thing the test measures. So the
|
|
253
|
+
* refusal is injected instead: the real `write()` branch logic, the lstat guard,
|
|
254
|
+
* the fallback selection, the stale-unlink and the stderr log all still run.
|
|
255
|
+
*
|
|
256
|
+
* Production callers never pass this.
|
|
257
|
+
*/
|
|
258
|
+
export interface BlockedApprovalStoreDeps {
|
|
259
|
+
/** Defaults to `node:fs`'s. Tests substitute one that throws EACCES for the
|
|
260
|
+
* shared path, reproducing a root-owned bind under a per-agent uid. */
|
|
261
|
+
writeFileSync?: typeof writeFileSync
|
|
262
|
+
}
|
|
263
|
+
|
|
236
264
|
export interface BlockedApprovalStore {
|
|
237
265
|
/** Write (or replace) the blocked record for this agent. */
|
|
238
266
|
write(rec: BlockedApprovalRecord): void
|
|
@@ -288,7 +316,10 @@ export function createBlockedApprovalStore(
|
|
|
288
316
|
* which must stay in sync with the filename below).
|
|
289
317
|
*/
|
|
290
318
|
fallbackDir?: string,
|
|
319
|
+
/** Test-only fs seam — see {@link BlockedApprovalStoreDeps}. */
|
|
320
|
+
deps: BlockedApprovalStoreDeps = {},
|
|
291
321
|
): BlockedApprovalStore {
|
|
322
|
+
const write_ = deps.writeFileSync ?? writeFileSync
|
|
292
323
|
const primary = join(dir, `${agent}.json`)
|
|
293
324
|
// Keep in sync with FALLBACK_RECORD_NAME in src/web/blocked-approvals-read.ts —
|
|
294
325
|
// the reader matches this filename exactly. Pinned by the fallback-contract
|
|
@@ -324,7 +355,7 @@ export function createBlockedApprovalStore(
|
|
|
324
355
|
if (dirMode != null) {
|
|
325
356
|
try { chmodSync(parent, dirMode) } catch { /* not ours to chmod */ }
|
|
326
357
|
}
|
|
327
|
-
|
|
358
|
+
write_(target, body, { encoding: 'utf-8', mode: BLOCKED_APPROVAL_FILE_MODE })
|
|
328
359
|
chmodSync(target, BLOCKED_APPROVAL_FILE_MODE)
|
|
329
360
|
return true
|
|
330
361
|
} catch {
|