switchroom 0.18.7 → 0.18.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/README.md +2 -2
  2. package/dist/cli/switchroom.js +905 -758
  3. package/dist/host-control/main.js +1 -1
  4. package/package.json +1 -1
  5. package/profiles/_base/start.sh.hbs +111 -34
  6. package/skills/switchroom-runtime/SKILL.md +2 -0
  7. package/telegram-plugin/dist/gateway/gateway.js +46273 -44324
  8. package/telegram-plugin/flood-circuit-breaker.ts +123 -0
  9. package/telegram-plugin/gateway/activity-card-store.ts +63 -18
  10. package/telegram-plugin/gateway/approval-card-stores.ts +99 -0
  11. package/telegram-plugin/gateway/boot-card.ts +27 -0
  12. package/telegram-plugin/gateway/bot-commands-ops-info.ts +194 -0
  13. package/telegram-plugin/gateway/busy-ack.ts +106 -0
  14. package/telegram-plugin/gateway/callback-query-handlers.ts +2660 -0
  15. package/telegram-plugin/gateway/gateway.ts +1169 -3043
  16. package/telegram-plugin/gateway/inbound-delivery-machine-dispatch.ts +181 -23
  17. package/telegram-plugin/gateway/inbound-delivery-machine.ts +8 -0
  18. package/telegram-plugin/gateway/mental-model-propose-diff.ts +61 -5
  19. package/telegram-plugin/gateway/model-command.ts +23 -11
  20. package/telegram-plugin/gateway/outbound-send-path.ts +375 -0
  21. package/telegram-plugin/gateway/pending-state-stores.ts +106 -0
  22. package/telegram-plugin/gateway/register-bot-commands.ts +30 -0
  23. package/telegram-plugin/gateway/session-model-file.ts +198 -0
  24. package/telegram-plugin/gateway/status-pin-store.ts +82 -22
  25. package/telegram-plugin/gateway/worker-pin-reaper.ts +114 -0
  26. package/telegram-plugin/hooks/hooks.json +10 -10
  27. package/telegram-plugin/hooks/run-hook.sh +84 -0
  28. package/telegram-plugin/model-unavailable.ts +26 -0
  29. package/telegram-plugin/pty-partial-handler.ts +39 -0
  30. package/telegram-plugin/render/rich-render.ts +79 -1
  31. package/telegram-plugin/retry-api-call.ts +62 -0
  32. package/telegram-plugin/shared/bot-runtime.ts +8 -1
  33. package/telegram-plugin/silence-poke.ts +14 -0
  34. package/telegram-plugin/stream-controller.ts +156 -38
  35. package/telegram-plugin/tests/activity-card-store.test.ts +47 -2
  36. package/telegram-plugin/tests/approval-card-restart-outcome.test.ts +218 -0
  37. package/telegram-plugin/tests/approval-card-stores.test.ts +124 -0
  38. package/telegram-plugin/tests/boot-card-flood-suppress.test.ts +111 -0
  39. package/telegram-plugin/tests/busy-ack-wiring.test.ts +118 -0
  40. package/telegram-plugin/tests/busy-ack.test.ts +121 -0
  41. package/telegram-plugin/tests/callback-query-handlers.test.ts +701 -0
  42. package/telegram-plugin/tests/emission-determinism-wiring.test.ts +11 -4
  43. package/telegram-plugin/tests/fixtures/cutover-killswitch-probe.ts +75 -0
  44. package/telegram-plugin/tests/flood-circuit-breaker.test.ts +74 -0
  45. package/telegram-plugin/tests/gateway-outbound-redact.test.ts +5 -1
  46. package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +177 -25
  47. package/telegram-plugin/tests/inbound-delivery-cutover-flip.test.ts +418 -0
  48. package/telegram-plugin/tests/inbound-delivery-dispatch-equivalence.test.ts +348 -0
  49. package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +141 -52
  50. package/telegram-plugin/tests/mental-model-name-entity-corruption.test.ts +119 -0
  51. package/telegram-plugin/tests/mental-model-propose-callback-gate.test.ts +8 -1
  52. package/telegram-plugin/tests/model-command.test.ts +2 -2
  53. package/telegram-plugin/tests/model-unavailable.test.ts +41 -0
  54. package/telegram-plugin/tests/outbound-send-chunks.test.ts +304 -0
  55. package/telegram-plugin/tests/outbound-send-path.test.ts +222 -0
  56. package/telegram-plugin/tests/pending-card-durability-wiring.test.ts +34 -15
  57. package/telegram-plugin/tests/pending-state-stores.test.ts +235 -0
  58. package/telegram-plugin/tests/pty-partial-handler.test.ts +56 -0
  59. package/telegram-plugin/tests/render/render-outbound-chunks.test.ts +98 -0
  60. package/telegram-plugin/tests/retry-api-call.test.ts +59 -0
  61. package/telegram-plugin/tests/run-hook-wrapper.test.ts +132 -0
  62. package/telegram-plugin/tests/session-model-file.test.ts +132 -0
  63. package/telegram-plugin/tests/slot-banner-boot-recovery.test.ts +3 -3
  64. package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
  65. package/telegram-plugin/tests/status-pin-store.test.ts +62 -6
  66. package/telegram-plugin/tests/stream-controller-chunk-cap.test.ts +122 -0
  67. package/telegram-plugin/tests/turn-flush-safety.test.ts +18 -4
  68. package/telegram-plugin/tests/vault-approval-posture.test.ts +15 -7
  69. package/telegram-plugin/tests/vault-grant-auto-resume.test.ts +8 -4
  70. package/telegram-plugin/tests/vault-grant-union.test.ts +8 -4
  71. package/telegram-plugin/tests/vault-grant-wizard.test.ts +8 -1
  72. package/telegram-plugin/tests/vault-grants-revoke.test.ts +8 -1
  73. package/telegram-plugin/tests/vault-key-regex-allows-slash.test.ts +8 -4
  74. package/telegram-plugin/tests/vault-request-access-tool.test.ts +8 -4
  75. package/telegram-plugin/tests/vault-request-access-unlock-resume.test.ts +8 -4
  76. package/telegram-plugin/tests/voice-send.test.ts +308 -0
  77. package/telegram-plugin/tests/worker-pin-reaper.test.ts +132 -0
  78. package/telegram-plugin/uat/scenarios/jtbd-deliberate-restart-resumes-dm.test.ts +118 -0
  79. package/telegram-plugin/uat/scenarios/jtbd-midflight-busy-ack-dm.test.ts +201 -0
  80. package/telegram-plugin/uat/scenarios/jtbd-worker-pin-lifecycle-dm.test.ts +208 -0
  81. package/telegram-plugin/uat/scenarios/vault-card-survives-gateway-restart-dm.test.ts +140 -0
  82. package/telegram-plugin/uat/scenarios/vault-deny-resumes-turn-dm.test.ts +84 -0
  83. package/telegram-plugin/uat/scenarios/vault-timeout-wakes-agent-dm.test.ts +91 -0
  84. package/telegram-plugin/voice-ondemand.ts +25 -1
  85. package/telegram-plugin/voice-send.ts +154 -0
@@ -0,0 +1,198 @@
1
+ /**
2
+ * Durable session-model stickiness — file helpers shared by the gateway.
3
+ *
4
+ * Two files in the bind-mounted agent state dir carry the contract
5
+ * (reference/rfcs/session-model-stickiness.md):
6
+ *
7
+ * - `.session-model` — the DURABLE session override written on every
8
+ * positively-confirmed `/model` switch. One-line JSON
9
+ * `{"model","configuredDefaultAtWrite","ts"}`. It is NOT consumed by a
10
+ * keep-path boot; start.sh deletes it on revert / invalidation /
11
+ * corruption / 7-day staleness, and the gateway deletes it on
12
+ * `/model default`.
13
+ *
14
+ * - `.relaunch-model-intent` — the ONE-SHOT intent bit for the next boot.
15
+ * One-line JSON `{"intent":"keep"|"revert","reason","ts"}`, atomic
16
+ * write, last-writer-wins. Boot default is REVERT (operator decision:
17
+ * a raw `docker restart` / host reboot / crash must revert to the yaml
18
+ * model), so every switchroom-managed KEEP path must stamp keep-intent
19
+ * BEFORE the bounce. start.sh consumes it (rm -f) every boot; a stale
20
+ * (>10 min by the embedded ts) or corrupt intent counts as no intent.
21
+ *
22
+ * The `model` token is always a canonical `claude --model` token (alias,
23
+ * `claude-*` id, or `sr-*` id) — NEVER a display label like "Opus 4.8".
24
+ * Shape-gated on write with the same regex start.sh greps with.
25
+ */
26
+
27
+ import { readFileSync, writeFileSync, renameSync, rmSync } from 'node:fs'
28
+ import { join } from 'node:path'
29
+ import { isValidModelArg } from './model-command.js'
30
+
31
+ export const SESSION_MODEL_FILE = '.session-model'
32
+ export const RELAUNCH_MODEL_INTENT_FILE = '.relaunch-model-intent'
33
+ export const CONFIGURED_DEFAULT_MODEL_FILE = '.configured-default-model'
34
+
35
+ export type RelaunchModelIntent = 'keep' | 'revert'
36
+
37
+ export interface SessionModelRecord {
38
+ model: string
39
+ configuredDefaultAtWrite: string
40
+ ts: number
41
+ }
42
+
43
+ /**
44
+ * Restart reasons whose semantics are "revert the session model to the
45
+ * configured default". Everything else `triggerSelfRestart` fires with is a
46
+ * switchroom-managed relaunch (watchdog recovery, drain-cap bounce,
47
+ * turn-complete deferred restart, fleet-fallback resume, sr-to-claude model
48
+ * switch, grant restarts) and KEEPS the override — the whole point of the
49
+ * stickiness contract. Enumerated in the RFC §3; default-keep here is safe
50
+ * because only gateway code calls triggerSelfRestart, and a bounce nobody
51
+ * stamped (crash, raw docker restart, deploy) reverts by boot default anyway.
52
+ */
53
+ const REVERT_RESTART_REASONS: ReadonlySet<string> = new Set(['inline-button-restart'])
54
+
55
+ /** Classify a triggerSelfRestart reason into the intent the boot should honor. */
56
+ export function intentForRestartReason(reason: string): RelaunchModelIntent {
57
+ return REVERT_RESTART_REASONS.has(reason) ? 'revert' : 'keep'
58
+ }
59
+
60
+ function atomicWrite(path: string, content: string): void {
61
+ const tmp = `${path}.tmp-${process.pid}-${Date.now()}`
62
+ writeFileSync(tmp, content, 'utf8')
63
+ renameSync(tmp, path)
64
+ }
65
+
66
+ /** Serialize a `.session-model` record (one line + trailing newline). */
67
+ export function serializeSessionModel(rec: SessionModelRecord): string {
68
+ return `${JSON.stringify({
69
+ model: rec.model,
70
+ configuredDefaultAtWrite: rec.configuredDefaultAtWrite,
71
+ ts: rec.ts,
72
+ })}\n`
73
+ }
74
+
75
+ /**
76
+ * Parse `.session-model` content. Returns null on corrupt JSON, a missing /
77
+ * non-string field, or a model token that fails the MODEL_ARG_RE shape gate
78
+ * (never trust an unvalidated string near `claude --model`).
79
+ */
80
+ export function parseSessionModel(text: string): SessionModelRecord | null {
81
+ try {
82
+ const raw = JSON.parse(text) as Partial<SessionModelRecord>
83
+ if (
84
+ typeof raw.model !== 'string' ||
85
+ typeof raw.configuredDefaultAtWrite !== 'string' ||
86
+ typeof raw.ts !== 'number' ||
87
+ !isValidModelArg(raw.model)
88
+ ) {
89
+ return null
90
+ }
91
+ return { model: raw.model, configuredDefaultAtWrite: raw.configuredDefaultAtWrite, ts: raw.ts }
92
+ } catch {
93
+ return null
94
+ }
95
+ }
96
+
97
+ /**
98
+ * Write the durable session override. Throws on a non-canonical token —
99
+ * callers must pass a `claude --model` token, never a display label
100
+ * (regression guard for the "Opus 4.5 persisted" class).
101
+ */
102
+ export function writeSessionModelFile(
103
+ agentDir: string,
104
+ model: string,
105
+ configuredDefaultAtWrite: string,
106
+ ): void {
107
+ if (!isValidModelArg(model)) {
108
+ throw new Error(`refusing to persist non-canonical session model token: ${JSON.stringify(model)}`)
109
+ }
110
+ atomicWrite(
111
+ join(agentDir, SESSION_MODEL_FILE),
112
+ serializeSessionModel({ model, configuredDefaultAtWrite, ts: Date.now() }),
113
+ )
114
+ }
115
+
116
+ /** Raw file text (for rollback snapshots), or null when absent/unreadable. */
117
+ export function readSessionModelFileRaw(agentDir: string): string | null {
118
+ try {
119
+ return readFileSync(join(agentDir, SESSION_MODEL_FILE), 'utf8')
120
+ } catch {
121
+ return null
122
+ }
123
+ }
124
+
125
+ /** Parsed durable override, or null when absent/corrupt. */
126
+ export function readSessionModelFile(agentDir: string): SessionModelRecord | null {
127
+ const raw = readSessionModelFileRaw(agentDir)
128
+ return raw == null ? null : parseSessionModel(raw)
129
+ }
130
+
131
+ /** Delete the durable override (`/model default`, rollback). Best-effort. */
132
+ export function clearSessionModelFile(agentDir: string): void {
133
+ try {
134
+ rmSync(join(agentDir, SESSION_MODEL_FILE), { force: true })
135
+ } catch {
136
+ /* best-effort */
137
+ }
138
+ }
139
+
140
+ /** Restore a rollback snapshot taken with readSessionModelFileRaw. */
141
+ export function restoreSessionModelFileRaw(agentDir: string, raw: string | null): void {
142
+ if (raw == null) {
143
+ clearSessionModelFile(agentDir)
144
+ return
145
+ }
146
+ try {
147
+ atomicWrite(join(agentDir, SESSION_MODEL_FILE), raw)
148
+ } catch {
149
+ /* best-effort */
150
+ }
151
+ }
152
+
153
+ /**
154
+ * Stamp the one-shot relaunch intent. MUST be called synchronously BEFORE
155
+ * the restart signal/dispatch it describes (write-before-kill invariant —
156
+ * the next start.sh boot reads this to decide keep vs revert). Best-effort:
157
+ * a failed write means the boot falls back to the default (revert), which
158
+ * is the safe side.
159
+ */
160
+ export function writeRelaunchModelIntent(
161
+ agentDir: string,
162
+ intent: RelaunchModelIntent,
163
+ reason: string,
164
+ ): void {
165
+ try {
166
+ atomicWrite(
167
+ join(agentDir, RELAUNCH_MODEL_INTENT_FILE),
168
+ `${JSON.stringify({ intent, reason, ts: Date.now() })}\n`,
169
+ )
170
+ } catch (err) {
171
+ process.stderr.write(
172
+ `telegram gateway: relaunch-model-intent write failed (boot will revert): ${(err as Error)?.message ?? String(err)}\n`,
173
+ )
174
+ }
175
+ }
176
+
177
+ /** Remove a stamped intent (rollback of a failed dispatch). Best-effort. */
178
+ export function clearRelaunchModelIntent(agentDir: string): void {
179
+ try {
180
+ rmSync(join(agentDir, RELAUNCH_MODEL_INTENT_FILE), { force: true })
181
+ } catch {
182
+ /* best-effort */
183
+ }
184
+ }
185
+
186
+ /**
187
+ * The resolved configured default start.sh recorded this boot
188
+ * (`.configured-default-model`, written before override resolution — the
189
+ * same resolver output the invalidation compare uses). Null when missing.
190
+ */
191
+ export function readConfiguredDefaultModel(agentDir: string): string | null {
192
+ try {
193
+ const v = readFileSync(join(agentDir, CONFIGURED_DEFAULT_MODEL_FILE), 'utf8').trim()
194
+ return v.length > 0 ? v : null
195
+ } catch {
196
+ return null
197
+ }
198
+ }
@@ -12,10 +12,13 @@
12
12
  * a dead session lingers.
13
13
  *
14
14
  * This makes cleanup self-contained across restart: every pin claim persists
15
- * here; on boot the gateway loads the persisted set and unpins each entry
16
- * (a status pin from a PRIOR session is stale by definition — the turn it
17
- * represented is over or crashed), then clears the store. It does NOT re-adopt
18
- * or re-pin it only cleans up.
15
+ * here; on boot the gateway loads the persisted set and unpins each
16
+ * work-scoped entry (a status pin from a PRIOR session is stale by definition
17
+ * — the turn it represented is over or crashed), dropping rows only after a
18
+ * successful unpin (failed ones are retained with an attempt counter for a
19
+ * next-boot retry — see runStatusPinBootCleanup). Time-scoped `tool:` rows
20
+ * (the `pin_message` MCP tool, #3001) survive boots until their `expiresAt`.
21
+ * It does NOT re-adopt or re-pin — it only cleans up.
19
22
  *
20
23
  * Shape choice — SNAPSHOT, not append-log, mirroring obligation-store.ts. The
21
24
  * claim set is tiny and bounded (one entry per in-flight pinned key, normally
@@ -55,8 +58,27 @@ export interface PersistedStatusPin {
55
58
  messageId: number
56
59
  /** True while the pin API call is in-flight / unconfirmed (see above). */
57
60
  pending?: boolean
61
+ /** Wall-clock ms after which this pin is stale and boot cleanup unpins it.
62
+ * Rows WITHOUT this field are work-scoped (fg:/wk:/banner:) — stale the
63
+ * moment their owning session dies, so boot cleanup unpins them
64
+ * unconditionally. Rows WITH it (the `tool:` pins written by the
65
+ * `pin_message` MCP tool, #3001) represent deliberate agent pins that have
66
+ * no "work finished" event: they SURVIVE restarts and are only swept once
67
+ * expired. */
68
+ expiresAt?: number
69
+ /** Boot-cleanup unpin retry counter (#3001). Incremented each boot the
70
+ * unpin fails (flood-wait exhausted / transient 5xx); the row is retained
71
+ * for retry until BOOT_UNPIN_MAX_ATTEMPTS, then forfeited. Absent = 0. */
72
+ attempts?: number
58
73
  }
59
74
 
75
+ /** How many boots may retry a failing boot-cleanup unpin before the row is
76
+ * forfeited. Unpins are idempotent (unpinning an already-unpinned or deleted
77
+ * message no-ops), so retrying across boots is safe; the cap only bounds a
78
+ * permanently-undeliverable unpin (chat gone, bot removed) so it cannot
79
+ * re-fail on every boot forever. */
80
+ export const BOOT_UNPIN_MAX_ATTEMPTS = 5
81
+
60
82
  /** Envelope version. v1 had no `pending` field; a v1 row loads as a confirmed
61
83
  * pin (pending undefined). v2 adds the optional `pending` flag. Both load
62
84
  * fail-open — an unknown/newer version yields []. */
@@ -74,7 +96,9 @@ function isPinRow(x: unknown): x is PersistedStatusPin {
74
96
  typeof o.chatId === 'string' &&
75
97
  o.chatId.length > 0 &&
76
98
  typeof o.messageId === 'number' &&
77
- (o.pending === undefined || typeof o.pending === 'boolean')
99
+ (o.pending === undefined || typeof o.pending === 'boolean') &&
100
+ (o.expiresAt === undefined || typeof o.expiresAt === 'number') &&
101
+ (o.attempts === undefined || typeof o.attempts === 'number')
78
102
  )
79
103
  }
80
104
 
@@ -170,16 +194,27 @@ export function pinnedMessageIsOurs(
170
194
  * the ordering + best-effort contract is unit-testable against the REAL code
171
195
  * (the gateway's thin wrapper just binds the live fs / unpin api / logger).
172
196
  *
173
- * Any pin persisted by a PRIOR session is stale by definition its turn ended
174
- * or the session crashed before its unpin reconcile ran. This includes records
175
- * left `pending` (the persist-intent-first write from `reconcileAndPersist-
176
- * StatusPin`): a crash between the pin API call and its confirming rewrite
177
- * leaves a pending record whose pin MAY have landed in Telegram, so we must
178
- * treat it exactly like a confirmed one and unpin it. We therefore best-effort
179
- * unpin EVERY persisted entry (confirmed and pending alike a failure is
180
- * non-fatal) and then EMPTY the store regardless, so a permanently-
181
- * undeliverable unpin can't re-run on every boot. We do NOT re-adopt or re-pin.
182
- * Returns the counts for logging/testing.
197
+ * The restart rule (#3001): a WORK-SCOPED pin persisted by a PRIOR session
198
+ * (fg:/wk:/banner: any row without `expiresAt`) is stale by definition its
199
+ * work ended or the session crashed before its unpin reconcile ran, so
200
+ * restart = reset: it is unpinned here. This includes records left `pending`
201
+ * (the persist-intent-first write from `reconcileAndPersistStatusPin`): a
202
+ * crash between the pin API call and its confirming rewrite leaves a pending
203
+ * record whose pin MAY have landed in Telegram, so we must treat it exactly
204
+ * like a confirmed one and unpin it.
205
+ *
206
+ * TIME-SCOPED rows (`tool:` pins from the `pin_message` MCP tool, carrying
207
+ * `expiresAt`) have no "work finished" event, so a restart does NOT reset
208
+ * them: an unexpired row is RETAINED untouched across boots and only unpinned
209
+ * once `now >= expiresAt`.
210
+ *
211
+ * RETRY-SAFETY (#3001): a row is dropped only AFTER its unpin resolves. A
212
+ * failing unpin (flood-wait exhausted / transient 5xx) retains the row with an
213
+ * incremented `attempts` counter so the NEXT boot retries, up to
214
+ * BOOT_UNPIN_MAX_ATTEMPTS — then the row is forfeited (a permanently-
215
+ * undeliverable unpin must not re-fail on every boot forever). Unpins are
216
+ * idempotent, so the retry can never double-unpin harmfully. We do NOT
217
+ * re-adopt or re-pin. Returns the counts for logging/testing.
183
218
  *
184
219
  * CRITICAL: the caller MUST only invoke this AFTER winning the startup mutex.
185
220
  * The store is a shared per-agent file; on a double-boot a losing gateway
@@ -189,27 +224,52 @@ export async function runStatusPinBootCleanup(args: {
189
224
  path: string
190
225
  fs: StatusPinStoreFsSeam
191
226
  unpin: (chatId: string, messageId: number) => Promise<unknown>
227
+ now?: number
192
228
  log?: (line: string) => void
193
- }): Promise<{ cleared: number; total: number }> {
229
+ }): Promise<{ cleared: number; retained: number; kept: number; total: number }> {
194
230
  const log = args.log ?? ((l: string) => process.stderr.write(l))
231
+ const now = args.now ?? Date.now()
195
232
  const persisted = loadStatusPins(args.path, args.fs)
196
- if (persisted.length === 0) return { cleared: 0, total: 0 }
233
+ if (persisted.length === 0) return { cleared: 0, retained: 0, kept: 0, total: 0 }
197
234
  let cleared = 0
235
+ let retained = 0
236
+ let kept = 0
237
+ const next: PersistedStatusPin[] = []
198
238
  for (const pin of persisted) {
239
+ // Unexpired time-scoped row (tool: pin): deliberately survives the
240
+ // restart — keep it as-is, no unpin.
241
+ if (pin.expiresAt != null && pin.expiresAt > now) {
242
+ next.push(pin)
243
+ kept++
244
+ continue
245
+ }
199
246
  try {
200
247
  await args.unpin(pin.chatId, pin.messageId)
201
248
  cleared++
202
249
  } catch (err) {
250
+ const attempts = (pin.attempts ?? 0) + 1
203
251
  log(
204
252
  `status-pin-store: boot cleanup unpin failed ` +
205
- `(chat=${pin.chatId} msg=${pin.messageId}): ${(err as Error).message}\n`,
253
+ `(chat=${pin.chatId} msg=${pin.messageId} attempt=${attempts}): ` +
254
+ `${(err as Error).message}\n`,
206
255
  )
256
+ if (attempts < BOOT_UNPIN_MAX_ATTEMPTS) {
257
+ // Retain for a retry on the next boot instead of forfeiting the
258
+ // orphan permanently (retry-safe boot sweep, #3001).
259
+ next.push({ ...pin, attempts })
260
+ retained++
261
+ } else {
262
+ log(
263
+ `status-pin-store: boot cleanup FORFEITING pin after ` +
264
+ `${attempts} failed unpin attempts ` +
265
+ `(key=${pin.pinKey} chat=${pin.chatId} msg=${pin.messageId}) — ` +
266
+ `will not retry again\n`,
267
+ )
268
+ }
207
269
  }
208
270
  }
209
- // Empty the store regardless — these claims belong to a dead session; leaving
210
- // them would re-attempt the same (already-tried) unpins on every future boot.
211
- persistStatusPins(args.path, args.fs, [], log)
212
- return { cleared, total: persisted.length }
271
+ persistStatusPins(args.path, args.fs, next, log)
272
+ return { cleared, retained, kept, total: persisted.length }
213
273
  }
214
274
 
215
275
  /**
@@ -0,0 +1,114 @@
1
+ /**
2
+ * worker-pin-reaper.ts — pure decision for the mid-session `wk:` pin sweep
3
+ * (#3001).
4
+ *
5
+ * Why this exists: the background-worker pin (`wk:<agentId>`, pinned on the
6
+ * `🛠 Worker` feed message while the worker runs) is normally unpinned by the
7
+ * worker's completion handler (`reconcileWorkerPin(agentId, null, false)` on
8
+ * the watcher's onFinish). But that event can be MISSED — watcher crash, SDK
9
+ * subprocess SIGKILL, a dropped JSONL tail — and then nothing ever unpins the
10
+ * worker's message until the next gateway boot. Log evidence on one agent:
11
+ * ~1120 pinChatMessage vs ~1014 unpinChatMessage with zero logged failures —
12
+ * a long tail of stale pins glued to the top of the chat.
13
+ *
14
+ * This module is the pure half (mirrors `runActivityCardMidSessionReaper`'s
15
+ * decide-over-injected-seams shape): given the currently-claimed `wk:` pins,
16
+ * a terminality predicate over the sub-agent registry, and a TTL, it returns
17
+ * the pins that should be unpinned NOW. The gateway executes each reap via
18
+ * `reconcileStatusPin(key, chat, { pinned: false })` so the in-memory claim
19
+ * AND the durable store row clear together.
20
+ *
21
+ * A pin is reaped when EITHER:
22
+ * - `terminal` — the registry says the worker reached a terminal status
23
+ * (completed | failed): its work is finished, the pin must go, however
24
+ * young it is. (A missed onFinish is exactly this case.)
25
+ * - `ttl` — the pin has been held past `ttlMs` AND the registry cannot
26
+ * vouch for the worker (no row / never linked / `stalled` / lookup
27
+ * error). A registry-confirmed RUNNING row exempts the pin from the TTL
28
+ * entirely: a healthy 7h worker keeps its pin for the whole run instead
29
+ * of churning unpin→re-pin every TTL. The stall detector demotes a dead
30
+ * worker's row out of 'running' within ~60s, so the TTL still catches
31
+ * true zombies.
32
+ *
33
+ * A worker the registry can't vouch for is still never touched before the
34
+ * TTL — the sweep can only ever shorten a stale pin's life, not a live one's.
35
+ */
36
+
37
+ /** Default TTL for a held worker pin: 6 hours. Rationale: worker turns are
38
+ * expected to run minutes-to-a-couple-of-hours (the watcher's own stall
39
+ * detection fires after ~60s of JSONL inactivity, and the longest sanctioned
40
+ * background dispatches are bounded by a single Claude session's lifetime).
41
+ * 6h comfortably exceeds any legitimate worker turn while bounding the
42
+ * stale-pin window to the same day instead of "until the next restart". */
43
+ export const WORKER_PIN_TTL_MS_DEFAULT = 6 * 60 * 60_000
44
+
45
+ export const WORKER_PIN_KEY_PREFIX = 'wk:'
46
+
47
+ /** One currently-claimed worker pin, flattened from the gateway's Maps. */
48
+ export interface WorkerPinCandidate {
49
+ /** Full pin key, `wk:<agentId>` shape. */
50
+ pinKey: string
51
+ /** Chat the pin lives in (from the gateway's pinKey → chatId registry). */
52
+ chatId: string
53
+ /** Wall-clock ms the claim was first taken (gateway's pinnedAt registry). */
54
+ pinnedAt: number
55
+ }
56
+
57
+ export interface WorkerPinReap extends WorkerPinCandidate {
58
+ reason: 'terminal' | 'ttl'
59
+ }
60
+
61
+ /** Extract the agentId from a `wk:<agentId>` pin key, or null for any other
62
+ * key shape (fg:/banner:/tool: keys are never worker-reaped). */
63
+ export function workerAgentIdOfPinKey(pinKey: string): string | null {
64
+ if (!pinKey.startsWith(WORKER_PIN_KEY_PREFIX)) return null
65
+ const agentId = pinKey.slice(WORKER_PIN_KEY_PREFIX.length)
66
+ return agentId.length > 0 ? agentId : null
67
+ }
68
+
69
+ /** The registry's view of a worker, distilled for the reap decision.
70
+ * - 'terminal' — row exists in completed | failed: reap now.
71
+ * - 'running' — row exists and is still 'running': NEVER reap, not even
72
+ * past the TTL. A healthy 7h worker must not get unpinned mid-run only
73
+ * for the next feed edit to re-pin it (pin/unpin churn every TTL). The
74
+ * subagent-watcher's stall detection demotes a dead worker's row out of
75
+ * 'running' within ~60s of its JSONL going quiet, so a true zombie can
76
+ * only hold 'running' briefly — the TTL still catches everything the
77
+ * registry has lost track of.
78
+ * - 'unknown' — no row / never linked / 'stalled' / lookup error: the
79
+ * TTL gate applies (the registry can't vouch for it). */
80
+ export type WorkerRegistryStatus = 'terminal' | 'running' | 'unknown'
81
+
82
+ /**
83
+ * Decide which claimed worker pins to unpin now. Pure: the registry lookup is
84
+ * an injected predicate (`statusOf` must return 'terminal' ONLY for a row in
85
+ * completed | failed, 'running' only for a live 'running' row, and 'unknown'
86
+ * for stalled / missing / lookup-error; a DB hiccup must degrade to 'unknown'
87
+ * — kept until the TTL — never to a spurious 'terminal' unpin).
88
+ */
89
+ export function decideWorkerPinReaps(args: {
90
+ pins: Iterable<WorkerPinCandidate>
91
+ statusOf: (agentId: string) => WorkerRegistryStatus
92
+ ttlMs: number
93
+ now: number
94
+ }): WorkerPinReap[] {
95
+ const reaps: WorkerPinReap[] = []
96
+ for (const pin of args.pins) {
97
+ const agentId = workerAgentIdOfPinKey(pin.pinKey)
98
+ if (agentId == null) continue // not a worker pin — never ours to reap
99
+ if (pin.chatId.length === 0) continue // can't unpin without a chat
100
+ const status = args.statusOf(agentId)
101
+ if (status === 'terminal') {
102
+ reaps.push({ ...pin, reason: 'terminal' })
103
+ continue
104
+ }
105
+ // A registry-confirmed RUNNING worker keeps its pin regardless of age —
106
+ // the TTL only reaps pins the registry can't vouch for (see
107
+ // WorkerRegistryStatus doc).
108
+ if (status === 'running') continue
109
+ if (args.now - pin.pinnedAt >= args.ttlMs) {
110
+ reaps.push({ ...pin, reason: 'ttl' })
111
+ }
112
+ }
113
+ return reaps
114
+ }
@@ -5,7 +5,7 @@
5
5
  "hooks": [
6
6
  {
7
7
  "type": "command",
8
- "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/secret-guard-pretool.mjs\"",
8
+ "command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/secret-guard-pretool.mjs\"",
9
9
  "timeout": 10
10
10
  }
11
11
  ]
@@ -14,7 +14,7 @@
14
14
  "hooks": [
15
15
  {
16
16
  "type": "command",
17
- "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/sentinel-reply-guard-pretool.mjs\"",
17
+ "command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/sentinel-reply-guard-pretool.mjs\"",
18
18
  "timeout": 5
19
19
  }
20
20
  ]
@@ -24,7 +24,7 @@
24
24
  "hooks": [
25
25
  {
26
26
  "type": "command",
27
- "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-tracker-pretool.mjs\"",
27
+ "command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-tracker-pretool.mjs\"",
28
28
  "timeout": 10
29
29
  }
30
30
  ]
@@ -33,7 +33,7 @@
33
33
  "hooks": [
34
34
  {
35
35
  "type": "command",
36
- "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/tool-label-pretool.mjs\"",
36
+ "command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/tool-label-pretool.mjs\"",
37
37
  "timeout": 5
38
38
  }
39
39
  ]
@@ -43,7 +43,7 @@
43
43
  "hooks": [
44
44
  {
45
45
  "type": "command",
46
- "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/repo-context-pretool.mjs\"",
46
+ "command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/repo-context-pretool.mjs\"",
47
47
  "timeout": 5
48
48
  }
49
49
  ]
@@ -55,7 +55,7 @@
55
55
  "hooks": [
56
56
  {
57
57
  "type": "command",
58
- "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-tracker-posttool.mjs\"",
58
+ "command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-tracker-posttool.mjs\"",
59
59
  "timeout": 10
60
60
  }
61
61
  ]
@@ -65,7 +65,7 @@
65
65
  "hooks": [
66
66
  {
67
67
  "type": "command",
68
- "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/sandbox-hint-posttool.mjs\"",
68
+ "command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/sandbox-hint-posttool.mjs\"",
69
69
  "timeout": 3
70
70
  }
71
71
  ]
@@ -76,7 +76,7 @@
76
76
  "hooks": [
77
77
  {
78
78
  "type": "command",
79
- "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/secret-scrub-stop.mjs\"",
79
+ "command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/secret-scrub-stop.mjs\"",
80
80
  "timeout": 15,
81
81
  "async": true
82
82
  }
@@ -86,7 +86,7 @@
86
86
  "hooks": [
87
87
  {
88
88
  "type": "command",
89
- "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/silent-end-interrupt-stop.mjs\"",
89
+ "command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/silent-end-interrupt-stop.mjs\"",
90
90
  "timeout": 5
91
91
  }
92
92
  ]
@@ -95,7 +95,7 @@
95
95
  "hooks": [
96
96
  {
97
97
  "type": "command",
98
- "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/tool-label-stop.mjs\"",
98
+ "command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/tool-label-stop.mjs\"",
99
99
  "timeout": 5,
100
100
  "async": true
101
101
  }
@@ -0,0 +1,84 @@
1
+ #!/bin/sh
2
+ # run-hook.sh — resilient launcher for Claude Code Node hooks (issue #2555).
3
+ #
4
+ # Under cgroup memory-ceiling pressure (the cgroup pinned at memory.max with
5
+ # reclaim lagging, page cache ~= cap), a freshly-spawned Node process can abort
6
+ # at STARTUP with exit 134 (SIGABRT) inside the libuv threadpool constructor —
7
+ # `Assertion failed: (0) == (uv_thread_create(...))` — BEFORE any hook code
8
+ # runs. It is a transient allocation failure, not a real hook error, but it
9
+ # surfaces a 🔴 issues card and skips the hook's work on that one tool call.
10
+ #
11
+ # This wrapper makes the invocation tolerant:
12
+ # 1. Shrink the libuv threadpool to 1 so Node needs the fewest possible
13
+ # thread-stack mmaps at startup (minimises the failure window).
14
+ # 2. Capture the hook payload from stdin ONCE and replay it on each attempt.
15
+ # The abort can land AFTER Node started draining the pipe, so a naive
16
+ # retry would feed the second attempt EMPTY stdin — a secret scanner would
17
+ # then scan nothing. Replaying the captured payload keeps the retry
18
+ # faithful.
19
+ # 3. Retry ONCE on a 134 abort after a brief backoff.
20
+ # 4. If it STILL aborts:
21
+ # - for SECURITY-CRITICAL hooks (secret-guard / secret-scrub) FAIL
22
+ # CLOSED — propagate 134 so the runtime cards it. 134 is a generic
23
+ # SIGABRT (assertion / OOM / any abort), not memory-pressure-specific,
24
+ # so silently returning 0 for a genuinely broken scanner would be a
25
+ # silent security BYPASS. Those hooks must never fail open.
26
+ # - for all other hooks, SKIP CLEANLY (exit 0) with a single stderr
27
+ # warn. A skipped label/context hook on one call is the documented,
28
+ # accepted degradation; a crash-card storm under memory pressure is
29
+ # not.
30
+ #
31
+ # Any non-134 exit status is passed through unchanged — real hook decisions
32
+ # (block/allow/non-zero) are never masked. `sleep 0.15` uses a fractional
33
+ # second (supported by GNU/BusyBox sleep, both present in the agent image).
34
+ #
35
+ # Usage (from hooks.json):
36
+ # sh "${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh" node "${CLAUDE_PLUGIN_ROOT}/hooks/<name>.mjs"
37
+
38
+ export UV_THREADPOOL_SIZE="${UV_THREADPOOL_SIZE:-1}"
39
+
40
+ # Identify the hook script (the .mjs argument) to decide fail-open vs closed.
41
+ hook_path=""
42
+ for a in "$@"; do
43
+ case "$a" in
44
+ *.mjs) hook_path="$a" ;;
45
+ esac
46
+ done
47
+ hook_name=$(basename "$hook_path" 2>/dev/null)
48
+
49
+ # Security-critical hooks MUST fail closed on a persistent abort.
50
+ fail_closed=0
51
+ case "$hook_name" in
52
+ secret-guard-pretool.mjs | secret-scrub-stop.mjs) fail_closed=1 ;;
53
+ esac
54
+
55
+ # Capture stdin once so both attempts see the SAME payload. Claude Code feeds
56
+ # the hook a single JSON object on stdin and closes it, so `cat` returns at
57
+ # EOF. (Command substitution strips trailing newlines, which JSON parsing does
58
+ # not care about.)
59
+ payload=$(cat)
60
+
61
+ run_hook() {
62
+ printf '%s' "$payload" | "$@"
63
+ }
64
+
65
+ run_hook "$@"
66
+ status=$?
67
+ if [ "$status" -ne 134 ]; then
68
+ exit "$status"
69
+ fi
70
+
71
+ # Transient thread-create abort — brief backoff, then retry once with the
72
+ # faithfully-replayed payload.
73
+ sleep 0.15
74
+ run_hook "$@"
75
+ status=$?
76
+ if [ "$status" -eq 134 ]; then
77
+ if [ "$fail_closed" -eq 1 ]; then
78
+ echo "run-hook: security hook '$hook_name' aborted twice with exit 134 — FAILING CLOSED (not skipping) (#2555)" >&2
79
+ exit 134
80
+ fi
81
+ echo "run-hook: '$hook_name' aborted twice with exit 134 (uv_thread_create under memory pressure) — skipping hook cleanly (#2555)" >&2
82
+ exit 0
83
+ fi
84
+ exit "$status"
@@ -69,6 +69,32 @@ export function detectModelUnavailable(
69
69
  const sample = stderr.length > 16_384 ? stderr.slice(0, 16_384) : stderr
70
70
  const lower = sample.toLowerCase()
71
71
 
72
+ // ── 0. Transient / server-side 429 (NOT account quota) — issue #2922 ────
73
+ // Anthropic emits a `rate_limit_error` whose message explicitly negates the
74
+ // account-quota reading: "Server is temporarily limiting requests (not your
75
+ // usage limit)". A negation-blind substring match on "usage limit" (step 1
76
+ // below) would misclassify this as `quota_exhausted`, firing a phantom fleet
77
+ // failover that self-cancels and leaves the turn dead. These are upstream
78
+ // throttles Claude Code retries internally with backoff — classify them as
79
+ // `overload` (the calm rate-limit path) BEFORE the quota substrings run, so
80
+ // the negation is honoured and no failover is announced.
81
+ const transientUpstreamSignals = [
82
+ 'not your usage limit',
83
+ 'not your account',
84
+ "not your account's",
85
+ 'temporarily limiting requests',
86
+ 'temporarily rate',
87
+ 'server is temporarily',
88
+ 'would exceed your account’s rate limit',
89
+ "would exceed your account's rate limit",
90
+ ]
91
+ if (transientUpstreamSignals.some(s => lower.includes(s))) {
92
+ const resetAt = parseResetTime(sample)
93
+ return resetAt !== undefined
94
+ ? { kind: 'overload', resetAt, raw: stderr }
95
+ : { kind: 'overload', raw: stderr }
96
+ }
97
+
72
98
  // ── 1. Quota / billing exhaustion ──────────────────────────────────────
73
99
  const quotaSignals = [
74
100
  'out of extra usage',