switchroom 0.18.7 → 0.18.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cli/switchroom.js +905 -758
- package/dist/host-control/main.js +1 -1
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +111 -34
- package/skills/switchroom-runtime/SKILL.md +2 -0
- package/telegram-plugin/dist/gateway/gateway.js +46273 -44324
- package/telegram-plugin/flood-circuit-breaker.ts +123 -0
- package/telegram-plugin/gateway/activity-card-store.ts +63 -18
- package/telegram-plugin/gateway/approval-card-stores.ts +99 -0
- package/telegram-plugin/gateway/boot-card.ts +27 -0
- package/telegram-plugin/gateway/bot-commands-ops-info.ts +194 -0
- package/telegram-plugin/gateway/busy-ack.ts +106 -0
- package/telegram-plugin/gateway/callback-query-handlers.ts +2660 -0
- package/telegram-plugin/gateway/gateway.ts +1169 -3043
- package/telegram-plugin/gateway/inbound-delivery-machine-dispatch.ts +181 -23
- package/telegram-plugin/gateway/inbound-delivery-machine.ts +8 -0
- package/telegram-plugin/gateway/mental-model-propose-diff.ts +61 -5
- package/telegram-plugin/gateway/model-command.ts +23 -11
- package/telegram-plugin/gateway/outbound-send-path.ts +375 -0
- package/telegram-plugin/gateway/pending-state-stores.ts +106 -0
- package/telegram-plugin/gateway/register-bot-commands.ts +30 -0
- package/telegram-plugin/gateway/session-model-file.ts +198 -0
- package/telegram-plugin/gateway/status-pin-store.ts +82 -22
- package/telegram-plugin/gateway/worker-pin-reaper.ts +114 -0
- package/telegram-plugin/hooks/hooks.json +10 -10
- package/telegram-plugin/hooks/run-hook.sh +84 -0
- package/telegram-plugin/model-unavailable.ts +26 -0
- package/telegram-plugin/pty-partial-handler.ts +39 -0
- package/telegram-plugin/render/rich-render.ts +79 -1
- package/telegram-plugin/retry-api-call.ts +62 -0
- package/telegram-plugin/shared/bot-runtime.ts +8 -1
- package/telegram-plugin/silence-poke.ts +14 -0
- package/telegram-plugin/stream-controller.ts +156 -38
- package/telegram-plugin/tests/activity-card-store.test.ts +47 -2
- package/telegram-plugin/tests/approval-card-restart-outcome.test.ts +218 -0
- package/telegram-plugin/tests/approval-card-stores.test.ts +124 -0
- package/telegram-plugin/tests/boot-card-flood-suppress.test.ts +111 -0
- package/telegram-plugin/tests/busy-ack-wiring.test.ts +118 -0
- package/telegram-plugin/tests/busy-ack.test.ts +121 -0
- package/telegram-plugin/tests/callback-query-handlers.test.ts +701 -0
- package/telegram-plugin/tests/emission-determinism-wiring.test.ts +11 -4
- package/telegram-plugin/tests/fixtures/cutover-killswitch-probe.ts +75 -0
- package/telegram-plugin/tests/flood-circuit-breaker.test.ts +74 -0
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +5 -1
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +177 -25
- package/telegram-plugin/tests/inbound-delivery-cutover-flip.test.ts +418 -0
- package/telegram-plugin/tests/inbound-delivery-dispatch-equivalence.test.ts +348 -0
- package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +141 -52
- package/telegram-plugin/tests/mental-model-name-entity-corruption.test.ts +119 -0
- package/telegram-plugin/tests/mental-model-propose-callback-gate.test.ts +8 -1
- package/telegram-plugin/tests/model-command.test.ts +2 -2
- package/telegram-plugin/tests/model-unavailable.test.ts +41 -0
- package/telegram-plugin/tests/outbound-send-chunks.test.ts +304 -0
- package/telegram-plugin/tests/outbound-send-path.test.ts +222 -0
- package/telegram-plugin/tests/pending-card-durability-wiring.test.ts +34 -15
- package/telegram-plugin/tests/pending-state-stores.test.ts +235 -0
- package/telegram-plugin/tests/pty-partial-handler.test.ts +56 -0
- package/telegram-plugin/tests/render/render-outbound-chunks.test.ts +98 -0
- package/telegram-plugin/tests/retry-api-call.test.ts +59 -0
- package/telegram-plugin/tests/run-hook-wrapper.test.ts +132 -0
- package/telegram-plugin/tests/session-model-file.test.ts +132 -0
- package/telegram-plugin/tests/slot-banner-boot-recovery.test.ts +3 -3
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
- package/telegram-plugin/tests/status-pin-store.test.ts +62 -6
- package/telegram-plugin/tests/stream-controller-chunk-cap.test.ts +122 -0
- package/telegram-plugin/tests/turn-flush-safety.test.ts +18 -4
- package/telegram-plugin/tests/vault-approval-posture.test.ts +15 -7
- package/telegram-plugin/tests/vault-grant-auto-resume.test.ts +8 -4
- package/telegram-plugin/tests/vault-grant-union.test.ts +8 -4
- package/telegram-plugin/tests/vault-grant-wizard.test.ts +8 -1
- package/telegram-plugin/tests/vault-grants-revoke.test.ts +8 -1
- package/telegram-plugin/tests/vault-key-regex-allows-slash.test.ts +8 -4
- package/telegram-plugin/tests/vault-request-access-tool.test.ts +8 -4
- package/telegram-plugin/tests/vault-request-access-unlock-resume.test.ts +8 -4
- package/telegram-plugin/tests/voice-send.test.ts +308 -0
- package/telegram-plugin/tests/worker-pin-reaper.test.ts +132 -0
- package/telegram-plugin/uat/scenarios/jtbd-deliberate-restart-resumes-dm.test.ts +118 -0
- package/telegram-plugin/uat/scenarios/jtbd-midflight-busy-ack-dm.test.ts +201 -0
- package/telegram-plugin/uat/scenarios/jtbd-worker-pin-lifecycle-dm.test.ts +208 -0
- package/telegram-plugin/uat/scenarios/vault-card-survives-gateway-restart-dm.test.ts +140 -0
- package/telegram-plugin/uat/scenarios/vault-deny-resumes-turn-dm.test.ts +84 -0
- package/telegram-plugin/uat/scenarios/vault-timeout-wakes-agent-dm.test.ts +91 -0
- package/telegram-plugin/voice-ondemand.ts +25 -1
- package/telegram-plugin/voice-send.ts +154 -0
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable session-model stickiness — file helpers shared by the gateway.
|
|
3
|
+
*
|
|
4
|
+
* Two files in the bind-mounted agent state dir carry the contract
|
|
5
|
+
* (reference/rfcs/session-model-stickiness.md):
|
|
6
|
+
*
|
|
7
|
+
* - `.session-model` — the DURABLE session override written on every
|
|
8
|
+
* positively-confirmed `/model` switch. One-line JSON
|
|
9
|
+
* `{"model","configuredDefaultAtWrite","ts"}`. It is NOT consumed by a
|
|
10
|
+
* keep-path boot; start.sh deletes it on revert / invalidation /
|
|
11
|
+
* corruption / 7-day staleness, and the gateway deletes it on
|
|
12
|
+
* `/model default`.
|
|
13
|
+
*
|
|
14
|
+
* - `.relaunch-model-intent` — the ONE-SHOT intent bit for the next boot.
|
|
15
|
+
* One-line JSON `{"intent":"keep"|"revert","reason","ts"}`, atomic
|
|
16
|
+
* write, last-writer-wins. Boot default is REVERT (operator decision:
|
|
17
|
+
* a raw `docker restart` / host reboot / crash must revert to the yaml
|
|
18
|
+
* model), so every switchroom-managed KEEP path must stamp keep-intent
|
|
19
|
+
* BEFORE the bounce. start.sh consumes it (rm -f) every boot; a stale
|
|
20
|
+
* (>10 min by the embedded ts) or corrupt intent counts as no intent.
|
|
21
|
+
*
|
|
22
|
+
* The `model` token is always a canonical `claude --model` token (alias,
|
|
23
|
+
* `claude-*` id, or `sr-*` id) — NEVER a display label like "Opus 4.8".
|
|
24
|
+
* Shape-gated on write with the same regex start.sh greps with.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { readFileSync, writeFileSync, renameSync, rmSync } from 'node:fs'
|
|
28
|
+
import { join } from 'node:path'
|
|
29
|
+
import { isValidModelArg } from './model-command.js'
|
|
30
|
+
|
|
31
|
+
export const SESSION_MODEL_FILE = '.session-model'
|
|
32
|
+
export const RELAUNCH_MODEL_INTENT_FILE = '.relaunch-model-intent'
|
|
33
|
+
export const CONFIGURED_DEFAULT_MODEL_FILE = '.configured-default-model'
|
|
34
|
+
|
|
35
|
+
export type RelaunchModelIntent = 'keep' | 'revert'
|
|
36
|
+
|
|
37
|
+
export interface SessionModelRecord {
|
|
38
|
+
model: string
|
|
39
|
+
configuredDefaultAtWrite: string
|
|
40
|
+
ts: number
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Restart reasons whose semantics are "revert the session model to the
|
|
45
|
+
* configured default". Everything else `triggerSelfRestart` fires with is a
|
|
46
|
+
* switchroom-managed relaunch (watchdog recovery, drain-cap bounce,
|
|
47
|
+
* turn-complete deferred restart, fleet-fallback resume, sr-to-claude model
|
|
48
|
+
* switch, grant restarts) and KEEPS the override — the whole point of the
|
|
49
|
+
* stickiness contract. Enumerated in the RFC §3; default-keep here is safe
|
|
50
|
+
* because only gateway code calls triggerSelfRestart, and a bounce nobody
|
|
51
|
+
* stamped (crash, raw docker restart, deploy) reverts by boot default anyway.
|
|
52
|
+
*/
|
|
53
|
+
const REVERT_RESTART_REASONS: ReadonlySet<string> = new Set(['inline-button-restart'])
|
|
54
|
+
|
|
55
|
+
/** Classify a triggerSelfRestart reason into the intent the boot should honor. */
|
|
56
|
+
export function intentForRestartReason(reason: string): RelaunchModelIntent {
|
|
57
|
+
return REVERT_RESTART_REASONS.has(reason) ? 'revert' : 'keep'
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function atomicWrite(path: string, content: string): void {
|
|
61
|
+
const tmp = `${path}.tmp-${process.pid}-${Date.now()}`
|
|
62
|
+
writeFileSync(tmp, content, 'utf8')
|
|
63
|
+
renameSync(tmp, path)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Serialize a `.session-model` record (one line + trailing newline). */
|
|
67
|
+
export function serializeSessionModel(rec: SessionModelRecord): string {
|
|
68
|
+
return `${JSON.stringify({
|
|
69
|
+
model: rec.model,
|
|
70
|
+
configuredDefaultAtWrite: rec.configuredDefaultAtWrite,
|
|
71
|
+
ts: rec.ts,
|
|
72
|
+
})}\n`
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Parse `.session-model` content. Returns null on corrupt JSON, a missing /
|
|
77
|
+
* non-string field, or a model token that fails the MODEL_ARG_RE shape gate
|
|
78
|
+
* (never trust an unvalidated string near `claude --model`).
|
|
79
|
+
*/
|
|
80
|
+
export function parseSessionModel(text: string): SessionModelRecord | null {
|
|
81
|
+
try {
|
|
82
|
+
const raw = JSON.parse(text) as Partial<SessionModelRecord>
|
|
83
|
+
if (
|
|
84
|
+
typeof raw.model !== 'string' ||
|
|
85
|
+
typeof raw.configuredDefaultAtWrite !== 'string' ||
|
|
86
|
+
typeof raw.ts !== 'number' ||
|
|
87
|
+
!isValidModelArg(raw.model)
|
|
88
|
+
) {
|
|
89
|
+
return null
|
|
90
|
+
}
|
|
91
|
+
return { model: raw.model, configuredDefaultAtWrite: raw.configuredDefaultAtWrite, ts: raw.ts }
|
|
92
|
+
} catch {
|
|
93
|
+
return null
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Write the durable session override. Throws on a non-canonical token —
|
|
99
|
+
* callers must pass a `claude --model` token, never a display label
|
|
100
|
+
* (regression guard for the "Opus 4.5 persisted" class).
|
|
101
|
+
*/
|
|
102
|
+
export function writeSessionModelFile(
|
|
103
|
+
agentDir: string,
|
|
104
|
+
model: string,
|
|
105
|
+
configuredDefaultAtWrite: string,
|
|
106
|
+
): void {
|
|
107
|
+
if (!isValidModelArg(model)) {
|
|
108
|
+
throw new Error(`refusing to persist non-canonical session model token: ${JSON.stringify(model)}`)
|
|
109
|
+
}
|
|
110
|
+
atomicWrite(
|
|
111
|
+
join(agentDir, SESSION_MODEL_FILE),
|
|
112
|
+
serializeSessionModel({ model, configuredDefaultAtWrite, ts: Date.now() }),
|
|
113
|
+
)
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Raw file text (for rollback snapshots), or null when absent/unreadable. */
|
|
117
|
+
export function readSessionModelFileRaw(agentDir: string): string | null {
|
|
118
|
+
try {
|
|
119
|
+
return readFileSync(join(agentDir, SESSION_MODEL_FILE), 'utf8')
|
|
120
|
+
} catch {
|
|
121
|
+
return null
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Parsed durable override, or null when absent/corrupt. */
|
|
126
|
+
export function readSessionModelFile(agentDir: string): SessionModelRecord | null {
|
|
127
|
+
const raw = readSessionModelFileRaw(agentDir)
|
|
128
|
+
return raw == null ? null : parseSessionModel(raw)
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Delete the durable override (`/model default`, rollback). Best-effort. */
|
|
132
|
+
export function clearSessionModelFile(agentDir: string): void {
|
|
133
|
+
try {
|
|
134
|
+
rmSync(join(agentDir, SESSION_MODEL_FILE), { force: true })
|
|
135
|
+
} catch {
|
|
136
|
+
/* best-effort */
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** Restore a rollback snapshot taken with readSessionModelFileRaw. */
|
|
141
|
+
export function restoreSessionModelFileRaw(agentDir: string, raw: string | null): void {
|
|
142
|
+
if (raw == null) {
|
|
143
|
+
clearSessionModelFile(agentDir)
|
|
144
|
+
return
|
|
145
|
+
}
|
|
146
|
+
try {
|
|
147
|
+
atomicWrite(join(agentDir, SESSION_MODEL_FILE), raw)
|
|
148
|
+
} catch {
|
|
149
|
+
/* best-effort */
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Stamp the one-shot relaunch intent. MUST be called synchronously BEFORE
|
|
155
|
+
* the restart signal/dispatch it describes (write-before-kill invariant —
|
|
156
|
+
* the next start.sh boot reads this to decide keep vs revert). Best-effort:
|
|
157
|
+
* a failed write means the boot falls back to the default (revert), which
|
|
158
|
+
* is the safe side.
|
|
159
|
+
*/
|
|
160
|
+
export function writeRelaunchModelIntent(
|
|
161
|
+
agentDir: string,
|
|
162
|
+
intent: RelaunchModelIntent,
|
|
163
|
+
reason: string,
|
|
164
|
+
): void {
|
|
165
|
+
try {
|
|
166
|
+
atomicWrite(
|
|
167
|
+
join(agentDir, RELAUNCH_MODEL_INTENT_FILE),
|
|
168
|
+
`${JSON.stringify({ intent, reason, ts: Date.now() })}\n`,
|
|
169
|
+
)
|
|
170
|
+
} catch (err) {
|
|
171
|
+
process.stderr.write(
|
|
172
|
+
`telegram gateway: relaunch-model-intent write failed (boot will revert): ${(err as Error)?.message ?? String(err)}\n`,
|
|
173
|
+
)
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** Remove a stamped intent (rollback of a failed dispatch). Best-effort. */
|
|
178
|
+
export function clearRelaunchModelIntent(agentDir: string): void {
|
|
179
|
+
try {
|
|
180
|
+
rmSync(join(agentDir, RELAUNCH_MODEL_INTENT_FILE), { force: true })
|
|
181
|
+
} catch {
|
|
182
|
+
/* best-effort */
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* The resolved configured default start.sh recorded this boot
|
|
188
|
+
* (`.configured-default-model`, written before override resolution — the
|
|
189
|
+
* same resolver output the invalidation compare uses). Null when missing.
|
|
190
|
+
*/
|
|
191
|
+
export function readConfiguredDefaultModel(agentDir: string): string | null {
|
|
192
|
+
try {
|
|
193
|
+
const v = readFileSync(join(agentDir, CONFIGURED_DEFAULT_MODEL_FILE), 'utf8').trim()
|
|
194
|
+
return v.length > 0 ? v : null
|
|
195
|
+
} catch {
|
|
196
|
+
return null
|
|
197
|
+
}
|
|
198
|
+
}
|
|
@@ -12,10 +12,13 @@
|
|
|
12
12
|
* a dead session lingers.
|
|
13
13
|
*
|
|
14
14
|
* This makes cleanup self-contained across restart: every pin claim persists
|
|
15
|
-
* here; on boot the gateway loads the persisted set and unpins each
|
|
16
|
-
* (a status pin from a PRIOR session is stale by definition
|
|
17
|
-
* represented is over or crashed),
|
|
18
|
-
*
|
|
15
|
+
* here; on boot the gateway loads the persisted set and unpins each
|
|
16
|
+
* work-scoped entry (a status pin from a PRIOR session is stale by definition
|
|
17
|
+
* — the turn it represented is over or crashed), dropping rows only after a
|
|
18
|
+
* successful unpin (failed ones are retained with an attempt counter for a
|
|
19
|
+
* next-boot retry — see runStatusPinBootCleanup). Time-scoped `tool:` rows
|
|
20
|
+
* (the `pin_message` MCP tool, #3001) survive boots until their `expiresAt`.
|
|
21
|
+
* It does NOT re-adopt or re-pin — it only cleans up.
|
|
19
22
|
*
|
|
20
23
|
* Shape choice — SNAPSHOT, not append-log, mirroring obligation-store.ts. The
|
|
21
24
|
* claim set is tiny and bounded (one entry per in-flight pinned key, normally
|
|
@@ -55,8 +58,27 @@ export interface PersistedStatusPin {
|
|
|
55
58
|
messageId: number
|
|
56
59
|
/** True while the pin API call is in-flight / unconfirmed (see above). */
|
|
57
60
|
pending?: boolean
|
|
61
|
+
/** Wall-clock ms after which this pin is stale and boot cleanup unpins it.
|
|
62
|
+
* Rows WITHOUT this field are work-scoped (fg:/wk:/banner:) — stale the
|
|
63
|
+
* moment their owning session dies, so boot cleanup unpins them
|
|
64
|
+
* unconditionally. Rows WITH it (the `tool:` pins written by the
|
|
65
|
+
* `pin_message` MCP tool, #3001) represent deliberate agent pins that have
|
|
66
|
+
* no "work finished" event: they SURVIVE restarts and are only swept once
|
|
67
|
+
* expired. */
|
|
68
|
+
expiresAt?: number
|
|
69
|
+
/** Boot-cleanup unpin retry counter (#3001). Incremented each boot the
|
|
70
|
+
* unpin fails (flood-wait exhausted / transient 5xx); the row is retained
|
|
71
|
+
* for retry until BOOT_UNPIN_MAX_ATTEMPTS, then forfeited. Absent = 0. */
|
|
72
|
+
attempts?: number
|
|
58
73
|
}
|
|
59
74
|
|
|
75
|
+
/** How many boots may retry a failing boot-cleanup unpin before the row is
|
|
76
|
+
* forfeited. Unpins are idempotent (unpinning an already-unpinned or deleted
|
|
77
|
+
* message no-ops), so retrying across boots is safe; the cap only bounds a
|
|
78
|
+
* permanently-undeliverable unpin (chat gone, bot removed) so it cannot
|
|
79
|
+
* re-fail on every boot forever. */
|
|
80
|
+
export const BOOT_UNPIN_MAX_ATTEMPTS = 5
|
|
81
|
+
|
|
60
82
|
/** Envelope version. v1 had no `pending` field; a v1 row loads as a confirmed
|
|
61
83
|
* pin (pending undefined). v2 adds the optional `pending` flag. Both load
|
|
62
84
|
* fail-open — an unknown/newer version yields []. */
|
|
@@ -74,7 +96,9 @@ function isPinRow(x: unknown): x is PersistedStatusPin {
|
|
|
74
96
|
typeof o.chatId === 'string' &&
|
|
75
97
|
o.chatId.length > 0 &&
|
|
76
98
|
typeof o.messageId === 'number' &&
|
|
77
|
-
(o.pending === undefined || typeof o.pending === 'boolean')
|
|
99
|
+
(o.pending === undefined || typeof o.pending === 'boolean') &&
|
|
100
|
+
(o.expiresAt === undefined || typeof o.expiresAt === 'number') &&
|
|
101
|
+
(o.attempts === undefined || typeof o.attempts === 'number')
|
|
78
102
|
)
|
|
79
103
|
}
|
|
80
104
|
|
|
@@ -170,16 +194,27 @@ export function pinnedMessageIsOurs(
|
|
|
170
194
|
* the ordering + best-effort contract is unit-testable against the REAL code
|
|
171
195
|
* (the gateway's thin wrapper just binds the live fs / unpin api / logger).
|
|
172
196
|
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
*
|
|
176
|
-
*
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
*
|
|
180
|
-
*
|
|
181
|
-
*
|
|
182
|
-
*
|
|
197
|
+
* The restart rule (#3001): a WORK-SCOPED pin persisted by a PRIOR session
|
|
198
|
+
* (fg:/wk:/banner: — any row without `expiresAt`) is stale by definition — its
|
|
199
|
+
* work ended or the session crashed before its unpin reconcile ran, so
|
|
200
|
+
* restart = reset: it is unpinned here. This includes records left `pending`
|
|
201
|
+
* (the persist-intent-first write from `reconcileAndPersistStatusPin`): a
|
|
202
|
+
* crash between the pin API call and its confirming rewrite leaves a pending
|
|
203
|
+
* record whose pin MAY have landed in Telegram, so we must treat it exactly
|
|
204
|
+
* like a confirmed one and unpin it.
|
|
205
|
+
*
|
|
206
|
+
* TIME-SCOPED rows (`tool:` pins from the `pin_message` MCP tool, carrying
|
|
207
|
+
* `expiresAt`) have no "work finished" event, so a restart does NOT reset
|
|
208
|
+
* them: an unexpired row is RETAINED untouched across boots and only unpinned
|
|
209
|
+
* once `now >= expiresAt`.
|
|
210
|
+
*
|
|
211
|
+
* RETRY-SAFETY (#3001): a row is dropped only AFTER its unpin resolves. A
|
|
212
|
+
* failing unpin (flood-wait exhausted / transient 5xx) retains the row with an
|
|
213
|
+
* incremented `attempts` counter so the NEXT boot retries, up to
|
|
214
|
+
* BOOT_UNPIN_MAX_ATTEMPTS — then the row is forfeited (a permanently-
|
|
215
|
+
* undeliverable unpin must not re-fail on every boot forever). Unpins are
|
|
216
|
+
* idempotent, so the retry can never double-unpin harmfully. We do NOT
|
|
217
|
+
* re-adopt or re-pin. Returns the counts for logging/testing.
|
|
183
218
|
*
|
|
184
219
|
* CRITICAL: the caller MUST only invoke this AFTER winning the startup mutex.
|
|
185
220
|
* The store is a shared per-agent file; on a double-boot a losing gateway
|
|
@@ -189,27 +224,52 @@ export async function runStatusPinBootCleanup(args: {
|
|
|
189
224
|
path: string
|
|
190
225
|
fs: StatusPinStoreFsSeam
|
|
191
226
|
unpin: (chatId: string, messageId: number) => Promise<unknown>
|
|
227
|
+
now?: number
|
|
192
228
|
log?: (line: string) => void
|
|
193
|
-
}): Promise<{ cleared: number; total: number }> {
|
|
229
|
+
}): Promise<{ cleared: number; retained: number; kept: number; total: number }> {
|
|
194
230
|
const log = args.log ?? ((l: string) => process.stderr.write(l))
|
|
231
|
+
const now = args.now ?? Date.now()
|
|
195
232
|
const persisted = loadStatusPins(args.path, args.fs)
|
|
196
|
-
if (persisted.length === 0) return { cleared: 0, total: 0 }
|
|
233
|
+
if (persisted.length === 0) return { cleared: 0, retained: 0, kept: 0, total: 0 }
|
|
197
234
|
let cleared = 0
|
|
235
|
+
let retained = 0
|
|
236
|
+
let kept = 0
|
|
237
|
+
const next: PersistedStatusPin[] = []
|
|
198
238
|
for (const pin of persisted) {
|
|
239
|
+
// Unexpired time-scoped row (tool: pin): deliberately survives the
|
|
240
|
+
// restart — keep it as-is, no unpin.
|
|
241
|
+
if (pin.expiresAt != null && pin.expiresAt > now) {
|
|
242
|
+
next.push(pin)
|
|
243
|
+
kept++
|
|
244
|
+
continue
|
|
245
|
+
}
|
|
199
246
|
try {
|
|
200
247
|
await args.unpin(pin.chatId, pin.messageId)
|
|
201
248
|
cleared++
|
|
202
249
|
} catch (err) {
|
|
250
|
+
const attempts = (pin.attempts ?? 0) + 1
|
|
203
251
|
log(
|
|
204
252
|
`status-pin-store: boot cleanup unpin failed ` +
|
|
205
|
-
`(chat=${pin.chatId} msg=${pin.messageId}):
|
|
253
|
+
`(chat=${pin.chatId} msg=${pin.messageId} attempt=${attempts}): ` +
|
|
254
|
+
`${(err as Error).message}\n`,
|
|
206
255
|
)
|
|
256
|
+
if (attempts < BOOT_UNPIN_MAX_ATTEMPTS) {
|
|
257
|
+
// Retain for a retry on the next boot instead of forfeiting the
|
|
258
|
+
// orphan permanently (retry-safe boot sweep, #3001).
|
|
259
|
+
next.push({ ...pin, attempts })
|
|
260
|
+
retained++
|
|
261
|
+
} else {
|
|
262
|
+
log(
|
|
263
|
+
`status-pin-store: boot cleanup FORFEITING pin after ` +
|
|
264
|
+
`${attempts} failed unpin attempts ` +
|
|
265
|
+
`(key=${pin.pinKey} chat=${pin.chatId} msg=${pin.messageId}) — ` +
|
|
266
|
+
`will not retry again\n`,
|
|
267
|
+
)
|
|
268
|
+
}
|
|
207
269
|
}
|
|
208
270
|
}
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
persistStatusPins(args.path, args.fs, [], log)
|
|
212
|
-
return { cleared, total: persisted.length }
|
|
271
|
+
persistStatusPins(args.path, args.fs, next, log)
|
|
272
|
+
return { cleared, retained, kept, total: persisted.length }
|
|
213
273
|
}
|
|
214
274
|
|
|
215
275
|
/**
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* worker-pin-reaper.ts — pure decision for the mid-session `wk:` pin sweep
|
|
3
|
+
* (#3001).
|
|
4
|
+
*
|
|
5
|
+
* Why this exists: the background-worker pin (`wk:<agentId>`, pinned on the
|
|
6
|
+
* `🛠 Worker` feed message while the worker runs) is normally unpinned by the
|
|
7
|
+
* worker's completion handler (`reconcileWorkerPin(agentId, null, false)` on
|
|
8
|
+
* the watcher's onFinish). But that event can be MISSED — watcher crash, SDK
|
|
9
|
+
* subprocess SIGKILL, a dropped JSONL tail — and then nothing ever unpins the
|
|
10
|
+
* worker's message until the next gateway boot. Log evidence on one agent:
|
|
11
|
+
* ~1120 pinChatMessage vs ~1014 unpinChatMessage with zero logged failures —
|
|
12
|
+
* a long tail of stale pins glued to the top of the chat.
|
|
13
|
+
*
|
|
14
|
+
* This module is the pure half (mirrors `runActivityCardMidSessionReaper`'s
|
|
15
|
+
* decide-over-injected-seams shape): given the currently-claimed `wk:` pins,
|
|
16
|
+
* a terminality predicate over the sub-agent registry, and a TTL, it returns
|
|
17
|
+
* the pins that should be unpinned NOW. The gateway executes each reap via
|
|
18
|
+
* `reconcileStatusPin(key, chat, { pinned: false })` so the in-memory claim
|
|
19
|
+
* AND the durable store row clear together.
|
|
20
|
+
*
|
|
21
|
+
* A pin is reaped when EITHER:
|
|
22
|
+
* - `terminal` — the registry says the worker reached a terminal status
|
|
23
|
+
* (completed | failed): its work is finished, the pin must go, however
|
|
24
|
+
* young it is. (A missed onFinish is exactly this case.)
|
|
25
|
+
* - `ttl` — the pin has been held past `ttlMs` AND the registry cannot
|
|
26
|
+
* vouch for the worker (no row / never linked / `stalled` / lookup
|
|
27
|
+
* error). A registry-confirmed RUNNING row exempts the pin from the TTL
|
|
28
|
+
* entirely: a healthy 7h worker keeps its pin for the whole run instead
|
|
29
|
+
* of churning unpin→re-pin every TTL. The stall detector demotes a dead
|
|
30
|
+
* worker's row out of 'running' within ~60s, so the TTL still catches
|
|
31
|
+
* true zombies.
|
|
32
|
+
*
|
|
33
|
+
* A worker the registry can't vouch for is still never touched before the
|
|
34
|
+
* TTL — the sweep can only ever shorten a stale pin's life, not a live one's.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
/** Default TTL for a held worker pin: 6 hours. Rationale: worker turns are
|
|
38
|
+
* expected to run minutes-to-a-couple-of-hours (the watcher's own stall
|
|
39
|
+
* detection fires after ~60s of JSONL inactivity, and the longest sanctioned
|
|
40
|
+
* background dispatches are bounded by a single Claude session's lifetime).
|
|
41
|
+
* 6h comfortably exceeds any legitimate worker turn while bounding the
|
|
42
|
+
* stale-pin window to the same day instead of "until the next restart". */
|
|
43
|
+
export const WORKER_PIN_TTL_MS_DEFAULT = 6 * 60 * 60_000
|
|
44
|
+
|
|
45
|
+
export const WORKER_PIN_KEY_PREFIX = 'wk:'
|
|
46
|
+
|
|
47
|
+
/** One currently-claimed worker pin, flattened from the gateway's Maps. */
|
|
48
|
+
export interface WorkerPinCandidate {
|
|
49
|
+
/** Full pin key, `wk:<agentId>` shape. */
|
|
50
|
+
pinKey: string
|
|
51
|
+
/** Chat the pin lives in (from the gateway's pinKey → chatId registry). */
|
|
52
|
+
chatId: string
|
|
53
|
+
/** Wall-clock ms the claim was first taken (gateway's pinnedAt registry). */
|
|
54
|
+
pinnedAt: number
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface WorkerPinReap extends WorkerPinCandidate {
|
|
58
|
+
reason: 'terminal' | 'ttl'
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Extract the agentId from a `wk:<agentId>` pin key, or null for any other
|
|
62
|
+
* key shape (fg:/banner:/tool: keys are never worker-reaped). */
|
|
63
|
+
export function workerAgentIdOfPinKey(pinKey: string): string | null {
|
|
64
|
+
if (!pinKey.startsWith(WORKER_PIN_KEY_PREFIX)) return null
|
|
65
|
+
const agentId = pinKey.slice(WORKER_PIN_KEY_PREFIX.length)
|
|
66
|
+
return agentId.length > 0 ? agentId : null
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** The registry's view of a worker, distilled for the reap decision.
|
|
70
|
+
* - 'terminal' — row exists in completed | failed: reap now.
|
|
71
|
+
* - 'running' — row exists and is still 'running': NEVER reap, not even
|
|
72
|
+
* past the TTL. A healthy 7h worker must not get unpinned mid-run only
|
|
73
|
+
* for the next feed edit to re-pin it (pin/unpin churn every TTL). The
|
|
74
|
+
* subagent-watcher's stall detection demotes a dead worker's row out of
|
|
75
|
+
* 'running' within ~60s of its JSONL going quiet, so a true zombie can
|
|
76
|
+
* only hold 'running' briefly — the TTL still catches everything the
|
|
77
|
+
* registry has lost track of.
|
|
78
|
+
* - 'unknown' — no row / never linked / 'stalled' / lookup error: the
|
|
79
|
+
* TTL gate applies (the registry can't vouch for it). */
|
|
80
|
+
export type WorkerRegistryStatus = 'terminal' | 'running' | 'unknown'
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Decide which claimed worker pins to unpin now. Pure: the registry lookup is
|
|
84
|
+
* an injected predicate (`statusOf` must return 'terminal' ONLY for a row in
|
|
85
|
+
* completed | failed, 'running' only for a live 'running' row, and 'unknown'
|
|
86
|
+
* for stalled / missing / lookup-error; a DB hiccup must degrade to 'unknown'
|
|
87
|
+
* — kept until the TTL — never to a spurious 'terminal' unpin).
|
|
88
|
+
*/
|
|
89
|
+
export function decideWorkerPinReaps(args: {
|
|
90
|
+
pins: Iterable<WorkerPinCandidate>
|
|
91
|
+
statusOf: (agentId: string) => WorkerRegistryStatus
|
|
92
|
+
ttlMs: number
|
|
93
|
+
now: number
|
|
94
|
+
}): WorkerPinReap[] {
|
|
95
|
+
const reaps: WorkerPinReap[] = []
|
|
96
|
+
for (const pin of args.pins) {
|
|
97
|
+
const agentId = workerAgentIdOfPinKey(pin.pinKey)
|
|
98
|
+
if (agentId == null) continue // not a worker pin — never ours to reap
|
|
99
|
+
if (pin.chatId.length === 0) continue // can't unpin without a chat
|
|
100
|
+
const status = args.statusOf(agentId)
|
|
101
|
+
if (status === 'terminal') {
|
|
102
|
+
reaps.push({ ...pin, reason: 'terminal' })
|
|
103
|
+
continue
|
|
104
|
+
}
|
|
105
|
+
// A registry-confirmed RUNNING worker keeps its pin regardless of age —
|
|
106
|
+
// the TTL only reaps pins the registry can't vouch for (see
|
|
107
|
+
// WorkerRegistryStatus doc).
|
|
108
|
+
if (status === 'running') continue
|
|
109
|
+
if (args.now - pin.pinnedAt >= args.ttlMs) {
|
|
110
|
+
reaps.push({ ...pin, reason: 'ttl' })
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return reaps
|
|
114
|
+
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"hooks": [
|
|
6
6
|
{
|
|
7
7
|
"type": "command",
|
|
8
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/secret-guard-pretool.mjs\"",
|
|
8
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/secret-guard-pretool.mjs\"",
|
|
9
9
|
"timeout": 10
|
|
10
10
|
}
|
|
11
11
|
]
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"hooks": [
|
|
15
15
|
{
|
|
16
16
|
"type": "command",
|
|
17
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/sentinel-reply-guard-pretool.mjs\"",
|
|
17
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/sentinel-reply-guard-pretool.mjs\"",
|
|
18
18
|
"timeout": 5
|
|
19
19
|
}
|
|
20
20
|
]
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
"hooks": [
|
|
25
25
|
{
|
|
26
26
|
"type": "command",
|
|
27
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-tracker-pretool.mjs\"",
|
|
27
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-tracker-pretool.mjs\"",
|
|
28
28
|
"timeout": 10
|
|
29
29
|
}
|
|
30
30
|
]
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
"hooks": [
|
|
34
34
|
{
|
|
35
35
|
"type": "command",
|
|
36
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/tool-label-pretool.mjs\"",
|
|
36
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/tool-label-pretool.mjs\"",
|
|
37
37
|
"timeout": 5
|
|
38
38
|
}
|
|
39
39
|
]
|
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
"hooks": [
|
|
44
44
|
{
|
|
45
45
|
"type": "command",
|
|
46
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/repo-context-pretool.mjs\"",
|
|
46
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/repo-context-pretool.mjs\"",
|
|
47
47
|
"timeout": 5
|
|
48
48
|
}
|
|
49
49
|
]
|
|
@@ -55,7 +55,7 @@
|
|
|
55
55
|
"hooks": [
|
|
56
56
|
{
|
|
57
57
|
"type": "command",
|
|
58
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-tracker-posttool.mjs\"",
|
|
58
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-tracker-posttool.mjs\"",
|
|
59
59
|
"timeout": 10
|
|
60
60
|
}
|
|
61
61
|
]
|
|
@@ -65,7 +65,7 @@
|
|
|
65
65
|
"hooks": [
|
|
66
66
|
{
|
|
67
67
|
"type": "command",
|
|
68
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/sandbox-hint-posttool.mjs\"",
|
|
68
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/sandbox-hint-posttool.mjs\"",
|
|
69
69
|
"timeout": 3
|
|
70
70
|
}
|
|
71
71
|
]
|
|
@@ -76,7 +76,7 @@
|
|
|
76
76
|
"hooks": [
|
|
77
77
|
{
|
|
78
78
|
"type": "command",
|
|
79
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/secret-scrub-stop.mjs\"",
|
|
79
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/secret-scrub-stop.mjs\"",
|
|
80
80
|
"timeout": 15,
|
|
81
81
|
"async": true
|
|
82
82
|
}
|
|
@@ -86,7 +86,7 @@
|
|
|
86
86
|
"hooks": [
|
|
87
87
|
{
|
|
88
88
|
"type": "command",
|
|
89
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/silent-end-interrupt-stop.mjs\"",
|
|
89
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/silent-end-interrupt-stop.mjs\"",
|
|
90
90
|
"timeout": 5
|
|
91
91
|
}
|
|
92
92
|
]
|
|
@@ -95,7 +95,7 @@
|
|
|
95
95
|
"hooks": [
|
|
96
96
|
{
|
|
97
97
|
"type": "command",
|
|
98
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/tool-label-stop.mjs\"",
|
|
98
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/tool-label-stop.mjs\"",
|
|
99
99
|
"timeout": 5,
|
|
100
100
|
"async": true
|
|
101
101
|
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# run-hook.sh — resilient launcher for Claude Code Node hooks (issue #2555).
|
|
3
|
+
#
|
|
4
|
+
# Under cgroup memory-ceiling pressure (the cgroup pinned at memory.max with
|
|
5
|
+
# reclaim lagging, page cache ~= cap), a freshly-spawned Node process can abort
|
|
6
|
+
# at STARTUP with exit 134 (SIGABRT) inside the libuv threadpool constructor —
|
|
7
|
+
# `Assertion failed: (0) == (uv_thread_create(...))` — BEFORE any hook code
|
|
8
|
+
# runs. It is a transient allocation failure, not a real hook error, but it
|
|
9
|
+
# surfaces a 🔴 issues card and skips the hook's work on that one tool call.
|
|
10
|
+
#
|
|
11
|
+
# This wrapper makes the invocation tolerant:
|
|
12
|
+
# 1. Shrink the libuv threadpool to 1 so Node needs the fewest possible
|
|
13
|
+
# thread-stack mmaps at startup (minimises the failure window).
|
|
14
|
+
# 2. Capture the hook payload from stdin ONCE and replay it on each attempt.
|
|
15
|
+
# The abort can land AFTER Node started draining the pipe, so a naive
|
|
16
|
+
# retry would feed the second attempt EMPTY stdin — a secret scanner would
|
|
17
|
+
# then scan nothing. Replaying the captured payload keeps the retry
|
|
18
|
+
# faithful.
|
|
19
|
+
# 3. Retry ONCE on a 134 abort after a brief backoff.
|
|
20
|
+
# 4. If it STILL aborts:
|
|
21
|
+
# - for SECURITY-CRITICAL hooks (secret-guard / secret-scrub) FAIL
|
|
22
|
+
# CLOSED — propagate 134 so the runtime cards it. 134 is a generic
|
|
23
|
+
# SIGABRT (assertion / OOM / any abort), not memory-pressure-specific,
|
|
24
|
+
# so silently returning 0 for a genuinely broken scanner would be a
|
|
25
|
+
# silent security BYPASS. Those hooks must never fail open.
|
|
26
|
+
# - for all other hooks, SKIP CLEANLY (exit 0) with a single stderr
|
|
27
|
+
# warn. A skipped label/context hook on one call is the documented,
|
|
28
|
+
# accepted degradation; a crash-card storm under memory pressure is
|
|
29
|
+
# not.
|
|
30
|
+
#
|
|
31
|
+
# Any non-134 exit status is passed through unchanged — real hook decisions
|
|
32
|
+
# (block/allow/non-zero) are never masked. `sleep 0.15` uses a fractional
|
|
33
|
+
# second (supported by GNU/BusyBox sleep, both present in the agent image).
|
|
34
|
+
#
|
|
35
|
+
# Usage (from hooks.json):
|
|
36
|
+
# sh "${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh" node "${CLAUDE_PLUGIN_ROOT}/hooks/<name>.mjs"
|
|
37
|
+
|
|
38
|
+
export UV_THREADPOOL_SIZE="${UV_THREADPOOL_SIZE:-1}"
|
|
39
|
+
|
|
40
|
+
# Identify the hook script (the .mjs argument) to decide fail-open vs closed.
|
|
41
|
+
hook_path=""
|
|
42
|
+
for a in "$@"; do
|
|
43
|
+
case "$a" in
|
|
44
|
+
*.mjs) hook_path="$a" ;;
|
|
45
|
+
esac
|
|
46
|
+
done
|
|
47
|
+
hook_name=$(basename "$hook_path" 2>/dev/null)
|
|
48
|
+
|
|
49
|
+
# Security-critical hooks MUST fail closed on a persistent abort.
|
|
50
|
+
fail_closed=0
|
|
51
|
+
case "$hook_name" in
|
|
52
|
+
secret-guard-pretool.mjs | secret-scrub-stop.mjs) fail_closed=1 ;;
|
|
53
|
+
esac
|
|
54
|
+
|
|
55
|
+
# Capture stdin once so both attempts see the SAME payload. Claude Code feeds
|
|
56
|
+
# the hook a single JSON object on stdin and closes it, so `cat` returns at
|
|
57
|
+
# EOF. (Command substitution strips trailing newlines, which JSON parsing does
|
|
58
|
+
# not care about.)
|
|
59
|
+
payload=$(cat)
|
|
60
|
+
|
|
61
|
+
run_hook() {
|
|
62
|
+
printf '%s' "$payload" | "$@"
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
run_hook "$@"
|
|
66
|
+
status=$?
|
|
67
|
+
if [ "$status" -ne 134 ]; then
|
|
68
|
+
exit "$status"
|
|
69
|
+
fi
|
|
70
|
+
|
|
71
|
+
# Transient thread-create abort — brief backoff, then retry once with the
|
|
72
|
+
# faithfully-replayed payload.
|
|
73
|
+
sleep 0.15
|
|
74
|
+
run_hook "$@"
|
|
75
|
+
status=$?
|
|
76
|
+
if [ "$status" -eq 134 ]; then
|
|
77
|
+
if [ "$fail_closed" -eq 1 ]; then
|
|
78
|
+
echo "run-hook: security hook '$hook_name' aborted twice with exit 134 — FAILING CLOSED (not skipping) (#2555)" >&2
|
|
79
|
+
exit 134
|
|
80
|
+
fi
|
|
81
|
+
echo "run-hook: '$hook_name' aborted twice with exit 134 (uv_thread_create under memory pressure) — skipping hook cleanly (#2555)" >&2
|
|
82
|
+
exit 0
|
|
83
|
+
fi
|
|
84
|
+
exit "$status"
|
|
@@ -69,6 +69,32 @@ export function detectModelUnavailable(
|
|
|
69
69
|
const sample = stderr.length > 16_384 ? stderr.slice(0, 16_384) : stderr
|
|
70
70
|
const lower = sample.toLowerCase()
|
|
71
71
|
|
|
72
|
+
// ── 0. Transient / server-side 429 (NOT account quota) — issue #2922 ────
|
|
73
|
+
// Anthropic emits a `rate_limit_error` whose message explicitly negates the
|
|
74
|
+
// account-quota reading: "Server is temporarily limiting requests (not your
|
|
75
|
+
// usage limit)". A negation-blind substring match on "usage limit" (step 1
|
|
76
|
+
// below) would misclassify this as `quota_exhausted`, firing a phantom fleet
|
|
77
|
+
// failover that self-cancels and leaves the turn dead. These are upstream
|
|
78
|
+
// throttles Claude Code retries internally with backoff — classify them as
|
|
79
|
+
// `overload` (the calm rate-limit path) BEFORE the quota substrings run, so
|
|
80
|
+
// the negation is honoured and no failover is announced.
|
|
81
|
+
const transientUpstreamSignals = [
|
|
82
|
+
'not your usage limit',
|
|
83
|
+
'not your account',
|
|
84
|
+
"not your account's",
|
|
85
|
+
'temporarily limiting requests',
|
|
86
|
+
'temporarily rate',
|
|
87
|
+
'server is temporarily',
|
|
88
|
+
'would exceed your account’s rate limit',
|
|
89
|
+
"would exceed your account's rate limit",
|
|
90
|
+
]
|
|
91
|
+
if (transientUpstreamSignals.some(s => lower.includes(s))) {
|
|
92
|
+
const resetAt = parseResetTime(sample)
|
|
93
|
+
return resetAt !== undefined
|
|
94
|
+
? { kind: 'overload', resetAt, raw: stderr }
|
|
95
|
+
: { kind: 'overload', raw: stderr }
|
|
96
|
+
}
|
|
97
|
+
|
|
72
98
|
// ── 1. Quota / billing exhaustion ──────────────────────────────────────
|
|
73
99
|
const quotaSignals = [
|
|
74
100
|
'out of extra usage',
|