switchroom 0.18.7 → 0.18.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/switchroom.js +905 -758
- package/dist/host-control/main.js +1 -1
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +111 -34
- package/skills/switchroom-runtime/SKILL.md +2 -0
- package/telegram-plugin/dist/gateway/gateway.js +1403 -657
- package/telegram-plugin/flood-circuit-breaker.ts +123 -0
- package/telegram-plugin/gateway/activity-card-store.ts +63 -18
- package/telegram-plugin/gateway/boot-card.ts +27 -0
- package/telegram-plugin/gateway/busy-ack.ts +106 -0
- package/telegram-plugin/gateway/gateway.ts +564 -85
- package/telegram-plugin/gateway/mental-model-propose-diff.ts +61 -5
- package/telegram-plugin/gateway/model-command.ts +23 -11
- package/telegram-plugin/gateway/session-model-file.ts +198 -0
- package/telegram-plugin/gateway/status-pin-store.ts +82 -22
- package/telegram-plugin/gateway/worker-pin-reaper.ts +114 -0
- package/telegram-plugin/hooks/hooks.json +10 -10
- package/telegram-plugin/hooks/run-hook.sh +84 -0
- package/telegram-plugin/model-unavailable.ts +26 -0
- package/telegram-plugin/pty-partial-handler.ts +39 -0
- package/telegram-plugin/render/rich-render.ts +79 -1
- package/telegram-plugin/retry-api-call.ts +62 -0
- package/telegram-plugin/shared/bot-runtime.ts +8 -1
- package/telegram-plugin/silence-poke.ts +14 -0
- package/telegram-plugin/stream-controller.ts +156 -38
- package/telegram-plugin/tests/activity-card-store.test.ts +47 -2
- package/telegram-plugin/tests/approval-card-restart-outcome.test.ts +218 -0
- package/telegram-plugin/tests/boot-card-flood-suppress.test.ts +111 -0
- package/telegram-plugin/tests/busy-ack-wiring.test.ts +118 -0
- package/telegram-plugin/tests/busy-ack.test.ts +121 -0
- package/telegram-plugin/tests/flood-circuit-breaker.test.ts +74 -0
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +177 -25
- package/telegram-plugin/tests/mental-model-name-entity-corruption.test.ts +119 -0
- package/telegram-plugin/tests/model-command.test.ts +2 -2
- package/telegram-plugin/tests/model-unavailable.test.ts +41 -0
- package/telegram-plugin/tests/pty-partial-handler.test.ts +56 -0
- package/telegram-plugin/tests/render/render-outbound-chunks.test.ts +98 -0
- package/telegram-plugin/tests/retry-api-call.test.ts +59 -0
- package/telegram-plugin/tests/run-hook-wrapper.test.ts +132 -0
- package/telegram-plugin/tests/session-model-file.test.ts +132 -0
- package/telegram-plugin/tests/slot-banner-boot-recovery.test.ts +3 -3
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
- package/telegram-plugin/tests/status-pin-store.test.ts +62 -6
- package/telegram-plugin/tests/stream-controller-chunk-cap.test.ts +122 -0
- package/telegram-plugin/tests/voice-send.test.ts +308 -0
- package/telegram-plugin/tests/worker-pin-reaper.test.ts +132 -0
- package/telegram-plugin/uat/scenarios/jtbd-deliberate-restart-resumes-dm.test.ts +118 -0
- package/telegram-plugin/uat/scenarios/jtbd-midflight-busy-ack-dm.test.ts +201 -0
- package/telegram-plugin/uat/scenarios/jtbd-worker-pin-lifecycle-dm.test.ts +208 -0
- package/telegram-plugin/uat/scenarios/vault-card-survives-gateway-restart-dm.test.ts +140 -0
- package/telegram-plugin/uat/scenarios/vault-deny-resumes-turn-dm.test.ts +84 -0
- package/telegram-plugin/uat/scenarios/vault-timeout-wakes-agent-dm.test.ts +91 -0
- package/telegram-plugin/voice-ondemand.ts +25 -1
- package/telegram-plugin/voice-send.ts +154 -0
|
@@ -27,6 +27,49 @@
|
|
|
27
27
|
import { parseDocument } from "yaml";
|
|
28
28
|
import { generateUnifiedDiff } from "../../src/web/config-diff.js";
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* Decode the canonical six HTML/XML entities to their literal characters — the
|
|
32
|
+
* config-write-boundary layer of the #2976 defense-in-depth.
|
|
33
|
+
*
|
|
34
|
+
* SCOPE / reachability (be honest about which half is load-bearing):
|
|
35
|
+
* - `source_query` decode is the REACHABLE half. A model can copy escaped
|
|
36
|
+
* entities out of its Telegram-HTML-rendered context into a proposal's
|
|
37
|
+
* free-form `source_query`, and undecoded it would steer recall on `R&D`
|
|
38
|
+
* instead of `R&D` — a durable, silent corruption of a memory-integrity
|
|
39
|
+
* field. Normalizing here means an escaped query can never LAND in
|
|
40
|
+
* `memory.mental_models[]`.
|
|
41
|
+
* - `name` decode is REDUNDANT belt-and-suspenders. The `mental_model_propose`
|
|
42
|
+
* gateway tool already slug-validates `name` against
|
|
43
|
+
* /^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/ BEFORE this runs, so an entity-bearing
|
|
44
|
+
* name is rejected upstream and can't reach here. It's kept only so this
|
|
45
|
+
* write boundary is self-contained if that gate ever moves.
|
|
46
|
+
* - The observed in-NAME corruption (`Nutrition Protocol & Deficit Status`,
|
|
47
|
+
* klanker 2026-07-06) actually arrived via the DIRECT `create_mental_model`
|
|
48
|
+
* Hindsight tool (`ensureMentalModel`, undecoded), NOT this propose path.
|
|
49
|
+
* That vector is OUT OF SCOPE here — covered by the steering.ts context-
|
|
50
|
+
* hygiene + in-service `Dockerfile.hindsight` normalization follow-ups.
|
|
51
|
+
*
|
|
52
|
+
* SINGLE PASS by design (matches issue #2976's success criteria): a
|
|
53
|
+
* double-escaped `R&D` decodes ONE layer to `R&D`, not all the way
|
|
54
|
+
* to `R&D` — we only undo the escaping switchroom itself applied when it
|
|
55
|
+
* rendered the model's prior output, never the user's literal intent. `&`
|
|
56
|
+
* is decoded LAST so `&lt;` collapses to `<` (one layer), not `<`.
|
|
57
|
+
*
|
|
58
|
+
* Accepted tradeoff: a `source_query` a human genuinely intended to contain a
|
|
59
|
+
* literal `&` / `<` / etc. (rare — a reflection question, not markup)
|
|
60
|
+
* will have that one layer decoded. Steering queries reading as escaped HTML is
|
|
61
|
+
* the far more common and more harmful case, so we optimize for it.
|
|
62
|
+
*/
|
|
63
|
+
export function decodeCanonicalEntities(s: string): string {
|
|
64
|
+
return s
|
|
65
|
+
.replace(/</g, "<")
|
|
66
|
+
.replace(/>/g, ">")
|
|
67
|
+
.replace(/"/g, '"')
|
|
68
|
+
.replace(/'/g, "'")
|
|
69
|
+
.replace(/'/g, "'")
|
|
70
|
+
.replace(/&/g, "&");
|
|
71
|
+
}
|
|
72
|
+
|
|
30
73
|
/**
|
|
31
74
|
* A proposed mental model, in the same snake_case shape the
|
|
32
75
|
* `memory.mental_models[]` schema (#2874) accepts. `name` + `source_query`
|
|
@@ -125,20 +168,33 @@ export function buildMentalModelAppendDiff(args: {
|
|
|
125
168
|
}
|
|
126
169
|
|
|
127
170
|
// Duplicate-name guard (defense-in-depth; the executor also checks up front
|
|
128
|
-
// so it never even posts a card for a dupe).
|
|
171
|
+
// so it never even posts a card for a dupe). Compare on the DECODED name so a
|
|
172
|
+
// re-propose of an entity-escaped variant collides with the already-stored
|
|
173
|
+
// literal (#2976) rather than sneaking in a corrupted twin.
|
|
174
|
+
const decodedName = decodeCanonicalEntities(spec.name);
|
|
129
175
|
const existing = readDeclaredMentalModelNames(configText, agentName);
|
|
130
|
-
if (existing.includes(
|
|
176
|
+
if (existing.includes(decodedName)) {
|
|
131
177
|
return {
|
|
132
178
|
ok: false,
|
|
133
179
|
error: "duplicate",
|
|
134
|
-
detail: `mental model "${
|
|
180
|
+
detail: `mental model "${decodedName}" is already declared for ${agentName}`,
|
|
135
181
|
};
|
|
136
182
|
}
|
|
137
183
|
|
|
184
|
+
// #2976 write-boundary normalization: decode any HTML/XML entities the model
|
|
185
|
+
// may have copied out of its own escaped context so the LITERAL characters
|
|
186
|
+
// land in config — never `&` / `<` etc. The load-bearing target is the
|
|
187
|
+
// free-form `source_query`; the `name` decode is redundant with the gateway
|
|
188
|
+
// slug gate (an entity-bearing name is rejected upstream) and kept only so
|
|
189
|
+
// this boundary is self-contained. Duplicate-name guard runs on the decoded
|
|
190
|
+
// name above.
|
|
191
|
+
const name = decodedName;
|
|
192
|
+
const source_query = decodeCanonicalEntities(spec.source_query);
|
|
193
|
+
|
|
138
194
|
// Assemble the minimal, schema-clean declaration node.
|
|
139
195
|
const item: Record<string, unknown> = {
|
|
140
|
-
name
|
|
141
|
-
source_query
|
|
196
|
+
name,
|
|
197
|
+
source_query,
|
|
142
198
|
};
|
|
143
199
|
if (spec.refresh_after_consolidation !== undefined) {
|
|
144
200
|
item.refresh_after_consolidation = spec.refresh_after_consolidation;
|
|
@@ -149,7 +149,7 @@ export interface ModelCommandDeps {
|
|
|
149
149
|
* Schedule a session-only switch TO a non-Claude (`sr-*` LiteLLM/OpenRouter)
|
|
150
150
|
* model. claude's in-REPL `/model` picker rejects unknown `sr-*` ids, so an
|
|
151
151
|
* inject can't set them. Instead the gateway writes the chosen token to the
|
|
152
|
-
* `.session-model
|
|
152
|
+
* durable `.session-model` override and gracefully restarts the agent;
|
|
153
153
|
* the next boot launches `claude --model <token>` directly (LiteLLM routes
|
|
154
154
|
* it, no picker validation). Session-only: reverts to the configured default
|
|
155
155
|
* on the following restart. Wired to the same restart dispatch as
|
|
@@ -175,7 +175,7 @@ export interface ModelCommandReply {
|
|
|
175
175
|
}
|
|
176
176
|
|
|
177
177
|
const PERSIST_NOTE =
|
|
178
|
-
'
|
|
178
|
+
'_Sticky across switchroom-managed relaunches (\`/new\`, watchdog recovery); reverts on \`/restart\`, agent restart, crash, or external container restart. \`/model default\` clears it. To persist, set \`model:\` in switchroom.yaml._'
|
|
179
179
|
|
|
180
180
|
function helpText(deps: ModelCommandDeps, reason?: string): ModelCommandReply {
|
|
181
181
|
const srAliasExamples = Object.keys(SR_MODEL_ALIASES).map(a => `\`${a}\``).join(' · ')
|
|
@@ -240,7 +240,7 @@ export async function handleModelCommand(
|
|
|
240
240
|
// the proxy at session start, not by claude's own REPL. A graceful restart
|
|
241
241
|
// is the only clean path back to the native OAuth route. Route it through the
|
|
242
242
|
// SAME carrier mechanism as a Claude → sr-* switch (scheduleModelRelaunch)
|
|
243
|
-
// so the requested Claude model is written to `.session-model
|
|
243
|
+
// so the requested Claude model is written to the durable `.session-model` and
|
|
244
244
|
// survives the restart — otherwise boot launches the configured default and
|
|
245
245
|
// the operator's choice is silently dropped. start.sh's LiteLLM-down guard
|
|
246
246
|
// only special-cases `sr-*` overrides, so a Claude token is never dropped.
|
|
@@ -294,7 +294,7 @@ export async function handleModelCommand(
|
|
|
294
294
|
return {
|
|
295
295
|
text: [
|
|
296
296
|
`Switching to \`${deps.escapeHtml(model)}\` — restarting session (~30s).`,
|
|
297
|
-
|
|
297
|
+
PERSIST_NOTE,
|
|
298
298
|
].join('\n'),
|
|
299
299
|
html: true,
|
|
300
300
|
}
|
|
@@ -744,13 +744,20 @@ export interface ModelCallbackOutcome {
|
|
|
744
744
|
/**
|
|
745
745
|
* The canonical `claude --model` token (alias or full `claude-*` id) for a
|
|
746
746
|
* Claude selection, when derivable — distinct from `selectedModel` (a display
|
|
747
|
-
* name for /status). The gateway
|
|
748
|
-
*
|
|
749
|
-
*
|
|
750
|
-
* target has no derivable token
|
|
751
|
-
* default).
|
|
747
|
+
* name for /status). The gateway persists this to the durable
|
|
748
|
+
* `.session-model` override so the confirmed switch survives
|
|
749
|
+
* switchroom-managed relaunches (and, on an sr-* → Claude transition, its own
|
|
750
|
+
* restart). Absent when the target has no derivable token.
|
|
752
751
|
*/
|
|
753
752
|
selectedModelToken?: string
|
|
753
|
+
/**
|
|
754
|
+
* True when the confirmed selection was the "Default (recommended)" row —
|
|
755
|
+
* i.e. the session is now on the configured default and any sticky
|
|
756
|
+
* `.session-model` override must be CLEARED (there is no token to persist;
|
|
757
|
+
* persisting nothing while leaving a stale override would re-apply the old
|
|
758
|
+
* model on the next keep-relaunch).
|
|
759
|
+
*/
|
|
760
|
+
clearedDefault?: boolean
|
|
754
761
|
/** Short toast for answerCallbackQuery. */
|
|
755
762
|
answer: string
|
|
756
763
|
/** Replacement dashboard (message edit). */
|
|
@@ -872,7 +879,7 @@ export async function handleModelMenuCallback(
|
|
|
872
879
|
answer: `Switching to ${friendlyName} — restarting (~30s)`,
|
|
873
880
|
reply: await menuWithBannerStatic(
|
|
874
881
|
deps,
|
|
875
|
-
`🔄 Switching session to **${deps.escapeHtml(friendlyName)}** — restarting (~30s)
|
|
882
|
+
`🔄 Switching session to **${deps.escapeHtml(friendlyName)}** — restarting (~30s).\n${PERSIST_NOTE}`,
|
|
876
883
|
),
|
|
877
884
|
selectedModel: srName,
|
|
878
885
|
}
|
|
@@ -949,11 +956,16 @@ export async function handleModelMenuCallback(
|
|
|
949
956
|
// "Default (recommended)". If neither resolves, record nothing rather than lie.
|
|
950
957
|
const token = canonicalClaudeToken(target.label)
|
|
951
958
|
const selectedModel = sessionModelFromConfirmation(result.confirmation) ?? token ?? undefined
|
|
959
|
+
// The "Default (recommended)" row has no derivable token BY DESIGN — a
|
|
960
|
+
// confirmed switch to it means "back on the configured default", which the
|
|
961
|
+
// gateway must translate into clearing the sticky override.
|
|
962
|
+
const clearedDefault = token == null && /^default\b/i.test(target.label.trim())
|
|
952
963
|
return {
|
|
953
964
|
answer: deps.escapeHtml(result.confirmation),
|
|
954
965
|
reply: await menuWithBanner(deps, `✅ ${deps.escapeHtml(result.confirmation)}`),
|
|
955
966
|
...(selectedModel ? { selectedModel } : {}),
|
|
956
967
|
...(token ? { selectedModelToken: token } : {}),
|
|
968
|
+
...(clearedDefault ? { clearedDefault: true } : {}),
|
|
957
969
|
}
|
|
958
970
|
}
|
|
959
971
|
|
|
@@ -1030,7 +1042,7 @@ export function isKeptModelConfirmation(confirmation: string): boolean {
|
|
|
1030
1042
|
|
|
1031
1043
|
/**
|
|
1032
1044
|
* Normalize a picker ROW LABEL to a canonical `claude --model` token suitable
|
|
1033
|
-
* for the `.session-model
|
|
1045
|
+
* for the durable `.session-model` override (aliases and full `claude-*` ids —
|
|
1034
1046
|
* NOT display strings like "Default (recommended)" or "Opus 4.8", which the CLI
|
|
1035
1047
|
* flag rejects). Returns null when the label is a pure display label with no
|
|
1036
1048
|
* derivable token: for the "Default" row that correctly means "boot the
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable session-model stickiness — file helpers shared by the gateway.
|
|
3
|
+
*
|
|
4
|
+
* Two files in the bind-mounted agent state dir carry the contract
|
|
5
|
+
* (reference/rfcs/session-model-stickiness.md):
|
|
6
|
+
*
|
|
7
|
+
* - `.session-model` — the DURABLE session override written on every
|
|
8
|
+
* positively-confirmed `/model` switch. One-line JSON
|
|
9
|
+
* `{"model","configuredDefaultAtWrite","ts"}`. It is NOT consumed by a
|
|
10
|
+
* keep-path boot; start.sh deletes it on revert / invalidation /
|
|
11
|
+
* corruption / 7-day staleness, and the gateway deletes it on
|
|
12
|
+
* `/model default`.
|
|
13
|
+
*
|
|
14
|
+
* - `.relaunch-model-intent` — the ONE-SHOT intent bit for the next boot.
|
|
15
|
+
* One-line JSON `{"intent":"keep"|"revert","reason","ts"}`, atomic
|
|
16
|
+
* write, last-writer-wins. Boot default is REVERT (operator decision:
|
|
17
|
+
* a raw `docker restart` / host reboot / crash must revert to the yaml
|
|
18
|
+
* model), so every switchroom-managed KEEP path must stamp keep-intent
|
|
19
|
+
* BEFORE the bounce. start.sh consumes it (rm -f) every boot; a stale
|
|
20
|
+
* (>10 min by the embedded ts) or corrupt intent counts as no intent.
|
|
21
|
+
*
|
|
22
|
+
* The `model` token is always a canonical `claude --model` token (alias,
|
|
23
|
+
* `claude-*` id, or `sr-*` id) — NEVER a display label like "Opus 4.8".
|
|
24
|
+
* Shape-gated on write with the same regex start.sh greps with.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { readFileSync, writeFileSync, renameSync, rmSync } from 'node:fs'
|
|
28
|
+
import { join } from 'node:path'
|
|
29
|
+
import { isValidModelArg } from './model-command.js'
|
|
30
|
+
|
|
31
|
+
export const SESSION_MODEL_FILE = '.session-model'
|
|
32
|
+
export const RELAUNCH_MODEL_INTENT_FILE = '.relaunch-model-intent'
|
|
33
|
+
export const CONFIGURED_DEFAULT_MODEL_FILE = '.configured-default-model'
|
|
34
|
+
|
|
35
|
+
export type RelaunchModelIntent = 'keep' | 'revert'
|
|
36
|
+
|
|
37
|
+
export interface SessionModelRecord {
|
|
38
|
+
model: string
|
|
39
|
+
configuredDefaultAtWrite: string
|
|
40
|
+
ts: number
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Restart reasons whose semantics are "revert the session model to the
|
|
45
|
+
* configured default". Everything else `triggerSelfRestart` fires with is a
|
|
46
|
+
* switchroom-managed relaunch (watchdog recovery, drain-cap bounce,
|
|
47
|
+
* turn-complete deferred restart, fleet-fallback resume, sr-to-claude model
|
|
48
|
+
* switch, grant restarts) and KEEPS the override — the whole point of the
|
|
49
|
+
* stickiness contract. Enumerated in the RFC §3; default-keep here is safe
|
|
50
|
+
* because only gateway code calls triggerSelfRestart, and a bounce nobody
|
|
51
|
+
* stamped (crash, raw docker restart, deploy) reverts by boot default anyway.
|
|
52
|
+
*/
|
|
53
|
+
const REVERT_RESTART_REASONS: ReadonlySet<string> = new Set(['inline-button-restart'])
|
|
54
|
+
|
|
55
|
+
/** Classify a triggerSelfRestart reason into the intent the boot should honor. */
|
|
56
|
+
export function intentForRestartReason(reason: string): RelaunchModelIntent {
|
|
57
|
+
return REVERT_RESTART_REASONS.has(reason) ? 'revert' : 'keep'
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function atomicWrite(path: string, content: string): void {
|
|
61
|
+
const tmp = `${path}.tmp-${process.pid}-${Date.now()}`
|
|
62
|
+
writeFileSync(tmp, content, 'utf8')
|
|
63
|
+
renameSync(tmp, path)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Serialize a `.session-model` record (one line + trailing newline). */
|
|
67
|
+
export function serializeSessionModel(rec: SessionModelRecord): string {
|
|
68
|
+
return `${JSON.stringify({
|
|
69
|
+
model: rec.model,
|
|
70
|
+
configuredDefaultAtWrite: rec.configuredDefaultAtWrite,
|
|
71
|
+
ts: rec.ts,
|
|
72
|
+
})}\n`
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Parse `.session-model` content. Returns null on corrupt JSON, a missing /
|
|
77
|
+
* non-string field, or a model token that fails the MODEL_ARG_RE shape gate
|
|
78
|
+
* (never trust an unvalidated string near `claude --model`).
|
|
79
|
+
*/
|
|
80
|
+
export function parseSessionModel(text: string): SessionModelRecord | null {
|
|
81
|
+
try {
|
|
82
|
+
const raw = JSON.parse(text) as Partial<SessionModelRecord>
|
|
83
|
+
if (
|
|
84
|
+
typeof raw.model !== 'string' ||
|
|
85
|
+
typeof raw.configuredDefaultAtWrite !== 'string' ||
|
|
86
|
+
typeof raw.ts !== 'number' ||
|
|
87
|
+
!isValidModelArg(raw.model)
|
|
88
|
+
) {
|
|
89
|
+
return null
|
|
90
|
+
}
|
|
91
|
+
return { model: raw.model, configuredDefaultAtWrite: raw.configuredDefaultAtWrite, ts: raw.ts }
|
|
92
|
+
} catch {
|
|
93
|
+
return null
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Write the durable session override. Throws on a non-canonical token —
|
|
99
|
+
* callers must pass a `claude --model` token, never a display label
|
|
100
|
+
* (regression guard for the "Opus 4.5 persisted" class).
|
|
101
|
+
*/
|
|
102
|
+
export function writeSessionModelFile(
|
|
103
|
+
agentDir: string,
|
|
104
|
+
model: string,
|
|
105
|
+
configuredDefaultAtWrite: string,
|
|
106
|
+
): void {
|
|
107
|
+
if (!isValidModelArg(model)) {
|
|
108
|
+
throw new Error(`refusing to persist non-canonical session model token: ${JSON.stringify(model)}`)
|
|
109
|
+
}
|
|
110
|
+
atomicWrite(
|
|
111
|
+
join(agentDir, SESSION_MODEL_FILE),
|
|
112
|
+
serializeSessionModel({ model, configuredDefaultAtWrite, ts: Date.now() }),
|
|
113
|
+
)
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Raw file text (for rollback snapshots), or null when absent/unreadable. */
|
|
117
|
+
export function readSessionModelFileRaw(agentDir: string): string | null {
|
|
118
|
+
try {
|
|
119
|
+
return readFileSync(join(agentDir, SESSION_MODEL_FILE), 'utf8')
|
|
120
|
+
} catch {
|
|
121
|
+
return null
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Parsed durable override, or null when absent/corrupt. */
|
|
126
|
+
export function readSessionModelFile(agentDir: string): SessionModelRecord | null {
|
|
127
|
+
const raw = readSessionModelFileRaw(agentDir)
|
|
128
|
+
return raw == null ? null : parseSessionModel(raw)
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Delete the durable override (`/model default`, rollback). Best-effort. */
|
|
132
|
+
export function clearSessionModelFile(agentDir: string): void {
|
|
133
|
+
try {
|
|
134
|
+
rmSync(join(agentDir, SESSION_MODEL_FILE), { force: true })
|
|
135
|
+
} catch {
|
|
136
|
+
/* best-effort */
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** Restore a rollback snapshot taken with readSessionModelFileRaw. */
|
|
141
|
+
export function restoreSessionModelFileRaw(agentDir: string, raw: string | null): void {
|
|
142
|
+
if (raw == null) {
|
|
143
|
+
clearSessionModelFile(agentDir)
|
|
144
|
+
return
|
|
145
|
+
}
|
|
146
|
+
try {
|
|
147
|
+
atomicWrite(join(agentDir, SESSION_MODEL_FILE), raw)
|
|
148
|
+
} catch {
|
|
149
|
+
/* best-effort */
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Stamp the one-shot relaunch intent. MUST be called synchronously BEFORE
|
|
155
|
+
* the restart signal/dispatch it describes (write-before-kill invariant —
|
|
156
|
+
* the next start.sh boot reads this to decide keep vs revert). Best-effort:
|
|
157
|
+
* a failed write means the boot falls back to the default (revert), which
|
|
158
|
+
* is the safe side.
|
|
159
|
+
*/
|
|
160
|
+
export function writeRelaunchModelIntent(
|
|
161
|
+
agentDir: string,
|
|
162
|
+
intent: RelaunchModelIntent,
|
|
163
|
+
reason: string,
|
|
164
|
+
): void {
|
|
165
|
+
try {
|
|
166
|
+
atomicWrite(
|
|
167
|
+
join(agentDir, RELAUNCH_MODEL_INTENT_FILE),
|
|
168
|
+
`${JSON.stringify({ intent, reason, ts: Date.now() })}\n`,
|
|
169
|
+
)
|
|
170
|
+
} catch (err) {
|
|
171
|
+
process.stderr.write(
|
|
172
|
+
`telegram gateway: relaunch-model-intent write failed (boot will revert): ${(err as Error)?.message ?? String(err)}\n`,
|
|
173
|
+
)
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** Remove a stamped intent (rollback of a failed dispatch). Best-effort. */
|
|
178
|
+
export function clearRelaunchModelIntent(agentDir: string): void {
|
|
179
|
+
try {
|
|
180
|
+
rmSync(join(agentDir, RELAUNCH_MODEL_INTENT_FILE), { force: true })
|
|
181
|
+
} catch {
|
|
182
|
+
/* best-effort */
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* The resolved configured default start.sh recorded this boot
|
|
188
|
+
* (`.configured-default-model`, written before override resolution — the
|
|
189
|
+
* same resolver output the invalidation compare uses). Null when missing.
|
|
190
|
+
*/
|
|
191
|
+
export function readConfiguredDefaultModel(agentDir: string): string | null {
|
|
192
|
+
try {
|
|
193
|
+
const v = readFileSync(join(agentDir, CONFIGURED_DEFAULT_MODEL_FILE), 'utf8').trim()
|
|
194
|
+
return v.length > 0 ? v : null
|
|
195
|
+
} catch {
|
|
196
|
+
return null
|
|
197
|
+
}
|
|
198
|
+
}
|
|
@@ -12,10 +12,13 @@
|
|
|
12
12
|
* a dead session lingers.
|
|
13
13
|
*
|
|
14
14
|
* This makes cleanup self-contained across restart: every pin claim persists
|
|
15
|
-
* here; on boot the gateway loads the persisted set and unpins each
|
|
16
|
-
* (a status pin from a PRIOR session is stale by definition
|
|
17
|
-
* represented is over or crashed),
|
|
18
|
-
*
|
|
15
|
+
* here; on boot the gateway loads the persisted set and unpins each
|
|
16
|
+
* work-scoped entry (a status pin from a PRIOR session is stale by definition
|
|
17
|
+
* — the turn it represented is over or crashed), dropping rows only after a
|
|
18
|
+
* successful unpin (failed ones are retained with an attempt counter for a
|
|
19
|
+
* next-boot retry — see runStatusPinBootCleanup). Time-scoped `tool:` rows
|
|
20
|
+
* (the `pin_message` MCP tool, #3001) survive boots until their `expiresAt`.
|
|
21
|
+
* It does NOT re-adopt or re-pin — it only cleans up.
|
|
19
22
|
*
|
|
20
23
|
* Shape choice — SNAPSHOT, not append-log, mirroring obligation-store.ts. The
|
|
21
24
|
* claim set is tiny and bounded (one entry per in-flight pinned key, normally
|
|
@@ -55,8 +58,27 @@ export interface PersistedStatusPin {
|
|
|
55
58
|
messageId: number
|
|
56
59
|
/** True while the pin API call is in-flight / unconfirmed (see above). */
|
|
57
60
|
pending?: boolean
|
|
61
|
+
/** Wall-clock ms after which this pin is stale and boot cleanup unpins it.
|
|
62
|
+
* Rows WITHOUT this field are work-scoped (fg:/wk:/banner:) — stale the
|
|
63
|
+
* moment their owning session dies, so boot cleanup unpins them
|
|
64
|
+
* unconditionally. Rows WITH it (the `tool:` pins written by the
|
|
65
|
+
* `pin_message` MCP tool, #3001) represent deliberate agent pins that have
|
|
66
|
+
* no "work finished" event: they SURVIVE restarts and are only swept once
|
|
67
|
+
* expired. */
|
|
68
|
+
expiresAt?: number
|
|
69
|
+
/** Boot-cleanup unpin retry counter (#3001). Incremented each boot the
|
|
70
|
+
* unpin fails (flood-wait exhausted / transient 5xx); the row is retained
|
|
71
|
+
* for retry until BOOT_UNPIN_MAX_ATTEMPTS, then forfeited. Absent = 0. */
|
|
72
|
+
attempts?: number
|
|
58
73
|
}
|
|
59
74
|
|
|
75
|
+
/** How many boots may retry a failing boot-cleanup unpin before the row is
|
|
76
|
+
* forfeited. Unpins are idempotent (unpinning an already-unpinned or deleted
|
|
77
|
+
* message no-ops), so retrying across boots is safe; the cap only bounds a
|
|
78
|
+
* permanently-undeliverable unpin (chat gone, bot removed) so it cannot
|
|
79
|
+
* re-fail on every boot forever. */
|
|
80
|
+
export const BOOT_UNPIN_MAX_ATTEMPTS = 5
|
|
81
|
+
|
|
60
82
|
/** Envelope version. v1 had no `pending` field; a v1 row loads as a confirmed
|
|
61
83
|
* pin (pending undefined). v2 adds the optional `pending` flag. Both load
|
|
62
84
|
* fail-open — an unknown/newer version yields []. */
|
|
@@ -74,7 +96,9 @@ function isPinRow(x: unknown): x is PersistedStatusPin {
|
|
|
74
96
|
typeof o.chatId === 'string' &&
|
|
75
97
|
o.chatId.length > 0 &&
|
|
76
98
|
typeof o.messageId === 'number' &&
|
|
77
|
-
(o.pending === undefined || typeof o.pending === 'boolean')
|
|
99
|
+
(o.pending === undefined || typeof o.pending === 'boolean') &&
|
|
100
|
+
(o.expiresAt === undefined || typeof o.expiresAt === 'number') &&
|
|
101
|
+
(o.attempts === undefined || typeof o.attempts === 'number')
|
|
78
102
|
)
|
|
79
103
|
}
|
|
80
104
|
|
|
@@ -170,16 +194,27 @@ export function pinnedMessageIsOurs(
|
|
|
170
194
|
* the ordering + best-effort contract is unit-testable against the REAL code
|
|
171
195
|
* (the gateway's thin wrapper just binds the live fs / unpin api / logger).
|
|
172
196
|
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
*
|
|
176
|
-
*
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
*
|
|
180
|
-
*
|
|
181
|
-
*
|
|
182
|
-
*
|
|
197
|
+
* The restart rule (#3001): a WORK-SCOPED pin persisted by a PRIOR session
|
|
198
|
+
* (fg:/wk:/banner: — any row without `expiresAt`) is stale by definition — its
|
|
199
|
+
* work ended or the session crashed before its unpin reconcile ran, so
|
|
200
|
+
* restart = reset: it is unpinned here. This includes records left `pending`
|
|
201
|
+
* (the persist-intent-first write from `reconcileAndPersistStatusPin`): a
|
|
202
|
+
* crash between the pin API call and its confirming rewrite leaves a pending
|
|
203
|
+
* record whose pin MAY have landed in Telegram, so we must treat it exactly
|
|
204
|
+
* like a confirmed one and unpin it.
|
|
205
|
+
*
|
|
206
|
+
* TIME-SCOPED rows (`tool:` pins from the `pin_message` MCP tool, carrying
|
|
207
|
+
* `expiresAt`) have no "work finished" event, so a restart does NOT reset
|
|
208
|
+
* them: an unexpired row is RETAINED untouched across boots and only unpinned
|
|
209
|
+
* once `now >= expiresAt`.
|
|
210
|
+
*
|
|
211
|
+
* RETRY-SAFETY (#3001): a row is dropped only AFTER its unpin resolves. A
|
|
212
|
+
* failing unpin (flood-wait exhausted / transient 5xx) retains the row with an
|
|
213
|
+
* incremented `attempts` counter so the NEXT boot retries, up to
|
|
214
|
+
* BOOT_UNPIN_MAX_ATTEMPTS — then the row is forfeited (a permanently-
|
|
215
|
+
* undeliverable unpin must not re-fail on every boot forever). Unpins are
|
|
216
|
+
* idempotent, so the retry can never double-unpin harmfully. We do NOT
|
|
217
|
+
* re-adopt or re-pin. Returns the counts for logging/testing.
|
|
183
218
|
*
|
|
184
219
|
* CRITICAL: the caller MUST only invoke this AFTER winning the startup mutex.
|
|
185
220
|
* The store is a shared per-agent file; on a double-boot a losing gateway
|
|
@@ -189,27 +224,52 @@ export async function runStatusPinBootCleanup(args: {
|
|
|
189
224
|
path: string
|
|
190
225
|
fs: StatusPinStoreFsSeam
|
|
191
226
|
unpin: (chatId: string, messageId: number) => Promise<unknown>
|
|
227
|
+
now?: number
|
|
192
228
|
log?: (line: string) => void
|
|
193
|
-
}): Promise<{ cleared: number; total: number }> {
|
|
229
|
+
}): Promise<{ cleared: number; retained: number; kept: number; total: number }> {
|
|
194
230
|
const log = args.log ?? ((l: string) => process.stderr.write(l))
|
|
231
|
+
const now = args.now ?? Date.now()
|
|
195
232
|
const persisted = loadStatusPins(args.path, args.fs)
|
|
196
|
-
if (persisted.length === 0) return { cleared: 0, total: 0 }
|
|
233
|
+
if (persisted.length === 0) return { cleared: 0, retained: 0, kept: 0, total: 0 }
|
|
197
234
|
let cleared = 0
|
|
235
|
+
let retained = 0
|
|
236
|
+
let kept = 0
|
|
237
|
+
const next: PersistedStatusPin[] = []
|
|
198
238
|
for (const pin of persisted) {
|
|
239
|
+
// Unexpired time-scoped row (tool: pin): deliberately survives the
|
|
240
|
+
// restart — keep it as-is, no unpin.
|
|
241
|
+
if (pin.expiresAt != null && pin.expiresAt > now) {
|
|
242
|
+
next.push(pin)
|
|
243
|
+
kept++
|
|
244
|
+
continue
|
|
245
|
+
}
|
|
199
246
|
try {
|
|
200
247
|
await args.unpin(pin.chatId, pin.messageId)
|
|
201
248
|
cleared++
|
|
202
249
|
} catch (err) {
|
|
250
|
+
const attempts = (pin.attempts ?? 0) + 1
|
|
203
251
|
log(
|
|
204
252
|
`status-pin-store: boot cleanup unpin failed ` +
|
|
205
|
-
`(chat=${pin.chatId} msg=${pin.messageId}):
|
|
253
|
+
`(chat=${pin.chatId} msg=${pin.messageId} attempt=${attempts}): ` +
|
|
254
|
+
`${(err as Error).message}\n`,
|
|
206
255
|
)
|
|
256
|
+
if (attempts < BOOT_UNPIN_MAX_ATTEMPTS) {
|
|
257
|
+
// Retain for a retry on the next boot instead of forfeiting the
|
|
258
|
+
// orphan permanently (retry-safe boot sweep, #3001).
|
|
259
|
+
next.push({ ...pin, attempts })
|
|
260
|
+
retained++
|
|
261
|
+
} else {
|
|
262
|
+
log(
|
|
263
|
+
`status-pin-store: boot cleanup FORFEITING pin after ` +
|
|
264
|
+
`${attempts} failed unpin attempts ` +
|
|
265
|
+
`(key=${pin.pinKey} chat=${pin.chatId} msg=${pin.messageId}) — ` +
|
|
266
|
+
`will not retry again\n`,
|
|
267
|
+
)
|
|
268
|
+
}
|
|
207
269
|
}
|
|
208
270
|
}
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
persistStatusPins(args.path, args.fs, [], log)
|
|
212
|
-
return { cleared, total: persisted.length }
|
|
271
|
+
persistStatusPins(args.path, args.fs, next, log)
|
|
272
|
+
return { cleared, retained, kept, total: persisted.length }
|
|
213
273
|
}
|
|
214
274
|
|
|
215
275
|
/**
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* worker-pin-reaper.ts — pure decision for the mid-session `wk:` pin sweep
|
|
3
|
+
* (#3001).
|
|
4
|
+
*
|
|
5
|
+
* Why this exists: the background-worker pin (`wk:<agentId>`, pinned on the
|
|
6
|
+
* `🛠 Worker` feed message while the worker runs) is normally unpinned by the
|
|
7
|
+
* worker's completion handler (`reconcileWorkerPin(agentId, null, false)` on
|
|
8
|
+
* the watcher's onFinish). But that event can be MISSED — watcher crash, SDK
|
|
9
|
+
* subprocess SIGKILL, a dropped JSONL tail — and then nothing ever unpins the
|
|
10
|
+
* worker's message until the next gateway boot. Log evidence on one agent:
|
|
11
|
+
* ~1120 pinChatMessage vs ~1014 unpinChatMessage with zero logged failures —
|
|
12
|
+
* a long tail of stale pins glued to the top of the chat.
|
|
13
|
+
*
|
|
14
|
+
* This module is the pure half (mirrors `runActivityCardMidSessionReaper`'s
|
|
15
|
+
* decide-over-injected-seams shape): given the currently-claimed `wk:` pins,
|
|
16
|
+
* a terminality predicate over the sub-agent registry, and a TTL, it returns
|
|
17
|
+
* the pins that should be unpinned NOW. The gateway executes each reap via
|
|
18
|
+
* `reconcileStatusPin(key, chat, { pinned: false })` so the in-memory claim
|
|
19
|
+
* AND the durable store row clear together.
|
|
20
|
+
*
|
|
21
|
+
* A pin is reaped when EITHER:
|
|
22
|
+
* - `terminal` — the registry says the worker reached a terminal status
|
|
23
|
+
* (completed | failed): its work is finished, the pin must go, however
|
|
24
|
+
* young it is. (A missed onFinish is exactly this case.)
|
|
25
|
+
* - `ttl` — the pin has been held past `ttlMs` AND the registry cannot
|
|
26
|
+
* vouch for the worker (no row / never linked / `stalled` / lookup
|
|
27
|
+
* error). A registry-confirmed RUNNING row exempts the pin from the TTL
|
|
28
|
+
* entirely: a healthy 7h worker keeps its pin for the whole run instead
|
|
29
|
+
* of churning unpin→re-pin every TTL. The stall detector demotes a dead
|
|
30
|
+
* worker's row out of 'running' within ~60s, so the TTL still catches
|
|
31
|
+
* true zombies.
|
|
32
|
+
*
|
|
33
|
+
* A worker the registry can't vouch for is still never touched before the
|
|
34
|
+
* TTL — the sweep can only ever shorten a stale pin's life, not a live one's.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
/** Default TTL for a held worker pin: 6 hours. Rationale: worker turns are
|
|
38
|
+
* expected to run minutes-to-a-couple-of-hours (the watcher's own stall
|
|
39
|
+
* detection fires after ~60s of JSONL inactivity, and the longest sanctioned
|
|
40
|
+
* background dispatches are bounded by a single Claude session's lifetime).
|
|
41
|
+
* 6h comfortably exceeds any legitimate worker turn while bounding the
|
|
42
|
+
* stale-pin window to the same day instead of "until the next restart". */
|
|
43
|
+
export const WORKER_PIN_TTL_MS_DEFAULT = 6 * 60 * 60_000
|
|
44
|
+
|
|
45
|
+
export const WORKER_PIN_KEY_PREFIX = 'wk:'
|
|
46
|
+
|
|
47
|
+
/** One currently-claimed worker pin, flattened from the gateway's Maps. */
|
|
48
|
+
export interface WorkerPinCandidate {
|
|
49
|
+
/** Full pin key, `wk:<agentId>` shape. */
|
|
50
|
+
pinKey: string
|
|
51
|
+
/** Chat the pin lives in (from the gateway's pinKey → chatId registry). */
|
|
52
|
+
chatId: string
|
|
53
|
+
/** Wall-clock ms the claim was first taken (gateway's pinnedAt registry). */
|
|
54
|
+
pinnedAt: number
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface WorkerPinReap extends WorkerPinCandidate {
|
|
58
|
+
reason: 'terminal' | 'ttl'
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Extract the agentId from a `wk:<agentId>` pin key, or null for any other
|
|
62
|
+
* key shape (fg:/banner:/tool: keys are never worker-reaped). */
|
|
63
|
+
export function workerAgentIdOfPinKey(pinKey: string): string | null {
|
|
64
|
+
if (!pinKey.startsWith(WORKER_PIN_KEY_PREFIX)) return null
|
|
65
|
+
const agentId = pinKey.slice(WORKER_PIN_KEY_PREFIX.length)
|
|
66
|
+
return agentId.length > 0 ? agentId : null
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** The registry's view of a worker, distilled for the reap decision.
|
|
70
|
+
* - 'terminal' — row exists in completed | failed: reap now.
|
|
71
|
+
* - 'running' — row exists and is still 'running': NEVER reap, not even
|
|
72
|
+
* past the TTL. A healthy 7h worker must not get unpinned mid-run only
|
|
73
|
+
* for the next feed edit to re-pin it (pin/unpin churn every TTL). The
|
|
74
|
+
* subagent-watcher's stall detection demotes a dead worker's row out of
|
|
75
|
+
* 'running' within ~60s of its JSONL going quiet, so a true zombie can
|
|
76
|
+
* only hold 'running' briefly — the TTL still catches everything the
|
|
77
|
+
* registry has lost track of.
|
|
78
|
+
* - 'unknown' — no row / never linked / 'stalled' / lookup error: the
|
|
79
|
+
* TTL gate applies (the registry can't vouch for it). */
|
|
80
|
+
export type WorkerRegistryStatus = 'terminal' | 'running' | 'unknown'
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Decide which claimed worker pins to unpin now. Pure: the registry lookup is
|
|
84
|
+
* an injected predicate (`statusOf` must return 'terminal' ONLY for a row in
|
|
85
|
+
* completed | failed, 'running' only for a live 'running' row, and 'unknown'
|
|
86
|
+
* for stalled / missing / lookup-error; a DB hiccup must degrade to 'unknown'
|
|
87
|
+
* — kept until the TTL — never to a spurious 'terminal' unpin).
|
|
88
|
+
*/
|
|
89
|
+
export function decideWorkerPinReaps(args: {
|
|
90
|
+
pins: Iterable<WorkerPinCandidate>
|
|
91
|
+
statusOf: (agentId: string) => WorkerRegistryStatus
|
|
92
|
+
ttlMs: number
|
|
93
|
+
now: number
|
|
94
|
+
}): WorkerPinReap[] {
|
|
95
|
+
const reaps: WorkerPinReap[] = []
|
|
96
|
+
for (const pin of args.pins) {
|
|
97
|
+
const agentId = workerAgentIdOfPinKey(pin.pinKey)
|
|
98
|
+
if (agentId == null) continue // not a worker pin — never ours to reap
|
|
99
|
+
if (pin.chatId.length === 0) continue // can't unpin without a chat
|
|
100
|
+
const status = args.statusOf(agentId)
|
|
101
|
+
if (status === 'terminal') {
|
|
102
|
+
reaps.push({ ...pin, reason: 'terminal' })
|
|
103
|
+
continue
|
|
104
|
+
}
|
|
105
|
+
// A registry-confirmed RUNNING worker keeps its pin regardless of age —
|
|
106
|
+
// the TTL only reaps pins the registry can't vouch for (see
|
|
107
|
+
// WorkerRegistryStatus doc).
|
|
108
|
+
if (status === 'running') continue
|
|
109
|
+
if (args.now - pin.pinnedAt >= args.ttlMs) {
|
|
110
|
+
reaps.push({ ...pin, reason: 'ttl' })
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return reaps
|
|
114
|
+
}
|