switchroom 0.18.7 → 0.18.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/cli/switchroom.js +905 -758
  2. package/dist/host-control/main.js +1 -1
  3. package/package.json +1 -1
  4. package/profiles/_base/start.sh.hbs +111 -34
  5. package/skills/switchroom-runtime/SKILL.md +2 -0
  6. package/telegram-plugin/dist/gateway/gateway.js +1403 -657
  7. package/telegram-plugin/flood-circuit-breaker.ts +123 -0
  8. package/telegram-plugin/gateway/activity-card-store.ts +63 -18
  9. package/telegram-plugin/gateway/boot-card.ts +27 -0
  10. package/telegram-plugin/gateway/busy-ack.ts +106 -0
  11. package/telegram-plugin/gateway/gateway.ts +564 -85
  12. package/telegram-plugin/gateway/mental-model-propose-diff.ts +61 -5
  13. package/telegram-plugin/gateway/model-command.ts +23 -11
  14. package/telegram-plugin/gateway/session-model-file.ts +198 -0
  15. package/telegram-plugin/gateway/status-pin-store.ts +82 -22
  16. package/telegram-plugin/gateway/worker-pin-reaper.ts +114 -0
  17. package/telegram-plugin/hooks/hooks.json +10 -10
  18. package/telegram-plugin/hooks/run-hook.sh +84 -0
  19. package/telegram-plugin/model-unavailable.ts +26 -0
  20. package/telegram-plugin/pty-partial-handler.ts +39 -0
  21. package/telegram-plugin/render/rich-render.ts +79 -1
  22. package/telegram-plugin/retry-api-call.ts +62 -0
  23. package/telegram-plugin/shared/bot-runtime.ts +8 -1
  24. package/telegram-plugin/silence-poke.ts +14 -0
  25. package/telegram-plugin/stream-controller.ts +156 -38
  26. package/telegram-plugin/tests/activity-card-store.test.ts +47 -2
  27. package/telegram-plugin/tests/approval-card-restart-outcome.test.ts +218 -0
  28. package/telegram-plugin/tests/boot-card-flood-suppress.test.ts +111 -0
  29. package/telegram-plugin/tests/busy-ack-wiring.test.ts +118 -0
  30. package/telegram-plugin/tests/busy-ack.test.ts +121 -0
  31. package/telegram-plugin/tests/flood-circuit-breaker.test.ts +74 -0
  32. package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +177 -25
  33. package/telegram-plugin/tests/mental-model-name-entity-corruption.test.ts +119 -0
  34. package/telegram-plugin/tests/model-command.test.ts +2 -2
  35. package/telegram-plugin/tests/model-unavailable.test.ts +41 -0
  36. package/telegram-plugin/tests/pty-partial-handler.test.ts +56 -0
  37. package/telegram-plugin/tests/render/render-outbound-chunks.test.ts +98 -0
  38. package/telegram-plugin/tests/retry-api-call.test.ts +59 -0
  39. package/telegram-plugin/tests/run-hook-wrapper.test.ts +132 -0
  40. package/telegram-plugin/tests/session-model-file.test.ts +132 -0
  41. package/telegram-plugin/tests/slot-banner-boot-recovery.test.ts +3 -3
  42. package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
  43. package/telegram-plugin/tests/status-pin-store.test.ts +62 -6
  44. package/telegram-plugin/tests/stream-controller-chunk-cap.test.ts +122 -0
  45. package/telegram-plugin/tests/voice-send.test.ts +308 -0
  46. package/telegram-plugin/tests/worker-pin-reaper.test.ts +132 -0
  47. package/telegram-plugin/uat/scenarios/jtbd-deliberate-restart-resumes-dm.test.ts +118 -0
  48. package/telegram-plugin/uat/scenarios/jtbd-midflight-busy-ack-dm.test.ts +201 -0
  49. package/telegram-plugin/uat/scenarios/jtbd-worker-pin-lifecycle-dm.test.ts +208 -0
  50. package/telegram-plugin/uat/scenarios/vault-card-survives-gateway-restart-dm.test.ts +140 -0
  51. package/telegram-plugin/uat/scenarios/vault-deny-resumes-turn-dm.test.ts +84 -0
  52. package/telegram-plugin/uat/scenarios/vault-timeout-wakes-agent-dm.test.ts +91 -0
  53. package/telegram-plugin/voice-ondemand.ts +25 -1
  54. package/telegram-plugin/voice-send.ts +154 -0
@@ -27,6 +27,49 @@
27
27
  import { parseDocument } from "yaml";
28
28
  import { generateUnifiedDiff } from "../../src/web/config-diff.js";
29
29
 
30
+ /**
31
+ * Decode the canonical six HTML/XML entities to their literal characters — the
32
+ * config-write-boundary layer of the #2976 defense-in-depth.
33
+ *
34
+ * SCOPE / reachability (be honest about which half is load-bearing):
35
+ * - `source_query` decode is the REACHABLE half. A model can copy escaped
36
+ * entities out of its Telegram-HTML-rendered context into a proposal's
37
+ * free-form `source_query`, and undecoded it would steer recall on `R&D`
38
+ * instead of `R&D` — a durable, silent corruption of a memory-integrity
39
+ * field. Normalizing here means an escaped query can never LAND in
40
+ * `memory.mental_models[]`.
41
+ * - `name` decode is REDUNDANT belt-and-suspenders. The `mental_model_propose`
42
+ * gateway tool already slug-validates `name` against
43
+ * /^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/ BEFORE this runs, so an entity-bearing
44
+ * name is rejected upstream and can't reach here. It's kept only so this
45
+ * write boundary is self-contained if that gate ever moves.
46
+ * - The observed in-NAME corruption (`Nutrition Protocol & Deficit Status`,
47
+ * klanker 2026-07-06) actually arrived via the DIRECT `create_mental_model`
48
+ * Hindsight tool (`ensureMentalModel`, undecoded), NOT this propose path.
49
+ * That vector is OUT OF SCOPE here — covered by the steering.ts context-
50
+ * hygiene + in-service `Dockerfile.hindsight` normalization follow-ups.
51
+ *
52
+ * SINGLE PASS by design (matches issue #2976's success criteria): a
53
+ * double-escaped `R&D` decodes ONE layer to `R&D`, not all the way
54
+ * to `R&D` — we only undo the escaping switchroom itself applied when it
55
+ * rendered the model's prior output, never the user's literal intent. `&`
56
+ * is decoded LAST so `&amp;lt;` collapses to `&lt;` (one layer), not `<`.
57
+ *
58
+ * Accepted tradeoff: a `source_query` a human genuinely intended to contain a
59
+ * literal `&amp;` / `&lt;` / etc. (rare — a reflection question, not markup)
60
+ * will have that one layer decoded. Steering queries reading as escaped HTML is
61
+ * the far more common and more harmful case, so we optimize for it.
62
+ */
63
+ export function decodeCanonicalEntities(s: string): string {
64
+ return s
65
+ .replace(/&lt;/g, "<")
66
+ .replace(/&gt;/g, ">")
67
+ .replace(/&quot;/g, '"')
68
+ .replace(/&apos;/g, "'")
69
+ .replace(/&#39;/g, "'")
70
+ .replace(/&amp;/g, "&");
71
+ }
72
+
30
73
  /**
31
74
  * A proposed mental model, in the same snake_case shape the
32
75
  * `memory.mental_models[]` schema (#2874) accepts. `name` + `source_query`
@@ -125,20 +168,33 @@ export function buildMentalModelAppendDiff(args: {
125
168
  }
126
169
 
127
170
  // Duplicate-name guard (defense-in-depth; the executor also checks up front
128
- // so it never even posts a card for a dupe).
171
+ // so it never even posts a card for a dupe). Compare on the DECODED name so a
172
+ // re-propose of an entity-escaped variant collides with the already-stored
173
+ // literal (#2976) rather than sneaking in a corrupted twin.
174
+ const decodedName = decodeCanonicalEntities(spec.name);
129
175
  const existing = readDeclaredMentalModelNames(configText, agentName);
130
- if (existing.includes(spec.name)) {
176
+ if (existing.includes(decodedName)) {
131
177
  return {
132
178
  ok: false,
133
179
  error: "duplicate",
134
- detail: `mental model "${spec.name}" is already declared for ${agentName}`,
180
+ detail: `mental model "${decodedName}" is already declared for ${agentName}`,
135
181
  };
136
182
  }
137
183
 
184
+ // #2976 write-boundary normalization: decode any HTML/XML entities the model
185
+ // may have copied out of its own escaped context so the LITERAL characters
186
+ // land in config — never `&amp;` / `&lt;` etc. The load-bearing target is the
187
+ // free-form `source_query`; the `name` decode is redundant with the gateway
188
+ // slug gate (an entity-bearing name is rejected upstream) and kept only so
189
+ // this boundary is self-contained. Duplicate-name guard runs on the decoded
190
+ // name above.
191
+ const name = decodedName;
192
+ const source_query = decodeCanonicalEntities(spec.source_query);
193
+
138
194
  // Assemble the minimal, schema-clean declaration node.
139
195
  const item: Record<string, unknown> = {
140
- name: spec.name,
141
- source_query: spec.source_query,
196
+ name,
197
+ source_query,
142
198
  };
143
199
  if (spec.refresh_after_consolidation !== undefined) {
144
200
  item.refresh_after_consolidation = spec.refresh_after_consolidation;
@@ -149,7 +149,7 @@ export interface ModelCommandDeps {
149
149
  * Schedule a session-only switch TO a non-Claude (`sr-*` LiteLLM/OpenRouter)
150
150
  * model. claude's in-REPL `/model` picker rejects unknown `sr-*` ids, so an
151
151
  * inject can't set them. Instead the gateway writes the chosen token to the
152
- * `.session-model-override` carrier file and gracefully restarts the agent;
152
+ * durable `.session-model` override and gracefully restarts the agent;
153
153
  * the next boot launches `claude --model <token>` directly (LiteLLM routes
154
154
  * it, no picker validation). Session-only: reverts to the configured default
155
155
  * on the following restart. Wired to the same restart dispatch as
@@ -175,7 +175,7 @@ export interface ModelCommandReply {
175
175
  }
176
176
 
177
177
  const PERSIST_NOTE =
178
- '_Session-only lasts until restart. To persist, set \`model:\` in switchroom.yaml and restart._'
178
+ '_Sticky across switchroom-managed relaunches (\`/new\`, watchdog recovery); reverts on \`/restart\`, agent restart, crash, or external container restart. \`/model default\` clears it. To persist, set \`model:\` in switchroom.yaml._'
179
179
 
180
180
  function helpText(deps: ModelCommandDeps, reason?: string): ModelCommandReply {
181
181
  const srAliasExamples = Object.keys(SR_MODEL_ALIASES).map(a => `\`${a}\``).join(' · ')
@@ -240,7 +240,7 @@ export async function handleModelCommand(
240
240
  // the proxy at session start, not by claude's own REPL. A graceful restart
241
241
  // is the only clean path back to the native OAuth route. Route it through the
242
242
  // SAME carrier mechanism as a Claude → sr-* switch (scheduleModelRelaunch)
243
- // so the requested Claude model is written to `.session-model-override` and
243
+ // so the requested Claude model is written to the durable `.session-model` and
244
244
  // survives the restart — otherwise boot launches the configured default and
245
245
  // the operator's choice is silently dropped. start.sh's LiteLLM-down guard
246
246
  // only special-cases `sr-*` overrides, so a Claude token is never dropped.
@@ -294,7 +294,7 @@ export async function handleModelCommand(
294
294
  return {
295
295
  text: [
296
296
  `Switching to \`${deps.escapeHtml(model)}\` — restarting session (~30s).`,
297
- '_Session-only — reverts to the configured default on the next restart._',
297
+ PERSIST_NOTE,
298
298
  ].join('\n'),
299
299
  html: true,
300
300
  }
@@ -744,13 +744,20 @@ export interface ModelCallbackOutcome {
744
744
  /**
745
745
  * The canonical `claude --model` token (alias or full `claude-*` id) for a
746
746
  * Claude selection, when derivable — distinct from `selectedModel` (a display
747
- * name for /status). The gateway writes this to the `.session-model-override`
748
- * carrier on an sr-* → Claude transition so the requested Claude model survives
749
- * the restart (otherwise boot launches the configured default). Absent when the
750
- * target has no derivable token (e.g. the "Default" row → boot the configured
751
- * default).
747
+ * name for /status). The gateway persists this to the durable
748
+ * `.session-model` override so the confirmed switch survives
749
+ * switchroom-managed relaunches (and, on an sr-* Claude transition, its own
750
+ * restart). Absent when the target has no derivable token.
752
751
  */
753
752
  selectedModelToken?: string
753
+ /**
754
+ * True when the confirmed selection was the "Default (recommended)" row —
755
+ * i.e. the session is now on the configured default and any sticky
756
+ * `.session-model` override must be CLEARED (there is no token to persist;
757
+ * persisting nothing while leaving a stale override would re-apply the old
758
+ * model on the next keep-relaunch).
759
+ */
760
+ clearedDefault?: boolean
754
761
  /** Short toast for answerCallbackQuery. */
755
762
  answer: string
756
763
  /** Replacement dashboard (message edit). */
@@ -872,7 +879,7 @@ export async function handleModelMenuCallback(
872
879
  answer: `Switching to ${friendlyName} — restarting (~30s)`,
873
880
  reply: await menuWithBannerStatic(
874
881
  deps,
875
- `🔄 Switching session to **${deps.escapeHtml(friendlyName)}** — restarting (~30s). _Session-only; reverts to the configured default on the next restart._`,
882
+ `🔄 Switching session to **${deps.escapeHtml(friendlyName)}** — restarting (~30s).\n${PERSIST_NOTE}`,
876
883
  ),
877
884
  selectedModel: srName,
878
885
  }
@@ -949,11 +956,16 @@ export async function handleModelMenuCallback(
949
956
  // "Default (recommended)". If neither resolves, record nothing rather than lie.
950
957
  const token = canonicalClaudeToken(target.label)
951
958
  const selectedModel = sessionModelFromConfirmation(result.confirmation) ?? token ?? undefined
959
+ // The "Default (recommended)" row has no derivable token BY DESIGN — a
960
+ // confirmed switch to it means "back on the configured default", which the
961
+ // gateway must translate into clearing the sticky override.
962
+ const clearedDefault = token == null && /^default\b/i.test(target.label.trim())
952
963
  return {
953
964
  answer: deps.escapeHtml(result.confirmation),
954
965
  reply: await menuWithBanner(deps, `✅ ${deps.escapeHtml(result.confirmation)}`),
955
966
  ...(selectedModel ? { selectedModel } : {}),
956
967
  ...(token ? { selectedModelToken: token } : {}),
968
+ ...(clearedDefault ? { clearedDefault: true } : {}),
957
969
  }
958
970
  }
959
971
 
@@ -1030,7 +1042,7 @@ export function isKeptModelConfirmation(confirmation: string): boolean {
1030
1042
 
1031
1043
  /**
1032
1044
  * Normalize a picker ROW LABEL to a canonical `claude --model` token suitable
1033
- * for the `.session-model-override` carrier (aliases and full `claude-*` ids —
1045
+ * for the durable `.session-model` override (aliases and full `claude-*` ids —
1034
1046
  * NOT display strings like "Default (recommended)" or "Opus 4.8", which the CLI
1035
1047
  * flag rejects). Returns null when the label is a pure display label with no
1036
1048
  * derivable token: for the "Default" row that correctly means "boot the
@@ -0,0 +1,198 @@
1
+ /**
2
+ * Durable session-model stickiness — file helpers shared by the gateway.
3
+ *
4
+ * Two files in the bind-mounted agent state dir carry the contract
5
+ * (reference/rfcs/session-model-stickiness.md):
6
+ *
7
+ * - `.session-model` — the DURABLE session override written on every
8
+ * positively-confirmed `/model` switch. One-line JSON
9
+ * `{"model","configuredDefaultAtWrite","ts"}`. It is NOT consumed by a
10
+ * keep-path boot; start.sh deletes it on revert / invalidation /
11
+ * corruption / 7-day staleness, and the gateway deletes it on
12
+ * `/model default`.
13
+ *
14
+ * - `.relaunch-model-intent` — the ONE-SHOT intent bit for the next boot.
15
+ * One-line JSON `{"intent":"keep"|"revert","reason","ts"}`, atomic
16
+ * write, last-writer-wins. Boot default is REVERT (operator decision:
17
+ * a raw `docker restart` / host reboot / crash must revert to the yaml
18
+ * model), so every switchroom-managed KEEP path must stamp keep-intent
19
+ * BEFORE the bounce. start.sh consumes it (rm -f) every boot; a stale
20
+ * (>10 min by the embedded ts) or corrupt intent counts as no intent.
21
+ *
22
+ * The `model` token is always a canonical `claude --model` token (alias,
23
+ * `claude-*` id, or `sr-*` id) — NEVER a display label like "Opus 4.8".
24
+ * Shape-gated on write with the same regex start.sh greps with.
25
+ */
26
+
27
+ import { readFileSync, writeFileSync, renameSync, rmSync } from 'node:fs'
28
+ import { join } from 'node:path'
29
+ import { isValidModelArg } from './model-command.js'
30
+
31
+ export const SESSION_MODEL_FILE = '.session-model'
32
+ export const RELAUNCH_MODEL_INTENT_FILE = '.relaunch-model-intent'
33
+ export const CONFIGURED_DEFAULT_MODEL_FILE = '.configured-default-model'
34
+
35
+ export type RelaunchModelIntent = 'keep' | 'revert'
36
+
37
+ export interface SessionModelRecord {
38
+ model: string
39
+ configuredDefaultAtWrite: string
40
+ ts: number
41
+ }
42
+
43
+ /**
44
+ * Restart reasons whose semantics are "revert the session model to the
45
+ * configured default". Everything else `triggerSelfRestart` fires with is a
46
+ * switchroom-managed relaunch (watchdog recovery, drain-cap bounce,
47
+ * turn-complete deferred restart, fleet-fallback resume, sr-to-claude model
48
+ * switch, grant restarts) and KEEPS the override — the whole point of the
49
+ * stickiness contract. Enumerated in the RFC §3; default-keep here is safe
50
+ * because only gateway code calls triggerSelfRestart, and a bounce nobody
51
+ * stamped (crash, raw docker restart, deploy) reverts by boot default anyway.
52
+ */
53
+ const REVERT_RESTART_REASONS: ReadonlySet<string> = new Set(['inline-button-restart'])
54
+
55
+ /** Classify a triggerSelfRestart reason into the intent the boot should honor. */
56
+ export function intentForRestartReason(reason: string): RelaunchModelIntent {
57
+ return REVERT_RESTART_REASONS.has(reason) ? 'revert' : 'keep'
58
+ }
59
+
60
+ function atomicWrite(path: string, content: string): void {
61
+ const tmp = `${path}.tmp-${process.pid}-${Date.now()}`
62
+ writeFileSync(tmp, content, 'utf8')
63
+ renameSync(tmp, path)
64
+ }
65
+
66
+ /** Serialize a `.session-model` record (one line + trailing newline). */
67
+ export function serializeSessionModel(rec: SessionModelRecord): string {
68
+ return `${JSON.stringify({
69
+ model: rec.model,
70
+ configuredDefaultAtWrite: rec.configuredDefaultAtWrite,
71
+ ts: rec.ts,
72
+ })}\n`
73
+ }
74
+
75
+ /**
76
+ * Parse `.session-model` content. Returns null on corrupt JSON, a missing /
77
+ * non-string field, or a model token that fails the MODEL_ARG_RE shape gate
78
+ * (never trust an unvalidated string near `claude --model`).
79
+ */
80
+ export function parseSessionModel(text: string): SessionModelRecord | null {
81
+ try {
82
+ const raw = JSON.parse(text) as Partial<SessionModelRecord>
83
+ if (
84
+ typeof raw.model !== 'string' ||
85
+ typeof raw.configuredDefaultAtWrite !== 'string' ||
86
+ typeof raw.ts !== 'number' ||
87
+ !isValidModelArg(raw.model)
88
+ ) {
89
+ return null
90
+ }
91
+ return { model: raw.model, configuredDefaultAtWrite: raw.configuredDefaultAtWrite, ts: raw.ts }
92
+ } catch {
93
+ return null
94
+ }
95
+ }
96
+
97
+ /**
98
+ * Write the durable session override. Throws on a non-canonical token —
99
+ * callers must pass a `claude --model` token, never a display label
100
+ * (regression guard for the "Opus 4.5 persisted" class).
101
+ */
102
+ export function writeSessionModelFile(
103
+ agentDir: string,
104
+ model: string,
105
+ configuredDefaultAtWrite: string,
106
+ ): void {
107
+ if (!isValidModelArg(model)) {
108
+ throw new Error(`refusing to persist non-canonical session model token: ${JSON.stringify(model)}`)
109
+ }
110
+ atomicWrite(
111
+ join(agentDir, SESSION_MODEL_FILE),
112
+ serializeSessionModel({ model, configuredDefaultAtWrite, ts: Date.now() }),
113
+ )
114
+ }
115
+
116
+ /** Raw file text (for rollback snapshots), or null when absent/unreadable. */
117
+ export function readSessionModelFileRaw(agentDir: string): string | null {
118
+ try {
119
+ return readFileSync(join(agentDir, SESSION_MODEL_FILE), 'utf8')
120
+ } catch {
121
+ return null
122
+ }
123
+ }
124
+
125
+ /** Parsed durable override, or null when absent/corrupt. */
126
+ export function readSessionModelFile(agentDir: string): SessionModelRecord | null {
127
+ const raw = readSessionModelFileRaw(agentDir)
128
+ return raw == null ? null : parseSessionModel(raw)
129
+ }
130
+
131
+ /** Delete the durable override (`/model default`, rollback). Best-effort. */
132
+ export function clearSessionModelFile(agentDir: string): void {
133
+ try {
134
+ rmSync(join(agentDir, SESSION_MODEL_FILE), { force: true })
135
+ } catch {
136
+ /* best-effort */
137
+ }
138
+ }
139
+
140
+ /** Restore a rollback snapshot taken with readSessionModelFileRaw. */
141
+ export function restoreSessionModelFileRaw(agentDir: string, raw: string | null): void {
142
+ if (raw == null) {
143
+ clearSessionModelFile(agentDir)
144
+ return
145
+ }
146
+ try {
147
+ atomicWrite(join(agentDir, SESSION_MODEL_FILE), raw)
148
+ } catch {
149
+ /* best-effort */
150
+ }
151
+ }
152
+
153
+ /**
154
+ * Stamp the one-shot relaunch intent. MUST be called synchronously BEFORE
155
+ * the restart signal/dispatch it describes (write-before-kill invariant —
156
+ * the next start.sh boot reads this to decide keep vs revert). Best-effort:
157
+ * a failed write means the boot falls back to the default (revert), which
158
+ * is the safe side.
159
+ */
160
+ export function writeRelaunchModelIntent(
161
+ agentDir: string,
162
+ intent: RelaunchModelIntent,
163
+ reason: string,
164
+ ): void {
165
+ try {
166
+ atomicWrite(
167
+ join(agentDir, RELAUNCH_MODEL_INTENT_FILE),
168
+ `${JSON.stringify({ intent, reason, ts: Date.now() })}\n`,
169
+ )
170
+ } catch (err) {
171
+ process.stderr.write(
172
+ `telegram gateway: relaunch-model-intent write failed (boot will revert): ${(err as Error)?.message ?? String(err)}\n`,
173
+ )
174
+ }
175
+ }
176
+
177
+ /** Remove a stamped intent (rollback of a failed dispatch). Best-effort. */
178
+ export function clearRelaunchModelIntent(agentDir: string): void {
179
+ try {
180
+ rmSync(join(agentDir, RELAUNCH_MODEL_INTENT_FILE), { force: true })
181
+ } catch {
182
+ /* best-effort */
183
+ }
184
+ }
185
+
186
+ /**
187
+ * The resolved configured default start.sh recorded this boot
188
+ * (`.configured-default-model`, written before override resolution — the
189
+ * same resolver output the invalidation compare uses). Null when missing.
190
+ */
191
+ export function readConfiguredDefaultModel(agentDir: string): string | null {
192
+ try {
193
+ const v = readFileSync(join(agentDir, CONFIGURED_DEFAULT_MODEL_FILE), 'utf8').trim()
194
+ return v.length > 0 ? v : null
195
+ } catch {
196
+ return null
197
+ }
198
+ }
@@ -12,10 +12,13 @@
12
12
  * a dead session lingers.
13
13
  *
14
14
  * This makes cleanup self-contained across restart: every pin claim persists
15
- * here; on boot the gateway loads the persisted set and unpins each entry
16
- * (a status pin from a PRIOR session is stale by definition — the turn it
17
- * represented is over or crashed), then clears the store. It does NOT re-adopt
18
- * or re-pin it only cleans up.
15
+ * here; on boot the gateway loads the persisted set and unpins each
16
+ * work-scoped entry (a status pin from a PRIOR session is stale by definition
17
+ * — the turn it represented is over or crashed), dropping rows only after a
18
+ * successful unpin (failed ones are retained with an attempt counter for a
19
+ * next-boot retry — see runStatusPinBootCleanup). Time-scoped `tool:` rows
20
+ * (the `pin_message` MCP tool, #3001) survive boots until their `expiresAt`.
21
+ * It does NOT re-adopt or re-pin — it only cleans up.
19
22
  *
20
23
  * Shape choice — SNAPSHOT, not append-log, mirroring obligation-store.ts. The
21
24
  * claim set is tiny and bounded (one entry per in-flight pinned key, normally
@@ -55,8 +58,27 @@ export interface PersistedStatusPin {
55
58
  messageId: number
56
59
  /** True while the pin API call is in-flight / unconfirmed (see above). */
57
60
  pending?: boolean
61
+ /** Wall-clock ms after which this pin is stale and boot cleanup unpins it.
62
+ * Rows WITHOUT this field are work-scoped (fg:/wk:/banner:) — stale the
63
+ * moment their owning session dies, so boot cleanup unpins them
64
+ * unconditionally. Rows WITH it (the `tool:` pins written by the
65
+ * `pin_message` MCP tool, #3001) represent deliberate agent pins that have
66
+ * no "work finished" event: they SURVIVE restarts and are only swept once
67
+ * expired. */
68
+ expiresAt?: number
69
+ /** Boot-cleanup unpin retry counter (#3001). Incremented each boot the
70
+ * unpin fails (flood-wait exhausted / transient 5xx); the row is retained
71
+ * for retry until BOOT_UNPIN_MAX_ATTEMPTS, then forfeited. Absent = 0. */
72
+ attempts?: number
58
73
  }
59
74
 
75
+ /** How many boots may retry a failing boot-cleanup unpin before the row is
76
+ * forfeited. Unpins are idempotent (unpinning an already-unpinned or deleted
77
+ * message no-ops), so retrying across boots is safe; the cap only bounds a
78
+ * permanently-undeliverable unpin (chat gone, bot removed) so it cannot
79
+ * re-fail on every boot forever. */
80
+ export const BOOT_UNPIN_MAX_ATTEMPTS = 5
81
+
60
82
  /** Envelope version. v1 had no `pending` field; a v1 row loads as a confirmed
61
83
  * pin (pending undefined). v2 adds the optional `pending` flag. Both load
62
84
  * fail-open — an unknown/newer version yields []. */
@@ -74,7 +96,9 @@ function isPinRow(x: unknown): x is PersistedStatusPin {
74
96
  typeof o.chatId === 'string' &&
75
97
  o.chatId.length > 0 &&
76
98
  typeof o.messageId === 'number' &&
77
- (o.pending === undefined || typeof o.pending === 'boolean')
99
+ (o.pending === undefined || typeof o.pending === 'boolean') &&
100
+ (o.expiresAt === undefined || typeof o.expiresAt === 'number') &&
101
+ (o.attempts === undefined || typeof o.attempts === 'number')
78
102
  )
79
103
  }
80
104
 
@@ -170,16 +194,27 @@ export function pinnedMessageIsOurs(
170
194
  * the ordering + best-effort contract is unit-testable against the REAL code
171
195
  * (the gateway's thin wrapper just binds the live fs / unpin api / logger).
172
196
  *
173
- * Any pin persisted by a PRIOR session is stale by definition its turn ended
174
- * or the session crashed before its unpin reconcile ran. This includes records
175
- * left `pending` (the persist-intent-first write from `reconcileAndPersist-
176
- * StatusPin`): a crash between the pin API call and its confirming rewrite
177
- * leaves a pending record whose pin MAY have landed in Telegram, so we must
178
- * treat it exactly like a confirmed one and unpin it. We therefore best-effort
179
- * unpin EVERY persisted entry (confirmed and pending alike a failure is
180
- * non-fatal) and then EMPTY the store regardless, so a permanently-
181
- * undeliverable unpin can't re-run on every boot. We do NOT re-adopt or re-pin.
182
- * Returns the counts for logging/testing.
197
+ * The restart rule (#3001): a WORK-SCOPED pin persisted by a PRIOR session
198
+ * (fg:/wk:/banner: any row without `expiresAt`) is stale by definition its
199
+ * work ended or the session crashed before its unpin reconcile ran, so
200
+ * restart = reset: it is unpinned here. This includes records left `pending`
201
+ * (the persist-intent-first write from `reconcileAndPersistStatusPin`): a
202
+ * crash between the pin API call and its confirming rewrite leaves a pending
203
+ * record whose pin MAY have landed in Telegram, so we must treat it exactly
204
+ * like a confirmed one and unpin it.
205
+ *
206
+ * TIME-SCOPED rows (`tool:` pins from the `pin_message` MCP tool, carrying
207
+ * `expiresAt`) have no "work finished" event, so a restart does NOT reset
208
+ * them: an unexpired row is RETAINED untouched across boots and only unpinned
209
+ * once `now >= expiresAt`.
210
+ *
211
+ * RETRY-SAFETY (#3001): a row is dropped only AFTER its unpin resolves. A
212
+ * failing unpin (flood-wait exhausted / transient 5xx) retains the row with an
213
+ * incremented `attempts` counter so the NEXT boot retries, up to
214
+ * BOOT_UNPIN_MAX_ATTEMPTS — then the row is forfeited (a permanently-
215
+ * undeliverable unpin must not re-fail on every boot forever). Unpins are
216
+ * idempotent, so the retry can never double-unpin harmfully. We do NOT
217
+ * re-adopt or re-pin. Returns the counts for logging/testing.
183
218
  *
184
219
  * CRITICAL: the caller MUST only invoke this AFTER winning the startup mutex.
185
220
  * The store is a shared per-agent file; on a double-boot a losing gateway
@@ -189,27 +224,52 @@ export async function runStatusPinBootCleanup(args: {
189
224
  path: string
190
225
  fs: StatusPinStoreFsSeam
191
226
  unpin: (chatId: string, messageId: number) => Promise<unknown>
227
+ now?: number
192
228
  log?: (line: string) => void
193
- }): Promise<{ cleared: number; total: number }> {
229
+ }): Promise<{ cleared: number; retained: number; kept: number; total: number }> {
194
230
  const log = args.log ?? ((l: string) => process.stderr.write(l))
231
+ const now = args.now ?? Date.now()
195
232
  const persisted = loadStatusPins(args.path, args.fs)
196
- if (persisted.length === 0) return { cleared: 0, total: 0 }
233
+ if (persisted.length === 0) return { cleared: 0, retained: 0, kept: 0, total: 0 }
197
234
  let cleared = 0
235
+ let retained = 0
236
+ let kept = 0
237
+ const next: PersistedStatusPin[] = []
198
238
  for (const pin of persisted) {
239
+ // Unexpired time-scoped row (tool: pin): deliberately survives the
240
+ // restart — keep it as-is, no unpin.
241
+ if (pin.expiresAt != null && pin.expiresAt > now) {
242
+ next.push(pin)
243
+ kept++
244
+ continue
245
+ }
199
246
  try {
200
247
  await args.unpin(pin.chatId, pin.messageId)
201
248
  cleared++
202
249
  } catch (err) {
250
+ const attempts = (pin.attempts ?? 0) + 1
203
251
  log(
204
252
  `status-pin-store: boot cleanup unpin failed ` +
205
- `(chat=${pin.chatId} msg=${pin.messageId}): ${(err as Error).message}\n`,
253
+ `(chat=${pin.chatId} msg=${pin.messageId} attempt=${attempts}): ` +
254
+ `${(err as Error).message}\n`,
206
255
  )
256
+ if (attempts < BOOT_UNPIN_MAX_ATTEMPTS) {
257
+ // Retain for a retry on the next boot instead of forfeiting the
258
+ // orphan permanently (retry-safe boot sweep, #3001).
259
+ next.push({ ...pin, attempts })
260
+ retained++
261
+ } else {
262
+ log(
263
+ `status-pin-store: boot cleanup FORFEITING pin after ` +
264
+ `${attempts} failed unpin attempts ` +
265
+ `(key=${pin.pinKey} chat=${pin.chatId} msg=${pin.messageId}) — ` +
266
+ `will not retry again\n`,
267
+ )
268
+ }
207
269
  }
208
270
  }
209
- // Empty the store regardless — these claims belong to a dead session; leaving
210
- // them would re-attempt the same (already-tried) unpins on every future boot.
211
- persistStatusPins(args.path, args.fs, [], log)
212
- return { cleared, total: persisted.length }
271
+ persistStatusPins(args.path, args.fs, next, log)
272
+ return { cleared, retained, kept, total: persisted.length }
213
273
  }
214
274
 
215
275
  /**
@@ -0,0 +1,114 @@
1
+ /**
2
+ * worker-pin-reaper.ts — pure decision for the mid-session `wk:` pin sweep
3
+ * (#3001).
4
+ *
5
+ * Why this exists: the background-worker pin (`wk:<agentId>`, pinned on the
6
+ * `🛠 Worker` feed message while the worker runs) is normally unpinned by the
7
+ * worker's completion handler (`reconcileWorkerPin(agentId, null, false)` on
8
+ * the watcher's onFinish). But that event can be MISSED — watcher crash, SDK
9
+ * subprocess SIGKILL, a dropped JSONL tail — and then nothing ever unpins the
10
+ * worker's message until the next gateway boot. Log evidence on one agent:
11
+ * ~1120 pinChatMessage vs ~1014 unpinChatMessage with zero logged failures —
12
+ * a long tail of stale pins glued to the top of the chat.
13
+ *
14
+ * This module is the pure half (mirrors `runActivityCardMidSessionReaper`'s
15
+ * decide-over-injected-seams shape): given the currently-claimed `wk:` pins,
16
+ * a terminality predicate over the sub-agent registry, and a TTL, it returns
17
+ * the pins that should be unpinned NOW. The gateway executes each reap via
18
+ * `reconcileStatusPin(key, chat, { pinned: false })` so the in-memory claim
19
+ * AND the durable store row clear together.
20
+ *
21
+ * A pin is reaped when EITHER:
22
+ * - `terminal` — the registry says the worker reached a terminal status
23
+ * (completed | failed): its work is finished, the pin must go, however
24
+ * young it is. (A missed onFinish is exactly this case.)
25
+ * - `ttl` — the pin has been held past `ttlMs` AND the registry cannot
26
+ * vouch for the worker (no row / never linked / `stalled` / lookup
27
+ * error). A registry-confirmed RUNNING row exempts the pin from the TTL
28
+ * entirely: a healthy 7h worker keeps its pin for the whole run instead
29
+ * of churning unpin→re-pin every TTL. The stall detector demotes a dead
30
+ * worker's row out of 'running' within ~60s, so the TTL still catches
31
+ * true zombies.
32
+ *
33
+ * A worker the registry can't vouch for is still never touched before the
34
+ * TTL — the sweep can only ever shorten a stale pin's life, not a live one's.
35
+ */
36
+
37
+ /** Default TTL for a held worker pin: 6 hours. Rationale: worker turns are
38
+ * expected to run minutes-to-a-couple-of-hours (the watcher's own stall
39
+ * detection fires after ~60s of JSONL inactivity, and the longest sanctioned
40
+ * background dispatches are bounded by a single Claude session's lifetime).
41
+ * 6h comfortably exceeds any legitimate worker turn while bounding the
42
+ * stale-pin window to the same day instead of "until the next restart". */
43
+ export const WORKER_PIN_TTL_MS_DEFAULT = 6 * 60 * 60_000
44
+
45
+ export const WORKER_PIN_KEY_PREFIX = 'wk:'
46
+
47
+ /** One currently-claimed worker pin, flattened from the gateway's Maps. */
48
+ export interface WorkerPinCandidate {
49
+ /** Full pin key, `wk:<agentId>` shape. */
50
+ pinKey: string
51
+ /** Chat the pin lives in (from the gateway's pinKey → chatId registry). */
52
+ chatId: string
53
+ /** Wall-clock ms the claim was first taken (gateway's pinnedAt registry). */
54
+ pinnedAt: number
55
+ }
56
+
57
+ export interface WorkerPinReap extends WorkerPinCandidate {
58
+ reason: 'terminal' | 'ttl'
59
+ }
60
+
61
+ /** Extract the agentId from a `wk:<agentId>` pin key, or null for any other
62
+ * key shape (fg:/banner:/tool: keys are never worker-reaped). */
63
+ export function workerAgentIdOfPinKey(pinKey: string): string | null {
64
+ if (!pinKey.startsWith(WORKER_PIN_KEY_PREFIX)) return null
65
+ const agentId = pinKey.slice(WORKER_PIN_KEY_PREFIX.length)
66
+ return agentId.length > 0 ? agentId : null
67
+ }
68
+
69
+ /** The registry's view of a worker, distilled for the reap decision.
70
+ * - 'terminal' — row exists in completed | failed: reap now.
71
+ * - 'running' — row exists and is still 'running': NEVER reap, not even
72
+ * past the TTL. A healthy 7h worker must not get unpinned mid-run only
73
+ * for the next feed edit to re-pin it (pin/unpin churn every TTL). The
74
+ * subagent-watcher's stall detection demotes a dead worker's row out of
75
+ * 'running' within ~60s of its JSONL going quiet, so a true zombie can
76
+ * only hold 'running' briefly — the TTL still catches everything the
77
+ * registry has lost track of.
78
+ * - 'unknown' — no row / never linked / 'stalled' / lookup error: the
79
+ * TTL gate applies (the registry can't vouch for it). */
80
+ export type WorkerRegistryStatus = 'terminal' | 'running' | 'unknown'
81
+
82
+ /**
83
+ * Decide which claimed worker pins to unpin now. Pure: the registry lookup is
84
+ * an injected predicate (`statusOf` must return 'terminal' ONLY for a row in
85
+ * completed | failed, 'running' only for a live 'running' row, and 'unknown'
86
+ * for stalled / missing / lookup-error; a DB hiccup must degrade to 'unknown'
87
+ * — kept until the TTL — never to a spurious 'terminal' unpin).
88
+ */
89
+ export function decideWorkerPinReaps(args: {
90
+ pins: Iterable<WorkerPinCandidate>
91
+ statusOf: (agentId: string) => WorkerRegistryStatus
92
+ ttlMs: number
93
+ now: number
94
+ }): WorkerPinReap[] {
95
+ const reaps: WorkerPinReap[] = []
96
+ for (const pin of args.pins) {
97
+ const agentId = workerAgentIdOfPinKey(pin.pinKey)
98
+ if (agentId == null) continue // not a worker pin — never ours to reap
99
+ if (pin.chatId.length === 0) continue // can't unpin without a chat
100
+ const status = args.statusOf(agentId)
101
+ if (status === 'terminal') {
102
+ reaps.push({ ...pin, reason: 'terminal' })
103
+ continue
104
+ }
105
+ // A registry-confirmed RUNNING worker keeps its pin regardless of age —
106
+ // the TTL only reaps pins the registry can't vouch for (see
107
+ // WorkerRegistryStatus doc).
108
+ if (status === 'running') continue
109
+ if (args.now - pin.pinnedAt >= args.ttlMs) {
110
+ reaps.push({ ...pin, reason: 'ttl' })
111
+ }
112
+ }
113
+ return reaps
114
+ }