switchroom 0.18.15 → 0.18.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +16 -0
- package/dist/auth-broker/index.js +445 -10
- package/dist/cli/notion-write-pretool.mjs +16 -0
- package/dist/cli/switchroom.js +654 -479
- package/dist/host-control/main.js +20 -1
- package/dist/vault/approvals/kernel-server.js +16 -0
- package/dist/vault/broker/server.js +16 -0
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +81 -139
- package/telegram-plugin/bridge/bridge.ts +7 -1
- package/telegram-plugin/dist/bridge/bridge.js +26 -1
- package/telegram-plugin/dist/gateway/gateway.js +1758 -661
- package/telegram-plugin/dist/server.js +26 -1
- package/telegram-plugin/draft-stream.ts +78 -3
- package/telegram-plugin/fleet-fallback-resume.ts +26 -3
- package/telegram-plugin/gateway/approval-hold.ts +49 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +64 -22
- package/telegram-plugin/gateway/effort-command.ts +9 -7
- package/telegram-plugin/gateway/gateway.ts +627 -291
- package/telegram-plugin/gateway/linear-activity.ts +20 -4
- package/telegram-plugin/gateway/litellm-local-notice-wiring.ts +200 -0
- package/telegram-plugin/gateway/model-command.ts +96 -18
- package/telegram-plugin/gateway/pending-session-command.ts +10 -8
- package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
- package/telegram-plugin/gateway/session-model-file.ts +141 -172
- package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
- package/telegram-plugin/litellm-local-notice.ts +189 -0
- package/telegram-plugin/llm-error-present.ts +436 -0
- package/telegram-plugin/operator-events.ts +7 -1
- package/telegram-plugin/permission-title.ts +172 -10
- package/telegram-plugin/premium-recovery.ts +101 -0
- package/telegram-plugin/quota-watch.ts +16 -4
- package/telegram-plugin/raw-error-scrub.ts +73 -0
- package/telegram-plugin/retry-api-call.ts +8 -2
- package/telegram-plugin/runtime-metrics.ts +16 -0
- package/telegram-plugin/send-gate-degraded.test.ts +161 -8
- package/telegram-plugin/send-gate-observability.test.ts +140 -0
- package/telegram-plugin/send-gate-observability.ts +65 -20
- package/telegram-plugin/send-gate.test.ts +143 -1
- package/telegram-plugin/send-gate.ts +246 -23
- package/telegram-plugin/session-tail.ts +16 -0
- package/telegram-plugin/shared/local-time.ts +69 -0
- package/telegram-plugin/stream-controller.ts +143 -20
- package/telegram-plugin/stream-reply-handler.ts +12 -2
- package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
- package/telegram-plugin/tests/bot-api.harness.ts +7 -2
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
- package/telegram-plugin/tests/draft-stream.test.ts +110 -1
- package/telegram-plugin/tests/effort-command.test.ts +4 -4
- package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +5 -4
- package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +33 -19
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +47 -127
- package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
- package/telegram-plugin/tests/litellm-local-notice.test.ts +417 -0
- package/telegram-plugin/tests/llm-error-present.test.ts +380 -0
- package/telegram-plugin/tests/model-command.test.ts +84 -1
- package/telegram-plugin/tests/permission-title.test.ts +167 -4
- package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
- package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
- package/telegram-plugin/tests/quota-watch.test.ts +21 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +8 -3
- package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
- package/telegram-plugin/tests/session-model-file.test.ts +7 -155
- package/telegram-plugin/tests/stream-controller-send-gate.test.ts +521 -0
- package/telegram-plugin/tests/stream-reply-handler.test.ts +44 -0
- package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
- package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
- package/telegram-plugin/tests/worker-activity-feed.test.ts +212 -2
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
- package/telegram-plugin/tier-downgrade.ts +198 -0
- package/telegram-plugin/tool-activity-summary.ts +99 -0
- package/telegram-plugin/worker-activity-feed.ts +543 -368
|
@@ -1,32 +1,31 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Session-scoped `/model` carrier — file helpers shared by the gateway.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* Contract: reference/rfcs/session-model-stickiness.md §0.1 (rev 4, operator
|
|
5
|
+
* decision 2026-07-12 — SESSION-SCOPED, superseding the rev-3 keep-by-default).
|
|
6
|
+
* Files in the bind-mounted agent state dir:
|
|
6
7
|
*
|
|
7
|
-
* - `.session-model` —
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* (
|
|
13
|
-
*
|
|
8
|
+
* - `.session-model` — a CONSUME-ONCE carrier written ONLY immediately
|
|
9
|
+
* before a relaunch that applies the switch (the sr-* / sr→Claude paths
|
|
10
|
+
* and a queued /model persisted at graceful shutdown). One-line JSON
|
|
11
|
+
* `{"model","configuredDefaultAtWrite","ts"}`. start.sh applies it on the
|
|
12
|
+
* single boot that reads it and then deletes it; every SUBSEQUENT restart
|
|
13
|
+
* (deploy, /restart, /new, watchdog recovery, crash, raw docker restart)
|
|
14
|
+
* finds no carrier and boots the configured default. Live Claude switches
|
|
15
|
+
* do NOT write a carrier — they apply in-session and the explicit
|
|
16
|
+
* `claude --model <configured>` flag reverts them on the next boot.
|
|
17
|
+
* Cleared live by `/model default`; invalidation (corruption / the
|
|
18
|
+
* configured yaml default changed at the apply-boot) drops it and notifies
|
|
19
|
+
* the operator chat via `.session-model-alert`.
|
|
14
20
|
*
|
|
15
|
-
* - `.
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
* - `.session-effort` — the DURABLE effort override (#3039), same shape
|
|
25
|
-
* and lifecycle as `.session-model` with `level` in place of `model`:
|
|
26
|
-
* `{"level","configuredDefaultAtWrite","ts"}`. Written on every
|
|
27
|
-
* positively-confirmed `/effort` apply, resolved by start.sh into the
|
|
28
|
-
* relaunch's `--effort`, cleared only by `/effort default` or
|
|
29
|
-
* invalidation (with a boot alert).
|
|
21
|
+
* - `.session-effort` — the CONSUME-ONCE effort carrier (#3186, session-
|
|
22
|
+
* scoped like `.session-model`). Same shape with `level` in place of
|
|
23
|
+
* `model`: `{"level","configuredDefaultAtWrite","ts"}`. Written ONLY by
|
|
24
|
+
* the queued-command shutdown persist (a mid-turn `/effort` carried
|
|
25
|
+
* across the bounce); start.sh resolves it into the apply-boot's
|
|
26
|
+
* `--effort` and deletes it, so any subsequent restart reverts to the
|
|
27
|
+
* configured `thinking_effort`. Live `/effort` applies record in memory
|
|
28
|
+
* only. Cleared live by `/effort default`; invalidation alerts at boot.
|
|
30
29
|
*
|
|
31
30
|
* The `model` token is always a canonical `claude --model` token (alias,
|
|
32
31
|
* `claude-*` id, or `sr-*` id) — NEVER a display label like "Opus 4.8".
|
|
@@ -38,12 +37,7 @@ import { join } from 'node:path'
|
|
|
38
37
|
import { isValidModelArg } from './model-command.js'
|
|
39
38
|
|
|
40
39
|
export const SESSION_MODEL_FILE = '.session-model'
|
|
41
|
-
export const RELAUNCH_MODEL_INTENT_FILE = '.relaunch-model-intent'
|
|
42
40
|
export const CONFIGURED_DEFAULT_MODEL_FILE = '.configured-default-model'
|
|
43
|
-
/** Crashloop self-heal counter (start.sh stamps `<count> <epoch>` per fast boot). */
|
|
44
|
-
export const SESSION_MODEL_BOOT_ATTEMPTS_FILE = '.session-model-boot-attempts'
|
|
45
|
-
|
|
46
|
-
export type RelaunchModelIntent = 'keep' | 'revert'
|
|
47
41
|
|
|
48
42
|
export interface SessionModelRecord {
|
|
49
43
|
model: string
|
|
@@ -51,21 +45,6 @@ export interface SessionModelRecord {
|
|
|
51
45
|
ts: number
|
|
52
46
|
}
|
|
53
47
|
|
|
54
|
-
/**
|
|
55
|
-
* Classify a triggerSelfRestart reason into the intent the boot should honor.
|
|
56
|
-
*
|
|
57
|
-
* Since #3039 (operator contract 2026-07-11) EVERY restart reason keeps the
|
|
58
|
-
* override: a restart — user-tapped, watchdog, deploy, or crash — is not
|
|
59
|
-
* "clear my model". The override is cleared only by explicit user action
|
|
60
|
-
* (`/model default`) or invalidation at boot, both of which run their own
|
|
61
|
-
* paths. The 'revert' intent value remains recognised by start.sh (and this
|
|
62
|
-
* classifier's signature keeps it) so an older gateway's stamp still parses,
|
|
63
|
-
* but current code never emits it.
|
|
64
|
-
*/
|
|
65
|
-
export function intentForRestartReason(_reason: string): RelaunchModelIntent {
|
|
66
|
-
return 'keep'
|
|
67
|
-
}
|
|
68
|
-
|
|
69
48
|
function atomicWrite(path: string, content: string): void {
|
|
70
49
|
const tmp = `${path}.tmp-${process.pid}-${Date.now()}`
|
|
71
50
|
writeFileSync(tmp, content, 'utf8')
|
|
@@ -104,8 +83,8 @@ export function parseSessionModel(text: string): SessionModelRecord | null {
|
|
|
104
83
|
}
|
|
105
84
|
|
|
106
85
|
/**
|
|
107
|
-
* Write the
|
|
108
|
-
* callers must pass a `claude --model` token, never a display label
|
|
86
|
+
* Write the consume-once session-model carrier. Throws on a non-canonical
|
|
87
|
+
* token — callers must pass a `claude --model` token, never a display label
|
|
109
88
|
* (regression guard for the "Opus 4.5 persisted" class).
|
|
110
89
|
*/
|
|
111
90
|
export function writeSessionModelFile(
|
|
@@ -137,37 +116,10 @@ export function readSessionModelFile(agentDir: string): SessionModelRecord | nul
|
|
|
137
116
|
return raw == null ? null : parseSessionModel(raw)
|
|
138
117
|
}
|
|
139
118
|
|
|
140
|
-
/** Delete the
|
|
119
|
+
/** Delete the session-model carrier (`/model default`, rollback). Best-effort. */
|
|
141
120
|
export function clearSessionModelFile(agentDir: string): void {
|
|
142
121
|
try {
|
|
143
122
|
rmSync(join(agentDir, SESSION_MODEL_FILE), { force: true })
|
|
144
|
-
// #3042 item 4: also drop the kept-alert dedup sentinel so a future
|
|
145
|
-
// override of the same name re-alerts on its first kept boot.
|
|
146
|
-
rmSync(join(agentDir, '.session-model-kept-notified'), { force: true })
|
|
147
|
-
rmSync(join(agentDir, SESSION_MODEL_BOOT_ATTEMPTS_FILE), { force: true })
|
|
148
|
-
} catch {
|
|
149
|
-
/* best-effort */
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
/**
|
|
154
|
-
* #3043 item 2: clear ONLY the crashloop boot-attempts counter — a positive
|
|
155
|
-
* health signal from the gateway, not a carrier change.
|
|
156
|
-
*
|
|
157
|
-
* start.sh's self-heal (start.sh.hbs "Override crashloop self-heal") increments
|
|
158
|
-
* the counter on every boot that re-enters within 150s with the override still
|
|
159
|
-
* active, and clears a healthy override after 3 fast boots. That window can't
|
|
160
|
-
* tell a genuine crashloop from three quick OPERATOR hand-bounces of a healthy
|
|
161
|
-
* agent — both look like fast successive boots — so three deliberate restarts
|
|
162
|
-
* would spuriously wipe a working override. A boot that reaches bridge
|
|
163
|
-
* registration is proven healthy (the session came all the way up and the
|
|
164
|
-
* bridge connected), so the gateway deletes the counter there. Only boots that
|
|
165
|
-
* genuinely FAIL before the bridge registers now accumulate toward the 3-strike
|
|
166
|
-
* clear. Best-effort; absent file is fine.
|
|
167
|
-
*/
|
|
168
|
-
export function clearSessionModelBootAttempts(agentDir: string): void {
|
|
169
|
-
try {
|
|
170
|
-
rmSync(join(agentDir, SESSION_MODEL_BOOT_ATTEMPTS_FILE), { force: true })
|
|
171
123
|
} catch {
|
|
172
124
|
/* best-effort */
|
|
173
125
|
}
|
|
@@ -186,94 +138,6 @@ export function restoreSessionModelFileRaw(agentDir: string, raw: string | null)
|
|
|
186
138
|
}
|
|
187
139
|
}
|
|
188
140
|
|
|
189
|
-
/**
|
|
190
|
-
* Stamp the one-shot relaunch intent. MUST be called synchronously BEFORE
|
|
191
|
-
* the restart signal/dispatch it describes (write-before-kill invariant —
|
|
192
|
-
* the next start.sh boot reads this to decide keep vs revert). Best-effort:
|
|
193
|
-
* a failed write means the boot falls back to the default (keep, #3039) —
|
|
194
|
-
* the user's choice is preserved either way.
|
|
195
|
-
*/
|
|
196
|
-
export function writeRelaunchModelIntent(
|
|
197
|
-
agentDir: string,
|
|
198
|
-
intent: RelaunchModelIntent,
|
|
199
|
-
reason: string,
|
|
200
|
-
): void {
|
|
201
|
-
try {
|
|
202
|
-
atomicWrite(
|
|
203
|
-
join(agentDir, RELAUNCH_MODEL_INTENT_FILE),
|
|
204
|
-
`${JSON.stringify({ intent, reason, ts: Date.now() })}\n`,
|
|
205
|
-
)
|
|
206
|
-
} catch (err) {
|
|
207
|
-
process.stderr.write(
|
|
208
|
-
`telegram gateway: relaunch-model-intent write failed (boot will revert): ${(err as Error)?.message ?? String(err)}\n`,
|
|
209
|
-
)
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
/**
|
|
214
|
-
* Reason prefix the gateway's SIGTERM/SIGINT shutdown handler stamps on its
|
|
215
|
-
* deploy-survival keep-intent (#3017/#3018). Distinguishable on purpose:
|
|
216
|
-
* a gateway-only bounce (supervisor relaunch, bare gateway-unit restart)
|
|
217
|
-
* leaves that stamp UNCONSUMED on disk — start.sh only runs on a container
|
|
218
|
-
* boot — and the next gateway boot uses this prefix to recognise and clear
|
|
219
|
-
* the stale stamp (see clearStaleGatewayShutdownIntent).
|
|
220
|
-
*/
|
|
221
|
-
export const GATEWAY_SHUTDOWN_INTENT_REASON_PREFIX = 'gateway-shutdown:'
|
|
222
|
-
|
|
223
|
-
export interface RelaunchModelIntentRecord {
|
|
224
|
-
intent: RelaunchModelIntent
|
|
225
|
-
reason: string
|
|
226
|
-
ts: number
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
/** Parsed `.relaunch-model-intent`, or null when absent / corrupt / malformed. */
|
|
230
|
-
export function readRelaunchModelIntent(agentDir: string): RelaunchModelIntentRecord | null {
|
|
231
|
-
try {
|
|
232
|
-
const raw = readFileSync(join(agentDir, RELAUNCH_MODEL_INTENT_FILE), 'utf8')
|
|
233
|
-
const parsed = JSON.parse(raw) as Partial<RelaunchModelIntentRecord>
|
|
234
|
-
if (
|
|
235
|
-
(parsed.intent !== 'keep' && parsed.intent !== 'revert') ||
|
|
236
|
-
typeof parsed.reason !== 'string' ||
|
|
237
|
-
typeof parsed.ts !== 'number'
|
|
238
|
-
) {
|
|
239
|
-
return null
|
|
240
|
-
}
|
|
241
|
-
return { intent: parsed.intent, reason: parsed.reason, ts: parsed.ts }
|
|
242
|
-
} catch {
|
|
243
|
-
return null
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
/**
|
|
248
|
-
* Boot-time cleanup for the gateway-only-bounce hole (#3018 finding 4).
|
|
249
|
-
*
|
|
250
|
-
* A container-level stop/deploy consumes `.relaunch-model-intent` in start.sh
|
|
251
|
-
* BEFORE any gateway boots. So if a freshly-booting GATEWAY still sees an
|
|
252
|
-
* intent that a gateway shutdown handler stamped (reason carries
|
|
253
|
-
* GATEWAY_SHUTDOWN_INTENT_REASON_PREFIX), the preceding bounce was
|
|
254
|
-
* gateway-only — the container never restarted and the stamp is stale.
|
|
255
|
-
* Left in place, it could convert a genuine crash within the 10-min
|
|
256
|
-
* freshness window into a "keep", breaking the crash-reverts policy.
|
|
257
|
-
* Clear it. Never touches a triggerSelfRestart / user-slash stamp (those
|
|
258
|
-
* use their own un-prefixed reasons and precede a container bounce).
|
|
259
|
-
* Returns true when a stale stamp was cleared.
|
|
260
|
-
*/
|
|
261
|
-
export function clearStaleGatewayShutdownIntent(agentDir: string): boolean {
|
|
262
|
-
const rec = readRelaunchModelIntent(agentDir)
|
|
263
|
-
if (rec == null || !rec.reason.startsWith(GATEWAY_SHUTDOWN_INTENT_REASON_PREFIX)) return false
|
|
264
|
-
clearRelaunchModelIntent(agentDir)
|
|
265
|
-
return true
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
/** Remove a stamped intent (rollback of a failed dispatch). Best-effort. */
|
|
269
|
-
export function clearRelaunchModelIntent(agentDir: string): void {
|
|
270
|
-
try {
|
|
271
|
-
rmSync(join(agentDir, RELAUNCH_MODEL_INTENT_FILE), { force: true })
|
|
272
|
-
} catch {
|
|
273
|
-
/* best-effort */
|
|
274
|
-
}
|
|
275
|
-
}
|
|
276
|
-
|
|
277
141
|
/**
|
|
278
142
|
* The resolved configured default start.sh recorded this boot
|
|
279
143
|
* (`.configured-default-model`, written before override resolution — the
|
|
@@ -288,12 +152,13 @@ export function readConfiguredDefaultModel(agentDir: string): string | null {
|
|
|
288
152
|
}
|
|
289
153
|
}
|
|
290
154
|
|
|
291
|
-
// ───
|
|
155
|
+
// ─── Consume-once session-effort carrier (#3186) ────────────────────────────
|
|
292
156
|
//
|
|
293
|
-
// The `/effort` sibling of `.session-model
|
|
294
|
-
//
|
|
295
|
-
//
|
|
296
|
-
//
|
|
157
|
+
// The `/effort` sibling of `.session-model`, session-scoped. Written ONLY by
|
|
158
|
+
// the queued-command shutdown persist (a mid-turn /effort carried across the
|
|
159
|
+
// bounce); start.sh applies it on the single boot that reads it (`--effort
|
|
160
|
+
// <level>`) and deletes it. Cleared live by `/effort default`; invalidation
|
|
161
|
+
// (configured `thinking_effort:` changed / corrupt file) alerts once at boot.
|
|
297
162
|
|
|
298
163
|
export const SESSION_EFFORT_FILE = '.session-effort'
|
|
299
164
|
|
|
@@ -330,8 +195,9 @@ export function parseSessionEffort(text: string): SessionEffortRecord | null {
|
|
|
330
195
|
}
|
|
331
196
|
|
|
332
197
|
/**
|
|
333
|
-
* Write the
|
|
334
|
-
* the value is passed verbatim to `claude --effort` at the next boot
|
|
198
|
+
* Write the consume-once effort carrier. Throws on a non-allowlisted level —
|
|
199
|
+
* the value is passed verbatim to `claude --effort` at the next boot (the
|
|
200
|
+
* apply-boot, which consumes the file).
|
|
335
201
|
*/
|
|
336
202
|
export function writeSessionEffortFile(
|
|
337
203
|
agentDir: string,
|
|
@@ -347,7 +213,7 @@ export function writeSessionEffortFile(
|
|
|
347
213
|
)
|
|
348
214
|
}
|
|
349
215
|
|
|
350
|
-
/** Parsed
|
|
216
|
+
/** Parsed effort carrier, or null when absent/corrupt. */
|
|
351
217
|
export function readSessionEffortFile(agentDir: string): SessionEffortRecord | null {
|
|
352
218
|
try {
|
|
353
219
|
return parseSessionEffort(readFileSync(join(agentDir, SESSION_EFFORT_FILE), 'utf8'))
|
|
@@ -356,7 +222,7 @@ export function readSessionEffortFile(agentDir: string): SessionEffortRecord | n
|
|
|
356
222
|
}
|
|
357
223
|
}
|
|
358
224
|
|
|
359
|
-
/** Delete the
|
|
225
|
+
/** Delete the effort carrier (`/effort default`, leftover hygiene). Best-effort. */
|
|
360
226
|
export function clearSessionEffortFile(agentDir: string): void {
|
|
361
227
|
try {
|
|
362
228
|
rmSync(join(agentDir, SESSION_EFFORT_FILE), { force: true })
|
|
@@ -364,3 +230,106 @@ export function clearSessionEffortFile(agentDir: string): void {
|
|
|
364
230
|
/* best-effort */
|
|
365
231
|
}
|
|
366
232
|
}
|
|
233
|
+
|
|
234
|
+
// ─── Consume-once premium-recovery marker (tier-downgrade companion) ─────────
|
|
235
|
+
//
|
|
236
|
+
// A DURABLE marker the tier-downgrade writes when it walls a premium `/model`
|
|
237
|
+
// selection fleet-wide. It records the DROPPED premium token + the chats to
|
|
238
|
+
// notify, and survives the downgrade self-restart (start.sh never touches this
|
|
239
|
+
// name — unlike `.session-model`, which is consumed at boot). The gateway's
|
|
240
|
+
// `runQuotaWatch` tick reads it, and when the broker's `list-state` shows the
|
|
241
|
+
// premium tier servable again (deterministic per-account eligibility), fires
|
|
242
|
+
// EXACTLY ONE "available again" ping with a one-tap switch-back button, then
|
|
243
|
+
// clears the marker (at-most-once). Also cleared the instant the user re-issues
|
|
244
|
+
// `/model <premium>` manually (no stale ping). Shape-gated like the carriers
|
|
245
|
+
// above — the model token passes the same MODEL_ARG_RE gate.
|
|
246
|
+
|
|
247
|
+
export const PREMIUM_RECOVERY_FILE = '.premium-recovery'
|
|
248
|
+
|
|
249
|
+
export interface PremiumRecoveryRecord {
|
|
250
|
+
/** The dropped premium `/model` token (e.g. `fable`) to offer switching back to. */
|
|
251
|
+
premiumModel: string
|
|
252
|
+
/** Telegram chat ids to ping on recovery (the downgrade notice's allowFrom). */
|
|
253
|
+
chats: string[]
|
|
254
|
+
ts: number
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Parse `.premium-recovery` content. Null on corrupt JSON, a missing/wrong-typed
|
|
259
|
+
* field, a model token that fails the MODEL_ARG_RE shape gate, or a `chats`
|
|
260
|
+
* array that is not a non-empty list of non-empty strings.
|
|
261
|
+
*/
|
|
262
|
+
export function parsePremiumRecovery(text: string): PremiumRecoveryRecord | null {
|
|
263
|
+
try {
|
|
264
|
+
const raw = JSON.parse(text) as Partial<PremiumRecoveryRecord>
|
|
265
|
+
if (
|
|
266
|
+
typeof raw.premiumModel !== 'string' ||
|
|
267
|
+
!isValidModelArg(raw.premiumModel) ||
|
|
268
|
+
typeof raw.ts !== 'number' ||
|
|
269
|
+
!Array.isArray(raw.chats) ||
|
|
270
|
+
raw.chats.length === 0 ||
|
|
271
|
+
!raw.chats.every((c) => typeof c === 'string' && c.length > 0)
|
|
272
|
+
) {
|
|
273
|
+
return null
|
|
274
|
+
}
|
|
275
|
+
return { premiumModel: raw.premiumModel, chats: raw.chats, ts: raw.ts }
|
|
276
|
+
} catch {
|
|
277
|
+
return null
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* Write the premium-recovery marker. Throws on a non-canonical token (parity
|
|
283
|
+
* with the carriers) or an empty chat list — a marker with nowhere to ping is a
|
|
284
|
+
* bug, not a silent no-op.
|
|
285
|
+
*/
|
|
286
|
+
export function writePremiumRecoveryFile(
|
|
287
|
+
agentDir: string,
|
|
288
|
+
premiumModel: string,
|
|
289
|
+
chats: string[],
|
|
290
|
+
): void {
|
|
291
|
+
if (!isValidModelArg(premiumModel)) {
|
|
292
|
+
throw new Error(`refusing to persist non-canonical premium-recovery token: ${JSON.stringify(premiumModel)}`)
|
|
293
|
+
}
|
|
294
|
+
const clean = chats.filter((c) => typeof c === 'string' && c.length > 0)
|
|
295
|
+
if (clean.length === 0) {
|
|
296
|
+
throw new Error('refusing to persist premium-recovery marker with no chats to notify')
|
|
297
|
+
}
|
|
298
|
+
atomicWrite(
|
|
299
|
+
join(agentDir, PREMIUM_RECOVERY_FILE),
|
|
300
|
+
`${JSON.stringify({ premiumModel, chats: clean, ts: Date.now() })}\n`,
|
|
301
|
+
)
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Parsed premium-recovery marker, or null when absent/corrupt. A present-but-
|
|
306
|
+
* corrupt marker (null parse of a file that exists) is SWEPT on read: neither
|
|
307
|
+
* the ping path nor `clearPremiumRecoveryOnManualSwitch` can act on a null
|
|
308
|
+
* parse, so a garbled file would otherwise linger forever. Deleting it here is
|
|
309
|
+
* best-effort hygiene — the read still returns null either way.
|
|
310
|
+
*/
|
|
311
|
+
export function readPremiumRecoveryFile(agentDir: string): PremiumRecoveryRecord | null {
|
|
312
|
+
let raw: string
|
|
313
|
+
try {
|
|
314
|
+
raw = readFileSync(join(agentDir, PREMIUM_RECOVERY_FILE), 'utf8')
|
|
315
|
+
} catch {
|
|
316
|
+
return null // absent / unreadable — nothing on disk to sweep.
|
|
317
|
+
}
|
|
318
|
+
const parsed = parsePremiumRecovery(raw)
|
|
319
|
+
if (parsed == null) {
|
|
320
|
+
// Corrupt marker present on disk — garbage-collect it so it can't wedge the
|
|
321
|
+
// recovery path (best-effort; a failed unlink still returns null).
|
|
322
|
+
clearPremiumRecoveryFile(agentDir)
|
|
323
|
+
return null
|
|
324
|
+
}
|
|
325
|
+
return parsed
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** Delete the premium-recovery marker (consumed on ping / manual re-issue). Best-effort. */
|
|
329
|
+
export function clearPremiumRecoveryFile(agentDir: string): void {
|
|
330
|
+
try {
|
|
331
|
+
rmSync(join(agentDir, PREMIUM_RECOVERY_FILE), { force: true })
|
|
332
|
+
} catch {
|
|
333
|
+
/* best-effort */
|
|
334
|
+
}
|
|
335
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* tier-downgrade-wiring.ts — the gateway GLUE for the model-tier downgrade,
|
|
3
|
+
* behind injected deps so the orchestration is unit-testable without importing
|
|
4
|
+
* the whole gateway (mirrors throttle-tier-wiring.ts).
|
|
5
|
+
*
|
|
6
|
+
* The pure decision (`decideTierDowngrade`), the give-up/suppress routing
|
|
7
|
+
* (`planTierDowngrade`), and the user-facing wording (`renderTierDowngradeNotice`)
|
|
8
|
+
* all live in tier-downgrade.ts. This module owns the ORDER-SENSITIVE side
|
|
9
|
+
* effects the gateway performs off those verdicts — and the invariants a review
|
|
10
|
+
* cares about:
|
|
11
|
+
*
|
|
12
|
+
* - the resume gate is PEEKed (not armed) first, so the fallible carrier write
|
|
13
|
+
* happens before the single-flight latch is committed;
|
|
14
|
+
* - the consume-once `.session-model` carrier is written BEFORE the arm; a
|
|
15
|
+
* throwing write aborts the downgrade and leaves NO armed latch;
|
|
16
|
+
* - the latch is armed only AFTER a successful carrier write;
|
|
17
|
+
* - the durable premium-recovery marker write is best-effort (a failure never
|
|
18
|
+
* aborts the downgrade — the recovery ping simply doesn't arm);
|
|
19
|
+
* - the broadcast notice is the HONEST resume-on-default text (never a
|
|
20
|
+
* revert-to-premium promise) and is sent only on a real downgrade;
|
|
21
|
+
* - a `skip-inflight` resume-gate verdict returns `restart-pending` and emits
|
|
22
|
+
* NO notice and NO restart (the concurrent turn's armed restart resumes).
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import {
|
|
26
|
+
decideTierDowngrade,
|
|
27
|
+
planTierDowngrade,
|
|
28
|
+
type ResumeGateVerdict,
|
|
29
|
+
} from '../tier-downgrade.js'
|
|
30
|
+
|
|
31
|
+
export type TierDowngradeOutcome = 'downgraded' | 'restart-pending' | 'skip'
|
|
32
|
+
|
|
33
|
+
export interface TierDowngradeRunnerDeps {
|
|
34
|
+
/** Bind-mounted agent state dir, or null when unresolvable (→ skip). */
|
|
35
|
+
getAgentDir: () => string | null
|
|
36
|
+
/** Resolved configured-default token; '' / null → unresolved (never downgrade blind). */
|
|
37
|
+
getConfiguredDefault: () => string | null
|
|
38
|
+
/** Live session `/model` override, or null on the plain configured default. */
|
|
39
|
+
getSessionOverride: () => string | null
|
|
40
|
+
/** Model canonicalizer (the gateway passes resolveMainModel). */
|
|
41
|
+
resolve: (token: string) => string
|
|
42
|
+
/** PEEK the resume gate WITHOUT arming it. */
|
|
43
|
+
peekResumeGate: () => ResumeGateVerdict
|
|
44
|
+
/** Write the consume-once `.session-model` carrier for the default. May throw. */
|
|
45
|
+
writeCarrier: (agentDir: string, toModel: string, configuredDefault: string) => void
|
|
46
|
+
/** Commit the single-flight arm — called ONLY after a successful carrier write. */
|
|
47
|
+
armResumeGate: () => void
|
|
48
|
+
/** Persist the durable premium-recovery marker (best-effort; may throw — caught). */
|
|
49
|
+
writeRecoveryMarker: (agentDir: string, premiumModel: string) => void
|
|
50
|
+
/** Broadcast the honest downgrade notice to the operator chats. */
|
|
51
|
+
broadcastNotice: (markdown: string) => void
|
|
52
|
+
/** Fire the resume self-restart for `agent`. */
|
|
53
|
+
selfRestart: (agent: string) => void
|
|
54
|
+
/** The agent whose session self-restarts (SWITCHROOM_AGENT_NAME ?? triggerAgent). */
|
|
55
|
+
selfAgent: (triggerAgent: string) => string
|
|
56
|
+
/** Structured logger (stderr in prod, captured in test). */
|
|
57
|
+
log: (msg: string) => void
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Run the tier-downgrade glue. Returns:
|
|
62
|
+
* 'downgraded' — carrier written, latch armed, notice broadcast, restart fired.
|
|
63
|
+
* 'restart-pending' — a resume restart is already armed (concurrent turn); no
|
|
64
|
+
* notice, no restart (its restart replays the dead turn).
|
|
65
|
+
* 'skip' — not applicable / carrier write failed; caller falls
|
|
66
|
+
* through to its all-blocked give-up card.
|
|
67
|
+
*/
|
|
68
|
+
export function runTierDowngrade(
|
|
69
|
+
triggerAgent: string,
|
|
70
|
+
deps: TierDowngradeRunnerDeps,
|
|
71
|
+
): TierDowngradeOutcome {
|
|
72
|
+
const agentDir = deps.getAgentDir()
|
|
73
|
+
if (!agentDir) return 'skip'
|
|
74
|
+
const configuredDefault = deps.getConfiguredDefault() ?? ''
|
|
75
|
+
const decision = decideTierDowngrade({
|
|
76
|
+
sessionOverride: deps.getSessionOverride(),
|
|
77
|
+
configuredDefault,
|
|
78
|
+
resolve: deps.resolve,
|
|
79
|
+
})
|
|
80
|
+
// PEEK (no arm): the fallible carrier write must precede the latch commit.
|
|
81
|
+
const gateVerdict = deps.peekResumeGate()
|
|
82
|
+
const plan = planTierDowngrade(decision, gateVerdict, triggerAgent)
|
|
83
|
+
if (plan.kind === 'skip') {
|
|
84
|
+
if (decision.action === 'downgrade') {
|
|
85
|
+
deps.log(`[tier-downgrade] restart suppressed (${gateVerdict}) agent=${triggerAgent}`)
|
|
86
|
+
}
|
|
87
|
+
return 'skip'
|
|
88
|
+
}
|
|
89
|
+
if (plan.kind === 'suppress') {
|
|
90
|
+
deps.log(
|
|
91
|
+
`[tier-downgrade] give-up suppressed — a resume restart is already armed agent=${triggerAgent}`,
|
|
92
|
+
)
|
|
93
|
+
return 'restart-pending'
|
|
94
|
+
}
|
|
95
|
+
// plan.kind === 'downgrade'. Carrier BEFORE arm: a throwing write must not
|
|
96
|
+
// leave an armed latch with no pending restart.
|
|
97
|
+
try {
|
|
98
|
+
deps.writeCarrier(agentDir, plan.toModel, configuredDefault)
|
|
99
|
+
} catch (err) {
|
|
100
|
+
deps.log(
|
|
101
|
+
`[tier-downgrade] failed to write session-model carrier — aborting downgrade: ${(err as Error)?.message ?? err}`,
|
|
102
|
+
)
|
|
103
|
+
return 'skip'
|
|
104
|
+
}
|
|
105
|
+
deps.armResumeGate()
|
|
106
|
+
deps.log(
|
|
107
|
+
`[tier-downgrade] downgrading ${plan.fromModel} → ${plan.toModel} and resuming via self-restart agent=${triggerAgent}`,
|
|
108
|
+
)
|
|
109
|
+
// Durable premium-recovery marker — best-effort (survives the restart; the
|
|
110
|
+
// downgrade is the critical path and must not abort on a marker failure).
|
|
111
|
+
try {
|
|
112
|
+
deps.writeRecoveryMarker(agentDir, plan.fromModel)
|
|
113
|
+
} catch (err) {
|
|
114
|
+
deps.log(
|
|
115
|
+
`[tier-downgrade] premium-recovery marker write failed (non-fatal): ${(err as Error)?.message ?? err}`,
|
|
116
|
+
)
|
|
117
|
+
}
|
|
118
|
+
deps.broadcastNotice(plan.notice)
|
|
119
|
+
deps.selfRestart(deps.selfAgent(triggerAgent))
|
|
120
|
+
return 'downgraded'
|
|
121
|
+
}
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
import { GrammyError, HttpError } from 'grammy'
|
|
14
14
|
|
|
15
|
-
import { FLOOD_WAIT_ACTIVE } from '../retry-api-call.js'
|
|
15
|
+
import { FLOOD_WAIT_ACTIVE, LOCAL_RESOURCE_EXHAUSTED } from '../retry-api-call.js'
|
|
16
16
|
|
|
17
17
|
export type RejectionAction = 'shutdown' | 'log_only'
|
|
18
18
|
|
|
@@ -78,6 +78,19 @@ export function classifyRejection(
|
|
|
78
78
|
// exact amplification the #2923 circuit breaker exists to stop.
|
|
79
79
|
if (err instanceof Error && err.message === FLOOD_WAIT_ACTIVE) return 'log_only'
|
|
80
80
|
|
|
81
|
+
// LOCAL_RESOURCE_EXHAUSTED (#3099, sibling of FLOOD_WAIT_ACTIVE above):
|
|
82
|
+
// retry-api-call throws this plain Error marker when a send fails on a LOCAL
|
|
83
|
+
// disk/memory exhaustion (ENOSPC/EDQUOT/EIO/ENOMEM) rather than retrying it
|
|
84
|
+
// (#2923) — retrying a local-resource failure in a tight loop is what tripped
|
|
85
|
+
// the per-bot flood ban in the first place. A leaked one (a fire-and-forget
|
|
86
|
+
// send that wasn't wrapped in swallowingApiCall) must NOT crash the gateway:
|
|
87
|
+
// the box is ALREADY out of disk/memory, and a crash→restart drives a fresh
|
|
88
|
+
// round of boot-time sends and staging writes at a resource that is already
|
|
89
|
+
// exhausted — the exact amplification the #2923 marker exists to avoid. The
|
|
90
|
+
// degraded-state marker already carries this signal; a crash loop is the
|
|
91
|
+
// wrong way to surface a full disk. Same log_only posture as its sibling.
|
|
92
|
+
if (err instanceof Error && err.message === LOCAL_RESOURCE_EXHAUSTED) return 'log_only'
|
|
93
|
+
|
|
81
94
|
if (!isGrammy) return 'shutdown'
|
|
82
95
|
|
|
83
96
|
const e = err as { error_code?: number; description?: string }
|