switchroom 0.18.29 → 0.18.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/handoff-briefing.sh +8 -2
- package/dist/agent-scheduler/index.js +111 -7
- package/dist/auth-broker/index.js +154 -16
- package/dist/cli/autoaccept-poll.js +8 -3
- package/dist/cli/drive-write-pretool.mjs +8 -3
- package/dist/cli/ms-365-write-pretool.mjs +158 -11
- package/dist/cli/notion-write-pretool.mjs +103 -4
- package/dist/cli/switchroom.js +2089 -1587
- package/dist/host-control/main.js +110 -13
- package/dist/vault/approvals/kernel-server.js +116 -13
- package/dist/vault/broker/server.js +314 -145
- package/package.json +3 -3
- package/profiles/_base/start.sh.hbs +172 -22
- package/telegram-plugin/dist/bridge/bridge.js +71 -47
- package/telegram-plugin/dist/gateway/gateway.js +601 -104
- package/telegram-plugin/dist/server.js +89 -64
- package/telegram-plugin/gateway/gateway.ts +280 -33
- package/telegram-plugin/gateway/model-command.ts +104 -0
- package/telegram-plugin/gateway/session-model-file.ts +40 -0
- package/telegram-plugin/gateway/turn-flush-suppression.ts +82 -0
- package/telegram-plugin/gateway/unhandled-message.ts +177 -0
- package/telegram-plugin/llm-error-present.ts +24 -0
- package/telegram-plugin/model-unavailable.ts +55 -0
- package/telegram-plugin/operator-events.ts +113 -0
- package/telegram-plugin/pending-user-notice.ts +88 -0
- package/telegram-plugin/shared/local-time.ts +43 -0
- package/telegram-plugin/tests/catch-all-forwarded-history.test.ts +103 -0
- package/telegram-plugin/tests/catch-all-unhandled-message.test.ts +264 -0
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +26 -2
- package/telegram-plugin/tests/litellm-proxy-auth-misconfig.test.ts +278 -0
- package/telegram-plugin/tests/local-time.test.ts +68 -1
- package/telegram-plugin/tests/model-command.test.ts +133 -0
- package/telegram-plugin/tests/session-model-file.test.ts +23 -0
- package/telegram-plugin/tests/turn-flush-suppression.test.ts +90 -0
- package/vendor/hindsight-memory/scripts/backfill_transcripts.py +399 -2
- package/vendor/hindsight-memory/scripts/lib/client.py +47 -0
- package/vendor/hindsight-memory/scripts/lib/content.py +53 -1
- package/vendor/hindsight-memory/scripts/lib/turnlog.py +450 -0
- package/vendor/hindsight-memory/scripts/tests/test_backfill_from_logs.py +467 -0
- package/vendor/hindsight-memory/tests/test_content.py +35 -0
|
@@ -84,6 +84,110 @@ export function isClaudeModel(name: string): boolean {
|
|
|
84
84
|
return lower.startsWith('claude-')
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
+
/**
|
|
88
|
+
* The outcome of a `/model` apply-boot, derived purely from the DETERMINISTIC
|
|
89
|
+
* post-boot signals (never optimistic):
|
|
90
|
+
*
|
|
91
|
+
* - `reason` — the clean-shutdown marker reason that keyed this boot as a
|
|
92
|
+
* `/model` apply-boot, e.g.
|
|
93
|
+
* `user: /model fable (session-only relaunch, menu)`.
|
|
94
|
+
* - `launched` — the contents of `.active-session-model`: the model start.sh
|
|
95
|
+
* actually passed to `claude --model` this boot.
|
|
96
|
+
* - `configured` — the resolved configured default model.
|
|
97
|
+
*
|
|
98
|
+
* Three outcomes:
|
|
99
|
+
* - `applied` — the launched model differs from the configured default, so
|
|
100
|
+
* the switch landed (a session-only override).
|
|
101
|
+
* - `default` — the operator asked for the configured default (`/model
|
|
102
|
+
* default`, or `/model <configured>`) and got it.
|
|
103
|
+
* - `not-applied` — the operator asked for a NON-default model, but the boot
|
|
104
|
+
* came back on the configured default: the switch SILENTLY
|
|
105
|
+
* failed to apply. The consume-once carrier was consumed by a
|
|
106
|
+
* boot that never launched the target (e.g. a wedged apply-boot
|
|
107
|
+
* that hit boot.lock_stale_recovered_boot_mismatch and reverted
|
|
108
|
+
* to the default). This is the case that previously emitted a
|
|
109
|
+
* MISLEADING green "✅ Now running <default> (the configured
|
|
110
|
+
* default)" card with no signal that the requested switch was
|
|
111
|
+
* lost. It self-corrects across boots: a later boot that
|
|
112
|
+
* genuinely launches the target reports `applied`.
|
|
113
|
+
*/
|
|
114
|
+
export type ModelSwitchConfirmation =
|
|
115
|
+
| { kind: 'applied'; launched: string }
|
|
116
|
+
| { kind: 'default'; launched: string }
|
|
117
|
+
| { kind: 'not-applied'; target: string; revertedTo: string }
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Extract the requested `/model <target>` token from a clean-shutdown reason
|
|
121
|
+
* (e.g. `user: /model fable (session-only relaunch, menu)` → `fable`). Returns
|
|
122
|
+
* null when the reason carries no `/model <token>` (a non-switch reason).
|
|
123
|
+
*/
|
|
124
|
+
export function parseModelSwitchTarget(reason: string): string | null {
|
|
125
|
+
const m = reason.match(/\/model\s+(\S+)/)
|
|
126
|
+
return m ? m[1] : null
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Reduce a model token to a comparable FAMILY key so an alias and its resolved
|
|
131
|
+
* full id compare equal (`sonnet` ≡ `claude-sonnet-5` ≡ `claude-sonnet-5-<date>`).
|
|
132
|
+
*
|
|
133
|
+
* This is the normalization the silent-revert classifier needs: `target` from
|
|
134
|
+
* the clean-shutdown reason is the RAW user token (an alias like `sonnet`),
|
|
135
|
+
* while `.active-session-model` / the configured default may be the RESOLVED
|
|
136
|
+
* full id start.sh wrote on a revert-with-alert path (config-default-changed at
|
|
137
|
+
* start.sh.hbs:1228, proxy-down at :1234). A naive string compare would flag
|
|
138
|
+
* `/model sonnet` on a `claude-sonnet-5`-default agent as "didn't apply" even
|
|
139
|
+
* though sonnet IS the default.
|
|
140
|
+
*
|
|
141
|
+
* NB `resolveMainModel` (scaffold.ts:1358) is NOT sufficient here — it only
|
|
142
|
+
* remaps the `default` alias / unset to the switchroom default; it passes
|
|
143
|
+
* `sonnet`/`opus`/etc. through unchanged. The alias↔full-id equivalence is a
|
|
144
|
+
* FAMILY reduction (same rule model-label.ts uses one-way), done here:
|
|
145
|
+
* - `claude-<family>-…` → `<family>` (e.g. `claude-sonnet-5` → `sonnet`)
|
|
146
|
+
* - a bare alias / any other token → itself, lowercased (`sonnet`, `sr-glm-5`)
|
|
147
|
+
* sr-* ids never carry a `claude-` prefix, so they stay verbatim — correct,
|
|
148
|
+
* since the reason token and `.active-session-model` both hold the same
|
|
149
|
+
* already-expanded sr-* id and compare equal directly.
|
|
150
|
+
*/
|
|
151
|
+
export function modelFamilyToken(token: string): string {
|
|
152
|
+
const t = token.trim().toLowerCase()
|
|
153
|
+
if (t.startsWith('claude-')) {
|
|
154
|
+
const family = t.slice('claude-'.length).split('-').filter((p) => p.length > 0)[0]
|
|
155
|
+
return family ?? t
|
|
156
|
+
}
|
|
157
|
+
return t
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Classify a `/model` apply-boot outcome from the post-boot signals. Pure so the
|
|
162
|
+
* confirmation-card decision is unit-testable without booting the gateway. The
|
|
163
|
+
* `not-applied` branch is the fix for the silent-revert bug: a non-default switch
|
|
164
|
+
* that reverted to the configured default must warn, not print a green ✅.
|
|
165
|
+
*/
|
|
166
|
+
export function classifyModelSwitchConfirmation(input: {
|
|
167
|
+
reason: string
|
|
168
|
+
launched: string
|
|
169
|
+
configured: string
|
|
170
|
+
}): ModelSwitchConfirmation {
|
|
171
|
+
const { reason, launched, configured } = input
|
|
172
|
+
const isApplyBoot = launched.length > 0 && launched !== configured
|
|
173
|
+
if (isApplyBoot) return { kind: 'applied', launched }
|
|
174
|
+
// launched === configured (or empty): either an intended default/revert, or a
|
|
175
|
+
// NON-default switch that silently reverted to the configured default.
|
|
176
|
+
const target = parseModelSwitchTarget(reason)
|
|
177
|
+
const revertedTo = launched.length > 0 ? launched : configured
|
|
178
|
+
// Family-normalize both sides so an alias target (`sonnet`) matches its
|
|
179
|
+
// resolved full-id revert (`claude-sonnet-5`) — otherwise a `/model sonnet`
|
|
180
|
+
// that reverted to the sonnet default would emit a WRONG "didn't apply" card.
|
|
181
|
+
if (
|
|
182
|
+
target != null &&
|
|
183
|
+
target.toLowerCase() !== 'default' &&
|
|
184
|
+
modelFamilyToken(target) !== modelFamilyToken(revertedTo)
|
|
185
|
+
) {
|
|
186
|
+
return { kind: 'not-applied', target, revertedTo }
|
|
187
|
+
}
|
|
188
|
+
return { kind: 'default', launched: revertedTo }
|
|
189
|
+
}
|
|
190
|
+
|
|
87
191
|
export type ParsedModelCommand =
|
|
88
192
|
| { kind: 'show' }
|
|
89
193
|
| { kind: 'set'; model: string }
|
|
@@ -38,6 +38,15 @@ import { isValidModelArg } from './model-command.js'
|
|
|
38
38
|
|
|
39
39
|
export const SESSION_MODEL_FILE = '.session-model'
|
|
40
40
|
export const CONFIGURED_DEFAULT_MODEL_FILE = '.configured-default-model'
|
|
41
|
+
/**
|
|
42
|
+
* Bounded-retry attempt counter for the session-model carrier (#3284).
|
|
43
|
+
* start.sh increments this on every boot that reads `.session-model` and
|
|
44
|
+
* applies it, and gives up (reverts + alerts) once it exceeds its bound. The
|
|
45
|
+
* gateway clears BOTH this file and the carrier on a healthy boot (see
|
|
46
|
+
* consumeSessionModelCarrierOnHealthyBoot) — the "healthy" signal a wedged
|
|
47
|
+
* boot cannot fake. Kept in sync with start.sh.hbs.
|
|
48
|
+
*/
|
|
49
|
+
export const SESSION_MODEL_BOOT_ATTEMPTS_FILE = '.session-model-boot-attempts'
|
|
41
50
|
|
|
42
51
|
export interface SessionModelRecord {
|
|
43
52
|
model: string
|
|
@@ -125,6 +134,37 @@ export function clearSessionModelFile(agentDir: string): void {
|
|
|
125
134
|
}
|
|
126
135
|
}
|
|
127
136
|
|
|
137
|
+
/**
|
|
138
|
+
* Consume the session-model carrier on a HEALTHY boot (#3284).
|
|
139
|
+
*
|
|
140
|
+
* Called once this gateway has ACQUIRED the boot lock (boot.lock_acquired) —
|
|
141
|
+
* the deterministic signal that THIS boot is the surviving healthy session,
|
|
142
|
+
* which a boot that wedges before lock-acquire cannot fake. Deletes BOTH the
|
|
143
|
+
* consume-once carrier and the bounded-retry attempt counter, so:
|
|
144
|
+
* - the next ordinary restart (deploy, /restart, crash) finds no carrier and
|
|
145
|
+
* reverts to the configured default (session-scoped semantics preserved);
|
|
146
|
+
* - a transient wedge BEFORE this point leaves the carrier in place, so the
|
|
147
|
+
* retry boot re-applies the intended model instead of silently reverting.
|
|
148
|
+
*
|
|
149
|
+
* This is what moved out of start.sh's old delete-before-apply: start.sh no
|
|
150
|
+
* longer consumes the carrier itself, it only APPLIES it and lets this healthy
|
|
151
|
+
* signal do the consume. Best-effort — a failure here just means the carrier is
|
|
152
|
+
* re-read (and re-applied, harmlessly) on the next boot, still bounded by the
|
|
153
|
+
* counter. Idempotent.
|
|
154
|
+
*/
|
|
155
|
+
export function consumeSessionModelCarrierOnHealthyBoot(agentDir: string): void {
|
|
156
|
+
try {
|
|
157
|
+
rmSync(join(agentDir, SESSION_MODEL_FILE), { force: true })
|
|
158
|
+
} catch {
|
|
159
|
+
/* best-effort */
|
|
160
|
+
}
|
|
161
|
+
try {
|
|
162
|
+
rmSync(join(agentDir, SESSION_MODEL_BOOT_ATTEMPTS_FILE), { force: true })
|
|
163
|
+
} catch {
|
|
164
|
+
/* best-effort */
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
128
168
|
/** Restore a rollback snapshot taken with readSessionModelFileRaw. */
|
|
129
169
|
export function restoreSessionModelFileRaw(agentDir: string, raw: string | null): void {
|
|
130
170
|
if (raw == null) {
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turn-flush pre-delivery suppression decision (S1 fix, fable red-team
|
|
3
|
+
* 2026-07-17 — `~klanker/work/fable-redteam-delivery-20260717/REDTEAM.md`).
|
|
4
|
+
*
|
|
5
|
+
* Before the turn-flush backstop posts the captured terminal answer, it asks:
|
|
6
|
+
* "did a reply already land for this turn in the last ~2s?" (a race guard for
|
|
7
|
+
* a reply whose IPC signal hadn't registered when `decideTurnFlush` ran, and
|
|
8
|
+
* for answer-stream materializations that already delivered the answer text).
|
|
9
|
+
*
|
|
10
|
+
* The OLD predicate — `getRecentOutboundCount(chatId, 2) > 0` — counted ANY
|
|
11
|
+
* assistant row in the WHOLE chat: a background worker's `progress_update`, a
|
|
12
|
+
* command ack / restart notice, or a reply in a DIFFERENT forum topic all
|
|
13
|
+
* suppressed the flush, and the branch then CLOSED the delivery obligation —
|
|
14
|
+
* the user's real answer was dropped with no re-present. This module scopes
|
|
15
|
+
* the predicate to a SUBSTANTIVE (≥ `FINAL_ANSWER_MIN_CHARS`) outbound in the
|
|
16
|
+
* SAME thread, via the injected `hasSubstantiveOutbound` (the gateway wires
|
|
17
|
+
* `hasOutboundDeliveredSince`, whose thread/length semantics are pinned by
|
|
18
|
+
* `history.test.ts`).
|
|
19
|
+
*
|
|
20
|
+
* Residual (documented, accepted): a ≥200-char same-thread non-answer outbound
|
|
21
|
+
* (e.g. an unusually long progress update) inside the 2s window still
|
|
22
|
+
* suppresses — the history schema has no message-kind column to discriminate
|
|
23
|
+
* further. That residual is why the CALLER must NOT close the obligation on
|
|
24
|
+
* suppression: a genuine reply closes its own obligation idempotently, while a
|
|
25
|
+
* false-positive suppression leaves it open for the liveness floor / sweep.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import { FINAL_ANSWER_MIN_CHARS } from '../final-answer-detect.js'
|
|
29
|
+
|
|
30
|
+
/** How far back a just-landed reply can be and still suppress the flush. */
|
|
31
|
+
export const FLUSH_SUPPRESSION_WINDOW_MS = 2000
|
|
32
|
+
|
|
33
|
+
export interface FlushSuppressionDeps {
|
|
34
|
+
/** Durable substantive-outbound oracle — the gateway passes
|
|
35
|
+
* `hasOutboundDeliveredSince` (thread-scoped, length-floored). */
|
|
36
|
+
hasSubstantiveOutbound(
|
|
37
|
+
chatId: string,
|
|
38
|
+
sinceMs: number,
|
|
39
|
+
threadId: number | null,
|
|
40
|
+
minChars: number,
|
|
41
|
+
): boolean
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface FlushSuppressionArgs {
|
|
45
|
+
chatId: string
|
|
46
|
+
/** The turn's origin thread. `null` = chat root / DM (matches the
|
|
47
|
+
* `hasOutboundDeliveredSince` explicit-null semantics — never pass
|
|
48
|
+
* `undefined`, which would match ANY thread and re-open the
|
|
49
|
+
* cross-topic false positive this module exists to close). */
|
|
50
|
+
threadId: number | null
|
|
51
|
+
/** Length of the captured answer the flush is about to deliver. The
|
|
52
|
+
* length floor is `min(FINAL_ANSWER_MIN_CHARS, answerLength)`: a row
|
|
53
|
+
* can only suppress the flush if it is at least as long as the answer
|
|
54
|
+
* itself (up to the 200-char substantive cap). A TERSE captured answer
|
|
55
|
+
* ("Yes — done.") is therefore still suppressible by its own short
|
|
56
|
+
* answer-stream materialization (avoiding a duplicate bubble), while a
|
|
57
|
+
* short progress ping can never suppress a LONG composed answer. */
|
|
58
|
+
answerLength: number
|
|
59
|
+
nowMs: number
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* True iff the flush should be suppressed: a substantive same-thread outbound
|
|
64
|
+
* landed within the suppression window. Fails open (no suppression) on oracle
|
|
65
|
+
* errors — delivering a possible duplicate beats dropping the only copy.
|
|
66
|
+
*/
|
|
67
|
+
export function shouldSuppressTurnFlush(
|
|
68
|
+
deps: FlushSuppressionDeps,
|
|
69
|
+
args: FlushSuppressionArgs,
|
|
70
|
+
): boolean {
|
|
71
|
+
const minChars = Math.max(1, Math.min(FINAL_ANSWER_MIN_CHARS, args.answerLength))
|
|
72
|
+
try {
|
|
73
|
+
return deps.hasSubstantiveOutbound(
|
|
74
|
+
args.chatId,
|
|
75
|
+
args.nowMs - FLUSH_SUPPRESSION_WINDOW_MS,
|
|
76
|
+
args.threadId,
|
|
77
|
+
minChars,
|
|
78
|
+
)
|
|
79
|
+
} catch {
|
|
80
|
+
return false
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Terminal catch-all + diagnostic tap for inbound Telegram messages (#3300).
|
|
3
|
+
*
|
|
4
|
+
* THE CLASS OF BUG THIS CLOSES: the gateway registers only content-specific
|
|
5
|
+
* handlers (`bot.on('message:text')`, `:photo`, … `:paid_media`). grammy
|
|
6
|
+
* ^1.44 routes `bot.on` as filtering middleware — internally
|
|
7
|
+
* `on → filter(pred, handler) → branch(pred, handler, pass)` — and a
|
|
8
|
+
* `message` update matching NONE of the registered predicates falls through
|
|
9
|
+
* every branch and is SILENTLY discarded: no log, no ack, no history row.
|
|
10
|
+
*
|
|
11
|
+
* Live signature (2026-07-16, klanker DM): message_id 19090 was allocated
|
|
12
|
+
* between an outbound reply (19089, 21:51:11Z) and the next inbound (19091,
|
|
13
|
+
* 21:59:31Z) with ZERO gateway trace — no early_ack, no gw-trace inbound, no
|
|
14
|
+
* gate-deny, no error — while polling stayed healthy on both sides. Exactly
|
|
15
|
+
* the zero-observability failure this module makes impossible: every update
|
|
16
|
+
* is now logged on receipt (tap) and every message either produces a turn or
|
|
17
|
+
* an explicit log-only line (catch-all).
|
|
18
|
+
*
|
|
19
|
+
* Extracted into its own module (rather than inline in gateway.ts) so tests
|
|
20
|
+
* can drive the REAL registration path on a real grammy Bot — gateway.ts is
|
|
21
|
+
* a side-effecting module that cannot be imported into a unit test.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import type { Bot, Context } from 'grammy'
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Top-level `message` envelope fields — identity/routing metadata, excluded
|
|
28
|
+
* when surfacing the CONTENT keys of a message (the fields that determine
|
|
29
|
+
* which `bot.on('message:*')` handler should match).
|
|
30
|
+
*/
|
|
31
|
+
export const MESSAGE_ENVELOPE_KEYS: ReadonlySet<string> = new Set<string>([
|
|
32
|
+
'message_id', 'message_thread_id', 'date', 'chat', 'from', 'sender_chat',
|
|
33
|
+
'forward_origin', 'reply_to_message', 'external_reply', 'quote',
|
|
34
|
+
'reply_to_story', 'edit_date', 'media_group_id', 'author_signature',
|
|
35
|
+
'is_topic_message', 'is_automatic_forward', 'via_bot', 'sender_boost_count',
|
|
36
|
+
'business_connection_id', 'effect_id', 'has_protected_content',
|
|
37
|
+
'is_from_offline', 'link_preview_options', 'show_caption_above_media',
|
|
38
|
+
'entities', 'caption_entities', 'paid_star_count',
|
|
39
|
+
])
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Known-noise SERVICE message content keys: chat/topic lifecycle events that
|
|
43
|
+
* carry no user intent for the agent. These are LOGGED (never silently
|
|
44
|
+
* dropped — the whole point of #3300) but do NOT become agent turns:
|
|
45
|
+
* forum-topic lifecycle events are a recurring spam vector in supergroups,
|
|
46
|
+
* and each spurious turn costs tokens and attention.
|
|
47
|
+
*
|
|
48
|
+
* Anything NOT in this set that reaches the catch-all is delivered as a turn
|
|
49
|
+
* (fail-toward-delivery: an unknown content type plausibly carries user
|
|
50
|
+
* intent; better a placeholder turn than a lost message).
|
|
51
|
+
*/
|
|
52
|
+
export const SERVICE_NOISE_KEYS: ReadonlySet<string> = new Set<string>([
|
|
53
|
+
'new_chat_members', 'left_chat_member', 'new_chat_title', 'new_chat_photo',
|
|
54
|
+
'delete_chat_photo', 'group_chat_created', 'supergroup_chat_created',
|
|
55
|
+
'channel_chat_created', 'message_auto_delete_timer_changed',
|
|
56
|
+
'migrate_to_chat_id', 'migrate_from_chat_id',
|
|
57
|
+
'forum_topic_created', 'forum_topic_edited', 'forum_topic_closed',
|
|
58
|
+
'forum_topic_reopened', 'general_forum_topic_hidden',
|
|
59
|
+
'general_forum_topic_unhidden',
|
|
60
|
+
'video_chat_scheduled', 'video_chat_started', 'video_chat_ended',
|
|
61
|
+
'video_chat_participants_invited',
|
|
62
|
+
'giveaway_created', 'giveaway', 'giveaway_winners', 'giveaway_completed',
|
|
63
|
+
'boost_added', 'chat_background_set', 'write_access_allowed',
|
|
64
|
+
'proximity_alert_triggered', 'chat_set_theme',
|
|
65
|
+
'connected_website', 'direct_message_price_changed',
|
|
66
|
+
])
|
|
67
|
+
|
|
68
|
+
/** The CONTENT keys of a message — top-level keys minus envelope metadata. */
|
|
69
|
+
export function messageContentKeys(msg: Record<string, unknown>): string[] {
|
|
70
|
+
return Object.keys(msg).filter(k => !MESSAGE_ENVELOPE_KEYS.has(k))
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export type UnhandledMessagePlan =
|
|
74
|
+
| { action: 'turn'; text: string; contentKeys: string[] }
|
|
75
|
+
| { action: 'log-only'; contentKeys: string[] }
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Decide what the catch-all does with a message no specific handler consumed:
|
|
79
|
+
* known-noise service messages → log-only (no turn); everything else → a turn
|
|
80
|
+
* with best-effort text (`text ?? caption ?? placeholder naming the type`).
|
|
81
|
+
*/
|
|
82
|
+
export function planUnhandledMessage(msg: Record<string, unknown>): UnhandledMessagePlan {
|
|
83
|
+
const contentKeys = messageContentKeys(msg)
|
|
84
|
+
if (contentKeys.length > 0 && contentKeys.every(k => SERVICE_NOISE_KEYS.has(k))) {
|
|
85
|
+
return { action: 'log-only', contentKeys }
|
|
86
|
+
}
|
|
87
|
+
const contentType = contentKeys[0] ?? 'unknown'
|
|
88
|
+
const text =
|
|
89
|
+
(typeof msg.text === 'string' ? msg.text : undefined) ??
|
|
90
|
+
(typeof msg.caption === 'string' ? msg.caption : undefined) ??
|
|
91
|
+
`(unhandled message content: ${contentType})`
|
|
92
|
+
return { action: 'turn', text, contentKeys }
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** One compact diagnostic line per update; cap prevents log flooding. */
|
|
96
|
+
export const TAP_MAX_LINES_PER_MINUTE = 300
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Install the diagnostic update tap: pass-through middleware (`bot.use` →
|
|
100
|
+
* always calls next()) logging every received update's type + content keys
|
|
101
|
+
* (never payload bodies — privacy). Rate-limited to TAP_MAX_LINES_PER_MINUTE
|
|
102
|
+
* per wall-clock minute; on rollover a single summary line reports how many
|
|
103
|
+
* were suppressed, so even a flood is never invisible.
|
|
104
|
+
*
|
|
105
|
+
* MUST be installed before all `bot.on` handlers so it observes every update.
|
|
106
|
+
*/
|
|
107
|
+
export function installUpdateTap(
|
|
108
|
+
bot: Pick<Bot, 'use'>,
|
|
109
|
+
log: (line: string) => void,
|
|
110
|
+
nowMs: () => number = Date.now,
|
|
111
|
+
): void {
|
|
112
|
+
let windowStart = 0
|
|
113
|
+
let windowCount = 0
|
|
114
|
+
let suppressed = 0
|
|
115
|
+
bot.use(async (ctx, next) => {
|
|
116
|
+
try {
|
|
117
|
+
const now = nowMs()
|
|
118
|
+
if (now - windowStart >= 60_000) {
|
|
119
|
+
if (suppressed > 0) {
|
|
120
|
+
log(`telegram gateway: rx tap suppressed ${suppressed} update lines in the last minute (cap ${TAP_MAX_LINES_PER_MINUTE}/min)\n`)
|
|
121
|
+
}
|
|
122
|
+
windowStart = now
|
|
123
|
+
windowCount = 0
|
|
124
|
+
suppressed = 0
|
|
125
|
+
}
|
|
126
|
+
if (windowCount < TAP_MAX_LINES_PER_MINUTE) {
|
|
127
|
+
windowCount++
|
|
128
|
+
const upd = ctx.update as unknown as Record<string, unknown>
|
|
129
|
+
const updateType = Object.keys(upd).find(k => k !== 'update_id') ?? 'unknown'
|
|
130
|
+
const msg = ctx.message as unknown as Record<string, unknown> | undefined
|
|
131
|
+
const detail = msg ? ` content=[${messageContentKeys(msg).join(',')}]` : ''
|
|
132
|
+
log(`telegram gateway: rx update_id=${ctx.update.update_id} type=${updateType}${detail}\n`)
|
|
133
|
+
} else {
|
|
134
|
+
suppressed++
|
|
135
|
+
}
|
|
136
|
+
} catch {
|
|
137
|
+
// The diagnostic tap must never break the inbound pipeline.
|
|
138
|
+
}
|
|
139
|
+
await next()
|
|
140
|
+
})
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Install the terminal catch-all. MUST be registered LAST among the
|
|
145
|
+
* `message`/`message:*` handlers: grammy leaf handlers never call `next()`,
|
|
146
|
+
* so a matched specific handler stops the chain (specific handler always
|
|
147
|
+
* wins) and this only fires for messages no specific handler consumed —
|
|
148
|
+
* registration order IS the no-double-handling guarantee.
|
|
149
|
+
*
|
|
150
|
+
* `onInbound` is the normal inbound pipeline (gateway.ts passes
|
|
151
|
+
* handleInboundCoalesced), which applies the same access gating and
|
|
152
|
+
* forward-origin parsing as every other content handler.
|
|
153
|
+
*/
|
|
154
|
+
export function installUnhandledMessageCatchAll(
|
|
155
|
+
bot: Pick<Bot, 'on'>,
|
|
156
|
+
onInbound: (ctx: Context, text: string) => Promise<void>,
|
|
157
|
+
log: (line: string) => void,
|
|
158
|
+
): void {
|
|
159
|
+
bot.on('message', async ctx => {
|
|
160
|
+
try {
|
|
161
|
+
const msg = ctx.message as unknown as Record<string, unknown>
|
|
162
|
+
const plan = planUnhandledMessage(msg)
|
|
163
|
+
// Log KEYS + ids only — never the payload bodies (privacy).
|
|
164
|
+
log(
|
|
165
|
+
`telegram gateway: catch-all inbound (no specific handler) ` +
|
|
166
|
+
`update_id=${ctx.update.update_id} chat_id=${ctx.chat?.id ?? '?'} ` +
|
|
167
|
+
`message_id=${ctx.message?.message_id ?? '?'} ` +
|
|
168
|
+
`content_keys=[${plan.contentKeys.join(',')}] action=${plan.action}\n`,
|
|
169
|
+
)
|
|
170
|
+
if (plan.action === 'turn') {
|
|
171
|
+
await onInbound(ctx, plan.text)
|
|
172
|
+
}
|
|
173
|
+
} catch (err) {
|
|
174
|
+
log(`telegram gateway: catch-all handler error: ${(err as Error).message}\n`)
|
|
175
|
+
}
|
|
176
|
+
})
|
|
177
|
+
}
|
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
import {
|
|
31
31
|
detectModelUnavailable,
|
|
32
32
|
isLitellmProxyLocal429,
|
|
33
|
+
isLitellmProxyAuthMisconfig,
|
|
33
34
|
parseResetTime,
|
|
34
35
|
} from './model-unavailable.js'
|
|
35
36
|
import { classify429Detail } from './throttle-tier.js'
|
|
@@ -46,6 +47,7 @@ export type LlmErrorKind =
|
|
|
46
47
|
| 'overload_529'
|
|
47
48
|
| 'quota_wall'
|
|
48
49
|
| 'auth'
|
|
50
|
+
| 'infra_misconfig'
|
|
49
51
|
| 'transient'
|
|
50
52
|
| 'unknown'
|
|
51
53
|
|
|
@@ -150,8 +152,22 @@ export function parseLlmError(
|
|
|
150
152
|
function classifyKindAndSource(text: string): { kind: LlmErrorKind; source: LlmErrorSource } {
|
|
151
153
|
const lower = text.toLowerCase()
|
|
152
154
|
|
|
155
|
+
// 0. LiteLLM-proxy AUTH misconfig — checked BEFORE the auth branch. The
|
|
156
|
+
// proxy's internal fallback re-dispatched WITHOUT the OAuth header onto a
|
|
157
|
+
// keyless deployment, so Anthropic 401'd it ("x-api-key header is
|
|
158
|
+
// required"). This is an OPERATOR-only infra fault, NOT an end-user login
|
|
159
|
+
// wall — never the user-facing 'auth' kind (which renders a re-auth card a
|
|
160
|
+
// non-operator user cannot act on). Source is the local proxy, not
|
|
161
|
+
// Anthropic. See isLitellmProxyAuthMisconfig for provenance.
|
|
162
|
+
if (isLitellmProxyAuthMisconfig(text)) {
|
|
163
|
+
return { kind: 'infra_misconfig', source: 'litellm-local' }
|
|
164
|
+
}
|
|
165
|
+
|
|
153
166
|
// 1. Auth — always terminal, always actionable.
|
|
154
167
|
const claudeKind = classifyClaudeError({ message: text, type: text })
|
|
168
|
+
if (claudeKind === 'proxy-misconfig') {
|
|
169
|
+
return { kind: 'infra_misconfig', source: 'litellm-local' }
|
|
170
|
+
}
|
|
155
171
|
if (claudeKind === 'credentials-expired' || claudeKind === 'credentials-invalid') {
|
|
156
172
|
return { kind: 'auth', source: 'anthropic' }
|
|
157
173
|
}
|
|
@@ -212,6 +228,8 @@ function buildCoreText(kind: LlmErrorKind, source: LlmErrorSource): string {
|
|
|
212
228
|
return 'Usage limit reached on this Claude subscription.'
|
|
213
229
|
case 'auth':
|
|
214
230
|
return 'Claude login needs re-authentication.'
|
|
231
|
+
case 'infra_misconfig':
|
|
232
|
+
return 'Local model-gateway auth misconfig (proxy fallback dropped the OAuth header).'
|
|
215
233
|
case 'transient':
|
|
216
234
|
return source === 'network'
|
|
217
235
|
? "Couldn't reach Anthropic (network) — retrying automatically."
|
|
@@ -280,6 +298,10 @@ function buildRecommendation(parsed: ParsedLlmError, tz: string): string | undef
|
|
|
280
298
|
switch (parsed.kind) {
|
|
281
299
|
case 'auth':
|
|
282
300
|
return '→ Re-authenticate this account to continue.'
|
|
301
|
+
case 'infra_misconfig':
|
|
302
|
+
// Operator-facing — NO re-auth wording (the login is fine). Points at the
|
|
303
|
+
// real remedy: the local LiteLLM proxy fallback config.
|
|
304
|
+
return '→ Fix the LiteLLM proxy fallback config (deployment missing OAuth passthrough).'
|
|
283
305
|
case 'quota_wall': {
|
|
284
306
|
const reset = formatResetClock(parsed.resetAt, tz)
|
|
285
307
|
return reset
|
|
@@ -353,6 +375,8 @@ function kindEmoji(kind: LlmErrorKind): string {
|
|
|
353
375
|
return '⚠️'
|
|
354
376
|
case 'auth':
|
|
355
377
|
return '🔑'
|
|
378
|
+
case 'infra_misconfig':
|
|
379
|
+
return '🛠️'
|
|
356
380
|
case 'transient':
|
|
357
381
|
return '🌐'
|
|
358
382
|
case 'unknown':
|
|
@@ -195,6 +195,61 @@ export function isLitellmProxyLocal429(text: string): boolean {
|
|
|
195
195
|
return litellmV3LimiterSignalPair.every(s => lower.includes(s))
|
|
196
196
|
}
|
|
197
197
|
|
|
198
|
+
/**
|
|
199
|
+
* True when `text` is a LiteLLM-proxy-LOCAL AUTH misconfiguration — the
|
|
200
|
+
* proxy-fallback keyless-401 class, NOT a genuine end-user credential wall.
|
|
201
|
+
*
|
|
202
|
+
* PROVENANCE (incident, 2026-07-16). LiteLLM's internal model-fallback chain
|
|
203
|
+
* (`gpt-oss-20b -> [gpt-oss-20b-openrouter, claude-sonnet-5]`) re-dispatches a
|
|
204
|
+
* failed primary WITHOUT forwarding the client's OAuth `Authorization` header.
|
|
205
|
+
* The `claude-sonnet-5` deployment is deliberately keyless (passthrough auth —
|
|
206
|
+
* it expects the forwarded OAuth), so Anthropic rejects the header-less
|
|
207
|
+
* fallback with a 401 `authentication_error` whose message is
|
|
208
|
+
* "x-api-key header is required". Switchroom agents authenticate the unmodified
|
|
209
|
+
* `claude` CLI with an OAuth Bearer token and NEVER an `x-api-key`, so that
|
|
210
|
+
* demand can only originate from the proxy dropping the header on a keyless
|
|
211
|
+
* fallback deployment — a host-side infra misconfig for the OPERATOR to fix,
|
|
212
|
+
* never a login problem an end user (or even the operator's OAuth) can act on.
|
|
213
|
+
*
|
|
214
|
+
* Misclassifying it (as `classifyClaudeError` did → `credentials-invalid` →
|
|
215
|
+
* a "🔑 re-authenticate" card) is doubly wrong: wrong AUDIENCE (a non-operator
|
|
216
|
+
* user like Lisa cannot re-auth anything) and wrong DIAGNOSIS (the login is
|
|
217
|
+
* fine; the proxy fallback config is not). Detecting it here lets both
|
|
218
|
+
* classifiers route it to the operator-only infra surface instead.
|
|
219
|
+
*
|
|
220
|
+
* Detection (deterministic wording, mirrors `isLitellmProxyLocal429`):
|
|
221
|
+
* - the definitive "x-api-key header is required" marker (impossible for our
|
|
222
|
+
* OAuth flow — only the keyless-proxy path emits it), OR
|
|
223
|
+
* - an `authentication_error` CO-OCCURRING with the proxy-FALLBACK-specific
|
|
224
|
+
* structural pair: "fallback" re-dispatch provenance AND an explicit
|
|
225
|
+
* "x-api-key" mention.
|
|
226
|
+
*
|
|
227
|
+
* PRECISION over recall (#3293 review finding 2): an earlier draft matched any
|
|
228
|
+
* `authentication_error` + (litellm|proxy) + (fallback|x-api-key|api_key) —
|
|
229
|
+
* broad enough that a GENUINE OAuth expiry wrapped in a proxy envelope that
|
|
230
|
+
* merely mentions "api_key" would misdiagnose as proxy-misconfig (still
|
|
231
|
+
* operator-visible, but the recommendation would point at the proxy config
|
|
232
|
+
* instead of a re-auth). Both classes route operator-only now, so an ambiguous
|
|
233
|
+
* envelope deliberately KEEPS the credentials-invalid/-expired diagnosis;
|
|
234
|
+
* only the unambiguous keyless-fallback signature classifies as misconfig.
|
|
235
|
+
* Never throws on weird input.
|
|
236
|
+
*/
|
|
237
|
+
export function isLitellmProxyAuthMisconfig(text: string): boolean {
|
|
238
|
+
if (typeof text !== 'string' || text.length === 0) return false
|
|
239
|
+
const sample = text.length > 16_384 ? text.slice(0, 16_384) : text
|
|
240
|
+
const lower = sample.toLowerCase()
|
|
241
|
+
// Definitive marker — see provenance above.
|
|
242
|
+
if (lower.includes('x-api-key header is required')) return true
|
|
243
|
+
// Structural signature: an authentication_error carrying BOTH the fallback
|
|
244
|
+
// re-dispatch provenance AND an explicit x-api-key mention. A proxy envelope
|
|
245
|
+
// that merely mentions litellm/proxy/api_key stays on the credentials
|
|
246
|
+
// diagnosis (ambiguous → not misconfig).
|
|
247
|
+
const isAuthErr =
|
|
248
|
+
lower.includes('authentication_error') || lower.includes('authenticationerror')
|
|
249
|
+
if (!isAuthErr) return false
|
|
250
|
+
return lower.includes('fallback') && lower.includes('x-api-key')
|
|
251
|
+
}
|
|
252
|
+
|
|
198
253
|
/**
|
|
199
254
|
* Best-effort extraction of the limit detail LiteLLM embeds in its
|
|
200
255
|
* proxy-local 429 bodies, for instrumentation (the `rate_limit_429_classified`
|