switchroom 0.18.12 → 0.18.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +57 -9
- package/dist/auth-broker/index.js +174 -72
- package/dist/cli/autoaccept-poll.js +23 -0
- package/dist/cli/drive-write-pretool.mjs +24 -1
- package/dist/cli/foreground-hog-pretool.mjs +264 -0
- package/dist/cli/ms-365-write-pretool.mjs +31 -8
- package/dist/cli/notion-write-pretool.mjs +9 -2
- package/dist/cli/skill-validate-pretool.mjs +144 -2847
- package/dist/cli/switchroom.js +986 -3131
- package/dist/host-control/main.js +216 -2863
- package/dist/vault/approvals/kernel-server.js +67 -1
- package/dist/vault/broker/server.js +98 -45
- package/package.json +1 -1
- package/profiles/coding/CLAUDE.md.hbs +2 -0
- package/profiles/default/CLAUDE.md.hbs +2 -0
- package/skills/switchroom-architecture/telegram.md +0 -1
- package/telegram-plugin/auth-snapshot-format.ts +37 -5
- package/telegram-plugin/auto-fallback-fleet.ts +29 -1
- package/telegram-plugin/bridge/bridge.ts +2 -0
- package/telegram-plugin/dist/bridge/bridge.js +51 -3
- package/telegram-plugin/dist/gateway/gateway.js +1251 -2368
- package/telegram-plugin/dist/server.js +67 -3
- package/telegram-plugin/format.ts +19 -0
- package/telegram-plugin/gateway/approval-hold.ts +21 -2
- package/telegram-plugin/gateway/auth-broker-client.ts +1 -0
- package/telegram-plugin/gateway/auth-command.ts +14 -0
- package/telegram-plugin/gateway/callback-query-handlers.ts +12 -0
- package/telegram-plugin/gateway/forward-origin.ts +235 -0
- package/telegram-plugin/gateway/gateway.ts +445 -83
- package/telegram-plugin/gateway/throttle-tier-wiring.ts +268 -0
- package/telegram-plugin/history.ts +106 -6
- package/telegram-plugin/inline-keyboard-callbacks.ts +94 -0
- package/telegram-plugin/model-unavailable.ts +61 -13
- package/telegram-plugin/outbound-field-redact.ts +69 -0
- package/telegram-plugin/render/render.ts +32 -14
- package/telegram-plugin/render/rich-render.ts +40 -32
- package/telegram-plugin/scoped-approval.ts +11 -2
- package/telegram-plugin/secret-detect/chunker.ts +18 -4
- package/telegram-plugin/secret-detect/index.ts +12 -56
- package/telegram-plugin/send-gate-degraded.test.ts +131 -0
- package/telegram-plugin/send-gate.test.ts +25 -6
- package/telegram-plugin/send-gate.ts +82 -8
- package/telegram-plugin/session-tail.ts +82 -7
- package/telegram-plugin/stream-controller.ts +3 -2
- package/telegram-plugin/subagent-watcher.ts +71 -16
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +36 -5
- package/telegram-plugin/tests/auto-fallback-fleet.test.ts +72 -0
- package/telegram-plugin/tests/callback-query-handlers.test.ts +65 -0
- package/telegram-plugin/tests/forward-origin.test.ts +309 -0
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +57 -0
- package/telegram-plugin/tests/history.test.ts +272 -0
- package/telegram-plugin/tests/inbound-message-types.test.ts +5 -1
- package/telegram-plugin/tests/inline-keyboard-callbacks.test.ts +164 -0
- package/telegram-plugin/tests/operator-events-session-tail.test.ts +74 -0
- package/telegram-plugin/tests/outbound-field-redact.test.ts +107 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +173 -0
- package/telegram-plugin/tests/render/render-outbound-chunks.test.ts +6 -4
- package/telegram-plugin/tests/render/render.test.ts +88 -0
- package/telegram-plugin/tests/render/rich-render.test.ts +41 -22
- package/telegram-plugin/tests/scoped-approval.test.ts +27 -0
- package/telegram-plugin/tests/secret-detect-chunk-overlap.test.ts +65 -0
- package/telegram-plugin/tests/secret-detect-oauth-code.test.ts +5 -4
- package/telegram-plugin/tests/session-tail-sidecar-reap.test.ts +268 -0
- package/telegram-plugin/tests/single-mode-stream-reply.test.ts +5 -3
- package/telegram-plugin/tests/status-accent.test.ts +5 -3
- package/telegram-plugin/tests/stream-controller-chunk-cap.test.ts +20 -20
- package/telegram-plugin/tests/stream-reply-handler.test.ts +5 -2
- package/telegram-plugin/tests/subagent-watcher-fd-leak.test.ts +275 -0
- package/telegram-plugin/tests/throttle-tier-wiring.test.ts +290 -0
- package/telegram-plugin/tests/throttle-tier.test.ts +278 -0
- package/telegram-plugin/tests/worktree-watch-cwds.test.ts +215 -1
- package/telegram-plugin/throttle-tier.ts +226 -0
- package/telegram-plugin/uat/scenarios/jtbd-rich-formatting-render-dm.test.ts +8 -7
- package/telegram-plugin/worktree-watch-cwds.ts +194 -5
- package/telegram-plugin/secret-detect/secretlint-source.ts +0 -95
- package/telegram-plugin/tests/secret-detect-secretlint.test.ts +0 -105
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* throttle-tier-wiring.ts — side-effect runner for the 429 throttle tier.
|
|
3
|
+
*
|
|
4
|
+
* The DECISION and notice text live in ../throttle-tier.ts (pure). This
|
|
5
|
+
* module owns the sequenced side effects, with every dependency injected so
|
|
6
|
+
* the wiring is unit-testable without importing gateway.ts:
|
|
7
|
+
*
|
|
8
|
+
* 1. Broker `mark-throttled` — records `throttled_until` in the quota
|
|
9
|
+
* ledger (no roll, no eligibility change) and runs the broker-side
|
|
10
|
+
* escalation guard (3 hits / 10 min → live probe → mark-exhausted when
|
|
11
|
+
* corroborated). EVERY fire reaches the broker — the notice cooldown
|
|
12
|
+
* below never suppresses the ledger write, because the escalation
|
|
13
|
+
* counter is what corroborates a wall hiding behind transient wording.
|
|
14
|
+
* 2. ONE lightweight operator notice — deduped per account BOTH locally
|
|
15
|
+
* (cooldown window) and FLEET-WIDE via the broker's claim-notification
|
|
16
|
+
* verb (N agents sharing a throttled account produce one copy per chat,
|
|
17
|
+
* not N). Claim failures FAIL OPEN (send anyway) per the claim
|
|
18
|
+
* contract.
|
|
19
|
+
* 3. A delayed retry nudge — after `throttled_until` (+slack +jitter so N
|
|
20
|
+
* agents don't restart-and-replay simultaneously into the just-cleared
|
|
21
|
+
* account), replay the turn the 429 killed via the existing resume
|
|
22
|
+
* lever (triggerSelfRestart → boot-resume). Guards, in order:
|
|
23
|
+
* - a LIVE turn that started AFTER the throttle was armed supersedes
|
|
24
|
+
* the dead turn — skip entirely (restarting would kill live work
|
|
25
|
+
* and boot-resume would replay the WRONG turn);
|
|
26
|
+
* - a live turn that started BEFORE the arm is the dead turn itself
|
|
27
|
+
* still holding the in-flight gate — defer the restart to the
|
|
28
|
+
* turn-complete drain (`pendingRestarts`) instead of SIGTERM-now;
|
|
29
|
+
* - otherwise consult the SHARED fleet-fallback resume gate
|
|
30
|
+
* (single-flight across throttle AND fallback resumes + staleness)
|
|
31
|
+
* and restart on a 'resume' verdict.
|
|
32
|
+
*
|
|
33
|
+
* The escalated outcome (broker corroborated a wall and already rolled)
|
|
34
|
+
* posts its own announcement — gateway-side, per the reactive-path doctrine
|
|
35
|
+
* (`LastFleetRoll` docstring in src/auth/broker/server.ts), which also
|
|
36
|
+
* covers PINNED (non-fleet-active) account rolls — then nudges the resume
|
|
37
|
+
* immediately through the same turn-safety guards.
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
import {
|
|
41
|
+
evaluateThrottleNotice,
|
|
42
|
+
renderThrottleEscalationNotice,
|
|
43
|
+
renderThrottleNotice,
|
|
44
|
+
THROTTLE_NOTICE_COOLDOWN_MS,
|
|
45
|
+
type ThrottleNoticeState,
|
|
46
|
+
} from '../throttle-tier.js'
|
|
47
|
+
|
|
48
|
+
/** Slack past throttled_until before the retry nudge fires. */
|
|
49
|
+
export const THROTTLE_RETRY_NUDGE_SLACK_MS = 5_000
|
|
50
|
+
|
|
51
|
+
/** Max random jitter added to the nudge so agents sharing the throttled
|
|
52
|
+
* account stagger their restart-and-replay instead of stampeding the
|
|
53
|
+
* just-cleared account. */
|
|
54
|
+
export const THROTTLE_RETRY_NUDGE_JITTER_MAX_MS = 30_000
|
|
55
|
+
|
|
56
|
+
/** The narrow broker surface the runner needs (structurally satisfied by
|
|
57
|
+
* the gateway's AuthBrokerClient). */
|
|
58
|
+
export interface ThrottleBrokerClient {
|
|
59
|
+
markThrottled(until: number): Promise<{
|
|
60
|
+
account: string
|
|
61
|
+
throttled_until: number
|
|
62
|
+
escalated: boolean
|
|
63
|
+
rolledTo?: string | null
|
|
64
|
+
}>
|
|
65
|
+
claimNotification(key: string, windowMs: number): Promise<{ granted: boolean }>
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface ThrottleTierRunnerDeps {
|
|
69
|
+
/** This gateway's own agent (SWITCHROOM_AGENT_NAME). */
|
|
70
|
+
agentName: string
|
|
71
|
+
getBrokerClient(): Promise<ThrottleBrokerClient | null>
|
|
72
|
+
/** Chats the notice broadcasts to (access.allowFrom, resolved per call). */
|
|
73
|
+
listNoticeChats(): Array<string | number>
|
|
74
|
+
/** Fire-and-forget rich send (gateway wraps swallowingApiCall). */
|
|
75
|
+
sendNotice(chatId: string | number, markdown: string): void
|
|
76
|
+
/** THE shared fleet-fallback resume gate (single-flight + staleness). */
|
|
77
|
+
resumeDecide(failedTurnStartedAtMs: number | null): 'resume' | 'skip-inflight' | 'skip-stale'
|
|
78
|
+
newestActiveTurnStartedAtMs(): number | null
|
|
79
|
+
/** True while a turn is in flight (gateway turnInFlightForGate()). */
|
|
80
|
+
turnInFlight(): boolean
|
|
81
|
+
/** Defer the restart to the turn-complete drain (pendingRestarts). */
|
|
82
|
+
deferRestartToTurnComplete(agentName: string, reason: string): void
|
|
83
|
+
/** Restart now (gateway triggerSelfRestart). */
|
|
84
|
+
restartNow(agentName: string, reason: string): void
|
|
85
|
+
log(msg: string): void
|
|
86
|
+
now?: () => number
|
|
87
|
+
/** Timer seam (tests drive synchronously). Default setTimeout+unref. */
|
|
88
|
+
schedule?: (fn: () => void, ms: number) => { cancel(): void }
|
|
89
|
+
/** Jitter source (tests pin it). Default uniform 0..JITTER_MAX. */
|
|
90
|
+
jitterMs?: () => number
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export interface ThrottleTierRunner {
|
|
94
|
+
/**
|
|
95
|
+
* Run the throttle path for one terminal transient 429. Fire-and-forget
|
|
96
|
+
* from the caller's perspective — never throws.
|
|
97
|
+
*/
|
|
98
|
+
fire(triggerAgent: string, throttledUntilMs: number, resetParsed: boolean): Promise<void>
|
|
99
|
+
/** Test/debug view of internal state. */
|
|
100
|
+
inspect(): { noticeState: ThrottleNoticeState; nudgePending: boolean }
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export function createThrottleTierRunner(deps: ThrottleTierRunnerDeps): ThrottleTierRunner {
|
|
104
|
+
const now = deps.now ?? (() => Date.now())
|
|
105
|
+
const schedule =
|
|
106
|
+
deps.schedule ??
|
|
107
|
+
((fn: () => void, ms: number) => {
|
|
108
|
+
const t = setTimeout(fn, ms)
|
|
109
|
+
if (typeof t.unref === 'function') t.unref()
|
|
110
|
+
return { cancel: () => clearTimeout(t) }
|
|
111
|
+
})
|
|
112
|
+
const jitterMs =
|
|
113
|
+
deps.jitterMs ?? (() => Math.floor(Math.random() * THROTTLE_RETRY_NUDGE_JITTER_MAX_MS))
|
|
114
|
+
|
|
115
|
+
let noticeState: ThrottleNoticeState = { lastSentAtMsByAccount: {} }
|
|
116
|
+
/** The LATEST throttle owns the nudge — a newer hit replaces an armed
|
|
117
|
+
* timer instead of stacking restarts. */
|
|
118
|
+
let pendingNudge: { cancel(): void } | null = null
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Post `markdown` to every authorized chat, fleet-deduped per chat via the
|
|
122
|
+
* broker claim verb when a client + account are available. Fail-open: a
|
|
123
|
+
* claim error or missing broker never drops the notice.
|
|
124
|
+
*/
|
|
125
|
+
async function broadcastDeduped(
|
|
126
|
+
client: ThrottleBrokerClient | null,
|
|
127
|
+
keyPrefix: string,
|
|
128
|
+
account: string | null,
|
|
129
|
+
markdown: string,
|
|
130
|
+
): Promise<void> {
|
|
131
|
+
for (const chatId of deps.listNoticeChats()) {
|
|
132
|
+
let granted = true
|
|
133
|
+
if (client && account) {
|
|
134
|
+
try {
|
|
135
|
+
granted = (
|
|
136
|
+
await client.claimNotification(
|
|
137
|
+
`${keyPrefix}:${account}:${chatId}`,
|
|
138
|
+
THROTTLE_NOTICE_COOLDOWN_MS,
|
|
139
|
+
)
|
|
140
|
+
).granted
|
|
141
|
+
} catch {
|
|
142
|
+
granted = true // fail open — a duplicated notice beats a dropped one
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
if (granted) {
|
|
146
|
+
deps.sendNotice(chatId, markdown)
|
|
147
|
+
} else {
|
|
148
|
+
deps.log(`[throttle-tier] notice suppressed (fleet claim) chat=${chatId}`)
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Replay the dead turn via restart, with the turn-safety guards (see
|
|
155
|
+
* module docstring). `armedAtMs` anchors the "newer turn supersedes"
|
|
156
|
+
* check: a turn that started after the throttle was armed is live user
|
|
157
|
+
* work, never to be killed for a replay.
|
|
158
|
+
*/
|
|
159
|
+
function nudgeResume(reason: string, armedAtMs: number): void {
|
|
160
|
+
const newest = deps.newestActiveTurnStartedAtMs()
|
|
161
|
+
if (deps.turnInFlight()) {
|
|
162
|
+
if (newest != null && newest > armedAtMs) {
|
|
163
|
+
deps.log(
|
|
164
|
+
`[throttle-tier] resume skipped (superseded by a live newer turn) reason=${reason}`,
|
|
165
|
+
)
|
|
166
|
+
return
|
|
167
|
+
}
|
|
168
|
+
// The in-flight gate is held by the dead throttled turn itself —
|
|
169
|
+
// defer to the turn-complete drain instead of SIGTERM-ing now.
|
|
170
|
+
deps.log(`[throttle-tier] resume deferred to turn-complete reason=${reason}`)
|
|
171
|
+
deps.deferRestartToTurnComplete(deps.agentName, reason)
|
|
172
|
+
return
|
|
173
|
+
}
|
|
174
|
+
const verdict = deps.resumeDecide(newest)
|
|
175
|
+
if (verdict === 'resume') {
|
|
176
|
+
deps.log(`[throttle-tier] resuming dead turn via self-restart reason=${reason}`)
|
|
177
|
+
deps.restartNow(deps.agentName, reason)
|
|
178
|
+
} else {
|
|
179
|
+
deps.log(`[throttle-tier] resume suppressed (${verdict}) reason=${reason}`)
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
async function fire(
|
|
184
|
+
triggerAgent: string,
|
|
185
|
+
throttledUntilMs: number,
|
|
186
|
+
resetParsed: boolean,
|
|
187
|
+
): Promise<void> {
|
|
188
|
+
const armedAtMs = now()
|
|
189
|
+
let client: ThrottleBrokerClient | null = null
|
|
190
|
+
let account: string | null = null
|
|
191
|
+
let escalated = false
|
|
192
|
+
let rolledTo: string | null = null
|
|
193
|
+
try {
|
|
194
|
+
client = await deps.getBrokerClient()
|
|
195
|
+
if (client) {
|
|
196
|
+
const r = await client.markThrottled(throttledUntilMs)
|
|
197
|
+
account = r.account
|
|
198
|
+
escalated = r.escalated
|
|
199
|
+
rolledTo = r.rolledTo ?? null
|
|
200
|
+
} else {
|
|
201
|
+
deps.log(
|
|
202
|
+
`[throttle-tier] broker unreachable — notice only, no ledger record agent=${triggerAgent}`,
|
|
203
|
+
)
|
|
204
|
+
}
|
|
205
|
+
} catch (err) {
|
|
206
|
+
deps.log(
|
|
207
|
+
`[throttle-tier] markThrottled failed agent=${triggerAgent}: ${(err as Error)?.message ?? err}`,
|
|
208
|
+
)
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
if (escalated) {
|
|
212
|
+
// The broker corroborated a genuine wall via a live probe and already
|
|
213
|
+
// ran mark-exhausted + roll (fleet active AND pinned accounts alike).
|
|
214
|
+
// The RAISING gateway announces — reactive-path doctrine — fleet-
|
|
215
|
+
// deduped so N gateways sharing the account produce one copy per chat.
|
|
216
|
+
deps.log(
|
|
217
|
+
`[throttle-tier] escalated to wall account=${account ?? '?'} ` +
|
|
218
|
+
`rolledTo=${rolledTo ?? 'none (all blocked)'}`,
|
|
219
|
+
)
|
|
220
|
+
await broadcastDeduped(
|
|
221
|
+
client,
|
|
222
|
+
'throttle-escalation',
|
|
223
|
+
account,
|
|
224
|
+
renderThrottleEscalationNotice({ account, agent: triggerAgent, rolledTo }),
|
|
225
|
+
)
|
|
226
|
+
if (rolledTo) nudgeResume('throttle-escalation-resume', armedAtMs)
|
|
227
|
+
return
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// ONE lightweight notice, deduped per account: locally by cooldown, then
|
|
231
|
+
// fleet-wide by broker claim. Unknown account (broker down) keys the
|
|
232
|
+
// local cooldown on the agent so the degraded path still can't spam.
|
|
233
|
+
const cooldownKey = account ?? `agent:${triggerAgent}`
|
|
234
|
+
const verdict = evaluateThrottleNotice(noticeState, cooldownKey, now())
|
|
235
|
+
if (verdict.send) {
|
|
236
|
+
noticeState = verdict.next
|
|
237
|
+
await broadcastDeduped(
|
|
238
|
+
client,
|
|
239
|
+
'throttle-notice',
|
|
240
|
+
account,
|
|
241
|
+
renderThrottleNotice({
|
|
242
|
+
account,
|
|
243
|
+
agent: triggerAgent,
|
|
244
|
+
throttledUntilMs,
|
|
245
|
+
resetParsed,
|
|
246
|
+
now: new Date(now()),
|
|
247
|
+
}),
|
|
248
|
+
)
|
|
249
|
+
} else {
|
|
250
|
+
deps.log(`[throttle-tier] notice suppressed (cooldown) key=${cooldownKey}`)
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// Arm (or re-arm) the retry nudge: slack past the reset + jitter so
|
|
254
|
+
// agents sharing the account stagger their replays.
|
|
255
|
+
const delayMs =
|
|
256
|
+
Math.max(throttledUntilMs - now(), 0) + THROTTLE_RETRY_NUDGE_SLACK_MS + jitterMs()
|
|
257
|
+
if (pendingNudge) pendingNudge.cancel()
|
|
258
|
+
pendingNudge = schedule(() => {
|
|
259
|
+
pendingNudge = null
|
|
260
|
+
nudgeResume('throttle-retry-resume', armedAtMs)
|
|
261
|
+
}, delayMs)
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
return {
|
|
265
|
+
fire,
|
|
266
|
+
inspect: () => ({ noticeState, nudgePending: pendingNudge != null }),
|
|
267
|
+
}
|
|
268
|
+
}
|
|
@@ -108,6 +108,25 @@ export interface RecordedMessage {
|
|
|
108
108
|
* Only emoji reactions are tracked — custom emoji are ignored for v1.
|
|
109
109
|
*/
|
|
110
110
|
user_reaction: string | null
|
|
111
|
+
/**
|
|
112
|
+
* Set when the inbound user message was FORWARDED: the server-stamped
|
|
113
|
+
* `forward_origin` of the original message (Bot API 7.0+), so
|
|
114
|
+
* get_recent_messages can surface who originally sent the content the
|
|
115
|
+
* agent saw at delivery time. `forwarded_from` is the raw (truncated,
|
|
116
|
+
* unescaped) human-readable name/title; `forwarded_from_type` is
|
|
117
|
+
* user|hidden_user|chat|channel (hidden_user = self-reported display
|
|
118
|
+
* name, no verifiable id); `forwarded_from_id` is the numeric id when
|
|
119
|
+
* the origin shape exposes one; `forwarded_date` is the original
|
|
120
|
+
* message's ISO timestamp; `forwarded_message_id` is the message id
|
|
121
|
+
* inside the origin channel (channel origins only). For a multi-origin
|
|
122
|
+
* coalesced burst only the PRIMARY (first) origin is persisted here —
|
|
123
|
+
* origins 2+ exist only in the delivered channel tag's numbered attrs.
|
|
124
|
+
*/
|
|
125
|
+
forwarded_from: string | null
|
|
126
|
+
forwarded_from_type: string | null
|
|
127
|
+
forwarded_from_id: string | null
|
|
128
|
+
forwarded_date: string | null
|
|
129
|
+
forwarded_message_id: number | null
|
|
111
130
|
}
|
|
112
131
|
|
|
113
132
|
export interface QueryOptions {
|
|
@@ -166,10 +185,20 @@ export function initHistory(stateDir: string, retentionDays = 30): void {
|
|
|
166
185
|
CREATE INDEX IF NOT EXISTS idx_messages_recent
|
|
167
186
|
ON messages (chat_id, thread_id, ts DESC)
|
|
168
187
|
`)
|
|
169
|
-
// Migration: add reply_to columns to existing DBs that pre-date issue #119
|
|
170
|
-
//
|
|
171
|
-
//
|
|
172
|
-
|
|
188
|
+
// Migration: add reply_to columns to existing DBs that pre-date issue #119,
|
|
189
|
+
// and the forwarded_* origin columns (forward_origin metadata) to DBs that
|
|
190
|
+
// pre-date them. SQLite has no IF NOT EXISTS for ALTER TABLE ADD COLUMN, so
|
|
191
|
+
// we tolerate "duplicate column name" errors and re-throw anything else.
|
|
192
|
+
for (const column of [
|
|
193
|
+
"reply_to_message_id INTEGER",
|
|
194
|
+
"reply_to_text TEXT",
|
|
195
|
+
"user_reaction TEXT",
|
|
196
|
+
"forwarded_from TEXT",
|
|
197
|
+
"forwarded_from_type TEXT",
|
|
198
|
+
"forwarded_from_id TEXT",
|
|
199
|
+
"forwarded_date TEXT",
|
|
200
|
+
"forwarded_message_id INTEGER",
|
|
201
|
+
]) {
|
|
173
202
|
try {
|
|
174
203
|
db.exec(`ALTER TABLE messages ADD COLUMN ${column}`)
|
|
175
204
|
} catch (err) {
|
|
@@ -178,6 +207,57 @@ export function initHistory(stateDir: string, retentionDays = 30): void {
|
|
|
178
207
|
}
|
|
179
208
|
}
|
|
180
209
|
|
|
210
|
+
// Migration (review finding H5): make INSERT OR REPLACE idempotent for thread-less rows.
|
|
211
|
+
//
|
|
212
|
+
// The table's PRIMARY KEY is (chat_id, thread_id, message_id), but thread_id
|
|
213
|
+
// is nullable and SQLite treats NULL as DISTINCT from NULL in a PK/UNIQUE
|
|
214
|
+
// index. Every DM and every non-topic group message has thread_id = NULL, so
|
|
215
|
+
// two inserts of the same (chat_id, message_id) with NULL thread do NOT
|
|
216
|
+
// conflict — `INSERT OR REPLACE` APPENDS a duplicate row instead of replacing
|
|
217
|
+
// it. On the documented at-least-once boot replay / synthesized-resume
|
|
218
|
+
// re-record, an already-stored message is duplicated, inflating
|
|
219
|
+
// get_recent_messages and the getRecentOutboundCount / hasOutboundDeliveredSince
|
|
220
|
+
// counters that feed the silence / over-ping detectors.
|
|
221
|
+
//
|
|
222
|
+
// Fix: a UNIQUE index over COALESCE(thread_id, '') gives every logical
|
|
223
|
+
// (chat, thread-or-general, message_id) key a NON-null uniqueness value, so
|
|
224
|
+
// REPLACE conflict-resolves and dedupes thread-less rows too. INSERT OR
|
|
225
|
+
// REPLACE resolves against ANY unique index, so no writer change is needed.
|
|
226
|
+
// A forum-topic row (non-null thread) and a general row (NULL thread) that
|
|
227
|
+
// share a message_id keep DISTINCT keys (the topic id vs ''), so they stay
|
|
228
|
+
// separate. Stored thread_id values remain real NULLs, so every read path
|
|
229
|
+
// (`thread_id IS NULL` / `thread_id = ?`) is unchanged.
|
|
230
|
+
const LOGICAL_KEY_INDEX = 'idx_messages_logical_key'
|
|
231
|
+
const logicalKeyIndexExists =
|
|
232
|
+
db
|
|
233
|
+
.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?`)
|
|
234
|
+
.get(LOGICAL_KEY_INDEX) != null
|
|
235
|
+
if (!logicalKeyIndexExists) {
|
|
236
|
+
// De-dupe rows an earlier (pre-fix) build already appended, keeping the
|
|
237
|
+
// NEWEST row per logical key (highest ts, then highest rowid = the last
|
|
238
|
+
// write, which is what INSERT OR REPLACE would have left). This MUST run
|
|
239
|
+
// before the UNIQUE index is created, or CREATE UNIQUE INDEX would fail on
|
|
240
|
+
// the existing duplicates. On a fresh/empty DB it is a harmless no-op.
|
|
241
|
+
db.exec(`
|
|
242
|
+
DELETE FROM messages
|
|
243
|
+
WHERE rowid NOT IN (
|
|
244
|
+
SELECT keep_rowid FROM (
|
|
245
|
+
SELECT rowid AS keep_rowid,
|
|
246
|
+
ROW_NUMBER() OVER (
|
|
247
|
+
PARTITION BY chat_id, COALESCE(thread_id, ''), message_id
|
|
248
|
+
ORDER BY ts DESC, rowid DESC
|
|
249
|
+
) AS rn
|
|
250
|
+
FROM messages
|
|
251
|
+
)
|
|
252
|
+
WHERE rn = 1
|
|
253
|
+
)
|
|
254
|
+
`)
|
|
255
|
+
db.exec(
|
|
256
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS ${LOGICAL_KEY_INDEX} ` +
|
|
257
|
+
`ON messages (chat_id, COALESCE(thread_id, ''), message_id)`,
|
|
258
|
+
)
|
|
259
|
+
}
|
|
260
|
+
|
|
181
261
|
// Readable by owner and others so the web dashboard (different uid than the
|
|
182
262
|
// agent) can stream replies back to Hermes Desktop. The WAL sidecar files
|
|
183
263
|
// (-shm/-wal) are also chmod'd so SQLite readonly opens succeed for uid=1000.
|
|
@@ -298,6 +378,19 @@ interface RecordInboundArgs {
|
|
|
298
378
|
*/
|
|
299
379
|
reply_to_message_id?: number | null | undefined
|
|
300
380
|
reply_to_text?: string | null | undefined
|
|
381
|
+
/**
|
|
382
|
+
* If the message was forwarded, the server-stamped origin metadata
|
|
383
|
+
* (Bot API 7.0 `forward_origin`). Populated from
|
|
384
|
+
* `ctx.message.forward_origin` in the gateway handler. `forwarded_from`
|
|
385
|
+
* is the RAW (truncated, unescaped) name — the XML-escaped form goes to
|
|
386
|
+
* the channel meta only. `forwarded_date` is the origin message's ISO
|
|
387
|
+
* timestamp; `forwarded_message_id` is set for channel origins only.
|
|
388
|
+
*/
|
|
389
|
+
forwarded_from?: string | null | undefined
|
|
390
|
+
forwarded_from_type?: string | null | undefined
|
|
391
|
+
forwarded_from_id?: string | null | undefined
|
|
392
|
+
forwarded_date?: string | null | undefined
|
|
393
|
+
forwarded_message_id?: number | null | undefined
|
|
301
394
|
}
|
|
302
395
|
|
|
303
396
|
/**
|
|
@@ -313,8 +406,8 @@ export function recordInbound(args: RecordInboundArgs): void {
|
|
|
313
406
|
if (args.message_id == null) return
|
|
314
407
|
const stmt = requireDb().prepare(`
|
|
315
408
|
INSERT OR REPLACE INTO messages
|
|
316
|
-
(chat_id, thread_id, message_id, role, user, user_id, ts, text, attachment_kind, group_id, reply_to_message_id, reply_to_text)
|
|
317
|
-
VALUES (?, ?, ?, 'user', ?, ?, ?, ?, ?, NULL, ?, ?)
|
|
409
|
+
(chat_id, thread_id, message_id, role, user, user_id, ts, text, attachment_kind, group_id, reply_to_message_id, reply_to_text, forwarded_from, forwarded_from_type, forwarded_from_id, forwarded_date, forwarded_message_id)
|
|
410
|
+
VALUES (?, ?, ?, 'user', ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?)
|
|
318
411
|
`)
|
|
319
412
|
// Defense-in-depth: never persist a detected secret to the message store.
|
|
320
413
|
// The inbound gate (server.ts handleInbound) already deletes + vaults a
|
|
@@ -331,6 +424,13 @@ export function recordInbound(args: RecordInboundArgs): void {
|
|
|
331
424
|
args.attachment_kind ?? null,
|
|
332
425
|
args.reply_to_message_id ?? null,
|
|
333
426
|
args.reply_to_text != null ? redact(args.reply_to_text) : (args.reply_to_text ?? null),
|
|
427
|
+
// Origin names/titles are user-controlled display strings; run them
|
|
428
|
+
// through the same secret-redaction backstop as message text.
|
|
429
|
+
args.forwarded_from != null ? redact(args.forwarded_from) : null,
|
|
430
|
+
args.forwarded_from_type ?? null,
|
|
431
|
+
args.forwarded_from_id ?? null,
|
|
432
|
+
args.forwarded_date ?? null,
|
|
433
|
+
args.forwarded_message_id ?? null,
|
|
334
434
|
)
|
|
335
435
|
}
|
|
336
436
|
|
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
|
|
31
31
|
import {
|
|
32
32
|
validateInlineKeyboard,
|
|
33
|
+
TELEGRAM_BUTTON_LIMITS,
|
|
33
34
|
type AnyButton,
|
|
34
35
|
type ButtonValidationError,
|
|
35
36
|
} from './telegram-button-constraints.js'
|
|
@@ -114,6 +115,99 @@ export function wrapAgentCallbacks(keyboard: AnyButton[][]): AnyButton[][] {
|
|
|
114
115
|
)
|
|
115
116
|
}
|
|
116
117
|
|
|
118
|
+
/**
|
|
119
|
+
* Redact agent-authored free-text on an inline keyboard BEFORE it is sent to
|
|
120
|
+
* Telegram (#3148 fast-follow secret-scrub coverage). `wrapAgentCallbacks`
|
|
121
|
+
* rewrites only `callback_data`; the visible `text` label, the `ack_text`
|
|
122
|
+
* toast, and any `copy_text.text` clipboard payload pass through VERBATIM. An
|
|
123
|
+
* agent that puts a secret in any of those transmits it unmasked, and it
|
|
124
|
+
* resurfaces on tap — the label is echoed back (`button_text`), re-rendered in
|
|
125
|
+
* the "✅ You chose: <label>" annotation (#789), and `ack_text` is shown as the
|
|
126
|
+
* toast. `switch_inline_query` / `switch_inline_query_current_chat` /
|
|
127
|
+
* `switch_inline_query_chosen_chat.query` are also agent-authored free text
|
|
128
|
+
* that Telegram pastes into a chat's input box on tap (the current chat for
|
|
129
|
+
* the `_current_chat` variant, a user-picked chat for `_chosen_chat` — both
|
|
130
|
+
* user-visible), so they carry the same leak class. Route every one of these free-text fields
|
|
131
|
+
* through the SAME outbound redactor the reply `text` body uses, at the
|
|
132
|
+
* outbound boundary, so every downstream resurface reads already-masked bytes.
|
|
133
|
+
*
|
|
134
|
+
* `callback_data` is the routing key and is NEVER touched — redacting it would
|
|
135
|
+
* break tap round-tripping. Button structure/order is preserved; the redactor
|
|
136
|
+
* (`redact()`) only replaces detected secret byte-ranges with a non-empty
|
|
137
|
+
* marker, so it never empties a label (the non-empty-text invariant holds).
|
|
138
|
+
* Returns a fresh keyboard; does not mutate the input. `redactFn` is injected
|
|
139
|
+
* so this stays pure + unit-testable and carries no gateway import cycle.
|
|
140
|
+
*
|
|
141
|
+
* Only agent-authored keyboards flow through here (the `reply` tool path);
|
|
142
|
+
* framework-internal keyboards (approval cards, vault wizard, model menus) are
|
|
143
|
+
* built separately and are NOT redacted by this function.
|
|
144
|
+
*/
|
|
145
|
+
export function redactAgentKeyboard(
|
|
146
|
+
keyboard: AnyButton[][],
|
|
147
|
+
redactFn: (s: string) => string,
|
|
148
|
+
): AnyButton[][] {
|
|
149
|
+
// The keyboard is validated for length BEFORE redaction, but the redaction
|
|
150
|
+
// marker (`[REDACTED:...]`) can be longer than the secret it replaces, so a
|
|
151
|
+
// field that was within a Telegram cap can exceed it after masking. An
|
|
152
|
+
// over-limit button field makes sendMessage 400 → the WHOLE reply is dropped
|
|
153
|
+
// (worse than the leak we just closed), so clamp each masked free-text field
|
|
154
|
+
// to its cap. Truncating a marker is harmless — it's already non-secret.
|
|
155
|
+
const clamp = (s: string, max: number): string =>
|
|
156
|
+
s.length > max ? s.slice(0, max) : s
|
|
157
|
+
return keyboard.map((row) =>
|
|
158
|
+
row.map((btn) => {
|
|
159
|
+
const out: AnyButton = { ...btn }
|
|
160
|
+
if (typeof out.text === 'string') {
|
|
161
|
+
out.text = clamp(redactFn(out.text), TELEGRAM_BUTTON_LIMITS.TEXT_MAX)
|
|
162
|
+
}
|
|
163
|
+
// ack_text is a switchroom-side toast (answerCallbackQuery), not a
|
|
164
|
+
// sendMessage field, so an over-length value can't drop the reply — no
|
|
165
|
+
// clamp needed, but redact it all the same.
|
|
166
|
+
if (typeof out.ack_text === 'string') out.ack_text = redactFn(out.ack_text)
|
|
167
|
+
// switch_inline_query* paste agent free text into a chat input box on tap
|
|
168
|
+
// — same leak class as the label. URL-class fields (url/web_app/login_url)
|
|
169
|
+
// are deliberately left exact: a [REDACTED] marker would corrupt a URL.
|
|
170
|
+
const siq = (out as { switch_inline_query?: unknown }).switch_inline_query
|
|
171
|
+
if (typeof siq === 'string') {
|
|
172
|
+
(out as { switch_inline_query?: string }).switch_inline_query = clamp(
|
|
173
|
+
redactFn(siq), TELEGRAM_BUTTON_LIMITS.SWITCH_INLINE_QUERY_MAX)
|
|
174
|
+
}
|
|
175
|
+
const siqc = (out as { switch_inline_query_current_chat?: unknown })
|
|
176
|
+
.switch_inline_query_current_chat
|
|
177
|
+
if (typeof siqc === 'string') {
|
|
178
|
+
(out as { switch_inline_query_current_chat?: string })
|
|
179
|
+
.switch_inline_query_current_chat = clamp(
|
|
180
|
+
redactFn(siqc), TELEGRAM_BUTTON_LIMITS.SWITCH_INLINE_QUERY_MAX)
|
|
181
|
+
}
|
|
182
|
+
// switch_inline_query_chosen_chat.query is the third variant: agent free
|
|
183
|
+
// text pasted into a user-picked chat's input box on tap — same leak class.
|
|
184
|
+
const cc = (out as { switch_inline_query_chosen_chat?: unknown })
|
|
185
|
+
.switch_inline_query_chosen_chat
|
|
186
|
+
if (cc != null && typeof cc === 'object' &&
|
|
187
|
+
typeof (cc as { query?: unknown }).query === 'string') {
|
|
188
|
+
(out as { switch_inline_query_chosen_chat?: Record<string, unknown> })
|
|
189
|
+
.switch_inline_query_chosen_chat = {
|
|
190
|
+
...(cc as Record<string, unknown>),
|
|
191
|
+
query: clamp(
|
|
192
|
+
redactFn((cc as { query: string }).query),
|
|
193
|
+
TELEGRAM_BUTTON_LIMITS.SWITCH_INLINE_QUERY_MAX),
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
const ct = out.copy_text
|
|
197
|
+
if (ct != null && typeof ct === 'object' &&
|
|
198
|
+
typeof (ct as { text?: unknown }).text === 'string') {
|
|
199
|
+
out.copy_text = {
|
|
200
|
+
...(ct as Record<string, unknown>),
|
|
201
|
+
text: clamp(
|
|
202
|
+
redactFn((ct as { text: string }).text),
|
|
203
|
+
TELEGRAM_BUTTON_LIMITS.COPY_TEXT_MAX),
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return out
|
|
207
|
+
}),
|
|
208
|
+
)
|
|
209
|
+
}
|
|
210
|
+
|
|
117
211
|
/**
|
|
118
212
|
* Extract per-button {@link AgentButtonMeta} from a raw (pre-wrap)
|
|
119
213
|
* keyboard. Returns a map keyed by the raw (unprefixed) callback_data
|
|
@@ -31,7 +31,20 @@ import { escapeMarkdown } from './card-format.js'
|
|
|
31
31
|
|
|
32
32
|
// ─── Public types ────────────────────────────────────────────────────────────
|
|
33
33
|
|
|
34
|
-
export type ModelUnavailableKind =
|
|
34
|
+
export type ModelUnavailableKind =
|
|
35
|
+
| 'overload'
|
|
36
|
+
| 'quota_exhausted'
|
|
37
|
+
| 'network'
|
|
38
|
+
/**
|
|
39
|
+
* 429 throttle tier — a TRANSIENT per-account 429 (explicit
|
|
40
|
+
* `transientUpstreamSignals` negation wording) whose parsed reset lies
|
|
41
|
+
* BEYOND the retry-in-place threshold, so the gateway escalates it to the
|
|
42
|
+
* standard mark-exhausted + fleet-failover machinery. Never produced by
|
|
43
|
+
* `detectModelUnavailable` (a transient-negation string classifies as
|
|
44
|
+
* `overload` there); constructed only by the gateway's throttle-tier
|
|
45
|
+
* branch so the card names the true cause instead of "quota exhausted".
|
|
46
|
+
*/
|
|
47
|
+
| 'rate_limited'
|
|
35
48
|
|
|
36
49
|
export interface ModelUnavailableDetection {
|
|
37
50
|
kind: ModelUnavailableKind
|
|
@@ -41,6 +54,44 @@ export interface ModelUnavailableDetection {
|
|
|
41
54
|
raw: string
|
|
42
55
|
}
|
|
43
56
|
|
|
57
|
+
// ─── Transient-burst signals (canonical, single source of truth) ─────────────
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Explicit markers of a TRANSIENT per-account burst / server-side throttle —
|
|
61
|
+
* a short-term RPM/burst 429 that Claude Code retries internally with backoff,
|
|
62
|
+
* NOT the 5h/7d subscription usage-limit wall. Anthropic emits these with a
|
|
63
|
+
* `rate_limit_error` whose wording explicitly NEGATES the account-quota reading
|
|
64
|
+
* ("not your usage limit" / "would exceed your account's rate limit … try again
|
|
65
|
+
* later"). Keyed on the explicit negation so a genuine wall that merely contains
|
|
66
|
+
* the word "limit" is never down-classified.
|
|
67
|
+
*
|
|
68
|
+
* Exported so `session-tail.ts` classifies a 429 by wording against THIS list
|
|
69
|
+
* rather than hand-rolling its own copy (keeps the two in sync — issue #2922).
|
|
70
|
+
*/
|
|
71
|
+
export const transientUpstreamSignals = [
|
|
72
|
+
'not your usage limit',
|
|
73
|
+
'not your account',
|
|
74
|
+
"not your account's",
|
|
75
|
+
'temporarily limiting requests',
|
|
76
|
+
'temporarily rate',
|
|
77
|
+
'server is temporarily',
|
|
78
|
+
'would exceed your account’s rate limit',
|
|
79
|
+
"would exceed your account's rate limit",
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* True when `text` carries an EXPLICIT transient-burst marker (see
|
|
84
|
+
* `transientUpstreamSignals`). Never throws on weird input. Used to keep the
|
|
85
|
+
* calm rate-limit path and the model-unavailable detector reading the same
|
|
86
|
+
* canonical signal list.
|
|
87
|
+
*/
|
|
88
|
+
export function isTransientUpstreamSignal(text: string): boolean {
|
|
89
|
+
if (typeof text !== 'string' || text.length === 0) return false
|
|
90
|
+
const sample = text.length > 16_384 ? text.slice(0, 16_384) : text
|
|
91
|
+
const lower = sample.toLowerCase()
|
|
92
|
+
return transientUpstreamSignals.some(s => lower.includes(s))
|
|
93
|
+
}
|
|
94
|
+
|
|
44
95
|
// ─── Detection ───────────────────────────────────────────────────────────────
|
|
45
96
|
|
|
46
97
|
/**
|
|
@@ -77,17 +128,9 @@ export function detectModelUnavailable(
|
|
|
77
128
|
// failover that self-cancels and leaves the turn dead. These are upstream
|
|
78
129
|
// throttles Claude Code retries internally with backoff — classify them as
|
|
79
130
|
// `overload` (the calm rate-limit path) BEFORE the quota substrings run, so
|
|
80
|
-
// the negation is honoured and no failover is announced.
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
'not your account',
|
|
84
|
-
"not your account's",
|
|
85
|
-
'temporarily limiting requests',
|
|
86
|
-
'temporarily rate',
|
|
87
|
-
'server is temporarily',
|
|
88
|
-
'would exceed your account’s rate limit',
|
|
89
|
-
"would exceed your account's rate limit",
|
|
90
|
-
]
|
|
131
|
+
// the negation is honoured and no failover is announced. The signal list is
|
|
132
|
+
// module-level (`transientUpstreamSignals`) so session-tail.ts classifies a
|
|
133
|
+
// 429 against the SAME canonical wording.
|
|
91
134
|
if (transientUpstreamSignals.some(s => lower.includes(s))) {
|
|
92
135
|
const resetAt = parseResetTime(sample)
|
|
93
136
|
return resetAt !== undefined
|
|
@@ -182,7 +225,7 @@ export function detectModelUnavailable(
|
|
|
182
225
|
* arg lets tests pin the relative-clock anchor; production callers omit
|
|
183
226
|
* it to use Date.now().
|
|
184
227
|
*/
|
|
185
|
-
function parseResetTime(text: string, parseTimeNow: Date = new Date()): Date | undefined {
|
|
228
|
+
export function parseResetTime(text: string, parseTimeNow: Date = new Date()): Date | undefined {
|
|
186
229
|
const lower = text.toLowerCase()
|
|
187
230
|
|
|
188
231
|
// "retry after 60 seconds" / "retry-after: 60"
|
|
@@ -429,6 +472,11 @@ function formatReason(d: ModelUnavailableDetection, now: Date): string {
|
|
|
429
472
|
return `quota exhausted${reset}`
|
|
430
473
|
case 'overload':
|
|
431
474
|
return `model overloaded${reset}`
|
|
475
|
+
case 'rate_limited':
|
|
476
|
+
// Throttle-tier escalation (429 with transient wording but a reset too
|
|
477
|
+
// far out to wait in place). Honest cause: the account is rate-limited,
|
|
478
|
+
// not quota-exhausted — the reset names when it frees.
|
|
479
|
+
return `account rate-limited${reset}`
|
|
432
480
|
case 'network':
|
|
433
481
|
return 'network unreachable'
|
|
434
482
|
}
|