switchroom 0.18.13 → 0.18.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +49 -9
- package/dist/auth-broker/index.js +111 -7
- package/dist/cli/autoaccept-poll.js +23 -0
- package/dist/cli/drive-write-pretool.mjs +24 -1
- package/dist/cli/foreground-hog-pretool.mjs +264 -0
- package/dist/cli/notion-write-pretool.mjs +0 -1
- package/dist/cli/switchroom.js +35 -6
- package/dist/host-control/main.js +1 -2
- package/dist/vault/approvals/kernel-server.js +0 -1
- package/dist/vault/broker/server.js +0 -1
- package/package.json +1 -1
- package/profiles/coding/CLAUDE.md.hbs +2 -0
- package/profiles/default/CLAUDE.md.hbs +2 -0
- package/skills/switchroom-architecture/telegram.md +0 -1
- package/telegram-plugin/auth-snapshot-format.ts +37 -5
- package/telegram-plugin/auto-fallback-fleet.ts +29 -1
- package/telegram-plugin/bridge/bridge.ts +2 -0
- package/telegram-plugin/dist/bridge/bridge.js +2 -0
- package/telegram-plugin/dist/gateway/gateway.js +620 -67
- package/telegram-plugin/dist/server.js +2 -0
- package/telegram-plugin/gateway/auth-broker-client.ts +1 -0
- package/telegram-plugin/gateway/auth-command.ts +14 -0
- package/telegram-plugin/gateway/forward-origin.ts +235 -0
- package/telegram-plugin/gateway/gateway.ts +224 -10
- package/telegram-plugin/gateway/throttle-tier-wiring.ts +268 -0
- package/telegram-plugin/history.ts +55 -6
- package/telegram-plugin/model-unavailable.ts +20 -2
- package/telegram-plugin/render/rich-render.ts +40 -32
- package/telegram-plugin/stream-controller.ts +3 -2
- package/telegram-plugin/tests/auto-fallback-fleet.test.ts +72 -0
- package/telegram-plugin/tests/forward-origin.test.ts +309 -0
- package/telegram-plugin/tests/history.test.ts +157 -0
- package/telegram-plugin/tests/render/render-outbound-chunks.test.ts +6 -4
- package/telegram-plugin/tests/render/rich-render.test.ts +41 -22
- package/telegram-plugin/tests/single-mode-stream-reply.test.ts +5 -3
- package/telegram-plugin/tests/status-accent.test.ts +5 -3
- package/telegram-plugin/tests/stream-controller-chunk-cap.test.ts +20 -20
- package/telegram-plugin/tests/stream-reply-handler.test.ts +5 -2
- package/telegram-plugin/tests/throttle-tier-wiring.test.ts +290 -0
- package/telegram-plugin/tests/throttle-tier.test.ts +278 -0
- package/telegram-plugin/throttle-tier.ts +226 -0
- package/telegram-plugin/uat/scenarios/jtbd-rich-formatting-render-dm.test.ts +8 -7
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* throttle-tier-wiring.ts — side-effect runner for the 429 throttle tier.
|
|
3
|
+
*
|
|
4
|
+
* The DECISION and notice text live in ../throttle-tier.ts (pure). This
|
|
5
|
+
* module owns the sequenced side effects, with every dependency injected so
|
|
6
|
+
* the wiring is unit-testable without importing gateway.ts:
|
|
7
|
+
*
|
|
8
|
+
* 1. Broker `mark-throttled` — records `throttled_until` in the quota
|
|
9
|
+
* ledger (no roll, no eligibility change) and runs the broker-side
|
|
10
|
+
* escalation guard (3 hits / 10 min → live probe → mark-exhausted when
|
|
11
|
+
* corroborated). EVERY fire reaches the broker — the notice cooldown
|
|
12
|
+
* below never suppresses the ledger write, because the escalation
|
|
13
|
+
* counter is what corroborates a wall hiding behind transient wording.
|
|
14
|
+
* 2. ONE lightweight operator notice — deduped per account BOTH locally
|
|
15
|
+
* (cooldown window) and FLEET-WIDE via the broker's claim-notification
|
|
16
|
+
* verb (N agents sharing a throttled account produce one copy per chat,
|
|
17
|
+
* not N). Claim failures FAIL OPEN (send anyway) per the claim
|
|
18
|
+
* contract.
|
|
19
|
+
* 3. A delayed retry nudge — after `throttled_until` (+slack +jitter so N
|
|
20
|
+
* agents don't restart-and-replay simultaneously into the just-cleared
|
|
21
|
+
* account), replay the turn the 429 killed via the existing resume
|
|
22
|
+
* lever (triggerSelfRestart → boot-resume). Guards, in order:
|
|
23
|
+
* - a LIVE turn that started AFTER the throttle was armed supersedes
|
|
24
|
+
* the dead turn — skip entirely (restarting would kill live work
|
|
25
|
+
* and boot-resume would replay the WRONG turn);
|
|
26
|
+
* - a live turn that started BEFORE the arm is the dead turn itself
|
|
27
|
+
* still holding the in-flight gate — defer the restart to the
|
|
28
|
+
* turn-complete drain (`pendingRestarts`) instead of SIGTERM-now;
|
|
29
|
+
* - otherwise consult the SHARED fleet-fallback resume gate
|
|
30
|
+
* (single-flight across throttle AND fallback resumes + staleness)
|
|
31
|
+
* and restart on a 'resume' verdict.
|
|
32
|
+
*
|
|
33
|
+
* The escalated outcome (broker corroborated a wall and already rolled)
|
|
34
|
+
* posts its own announcement — gateway-side, per the reactive-path doctrine
|
|
35
|
+
* (`LastFleetRoll` docstring in src/auth/broker/server.ts), which also
|
|
36
|
+
* covers PINNED (non-fleet-active) account rolls — then nudges the resume
|
|
37
|
+
* immediately through the same turn-safety guards.
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
import {
|
|
41
|
+
evaluateThrottleNotice,
|
|
42
|
+
renderThrottleEscalationNotice,
|
|
43
|
+
renderThrottleNotice,
|
|
44
|
+
THROTTLE_NOTICE_COOLDOWN_MS,
|
|
45
|
+
type ThrottleNoticeState,
|
|
46
|
+
} from '../throttle-tier.js'
|
|
47
|
+
|
|
48
|
+
/** Slack past throttled_until before the retry nudge fires. */
|
|
49
|
+
export const THROTTLE_RETRY_NUDGE_SLACK_MS = 5_000
|
|
50
|
+
|
|
51
|
+
/** Max random jitter added to the nudge so agents sharing the throttled
|
|
52
|
+
* account stagger their restart-and-replay instead of stampeding the
|
|
53
|
+
* just-cleared account. */
|
|
54
|
+
export const THROTTLE_RETRY_NUDGE_JITTER_MAX_MS = 30_000
|
|
55
|
+
|
|
56
|
+
/** The narrow broker surface the runner needs (structurally satisfied by
|
|
57
|
+
* the gateway's AuthBrokerClient). */
|
|
58
|
+
export interface ThrottleBrokerClient {
|
|
59
|
+
markThrottled(until: number): Promise<{
|
|
60
|
+
account: string
|
|
61
|
+
throttled_until: number
|
|
62
|
+
escalated: boolean
|
|
63
|
+
rolledTo?: string | null
|
|
64
|
+
}>
|
|
65
|
+
claimNotification(key: string, windowMs: number): Promise<{ granted: boolean }>
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface ThrottleTierRunnerDeps {
|
|
69
|
+
/** This gateway's own agent (SWITCHROOM_AGENT_NAME). */
|
|
70
|
+
agentName: string
|
|
71
|
+
getBrokerClient(): Promise<ThrottleBrokerClient | null>
|
|
72
|
+
/** Chats the notice broadcasts to (access.allowFrom, resolved per call). */
|
|
73
|
+
listNoticeChats(): Array<string | number>
|
|
74
|
+
/** Fire-and-forget rich send (gateway wraps swallowingApiCall). */
|
|
75
|
+
sendNotice(chatId: string | number, markdown: string): void
|
|
76
|
+
/** THE shared fleet-fallback resume gate (single-flight + staleness). */
|
|
77
|
+
resumeDecide(failedTurnStartedAtMs: number | null): 'resume' | 'skip-inflight' | 'skip-stale'
|
|
78
|
+
newestActiveTurnStartedAtMs(): number | null
|
|
79
|
+
/** True while a turn is in flight (gateway turnInFlightForGate()). */
|
|
80
|
+
turnInFlight(): boolean
|
|
81
|
+
/** Defer the restart to the turn-complete drain (pendingRestarts). */
|
|
82
|
+
deferRestartToTurnComplete(agentName: string, reason: string): void
|
|
83
|
+
/** Restart now (gateway triggerSelfRestart). */
|
|
84
|
+
restartNow(agentName: string, reason: string): void
|
|
85
|
+
log(msg: string): void
|
|
86
|
+
now?: () => number
|
|
87
|
+
/** Timer seam (tests drive synchronously). Default setTimeout+unref. */
|
|
88
|
+
schedule?: (fn: () => void, ms: number) => { cancel(): void }
|
|
89
|
+
/** Jitter source (tests pin it). Default uniform 0..JITTER_MAX. */
|
|
90
|
+
jitterMs?: () => number
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export interface ThrottleTierRunner {
|
|
94
|
+
/**
|
|
95
|
+
* Run the throttle path for one terminal transient 429. Fire-and-forget
|
|
96
|
+
* from the caller's perspective — never throws.
|
|
97
|
+
*/
|
|
98
|
+
fire(triggerAgent: string, throttledUntilMs: number, resetParsed: boolean): Promise<void>
|
|
99
|
+
/** Test/debug view of internal state. */
|
|
100
|
+
inspect(): { noticeState: ThrottleNoticeState; nudgePending: boolean }
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export function createThrottleTierRunner(deps: ThrottleTierRunnerDeps): ThrottleTierRunner {
|
|
104
|
+
const now = deps.now ?? (() => Date.now())
|
|
105
|
+
const schedule =
|
|
106
|
+
deps.schedule ??
|
|
107
|
+
((fn: () => void, ms: number) => {
|
|
108
|
+
const t = setTimeout(fn, ms)
|
|
109
|
+
if (typeof t.unref === 'function') t.unref()
|
|
110
|
+
return { cancel: () => clearTimeout(t) }
|
|
111
|
+
})
|
|
112
|
+
const jitterMs =
|
|
113
|
+
deps.jitterMs ?? (() => Math.floor(Math.random() * THROTTLE_RETRY_NUDGE_JITTER_MAX_MS))
|
|
114
|
+
|
|
115
|
+
let noticeState: ThrottleNoticeState = { lastSentAtMsByAccount: {} }
|
|
116
|
+
/** The LATEST throttle owns the nudge — a newer hit replaces an armed
|
|
117
|
+
* timer instead of stacking restarts. */
|
|
118
|
+
let pendingNudge: { cancel(): void } | null = null
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Post `markdown` to every authorized chat, fleet-deduped per chat via the
|
|
122
|
+
* broker claim verb when a client + account are available. Fail-open: a
|
|
123
|
+
* claim error or missing broker never drops the notice.
|
|
124
|
+
*/
|
|
125
|
+
async function broadcastDeduped(
|
|
126
|
+
client: ThrottleBrokerClient | null,
|
|
127
|
+
keyPrefix: string,
|
|
128
|
+
account: string | null,
|
|
129
|
+
markdown: string,
|
|
130
|
+
): Promise<void> {
|
|
131
|
+
for (const chatId of deps.listNoticeChats()) {
|
|
132
|
+
let granted = true
|
|
133
|
+
if (client && account) {
|
|
134
|
+
try {
|
|
135
|
+
granted = (
|
|
136
|
+
await client.claimNotification(
|
|
137
|
+
`${keyPrefix}:${account}:${chatId}`,
|
|
138
|
+
THROTTLE_NOTICE_COOLDOWN_MS,
|
|
139
|
+
)
|
|
140
|
+
).granted
|
|
141
|
+
} catch {
|
|
142
|
+
granted = true // fail open — a duplicated notice beats a dropped one
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
if (granted) {
|
|
146
|
+
deps.sendNotice(chatId, markdown)
|
|
147
|
+
} else {
|
|
148
|
+
deps.log(`[throttle-tier] notice suppressed (fleet claim) chat=${chatId}`)
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Replay the dead turn via restart, with the turn-safety guards (see
|
|
155
|
+
* module docstring). `armedAtMs` anchors the "newer turn supersedes"
|
|
156
|
+
* check: a turn that started after the throttle was armed is live user
|
|
157
|
+
* work, never to be killed for a replay.
|
|
158
|
+
*/
|
|
159
|
+
function nudgeResume(reason: string, armedAtMs: number): void {
|
|
160
|
+
const newest = deps.newestActiveTurnStartedAtMs()
|
|
161
|
+
if (deps.turnInFlight()) {
|
|
162
|
+
if (newest != null && newest > armedAtMs) {
|
|
163
|
+
deps.log(
|
|
164
|
+
`[throttle-tier] resume skipped (superseded by a live newer turn) reason=${reason}`,
|
|
165
|
+
)
|
|
166
|
+
return
|
|
167
|
+
}
|
|
168
|
+
// The in-flight gate is held by the dead throttled turn itself —
|
|
169
|
+
// defer to the turn-complete drain instead of SIGTERM-ing now.
|
|
170
|
+
deps.log(`[throttle-tier] resume deferred to turn-complete reason=${reason}`)
|
|
171
|
+
deps.deferRestartToTurnComplete(deps.agentName, reason)
|
|
172
|
+
return
|
|
173
|
+
}
|
|
174
|
+
const verdict = deps.resumeDecide(newest)
|
|
175
|
+
if (verdict === 'resume') {
|
|
176
|
+
deps.log(`[throttle-tier] resuming dead turn via self-restart reason=${reason}`)
|
|
177
|
+
deps.restartNow(deps.agentName, reason)
|
|
178
|
+
} else {
|
|
179
|
+
deps.log(`[throttle-tier] resume suppressed (${verdict}) reason=${reason}`)
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
async function fire(
|
|
184
|
+
triggerAgent: string,
|
|
185
|
+
throttledUntilMs: number,
|
|
186
|
+
resetParsed: boolean,
|
|
187
|
+
): Promise<void> {
|
|
188
|
+
const armedAtMs = now()
|
|
189
|
+
let client: ThrottleBrokerClient | null = null
|
|
190
|
+
let account: string | null = null
|
|
191
|
+
let escalated = false
|
|
192
|
+
let rolledTo: string | null = null
|
|
193
|
+
try {
|
|
194
|
+
client = await deps.getBrokerClient()
|
|
195
|
+
if (client) {
|
|
196
|
+
const r = await client.markThrottled(throttledUntilMs)
|
|
197
|
+
account = r.account
|
|
198
|
+
escalated = r.escalated
|
|
199
|
+
rolledTo = r.rolledTo ?? null
|
|
200
|
+
} else {
|
|
201
|
+
deps.log(
|
|
202
|
+
`[throttle-tier] broker unreachable — notice only, no ledger record agent=${triggerAgent}`,
|
|
203
|
+
)
|
|
204
|
+
}
|
|
205
|
+
} catch (err) {
|
|
206
|
+
deps.log(
|
|
207
|
+
`[throttle-tier] markThrottled failed agent=${triggerAgent}: ${(err as Error)?.message ?? err}`,
|
|
208
|
+
)
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
if (escalated) {
|
|
212
|
+
// The broker corroborated a genuine wall via a live probe and already
|
|
213
|
+
// ran mark-exhausted + roll (fleet active AND pinned accounts alike).
|
|
214
|
+
// The RAISING gateway announces — reactive-path doctrine — fleet-
|
|
215
|
+
// deduped so N gateways sharing the account produce one copy per chat.
|
|
216
|
+
deps.log(
|
|
217
|
+
`[throttle-tier] escalated to wall account=${account ?? '?'} ` +
|
|
218
|
+
`rolledTo=${rolledTo ?? 'none (all blocked)'}`,
|
|
219
|
+
)
|
|
220
|
+
await broadcastDeduped(
|
|
221
|
+
client,
|
|
222
|
+
'throttle-escalation',
|
|
223
|
+
account,
|
|
224
|
+
renderThrottleEscalationNotice({ account, agent: triggerAgent, rolledTo }),
|
|
225
|
+
)
|
|
226
|
+
if (rolledTo) nudgeResume('throttle-escalation-resume', armedAtMs)
|
|
227
|
+
return
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// ONE lightweight notice, deduped per account: locally by cooldown, then
|
|
231
|
+
// fleet-wide by broker claim. Unknown account (broker down) keys the
|
|
232
|
+
// local cooldown on the agent so the degraded path still can't spam.
|
|
233
|
+
const cooldownKey = account ?? `agent:${triggerAgent}`
|
|
234
|
+
const verdict = evaluateThrottleNotice(noticeState, cooldownKey, now())
|
|
235
|
+
if (verdict.send) {
|
|
236
|
+
noticeState = verdict.next
|
|
237
|
+
await broadcastDeduped(
|
|
238
|
+
client,
|
|
239
|
+
'throttle-notice',
|
|
240
|
+
account,
|
|
241
|
+
renderThrottleNotice({
|
|
242
|
+
account,
|
|
243
|
+
agent: triggerAgent,
|
|
244
|
+
throttledUntilMs,
|
|
245
|
+
resetParsed,
|
|
246
|
+
now: new Date(now()),
|
|
247
|
+
}),
|
|
248
|
+
)
|
|
249
|
+
} else {
|
|
250
|
+
deps.log(`[throttle-tier] notice suppressed (cooldown) key=${cooldownKey}`)
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// Arm (or re-arm) the retry nudge: slack past the reset + jitter so
|
|
254
|
+
// agents sharing the account stagger their replays.
|
|
255
|
+
const delayMs =
|
|
256
|
+
Math.max(throttledUntilMs - now(), 0) + THROTTLE_RETRY_NUDGE_SLACK_MS + jitterMs()
|
|
257
|
+
if (pendingNudge) pendingNudge.cancel()
|
|
258
|
+
pendingNudge = schedule(() => {
|
|
259
|
+
pendingNudge = null
|
|
260
|
+
nudgeResume('throttle-retry-resume', armedAtMs)
|
|
261
|
+
}, delayMs)
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
return {
|
|
265
|
+
fire,
|
|
266
|
+
inspect: () => ({ noticeState, nudgePending: pendingNudge != null }),
|
|
267
|
+
}
|
|
268
|
+
}
|
|
@@ -108,6 +108,25 @@ export interface RecordedMessage {
|
|
|
108
108
|
* Only emoji reactions are tracked — custom emoji are ignored for v1.
|
|
109
109
|
*/
|
|
110
110
|
user_reaction: string | null
|
|
111
|
+
/**
|
|
112
|
+
* Set when the inbound user message was FORWARDED: the server-stamped
|
|
113
|
+
* `forward_origin` of the original message (Bot API 7.0+), so
|
|
114
|
+
* get_recent_messages can surface who originally sent the content the
|
|
115
|
+
* agent saw at delivery time. `forwarded_from` is the raw (truncated,
|
|
116
|
+
* unescaped) human-readable name/title; `forwarded_from_type` is
|
|
117
|
+
* user|hidden_user|chat|channel (hidden_user = self-reported display
|
|
118
|
+
* name, no verifiable id); `forwarded_from_id` is the numeric id when
|
|
119
|
+
* the origin shape exposes one; `forwarded_date` is the original
|
|
120
|
+
* message's ISO timestamp; `forwarded_message_id` is the message id
|
|
121
|
+
* inside the origin channel (channel origins only). For a multi-origin
|
|
122
|
+
* coalesced burst only the PRIMARY (first) origin is persisted here —
|
|
123
|
+
* origins 2+ exist only in the delivered channel tag's numbered attrs.
|
|
124
|
+
*/
|
|
125
|
+
forwarded_from: string | null
|
|
126
|
+
forwarded_from_type: string | null
|
|
127
|
+
forwarded_from_id: string | null
|
|
128
|
+
forwarded_date: string | null
|
|
129
|
+
forwarded_message_id: number | null
|
|
111
130
|
}
|
|
112
131
|
|
|
113
132
|
export interface QueryOptions {
|
|
@@ -166,10 +185,20 @@ export function initHistory(stateDir: string, retentionDays = 30): void {
|
|
|
166
185
|
CREATE INDEX IF NOT EXISTS idx_messages_recent
|
|
167
186
|
ON messages (chat_id, thread_id, ts DESC)
|
|
168
187
|
`)
|
|
169
|
-
// Migration: add reply_to columns to existing DBs that pre-date issue #119
|
|
170
|
-
//
|
|
171
|
-
//
|
|
172
|
-
|
|
188
|
+
// Migration: add reply_to columns to existing DBs that pre-date issue #119,
|
|
189
|
+
// and the forwarded_* origin columns (forward_origin metadata) to DBs that
|
|
190
|
+
// pre-date them. SQLite has no IF NOT EXISTS for ALTER TABLE ADD COLUMN, so
|
|
191
|
+
// we tolerate "duplicate column name" errors and re-throw anything else.
|
|
192
|
+
for (const column of [
|
|
193
|
+
"reply_to_message_id INTEGER",
|
|
194
|
+
"reply_to_text TEXT",
|
|
195
|
+
"user_reaction TEXT",
|
|
196
|
+
"forwarded_from TEXT",
|
|
197
|
+
"forwarded_from_type TEXT",
|
|
198
|
+
"forwarded_from_id TEXT",
|
|
199
|
+
"forwarded_date TEXT",
|
|
200
|
+
"forwarded_message_id INTEGER",
|
|
201
|
+
]) {
|
|
173
202
|
try {
|
|
174
203
|
db.exec(`ALTER TABLE messages ADD COLUMN ${column}`)
|
|
175
204
|
} catch (err) {
|
|
@@ -349,6 +378,19 @@ interface RecordInboundArgs {
|
|
|
349
378
|
*/
|
|
350
379
|
reply_to_message_id?: number | null | undefined
|
|
351
380
|
reply_to_text?: string | null | undefined
|
|
381
|
+
/**
|
|
382
|
+
* If the message was forwarded, the server-stamped origin metadata
|
|
383
|
+
* (Bot API 7.0 `forward_origin`). Populated from
|
|
384
|
+
* `ctx.message.forward_origin` in the gateway handler. `forwarded_from`
|
|
385
|
+
* is the RAW (truncated, unescaped) name — the XML-escaped form goes to
|
|
386
|
+
* the channel meta only. `forwarded_date` is the origin message's ISO
|
|
387
|
+
* timestamp; `forwarded_message_id` is set for channel origins only.
|
|
388
|
+
*/
|
|
389
|
+
forwarded_from?: string | null | undefined
|
|
390
|
+
forwarded_from_type?: string | null | undefined
|
|
391
|
+
forwarded_from_id?: string | null | undefined
|
|
392
|
+
forwarded_date?: string | null | undefined
|
|
393
|
+
forwarded_message_id?: number | null | undefined
|
|
352
394
|
}
|
|
353
395
|
|
|
354
396
|
/**
|
|
@@ -364,8 +406,8 @@ export function recordInbound(args: RecordInboundArgs): void {
|
|
|
364
406
|
if (args.message_id == null) return
|
|
365
407
|
const stmt = requireDb().prepare(`
|
|
366
408
|
INSERT OR REPLACE INTO messages
|
|
367
|
-
(chat_id, thread_id, message_id, role, user, user_id, ts, text, attachment_kind, group_id, reply_to_message_id, reply_to_text)
|
|
368
|
-
VALUES (?, ?, ?, 'user', ?, ?, ?, ?, ?, NULL, ?, ?)
|
|
409
|
+
(chat_id, thread_id, message_id, role, user, user_id, ts, text, attachment_kind, group_id, reply_to_message_id, reply_to_text, forwarded_from, forwarded_from_type, forwarded_from_id, forwarded_date, forwarded_message_id)
|
|
410
|
+
VALUES (?, ?, ?, 'user', ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?)
|
|
369
411
|
`)
|
|
370
412
|
// Defense-in-depth: never persist a detected secret to the message store.
|
|
371
413
|
// The inbound gate (server.ts handleInbound) already deletes + vaults a
|
|
@@ -382,6 +424,13 @@ export function recordInbound(args: RecordInboundArgs): void {
|
|
|
382
424
|
args.attachment_kind ?? null,
|
|
383
425
|
args.reply_to_message_id ?? null,
|
|
384
426
|
args.reply_to_text != null ? redact(args.reply_to_text) : (args.reply_to_text ?? null),
|
|
427
|
+
// Origin names/titles are user-controlled display strings; run them
|
|
428
|
+
// through the same secret-redaction backstop as message text.
|
|
429
|
+
args.forwarded_from != null ? redact(args.forwarded_from) : null,
|
|
430
|
+
args.forwarded_from_type ?? null,
|
|
431
|
+
args.forwarded_from_id ?? null,
|
|
432
|
+
args.forwarded_date ?? null,
|
|
433
|
+
args.forwarded_message_id ?? null,
|
|
385
434
|
)
|
|
386
435
|
}
|
|
387
436
|
|
|
@@ -31,7 +31,20 @@ import { escapeMarkdown } from './card-format.js'
|
|
|
31
31
|
|
|
32
32
|
// ─── Public types ────────────────────────────────────────────────────────────
|
|
33
33
|
|
|
34
|
-
export type ModelUnavailableKind =
|
|
34
|
+
export type ModelUnavailableKind =
|
|
35
|
+
| 'overload'
|
|
36
|
+
| 'quota_exhausted'
|
|
37
|
+
| 'network'
|
|
38
|
+
/**
|
|
39
|
+
* 429 throttle tier — a TRANSIENT per-account 429 (explicit
|
|
40
|
+
* `transientUpstreamSignals` negation wording) whose parsed reset lies
|
|
41
|
+
* BEYOND the retry-in-place threshold, so the gateway escalates it to the
|
|
42
|
+
* standard mark-exhausted + fleet-failover machinery. Never produced by
|
|
43
|
+
* `detectModelUnavailable` (a transient-negation string classifies as
|
|
44
|
+
* `overload` there); constructed only by the gateway's throttle-tier
|
|
45
|
+
* branch so the card names the true cause instead of "quota exhausted".
|
|
46
|
+
*/
|
|
47
|
+
| 'rate_limited'
|
|
35
48
|
|
|
36
49
|
export interface ModelUnavailableDetection {
|
|
37
50
|
kind: ModelUnavailableKind
|
|
@@ -212,7 +225,7 @@ export function detectModelUnavailable(
|
|
|
212
225
|
* arg lets tests pin the relative-clock anchor; production callers omit
|
|
213
226
|
* it to use Date.now().
|
|
214
227
|
*/
|
|
215
|
-
function parseResetTime(text: string, parseTimeNow: Date = new Date()): Date | undefined {
|
|
228
|
+
export function parseResetTime(text: string, parseTimeNow: Date = new Date()): Date | undefined {
|
|
216
229
|
const lower = text.toLowerCase()
|
|
217
230
|
|
|
218
231
|
// "retry after 60 seconds" / "retry-after: 60"
|
|
@@ -459,6 +472,11 @@ function formatReason(d: ModelUnavailableDetection, now: Date): string {
|
|
|
459
472
|
return `quota exhausted${reset}`
|
|
460
473
|
case 'overload':
|
|
461
474
|
return `model overloaded${reset}`
|
|
475
|
+
case 'rate_limited':
|
|
476
|
+
// Throttle-tier escalation (429 with transient wording but a reset too
|
|
477
|
+
// far out to wait in place). Honest cause: the account is rate-limited,
|
|
478
|
+
// not quota-exhausted — the reset names when it frees.
|
|
479
|
+
return `account rate-limited${reset}`
|
|
462
480
|
case 'network':
|
|
463
481
|
return 'network unreachable'
|
|
464
482
|
}
|
|
@@ -1,21 +1,24 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
1
|
+
// Wiring for the Bot API 10.1 rich renderer (parse.ts + render.ts) into the
|
|
2
|
+
// live outbound send path.
|
|
3
3
|
//
|
|
4
4
|
// The renderer (`render/render.ts`, PR #2930) and parser (`render/parse.ts`)
|
|
5
5
|
// have full unit coverage but were, until this module, wired into NOTHING —
|
|
6
|
-
// no outbound message ever flowed through them. This module is the single
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
6
|
+
// no outbound message ever flowed through them. This module is the single
|
|
7
|
+
// bridge: the gateway's rich send path (stream-controller.ts) runs the
|
|
8
|
+
// assistant's raw markdown through `parse → renderSafe` before it reaches
|
|
9
|
+
// `sendRichMessage`, unless the escape hatch below disables it.
|
|
10
10
|
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
//
|
|
11
|
+
// Escape hatch — `SWITCHROOM_RICH_RENDER`, default ON:
|
|
12
|
+
// ON BY DEFAULT in every install — an escape hatch, not an opt-in feature,
|
|
13
|
+
// mirroring the send gate's convention (`sendGateEnabledFromEnv` in
|
|
14
|
+
// `send-gate.ts`, default-on since #3153; also `midTurnFloorEnabled`,
|
|
15
|
+
// `SWITCHROOM_RATE_LIMIT_OVERAGE=0`). Enabled unless the var is explicitly
|
|
16
|
+
// set to a falsey/off value (`0`/`false`/`off`/`no`, case-insensitive,
|
|
17
|
+
// trimmed) — per agent via the `env:` block in `switchroom.yaml`
|
|
18
|
+
// (propagated into the container `environment:` by
|
|
19
|
+
// `src/agents/compose.ts`). When disabled, `maybeRenderOutbound` returns
|
|
20
|
+
// the input untouched, so the live send path is byte-for-byte the
|
|
21
|
+
// pre-renderer behaviour (raw transcript markdown to `sendRichMessage`).
|
|
19
22
|
//
|
|
20
23
|
// Why route through `renderSafe` and not bare `render`:
|
|
21
24
|
// `renderSafe` guarantees the returned body is never a rich-markdown string
|
|
@@ -38,17 +41,19 @@ import { RICH_MESSAGE_MAX_CHARS, splitMarkdownChunks } from "../format.js";
|
|
|
38
41
|
*/
|
|
39
42
|
export const PLAIN_TEXT_MAX_CHARS = 4096;
|
|
40
43
|
|
|
41
|
-
/** Parse the `SWITCHROOM_RICH_RENDER`
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
+
/** Parse the `SWITCHROOM_RICH_RENDER` kill-switch value. Default ON; disabled
|
|
45
|
+
* only by an explicit falsey/off token (`0`/`false`/`off`/`no`,
|
|
46
|
+
* case-insensitive, trimmed) — the same disable vocabulary as
|
|
47
|
+
* `sendGateEnabledFromEnv`. Unset, empty, or unrecognised values → ON. Pure
|
|
48
|
+
* so the default + parsing are unit-testable. */
|
|
44
49
|
export function parseRichRenderEnabled(raw: string | undefined): boolean {
|
|
45
|
-
if (raw == null) return
|
|
50
|
+
if (raw == null) return true;
|
|
46
51
|
const v = raw.trim().toLowerCase();
|
|
47
|
-
return v === "
|
|
52
|
+
return !(v === "0" || v === "false" || v === "off" || v === "no");
|
|
48
53
|
}
|
|
49
54
|
|
|
50
55
|
/** Is the rich renderer enabled in this process? Reads the env flag live so a
|
|
51
|
-
* test can set/unset it per-case; defaults
|
|
56
|
+
* test can set/unset it per-case; defaults ON (escape hatch, not opt-in). */
|
|
52
57
|
export function richRenderEnabled(
|
|
53
58
|
env: NodeJS.ProcessEnv = process.env,
|
|
54
59
|
): boolean {
|
|
@@ -65,13 +70,14 @@ export function renderOutbound(
|
|
|
65
70
|
}
|
|
66
71
|
|
|
67
72
|
/**
|
|
68
|
-
*
|
|
73
|
+
* Kill-switch-gated transform for the live send path.
|
|
69
74
|
*
|
|
70
|
-
* -
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
* -
|
|
74
|
-
*
|
|
75
|
+
* - enabled (default): returns `parse → renderSafe` output. `mode: "plain"`
|
|
76
|
+
* signals the caller to send WITHOUT the rich wrapper (oversized/unsafe
|
|
77
|
+
* content).
|
|
78
|
+
* - disabled (`SWITCHROOM_RICH_RENDER=0`): returns the input untouched,
|
|
79
|
+
* `mode: "markdown"` — identical to the pre-renderer behaviour (raw
|
|
80
|
+
* transcript markdown sent straight to `sendRichMessage`).
|
|
75
81
|
*/
|
|
76
82
|
export function maybeRenderOutbound(
|
|
77
83
|
text: string,
|
|
@@ -83,7 +89,7 @@ export function maybeRenderOutbound(
|
|
|
83
89
|
}
|
|
84
90
|
|
|
85
91
|
/**
|
|
86
|
-
*
|
|
92
|
+
* Kill-switch-gated, CAP-ENFORCING transform for the live send path.
|
|
87
93
|
*
|
|
88
94
|
* `maybeRenderOutbound` returns ONE `RenderResult` and can only ever fit a
|
|
89
95
|
* body into a single wire message. But `renderSafe`'s markdown re-escaping
|
|
@@ -104,11 +110,13 @@ export function maybeRenderOutbound(
|
|
|
104
110
|
* plain degradation would have thrown away: the smaller pieces individually
|
|
105
111
|
* escape under `maxLen` and come back as `markdown`.
|
|
106
112
|
*
|
|
107
|
-
* -
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
113
|
+
* - disabled (`SWITCHROOM_RICH_RENDER=0`):
|
|
114
|
+
* `[{ text, mode: "markdown", degradations: [] }]` — a single passthrough
|
|
115
|
+
* piece, identical to `maybeRenderOutbound`.
|
|
116
|
+
* - enabled (default), body fits: `[renderSafe(...)]` — a single piece,
|
|
117
|
+
* identical to `maybeRenderOutbound` (byte-for-byte for the common case).
|
|
118
|
+
* - enabled (default), body oversize: 2+ cap-respecting pieces in send
|
|
119
|
+
* order.
|
|
112
120
|
*/
|
|
113
121
|
export function renderOutboundChunks(
|
|
114
122
|
text: string,
|
|
@@ -228,8 +228,9 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
|
|
|
228
228
|
// `maybeRenderOutbound` path; a body whose markdown-escaping pushed the
|
|
229
229
|
// rendered form past the cap splits into several pieces, each of which fits
|
|
230
230
|
// its own wire cap and never bisects a fenced block / table row (see
|
|
231
|
-
// `renderOutboundChunks`).
|
|
232
|
-
// stream both yield a single passthrough
|
|
231
|
+
// `renderOutboundChunks`). Rendering disabled (SWITCHROOM_RICH_RENDER=0)
|
|
232
|
+
// and a literal `format:'text'` stream both yield a single passthrough
|
|
233
|
+
// piece — byte-for-byte the pre-renderer send path.
|
|
233
234
|
const renderPieces = (text: string): { text: string; rich: boolean }[] => {
|
|
234
235
|
if (literalText) return [{ text, rich: false }]
|
|
235
236
|
// A `plain`-mode piece (oversized/unsafe content renderSafe declined to
|
|
@@ -118,6 +118,78 @@ describe('runFleetAutoFallback', () => {
|
|
|
118
118
|
}
|
|
119
119
|
});
|
|
120
120
|
|
|
121
|
+
it('parsedResetAt (429 throttle tier) names the recovery when the old probe carried no reset', async () => {
|
|
122
|
+
const failover = vi.fn(async () => ({ rolledTo: 'you@x', rolled: ['ken@x'] }));
|
|
123
|
+
const out = await runFleetAutoFallback({
|
|
124
|
+
state: state('ken@x', ['ken@x', 'you@x']),
|
|
125
|
+
quotas: [
|
|
126
|
+
// ken: walled, but the probe carried NO reset time — pre-fix the
|
|
127
|
+
// announcement's recovery line was silently dropped.
|
|
128
|
+
qOk({ fiveHourUtilizationPct: 100, representativeClaim: 'five_hour' }),
|
|
129
|
+
qOk({ fiveHourUtilizationPct: 8, sevenDayUtilizationPct: 20 }),
|
|
130
|
+
],
|
|
131
|
+
failover,
|
|
132
|
+
triggerAgent: 'carrie',
|
|
133
|
+
now: NOW,
|
|
134
|
+
tz: 'UTC',
|
|
135
|
+
// Parsed from the error prose ("resets 5:50am") by the gateway.
|
|
136
|
+
parsedResetAt: new Date('2026-05-15T05:50:00Z'),
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
expect(out.kind).toBe('switched');
|
|
140
|
+
if (out.kind === 'switched') {
|
|
141
|
+
expect(out.announcement).toContain('recovers');
|
|
142
|
+
expect(out.announcement).toContain('in 4h 57m');
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
it('rateLimitTrigger: swaps even when the old account probes HEALTHY (the >threshold leg must execute)', async () => {
|
|
147
|
+
// A terminal transient 429 NEGATES the usage-limit reading, so healthy
|
|
148
|
+
// utilization is the EXPECTED state for the rate-limited account. The
|
|
149
|
+
// healthy-idempotency guard must not self-cancel this swap into a
|
|
150
|
+
// "probed healthy / Stale event?" no-op.
|
|
151
|
+
const failover = vi.fn(async () => ({ rolledTo: 'you@x', rolled: ['ken@x'] }));
|
|
152
|
+
const out = await runFleetAutoFallback({
|
|
153
|
+
state: state('ken@x', ['ken@x', 'you@x']),
|
|
154
|
+
quotas: [
|
|
155
|
+
qOk({ fiveHourUtilizationPct: 12, sevenDayUtilizationPct: 30 }), // healthy!
|
|
156
|
+
qOk({ fiveHourUtilizationPct: 8, sevenDayUtilizationPct: 20 }),
|
|
157
|
+
],
|
|
158
|
+
failover,
|
|
159
|
+
triggerAgent: 'carrie',
|
|
160
|
+
now: NOW,
|
|
161
|
+
tz: 'UTC',
|
|
162
|
+
parsedResetAt: new Date('2026-05-15T02:53:00Z'),
|
|
163
|
+
rateLimitTrigger: true,
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
expect(out.kind).toBe('switched');
|
|
167
|
+
expect(failover).toHaveBeenCalledTimes(1);
|
|
168
|
+
if (out.kind === 'switched') {
|
|
169
|
+
// Honest headline: a rate limit, not a utilization-derived window cap.
|
|
170
|
+
expect(out.announcement).toContain('rate limit on ken@x');
|
|
171
|
+
expect(out.announcement).not.toContain('5-hour limit');
|
|
172
|
+
// Recovery line carries the parsed reset (no window was maxed).
|
|
173
|
+
expect(out.announcement).toContain('recovers');
|
|
174
|
+
expect(out.announcement).toContain('in 2h');
|
|
175
|
+
}
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
it('rateLimitTrigger stays subject to the broker outcome (all-blocked passes through)', async () => {
|
|
179
|
+
const failover = vi.fn(async () => ({ rolledTo: null, rolled: [] }));
|
|
180
|
+
const out = await runFleetAutoFallback({
|
|
181
|
+
state: state('ken@x', ['ken@x']),
|
|
182
|
+
quotas: [qOk({ fiveHourUtilizationPct: 12, sevenDayUtilizationPct: 30 })],
|
|
183
|
+
failover,
|
|
184
|
+
triggerAgent: 'carrie',
|
|
185
|
+
now: NOW,
|
|
186
|
+
tz: 'UTC',
|
|
187
|
+
rateLimitTrigger: true,
|
|
188
|
+
});
|
|
189
|
+
expect(out.kind).toBe('all-blocked');
|
|
190
|
+
expect(failover).toHaveBeenCalledTimes(1);
|
|
191
|
+
});
|
|
192
|
+
|
|
121
193
|
it('idempotency: skips the swap WITHOUT calling failover when active probes healthy', async () => {
|
|
122
194
|
const failover = vi.fn();
|
|
123
195
|
const out = await runFleetAutoFallback({
|