switchroom 0.18.15 → 0.18.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +3 -0
- package/dist/auth-broker/index.js +432 -10
- package/dist/cli/notion-write-pretool.mjs +3 -0
- package/dist/cli/switchroom.js +50 -1
- package/dist/host-control/main.js +4 -1
- package/dist/vault/approvals/kernel-server.js +3 -0
- package/dist/vault/broker/server.js +3 -0
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +81 -139
- package/telegram-plugin/dist/gateway/gateway.js +386 -259
- package/telegram-plugin/draft-stream.ts +78 -3
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +3 -4
- package/telegram-plugin/gateway/effort-command.ts +9 -7
- package/telegram-plugin/gateway/gateway.ts +265 -220
- package/telegram-plugin/gateway/litellm-local-notice-wiring.ts +200 -0
- package/telegram-plugin/gateway/model-command.ts +96 -18
- package/telegram-plugin/gateway/pending-session-command.ts +10 -8
- package/telegram-plugin/gateway/session-model-file.ts +38 -172
- package/telegram-plugin/litellm-local-notice.ts +189 -0
- package/telegram-plugin/quota-watch.ts +16 -4
- package/telegram-plugin/runtime-metrics.ts +16 -0
- package/telegram-plugin/send-gate-degraded.test.ts +9 -7
- package/telegram-plugin/send-gate.ts +34 -4
- package/telegram-plugin/stream-controller.ts +143 -20
- package/telegram-plugin/stream-reply-handler.ts +12 -2
- package/telegram-plugin/tests/bot-api.harness.ts +7 -2
- package/telegram-plugin/tests/draft-stream.test.ts +110 -1
- package/telegram-plugin/tests/effort-command.test.ts +4 -4
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +2 -2
- package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +33 -19
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +47 -127
- package/telegram-plugin/tests/litellm-local-notice.test.ts +417 -0
- package/telegram-plugin/tests/model-command.test.ts +84 -1
- package/telegram-plugin/tests/quota-watch.test.ts +21 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +2 -2
- package/telegram-plugin/tests/session-model-file.test.ts +7 -155
- package/telegram-plugin/tests/stream-controller-send-gate.test.ts +521 -0
- package/telegram-plugin/tests/stream-reply-handler.test.ts +44 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +207 -0
- package/telegram-plugin/worker-activity-feed.ts +83 -8
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* litellm-local-notice.ts — debounced operator notice for LiteLLM-proxy-local
|
|
3
|
+
* 429s (pure module: state machine + text + config parsing, no IPC, no bot,
|
|
4
|
+
* no clock except the injected `now`).
|
|
5
|
+
*
|
|
6
|
+
* When an agent routes through the LiteLLM gateway and trips the proxy's OWN
|
|
7
|
+
* `tpm_limit`/`rpm_limit` limiter, `classify429Detail` (throttle-tier.ts)
|
|
8
|
+
* classifies the terminal 429 `litellm-local` and the gateway takes the calm
|
|
9
|
+
* path — no broker mark, no failover, no throttle tier (the request never
|
|
10
|
+
* reached Anthropic, so the condition says nothing about the account). But
|
|
11
|
+
* pre-notice the user-facing surface was the GENERIC "🚦 Rate limited" card,
|
|
12
|
+
* which reads like an Anthropic problem. This module owns the honest
|
|
13
|
+
* replacement:
|
|
14
|
+
*
|
|
15
|
+
* - ONE calm notice per agent per cooldown window (default 15 min,
|
|
16
|
+
* operator-tunable via channels.telegram.litellm_notice.window_ms in
|
|
17
|
+
* switchroom.yaml, projected into access.json by scaffold).
|
|
18
|
+
* - Further litellm-local 429s inside the window are counted SILENTLY;
|
|
19
|
+
* the first notice after the window expires carries "throttled N more
|
|
20
|
+
* times since the last notice".
|
|
21
|
+
* - A notice only ever fires on an actual throttle event — a quiet agent
|
|
22
|
+
* posts nothing (the state machine is evaluate-on-event, no timers).
|
|
23
|
+
*
|
|
24
|
+
* The copy must make clear this is the fleet token limiter (LiteLLM
|
|
25
|
+
* `tpm_limit`/`rpm_limit`), NOT an Anthropic account limit, and that the
|
|
26
|
+
* turn retries — no action needed.
|
|
27
|
+
*
|
|
28
|
+
* Side-effect sequencing lives in gateway/litellm-local-notice-wiring.ts.
|
|
29
|
+
* This module MUST NOT change classification, failover, or quota-ledger
|
|
30
|
+
* behavior — it only renders and debounces the notice.
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
import { escapeMarkdown } from './card-format.js'
|
|
34
|
+
import type { RateLimit429Classification } from './throttle-tier.js'
|
|
35
|
+
import type { RuntimeMetricEvent } from './runtime-metrics.js'
|
|
36
|
+
|
|
37
|
+
// ─── Cooldown window config ──────────────────────────────────────────────────
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Default per-agent notice cooldown. 15 minutes: long enough that a sustained
|
|
41
|
+
* burst produces a handful of notices per hour at most, short enough that the
|
|
42
|
+
* operator still sees the limiter working while it's working.
|
|
43
|
+
*/
|
|
44
|
+
export const LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT = 15 * 60_000
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Resolve the cooldown window from the raw access-file value
|
|
48
|
+
* (`access.litellmNoticeWindowMs`, projected by scaffold from
|
|
49
|
+
* channels.telegram.litellm_notice.window_ms). Any non-finite, non-positive,
|
|
50
|
+
* or non-number value falls back to the default — an operator typo must
|
|
51
|
+
* never produce a zero-width window (notice storm) or a NaN comparison
|
|
52
|
+
* (notices never fire again).
|
|
53
|
+
*/
|
|
54
|
+
export function parseLitellmNoticeWindowMs(raw: unknown): number {
|
|
55
|
+
if (typeof raw !== 'number') return LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT
|
|
56
|
+
if (!Number.isFinite(raw) || raw <= 0) return LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT
|
|
57
|
+
return raw
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// ─── Per-agent cooldown state machine ────────────────────────────────────────
|
|
61
|
+
|
|
62
|
+
export interface LitellmLocalNoticeState {
|
|
63
|
+
/** agent → unix ms of the last notice sent for it. */
|
|
64
|
+
lastSentAtMsByAgent: Record<string, number>
|
|
65
|
+
/** agent → litellm-local 429s counted silently since the last notice. */
|
|
66
|
+
suppressedCountByAgent: Record<string, number>
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function initialLitellmLocalNoticeState(): LitellmLocalNoticeState {
|
|
70
|
+
return { lastSentAtMsByAgent: {}, suppressedCountByAgent: {} }
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface LitellmLocalNoticeVerdict {
|
|
74
|
+
send: boolean
|
|
75
|
+
/** Silent throttle events since the previous notice (0 on the first). Only
|
|
76
|
+
* meaningful when `send` is true — the renderer adds the "N more times"
|
|
77
|
+
* line when > 0. */
|
|
78
|
+
suppressedSinceLastNotice: number
|
|
79
|
+
next: LitellmLocalNoticeState
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Evaluate one litellm-local 429 for `agent` at `now`.
|
|
84
|
+
*
|
|
85
|
+
* - First event ever (or ≥ windowMs since the last notice) → send; the
|
|
86
|
+
* verdict carries the silently-counted events since the previous notice
|
|
87
|
+
* and the counter resets.
|
|
88
|
+
* - Inside the window → suppress; the counter increments.
|
|
89
|
+
*
|
|
90
|
+
* Boundary is INCLUSIVE on expiry (elapsed === windowMs sends), matching
|
|
91
|
+
* `evaluateThrottleNotice` in throttle-tier.ts. Pure: returns the next
|
|
92
|
+
* state, never mutates `prev`.
|
|
93
|
+
*/
|
|
94
|
+
export function evaluateLitellmLocalNotice(
|
|
95
|
+
prev: LitellmLocalNoticeState,
|
|
96
|
+
agent: string,
|
|
97
|
+
now: number,
|
|
98
|
+
windowMs: number = LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT,
|
|
99
|
+
): LitellmLocalNoticeVerdict {
|
|
100
|
+
const last = prev.lastSentAtMsByAgent[agent]
|
|
101
|
+
if (last == null || now - last >= windowMs) {
|
|
102
|
+
return {
|
|
103
|
+
send: true,
|
|
104
|
+
suppressedSinceLastNotice: prev.suppressedCountByAgent[agent] ?? 0,
|
|
105
|
+
next: {
|
|
106
|
+
lastSentAtMsByAgent: { ...prev.lastSentAtMsByAgent, [agent]: now },
|
|
107
|
+
suppressedCountByAgent: { ...prev.suppressedCountByAgent, [agent]: 0 },
|
|
108
|
+
},
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return {
|
|
112
|
+
send: false,
|
|
113
|
+
suppressedSinceLastNotice: 0,
|
|
114
|
+
next: {
|
|
115
|
+
lastSentAtMsByAgent: prev.lastSentAtMsByAgent,
|
|
116
|
+
suppressedCountByAgent: {
|
|
117
|
+
...prev.suppressedCountByAgent,
|
|
118
|
+
[agent]: (prev.suppressedCountByAgent[agent] ?? 0) + 1,
|
|
119
|
+
},
|
|
120
|
+
},
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// ─── Notice rendering ────────────────────────────────────────────────────────
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* The ONE calm notice for a litellm-local 429. Markdown (not HTML) per the
|
|
128
|
+
* card-format.ts convention. Copy contract (operator spec):
|
|
129
|
+
* - names the fleet token limiter (LiteLLM `tpm_limit`/`rpm_limit`)
|
|
130
|
+
* - explicitly NOT an Anthropic account limit — nothing exhausted, no
|
|
131
|
+
* account touched
|
|
132
|
+
* - the turn retries automatically; no action needed
|
|
133
|
+
* - after a window with silent suppressions, says "throttled N more
|
|
134
|
+
* time(s) since the last notice"
|
|
135
|
+
*/
|
|
136
|
+
export function renderLitellmLocalNotice(opts: {
|
|
137
|
+
agent: string
|
|
138
|
+
/** Silent litellm-local 429s since the previous notice (0 on the first). */
|
|
139
|
+
suppressedSinceLastNotice: number
|
|
140
|
+
}): string {
|
|
141
|
+
const agent = escapeMarkdown(opts.agent)
|
|
142
|
+
const n = opts.suppressedSinceLastNotice
|
|
143
|
+
const lines = [
|
|
144
|
+
`🚦 **Fleet token limiter engaged** — **${agent}** hit the local LiteLLM proxy cap (\`tpm_limit\`/\`rpm_limit\`).`,
|
|
145
|
+
`This is switchroom's own fleet limiter smoothing a burst, not an Anthropic account limit — nothing is exhausted and no account was touched.`,
|
|
146
|
+
]
|
|
147
|
+
if (n > 0) {
|
|
148
|
+
lines.push(`Throttled ${n} more ${n === 1 ? 'time' : 'times'} since the last notice.`)
|
|
149
|
+
}
|
|
150
|
+
lines.push(`_The turn retries automatically — no action needed._`)
|
|
151
|
+
return lines.join('\n')
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// ─── Metric builder ──────────────────────────────────────────────────────────
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Build the `litellm_local_429_notice` runtime metric for one SENT notice.
|
|
158
|
+
* Distinct from `rate_limit_429_classified` (which fires on EVERY classified
|
|
159
|
+
* event): this one fires only when a notice actually posts, and carries how
|
|
160
|
+
* many events the window silently absorbed — the debounce's effectiveness in
|
|
161
|
+
* one number. Pure builder so the payload shape is unit-testable; the wiring
|
|
162
|
+
* emits the result via emitRuntimeMetric (PostHog + JSONL dual sink).
|
|
163
|
+
*/
|
|
164
|
+
export function buildLitellmLocalNoticeMetric(opts: {
|
|
165
|
+
agent: string
|
|
166
|
+
suppressedCount: number
|
|
167
|
+
windowMs: number
|
|
168
|
+
}): Extract<RuntimeMetricEvent, { kind: 'litellm_local_429_notice' }> {
|
|
169
|
+
return {
|
|
170
|
+
kind: 'litellm_local_429_notice',
|
|
171
|
+
agent: opts.agent,
|
|
172
|
+
suppressed_count: opts.suppressedCount,
|
|
173
|
+
window_ms: opts.windowMs,
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// ─── Classification guard ────────────────────────────────────────────────────
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* True only for the `litellm-local` classification. The wiring runs this
|
|
181
|
+
* guard so "a notice never fires for non-litellm-local classifications" is a
|
|
182
|
+
* deterministic mechanism, not caller discipline — account-scoped 429s keep
|
|
183
|
+
* the throttle tier, generic transients keep the calm rate-limited card.
|
|
184
|
+
*/
|
|
185
|
+
export function isLitellmLocalNoticeEligible(
|
|
186
|
+
classification: RateLimit429Classification | null | undefined,
|
|
187
|
+
): classification is 'litellm-local' {
|
|
188
|
+
return classification === 'litellm-local'
|
|
189
|
+
}
|
|
@@ -192,10 +192,14 @@ export interface FleetRollInfo {
|
|
|
192
192
|
/**
|
|
193
193
|
* Trigger attribution (#3031 PR 2): "soft-avoid" = proactive
|
|
194
194
|
* serving-preference roll off an account APPROACHING its limits;
|
|
195
|
-
* "hard-exhaustion" = the probe saw a genuine quota wall
|
|
196
|
-
*
|
|
195
|
+
* "hard-exhaustion" = the probe saw a genuine quota wall;
|
|
196
|
+
* "model-tier-wall" = the flagship-tier (7d_oi) canary saw the account
|
|
197
|
+
* walled on the premium tier (#3176). Absent on pre-PR-2 brokers —
|
|
198
|
+
* rendered as hard exhaustion.
|
|
197
199
|
*/
|
|
198
|
-
reason?: "soft-avoid" | "hard-exhaustion";
|
|
200
|
+
reason?: "soft-avoid" | "hard-exhaustion" | "model-tier-wall";
|
|
201
|
+
/** #3176 — the binding tier bucket, set only when reason is model-tier-wall. */
|
|
202
|
+
bucket?: string;
|
|
199
203
|
}
|
|
200
204
|
|
|
201
205
|
export type FleetRollAnnounceDecision =
|
|
@@ -241,9 +245,17 @@ export function buildFleetRollMessage(roll: FleetRollInfo, now: number): string
|
|
|
241
245
|
? ` (resets ${formatRelative(new Date(roll.exhausted_until), new Date(now))})`
|
|
242
246
|
: "";
|
|
243
247
|
const softAvoid = roll.reason === "soft-avoid";
|
|
248
|
+
// #3176 — a model-tier wall binds on the flagship (premium) 7d_oi bucket, not
|
|
249
|
+
// the 5h/7d windows (which read healthy — the whole point of the bug), so it
|
|
250
|
+
// has no window/pct to cite. Name the flagship tier explicitly and reassure
|
|
251
|
+
// that opus/haiku on the walled account are unaffected (the mark is
|
|
252
|
+
// tier-scoped), rather than rendering a generic, misleading "quota window".
|
|
253
|
+
const modelTierWall = roll.reason === "model-tier-wall";
|
|
244
254
|
const causeLine = softAvoid
|
|
245
255
|
? `Proactive switch — \`${codeSpanSafe(roll.from)}\` is approaching its limits (${winLabel}${pctPart})${resetPart}, so the fleet moved early instead of hitting the wall.`
|
|
246
|
-
:
|
|
256
|
+
: modelTierWall
|
|
257
|
+
? `Flagship (premium) tier weekly limit reached on \`${codeSpanSafe(roll.from)}\`${resetPart} — opus/haiku on that account are unaffected.`
|
|
258
|
+
: `${winLabel}${pctPart} on \`${codeSpanSafe(roll.from)}\`${resetPart}.`;
|
|
247
259
|
return [
|
|
248
260
|
`🔁 **Switched fleet to \`${codeSpanSafe(roll.to)}\`**`,
|
|
249
261
|
``,
|
|
@@ -195,6 +195,22 @@ export type RuntimeMetricEvent =
|
|
|
195
195
|
limit: number | null
|
|
196
196
|
current_usage: number | null
|
|
197
197
|
}
|
|
198
|
+
/**
|
|
199
|
+
* A litellm-local throttle NOTICE actually posted (litellm-local-notice.ts
|
|
200
|
+
* — the debounced "fleet token limiter engaged" message). Distinct from
|
|
201
|
+
* `rate_limit_429_classified`, which fires on EVERY classified 429: this
|
|
202
|
+
* fires once per sent notice, and `suppressed_count` is how many
|
|
203
|
+
* litellm-local 429s the cooldown window silently absorbed since the
|
|
204
|
+
* previous notice (0 on the first) — the debounce's effectiveness in one
|
|
205
|
+
* number. `window_ms` is the resolved cooldown window (default 15 min;
|
|
206
|
+
* channels.telegram.litellm_notice.window_ms).
|
|
207
|
+
*/
|
|
208
|
+
| {
|
|
209
|
+
kind: 'litellm_local_429_notice'
|
|
210
|
+
agent: string
|
|
211
|
+
suppressed_count: number
|
|
212
|
+
window_ms: number
|
|
213
|
+
}
|
|
198
214
|
|
|
199
215
|
/**
|
|
200
216
|
* The JSONL sink lives under the runtime state dir so it's per-agent
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, expect, it } from 'vitest'
|
|
2
|
-
import { createSendGate, type Clock } from './send-gate.js'
|
|
2
|
+
import { createSendGate, SEND_GATE_SHED, type Clock } from './send-gate.js'
|
|
3
3
|
import { isFloodWaitActiveError } from './retry-api-call.js'
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -73,8 +73,10 @@ describe('send-gate PR2: cosmetic shedding', () => {
|
|
|
73
73
|
|
|
74
74
|
const res = await gate.gate(fn('cosmetic'), { priorityClass: 'cosmetic' })
|
|
75
75
|
|
|
76
|
-
// Shed resolves
|
|
77
|
-
|
|
76
|
+
// Shed resolves the distinguishable sentinel (#3110 F1 — never a bare
|
|
77
|
+
// undefined, which is reserved for benign no-op drops); fn never ran;
|
|
78
|
+
// the shed counter moved.
|
|
79
|
+
expect(res).toBe(SEND_GATE_SHED)
|
|
78
80
|
expect(calls).toHaveLength(0)
|
|
79
81
|
expect(gate.stats().global.shed).toBe(1)
|
|
80
82
|
expect(gate.stats().global.sent).toBe(0)
|
|
@@ -95,7 +97,7 @@ describe('send-gate PR2: cosmetic shedding', () => {
|
|
|
95
97
|
const shed = await gate.gate(fn('b'), { chat_id: '5', priorityClass: 'cosmetic' })
|
|
96
98
|
|
|
97
99
|
expect(calls.map((c) => c.label)).toEqual(['a'])
|
|
98
|
-
expect(shed).
|
|
100
|
+
expect(shed).toBe(SEND_GATE_SHED)
|
|
99
101
|
expect(gate.stats().global.shed).toBe(1)
|
|
100
102
|
})
|
|
101
103
|
|
|
@@ -128,7 +130,7 @@ describe('send-gate PR2: cosmetic shedding', () => {
|
|
|
128
130
|
priorityClass: 'cosmetic',
|
|
129
131
|
})
|
|
130
132
|
|
|
131
|
-
expect(res).
|
|
133
|
+
expect(res).toBe(SEND_GATE_SHED)
|
|
132
134
|
expect(calls).toHaveLength(0)
|
|
133
135
|
expect(gate.stats().global.shed).toBe(1)
|
|
134
136
|
})
|
|
@@ -366,7 +368,7 @@ describe('send-gate PR2: H1 cross-chat message_id isolation', () => {
|
|
|
366
368
|
priorityClass: 'cosmetic',
|
|
367
369
|
})
|
|
368
370
|
|
|
369
|
-
expect(rA).
|
|
371
|
+
expect(rA).toBe(SEND_GATE_SHED) // A shed
|
|
370
372
|
expect(rB).toBe('B-edit') // B sent
|
|
371
373
|
expect(calls.map((c) => c.label)).toEqual(['B-edit'])
|
|
372
374
|
expect(gate.stats().global.shed).toBe(1)
|
|
@@ -568,7 +570,7 @@ describe('send-gate PR2: opening a window from a 429, and persistence hook', ()
|
|
|
568
570
|
expect(scopes).toEqual(['chat:7', 'global', 'group:7', 'msg-edit:7:3'])
|
|
569
571
|
// After the window opens, a later cosmetic on the same chat sheds.
|
|
570
572
|
const shed = await gate.gate(async () => 'x', { chat_id: '7', priorityClass: 'cosmetic' })
|
|
571
|
-
expect(shed).
|
|
573
|
+
expect(shed).toBe(SEND_GATE_SHED)
|
|
572
574
|
expect(gate.stats().global.shed).toBe(1)
|
|
573
575
|
})
|
|
574
576
|
})
|
|
@@ -118,6 +118,32 @@ export type PriorityClass = 'critical' | 'useful' | 'cosmetic'
|
|
|
118
118
|
*/
|
|
119
119
|
export const UNTAGGED_SEND_CLASS: PriorityClass = 'critical'
|
|
120
120
|
|
|
121
|
+
/**
|
|
122
|
+
* Distinguishable resolution value for a SHED call (#3110 review F1).
|
|
123
|
+
*
|
|
124
|
+
* `undefined` was overloaded three ways at the robustApiCall seam: a gate
|
|
125
|
+
* SHED (cosmetic call dropped under pressure — did NOT land), a gate no-op
|
|
126
|
+
* drop (identical payload already on screen — benign), and the retry
|
|
127
|
+
* policy's swallowed benign 400s ("message is not modified" — also benign).
|
|
128
|
+
* A caller that needs to know "did my edit land?" (the draft stream's
|
|
129
|
+
* shed-honesty handling in stream-controller.ts) could not tell these
|
|
130
|
+
* apart, and treating every `undefined` as a shed froze multi-piece
|
|
131
|
+
* streams on perfectly healthy chats.
|
|
132
|
+
*
|
|
133
|
+
* A shed now resolves THIS sentinel instead. `Symbol.for` keys it in the
|
|
134
|
+
* global symbol registry so duplicated module instances (server + gateway
|
|
135
|
+
* builds) agree on identity. The no-op drop and coalesced-revert drop keep
|
|
136
|
+
* resolving `undefined` — for those the payload IS on screen. Callers that
|
|
137
|
+
* ignore the result (reactions, typing, fire-and-forget card edits) are
|
|
138
|
+
* unaffected either way.
|
|
139
|
+
*/
|
|
140
|
+
export const SEND_GATE_SHED: unique symbol = Symbol.for('switchroom.send-gate.shed')
|
|
141
|
+
|
|
142
|
+
/** True when a gate result is the {@link SEND_GATE_SHED} sentinel. */
|
|
143
|
+
export function isSendGateShed(value: unknown): value is typeof SEND_GATE_SHED {
|
|
144
|
+
return value === SEND_GATE_SHED
|
|
145
|
+
}
|
|
146
|
+
|
|
121
147
|
/**
|
|
122
148
|
* Extra metadata a call site can attach so the gate can key the right buckets.
|
|
123
149
|
* All fields optional — a call with none still passes the global bucket. These
|
|
@@ -159,8 +185,9 @@ export interface BucketCounters {
|
|
|
159
185
|
dropped: number
|
|
160
186
|
/**
|
|
161
187
|
* Cosmetic calls shed under pressure — no token free OR a flood window open
|
|
162
|
-
* (part3-design §2). A shed resolves
|
|
163
|
-
*
|
|
188
|
+
* (part3-design §2). A shed resolves the SEND_GATE_SHED sentinel (F1 —
|
|
189
|
+
* distinguishable from a benign no-op drop's `undefined`); the next send
|
|
190
|
+
* carries full state.
|
|
164
191
|
*/
|
|
165
192
|
shed: number
|
|
166
193
|
/** Useful calls dropped because they exceeded their queue TTL (part3-design §2). */
|
|
@@ -913,7 +940,9 @@ export function createSendGate(config: SendGateConfig): SendGate {
|
|
|
913
940
|
const msgWait = state.suppressedUntilMs > now ? state.suppressedUntilMs - now : 0
|
|
914
941
|
if (wait > 0 || msgWait > 0) {
|
|
915
942
|
counters.shed++
|
|
916
|
-
|
|
943
|
+
// Distinguishable from the no-op drop below (undefined): a shed did
|
|
944
|
+
// NOT land, and edit-driving callers must be able to tell (F1).
|
|
945
|
+
return Promise.resolve(SEND_GATE_SHED as unknown as T)
|
|
917
946
|
}
|
|
918
947
|
}
|
|
919
948
|
|
|
@@ -991,7 +1020,8 @@ export function createSendGate(config: SendGateConfig): SendGate {
|
|
|
991
1020
|
const outcome = await admitPriority(bucketsFor(opts), priority)
|
|
992
1021
|
if (outcome.result === 'shed') {
|
|
993
1022
|
counters.shed++
|
|
994
|
-
|
|
1023
|
+
// See SEND_GATE_SHED: distinguishable "did not land" resolution (F1).
|
|
1024
|
+
return SEND_GATE_SHED as unknown as T
|
|
995
1025
|
}
|
|
996
1026
|
if (outcome.result === 'expired') {
|
|
997
1027
|
counters.expired++
|
|
@@ -19,9 +19,14 @@
|
|
|
19
19
|
* entire server.ts top-level initialization.
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
|
-
import {
|
|
22
|
+
import {
|
|
23
|
+
createDraftStream,
|
|
24
|
+
makeDraftEditShedError,
|
|
25
|
+
type DraftStreamHandle,
|
|
26
|
+
} from './draft-stream.js'
|
|
23
27
|
import { richMessage, isParseEntitiesError } from './rich-send.js'
|
|
24
28
|
import { renderOutboundChunks } from './render/rich-render.js'
|
|
29
|
+
import { isSendGateShed, type SendGateOpts } from './send-gate.js'
|
|
25
30
|
|
|
26
31
|
/**
|
|
27
32
|
* Minimal bot.api surface the controller needs. Real callers pass grammy's
|
|
@@ -77,9 +82,21 @@ export interface StreamSendOpts {
|
|
|
77
82
|
disable_notification?: boolean
|
|
78
83
|
}
|
|
79
84
|
|
|
85
|
+
/**
|
|
86
|
+
* Options a stream-controller call passes to its retry wrapper. A structural
|
|
87
|
+
* superset of `SendGateOpts` (send-gate.ts) plus the retry policy's own
|
|
88
|
+
* `threadId`, and assignable to `RetryCallOpts` (retry-api-call.ts) — so the
|
|
89
|
+
* production wrapper (`robustApiCall` = send gate over `createRetryApiCall`)
|
|
90
|
+
* receives `messageId` / `editPayload` / `priorityClass` and the gate's
|
|
91
|
+
* per-message edit floor, last-write-wins coalescing, no-op skip and
|
|
92
|
+
* cosmetic shedding govern the draft/answer stream (#3110; part3-design §4
|
|
93
|
+
* names rapid same-message editMessageText as the #1 flood-ban trigger).
|
|
94
|
+
*/
|
|
95
|
+
export type RetryPolicyOpts = SendGateOpts & { threadId?: number }
|
|
96
|
+
|
|
80
97
|
export type RetryPolicy = <T>(
|
|
81
98
|
fn: () => Promise<T>,
|
|
82
|
-
opts?:
|
|
99
|
+
opts?: RetryPolicyOpts,
|
|
83
100
|
) => Promise<T>
|
|
84
101
|
|
|
85
102
|
export interface StreamControllerConfig {
|
|
@@ -251,6 +268,70 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
|
|
|
251
268
|
return bot.api.editMessageText(chatId, id, richMessage(piece.text), opts)
|
|
252
269
|
}
|
|
253
270
|
|
|
271
|
+
// ---- Send-gate wiring for the edit path (#3110) -------------------------
|
|
272
|
+
//
|
|
273
|
+
// Every edit call passes `messageId` / `editPayload` / `priorityClass`
|
|
274
|
+
// through the retry wrapper so the send gate's per-message edit floor
|
|
275
|
+
// (>=1.5s), last-write-wins coalescing, and no-op skip govern the draft
|
|
276
|
+
// stream — previously these edits carried only `{ threadId, chat_id }`,
|
|
277
|
+
// so the gate treated them as ordinary sends and the stream's own 400 ms
|
|
278
|
+
// DM throttle drove same-message editMessageText well under the floor
|
|
279
|
+
// (the #1 documented flood-ban trigger, part3-design §4; production ban
|
|
280
|
+
// 2026-07-12 on #3110). The local per-surface throttle stays as a cheap
|
|
281
|
+
// pre-filter; the gate is the authority.
|
|
282
|
+
//
|
|
283
|
+
// Priority classes: intermediate draft edits are `cosmetic` (part3-design
|
|
284
|
+
// §2 lists "stream updates" there) — shed under pressure / an open flood
|
|
285
|
+
// window; the next flush carries full state. The FINALIZE flush (the edit
|
|
286
|
+
// that renders the completed answer) is `critical`, mirroring the reply
|
|
287
|
+
// path's preview-finalize convention (gateway.ts editPreview): never shed,
|
|
288
|
+
// waits out a short window, fails fast with a structured FLOOD_WAIT_ACTIVE
|
|
289
|
+
// on a long one. SENDS stay untagged (the gate admits untagged non-edit
|
|
290
|
+
// sends as `critical`) — a shed send would resolve the gate's shed
|
|
291
|
+
// sentinel instead of `{ message_id }` and break message-id capture, and
|
|
292
|
+
// the anchor/tail sends ARE the answer surface.
|
|
293
|
+
//
|
|
294
|
+
// `handleRef.isFinal()` is true from the moment finalize() is entered
|
|
295
|
+
// (draft-stream sets `final` before its last flush), so the closures below
|
|
296
|
+
// classify exactly the finalize flush — and anything after it — as
|
|
297
|
+
// critical. The ref is assigned right after createDraftStream returns,
|
|
298
|
+
// before any closure can run (closures only fire from update/finalize).
|
|
299
|
+
//
|
|
300
|
+
// KNOWN MISSED BENEFIT (review F5, deliberate): the gate's last-write-wins
|
|
301
|
+
// coalescing never engages for THIS surface, because draft-stream
|
|
302
|
+
// serializes its flushes — it awaits each edit before issuing the next, so
|
|
303
|
+
// at most one edit per message is ever inside the gate. Consequence: a
|
|
304
|
+
// stale draft sleeping on the gate's floor still lands (one API call the
|
|
305
|
+
// coalescer would have replaced) before the newer snapshot, and a finalize
|
|
306
|
+
// issued mid-floor can trail by up to ~2x editFloorMs (floor wait for the
|
|
307
|
+
// stale draft, then floor wait for the final). Correctness is unaffected —
|
|
308
|
+
// the latest state always lands, floor-paced — and the gate coalescing
|
|
309
|
+
// remains live protection for CONCURRENT writers to one message (e.g. a
|
|
310
|
+
// re-attached #626 controller racing its predecessor).
|
|
311
|
+
let handleRef: DraftStreamHandle | null = null
|
|
312
|
+
const editGateOpts = (id: number, payload: unknown): RetryPolicyOpts => ({
|
|
313
|
+
threadId,
|
|
314
|
+
chat_id: chatId,
|
|
315
|
+
messageId: id,
|
|
316
|
+
editPayload: payload,
|
|
317
|
+
priorityClass: handleRef?.isFinal() === true ? 'critical' : 'cosmetic',
|
|
318
|
+
})
|
|
319
|
+
// The rendered payload the gate hashes for the no-op skip / coalescing —
|
|
320
|
+
// exactly what goes over the wire (rich wrapper included), so a plain
|
|
321
|
+
// fallback of the same text never hashes equal to its rich form.
|
|
322
|
+
const piecePayload = (piece: { text: string; rich: boolean }): unknown =>
|
|
323
|
+
piece.rich ? richMessage(piece.text) : piece.text
|
|
324
|
+
// Shed detection (#3110 review F1): keyed EXACTLY off the gate's
|
|
325
|
+
// SEND_GATE_SHED sentinel — never off `undefined`, which is overloaded
|
|
326
|
+
// (gate no-op drop; robustApiCall's swallowed benign 400s like "message is
|
|
327
|
+
// not modified"). Those benign cases mean the payload is ALREADY on screen
|
|
328
|
+
// and are treated as delivered, exactly as before this wiring existed. A
|
|
329
|
+
// true shed means the edit did NOT land: draft-stream must not record the
|
|
330
|
+
// snapshot as on-screen (its dedupe would skip a later flush of the same
|
|
331
|
+
// text — the completed answer would never render), so the edit closure
|
|
332
|
+
// throws the marker error draft-stream recognizes and recovers from
|
|
333
|
+
// (`makeDraftEditShedError` → snapshot preserved for finalize, review F2).
|
|
334
|
+
|
|
254
335
|
// Overflow-tail bookkeeping, shared across the send + edit closures for the
|
|
255
336
|
// whole stream lifetime. A body large enough to split into several
|
|
256
337
|
// wire-cap pieces anchors on piece[0] (edited in place by draft-stream) and
|
|
@@ -269,12 +350,25 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
|
|
|
269
350
|
// text) thereafter. A non-parse failure is logged as a partial-delivery
|
|
270
351
|
// warning and swallowed so the remaining tail pieces still get a chance to
|
|
271
352
|
// land — never a silent drop, never an abort of pieces K..N (concern C1).
|
|
272
|
-
|
|
353
|
+
//
|
|
354
|
+
// Returns TRUE when this piece's edit was SHED by the send gate (did not
|
|
355
|
+
// land): the caller must then report the whole flush as shed so
|
|
356
|
+
// draft-stream does not record the full body as delivered while a tail is
|
|
357
|
+
// stale (review F3). The piece's own tailLastText stays stale too, so the
|
|
358
|
+
// recovery flush re-attempts exactly the shed piece.
|
|
359
|
+
const upsertTail = async (
|
|
360
|
+
ti: number,
|
|
361
|
+
piece: { text: string; rich: boolean },
|
|
362
|
+
): Promise<boolean> => {
|
|
273
363
|
const existingId = tailIds[ti]
|
|
274
364
|
if (existingId != null) {
|
|
275
|
-
if (tailLastText[ti] === piece.text) return // unchanged — skip the API call
|
|
365
|
+
if (tailLastText[ti] === piece.text) return false // unchanged — skip the API call
|
|
276
366
|
try {
|
|
277
|
-
await retry(
|
|
367
|
+
const res = await retry(
|
|
368
|
+
() => editPiece(existingId, piece, baseOpts),
|
|
369
|
+
editGateOpts(existingId, piecePayload(piece)),
|
|
370
|
+
)
|
|
371
|
+
if (isSendGateShed(res)) return true // shed — stale; retried by the recovery flush
|
|
278
372
|
tailLastText[ti] = piece.text
|
|
279
373
|
onEdit?.(existingId, piece.text.length)
|
|
280
374
|
} catch (err) {
|
|
@@ -282,10 +376,11 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
|
|
|
282
376
|
warn?.(
|
|
283
377
|
`stream-controller: tail-piece #${ti + 1} edit parse-entities rejected — retrying same id=${existingId} as plain text (${err instanceof Error ? err.message : String(err)})`,
|
|
284
378
|
)
|
|
285
|
-
await retry(
|
|
379
|
+
const res = await retry(
|
|
286
380
|
() => bot.api.editMessageText(chatId, existingId, piece.text, baseOpts),
|
|
287
|
-
|
|
381
|
+
editGateOpts(existingId, piece.text),
|
|
288
382
|
)
|
|
383
|
+
if (isSendGateShed(res)) return true // shed — stale; retried by the recovery flush
|
|
289
384
|
tailLastText[ti] = piece.text
|
|
290
385
|
onEdit?.(existingId, piece.text.length)
|
|
291
386
|
} else {
|
|
@@ -296,7 +391,7 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
|
|
|
296
391
|
)
|
|
297
392
|
}
|
|
298
393
|
}
|
|
299
|
-
return
|
|
394
|
+
return false
|
|
300
395
|
}
|
|
301
396
|
// First emission of this tail piece → a fresh follow-up message.
|
|
302
397
|
try {
|
|
@@ -324,9 +419,12 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
|
|
|
324
419
|
)
|
|
325
420
|
}
|
|
326
421
|
}
|
|
422
|
+
// First-emission SENDS are untagged (critical) — the gate never sheds
|
|
423
|
+
// them, so this path can only land or fail (handled above).
|
|
424
|
+
return false
|
|
327
425
|
}
|
|
328
426
|
|
|
329
|
-
|
|
427
|
+
const handle = createDraftStream(
|
|
330
428
|
async (text) => {
|
|
331
429
|
// Render → 1+ cap-respecting pieces. The FIRST piece's message_id anchors
|
|
332
430
|
// the stream (later edits target it); any overflow pieces are parked as
|
|
@@ -378,14 +476,26 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
|
|
|
378
476
|
async (id, text) => {
|
|
379
477
|
const pieces = renderPieces(text)
|
|
380
478
|
const head = pieces[0]
|
|
479
|
+
// Whether any piece of THIS flush was shed by the gate (did not land).
|
|
480
|
+
// Decided at the very end — AFTER the tail loop — so a benign anchor
|
|
481
|
+
// outcome (or even a shed anchor) never starves the tail pieces of
|
|
482
|
+
// their own upsert attempt (review F1: an anchor whose payload stopped
|
|
483
|
+
// changing resolves benignly every flush while only the tail grows).
|
|
484
|
+
let anchorShed = false
|
|
381
485
|
// Edit the anchor message in place with the FIRST piece.
|
|
382
486
|
try {
|
|
383
|
-
await retry(
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
487
|
+
const res = await retry(() => editPiece(id, head, baseOpts), editGateOpts(id, piecePayload(head)))
|
|
488
|
+
if (isSendGateShed(res)) {
|
|
489
|
+
// Shed by the gate (cosmetic under pressure / an open flood
|
|
490
|
+
// window) — did NOT land. Benign `undefined` resolutions (gate
|
|
491
|
+
// no-op drop, robustApiCall's swallowed "message is not modified")
|
|
492
|
+
// deliberately do NOT take this branch: the payload is already on
|
|
493
|
+
// screen and the flush proceeds as delivered.
|
|
494
|
+
anchorShed = true
|
|
495
|
+
} else {
|
|
496
|
+
// C2: report the actual head-piece length, not the full body length.
|
|
497
|
+
onEdit?.(id, head.text.length)
|
|
498
|
+
}
|
|
389
499
|
} catch (err) {
|
|
390
500
|
if (!literalText && head.rich && isParseEntitiesError(err)) {
|
|
391
501
|
// Edit rejected because the markdown couldn't be parsed — DO NOT
|
|
@@ -400,11 +510,12 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
|
|
|
400
510
|
// contract). For a single-piece stream (common case) this is the
|
|
401
511
|
// whole body; a rare oversize split edits the head piece's body.
|
|
402
512
|
const fallbackBody = pieces.length === 1 ? text : head.text
|
|
403
|
-
await retry(
|
|
513
|
+
const res = await retry(
|
|
404
514
|
() => bot.api.editMessageText(chatId, id, fallbackBody, baseOpts),
|
|
405
|
-
|
|
515
|
+
editGateOpts(id, fallbackBody),
|
|
406
516
|
)
|
|
407
|
-
|
|
517
|
+
if (isSendGateShed(res)) anchorShed = true
|
|
518
|
+
else onEdit?.(id, head.text.length)
|
|
408
519
|
} else {
|
|
409
520
|
throw err
|
|
410
521
|
}
|
|
@@ -414,10 +525,20 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
|
|
|
414
525
|
// that only just came into existence) — we never re-send tails already
|
|
415
526
|
// emitted on a prior flush. This is the fix for the duplicate-flood
|
|
416
527
|
// blocker: the previous code re-sent pieces[1..n] as brand-new messages
|
|
417
|
-
// on each throttled edit tick.
|
|
528
|
+
// on each throttled edit tick. Runs BEFORE the shed decision below so a
|
|
529
|
+
// shed (or benign) anchor never starves the tails (review F1).
|
|
530
|
+
let anyTailShed = false
|
|
418
531
|
for (let pi = 1; pi < pieces.length; pi++) {
|
|
419
|
-
await upsertTail(pi - 1, pieces[pi])
|
|
532
|
+
if (await upsertTail(pi - 1, pieces[pi])) anyTailShed = true
|
|
420
533
|
}
|
|
534
|
+
// Review F3: ANY shed piece — anchor or tail — means this flush did not
|
|
535
|
+
// fully land. Throw the marker error so draft-stream does not record
|
|
536
|
+
// the body as delivered (its dedupe would freeze the shed piece
|
|
537
|
+
// forever) and instead preserves the snapshot for the finalize
|
|
538
|
+
// re-flush. Pieces that DID land are unaffected on that re-flush: the
|
|
539
|
+
// gate's no-op skip drops their identical payloads before the API, and
|
|
540
|
+
// landed tails short-circuit on tailLastText.
|
|
541
|
+
if (anchorShed || anyTailShed) throw makeDraftEditShedError(id)
|
|
421
542
|
},
|
|
422
543
|
{
|
|
423
544
|
...(throttleMs != null ? { throttleMs } : {}),
|
|
@@ -429,4 +550,6 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
|
|
|
429
550
|
chatId,
|
|
430
551
|
},
|
|
431
552
|
)
|
|
553
|
+
handleRef = handle
|
|
554
|
+
return handle
|
|
432
555
|
}
|
|
@@ -516,10 +516,20 @@ export async function handleStreamReply(
|
|
|
516
516
|
state.activeDraftStreams.set(sKey, stream)
|
|
517
517
|
}
|
|
518
518
|
|
|
519
|
-
|
|
519
|
+
if (!done) {
|
|
520
|
+
// Intermediate snapshot — an ordinary throttled draft update.
|
|
521
|
+
await stream.update(effectiveText)
|
|
522
|
+
}
|
|
520
523
|
|
|
521
524
|
if (done) {
|
|
522
|
-
|
|
525
|
+
// #3110: route the FINAL text through finalize(text) so the flush that
|
|
526
|
+
// renders the completed answer runs with the stream already final —
|
|
527
|
+
// stream-controller then classifies that edit `critical` for the send
|
|
528
|
+
// gate (never shed; fails fast with a structured FLOOD_WAIT_ACTIVE on a
|
|
529
|
+
// long flood window) while intermediate draft edits stay `cosmetic`
|
|
530
|
+
// (sheddable). The previous update()-then-finalize() pair flushed the
|
|
531
|
+
// final text inside update(), i.e. as an ordinary sheddable draft edit.
|
|
532
|
+
await stream.finalize(effectiveText)
|
|
523
533
|
state.activeDraftStreams.delete(sKey)
|
|
524
534
|
// #1713: stream_reply done=true is a NON-EVENT for the status
|
|
525
535
|
// reaction. The reaction reflects current turn activity, not
|