switchroom 0.18.15 → 0.18.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/dist/agent-scheduler/index.js +3 -0
  2. package/dist/auth-broker/index.js +432 -10
  3. package/dist/cli/notion-write-pretool.mjs +3 -0
  4. package/dist/cli/switchroom.js +50 -1
  5. package/dist/host-control/main.js +4 -1
  6. package/dist/vault/approvals/kernel-server.js +3 -0
  7. package/dist/vault/broker/server.js +3 -0
  8. package/package.json +1 -1
  9. package/profiles/_base/start.sh.hbs +81 -139
  10. package/telegram-plugin/dist/gateway/gateway.js +386 -259
  11. package/telegram-plugin/draft-stream.ts +78 -3
  12. package/telegram-plugin/gateway/bridge-dead-watchdog.ts +3 -4
  13. package/telegram-plugin/gateway/effort-command.ts +9 -7
  14. package/telegram-plugin/gateway/gateway.ts +265 -220
  15. package/telegram-plugin/gateway/litellm-local-notice-wiring.ts +200 -0
  16. package/telegram-plugin/gateway/model-command.ts +96 -18
  17. package/telegram-plugin/gateway/pending-session-command.ts +10 -8
  18. package/telegram-plugin/gateway/session-model-file.ts +38 -172
  19. package/telegram-plugin/litellm-local-notice.ts +189 -0
  20. package/telegram-plugin/quota-watch.ts +16 -4
  21. package/telegram-plugin/runtime-metrics.ts +16 -0
  22. package/telegram-plugin/send-gate-degraded.test.ts +9 -7
  23. package/telegram-plugin/send-gate.ts +34 -4
  24. package/telegram-plugin/stream-controller.ts +143 -20
  25. package/telegram-plugin/stream-reply-handler.ts +12 -2
  26. package/telegram-plugin/tests/bot-api.harness.ts +7 -2
  27. package/telegram-plugin/tests/draft-stream.test.ts +110 -1
  28. package/telegram-plugin/tests/effort-command.test.ts +4 -4
  29. package/telegram-plugin/tests/flood-windows-persistence.test.ts +2 -2
  30. package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +33 -19
  31. package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +47 -127
  32. package/telegram-plugin/tests/litellm-local-notice.test.ts +417 -0
  33. package/telegram-plugin/tests/model-command.test.ts +84 -1
  34. package/telegram-plugin/tests/quota-watch.test.ts +21 -0
  35. package/telegram-plugin/tests/reaction-gate-routing.test.ts +2 -2
  36. package/telegram-plugin/tests/session-model-file.test.ts +7 -155
  37. package/telegram-plugin/tests/stream-controller-send-gate.test.ts +521 -0
  38. package/telegram-plugin/tests/stream-reply-handler.test.ts +44 -0
  39. package/telegram-plugin/tests/worker-activity-feed.test.ts +207 -0
  40. package/telegram-plugin/worker-activity-feed.ts +83 -8
@@ -0,0 +1,189 @@
1
+ /**
2
+ * litellm-local-notice.ts — debounced operator notice for LiteLLM-proxy-local
3
+ * 429s (pure module: state machine + text + config parsing, no IPC, no bot,
4
+ * no clock except the injected `now`).
5
+ *
6
+ * When an agent routes through the LiteLLM gateway and trips the proxy's OWN
7
+ * `tpm_limit`/`rpm_limit` limiter, `classify429Detail` (throttle-tier.ts)
8
+ * classifies the terminal 429 `litellm-local` and the gateway takes the calm
9
+ * path — no broker mark, no failover, no throttle tier (the request never
10
+ * reached Anthropic, so the condition says nothing about the account). But
11
+ * pre-notice the user-facing surface was the GENERIC "🚦 Rate limited" card,
12
+ * which reads like an Anthropic problem. This module owns the honest
13
+ * replacement:
14
+ *
15
+ * - ONE calm notice per agent per cooldown window (default 15 min,
16
+ * operator-tunable via channels.telegram.litellm_notice.window_ms in
17
+ * switchroom.yaml, projected into access.json by scaffold).
18
+ * - Further litellm-local 429s inside the window are counted SILENTLY;
19
+ * the first notice after the window expires carries "throttled N more
20
+ * times since the last notice".
21
+ * - A notice only ever fires on an actual throttle event — a quiet agent
22
+ * posts nothing (the state machine is evaluate-on-event, no timers).
23
+ *
24
+ * The copy must make clear this is the fleet token limiter (LiteLLM
25
+ * `tpm_limit`/`rpm_limit`), NOT an Anthropic account limit, and that the
26
+ * turn retries — no action needed.
27
+ *
28
+ * Side-effect sequencing lives in gateway/litellm-local-notice-wiring.ts.
29
+ * This module MUST NOT change classification, failover, or quota-ledger
30
+ * behavior — it only renders and debounces the notice.
31
+ */
32
+
33
+ import { escapeMarkdown } from './card-format.js'
34
+ import type { RateLimit429Classification } from './throttle-tier.js'
35
+ import type { RuntimeMetricEvent } from './runtime-metrics.js'
36
+
37
+ // ─── Cooldown window config ──────────────────────────────────────────────────
38
+
39
+ /**
40
+ * Default per-agent notice cooldown. 15 minutes: long enough that a sustained
41
+ * burst produces a handful of notices per hour at most, short enough that the
42
+ * operator still sees the limiter working while it's working.
43
+ */
44
+ export const LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT = 15 * 60_000
45
+
46
+ /**
47
+ * Resolve the cooldown window from the raw access-file value
48
+ * (`access.litellmNoticeWindowMs`, projected by scaffold from
49
+ * channels.telegram.litellm_notice.window_ms). Any non-finite, non-positive,
50
+ * or non-number value falls back to the default — an operator typo must
51
+ * never produce a zero-width window (notice storm) or a NaN comparison
52
+ * (notices never fire again).
53
+ */
54
+ export function parseLitellmNoticeWindowMs(raw: unknown): number {
55
+ if (typeof raw !== 'number') return LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT
56
+ if (!Number.isFinite(raw) || raw <= 0) return LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT
57
+ return raw
58
+ }
59
+
60
+ // ─── Per-agent cooldown state machine ────────────────────────────────────────
61
+
62
+ export interface LitellmLocalNoticeState {
63
+ /** agent → unix ms of the last notice sent for it. */
64
+ lastSentAtMsByAgent: Record<string, number>
65
+ /** agent → litellm-local 429s counted silently since the last notice. */
66
+ suppressedCountByAgent: Record<string, number>
67
+ }
68
+
69
+ export function initialLitellmLocalNoticeState(): LitellmLocalNoticeState {
70
+ return { lastSentAtMsByAgent: {}, suppressedCountByAgent: {} }
71
+ }
72
+
73
+ export interface LitellmLocalNoticeVerdict {
74
+ send: boolean
75
+ /** Silent throttle events since the previous notice (0 on the first). Only
76
+ * meaningful when `send` is true — the renderer adds the "N more times"
77
+ * line when > 0. */
78
+ suppressedSinceLastNotice: number
79
+ next: LitellmLocalNoticeState
80
+ }
81
+
82
+ /**
83
+ * Evaluate one litellm-local 429 for `agent` at `now`.
84
+ *
85
+ * - First event ever (or ≥ windowMs since the last notice) → send; the
86
+ * verdict carries the silently-counted events since the previous notice
87
+ * and the counter resets.
88
+ * - Inside the window → suppress; the counter increments.
89
+ *
90
+ * Boundary is INCLUSIVE on expiry (elapsed === windowMs sends), matching
91
+ * `evaluateThrottleNotice` in throttle-tier.ts. Pure: returns the next
92
+ * state, never mutates `prev`.
93
+ */
94
+ export function evaluateLitellmLocalNotice(
95
+ prev: LitellmLocalNoticeState,
96
+ agent: string,
97
+ now: number,
98
+ windowMs: number = LITELLM_LOCAL_NOTICE_WINDOW_MS_DEFAULT,
99
+ ): LitellmLocalNoticeVerdict {
100
+ const last = prev.lastSentAtMsByAgent[agent]
101
+ if (last == null || now - last >= windowMs) {
102
+ return {
103
+ send: true,
104
+ suppressedSinceLastNotice: prev.suppressedCountByAgent[agent] ?? 0,
105
+ next: {
106
+ lastSentAtMsByAgent: { ...prev.lastSentAtMsByAgent, [agent]: now },
107
+ suppressedCountByAgent: { ...prev.suppressedCountByAgent, [agent]: 0 },
108
+ },
109
+ }
110
+ }
111
+ return {
112
+ send: false,
113
+ suppressedSinceLastNotice: 0,
114
+ next: {
115
+ lastSentAtMsByAgent: prev.lastSentAtMsByAgent,
116
+ suppressedCountByAgent: {
117
+ ...prev.suppressedCountByAgent,
118
+ [agent]: (prev.suppressedCountByAgent[agent] ?? 0) + 1,
119
+ },
120
+ },
121
+ }
122
+ }
123
+
124
+ // ─── Notice rendering ────────────────────────────────────────────────────────
125
+
126
+ /**
127
+ * The ONE calm notice for a litellm-local 429. Markdown (not HTML) per the
128
+ * card-format.ts convention. Copy contract (operator spec):
129
+ * - names the fleet token limiter (LiteLLM `tpm_limit`/`rpm_limit`)
130
+ * - explicitly NOT an Anthropic account limit — nothing exhausted, no
131
+ * account touched
132
+ * - the turn retries automatically; no action needed
133
+ * - after a window with silent suppressions, says "throttled N more
134
+ * time(s) since the last notice"
135
+ */
136
+ export function renderLitellmLocalNotice(opts: {
137
+ agent: string
138
+ /** Silent litellm-local 429s since the previous notice (0 on the first). */
139
+ suppressedSinceLastNotice: number
140
+ }): string {
141
+ const agent = escapeMarkdown(opts.agent)
142
+ const n = opts.suppressedSinceLastNotice
143
+ const lines = [
144
+ `🚦 **Fleet token limiter engaged** — **${agent}** hit the local LiteLLM proxy cap (\`tpm_limit\`/\`rpm_limit\`).`,
145
+ `This is switchroom's own fleet limiter smoothing a burst, not an Anthropic account limit — nothing is exhausted and no account was touched.`,
146
+ ]
147
+ if (n > 0) {
148
+ lines.push(`Throttled ${n} more ${n === 1 ? 'time' : 'times'} since the last notice.`)
149
+ }
150
+ lines.push(`_The turn retries automatically — no action needed._`)
151
+ return lines.join('\n')
152
+ }
153
+
154
+ // ─── Metric builder ──────────────────────────────────────────────────────────
155
+
156
+ /**
157
+ * Build the `litellm_local_429_notice` runtime metric for one SENT notice.
158
+ * Distinct from `rate_limit_429_classified` (which fires on EVERY classified
159
+ * event): this one fires only when a notice actually posts, and carries how
160
+ * many events the window silently absorbed — the debounce's effectiveness in
161
+ * one number. Pure builder so the payload shape is unit-testable; the wiring
162
+ * emits the result via emitRuntimeMetric (PostHog + JSONL dual sink).
163
+ */
164
+ export function buildLitellmLocalNoticeMetric(opts: {
165
+ agent: string
166
+ suppressedCount: number
167
+ windowMs: number
168
+ }): Extract<RuntimeMetricEvent, { kind: 'litellm_local_429_notice' }> {
169
+ return {
170
+ kind: 'litellm_local_429_notice',
171
+ agent: opts.agent,
172
+ suppressed_count: opts.suppressedCount,
173
+ window_ms: opts.windowMs,
174
+ }
175
+ }
176
+
177
+ // ─── Classification guard ────────────────────────────────────────────────────
178
+
179
+ /**
180
+ * True only for the `litellm-local` classification. The wiring runs this
181
+ * guard so "a notice never fires for non-litellm-local classifications" is a
182
+ * deterministic mechanism, not caller discipline — account-scoped 429s keep
183
+ * the throttle tier, generic transients keep the calm rate-limited card.
184
+ */
185
+ export function isLitellmLocalNoticeEligible(
186
+ classification: RateLimit429Classification | null | undefined,
187
+ ): classification is 'litellm-local' {
188
+ return classification === 'litellm-local'
189
+ }
@@ -192,10 +192,14 @@ export interface FleetRollInfo {
192
192
  /**
193
193
  * Trigger attribution (#3031 PR 2): "soft-avoid" = proactive
194
194
  * serving-preference roll off an account APPROACHING its limits;
195
- * "hard-exhaustion" = the probe saw a genuine quota wall. Absent on
196
- * pre-PR-2 brokers — rendered as hard exhaustion.
195
+ * "hard-exhaustion" = the probe saw a genuine quota wall;
196
+ * "model-tier-wall" = the flagship-tier (7d_oi) canary saw the account
197
+ * walled on the premium tier (#3176). Absent on pre-PR-2 brokers —
198
+ * rendered as hard exhaustion.
197
199
  */
198
- reason?: "soft-avoid" | "hard-exhaustion";
200
+ reason?: "soft-avoid" | "hard-exhaustion" | "model-tier-wall";
201
+ /** #3176 — the binding tier bucket, set only when reason is model-tier-wall. */
202
+ bucket?: string;
199
203
  }
200
204
 
201
205
  export type FleetRollAnnounceDecision =
@@ -241,9 +245,17 @@ export function buildFleetRollMessage(roll: FleetRollInfo, now: number): string
241
245
  ? ` (resets ${formatRelative(new Date(roll.exhausted_until), new Date(now))})`
242
246
  : "";
243
247
  const softAvoid = roll.reason === "soft-avoid";
248
+ // #3176 — a model-tier wall binds on the flagship (premium) 7d_oi bucket, not
249
+ // the 5h/7d windows (which read healthy — the whole point of the bug), so it
250
+ // has no window/pct to cite. Name the flagship tier explicitly and reassure
251
+ // that opus/haiku on the walled account are unaffected (the mark is
252
+ // tier-scoped), rather than rendering a generic, misleading "quota window".
253
+ const modelTierWall = roll.reason === "model-tier-wall";
244
254
  const causeLine = softAvoid
245
255
  ? `Proactive switch — \`${codeSpanSafe(roll.from)}\` is approaching its limits (${winLabel}${pctPart})${resetPart}, so the fleet moved early instead of hitting the wall.`
246
- : `${winLabel}${pctPart} on \`${codeSpanSafe(roll.from)}\`${resetPart}.`;
256
+ : modelTierWall
257
+ ? `Flagship (premium) tier weekly limit reached on \`${codeSpanSafe(roll.from)}\`${resetPart} — opus/haiku on that account are unaffected.`
258
+ : `${winLabel}${pctPart} on \`${codeSpanSafe(roll.from)}\`${resetPart}.`;
247
259
  return [
248
260
  `🔁 **Switched fleet to \`${codeSpanSafe(roll.to)}\`**`,
249
261
  ``,
@@ -195,6 +195,22 @@ export type RuntimeMetricEvent =
195
195
  limit: number | null
196
196
  current_usage: number | null
197
197
  }
198
+ /**
199
+ * A litellm-local throttle NOTICE actually posted (litellm-local-notice.ts
200
+ * — the debounced "fleet token limiter engaged" message). Distinct from
201
+ * `rate_limit_429_classified`, which fires on EVERY classified 429: this
202
+ * fires once per sent notice, and `suppressed_count` is how many
203
+ * litellm-local 429s the cooldown window silently absorbed since the
204
+ * previous notice (0 on the first) — the debounce's effectiveness in one
205
+ * number. `window_ms` is the resolved cooldown window (default 15 min;
206
+ * channels.telegram.litellm_notice.window_ms).
207
+ */
208
+ | {
209
+ kind: 'litellm_local_429_notice'
210
+ agent: string
211
+ suppressed_count: number
212
+ window_ms: number
213
+ }
198
214
 
199
215
  /**
200
216
  * The JSONL sink lives under the runtime state dir so it's per-agent
@@ -1,5 +1,5 @@
1
1
  import { describe, expect, it } from 'vitest'
2
- import { createSendGate, type Clock } from './send-gate.js'
2
+ import { createSendGate, SEND_GATE_SHED, type Clock } from './send-gate.js'
3
3
  import { isFloodWaitActiveError } from './retry-api-call.js'
4
4
 
5
5
  /**
@@ -73,8 +73,10 @@ describe('send-gate PR2: cosmetic shedding', () => {
73
73
 
74
74
  const res = await gate.gate(fn('cosmetic'), { priorityClass: 'cosmetic' })
75
75
 
76
- // Shed resolves undefined; fn never ran; the shed counter moved.
77
- expect(res).toBeUndefined()
76
+ // Shed resolves the distinguishable sentinel (#3110 F1 — never a bare
77
+ // undefined, which is reserved for benign no-op drops); fn never ran;
78
+ // the shed counter moved.
79
+ expect(res).toBe(SEND_GATE_SHED)
78
80
  expect(calls).toHaveLength(0)
79
81
  expect(gate.stats().global.shed).toBe(1)
80
82
  expect(gate.stats().global.sent).toBe(0)
@@ -95,7 +97,7 @@ describe('send-gate PR2: cosmetic shedding', () => {
95
97
  const shed = await gate.gate(fn('b'), { chat_id: '5', priorityClass: 'cosmetic' })
96
98
 
97
99
  expect(calls.map((c) => c.label)).toEqual(['a'])
98
- expect(shed).toBeUndefined()
100
+ expect(shed).toBe(SEND_GATE_SHED)
99
101
  expect(gate.stats().global.shed).toBe(1)
100
102
  })
101
103
 
@@ -128,7 +130,7 @@ describe('send-gate PR2: cosmetic shedding', () => {
128
130
  priorityClass: 'cosmetic',
129
131
  })
130
132
 
131
- expect(res).toBeUndefined()
133
+ expect(res).toBe(SEND_GATE_SHED)
132
134
  expect(calls).toHaveLength(0)
133
135
  expect(gate.stats().global.shed).toBe(1)
134
136
  })
@@ -366,7 +368,7 @@ describe('send-gate PR2: H1 cross-chat message_id isolation', () => {
366
368
  priorityClass: 'cosmetic',
367
369
  })
368
370
 
369
- expect(rA).toBeUndefined() // A shed
371
+ expect(rA).toBe(SEND_GATE_SHED) // A shed
370
372
  expect(rB).toBe('B-edit') // B sent
371
373
  expect(calls.map((c) => c.label)).toEqual(['B-edit'])
372
374
  expect(gate.stats().global.shed).toBe(1)
@@ -568,7 +570,7 @@ describe('send-gate PR2: opening a window from a 429, and persistence hook', ()
568
570
  expect(scopes).toEqual(['chat:7', 'global', 'group:7', 'msg-edit:7:3'])
569
571
  // After the window opens, a later cosmetic on the same chat sheds.
570
572
  const shed = await gate.gate(async () => 'x', { chat_id: '7', priorityClass: 'cosmetic' })
571
- expect(shed).toBeUndefined()
573
+ expect(shed).toBe(SEND_GATE_SHED)
572
574
  expect(gate.stats().global.shed).toBe(1)
573
575
  })
574
576
  })
@@ -118,6 +118,32 @@ export type PriorityClass = 'critical' | 'useful' | 'cosmetic'
118
118
  */
119
119
  export const UNTAGGED_SEND_CLASS: PriorityClass = 'critical'
120
120
 
121
+ /**
122
+ * Distinguishable resolution value for a SHED call (#3110 review F1).
123
+ *
124
+ * `undefined` was overloaded three ways at the robustApiCall seam: a gate
125
+ * SHED (cosmetic call dropped under pressure — did NOT land), a gate no-op
126
+ * drop (identical payload already on screen — benign), and the retry
127
+ * policy's swallowed benign 400s ("message is not modified" — also benign).
128
+ * A caller that needs to know "did my edit land?" (the draft stream's
129
+ * shed-honesty handling in stream-controller.ts) could not tell these
130
+ * apart, and treating every `undefined` as a shed froze multi-piece
131
+ * streams on perfectly healthy chats.
132
+ *
133
+ * A shed now resolves THIS sentinel instead. `Symbol.for` keys it in the
134
+ * global symbol registry so duplicated module instances (server + gateway
135
+ * builds) agree on identity. The no-op drop and coalesced-revert drop keep
136
+ * resolving `undefined` — for those the payload IS on screen. Callers that
137
+ * ignore the result (reactions, typing, fire-and-forget card edits) are
138
+ * unaffected either way.
139
+ */
140
+ export const SEND_GATE_SHED: unique symbol = Symbol.for('switchroom.send-gate.shed')
141
+
142
+ /** True when a gate result is the {@link SEND_GATE_SHED} sentinel. */
143
+ export function isSendGateShed(value: unknown): value is typeof SEND_GATE_SHED {
144
+ return value === SEND_GATE_SHED
145
+ }
146
+
121
147
  /**
122
148
  * Extra metadata a call site can attach so the gate can key the right buckets.
123
149
  * All fields optional — a call with none still passes the global bucket. These
@@ -159,8 +185,9 @@ export interface BucketCounters {
159
185
  dropped: number
160
186
  /**
161
187
  * Cosmetic calls shed under pressure — no token free OR a flood window open
162
- * (part3-design §2). A shed resolves as `undefined`; the next send carries
163
- * full state.
188
+ * (part3-design §2). A shed resolves the SEND_GATE_SHED sentinel (F1 —
189
+ * distinguishable from a benign no-op drop's `undefined`); the next send
190
+ * carries full state.
164
191
  */
165
192
  shed: number
166
193
  /** Useful calls dropped because they exceeded their queue TTL (part3-design §2). */
@@ -913,7 +940,9 @@ export function createSendGate(config: SendGateConfig): SendGate {
913
940
  const msgWait = state.suppressedUntilMs > now ? state.suppressedUntilMs - now : 0
914
941
  if (wait > 0 || msgWait > 0) {
915
942
  counters.shed++
916
- return Promise.resolve(undefined as unknown as T)
943
+ // Distinguishable from the no-op drop below (undefined): a shed did
944
+ // NOT land, and edit-driving callers must be able to tell (F1).
945
+ return Promise.resolve(SEND_GATE_SHED as unknown as T)
917
946
  }
918
947
  }
919
948
 
@@ -991,7 +1020,8 @@ export function createSendGate(config: SendGateConfig): SendGate {
991
1020
  const outcome = await admitPriority(bucketsFor(opts), priority)
992
1021
  if (outcome.result === 'shed') {
993
1022
  counters.shed++
994
- return undefined as unknown as T
1023
+ // See SEND_GATE_SHED: distinguishable "did not land" resolution (F1).
1024
+ return SEND_GATE_SHED as unknown as T
995
1025
  }
996
1026
  if (outcome.result === 'expired') {
997
1027
  counters.expired++
@@ -19,9 +19,14 @@
19
19
  * entire server.ts top-level initialization.
20
20
  */
21
21
 
22
- import { createDraftStream, type DraftStreamHandle } from './draft-stream.js'
22
+ import {
23
+ createDraftStream,
24
+ makeDraftEditShedError,
25
+ type DraftStreamHandle,
26
+ } from './draft-stream.js'
23
27
  import { richMessage, isParseEntitiesError } from './rich-send.js'
24
28
  import { renderOutboundChunks } from './render/rich-render.js'
29
+ import { isSendGateShed, type SendGateOpts } from './send-gate.js'
25
30
 
26
31
  /**
27
32
  * Minimal bot.api surface the controller needs. Real callers pass grammy's
@@ -77,9 +82,21 @@ export interface StreamSendOpts {
77
82
  disable_notification?: boolean
78
83
  }
79
84
 
85
+ /**
86
+ * Options a stream-controller call passes to its retry wrapper. A structural
87
+ * superset of `SendGateOpts` (send-gate.ts) plus the retry policy's own
88
+ * `threadId`, and assignable to `RetryCallOpts` (retry-api-call.ts) — so the
89
+ * production wrapper (`robustApiCall` = send gate over `createRetryApiCall`)
90
+ * receives `messageId` / `editPayload` / `priorityClass` and the gate's
91
+ * per-message edit floor, last-write-wins coalescing, no-op skip and
92
+ * cosmetic shedding govern the draft/answer stream (#3110; part3-design §4
93
+ * names rapid same-message editMessageText as the #1 flood-ban trigger).
94
+ */
95
+ export type RetryPolicyOpts = SendGateOpts & { threadId?: number }
96
+
80
97
  export type RetryPolicy = <T>(
81
98
  fn: () => Promise<T>,
82
- opts?: { threadId?: number; chat_id?: string },
99
+ opts?: RetryPolicyOpts,
83
100
  ) => Promise<T>
84
101
 
85
102
  export interface StreamControllerConfig {
@@ -251,6 +268,70 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
251
268
  return bot.api.editMessageText(chatId, id, richMessage(piece.text), opts)
252
269
  }
253
270
 
271
+ // ---- Send-gate wiring for the edit path (#3110) -------------------------
272
+ //
273
+ // Every edit call passes `messageId` / `editPayload` / `priorityClass`
274
+ // through the retry wrapper so the send gate's per-message edit floor
275
+ // (>=1.5s), last-write-wins coalescing, and no-op skip govern the draft
276
+ // stream — previously these edits carried only `{ threadId, chat_id }`,
277
+ // so the gate treated them as ordinary sends and the stream's own 400 ms
278
+ // DM throttle drove same-message editMessageText well under the floor
279
+ // (the #1 documented flood-ban trigger, part3-design §4; production ban
280
+ // 2026-07-12 on #3110). The local per-surface throttle stays as a cheap
281
+ // pre-filter; the gate is the authority.
282
+ //
283
+ // Priority classes: intermediate draft edits are `cosmetic` (part3-design
284
+ // §2 lists "stream updates" there) — shed under pressure / an open flood
285
+ // window; the next flush carries full state. The FINALIZE flush (the edit
286
+ // that renders the completed answer) is `critical`, mirroring the reply
287
+ // path's preview-finalize convention (gateway.ts editPreview): never shed,
288
+ // waits out a short window, fails fast with a structured FLOOD_WAIT_ACTIVE
289
+ // on a long one. SENDS stay untagged (the gate admits untagged non-edit
290
+ // sends as `critical`) — a shed send would resolve the gate's shed
291
+ // sentinel instead of `{ message_id }` and break message-id capture, and
292
+ // the anchor/tail sends ARE the answer surface.
293
+ //
294
+ // `handleRef.isFinal()` is true from the moment finalize() is entered
295
+ // (draft-stream sets `final` before its last flush), so the closures below
296
+ // classify exactly the finalize flush — and anything after it — as
297
+ // critical. The ref is assigned right after createDraftStream returns,
298
+ // before any closure can run (closures only fire from update/finalize).
299
+ //
300
+ // KNOWN MISSED BENEFIT (review F5, deliberate): the gate's last-write-wins
301
+ // coalescing never engages for THIS surface, because draft-stream
302
+ // serializes its flushes — it awaits each edit before issuing the next, so
303
+ // at most one edit per message is ever inside the gate. Consequence: a
304
+ // stale draft sleeping on the gate's floor still lands (one API call the
305
+ // coalescer would have replaced) before the newer snapshot, and a finalize
306
+ // issued mid-floor can trail by up to ~2x editFloorMs (floor wait for the
307
+ // stale draft, then floor wait for the final). Correctness is unaffected —
308
+ // the latest state always lands, floor-paced — and the gate coalescing
309
+ // remains live protection for CONCURRENT writers to one message (e.g. a
310
+ // re-attached #626 controller racing its predecessor).
311
+ let handleRef: DraftStreamHandle | null = null
312
+ const editGateOpts = (id: number, payload: unknown): RetryPolicyOpts => ({
313
+ threadId,
314
+ chat_id: chatId,
315
+ messageId: id,
316
+ editPayload: payload,
317
+ priorityClass: handleRef?.isFinal() === true ? 'critical' : 'cosmetic',
318
+ })
319
+ // The rendered payload the gate hashes for the no-op skip / coalescing —
320
+ // exactly what goes over the wire (rich wrapper included), so a plain
321
+ // fallback of the same text never hashes equal to its rich form.
322
+ const piecePayload = (piece: { text: string; rich: boolean }): unknown =>
323
+ piece.rich ? richMessage(piece.text) : piece.text
324
+ // Shed detection (#3110 review F1): keyed EXACTLY off the gate's
325
+ // SEND_GATE_SHED sentinel — never off `undefined`, which is overloaded
326
+ // (gate no-op drop; robustApiCall's swallowed benign 400s like "message is
327
+ // not modified"). Those benign cases mean the payload is ALREADY on screen
328
+ // and are treated as delivered, exactly as before this wiring existed. A
329
+ // true shed means the edit did NOT land: draft-stream must not record the
330
+ // snapshot as on-screen (its dedupe would skip a later flush of the same
331
+ // text — the completed answer would never render), so the edit closure
332
+ // throws the marker error draft-stream recognizes and recovers from
333
+ // (`makeDraftEditShedError` → snapshot preserved for finalize, review F2).
334
+
254
335
  // Overflow-tail bookkeeping, shared across the send + edit closures for the
255
336
  // whole stream lifetime. A body large enough to split into several
256
337
  // wire-cap pieces anchors on piece[0] (edited in place by draft-stream) and
@@ -269,12 +350,25 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
269
350
  // text) thereafter. A non-parse failure is logged as a partial-delivery
270
351
  // warning and swallowed so the remaining tail pieces still get a chance to
271
352
  // land — never a silent drop, never an abort of pieces K..N (concern C1).
272
- const upsertTail = async (ti: number, piece: { text: string; rich: boolean }): Promise<void> => {
353
+ //
354
+ // Returns TRUE when this piece's edit was SHED by the send gate (did not
355
+ // land): the caller must then report the whole flush as shed so
356
+ // draft-stream does not record the full body as delivered while a tail is
357
+ // stale (review F3). The piece's own tailLastText stays stale too, so the
358
+ // recovery flush re-attempts exactly the shed piece.
359
+ const upsertTail = async (
360
+ ti: number,
361
+ piece: { text: string; rich: boolean },
362
+ ): Promise<boolean> => {
273
363
  const existingId = tailIds[ti]
274
364
  if (existingId != null) {
275
- if (tailLastText[ti] === piece.text) return // unchanged — skip the API call
365
+ if (tailLastText[ti] === piece.text) return false // unchanged — skip the API call
276
366
  try {
277
- await retry(() => editPiece(existingId, piece, baseOpts), { threadId, chat_id: chatId })
367
+ const res = await retry(
368
+ () => editPiece(existingId, piece, baseOpts),
369
+ editGateOpts(existingId, piecePayload(piece)),
370
+ )
371
+ if (isSendGateShed(res)) return true // shed — stale; retried by the recovery flush
278
372
  tailLastText[ti] = piece.text
279
373
  onEdit?.(existingId, piece.text.length)
280
374
  } catch (err) {
@@ -282,10 +376,11 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
282
376
  warn?.(
283
377
  `stream-controller: tail-piece #${ti + 1} edit parse-entities rejected — retrying same id=${existingId} as plain text (${err instanceof Error ? err.message : String(err)})`,
284
378
  )
285
- await retry(
379
+ const res = await retry(
286
380
  () => bot.api.editMessageText(chatId, existingId, piece.text, baseOpts),
287
- { threadId, chat_id: chatId },
381
+ editGateOpts(existingId, piece.text),
288
382
  )
383
+ if (isSendGateShed(res)) return true // shed — stale; retried by the recovery flush
289
384
  tailLastText[ti] = piece.text
290
385
  onEdit?.(existingId, piece.text.length)
291
386
  } else {
@@ -296,7 +391,7 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
296
391
  )
297
392
  }
298
393
  }
299
- return
394
+ return false
300
395
  }
301
396
  // First emission of this tail piece → a fresh follow-up message.
302
397
  try {
@@ -324,9 +419,12 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
324
419
  )
325
420
  }
326
421
  }
422
+ // First-emission SENDS are untagged (critical) — the gate never sheds
423
+ // them, so this path can only land or fail (handled above).
424
+ return false
327
425
  }
328
426
 
329
- return createDraftStream(
427
+ const handle = createDraftStream(
330
428
  async (text) => {
331
429
  // Render → 1+ cap-respecting pieces. The FIRST piece's message_id anchors
332
430
  // the stream (later edits target it); any overflow pieces are parked as
@@ -378,14 +476,26 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
378
476
  async (id, text) => {
379
477
  const pieces = renderPieces(text)
380
478
  const head = pieces[0]
479
+ // Whether any piece of THIS flush was shed by the gate (did not land).
480
+ // Decided at the very end — AFTER the tail loop — so a benign anchor
481
+ // outcome (or even a shed anchor) never starves the tail pieces of
482
+ // their own upsert attempt (review F1: an anchor whose payload stopped
483
+ // changing resolves benignly every flush while only the tail grows).
484
+ let anchorShed = false
381
485
  // Edit the anchor message in place with the FIRST piece.
382
486
  try {
383
- await retry(
384
- () => editPiece(id, head, baseOpts),
385
- { threadId, chat_id: chatId },
386
- )
387
- // C2: report the actual head-piece length, not the full body length.
388
- onEdit?.(id, head.text.length)
487
+ const res = await retry(() => editPiece(id, head, baseOpts), editGateOpts(id, piecePayload(head)))
488
+ if (isSendGateShed(res)) {
489
+ // Shed by the gate (cosmetic under pressure / an open flood
490
+ // window) — did NOT land. Benign `undefined` resolutions (gate
491
+ // no-op drop, robustApiCall's swallowed "message is not modified")
492
+ // deliberately do NOT take this branch: the payload is already on
493
+ // screen and the flush proceeds as delivered.
494
+ anchorShed = true
495
+ } else {
496
+ // C2: report the actual head-piece length, not the full body length.
497
+ onEdit?.(id, head.text.length)
498
+ }
389
499
  } catch (err) {
390
500
  if (!literalText && head.rich && isParseEntitiesError(err)) {
391
501
  // Edit rejected because the markdown couldn't be parsed — DO NOT
@@ -400,11 +510,12 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
400
510
  // contract). For a single-piece stream (common case) this is the
401
511
  // whole body; a rare oversize split edits the head piece's body.
402
512
  const fallbackBody = pieces.length === 1 ? text : head.text
403
- await retry(
513
+ const res = await retry(
404
514
  () => bot.api.editMessageText(chatId, id, fallbackBody, baseOpts),
405
- { threadId, chat_id: chatId },
515
+ editGateOpts(id, fallbackBody),
406
516
  )
407
- onEdit?.(id, head.text.length)
517
+ if (isSendGateShed(res)) anchorShed = true
518
+ else onEdit?.(id, head.text.length)
408
519
  } else {
409
520
  throw err
410
521
  }
@@ -414,10 +525,20 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
414
525
  // that only just came into existence) — we never re-send tails already
415
526
  // emitted on a prior flush. This is the fix for the duplicate-flood
416
527
  // blocker: the previous code re-sent pieces[1..n] as brand-new messages
417
- // on each throttled edit tick.
528
+ // on each throttled edit tick. Runs BEFORE the shed decision below so a
529
+ // shed (or benign) anchor never starves the tails (review F1).
530
+ let anyTailShed = false
418
531
  for (let pi = 1; pi < pieces.length; pi++) {
419
- await upsertTail(pi - 1, pieces[pi])
532
+ if (await upsertTail(pi - 1, pieces[pi])) anyTailShed = true
420
533
  }
534
+ // Review F3: ANY shed piece — anchor or tail — means this flush did not
535
+ // fully land. Throw the marker error so draft-stream does not record
536
+ // the body as delivered (its dedupe would freeze the shed piece
537
+ // forever) and instead preserves the snapshot for the finalize
538
+ // re-flush. Pieces that DID land are unaffected on that re-flush: the
539
+ // gate's no-op skip drops their identical payloads before the API, and
540
+ // landed tails short-circuit on tailLastText.
541
+ if (anchorShed || anyTailShed) throw makeDraftEditShedError(id)
421
542
  },
422
543
  {
423
544
  ...(throttleMs != null ? { throttleMs } : {}),
@@ -429,4 +550,6 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
429
550
  chatId,
430
551
  },
431
552
  )
553
+ handleRef = handle
554
+ return handle
432
555
  }
@@ -516,10 +516,20 @@ export async function handleStreamReply(
516
516
  state.activeDraftStreams.set(sKey, stream)
517
517
  }
518
518
 
519
- await stream.update(effectiveText)
519
+ if (!done) {
520
+ // Intermediate snapshot — an ordinary throttled draft update.
521
+ await stream.update(effectiveText)
522
+ }
520
523
 
521
524
  if (done) {
522
- await stream.finalize()
525
+ // #3110: route the FINAL text through finalize(text) so the flush that
526
+ // renders the completed answer runs with the stream already final —
527
+ // stream-controller then classifies that edit `critical` for the send
528
+ // gate (never shed; fails fast with a structured FLOOD_WAIT_ACTIVE on a
529
+ // long flood window) while intermediate draft edits stay `cosmetic`
530
+ // (sheddable). The previous update()-then-finalize() pair flushed the
531
+ // final text inside update(), i.e. as an ordinary sheddable draft edit.
532
+ await stream.finalize(effectiveText)
523
533
  state.activeDraftStreams.delete(sKey)
524
534
  // #1713: stream_reply done=true is a NON-EVENT for the status
525
535
  // reaction. The reaction reflects current turn activity, not