switchroom 0.19.26 → 0.19.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/bin/git-agent-attribution-hook.sh +144 -0
  2. package/dist/agent-scheduler/index.js +60 -2
  3. package/dist/auth-broker/index.js +244 -13
  4. package/dist/cli/autoaccept-poll.js +225 -17
  5. package/dist/cli/notion-write-pretool.mjs +60 -2
  6. package/dist/cli/switchroom.js +2843 -1220
  7. package/dist/host-control/main.js +245 -14
  8. package/dist/vault/approvals/kernel-server.js +242 -13
  9. package/dist/vault/broker/server.js +242 -13
  10. package/package.json +7 -2
  11. package/profiles/_base/cron-session.sh.hbs +8 -0
  12. package/profiles/_base/start.sh.hbs +175 -15
  13. package/telegram-plugin/card-layout.ts +328 -0
  14. package/telegram-plugin/dist/bridge/bridge.js +94 -1
  15. package/telegram-plugin/dist/gateway/gateway.js +2544 -1182
  16. package/telegram-plugin/dist/server.js +97 -1
  17. package/telegram-plugin/edit-flood-fuse.ts +841 -57
  18. package/telegram-plugin/flood-429-ledger.ts +526 -0
  19. package/telegram-plugin/flood-circuit-breaker.ts +18 -0
  20. package/telegram-plugin/gateway/callback-query-handlers.ts +6 -0
  21. package/telegram-plugin/gateway/flood-reply-queue.ts +168 -0
  22. package/telegram-plugin/gateway/gateway.ts +67 -70
  23. package/telegram-plugin/gateway/mcp-failure-hook.ts +74 -0
  24. package/telegram-plugin/gateway/narrative-lane.ts +14 -0
  25. package/telegram-plugin/gateway/outbound-send-path.ts +36 -0
  26. package/telegram-plugin/gateway/outbox-sweep.ts +183 -6
  27. package/telegram-plugin/gateway/pinned-message-handler.ts +12 -16
  28. package/telegram-plugin/gateway/status-pin-retarget.ts +72 -36
  29. package/telegram-plugin/gateway/status-pin-store.ts +58 -9
  30. package/telegram-plugin/gateway/worker-pin-reaper.ts +56 -7
  31. package/telegram-plugin/inline-keyboard-callbacks.ts +202 -21
  32. package/telegram-plugin/llm-error-present.ts +61 -2
  33. package/telegram-plugin/mcp-credential-failure.ts +459 -0
  34. package/telegram-plugin/model-unavailable.ts +8 -0
  35. package/telegram-plugin/operator-events.ts +110 -5
  36. package/telegram-plugin/outbound-class.ts +81 -0
  37. package/telegram-plugin/provider-credit.ts +237 -0
  38. package/telegram-plugin/scripts/bun-test-ci.sh +36 -6
  39. package/telegram-plugin/send-gate.ts +24 -2
  40. package/telegram-plugin/status-no-truncate.ts +10 -48
  41. package/telegram-plugin/status-pin-driver.ts +33 -45
  42. package/telegram-plugin/status-pin.ts +18 -1
  43. package/telegram-plugin/tests/card-golden.test.ts +69 -0
  44. package/telegram-plugin/tests/card-lifecycle-render.test.ts +362 -0
  45. package/telegram-plugin/tests/card-type-distinguishability.test.ts +187 -164
  46. package/telegram-plugin/tests/card-variants.golden.txt +211 -0
  47. package/telegram-plugin/tests/card-variants.ts +366 -0
  48. package/telegram-plugin/tests/edit-flood-fuse-ban-awareness.test.ts +373 -0
  49. package/telegram-plugin/tests/edit-flood-fuse-default-deny.test.ts +319 -0
  50. package/telegram-plugin/tests/edit-flood-fuse-reply-reserve.test.ts +340 -0
  51. package/telegram-plugin/tests/edit-flood-fuse.test.ts +11 -2
  52. package/telegram-plugin/tests/feed-edit-rate-ceiling.test.ts +462 -0
  53. package/telegram-plugin/tests/finalize-callback-flood-policy.test.ts +298 -0
  54. package/telegram-plugin/tests/finalize-callback.test.ts +41 -8
  55. package/telegram-plugin/tests/fixtures/real-429-stream.ts +220 -0
  56. package/telegram-plugin/tests/flood-429-ledger.test.ts +278 -0
  57. package/telegram-plugin/tests/flood-429-recorder-wiring.test.ts +128 -0
  58. package/telegram-plugin/tests/flood-reply-queue.test.ts +418 -0
  59. package/telegram-plugin/tests/mcp-credential-failure.test.ts +310 -0
  60. package/telegram-plugin/tests/outbox-sweep-flood-breaker.test.ts +221 -0
  61. package/telegram-plugin/tests/pinned-card-collapse.test.ts +19 -24
  62. package/telegram-plugin/tests/pinned-message-handler.test.ts +15 -15
  63. package/telegram-plugin/tests/provider-credit-402.test.ts +243 -0
  64. package/telegram-plugin/tests/status-pin-api.test.ts +11 -11
  65. package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +36 -37
  66. package/telegram-plugin/tests/status-pin-lifecycle.test.ts +602 -0
  67. package/telegram-plugin/tests/status-pin-retarget.test.ts +90 -62
  68. package/telegram-plugin/tests/status-pin-service-message-suppression.test.ts +7 -3
  69. package/telegram-plugin/tests/status-pin-store.test.ts +109 -60
  70. package/telegram-plugin/tests/status-pin.test.ts +56 -5
  71. package/telegram-plugin/tests/test-runner-coverage.test.ts +133 -0
  72. package/telegram-plugin/tests/worker-activity-feed.test.ts +12 -10
  73. package/telegram-plugin/tests/worker-feed-coalesce.test.ts +23 -29
  74. package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +56 -59
  75. package/telegram-plugin/tests/worker-feed-terminal-edit-class.test.ts +335 -0
  76. package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +1 -1
  77. package/telegram-plugin/tool-activity-summary.ts +239 -365
  78. package/telegram-plugin/uat/assertions.ts +22 -11
  79. package/telegram-plugin/uat/feed-matcher.test.ts +24 -17
  80. package/telegram-plugin/worker-activity-feed.ts +105 -47
  81. package/vendor/hindsight-memory/CLAUDE.md +45 -0
  82. package/vendor/hindsight-memory/scripts/drain_pending.py +433 -11
  83. package/vendor/hindsight-memory/scripts/lib/config.py +33 -0
  84. package/vendor/hindsight-memory/scripts/lib/pending.py +193 -28
  85. package/vendor/hindsight-memory/scripts/recall.py +176 -7
  86. package/vendor/hindsight-memory/scripts/tests/test_config_recall_passthrough_env.py +170 -0
  87. package/vendor/hindsight-memory/scripts/tests/test_drain_circuit_breaker.py +401 -0
  88. package/vendor/hindsight-memory/scripts/tests/test_drain_serialisation.py +286 -0
  89. package/vendor/hindsight-memory/scripts/tests/test_pending_drops.py +817 -8
  90. package/vendor/hindsight-memory/scripts/tests/test_recall_min_score.py +464 -0
  91. package/vendor/hindsight-memory/settings.json +1 -1
  92. package/vendor/hindsight-memory/tests/test_hooks.py +11 -2
@@ -56,11 +56,12 @@
56
56
  * - send gate `editFloorMs` = 1500ms ⇒ up to 40 edits/min/message
57
57
  * - send gate cosmetic budget 150 / 300s ⇒ 30 edits/min/message
58
58
  * - worker feed's own floor 2500ms ⇒ 24 edits/min/message
59
+ * - gateway `FEED_HEARTBEAT_TICK_MS` = 6000ms ⇒ 10 repaints/min/turn
59
60
  *
60
- * The per-message ceiling here is 20/60s, i.e. BELOW all three. That is
61
- * deliberate — Telegram's own per-group limit is ~20 messages/minute and
62
- * edits count against it, so the *legitimate* cadences are themselves above
63
- * what Telegram tolerates; that is precisely how `overlord` got banned while
61
+ * The fuse's cosmetic ceilings sit BELOW all of these. That is deliberate —
62
+ * Telegram's own per-group limit is ~20 messages/minute and edits count
63
+ * against it, so the *legitimate* cadences are themselves above what
64
+ * Telegram tolerates; that is precisely how `overlord` got banned while
64
65
  * every in-repo pacer believed it was behaving. The fuse is therefore the
65
66
  * BINDING constraint on a hot card, not a never-reached backstop.
66
67
  *
@@ -74,12 +75,150 @@
74
75
  * RELEASED late instead of dropped. Dropping it would freeze the card
75
76
  * mid-run AND return `true` to the send gate, which would then record a
76
77
  * never-painted payload as on-screen and no-op-skip every retry.
78
+ *
79
+ * ── 2026-07-27: the fuse existed and the ban happened anyway ──────────────
80
+ * The ceilings above were sized against the in-repo pacers, not against what
81
+ * Telegram actually tolerates. Agent `overlord` then sustained ~17
82
+ * `editMessageText`/min on ONE DM chat for hours (326 edits against 6 real
83
+ * replies in the final 20 minutes) and took a **15908-second** flood ban that
84
+ * severed every outbound reply. Both original ceilings — 20 edits/60s per
85
+ * message, 30 edits/60s per chat — sat ABOVE that observed rate, so the fuse
86
+ * never bound once. Three structural changes follow, and each is a separate,
87
+ * independently-sufficient reason the same incident cannot recur:
88
+ *
89
+ * 1. **Class awareness.** The fuse now reads the send gate's priority class
90
+ * off `outbound-class.ts` (AsyncLocalStorage). `cosmetic` traffic — every
91
+ * activity/worker/liveness repaint — is governed by its own, much tighter
92
+ * ceilings; `useful` / `critical` edits (approval cards, answers) are not
93
+ * throttled by them. Previously the fuse was class-blind and had to pick
94
+ * one number for both, which is why the number was too high.
95
+ * 2. **A hard cosmetic rate ceiling.** 4 edits/60s per message (one per 15s,
96
+ * matching the worker feed's own `elapsedRefreshMs`) and 6 edits/60s per
97
+ * chat across ALL cosmetic surfaces. 6/min is below the ~4-6/min band the
98
+ * live chat survived for hours and far below the 15-17/min that earned
99
+ * the ban. Coalescing (supersede) means the card still shows CURRENT
100
+ * state at every permitted edit — the operator loses refresh frequency,
101
+ * never accuracy.
102
+ * 3. **A shared per-chat budget with a reply reservation.** Telegram meters
103
+ * sends and edits against the SAME per-chat allowance, so separate
104
+ * edit/send windows could each be "in budget" while their sum was not.
105
+ * One `perChatTotalMaxPerWindow` window (default 20/60s) counts every
106
+ * chat-targeted call, and `perChatReplyReserve` (default 8) slots of it
107
+ * are unreachable by `cosmetic` traffic. A reply therefore cannot be
108
+ * starved by repaints even if a future call site is misclassified.
109
+ *
110
+ * ── 2026-07-28: the meter was reading ~58% of the traffic (#3855) ─────────
111
+ * Point 3 above USED to claim `perChatTotalMaxPerWindow` "counts every
112
+ * admitted call" and "matches Telegram's own per-chat metering". Both were
113
+ * false, and the gap is the reason a "20/60s" ceiling did not stop a ban.
114
+ * `apply` opened with `if (!isEdit && !isSend) return runObserved(next)` —
115
+ * an ALLOWLIST. Anything outside `EDIT_METHODS`/`SEND_METHODS` was entirely
116
+ * unmetered: it took no slot, and the ceiling therefore governed only the
117
+ * subset of traffic it happened to recognise.
118
+ *
119
+ * Measured on overlord's own gateway log (46791 `tg-post` lines), the
120
+ * unmetered share was 29% overall and 42% inside the 30 minutes before the
121
+ * 20:19:56 ban. By method:
122
+ *
123
+ * sendChatAction 10457 ← unmetered, and up to 18/min on ONE DM
124
+ * setMessageReaction 2098 ← unmetered
125
+ * deleteMessage 370 ← unmetered
126
+ * pinChatMessage 366 ← unmetered
127
+ * unpinChatMessage 365 ← unmetered
128
+ * answerCallbackQuery 31 ← unmetered (carries no chat_id)
129
+ *
130
+ * `sendChatAction` alone ran at Telegram's entire documented per-chat/minute
131
+ * allowance while the fuse counted none of it. So the fix is not a smaller
132
+ * number, it is an honest meter:
133
+ *
134
+ * 4. **DEFAULT-DENY metering.** Every method whose payload carries a
135
+ * `chat_id` is charged to `perChatTotalMaxPerWindow`, whether or not this
136
+ * file has heard of it. There is no per-chat exemption list to forget to
137
+ * update — a new Bot API method, or one this gateway starts calling
138
+ * tomorrow (`sendChecklist` is exactly that case: it carries `chat_id`,
139
+ * creates a message, and was in neither set), is metered on arrival.
140
+ * Methods that are neither edits nor sends are PACED and never dropped,
141
+ * except `CHAT_ACTION_METHODS` — a typing indicator expires by itself in
142
+ * ~5s and carries no content, so shedding one loses nothing.
143
+ * The ceiling NUMBER is unchanged at 20/60s; what changed is that 20
144
+ * admitted calls now means 20 on the wire rather than ~34.
145
+ * 5. **A per-BOT-TOKEN window.** Telegram's flood ban is scoped to the bot
146
+ * token, not the chat, and enforces a global rate across all chats. A
147
+ * second window keyed by the bot's PUBLIC numeric id (`perTokenWindowMs`
148
+ * / `perTokenMaxPerWindow`, default 25 per 1000ms) is now charged for
149
+ * EVERY outbound call — including the chat-less ones like
150
+ * `answerCallbackQuery`, which the per-chat tier structurally cannot key.
151
+ * A call is admitted only if BOTH windows have room.
152
+ * LIMITATION, stated plainly: this window is per-PROCESS. Two agents
153
+ * sharing one bot token run in separate containers with separate state
154
+ * dirs, so neither sees the other's window. It is a real improvement on
155
+ * the single process that actually floods and it is NOT full coverage
156
+ * under a shared token; a cross-process view would need a token-keyed
157
+ * store on a path both processes can write, which does not exist today.
158
+ * Only the public bot id (the prefix before the `:`) is ever used as a
159
+ * key, and it is never logged — see {@link deriveBotScopeKey}.
160
+ *
161
+ * Plus **escalating** backoff: a 429 used to apply ONE 0.5× tightening for a
162
+ * flat 10 minutes, so a stream that took repeated small 429s re-tightened to
163
+ * the same level forever and walked into the big ban. Tightening is now
164
+ * multiplicative PER 429 (`tightenFactor ^ level`, level capped by
165
+ * `maxTightenLevel`) and decays one level at a time, so sustained pressure
166
+ * ratchets the cosmetic rate down towards the floor of 1/window.
167
+ *
168
+ * ── 2026-07-28: the reserve could not fire, and the reply paid (#3885) ────
169
+ * Points 3 and 5 above, and the escalating backoff, combined into a defect
170
+ * that hit the operator directly: during a 12-agent staggered restart that drew
171
+ * **three** genuine 429s, one chat logged 792 fuse events in an hour against a
172
+ * 35-50/hr baseline, shed 221 typing indicators, deferred 18 reactions, and
173
+ * deferred real replies long enough that the MCP `reply` tool's 60s timeout
174
+ * fired — so the agent retried and the operator got the same answer twice. One
175
+ * ~1900-character reply was lost outright.
176
+ *
177
+ * Three 3-second 429s ratchet `tightenLevel` to 3, and each one also writes
178
+ * `flood-wait.json`, which pinned `levelAt` at `maxTightenLevel` outright. At
179
+ * level 4 the shared per-chat budget is `max(1, floor(20 * 0.5^4))` = **1 call
180
+ * per minute for the entire chat**. The reserve that exists for precisely this
181
+ * case could not help, because it was a fixed count subtracted from the BASE:
182
+ * `max(1, 20 - 8)` = 12, tightened to 1 — the same 1 the reply had. Reply and
183
+ * repaint ended up with an identical budget, which is the opposite of a
184
+ * reservation. Three changes:
185
+ *
186
+ * 6. **The reserve is a PROPORTION, re-derived per window.** `cosmeticTotalMax`
187
+ * computes it against the EFFECTIVE ceiling, so "8 of 20" means "40% of
188
+ * whatever the budget currently is", and at least one slot is always
189
+ * reserved. When the effective budget shrinks to the reserve, cosmetic
190
+ * traffic gets 0 and every remaining slot belongs to a reply.
191
+ * 7. **A hard floor under `critical`.** `perChatCriticalMinPerWindow`
192
+ * (default 3) is a rate no tightening may cross, on every per-chat tier.
193
+ * Shedding a typing indicator under pressure is correct; holding the
194
+ * answer past the caller's own timeout is not — it does not save a call,
195
+ * it doubles it.
196
+ * 8. **A persisted flood window tightens IN PROPORTION to what remains**,
197
+ * through the same severity ladder `noteFlood` uses, instead of jumping
198
+ * straight to `maxTightenLevel`. `makeFloodWaitRecorder` writes the marker
199
+ * for every 429 including a 3-second nudge, so the flat jump reintroduced
200
+ * on the persisted path exactly the "a nudge and a 4.4h ban are the same
201
+ * signal" defect that #3856 had just fixed on the in-memory path.
202
+ *
203
+ * Every ceiling is operator-overridable via env — see
204
+ * {@link editFloodFuseConfigFromEnv}. Previously none of them were, and until
205
+ * #3885 the tightening CURVE (`maxTightenLevel`, `tightenFactor`) still was
206
+ * not, which is why the only available workaround for the above was to inflate
207
+ * the base ceiling fleet-wide.
77
208
  */
78
209
 
210
+ import { createHash } from 'node:crypto'
211
+
79
212
  import type { Bot } from 'grammy'
80
213
 
81
214
  import type { Clock } from './send-gate.js'
82
215
  import { systemClock } from './send-gate.js'
216
+ import { floodStatePath, makeFloodWaitProbe } from './flood-circuit-breaker.js'
217
+ import {
218
+ currentOutboundClass,
219
+ defaultOutboundClass,
220
+ type OutboundClass,
221
+ } from './outbound-class.js'
83
222
 
84
223
  /** Methods that mutate an EXISTING message — droppable, coalescible. */
85
224
  const EDIT_METHODS: ReadonlySet<string> = new Set([
@@ -112,26 +251,185 @@ const SEND_METHODS: ReadonlySet<string> = new Set([
112
251
  'sendRichMessage',
113
252
  ])
114
253
 
254
+ /**
255
+ * Chat-targeted methods whose effect EXPIRES on its own and carries no content,
256
+ * so shedding one under pressure loses nothing a user would notice.
257
+ *
258
+ * `sendChatAction` is the entire set and the reason this tier exists: it was
259
+ * the single largest unmetered method in the incident log (10457 calls, up to
260
+ * 18/min on ONE DM — Telegram's whole documented per-chat/minute allowance),
261
+ * and the typing status it sets clears itself after ~5 seconds regardless.
262
+ * Every OTHER non-edit, non-send method with a `chat_id` (`deleteMessage`,
263
+ * `pinChatMessage`, `setMessageReaction`, …) has a durable, user-visible
264
+ * effect and is paced but never dropped.
265
+ */
266
+ const CHAT_ACTION_METHODS: ReadonlySet<string> = new Set(['sendChatAction'])
267
+
268
+ /**
269
+ * The ONLY methods exempt from metering entirely. Deliberately one entry.
270
+ *
271
+ * `getUpdates` is the long-poll RECEIVE loop, not outbound traffic: it is how
272
+ * inbound messages arrive at all. Pacing it would stall message delivery
273
+ * rather than protect anything, and it targets no chat, so it consumes no
274
+ * per-chat budget by construction.
275
+ *
276
+ * Nothing else is listed. The one-off administrative calls (`setMyCommands`,
277
+ * `deleteWebhook`, `getFile`, `getChat` — 37 calls TOTAL across a 46791-call
278
+ * log) are metered like everything else: at 25/second the cost is nil, and
279
+ * exempting them would mean asserting something about Telegram's accounting
280
+ * that this repo cannot verify from first-party evidence. Default-deny means
281
+ * the burden of proof is on the exemption.
282
+ */
283
+ const UNMETERED_METHODS: ReadonlySet<string> = new Set(['getUpdates'])
284
+
285
+ /**
286
+ * Reduce a bot token to a key safe to hold in memory.
287
+ *
288
+ * A Telegram token is `<bot_id>:<secret>`. The bot id is PUBLIC (it is the
289
+ * bot's user id, visible on every message it sends); the secret half is not.
290
+ * This returns the bot id alone when the token has that shape, so nothing
291
+ * secret is ever used as a map key, logged, or persisted. A token that does
292
+ * not match the shape falls back to a truncated SHA-256, which is stable
293
+ * across processes and irreversible.
294
+ */
295
+ export function deriveBotScopeKey(token: string | undefined): string {
296
+ if (token == null || token === '') return 'unknown'
297
+ const id = token.slice(0, token.indexOf(':'))
298
+ if (/^\d+$/.test(id)) return id
299
+ return `h${createHash('sha256').update(token).digest('hex').slice(0, 16)}`
300
+ }
301
+
115
302
  export interface EditFloodFuseConfig {
116
303
  /** Master switch. When false, `apply` is a pure passthrough. Default true. */
117
304
  enabled?: boolean
118
305
  clock?: Clock
119
- /** Hard ceiling on edits to ONE message id. Default 20 per 60s. */
306
+ /**
307
+ * Hard ceiling on NON-cosmetic (`useful` / `critical`) edits to ONE message
308
+ * id. Default 20 per 60s. Approval cards and answer finalisations live here;
309
+ * they are low-cadence by nature, so this stays a backstop, not a pacer.
310
+ */
120
311
  perMessageMaxPerWindow?: number
121
312
  perMessageWindowMs?: number
122
- /** Hard ceiling on edits to ONE chat across all messages. Default 30 per 60s. */
313
+ /** Hard ceiling on ALL edits to ONE chat across all messages. Default 30 per 60s. */
123
314
  perChatEditMaxPerWindow?: number
124
315
  /** Pacing ceiling on non-edit sends to ONE chat. Default 25 per 60s. */
125
316
  perChatSendMaxPerWindow?: number
317
+ /**
318
+ * ADDITIONAL ceiling applied to COSMETIC edits of ONE message id, on top of
319
+ * `perMessageMaxPerWindow` (the effective ceiling is the lower of the two).
320
+ * Default 4 per 60s — one per 15s, matching the worker feed's
321
+ * `elapsedRefreshMs`. This is the number that bounds a hot progress card.
322
+ */
323
+ cosmeticPerMessageMaxPerWindow?: number
324
+ /**
325
+ * ADDITIONAL ceiling applied to COSMETIC edits in ONE chat, summed across
326
+ * every cosmetic surface in it (lower of this and `perChatEditMaxPerWindow`
327
+ * binds). Default 6 per 60s. Without it, N concurrent cards each inside their
328
+ * own per-message ceiling still add up to a flood.
329
+ */
330
+ cosmeticPerChatMaxPerWindow?: number
331
+ /**
332
+ * Shared per-chat allowance counting EVERY admitted call — edits and sends,
333
+ * every class. Default 20 per 60s, matching Telegram's own per-chat/group
334
+ * metering (which does not separate the two). Sends and non-cosmetic edits
335
+ * are paced against it and never dropped.
336
+ */
337
+ perChatTotalMaxPerWindow?: number
338
+ /**
339
+ * Slots of `perChatTotalMaxPerWindow` that `cosmetic` traffic may NEVER
340
+ * consume. Default 8. This is the reply-starvation guarantee: whatever the
341
+ * repaint surfaces do, at least this many calls per window remain available
342
+ * to an actual answer.
343
+ *
344
+ * Read as a PROPORTION of the base, not an absolute count (#3885). The
345
+ * reserve used to be a fixed number subtracted from the base ceiling, which
346
+ * made it arithmetically dead under tightening: at `maxTightenLevel` the base
347
+ * 20 collapses to an effective 1, and `1 - 8` clamps to the same floor of 1
348
+ * that cosmetic traffic already had — so reply and repaint ended up with an
349
+ * identical budget in exactly the situation the reserve exists for. It is now
350
+ * re-derived against the EFFECTIVE ceiling every window, and always leaves at
351
+ * least one slot reserved.
352
+ */
353
+ perChatReplyReserve?: number
354
+ /**
355
+ * Absolute floor, per `perChatWindowMs`, on `critical`-class traffic in one
356
+ * chat — the operator's actual answer, an approval card, a reaction. Default
357
+ * 3. No amount of tightening may take a chat below this: a reply held past
358
+ * the caller's own timeout is not "shedding under pressure", it is a lost
359
+ * answer plus a retry that sends it twice (#3885). Cosmetic traffic has no
360
+ * floor and is still shed to zero, which is the correct trade.
361
+ *
362
+ * Capped by the tier's own base ceiling, so an operator who deliberately
363
+ * configures a base below this floor still gets what they configured.
364
+ */
365
+ perChatCriticalMinPerWindow?: number
366
+ /**
367
+ * Ceiling on EVERY outbound call made with this bot token, across all chats
368
+ * and including the chat-less ones. Default 25 per `perTokenWindowMs`
369
+ * (1000ms), just under the ~30/second Telegram enforces at the token level.
370
+ * The flood ban is token-scoped, so this is the tier that matches the thing
371
+ * that actually gets banned. Per-PROCESS — see the docblock's point 5.
372
+ */
373
+ perTokenMaxPerWindow?: number
374
+ perTokenWindowMs?: number
375
+ /**
376
+ * Scope key for the token window: the bot's PUBLIC numeric id. Never pass a
377
+ * raw token — {@link installEditFloodFuse} derives this via
378
+ * {@link deriveBotScopeKey}. Default `'unknown'` (a single shared window,
379
+ * which is still correct for the one-bot-per-process case).
380
+ */
381
+ botScopeKey?: string
382
+ /**
383
+ * Longest a `CHAT_ACTION_METHODS` call may be held before it is shed.
384
+ * Default 3000ms — a typing indicator that arrives after the status it
385
+ * describes has already expired is worse than no typing indicator.
386
+ */
387
+ chatActionMaxDeferMs?: number
388
+ /**
389
+ * Cap on COSMETIC edits that may be released late (over budget) because
390
+ * nothing newer will repaint them — the bounded form of the R1 rule. Default
391
+ * 2 per `perChatWindowMs`, so the worst-case sustained cosmetic rate is
392
+ * `cosmeticPerChatMaxPerWindow + lateReleaseMaxPerWindow`.
393
+ */
394
+ lateReleaseMaxPerWindow?: number
126
395
  perChatWindowMs?: number
127
396
  /** Longest a call may be held before it is dropped (edit) / released (send). Default 30s. */
128
397
  maxDeferMs?: number
129
- /** Multiplicative decrease applied to every ceiling after a 429. Default 0.5. */
398
+ /** Multiplicative decrease applied per observed 429. Default 0.5. */
130
399
  tightenFactor?: number
131
- /** How long a tightened ceiling stays in force. Default 10 minutes. */
400
+ /** How long ONE level of tightening stays in force before decaying. Default 10 minutes. */
132
401
  tightenMs?: number
402
+ /**
403
+ * Cap on compounding 429 tightening levels. Default 4 (⇒ 0.5^4 = 1/16).
404
+ * Operator-overridable via `SWITCHROOM_EDIT_FUSE_MAX_TIGHTEN_LEVEL` (#3885);
405
+ * `0` disables tightening entirely.
406
+ */
407
+ maxTightenLevel?: number
408
+ /**
409
+ * Remaining ms of a KNOWN-OPEN Telegram flood window, or 0 when none — the
410
+ * persisted breaker state (`flood-circuit-breaker.ts`) that every other
411
+ * outbound path already consults. Wired by {@link editFloodFuseConfigFromEnv}.
412
+ *
413
+ * The fuse's tightening lived only in process memory, so before #3856 a
414
+ * gateway restart during a ban came back FULLY UNTIGHTENED and resumed at
415
+ * the rate that earned the ban — and a restart is the single most likely
416
+ * thing to happen during a multi-hour outage. Consulting the persisted
417
+ * window makes the response durable: while it is open the fuse holds
418
+ * `maxTightenLevel` regardless of what this process remembers.
419
+ *
420
+ * FAILS OPEN (throwing or absent probe ⇒ untightened): a breaker that cannot
421
+ * read its own state must never gag the bot.
422
+ */
423
+ floodWaitRemainingMs?: () => number
424
+ /** How often the persisted window may be re-read. Default 1000ms. */
425
+ floodProbeIntervalMs?: number
133
426
  /** Observability hook; fired whenever the fuse binds. */
134
- onTrip?: (info: { method: string; key: string; action: 'deferred' | 'dropped' | 'superseded' }) => void
427
+ onTrip?: (info: {
428
+ method: string
429
+ key: string
430
+ action: 'deferred' | 'dropped' | 'superseded'
431
+ cls: OutboundClass
432
+ }) => void
135
433
  }
136
434
 
137
435
  export interface EditFloodFuseStats {
@@ -146,8 +444,41 @@ export interface EditFloodFuseStats {
146
444
  floodObserved: number
147
445
  /** Whether a tightened ceiling is in force right now. */
148
446
  tightened: boolean
149
- /** Live per-message ceiling (post-AIMD). */
447
+ /** Compounding 429 tightening level currently in force (0 = untightened). */
448
+ tightenLevel: number
449
+ /** Live NON-cosmetic per-message ceiling (post-AIMD). */
150
450
  perMessageCeiling: number
451
+ /** Live COSMETIC per-message ceiling (post-AIMD) — the incident-relevant one. */
452
+ cosmeticPerMessageCeiling: number
453
+ /** Live COSMETIC per-chat ceiling (post-AIMD). */
454
+ cosmeticPerChatCeiling: number
455
+ /**
456
+ * Live shared per-chat budget available to COSMETIC traffic (post-AIMD, after
457
+ * the reply reserve). Goes to 0 under heavy tightening — that is the reserve
458
+ * working, not a fault.
459
+ */
460
+ cosmeticPerChatTotalCeiling: number
461
+ /**
462
+ * Live shared per-chat budget available to CRITICAL traffic (post-AIMD, after
463
+ * the critical floor). The direct measure of #3885: this must never drop
464
+ * below `perChatCriticalMinPerWindow`, at any tighten level.
465
+ */
466
+ criticalPerChatTotalCeiling: number
467
+ /**
468
+ * Chat-targeted calls charged by the DEFAULT-DENY rule — every one of these
469
+ * was completely unmetered before #3855. A non-zero value here is the direct
470
+ * measure of the hole this closed.
471
+ */
472
+ meteredByDefault: number
473
+ /** Calls with no `chat_id`, charged to the token window only. */
474
+ chatless: number
475
+ /** Live per-bot-token ceiling (post-AIMD). */
476
+ perTokenCeiling: number
477
+ /**
478
+ * True when a PERSISTED flood window is open right now — i.e. the tightening
479
+ * in force is durable rather than remembered, and survives a restart (#3856).
480
+ */
481
+ persistedFloodOpen: boolean
151
482
  }
152
483
 
153
484
  export const EDIT_FLOOD_FUSE_DEFAULTS = {
@@ -155,12 +486,119 @@ export const EDIT_FLOOD_FUSE_DEFAULTS = {
155
486
  perMessageWindowMs: 60_000,
156
487
  perChatEditMaxPerWindow: 30,
157
488
  perChatSendMaxPerWindow: 25,
489
+ /**
490
+ * 4/60s. One cosmetic repaint per 15s per card. Chosen to equal the worker
491
+ * feed's `elapsedRefreshMs` (15000) — the slowest cadence at which the feed
492
+ * itself considers a repaint worth making — so the fuse binds the runaway
493
+ * case without ever throttling the feed's own intended pace.
494
+ */
495
+ cosmeticPerMessageMaxPerWindow: 4,
496
+ /**
497
+ * 6/60s across every cosmetic surface in a chat. The incident's own timeline
498
+ * is the evidence: 10-minute buckets of 58/53/41 edits (≈4-6/min) ran for
499
+ * hours without a ban; 115/78/73 then 152/174 (≈8-17/min) earned one. 6/min
500
+ * sits at the top of the survived band and less than half the banned rate.
501
+ */
502
+ cosmeticPerChatMaxPerWindow: 6,
503
+ /**
504
+ * 20/60s. Telegram's documented per-group ceiling, and it meters sends and
505
+ * edits together — so this is the only window that reflects the real budget.
506
+ */
507
+ perChatTotalMaxPerWindow: 20,
508
+ /**
509
+ * 8 of those 20 slots/min are unreachable by cosmetic traffic — i.e. 40% of
510
+ * whatever the EFFECTIVE ceiling is, re-derived per window (#3885).
511
+ */
512
+ perChatReplyReserve: 8,
513
+ /**
514
+ * 3/60s. The hard floor on `critical` traffic in a chat. Sized against the
515
+ * thing that broke: the MCP `reply` tool times out at 60s, so a chat that can
516
+ * pass fewer than a couple of criticals a minute turns one answer into a
517
+ * timeout, a retry, and a duplicate message.
518
+ */
519
+ perChatCriticalMinPerWindow: 3,
520
+ /**
521
+ * 25 per second across the whole bot token. Telegram enforces ~30/s at the
522
+ * token level and the ban it issues is token-scoped; 25 leaves headroom for
523
+ * traffic this process cannot see (a peer agent sharing the token, or a
524
+ * retry issued below the transformer stack).
525
+ */
526
+ perTokenMaxPerWindow: 25,
527
+ perTokenWindowMs: 1_000,
528
+ /** A typing indicator held longer than this is not worth sending. */
529
+ chatActionMaxDeferMs: 3_000,
530
+ /** At most 2 cosmetic frames/min may exceed the ceiling as "lone" releases. */
531
+ lateReleaseMaxPerWindow: 2,
158
532
  perChatWindowMs: 60_000,
159
533
  maxDeferMs: 30_000,
160
534
  tightenFactor: 0.5,
161
535
  tightenMs: 600_000,
536
+ maxTightenLevel: 4,
537
+ /** The persisted flood window is re-read at most once a second (#3856). */
538
+ floodProbeIntervalMs: 1_000,
162
539
  } as const
163
540
 
541
+ /** Parse a positive integer from env, or undefined when unset/invalid. */
542
+ function envInt(raw: string | undefined): number | undefined {
543
+ if (raw == null || raw.trim() === '') return undefined
544
+ const n = Number(raw)
545
+ return Number.isFinite(n) && n >= 0 ? Math.floor(n) : undefined
546
+ }
547
+
548
+ /**
549
+ * Parse a multiplicative-decrease factor from env: a real in (0, 1].
550
+ *
551
+ * Rejects 0 and negatives (which would zero every ceiling) and anything above
552
+ * 1 (which would make a 429 LOOSEN the fuse). An out-of-range value is treated
553
+ * as unset rather than clamped — silently reinterpreting an operator's number
554
+ * is how a fuse ends up at a rate nobody chose.
555
+ */
556
+ function envFactor(raw: string | undefined): number | undefined {
557
+ if (raw == null || raw.trim() === '') return undefined
558
+ const n = Number(raw)
559
+ return Number.isFinite(n) && n > 0 && n <= 1 ? n : undefined
560
+ }
561
+
562
+ /**
563
+ * Operator config surface. Before the 2026-07-27 incident the fuse had exactly
564
+ * one knob — `SWITCHROOM_EDIT_FUSE=0`, which turns the whole failsafe OFF —
565
+ * so an operator whose chat was being flooded had no way to tighten it and no
566
+ * way to loosen it for a chat that could take more. Every ceiling is now
567
+ * overridable; unset values keep the defaults above.
568
+ */
569
+ export function editFloodFuseConfigFromEnv(
570
+ env: Record<string, string | undefined>,
571
+ ): EditFloodFuseConfig {
572
+ const cfg: EditFloodFuseConfig = { enabled: env.SWITCHROOM_EDIT_FUSE !== '0' }
573
+ const assign = <K extends keyof EditFloodFuseConfig>(k: K, v: number | undefined): void => {
574
+ if (v !== undefined) (cfg[k] as number) = v
575
+ }
576
+ assign('cosmeticPerMessageMaxPerWindow', envInt(env.SWITCHROOM_FEED_EDIT_MAX_PER_MSG_PER_MIN))
577
+ assign('cosmeticPerChatMaxPerWindow', envInt(env.SWITCHROOM_FEED_EDIT_MAX_PER_CHAT_PER_MIN))
578
+ assign('perChatTotalMaxPerWindow', envInt(env.SWITCHROOM_CHAT_TOTAL_MAX_PER_MIN))
579
+ assign('perChatReplyReserve', envInt(env.SWITCHROOM_CHAT_REPLY_RESERVE))
580
+ assign('perChatCriticalMinPerWindow', envInt(env.SWITCHROOM_CHAT_CRITICAL_MIN_PER_MIN))
581
+ assign('perTokenMaxPerWindow', envInt(env.SWITCHROOM_TOKEN_MAX_PER_SEC))
582
+ assign('maxDeferMs', envInt(env.SWITCHROOM_EDIT_FUSE_MAX_DEFER_MS))
583
+ // #3885 — the tightening curve itself is now operator-tunable. Before this,
584
+ // `SWITCHROOM_CHAT_TOTAL_MAX_PER_MIN` was the ONLY lever on it, so an
585
+ // operator whose chats were collapsing under `factor^maxLevel` had to inflate
586
+ // the BASE ceiling fleet-wide to compensate — raising the untightened rate
587
+ // (the one that earns bans) to fix the tightened one.
588
+ assign('maxTightenLevel', envInt(env.SWITCHROOM_EDIT_FUSE_MAX_TIGHTEN_LEVEL))
589
+ assign('tightenFactor', envFactor(env.SWITCHROOM_EDIT_FUSE_TIGHTEN_FACTOR))
590
+ // #3856 — durable ban awareness. The fuse reads the SAME persisted marker
591
+ // (`flood-wait.json`) that `robustApiCall` and the outbox sweep consult, so a
592
+ // restart mid-ban comes back tightened instead of at full rate. Wired here
593
+ // rather than at the call site so no caller can forget it, and because
594
+ // `gateway.ts` is under a zero-slack line ratchet.
595
+ const stateDir = env.TELEGRAM_STATE_DIR
596
+ if (stateDir != null && stateDir !== '') {
597
+ cfg.floodWaitRemainingMs = makeFloodWaitProbe(floodStatePath(stateDir))
598
+ }
599
+ return cfg
600
+ }
601
+
164
602
  /** grammY resolves an edit with `true` when there is nothing to return. */
165
603
  const DROPPED_RESULT = true
166
604
 
@@ -189,28 +627,175 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
189
627
  const perMessageWindowMs = config.perMessageWindowMs ?? D.perMessageWindowMs
190
628
  const perChatEditMax = config.perChatEditMaxPerWindow ?? D.perChatEditMaxPerWindow
191
629
  const perChatSendMax = config.perChatSendMaxPerWindow ?? D.perChatSendMaxPerWindow
630
+ // Cosmetic tiers are an ADDITIONAL constraint on the same window keys, so the
631
+ // effective ceiling is the lower of the two. Folding them in here (rather
632
+ // than as extra tiers) keeps the admission path at the same number of awaits.
633
+ const cosmeticPerMessageMax = Math.min(
634
+ perMessageMax, config.cosmeticPerMessageMaxPerWindow ?? D.cosmeticPerMessageMaxPerWindow)
635
+ const cosmeticPerChatMax = Math.min(
636
+ perChatEditMax, config.cosmeticPerChatMaxPerWindow ?? D.cosmeticPerChatMaxPerWindow)
637
+ const perChatTotalMax = config.perChatTotalMaxPerWindow ?? D.perChatTotalMaxPerWindow
192
638
  const perChatWindowMs = config.perChatWindowMs ?? D.perChatWindowMs
639
+ // Clamped so a misconfigured reserve can never eat more than the whole
640
+ // budget. Note this clamp is on the BASE only — the reserve that actually
641
+ // binds is re-derived per window by `cosmeticTotalMax` (#3885).
642
+ const perChatReplyReserve = Math.min(
643
+ Math.max(0, config.perChatReplyReserve ?? D.perChatReplyReserve),
644
+ Math.max(0, perChatTotalMax - 1),
645
+ )
646
+ /**
647
+ * The reserve as a FRACTION of the base. This is the whole #3885 fix: the
648
+ * reserve has to mean "40% of the chat's budget belongs to replies", not "8
649
+ * calls", because 8 is meaningless once the effective budget is 1.
650
+ */
651
+ const replyReserveFraction = perChatTotalMax > 0 ? perChatReplyReserve / perChatTotalMax : 0
652
+ const perChatCriticalMin = Math.max(
653
+ 0, config.perChatCriticalMinPerWindow ?? D.perChatCriticalMinPerWindow)
654
+ const perTokenMax = Math.max(1, config.perTokenMaxPerWindow ?? D.perTokenMaxPerWindow)
655
+ const perTokenWindowMs = Math.max(1, config.perTokenWindowMs ?? D.perTokenWindowMs)
656
+ // Only ever the PUBLIC bot id (or an irreversible hash) reaches this key.
657
+ const tokenKey = `g:${config.botScopeKey ?? 'unknown'}`
658
+ const chatActionMaxDeferMs = config.chatActionMaxDeferMs ?? D.chatActionMaxDeferMs
659
+ const lateReleaseMax = Math.max(0, config.lateReleaseMaxPerWindow ?? D.lateReleaseMaxPerWindow)
193
660
  const maxDeferMs = config.maxDeferMs ?? D.maxDeferMs
194
661
  const tightenFactor = config.tightenFactor ?? D.tightenFactor
195
662
  const tightenMs = config.tightenMs ?? D.tightenMs
663
+ const maxTightenLevel = Math.max(0, config.maxTightenLevel ?? D.maxTightenLevel)
664
+ const floodProbe = config.floodWaitRemainingMs
665
+ const floodProbeIntervalMs = Math.max(0, config.floodProbeIntervalMs ?? D.floodProbeIntervalMs)
196
666
  const onTrip = config.onTrip
197
667
 
198
668
  const windows = new Map<string, Window>()
199
- const counters = { deferred: 0, dropped: 0, superseded: 0, floodObserved: 0 }
200
- /** Timestamp until which the AIMD tightening is in force (0 = untightened). */
669
+ const counters = {
670
+ deferred: 0, dropped: 0, superseded: 0, floodObserved: 0,
671
+ /** Chat-targeted calls charged by the DEFAULT-DENY rule that the old
672
+ * allowlist would have let through unmetered (#3855 observability). */
673
+ meteredByDefault: 0,
674
+ /** Calls with no `chat_id`, charged to the token window only. */
675
+ chatless: 0,
676
+ }
677
+ /**
678
+ * Compounding 429 backoff. `tightenLevel` is the number of multiplicative
679
+ * decreases currently in force; `tightenedUntil` is when the NEXT level
680
+ * decays. A single flat tightening (the pre-2026-07-27 behaviour) let a
681
+ * stream that kept taking small 429s sit at 0.5× indefinitely and walk into
682
+ * a long ban; escalating means repeated 429s ratchet the rate down.
683
+ */
684
+ let tightenLevel = 0
201
685
  let tightenedUntil = 0
202
686
 
687
+ /**
688
+ * Cached read of the PERSISTED flood window (#3856). Throttled to
689
+ * `floodProbeIntervalMs` because `levelAt` runs on every admission and the
690
+ * probe is a file read — an unthrottled read would make the fuse the latency
691
+ * problem it exists to prevent. Between reads the cached remaining time is
692
+ * decayed by elapsed wall time, so a window never appears to last longer
693
+ * than it does.
694
+ *
695
+ * FAILS OPEN: any throw is treated as "no window". `makeFloodWaitProbe`
696
+ * already fails open on an unreadable marker and warns loudly (#3106); this
697
+ * catch covers the rest.
698
+ */
699
+ let probedAt = Number.NEGATIVE_INFINITY
700
+ let probedRemainingMs = 0
701
+ function persistedFloodRemainingMs(now: number): number {
702
+ if (floodProbe === undefined) return 0
703
+ const since = now - probedAt
704
+ if (since < floodProbeIntervalMs) return Math.max(0, probedRemainingMs - since)
705
+ probedAt = now
706
+ try {
707
+ const ms = floodProbe()
708
+ probedRemainingMs = Number.isFinite(ms) && ms > 0 ? ms : 0
709
+ } catch {
710
+ probedRemainingMs = 0
711
+ }
712
+ return probedRemainingMs
713
+ }
714
+
715
+ /** Decay one level per `tightenMs` of quiet, rather than a single cliff. */
716
+ function levelAt(now: number): number {
717
+ while (tightenLevel > 0 && tightenedUntil <= now) {
718
+ tightenLevel--
719
+ tightenedUntil = tightenLevel > 0 ? tightenedUntil + tightenMs : 0
720
+ }
721
+ // #3856 — a KNOWN-OPEN persisted window pins the fuse regardless of
722
+ // in-memory state. This is what survives a restart: the process forgets,
723
+ // the marker on disk does not. It does not MUTATE `tightenLevel`, so when
724
+ // the window closes the fuse returns to whatever this process actually
725
+ // earned rather than staying pinned.
726
+ //
727
+ // #3885 — but pinned at a level PROPORTIONAL to the window still open, not
728
+ // unconditionally at `maxTightenLevel`. `flood-wait.json` is written for
729
+ // EVERY 429 including a routine 3-second nudge (`makeFloodWaitRecorder` →
730
+ // `computeFloodWait`), so the flat jump meant a 3s burst nudge and a 4.4h
731
+ // ban produced the identical maximal response — the exact asymmetry #3856
732
+ // fixed for the in-memory path and left in place here. The remaining window
733
+ // is graded through the SAME severity ladder `noteFlood` uses, so a big ban
734
+ // still pins at maximum and a nudge costs one level.
735
+ //
736
+ // Clamped by `maxTightenLevel` like every other path: `tightenStepFor`
737
+ // returns a fixed 1..3 for the smaller bands, so an operator who sets
738
+ // `SWITCHROOM_EDIT_FUSE_MAX_TIGHTEN_LEVEL=0` to disable tightening entirely
739
+ // must not find the fuse tightening anyway the moment a marker exists.
740
+ const remainingMs = persistedFloodRemainingMs(now)
741
+ if (remainingMs > 0) {
742
+ return Math.max(tightenLevel, Math.min(maxTightenLevel, tightenStepFor(remainingMs / 1000)))
743
+ }
744
+ return tightenLevel
745
+ }
746
+
203
747
  function isTightened(now: number): boolean {
204
- return tightenedUntil > now
748
+ return levelAt(now) > 0
205
749
  }
206
750
 
207
751
  /**
208
- * AIMD multiplicative decrease. Applied to every ceiling while tightened;
209
- * floored at 1 so the fuse never deadlocks a surface completely.
752
+ * AIMD multiplicative decrease, compounding per observed 429. Applied to
753
+ * every ceiling while tightened; floored at 1 so the fuse never deadlocks a
754
+ * surface completely.
210
755
  */
211
756
  function ceiling(base: number, now: number): number {
212
- if (!isTightened(now)) return base
213
- return Math.max(1, Math.floor(base * tightenFactor))
757
+ const level = levelAt(now)
758
+ if (level === 0) return base
759
+ return Math.max(1, Math.floor(base * Math.pow(tightenFactor, level)))
760
+ }
761
+
762
+ /**
763
+ * The ceiling a given CLASS sees on a tier whose base is `base` (#3885).
764
+ *
765
+ * `critical` traffic — the operator's answer, an approval card, a reaction —
766
+ * gets an absolute floor that tightening cannot cross. Everything else takes
767
+ * the tightened ceiling as-is, so shedding still happens; it just happens to
768
+ * the traffic that can afford it.
769
+ *
770
+ * The floor is capped by `base` so it can only ever RAISE a tightened ceiling
771
+ * back towards what the operator configured, never above it.
772
+ */
773
+ function classCeiling(base: number, cls: OutboundClass, now: number): number {
774
+ const eff = ceiling(base, now)
775
+ if (cls !== 'critical') return eff
776
+ return Math.max(eff, Math.min(base, perChatCriticalMin))
777
+ }
778
+
779
+ /**
780
+ * The shared per-chat budget COSMETIC traffic may reach, re-derived against
781
+ * the EFFECTIVE ceiling (#3885).
782
+ *
783
+ * Returns 0 when the effective budget has shrunk to the reserve — at that
784
+ * point every remaining slot belongs to replies, and a repaint waits for the
785
+ * window or is shed. That is the intended behaviour under real flood
786
+ * pressure: the operator loses refresh frequency, never the answer.
787
+ */
788
+ function cosmeticTotalMax(now: number): number {
789
+ const eff = ceiling(perChatTotalMax, now)
790
+ if (replyReserveFraction <= 0) return eff
791
+ const reserve = Math.min(eff, Math.max(1, Math.round(eff * replyReserveFraction)))
792
+ return Math.max(0, eff - reserve)
793
+ }
794
+
795
+ /** Shared per-chat budget resolver, per class, evaluated at admission time. */
796
+ function totalMaxFor(cls: OutboundClass): (now: number) => number {
797
+ if (cls === 'cosmetic') return cosmeticTotalMax
798
+ return (now: number) => classCeiling(perChatTotalMax, cls, now)
214
799
  }
215
800
 
216
801
  function win(key: string): Window {
@@ -255,6 +840,14 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
255
840
  /** ms until `w` has room, or 0 if it has room now. */
256
841
  function waitFor(w: Window, now: number, windowMs: number, max: number): number {
257
842
  prune(w, now, windowMs)
843
+ // A ceiling of 0 is reachable now that the reply reserve is derived against
844
+ // the EFFECTIVE budget (#3885): under heavy tightening the whole remaining
845
+ // budget belongs to replies, so cosmetic traffic has none. There is no
846
+ // "oldest slot" to wait behind in that case, so wait out the window rather
847
+ // than indexing an empty array (which would produce NaN and admit).
848
+ if (max <= 0) {
849
+ return w.ts.length > 0 ? Math.max(1, w.ts[0]! + windowMs - now) : Math.max(1, windowMs)
850
+ }
258
851
  if (w.ts.length < max) return 0
259
852
  return Math.max(1, w.ts[0]! + windowMs - now)
260
853
  }
@@ -266,23 +859,69 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
266
859
  return { chat, msg }
267
860
  }
268
861
 
269
- /** Record a 429 (whatever shape it arrives in) and tighten. */
270
- function noteFlood(now: number): void {
862
+ /**
863
+ * Record a 429 and tighten IN PROPORTION TO THE PENALTY (#3856).
864
+ *
865
+ * Before this, every 429 was worth exactly one level: a 3-second "slow down
866
+ * a touch" nudge and a **15908-second** ban produced the identical response.
867
+ * The whole point of AIMD is that the decrease matches the signal, and a
868
+ * four-hour ban is not one nudge's worth of signal. `retryAfterSec` is the
869
+ * severity Telegram itself states, so it is what the step is scaled by.
870
+ *
871
+ * The tightening is also held for at least the ban's own duration. A flat
872
+ * 10-minute `tightenMs` expired ~25× over during the 2026-07-27 ban, so the
873
+ * fuse would have been back at full rate long before the window closed.
874
+ */
875
+ function noteFlood(now: number, retryAfterSec: number): void {
271
876
  counters.floodObserved++
272
- tightenedUntil = now + tightenMs
877
+ levelAt(now)
878
+ tightenLevel = Math.min(maxTightenLevel, tightenLevel + tightenStepFor(retryAfterSec))
879
+ // Every level currently in force is re-armed; the decay clock restarts
880
+ // from the newest 429, and never expires before the penalty itself does.
881
+ tightenedUntil = now + Math.max(tightenMs, retryAfterSec * 1000 + tightenMs)
273
882
  }
274
883
 
275
- function looksLikeFlood(err: unknown): boolean {
884
+ /**
885
+ * Levels of multiplicative decrease one 429 is worth, by stated penalty:
886
+ *
887
+ * ≤ 5s 1 a routine burst nudge
888
+ * ≤ 60s 2 sustained overrate
889
+ * ≤ 600s 3 a real ban
890
+ * > 600s ALL the way to `maxTightenLevel` — at this magnitude the rate
891
+ * that produced it is categorically wrong, and stepping down
892
+ * one level at a time would spend the next window earning the
893
+ * next ban. The 2026-07-25 (3713s) and 2026-07-27 (15908s)
894
+ * bans are both in this band.
895
+ */
896
+ function tightenStepFor(retryAfterSec: number): number {
897
+ if (!Number.isFinite(retryAfterSec) || retryAfterSec <= 5) return 1
898
+ if (retryAfterSec <= 60) return 2
899
+ if (retryAfterSec <= 600) return 3
900
+ return maxTightenLevel
901
+ }
902
+
903
+ /**
904
+ * The stated `retry_after` in seconds when `err` is a flood rejection, else
905
+ * null. Returns 0 for a flood whose magnitude is not stated (the text-match
906
+ * path), which `tightenStepFor` treats as the mildest case — an unquantified
907
+ * signal must not be inflated into a maximal response.
908
+ */
909
+ function floodRetryAfterSec(err: unknown): number | null {
276
910
  const e = err as { error_code?: number; parameters?: { retry_after?: number } } | null
277
911
  if (e != null && typeof e === 'object') {
278
- if (e.error_code === 429) return true
279
- if (e.parameters != null && typeof e.parameters.retry_after === 'number') return true
912
+ const stated = e.parameters?.retry_after
913
+ if (typeof stated === 'number') return stated
914
+ if (e.error_code === 429) return 0
280
915
  }
281
916
  // Match on the SEMANTIC markers only. A bare "429" substring
282
917
  // false-positives on ordinary server text (e.g. "message 429 not found"),
283
918
  // and a false positive here halves every ceiling for ten minutes.
284
919
  const msg = err instanceof Error ? err.message : String(err ?? '')
285
- return /too many requests/i.test(msg) || /retry[ _-]?after/i.test(msg)
920
+ if (!/too many requests/i.test(msg) && !/retry[ _-]?after/i.test(msg)) return null
921
+ // grammY and the Bot API both surface the magnitude in the text; use it
922
+ // when it is there rather than throwing the severity away.
923
+ const m = /retry[ _-]?after[^0-9]{0,4}(\d+)/i.exec(msg)
924
+ return m != null ? Number(m[1]) : 0
286
925
  }
287
926
 
288
927
  /**
@@ -311,7 +950,16 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
311
950
  * Returns the reserved timestamp, or null when the call must be dropped.
312
951
  */
313
952
  async function awaitRoom(
314
- key: string, windowMs: number, max: number, method: string, mode: WaitMode,
953
+ key: string, windowMs: number,
954
+ /**
955
+ * The ceiling, resolved at EVERY loop iteration rather than once at entry.
956
+ * It has to be a function: the effective ceiling depends on the tighten
957
+ * level, the tighten level decays with time, and a call can sit in this
958
+ * loop for `maxDeferMs`. Passing a number would pin a waiter to the ceiling
959
+ * that was in force when it arrived.
960
+ */
961
+ maxFor: (now: number) => number,
962
+ method: string, mode: WaitMode, cls: OutboundClass,
315
963
  /**
316
964
  * Consulted ONLY at the defer deadline for `mode: 'drop'`. Returning false
317
965
  * converts the drop into a late release: this call is the last thing that
@@ -319,14 +967,34 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
319
967
  * frame", it is freezing a card. Absent ⇒ drop unconditionally (the
320
968
  * pre-review behaviour).
321
969
  */
322
- dropGuard?: () => boolean,
970
+ dropGuard: (() => boolean) | undefined,
971
+ /**
972
+ * Absolute deadline SHARED by every tier of one `apply` call. Each tier
973
+ * used to start its own `maxDeferMs`, so a call crossing three tiers could
974
+ * be held 3×`maxDeferMs` — an unbounded-in-practice hold that a caller's
975
+ * own timeout, not this fuse, would have to end. One deadline per call
976
+ * makes `maxDeferMs` mean what it says.
977
+ */
978
+ deadline: number,
979
+ /**
980
+ * Bounded overshoot budget for the R1 late-release path (cosmetic edits
981
+ * only). R1 says an over-budget edit that nothing newer will repaint must
982
+ * be RELEASED late rather than dropped, so a card cannot freeze mid-run.
983
+ * Taken literally that rule has no ceiling: a stream of cosmetic edits to
984
+ * DISTINCT message ids is a stream of "lone" frames, every one of which
985
+ * releases over budget — which is how 6 concurrent worker cards can sail
986
+ * past a 6/min chat ceiling. Charging each late release to a small extra
987
+ * window keeps R1's guarantee for the rare genuinely-lone frame (a
988
+ * worker's terminal recap) while capping the leak at `lateReleaseMax` per
989
+ * window. Absent ⇒ unbounded late release (sends, non-cosmetic edits).
990
+ */
991
+ lateReleaseKey?: string,
323
992
  ): Promise<number | null> {
324
993
  const w = win(key)
325
- const deadline = clock.now() + maxDeferMs
326
994
  let counted = false
327
995
  for (;;) {
328
996
  const now = clock.now()
329
- const wait = waitFor(w, now, windowMs, ceiling(max, now))
997
+ const wait = waitFor(w, now, windowMs, maxFor(now))
330
998
  if (wait === 0) {
331
999
  w.ts.push(now)
332
1000
  return now
@@ -337,19 +1005,32 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
337
1005
  // worse than a late one).
338
1006
  if (mode !== 'release' && (dropGuard === undefined || dropGuard())) {
339
1007
  counters.dropped++
340
- onTrip?.({ method, key, action: 'dropped' })
1008
+ onTrip?.({ method, key, action: 'dropped', cls })
341
1009
  return null
342
1010
  }
343
1011
  // A send — or an edit that nothing newer will repaint — is released
344
1012
  // rather than dropped, and still takes a slot so the window reflects
345
- // what actually went out.
1013
+ // what actually went out. Cosmetic late releases are additionally
1014
+ // charged to the bounded overshoot budget; when that is exhausted the
1015
+ // frame is dropped after all, so the ceiling cannot be walked past one
1016
+ // "lone" frame at a time.
1017
+ if (lateReleaseKey !== undefined) {
1018
+ const lw = win(lateReleaseKey)
1019
+ prune(lw, now, perChatWindowMs)
1020
+ if (lw.ts.length >= ceiling(lateReleaseMax, now)) {
1021
+ counters.dropped++
1022
+ onTrip?.({ method, key, action: 'dropped', cls })
1023
+ return null
1024
+ }
1025
+ lw.ts.push(now)
1026
+ }
346
1027
  w.ts.push(now)
347
1028
  return now
348
1029
  }
349
1030
  if (!counted) {
350
1031
  counters.deferred++
351
1032
  counted = true
352
- onTrip?.({ method, key, action: 'deferred' })
1033
+ onTrip?.({ method, key, action: 'deferred', cls })
353
1034
  }
354
1035
  // Last-write-wins: a newer edit to the same message kills this one. Only
355
1036
  // valid on the per-MESSAGE tier — on a per-chat key the "newer" edit is
@@ -364,7 +1045,7 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
364
1045
  await Promise.race([clock.sleep(Math.min(wait, deadline - now)), superseded])
365
1046
  if (killed) {
366
1047
  counters.superseded++
367
- onTrip?.({ method, key, action: 'superseded' })
1048
+ onTrip?.({ method, key, action: 'superseded', cls })
368
1049
  return null
369
1050
  }
370
1051
  w.waiter = null
@@ -381,51 +1062,136 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
381
1062
  next: () => Promise<R>,
382
1063
  ): Promise<R> {
383
1064
  if (!enabled) return next()
1065
+ if (UNMETERED_METHODS.has(method)) return runObserved(next)
384
1066
 
385
1067
  const isEdit = EDIT_METHODS.has(method)
386
1068
  const isSend = SEND_METHODS.has(method)
387
- if (!isEdit && !isSend) return runObserved(next)
1069
+ const isChatAction = CHAT_ACTION_METHODS.has(method)
388
1070
 
389
1071
  const { chat, msg } = payloadKeys(payload)
390
- // Inline-message edits carry no chat/message id — nothing to key on, and
391
- // they are not part of any card loop. Pass through (still observed).
392
- if (chat == null) return runObserved(next)
393
1072
 
394
1073
  const now = clock.now()
395
1074
  evict(now)
1075
+ // ONE deadline for the whole call, shared by every tier it crosses.
1076
+ const deadline = now + maxDeferMs
1077
+
1078
+ // The send gate's priority class, propagated through AsyncLocalStorage.
1079
+ // An untagged edit is COSMETIC by default — see `defaultOutboundClass`.
1080
+ // A chat action is cosmetic too: it is a self-expiring status, not content.
1081
+ const cls =
1082
+ currentOutboundClass() ?? (isChatAction ? 'cosmetic' : defaultOutboundClass(isEdit))
1083
+
1084
+ // DEFAULT-DENY (#3855): a call the fuse cannot key to a chat — an inline
1085
+ // edit, `answerCallbackQuery`, `getFile`, `setMyCommands` — is NOT free.
1086
+ // It cannot take a per-chat slot (there is no chat to charge), but it does
1087
+ // consume the bot token's global allowance, which is the thing Telegram
1088
+ // actually bans. Charge it there and pass.
1089
+ // The token tier is per-SECOND and global; a class floor there would be
1090
+ // meaningless (even at maximum tightening it admits 60/min), so it takes
1091
+ // the plain tightened ceiling.
1092
+ const tokenMaxFor = (t: number): number => ceiling(perTokenMax, t)
1093
+
1094
+ if (chat == null) {
1095
+ counters.chatless++
1096
+ await awaitRoom(tokenKey, perTokenWindowMs, tokenMaxFor, method, 'release', cls, undefined, deadline)
1097
+ return runObserved(next)
1098
+ }
1099
+
1100
+ const totalKey = `t:${chat}`
1101
+ // Cosmetic traffic may only reach the effective ceiling MINUS the reply
1102
+ // reserve; the remaining slots stay available to a real reply no matter how
1103
+ // hot the repaint surfaces are, and `critical` additionally cannot be
1104
+ // tightened below `perChatCriticalMinPerWindow`. Together these are the
1105
+ // reply-starvation guarantee, and unlike the pre-#3885 shape they hold at
1106
+ // every tighten level rather than only at level 0.
1107
+ const chatTotalMax = totalMaxFor(cls)
1108
+
1109
+ /** Final tier for every chat-targeted path: the token-scoped ceiling. */
1110
+ const passToken = async (): Promise<R> => {
1111
+ await awaitRoom(tokenKey, perTokenWindowMs, tokenMaxFor, method, 'release', cls, undefined, deadline)
1112
+ return runObserved(next)
1113
+ }
396
1114
 
397
- if (isEdit) {
398
- if (msg == null) return runObserved(next)
1115
+ if (isEdit && msg != null) {
399
1116
  const msgKey = `m:${chat}:${msg}`
400
1117
  const mw = win(msgKey)
401
1118
  // Counted BEFORE the first await so a frame that arrives while an older
402
1119
  // one is waiting is visible to that older one's `dropGuard`.
403
1120
  mw.inflight++
404
1121
  try {
405
- const msgSlot = await awaitRoom(msgKey, perMessageWindowMs, perMessageMax, method, 'supersede')
1122
+ const msgSlot = await awaitRoom(
1123
+ msgKey, perMessageWindowMs,
1124
+ cls === 'cosmetic'
1125
+ ? (t) => ceiling(cosmeticPerMessageMax, t)
1126
+ : (t) => classCeiling(perMessageMax, cls, t),
1127
+ method, 'supersede', cls, undefined, deadline,
1128
+ )
406
1129
  if (msgSlot === null) return DROPPED_RESULT as unknown as R
1130
+ // R1: only drop while something newer for THIS message is still in
1131
+ // flight to repaint it. `> 1` = this frame plus at least one newer.
1132
+ const dropGuard = (): boolean => mw.inflight > 1
1133
+ // Cosmetic frames share ONE overshoot budget per chat across both
1134
+ // per-chat tiers, so a frame cannot late-release twice on its way out.
1135
+ const lateKey = cls === 'cosmetic' ? `lr:${chat}` : undefined
1136
+ const reserved: Array<[string, number]> = [[msgKey, msgSlot]]
1137
+ const giveBack = (): void => { for (const [k, at] of reserved) unreserve(k, at) }
1138
+
407
1139
  const chatKey = `ce:${chat}`
408
1140
  const chatSlot = await awaitRoom(
409
- chatKey, perChatWindowMs, perChatEditMax, method, 'drop',
410
- // R1: only drop while something newer for THIS message is still in
411
- // flight to repaint it. `> 1` = this frame plus at least one newer.
412
- () => mw.inflight > 1,
1141
+ chatKey, perChatWindowMs,
1142
+ cls === 'cosmetic'
1143
+ ? (t) => ceiling(cosmeticPerChatMax, t)
1144
+ : (t) => classCeiling(perChatEditMax, cls, t),
1145
+ method, 'drop', cls, dropGuard, deadline, lateKey,
413
1146
  )
414
- if (chatSlot === null) {
415
- // Denied by the second tier — hand the first tier's slot back so a
416
- // dropped edit never consumes budget it did not use.
417
- unreserve(msgKey, msgSlot)
418
- return DROPPED_RESULT as unknown as R
419
- }
420
- return runObserved(next)
1147
+ if (chatSlot === null) { giveBack(); return DROPPED_RESULT as unknown as R }
1148
+ reserved.push([chatKey, chatSlot])
1149
+
1150
+ // Shared per-chat budget — the one that mirrors Telegram's real
1151
+ // metering. A non-cosmetic edit is RELEASED late rather than dropped:
1152
+ // an approval card or answer finalisation has no newer frame coming.
1153
+ const totalSlot = await awaitRoom(
1154
+ totalKey, perChatWindowMs, chatTotalMax, method,
1155
+ cls === 'cosmetic' ? 'drop' : 'release', cls, dropGuard, deadline, lateKey,
1156
+ )
1157
+ if (totalSlot === null) { giveBack(); return DROPPED_RESULT as unknown as R }
1158
+ return passToken()
421
1159
  } finally {
422
1160
  mw.inflight--
423
1161
  }
424
1162
  }
425
1163
 
426
- const chatKey = `cs:${chat}`
427
- await awaitRoom(chatKey, perChatWindowMs, perChatSendMax, method, 'release')
428
- return runObserved(next)
1164
+ if (isSend) {
1165
+ // Sends create user-visible output and are NEVER dropped — only paced.
1166
+ // The shared budget's reserve is what guarantees they still have room
1167
+ // when the repaint surfaces are saturated.
1168
+ await awaitRoom(`cs:${chat}`, perChatWindowMs,
1169
+ (t) => classCeiling(perChatSendMax, cls, t), method, 'release', cls, undefined, deadline)
1170
+ await awaitRoom(totalKey, perChatWindowMs, chatTotalMax, method, 'release', cls, undefined, deadline)
1171
+ return passToken()
1172
+ }
1173
+
1174
+ // DEFAULT-DENY (#3855). Everything else with a `chat_id` — `sendChatAction`,
1175
+ // `setMessageReaction`, `deleteMessage`, `pinChatMessage`, `sendChecklist`,
1176
+ // an inline edit that carries a chat but no message id, and any Bot API
1177
+ // method added after this file was written — is charged to the SAME shared
1178
+ // per-chat window. Previously all of it was free, which is how a "20/60s"
1179
+ // ceiling admitted ~34 calls/60s on the wire.
1180
+ counters.meteredByDefault++
1181
+ if (isChatAction) {
1182
+ // Self-expiring status: shed it rather than deliver it late. Its own
1183
+ // short deadline also keeps a typing ping from occupying a 30s hold.
1184
+ const actionDeadline = Math.min(deadline, now + chatActionMaxDeferMs)
1185
+ const slot = await awaitRoom(
1186
+ totalKey, perChatWindowMs, chatTotalMax, method, 'drop', cls, undefined, actionDeadline,
1187
+ )
1188
+ if (slot === null) return DROPPED_RESULT as unknown as R
1189
+ return passToken()
1190
+ }
1191
+ // Durable, user-visible effect (a deletion, a pin, a reaction): paced,
1192
+ // never dropped — the same contract sends get.
1193
+ await awaitRoom(totalKey, perChatWindowMs, chatTotalMax, method, 'release', cls, undefined, deadline)
1194
+ return passToken()
429
1195
  }
430
1196
 
431
1197
  /** Run the downstream call, tightening the ceilings if it floods. */
@@ -434,13 +1200,16 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
434
1200
  const res = await next()
435
1201
  // grammY throws on ok:false, but a transformer installed BELOW another
436
1202
  // one can still observe a raw ApiResponse — handle both shapes.
437
- const r = res as unknown as { ok?: boolean; error_code?: number }
1203
+ const r = res as unknown as {
1204
+ ok?: boolean; error_code?: number; parameters?: { retry_after?: number }
1205
+ }
438
1206
  if (r != null && typeof r === 'object' && r.ok === false && r.error_code === 429) {
439
- noteFlood(clock.now())
1207
+ noteFlood(clock.now(), r.parameters?.retry_after ?? 0)
440
1208
  }
441
1209
  return res
442
1210
  } catch (err) {
443
- if (looksLikeFlood(err)) noteFlood(clock.now())
1211
+ const retryAfterSec = floodRetryAfterSec(err)
1212
+ if (retryAfterSec !== null) noteFlood(clock.now(), retryAfterSec)
444
1213
  throw err
445
1214
  }
446
1215
  }
@@ -454,7 +1223,16 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
454
1223
  superseded: counters.superseded,
455
1224
  floodObserved: counters.floodObserved,
456
1225
  tightened: isTightened(now),
1226
+ tightenLevel: levelAt(now),
457
1227
  perMessageCeiling: ceiling(perMessageMax, now),
1228
+ cosmeticPerMessageCeiling: ceiling(cosmeticPerMessageMax, now),
1229
+ cosmeticPerChatCeiling: ceiling(cosmeticPerChatMax, now),
1230
+ cosmeticPerChatTotalCeiling: cosmeticTotalMax(now),
1231
+ criticalPerChatTotalCeiling: classCeiling(perChatTotalMax, 'critical', now),
1232
+ meteredByDefault: counters.meteredByDefault,
1233
+ chatless: counters.chatless,
1234
+ perTokenCeiling: ceiling(perTokenMax, now),
1235
+ persistedFloodOpen: persistedFloodRemainingMs(now) > 0,
458
1236
  }
459
1237
  }
460
1238
 
@@ -470,7 +1248,13 @@ export type EditFloodFuse = ReturnType<typeof createEditFloodFuse>
470
1248
  export type FuseInstallable = Bot
471
1249
 
472
1250
  export function installEditFloodFuse(bot: FuseInstallable, config: EditFloodFuseConfig = {}): EditFloodFuse {
473
- const fuse = createEditFloodFuse(config)
1251
+ // The token window is keyed by the bot's PUBLIC id only. `deriveBotScopeKey`
1252
+ // never returns any part of the secret half, and the key is used solely as an
1253
+ // in-memory Map key — never logged, never persisted.
1254
+ const fuse = createEditFloodFuse({
1255
+ botScopeKey: deriveBotScopeKey((bot as { token?: string }).token),
1256
+ ...config,
1257
+ })
474
1258
  bot.api.config.use(async (prev, method, payload, signal) =>
475
1259
  fuse.apply(method, payload, () => prev(method, payload, signal)))
476
1260
  return fuse