switchroom 0.19.26 → 0.19.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/git-agent-attribution-hook.sh +144 -0
- package/dist/agent-scheduler/index.js +60 -2
- package/dist/auth-broker/index.js +244 -13
- package/dist/cli/autoaccept-poll.js +225 -17
- package/dist/cli/notion-write-pretool.mjs +60 -2
- package/dist/cli/switchroom.js +2843 -1220
- package/dist/host-control/main.js +245 -14
- package/dist/vault/approvals/kernel-server.js +242 -13
- package/dist/vault/broker/server.js +242 -13
- package/package.json +7 -2
- package/profiles/_base/cron-session.sh.hbs +8 -0
- package/profiles/_base/start.sh.hbs +175 -15
- package/telegram-plugin/card-layout.ts +328 -0
- package/telegram-plugin/dist/bridge/bridge.js +94 -1
- package/telegram-plugin/dist/gateway/gateway.js +2544 -1182
- package/telegram-plugin/dist/server.js +97 -1
- package/telegram-plugin/edit-flood-fuse.ts +841 -57
- package/telegram-plugin/flood-429-ledger.ts +526 -0
- package/telegram-plugin/flood-circuit-breaker.ts +18 -0
- package/telegram-plugin/gateway/callback-query-handlers.ts +6 -0
- package/telegram-plugin/gateway/flood-reply-queue.ts +168 -0
- package/telegram-plugin/gateway/gateway.ts +67 -70
- package/telegram-plugin/gateway/mcp-failure-hook.ts +74 -0
- package/telegram-plugin/gateway/narrative-lane.ts +14 -0
- package/telegram-plugin/gateway/outbound-send-path.ts +36 -0
- package/telegram-plugin/gateway/outbox-sweep.ts +183 -6
- package/telegram-plugin/gateway/pinned-message-handler.ts +12 -16
- package/telegram-plugin/gateway/status-pin-retarget.ts +72 -36
- package/telegram-plugin/gateway/status-pin-store.ts +58 -9
- package/telegram-plugin/gateway/worker-pin-reaper.ts +56 -7
- package/telegram-plugin/inline-keyboard-callbacks.ts +202 -21
- package/telegram-plugin/llm-error-present.ts +61 -2
- package/telegram-plugin/mcp-credential-failure.ts +459 -0
- package/telegram-plugin/model-unavailable.ts +8 -0
- package/telegram-plugin/operator-events.ts +110 -5
- package/telegram-plugin/outbound-class.ts +81 -0
- package/telegram-plugin/provider-credit.ts +237 -0
- package/telegram-plugin/scripts/bun-test-ci.sh +36 -6
- package/telegram-plugin/send-gate.ts +24 -2
- package/telegram-plugin/status-no-truncate.ts +10 -48
- package/telegram-plugin/status-pin-driver.ts +33 -45
- package/telegram-plugin/status-pin.ts +18 -1
- package/telegram-plugin/tests/card-golden.test.ts +69 -0
- package/telegram-plugin/tests/card-lifecycle-render.test.ts +362 -0
- package/telegram-plugin/tests/card-type-distinguishability.test.ts +187 -164
- package/telegram-plugin/tests/card-variants.golden.txt +211 -0
- package/telegram-plugin/tests/card-variants.ts +366 -0
- package/telegram-plugin/tests/edit-flood-fuse-ban-awareness.test.ts +373 -0
- package/telegram-plugin/tests/edit-flood-fuse-default-deny.test.ts +319 -0
- package/telegram-plugin/tests/edit-flood-fuse-reply-reserve.test.ts +340 -0
- package/telegram-plugin/tests/edit-flood-fuse.test.ts +11 -2
- package/telegram-plugin/tests/feed-edit-rate-ceiling.test.ts +462 -0
- package/telegram-plugin/tests/finalize-callback-flood-policy.test.ts +298 -0
- package/telegram-plugin/tests/finalize-callback.test.ts +41 -8
- package/telegram-plugin/tests/fixtures/real-429-stream.ts +220 -0
- package/telegram-plugin/tests/flood-429-ledger.test.ts +278 -0
- package/telegram-plugin/tests/flood-429-recorder-wiring.test.ts +128 -0
- package/telegram-plugin/tests/flood-reply-queue.test.ts +418 -0
- package/telegram-plugin/tests/mcp-credential-failure.test.ts +310 -0
- package/telegram-plugin/tests/outbox-sweep-flood-breaker.test.ts +221 -0
- package/telegram-plugin/tests/pinned-card-collapse.test.ts +19 -24
- package/telegram-plugin/tests/pinned-message-handler.test.ts +15 -15
- package/telegram-plugin/tests/provider-credit-402.test.ts +243 -0
- package/telegram-plugin/tests/status-pin-api.test.ts +11 -11
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +36 -37
- package/telegram-plugin/tests/status-pin-lifecycle.test.ts +602 -0
- package/telegram-plugin/tests/status-pin-retarget.test.ts +90 -62
- package/telegram-plugin/tests/status-pin-service-message-suppression.test.ts +7 -3
- package/telegram-plugin/tests/status-pin-store.test.ts +109 -60
- package/telegram-plugin/tests/status-pin.test.ts +56 -5
- package/telegram-plugin/tests/test-runner-coverage.test.ts +133 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +12 -10
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +23 -29
- package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +56 -59
- package/telegram-plugin/tests/worker-feed-terminal-edit-class.test.ts +335 -0
- package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +1 -1
- package/telegram-plugin/tool-activity-summary.ts +239 -365
- package/telegram-plugin/uat/assertions.ts +22 -11
- package/telegram-plugin/uat/feed-matcher.test.ts +24 -17
- package/telegram-plugin/worker-activity-feed.ts +105 -47
- package/vendor/hindsight-memory/CLAUDE.md +45 -0
- package/vendor/hindsight-memory/scripts/drain_pending.py +433 -11
- package/vendor/hindsight-memory/scripts/lib/config.py +33 -0
- package/vendor/hindsight-memory/scripts/lib/pending.py +193 -28
- package/vendor/hindsight-memory/scripts/recall.py +176 -7
- package/vendor/hindsight-memory/scripts/tests/test_config_recall_passthrough_env.py +170 -0
- package/vendor/hindsight-memory/scripts/tests/test_drain_circuit_breaker.py +401 -0
- package/vendor/hindsight-memory/scripts/tests/test_drain_serialisation.py +286 -0
- package/vendor/hindsight-memory/scripts/tests/test_pending_drops.py +817 -8
- package/vendor/hindsight-memory/scripts/tests/test_recall_min_score.py +464 -0
- package/vendor/hindsight-memory/settings.json +1 -1
- package/vendor/hindsight-memory/tests/test_hooks.py +11 -2
|
@@ -56,11 +56,12 @@
|
|
|
56
56
|
* - send gate `editFloorMs` = 1500ms ⇒ up to 40 edits/min/message
|
|
57
57
|
* - send gate cosmetic budget 150 / 300s ⇒ 30 edits/min/message
|
|
58
58
|
* - worker feed's own floor 2500ms ⇒ 24 edits/min/message
|
|
59
|
+
* - gateway `FEED_HEARTBEAT_TICK_MS` = 6000ms ⇒ 10 repaints/min/turn
|
|
59
60
|
*
|
|
60
|
-
* The
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
61
|
+
* The fuse's cosmetic ceilings sit BELOW all of these. That is deliberate —
|
|
62
|
+
* Telegram's own per-group limit is ~20 messages/minute and edits count
|
|
63
|
+
* against it, so the *legitimate* cadences are themselves above what
|
|
64
|
+
* Telegram tolerates; that is precisely how `overlord` got banned while
|
|
64
65
|
* every in-repo pacer believed it was behaving. The fuse is therefore the
|
|
65
66
|
* BINDING constraint on a hot card, not a never-reached backstop.
|
|
66
67
|
*
|
|
@@ -74,12 +75,150 @@
|
|
|
74
75
|
* RELEASED late instead of dropped. Dropping it would freeze the card
|
|
75
76
|
* mid-run AND return `true` to the send gate, which would then record a
|
|
76
77
|
* never-painted payload as on-screen and no-op-skip every retry.
|
|
78
|
+
*
|
|
79
|
+
* ── 2026-07-27: the fuse existed and the ban happened anyway ──────────────
|
|
80
|
+
* The ceilings above were sized against the in-repo pacers, not against what
|
|
81
|
+
* Telegram actually tolerates. Agent `overlord` then sustained ~17
|
|
82
|
+
* `editMessageText`/min on ONE DM chat for hours (326 edits against 6 real
|
|
83
|
+
* replies in the final 20 minutes) and took a **15908-second** flood ban that
|
|
84
|
+
* severed every outbound reply. Both original ceilings — 20 edits/60s per
|
|
85
|
+
* message, 30 edits/60s per chat — sat ABOVE that observed rate, so the fuse
|
|
86
|
+
* never bound once. Three structural changes follow, and each is a separate,
|
|
87
|
+
* independently-sufficient reason the same incident cannot recur:
|
|
88
|
+
*
|
|
89
|
+
* 1. **Class awareness.** The fuse now reads the send gate's priority class
|
|
90
|
+
* off `outbound-class.ts` (AsyncLocalStorage). `cosmetic` traffic — every
|
|
91
|
+
* activity/worker/liveness repaint — is governed by its own, much tighter
|
|
92
|
+
* ceilings; `useful` / `critical` edits (approval cards, answers) are not
|
|
93
|
+
* throttled by them. Previously the fuse was class-blind and had to pick
|
|
94
|
+
* one number for both, which is why the number was too high.
|
|
95
|
+
* 2. **A hard cosmetic rate ceiling.** 4 edits/60s per message (one per 15s,
|
|
96
|
+
* matching the worker feed's own `elapsedRefreshMs`) and 6 edits/60s per
|
|
97
|
+
* chat across ALL cosmetic surfaces. 6/min is below the ~4-6/min band the
|
|
98
|
+
* live chat survived for hours and far below the 15-17/min that earned
|
|
99
|
+
* the ban. Coalescing (supersede) means the card still shows CURRENT
|
|
100
|
+
* state at every permitted edit — the operator loses refresh frequency,
|
|
101
|
+
* never accuracy.
|
|
102
|
+
* 3. **A shared per-chat budget with a reply reservation.** Telegram meters
|
|
103
|
+
* sends and edits against the SAME per-chat allowance, so separate
|
|
104
|
+
* edit/send windows could each be "in budget" while their sum was not.
|
|
105
|
+
* One `perChatTotalMaxPerWindow` window (default 20/60s) counts every
|
|
106
|
+
* chat-targeted call, and `perChatReplyReserve` (default 8) slots of it
|
|
107
|
+
* are unreachable by `cosmetic` traffic. A reply therefore cannot be
|
|
108
|
+
* starved by repaints even if a future call site is misclassified.
|
|
109
|
+
*
|
|
110
|
+
* ── 2026-07-28: the meter was reading ~58% of the traffic (#3855) ─────────
|
|
111
|
+
* Point 3 above USED to claim `perChatTotalMaxPerWindow` "counts every
|
|
112
|
+
* admitted call" and "matches Telegram's own per-chat metering". Both were
|
|
113
|
+
* false, and the gap is the reason a "20/60s" ceiling did not stop a ban.
|
|
114
|
+
* `apply` opened with `if (!isEdit && !isSend) return runObserved(next)` —
|
|
115
|
+
* an ALLOWLIST. Anything outside `EDIT_METHODS`/`SEND_METHODS` was entirely
|
|
116
|
+
* unmetered: it took no slot, and the ceiling therefore governed only the
|
|
117
|
+
* subset of traffic it happened to recognise.
|
|
118
|
+
*
|
|
119
|
+
* Measured on overlord's own gateway log (46791 `tg-post` lines), the
|
|
120
|
+
* unmetered share was 29% overall and 42% inside the 30 minutes before the
|
|
121
|
+
* 20:19:56 ban. By method:
|
|
122
|
+
*
|
|
123
|
+
* sendChatAction 10457 ← unmetered, and up to 18/min on ONE DM
|
|
124
|
+
* setMessageReaction 2098 ← unmetered
|
|
125
|
+
* deleteMessage 370 ← unmetered
|
|
126
|
+
* pinChatMessage 366 ← unmetered
|
|
127
|
+
* unpinChatMessage 365 ← unmetered
|
|
128
|
+
* answerCallbackQuery 31 ← unmetered (carries no chat_id)
|
|
129
|
+
*
|
|
130
|
+
* `sendChatAction` alone ran at Telegram's entire documented per-chat/minute
|
|
131
|
+
* allowance while the fuse counted none of it. So the fix is not a smaller
|
|
132
|
+
* number, it is an honest meter:
|
|
133
|
+
*
|
|
134
|
+
* 4. **DEFAULT-DENY metering.** Every method whose payload carries a
|
|
135
|
+
* `chat_id` is charged to `perChatTotalMaxPerWindow`, whether or not this
|
|
136
|
+
* file has heard of it. There is no per-chat exemption list to forget to
|
|
137
|
+
* update — a new Bot API method, or one this gateway starts calling
|
|
138
|
+
* tomorrow (`sendChecklist` is exactly that case: it carries `chat_id`,
|
|
139
|
+
* creates a message, and was in neither set), is metered on arrival.
|
|
140
|
+
* Methods that are neither edits nor sends are PACED and never dropped,
|
|
141
|
+
* except `CHAT_ACTION_METHODS` — a typing indicator expires by itself in
|
|
142
|
+
* ~5s and carries no content, so shedding one loses nothing.
|
|
143
|
+
* The ceiling NUMBER is unchanged at 20/60s; what changed is that 20
|
|
144
|
+
* admitted calls now means 20 on the wire rather than ~34.
|
|
145
|
+
* 5. **A per-BOT-TOKEN window.** Telegram's flood ban is scoped to the bot
|
|
146
|
+
* token, not the chat, and enforces a global rate across all chats. A
|
|
147
|
+
* second window keyed by the bot's PUBLIC numeric id (`perTokenWindowMs`
|
|
148
|
+
* / `perTokenMaxPerWindow`, default 25 per 1000ms) is now charged for
|
|
149
|
+
* EVERY outbound call — including the chat-less ones like
|
|
150
|
+
* `answerCallbackQuery`, which the per-chat tier structurally cannot key.
|
|
151
|
+
* A call is admitted only if BOTH windows have room.
|
|
152
|
+
* LIMITATION, stated plainly: this window is per-PROCESS. Two agents
|
|
153
|
+
* sharing one bot token run in separate containers with separate state
|
|
154
|
+
* dirs, so neither sees the other's window. It is a real improvement on
|
|
155
|
+
* the single process that actually floods and it is NOT full coverage
|
|
156
|
+
* under a shared token; a cross-process view would need a token-keyed
|
|
157
|
+
* store on a path both processes can write, which does not exist today.
|
|
158
|
+
* Only the public bot id (the prefix before the `:`) is ever used as a
|
|
159
|
+
* key, and it is never logged — see {@link deriveBotScopeKey}.
|
|
160
|
+
*
|
|
161
|
+
* Plus **escalating** backoff: a 429 used to apply ONE 0.5× tightening for a
|
|
162
|
+
* flat 10 minutes, so a stream that took repeated small 429s re-tightened to
|
|
163
|
+
* the same level forever and walked into the big ban. Tightening is now
|
|
164
|
+
* multiplicative PER 429 (`tightenFactor ^ level`, level capped by
|
|
165
|
+
* `maxTightenLevel`) and decays one level at a time, so sustained pressure
|
|
166
|
+
* ratchets the cosmetic rate down towards the floor of 1/window.
|
|
167
|
+
*
|
|
168
|
+
* ── 2026-07-28: the reserve could not fire, and the reply paid (#3885) ────
|
|
169
|
+
* Points 3 and 5 above, and the escalating backoff, combined into a defect
|
|
170
|
+
* that hit the operator directly: during a 12-agent staggered restart that drew
|
|
171
|
+
* **three** genuine 429s, one chat logged 792 fuse events in an hour against a
|
|
172
|
+
* 35-50/hr baseline, shed 221 typing indicators, deferred 18 reactions, and
|
|
173
|
+
* deferred real replies long enough that the MCP `reply` tool's 60s timeout
|
|
174
|
+
* fired — so the agent retried and the operator got the same answer twice. One
|
|
175
|
+
* ~1900-character reply was lost outright.
|
|
176
|
+
*
|
|
177
|
+
* Three 3-second 429s ratchet `tightenLevel` to 3, and each one also writes
|
|
178
|
+
* `flood-wait.json`, which pinned `levelAt` at `maxTightenLevel` outright. At
|
|
179
|
+
* level 4 the shared per-chat budget is `max(1, floor(20 * 0.5^4))` = **1 call
|
|
180
|
+
* per minute for the entire chat**. The reserve that exists for precisely this
|
|
181
|
+
* case could not help, because it was a fixed count subtracted from the BASE:
|
|
182
|
+
* `max(1, 20 - 8)` = 12, tightened to 1 — the same 1 the reply had. Reply and
|
|
183
|
+
* repaint ended up with an identical budget, which is the opposite of a
|
|
184
|
+
* reservation. Three changes:
|
|
185
|
+
*
|
|
186
|
+
* 6. **The reserve is a PROPORTION, re-derived per window.** `cosmeticTotalMax`
|
|
187
|
+
* computes it against the EFFECTIVE ceiling, so "8 of 20" means "40% of
|
|
188
|
+
* whatever the budget currently is", and at least one slot is always
|
|
189
|
+
* reserved. When the effective budget shrinks to the reserve, cosmetic
|
|
190
|
+
* traffic gets 0 and every remaining slot belongs to a reply.
|
|
191
|
+
* 7. **A hard floor under `critical`.** `perChatCriticalMinPerWindow`
|
|
192
|
+
* (default 3) is a rate no tightening may cross, on every per-chat tier.
|
|
193
|
+
* Shedding a typing indicator under pressure is correct; holding the
|
|
194
|
+
* answer past the caller's own timeout is not — it does not save a call,
|
|
195
|
+
* it doubles it.
|
|
196
|
+
* 8. **A persisted flood window tightens IN PROPORTION to what remains**,
|
|
197
|
+
* through the same severity ladder `noteFlood` uses, instead of jumping
|
|
198
|
+
* straight to `maxTightenLevel`. `makeFloodWaitRecorder` writes the marker
|
|
199
|
+
* for every 429 including a 3-second nudge, so the flat jump reintroduced
|
|
200
|
+
* on the persisted path exactly the "a nudge and a 4.4h ban are the same
|
|
201
|
+
* signal" defect that #3856 had just fixed on the in-memory path.
|
|
202
|
+
*
|
|
203
|
+
* Every ceiling is operator-overridable via env — see
|
|
204
|
+
* {@link editFloodFuseConfigFromEnv}. Previously none of them were, and until
|
|
205
|
+
* #3885 the tightening CURVE (`maxTightenLevel`, `tightenFactor`) still was
|
|
206
|
+
* not, which is why the only available workaround for the above was to inflate
|
|
207
|
+
* the base ceiling fleet-wide.
|
|
77
208
|
*/
|
|
78
209
|
|
|
210
|
+
import { createHash } from 'node:crypto'
|
|
211
|
+
|
|
79
212
|
import type { Bot } from 'grammy'
|
|
80
213
|
|
|
81
214
|
import type { Clock } from './send-gate.js'
|
|
82
215
|
import { systemClock } from './send-gate.js'
|
|
216
|
+
import { floodStatePath, makeFloodWaitProbe } from './flood-circuit-breaker.js'
|
|
217
|
+
import {
|
|
218
|
+
currentOutboundClass,
|
|
219
|
+
defaultOutboundClass,
|
|
220
|
+
type OutboundClass,
|
|
221
|
+
} from './outbound-class.js'
|
|
83
222
|
|
|
84
223
|
/** Methods that mutate an EXISTING message — droppable, coalescible. */
|
|
85
224
|
const EDIT_METHODS: ReadonlySet<string> = new Set([
|
|
@@ -112,26 +251,185 @@ const SEND_METHODS: ReadonlySet<string> = new Set([
|
|
|
112
251
|
'sendRichMessage',
|
|
113
252
|
])
|
|
114
253
|
|
|
254
|
+
/**
|
|
255
|
+
* Chat-targeted methods whose effect EXPIRES on its own and carries no content,
|
|
256
|
+
* so shedding one under pressure loses nothing a user would notice.
|
|
257
|
+
*
|
|
258
|
+
* `sendChatAction` is the entire set and the reason this tier exists: it was
|
|
259
|
+
* the single largest unmetered method in the incident log (10457 calls, up to
|
|
260
|
+
* 18/min on ONE DM — Telegram's whole documented per-chat/minute allowance),
|
|
261
|
+
* and the typing status it sets clears itself after ~5 seconds regardless.
|
|
262
|
+
* Every OTHER non-edit, non-send method with a `chat_id` (`deleteMessage`,
|
|
263
|
+
* `pinChatMessage`, `setMessageReaction`, …) has a durable, user-visible
|
|
264
|
+
* effect and is paced but never dropped.
|
|
265
|
+
*/
|
|
266
|
+
const CHAT_ACTION_METHODS: ReadonlySet<string> = new Set(['sendChatAction'])
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* The ONLY methods exempt from metering entirely. Deliberately one entry.
|
|
270
|
+
*
|
|
271
|
+
* `getUpdates` is the long-poll RECEIVE loop, not outbound traffic: it is how
|
|
272
|
+
* inbound messages arrive at all. Pacing it would stall message delivery
|
|
273
|
+
* rather than protect anything, and it targets no chat, so it consumes no
|
|
274
|
+
* per-chat budget by construction.
|
|
275
|
+
*
|
|
276
|
+
* Nothing else is listed. The one-off administrative calls (`setMyCommands`,
|
|
277
|
+
* `deleteWebhook`, `getFile`, `getChat` — 37 calls TOTAL across a 46791-call
|
|
278
|
+
* log) are metered like everything else: at 25/second the cost is nil, and
|
|
279
|
+
* exempting them would mean asserting something about Telegram's accounting
|
|
280
|
+
* that this repo cannot verify from first-party evidence. Default-deny means
|
|
281
|
+
* the burden of proof is on the exemption.
|
|
282
|
+
*/
|
|
283
|
+
const UNMETERED_METHODS: ReadonlySet<string> = new Set(['getUpdates'])
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* Reduce a bot token to a key safe to hold in memory.
|
|
287
|
+
*
|
|
288
|
+
* A Telegram token is `<bot_id>:<secret>`. The bot id is PUBLIC (it is the
|
|
289
|
+
* bot's user id, visible on every message it sends); the secret half is not.
|
|
290
|
+
* This returns the bot id alone when the token has that shape, so nothing
|
|
291
|
+
* secret is ever used as a map key, logged, or persisted. A token that does
|
|
292
|
+
* not match the shape falls back to a truncated SHA-256, which is stable
|
|
293
|
+
* across processes and irreversible.
|
|
294
|
+
*/
|
|
295
|
+
export function deriveBotScopeKey(token: string | undefined): string {
|
|
296
|
+
if (token == null || token === '') return 'unknown'
|
|
297
|
+
const id = token.slice(0, token.indexOf(':'))
|
|
298
|
+
if (/^\d+$/.test(id)) return id
|
|
299
|
+
return `h${createHash('sha256').update(token).digest('hex').slice(0, 16)}`
|
|
300
|
+
}
|
|
301
|
+
|
|
115
302
|
export interface EditFloodFuseConfig {
|
|
116
303
|
/** Master switch. When false, `apply` is a pure passthrough. Default true. */
|
|
117
304
|
enabled?: boolean
|
|
118
305
|
clock?: Clock
|
|
119
|
-
/**
|
|
306
|
+
/**
|
|
307
|
+
* Hard ceiling on NON-cosmetic (`useful` / `critical`) edits to ONE message
|
|
308
|
+
* id. Default 20 per 60s. Approval cards and answer finalisations live here;
|
|
309
|
+
* they are low-cadence by nature, so this stays a backstop, not a pacer.
|
|
310
|
+
*/
|
|
120
311
|
perMessageMaxPerWindow?: number
|
|
121
312
|
perMessageWindowMs?: number
|
|
122
|
-
/** Hard ceiling on edits to ONE chat across all messages. Default 30 per 60s. */
|
|
313
|
+
/** Hard ceiling on ALL edits to ONE chat across all messages. Default 30 per 60s. */
|
|
123
314
|
perChatEditMaxPerWindow?: number
|
|
124
315
|
/** Pacing ceiling on non-edit sends to ONE chat. Default 25 per 60s. */
|
|
125
316
|
perChatSendMaxPerWindow?: number
|
|
317
|
+
/**
|
|
318
|
+
* ADDITIONAL ceiling applied to COSMETIC edits of ONE message id, on top of
|
|
319
|
+
* `perMessageMaxPerWindow` (the effective ceiling is the lower of the two).
|
|
320
|
+
* Default 4 per 60s — one per 15s, matching the worker feed's
|
|
321
|
+
* `elapsedRefreshMs`. This is the number that bounds a hot progress card.
|
|
322
|
+
*/
|
|
323
|
+
cosmeticPerMessageMaxPerWindow?: number
|
|
324
|
+
/**
|
|
325
|
+
* ADDITIONAL ceiling applied to COSMETIC edits in ONE chat, summed across
|
|
326
|
+
* every cosmetic surface in it (lower of this and `perChatEditMaxPerWindow`
|
|
327
|
+
* binds). Default 6 per 60s. Without it, N concurrent cards each inside their
|
|
328
|
+
* own per-message ceiling still add up to a flood.
|
|
329
|
+
*/
|
|
330
|
+
cosmeticPerChatMaxPerWindow?: number
|
|
331
|
+
/**
|
|
332
|
+
* Shared per-chat allowance counting EVERY admitted call — edits and sends,
|
|
333
|
+
* every class. Default 20 per 60s, matching Telegram's own per-chat/group
|
|
334
|
+
* metering (which does not separate the two). Sends and non-cosmetic edits
|
|
335
|
+
* are paced against it and never dropped.
|
|
336
|
+
*/
|
|
337
|
+
perChatTotalMaxPerWindow?: number
|
|
338
|
+
/**
|
|
339
|
+
* Slots of `perChatTotalMaxPerWindow` that `cosmetic` traffic may NEVER
|
|
340
|
+
* consume. Default 8. This is the reply-starvation guarantee: whatever the
|
|
341
|
+
* repaint surfaces do, at least this many calls per window remain available
|
|
342
|
+
* to an actual answer.
|
|
343
|
+
*
|
|
344
|
+
* Read as a PROPORTION of the base, not an absolute count (#3885). The
|
|
345
|
+
* reserve used to be a fixed number subtracted from the base ceiling, which
|
|
346
|
+
* made it arithmetically dead under tightening: at `maxTightenLevel` the base
|
|
347
|
+
* 20 collapses to an effective 1, and `1 - 8` clamps to the same floor of 1
|
|
348
|
+
* that cosmetic traffic already had — so reply and repaint ended up with an
|
|
349
|
+
* identical budget in exactly the situation the reserve exists for. It is now
|
|
350
|
+
* re-derived against the EFFECTIVE ceiling every window, and always leaves at
|
|
351
|
+
* least one slot reserved.
|
|
352
|
+
*/
|
|
353
|
+
perChatReplyReserve?: number
|
|
354
|
+
/**
|
|
355
|
+
* Absolute floor, per `perChatWindowMs`, on `critical`-class traffic in one
|
|
356
|
+
* chat — the operator's actual answer, an approval card, a reaction. Default
|
|
357
|
+
* 3. No amount of tightening may take a chat below this: a reply held past
|
|
358
|
+
* the caller's own timeout is not "shedding under pressure", it is a lost
|
|
359
|
+
* answer plus a retry that sends it twice (#3885). Cosmetic traffic has no
|
|
360
|
+
* floor and is still shed to zero, which is the correct trade.
|
|
361
|
+
*
|
|
362
|
+
* Capped by the tier's own base ceiling, so an operator who deliberately
|
|
363
|
+
* configures a base below this floor still gets what they configured.
|
|
364
|
+
*/
|
|
365
|
+
perChatCriticalMinPerWindow?: number
|
|
366
|
+
/**
|
|
367
|
+
* Ceiling on EVERY outbound call made with this bot token, across all chats
|
|
368
|
+
* and including the chat-less ones. Default 25 per `perTokenWindowMs`
|
|
369
|
+
* (1000ms), just under the ~30/second Telegram enforces at the token level.
|
|
370
|
+
* The flood ban is token-scoped, so this is the tier that matches the thing
|
|
371
|
+
* that actually gets banned. Per-PROCESS — see the docblock's point 5.
|
|
372
|
+
*/
|
|
373
|
+
perTokenMaxPerWindow?: number
|
|
374
|
+
perTokenWindowMs?: number
|
|
375
|
+
/**
|
|
376
|
+
* Scope key for the token window: the bot's PUBLIC numeric id. Never pass a
|
|
377
|
+
* raw token — {@link installEditFloodFuse} derives this via
|
|
378
|
+
* {@link deriveBotScopeKey}. Default `'unknown'` (a single shared window,
|
|
379
|
+
* which is still correct for the one-bot-per-process case).
|
|
380
|
+
*/
|
|
381
|
+
botScopeKey?: string
|
|
382
|
+
/**
|
|
383
|
+
* Longest a `CHAT_ACTION_METHODS` call may be held before it is shed.
|
|
384
|
+
* Default 3000ms — a typing indicator that arrives after the status it
|
|
385
|
+
* describes has already expired is worse than no typing indicator.
|
|
386
|
+
*/
|
|
387
|
+
chatActionMaxDeferMs?: number
|
|
388
|
+
/**
|
|
389
|
+
* Cap on COSMETIC edits that may be released late (over budget) because
|
|
390
|
+
* nothing newer will repaint them — the bounded form of the R1 rule. Default
|
|
391
|
+
* 2 per `perChatWindowMs`, so the worst-case sustained cosmetic rate is
|
|
392
|
+
* `cosmeticPerChatMaxPerWindow + lateReleaseMaxPerWindow`.
|
|
393
|
+
*/
|
|
394
|
+
lateReleaseMaxPerWindow?: number
|
|
126
395
|
perChatWindowMs?: number
|
|
127
396
|
/** Longest a call may be held before it is dropped (edit) / released (send). Default 30s. */
|
|
128
397
|
maxDeferMs?: number
|
|
129
|
-
/** Multiplicative decrease applied
|
|
398
|
+
/** Multiplicative decrease applied per observed 429. Default 0.5. */
|
|
130
399
|
tightenFactor?: number
|
|
131
|
-
/** How long
|
|
400
|
+
/** How long ONE level of tightening stays in force before decaying. Default 10 minutes. */
|
|
132
401
|
tightenMs?: number
|
|
402
|
+
/**
|
|
403
|
+
* Cap on compounding 429 tightening levels. Default 4 (⇒ 0.5^4 = 1/16).
|
|
404
|
+
* Operator-overridable via `SWITCHROOM_EDIT_FUSE_MAX_TIGHTEN_LEVEL` (#3885);
|
|
405
|
+
* `0` disables tightening entirely.
|
|
406
|
+
*/
|
|
407
|
+
maxTightenLevel?: number
|
|
408
|
+
/**
|
|
409
|
+
* Remaining ms of a KNOWN-OPEN Telegram flood window, or 0 when none — the
|
|
410
|
+
* persisted breaker state (`flood-circuit-breaker.ts`) that every other
|
|
411
|
+
* outbound path already consults. Wired by {@link editFloodFuseConfigFromEnv}.
|
|
412
|
+
*
|
|
413
|
+
* The fuse's tightening lived only in process memory, so before #3856 a
|
|
414
|
+
* gateway restart during a ban came back FULLY UNTIGHTENED and resumed at
|
|
415
|
+
* the rate that earned the ban — and a restart is the single most likely
|
|
416
|
+
* thing to happen during a multi-hour outage. Consulting the persisted
|
|
417
|
+
* window makes the response durable: while it is open the fuse holds
|
|
418
|
+
* `maxTightenLevel` regardless of what this process remembers.
|
|
419
|
+
*
|
|
420
|
+
* FAILS OPEN (throwing or absent probe ⇒ untightened): a breaker that cannot
|
|
421
|
+
* read its own state must never gag the bot.
|
|
422
|
+
*/
|
|
423
|
+
floodWaitRemainingMs?: () => number
|
|
424
|
+
/** How often the persisted window may be re-read. Default 1000ms. */
|
|
425
|
+
floodProbeIntervalMs?: number
|
|
133
426
|
/** Observability hook; fired whenever the fuse binds. */
|
|
134
|
-
onTrip?: (info: {
|
|
427
|
+
onTrip?: (info: {
|
|
428
|
+
method: string
|
|
429
|
+
key: string
|
|
430
|
+
action: 'deferred' | 'dropped' | 'superseded'
|
|
431
|
+
cls: OutboundClass
|
|
432
|
+
}) => void
|
|
135
433
|
}
|
|
136
434
|
|
|
137
435
|
export interface EditFloodFuseStats {
|
|
@@ -146,8 +444,41 @@ export interface EditFloodFuseStats {
|
|
|
146
444
|
floodObserved: number
|
|
147
445
|
/** Whether a tightened ceiling is in force right now. */
|
|
148
446
|
tightened: boolean
|
|
149
|
-
/**
|
|
447
|
+
/** Compounding 429 tightening level currently in force (0 = untightened). */
|
|
448
|
+
tightenLevel: number
|
|
449
|
+
/** Live NON-cosmetic per-message ceiling (post-AIMD). */
|
|
150
450
|
perMessageCeiling: number
|
|
451
|
+
/** Live COSMETIC per-message ceiling (post-AIMD) — the incident-relevant one. */
|
|
452
|
+
cosmeticPerMessageCeiling: number
|
|
453
|
+
/** Live COSMETIC per-chat ceiling (post-AIMD). */
|
|
454
|
+
cosmeticPerChatCeiling: number
|
|
455
|
+
/**
|
|
456
|
+
* Live shared per-chat budget available to COSMETIC traffic (post-AIMD, after
|
|
457
|
+
* the reply reserve). Goes to 0 under heavy tightening — that is the reserve
|
|
458
|
+
* working, not a fault.
|
|
459
|
+
*/
|
|
460
|
+
cosmeticPerChatTotalCeiling: number
|
|
461
|
+
/**
|
|
462
|
+
* Live shared per-chat budget available to CRITICAL traffic (post-AIMD, after
|
|
463
|
+
* the critical floor). The direct measure of #3885: this must never drop
|
|
464
|
+
* below `perChatCriticalMinPerWindow`, at any tighten level.
|
|
465
|
+
*/
|
|
466
|
+
criticalPerChatTotalCeiling: number
|
|
467
|
+
/**
|
|
468
|
+
* Chat-targeted calls charged by the DEFAULT-DENY rule — every one of these
|
|
469
|
+
* was completely unmetered before #3855. A non-zero value here is the direct
|
|
470
|
+
* measure of the hole this closed.
|
|
471
|
+
*/
|
|
472
|
+
meteredByDefault: number
|
|
473
|
+
/** Calls with no `chat_id`, charged to the token window only. */
|
|
474
|
+
chatless: number
|
|
475
|
+
/** Live per-bot-token ceiling (post-AIMD). */
|
|
476
|
+
perTokenCeiling: number
|
|
477
|
+
/**
|
|
478
|
+
* True when a PERSISTED flood window is open right now — i.e. the tightening
|
|
479
|
+
* in force is durable rather than remembered, and survives a restart (#3856).
|
|
480
|
+
*/
|
|
481
|
+
persistedFloodOpen: boolean
|
|
151
482
|
}
|
|
152
483
|
|
|
153
484
|
export const EDIT_FLOOD_FUSE_DEFAULTS = {
|
|
@@ -155,12 +486,119 @@ export const EDIT_FLOOD_FUSE_DEFAULTS = {
|
|
|
155
486
|
perMessageWindowMs: 60_000,
|
|
156
487
|
perChatEditMaxPerWindow: 30,
|
|
157
488
|
perChatSendMaxPerWindow: 25,
|
|
489
|
+
/**
|
|
490
|
+
* 4/60s. One cosmetic repaint per 15s per card. Chosen to equal the worker
|
|
491
|
+
* feed's `elapsedRefreshMs` (15000) — the slowest cadence at which the feed
|
|
492
|
+
* itself considers a repaint worth making — so the fuse binds the runaway
|
|
493
|
+
* case without ever throttling the feed's own intended pace.
|
|
494
|
+
*/
|
|
495
|
+
cosmeticPerMessageMaxPerWindow: 4,
|
|
496
|
+
/**
|
|
497
|
+
* 6/60s across every cosmetic surface in a chat. The incident's own timeline
|
|
498
|
+
* is the evidence: 10-minute buckets of 58/53/41 edits (≈4-6/min) ran for
|
|
499
|
+
* hours without a ban; 115/78/73 then 152/174 (≈8-17/min) earned one. 6/min
|
|
500
|
+
* sits at the top of the survived band and less than half the banned rate.
|
|
501
|
+
*/
|
|
502
|
+
cosmeticPerChatMaxPerWindow: 6,
|
|
503
|
+
/**
|
|
504
|
+
* 20/60s. Telegram's documented per-group ceiling, and it meters sends and
|
|
505
|
+
* edits together — so this is the only window that reflects the real budget.
|
|
506
|
+
*/
|
|
507
|
+
perChatTotalMaxPerWindow: 20,
|
|
508
|
+
/**
|
|
509
|
+
* 8 of those 20 slots/min are unreachable by cosmetic traffic — i.e. 40% of
|
|
510
|
+
* whatever the EFFECTIVE ceiling is, re-derived per window (#3885).
|
|
511
|
+
*/
|
|
512
|
+
perChatReplyReserve: 8,
|
|
513
|
+
/**
|
|
514
|
+
* 3/60s. The hard floor on `critical` traffic in a chat. Sized against the
|
|
515
|
+
* thing that broke: the MCP `reply` tool times out at 60s, so a chat that can
|
|
516
|
+
* pass fewer than a couple of criticals a minute turns one answer into a
|
|
517
|
+
* timeout, a retry, and a duplicate message.
|
|
518
|
+
*/
|
|
519
|
+
perChatCriticalMinPerWindow: 3,
|
|
520
|
+
/**
|
|
521
|
+
* 25 per second across the whole bot token. Telegram enforces ~30/s at the
|
|
522
|
+
* token level and the ban it issues is token-scoped; 25 leaves headroom for
|
|
523
|
+
* traffic this process cannot see (a peer agent sharing the token, or a
|
|
524
|
+
* retry issued below the transformer stack).
|
|
525
|
+
*/
|
|
526
|
+
perTokenMaxPerWindow: 25,
|
|
527
|
+
perTokenWindowMs: 1_000,
|
|
528
|
+
/** A typing indicator held longer than this is not worth sending. */
|
|
529
|
+
chatActionMaxDeferMs: 3_000,
|
|
530
|
+
/** At most 2 cosmetic frames/min may exceed the ceiling as "lone" releases. */
|
|
531
|
+
lateReleaseMaxPerWindow: 2,
|
|
158
532
|
perChatWindowMs: 60_000,
|
|
159
533
|
maxDeferMs: 30_000,
|
|
160
534
|
tightenFactor: 0.5,
|
|
161
535
|
tightenMs: 600_000,
|
|
536
|
+
maxTightenLevel: 4,
|
|
537
|
+
/** The persisted flood window is re-read at most once a second (#3856). */
|
|
538
|
+
floodProbeIntervalMs: 1_000,
|
|
162
539
|
} as const
|
|
163
540
|
|
|
541
|
+
/** Parse a positive integer from env, or undefined when unset/invalid. */
|
|
542
|
+
function envInt(raw: string | undefined): number | undefined {
|
|
543
|
+
if (raw == null || raw.trim() === '') return undefined
|
|
544
|
+
const n = Number(raw)
|
|
545
|
+
return Number.isFinite(n) && n >= 0 ? Math.floor(n) : undefined
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/**
|
|
549
|
+
* Parse a multiplicative-decrease factor from env: a real in (0, 1].
|
|
550
|
+
*
|
|
551
|
+
* Rejects 0 and negatives (which would zero every ceiling) and anything above
|
|
552
|
+
* 1 (which would make a 429 LOOSEN the fuse). An out-of-range value is treated
|
|
553
|
+
* as unset rather than clamped — silently reinterpreting an operator's number
|
|
554
|
+
* is how a fuse ends up at a rate nobody chose.
|
|
555
|
+
*/
|
|
556
|
+
function envFactor(raw: string | undefined): number | undefined {
|
|
557
|
+
if (raw == null || raw.trim() === '') return undefined
|
|
558
|
+
const n = Number(raw)
|
|
559
|
+
return Number.isFinite(n) && n > 0 && n <= 1 ? n : undefined
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* Operator config surface. Before the 2026-07-27 incident the fuse had exactly
|
|
564
|
+
* one knob — `SWITCHROOM_EDIT_FUSE=0`, which turns the whole failsafe OFF —
|
|
565
|
+
* so an operator whose chat was being flooded had no way to tighten it and no
|
|
566
|
+
* way to loosen it for a chat that could take more. Every ceiling is now
|
|
567
|
+
* overridable; unset values keep the defaults above.
|
|
568
|
+
*/
|
|
569
|
+
export function editFloodFuseConfigFromEnv(
|
|
570
|
+
env: Record<string, string | undefined>,
|
|
571
|
+
): EditFloodFuseConfig {
|
|
572
|
+
const cfg: EditFloodFuseConfig = { enabled: env.SWITCHROOM_EDIT_FUSE !== '0' }
|
|
573
|
+
const assign = <K extends keyof EditFloodFuseConfig>(k: K, v: number | undefined): void => {
|
|
574
|
+
if (v !== undefined) (cfg[k] as number) = v
|
|
575
|
+
}
|
|
576
|
+
assign('cosmeticPerMessageMaxPerWindow', envInt(env.SWITCHROOM_FEED_EDIT_MAX_PER_MSG_PER_MIN))
|
|
577
|
+
assign('cosmeticPerChatMaxPerWindow', envInt(env.SWITCHROOM_FEED_EDIT_MAX_PER_CHAT_PER_MIN))
|
|
578
|
+
assign('perChatTotalMaxPerWindow', envInt(env.SWITCHROOM_CHAT_TOTAL_MAX_PER_MIN))
|
|
579
|
+
assign('perChatReplyReserve', envInt(env.SWITCHROOM_CHAT_REPLY_RESERVE))
|
|
580
|
+
assign('perChatCriticalMinPerWindow', envInt(env.SWITCHROOM_CHAT_CRITICAL_MIN_PER_MIN))
|
|
581
|
+
assign('perTokenMaxPerWindow', envInt(env.SWITCHROOM_TOKEN_MAX_PER_SEC))
|
|
582
|
+
assign('maxDeferMs', envInt(env.SWITCHROOM_EDIT_FUSE_MAX_DEFER_MS))
|
|
583
|
+
// #3885 — the tightening curve itself is now operator-tunable. Before this,
|
|
584
|
+
// `SWITCHROOM_CHAT_TOTAL_MAX_PER_MIN` was the ONLY lever on it, so an
|
|
585
|
+
// operator whose chats were collapsing under `factor^maxLevel` had to inflate
|
|
586
|
+
// the BASE ceiling fleet-wide to compensate — raising the untightened rate
|
|
587
|
+
// (the one that earns bans) to fix the tightened one.
|
|
588
|
+
assign('maxTightenLevel', envInt(env.SWITCHROOM_EDIT_FUSE_MAX_TIGHTEN_LEVEL))
|
|
589
|
+
assign('tightenFactor', envFactor(env.SWITCHROOM_EDIT_FUSE_TIGHTEN_FACTOR))
|
|
590
|
+
// #3856 — durable ban awareness. The fuse reads the SAME persisted marker
|
|
591
|
+
// (`flood-wait.json`) that `robustApiCall` and the outbox sweep consult, so a
|
|
592
|
+
// restart mid-ban comes back tightened instead of at full rate. Wired here
|
|
593
|
+
// rather than at the call site so no caller can forget it, and because
|
|
594
|
+
// `gateway.ts` is under a zero-slack line ratchet.
|
|
595
|
+
const stateDir = env.TELEGRAM_STATE_DIR
|
|
596
|
+
if (stateDir != null && stateDir !== '') {
|
|
597
|
+
cfg.floodWaitRemainingMs = makeFloodWaitProbe(floodStatePath(stateDir))
|
|
598
|
+
}
|
|
599
|
+
return cfg
|
|
600
|
+
}
|
|
601
|
+
|
|
164
602
|
/** grammY resolves an edit with `true` when there is nothing to return. */
|
|
165
603
|
const DROPPED_RESULT = true
|
|
166
604
|
|
|
@@ -189,28 +627,175 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
|
|
|
189
627
|
const perMessageWindowMs = config.perMessageWindowMs ?? D.perMessageWindowMs
|
|
190
628
|
const perChatEditMax = config.perChatEditMaxPerWindow ?? D.perChatEditMaxPerWindow
|
|
191
629
|
const perChatSendMax = config.perChatSendMaxPerWindow ?? D.perChatSendMaxPerWindow
|
|
630
|
+
// Cosmetic tiers are an ADDITIONAL constraint on the same window keys, so the
|
|
631
|
+
// effective ceiling is the lower of the two. Folding them in here (rather
|
|
632
|
+
// than as extra tiers) keeps the admission path at the same number of awaits.
|
|
633
|
+
const cosmeticPerMessageMax = Math.min(
|
|
634
|
+
perMessageMax, config.cosmeticPerMessageMaxPerWindow ?? D.cosmeticPerMessageMaxPerWindow)
|
|
635
|
+
const cosmeticPerChatMax = Math.min(
|
|
636
|
+
perChatEditMax, config.cosmeticPerChatMaxPerWindow ?? D.cosmeticPerChatMaxPerWindow)
|
|
637
|
+
const perChatTotalMax = config.perChatTotalMaxPerWindow ?? D.perChatTotalMaxPerWindow
|
|
192
638
|
const perChatWindowMs = config.perChatWindowMs ?? D.perChatWindowMs
|
|
639
|
+
// Clamped so a misconfigured reserve can never eat more than the whole
|
|
640
|
+
// budget. Note this clamp is on the BASE only — the reserve that actually
|
|
641
|
+
// binds is re-derived per window by `cosmeticTotalMax` (#3885).
|
|
642
|
+
const perChatReplyReserve = Math.min(
|
|
643
|
+
Math.max(0, config.perChatReplyReserve ?? D.perChatReplyReserve),
|
|
644
|
+
Math.max(0, perChatTotalMax - 1),
|
|
645
|
+
)
|
|
646
|
+
/**
|
|
647
|
+
* The reserve as a FRACTION of the base. This is the whole #3885 fix: the
|
|
648
|
+
* reserve has to mean "40% of the chat's budget belongs to replies", not "8
|
|
649
|
+
* calls", because 8 is meaningless once the effective budget is 1.
|
|
650
|
+
*/
|
|
651
|
+
const replyReserveFraction = perChatTotalMax > 0 ? perChatReplyReserve / perChatTotalMax : 0
|
|
652
|
+
const perChatCriticalMin = Math.max(
|
|
653
|
+
0, config.perChatCriticalMinPerWindow ?? D.perChatCriticalMinPerWindow)
|
|
654
|
+
const perTokenMax = Math.max(1, config.perTokenMaxPerWindow ?? D.perTokenMaxPerWindow)
|
|
655
|
+
const perTokenWindowMs = Math.max(1, config.perTokenWindowMs ?? D.perTokenWindowMs)
|
|
656
|
+
// Only ever the PUBLIC bot id (or an irreversible hash) reaches this key.
|
|
657
|
+
const tokenKey = `g:${config.botScopeKey ?? 'unknown'}`
|
|
658
|
+
const chatActionMaxDeferMs = config.chatActionMaxDeferMs ?? D.chatActionMaxDeferMs
|
|
659
|
+
const lateReleaseMax = Math.max(0, config.lateReleaseMaxPerWindow ?? D.lateReleaseMaxPerWindow)
|
|
193
660
|
const maxDeferMs = config.maxDeferMs ?? D.maxDeferMs
|
|
194
661
|
const tightenFactor = config.tightenFactor ?? D.tightenFactor
|
|
195
662
|
const tightenMs = config.tightenMs ?? D.tightenMs
|
|
663
|
+
const maxTightenLevel = Math.max(0, config.maxTightenLevel ?? D.maxTightenLevel)
|
|
664
|
+
const floodProbe = config.floodWaitRemainingMs
|
|
665
|
+
const floodProbeIntervalMs = Math.max(0, config.floodProbeIntervalMs ?? D.floodProbeIntervalMs)
|
|
196
666
|
const onTrip = config.onTrip
|
|
197
667
|
|
|
198
668
|
const windows = new Map<string, Window>()
|
|
199
|
-
const counters = {
|
|
200
|
-
|
|
669
|
+
const counters = {
|
|
670
|
+
deferred: 0, dropped: 0, superseded: 0, floodObserved: 0,
|
|
671
|
+
/** Chat-targeted calls charged by the DEFAULT-DENY rule that the old
|
|
672
|
+
* allowlist would have let through unmetered (#3855 observability). */
|
|
673
|
+
meteredByDefault: 0,
|
|
674
|
+
/** Calls with no `chat_id`, charged to the token window only. */
|
|
675
|
+
chatless: 0,
|
|
676
|
+
}
|
|
677
|
+
/**
|
|
678
|
+
* Compounding 429 backoff. `tightenLevel` is the number of multiplicative
|
|
679
|
+
* decreases currently in force; `tightenedUntil` is when the NEXT level
|
|
680
|
+
* decays. A single flat tightening (the pre-2026-07-27 behaviour) let a
|
|
681
|
+
* stream that kept taking small 429s sit at 0.5× indefinitely and walk into
|
|
682
|
+
* a long ban; escalating means repeated 429s ratchet the rate down.
|
|
683
|
+
*/
|
|
684
|
+
let tightenLevel = 0
|
|
201
685
|
let tightenedUntil = 0
|
|
202
686
|
|
|
687
|
+
/**
|
|
688
|
+
* Cached read of the PERSISTED flood window (#3856). Throttled to
|
|
689
|
+
* `floodProbeIntervalMs` because `levelAt` runs on every admission and the
|
|
690
|
+
* probe is a file read — an unthrottled read would make the fuse the latency
|
|
691
|
+
* problem it exists to prevent. Between reads the cached remaining time is
|
|
692
|
+
* decayed by elapsed wall time, so a window never appears to last longer
|
|
693
|
+
* than it does.
|
|
694
|
+
*
|
|
695
|
+
* FAILS OPEN: any throw is treated as "no window". `makeFloodWaitProbe`
|
|
696
|
+
* already fails open on an unreadable marker and warns loudly (#3106); this
|
|
697
|
+
* catch covers the rest.
|
|
698
|
+
*/
|
|
699
|
+
let probedAt = Number.NEGATIVE_INFINITY
|
|
700
|
+
let probedRemainingMs = 0
|
|
701
|
+
function persistedFloodRemainingMs(now: number): number {
|
|
702
|
+
if (floodProbe === undefined) return 0
|
|
703
|
+
const since = now - probedAt
|
|
704
|
+
if (since < floodProbeIntervalMs) return Math.max(0, probedRemainingMs - since)
|
|
705
|
+
probedAt = now
|
|
706
|
+
try {
|
|
707
|
+
const ms = floodProbe()
|
|
708
|
+
probedRemainingMs = Number.isFinite(ms) && ms > 0 ? ms : 0
|
|
709
|
+
} catch {
|
|
710
|
+
probedRemainingMs = 0
|
|
711
|
+
}
|
|
712
|
+
return probedRemainingMs
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
/** Decay one level per `tightenMs` of quiet, rather than a single cliff. */
|
|
716
|
+
function levelAt(now: number): number {
|
|
717
|
+
while (tightenLevel > 0 && tightenedUntil <= now) {
|
|
718
|
+
tightenLevel--
|
|
719
|
+
tightenedUntil = tightenLevel > 0 ? tightenedUntil + tightenMs : 0
|
|
720
|
+
}
|
|
721
|
+
// #3856 — a KNOWN-OPEN persisted window pins the fuse regardless of
|
|
722
|
+
// in-memory state. This is what survives a restart: the process forgets,
|
|
723
|
+
// the marker on disk does not. It does not MUTATE `tightenLevel`, so when
|
|
724
|
+
// the window closes the fuse returns to whatever this process actually
|
|
725
|
+
// earned rather than staying pinned.
|
|
726
|
+
//
|
|
727
|
+
// #3885 — but pinned at a level PROPORTIONAL to the window still open, not
|
|
728
|
+
// unconditionally at `maxTightenLevel`. `flood-wait.json` is written for
|
|
729
|
+
// EVERY 429 including a routine 3-second nudge (`makeFloodWaitRecorder` →
|
|
730
|
+
// `computeFloodWait`), so the flat jump meant a 3s burst nudge and a 4.4h
|
|
731
|
+
// ban produced the identical maximal response — the exact asymmetry #3856
|
|
732
|
+
// fixed for the in-memory path and left in place here. The remaining window
|
|
733
|
+
// is graded through the SAME severity ladder `noteFlood` uses, so a big ban
|
|
734
|
+
// still pins at maximum and a nudge costs one level.
|
|
735
|
+
//
|
|
736
|
+
// Clamped by `maxTightenLevel` like every other path: `tightenStepFor`
|
|
737
|
+
// returns a fixed 1..3 for the smaller bands, so an operator who sets
|
|
738
|
+
// `SWITCHROOM_EDIT_FUSE_MAX_TIGHTEN_LEVEL=0` to disable tightening entirely
|
|
739
|
+
// must not find the fuse tightening anyway the moment a marker exists.
|
|
740
|
+
const remainingMs = persistedFloodRemainingMs(now)
|
|
741
|
+
if (remainingMs > 0) {
|
|
742
|
+
return Math.max(tightenLevel, Math.min(maxTightenLevel, tightenStepFor(remainingMs / 1000)))
|
|
743
|
+
}
|
|
744
|
+
return tightenLevel
|
|
745
|
+
}
|
|
746
|
+
|
|
203
747
|
function isTightened(now: number): boolean {
|
|
204
|
-
return
|
|
748
|
+
return levelAt(now) > 0
|
|
205
749
|
}
|
|
206
750
|
|
|
207
751
|
/**
|
|
208
|
-
* AIMD multiplicative decrease
|
|
209
|
-
* floored at 1 so the fuse never deadlocks a
|
|
752
|
+
* AIMD multiplicative decrease, compounding per observed 429. Applied to
|
|
753
|
+
* every ceiling while tightened; floored at 1 so the fuse never deadlocks a
|
|
754
|
+
* surface completely.
|
|
210
755
|
*/
|
|
211
756
|
function ceiling(base: number, now: number): number {
|
|
212
|
-
|
|
213
|
-
|
|
757
|
+
const level = levelAt(now)
|
|
758
|
+
if (level === 0) return base
|
|
759
|
+
return Math.max(1, Math.floor(base * Math.pow(tightenFactor, level)))
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
/**
|
|
763
|
+
* The ceiling a given CLASS sees on a tier whose base is `base` (#3885).
|
|
764
|
+
*
|
|
765
|
+
* `critical` traffic — the operator's answer, an approval card, a reaction —
|
|
766
|
+
* gets an absolute floor that tightening cannot cross. Everything else takes
|
|
767
|
+
* the tightened ceiling as-is, so shedding still happens; it just happens to
|
|
768
|
+
* the traffic that can afford it.
|
|
769
|
+
*
|
|
770
|
+
* The floor is capped by `base` so it can only ever RAISE a tightened ceiling
|
|
771
|
+
* back towards what the operator configured, never above it.
|
|
772
|
+
*/
|
|
773
|
+
function classCeiling(base: number, cls: OutboundClass, now: number): number {
|
|
774
|
+
const eff = ceiling(base, now)
|
|
775
|
+
if (cls !== 'critical') return eff
|
|
776
|
+
return Math.max(eff, Math.min(base, perChatCriticalMin))
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
/**
|
|
780
|
+
* The shared per-chat budget COSMETIC traffic may reach, re-derived against
|
|
781
|
+
* the EFFECTIVE ceiling (#3885).
|
|
782
|
+
*
|
|
783
|
+
* Returns 0 when the effective budget has shrunk to the reserve — at that
|
|
784
|
+
* point every remaining slot belongs to replies, and a repaint waits for the
|
|
785
|
+
* window or is shed. That is the intended behaviour under real flood
|
|
786
|
+
* pressure: the operator loses refresh frequency, never the answer.
|
|
787
|
+
*/
|
|
788
|
+
function cosmeticTotalMax(now: number): number {
|
|
789
|
+
const eff = ceiling(perChatTotalMax, now)
|
|
790
|
+
if (replyReserveFraction <= 0) return eff
|
|
791
|
+
const reserve = Math.min(eff, Math.max(1, Math.round(eff * replyReserveFraction)))
|
|
792
|
+
return Math.max(0, eff - reserve)
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
/** Shared per-chat budget resolver, per class, evaluated at admission time. */
|
|
796
|
+
function totalMaxFor(cls: OutboundClass): (now: number) => number {
|
|
797
|
+
if (cls === 'cosmetic') return cosmeticTotalMax
|
|
798
|
+
return (now: number) => classCeiling(perChatTotalMax, cls, now)
|
|
214
799
|
}
|
|
215
800
|
|
|
216
801
|
function win(key: string): Window {
|
|
@@ -255,6 +840,14 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
|
|
|
255
840
|
/** ms until `w` has room, or 0 if it has room now. */
|
|
256
841
|
function waitFor(w: Window, now: number, windowMs: number, max: number): number {
|
|
257
842
|
prune(w, now, windowMs)
|
|
843
|
+
// A ceiling of 0 is reachable now that the reply reserve is derived against
|
|
844
|
+
// the EFFECTIVE budget (#3885): under heavy tightening the whole remaining
|
|
845
|
+
// budget belongs to replies, so cosmetic traffic has none. There is no
|
|
846
|
+
// "oldest slot" to wait behind in that case, so wait out the window rather
|
|
847
|
+
// than indexing an empty array (which would produce NaN and admit).
|
|
848
|
+
if (max <= 0) {
|
|
849
|
+
return w.ts.length > 0 ? Math.max(1, w.ts[0]! + windowMs - now) : Math.max(1, windowMs)
|
|
850
|
+
}
|
|
258
851
|
if (w.ts.length < max) return 0
|
|
259
852
|
return Math.max(1, w.ts[0]! + windowMs - now)
|
|
260
853
|
}
|
|
@@ -266,23 +859,69 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
|
|
|
266
859
|
return { chat, msg }
|
|
267
860
|
}
|
|
268
861
|
|
|
269
|
-
/**
|
|
270
|
-
|
|
862
|
+
/**
|
|
863
|
+
* Record a 429 and tighten IN PROPORTION TO THE PENALTY (#3856).
|
|
864
|
+
*
|
|
865
|
+
* Before this, every 429 was worth exactly one level: a 3-second "slow down
|
|
866
|
+
* a touch" nudge and a **15908-second** ban produced the identical response.
|
|
867
|
+
* The whole point of AIMD is that the decrease matches the signal, and a
|
|
868
|
+
* four-hour ban is not one nudge's worth of signal. `retryAfterSec` is the
|
|
869
|
+
* severity Telegram itself states, so it is what the step is scaled by.
|
|
870
|
+
*
|
|
871
|
+
* The tightening is also held for at least the ban's own duration. A flat
|
|
872
|
+
* 10-minute `tightenMs` expired ~25× over during the 2026-07-27 ban, so the
|
|
873
|
+
* fuse would have been back at full rate long before the window closed.
|
|
874
|
+
*/
|
|
875
|
+
function noteFlood(now: number, retryAfterSec: number): void {
|
|
271
876
|
counters.floodObserved++
|
|
272
|
-
|
|
877
|
+
levelAt(now)
|
|
878
|
+
tightenLevel = Math.min(maxTightenLevel, tightenLevel + tightenStepFor(retryAfterSec))
|
|
879
|
+
// Every level currently in force is re-armed; the decay clock restarts
|
|
880
|
+
// from the newest 429, and never expires before the penalty itself does.
|
|
881
|
+
tightenedUntil = now + Math.max(tightenMs, retryAfterSec * 1000 + tightenMs)
|
|
273
882
|
}
|
|
274
883
|
|
|
275
|
-
|
|
884
|
+
/**
|
|
885
|
+
* Levels of multiplicative decrease one 429 is worth, by stated penalty:
|
|
886
|
+
*
|
|
887
|
+
* ≤ 5s 1 a routine burst nudge
|
|
888
|
+
* ≤ 60s 2 sustained overrate
|
|
889
|
+
* ≤ 600s 3 a real ban
|
|
890
|
+
* > 600s ALL the way to `maxTightenLevel` — at this magnitude the rate
|
|
891
|
+
* that produced it is categorically wrong, and stepping down
|
|
892
|
+
* one level at a time would spend the next window earning the
|
|
893
|
+
* next ban. The 2026-07-25 (3713s) and 2026-07-27 (15908s)
|
|
894
|
+
* bans are both in this band.
|
|
895
|
+
*/
|
|
896
|
+
function tightenStepFor(retryAfterSec: number): number {
|
|
897
|
+
if (!Number.isFinite(retryAfterSec) || retryAfterSec <= 5) return 1
|
|
898
|
+
if (retryAfterSec <= 60) return 2
|
|
899
|
+
if (retryAfterSec <= 600) return 3
|
|
900
|
+
return maxTightenLevel
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
/**
|
|
904
|
+
* The stated `retry_after` in seconds when `err` is a flood rejection, else
|
|
905
|
+
* null. Returns 0 for a flood whose magnitude is not stated (the text-match
|
|
906
|
+
* path), which `tightenStepFor` treats as the mildest case — an unquantified
|
|
907
|
+
* signal must not be inflated into a maximal response.
|
|
908
|
+
*/
|
|
909
|
+
function floodRetryAfterSec(err: unknown): number | null {
|
|
276
910
|
const e = err as { error_code?: number; parameters?: { retry_after?: number } } | null
|
|
277
911
|
if (e != null && typeof e === 'object') {
|
|
278
|
-
|
|
279
|
-
if (
|
|
912
|
+
const stated = e.parameters?.retry_after
|
|
913
|
+
if (typeof stated === 'number') return stated
|
|
914
|
+
if (e.error_code === 429) return 0
|
|
280
915
|
}
|
|
281
916
|
// Match on the SEMANTIC markers only. A bare "429" substring
|
|
282
917
|
// false-positives on ordinary server text (e.g. "message 429 not found"),
|
|
283
918
|
// and a false positive here halves every ceiling for ten minutes.
|
|
284
919
|
const msg = err instanceof Error ? err.message : String(err ?? '')
|
|
285
|
-
|
|
920
|
+
if (!/too many requests/i.test(msg) && !/retry[ _-]?after/i.test(msg)) return null
|
|
921
|
+
// grammY and the Bot API both surface the magnitude in the text; use it
|
|
922
|
+
// when it is there rather than throwing the severity away.
|
|
923
|
+
const m = /retry[ _-]?after[^0-9]{0,4}(\d+)/i.exec(msg)
|
|
924
|
+
return m != null ? Number(m[1]) : 0
|
|
286
925
|
}
|
|
287
926
|
|
|
288
927
|
/**
|
|
@@ -311,7 +950,16 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
|
|
|
311
950
|
* Returns the reserved timestamp, or null when the call must be dropped.
|
|
312
951
|
*/
|
|
313
952
|
async function awaitRoom(
|
|
314
|
-
key: string, windowMs: number,
|
|
953
|
+
key: string, windowMs: number,
|
|
954
|
+
/**
|
|
955
|
+
* The ceiling, resolved at EVERY loop iteration rather than once at entry.
|
|
956
|
+
* It has to be a function: the effective ceiling depends on the tighten
|
|
957
|
+
* level, the tighten level decays with time, and a call can sit in this
|
|
958
|
+
* loop for `maxDeferMs`. Passing a number would pin a waiter to the ceiling
|
|
959
|
+
* that was in force when it arrived.
|
|
960
|
+
*/
|
|
961
|
+
maxFor: (now: number) => number,
|
|
962
|
+
method: string, mode: WaitMode, cls: OutboundClass,
|
|
315
963
|
/**
|
|
316
964
|
* Consulted ONLY at the defer deadline for `mode: 'drop'`. Returning false
|
|
317
965
|
* converts the drop into a late release: this call is the last thing that
|
|
@@ -319,14 +967,34 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
|
|
|
319
967
|
* frame", it is freezing a card. Absent ⇒ drop unconditionally (the
|
|
320
968
|
* pre-review behaviour).
|
|
321
969
|
*/
|
|
322
|
-
dropGuard
|
|
970
|
+
dropGuard: (() => boolean) | undefined,
|
|
971
|
+
/**
|
|
972
|
+
* Absolute deadline SHARED by every tier of one `apply` call. Each tier
|
|
973
|
+
* used to start its own `maxDeferMs`, so a call crossing three tiers could
|
|
974
|
+
* be held 3×`maxDeferMs` — an unbounded-in-practice hold that a caller's
|
|
975
|
+
* own timeout, not this fuse, would have to end. One deadline per call
|
|
976
|
+
* makes `maxDeferMs` mean what it says.
|
|
977
|
+
*/
|
|
978
|
+
deadline: number,
|
|
979
|
+
/**
|
|
980
|
+
* Bounded overshoot budget for the R1 late-release path (cosmetic edits
|
|
981
|
+
* only). R1 says an over-budget edit that nothing newer will repaint must
|
|
982
|
+
* be RELEASED late rather than dropped, so a card cannot freeze mid-run.
|
|
983
|
+
* Taken literally that rule has no ceiling: a stream of cosmetic edits to
|
|
984
|
+
* DISTINCT message ids is a stream of "lone" frames, every one of which
|
|
985
|
+
* releases over budget — which is how 6 concurrent worker cards can sail
|
|
986
|
+
* past a 6/min chat ceiling. Charging each late release to a small extra
|
|
987
|
+
* window keeps R1's guarantee for the rare genuinely-lone frame (a
|
|
988
|
+
* worker's terminal recap) while capping the leak at `lateReleaseMax` per
|
|
989
|
+
* window. Absent ⇒ unbounded late release (sends, non-cosmetic edits).
|
|
990
|
+
*/
|
|
991
|
+
lateReleaseKey?: string,
|
|
323
992
|
): Promise<number | null> {
|
|
324
993
|
const w = win(key)
|
|
325
|
-
const deadline = clock.now() + maxDeferMs
|
|
326
994
|
let counted = false
|
|
327
995
|
for (;;) {
|
|
328
996
|
const now = clock.now()
|
|
329
|
-
const wait = waitFor(w, now, windowMs,
|
|
997
|
+
const wait = waitFor(w, now, windowMs, maxFor(now))
|
|
330
998
|
if (wait === 0) {
|
|
331
999
|
w.ts.push(now)
|
|
332
1000
|
return now
|
|
@@ -337,19 +1005,32 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
|
|
|
337
1005
|
// worse than a late one).
|
|
338
1006
|
if (mode !== 'release' && (dropGuard === undefined || dropGuard())) {
|
|
339
1007
|
counters.dropped++
|
|
340
|
-
onTrip?.({ method, key, action: 'dropped' })
|
|
1008
|
+
onTrip?.({ method, key, action: 'dropped', cls })
|
|
341
1009
|
return null
|
|
342
1010
|
}
|
|
343
1011
|
// A send — or an edit that nothing newer will repaint — is released
|
|
344
1012
|
// rather than dropped, and still takes a slot so the window reflects
|
|
345
|
-
// what actually went out.
|
|
1013
|
+
// what actually went out. Cosmetic late releases are additionally
|
|
1014
|
+
// charged to the bounded overshoot budget; when that is exhausted the
|
|
1015
|
+
// frame is dropped after all, so the ceiling cannot be walked past one
|
|
1016
|
+
// "lone" frame at a time.
|
|
1017
|
+
if (lateReleaseKey !== undefined) {
|
|
1018
|
+
const lw = win(lateReleaseKey)
|
|
1019
|
+
prune(lw, now, perChatWindowMs)
|
|
1020
|
+
if (lw.ts.length >= ceiling(lateReleaseMax, now)) {
|
|
1021
|
+
counters.dropped++
|
|
1022
|
+
onTrip?.({ method, key, action: 'dropped', cls })
|
|
1023
|
+
return null
|
|
1024
|
+
}
|
|
1025
|
+
lw.ts.push(now)
|
|
1026
|
+
}
|
|
346
1027
|
w.ts.push(now)
|
|
347
1028
|
return now
|
|
348
1029
|
}
|
|
349
1030
|
if (!counted) {
|
|
350
1031
|
counters.deferred++
|
|
351
1032
|
counted = true
|
|
352
|
-
onTrip?.({ method, key, action: 'deferred' })
|
|
1033
|
+
onTrip?.({ method, key, action: 'deferred', cls })
|
|
353
1034
|
}
|
|
354
1035
|
// Last-write-wins: a newer edit to the same message kills this one. Only
|
|
355
1036
|
// valid on the per-MESSAGE tier — on a per-chat key the "newer" edit is
|
|
@@ -364,7 +1045,7 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
|
|
|
364
1045
|
await Promise.race([clock.sleep(Math.min(wait, deadline - now)), superseded])
|
|
365
1046
|
if (killed) {
|
|
366
1047
|
counters.superseded++
|
|
367
|
-
onTrip?.({ method, key, action: 'superseded' })
|
|
1048
|
+
onTrip?.({ method, key, action: 'superseded', cls })
|
|
368
1049
|
return null
|
|
369
1050
|
}
|
|
370
1051
|
w.waiter = null
|
|
@@ -381,51 +1062,136 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
|
|
|
381
1062
|
next: () => Promise<R>,
|
|
382
1063
|
): Promise<R> {
|
|
383
1064
|
if (!enabled) return next()
|
|
1065
|
+
if (UNMETERED_METHODS.has(method)) return runObserved(next)
|
|
384
1066
|
|
|
385
1067
|
const isEdit = EDIT_METHODS.has(method)
|
|
386
1068
|
const isSend = SEND_METHODS.has(method)
|
|
387
|
-
|
|
1069
|
+
const isChatAction = CHAT_ACTION_METHODS.has(method)
|
|
388
1070
|
|
|
389
1071
|
const { chat, msg } = payloadKeys(payload)
|
|
390
|
-
// Inline-message edits carry no chat/message id — nothing to key on, and
|
|
391
|
-
// they are not part of any card loop. Pass through (still observed).
|
|
392
|
-
if (chat == null) return runObserved(next)
|
|
393
1072
|
|
|
394
1073
|
const now = clock.now()
|
|
395
1074
|
evict(now)
|
|
1075
|
+
// ONE deadline for the whole call, shared by every tier it crosses.
|
|
1076
|
+
const deadline = now + maxDeferMs
|
|
1077
|
+
|
|
1078
|
+
// The send gate's priority class, propagated through AsyncLocalStorage.
|
|
1079
|
+
// An untagged edit is COSMETIC by default — see `defaultOutboundClass`.
|
|
1080
|
+
// A chat action is cosmetic too: it is a self-expiring status, not content.
|
|
1081
|
+
const cls =
|
|
1082
|
+
currentOutboundClass() ?? (isChatAction ? 'cosmetic' : defaultOutboundClass(isEdit))
|
|
1083
|
+
|
|
1084
|
+
// DEFAULT-DENY (#3855): a call the fuse cannot key to a chat — an inline
|
|
1085
|
+
// edit, `answerCallbackQuery`, `getFile`, `setMyCommands` — is NOT free.
|
|
1086
|
+
// It cannot take a per-chat slot (there is no chat to charge), but it does
|
|
1087
|
+
// consume the bot token's global allowance, which is the thing Telegram
|
|
1088
|
+
// actually bans. Charge it there and pass.
|
|
1089
|
+
// The token tier is per-SECOND and global; a class floor there would be
|
|
1090
|
+
// meaningless (even at maximum tightening it admits 60/min), so it takes
|
|
1091
|
+
// the plain tightened ceiling.
|
|
1092
|
+
const tokenMaxFor = (t: number): number => ceiling(perTokenMax, t)
|
|
1093
|
+
|
|
1094
|
+
if (chat == null) {
|
|
1095
|
+
counters.chatless++
|
|
1096
|
+
await awaitRoom(tokenKey, perTokenWindowMs, tokenMaxFor, method, 'release', cls, undefined, deadline)
|
|
1097
|
+
return runObserved(next)
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
const totalKey = `t:${chat}`
|
|
1101
|
+
// Cosmetic traffic may only reach the effective ceiling MINUS the reply
|
|
1102
|
+
// reserve; the remaining slots stay available to a real reply no matter how
|
|
1103
|
+
// hot the repaint surfaces are, and `critical` additionally cannot be
|
|
1104
|
+
// tightened below `perChatCriticalMinPerWindow`. Together these are the
|
|
1105
|
+
// reply-starvation guarantee, and unlike the pre-#3885 shape they hold at
|
|
1106
|
+
// every tighten level rather than only at level 0.
|
|
1107
|
+
const chatTotalMax = totalMaxFor(cls)
|
|
1108
|
+
|
|
1109
|
+
/** Final tier for every chat-targeted path: the token-scoped ceiling. */
|
|
1110
|
+
const passToken = async (): Promise<R> => {
|
|
1111
|
+
await awaitRoom(tokenKey, perTokenWindowMs, tokenMaxFor, method, 'release', cls, undefined, deadline)
|
|
1112
|
+
return runObserved(next)
|
|
1113
|
+
}
|
|
396
1114
|
|
|
397
|
-
if (isEdit) {
|
|
398
|
-
if (msg == null) return runObserved(next)
|
|
1115
|
+
if (isEdit && msg != null) {
|
|
399
1116
|
const msgKey = `m:${chat}:${msg}`
|
|
400
1117
|
const mw = win(msgKey)
|
|
401
1118
|
// Counted BEFORE the first await so a frame that arrives while an older
|
|
402
1119
|
// one is waiting is visible to that older one's `dropGuard`.
|
|
403
1120
|
mw.inflight++
|
|
404
1121
|
try {
|
|
405
|
-
const msgSlot = await awaitRoom(
|
|
1122
|
+
const msgSlot = await awaitRoom(
|
|
1123
|
+
msgKey, perMessageWindowMs,
|
|
1124
|
+
cls === 'cosmetic'
|
|
1125
|
+
? (t) => ceiling(cosmeticPerMessageMax, t)
|
|
1126
|
+
: (t) => classCeiling(perMessageMax, cls, t),
|
|
1127
|
+
method, 'supersede', cls, undefined, deadline,
|
|
1128
|
+
)
|
|
406
1129
|
if (msgSlot === null) return DROPPED_RESULT as unknown as R
|
|
1130
|
+
// R1: only drop while something newer for THIS message is still in
|
|
1131
|
+
// flight to repaint it. `> 1` = this frame plus at least one newer.
|
|
1132
|
+
const dropGuard = (): boolean => mw.inflight > 1
|
|
1133
|
+
// Cosmetic frames share ONE overshoot budget per chat across both
|
|
1134
|
+
// per-chat tiers, so a frame cannot late-release twice on its way out.
|
|
1135
|
+
const lateKey = cls === 'cosmetic' ? `lr:${chat}` : undefined
|
|
1136
|
+
const reserved: Array<[string, number]> = [[msgKey, msgSlot]]
|
|
1137
|
+
const giveBack = (): void => { for (const [k, at] of reserved) unreserve(k, at) }
|
|
1138
|
+
|
|
407
1139
|
const chatKey = `ce:${chat}`
|
|
408
1140
|
const chatSlot = await awaitRoom(
|
|
409
|
-
chatKey, perChatWindowMs,
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
1141
|
+
chatKey, perChatWindowMs,
|
|
1142
|
+
cls === 'cosmetic'
|
|
1143
|
+
? (t) => ceiling(cosmeticPerChatMax, t)
|
|
1144
|
+
: (t) => classCeiling(perChatEditMax, cls, t),
|
|
1145
|
+
method, 'drop', cls, dropGuard, deadline, lateKey,
|
|
413
1146
|
)
|
|
414
|
-
if (chatSlot === null) {
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
1147
|
+
if (chatSlot === null) { giveBack(); return DROPPED_RESULT as unknown as R }
|
|
1148
|
+
reserved.push([chatKey, chatSlot])
|
|
1149
|
+
|
|
1150
|
+
// Shared per-chat budget — the one that mirrors Telegram's real
|
|
1151
|
+
// metering. A non-cosmetic edit is RELEASED late rather than dropped:
|
|
1152
|
+
// an approval card or answer finalisation has no newer frame coming.
|
|
1153
|
+
const totalSlot = await awaitRoom(
|
|
1154
|
+
totalKey, perChatWindowMs, chatTotalMax, method,
|
|
1155
|
+
cls === 'cosmetic' ? 'drop' : 'release', cls, dropGuard, deadline, lateKey,
|
|
1156
|
+
)
|
|
1157
|
+
if (totalSlot === null) { giveBack(); return DROPPED_RESULT as unknown as R }
|
|
1158
|
+
return passToken()
|
|
421
1159
|
} finally {
|
|
422
1160
|
mw.inflight--
|
|
423
1161
|
}
|
|
424
1162
|
}
|
|
425
1163
|
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
1164
|
+
if (isSend) {
|
|
1165
|
+
// Sends create user-visible output and are NEVER dropped — only paced.
|
|
1166
|
+
// The shared budget's reserve is what guarantees they still have room
|
|
1167
|
+
// when the repaint surfaces are saturated.
|
|
1168
|
+
await awaitRoom(`cs:${chat}`, perChatWindowMs,
|
|
1169
|
+
(t) => classCeiling(perChatSendMax, cls, t), method, 'release', cls, undefined, deadline)
|
|
1170
|
+
await awaitRoom(totalKey, perChatWindowMs, chatTotalMax, method, 'release', cls, undefined, deadline)
|
|
1171
|
+
return passToken()
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
// DEFAULT-DENY (#3855). Everything else with a `chat_id` — `sendChatAction`,
|
|
1175
|
+
// `setMessageReaction`, `deleteMessage`, `pinChatMessage`, `sendChecklist`,
|
|
1176
|
+
// an inline edit that carries a chat but no message id, and any Bot API
|
|
1177
|
+
// method added after this file was written — is charged to the SAME shared
|
|
1178
|
+
// per-chat window. Previously all of it was free, which is how a "20/60s"
|
|
1179
|
+
// ceiling admitted ~34 calls/60s on the wire.
|
|
1180
|
+
counters.meteredByDefault++
|
|
1181
|
+
if (isChatAction) {
|
|
1182
|
+
// Self-expiring status: shed it rather than deliver it late. Its own
|
|
1183
|
+
// short deadline also keeps a typing ping from occupying a 30s hold.
|
|
1184
|
+
const actionDeadline = Math.min(deadline, now + chatActionMaxDeferMs)
|
|
1185
|
+
const slot = await awaitRoom(
|
|
1186
|
+
totalKey, perChatWindowMs, chatTotalMax, method, 'drop', cls, undefined, actionDeadline,
|
|
1187
|
+
)
|
|
1188
|
+
if (slot === null) return DROPPED_RESULT as unknown as R
|
|
1189
|
+
return passToken()
|
|
1190
|
+
}
|
|
1191
|
+
// Durable, user-visible effect (a deletion, a pin, a reaction): paced,
|
|
1192
|
+
// never dropped — the same contract sends get.
|
|
1193
|
+
await awaitRoom(totalKey, perChatWindowMs, chatTotalMax, method, 'release', cls, undefined, deadline)
|
|
1194
|
+
return passToken()
|
|
429
1195
|
}
|
|
430
1196
|
|
|
431
1197
|
/** Run the downstream call, tightening the ceilings if it floods. */
|
|
@@ -434,13 +1200,16 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
|
|
|
434
1200
|
const res = await next()
|
|
435
1201
|
// grammY throws on ok:false, but a transformer installed BELOW another
|
|
436
1202
|
// one can still observe a raw ApiResponse — handle both shapes.
|
|
437
|
-
const r = res as unknown as {
|
|
1203
|
+
const r = res as unknown as {
|
|
1204
|
+
ok?: boolean; error_code?: number; parameters?: { retry_after?: number }
|
|
1205
|
+
}
|
|
438
1206
|
if (r != null && typeof r === 'object' && r.ok === false && r.error_code === 429) {
|
|
439
|
-
noteFlood(clock.now())
|
|
1207
|
+
noteFlood(clock.now(), r.parameters?.retry_after ?? 0)
|
|
440
1208
|
}
|
|
441
1209
|
return res
|
|
442
1210
|
} catch (err) {
|
|
443
|
-
|
|
1211
|
+
const retryAfterSec = floodRetryAfterSec(err)
|
|
1212
|
+
if (retryAfterSec !== null) noteFlood(clock.now(), retryAfterSec)
|
|
444
1213
|
throw err
|
|
445
1214
|
}
|
|
446
1215
|
}
|
|
@@ -454,7 +1223,16 @@ export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
|
|
|
454
1223
|
superseded: counters.superseded,
|
|
455
1224
|
floodObserved: counters.floodObserved,
|
|
456
1225
|
tightened: isTightened(now),
|
|
1226
|
+
tightenLevel: levelAt(now),
|
|
457
1227
|
perMessageCeiling: ceiling(perMessageMax, now),
|
|
1228
|
+
cosmeticPerMessageCeiling: ceiling(cosmeticPerMessageMax, now),
|
|
1229
|
+
cosmeticPerChatCeiling: ceiling(cosmeticPerChatMax, now),
|
|
1230
|
+
cosmeticPerChatTotalCeiling: cosmeticTotalMax(now),
|
|
1231
|
+
criticalPerChatTotalCeiling: classCeiling(perChatTotalMax, 'critical', now),
|
|
1232
|
+
meteredByDefault: counters.meteredByDefault,
|
|
1233
|
+
chatless: counters.chatless,
|
|
1234
|
+
perTokenCeiling: ceiling(perTokenMax, now),
|
|
1235
|
+
persistedFloodOpen: persistedFloodRemainingMs(now) > 0,
|
|
458
1236
|
}
|
|
459
1237
|
}
|
|
460
1238
|
|
|
@@ -470,7 +1248,13 @@ export type EditFloodFuse = ReturnType<typeof createEditFloodFuse>
|
|
|
470
1248
|
export type FuseInstallable = Bot
|
|
471
1249
|
|
|
472
1250
|
export function installEditFloodFuse(bot: FuseInstallable, config: EditFloodFuseConfig = {}): EditFloodFuse {
|
|
473
|
-
|
|
1251
|
+
// The token window is keyed by the bot's PUBLIC id only. `deriveBotScopeKey`
|
|
1252
|
+
// never returns any part of the secret half, and the key is used solely as an
|
|
1253
|
+
// in-memory Map key — never logged, never persisted.
|
|
1254
|
+
const fuse = createEditFloodFuse({
|
|
1255
|
+
botScopeKey: deriveBotScopeKey((bot as { token?: string }).token),
|
|
1256
|
+
...config,
|
|
1257
|
+
})
|
|
474
1258
|
bot.api.config.use(async (prev, method, payload, signal) =>
|
|
475
1259
|
fuse.apply(method, payload, () => prev(method, payload, signal)))
|
|
476
1260
|
return fuse
|