switchroom 0.19.19 → 0.19.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-broker/index.js +53 -0
- package/dist/cli/switchroom.js +2444 -1264
- package/dist/host-control/main.js +54 -1
- package/dist/vault/approvals/kernel-server.js +53 -0
- package/dist/vault/broker/server.js +53 -0
- package/package.json +4 -2
- package/skills/switchroom-release/SKILL.md +103 -20
- package/telegram-plugin/card-format.ts +92 -3
- package/telegram-plugin/dist/gateway/gateway.js +769 -172
- package/telegram-plugin/edit-flood-fuse.ts +477 -0
- package/telegram-plugin/format.ts +19 -7
- package/telegram-plugin/gateway/boot-sweep-gate.ts +164 -0
- package/telegram-plugin/gateway/callback-query-handlers.ts +454 -81
- package/telegram-plugin/gateway/gateway.ts +66 -56
- package/telegram-plugin/gateway/inbound-interceptors.ts +27 -4
- package/telegram-plugin/gateway/narrative-lane.ts +49 -3
- package/telegram-plugin/gateway/status-pin-api.ts +145 -0
- package/telegram-plugin/hooks/subagent-tracker-posttool.mjs +325 -45
- package/telegram-plugin/retry-api-call.ts +15 -2
- package/telegram-plugin/send-gate.ts +1 -1
- package/telegram-plugin/status-no-truncate.ts +64 -1
- package/telegram-plugin/status-pin-driver.ts +50 -27
- package/telegram-plugin/status-pin.ts +43 -5
- package/telegram-plugin/tests/activity-card-send-gate.test.ts +275 -0
- package/telegram-plugin/tests/activity-card-wiring.test.ts +16 -7
- package/telegram-plugin/tests/boot-pin-sweep-wiring.test.ts +101 -0
- package/telegram-plugin/tests/boot-sweep-gate.test.ts +293 -0
- package/telegram-plugin/tests/boot-version-string.test.ts +0 -0
- package/telegram-plugin/tests/edit-flood-fuse.test.ts +431 -0
- package/telegram-plugin/tests/pinned-card-collapse.test.ts +356 -0
- package/telegram-plugin/tests/status-pin-api.test.ts +178 -0
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +94 -11
- package/telegram-plugin/tests/status-pin.test.ts +106 -5
- package/telegram-plugin/tests/subagent-tracker-hooks.test.ts +631 -1
- package/telegram-plugin/tests/tool-activity-summary.test.ts +19 -10
- package/telegram-plugin/tests/vault-approval-posture.test.ts +6 -1
- package/telegram-plugin/tests/vault-passphrase-retry.test.ts +666 -0
- package/telegram-plugin/tests/vault-request-access-unlock-resume.test.ts +42 -21
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +233 -1
- package/telegram-plugin/tool-activity-summary.ts +85 -13
- package/telegram-plugin/worker-activity-feed.ts +5 -1
- package/vendor/hindsight-memory/scripts/drain_pending.py +193 -25
- package/vendor/hindsight-memory/scripts/lib/pending.py +84 -5
- package/vendor/hindsight-memory/scripts/lib/retain_split.py +21 -10
- package/vendor/hindsight-memory/scripts/recall.py +74 -5
- package/vendor/hindsight-memory/scripts/tests/test_pending_drops.py +158 -4
- package/vendor/hindsight-memory/scripts/tests/test_pending_failure_class.py +105 -0
- package/vendor/hindsight-memory/scripts/tests/test_pending_wedge.py +300 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_degraded_notice.py +365 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_envelope_strip_telemetry.py +12 -4
- package/vendor/hindsight-memory/scripts/tests/test_recall_transcript_fallback.py +27 -2
- package/vendor/hindsight-memory/scripts/tests/test_retain_split.py +19 -11
- package/vendor/hindsight-memory/tests/test_drain_pending.py +28 -2
|
@@ -0,0 +1,477 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Edit-flood fuse — a LAST-RESORT rate ceiling installed at the one seam no
|
|
3
|
+
* outbound call can bypass (#3620).
|
|
4
|
+
*
|
|
5
|
+
* ── Why this exists ───────────────────────────────────────────────────────
|
|
6
|
+
* The send gate (`send-gate.ts`) is the primary pacer, and every call that
|
|
7
|
+
* goes through `robustApiCall` is admitted by it. But two properties make it
|
|
8
|
+
* a gate with side doors rather than a failsafe:
|
|
9
|
+
*
|
|
10
|
+
* 1. Its per-message protections (edit floor, last-write-wins coalescing,
|
|
11
|
+
* no-op skip, long-horizon edit budget) are OPT-IN: `gate()` routes to
|
|
12
|
+
* the edit path only when the caller passes BOTH `messageId` and
|
|
13
|
+
* `editPayload` (send-gate.ts, `gate()`). A caller that forgets gets a
|
|
14
|
+
* plain send — no floor, no coalescing.
|
|
15
|
+
* 2. An untagged call is admitted as `critical` (UNTAGGED_SEND_CLASS), the
|
|
16
|
+
* class that is never shed and queues unbounded.
|
|
17
|
+
*
|
|
18
|
+
* That combination is exactly what earned agent `overlord` a 3713-second
|
|
19
|
+
* flood ban on 2026-07-25: the live activity card edited ONE message id with
|
|
20
|
+
* neither key nor class, so it ran at the per-chat bucket rate (60/min) for
|
|
21
|
+
* an hour. Keying that call site fixes that call site — it does not make the
|
|
22
|
+
* next unkeyed call site impossible.
|
|
23
|
+
*
|
|
24
|
+
* ── Why a grammY transformer ──────────────────────────────────────────────
|
|
25
|
+
* The gateway constructs exactly ONE `new Bot(TOKEN)`, and grammY routes
|
|
26
|
+
* 100% of outbound API traffic through that instance's transformer stack —
|
|
27
|
+
* `bot.api.*`, `ctx.api.*`, anything holding a reference to the `Api` object,
|
|
28
|
+
* wrapped or raw, gated or not. Installing here is therefore STRUCTURAL, not
|
|
29
|
+
* conventional: a new call site cannot opt out, because there is no code path
|
|
30
|
+
* to the network that skips the transformer stack. (`shared/bot-runtime.ts`
|
|
31
|
+
* already proves the seam with `installTgPostLogger`.)
|
|
32
|
+
*
|
|
33
|
+
* ── What it does (and deliberately does NOT do) ───────────────────────────
|
|
34
|
+
* - EDITS of the same `${chat_id}:${message_id}` are capped at
|
|
35
|
+
* `perMessageMaxPerWindow` per `perMessageWindowMs` (default 20/60s).
|
|
36
|
+
* Over-budget edits WAIT for the window to slide, and while they wait a
|
|
37
|
+
* newer edit to the same message SUPERSEDES them — last-write-wins at the
|
|
38
|
+
* chokepoint, for every caller, including ones that never heard of the
|
|
39
|
+
* send gate. A superseded or over-deferred edit is DROPPED (resolved as a
|
|
40
|
+
* benign no-op), which is safe precisely because the next render of any
|
|
41
|
+
* card carries full state.
|
|
42
|
+
* - NON-EDIT sends are PACED (per-chat rolling ceiling) but NEVER dropped.
|
|
43
|
+
* Dropping a reply would lose a user-visible answer; dropping a repaint
|
|
44
|
+
* costs nothing. Pacing is capped at `maxDeferMs`, after which the call
|
|
45
|
+
* passes through rather than blocking a reply for an unbounded time.
|
|
46
|
+
* - On an observed 429 the ceilings TIGHTEN multiplicatively (AIMD) for
|
|
47
|
+
* `tightenMs`, then restore. `retry-api-call.ts` sleeps `retry_after` and
|
|
48
|
+
* resumes at FULL rate; nothing in the stack previously reduced the
|
|
49
|
+
* sustained rate after a soft 429.
|
|
50
|
+
*
|
|
51
|
+
* ── Where the ceiling actually sits (review 2026-07-26, R1) ───────────────
|
|
52
|
+
* An earlier draft of this docblock claimed the fuse "sits ABOVE every
|
|
53
|
+
* legitimate cadence". That is FALSE and the claim mattered, so it is
|
|
54
|
+
* corrected here rather than deleted. The in-repo cadences are:
|
|
55
|
+
*
|
|
56
|
+
* - send gate `editFloorMs` = 1500ms ⇒ up to 40 edits/min/message
|
|
57
|
+
* - send gate cosmetic budget 150 / 300s ⇒ 30 edits/min/message
|
|
58
|
+
* - worker feed's own floor 2500ms ⇒ 24 edits/min/message
|
|
59
|
+
*
|
|
60
|
+
* The per-message ceiling here is 20/60s, i.e. BELOW all three. That is
|
|
61
|
+
* deliberate — Telegram's own per-group limit is ~20 messages/minute and
|
|
62
|
+
* edits count against it, so the *legitimate* cadences are themselves above
|
|
63
|
+
* what Telegram tolerates; that is precisely how `overlord` got banned while
|
|
64
|
+
* every in-repo pacer believed it was behaving. The fuse is therefore the
|
|
65
|
+
* BINDING constraint on a hot card, not a never-reached backstop.
|
|
66
|
+
*
|
|
67
|
+
* Because it binds routinely, "over budget" must never mean "silently
|
|
68
|
+
* lost". Two rules follow, and both are load-bearing:
|
|
69
|
+
* - per-MESSAGE over-budget edits SUPERSEDE (newest wins, older frame is
|
|
70
|
+
* discarded) — safe, because the frame that killed it will paint;
|
|
71
|
+
* - per-CHAT over-budget edits may only be DROPPED while a NEWER frame for
|
|
72
|
+
* that same message is still in flight to repaint it. When the waiting
|
|
73
|
+
* frame is the only one for its card (a turn-final `finalize`, say) it is
|
|
74
|
+
* RELEASED late instead of dropped. Dropping it would freeze the card
|
|
75
|
+
* mid-run AND return `true` to the send gate, which would then record a
|
|
76
|
+
* never-painted payload as on-screen and no-op-skip every retry.
|
|
77
|
+
*/
|
|
78
|
+
|
|
79
|
+
import type { Bot } from 'grammy'
|
|
80
|
+
|
|
81
|
+
import type { Clock } from './send-gate.js'
|
|
82
|
+
import { systemClock } from './send-gate.js'
|
|
83
|
+
|
|
84
|
+
/** Methods that mutate an EXISTING message — droppable, coalescible. */
|
|
85
|
+
const EDIT_METHODS: ReadonlySet<string> = new Set([
|
|
86
|
+
'editMessageText',
|
|
87
|
+
'editMessageCaption',
|
|
88
|
+
'editMessageMedia',
|
|
89
|
+
'editMessageReplyMarkup',
|
|
90
|
+
'editMessageLiveLocation',
|
|
91
|
+
// Review 2026-07-26 (R2): the checklist tools drive this one exactly like a
|
|
92
|
+
// card — `update_checklist` re-edits ONE message id in a loop
|
|
93
|
+
// (gateway.ts `_rawEditMessageChecklist`, called through `bot.api.raw`, which
|
|
94
|
+
// grammY routes through the transformer stack like everything else). Omitting
|
|
95
|
+
// it left a same-shaped flood path completely unfused.
|
|
96
|
+
'editMessageChecklist',
|
|
97
|
+
])
|
|
98
|
+
|
|
99
|
+
/** Methods that CREATE user-visible output — paced, never dropped. */
|
|
100
|
+
const SEND_METHODS: ReadonlySet<string> = new Set([
|
|
101
|
+
'sendMessage', 'sendPhoto', 'sendDocument', 'sendMediaGroup', 'sendAnimation',
|
|
102
|
+
'sendVideo', 'sendVoice', 'sendAudio', 'sendSticker', 'sendLocation',
|
|
103
|
+
'forwardMessage', 'forwardMessages', 'copyMessage', 'copyMessages',
|
|
104
|
+
// Review 2026-07-26 (R3): `sendRichMessage` is the plugin's PRIMARY send —
|
|
105
|
+
// every rich reply, every activity-card OPEN, every stream finalisation goes
|
|
106
|
+
// through it (`rich-send.ts`, `narrative-lane.ts`, `stream-render.ts`).
|
|
107
|
+
// Leaving it out meant the "non-edit sends are paced" property applied to
|
|
108
|
+
// almost nothing that this gateway actually sends. Sends are never dropped,
|
|
109
|
+
// so this only ever adds a bounded (`maxDeferMs`) delay under real pressure.
|
|
110
|
+
// NOTE: `sendRichMessageDraft` is deliberately NOT here — an ephemeral
|
|
111
|
+
// 30s-preview draft is high-cadence by design and is not a persisted message.
|
|
112
|
+
'sendRichMessage',
|
|
113
|
+
])
|
|
114
|
+
|
|
115
|
+
export interface EditFloodFuseConfig {
|
|
116
|
+
/** Master switch. When false, `apply` is a pure passthrough. Default true. */
|
|
117
|
+
enabled?: boolean
|
|
118
|
+
clock?: Clock
|
|
119
|
+
/** Hard ceiling on edits to ONE message id. Default 20 per 60s. */
|
|
120
|
+
perMessageMaxPerWindow?: number
|
|
121
|
+
perMessageWindowMs?: number
|
|
122
|
+
/** Hard ceiling on edits to ONE chat across all messages. Default 30 per 60s. */
|
|
123
|
+
perChatEditMaxPerWindow?: number
|
|
124
|
+
/** Pacing ceiling on non-edit sends to ONE chat. Default 25 per 60s. */
|
|
125
|
+
perChatSendMaxPerWindow?: number
|
|
126
|
+
perChatWindowMs?: number
|
|
127
|
+
/** Longest a call may be held before it is dropped (edit) / released (send). Default 30s. */
|
|
128
|
+
maxDeferMs?: number
|
|
129
|
+
/** Multiplicative decrease applied to every ceiling after a 429. Default 0.5. */
|
|
130
|
+
tightenFactor?: number
|
|
131
|
+
/** How long a tightened ceiling stays in force. Default 10 minutes. */
|
|
132
|
+
tightenMs?: number
|
|
133
|
+
/** Observability hook; fired whenever the fuse binds. */
|
|
134
|
+
onTrip?: (info: { method: string; key: string; action: 'deferred' | 'dropped' | 'superseded' }) => void
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export interface EditFloodFuseStats {
|
|
138
|
+
enabled: boolean
|
|
139
|
+
/** Calls the fuse held back waiting for a window to slide. */
|
|
140
|
+
deferred: number
|
|
141
|
+
/** Edits dropped because the window never opened inside `maxDeferMs`. */
|
|
142
|
+
dropped: number
|
|
143
|
+
/** Edits dropped because a NEWER edit to the same message arrived. */
|
|
144
|
+
superseded: number
|
|
145
|
+
/** 429s observed (each one tightens the ceilings). */
|
|
146
|
+
floodObserved: number
|
|
147
|
+
/** Whether a tightened ceiling is in force right now. */
|
|
148
|
+
tightened: boolean
|
|
149
|
+
/** Live per-message ceiling (post-AIMD). */
|
|
150
|
+
perMessageCeiling: number
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
export const EDIT_FLOOD_FUSE_DEFAULTS = {
|
|
154
|
+
perMessageMaxPerWindow: 20,
|
|
155
|
+
perMessageWindowMs: 60_000,
|
|
156
|
+
perChatEditMaxPerWindow: 30,
|
|
157
|
+
perChatSendMaxPerWindow: 25,
|
|
158
|
+
perChatWindowMs: 60_000,
|
|
159
|
+
maxDeferMs: 30_000,
|
|
160
|
+
tightenFactor: 0.5,
|
|
161
|
+
tightenMs: 600_000,
|
|
162
|
+
} as const
|
|
163
|
+
|
|
164
|
+
/** grammY resolves an edit with `true` when there is nothing to return. */
|
|
165
|
+
const DROPPED_RESULT = true
|
|
166
|
+
|
|
167
|
+
interface Window {
|
|
168
|
+
ts: number[]
|
|
169
|
+
/** The newest waiter on this key; a fresh waiter supersedes it. */
|
|
170
|
+
waiter: { kill: () => void } | null
|
|
171
|
+
/**
|
|
172
|
+
* How many calls for this key are currently inside `apply` (per-MESSAGE keys
|
|
173
|
+
* only). Read at the per-chat defer deadline: dropping is only honest while
|
|
174
|
+
* a NEWER frame for the same message is still in flight to repaint. See the
|
|
175
|
+
* docblock's "Where the ceiling actually sits" (R1).
|
|
176
|
+
*/
|
|
177
|
+
inflight: number
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* The fuse as a pure, clock-injectable unit. `apply` has the grammY
|
|
182
|
+
* transformer shape so it can be handed straight to `bot.api.config.use`.
|
|
183
|
+
*/
|
|
184
|
+
export function createEditFloodFuse(config: EditFloodFuseConfig = {}) {
|
|
185
|
+
const enabled = config.enabled ?? true
|
|
186
|
+
const clock = config.clock ?? systemClock
|
|
187
|
+
const D = EDIT_FLOOD_FUSE_DEFAULTS
|
|
188
|
+
const perMessageMax = config.perMessageMaxPerWindow ?? D.perMessageMaxPerWindow
|
|
189
|
+
const perMessageWindowMs = config.perMessageWindowMs ?? D.perMessageWindowMs
|
|
190
|
+
const perChatEditMax = config.perChatEditMaxPerWindow ?? D.perChatEditMaxPerWindow
|
|
191
|
+
const perChatSendMax = config.perChatSendMaxPerWindow ?? D.perChatSendMaxPerWindow
|
|
192
|
+
const perChatWindowMs = config.perChatWindowMs ?? D.perChatWindowMs
|
|
193
|
+
const maxDeferMs = config.maxDeferMs ?? D.maxDeferMs
|
|
194
|
+
const tightenFactor = config.tightenFactor ?? D.tightenFactor
|
|
195
|
+
const tightenMs = config.tightenMs ?? D.tightenMs
|
|
196
|
+
const onTrip = config.onTrip
|
|
197
|
+
|
|
198
|
+
const windows = new Map<string, Window>()
|
|
199
|
+
const counters = { deferred: 0, dropped: 0, superseded: 0, floodObserved: 0 }
|
|
200
|
+
/** Timestamp until which the AIMD tightening is in force (0 = untightened). */
|
|
201
|
+
let tightenedUntil = 0
|
|
202
|
+
|
|
203
|
+
function isTightened(now: number): boolean {
|
|
204
|
+
return tightenedUntil > now
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* AIMD multiplicative decrease. Applied to every ceiling while tightened;
|
|
209
|
+
* floored at 1 so the fuse never deadlocks a surface completely.
|
|
210
|
+
*/
|
|
211
|
+
function ceiling(base: number, now: number): number {
|
|
212
|
+
if (!isTightened(now)) return base
|
|
213
|
+
return Math.max(1, Math.floor(base * tightenFactor))
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function win(key: string): Window {
|
|
217
|
+
let w = windows.get(key)
|
|
218
|
+
if (w === undefined) {
|
|
219
|
+
w = { ts: [], waiter: null, inflight: 0 }
|
|
220
|
+
windows.set(key, w)
|
|
221
|
+
}
|
|
222
|
+
return w
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function prune(w: Window, now: number, windowMs: number): void {
|
|
226
|
+
const cutoff = now - windowMs
|
|
227
|
+
while (w.ts.length > 0 && w.ts[0]! <= cutoff) w.ts.shift()
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Bounded LRU-ish eviction so a long-lived process cannot grow the map
|
|
232
|
+
* without limit. A window with no in-flight waiter and no timestamps inside
|
|
233
|
+
* the widest window is dead state.
|
|
234
|
+
*/
|
|
235
|
+
let sinceEvict = 0
|
|
236
|
+
function evict(now: number): void {
|
|
237
|
+
if (windows.size < 4096) return
|
|
238
|
+
// Sweep at most once per 1024 admissions: an unconditional sweep would be
|
|
239
|
+
// O(n) on EVERY outbound call once the map is large, i.e. the fuse itself
|
|
240
|
+
// would become the latency problem it exists to prevent.
|
|
241
|
+
if (++sinceEvict < 1024) return
|
|
242
|
+
sinceEvict = 0
|
|
243
|
+
const widest = Math.max(perMessageWindowMs, perChatWindowMs)
|
|
244
|
+
for (const [k, w] of windows) {
|
|
245
|
+
// `inflight > 0` keeps a key whose caller is mid-wait: evicting it would
|
|
246
|
+
// hand that caller a stale Window object while a FRESH one (empty `ts`)
|
|
247
|
+
// took its place in the map, resetting the ceiling for everyone else.
|
|
248
|
+
if (w.waiter === null && w.inflight === 0
|
|
249
|
+
&& (w.ts.length === 0 || w.ts[w.ts.length - 1]! <= now - widest)) {
|
|
250
|
+
windows.delete(k)
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** ms until `w` has room, or 0 if it has room now. */
|
|
256
|
+
function waitFor(w: Window, now: number, windowMs: number, max: number): number {
|
|
257
|
+
prune(w, now, windowMs)
|
|
258
|
+
if (w.ts.length < max) return 0
|
|
259
|
+
return Math.max(1, w.ts[0]! + windowMs - now)
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function payloadKeys(payload: unknown): { chat: string | null; msg: string | null } {
|
|
263
|
+
const p = (payload ?? {}) as Record<string, unknown>
|
|
264
|
+
const chat = p.chat_id != null ? String(p.chat_id) : null
|
|
265
|
+
const msg = p.message_id != null ? String(p.message_id) : null
|
|
266
|
+
return { chat, msg }
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/** Record a 429 (whatever shape it arrives in) and tighten. */
|
|
270
|
+
function noteFlood(now: number): void {
|
|
271
|
+
counters.floodObserved++
|
|
272
|
+
tightenedUntil = now + tightenMs
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
function looksLikeFlood(err: unknown): boolean {
|
|
276
|
+
const e = err as { error_code?: number; parameters?: { retry_after?: number } } | null
|
|
277
|
+
if (e != null && typeof e === 'object') {
|
|
278
|
+
if (e.error_code === 429) return true
|
|
279
|
+
if (e.parameters != null && typeof e.parameters.retry_after === 'number') return true
|
|
280
|
+
}
|
|
281
|
+
// Match on the SEMANTIC markers only. A bare "429" substring
|
|
282
|
+
// false-positives on ordinary server text (e.g. "message 429 not found"),
|
|
283
|
+
// and a false positive here halves every ceiling for ten minutes.
|
|
284
|
+
const msg = err instanceof Error ? err.message : String(err ?? '')
|
|
285
|
+
return /too many requests/i.test(msg) || /retry[ _-]?after/i.test(msg)
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* How a call that finds no room behaves:
|
|
290
|
+
* supersede — per-message edits: a newer edit to the SAME message kills it
|
|
291
|
+
* drop — per-chat edits: waits, then drops at the defer deadline
|
|
292
|
+
* release — sends: waits, then passes through (never lost)
|
|
293
|
+
*/
|
|
294
|
+
type WaitMode = 'supersede' | 'drop' | 'release'
|
|
295
|
+
|
|
296
|
+
/** Give back a slot reserved by `awaitRoom` (used when a later tier denies). */
|
|
297
|
+
function unreserve(key: string, at: number): void {
|
|
298
|
+
const w = windows.get(key)
|
|
299
|
+
if (w === undefined) return
|
|
300
|
+
const i = w.ts.lastIndexOf(at)
|
|
301
|
+
if (i >= 0) w.ts.splice(i, 1)
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Wait for room on `key` and RESERVE the slot on success. Reserving inside
|
|
306
|
+
* the same synchronous step as the check is what makes the ceiling hold
|
|
307
|
+
* under concurrency: N producers that all pass a check-then-act test in the
|
|
308
|
+
* same tick would each admit, and N parallel sub-agents is exactly the shape
|
|
309
|
+
* that turns a ceiling into N× the ceiling.
|
|
310
|
+
*
|
|
311
|
+
* Returns the reserved timestamp, or null when the call must be dropped.
|
|
312
|
+
*/
|
|
313
|
+
async function awaitRoom(
|
|
314
|
+
key: string, windowMs: number, max: number, method: string, mode: WaitMode,
|
|
315
|
+
/**
|
|
316
|
+
* Consulted ONLY at the defer deadline for `mode: 'drop'`. Returning false
|
|
317
|
+
* converts the drop into a late release: this call is the last thing that
|
|
318
|
+
* will ever paint its message, so losing it is not "shedding a stale
|
|
319
|
+
* frame", it is freezing a card. Absent ⇒ drop unconditionally (the
|
|
320
|
+
* pre-review behaviour).
|
|
321
|
+
*/
|
|
322
|
+
dropGuard?: () => boolean,
|
|
323
|
+
): Promise<number | null> {
|
|
324
|
+
const w = win(key)
|
|
325
|
+
const deadline = clock.now() + maxDeferMs
|
|
326
|
+
let counted = false
|
|
327
|
+
for (;;) {
|
|
328
|
+
const now = clock.now()
|
|
329
|
+
const wait = waitFor(w, now, windowMs, ceiling(max, now))
|
|
330
|
+
if (wait === 0) {
|
|
331
|
+
w.ts.push(now)
|
|
332
|
+
return now
|
|
333
|
+
}
|
|
334
|
+
if (now >= deadline) {
|
|
335
|
+
// Held as long as we are willing to hold. An edit is dropped (the next
|
|
336
|
+
// render carries full state); a send is released (losing a reply is
|
|
337
|
+
// worse than a late one).
|
|
338
|
+
if (mode !== 'release' && (dropGuard === undefined || dropGuard())) {
|
|
339
|
+
counters.dropped++
|
|
340
|
+
onTrip?.({ method, key, action: 'dropped' })
|
|
341
|
+
return null
|
|
342
|
+
}
|
|
343
|
+
// A send — or an edit that nothing newer will repaint — is released
|
|
344
|
+
// rather than dropped, and still takes a slot so the window reflects
|
|
345
|
+
// what actually went out.
|
|
346
|
+
w.ts.push(now)
|
|
347
|
+
return now
|
|
348
|
+
}
|
|
349
|
+
if (!counted) {
|
|
350
|
+
counters.deferred++
|
|
351
|
+
counted = true
|
|
352
|
+
onTrip?.({ method, key, action: 'deferred' })
|
|
353
|
+
}
|
|
354
|
+
// Last-write-wins: a newer edit to the same message kills this one. Only
|
|
355
|
+
// valid on the per-MESSAGE tier — on a per-chat key the "newer" edit is
|
|
356
|
+
// usually to a DIFFERENT message, and killing an unrelated card's edit is
|
|
357
|
+
// not last-write-wins, it is data loss.
|
|
358
|
+
let killed = false
|
|
359
|
+
if (mode === 'supersede') {
|
|
360
|
+
w.waiter?.kill()
|
|
361
|
+
const superseded = new Promise<void>((resolve) => {
|
|
362
|
+
w.waiter = { kill: () => { killed = true; resolve() } }
|
|
363
|
+
})
|
|
364
|
+
await Promise.race([clock.sleep(Math.min(wait, deadline - now)), superseded])
|
|
365
|
+
if (killed) {
|
|
366
|
+
counters.superseded++
|
|
367
|
+
onTrip?.({ method, key, action: 'superseded' })
|
|
368
|
+
return null
|
|
369
|
+
}
|
|
370
|
+
w.waiter = null
|
|
371
|
+
} else {
|
|
372
|
+
await clock.sleep(Math.min(wait, deadline - now))
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
/** The grammY transformer body. */
|
|
378
|
+
async function apply<R>(
|
|
379
|
+
method: string,
|
|
380
|
+
payload: unknown,
|
|
381
|
+
next: () => Promise<R>,
|
|
382
|
+
): Promise<R> {
|
|
383
|
+
if (!enabled) return next()
|
|
384
|
+
|
|
385
|
+
const isEdit = EDIT_METHODS.has(method)
|
|
386
|
+
const isSend = SEND_METHODS.has(method)
|
|
387
|
+
if (!isEdit && !isSend) return runObserved(next)
|
|
388
|
+
|
|
389
|
+
const { chat, msg } = payloadKeys(payload)
|
|
390
|
+
// Inline-message edits carry no chat/message id — nothing to key on, and
|
|
391
|
+
// they are not part of any card loop. Pass through (still observed).
|
|
392
|
+
if (chat == null) return runObserved(next)
|
|
393
|
+
|
|
394
|
+
const now = clock.now()
|
|
395
|
+
evict(now)
|
|
396
|
+
|
|
397
|
+
if (isEdit) {
|
|
398
|
+
if (msg == null) return runObserved(next)
|
|
399
|
+
const msgKey = `m:${chat}:${msg}`
|
|
400
|
+
const mw = win(msgKey)
|
|
401
|
+
// Counted BEFORE the first await so a frame that arrives while an older
|
|
402
|
+
// one is waiting is visible to that older one's `dropGuard`.
|
|
403
|
+
mw.inflight++
|
|
404
|
+
try {
|
|
405
|
+
const msgSlot = await awaitRoom(msgKey, perMessageWindowMs, perMessageMax, method, 'supersede')
|
|
406
|
+
if (msgSlot === null) return DROPPED_RESULT as unknown as R
|
|
407
|
+
const chatKey = `ce:${chat}`
|
|
408
|
+
const chatSlot = await awaitRoom(
|
|
409
|
+
chatKey, perChatWindowMs, perChatEditMax, method, 'drop',
|
|
410
|
+
// R1: only drop while something newer for THIS message is still in
|
|
411
|
+
// flight to repaint it. `> 1` = this frame plus at least one newer.
|
|
412
|
+
() => mw.inflight > 1,
|
|
413
|
+
)
|
|
414
|
+
if (chatSlot === null) {
|
|
415
|
+
// Denied by the second tier — hand the first tier's slot back so a
|
|
416
|
+
// dropped edit never consumes budget it did not use.
|
|
417
|
+
unreserve(msgKey, msgSlot)
|
|
418
|
+
return DROPPED_RESULT as unknown as R
|
|
419
|
+
}
|
|
420
|
+
return runObserved(next)
|
|
421
|
+
} finally {
|
|
422
|
+
mw.inflight--
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
const chatKey = `cs:${chat}`
|
|
427
|
+
await awaitRoom(chatKey, perChatWindowMs, perChatSendMax, method, 'release')
|
|
428
|
+
return runObserved(next)
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/** Run the downstream call, tightening the ceilings if it floods. */
|
|
432
|
+
async function runObserved<R>(next: () => Promise<R>): Promise<R> {
|
|
433
|
+
try {
|
|
434
|
+
const res = await next()
|
|
435
|
+
// grammY throws on ok:false, but a transformer installed BELOW another
|
|
436
|
+
// one can still observe a raw ApiResponse — handle both shapes.
|
|
437
|
+
const r = res as unknown as { ok?: boolean; error_code?: number }
|
|
438
|
+
if (r != null && typeof r === 'object' && r.ok === false && r.error_code === 429) {
|
|
439
|
+
noteFlood(clock.now())
|
|
440
|
+
}
|
|
441
|
+
return res
|
|
442
|
+
} catch (err) {
|
|
443
|
+
if (looksLikeFlood(err)) noteFlood(clock.now())
|
|
444
|
+
throw err
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
function stats(): EditFloodFuseStats {
|
|
449
|
+
const now = clock.now()
|
|
450
|
+
return {
|
|
451
|
+
enabled,
|
|
452
|
+
deferred: counters.deferred,
|
|
453
|
+
dropped: counters.dropped,
|
|
454
|
+
superseded: counters.superseded,
|
|
455
|
+
floodObserved: counters.floodObserved,
|
|
456
|
+
tightened: isTightened(now),
|
|
457
|
+
perMessageCeiling: ceiling(perMessageMax, now),
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
return { apply, stats }
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
export type EditFloodFuse = ReturnType<typeof createEditFloodFuse>
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* The bot handle the fuse installs on. Typed against grammY's own `Bot` so the
|
|
468
|
+
* transformer signature is checked by the compiler, not by convention.
|
|
469
|
+
*/
|
|
470
|
+
export type FuseInstallable = Bot
|
|
471
|
+
|
|
472
|
+
export function installEditFloodFuse(bot: FuseInstallable, config: EditFloodFuseConfig = {}): EditFloodFuse {
|
|
473
|
+
const fuse = createEditFloodFuse(config)
|
|
474
|
+
bot.api.config.use(async (prev, method, payload, signal) =>
|
|
475
|
+
fuse.apply(method, payload, () => prev(method, payload, signal)))
|
|
476
|
+
return fuse
|
|
477
|
+
}
|
|
@@ -326,13 +326,25 @@ export function normalizeParagraphBreaks(text: string): string {
|
|
|
326
326
|
// it was the "stray blank line" seen in real replies. Collapse it.
|
|
327
327
|
//
|
|
328
328
|
// Deliberately ASCII-only: the `[ \t\r]` character class excludes U+00A0 by
|
|
329
|
-
// construction, so a line whose only content is a non-breaking space
|
|
330
|
-
//
|
|
331
|
-
//
|
|
332
|
-
//
|
|
333
|
-
//
|
|
334
|
-
//
|
|
335
|
-
//
|
|
329
|
+
// construction, so a line whose only content is a non-breaking space is left
|
|
330
|
+
// intact rather than silently collapsed.
|
|
331
|
+
//
|
|
332
|
+
// That is load-bearing, not merely conservative. `addParagraphSpacers` (below)
|
|
333
|
+
// wedges a U+00A0-only line into every prose paragraph gap to force a VISIBLE
|
|
334
|
+
// gap (#2692) and is LIVE on the outbound path today — gateway/
|
|
335
|
+
// outbound-send-path.ts (`normalizeOutboundBody`) and gateway/gateway.ts (the
|
|
336
|
+
// rich `sendRichMessage` chunker). Within a single `normalizeOutboundBody`
|
|
337
|
+
// pass the spacer is injected AFTER this normalizer, so this pass usually does
|
|
338
|
+
// not see it — but the ASCII-only class is a deliberate, pinned invariant, not
|
|
339
|
+
// an accident of that ordering: tests/paragraph-normalizer.test.ts ("the
|
|
340
|
+
// deliberate U+00A0 spacer … is preserved") asserts
|
|
341
|
+
// `normalizeParagraphBreaks(spaced) === spaced`, so this function stays safe
|
|
342
|
+
// to re-enter over already-spacered text. Widening `[ \t\r]` to catch U+00A0
|
|
343
|
+
// would eat the spacer, fail that test, and regress #2692.
|
|
344
|
+
//
|
|
345
|
+
// It also leaves a non-breaking space a user legitimately typed alone. Runs on
|
|
346
|
+
// code-masked text, so a blank-ish line inside a fenced block is parked and
|
|
347
|
+
// never touched.
|
|
336
348
|
let out = masked
|
|
337
349
|
// Collapse any run of newlines interleaved with ASCII whitespace-only
|
|
338
350
|
// interior lines down to a single clean `\n\n`. Requires at least one
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* boot-sweep-gate.ts — control flow for the boot pin sweep (#3664): WHEN it may
|
|
3
|
+
* start (`createBootSweepGate`, the two-condition arming gate) and HOW its steps
|
|
4
|
+
* are sequenced once it does (`runBootPinSweepSteps`, per-step isolation).
|
|
5
|
+
*
|
|
6
|
+
* Why this exists
|
|
7
|
+
* ---------------
|
|
8
|
+
* The boot orphan sweep (`runBootPinCleanupAndDmSweep` → `statusPinBootCleanup`
|
|
9
|
+
* → `unpinChatMessage`) has TWO independent preconditions, and they are
|
|
10
|
+
* satisfied at two different points of the gateway's boot:
|
|
11
|
+
*
|
|
12
|
+
* 1. **Ownership** — this process won the startup mutex, so it is the sole
|
|
13
|
+
* live owner of the shared per-agent pin state. Established during module
|
|
14
|
+
* evaluation (`acquireStartupLock`, plus the non-atomic fallback path).
|
|
15
|
+
* A LOSING double-boot must never unpin the live holder's pins.
|
|
16
|
+
* 2. **A Telegram handle** — `lockedBot` is assigned, so an unpin can
|
|
17
|
+
* actually reach Telegram. Established much later, inside
|
|
18
|
+
* `initGatewayBot()` at boot.
|
|
19
|
+
*
|
|
20
|
+
* Before this gate the sweep was kicked off fire-and-forget the moment (1) was
|
|
21
|
+
* satisfied, i.e. at module-eval time — thousands of lines before (2). Every
|
|
22
|
+
* boot-cleanup unpin therefore threw `undefined is not an object (evaluating
|
|
23
|
+
* 'lockedBot.api')` before touching the network, the row was retained with an
|
|
24
|
+
* incremented attempt counter, and after `BOOT_UNPIN_MAX_ATTEMPTS` the only
|
|
25
|
+
* record of a still-pinned message was forfeited. That was a regression from
|
|
26
|
+
* `caa4d7568` (#3310), which moved the Bot construction — and with it the
|
|
27
|
+
* `lockedBot` assignment — out of module scope into `initGatewayBot()` while
|
|
28
|
+
* leaving the sweep's kick-off where it was.
|
|
29
|
+
*
|
|
30
|
+
* The gate makes the ordering DETERMINISTIC rather than a race: the sweep runs
|
|
31
|
+
* exactly once, when BOTH signals have arrived, in whichever order they arrive.
|
|
32
|
+
* It cannot regress by code motion — moving either signal site around cannot
|
|
33
|
+
* make the sweep fire before the other signal.
|
|
34
|
+
*
|
|
35
|
+
* Deliberately dependency-free and side-effect-free so it is provable in
|
|
36
|
+
* isolation (`telegram-plugin/tests/boot-sweep-gate.test.ts`); the gateway owns
|
|
37
|
+
* the wiring.
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
export interface BootSweepGate {
|
|
41
|
+
/** Signal (1): this gateway owns the shared per-agent pin state. */
|
|
42
|
+
arm(): void
|
|
43
|
+
/** Signal (2): the Telegram bot handle exists — API calls can be made. */
|
|
44
|
+
botReady(): void
|
|
45
|
+
/** True once the sweep has been dispatched. Test/introspection seam. */
|
|
46
|
+
hasRun(): boolean
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Build a gate that dispatches `run` exactly once, after BOTH `arm()` and
|
|
51
|
+
* `botReady()` have been called. Both signals are idempotent and
|
|
52
|
+
* order-independent. `run` is dispatched fire-and-forget; a rejection is routed
|
|
53
|
+
* to `onError` so it can never surface as an unhandled rejection (the sweep is
|
|
54
|
+
* best-effort and must never take the gateway down).
|
|
55
|
+
*/
|
|
56
|
+
export function createBootSweepGate(args: {
|
|
57
|
+
run: () => Promise<void>
|
|
58
|
+
onError?: (err: unknown) => void
|
|
59
|
+
}): BootSweepGate {
|
|
60
|
+
let armed = false
|
|
61
|
+
let ready = false
|
|
62
|
+
let started = false
|
|
63
|
+
|
|
64
|
+
const maybeRun = (): void => {
|
|
65
|
+
if (started || !armed || !ready) return
|
|
66
|
+
started = true
|
|
67
|
+
void (async () => {
|
|
68
|
+
try {
|
|
69
|
+
await args.run()
|
|
70
|
+
} catch (err) {
|
|
71
|
+
args.onError?.(err)
|
|
72
|
+
}
|
|
73
|
+
})()
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
return {
|
|
77
|
+
arm(): void {
|
|
78
|
+
armed = true
|
|
79
|
+
maybeRun()
|
|
80
|
+
},
|
|
81
|
+
botReady(): void {
|
|
82
|
+
ready = true
|
|
83
|
+
maybeRun()
|
|
84
|
+
},
|
|
85
|
+
hasRun(): boolean {
|
|
86
|
+
return started
|
|
87
|
+
},
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Dependencies of {@link runBootPinSweepSteps}. All side effects are injected
|
|
92
|
+
* so the sequencing contract is provable without a gateway. */
|
|
93
|
+
export interface BootPinSweepSteps {
|
|
94
|
+
/** Pure fs scan for DM chat ids with a prior-session pin record. MUST run
|
|
95
|
+
* BEFORE the reapers, which empty those same stores. */
|
|
96
|
+
scanDmChatIds: () => string[]
|
|
97
|
+
statusPinCleanup: () => Promise<unknown>
|
|
98
|
+
activityCardReaper: () => Promise<unknown>
|
|
99
|
+
queuedCardReaper: () => Promise<unknown>
|
|
100
|
+
/** Flips the flag authorising the DM unpin-all path — for this sweep AND for
|
|
101
|
+
* later lazy first-inbound sweeps. */
|
|
102
|
+
enableDmSweep: () => void
|
|
103
|
+
sweepDm: (chatId: string) => Promise<unknown>
|
|
104
|
+
log?: (line: string) => void
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Run one sweep step, absorbing (and logging) a throw. See
|
|
109
|
+
* {@link runBootPinSweepSteps} for why isolation is mandatory here.
|
|
110
|
+
*
|
|
111
|
+
* Absorbs unconditionally: a non-`Error` throw is stringified rather than
|
|
112
|
+
* logged as `undefined`, and a `log` that itself throws (a closed stderr on a
|
|
113
|
+
* dying process) is swallowed too — otherwise the one function whose whole job
|
|
114
|
+
* is "never let a step take the sweep down" could take the sweep down.
|
|
115
|
+
*/
|
|
116
|
+
async function step(name: string, fn: () => Promise<unknown>, log: (line: string) => void) {
|
|
117
|
+
try {
|
|
118
|
+
await fn()
|
|
119
|
+
} catch (err) {
|
|
120
|
+
const msg = err instanceof Error ? err.message : String(err)
|
|
121
|
+
try {
|
|
122
|
+
log(`telegram gateway: boot pin sweep step '${name}' failed: ${msg}\n`)
|
|
123
|
+
} catch {
|
|
124
|
+
/* stderr is gone — nothing useful left to do */
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Sequence the boot sweep's steps, each individually absorbed.
|
|
131
|
+
*
|
|
132
|
+
* Why isolation is mandatory (#3664 salvage S2). The steps used to run as a
|
|
133
|
+
* bare sequential `await` chain. `runStatusPinBootCleanup` absorbs its own
|
|
134
|
+
* per-row throws, but the two card reapers issue real Bot API calls and can
|
|
135
|
+
* reject outright — and a rejection there did not merely skip the later
|
|
136
|
+
* reapers, it skipped `enableDmSweep()`. That flag authorises the DM
|
|
137
|
+
* unpin-all path for the WHOLE SESSION (the boot sweep AND every later lazy
|
|
138
|
+
* first-inbound sweep), so one throwing reaper silently disabled DM stale-pin
|
|
139
|
+
* cleanup entirely.
|
|
140
|
+
*
|
|
141
|
+
* That was unreachable before #3664 only because the sweep never actually ran
|
|
142
|
+
* — every step died on `undefined is not an object (evaluating
|
|
143
|
+
* 'lockedBot.api')` before doing anything. Making the sweep work for the first
|
|
144
|
+
* time is precisely what makes an earlier step's throw newly reachable, so the
|
|
145
|
+
* isolation lands with the fix rather than after it.
|
|
146
|
+
*
|
|
147
|
+
* Never rejects: every step is best-effort and the caller fire-and-forgets it.
|
|
148
|
+
*/
|
|
149
|
+
export async function runBootPinSweepSteps(deps: BootPinSweepSteps): Promise<void> {
|
|
150
|
+
const log = deps.log ?? ((l: string) => process.stderr.write(l))
|
|
151
|
+
|
|
152
|
+
let dmChatIds: string[] = []
|
|
153
|
+
await step('dm-chat-scan', async () => {
|
|
154
|
+
dmChatIds = deps.scanDmChatIds()
|
|
155
|
+
}, log)
|
|
156
|
+
|
|
157
|
+
await step('status-pin-cleanup', deps.statusPinCleanup, log)
|
|
158
|
+
await step('activity-card-reaper', deps.activityCardReaper, log)
|
|
159
|
+
await step('queued-card-reaper', deps.queuedCardReaper, log)
|
|
160
|
+
|
|
161
|
+
// Unconditional: reached even when every step above threw. See the docblock.
|
|
162
|
+
await step('enable-dm-sweep', async () => deps.enableDmSweep(), log)
|
|
163
|
+
for (const id of dmChatIds) await step(`dm-sweep:${id}`, () => deps.sweepDm(id), log)
|
|
164
|
+
}
|