switchroom 0.19.30 → 0.19.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/cli/switchroom.js +1121 -584
  2. package/dist/host-control/main.js +151 -73
  3. package/package.json +1 -1
  4. package/profiles/_base/cron-session.sh.hbs +5 -1
  5. package/profiles/_base/start.sh.hbs +15 -1
  6. package/telegram-plugin/dist/gateway/gateway.js +1435 -551
  7. package/telegram-plugin/edit-flood-fuse.ts +70 -20
  8. package/telegram-plugin/gateway/boot-beacon.ts +364 -0
  9. package/telegram-plugin/gateway/boot-sweep-gate.ts +20 -15
  10. package/telegram-plugin/gateway/gateway.ts +87 -88
  11. package/telegram-plugin/gateway/inbound-spool.ts +39 -0
  12. package/telegram-plugin/gateway/narrative-lane.ts +12 -0
  13. package/telegram-plugin/gateway/obligation-store.ts +28 -0
  14. package/telegram-plugin/gateway/stale-pin-sweep-store.ts +221 -0
  15. package/telegram-plugin/gateway/stale-pin-sweep-wiring.ts +211 -0
  16. package/telegram-plugin/gateway/stale-pin-sweep.test.ts +804 -0
  17. package/telegram-plugin/gateway/stale-pin-sweep.ts +1146 -0
  18. package/telegram-plugin/gateway/status-pin-retarget.ts +15 -2
  19. package/telegram-plugin/gateway/status-pin-store.ts +33 -11
  20. package/telegram-plugin/registry/turns-schema.ts +21 -1
  21. package/telegram-plugin/retry-api-call.ts +46 -21
  22. package/telegram-plugin/shared/bot-runtime.ts +61 -17
  23. package/telegram-plugin/shared/gw-trace-gate.ts +18 -2
  24. package/telegram-plugin/tests/activity-card-wiring.test.ts +7 -7
  25. package/telegram-plugin/tests/activity-drain-fuse-drop-not-failure.test.ts +324 -0
  26. package/telegram-plugin/tests/agent-card-result-footer.test.ts +193 -0
  27. package/telegram-plugin/tests/boot-beacon.test.ts +462 -0
  28. package/telegram-plugin/tests/boot-pin-sweep-wiring.test.ts +6 -6
  29. package/telegram-plugin/tests/boot-sweep-gate.test.ts +42 -31
  30. package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +2 -0
  31. package/telegram-plugin/tests/inbound-spool-progress.test.ts +2 -0
  32. package/telegram-plugin/tests/inbound-spool.test.ts +134 -5
  33. package/telegram-plugin/tests/narrative-lane-golden.test.ts +28 -0
  34. package/telegram-plugin/tests/obligation-determinism.test.ts +2 -0
  35. package/telegram-plugin/tests/obligation-store.test.ts +67 -1
  36. package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
  37. package/telegram-plugin/tests/status-pin-store.test.ts +26 -5
  38. package/telegram-plugin/tests/tg-post-logger-error-shape.test.ts +161 -0
  39. package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +30 -0
  40. package/telegram-plugin/tool-activity-summary.ts +104 -38
  41. package/telegram-plugin/worker-activity-feed.ts +33 -16
  42. package/telegram-plugin/gateway/dm-pin-sweep.test.ts +0 -251
  43. package/telegram-plugin/gateway/dm-pin-sweep.ts +0 -178
@@ -0,0 +1,1146 @@
1
+ /**
2
+ * stale-pin-sweep.ts — drains the STACK of orphaned bot pins a crashed or
3
+ * pre-fix session left behind, per chat class, under hard rate gates.
4
+ *
5
+ * ── What was actually broken ─────────────────────────────────────────────────
6
+ *
7
+ * A Telegram chat holds a STACK of pinned messages. `getChat().pinned_message`
8
+ * exposes only the TOP one and the Bot API has no list-pins method, so a
9
+ * probe-based cleanup can never see the rest. Worse — verified live on the
10
+ * fleet 2026-07-29 — a bare `unpinChatMessage(chat_id, message_id)` against a
11
+ * stale entry is a SILENT NO-OP that returns `{"ok":true}`. (Control: unpinning
12
+ * a message that was NEVER pinned also returns `ok:true`, so the response
13
+ * carries no information about whether anything was popped.) The gateway
14
+ * therefore logged a successful cleanup on every boot while the stack grew
15
+ * without bound: 37 orphaned pins across 5 agents, 21 of them in one chat.
16
+ *
17
+ * The load-bearing consequence, and the invariant this module is built around:
18
+ *
19
+ * ► A resolved `unpinChatMessage` is NOT evidence that anything was unpinned.
20
+ * Drain progress is only ever concluded from OBSERVED STATE, never from an
21
+ * API result.
22
+ *
23
+ * ── The drain primitives, per chat class ─────────────────────────────────────
24
+ *
25
+ * ONE sweep path (`sweepTarget`) that owns eligibility, the rights precheck,
26
+ * the rate gates, the circuit breaker, the cursor and the verification. It
27
+ * branches on chat class only for the drain PRIMITIVE, because DMs and groups
28
+ * genuinely have different Telegram semantics — measured, not assumed, in the
29
+ * test supergroup on 2026-07-29 with the bot admin + `can_pin_messages`:
30
+ *
31
+ * • DM (`dm`) — targeted `unpinChatMessage` does NOT pop a stale entry here;
32
+ * the only sequence that does is RE-PIN then unpin:
33
+ * `pinChatMessage(chat, id, disable_notification)` then
34
+ * `unpinChatMessage(chat, id)`. Loop until verified empty. A DM emits no
35
+ * service message for a pin, so the loop is invisible to the user.
36
+ *
37
+ * • Any group (`supergroup` / `forum-topic`) — targeted
38
+ * `unpinChatMessage(chat_id, message_id)` WORKS ON ANY STACK POSITION.
39
+ * Observed: an entry sitting third from the top was unpinned directly and
40
+ * vanished from the middle of the stack; the top then drained cleanly by
41
+ * plain targeted unpins. So a group needs NO re-pin, and with the re-pin
42
+ * goes the only thing that ever spams a group (see below).
43
+ *
44
+ * ── We unpin BY ID, never "unpin all" ────────────────────────────────────────
45
+ *
46
+ * This is a correctness rule, not a preference. `unpinAllChatMessages` works
47
+ * fine on a group stack — one call, silent, clears everything — and that is
48
+ * exactly the problem: "everything" includes OTHER PEOPLE'S pins. Measured, it
49
+ * destroyed a pre-existing pin that predated the test. In a real team chat that
50
+ * is unrecoverable data loss.
51
+ *
52
+ * The gateway already knows precisely which message ids it pinned
53
+ * (`status-pins.json`, plus the pinned activity cards). The group drain unpins
54
+ * exactly those and nothing else. A pin the gateway did not place is not its
55
+ * to remove.
56
+ *
57
+ * `unpinAllForumTopicMessages(chat_id, message_thread_id)` is genuinely
58
+ * topic-scoped (`telegram-bot-api` Client.cpp resolves `get_forum_topic_id` for
59
+ * it; `pinChatMessage`/`unpinChatMessage` never do) and needs only
60
+ * `can_pin_messages`, verified with `can_manage_topics: false`. But it is just
61
+ * as indiscriminate WITHIN the topic, so it is a LAST-RESORT REMEDY behind an
62
+ * explicit opt-in (`allowUnpinAllForumTopic`), never the default sweep. When it
63
+ * does run it runs ONCE: on a deep stack the unpin-all family can peg at `429
64
+ * retry_after: 3` with zero progress, because TDLib services it through
65
+ * `run_affected_history_query_until_complete`, which re-issues the IDENTICAL
66
+ * query while `!is_final_`. Retrying that is not slow, it is non-terminating.
67
+ *
68
+ * ── Pinning is what costs; unpinning is free ─────────────────────────────────
69
+ *
70
+ * Id-gap probe in the test supergroup: control gap 1, pin gap 2 — a pin DOES
71
+ * emit a service message, and `disable_notification` does not suppress it (gap 2
72
+ * either way). Every unpin verb measured gap 1: `unpinChatMessage`,
73
+ * `unpinAllChatMessages` and `unpinAllForumTopicMessages` are all silent.
74
+ *
75
+ * Since the group drain is unpin-only, the group sweep emits nothing visible at
76
+ * all — which is why there is no "groups are manual-only" restriction here. The
77
+ * DM repin loop does pin, but a DM has no service messages.
78
+ *
79
+ * ── Rate gates ───────────────────────────────────────────────────────────────
80
+ *
81
+ * All gates are named constants (`DM_SWEEP_GATE` / `GROUP_SWEEP_GATE`), never
82
+ * inline numbers:
83
+ *
84
+ * • GROUP — measured headroom is large (~15 mutating calls at 2-3s spacing
85
+ * plus `getChat` reads at 0.15-0.4s produced zero 429s and no `PEER_FLOOD`),
86
+ * so 6000ms is a deliberately conservative default rather than a measured
87
+ * ceiling; it costs nothing on the shallow stacks that are the common case.
88
+ * There is NO per-chat pop ceiling in a group: the drain is bounded by the
89
+ * id list, and with no service-message cost there is nothing to ration.
90
+ * • DM — the rule is ~1 message/second/chat. 1500ms across a two-call pop is
91
+ * ~0.67 writes/sec, comfortably inside it. The 40-pop cap bounds one blind
92
+ * drain's blast radius (deepest stack observed: 21).
93
+ *
94
+ * The per-minute cap bounds the BOT across all chats simultaneously (a fleet
95
+ * boot sweeping six chats at once is what actually earns a flood wait). Anything
96
+ * not finished inside the caps persists its cursor and yields to the next boot
97
+ * — which is safe precisely because the obligation is durable.
98
+ *
99
+ * Give-up and circuit-breaker rules are likewise constants: 3 consecutive 429s
100
+ * or 2 identical `retry_after` values with zero observed progress abort THIS
101
+ * target; a `400 … PEER_FLOOD` or a `429 retry_after > 60` halts the whole
102
+ * bot's sweep immediately (those two say "you are already in trouble", and the
103
+ * only correct response is to stop writing).
104
+ *
105
+ * PURE over injected seams — no Telegram import, no fs import, no timers — so
106
+ * every gate, the breaker, the resume and the drain are provable in
107
+ * `stale-pin-sweep.test.ts` against a fake that models the `ok:true` no-op
108
+ * faithfully.
109
+ */
110
+
111
+ import type { SweepChatKind, SweepCursor, SweepStoreFsSeam } from './stale-pin-sweep-store.js'
112
+ export type { SweepChatKind, SweepCursor, SweepStoreFsSeam } from './stale-pin-sweep-store.js'
113
+ import {
114
+ loadSweepCursors,
115
+ sweepTargetKey,
116
+ upsertSweepCursor,
117
+ SWEEP_MAX_ATTEMPTS,
118
+ } from './stale-pin-sweep-store.js'
119
+
120
+ // ─── Rate gates (operator-mandated; do not inline these numbers) ─────────────
121
+
122
+ /** The four rate dials one chat class is swept under. */
123
+ export interface SweepGate {
124
+ /** Minimum wall-clock gap between two pin/unpin writes in this class. */
125
+ minCallDelayMs: number
126
+ /**
127
+ * Hard cap on pops for ONE chat in ONE sweep; the rest yields to next boot.
128
+ * `Infinity` for groups: the group drain is bounded by the recorded id list
129
+ * and emits nothing visible, so there is nothing to ration.
130
+ */
131
+ maxPopsPerChatPerSweep: number
132
+ /** Cap on pin ops for the whole BOT per minute in this class, all chats. */
133
+ maxPinOpsPerMinute: number
134
+ }
135
+
136
+ /**
137
+ * DM gates. 1500ms across a two-write pop ≈ 0.67 writes/sec, inside Telegram's
138
+ * ~1 message/second/chat rule. 40 pops covers the deepest stack observed (21)
139
+ * with headroom; 30 ops/min bounds a fleet boot sweeping several DMs at once.
140
+ */
141
+ export const DM_SWEEP_GATE: SweepGate = {
142
+ minCallDelayMs: 1_500,
143
+ maxPopsPerChatPerSweep: 40,
144
+ maxPinOpsPerMinute: 30,
145
+ }
146
+
147
+ /**
148
+ * Supergroup / forum gates.
149
+ *
150
+ * 6000ms is a CONSERVATIVE DEFAULT, not a measured ceiling: a live session of
151
+ * ~15 mutating calls at 2-3s spacing (plus `getChat` reads at 0.15-0.4s) drew
152
+ * zero 429s and no `PEER_FLOOD`, so there is large headroom. It is kept because
153
+ * it costs nothing on the shallow stacks that are the common case.
154
+ *
155
+ * NO per-chat pop ceiling: a group drain issues one targeted unpin per RECORDED
156
+ * id, so it is already bounded by that list, and unpinning emits no service
157
+ * message — the thing the old ceiling existed to ration does not exist. The
158
+ * per-minute budget still applies and still yields to the next boot.
159
+ */
160
+ export const GROUP_SWEEP_GATE: SweepGate = {
161
+ minCallDelayMs: 6_000,
162
+ maxPopsPerChatPerSweep: Number.POSITIVE_INFINITY,
163
+ maxPinOpsPerMinute: 10,
164
+ }
165
+
166
+ /** Backoff after a 429: honour `retry_after`, then double it (3→6→12→24s). */
167
+ export const FLOOD_BACKOFF_FACTOR = 2
168
+
169
+ /** Consecutive 429s for one target before we abort it and defer to next boot. */
170
+ export const MAX_CONSECUTIVE_FLOOD_WAITS = 3
171
+
172
+ /**
173
+ * Identical `retry_after` values seen with ZERO observed progress before we
174
+ * abort. This is the `unpinAllChatMessages` signature — the server hands back
175
+ * the same wait forever because the query never becomes final.
176
+ */
177
+ export const MAX_IDENTICAL_RETRY_AFTER_NO_PROGRESS = 2
178
+
179
+ /** A 429 asking for longer than this trips the breaker outright. */
180
+ export const CIRCUIT_BREAKER_RETRY_AFTER_SEC = 60
181
+
182
+ /**
183
+ * Gap between the two `getChat` reads that must AGREE before a drain is called
184
+ * verified. `getChat` is a CACHE and it lies right after a mutation: a lone
185
+ * empty read taken immediately after an unpin means nothing. Two agreeing reads
186
+ * ~1.8s apart is the cheapest observation that survives the cache.
187
+ */
188
+ export const VERIFY_READ_GAP_MS = 1_800
189
+
190
+ /**
191
+ * How many read PAIRS may be taken while chasing two agreeing values. More than
192
+ * one because the observed lie (`pinned_message: None` on a non-empty stack,
193
+ * ~0.15s after an unpin) settles within ~0.55s: the first pair straddles the
194
+ * transient and disagrees, the second reads the settled value. Bounded so an
195
+ * genuinely flapping chat degrades to "unknown" rather than spinning.
196
+ */
197
+ export const VERIFY_READ_MAX_PAIRS = 3
198
+
199
+ /**
200
+ * THE destructive-remedy policy, in ONE place.
201
+ *
202
+ * `false` ⇒ `unpinAllForumTopicMessages` is never used by the routine sweep.
203
+ *
204
+ * Rationale: it is genuinely topic-scoped (unlike every other pin verb a bot
205
+ * has) and it drains a topic in one silent call, which makes it tempting. But
206
+ * within that topic it is INDISCRIMINATE — measured, the unpin-all family
207
+ * destroyed a pre-existing pin that the gateway never placed. The routine
208
+ * remedy is therefore a targeted unpin of the ids the gateway recorded, and
209
+ * this verb stays behind an explicit per-deployment opt-in for the case where
210
+ * an operator knowingly wants a topic's pins cleared wholesale.
211
+ *
212
+ * Kept as one constant + {@link mayUnpinAllForumTopic} rather than a condition
213
+ * spread through the control flow, so the policy is auditable in one place.
214
+ */
215
+ export const UNPIN_ALL_FORUM_TOPIC_ENABLED = false
216
+
217
+ /**
218
+ * May this sweep use the wholesale topic drain?
219
+ *
220
+ * `override` is the per-deployment escape hatch (`allowUnpinAllForumTopic`,
221
+ * bound to `SWITCHROOM_PIN_SWEEP_UNPIN_ALL_TOPIC=1`); when it is undefined the
222
+ * standing policy above decides.
223
+ */
224
+ export function mayUnpinAllForumTopic(kind: SweepChatKind, override?: boolean): boolean {
225
+ if (kind !== 'forum-topic') return false
226
+ return override ?? UNPIN_ALL_FORUM_TOPIC_ENABLED
227
+ }
228
+
229
+ /** Gates for a chat class. Forum topics ride the group gates — same chat. */
230
+ export function gateFor(kind: SweepChatKind): SweepGate {
231
+ return kind === 'dm' ? DM_SWEEP_GATE : GROUP_SWEEP_GATE
232
+ }
233
+
234
+ /** Which per-minute budget a class draws from. Forum topics are groups. */
235
+ export function budgetClassFor(kind: SweepChatKind): 'dm' | 'group' {
236
+ return kind === 'dm' ? 'dm' : 'group'
237
+ }
238
+
239
+ // ─── Chat classification ─────────────────────────────────────────────────────
240
+
241
+ /** True iff `chatId` is a user DM: a positive integer. Groups are negative. */
242
+ export function isDmChatId(chatId: string): boolean {
243
+ const n = Number(chatId)
244
+ return Number.isFinite(n) && Number.isInteger(n) && n > 0
245
+ }
246
+
247
+ /**
248
+ * Classify a sweep target.
249
+ *
250
+ * `forum-topic` requires BOTH a forum chat and a known `threadId`, because the
251
+ * only thing that distinction now buys is eligibility for the opt-in wholesale
252
+ * topic drain, whose verb takes a `message_thread_id`. Both group classes take
253
+ * the same routine path (targeted unpin of recorded ids), so a forum chat whose
254
+ * orphans have no recorded thread degrades to `supergroup` with no loss of
255
+ * capability.
256
+ */
257
+ export function classifyChatForSweep(args: {
258
+ chatId: string
259
+ threadId?: number
260
+ isForum?: boolean
261
+ }): SweepChatKind {
262
+ if (isDmChatId(args.chatId)) return 'dm'
263
+ if (args.isForum === true && args.threadId != null) return 'forum-topic'
264
+ return 'supergroup'
265
+ }
266
+
267
+ // ─── Seeding the obligation set from the durable stores ─────────────────────
268
+
269
+ /**
270
+ * The minimum a persisted row must expose to become a sweep target.
271
+ *
272
+ * `messageId` / `activityMessageId` are read too, because the GROUP drain is
273
+ * not a blind stack walk — it unpins exactly the ids the gateway is on record
274
+ * as having pinned, and those ids must be captured at SCAN time, before the
275
+ * boot reapers empty the very stores they come from.
276
+ */
277
+ export interface SweepCandidateRow {
278
+ chatId: string
279
+ threadId?: number | null
280
+ /** status-pin rows: the pinned message. */
281
+ messageId?: number
282
+ /** activity-card rows name it differently. */
283
+ activityMessageId?: number
284
+ /** activity-card rows: only a card that was actually pinned is on the stack. */
285
+ pinned?: boolean
286
+ }
287
+
288
+ /**
289
+ * The DISTINCT `(chat, thread)` targets that appear in ANY persisted store —
290
+ * the set of places a prior session is on record as having pinned something.
291
+ * These, and only these, are worth sweeping: an unpin-all is wasted on a chat
292
+ * the agent never pinned in, and blind-sweeping every known chat is exactly the
293
+ * kind of unbounded write volume the rate gates exist to prevent.
294
+ *
295
+ * Pure over its inputs; the gateway passes the loaded snapshots (status pins,
296
+ * activity cards, queued/busy-ack cards). Rows are deduped on the FULL
297
+ * `(chatId, threadId)` key — the old collector deduped on chat alone, which
298
+ * silently collapsed every topic of a forum into one target and threw away the
299
+ * `message_thread_id` the topic-scoped drain verb needs.
300
+ */
301
+ export function collectSweepTargets(input: {
302
+ statusPins?: readonly SweepCandidateRow[]
303
+ activityCards?: readonly SweepCandidateRow[]
304
+ queuedCards?: readonly SweepCandidateRow[]
305
+ }): SweepTarget[] {
306
+ const out = new Map<string, SweepTarget>()
307
+ const add = (rows?: readonly SweepCandidateRow[]): void => {
308
+ if (rows == null) return
309
+ for (const r of rows) {
310
+ if (typeof r.chatId !== 'string' || r.chatId.length === 0) continue
311
+ const threadId = r.threadId ?? undefined
312
+ const key = sweepTargetKey(r.chatId, threadId)
313
+ let target = out.get(key)
314
+ if (target == null) {
315
+ target = { chatId: r.chatId, threadId, messageIds: [] }
316
+ out.set(key, target)
317
+ }
318
+ // An activity card that was never pinned is not on the pin stack, so its
319
+ // id must NOT join the unpin list: issuing an unpin for it would spend
320
+ // rate budget on a call that provably cannot pop anything.
321
+ const id = r.messageId ?? (r.pinned === true ? r.activityMessageId : undefined)
322
+ if (typeof id === 'number' && !target.messageIds!.includes(id)) target.messageIds!.push(id)
323
+ }
324
+ }
325
+ add(input.statusPins)
326
+ add(input.activityCards)
327
+ add(input.queuedCards)
328
+ return [...out.values()]
329
+ }
330
+
331
+ /**
332
+ * The messageIds in `chatId` that must SURVIVE a drain because they belong to
333
+ * deliberately-retained store rows: unexpired time-scoped `tool:` pins (the
334
+ * `pin_message` MCP tool, #3001), which `runStatusPinBootCleanup` intentionally
335
+ * KEEPS across restarts. Work-scoped rows (no `expiresAt`) are stale by
336
+ * definition after a restart and are NOT re-pin candidates; expired
337
+ * time-scoped rows are due for sweeping.
338
+ */
339
+ export function unexpiredStoreRepinIds(
340
+ rows: readonly { chatId: string; messageId: number; expiresAt?: number }[],
341
+ chatId: string,
342
+ now: number,
343
+ ): number[] {
344
+ const out: number[] = []
345
+ for (const r of rows) {
346
+ if (r.chatId !== chatId) continue
347
+ if (r.expiresAt != null && r.expiresAt > now) out.push(r.messageId)
348
+ }
349
+ return out
350
+ }
351
+
352
+ // ─── Telegram error shapes ───────────────────────────────────────────────────
353
+
354
+ function description(err: unknown): string {
355
+ if (err != null && typeof err === 'object') {
356
+ const o = err as { description?: unknown; message?: unknown }
357
+ if (typeof o.description === 'string' && o.description.length > 0) {
358
+ return o.description.toLowerCase()
359
+ }
360
+ if (typeof o.message === 'string' && o.message.length > 0) return o.message.toLowerCase()
361
+ }
362
+ return String(err).toLowerCase()
363
+ }
364
+
365
+ /** Seconds Telegram asked us to wait, or null when this is not a flood wait. */
366
+ export function retryAfterSeconds(err: unknown): number | null {
367
+ if (err == null || typeof err !== 'object') return null
368
+ const o = err as { error_code?: unknown; parameters?: { retry_after?: unknown } }
369
+ const fromParams = o.parameters?.retry_after
370
+ if (typeof fromParams === 'number' && fromParams >= 0) return fromParams
371
+ if (o.error_code === 429) {
372
+ const m = /retry after (\d+)/.exec(description(err))
373
+ if (m != null) return Number(m[1])
374
+ return 0
375
+ }
376
+ return null
377
+ }
378
+
379
+ /**
380
+ * `400 … PEER_FLOOD` — Telegram telling the bot it is already flood-limited at
381
+ * the account level. Never retry through this; stop writing entirely.
382
+ */
383
+ export function isPeerFloodError(err: unknown): boolean {
384
+ return description(err).includes('peer_flood')
385
+ }
386
+
387
+ /**
388
+ * The bot is not allowed to manage pins here. Verified live: a bot without
389
+ * `can_pin_messages` gets `400 "not enough rights to manage pinned messages in
390
+ * the chat"` from EVERY pin method — an honest, stable rejection, unlike the
391
+ * silent-success unpin. So a rights failure is terminal for the chat, not a
392
+ * retry.
393
+ */
394
+ export function isPinRightsError(err: unknown): boolean {
395
+ return description(err).includes('not enough rights')
396
+ }
397
+
398
+ /**
399
+ * `unpinChatMessage` with NO `message_id` on an EMPTY stack answers `400
400
+ * "message to unpin not found"`. That is a real, positive termination signal —
401
+ * one of the very few honest answers in this API surface — so we use it as a
402
+ * corroborating check alongside the two agreeing `getChat` reads.
403
+ */
404
+ export function isNothingToUnpinError(err: unknown): boolean {
405
+ return description(err).includes('message to unpin not found')
406
+ }
407
+
408
+ // ─── Per-bot op budget ───────────────────────────────────────────────────────
409
+
410
+ /**
411
+ * Sliding-window budget for pin ops per bot per minute, per class. Shared
412
+ * across every chat in one sweep — the flood ledger Telegram keeps is per BOT,
413
+ * not per chat, so a per-chat-only gate is no gate at all when six chats are
414
+ * swept concurrently at boot.
415
+ */
416
+ export interface PinOpBudget {
417
+ /** Ms to wait before the next op in `cls` is within budget (0 = go now). */
418
+ waitMs(cls: 'dm' | 'group'): number
419
+ /** Record that an op just went out. */
420
+ record(cls: 'dm' | 'group'): void
421
+ }
422
+
423
+ export function createPinOpBudget(now: () => number): PinOpBudget {
424
+ const windows: Record<'dm' | 'group', number[]> = { dm: [], group: [] }
425
+ const limits: Record<'dm' | 'group', number> = {
426
+ dm: DM_SWEEP_GATE.maxPinOpsPerMinute,
427
+ group: GROUP_SWEEP_GATE.maxPinOpsPerMinute,
428
+ }
429
+ const prune = (cls: 'dm' | 'group', t: number): void => {
430
+ const w = windows[cls]
431
+ while (w.length > 0 && t - w[0] >= 60_000) w.shift()
432
+ }
433
+ return {
434
+ waitMs(cls) {
435
+ const t = now()
436
+ prune(cls, t)
437
+ const w = windows[cls]
438
+ if (w.length < limits[cls]) return 0
439
+ // Wait until the oldest op in the window ages out of the minute.
440
+ return Math.max(0, 60_000 - (t - w[0]))
441
+ },
442
+ record(cls) {
443
+ windows[cls].push(now())
444
+ },
445
+ }
446
+ }
447
+
448
+ // ─── Sweeper ─────────────────────────────────────────────────────────────────
449
+
450
+ export interface SweepTarget {
451
+ chatId: string
452
+ threadId?: number
453
+ /** Whether the chat is a forum supergroup, when the caller knows. */
454
+ isForum?: boolean
455
+ /**
456
+ * Message ids the gateway is on record as having pinned here, captured at
457
+ * scan time. The GROUP drain unpins exactly these — a pin the gateway did not
458
+ * place is not its to remove. Unioned with `deps.recordedPinIds` so the lazy
459
+ * first-inbound caller, which has no scan snapshot, still gets a list.
460
+ */
461
+ messageIds?: number[]
462
+ }
463
+
464
+ export type SweepStatus =
465
+ /** Verified empty (two agreeing reads). The obligation is discharged. */
466
+ | 'drained'
467
+ /** Never eligible this boot (lost the startup mutex). Nothing was written. */
468
+ | 'skipped-not-eligible'
469
+ /** Bot is not an admin with can_pin_messages here. Nothing was written. */
470
+ | 'skipped-no-rights'
471
+ /**
472
+ * A GROUP target with no recorded pin ids. Nothing is owed and nothing may be
473
+ * done: the gateway only ever removes pins it placed, so with no record there
474
+ * is no safe action (an unpin-all would take other people's pins with it).
475
+ */
476
+ | 'skipped-nothing-recorded'
477
+ /** Already discharged in a previous boot / earlier in this one. */
478
+ | 'already-drained'
479
+ /**
480
+ * This process already attempted this target. NOT an error: a re-attempt in
481
+ * the same process reads the same state through the same gates and can only
482
+ * burn the attempt budget and the flood ledger.
483
+ */
484
+ | 'already-attempted'
485
+ /** Attempt budget exhausted (SWEEP_MAX_ATTEMPTS boots). Never retried. */
486
+ | 'forfeited'
487
+ /** Hit the per-chat pop cap or the per-minute budget. Cursor persisted. */
488
+ | 'deferred-budget'
489
+ /** Flood-waited out (consecutive 429s, or a pegged identical retry_after). */
490
+ | 'deferred-flood'
491
+ /** Writes stopped: PEER_FLOOD or retry_after > 60s. Cursor persisted. */
492
+ | 'aborted-circuit-breaker'
493
+ /** Drain ran but the stack is provably NOT empty — never reported drained. */
494
+ | 'incomplete'
495
+ /** An unexpected failure. Cursor persisted; retried next boot. */
496
+ | 'error'
497
+
498
+ export interface SweepResult {
499
+ status: SweepStatus
500
+ /** Pops OBSERVED to have landed in THIS sweep (never counted from an API
501
+ * result — see the module docblock). */
502
+ popped: number
503
+ /**
504
+ * Targeted unpins ISSUED in this sweep. Deliberately separate from `popped`:
505
+ * an unpin aimed at a mid-stack id has no observable effect on the only thing
506
+ * a bot can read (`getChat().pinned_message`, the top), so issuing one is not
507
+ * evidence it landed. Keeping the two apart is what stops the `ok:true`
508
+ * silent no-op from being laundered back into a success count.
509
+ */
510
+ issued?: number
511
+ /** Detail for the log line. */
512
+ detail?: string
513
+ }
514
+
515
+ export interface StalePinSweepDeps {
516
+ /**
517
+ * Top of the pin stack (`getChat().pinned_message.message_id`), or null when
518
+ * the chat reports nothing pinned. This is the ONLY source of drain progress
519
+ * — deliberately, because the unpin RESULT is worthless (see the module
520
+ * docblock). May throw; a throw is treated as "unknown", never as "empty".
521
+ */
522
+ getTopPinnedMessageId: (chatId: string) => Promise<number | null>
523
+ /** `pinChatMessage(chat, id, { disable_notification: true })`. */
524
+ pinSilent: (chatId: string, messageId: number) => Promise<unknown>
525
+ /** `unpinChatMessage(chat, id)`. Its resolution proves NOTHING. */
526
+ unpin: (chatId: string, messageId: number) => Promise<unknown>
527
+ /** `unpinAllForumTopicMessages(chat, thread)` — the one topic-scoped verb. */
528
+ unpinAllForumTopicMessages: (chatId: string, threadId: number) => Promise<unknown>
529
+ /**
530
+ * `getChatMember(chat, self)` folded to "is administrator AND can_pin_messages".
531
+ * Consulted BEFORE any write in a group. A throw is treated as "no rights".
532
+ */
533
+ canPinInChat: (chatId: string) => Promise<boolean>
534
+ /**
535
+ * Message ids in this chat that must be RE-PINNED once the drain finishes:
536
+ * live in-memory claims plus the deliberately-retained store rows (unexpired
537
+ * `tool:` pins). Read at the START of the drain, restored at the end.
538
+ */
539
+ protectedMessageIds: (chatId: string) => number[]
540
+ /**
541
+ * Message ids the gateway is on record as having pinned in this target, read
542
+ * LIVE from the durable stores. Unioned with `SweepTarget.messageIds` so the
543
+ * lazy first-inbound sweep (which carries no boot scan) still has a list.
544
+ */
545
+ recordedPinIds: (chatId: string, threadId?: number) => number[]
546
+ /** Gate: this gateway won the startup mutex and owns the shared pin state. */
547
+ eligible: () => boolean
548
+ sleep: (ms: number) => Promise<void>
549
+ now: () => number
550
+ /** Durable obligation ledger. */
551
+ store: { path: string; fs: SweepStoreFsSeam }
552
+ /**
553
+ * Per-deployment override of {@link UNPIN_ALL_FORUM_TOPIC_ENABLED}. Leave
554
+ * undefined to take the standing policy; `true` opts this deployment into the
555
+ * WHOLESALE topic drain, which also removes pins the gateway did not place.
556
+ */
557
+ allowUnpinAllForumTopic?: boolean
558
+ log?: (line: string) => void
559
+ }
560
+
561
+ export interface StalePinSweeper {
562
+ /**
563
+ * Sweep one `(chat, thread)` target to completion or to a gate.
564
+ *
565
+ * AT MOST ONCE PER TARGET PER PROCESS. The lazy first-inbound caller fires on
566
+ * every inbound message, so without this a busy chat whose drain did not
567
+ * finish would re-attempt on every message — burning the 8-boot attempt
568
+ * budget in 8 messages and running overlapping drains against the same chat.
569
+ * Concurrent callers for the same target share the in-flight promise.
570
+ */
571
+ sweepTarget: (target: SweepTarget) => Promise<SweepResult>
572
+ /** True once the breaker tripped — every later target short-circuits. */
573
+ isCircuitOpen: () => boolean
574
+ }
575
+
576
+ export function createStalePinSweeper(deps: StalePinSweepDeps): StalePinSweeper {
577
+ const log = deps.log ?? ((l: string) => process.stderr.write(l))
578
+ const budget = createPinOpBudget(deps.now)
579
+ let circuitOpen = false
580
+ let lastWriteAt = 0
581
+
582
+ /** Persist the obligation. Never throws — durability is best-effort. */
583
+ const commit = (cursor: SweepCursor): void => {
584
+ try {
585
+ upsertSweepCursor(deps.store.path, deps.store.fs, cursor, log)
586
+ } catch (err) {
587
+ log(`stale-pin-sweep: cursor persist failed: ${(err as Error).message}\n`)
588
+ }
589
+ }
590
+
591
+ const loadCursor = (t: SweepTarget, kind: SweepChatKind): SweepCursor => {
592
+ const key = sweepTargetKey(t.chatId, t.threadId)
593
+ let rows: SweepCursor[] = []
594
+ try {
595
+ rows = loadSweepCursors(deps.store.path, deps.store.fs)
596
+ } catch {
597
+ rows = []
598
+ }
599
+ const existing = rows.find((c) => sweepTargetKey(c.chatId, c.threadId) === key)
600
+ if (existing != null) return { ...existing, kind }
601
+ return {
602
+ chatId: t.chatId,
603
+ threadId: t.threadId,
604
+ kind,
605
+ popped: 0,
606
+ done: false,
607
+ attempts: 0,
608
+ updatedAt: deps.now(),
609
+ }
610
+ }
611
+
612
+ /**
613
+ * Wait until BOTH the per-class minimum inter-call delay and the per-minute
614
+ * bot budget allow the next write. Returns false when the budget cannot be
615
+ * satisfied inside this sweep (caller defers to the next boot rather than
616
+ * sleeping a whole minute inside a boot path).
617
+ */
618
+ const awaitWriteSlot = async (kind: SweepChatKind): Promise<boolean> => {
619
+ const cls = budgetClassFor(kind)
620
+ const wait = budget.waitMs(cls)
621
+ // A budget wait means we've already spent this minute's whole allowance.
622
+ // Sleeping it off inside boot would stall the sweep for a full minute for
623
+ // no benefit — the cursor is durable, so yielding to the next boot is
624
+ // strictly better.
625
+ if (wait > 0) return false
626
+ const gate = gateFor(kind)
627
+ const since = deps.now() - lastWriteAt
628
+ if (lastWriteAt > 0 && since < gate.minCallDelayMs) {
629
+ await deps.sleep(gate.minCallDelayMs - since)
630
+ }
631
+ lastWriteAt = deps.now()
632
+ budget.record(cls)
633
+ return true
634
+ }
635
+
636
+ /**
637
+ * Two `getChat` reads separated by {@link VERIFY_READ_GAP_MS} that must
638
+ * AGREE, retried up to {@link VERIFY_READ_MAX_PAIRS} times. A read that
639
+ * THROWS is "unknown", never "empty".
640
+ *
641
+ * This exists because of a measured lie with a counter-intuitive shape: right
642
+ * after an unpin, `getChat` transiently reports `pinned_message: None` on a
643
+ * stack that is NOT empty. Tight sampling: at ~0.15s it read `None`; at
644
+ * ~0.55s and every read after, it read the next real pin. The transient value
645
+ * is the EMPTY one, not a stale id — the opposite of what you would guess —
646
+ * so a drain loop that terminates on a single `None` exits early and orphans
647
+ * the rest of the stack. Never treat one `None` as empty.
648
+ *
649
+ * Retrying (rather than giving up on the first disagreement) is what keeps the
650
+ * transient from ending an otherwise healthy drain: the second pair reads the
651
+ * settled value and the loop carries on.
652
+ */
653
+ const readTopVerified = async (chatId: string): Promise<{ top: number | null } | null> => {
654
+ const read = async (): Promise<{ v: number | null } | null> => {
655
+ try {
656
+ return { v: await deps.getTopPinnedMessageId(chatId) }
657
+ } catch {
658
+ return null
659
+ }
660
+ }
661
+ let prev = await read()
662
+ if (prev == null) return null
663
+ for (let i = 0; i < VERIFY_READ_MAX_PAIRS; i++) {
664
+ await deps.sleep(VERIFY_READ_GAP_MS)
665
+ const next = await read()
666
+ if (next == null) return null
667
+ if (next.v === prev.v) return { top: next.v }
668
+ prev = next
669
+ }
670
+ return null
671
+ }
672
+
673
+ /** Fold an error into a control decision shared by every drain primitive. */
674
+ type FloodDecision =
675
+ | { kind: 'breaker'; detail: string }
676
+ | { kind: 'flood'; seconds: number }
677
+ | { kind: 'rights' }
678
+ | { kind: 'other'; detail: string }
679
+
680
+ const classify = (err: unknown): FloodDecision => {
681
+ if (isPeerFloodError(err)) return { kind: 'breaker', detail: 'PEER_FLOOD' }
682
+ const retry = retryAfterSeconds(err)
683
+ if (retry != null) {
684
+ if (retry > CIRCUIT_BREAKER_RETRY_AFTER_SEC) {
685
+ return { kind: 'breaker', detail: `retry_after=${retry}s > ${CIRCUIT_BREAKER_RETRY_AFTER_SEC}s` }
686
+ }
687
+ return { kind: 'flood', seconds: retry }
688
+ }
689
+ if (isPinRightsError(err)) return { kind: 'rights' }
690
+ return { kind: 'other', detail: (err as Error)?.message ?? String(err) }
691
+ }
692
+
693
+ /**
694
+ * The DM / opted-in-supergroup drain: RE-PIN the top then UNPIN it, which is
695
+ * the only sequence that actually pops an entry, and re-read the observed top
696
+ * to decide whether anything moved. A resolved unpin is never counted.
697
+ */
698
+ const repinUnpinLoop = async (
699
+ target: SweepTarget,
700
+ kind: SweepChatKind,
701
+ cursor: SweepCursor,
702
+ ): Promise<SweepResult> => {
703
+ const gate = gateFor(kind)
704
+ let popped = 0
705
+ let consecutiveFloods = 0
706
+ let lastRetryAfter: number | null = null
707
+ let noProgressAtSameRetryAfter = 0
708
+ let lastObservedTop: number | null = null
709
+ // True between "the unpin call resolved" and "we have re-read the top and
710
+ // decided whether it actually did anything". A resolved unpin is a CLAIM,
711
+ // never a pop.
712
+ let unverifiedPop = false
713
+
714
+ /** Count a pop ONLY once the observed top has moved. */
715
+ const creditPop = (): void => {
716
+ popped++
717
+ cursor.popped++
718
+ cursor.updatedAt = deps.now()
719
+ commit(cursor)
720
+ }
721
+
722
+ for (let i = 0; i < gate.maxPopsPerChatPerSweep; i++) {
723
+ if (circuitOpen) {
724
+ return { status: 'aborted-circuit-breaker', popped, detail: 'breaker open' }
725
+ }
726
+ const observed = await readTopVerified(target.chatId)
727
+ if (observed == null) {
728
+ // Ambiguous read (disagreeing reads or a throw). Not a drain, and NOT
729
+ // an empty stack. Defer rather than assert either way.
730
+ return { status: 'incomplete', popped, detail: 'pin-stack read unverifiable' }
731
+ }
732
+ if (unverifiedPop) {
733
+ // THE bug this module exists for: `unpinChatMessage` resolves ok:true on
734
+ // a stale entry and pops NOTHING. The same id still on top after a full
735
+ // repin+unpin is proof the pop did not land, whatever the API answered.
736
+ if (observed.top != null && observed.top === lastObservedTop) {
737
+ return {
738
+ status: 'incomplete',
739
+ popped,
740
+ detail: `no progress: message ${observed.top} still on top after a repin+unpin`,
741
+ }
742
+ }
743
+ creditPop()
744
+ unverifiedPop = false
745
+ }
746
+ if (observed.top == null) {
747
+ return { status: 'drained', popped }
748
+ }
749
+ lastObservedTop = observed.top
750
+
751
+ if (!(await awaitWriteSlot(kind))) {
752
+ return { status: 'deferred-budget', popped, detail: 'per-minute pin-op budget spent' }
753
+ }
754
+ try {
755
+ await deps.pinSilent(target.chatId, observed.top)
756
+ } catch (err) {
757
+ const d = classify(err)
758
+ if (d.kind === 'breaker') {
759
+ circuitOpen = true
760
+ return { status: 'aborted-circuit-breaker', popped, detail: d.detail }
761
+ }
762
+ if (d.kind === 'rights') return { status: 'skipped-no-rights', popped }
763
+ if (d.kind === 'flood') {
764
+ consecutiveFloods++
765
+ if (lastRetryAfter === d.seconds) noProgressAtSameRetryAfter++
766
+ else noProgressAtSameRetryAfter = 1
767
+ lastRetryAfter = d.seconds
768
+ if (
769
+ consecutiveFloods >= MAX_CONSECUTIVE_FLOOD_WAITS ||
770
+ noProgressAtSameRetryAfter >= MAX_IDENTICAL_RETRY_AFTER_NO_PROGRESS
771
+ ) {
772
+ return {
773
+ status: 'deferred-flood',
774
+ popped,
775
+ detail: `flood: ${consecutiveFloods} consecutive 429s, retry_after=${d.seconds}s`,
776
+ }
777
+ }
778
+ // Honour retry_after, then double it for the next one (3→6→12→24).
779
+ await deps.sleep(d.seconds * 1000 * Math.pow(FLOOD_BACKOFF_FACTOR, consecutiveFloods - 1))
780
+ continue
781
+ }
782
+ return { status: 'error', popped, detail: d.detail }
783
+ }
784
+ // The unpin's RESULT is deliberately ignored — it resolves ok:true on a
785
+ // silent no-op. Only the next observed top decides whether this popped.
786
+ if (!(await awaitWriteSlot(kind))) {
787
+ return { status: 'deferred-budget', popped, detail: 'per-minute pin-op budget spent' }
788
+ }
789
+ try {
790
+ await deps.unpin(target.chatId, observed.top)
791
+ } catch (err) {
792
+ const d = classify(err)
793
+ if (d.kind === 'breaker') {
794
+ circuitOpen = true
795
+ return { status: 'aborted-circuit-breaker', popped, detail: d.detail }
796
+ }
797
+ if (d.kind === 'rights') return { status: 'skipped-no-rights', popped }
798
+ if (isNothingToUnpinError(err)) return { status: 'drained', popped }
799
+ if (d.kind === 'flood') {
800
+ consecutiveFloods++
801
+ if (lastRetryAfter === d.seconds) noProgressAtSameRetryAfter++
802
+ else noProgressAtSameRetryAfter = 1
803
+ lastRetryAfter = d.seconds
804
+ if (
805
+ consecutiveFloods >= MAX_CONSECUTIVE_FLOOD_WAITS ||
806
+ noProgressAtSameRetryAfter >= MAX_IDENTICAL_RETRY_AFTER_NO_PROGRESS
807
+ ) {
808
+ return {
809
+ status: 'deferred-flood',
810
+ popped,
811
+ detail: `flood: ${consecutiveFloods} consecutive 429s, retry_after=${d.seconds}s`,
812
+ }
813
+ }
814
+ await deps.sleep(d.seconds * 1000 * Math.pow(FLOOD_BACKOFF_FACTOR, consecutiveFloods - 1))
815
+ continue
816
+ }
817
+ return { status: 'error', popped, detail: d.detail }
818
+ }
819
+ consecutiveFloods = 0
820
+ unverifiedPop = true
821
+ }
822
+
823
+ // Pop cap reached. Is it actually empty? One verified read decides, and it
824
+ // is the ONLY thing allowed to report `drained`.
825
+ const finalRead = await readTopVerified(target.chatId)
826
+ if (unverifiedPop && finalRead != null && finalRead.top !== lastObservedTop) creditPop()
827
+ if (finalRead != null && finalRead.top == null) return { status: 'drained', popped }
828
+ return {
829
+ status: 'deferred-budget',
830
+ popped,
831
+ detail: `hit the ${gate.maxPopsPerChatPerSweep}-pop cap for this chat; yielding to next boot`,
832
+ }
833
+ }
834
+
835
+ /**
836
+ * The GROUP drain: one targeted `unpinChatMessage` per RECORDED id.
837
+ *
838
+ * Measured, a targeted unpin in a supergroup works at ANY stack position — an
839
+ * entry third from the top vanished from the middle — so no re-pin is needed
840
+ * and nothing visible is emitted. It is deliberately scoped to ids the gateway
841
+ * placed: `unpinAllChatMessages` would clear the stack in one silent call and
842
+ * take other people's pins with it (measured, it destroyed a pre-existing
843
+ * pin), which in a real team chat is unrecoverable data loss.
844
+ *
845
+ * Because a mid-stack unpin is unobservable through `getChat` (which exposes
846
+ * only the top), `popped` — the OBSERVED count — is credited only from a top
847
+ * that actually moved. Every issued call is reported separately as `issued`.
848
+ */
849
+ const targetedUnpinDrain = async (
850
+ target: SweepTarget,
851
+ kind: SweepChatKind,
852
+ cursor: SweepCursor,
853
+ ids: readonly number[],
854
+ ): Promise<SweepResult> => {
855
+ const alreadyDone = new Set(cursor.doneIds ?? [])
856
+ const pending = ids.filter((id) => !alreadyDone.has(id))
857
+ if (pending.length === 0) {
858
+ return { status: 'drained', popped: 0, issued: 0, detail: 'every recorded id already unpinned' }
859
+ }
860
+
861
+ const before = await readTopVerified(target.chatId)
862
+ let issued = 0
863
+ let deferred: SweepResult | null = null
864
+
865
+ for (const id of pending) {
866
+ if (circuitOpen) {
867
+ deferred = { status: 'aborted-circuit-breaker', popped: 0, issued, detail: 'breaker open' }
868
+ break
869
+ }
870
+ if (!(await awaitWriteSlot(kind))) {
871
+ deferred = {
872
+ status: 'deferred-budget',
873
+ popped: 0,
874
+ issued,
875
+ detail: `per-minute pin-op budget spent with ${pending.length - issued} ids left`,
876
+ }
877
+ break
878
+ }
879
+ try {
880
+ // The RESULT is deliberately ignored: `unpinChatMessage` answers ok:true
881
+ // even when it pops nothing. Only the observed top below is evidence.
882
+ await deps.unpin(target.chatId, id)
883
+ } catch (err) {
884
+ const d = classify(err)
885
+ if (d.kind === 'breaker') {
886
+ circuitOpen = true
887
+ deferred = { status: 'aborted-circuit-breaker', popped: 0, issued, detail: d.detail }
888
+ break
889
+ }
890
+ if (d.kind === 'rights') {
891
+ deferred = { status: 'skipped-no-rights', popped: 0, issued }
892
+ break
893
+ }
894
+ if (d.kind === 'flood') {
895
+ deferred = {
896
+ status: 'deferred-flood',
897
+ popped: 0,
898
+ issued,
899
+ detail: `flood: retry_after=${d.seconds}s`,
900
+ }
901
+ break
902
+ }
903
+ if (!isNothingToUnpinError(err)) {
904
+ deferred = { status: 'error', popped: 0, issued, detail: d.detail }
905
+ break
906
+ }
907
+ }
908
+ issued++
909
+ // Durable resume point: a boot that dies here must not re-issue this id.
910
+ alreadyDone.add(id)
911
+ cursor.doneIds = [...alreadyDone]
912
+ cursor.updatedAt = deps.now()
913
+ commit(cursor)
914
+ }
915
+
916
+ const after = await readTopVerified(target.chatId)
917
+ if (after == null) {
918
+ return {
919
+ status: deferred?.status ?? 'incomplete',
920
+ popped: 0,
921
+ issued,
922
+ detail: 'pin-stack read unverifiable after the unpin pass',
923
+ }
924
+ }
925
+ // The whole stack is gone ⇒ every issued unpin demonstrably landed.
926
+ if (after.top == null) {
927
+ const popped = issued
928
+ cursor.popped += popped
929
+ commit(cursor)
930
+ return { status: 'drained', popped, issued }
931
+ }
932
+ if (deferred != null) return { ...deferred, popped: 0 }
933
+ const ours = new Set(ids)
934
+ if (ours.has(after.top)) {
935
+ // One of OUR ids is still on top after we unpinned it — the silent no-op,
936
+ // observed. Never call that drained.
937
+ return {
938
+ status: 'incomplete',
939
+ popped: 0,
940
+ issued,
941
+ detail: `recorded pin ${after.top} is still on top after a targeted unpin`,
942
+ }
943
+ }
944
+ // Top is a pin the gateway never placed. Ours were mid-stack, where the API
945
+ // gives no read-back, so this is the honest limit of what can be observed:
946
+ // the obligation is discharged, the effect is unverifiable. Tracked as
947
+ // `issued`, never as `popped`.
948
+ const popped = before != null && before.top !== after.top ? 1 : 0
949
+ cursor.popped += popped
950
+ commit(cursor)
951
+ return {
952
+ status: 'drained',
953
+ popped,
954
+ issued,
955
+ detail: `unpinned ${issued} recorded id(s); top ${after.top} is not ours and was left alone`,
956
+ }
957
+ }
958
+
959
+ /**
960
+ * The OPT-IN wholesale topic drain: ONE topic-scoped
961
+ * `unpinAllForumTopicMessages`, no loop. Removes pins the gateway did not
962
+ * place, which is why it is gated on {@link mayUnpinAllForumTopic}. A 429 here
963
+ * is the pegged TDLib re-issue loop, so it is a give-up, not a backoff — see
964
+ * the module docblock.
965
+ */
966
+ const forumTopicDrain = async (
967
+ target: SweepTarget,
968
+ cursor: SweepCursor,
969
+ ): Promise<SweepResult> => {
970
+ const threadId = target.threadId
971
+ if (threadId == null) {
972
+ return { status: 'skipped-nothing-recorded', popped: 0, detail: 'no message_thread_id' }
973
+ }
974
+ if (!(await awaitWriteSlot('forum-topic'))) {
975
+ return { status: 'deferred-budget', popped: 0, detail: 'per-minute pin-op budget spent' }
976
+ }
977
+ try {
978
+ await deps.unpinAllForumTopicMessages(target.chatId, threadId)
979
+ } catch (err) {
980
+ const d = classify(err)
981
+ if (d.kind === 'breaker') {
982
+ circuitOpen = true
983
+ return { status: 'aborted-circuit-breaker', popped: 0, detail: d.detail }
984
+ }
985
+ if (d.kind === 'rights') return { status: 'skipped-no-rights', popped: 0 }
986
+ if (d.kind === 'flood') {
987
+ // DO NOT retry. `run_affected_history_query_until_complete` re-issues
988
+ // the identical query while `!is_final_`, re-tripping the same server
989
+ // flood wait — a loop here never terminates.
990
+ return {
991
+ status: 'deferred-flood',
992
+ popped: 0,
993
+ detail: `unpinAllForumTopicMessages pegged at retry_after=${d.seconds}s; not looping`,
994
+ }
995
+ }
996
+ return { status: 'error', popped: 0, detail: d.detail }
997
+ }
998
+ cursor.popped++
999
+ cursor.updatedAt = deps.now()
1000
+ commit(cursor)
1001
+ const observed = await readTopVerified(target.chatId)
1002
+ if (observed != null && observed.top == null) return { status: 'drained', popped: 1 }
1003
+ // The chat-wide stack may legitimately still hold pins belonging to OTHER
1004
+ // topics — the topic-scoped verb only drains this one. That is a success
1005
+ // for this obligation, not an incomplete drain.
1006
+ return { status: 'drained', popped: 1, detail: 'topic drained; chat-wide stack may hold other topics' }
1007
+ }
1008
+
1009
+ const sweepOnce = async (target: SweepTarget): Promise<SweepResult> => {
1010
+ if (!deps.eligible()) return { status: 'skipped-not-eligible', popped: 0 }
1011
+ if (circuitOpen) {
1012
+ return { status: 'aborted-circuit-breaker', popped: 0, detail: 'breaker already open' }
1013
+ }
1014
+ const kind = classifyChatForSweep(target)
1015
+ const cursor = loadCursor(target, kind)
1016
+ if (cursor.done) return { status: 'already-drained', popped: 0 }
1017
+ // Attempt budget exhausted: a permanently-undrainable chat (bot kicked,
1018
+ // rights revoked, a chat that flood-waits forever) must not re-burn the pop
1019
+ // budget on every boot for the rest of time.
1020
+ if (cursor.attempts >= SWEEP_MAX_ATTEMPTS) {
1021
+ return { status: 'forfeited', popped: 0, detail: `${cursor.attempts} attempts exhausted` }
1022
+ }
1023
+
1024
+ // RIGHTS PRECHECK, before ANY write, in every group. A bot without
1025
+ // can_pin_messages is honestly rejected by Telegram, but burning a rejected
1026
+ // write per orphan against the flood ledger is pointless. DMs need no
1027
+ // precheck — getChatMember is not meaningful there.
1028
+ if (kind !== 'dm') {
1029
+ let allowed = false
1030
+ try {
1031
+ allowed = await deps.canPinInChat(target.chatId)
1032
+ } catch {
1033
+ allowed = false
1034
+ }
1035
+ if (!allowed) {
1036
+ cursor.attempts++
1037
+ cursor.lastStatus = 'skipped-no-rights'
1038
+ cursor.updatedAt = deps.now()
1039
+ commit(cursor)
1040
+ return { status: 'skipped-no-rights', popped: 0 }
1041
+ }
1042
+ }
1043
+
1044
+ cursor.attempts++
1045
+ cursor.updatedAt = deps.now()
1046
+ commit(cursor)
1047
+
1048
+ // Snapshot the ids that must survive BEFORE draining.
1049
+ const protectedIds = [...new Set(deps.protectedMessageIds(target.chatId))]
1050
+
1051
+ let result: SweepResult
1052
+ try {
1053
+ if (kind === 'dm') {
1054
+ // A DM is a blind drain: targeted unpin does not pop there, so the only
1055
+ // primitive is repin+unpin against the observed top. It needs no id
1056
+ // list, and there are no third-party pins in a DM to protect.
1057
+ result = await repinUnpinLoop(target, kind, cursor)
1058
+ } else if (mayUnpinAllForumTopic(kind, deps.allowUnpinAllForumTopic)) {
1059
+ result = await forumTopicDrain(target, cursor)
1060
+ } else {
1061
+ // The routine group path: unpin exactly the ids we recorded, minus the
1062
+ // ones we deliberately keep (unexpired `tool:` pins).
1063
+ const keep = new Set(protectedIds)
1064
+ const ids = [
1065
+ ...new Set([
1066
+ ...(target.messageIds ?? []),
1067
+ ...deps.recordedPinIds(target.chatId, target.threadId),
1068
+ ]),
1069
+ ].filter((id) => !keep.has(id))
1070
+ result =
1071
+ ids.length === 0
1072
+ ? {
1073
+ status: 'skipped-nothing-recorded',
1074
+ popped: 0,
1075
+ issued: 0,
1076
+ detail: 'no recorded pin ids for this target — nothing this gateway may remove',
1077
+ }
1078
+ : await targetedUnpinDrain(target, kind, cursor, ids)
1079
+ }
1080
+ } catch (err) {
1081
+ result = { status: 'error', popped: 0, detail: (err as Error).message }
1082
+ }
1083
+
1084
+ // `skipped-nothing-recorded` discharges the obligation too: there is no id
1085
+ // this gateway may legitimately remove here, so re-attempting next boot can
1086
+ // only burn the attempt budget until it forfeits.
1087
+ if (result.status === 'drained' || result.status === 'skipped-nothing-recorded') {
1088
+ cursor.done = true
1089
+ }
1090
+ cursor.lastStatus = result.status
1091
+ cursor.updatedAt = deps.now()
1092
+ commit(cursor)
1093
+
1094
+ // Restore the deliberately-retained pins the blind DM drain cleared.
1095
+ // Rate-gated like any other write; a failure is logged and never fatal.
1096
+ //
1097
+ // DM ONLY, deliberately. The group path never removes a protected id in the
1098
+ // first place (they are filtered out of the unpin list), so re-pinning there
1099
+ // would double-pin — and a pin is the ONE group op that emits a visible
1100
+ // service message, so it would also be the only spam this sweep produces.
1101
+ if (kind === 'dm' && (result.status === 'drained' || result.popped > 0)) {
1102
+ for (const messageId of protectedIds) {
1103
+ if (circuitOpen) break
1104
+ if (!(await awaitWriteSlot(kind))) break
1105
+ try {
1106
+ await deps.pinSilent(target.chatId, messageId)
1107
+ } catch (err) {
1108
+ log(
1109
+ `telegram gateway: stale-pin-sweep: re-pin of protected message failed ` +
1110
+ `(chat=${target.chatId} msg=${messageId}): ${(err as Error).message}\n`,
1111
+ )
1112
+ }
1113
+ }
1114
+ }
1115
+
1116
+ log(
1117
+ `telegram gateway: stale-pin-sweep: chat=${target.chatId} ` +
1118
+ `thread=${target.threadId ?? '-'} kind=${kind} status=${result.status} ` +
1119
+ `popped=${result.popped} issued=${result.issued ?? 0} total=${cursor.popped}` +
1120
+ (result.detail != null ? ` (${result.detail})` : '') +
1121
+ `\n`,
1122
+ )
1123
+ return result
1124
+ }
1125
+
1126
+ // At-most-once-per-process, plus in-flight coalescing. See StalePinSweeper.
1127
+ const attempted = new Set<string>()
1128
+ const inFlight = new Map<string, Promise<SweepResult>>()
1129
+
1130
+ const sweepTarget = (target: SweepTarget): Promise<SweepResult> => {
1131
+ const key = sweepTargetKey(target.chatId, target.threadId)
1132
+ const running = inFlight.get(key)
1133
+ if (running != null) return running
1134
+ // Not eligible yet ⇒ nothing was attempted, so do NOT consume the one
1135
+ // allowed attempt: the boot sweep must still get its turn once the mutex
1136
+ // is won.
1137
+ if (!deps.eligible()) return Promise.resolve({ status: 'skipped-not-eligible', popped: 0 })
1138
+ if (attempted.has(key)) return Promise.resolve({ status: 'already-attempted', popped: 0 })
1139
+ attempted.add(key)
1140
+ const p = sweepOnce(target).finally(() => inFlight.delete(key))
1141
+ inFlight.set(key, p)
1142
+ return p
1143
+ }
1144
+
1145
+ return { sweepTarget, isCircuitOpen: () => circuitOpen }
1146
+ }