switchroom 0.19.30 → 0.19.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/switchroom.js +1121 -584
- package/dist/host-control/main.js +151 -73
- package/package.json +1 -1
- package/profiles/_base/cron-session.sh.hbs +5 -1
- package/profiles/_base/start.sh.hbs +15 -1
- package/telegram-plugin/dist/gateway/gateway.js +1435 -551
- package/telegram-plugin/edit-flood-fuse.ts +70 -20
- package/telegram-plugin/gateway/boot-beacon.ts +364 -0
- package/telegram-plugin/gateway/boot-sweep-gate.ts +20 -15
- package/telegram-plugin/gateway/gateway.ts +87 -88
- package/telegram-plugin/gateway/inbound-spool.ts +39 -0
- package/telegram-plugin/gateway/narrative-lane.ts +12 -0
- package/telegram-plugin/gateway/obligation-store.ts +28 -0
- package/telegram-plugin/gateway/stale-pin-sweep-store.ts +221 -0
- package/telegram-plugin/gateway/stale-pin-sweep-wiring.ts +211 -0
- package/telegram-plugin/gateway/stale-pin-sweep.test.ts +804 -0
- package/telegram-plugin/gateway/stale-pin-sweep.ts +1146 -0
- package/telegram-plugin/gateway/status-pin-retarget.ts +15 -2
- package/telegram-plugin/gateway/status-pin-store.ts +33 -11
- package/telegram-plugin/registry/turns-schema.ts +21 -1
- package/telegram-plugin/retry-api-call.ts +46 -21
- package/telegram-plugin/shared/bot-runtime.ts +61 -17
- package/telegram-plugin/shared/gw-trace-gate.ts +18 -2
- package/telegram-plugin/tests/activity-card-wiring.test.ts +7 -7
- package/telegram-plugin/tests/activity-drain-fuse-drop-not-failure.test.ts +324 -0
- package/telegram-plugin/tests/agent-card-result-footer.test.ts +193 -0
- package/telegram-plugin/tests/boot-beacon.test.ts +462 -0
- package/telegram-plugin/tests/boot-pin-sweep-wiring.test.ts +6 -6
- package/telegram-plugin/tests/boot-sweep-gate.test.ts +42 -31
- package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +2 -0
- package/telegram-plugin/tests/inbound-spool-progress.test.ts +2 -0
- package/telegram-plugin/tests/inbound-spool.test.ts +134 -5
- package/telegram-plugin/tests/narrative-lane-golden.test.ts +28 -0
- package/telegram-plugin/tests/obligation-determinism.test.ts +2 -0
- package/telegram-plugin/tests/obligation-store.test.ts +67 -1
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
- package/telegram-plugin/tests/status-pin-store.test.ts +26 -5
- package/telegram-plugin/tests/tg-post-logger-error-shape.test.ts +161 -0
- package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +30 -0
- package/telegram-plugin/tool-activity-summary.ts +104 -38
- package/telegram-plugin/worker-activity-feed.ts +33 -16
- package/telegram-plugin/gateway/dm-pin-sweep.test.ts +0 -251
- package/telegram-plugin/gateway/dm-pin-sweep.ts +0 -178
|
@@ -0,0 +1,1146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* stale-pin-sweep.ts — drains the STACK of orphaned bot pins a crashed or
|
|
3
|
+
* pre-fix session left behind, per chat class, under hard rate gates.
|
|
4
|
+
*
|
|
5
|
+
* ── What was actually broken ─────────────────────────────────────────────────
|
|
6
|
+
*
|
|
7
|
+
* A Telegram chat holds a STACK of pinned messages. `getChat().pinned_message`
|
|
8
|
+
* exposes only the TOP one and the Bot API has no list-pins method, so a
|
|
9
|
+
* probe-based cleanup can never see the rest. Worse — verified live on the
|
|
10
|
+
* fleet 2026-07-29 — a bare `unpinChatMessage(chat_id, message_id)` against a
|
|
11
|
+
* stale entry is a SILENT NO-OP that returns `{"ok":true}`. (Control: unpinning
|
|
12
|
+
* a message that was NEVER pinned also returns `ok:true`, so the response
|
|
13
|
+
* carries no information about whether anything was popped.) The gateway
|
|
14
|
+
* therefore logged a successful cleanup on every boot while the stack grew
|
|
15
|
+
* without bound: 37 orphaned pins across 5 agents, 21 of them in one chat.
|
|
16
|
+
*
|
|
17
|
+
* The load-bearing consequence, and the invariant this module is built around:
|
|
18
|
+
*
|
|
19
|
+
* ► A resolved `unpinChatMessage` is NOT evidence that anything was unpinned.
|
|
20
|
+
* Drain progress is only ever concluded from OBSERVED STATE, never from an
|
|
21
|
+
* API result.
|
|
22
|
+
*
|
|
23
|
+
* ── The drain primitives, per chat class ─────────────────────────────────────
|
|
24
|
+
*
|
|
25
|
+
* ONE sweep path (`sweepTarget`) that owns eligibility, the rights precheck,
|
|
26
|
+
* the rate gates, the circuit breaker, the cursor and the verification. It
|
|
27
|
+
* branches on chat class only for the drain PRIMITIVE, because DMs and groups
|
|
28
|
+
* genuinely have different Telegram semantics — measured, not assumed, in the
|
|
29
|
+
* test supergroup on 2026-07-29 with the bot admin + `can_pin_messages`:
|
|
30
|
+
*
|
|
31
|
+
* • DM (`dm`) — targeted `unpinChatMessage` does NOT pop a stale entry here;
|
|
32
|
+
* the only sequence that does is RE-PIN then unpin:
|
|
33
|
+
* `pinChatMessage(chat, id, disable_notification)` then
|
|
34
|
+
* `unpinChatMessage(chat, id)`. Loop until verified empty. A DM emits no
|
|
35
|
+
* service message for a pin, so the loop is invisible to the user.
|
|
36
|
+
*
|
|
37
|
+
* • Any group (`supergroup` / `forum-topic`) — targeted
|
|
38
|
+
* `unpinChatMessage(chat_id, message_id)` WORKS ON ANY STACK POSITION.
|
|
39
|
+
* Observed: an entry sitting third from the top was unpinned directly and
|
|
40
|
+
* vanished from the middle of the stack; the top then drained cleanly by
|
|
41
|
+
* plain targeted unpins. So a group needs NO re-pin, and with the re-pin
|
|
42
|
+
* goes the only thing that ever spams a group (see below).
|
|
43
|
+
*
|
|
44
|
+
* ── We unpin BY ID, never "unpin all" ────────────────────────────────────────
|
|
45
|
+
*
|
|
46
|
+
* This is a correctness rule, not a preference. `unpinAllChatMessages` works
|
|
47
|
+
* fine on a group stack — one call, silent, clears everything — and that is
|
|
48
|
+
* exactly the problem: "everything" includes OTHER PEOPLE'S pins. Measured, it
|
|
49
|
+
* destroyed a pre-existing pin that predated the test. In a real team chat that
|
|
50
|
+
* is unrecoverable data loss.
|
|
51
|
+
*
|
|
52
|
+
* The gateway already knows precisely which message ids it pinned
|
|
53
|
+
* (`status-pins.json`, plus the pinned activity cards). The group drain unpins
|
|
54
|
+
* exactly those and nothing else. A pin the gateway did not place is not its
|
|
55
|
+
* to remove.
|
|
56
|
+
*
|
|
57
|
+
* `unpinAllForumTopicMessages(chat_id, message_thread_id)` is genuinely
|
|
58
|
+
* topic-scoped (`telegram-bot-api` Client.cpp resolves `get_forum_topic_id` for
|
|
59
|
+
* it; `pinChatMessage`/`unpinChatMessage` never do) and needs only
|
|
60
|
+
* `can_pin_messages`, verified with `can_manage_topics: false`. But it is just
|
|
61
|
+
* as indiscriminate WITHIN the topic, so it is a LAST-RESORT REMEDY behind an
|
|
62
|
+
* explicit opt-in (`allowUnpinAllForumTopic`), never the default sweep. When it
|
|
63
|
+
* does run it runs ONCE: on a deep stack the unpin-all family can peg at `429
|
|
64
|
+
* retry_after: 3` with zero progress, because TDLib services it through
|
|
65
|
+
* `run_affected_history_query_until_complete`, which re-issues the IDENTICAL
|
|
66
|
+
* query while `!is_final_`. Retrying that is not slow, it is non-terminating.
|
|
67
|
+
*
|
|
68
|
+
* ── Pinning is what costs; unpinning is free ─────────────────────────────────
|
|
69
|
+
*
|
|
70
|
+
* Id-gap probe in the test supergroup: control gap 1, pin gap 2 — a pin DOES
|
|
71
|
+
* emit a service message, and `disable_notification` does not suppress it (gap 2
|
|
72
|
+
* either way). Every unpin verb measured gap 1: `unpinChatMessage`,
|
|
73
|
+
* `unpinAllChatMessages` and `unpinAllForumTopicMessages` are all silent.
|
|
74
|
+
*
|
|
75
|
+
* Since the group drain is unpin-only, the group sweep emits nothing visible at
|
|
76
|
+
* all — which is why there is no "groups are manual-only" restriction here. The
|
|
77
|
+
* DM repin loop does pin, but a DM has no service messages.
|
|
78
|
+
*
|
|
79
|
+
* ── Rate gates ───────────────────────────────────────────────────────────────
|
|
80
|
+
*
|
|
81
|
+
* All gates are named constants (`DM_SWEEP_GATE` / `GROUP_SWEEP_GATE`), never
|
|
82
|
+
* inline numbers:
|
|
83
|
+
*
|
|
84
|
+
* • GROUP — measured headroom is large (~15 mutating calls at 2-3s spacing
|
|
85
|
+
* plus `getChat` reads at 0.15-0.4s produced zero 429s and no `PEER_FLOOD`),
|
|
86
|
+
* so 6000ms is a deliberately conservative default rather than a measured
|
|
87
|
+
* ceiling; it costs nothing on the shallow stacks that are the common case.
|
|
88
|
+
* There is NO per-chat pop ceiling in a group: the drain is bounded by the
|
|
89
|
+
* id list, and with no service-message cost there is nothing to ration.
|
|
90
|
+
* • DM — the rule is ~1 message/second/chat. 1500ms across a two-call pop is
|
|
91
|
+
* ~0.67 writes/sec, comfortably inside it. The 40-pop cap bounds one blind
|
|
92
|
+
* drain's blast radius (deepest stack observed: 21).
|
|
93
|
+
*
|
|
94
|
+
* The per-minute cap bounds the BOT across all chats simultaneously (a fleet
|
|
95
|
+
* boot sweeping six chats at once is what actually earns a flood wait). Anything
|
|
96
|
+
* not finished inside the caps persists its cursor and yields to the next boot
|
|
97
|
+
* — which is safe precisely because the obligation is durable.
|
|
98
|
+
*
|
|
99
|
+
* Give-up and circuit-breaker rules are likewise constants: 3 consecutive 429s
|
|
100
|
+
* or 2 identical `retry_after` values with zero observed progress abort THIS
|
|
101
|
+
* target; a `400 … PEER_FLOOD` or a `429 retry_after > 60` halts the whole
|
|
102
|
+
* bot's sweep immediately (those two say "you are already in trouble", and the
|
|
103
|
+
* only correct response is to stop writing).
|
|
104
|
+
*
|
|
105
|
+
* PURE over injected seams — no Telegram import, no fs import, no timers — so
|
|
106
|
+
* every gate, the breaker, the resume and the drain are provable in
|
|
107
|
+
* `stale-pin-sweep.test.ts` against a fake that models the `ok:true` no-op
|
|
108
|
+
* faithfully.
|
|
109
|
+
*/
|
|
110
|
+
|
|
111
|
+
import type { SweepChatKind, SweepCursor, SweepStoreFsSeam } from './stale-pin-sweep-store.js'
|
|
112
|
+
export type { SweepChatKind, SweepCursor, SweepStoreFsSeam } from './stale-pin-sweep-store.js'
|
|
113
|
+
import {
|
|
114
|
+
loadSweepCursors,
|
|
115
|
+
sweepTargetKey,
|
|
116
|
+
upsertSweepCursor,
|
|
117
|
+
SWEEP_MAX_ATTEMPTS,
|
|
118
|
+
} from './stale-pin-sweep-store.js'
|
|
119
|
+
|
|
120
|
+
// ─── Rate gates (operator-mandated; do not inline these numbers) ─────────────
|
|
121
|
+
|
|
122
|
+
/** The four rate dials one chat class is swept under. */
|
|
123
|
+
export interface SweepGate {
|
|
124
|
+
/** Minimum wall-clock gap between two pin/unpin writes in this class. */
|
|
125
|
+
minCallDelayMs: number
|
|
126
|
+
/**
|
|
127
|
+
* Hard cap on pops for ONE chat in ONE sweep; the rest yields to next boot.
|
|
128
|
+
* `Infinity` for groups: the group drain is bounded by the recorded id list
|
|
129
|
+
* and emits nothing visible, so there is nothing to ration.
|
|
130
|
+
*/
|
|
131
|
+
maxPopsPerChatPerSweep: number
|
|
132
|
+
/** Cap on pin ops for the whole BOT per minute in this class, all chats. */
|
|
133
|
+
maxPinOpsPerMinute: number
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* DM gates. 1500ms across a two-write pop ≈ 0.67 writes/sec, inside Telegram's
|
|
138
|
+
* ~1 message/second/chat rule. 40 pops covers the deepest stack observed (21)
|
|
139
|
+
* with headroom; 30 ops/min bounds a fleet boot sweeping several DMs at once.
|
|
140
|
+
*/
|
|
141
|
+
export const DM_SWEEP_GATE: SweepGate = {
|
|
142
|
+
minCallDelayMs: 1_500,
|
|
143
|
+
maxPopsPerChatPerSweep: 40,
|
|
144
|
+
maxPinOpsPerMinute: 30,
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Supergroup / forum gates.
|
|
149
|
+
*
|
|
150
|
+
* 6000ms is a CONSERVATIVE DEFAULT, not a measured ceiling: a live session of
|
|
151
|
+
* ~15 mutating calls at 2-3s spacing (plus `getChat` reads at 0.15-0.4s) drew
|
|
152
|
+
* zero 429s and no `PEER_FLOOD`, so there is large headroom. It is kept because
|
|
153
|
+
* it costs nothing on the shallow stacks that are the common case.
|
|
154
|
+
*
|
|
155
|
+
* NO per-chat pop ceiling: a group drain issues one targeted unpin per RECORDED
|
|
156
|
+
* id, so it is already bounded by that list, and unpinning emits no service
|
|
157
|
+
* message — the thing the old ceiling existed to ration does not exist. The
|
|
158
|
+
* per-minute budget still applies and still yields to the next boot.
|
|
159
|
+
*/
|
|
160
|
+
export const GROUP_SWEEP_GATE: SweepGate = {
|
|
161
|
+
minCallDelayMs: 6_000,
|
|
162
|
+
maxPopsPerChatPerSweep: Number.POSITIVE_INFINITY,
|
|
163
|
+
maxPinOpsPerMinute: 10,
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** Backoff after a 429: honour `retry_after`, then double it (3→6→12→24s). */
|
|
167
|
+
export const FLOOD_BACKOFF_FACTOR = 2
|
|
168
|
+
|
|
169
|
+
/** Consecutive 429s for one target before we abort it and defer to next boot. */
|
|
170
|
+
export const MAX_CONSECUTIVE_FLOOD_WAITS = 3
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Identical `retry_after` values seen with ZERO observed progress before we
|
|
174
|
+
* abort. This is the `unpinAllChatMessages` signature — the server hands back
|
|
175
|
+
* the same wait forever because the query never becomes final.
|
|
176
|
+
*/
|
|
177
|
+
export const MAX_IDENTICAL_RETRY_AFTER_NO_PROGRESS = 2
|
|
178
|
+
|
|
179
|
+
/** A 429 asking for longer than this trips the breaker outright. */
|
|
180
|
+
export const CIRCUIT_BREAKER_RETRY_AFTER_SEC = 60
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Gap between the two `getChat` reads that must AGREE before a drain is called
|
|
184
|
+
* verified. `getChat` is a CACHE and it lies right after a mutation: a lone
|
|
185
|
+
* empty read taken immediately after an unpin means nothing. Two agreeing reads
|
|
186
|
+
* ~1.8s apart is the cheapest observation that survives the cache.
|
|
187
|
+
*/
|
|
188
|
+
export const VERIFY_READ_GAP_MS = 1_800
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* How many read PAIRS may be taken while chasing two agreeing values. More than
|
|
192
|
+
* one because the observed lie (`pinned_message: None` on a non-empty stack,
|
|
193
|
+
* ~0.15s after an unpin) settles within ~0.55s: the first pair straddles the
|
|
194
|
+
* transient and disagrees, the second reads the settled value. Bounded so an
|
|
195
|
+
* genuinely flapping chat degrades to "unknown" rather than spinning.
|
|
196
|
+
*/
|
|
197
|
+
export const VERIFY_READ_MAX_PAIRS = 3
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* THE destructive-remedy policy, in ONE place.
|
|
201
|
+
*
|
|
202
|
+
* `false` ⇒ `unpinAllForumTopicMessages` is never used by the routine sweep.
|
|
203
|
+
*
|
|
204
|
+
* Rationale: it is genuinely topic-scoped (unlike every other pin verb a bot
|
|
205
|
+
* has) and it drains a topic in one silent call, which makes it tempting. But
|
|
206
|
+
* within that topic it is INDISCRIMINATE — measured, the unpin-all family
|
|
207
|
+
* destroyed a pre-existing pin that the gateway never placed. The routine
|
|
208
|
+
* remedy is therefore a targeted unpin of the ids the gateway recorded, and
|
|
209
|
+
* this verb stays behind an explicit per-deployment opt-in for the case where
|
|
210
|
+
* an operator knowingly wants a topic's pins cleared wholesale.
|
|
211
|
+
*
|
|
212
|
+
* Kept as one constant + {@link mayUnpinAllForumTopic} rather than a condition
|
|
213
|
+
* spread through the control flow, so the policy is auditable in one place.
|
|
214
|
+
*/
|
|
215
|
+
export const UNPIN_ALL_FORUM_TOPIC_ENABLED = false
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* May this sweep use the wholesale topic drain?
|
|
219
|
+
*
|
|
220
|
+
* `override` is the per-deployment escape hatch (`allowUnpinAllForumTopic`,
|
|
221
|
+
* bound to `SWITCHROOM_PIN_SWEEP_UNPIN_ALL_TOPIC=1`); when it is undefined the
|
|
222
|
+
* standing policy above decides.
|
|
223
|
+
*/
|
|
224
|
+
export function mayUnpinAllForumTopic(kind: SweepChatKind, override?: boolean): boolean {
|
|
225
|
+
if (kind !== 'forum-topic') return false
|
|
226
|
+
return override ?? UNPIN_ALL_FORUM_TOPIC_ENABLED
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/** Gates for a chat class. Forum topics ride the group gates — same chat. */
|
|
230
|
+
export function gateFor(kind: SweepChatKind): SweepGate {
|
|
231
|
+
return kind === 'dm' ? DM_SWEEP_GATE : GROUP_SWEEP_GATE
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/** Which per-minute budget a class draws from. Forum topics are groups. */
|
|
235
|
+
export function budgetClassFor(kind: SweepChatKind): 'dm' | 'group' {
|
|
236
|
+
return kind === 'dm' ? 'dm' : 'group'
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// ─── Chat classification ─────────────────────────────────────────────────────
|
|
240
|
+
|
|
241
|
+
/** True iff `chatId` is a user DM: a positive integer. Groups are negative. */
|
|
242
|
+
export function isDmChatId(chatId: string): boolean {
|
|
243
|
+
const n = Number(chatId)
|
|
244
|
+
return Number.isFinite(n) && Number.isInteger(n) && n > 0
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* Classify a sweep target.
|
|
249
|
+
*
|
|
250
|
+
* `forum-topic` requires BOTH a forum chat and a known `threadId`, because the
|
|
251
|
+
* only thing that distinction now buys is eligibility for the opt-in wholesale
|
|
252
|
+
* topic drain, whose verb takes a `message_thread_id`. Both group classes take
|
|
253
|
+
* the same routine path (targeted unpin of recorded ids), so a forum chat whose
|
|
254
|
+
* orphans have no recorded thread degrades to `supergroup` with no loss of
|
|
255
|
+
* capability.
|
|
256
|
+
*/
|
|
257
|
+
export function classifyChatForSweep(args: {
|
|
258
|
+
chatId: string
|
|
259
|
+
threadId?: number
|
|
260
|
+
isForum?: boolean
|
|
261
|
+
}): SweepChatKind {
|
|
262
|
+
if (isDmChatId(args.chatId)) return 'dm'
|
|
263
|
+
if (args.isForum === true && args.threadId != null) return 'forum-topic'
|
|
264
|
+
return 'supergroup'
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// ─── Seeding the obligation set from the durable stores ─────────────────────
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* The minimum a persisted row must expose to become a sweep target.
|
|
271
|
+
*
|
|
272
|
+
* `messageId` / `activityMessageId` are read too, because the GROUP drain is
|
|
273
|
+
* not a blind stack walk — it unpins exactly the ids the gateway is on record
|
|
274
|
+
* as having pinned, and those ids must be captured at SCAN time, before the
|
|
275
|
+
* boot reapers empty the very stores they come from.
|
|
276
|
+
*/
|
|
277
|
+
export interface SweepCandidateRow {
|
|
278
|
+
chatId: string
|
|
279
|
+
threadId?: number | null
|
|
280
|
+
/** status-pin rows: the pinned message. */
|
|
281
|
+
messageId?: number
|
|
282
|
+
/** activity-card rows name it differently. */
|
|
283
|
+
activityMessageId?: number
|
|
284
|
+
/** activity-card rows: only a card that was actually pinned is on the stack. */
|
|
285
|
+
pinned?: boolean
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* The DISTINCT `(chat, thread)` targets that appear in ANY persisted store —
|
|
290
|
+
* the set of places a prior session is on record as having pinned something.
|
|
291
|
+
* These, and only these, are worth sweeping: an unpin-all is wasted on a chat
|
|
292
|
+
* the agent never pinned in, and blind-sweeping every known chat is exactly the
|
|
293
|
+
* kind of unbounded write volume the rate gates exist to prevent.
|
|
294
|
+
*
|
|
295
|
+
* Pure over its inputs; the gateway passes the loaded snapshots (status pins,
|
|
296
|
+
* activity cards, queued/busy-ack cards). Rows are deduped on the FULL
|
|
297
|
+
* `(chatId, threadId)` key — the old collector deduped on chat alone, which
|
|
298
|
+
* silently collapsed every topic of a forum into one target and threw away the
|
|
299
|
+
* `message_thread_id` the topic-scoped drain verb needs.
|
|
300
|
+
*/
|
|
301
|
+
export function collectSweepTargets(input: {
|
|
302
|
+
statusPins?: readonly SweepCandidateRow[]
|
|
303
|
+
activityCards?: readonly SweepCandidateRow[]
|
|
304
|
+
queuedCards?: readonly SweepCandidateRow[]
|
|
305
|
+
}): SweepTarget[] {
|
|
306
|
+
const out = new Map<string, SweepTarget>()
|
|
307
|
+
const add = (rows?: readonly SweepCandidateRow[]): void => {
|
|
308
|
+
if (rows == null) return
|
|
309
|
+
for (const r of rows) {
|
|
310
|
+
if (typeof r.chatId !== 'string' || r.chatId.length === 0) continue
|
|
311
|
+
const threadId = r.threadId ?? undefined
|
|
312
|
+
const key = sweepTargetKey(r.chatId, threadId)
|
|
313
|
+
let target = out.get(key)
|
|
314
|
+
if (target == null) {
|
|
315
|
+
target = { chatId: r.chatId, threadId, messageIds: [] }
|
|
316
|
+
out.set(key, target)
|
|
317
|
+
}
|
|
318
|
+
// An activity card that was never pinned is not on the pin stack, so its
|
|
319
|
+
// id must NOT join the unpin list: issuing an unpin for it would spend
|
|
320
|
+
// rate budget on a call that provably cannot pop anything.
|
|
321
|
+
const id = r.messageId ?? (r.pinned === true ? r.activityMessageId : undefined)
|
|
322
|
+
if (typeof id === 'number' && !target.messageIds!.includes(id)) target.messageIds!.push(id)
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
add(input.statusPins)
|
|
326
|
+
add(input.activityCards)
|
|
327
|
+
add(input.queuedCards)
|
|
328
|
+
return [...out.values()]
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* The messageIds in `chatId` that must SURVIVE a drain because they belong to
|
|
333
|
+
* deliberately-retained store rows: unexpired time-scoped `tool:` pins (the
|
|
334
|
+
* `pin_message` MCP tool, #3001), which `runStatusPinBootCleanup` intentionally
|
|
335
|
+
* KEEPS across restarts. Work-scoped rows (no `expiresAt`) are stale by
|
|
336
|
+
* definition after a restart and are NOT re-pin candidates; expired
|
|
337
|
+
* time-scoped rows are due for sweeping.
|
|
338
|
+
*/
|
|
339
|
+
export function unexpiredStoreRepinIds(
|
|
340
|
+
rows: readonly { chatId: string; messageId: number; expiresAt?: number }[],
|
|
341
|
+
chatId: string,
|
|
342
|
+
now: number,
|
|
343
|
+
): number[] {
|
|
344
|
+
const out: number[] = []
|
|
345
|
+
for (const r of rows) {
|
|
346
|
+
if (r.chatId !== chatId) continue
|
|
347
|
+
if (r.expiresAt != null && r.expiresAt > now) out.push(r.messageId)
|
|
348
|
+
}
|
|
349
|
+
return out
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
// ─── Telegram error shapes ───────────────────────────────────────────────────
|
|
353
|
+
|
|
354
|
+
function description(err: unknown): string {
|
|
355
|
+
if (err != null && typeof err === 'object') {
|
|
356
|
+
const o = err as { description?: unknown; message?: unknown }
|
|
357
|
+
if (typeof o.description === 'string' && o.description.length > 0) {
|
|
358
|
+
return o.description.toLowerCase()
|
|
359
|
+
}
|
|
360
|
+
if (typeof o.message === 'string' && o.message.length > 0) return o.message.toLowerCase()
|
|
361
|
+
}
|
|
362
|
+
return String(err).toLowerCase()
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/** Seconds Telegram asked us to wait, or null when this is not a flood wait. */
|
|
366
|
+
export function retryAfterSeconds(err: unknown): number | null {
|
|
367
|
+
if (err == null || typeof err !== 'object') return null
|
|
368
|
+
const o = err as { error_code?: unknown; parameters?: { retry_after?: unknown } }
|
|
369
|
+
const fromParams = o.parameters?.retry_after
|
|
370
|
+
if (typeof fromParams === 'number' && fromParams >= 0) return fromParams
|
|
371
|
+
if (o.error_code === 429) {
|
|
372
|
+
const m = /retry after (\d+)/.exec(description(err))
|
|
373
|
+
if (m != null) return Number(m[1])
|
|
374
|
+
return 0
|
|
375
|
+
}
|
|
376
|
+
return null
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* `400 … PEER_FLOOD` — Telegram telling the bot it is already flood-limited at
|
|
381
|
+
* the account level. Never retry through this; stop writing entirely.
|
|
382
|
+
*/
|
|
383
|
+
export function isPeerFloodError(err: unknown): boolean {
|
|
384
|
+
return description(err).includes('peer_flood')
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* The bot is not allowed to manage pins here. Verified live: a bot without
|
|
389
|
+
* `can_pin_messages` gets `400 "not enough rights to manage pinned messages in
|
|
390
|
+
* the chat"` from EVERY pin method — an honest, stable rejection, unlike the
|
|
391
|
+
* silent-success unpin. So a rights failure is terminal for the chat, not a
|
|
392
|
+
* retry.
|
|
393
|
+
*/
|
|
394
|
+
export function isPinRightsError(err: unknown): boolean {
|
|
395
|
+
return description(err).includes('not enough rights')
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/**
|
|
399
|
+
* `unpinChatMessage` with NO `message_id` on an EMPTY stack answers `400
|
|
400
|
+
* "message to unpin not found"`. That is a real, positive termination signal —
|
|
401
|
+
* one of the very few honest answers in this API surface — so we use it as a
|
|
402
|
+
* corroborating check alongside the two agreeing `getChat` reads.
|
|
403
|
+
*/
|
|
404
|
+
export function isNothingToUnpinError(err: unknown): boolean {
|
|
405
|
+
return description(err).includes('message to unpin not found')
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
// ─── Per-bot op budget ───────────────────────────────────────────────────────
|
|
409
|
+
|
|
410
|
+
/**
|
|
411
|
+
* Sliding-window budget for pin ops per bot per minute, per class. Shared
|
|
412
|
+
* across every chat in one sweep — the flood ledger Telegram keeps is per BOT,
|
|
413
|
+
* not per chat, so a per-chat-only gate is no gate at all when six chats are
|
|
414
|
+
* swept concurrently at boot.
|
|
415
|
+
*/
|
|
416
|
+
export interface PinOpBudget {
|
|
417
|
+
/** Ms to wait before the next op in `cls` is within budget (0 = go now). */
|
|
418
|
+
waitMs(cls: 'dm' | 'group'): number
|
|
419
|
+
/** Record that an op just went out. */
|
|
420
|
+
record(cls: 'dm' | 'group'): void
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
export function createPinOpBudget(now: () => number): PinOpBudget {
|
|
424
|
+
const windows: Record<'dm' | 'group', number[]> = { dm: [], group: [] }
|
|
425
|
+
const limits: Record<'dm' | 'group', number> = {
|
|
426
|
+
dm: DM_SWEEP_GATE.maxPinOpsPerMinute,
|
|
427
|
+
group: GROUP_SWEEP_GATE.maxPinOpsPerMinute,
|
|
428
|
+
}
|
|
429
|
+
const prune = (cls: 'dm' | 'group', t: number): void => {
|
|
430
|
+
const w = windows[cls]
|
|
431
|
+
while (w.length > 0 && t - w[0] >= 60_000) w.shift()
|
|
432
|
+
}
|
|
433
|
+
return {
|
|
434
|
+
waitMs(cls) {
|
|
435
|
+
const t = now()
|
|
436
|
+
prune(cls, t)
|
|
437
|
+
const w = windows[cls]
|
|
438
|
+
if (w.length < limits[cls]) return 0
|
|
439
|
+
// Wait until the oldest op in the window ages out of the minute.
|
|
440
|
+
return Math.max(0, 60_000 - (t - w[0]))
|
|
441
|
+
},
|
|
442
|
+
record(cls) {
|
|
443
|
+
windows[cls].push(now())
|
|
444
|
+
},
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// ─── Sweeper ─────────────────────────────────────────────────────────────────
|
|
449
|
+
|
|
450
|
+
export interface SweepTarget {
|
|
451
|
+
chatId: string
|
|
452
|
+
threadId?: number
|
|
453
|
+
/** Whether the chat is a forum supergroup, when the caller knows. */
|
|
454
|
+
isForum?: boolean
|
|
455
|
+
/**
|
|
456
|
+
* Message ids the gateway is on record as having pinned here, captured at
|
|
457
|
+
* scan time. The GROUP drain unpins exactly these — a pin the gateway did not
|
|
458
|
+
* place is not its to remove. Unioned with `deps.recordedPinIds` so the lazy
|
|
459
|
+
* first-inbound caller, which has no scan snapshot, still gets a list.
|
|
460
|
+
*/
|
|
461
|
+
messageIds?: number[]
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
export type SweepStatus =
|
|
465
|
+
/** Verified empty (two agreeing reads). The obligation is discharged. */
|
|
466
|
+
| 'drained'
|
|
467
|
+
/** Never eligible this boot (lost the startup mutex). Nothing was written. */
|
|
468
|
+
| 'skipped-not-eligible'
|
|
469
|
+
/** Bot is not an admin with can_pin_messages here. Nothing was written. */
|
|
470
|
+
| 'skipped-no-rights'
|
|
471
|
+
/**
|
|
472
|
+
* A GROUP target with no recorded pin ids. Nothing is owed and nothing may be
|
|
473
|
+
* done: the gateway only ever removes pins it placed, so with no record there
|
|
474
|
+
* is no safe action (an unpin-all would take other people's pins with it).
|
|
475
|
+
*/
|
|
476
|
+
| 'skipped-nothing-recorded'
|
|
477
|
+
/** Already discharged in a previous boot / earlier in this one. */
|
|
478
|
+
| 'already-drained'
|
|
479
|
+
/**
|
|
480
|
+
* This process already attempted this target. NOT an error: a re-attempt in
|
|
481
|
+
* the same process reads the same state through the same gates and can only
|
|
482
|
+
* burn the attempt budget and the flood ledger.
|
|
483
|
+
*/
|
|
484
|
+
| 'already-attempted'
|
|
485
|
+
/** Attempt budget exhausted (SWEEP_MAX_ATTEMPTS boots). Never retried. */
|
|
486
|
+
| 'forfeited'
|
|
487
|
+
/** Hit the per-chat pop cap or the per-minute budget. Cursor persisted. */
|
|
488
|
+
| 'deferred-budget'
|
|
489
|
+
/** Flood-waited out (consecutive 429s, or a pegged identical retry_after). */
|
|
490
|
+
| 'deferred-flood'
|
|
491
|
+
/** Writes stopped: PEER_FLOOD or retry_after > 60s. Cursor persisted. */
|
|
492
|
+
| 'aborted-circuit-breaker'
|
|
493
|
+
/** Drain ran but the stack is provably NOT empty — never reported drained. */
|
|
494
|
+
| 'incomplete'
|
|
495
|
+
/** An unexpected failure. Cursor persisted; retried next boot. */
|
|
496
|
+
| 'error'
|
|
497
|
+
|
|
498
|
+
export interface SweepResult {
|
|
499
|
+
status: SweepStatus
|
|
500
|
+
/** Pops OBSERVED to have landed in THIS sweep (never counted from an API
|
|
501
|
+
* result — see the module docblock). */
|
|
502
|
+
popped: number
|
|
503
|
+
/**
|
|
504
|
+
* Targeted unpins ISSUED in this sweep. Deliberately separate from `popped`:
|
|
505
|
+
* an unpin aimed at a mid-stack id has no observable effect on the only thing
|
|
506
|
+
* a bot can read (`getChat().pinned_message`, the top), so issuing one is not
|
|
507
|
+
* evidence it landed. Keeping the two apart is what stops the `ok:true`
|
|
508
|
+
* silent no-op from being laundered back into a success count.
|
|
509
|
+
*/
|
|
510
|
+
issued?: number
|
|
511
|
+
/** Detail for the log line. */
|
|
512
|
+
detail?: string
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
export interface StalePinSweepDeps {
|
|
516
|
+
/**
|
|
517
|
+
* Top of the pin stack (`getChat().pinned_message.message_id`), or null when
|
|
518
|
+
* the chat reports nothing pinned. This is the ONLY source of drain progress
|
|
519
|
+
* — deliberately, because the unpin RESULT is worthless (see the module
|
|
520
|
+
* docblock). May throw; a throw is treated as "unknown", never as "empty".
|
|
521
|
+
*/
|
|
522
|
+
getTopPinnedMessageId: (chatId: string) => Promise<number | null>
|
|
523
|
+
/** `pinChatMessage(chat, id, { disable_notification: true })`. */
|
|
524
|
+
pinSilent: (chatId: string, messageId: number) => Promise<unknown>
|
|
525
|
+
/** `unpinChatMessage(chat, id)`. Its resolution proves NOTHING. */
|
|
526
|
+
unpin: (chatId: string, messageId: number) => Promise<unknown>
|
|
527
|
+
/** `unpinAllForumTopicMessages(chat, thread)` — the one topic-scoped verb. */
|
|
528
|
+
unpinAllForumTopicMessages: (chatId: string, threadId: number) => Promise<unknown>
|
|
529
|
+
/**
|
|
530
|
+
* `getChatMember(chat, self)` folded to "is administrator AND can_pin_messages".
|
|
531
|
+
* Consulted BEFORE any write in a group. A throw is treated as "no rights".
|
|
532
|
+
*/
|
|
533
|
+
canPinInChat: (chatId: string) => Promise<boolean>
|
|
534
|
+
/**
|
|
535
|
+
* Message ids in this chat that must be RE-PINNED once the drain finishes:
|
|
536
|
+
* live in-memory claims plus the deliberately-retained store rows (unexpired
|
|
537
|
+
* `tool:` pins). Read at the START of the drain, restored at the end.
|
|
538
|
+
*/
|
|
539
|
+
protectedMessageIds: (chatId: string) => number[]
|
|
540
|
+
/**
|
|
541
|
+
* Message ids the gateway is on record as having pinned in this target, read
|
|
542
|
+
* LIVE from the durable stores. Unioned with `SweepTarget.messageIds` so the
|
|
543
|
+
* lazy first-inbound sweep (which carries no boot scan) still has a list.
|
|
544
|
+
*/
|
|
545
|
+
recordedPinIds: (chatId: string, threadId?: number) => number[]
|
|
546
|
+
/** Gate: this gateway won the startup mutex and owns the shared pin state. */
|
|
547
|
+
eligible: () => boolean
|
|
548
|
+
sleep: (ms: number) => Promise<void>
|
|
549
|
+
now: () => number
|
|
550
|
+
/** Durable obligation ledger. */
|
|
551
|
+
store: { path: string; fs: SweepStoreFsSeam }
|
|
552
|
+
/**
|
|
553
|
+
* Per-deployment override of {@link UNPIN_ALL_FORUM_TOPIC_ENABLED}. Leave
|
|
554
|
+
* undefined to take the standing policy; `true` opts this deployment into the
|
|
555
|
+
* WHOLESALE topic drain, which also removes pins the gateway did not place.
|
|
556
|
+
*/
|
|
557
|
+
allowUnpinAllForumTopic?: boolean
|
|
558
|
+
log?: (line: string) => void
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
export interface StalePinSweeper {
|
|
562
|
+
/**
|
|
563
|
+
* Sweep one `(chat, thread)` target to completion or to a gate.
|
|
564
|
+
*
|
|
565
|
+
* AT MOST ONCE PER TARGET PER PROCESS. The lazy first-inbound caller fires on
|
|
566
|
+
* every inbound message, so without this a busy chat whose drain did not
|
|
567
|
+
* finish would re-attempt on every message — burning the 8-boot attempt
|
|
568
|
+
* budget in 8 messages and running overlapping drains against the same chat.
|
|
569
|
+
* Concurrent callers for the same target share the in-flight promise.
|
|
570
|
+
*/
|
|
571
|
+
sweepTarget: (target: SweepTarget) => Promise<SweepResult>
|
|
572
|
+
/** True once the breaker tripped — every later target short-circuits. */
|
|
573
|
+
isCircuitOpen: () => boolean
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
export function createStalePinSweeper(deps: StalePinSweepDeps): StalePinSweeper {
|
|
577
|
+
const log = deps.log ?? ((l: string) => process.stderr.write(l))
|
|
578
|
+
const budget = createPinOpBudget(deps.now)
|
|
579
|
+
let circuitOpen = false
|
|
580
|
+
let lastWriteAt = 0
|
|
581
|
+
|
|
582
|
+
/** Persist the obligation. Never throws — durability is best-effort. */
|
|
583
|
+
const commit = (cursor: SweepCursor): void => {
|
|
584
|
+
try {
|
|
585
|
+
upsertSweepCursor(deps.store.path, deps.store.fs, cursor, log)
|
|
586
|
+
} catch (err) {
|
|
587
|
+
log(`stale-pin-sweep: cursor persist failed: ${(err as Error).message}\n`)
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
const loadCursor = (t: SweepTarget, kind: SweepChatKind): SweepCursor => {
|
|
592
|
+
const key = sweepTargetKey(t.chatId, t.threadId)
|
|
593
|
+
let rows: SweepCursor[] = []
|
|
594
|
+
try {
|
|
595
|
+
rows = loadSweepCursors(deps.store.path, deps.store.fs)
|
|
596
|
+
} catch {
|
|
597
|
+
rows = []
|
|
598
|
+
}
|
|
599
|
+
const existing = rows.find((c) => sweepTargetKey(c.chatId, c.threadId) === key)
|
|
600
|
+
if (existing != null) return { ...existing, kind }
|
|
601
|
+
return {
|
|
602
|
+
chatId: t.chatId,
|
|
603
|
+
threadId: t.threadId,
|
|
604
|
+
kind,
|
|
605
|
+
popped: 0,
|
|
606
|
+
done: false,
|
|
607
|
+
attempts: 0,
|
|
608
|
+
updatedAt: deps.now(),
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
/**
|
|
613
|
+
* Wait until BOTH the per-class minimum inter-call delay and the per-minute
|
|
614
|
+
* bot budget allow the next write. Returns false when the budget cannot be
|
|
615
|
+
* satisfied inside this sweep (caller defers to the next boot rather than
|
|
616
|
+
* sleeping a whole minute inside a boot path).
|
|
617
|
+
*/
|
|
618
|
+
const awaitWriteSlot = async (kind: SweepChatKind): Promise<boolean> => {
|
|
619
|
+
const cls = budgetClassFor(kind)
|
|
620
|
+
const wait = budget.waitMs(cls)
|
|
621
|
+
// A budget wait means we've already spent this minute's whole allowance.
|
|
622
|
+
// Sleeping it off inside boot would stall the sweep for a full minute for
|
|
623
|
+
// no benefit — the cursor is durable, so yielding to the next boot is
|
|
624
|
+
// strictly better.
|
|
625
|
+
if (wait > 0) return false
|
|
626
|
+
const gate = gateFor(kind)
|
|
627
|
+
const since = deps.now() - lastWriteAt
|
|
628
|
+
if (lastWriteAt > 0 && since < gate.minCallDelayMs) {
|
|
629
|
+
await deps.sleep(gate.minCallDelayMs - since)
|
|
630
|
+
}
|
|
631
|
+
lastWriteAt = deps.now()
|
|
632
|
+
budget.record(cls)
|
|
633
|
+
return true
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
/**
|
|
637
|
+
* Two `getChat` reads separated by {@link VERIFY_READ_GAP_MS} that must
|
|
638
|
+
* AGREE, retried up to {@link VERIFY_READ_MAX_PAIRS} times. A read that
|
|
639
|
+
* THROWS is "unknown", never "empty".
|
|
640
|
+
*
|
|
641
|
+
* This exists because of a measured lie with a counter-intuitive shape: right
|
|
642
|
+
* after an unpin, `getChat` transiently reports `pinned_message: None` on a
|
|
643
|
+
* stack that is NOT empty. Tight sampling: at ~0.15s it read `None`; at
|
|
644
|
+
* ~0.55s and every read after, it read the next real pin. The transient value
|
|
645
|
+
* is the EMPTY one, not a stale id — the opposite of what you would guess —
|
|
646
|
+
* so a drain loop that terminates on a single `None` exits early and orphans
|
|
647
|
+
* the rest of the stack. Never treat one `None` as empty.
|
|
648
|
+
*
|
|
649
|
+
* Retrying (rather than giving up on the first disagreement) is what keeps the
|
|
650
|
+
* transient from ending an otherwise healthy drain: the second pair reads the
|
|
651
|
+
* settled value and the loop carries on.
|
|
652
|
+
*/
|
|
653
|
+
const readTopVerified = async (chatId: string): Promise<{ top: number | null } | null> => {
|
|
654
|
+
const read = async (): Promise<{ v: number | null } | null> => {
|
|
655
|
+
try {
|
|
656
|
+
return { v: await deps.getTopPinnedMessageId(chatId) }
|
|
657
|
+
} catch {
|
|
658
|
+
return null
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
let prev = await read()
|
|
662
|
+
if (prev == null) return null
|
|
663
|
+
for (let i = 0; i < VERIFY_READ_MAX_PAIRS; i++) {
|
|
664
|
+
await deps.sleep(VERIFY_READ_GAP_MS)
|
|
665
|
+
const next = await read()
|
|
666
|
+
if (next == null) return null
|
|
667
|
+
if (next.v === prev.v) return { top: next.v }
|
|
668
|
+
prev = next
|
|
669
|
+
}
|
|
670
|
+
return null
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
/** Fold an error into a control decision shared by every drain primitive. */
|
|
674
|
+
type FloodDecision =
|
|
675
|
+
| { kind: 'breaker'; detail: string }
|
|
676
|
+
| { kind: 'flood'; seconds: number }
|
|
677
|
+
| { kind: 'rights' }
|
|
678
|
+
| { kind: 'other'; detail: string }
|
|
679
|
+
|
|
680
|
+
const classify = (err: unknown): FloodDecision => {
|
|
681
|
+
if (isPeerFloodError(err)) return { kind: 'breaker', detail: 'PEER_FLOOD' }
|
|
682
|
+
const retry = retryAfterSeconds(err)
|
|
683
|
+
if (retry != null) {
|
|
684
|
+
if (retry > CIRCUIT_BREAKER_RETRY_AFTER_SEC) {
|
|
685
|
+
return { kind: 'breaker', detail: `retry_after=${retry}s > ${CIRCUIT_BREAKER_RETRY_AFTER_SEC}s` }
|
|
686
|
+
}
|
|
687
|
+
return { kind: 'flood', seconds: retry }
|
|
688
|
+
}
|
|
689
|
+
if (isPinRightsError(err)) return { kind: 'rights' }
|
|
690
|
+
return { kind: 'other', detail: (err as Error)?.message ?? String(err) }
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
/**
|
|
694
|
+
* The DM / opted-in-supergroup drain: RE-PIN the top then UNPIN it, which is
|
|
695
|
+
* the only sequence that actually pops an entry, and re-read the observed top
|
|
696
|
+
* to decide whether anything moved. A resolved unpin is never counted.
|
|
697
|
+
*/
|
|
698
|
+
const repinUnpinLoop = async (
|
|
699
|
+
target: SweepTarget,
|
|
700
|
+
kind: SweepChatKind,
|
|
701
|
+
cursor: SweepCursor,
|
|
702
|
+
): Promise<SweepResult> => {
|
|
703
|
+
const gate = gateFor(kind)
|
|
704
|
+
let popped = 0
|
|
705
|
+
let consecutiveFloods = 0
|
|
706
|
+
let lastRetryAfter: number | null = null
|
|
707
|
+
let noProgressAtSameRetryAfter = 0
|
|
708
|
+
let lastObservedTop: number | null = null
|
|
709
|
+
// True between "the unpin call resolved" and "we have re-read the top and
|
|
710
|
+
// decided whether it actually did anything". A resolved unpin is a CLAIM,
|
|
711
|
+
// never a pop.
|
|
712
|
+
let unverifiedPop = false
|
|
713
|
+
|
|
714
|
+
/** Count a pop ONLY once the observed top has moved. */
|
|
715
|
+
const creditPop = (): void => {
|
|
716
|
+
popped++
|
|
717
|
+
cursor.popped++
|
|
718
|
+
cursor.updatedAt = deps.now()
|
|
719
|
+
commit(cursor)
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
for (let i = 0; i < gate.maxPopsPerChatPerSweep; i++) {
|
|
723
|
+
if (circuitOpen) {
|
|
724
|
+
return { status: 'aborted-circuit-breaker', popped, detail: 'breaker open' }
|
|
725
|
+
}
|
|
726
|
+
const observed = await readTopVerified(target.chatId)
|
|
727
|
+
if (observed == null) {
|
|
728
|
+
// Ambiguous read (disagreeing reads or a throw). Not a drain, and NOT
|
|
729
|
+
// an empty stack. Defer rather than assert either way.
|
|
730
|
+
return { status: 'incomplete', popped, detail: 'pin-stack read unverifiable' }
|
|
731
|
+
}
|
|
732
|
+
if (unverifiedPop) {
|
|
733
|
+
// THE bug this module exists for: `unpinChatMessage` resolves ok:true on
|
|
734
|
+
// a stale entry and pops NOTHING. The same id still on top after a full
|
|
735
|
+
// repin+unpin is proof the pop did not land, whatever the API answered.
|
|
736
|
+
if (observed.top != null && observed.top === lastObservedTop) {
|
|
737
|
+
return {
|
|
738
|
+
status: 'incomplete',
|
|
739
|
+
popped,
|
|
740
|
+
detail: `no progress: message ${observed.top} still on top after a repin+unpin`,
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
creditPop()
|
|
744
|
+
unverifiedPop = false
|
|
745
|
+
}
|
|
746
|
+
if (observed.top == null) {
|
|
747
|
+
return { status: 'drained', popped }
|
|
748
|
+
}
|
|
749
|
+
lastObservedTop = observed.top
|
|
750
|
+
|
|
751
|
+
if (!(await awaitWriteSlot(kind))) {
|
|
752
|
+
return { status: 'deferred-budget', popped, detail: 'per-minute pin-op budget spent' }
|
|
753
|
+
}
|
|
754
|
+
try {
|
|
755
|
+
await deps.pinSilent(target.chatId, observed.top)
|
|
756
|
+
} catch (err) {
|
|
757
|
+
const d = classify(err)
|
|
758
|
+
if (d.kind === 'breaker') {
|
|
759
|
+
circuitOpen = true
|
|
760
|
+
return { status: 'aborted-circuit-breaker', popped, detail: d.detail }
|
|
761
|
+
}
|
|
762
|
+
if (d.kind === 'rights') return { status: 'skipped-no-rights', popped }
|
|
763
|
+
if (d.kind === 'flood') {
|
|
764
|
+
consecutiveFloods++
|
|
765
|
+
if (lastRetryAfter === d.seconds) noProgressAtSameRetryAfter++
|
|
766
|
+
else noProgressAtSameRetryAfter = 1
|
|
767
|
+
lastRetryAfter = d.seconds
|
|
768
|
+
if (
|
|
769
|
+
consecutiveFloods >= MAX_CONSECUTIVE_FLOOD_WAITS ||
|
|
770
|
+
noProgressAtSameRetryAfter >= MAX_IDENTICAL_RETRY_AFTER_NO_PROGRESS
|
|
771
|
+
) {
|
|
772
|
+
return {
|
|
773
|
+
status: 'deferred-flood',
|
|
774
|
+
popped,
|
|
775
|
+
detail: `flood: ${consecutiveFloods} consecutive 429s, retry_after=${d.seconds}s`,
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
// Honour retry_after, then double it for the next one (3→6→12→24).
|
|
779
|
+
await deps.sleep(d.seconds * 1000 * Math.pow(FLOOD_BACKOFF_FACTOR, consecutiveFloods - 1))
|
|
780
|
+
continue
|
|
781
|
+
}
|
|
782
|
+
return { status: 'error', popped, detail: d.detail }
|
|
783
|
+
}
|
|
784
|
+
// The unpin's RESULT is deliberately ignored — it resolves ok:true on a
|
|
785
|
+
// silent no-op. Only the next observed top decides whether this popped.
|
|
786
|
+
if (!(await awaitWriteSlot(kind))) {
|
|
787
|
+
return { status: 'deferred-budget', popped, detail: 'per-minute pin-op budget spent' }
|
|
788
|
+
}
|
|
789
|
+
try {
|
|
790
|
+
await deps.unpin(target.chatId, observed.top)
|
|
791
|
+
} catch (err) {
|
|
792
|
+
const d = classify(err)
|
|
793
|
+
if (d.kind === 'breaker') {
|
|
794
|
+
circuitOpen = true
|
|
795
|
+
return { status: 'aborted-circuit-breaker', popped, detail: d.detail }
|
|
796
|
+
}
|
|
797
|
+
if (d.kind === 'rights') return { status: 'skipped-no-rights', popped }
|
|
798
|
+
if (isNothingToUnpinError(err)) return { status: 'drained', popped }
|
|
799
|
+
if (d.kind === 'flood') {
|
|
800
|
+
consecutiveFloods++
|
|
801
|
+
if (lastRetryAfter === d.seconds) noProgressAtSameRetryAfter++
|
|
802
|
+
else noProgressAtSameRetryAfter = 1
|
|
803
|
+
lastRetryAfter = d.seconds
|
|
804
|
+
if (
|
|
805
|
+
consecutiveFloods >= MAX_CONSECUTIVE_FLOOD_WAITS ||
|
|
806
|
+
noProgressAtSameRetryAfter >= MAX_IDENTICAL_RETRY_AFTER_NO_PROGRESS
|
|
807
|
+
) {
|
|
808
|
+
return {
|
|
809
|
+
status: 'deferred-flood',
|
|
810
|
+
popped,
|
|
811
|
+
detail: `flood: ${consecutiveFloods} consecutive 429s, retry_after=${d.seconds}s`,
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
await deps.sleep(d.seconds * 1000 * Math.pow(FLOOD_BACKOFF_FACTOR, consecutiveFloods - 1))
|
|
815
|
+
continue
|
|
816
|
+
}
|
|
817
|
+
return { status: 'error', popped, detail: d.detail }
|
|
818
|
+
}
|
|
819
|
+
consecutiveFloods = 0
|
|
820
|
+
unverifiedPop = true
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
// Pop cap reached. Is it actually empty? One verified read decides, and it
|
|
824
|
+
// is the ONLY thing allowed to report `drained`.
|
|
825
|
+
const finalRead = await readTopVerified(target.chatId)
|
|
826
|
+
if (unverifiedPop && finalRead != null && finalRead.top !== lastObservedTop) creditPop()
|
|
827
|
+
if (finalRead != null && finalRead.top == null) return { status: 'drained', popped }
|
|
828
|
+
return {
|
|
829
|
+
status: 'deferred-budget',
|
|
830
|
+
popped,
|
|
831
|
+
detail: `hit the ${gate.maxPopsPerChatPerSweep}-pop cap for this chat; yielding to next boot`,
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
/**
|
|
836
|
+
* The GROUP drain: one targeted `unpinChatMessage` per RECORDED id.
|
|
837
|
+
*
|
|
838
|
+
* Measured, a targeted unpin in a supergroup works at ANY stack position — an
|
|
839
|
+
* entry third from the top vanished from the middle — so no re-pin is needed
|
|
840
|
+
* and nothing visible is emitted. It is deliberately scoped to ids the gateway
|
|
841
|
+
* placed: `unpinAllChatMessages` would clear the stack in one silent call and
|
|
842
|
+
* take other people's pins with it (measured, it destroyed a pre-existing
|
|
843
|
+
* pin), which in a real team chat is unrecoverable data loss.
|
|
844
|
+
*
|
|
845
|
+
* Because a mid-stack unpin is unobservable through `getChat` (which exposes
|
|
846
|
+
* only the top), `popped` — the OBSERVED count — is credited only from a top
|
|
847
|
+
* that actually moved. Every issued call is reported separately as `issued`.
|
|
848
|
+
*/
|
|
849
|
+
const targetedUnpinDrain = async (
|
|
850
|
+
target: SweepTarget,
|
|
851
|
+
kind: SweepChatKind,
|
|
852
|
+
cursor: SweepCursor,
|
|
853
|
+
ids: readonly number[],
|
|
854
|
+
): Promise<SweepResult> => {
|
|
855
|
+
const alreadyDone = new Set(cursor.doneIds ?? [])
|
|
856
|
+
const pending = ids.filter((id) => !alreadyDone.has(id))
|
|
857
|
+
if (pending.length === 0) {
|
|
858
|
+
return { status: 'drained', popped: 0, issued: 0, detail: 'every recorded id already unpinned' }
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
const before = await readTopVerified(target.chatId)
|
|
862
|
+
let issued = 0
|
|
863
|
+
let deferred: SweepResult | null = null
|
|
864
|
+
|
|
865
|
+
for (const id of pending) {
|
|
866
|
+
if (circuitOpen) {
|
|
867
|
+
deferred = { status: 'aborted-circuit-breaker', popped: 0, issued, detail: 'breaker open' }
|
|
868
|
+
break
|
|
869
|
+
}
|
|
870
|
+
if (!(await awaitWriteSlot(kind))) {
|
|
871
|
+
deferred = {
|
|
872
|
+
status: 'deferred-budget',
|
|
873
|
+
popped: 0,
|
|
874
|
+
issued,
|
|
875
|
+
detail: `per-minute pin-op budget spent with ${pending.length - issued} ids left`,
|
|
876
|
+
}
|
|
877
|
+
break
|
|
878
|
+
}
|
|
879
|
+
try {
|
|
880
|
+
// The RESULT is deliberately ignored: `unpinChatMessage` answers ok:true
|
|
881
|
+
// even when it pops nothing. Only the observed top below is evidence.
|
|
882
|
+
await deps.unpin(target.chatId, id)
|
|
883
|
+
} catch (err) {
|
|
884
|
+
const d = classify(err)
|
|
885
|
+
if (d.kind === 'breaker') {
|
|
886
|
+
circuitOpen = true
|
|
887
|
+
deferred = { status: 'aborted-circuit-breaker', popped: 0, issued, detail: d.detail }
|
|
888
|
+
break
|
|
889
|
+
}
|
|
890
|
+
if (d.kind === 'rights') {
|
|
891
|
+
deferred = { status: 'skipped-no-rights', popped: 0, issued }
|
|
892
|
+
break
|
|
893
|
+
}
|
|
894
|
+
if (d.kind === 'flood') {
|
|
895
|
+
deferred = {
|
|
896
|
+
status: 'deferred-flood',
|
|
897
|
+
popped: 0,
|
|
898
|
+
issued,
|
|
899
|
+
detail: `flood: retry_after=${d.seconds}s`,
|
|
900
|
+
}
|
|
901
|
+
break
|
|
902
|
+
}
|
|
903
|
+
if (!isNothingToUnpinError(err)) {
|
|
904
|
+
deferred = { status: 'error', popped: 0, issued, detail: d.detail }
|
|
905
|
+
break
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
issued++
|
|
909
|
+
// Durable resume point: a boot that dies here must not re-issue this id.
|
|
910
|
+
alreadyDone.add(id)
|
|
911
|
+
cursor.doneIds = [...alreadyDone]
|
|
912
|
+
cursor.updatedAt = deps.now()
|
|
913
|
+
commit(cursor)
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
const after = await readTopVerified(target.chatId)
|
|
917
|
+
if (after == null) {
|
|
918
|
+
return {
|
|
919
|
+
status: deferred?.status ?? 'incomplete',
|
|
920
|
+
popped: 0,
|
|
921
|
+
issued,
|
|
922
|
+
detail: 'pin-stack read unverifiable after the unpin pass',
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
// The whole stack is gone ⇒ every issued unpin demonstrably landed.
|
|
926
|
+
if (after.top == null) {
|
|
927
|
+
const popped = issued
|
|
928
|
+
cursor.popped += popped
|
|
929
|
+
commit(cursor)
|
|
930
|
+
return { status: 'drained', popped, issued }
|
|
931
|
+
}
|
|
932
|
+
if (deferred != null) return { ...deferred, popped: 0 }
|
|
933
|
+
const ours = new Set(ids)
|
|
934
|
+
if (ours.has(after.top)) {
|
|
935
|
+
// One of OUR ids is still on top after we unpinned it — the silent no-op,
|
|
936
|
+
// observed. Never call that drained.
|
|
937
|
+
return {
|
|
938
|
+
status: 'incomplete',
|
|
939
|
+
popped: 0,
|
|
940
|
+
issued,
|
|
941
|
+
detail: `recorded pin ${after.top} is still on top after a targeted unpin`,
|
|
942
|
+
}
|
|
943
|
+
}
|
|
944
|
+
// Top is a pin the gateway never placed. Ours were mid-stack, where the API
|
|
945
|
+
// gives no read-back, so this is the honest limit of what can be observed:
|
|
946
|
+
// the obligation is discharged, the effect is unverifiable. Tracked as
|
|
947
|
+
// `issued`, never as `popped`.
|
|
948
|
+
const popped = before != null && before.top !== after.top ? 1 : 0
|
|
949
|
+
cursor.popped += popped
|
|
950
|
+
commit(cursor)
|
|
951
|
+
return {
|
|
952
|
+
status: 'drained',
|
|
953
|
+
popped,
|
|
954
|
+
issued,
|
|
955
|
+
detail: `unpinned ${issued} recorded id(s); top ${after.top} is not ours and was left alone`,
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
/**
|
|
960
|
+
* The OPT-IN wholesale topic drain: ONE topic-scoped
|
|
961
|
+
* `unpinAllForumTopicMessages`, no loop. Removes pins the gateway did not
|
|
962
|
+
* place, which is why it is gated on {@link mayUnpinAllForumTopic}. A 429 here
|
|
963
|
+
* is the pegged TDLib re-issue loop, so it is a give-up, not a backoff — see
|
|
964
|
+
* the module docblock.
|
|
965
|
+
*/
|
|
966
|
+
const forumTopicDrain = async (
|
|
967
|
+
target: SweepTarget,
|
|
968
|
+
cursor: SweepCursor,
|
|
969
|
+
): Promise<SweepResult> => {
|
|
970
|
+
const threadId = target.threadId
|
|
971
|
+
if (threadId == null) {
|
|
972
|
+
return { status: 'skipped-nothing-recorded', popped: 0, detail: 'no message_thread_id' }
|
|
973
|
+
}
|
|
974
|
+
if (!(await awaitWriteSlot('forum-topic'))) {
|
|
975
|
+
return { status: 'deferred-budget', popped: 0, detail: 'per-minute pin-op budget spent' }
|
|
976
|
+
}
|
|
977
|
+
try {
|
|
978
|
+
await deps.unpinAllForumTopicMessages(target.chatId, threadId)
|
|
979
|
+
} catch (err) {
|
|
980
|
+
const d = classify(err)
|
|
981
|
+
if (d.kind === 'breaker') {
|
|
982
|
+
circuitOpen = true
|
|
983
|
+
return { status: 'aborted-circuit-breaker', popped: 0, detail: d.detail }
|
|
984
|
+
}
|
|
985
|
+
if (d.kind === 'rights') return { status: 'skipped-no-rights', popped: 0 }
|
|
986
|
+
if (d.kind === 'flood') {
|
|
987
|
+
// DO NOT retry. `run_affected_history_query_until_complete` re-issues
|
|
988
|
+
// the identical query while `!is_final_`, re-tripping the same server
|
|
989
|
+
// flood wait — a loop here never terminates.
|
|
990
|
+
return {
|
|
991
|
+
status: 'deferred-flood',
|
|
992
|
+
popped: 0,
|
|
993
|
+
detail: `unpinAllForumTopicMessages pegged at retry_after=${d.seconds}s; not looping`,
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
return { status: 'error', popped: 0, detail: d.detail }
|
|
997
|
+
}
|
|
998
|
+
cursor.popped++
|
|
999
|
+
cursor.updatedAt = deps.now()
|
|
1000
|
+
commit(cursor)
|
|
1001
|
+
const observed = await readTopVerified(target.chatId)
|
|
1002
|
+
if (observed != null && observed.top == null) return { status: 'drained', popped: 1 }
|
|
1003
|
+
// The chat-wide stack may legitimately still hold pins belonging to OTHER
|
|
1004
|
+
// topics — the topic-scoped verb only drains this one. That is a success
|
|
1005
|
+
// for this obligation, not an incomplete drain.
|
|
1006
|
+
return { status: 'drained', popped: 1, detail: 'topic drained; chat-wide stack may hold other topics' }
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
const sweepOnce = async (target: SweepTarget): Promise<SweepResult> => {
|
|
1010
|
+
if (!deps.eligible()) return { status: 'skipped-not-eligible', popped: 0 }
|
|
1011
|
+
if (circuitOpen) {
|
|
1012
|
+
return { status: 'aborted-circuit-breaker', popped: 0, detail: 'breaker already open' }
|
|
1013
|
+
}
|
|
1014
|
+
const kind = classifyChatForSweep(target)
|
|
1015
|
+
const cursor = loadCursor(target, kind)
|
|
1016
|
+
if (cursor.done) return { status: 'already-drained', popped: 0 }
|
|
1017
|
+
// Attempt budget exhausted: a permanently-undrainable chat (bot kicked,
|
|
1018
|
+
// rights revoked, a chat that flood-waits forever) must not re-burn the pop
|
|
1019
|
+
// budget on every boot for the rest of time.
|
|
1020
|
+
if (cursor.attempts >= SWEEP_MAX_ATTEMPTS) {
|
|
1021
|
+
return { status: 'forfeited', popped: 0, detail: `${cursor.attempts} attempts exhausted` }
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
// RIGHTS PRECHECK, before ANY write, in every group. A bot without
|
|
1025
|
+
// can_pin_messages is honestly rejected by Telegram, but burning a rejected
|
|
1026
|
+
// write per orphan against the flood ledger is pointless. DMs need no
|
|
1027
|
+
// precheck — getChatMember is not meaningful there.
|
|
1028
|
+
if (kind !== 'dm') {
|
|
1029
|
+
let allowed = false
|
|
1030
|
+
try {
|
|
1031
|
+
allowed = await deps.canPinInChat(target.chatId)
|
|
1032
|
+
} catch {
|
|
1033
|
+
allowed = false
|
|
1034
|
+
}
|
|
1035
|
+
if (!allowed) {
|
|
1036
|
+
cursor.attempts++
|
|
1037
|
+
cursor.lastStatus = 'skipped-no-rights'
|
|
1038
|
+
cursor.updatedAt = deps.now()
|
|
1039
|
+
commit(cursor)
|
|
1040
|
+
return { status: 'skipped-no-rights', popped: 0 }
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
cursor.attempts++
|
|
1045
|
+
cursor.updatedAt = deps.now()
|
|
1046
|
+
commit(cursor)
|
|
1047
|
+
|
|
1048
|
+
// Snapshot the ids that must survive BEFORE draining.
|
|
1049
|
+
const protectedIds = [...new Set(deps.protectedMessageIds(target.chatId))]
|
|
1050
|
+
|
|
1051
|
+
let result: SweepResult
|
|
1052
|
+
try {
|
|
1053
|
+
if (kind === 'dm') {
|
|
1054
|
+
// A DM is a blind drain: targeted unpin does not pop there, so the only
|
|
1055
|
+
// primitive is repin+unpin against the observed top. It needs no id
|
|
1056
|
+
// list, and there are no third-party pins in a DM to protect.
|
|
1057
|
+
result = await repinUnpinLoop(target, kind, cursor)
|
|
1058
|
+
} else if (mayUnpinAllForumTopic(kind, deps.allowUnpinAllForumTopic)) {
|
|
1059
|
+
result = await forumTopicDrain(target, cursor)
|
|
1060
|
+
} else {
|
|
1061
|
+
// The routine group path: unpin exactly the ids we recorded, minus the
|
|
1062
|
+
// ones we deliberately keep (unexpired `tool:` pins).
|
|
1063
|
+
const keep = new Set(protectedIds)
|
|
1064
|
+
const ids = [
|
|
1065
|
+
...new Set([
|
|
1066
|
+
...(target.messageIds ?? []),
|
|
1067
|
+
...deps.recordedPinIds(target.chatId, target.threadId),
|
|
1068
|
+
]),
|
|
1069
|
+
].filter((id) => !keep.has(id))
|
|
1070
|
+
result =
|
|
1071
|
+
ids.length === 0
|
|
1072
|
+
? {
|
|
1073
|
+
status: 'skipped-nothing-recorded',
|
|
1074
|
+
popped: 0,
|
|
1075
|
+
issued: 0,
|
|
1076
|
+
detail: 'no recorded pin ids for this target — nothing this gateway may remove',
|
|
1077
|
+
}
|
|
1078
|
+
: await targetedUnpinDrain(target, kind, cursor, ids)
|
|
1079
|
+
}
|
|
1080
|
+
} catch (err) {
|
|
1081
|
+
result = { status: 'error', popped: 0, detail: (err as Error).message }
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
// `skipped-nothing-recorded` discharges the obligation too: there is no id
|
|
1085
|
+
// this gateway may legitimately remove here, so re-attempting next boot can
|
|
1086
|
+
// only burn the attempt budget until it forfeits.
|
|
1087
|
+
if (result.status === 'drained' || result.status === 'skipped-nothing-recorded') {
|
|
1088
|
+
cursor.done = true
|
|
1089
|
+
}
|
|
1090
|
+
cursor.lastStatus = result.status
|
|
1091
|
+
cursor.updatedAt = deps.now()
|
|
1092
|
+
commit(cursor)
|
|
1093
|
+
|
|
1094
|
+
// Restore the deliberately-retained pins the blind DM drain cleared.
|
|
1095
|
+
// Rate-gated like any other write; a failure is logged and never fatal.
|
|
1096
|
+
//
|
|
1097
|
+
// DM ONLY, deliberately. The group path never removes a protected id in the
|
|
1098
|
+
// first place (they are filtered out of the unpin list), so re-pinning there
|
|
1099
|
+
// would double-pin — and a pin is the ONE group op that emits a visible
|
|
1100
|
+
// service message, so it would also be the only spam this sweep produces.
|
|
1101
|
+
if (kind === 'dm' && (result.status === 'drained' || result.popped > 0)) {
|
|
1102
|
+
for (const messageId of protectedIds) {
|
|
1103
|
+
if (circuitOpen) break
|
|
1104
|
+
if (!(await awaitWriteSlot(kind))) break
|
|
1105
|
+
try {
|
|
1106
|
+
await deps.pinSilent(target.chatId, messageId)
|
|
1107
|
+
} catch (err) {
|
|
1108
|
+
log(
|
|
1109
|
+
`telegram gateway: stale-pin-sweep: re-pin of protected message failed ` +
|
|
1110
|
+
`(chat=${target.chatId} msg=${messageId}): ${(err as Error).message}\n`,
|
|
1111
|
+
)
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
log(
|
|
1117
|
+
`telegram gateway: stale-pin-sweep: chat=${target.chatId} ` +
|
|
1118
|
+
`thread=${target.threadId ?? '-'} kind=${kind} status=${result.status} ` +
|
|
1119
|
+
`popped=${result.popped} issued=${result.issued ?? 0} total=${cursor.popped}` +
|
|
1120
|
+
(result.detail != null ? ` (${result.detail})` : '') +
|
|
1121
|
+
`\n`,
|
|
1122
|
+
)
|
|
1123
|
+
return result
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
// At-most-once-per-process, plus in-flight coalescing. See StalePinSweeper.
|
|
1127
|
+
const attempted = new Set<string>()
|
|
1128
|
+
const inFlight = new Map<string, Promise<SweepResult>>()
|
|
1129
|
+
|
|
1130
|
+
const sweepTarget = (target: SweepTarget): Promise<SweepResult> => {
|
|
1131
|
+
const key = sweepTargetKey(target.chatId, target.threadId)
|
|
1132
|
+
const running = inFlight.get(key)
|
|
1133
|
+
if (running != null) return running
|
|
1134
|
+
// Not eligible yet ⇒ nothing was attempted, so do NOT consume the one
|
|
1135
|
+
// allowed attempt: the boot sweep must still get its turn once the mutex
|
|
1136
|
+
// is won.
|
|
1137
|
+
if (!deps.eligible()) return Promise.resolve({ status: 'skipped-not-eligible', popped: 0 })
|
|
1138
|
+
if (attempted.has(key)) return Promise.resolve({ status: 'already-attempted', popped: 0 })
|
|
1139
|
+
attempted.add(key)
|
|
1140
|
+
const p = sweepOnce(target).finally(() => inFlight.delete(key))
|
|
1141
|
+
inFlight.set(key, p)
|
|
1142
|
+
return p
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
return { sweepTarget, isCircuitOpen: () => circuitOpen }
|
|
1146
|
+
}
|