switchroom 0.19.26 → 0.19.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/git-agent-attribution-hook.sh +144 -0
- package/dist/agent-scheduler/index.js +55 -0
- package/dist/auth-broker/index.js +116 -6
- package/dist/cli/notion-write-pretool.mjs +55 -0
- package/dist/cli/switchroom.js +2055 -1193
- package/dist/host-control/main.js +117 -7
- package/dist/vault/approvals/kernel-server.js +114 -6
- package/dist/vault/broker/server.js +114 -6
- package/package.json +6 -2
- package/profiles/_base/cron-session.sh.hbs +8 -0
- package/profiles/_base/start.sh.hbs +105 -0
- package/telegram-plugin/card-layout.ts +328 -0
- package/telegram-plugin/dist/bridge/bridge.js +93 -1
- package/telegram-plugin/dist/gateway/gateway.js +2009 -1166
- package/telegram-plugin/dist/server.js +96 -1
- package/telegram-plugin/edit-flood-fuse.ts +637 -56
- package/telegram-plugin/flood-429-ledger.ts +526 -0
- package/telegram-plugin/flood-circuit-breaker.ts +18 -0
- package/telegram-plugin/gateway/flood-reply-queue.ts +168 -0
- package/telegram-plugin/gateway/gateway.ts +58 -68
- package/telegram-plugin/gateway/narrative-lane.ts +14 -0
- package/telegram-plugin/gateway/outbound-send-path.ts +36 -0
- package/telegram-plugin/gateway/outbox-sweep.ts +183 -6
- package/telegram-plugin/gateway/pinned-message-handler.ts +12 -16
- package/telegram-plugin/gateway/status-pin-retarget.ts +72 -36
- package/telegram-plugin/gateway/status-pin-store.ts +58 -9
- package/telegram-plugin/gateway/worker-pin-reaper.ts +56 -7
- package/telegram-plugin/llm-error-present.ts +61 -2
- package/telegram-plugin/model-unavailable.ts +8 -0
- package/telegram-plugin/operator-events.ts +72 -5
- package/telegram-plugin/outbound-class.ts +81 -0
- package/telegram-plugin/provider-credit.ts +237 -0
- package/telegram-plugin/scripts/bun-test-ci.sh +36 -6
- package/telegram-plugin/send-gate.ts +24 -2
- package/telegram-plugin/status-no-truncate.ts +10 -48
- package/telegram-plugin/status-pin-driver.ts +33 -45
- package/telegram-plugin/status-pin.ts +18 -1
- package/telegram-plugin/tests/card-golden.test.ts +69 -0
- package/telegram-plugin/tests/card-lifecycle-render.test.ts +362 -0
- package/telegram-plugin/tests/card-type-distinguishability.test.ts +187 -164
- package/telegram-plugin/tests/card-variants.golden.txt +211 -0
- package/telegram-plugin/tests/card-variants.ts +366 -0
- package/telegram-plugin/tests/edit-flood-fuse-ban-awareness.test.ts +316 -0
- package/telegram-plugin/tests/edit-flood-fuse-default-deny.test.ts +319 -0
- package/telegram-plugin/tests/edit-flood-fuse.test.ts +11 -2
- package/telegram-plugin/tests/feed-edit-rate-ceiling.test.ts +462 -0
- package/telegram-plugin/tests/fixtures/real-429-stream.ts +220 -0
- package/telegram-plugin/tests/flood-429-ledger.test.ts +278 -0
- package/telegram-plugin/tests/flood-429-recorder-wiring.test.ts +128 -0
- package/telegram-plugin/tests/flood-reply-queue.test.ts +418 -0
- package/telegram-plugin/tests/outbox-sweep-flood-breaker.test.ts +221 -0
- package/telegram-plugin/tests/pinned-card-collapse.test.ts +19 -24
- package/telegram-plugin/tests/pinned-message-handler.test.ts +15 -15
- package/telegram-plugin/tests/provider-credit-402.test.ts +243 -0
- package/telegram-plugin/tests/status-pin-api.test.ts +11 -11
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +36 -37
- package/telegram-plugin/tests/status-pin-lifecycle.test.ts +602 -0
- package/telegram-plugin/tests/status-pin-retarget.test.ts +90 -62
- package/telegram-plugin/tests/status-pin-service-message-suppression.test.ts +7 -3
- package/telegram-plugin/tests/status-pin-store.test.ts +109 -60
- package/telegram-plugin/tests/status-pin.test.ts +56 -5
- package/telegram-plugin/tests/test-runner-coverage.test.ts +133 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +12 -10
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +23 -29
- package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +56 -59
- package/telegram-plugin/tests/worker-feed-terminal-edit-class.test.ts +335 -0
- package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +1 -1
- package/telegram-plugin/tool-activity-summary.ts +239 -365
- package/telegram-plugin/uat/assertions.ts +22 -11
- package/telegram-plugin/uat/feed-matcher.test.ts +24 -17
- package/telegram-plugin/worker-activity-feed.ts +105 -47
- package/vendor/hindsight-memory/CLAUDE.md +45 -0
- package/vendor/hindsight-memory/scripts/lib/config.py +33 -0
- package/vendor/hindsight-memory/scripts/recall.py +176 -7
- package/vendor/hindsight-memory/scripts/tests/test_config_recall_passthrough_env.py +170 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_min_score.py +464 -0
- package/vendor/hindsight-memory/settings.json +1 -1
|
@@ -45,6 +45,11 @@ import {
|
|
|
45
45
|
import { isShownBlock } from '../shown-ledger.js'
|
|
46
46
|
import { resolveSubagentOriginTurnKey } from '../registry/subagents-schema.js'
|
|
47
47
|
import { createRetryApiCall, retryWithThreadFallback } from '../retry-api-call.js'
|
|
48
|
+
import {
|
|
49
|
+
floodStatePath,
|
|
50
|
+
makeFloodWaitProbe,
|
|
51
|
+
makeFloodWaitRecorder,
|
|
52
|
+
} from '../flood-circuit-breaker.js'
|
|
48
53
|
|
|
49
54
|
export interface OutboxSweepDeps {
|
|
50
55
|
/** Deliver `text` to the chat. Resolves to the primary message id (best-effort). */
|
|
@@ -67,12 +72,39 @@ export interface OutboxSweepDeps {
|
|
|
67
72
|
now?: () => number
|
|
68
73
|
log?: (line: string) => void
|
|
69
74
|
quietMs?: number
|
|
75
|
+
/**
|
|
76
|
+
* Remaining ms of a KNOWN-OPEN Telegram flood window, or 0 when none — the
|
|
77
|
+
* SAME persisted breaker state (`flood-wait.json`) `robustApiCall` consults
|
|
78
|
+
* (`makeFloodWaitProbe`). When it reports an open window the sweep does not
|
|
79
|
+
* scan, claim, or send at all.
|
|
80
|
+
*
|
|
81
|
+
* Why the sweep needs its own check on top of the retry policy's: the retry
|
|
82
|
+
* policy only short-circuits windows LONGER than its in-process sleep ceiling
|
|
83
|
+
* (120s), and even then it does so per-CALL — it still costs a claim/release
|
|
84
|
+
* cycle and a log line every tick. The outbox is a safety net with no latency
|
|
85
|
+
* SLA; deferring the whole sweep while ANY window is open is strictly correct
|
|
86
|
+
* and is the only thing that makes "does not hit the wire" true.
|
|
87
|
+
*
|
|
88
|
+
* FAILS OPEN (a throwing probe ⇒ 0) for the same reason `makeFloodWaitProbe`
|
|
89
|
+
* does: a broken marker file must never permanently strand the outbox.
|
|
90
|
+
*/
|
|
91
|
+
floodWaitRemainingMs?: () => number
|
|
70
92
|
}
|
|
71
93
|
|
|
72
94
|
export interface OutboxSweepSummary {
|
|
73
95
|
scanned: number
|
|
74
96
|
delivered: number
|
|
75
97
|
skipped: number
|
|
98
|
+
/**
|
|
99
|
+
* Records whose send THREW this sweep (claim released, retried next tick).
|
|
100
|
+
* The tick-level backoff keys off this: a 5s fixed retry against a failing
|
|
101
|
+
* Telegram is how the 2026-07-27 sweep hit an open 4.4h ban 228 times.
|
|
102
|
+
*/
|
|
103
|
+
sendFailures: number
|
|
104
|
+
/** True when the whole sweep was skipped because a flood window is open. */
|
|
105
|
+
floodDeferred?: boolean
|
|
106
|
+
/** Remaining ms of the window that caused `floodDeferred`. */
|
|
107
|
+
floodRemainingMs?: number
|
|
76
108
|
}
|
|
77
109
|
|
|
78
110
|
/**
|
|
@@ -81,7 +113,20 @@ export interface OutboxSweepSummary {
|
|
|
81
113
|
export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSummary> {
|
|
82
114
|
const now = deps.now?.() ?? Date.now()
|
|
83
115
|
const log = deps.log ?? (() => {})
|
|
84
|
-
const summary: OutboxSweepSummary = { scanned: 0, delivered: 0, skipped: 0 }
|
|
116
|
+
const summary: OutboxSweepSummary = { scanned: 0, delivered: 0, skipped: 0, sendFailures: 0 }
|
|
117
|
+
|
|
118
|
+
// #3853 — do not sweep INTO a known-open flood window. Every record is
|
|
119
|
+
// preserved on disk and re-scanned on the next tick after the window closes,
|
|
120
|
+
// so this loses no delivery; issuing the send would only feed the ban.
|
|
121
|
+
const remainingMs = probeFloodWindow(deps.floodWaitRemainingMs)
|
|
122
|
+
if (remainingMs > 0) {
|
|
123
|
+
summary.floodDeferred = true
|
|
124
|
+
summary.floodRemainingMs = remainingMs
|
|
125
|
+
// NOT logged here: a 4.4h ban is ~3181 ticks, and one line per tick just
|
|
126
|
+
// moves the flood from the wire to the disk. `startOutboxSweep` logs the
|
|
127
|
+
// deferral at most once per DEFER_LOG_INTERVAL_MS.
|
|
128
|
+
return summary
|
|
129
|
+
}
|
|
85
130
|
|
|
86
131
|
// Re-queue crashed claims first (claim-then-crash before send).
|
|
87
132
|
reclaimStaleSending(deps.stateDir, now)
|
|
@@ -90,9 +135,27 @@ export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSum
|
|
|
90
135
|
if (pending.length === 0) return summary
|
|
91
136
|
const deliveredNonces = readDeliveredNonces(deps.stateDir)
|
|
92
137
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
138
|
+
// #3861 — deliver in CAPTURE order. `listPendingRecords` returns readdir
|
|
139
|
+
// order, which is filesystem-dependent (hash order on most Linux filesystems,
|
|
140
|
+
// i.e. effectively the nonce's hash). That was harmless while the outbox held
|
|
141
|
+
// at most one stray handback per turn; it is not harmless now that a
|
|
142
|
+
// multi-hour flood window can park a whole conversation's worth of queued
|
|
143
|
+
// replies, which would then land shuffled. Sort by `createdAt`, tie-broken by
|
|
144
|
+
// nonce so the order is total and deterministic.
|
|
145
|
+
const records = pending
|
|
146
|
+
.map((fileName) => readOutboxRecord(fileName, deps.stateDir))
|
|
147
|
+
.filter((r): r is OutboxRecord => r != null)
|
|
148
|
+
.sort((a, b) =>
|
|
149
|
+
a.createdAt !== b.createdAt
|
|
150
|
+
? a.createdAt - b.createdAt
|
|
151
|
+
: a.turnNonce < b.turnNonce
|
|
152
|
+
? -1
|
|
153
|
+
: a.turnNonce > b.turnNonce
|
|
154
|
+
? 1
|
|
155
|
+
: 0,
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
for (const record of records) {
|
|
96
159
|
summary.scanned++
|
|
97
160
|
|
|
98
161
|
// Resolve destination (anchor → registry chain → per-session origin). Fails
|
|
@@ -209,12 +272,24 @@ export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSum
|
|
|
209
272
|
// preserved on disk; no loss.
|
|
210
273
|
releaseClaim(record.turnNonce, deps.stateDir)
|
|
211
274
|
summary.skipped++
|
|
275
|
+
summary.sendFailures++
|
|
212
276
|
log(`outbox-sweep: send failed nonce=${record.turnNonce}: ${(err as Error).message} — will retry\n`)
|
|
213
277
|
}
|
|
214
278
|
}
|
|
215
279
|
return summary
|
|
216
280
|
}
|
|
217
281
|
|
|
282
|
+
/** Read the flood probe, failing OPEN on any throw. */
|
|
283
|
+
function probeFloodWindow(probe: (() => number) | undefined): number {
|
|
284
|
+
if (probe == null) return 0
|
|
285
|
+
try {
|
|
286
|
+
const ms = probe()
|
|
287
|
+
return Number.isFinite(ms) && ms > 0 ? ms : 0
|
|
288
|
+
} catch {
|
|
289
|
+
return 0
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
218
293
|
/**
|
|
219
294
|
* Parse a registry `turn_key` (`chatId:threadId`, `_` → null thread) into a
|
|
220
295
|
* routable chat. Exported for the gateway wiring + tests.
|
|
@@ -230,6 +305,59 @@ export function chatFromTurnKey(turnKey: string): { chatId: string; threadId: nu
|
|
|
230
305
|
/** How often the sweep ticks (aligned with the delivery-confirm sweep cadence). */
|
|
231
306
|
export const OUTBOX_SWEEP_INTERVAL_MS = 5_000
|
|
232
307
|
|
|
308
|
+
/** Ceiling on the exponential send-failure backoff. */
|
|
309
|
+
export const OUTBOX_SWEEP_BACKOFF_MAX_MS = 5 * 60_000
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Minimum gap between two "flood window still open" log lines. A 4.4h ban is
|
|
313
|
+
* ~3181 ticks; logging each one just relocates the flood from the wire to the
|
|
314
|
+
* disk, so the deferral is reported once a minute instead.
|
|
315
|
+
*/
|
|
316
|
+
export const OUTBOX_SWEEP_DEFER_LOG_INTERVAL_MS = 60_000
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Exponential backoff over the fixed 5s sweep tick (#3853).
|
|
320
|
+
*
|
|
321
|
+
* The tick interval alone is not a retry policy. On 2026-07-27 the sweep held
|
|
322
|
+
* ONE undeliverable record and re-issued its `sendMessage` every 5s for the
|
|
323
|
+
* whole 4.4h ban — 228 requests into a window the breaker already knew was
|
|
324
|
+
* open, each one answered with the same counting-down `retry_after`. The flood
|
|
325
|
+
* probe (above) closes the case where the breaker HAS a window recorded; this
|
|
326
|
+
* closes the case where it does not — a persistent failure of any other kind
|
|
327
|
+
* must not be re-driven at 12 requests/minute forever.
|
|
328
|
+
*
|
|
329
|
+
* Pure and clock-injected so the schedule is asserted in tests rather than
|
|
330
|
+
* inferred from timer behaviour.
|
|
331
|
+
*/
|
|
332
|
+
export function createSweepBackoff(cfg: { baseMs?: number; maxMs?: number } = {}): {
|
|
333
|
+
ready: (now: number) => boolean
|
|
334
|
+
noteFailure: (now: number) => number
|
|
335
|
+
noteSuccess: () => void
|
|
336
|
+
failures: () => number
|
|
337
|
+
} {
|
|
338
|
+
const baseMs = cfg.baseMs ?? OUTBOX_SWEEP_INTERVAL_MS
|
|
339
|
+
const maxMs = cfg.maxMs ?? OUTBOX_SWEEP_BACKOFF_MAX_MS
|
|
340
|
+
let failures = 0
|
|
341
|
+
let nextAllowedAt = 0
|
|
342
|
+
return {
|
|
343
|
+
ready: (now) => now >= nextAllowedAt,
|
|
344
|
+
noteFailure: (now) => {
|
|
345
|
+
failures++
|
|
346
|
+
// 5s, 10s, 20s, … capped. `failures - 1` so the FIRST failure still
|
|
347
|
+
// retries at the normal tick cadence — a one-off transient send error
|
|
348
|
+
// should not delay a real answer.
|
|
349
|
+
const delay = Math.min(maxMs, baseMs * Math.pow(2, failures - 1))
|
|
350
|
+
nextAllowedAt = now + delay
|
|
351
|
+
return delay
|
|
352
|
+
},
|
|
353
|
+
noteSuccess: () => {
|
|
354
|
+
failures = 0
|
|
355
|
+
nextAllowedAt = 0
|
|
356
|
+
},
|
|
357
|
+
failures: () => failures,
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
233
361
|
/**
|
|
234
362
|
* Own the heartbeat-tick outbox sweep entirely, keeping the timer / chunking /
|
|
235
363
|
* turn-key parse / dedup / registry-chain wiring OUT of `gateway.ts` (the line
|
|
@@ -336,21 +464,43 @@ export function startOutboxSweep(deps: {
|
|
|
336
464
|
text: string,
|
|
337
465
|
) => OutboxDeliveryMarkup | undefined
|
|
338
466
|
log?: (line: string) => void
|
|
467
|
+
/** Test seam — override the flood probe (default: the persisted breaker). */
|
|
468
|
+
floodWaitRemainingMs?: () => number
|
|
469
|
+
/** Test seam — override the tick backoff. */
|
|
470
|
+
backoff?: ReturnType<typeof createSweepBackoff>
|
|
339
471
|
}): ReturnType<typeof setInterval> | undefined {
|
|
340
472
|
if (!deps.isGatewayMain || process.env.SWITCHROOM_TG_OUTBOX_DELIVERY === '0') return undefined
|
|
341
|
-
|
|
473
|
+
// #3853 — the sweep is a full outbound machine and must consult the SAME
|
|
474
|
+
// persisted flood breaker every other outbound path does. Before this it was
|
|
475
|
+
// the one `createRetryApiCall` wiring with neither hook: it could not see an
|
|
476
|
+
// open ban (no `floodWaitRemainingMs`) and could not record one it caused (no
|
|
477
|
+
// `onFloodWait`). `scripts/check-retry-flood-hooks.mjs` now fails lint if any
|
|
478
|
+
// wiring omits either.
|
|
479
|
+
const floodProbe =
|
|
480
|
+
deps.floodWaitRemainingMs ?? makeFloodWaitProbe(floodStatePath(deps.stateDir))
|
|
481
|
+
const recordFloodWait = makeFloodWaitRecorder(floodStatePath(deps.stateDir))
|
|
482
|
+
const retry = createRetryApiCall({
|
|
483
|
+
log: deps.log,
|
|
484
|
+
floodWaitRemainingMs: floodProbe,
|
|
485
|
+
onFloodWait: (retryAfterSec) => recordFloodWait(retryAfterSec),
|
|
486
|
+
})
|
|
342
487
|
const send = createOutboxSend({
|
|
343
488
|
getBot: deps.getBot,
|
|
344
489
|
retry,
|
|
345
490
|
...(deps.resolveReplyMarkup != null ? { resolveReplyMarkup: deps.resolveReplyMarkup } : {}),
|
|
346
491
|
})
|
|
492
|
+
const backoff = deps.backoff ?? createSweepBackoff()
|
|
493
|
+
let lastDeferLogAt = 0
|
|
347
494
|
const tick = () => {
|
|
348
495
|
const bot = deps.getBot()
|
|
349
496
|
if (bot == null) return
|
|
497
|
+
const now = Date.now()
|
|
498
|
+
if (!backoff.ready(now)) return
|
|
350
499
|
void sweepOutbox({
|
|
351
500
|
stateDir: deps.stateDir,
|
|
352
501
|
log: deps.log,
|
|
353
502
|
send,
|
|
503
|
+
floodWaitRemainingMs: floodProbe,
|
|
354
504
|
textAlreadyDelivered: (chatId, threadId, text) => deps.dedupCheck(chatId, threadId ?? undefined, text),
|
|
355
505
|
registryChainLookup: (taskId) => {
|
|
356
506
|
const db = deps.getTurnsDb()
|
|
@@ -358,7 +508,34 @@ export function startOutboxSweep(deps: {
|
|
|
358
508
|
const turnKey = resolveSubagentOriginTurnKey(db, taskId)
|
|
359
509
|
return turnKey == null ? null : chatFromTurnKey(turnKey)
|
|
360
510
|
},
|
|
361
|
-
})
|
|
511
|
+
})
|
|
512
|
+
.then((summary) => {
|
|
513
|
+
// A flood-deferred sweep is neither success nor failure: it never
|
|
514
|
+
// reached the wire, so it must not clear a backoff earned by real
|
|
515
|
+
// failures, and it must not deepen one either.
|
|
516
|
+
if (summary.floodDeferred === true) {
|
|
517
|
+
const at = Date.now()
|
|
518
|
+
if (at - lastDeferLogAt >= OUTBOX_SWEEP_DEFER_LOG_INTERVAL_MS) {
|
|
519
|
+
lastDeferLogAt = at
|
|
520
|
+
deps.log?.(
|
|
521
|
+
`outbox-sweep: deferred — Telegram flood window still open for ` +
|
|
522
|
+
`${Math.ceil((summary.floodRemainingMs ?? 0) / 1000)}s; not issuing any ` +
|
|
523
|
+
`send (would feed the ban)\n`,
|
|
524
|
+
)
|
|
525
|
+
}
|
|
526
|
+
return
|
|
527
|
+
}
|
|
528
|
+
if (summary.sendFailures > 0) {
|
|
529
|
+
const delay = backoff.noteFailure(Date.now())
|
|
530
|
+
deps.log?.(
|
|
531
|
+
`outbox-sweep: ${summary.sendFailures} send failure(s) — backing off ` +
|
|
532
|
+
`${Math.round(delay / 1000)}s before the next sweep (attempt ${backoff.failures()})\n`,
|
|
533
|
+
)
|
|
534
|
+
} else {
|
|
535
|
+
backoff.noteSuccess()
|
|
536
|
+
}
|
|
537
|
+
})
|
|
538
|
+
.catch((err) => deps.log?.(`outbox-sweep: tick failed: ${(err as Error).message}\n`))
|
|
362
539
|
}
|
|
363
540
|
const timer = setInterval(tick, OUTBOX_SWEEP_INTERVAL_MS)
|
|
364
541
|
timer.unref?.()
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* silent status-pin fires. The pin call passes `disable_notification: true`,
|
|
6
6
|
* which kills the PUSH notification but NOT the in-chat service message — so we
|
|
7
7
|
* delete that service message as it lands, but ONLY for pins we own (tracked in
|
|
8
|
-
* `
|
|
8
|
+
* `statusPinClaims`). Manual/operator pins are never silent and are never
|
|
9
9
|
* touched. Only silent status pins reach here as an OUR-pin match, so the
|
|
10
10
|
* ownership check is the guard.
|
|
11
11
|
*
|
|
@@ -21,13 +21,12 @@
|
|
|
21
21
|
|
|
22
22
|
import type { Context, Filter } from 'grammy'
|
|
23
23
|
import { pinnedMessageIsOurs, type TrackedStatusPin } from './status-pin-store.js'
|
|
24
|
-
import type {
|
|
24
|
+
import type { StatusPinClaim } from './status-pin-retarget.js'
|
|
25
25
|
|
|
26
26
|
export interface PinnedMessageHandlerDeps {
|
|
27
|
-
/** Live pinKey→
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
statusPinChatIds: ReadonlyMap<string, string>
|
|
27
|
+
/** Live pinKey→claim registry of pins we own. ONE record per key (#3809):
|
|
28
|
+
* the chat id is a field of the claim, not a parallel map that can diverge. */
|
|
29
|
+
statusPinClaims: ReadonlyMap<string, StatusPinClaim>
|
|
31
30
|
/**
|
|
32
31
|
* Delete the pin service message. Bound in gateway.ts through
|
|
33
32
|
* `robustApiCall(() => lockedBot.api.deleteMessage(...))` so the raw Bot API
|
|
@@ -50,16 +49,13 @@ export async function handlePinnedMessage(
|
|
|
50
49
|
// Chat-scoped ownership (see pinnedMessageIsOurs): the match requires BOTH
|
|
51
50
|
// the messageId AND that the tracked entry lives in THIS chat, so a pin id
|
|
52
51
|
// colliding across chats can't delete a foreign (e.g. operator-manual) pin
|
|
53
|
-
// notice.
|
|
54
|
-
//
|
|
55
|
-
const trackedPins = (): TrackedStatusPin[] =>
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
}
|
|
61
|
-
return out
|
|
62
|
-
}
|
|
52
|
+
// notice. Each claim carries its own chat id (#3809), so the match can never
|
|
53
|
+
// silently degrade to messageId-only because a companion map went missing.
|
|
54
|
+
const trackedPins = (): TrackedStatusPin[] =>
|
|
55
|
+
[...deps.statusPinClaims.values()].map((c) => ({
|
|
56
|
+
chatId: c.chatId,
|
|
57
|
+
messageId: c.messageId,
|
|
58
|
+
}))
|
|
63
59
|
const isOurs = () => pinnedMessageIsOurs(trackedPins(), chatId, pinnedId)
|
|
64
60
|
|
|
65
61
|
if (!isOurs()) {
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
* leg 1 unpin the stale claim, with the row still naming the OLD id;
|
|
35
35
|
* leg 2 pin the new message from a null claim, with its own pending row.
|
|
36
36
|
*
|
|
37
|
-
* Failure semantics are inherited from `
|
|
37
|
+
* Failure semantics are inherited from `executePinLeg`, not re-invented here: a
|
|
38
38
|
* leg-1 result that is NON-NULL means the unpin was never confirmed (#3664
|
|
39
39
|
* Defect B — the old message is provably still pinned and the claim was
|
|
40
40
|
* deliberately retained). Pinning the new message then would leave two pins in
|
|
@@ -42,24 +42,45 @@
|
|
|
42
42
|
* claim is left for the next reconcile / the mid-session reaper / the boot
|
|
43
43
|
* sweep to retry.
|
|
44
44
|
*
|
|
45
|
+
* THE SINGLE PATH (#3831)
|
|
46
|
+
* -----------------------
|
|
47
|
+
* This module is the ONLY decider. `status-pin-driver.ts` used to call
|
|
48
|
+
* `decidePinAction` itself and carry its OWN copy of the expansion above — a
|
|
49
|
+
* copy that recursed without the per-leg persistence, so which of the two
|
|
50
|
+
* implementations ran decided whether the durable row survived. That copy is
|
|
51
|
+
* deleted, and the driver's parameter type (`PinLegAction`) can no longer even
|
|
52
|
+
* express a repin, so a second one cannot grow back.
|
|
53
|
+
*
|
|
45
54
|
* Dependency-free apart from the pure decision, so it is provable in isolation
|
|
46
55
|
* (`telegram-plugin/tests/status-pin-retarget.test.ts`); the gateway owns the
|
|
47
|
-
* wiring of `
|
|
56
|
+
* wiring of `runPin` / `commit`.
|
|
48
57
|
*/
|
|
49
58
|
|
|
50
|
-
import type { DesiredPin, PinState } from '../status-pin.js'
|
|
59
|
+
import type { DesiredPin, PinLegAction, PinState } from '../status-pin.js'
|
|
51
60
|
import { decidePinAction } from '../status-pin.js'
|
|
52
61
|
import type { StatusPinPersistOp, StatusPinStoreFsSeam } from './status-pin-store.js'
|
|
53
62
|
import { reconcileAndPersistStatusPin } from './status-pin-store.js'
|
|
54
63
|
|
|
55
|
-
/**
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
64
|
+
/**
|
|
65
|
+
* ONE in-memory claim for a pin key — the message we believe is pinned, the
|
|
66
|
+
* chat it lives in, and when the claim was FIRST taken (the reaper's TTL age).
|
|
67
|
+
*
|
|
68
|
+
* #3809 — these were three parallel `Map`s (`statusPinState` / `statusPinChatIds`
|
|
69
|
+
* / `statusPinPinnedAt`) written together but able to diverge. A `wk:` claim
|
|
70
|
+
* whose chatId entry went missing was skipped by the mid-session reaper on
|
|
71
|
+
* EVERY pass (it cannot unpin without a chat) and excluded from the durable
|
|
72
|
+
* store-orphan net (which skips keys that DO have an in-memory claim) — so it
|
|
73
|
+
* was permanently unreapable until the next boot, silently. Collapsing the
|
|
74
|
+
* three maps into one record makes the invariant STRUCTURAL: a claim without a
|
|
75
|
+
* chat id cannot be represented, so that gap cannot be reintroduced.
|
|
76
|
+
*/
|
|
77
|
+
export interface StatusPinClaim {
|
|
78
|
+
/** Telegram message_id we believe is pinned for this key. */
|
|
79
|
+
messageId: number
|
|
80
|
+
/** Chat the pin lives in. Never empty — the reconcile refuses an empty chat. */
|
|
81
|
+
chatId: string
|
|
82
|
+
/** Wall-clock ms the claim was FIRST taken (survives a same-id re-pin). */
|
|
83
|
+
pinnedAt: number
|
|
63
84
|
}
|
|
64
85
|
|
|
65
86
|
export interface StatusPinReconcileArgs {
|
|
@@ -72,48 +93,56 @@ export interface StatusPinReconcileArgs {
|
|
|
72
93
|
/** Durable-store binding, or null when persistence is off (STATIC). */
|
|
73
94
|
persist: { path: string; fs: StatusPinStoreFsSeam } | null
|
|
74
95
|
/**
|
|
75
|
-
* Execute
|
|
76
|
-
* `
|
|
96
|
+
* Execute ONE already-decided leg against Telegram (the gateway binds
|
|
97
|
+
* `executePinLeg`). Must never throw — API errors are absorbed inside.
|
|
77
98
|
*/
|
|
78
|
-
runPin: (from: PinState | null
|
|
79
|
-
|
|
99
|
+
runPin: (action: PinLegAction, from: PinState | null) => Promise<PinState | null>
|
|
100
|
+
/** The single claim registry, keyed by pinKey. */
|
|
101
|
+
claims: Map<string, StatusPinClaim>
|
|
102
|
+
/** Injectable clock for the `pinnedAt` stamp (tests). */
|
|
103
|
+
now?: () => number
|
|
80
104
|
}
|
|
81
105
|
|
|
82
106
|
/**
|
|
83
107
|
* Drive `prev → desired` for one pin key, expanding a RETARGET into its two
|
|
84
108
|
* legs. Every other action (pin / unpin / noop) is a single leg, unchanged.
|
|
85
109
|
*
|
|
86
|
-
* Each leg is committed to the
|
|
87
|
-
* never lags the durable row — including the intermediate cleared state
|
|
88
|
-
* a retarget's two legs, which is what lets a concurrent reader (the
|
|
89
|
-
* reaper) see an honest snapshot rather than a claim on an unpinned id.
|
|
110
|
+
* Each leg is committed to the claim registry as it lands, so the in-memory
|
|
111
|
+
* claim never lags the durable row — including the intermediate cleared state
|
|
112
|
+
* between a retarget's two legs, which is what lets a concurrent reader (the
|
|
113
|
+
* `wk:` reaper) see an honest snapshot rather than a claim on an unpinned id.
|
|
90
114
|
*
|
|
91
115
|
* Callers must hold the per-key reconcile lock (`withPinReconcileLock`); `prev`
|
|
92
116
|
* is read once by the caller and both legs run inside that critical section.
|
|
93
117
|
*/
|
|
94
118
|
export async function runStatusPinReconcile(args: StatusPinReconcileArgs): Promise<void> {
|
|
95
|
-
const { pinKey, chatId, prev, desired, persist, runPin,
|
|
119
|
+
const { pinKey, chatId, prev, desired, persist, runPin, claims } = args
|
|
120
|
+
const now = args.now ?? Date.now
|
|
96
121
|
|
|
97
|
-
// Publish a leg's outcome. `pinnedAt` is
|
|
98
|
-
//
|
|
122
|
+
// Publish a leg's outcome into the single claim registry. `pinnedAt` is
|
|
123
|
+
// carried forward when a claim already exists so it keeps the FIRST-taken
|
|
124
|
+
// timestamp; clearing the claim drops it, so a re-pin ages fresh.
|
|
99
125
|
const commit = (next: PinState | null): void => {
|
|
100
126
|
if (next == null) {
|
|
101
|
-
|
|
102
|
-
registries.chatIds.delete(pinKey)
|
|
103
|
-
registries.pinnedAt.delete(pinKey)
|
|
127
|
+
claims.delete(pinKey)
|
|
104
128
|
return
|
|
105
129
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
if (!registries.pinnedAt.has(pinKey)) registries.pinnedAt.set(pinKey, Date.now())
|
|
130
|
+
const pinnedAt = claims.get(pinKey)?.pinnedAt ?? now()
|
|
131
|
+
claims.set(pinKey, { messageId: next.messageId, chatId, pinnedAt })
|
|
109
132
|
}
|
|
110
133
|
|
|
111
134
|
// One leg, with the persist ordering its action requires. Only a fresh `pin`
|
|
112
135
|
// opens the crash window that persist-BEFORE-pin closes; unpin / noop / re-pin
|
|
113
136
|
// of the same id clear or leave the row and are safe to persist after.
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
137
|
+
//
|
|
138
|
+
// The leg action is decided ONCE, below, and threaded through to both the
|
|
139
|
+
// persist-op mapping and the driver — the driver no longer re-derives it
|
|
140
|
+
// (#3831).
|
|
141
|
+
const runLeg = (
|
|
142
|
+
legAction: PinLegAction,
|
|
143
|
+
from: PinState | null,
|
|
144
|
+
): Promise<PinState | null> => {
|
|
145
|
+
if (persist == null) return runPin(legAction, from)
|
|
117
146
|
const op: StatusPinPersistOp =
|
|
118
147
|
legAction.kind === 'pin'
|
|
119
148
|
? { kind: 'pin', messageId: legAction.messageId }
|
|
@@ -124,21 +153,28 @@ export async function runStatusPinReconcile(args: StatusPinReconcileArgs): Promi
|
|
|
124
153
|
pinKey,
|
|
125
154
|
chatId,
|
|
126
155
|
op,
|
|
127
|
-
applyPin: () => runPin(
|
|
156
|
+
applyPin: () => runPin(legAction, from),
|
|
157
|
+
now: now(),
|
|
128
158
|
})
|
|
129
159
|
}
|
|
130
160
|
|
|
131
161
|
const action = decidePinAction(prev, desired)
|
|
132
162
|
|
|
133
|
-
|
|
134
|
-
|
|
163
|
+
// THE retarget expansion. This is the only copy in the codebase; the driver's
|
|
164
|
+
// parameter type (`PinLegAction`) cannot express a repin, so a second one
|
|
165
|
+
// cannot be written there again (#3831).
|
|
166
|
+
if (action.kind === 'repin') {
|
|
167
|
+
const afterUnpin = await runLeg(
|
|
168
|
+
{ kind: 'unpin', messageId: action.unpinMessageId },
|
|
169
|
+
prev,
|
|
170
|
+
)
|
|
135
171
|
commit(afterUnpin)
|
|
136
172
|
// Never-confirmed unpin (#3664 Defect B): keep the retained claim, skip the
|
|
137
173
|
// pin leg. See the docblock.
|
|
138
174
|
if (afterUnpin != null) return
|
|
139
|
-
commit(await runLeg(
|
|
175
|
+
commit(await runLeg({ kind: 'pin', messageId: action.pinMessageId }, null))
|
|
140
176
|
return
|
|
141
177
|
}
|
|
142
178
|
|
|
143
|
-
commit(await runLeg(
|
|
179
|
+
commit(await runLeg(action, prev))
|
|
144
180
|
}
|
|
@@ -70,6 +70,27 @@ export interface PersistedStatusPin {
|
|
|
70
70
|
* unpin fails (flood-wait exhausted / transient 5xx); the row is retained
|
|
71
71
|
* for retry until BOOT_UNPIN_MAX_ATTEMPTS, then forfeited. Absent = 0. */
|
|
72
72
|
attempts?: number
|
|
73
|
+
/**
|
|
74
|
+
* Wall-clock ms the pin was FIRST claimed for this key+message (#3810).
|
|
75
|
+
*
|
|
76
|
+
* Why the store needs it: the mid-session `wk:` reaper folds in rows that
|
|
77
|
+
* exist on disk but have NO in-memory claim (`storeOnlyWorkerPinCandidates`).
|
|
78
|
+
* The store used to persist no timestamp, so those candidates were stamped
|
|
79
|
+
* `pinnedAt = now` on every pass — deliberately, to avoid a spurious unpin of
|
|
80
|
+
* a pin whose real age was unknown, but with the side effect that the TTL
|
|
81
|
+
* gate could NEVER fire for a store orphan. Combined with a `wk:<agentId>`
|
|
82
|
+
* whose turnsDb row is gone (verdict `'unknown'`, never `'terminal'`), such a
|
|
83
|
+
* row was never mid-session reaped AT ALL — it waited for the next boot, the
|
|
84
|
+
* exact "stale pin glued to the top of the chat" failure the reaper exists to
|
|
85
|
+
* prevent.
|
|
86
|
+
*
|
|
87
|
+
* Recording the real claim time removes the guess: a store orphan now ages
|
|
88
|
+
* honestly and the ordinary TTL applies. Optional so a v1/v2 snapshot still
|
|
89
|
+
* loads; a row without it keeps the old conservative `now` stamp (terminal-
|
|
90
|
+
* only reaping), which self-clears within one restart since every write from
|
|
91
|
+
* this version onward carries the field.
|
|
92
|
+
*/
|
|
93
|
+
pinnedAt?: number
|
|
73
94
|
}
|
|
74
95
|
|
|
75
96
|
/** How many boots may retry a failing boot-cleanup unpin before the row is
|
|
@@ -80,13 +101,18 @@ export interface PersistedStatusPin {
|
|
|
80
101
|
export const BOOT_UNPIN_MAX_ATTEMPTS = 5
|
|
81
102
|
|
|
82
103
|
/** Envelope version. v1 had no `pending` field; a v1 row loads as a confirmed
|
|
83
|
-
* pin (pending undefined). v2 adds the optional `pending` flag.
|
|
84
|
-
* fail-open — an
|
|
104
|
+
* pin (pending undefined). v2 adds the optional `pending` flag. v3 adds the
|
|
105
|
+
* optional `pinnedAt` claim timestamp (#3810). All load fail-open — an
|
|
106
|
+
* unknown/newer version yields []. Every field added since v1 is optional, so
|
|
107
|
+
* the versions are mutually readable and a downgrade degrades rather than
|
|
108
|
+
* breaks. */
|
|
85
109
|
interface SnapshotEnvelope {
|
|
86
|
-
v: 1 | 2
|
|
110
|
+
v: 1 | 2 | 3
|
|
87
111
|
pins: PersistedStatusPin[]
|
|
88
112
|
}
|
|
89
113
|
|
|
114
|
+
const SNAPSHOT_VERSIONS = new Set([1, 2, 3])
|
|
115
|
+
|
|
90
116
|
function isPinRow(x: unknown): x is PersistedStatusPin {
|
|
91
117
|
if (x == null || typeof x !== 'object') return false
|
|
92
118
|
const o = x as Record<string, unknown>
|
|
@@ -98,7 +124,8 @@ function isPinRow(x: unknown): x is PersistedStatusPin {
|
|
|
98
124
|
typeof o.messageId === 'number' &&
|
|
99
125
|
(o.pending === undefined || typeof o.pending === 'boolean') &&
|
|
100
126
|
(o.expiresAt === undefined || typeof o.expiresAt === 'number') &&
|
|
101
|
-
(o.attempts === undefined || typeof o.attempts === 'number')
|
|
127
|
+
(o.attempts === undefined || typeof o.attempts === 'number') &&
|
|
128
|
+
(o.pinnedAt === undefined || typeof o.pinnedAt === 'number')
|
|
102
129
|
)
|
|
103
130
|
}
|
|
104
131
|
|
|
@@ -127,7 +154,7 @@ export function loadStatusPins(
|
|
|
127
154
|
}
|
|
128
155
|
if (parsed == null || typeof parsed !== 'object') return []
|
|
129
156
|
const env = parsed as Record<string, unknown>
|
|
130
|
-
if (
|
|
157
|
+
if (typeof env.v !== 'number' || !SNAPSHOT_VERSIONS.has(env.v) || !Array.isArray(env.pins)) return []
|
|
131
158
|
return env.pins.filter(isPinRow)
|
|
132
159
|
}
|
|
133
160
|
|
|
@@ -143,7 +170,7 @@ export function persistStatusPins(
|
|
|
143
170
|
snapshot: readonly PersistedStatusPin[],
|
|
144
171
|
log: (line: string) => void = (l) => process.stderr.write(l),
|
|
145
172
|
): void {
|
|
146
|
-
const env: SnapshotEnvelope = { v:
|
|
173
|
+
const env: SnapshotEnvelope = { v: 3, pins: [...snapshot] }
|
|
147
174
|
const tmp = path + '.tmp'
|
|
148
175
|
try {
|
|
149
176
|
fs.writeFileSync(tmp, JSON.stringify(env))
|
|
@@ -427,16 +454,34 @@ export function reconcileAndPersistStatusPin(args: {
|
|
|
427
454
|
/** Execute the real pin/unpin; returns the confirmed message id (pin) or
|
|
428
455
|
* null (cleared). Must never throw — API errors are swallowed inside. */
|
|
429
456
|
applyPin: () => Promise<{ messageId: number } | null>
|
|
457
|
+
/** Clock for the `pinnedAt` stamp (#3810). Defaults to wall clock. */
|
|
458
|
+
now?: number
|
|
430
459
|
log?: (line: string) => void
|
|
431
460
|
}): Promise<{ messageId: number } | null> {
|
|
432
461
|
const { path, fs, pinKey, chatId, op } = args
|
|
433
462
|
const log = args.log ?? ((l: string) => process.stderr.write(l))
|
|
463
|
+
const now = args.now ?? Date.now()
|
|
464
|
+
|
|
465
|
+
/**
|
|
466
|
+
* The claim age to persist for `messageId`. Carried forward from the row
|
|
467
|
+
* already on disk when it names the SAME message (a steady-state re-write
|
|
468
|
+
* must not reset the age and hand a stale pin a fresh TTL lease); stamped
|
|
469
|
+
* `now` for a genuinely new pin. Mirrors the in-memory `pinnedAt` rule in
|
|
470
|
+
* `status-pin-retarget.ts` so disk and memory can't disagree about age.
|
|
471
|
+
*/
|
|
472
|
+
const claimAge = (messageId: number): number => {
|
|
473
|
+
const existing = loadStatusPins(path, fs).find((p) => p.pinKey === pinKey)
|
|
474
|
+
return existing?.messageId === messageId && existing.pinnedAt != null
|
|
475
|
+
? existing.pinnedAt
|
|
476
|
+
: now
|
|
477
|
+
}
|
|
434
478
|
|
|
435
479
|
// Hold the per-path lock across the WHOLE op so no other writer (a banner
|
|
436
480
|
// persist, or another key's reconcile) can rebuild/overwrite the file during
|
|
437
481
|
// the applyPin await window and drop this key's pending/confirmed row.
|
|
438
482
|
return withStoreLock(path, async () => {
|
|
439
483
|
if (op.kind === 'pin') {
|
|
484
|
+
const pinnedAt = claimAge(op.messageId)
|
|
440
485
|
// Persist INTENT first, marked pending — BEFORE the pin API call. If we
|
|
441
486
|
// crash after the pin lands but before the confirm rewrite, this pending
|
|
442
487
|
// record is what boot cleanup uses to unpin the orphan.
|
|
@@ -444,7 +489,7 @@ export function reconcileAndPersistStatusPin(args: {
|
|
|
444
489
|
path,
|
|
445
490
|
fs,
|
|
446
491
|
pinKey,
|
|
447
|
-
{ pinKey, chatId, messageId: op.messageId, pending: true },
|
|
492
|
+
{ pinKey, chatId, messageId: op.messageId, pending: true, pinnedAt },
|
|
448
493
|
log,
|
|
449
494
|
)
|
|
450
495
|
const next = await args.applyPin()
|
|
@@ -459,7 +504,7 @@ export function reconcileAndPersistStatusPin(args: {
|
|
|
459
504
|
path,
|
|
460
505
|
fs,
|
|
461
506
|
pinKey,
|
|
462
|
-
{ pinKey, chatId, messageId: next.messageId },
|
|
507
|
+
{ pinKey, chatId, messageId: next.messageId, pinnedAt: claimAge(next.messageId) },
|
|
463
508
|
log,
|
|
464
509
|
)
|
|
465
510
|
return next
|
|
@@ -488,11 +533,15 @@ export function reconcileAndPersistStatusPin(args: {
|
|
|
488
533
|
if (next == null) {
|
|
489
534
|
applyStatusPinRow(path, fs, pinKey, null, log)
|
|
490
535
|
} else {
|
|
536
|
+
// Steady-state noop-clear (the worker feed calls syncPin on every edit):
|
|
537
|
+
// the pin is still up, so the row is preserved AND so is its original
|
|
538
|
+
// claim age — re-stamping it here would hand a genuinely old pin a fresh
|
|
539
|
+
// TTL lease on every feed edit and make it immortal (#3810).
|
|
491
540
|
applyStatusPinRow(
|
|
492
541
|
path,
|
|
493
542
|
fs,
|
|
494
543
|
pinKey,
|
|
495
|
-
{ pinKey, chatId, messageId: next.messageId },
|
|
544
|
+
{ pinKey, chatId, messageId: next.messageId, pinnedAt: claimAge(next.messageId) },
|
|
496
545
|
log,
|
|
497
546
|
)
|
|
498
547
|
}
|