switchroom 0.19.26 → 0.19.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/bin/git-agent-attribution-hook.sh +144 -0
  2. package/dist/agent-scheduler/index.js +55 -0
  3. package/dist/auth-broker/index.js +116 -6
  4. package/dist/cli/notion-write-pretool.mjs +55 -0
  5. package/dist/cli/switchroom.js +2055 -1193
  6. package/dist/host-control/main.js +117 -7
  7. package/dist/vault/approvals/kernel-server.js +114 -6
  8. package/dist/vault/broker/server.js +114 -6
  9. package/package.json +6 -2
  10. package/profiles/_base/cron-session.sh.hbs +8 -0
  11. package/profiles/_base/start.sh.hbs +105 -0
  12. package/telegram-plugin/card-layout.ts +328 -0
  13. package/telegram-plugin/dist/bridge/bridge.js +93 -1
  14. package/telegram-plugin/dist/gateway/gateway.js +2009 -1166
  15. package/telegram-plugin/dist/server.js +96 -1
  16. package/telegram-plugin/edit-flood-fuse.ts +637 -56
  17. package/telegram-plugin/flood-429-ledger.ts +526 -0
  18. package/telegram-plugin/flood-circuit-breaker.ts +18 -0
  19. package/telegram-plugin/gateway/flood-reply-queue.ts +168 -0
  20. package/telegram-plugin/gateway/gateway.ts +58 -68
  21. package/telegram-plugin/gateway/narrative-lane.ts +14 -0
  22. package/telegram-plugin/gateway/outbound-send-path.ts +36 -0
  23. package/telegram-plugin/gateway/outbox-sweep.ts +183 -6
  24. package/telegram-plugin/gateway/pinned-message-handler.ts +12 -16
  25. package/telegram-plugin/gateway/status-pin-retarget.ts +72 -36
  26. package/telegram-plugin/gateway/status-pin-store.ts +58 -9
  27. package/telegram-plugin/gateway/worker-pin-reaper.ts +56 -7
  28. package/telegram-plugin/llm-error-present.ts +61 -2
  29. package/telegram-plugin/model-unavailable.ts +8 -0
  30. package/telegram-plugin/operator-events.ts +72 -5
  31. package/telegram-plugin/outbound-class.ts +81 -0
  32. package/telegram-plugin/provider-credit.ts +237 -0
  33. package/telegram-plugin/scripts/bun-test-ci.sh +36 -6
  34. package/telegram-plugin/send-gate.ts +24 -2
  35. package/telegram-plugin/status-no-truncate.ts +10 -48
  36. package/telegram-plugin/status-pin-driver.ts +33 -45
  37. package/telegram-plugin/status-pin.ts +18 -1
  38. package/telegram-plugin/tests/card-golden.test.ts +69 -0
  39. package/telegram-plugin/tests/card-lifecycle-render.test.ts +362 -0
  40. package/telegram-plugin/tests/card-type-distinguishability.test.ts +187 -164
  41. package/telegram-plugin/tests/card-variants.golden.txt +211 -0
  42. package/telegram-plugin/tests/card-variants.ts +366 -0
  43. package/telegram-plugin/tests/edit-flood-fuse-ban-awareness.test.ts +316 -0
  44. package/telegram-plugin/tests/edit-flood-fuse-default-deny.test.ts +319 -0
  45. package/telegram-plugin/tests/edit-flood-fuse.test.ts +11 -2
  46. package/telegram-plugin/tests/feed-edit-rate-ceiling.test.ts +462 -0
  47. package/telegram-plugin/tests/fixtures/real-429-stream.ts +220 -0
  48. package/telegram-plugin/tests/flood-429-ledger.test.ts +278 -0
  49. package/telegram-plugin/tests/flood-429-recorder-wiring.test.ts +128 -0
  50. package/telegram-plugin/tests/flood-reply-queue.test.ts +418 -0
  51. package/telegram-plugin/tests/outbox-sweep-flood-breaker.test.ts +221 -0
  52. package/telegram-plugin/tests/pinned-card-collapse.test.ts +19 -24
  53. package/telegram-plugin/tests/pinned-message-handler.test.ts +15 -15
  54. package/telegram-plugin/tests/provider-credit-402.test.ts +243 -0
  55. package/telegram-plugin/tests/status-pin-api.test.ts +11 -11
  56. package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +36 -37
  57. package/telegram-plugin/tests/status-pin-lifecycle.test.ts +602 -0
  58. package/telegram-plugin/tests/status-pin-retarget.test.ts +90 -62
  59. package/telegram-plugin/tests/status-pin-service-message-suppression.test.ts +7 -3
  60. package/telegram-plugin/tests/status-pin-store.test.ts +109 -60
  61. package/telegram-plugin/tests/status-pin.test.ts +56 -5
  62. package/telegram-plugin/tests/test-runner-coverage.test.ts +133 -0
  63. package/telegram-plugin/tests/worker-activity-feed.test.ts +12 -10
  64. package/telegram-plugin/tests/worker-feed-coalesce.test.ts +23 -29
  65. package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +56 -59
  66. package/telegram-plugin/tests/worker-feed-terminal-edit-class.test.ts +335 -0
  67. package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +1 -1
  68. package/telegram-plugin/tool-activity-summary.ts +239 -365
  69. package/telegram-plugin/uat/assertions.ts +22 -11
  70. package/telegram-plugin/uat/feed-matcher.test.ts +24 -17
  71. package/telegram-plugin/worker-activity-feed.ts +105 -47
  72. package/vendor/hindsight-memory/CLAUDE.md +45 -0
  73. package/vendor/hindsight-memory/scripts/lib/config.py +33 -0
  74. package/vendor/hindsight-memory/scripts/recall.py +176 -7
  75. package/vendor/hindsight-memory/scripts/tests/test_config_recall_passthrough_env.py +170 -0
  76. package/vendor/hindsight-memory/scripts/tests/test_recall_min_score.py +464 -0
  77. package/vendor/hindsight-memory/settings.json +1 -1
@@ -45,6 +45,11 @@ import {
45
45
  import { isShownBlock } from '../shown-ledger.js'
46
46
  import { resolveSubagentOriginTurnKey } from '../registry/subagents-schema.js'
47
47
  import { createRetryApiCall, retryWithThreadFallback } from '../retry-api-call.js'
48
+ import {
49
+ floodStatePath,
50
+ makeFloodWaitProbe,
51
+ makeFloodWaitRecorder,
52
+ } from '../flood-circuit-breaker.js'
48
53
 
49
54
  export interface OutboxSweepDeps {
50
55
  /** Deliver `text` to the chat. Resolves to the primary message id (best-effort). */
@@ -67,12 +72,39 @@ export interface OutboxSweepDeps {
67
72
  now?: () => number
68
73
  log?: (line: string) => void
69
74
  quietMs?: number
75
+ /**
76
+ * Remaining ms of a KNOWN-OPEN Telegram flood window, or 0 when none — the
77
+ * SAME persisted breaker state (`flood-wait.json`) `robustApiCall` consults
78
+ * (`makeFloodWaitProbe`). When it reports an open window the sweep does not
79
+ * scan, claim, or send at all.
80
+ *
81
+ * Why the sweep needs its own check on top of the retry policy's: the retry
82
+ * policy only short-circuits windows LONGER than its in-process sleep ceiling
83
+ * (120s), and even then it does so per-CALL — it still costs a claim/release
84
+ * cycle and a log line every tick. The outbox is a safety net with no latency
85
+ * SLA; deferring the whole sweep while ANY window is open is strictly correct
86
+ * and is the only thing that makes "does not hit the wire" true.
87
+ *
88
+ * FAILS OPEN (a throwing probe ⇒ 0) for the same reason `makeFloodWaitProbe`
89
+ * does: a broken marker file must never permanently strand the outbox.
90
+ */
91
+ floodWaitRemainingMs?: () => number
70
92
  }
71
93
 
72
94
  export interface OutboxSweepSummary {
73
95
  scanned: number
74
96
  delivered: number
75
97
  skipped: number
98
+ /**
99
+ * Records whose send THREW this sweep (claim released, retried next tick).
100
+ * The tick-level backoff keys off this: a 5s fixed retry against a failing
101
+ * Telegram is how the 2026-07-27 sweep hit an open 4.4h ban 228 times.
102
+ */
103
+ sendFailures: number
104
+ /** True when the whole sweep was skipped because a flood window is open. */
105
+ floodDeferred?: boolean
106
+ /** Remaining ms of the window that caused `floodDeferred`. */
107
+ floodRemainingMs?: number
76
108
  }
77
109
 
78
110
  /**
@@ -81,7 +113,20 @@ export interface OutboxSweepSummary {
81
113
  export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSummary> {
82
114
  const now = deps.now?.() ?? Date.now()
83
115
  const log = deps.log ?? (() => {})
84
- const summary: OutboxSweepSummary = { scanned: 0, delivered: 0, skipped: 0 }
116
+ const summary: OutboxSweepSummary = { scanned: 0, delivered: 0, skipped: 0, sendFailures: 0 }
117
+
118
+ // #3853 — do not sweep INTO a known-open flood window. Every record is
119
+ // preserved on disk and re-scanned on the next tick after the window closes,
120
+ // so this loses no delivery; issuing the send would only feed the ban.
121
+ const remainingMs = probeFloodWindow(deps.floodWaitRemainingMs)
122
+ if (remainingMs > 0) {
123
+ summary.floodDeferred = true
124
+ summary.floodRemainingMs = remainingMs
125
+ // NOT logged here: a 4.4h ban is ~3181 ticks, and one line per tick just
126
+ // moves the flood from the wire to the disk. `startOutboxSweep` logs the
127
+ // deferral at most once per DEFER_LOG_INTERVAL_MS.
128
+ return summary
129
+ }
85
130
 
86
131
  // Re-queue crashed claims first (claim-then-crash before send).
87
132
  reclaimStaleSending(deps.stateDir, now)
@@ -90,9 +135,27 @@ export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSum
90
135
  if (pending.length === 0) return summary
91
136
  const deliveredNonces = readDeliveredNonces(deps.stateDir)
92
137
 
93
- for (const fileName of pending) {
94
- const record = readOutboxRecord(fileName, deps.stateDir)
95
- if (record == null) continue
138
+ // #3861 deliver in CAPTURE order. `listPendingRecords` returns readdir
139
+ // order, which is filesystem-dependent (hash order on most Linux filesystems,
140
+ // i.e. effectively the nonce's hash). That was harmless while the outbox held
141
+ // at most one stray handback per turn; it is not harmless now that a
142
+ // multi-hour flood window can park a whole conversation's worth of queued
143
+ // replies, which would then land shuffled. Sort by `createdAt`, tie-broken by
144
+ // nonce so the order is total and deterministic.
145
+ const records = pending
146
+ .map((fileName) => readOutboxRecord(fileName, deps.stateDir))
147
+ .filter((r): r is OutboxRecord => r != null)
148
+ .sort((a, b) =>
149
+ a.createdAt !== b.createdAt
150
+ ? a.createdAt - b.createdAt
151
+ : a.turnNonce < b.turnNonce
152
+ ? -1
153
+ : a.turnNonce > b.turnNonce
154
+ ? 1
155
+ : 0,
156
+ )
157
+
158
+ for (const record of records) {
96
159
  summary.scanned++
97
160
 
98
161
  // Resolve destination (anchor → registry chain → per-session origin). Fails
@@ -209,12 +272,24 @@ export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSum
209
272
  // preserved on disk; no loss.
210
273
  releaseClaim(record.turnNonce, deps.stateDir)
211
274
  summary.skipped++
275
+ summary.sendFailures++
212
276
  log(`outbox-sweep: send failed nonce=${record.turnNonce}: ${(err as Error).message} — will retry\n`)
213
277
  }
214
278
  }
215
279
  return summary
216
280
  }
217
281
 
282
+ /** Read the flood probe, failing OPEN on any throw. */
283
+ function probeFloodWindow(probe: (() => number) | undefined): number {
284
+ if (probe == null) return 0
285
+ try {
286
+ const ms = probe()
287
+ return Number.isFinite(ms) && ms > 0 ? ms : 0
288
+ } catch {
289
+ return 0
290
+ }
291
+ }
292
+
218
293
  /**
219
294
  * Parse a registry `turn_key` (`chatId:threadId`, `_` → null thread) into a
220
295
  * routable chat. Exported for the gateway wiring + tests.
@@ -230,6 +305,59 @@ export function chatFromTurnKey(turnKey: string): { chatId: string; threadId: nu
230
305
  /** How often the sweep ticks (aligned with the delivery-confirm sweep cadence). */
231
306
  export const OUTBOX_SWEEP_INTERVAL_MS = 5_000
232
307
 
308
+ /** Ceiling on the exponential send-failure backoff. */
309
+ export const OUTBOX_SWEEP_BACKOFF_MAX_MS = 5 * 60_000
310
+
311
+ /**
312
+ * Minimum gap between two "flood window still open" log lines. A 4.4h ban is
313
+ * ~3181 ticks; logging each one just relocates the flood from the wire to the
314
+ * disk, so the deferral is reported once a minute instead.
315
+ */
316
+ export const OUTBOX_SWEEP_DEFER_LOG_INTERVAL_MS = 60_000
317
+
318
+ /**
319
+ * Exponential backoff over the fixed 5s sweep tick (#3853).
320
+ *
321
+ * The tick interval alone is not a retry policy. On 2026-07-27 the sweep held
322
+ * ONE undeliverable record and re-issued its `sendMessage` every 5s for the
323
+ * whole 4.4h ban — 228 requests into a window the breaker already knew was
324
+ * open, each one answered with the same counting-down `retry_after`. The flood
325
+ * probe (above) closes the case where the breaker HAS a window recorded; this
326
+ * closes the case where it does not — a persistent failure of any other kind
327
+ * must not be re-driven at 12 requests/minute forever.
328
+ *
329
+ * Pure and clock-injected so the schedule is asserted in tests rather than
330
+ * inferred from timer behaviour.
331
+ */
332
+ export function createSweepBackoff(cfg: { baseMs?: number; maxMs?: number } = {}): {
333
+ ready: (now: number) => boolean
334
+ noteFailure: (now: number) => number
335
+ noteSuccess: () => void
336
+ failures: () => number
337
+ } {
338
+ const baseMs = cfg.baseMs ?? OUTBOX_SWEEP_INTERVAL_MS
339
+ const maxMs = cfg.maxMs ?? OUTBOX_SWEEP_BACKOFF_MAX_MS
340
+ let failures = 0
341
+ let nextAllowedAt = 0
342
+ return {
343
+ ready: (now) => now >= nextAllowedAt,
344
+ noteFailure: (now) => {
345
+ failures++
346
+ // 5s, 10s, 20s, … capped. `failures - 1` so the FIRST failure still
347
+ // retries at the normal tick cadence — a one-off transient send error
348
+ // should not delay a real answer.
349
+ const delay = Math.min(maxMs, baseMs * Math.pow(2, failures - 1))
350
+ nextAllowedAt = now + delay
351
+ return delay
352
+ },
353
+ noteSuccess: () => {
354
+ failures = 0
355
+ nextAllowedAt = 0
356
+ },
357
+ failures: () => failures,
358
+ }
359
+ }
360
+
233
361
  /**
234
362
  * Own the heartbeat-tick outbox sweep entirely, keeping the timer / chunking /
235
363
  * turn-key parse / dedup / registry-chain wiring OUT of `gateway.ts` (the line
@@ -336,21 +464,43 @@ export function startOutboxSweep(deps: {
336
464
  text: string,
337
465
  ) => OutboxDeliveryMarkup | undefined
338
466
  log?: (line: string) => void
467
+ /** Test seam — override the flood probe (default: the persisted breaker). */
468
+ floodWaitRemainingMs?: () => number
469
+ /** Test seam — override the tick backoff. */
470
+ backoff?: ReturnType<typeof createSweepBackoff>
339
471
  }): ReturnType<typeof setInterval> | undefined {
340
472
  if (!deps.isGatewayMain || process.env.SWITCHROOM_TG_OUTBOX_DELIVERY === '0') return undefined
341
- const retry = createRetryApiCall({ log: deps.log })
473
+ // #3853 the sweep is a full outbound machine and must consult the SAME
474
+ // persisted flood breaker every other outbound path does. Before this it was
475
+ // the one `createRetryApiCall` wiring with neither hook: it could not see an
476
+ // open ban (no `floodWaitRemainingMs`) and could not record one it caused (no
477
+ // `onFloodWait`). `scripts/check-retry-flood-hooks.mjs` now fails lint if any
478
+ // wiring omits either.
479
+ const floodProbe =
480
+ deps.floodWaitRemainingMs ?? makeFloodWaitProbe(floodStatePath(deps.stateDir))
481
+ const recordFloodWait = makeFloodWaitRecorder(floodStatePath(deps.stateDir))
482
+ const retry = createRetryApiCall({
483
+ log: deps.log,
484
+ floodWaitRemainingMs: floodProbe,
485
+ onFloodWait: (retryAfterSec) => recordFloodWait(retryAfterSec),
486
+ })
342
487
  const send = createOutboxSend({
343
488
  getBot: deps.getBot,
344
489
  retry,
345
490
  ...(deps.resolveReplyMarkup != null ? { resolveReplyMarkup: deps.resolveReplyMarkup } : {}),
346
491
  })
492
+ const backoff = deps.backoff ?? createSweepBackoff()
493
+ let lastDeferLogAt = 0
347
494
  const tick = () => {
348
495
  const bot = deps.getBot()
349
496
  if (bot == null) return
497
+ const now = Date.now()
498
+ if (!backoff.ready(now)) return
350
499
  void sweepOutbox({
351
500
  stateDir: deps.stateDir,
352
501
  log: deps.log,
353
502
  send,
503
+ floodWaitRemainingMs: floodProbe,
354
504
  textAlreadyDelivered: (chatId, threadId, text) => deps.dedupCheck(chatId, threadId ?? undefined, text),
355
505
  registryChainLookup: (taskId) => {
356
506
  const db = deps.getTurnsDb()
@@ -358,7 +508,34 @@ export function startOutboxSweep(deps: {
358
508
  const turnKey = resolveSubagentOriginTurnKey(db, taskId)
359
509
  return turnKey == null ? null : chatFromTurnKey(turnKey)
360
510
  },
361
- }).catch((err) => deps.log?.(`outbox-sweep: tick failed: ${(err as Error).message}\n`))
511
+ })
512
+ .then((summary) => {
513
+ // A flood-deferred sweep is neither success nor failure: it never
514
+ // reached the wire, so it must not clear a backoff earned by real
515
+ // failures, and it must not deepen one either.
516
+ if (summary.floodDeferred === true) {
517
+ const at = Date.now()
518
+ if (at - lastDeferLogAt >= OUTBOX_SWEEP_DEFER_LOG_INTERVAL_MS) {
519
+ lastDeferLogAt = at
520
+ deps.log?.(
521
+ `outbox-sweep: deferred — Telegram flood window still open for ` +
522
+ `${Math.ceil((summary.floodRemainingMs ?? 0) / 1000)}s; not issuing any ` +
523
+ `send (would feed the ban)\n`,
524
+ )
525
+ }
526
+ return
527
+ }
528
+ if (summary.sendFailures > 0) {
529
+ const delay = backoff.noteFailure(Date.now())
530
+ deps.log?.(
531
+ `outbox-sweep: ${summary.sendFailures} send failure(s) — backing off ` +
532
+ `${Math.round(delay / 1000)}s before the next sweep (attempt ${backoff.failures()})\n`,
533
+ )
534
+ } else {
535
+ backoff.noteSuccess()
536
+ }
537
+ })
538
+ .catch((err) => deps.log?.(`outbox-sweep: tick failed: ${(err as Error).message}\n`))
362
539
  }
363
540
  const timer = setInterval(tick, OUTBOX_SWEEP_INTERVAL_MS)
364
541
  timer.unref?.()
@@ -5,7 +5,7 @@
5
5
  * silent status-pin fires. The pin call passes `disable_notification: true`,
6
6
  * which kills the PUSH notification but NOT the in-chat service message — so we
7
7
  * delete that service message as it lands, but ONLY for pins we own (tracked in
8
- * `statusPinState`). Manual/operator pins are never silent and are never
8
+ * `statusPinClaims`). Manual/operator pins are never silent and are never
9
9
  * touched. Only silent status pins reach here as an OUR-pin match, so the
10
10
  * ownership check is the guard.
11
11
  *
@@ -21,13 +21,12 @@
21
21
 
22
22
  import type { Context, Filter } from 'grammy'
23
23
  import { pinnedMessageIsOurs, type TrackedStatusPin } from './status-pin-store.js'
24
- import type { PinState } from '../status-pin.js'
24
+ import type { StatusPinClaim } from './status-pin-retarget.js'
25
25
 
26
26
  export interface PinnedMessageHandlerDeps {
27
- /** Live pinKey→PinState registry of pins we own. */
28
- statusPinState: ReadonlyMap<string, PinState>
29
- /** Companion pinKey→chatId registry (chat-scoped ownership). */
30
- statusPinChatIds: ReadonlyMap<string, string>
27
+ /** Live pinKey→claim registry of pins we own. ONE record per key (#3809):
28
+ * the chat id is a field of the claim, not a parallel map that can diverge. */
29
+ statusPinClaims: ReadonlyMap<string, StatusPinClaim>
31
30
  /**
32
31
  * Delete the pin service message. Bound in gateway.ts through
33
32
  * `robustApiCall(() => lockedBot.api.deleteMessage(...))` so the raw Bot API
@@ -50,16 +49,13 @@ export async function handlePinnedMessage(
50
49
  // Chat-scoped ownership (see pinnedMessageIsOurs): the match requires BOTH
51
50
  // the messageId AND that the tracked entry lives in THIS chat, so a pin id
52
51
  // colliding across chats can't delete a foreign (e.g. operator-manual) pin
53
- // notice. statusPinChatIds is the companion pinKey→chatId map written on
54
- // every desired-pinned reconcile.
55
- const trackedPins = (): TrackedStatusPin[] => {
56
- const out: TrackedStatusPin[] = []
57
- for (const [pinKey, state] of deps.statusPinState) {
58
- const c = deps.statusPinChatIds.get(pinKey)
59
- if (c != null) out.push({ chatId: c, messageId: state.messageId })
60
- }
61
- return out
62
- }
52
+ // notice. Each claim carries its own chat id (#3809), so the match can never
53
+ // silently degrade to messageId-only because a companion map went missing.
54
+ const trackedPins = (): TrackedStatusPin[] =>
55
+ [...deps.statusPinClaims.values()].map((c) => ({
56
+ chatId: c.chatId,
57
+ messageId: c.messageId,
58
+ }))
63
59
  const isOurs = () => pinnedMessageIsOurs(trackedPins(), chatId, pinnedId)
64
60
 
65
61
  if (!isOurs()) {
@@ -34,7 +34,7 @@
34
34
  * leg 1 unpin the stale claim, with the row still naming the OLD id;
35
35
  * leg 2 pin the new message from a null claim, with its own pending row.
36
36
  *
37
- * Failure semantics are inherited from `reconcilePin`, not re-invented here: a
37
+ * Failure semantics are inherited from `executePinLeg`, not re-invented here: a
38
38
  * leg-1 result that is NON-NULL means the unpin was never confirmed (#3664
39
39
  * Defect B — the old message is provably still pinned and the claim was
40
40
  * deliberately retained). Pinning the new message then would leave two pins in
@@ -42,24 +42,45 @@
42
42
  * claim is left for the next reconcile / the mid-session reaper / the boot
43
43
  * sweep to retry.
44
44
  *
45
+ * THE SINGLE PATH (#3831)
46
+ * -----------------------
47
+ * This module is the ONLY decider. `status-pin-driver.ts` used to call
48
+ * `decidePinAction` itself and carry its OWN copy of the expansion above — a
49
+ * copy that recursed without the per-leg persistence, so which of the two
50
+ * implementations ran decided whether the durable row survived. That copy is
51
+ * deleted, and the driver's parameter type (`PinLegAction`) can no longer even
52
+ * express a repin, so a second one cannot grow back.
53
+ *
45
54
  * Dependency-free apart from the pure decision, so it is provable in isolation
46
55
  * (`telegram-plugin/tests/status-pin-retarget.test.ts`); the gateway owns the
47
- * wiring of `runLeg` / `commit`.
56
+ * wiring of `runPin` / `commit`.
48
57
  */
49
58
 
50
- import type { DesiredPin, PinState } from '../status-pin.js'
59
+ import type { DesiredPin, PinLegAction, PinState } from '../status-pin.js'
51
60
  import { decidePinAction } from '../status-pin.js'
52
61
  import type { StatusPinPersistOp, StatusPinStoreFsSeam } from './status-pin-store.js'
53
62
  import { reconcileAndPersistStatusPin } from './status-pin-store.js'
54
63
 
55
- /** The three parallel in-memory maps the gateway keys by pinKey. */
56
- export interface StatusPinRegistries {
57
- /** The claim: which message we believe is pinned for this key. */
58
- state: Map<string, PinState>
59
- /** Which chat that claim lives in (the `wk:` reaper needs it to unpin). */
60
- chatIds: Map<string, string>
61
- /** When the claim was FIRST taken the reaper's TTL age. */
62
- pinnedAt: Map<string, number>
64
+ /**
65
+ * ONE in-memory claim for a pin key — the message we believe is pinned, the
66
+ * chat it lives in, and when the claim was FIRST taken (the reaper's TTL age).
67
+ *
68
+ * #3809 these were three parallel `Map`s (`statusPinState` / `statusPinChatIds`
69
+ * / `statusPinPinnedAt`) written together but able to diverge. A `wk:` claim
70
+ * whose chatId entry went missing was skipped by the mid-session reaper on
71
+ * EVERY pass (it cannot unpin without a chat) and excluded from the durable
72
+ * store-orphan net (which skips keys that DO have an in-memory claim) — so it
73
+ * was permanently unreapable until the next boot, silently. Collapsing the
74
+ * three maps into one record makes the invariant STRUCTURAL: a claim without a
75
+ * chat id cannot be represented, so that gap cannot be reintroduced.
76
+ */
77
+ export interface StatusPinClaim {
78
+ /** Telegram message_id we believe is pinned for this key. */
79
+ messageId: number
80
+ /** Chat the pin lives in. Never empty — the reconcile refuses an empty chat. */
81
+ chatId: string
82
+ /** Wall-clock ms the claim was FIRST taken (survives a same-id re-pin). */
83
+ pinnedAt: number
63
84
  }
64
85
 
65
86
  export interface StatusPinReconcileArgs {
@@ -72,48 +93,56 @@ export interface StatusPinReconcileArgs {
72
93
  /** Durable-store binding, or null when persistence is off (STATIC). */
73
94
  persist: { path: string; fs: StatusPinStoreFsSeam } | null
74
95
  /**
75
- * Execute the real Telegram pin/unpin for one leg (the gateway binds
76
- * `reconcilePin`). Must never throw — API errors are absorbed inside.
96
+ * Execute ONE already-decided leg against Telegram (the gateway binds
97
+ * `executePinLeg`). Must never throw — API errors are absorbed inside.
77
98
  */
78
- runPin: (from: PinState | null, want: DesiredPin) => Promise<PinState | null>
79
- registries: StatusPinRegistries
99
+ runPin: (action: PinLegAction, from: PinState | null) => Promise<PinState | null>
100
+ /** The single claim registry, keyed by pinKey. */
101
+ claims: Map<string, StatusPinClaim>
102
+ /** Injectable clock for the `pinnedAt` stamp (tests). */
103
+ now?: () => number
80
104
  }
81
105
 
82
106
  /**
83
107
  * Drive `prev → desired` for one pin key, expanding a RETARGET into its two
84
108
  * legs. Every other action (pin / unpin / noop) is a single leg, unchanged.
85
109
  *
86
- * Each leg is committed to the registries as it lands, so the in-memory claim
87
- * never lags the durable row — including the intermediate cleared state between
88
- * a retarget's two legs, which is what lets a concurrent reader (the `wk:`
89
- * reaper) see an honest snapshot rather than a claim on an unpinned id.
110
+ * Each leg is committed to the claim registry as it lands, so the in-memory
111
+ * claim never lags the durable row — including the intermediate cleared state
112
+ * between a retarget's two legs, which is what lets a concurrent reader (the
113
+ * `wk:` reaper) see an honest snapshot rather than a claim on an unpinned id.
90
114
  *
91
115
  * Callers must hold the per-key reconcile lock (`withPinReconcileLock`); `prev`
92
116
  * is read once by the caller and both legs run inside that critical section.
93
117
  */
94
118
  export async function runStatusPinReconcile(args: StatusPinReconcileArgs): Promise<void> {
95
- const { pinKey, chatId, prev, desired, persist, runPin, registries } = args
119
+ const { pinKey, chatId, prev, desired, persist, runPin, claims } = args
120
+ const now = args.now ?? Date.now
96
121
 
97
- // Publish a leg's outcome. `pinnedAt` is set only when absent so it keeps the
98
- // FIRST-taken timestamp; clearing the claim drops it, so a re-pin ages fresh.
122
+ // Publish a leg's outcome into the single claim registry. `pinnedAt` is
123
+ // carried forward when a claim already exists so it keeps the FIRST-taken
124
+ // timestamp; clearing the claim drops it, so a re-pin ages fresh.
99
125
  const commit = (next: PinState | null): void => {
100
126
  if (next == null) {
101
- registries.state.delete(pinKey)
102
- registries.chatIds.delete(pinKey)
103
- registries.pinnedAt.delete(pinKey)
127
+ claims.delete(pinKey)
104
128
  return
105
129
  }
106
- registries.state.set(pinKey, next)
107
- registries.chatIds.set(pinKey, chatId)
108
- if (!registries.pinnedAt.has(pinKey)) registries.pinnedAt.set(pinKey, Date.now())
130
+ const pinnedAt = claims.get(pinKey)?.pinnedAt ?? now()
131
+ claims.set(pinKey, { messageId: next.messageId, chatId, pinnedAt })
109
132
  }
110
133
 
111
134
  // One leg, with the persist ordering its action requires. Only a fresh `pin`
112
135
  // opens the crash window that persist-BEFORE-pin closes; unpin / noop / re-pin
113
136
  // of the same id clear or leave the row and are safe to persist after.
114
- const runLeg = (from: PinState | null, want: DesiredPin): Promise<PinState | null> => {
115
- if (persist == null) return runPin(from, want)
116
- const legAction = decidePinAction(from, want)
137
+ //
138
+ // The leg action is decided ONCE, below, and threaded through to both the
139
+ // persist-op mapping and the driver — the driver no longer re-derives it
140
+ // (#3831).
141
+ const runLeg = (
142
+ legAction: PinLegAction,
143
+ from: PinState | null,
144
+ ): Promise<PinState | null> => {
145
+ if (persist == null) return runPin(legAction, from)
117
146
  const op: StatusPinPersistOp =
118
147
  legAction.kind === 'pin'
119
148
  ? { kind: 'pin', messageId: legAction.messageId }
@@ -124,21 +153,28 @@ export async function runStatusPinReconcile(args: StatusPinReconcileArgs): Promi
124
153
  pinKey,
125
154
  chatId,
126
155
  op,
127
- applyPin: () => runPin(from, want),
156
+ applyPin: () => runPin(legAction, from),
157
+ now: now(),
128
158
  })
129
159
  }
130
160
 
131
161
  const action = decidePinAction(prev, desired)
132
162
 
133
- if (action.kind === 'repin' && desired.pinned) {
134
- const afterUnpin = await runLeg(prev, { pinned: false })
163
+ // THE retarget expansion. This is the only copy in the codebase; the driver's
164
+ // parameter type (`PinLegAction`) cannot express a repin, so a second one
165
+ // cannot be written there again (#3831).
166
+ if (action.kind === 'repin') {
167
+ const afterUnpin = await runLeg(
168
+ { kind: 'unpin', messageId: action.unpinMessageId },
169
+ prev,
170
+ )
135
171
  commit(afterUnpin)
136
172
  // Never-confirmed unpin (#3664 Defect B): keep the retained claim, skip the
137
173
  // pin leg. See the docblock.
138
174
  if (afterUnpin != null) return
139
- commit(await runLeg(null, desired))
175
+ commit(await runLeg({ kind: 'pin', messageId: action.pinMessageId }, null))
140
176
  return
141
177
  }
142
178
 
143
- commit(await runLeg(prev, desired))
179
+ commit(await runLeg(action, prev))
144
180
  }
@@ -70,6 +70,27 @@ export interface PersistedStatusPin {
70
70
  * unpin fails (flood-wait exhausted / transient 5xx); the row is retained
71
71
  * for retry until BOOT_UNPIN_MAX_ATTEMPTS, then forfeited. Absent = 0. */
72
72
  attempts?: number
73
+ /**
74
+ * Wall-clock ms the pin was FIRST claimed for this key+message (#3810).
75
+ *
76
+ * Why the store needs it: the mid-session `wk:` reaper folds in rows that
77
+ * exist on disk but have NO in-memory claim (`storeOnlyWorkerPinCandidates`).
78
+ * The store used to persist no timestamp, so those candidates were stamped
79
+ * `pinnedAt = now` on every pass — deliberately, to avoid a spurious unpin of
80
+ * a pin whose real age was unknown, but with the side effect that the TTL
81
+ * gate could NEVER fire for a store orphan. Combined with a `wk:<agentId>`
82
+ * whose turnsDb row is gone (verdict `'unknown'`, never `'terminal'`), such a
83
+ * row was never mid-session reaped AT ALL — it waited for the next boot, the
84
+ * exact "stale pin glued to the top of the chat" failure the reaper exists to
85
+ * prevent.
86
+ *
87
+ * Recording the real claim time removes the guess: a store orphan now ages
88
+ * honestly and the ordinary TTL applies. Optional so a v1/v2 snapshot still
89
+ * loads; a row without it keeps the old conservative `now` stamp (terminal-
90
+ * only reaping), which self-clears within one restart since every write from
91
+ * this version onward carries the field.
92
+ */
93
+ pinnedAt?: number
73
94
  }
74
95
 
75
96
  /** How many boots may retry a failing boot-cleanup unpin before the row is
@@ -80,13 +101,18 @@ export interface PersistedStatusPin {
80
101
  export const BOOT_UNPIN_MAX_ATTEMPTS = 5
81
102
 
82
103
  /** Envelope version. v1 had no `pending` field; a v1 row loads as a confirmed
83
- * pin (pending undefined). v2 adds the optional `pending` flag. Both load
84
- * fail-open — an unknown/newer version yields []. */
104
+ * pin (pending undefined). v2 adds the optional `pending` flag. v3 adds the
105
+ * optional `pinnedAt` claim timestamp (#3810). All load fail-open — an
106
+ * unknown/newer version yields []. Every field added since v1 is optional, so
107
+ * the versions are mutually readable and a downgrade degrades rather than
108
+ * breaks. */
85
109
  interface SnapshotEnvelope {
86
- v: 1 | 2
110
+ v: 1 | 2 | 3
87
111
  pins: PersistedStatusPin[]
88
112
  }
89
113
 
114
+ const SNAPSHOT_VERSIONS = new Set([1, 2, 3])
115
+
90
116
  function isPinRow(x: unknown): x is PersistedStatusPin {
91
117
  if (x == null || typeof x !== 'object') return false
92
118
  const o = x as Record<string, unknown>
@@ -98,7 +124,8 @@ function isPinRow(x: unknown): x is PersistedStatusPin {
98
124
  typeof o.messageId === 'number' &&
99
125
  (o.pending === undefined || typeof o.pending === 'boolean') &&
100
126
  (o.expiresAt === undefined || typeof o.expiresAt === 'number') &&
101
- (o.attempts === undefined || typeof o.attempts === 'number')
127
+ (o.attempts === undefined || typeof o.attempts === 'number') &&
128
+ (o.pinnedAt === undefined || typeof o.pinnedAt === 'number')
102
129
  )
103
130
  }
104
131
 
@@ -127,7 +154,7 @@ export function loadStatusPins(
127
154
  }
128
155
  if (parsed == null || typeof parsed !== 'object') return []
129
156
  const env = parsed as Record<string, unknown>
130
- if ((env.v !== 1 && env.v !== 2) || !Array.isArray(env.pins)) return []
157
+ if (typeof env.v !== 'number' || !SNAPSHOT_VERSIONS.has(env.v) || !Array.isArray(env.pins)) return []
131
158
  return env.pins.filter(isPinRow)
132
159
  }
133
160
 
@@ -143,7 +170,7 @@ export function persistStatusPins(
143
170
  snapshot: readonly PersistedStatusPin[],
144
171
  log: (line: string) => void = (l) => process.stderr.write(l),
145
172
  ): void {
146
- const env: SnapshotEnvelope = { v: 2, pins: [...snapshot] }
173
+ const env: SnapshotEnvelope = { v: 3, pins: [...snapshot] }
147
174
  const tmp = path + '.tmp'
148
175
  try {
149
176
  fs.writeFileSync(tmp, JSON.stringify(env))
@@ -427,16 +454,34 @@ export function reconcileAndPersistStatusPin(args: {
427
454
  /** Execute the real pin/unpin; returns the confirmed message id (pin) or
428
455
  * null (cleared). Must never throw — API errors are swallowed inside. */
429
456
  applyPin: () => Promise<{ messageId: number } | null>
457
+ /** Clock for the `pinnedAt` stamp (#3810). Defaults to wall clock. */
458
+ now?: number
430
459
  log?: (line: string) => void
431
460
  }): Promise<{ messageId: number } | null> {
432
461
  const { path, fs, pinKey, chatId, op } = args
433
462
  const log = args.log ?? ((l: string) => process.stderr.write(l))
463
+ const now = args.now ?? Date.now()
464
+
465
+ /**
466
+ * The claim age to persist for `messageId`. Carried forward from the row
467
+ * already on disk when it names the SAME message (a steady-state re-write
468
+ * must not reset the age and hand a stale pin a fresh TTL lease); stamped
469
+ * `now` for a genuinely new pin. Mirrors the in-memory `pinnedAt` rule in
470
+ * `status-pin-retarget.ts` so disk and memory can't disagree about age.
471
+ */
472
+ const claimAge = (messageId: number): number => {
473
+ const existing = loadStatusPins(path, fs).find((p) => p.pinKey === pinKey)
474
+ return existing?.messageId === messageId && existing.pinnedAt != null
475
+ ? existing.pinnedAt
476
+ : now
477
+ }
434
478
 
435
479
  // Hold the per-path lock across the WHOLE op so no other writer (a banner
436
480
  // persist, or another key's reconcile) can rebuild/overwrite the file during
437
481
  // the applyPin await window and drop this key's pending/confirmed row.
438
482
  return withStoreLock(path, async () => {
439
483
  if (op.kind === 'pin') {
484
+ const pinnedAt = claimAge(op.messageId)
440
485
  // Persist INTENT first, marked pending — BEFORE the pin API call. If we
441
486
  // crash after the pin lands but before the confirm rewrite, this pending
442
487
  // record is what boot cleanup uses to unpin the orphan.
@@ -444,7 +489,7 @@ export function reconcileAndPersistStatusPin(args: {
444
489
  path,
445
490
  fs,
446
491
  pinKey,
447
- { pinKey, chatId, messageId: op.messageId, pending: true },
492
+ { pinKey, chatId, messageId: op.messageId, pending: true, pinnedAt },
448
493
  log,
449
494
  )
450
495
  const next = await args.applyPin()
@@ -459,7 +504,7 @@ export function reconcileAndPersistStatusPin(args: {
459
504
  path,
460
505
  fs,
461
506
  pinKey,
462
- { pinKey, chatId, messageId: next.messageId },
507
+ { pinKey, chatId, messageId: next.messageId, pinnedAt: claimAge(next.messageId) },
463
508
  log,
464
509
  )
465
510
  return next
@@ -488,11 +533,15 @@ export function reconcileAndPersistStatusPin(args: {
488
533
  if (next == null) {
489
534
  applyStatusPinRow(path, fs, pinKey, null, log)
490
535
  } else {
536
+ // Steady-state noop-clear (the worker feed calls syncPin on every edit):
537
+ // the pin is still up, so the row is preserved AND so is its original
538
+ // claim age — re-stamping it here would hand a genuinely old pin a fresh
539
+ // TTL lease on every feed edit and make it immortal (#3810).
491
540
  applyStatusPinRow(
492
541
  path,
493
542
  fs,
494
543
  pinKey,
495
- { pinKey, chatId, messageId: next.messageId },
544
+ { pinKey, chatId, messageId: next.messageId, pinnedAt: claimAge(next.messageId) },
496
545
  log,
497
546
  )
498
547
  }