switchroom 0.18.27 → 0.18.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2120,7 +2120,7 @@ var init_esm = __esm(() => {
2120
2120
  });
2121
2121
 
2122
2122
  // src/build-info.ts
2123
- var VERSION = "0.18.27", COMMIT_SHA = "93871829";
2123
+ var VERSION = "0.18.28", COMMIT_SHA = "ef0be9a2";
2124
2124
 
2125
2125
  // src/cli/resolve-version.ts
2126
2126
  import { existsSync, readFileSync } from "node:fs";
@@ -26605,7 +26605,7 @@ import { existsSync as existsSync9, readFileSync as readFileSync7 } from "node:f
26605
26605
  import { dirname as dirname4, join as join7 } from "node:path";
26606
26606
 
26607
26607
  // src/build-info.ts
26608
- var VERSION = "0.18.27";
26608
+ var VERSION = "0.18.28";
26609
26609
 
26610
26610
  // src/cli/resolve-version.ts
26611
26611
  function readPackageVersion() {
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "switchroom",
3
3
  "//version": "NOT the release version — source of truth is the git tag, resolved by scripts/build.mjs:resolveVersion() (see CLAUDE.md > Standard release process). This field is stale by design and only the Layer-4 dev/non-tag fallback for build.mjs + src/cli/resolve-version.ts; do NOT bump it expecting a release to pick it up. npm-pack tarball naming needs a real version — do that as an UNCOMMITTED pack-time bump (see release step 6), never a committed one.",
4
- "version": "0.18.27",
4
+ "version": "0.18.28",
5
5
  "description": "Run Claude Code 24/7 on your Claude Pro/Max subscription over Telegram. Open-source alternative to OpenClaw and NanoClaw — no API keys.",
6
6
  "type": "module",
7
7
  "bin": {
@@ -41118,6 +41118,12 @@ function createWorkerActivityFeed(opts) {
41118
41118
  agentIndex.delete(agentId);
41119
41119
  maybeDeleteGroup(g);
41120
41120
  }
41121
+ function evictRowFromGroup(g, agentId) {
41122
+ g.workers.delete(agentId);
41123
+ if (agentIndex.get(agentId) === g.feedKey)
41124
+ agentIndex.delete(agentId);
41125
+ maybeDeleteGroup(g);
41126
+ }
41121
41127
  function maybeDeleteGroup(g) {
41122
41128
  if (g.workers.size === 0 && g.pendingFinalize.size === 0)
41123
41129
  groups.delete(g.feedKey);
@@ -41309,7 +41315,7 @@ function createWorkerActivityFeed(opts) {
41309
41315
  if (row.finished)
41310
41316
  staleFinished.push({ g, agentId: row.agentId, reason });
41311
41317
  else
41312
- staleAgentIds.push({ agentId: row.agentId, reason });
41318
+ staleAgentIds.push({ g, agentId: row.agentId, reason });
41313
41319
  }
41314
41320
  }
41315
41321
  for (const { g, agentId, reason } of staleFinished) {
@@ -41323,15 +41329,22 @@ function createWorkerActivityFeed(opts) {
41323
41329
  removeWorker(g, agentId);
41324
41330
  syncPin(g);
41325
41331
  }
41326
- for (const { agentId, reason } of staleAgentIds) {
41327
- const row = groupOfAgent(agentId)?.workers.get(agentId);
41332
+ for (const { g, agentId, reason } of staleAgentIds) {
41333
+ const row = g.workers.get(agentId);
41328
41334
  if (reason === "absolute") {
41329
41335
  const age = Math.floor((now - (row?.createdAtMs ?? now)) / 1000);
41330
41336
  log(`worker-feed: ABSOLUTE cap reap agent=${agentId} \u2014 row age ${age}s (>= ${Math.floor(absoluteRowLifetimeCapMs / 1000)}s); force-terminating immortal row (survives lastUpdateAt reset)`);
41331
41337
  } else {
41332
41338
  log(`worker-feed: TTL reap agent=${agentId} \u2014 no update in ${Math.floor((now - (row?.lastUpdateAt ?? now)) / 1000)}s (>= ${Math.floor(staleWorkerTtlMs / 1000)}s); force-terminating leaked row`);
41333
41339
  }
41334
- terminateWorker(agentId);
41340
+ if (groupOfAgent(agentId) === g) {
41341
+ terminateWorker(agentId);
41342
+ } else {
41343
+ log(`worker-feed: force-evicting leaked row agent=${agentId} feed=${g.feedKey} \u2014 agentIndex desynced (points elsewhere); removing directly`);
41344
+ markFinalized(agentId);
41345
+ evictRowFromGroup(g, agentId);
41346
+ syncPin(g);
41347
+ }
41335
41348
  }
41336
41349
  for (const g of [...groups.values()]) {
41337
41350
  if (g.pendingFinalize.size > 0 && now >= g.cooldownUntil) {
@@ -41378,6 +41391,14 @@ function createWorkerActivityFeed(opts) {
41378
41391
  }
41379
41392
  }
41380
41393
  heartbeatTimer = setIntervalFn(heartbeatTick, heartbeatTickMs);
41394
+ opts.exposeTestControls?.({
41395
+ repointAgentIndex: (agentId, feedKey) => {
41396
+ if (feedKey == null)
41397
+ agentIndex.delete(agentId);
41398
+ else
41399
+ agentIndex.set(agentId, feedKey);
41400
+ }
41401
+ });
41381
41402
  return {
41382
41403
  has(agentId) {
41383
41404
  const g = groupOfAgent(agentId);
@@ -41405,6 +41426,14 @@ function createWorkerActivityFeed(opts) {
41405
41426
  if (existingRow?.finished === true)
41406
41427
  return Promise.resolve();
41407
41428
  const feedKey = feedKeyOf(chatId, threadId);
41429
+ const priorFeedKey = agentIndex.get(agentId);
41430
+ if (priorFeedKey != null && priorFeedKey !== feedKey) {
41431
+ const priorGroup = groups.get(priorFeedKey);
41432
+ if (priorGroup != null) {
41433
+ evictRowFromGroup(priorGroup, agentId);
41434
+ syncPin(priorGroup);
41435
+ }
41436
+ }
41408
41437
  let g = groups.get(feedKey);
41409
41438
  if (g == null) {
41410
41439
  g = {
@@ -76759,6 +76788,28 @@ function workerAgentIdOfPinKey(pinKey) {
76759
76788
  const agentId = pinKey.slice(WORKER_PIN_KEY_PREFIX.length);
76760
76789
  return agentId.length > 0 ? agentId : null;
76761
76790
  }
76791
+ function storeOnlyWorkerPinCandidates(args) {
76792
+ const out = [];
76793
+ for (const r of args.rows) {
76794
+ if (workerAgentIdOfPinKey(r.pinKey) == null)
76795
+ continue;
76796
+ if (r.pending)
76797
+ continue;
76798
+ if (r.expiresAt != null)
76799
+ continue;
76800
+ if (r.chatId.length === 0)
76801
+ continue;
76802
+ if (args.inMemoryPinKeys.has(r.pinKey))
76803
+ continue;
76804
+ out.push({
76805
+ pinKey: r.pinKey,
76806
+ chatId: r.chatId,
76807
+ pinnedAt: args.now,
76808
+ messageId: r.messageId
76809
+ });
76810
+ }
76811
+ return out;
76812
+ }
76762
76813
  function decideWorkerPinReaps(args) {
76763
76814
  const reaps = [];
76764
76815
  for (const pin of args.pins) {
@@ -84131,10 +84182,10 @@ function effectiveTurnAgeMs(markerAgeMs, turnStartedAt, now) {
84131
84182
  }
84132
84183
 
84133
84184
  // ../src/build-info.ts
84134
- var VERSION = "0.18.27";
84135
- var COMMIT_SHA = "93871829";
84136
- var COMMIT_DATE = "2026-07-16T01:31:20Z";
84137
- var LATEST_PR = 3273;
84185
+ var VERSION = "0.18.28";
84186
+ var COMMIT_SHA = "ef0be9a2";
84187
+ var COMMIT_DATE = "2026-07-16T14:42:35+10:00";
84188
+ var LATEST_PR = 3274;
84138
84189
  var COMMITS_AHEAD_OF_TAG = 0;
84139
84190
 
84140
84191
  // gateway/boot-version.ts
@@ -89325,11 +89376,19 @@ async function runMidSessionCardReaper() {
89325
89376
  }
89326
89377
  if (WORKER_PIN_REAPER_ENABLED && PIN_STATUS_WHILE_WORKING) {
89327
89378
  try {
89328
- const candidates = [...statusPinState.keys()].filter((k) => k.startsWith("wk:")).map((k) => ({
89379
+ const inMemoryKeys = new Set([...statusPinState.keys()].filter((k) => k.startsWith("wk:")));
89380
+ const inMemoryCandidates = [...inMemoryKeys].map((k) => ({
89329
89381
  pinKey: k,
89330
89382
  chatId: statusPinChatIds.get(k) ?? "",
89331
89383
  pinnedAt: statusPinPinnedAt.get(k) ?? now
89332
89384
  }));
89385
+ const storeOnlyCandidates = statusPinPersistEnabled ? storeOnlyWorkerPinCandidates({
89386
+ rows: loadStatusPins(STATUS_PIN_STORE_PATH, statusPinStoreFs),
89387
+ inMemoryPinKeys: inMemoryKeys,
89388
+ now
89389
+ }) : [];
89390
+ const storeOnlyKeys = new Set(storeOnlyCandidates.map((c) => c.pinKey));
89391
+ const candidates = [...inMemoryCandidates, ...storeOnlyCandidates];
89333
89392
  const reaps = decideWorkerPinReaps({
89334
89393
  pins: candidates,
89335
89394
  statusOf: (agentId) => {
@@ -89356,9 +89415,20 @@ async function runMidSessionCardReaper() {
89356
89415
  now
89357
89416
  });
89358
89417
  for (const reap of reaps) {
89359
- process.stderr.write(`telegram gateway: worker-pin reaper unpinning ${reap.pinKey} (chat=${reap.chatId} reason=${reap.reason})
89418
+ const storeOnly = storeOnlyKeys.has(reap.pinKey);
89419
+ process.stderr.write(`telegram gateway: worker-pin reaper unpinning ${reap.pinKey} (chat=${reap.chatId} reason=${reap.reason}${storeOnly ? " source=store-orphan" : ""})
89420
+ `);
89421
+ if (storeOnly && reap.messageId != null) {
89422
+ try {
89423
+ await statusPinApi().unpinChatMessage(reap.chatId, reap.messageId);
89424
+ } catch (err) {
89425
+ process.stderr.write(`telegram gateway: worker-pin reaper store-orphan unpin failed (${reap.pinKey} chat=${reap.chatId} msg=${reap.messageId}): ${err.message}
89360
89426
  `);
89361
- await reconcileStatusPin(reap.pinKey, reap.chatId, { pinned: false });
89427
+ }
89428
+ await mutateStatusPinRow(STATUS_PIN_STORE_PATH, statusPinStoreFs, reap.pinKey, null);
89429
+ } else {
89430
+ await reconcileStatusPin(reap.pinKey, reap.chatId, { pinned: false });
89431
+ }
89362
89432
  }
89363
89433
  } catch (err) {
89364
89434
  process.stderr.write(`telegram gateway: worker-pin reaper error: ${err.message}
@@ -604,6 +604,7 @@ import {
604
604
  } from './queued-card-store.js'
605
605
  import {
606
606
  decideWorkerPinReaps,
607
+ storeOnlyWorkerPinCandidates,
607
608
  WORKER_PIN_TTL_MS_DEFAULT,
608
609
  } from './worker-pin-reaper.js'
609
610
  import { driveEscalation } from './escalation-drive.js'
@@ -9023,16 +9024,32 @@ async function runMidSessionCardReaper(): Promise<void> {
9023
9024
  // the durable store row clear together.
9024
9025
  if (WORKER_PIN_REAPER_ENABLED && PIN_STATUS_WHILE_WORKING) {
9025
9026
  try {
9026
- const candidates = [...statusPinState.keys()]
9027
- .filter((k) => k.startsWith('wk:'))
9028
- .map((k) => ({
9029
- pinKey: k,
9030
- chatId: statusPinChatIds.get(k) ?? '',
9031
- // A missing timestamp (should not happen — set on every claim)
9032
- // degrades to "claimed just now": terminality can still reap it,
9033
- // the TTL gate never can. Conservative, never a spurious unpin.
9034
- pinnedAt: statusPinPinnedAt.get(k) ?? now,
9035
- }))
9027
+ const inMemoryKeys = new Set(
9028
+ [...statusPinState.keys()].filter((k) => k.startsWith('wk:')),
9029
+ )
9030
+ const inMemoryCandidates = [...inMemoryKeys].map((k) => ({
9031
+ pinKey: k,
9032
+ chatId: statusPinChatIds.get(k) ?? '',
9033
+ // A missing timestamp (should not happen set on every claim)
9034
+ // degrades to "claimed just now": terminality can still reap it,
9035
+ // the TTL gate never can. Conservative, never a spurious unpin.
9036
+ pinnedAt: statusPinPinnedAt.get(k) ?? now,
9037
+ }))
9038
+ // #3001 durable group net: fold in `wk:` rows that live in the DURABLE
9039
+ // store but have NO in-memory claim — the divergence window where the
9040
+ // claim was lost but the Telegram pin + store row survive. These would
9041
+ // otherwise linger until the next boot cleanup; the reconciling sweep
9042
+ // recovers them mid-session too, group-safely (per-message unpin of a
9043
+ // bot-tracked row, NEVER an unpin-all, NEVER an untracked/human pin).
9044
+ const storeOnlyCandidates = statusPinPersistEnabled
9045
+ ? storeOnlyWorkerPinCandidates({
9046
+ rows: loadStatusPins(STATUS_PIN_STORE_PATH, statusPinStoreFs),
9047
+ inMemoryPinKeys: inMemoryKeys,
9048
+ now,
9049
+ })
9050
+ : []
9051
+ const storeOnlyKeys = new Set(storeOnlyCandidates.map((c) => c.pinKey))
9052
+ const candidates = [...inMemoryCandidates, ...storeOnlyCandidates]
9036
9053
  const reaps = decideWorkerPinReaps({
9037
9054
  pins: candidates,
9038
9055
  statusOf: (agentId) => {
@@ -9064,11 +9081,35 @@ async function runMidSessionCardReaper(): Promise<void> {
9064
9081
  now,
9065
9082
  })
9066
9083
  for (const reap of reaps) {
9084
+ const storeOnly = storeOnlyKeys.has(reap.pinKey)
9067
9085
  process.stderr.write(
9068
9086
  `telegram gateway: worker-pin reaper unpinning ${reap.pinKey} ` +
9069
- `(chat=${reap.chatId} reason=${reap.reason})\n`,
9087
+ `(chat=${reap.chatId} reason=${reap.reason}` +
9088
+ `${storeOnly ? ' source=store-orphan' : ''})\n`,
9070
9089
  )
9071
- await reconcileStatusPin(reap.pinKey, reap.chatId, { pinned: false })
9090
+ if (storeOnly && reap.messageId != null) {
9091
+ // No in-memory claim to reconcile: unpin the exact tracked message
9092
+ // (per-message — group-safe) and drop the store row directly. Both
9093
+ // are best-effort/idempotent; a failed unpin still drops the row so
9094
+ // the next boot's cleanup is the final backstop.
9095
+ try {
9096
+ await statusPinApi().unpinChatMessage(reap.chatId, reap.messageId)
9097
+ } catch (err) {
9098
+ process.stderr.write(
9099
+ `telegram gateway: worker-pin reaper store-orphan unpin failed ` +
9100
+ `(${reap.pinKey} chat=${reap.chatId} msg=${reap.messageId}): ` +
9101
+ `${(err as Error).message}\n`,
9102
+ )
9103
+ }
9104
+ await mutateStatusPinRow(
9105
+ STATUS_PIN_STORE_PATH,
9106
+ statusPinStoreFs,
9107
+ reap.pinKey,
9108
+ null,
9109
+ )
9110
+ } else {
9111
+ await reconcileStatusPin(reap.pinKey, reap.chatId, { pinned: false })
9112
+ }
9072
9113
  }
9073
9114
  } catch (err) {
9074
9115
  process.stderr.write(
@@ -52,6 +52,10 @@ export interface WorkerPinCandidate {
52
52
  chatId: string
53
53
  /** Wall-clock ms the claim was first taken (gateway's pinnedAt registry). */
54
54
  pinnedAt: number
55
+ /** The pinned message id. Present for store-only candidates (whose reap is a
56
+ * raw per-message unpin — no in-memory claim to reconcile through). Omitted
57
+ * for in-memory candidates, which reap via reconcileStatusPin off the claim. */
58
+ messageId?: number
55
59
  }
56
60
 
57
61
  export interface WorkerPinReap extends WorkerPinCandidate {
@@ -86,6 +90,56 @@ export type WorkerRegistryStatus = 'terminal' | 'running' | 'unknown'
86
90
  * for stalled / missing / lookup-error; a DB hiccup must degrade to 'unknown'
87
91
  * — kept until the TTL — never to a spurious 'terminal' unpin).
88
92
  */
93
+ /** A durable-store row (status-pins.json) as seen by the reconciling sweep. */
94
+ export interface StoreOrphanRow {
95
+ pinKey: string
96
+ chatId: string
97
+ messageId: number
98
+ /** Pin API call still in-flight (persist-intent-first). Never reaped. */
99
+ pending?: boolean
100
+ /** Time-scoped `tool:` pin. Never a worker orphan; excluded. */
101
+ expiresAt?: number
102
+ }
103
+
104
+ /**
105
+ * Build the extra `wk:` reap candidates that exist ONLY in the durable store
106
+ * (status-pins.json) and NOT in the in-memory claim map — the divergence window
107
+ * where the in-memory claim was lost/never-held but the Telegram pin AND its
108
+ * store row survive. Without this, such an orphan lingers until the next
109
+ * gateway boot (runStatusPinBootCleanup); this lets the PERIODIC sweep recover
110
+ * it too, group-safely (per-message unpin of a bot-tracked row — never an
111
+ * unpin-all, never a human pin).
112
+ *
113
+ * The store persists NO timestamp, so a store-only candidate carries
114
+ * `pinnedAt = now`: only a TERMINAL registry verdict can reap it. It is never
115
+ * TTL-reaped (no trustworthy age to age it out), never touched while its worker
116
+ * is `running`, and it is always a `wk:` row (the documented leak class).
117
+ * `pending` rows (pin API in-flight) and time-scoped `tool:` rows are excluded,
118
+ * as are rows already tracked in memory (the in-memory reaper owns those, with
119
+ * a real `pinnedAt` that can drive the TTL).
120
+ */
121
+ export function storeOnlyWorkerPinCandidates(args: {
122
+ rows: Iterable<StoreOrphanRow>
123
+ inMemoryPinKeys: ReadonlySet<string>
124
+ now: number
125
+ }): WorkerPinCandidate[] {
126
+ const out: WorkerPinCandidate[] = []
127
+ for (const r of args.rows) {
128
+ if (workerAgentIdOfPinKey(r.pinKey) == null) continue // only wk: rows
129
+ if (r.pending) continue // pin API in-flight — do not race the write
130
+ if (r.expiresAt != null) continue // time-scoped tool pin — not a worker
131
+ if (r.chatId.length === 0) continue // can't unpin without a chat
132
+ if (args.inMemoryPinKeys.has(r.pinKey)) continue // in-memory reaper owns it
133
+ out.push({
134
+ pinKey: r.pinKey,
135
+ chatId: r.chatId,
136
+ pinnedAt: args.now,
137
+ messageId: r.messageId,
138
+ })
139
+ }
140
+ return out
141
+ }
142
+
89
143
  export function decideWorkerPinReaps(args: {
90
144
  pins: Iterable<WorkerPinCandidate>
91
145
  statusOf: (agentId: string) => WorkerRegistryStatus
@@ -0,0 +1,140 @@
1
+ /**
2
+ * Regression guard for the feed-registry churn bug (FIX 2): a worker row whose
3
+ * `agentId` migrated feeds (a fresh `update()` under a different chat/thread)
4
+ * used to leave the OLD group's `workers` map holding an orphan row that the
5
+ * `agentIndex` no longer pointed at. `groupOfAgent(agentId)` could then never
6
+ * resolve it, so the heartbeat TTL sweep's `terminateWorker(agentId)` no-op'd
7
+ * and the row churned every tick forever — the self-contradictory
8
+ * `worker-feed: TTL reap … no update in 0s (>= 3000s); force-terminating
9
+ * leaked row` log, repeating ~every 6s.
10
+ *
11
+ * Two guarantees are asserted as OUTCOMES:
12
+ * 1. Migration eviction (root cause): after the same agentId updates a
13
+ * different feed, the old group's row is gone immediately — total tracked
14
+ * rows never double-count the migrated worker.
15
+ * 2. Sweep force-eviction is idempotent: once every worker has ended, the TTL
16
+ * sweep drives `size` to 0 and it STAYS 0 across repeated heartbeat ticks
17
+ * (no churning row that re-reaps forever).
18
+ */
19
+ import { describe, it, expect } from 'vitest'
20
+ import {
21
+ createWorkerActivityFeed,
22
+ type BotApiForWorkerFeed,
23
+ type WorkerActivityView,
24
+ } from '../worker-activity-feed.js'
25
+
26
+ function view(desc: string, elapsedMs: number): WorkerActivityView {
27
+ return { description: desc, lastTool: null, toolCount: 1, latestSummary: 'step', elapsedMs, state: 'running' }
28
+ }
29
+
30
+ type IndexControl = { repointAgentIndex: (agentId: string, feedKey: string | null) => void }
31
+
32
+ function makeFeed(nowRef: { t: number }) {
33
+ const pins: { messageId: number | null }[] = []
34
+ let seq = 900
35
+ let control: IndexControl | null = null
36
+ const bot: BotApiForWorkerFeed = {
37
+ sendMessage: async () => ({ message_id: seq++ }),
38
+ editMessageText: async () => true,
39
+ }
40
+ const feed = createWorkerActivityFeed({
41
+ bot,
42
+ now: () => nowRef.t,
43
+ minEditIntervalMs: 0,
44
+ heartbeatTickMs: 1000,
45
+ firstPaintMinMs: 0,
46
+ staleWorkerTtlMs: 3_000_000, // 3000s — matches the observed churn log
47
+ setInterval: () => 0,
48
+ clearInterval: () => {},
49
+ reconcilePin: ({ messageId }) => pins.push({ messageId }),
50
+ exposeTestControls: (c) => {
51
+ control = c
52
+ },
53
+ })
54
+ return { feed, pins, control: () => control as IndexControl }
55
+ }
56
+
57
+ async function drain(): Promise<void> {
58
+ for (let i = 0; i < 12; i++) await new Promise((r) => setImmediate(r))
59
+ }
60
+
61
+ describe('worker-feed migration eviction + churn-free sweep (FIX 2)', () => {
62
+ it('evicts the old-feed orphan when an agentId migrates feeds — no double-count', async () => {
63
+ const nowRef = { t: 1_000_000 }
64
+ const { feed } = makeFeed(nowRef)
65
+
66
+ // Worker registers in feed A (chat -100).
67
+ await feed.update('agent-1', '-100', view('task', 0))
68
+ await drain()
69
+ expect(feed.size).toBe(1)
70
+
71
+ // SAME agentId now updates a DIFFERENT feed (chat -200) — a migration. The
72
+ // old feed-A row must be evicted, not orphaned. Total rows stay 1.
73
+ nowRef.t += 1000
74
+ await feed.update('agent-1', '-200', view('task', 1000))
75
+ await drain()
76
+ expect(feed.size).toBe(1)
77
+ })
78
+
79
+ it('TTL sweep drives size to 0 and stays 0 across repeated ticks (no churn)', async () => {
80
+ const nowRef = { t: 2_000_000 }
81
+ const { feed } = makeFeed(nowRef)
82
+
83
+ await feed.update('agent-1', '-100', view('task', 0))
84
+ await drain()
85
+ // Migrate to a second feed to exercise the exact desync path that leaked.
86
+ nowRef.t += 1000
87
+ await feed.update('agent-1', '-200', view('task', 1000))
88
+ await drain()
89
+ expect(feed.size).toBe(1)
90
+
91
+ // Advance past the silence TTL so the sweep force-terminates the last row.
92
+ nowRef.t += 3_000_001
93
+ feed.heartbeatTick()
94
+ await drain()
95
+ expect(feed.size).toBe(0)
96
+
97
+ // Idempotency: further ticks must not resurrect / re-reap a churning row.
98
+ for (let i = 0; i < 3; i++) {
99
+ nowRef.t += 3_000_001
100
+ feed.heartbeatTick()
101
+ await drain()
102
+ }
103
+ expect(feed.size).toBe(0)
104
+ })
105
+
106
+ it('TTL sweep FORCE-EVICTS a leaked row whose agentIndex has desynced from its group', async () => {
107
+ // Reproduce the exact desync the migration-eviction fix now prevents any
108
+ // public sequence from producing: a live row physically in group A while
109
+ // the agentIndex points somewhere else, so `groupOfAgent(agentId) !== g`.
110
+ // This drives the heartbeat sweep's FORCE-EVICT branch specifically — the
111
+ // one that removes the orphan directly from the group it lives in instead
112
+ // of no-op'ing through `terminateWorker` (which would churn forever).
113
+ const nowRef = { t: 3_000_000 }
114
+ const { feed, control } = makeFeed(nowRef)
115
+
116
+ await feed.update('agent-1', '-100', view('task', 0))
117
+ await drain()
118
+ expect(feed.size).toBe(1)
119
+
120
+ // Corrupt the index so it no longer resolves to the group holding the row.
121
+ // `terminateWorker('agent-1')` would now look up a non-existent group and
122
+ // no-op — the leak the force-evict branch exists to catch.
123
+ control().repointAgentIndex('agent-1', 'bogus-feed-key')
124
+
125
+ // Past the silence TTL: the sweep must force-evict the row from its real
126
+ // group, NOT rely on terminateWorker.
127
+ nowRef.t += 3_000_001
128
+ feed.heartbeatTick()
129
+ await drain()
130
+ expect(feed.size).toBe(0)
131
+
132
+ // And it stays gone — no churning row re-reaped every tick.
133
+ for (let i = 0; i < 3; i++) {
134
+ nowRef.t += 3_000_001
135
+ feed.heartbeatTick()
136
+ await drain()
137
+ }
138
+ expect(feed.size).toBe(0)
139
+ })
140
+ })
@@ -1,6 +1,7 @@
1
1
  import { describe, it, expect } from "vitest";
2
2
  import {
3
3
  decideWorkerPinReaps,
4
+ storeOnlyWorkerPinCandidates,
4
5
  workerAgentIdOfPinKey,
5
6
  WORKER_PIN_TTL_MS_DEFAULT,
6
7
  type WorkerPinCandidate,
@@ -130,3 +131,80 @@ describe("decideWorkerPinReaps (#3001 mid-session wk: sweep)", () => {
130
131
  expect(reaps).toHaveLength(0);
131
132
  });
132
133
  });
134
+
135
+ describe("storeOnlyWorkerPinCandidates (#3001 durable group net)", () => {
136
+ const inMem = (keys: string[] = []) => new Set(keys);
137
+
138
+ it("promotes a wk: store row that has NO in-memory claim into a reap candidate (group chat)", () => {
139
+ // A GROUP chat id is negative; the reconciling sweep must recover a
140
+ // bot-tracked orphan there via a per-message unpin (never an unpin-all).
141
+ const cands = storeOnlyWorkerPinCandidates({
142
+ rows: [{ pinKey: "wk:group:-100:5", chatId: "-100", messageId: 42 }],
143
+ inMemoryPinKeys: inMem(),
144
+ now: NOW,
145
+ });
146
+ expect(cands).toHaveLength(1);
147
+ expect(cands[0].pinKey).toBe("wk:group:-100:5");
148
+ expect(cands[0].chatId).toBe("-100");
149
+ // messageId is carried so the gateway can per-message unpin (group-safe).
150
+ expect(cands[0].messageId).toBe(42);
151
+ // A terminal registry verdict reaps it now, however "young" (pinnedAt=now).
152
+ const reaps = decideWorkerPinReaps({
153
+ pins: cands,
154
+ statusOf: () => "terminal" as const,
155
+ ttlMs: TTL,
156
+ now: NOW,
157
+ });
158
+ expect(reaps.map((r) => r.pinKey)).toEqual(["wk:group:-100:5"]);
159
+ });
160
+
161
+ it("NEVER promotes an untracked/human pin — only wk: rows qualify (fg:/tool:/banner: excluded)", () => {
162
+ const cands = storeOnlyWorkerPinCandidates({
163
+ rows: [
164
+ { pinKey: "fg:-100:9", chatId: "-100", messageId: 1 },
165
+ { pinKey: "tool:-100:9", chatId: "-100", messageId: 2, expiresAt: NOW + 1 },
166
+ { pinKey: "banner:owner", chatId: "-100", messageId: 3 },
167
+ ],
168
+ inMemoryPinKeys: inMem(),
169
+ now: NOW,
170
+ });
171
+ expect(cands).toHaveLength(0);
172
+ });
173
+
174
+ it("skips rows the in-memory reaper already owns (dedup by pinKey)", () => {
175
+ const cands = storeOnlyWorkerPinCandidates({
176
+ rows: [{ pinKey: "wk:a", chatId: "-100", messageId: 7 }],
177
+ inMemoryPinKeys: inMem(["wk:a"]),
178
+ now: NOW,
179
+ });
180
+ expect(cands).toHaveLength(0);
181
+ });
182
+
183
+ it("skips pending (pin API in-flight) and time-scoped tool rows and empty-chat rows", () => {
184
+ const cands = storeOnlyWorkerPinCandidates({
185
+ rows: [
186
+ { pinKey: "wk:pending", chatId: "-100", messageId: 1, pending: true },
187
+ { pinKey: "wk:tool", chatId: "-100", messageId: 2, expiresAt: NOW + 1000 },
188
+ { pinKey: "wk:nochat", chatId: "", messageId: 3 },
189
+ ],
190
+ inMemoryPinKeys: inMem(),
191
+ now: NOW,
192
+ });
193
+ expect(cands).toHaveLength(0);
194
+ });
195
+
196
+ it("a store-orphan with a live 'running' worker is kept (never reaped mid-run)", () => {
197
+ const cands = storeOnlyWorkerPinCandidates({
198
+ rows: [{ pinKey: "wk:live", chatId: "-100", messageId: 5 }],
199
+ inMemoryPinKeys: inMem(),
200
+ now: NOW,
201
+ });
202
+ const reaps = decideWorkerPinReaps({
203
+ pins: cands,
204
+ statusOf: () => "running" as const,
205
+ ttlMs: TTL,
206
+ now: NOW,
207
+ });
208
+ expect(reaps).toHaveLength(0);
209
+ });
210
+ });
@@ -393,6 +393,21 @@ export interface WorkerActivityFeedOpts {
393
393
  threadId?: number
394
394
  messageId: number | null
395
395
  }) => void
396
+ /**
397
+ * TEST-ONLY (never set in production). Invoked once at construction with a
398
+ * narrow control that repoints the internal `agentId → feedKey` index WITHOUT
399
+ * moving the worker's row. Its sole purpose is to reproduce the
400
+ * `agentIndex`/`workers` DESYNC that the migration-eviction root-cause fix now
401
+ * prevents any public API sequence from producing — so the heartbeat sweep's
402
+ * force-evict backstop (the branch that removes a leaked row when
403
+ * `groupOfAgent(agentId)` no longer resolves to the group physically holding
404
+ * it) can be covered by a deterministic test. A no-op when unset.
405
+ */
406
+ exposeTestControls?: (controls: {
407
+ /** Set the internal index entry for `agentId` to `feedKey` (or delete it
408
+ * when `feedKey` is null) without touching any group's `workers` map. */
409
+ repointAgentIndex: (agentId: string, feedKey: string | null) => void
410
+ }) => void
396
411
  }
397
412
 
398
413
  /**
@@ -897,6 +912,20 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
897
912
  maybeDeleteGroup(g)
898
913
  }
899
914
 
915
+ /**
916
+ * Force-evict a LEAKED row from the specific group it lives in, used when the
917
+ * agentIndex has desynced (points at a different feed, or none) so the normal
918
+ * `removeWorker`/`terminateWorker` path — which resolves the group via the
919
+ * index — can't reach it. Deletes the index entry ONLY if it still points at
920
+ * THIS group, so a live row for the same agentId in another feed keeps its
921
+ * mapping intact.
922
+ */
923
+ function evictRowFromGroup(g: FeedGroup, agentId: string): void {
924
+ g.workers.delete(agentId)
925
+ if (agentIndex.get(agentId) === g.feedKey) agentIndex.delete(agentId)
926
+ maybeDeleteGroup(g)
927
+ }
928
+
900
929
  /**
901
930
  * Delete an empty group. A group is gone only when it has NO tracked workers
902
931
  * AND no staged terminal repaints pending — the latter keeps the group object
@@ -1243,7 +1272,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
1243
1272
  // that keeps getting `update()` cues every heartbeat resets
1244
1273
  // `lastUpdateAt` forever, so only an age anchor immune to that reset can
1245
1274
  // reap it (Carrie 5h zombie pin, re-edited 3000+ times).
1246
- const staleAgentIds: Array<{ agentId: string; reason: 'silence' | 'absolute' }> = []
1275
+ const staleAgentIds: Array<{ g: FeedGroup; agentId: string; reason: 'silence' | 'absolute' }> = []
1247
1276
  const staleFinished: Array<{ g: FeedGroup; agentId: string; reason: 'silence' | 'absolute' }> = []
1248
1277
  for (const g of groups.values()) {
1249
1278
  for (const row of g.workers.values()) {
@@ -1253,8 +1282,12 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
1253
1282
  // Attribute the reap to the absolute cap only when silence alone would
1254
1283
  // NOT have fired — so the log names the trigger that actually caught it.
1255
1284
  const reason: 'silence' | 'absolute' = silent ? 'silence' : 'absolute'
1285
+ // Carry the group the row was ACTUALLY found in (not `groupOfAgent`,
1286
+ // which resolves via the agentIndex — that index can desync from a
1287
+ // group's `workers` map when an agentId migrated feeds, orphaning the
1288
+ // old row. Using `g` guarantees the reap evicts the row that exists.)
1256
1289
  if (row.finished) staleFinished.push({ g, agentId: row.agentId, reason })
1257
- else staleAgentIds.push({ agentId: row.agentId, reason })
1290
+ else staleAgentIds.push({ g, agentId: row.agentId, reason })
1258
1291
  }
1259
1292
  }
1260
1293
  // GC any FINISHED row still lingering past the TTL (#3207: finished rows
@@ -1274,15 +1307,32 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
1274
1307
  removeWorker(g, agentId)
1275
1308
  syncPin(g)
1276
1309
  }
1277
- for (const { agentId, reason } of staleAgentIds) {
1278
- const row = groupOfAgent(agentId)?.workers.get(agentId)
1310
+ for (const { g, agentId, reason } of staleAgentIds) {
1311
+ // Read the row from the group it was FOUND in, so the log reports the
1312
+ // real age even when the agentIndex has desynced (the old bug printed
1313
+ // "no update in 0s" because it read `groupOfAgent(agentId)`, which
1314
+ // resolved to a different/absent group and fell back to `now`).
1315
+ const row = g.workers.get(agentId)
1279
1316
  if (reason === 'absolute') {
1280
1317
  const age = Math.floor((now - (row?.createdAtMs ?? now)) / 1000)
1281
1318
  log(`worker-feed: ABSOLUTE cap reap agent=${agentId} — row age ${age}s (>= ${Math.floor(absoluteRowLifetimeCapMs / 1000)}s); force-terminating immortal row (survives lastUpdateAt reset)`)
1282
1319
  } else {
1283
1320
  log(`worker-feed: TTL reap agent=${agentId} — no update in ${Math.floor((now - (row?.lastUpdateAt ?? now)) / 1000)}s (>= ${Math.floor(staleWorkerTtlMs / 1000)}s); force-terminating leaked row`)
1284
1321
  }
1285
- void terminateWorker(agentId)
1322
+ // Normal honest finalize when the agentIndex still points at THIS group
1323
+ // (renders the terminal recap + releases the pin through the chain). But
1324
+ // when the index has desynced — it resolves to a different group or none
1325
+ // — `terminateWorker` would look up the wrong/no row and no-op, leaving
1326
+ // this row to churn every heartbeat forever. Detect that and force-evict
1327
+ // the leaked row directly from the group it actually lives in.
1328
+ if (groupOfAgent(agentId) === g) {
1329
+ void terminateWorker(agentId)
1330
+ } else {
1331
+ log(`worker-feed: force-evicting leaked row agent=${agentId} feed=${g.feedKey} — agentIndex desynced (points elsewhere); removing directly`)
1332
+ markFinalized(agentId)
1333
+ evictRowFromGroup(g, agentId)
1334
+ syncPin(g)
1335
+ }
1286
1336
  }
1287
1337
  for (const g of [...groups.values()]) {
1288
1338
  // Deferred-finalize re-drive: a terminal edit that hit a cooldown/flood
@@ -1369,6 +1419,15 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
1369
1419
 
1370
1420
  heartbeatTimer = setIntervalFn(heartbeatTick, heartbeatTickMs)
1371
1421
 
1422
+ // TEST-ONLY: hand out the narrow index-repoint control (see opts doc). Never
1423
+ // provided in production wiring, so this is a no-op there.
1424
+ opts.exposeTestControls?.({
1425
+ repointAgentIndex: (agentId, feedKey) => {
1426
+ if (feedKey == null) agentIndex.delete(agentId)
1427
+ else agentIndex.set(agentId, feedKey)
1428
+ },
1429
+ })
1430
+
1372
1431
  return {
1373
1432
  has(agentId) {
1374
1433
  const g = groupOfAgent(agentId)
@@ -1401,6 +1460,20 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
1401
1460
  if (existingRow?.finished === true) return Promise.resolve()
1402
1461
 
1403
1462
  const feedKey = feedKeyOf(chatId, threadId)
1463
+ // Feed-migration cleanup (root-cause of the "no update in 0s" churn): if
1464
+ // this agentId is already indexed to a DIFFERENT feed, its prior row is
1465
+ // about to be orphaned (we re-index below to the new feed, but the old
1466
+ // group's `workers` map still holds the stale row — invisible to
1467
+ // `groupOfAgent` forever after, so the TTL sweep can never evict it and
1468
+ // it churns every heartbeat). Evict the stale row from its old group now.
1469
+ const priorFeedKey = agentIndex.get(agentId)
1470
+ if (priorFeedKey != null && priorFeedKey !== feedKey) {
1471
+ const priorGroup = groups.get(priorFeedKey)
1472
+ if (priorGroup != null) {
1473
+ evictRowFromGroup(priorGroup, agentId)
1474
+ syncPin(priorGroup)
1475
+ }
1476
+ }
1404
1477
  let g = groups.get(feedKey)
1405
1478
  if (g == null) {
1406
1479
  g = {