switchroom 0.18.26 → 0.18.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/README.md +6 -2
  2. package/dist/cli/ms-365-write-pretool.mjs +4953 -14
  3. package/dist/cli/switchroom.js +1 -1
  4. package/dist/host-control/main.js +1 -1
  5. package/package.json +1 -1
  6. package/profiles/_base/start.sh.hbs +16 -0
  7. package/telegram-plugin/dist/gateway/gateway.js +571 -43
  8. package/telegram-plugin/flushed-turn-supersede.ts +58 -0
  9. package/telegram-plugin/gateway/derive-turn-id.ts +32 -0
  10. package/telegram-plugin/gateway/gateway.ts +358 -53
  11. package/telegram-plugin/gateway/handback-preturn-signal.ts +442 -0
  12. package/telegram-plugin/gateway/model-command.ts +68 -0
  13. package/telegram-plugin/gateway/ms365-write-approval.test.ts +101 -0
  14. package/telegram-plugin/gateway/ms365-write-approval.ts +65 -3
  15. package/telegram-plugin/gateway/subagent-handback-inbound-builder.ts +12 -0
  16. package/telegram-plugin/gateway/turn-active-marker.ts +35 -0
  17. package/telegram-plugin/gateway/worker-pin-reaper.ts +54 -0
  18. package/telegram-plugin/send-gate.test.ts +138 -0
  19. package/telegram-plugin/send-gate.ts +104 -1
  20. package/telegram-plugin/tests/activity-ever-opened-sticky.test.ts +14 -4
  21. package/telegram-plugin/tests/effort-command.test.ts +47 -0
  22. package/telegram-plugin/tests/flushed-turn-supersede.test.ts +60 -0
  23. package/telegram-plugin/tests/handback-preturn-adoption-roundtrip.test.ts +211 -0
  24. package/telegram-plugin/tests/handback-preturn-signal.test.ts +346 -0
  25. package/telegram-plugin/tests/model-command.test.ts +112 -0
  26. package/telegram-plugin/tests/multitopic-routing-wiring.test.ts +14 -2
  27. package/telegram-plugin/tests/outbound-send-chunks.test.ts +57 -0
  28. package/telegram-plugin/tests/permission-no-repeat-wiring.test.ts +18 -11
  29. package/telegram-plugin/tests/reply-owner-resolve.test.ts +90 -0
  30. package/telegram-plugin/tests/subagent-handback-inbound-builder.test.ts +5 -0
  31. package/telegram-plugin/tests/turn-active-marker.test.ts +29 -0
  32. package/telegram-plugin/tests/worker-activity-feed.test.ts +121 -0
  33. package/telegram-plugin/tests/worker-feed-migration-eviction.test.ts +140 -0
  34. package/telegram-plugin/tests/worker-pin-reaper.test.ts +78 -0
  35. package/telegram-plugin/worker-activity-feed.ts +169 -6
@@ -0,0 +1,140 @@
1
+ /**
2
+ * Regression guard for the feed-registry churn bug (FIX 2): a worker row whose
3
+ * `agentId` migrated feeds (a fresh `update()` under a different chat/thread)
4
+ * used to leave the OLD group's `workers` map holding an orphan row that the
5
+ * `agentIndex` no longer pointed at. `groupOfAgent(agentId)` could then never
6
+ * resolve it, so the heartbeat TTL sweep's `terminateWorker(agentId)` no-op'd
7
+ * and the row churned every tick forever — the self-contradictory
8
+ * `worker-feed: TTL reap … no update in 0s (>= 3000s); force-terminating
9
+ * leaked row` log, repeating ~every 6s.
10
+ *
11
+ * Two guarantees are asserted as OUTCOMES:
12
+ * 1. Migration eviction (root cause): after the same agentId updates a
13
+ * different feed, the old group's row is gone immediately — total tracked
14
+ * rows never double-count the migrated worker.
15
+ * 2. Sweep force-eviction is idempotent: once every worker has ended, the TTL
16
+ * sweep drives `size` to 0 and it STAYS 0 across repeated heartbeat ticks
17
+ * (no churning row that re-reaps forever).
18
+ */
19
+ import { describe, it, expect } from 'vitest'
20
+ import {
21
+ createWorkerActivityFeed,
22
+ type BotApiForWorkerFeed,
23
+ type WorkerActivityView,
24
+ } from '../worker-activity-feed.js'
25
+
26
+ function view(desc: string, elapsedMs: number): WorkerActivityView {
27
+ return { description: desc, lastTool: null, toolCount: 1, latestSummary: 'step', elapsedMs, state: 'running' }
28
+ }
29
+
30
+ type IndexControl = { repointAgentIndex: (agentId: string, feedKey: string | null) => void }
31
+
32
+ function makeFeed(nowRef: { t: number }) {
33
+ const pins: { messageId: number | null }[] = []
34
+ let seq = 900
35
+ let control: IndexControl | null = null
36
+ const bot: BotApiForWorkerFeed = {
37
+ sendMessage: async () => ({ message_id: seq++ }),
38
+ editMessageText: async () => true,
39
+ }
40
+ const feed = createWorkerActivityFeed({
41
+ bot,
42
+ now: () => nowRef.t,
43
+ minEditIntervalMs: 0,
44
+ heartbeatTickMs: 1000,
45
+ firstPaintMinMs: 0,
46
+ staleWorkerTtlMs: 3_000_000, // 3000s — matches the observed churn log
47
+ setInterval: () => 0,
48
+ clearInterval: () => {},
49
+ reconcilePin: ({ messageId }) => pins.push({ messageId }),
50
+ exposeTestControls: (c) => {
51
+ control = c
52
+ },
53
+ })
54
+ return { feed, pins, control: () => control as IndexControl }
55
+ }
56
+
57
+ async function drain(): Promise<void> {
58
+ for (let i = 0; i < 12; i++) await new Promise((r) => setImmediate(r))
59
+ }
60
+
61
+ describe('worker-feed migration eviction + churn-free sweep (FIX 2)', () => {
62
+ it('evicts the old-feed orphan when an agentId migrates feeds — no double-count', async () => {
63
+ const nowRef = { t: 1_000_000 }
64
+ const { feed } = makeFeed(nowRef)
65
+
66
+ // Worker registers in feed A (chat -100).
67
+ await feed.update('agent-1', '-100', view('task', 0))
68
+ await drain()
69
+ expect(feed.size).toBe(1)
70
+
71
+ // SAME agentId now updates a DIFFERENT feed (chat -200) — a migration. The
72
+ // old feed-A row must be evicted, not orphaned. Total rows stay 1.
73
+ nowRef.t += 1000
74
+ await feed.update('agent-1', '-200', view('task', 1000))
75
+ await drain()
76
+ expect(feed.size).toBe(1)
77
+ })
78
+
79
+ it('TTL sweep drives size to 0 and stays 0 across repeated ticks (no churn)', async () => {
80
+ const nowRef = { t: 2_000_000 }
81
+ const { feed } = makeFeed(nowRef)
82
+
83
+ await feed.update('agent-1', '-100', view('task', 0))
84
+ await drain()
85
+ // Migrate to a second feed to exercise the exact desync path that leaked.
86
+ nowRef.t += 1000
87
+ await feed.update('agent-1', '-200', view('task', 1000))
88
+ await drain()
89
+ expect(feed.size).toBe(1)
90
+
91
+ // Advance past the silence TTL so the sweep force-terminates the last row.
92
+ nowRef.t += 3_000_001
93
+ feed.heartbeatTick()
94
+ await drain()
95
+ expect(feed.size).toBe(0)
96
+
97
+ // Idempotency: further ticks must not resurrect / re-reap a churning row.
98
+ for (let i = 0; i < 3; i++) {
99
+ nowRef.t += 3_000_001
100
+ feed.heartbeatTick()
101
+ await drain()
102
+ }
103
+ expect(feed.size).toBe(0)
104
+ })
105
+
106
+ it('TTL sweep FORCE-EVICTS a leaked row whose agentIndex has desynced from its group', async () => {
107
+ // Reproduce the exact desync the migration-eviction fix now prevents any
108
+ // public sequence from producing: a live row physically in group A while
109
+ // the agentIndex points somewhere else, so `groupOfAgent(agentId) !== g`.
110
+ // This drives the heartbeat sweep's FORCE-EVICT branch specifically — the
111
+ // one that removes the orphan directly from the group it lives in instead
112
+ // of no-op'ing through `terminateWorker` (which would churn forever).
113
+ const nowRef = { t: 3_000_000 }
114
+ const { feed, control } = makeFeed(nowRef)
115
+
116
+ await feed.update('agent-1', '-100', view('task', 0))
117
+ await drain()
118
+ expect(feed.size).toBe(1)
119
+
120
+ // Corrupt the index so it no longer resolves to the group holding the row.
121
+ // `terminateWorker('agent-1')` would now look up a non-existent group and
122
+ // no-op — the leak the force-evict branch exists to catch.
123
+ control().repointAgentIndex('agent-1', 'bogus-feed-key')
124
+
125
+ // Past the silence TTL: the sweep must force-evict the row from its real
126
+ // group, NOT rely on terminateWorker.
127
+ nowRef.t += 3_000_001
128
+ feed.heartbeatTick()
129
+ await drain()
130
+ expect(feed.size).toBe(0)
131
+
132
+ // And it stays gone — no churning row re-reaped every tick.
133
+ for (let i = 0; i < 3; i++) {
134
+ nowRef.t += 3_000_001
135
+ feed.heartbeatTick()
136
+ await drain()
137
+ }
138
+ expect(feed.size).toBe(0)
139
+ })
140
+ })
@@ -1,6 +1,7 @@
1
1
  import { describe, it, expect } from "vitest";
2
2
  import {
3
3
  decideWorkerPinReaps,
4
+ storeOnlyWorkerPinCandidates,
4
5
  workerAgentIdOfPinKey,
5
6
  WORKER_PIN_TTL_MS_DEFAULT,
6
7
  type WorkerPinCandidate,
@@ -130,3 +131,80 @@ describe("decideWorkerPinReaps (#3001 mid-session wk: sweep)", () => {
130
131
  expect(reaps).toHaveLength(0);
131
132
  });
132
133
  });
134
+
135
+ describe("storeOnlyWorkerPinCandidates (#3001 durable group net)", () => {
136
+ const inMem = (keys: string[] = []) => new Set(keys);
137
+
138
+ it("promotes a wk: store row that has NO in-memory claim into a reap candidate (group chat)", () => {
139
+ // A GROUP chat id is negative; the reconciling sweep must recover a
140
+ // bot-tracked orphan there via a per-message unpin (never an unpin-all).
141
+ const cands = storeOnlyWorkerPinCandidates({
142
+ rows: [{ pinKey: "wk:group:-100:5", chatId: "-100", messageId: 42 }],
143
+ inMemoryPinKeys: inMem(),
144
+ now: NOW,
145
+ });
146
+ expect(cands).toHaveLength(1);
147
+ expect(cands[0].pinKey).toBe("wk:group:-100:5");
148
+ expect(cands[0].chatId).toBe("-100");
149
+ // messageId is carried so the gateway can per-message unpin (group-safe).
150
+ expect(cands[0].messageId).toBe(42);
151
+ // A terminal registry verdict reaps it now, however "young" (pinnedAt=now).
152
+ const reaps = decideWorkerPinReaps({
153
+ pins: cands,
154
+ statusOf: () => "terminal" as const,
155
+ ttlMs: TTL,
156
+ now: NOW,
157
+ });
158
+ expect(reaps.map((r) => r.pinKey)).toEqual(["wk:group:-100:5"]);
159
+ });
160
+
161
+ it("NEVER promotes an untracked/human pin — only wk: rows qualify (fg:/tool:/banner: excluded)", () => {
162
+ const cands = storeOnlyWorkerPinCandidates({
163
+ rows: [
164
+ { pinKey: "fg:-100:9", chatId: "-100", messageId: 1 },
165
+ { pinKey: "tool:-100:9", chatId: "-100", messageId: 2, expiresAt: NOW + 1 },
166
+ { pinKey: "banner:owner", chatId: "-100", messageId: 3 },
167
+ ],
168
+ inMemoryPinKeys: inMem(),
169
+ now: NOW,
170
+ });
171
+ expect(cands).toHaveLength(0);
172
+ });
173
+
174
+ it("skips rows the in-memory reaper already owns (dedup by pinKey)", () => {
175
+ const cands = storeOnlyWorkerPinCandidates({
176
+ rows: [{ pinKey: "wk:a", chatId: "-100", messageId: 7 }],
177
+ inMemoryPinKeys: inMem(["wk:a"]),
178
+ now: NOW,
179
+ });
180
+ expect(cands).toHaveLength(0);
181
+ });
182
+
183
+ it("skips pending (pin API in-flight) and time-scoped tool rows and empty-chat rows", () => {
184
+ const cands = storeOnlyWorkerPinCandidates({
185
+ rows: [
186
+ { pinKey: "wk:pending", chatId: "-100", messageId: 1, pending: true },
187
+ { pinKey: "wk:tool", chatId: "-100", messageId: 2, expiresAt: NOW + 1000 },
188
+ { pinKey: "wk:nochat", chatId: "", messageId: 3 },
189
+ ],
190
+ inMemoryPinKeys: inMem(),
191
+ now: NOW,
192
+ });
193
+ expect(cands).toHaveLength(0);
194
+ });
195
+
196
+ it("a store-orphan with a live 'running' worker is kept (never reaped mid-run)", () => {
197
+ const cands = storeOnlyWorkerPinCandidates({
198
+ rows: [{ pinKey: "wk:live", chatId: "-100", messageId: 5 }],
199
+ inMemoryPinKeys: inMem(),
200
+ now: NOW,
201
+ });
202
+ const reaps = decideWorkerPinReaps({
203
+ pins: cands,
204
+ statusOf: () => "running" as const,
205
+ ttlMs: TTL,
206
+ now: NOW,
207
+ });
208
+ expect(reaps).toHaveLength(0);
209
+ });
210
+ });
@@ -238,6 +238,21 @@ export interface WorkerActivityFeedOpts {
238
238
  * `finish` edit bypass it.
239
239
  */
240
240
  minEditIntervalMs?: number
241
+ /**
242
+ * Coarse cadence (ms) for elapsed-ONLY refreshes. When the only thing that
243
+ * changed since the last edit is the volatile elapsed clock (no new step, no
244
+ * state/toolCount/token change), the card is NOT re-edited until this much
245
+ * time has passed — so a worker sitting in one long tool call advances its
246
+ * clock only every ~`elapsedRefreshMs`, not every jsonl tick. This is the
247
+ * primary defense against the sustained same-message edit stream that earns a
248
+ * Telegram flood ban (finn incident: 26,460 clock-only edits → ~88min 429).
249
+ * A SUBSTANTIVE change (new narrative step, done/failed, toolCount/token
250
+ * delta) still renders promptly under `minEditIntervalMs`. Default 15000ms.
251
+ * The terminal finish edit and forced edits bypass it. Liveness for a truly
252
+ * silent worker is already carried by the typing loop, so a slow clock is not
253
+ * a liveness regression.
254
+ */
255
+ elapsedRefreshMs?: number
241
256
  /**
242
257
  * A worker must have been running at least this long before its first
243
258
  * message is posted. Sub-second / trivial workers never surface a live
@@ -378,6 +393,21 @@ export interface WorkerActivityFeedOpts {
378
393
  threadId?: number
379
394
  messageId: number | null
380
395
  }) => void
396
+ /**
397
+ * TEST-ONLY (never set in production). Invoked once at construction with a
398
+ * narrow control that repoints the internal `agentId → feedKey` index WITHOUT
399
+ * moving the worker's row. Its sole purpose is to reproduce the
400
+ * `agentIndex`/`workers` DESYNC that the migration-eviction root-cause fix now
401
+ * prevents any public API sequence from producing — so the heartbeat sweep's
402
+ * force-evict backstop (the branch that removes a leaked row when
403
+ * `groupOfAgent(agentId)` no longer resolves to the group physically holding
404
+ * it) can be covered by a deterministic test. A no-op when unset.
405
+ */
406
+ exposeTestControls?: (controls: {
407
+ /** Set the internal index entry for `agentId` to `feedKey` (or delete it
408
+ * when `feedKey` is null) without touching any group's `workers` map. */
409
+ repointAgentIndex: (agentId: string, feedKey: string | null) => void
410
+ }) => void
381
411
  }
382
412
 
383
413
  /**
@@ -467,6 +497,14 @@ interface FeedGroup {
467
497
  */
468
498
  messageCreatedAtMs: number
469
499
  lastBody: string | null
500
+ /**
501
+ * Substance signature of the last EDITED body — every rendered field EXCEPT
502
+ * the volatile elapsed clock (and heartbeat live suffix). Used to distinguish
503
+ * an elapsed-only change (paced to `elapsedRefreshMs`) from a real update
504
+ * (new step / state / toolCount / tokens, rendered promptly). Null until the
505
+ * group's first paint. See {@link groupSubstanceKey}.
506
+ */
507
+ lastSubstanceKey: string | null
470
508
  lastEditAt: number
471
509
  cooldownUntil: number
472
510
  /** Single serialization chain for the shared message — ticks can't interleave sends. */
@@ -647,6 +685,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
647
685
  const nowFn = opts.now ?? Date.now
648
686
  const floodWaitRemainingMs = opts.floodWaitRemainingMs ?? (() => 0)
649
687
  const minEditInterval = opts.minEditIntervalMs ?? 2500
688
+ const elapsedRefreshMs = Math.max(minEditInterval, Math.floor(opts.elapsedRefreshMs ?? 15000))
650
689
  const firstPaintMin = opts.firstPaintMinMs ?? 8000
651
690
  const heartbeatTickMs = opts.heartbeatTickMs ?? 6000
652
691
  const maxRows = Math.max(1, Math.floor(opts.maxRows ?? 8))
@@ -820,6 +859,52 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
820
859
  return renderCombinedWorkerFeed(rows, { maxRows })
821
860
  }
822
861
 
862
+ /**
863
+ * Substance signature of a group's render at `now` — a stable string of every
864
+ * field that renders EXCEPT the volatile elapsed clock (and the heartbeat
865
+ * live suffix, which is elapsed-derived). Two renders with the same substance
866
+ * key differ only in their clock; an edit between them advances nothing the
867
+ * user cares about and is exactly the churn that accumulates into a flood ban.
868
+ *
869
+ * Built from the group data (not by string-stripping the rendered body) so it
870
+ * is deterministic and robust to render-format changes: description, state,
871
+ * toolCount, totalTokens, and the full narrative trail per running row (or the
872
+ * terminal recap's state/result/narrative when finalizing).
873
+ */
874
+ function groupSubstanceKey(g: FeedGroup, terminalRecap: WorkerActivityView | null): string {
875
+ // Unambiguous delimiters so adjacent fields can never blur into a boundary
876
+ // collision (e.g. toolCount `1`+desc `"23"` vs toolCount `12`+desc `"3"`).
877
+ // FS separates fields within a row; RS separates rows in the combined body.
878
+ const FS = '\x00'
879
+ const RS = '\x1e'
880
+ if (terminalRecap != null) {
881
+ return [
882
+ 'T',
883
+ terminalRecap.state,
884
+ terminalRecap.description,
885
+ terminalRecap.toolCount,
886
+ terminalRecap.totalTokens ?? '',
887
+ terminalRecap.latestSummary,
888
+ ...(terminalRecap.narrativeLines ?? []),
889
+ ].join(FS)
890
+ }
891
+ const running = runningRows(g)
892
+ if (running.length === 0) return 'EMPTY'
893
+ return running
894
+ .map((r) => {
895
+ const v = r.lastView as WorkerActivityView
896
+ return [
897
+ r.agentId,
898
+ v.state,
899
+ v.description,
900
+ v.toolCount,
901
+ v.totalTokens ?? '',
902
+ ...r.narrative,
903
+ ].join(FS)
904
+ })
905
+ .join(RS)
906
+ }
907
+
823
908
  /** Remove a worker's row + index entry; delete the group if it is now empty. */
824
909
  function removeWorker(g: FeedGroup, agentId: string): void {
825
910
  g.workers.delete(agentId)
@@ -827,6 +912,20 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
827
912
  maybeDeleteGroup(g)
828
913
  }
829
914
 
915
+ /**
916
+ * Force-evict a LEAKED row from the specific group it lives in, used when the
917
+ * agentIndex has desynced (points at a different feed, or none) so the normal
918
+ * `removeWorker`/`terminateWorker` path — which resolves the group via the
919
+ * index — can't reach it. Deletes the index entry ONLY if it still points at
920
+ * THIS group, so a live row for the same agentId in another feed keeps its
921
+ * mapping intact.
922
+ */
923
+ function evictRowFromGroup(g: FeedGroup, agentId: string): void {
924
+ g.workers.delete(agentId)
925
+ if (agentIndex.get(agentId) === g.feedKey) agentIndex.delete(agentId)
926
+ maybeDeleteGroup(g)
927
+ }
928
+
830
929
  /**
831
930
  * Delete an empty group. A group is gone only when it has NO tracked workers
832
931
  * AND no staged terminal repaints pending — the latter keeps the group object
@@ -926,6 +1025,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
926
1025
  }
927
1026
 
928
1027
  const body = renderGroupBody(g, now, opts2.terminalRecap ?? null, opts2.heartbeat ?? false)
1028
+ const substanceKey = groupSubstanceKey(g, opts2.terminalRecap ?? null)
929
1029
  if (body == null) {
930
1030
  // Nothing to show. On a terminal finalize the row is already dropped;
931
1031
  // clear the staged repaint (the handback carries the result).
@@ -961,6 +1061,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
961
1061
  // group-message lifetime cap measures the message's true absolute age.
962
1062
  g.messageCreatedAtMs = now
963
1063
  g.lastBody = body
1064
+ g.lastSubstanceKey = substanceKey
964
1065
  g.lastEditAt = now
965
1066
  // A fresh message is a live running paint, never a terminal recap.
966
1067
  g.terminalPainted = false
@@ -982,7 +1083,19 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
982
1083
  if (isTerminal) clearStaged()
983
1084
  return
984
1085
  }
985
- if (!opts2.force && now - g.lastEditAt < minEditInterval) return
1086
+ if (!opts2.force && !isTerminal) {
1087
+ // Spacing floor: never edit the same message faster than the 429 floor.
1088
+ if (now - g.lastEditAt < minEditInterval) return
1089
+ // Elapsed-only pacing (flood-ban defense): if the SUBSTANCE is unchanged
1090
+ // since the last edit — the body differs only because the clock advanced —
1091
+ // hold the edit until the coarse `elapsedRefreshMs` cadence. A real change
1092
+ // (new step / state / toolCount / tokens) shifts `substanceKey` and falls
1093
+ // through immediately (subject only to the spacing floor above). This is
1094
+ // what collapses the sustained ~0.33/s clock-only edit stream that
1095
+ // accumulates into Telegram's per-message flood counter.
1096
+ const substanceChanged = substanceKey !== g.lastSubstanceKey
1097
+ if (!substanceChanged && now - g.lastEditAt < elapsedRefreshMs) return
1098
+ }
986
1099
 
987
1100
  try {
988
1101
  const res = await opts.bot.editMessageText(g.chatId, g.messageId, body, sendOptsFor(g))
@@ -996,6 +1109,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
996
1109
  return
997
1110
  }
998
1111
  g.lastBody = body
1112
+ g.lastSubstanceKey = substanceKey
999
1113
  g.lastEditAt = now
1000
1114
  if (isTerminal) {
1001
1115
  log(
@@ -1029,6 +1143,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
1029
1143
  }
1030
1144
  if (outcome === 'not_modified') {
1031
1145
  g.lastBody = body
1146
+ g.lastSubstanceKey = substanceKey
1032
1147
  g.lastEditAt = now
1033
1148
  if (isTerminal) clearStaged()
1034
1149
  return
@@ -1040,6 +1155,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
1040
1155
  g.messageId = null
1041
1156
  g.messageCreatedAtMs = 0
1042
1157
  g.lastBody = null
1158
+ g.lastSubstanceKey = null
1043
1159
  // The pinned message no longer exists → release the group pin claim
1044
1160
  // (clearStaged already re-syncs on the terminal path).
1045
1161
  if (isTerminal) clearStaged()
@@ -1156,7 +1272,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
1156
1272
  // that keeps getting `update()` cues every heartbeat resets
1157
1273
  // `lastUpdateAt` forever, so only an age anchor immune to that reset can
1158
1274
  // reap it (Carrie 5h zombie pin, re-edited 3000+ times).
1159
- const staleAgentIds: Array<{ agentId: string; reason: 'silence' | 'absolute' }> = []
1275
+ const staleAgentIds: Array<{ g: FeedGroup; agentId: string; reason: 'silence' | 'absolute' }> = []
1160
1276
  const staleFinished: Array<{ g: FeedGroup; agentId: string; reason: 'silence' | 'absolute' }> = []
1161
1277
  for (const g of groups.values()) {
1162
1278
  for (const row of g.workers.values()) {
@@ -1166,8 +1282,12 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
1166
1282
  // Attribute the reap to the absolute cap only when silence alone would
1167
1283
  // NOT have fired — so the log names the trigger that actually caught it.
1168
1284
  const reason: 'silence' | 'absolute' = silent ? 'silence' : 'absolute'
1285
+ // Carry the group the row was ACTUALLY found in (not `groupOfAgent`,
1286
+ // which resolves via the agentIndex — that index can desync from a
1287
+ // group's `workers` map when an agentId migrated feeds, orphaning the
1288
+ // old row. Using `g` guarantees the reap evicts the row that exists.)
1169
1289
  if (row.finished) staleFinished.push({ g, agentId: row.agentId, reason })
1170
- else staleAgentIds.push({ agentId: row.agentId, reason })
1290
+ else staleAgentIds.push({ g, agentId: row.agentId, reason })
1171
1291
  }
1172
1292
  }
1173
1293
  // GC any FINISHED row still lingering past the TTL (#3207: finished rows
@@ -1187,15 +1307,32 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
1187
1307
  removeWorker(g, agentId)
1188
1308
  syncPin(g)
1189
1309
  }
1190
- for (const { agentId, reason } of staleAgentIds) {
1191
- const row = groupOfAgent(agentId)?.workers.get(agentId)
1310
+ for (const { g, agentId, reason } of staleAgentIds) {
1311
+ // Read the row from the group it was FOUND in, so the log reports the
1312
+ // real age even when the agentIndex has desynced (the old bug printed
1313
+ // "no update in 0s" because it read `groupOfAgent(agentId)`, which
1314
+ // resolved to a different/absent group and fell back to `now`).
1315
+ const row = g.workers.get(agentId)
1192
1316
  if (reason === 'absolute') {
1193
1317
  const age = Math.floor((now - (row?.createdAtMs ?? now)) / 1000)
1194
1318
  log(`worker-feed: ABSOLUTE cap reap agent=${agentId} — row age ${age}s (>= ${Math.floor(absoluteRowLifetimeCapMs / 1000)}s); force-terminating immortal row (survives lastUpdateAt reset)`)
1195
1319
  } else {
1196
1320
  log(`worker-feed: TTL reap agent=${agentId} — no update in ${Math.floor((now - (row?.lastUpdateAt ?? now)) / 1000)}s (>= ${Math.floor(staleWorkerTtlMs / 1000)}s); force-terminating leaked row`)
1197
1321
  }
1198
- void terminateWorker(agentId)
1322
+ // Normal honest finalize when the agentIndex still points at THIS group
1323
+ // (renders the terminal recap + releases the pin through the chain). But
1324
+ // when the index has desynced — it resolves to a different group or none
1325
+ // — `terminateWorker` would look up the wrong/no row and no-op, leaving
1326
+ // this row to churn every heartbeat forever. Detect that and force-evict
1327
+ // the leaked row directly from the group it actually lives in.
1328
+ if (groupOfAgent(agentId) === g) {
1329
+ void terminateWorker(agentId)
1330
+ } else {
1331
+ log(`worker-feed: force-evicting leaked row agent=${agentId} feed=${g.feedKey} — agentIndex desynced (points elsewhere); removing directly`)
1332
+ markFinalized(agentId)
1333
+ evictRowFromGroup(g, agentId)
1334
+ syncPin(g)
1335
+ }
1199
1336
  }
1200
1337
  for (const g of [...groups.values()]) {
1201
1338
  // Deferred-finalize re-drive: a terminal edit that hit a cooldown/flood
@@ -1242,6 +1379,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
1242
1379
  g.messageId = null
1243
1380
  g.messageCreatedAtMs = 0
1244
1381
  g.lastBody = null
1382
+ g.lastSubstanceKey = null
1245
1383
  // Clear the (possibly stale) pin claim for the retired message; the
1246
1384
  // fresh first-paint below re-pins the new one from a null claim.
1247
1385
  syncPin(g)
@@ -1281,6 +1419,15 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
1281
1419
 
1282
1420
  heartbeatTimer = setIntervalFn(heartbeatTick, heartbeatTickMs)
1283
1421
 
1422
+ // TEST-ONLY: hand out the narrow index-repoint control (see opts doc). Never
1423
+ // provided in production wiring, so this is a no-op there.
1424
+ opts.exposeTestControls?.({
1425
+ repointAgentIndex: (agentId, feedKey) => {
1426
+ if (feedKey == null) agentIndex.delete(agentId)
1427
+ else agentIndex.set(agentId, feedKey)
1428
+ },
1429
+ })
1430
+
1284
1431
  return {
1285
1432
  has(agentId) {
1286
1433
  const g = groupOfAgent(agentId)
@@ -1313,6 +1460,20 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
1313
1460
  if (existingRow?.finished === true) return Promise.resolve()
1314
1461
 
1315
1462
  const feedKey = feedKeyOf(chatId, threadId)
1463
+ // Feed-migration cleanup (root-cause of the "no update in 0s" churn): if
1464
+ // this agentId is already indexed to a DIFFERENT feed, its prior row is
1465
+ // about to be orphaned (we re-index below to the new feed, but the old
1466
+ // group's `workers` map still holds the stale row — invisible to
1467
+ // `groupOfAgent` forever after, so the TTL sweep can never evict it and
1468
+ // it churns every heartbeat). Evict the stale row from its old group now.
1469
+ const priorFeedKey = agentIndex.get(agentId)
1470
+ if (priorFeedKey != null && priorFeedKey !== feedKey) {
1471
+ const priorGroup = groups.get(priorFeedKey)
1472
+ if (priorGroup != null) {
1473
+ evictRowFromGroup(priorGroup, agentId)
1474
+ syncPin(priorGroup)
1475
+ }
1476
+ }
1316
1477
  let g = groups.get(feedKey)
1317
1478
  if (g == null) {
1318
1479
  g = {
@@ -1322,6 +1483,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
1322
1483
  messageId: null,
1323
1484
  messageCreatedAtMs: 0,
1324
1485
  lastBody: null,
1486
+ lastSubstanceKey: null,
1325
1487
  lastEditAt: 0,
1326
1488
  cooldownUntil: 0,
1327
1489
  chain: Promise.resolve(),
@@ -1342,6 +1504,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
1342
1504
  g.messageId = null
1343
1505
  g.messageCreatedAtMs = 0
1344
1506
  g.lastBody = null
1507
+ g.lastSubstanceKey = null
1345
1508
  g.pendingFinalize.clear()
1346
1509
  g.terminalPainted = false
1347
1510
  // The stale terminal message is no longer this group's pinned surface.