switchroom 0.18.26 → 0.18.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/cli/ms-365-write-pretool.mjs +4953 -14
- package/dist/cli/switchroom.js +1 -1
- package/dist/host-control/main.js +1 -1
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +16 -0
- package/telegram-plugin/dist/gateway/gateway.js +571 -43
- package/telegram-plugin/flushed-turn-supersede.ts +58 -0
- package/telegram-plugin/gateway/derive-turn-id.ts +32 -0
- package/telegram-plugin/gateway/gateway.ts +358 -53
- package/telegram-plugin/gateway/handback-preturn-signal.ts +442 -0
- package/telegram-plugin/gateway/model-command.ts +68 -0
- package/telegram-plugin/gateway/ms365-write-approval.test.ts +101 -0
- package/telegram-plugin/gateway/ms365-write-approval.ts +65 -3
- package/telegram-plugin/gateway/subagent-handback-inbound-builder.ts +12 -0
- package/telegram-plugin/gateway/turn-active-marker.ts +35 -0
- package/telegram-plugin/gateway/worker-pin-reaper.ts +54 -0
- package/telegram-plugin/send-gate.test.ts +138 -0
- package/telegram-plugin/send-gate.ts +104 -1
- package/telegram-plugin/tests/activity-ever-opened-sticky.test.ts +14 -4
- package/telegram-plugin/tests/effort-command.test.ts +47 -0
- package/telegram-plugin/tests/flushed-turn-supersede.test.ts +60 -0
- package/telegram-plugin/tests/handback-preturn-adoption-roundtrip.test.ts +211 -0
- package/telegram-plugin/tests/handback-preturn-signal.test.ts +346 -0
- package/telegram-plugin/tests/model-command.test.ts +112 -0
- package/telegram-plugin/tests/multitopic-routing-wiring.test.ts +14 -2
- package/telegram-plugin/tests/outbound-send-chunks.test.ts +57 -0
- package/telegram-plugin/tests/permission-no-repeat-wiring.test.ts +18 -11
- package/telegram-plugin/tests/reply-owner-resolve.test.ts +90 -0
- package/telegram-plugin/tests/subagent-handback-inbound-builder.test.ts +5 -0
- package/telegram-plugin/tests/turn-active-marker.test.ts +29 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +121 -0
- package/telegram-plugin/tests/worker-feed-migration-eviction.test.ts +140 -0
- package/telegram-plugin/tests/worker-pin-reaper.test.ts +78 -0
- package/telegram-plugin/worker-activity-feed.ts +169 -6
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Regression guard for the feed-registry churn bug (FIX 2): a worker row whose
|
|
3
|
+
* `agentId` migrated feeds (a fresh `update()` under a different chat/thread)
|
|
4
|
+
* used to leave the OLD group's `workers` map holding an orphan row that the
|
|
5
|
+
* `agentIndex` no longer pointed at. `groupOfAgent(agentId)` could then never
|
|
6
|
+
* resolve it, so the heartbeat TTL sweep's `terminateWorker(agentId)` no-op'd
|
|
7
|
+
* and the row churned every tick forever — the self-contradictory
|
|
8
|
+
* `worker-feed: TTL reap … no update in 0s (>= 3000s); force-terminating
|
|
9
|
+
* leaked row` log, repeating ~every 6s.
|
|
10
|
+
*
|
|
11
|
+
* Two guarantees are asserted as OUTCOMES:
|
|
12
|
+
* 1. Migration eviction (root cause): after the same agentId updates a
|
|
13
|
+
* different feed, the old group's row is gone immediately — total tracked
|
|
14
|
+
* rows never double-count the migrated worker.
|
|
15
|
+
* 2. Sweep force-eviction is idempotent: once every worker has ended, the TTL
|
|
16
|
+
* sweep drives `size` to 0 and it STAYS 0 across repeated heartbeat ticks
|
|
17
|
+
* (no churning row that re-reaps forever).
|
|
18
|
+
*/
|
|
19
|
+
import { describe, it, expect } from 'vitest'
|
|
20
|
+
import {
|
|
21
|
+
createWorkerActivityFeed,
|
|
22
|
+
type BotApiForWorkerFeed,
|
|
23
|
+
type WorkerActivityView,
|
|
24
|
+
} from '../worker-activity-feed.js'
|
|
25
|
+
|
|
26
|
+
function view(desc: string, elapsedMs: number): WorkerActivityView {
|
|
27
|
+
return { description: desc, lastTool: null, toolCount: 1, latestSummary: 'step', elapsedMs, state: 'running' }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
type IndexControl = { repointAgentIndex: (agentId: string, feedKey: string | null) => void }
|
|
31
|
+
|
|
32
|
+
function makeFeed(nowRef: { t: number }) {
|
|
33
|
+
const pins: { messageId: number | null }[] = []
|
|
34
|
+
let seq = 900
|
|
35
|
+
let control: IndexControl | null = null
|
|
36
|
+
const bot: BotApiForWorkerFeed = {
|
|
37
|
+
sendMessage: async () => ({ message_id: seq++ }),
|
|
38
|
+
editMessageText: async () => true,
|
|
39
|
+
}
|
|
40
|
+
const feed = createWorkerActivityFeed({
|
|
41
|
+
bot,
|
|
42
|
+
now: () => nowRef.t,
|
|
43
|
+
minEditIntervalMs: 0,
|
|
44
|
+
heartbeatTickMs: 1000,
|
|
45
|
+
firstPaintMinMs: 0,
|
|
46
|
+
staleWorkerTtlMs: 3_000_000, // 3000s — matches the observed churn log
|
|
47
|
+
setInterval: () => 0,
|
|
48
|
+
clearInterval: () => {},
|
|
49
|
+
reconcilePin: ({ messageId }) => pins.push({ messageId }),
|
|
50
|
+
exposeTestControls: (c) => {
|
|
51
|
+
control = c
|
|
52
|
+
},
|
|
53
|
+
})
|
|
54
|
+
return { feed, pins, control: () => control as IndexControl }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function drain(): Promise<void> {
|
|
58
|
+
for (let i = 0; i < 12; i++) await new Promise((r) => setImmediate(r))
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
describe('worker-feed migration eviction + churn-free sweep (FIX 2)', () => {
|
|
62
|
+
it('evicts the old-feed orphan when an agentId migrates feeds — no double-count', async () => {
|
|
63
|
+
const nowRef = { t: 1_000_000 }
|
|
64
|
+
const { feed } = makeFeed(nowRef)
|
|
65
|
+
|
|
66
|
+
// Worker registers in feed A (chat -100).
|
|
67
|
+
await feed.update('agent-1', '-100', view('task', 0))
|
|
68
|
+
await drain()
|
|
69
|
+
expect(feed.size).toBe(1)
|
|
70
|
+
|
|
71
|
+
// SAME agentId now updates a DIFFERENT feed (chat -200) — a migration. The
|
|
72
|
+
// old feed-A row must be evicted, not orphaned. Total rows stay 1.
|
|
73
|
+
nowRef.t += 1000
|
|
74
|
+
await feed.update('agent-1', '-200', view('task', 1000))
|
|
75
|
+
await drain()
|
|
76
|
+
expect(feed.size).toBe(1)
|
|
77
|
+
})
|
|
78
|
+
|
|
79
|
+
it('TTL sweep drives size to 0 and stays 0 across repeated ticks (no churn)', async () => {
|
|
80
|
+
const nowRef = { t: 2_000_000 }
|
|
81
|
+
const { feed } = makeFeed(nowRef)
|
|
82
|
+
|
|
83
|
+
await feed.update('agent-1', '-100', view('task', 0))
|
|
84
|
+
await drain()
|
|
85
|
+
// Migrate to a second feed to exercise the exact desync path that leaked.
|
|
86
|
+
nowRef.t += 1000
|
|
87
|
+
await feed.update('agent-1', '-200', view('task', 1000))
|
|
88
|
+
await drain()
|
|
89
|
+
expect(feed.size).toBe(1)
|
|
90
|
+
|
|
91
|
+
// Advance past the silence TTL so the sweep force-terminates the last row.
|
|
92
|
+
nowRef.t += 3_000_001
|
|
93
|
+
feed.heartbeatTick()
|
|
94
|
+
await drain()
|
|
95
|
+
expect(feed.size).toBe(0)
|
|
96
|
+
|
|
97
|
+
// Idempotency: further ticks must not resurrect / re-reap a churning row.
|
|
98
|
+
for (let i = 0; i < 3; i++) {
|
|
99
|
+
nowRef.t += 3_000_001
|
|
100
|
+
feed.heartbeatTick()
|
|
101
|
+
await drain()
|
|
102
|
+
}
|
|
103
|
+
expect(feed.size).toBe(0)
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
it('TTL sweep FORCE-EVICTS a leaked row whose agentIndex has desynced from its group', async () => {
|
|
107
|
+
// Reproduce the exact desync the migration-eviction fix now prevents any
|
|
108
|
+
// public sequence from producing: a live row physically in group A while
|
|
109
|
+
// the agentIndex points somewhere else, so `groupOfAgent(agentId) !== g`.
|
|
110
|
+
// This drives the heartbeat sweep's FORCE-EVICT branch specifically — the
|
|
111
|
+
// one that removes the orphan directly from the group it lives in instead
|
|
112
|
+
// of no-op'ing through `terminateWorker` (which would churn forever).
|
|
113
|
+
const nowRef = { t: 3_000_000 }
|
|
114
|
+
const { feed, control } = makeFeed(nowRef)
|
|
115
|
+
|
|
116
|
+
await feed.update('agent-1', '-100', view('task', 0))
|
|
117
|
+
await drain()
|
|
118
|
+
expect(feed.size).toBe(1)
|
|
119
|
+
|
|
120
|
+
// Corrupt the index so it no longer resolves to the group holding the row.
|
|
121
|
+
// `terminateWorker('agent-1')` would now look up a non-existent group and
|
|
122
|
+
// no-op — the leak the force-evict branch exists to catch.
|
|
123
|
+
control().repointAgentIndex('agent-1', 'bogus-feed-key')
|
|
124
|
+
|
|
125
|
+
// Past the silence TTL: the sweep must force-evict the row from its real
|
|
126
|
+
// group, NOT rely on terminateWorker.
|
|
127
|
+
nowRef.t += 3_000_001
|
|
128
|
+
feed.heartbeatTick()
|
|
129
|
+
await drain()
|
|
130
|
+
expect(feed.size).toBe(0)
|
|
131
|
+
|
|
132
|
+
// And it stays gone — no churning row re-reaped every tick.
|
|
133
|
+
for (let i = 0; i < 3; i++) {
|
|
134
|
+
nowRef.t += 3_000_001
|
|
135
|
+
feed.heartbeatTick()
|
|
136
|
+
await drain()
|
|
137
|
+
}
|
|
138
|
+
expect(feed.size).toBe(0)
|
|
139
|
+
})
|
|
140
|
+
})
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { describe, it, expect } from "vitest";
|
|
2
2
|
import {
|
|
3
3
|
decideWorkerPinReaps,
|
|
4
|
+
storeOnlyWorkerPinCandidates,
|
|
4
5
|
workerAgentIdOfPinKey,
|
|
5
6
|
WORKER_PIN_TTL_MS_DEFAULT,
|
|
6
7
|
type WorkerPinCandidate,
|
|
@@ -130,3 +131,80 @@ describe("decideWorkerPinReaps (#3001 mid-session wk: sweep)", () => {
|
|
|
130
131
|
expect(reaps).toHaveLength(0);
|
|
131
132
|
});
|
|
132
133
|
});
|
|
134
|
+
|
|
135
|
+
describe("storeOnlyWorkerPinCandidates (#3001 durable group net)", () => {
|
|
136
|
+
const inMem = (keys: string[] = []) => new Set(keys);
|
|
137
|
+
|
|
138
|
+
it("promotes a wk: store row that has NO in-memory claim into a reap candidate (group chat)", () => {
|
|
139
|
+
// A GROUP chat id is negative; the reconciling sweep must recover a
|
|
140
|
+
// bot-tracked orphan there via a per-message unpin (never an unpin-all).
|
|
141
|
+
const cands = storeOnlyWorkerPinCandidates({
|
|
142
|
+
rows: [{ pinKey: "wk:group:-100:5", chatId: "-100", messageId: 42 }],
|
|
143
|
+
inMemoryPinKeys: inMem(),
|
|
144
|
+
now: NOW,
|
|
145
|
+
});
|
|
146
|
+
expect(cands).toHaveLength(1);
|
|
147
|
+
expect(cands[0].pinKey).toBe("wk:group:-100:5");
|
|
148
|
+
expect(cands[0].chatId).toBe("-100");
|
|
149
|
+
// messageId is carried so the gateway can per-message unpin (group-safe).
|
|
150
|
+
expect(cands[0].messageId).toBe(42);
|
|
151
|
+
// A terminal registry verdict reaps it now, however "young" (pinnedAt=now).
|
|
152
|
+
const reaps = decideWorkerPinReaps({
|
|
153
|
+
pins: cands,
|
|
154
|
+
statusOf: () => "terminal" as const,
|
|
155
|
+
ttlMs: TTL,
|
|
156
|
+
now: NOW,
|
|
157
|
+
});
|
|
158
|
+
expect(reaps.map((r) => r.pinKey)).toEqual(["wk:group:-100:5"]);
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
it("NEVER promotes an untracked/human pin — only wk: rows qualify (fg:/tool:/banner: excluded)", () => {
|
|
162
|
+
const cands = storeOnlyWorkerPinCandidates({
|
|
163
|
+
rows: [
|
|
164
|
+
{ pinKey: "fg:-100:9", chatId: "-100", messageId: 1 },
|
|
165
|
+
{ pinKey: "tool:-100:9", chatId: "-100", messageId: 2, expiresAt: NOW + 1 },
|
|
166
|
+
{ pinKey: "banner:owner", chatId: "-100", messageId: 3 },
|
|
167
|
+
],
|
|
168
|
+
inMemoryPinKeys: inMem(),
|
|
169
|
+
now: NOW,
|
|
170
|
+
});
|
|
171
|
+
expect(cands).toHaveLength(0);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it("skips rows the in-memory reaper already owns (dedup by pinKey)", () => {
|
|
175
|
+
const cands = storeOnlyWorkerPinCandidates({
|
|
176
|
+
rows: [{ pinKey: "wk:a", chatId: "-100", messageId: 7 }],
|
|
177
|
+
inMemoryPinKeys: inMem(["wk:a"]),
|
|
178
|
+
now: NOW,
|
|
179
|
+
});
|
|
180
|
+
expect(cands).toHaveLength(0);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
it("skips pending (pin API in-flight) and time-scoped tool rows and empty-chat rows", () => {
|
|
184
|
+
const cands = storeOnlyWorkerPinCandidates({
|
|
185
|
+
rows: [
|
|
186
|
+
{ pinKey: "wk:pending", chatId: "-100", messageId: 1, pending: true },
|
|
187
|
+
{ pinKey: "wk:tool", chatId: "-100", messageId: 2, expiresAt: NOW + 1000 },
|
|
188
|
+
{ pinKey: "wk:nochat", chatId: "", messageId: 3 },
|
|
189
|
+
],
|
|
190
|
+
inMemoryPinKeys: inMem(),
|
|
191
|
+
now: NOW,
|
|
192
|
+
});
|
|
193
|
+
expect(cands).toHaveLength(0);
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
it("a store-orphan with a live 'running' worker is kept (never reaped mid-run)", () => {
|
|
197
|
+
const cands = storeOnlyWorkerPinCandidates({
|
|
198
|
+
rows: [{ pinKey: "wk:live", chatId: "-100", messageId: 5 }],
|
|
199
|
+
inMemoryPinKeys: inMem(),
|
|
200
|
+
now: NOW,
|
|
201
|
+
});
|
|
202
|
+
const reaps = decideWorkerPinReaps({
|
|
203
|
+
pins: cands,
|
|
204
|
+
statusOf: () => "running" as const,
|
|
205
|
+
ttlMs: TTL,
|
|
206
|
+
now: NOW,
|
|
207
|
+
});
|
|
208
|
+
expect(reaps).toHaveLength(0);
|
|
209
|
+
});
|
|
210
|
+
});
|
|
@@ -238,6 +238,21 @@ export interface WorkerActivityFeedOpts {
|
|
|
238
238
|
* `finish` edit bypass it.
|
|
239
239
|
*/
|
|
240
240
|
minEditIntervalMs?: number
|
|
241
|
+
/**
|
|
242
|
+
* Coarse cadence (ms) for elapsed-ONLY refreshes. When the only thing that
|
|
243
|
+
* changed since the last edit is the volatile elapsed clock (no new step, no
|
|
244
|
+
* state/toolCount/token change), the card is NOT re-edited until this much
|
|
245
|
+
* time has passed — so a worker sitting in one long tool call advances its
|
|
246
|
+
* clock only every ~`elapsedRefreshMs`, not every jsonl tick. This is the
|
|
247
|
+
* primary defense against the sustained same-message edit stream that earns a
|
|
248
|
+
* Telegram flood ban (finn incident: 26,460 clock-only edits → ~88min 429).
|
|
249
|
+
* A SUBSTANTIVE change (new narrative step, done/failed, toolCount/token
|
|
250
|
+
* delta) still renders promptly under `minEditIntervalMs`. Default 15000ms.
|
|
251
|
+
* The terminal finish edit and forced edits bypass it. Liveness for a truly
|
|
252
|
+
* silent worker is already carried by the typing loop, so a slow clock is not
|
|
253
|
+
* a liveness regression.
|
|
254
|
+
*/
|
|
255
|
+
elapsedRefreshMs?: number
|
|
241
256
|
/**
|
|
242
257
|
* A worker must have been running at least this long before its first
|
|
243
258
|
* message is posted. Sub-second / trivial workers never surface a live
|
|
@@ -378,6 +393,21 @@ export interface WorkerActivityFeedOpts {
|
|
|
378
393
|
threadId?: number
|
|
379
394
|
messageId: number | null
|
|
380
395
|
}) => void
|
|
396
|
+
/**
|
|
397
|
+
* TEST-ONLY (never set in production). Invoked once at construction with a
|
|
398
|
+
* narrow control that repoints the internal `agentId → feedKey` index WITHOUT
|
|
399
|
+
* moving the worker's row. Its sole purpose is to reproduce the
|
|
400
|
+
* `agentIndex`/`workers` DESYNC that the migration-eviction root-cause fix now
|
|
401
|
+
* prevents any public API sequence from producing — so the heartbeat sweep's
|
|
402
|
+
* force-evict backstop (the branch that removes a leaked row when
|
|
403
|
+
* `groupOfAgent(agentId)` no longer resolves to the group physically holding
|
|
404
|
+
* it) can be covered by a deterministic test. A no-op when unset.
|
|
405
|
+
*/
|
|
406
|
+
exposeTestControls?: (controls: {
|
|
407
|
+
/** Set the internal index entry for `agentId` to `feedKey` (or delete it
|
|
408
|
+
* when `feedKey` is null) without touching any group's `workers` map. */
|
|
409
|
+
repointAgentIndex: (agentId: string, feedKey: string | null) => void
|
|
410
|
+
}) => void
|
|
381
411
|
}
|
|
382
412
|
|
|
383
413
|
/**
|
|
@@ -467,6 +497,14 @@ interface FeedGroup {
|
|
|
467
497
|
*/
|
|
468
498
|
messageCreatedAtMs: number
|
|
469
499
|
lastBody: string | null
|
|
500
|
+
/**
|
|
501
|
+
* Substance signature of the last EDITED body — every rendered field EXCEPT
|
|
502
|
+
* the volatile elapsed clock (and heartbeat live suffix). Used to distinguish
|
|
503
|
+
* an elapsed-only change (paced to `elapsedRefreshMs`) from a real update
|
|
504
|
+
* (new step / state / toolCount / tokens, rendered promptly). Null until the
|
|
505
|
+
* group's first paint. See {@link groupSubstanceKey}.
|
|
506
|
+
*/
|
|
507
|
+
lastSubstanceKey: string | null
|
|
470
508
|
lastEditAt: number
|
|
471
509
|
cooldownUntil: number
|
|
472
510
|
/** Single serialization chain for the shared message — ticks can't interleave sends. */
|
|
@@ -647,6 +685,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
647
685
|
const nowFn = opts.now ?? Date.now
|
|
648
686
|
const floodWaitRemainingMs = opts.floodWaitRemainingMs ?? (() => 0)
|
|
649
687
|
const minEditInterval = opts.minEditIntervalMs ?? 2500
|
|
688
|
+
const elapsedRefreshMs = Math.max(minEditInterval, Math.floor(opts.elapsedRefreshMs ?? 15000))
|
|
650
689
|
const firstPaintMin = opts.firstPaintMinMs ?? 8000
|
|
651
690
|
const heartbeatTickMs = opts.heartbeatTickMs ?? 6000
|
|
652
691
|
const maxRows = Math.max(1, Math.floor(opts.maxRows ?? 8))
|
|
@@ -820,6 +859,52 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
820
859
|
return renderCombinedWorkerFeed(rows, { maxRows })
|
|
821
860
|
}
|
|
822
861
|
|
|
862
|
+
/**
|
|
863
|
+
* Substance signature of a group's render at `now` — a stable string of every
|
|
864
|
+
* field that renders EXCEPT the volatile elapsed clock (and the heartbeat
|
|
865
|
+
* live suffix, which is elapsed-derived). Two renders with the same substance
|
|
866
|
+
* key differ only in their clock; an edit between them advances nothing the
|
|
867
|
+
* user cares about and is exactly the churn that accumulates into a flood ban.
|
|
868
|
+
*
|
|
869
|
+
* Built from the group data (not by string-stripping the rendered body) so it
|
|
870
|
+
* is deterministic and robust to render-format changes: description, state,
|
|
871
|
+
* toolCount, totalTokens, and the full narrative trail per running row (or the
|
|
872
|
+
* terminal recap's state/result/narrative when finalizing).
|
|
873
|
+
*/
|
|
874
|
+
function groupSubstanceKey(g: FeedGroup, terminalRecap: WorkerActivityView | null): string {
|
|
875
|
+
// Unambiguous delimiters so adjacent fields can never blur into a boundary
|
|
876
|
+
// collision (e.g. toolCount `1`+desc `"23"` vs toolCount `12`+desc `"3"`).
|
|
877
|
+
// FS separates fields within a row; RS separates rows in the combined body.
|
|
878
|
+
const FS = '\x00'
|
|
879
|
+
const RS = '\x1e'
|
|
880
|
+
if (terminalRecap != null) {
|
|
881
|
+
return [
|
|
882
|
+
'T',
|
|
883
|
+
terminalRecap.state,
|
|
884
|
+
terminalRecap.description,
|
|
885
|
+
terminalRecap.toolCount,
|
|
886
|
+
terminalRecap.totalTokens ?? '',
|
|
887
|
+
terminalRecap.latestSummary,
|
|
888
|
+
...(terminalRecap.narrativeLines ?? []),
|
|
889
|
+
].join(FS)
|
|
890
|
+
}
|
|
891
|
+
const running = runningRows(g)
|
|
892
|
+
if (running.length === 0) return 'EMPTY'
|
|
893
|
+
return running
|
|
894
|
+
.map((r) => {
|
|
895
|
+
const v = r.lastView as WorkerActivityView
|
|
896
|
+
return [
|
|
897
|
+
r.agentId,
|
|
898
|
+
v.state,
|
|
899
|
+
v.description,
|
|
900
|
+
v.toolCount,
|
|
901
|
+
v.totalTokens ?? '',
|
|
902
|
+
...r.narrative,
|
|
903
|
+
].join(FS)
|
|
904
|
+
})
|
|
905
|
+
.join(RS)
|
|
906
|
+
}
|
|
907
|
+
|
|
823
908
|
/** Remove a worker's row + index entry; delete the group if it is now empty. */
|
|
824
909
|
function removeWorker(g: FeedGroup, agentId: string): void {
|
|
825
910
|
g.workers.delete(agentId)
|
|
@@ -827,6 +912,20 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
827
912
|
maybeDeleteGroup(g)
|
|
828
913
|
}
|
|
829
914
|
|
|
915
|
+
/**
|
|
916
|
+
* Force-evict a LEAKED row from the specific group it lives in, used when the
|
|
917
|
+
* agentIndex has desynced (points at a different feed, or none) so the normal
|
|
918
|
+
* `removeWorker`/`terminateWorker` path — which resolves the group via the
|
|
919
|
+
* index — can't reach it. Deletes the index entry ONLY if it still points at
|
|
920
|
+
* THIS group, so a live row for the same agentId in another feed keeps its
|
|
921
|
+
* mapping intact.
|
|
922
|
+
*/
|
|
923
|
+
function evictRowFromGroup(g: FeedGroup, agentId: string): void {
|
|
924
|
+
g.workers.delete(agentId)
|
|
925
|
+
if (agentIndex.get(agentId) === g.feedKey) agentIndex.delete(agentId)
|
|
926
|
+
maybeDeleteGroup(g)
|
|
927
|
+
}
|
|
928
|
+
|
|
830
929
|
/**
|
|
831
930
|
* Delete an empty group. A group is gone only when it has NO tracked workers
|
|
832
931
|
* AND no staged terminal repaints pending — the latter keeps the group object
|
|
@@ -926,6 +1025,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
926
1025
|
}
|
|
927
1026
|
|
|
928
1027
|
const body = renderGroupBody(g, now, opts2.terminalRecap ?? null, opts2.heartbeat ?? false)
|
|
1028
|
+
const substanceKey = groupSubstanceKey(g, opts2.terminalRecap ?? null)
|
|
929
1029
|
if (body == null) {
|
|
930
1030
|
// Nothing to show. On a terminal finalize the row is already dropped;
|
|
931
1031
|
// clear the staged repaint (the handback carries the result).
|
|
@@ -961,6 +1061,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
961
1061
|
// group-message lifetime cap measures the message's true absolute age.
|
|
962
1062
|
g.messageCreatedAtMs = now
|
|
963
1063
|
g.lastBody = body
|
|
1064
|
+
g.lastSubstanceKey = substanceKey
|
|
964
1065
|
g.lastEditAt = now
|
|
965
1066
|
// A fresh message is a live running paint, never a terminal recap.
|
|
966
1067
|
g.terminalPainted = false
|
|
@@ -982,7 +1083,19 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
982
1083
|
if (isTerminal) clearStaged()
|
|
983
1084
|
return
|
|
984
1085
|
}
|
|
985
|
-
if (!opts2.force &&
|
|
1086
|
+
if (!opts2.force && !isTerminal) {
|
|
1087
|
+
// Spacing floor: never edit the same message faster than the 429 floor.
|
|
1088
|
+
if (now - g.lastEditAt < minEditInterval) return
|
|
1089
|
+
// Elapsed-only pacing (flood-ban defense): if the SUBSTANCE is unchanged
|
|
1090
|
+
// since the last edit — the body differs only because the clock advanced —
|
|
1091
|
+
// hold the edit until the coarse `elapsedRefreshMs` cadence. A real change
|
|
1092
|
+
// (new step / state / toolCount / tokens) shifts `substanceKey` and falls
|
|
1093
|
+
// through immediately (subject only to the spacing floor above). This is
|
|
1094
|
+
// what collapses the sustained ~0.33/s clock-only edit stream that
|
|
1095
|
+
// accumulates into Telegram's per-message flood counter.
|
|
1096
|
+
const substanceChanged = substanceKey !== g.lastSubstanceKey
|
|
1097
|
+
if (!substanceChanged && now - g.lastEditAt < elapsedRefreshMs) return
|
|
1098
|
+
}
|
|
986
1099
|
|
|
987
1100
|
try {
|
|
988
1101
|
const res = await opts.bot.editMessageText(g.chatId, g.messageId, body, sendOptsFor(g))
|
|
@@ -996,6 +1109,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
996
1109
|
return
|
|
997
1110
|
}
|
|
998
1111
|
g.lastBody = body
|
|
1112
|
+
g.lastSubstanceKey = substanceKey
|
|
999
1113
|
g.lastEditAt = now
|
|
1000
1114
|
if (isTerminal) {
|
|
1001
1115
|
log(
|
|
@@ -1029,6 +1143,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1029
1143
|
}
|
|
1030
1144
|
if (outcome === 'not_modified') {
|
|
1031
1145
|
g.lastBody = body
|
|
1146
|
+
g.lastSubstanceKey = substanceKey
|
|
1032
1147
|
g.lastEditAt = now
|
|
1033
1148
|
if (isTerminal) clearStaged()
|
|
1034
1149
|
return
|
|
@@ -1040,6 +1155,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1040
1155
|
g.messageId = null
|
|
1041
1156
|
g.messageCreatedAtMs = 0
|
|
1042
1157
|
g.lastBody = null
|
|
1158
|
+
g.lastSubstanceKey = null
|
|
1043
1159
|
// The pinned message no longer exists → release the group pin claim
|
|
1044
1160
|
// (clearStaged already re-syncs on the terminal path).
|
|
1045
1161
|
if (isTerminal) clearStaged()
|
|
@@ -1156,7 +1272,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1156
1272
|
// that keeps getting `update()` cues every heartbeat resets
|
|
1157
1273
|
// `lastUpdateAt` forever, so only an age anchor immune to that reset can
|
|
1158
1274
|
// reap it (Carrie 5h zombie pin, re-edited 3000+ times).
|
|
1159
|
-
const staleAgentIds: Array<{ agentId: string; reason: 'silence' | 'absolute' }> = []
|
|
1275
|
+
const staleAgentIds: Array<{ g: FeedGroup; agentId: string; reason: 'silence' | 'absolute' }> = []
|
|
1160
1276
|
const staleFinished: Array<{ g: FeedGroup; agentId: string; reason: 'silence' | 'absolute' }> = []
|
|
1161
1277
|
for (const g of groups.values()) {
|
|
1162
1278
|
for (const row of g.workers.values()) {
|
|
@@ -1166,8 +1282,12 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1166
1282
|
// Attribute the reap to the absolute cap only when silence alone would
|
|
1167
1283
|
// NOT have fired — so the log names the trigger that actually caught it.
|
|
1168
1284
|
const reason: 'silence' | 'absolute' = silent ? 'silence' : 'absolute'
|
|
1285
|
+
// Carry the group the row was ACTUALLY found in (not `groupOfAgent`,
|
|
1286
|
+
// which resolves via the agentIndex — that index can desync from a
|
|
1287
|
+
// group's `workers` map when an agentId migrated feeds, orphaning the
|
|
1288
|
+
// old row. Using `g` guarantees the reap evicts the row that exists.)
|
|
1169
1289
|
if (row.finished) staleFinished.push({ g, agentId: row.agentId, reason })
|
|
1170
|
-
else staleAgentIds.push({ agentId: row.agentId, reason })
|
|
1290
|
+
else staleAgentIds.push({ g, agentId: row.agentId, reason })
|
|
1171
1291
|
}
|
|
1172
1292
|
}
|
|
1173
1293
|
// GC any FINISHED row still lingering past the TTL (#3207: finished rows
|
|
@@ -1187,15 +1307,32 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1187
1307
|
removeWorker(g, agentId)
|
|
1188
1308
|
syncPin(g)
|
|
1189
1309
|
}
|
|
1190
|
-
for (const { agentId, reason } of staleAgentIds) {
|
|
1191
|
-
|
|
1310
|
+
for (const { g, agentId, reason } of staleAgentIds) {
|
|
1311
|
+
// Read the row from the group it was FOUND in, so the log reports the
|
|
1312
|
+
// real age even when the agentIndex has desynced (the old bug printed
|
|
1313
|
+
// "no update in 0s" because it read `groupOfAgent(agentId)`, which
|
|
1314
|
+
// resolved to a different/absent group and fell back to `now`).
|
|
1315
|
+
const row = g.workers.get(agentId)
|
|
1192
1316
|
if (reason === 'absolute') {
|
|
1193
1317
|
const age = Math.floor((now - (row?.createdAtMs ?? now)) / 1000)
|
|
1194
1318
|
log(`worker-feed: ABSOLUTE cap reap agent=${agentId} — row age ${age}s (>= ${Math.floor(absoluteRowLifetimeCapMs / 1000)}s); force-terminating immortal row (survives lastUpdateAt reset)`)
|
|
1195
1319
|
} else {
|
|
1196
1320
|
log(`worker-feed: TTL reap agent=${agentId} — no update in ${Math.floor((now - (row?.lastUpdateAt ?? now)) / 1000)}s (>= ${Math.floor(staleWorkerTtlMs / 1000)}s); force-terminating leaked row`)
|
|
1197
1321
|
}
|
|
1198
|
-
|
|
1322
|
+
// Normal honest finalize when the agentIndex still points at THIS group
|
|
1323
|
+
// (renders the terminal recap + releases the pin through the chain). But
|
|
1324
|
+
// when the index has desynced — it resolves to a different group or none
|
|
1325
|
+
// — `terminateWorker` would look up the wrong/no row and no-op, leaving
|
|
1326
|
+
// this row to churn every heartbeat forever. Detect that and force-evict
|
|
1327
|
+
// the leaked row directly from the group it actually lives in.
|
|
1328
|
+
if (groupOfAgent(agentId) === g) {
|
|
1329
|
+
void terminateWorker(agentId)
|
|
1330
|
+
} else {
|
|
1331
|
+
log(`worker-feed: force-evicting leaked row agent=${agentId} feed=${g.feedKey} — agentIndex desynced (points elsewhere); removing directly`)
|
|
1332
|
+
markFinalized(agentId)
|
|
1333
|
+
evictRowFromGroup(g, agentId)
|
|
1334
|
+
syncPin(g)
|
|
1335
|
+
}
|
|
1199
1336
|
}
|
|
1200
1337
|
for (const g of [...groups.values()]) {
|
|
1201
1338
|
// Deferred-finalize re-drive: a terminal edit that hit a cooldown/flood
|
|
@@ -1242,6 +1379,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1242
1379
|
g.messageId = null
|
|
1243
1380
|
g.messageCreatedAtMs = 0
|
|
1244
1381
|
g.lastBody = null
|
|
1382
|
+
g.lastSubstanceKey = null
|
|
1245
1383
|
// Clear the (possibly stale) pin claim for the retired message; the
|
|
1246
1384
|
// fresh first-paint below re-pins the new one from a null claim.
|
|
1247
1385
|
syncPin(g)
|
|
@@ -1281,6 +1419,15 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1281
1419
|
|
|
1282
1420
|
heartbeatTimer = setIntervalFn(heartbeatTick, heartbeatTickMs)
|
|
1283
1421
|
|
|
1422
|
+
// TEST-ONLY: hand out the narrow index-repoint control (see opts doc). Never
|
|
1423
|
+
// provided in production wiring, so this is a no-op there.
|
|
1424
|
+
opts.exposeTestControls?.({
|
|
1425
|
+
repointAgentIndex: (agentId, feedKey) => {
|
|
1426
|
+
if (feedKey == null) agentIndex.delete(agentId)
|
|
1427
|
+
else agentIndex.set(agentId, feedKey)
|
|
1428
|
+
},
|
|
1429
|
+
})
|
|
1430
|
+
|
|
1284
1431
|
return {
|
|
1285
1432
|
has(agentId) {
|
|
1286
1433
|
const g = groupOfAgent(agentId)
|
|
@@ -1313,6 +1460,20 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1313
1460
|
if (existingRow?.finished === true) return Promise.resolve()
|
|
1314
1461
|
|
|
1315
1462
|
const feedKey = feedKeyOf(chatId, threadId)
|
|
1463
|
+
// Feed-migration cleanup (root-cause of the "no update in 0s" churn): if
|
|
1464
|
+
// this agentId is already indexed to a DIFFERENT feed, its prior row is
|
|
1465
|
+
// about to be orphaned (we re-index below to the new feed, but the old
|
|
1466
|
+
// group's `workers` map still holds the stale row — invisible to
|
|
1467
|
+
// `groupOfAgent` forever after, so the TTL sweep can never evict it and
|
|
1468
|
+
// it churns every heartbeat). Evict the stale row from its old group now.
|
|
1469
|
+
const priorFeedKey = agentIndex.get(agentId)
|
|
1470
|
+
if (priorFeedKey != null && priorFeedKey !== feedKey) {
|
|
1471
|
+
const priorGroup = groups.get(priorFeedKey)
|
|
1472
|
+
if (priorGroup != null) {
|
|
1473
|
+
evictRowFromGroup(priorGroup, agentId)
|
|
1474
|
+
syncPin(priorGroup)
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1316
1477
|
let g = groups.get(feedKey)
|
|
1317
1478
|
if (g == null) {
|
|
1318
1479
|
g = {
|
|
@@ -1322,6 +1483,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1322
1483
|
messageId: null,
|
|
1323
1484
|
messageCreatedAtMs: 0,
|
|
1324
1485
|
lastBody: null,
|
|
1486
|
+
lastSubstanceKey: null,
|
|
1325
1487
|
lastEditAt: 0,
|
|
1326
1488
|
cooldownUntil: 0,
|
|
1327
1489
|
chain: Promise.resolve(),
|
|
@@ -1342,6 +1504,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1342
1504
|
g.messageId = null
|
|
1343
1505
|
g.messageCreatedAtMs = 0
|
|
1344
1506
|
g.lastBody = null
|
|
1507
|
+
g.lastSubstanceKey = null
|
|
1345
1508
|
g.pendingFinalize.clear()
|
|
1346
1509
|
g.terminalPainted = false
|
|
1347
1510
|
// The stale terminal message is no longer this group's pinned surface.
|