switchroom 0.18.21 → 0.18.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/switchroom.js +1 -1
- package/dist/host-control/main.js +1 -1
- package/package.json +1 -1
- package/telegram-plugin/dist/gateway/gateway.js +227 -59
- package/telegram-plugin/flushed-turn-supersede.ts +230 -0
- package/telegram-plugin/gateway/gateway.ts +69 -0
- package/telegram-plugin/tests/flushed-turn-supersede.test.ts +206 -0
- package/telegram-plugin/tests/turn-flush-safety.test.ts +71 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +13 -8
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +196 -0
- package/telegram-plugin/turn-flush-safety.ts +74 -1
- package/telegram-plugin/worker-activity-feed.ts +155 -45
|
@@ -443,6 +443,202 @@ describe('coalesced worker feed — GROUP-level pin lifecycle (#3207 review)', (
|
|
|
443
443
|
})
|
|
444
444
|
})
|
|
445
445
|
|
|
446
|
+
// ─── #3207 leaked-finished-row fix ────────────────────────────────────────────
|
|
447
|
+
//
|
|
448
|
+
// Root cause of the "progress card never unpins; later workers append to a
|
|
449
|
+
// stale pinned message" incident (a card observed pinned + edited for 2h12m):
|
|
450
|
+
// a finished worker row whose terminal edit hit a 429/flood window was left in
|
|
451
|
+
// `g.workers`, so the group never emptied → never deleted → never unpinned, and
|
|
452
|
+
// a later worker inherited the stale message. These outcome tests pin the fix:
|
|
453
|
+
// group emptiness / deletion / unpin are decoupled from terminal-edit success,
|
|
454
|
+
// the finalize staging is per-agent (no single-slot overwrite), and a later
|
|
455
|
+
// worker never reuses a terminated group's message.
|
|
456
|
+
|
|
457
|
+
describe('coalesced worker feed — leaked finished-row fix (#3207)', () => {
|
|
458
|
+
interface PinCall {
|
|
459
|
+
feedKey: string
|
|
460
|
+
chatId: string
|
|
461
|
+
messageId: number | null
|
|
462
|
+
}
|
|
463
|
+
function leakHarness(opts: { failEdits?: () => unknown } = {}) {
|
|
464
|
+
const pins: PinCall[] = []
|
|
465
|
+
const sent: { chatId: string; text: string; messageId: number }[] = []
|
|
466
|
+
const edits: { messageId: number; text: string }[] = []
|
|
467
|
+
let seq = 700
|
|
468
|
+
let clock = 0
|
|
469
|
+
const bot: BotApiForWorkerFeed = {
|
|
470
|
+
sendMessage: async (chatId, text) => {
|
|
471
|
+
const messageId = seq++
|
|
472
|
+
sent.push({ chatId, text, messageId })
|
|
473
|
+
return { message_id: messageId }
|
|
474
|
+
},
|
|
475
|
+
editMessageText: async (_chatId, messageId, text) => {
|
|
476
|
+
const err = opts.failEdits?.()
|
|
477
|
+
if (err != null) throw err
|
|
478
|
+
edits.push({ messageId, text })
|
|
479
|
+
return true
|
|
480
|
+
},
|
|
481
|
+
}
|
|
482
|
+
const feed = createWorkerActivityFeed({
|
|
483
|
+
bot,
|
|
484
|
+
now: () => clock,
|
|
485
|
+
minEditIntervalMs: 0,
|
|
486
|
+
firstPaintMinMs: 0,
|
|
487
|
+
heartbeatTickMs: 6000,
|
|
488
|
+
setInterval: () => 0,
|
|
489
|
+
clearInterval: () => {},
|
|
490
|
+
reconcilePin: ({ feedKey, chatId, messageId }) => pins.push({ feedKey, chatId, messageId }),
|
|
491
|
+
})
|
|
492
|
+
return { feed, pins, sent, edits, setClock: (t: number) => (clock = t) }
|
|
493
|
+
}
|
|
494
|
+
const done = (desc: string, elapsedMs: number): WorkerActivityView => ({
|
|
495
|
+
description: desc,
|
|
496
|
+
lastTool: null,
|
|
497
|
+
toolCount: 3,
|
|
498
|
+
latestSummary: `${desc} result`,
|
|
499
|
+
elapsedMs,
|
|
500
|
+
state: 'done',
|
|
501
|
+
})
|
|
502
|
+
const drain = () => new Promise((r) => setTimeout(r, 0))
|
|
503
|
+
|
|
504
|
+
it('drops the finished row + requests unpin even when the last terminal edit floods (429)', async () => {
|
|
505
|
+
let flooding = true
|
|
506
|
+
const { feed, pins, sent, edits, setClock } = leakHarness({
|
|
507
|
+
failEdits: () => (flooding ? { error_code: 429, parameters: { retry_after: 2 } } : null),
|
|
508
|
+
})
|
|
509
|
+
setClock(1000)
|
|
510
|
+
await feed.update('w1', 'chat', view('task 1', 'doing', 1000))
|
|
511
|
+
expect(sent).toHaveLength(1)
|
|
512
|
+
const feedKey = pins[pins.length - 1].feedKey
|
|
513
|
+
|
|
514
|
+
// The LAST worker finishes but the terminal recap edit 429s.
|
|
515
|
+
setClock(2000)
|
|
516
|
+
await feed.finish('w1', done('task 1', 2000))
|
|
517
|
+
await drain()
|
|
518
|
+
|
|
519
|
+
// Outcome: the finished row is gone and the pin is RELEASED immediately —
|
|
520
|
+
// neither is blocked by the failed terminal edit.
|
|
521
|
+
expect(feed.has('w1')).toBe(false)
|
|
522
|
+
expect(feed.size).toBe(0)
|
|
523
|
+
// Pin-reaper backstop: no running rows remain → the group is not "running".
|
|
524
|
+
expect(feed.hasRunningInFeed(feedKey)).toBe(false)
|
|
525
|
+
expect(pins[pins.length - 1].messageId).toBeNull()
|
|
526
|
+
// The terminal recap has NOT landed yet (still flooding) — proves the unpin
|
|
527
|
+
// did not wait on it.
|
|
528
|
+
expect(edits.some((e) => e.text.includes('_done ·'))).toBe(false)
|
|
529
|
+
|
|
530
|
+
// Flood clears; a heartbeat past the cooldown re-drives the staged recap.
|
|
531
|
+
flooding = false
|
|
532
|
+
setClock(5000)
|
|
533
|
+
feed.heartbeatTick()
|
|
534
|
+
await drain()
|
|
535
|
+
expect(edits.some((e) => e.text.includes('_done ·'))).toBe(true)
|
|
536
|
+
|
|
537
|
+
// The group is fully reaped: a later worker in the SAME chat paints a FRESH
|
|
538
|
+
// message — it does NOT reuse/edit the terminated group's message id.
|
|
539
|
+
setClock(6000)
|
|
540
|
+
await feed.update('w2', 'chat', view('task 2', 'doing2', 6000))
|
|
541
|
+
await drain()
|
|
542
|
+
expect(sent).toHaveLength(2)
|
|
543
|
+
expect(feed.messageIdOf('w2')).toBe(sent[1].messageId)
|
|
544
|
+
expect(feed.messageIdOf('w2')).not.toBe(sent[0].messageId)
|
|
545
|
+
})
|
|
546
|
+
|
|
547
|
+
it('TWO workers finishing in the same flood window are BOTH reaped (no single-slot overwrite)', async () => {
|
|
548
|
+
let flooding = true
|
|
549
|
+
const { feed, pins, sent, edits, setClock } = leakHarness({
|
|
550
|
+
failEdits: () => (flooding ? { error_code: 429, parameters: { retry_after: 2 } } : null),
|
|
551
|
+
})
|
|
552
|
+
setClock(1000)
|
|
553
|
+
await feed.update('a', 'chat', view('task A', 'a-doing', 1000))
|
|
554
|
+
await feed.update('b', 'chat', view('task B', 'b-doing', 1000))
|
|
555
|
+
expect(feed.size).toBe(2)
|
|
556
|
+
const feedKey = pins[pins.length - 1].feedKey
|
|
557
|
+
|
|
558
|
+
// Both finish before either finalize chain drains, so both latch terminal in
|
|
559
|
+
// the SAME cooldown window. The old single `pendingFinalize` slot dropped all
|
|
560
|
+
// but one → the other row leaked forever. The per-agent map keeps both.
|
|
561
|
+
setClock(2000)
|
|
562
|
+
const f1 = feed.finish('a', done('task A', 2000))
|
|
563
|
+
const f2 = feed.finish('b', done('task B', 2000))
|
|
564
|
+
await Promise.all([f1, f2])
|
|
565
|
+
await drain()
|
|
566
|
+
|
|
567
|
+
// NEITHER finished row leaks — the group is empty and unpinned.
|
|
568
|
+
expect(feed.size).toBe(0)
|
|
569
|
+
expect(feed.hasRunningInFeed(feedKey)).toBe(false)
|
|
570
|
+
expect(pins[pins.length - 1].messageId).toBeNull()
|
|
571
|
+
|
|
572
|
+
// Flood clears; one heartbeat drains ALL staged recaps and reaps the group.
|
|
573
|
+
flooding = false
|
|
574
|
+
setClock(5000)
|
|
575
|
+
feed.heartbeatTick()
|
|
576
|
+
await drain()
|
|
577
|
+
|
|
578
|
+
// BOTH staged terminal recaps must actually repaint — not just one. This is
|
|
579
|
+
// the contract the per-agent `pendingFinalize` map exists to hold: a single-
|
|
580
|
+
// slot `pendingFinalize` would have let worker B's stage overwrite worker A's,
|
|
581
|
+
// so only ONE recap edit would land and this assertion would FAIL. The
|
|
582
|
+
// decoupled row-removal (fix 1) alone would pass size===0 + unpin above WITHOUT
|
|
583
|
+
// repainting either recap, so those assertions do not pin this contract — these
|
|
584
|
+
// do. Each recap carries its own worker's description ('task A' / 'task B').
|
|
585
|
+
const doneEdits = edits.filter((e) => e.text.includes('_done ·'))
|
|
586
|
+
expect(doneEdits.some((e) => e.text.includes('task A'))).toBe(true)
|
|
587
|
+
expect(doneEdits.some((e) => e.text.includes('task B'))).toBe(true)
|
|
588
|
+
|
|
589
|
+
// The group is gone: a later worker paints a FRESH message, never editing a
|
|
590
|
+
// terminated sibling's message.
|
|
591
|
+
setClock(6000)
|
|
592
|
+
await feed.update('c', 'chat', view('task C', 'c-doing', 6000))
|
|
593
|
+
await drain()
|
|
594
|
+
const firstMsgId = sent[0].messageId
|
|
595
|
+
expect(feed.messageIdOf('c')).toBe(sent[sent.length - 1].messageId)
|
|
596
|
+
expect(feed.messageIdOf('c')).not.toBe(firstMsgId)
|
|
597
|
+
})
|
|
598
|
+
|
|
599
|
+
it('a later worker after all prior workers finished does NOT reuse the prior message id', async () => {
|
|
600
|
+
const { feed, sent, setClock } = leakHarness()
|
|
601
|
+
setClock(1000)
|
|
602
|
+
await feed.update('w1', 'chat', view('task 1', 'doing', 1000))
|
|
603
|
+
setClock(2000)
|
|
604
|
+
await feed.finish('w1', done('task 1', 2000))
|
|
605
|
+
await drain()
|
|
606
|
+
expect(feed.size).toBe(0)
|
|
607
|
+
|
|
608
|
+
// A brand-new worker in the same chat gets its OWN fresh message.
|
|
609
|
+
setClock(3000)
|
|
610
|
+
await feed.update('w2', 'chat', view('task 2', 'doing2', 3000))
|
|
611
|
+
await drain()
|
|
612
|
+
expect(sent).toHaveLength(2)
|
|
613
|
+
expect(feed.messageIdOf('w2')).toBe(sent[1].messageId)
|
|
614
|
+
expect(feed.messageIdOf('w2')).not.toBe(sent[0].messageId)
|
|
615
|
+
})
|
|
616
|
+
|
|
617
|
+
it('hasRunningInFeed is false once only finished rows remain (pin-reaper backstop fires)', async () => {
|
|
618
|
+
// A sibling keeps the group running; when the LAST running worker finishes,
|
|
619
|
+
// no running row remains → hasRunningInFeed flips to false so the gateway's
|
|
620
|
+
// wk:group: reaper is free to unpin (finished rows never count as running).
|
|
621
|
+
const { feed, pins, setClock } = leakHarness()
|
|
622
|
+
setClock(1000)
|
|
623
|
+
await feed.update('a', 'chat', view('task A', 'a-doing', 1000))
|
|
624
|
+
await feed.update('b', 'chat', view('task B', 'b-doing', 1000))
|
|
625
|
+
const feedKey = pins[pins.length - 1].feedKey
|
|
626
|
+
expect(feed.hasRunningInFeed(feedKey)).toBe(true)
|
|
627
|
+
|
|
628
|
+
setClock(2000)
|
|
629
|
+
await feed.finish('a', done('task A', 2000))
|
|
630
|
+
await drain()
|
|
631
|
+
// One still runs → still counts.
|
|
632
|
+
expect(feed.hasRunningInFeed(feedKey)).toBe(true)
|
|
633
|
+
|
|
634
|
+
setClock(3000)
|
|
635
|
+
await feed.finish('b', done('task B', 3000))
|
|
636
|
+
await drain()
|
|
637
|
+
// Only finished work remains → NOT running.
|
|
638
|
+
expect(feed.hasRunningInFeed(feedKey)).toBe(false)
|
|
639
|
+
})
|
|
640
|
+
})
|
|
641
|
+
|
|
446
642
|
describe('renderCombinedWorkerFeed (pure)', () => {
|
|
447
643
|
const row = (i: number, step: string) => ({
|
|
448
644
|
description: `task number ${i}`,
|
|
@@ -128,6 +128,75 @@ export function endsWithSilentMarker(text: string | undefined): boolean {
|
|
|
128
128
|
return isSilentFlushMarker(lines[lines.length - 1])
|
|
129
129
|
}
|
|
130
130
|
|
|
131
|
+
/**
|
|
132
|
+
* Substantive-answer floor (chars, trimmed). Mirrors
|
|
133
|
+
* `final-answer-detect.ts` `FINAL_ANSWER_MIN_CHARS` and
|
|
134
|
+
* `hooks/silent-end-scan.mjs` — the same bar the codebase uses everywhere to
|
|
135
|
+
* recognise "this text block is a real answer, not a short narration/closer".
|
|
136
|
+
*/
|
|
137
|
+
export const FLUSH_SUBSTANTIVE_MIN_CHARS = 200
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Pick the text a turn-flush should actually DELIVER from the captured
|
|
141
|
+
* assistant blocks.
|
|
142
|
+
*
|
|
143
|
+
* The duplicate-reply bug (2026-07) had the answer-ready flush fire while the
|
|
144
|
+
* model was still composing its `reply` tool call: `capturedText` at that
|
|
145
|
+
* moment holds intent-narration blocks ("Let me check X", "I'll now …") FOLLOWED
|
|
146
|
+
* by the composed answer block. Flushing `capturedText.join('\n\n')` dumped the
|
|
147
|
+
* whole narration+answer blob into chat — which then never matched the clean
|
|
148
|
+
* answer-only `reply` on the outbound dedup (containment, not equality), so the
|
|
149
|
+
* user got a narration-laden duplicate.
|
|
150
|
+
*
|
|
151
|
+
* The answer is the TERMINAL block — the last thing the model wrote is what it
|
|
152
|
+
* settled on, REGARDLESS of length. A real dropped answer is often short
|
|
153
|
+
* (a one-line confirmation, a two-sentence reply), so gating delivery on the
|
|
154
|
+
* 200-char substantive floor was wrong: with blocks
|
|
155
|
+
* `[verboseNarration(250), realAnswer(150)]` a reversed length scan returns the
|
|
156
|
+
* 250-char narration and DROPS the 150-char real answer. Instead we take the
|
|
157
|
+
* last non-empty block as the answer and strip only the EARLIER blocks — and
|
|
158
|
+
* only when they look like intent-narration (short, or the classic "Let me…" /
|
|
159
|
+
* "I'll…" openers). If the earlier blocks are themselves substantial (a genuine
|
|
160
|
+
* multi-paragraph answer written as several blocks) we keep the whole thing
|
|
161
|
+
* joined, so we never truncate a real long answer down to its last paragraph.
|
|
162
|
+
*
|
|
163
|
+
* `blocks` are already trimmed/non-empty candidates (silent markers removed by
|
|
164
|
+
* the caller's guards). Returns the chosen delivery text.
|
|
165
|
+
*/
|
|
166
|
+
export function selectFlushDeliveryText(blocks: string[]): string {
|
|
167
|
+
const candidates = blocks
|
|
168
|
+
.map(b => b.trim())
|
|
169
|
+
.filter(b => b.length > 0)
|
|
170
|
+
if (candidates.length === 0) return ''
|
|
171
|
+
if (candidates.length === 1) return candidates[0]
|
|
172
|
+
const answer = candidates[candidates.length - 1]
|
|
173
|
+
const preceding = candidates.slice(0, -1)
|
|
174
|
+
// Deliver only the terminal answer when every earlier block is
|
|
175
|
+
// intent-narration (a short block, or a "Let me…/I'll…/I'm going to…" opener).
|
|
176
|
+
// Otherwise the earlier blocks carry real content — keep the full joined text
|
|
177
|
+
// so a legitimate multi-block answer is never truncated to its last paragraph.
|
|
178
|
+
const allNarration = preceding.every(isNarrationBlock)
|
|
179
|
+
return allNarration ? answer : candidates.join('\n\n')
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Narration heuristic: a block that opens with a first-person "about to do X"
|
|
184
|
+
* phrase the model emits BEFORE composing its real answer ("Let me check…",
|
|
185
|
+
* "I'll look it up…", "Now let me…"). Deliberately NOT length-based — the
|
|
186
|
+
* narration that shadowed the real answer in the observed bug was a LONG
|
|
187
|
+
* (≥200-char) "Let me pull the numbers…" block, and short blocks are frequently
|
|
188
|
+
* legitimate multi-paragraph answer content — so gating on length either drops a
|
|
189
|
+
* short real answer or keeps a long narration. When earlier blocks don't match
|
|
190
|
+
* this opener we keep the full joined text (never drop content we can't
|
|
191
|
+
* confidently attribute to narration).
|
|
192
|
+
*/
|
|
193
|
+
const NARRATION_OPENER =
|
|
194
|
+
/^(let me\b|lemme\b|i'?ll\b|i will\b|i am going to\b|i'?m going to\b|i'?m about to\b|going to\b|first,?\s+(?:let me|i'?ll|i will)\b|now,?\s+(?:let me|i'?ll|i will)\b|next,?\s+(?:let me|i'?ll|i will)\b|let'?s\b)/i
|
|
195
|
+
|
|
196
|
+
function isNarrationBlock(block: string): boolean {
|
|
197
|
+
return NARRATION_OPENER.test(block.trimStart())
|
|
198
|
+
}
|
|
199
|
+
|
|
131
200
|
export type FlushDecision =
|
|
132
201
|
| { kind: 'flush'; text: string }
|
|
133
202
|
| { kind: 'skip'; reason: FlushSkipReason }
|
|
@@ -219,7 +288,11 @@ export function decideTurnFlush(input: FlushDecisionInput): FlushDecision {
|
|
|
219
288
|
// sentinel — treat the whole turn as intentionally silent rather than
|
|
220
289
|
// flush the prose with the sentinel glued on.
|
|
221
290
|
if (endsWithSilentMarker(joined)) return { kind: 'skip', reason: 'silent-marker' }
|
|
222
|
-
|
|
291
|
+
// Deliver only the substantive answer block, never the whole narration+answer
|
|
292
|
+
// blob (see `selectFlushDeliveryText`). The silent-marker / empty guards above
|
|
293
|
+
// still run on the full `joined` string so a partly-silent turn is classified
|
|
294
|
+
// correctly; only the DELIVERED text is narrowed to the answer.
|
|
295
|
+
return { kind: 'flush', text: selectFlushDeliveryText(input.capturedText) }
|
|
223
296
|
}
|
|
224
297
|
|
|
225
298
|
/**
|
|
@@ -390,12 +390,29 @@ interface FeedGroup {
|
|
|
390
390
|
/** Live workers in this group, keyed by agentId (insertion ≈ dispatch order). */
|
|
391
391
|
workers: Map<string, WorkerRow>
|
|
392
392
|
/**
|
|
393
|
-
*
|
|
394
|
-
* flood window blocked the edit.
|
|
395
|
-
*
|
|
396
|
-
*
|
|
393
|
+
* Terminal renders (per finishing worker's recap) staged because a 429
|
|
394
|
+
* cooldown / flood window blocked the edit. Keyed by agentId so a SECOND
|
|
395
|
+
* worker finishing in the same window can't overwrite the first's staged
|
|
396
|
+
* recap (the single-slot bug: two near-simultaneous last-worker finishes
|
|
397
|
+
* leaked the earlier finished row forever). The heartbeat drains EVERY entry
|
|
398
|
+
* once the cooldown expires so a finished feed can't get stuck on its last
|
|
399
|
+
* running render. Empty when no finalize is pending.
|
|
400
|
+
*
|
|
401
|
+
* Note: the finishing row is dropped from `workers` immediately at finalize
|
|
402
|
+
* time (decoupled from edit success — #3207 leaked-finished-row fix), so a
|
|
403
|
+
* staged entry here is purely the best-effort terminal REPAINT; group
|
|
404
|
+
* emptiness / deletion / unpin never wait on it.
|
|
405
|
+
*/
|
|
406
|
+
pendingFinalize: Map<string, WorkerActivityView>
|
|
407
|
+
/**
|
|
408
|
+
* True once this group's shared message last rendered a TERMINAL recap (a
|
|
409
|
+
* finished worker's done/failed/incomplete card), and no live worker has
|
|
410
|
+
* repainted since. A later worker joining the group must NOT edit that
|
|
411
|
+
* terminal message — it forces a fresh first-paint (new message) instead, so
|
|
412
|
+
* a new dispatch never appends to a stale "done" card. Reset to false on any
|
|
413
|
+
* fresh first-paint.
|
|
397
414
|
*/
|
|
398
|
-
|
|
415
|
+
terminalPainted: boolean
|
|
399
416
|
}
|
|
400
417
|
|
|
401
418
|
const COOLDOWN_JITTER_MS = 500
|
|
@@ -693,7 +710,24 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
693
710
|
function removeWorker(g: FeedGroup, agentId: string): void {
|
|
694
711
|
g.workers.delete(agentId)
|
|
695
712
|
agentIndex.delete(agentId)
|
|
696
|
-
|
|
713
|
+
maybeDeleteGroup(g)
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
/**
|
|
717
|
+
* Delete an empty group. A group is gone only when it has NO tracked workers
|
|
718
|
+
* AND no staged terminal repaints pending — the latter keeps the group object
|
|
719
|
+
* reachable for the heartbeat re-drive after a flood/429 deferred the recap,
|
|
720
|
+
* without keeping it "alive" for the pin (syncPin / hasRunningInFeed count
|
|
721
|
+
* only live workers, so an emptied-but-pending group is already unpinned).
|
|
722
|
+
*/
|
|
723
|
+
function maybeDeleteGroup(g: FeedGroup): void {
|
|
724
|
+
if (g.workers.size === 0 && g.pendingFinalize.size === 0) groups.delete(g.feedKey)
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
/** Live (not-yet-finished) workers still tracked in a group. */
|
|
728
|
+
function hasLiveWorker(g: FeedGroup): boolean {
|
|
729
|
+
for (const w of g.workers.values()) if (!w.finished) return true
|
|
730
|
+
return false
|
|
697
731
|
}
|
|
698
732
|
|
|
699
733
|
/**
|
|
@@ -704,7 +738,10 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
704
738
|
* still needs — the survivors keep the pin until the LAST worker is done.
|
|
705
739
|
*/
|
|
706
740
|
function syncPin(g: FeedGroup): void {
|
|
707
|
-
|
|
741
|
+
// Pin follows LIVE membership, not row count: a finished-but-not-yet-swept
|
|
742
|
+
// row (or a staged terminal repaint) must NOT keep the pin. Unpin the
|
|
743
|
+
// instant the last running worker is gone.
|
|
744
|
+
const messageId = g.messageId != null && hasLiveWorker(g) ? g.messageId : null
|
|
708
745
|
reconcilePinFn({ feedKey: g.feedKey, chatId: g.chatId, threadId: g.threadId, messageId })
|
|
709
746
|
}
|
|
710
747
|
|
|
@@ -721,45 +758,78 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
721
758
|
): Promise<void> {
|
|
722
759
|
const now = nowFn()
|
|
723
760
|
const isTerminal = opts2.terminalRecap != null
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
// the
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
761
|
+
const finishingAgentId = opts2.finishingAgentId
|
|
762
|
+
|
|
763
|
+
// Stage the terminal recap for the heartbeat re-drive, keyed by the
|
|
764
|
+
// FINISHING agent so a second worker finishing in the same cooldown/flood
|
|
765
|
+
// window can't overwrite an earlier staged recap (#3207: the single-slot
|
|
766
|
+
// overwrite that orphaned finished rows). Best-effort repaint only — the
|
|
767
|
+
// row is dropped and the pin released independently, below.
|
|
768
|
+
const stageRecap = (): void => {
|
|
769
|
+
if (isTerminal && finishingAgentId != null && opts2.terminalRecap != null) {
|
|
770
|
+
g.pendingFinalize.set(finishingAgentId, opts2.terminalRecap)
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
// Terminal repaint no longer pending (landed / not-modified / gone / never
|
|
774
|
+
// had a message): stop re-driving it and reap the group if now fully empty.
|
|
775
|
+
const clearStaged = (): void => {
|
|
776
|
+
if (finishingAgentId != null) g.pendingFinalize.delete(finishingAgentId)
|
|
777
|
+
maybeDeleteGroup(g)
|
|
778
|
+
syncPin(g)
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
// #3207 leaked-finished-row fix — decouple row removal + unpin from the
|
|
782
|
+
// terminal edit's SUCCESS. The instant a worker finalizes we drop its row
|
|
783
|
+
// and release the group pin; a 429 / flood / transport failure on the
|
|
784
|
+
// cosmetic recap edit must NEVER keep the group, its message, or its pin
|
|
785
|
+
// alive. The recap is staged (above) purely as a best-effort repaint.
|
|
786
|
+
if (isTerminal) {
|
|
787
|
+
stageRecap()
|
|
788
|
+
if (finishingAgentId != null && g.workers.has(finishingAgentId)) {
|
|
789
|
+
removeWorker(g, finishingAgentId)
|
|
790
|
+
}
|
|
791
|
+
// Only latch terminalPainted when NO live worker remains at paint time.
|
|
792
|
+
// Race: this finalize was the last live worker when the chain latched, but
|
|
793
|
+
// a fresh worker B may have called update() and joined g.workers before
|
|
794
|
+
// this body runs — in which case renderGroupBody paints B's RUNNING card,
|
|
795
|
+
// not a terminal recap. Setting the flag true unconditionally would
|
|
796
|
+
// mislabel that running paint as terminal (a spurious group-reuse fresh-
|
|
797
|
+
// paint on B's next update). Reflect reality: the group only went terminal
|
|
798
|
+
// if it's actually empty of live workers.
|
|
799
|
+
g.terminalPainted = !hasLiveWorker(g)
|
|
732
800
|
syncPin(g)
|
|
733
801
|
}
|
|
802
|
+
|
|
734
803
|
if (now < g.cooldownUntil) {
|
|
735
|
-
|
|
804
|
+
// Row already dropped + unpinned above; the recap stays staged so the
|
|
805
|
+
// heartbeat re-drives the repaint once the cooldown expires.
|
|
736
806
|
return
|
|
737
807
|
}
|
|
738
808
|
// A flood window is open: the gate would SHED every call. Park in cooldown
|
|
739
809
|
// and make ZERO api calls until it closes; the heartbeat re-drives.
|
|
740
810
|
if (parkIfFloodWindowOpen(g)) {
|
|
741
|
-
if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
|
|
742
811
|
return
|
|
743
812
|
}
|
|
744
813
|
|
|
745
814
|
const body = renderGroupBody(g, now, opts2.terminalRecap ?? null, opts2.heartbeat ?? false)
|
|
746
815
|
if (body == null) {
|
|
747
|
-
// Nothing to show. On a terminal finalize
|
|
748
|
-
//
|
|
749
|
-
if (isTerminal)
|
|
816
|
+
// Nothing to show. On a terminal finalize the row is already dropped;
|
|
817
|
+
// clear the staged repaint (the handback carries the result).
|
|
818
|
+
if (isTerminal) clearStaged()
|
|
750
819
|
return
|
|
751
820
|
}
|
|
752
821
|
|
|
753
822
|
// First paint: hold until some worker in the group has run long enough.
|
|
754
823
|
if (g.messageId == null) {
|
|
755
|
-
const maxElapsed = Math.max(0, ...runningRows(g).map((r) => liveElapsed(r, now)))
|
|
756
824
|
// A terminal recap for a group that never painted → nothing to finalize;
|
|
757
825
|
// never first-paint a terminal card (trivial workers stay silent, the
|
|
758
826
|
// handback carries the result — matches the pre-coalesce doFinish guard).
|
|
827
|
+
// The finished row is already dropped; just clear the staged repaint.
|
|
759
828
|
if (isTerminal) {
|
|
760
|
-
|
|
829
|
+
clearStaged()
|
|
761
830
|
return
|
|
762
831
|
}
|
|
832
|
+
const maxElapsed = Math.max(0, ...runningRows(g).map((r) => liveElapsed(r, now)))
|
|
763
833
|
if (maxElapsed < firstPaintMin) return
|
|
764
834
|
try {
|
|
765
835
|
const sent = await opts.bot.sendMessage(g.chatId, body, sendOptsFor(g))
|
|
@@ -775,6 +845,8 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
775
845
|
g.messageId = sent.message_id
|
|
776
846
|
g.lastBody = body
|
|
777
847
|
g.lastEditAt = now
|
|
848
|
+
// A fresh message is a live running paint, never a terminal recap.
|
|
849
|
+
g.terminalPainted = false
|
|
778
850
|
// Group's first (or re-established) message is up → pin it for the group.
|
|
779
851
|
syncPin(g)
|
|
780
852
|
log(
|
|
@@ -790,7 +862,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
790
862
|
|
|
791
863
|
// Dedup + proactive throttle (finish/terminal edits force through).
|
|
792
864
|
if (body === g.lastBody) {
|
|
793
|
-
if (isTerminal)
|
|
865
|
+
if (isTerminal) clearStaged()
|
|
794
866
|
return
|
|
795
867
|
}
|
|
796
868
|
if (!opts2.force && now - g.lastEditAt < minEditInterval) return
|
|
@@ -803,7 +875,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
803
875
|
// The shed payload is NOT on screen, so do not record it as `lastBody`.
|
|
804
876
|
if (isSendGateShed(res)) {
|
|
805
877
|
parkIfFloodWindowOpen(g)
|
|
806
|
-
|
|
878
|
+
// Recap stays staged (above); row already dropped + unpinned.
|
|
807
879
|
return
|
|
808
880
|
}
|
|
809
881
|
g.lastBody = body
|
|
@@ -811,7 +883,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
811
883
|
if (isTerminal) {
|
|
812
884
|
log(
|
|
813
885
|
`worker-feed: finish feed=${g.feedKey} chat=${g.chatId} thread=${g.threadId ?? '-'} ` +
|
|
814
|
-
`msgId=${g.messageId} agent=${
|
|
886
|
+
`msgId=${g.messageId} agent=${finishingAgentId ?? '-'} ` +
|
|
815
887
|
`state=${opts2.terminalRecap?.state ?? 'done'} bytes=${body.length}`,
|
|
816
888
|
)
|
|
817
889
|
} else {
|
|
@@ -820,34 +892,34 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
820
892
|
`thread=${g.threadId ?? '-'} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`,
|
|
821
893
|
)
|
|
822
894
|
}
|
|
823
|
-
if (isTerminal)
|
|
895
|
+
if (isTerminal) clearStaged()
|
|
824
896
|
} catch (err) {
|
|
825
897
|
const outcome = classifyEditError(err)
|
|
826
898
|
if (outcome === 'rate_limited') {
|
|
827
899
|
noteRateLimited(g, err, isTerminal ? 'finish' : 'edit')
|
|
828
|
-
|
|
900
|
+
// Recap stays staged; the heartbeat re-drives after the cooldown.
|
|
829
901
|
return
|
|
830
902
|
}
|
|
831
903
|
if (outcome === 'not_modified') {
|
|
832
904
|
g.lastBody = body
|
|
833
905
|
g.lastEditAt = now
|
|
834
|
-
if (isTerminal)
|
|
906
|
+
if (isTerminal) clearStaged()
|
|
835
907
|
return
|
|
836
908
|
}
|
|
837
909
|
if (outcome === 'gone') {
|
|
838
910
|
// Message/chat gone or edit window closed — no card to update. Drop the
|
|
839
911
|
// stale message id; a fresh first-paint re-establishes one if workers
|
|
840
|
-
// are still live. On a terminal finalize,
|
|
912
|
+
// are still live. On a terminal finalize, clear the staged repaint.
|
|
841
913
|
g.messageId = null
|
|
842
914
|
g.lastBody = null
|
|
843
915
|
// The pinned message no longer exists → release the group pin claim
|
|
844
|
-
// (
|
|
845
|
-
if (isTerminal)
|
|
916
|
+
// (clearStaged already re-syncs on the terminal path).
|
|
917
|
+
if (isTerminal) clearStaged()
|
|
846
918
|
else syncPin(g)
|
|
847
919
|
return
|
|
848
920
|
}
|
|
849
921
|
// 'transient' — leave the message intact; the heartbeat re-attempts.
|
|
850
|
-
|
|
922
|
+
// Recap stays staged for the terminal path.
|
|
851
923
|
log(`worker-feed: edit transient error feed=${g.feedKey}: ${(err as Error).message}`)
|
|
852
924
|
}
|
|
853
925
|
}
|
|
@@ -945,13 +1017,27 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
945
1017
|
// mutates the group's worker map), then terminate each through its chain so
|
|
946
1018
|
// the render/unpin happens under the normal cooldown/flood guards.
|
|
947
1019
|
const staleAgentIds: string[] = []
|
|
1020
|
+
const staleFinished: Array<{ g: FeedGroup; agentId: string }> = []
|
|
948
1021
|
for (const g of groups.values()) {
|
|
949
1022
|
for (const row of g.workers.values()) {
|
|
950
|
-
if (
|
|
951
|
-
|
|
1023
|
+
if (now - row.lastUpdateAt >= staleWorkerTtlMs) {
|
|
1024
|
+
if (row.finished) staleFinished.push({ g, agentId: row.agentId })
|
|
1025
|
+
else staleAgentIds.push(row.agentId)
|
|
952
1026
|
}
|
|
953
1027
|
}
|
|
954
1028
|
}
|
|
1029
|
+
// GC any FINISHED row still lingering past the TTL (#3207: finished rows
|
|
1030
|
+
// were exempt from the sweep — `if (!row.finished …)` — so a stuck finished
|
|
1031
|
+
// row could keep a group, its message, and its pin alive forever). The
|
|
1032
|
+
// normal finalize path drops the row immediately now, but reap directly
|
|
1033
|
+
// here as a durable backstop: `terminate()` no-ops on a finished row, so
|
|
1034
|
+
// remove it and release the pin without routing through it.
|
|
1035
|
+
for (const { g, agentId } of staleFinished) {
|
|
1036
|
+
log(`worker-feed: TTL GC finished row agent=${agentId} feed=${g.feedKey} — reaping leaked finished row`)
|
|
1037
|
+
g.pendingFinalize.delete(agentId)
|
|
1038
|
+
removeWorker(g, agentId)
|
|
1039
|
+
syncPin(g)
|
|
1040
|
+
}
|
|
955
1041
|
for (const agentId of staleAgentIds) {
|
|
956
1042
|
log(`worker-feed: TTL reap agent=${agentId} — no update in ${Math.floor((now - (groupOfAgent(agentId)?.workers.get(agentId)?.lastUpdateAt ?? now)) / 1000)}s (>= ${Math.floor(staleWorkerTtlMs / 1000)}s); force-terminating leaked row`)
|
|
957
1043
|
void terminateWorker(agentId)
|
|
@@ -960,15 +1046,19 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
960
1046
|
// Deferred-finalize re-drive: a terminal edit that hit a cooldown/flood
|
|
961
1047
|
// window was staged on `pendingFinalize`. Re-drive it once the cooldown
|
|
962
1048
|
// expires so a finished feed can't get stuck on its last running render.
|
|
963
|
-
if (g.pendingFinalize
|
|
964
|
-
|
|
965
|
-
//
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
1049
|
+
if (g.pendingFinalize.size > 0 && now >= g.cooldownUntil) {
|
|
1050
|
+
// Drain EVERY staged terminal recap, keyed by its finishing agent
|
|
1051
|
+
// (#3207: the single `pendingFinalize` slot dropped all but one when
|
|
1052
|
+
// multiple workers finished inside the same cooldown/flood window, so
|
|
1053
|
+
// the earlier finished rows leaked). Each re-drive removes its own
|
|
1054
|
+
// staged entry on success and reaps the group once fully empty.
|
|
1055
|
+
for (const [agentId, recap] of [...g.pendingFinalize]) {
|
|
1056
|
+
g.chain = g.chain
|
|
1057
|
+
.then(() => doRender(g, { force: true, terminalRecap: recap, finishingAgentId: agentId }))
|
|
1058
|
+
.catch((err) => {
|
|
1059
|
+
log(`worker-feed: heartbeat finalize re-drive error feed=${g.feedKey}: ${(err as Error).message}`)
|
|
1060
|
+
})
|
|
1061
|
+
}
|
|
972
1062
|
continue
|
|
973
1063
|
}
|
|
974
1064
|
|
|
@@ -1013,7 +1103,11 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1013
1103
|
},
|
|
1014
1104
|
hasRunningInFeed(feedKey) {
|
|
1015
1105
|
const g = groups.get(feedKey)
|
|
1016
|
-
|
|
1106
|
+
// Count only RUNNING (not-yet-finished) rows (#3207): a finished row
|
|
1107
|
+
// lingering pre-sweep — or a group kept alive solely for a staged
|
|
1108
|
+
// terminal repaint — is NOT running, so the gateway's `wk:group:` pin
|
|
1109
|
+
// reaper must be free to unpin it.
|
|
1110
|
+
return g != null && hasLiveWorker(g)
|
|
1017
1111
|
},
|
|
1018
1112
|
get size() {
|
|
1019
1113
|
let n = 0
|
|
@@ -1043,10 +1137,26 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1043
1137
|
cooldownUntil: 0,
|
|
1044
1138
|
chain: Promise.resolve(),
|
|
1045
1139
|
workers: new Map(),
|
|
1046
|
-
pendingFinalize:
|
|
1140
|
+
pendingFinalize: new Map(),
|
|
1141
|
+
terminalPainted: false,
|
|
1047
1142
|
}
|
|
1048
1143
|
groups.set(feedKey, g)
|
|
1049
1144
|
}
|
|
1145
|
+
// Group-reuse-after-terminal (#3207): a later worker landing in a group
|
|
1146
|
+
// whose shared message last rendered a TERMINAL recap (its prior workers
|
|
1147
|
+
// all finished, but the group lingered — e.g. a staged repaint, or a
|
|
1148
|
+
// finished row not yet swept) must NOT inherit that message id and edit
|
|
1149
|
+
// the "done" card. Force a fresh first-paint (new message) so the new
|
|
1150
|
+
// dispatch never appends to a stale terminal card. Only triggers when no
|
|
1151
|
+
// live worker remains AND the group last went terminal.
|
|
1152
|
+
if (!g.workers.has(agentId) && g.terminalPainted && !hasLiveWorker(g)) {
|
|
1153
|
+
g.messageId = null
|
|
1154
|
+
g.lastBody = null
|
|
1155
|
+
g.pendingFinalize.clear()
|
|
1156
|
+
g.terminalPainted = false
|
|
1157
|
+
// The stale terminal message is no longer this group's pinned surface.
|
|
1158
|
+
syncPin(g)
|
|
1159
|
+
}
|
|
1050
1160
|
let row = g.workers.get(agentId)
|
|
1051
1161
|
if (row == null) {
|
|
1052
1162
|
row = {
|