switchroom 0.18.17 → 0.18.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/agent-scheduler/index.js +13 -0
  2. package/dist/auth-broker/index.js +13 -0
  3. package/dist/cli/notion-write-pretool.mjs +13 -0
  4. package/dist/cli/switchroom.js +605 -479
  5. package/dist/host-control/main.js +17 -1
  6. package/dist/vault/approvals/kernel-server.js +13 -0
  7. package/dist/vault/broker/server.js +13 -0
  8. package/package.json +1 -1
  9. package/telegram-plugin/bridge/bridge.ts +7 -1
  10. package/telegram-plugin/dist/bridge/bridge.js +26 -1
  11. package/telegram-plugin/dist/gateway/gateway.js +1401 -431
  12. package/telegram-plugin/dist/server.js +26 -1
  13. package/telegram-plugin/fleet-fallback-resume.ts +26 -3
  14. package/telegram-plugin/gateway/approval-hold.ts +49 -0
  15. package/telegram-plugin/gateway/bridge-dead-watchdog.ts +61 -18
  16. package/telegram-plugin/gateway/gateway.ts +362 -71
  17. package/telegram-plugin/gateway/linear-activity.ts +20 -4
  18. package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
  19. package/telegram-plugin/gateway/session-model-file.ts +103 -0
  20. package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
  21. package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
  22. package/telegram-plugin/llm-error-present.ts +436 -0
  23. package/telegram-plugin/operator-events.ts +7 -1
  24. package/telegram-plugin/permission-title.ts +172 -10
  25. package/telegram-plugin/premium-recovery.ts +101 -0
  26. package/telegram-plugin/raw-error-scrub.ts +73 -0
  27. package/telegram-plugin/retry-api-call.ts +8 -2
  28. package/telegram-plugin/send-gate-degraded.test.ts +152 -1
  29. package/telegram-plugin/send-gate-observability.test.ts +140 -0
  30. package/telegram-plugin/send-gate-observability.ts +65 -20
  31. package/telegram-plugin/send-gate.test.ts +143 -1
  32. package/telegram-plugin/send-gate.ts +212 -19
  33. package/telegram-plugin/session-tail.ts +16 -0
  34. package/telegram-plugin/shared/local-time.ts +69 -0
  35. package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
  36. package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
  37. package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
  38. package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
  39. package/telegram-plugin/tests/flood-windows-persistence.test.ts +3 -2
  40. package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
  41. package/telegram-plugin/tests/llm-error-present.test.ts +380 -0
  42. package/telegram-plugin/tests/permission-title.test.ts +167 -4
  43. package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
  44. package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
  45. package/telegram-plugin/tests/reaction-gate-routing.test.ts +6 -1
  46. package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
  47. package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
  48. package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
  49. package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
  50. package/telegram-plugin/tests/worker-activity-feed.test.ts +5 -2
  51. package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
  52. package/telegram-plugin/tier-downgrade.ts +198 -0
  53. package/telegram-plugin/tool-activity-summary.ts +99 -0
  54. package/telegram-plugin/worker-activity-feed.ts +509 -409
@@ -13,17 +13,34 @@
13
13
  * that grows/edits as work happens — indistinguishable from "the agent
14
14
  * is typing live", not a status widget.
15
15
  *
16
- * Pure render (`renderWorkerActivity`) + an injected bot API
17
- * (`BotApiForWorkerFeed`), mirroring `issues-card.ts` so the gateway
18
- * reuses the same wiring. The manager (`createWorkerActivityFeed`) owns
19
- * one edit-in-place message per worker, keyed by jsonl agent id, with:
16
+ * COALESCED (#3084 follow-up): all live workers dispatched to the SAME
17
+ * chat/thread render into ONE shared message (a {@link FeedGroup}), not one
18
+ * message each. With N workers, N separate messages each coalesce their own
19
+ * edit stream but ALL draw from the send gate's 1/sec per-chat bucket — so
20
+ * ~N-1 of every second's edits SHED and every card refreshes only ~once per N
21
+ * seconds (liveness collapse). One combined message = ONE per-message edit
22
+ * stream: it coalesces (last-write-wins) and never contends with siblings, so
23
+ * every worker's row refreshes together within the ~1.5s edit floor. A single-
24
+ * worker chat still renders the full 🛠 Worker card (identical to before); a
25
+ * 2+ worker chat renders `renderCombinedWorkerFeed`.
26
+ *
27
+ * Pure render (`renderWorkerActivity` / `renderCombinedWorkerFeed`) + an
28
+ * injected bot API (`BotApiForWorkerFeed`). The manager
29
+ * (`createWorkerActivityFeed`) owns one edit-in-place message per (chat,thread)
30
+ * with:
20
31
  * - a first-paint delay so trivial sub-second workers never post a
21
32
  * message (their result still lands via the handback reply),
22
33
  * - a proactive min-edit-interval throttle (worker jsonl ticks ~1/s;
23
34
  * Telegram rate-limits edits) plus body-dedup,
24
- * - per-worker serialization so two rapid ticks can't double-send,
35
+ * - per-message serialization so two rapid ticks can't double-send,
25
36
  * - 429 cooldown + message_id drift resilience (re-post on stale edit),
26
- * - a forced terminal edit on `finish` regardless of throttle.
37
+ * - a forced terminal edit on the LAST worker's `finish`.
38
+ *
39
+ * The completed-worker RESULT is NOT folded into this cosmetic feed — it
40
+ * reaches the user as its own `useful` handback reply (gateway onFinish). The
41
+ * feed shows only *live* work; a finished worker's row is dropped from the
42
+ * combined body (or, when it is the last worker, the message finalizes to its
43
+ * terminal recap).
27
44
  *
28
45
  * The feed is gated to BACKGROUND workers and is ON by default; set
29
46
  * `SWITCHROOM_WORKER_ACTIVITY_FEED=0` to disable it — see the gateway
@@ -38,7 +55,12 @@ import {
38
55
  truncate,
39
56
  } from './card-format.js'
40
57
  import { STATUS_ROLLING_LINES } from './status-no-truncate.js'
41
- import { renderStatusCard, formatStepSuffix } from './tool-activity-summary.js'
58
+ import {
59
+ renderStatusCard,
60
+ formatStepSuffix,
61
+ renderCombinedWorkerFeed,
62
+ type CombinedWorkerRow,
63
+ } from './tool-activity-summary.js'
42
64
  import { isSendGateShed } from './send-gate.js'
43
65
 
44
66
  /** Worker-activity feed is ON by default; an operator opts out with
@@ -208,7 +230,7 @@ export interface WorkerActivityFeedOpts {
208
230
  * shed send every `heartbeatTickMs` (~6s) for the WHOLE ban — thousands of
209
231
  * gate admissions, and (pre-fix) a `sent.message_id` crash on the `undefined`
210
232
  * every tick. With it, a running/first-paint tick that sees an open window
211
- * parks the handle in cooldown for the window's remaining and makes ZERO api
233
+ * parks the group in cooldown for the window's remaining and makes ZERO api
212
234
  * calls until it closes, mirroring the held-card sweep's pre-send probe.
213
235
  * Defaults to `() => 0` (no window) so tests and non-gateway callers are
214
236
  * unchanged.
@@ -222,69 +244,110 @@ export interface WorkerActivityFeedOpts {
222
244
  /** Heartbeat timer disposer. Injectable for tests. Defaults to `clearInterval`. */
223
245
  clearInterval?: (handle: unknown) => void
224
246
  /**
225
- * Heartbeat tick cadence in ms. On each tick a stale, running worker is
226
- * re-rendered with a climbing `· Ns` suffix so a worker that emits no new
227
- * narrative still visibly advances. Default 6000ms.
247
+ * Heartbeat tick cadence in ms. On each tick a stale, running feed is
248
+ * re-rendered with climbing elapsed so a worker that emits no new narrative
249
+ * still visibly advances. Default 6000ms.
228
250
  */
229
251
  heartbeatTickMs?: number
252
+ /**
253
+ * Max worker rows rendered in a COMBINED feed (2+ workers in one chat/thread)
254
+ * before the `+M more working…` spill line. Keeps the coalesced body compact
255
+ * and legible (and under the rich-message wire ceiling). Default 8. A single-
256
+ * worker chat renders the full 🛠 Worker card and ignores this. Sourced from
257
+ * `channels.telegram.worker_feed.max_rows` via the config cascade.
258
+ */
259
+ maxRows?: number
260
+ /**
261
+ * Group-level status-pin reconcile hook (#3207 review). Because workers now
262
+ * COALESCE into one shared message, the pin MUST follow the GROUP lifecycle,
263
+ * not a single worker's: pin the shared message when a group's first worker
264
+ * paints, and unpin ONLY when the group empties (its LAST worker finishes / is
265
+ * dropped). The feed alone knows group membership + running-count, so it owns
266
+ * the pin. `messageId: null` means "unpin this group". A per-worker unpin would
267
+ * physically unpin a message a SIBLING still needs, and the survivor's next
268
+ * pin request NO-OPs (its claim still names that id) — so it would run
269
+ * unpinned for the rest of its life (the shared-message NOOP trap). Best-
270
+ * effort; defaults to a noop for tests / non-gateway callers.
271
+ */
272
+ reconcilePin?: (args: {
273
+ feedKey: string
274
+ chatId: string
275
+ threadId?: number
276
+ messageId: number | null
277
+ }) => void
230
278
  }
231
279
 
232
- interface WorkerHandle {
280
+ /**
281
+ * One live worker's per-row state inside a chat/thread feed group. The row
282
+ * carries everything the render needs for THIS worker; the shared message
283
+ * (id, cooldown, chain) lives on the enclosing {@link FeedGroup}.
284
+ */
285
+ interface WorkerRow {
233
286
  /** jsonl agent id — carried so success/failure log lines can name the worker. */
234
287
  agentId: string
235
- chatId: string
236
- threadId?: number
237
- messageId: number | null
238
- lastBody: string | null
239
- lastEditAt: number
240
- cooldownUntil: number
241
288
  /**
242
- * Accumulated narrative lines (oldest→newest), deduped against the
243
- * immediately-preceding line. Rolling-window capped to STATUS_ROLLING_LINES.
244
- * Grows the live render so the feed reads like the main agent's answer.
289
+ * Accumulated narrative lines (oldest→newest), deduped within the whole
290
+ * rolling window. Rolling-window capped to STATUS_ROLLING_LINES. Grows the
291
+ * live render so the feed reads like the main agent's answer.
245
292
  */
246
293
  narrative: string[]
247
- /** Per-worker serialization chain so ticks can't interleave sends. */
248
- chain: Promise<void>
249
- /** Last view rendered into the message (drives the heartbeat re-render). */
294
+ /** Last view for this worker (drives the heartbeat re-render + combined row). */
250
295
  lastView: WorkerActivityView | null
296
+ /** Latest state observed for this worker; excluded from the running set once
297
+ * terminal (its result reaches the user via the separate handback reply). */
298
+ state: WorkerActivityState
251
299
  /**
252
- * A terminal (`finish`) view whose edit could not land yet — most often
253
- * because a 429 cooldown was in effect when `doFinish` ran. The heartbeat
254
- * re-drives `doFinish` with this view once the cooldown expires so a
255
- * transport hiccup can't leave the card stuck on its last running render
256
- * ("worker done, card says running"). Cleared on a successful terminal
257
- * edit, on a permanent failure (message gone), or when the handle is
258
- * deleted. Null when no finalize is pending.
259
- */
260
- pendingFinish: WorkerActivityView | null
261
- /**
262
- * Latched in `doFinish` before the terminal edit. A late watcher
263
- * `onProgress` tick that arrives after `finish()` queued its chain (but
264
- * before the `.finally(handles.delete)` microtask drains) must NOT
265
- * resurrect the handle and paint a fresh `running` message on an
266
- * already-finalized worker. The heartbeat's orphan-paint guard
267
- * (`if (!handles.has(h.agentId)) continue`) only covers the heartbeat
268
- * tick — this flag covers the `update` entry point. Set synchronously
269
- * inside `doFinish` (runs on the chain), checked synchronously in
270
- * `update` before handle creation.
300
+ * Latched in `finish` before the terminal edit. A late watcher `onProgress`
301
+ * tick that arrives after `finish()` queued its chain must NOT resurrect the
302
+ * row and paint a fresh `running` state on an already-finalized worker.
271
303
  */
272
304
  finished: boolean
273
305
  /**
274
306
  * Wall-clock ms the worker was dispatched, derived from `now - view.elapsedMs`
275
307
  * on the first update. The heartbeat computes a live elapsed from this so the
276
- * `· Ns` suffix climbs even when no fresh view arrives.
308
+ * elapsed climbs even when no fresh view arrives, and it fixes the worker's
309
+ * stable sort order within the combined feed.
277
310
  */
278
311
  dispatchAtMs: number | null
279
312
  /**
280
313
  * Wall-clock ms the CURRENT step started — stamped whenever a NEW narrative
281
314
  * line lands (the `→` line changes). The heartbeat's step suffix shows the
282
- * step's OWN elapsed from this anchor (not the worker total, which the
283
- * header already carries), and only once past STEP_TIMER_MIN_MS.
315
+ * step's OWN elapsed from this anchor (single-worker card only), and only
316
+ * once past STEP_TIMER_MIN_MS.
284
317
  */
285
318
  stepStartedAtMs: number | null
286
319
  }
287
320
 
321
+ /**
322
+ * One shared feed message per (chatId, threadId). ALL live workers dispatched
323
+ * to the same chat/thread render into this one message — so their edits form a
324
+ * single per-message edit stream under the send gate's 1/sec per-chat ceiling
325
+ * (last-write-wins coalescing + no-op skip) instead of N contending streams
326
+ * that shed (#3084). A single-worker group renders the full 🛠 Worker card;
327
+ * a 2+ worker group renders the combined `renderCombinedWorkerFeed` body.
328
+ */
329
+ interface FeedGroup {
330
+ /** Stable key `${chatId} ${threadId ?? ''}`. */
331
+ feedKey: string
332
+ chatId: string
333
+ threadId?: number
334
+ messageId: number | null
335
+ lastBody: string | null
336
+ lastEditAt: number
337
+ cooldownUntil: number
338
+ /** Single serialization chain for the shared message — ticks can't interleave sends. */
339
+ chain: Promise<void>
340
+ /** Live workers in this group, keyed by agentId (insertion ≈ dispatch order). */
341
+ workers: Map<string, WorkerRow>
342
+ /**
343
+ * A terminal render (the last worker's recap) staged because a 429 cooldown /
344
+ * flood window blocked the edit. The heartbeat re-drives it once the cooldown
345
+ * expires so a finished feed can't get stuck on its last running render.
346
+ * Null when no finalize is pending.
347
+ */
348
+ pendingFinalize: WorkerActivityView | null
349
+ }
350
+
288
351
  const COOLDOWN_JITTER_MS = 500
289
352
 
290
353
  function extractRetryAfterSecs(err: unknown): number | null {
@@ -307,7 +370,7 @@ function extractRetryAfterSecs(err: unknown): number | null {
307
370
  * 'rate_limited' — 429 with retry_after. Back off; the heartbeat re-drives
308
371
  * the edit after cooldown (running renders + deferred terminal edits).
309
372
  * 'gone' — message/chat deleted or edit window expired. Nothing to
310
- * update; drop the handle silently (no warning — there is no card).
373
+ * update; drop the message silently (no warning — there is no card).
311
374
  * 'transient' — anything else (network blip, 5xx). Retry on the next
312
375
  * heartbeat tick; don't spam stderr.
313
376
  */
@@ -338,18 +401,25 @@ function classifyEditError(err: unknown): EditOutcome {
338
401
  }
339
402
 
340
403
  /**
341
- * Manager owning one live message per background worker. Keyed by jsonl
342
- * agent id. The gateway calls `update` on each watcher activity cue and
343
- * `finish` on terminal; `drop` discards a worker's state without a final
344
- * edit (error / supersession paths).
404
+ * Manager owning one live message per (chat,thread) into which all live
405
+ * workers there coalesce. Public methods stay keyed by jsonl agent id (the
406
+ * gateway wiring is unchanged): the manager resolves the enclosing feed group
407
+ * internally. The gateway calls `update` on each watcher activity cue and
408
+ * `finish` on terminal; `drop` discards a worker's state without a final edit
409
+ * (error / supersession paths).
345
410
  */
346
411
  export interface WorkerActivityFeed {
347
- /** True if a message is currently posted for this worker. */
412
+ /** True if a message is currently posted for this worker's feed group. */
348
413
  has(agentId: string): boolean
349
- /** The Telegram message_id currently posted for this worker, or null if
350
- * none is posted (never painted, or dropped after a stale-edit re-post).
351
- * Lets the gateway pin the EXISTING `🛠 Worker` message (status-pin). */
414
+ /** The Telegram message_id currently posted for this worker's feed group, or
415
+ * null if none is posted (never painted, or dropped after a stale-edit
416
+ * re-post). Lets the gateway pin the EXISTING `🛠 Worker` message. Note:
417
+ * siblings sharing the chat/thread return the SAME id (one message). */
352
418
  messageIdOf(agentId: string): number | null
419
+ /** True while the feed group `feedKey` (`${chatId} ${threadId ?? ''}`) still
420
+ * tracks live work — used by the gateway's `wk:group:` pin reaper to exempt
421
+ * a live group's pin from the stale-TTL sweep (#3207). */
422
+ hasRunningInFeed(feedKey: string): boolean
353
423
  /** Push a running-state cue. Returns the serialized op for tests. */
354
424
  update(
355
425
  agentId: string,
@@ -357,13 +427,17 @@ export interface WorkerActivityFeed {
357
427
  view: WorkerActivityView,
358
428
  threadId?: number,
359
429
  ): Promise<void>
360
- /** Force the terminal recap edit. No-op if no message was ever posted. */
430
+ /** Finalize a worker: drop its row from the combined feed (its result reaches
431
+ * the user via the separate handback), or — when it is the last live worker
432
+ * in the group — force the terminal recap edit. No-op if the worker was
433
+ * never tracked. */
361
434
  finish(agentId: string, view: WorkerActivityView): Promise<void>
362
- /** Forget a worker's state without editing (e.g. error path). */
435
+ /** Forget a worker's state without a recap edit (e.g. error path); re-renders
436
+ * the group so the dropped worker disappears from the combined body. */
363
437
  drop(agentId: string): void
364
438
  /**
365
439
  * Issue #3023 (card resurrection). Undo a finalization: clear the durable
366
- * `finalized` gate (and any lingering per-handle `finished` latch) so a
440
+ * `finalized` gate (and any lingering per-row `finished` latch) so a
367
441
  * worker whose card was FALSELY finalized can be painted/edited again. The
368
442
  * watcher calls this (via the gateway's `onResurrect` wiring) when a
369
443
  * falsely-finalized worker's JSONL resumes growing. A fresh `running` cue
@@ -375,7 +449,7 @@ export interface WorkerActivityFeed {
375
449
  stop(): void
376
450
  /** Manually fire one heartbeat tick (test hook). */
377
451
  heartbeatTick(): void
378
- /** Number of tracked workers (test/inspection hook). */
452
+ /** Number of tracked workers across all feed groups (test/inspection hook). */
379
453
  readonly size: number
380
454
  }
381
455
 
@@ -386,6 +460,8 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
386
460
  const minEditInterval = opts.minEditIntervalMs ?? 2500
387
461
  const firstPaintMin = opts.firstPaintMinMs ?? 8000
388
462
  const heartbeatTickMs = opts.heartbeatTickMs ?? 6000
463
+ const maxRows = Math.max(1, Math.floor(opts.maxRows ?? 8))
464
+ const reconcilePinFn = opts.reconcilePin ?? (() => {})
389
465
  const setIntervalFn =
390
466
  opts.setInterval ??
391
467
  ((cb: () => void, ms: number): unknown => {
@@ -395,17 +471,21 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
395
471
  return t
396
472
  })
397
473
  const clearIntervalFn = opts.clearInterval ?? ((handle: unknown) => clearInterval(handle as ReturnType<typeof setInterval>))
398
- const handles = new Map<string, WorkerHandle>()
474
+
475
+ /** Feed groups keyed by `${chatId} ${threadId ?? ''}`. */
476
+ const groups = new Map<string, FeedGroup>()
477
+ /** Reverse index agentId → feedKey, so the agentId-keyed public API resolves
478
+ * its group in O(1). Cleared when a worker's row is removed. */
479
+ const agentIndex = new Map<string, string>()
480
+
399
481
  /**
400
- * Agent ids that have been finalized (`doFinish` latched). Survives handle
482
+ * Agent ids that have been finalized (`finish` latched). Survives row/group
401
483
  * deletion so a LATE watcher `onProgress` tick — which can arrive after
402
- * `finish()`'s chain has fully settled and the handle was deleted — cannot
403
- * resurrect a fresh handle and paint a running card on a worker that is
404
- * already done. The per-handle `finished` flag only covers the narrow
405
- * window between latch and delete; this set is the durable gate. A late
406
- * tick arrives within seconds of finish (watcher poll cadence), so the set
407
- * only needs to cover recent finalizations — capped at FINALIZED_CAP and
408
- * trimmed FIFO to stay bounded across a long gateway lifetime.
484
+ * `finish()`'s chain has fully settled — cannot resurrect a fresh row and
485
+ * paint a running card on a worker that is already done. A late tick arrives
486
+ * within seconds of finish (watcher poll cadence), so the set only needs to
487
+ * cover recent finalizations — capped at FINALIZED_CAP and trimmed FIFO to
488
+ * stay bounded across a long gateway lifetime.
409
489
  */
410
490
  const finalized = new Set<string>()
411
491
  const FINALIZED_CAP = 256
@@ -413,469 +493,489 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
413
493
  if (finalized.has(agentId)) return
414
494
  finalized.add(agentId)
415
495
  if (finalized.size > FINALIZED_CAP) {
416
- // Map-free FIFO trim: Set iterates in insertion order; drop the oldest.
417
496
  const oldest = finalized.values().next().value
418
497
  if (oldest != null) finalized.delete(oldest)
419
498
  }
420
499
  }
421
500
  let heartbeatTimer: unknown = null
422
501
 
423
- function sendOptsFor(h: WorkerHandle): Record<string, unknown> {
502
+ function feedKeyOf(chatId: string, threadId?: number): string {
503
+ return `${chatId} ${threadId ?? ''}`
504
+ }
505
+ function groupOfAgent(agentId: string): FeedGroup | undefined {
506
+ const key = agentIndex.get(agentId)
507
+ return key != null ? groups.get(key) : undefined
508
+ }
509
+
510
+ function sendOptsFor(g: FeedGroup): Record<string, unknown> {
424
511
  return {
425
512
  disable_web_page_preview: true,
426
- // Sub-agent progress card is a status surface, never the user's
427
- // answer — silence the open ping. (editMessageText ignores
428
- // disable_notification, so this is a no-op on the in-place edits
429
- // that share these opts.)
513
+ // Sub-agent progress feed is a status surface, never the user's answer —
514
+ // silence the open ping. (editMessageText ignores disable_notification,
515
+ // so this is a no-op on the in-place edits that share these opts.)
430
516
  disable_notification: true,
431
- ...(h.threadId != null ? { message_thread_id: h.threadId } : {}),
517
+ ...(g.threadId != null ? { message_thread_id: g.threadId } : {}),
432
518
  }
433
519
  }
434
520
 
435
- function noteRateLimited(h: WorkerHandle, err: unknown, label: string): void {
521
+ function noteRateLimited(g: FeedGroup, err: unknown, label: string): void {
436
522
  const retryAfter = extractRetryAfterSecs(err)
437
523
  if (retryAfter == null) return
438
- h.cooldownUntil = nowFn() + retryAfter * 1000 + COOLDOWN_JITTER_MS
524
+ g.cooldownUntil = nowFn() + retryAfter * 1000 + COOLDOWN_JITTER_MS
439
525
  log(`worker-feed: ${label} 429 — backing off ${retryAfter}s`)
440
526
  }
441
527
 
442
528
  /**
443
- * Park a handle in cooldown for the remaining of an open flood window (if
444
- * any), so the heartbeat's `nowFn() < h.cooldownUntil` guard suppresses every
529
+ * Park a group in cooldown for the remaining of an open flood window (if
530
+ * any), so the heartbeat's `nowFn() < g.cooldownUntil` guard suppresses every
445
531
  * further send/edit until the ban closes. Returns true when a window was open
446
532
  * (caller should abandon the current attempt). Two reads of the SAME on-disk
447
533
  * marker `robustApiCall` gates on — never a second notion of "channel open".
448
534
  */
449
- function parkIfFloodWindowOpen(h: WorkerHandle): boolean {
535
+ function parkIfFloodWindowOpen(g: FeedGroup): boolean {
450
536
  const remaining = floodWaitRemainingMs()
451
537
  if (remaining <= 0) return false
452
- // Only extend the cooldown — never shorten one a 429 already set longer.
453
538
  const until = nowFn() + remaining + COOLDOWN_JITTER_MS
454
- if (until > h.cooldownUntil) h.cooldownUntil = until
539
+ if (until > g.cooldownUntil) g.cooldownUntil = until
455
540
  return true
456
541
  }
457
542
 
458
- function accumulateNarrative(h: WorkerHandle, view: WorkerActivityView): void {
543
+ function accumulateNarrative(row: WorkerRow, view: WorkerActivityView): void {
459
544
  const line = view.latestSummary.trim()
460
545
  if (line.length === 0) return
461
- // Dedup within the whole rolling window, not just the immediately-
462
- // preceding line. The watcher re-emits the same narrative across ticks
463
- // while a tool runs (adjacent repeats), AND one logical step can surface
464
- // twice non-adjacently — e.g. a "Look for X" preamble followed later by
465
- // the Task tool whose describeToolUse label is the same "Look for X"
466
- // description, interleaved with another step (the A,B,A duplication the
467
- // operator observed on live cards). A legitimate later re-visit of the
468
- // same step re-appears once the earlier copy scrolls out of the window.
469
- if (h.narrative.includes(line)) return
470
- h.narrative.push(line)
471
- // The `→` current-step line just CHANGED — reset the per-step timer so the
472
- // heartbeat's `· Ns` suffix measures THIS step, not the whole worker run.
473
- h.stepStartedAtMs = nowFn()
474
- // Rolling window — keep only the last STATUS_ROLLING_LINES in memory. The
475
- // render shows exactly those lines (clipped per-line by the unified pipeline);
476
- // fitCardToBudget is the wire-limit backstop.
477
- if (h.narrative.length > STATUS_ROLLING_LINES) {
478
- h.narrative.splice(0, h.narrative.length - STATUS_ROLLING_LINES)
546
+ // Dedup within the whole rolling window (the watcher re-emits the same
547
+ // narrative across ticks, and a preamble + its tool label can repeat
548
+ // non-adjacently — the A,B,A duplication observed on live cards).
549
+ if (row.narrative.includes(line)) return
550
+ row.narrative.push(line)
551
+ // The `→` current-step line just CHANGED — reset the per-step timer.
552
+ row.stepStartedAtMs = nowFn()
553
+ if (row.narrative.length > STATUS_ROLLING_LINES) {
554
+ row.narrative.splice(0, row.narrative.length - STATUS_ROLLING_LINES)
555
+ }
556
+ }
557
+
558
+ /** Live wall-clock elapsed for a worker (climbs between fresh views). */
559
+ function liveElapsed(row: WorkerRow, now: number): number {
560
+ const base = row.dispatchAtMs != null ? now - row.dispatchAtMs : row.lastView?.elapsedMs ?? 0
561
+ return Math.max(base, row.lastView?.elapsedMs ?? 0)
562
+ }
563
+
564
+ /** The running rows of a group, dispatch-ordered (oldest first, stable). */
565
+ function runningRows(g: FeedGroup): WorkerRow[] {
566
+ return [...g.workers.values()]
567
+ .filter((w) => w.state === 'running' && w.lastView != null)
568
+ .sort((a, b) => (a.dispatchAtMs ?? 0) - (b.dispatchAtMs ?? 0))
569
+ }
570
+
571
+ /**
572
+ * Render a group's shared message body at `now`.
573
+ * - `terminalRecap` set + zero running → the last worker's terminal recap
574
+ * (single 🛠 Worker card, done/failed).
575
+ * - exactly one running → the full 🛠 Worker card (single-worker parity).
576
+ * - 2+ running → the combined `renderCombinedWorkerFeed` body.
577
+ * - zero running, no recap → null (nothing to show).
578
+ * `heartbeat` toggles the single-worker climbing `· Ns` step suffix.
579
+ */
580
+ function renderGroupBody(
581
+ g: FeedGroup,
582
+ now: number,
583
+ terminalRecap: WorkerActivityView | null,
584
+ heartbeat: boolean,
585
+ ): string | null {
586
+ const running = runningRows(g)
587
+ if (running.length === 0) {
588
+ if (terminalRecap == null) return null
589
+ return renderWorkerActivity(terminalRecap)
590
+ }
591
+ // On a normal update/finish the header shows the worker's LAST-REPORTED
592
+ // elapsed (byte-stable for the dedup / no-op skip). Only the heartbeat —
593
+ // which fires when no fresh view arrived — climbs a live wall-clock elapsed
594
+ // so a silent worker still visibly advances.
595
+ const elapsedFor = (r: WorkerRow): number =>
596
+ heartbeat ? liveElapsed(r, now) : r.lastView?.elapsedMs ?? liveElapsed(r, now)
597
+ if (running.length === 1) {
598
+ const r = running[0]
599
+ const view: WorkerActivityView = {
600
+ ...(r.lastView as WorkerActivityView),
601
+ elapsedMs: elapsedFor(r),
602
+ narrativeLines: [...r.narrative],
603
+ }
604
+ let liveSuffix = ''
605
+ if (heartbeat) {
606
+ const stepElapsed = r.stepStartedAtMs != null ? now - r.stepStartedAtMs : liveElapsed(r, now)
607
+ liveSuffix = formatStepSuffix(stepElapsed)
608
+ }
609
+ return renderWorkerActivity(view, liveSuffix)
479
610
  }
611
+ const rows: CombinedWorkerRow[] = running.map((r) => {
612
+ const v = r.lastView as WorkerActivityView
613
+ const currentStep = r.narrative.length > 0 ? r.narrative[r.narrative.length - 1] : v.latestSummary
614
+ return {
615
+ description: v.description,
616
+ elapsedMs: elapsedFor(r),
617
+ toolCount: v.toolCount,
618
+ currentStep,
619
+ model: v.model,
620
+ }
621
+ })
622
+ return renderCombinedWorkerFeed(rows, { maxRows })
623
+ }
624
+
625
+ /** Remove a worker's row + index entry; delete the group if it is now empty. */
626
+ function removeWorker(g: FeedGroup, agentId: string): void {
627
+ g.workers.delete(agentId)
628
+ agentIndex.delete(agentId)
629
+ if (g.workers.size === 0) groups.delete(g.feedKey)
630
+ }
631
+
632
+ /**
633
+ * Reconcile the GROUP-level status pin (#3207 review). Pin the shared message
634
+ * while the group has a posted message AND at least one tracked worker;
635
+ * unpin the instant the group empties. Because it is keyed by the whole group
636
+ * (not a single worker), a per-worker finish never unpins a message a sibling
637
+ * still needs — the survivors keep the pin until the LAST worker is done.
638
+ */
639
+ function syncPin(g: FeedGroup): void {
640
+ const messageId = g.messageId != null && g.workers.size > 0 ? g.messageId : null
641
+ reconcilePinFn({ feedKey: g.feedKey, chatId: g.chatId, threadId: g.threadId, messageId })
480
642
  }
481
643
 
482
- async function doUpdate(h: WorkerHandle, view: WorkerActivityView, liveSuffix = ''): Promise<void> {
483
- // Accumulate before any gate so a throttled/cooled-down tick still grows
484
- // the narrative — the line surfaces on the next edit that does fire.
485
- accumulateNarrative(h, view)
486
- // Stamp the dispatch wall-clock once so the heartbeat can climb a live
487
- // elapsed even between fresh views. lastView feeds the heartbeat re-render.
488
- const merged: WorkerActivityView = { ...view, narrativeLines: [...h.narrative] }
489
- h.lastView = merged
490
- if (h.dispatchAtMs == null) h.dispatchAtMs = nowFn() - view.elapsedMs
491
- if (nowFn() < h.cooldownUntil) return
492
- // A flood window is open: the send gate would SHED every call made now
493
- // (resolving `undefined`). Park in cooldown for the window's remaining and
494
- // make ZERO api calls until it closes — the heartbeat re-drives the paint
495
- // with full state on the first tick past the window. Same source of truth
496
- // as robustApiCall's own pre-call probe.
497
- if (parkIfFloodWindowOpen(h)) return
498
- const body = renderWorkerActivity(merged, liveSuffix)
499
-
500
- // First paint: hold off until the worker has run long enough to be
501
- // worth a message; trivial workers stay silent (handback covers them).
502
- if (h.messageId == null) {
503
- if (view.elapsedMs < firstPaintMin) return
644
+ /**
645
+ * Drive the group's shared message to the current combined body. Handles
646
+ * first-paint gating, the proactive throttle (bypassed by `force`), the
647
+ * dedup/no-op skip, the send-gate SHED contract, and 429/flood cooldown.
648
+ * `terminalRecap` (set on the last worker's finish) renders + finalizes the
649
+ * message, then removes the finished row.
650
+ */
651
+ async function doRender(
652
+ g: FeedGroup,
653
+ opts2: { force?: boolean; heartbeat?: boolean; terminalRecap?: WorkerActivityView; finishingAgentId?: string } = {},
654
+ ): Promise<void> {
655
+ const now = nowFn()
656
+ const isTerminal = opts2.terminalRecap != null
657
+ // Terminal edit RESOLVED (landed / not-modified / gone): clear the staged
658
+ // re-drive unconditionally so a heartbeat re-drive can never loop, and drop
659
+ // the finished row when its id is known (the last-worker finalize path).
660
+ const settleTerminal = (): void => {
661
+ g.pendingFinalize = null
662
+ if (opts2.finishingAgentId != null) removeWorker(g, opts2.finishingAgentId)
663
+ // Group-level pin follows membership: unpin once this drops the last
664
+ // worker; a NOOP-pin (siblings remain) keeps the shared message pinned.
665
+ syncPin(g)
666
+ }
667
+ if (now < g.cooldownUntil) {
668
+ if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
669
+ return
670
+ }
671
+ // A flood window is open: the gate would SHED every call. Park in cooldown
672
+ // and make ZERO api calls until it closes; the heartbeat re-drives.
673
+ if (parkIfFloodWindowOpen(g)) {
674
+ if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
675
+ return
676
+ }
677
+
678
+ const body = renderGroupBody(g, now, opts2.terminalRecap ?? null, opts2.heartbeat ?? false)
679
+ if (body == null) {
680
+ // Nothing to show. On a terminal finalize with no message ever posted,
681
+ // just drop the finished row (the handback carries the result).
682
+ if (isTerminal) settleTerminal()
683
+ return
684
+ }
685
+
686
+ // First paint: hold until some worker in the group has run long enough.
687
+ if (g.messageId == null) {
688
+ const maxElapsed = Math.max(0, ...runningRows(g).map((r) => liveElapsed(r, now)))
689
+ // A terminal recap for a group that never painted → nothing to finalize;
690
+ // never first-paint a terminal card (trivial workers stay silent, the
691
+ // handback carries the result — matches the pre-coalesce doFinish guard).
692
+ if (isTerminal) {
693
+ settleTerminal()
694
+ return
695
+ }
696
+ if (maxElapsed < firstPaintMin) return
504
697
  try {
505
- const sent = await opts.bot.sendMessage(h.chatId, body, sendOptsFor(h))
506
- // Shed contract (#3084): the gate resolves `undefined` for a send it
507
- // shed (open flood window) or dropped as stale (`useful` TTL). That is
508
- // NOT a delivered message — dereferencing `sent.message_id` here was
509
- // the `undefined is not an object` crash that fired every ~6s for a
510
- // whole 6h ban. Treat it as not-delivered: record no message_id, park
511
- // on any open window, and let the heartbeat re-drive the paint.
698
+ const sent = await opts.bot.sendMessage(g.chatId, body, sendOptsFor(g))
699
+ // Shed contract (#3084): the gate resolves `undefined`/non-object for a
700
+ // shed send (open flood window) or dropped-stale `useful` TTL. That is
701
+ // NOT a delivered message — record no id, park on any window, let the
702
+ // heartbeat re-drive.
512
703
  if (sent == null || typeof sent.message_id !== 'number') {
513
- parkIfFloodWindowOpen(h)
514
- log(`worker-feed: first paint shed by send gate agent=${h.agentId} — not delivered`)
704
+ parkIfFloodWindowOpen(g)
705
+ log(`worker-feed: first paint shed by send gate feed=${g.feedKey} — not delivered`)
515
706
  return
516
707
  }
517
- h.messageId = sent.message_id
518
- h.lastBody = body
519
- h.lastEditAt = nowFn()
708
+ g.messageId = sent.message_id
709
+ g.lastBody = body
710
+ g.lastEditAt = now
711
+ // Group's first (or re-established) message is up → pin it for the group.
712
+ syncPin(g)
520
713
  log(
521
- `worker-feed: paint agent=${h.agentId} chat=${h.chatId} ` +
522
- `thread=${h.threadId ?? '-'} msgId=${h.messageId} bytes=${body.length}`,
714
+ `worker-feed: paint feed=${g.feedKey} chat=${g.chatId} ` +
715
+ `thread=${g.threadId ?? '-'} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`,
523
716
  )
524
717
  } catch (err) {
525
- noteRateLimited(h, err, 'send')
718
+ noteRateLimited(g, err, 'send')
526
719
  log(`worker-feed: send failed: ${(err as Error).message}`)
527
720
  }
528
721
  return
529
722
  }
530
723
 
531
- // Dedup + proactive throttle.
532
- if (body === h.lastBody) return
533
- if (nowFn() - h.lastEditAt < minEditInterval) return
534
-
535
- try {
536
- const res = await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h))
537
- // Shed honesty (#3084, mirrors #3173): a cosmetic edit the gate shed
538
- // resolves the distinguishable SEND_GATE_SHED sentinel (#3110 F1 — NOT a
539
- // bare `undefined`, which the gate reserves for a benign no-op drop whose
540
- // payload IS already on screen). The shed payload is NOT on screen, so do
541
- // not record it as `lastBody`, or the next tick's dedup would skip
542
- // re-sending the very update the gate dropped. Park on any open window and
543
- // let the heartbeat re-drive once it closes. A benign `undefined`/`true`
544
- // falls through below and is correctly recorded as delivered.
545
- if (isSendGateShed(res)) {
546
- parkIfFloodWindowOpen(h)
547
- return
548
- }
549
- h.lastBody = body
550
- h.lastEditAt = nowFn()
551
- log(
552
- `worker-feed: edit agent=${h.agentId} chat=${h.chatId} ` +
553
- `thread=${h.threadId ?? '-'} msgId=${h.messageId} bytes=${body.length}`,
554
- )
555
- } catch (err) {
556
- const outcome = classifyEditError(err)
557
- if (outcome === 'rate_limited') {
558
- noteRateLimited(h, err, 'edit')
559
- return
560
- }
561
- if (outcome === 'not_modified') {
562
- // Card already shows this body — record it as landed and move on.
563
- h.lastBody = body
564
- h.lastEditAt = nowFn()
565
- return
566
- }
567
- if (outcome === 'gone') {
568
- // Message/chat deleted or edit window closed — there is no card to
569
- // update. Drop the handle silently; a fresh first-paint on the next
570
- // running tick re-establishes one if the worker is still active. No
571
- // warning: "no card" is not a liveness-logic error.
572
- h.messageId = null
573
- h.lastBody = null
574
- return
575
- }
576
- // 'transient' — network blip / 5xx. Leave the handle intact; the
577
- // heartbeat re-attempts on its next tick. Log at debug, not stderr-warn:
578
- // a transport hiccup on a best-effort card is not "shit code", it's a
579
- // retryable blip the framework rides out deterministically.
580
- log(`worker-feed: edit transient error agent=${h.agentId}: ${(err as Error).message}`)
581
- }
582
- }
583
-
584
- async function doFinish(h: WorkerHandle, view: WorkerActivityView): Promise<void> {
585
- // Latch FIRST, before any early return. A `running`-cue tick arriving
586
- // after `finish()` queued this chain (but before its `.finally(delete)`
587
- // drains) would otherwise resurrect a handle via `update()` and paint a
588
- // fresh running message on a finalized worker. Setting this synchronously
589
- // on the chain — ahead of the cooldown/no-message guards — makes the
590
- // gate in `update()` authoritative regardless of which guard path runs.
591
- // The durable `finalized` set survives the subsequent handle deletion so
592
- // a tick arriving AFTER the full settle still can't resurrect.
593
- h.finished = true
594
- markFinalized(h.agentId)
595
- // No message ever posted → nothing to finalize. The worker's result
596
- // reaches the user via the handback reply; a bare "done" recap with
597
- // no preceding activity would be noise.
598
- if (h.messageId == null) {
599
- h.pendingFinish = null
600
- return
601
- }
602
- // A flood window is open (or a prior 429 set a cooldown): honour it — a
603
- // terminal edit isn't worth extending a ban. STAGE the terminal view so the
604
- // heartbeat re-drives the finalize the instant the window/cooldown expires,
605
- // so a finished worker's card can't get stuck on its last running render.
606
- if (parkIfFloodWindowOpen(h) || nowFn() < h.cooldownUntil) {
607
- h.pendingFinish = view
608
- return
609
- }
610
- const body = renderWorkerActivity({ ...view, narrativeLines: h.narrative })
611
- if (body === h.lastBody) {
612
- h.pendingFinish = null
724
+ // Dedup + proactive throttle (finish/terminal edits force through).
725
+ if (body === g.lastBody) {
726
+ if (isTerminal) settleTerminal()
613
727
  return
614
728
  }
729
+ if (!opts2.force && now - g.lastEditAt < minEditInterval) return
730
+
615
731
  try {
616
- const res = await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h))
617
- // Shed honesty (#3084): a shed terminal edit resolves the distinguishable
618
- // SEND_GATE_SHED sentinel (#3110 F1 — NOT a bare `undefined`, which is the
619
- // gate's benign no-op drop) — it did NOT land. Keep `pendingFinish` staged
620
- // so the heartbeat re-drives it once the window closes; do NOT clear it or
621
- // record `lastBody` (that would be a false finalization — card frozen on
622
- // its running render while we believe it's done). Park on any open window.
732
+ const res = await opts.bot.editMessageText(g.chatId, g.messageId, body, sendOptsFor(g))
733
+ // Shed honesty (#3084): a cosmetic edit the gate shed resolves the
734
+ // distinguishable SEND_GATE_SHED sentinel (NOT a bare `undefined`, which
735
+ // the gate reserves for a benign no-op drop whose payload IS on screen).
736
+ // The shed payload is NOT on screen, so do not record it as `lastBody`.
623
737
  if (isSendGateShed(res)) {
624
- parkIfFloodWindowOpen(h)
625
- h.pendingFinish = view
738
+ parkIfFloodWindowOpen(g)
739
+ if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
626
740
  return
627
741
  }
628
- h.lastBody = body
629
- h.lastEditAt = nowFn()
630
- h.pendingFinish = null
631
- log(
632
- `worker-feed: finish agent=${h.agentId} chat=${h.chatId} ` +
633
- `thread=${h.threadId ?? '-'} msgId=${h.messageId} state=${view.state} bytes=${body.length}`,
634
- )
742
+ g.lastBody = body
743
+ g.lastEditAt = now
744
+ if (isTerminal) {
745
+ log(
746
+ `worker-feed: finish feed=${g.feedKey} chat=${g.chatId} thread=${g.threadId ?? '-'} ` +
747
+ `msgId=${g.messageId} agent=${opts2.finishingAgentId ?? '-'} ` +
748
+ `state=${opts2.terminalRecap?.state ?? 'done'} bytes=${body.length}`,
749
+ )
750
+ } else {
751
+ log(
752
+ `worker-feed: edit feed=${g.feedKey} chat=${g.chatId} ` +
753
+ `thread=${g.threadId ?? '-'} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`,
754
+ )
755
+ }
756
+ if (isTerminal) settleTerminal()
635
757
  } catch (err) {
636
758
  const outcome = classifyEditError(err)
637
759
  if (outcome === 'rate_limited') {
638
- noteRateLimited(h, err, 'finish')
639
- // Re-stage for the heartbeat to re-drive after cooldown.
640
- h.pendingFinish = view
760
+ noteRateLimited(g, err, isTerminal ? 'finish' : 'edit')
761
+ if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
641
762
  return
642
763
  }
643
764
  if (outcome === 'not_modified') {
644
- // Card already shows the finalized body — terminal edit succeeded.
645
- h.lastBody = body
646
- h.lastEditAt = nowFn()
647
- h.pendingFinish = null
765
+ g.lastBody = body
766
+ g.lastEditAt = now
767
+ if (isTerminal) settleTerminal()
648
768
  return
649
769
  }
650
770
  if (outcome === 'gone') {
651
- // Message/chat gone — no card to finalize. Drop silently; the
652
- // handback reply carries the result regardless.
653
- h.pendingFinish = null
771
+ // Message/chat gone or edit window closed — no card to update. Drop the
772
+ // stale message id; a fresh first-paint re-establishes one if workers
773
+ // are still live. On a terminal finalize, also drop the finished row.
774
+ g.messageId = null
775
+ g.lastBody = null
776
+ // The pinned message no longer exists → release the group pin claim
777
+ // (settleTerminal already re-syncs on the terminal path).
778
+ if (isTerminal) settleTerminal()
779
+ else syncPin(g)
654
780
  return
655
781
  }
656
- // 'transient' — re-stage for a heartbeat retry; log at debug.
657
- h.pendingFinish = view
658
- log(`worker-feed: finish transient error agent=${h.agentId}: ${(err as Error).message}`)
782
+ // 'transient' — leave the message intact; the heartbeat re-attempts.
783
+ if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
784
+ log(`worker-feed: edit transient error feed=${g.feedKey}: ${(err as Error).message}`)
659
785
  }
660
786
  }
661
787
 
662
- /**
663
- * Heartbeat — keeps a running worker's message alive AND performs the
664
- * FIRST paint for a prose-silent worker whose only tick arrived before
665
- * `firstPaintMin`.
666
- *
667
- * Why the first-paint branch exists: a background worker that dives
668
- * straight into quiet work (e.g. a long `Bash` / `npm test`) emits a
669
- * single `sub_agent_tool_use` event when the command is invoked, then no
670
- * further JSONL lines for the whole run. That one tick drives `update`
671
- * once — but if it lands before `firstPaintMin` the paint is held, and
672
- * with no subsequent tick nothing ever re-drives it, so the worker shows
673
- * NOTHING for its entire run (the "I can't see the worker" gap). The
674
- * heartbeat closes it: once such a handle is past `firstPaintMin`, drive a
675
- * paint here through the same chain → doUpdate path. After first paint the
676
- * suffix-only maintenance branch keeps it advancing.
677
- *
678
- * For handles that already have a posted message, this is the original
679
- * option-(a), suffix-only re-render (never editMessageText directly). Skips:
680
- * - handles inside a 429 cooldown,
681
- * - handles with no `lastView` (no update ever arrived) or non-running,
682
- * - for the maintenance branch: handles edited within minEditInterval
683
- * (no stampede) or whose current step isn't yet stale.
684
- * The `· Ns` liveSuffix is applied ONLY when the worker's current step is
685
- * stale (now - lastEditAt >= heartbeatTickMs) so a normally-ticking worker is
686
- * untouched and its body stays byte-stable for the dedup.
687
- */
788
+ // Arm the heartbeat once at construction. The real timer is `.unref()`'d so
789
+ // it never keeps the process alive; tests inject setInterval/clearInterval.
688
790
  function heartbeatTick(): void {
689
791
  const now = nowFn()
690
- for (const h of handles.values()) {
691
- // Orphan-paint guard: `finish()` deletes the handle in a `.finally` that
692
- // may not have drained if a tick fires in the same synchronous stretch.
693
- // Skip any handle no longer in the map so the first-paint branch below
694
- // can never send a fresh `running` message on an already-finished worker
695
- // (which would orphan a card that never finalizes). Restores the
696
- // structural safety the pre-first-paint `messageId == null` skip gave.
697
- if (!handles.has(h.agentId)) continue
698
-
699
- // Deferred-finalize re-drive: a terminal edit that hit a 429 cooldown
700
- // (or a transient error) was staged on `pendingFinish` by `doFinish`.
701
- // Re-drive it once the cooldown has expired so a finished worker's card
702
- // can't get stuck on its last running render. This is the deterministic
703
- // backstop that replaces the old "stale but harmless" surrender — the
704
- // framework owns ALIVE-and-done, wall-clock driven, no model in the loop.
705
- // (The handle is still in the map because `finish()`'s `.finally(delete)`
706
- // is chained AFTER `doFinish` and won't drain while a re-drive keeps the
707
- // chain busy; once the terminal edit lands, `pendingFinish` is cleared
708
- // and the `.finally` runs on the next chain settle.)
709
- if (h.pendingFinish != null && now >= h.cooldownUntil) {
710
- const view = h.pendingFinish
711
- h.chain = h.chain
712
- .then(() => doFinish(h, view))
792
+ for (const g of [...groups.values()]) {
793
+ // Deferred-finalize re-drive: a terminal edit that hit a cooldown/flood
794
+ // window was staged on `pendingFinalize`. Re-drive it once the cooldown
795
+ // expires so a finished feed can't get stuck on its last running render.
796
+ if (g.pendingFinalize != null && now >= g.cooldownUntil) {
797
+ const recap = g.pendingFinalize
798
+ // The finishing agent is whatever finished row remains (state terminal).
799
+ const finishingAgentId = [...g.workers.values()].find((w) => w.finished)?.agentId
800
+ g.chain = g.chain
801
+ .then(() => doRender(g, { force: true, terminalRecap: recap, finishingAgentId }))
713
802
  .catch((err) => {
714
- log(`worker-feed: heartbeat finalize re-drive error ${h.agentId}: ${(err as Error).message}`)
715
- })
716
- .finally(() => {
717
- // Mirror `finish()`'s teardown: once the re-driven `doFinish`
718
- // clears `pendingFinish` (terminal edit landed OR permanently
719
- // failed), drop the handle. If it re-staged (another 429), the
720
- // handle survives for the next heartbeat tick to retry.
721
- if (handles.get(h.agentId)?.pendingFinish == null) {
722
- handles.delete(h.agentId)
723
- }
803
+ log(`worker-feed: heartbeat finalize re-drive error feed=${g.feedKey}: ${(err as Error).message}`)
724
804
  })
725
805
  continue
726
806
  }
727
807
 
728
- if (h.lastView == null) continue
729
- if (h.lastView.state !== 'running') continue
730
- if (now < h.cooldownUntil) continue
731
-
732
- const liveElapsed = h.dispatchAtMs != null ? now - h.dispatchAtMs : h.lastView.elapsedMs
733
-
734
- // First-paint path: a prose-silent worker's single early tick was held
735
- // (elapsed < firstPaintMin) and no further tick re-drove it. Once it is
736
- // past firstPaintMin, drive the paint. doUpdate's send branch re-checks
737
- // firstPaintMin against the refreshed elapsed, so this is exact.
738
- if (h.messageId == null) {
739
- if (liveElapsed < firstPaintMin) continue
740
- const view = { ...h.lastView, elapsedMs: Math.max(h.lastView.elapsedMs, liveElapsed) }
741
- h.chain = h.chain
742
- .then(() => doUpdate(h, view))
808
+ if (now < g.cooldownUntil) continue
809
+ const running = runningRows(g)
810
+ if (running.length === 0) continue
811
+
812
+ // First-paint path: no message yet and some worker has now crossed
813
+ // firstPaintMin (a prose-silent worker's single early tick was held).
814
+ if (g.messageId == null) {
815
+ const maxElapsed = Math.max(0, ...running.map((r) => liveElapsed(r, now)))
816
+ if (maxElapsed < firstPaintMin) continue
817
+ g.chain = g.chain
818
+ .then(() => doRender(g, {}))
743
819
  .catch((err) => {
744
- log(`worker-feed: heartbeat first-paint chain error ${h.agentId}: ${(err as Error).message}`)
820
+ log(`worker-feed: heartbeat first-paint chain error feed=${g.feedKey}: ${(err as Error).message}`)
745
821
  })
746
822
  continue
747
823
  }
748
824
 
749
- if (now - h.lastEditAt < minEditInterval) continue
750
- const stale = now - h.lastEditAt >= heartbeatTickMs
825
+ if (now - g.lastEditAt < minEditInterval) continue
826
+ const stale = now - g.lastEditAt >= heartbeatTickMs
751
827
  if (!stale) continue
752
- // Per-step suffix: the CURRENT step's own elapsed (since the `→` line
753
- // last changed), never the worker total — the header already shows the
754
- // total, and repeating it on the step line was the Ken-observed dupe.
755
- // Under STEP_TIMER_MIN_MS formatStepSuffix returns '' (no timer yet);
756
- // the header elapsed still climbs via the refreshed view below.
757
- const stepElapsed = h.stepStartedAtMs != null ? now - h.stepStartedAtMs : liveElapsed
758
- const liveSuffix = formatStepSuffix(stepElapsed)
759
- // Re-render THROUGH the chain + doUpdate path — never editMessageText directly.
760
- //
761
- // CLOCK-ANCHOR PARITY: refresh the view's elapsedMs to the same `now`
762
- // anchor the step suffix uses. The header renders
763
- // `view.elapsedMs`; passing the stale lastView froze the header at the
764
- // last watcher event while the `· Ns` suffix kept ticking, so the
765
- // current step's timer could read MORE than the card's master elapsed
766
- // (Ken-observed defect). Both numbers now derive from one anchor
767
- // (dispatchAtMs) at one `now`, so header elapsed >= step suffix always.
768
- const view = { ...h.lastView, elapsedMs: Math.max(h.lastView.elapsedMs, liveElapsed) }
769
- h.chain = h.chain
770
- .then(() => doUpdate(h, view, liveSuffix))
828
+ // Re-render THROUGH the chain → doRender path with climbing elapsed.
829
+ g.chain = g.chain
830
+ .then(() => doRender(g, { heartbeat: true }))
771
831
  .catch((err) => {
772
- log(`worker-feed: heartbeat chain error ${h.agentId}: ${(err as Error).message}`)
832
+ log(`worker-feed: heartbeat chain error feed=${g.feedKey}: ${(err as Error).message}`)
773
833
  })
774
834
  }
775
835
  }
776
836
 
777
- // Arm the heartbeat once at construction. The real timer is `.unref()`'d so
778
- // it never keeps the process alive; tests inject setInterval/clearInterval.
779
837
  heartbeatTimer = setIntervalFn(heartbeatTick, heartbeatTickMs)
780
838
 
781
839
  return {
782
840
  has(agentId) {
783
- return handles.get(agentId)?.messageId != null
841
+ const g = groupOfAgent(agentId)
842
+ return g != null && g.messageId != null && g.workers.has(agentId)
784
843
  },
785
844
  messageIdOf(agentId) {
786
- return handles.get(agentId)?.messageId ?? null
845
+ return groupOfAgent(agentId)?.messageId ?? null
846
+ },
847
+ hasRunningInFeed(feedKey) {
848
+ const g = groups.get(feedKey)
849
+ return g != null && g.workers.size > 0
787
850
  },
788
851
  get size() {
789
- return handles.size
852
+ let n = 0
853
+ for (const g of groups.values()) n += g.workers.size
854
+ return n
790
855
  },
791
856
  update(agentId, chatId, view, threadId) {
792
- // No chat to post to (owner DM unconfigured) — don't create a
793
- // handle that would retry a failing send('') every tick.
857
+ // No chat to post to (owner DM unconfigured) — don't create state that
858
+ // would retry a failing send('') every tick.
794
859
  if (chatId.length === 0) return Promise.resolve()
795
- // Resurrection guard: a worker that has already been finalized
796
- // (`doFinish` latched `finalized`) must not get a fresh running cue.
797
- // A late watcher `onProgress` tick can arrive after `finish()`'s chain
798
- // has fully settled and the handle was deleted — without this durable
799
- // gate the tick would create a brand-new handle and paint a fresh
800
- // `running` message on an already-done worker (the card lies). The
801
- // heartbeat's orphan-paint guard covers the heartbeat tick only; the
802
- // per-handle `finished` flag covers the pre-delete window; this set
803
- // covers the post-delete window.
860
+ // Resurrection guard: a worker already finalized must not get a fresh
861
+ // running cue (a late watcher tick would repaint a done worker as live).
804
862
  if (finalized.has(agentId)) return Promise.resolve()
805
- const existing = handles.get(agentId)
806
- if (existing?.finished === true) return Promise.resolve()
807
- let h = existing
808
- if (h == null) {
809
- h = {
810
- agentId,
863
+ const existingRow = groupOfAgent(agentId)?.workers.get(agentId)
864
+ if (existingRow?.finished === true) return Promise.resolve()
865
+
866
+ const feedKey = feedKeyOf(chatId, threadId)
867
+ let g = groups.get(feedKey)
868
+ if (g == null) {
869
+ g = {
870
+ feedKey,
811
871
  chatId,
812
872
  threadId,
813
873
  messageId: null,
814
874
  lastBody: null,
815
875
  lastEditAt: 0,
816
876
  cooldownUntil: 0,
817
- narrative: [],
818
877
  chain: Promise.resolve(),
878
+ workers: new Map(),
879
+ pendingFinalize: null,
880
+ }
881
+ groups.set(feedKey, g)
882
+ }
883
+ let row = g.workers.get(agentId)
884
+ if (row == null) {
885
+ row = {
886
+ agentId,
887
+ narrative: [],
819
888
  lastView: null,
889
+ state: 'running',
890
+ finished: false,
820
891
  dispatchAtMs: null,
821
892
  stepStartedAtMs: null,
822
- finished: false,
823
- pendingFinish: null,
824
893
  }
825
- handles.set(agentId, h)
894
+ g.workers.set(agentId, row)
895
+ agentIndex.set(agentId, feedKey)
826
896
  }
827
- const handle = h
828
- handle.chain = handle.chain.then(() => doUpdate(handle, view)).catch((err) => {
897
+ // Accumulate before the gate so a throttled tick still grows the
898
+ // narrative — it surfaces on the next edit that does fire.
899
+ accumulateNarrative(row, view)
900
+ row.state = 'running'
901
+ row.lastView = { ...view, narrativeLines: [...row.narrative] }
902
+ if (row.dispatchAtMs == null) row.dispatchAtMs = nowFn() - view.elapsedMs
903
+
904
+ const group = g
905
+ group.chain = group.chain.then(() => doRender(group)).catch((err) => {
829
906
  log(`worker-feed: update chain error ${agentId}: ${(err as Error).message}`)
830
907
  })
831
- return handle.chain
908
+ return group.chain
832
909
  },
833
910
  finish(agentId, view) {
834
- const h = handles.get(agentId)
835
- if (h == null) return Promise.resolve()
836
- h.chain = h.chain
837
- .then(() => doFinish(h, view))
911
+ const g = groupOfAgent(agentId)
912
+ const row = g?.workers.get(agentId)
913
+ if (g == null || row == null) {
914
+ // Never tracked (trivial worker) — mark finalized so a late tick can't
915
+ // resurrect, and let the handback carry the result.
916
+ markFinalized(agentId)
917
+ return Promise.resolve()
918
+ }
919
+ // Latch synchronously so a late `running` cue on the chain can't resurrect.
920
+ row.finished = true
921
+ row.state = view.state === 'failed' ? 'failed' : 'done'
922
+ markFinalized(agentId)
923
+
924
+ const group = g
925
+ group.chain = group.chain
926
+ .then(() => {
927
+ const others = runningRows(group).filter((w) => w.agentId !== agentId)
928
+ if (others.length > 0) {
929
+ // Siblings still live → drop this row from the combined body and
930
+ // re-render the running set. The result reaches the user via the
931
+ // separate handback, never folded into this cosmetic edit. The
932
+ // group pin STAYS (siblings still need the shared message) — a
933
+ // per-worker unpin here was the #3207 review blocker.
934
+ removeWorker(group, agentId)
935
+ syncPin(group)
936
+ return doRender(group, { force: true })
937
+ }
938
+ // Last live worker → finalize the shared message to its terminal recap.
939
+ const recap: WorkerActivityView = { ...view, narrativeLines: [...row.narrative] }
940
+ return doRender(group, { force: true, terminalRecap: recap, finishingAgentId: agentId })
941
+ })
838
942
  .catch((err) => {
839
943
  log(`worker-feed: finish chain error ${agentId}: ${(err as Error).message}`)
840
944
  })
841
- .finally(() => {
842
- // Only tear down the handle once the terminal edit has actually
843
- // landed (or permanently failed). If `doFinish` staged the edit on
844
- // `pendingFinish` (a 429 cooldown / transient error was in effect),
845
- // the handle must survive so the heartbeat can re-drive the
846
- // finalize after cooldown. The heartbeat's re-drive chain ends by
847
- // re-entering `doFinish`, which clears `pendingFinish` on success
848
- // or permanent-failure — so this `.finally` deletes on the NEXT
849
- // chain settle once there is nothing left to finalize. Without this
850
- // guard, the `.finally` would delete the handle (and its staged
851
- // pendingFinish) immediately after the first staged doFinish,
852
- // stranding the card on its last running render.
853
- if (handles.get(agentId)?.pendingFinish == null) {
854
- handles.delete(agentId)
855
- }
856
- })
857
- return h.chain
945
+ return group.chain
858
946
  },
859
947
  drop(agentId) {
860
- // A dropped worker is also done — mark finalized so a late watcher
861
- // tick can't resurrect a running card on it (same gate as `finish`).
948
+ // A dropped worker is also done — mark finalized so a late tick can't
949
+ // resurrect a running card on it (same gate as `finish`).
862
950
  markFinalized(agentId)
863
- handles.delete(agentId)
951
+ const g = groupOfAgent(agentId)
952
+ if (g == null) return
953
+ const hadMessage = g.messageId != null
954
+ removeWorker(g, agentId)
955
+ // Group pin follows membership: unpin if this emptied the group, else
956
+ // keep it (siblings still need the shared message).
957
+ if (hadMessage) syncPin(g)
958
+ // Re-render so the dropped worker disappears from a combined body. Skip
959
+ // when the group is gone (removeWorker deleted it) or never painted.
960
+ if (hadMessage && groups.has(g.feedKey) && runningRows(g).length > 0) {
961
+ g.chain = g.chain
962
+ .then(() => doRender(g, { force: true }))
963
+ .catch((err) => {
964
+ log(`worker-feed: drop re-render error ${agentId}: ${(err as Error).message}`)
965
+ })
966
+ }
864
967
  },
865
968
  resurrect(agentId) {
866
969
  // Issue #3023: the worker's card was falsely finalized and its JSONL has
867
- // resumed. Re-open the paint path: drop the durable finalized gate so a
868
- // fresh `running` cue creates a new handle and first-paints a live card
869
- // again, and un-latch any surviving handle (finish deleted it in the
870
- // common case, but a staged pendingFinish could keep it alive). The next
871
- // `update` tick from the watcher's replayed progress does the repaint.
970
+ // resumed. Re-open the paint path: drop the durable finalized gate + any
971
+ // surviving per-row latch so a fresh `running` cue repaints a live card.
872
972
  const wasFinalized = finalized.delete(agentId)
873
- const h = handles.get(agentId)
874
- if (h != null) {
875
- h.finished = false
876
- h.pendingFinish = null
973
+ const row = groupOfAgent(agentId)?.workers.get(agentId)
974
+ if (row != null) {
975
+ row.finished = false
976
+ row.state = 'running'
877
977
  }
878
- if (wasFinalized || h != null) {
978
+ if (wasFinalized || row != null) {
879
979
  log(`worker-feed: resurrect agent=${agentId} — cleared finalized gate; card will repaint on next running cue`)
880
980
  }
881
981
  },