switchroom 0.18.15 → 0.18.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/dist/agent-scheduler/index.js +16 -0
  2. package/dist/auth-broker/index.js +445 -10
  3. package/dist/cli/notion-write-pretool.mjs +16 -0
  4. package/dist/cli/switchroom.js +654 -479
  5. package/dist/host-control/main.js +20 -1
  6. package/dist/vault/approvals/kernel-server.js +16 -0
  7. package/dist/vault/broker/server.js +16 -0
  8. package/package.json +1 -1
  9. package/profiles/_base/start.sh.hbs +81 -139
  10. package/telegram-plugin/bridge/bridge.ts +7 -1
  11. package/telegram-plugin/dist/bridge/bridge.js +26 -1
  12. package/telegram-plugin/dist/gateway/gateway.js +1758 -661
  13. package/telegram-plugin/dist/server.js +26 -1
  14. package/telegram-plugin/draft-stream.ts +78 -3
  15. package/telegram-plugin/fleet-fallback-resume.ts +26 -3
  16. package/telegram-plugin/gateway/approval-hold.ts +49 -0
  17. package/telegram-plugin/gateway/bridge-dead-watchdog.ts +64 -22
  18. package/telegram-plugin/gateway/effort-command.ts +9 -7
  19. package/telegram-plugin/gateway/gateway.ts +627 -291
  20. package/telegram-plugin/gateway/linear-activity.ts +20 -4
  21. package/telegram-plugin/gateway/litellm-local-notice-wiring.ts +200 -0
  22. package/telegram-plugin/gateway/model-command.ts +96 -18
  23. package/telegram-plugin/gateway/pending-session-command.ts +10 -8
  24. package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
  25. package/telegram-plugin/gateway/session-model-file.ts +141 -172
  26. package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
  27. package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
  28. package/telegram-plugin/litellm-local-notice.ts +189 -0
  29. package/telegram-plugin/llm-error-present.ts +436 -0
  30. package/telegram-plugin/operator-events.ts +7 -1
  31. package/telegram-plugin/permission-title.ts +172 -10
  32. package/telegram-plugin/premium-recovery.ts +101 -0
  33. package/telegram-plugin/quota-watch.ts +16 -4
  34. package/telegram-plugin/raw-error-scrub.ts +73 -0
  35. package/telegram-plugin/retry-api-call.ts +8 -2
  36. package/telegram-plugin/runtime-metrics.ts +16 -0
  37. package/telegram-plugin/send-gate-degraded.test.ts +161 -8
  38. package/telegram-plugin/send-gate-observability.test.ts +140 -0
  39. package/telegram-plugin/send-gate-observability.ts +65 -20
  40. package/telegram-plugin/send-gate.test.ts +143 -1
  41. package/telegram-plugin/send-gate.ts +246 -23
  42. package/telegram-plugin/session-tail.ts +16 -0
  43. package/telegram-plugin/shared/local-time.ts +69 -0
  44. package/telegram-plugin/stream-controller.ts +143 -20
  45. package/telegram-plugin/stream-reply-handler.ts +12 -2
  46. package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
  47. package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
  48. package/telegram-plugin/tests/bot-api.harness.ts +7 -2
  49. package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
  50. package/telegram-plugin/tests/draft-stream.test.ts +110 -1
  51. package/telegram-plugin/tests/effort-command.test.ts +4 -4
  52. package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
  53. package/telegram-plugin/tests/flood-windows-persistence.test.ts +5 -4
  54. package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +33 -19
  55. package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +47 -127
  56. package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
  57. package/telegram-plugin/tests/litellm-local-notice.test.ts +417 -0
  58. package/telegram-plugin/tests/llm-error-present.test.ts +380 -0
  59. package/telegram-plugin/tests/model-command.test.ts +84 -1
  60. package/telegram-plugin/tests/permission-title.test.ts +167 -4
  61. package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
  62. package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
  63. package/telegram-plugin/tests/quota-watch.test.ts +21 -0
  64. package/telegram-plugin/tests/reaction-gate-routing.test.ts +8 -3
  65. package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
  66. package/telegram-plugin/tests/session-model-file.test.ts +7 -155
  67. package/telegram-plugin/tests/stream-controller-send-gate.test.ts +521 -0
  68. package/telegram-plugin/tests/stream-reply-handler.test.ts +44 -0
  69. package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
  70. package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
  71. package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
  72. package/telegram-plugin/tests/worker-activity-feed.test.ts +212 -2
  73. package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
  74. package/telegram-plugin/tier-downgrade.ts +198 -0
  75. package/telegram-plugin/tool-activity-summary.ts +99 -0
  76. package/telegram-plugin/worker-activity-feed.ts +543 -368
@@ -13,17 +13,34 @@
13
13
  * that grows/edits as work happens — indistinguishable from "the agent
14
14
  * is typing live", not a status widget.
15
15
  *
16
- * Pure render (`renderWorkerActivity`) + an injected bot API
17
- * (`BotApiForWorkerFeed`), mirroring `issues-card.ts` so the gateway
18
- * reuses the same wiring. The manager (`createWorkerActivityFeed`) owns
19
- * one edit-in-place message per worker, keyed by jsonl agent id, with:
16
+ * COALESCED (#3084 follow-up): all live workers dispatched to the SAME
17
+ * chat/thread render into ONE shared message (a {@link FeedGroup}), not one
18
+ * message each. With N workers, N separate messages each coalesce their own
19
+ * edit stream but ALL draw from the send gate's 1/sec per-chat bucket — so
20
+ * ~N-1 of every second's edits SHED and every card refreshes only ~once per N
21
+ * seconds (liveness collapse). One combined message = ONE per-message edit
22
+ * stream: it coalesces (last-write-wins) and never contends with siblings, so
23
+ * every worker's row refreshes together within the ~1.5s edit floor. A single-
24
+ * worker chat still renders the full 🛠 Worker card (identical to before); a
25
+ * 2+ worker chat renders `renderCombinedWorkerFeed`.
26
+ *
27
+ * Pure render (`renderWorkerActivity` / `renderCombinedWorkerFeed`) + an
28
+ * injected bot API (`BotApiForWorkerFeed`). The manager
29
+ * (`createWorkerActivityFeed`) owns one edit-in-place message per (chat,thread)
30
+ * with:
20
31
  * - a first-paint delay so trivial sub-second workers never post a
21
32
  * message (their result still lands via the handback reply),
22
33
  * - a proactive min-edit-interval throttle (worker jsonl ticks ~1/s;
23
34
  * Telegram rate-limits edits) plus body-dedup,
24
- * - per-worker serialization so two rapid ticks can't double-send,
35
+ * - per-message serialization so two rapid ticks can't double-send,
25
36
  * - 429 cooldown + message_id drift resilience (re-post on stale edit),
26
- * - a forced terminal edit on `finish` regardless of throttle.
37
+ * - a forced terminal edit on the LAST worker's `finish`.
38
+ *
39
+ * The completed-worker RESULT is NOT folded into this cosmetic feed — it
40
+ * reaches the user as its own `useful` handback reply (gateway onFinish). The
41
+ * feed shows only *live* work; a finished worker's row is dropped from the
42
+ * combined body (or, when it is the last worker, the message finalizes to its
43
+ * terminal recap).
27
44
  *
28
45
  * The feed is gated to BACKGROUND workers and is ON by default; set
29
46
  * `SWITCHROOM_WORKER_ACTIVITY_FEED=0` to disable it — see the gateway
@@ -38,7 +55,13 @@ import {
38
55
  truncate,
39
56
  } from './card-format.js'
40
57
  import { STATUS_ROLLING_LINES } from './status-no-truncate.js'
41
- import { renderStatusCard, formatStepSuffix } from './tool-activity-summary.js'
58
+ import {
59
+ renderStatusCard,
60
+ formatStepSuffix,
61
+ renderCombinedWorkerFeed,
62
+ type CombinedWorkerRow,
63
+ } from './tool-activity-summary.js'
64
+ import { isSendGateShed } from './send-gate.js'
42
65
 
43
66
  /** Worker-activity feed is ON by default; an operator opts out with
44
67
  * SWITCHROOM_WORKER_ACTIVITY_FEED=0. */
@@ -195,6 +218,24 @@ export interface WorkerActivityFeedOpts {
195
218
  firstPaintMinMs?: number
196
219
  /** stderr-style log sink. Defaults to noop. */
197
220
  log?: (msg: string) => void
221
+ /**
222
+ * Remaining ms of the currently-open per-bot flood window, read from the
223
+ * SAME persisted marker the gateway's `robustApiCall` and held-card sweep
224
+ * consult (`makeFloodWaitProbe(FLOOD_STATE_PATH)`). Two independent reads of
225
+ * one source of truth — not a second notion of "is the channel open".
226
+ *
227
+ * Why the feed needs it (#3084 follow-up): the feed's send/edit adapters run
228
+ * through the send gate, which SHEDS (resolves `undefined`) any call made
229
+ * while a flood window is open. Without this probe the heartbeat re-fires a
230
+ * shed send every `heartbeatTickMs` (~6s) for the WHOLE ban — thousands of
231
+ * gate admissions, and (pre-fix) a `sent.message_id` crash on the `undefined`
232
+ * every tick. With it, a running/first-paint tick that sees an open window
233
+ * parks the group in cooldown for the window's remaining and makes ZERO api
234
+ * calls until it closes, mirroring the held-card sweep's pre-send probe.
235
+ * Defaults to `() => 0` (no window) so tests and non-gateway callers are
236
+ * unchanged.
237
+ */
238
+ floodWaitRemainingMs?: () => number
198
239
  /**
199
240
  * Heartbeat timer factory. Injectable for tests. Defaults to the real
200
241
  * `setInterval`, `.unref()`'d so it never keeps the process alive.
@@ -203,69 +244,110 @@ export interface WorkerActivityFeedOpts {
203
244
  /** Heartbeat timer disposer. Injectable for tests. Defaults to `clearInterval`. */
204
245
  clearInterval?: (handle: unknown) => void
205
246
  /**
206
- * Heartbeat tick cadence in ms. On each tick a stale, running worker is
207
- * re-rendered with a climbing `· Ns` suffix so a worker that emits no new
208
- * narrative still visibly advances. Default 6000ms.
247
+ * Heartbeat tick cadence in ms. On each tick a stale, running feed is
248
+ * re-rendered with climbing elapsed so a worker that emits no new narrative
249
+ * still visibly advances. Default 6000ms.
209
250
  */
210
251
  heartbeatTickMs?: number
252
+ /**
253
+ * Max worker rows rendered in a COMBINED feed (2+ workers in one chat/thread)
254
+ * before the `+M more working…` spill line. Keeps the coalesced body compact
255
+ * and legible (and under the rich-message wire ceiling). Default 8. A single-
256
+ * worker chat renders the full 🛠 Worker card and ignores this. Sourced from
257
+ * `channels.telegram.worker_feed.max_rows` via the config cascade.
258
+ */
259
+ maxRows?: number
260
+ /**
261
+ * Group-level status-pin reconcile hook (#3207 review). Because workers now
262
+ * COALESCE into one shared message, the pin MUST follow the GROUP lifecycle,
263
+ * not a single worker's: pin the shared message when a group's first worker
264
+ * paints, and unpin ONLY when the group empties (its LAST worker finishes / is
265
+ * dropped). The feed alone knows group membership + running-count, so it owns
266
+ * the pin. `messageId: null` means "unpin this group". A per-worker unpin would
267
+ * physically unpin a message a SIBLING still needs, and the survivor's next
268
+ * pin request NO-OPs (its claim still names that id) — so it would run
269
+ * unpinned for the rest of its life (the shared-message NOOP trap). Best-
270
+ * effort; defaults to a noop for tests / non-gateway callers.
271
+ */
272
+ reconcilePin?: (args: {
273
+ feedKey: string
274
+ chatId: string
275
+ threadId?: number
276
+ messageId: number | null
277
+ }) => void
211
278
  }
212
279
 
213
- interface WorkerHandle {
280
+ /**
281
+ * One live worker's per-row state inside a chat/thread feed group. The row
282
+ * carries everything the render needs for THIS worker; the shared message
283
+ * (id, cooldown, chain) lives on the enclosing {@link FeedGroup}.
284
+ */
285
+ interface WorkerRow {
214
286
  /** jsonl agent id — carried so success/failure log lines can name the worker. */
215
287
  agentId: string
216
- chatId: string
217
- threadId?: number
218
- messageId: number | null
219
- lastBody: string | null
220
- lastEditAt: number
221
- cooldownUntil: number
222
288
  /**
223
- * Accumulated narrative lines (oldest→newest), deduped against the
224
- * immediately-preceding line. Rolling-window capped to STATUS_ROLLING_LINES.
225
- * Grows the live render so the feed reads like the main agent's answer.
289
+ * Accumulated narrative lines (oldest→newest), deduped within the whole
290
+ * rolling window. Rolling-window capped to STATUS_ROLLING_LINES. Grows the
291
+ * live render so the feed reads like the main agent's answer.
226
292
  */
227
293
  narrative: string[]
228
- /** Per-worker serialization chain so ticks can't interleave sends. */
229
- chain: Promise<void>
230
- /** Last view rendered into the message (drives the heartbeat re-render). */
294
+ /** Last view for this worker (drives the heartbeat re-render + combined row). */
231
295
  lastView: WorkerActivityView | null
296
+ /** Latest state observed for this worker; excluded from the running set once
297
+ * terminal (its result reaches the user via the separate handback reply). */
298
+ state: WorkerActivityState
232
299
  /**
233
- * A terminal (`finish`) view whose edit could not land yet — most often
234
- * because a 429 cooldown was in effect when `doFinish` ran. The heartbeat
235
- * re-drives `doFinish` with this view once the cooldown expires so a
236
- * transport hiccup can't leave the card stuck on its last running render
237
- * ("worker done, card says running"). Cleared on a successful terminal
238
- * edit, on a permanent failure (message gone), or when the handle is
239
- * deleted. Null when no finalize is pending.
240
- */
241
- pendingFinish: WorkerActivityView | null
242
- /**
243
- * Latched in `doFinish` before the terminal edit. A late watcher
244
- * `onProgress` tick that arrives after `finish()` queued its chain (but
245
- * before the `.finally(handles.delete)` microtask drains) must NOT
246
- * resurrect the handle and paint a fresh `running` message on an
247
- * already-finalized worker. The heartbeat's orphan-paint guard
248
- * (`if (!handles.has(h.agentId)) continue`) only covers the heartbeat
249
- * tick — this flag covers the `update` entry point. Set synchronously
250
- * inside `doFinish` (runs on the chain), checked synchronously in
251
- * `update` before handle creation.
300
+ * Latched in `finish` before the terminal edit. A late watcher `onProgress`
301
+ * tick that arrives after `finish()` queued its chain must NOT resurrect the
302
+ * row and paint a fresh `running` state on an already-finalized worker.
252
303
  */
253
304
  finished: boolean
254
305
  /**
255
306
  * Wall-clock ms the worker was dispatched, derived from `now - view.elapsedMs`
256
307
  * on the first update. The heartbeat computes a live elapsed from this so the
257
- * `· Ns` suffix climbs even when no fresh view arrives.
308
+ * elapsed climbs even when no fresh view arrives, and it fixes the worker's
309
+ * stable sort order within the combined feed.
258
310
  */
259
311
  dispatchAtMs: number | null
260
312
  /**
261
313
  * Wall-clock ms the CURRENT step started — stamped whenever a NEW narrative
262
314
  * line lands (the `→` line changes). The heartbeat's step suffix shows the
263
- * step's OWN elapsed from this anchor (not the worker total, which the
264
- * header already carries), and only once past STEP_TIMER_MIN_MS.
315
+ * step's OWN elapsed from this anchor (single-worker card only), and only
316
+ * once past STEP_TIMER_MIN_MS.
265
317
  */
266
318
  stepStartedAtMs: number | null
267
319
  }
268
320
 
321
+ /**
322
+ * One shared feed message per (chatId, threadId). ALL live workers dispatched
323
+ * to the same chat/thread render into this one message — so their edits form a
324
+ * single per-message edit stream under the send gate's 1/sec per-chat ceiling
325
+ * (last-write-wins coalescing + no-op skip) instead of N contending streams
326
+ * that shed (#3084). A single-worker group renders the full 🛠 Worker card;
327
+ * a 2+ worker group renders the combined `renderCombinedWorkerFeed` body.
328
+ */
329
+ interface FeedGroup {
330
+ /** Stable key `${chatId} ${threadId ?? ''}`. */
331
+ feedKey: string
332
+ chatId: string
333
+ threadId?: number
334
+ messageId: number | null
335
+ lastBody: string | null
336
+ lastEditAt: number
337
+ cooldownUntil: number
338
+ /** Single serialization chain for the shared message — ticks can't interleave sends. */
339
+ chain: Promise<void>
340
+ /** Live workers in this group, keyed by agentId (insertion ≈ dispatch order). */
341
+ workers: Map<string, WorkerRow>
342
+ /**
343
+ * A terminal render (the last worker's recap) staged because a 429 cooldown /
344
+ * flood window blocked the edit. The heartbeat re-drives it once the cooldown
345
+ * expires so a finished feed can't get stuck on its last running render.
346
+ * Null when no finalize is pending.
347
+ */
348
+ pendingFinalize: WorkerActivityView | null
349
+ }
350
+
269
351
  const COOLDOWN_JITTER_MS = 500
270
352
 
271
353
  function extractRetryAfterSecs(err: unknown): number | null {
@@ -288,7 +370,7 @@ function extractRetryAfterSecs(err: unknown): number | null {
288
370
  * 'rate_limited' — 429 with retry_after. Back off; the heartbeat re-drives
289
371
  * the edit after cooldown (running renders + deferred terminal edits).
290
372
  * 'gone' — message/chat deleted or edit window expired. Nothing to
291
- * update; drop the handle silently (no warning — there is no card).
373
+ * update; drop the message silently (no warning — there is no card).
292
374
  * 'transient' — anything else (network blip, 5xx). Retry on the next
293
375
  * heartbeat tick; don't spam stderr.
294
376
  */
@@ -319,18 +401,25 @@ function classifyEditError(err: unknown): EditOutcome {
319
401
  }
320
402
 
321
403
  /**
322
- * Manager owning one live message per background worker. Keyed by jsonl
323
- * agent id. The gateway calls `update` on each watcher activity cue and
324
- * `finish` on terminal; `drop` discards a worker's state without a final
325
- * edit (error / supersession paths).
404
+ * Manager owning one live message per (chat,thread) into which all live
405
+ * workers there coalesce. Public methods stay keyed by jsonl agent id (the
406
+ * gateway wiring is unchanged): the manager resolves the enclosing feed group
407
+ * internally. The gateway calls `update` on each watcher activity cue and
408
+ * `finish` on terminal; `drop` discards a worker's state without a final edit
409
+ * (error / supersession paths).
326
410
  */
327
411
  export interface WorkerActivityFeed {
328
- /** True if a message is currently posted for this worker. */
412
+ /** True if a message is currently posted for this worker's feed group. */
329
413
  has(agentId: string): boolean
330
- /** The Telegram message_id currently posted for this worker, or null if
331
- * none is posted (never painted, or dropped after a stale-edit re-post).
332
- * Lets the gateway pin the EXISTING `🛠 Worker` message (status-pin). */
414
+ /** The Telegram message_id currently posted for this worker's feed group, or
415
+ * null if none is posted (never painted, or dropped after a stale-edit
416
+ * re-post). Lets the gateway pin the EXISTING `🛠 Worker` message. Note:
417
+ * siblings sharing the chat/thread return the SAME id (one message). */
333
418
  messageIdOf(agentId: string): number | null
419
+ /** True while the feed group `feedKey` (`${chatId} ${threadId ?? ''}`) still
420
+ * tracks live work — used by the gateway's `wk:group:` pin reaper to exempt
421
+ * a live group's pin from the stale-TTL sweep (#3207). */
422
+ hasRunningInFeed(feedKey: string): boolean
334
423
  /** Push a running-state cue. Returns the serialized op for tests. */
335
424
  update(
336
425
  agentId: string,
@@ -338,13 +427,17 @@ export interface WorkerActivityFeed {
338
427
  view: WorkerActivityView,
339
428
  threadId?: number,
340
429
  ): Promise<void>
341
- /** Force the terminal recap edit. No-op if no message was ever posted. */
430
+ /** Finalize a worker: drop its row from the combined feed (its result reaches
431
+ * the user via the separate handback), or — when it is the last live worker
432
+ * in the group — force the terminal recap edit. No-op if the worker was
433
+ * never tracked. */
342
434
  finish(agentId: string, view: WorkerActivityView): Promise<void>
343
- /** Forget a worker's state without editing (e.g. error path). */
435
+ /** Forget a worker's state without a recap edit (e.g. error path); re-renders
436
+ * the group so the dropped worker disappears from the combined body. */
344
437
  drop(agentId: string): void
345
438
  /**
346
439
  * Issue #3023 (card resurrection). Undo a finalization: clear the durable
347
- * `finalized` gate (and any lingering per-handle `finished` latch) so a
440
+ * `finalized` gate (and any lingering per-row `finished` latch) so a
348
441
  * worker whose card was FALSELY finalized can be painted/edited again. The
349
442
  * watcher calls this (via the gateway's `onResurrect` wiring) when a
350
443
  * falsely-finalized worker's JSONL resumes growing. A fresh `running` cue
@@ -356,16 +449,19 @@ export interface WorkerActivityFeed {
356
449
  stop(): void
357
450
  /** Manually fire one heartbeat tick (test hook). */
358
451
  heartbeatTick(): void
359
- /** Number of tracked workers (test/inspection hook). */
452
+ /** Number of tracked workers across all feed groups (test/inspection hook). */
360
453
  readonly size: number
361
454
  }
362
455
 
363
456
  export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerActivityFeed {
364
457
  const log = opts.log ?? (() => {})
365
458
  const nowFn = opts.now ?? Date.now
459
+ const floodWaitRemainingMs = opts.floodWaitRemainingMs ?? (() => 0)
366
460
  const minEditInterval = opts.minEditIntervalMs ?? 2500
367
461
  const firstPaintMin = opts.firstPaintMinMs ?? 8000
368
462
  const heartbeatTickMs = opts.heartbeatTickMs ?? 6000
463
+ const maxRows = Math.max(1, Math.floor(opts.maxRows ?? 8))
464
+ const reconcilePinFn = opts.reconcilePin ?? (() => {})
369
465
  const setIntervalFn =
370
466
  opts.setInterval ??
371
467
  ((cb: () => void, ms: number): unknown => {
@@ -375,17 +471,21 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
375
471
  return t
376
472
  })
377
473
  const clearIntervalFn = opts.clearInterval ?? ((handle: unknown) => clearInterval(handle as ReturnType<typeof setInterval>))
378
- const handles = new Map<string, WorkerHandle>()
474
+
475
+ /** Feed groups keyed by `${chatId} ${threadId ?? ''}`. */
476
+ const groups = new Map<string, FeedGroup>()
477
+ /** Reverse index agentId → feedKey, so the agentId-keyed public API resolves
478
+ * its group in O(1). Cleared when a worker's row is removed. */
479
+ const agentIndex = new Map<string, string>()
480
+
379
481
  /**
380
- * Agent ids that have been finalized (`doFinish` latched). Survives handle
482
+ * Agent ids that have been finalized (`finish` latched). Survives row/group
381
483
  * deletion so a LATE watcher `onProgress` tick — which can arrive after
382
- * `finish()`'s chain has fully settled and the handle was deleted — cannot
383
- * resurrect a fresh handle and paint a running card on a worker that is
384
- * already done. The per-handle `finished` flag only covers the narrow
385
- * window between latch and delete; this set is the durable gate. A late
386
- * tick arrives within seconds of finish (watcher poll cadence), so the set
387
- * only needs to cover recent finalizations — capped at FINALIZED_CAP and
388
- * trimmed FIFO to stay bounded across a long gateway lifetime.
484
+ * `finish()`'s chain has fully settled — cannot resurrect a fresh row and
485
+ * paint a running card on a worker that is already done. A late tick arrives
486
+ * within seconds of finish (watcher poll cadence), so the set only needs to
487
+ * cover recent finalizations — capped at FINALIZED_CAP and trimmed FIFO to
488
+ * stay bounded across a long gateway lifetime.
389
489
  */
390
490
  const finalized = new Set<string>()
391
491
  const FINALIZED_CAP = 256
@@ -393,414 +493,489 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
393
493
  if (finalized.has(agentId)) return
394
494
  finalized.add(agentId)
395
495
  if (finalized.size > FINALIZED_CAP) {
396
- // Map-free FIFO trim: Set iterates in insertion order; drop the oldest.
397
496
  const oldest = finalized.values().next().value
398
497
  if (oldest != null) finalized.delete(oldest)
399
498
  }
400
499
  }
401
500
  let heartbeatTimer: unknown = null
402
501
 
403
- function sendOptsFor(h: WorkerHandle): Record<string, unknown> {
502
+ function feedKeyOf(chatId: string, threadId?: number): string {
503
+ return `${chatId} ${threadId ?? ''}`
504
+ }
505
+ function groupOfAgent(agentId: string): FeedGroup | undefined {
506
+ const key = agentIndex.get(agentId)
507
+ return key != null ? groups.get(key) : undefined
508
+ }
509
+
510
+ function sendOptsFor(g: FeedGroup): Record<string, unknown> {
404
511
  return {
405
512
  disable_web_page_preview: true,
406
- // Sub-agent progress card is a status surface, never the user's
407
- // answer — silence the open ping. (editMessageText ignores
408
- // disable_notification, so this is a no-op on the in-place edits
409
- // that share these opts.)
513
+ // Sub-agent progress feed is a status surface, never the user's answer —
514
+ // silence the open ping. (editMessageText ignores disable_notification,
515
+ // so this is a no-op on the in-place edits that share these opts.)
410
516
  disable_notification: true,
411
- ...(h.threadId != null ? { message_thread_id: h.threadId } : {}),
517
+ ...(g.threadId != null ? { message_thread_id: g.threadId } : {}),
412
518
  }
413
519
  }
414
520
 
415
- function noteRateLimited(h: WorkerHandle, err: unknown, label: string): void {
521
+ function noteRateLimited(g: FeedGroup, err: unknown, label: string): void {
416
522
  const retryAfter = extractRetryAfterSecs(err)
417
523
  if (retryAfter == null) return
418
- h.cooldownUntil = nowFn() + retryAfter * 1000 + COOLDOWN_JITTER_MS
524
+ g.cooldownUntil = nowFn() + retryAfter * 1000 + COOLDOWN_JITTER_MS
419
525
  log(`worker-feed: ${label} 429 — backing off ${retryAfter}s`)
420
526
  }
421
527
 
422
- function accumulateNarrative(h: WorkerHandle, view: WorkerActivityView): void {
528
+ /**
529
+ * Park a group in cooldown for the remaining of an open flood window (if
530
+ * any), so the heartbeat's `nowFn() < g.cooldownUntil` guard suppresses every
531
+ * further send/edit until the ban closes. Returns true when a window was open
532
+ * (caller should abandon the current attempt). Two reads of the SAME on-disk
533
+ * marker `robustApiCall` gates on — never a second notion of "channel open".
534
+ */
535
+ function parkIfFloodWindowOpen(g: FeedGroup): boolean {
536
+ const remaining = floodWaitRemainingMs()
537
+ if (remaining <= 0) return false
538
+ const until = nowFn() + remaining + COOLDOWN_JITTER_MS
539
+ if (until > g.cooldownUntil) g.cooldownUntil = until
540
+ return true
541
+ }
542
+
543
+ function accumulateNarrative(row: WorkerRow, view: WorkerActivityView): void {
423
544
  const line = view.latestSummary.trim()
424
545
  if (line.length === 0) return
425
- // Dedup within the whole rolling window, not just the immediately-
426
- // preceding line. The watcher re-emits the same narrative across ticks
427
- // while a tool runs (adjacent repeats), AND one logical step can surface
428
- // twice non-adjacently — e.g. a "Look for X" preamble followed later by
429
- // the Task tool whose describeToolUse label is the same "Look for X"
430
- // description, interleaved with another step (the A,B,A duplication the
431
- // operator observed on live cards). A legitimate later re-visit of the
432
- // same step re-appears once the earlier copy scrolls out of the window.
433
- if (h.narrative.includes(line)) return
434
- h.narrative.push(line)
435
- // The `→` current-step line just CHANGED — reset the per-step timer so the
436
- // heartbeat's `· Ns` suffix measures THIS step, not the whole worker run.
437
- h.stepStartedAtMs = nowFn()
438
- // Rolling window — keep only the last STATUS_ROLLING_LINES in memory. The
439
- // render shows exactly those lines (clipped per-line by the unified pipeline);
440
- // fitCardToBudget is the wire-limit backstop.
441
- if (h.narrative.length > STATUS_ROLLING_LINES) {
442
- h.narrative.splice(0, h.narrative.length - STATUS_ROLLING_LINES)
546
+ // Dedup within the whole rolling window (the watcher re-emits the same
547
+ // narrative across ticks, and a preamble + its tool label can repeat
548
+ // non-adjacently — the A,B,A duplication observed on live cards).
549
+ if (row.narrative.includes(line)) return
550
+ row.narrative.push(line)
551
+ // The `→` current-step line just CHANGED — reset the per-step timer.
552
+ row.stepStartedAtMs = nowFn()
553
+ if (row.narrative.length > STATUS_ROLLING_LINES) {
554
+ row.narrative.splice(0, row.narrative.length - STATUS_ROLLING_LINES)
443
555
  }
444
556
  }
445
557
 
446
- async function doUpdate(h: WorkerHandle, view: WorkerActivityView, liveSuffix = ''): Promise<void> {
447
- // Accumulate before any gate so a throttled/cooled-down tick still grows
448
- // the narrative — the line surfaces on the next edit that does fire.
449
- accumulateNarrative(h, view)
450
- // Stamp the dispatch wall-clock once so the heartbeat can climb a live
451
- // elapsed even between fresh views. lastView feeds the heartbeat re-render.
452
- const merged: WorkerActivityView = { ...view, narrativeLines: [...h.narrative] }
453
- h.lastView = merged
454
- if (h.dispatchAtMs == null) h.dispatchAtMs = nowFn() - view.elapsedMs
455
- if (nowFn() < h.cooldownUntil) return
456
- const body = renderWorkerActivity(merged, liveSuffix)
457
-
458
- // First paint: hold off until the worker has run long enough to be
459
- // worth a message; trivial workers stay silent (handback covers them).
460
- if (h.messageId == null) {
461
- if (view.elapsedMs < firstPaintMin) return
462
- try {
463
- const sent = await opts.bot.sendMessage(h.chatId, body, sendOptsFor(h))
464
- h.messageId = sent.message_id
465
- h.lastBody = body
466
- h.lastEditAt = nowFn()
467
- log(
468
- `worker-feed: paint agent=${h.agentId} chat=${h.chatId} ` +
469
- `thread=${h.threadId ?? '-'} msgId=${h.messageId} bytes=${body.length}`,
470
- )
471
- } catch (err) {
472
- noteRateLimited(h, err, 'send')
473
- log(`worker-feed: send failed: ${(err as Error).message}`)
474
- }
475
- return
476
- }
558
+ /** Live wall-clock elapsed for a worker (climbs between fresh views). */
559
+ function liveElapsed(row: WorkerRow, now: number): number {
560
+ const base = row.dispatchAtMs != null ? now - row.dispatchAtMs : row.lastView?.elapsedMs ?? 0
561
+ return Math.max(base, row.lastView?.elapsedMs ?? 0)
562
+ }
477
563
 
478
- // Dedup + proactive throttle.
479
- if (body === h.lastBody) return
480
- if (nowFn() - h.lastEditAt < minEditInterval) return
564
+ /** The running rows of a group, dispatch-ordered (oldest first, stable). */
565
+ function runningRows(g: FeedGroup): WorkerRow[] {
566
+ return [...g.workers.values()]
567
+ .filter((w) => w.state === 'running' && w.lastView != null)
568
+ .sort((a, b) => (a.dispatchAtMs ?? 0) - (b.dispatchAtMs ?? 0))
569
+ }
481
570
 
482
- try {
483
- await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h))
484
- h.lastBody = body
485
- h.lastEditAt = nowFn()
486
- log(
487
- `worker-feed: edit agent=${h.agentId} chat=${h.chatId} ` +
488
- `thread=${h.threadId ?? '-'} msgId=${h.messageId} bytes=${body.length}`,
489
- )
490
- } catch (err) {
491
- const outcome = classifyEditError(err)
492
- if (outcome === 'rate_limited') {
493
- noteRateLimited(h, err, 'edit')
494
- return
571
+ /**
572
+ * Render a group's shared message body at `now`.
573
+ * - `terminalRecap` set + zero running → the last worker's terminal recap
574
+ * (single 🛠 Worker card, done/failed).
575
+ * - exactly one running → the full 🛠 Worker card (single-worker parity).
576
+ * - 2+ running → the combined `renderCombinedWorkerFeed` body.
577
+ * - zero running, no recap → null (nothing to show).
578
+ * `heartbeat` toggles the single-worker climbing `· Ns` step suffix.
579
+ */
580
+ function renderGroupBody(
581
+ g: FeedGroup,
582
+ now: number,
583
+ terminalRecap: WorkerActivityView | null,
584
+ heartbeat: boolean,
585
+ ): string | null {
586
+ const running = runningRows(g)
587
+ if (running.length === 0) {
588
+ if (terminalRecap == null) return null
589
+ return renderWorkerActivity(terminalRecap)
590
+ }
591
+ // On a normal update/finish the header shows the worker's LAST-REPORTED
592
+ // elapsed (byte-stable for the dedup / no-op skip). Only the heartbeat —
593
+ // which fires when no fresh view arrived — climbs a live wall-clock elapsed
594
+ // so a silent worker still visibly advances.
595
+ const elapsedFor = (r: WorkerRow): number =>
596
+ heartbeat ? liveElapsed(r, now) : r.lastView?.elapsedMs ?? liveElapsed(r, now)
597
+ if (running.length === 1) {
598
+ const r = running[0]
599
+ const view: WorkerActivityView = {
600
+ ...(r.lastView as WorkerActivityView),
601
+ elapsedMs: elapsedFor(r),
602
+ narrativeLines: [...r.narrative],
495
603
  }
496
- if (outcome === 'not_modified') {
497
- // Card already shows this body — record it as landed and move on.
498
- h.lastBody = body
499
- h.lastEditAt = nowFn()
500
- return
604
+ let liveSuffix = ''
605
+ if (heartbeat) {
606
+ const stepElapsed = r.stepStartedAtMs != null ? now - r.stepStartedAtMs : liveElapsed(r, now)
607
+ liveSuffix = formatStepSuffix(stepElapsed)
501
608
  }
502
- if (outcome === 'gone') {
503
- // Message/chat deleted or edit window closed — there is no card to
504
- // update. Drop the handle silently; a fresh first-paint on the next
505
- // running tick re-establishes one if the worker is still active. No
506
- // warning: "no card" is not a liveness-logic error.
507
- h.messageId = null
508
- h.lastBody = null
509
- return
510
- }
511
- // 'transient' — network blip / 5xx. Leave the handle intact; the
512
- // heartbeat re-attempts on its next tick. Log at debug, not stderr-warn:
513
- // a transport hiccup on a best-effort card is not "shit code", it's a
514
- // retryable blip the framework rides out deterministically.
515
- log(`worker-feed: edit transient error agent=${h.agentId}: ${(err as Error).message}`)
609
+ return renderWorkerActivity(view, liveSuffix)
516
610
  }
611
+ const rows: CombinedWorkerRow[] = running.map((r) => {
612
+ const v = r.lastView as WorkerActivityView
613
+ const currentStep = r.narrative.length > 0 ? r.narrative[r.narrative.length - 1] : v.latestSummary
614
+ return {
615
+ description: v.description,
616
+ elapsedMs: elapsedFor(r),
617
+ toolCount: v.toolCount,
618
+ currentStep,
619
+ model: v.model,
620
+ }
621
+ })
622
+ return renderCombinedWorkerFeed(rows, { maxRows })
623
+ }
624
+
625
+ /** Remove a worker's row + index entry; delete the group if it is now empty. */
626
+ function removeWorker(g: FeedGroup, agentId: string): void {
627
+ g.workers.delete(agentId)
628
+ agentIndex.delete(agentId)
629
+ if (g.workers.size === 0) groups.delete(g.feedKey)
630
+ }
631
+
632
+ /**
633
+ * Reconcile the GROUP-level status pin (#3207 review). Pin the shared message
634
+ * while the group has a posted message AND at least one tracked worker;
635
+ * unpin the instant the group empties. Because it is keyed by the whole group
636
+ * (not a single worker), a per-worker finish never unpins a message a sibling
637
+ * still needs — the survivors keep the pin until the LAST worker is done.
638
+ */
639
+ function syncPin(g: FeedGroup): void {
640
+ const messageId = g.messageId != null && g.workers.size > 0 ? g.messageId : null
641
+ reconcilePinFn({ feedKey: g.feedKey, chatId: g.chatId, threadId: g.threadId, messageId })
517
642
  }
518
643
 
519
- async function doFinish(h: WorkerHandle, view: WorkerActivityView): Promise<void> {
520
- // Latch FIRST, before any early return. A `running`-cue tick arriving
521
- // after `finish()` queued this chain (but before its `.finally(delete)`
522
- // drains) would otherwise resurrect a handle via `update()` and paint a
523
- // fresh running message on a finalized worker. Setting this synchronously
524
- // on the chain — ahead of the cooldown/no-message guards — makes the
525
- // gate in `update()` authoritative regardless of which guard path runs.
526
- // The durable `finalized` set survives the subsequent handle deletion so
527
- // a tick arriving AFTER the full settle still can't resurrect.
528
- h.finished = true
529
- markFinalized(h.agentId)
530
- // No message ever posted → nothing to finalize. The worker's result
531
- // reaches the user via the handback reply; a bare "done" recap with
532
- // no preceding activity would be noise.
533
- if (h.messageId == null) {
534
- h.pendingFinish = null
644
+ /**
645
+ * Drive the group's shared message to the current combined body. Handles
646
+ * first-paint gating, the proactive throttle (bypassed by `force`), the
647
+ * dedup/no-op skip, the send-gate SHED contract, and 429/flood cooldown.
648
+ * `terminalRecap` (set on the last worker's finish) renders + finalizes the
649
+ * message, then removes the finished row.
650
+ */
651
+ async function doRender(
652
+ g: FeedGroup,
653
+ opts2: { force?: boolean; heartbeat?: boolean; terminalRecap?: WorkerActivityView; finishingAgentId?: string } = {},
654
+ ): Promise<void> {
655
+ const now = nowFn()
656
+ const isTerminal = opts2.terminalRecap != null
657
+ // Terminal edit RESOLVED (landed / not-modified / gone): clear the staged
658
+ // re-drive unconditionally so a heartbeat re-drive can never loop, and drop
659
+ // the finished row when its id is known (the last-worker finalize path).
660
+ const settleTerminal = (): void => {
661
+ g.pendingFinalize = null
662
+ if (opts2.finishingAgentId != null) removeWorker(g, opts2.finishingAgentId)
663
+ // Group-level pin follows membership: unpin once this drops the last
664
+ // worker; a NOOP-pin (siblings remain) keeps the shared message pinned.
665
+ syncPin(g)
666
+ }
667
+ if (now < g.cooldownUntil) {
668
+ if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
535
669
  return
536
670
  }
537
- if (nowFn() < h.cooldownUntil) {
538
- // Honour the flood-wait; a terminal edit isn't worth a ban. But
539
- // unlike the prior "stale but harmless" surrender, STAGE the terminal
540
- // view so the heartbeat re-drives the finalize edit the instant the
541
- // cooldown expires — a transport hiccup can no longer leave a
542
- // finished worker's card stuck on its last running render.
543
- h.pendingFinish = view
671
+ // A flood window is open: the gate would SHED every call. Park in cooldown
672
+ // and make ZERO api calls until it closes; the heartbeat re-drives.
673
+ if (parkIfFloodWindowOpen(g)) {
674
+ if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
544
675
  return
545
676
  }
546
- const body = renderWorkerActivity({ ...view, narrativeLines: h.narrative })
547
- if (body === h.lastBody) {
548
- h.pendingFinish = null
677
+
678
+ const body = renderGroupBody(g, now, opts2.terminalRecap ?? null, opts2.heartbeat ?? false)
679
+ if (body == null) {
680
+ // Nothing to show. On a terminal finalize with no message ever posted,
681
+ // just drop the finished row (the handback carries the result).
682
+ if (isTerminal) settleTerminal()
549
683
  return
550
684
  }
685
+
686
+ // First paint: hold until some worker in the group has run long enough.
687
+ if (g.messageId == null) {
688
+ const maxElapsed = Math.max(0, ...runningRows(g).map((r) => liveElapsed(r, now)))
689
+ // A terminal recap for a group that never painted → nothing to finalize;
690
+ // never first-paint a terminal card (trivial workers stay silent, the
691
+ // handback carries the result — matches the pre-coalesce doFinish guard).
692
+ if (isTerminal) {
693
+ settleTerminal()
694
+ return
695
+ }
696
+ if (maxElapsed < firstPaintMin) return
697
+ try {
698
+ const sent = await opts.bot.sendMessage(g.chatId, body, sendOptsFor(g))
699
+ // Shed contract (#3084): the gate resolves `undefined`/non-object for a
700
+ // shed send (open flood window) or dropped-stale `useful` TTL. That is
701
+ // NOT a delivered message — record no id, park on any window, let the
702
+ // heartbeat re-drive.
703
+ if (sent == null || typeof sent.message_id !== 'number') {
704
+ parkIfFloodWindowOpen(g)
705
+ log(`worker-feed: first paint shed by send gate feed=${g.feedKey} — not delivered`)
706
+ return
707
+ }
708
+ g.messageId = sent.message_id
709
+ g.lastBody = body
710
+ g.lastEditAt = now
711
+ // Group's first (or re-established) message is up → pin it for the group.
712
+ syncPin(g)
713
+ log(
714
+ `worker-feed: paint feed=${g.feedKey} chat=${g.chatId} ` +
715
+ `thread=${g.threadId ?? '-'} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`,
716
+ )
717
+ } catch (err) {
718
+ noteRateLimited(g, err, 'send')
719
+ log(`worker-feed: send failed: ${(err as Error).message}`)
720
+ }
721
+ return
722
+ }
723
+
724
+ // Dedup + proactive throttle (finish/terminal edits force through).
725
+ if (body === g.lastBody) {
726
+ if (isTerminal) settleTerminal()
727
+ return
728
+ }
729
+ if (!opts2.force && now - g.lastEditAt < minEditInterval) return
730
+
551
731
  try {
552
- await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h))
553
- h.lastBody = body
554
- h.lastEditAt = nowFn()
555
- h.pendingFinish = null
556
- log(
557
- `worker-feed: finish agent=${h.agentId} chat=${h.chatId} ` +
558
- `thread=${h.threadId ?? '-'} msgId=${h.messageId} state=${view.state} bytes=${body.length}`,
559
- )
732
+ const res = await opts.bot.editMessageText(g.chatId, g.messageId, body, sendOptsFor(g))
733
+ // Shed honesty (#3084): a cosmetic edit the gate shed resolves the
734
+ // distinguishable SEND_GATE_SHED sentinel (NOT a bare `undefined`, which
735
+ // the gate reserves for a benign no-op drop whose payload IS on screen).
736
+ // The shed payload is NOT on screen, so do not record it as `lastBody`.
737
+ if (isSendGateShed(res)) {
738
+ parkIfFloodWindowOpen(g)
739
+ if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
740
+ return
741
+ }
742
+ g.lastBody = body
743
+ g.lastEditAt = now
744
+ if (isTerminal) {
745
+ log(
746
+ `worker-feed: finish feed=${g.feedKey} chat=${g.chatId} thread=${g.threadId ?? '-'} ` +
747
+ `msgId=${g.messageId} agent=${opts2.finishingAgentId ?? '-'} ` +
748
+ `state=${opts2.terminalRecap?.state ?? 'done'} bytes=${body.length}`,
749
+ )
750
+ } else {
751
+ log(
752
+ `worker-feed: edit feed=${g.feedKey} chat=${g.chatId} ` +
753
+ `thread=${g.threadId ?? '-'} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`,
754
+ )
755
+ }
756
+ if (isTerminal) settleTerminal()
560
757
  } catch (err) {
561
758
  const outcome = classifyEditError(err)
562
759
  if (outcome === 'rate_limited') {
563
- noteRateLimited(h, err, 'finish')
564
- // Re-stage for the heartbeat to re-drive after cooldown.
565
- h.pendingFinish = view
760
+ noteRateLimited(g, err, isTerminal ? 'finish' : 'edit')
761
+ if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
566
762
  return
567
763
  }
568
764
  if (outcome === 'not_modified') {
569
- // Card already shows the finalized body — terminal edit succeeded.
570
- h.lastBody = body
571
- h.lastEditAt = nowFn()
572
- h.pendingFinish = null
765
+ g.lastBody = body
766
+ g.lastEditAt = now
767
+ if (isTerminal) settleTerminal()
573
768
  return
574
769
  }
575
770
  if (outcome === 'gone') {
576
- // Message/chat gone — no card to finalize. Drop silently; the
577
- // handback reply carries the result regardless.
578
- h.pendingFinish = null
771
+ // Message/chat gone or edit window closed — no card to update. Drop the
772
+ // stale message id; a fresh first-paint re-establishes one if workers
773
+ // are still live. On a terminal finalize, also drop the finished row.
774
+ g.messageId = null
775
+ g.lastBody = null
776
+ // The pinned message no longer exists → release the group pin claim
777
+ // (settleTerminal already re-syncs on the terminal path).
778
+ if (isTerminal) settleTerminal()
779
+ else syncPin(g)
579
780
  return
580
781
  }
581
- // 'transient' — re-stage for a heartbeat retry; log at debug.
582
- h.pendingFinish = view
583
- log(`worker-feed: finish transient error agent=${h.agentId}: ${(err as Error).message}`)
782
+ // 'transient' — leave the message intact; the heartbeat re-attempts.
783
+ if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
784
+ log(`worker-feed: edit transient error feed=${g.feedKey}: ${(err as Error).message}`)
584
785
  }
585
786
  }
586
787
 
587
- /**
588
- * Heartbeat — keeps a running worker's message alive AND performs the
589
- * FIRST paint for a prose-silent worker whose only tick arrived before
590
- * `firstPaintMin`.
591
- *
592
- * Why the first-paint branch exists: a background worker that dives
593
- * straight into quiet work (e.g. a long `Bash` / `npm test`) emits a
594
- * single `sub_agent_tool_use` event when the command is invoked, then no
595
- * further JSONL lines for the whole run. That one tick drives `update`
596
- * once — but if it lands before `firstPaintMin` the paint is held, and
597
- * with no subsequent tick nothing ever re-drives it, so the worker shows
598
- * NOTHING for its entire run (the "I can't see the worker" gap). The
599
- * heartbeat closes it: once such a handle is past `firstPaintMin`, drive a
600
- * paint here through the same chain → doUpdate path. After first paint the
601
- * suffix-only maintenance branch keeps it advancing.
602
- *
603
- * For handles that already have a posted message, this is the original
604
- * option-(a), suffix-only re-render (never editMessageText directly). Skips:
605
- * - handles inside a 429 cooldown,
606
- * - handles with no `lastView` (no update ever arrived) or non-running,
607
- * - for the maintenance branch: handles edited within minEditInterval
608
- * (no stampede) or whose current step isn't yet stale.
609
- * The `· Ns` liveSuffix is applied ONLY when the worker's current step is
610
- * stale (now - lastEditAt >= heartbeatTickMs) so a normally-ticking worker is
611
- * untouched and its body stays byte-stable for the dedup.
612
- */
788
+ // Arm the heartbeat once at construction. The real timer is `.unref()`'d so
789
+ // it never keeps the process alive; tests inject setInterval/clearInterval.
613
790
  function heartbeatTick(): void {
614
791
  const now = nowFn()
615
- for (const h of handles.values()) {
616
- // Orphan-paint guard: `finish()` deletes the handle in a `.finally` that
617
- // may not have drained if a tick fires in the same synchronous stretch.
618
- // Skip any handle no longer in the map so the first-paint branch below
619
- // can never send a fresh `running` message on an already-finished worker
620
- // (which would orphan a card that never finalizes). Restores the
621
- // structural safety the pre-first-paint `messageId == null` skip gave.
622
- if (!handles.has(h.agentId)) continue
623
-
624
- // Deferred-finalize re-drive: a terminal edit that hit a 429 cooldown
625
- // (or a transient error) was staged on `pendingFinish` by `doFinish`.
626
- // Re-drive it once the cooldown has expired so a finished worker's card
627
- // can't get stuck on its last running render. This is the deterministic
628
- // backstop that replaces the old "stale but harmless" surrender — the
629
- // framework owns ALIVE-and-done, wall-clock driven, no model in the loop.
630
- // (The handle is still in the map because `finish()`'s `.finally(delete)`
631
- // is chained AFTER `doFinish` and won't drain while a re-drive keeps the
632
- // chain busy; once the terminal edit lands, `pendingFinish` is cleared
633
- // and the `.finally` runs on the next chain settle.)
634
- if (h.pendingFinish != null && now >= h.cooldownUntil) {
635
- const view = h.pendingFinish
636
- h.chain = h.chain
637
- .then(() => doFinish(h, view))
792
+ for (const g of [...groups.values()]) {
793
+ // Deferred-finalize re-drive: a terminal edit that hit a cooldown/flood
794
+ // window was staged on `pendingFinalize`. Re-drive it once the cooldown
795
+ // expires so a finished feed can't get stuck on its last running render.
796
+ if (g.pendingFinalize != null && now >= g.cooldownUntil) {
797
+ const recap = g.pendingFinalize
798
+ // The finishing agent is whatever finished row remains (state terminal).
799
+ const finishingAgentId = [...g.workers.values()].find((w) => w.finished)?.agentId
800
+ g.chain = g.chain
801
+ .then(() => doRender(g, { force: true, terminalRecap: recap, finishingAgentId }))
638
802
  .catch((err) => {
639
- log(`worker-feed: heartbeat finalize re-drive error ${h.agentId}: ${(err as Error).message}`)
640
- })
641
- .finally(() => {
642
- // Mirror `finish()`'s teardown: once the re-driven `doFinish`
643
- // clears `pendingFinish` (terminal edit landed OR permanently
644
- // failed), drop the handle. If it re-staged (another 429), the
645
- // handle survives for the next heartbeat tick to retry.
646
- if (handles.get(h.agentId)?.pendingFinish == null) {
647
- handles.delete(h.agentId)
648
- }
803
+ log(`worker-feed: heartbeat finalize re-drive error feed=${g.feedKey}: ${(err as Error).message}`)
649
804
  })
650
805
  continue
651
806
  }
652
807
 
653
- if (h.lastView == null) continue
654
- if (h.lastView.state !== 'running') continue
655
- if (now < h.cooldownUntil) continue
656
-
657
- const liveElapsed = h.dispatchAtMs != null ? now - h.dispatchAtMs : h.lastView.elapsedMs
658
-
659
- // First-paint path: a prose-silent worker's single early tick was held
660
- // (elapsed < firstPaintMin) and no further tick re-drove it. Once it is
661
- // past firstPaintMin, drive the paint. doUpdate's send branch re-checks
662
- // firstPaintMin against the refreshed elapsed, so this is exact.
663
- if (h.messageId == null) {
664
- if (liveElapsed < firstPaintMin) continue
665
- const view = { ...h.lastView, elapsedMs: Math.max(h.lastView.elapsedMs, liveElapsed) }
666
- h.chain = h.chain
667
- .then(() => doUpdate(h, view))
808
+ if (now < g.cooldownUntil) continue
809
+ const running = runningRows(g)
810
+ if (running.length === 0) continue
811
+
812
+ // First-paint path: no message yet and some worker has now crossed
813
+ // firstPaintMin (a prose-silent worker's single early tick was held).
814
+ if (g.messageId == null) {
815
+ const maxElapsed = Math.max(0, ...running.map((r) => liveElapsed(r, now)))
816
+ if (maxElapsed < firstPaintMin) continue
817
+ g.chain = g.chain
818
+ .then(() => doRender(g, {}))
668
819
  .catch((err) => {
669
- log(`worker-feed: heartbeat first-paint chain error ${h.agentId}: ${(err as Error).message}`)
820
+ log(`worker-feed: heartbeat first-paint chain error feed=${g.feedKey}: ${(err as Error).message}`)
670
821
  })
671
822
  continue
672
823
  }
673
824
 
674
- if (now - h.lastEditAt < minEditInterval) continue
675
- const stale = now - h.lastEditAt >= heartbeatTickMs
825
+ if (now - g.lastEditAt < minEditInterval) continue
826
+ const stale = now - g.lastEditAt >= heartbeatTickMs
676
827
  if (!stale) continue
677
- // Per-step suffix: the CURRENT step's own elapsed (since the `→` line
678
- // last changed), never the worker total — the header already shows the
679
- // total, and repeating it on the step line was the Ken-observed dupe.
680
- // Under STEP_TIMER_MIN_MS formatStepSuffix returns '' (no timer yet);
681
- // the header elapsed still climbs via the refreshed view below.
682
- const stepElapsed = h.stepStartedAtMs != null ? now - h.stepStartedAtMs : liveElapsed
683
- const liveSuffix = formatStepSuffix(stepElapsed)
684
- // Re-render THROUGH the chain + doUpdate path — never editMessageText directly.
685
- //
686
- // CLOCK-ANCHOR PARITY: refresh the view's elapsedMs to the same `now`
687
- // anchor the step suffix uses. The header renders
688
- // `view.elapsedMs`; passing the stale lastView froze the header at the
689
- // last watcher event while the `· Ns` suffix kept ticking, so the
690
- // current step's timer could read MORE than the card's master elapsed
691
- // (Ken-observed defect). Both numbers now derive from one anchor
692
- // (dispatchAtMs) at one `now`, so header elapsed >= step suffix always.
693
- const view = { ...h.lastView, elapsedMs: Math.max(h.lastView.elapsedMs, liveElapsed) }
694
- h.chain = h.chain
695
- .then(() => doUpdate(h, view, liveSuffix))
828
+ // Re-render THROUGH the chain → doRender path with climbing elapsed.
829
+ g.chain = g.chain
830
+ .then(() => doRender(g, { heartbeat: true }))
696
831
  .catch((err) => {
697
- log(`worker-feed: heartbeat chain error ${h.agentId}: ${(err as Error).message}`)
832
+ log(`worker-feed: heartbeat chain error feed=${g.feedKey}: ${(err as Error).message}`)
698
833
  })
699
834
  }
700
835
  }
701
836
 
702
- // Arm the heartbeat once at construction. The real timer is `.unref()`'d so
703
- // it never keeps the process alive; tests inject setInterval/clearInterval.
704
837
  heartbeatTimer = setIntervalFn(heartbeatTick, heartbeatTickMs)
705
838
 
706
839
  return {
707
840
  has(agentId) {
708
- return handles.get(agentId)?.messageId != null
841
+ const g = groupOfAgent(agentId)
842
+ return g != null && g.messageId != null && g.workers.has(agentId)
709
843
  },
710
844
  messageIdOf(agentId) {
711
- return handles.get(agentId)?.messageId ?? null
845
+ return groupOfAgent(agentId)?.messageId ?? null
846
+ },
847
+ hasRunningInFeed(feedKey) {
848
+ const g = groups.get(feedKey)
849
+ return g != null && g.workers.size > 0
712
850
  },
713
851
  get size() {
714
- return handles.size
852
+ let n = 0
853
+ for (const g of groups.values()) n += g.workers.size
854
+ return n
715
855
  },
716
856
  update(agentId, chatId, view, threadId) {
717
- // No chat to post to (owner DM unconfigured) — don't create a
718
- // handle that would retry a failing send('') every tick.
857
+ // No chat to post to (owner DM unconfigured) — don't create state that
858
+ // would retry a failing send('') every tick.
719
859
  if (chatId.length === 0) return Promise.resolve()
720
- // Resurrection guard: a worker that has already been finalized
721
- // (`doFinish` latched `finalized`) must not get a fresh running cue.
722
- // A late watcher `onProgress` tick can arrive after `finish()`'s chain
723
- // has fully settled and the handle was deleted — without this durable
724
- // gate the tick would create a brand-new handle and paint a fresh
725
- // `running` message on an already-done worker (the card lies). The
726
- // heartbeat's orphan-paint guard covers the heartbeat tick only; the
727
- // per-handle `finished` flag covers the pre-delete window; this set
728
- // covers the post-delete window.
860
+ // Resurrection guard: a worker already finalized must not get a fresh
861
+ // running cue (a late watcher tick would repaint a done worker as live).
729
862
  if (finalized.has(agentId)) return Promise.resolve()
730
- const existing = handles.get(agentId)
731
- if (existing?.finished === true) return Promise.resolve()
732
- let h = existing
733
- if (h == null) {
734
- h = {
735
- agentId,
863
+ const existingRow = groupOfAgent(agentId)?.workers.get(agentId)
864
+ if (existingRow?.finished === true) return Promise.resolve()
865
+
866
+ const feedKey = feedKeyOf(chatId, threadId)
867
+ let g = groups.get(feedKey)
868
+ if (g == null) {
869
+ g = {
870
+ feedKey,
736
871
  chatId,
737
872
  threadId,
738
873
  messageId: null,
739
874
  lastBody: null,
740
875
  lastEditAt: 0,
741
876
  cooldownUntil: 0,
742
- narrative: [],
743
877
  chain: Promise.resolve(),
878
+ workers: new Map(),
879
+ pendingFinalize: null,
880
+ }
881
+ groups.set(feedKey, g)
882
+ }
883
+ let row = g.workers.get(agentId)
884
+ if (row == null) {
885
+ row = {
886
+ agentId,
887
+ narrative: [],
744
888
  lastView: null,
889
+ state: 'running',
890
+ finished: false,
745
891
  dispatchAtMs: null,
746
892
  stepStartedAtMs: null,
747
- finished: false,
748
- pendingFinish: null,
749
893
  }
750
- handles.set(agentId, h)
894
+ g.workers.set(agentId, row)
895
+ agentIndex.set(agentId, feedKey)
751
896
  }
752
- const handle = h
753
- handle.chain = handle.chain.then(() => doUpdate(handle, view)).catch((err) => {
897
+ // Accumulate before the gate so a throttled tick still grows the
898
+ // narrative — it surfaces on the next edit that does fire.
899
+ accumulateNarrative(row, view)
900
+ row.state = 'running'
901
+ row.lastView = { ...view, narrativeLines: [...row.narrative] }
902
+ if (row.dispatchAtMs == null) row.dispatchAtMs = nowFn() - view.elapsedMs
903
+
904
+ const group = g
905
+ group.chain = group.chain.then(() => doRender(group)).catch((err) => {
754
906
  log(`worker-feed: update chain error ${agentId}: ${(err as Error).message}`)
755
907
  })
756
- return handle.chain
908
+ return group.chain
757
909
  },
758
910
  finish(agentId, view) {
759
- const h = handles.get(agentId)
760
- if (h == null) return Promise.resolve()
761
- h.chain = h.chain
762
- .then(() => doFinish(h, view))
911
+ const g = groupOfAgent(agentId)
912
+ const row = g?.workers.get(agentId)
913
+ if (g == null || row == null) {
914
+ // Never tracked (trivial worker) — mark finalized so a late tick can't
915
+ // resurrect, and let the handback carry the result.
916
+ markFinalized(agentId)
917
+ return Promise.resolve()
918
+ }
919
+ // Latch synchronously so a late `running` cue on the chain can't resurrect.
920
+ row.finished = true
921
+ row.state = view.state === 'failed' ? 'failed' : 'done'
922
+ markFinalized(agentId)
923
+
924
+ const group = g
925
+ group.chain = group.chain
926
+ .then(() => {
927
+ const others = runningRows(group).filter((w) => w.agentId !== agentId)
928
+ if (others.length > 0) {
929
+ // Siblings still live → drop this row from the combined body and
930
+ // re-render the running set. The result reaches the user via the
931
+ // separate handback, never folded into this cosmetic edit. The
932
+ // group pin STAYS (siblings still need the shared message) — a
933
+ // per-worker unpin here was the #3207 review blocker.
934
+ removeWorker(group, agentId)
935
+ syncPin(group)
936
+ return doRender(group, { force: true })
937
+ }
938
+ // Last live worker → finalize the shared message to its terminal recap.
939
+ const recap: WorkerActivityView = { ...view, narrativeLines: [...row.narrative] }
940
+ return doRender(group, { force: true, terminalRecap: recap, finishingAgentId: agentId })
941
+ })
763
942
  .catch((err) => {
764
943
  log(`worker-feed: finish chain error ${agentId}: ${(err as Error).message}`)
765
944
  })
766
- .finally(() => {
767
- // Only tear down the handle once the terminal edit has actually
768
- // landed (or permanently failed). If `doFinish` staged the edit on
769
- // `pendingFinish` (a 429 cooldown / transient error was in effect),
770
- // the handle must survive so the heartbeat can re-drive the
771
- // finalize after cooldown. The heartbeat's re-drive chain ends by
772
- // re-entering `doFinish`, which clears `pendingFinish` on success
773
- // or permanent-failure — so this `.finally` deletes on the NEXT
774
- // chain settle once there is nothing left to finalize. Without this
775
- // guard, the `.finally` would delete the handle (and its staged
776
- // pendingFinish) immediately after the first staged doFinish,
777
- // stranding the card on its last running render.
778
- if (handles.get(agentId)?.pendingFinish == null) {
779
- handles.delete(agentId)
780
- }
781
- })
782
- return h.chain
945
+ return group.chain
783
946
  },
784
947
  drop(agentId) {
785
- // A dropped worker is also done — mark finalized so a late watcher
786
- // tick can't resurrect a running card on it (same gate as `finish`).
948
+ // A dropped worker is also done — mark finalized so a late tick can't
949
+ // resurrect a running card on it (same gate as `finish`).
787
950
  markFinalized(agentId)
788
- handles.delete(agentId)
951
+ const g = groupOfAgent(agentId)
952
+ if (g == null) return
953
+ const hadMessage = g.messageId != null
954
+ removeWorker(g, agentId)
955
+ // Group pin follows membership: unpin if this emptied the group, else
956
+ // keep it (siblings still need the shared message).
957
+ if (hadMessage) syncPin(g)
958
+ // Re-render so the dropped worker disappears from a combined body. Skip
959
+ // when the group is gone (removeWorker deleted it) or never painted.
960
+ if (hadMessage && groups.has(g.feedKey) && runningRows(g).length > 0) {
961
+ g.chain = g.chain
962
+ .then(() => doRender(g, { force: true }))
963
+ .catch((err) => {
964
+ log(`worker-feed: drop re-render error ${agentId}: ${(err as Error).message}`)
965
+ })
966
+ }
789
967
  },
790
968
  resurrect(agentId) {
791
969
  // Issue #3023: the worker's card was falsely finalized and its JSONL has
792
- // resumed. Re-open the paint path: drop the durable finalized gate so a
793
- // fresh `running` cue creates a new handle and first-paints a live card
794
- // again, and un-latch any surviving handle (finish deleted it in the
795
- // common case, but a staged pendingFinish could keep it alive). The next
796
- // `update` tick from the watcher's replayed progress does the repaint.
970
+ // resumed. Re-open the paint path: drop the durable finalized gate + any
971
+ // surviving per-row latch so a fresh `running` cue repaints a live card.
797
972
  const wasFinalized = finalized.delete(agentId)
798
- const h = handles.get(agentId)
799
- if (h != null) {
800
- h.finished = false
801
- h.pendingFinish = null
973
+ const row = groupOfAgent(agentId)?.workers.get(agentId)
974
+ if (row != null) {
975
+ row.finished = false
976
+ row.state = 'running'
802
977
  }
803
- if (wasFinalized || h != null) {
978
+ if (wasFinalized || row != null) {
804
979
  log(`worker-feed: resurrect agent=${agentId} — cleared finalized gate; card will repaint on next running cue`)
805
980
  }
806
981
  },