switchroom 0.18.17 → 0.18.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +13 -0
- package/dist/auth-broker/index.js +13 -0
- package/dist/cli/notion-write-pretool.mjs +13 -0
- package/dist/cli/switchroom.js +605 -479
- package/dist/host-control/main.js +17 -1
- package/dist/vault/approvals/kernel-server.js +13 -0
- package/dist/vault/broker/server.js +13 -0
- package/package.json +1 -1
- package/telegram-plugin/bridge/bridge.ts +7 -1
- package/telegram-plugin/dist/bridge/bridge.js +26 -1
- package/telegram-plugin/dist/gateway/gateway.js +1401 -431
- package/telegram-plugin/dist/server.js +26 -1
- package/telegram-plugin/fleet-fallback-resume.ts +26 -3
- package/telegram-plugin/gateway/approval-hold.ts +49 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +61 -18
- package/telegram-plugin/gateway/gateway.ts +362 -71
- package/telegram-plugin/gateway/linear-activity.ts +20 -4
- package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
- package/telegram-plugin/gateway/session-model-file.ts +103 -0
- package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
- package/telegram-plugin/llm-error-present.ts +436 -0
- package/telegram-plugin/operator-events.ts +7 -1
- package/telegram-plugin/permission-title.ts +172 -10
- package/telegram-plugin/premium-recovery.ts +101 -0
- package/telegram-plugin/raw-error-scrub.ts +73 -0
- package/telegram-plugin/retry-api-call.ts +8 -2
- package/telegram-plugin/send-gate-degraded.test.ts +152 -1
- package/telegram-plugin/send-gate-observability.test.ts +140 -0
- package/telegram-plugin/send-gate-observability.ts +65 -20
- package/telegram-plugin/send-gate.test.ts +143 -1
- package/telegram-plugin/send-gate.ts +212 -19
- package/telegram-plugin/session-tail.ts +16 -0
- package/telegram-plugin/shared/local-time.ts +69 -0
- package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
- package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +3 -2
- package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
- package/telegram-plugin/tests/llm-error-present.test.ts +380 -0
- package/telegram-plugin/tests/permission-title.test.ts +167 -4
- package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
- package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +6 -1
- package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
- package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
- package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
- package/telegram-plugin/tests/worker-activity-feed.test.ts +5 -2
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
- package/telegram-plugin/tier-downgrade.ts +198 -0
- package/telegram-plugin/tool-activity-summary.ts +99 -0
- package/telegram-plugin/worker-activity-feed.ts +509 -409
|
@@ -13,17 +13,34 @@
|
|
|
13
13
|
* that grows/edits as work happens — indistinguishable from "the agent
|
|
14
14
|
* is typing live", not a status widget.
|
|
15
15
|
*
|
|
16
|
-
*
|
|
17
|
-
* (
|
|
18
|
-
*
|
|
19
|
-
*
|
|
16
|
+
* COALESCED (#3084 follow-up): all live workers dispatched to the SAME
|
|
17
|
+
* chat/thread render into ONE shared message (a {@link FeedGroup}), not one
|
|
18
|
+
* message each. With N workers, N separate messages each coalesce their own
|
|
19
|
+
* edit stream but ALL draw from the send gate's 1/sec per-chat bucket — so
|
|
20
|
+
* ~N-1 of every second's edits SHED and every card refreshes only ~once per N
|
|
21
|
+
* seconds (liveness collapse). One combined message = ONE per-message edit
|
|
22
|
+
* stream: it coalesces (last-write-wins) and never contends with siblings, so
|
|
23
|
+
* every worker's row refreshes together within the ~1.5s edit floor. A single-
|
|
24
|
+
* worker chat still renders the full 🛠 Worker card (identical to before); a
|
|
25
|
+
* 2+ worker chat renders `renderCombinedWorkerFeed`.
|
|
26
|
+
*
|
|
27
|
+
* Pure render (`renderWorkerActivity` / `renderCombinedWorkerFeed`) + an
|
|
28
|
+
* injected bot API (`BotApiForWorkerFeed`). The manager
|
|
29
|
+
* (`createWorkerActivityFeed`) owns one edit-in-place message per (chat,thread)
|
|
30
|
+
* with:
|
|
20
31
|
* - a first-paint delay so trivial sub-second workers never post a
|
|
21
32
|
* message (their result still lands via the handback reply),
|
|
22
33
|
* - a proactive min-edit-interval throttle (worker jsonl ticks ~1/s;
|
|
23
34
|
* Telegram rate-limits edits) plus body-dedup,
|
|
24
|
-
* - per-
|
|
35
|
+
* - per-message serialization so two rapid ticks can't double-send,
|
|
25
36
|
* - 429 cooldown + message_id drift resilience (re-post on stale edit),
|
|
26
|
-
* - a forced terminal edit on
|
|
37
|
+
* - a forced terminal edit on the LAST worker's `finish`.
|
|
38
|
+
*
|
|
39
|
+
* The completed-worker RESULT is NOT folded into this cosmetic feed — it
|
|
40
|
+
* reaches the user as its own `useful` handback reply (gateway onFinish). The
|
|
41
|
+
* feed shows only *live* work; a finished worker's row is dropped from the
|
|
42
|
+
* combined body (or, when it is the last worker, the message finalizes to its
|
|
43
|
+
* terminal recap).
|
|
27
44
|
*
|
|
28
45
|
* The feed is gated to BACKGROUND workers and is ON by default; set
|
|
29
46
|
* `SWITCHROOM_WORKER_ACTIVITY_FEED=0` to disable it — see the gateway
|
|
@@ -38,7 +55,12 @@ import {
|
|
|
38
55
|
truncate,
|
|
39
56
|
} from './card-format.js'
|
|
40
57
|
import { STATUS_ROLLING_LINES } from './status-no-truncate.js'
|
|
41
|
-
import {
|
|
58
|
+
import {
|
|
59
|
+
renderStatusCard,
|
|
60
|
+
formatStepSuffix,
|
|
61
|
+
renderCombinedWorkerFeed,
|
|
62
|
+
type CombinedWorkerRow,
|
|
63
|
+
} from './tool-activity-summary.js'
|
|
42
64
|
import { isSendGateShed } from './send-gate.js'
|
|
43
65
|
|
|
44
66
|
/** Worker-activity feed is ON by default; an operator opts out with
|
|
@@ -208,7 +230,7 @@ export interface WorkerActivityFeedOpts {
|
|
|
208
230
|
* shed send every `heartbeatTickMs` (~6s) for the WHOLE ban — thousands of
|
|
209
231
|
* gate admissions, and (pre-fix) a `sent.message_id` crash on the `undefined`
|
|
210
232
|
* every tick. With it, a running/first-paint tick that sees an open window
|
|
211
|
-
* parks the
|
|
233
|
+
* parks the group in cooldown for the window's remaining and makes ZERO api
|
|
212
234
|
* calls until it closes, mirroring the held-card sweep's pre-send probe.
|
|
213
235
|
* Defaults to `() => 0` (no window) so tests and non-gateway callers are
|
|
214
236
|
* unchanged.
|
|
@@ -222,69 +244,110 @@ export interface WorkerActivityFeedOpts {
|
|
|
222
244
|
/** Heartbeat timer disposer. Injectable for tests. Defaults to `clearInterval`. */
|
|
223
245
|
clearInterval?: (handle: unknown) => void
|
|
224
246
|
/**
|
|
225
|
-
* Heartbeat tick cadence in ms. On each tick a stale, running
|
|
226
|
-
* re-rendered with
|
|
227
|
-
*
|
|
247
|
+
* Heartbeat tick cadence in ms. On each tick a stale, running feed is
|
|
248
|
+
* re-rendered with climbing elapsed so a worker that emits no new narrative
|
|
249
|
+
* still visibly advances. Default 6000ms.
|
|
228
250
|
*/
|
|
229
251
|
heartbeatTickMs?: number
|
|
252
|
+
/**
|
|
253
|
+
* Max worker rows rendered in a COMBINED feed (2+ workers in one chat/thread)
|
|
254
|
+
* before the `+M more working…` spill line. Keeps the coalesced body compact
|
|
255
|
+
* and legible (and under the rich-message wire ceiling). Default 8. A single-
|
|
256
|
+
* worker chat renders the full 🛠 Worker card and ignores this. Sourced from
|
|
257
|
+
* `channels.telegram.worker_feed.max_rows` via the config cascade.
|
|
258
|
+
*/
|
|
259
|
+
maxRows?: number
|
|
260
|
+
/**
|
|
261
|
+
* Group-level status-pin reconcile hook (#3207 review). Because workers now
|
|
262
|
+
* COALESCE into one shared message, the pin MUST follow the GROUP lifecycle,
|
|
263
|
+
* not a single worker's: pin the shared message when a group's first worker
|
|
264
|
+
* paints, and unpin ONLY when the group empties (its LAST worker finishes / is
|
|
265
|
+
* dropped). The feed alone knows group membership + running-count, so it owns
|
|
266
|
+
* the pin. `messageId: null` means "unpin this group". A per-worker unpin would
|
|
267
|
+
* physically unpin a message a SIBLING still needs, and the survivor's next
|
|
268
|
+
* pin request NO-OPs (its claim still names that id) — so it would run
|
|
269
|
+
* unpinned for the rest of its life (the shared-message NOOP trap). Best-
|
|
270
|
+
* effort; defaults to a noop for tests / non-gateway callers.
|
|
271
|
+
*/
|
|
272
|
+
reconcilePin?: (args: {
|
|
273
|
+
feedKey: string
|
|
274
|
+
chatId: string
|
|
275
|
+
threadId?: number
|
|
276
|
+
messageId: number | null
|
|
277
|
+
}) => void
|
|
230
278
|
}
|
|
231
279
|
|
|
232
|
-
|
|
280
|
+
/**
|
|
281
|
+
* One live worker's per-row state inside a chat/thread feed group. The row
|
|
282
|
+
* carries everything the render needs for THIS worker; the shared message
|
|
283
|
+
* (id, cooldown, chain) lives on the enclosing {@link FeedGroup}.
|
|
284
|
+
*/
|
|
285
|
+
interface WorkerRow {
|
|
233
286
|
/** jsonl agent id — carried so success/failure log lines can name the worker. */
|
|
234
287
|
agentId: string
|
|
235
|
-
chatId: string
|
|
236
|
-
threadId?: number
|
|
237
|
-
messageId: number | null
|
|
238
|
-
lastBody: string | null
|
|
239
|
-
lastEditAt: number
|
|
240
|
-
cooldownUntil: number
|
|
241
288
|
/**
|
|
242
|
-
* Accumulated narrative lines (oldest→newest), deduped
|
|
243
|
-
*
|
|
244
|
-
*
|
|
289
|
+
* Accumulated narrative lines (oldest→newest), deduped within the whole
|
|
290
|
+
* rolling window. Rolling-window capped to STATUS_ROLLING_LINES. Grows the
|
|
291
|
+
* live render so the feed reads like the main agent's answer.
|
|
245
292
|
*/
|
|
246
293
|
narrative: string[]
|
|
247
|
-
/**
|
|
248
|
-
chain: Promise<void>
|
|
249
|
-
/** Last view rendered into the message (drives the heartbeat re-render). */
|
|
294
|
+
/** Last view for this worker (drives the heartbeat re-render + combined row). */
|
|
250
295
|
lastView: WorkerActivityView | null
|
|
296
|
+
/** Latest state observed for this worker; excluded from the running set once
|
|
297
|
+
* terminal (its result reaches the user via the separate handback reply). */
|
|
298
|
+
state: WorkerActivityState
|
|
251
299
|
/**
|
|
252
|
-
*
|
|
253
|
-
*
|
|
254
|
-
*
|
|
255
|
-
* transport hiccup can't leave the card stuck on its last running render
|
|
256
|
-
* ("worker done, card says running"). Cleared on a successful terminal
|
|
257
|
-
* edit, on a permanent failure (message gone), or when the handle is
|
|
258
|
-
* deleted. Null when no finalize is pending.
|
|
259
|
-
*/
|
|
260
|
-
pendingFinish: WorkerActivityView | null
|
|
261
|
-
/**
|
|
262
|
-
* Latched in `doFinish` before the terminal edit. A late watcher
|
|
263
|
-
* `onProgress` tick that arrives after `finish()` queued its chain (but
|
|
264
|
-
* before the `.finally(handles.delete)` microtask drains) must NOT
|
|
265
|
-
* resurrect the handle and paint a fresh `running` message on an
|
|
266
|
-
* already-finalized worker. The heartbeat's orphan-paint guard
|
|
267
|
-
* (`if (!handles.has(h.agentId)) continue`) only covers the heartbeat
|
|
268
|
-
* tick — this flag covers the `update` entry point. Set synchronously
|
|
269
|
-
* inside `doFinish` (runs on the chain), checked synchronously in
|
|
270
|
-
* `update` before handle creation.
|
|
300
|
+
* Latched in `finish` before the terminal edit. A late watcher `onProgress`
|
|
301
|
+
* tick that arrives after `finish()` queued its chain must NOT resurrect the
|
|
302
|
+
* row and paint a fresh `running` state on an already-finalized worker.
|
|
271
303
|
*/
|
|
272
304
|
finished: boolean
|
|
273
305
|
/**
|
|
274
306
|
* Wall-clock ms the worker was dispatched, derived from `now - view.elapsedMs`
|
|
275
307
|
* on the first update. The heartbeat computes a live elapsed from this so the
|
|
276
|
-
*
|
|
308
|
+
* elapsed climbs even when no fresh view arrives, and it fixes the worker's
|
|
309
|
+
* stable sort order within the combined feed.
|
|
277
310
|
*/
|
|
278
311
|
dispatchAtMs: number | null
|
|
279
312
|
/**
|
|
280
313
|
* Wall-clock ms the CURRENT step started — stamped whenever a NEW narrative
|
|
281
314
|
* line lands (the `→` line changes). The heartbeat's step suffix shows the
|
|
282
|
-
* step's OWN elapsed from this anchor (
|
|
283
|
-
*
|
|
315
|
+
* step's OWN elapsed from this anchor (single-worker card only), and only
|
|
316
|
+
* once past STEP_TIMER_MIN_MS.
|
|
284
317
|
*/
|
|
285
318
|
stepStartedAtMs: number | null
|
|
286
319
|
}
|
|
287
320
|
|
|
321
|
+
/**
|
|
322
|
+
* One shared feed message per (chatId, threadId). ALL live workers dispatched
|
|
323
|
+
* to the same chat/thread render into this one message — so their edits form a
|
|
324
|
+
* single per-message edit stream under the send gate's 1/sec per-chat ceiling
|
|
325
|
+
* (last-write-wins coalescing + no-op skip) instead of N contending streams
|
|
326
|
+
* that shed (#3084). A single-worker group renders the full 🛠 Worker card;
|
|
327
|
+
* a 2+ worker group renders the combined `renderCombinedWorkerFeed` body.
|
|
328
|
+
*/
|
|
329
|
+
interface FeedGroup {
|
|
330
|
+
/** Stable key `${chatId} ${threadId ?? ''}`. */
|
|
331
|
+
feedKey: string
|
|
332
|
+
chatId: string
|
|
333
|
+
threadId?: number
|
|
334
|
+
messageId: number | null
|
|
335
|
+
lastBody: string | null
|
|
336
|
+
lastEditAt: number
|
|
337
|
+
cooldownUntil: number
|
|
338
|
+
/** Single serialization chain for the shared message — ticks can't interleave sends. */
|
|
339
|
+
chain: Promise<void>
|
|
340
|
+
/** Live workers in this group, keyed by agentId (insertion ≈ dispatch order). */
|
|
341
|
+
workers: Map<string, WorkerRow>
|
|
342
|
+
/**
|
|
343
|
+
* A terminal render (the last worker's recap) staged because a 429 cooldown /
|
|
344
|
+
* flood window blocked the edit. The heartbeat re-drives it once the cooldown
|
|
345
|
+
* expires so a finished feed can't get stuck on its last running render.
|
|
346
|
+
* Null when no finalize is pending.
|
|
347
|
+
*/
|
|
348
|
+
pendingFinalize: WorkerActivityView | null
|
|
349
|
+
}
|
|
350
|
+
|
|
288
351
|
const COOLDOWN_JITTER_MS = 500
|
|
289
352
|
|
|
290
353
|
function extractRetryAfterSecs(err: unknown): number | null {
|
|
@@ -307,7 +370,7 @@ function extractRetryAfterSecs(err: unknown): number | null {
|
|
|
307
370
|
* 'rate_limited' — 429 with retry_after. Back off; the heartbeat re-drives
|
|
308
371
|
* the edit after cooldown (running renders + deferred terminal edits).
|
|
309
372
|
* 'gone' — message/chat deleted or edit window expired. Nothing to
|
|
310
|
-
* update; drop the
|
|
373
|
+
* update; drop the message silently (no warning — there is no card).
|
|
311
374
|
* 'transient' — anything else (network blip, 5xx). Retry on the next
|
|
312
375
|
* heartbeat tick; don't spam stderr.
|
|
313
376
|
*/
|
|
@@ -338,18 +401,25 @@ function classifyEditError(err: unknown): EditOutcome {
|
|
|
338
401
|
}
|
|
339
402
|
|
|
340
403
|
/**
|
|
341
|
-
* Manager owning one live message per
|
|
342
|
-
*
|
|
343
|
-
*
|
|
344
|
-
*
|
|
404
|
+
* Manager owning one live message per (chat,thread) into which all live
|
|
405
|
+
* workers there coalesce. Public methods stay keyed by jsonl agent id (the
|
|
406
|
+
* gateway wiring is unchanged): the manager resolves the enclosing feed group
|
|
407
|
+
* internally. The gateway calls `update` on each watcher activity cue and
|
|
408
|
+
* `finish` on terminal; `drop` discards a worker's state without a final edit
|
|
409
|
+
* (error / supersession paths).
|
|
345
410
|
*/
|
|
346
411
|
export interface WorkerActivityFeed {
|
|
347
|
-
/** True if a message is currently posted for this worker. */
|
|
412
|
+
/** True if a message is currently posted for this worker's feed group. */
|
|
348
413
|
has(agentId: string): boolean
|
|
349
|
-
/** The Telegram message_id currently posted for this worker, or
|
|
350
|
-
* none is posted (never painted, or dropped after a stale-edit
|
|
351
|
-
* Lets the gateway pin the EXISTING `🛠 Worker` message
|
|
414
|
+
/** The Telegram message_id currently posted for this worker's feed group, or
|
|
415
|
+
* null if none is posted (never painted, or dropped after a stale-edit
|
|
416
|
+
* re-post). Lets the gateway pin the EXISTING `🛠 Worker` message. Note:
|
|
417
|
+
* siblings sharing the chat/thread return the SAME id (one message). */
|
|
352
418
|
messageIdOf(agentId: string): number | null
|
|
419
|
+
/** True while the feed group `feedKey` (`${chatId} ${threadId ?? ''}`) still
|
|
420
|
+
* tracks live work — used by the gateway's `wk:group:` pin reaper to exempt
|
|
421
|
+
* a live group's pin from the stale-TTL sweep (#3207). */
|
|
422
|
+
hasRunningInFeed(feedKey: string): boolean
|
|
353
423
|
/** Push a running-state cue. Returns the serialized op for tests. */
|
|
354
424
|
update(
|
|
355
425
|
agentId: string,
|
|
@@ -357,13 +427,17 @@ export interface WorkerActivityFeed {
|
|
|
357
427
|
view: WorkerActivityView,
|
|
358
428
|
threadId?: number,
|
|
359
429
|
): Promise<void>
|
|
360
|
-
/**
|
|
430
|
+
/** Finalize a worker: drop its row from the combined feed (its result reaches
|
|
431
|
+
* the user via the separate handback), or — when it is the last live worker
|
|
432
|
+
* in the group — force the terminal recap edit. No-op if the worker was
|
|
433
|
+
* never tracked. */
|
|
361
434
|
finish(agentId: string, view: WorkerActivityView): Promise<void>
|
|
362
|
-
/** Forget a worker's state without
|
|
435
|
+
/** Forget a worker's state without a recap edit (e.g. error path); re-renders
|
|
436
|
+
* the group so the dropped worker disappears from the combined body. */
|
|
363
437
|
drop(agentId: string): void
|
|
364
438
|
/**
|
|
365
439
|
* Issue #3023 (card resurrection). Undo a finalization: clear the durable
|
|
366
|
-
* `finalized` gate (and any lingering per-
|
|
440
|
+
* `finalized` gate (and any lingering per-row `finished` latch) so a
|
|
367
441
|
* worker whose card was FALSELY finalized can be painted/edited again. The
|
|
368
442
|
* watcher calls this (via the gateway's `onResurrect` wiring) when a
|
|
369
443
|
* falsely-finalized worker's JSONL resumes growing. A fresh `running` cue
|
|
@@ -375,7 +449,7 @@ export interface WorkerActivityFeed {
|
|
|
375
449
|
stop(): void
|
|
376
450
|
/** Manually fire one heartbeat tick (test hook). */
|
|
377
451
|
heartbeatTick(): void
|
|
378
|
-
/** Number of tracked workers (test/inspection hook). */
|
|
452
|
+
/** Number of tracked workers across all feed groups (test/inspection hook). */
|
|
379
453
|
readonly size: number
|
|
380
454
|
}
|
|
381
455
|
|
|
@@ -386,6 +460,8 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
386
460
|
const minEditInterval = opts.minEditIntervalMs ?? 2500
|
|
387
461
|
const firstPaintMin = opts.firstPaintMinMs ?? 8000
|
|
388
462
|
const heartbeatTickMs = opts.heartbeatTickMs ?? 6000
|
|
463
|
+
const maxRows = Math.max(1, Math.floor(opts.maxRows ?? 8))
|
|
464
|
+
const reconcilePinFn = opts.reconcilePin ?? (() => {})
|
|
389
465
|
const setIntervalFn =
|
|
390
466
|
opts.setInterval ??
|
|
391
467
|
((cb: () => void, ms: number): unknown => {
|
|
@@ -395,17 +471,21 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
395
471
|
return t
|
|
396
472
|
})
|
|
397
473
|
const clearIntervalFn = opts.clearInterval ?? ((handle: unknown) => clearInterval(handle as ReturnType<typeof setInterval>))
|
|
398
|
-
|
|
474
|
+
|
|
475
|
+
/** Feed groups keyed by `${chatId} ${threadId ?? ''}`. */
|
|
476
|
+
const groups = new Map<string, FeedGroup>()
|
|
477
|
+
/** Reverse index agentId → feedKey, so the agentId-keyed public API resolves
|
|
478
|
+
* its group in O(1). Cleared when a worker's row is removed. */
|
|
479
|
+
const agentIndex = new Map<string, string>()
|
|
480
|
+
|
|
399
481
|
/**
|
|
400
|
-
* Agent ids that have been finalized (`
|
|
482
|
+
* Agent ids that have been finalized (`finish` latched). Survives row/group
|
|
401
483
|
* deletion so a LATE watcher `onProgress` tick — which can arrive after
|
|
402
|
-
* `finish()`'s chain has fully settled
|
|
403
|
-
*
|
|
404
|
-
*
|
|
405
|
-
*
|
|
406
|
-
*
|
|
407
|
-
* only needs to cover recent finalizations — capped at FINALIZED_CAP and
|
|
408
|
-
* trimmed FIFO to stay bounded across a long gateway lifetime.
|
|
484
|
+
* `finish()`'s chain has fully settled — cannot resurrect a fresh row and
|
|
485
|
+
* paint a running card on a worker that is already done. A late tick arrives
|
|
486
|
+
* within seconds of finish (watcher poll cadence), so the set only needs to
|
|
487
|
+
* cover recent finalizations — capped at FINALIZED_CAP and trimmed FIFO to
|
|
488
|
+
* stay bounded across a long gateway lifetime.
|
|
409
489
|
*/
|
|
410
490
|
const finalized = new Set<string>()
|
|
411
491
|
const FINALIZED_CAP = 256
|
|
@@ -413,469 +493,489 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
413
493
|
if (finalized.has(agentId)) return
|
|
414
494
|
finalized.add(agentId)
|
|
415
495
|
if (finalized.size > FINALIZED_CAP) {
|
|
416
|
-
// Map-free FIFO trim: Set iterates in insertion order; drop the oldest.
|
|
417
496
|
const oldest = finalized.values().next().value
|
|
418
497
|
if (oldest != null) finalized.delete(oldest)
|
|
419
498
|
}
|
|
420
499
|
}
|
|
421
500
|
let heartbeatTimer: unknown = null
|
|
422
501
|
|
|
423
|
-
function
|
|
502
|
+
function feedKeyOf(chatId: string, threadId?: number): string {
|
|
503
|
+
return `${chatId} ${threadId ?? ''}`
|
|
504
|
+
}
|
|
505
|
+
function groupOfAgent(agentId: string): FeedGroup | undefined {
|
|
506
|
+
const key = agentIndex.get(agentId)
|
|
507
|
+
return key != null ? groups.get(key) : undefined
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
function sendOptsFor(g: FeedGroup): Record<string, unknown> {
|
|
424
511
|
return {
|
|
425
512
|
disable_web_page_preview: true,
|
|
426
|
-
// Sub-agent progress
|
|
427
|
-
//
|
|
428
|
-
//
|
|
429
|
-
// that share these opts.)
|
|
513
|
+
// Sub-agent progress feed is a status surface, never the user's answer —
|
|
514
|
+
// silence the open ping. (editMessageText ignores disable_notification,
|
|
515
|
+
// so this is a no-op on the in-place edits that share these opts.)
|
|
430
516
|
disable_notification: true,
|
|
431
|
-
...(
|
|
517
|
+
...(g.threadId != null ? { message_thread_id: g.threadId } : {}),
|
|
432
518
|
}
|
|
433
519
|
}
|
|
434
520
|
|
|
435
|
-
function noteRateLimited(
|
|
521
|
+
function noteRateLimited(g: FeedGroup, err: unknown, label: string): void {
|
|
436
522
|
const retryAfter = extractRetryAfterSecs(err)
|
|
437
523
|
if (retryAfter == null) return
|
|
438
|
-
|
|
524
|
+
g.cooldownUntil = nowFn() + retryAfter * 1000 + COOLDOWN_JITTER_MS
|
|
439
525
|
log(`worker-feed: ${label} 429 — backing off ${retryAfter}s`)
|
|
440
526
|
}
|
|
441
527
|
|
|
442
528
|
/**
|
|
443
|
-
* Park a
|
|
444
|
-
* any), so the heartbeat's `nowFn() <
|
|
529
|
+
* Park a group in cooldown for the remaining of an open flood window (if
|
|
530
|
+
* any), so the heartbeat's `nowFn() < g.cooldownUntil` guard suppresses every
|
|
445
531
|
* further send/edit until the ban closes. Returns true when a window was open
|
|
446
532
|
* (caller should abandon the current attempt). Two reads of the SAME on-disk
|
|
447
533
|
* marker `robustApiCall` gates on — never a second notion of "channel open".
|
|
448
534
|
*/
|
|
449
|
-
function parkIfFloodWindowOpen(
|
|
535
|
+
function parkIfFloodWindowOpen(g: FeedGroup): boolean {
|
|
450
536
|
const remaining = floodWaitRemainingMs()
|
|
451
537
|
if (remaining <= 0) return false
|
|
452
|
-
// Only extend the cooldown — never shorten one a 429 already set longer.
|
|
453
538
|
const until = nowFn() + remaining + COOLDOWN_JITTER_MS
|
|
454
|
-
if (until >
|
|
539
|
+
if (until > g.cooldownUntil) g.cooldownUntil = until
|
|
455
540
|
return true
|
|
456
541
|
}
|
|
457
542
|
|
|
458
|
-
function accumulateNarrative(
|
|
543
|
+
function accumulateNarrative(row: WorkerRow, view: WorkerActivityView): void {
|
|
459
544
|
const line = view.latestSummary.trim()
|
|
460
545
|
if (line.length === 0) return
|
|
461
|
-
// Dedup within the whole rolling window
|
|
462
|
-
//
|
|
463
|
-
//
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
//
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
546
|
+
// Dedup within the whole rolling window (the watcher re-emits the same
|
|
547
|
+
// narrative across ticks, and a preamble + its tool label can repeat
|
|
548
|
+
// non-adjacently — the A,B,A duplication observed on live cards).
|
|
549
|
+
if (row.narrative.includes(line)) return
|
|
550
|
+
row.narrative.push(line)
|
|
551
|
+
// The `→` current-step line just CHANGED — reset the per-step timer.
|
|
552
|
+
row.stepStartedAtMs = nowFn()
|
|
553
|
+
if (row.narrative.length > STATUS_ROLLING_LINES) {
|
|
554
|
+
row.narrative.splice(0, row.narrative.length - STATUS_ROLLING_LINES)
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/** Live wall-clock elapsed for a worker (climbs between fresh views). */
|
|
559
|
+
function liveElapsed(row: WorkerRow, now: number): number {
|
|
560
|
+
const base = row.dispatchAtMs != null ? now - row.dispatchAtMs : row.lastView?.elapsedMs ?? 0
|
|
561
|
+
return Math.max(base, row.lastView?.elapsedMs ?? 0)
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
/** The running rows of a group, dispatch-ordered (oldest first, stable). */
|
|
565
|
+
function runningRows(g: FeedGroup): WorkerRow[] {
|
|
566
|
+
return [...g.workers.values()]
|
|
567
|
+
.filter((w) => w.state === 'running' && w.lastView != null)
|
|
568
|
+
.sort((a, b) => (a.dispatchAtMs ?? 0) - (b.dispatchAtMs ?? 0))
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
/**
|
|
572
|
+
* Render a group's shared message body at `now`.
|
|
573
|
+
* - `terminalRecap` set + zero running → the last worker's terminal recap
|
|
574
|
+
* (single 🛠 Worker card, done/failed).
|
|
575
|
+
* - exactly one running → the full 🛠 Worker card (single-worker parity).
|
|
576
|
+
* - 2+ running → the combined `renderCombinedWorkerFeed` body.
|
|
577
|
+
* - zero running, no recap → null (nothing to show).
|
|
578
|
+
* `heartbeat` toggles the single-worker climbing `· Ns` step suffix.
|
|
579
|
+
*/
|
|
580
|
+
function renderGroupBody(
|
|
581
|
+
g: FeedGroup,
|
|
582
|
+
now: number,
|
|
583
|
+
terminalRecap: WorkerActivityView | null,
|
|
584
|
+
heartbeat: boolean,
|
|
585
|
+
): string | null {
|
|
586
|
+
const running = runningRows(g)
|
|
587
|
+
if (running.length === 0) {
|
|
588
|
+
if (terminalRecap == null) return null
|
|
589
|
+
return renderWorkerActivity(terminalRecap)
|
|
590
|
+
}
|
|
591
|
+
// On a normal update/finish the header shows the worker's LAST-REPORTED
|
|
592
|
+
// elapsed (byte-stable for the dedup / no-op skip). Only the heartbeat —
|
|
593
|
+
// which fires when no fresh view arrived — climbs a live wall-clock elapsed
|
|
594
|
+
// so a silent worker still visibly advances.
|
|
595
|
+
const elapsedFor = (r: WorkerRow): number =>
|
|
596
|
+
heartbeat ? liveElapsed(r, now) : r.lastView?.elapsedMs ?? liveElapsed(r, now)
|
|
597
|
+
if (running.length === 1) {
|
|
598
|
+
const r = running[0]
|
|
599
|
+
const view: WorkerActivityView = {
|
|
600
|
+
...(r.lastView as WorkerActivityView),
|
|
601
|
+
elapsedMs: elapsedFor(r),
|
|
602
|
+
narrativeLines: [...r.narrative],
|
|
603
|
+
}
|
|
604
|
+
let liveSuffix = ''
|
|
605
|
+
if (heartbeat) {
|
|
606
|
+
const stepElapsed = r.stepStartedAtMs != null ? now - r.stepStartedAtMs : liveElapsed(r, now)
|
|
607
|
+
liveSuffix = formatStepSuffix(stepElapsed)
|
|
608
|
+
}
|
|
609
|
+
return renderWorkerActivity(view, liveSuffix)
|
|
479
610
|
}
|
|
611
|
+
const rows: CombinedWorkerRow[] = running.map((r) => {
|
|
612
|
+
const v = r.lastView as WorkerActivityView
|
|
613
|
+
const currentStep = r.narrative.length > 0 ? r.narrative[r.narrative.length - 1] : v.latestSummary
|
|
614
|
+
return {
|
|
615
|
+
description: v.description,
|
|
616
|
+
elapsedMs: elapsedFor(r),
|
|
617
|
+
toolCount: v.toolCount,
|
|
618
|
+
currentStep,
|
|
619
|
+
model: v.model,
|
|
620
|
+
}
|
|
621
|
+
})
|
|
622
|
+
return renderCombinedWorkerFeed(rows, { maxRows })
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/** Remove a worker's row + index entry; delete the group if it is now empty. */
|
|
626
|
+
function removeWorker(g: FeedGroup, agentId: string): void {
|
|
627
|
+
g.workers.delete(agentId)
|
|
628
|
+
agentIndex.delete(agentId)
|
|
629
|
+
if (g.workers.size === 0) groups.delete(g.feedKey)
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
/**
|
|
633
|
+
* Reconcile the GROUP-level status pin (#3207 review). Pin the shared message
|
|
634
|
+
* while the group has a posted message AND at least one tracked worker;
|
|
635
|
+
* unpin the instant the group empties. Because it is keyed by the whole group
|
|
636
|
+
* (not a single worker), a per-worker finish never unpins a message a sibling
|
|
637
|
+
* still needs — the survivors keep the pin until the LAST worker is done.
|
|
638
|
+
*/
|
|
639
|
+
function syncPin(g: FeedGroup): void {
|
|
640
|
+
const messageId = g.messageId != null && g.workers.size > 0 ? g.messageId : null
|
|
641
|
+
reconcilePinFn({ feedKey: g.feedKey, chatId: g.chatId, threadId: g.threadId, messageId })
|
|
480
642
|
}
|
|
481
643
|
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
//
|
|
496
|
-
//
|
|
497
|
-
|
|
498
|
-
const
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
644
|
+
/**
|
|
645
|
+
* Drive the group's shared message to the current combined body. Handles
|
|
646
|
+
* first-paint gating, the proactive throttle (bypassed by `force`), the
|
|
647
|
+
* dedup/no-op skip, the send-gate SHED contract, and 429/flood cooldown.
|
|
648
|
+
* `terminalRecap` (set on the last worker's finish) renders + finalizes the
|
|
649
|
+
* message, then removes the finished row.
|
|
650
|
+
*/
|
|
651
|
+
async function doRender(
|
|
652
|
+
g: FeedGroup,
|
|
653
|
+
opts2: { force?: boolean; heartbeat?: boolean; terminalRecap?: WorkerActivityView; finishingAgentId?: string } = {},
|
|
654
|
+
): Promise<void> {
|
|
655
|
+
const now = nowFn()
|
|
656
|
+
const isTerminal = opts2.terminalRecap != null
|
|
657
|
+
// Terminal edit RESOLVED (landed / not-modified / gone): clear the staged
|
|
658
|
+
// re-drive unconditionally so a heartbeat re-drive can never loop, and drop
|
|
659
|
+
// the finished row when its id is known (the last-worker finalize path).
|
|
660
|
+
const settleTerminal = (): void => {
|
|
661
|
+
g.pendingFinalize = null
|
|
662
|
+
if (opts2.finishingAgentId != null) removeWorker(g, opts2.finishingAgentId)
|
|
663
|
+
// Group-level pin follows membership: unpin once this drops the last
|
|
664
|
+
// worker; a NOOP-pin (siblings remain) keeps the shared message pinned.
|
|
665
|
+
syncPin(g)
|
|
666
|
+
}
|
|
667
|
+
if (now < g.cooldownUntil) {
|
|
668
|
+
if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
|
|
669
|
+
return
|
|
670
|
+
}
|
|
671
|
+
// A flood window is open: the gate would SHED every call. Park in cooldown
|
|
672
|
+
// and make ZERO api calls until it closes; the heartbeat re-drives.
|
|
673
|
+
if (parkIfFloodWindowOpen(g)) {
|
|
674
|
+
if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
|
|
675
|
+
return
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
const body = renderGroupBody(g, now, opts2.terminalRecap ?? null, opts2.heartbeat ?? false)
|
|
679
|
+
if (body == null) {
|
|
680
|
+
// Nothing to show. On a terminal finalize with no message ever posted,
|
|
681
|
+
// just drop the finished row (the handback carries the result).
|
|
682
|
+
if (isTerminal) settleTerminal()
|
|
683
|
+
return
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
// First paint: hold until some worker in the group has run long enough.
|
|
687
|
+
if (g.messageId == null) {
|
|
688
|
+
const maxElapsed = Math.max(0, ...runningRows(g).map((r) => liveElapsed(r, now)))
|
|
689
|
+
// A terminal recap for a group that never painted → nothing to finalize;
|
|
690
|
+
// never first-paint a terminal card (trivial workers stay silent, the
|
|
691
|
+
// handback carries the result — matches the pre-coalesce doFinish guard).
|
|
692
|
+
if (isTerminal) {
|
|
693
|
+
settleTerminal()
|
|
694
|
+
return
|
|
695
|
+
}
|
|
696
|
+
if (maxElapsed < firstPaintMin) return
|
|
504
697
|
try {
|
|
505
|
-
const sent = await opts.bot.sendMessage(
|
|
506
|
-
// Shed contract (#3084): the gate resolves `undefined
|
|
507
|
-
// shed (open flood window) or dropped
|
|
508
|
-
// NOT a delivered message —
|
|
509
|
-
//
|
|
510
|
-
// whole 6h ban. Treat it as not-delivered: record no message_id, park
|
|
511
|
-
// on any open window, and let the heartbeat re-drive the paint.
|
|
698
|
+
const sent = await opts.bot.sendMessage(g.chatId, body, sendOptsFor(g))
|
|
699
|
+
// Shed contract (#3084): the gate resolves `undefined`/non-object for a
|
|
700
|
+
// shed send (open flood window) or dropped-stale `useful` TTL. That is
|
|
701
|
+
// NOT a delivered message — record no id, park on any window, let the
|
|
702
|
+
// heartbeat re-drive.
|
|
512
703
|
if (sent == null || typeof sent.message_id !== 'number') {
|
|
513
|
-
parkIfFloodWindowOpen(
|
|
514
|
-
log(`worker-feed: first paint shed by send gate
|
|
704
|
+
parkIfFloodWindowOpen(g)
|
|
705
|
+
log(`worker-feed: first paint shed by send gate feed=${g.feedKey} — not delivered`)
|
|
515
706
|
return
|
|
516
707
|
}
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
708
|
+
g.messageId = sent.message_id
|
|
709
|
+
g.lastBody = body
|
|
710
|
+
g.lastEditAt = now
|
|
711
|
+
// Group's first (or re-established) message is up → pin it for the group.
|
|
712
|
+
syncPin(g)
|
|
520
713
|
log(
|
|
521
|
-
`worker-feed: paint
|
|
522
|
-
`thread=${
|
|
714
|
+
`worker-feed: paint feed=${g.feedKey} chat=${g.chatId} ` +
|
|
715
|
+
`thread=${g.threadId ?? '-'} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`,
|
|
523
716
|
)
|
|
524
717
|
} catch (err) {
|
|
525
|
-
noteRateLimited(
|
|
718
|
+
noteRateLimited(g, err, 'send')
|
|
526
719
|
log(`worker-feed: send failed: ${(err as Error).message}`)
|
|
527
720
|
}
|
|
528
721
|
return
|
|
529
722
|
}
|
|
530
723
|
|
|
531
|
-
// Dedup + proactive throttle.
|
|
532
|
-
if (body ===
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
try {
|
|
536
|
-
const res = await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h))
|
|
537
|
-
// Shed honesty (#3084, mirrors #3173): a cosmetic edit the gate shed
|
|
538
|
-
// resolves the distinguishable SEND_GATE_SHED sentinel (#3110 F1 — NOT a
|
|
539
|
-
// bare `undefined`, which the gate reserves for a benign no-op drop whose
|
|
540
|
-
// payload IS already on screen). The shed payload is NOT on screen, so do
|
|
541
|
-
// not record it as `lastBody`, or the next tick's dedup would skip
|
|
542
|
-
// re-sending the very update the gate dropped. Park on any open window and
|
|
543
|
-
// let the heartbeat re-drive once it closes. A benign `undefined`/`true`
|
|
544
|
-
// falls through below and is correctly recorded as delivered.
|
|
545
|
-
if (isSendGateShed(res)) {
|
|
546
|
-
parkIfFloodWindowOpen(h)
|
|
547
|
-
return
|
|
548
|
-
}
|
|
549
|
-
h.lastBody = body
|
|
550
|
-
h.lastEditAt = nowFn()
|
|
551
|
-
log(
|
|
552
|
-
`worker-feed: edit agent=${h.agentId} chat=${h.chatId} ` +
|
|
553
|
-
`thread=${h.threadId ?? '-'} msgId=${h.messageId} bytes=${body.length}`,
|
|
554
|
-
)
|
|
555
|
-
} catch (err) {
|
|
556
|
-
const outcome = classifyEditError(err)
|
|
557
|
-
if (outcome === 'rate_limited') {
|
|
558
|
-
noteRateLimited(h, err, 'edit')
|
|
559
|
-
return
|
|
560
|
-
}
|
|
561
|
-
if (outcome === 'not_modified') {
|
|
562
|
-
// Card already shows this body — record it as landed and move on.
|
|
563
|
-
h.lastBody = body
|
|
564
|
-
h.lastEditAt = nowFn()
|
|
565
|
-
return
|
|
566
|
-
}
|
|
567
|
-
if (outcome === 'gone') {
|
|
568
|
-
// Message/chat deleted or edit window closed — there is no card to
|
|
569
|
-
// update. Drop the handle silently; a fresh first-paint on the next
|
|
570
|
-
// running tick re-establishes one if the worker is still active. No
|
|
571
|
-
// warning: "no card" is not a liveness-logic error.
|
|
572
|
-
h.messageId = null
|
|
573
|
-
h.lastBody = null
|
|
574
|
-
return
|
|
575
|
-
}
|
|
576
|
-
// 'transient' — network blip / 5xx. Leave the handle intact; the
|
|
577
|
-
// heartbeat re-attempts on its next tick. Log at debug, not stderr-warn:
|
|
578
|
-
// a transport hiccup on a best-effort card is not "shit code", it's a
|
|
579
|
-
// retryable blip the framework rides out deterministically.
|
|
580
|
-
log(`worker-feed: edit transient error agent=${h.agentId}: ${(err as Error).message}`)
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
async function doFinish(h: WorkerHandle, view: WorkerActivityView): Promise<void> {
|
|
585
|
-
// Latch FIRST, before any early return. A `running`-cue tick arriving
|
|
586
|
-
// after `finish()` queued this chain (but before its `.finally(delete)`
|
|
587
|
-
// drains) would otherwise resurrect a handle via `update()` and paint a
|
|
588
|
-
// fresh running message on a finalized worker. Setting this synchronously
|
|
589
|
-
// on the chain — ahead of the cooldown/no-message guards — makes the
|
|
590
|
-
// gate in `update()` authoritative regardless of which guard path runs.
|
|
591
|
-
// The durable `finalized` set survives the subsequent handle deletion so
|
|
592
|
-
// a tick arriving AFTER the full settle still can't resurrect.
|
|
593
|
-
h.finished = true
|
|
594
|
-
markFinalized(h.agentId)
|
|
595
|
-
// No message ever posted → nothing to finalize. The worker's result
|
|
596
|
-
// reaches the user via the handback reply; a bare "done" recap with
|
|
597
|
-
// no preceding activity would be noise.
|
|
598
|
-
if (h.messageId == null) {
|
|
599
|
-
h.pendingFinish = null
|
|
600
|
-
return
|
|
601
|
-
}
|
|
602
|
-
// A flood window is open (or a prior 429 set a cooldown): honour it — a
|
|
603
|
-
// terminal edit isn't worth extending a ban. STAGE the terminal view so the
|
|
604
|
-
// heartbeat re-drives the finalize the instant the window/cooldown expires,
|
|
605
|
-
// so a finished worker's card can't get stuck on its last running render.
|
|
606
|
-
if (parkIfFloodWindowOpen(h) || nowFn() < h.cooldownUntil) {
|
|
607
|
-
h.pendingFinish = view
|
|
608
|
-
return
|
|
609
|
-
}
|
|
610
|
-
const body = renderWorkerActivity({ ...view, narrativeLines: h.narrative })
|
|
611
|
-
if (body === h.lastBody) {
|
|
612
|
-
h.pendingFinish = null
|
|
724
|
+
// Dedup + proactive throttle (finish/terminal edits force through).
|
|
725
|
+
if (body === g.lastBody) {
|
|
726
|
+
if (isTerminal) settleTerminal()
|
|
613
727
|
return
|
|
614
728
|
}
|
|
729
|
+
if (!opts2.force && now - g.lastEditAt < minEditInterval) return
|
|
730
|
+
|
|
615
731
|
try {
|
|
616
|
-
const res = await opts.bot.editMessageText(
|
|
617
|
-
// Shed honesty (#3084): a
|
|
618
|
-
// SEND_GATE_SHED sentinel (
|
|
619
|
-
// gate
|
|
620
|
-
//
|
|
621
|
-
// record `lastBody` (that would be a false finalization — card frozen on
|
|
622
|
-
// its running render while we believe it's done). Park on any open window.
|
|
732
|
+
const res = await opts.bot.editMessageText(g.chatId, g.messageId, body, sendOptsFor(g))
|
|
733
|
+
// Shed honesty (#3084): a cosmetic edit the gate shed resolves the
|
|
734
|
+
// distinguishable SEND_GATE_SHED sentinel (NOT a bare `undefined`, which
|
|
735
|
+
// the gate reserves for a benign no-op drop whose payload IS on screen).
|
|
736
|
+
// The shed payload is NOT on screen, so do not record it as `lastBody`.
|
|
623
737
|
if (isSendGateShed(res)) {
|
|
624
|
-
parkIfFloodWindowOpen(
|
|
625
|
-
|
|
738
|
+
parkIfFloodWindowOpen(g)
|
|
739
|
+
if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
|
|
626
740
|
return
|
|
627
741
|
}
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
742
|
+
g.lastBody = body
|
|
743
|
+
g.lastEditAt = now
|
|
744
|
+
if (isTerminal) {
|
|
745
|
+
log(
|
|
746
|
+
`worker-feed: finish feed=${g.feedKey} chat=${g.chatId} thread=${g.threadId ?? '-'} ` +
|
|
747
|
+
`msgId=${g.messageId} agent=${opts2.finishingAgentId ?? '-'} ` +
|
|
748
|
+
`state=${opts2.terminalRecap?.state ?? 'done'} bytes=${body.length}`,
|
|
749
|
+
)
|
|
750
|
+
} else {
|
|
751
|
+
log(
|
|
752
|
+
`worker-feed: edit feed=${g.feedKey} chat=${g.chatId} ` +
|
|
753
|
+
`thread=${g.threadId ?? '-'} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`,
|
|
754
|
+
)
|
|
755
|
+
}
|
|
756
|
+
if (isTerminal) settleTerminal()
|
|
635
757
|
} catch (err) {
|
|
636
758
|
const outcome = classifyEditError(err)
|
|
637
759
|
if (outcome === 'rate_limited') {
|
|
638
|
-
noteRateLimited(
|
|
639
|
-
|
|
640
|
-
h.pendingFinish = view
|
|
760
|
+
noteRateLimited(g, err, isTerminal ? 'finish' : 'edit')
|
|
761
|
+
if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
|
|
641
762
|
return
|
|
642
763
|
}
|
|
643
764
|
if (outcome === 'not_modified') {
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
h.pendingFinish = null
|
|
765
|
+
g.lastBody = body
|
|
766
|
+
g.lastEditAt = now
|
|
767
|
+
if (isTerminal) settleTerminal()
|
|
648
768
|
return
|
|
649
769
|
}
|
|
650
770
|
if (outcome === 'gone') {
|
|
651
|
-
// Message/chat gone — no card to
|
|
652
|
-
//
|
|
653
|
-
|
|
771
|
+
// Message/chat gone or edit window closed — no card to update. Drop the
|
|
772
|
+
// stale message id; a fresh first-paint re-establishes one if workers
|
|
773
|
+
// are still live. On a terminal finalize, also drop the finished row.
|
|
774
|
+
g.messageId = null
|
|
775
|
+
g.lastBody = null
|
|
776
|
+
// The pinned message no longer exists → release the group pin claim
|
|
777
|
+
// (settleTerminal already re-syncs on the terminal path).
|
|
778
|
+
if (isTerminal) settleTerminal()
|
|
779
|
+
else syncPin(g)
|
|
654
780
|
return
|
|
655
781
|
}
|
|
656
|
-
// 'transient' —
|
|
657
|
-
|
|
658
|
-
log(`worker-feed:
|
|
782
|
+
// 'transient' — leave the message intact; the heartbeat re-attempts.
|
|
783
|
+
if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
|
|
784
|
+
log(`worker-feed: edit transient error feed=${g.feedKey}: ${(err as Error).message}`)
|
|
659
785
|
}
|
|
660
786
|
}
|
|
661
787
|
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
* FIRST paint for a prose-silent worker whose only tick arrived before
|
|
665
|
-
* `firstPaintMin`.
|
|
666
|
-
*
|
|
667
|
-
* Why the first-paint branch exists: a background worker that dives
|
|
668
|
-
* straight into quiet work (e.g. a long `Bash` / `npm test`) emits a
|
|
669
|
-
* single `sub_agent_tool_use` event when the command is invoked, then no
|
|
670
|
-
* further JSONL lines for the whole run. That one tick drives `update`
|
|
671
|
-
* once — but if it lands before `firstPaintMin` the paint is held, and
|
|
672
|
-
* with no subsequent tick nothing ever re-drives it, so the worker shows
|
|
673
|
-
* NOTHING for its entire run (the "I can't see the worker" gap). The
|
|
674
|
-
* heartbeat closes it: once such a handle is past `firstPaintMin`, drive a
|
|
675
|
-
* paint here through the same chain → doUpdate path. After first paint the
|
|
676
|
-
* suffix-only maintenance branch keeps it advancing.
|
|
677
|
-
*
|
|
678
|
-
* For handles that already have a posted message, this is the original
|
|
679
|
-
* option-(a), suffix-only re-render (never editMessageText directly). Skips:
|
|
680
|
-
* - handles inside a 429 cooldown,
|
|
681
|
-
* - handles with no `lastView` (no update ever arrived) or non-running,
|
|
682
|
-
* - for the maintenance branch: handles edited within minEditInterval
|
|
683
|
-
* (no stampede) or whose current step isn't yet stale.
|
|
684
|
-
* The `· Ns` liveSuffix is applied ONLY when the worker's current step is
|
|
685
|
-
* stale (now - lastEditAt >= heartbeatTickMs) so a normally-ticking worker is
|
|
686
|
-
* untouched and its body stays byte-stable for the dedup.
|
|
687
|
-
*/
|
|
788
|
+
// Arm the heartbeat once at construction. The real timer is `.unref()`'d so
|
|
789
|
+
// it never keeps the process alive; tests inject setInterval/clearInterval.
|
|
688
790
|
function heartbeatTick(): void {
|
|
689
791
|
const now = nowFn()
|
|
690
|
-
for (const
|
|
691
|
-
//
|
|
692
|
-
//
|
|
693
|
-
//
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
// (or a transient error) was staged on `pendingFinish` by `doFinish`.
|
|
701
|
-
// Re-drive it once the cooldown has expired so a finished worker's card
|
|
702
|
-
// can't get stuck on its last running render. This is the deterministic
|
|
703
|
-
// backstop that replaces the old "stale but harmless" surrender — the
|
|
704
|
-
// framework owns ALIVE-and-done, wall-clock driven, no model in the loop.
|
|
705
|
-
// (The handle is still in the map because `finish()`'s `.finally(delete)`
|
|
706
|
-
// is chained AFTER `doFinish` and won't drain while a re-drive keeps the
|
|
707
|
-
// chain busy; once the terminal edit lands, `pendingFinish` is cleared
|
|
708
|
-
// and the `.finally` runs on the next chain settle.)
|
|
709
|
-
if (h.pendingFinish != null && now >= h.cooldownUntil) {
|
|
710
|
-
const view = h.pendingFinish
|
|
711
|
-
h.chain = h.chain
|
|
712
|
-
.then(() => doFinish(h, view))
|
|
792
|
+
for (const g of [...groups.values()]) {
|
|
793
|
+
// Deferred-finalize re-drive: a terminal edit that hit a cooldown/flood
|
|
794
|
+
// window was staged on `pendingFinalize`. Re-drive it once the cooldown
|
|
795
|
+
// expires so a finished feed can't get stuck on its last running render.
|
|
796
|
+
if (g.pendingFinalize != null && now >= g.cooldownUntil) {
|
|
797
|
+
const recap = g.pendingFinalize
|
|
798
|
+
// The finishing agent is whatever finished row remains (state terminal).
|
|
799
|
+
const finishingAgentId = [...g.workers.values()].find((w) => w.finished)?.agentId
|
|
800
|
+
g.chain = g.chain
|
|
801
|
+
.then(() => doRender(g, { force: true, terminalRecap: recap, finishingAgentId }))
|
|
713
802
|
.catch((err) => {
|
|
714
|
-
log(`worker-feed: heartbeat finalize re-drive error
|
|
715
|
-
})
|
|
716
|
-
.finally(() => {
|
|
717
|
-
// Mirror `finish()`'s teardown: once the re-driven `doFinish`
|
|
718
|
-
// clears `pendingFinish` (terminal edit landed OR permanently
|
|
719
|
-
// failed), drop the handle. If it re-staged (another 429), the
|
|
720
|
-
// handle survives for the next heartbeat tick to retry.
|
|
721
|
-
if (handles.get(h.agentId)?.pendingFinish == null) {
|
|
722
|
-
handles.delete(h.agentId)
|
|
723
|
-
}
|
|
803
|
+
log(`worker-feed: heartbeat finalize re-drive error feed=${g.feedKey}: ${(err as Error).message}`)
|
|
724
804
|
})
|
|
725
805
|
continue
|
|
726
806
|
}
|
|
727
807
|
|
|
728
|
-
if (
|
|
729
|
-
|
|
730
|
-
if (
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
if (liveElapsed < firstPaintMin) continue
|
|
740
|
-
const view = { ...h.lastView, elapsedMs: Math.max(h.lastView.elapsedMs, liveElapsed) }
|
|
741
|
-
h.chain = h.chain
|
|
742
|
-
.then(() => doUpdate(h, view))
|
|
808
|
+
if (now < g.cooldownUntil) continue
|
|
809
|
+
const running = runningRows(g)
|
|
810
|
+
if (running.length === 0) continue
|
|
811
|
+
|
|
812
|
+
// First-paint path: no message yet and some worker has now crossed
|
|
813
|
+
// firstPaintMin (a prose-silent worker's single early tick was held).
|
|
814
|
+
if (g.messageId == null) {
|
|
815
|
+
const maxElapsed = Math.max(0, ...running.map((r) => liveElapsed(r, now)))
|
|
816
|
+
if (maxElapsed < firstPaintMin) continue
|
|
817
|
+
g.chain = g.chain
|
|
818
|
+
.then(() => doRender(g, {}))
|
|
743
819
|
.catch((err) => {
|
|
744
|
-
log(`worker-feed: heartbeat first-paint chain error
|
|
820
|
+
log(`worker-feed: heartbeat first-paint chain error feed=${g.feedKey}: ${(err as Error).message}`)
|
|
745
821
|
})
|
|
746
822
|
continue
|
|
747
823
|
}
|
|
748
824
|
|
|
749
|
-
if (now -
|
|
750
|
-
const stale = now -
|
|
825
|
+
if (now - g.lastEditAt < minEditInterval) continue
|
|
826
|
+
const stale = now - g.lastEditAt >= heartbeatTickMs
|
|
751
827
|
if (!stale) continue
|
|
752
|
-
//
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
// Under STEP_TIMER_MIN_MS formatStepSuffix returns '' (no timer yet);
|
|
756
|
-
// the header elapsed still climbs via the refreshed view below.
|
|
757
|
-
const stepElapsed = h.stepStartedAtMs != null ? now - h.stepStartedAtMs : liveElapsed
|
|
758
|
-
const liveSuffix = formatStepSuffix(stepElapsed)
|
|
759
|
-
// Re-render THROUGH the chain + doUpdate path — never editMessageText directly.
|
|
760
|
-
//
|
|
761
|
-
// CLOCK-ANCHOR PARITY: refresh the view's elapsedMs to the same `now`
|
|
762
|
-
// anchor the step suffix uses. The header renders
|
|
763
|
-
// `view.elapsedMs`; passing the stale lastView froze the header at the
|
|
764
|
-
// last watcher event while the `· Ns` suffix kept ticking, so the
|
|
765
|
-
// current step's timer could read MORE than the card's master elapsed
|
|
766
|
-
// (Ken-observed defect). Both numbers now derive from one anchor
|
|
767
|
-
// (dispatchAtMs) at one `now`, so header elapsed >= step suffix always.
|
|
768
|
-
const view = { ...h.lastView, elapsedMs: Math.max(h.lastView.elapsedMs, liveElapsed) }
|
|
769
|
-
h.chain = h.chain
|
|
770
|
-
.then(() => doUpdate(h, view, liveSuffix))
|
|
828
|
+
// Re-render THROUGH the chain → doRender path with climbing elapsed.
|
|
829
|
+
g.chain = g.chain
|
|
830
|
+
.then(() => doRender(g, { heartbeat: true }))
|
|
771
831
|
.catch((err) => {
|
|
772
|
-
log(`worker-feed: heartbeat chain error
|
|
832
|
+
log(`worker-feed: heartbeat chain error feed=${g.feedKey}: ${(err as Error).message}`)
|
|
773
833
|
})
|
|
774
834
|
}
|
|
775
835
|
}
|
|
776
836
|
|
|
777
|
-
// Arm the heartbeat once at construction. The real timer is `.unref()`'d so
|
|
778
|
-
// it never keeps the process alive; tests inject setInterval/clearInterval.
|
|
779
837
|
heartbeatTimer = setIntervalFn(heartbeatTick, heartbeatTickMs)
|
|
780
838
|
|
|
781
839
|
return {
|
|
782
840
|
has(agentId) {
|
|
783
|
-
|
|
841
|
+
const g = groupOfAgent(agentId)
|
|
842
|
+
return g != null && g.messageId != null && g.workers.has(agentId)
|
|
784
843
|
},
|
|
785
844
|
messageIdOf(agentId) {
|
|
786
|
-
return
|
|
845
|
+
return groupOfAgent(agentId)?.messageId ?? null
|
|
846
|
+
},
|
|
847
|
+
hasRunningInFeed(feedKey) {
|
|
848
|
+
const g = groups.get(feedKey)
|
|
849
|
+
return g != null && g.workers.size > 0
|
|
787
850
|
},
|
|
788
851
|
get size() {
|
|
789
|
-
|
|
852
|
+
let n = 0
|
|
853
|
+
for (const g of groups.values()) n += g.workers.size
|
|
854
|
+
return n
|
|
790
855
|
},
|
|
791
856
|
update(agentId, chatId, view, threadId) {
|
|
792
|
-
// No chat to post to (owner DM unconfigured) — don't create
|
|
793
|
-
//
|
|
857
|
+
// No chat to post to (owner DM unconfigured) — don't create state that
|
|
858
|
+
// would retry a failing send('') every tick.
|
|
794
859
|
if (chatId.length === 0) return Promise.resolve()
|
|
795
|
-
// Resurrection guard: a worker
|
|
796
|
-
// (
|
|
797
|
-
// A late watcher `onProgress` tick can arrive after `finish()`'s chain
|
|
798
|
-
// has fully settled and the handle was deleted — without this durable
|
|
799
|
-
// gate the tick would create a brand-new handle and paint a fresh
|
|
800
|
-
// `running` message on an already-done worker (the card lies). The
|
|
801
|
-
// heartbeat's orphan-paint guard covers the heartbeat tick only; the
|
|
802
|
-
// per-handle `finished` flag covers the pre-delete window; this set
|
|
803
|
-
// covers the post-delete window.
|
|
860
|
+
// Resurrection guard: a worker already finalized must not get a fresh
|
|
861
|
+
// running cue (a late watcher tick would repaint a done worker as live).
|
|
804
862
|
if (finalized.has(agentId)) return Promise.resolve()
|
|
805
|
-
const
|
|
806
|
-
if (
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
863
|
+
const existingRow = groupOfAgent(agentId)?.workers.get(agentId)
|
|
864
|
+
if (existingRow?.finished === true) return Promise.resolve()
|
|
865
|
+
|
|
866
|
+
const feedKey = feedKeyOf(chatId, threadId)
|
|
867
|
+
let g = groups.get(feedKey)
|
|
868
|
+
if (g == null) {
|
|
869
|
+
g = {
|
|
870
|
+
feedKey,
|
|
811
871
|
chatId,
|
|
812
872
|
threadId,
|
|
813
873
|
messageId: null,
|
|
814
874
|
lastBody: null,
|
|
815
875
|
lastEditAt: 0,
|
|
816
876
|
cooldownUntil: 0,
|
|
817
|
-
narrative: [],
|
|
818
877
|
chain: Promise.resolve(),
|
|
878
|
+
workers: new Map(),
|
|
879
|
+
pendingFinalize: null,
|
|
880
|
+
}
|
|
881
|
+
groups.set(feedKey, g)
|
|
882
|
+
}
|
|
883
|
+
let row = g.workers.get(agentId)
|
|
884
|
+
if (row == null) {
|
|
885
|
+
row = {
|
|
886
|
+
agentId,
|
|
887
|
+
narrative: [],
|
|
819
888
|
lastView: null,
|
|
889
|
+
state: 'running',
|
|
890
|
+
finished: false,
|
|
820
891
|
dispatchAtMs: null,
|
|
821
892
|
stepStartedAtMs: null,
|
|
822
|
-
finished: false,
|
|
823
|
-
pendingFinish: null,
|
|
824
893
|
}
|
|
825
|
-
|
|
894
|
+
g.workers.set(agentId, row)
|
|
895
|
+
agentIndex.set(agentId, feedKey)
|
|
826
896
|
}
|
|
827
|
-
|
|
828
|
-
|
|
897
|
+
// Accumulate before the gate so a throttled tick still grows the
|
|
898
|
+
// narrative — it surfaces on the next edit that does fire.
|
|
899
|
+
accumulateNarrative(row, view)
|
|
900
|
+
row.state = 'running'
|
|
901
|
+
row.lastView = { ...view, narrativeLines: [...row.narrative] }
|
|
902
|
+
if (row.dispatchAtMs == null) row.dispatchAtMs = nowFn() - view.elapsedMs
|
|
903
|
+
|
|
904
|
+
const group = g
|
|
905
|
+
group.chain = group.chain.then(() => doRender(group)).catch((err) => {
|
|
829
906
|
log(`worker-feed: update chain error ${agentId}: ${(err as Error).message}`)
|
|
830
907
|
})
|
|
831
|
-
return
|
|
908
|
+
return group.chain
|
|
832
909
|
},
|
|
833
910
|
finish(agentId, view) {
|
|
834
|
-
const
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
911
|
+
const g = groupOfAgent(agentId)
|
|
912
|
+
const row = g?.workers.get(agentId)
|
|
913
|
+
if (g == null || row == null) {
|
|
914
|
+
// Never tracked (trivial worker) — mark finalized so a late tick can't
|
|
915
|
+
// resurrect, and let the handback carry the result.
|
|
916
|
+
markFinalized(agentId)
|
|
917
|
+
return Promise.resolve()
|
|
918
|
+
}
|
|
919
|
+
// Latch synchronously so a late `running` cue on the chain can't resurrect.
|
|
920
|
+
row.finished = true
|
|
921
|
+
row.state = view.state === 'failed' ? 'failed' : 'done'
|
|
922
|
+
markFinalized(agentId)
|
|
923
|
+
|
|
924
|
+
const group = g
|
|
925
|
+
group.chain = group.chain
|
|
926
|
+
.then(() => {
|
|
927
|
+
const others = runningRows(group).filter((w) => w.agentId !== agentId)
|
|
928
|
+
if (others.length > 0) {
|
|
929
|
+
// Siblings still live → drop this row from the combined body and
|
|
930
|
+
// re-render the running set. The result reaches the user via the
|
|
931
|
+
// separate handback, never folded into this cosmetic edit. The
|
|
932
|
+
// group pin STAYS (siblings still need the shared message) — a
|
|
933
|
+
// per-worker unpin here was the #3207 review blocker.
|
|
934
|
+
removeWorker(group, agentId)
|
|
935
|
+
syncPin(group)
|
|
936
|
+
return doRender(group, { force: true })
|
|
937
|
+
}
|
|
938
|
+
// Last live worker → finalize the shared message to its terminal recap.
|
|
939
|
+
const recap: WorkerActivityView = { ...view, narrativeLines: [...row.narrative] }
|
|
940
|
+
return doRender(group, { force: true, terminalRecap: recap, finishingAgentId: agentId })
|
|
941
|
+
})
|
|
838
942
|
.catch((err) => {
|
|
839
943
|
log(`worker-feed: finish chain error ${agentId}: ${(err as Error).message}`)
|
|
840
944
|
})
|
|
841
|
-
|
|
842
|
-
// Only tear down the handle once the terminal edit has actually
|
|
843
|
-
// landed (or permanently failed). If `doFinish` staged the edit on
|
|
844
|
-
// `pendingFinish` (a 429 cooldown / transient error was in effect),
|
|
845
|
-
// the handle must survive so the heartbeat can re-drive the
|
|
846
|
-
// finalize after cooldown. The heartbeat's re-drive chain ends by
|
|
847
|
-
// re-entering `doFinish`, which clears `pendingFinish` on success
|
|
848
|
-
// or permanent-failure — so this `.finally` deletes on the NEXT
|
|
849
|
-
// chain settle once there is nothing left to finalize. Without this
|
|
850
|
-
// guard, the `.finally` would delete the handle (and its staged
|
|
851
|
-
// pendingFinish) immediately after the first staged doFinish,
|
|
852
|
-
// stranding the card on its last running render.
|
|
853
|
-
if (handles.get(agentId)?.pendingFinish == null) {
|
|
854
|
-
handles.delete(agentId)
|
|
855
|
-
}
|
|
856
|
-
})
|
|
857
|
-
return h.chain
|
|
945
|
+
return group.chain
|
|
858
946
|
},
|
|
859
947
|
drop(agentId) {
|
|
860
|
-
// A dropped worker is also done — mark finalized so a late
|
|
861
|
-
//
|
|
948
|
+
// A dropped worker is also done — mark finalized so a late tick can't
|
|
949
|
+
// resurrect a running card on it (same gate as `finish`).
|
|
862
950
|
markFinalized(agentId)
|
|
863
|
-
|
|
951
|
+
const g = groupOfAgent(agentId)
|
|
952
|
+
if (g == null) return
|
|
953
|
+
const hadMessage = g.messageId != null
|
|
954
|
+
removeWorker(g, agentId)
|
|
955
|
+
// Group pin follows membership: unpin if this emptied the group, else
|
|
956
|
+
// keep it (siblings still need the shared message).
|
|
957
|
+
if (hadMessage) syncPin(g)
|
|
958
|
+
// Re-render so the dropped worker disappears from a combined body. Skip
|
|
959
|
+
// when the group is gone (removeWorker deleted it) or never painted.
|
|
960
|
+
if (hadMessage && groups.has(g.feedKey) && runningRows(g).length > 0) {
|
|
961
|
+
g.chain = g.chain
|
|
962
|
+
.then(() => doRender(g, { force: true }))
|
|
963
|
+
.catch((err) => {
|
|
964
|
+
log(`worker-feed: drop re-render error ${agentId}: ${(err as Error).message}`)
|
|
965
|
+
})
|
|
966
|
+
}
|
|
864
967
|
},
|
|
865
968
|
resurrect(agentId) {
|
|
866
969
|
// Issue #3023: the worker's card was falsely finalized and its JSONL has
|
|
867
|
-
// resumed. Re-open the paint path: drop the durable finalized gate
|
|
868
|
-
//
|
|
869
|
-
// again, and un-latch any surviving handle (finish deleted it in the
|
|
870
|
-
// common case, but a staged pendingFinish could keep it alive). The next
|
|
871
|
-
// `update` tick from the watcher's replayed progress does the repaint.
|
|
970
|
+
// resumed. Re-open the paint path: drop the durable finalized gate + any
|
|
971
|
+
// surviving per-row latch so a fresh `running` cue repaints a live card.
|
|
872
972
|
const wasFinalized = finalized.delete(agentId)
|
|
873
|
-
const
|
|
874
|
-
if (
|
|
875
|
-
|
|
876
|
-
|
|
973
|
+
const row = groupOfAgent(agentId)?.workers.get(agentId)
|
|
974
|
+
if (row != null) {
|
|
975
|
+
row.finished = false
|
|
976
|
+
row.state = 'running'
|
|
877
977
|
}
|
|
878
|
-
if (wasFinalized ||
|
|
978
|
+
if (wasFinalized || row != null) {
|
|
879
979
|
log(`worker-feed: resurrect agent=${agentId} — cleared finalized gate; card will repaint on next running cue`)
|
|
880
980
|
}
|
|
881
981
|
},
|