switchroom 0.18.15 → 0.18.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +16 -0
- package/dist/auth-broker/index.js +445 -10
- package/dist/cli/notion-write-pretool.mjs +16 -0
- package/dist/cli/switchroom.js +654 -479
- package/dist/host-control/main.js +20 -1
- package/dist/vault/approvals/kernel-server.js +16 -0
- package/dist/vault/broker/server.js +16 -0
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +81 -139
- package/telegram-plugin/bridge/bridge.ts +7 -1
- package/telegram-plugin/dist/bridge/bridge.js +26 -1
- package/telegram-plugin/dist/gateway/gateway.js +1758 -661
- package/telegram-plugin/dist/server.js +26 -1
- package/telegram-plugin/draft-stream.ts +78 -3
- package/telegram-plugin/fleet-fallback-resume.ts +26 -3
- package/telegram-plugin/gateway/approval-hold.ts +49 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +64 -22
- package/telegram-plugin/gateway/effort-command.ts +9 -7
- package/telegram-plugin/gateway/gateway.ts +627 -291
- package/telegram-plugin/gateway/linear-activity.ts +20 -4
- package/telegram-plugin/gateway/litellm-local-notice-wiring.ts +200 -0
- package/telegram-plugin/gateway/model-command.ts +96 -18
- package/telegram-plugin/gateway/pending-session-command.ts +10 -8
- package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
- package/telegram-plugin/gateway/session-model-file.ts +141 -172
- package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
- package/telegram-plugin/litellm-local-notice.ts +189 -0
- package/telegram-plugin/llm-error-present.ts +436 -0
- package/telegram-plugin/operator-events.ts +7 -1
- package/telegram-plugin/permission-title.ts +172 -10
- package/telegram-plugin/premium-recovery.ts +101 -0
- package/telegram-plugin/quota-watch.ts +16 -4
- package/telegram-plugin/raw-error-scrub.ts +73 -0
- package/telegram-plugin/retry-api-call.ts +8 -2
- package/telegram-plugin/runtime-metrics.ts +16 -0
- package/telegram-plugin/send-gate-degraded.test.ts +161 -8
- package/telegram-plugin/send-gate-observability.test.ts +140 -0
- package/telegram-plugin/send-gate-observability.ts +65 -20
- package/telegram-plugin/send-gate.test.ts +143 -1
- package/telegram-plugin/send-gate.ts +246 -23
- package/telegram-plugin/session-tail.ts +16 -0
- package/telegram-plugin/shared/local-time.ts +69 -0
- package/telegram-plugin/stream-controller.ts +143 -20
- package/telegram-plugin/stream-reply-handler.ts +12 -2
- package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
- package/telegram-plugin/tests/bot-api.harness.ts +7 -2
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
- package/telegram-plugin/tests/draft-stream.test.ts +110 -1
- package/telegram-plugin/tests/effort-command.test.ts +4 -4
- package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +5 -4
- package/telegram-plugin/tests/gateway-pending-command-wiring.test.ts +33 -19
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +47 -127
- package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
- package/telegram-plugin/tests/litellm-local-notice.test.ts +417 -0
- package/telegram-plugin/tests/llm-error-present.test.ts +380 -0
- package/telegram-plugin/tests/model-command.test.ts +84 -1
- package/telegram-plugin/tests/permission-title.test.ts +167 -4
- package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
- package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
- package/telegram-plugin/tests/quota-watch.test.ts +21 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +8 -3
- package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
- package/telegram-plugin/tests/session-model-file.test.ts +7 -155
- package/telegram-plugin/tests/stream-controller-send-gate.test.ts +521 -0
- package/telegram-plugin/tests/stream-reply-handler.test.ts +44 -0
- package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
- package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
- package/telegram-plugin/tests/worker-activity-feed.test.ts +212 -2
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
- package/telegram-plugin/tier-downgrade.ts +198 -0
- package/telegram-plugin/tool-activity-summary.ts +99 -0
- package/telegram-plugin/worker-activity-feed.ts +543 -368
|
@@ -13,17 +13,34 @@
|
|
|
13
13
|
* that grows/edits as work happens — indistinguishable from "the agent
|
|
14
14
|
* is typing live", not a status widget.
|
|
15
15
|
*
|
|
16
|
-
*
|
|
17
|
-
* (
|
|
18
|
-
*
|
|
19
|
-
*
|
|
16
|
+
* COALESCED (#3084 follow-up): all live workers dispatched to the SAME
|
|
17
|
+
* chat/thread render into ONE shared message (a {@link FeedGroup}), not one
|
|
18
|
+
* message each. With N workers, N separate messages each coalesce their own
|
|
19
|
+
* edit stream but ALL draw from the send gate's 1/sec per-chat bucket — so
|
|
20
|
+
* ~N-1 of every second's edits SHED and every card refreshes only ~once per N
|
|
21
|
+
* seconds (liveness collapse). One combined message = ONE per-message edit
|
|
22
|
+
* stream: it coalesces (last-write-wins) and never contends with siblings, so
|
|
23
|
+
* every worker's row refreshes together within the ~1.5s edit floor. A single-
|
|
24
|
+
* worker chat still renders the full 🛠 Worker card (identical to before); a
|
|
25
|
+
* 2+ worker chat renders `renderCombinedWorkerFeed`.
|
|
26
|
+
*
|
|
27
|
+
* Pure render (`renderWorkerActivity` / `renderCombinedWorkerFeed`) + an
|
|
28
|
+
* injected bot API (`BotApiForWorkerFeed`). The manager
|
|
29
|
+
* (`createWorkerActivityFeed`) owns one edit-in-place message per (chat,thread)
|
|
30
|
+
* with:
|
|
20
31
|
* - a first-paint delay so trivial sub-second workers never post a
|
|
21
32
|
* message (their result still lands via the handback reply),
|
|
22
33
|
* - a proactive min-edit-interval throttle (worker jsonl ticks ~1/s;
|
|
23
34
|
* Telegram rate-limits edits) plus body-dedup,
|
|
24
|
-
* - per-
|
|
35
|
+
* - per-message serialization so two rapid ticks can't double-send,
|
|
25
36
|
* - 429 cooldown + message_id drift resilience (re-post on stale edit),
|
|
26
|
-
* - a forced terminal edit on
|
|
37
|
+
* - a forced terminal edit on the LAST worker's `finish`.
|
|
38
|
+
*
|
|
39
|
+
* The completed-worker RESULT is NOT folded into this cosmetic feed — it
|
|
40
|
+
* reaches the user as its own `useful` handback reply (gateway onFinish). The
|
|
41
|
+
* feed shows only *live* work; a finished worker's row is dropped from the
|
|
42
|
+
* combined body (or, when it is the last worker, the message finalizes to its
|
|
43
|
+
* terminal recap).
|
|
27
44
|
*
|
|
28
45
|
* The feed is gated to BACKGROUND workers and is ON by default; set
|
|
29
46
|
* `SWITCHROOM_WORKER_ACTIVITY_FEED=0` to disable it — see the gateway
|
|
@@ -38,7 +55,13 @@ import {
|
|
|
38
55
|
truncate,
|
|
39
56
|
} from './card-format.js'
|
|
40
57
|
import { STATUS_ROLLING_LINES } from './status-no-truncate.js'
|
|
41
|
-
import {
|
|
58
|
+
import {
|
|
59
|
+
renderStatusCard,
|
|
60
|
+
formatStepSuffix,
|
|
61
|
+
renderCombinedWorkerFeed,
|
|
62
|
+
type CombinedWorkerRow,
|
|
63
|
+
} from './tool-activity-summary.js'
|
|
64
|
+
import { isSendGateShed } from './send-gate.js'
|
|
42
65
|
|
|
43
66
|
/** Worker-activity feed is ON by default; an operator opts out with
|
|
44
67
|
* SWITCHROOM_WORKER_ACTIVITY_FEED=0. */
|
|
@@ -195,6 +218,24 @@ export interface WorkerActivityFeedOpts {
|
|
|
195
218
|
firstPaintMinMs?: number
|
|
196
219
|
/** stderr-style log sink. Defaults to noop. */
|
|
197
220
|
log?: (msg: string) => void
|
|
221
|
+
/**
|
|
222
|
+
* Remaining ms of the currently-open per-bot flood window, read from the
|
|
223
|
+
* SAME persisted marker the gateway's `robustApiCall` and held-card sweep
|
|
224
|
+
* consult (`makeFloodWaitProbe(FLOOD_STATE_PATH)`). Two independent reads of
|
|
225
|
+
* one source of truth — not a second notion of "is the channel open".
|
|
226
|
+
*
|
|
227
|
+
* Why the feed needs it (#3084 follow-up): the feed's send/edit adapters run
|
|
228
|
+
* through the send gate, which SHEDS (resolves `undefined`) any call made
|
|
229
|
+
* while a flood window is open. Without this probe the heartbeat re-fires a
|
|
230
|
+
* shed send every `heartbeatTickMs` (~6s) for the WHOLE ban — thousands of
|
|
231
|
+
* gate admissions, and (pre-fix) a `sent.message_id` crash on the `undefined`
|
|
232
|
+
* every tick. With it, a running/first-paint tick that sees an open window
|
|
233
|
+
* parks the group in cooldown for the window's remaining and makes ZERO api
|
|
234
|
+
* calls until it closes, mirroring the held-card sweep's pre-send probe.
|
|
235
|
+
* Defaults to `() => 0` (no window) so tests and non-gateway callers are
|
|
236
|
+
* unchanged.
|
|
237
|
+
*/
|
|
238
|
+
floodWaitRemainingMs?: () => number
|
|
198
239
|
/**
|
|
199
240
|
* Heartbeat timer factory. Injectable for tests. Defaults to the real
|
|
200
241
|
* `setInterval`, `.unref()`'d so it never keeps the process alive.
|
|
@@ -203,69 +244,110 @@ export interface WorkerActivityFeedOpts {
|
|
|
203
244
|
/** Heartbeat timer disposer. Injectable for tests. Defaults to `clearInterval`. */
|
|
204
245
|
clearInterval?: (handle: unknown) => void
|
|
205
246
|
/**
|
|
206
|
-
* Heartbeat tick cadence in ms. On each tick a stale, running
|
|
207
|
-
* re-rendered with
|
|
208
|
-
*
|
|
247
|
+
* Heartbeat tick cadence in ms. On each tick a stale, running feed is
|
|
248
|
+
* re-rendered with climbing elapsed so a worker that emits no new narrative
|
|
249
|
+
* still visibly advances. Default 6000ms.
|
|
209
250
|
*/
|
|
210
251
|
heartbeatTickMs?: number
|
|
252
|
+
/**
|
|
253
|
+
* Max worker rows rendered in a COMBINED feed (2+ workers in one chat/thread)
|
|
254
|
+
* before the `+M more working…` spill line. Keeps the coalesced body compact
|
|
255
|
+
* and legible (and under the rich-message wire ceiling). Default 8. A single-
|
|
256
|
+
* worker chat renders the full 🛠 Worker card and ignores this. Sourced from
|
|
257
|
+
* `channels.telegram.worker_feed.max_rows` via the config cascade.
|
|
258
|
+
*/
|
|
259
|
+
maxRows?: number
|
|
260
|
+
/**
|
|
261
|
+
* Group-level status-pin reconcile hook (#3207 review). Because workers now
|
|
262
|
+
* COALESCE into one shared message, the pin MUST follow the GROUP lifecycle,
|
|
263
|
+
* not a single worker's: pin the shared message when a group's first worker
|
|
264
|
+
* paints, and unpin ONLY when the group empties (its LAST worker finishes / is
|
|
265
|
+
* dropped). The feed alone knows group membership + running-count, so it owns
|
|
266
|
+
* the pin. `messageId: null` means "unpin this group". A per-worker unpin would
|
|
267
|
+
* physically unpin a message a SIBLING still needs, and the survivor's next
|
|
268
|
+
* pin request NO-OPs (its claim still names that id) — so it would run
|
|
269
|
+
* unpinned for the rest of its life (the shared-message NOOP trap). Best-
|
|
270
|
+
* effort; defaults to a noop for tests / non-gateway callers.
|
|
271
|
+
*/
|
|
272
|
+
reconcilePin?: (args: {
|
|
273
|
+
feedKey: string
|
|
274
|
+
chatId: string
|
|
275
|
+
threadId?: number
|
|
276
|
+
messageId: number | null
|
|
277
|
+
}) => void
|
|
211
278
|
}
|
|
212
279
|
|
|
213
|
-
|
|
280
|
+
/**
|
|
281
|
+
* One live worker's per-row state inside a chat/thread feed group. The row
|
|
282
|
+
* carries everything the render needs for THIS worker; the shared message
|
|
283
|
+
* (id, cooldown, chain) lives on the enclosing {@link FeedGroup}.
|
|
284
|
+
*/
|
|
285
|
+
interface WorkerRow {
|
|
214
286
|
/** jsonl agent id — carried so success/failure log lines can name the worker. */
|
|
215
287
|
agentId: string
|
|
216
|
-
chatId: string
|
|
217
|
-
threadId?: number
|
|
218
|
-
messageId: number | null
|
|
219
|
-
lastBody: string | null
|
|
220
|
-
lastEditAt: number
|
|
221
|
-
cooldownUntil: number
|
|
222
288
|
/**
|
|
223
|
-
* Accumulated narrative lines (oldest→newest), deduped
|
|
224
|
-
*
|
|
225
|
-
*
|
|
289
|
+
* Accumulated narrative lines (oldest→newest), deduped within the whole
|
|
290
|
+
* rolling window. Rolling-window capped to STATUS_ROLLING_LINES. Grows the
|
|
291
|
+
* live render so the feed reads like the main agent's answer.
|
|
226
292
|
*/
|
|
227
293
|
narrative: string[]
|
|
228
|
-
/**
|
|
229
|
-
chain: Promise<void>
|
|
230
|
-
/** Last view rendered into the message (drives the heartbeat re-render). */
|
|
294
|
+
/** Last view for this worker (drives the heartbeat re-render + combined row). */
|
|
231
295
|
lastView: WorkerActivityView | null
|
|
296
|
+
/** Latest state observed for this worker; excluded from the running set once
|
|
297
|
+
* terminal (its result reaches the user via the separate handback reply). */
|
|
298
|
+
state: WorkerActivityState
|
|
232
299
|
/**
|
|
233
|
-
*
|
|
234
|
-
*
|
|
235
|
-
*
|
|
236
|
-
* transport hiccup can't leave the card stuck on its last running render
|
|
237
|
-
* ("worker done, card says running"). Cleared on a successful terminal
|
|
238
|
-
* edit, on a permanent failure (message gone), or when the handle is
|
|
239
|
-
* deleted. Null when no finalize is pending.
|
|
240
|
-
*/
|
|
241
|
-
pendingFinish: WorkerActivityView | null
|
|
242
|
-
/**
|
|
243
|
-
* Latched in `doFinish` before the terminal edit. A late watcher
|
|
244
|
-
* `onProgress` tick that arrives after `finish()` queued its chain (but
|
|
245
|
-
* before the `.finally(handles.delete)` microtask drains) must NOT
|
|
246
|
-
* resurrect the handle and paint a fresh `running` message on an
|
|
247
|
-
* already-finalized worker. The heartbeat's orphan-paint guard
|
|
248
|
-
* (`if (!handles.has(h.agentId)) continue`) only covers the heartbeat
|
|
249
|
-
* tick — this flag covers the `update` entry point. Set synchronously
|
|
250
|
-
* inside `doFinish` (runs on the chain), checked synchronously in
|
|
251
|
-
* `update` before handle creation.
|
|
300
|
+
* Latched in `finish` before the terminal edit. A late watcher `onProgress`
|
|
301
|
+
* tick that arrives after `finish()` queued its chain must NOT resurrect the
|
|
302
|
+
* row and paint a fresh `running` state on an already-finalized worker.
|
|
252
303
|
*/
|
|
253
304
|
finished: boolean
|
|
254
305
|
/**
|
|
255
306
|
* Wall-clock ms the worker was dispatched, derived from `now - view.elapsedMs`
|
|
256
307
|
* on the first update. The heartbeat computes a live elapsed from this so the
|
|
257
|
-
*
|
|
308
|
+
* elapsed climbs even when no fresh view arrives, and it fixes the worker's
|
|
309
|
+
* stable sort order within the combined feed.
|
|
258
310
|
*/
|
|
259
311
|
dispatchAtMs: number | null
|
|
260
312
|
/**
|
|
261
313
|
* Wall-clock ms the CURRENT step started — stamped whenever a NEW narrative
|
|
262
314
|
* line lands (the `→` line changes). The heartbeat's step suffix shows the
|
|
263
|
-
* step's OWN elapsed from this anchor (
|
|
264
|
-
*
|
|
315
|
+
* step's OWN elapsed from this anchor (single-worker card only), and only
|
|
316
|
+
* once past STEP_TIMER_MIN_MS.
|
|
265
317
|
*/
|
|
266
318
|
stepStartedAtMs: number | null
|
|
267
319
|
}
|
|
268
320
|
|
|
321
|
+
/**
|
|
322
|
+
* One shared feed message per (chatId, threadId). ALL live workers dispatched
|
|
323
|
+
* to the same chat/thread render into this one message — so their edits form a
|
|
324
|
+
* single per-message edit stream under the send gate's 1/sec per-chat ceiling
|
|
325
|
+
* (last-write-wins coalescing + no-op skip) instead of N contending streams
|
|
326
|
+
* that shed (#3084). A single-worker group renders the full 🛠 Worker card;
|
|
327
|
+
* a 2+ worker group renders the combined `renderCombinedWorkerFeed` body.
|
|
328
|
+
*/
|
|
329
|
+
interface FeedGroup {
|
|
330
|
+
/** Stable key `${chatId} ${threadId ?? ''}`. */
|
|
331
|
+
feedKey: string
|
|
332
|
+
chatId: string
|
|
333
|
+
threadId?: number
|
|
334
|
+
messageId: number | null
|
|
335
|
+
lastBody: string | null
|
|
336
|
+
lastEditAt: number
|
|
337
|
+
cooldownUntil: number
|
|
338
|
+
/** Single serialization chain for the shared message — ticks can't interleave sends. */
|
|
339
|
+
chain: Promise<void>
|
|
340
|
+
/** Live workers in this group, keyed by agentId (insertion ≈ dispatch order). */
|
|
341
|
+
workers: Map<string, WorkerRow>
|
|
342
|
+
/**
|
|
343
|
+
* A terminal render (the last worker's recap) staged because a 429 cooldown /
|
|
344
|
+
* flood window blocked the edit. The heartbeat re-drives it once the cooldown
|
|
345
|
+
* expires so a finished feed can't get stuck on its last running render.
|
|
346
|
+
* Null when no finalize is pending.
|
|
347
|
+
*/
|
|
348
|
+
pendingFinalize: WorkerActivityView | null
|
|
349
|
+
}
|
|
350
|
+
|
|
269
351
|
const COOLDOWN_JITTER_MS = 500
|
|
270
352
|
|
|
271
353
|
function extractRetryAfterSecs(err: unknown): number | null {
|
|
@@ -288,7 +370,7 @@ function extractRetryAfterSecs(err: unknown): number | null {
|
|
|
288
370
|
* 'rate_limited' — 429 with retry_after. Back off; the heartbeat re-drives
|
|
289
371
|
* the edit after cooldown (running renders + deferred terminal edits).
|
|
290
372
|
* 'gone' — message/chat deleted or edit window expired. Nothing to
|
|
291
|
-
* update; drop the
|
|
373
|
+
* update; drop the message silently (no warning — there is no card).
|
|
292
374
|
* 'transient' — anything else (network blip, 5xx). Retry on the next
|
|
293
375
|
* heartbeat tick; don't spam stderr.
|
|
294
376
|
*/
|
|
@@ -319,18 +401,25 @@ function classifyEditError(err: unknown): EditOutcome {
|
|
|
319
401
|
}
|
|
320
402
|
|
|
321
403
|
/**
|
|
322
|
-
* Manager owning one live message per
|
|
323
|
-
*
|
|
324
|
-
*
|
|
325
|
-
*
|
|
404
|
+
* Manager owning one live message per (chat,thread) into which all live
|
|
405
|
+
* workers there coalesce. Public methods stay keyed by jsonl agent id (the
|
|
406
|
+
* gateway wiring is unchanged): the manager resolves the enclosing feed group
|
|
407
|
+
* internally. The gateway calls `update` on each watcher activity cue and
|
|
408
|
+
* `finish` on terminal; `drop` discards a worker's state without a final edit
|
|
409
|
+
* (error / supersession paths).
|
|
326
410
|
*/
|
|
327
411
|
export interface WorkerActivityFeed {
|
|
328
|
-
/** True if a message is currently posted for this worker. */
|
|
412
|
+
/** True if a message is currently posted for this worker's feed group. */
|
|
329
413
|
has(agentId: string): boolean
|
|
330
|
-
/** The Telegram message_id currently posted for this worker, or
|
|
331
|
-
* none is posted (never painted, or dropped after a stale-edit
|
|
332
|
-
* Lets the gateway pin the EXISTING `🛠 Worker` message
|
|
414
|
+
/** The Telegram message_id currently posted for this worker's feed group, or
|
|
415
|
+
* null if none is posted (never painted, or dropped after a stale-edit
|
|
416
|
+
* re-post). Lets the gateway pin the EXISTING `🛠 Worker` message. Note:
|
|
417
|
+
* siblings sharing the chat/thread return the SAME id (one message). */
|
|
333
418
|
messageIdOf(agentId: string): number | null
|
|
419
|
+
/** True while the feed group `feedKey` (`${chatId} ${threadId ?? ''}`) still
|
|
420
|
+
* tracks live work — used by the gateway's `wk:group:` pin reaper to exempt
|
|
421
|
+
* a live group's pin from the stale-TTL sweep (#3207). */
|
|
422
|
+
hasRunningInFeed(feedKey: string): boolean
|
|
334
423
|
/** Push a running-state cue. Returns the serialized op for tests. */
|
|
335
424
|
update(
|
|
336
425
|
agentId: string,
|
|
@@ -338,13 +427,17 @@ export interface WorkerActivityFeed {
|
|
|
338
427
|
view: WorkerActivityView,
|
|
339
428
|
threadId?: number,
|
|
340
429
|
): Promise<void>
|
|
341
|
-
/**
|
|
430
|
+
/** Finalize a worker: drop its row from the combined feed (its result reaches
|
|
431
|
+
* the user via the separate handback), or — when it is the last live worker
|
|
432
|
+
* in the group — force the terminal recap edit. No-op if the worker was
|
|
433
|
+
* never tracked. */
|
|
342
434
|
finish(agentId: string, view: WorkerActivityView): Promise<void>
|
|
343
|
-
/** Forget a worker's state without
|
|
435
|
+
/** Forget a worker's state without a recap edit (e.g. error path); re-renders
|
|
436
|
+
* the group so the dropped worker disappears from the combined body. */
|
|
344
437
|
drop(agentId: string): void
|
|
345
438
|
/**
|
|
346
439
|
* Issue #3023 (card resurrection). Undo a finalization: clear the durable
|
|
347
|
-
* `finalized` gate (and any lingering per-
|
|
440
|
+
* `finalized` gate (and any lingering per-row `finished` latch) so a
|
|
348
441
|
* worker whose card was FALSELY finalized can be painted/edited again. The
|
|
349
442
|
* watcher calls this (via the gateway's `onResurrect` wiring) when a
|
|
350
443
|
* falsely-finalized worker's JSONL resumes growing. A fresh `running` cue
|
|
@@ -356,16 +449,19 @@ export interface WorkerActivityFeed {
|
|
|
356
449
|
stop(): void
|
|
357
450
|
/** Manually fire one heartbeat tick (test hook). */
|
|
358
451
|
heartbeatTick(): void
|
|
359
|
-
/** Number of tracked workers (test/inspection hook). */
|
|
452
|
+
/** Number of tracked workers across all feed groups (test/inspection hook). */
|
|
360
453
|
readonly size: number
|
|
361
454
|
}
|
|
362
455
|
|
|
363
456
|
export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerActivityFeed {
|
|
364
457
|
const log = opts.log ?? (() => {})
|
|
365
458
|
const nowFn = opts.now ?? Date.now
|
|
459
|
+
const floodWaitRemainingMs = opts.floodWaitRemainingMs ?? (() => 0)
|
|
366
460
|
const minEditInterval = opts.minEditIntervalMs ?? 2500
|
|
367
461
|
const firstPaintMin = opts.firstPaintMinMs ?? 8000
|
|
368
462
|
const heartbeatTickMs = opts.heartbeatTickMs ?? 6000
|
|
463
|
+
const maxRows = Math.max(1, Math.floor(opts.maxRows ?? 8))
|
|
464
|
+
const reconcilePinFn = opts.reconcilePin ?? (() => {})
|
|
369
465
|
const setIntervalFn =
|
|
370
466
|
opts.setInterval ??
|
|
371
467
|
((cb: () => void, ms: number): unknown => {
|
|
@@ -375,17 +471,21 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
375
471
|
return t
|
|
376
472
|
})
|
|
377
473
|
const clearIntervalFn = opts.clearInterval ?? ((handle: unknown) => clearInterval(handle as ReturnType<typeof setInterval>))
|
|
378
|
-
|
|
474
|
+
|
|
475
|
+
/** Feed groups keyed by `${chatId} ${threadId ?? ''}`. */
|
|
476
|
+
const groups = new Map<string, FeedGroup>()
|
|
477
|
+
/** Reverse index agentId → feedKey, so the agentId-keyed public API resolves
|
|
478
|
+
* its group in O(1). Cleared when a worker's row is removed. */
|
|
479
|
+
const agentIndex = new Map<string, string>()
|
|
480
|
+
|
|
379
481
|
/**
|
|
380
|
-
* Agent ids that have been finalized (`
|
|
482
|
+
* Agent ids that have been finalized (`finish` latched). Survives row/group
|
|
381
483
|
* deletion so a LATE watcher `onProgress` tick — which can arrive after
|
|
382
|
-
* `finish()`'s chain has fully settled
|
|
383
|
-
*
|
|
384
|
-
*
|
|
385
|
-
*
|
|
386
|
-
*
|
|
387
|
-
* only needs to cover recent finalizations — capped at FINALIZED_CAP and
|
|
388
|
-
* trimmed FIFO to stay bounded across a long gateway lifetime.
|
|
484
|
+
* `finish()`'s chain has fully settled — cannot resurrect a fresh row and
|
|
485
|
+
* paint a running card on a worker that is already done. A late tick arrives
|
|
486
|
+
* within seconds of finish (watcher poll cadence), so the set only needs to
|
|
487
|
+
* cover recent finalizations — capped at FINALIZED_CAP and trimmed FIFO to
|
|
488
|
+
* stay bounded across a long gateway lifetime.
|
|
389
489
|
*/
|
|
390
490
|
const finalized = new Set<string>()
|
|
391
491
|
const FINALIZED_CAP = 256
|
|
@@ -393,414 +493,489 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
393
493
|
if (finalized.has(agentId)) return
|
|
394
494
|
finalized.add(agentId)
|
|
395
495
|
if (finalized.size > FINALIZED_CAP) {
|
|
396
|
-
// Map-free FIFO trim: Set iterates in insertion order; drop the oldest.
|
|
397
496
|
const oldest = finalized.values().next().value
|
|
398
497
|
if (oldest != null) finalized.delete(oldest)
|
|
399
498
|
}
|
|
400
499
|
}
|
|
401
500
|
let heartbeatTimer: unknown = null
|
|
402
501
|
|
|
403
|
-
function
|
|
502
|
+
function feedKeyOf(chatId: string, threadId?: number): string {
|
|
503
|
+
return `${chatId} ${threadId ?? ''}`
|
|
504
|
+
}
|
|
505
|
+
function groupOfAgent(agentId: string): FeedGroup | undefined {
|
|
506
|
+
const key = agentIndex.get(agentId)
|
|
507
|
+
return key != null ? groups.get(key) : undefined
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
function sendOptsFor(g: FeedGroup): Record<string, unknown> {
|
|
404
511
|
return {
|
|
405
512
|
disable_web_page_preview: true,
|
|
406
|
-
// Sub-agent progress
|
|
407
|
-
//
|
|
408
|
-
//
|
|
409
|
-
// that share these opts.)
|
|
513
|
+
// Sub-agent progress feed is a status surface, never the user's answer —
|
|
514
|
+
// silence the open ping. (editMessageText ignores disable_notification,
|
|
515
|
+
// so this is a no-op on the in-place edits that share these opts.)
|
|
410
516
|
disable_notification: true,
|
|
411
|
-
...(
|
|
517
|
+
...(g.threadId != null ? { message_thread_id: g.threadId } : {}),
|
|
412
518
|
}
|
|
413
519
|
}
|
|
414
520
|
|
|
415
|
-
function noteRateLimited(
|
|
521
|
+
function noteRateLimited(g: FeedGroup, err: unknown, label: string): void {
|
|
416
522
|
const retryAfter = extractRetryAfterSecs(err)
|
|
417
523
|
if (retryAfter == null) return
|
|
418
|
-
|
|
524
|
+
g.cooldownUntil = nowFn() + retryAfter * 1000 + COOLDOWN_JITTER_MS
|
|
419
525
|
log(`worker-feed: ${label} 429 — backing off ${retryAfter}s`)
|
|
420
526
|
}
|
|
421
527
|
|
|
422
|
-
|
|
528
|
+
/**
|
|
529
|
+
* Park a group in cooldown for the remaining of an open flood window (if
|
|
530
|
+
* any), so the heartbeat's `nowFn() < g.cooldownUntil` guard suppresses every
|
|
531
|
+
* further send/edit until the ban closes. Returns true when a window was open
|
|
532
|
+
* (caller should abandon the current attempt). Two reads of the SAME on-disk
|
|
533
|
+
* marker `robustApiCall` gates on — never a second notion of "channel open".
|
|
534
|
+
*/
|
|
535
|
+
function parkIfFloodWindowOpen(g: FeedGroup): boolean {
|
|
536
|
+
const remaining = floodWaitRemainingMs()
|
|
537
|
+
if (remaining <= 0) return false
|
|
538
|
+
const until = nowFn() + remaining + COOLDOWN_JITTER_MS
|
|
539
|
+
if (until > g.cooldownUntil) g.cooldownUntil = until
|
|
540
|
+
return true
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
function accumulateNarrative(row: WorkerRow, view: WorkerActivityView): void {
|
|
423
544
|
const line = view.latestSummary.trim()
|
|
424
545
|
if (line.length === 0) return
|
|
425
|
-
// Dedup within the whole rolling window
|
|
426
|
-
//
|
|
427
|
-
//
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
//
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
h.narrative.push(line)
|
|
435
|
-
// The `→` current-step line just CHANGED — reset the per-step timer so the
|
|
436
|
-
// heartbeat's `· Ns` suffix measures THIS step, not the whole worker run.
|
|
437
|
-
h.stepStartedAtMs = nowFn()
|
|
438
|
-
// Rolling window — keep only the last STATUS_ROLLING_LINES in memory. The
|
|
439
|
-
// render shows exactly those lines (clipped per-line by the unified pipeline);
|
|
440
|
-
// fitCardToBudget is the wire-limit backstop.
|
|
441
|
-
if (h.narrative.length > STATUS_ROLLING_LINES) {
|
|
442
|
-
h.narrative.splice(0, h.narrative.length - STATUS_ROLLING_LINES)
|
|
546
|
+
// Dedup within the whole rolling window (the watcher re-emits the same
|
|
547
|
+
// narrative across ticks, and a preamble + its tool label can repeat
|
|
548
|
+
// non-adjacently — the A,B,A duplication observed on live cards).
|
|
549
|
+
if (row.narrative.includes(line)) return
|
|
550
|
+
row.narrative.push(line)
|
|
551
|
+
// The `→` current-step line just CHANGED — reset the per-step timer.
|
|
552
|
+
row.stepStartedAtMs = nowFn()
|
|
553
|
+
if (row.narrative.length > STATUS_ROLLING_LINES) {
|
|
554
|
+
row.narrative.splice(0, row.narrative.length - STATUS_ROLLING_LINES)
|
|
443
555
|
}
|
|
444
556
|
}
|
|
445
557
|
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
// elapsed even between fresh views. lastView feeds the heartbeat re-render.
|
|
452
|
-
const merged: WorkerActivityView = { ...view, narrativeLines: [...h.narrative] }
|
|
453
|
-
h.lastView = merged
|
|
454
|
-
if (h.dispatchAtMs == null) h.dispatchAtMs = nowFn() - view.elapsedMs
|
|
455
|
-
if (nowFn() < h.cooldownUntil) return
|
|
456
|
-
const body = renderWorkerActivity(merged, liveSuffix)
|
|
457
|
-
|
|
458
|
-
// First paint: hold off until the worker has run long enough to be
|
|
459
|
-
// worth a message; trivial workers stay silent (handback covers them).
|
|
460
|
-
if (h.messageId == null) {
|
|
461
|
-
if (view.elapsedMs < firstPaintMin) return
|
|
462
|
-
try {
|
|
463
|
-
const sent = await opts.bot.sendMessage(h.chatId, body, sendOptsFor(h))
|
|
464
|
-
h.messageId = sent.message_id
|
|
465
|
-
h.lastBody = body
|
|
466
|
-
h.lastEditAt = nowFn()
|
|
467
|
-
log(
|
|
468
|
-
`worker-feed: paint agent=${h.agentId} chat=${h.chatId} ` +
|
|
469
|
-
`thread=${h.threadId ?? '-'} msgId=${h.messageId} bytes=${body.length}`,
|
|
470
|
-
)
|
|
471
|
-
} catch (err) {
|
|
472
|
-
noteRateLimited(h, err, 'send')
|
|
473
|
-
log(`worker-feed: send failed: ${(err as Error).message}`)
|
|
474
|
-
}
|
|
475
|
-
return
|
|
476
|
-
}
|
|
558
|
+
/** Live wall-clock elapsed for a worker (climbs between fresh views). */
|
|
559
|
+
function liveElapsed(row: WorkerRow, now: number): number {
|
|
560
|
+
const base = row.dispatchAtMs != null ? now - row.dispatchAtMs : row.lastView?.elapsedMs ?? 0
|
|
561
|
+
return Math.max(base, row.lastView?.elapsedMs ?? 0)
|
|
562
|
+
}
|
|
477
563
|
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
564
|
+
/** The running rows of a group, dispatch-ordered (oldest first, stable). */
|
|
565
|
+
function runningRows(g: FeedGroup): WorkerRow[] {
|
|
566
|
+
return [...g.workers.values()]
|
|
567
|
+
.filter((w) => w.state === 'running' && w.lastView != null)
|
|
568
|
+
.sort((a, b) => (a.dispatchAtMs ?? 0) - (b.dispatchAtMs ?? 0))
|
|
569
|
+
}
|
|
481
570
|
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
571
|
+
/**
|
|
572
|
+
* Render a group's shared message body at `now`.
|
|
573
|
+
* - `terminalRecap` set + zero running → the last worker's terminal recap
|
|
574
|
+
* (single 🛠 Worker card, done/failed).
|
|
575
|
+
* - exactly one running → the full 🛠 Worker card (single-worker parity).
|
|
576
|
+
* - 2+ running → the combined `renderCombinedWorkerFeed` body.
|
|
577
|
+
* - zero running, no recap → null (nothing to show).
|
|
578
|
+
* `heartbeat` toggles the single-worker climbing `· Ns` step suffix.
|
|
579
|
+
*/
|
|
580
|
+
function renderGroupBody(
|
|
581
|
+
g: FeedGroup,
|
|
582
|
+
now: number,
|
|
583
|
+
terminalRecap: WorkerActivityView | null,
|
|
584
|
+
heartbeat: boolean,
|
|
585
|
+
): string | null {
|
|
586
|
+
const running = runningRows(g)
|
|
587
|
+
if (running.length === 0) {
|
|
588
|
+
if (terminalRecap == null) return null
|
|
589
|
+
return renderWorkerActivity(terminalRecap)
|
|
590
|
+
}
|
|
591
|
+
// On a normal update/finish the header shows the worker's LAST-REPORTED
|
|
592
|
+
// elapsed (byte-stable for the dedup / no-op skip). Only the heartbeat —
|
|
593
|
+
// which fires when no fresh view arrived — climbs a live wall-clock elapsed
|
|
594
|
+
// so a silent worker still visibly advances.
|
|
595
|
+
const elapsedFor = (r: WorkerRow): number =>
|
|
596
|
+
heartbeat ? liveElapsed(r, now) : r.lastView?.elapsedMs ?? liveElapsed(r, now)
|
|
597
|
+
if (running.length === 1) {
|
|
598
|
+
const r = running[0]
|
|
599
|
+
const view: WorkerActivityView = {
|
|
600
|
+
...(r.lastView as WorkerActivityView),
|
|
601
|
+
elapsedMs: elapsedFor(r),
|
|
602
|
+
narrativeLines: [...r.narrative],
|
|
495
603
|
}
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
return
|
|
604
|
+
let liveSuffix = ''
|
|
605
|
+
if (heartbeat) {
|
|
606
|
+
const stepElapsed = r.stepStartedAtMs != null ? now - r.stepStartedAtMs : liveElapsed(r, now)
|
|
607
|
+
liveSuffix = formatStepSuffix(stepElapsed)
|
|
501
608
|
}
|
|
502
|
-
|
|
503
|
-
// Message/chat deleted or edit window closed — there is no card to
|
|
504
|
-
// update. Drop the handle silently; a fresh first-paint on the next
|
|
505
|
-
// running tick re-establishes one if the worker is still active. No
|
|
506
|
-
// warning: "no card" is not a liveness-logic error.
|
|
507
|
-
h.messageId = null
|
|
508
|
-
h.lastBody = null
|
|
509
|
-
return
|
|
510
|
-
}
|
|
511
|
-
// 'transient' — network blip / 5xx. Leave the handle intact; the
|
|
512
|
-
// heartbeat re-attempts on its next tick. Log at debug, not stderr-warn:
|
|
513
|
-
// a transport hiccup on a best-effort card is not "shit code", it's a
|
|
514
|
-
// retryable blip the framework rides out deterministically.
|
|
515
|
-
log(`worker-feed: edit transient error agent=${h.agentId}: ${(err as Error).message}`)
|
|
609
|
+
return renderWorkerActivity(view, liveSuffix)
|
|
516
610
|
}
|
|
611
|
+
const rows: CombinedWorkerRow[] = running.map((r) => {
|
|
612
|
+
const v = r.lastView as WorkerActivityView
|
|
613
|
+
const currentStep = r.narrative.length > 0 ? r.narrative[r.narrative.length - 1] : v.latestSummary
|
|
614
|
+
return {
|
|
615
|
+
description: v.description,
|
|
616
|
+
elapsedMs: elapsedFor(r),
|
|
617
|
+
toolCount: v.toolCount,
|
|
618
|
+
currentStep,
|
|
619
|
+
model: v.model,
|
|
620
|
+
}
|
|
621
|
+
})
|
|
622
|
+
return renderCombinedWorkerFeed(rows, { maxRows })
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/** Remove a worker's row + index entry; delete the group if it is now empty. */
|
|
626
|
+
function removeWorker(g: FeedGroup, agentId: string): void {
|
|
627
|
+
g.workers.delete(agentId)
|
|
628
|
+
agentIndex.delete(agentId)
|
|
629
|
+
if (g.workers.size === 0) groups.delete(g.feedKey)
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
/**
|
|
633
|
+
* Reconcile the GROUP-level status pin (#3207 review). Pin the shared message
|
|
634
|
+
* while the group has a posted message AND at least one tracked worker;
|
|
635
|
+
* unpin the instant the group empties. Because it is keyed by the whole group
|
|
636
|
+
* (not a single worker), a per-worker finish never unpins a message a sibling
|
|
637
|
+
* still needs — the survivors keep the pin until the LAST worker is done.
|
|
638
|
+
*/
|
|
639
|
+
function syncPin(g: FeedGroup): void {
|
|
640
|
+
const messageId = g.messageId != null && g.workers.size > 0 ? g.messageId : null
|
|
641
|
+
reconcilePinFn({ feedKey: g.feedKey, chatId: g.chatId, threadId: g.threadId, messageId })
|
|
517
642
|
}
|
|
518
643
|
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
//
|
|
533
|
-
|
|
534
|
-
|
|
644
|
+
/**
|
|
645
|
+
* Drive the group's shared message to the current combined body. Handles
|
|
646
|
+
* first-paint gating, the proactive throttle (bypassed by `force`), the
|
|
647
|
+
* dedup/no-op skip, the send-gate SHED contract, and 429/flood cooldown.
|
|
648
|
+
* `terminalRecap` (set on the last worker's finish) renders + finalizes the
|
|
649
|
+
* message, then removes the finished row.
|
|
650
|
+
*/
|
|
651
|
+
async function doRender(
|
|
652
|
+
g: FeedGroup,
|
|
653
|
+
opts2: { force?: boolean; heartbeat?: boolean; terminalRecap?: WorkerActivityView; finishingAgentId?: string } = {},
|
|
654
|
+
): Promise<void> {
|
|
655
|
+
const now = nowFn()
|
|
656
|
+
const isTerminal = opts2.terminalRecap != null
|
|
657
|
+
// Terminal edit RESOLVED (landed / not-modified / gone): clear the staged
|
|
658
|
+
// re-drive unconditionally so a heartbeat re-drive can never loop, and drop
|
|
659
|
+
// the finished row when its id is known (the last-worker finalize path).
|
|
660
|
+
const settleTerminal = (): void => {
|
|
661
|
+
g.pendingFinalize = null
|
|
662
|
+
if (opts2.finishingAgentId != null) removeWorker(g, opts2.finishingAgentId)
|
|
663
|
+
// Group-level pin follows membership: unpin once this drops the last
|
|
664
|
+
// worker; a NOOP-pin (siblings remain) keeps the shared message pinned.
|
|
665
|
+
syncPin(g)
|
|
666
|
+
}
|
|
667
|
+
if (now < g.cooldownUntil) {
|
|
668
|
+
if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
|
|
535
669
|
return
|
|
536
670
|
}
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
// cooldown expires — a transport hiccup can no longer leave a
|
|
542
|
-
// finished worker's card stuck on its last running render.
|
|
543
|
-
h.pendingFinish = view
|
|
671
|
+
// A flood window is open: the gate would SHED every call. Park in cooldown
|
|
672
|
+
// and make ZERO api calls until it closes; the heartbeat re-drives.
|
|
673
|
+
if (parkIfFloodWindowOpen(g)) {
|
|
674
|
+
if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
|
|
544
675
|
return
|
|
545
676
|
}
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
677
|
+
|
|
678
|
+
const body = renderGroupBody(g, now, opts2.terminalRecap ?? null, opts2.heartbeat ?? false)
|
|
679
|
+
if (body == null) {
|
|
680
|
+
// Nothing to show. On a terminal finalize with no message ever posted,
|
|
681
|
+
// just drop the finished row (the handback carries the result).
|
|
682
|
+
if (isTerminal) settleTerminal()
|
|
549
683
|
return
|
|
550
684
|
}
|
|
685
|
+
|
|
686
|
+
// First paint: hold until some worker in the group has run long enough.
|
|
687
|
+
if (g.messageId == null) {
|
|
688
|
+
const maxElapsed = Math.max(0, ...runningRows(g).map((r) => liveElapsed(r, now)))
|
|
689
|
+
// A terminal recap for a group that never painted → nothing to finalize;
|
|
690
|
+
// never first-paint a terminal card (trivial workers stay silent, the
|
|
691
|
+
// handback carries the result — matches the pre-coalesce doFinish guard).
|
|
692
|
+
if (isTerminal) {
|
|
693
|
+
settleTerminal()
|
|
694
|
+
return
|
|
695
|
+
}
|
|
696
|
+
if (maxElapsed < firstPaintMin) return
|
|
697
|
+
try {
|
|
698
|
+
const sent = await opts.bot.sendMessage(g.chatId, body, sendOptsFor(g))
|
|
699
|
+
// Shed contract (#3084): the gate resolves `undefined`/non-object for a
|
|
700
|
+
// shed send (open flood window) or dropped-stale `useful` TTL. That is
|
|
701
|
+
// NOT a delivered message — record no id, park on any window, let the
|
|
702
|
+
// heartbeat re-drive.
|
|
703
|
+
if (sent == null || typeof sent.message_id !== 'number') {
|
|
704
|
+
parkIfFloodWindowOpen(g)
|
|
705
|
+
log(`worker-feed: first paint shed by send gate feed=${g.feedKey} — not delivered`)
|
|
706
|
+
return
|
|
707
|
+
}
|
|
708
|
+
g.messageId = sent.message_id
|
|
709
|
+
g.lastBody = body
|
|
710
|
+
g.lastEditAt = now
|
|
711
|
+
// Group's first (or re-established) message is up → pin it for the group.
|
|
712
|
+
syncPin(g)
|
|
713
|
+
log(
|
|
714
|
+
`worker-feed: paint feed=${g.feedKey} chat=${g.chatId} ` +
|
|
715
|
+
`thread=${g.threadId ?? '-'} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`,
|
|
716
|
+
)
|
|
717
|
+
} catch (err) {
|
|
718
|
+
noteRateLimited(g, err, 'send')
|
|
719
|
+
log(`worker-feed: send failed: ${(err as Error).message}`)
|
|
720
|
+
}
|
|
721
|
+
return
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
// Dedup + proactive throttle (finish/terminal edits force through).
|
|
725
|
+
if (body === g.lastBody) {
|
|
726
|
+
if (isTerminal) settleTerminal()
|
|
727
|
+
return
|
|
728
|
+
}
|
|
729
|
+
if (!opts2.force && now - g.lastEditAt < minEditInterval) return
|
|
730
|
+
|
|
551
731
|
try {
|
|
552
|
-
await opts.bot.editMessageText(
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
732
|
+
const res = await opts.bot.editMessageText(g.chatId, g.messageId, body, sendOptsFor(g))
|
|
733
|
+
// Shed honesty (#3084): a cosmetic edit the gate shed resolves the
|
|
734
|
+
// distinguishable SEND_GATE_SHED sentinel (NOT a bare `undefined`, which
|
|
735
|
+
// the gate reserves for a benign no-op drop whose payload IS on screen).
|
|
736
|
+
// The shed payload is NOT on screen, so do not record it as `lastBody`.
|
|
737
|
+
if (isSendGateShed(res)) {
|
|
738
|
+
parkIfFloodWindowOpen(g)
|
|
739
|
+
if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
|
|
740
|
+
return
|
|
741
|
+
}
|
|
742
|
+
g.lastBody = body
|
|
743
|
+
g.lastEditAt = now
|
|
744
|
+
if (isTerminal) {
|
|
745
|
+
log(
|
|
746
|
+
`worker-feed: finish feed=${g.feedKey} chat=${g.chatId} thread=${g.threadId ?? '-'} ` +
|
|
747
|
+
`msgId=${g.messageId} agent=${opts2.finishingAgentId ?? '-'} ` +
|
|
748
|
+
`state=${opts2.terminalRecap?.state ?? 'done'} bytes=${body.length}`,
|
|
749
|
+
)
|
|
750
|
+
} else {
|
|
751
|
+
log(
|
|
752
|
+
`worker-feed: edit feed=${g.feedKey} chat=${g.chatId} ` +
|
|
753
|
+
`thread=${g.threadId ?? '-'} msgId=${g.messageId} workers=${g.workers.size} bytes=${body.length}`,
|
|
754
|
+
)
|
|
755
|
+
}
|
|
756
|
+
if (isTerminal) settleTerminal()
|
|
560
757
|
} catch (err) {
|
|
561
758
|
const outcome = classifyEditError(err)
|
|
562
759
|
if (outcome === 'rate_limited') {
|
|
563
|
-
noteRateLimited(
|
|
564
|
-
|
|
565
|
-
h.pendingFinish = view
|
|
760
|
+
noteRateLimited(g, err, isTerminal ? 'finish' : 'edit')
|
|
761
|
+
if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
|
|
566
762
|
return
|
|
567
763
|
}
|
|
568
764
|
if (outcome === 'not_modified') {
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
h.pendingFinish = null
|
|
765
|
+
g.lastBody = body
|
|
766
|
+
g.lastEditAt = now
|
|
767
|
+
if (isTerminal) settleTerminal()
|
|
573
768
|
return
|
|
574
769
|
}
|
|
575
770
|
if (outcome === 'gone') {
|
|
576
|
-
// Message/chat gone — no card to
|
|
577
|
-
//
|
|
578
|
-
|
|
771
|
+
// Message/chat gone or edit window closed — no card to update. Drop the
|
|
772
|
+
// stale message id; a fresh first-paint re-establishes one if workers
|
|
773
|
+
// are still live. On a terminal finalize, also drop the finished row.
|
|
774
|
+
g.messageId = null
|
|
775
|
+
g.lastBody = null
|
|
776
|
+
// The pinned message no longer exists → release the group pin claim
|
|
777
|
+
// (settleTerminal already re-syncs on the terminal path).
|
|
778
|
+
if (isTerminal) settleTerminal()
|
|
779
|
+
else syncPin(g)
|
|
579
780
|
return
|
|
580
781
|
}
|
|
581
|
-
// 'transient' —
|
|
582
|
-
|
|
583
|
-
log(`worker-feed:
|
|
782
|
+
// 'transient' — leave the message intact; the heartbeat re-attempts.
|
|
783
|
+
if (isTerminal && opts2.terminalRecap != null) g.pendingFinalize = opts2.terminalRecap
|
|
784
|
+
log(`worker-feed: edit transient error feed=${g.feedKey}: ${(err as Error).message}`)
|
|
584
785
|
}
|
|
585
786
|
}
|
|
586
787
|
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
* FIRST paint for a prose-silent worker whose only tick arrived before
|
|
590
|
-
* `firstPaintMin`.
|
|
591
|
-
*
|
|
592
|
-
* Why the first-paint branch exists: a background worker that dives
|
|
593
|
-
* straight into quiet work (e.g. a long `Bash` / `npm test`) emits a
|
|
594
|
-
* single `sub_agent_tool_use` event when the command is invoked, then no
|
|
595
|
-
* further JSONL lines for the whole run. That one tick drives `update`
|
|
596
|
-
* once — but if it lands before `firstPaintMin` the paint is held, and
|
|
597
|
-
* with no subsequent tick nothing ever re-drives it, so the worker shows
|
|
598
|
-
* NOTHING for its entire run (the "I can't see the worker" gap). The
|
|
599
|
-
* heartbeat closes it: once such a handle is past `firstPaintMin`, drive a
|
|
600
|
-
* paint here through the same chain → doUpdate path. After first paint the
|
|
601
|
-
* suffix-only maintenance branch keeps it advancing.
|
|
602
|
-
*
|
|
603
|
-
* For handles that already have a posted message, this is the original
|
|
604
|
-
* option-(a), suffix-only re-render (never editMessageText directly). Skips:
|
|
605
|
-
* - handles inside a 429 cooldown,
|
|
606
|
-
* - handles with no `lastView` (no update ever arrived) or non-running,
|
|
607
|
-
* - for the maintenance branch: handles edited within minEditInterval
|
|
608
|
-
* (no stampede) or whose current step isn't yet stale.
|
|
609
|
-
* The `· Ns` liveSuffix is applied ONLY when the worker's current step is
|
|
610
|
-
* stale (now - lastEditAt >= heartbeatTickMs) so a normally-ticking worker is
|
|
611
|
-
* untouched and its body stays byte-stable for the dedup.
|
|
612
|
-
*/
|
|
788
|
+
// Arm the heartbeat once at construction. The real timer is `.unref()`'d so
|
|
789
|
+
// it never keeps the process alive; tests inject setInterval/clearInterval.
|
|
613
790
|
function heartbeatTick(): void {
|
|
614
791
|
const now = nowFn()
|
|
615
|
-
for (const
|
|
616
|
-
//
|
|
617
|
-
//
|
|
618
|
-
//
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
// (or a transient error) was staged on `pendingFinish` by `doFinish`.
|
|
626
|
-
// Re-drive it once the cooldown has expired so a finished worker's card
|
|
627
|
-
// can't get stuck on its last running render. This is the deterministic
|
|
628
|
-
// backstop that replaces the old "stale but harmless" surrender — the
|
|
629
|
-
// framework owns ALIVE-and-done, wall-clock driven, no model in the loop.
|
|
630
|
-
// (The handle is still in the map because `finish()`'s `.finally(delete)`
|
|
631
|
-
// is chained AFTER `doFinish` and won't drain while a re-drive keeps the
|
|
632
|
-
// chain busy; once the terminal edit lands, `pendingFinish` is cleared
|
|
633
|
-
// and the `.finally` runs on the next chain settle.)
|
|
634
|
-
if (h.pendingFinish != null && now >= h.cooldownUntil) {
|
|
635
|
-
const view = h.pendingFinish
|
|
636
|
-
h.chain = h.chain
|
|
637
|
-
.then(() => doFinish(h, view))
|
|
792
|
+
for (const g of [...groups.values()]) {
|
|
793
|
+
// Deferred-finalize re-drive: a terminal edit that hit a cooldown/flood
|
|
794
|
+
// window was staged on `pendingFinalize`. Re-drive it once the cooldown
|
|
795
|
+
// expires so a finished feed can't get stuck on its last running render.
|
|
796
|
+
if (g.pendingFinalize != null && now >= g.cooldownUntil) {
|
|
797
|
+
const recap = g.pendingFinalize
|
|
798
|
+
// The finishing agent is whatever finished row remains (state terminal).
|
|
799
|
+
const finishingAgentId = [...g.workers.values()].find((w) => w.finished)?.agentId
|
|
800
|
+
g.chain = g.chain
|
|
801
|
+
.then(() => doRender(g, { force: true, terminalRecap: recap, finishingAgentId }))
|
|
638
802
|
.catch((err) => {
|
|
639
|
-
log(`worker-feed: heartbeat finalize re-drive error
|
|
640
|
-
})
|
|
641
|
-
.finally(() => {
|
|
642
|
-
// Mirror `finish()`'s teardown: once the re-driven `doFinish`
|
|
643
|
-
// clears `pendingFinish` (terminal edit landed OR permanently
|
|
644
|
-
// failed), drop the handle. If it re-staged (another 429), the
|
|
645
|
-
// handle survives for the next heartbeat tick to retry.
|
|
646
|
-
if (handles.get(h.agentId)?.pendingFinish == null) {
|
|
647
|
-
handles.delete(h.agentId)
|
|
648
|
-
}
|
|
803
|
+
log(`worker-feed: heartbeat finalize re-drive error feed=${g.feedKey}: ${(err as Error).message}`)
|
|
649
804
|
})
|
|
650
805
|
continue
|
|
651
806
|
}
|
|
652
807
|
|
|
653
|
-
if (
|
|
654
|
-
|
|
655
|
-
if (
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
if (liveElapsed < firstPaintMin) continue
|
|
665
|
-
const view = { ...h.lastView, elapsedMs: Math.max(h.lastView.elapsedMs, liveElapsed) }
|
|
666
|
-
h.chain = h.chain
|
|
667
|
-
.then(() => doUpdate(h, view))
|
|
808
|
+
if (now < g.cooldownUntil) continue
|
|
809
|
+
const running = runningRows(g)
|
|
810
|
+
if (running.length === 0) continue
|
|
811
|
+
|
|
812
|
+
// First-paint path: no message yet and some worker has now crossed
|
|
813
|
+
// firstPaintMin (a prose-silent worker's single early tick was held).
|
|
814
|
+
if (g.messageId == null) {
|
|
815
|
+
const maxElapsed = Math.max(0, ...running.map((r) => liveElapsed(r, now)))
|
|
816
|
+
if (maxElapsed < firstPaintMin) continue
|
|
817
|
+
g.chain = g.chain
|
|
818
|
+
.then(() => doRender(g, {}))
|
|
668
819
|
.catch((err) => {
|
|
669
|
-
log(`worker-feed: heartbeat first-paint chain error
|
|
820
|
+
log(`worker-feed: heartbeat first-paint chain error feed=${g.feedKey}: ${(err as Error).message}`)
|
|
670
821
|
})
|
|
671
822
|
continue
|
|
672
823
|
}
|
|
673
824
|
|
|
674
|
-
if (now -
|
|
675
|
-
const stale = now -
|
|
825
|
+
if (now - g.lastEditAt < minEditInterval) continue
|
|
826
|
+
const stale = now - g.lastEditAt >= heartbeatTickMs
|
|
676
827
|
if (!stale) continue
|
|
677
|
-
//
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
// Under STEP_TIMER_MIN_MS formatStepSuffix returns '' (no timer yet);
|
|
681
|
-
// the header elapsed still climbs via the refreshed view below.
|
|
682
|
-
const stepElapsed = h.stepStartedAtMs != null ? now - h.stepStartedAtMs : liveElapsed
|
|
683
|
-
const liveSuffix = formatStepSuffix(stepElapsed)
|
|
684
|
-
// Re-render THROUGH the chain + doUpdate path — never editMessageText directly.
|
|
685
|
-
//
|
|
686
|
-
// CLOCK-ANCHOR PARITY: refresh the view's elapsedMs to the same `now`
|
|
687
|
-
// anchor the step suffix uses. The header renders
|
|
688
|
-
// `view.elapsedMs`; passing the stale lastView froze the header at the
|
|
689
|
-
// last watcher event while the `· Ns` suffix kept ticking, so the
|
|
690
|
-
// current step's timer could read MORE than the card's master elapsed
|
|
691
|
-
// (Ken-observed defect). Both numbers now derive from one anchor
|
|
692
|
-
// (dispatchAtMs) at one `now`, so header elapsed >= step suffix always.
|
|
693
|
-
const view = { ...h.lastView, elapsedMs: Math.max(h.lastView.elapsedMs, liveElapsed) }
|
|
694
|
-
h.chain = h.chain
|
|
695
|
-
.then(() => doUpdate(h, view, liveSuffix))
|
|
828
|
+
// Re-render THROUGH the chain → doRender path with climbing elapsed.
|
|
829
|
+
g.chain = g.chain
|
|
830
|
+
.then(() => doRender(g, { heartbeat: true }))
|
|
696
831
|
.catch((err) => {
|
|
697
|
-
log(`worker-feed: heartbeat chain error
|
|
832
|
+
log(`worker-feed: heartbeat chain error feed=${g.feedKey}: ${(err as Error).message}`)
|
|
698
833
|
})
|
|
699
834
|
}
|
|
700
835
|
}
|
|
701
836
|
|
|
702
|
-
// Arm the heartbeat once at construction. The real timer is `.unref()`'d so
|
|
703
|
-
// it never keeps the process alive; tests inject setInterval/clearInterval.
|
|
704
837
|
heartbeatTimer = setIntervalFn(heartbeatTick, heartbeatTickMs)
|
|
705
838
|
|
|
706
839
|
return {
|
|
707
840
|
has(agentId) {
|
|
708
|
-
|
|
841
|
+
const g = groupOfAgent(agentId)
|
|
842
|
+
return g != null && g.messageId != null && g.workers.has(agentId)
|
|
709
843
|
},
|
|
710
844
|
messageIdOf(agentId) {
|
|
711
|
-
return
|
|
845
|
+
return groupOfAgent(agentId)?.messageId ?? null
|
|
846
|
+
},
|
|
847
|
+
hasRunningInFeed(feedKey) {
|
|
848
|
+
const g = groups.get(feedKey)
|
|
849
|
+
return g != null && g.workers.size > 0
|
|
712
850
|
},
|
|
713
851
|
get size() {
|
|
714
|
-
|
|
852
|
+
let n = 0
|
|
853
|
+
for (const g of groups.values()) n += g.workers.size
|
|
854
|
+
return n
|
|
715
855
|
},
|
|
716
856
|
update(agentId, chatId, view, threadId) {
|
|
717
|
-
// No chat to post to (owner DM unconfigured) — don't create
|
|
718
|
-
//
|
|
857
|
+
// No chat to post to (owner DM unconfigured) — don't create state that
|
|
858
|
+
// would retry a failing send('') every tick.
|
|
719
859
|
if (chatId.length === 0) return Promise.resolve()
|
|
720
|
-
// Resurrection guard: a worker
|
|
721
|
-
// (
|
|
722
|
-
// A late watcher `onProgress` tick can arrive after `finish()`'s chain
|
|
723
|
-
// has fully settled and the handle was deleted — without this durable
|
|
724
|
-
// gate the tick would create a brand-new handle and paint a fresh
|
|
725
|
-
// `running` message on an already-done worker (the card lies). The
|
|
726
|
-
// heartbeat's orphan-paint guard covers the heartbeat tick only; the
|
|
727
|
-
// per-handle `finished` flag covers the pre-delete window; this set
|
|
728
|
-
// covers the post-delete window.
|
|
860
|
+
// Resurrection guard: a worker already finalized must not get a fresh
|
|
861
|
+
// running cue (a late watcher tick would repaint a done worker as live).
|
|
729
862
|
if (finalized.has(agentId)) return Promise.resolve()
|
|
730
|
-
const
|
|
731
|
-
if (
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
863
|
+
const existingRow = groupOfAgent(agentId)?.workers.get(agentId)
|
|
864
|
+
if (existingRow?.finished === true) return Promise.resolve()
|
|
865
|
+
|
|
866
|
+
const feedKey = feedKeyOf(chatId, threadId)
|
|
867
|
+
let g = groups.get(feedKey)
|
|
868
|
+
if (g == null) {
|
|
869
|
+
g = {
|
|
870
|
+
feedKey,
|
|
736
871
|
chatId,
|
|
737
872
|
threadId,
|
|
738
873
|
messageId: null,
|
|
739
874
|
lastBody: null,
|
|
740
875
|
lastEditAt: 0,
|
|
741
876
|
cooldownUntil: 0,
|
|
742
|
-
narrative: [],
|
|
743
877
|
chain: Promise.resolve(),
|
|
878
|
+
workers: new Map(),
|
|
879
|
+
pendingFinalize: null,
|
|
880
|
+
}
|
|
881
|
+
groups.set(feedKey, g)
|
|
882
|
+
}
|
|
883
|
+
let row = g.workers.get(agentId)
|
|
884
|
+
if (row == null) {
|
|
885
|
+
row = {
|
|
886
|
+
agentId,
|
|
887
|
+
narrative: [],
|
|
744
888
|
lastView: null,
|
|
889
|
+
state: 'running',
|
|
890
|
+
finished: false,
|
|
745
891
|
dispatchAtMs: null,
|
|
746
892
|
stepStartedAtMs: null,
|
|
747
|
-
finished: false,
|
|
748
|
-
pendingFinish: null,
|
|
749
893
|
}
|
|
750
|
-
|
|
894
|
+
g.workers.set(agentId, row)
|
|
895
|
+
agentIndex.set(agentId, feedKey)
|
|
751
896
|
}
|
|
752
|
-
|
|
753
|
-
|
|
897
|
+
// Accumulate before the gate so a throttled tick still grows the
|
|
898
|
+
// narrative — it surfaces on the next edit that does fire.
|
|
899
|
+
accumulateNarrative(row, view)
|
|
900
|
+
row.state = 'running'
|
|
901
|
+
row.lastView = { ...view, narrativeLines: [...row.narrative] }
|
|
902
|
+
if (row.dispatchAtMs == null) row.dispatchAtMs = nowFn() - view.elapsedMs
|
|
903
|
+
|
|
904
|
+
const group = g
|
|
905
|
+
group.chain = group.chain.then(() => doRender(group)).catch((err) => {
|
|
754
906
|
log(`worker-feed: update chain error ${agentId}: ${(err as Error).message}`)
|
|
755
907
|
})
|
|
756
|
-
return
|
|
908
|
+
return group.chain
|
|
757
909
|
},
|
|
758
910
|
finish(agentId, view) {
|
|
759
|
-
const
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
911
|
+
const g = groupOfAgent(agentId)
|
|
912
|
+
const row = g?.workers.get(agentId)
|
|
913
|
+
if (g == null || row == null) {
|
|
914
|
+
// Never tracked (trivial worker) — mark finalized so a late tick can't
|
|
915
|
+
// resurrect, and let the handback carry the result.
|
|
916
|
+
markFinalized(agentId)
|
|
917
|
+
return Promise.resolve()
|
|
918
|
+
}
|
|
919
|
+
// Latch synchronously so a late `running` cue on the chain can't resurrect.
|
|
920
|
+
row.finished = true
|
|
921
|
+
row.state = view.state === 'failed' ? 'failed' : 'done'
|
|
922
|
+
markFinalized(agentId)
|
|
923
|
+
|
|
924
|
+
const group = g
|
|
925
|
+
group.chain = group.chain
|
|
926
|
+
.then(() => {
|
|
927
|
+
const others = runningRows(group).filter((w) => w.agentId !== agentId)
|
|
928
|
+
if (others.length > 0) {
|
|
929
|
+
// Siblings still live → drop this row from the combined body and
|
|
930
|
+
// re-render the running set. The result reaches the user via the
|
|
931
|
+
// separate handback, never folded into this cosmetic edit. The
|
|
932
|
+
// group pin STAYS (siblings still need the shared message) — a
|
|
933
|
+
// per-worker unpin here was the #3207 review blocker.
|
|
934
|
+
removeWorker(group, agentId)
|
|
935
|
+
syncPin(group)
|
|
936
|
+
return doRender(group, { force: true })
|
|
937
|
+
}
|
|
938
|
+
// Last live worker → finalize the shared message to its terminal recap.
|
|
939
|
+
const recap: WorkerActivityView = { ...view, narrativeLines: [...row.narrative] }
|
|
940
|
+
return doRender(group, { force: true, terminalRecap: recap, finishingAgentId: agentId })
|
|
941
|
+
})
|
|
763
942
|
.catch((err) => {
|
|
764
943
|
log(`worker-feed: finish chain error ${agentId}: ${(err as Error).message}`)
|
|
765
944
|
})
|
|
766
|
-
|
|
767
|
-
// Only tear down the handle once the terminal edit has actually
|
|
768
|
-
// landed (or permanently failed). If `doFinish` staged the edit on
|
|
769
|
-
// `pendingFinish` (a 429 cooldown / transient error was in effect),
|
|
770
|
-
// the handle must survive so the heartbeat can re-drive the
|
|
771
|
-
// finalize after cooldown. The heartbeat's re-drive chain ends by
|
|
772
|
-
// re-entering `doFinish`, which clears `pendingFinish` on success
|
|
773
|
-
// or permanent-failure — so this `.finally` deletes on the NEXT
|
|
774
|
-
// chain settle once there is nothing left to finalize. Without this
|
|
775
|
-
// guard, the `.finally` would delete the handle (and its staged
|
|
776
|
-
// pendingFinish) immediately after the first staged doFinish,
|
|
777
|
-
// stranding the card on its last running render.
|
|
778
|
-
if (handles.get(agentId)?.pendingFinish == null) {
|
|
779
|
-
handles.delete(agentId)
|
|
780
|
-
}
|
|
781
|
-
})
|
|
782
|
-
return h.chain
|
|
945
|
+
return group.chain
|
|
783
946
|
},
|
|
784
947
|
drop(agentId) {
|
|
785
|
-
// A dropped worker is also done — mark finalized so a late
|
|
786
|
-
//
|
|
948
|
+
// A dropped worker is also done — mark finalized so a late tick can't
|
|
949
|
+
// resurrect a running card on it (same gate as `finish`).
|
|
787
950
|
markFinalized(agentId)
|
|
788
|
-
|
|
951
|
+
const g = groupOfAgent(agentId)
|
|
952
|
+
if (g == null) return
|
|
953
|
+
const hadMessage = g.messageId != null
|
|
954
|
+
removeWorker(g, agentId)
|
|
955
|
+
// Group pin follows membership: unpin if this emptied the group, else
|
|
956
|
+
// keep it (siblings still need the shared message).
|
|
957
|
+
if (hadMessage) syncPin(g)
|
|
958
|
+
// Re-render so the dropped worker disappears from a combined body. Skip
|
|
959
|
+
// when the group is gone (removeWorker deleted it) or never painted.
|
|
960
|
+
if (hadMessage && groups.has(g.feedKey) && runningRows(g).length > 0) {
|
|
961
|
+
g.chain = g.chain
|
|
962
|
+
.then(() => doRender(g, { force: true }))
|
|
963
|
+
.catch((err) => {
|
|
964
|
+
log(`worker-feed: drop re-render error ${agentId}: ${(err as Error).message}`)
|
|
965
|
+
})
|
|
966
|
+
}
|
|
789
967
|
},
|
|
790
968
|
resurrect(agentId) {
|
|
791
969
|
// Issue #3023: the worker's card was falsely finalized and its JSONL has
|
|
792
|
-
// resumed. Re-open the paint path: drop the durable finalized gate
|
|
793
|
-
//
|
|
794
|
-
// again, and un-latch any surviving handle (finish deleted it in the
|
|
795
|
-
// common case, but a staged pendingFinish could keep it alive). The next
|
|
796
|
-
// `update` tick from the watcher's replayed progress does the repaint.
|
|
970
|
+
// resumed. Re-open the paint path: drop the durable finalized gate + any
|
|
971
|
+
// surviving per-row latch so a fresh `running` cue repaints a live card.
|
|
797
972
|
const wasFinalized = finalized.delete(agentId)
|
|
798
|
-
const
|
|
799
|
-
if (
|
|
800
|
-
|
|
801
|
-
|
|
973
|
+
const row = groupOfAgent(agentId)?.workers.get(agentId)
|
|
974
|
+
if (row != null) {
|
|
975
|
+
row.finished = false
|
|
976
|
+
row.state = 'running'
|
|
802
977
|
}
|
|
803
|
-
if (wasFinalized ||
|
|
978
|
+
if (wasFinalized || row != null) {
|
|
804
979
|
log(`worker-feed: resurrect agent=${agentId} — cleared finalized gate; card will repaint on next running cue`)
|
|
805
980
|
}
|
|
806
981
|
},
|