pi-crew 0.11.40 → 0.11.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/dist/index.mjs +495 -495
- package/package.json +1 -1
- package/src/runtime/foreground-watchdog.ts +213 -26
- package/src/runtime/task-runner/live-executor.ts +146 -45
package/package.json
CHANGED
|
@@ -6,7 +6,9 @@
|
|
|
6
6
|
* assistant has no way to know when a run completes or gets stuck without
|
|
7
7
|
* manual polling. This watchdog monitors active runs and:
|
|
8
8
|
*
|
|
9
|
-
* 1. Detects hung runs (active status,
|
|
9
|
+
* 1. Detects hung runs (active status, manifest not updated for >2 min with
|
|
10
|
+
* no agent activity evidence — wave 13: a live registry claim or a fresh
|
|
11
|
+
* task heartbeat withholds the verdict, see CLAIMED_RUN_HUNG_MS)
|
|
10
12
|
* 2. Injects a followUp message via pi.sendUserMessage() so the assistant
|
|
11
13
|
* is automatically notified — no manual sleep+check needed.
|
|
12
14
|
* 3. Cleans up after itself when the run completes or the session ends.
|
|
@@ -19,6 +21,7 @@
|
|
|
19
21
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
20
22
|
import { getCrewEnv, getCrewEnvInt } from "../config/env-vars.ts";
|
|
21
23
|
import { TEAM_TERMINAL_TASK_STATUSES } from "../state/contracts.ts";
|
|
24
|
+
import { activeRunEntries } from "../state/stores/active-run-registry.ts";
|
|
22
25
|
import { loadRunManifestById } from "../state/stores/state-store.ts";
|
|
23
26
|
import type { TeamTaskState } from "../state/types.ts";
|
|
24
27
|
import { readCrewAgents } from "./crew-agent-records.ts";
|
|
@@ -128,6 +131,170 @@ function resolveCheckIntervalMs(overrideMs: number | undefined): number {
|
|
|
128
131
|
return DEFAULT_CHECK_INTERVAL_MS;
|
|
129
132
|
}
|
|
130
133
|
|
|
134
|
+
// ── Wave 13 (F1, GAP-0): hung-verdict liveness short-circuit ────────────────
|
|
135
|
+
//
|
|
136
|
+
// Live evidence (2026-10-24/25): five false "appears hung" alerts in one
|
|
137
|
+
// session, all direct single-agent runs executing ONE long tool call (a
|
|
138
|
+
// 20-min CI poll loop, a 20-min test suite). The hung verdict
|
|
139
|
+
// (isLikelyOrphanedActiveRun: manifest stale ≥2 min + no agent activity
|
|
140
|
+
// within 5 min) is structurally blind while a tool executes — every
|
|
141
|
+
// liveness write is event-driven and no pi events fire mid-tool — so a
|
|
142
|
+
// HEALTHY run reads as hung ~5-10 min into a long tool. The two signals
|
|
143
|
+
// other consumers already trust, and this verdict ignores, are consulted
|
|
144
|
+
// here IN THE WATCHDOG (never inside isLikelyOrphanedActiveRun — that
|
|
145
|
+
// predicate is shared with the dashboard's isDisplayActiveRun, GAP-4):
|
|
146
|
+
// (a) a LIVE active-run-registry claim, and
|
|
147
|
+
// (b) a fresh task heartbeat (kept fresh through long tools by F2's
|
|
148
|
+
// live-executor timer).
|
|
149
|
+
// The two branches are INDEPENDENT ORs: the registry drops non-async
|
|
150
|
+
// entries once the manifest is >30 min stale, so past that horizon only the
|
|
151
|
+
// heartbeat carries liveness — never require both.
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Wave 13 (F1, GAP-0): hard ceiling for how long a LIVE active-run-registry
|
|
155
|
+
* claim can shield a run from the watchdog's hung verdict. A claimed run
|
|
156
|
+
* whose NEWEST liveness evidence (claim assertion time, task heartbeats/
|
|
157
|
+
* activity) is older than this still alerts — a genuine in-process deadlock
|
|
158
|
+
* (claim held, nothing progressing) is still caught — while healthy claimed
|
|
159
|
+
* runs mid-long-tool are not false-alarmed by the 2-min ladder. 45 min is
|
|
160
|
+
* 1.5× the registry's own 30-min freshness horizon and 9× the 5-min activity
|
|
161
|
+
* shield, so only a truly stuck claim trips it. Unclaimed runs keep the
|
|
162
|
+
* fast 2-min ladder unchanged; ORPHANED_ACTIVE_RUN_MS (process-status.ts,
|
|
163
|
+
* shared with the dashboard — GAP-4) is NOT touched.
|
|
164
|
+
*/
|
|
165
|
+
export const CLAIMED_RUN_HUNG_MS = 45 * 60 * 1000;
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Wave 13 (F1): freshness window for task-heartbeat liveness evidence —
|
|
169
|
+
* mirrors ACTIVE_EVIDENCE_TTL_MS in stale-reconciler.ts (5 min).
|
|
170
|
+
*/
|
|
171
|
+
const TASK_HEARTBEAT_EVIDENCE_MS = 5 * 60 * 1000;
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Wave 13 (F1): read-only active-run-registry claim consult result.
|
|
175
|
+
* `claimedAtMs` is the registry entry's `updatedAt` (the moment the claim
|
|
176
|
+
* was asserted — registerActiveRun fires at dispatch); undefined when the
|
|
177
|
+
* entry carries no parseable timestamp.
|
|
178
|
+
*/
|
|
179
|
+
interface LiveRunClaim {
|
|
180
|
+
claimedAtMs?: number;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Wave 13 (F1, GAP-0): consult the active-run registry for a LIVE claim on
|
|
185
|
+
* `runId`. Read-only mirror of findLiveRegistryEntry (stale-reconciler G24):
|
|
186
|
+
* activeRunEntries() applies the full liveness filter itself (terminal
|
|
187
|
+
* status, dead async PID, >30-min-stale non-async manifests, symlink
|
|
188
|
+
* safety), so an entry found here means some live session/runner claims the
|
|
189
|
+
* run RIGHT NOW. Inlined via the registry store instead of importing
|
|
190
|
+
* stale-reconciler (its reaping graph must not enter the watchdog's import
|
|
191
|
+
* closure); the async-PID half of hasLiveRunClaim is deliberately omitted —
|
|
192
|
+
* async runs never reach this verdict path (isLikelyOrphanedActiveRun
|
|
193
|
+
* excludes them). Fail-open: any registry error ⇒ no claim — a broken
|
|
194
|
+
* registry must never shield a genuinely hung run (G24 contract).
|
|
195
|
+
*/
|
|
196
|
+
function consultActiveRunRegistry(runId: string): LiveRunClaim | undefined {
|
|
197
|
+
try {
|
|
198
|
+
const entry = activeRunEntries().find((item) => item.runId === runId);
|
|
199
|
+
if (!entry) return undefined;
|
|
200
|
+
const claimedAtMs = entry.updatedAt ? Date.parse(entry.updatedAt) : Number.NaN;
|
|
201
|
+
return { claimedAtMs: Number.isFinite(claimedAtMs) ? claimedAtMs : undefined };
|
|
202
|
+
} catch {
|
|
203
|
+
return undefined;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
let liveClaimConsult: (runId: string) => LiveRunClaim | undefined = consultActiveRunRegistry;
|
|
208
|
+
|
|
209
|
+
/** @internal Test seam — replace the registry claim consult (null = real). */
|
|
210
|
+
export function setLiveClaimConsultForTests(consult: ((runId: string) => LiveRunClaim | undefined) | null): void {
|
|
211
|
+
liveClaimConsult = consult ?? consultActiveRunRegistry;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Wave 13 (F1): does any running/waiting task carry FRESH live evidence?
|
|
216
|
+
* Behavioral mirror of hasRecentActiveEvidence (stale-reconciler.ts): a
|
|
217
|
+
* running/waiting task whose `heartbeat.lastSeenAt` (with `alive !== false`)
|
|
218
|
+
* or `agentProgress.lastActivityAt` is within TASK_HEARTBEAT_EVIDENCE_MS
|
|
219
|
+
* counts. Absent/unreadable tasks (`[]`) ⇒ false — fail-open to the ladder,
|
|
220
|
+
* same as WD-STALE-RECORD.
|
|
221
|
+
*/
|
|
222
|
+
function hasFreshTaskHeartbeatEvidence(tasks: TeamTaskState[] | undefined, now: number): boolean {
|
|
223
|
+
if (!Array.isArray(tasks)) return false;
|
|
224
|
+
return tasks.some((task) => {
|
|
225
|
+
if (task.status !== "running" && task.status !== "waiting") return false;
|
|
226
|
+
const heartbeatAt = task.heartbeat?.lastSeenAt ? new Date(task.heartbeat.lastSeenAt).getTime() : Number.NaN;
|
|
227
|
+
if (task.heartbeat?.alive !== false && Number.isFinite(heartbeatAt) && now - heartbeatAt <= TASK_HEARTBEAT_EVIDENCE_MS) return true;
|
|
228
|
+
const activityAt = task.agentProgress?.lastActivityAt ? new Date(task.agentProgress.lastActivityAt).getTime() : Number.NaN;
|
|
229
|
+
return Number.isFinite(activityAt) && now - activityAt <= TASK_HEARTBEAT_EVIDENCE_MS;
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Wave 13 (F1): newest liveness-evidence timestamp (epoch ms) among the
|
|
235
|
+
* claim assertion time and the running/waiting tasks' heartbeat/activity
|
|
236
|
+
* marks — everything EXCEPT manifest.updatedAt, which is the frozen
|
|
237
|
+
* artifact under judgment, not evidence (the registry's own filter already
|
|
238
|
+
* bounds it at 30 min for entry presence; counting it here would make the
|
|
239
|
+
* CLAIMED_RUN_HUNG_MS ceiling unreachable). undefined when no signal
|
|
240
|
+
* carries a finite timestamp.
|
|
241
|
+
*/
|
|
242
|
+
function newestClaimEvidenceMs(tasks: TeamTaskState[] | undefined, claim: LiveRunClaim): number | undefined {
|
|
243
|
+
let newest: number | undefined;
|
|
244
|
+
const consider = (ms: number): void => {
|
|
245
|
+
if (Number.isFinite(ms) && (newest === undefined || ms > newest)) newest = ms;
|
|
246
|
+
};
|
|
247
|
+
if (claim.claimedAtMs !== undefined) consider(claim.claimedAtMs);
|
|
248
|
+
for (const task of tasks ?? []) {
|
|
249
|
+
if (task.status !== "running" && task.status !== "waiting") continue;
|
|
250
|
+
const heartbeatAt = task.heartbeat?.lastSeenAt ? new Date(task.heartbeat.lastSeenAt).getTime() : Number.NaN;
|
|
251
|
+
consider(heartbeatAt);
|
|
252
|
+
const activityAt = task.agentProgress?.lastActivityAt ? new Date(task.agentProgress.lastActivityAt).getTime() : Number.NaN;
|
|
253
|
+
consider(activityAt);
|
|
254
|
+
}
|
|
255
|
+
return newest;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Wave 13 (F1): hung-verdict liveness short-circuit result. */
|
|
259
|
+
interface LivenessShortCircuit {
|
|
260
|
+
/** True ⇒ the hung verdict is withheld this cycle (live evidence present). */
|
|
261
|
+
suppress: boolean;
|
|
262
|
+
/** Debug-log reason (PI_CREW_DEBUG-gated). */
|
|
263
|
+
reason: string;
|
|
264
|
+
/** Claim present but its evidence aged past CLAIMED_RUN_HUNG_MS — the alert detail must name it. */
|
|
265
|
+
claimExhausted: boolean;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Wave 13 (F1, GAP-0): should the hung verdict for this cycle be withheld?
|
|
270
|
+
* Two INDEPENDENT OR branches (never require both):
|
|
271
|
+
* (b) a fresh task heartbeat (≤5 min) ⇒ suppress outright;
|
|
272
|
+
* (a) a live registry claim ⇒ suppress while the newest liveness evidence
|
|
273
|
+
* is ≤ CLAIMED_RUN_HUNG_MS old; past the ceiling the ladder proceeds
|
|
274
|
+
* (claimExhausted ⇒ the alert names the claimed-run threshold).
|
|
275
|
+
* No claim and no fresh heartbeat ⇒ the existing 2-min ladder is unchanged
|
|
276
|
+
* (an unclaimed zombie still alerts fast).
|
|
277
|
+
*/
|
|
278
|
+
function assessClaimedRunLiveness(runId: string, tasks: TeamTaskState[] | undefined, now: number): LivenessShortCircuit {
|
|
279
|
+
if (hasFreshTaskHeartbeatEvidence(tasks, now)) {
|
|
280
|
+
return { suppress: true, reason: "fresh task heartbeat", claimExhausted: false };
|
|
281
|
+
}
|
|
282
|
+
const claim = liveClaimConsult(runId);
|
|
283
|
+
if (claim) {
|
|
284
|
+
const evidenceMs = newestClaimEvidenceMs(tasks, claim);
|
|
285
|
+
const evidenceAgeMs = evidenceMs !== undefined ? now - evidenceMs : undefined;
|
|
286
|
+
if (evidenceAgeMs !== undefined && evidenceAgeMs <= CLAIMED_RUN_HUNG_MS) {
|
|
287
|
+
return {
|
|
288
|
+
suppress: true,
|
|
289
|
+
reason: `live registry claim (newest evidence ${Math.max(0, Math.round(evidenceAgeMs / 1000))}s old)`,
|
|
290
|
+
claimExhausted: false,
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
return { suppress: false, reason: "claim evidence older than CLAIMED_RUN_HUNG_MS", claimExhausted: true };
|
|
294
|
+
}
|
|
295
|
+
return { suppress: false, reason: "no live claim", claimExhausted: false };
|
|
296
|
+
}
|
|
297
|
+
|
|
131
298
|
/**
|
|
132
299
|
* Pre-emit state re-verify loader. Injectable for tests to simulate the run
|
|
133
300
|
* going terminal between the cycle's initial read and the emit point.
|
|
@@ -471,39 +638,59 @@ export function startForegroundWatchdog(opts: WatchdogOptions): void {
|
|
|
471
638
|
// Keep monitoring: if the manifest eventually finalizes, the
|
|
472
639
|
// terminal branch delivers the completion notice.
|
|
473
640
|
} else {
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
641
|
+
// Wave 13 (F1, GAP-0): liveness short-circuit BEFORE any hung alert.
|
|
642
|
+
// The verdict above is structurally blind while a tool executes; a
|
|
643
|
+
// live registry claim or fresh task heartbeat withholds it. A
|
|
644
|
+
// suppressed cycle means the run left the hung state — reset the
|
|
645
|
+
// no-progress counter so a later genuine alert gets the full drip cap.
|
|
646
|
+
const liveness = assessClaimedRunLiveness(runId, reverified.tasks, now);
|
|
647
|
+
if (liveness.suppress) {
|
|
648
|
+
consecutiveHungNotices = 0;
|
|
649
|
+
debugLog(`[foreground-watchdog] suppress hung verdict for ${runId}: ${liveness.reason}`);
|
|
650
|
+
} else {
|
|
651
|
+
// Wave 13 (F3, GAP-2): the detail carries its EVIDENCE time (the
|
|
652
|
+
// manifest read the verdict was based on) — "as of <ISO>" — not the
|
|
653
|
+
// emit time, so a user re-checking can tell a stale/queued alert
|
|
654
|
+
// apart from one about the last few minutes. An exhausted claim
|
|
655
|
+
// names the claimed-run threshold so the user knows it is not the
|
|
656
|
+
// 2-min ladder.
|
|
657
|
+
const claimNote = liveness.claimExhausted
|
|
658
|
+
? `, claimed-run hung threshold exceeded (no live evidence for >${Math.round(CLAIMED_RUN_HUNG_MS / 60_000)} min)`
|
|
659
|
+
: "";
|
|
660
|
+
const detail = `status=${manifest.status}, updatedAt=${manifest.updatedAt}, agents=${agents.length}, as of ${manifest.updatedAt}${claimNote}`;
|
|
661
|
+
consecutiveHungNotices += 1;
|
|
662
|
+
try {
|
|
663
|
+
if (consecutiveHungNotices <= HUNG_NOTICE_CAP) {
|
|
664
|
+
// WD-DEDUP: alert-once per runId at module scope — even a
|
|
665
|
+
// fresh watchdog instance must not re-alert the same run.
|
|
666
|
+
if (!hungAlertAlreadySent(runId)) {
|
|
667
|
+
// Mark BEFORE queueing so a racing cycle cannot double-queue.
|
|
668
|
+
updateHungAlertRecord(runId, { alerted: true });
|
|
669
|
+
// WD-DELIVER-GUARD: the host queues a followUp while the
|
|
670
|
+
// session streams and delivers it arbitrarily later — so
|
|
671
|
+
// hand it over only at the idle boundary, re-verifying the
|
|
672
|
+
// run state at THAT moment.
|
|
673
|
+
queueHungAlertDelivery(
|
|
674
|
+
pi,
|
|
675
|
+
cwd,
|
|
676
|
+
runId,
|
|
677
|
+
`pi-crew watchdog: run ${runId} appears hung (${detail}). Consider running team action='cancel' runId='${runId}' or team action='doctor'.`,
|
|
678
|
+
opts.isSessionIdle,
|
|
679
|
+
);
|
|
680
|
+
}
|
|
681
|
+
} else if (consecutiveHungNotices === HUNG_NOTICE_CAP + 1) {
|
|
682
|
+
// One-time handoff, then silence — still monitoring, no more drips.
|
|
487
683
|
queueHungAlertDelivery(
|
|
488
684
|
pi,
|
|
489
685
|
cwd,
|
|
490
686
|
runId,
|
|
491
|
-
`pi-crew watchdog: run ${runId}
|
|
687
|
+
`pi-crew watchdog: run ${runId} is still hung after ${consecutiveHungNotices} checks — going quiet now. Intervene via team action='cancel' runId='${runId}', team action='doctor', or leave it; a completion notice will still fire.`,
|
|
492
688
|
opts.isSessionIdle,
|
|
493
689
|
);
|
|
494
690
|
}
|
|
495
|
-
}
|
|
496
|
-
|
|
497
|
-
queueHungAlertDelivery(
|
|
498
|
-
pi,
|
|
499
|
-
cwd,
|
|
500
|
-
runId,
|
|
501
|
-
`pi-crew watchdog: run ${runId} is still hung after ${consecutiveHungNotices} checks — going quiet now. Intervene via team action='cancel' runId='${runId}', team action='doctor', or leave it; a completion notice will still fire.`,
|
|
502
|
-
opts.isSessionIdle,
|
|
503
|
-
);
|
|
691
|
+
} catch {
|
|
692
|
+
/* non-critical */
|
|
504
693
|
}
|
|
505
|
-
} catch {
|
|
506
|
-
/* non-critical */
|
|
507
694
|
}
|
|
508
695
|
}
|
|
509
696
|
// Keep monitoring (ARCH-5: silently past the cap — no more notices).
|
|
@@ -5,6 +5,7 @@ import { appendEventFireAndForget } from "../../state/event-log/event-log.ts";
|
|
|
5
5
|
import { writeArtifact } from "../../state/stores/artifact-store.ts";
|
|
6
6
|
import { loadRunManifestById } from "../../state/stores/state-store.ts";
|
|
7
7
|
import type { ArtifactDescriptor, TeamRunManifest, TeamTaskState } from "../../state/types.ts";
|
|
8
|
+
import { logInternalError } from "../../utils/internal-error.ts";
|
|
8
9
|
import type { WorkflowStep } from "../../workflows/workflow-config.ts";
|
|
9
10
|
import {
|
|
10
11
|
appendCrewAgentEvent,
|
|
@@ -104,6 +105,79 @@ function updateTask(tasks: TeamTaskState[], updated: TeamTaskState): TeamTaskSta
|
|
|
104
105
|
return tasks.map((task) => (task.id === updated.id ? updated : task));
|
|
105
106
|
}
|
|
106
107
|
|
|
108
|
+
/**
|
|
109
|
+
* Wave 13 (F2, GAP-1): default cadence (ms) of the live-executor tool
|
|
110
|
+
* heartbeat timer. 30s sits far inside every liveness window that consumes
|
|
111
|
+
* the task heartbeat (watchdog task-heartbeat evidence 5 min, stale-reconciler
|
|
112
|
+
* ACTIVE_EVIDENCE_TTL_MS 5 min, CLAIMED_RUN_HUNG_MS 45 min), costs at most one
|
|
113
|
+
* throttled tasks.json persist per running tool per 30s, and never touches
|
|
114
|
+
* the event log. Fixed constant, NOT env-configurable on purpose (SPEC §3.2):
|
|
115
|
+
* wiring an env kill-switch would require registering
|
|
116
|
+
* PI_CREW_LIVE_HEARTBEAT_MS in src/config/env-vars.ts AND a forwarding
|
|
117
|
+
* decision in src/config/env-allowlist-registry.ts (fail-fast at module
|
|
118
|
+
* load) — both outside this wave's file scope.
|
|
119
|
+
*/
|
|
120
|
+
export const LIVE_TOOL_HEARTBEAT_INTERVAL_MS = 30_000;
|
|
121
|
+
|
|
122
|
+
/** Opaque handle for the per-attempt live tool heartbeat timer. */
|
|
123
|
+
export interface LiveToolHeartbeatTimer {
|
|
124
|
+
/** Stop the timer and clear its interval. Idempotent. */
|
|
125
|
+
stop(): void;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Wave 13 (F2, GAP-1) — start the tool heartbeat timer for one live attempt
|
|
130
|
+
* (SPEC §3.2). Every `intervalMs`, IF a tool is currently executing
|
|
131
|
+
* (`isToolInFlight()` — `agentProgress.currentToolStartedAt` set by
|
|
132
|
+
* tool_execution_start, cleared by tool_execution_end), call `touch()`
|
|
133
|
+
* (the heartbeat-only persist, shared 1s throttle with event-driven persists).
|
|
134
|
+
* Idle between tools ⇒ no write. `touch` must NOT append task.progress events
|
|
135
|
+
* so the event log cannot grow from heartbeats.
|
|
136
|
+
*
|
|
137
|
+
* Failure containment (mirrors startWorkerHeartbeatPulse [pulse-3]): a
|
|
138
|
+
* throwing gate/touch is logged and the timer keeps ticking — the heartbeat
|
|
139
|
+
* must never crash the host process. The interval is `unref`'d so it can
|
|
140
|
+
* never hold the event loop open.
|
|
141
|
+
*/
|
|
142
|
+
export function startLiveToolHeartbeatTimer(opts: {
|
|
143
|
+
intervalMs: number;
|
|
144
|
+
isToolInFlight: () => boolean;
|
|
145
|
+
touch: () => void;
|
|
146
|
+
}): LiveToolHeartbeatTimer {
|
|
147
|
+
// Kill-switch shape (interval ≤ 0 ⇒ disabled, no timer) kept so a future
|
|
148
|
+
// env-configurable interval can slot in without touching call sites.
|
|
149
|
+
if (opts.intervalMs <= 0) {
|
|
150
|
+
return {
|
|
151
|
+
stop() {
|
|
152
|
+
// Intentional no-op: a disabled timer (interval ≤ 0) has no handle.
|
|
153
|
+
},
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
const timer = setInterval(() => {
|
|
157
|
+
try {
|
|
158
|
+
if (opts.isToolInFlight()) opts.touch();
|
|
159
|
+
} catch (err) {
|
|
160
|
+
logInternalError("task-runner.live-tool-heartbeat", err as Error);
|
|
161
|
+
}
|
|
162
|
+
}, opts.intervalMs);
|
|
163
|
+
timer.unref?.();
|
|
164
|
+
return {
|
|
165
|
+
stop() {
|
|
166
|
+
clearInterval(timer);
|
|
167
|
+
},
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
let armLiveToolHeartbeatTimer: typeof startLiveToolHeartbeatTimer = startLiveToolHeartbeatTimer;
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* @internal Test seam — replace the timer constructor used inside runLiveTask
|
|
175
|
+
* (null = real). Mirrors the loader seams in foreground-watchdog.ts.
|
|
176
|
+
*/
|
|
177
|
+
export function setLiveToolHeartbeatTimerForTests(starter: typeof startLiveToolHeartbeatTimer | null): void {
|
|
178
|
+
armLiveToolHeartbeatTimer = starter ?? startLiveToolHeartbeatTimer;
|
|
179
|
+
}
|
|
180
|
+
|
|
107
181
|
export async function runLiveTask(input: RunLiveTaskInput): Promise<RunLiveTaskOutput> {
|
|
108
182
|
const { manifest, step, agent, prompt } = input;
|
|
109
183
|
let task = input.task;
|
|
@@ -122,6 +196,24 @@ export async function runLiveTask(input: RunLiveTaskInput): Promise<RunLiveTaskO
|
|
|
122
196
|
let lastRunProgressPersistedAt = 0;
|
|
123
197
|
let lastHeartbeatPersistedAt = 0;
|
|
124
198
|
let lastRunProgressSummary: ProgressEventSummary | undefined;
|
|
199
|
+
/**
|
|
200
|
+
* Wave 13 (F2, GAP-1): heartbeat-only persist, extracted from
|
|
201
|
+
* persistLiveProgress so the tool heartbeat timer can reuse the EXACT
|
|
202
|
+
* event-driven path (same 1s throttle via the shared lastHeartbeatPersistedAt,
|
|
203
|
+
* same persistSingleTaskUpdate write, same isCurrent gate). Touching only
|
|
204
|
+
* tasks.json — never the event log — so timer ticks cannot inflate it.
|
|
205
|
+
*/
|
|
206
|
+
const touchTaskHeartbeat = (force = false): void => {
|
|
207
|
+
if (!isCurrent()) return;
|
|
208
|
+
const now = Date.now();
|
|
209
|
+
if (!force && now - lastHeartbeatPersistedAt < 1000) return;
|
|
210
|
+
task = {
|
|
211
|
+
...task,
|
|
212
|
+
heartbeat: touchWorkerHeartbeat(task.heartbeat ?? createWorkerHeartbeat(task.id)),
|
|
213
|
+
};
|
|
214
|
+
tasks = persistSingleTaskUpdate(manifest, tasks, task);
|
|
215
|
+
lastHeartbeatPersistedAt = now;
|
|
216
|
+
};
|
|
125
217
|
const persistLiveProgress = (event: unknown, force = false): void => {
|
|
126
218
|
if (!isCurrent()) return;
|
|
127
219
|
const now = Date.now();
|
|
@@ -130,14 +222,7 @@ export async function runLiveTask(input: RunLiveTaskInput): Promise<RunLiveTaskO
|
|
|
130
222
|
lastAgentRecordPersistedAt = now;
|
|
131
223
|
}
|
|
132
224
|
// Bug #5 fix: also persist heartbeat to tasks.json so heartbeat-watcher can detect live-session activity
|
|
133
|
-
|
|
134
|
-
task = {
|
|
135
|
-
...task,
|
|
136
|
-
heartbeat: touchWorkerHeartbeat(task.heartbeat ?? createWorkerHeartbeat(task.id)),
|
|
137
|
-
};
|
|
138
|
-
tasks = persistSingleTaskUpdate(manifest, tasks, task);
|
|
139
|
-
lastHeartbeatPersistedAt = now;
|
|
140
|
-
}
|
|
225
|
+
touchTaskHeartbeat(force);
|
|
141
226
|
const summary = progressEventSummary(task, event);
|
|
142
227
|
const decision = shouldAppendProgressEventUpdate({
|
|
143
228
|
previous: lastRunProgressSummary,
|
|
@@ -166,44 +251,60 @@ export async function runLiveTask(input: RunLiveTaskInput): Promise<RunLiveTaskO
|
|
|
166
251
|
agentMax != null && agentMax > 0 && (!input.runtimeConfig?.maxTurns || agentMax < input.runtimeConfig.maxTurns)
|
|
167
252
|
? { ...input.runtimeConfig, maxTurns: agentMax }
|
|
168
253
|
: input.runtimeConfig;
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
parentModel: input.parentModel,
|
|
180
|
-
modelRegistry: input.modelRegistry,
|
|
181
|
-
modelOverride: input.modelOverride,
|
|
182
|
-
teamRoleModel: input.teamRoleModel,
|
|
183
|
-
teamRoleFallbackModels: input.teamRoleFallbackModels,
|
|
184
|
-
teamRoleThinking: input.teamRoleThinking,
|
|
185
|
-
isCurrent,
|
|
186
|
-
workspaceId: input.workspaceId,
|
|
187
|
-
// U11: embedded-tier mitigation knobs (lane-B contract) — forwarded
|
|
188
|
-
// verbatim; undefined keeps every mitigation inert (legacy path).
|
|
189
|
-
embeddedMitigations: input.embeddedMitigations,
|
|
190
|
-
// Phase 2: Pass output schema for yield validation
|
|
191
|
-
outputSchema: undefined,
|
|
192
|
-
onOutput: (text) => {
|
|
193
|
-
if (!isCurrent()) return;
|
|
194
|
-
appendCrewAgentOutput(manifest, task.id, text);
|
|
195
|
-
},
|
|
196
|
-
onEvent: (event) => {
|
|
197
|
-
if (!isCurrent()) return;
|
|
198
|
-
appendCrewAgentEvent(manifest, task.id, event);
|
|
199
|
-
task = {
|
|
200
|
-
...task,
|
|
201
|
-
agentProgress: applyAgentProgressEvent(task.agentProgress ?? emptyCrewAgentProgress(), event, task.startedAt),
|
|
202
|
-
};
|
|
203
|
-
tasks = updateTask(tasks, task);
|
|
204
|
-
persistLiveProgress(event);
|
|
205
|
-
},
|
|
254
|
+
// Wave 13 (F2, GAP-1): arm the tool heartbeat timer for this attempt. The
|
|
255
|
+
// tick no-ops unless a tool is executing (agentProgress.currentToolStartedAt
|
|
256
|
+
// set by tool_execution_start / cleared by tool_execution_end), so idle gaps
|
|
257
|
+
// between tools write nothing; the shared 1s throttle inside
|
|
258
|
+
// touchTaskHeartbeat bounds disk writes; and the finally below clears the
|
|
259
|
+
// interval on completion, throw AND cancel — no timer leak across attempts.
|
|
260
|
+
const heartbeatTimer = armLiveToolHeartbeatTimer({
|
|
261
|
+
intervalMs: LIVE_TOOL_HEARTBEAT_INTERVAL_MS,
|
|
262
|
+
isToolInFlight: () => task.agentProgress?.currentToolStartedAt !== undefined,
|
|
263
|
+
touch: () => touchTaskHeartbeat(false),
|
|
206
264
|
});
|
|
265
|
+
let liveResult: Awaited<ReturnType<typeof runLiveSessionTask>>;
|
|
266
|
+
try {
|
|
267
|
+
liveResult = await runLiveSessionTask({
|
|
268
|
+
manifest,
|
|
269
|
+
task,
|
|
270
|
+
step,
|
|
271
|
+
agent,
|
|
272
|
+
prompt,
|
|
273
|
+
signal: input.signal,
|
|
274
|
+
transcriptPath,
|
|
275
|
+
runtimeConfig: effectiveRuntimeConfig,
|
|
276
|
+
parentContext: input.parentContext,
|
|
277
|
+
parentModel: input.parentModel,
|
|
278
|
+
modelRegistry: input.modelRegistry,
|
|
279
|
+
modelOverride: input.modelOverride,
|
|
280
|
+
teamRoleModel: input.teamRoleModel,
|
|
281
|
+
teamRoleFallbackModels: input.teamRoleFallbackModels,
|
|
282
|
+
teamRoleThinking: input.teamRoleThinking,
|
|
283
|
+
isCurrent,
|
|
284
|
+
workspaceId: input.workspaceId,
|
|
285
|
+
// U11: embedded-tier mitigation knobs (lane-B contract) — forwarded
|
|
286
|
+
// verbatim; undefined keeps every mitigation inert (legacy path).
|
|
287
|
+
embeddedMitigations: input.embeddedMitigations,
|
|
288
|
+
// Phase 2: Pass output schema for yield validation
|
|
289
|
+
outputSchema: undefined,
|
|
290
|
+
onOutput: (text) => {
|
|
291
|
+
if (!isCurrent()) return;
|
|
292
|
+
appendCrewAgentOutput(manifest, task.id, text);
|
|
293
|
+
},
|
|
294
|
+
onEvent: (event) => {
|
|
295
|
+
if (!isCurrent()) return;
|
|
296
|
+
appendCrewAgentEvent(manifest, task.id, event);
|
|
297
|
+
task = {
|
|
298
|
+
...task,
|
|
299
|
+
agentProgress: applyAgentProgressEvent(task.agentProgress ?? emptyCrewAgentProgress(), event, task.startedAt),
|
|
300
|
+
};
|
|
301
|
+
tasks = updateTask(tasks, task);
|
|
302
|
+
persistLiveProgress(event);
|
|
303
|
+
},
|
|
304
|
+
});
|
|
305
|
+
} finally {
|
|
306
|
+
heartbeatTimer.stop();
|
|
307
|
+
}
|
|
207
308
|
const startupEvidence = createStartupEvidence({
|
|
208
309
|
command: "live-session",
|
|
209
310
|
startedAt: attemptStartedAt,
|