pi-crew 0.11.51 → 0.11.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/dist/index.mjs +488 -488
- package/package.json +1 -1
- package/src/extension/team-tool/cancel.ts +15 -1
- package/src/runtime/background-runner.ts +101 -81
- package/src/runtime/foreground-control.ts +95 -1
- package/src/runtime/heartbeat/heartbeat-watcher.ts +32 -11
- package/src/runtime/task-runner/child-executor.ts +23 -2
- package/src/runtime/task-runner/live-executor.ts +3 -0
- package/src/runtime/task-runner/post-execution.ts +5 -0
package/package.json
CHANGED
|
@@ -209,7 +209,21 @@ export async function handleRetry(params: TeamToolParamsValue, ctx: TeamContext,
|
|
|
209
209
|
const retriedIds = new Set(matchingTasks.map((t) => t.id));
|
|
210
210
|
const tasks = fresh.tasks.map((task) => {
|
|
211
211
|
if (!retriedIds.has(task.id)) return task;
|
|
212
|
-
|
|
212
|
+
// FIX-2 (live catch, run team_20261030044630): also strip the stale
|
|
213
|
+
// liveness fields — heartbeat-watcher treats evidence older than the
|
|
214
|
+
// NEXT attempt's startedAt as inadmissible, but the persisted fields
|
|
215
|
+
// also feed OTHER consumers (UI gradients, reconciler). Attempt-2
|
|
216
|
+
// must not inherit attempt-1's heartbeat/agentProgress timestamps.
|
|
217
|
+
// (checkpoint.childPid has its own per-spawn reset semantics in
|
|
218
|
+
// state-helpers.ts — deliberately untouched here.)
|
|
219
|
+
const {
|
|
220
|
+
error: _error,
|
|
221
|
+
finishedAt: _finishedAt,
|
|
222
|
+
terminalEvidence: _terminalEvidence,
|
|
223
|
+
heartbeat: _heartbeat,
|
|
224
|
+
agentProgress: _agentProgress,
|
|
225
|
+
...rest
|
|
226
|
+
} = task;
|
|
213
227
|
return { ...rest, status: "queued" as const };
|
|
214
228
|
});
|
|
215
229
|
saveRunTasks(fresh.manifest, tasks);
|
|
@@ -48,6 +48,7 @@ import { writeAsyncStartMarker } from "./async-marker.ts";
|
|
|
48
48
|
import { parseStdinBrokerPayload, readStdinFirstLine } from "./broker/stdin-handshake.ts";
|
|
49
49
|
import { terminateActiveChildPiProcesses } from "./child-pi/child-pi.ts";
|
|
50
50
|
import { directTeamAndWorkflowFromRun } from "./direct-run.ts";
|
|
51
|
+
import { startRunInterruptGuard } from "./foreground-control.ts";
|
|
51
52
|
import { resolveCrewRuntime, runtimeResolutionState } from "./model/runtime-resolver.ts";
|
|
52
53
|
import { registryFromModelContext } from "./model/session-model.ts";
|
|
53
54
|
import { unregisterWorker } from "./orphan-worker-registry.ts";
|
|
@@ -227,86 +228,47 @@ export function installBackgroundRunnerSigintHandler(abortController: AbortContr
|
|
|
227
228
|
});
|
|
228
229
|
}
|
|
229
230
|
|
|
231
|
+
/**
|
|
232
|
+
* Process-wide (detached-runner) variant of the interrupt guard. Thin wrapper
|
|
233
|
+
* over the extracted core in ./foreground-control.ts (FIX-1, live catch run
|
|
234
|
+
* team_20261030044630): the polling/ack/event semantics live there; THIS
|
|
235
|
+
* wrapper owns the process-wide consequences only the detached runner may
|
|
236
|
+
* take (kill ALL child-pi processes, stop the parent guard, exit 130). The
|
|
237
|
+
* INLINE core (executeBackgroundRun) uses startRunInterruptGuard directly
|
|
238
|
+
* with a run-scoped onInterrupt instead — it must NEVER take these actions
|
|
239
|
+
* inside a host process that may own sibling runs.
|
|
240
|
+
*
|
|
241
|
+
* Kept exported from background-runner.ts so existing imports/tests
|
|
242
|
+
* (interrupt-guard-ack.test.ts) keep working unchanged.
|
|
243
|
+
*/
|
|
230
244
|
export function startInterruptGuard(
|
|
231
245
|
manifest: { runId: string; stateRoot: string; eventsPath: string },
|
|
232
246
|
abortController: AbortController,
|
|
233
247
|
stopParentGuard: () => void,
|
|
234
248
|
): () => void {
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
const last = parsed.requests?.at(-1);
|
|
259
|
-
if (last?.type === "interrupt" && last?.acknowledged !== true) {
|
|
260
|
-
// RT-4 FIX: Gate ensures the interrupt body runs only once even if
|
|
261
|
-
// the synchronous ack write below fails.
|
|
262
|
-
if (interruptHandled) return;
|
|
263
|
-
interruptHandled = true;
|
|
264
|
-
|
|
265
|
-
// RT-4 FIX: Write acknowledged:true back to foreground-control.json
|
|
266
|
-
// SYNCHRONOUSLY. This stops the guard from re-firing on the next tick
|
|
267
|
-
// (interruptGuardInterval). Must be sync because this is a setInterval
|
|
268
|
-
// polling callback — we cannot await in a polling callback.
|
|
269
|
-
try {
|
|
270
|
-
const reqs = parsed.requests ?? [];
|
|
271
|
-
if (reqs.length > 0) {
|
|
272
|
-
reqs[reqs.length - 1].acknowledged = true;
|
|
273
|
-
atomicWriteFile(controlPath, JSON.stringify(parsed, null, 2));
|
|
274
|
-
}
|
|
275
|
-
} catch {
|
|
276
|
-
/* best-effort ack — interruptHandled gate prevents re-fire */
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
appendEventBuffered(manifest.eventsPath, {
|
|
280
|
-
type: "async.interrupt_detected",
|
|
281
|
-
runId: manifest.runId,
|
|
282
|
-
message: "Background runner detected foreground interrupt — killing child processes and exiting.",
|
|
283
|
-
}).catch((e) => logInternalError("background-runner.buffered", e, "type=async.interrupt_detected"));
|
|
284
|
-
// FIX: Terminate ALL child-pi processes IMMEDIATELY before exiting.
|
|
285
|
-
// Previously this was missing, causing orphaned child processes to run forever
|
|
286
|
-
// after the background-runner exited. terminateActiveChildPiProcesses sends
|
|
287
|
-
// SIGTERM then SIGKILL (after HARD_KILL_MS=3s) to every active child.
|
|
288
|
-
const killed = terminateActiveChildPiProcesses();
|
|
289
|
-
console.log(`[background-runner] interrupt: killed ${killed} child processes`);
|
|
290
|
-
// Also abort the run signal so executeTeamRun exits quickly via its signal check.
|
|
291
|
-
abortController.abort();
|
|
292
|
-
// Call stopParentGuard() explicitly in addition to the finally block in
|
|
293
|
-
// main() — defense-in-depth ensures the guard is torn down promptly.
|
|
294
|
-
stopParentGuard();
|
|
295
|
-
// CORE-7 fix: Do NOT call process.exit() — it skips the finally block in
|
|
296
|
-
// main() (which runs unregisterWorker + stopParentGuard), leaving a stale
|
|
297
|
-
// orphan registry entry. Setting exitCode lets the event loop drain and
|
|
298
|
-
// main()'s finally block run cleanup before the process exits naturally
|
|
299
|
-
// with code 130.
|
|
300
|
-
process.exitCode = 130;
|
|
301
|
-
} else if (last) {
|
|
302
|
-
console.warn(`[background-runner] Ignoring unknown foreground control request: ${last.type}`);
|
|
303
|
-
}
|
|
304
|
-
} catch {
|
|
305
|
-
/* ignore read/parse errors */
|
|
306
|
-
}
|
|
307
|
-
}, interruptGuardInterval);
|
|
308
|
-
interval.unref();
|
|
309
|
-
return () => clearInterval(interval);
|
|
249
|
+
return startRunInterruptGuard({
|
|
250
|
+
manifest,
|
|
251
|
+
eventMessage: "Background runner detected foreground interrupt — killing child processes and exiting.",
|
|
252
|
+
onInterrupt: () => {
|
|
253
|
+
// FIX: Terminate ALL child-pi processes IMMEDIATELY before exiting.
|
|
254
|
+
// Previously this was missing, causing orphaned child processes to run forever
|
|
255
|
+
// after the background-runner exited. terminateActiveChildPiProcesses sends
|
|
256
|
+
// SIGTERM then SIGKILL (after HARD_KILL_MS=3s) to every active child.
|
|
257
|
+
const killed = terminateActiveChildPiProcesses();
|
|
258
|
+
console.log(`[background-runner] interrupt: killed ${killed} child processes`);
|
|
259
|
+
// Also abort the run signal so executeTeamRun exits quickly via its signal check.
|
|
260
|
+
abortController.abort();
|
|
261
|
+
// Call stopParentGuard() explicitly in addition to the finally block in
|
|
262
|
+
// main() — defense-in-depth ensures the guard is torn down promptly.
|
|
263
|
+
stopParentGuard();
|
|
264
|
+
// CORE-7 fix: Do NOT call process.exit() — it skips the finally block in
|
|
265
|
+
// main() (which runs unregisterWorker + stopParentGuard), leaving a stale
|
|
266
|
+
// orphan registry entry. Setting exitCode lets the event loop drain and
|
|
267
|
+
// main()'s finally block run cleanup before the process exits naturally
|
|
268
|
+
// with code 130.
|
|
269
|
+
process.exitCode = 130;
|
|
270
|
+
},
|
|
271
|
+
});
|
|
310
272
|
}
|
|
311
273
|
|
|
312
274
|
/**
|
|
@@ -421,7 +383,15 @@ function runCleanup(
|
|
|
421
383
|
let exitDueToRejection = false;
|
|
422
384
|
|
|
423
385
|
export interface ExecuteBackgroundRunOptions {
|
|
424
|
-
/**
|
|
386
|
+
/**
|
|
387
|
+
* Abort signal — the detached runner passes its controller's signal
|
|
388
|
+
* (SIGINT/SIGTERM/watchdog); the inline test seam passes none.
|
|
389
|
+
*
|
|
390
|
+
* FIX-1 guard ownership: passing a signal means the HOST owns interrupt
|
|
391
|
+
* consumption (detached main() mounts the process-wide guard). Passing
|
|
392
|
+
* NONE makes executeBackgroundRun mount its own run-scoped guard and
|
|
393
|
+
* abort THIS run on a foreground-control interrupt (see wrapper doc).
|
|
394
|
+
*/
|
|
425
395
|
signal?: AbortSignal;
|
|
426
396
|
/**
|
|
427
397
|
* Set PI_CREW_BACKGROUND_MODE=1 (detached-runner observability marker).
|
|
@@ -449,19 +419,65 @@ export interface ExecuteBackgroundRunOptions {
|
|
|
449
419
|
* first-spawn stall (no second process to come up inside a test deadline)
|
|
450
420
|
* and the orphan-tmpdir leak (the run dies with the test process — nothing
|
|
451
421
|
* detached survives to write state after the test's rmSync).
|
|
422
|
+
*
|
|
423
|
+
* FIX-1 (live catch, run team_20261030044630_3935473bc4886682): INTERRUPT
|
|
424
|
+
* GUARD OWNERSHIP. When the host passes opts.signal (the detached runner
|
|
425
|
+
* does — it mounts the process-wide guard at :959 and forwards its
|
|
426
|
+
* controller's signal), the host owns interrupt consumption and this core
|
|
427
|
+
* mounts NOTHING (no double guard on one run). When NO signal is passed
|
|
428
|
+
* (inline execution in the owner session process), THIS core owns it: it
|
|
429
|
+
* mounts a run-scoped guard (startRunInterruptGuard) that acks the request,
|
|
430
|
+
* appends async.interrupt_detected and aborts the run's own controller — and
|
|
431
|
+
* NEVER sweeps the process (terminateActiveChildPiProcesses would murder
|
|
432
|
+
* sibling runs' children in the same process; the abort's own cancellation
|
|
433
|
+
* ladder SIGTERMs this run's children scoped). The guard is stopped in the
|
|
434
|
+
* finally — no leaked interval, whatever the outcome.
|
|
452
435
|
*/
|
|
453
436
|
export async function executeBackgroundRun(
|
|
454
437
|
input: { manifest: TeamRunManifest; tasks: TeamTaskState[] },
|
|
455
438
|
opts: ExecuteBackgroundRunOptions = {},
|
|
439
|
+
): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
|
|
440
|
+
let signal: AbortSignal;
|
|
441
|
+
let stopRunInterruptGuard: (() => void) | undefined;
|
|
442
|
+
if (opts.signal) {
|
|
443
|
+
// Host-provided signal ⇒ host owns the interrupt guard (detached
|
|
444
|
+
// main() mounts the process-wide variant). Pass through unchanged.
|
|
445
|
+
signal = opts.signal;
|
|
446
|
+
} else {
|
|
447
|
+
// Inline execution ⇒ this core owns interrupt consumption for its run.
|
|
448
|
+
// Retain the controller so the guard can abort THIS run only.
|
|
449
|
+
const runController = new AbortController();
|
|
450
|
+
signal = runController.signal;
|
|
451
|
+
stopRunInterruptGuard = startRunInterruptGuard({
|
|
452
|
+
manifest: input.manifest,
|
|
453
|
+
eventMessage: `Inline background run detected foreground interrupt — aborting run ${input.manifest.runId}.`,
|
|
454
|
+
onInterrupt: () => {
|
|
455
|
+
// INLINE SAFETY: run-scoped abort ONLY. No
|
|
456
|
+
// terminateActiveChildPiProcesses() (kills EVERY child-pi in the
|
|
457
|
+
// process — sibling runs in the same owner session would be
|
|
458
|
+
// murdered), no stopParentGuard (host owns it), no exitCode
|
|
459
|
+
// (this is not the process's decision). The run's cancellation
|
|
460
|
+
// ladder SIGTERMs this run's children scoped to the controller.
|
|
461
|
+
runController.abort();
|
|
462
|
+
},
|
|
463
|
+
});
|
|
464
|
+
}
|
|
465
|
+
try {
|
|
466
|
+
return await executeBackgroundRunCore(input, opts, signal);
|
|
467
|
+
} finally {
|
|
468
|
+
stopRunInterruptGuard?.();
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
async function executeBackgroundRunCore(
|
|
473
|
+
input: { manifest: TeamRunManifest; tasks: TeamTaskState[] },
|
|
474
|
+
opts: ExecuteBackgroundRunOptions,
|
|
475
|
+
signal: AbortSignal,
|
|
456
476
|
): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
|
|
457
477
|
const log = opts.log ?? ((message: string) => console.log(message));
|
|
458
478
|
let manifest = input.manifest;
|
|
459
479
|
let tasks = input.tasks;
|
|
460
480
|
const cwd = manifest.cwd;
|
|
461
|
-
// Inline seam passes no signal — a never-aborted dummy is behaviorally
|
|
462
|
-
// identical ("no cancellation source") for the goal-loop/dwf/team-run
|
|
463
|
-
// inputs below that require a non-optional AbortSignal.
|
|
464
|
-
const signal: AbortSignal = opts.signal ?? new AbortController().signal;
|
|
465
481
|
debugLog(`[background-runner] about to call discoverAgents`);
|
|
466
482
|
const agents = allAgents(discoverAgents(cwd));
|
|
467
483
|
debugLog(`[background-runner] discoverAgents done, ${agents.length} agents`);
|
|
@@ -641,7 +657,11 @@ export async function executeBackgroundRun(
|
|
|
641
657
|
...restoredModelRouting(manifest),
|
|
642
658
|
reliability: runConfig.reliability,
|
|
643
659
|
workspaceId: manifest.ownerSessionId ?? manifest.cwd,
|
|
644
|
-
|
|
660
|
+
// FIX-1: pass the RESOLVED run signal (host's, or the inline
|
|
661
|
+
// guard-owned controller). Was `signal: opts.signal` — the inline
|
|
662
|
+
// host got NO cancellation source, so a consumed interrupt could
|
|
663
|
+
// never abort the executing team run.
|
|
664
|
+
signal,
|
|
645
665
|
...(manifest.budgetTotal !== undefined ? { budgetTotal: manifest.budgetTotal } : {}),
|
|
646
666
|
...(manifest.budgetWarning !== undefined ? { budgetWarning: manifest.budgetWarning } : {}),
|
|
647
667
|
...(manifest.budgetAbort !== undefined ? { budgetAbort: manifest.budgetAbort } : {}),
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as path from "node:path";
|
|
3
|
+
import { getCrewEnv } from "../config/env-vars.ts";
|
|
3
4
|
import { atomicWriteFile } from "../state/atomic-write.ts";
|
|
4
|
-
import { appendEvent } from "../state/event-log/event-log.ts";
|
|
5
|
+
import { appendEvent, appendEventBuffered } from "../state/event-log/event-log.ts";
|
|
5
6
|
import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
|
|
6
7
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
7
8
|
import { sleepSync } from "../utils/sleep.ts";
|
|
@@ -204,3 +205,96 @@ export function writeForegroundInterruptRequest(
|
|
|
204
205
|
releaseLock();
|
|
205
206
|
}
|
|
206
207
|
}
|
|
208
|
+
|
|
209
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
210
|
+
// FIX-1 (live catch, run team_20261030044630_3935473bc4886682): the
|
|
211
|
+
// foreground-control interrupt CONSUMER. The guard used to live ONLY inside
|
|
212
|
+
// background-runner's detached main(); runs executed INLINE in the owner
|
|
213
|
+
// session process (executeBackgroundRun core — also the PI_CREW_TEST_ASYNC_INLINE
|
|
214
|
+
// seam) never polled foreground-control.json, so cross-session cancels sat
|
|
215
|
+
// acknowledged:false forever (two requests in that run) while the worker ran
|
|
216
|
+
// ≥75s post-cancel with fresh heartbeats. Colocated with the PRODUCER
|
|
217
|
+
// (writeForegroundInterruptRequest above) so the control-file contract has
|
|
218
|
+
// one home.
|
|
219
|
+
//
|
|
220
|
+
// Polling/ack semantics carried over byte-identically from the previous
|
|
221
|
+
// inline guard in background-runner.ts: existsSync+readFileSync+JSON.Parse
|
|
222
|
+
// per tick, interval PI_CREW_INTERRUPT_GUARD_INTERVAL_MS (default 1000ms),
|
|
223
|
+
// synchronous atomic ack write, once-per-request gate (defense-in-depth when
|
|
224
|
+
// the ack write fails), appendEventBuffered("async.interrupt_detected"),
|
|
225
|
+
// interval.unref().
|
|
226
|
+
//
|
|
227
|
+
// What an interrupt DOES is host-scoped via onInterrupt:
|
|
228
|
+
// - detached runner (background-runner.startInterruptGuard): process-wide
|
|
229
|
+
// sweep — terminateActiveChildPiProcesses + stopParentGuard + exitCode 130.
|
|
230
|
+
// - inline core (executeBackgroundRun with no host signal): abort ONLY this
|
|
231
|
+
// run's controller. NEVER sweep: the owner session process may host
|
|
232
|
+
// sibling runs whose child-pi processes would be murdered by a global
|
|
233
|
+
// terminateActiveChildPiProcesses() call.
|
|
234
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
235
|
+
|
|
236
|
+
export interface RunInterruptGuardOptions {
|
|
237
|
+
manifest: { runId: string; stateRoot: string; eventsPath: string };
|
|
238
|
+
/**
|
|
239
|
+
* Host-scoped interrupt action. Runs ONCE per unacknowledged request,
|
|
240
|
+
* after the synchronous ack write. Keep it synchronous: this is a
|
|
241
|
+
* setInterval polling callback — it cannot await.
|
|
242
|
+
*/
|
|
243
|
+
onInterrupt: () => void;
|
|
244
|
+
/** Message for the async.interrupt_detected event (hosts describe their own follow-up). */
|
|
245
|
+
eventMessage?: string;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
export function startRunInterruptGuard(opts: RunInterruptGuardOptions): () => void {
|
|
249
|
+
const controlPath = path.join(opts.manifest.stateRoot, "foreground-control.json");
|
|
250
|
+
// PERF (2026-08-24, carried over verbatim): default 1000ms. Each tick does
|
|
251
|
+
// existsSync + readFileSync + JSON.parse for the whole run, and the guard
|
|
252
|
+
// is best-effort interrupt LATENCY, not a correctness deadline — a late
|
|
253
|
+
// trigger merely delays the SIGTERM (children are SIGKILLed after
|
|
254
|
+
// HARD_KILL_MS=3s regardless) and no state machine aborts on a deadline
|
|
255
|
+
// when the guard fires late. Nothing downstream requires a tighter cadence.
|
|
256
|
+
const interruptGuardInterval = Number(getCrewEnv("PI_CREW_INTERRUPT_GUARD_INTERVAL_MS")) || 1000;
|
|
257
|
+
// RT-4 FIX (carried over verbatim): local gate so the interrupt body runs
|
|
258
|
+
// only once per request. The ack write stops the re-fire; this gate is
|
|
259
|
+
// defense-in-depth if the ack write fails (e.g. transient fs error).
|
|
260
|
+
let interruptHandled = false;
|
|
261
|
+
const interval = setInterval(() => {
|
|
262
|
+
try {
|
|
263
|
+
if (!fs.existsSync(controlPath)) return;
|
|
264
|
+
const parsed = JSON.parse(fs.readFileSync(controlPath, "utf-8")) as {
|
|
265
|
+
requests?: Array<{ type: string; acknowledged?: boolean }>;
|
|
266
|
+
};
|
|
267
|
+
const last = parsed.requests?.at(-1);
|
|
268
|
+
if (last?.type === "interrupt" && last?.acknowledged !== true) {
|
|
269
|
+
if (interruptHandled) return;
|
|
270
|
+
interruptHandled = true;
|
|
271
|
+
// RT-4 FIX: write acknowledged:true back SYNCHRONOUSLY so the next
|
|
272
|
+
// tick does not re-fire. Sync because a polling callback cannot await.
|
|
273
|
+
try {
|
|
274
|
+
const reqs = parsed.requests ?? [];
|
|
275
|
+
if (reqs.length > 0) {
|
|
276
|
+
reqs[reqs.length - 1].acknowledged = true;
|
|
277
|
+
atomicWriteFile(controlPath, JSON.stringify(parsed, null, 2));
|
|
278
|
+
}
|
|
279
|
+
} catch {
|
|
280
|
+
/* best-effort ack — interruptHandled gate prevents re-fire */
|
|
281
|
+
}
|
|
282
|
+
appendEventBuffered(opts.manifest.eventsPath, {
|
|
283
|
+
type: "async.interrupt_detected",
|
|
284
|
+
runId: opts.manifest.runId,
|
|
285
|
+
message: opts.eventMessage ?? "Background runner detected foreground interrupt — killing child processes and exiting.",
|
|
286
|
+
}).catch((e) => logInternalError("interrupt-guard.buffered", e, "type=async.interrupt_detected"));
|
|
287
|
+
opts.onInterrupt();
|
|
288
|
+
// Cosmetic (verifier-flagged, live catch follow-up): an ALREADY-ACKED
|
|
289
|
+
// interrupt must not fall through to this branch — it used to warn
|
|
290
|
+
// "Ignoring unknown ... interrupt" every tick until guard stop.
|
|
291
|
+
} else if (last && !(last.type === "interrupt" && last.acknowledged === true)) {
|
|
292
|
+
console.warn(`[background-runner] Ignoring unknown foreground control request: ${last.type}`);
|
|
293
|
+
}
|
|
294
|
+
} catch {
|
|
295
|
+
/* ignore read/parse errors */
|
|
296
|
+
}
|
|
297
|
+
}, interruptGuardInterval);
|
|
298
|
+
interval.unref();
|
|
299
|
+
return () => clearInterval(interval);
|
|
300
|
+
}
|
|
@@ -109,22 +109,43 @@ export class HeartbeatWatcher {
|
|
|
109
109
|
activeKeys.add(key);
|
|
110
110
|
this.lastSeen.set(key, now);
|
|
111
111
|
|
|
112
|
-
//
|
|
113
|
-
|
|
112
|
+
// FIX-2 (live catch, run team_20261030044630_3935473bc4886682):
|
|
113
|
+
// attempt-clamp. handleRetry re-queues tasks WITHOUT resetting
|
|
114
|
+
// liveness fields, so a retry attempt inherited attempt-1's stale
|
|
115
|
+
// heartbeat/agentProgress timestamps and this watcher fired
|
|
116
|
+
// heartbeat_dead 3× for a worker that was ALIVE (fresh heartbeats,
|
|
117
|
+
// active tool calls). Evidence predating the CURRENT attempt's
|
|
118
|
+
// task.startedAt is meaningless and must be ignored; startedAt is
|
|
119
|
+
// reset per attempt at dispatch (child-executor attemptStartedAt),
|
|
120
|
+
// so it is the authoritative attempt boundary.
|
|
121
|
+
const startedAtMs = task.startedAt !== undefined ? Date.parse(task.startedAt) : Number.NaN;
|
|
122
|
+
const attemptStartMs = Number.isFinite(startedAtMs) ? startedAtMs : undefined;
|
|
123
|
+
const evidenceUsable = (iso: string | undefined): boolean => {
|
|
124
|
+
if (iso === undefined) return false;
|
|
125
|
+
const ms = Date.parse(iso);
|
|
126
|
+
return Number.isFinite(ms) && (attemptStartMs === undefined || ms >= attemptStartMs);
|
|
127
|
+
};
|
|
128
|
+
// Check heartbeat staleness with lastActivityAt fallback (evidence
|
|
129
|
+
// older than startedAt is inadmissible — see clamp above).
|
|
130
|
+
let elapsed = evidenceUsable(task.heartbeat?.lastSeenAt) ? heartbeatAgeMs(task.heartbeat, now) : Number.POSITIVE_INFINITY;
|
|
114
131
|
// PR #6 partial: use lastActivityAt as fallback when heartbeat is stale
|
|
115
132
|
// If heartbeat is stale but lastActivityAt is fresher, use activity age instead.
|
|
116
133
|
// This prevents false-positive dead detection for live-session tasks during long operations.
|
|
117
|
-
if (task.agentProgress?.lastActivityAt) {
|
|
118
|
-
const
|
|
119
|
-
if
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
if (activityAge < elapsed) {
|
|
124
|
-
elapsed = activityAge;
|
|
125
|
-
}
|
|
134
|
+
if (evidenceUsable(task.agentProgress?.lastActivityAt)) {
|
|
135
|
+
const activityAge = now - new Date(task.agentProgress!.lastActivityAt!).getTime();
|
|
136
|
+
// Use activity age if it's fresher than heartbeat age
|
|
137
|
+
// (no upper bound - if agent has recent activity, trust it even if old)
|
|
138
|
+
if (activityAge < elapsed) {
|
|
139
|
+
elapsed = activityAge;
|
|
126
140
|
}
|
|
127
141
|
}
|
|
142
|
+
// No usable liveness evidence at all → compute elapsed from the
|
|
143
|
+
// attempt's own start: a task younger than deadMs can never be dead
|
|
144
|
+
// (previously a missing heartbeat meant elapsed=∞ → instant "dead").
|
|
145
|
+
// Tasks without startedAt keep the legacy ∞ (unclassifiable).
|
|
146
|
+
if (!Number.isFinite(elapsed) && attemptStartMs !== undefined) {
|
|
147
|
+
elapsed = Math.max(0, now - attemptStartMs);
|
|
148
|
+
}
|
|
128
149
|
// PID liveness gate: if the worker process is still alive, downgrade
|
|
129
150
|
// "dead" to "stale". This prevents false positives when the LLM spends
|
|
130
151
|
// a long time generating a response (>5 min) without tool calls.
|
|
@@ -639,6 +639,13 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
|
|
|
639
639
|
let lastRunProgressPersistedAt = 0;
|
|
640
640
|
let lastTaskProgressPersistedAt = 0;
|
|
641
641
|
let lastRunProgressSummary: ProgressEventSummary | undefined;
|
|
642
|
+
// FIX-2 (live catch, run team_20261030044630): the current attempt's worker
|
|
643
|
+
// pid, pinned by onSpawn below (hoisted to task scope so the
|
|
644
|
+
// persistHeartbeat closure can pin it into task.heartbeat — see the touch
|
|
645
|
+
// below). Reset to undefined at the TOP of each model-fallback attempt:
|
|
646
|
+
// each iteration spawns a fresh worker process, and a dead prior-attempt
|
|
647
|
+
// pid must never masquerade as this attempt's liveness evidence.
|
|
648
|
+
let attemptWorkerPid: number | undefined;
|
|
642
649
|
const persistHeartbeat = (force = false): void => {
|
|
643
650
|
const now = Date.now();
|
|
644
651
|
// Skip disk write if throttled (unless forced).
|
|
@@ -657,9 +664,21 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
|
|
|
657
664
|
throw err;
|
|
658
665
|
}
|
|
659
666
|
// Now update in-memory heartbeat so it is always >= persisted state.
|
|
667
|
+
// FIX-2: pin the current attempt's worker pid so the heartbeat-watcher
|
|
668
|
+
// PID-liveness gate (heartbeat?.pid ?? checkpoint?.childPid — it
|
|
669
|
+
// downgrades dead→stale when the worker process is ALIVE) actually has
|
|
670
|
+
// something to read for child-process tasks. In the live run every
|
|
671
|
+
// task.heartbeat had pid:undefined, so a >5min-silent-but-alive worker
|
|
672
|
+
// could not be vindicated. Guarded: only overwrite when a pid is known —
|
|
673
|
+
// an unconditional spread would stamp pid:undefined over an
|
|
674
|
+
// already-pinned pid (call shapes pinned in
|
|
675
|
+
// child-executor-heartbeat-pid.test.ts).
|
|
660
676
|
task = {
|
|
661
677
|
...task,
|
|
662
|
-
heartbeat: touchWorkerHeartbeat(
|
|
678
|
+
heartbeat: touchWorkerHeartbeat(
|
|
679
|
+
task.heartbeat ?? createWorkerHeartbeat(task.id, attemptWorkerPid),
|
|
680
|
+
attemptWorkerPid !== undefined ? { pid: attemptWorkerPid } : {},
|
|
681
|
+
),
|
|
663
682
|
};
|
|
664
683
|
lastHeartbeatPersistedAt = now;
|
|
665
684
|
};
|
|
@@ -795,7 +814,9 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
|
|
|
795
814
|
// evidence of an in-flight incarnation; touch (fail-open, bounded:
|
|
796
815
|
// real spawns pin the pid within ms, and the reconciler's ESRCH
|
|
797
816
|
// verdict can never be overridden by a fresh heartbeat).
|
|
798
|
-
let attemptWorkerPid
|
|
817
|
+
// FIX-2: no `let` here — attemptWorkerPid is hoisted to task scope
|
|
818
|
+
// (persistHeartbeat pins it into task.heartbeat); reset per attempt.
|
|
819
|
+
attemptWorkerPid = undefined;
|
|
799
820
|
const heartbeatPulse = startWorkerHeartbeatPulse({
|
|
800
821
|
intervalMs: resolveHeartbeatPulseIntervalMs(),
|
|
801
822
|
isAlive: () => {
|
|
@@ -207,6 +207,9 @@ export async function runLiveTask(input: RunLiveTaskInput): Promise<RunLiveTaskO
|
|
|
207
207
|
if (!isCurrent()) return;
|
|
208
208
|
const now = Date.now();
|
|
209
209
|
if (!force && now - lastHeartbeatPersistedAt < 1000) return;
|
|
210
|
+
// FIX-2 note: no pid at write time — live-session workers run IN this
|
|
211
|
+
// process (no child pid exists); the watcher PID-gate stays unarmed for
|
|
212
|
+
// live-session tasks (their activity stream is the liveness evidence).
|
|
210
213
|
task = {
|
|
211
214
|
...task,
|
|
212
215
|
heartbeat: touchWorkerHeartbeat(task.heartbeat ?? createWorkerHeartbeat(task.id)),
|
|
@@ -386,6 +386,9 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
|
|
|
386
386
|
exitCode: null,
|
|
387
387
|
error: undefined,
|
|
388
388
|
claim: undefined,
|
|
389
|
+
// FIX-2 note: no pid at write time — this terminal touch only marks
|
|
390
|
+
// alive:false on the EXISTING heartbeat (worker already exited); a
|
|
391
|
+
// live pid is pinned at spawn by child-executor's persistHeartbeat.
|
|
389
392
|
heartbeat: touchWorkerHeartbeat(task.heartbeat ?? createWorkerHeartbeat(task.id), { alive: false }),
|
|
390
393
|
terminalEvidence: terminalEvidence.length ? [...(task.terminalEvidence ?? []), ...terminalEvidence] : task.terminalEvidence,
|
|
391
394
|
diagnostics: {
|
|
@@ -880,6 +883,8 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
|
|
|
880
883
|
...(specGate ? { specGate } : {}),
|
|
881
884
|
resultArtifact,
|
|
882
885
|
claim: undefined,
|
|
886
|
+
// FIX-2 note: no pid at write time — terminal touch, worker already
|
|
887
|
+
// exited; the live pid was pinned at spawn (child-executor).
|
|
883
888
|
heartbeat: touchWorkerHeartbeat(task.heartbeat ?? createWorkerHeartbeat(task.id), { alive: false }),
|
|
884
889
|
terminalEvidence: terminalEvidence.length ? [...(task.terminalEvidence ?? []), ...terminalEvidence] : task.terminalEvidence,
|
|
885
890
|
...(logArtifact ? { logArtifact } : {}),
|