pi-crew 0.11.21 → 0.11.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/dist/index.mjs +542 -531
- package/package.json +2 -1
- package/src/errors.ts +14 -1
- package/src/extension/async-notifier.ts +42 -17
- package/src/extension/cross-extension-rpc.ts +42 -5
- package/src/extension/notification-router.ts +15 -2
- package/src/extension/registration/lifecycle-handlers.ts +36 -13
- package/src/extension/registration/runtime-cleanup.ts +17 -0
- package/src/extension/registration/wire-cross-extension.ts +26 -0
- package/src/extension/rpc-hmac.ts +53 -2
- package/src/extension/team-tool/cancel.ts +34 -21
- package/src/extension/team-tool/dispatch/index.ts +6 -5
- package/src/extension/team-tool/dispatch/status.ts +42 -2
- package/src/extension/team-tool/respond.ts +8 -6
- package/src/extension/team-tool/run.ts +4 -4
- package/src/extension/team-tool.ts +24 -2
- package/src/prompt/inbox-poll.ts +77 -25
- package/src/prompt/message-tool.ts +62 -4
- package/src/prompt/prompt-runtime.ts +82 -32
- package/src/prompt/surface-worker.ts +13 -1
- package/src/runtime/broker/crew-broker-client.ts +74 -16
- package/src/runtime/broker/crew-broker.ts +120 -97
- package/src/runtime/broker/protocol/msg-send-targets.ts +74 -0
- package/src/runtime/broker/protocol/request-parsers.ts +68 -0
- package/src/runtime/broker/wait-status-cache.ts +14 -5
- package/src/runtime/child-pi/child-pi-spawn.ts +47 -9
- package/src/runtime/child-pi/child-pi.ts +20 -2
- package/src/runtime/child-pi/mock-fixtures.ts +23 -0
- package/src/runtime/compaction/compact-stages/truncation-stage.ts +45 -3
- package/src/runtime/compaction/important-line-classifier.ts +85 -11
- package/src/runtime/compaction/tool-output-pruner.ts +35 -11
- package/src/runtime/crew-agent-records.ts +32 -14
- package/src/runtime/delivery-coordinator.ts +30 -0
- package/src/runtime/dispatch-batch.ts +9 -6
- package/src/runtime/dwf-state-store.ts +41 -7
- package/src/runtime/finalize-run.ts +13 -2
- package/src/runtime/goal-workflow/dynamic-workflow-context.ts +192 -64
- package/src/runtime/goal-workflow/goal-loop-runner.ts +130 -3
- package/src/runtime/heartbeat/heartbeat-watcher.ts +7 -0
- package/src/runtime/live-session/embedded-tier.ts +3 -1
- package/src/runtime/live-session/live-session-runtime.ts +54 -1
- package/src/runtime/model/model-fallback.ts +23 -1
- package/src/runtime/output/pi-json-output.ts +26 -1
- package/src/runtime/policy-engine.ts +18 -0
- package/src/runtime/process/zombie-scanner.ts +84 -16
- package/src/runtime/process-status.ts +6 -1
- package/src/runtime/recovery/crash-recovery.ts +170 -28
- package/src/runtime/rpc/rpc-worker.ts +44 -1
- package/src/runtime/scheduling/run-coalesced-task-group.ts +73 -6
- package/src/runtime/scheduling/scheduler.ts +74 -11
- package/src/runtime/scratchpad/engine.ts +26 -3
- package/src/runtime/settings-store.ts +22 -2
- package/src/runtime/stale-reconciler.ts +4 -1
- package/src/runtime/surface/tmux-provider.ts +24 -3
- package/src/runtime/task-health.ts +16 -1
- package/src/runtime/task-output-context.ts +143 -71
- package/src/runtime/task-packet.ts +70 -17
- package/src/runtime/task-runner/child-executor.ts +27 -0
- package/src/runtime/task-runner/output-splitter.ts +79 -16
- package/src/runtime/task-runner/post-execution.ts +75 -15
- package/src/runtime/task-runner/state-helpers.ts +92 -2
- package/src/runtime/usage-tracker.ts +11 -0
- package/src/runtime/verification/green-contract.ts +17 -0
- package/src/runtime/verification/verification-gates.ts +24 -6
- package/src/runtime/workspace-lock.ts +53 -0
- package/src/schema/team-tool-schema.ts +10 -4
- package/src/state/coordination/mailbox.ts +133 -38
- package/src/state/decision-ledger.ts +56 -8
- package/src/state/event-log/cursor.ts +14 -2
- package/src/state/event-log/event-log-rotation.ts +83 -5
- package/src/state/event-log/event-log.ts +181 -12
- package/src/state/stores/active-run-registry.ts +10 -3
- package/src/state/types.ts +9 -1
- package/src/ui/agents-jobs-browser.ts +22 -8
- package/src/ui/dashboard-panes/agents-pane.ts +5 -2
- package/src/ui/events-state-source.ts +67 -28
- package/src/ui/inline-panel/agent-pane.ts +13 -6
- package/src/ui/inline-panel/agent-transcript.ts +33 -21
- package/src/ui/inline-panel/agent-view-overlay.ts +18 -3
- package/src/ui/inline-panel/index.ts +21 -1
- package/src/ui/live-run-sidebar.ts +14 -3
- package/src/ui/overlays/mailbox-compose-overlay.ts +10 -4
- package/src/ui/powerbar-publisher.ts +6 -3
- package/src/ui/rail.ts +7 -1
- package/src/ui/render-scheduler.ts +53 -1
- package/src/ui/run-action-dispatcher.ts +68 -20
- package/src/ui/status-colors.ts +9 -0
- package/src/utils/env-filter.ts +8 -0
|
@@ -20,6 +20,8 @@ import { createRequire } from "node:module";
|
|
|
20
20
|
import type { AgentConfig } from "../../agents/agent-config.ts";
|
|
21
21
|
import { loadConfig } from "../../config/config.ts";
|
|
22
22
|
import { effectiveRunConfig } from "../../extension/team-tool/config-patch.ts";
|
|
23
|
+
import { isTerminalTaskStatus } from "../../state/contracts.ts";
|
|
24
|
+
import { withRunLockSync } from "../../state/coordination/locks.ts";
|
|
23
25
|
import { appendEventBuffered } from "../../state/event-log/event-log.ts";
|
|
24
26
|
import { registerActiveRun, unregisterActiveRun } from "../../state/stores/active-run-registry.ts";
|
|
25
27
|
import { collectRunMetrics } from "../../state/stores/run-metrics.ts";
|
|
@@ -28,14 +30,52 @@ import type { GoalLoopState, GoalLoopStatus, GoalVerdict, TeamRunManifest, TeamT
|
|
|
28
30
|
import type { TeamConfig } from "../../teams/team-config.ts";
|
|
29
31
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
30
32
|
import type { WorkflowConfig } from "../../workflows/workflow-config.ts";
|
|
33
|
+
import { isRunLockBusyError } from "../broker/protocol/lock-busy.ts";
|
|
31
34
|
import { resolveCrewRuntime } from "../model/runtime-resolver.ts";
|
|
32
35
|
import { registryFromModelContext } from "../model/session-model.ts";
|
|
33
36
|
import { executeTeamRun } from "../team-runner.ts";
|
|
34
37
|
import { compareSnapshot, snapshotManifests } from "../verification/verification-integrity.ts";
|
|
35
|
-
import { acquireWorkspaceLock, type WorkspaceLockHandle } from "../workspace-lock.ts";
|
|
38
|
+
import { acquireWorkspaceLock, refreshWorkspaceLockHeartbeat, type WorkspaceLockHandle } from "../workspace-lock.ts";
|
|
36
39
|
import { bundleEvidence, evaluateGoal } from "./goal-evaluator.ts";
|
|
37
40
|
import { GoalStore } from "./goal-state-store.ts";
|
|
38
41
|
|
|
42
|
+
/**
|
|
43
|
+
* D5 (DR-02 Wave 2 Cluster C): merge the turn's FINAL task states onto a
|
|
44
|
+
* FRESH tasks.json read. The turn owns exactly the terminal-evidence fields
|
|
45
|
+
* its executor wrote (status/finishedAt/resultArtifact/error/exitCode);
|
|
46
|
+
* everything else on the fresh task (heartbeat/lastSeenAt, claim, delegate
|
|
47
|
+
* state, usage…) may be NEWER than the turn's in-memory snapshot and must
|
|
48
|
+
* never be regressed by the post-turn persist.
|
|
49
|
+
*
|
|
50
|
+
* Terminal-preserve: a task that is ALREADY terminal in the fresh read (an
|
|
51
|
+
* external decision — operator cancel, crash-recovery, the async-died
|
|
52
|
+
* notifier) keeps its fresh state entirely; the turn must not resurrect or
|
|
53
|
+
* re-decide it (same guard class as heartbeat.ts R13-S1). Turn tasks missing
|
|
54
|
+
* from the fresh read are appended — the turn created them and the
|
|
55
|
+
* budget/audit reads this persist exists for expect them on disk.
|
|
56
|
+
*/
|
|
57
|
+
export function mergeTurnFinalTasksOntoFresh(freshTasks: readonly TeamTaskState[], turnTasks: readonly TeamTaskState[]): TeamTaskState[] {
|
|
58
|
+
const freshById = new Map(freshTasks.map((task) => [task.id, task]));
|
|
59
|
+
const turnById = new Map(turnTasks.map((task) => [task.id, task]));
|
|
60
|
+
const merged = freshTasks.map((freshTask) => {
|
|
61
|
+
const turnTask = turnById.get(freshTask.id);
|
|
62
|
+
if (!turnTask) return freshTask;
|
|
63
|
+
if (isTerminalTaskStatus(freshTask.status)) return freshTask;
|
|
64
|
+
return {
|
|
65
|
+
...freshTask,
|
|
66
|
+
status: turnTask.status,
|
|
67
|
+
...(turnTask.finishedAt !== undefined ? { finishedAt: turnTask.finishedAt } : {}),
|
|
68
|
+
...(turnTask.resultArtifact !== undefined ? { resultArtifact: turnTask.resultArtifact } : {}),
|
|
69
|
+
...(turnTask.error !== undefined ? { error: turnTask.error } : {}),
|
|
70
|
+
...(turnTask.exitCode !== undefined ? { exitCode: turnTask.exitCode } : {}),
|
|
71
|
+
};
|
|
72
|
+
});
|
|
73
|
+
for (const turnTask of turnTasks) {
|
|
74
|
+
if (!freshById.has(turnTask.id)) merged.push(turnTask);
|
|
75
|
+
}
|
|
76
|
+
return merged;
|
|
77
|
+
}
|
|
78
|
+
|
|
39
79
|
/** Required minimal shape for the worker + agents discovery (P0 uses the goal's workerAgent). */
|
|
40
80
|
export interface GoalLoopRuntimeDeps {
|
|
41
81
|
/** Resolve the agent configs reachable from cwd (used for executeTeamRun's agents arg). */
|
|
@@ -538,6 +578,48 @@ export function deriveTranscriptPath(artifactsRoot: string, tasks: import("../..
|
|
|
538
578
|
return undefined;
|
|
539
579
|
}
|
|
540
580
|
|
|
581
|
+
/** Interval (ms) between workspace-lock heartbeat refreshes while a goal loop runs (WI-2 / DR-02 R4-3). 15s = 4x margin under the 60s staleness threshold. */
|
|
582
|
+
export const WORKSPACE_LOCK_HEARTBEAT_REFRESH_MS = 15_000;
|
|
583
|
+
|
|
584
|
+
export interface StartWorkspaceLockHeartbeatRefreshOptions {
|
|
585
|
+
/** Test injection: override the refresh interval (ms). */
|
|
586
|
+
refreshMs?: number;
|
|
587
|
+
/** Test injection: override the refresh implementation (default: refreshWorkspaceLockHeartbeat). */
|
|
588
|
+
refresh?: (lockPath: string, goalId: string) => boolean;
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
/**
|
|
592
|
+
* Start the periodic workspace-lock heartbeat refresh for a lock acquired by
|
|
593
|
+
* runGoalLoop (WI-2 / DR-02 R4-3).
|
|
594
|
+
*
|
|
595
|
+
* Without refresh, workspace-lock isLockStale clause (b) marks the lock stale
|
|
596
|
+
* after 60s REGARDLESS of holder pid liveness — a goal whose turn takes >60s
|
|
597
|
+
* gets its workspace lock force-stolen by a second goal on the same cwd
|
|
598
|
+
* (multi-goal clobber). This timer keeps the heartbeat young for the whole
|
|
599
|
+
* loop lifetime. It is unref()-ed (must not keep the background process
|
|
600
|
+
* alive) and the returned stop() MUST be called in the same finally that
|
|
601
|
+
* releases the lock. Refresh failures are logged as warn and swallowed — a
|
|
602
|
+
* heartbeat miss must never throw into the goal loop.
|
|
603
|
+
*/
|
|
604
|
+
export function startWorkspaceLockHeartbeatRefresh(
|
|
605
|
+
handle: Pick<WorkspaceLockHandle, "lockPath" | "goalId">,
|
|
606
|
+
opts: StartWorkspaceLockHeartbeatRefreshOptions = {},
|
|
607
|
+
): { stop: () => void } {
|
|
608
|
+
const refreshMs = opts.refreshMs ?? WORKSPACE_LOCK_HEARTBEAT_REFRESH_MS;
|
|
609
|
+
const refresh = opts.refresh ?? refreshWorkspaceLockHeartbeat;
|
|
610
|
+
const timer = setInterval(() => {
|
|
611
|
+
try {
|
|
612
|
+
refresh(handle.lockPath, handle.goalId);
|
|
613
|
+
} catch (error) {
|
|
614
|
+
// Never throw into the loop (interval callbacks crashing node:test aside,
|
|
615
|
+
// an escaping throw here would kill the background goal process).
|
|
616
|
+
logInternalError("goal-loop.workspaceLockHeartbeat", error, `goalId=${handle.goalId} lockPath=${handle.lockPath}`, "warn");
|
|
617
|
+
}
|
|
618
|
+
}, refreshMs);
|
|
619
|
+
timer.unref(); // the refresh timer must not keep the detached goal-loop process alive
|
|
620
|
+
return { stop: () => clearInterval(timer) };
|
|
621
|
+
}
|
|
622
|
+
|
|
541
623
|
/**
|
|
542
624
|
* Run the autonomous goal loop. Returns {manifest, tasks, goalState} — the OUTER
|
|
543
625
|
* manifest is the synthetic goal-loop manifest (runKind:"goal-loop"); per-turn
|
|
@@ -575,10 +657,18 @@ export async function runGoalLoop(input: RunGoalLoopInput): Promise<RunGoalLoopR
|
|
|
575
657
|
// the background-process boundary. `goal start` / `goal resume` pre-check via isWorkspaceBusy
|
|
576
658
|
// for a good error message; this acquisition is the authoritative claim.
|
|
577
659
|
let workspaceLock: WorkspaceLockHandle | undefined;
|
|
660
|
+
// WI-2 (DR-02 R4-3): periodic heartbeat refresh for the lock above — stopped
|
|
661
|
+
// in the same finally that releases it (see startWorkspaceLockHeartbeatRefresh).
|
|
662
|
+
let lockHeartbeat: { stop: () => void } | undefined;
|
|
578
663
|
try {
|
|
579
664
|
workspaceLock = await acquireWorkspaceLock(goal.cwd, goal.goalId, {
|
|
580
665
|
signal,
|
|
581
666
|
});
|
|
667
|
+
// WI-2 (DR-02 R4-3): keep the heartbeat young for the whole loop lifetime —
|
|
668
|
+
// without this, isLockStale clause (b) force-steals the lock after 60s even
|
|
669
|
+
// though this pid is alive (multi-goal clobber). Started immediately after
|
|
670
|
+
// acquire; the timer is unref()-ed and never throws into the loop.
|
|
671
|
+
lockHeartbeat = startWorkspaceLockHeartbeatRefresh(workspaceLock);
|
|
582
672
|
} catch (error) {
|
|
583
673
|
logInternalError("goal-loop.workspaceLock", error, `goalId=${goal.goalId} cwd=${goal.cwd}`);
|
|
584
674
|
goal = safeSetStatus(store, goal.goalId, "blocked", goal, eventsPath);
|
|
@@ -753,10 +843,41 @@ export async function runGoalLoop(input: RunGoalLoopInput): Promise<RunGoalLoopR
|
|
|
753
843
|
}
|
|
754
844
|
|
|
755
845
|
// Persist final task states for budget/audit reads.
|
|
846
|
+
// D5 (DR-02 Wave 2 Cluster C): the bare saveRunTasks(turnResult.tasks) was
|
|
847
|
+
// an UNLOCKED write of the executor's in-memory snapshot (goal-loop-runner.ts
|
|
848
|
+
// ":757") — it clobbered any NEWER state written under the run lock between
|
|
849
|
+
// the executor's last persist and this save (worker heartbeats/lastSeenAt,
|
|
850
|
+
// task claims, delegate statuses). Now a run-locked RMW: fresh re-read inside
|
|
851
|
+
// the lock, then a field-level merge — the turn re-decides only the fields it
|
|
852
|
+
// owns and never regresses fresh heartbeat/claim/delegate data.
|
|
756
853
|
try {
|
|
757
|
-
|
|
854
|
+
withRunLockSync(turnResult.manifest, () => {
|
|
855
|
+
const fresh = loadRunManifestById(turnResult.manifest.cwd, turnResult.manifest.runId);
|
|
856
|
+
if (!fresh) {
|
|
857
|
+
logInternalError(
|
|
858
|
+
"goal-loop.saveTurnTasks",
|
|
859
|
+
new Error(`turn run '${turnResult.manifest.runId}' vanished before the final task persist — skipped`),
|
|
860
|
+
"best-effort",
|
|
861
|
+
"warn",
|
|
862
|
+
);
|
|
863
|
+
return;
|
|
864
|
+
}
|
|
865
|
+
saveRunTasks(fresh.manifest, mergeTurnFinalTasksOntoFresh(fresh.tasks, turnResult.tasks));
|
|
866
|
+
});
|
|
758
867
|
} catch (error) {
|
|
759
|
-
|
|
868
|
+
if (isRunLockBusyError(error)) {
|
|
869
|
+
// Best-effort persist: the executor already persisted terminal task
|
|
870
|
+
// states during the turn — this merge is belt-and-suspenders for
|
|
871
|
+
// budget/audit reads, so contention skips it with a warn.
|
|
872
|
+
logInternalError(
|
|
873
|
+
"goal-loop.saveTurnTasks",
|
|
874
|
+
error,
|
|
875
|
+
`turnRunId=${created.manifest.runId} run lock busy — skipped final task merge`,
|
|
876
|
+
"warn",
|
|
877
|
+
);
|
|
878
|
+
} else {
|
|
879
|
+
logInternalError("goal-loop.saveTurnTasks", error, `turnRunId=${created.manifest.runId}`);
|
|
880
|
+
}
|
|
760
881
|
}
|
|
761
882
|
|
|
762
883
|
// GL-1: if the turn ended in blocked/failed status, stop the loop without spawning the judge.
|
|
@@ -908,6 +1029,12 @@ export async function runGoalLoop(input: RunGoalLoopInput): Promise<RunGoalLoopR
|
|
|
908
1029
|
}).catch((e) => logInternalError("goal-loop.buffered", e, "type=goal.loop_error"));
|
|
909
1030
|
goal = safeSetStatus(store, goal.goalId, "blocked", goal, eventsPath);
|
|
910
1031
|
} finally {
|
|
1032
|
+
// WI-2: stop the heartbeat refresh BEFORE releasing the lock it was keeping alive.
|
|
1033
|
+
try {
|
|
1034
|
+
lockHeartbeat?.stop();
|
|
1035
|
+
} catch {
|
|
1036
|
+
/* best-effort */
|
|
1037
|
+
}
|
|
911
1038
|
// P1g: release the workspace lock (held since loop start).
|
|
912
1039
|
try {
|
|
913
1040
|
workspaceLock?.release();
|
|
@@ -231,6 +231,11 @@ export class HeartbeatWatcher {
|
|
|
231
231
|
if (!activeKeys.has(key) && ts < cutoff) {
|
|
232
232
|
this.lastLevel.delete(key);
|
|
233
233
|
this.consecutiveDead.delete(key);
|
|
234
|
+
// M-8 (2026-10-16): also drop the deadletter cooldown marker —
|
|
235
|
+
// before this the map leaked one entry per vanished run:taskId key
|
|
236
|
+
// and a stale cooldown timestamp could suppress a legitimate
|
|
237
|
+
// re-trigger after the same task reappeared.
|
|
238
|
+
this.lastDeadletterTriggerAt.delete(key);
|
|
234
239
|
this.lastSeen.delete(key);
|
|
235
240
|
}
|
|
236
241
|
}
|
|
@@ -242,5 +247,7 @@ export class HeartbeatWatcher {
|
|
|
242
247
|
this.lastLevel.clear();
|
|
243
248
|
this.consecutiveDead.clear();
|
|
244
249
|
this.lastSeen.clear();
|
|
250
|
+
// M-8: clear the deadletter cooldown map too (was leaked before).
|
|
251
|
+
this.lastDeadletterTriggerAt.clear();
|
|
245
252
|
}
|
|
246
253
|
}
|
|
@@ -908,7 +908,9 @@ export function admitEmbeddedSession(input: {
|
|
|
908
908
|
});
|
|
909
909
|
// Raced against the prompt; a pre-race rejection must not surface as an
|
|
910
910
|
// unhandled rejection (which would trip our own process guard).
|
|
911
|
-
escapeSignal.catch(() => {
|
|
911
|
+
escapeSignal.catch(() => {
|
|
912
|
+
/* intentional UR-guard — see comment above */
|
|
913
|
+
});
|
|
912
914
|
let tornDown = false;
|
|
913
915
|
registerEmbeddedSessionGuard(
|
|
914
916
|
input.agentId,
|
|
@@ -868,6 +868,49 @@ export async function probeLiveSessionRuntime(): Promise<LiveSessionUnavailableR
|
|
|
868
868
|
};
|
|
869
869
|
}
|
|
870
870
|
|
|
871
|
+
// DR-02 C1: the per-agent control-request dedupe Set previously grew for
|
|
872
|
+
// the whole run lifetime — fed by BOTH the 500ms/1000ms control poll and the
|
|
873
|
+
// realtime subscription, with no eviction. Bounded FIFO eviction: a Set
|
|
874
|
+
// preserves insertion order, so deleting the first-added id evicts the OLDEST
|
|
875
|
+
// request id (mirrors the NotificationRouter.SEEN_MAP_MAX_SIZE bounded-seen
|
|
876
|
+
// pattern). Worst case after an eviction is one redelivery of a very old
|
|
877
|
+
// control request; the consumers (steer / follow-up) are idempotent by design.
|
|
878
|
+
export const MAX_SEEN_CONTROL_REQUEST_IDS = 10_000;
|
|
879
|
+
|
|
880
|
+
class BoundedSeenControlRequestIdSet extends Set<string> {
|
|
881
|
+
override add(id: string): this {
|
|
882
|
+
super.add(id);
|
|
883
|
+
while (this.size > MAX_SEEN_CONTROL_REQUEST_IDS) {
|
|
884
|
+
const oldest = this.values().next().value;
|
|
885
|
+
if (oldest === undefined) break;
|
|
886
|
+
this.delete(oldest);
|
|
887
|
+
}
|
|
888
|
+
return this;
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
/** @internal — test seam for the C1 cap (see live-agent-control-dedupe tests). */
|
|
893
|
+
export const __test__BoundedSeenControlRequestIdSet = BoundedSeenControlRequestIdSet;
|
|
894
|
+
|
|
895
|
+
/**
|
|
896
|
+
* R6-9 (DR-03 round-6): best-effort usage extraction for the fault/watchdog
|
|
897
|
+
* escape returns. The success return reports usageFromSessionStats(session),
|
|
898
|
+
* but the fault/watchdog catch paths previously carried NO usage — whenever
|
|
899
|
+
* embeddedWorkers routed work to the embedded tier and the task faulted, the
|
|
900
|
+
* burned turns vanished from the budget (under-count). The session may be
|
|
901
|
+
* disposed or partially unusable after the fault, so stats collection is
|
|
902
|
+
* guarded — it can never turn a fault into a crash. Returns undefined when no
|
|
903
|
+
* usable stats surface exists.
|
|
904
|
+
*/
|
|
905
|
+
function faultPathUsage(session: LiveSessionLike | undefined): UsageState | undefined {
|
|
906
|
+
try {
|
|
907
|
+
return usageFromSessionStats(session);
|
|
908
|
+
} catch (error) {
|
|
909
|
+
logInternalError("live-session.fault-usage-stats", error, "session stats unavailable on fault path");
|
|
910
|
+
return undefined;
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
|
|
871
914
|
export async function runLiveSessionTask(input: LiveSessionSpawnInput): Promise<LiveSessionRunResult> {
|
|
872
915
|
// Cold-start race fix: ensure the hot module graph is warm before touching
|
|
873
916
|
// any module. Under tsx, concurrent first-imports race module-record
|
|
@@ -1406,7 +1449,7 @@ export async function runLiveSessionTask(input: LiveSessionSpawnInput): Promise<
|
|
|
1406
1449
|
registerLiveAgentModel(agentId, resolvedModelRef ?? "");
|
|
1407
1450
|
streamOut = createStreamingOutput(input.manifest, input.task.id);
|
|
1408
1451
|
let controlCursor: LiveAgentControlCursor = { offset: 0 };
|
|
1409
|
-
const seenControlRequestIds = new
|
|
1452
|
+
const seenControlRequestIds: Set<string> = new BoundedSeenControlRequestIdSet();
|
|
1410
1453
|
let controlBusy = false;
|
|
1411
1454
|
const pollControl = async () => {
|
|
1412
1455
|
if (!isCurrent() || controlBusy || !session) return;
|
|
@@ -1793,12 +1836,16 @@ export async function runLiveSessionTask(input: LiveSessionSpawnInput): Promise<
|
|
|
1793
1836
|
},
|
|
1794
1837
|
});
|
|
1795
1838
|
updateLiveAgentStatus(agentId, "failed");
|
|
1839
|
+
// R6-9: turns burned before the watchdog fired still count — capture the
|
|
1840
|
+
// partial usage (guarded; session was disposed by the watchdog timer).
|
|
1841
|
+
const watchdogUsage = faultPathUsage(session);
|
|
1796
1842
|
return {
|
|
1797
1843
|
available: true,
|
|
1798
1844
|
exitCode: 1,
|
|
1799
1845
|
stdout: stdoutTail.value().trim(),
|
|
1800
1846
|
stderr: error.message,
|
|
1801
1847
|
jsonEvents,
|
|
1848
|
+
usage: watchdogUsage,
|
|
1802
1849
|
error: error.message,
|
|
1803
1850
|
...(embedded ? { fault: { kind: "turn-watchdog" as const, detail: error.message, countAsFault: true } } : {}),
|
|
1804
1851
|
};
|
|
@@ -1820,12 +1867,15 @@ export async function runLiveSessionTask(input: LiveSessionSpawnInput): Promise<
|
|
|
1820
1867
|
// drop-reference (synchronous — never awaits the stuck abort): drain +
|
|
1821
1868
|
// terminate + dispose + UNREGISTER, so the hung session holds no slot.
|
|
1822
1869
|
dropLiveAgentReference(agentId);
|
|
1870
|
+
// R6-9: partial usage from the turns completed before the tool hung.
|
|
1871
|
+
const hungUsage = faultPathUsage(session);
|
|
1823
1872
|
return {
|
|
1824
1873
|
available: true,
|
|
1825
1874
|
exitCode: 1,
|
|
1826
1875
|
stdout: stdoutTail.value().trim(),
|
|
1827
1876
|
stderr: error.message,
|
|
1828
1877
|
jsonEvents,
|
|
1878
|
+
usage: hungUsage,
|
|
1829
1879
|
error: error.message,
|
|
1830
1880
|
...(embedded ? { fault: { kind: "hung-tool" as const, timeoutMs: error.timeoutMs, countAsFault: true } } : {}),
|
|
1831
1881
|
};
|
|
@@ -1861,6 +1911,9 @@ export async function runLiveSessionTask(input: LiveSessionSpawnInput): Promise<
|
|
|
1861
1911
|
stdout: stdoutTail.value().trim(),
|
|
1862
1912
|
stderr: message,
|
|
1863
1913
|
jsonEvents,
|
|
1914
|
+
// R6-9: generic fault path — partial usage from completed turns so the
|
|
1915
|
+
// budget sees the real spend even when the task failed.
|
|
1916
|
+
usage: faultPathUsage(session),
|
|
1864
1917
|
error: message,
|
|
1865
1918
|
...(embeddedCatchFault ? { fault: embeddedCatchFault } : {}),
|
|
1866
1919
|
};
|
|
@@ -606,7 +606,29 @@ export function resolveModelFallbackPolicy(
|
|
|
606
606
|
} else {
|
|
607
607
|
maxAutoFallbacks = config?.maxAutoFallbacks;
|
|
608
608
|
}
|
|
609
|
-
|
|
609
|
+
// J5 (DR-03 round-6): validate the order env against the two legal values.
|
|
610
|
+
// The previous unvalidated cast let arbitrary strings flow into
|
|
611
|
+
// orderAutoFallbacks' `order === "asIs"` checks — any typo (e.g.
|
|
612
|
+
// "parent-first") silently behaved as parentFirst. Invalid → warn + fall
|
|
613
|
+
// back to config/default, mirroring the PI_CREW_MAX_AUTO_FALLBACKS
|
|
614
|
+
// clamp-warn sibling pattern above.
|
|
615
|
+
const envOrder = env.PI_CREW_MODEL_FALLBACK_ORDER;
|
|
616
|
+
let order: "parentFirst" | "asIs" | undefined;
|
|
617
|
+
if (envOrder) {
|
|
618
|
+
if (envOrder === "parentFirst" || envOrder === "asIs") {
|
|
619
|
+
order = envOrder;
|
|
620
|
+
} else {
|
|
621
|
+
logInternalError(
|
|
622
|
+
"model-fallback.model-fallback-order-invalid",
|
|
623
|
+
undefined,
|
|
624
|
+
`PI_CREW_MODEL_FALLBACK_ORDER="${envOrder}" is invalid (expected "parentFirst" or "asIs"), falling back to config`,
|
|
625
|
+
"warn",
|
|
626
|
+
);
|
|
627
|
+
order = config?.order;
|
|
628
|
+
}
|
|
629
|
+
} else {
|
|
630
|
+
order = config?.order;
|
|
631
|
+
}
|
|
610
632
|
const requireCredentials =
|
|
611
633
|
env.PI_CREW_MODEL_REQUIRE_CREDENTIALS === "1"
|
|
612
634
|
? true
|
|
@@ -20,6 +20,14 @@ export interface ParsedPiJsonOutput {
|
|
|
20
20
|
* "429 ... overloaded"). Used to detect runs that exited 0 but produced
|
|
21
21
|
* nothing because the model was rate-limited — see task-runner 429 fix. */
|
|
22
22
|
errorMessages?: string[];
|
|
23
|
+
/** DR-01 (deep review 2026-10-14): true when the LAST assistant
|
|
24
|
+
* `message_end` event in the stream carried no non-empty text — the
|
|
25
|
+
* session settled without a final answer. Upstream 429 exhaustion
|
|
26
|
+
* manifests as a chain of EMPTY assistant message_end events (content [],
|
|
27
|
+
* usage 0) between auto-retry cycles — never as stopReason:"error" — so
|
|
28
|
+
* `errorMessages` stays empty and `finalText` keeps an early pre-outage
|
|
29
|
+
* sentence. undefined when no assistant message_end was parsed at all. */
|
|
30
|
+
settledWithoutAnswer?: boolean;
|
|
23
31
|
}
|
|
24
32
|
|
|
25
33
|
function asRecord(value: unknown): Record<string, unknown> | undefined {
|
|
@@ -82,12 +90,23 @@ export function extractText(value: unknown): string[] {
|
|
|
82
90
|
return text.filter((entry) => entry.trim().length > 0);
|
|
83
91
|
}
|
|
84
92
|
|
|
93
|
+
/** True for `message_end` events whose message is assistant-role. Feeds the
|
|
94
|
+
* settledWithoutAnswer flag (DR-01): see ParsedPiJsonOutput.settledWithoutAnswer. */
|
|
95
|
+
function isAssistantMessageEnd(event: unknown): boolean {
|
|
96
|
+
const obj = asRecord(event);
|
|
97
|
+
if (obj?.type !== "message_end") return false;
|
|
98
|
+
const message = asRecord(obj.message);
|
|
99
|
+
return message?.role === "assistant";
|
|
100
|
+
}
|
|
101
|
+
|
|
85
102
|
export function parsePiJsonOutput(stdout: string): ParsedPiJsonOutput {
|
|
86
103
|
let jsonEvents = 0;
|
|
87
104
|
const textEvents: string[] = [];
|
|
88
105
|
const patches: string[] = [];
|
|
89
106
|
const errorMessages: string[] = [];
|
|
90
107
|
let usage: ParsedPiUsage | undefined;
|
|
108
|
+
let sawAssistantMessageEnd = false;
|
|
109
|
+
let lastAssistantMessageHadText = false;
|
|
91
110
|
for (const line of stdout.split("\n")) {
|
|
92
111
|
const trimmed = line.trim();
|
|
93
112
|
if (!trimmed) continue;
|
|
@@ -98,7 +117,12 @@ export function parsePiJsonOutput(stdout: string): ParsedPiJsonOutput {
|
|
|
98
117
|
continue;
|
|
99
118
|
}
|
|
100
119
|
jsonEvents++;
|
|
101
|
-
|
|
120
|
+
const eventTexts = extractText(event);
|
|
121
|
+
textEvents.push(...eventTexts);
|
|
122
|
+
if (isAssistantMessageEnd(event)) {
|
|
123
|
+
sawAssistantMessageEnd = true;
|
|
124
|
+
lastAssistantMessageHadText = eventTexts.length > 0;
|
|
125
|
+
}
|
|
102
126
|
// Extract unified patches from tool_result events
|
|
103
127
|
extractPatch(event, patches);
|
|
104
128
|
// Extract provider/model error messages from message_end events (429 fix).
|
|
@@ -114,6 +138,7 @@ export function parsePiJsonOutput(stdout: string): ParsedPiJsonOutput {
|
|
|
114
138
|
usage,
|
|
115
139
|
patches: patches.length > 0 ? patches : undefined,
|
|
116
140
|
errorMessages: errorMessages.length > 0 ? errorMessages : undefined,
|
|
141
|
+
settledWithoutAnswer: sawAssistantMessageEnd ? !lastAssistantMessageHadText : undefined,
|
|
117
142
|
};
|
|
118
143
|
}
|
|
119
144
|
|
|
@@ -104,6 +104,24 @@ export function evaluateCrewPolicy(input: PolicyEngineInput): PolicyDecision[] {
|
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
+
// R4-4 (DR-02 wave 2): needs_attention is a TERMINAL-without-result status
|
|
108
|
+
// (surface-lost / no-yield paths in post-execution) and the scheduler counts
|
|
109
|
+
// it as done, so a run can otherwise complete green while a task never
|
|
110
|
+
// submitted its result. Surface it as a NON-blocking notify decision —
|
|
111
|
+
// detection-first: no status flip, no block, no retry (re-queue/recovery is
|
|
112
|
+
// owned by the surface-degrade drain machinery, not the policy engine).
|
|
113
|
+
const needsAttentionTasks = input.tasks.filter((t) => t.status === "needs_attention");
|
|
114
|
+
if (needsAttentionTasks.length > 0) {
|
|
115
|
+
const ids = needsAttentionTasks.map((t) => t.id).join(", ");
|
|
116
|
+
decisions.push(
|
|
117
|
+
decision(
|
|
118
|
+
"notify",
|
|
119
|
+
"task_needs_attention",
|
|
120
|
+
`${needsAttentionTasks.length} task(s) finished needs_attention without submitting a result: ${ids}.`,
|
|
121
|
+
),
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
|
|
107
125
|
if (decisions.length === 0 && input.tasks.length > 0 && input.tasks.every((task) => task.status === "completed")) {
|
|
108
126
|
decisions.push(decision("closeout", "run_complete", "All tasks completed and no policy blockers were found."));
|
|
109
127
|
}
|
|
@@ -60,6 +60,13 @@ export interface ZombieSubagent {
|
|
|
60
60
|
surfacePaneId?: string;
|
|
61
61
|
rssKb: number;
|
|
62
62
|
elapsedSec: number | undefined;
|
|
63
|
+
/** M-7 (DR-04 round-7): /proc/<pid>/stat starttime — clock ticks since boot.
|
|
64
|
+
* Stable pid identity: unlike the pid number, starttime changes when the OS
|
|
65
|
+
* recycles a dead pid for a NEW process, so kill-time re-checks can detect
|
|
66
|
+
* PID reuse between scan and kill. */
|
|
67
|
+
starttime?: number;
|
|
68
|
+
/** M-7: wall-clock start time (ISO) derived from starttime + /proc/uptime. */
|
|
69
|
+
startedAt?: string;
|
|
63
70
|
cmd: string;
|
|
64
71
|
}
|
|
65
72
|
|
|
@@ -106,8 +113,8 @@ function readProcEnviron(pid: number): Record<string, string> {
|
|
|
106
113
|
}
|
|
107
114
|
}
|
|
108
115
|
|
|
109
|
-
/** Read /proc/<pid>/stat to get ppid + elapsed. Returns undefined if unreadable. */
|
|
110
|
-
function readProcStat(pid: number): { ppid: number; elapsedSec: number | undefined } | undefined {
|
|
116
|
+
/** Read /proc/<pid>/stat to get ppid + starttime + elapsed. Returns undefined if unreadable. */
|
|
117
|
+
function readProcStat(pid: number): { ppid: number; starttime: number | undefined; elapsedSec: number | undefined } | undefined {
|
|
111
118
|
try {
|
|
112
119
|
const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf-8");
|
|
113
120
|
// stat format: pid (comm) state ppid ... starttime ...
|
|
@@ -117,14 +124,38 @@ function readProcStat(pid: number): { ppid: number; elapsedSec: number | undefin
|
|
|
117
124
|
if (!rest) return undefined;
|
|
118
125
|
const ppid = Number.parseInt(rest[PROC_STAT_PPID_INDEX] ?? "", 10);
|
|
119
126
|
const starttimeTicksRaw = Number.parseInt(rest[PROC_STAT_STARTTIME_INDEX] ?? "", 10);
|
|
120
|
-
const
|
|
121
|
-
const elapsedSec = computeElapsedSec(
|
|
122
|
-
return { ppid: Number.isFinite(ppid) ? ppid : 0, elapsedSec };
|
|
127
|
+
const starttime = Number.isFinite(starttimeTicksRaw) ? starttimeTicksRaw : undefined;
|
|
128
|
+
const elapsedSec = computeElapsedSec(starttime);
|
|
129
|
+
return { ppid: Number.isFinite(ppid) ? ppid : 0, starttime, elapsedSec };
|
|
123
130
|
} catch {
|
|
124
131
|
return undefined;
|
|
125
132
|
}
|
|
126
133
|
}
|
|
127
134
|
|
|
135
|
+
/** /proc/uptime first field (seconds since boot); undefined when unreadable. */
|
|
136
|
+
function readUptimeSec(): number | undefined {
|
|
137
|
+
try {
|
|
138
|
+
const uptimeRaw = fs.readFileSync("/proc/uptime", "utf-8");
|
|
139
|
+
const uptimeSec = Number.parseFloat(uptimeRaw.split(" ")[0] ?? "");
|
|
140
|
+
return Number.isFinite(uptimeSec) ? uptimeSec : undefined;
|
|
141
|
+
} catch {
|
|
142
|
+
return undefined;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* M-7 (DR-04 round-7): wall-clock start time of a pid from its starttime
|
|
148
|
+
* ticks — boot time = now - uptime, so startedAt = boot + starttime/CLK_TCK.
|
|
149
|
+
* Best-effort ISO string; undefined when /proc/uptime is unavailable.
|
|
150
|
+
*/
|
|
151
|
+
function computeStartedAtIso(starttime: number | undefined): string | undefined {
|
|
152
|
+
if (starttime === undefined || !Number.isFinite(starttime)) return undefined;
|
|
153
|
+
const uptimeSec = readUptimeSec();
|
|
154
|
+
if (uptimeSec === undefined) return undefined;
|
|
155
|
+
const bootMs = Date.now() - uptimeSec * 1000;
|
|
156
|
+
return new Date(bootMs + (starttime / 100) * 1000).toISOString();
|
|
157
|
+
}
|
|
158
|
+
|
|
128
159
|
function computeElapsedSec(starttimeTicks: number | undefined): number | undefined {
|
|
129
160
|
if (starttimeTicks === undefined || !Number.isFinite(starttimeTicks)) return undefined;
|
|
130
161
|
try {
|
|
@@ -132,10 +163,8 @@ function computeElapsedSec(starttimeTicks: number | undefined): number | undefin
|
|
|
132
163
|
// portably from Node requires a native addon; hardcoding 100 matches every
|
|
133
164
|
// mainstream Linux distro and keeps this dependency-free.
|
|
134
165
|
const ticksPerSec = 100;
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
const uptimeSec = Number.parseFloat(uptimeRaw.split(" ")[0] ?? "");
|
|
138
|
-
if (!Number.isFinite(uptimeSec)) return undefined;
|
|
166
|
+
const uptimeSec = readUptimeSec();
|
|
167
|
+
if (uptimeSec === undefined) return undefined;
|
|
139
168
|
// starttime (ticks since boot) → process age in seconds = uptime - starttime/ticksPerSec.
|
|
140
169
|
const startAgeSec = starttimeTicks / ticksPerSec;
|
|
141
170
|
return Math.max(0, uptimeSec - startAgeSec);
|
|
@@ -150,8 +179,12 @@ function isPidAlive(pid: number): boolean {
|
|
|
150
179
|
// process.kill(pid, 0) throws if the pid is not alive (or not ours).
|
|
151
180
|
process.kill(pid, 0);
|
|
152
181
|
return true;
|
|
153
|
-
} catch {
|
|
154
|
-
|
|
182
|
+
} catch (error) {
|
|
183
|
+
// R7-A2 (2026-10-17): only ESRCH/ENOENT mean the process is gone.
|
|
184
|
+
// EPERM means the process is alive but protected (different security
|
|
185
|
+
// context); treat as alive — mirror of active-run-registry.ts fix.
|
|
186
|
+
const code = (error as NodeJS.ErrnoException).code;
|
|
187
|
+
return code !== "ESRCH" && code !== "ENOENT";
|
|
155
188
|
}
|
|
156
189
|
}
|
|
157
190
|
|
|
@@ -271,6 +304,10 @@ export function scanZombieSubagents(): ZombieScanResult {
|
|
|
271
304
|
surfacePaneId: environ.PI_CREW_SURFACE_PANE || undefined,
|
|
272
305
|
rssKb: readProcRssKb(pid),
|
|
273
306
|
elapsedSec: stat?.elapsedSec,
|
|
307
|
+
// M-7: starttime (ticks) + derived wall-clock start — pid identity pin
|
|
308
|
+
// for kill-time PID-reuse checks and prompt-kill triage.
|
|
309
|
+
starttime: stat?.starttime,
|
|
310
|
+
startedAt: computeStartedAtIso(stat?.starttime),
|
|
274
311
|
cmd: readProcCmdline(pid),
|
|
275
312
|
};
|
|
276
313
|
|
|
@@ -392,9 +429,13 @@ export function formatZombieReport(scan: ZombieScanResult): string {
|
|
|
392
429
|
|
|
393
430
|
if (scan.zombies.length > 0) {
|
|
394
431
|
lines.push(`### Zombies — parent dead (${scan.zombies.length})`);
|
|
395
|
-
lines.push("These sub-agents are orphaned. Safe to kill after
|
|
432
|
+
lines.push("These sub-agents are orphaned. Safe to kill, but do it PROMPTLY after this");
|
|
433
|
+
lines.push("scan: an unreviewed delay re-opens the PID-reuse window (the OS can recycle");
|
|
434
|
+
lines.push("a dead zombie's PID for an unrelated process). Before killing, re-check that");
|
|
435
|
+
lines.push("the PID's STARTED column below still matches — a newer starttime means the");
|
|
436
|
+
lines.push("process you are about to kill is NOT the zombie that was scanned.");
|
|
396
437
|
lines.push("");
|
|
397
|
-
lines.push(" PID PARENT RSS ROLE SURFACE CMD");
|
|
438
|
+
lines.push(" PID PARENT AGE STARTED RSS ROLE SURFACE CMD");
|
|
398
439
|
for (const z of scan.zombies) {
|
|
399
440
|
lines.push(formatZombieRow(z));
|
|
400
441
|
}
|
|
@@ -405,7 +446,7 @@ export function formatZombieReport(scan: ZombieScanResult): string {
|
|
|
405
446
|
lines.push(`### Live — parent still running (${scan.live.length})`);
|
|
406
447
|
lines.push("NOT zombies. Do not kill (parent PID is alive and may still reap them).");
|
|
407
448
|
lines.push("");
|
|
408
|
-
lines.push(" PID PARENT RSS ROLE SURFACE CMD");
|
|
449
|
+
lines.push(" PID PARENT AGE STARTED RSS ROLE SURFACE CMD");
|
|
409
450
|
for (const l of scan.live) {
|
|
410
451
|
lines.push(formatZombieRow(l));
|
|
411
452
|
}
|
|
@@ -435,7 +476,9 @@ export function formatZombieReport(scan: ZombieScanResult): string {
|
|
|
435
476
|
lines.push("");
|
|
436
477
|
}
|
|
437
478
|
|
|
438
|
-
lines.push("To kill a zombie: `kill <PID>` (the OS will reap it).
|
|
479
|
+
lines.push("To kill a zombie: `kill <PID>` (the OS will reap it). Kill soon after the");
|
|
480
|
+
lines.push("scan — a PID listed here can be reused by an unrelated process if you wait;");
|
|
481
|
+
lines.push("verify its starttime/age first. This tool never kills.");
|
|
439
482
|
return lines.join("\n");
|
|
440
483
|
}
|
|
441
484
|
|
|
@@ -443,7 +486,30 @@ export function formatZombieReport(scan: ZombieScanResult): string {
|
|
|
443
486
|
function formatZombieRow(z: ZombieSubagent): string {
|
|
444
487
|
// SURFACE = "kind:paneId" (T12) — the pane doctor closes when orphaned.
|
|
445
488
|
const surface = z.surface ? `${z.surface}:${z.surfacePaneId ?? "?"}` : "-";
|
|
446
|
-
|
|
489
|
+
// M-7: AGE + STARTED — pid identity + triage (old zombies = long-leaked;
|
|
490
|
+
// STARTED mismatch at kill time = PID was reused, do NOT kill).
|
|
491
|
+
return (
|
|
492
|
+
` ${String(z.pid).padEnd(9)}${String(z.crewParentPid).padEnd(8)}${formatAge(z.elapsedSec).padEnd(9)}` +
|
|
493
|
+
`${formatStartedAt(z.startedAt).padEnd(21)}${formatRss(z.rssKb).padEnd(10)}${(z.role ?? "?").padEnd(14)}` +
|
|
494
|
+
`${surface.padEnd(16)}${z.cmd.slice(0, 28)}`
|
|
495
|
+
);
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/** M-7: age as a compact human duration ("37s", "5m", "3h", "2d"). */
|
|
499
|
+
function formatAge(sec: number | undefined): string {
|
|
500
|
+
if (sec === undefined || !Number.isFinite(sec) || sec < 0) return "-";
|
|
501
|
+
if (sec < 60) return `${Math.floor(sec)}s`;
|
|
502
|
+
if (sec < 3600) return `${Math.floor(sec / 60)}m`;
|
|
503
|
+
if (sec < 86400) return `${Math.floor(sec / 3600)}h`;
|
|
504
|
+
return `${Math.floor(sec / 86400)}d`;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/** M-7: ISO startedAt → "MM-DD HH:MM:SS" (UTC, no year — scan-time triage). */
|
|
508
|
+
function formatStartedAt(iso: string | undefined): string {
|
|
509
|
+
if (!iso) return "-";
|
|
510
|
+
const d = new Date(iso);
|
|
511
|
+
if (Number.isNaN(d.getTime())) return "-";
|
|
512
|
+
return d.toISOString().slice(5, 19).replace("T", " ");
|
|
447
513
|
}
|
|
448
514
|
|
|
449
515
|
/** One table row for the R3-16 foreign section — state carries the triage flags. */
|
|
@@ -469,4 +535,6 @@ export const __test = {
|
|
|
469
535
|
looksLikePiBinary,
|
|
470
536
|
isHeadlessArgv,
|
|
471
537
|
collectAncestorPids,
|
|
538
|
+
// M-7: starttime helpers (ticks → wall-clock start).
|
|
539
|
+
computeStartedAtIso,
|
|
472
540
|
};
|
|
@@ -140,7 +140,12 @@ export function isDisplayActiveRun(run: TeamRunManifest, agents: CrewAgentRecord
|
|
|
140
140
|
if (Number.isFinite(updatedAt) && now - updatedAt > STALE_ACTIVE_RUN_MS) return false;
|
|
141
141
|
}
|
|
142
142
|
// Grace period: show completed runs for a few seconds so users see the result.
|
|
143
|
-
|
|
143
|
+
// R5-4 (DR-02 wave 2): "blocked" is terminal (isFinishedRunStatus) but NOT an
|
|
144
|
+
// error the user caused — without the grace branch it fell through to
|
|
145
|
+
// `!isActiveRunStatus` and vanished from the widget instantly while "failed"
|
|
146
|
+
// lingered 10min. Blocked now gets the same ERROR_VISIBILITY_GRACE_MS so the
|
|
147
|
+
// block reason stays inspectable.
|
|
148
|
+
if (run.status === "completed" || run.status === "failed" || run.status === "cancelled" || run.status === "blocked") {
|
|
144
149
|
const grace = run.status === "completed" ? COMPLETED_VISIBILITY_GRACE_MS : ERROR_VISIBILITY_GRACE_MS;
|
|
145
150
|
const lastAgentActivity = agents.reduce<number>((max, agent) => {
|
|
146
151
|
const ts = agent.completedAt ?? agent.startedAt;
|