pi-crew 0.11.21 → 0.11.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/CHANGELOG.md +41 -0
  2. package/dist/index.mjs +542 -531
  3. package/package.json +2 -1
  4. package/src/errors.ts +14 -1
  5. package/src/extension/async-notifier.ts +42 -17
  6. package/src/extension/cross-extension-rpc.ts +42 -5
  7. package/src/extension/notification-router.ts +15 -2
  8. package/src/extension/registration/lifecycle-handlers.ts +36 -13
  9. package/src/extension/registration/runtime-cleanup.ts +17 -0
  10. package/src/extension/registration/wire-cross-extension.ts +26 -0
  11. package/src/extension/rpc-hmac.ts +53 -2
  12. package/src/extension/team-tool/cancel.ts +34 -21
  13. package/src/extension/team-tool/dispatch/index.ts +6 -5
  14. package/src/extension/team-tool/dispatch/status.ts +42 -2
  15. package/src/extension/team-tool/respond.ts +8 -6
  16. package/src/extension/team-tool/run.ts +4 -4
  17. package/src/extension/team-tool.ts +24 -2
  18. package/src/prompt/inbox-poll.ts +77 -25
  19. package/src/prompt/message-tool.ts +62 -4
  20. package/src/prompt/prompt-runtime.ts +82 -32
  21. package/src/prompt/surface-worker.ts +13 -1
  22. package/src/runtime/broker/crew-broker-client.ts +74 -16
  23. package/src/runtime/broker/crew-broker.ts +120 -97
  24. package/src/runtime/broker/protocol/msg-send-targets.ts +74 -0
  25. package/src/runtime/broker/protocol/request-parsers.ts +68 -0
  26. package/src/runtime/broker/wait-status-cache.ts +14 -5
  27. package/src/runtime/child-pi/child-pi-spawn.ts +47 -9
  28. package/src/runtime/child-pi/child-pi.ts +20 -2
  29. package/src/runtime/child-pi/mock-fixtures.ts +23 -0
  30. package/src/runtime/compaction/compact-stages/truncation-stage.ts +45 -3
  31. package/src/runtime/compaction/important-line-classifier.ts +85 -11
  32. package/src/runtime/compaction/tool-output-pruner.ts +35 -11
  33. package/src/runtime/crew-agent-records.ts +32 -14
  34. package/src/runtime/delivery-coordinator.ts +30 -0
  35. package/src/runtime/dispatch-batch.ts +9 -6
  36. package/src/runtime/dwf-state-store.ts +41 -7
  37. package/src/runtime/finalize-run.ts +13 -2
  38. package/src/runtime/goal-workflow/dynamic-workflow-context.ts +192 -64
  39. package/src/runtime/goal-workflow/goal-loop-runner.ts +130 -3
  40. package/src/runtime/heartbeat/heartbeat-watcher.ts +7 -0
  41. package/src/runtime/live-session/embedded-tier.ts +3 -1
  42. package/src/runtime/live-session/live-session-runtime.ts +54 -1
  43. package/src/runtime/model/model-fallback.ts +23 -1
  44. package/src/runtime/output/pi-json-output.ts +26 -1
  45. package/src/runtime/policy-engine.ts +18 -0
  46. package/src/runtime/process/zombie-scanner.ts +84 -16
  47. package/src/runtime/process-status.ts +6 -1
  48. package/src/runtime/recovery/crash-recovery.ts +170 -28
  49. package/src/runtime/rpc/rpc-worker.ts +44 -1
  50. package/src/runtime/scheduling/run-coalesced-task-group.ts +73 -6
  51. package/src/runtime/scheduling/scheduler.ts +74 -11
  52. package/src/runtime/scratchpad/engine.ts +26 -3
  53. package/src/runtime/settings-store.ts +22 -2
  54. package/src/runtime/stale-reconciler.ts +4 -1
  55. package/src/runtime/surface/tmux-provider.ts +24 -3
  56. package/src/runtime/task-health.ts +16 -1
  57. package/src/runtime/task-output-context.ts +143 -71
  58. package/src/runtime/task-packet.ts +70 -17
  59. package/src/runtime/task-runner/child-executor.ts +27 -0
  60. package/src/runtime/task-runner/output-splitter.ts +79 -16
  61. package/src/runtime/task-runner/post-execution.ts +75 -15
  62. package/src/runtime/task-runner/state-helpers.ts +92 -2
  63. package/src/runtime/usage-tracker.ts +11 -0
  64. package/src/runtime/verification/green-contract.ts +17 -0
  65. package/src/runtime/verification/verification-gates.ts +24 -6
  66. package/src/runtime/workspace-lock.ts +53 -0
  67. package/src/schema/team-tool-schema.ts +10 -4
  68. package/src/state/coordination/mailbox.ts +133 -38
  69. package/src/state/decision-ledger.ts +56 -8
  70. package/src/state/event-log/cursor.ts +14 -2
  71. package/src/state/event-log/event-log-rotation.ts +83 -5
  72. package/src/state/event-log/event-log.ts +181 -12
  73. package/src/state/stores/active-run-registry.ts +10 -3
  74. package/src/state/types.ts +9 -1
  75. package/src/ui/agents-jobs-browser.ts +22 -8
  76. package/src/ui/dashboard-panes/agents-pane.ts +5 -2
  77. package/src/ui/events-state-source.ts +67 -28
  78. package/src/ui/inline-panel/agent-pane.ts +13 -6
  79. package/src/ui/inline-panel/agent-transcript.ts +33 -21
  80. package/src/ui/inline-panel/agent-view-overlay.ts +18 -3
  81. package/src/ui/inline-panel/index.ts +21 -1
  82. package/src/ui/live-run-sidebar.ts +14 -3
  83. package/src/ui/overlays/mailbox-compose-overlay.ts +10 -4
  84. package/src/ui/powerbar-publisher.ts +6 -3
  85. package/src/ui/rail.ts +7 -1
  86. package/src/ui/render-scheduler.ts +53 -1
  87. package/src/ui/run-action-dispatcher.ts +68 -20
  88. package/src/ui/status-colors.ts +9 -0
  89. package/src/utils/env-filter.ts +8 -0
@@ -20,6 +20,8 @@ import { createRequire } from "node:module";
20
20
  import type { AgentConfig } from "../../agents/agent-config.ts";
21
21
  import { loadConfig } from "../../config/config.ts";
22
22
  import { effectiveRunConfig } from "../../extension/team-tool/config-patch.ts";
23
+ import { isTerminalTaskStatus } from "../../state/contracts.ts";
24
+ import { withRunLockSync } from "../../state/coordination/locks.ts";
23
25
  import { appendEventBuffered } from "../../state/event-log/event-log.ts";
24
26
  import { registerActiveRun, unregisterActiveRun } from "../../state/stores/active-run-registry.ts";
25
27
  import { collectRunMetrics } from "../../state/stores/run-metrics.ts";
@@ -28,14 +30,52 @@ import type { GoalLoopState, GoalLoopStatus, GoalVerdict, TeamRunManifest, TeamT
28
30
  import type { TeamConfig } from "../../teams/team-config.ts";
29
31
  import { logInternalError } from "../../utils/internal-error.ts";
30
32
  import type { WorkflowConfig } from "../../workflows/workflow-config.ts";
33
+ import { isRunLockBusyError } from "../broker/protocol/lock-busy.ts";
31
34
  import { resolveCrewRuntime } from "../model/runtime-resolver.ts";
32
35
  import { registryFromModelContext } from "../model/session-model.ts";
33
36
  import { executeTeamRun } from "../team-runner.ts";
34
37
  import { compareSnapshot, snapshotManifests } from "../verification/verification-integrity.ts";
35
- import { acquireWorkspaceLock, type WorkspaceLockHandle } from "../workspace-lock.ts";
38
+ import { acquireWorkspaceLock, refreshWorkspaceLockHeartbeat, type WorkspaceLockHandle } from "../workspace-lock.ts";
36
39
  import { bundleEvidence, evaluateGoal } from "./goal-evaluator.ts";
37
40
  import { GoalStore } from "./goal-state-store.ts";
38
41
 
42
+ /**
43
+ * D5 (DR-02 Wave 2 Cluster C): merge the turn's FINAL task states onto a
44
+ * FRESH tasks.json read. The turn owns exactly the terminal-evidence fields
45
+ * its executor wrote (status/finishedAt/resultArtifact/error/exitCode);
46
+ * everything else on the fresh task (heartbeat/lastSeenAt, claim, delegate
47
+ * state, usage…) may be NEWER than the turn's in-memory snapshot and must
48
+ * never be regressed by the post-turn persist.
49
+ *
50
+ * Terminal-preserve: a task that is ALREADY terminal in the fresh read (an
51
+ * external decision — operator cancel, crash-recovery, the async-died
52
+ * notifier) keeps its fresh state entirely; the turn must not resurrect or
53
+ * re-decide it (same guard class as heartbeat.ts R13-S1). Turn tasks missing
54
+ * from the fresh read are appended — the turn created them and the
55
+ * budget/audit reads this persist exists for expect them on disk.
56
+ */
57
+ export function mergeTurnFinalTasksOntoFresh(freshTasks: readonly TeamTaskState[], turnTasks: readonly TeamTaskState[]): TeamTaskState[] {
58
+ const freshById = new Map(freshTasks.map((task) => [task.id, task]));
59
+ const turnById = new Map(turnTasks.map((task) => [task.id, task]));
60
+ const merged = freshTasks.map((freshTask) => {
61
+ const turnTask = turnById.get(freshTask.id);
62
+ if (!turnTask) return freshTask;
63
+ if (isTerminalTaskStatus(freshTask.status)) return freshTask;
64
+ return {
65
+ ...freshTask,
66
+ status: turnTask.status,
67
+ ...(turnTask.finishedAt !== undefined ? { finishedAt: turnTask.finishedAt } : {}),
68
+ ...(turnTask.resultArtifact !== undefined ? { resultArtifact: turnTask.resultArtifact } : {}),
69
+ ...(turnTask.error !== undefined ? { error: turnTask.error } : {}),
70
+ ...(turnTask.exitCode !== undefined ? { exitCode: turnTask.exitCode } : {}),
71
+ };
72
+ });
73
+ for (const turnTask of turnTasks) {
74
+ if (!freshById.has(turnTask.id)) merged.push(turnTask);
75
+ }
76
+ return merged;
77
+ }
78
+
39
79
  /** Required minimal shape for the worker + agents discovery (P0 uses the goal's workerAgent). */
40
80
  export interface GoalLoopRuntimeDeps {
41
81
  /** Resolve the agent configs reachable from cwd (used for executeTeamRun's agents arg). */
@@ -538,6 +578,48 @@ export function deriveTranscriptPath(artifactsRoot: string, tasks: import("../..
538
578
  return undefined;
539
579
  }
540
580
 
581
+ /** Interval (ms) between workspace-lock heartbeat refreshes while a goal loop runs (WI-2 / DR-02 R4-3). 15s = 4x margin under the 60s staleness threshold. */
582
+ export const WORKSPACE_LOCK_HEARTBEAT_REFRESH_MS = 15_000;
583
+
584
+ export interface StartWorkspaceLockHeartbeatRefreshOptions {
585
+ /** Test injection: override the refresh interval (ms). */
586
+ refreshMs?: number;
587
+ /** Test injection: override the refresh implementation (default: refreshWorkspaceLockHeartbeat). */
588
+ refresh?: (lockPath: string, goalId: string) => boolean;
589
+ }
590
+
591
+ /**
592
+ * Start the periodic workspace-lock heartbeat refresh for a lock acquired by
593
+ * runGoalLoop (WI-2 / DR-02 R4-3).
594
+ *
595
+ * Without refresh, workspace-lock isLockStale clause (b) marks the lock stale
596
+ * after 60s REGARDLESS of holder pid liveness — a goal whose turn takes >60s
597
+ * gets its workspace lock force-stolen by a second goal on the same cwd
598
+ * (multi-goal clobber). This timer keeps the heartbeat young for the whole
599
+ * loop lifetime. It is unref()-ed (must not keep the background process
600
+ * alive) and the returned stop() MUST be called in the same finally that
601
+ * releases the lock. Refresh failures are logged as warn and swallowed — a
602
+ * heartbeat miss must never throw into the goal loop.
603
+ */
604
+ export function startWorkspaceLockHeartbeatRefresh(
605
+ handle: Pick<WorkspaceLockHandle, "lockPath" | "goalId">,
606
+ opts: StartWorkspaceLockHeartbeatRefreshOptions = {},
607
+ ): { stop: () => void } {
608
+ const refreshMs = opts.refreshMs ?? WORKSPACE_LOCK_HEARTBEAT_REFRESH_MS;
609
+ const refresh = opts.refresh ?? refreshWorkspaceLockHeartbeat;
610
+ const timer = setInterval(() => {
611
+ try {
612
+ refresh(handle.lockPath, handle.goalId);
613
+ } catch (error) {
614
+ // Never throw into the loop (interval callbacks crashing node:test aside,
615
+ // an escaping throw here would kill the background goal process).
616
+ logInternalError("goal-loop.workspaceLockHeartbeat", error, `goalId=${handle.goalId} lockPath=${handle.lockPath}`, "warn");
617
+ }
618
+ }, refreshMs);
619
+ timer.unref(); // the refresh timer must not keep the detached goal-loop process alive
620
+ return { stop: () => clearInterval(timer) };
621
+ }
622
+
541
623
  /**
542
624
  * Run the autonomous goal loop. Returns {manifest, tasks, goalState} — the OUTER
543
625
  * manifest is the synthetic goal-loop manifest (runKind:"goal-loop"); per-turn
@@ -575,10 +657,18 @@ export async function runGoalLoop(input: RunGoalLoopInput): Promise<RunGoalLoopR
575
657
  // the background-process boundary. `goal start` / `goal resume` pre-check via isWorkspaceBusy
576
658
  // for a good error message; this acquisition is the authoritative claim.
577
659
  let workspaceLock: WorkspaceLockHandle | undefined;
660
+ // WI-2 (DR-02 R4-3): periodic heartbeat refresh for the lock above — stopped
661
+ // in the same finally that releases it (see startWorkspaceLockHeartbeatRefresh).
662
+ let lockHeartbeat: { stop: () => void } | undefined;
578
663
  try {
579
664
  workspaceLock = await acquireWorkspaceLock(goal.cwd, goal.goalId, {
580
665
  signal,
581
666
  });
667
+ // WI-2 (DR-02 R4-3): keep the heartbeat young for the whole loop lifetime —
668
+ // without this, isLockStale clause (b) force-steals the lock after 60s even
669
+ // though this pid is alive (multi-goal clobber). Started immediately after
670
+ // acquire; the timer is unref()-ed and never throws into the loop.
671
+ lockHeartbeat = startWorkspaceLockHeartbeatRefresh(workspaceLock);
582
672
  } catch (error) {
583
673
  logInternalError("goal-loop.workspaceLock", error, `goalId=${goal.goalId} cwd=${goal.cwd}`);
584
674
  goal = safeSetStatus(store, goal.goalId, "blocked", goal, eventsPath);
@@ -753,10 +843,41 @@ export async function runGoalLoop(input: RunGoalLoopInput): Promise<RunGoalLoopR
753
843
  }
754
844
 
755
845
  // Persist final task states for budget/audit reads.
846
+ // D5 (DR-02 Wave 2 Cluster C): the bare saveRunTasks(turnResult.tasks) was
847
+ // an UNLOCKED write of the executor's in-memory snapshot (goal-loop-runner.ts
848
+ // ":757") — it clobbered any NEWER state written under the run lock between
849
+ // the executor's last persist and this save (worker heartbeats/lastSeenAt,
850
+ // task claims, delegate statuses). Now a run-locked RMW: fresh re-read inside
851
+ // the lock, then a field-level merge — the turn re-decides only the fields it
852
+ // owns and never regresses fresh heartbeat/claim/delegate data.
756
853
  try {
757
- saveRunTasks(turnResult.manifest, turnResult.tasks);
854
+ withRunLockSync(turnResult.manifest, () => {
855
+ const fresh = loadRunManifestById(turnResult.manifest.cwd, turnResult.manifest.runId);
856
+ if (!fresh) {
857
+ logInternalError(
858
+ "goal-loop.saveTurnTasks",
859
+ new Error(`turn run '${turnResult.manifest.runId}' vanished before the final task persist — skipped`),
860
+ "best-effort",
861
+ "warn",
862
+ );
863
+ return;
864
+ }
865
+ saveRunTasks(fresh.manifest, mergeTurnFinalTasksOntoFresh(fresh.tasks, turnResult.tasks));
866
+ });
758
867
  } catch (error) {
759
- logInternalError("goal-loop.saveTurnTasks", error, `turnRunId=${created.manifest.runId}`);
868
+ if (isRunLockBusyError(error)) {
869
+ // Best-effort persist: the executor already persisted terminal task
870
+ // states during the turn — this merge is belt-and-suspenders for
871
+ // budget/audit reads, so contention skips it with a warn.
872
+ logInternalError(
873
+ "goal-loop.saveTurnTasks",
874
+ error,
875
+ `turnRunId=${created.manifest.runId} run lock busy — skipped final task merge`,
876
+ "warn",
877
+ );
878
+ } else {
879
+ logInternalError("goal-loop.saveTurnTasks", error, `turnRunId=${created.manifest.runId}`);
880
+ }
760
881
  }
761
882
 
762
883
  // GL-1: if the turn ended in blocked/failed status, stop the loop without spawning the judge.
@@ -908,6 +1029,12 @@ export async function runGoalLoop(input: RunGoalLoopInput): Promise<RunGoalLoopR
908
1029
  }).catch((e) => logInternalError("goal-loop.buffered", e, "type=goal.loop_error"));
909
1030
  goal = safeSetStatus(store, goal.goalId, "blocked", goal, eventsPath);
910
1031
  } finally {
1032
+ // WI-2: stop the heartbeat refresh BEFORE releasing the lock it was keeping alive.
1033
+ try {
1034
+ lockHeartbeat?.stop();
1035
+ } catch {
1036
+ /* best-effort */
1037
+ }
911
1038
  // P1g: release the workspace lock (held since loop start).
912
1039
  try {
913
1040
  workspaceLock?.release();
@@ -231,6 +231,11 @@ export class HeartbeatWatcher {
231
231
  if (!activeKeys.has(key) && ts < cutoff) {
232
232
  this.lastLevel.delete(key);
233
233
  this.consecutiveDead.delete(key);
234
+ // M-8 (2026-10-16): also drop the deadletter cooldown marker —
235
+ // before this the map leaked one entry per vanished run:taskId key
236
+ // and a stale cooldown timestamp could suppress a legitimate
237
+ // re-trigger after the same task reappeared.
238
+ this.lastDeadletterTriggerAt.delete(key);
234
239
  this.lastSeen.delete(key);
235
240
  }
236
241
  }
@@ -242,5 +247,7 @@ export class HeartbeatWatcher {
242
247
  this.lastLevel.clear();
243
248
  this.consecutiveDead.clear();
244
249
  this.lastSeen.clear();
250
+ // M-8: clear the deadletter cooldown map too (was leaked before).
251
+ this.lastDeadletterTriggerAt.clear();
245
252
  }
246
253
  }
@@ -908,7 +908,9 @@ export function admitEmbeddedSession(input: {
908
908
  });
909
909
  // Raced against the prompt; a pre-race rejection must not surface as an
910
910
  // unhandled rejection (which would trip our own process guard).
911
- escapeSignal.catch(() => {});
911
+ escapeSignal.catch(() => {
912
+ /* intentional UR-guard — see comment above */
913
+ });
912
914
  let tornDown = false;
913
915
  registerEmbeddedSessionGuard(
914
916
  input.agentId,
@@ -868,6 +868,49 @@ export async function probeLiveSessionRuntime(): Promise<LiveSessionUnavailableR
868
868
  };
869
869
  }
870
870
 
871
+ // DR-02 C1: the per-agent control-request dedupe Set previously grew for
872
+ // the whole run lifetime — fed by BOTH the 500ms/1000ms control poll and the
873
+ // realtime subscription, with no eviction. Bounded FIFO eviction: a Set
874
+ // preserves insertion order, so deleting the first-added id evicts the OLDEST
875
+ // request id (mirrors the NotificationRouter.SEEN_MAP_MAX_SIZE bounded-seen
876
+ // pattern). Worst case after an eviction is one redelivery of a very old
877
+ // control request; the consumers (steer / follow-up) are idempotent by design.
878
+ export const MAX_SEEN_CONTROL_REQUEST_IDS = 10_000;
879
+
880
+ class BoundedSeenControlRequestIdSet extends Set<string> {
881
+ override add(id: string): this {
882
+ super.add(id);
883
+ while (this.size > MAX_SEEN_CONTROL_REQUEST_IDS) {
884
+ const oldest = this.values().next().value;
885
+ if (oldest === undefined) break;
886
+ this.delete(oldest);
887
+ }
888
+ return this;
889
+ }
890
+ }
891
+
892
+ /** @internal — test seam for the C1 cap (see live-agent-control-dedupe tests). */
893
+ export const __test__BoundedSeenControlRequestIdSet = BoundedSeenControlRequestIdSet;
894
+
895
+ /**
896
+ * R6-9 (DR-03 round-6): best-effort usage extraction for the fault/watchdog
897
+ * escape returns. The success return reports usageFromSessionStats(session),
898
+ * but the fault/watchdog catch paths previously carried NO usage — whenever
899
+ * embeddedWorkers routed work to the embedded tier and the task faulted, the
900
+ * burned turns vanished from the budget (under-count). The session may be
901
+ * disposed or partially unusable after the fault, so stats collection is
902
+ * guarded — it can never turn a fault into a crash. Returns undefined when no
903
+ * usable stats surface exists.
904
+ */
905
+ function faultPathUsage(session: LiveSessionLike | undefined): UsageState | undefined {
906
+ try {
907
+ return usageFromSessionStats(session);
908
+ } catch (error) {
909
+ logInternalError("live-session.fault-usage-stats", error, "session stats unavailable on fault path");
910
+ return undefined;
911
+ }
912
+ }
913
+
871
914
  export async function runLiveSessionTask(input: LiveSessionSpawnInput): Promise<LiveSessionRunResult> {
872
915
  // Cold-start race fix: ensure the hot module graph is warm before touching
873
916
  // any module. Under tsx, concurrent first-imports race module-record
@@ -1406,7 +1449,7 @@ export async function runLiveSessionTask(input: LiveSessionSpawnInput): Promise<
1406
1449
  registerLiveAgentModel(agentId, resolvedModelRef ?? "");
1407
1450
  streamOut = createStreamingOutput(input.manifest, input.task.id);
1408
1451
  let controlCursor: LiveAgentControlCursor = { offset: 0 };
1409
- const seenControlRequestIds = new Set<string>();
1452
+ const seenControlRequestIds: Set<string> = new BoundedSeenControlRequestIdSet();
1410
1453
  let controlBusy = false;
1411
1454
  const pollControl = async () => {
1412
1455
  if (!isCurrent() || controlBusy || !session) return;
@@ -1793,12 +1836,16 @@ export async function runLiveSessionTask(input: LiveSessionSpawnInput): Promise<
1793
1836
  },
1794
1837
  });
1795
1838
  updateLiveAgentStatus(agentId, "failed");
1839
+ // R6-9: turns burned before the watchdog fired still count — capture the
1840
+ // partial usage (guarded; session was disposed by the watchdog timer).
1841
+ const watchdogUsage = faultPathUsage(session);
1796
1842
  return {
1797
1843
  available: true,
1798
1844
  exitCode: 1,
1799
1845
  stdout: stdoutTail.value().trim(),
1800
1846
  stderr: error.message,
1801
1847
  jsonEvents,
1848
+ usage: watchdogUsage,
1802
1849
  error: error.message,
1803
1850
  ...(embedded ? { fault: { kind: "turn-watchdog" as const, detail: error.message, countAsFault: true } } : {}),
1804
1851
  };
@@ -1820,12 +1867,15 @@ export async function runLiveSessionTask(input: LiveSessionSpawnInput): Promise<
1820
1867
  // drop-reference (synchronous — never awaits the stuck abort): drain +
1821
1868
  // terminate + dispose + UNREGISTER, so the hung session holds no slot.
1822
1869
  dropLiveAgentReference(agentId);
1870
+ // R6-9: partial usage from the turns completed before the tool hung.
1871
+ const hungUsage = faultPathUsage(session);
1823
1872
  return {
1824
1873
  available: true,
1825
1874
  exitCode: 1,
1826
1875
  stdout: stdoutTail.value().trim(),
1827
1876
  stderr: error.message,
1828
1877
  jsonEvents,
1878
+ usage: hungUsage,
1829
1879
  error: error.message,
1830
1880
  ...(embedded ? { fault: { kind: "hung-tool" as const, timeoutMs: error.timeoutMs, countAsFault: true } } : {}),
1831
1881
  };
@@ -1861,6 +1911,9 @@ export async function runLiveSessionTask(input: LiveSessionSpawnInput): Promise<
1861
1911
  stdout: stdoutTail.value().trim(),
1862
1912
  stderr: message,
1863
1913
  jsonEvents,
1914
+ // R6-9: generic fault path — partial usage from completed turns so the
1915
+ // budget sees the real spend even when the task failed.
1916
+ usage: faultPathUsage(session),
1864
1917
  error: message,
1865
1918
  ...(embeddedCatchFault ? { fault: embeddedCatchFault } : {}),
1866
1919
  };
@@ -606,7 +606,29 @@ export function resolveModelFallbackPolicy(
606
606
  } else {
607
607
  maxAutoFallbacks = config?.maxAutoFallbacks;
608
608
  }
609
- const order = (env.PI_CREW_MODEL_FALLBACK_ORDER as "parentFirst" | "asIs" | undefined) ?? config?.order;
609
+ // J5 (DR-03 round-6): validate the order env against the two legal values.
610
+ // The previous unvalidated cast let arbitrary strings flow into
611
+ // orderAutoFallbacks' `order === "asIs"` checks — any typo (e.g.
612
+ // "parent-first") silently behaved as parentFirst. Invalid → warn + fall
613
+ // back to config/default, mirroring the PI_CREW_MAX_AUTO_FALLBACKS
614
+ // clamp-warn sibling pattern above.
615
+ const envOrder = env.PI_CREW_MODEL_FALLBACK_ORDER;
616
+ let order: "parentFirst" | "asIs" | undefined;
617
+ if (envOrder) {
618
+ if (envOrder === "parentFirst" || envOrder === "asIs") {
619
+ order = envOrder;
620
+ } else {
621
+ logInternalError(
622
+ "model-fallback.model-fallback-order-invalid",
623
+ undefined,
624
+ `PI_CREW_MODEL_FALLBACK_ORDER="${envOrder}" is invalid (expected "parentFirst" or "asIs"), falling back to config`,
625
+ "warn",
626
+ );
627
+ order = config?.order;
628
+ }
629
+ } else {
630
+ order = config?.order;
631
+ }
610
632
  const requireCredentials =
611
633
  env.PI_CREW_MODEL_REQUIRE_CREDENTIALS === "1"
612
634
  ? true
@@ -20,6 +20,14 @@ export interface ParsedPiJsonOutput {
20
20
  * "429 ... overloaded"). Used to detect runs that exited 0 but produced
21
21
  * nothing because the model was rate-limited — see task-runner 429 fix. */
22
22
  errorMessages?: string[];
23
+ /** DR-01 (deep review 2026-10-14): true when the LAST assistant
24
+ * `message_end` event in the stream carried no non-empty text — the
25
+ * session settled without a final answer. Upstream 429 exhaustion
26
+ * manifests as a chain of EMPTY assistant message_end events (content [],
27
+ * usage 0) between auto-retry cycles — never as stopReason:"error" — so
28
+ * `errorMessages` stays empty and `finalText` keeps an early pre-outage
29
+ * sentence. undefined when no assistant message_end was parsed at all. */
30
+ settledWithoutAnswer?: boolean;
23
31
  }
24
32
 
25
33
  function asRecord(value: unknown): Record<string, unknown> | undefined {
@@ -82,12 +90,23 @@ export function extractText(value: unknown): string[] {
82
90
  return text.filter((entry) => entry.trim().length > 0);
83
91
  }
84
92
 
93
+ /** True for `message_end` events whose message is assistant-role. Feeds the
94
+ * settledWithoutAnswer flag (DR-01): see ParsedPiJsonOutput.settledWithoutAnswer. */
95
+ function isAssistantMessageEnd(event: unknown): boolean {
96
+ const obj = asRecord(event);
97
+ if (obj?.type !== "message_end") return false;
98
+ const message = asRecord(obj.message);
99
+ return message?.role === "assistant";
100
+ }
101
+
85
102
  export function parsePiJsonOutput(stdout: string): ParsedPiJsonOutput {
86
103
  let jsonEvents = 0;
87
104
  const textEvents: string[] = [];
88
105
  const patches: string[] = [];
89
106
  const errorMessages: string[] = [];
90
107
  let usage: ParsedPiUsage | undefined;
108
+ let sawAssistantMessageEnd = false;
109
+ let lastAssistantMessageHadText = false;
91
110
  for (const line of stdout.split("\n")) {
92
111
  const trimmed = line.trim();
93
112
  if (!trimmed) continue;
@@ -98,7 +117,12 @@ export function parsePiJsonOutput(stdout: string): ParsedPiJsonOutput {
98
117
  continue;
99
118
  }
100
119
  jsonEvents++;
101
- textEvents.push(...extractText(event));
120
+ const eventTexts = extractText(event);
121
+ textEvents.push(...eventTexts);
122
+ if (isAssistantMessageEnd(event)) {
123
+ sawAssistantMessageEnd = true;
124
+ lastAssistantMessageHadText = eventTexts.length > 0;
125
+ }
102
126
  // Extract unified patches from tool_result events
103
127
  extractPatch(event, patches);
104
128
  // Extract provider/model error messages from message_end events (429 fix).
@@ -114,6 +138,7 @@ export function parsePiJsonOutput(stdout: string): ParsedPiJsonOutput {
114
138
  usage,
115
139
  patches: patches.length > 0 ? patches : undefined,
116
140
  errorMessages: errorMessages.length > 0 ? errorMessages : undefined,
141
+ settledWithoutAnswer: sawAssistantMessageEnd ? !lastAssistantMessageHadText : undefined,
117
142
  };
118
143
  }
119
144
 
@@ -104,6 +104,24 @@ export function evaluateCrewPolicy(input: PolicyEngineInput): PolicyDecision[] {
104
104
  }
105
105
  }
106
106
 
107
+ // R4-4 (DR-02 wave 2): needs_attention is a TERMINAL-without-result status
108
+ // (surface-lost / no-yield paths in post-execution) and the scheduler counts
109
+ // it as done, so a run can otherwise complete green while a task never
110
+ // submitted its result. Surface it as a NON-blocking notify decision —
111
+ // detection-first: no status flip, no block, no retry (re-queue/recovery is
112
+ // owned by the surface-degrade drain machinery, not the policy engine).
113
+ const needsAttentionTasks = input.tasks.filter((t) => t.status === "needs_attention");
114
+ if (needsAttentionTasks.length > 0) {
115
+ const ids = needsAttentionTasks.map((t) => t.id).join(", ");
116
+ decisions.push(
117
+ decision(
118
+ "notify",
119
+ "task_needs_attention",
120
+ `${needsAttentionTasks.length} task(s) finished needs_attention without submitting a result: ${ids}.`,
121
+ ),
122
+ );
123
+ }
124
+
107
125
  if (decisions.length === 0 && input.tasks.length > 0 && input.tasks.every((task) => task.status === "completed")) {
108
126
  decisions.push(decision("closeout", "run_complete", "All tasks completed and no policy blockers were found."));
109
127
  }
@@ -60,6 +60,13 @@ export interface ZombieSubagent {
60
60
  surfacePaneId?: string;
61
61
  rssKb: number;
62
62
  elapsedSec: number | undefined;
63
+ /** M-7 (DR-04 round-7): /proc/<pid>/stat starttime — clock ticks since boot.
64
+ * Stable pid identity: unlike the pid number, starttime changes when the OS
65
+ * recycles a dead pid for a NEW process, so kill-time re-checks can detect
66
+ * PID reuse between scan and kill. */
67
+ starttime?: number;
68
+ /** M-7: wall-clock start time (ISO) derived from starttime + /proc/uptime. */
69
+ startedAt?: string;
63
70
  cmd: string;
64
71
  }
65
72
 
@@ -106,8 +113,8 @@ function readProcEnviron(pid: number): Record<string, string> {
106
113
  }
107
114
  }
108
115
 
109
- /** Read /proc/<pid>/stat to get ppid + elapsed. Returns undefined if unreadable. */
110
- function readProcStat(pid: number): { ppid: number; elapsedSec: number | undefined } | undefined {
116
+ /** Read /proc/<pid>/stat to get ppid + starttime + elapsed. Returns undefined if unreadable. */
117
+ function readProcStat(pid: number): { ppid: number; starttime: number | undefined; elapsedSec: number | undefined } | undefined {
111
118
  try {
112
119
  const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf-8");
113
120
  // stat format: pid (comm) state ppid ... starttime ...
@@ -117,14 +124,38 @@ function readProcStat(pid: number): { ppid: number; elapsedSec: number | undefin
117
124
  if (!rest) return undefined;
118
125
  const ppid = Number.parseInt(rest[PROC_STAT_PPID_INDEX] ?? "", 10);
119
126
  const starttimeTicksRaw = Number.parseInt(rest[PROC_STAT_STARTTIME_INDEX] ?? "", 10);
120
- const starttimeTicks = Number.isFinite(starttimeTicksRaw) ? starttimeTicksRaw : undefined;
121
- const elapsedSec = computeElapsedSec(starttimeTicks);
122
- return { ppid: Number.isFinite(ppid) ? ppid : 0, elapsedSec };
127
+ const starttime = Number.isFinite(starttimeTicksRaw) ? starttimeTicksRaw : undefined;
128
+ const elapsedSec = computeElapsedSec(starttime);
129
+ return { ppid: Number.isFinite(ppid) ? ppid : 0, starttime, elapsedSec };
123
130
  } catch {
124
131
  return undefined;
125
132
  }
126
133
  }
127
134
 
135
+ /** /proc/uptime first field (seconds since boot); undefined when unreadable. */
136
+ function readUptimeSec(): number | undefined {
137
+ try {
138
+ const uptimeRaw = fs.readFileSync("/proc/uptime", "utf-8");
139
+ const uptimeSec = Number.parseFloat(uptimeRaw.split(" ")[0] ?? "");
140
+ return Number.isFinite(uptimeSec) ? uptimeSec : undefined;
141
+ } catch {
142
+ return undefined;
143
+ }
144
+ }
145
+
146
+ /**
147
+ * M-7 (DR-04 round-7): wall-clock start time of a pid from its starttime
148
+ * ticks — boot time = now - uptime, so startedAt = boot + starttime/CLK_TCK.
149
+ * Best-effort ISO string; undefined when /proc/uptime is unavailable.
150
+ */
151
+ function computeStartedAtIso(starttime: number | undefined): string | undefined {
152
+ if (starttime === undefined || !Number.isFinite(starttime)) return undefined;
153
+ const uptimeSec = readUptimeSec();
154
+ if (uptimeSec === undefined) return undefined;
155
+ const bootMs = Date.now() - uptimeSec * 1000;
156
+ return new Date(bootMs + (starttime / 100) * 1000).toISOString();
157
+ }
158
+
128
159
  function computeElapsedSec(starttimeTicks: number | undefined): number | undefined {
129
160
  if (starttimeTicks === undefined || !Number.isFinite(starttimeTicks)) return undefined;
130
161
  try {
@@ -132,10 +163,8 @@ function computeElapsedSec(starttimeTicks: number | undefined): number | undefin
132
163
  // portably from Node requires a native addon; hardcoding 100 matches every
133
164
  // mainstream Linux distro and keeps this dependency-free.
134
165
  const ticksPerSec = 100;
135
- // /proc/uptime: first field is seconds since boot.
136
- const uptimeRaw = fs.readFileSync("/proc/uptime", "utf-8");
137
- const uptimeSec = Number.parseFloat(uptimeRaw.split(" ")[0] ?? "");
138
- if (!Number.isFinite(uptimeSec)) return undefined;
166
+ const uptimeSec = readUptimeSec();
167
+ if (uptimeSec === undefined) return undefined;
139
168
  // starttime (ticks since boot) → process age in seconds = uptime - starttime/ticksPerSec.
140
169
  const startAgeSec = starttimeTicks / ticksPerSec;
141
170
  return Math.max(0, uptimeSec - startAgeSec);
@@ -150,8 +179,12 @@ function isPidAlive(pid: number): boolean {
150
179
  // process.kill(pid, 0) throws if the pid is not alive (or not ours).
151
180
  process.kill(pid, 0);
152
181
  return true;
153
- } catch {
154
- return false;
182
+ } catch (error) {
183
+ // R7-A2 (2026-10-17): only ESRCH/ENOENT mean the process is gone.
184
+ // EPERM means the process is alive but protected (different security
185
+ // context); treat as alive — mirror of active-run-registry.ts fix.
186
+ const code = (error as NodeJS.ErrnoException).code;
187
+ return code !== "ESRCH" && code !== "ENOENT";
155
188
  }
156
189
  }
157
190
 
@@ -271,6 +304,10 @@ export function scanZombieSubagents(): ZombieScanResult {
271
304
  surfacePaneId: environ.PI_CREW_SURFACE_PANE || undefined,
272
305
  rssKb: readProcRssKb(pid),
273
306
  elapsedSec: stat?.elapsedSec,
307
+ // M-7: starttime (ticks) + derived wall-clock start — pid identity pin
308
+ // for kill-time PID-reuse checks and prompt-kill triage.
309
+ starttime: stat?.starttime,
310
+ startedAt: computeStartedAtIso(stat?.starttime),
274
311
  cmd: readProcCmdline(pid),
275
312
  };
276
313
 
@@ -392,9 +429,13 @@ export function formatZombieReport(scan: ZombieScanResult): string {
392
429
 
393
430
  if (scan.zombies.length > 0) {
394
431
  lines.push(`### Zombies — parent dead (${scan.zombies.length})`);
395
- lines.push("These sub-agents are orphaned. Safe to kill after review:");
432
+ lines.push("These sub-agents are orphaned. Safe to kill, but do it PROMPTLY after this");
433
+ lines.push("scan: an unreviewed delay re-opens the PID-reuse window (the OS can recycle");
434
+ lines.push("a dead zombie's PID for an unrelated process). Before killing, re-check that");
435
+ lines.push("the PID's STARTED column below still matches — a newer starttime means the");
436
+ lines.push("process you are about to kill is NOT the zombie that was scanned.");
396
437
  lines.push("");
397
- lines.push(" PID PARENT RSS ROLE SURFACE CMD");
438
+ lines.push(" PID PARENT AGE STARTED RSS ROLE SURFACE CMD");
398
439
  for (const z of scan.zombies) {
399
440
  lines.push(formatZombieRow(z));
400
441
  }
@@ -405,7 +446,7 @@ export function formatZombieReport(scan: ZombieScanResult): string {
405
446
  lines.push(`### Live — parent still running (${scan.live.length})`);
406
447
  lines.push("NOT zombies. Do not kill (parent PID is alive and may still reap them).");
407
448
  lines.push("");
408
- lines.push(" PID PARENT RSS ROLE SURFACE CMD");
449
+ lines.push(" PID PARENT AGE STARTED RSS ROLE SURFACE CMD");
409
450
  for (const l of scan.live) {
410
451
  lines.push(formatZombieRow(l));
411
452
  }
@@ -435,7 +476,9 @@ export function formatZombieReport(scan: ZombieScanResult): string {
435
476
  lines.push("");
436
477
  }
437
478
 
438
- lines.push("To kill a zombie: `kill <PID>` (the OS will reap it). This tool never kills.");
479
+ lines.push("To kill a zombie: `kill <PID>` (the OS will reap it). Kill soon after the");
480
+ lines.push("scan — a PID listed here can be reused by an unrelated process if you wait;");
481
+ lines.push("verify its starttime/age first. This tool never kills.");
439
482
  return lines.join("\n");
440
483
  }
441
484
 
@@ -443,7 +486,30 @@ export function formatZombieReport(scan: ZombieScanResult): string {
443
486
  function formatZombieRow(z: ZombieSubagent): string {
444
487
  // SURFACE = "kind:paneId" (T12) — the pane doctor closes when orphaned.
445
488
  const surface = z.surface ? `${z.surface}:${z.surfacePaneId ?? "?"}` : "-";
446
- return ` ${String(z.pid).padEnd(9)}${String(z.crewParentPid).padEnd(8)}${formatRss(z.rssKb).padEnd(10)}${(z.role ?? "?").padEnd(14)}${surface.padEnd(16)}${z.cmd.slice(0, 44)}`;
489
+ // M-7: AGE + STARTED — pid identity + triage (old zombies = long-leaked;
490
+ // STARTED mismatch at kill time = PID was reused, do NOT kill).
491
+ return (
492
+ ` ${String(z.pid).padEnd(9)}${String(z.crewParentPid).padEnd(8)}${formatAge(z.elapsedSec).padEnd(9)}` +
493
+ `${formatStartedAt(z.startedAt).padEnd(21)}${formatRss(z.rssKb).padEnd(10)}${(z.role ?? "?").padEnd(14)}` +
494
+ `${surface.padEnd(16)}${z.cmd.slice(0, 28)}`
495
+ );
496
+ }
497
+
498
+ /** M-7: age as a compact human duration ("37s", "5m", "3h", "2d"). */
499
+ function formatAge(sec: number | undefined): string {
500
+ if (sec === undefined || !Number.isFinite(sec) || sec < 0) return "-";
501
+ if (sec < 60) return `${Math.floor(sec)}s`;
502
+ if (sec < 3600) return `${Math.floor(sec / 60)}m`;
503
+ if (sec < 86400) return `${Math.floor(sec / 3600)}h`;
504
+ return `${Math.floor(sec / 86400)}d`;
505
+ }
506
+
507
+ /** M-7: ISO startedAt → "MM-DD HH:MM:SS" (UTC, no year — scan-time triage). */
508
+ function formatStartedAt(iso: string | undefined): string {
509
+ if (!iso) return "-";
510
+ const d = new Date(iso);
511
+ if (Number.isNaN(d.getTime())) return "-";
512
+ return d.toISOString().slice(5, 19).replace("T", " ");
447
513
  }
448
514
 
449
515
  /** One table row for the R3-16 foreign section — state carries the triage flags. */
@@ -469,4 +535,6 @@ export const __test = {
469
535
  looksLikePiBinary,
470
536
  isHeadlessArgv,
471
537
  collectAncestorPids,
538
+ // M-7: starttime helpers (ticks → wall-clock start).
539
+ computeStartedAtIso,
472
540
  };
@@ -140,7 +140,12 @@ export function isDisplayActiveRun(run: TeamRunManifest, agents: CrewAgentRecord
140
140
  if (Number.isFinite(updatedAt) && now - updatedAt > STALE_ACTIVE_RUN_MS) return false;
141
141
  }
142
142
  // Grace period: show completed runs for a few seconds so users see the result.
143
- if (run.status === "completed" || run.status === "failed" || run.status === "cancelled") {
143
+ // R5-4 (DR-02 wave 2): "blocked" is terminal (isFinishedRunStatus) but NOT an
144
+ // error the user caused — without the grace branch it fell through to
145
+ // `!isActiveRunStatus` and vanished from the widget instantly while "failed"
146
+ // lingered 10min. Blocked now gets the same ERROR_VISIBILITY_GRACE_MS so the
147
+ // block reason stays inspectable.
148
+ if (run.status === "completed" || run.status === "failed" || run.status === "cancelled" || run.status === "blocked") {
144
149
  const grace = run.status === "completed" ? COMPLETED_VISIBILITY_GRACE_MS : ERROR_VISIBILITY_GRACE_MS;
145
150
  const lastAgentActivity = agents.reduce<number>((max, agent) => {
146
151
  const ts = agent.completedAt ?? agent.startedAt;