pi-crew 0.11.11 → 0.11.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/CHANGELOG.md +102 -0
  2. package/dist/index.mjs +551 -519
  3. package/package.json +1 -1
  4. package/src/config/config-validation.ts +9 -0
  5. package/src/config/defaults.ts +7 -0
  6. package/src/config/env-vars.ts +24 -0
  7. package/src/config/types.ts +11 -0
  8. package/src/extension/registration/tool-registration.ts +7 -0
  9. package/src/extension/registration/verify-gate-tool.ts +263 -0
  10. package/src/extension/team-tool/run-deadline.ts +22 -1
  11. package/src/extension/team-tool/run.ts +38 -2
  12. package/src/extension/team-tool-types.ts +4 -0
  13. package/src/extension/team-tool.ts +31 -3
  14. package/src/prompt/inbox-poll.ts +141 -3
  15. package/src/runtime/broker/crew-broker.ts +48 -2
  16. package/src/runtime/broker/protocol/lock-busy.ts +47 -1
  17. package/src/runtime/broker/protocol/request-parsers.ts +8 -1
  18. package/src/runtime/budget-enforcement.ts +36 -20
  19. package/src/runtime/child-pi/child-pi-constants.ts +24 -0
  20. package/src/runtime/child-pi/child-pi-kill.ts +59 -2
  21. package/src/runtime/child-pi/child-pi-rpc.ts +1021 -0
  22. package/src/runtime/child-pi/child-pi-steering.ts +10 -8
  23. package/src/runtime/child-pi/child-pi-streams.ts +69 -19
  24. package/src/runtime/child-pi/child-pi.ts +29 -30
  25. package/src/runtime/child-pi/owned-process.ts +434 -0
  26. package/src/runtime/child-pi/postmortem-registry.ts +134 -0
  27. package/src/runtime/classifier/ambient-noise-gate.ts +126 -0
  28. package/src/runtime/classifier/classifier-service.ts +277 -61
  29. package/src/runtime/dispatch-batch.ts +32 -0
  30. package/src/runtime/live-session/live-agent-control.ts +2 -0
  31. package/src/runtime/live-session/live-agent-manager.ts +72 -2
  32. package/src/runtime/live-session/live-session-runtime.ts +435 -40
  33. package/src/runtime/merge-loop.ts +63 -35
  34. package/src/runtime/recovery/crash-recovery.ts +141 -3
  35. package/src/runtime/recovery/retry-executor.ts +29 -0
  36. package/src/runtime/run-tracker.ts +28 -0
  37. package/src/runtime/stale-reconciler.ts +36 -1
  38. package/src/runtime/task-runner.ts +103 -0
  39. package/src/runtime/verification/classifier-pre-gate.ts +262 -0
  40. package/src/runtime/verification/verification-gates.ts +5 -1
  41. package/src/runtime/verification/verify-gate.ts +394 -0
  42. package/src/schema/config-schema.ts +13 -0
  43. package/src/state/contracts.ts +18 -0
  44. package/src/state/coordination/mailbox.ts +203 -55
  45. package/src/state/stores/sqlite-run-state.ts +378 -0
  46. package/src/state/stores/state-store.ts +134 -6
  47. package/src/ui/agents-jobs-browser.ts +154 -8
  48. package/src/ui/events-state-source.ts +358 -0
  49. package/src/ui/run-snapshot-cache.ts +87 -3
  50. package/src/ui/widget/widget-model.ts +13 -2
@@ -13,10 +13,11 @@
13
13
  * keeping it here avoids a merge-loop ↔ finalize-run import cycle.
14
14
  */
15
15
  import { flushPendingAtomicWrites } from "../state/atomic-write.ts";
16
- import { withRunLock } from "../state/coordination/locks.ts";
16
+ import { appendEventAsync } from "../state/event-log/event-log.ts";
17
17
  import { loadRunManifestById, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/stores/state-store.ts";
18
18
  import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
19
19
  import { classifyFatalFsError } from "../utils/fs-errno.ts";
20
+ import { MERGE_LOCK_BUSY_RETRY_DELAYS_MS, withRunLockBusyRetryAsync } from "./broker/protocol/lock-busy.ts";
20
21
  import { cancelNonTerminalTasks } from "./dispatch-batch.ts";
21
22
  import { mergeTaskUpdatesPreservingTerminal } from "./merge-gate.ts";
22
23
  import type { SchedulerContext, SchedulerDecision } from "./scheduler-context.ts";
@@ -86,40 +87,67 @@ export async function mergeUnitResult(ctx: SchedulerContext): Promise<SchedulerD
86
87
  // not the arbitrary order in which mapConcurrent returned results.
87
88
  // Read committed manifest from disk inside the lock so artifact merge is based
88
89
  // on committed state, not in-memory state that may differ from disk.
89
- const mergeResult = await withRunLock(ctx.manifest, async () => {
90
- // NEW-D1: flush any pending coalesced atomic writes before reading from
91
- // disk. Without this, a worker's async manifest save (coalesced by
92
- // atomic-write) may not be committed yet, causing a lost-update on the
93
- // merge read. flushPendingAtomicWrites forces all queued writes to disk.
94
- flushPendingAtomicWrites();
95
- const disk = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
96
- const diskManifest = disk?.manifest ?? ctx.manifest;
97
- const diskArtifacts = diskManifest.artifacts;
98
- const reconciledArtifacts = mergeArtifacts([...diskArtifacts, ...validResults.map((item) => item.manifest.artifacts)].flat());
99
- // R15-2: only force "running" when the disk manifest is NON-terminal
100
- // (queued/planning/running/blocked). If the disk status is already terminal
101
- // (cancelled/failed/completed — an external cancel or reconciler write
102
- // landing during the batch), PRESERVE that terminal status: forcing
103
- // "running" would legally erase it (contracts.ts allows cancelled/failed/
104
- // completed → running) and the loop would never observe the disk-terminal
105
- // (CANCEL-1/CANCEL-2 only catch worker-reported cancel or signal abort).
106
- const mergedBase = { ...diskManifest, artifacts: reconciledArtifacts };
107
- const resultManifest = isRunTerminalPreserved(diskManifest.status)
108
- ? mergedBase
109
- : updateRunStatus(mergedBase, "running", "Merged task updates from parallel batch.");
110
- // CANCEL-1: use the freshly-loaded disk tasks as the merge base instead
111
- // of the in-memory `tasks` closure variable. The in-memory tasks reflect
112
- // only team-runner's view; an external cancel (handleCancel, background
113
- // race with SIGTERM arriving after cancel wrote but before merge ran)
114
- // writes 'cancelled' to disk.tasks — using disk.tasks as base preserves
115
- // that cancellation through the merge instead of overwriting it with the
116
- // stale in-memory view. disk was loaded inside this lock, so it reflects
117
- // the freshest committed state.
118
- const resultTasks = mergeTaskUpdatesPreservingTerminal(disk?.tasks ?? ctx.tasks, validResults);
119
- await saveRunManifestAsync(resultManifest);
120
- await saveRunTasksAsync(resultManifest, resultTasks);
121
- return { resultManifest, resultTasks };
122
- });
90
+ // RELIABILITY FIX 2026-10-10 (run team_20261010100956 incident): the merge
91
+ // used a bare withRunLock. Under run.lock contention (a second writer —
92
+ // stale reconciler, resumed runner, or a slow ENOSPC-degraded writer —
93
+ // holding run.lock past the acquire retry budget) withRunLock throws the
94
+ // plain "Run 'run.lock' is locked by another operation." error, which
95
+ // escaped executeTeamRun's catch → rejectRunPromise → an UNHANDLED promise
96
+ // rejection (no waitForRun waiter in background-runner mode) → the runner's
97
+ // unhandledRejection guard killed the whole runner mid-run. The merge now
98
+ // retries transient contention with back-off (logging a run.lock_retry
99
+ // event per attempt); only after the bounded schedule is exhausted does
100
+ // the busy error propagate for the run-level handler to classify. Non-busy
101
+ // errors (real faults) keep their existing fatal path — this is NOT a
102
+ // swallow-all.
103
+ const mergeResult = await withRunLockBusyRetryAsync(
104
+ ctx.manifest,
105
+ MERGE_LOCK_BUSY_RETRY_DELAYS_MS,
106
+ async () => {
107
+ // NEW-D1: flush any pending coalesced atomic writes before reading from
108
+ // disk. Without this, a worker's async manifest save (coalesced by
109
+ // atomic-write) may not be committed yet, causing a lost-update on the
110
+ // merge read. flushPendingAtomicWrites forces all queued writes to disk.
111
+ flushPendingAtomicWrites();
112
+ const disk = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
113
+ const diskManifest = disk?.manifest ?? ctx.manifest;
114
+ const diskArtifacts = diskManifest.artifacts;
115
+ const reconciledArtifacts = mergeArtifacts([...diskArtifacts, ...validResults.map((item) => item.manifest.artifacts)].flat());
116
+ // R15-2: only force "running" when the disk manifest is NON-terminal
117
+ // (queued/planning/running/blocked). If the disk status is already terminal
118
+ // (cancelled/failed/completed — an external cancel or reconciler write
119
+ // landing during the batch), PRESERVE that terminal status: forcing
120
+ // "running" would legally erase it (contracts.ts allows cancelled/failed/
121
+ // completed → running) and the loop would never observe the disk-terminal
122
+ // (CANCEL-1/CANCEL-2 only catch worker-reported cancel or signal abort).
123
+ const mergedBase = { ...diskManifest, artifacts: reconciledArtifacts };
124
+ const resultManifest = isRunTerminalPreserved(diskManifest.status)
125
+ ? mergedBase
126
+ : updateRunStatus(mergedBase, "running", "Merged task updates from parallel batch.");
127
+ // CANCEL-1: use the freshly-loaded disk tasks as the merge base instead
128
+ // of the in-memory `tasks` closure variable. The in-memory tasks reflect
129
+ // only team-runner's view; an external cancel (handleCancel, background
130
+ // race with SIGTERM arriving after cancel wrote but before merge ran)
131
+ // writes 'cancelled' to disk.tasks — using disk.tasks as base preserves
132
+ // that cancellation through the merge instead of overwriting it with the
133
+ // stale in-memory view. disk was loaded inside this lock, so it reflects
134
+ // the freshest committed state.
135
+ const resultTasks = mergeTaskUpdatesPreservingTerminal(disk?.tasks ?? ctx.tasks, validResults);
136
+ await saveRunManifestAsync(resultManifest);
137
+ await saveRunTasksAsync(resultManifest, resultTasks);
138
+ return { resultManifest, resultTasks };
139
+ },
140
+ (attempt, delayMs, error) => {
141
+ // Best-effort observability only — a failed event append (e.g. the same
142
+ // degraded disk that caused the contention) must not break the retry.
143
+ appendEventAsync(ctx.manifest.eventsPath, {
144
+ type: "run.lock_retry",
145
+ runId: ctx.manifest.runId,
146
+ message: `run.lock contention during unit merge (attempt ${attempt}); retrying in ${delayMs}ms: ${error.message}`,
147
+ data: { attempt, delayMs, site: "mergeUnitResult" },
148
+ }).catch(() => undefined);
149
+ },
150
+ );
123
151
  ctx.manifest = mergeResult.resultManifest;
124
152
  ctx.tasks = mergeResult.resultTasks;
125
153
  ctx.settledMerge = { taskIds: completedUnit.taskIds, result: resultToMerge };
@@ -15,7 +15,7 @@ import {
15
15
  saveRunTasks,
16
16
  updateRunStatus,
17
17
  } from "../../state/stores/state-store.ts";
18
- import type { TeamTaskState } from "../../state/types.ts";
18
+ import type { TeamRunManifest, TeamTaskState } from "../../state/types.ts";
19
19
  import { logInternalError } from "../../utils/internal-error.ts";
20
20
  import { projectCrewRoot, userCrewRoot } from "../../utils/paths.ts";
21
21
  import { resolveRealContainedPath } from "../../utils/safe-paths.ts";
@@ -26,7 +26,13 @@ import { terminateLiveAgentsForRun } from "../live-session/live-agent-manager.ts
26
26
  import type { ManifestCache } from "../manifest-cache.ts";
27
27
  import { checkProcessLiveness } from "../process-status.ts";
28
28
  import { mapConcurrent } from "../scheduling/parallel-utils.ts";
29
- import { isIntentionalWait, isPlanApprovalPendingEffective, type ReconcileResult, reconcileStaleRun } from "../stale-reconciler.ts";
29
+ import {
30
+ hasLiveRunClaim,
31
+ isIntentionalWait,
32
+ isPlanApprovalPendingEffective,
33
+ type ReconcileResult,
34
+ reconcileStaleRun,
35
+ } from "../stale-reconciler.ts";
30
36
 
31
37
  export interface RecoveryPlan {
32
38
  runId: string;
@@ -134,7 +140,17 @@ export function detectInterruptedRuns(
134
140
  // NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
135
141
  const loaded = loadRunManifestById(cwd, manifest.runId); // NOTE: no withRunLock - best-effort only; concurrent writes may cause inconsistency
136
142
  if (!loaded) continue;
137
- const resumableTasks = loaded.tasks.filter((task) => shouldRecoverTask(task, deadMs)).map((task) => task.id);
143
+ // U9 (upgrade-spec 2026-10-09 — reconcile-at-open fast-path DETECTION): a
144
+ // dead detached runner PID + no live active-run-registry claim is
145
+ // AUTHORITATIVE crash evidence. The deadMs heartbeat-staleness wait in
146
+ // shouldRecoverTask exists to protect runs whose liveness we cannot probe
147
+ // (foreground/no-PID runs); for a dead async runner it only delays an
148
+ // interrupted run from becoming actionable by the full deadMs (300s).
149
+ // Skip the wait here so a crashed run is resumable the moment it is opened.
150
+ const fastPathEligible = loaded.manifest.async?.pid !== undefined && !hasLiveRunClaim(loaded.manifest.runId, loaded.manifest);
151
+ const resumableTasks = loaded.tasks
152
+ .filter((task) => (fastPathEligible ? task.status === "running" : shouldRecoverTask(task, deadMs)))
153
+ .map((task) => task.id);
138
154
  if (!resumableTasks.length) continue;
139
155
  plans.push({
140
156
  runId: manifest.runId,
@@ -725,6 +741,113 @@ export function purgeStaleActiveRunIndex(
725
741
  return { purged, kept };
726
742
  }
727
743
 
744
+ /**
745
+ * U9 (upgrade-spec 2026-10-09): reconcile-at-open FAST-PATH result — the
746
+ * durable-scheduler-inspired "1 pass" that replaces the verdict tree for
747
+ * crashed runs WITHOUT a live claim. `event` is appended by the CALLER via the
748
+ * async primitive after the sync writes land (U2 discipline — mirrors
749
+ * applyRecoveryPlan's `lifecycle` pattern; the run lock is held by the caller).
750
+ */
751
+ export interface ReconcileAtOpenFastPathOutcome {
752
+ result: ReconcileResult;
753
+ eventsPath: string;
754
+ event: AppendTeamEvent;
755
+ }
756
+
757
+ /**
758
+ * U9 (upgrade-spec 2026-10-09): reconcile-at-open fast-path.
759
+ *
760
+ * When a run is opened after a crash and there is NO live claim (dead async
761
+ * runner PID + no live active-run-registry entry), ONE pass requeues every
762
+ * `running` task → `queued`, PRESERVING attempt/deps/partial state (attempts,
763
+ * checkpoint, claim, artifacts, usage — only per-attempt transient fields are
764
+ * reset, mirroring applyRecoveryPlan), then marks the run `failed` so later
765
+ * reconcile passes leave the requeued tasks alone (the failed status is
766
+ * load-bearing idempotency: reconcileAllStaleRuns only visits running/blocked
767
+ * runs, so the fast-path fires exactly once; a later `team resume` requeues
768
+ * failed/cancelled/skipped/running → queued and re-executes what remains).
769
+ *
770
+ * G12 preserved: a run WITH a live claim NEVER takes this path — callers fall
771
+ * through to the full stale-reconciler verdict tree
772
+ * (healthy/waiting_answer/pid_dead/...), unchanged. Intentional waits (plan
773
+ * approval / pending ask) are likewise left to the verdict tree. Runs without
774
+ * an async runner PID (foreground/live-session) also defer — their liveness
775
+ * evidence is the registry entry plus heartbeat staleness, which the verdict
776
+ * tree already weighs.
777
+ *
778
+ * MUST be called under the run lock on a freshly-loaded state (the
779
+ * reconcileAllStaleRuns call site does exactly that). Returns undefined when
780
+ * the fast-path does not apply (live claim / intentional wait / no async PID /
781
+ * nothing left in `running`/`waiting` — the idempotent re-entry case).
782
+ */
783
+ export function applyReconcileAtOpenFastPath(
784
+ loaded: { manifest: TeamRunManifest; tasks: TeamTaskState[] },
785
+ now = Date.now(),
786
+ ): ReconcileAtOpenFastPathOutcome | undefined {
787
+ const { manifest } = loaded;
788
+ const pid = manifest.async?.pid;
789
+ // Only a dead, well-formed detached-runner PID is authoritative crash
790
+ // evidence. No PID (foreground/live-session runs) → verdict tree.
791
+ if (pid === undefined || !Number.isInteger(pid) || pid <= 0) return undefined;
792
+ if (checkProcessLiveness(pid).alive) return undefined; // live claim → verdict tree
793
+ if (hasLiveRunClaim(manifest.runId, manifest)) return undefined; // G12: never touch a claimed run
794
+ // Intentional waits (plan approval / pending ask within TTL) are not crashes.
795
+ if (isIntentionalWait(manifest, now)) return undefined;
796
+ // `waiting` tasks requeue alongside `running` ones (review round-2 MINOR-1):
797
+ // an ask parked past WAITING_TTL — or whose waitState was lost in the crash —
798
+ // is NOT an intentional wait (isIntentionalWait above already declined), and
799
+ // `team resume` only requeues failed/cancelled/skipped/running, so leaving a
800
+ // `waiting` task inside the failed run would strand it for manual repair.
801
+ const requeueEligible = loaded.tasks.filter((task) => task.status === "running" || task.status === "waiting");
802
+ if (requeueEligible.length === 0) return undefined; // idempotent: nothing to requeue
803
+
804
+ const requeued = new Set(requeueEligible.map((task) => task.id));
805
+ const tasks = loaded.tasks.map((task) =>
806
+ requeued.has(task.id)
807
+ ? {
808
+ ...task,
809
+ status: "queued" as const,
810
+ startedAt: undefined,
811
+ finishedAt: undefined,
812
+ error: undefined,
813
+ heartbeat: undefined,
814
+ // WP-2/R2 discipline (same as applyRecoveryPlan): a requeued task must
815
+ // not resume parked ask state.
816
+ waiting: undefined,
817
+ // attempt/deps/partial state PRESERVED (attempts, checkpoint, claim,
818
+ // artifacts, usage, modelAttempts) — only per-attempt transient
819
+ // fields above are reset, exactly like applyRecoveryPlan.
820
+ }
821
+ : task,
822
+ );
823
+ saveRunTasks(manifest, tasks);
824
+ updateRunStatus(
825
+ manifest,
826
+ "failed",
827
+ `Interrupted: async runner dead (no live claim); ${requeueEligible.length} task(s) requeued for resume`,
828
+ );
829
+ return {
830
+ result: {
831
+ runId: manifest.runId,
832
+ verdict: "requeued_no_claim",
833
+ repaired: true,
834
+ detail: `No live claim (async pid ${pid} dead, no live registry entry); requeued ${requeueEligible.length} interrupted task(s) (running or abandoned waiting) → queued for resume (attempt state preserved); run marked failed so later reconcile passes leave it resumable`,
835
+ repairedTasks: tasks,
836
+ },
837
+ eventsPath: manifest.eventsPath,
838
+ event: {
839
+ type: "crew.run.resumed",
840
+ runId: manifest.runId,
841
+ message: `Reconcile-at-open fast-path: requeued ${requeueEligible.length} interrupted task(s) for resume (no live claim).`,
842
+ data: {
843
+ fastPath: true,
844
+ requeuedTasks: [...requeued],
845
+ recoveredFromSeq: scanSequence(manifest.eventsPath),
846
+ },
847
+ },
848
+ };
849
+ }
850
+
728
851
  export async function reconcileAllStaleRuns(
729
852
  cwd: string,
730
853
  manifestCache: ManifestCache,
@@ -775,6 +898,21 @@ export async function reconcileAllStaleRuns(
775
898
  });
776
899
  return;
777
900
  }
901
+ // U9 (upgrade-spec 2026-10-09): reconcile-at-open FAST-PATH. A crashed run
902
+ // with NO live claim (dead async runner PID + no live registry entry) is
903
+ // requeued in one pass instead of going through the verdict tree, whose
904
+ // pid_dead branch would CANCEL the running tasks and destroy resume
905
+ // eligibility. Runs WITH a live claim (or intentional waits / no async
906
+ // PID) skip this and keep the full verdict-tree semantics, unchanged.
907
+ const fastPath = applyReconcileAtOpenFastPath(fresh, now);
908
+ if (fastPath) {
909
+ // Async lifecycle append AFTER the sync saveRunTasks/updateRunStatus
910
+ // writes (U2 discipline — same ordering as the verdict branch below:
911
+ // run.failed lands first, then the fast-path recovery record).
912
+ await appendEventAsync(fastPath.eventsPath, fastPath.event);
913
+ out.push(fastPath.result);
914
+ return;
915
+ }
778
916
  const result = reconcileStaleRun(fresh.manifest, fresh.tasks, now);
779
917
  if (result.repaired || result.verdict === "result_exists") {
780
918
  if (result.repairedTasks) {
@@ -1,3 +1,4 @@
1
+ import { logInternalError } from "../../utils/internal-error.ts";
1
2
  import { sleep } from "../../utils/sleep.ts";
2
3
  import { throwIfCancelled } from "../process/cancellation.ts";
3
4
 
@@ -28,6 +29,17 @@ export interface RetryHooks {
28
29
  onRetryGivenUp?: (attempts: number, error: Error, info: RetryAttemptInfo) => void;
29
30
  attemptId?: (attempt: number) => string;
30
31
  signal?: AbortSignal;
32
+ /**
33
+ * U6A (upgrade spec 2026-10-09 §U6 Phase A): async pre-gate consulted
34
+ * after a failed attempt and BEFORE the next one is queued. Returning
35
+ * false gives up exactly like non-retryable exhaustion (onRetryGivenUp +
36
+ * throw of the ORIGINAL error — the gate itself can never introduce a new
37
+ * failure). SOFT contract: a throwing/rejecting hook is swallowed and
38
+ * treated as "retry" — a broken gate must never change retry behavior.
39
+ * Consumer: dispatch-batch's ambient-noise classifier gate
40
+ * (classifyAmbientNoiseGate), dormant unless runtime.classifierEnabled.
41
+ */
42
+ shouldRetry?: (attempt: number, error: Error, info: RetryAttemptInfo) => boolean | Promise<boolean>;
31
43
  }
32
44
 
33
45
  export const DEFAULT_RETRY_POLICY: RetryPolicy = {
@@ -95,6 +107,23 @@ export async function executeWithRetry<T>(
95
107
  hooks.onRetryGivenUp?.(attempt, lastError, info);
96
108
  throw lastError;
97
109
  }
110
+ // U6A: ambient-noise pre-gate — consulted once per failed attempt,
111
+ // after retryability, BEFORE the backoff sleep + next spawn. SOFT:
112
+ // hook errors are logged and treated as "retry" (fallback true), so
113
+ // the gate can only skip a queued retry, never break the loop.
114
+ if (hooks.shouldRetry) {
115
+ let gateDecision = true;
116
+ try {
117
+ gateDecision = await hooks.shouldRetry(attempt, lastError, info);
118
+ } catch (gateError) {
119
+ logInternalError("retry-executor.shouldRetry", gateError, "soft-fail: proceeding with the queued retry", "warn");
120
+ gateDecision = true;
121
+ }
122
+ if (gateDecision === false) {
123
+ hooks.onRetryGivenUp?.(attempt, lastError, info);
124
+ throw lastError;
125
+ }
126
+ }
98
127
  const delay = calculateRetryDelay(attempt, normalized);
99
128
  hooks.onAttemptFailed?.(attempt, lastError, delay, info);
100
129
  try {
@@ -67,6 +67,20 @@ export function registerRunPromise(runId: string): ActiveRunPromise {
67
67
  reject = rej;
68
68
  });
69
69
  const entry: ActiveRunPromise = { promise, resolve, reject };
70
+ // RELIABILITY FIX 2026-10-10 (run team_20261010100956 incident): a run
71
+ // registered in a process with NO waitForRun waiter (background-runner
72
+ // mode — the runner awaits executeTeamRun directly, nothing races
73
+ // entry.promise) and later REJECTED (executeTeamRun's catch calls
74
+ // rejectRunPromise with the original error) left the rejection unobserved
75
+ // → Node unhandledRejection → the background-runner's guard aborted the
76
+ // whole runner AFTER its cleanup had already run (heartbeat stopped →
77
+ // stale-reconciler mass-cancelled every task). Attaching a no-op rejection
78
+ // handler marks the promise handled; consumers still observe the rejection
79
+ // through their own .then/.race attachments — this changes nothing for
80
+ // foreground waiters, it only stops a waiter-less rejection from killing
81
+ // the process. Run failures themselves remain fully reported through the
82
+ // normal executeTeamRun catch path (manifest failed + async.failed event).
83
+ promise.catch(() => undefined);
70
84
  activeRunPromises.set(runId, entry);
71
85
  return entry;
72
86
  }
@@ -128,6 +142,20 @@ export function rejectRunPromise(runId: string, reason: unknown): void {
128
142
  }
129
143
  }
130
144
 
145
+ /** Identity matcher for waitForRun's timeout error — thrown by BOTH wait
146
+ * paths (the promise-race fast path and the polling slow path) as
147
+ * `waitForRun timed out after <N>ms`. Callers use it to distinguish "the
148
+ * WATCH window expired — the run itself may still be executing" from real
149
+ * wait failures (run not found, corrupt state). RELIABILITY FIX 2026-10-10
150
+ * (bug #2): a watch expiry must be reported as a partial-watch, never
151
+ * conflated with a run failure. Message identity — run-tracker exports no
152
+ * error class (plain Error), mirroring lock-busy.ts's matcher precedent. */
153
+ const WAIT_FOR_RUN_TIMEOUT_MESSAGE = /^waitForRun timed out after \d+ms$/;
154
+
155
+ export function isWaitForRunTimeoutError(error: unknown): boolean {
156
+ return error instanceof Error && WAIT_FOR_RUN_TIMEOUT_MESSAGE.test(error.message);
157
+ }
158
+
131
159
  function raceRunPromise(entry: ActiveRunPromise, timeoutMs: number, deadline: number): Promise<RunWaitResult> {
132
160
  let timer: ReturnType<typeof setTimeout> | undefined;
133
161
  const remaining = Math.max(0, deadline - Date.now());
@@ -38,7 +38,19 @@ const ORPHAN_TEMP_SCAN_BATCH_SIZE = 50;
38
38
  export interface ReconcileResult {
39
39
  runId: string;
40
40
  /** What was found and what action was taken */
41
- verdict: "healthy" | "blocked_awaiting_approval" | "waiting_answer" | "result_exists" | "pid_dead" | "pid_alive_stale" | "no_status";
41
+ verdict:
42
+ | "healthy"
43
+ | "blocked_awaiting_approval"
44
+ | "waiting_answer"
45
+ | "result_exists"
46
+ | "pid_dead"
47
+ | "pid_alive_stale"
48
+ | "no_status"
49
+ /** U9 (upgrade-spec 2026-10-09): reconcile-at-open fast-path fired — no
50
+ * live claim (dead async runner PID + no live registry entry), running
51
+ * tasks requeued → queued (attempt/deps/partial state preserved) and the
52
+ * run marked failed-for-resume instead of cancelled by the verdict tree. */
53
+ | "requeued_no_claim";
42
54
  /** Whether repair was applied */
43
55
  repaired: boolean;
44
56
  /** Human-readable detail */
@@ -380,6 +392,29 @@ function findLiveRegistryEntry(runId: string): ActiveRunRegistryEntry | undefine
380
392
  }
381
393
  }
382
394
 
395
+ /**
396
+ * U9 (upgrade-spec 2026-10-09 — reconcile-at-open fast-path): does ANY live
397
+ * claim exist for this run RIGHT NOW? Two independent signals, either ⇒ live:
398
+ * 1. a LIVE active-run-registry entry (registerActiveRun fires at dispatch;
399
+ * activeRunEntries() applies terminal-status, dead-async-PID and
400
+ * 30-min-freshness filtering itself — an entry found here means a live
401
+ * session/runner owns the run), or
402
+ * 2. the detached async runner PID is alive (authoritative process probe —
403
+ * ESRCH/ENOENT dead, EPERM alive).
404
+ * A run with NEITHER signal cannot have a worker executing tasks right now,
405
+ * so requeueing its `running` tasks can never double-dispatch a live worker.
406
+ * This is the G12-preserving precondition for the crash-recovery fast-path:
407
+ * a run WITH a live claim always defers to the full verdict tree instead.
408
+ * Registry read failures fail-open (undefined ⇒ "no registry claim"), mirroring
409
+ * findLiveRegistryEntry/G24: a broken registry must not shield runs forever.
410
+ */
411
+ export function hasLiveRunClaim(runId: string, manifest: TeamRunManifest): boolean {
412
+ if (findLiveRegistryEntry(runId)) return true;
413
+ const pid = manifest.async?.pid;
414
+ if (pid !== undefined && Number.isInteger(pid) && pid > 0 && checkProcessLiveness(pid).alive) return true;
415
+ return false;
416
+ }
417
+
383
418
  function repairStaleRun(manifest: TeamRunManifest, tasks: TeamTaskState[], reason: string): TeamTaskState[] {
384
419
  const now = new Date().toISOString();
385
420
  const repairedTasks = tasks.map((task) => {
@@ -1,9 +1,11 @@
1
1
  import type { AgentConfig } from "../agents/agent-config.ts";
2
2
  import type { CrewLimitsConfig, CrewRuntimeConfig } from "../config/config.ts";
3
3
  import type { MetricRegistry } from "../observability/metric-registry.ts";
4
+ import { appendEventFireAndForget } from "../state/event-log/event-log.ts";
4
5
  import { writeArtifact } from "../state/stores/artifact-store.ts";
5
6
  import type { ArtifactDescriptor, OperationTerminalEvidence, TeamRunManifest, TeamTaskState } from "../state/types.ts";
6
7
  import type { WorkflowStep } from "../workflows/workflow-config.ts";
8
+ import { resolveClassifierEnabled, resolveClassifierModel } from "./classifier/classifier-service.ts";
7
9
  import type { CrewRuntimeKind } from "./crew-agent-runtime.ts";
8
10
  import { registerStreamBridge } from "./event-stream-bridge.ts";
9
11
  import type { ModelAttemptSummary } from "./model/model-fallback.ts";
@@ -15,6 +17,7 @@ import { finalizeTaskResult, type TaskExecutionResult } from "./task-runner/post
15
17
  import { prepareTaskExecutionContext } from "./task-runner/pre-execution.ts";
16
18
  import { cleanResultText } from "./task-runner/result-utils.ts";
17
19
  import { runScaffoldTask } from "./task-runner/scaffold-executor.ts";
20
+ import { renderVerifyGateSummary, resolveVerifyGateEnabled, runVerifierGatePreGate } from "./verification/verify-gate.ts";
18
21
  import { registerYieldTool } from "./yield-handler.ts";
19
22
 
20
23
  // Register the submit_result tool handler so subprocess events can extract yield data.
@@ -106,6 +109,106 @@ export async function runTeamTask(input: TaskRunnerInput): Promise<{ manifest: T
106
109
  const prepared = await prepareTaskExecutionContext(input, manifest, streamBridge);
107
110
  if (prepared.kind === "cancelled") return prepared.result;
108
111
  const ctx = prepared.ctx;
112
+ // ── U13 (upgrade spec 2026-10-09): deterministic verifier pre-gate ──
113
+ // For verifier-role tasks, run the deterministic gate (typecheck /
114
+ // test:critical via package.json scripts, subprocess executor) BEFORE
115
+ // spawning the LLM verifier worker:
116
+ // - gate PASS → SKIP the spawn: terminalize the task as completed
117
+ // with the gate evidence as its result (spec acceptance: "một run
118
+ // verification với gate xanh không spawn LLM verifier");
119
+ // - gate FAILED/INCONCLUSIVE (+ U6B classifier layer on the latter)
120
+ // → spawn the LLM verifier as today, with the gate results appended
121
+ // to its prompt (it diagnoses; it does not re-derive exit codes).
122
+ // Disabled by PI_CREW_VERIFY_GATE=0; scaffold runtime never gates.
123
+ if (ctx.task.role === "verifier" && ctx.runtimeKind !== "scaffold" && resolveVerifyGateEnabled()) {
124
+ const preGate = await runVerifierGatePreGate({
125
+ cwd: ctx.task.cwd,
126
+ signal: input.signal,
127
+ classifier: {
128
+ enabled: resolveClassifierEnabled(input.runtimeConfig?.classifierEnabled),
129
+ modelRegistry: input.modelRegistry,
130
+ classifierModel: resolveClassifierModel(input.runtimeConfig?.classifierModel),
131
+ eventsPath: manifest.eventsPath,
132
+ runId: manifest.runId,
133
+ taskId: ctx.task.id,
134
+ },
135
+ metricRegistry: input.metricRegistry,
136
+ });
137
+ // Diagnostic event (task.verifier_pre_gate — U6B's event type; the
138
+ // TEAM_EVENT_TYPES registry entry stays report-only pending a
139
+ // src/state/** lane, same as classifier-pre-gate's own appends).
140
+ appendEventFireAndForget(manifest.eventsPath, {
141
+ type: "task.verifier_pre_gate",
142
+ runId: manifest.runId,
143
+ taskId: ctx.task.id,
144
+ message: preGate.decision.skipVerifier
145
+ ? `Deterministic verify_gate ${preGate.outcome.verdict} — LLM verifier spawn SKIPPED (${preGate.decision.reason})`
146
+ : `Deterministic verify_gate ${preGate.outcome.verdict} — LLM verifier spawns with gate context (${preGate.decision.reason})`,
147
+ data: {
148
+ verdict: preGate.outcome.verdict,
149
+ decision: preGate.decision,
150
+ evidenceLines: preGate.outcome.evidenceLines,
151
+ ...(preGate.preGate ? { classifierAction: preGate.preGate.action } : {}),
152
+ },
153
+ });
154
+ if (preGate.decision.skipVerifier) {
155
+ // Terminalize WITHOUT spawning: synthesize the branch result (gate
156
+ // summary as the task result artifact + stdout) and reuse the full
157
+ // finalizer — events, hooks, persistence, verification evidence.
158
+ const summary = renderVerifyGateSummary(preGate.outcome, preGate.decision.reason);
159
+ const resultArtifact = writeArtifact(manifest.artifactsRoot, {
160
+ kind: "result",
161
+ relativePath: `results/${ctx.task.id}.txt`,
162
+ content: `${summary}\n`,
163
+ producer: ctx.task.id,
164
+ });
165
+ ctx.task = {
166
+ ...ctx.task,
167
+ // Provenance: this verdict came from the deterministic gate, not a
168
+ // model attempt (U6B mount note: evaluator/model provenance marker).
169
+ diagnostics: {
170
+ ...(ctx.task.diagnostics ?? {}),
171
+ verifyGate: {
172
+ verdict: preGate.outcome.verdict,
173
+ reason: preGate.decision.reason,
174
+ evidenceLines: preGate.outcome.evidenceLines,
175
+ },
176
+ },
177
+ };
178
+ return await finalizeTaskResult(ctx, {
179
+ resultArtifact,
180
+ logArtifact: undefined,
181
+ transcriptArtifact: undefined,
182
+ exitCode: 0,
183
+ error: undefined,
184
+ modelAttempts: [],
185
+ parsedOutput: undefined,
186
+ finalStdout: summary,
187
+ transcriptPath: undefined,
188
+ terminalEvidence: [],
189
+ startupEvidence: ctx.startupEvidence,
190
+ });
191
+ }
192
+ if (preGate.promptContext) {
193
+ // Spawn proceeds — attach the gate results to the verifier prompt
194
+ // (both channels: full prompt + the user-message half). The prompt
195
+ // artifact was written in pre-execution with the PRE-gate prompt;
196
+ // rewrite it so the persisted artifact matches what the worker saw.
197
+ ctx.prompt = `${ctx.prompt}\n\n${preGate.promptContext}`;
198
+ if (ctx.userPrompt !== undefined) ctx.userPrompt = `${ctx.userPrompt}\n\n${preGate.promptContext}`;
199
+ if (ctx.promptArtifact) {
200
+ ctx.promptArtifact = writeArtifact(manifest.artifactsRoot, {
201
+ kind: "prompt",
202
+ // Same relative-path convention pre-execution used
203
+ // (prompts/<taskId>.md) — ArtifactDescriptor carries only the
204
+ // absolute path, so the rewrite re-derives the relative path.
205
+ relativePath: `prompts/${ctx.task.id}.md`,
206
+ content: `${ctx.prompt}\n`,
207
+ producer: ctx.task.id,
208
+ });
209
+ }
210
+ }
211
+ }
109
212
  // Destructure mutable fields for branch closures + sync-back
110
213
  let task = ctx.task;
111
214
  let tasks = ctx.tasks;