pi-crew 0.11.11 → 0.11.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +102 -0
- package/dist/index.mjs +551 -519
- package/package.json +1 -1
- package/src/config/config-validation.ts +9 -0
- package/src/config/defaults.ts +7 -0
- package/src/config/env-vars.ts +24 -0
- package/src/config/types.ts +11 -0
- package/src/extension/registration/tool-registration.ts +7 -0
- package/src/extension/registration/verify-gate-tool.ts +263 -0
- package/src/extension/team-tool/run-deadline.ts +22 -1
- package/src/extension/team-tool/run.ts +38 -2
- package/src/extension/team-tool-types.ts +4 -0
- package/src/extension/team-tool.ts +31 -3
- package/src/prompt/inbox-poll.ts +141 -3
- package/src/runtime/broker/crew-broker.ts +48 -2
- package/src/runtime/broker/protocol/lock-busy.ts +47 -1
- package/src/runtime/broker/protocol/request-parsers.ts +8 -1
- package/src/runtime/budget-enforcement.ts +36 -20
- package/src/runtime/child-pi/child-pi-constants.ts +24 -0
- package/src/runtime/child-pi/child-pi-kill.ts +59 -2
- package/src/runtime/child-pi/child-pi-rpc.ts +1021 -0
- package/src/runtime/child-pi/child-pi-steering.ts +10 -8
- package/src/runtime/child-pi/child-pi-streams.ts +69 -19
- package/src/runtime/child-pi/child-pi.ts +29 -30
- package/src/runtime/child-pi/owned-process.ts +434 -0
- package/src/runtime/child-pi/postmortem-registry.ts +134 -0
- package/src/runtime/classifier/ambient-noise-gate.ts +126 -0
- package/src/runtime/classifier/classifier-service.ts +277 -61
- package/src/runtime/dispatch-batch.ts +32 -0
- package/src/runtime/live-session/live-agent-control.ts +2 -0
- package/src/runtime/live-session/live-agent-manager.ts +72 -2
- package/src/runtime/live-session/live-session-runtime.ts +435 -40
- package/src/runtime/merge-loop.ts +63 -35
- package/src/runtime/recovery/crash-recovery.ts +141 -3
- package/src/runtime/recovery/retry-executor.ts +29 -0
- package/src/runtime/run-tracker.ts +28 -0
- package/src/runtime/stale-reconciler.ts +36 -1
- package/src/runtime/task-runner.ts +103 -0
- package/src/runtime/verification/classifier-pre-gate.ts +262 -0
- package/src/runtime/verification/verification-gates.ts +5 -1
- package/src/runtime/verification/verify-gate.ts +394 -0
- package/src/schema/config-schema.ts +13 -0
- package/src/state/contracts.ts +18 -0
- package/src/state/coordination/mailbox.ts +203 -55
- package/src/state/stores/sqlite-run-state.ts +378 -0
- package/src/state/stores/state-store.ts +134 -6
- package/src/ui/agents-jobs-browser.ts +154 -8
- package/src/ui/events-state-source.ts +358 -0
- package/src/ui/run-snapshot-cache.ts +87 -3
- package/src/ui/widget/widget-model.ts +13 -2
|
@@ -13,10 +13,11 @@
|
|
|
13
13
|
* keeping it here avoids a merge-loop ↔ finalize-run import cycle.
|
|
14
14
|
*/
|
|
15
15
|
import { flushPendingAtomicWrites } from "../state/atomic-write.ts";
|
|
16
|
-
import {
|
|
16
|
+
import { appendEventAsync } from "../state/event-log/event-log.ts";
|
|
17
17
|
import { loadRunManifestById, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/stores/state-store.ts";
|
|
18
18
|
import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
|
|
19
19
|
import { classifyFatalFsError } from "../utils/fs-errno.ts";
|
|
20
|
+
import { MERGE_LOCK_BUSY_RETRY_DELAYS_MS, withRunLockBusyRetryAsync } from "./broker/protocol/lock-busy.ts";
|
|
20
21
|
import { cancelNonTerminalTasks } from "./dispatch-batch.ts";
|
|
21
22
|
import { mergeTaskUpdatesPreservingTerminal } from "./merge-gate.ts";
|
|
22
23
|
import type { SchedulerContext, SchedulerDecision } from "./scheduler-context.ts";
|
|
@@ -86,40 +87,67 @@ export async function mergeUnitResult(ctx: SchedulerContext): Promise<SchedulerD
|
|
|
86
87
|
// not the arbitrary order in which mapConcurrent returned results.
|
|
87
88
|
// Read committed manifest from disk inside the lock so artifact merge is based
|
|
88
89
|
// on committed state, not in-memory state that may differ from disk.
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
90
|
+
// RELIABILITY FIX 2026-10-10 (run team_20261010100956 incident): the merge
|
|
91
|
+
// used a bare withRunLock. Under run.lock contention (a second writer —
|
|
92
|
+
// stale reconciler, resumed runner, or a slow ENOSPC-degraded writer —
|
|
93
|
+
// holding run.lock past the acquire retry budget) withRunLock throws the
|
|
94
|
+
// plain "Run 'run.lock' is locked by another operation." error, which
|
|
95
|
+
// escaped executeTeamRun's catch → rejectRunPromise → an UNHANDLED promise
|
|
96
|
+
// rejection (no waitForRun waiter in background-runner mode) → the runner's
|
|
97
|
+
// unhandledRejection guard killed the whole runner mid-run. The merge now
|
|
98
|
+
// retries transient contention with back-off (logging a run.lock_retry
|
|
99
|
+
// event per attempt); only after the bounded schedule is exhausted does
|
|
100
|
+
// the busy error propagate for the run-level handler to classify. Non-busy
|
|
101
|
+
// errors (real faults) keep their existing fatal path — this is NOT a
|
|
102
|
+
// swallow-all.
|
|
103
|
+
const mergeResult = await withRunLockBusyRetryAsync(
|
|
104
|
+
ctx.manifest,
|
|
105
|
+
MERGE_LOCK_BUSY_RETRY_DELAYS_MS,
|
|
106
|
+
async () => {
|
|
107
|
+
// NEW-D1: flush any pending coalesced atomic writes before reading from
|
|
108
|
+
// disk. Without this, a worker's async manifest save (coalesced by
|
|
109
|
+
// atomic-write) may not be committed yet, causing a lost-update on the
|
|
110
|
+
// merge read. flushPendingAtomicWrites forces all queued writes to disk.
|
|
111
|
+
flushPendingAtomicWrites();
|
|
112
|
+
const disk = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
|
|
113
|
+
const diskManifest = disk?.manifest ?? ctx.manifest;
|
|
114
|
+
const diskArtifacts = diskManifest.artifacts;
|
|
115
|
+
const reconciledArtifacts = mergeArtifacts([...diskArtifacts, ...validResults.map((item) => item.manifest.artifacts)].flat());
|
|
116
|
+
// R15-2: only force "running" when the disk manifest is NON-terminal
|
|
117
|
+
// (queued/planning/running/blocked). If the disk status is already terminal
|
|
118
|
+
// (cancelled/failed/completed — an external cancel or reconciler write
|
|
119
|
+
// landing during the batch), PRESERVE that terminal status: forcing
|
|
120
|
+
// "running" would legally erase it (contracts.ts allows cancelled/failed/
|
|
121
|
+
// completed → running) and the loop would never observe the disk-terminal
|
|
122
|
+
// (CANCEL-1/CANCEL-2 only catch worker-reported cancel or signal abort).
|
|
123
|
+
const mergedBase = { ...diskManifest, artifacts: reconciledArtifacts };
|
|
124
|
+
const resultManifest = isRunTerminalPreserved(diskManifest.status)
|
|
125
|
+
? mergedBase
|
|
126
|
+
: updateRunStatus(mergedBase, "running", "Merged task updates from parallel batch.");
|
|
127
|
+
// CANCEL-1: use the freshly-loaded disk tasks as the merge base instead
|
|
128
|
+
// of the in-memory `tasks` closure variable. The in-memory tasks reflect
|
|
129
|
+
// only team-runner's view; an external cancel (handleCancel, background
|
|
130
|
+
// race with SIGTERM arriving after cancel wrote but before merge ran)
|
|
131
|
+
// writes 'cancelled' to disk.tasks — using disk.tasks as base preserves
|
|
132
|
+
// that cancellation through the merge instead of overwriting it with the
|
|
133
|
+
// stale in-memory view. disk was loaded inside this lock, so it reflects
|
|
134
|
+
// the freshest committed state.
|
|
135
|
+
const resultTasks = mergeTaskUpdatesPreservingTerminal(disk?.tasks ?? ctx.tasks, validResults);
|
|
136
|
+
await saveRunManifestAsync(resultManifest);
|
|
137
|
+
await saveRunTasksAsync(resultManifest, resultTasks);
|
|
138
|
+
return { resultManifest, resultTasks };
|
|
139
|
+
},
|
|
140
|
+
(attempt, delayMs, error) => {
|
|
141
|
+
// Best-effort observability only — a failed event append (e.g. the same
|
|
142
|
+
// degraded disk that caused the contention) must not break the retry.
|
|
143
|
+
appendEventAsync(ctx.manifest.eventsPath, {
|
|
144
|
+
type: "run.lock_retry",
|
|
145
|
+
runId: ctx.manifest.runId,
|
|
146
|
+
message: `run.lock contention during unit merge (attempt ${attempt}); retrying in ${delayMs}ms: ${error.message}`,
|
|
147
|
+
data: { attempt, delayMs, site: "mergeUnitResult" },
|
|
148
|
+
}).catch(() => undefined);
|
|
149
|
+
},
|
|
150
|
+
);
|
|
123
151
|
ctx.manifest = mergeResult.resultManifest;
|
|
124
152
|
ctx.tasks = mergeResult.resultTasks;
|
|
125
153
|
ctx.settledMerge = { taskIds: completedUnit.taskIds, result: resultToMerge };
|
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
saveRunTasks,
|
|
16
16
|
updateRunStatus,
|
|
17
17
|
} from "../../state/stores/state-store.ts";
|
|
18
|
-
import type { TeamTaskState } from "../../state/types.ts";
|
|
18
|
+
import type { TeamRunManifest, TeamTaskState } from "../../state/types.ts";
|
|
19
19
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
20
20
|
import { projectCrewRoot, userCrewRoot } from "../../utils/paths.ts";
|
|
21
21
|
import { resolveRealContainedPath } from "../../utils/safe-paths.ts";
|
|
@@ -26,7 +26,13 @@ import { terminateLiveAgentsForRun } from "../live-session/live-agent-manager.ts
|
|
|
26
26
|
import type { ManifestCache } from "../manifest-cache.ts";
|
|
27
27
|
import { checkProcessLiveness } from "../process-status.ts";
|
|
28
28
|
import { mapConcurrent } from "../scheduling/parallel-utils.ts";
|
|
29
|
-
import {
|
|
29
|
+
import {
|
|
30
|
+
hasLiveRunClaim,
|
|
31
|
+
isIntentionalWait,
|
|
32
|
+
isPlanApprovalPendingEffective,
|
|
33
|
+
type ReconcileResult,
|
|
34
|
+
reconcileStaleRun,
|
|
35
|
+
} from "../stale-reconciler.ts";
|
|
30
36
|
|
|
31
37
|
export interface RecoveryPlan {
|
|
32
38
|
runId: string;
|
|
@@ -134,7 +140,17 @@ export function detectInterruptedRuns(
|
|
|
134
140
|
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
135
141
|
const loaded = loadRunManifestById(cwd, manifest.runId); // NOTE: no withRunLock - best-effort only; concurrent writes may cause inconsistency
|
|
136
142
|
if (!loaded) continue;
|
|
137
|
-
|
|
143
|
+
// U9 (upgrade-spec 2026-10-09 — reconcile-at-open fast-path DETECTION): a
|
|
144
|
+
// dead detached runner PID + no live active-run-registry claim is
|
|
145
|
+
// AUTHORITATIVE crash evidence. The deadMs heartbeat-staleness wait in
|
|
146
|
+
// shouldRecoverTask exists to protect runs whose liveness we cannot probe
|
|
147
|
+
// (foreground/no-PID runs); for a dead async runner it only delays an
|
|
148
|
+
// interrupted run from becoming actionable by the full deadMs (300s).
|
|
149
|
+
// Skip the wait here so a crashed run is resumable the moment it is opened.
|
|
150
|
+
const fastPathEligible = loaded.manifest.async?.pid !== undefined && !hasLiveRunClaim(loaded.manifest.runId, loaded.manifest);
|
|
151
|
+
const resumableTasks = loaded.tasks
|
|
152
|
+
.filter((task) => (fastPathEligible ? task.status === "running" : shouldRecoverTask(task, deadMs)))
|
|
153
|
+
.map((task) => task.id);
|
|
138
154
|
if (!resumableTasks.length) continue;
|
|
139
155
|
plans.push({
|
|
140
156
|
runId: manifest.runId,
|
|
@@ -725,6 +741,113 @@ export function purgeStaleActiveRunIndex(
|
|
|
725
741
|
return { purged, kept };
|
|
726
742
|
}
|
|
727
743
|
|
|
744
|
+
/**
|
|
745
|
+
* U9 (upgrade-spec 2026-10-09): reconcile-at-open FAST-PATH result — the
|
|
746
|
+
* durable-scheduler-inspired "1 pass" that replaces the verdict tree for
|
|
747
|
+
* crashed runs WITHOUT a live claim. `event` is appended by the CALLER via the
|
|
748
|
+
* async primitive after the sync writes land (U2 discipline — mirrors
|
|
749
|
+
* applyRecoveryPlan's `lifecycle` pattern; the run lock is held by the caller).
|
|
750
|
+
*/
|
|
751
|
+
export interface ReconcileAtOpenFastPathOutcome {
|
|
752
|
+
result: ReconcileResult;
|
|
753
|
+
eventsPath: string;
|
|
754
|
+
event: AppendTeamEvent;
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
/**
|
|
758
|
+
* U9 (upgrade-spec 2026-10-09): reconcile-at-open fast-path.
|
|
759
|
+
*
|
|
760
|
+
* When a run is opened after a crash and there is NO live claim (dead async
|
|
761
|
+
* runner PID + no live active-run-registry entry), ONE pass requeues every
|
|
762
|
+
* `running` task → `queued`, PRESERVING attempt/deps/partial state (attempts,
|
|
763
|
+
* checkpoint, claim, artifacts, usage — only per-attempt transient fields are
|
|
764
|
+
* reset, mirroring applyRecoveryPlan), then marks the run `failed` so later
|
|
765
|
+
* reconcile passes leave the requeued tasks alone (the failed status is
|
|
766
|
+
* load-bearing idempotency: reconcileAllStaleRuns only visits running/blocked
|
|
767
|
+
* runs, so the fast-path fires exactly once; a later `team resume` requeues
|
|
768
|
+
* failed/cancelled/skipped/running → queued and re-executes what remains).
|
|
769
|
+
*
|
|
770
|
+
* G12 preserved: a run WITH a live claim NEVER takes this path — callers fall
|
|
771
|
+
* through to the full stale-reconciler verdict tree
|
|
772
|
+
* (healthy/waiting_answer/pid_dead/...), unchanged. Intentional waits (plan
|
|
773
|
+
* approval / pending ask) are likewise left to the verdict tree. Runs without
|
|
774
|
+
* an async runner PID (foreground/live-session) also defer — their liveness
|
|
775
|
+
* evidence is the registry entry plus heartbeat staleness, which the verdict
|
|
776
|
+
* tree already weighs.
|
|
777
|
+
*
|
|
778
|
+
* MUST be called under the run lock on a freshly-loaded state (the
|
|
779
|
+
* reconcileAllStaleRuns call site does exactly that). Returns undefined when
|
|
780
|
+
* the fast-path does not apply (live claim / intentional wait / no async PID /
|
|
781
|
+
* nothing left in `running`/`waiting` — the idempotent re-entry case).
|
|
782
|
+
*/
|
|
783
|
+
export function applyReconcileAtOpenFastPath(
|
|
784
|
+
loaded: { manifest: TeamRunManifest; tasks: TeamTaskState[] },
|
|
785
|
+
now = Date.now(),
|
|
786
|
+
): ReconcileAtOpenFastPathOutcome | undefined {
|
|
787
|
+
const { manifest } = loaded;
|
|
788
|
+
const pid = manifest.async?.pid;
|
|
789
|
+
// Only a dead, well-formed detached-runner PID is authoritative crash
|
|
790
|
+
// evidence. No PID (foreground/live-session runs) → verdict tree.
|
|
791
|
+
if (pid === undefined || !Number.isInteger(pid) || pid <= 0) return undefined;
|
|
792
|
+
if (checkProcessLiveness(pid).alive) return undefined; // live claim → verdict tree
|
|
793
|
+
if (hasLiveRunClaim(manifest.runId, manifest)) return undefined; // G12: never touch a claimed run
|
|
794
|
+
// Intentional waits (plan approval / pending ask within TTL) are not crashes.
|
|
795
|
+
if (isIntentionalWait(manifest, now)) return undefined;
|
|
796
|
+
// `waiting` tasks requeue alongside `running` ones (review round-2 MINOR-1):
|
|
797
|
+
// an ask parked past WAITING_TTL — or whose waitState was lost in the crash —
|
|
798
|
+
// is NOT an intentional wait (isIntentionalWait above already declined), and
|
|
799
|
+
// `team resume` only requeues failed/cancelled/skipped/running, so leaving a
|
|
800
|
+
// `waiting` task inside the failed run would strand it for manual repair.
|
|
801
|
+
const requeueEligible = loaded.tasks.filter((task) => task.status === "running" || task.status === "waiting");
|
|
802
|
+
if (requeueEligible.length === 0) return undefined; // idempotent: nothing to requeue
|
|
803
|
+
|
|
804
|
+
const requeued = new Set(requeueEligible.map((task) => task.id));
|
|
805
|
+
const tasks = loaded.tasks.map((task) =>
|
|
806
|
+
requeued.has(task.id)
|
|
807
|
+
? {
|
|
808
|
+
...task,
|
|
809
|
+
status: "queued" as const,
|
|
810
|
+
startedAt: undefined,
|
|
811
|
+
finishedAt: undefined,
|
|
812
|
+
error: undefined,
|
|
813
|
+
heartbeat: undefined,
|
|
814
|
+
// WP-2/R2 discipline (same as applyRecoveryPlan): a requeued task must
|
|
815
|
+
// not resume parked ask state.
|
|
816
|
+
waiting: undefined,
|
|
817
|
+
// attempt/deps/partial state PRESERVED (attempts, checkpoint, claim,
|
|
818
|
+
// artifacts, usage, modelAttempts) — only per-attempt transient
|
|
819
|
+
// fields above are reset, exactly like applyRecoveryPlan.
|
|
820
|
+
}
|
|
821
|
+
: task,
|
|
822
|
+
);
|
|
823
|
+
saveRunTasks(manifest, tasks);
|
|
824
|
+
updateRunStatus(
|
|
825
|
+
manifest,
|
|
826
|
+
"failed",
|
|
827
|
+
`Interrupted: async runner dead (no live claim); ${requeueEligible.length} task(s) requeued for resume`,
|
|
828
|
+
);
|
|
829
|
+
return {
|
|
830
|
+
result: {
|
|
831
|
+
runId: manifest.runId,
|
|
832
|
+
verdict: "requeued_no_claim",
|
|
833
|
+
repaired: true,
|
|
834
|
+
detail: `No live claim (async pid ${pid} dead, no live registry entry); requeued ${requeueEligible.length} interrupted task(s) (running or abandoned waiting) → queued for resume (attempt state preserved); run marked failed so later reconcile passes leave it resumable`,
|
|
835
|
+
repairedTasks: tasks,
|
|
836
|
+
},
|
|
837
|
+
eventsPath: manifest.eventsPath,
|
|
838
|
+
event: {
|
|
839
|
+
type: "crew.run.resumed",
|
|
840
|
+
runId: manifest.runId,
|
|
841
|
+
message: `Reconcile-at-open fast-path: requeued ${requeueEligible.length} interrupted task(s) for resume (no live claim).`,
|
|
842
|
+
data: {
|
|
843
|
+
fastPath: true,
|
|
844
|
+
requeuedTasks: [...requeued],
|
|
845
|
+
recoveredFromSeq: scanSequence(manifest.eventsPath),
|
|
846
|
+
},
|
|
847
|
+
},
|
|
848
|
+
};
|
|
849
|
+
}
|
|
850
|
+
|
|
728
851
|
export async function reconcileAllStaleRuns(
|
|
729
852
|
cwd: string,
|
|
730
853
|
manifestCache: ManifestCache,
|
|
@@ -775,6 +898,21 @@ export async function reconcileAllStaleRuns(
|
|
|
775
898
|
});
|
|
776
899
|
return;
|
|
777
900
|
}
|
|
901
|
+
// U9 (upgrade-spec 2026-10-09): reconcile-at-open FAST-PATH. A crashed run
|
|
902
|
+
// with NO live claim (dead async runner PID + no live registry entry) is
|
|
903
|
+
// requeued in one pass instead of going through the verdict tree, whose
|
|
904
|
+
// pid_dead branch would CANCEL the running tasks and destroy resume
|
|
905
|
+
// eligibility. Runs WITH a live claim (or intentional waits / no async
|
|
906
|
+
// PID) skip this and keep the full verdict-tree semantics, unchanged.
|
|
907
|
+
const fastPath = applyReconcileAtOpenFastPath(fresh, now);
|
|
908
|
+
if (fastPath) {
|
|
909
|
+
// Async lifecycle append AFTER the sync saveRunTasks/updateRunStatus
|
|
910
|
+
// writes (U2 discipline — same ordering as the verdict branch below:
|
|
911
|
+
// run.failed lands first, then the fast-path recovery record).
|
|
912
|
+
await appendEventAsync(fastPath.eventsPath, fastPath.event);
|
|
913
|
+
out.push(fastPath.result);
|
|
914
|
+
return;
|
|
915
|
+
}
|
|
778
916
|
const result = reconcileStaleRun(fresh.manifest, fresh.tasks, now);
|
|
779
917
|
if (result.repaired || result.verdict === "result_exists") {
|
|
780
918
|
if (result.repairedTasks) {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { logInternalError } from "../../utils/internal-error.ts";
|
|
1
2
|
import { sleep } from "../../utils/sleep.ts";
|
|
2
3
|
import { throwIfCancelled } from "../process/cancellation.ts";
|
|
3
4
|
|
|
@@ -28,6 +29,17 @@ export interface RetryHooks {
|
|
|
28
29
|
onRetryGivenUp?: (attempts: number, error: Error, info: RetryAttemptInfo) => void;
|
|
29
30
|
attemptId?: (attempt: number) => string;
|
|
30
31
|
signal?: AbortSignal;
|
|
32
|
+
/**
|
|
33
|
+
* U6A (upgrade spec 2026-10-09 §U6 Phase A): async pre-gate consulted
|
|
34
|
+
* after a failed attempt and BEFORE the next one is queued. Returning
|
|
35
|
+
* false gives up exactly like non-retryable exhaustion (onRetryGivenUp +
|
|
36
|
+
* throw of the ORIGINAL error — the gate itself can never introduce a new
|
|
37
|
+
* failure). SOFT contract: a throwing/rejecting hook is swallowed and
|
|
38
|
+
* treated as "retry" — a broken gate must never change retry behavior.
|
|
39
|
+
* Consumer: dispatch-batch's ambient-noise classifier gate
|
|
40
|
+
* (classifyAmbientNoiseGate), dormant unless runtime.classifierEnabled.
|
|
41
|
+
*/
|
|
42
|
+
shouldRetry?: (attempt: number, error: Error, info: RetryAttemptInfo) => boolean | Promise<boolean>;
|
|
31
43
|
}
|
|
32
44
|
|
|
33
45
|
export const DEFAULT_RETRY_POLICY: RetryPolicy = {
|
|
@@ -95,6 +107,23 @@ export async function executeWithRetry<T>(
|
|
|
95
107
|
hooks.onRetryGivenUp?.(attempt, lastError, info);
|
|
96
108
|
throw lastError;
|
|
97
109
|
}
|
|
110
|
+
// U6A: ambient-noise pre-gate — consulted once per failed attempt,
|
|
111
|
+
// after retryability, BEFORE the backoff sleep + next spawn. SOFT:
|
|
112
|
+
// hook errors are logged and treated as "retry" (fallback true), so
|
|
113
|
+
// the gate can only skip a queued retry, never break the loop.
|
|
114
|
+
if (hooks.shouldRetry) {
|
|
115
|
+
let gateDecision = true;
|
|
116
|
+
try {
|
|
117
|
+
gateDecision = await hooks.shouldRetry(attempt, lastError, info);
|
|
118
|
+
} catch (gateError) {
|
|
119
|
+
logInternalError("retry-executor.shouldRetry", gateError, "soft-fail: proceeding with the queued retry", "warn");
|
|
120
|
+
gateDecision = true;
|
|
121
|
+
}
|
|
122
|
+
if (gateDecision === false) {
|
|
123
|
+
hooks.onRetryGivenUp?.(attempt, lastError, info);
|
|
124
|
+
throw lastError;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
98
127
|
const delay = calculateRetryDelay(attempt, normalized);
|
|
99
128
|
hooks.onAttemptFailed?.(attempt, lastError, delay, info);
|
|
100
129
|
try {
|
|
@@ -67,6 +67,20 @@ export function registerRunPromise(runId: string): ActiveRunPromise {
|
|
|
67
67
|
reject = rej;
|
|
68
68
|
});
|
|
69
69
|
const entry: ActiveRunPromise = { promise, resolve, reject };
|
|
70
|
+
// RELIABILITY FIX 2026-10-10 (run team_20261010100956 incident): a run
|
|
71
|
+
// registered in a process with NO waitForRun waiter (background-runner
|
|
72
|
+
// mode — the runner awaits executeTeamRun directly, nothing races
|
|
73
|
+
// entry.promise) and later REJECTED (executeTeamRun's catch calls
|
|
74
|
+
// rejectRunPromise with the original error) left the rejection unobserved
|
|
75
|
+
// → Node unhandledRejection → the background-runner's guard aborted the
|
|
76
|
+
// whole runner AFTER its cleanup had already run (heartbeat stopped →
|
|
77
|
+
// stale-reconciler mass-cancelled every task). Attaching a no-op rejection
|
|
78
|
+
// handler marks the promise handled; consumers still observe the rejection
|
|
79
|
+
// through their own .then/.race attachments — this changes nothing for
|
|
80
|
+
// foreground waiters, it only stops a waiter-less rejection from killing
|
|
81
|
+
// the process. Run failures themselves remain fully reported through the
|
|
82
|
+
// normal executeTeamRun catch path (manifest failed + async.failed event).
|
|
83
|
+
promise.catch(() => undefined);
|
|
70
84
|
activeRunPromises.set(runId, entry);
|
|
71
85
|
return entry;
|
|
72
86
|
}
|
|
@@ -128,6 +142,20 @@ export function rejectRunPromise(runId: string, reason: unknown): void {
|
|
|
128
142
|
}
|
|
129
143
|
}
|
|
130
144
|
|
|
145
|
+
/** Identity matcher for waitForRun's timeout error — thrown by BOTH wait
|
|
146
|
+
* paths (the promise-race fast path and the polling slow path) as
|
|
147
|
+
* `waitForRun timed out after <N>ms`. Callers use it to distinguish "the
|
|
148
|
+
* WATCH window expired — the run itself may still be executing" from real
|
|
149
|
+
* wait failures (run not found, corrupt state). RELIABILITY FIX 2026-10-10
|
|
150
|
+
* (bug #2): a watch expiry must be reported as a partial-watch, never
|
|
151
|
+
* conflated with a run failure. Message identity — run-tracker exports no
|
|
152
|
+
* error class (plain Error), mirroring lock-busy.ts's matcher precedent. */
|
|
153
|
+
const WAIT_FOR_RUN_TIMEOUT_MESSAGE = /^waitForRun timed out after \d+ms$/;
|
|
154
|
+
|
|
155
|
+
export function isWaitForRunTimeoutError(error: unknown): boolean {
|
|
156
|
+
return error instanceof Error && WAIT_FOR_RUN_TIMEOUT_MESSAGE.test(error.message);
|
|
157
|
+
}
|
|
158
|
+
|
|
131
159
|
function raceRunPromise(entry: ActiveRunPromise, timeoutMs: number, deadline: number): Promise<RunWaitResult> {
|
|
132
160
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
133
161
|
const remaining = Math.max(0, deadline - Date.now());
|
|
@@ -38,7 +38,19 @@ const ORPHAN_TEMP_SCAN_BATCH_SIZE = 50;
|
|
|
38
38
|
export interface ReconcileResult {
|
|
39
39
|
runId: string;
|
|
40
40
|
/** What was found and what action was taken */
|
|
41
|
-
verdict:
|
|
41
|
+
verdict:
|
|
42
|
+
| "healthy"
|
|
43
|
+
| "blocked_awaiting_approval"
|
|
44
|
+
| "waiting_answer"
|
|
45
|
+
| "result_exists"
|
|
46
|
+
| "pid_dead"
|
|
47
|
+
| "pid_alive_stale"
|
|
48
|
+
| "no_status"
|
|
49
|
+
/** U9 (upgrade-spec 2026-10-09): reconcile-at-open fast-path fired — no
|
|
50
|
+
* live claim (dead async runner PID + no live registry entry), running
|
|
51
|
+
* tasks requeued → queued (attempt/deps/partial state preserved) and the
|
|
52
|
+
* run marked failed-for-resume instead of cancelled by the verdict tree. */
|
|
53
|
+
| "requeued_no_claim";
|
|
42
54
|
/** Whether repair was applied */
|
|
43
55
|
repaired: boolean;
|
|
44
56
|
/** Human-readable detail */
|
|
@@ -380,6 +392,29 @@ function findLiveRegistryEntry(runId: string): ActiveRunRegistryEntry | undefine
|
|
|
380
392
|
}
|
|
381
393
|
}
|
|
382
394
|
|
|
395
|
+
/**
|
|
396
|
+
* U9 (upgrade-spec 2026-10-09 — reconcile-at-open fast-path): does ANY live
|
|
397
|
+
* claim exist for this run RIGHT NOW? Two independent signals, either ⇒ live:
|
|
398
|
+
* 1. a LIVE active-run-registry entry (registerActiveRun fires at dispatch;
|
|
399
|
+
* activeRunEntries() applies terminal-status, dead-async-PID and
|
|
400
|
+
* 30-min-freshness filtering itself — an entry found here means a live
|
|
401
|
+
* session/runner owns the run), or
|
|
402
|
+
* 2. the detached async runner PID is alive (authoritative process probe —
|
|
403
|
+
* ESRCH/ENOENT dead, EPERM alive).
|
|
404
|
+
* A run with NEITHER signal cannot have a worker executing tasks right now,
|
|
405
|
+
* so requeueing its `running` tasks can never double-dispatch a live worker.
|
|
406
|
+
* This is the G12-preserving precondition for the crash-recovery fast-path:
|
|
407
|
+
* a run WITH a live claim always defers to the full verdict tree instead.
|
|
408
|
+
* Registry read failures fail-open (undefined ⇒ "no registry claim"), mirroring
|
|
409
|
+
* findLiveRegistryEntry/G24: a broken registry must not shield runs forever.
|
|
410
|
+
*/
|
|
411
|
+
export function hasLiveRunClaim(runId: string, manifest: TeamRunManifest): boolean {
|
|
412
|
+
if (findLiveRegistryEntry(runId)) return true;
|
|
413
|
+
const pid = manifest.async?.pid;
|
|
414
|
+
if (pid !== undefined && Number.isInteger(pid) && pid > 0 && checkProcessLiveness(pid).alive) return true;
|
|
415
|
+
return false;
|
|
416
|
+
}
|
|
417
|
+
|
|
383
418
|
function repairStaleRun(manifest: TeamRunManifest, tasks: TeamTaskState[], reason: string): TeamTaskState[] {
|
|
384
419
|
const now = new Date().toISOString();
|
|
385
420
|
const repairedTasks = tasks.map((task) => {
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import type { AgentConfig } from "../agents/agent-config.ts";
|
|
2
2
|
import type { CrewLimitsConfig, CrewRuntimeConfig } from "../config/config.ts";
|
|
3
3
|
import type { MetricRegistry } from "../observability/metric-registry.ts";
|
|
4
|
+
import { appendEventFireAndForget } from "../state/event-log/event-log.ts";
|
|
4
5
|
import { writeArtifact } from "../state/stores/artifact-store.ts";
|
|
5
6
|
import type { ArtifactDescriptor, OperationTerminalEvidence, TeamRunManifest, TeamTaskState } from "../state/types.ts";
|
|
6
7
|
import type { WorkflowStep } from "../workflows/workflow-config.ts";
|
|
8
|
+
import { resolveClassifierEnabled, resolveClassifierModel } from "./classifier/classifier-service.ts";
|
|
7
9
|
import type { CrewRuntimeKind } from "./crew-agent-runtime.ts";
|
|
8
10
|
import { registerStreamBridge } from "./event-stream-bridge.ts";
|
|
9
11
|
import type { ModelAttemptSummary } from "./model/model-fallback.ts";
|
|
@@ -15,6 +17,7 @@ import { finalizeTaskResult, type TaskExecutionResult } from "./task-runner/post
|
|
|
15
17
|
import { prepareTaskExecutionContext } from "./task-runner/pre-execution.ts";
|
|
16
18
|
import { cleanResultText } from "./task-runner/result-utils.ts";
|
|
17
19
|
import { runScaffoldTask } from "./task-runner/scaffold-executor.ts";
|
|
20
|
+
import { renderVerifyGateSummary, resolveVerifyGateEnabled, runVerifierGatePreGate } from "./verification/verify-gate.ts";
|
|
18
21
|
import { registerYieldTool } from "./yield-handler.ts";
|
|
19
22
|
|
|
20
23
|
// Register the submit_result tool handler so subprocess events can extract yield data.
|
|
@@ -106,6 +109,106 @@ export async function runTeamTask(input: TaskRunnerInput): Promise<{ manifest: T
|
|
|
106
109
|
const prepared = await prepareTaskExecutionContext(input, manifest, streamBridge);
|
|
107
110
|
if (prepared.kind === "cancelled") return prepared.result;
|
|
108
111
|
const ctx = prepared.ctx;
|
|
112
|
+
// ── U13 (upgrade spec 2026-10-09): deterministic verifier pre-gate ──
|
|
113
|
+
// For verifier-role tasks, run the deterministic gate (typecheck /
|
|
114
|
+
// test:critical via package.json scripts, subprocess executor) BEFORE
|
|
115
|
+
// spawning the LLM verifier worker:
|
|
116
|
+
// - gate PASS → SKIP the spawn: terminalize the task as completed
|
|
117
|
+
// with the gate evidence as its result (spec acceptance: "một run
|
|
118
|
+
// verification với gate xanh không spawn LLM verifier");
|
|
119
|
+
// - gate FAILED/INCONCLUSIVE (+ U6B classifier layer on the latter)
|
|
120
|
+
// → spawn the LLM verifier as today, with the gate results appended
|
|
121
|
+
// to its prompt (it diagnoses; it does not re-derive exit codes).
|
|
122
|
+
// Disabled by PI_CREW_VERIFY_GATE=0; scaffold runtime never gates.
|
|
123
|
+
if (ctx.task.role === "verifier" && ctx.runtimeKind !== "scaffold" && resolveVerifyGateEnabled()) {
|
|
124
|
+
const preGate = await runVerifierGatePreGate({
|
|
125
|
+
cwd: ctx.task.cwd,
|
|
126
|
+
signal: input.signal,
|
|
127
|
+
classifier: {
|
|
128
|
+
enabled: resolveClassifierEnabled(input.runtimeConfig?.classifierEnabled),
|
|
129
|
+
modelRegistry: input.modelRegistry,
|
|
130
|
+
classifierModel: resolveClassifierModel(input.runtimeConfig?.classifierModel),
|
|
131
|
+
eventsPath: manifest.eventsPath,
|
|
132
|
+
runId: manifest.runId,
|
|
133
|
+
taskId: ctx.task.id,
|
|
134
|
+
},
|
|
135
|
+
metricRegistry: input.metricRegistry,
|
|
136
|
+
});
|
|
137
|
+
// Diagnostic event (task.verifier_pre_gate — U6B's event type; the
|
|
138
|
+
// TEAM_EVENT_TYPES registry entry stays report-only pending a
|
|
139
|
+
// src/state/** lane, same as classifier-pre-gate's own appends).
|
|
140
|
+
appendEventFireAndForget(manifest.eventsPath, {
|
|
141
|
+
type: "task.verifier_pre_gate",
|
|
142
|
+
runId: manifest.runId,
|
|
143
|
+
taskId: ctx.task.id,
|
|
144
|
+
message: preGate.decision.skipVerifier
|
|
145
|
+
? `Deterministic verify_gate ${preGate.outcome.verdict} — LLM verifier spawn SKIPPED (${preGate.decision.reason})`
|
|
146
|
+
: `Deterministic verify_gate ${preGate.outcome.verdict} — LLM verifier spawns with gate context (${preGate.decision.reason})`,
|
|
147
|
+
data: {
|
|
148
|
+
verdict: preGate.outcome.verdict,
|
|
149
|
+
decision: preGate.decision,
|
|
150
|
+
evidenceLines: preGate.outcome.evidenceLines,
|
|
151
|
+
...(preGate.preGate ? { classifierAction: preGate.preGate.action } : {}),
|
|
152
|
+
},
|
|
153
|
+
});
|
|
154
|
+
if (preGate.decision.skipVerifier) {
|
|
155
|
+
// Terminalize WITHOUT spawning: synthesize the branch result (gate
|
|
156
|
+
// summary as the task result artifact + stdout) and reuse the full
|
|
157
|
+
// finalizer — events, hooks, persistence, verification evidence.
|
|
158
|
+
const summary = renderVerifyGateSummary(preGate.outcome, preGate.decision.reason);
|
|
159
|
+
const resultArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
160
|
+
kind: "result",
|
|
161
|
+
relativePath: `results/${ctx.task.id}.txt`,
|
|
162
|
+
content: `${summary}\n`,
|
|
163
|
+
producer: ctx.task.id,
|
|
164
|
+
});
|
|
165
|
+
ctx.task = {
|
|
166
|
+
...ctx.task,
|
|
167
|
+
// Provenance: this verdict came from the deterministic gate, not a
|
|
168
|
+
// model attempt (U6B mount note: evaluator/model provenance marker).
|
|
169
|
+
diagnostics: {
|
|
170
|
+
...(ctx.task.diagnostics ?? {}),
|
|
171
|
+
verifyGate: {
|
|
172
|
+
verdict: preGate.outcome.verdict,
|
|
173
|
+
reason: preGate.decision.reason,
|
|
174
|
+
evidenceLines: preGate.outcome.evidenceLines,
|
|
175
|
+
},
|
|
176
|
+
},
|
|
177
|
+
};
|
|
178
|
+
return await finalizeTaskResult(ctx, {
|
|
179
|
+
resultArtifact,
|
|
180
|
+
logArtifact: undefined,
|
|
181
|
+
transcriptArtifact: undefined,
|
|
182
|
+
exitCode: 0,
|
|
183
|
+
error: undefined,
|
|
184
|
+
modelAttempts: [],
|
|
185
|
+
parsedOutput: undefined,
|
|
186
|
+
finalStdout: summary,
|
|
187
|
+
transcriptPath: undefined,
|
|
188
|
+
terminalEvidence: [],
|
|
189
|
+
startupEvidence: ctx.startupEvidence,
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
if (preGate.promptContext) {
|
|
193
|
+
// Spawn proceeds — attach the gate results to the verifier prompt
|
|
194
|
+
// (both channels: full prompt + the user-message half). The prompt
|
|
195
|
+
// artifact was written in pre-execution with the PRE-gate prompt;
|
|
196
|
+
// rewrite it so the persisted artifact matches what the worker saw.
|
|
197
|
+
ctx.prompt = `${ctx.prompt}\n\n${preGate.promptContext}`;
|
|
198
|
+
if (ctx.userPrompt !== undefined) ctx.userPrompt = `${ctx.userPrompt}\n\n${preGate.promptContext}`;
|
|
199
|
+
if (ctx.promptArtifact) {
|
|
200
|
+
ctx.promptArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
201
|
+
kind: "prompt",
|
|
202
|
+
// Same relative-path convention pre-execution used
|
|
203
|
+
// (prompts/<taskId>.md) — ArtifactDescriptor carries only the
|
|
204
|
+
// absolute path, so the rewrite re-derives the relative path.
|
|
205
|
+
relativePath: `prompts/${ctx.task.id}.md`,
|
|
206
|
+
content: `${ctx.prompt}\n`,
|
|
207
|
+
producer: ctx.task.id,
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
109
212
|
// Destructure mutable fields for branch closures + sync-back
|
|
110
213
|
let task = ctx.task;
|
|
111
214
|
let tasks = ctx.tasks;
|