pi-crew 0.11.2 → 0.11.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +83 -1
- package/README.md +1 -1
- package/dist/index.mjs +492 -493
- package/docs/actions-reference.md +22 -0
- package/index.ts +9 -5
- package/package.json +12 -9
- package/scripts/analyze-run.mjs +6 -3
- package/skills/real-test-pi-crew/SKILL.md +20 -12
- package/src/extension/async-notifier.ts +32 -8
- package/src/extension/crew-input-router.ts +9 -2
- package/src/extension/plan-orchestrate.ts +19 -3
- package/src/extension/registration/commands/shared.ts +11 -10
- package/src/extension/registration/foreground-run-controller.ts +2 -2
- package/src/extension/registration/lazy-configurers.ts +4 -1
- package/src/extension/registration/lifecycle-handlers.ts +16 -10
- package/src/extension/registration/lifecycle.ts +2 -0
- package/src/extension/registration/observability.ts +27 -18
- package/src/extension/registration/team-tool.ts +9 -2
- package/src/extension/registration/wire-cross-extension.ts +2 -0
- package/src/extension/team-manager-command.ts +3 -2
- package/src/extension/team-tool/dispatch/index.ts +4 -3
- package/src/extension/team-tool/dispatch/status.ts +7 -3
- package/src/extension/team-tool/goal-wrap.ts +7 -0
- package/src/extension/team-tool/goal.ts +12 -0
- package/src/extension/team-tool/inspect.ts +90 -0
- package/src/extension/team-tool/parallel-dispatch.ts +16 -1
- package/src/extension/team-tool/run-intent.ts +1 -1
- package/src/extension/team-tool/run.ts +1 -1
- package/src/runtime/async-runner.ts +36 -0
- package/src/runtime/broker/crew-broker.ts +61 -62
- package/src/runtime/broker/delegate/delegate-event.ts +12 -8
- package/src/runtime/broker/protocol/connection-state.ts +10 -0
- package/src/runtime/broker/protocol/lock-busy.ts +76 -0
- package/src/runtime/delegate-spawn.ts +16 -2
- package/src/runtime/deterministic-ast.ts +14 -2
- package/src/runtime/goal-workflow/dynamic-workflow-context.ts +78 -58
- package/src/runtime/goal-workflow/dynamic-workflow-runner.ts +14 -2
- package/src/runtime/goal-workflow/goal-loop-runner.ts +32 -0
- package/src/runtime/group-join.ts +11 -3
- package/src/runtime/live-session/live-session-runtime.ts +1 -1
- package/src/runtime/model/runtime-warmup.ts +2 -0
- package/src/runtime/output/output-validator.ts +10 -5
- package/src/runtime/plan-approval.ts +14 -1
- package/src/runtime/recovery/crash-recovery.ts +80 -52
- package/src/runtime/stale-reconciler.ts +8 -0
- package/src/runtime/subagent-manager.ts +43 -13
- package/src/runtime/task-output-context.ts +11 -1
- package/src/runtime/task-runner/child-executor.ts +17 -6
- package/src/runtime/task-runner/post-execution.ts +46 -14
- package/src/runtime/task-runner/pre-execution.ts +29 -21
- package/src/runtime/task-runner/prompt-builder.ts +8 -0
- package/src/runtime/task-runner/retrieval-orchestrator.ts +36 -14
- package/src/runtime/task-runner.ts +7 -0
- package/src/runtime/team-runner.ts +4 -1
- package/src/runtime/verification/verification-worktree.ts +3 -0
- package/src/schema/team-tool-schema.ts +2 -1
- package/src/state/coordination/mailbox.ts +7 -2
- package/src/state/stores/state-store.ts +8 -1
- package/src/ui/dashboard-panes/agents-pane.ts +11 -8
- package/src/ui/inline-panel/index.ts +1 -1
- package/src/ui/render-diff.ts +19 -6
- package/src/ui/run-dashboard.ts +11 -1
- package/src/ui/run-snapshot-cache.ts +9 -3
- package/src/ui/widget/task-list.ts +8 -5
- package/src/ui/widget/widget-model.ts +3 -1
- package/tsconfig.json +2 -0
- package/src/ui/card-colors.ts +0 -168
- package/src/utils/completion-dedupe.ts +0 -63
- package/src/utils/resolve-shell.ts +0 -36
|
@@ -29,6 +29,7 @@ import type { TeamConfig } from "../../teams/team-config.ts";
|
|
|
29
29
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
30
30
|
import type { WorkflowConfig } from "../../workflows/workflow-config.ts";
|
|
31
31
|
import { resolveCrewRuntime } from "../model/runtime-resolver.ts";
|
|
32
|
+
import { registryFromModelContext } from "../model/session-model.ts";
|
|
32
33
|
import { executeTeamRun } from "../team-runner.ts";
|
|
33
34
|
import { compareSnapshot, snapshotManifests } from "../verification/verification-integrity.ts";
|
|
34
35
|
import { acquireWorkspaceLock, type WorkspaceLockHandle } from "../workspace-lock.ts";
|
|
@@ -65,6 +66,29 @@ export interface RunGoalLoopResult {
|
|
|
65
66
|
goalState: GoalLoopState;
|
|
66
67
|
}
|
|
67
68
|
|
|
69
|
+
/**
|
|
70
|
+
* Finding #1 (battery 2026-09-29): re-hydrate the model routing inputs the
|
|
71
|
+
* OUTER goal-loop manifest persisted at dispatch time (captureRunModelContext
|
|
72
|
+
* in goal.ts / goal-wrap.ts). The detached goal-loop runs in the background
|
|
73
|
+
* runner with no ExtensionContext — without this, every turn's executeTeamRun
|
|
74
|
+
* resolves the DEFAULT chain (live: minimax/MiniMax-M3 402 while the parent
|
|
75
|
+
* session ran zai/glm-5.3). Mirrors background-runner's restoredModelRouting().
|
|
76
|
+
*/
|
|
77
|
+
function restoredTurnModelRouting(manifest: TeamRunManifest): {
|
|
78
|
+
modelOverride?: string;
|
|
79
|
+
parentModel?: string;
|
|
80
|
+
modelRegistry?: { getAvailable: () => unknown[] };
|
|
81
|
+
} {
|
|
82
|
+
const context = manifest.modelContext;
|
|
83
|
+
if (!context) return {};
|
|
84
|
+
const modelRegistry = registryFromModelContext(context);
|
|
85
|
+
return {
|
|
86
|
+
...(context.override ? { modelOverride: context.override } : {}),
|
|
87
|
+
...(context.parentModel ? { parentModel: context.parentModel } : {}),
|
|
88
|
+
...(modelRegistry ? { modelRegistry } : {}),
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
68
92
|
/**
|
|
69
93
|
* The placeholder evaluator for P0: always returns {achieved:false}.
|
|
70
94
|
* Kept for unit tests of the loop's max_turns exit path. The production loop
|
|
@@ -630,6 +654,10 @@ export async function runGoalLoop(input: RunGoalLoopInput): Promise<RunGoalLoopR
|
|
|
630
654
|
workspaceMode: "single",
|
|
631
655
|
ownerSessionId: goal.ownerSessionId,
|
|
632
656
|
runKind: "team-run", // §0a v2 note: turns are normal team-runs; the OUTER loop is goal-loop
|
|
657
|
+
// Finding #1: turns execute in the detached process — carry the outer
|
|
658
|
+
// manifest's model routing snapshot onto each turn manifest too, so
|
|
659
|
+
// status/resume paths see the same routing the turn was dispatched with.
|
|
660
|
+
...(manifest.modelContext ? { modelContext: manifest.modelContext } : {}),
|
|
633
661
|
});
|
|
634
662
|
goal =
|
|
635
663
|
store.patch(
|
|
@@ -689,6 +717,10 @@ export async function runGoalLoop(input: RunGoalLoopInput): Promise<RunGoalLoopR
|
|
|
689
717
|
limits: turnExecutedConfig.limits,
|
|
690
718
|
runtime: turnRuntime,
|
|
691
719
|
runtimeConfig: turnExecutedConfig.runtime,
|
|
720
|
+
// Finding #1: restore the dispatch-time model routing (see
|
|
721
|
+
// restoredTurnModelRouting) — executeTeamRun in THIS process has no
|
|
722
|
+
// ExtensionContext to inherit the session model from.
|
|
723
|
+
...restoredTurnModelRouting(manifest),
|
|
692
724
|
reliability: turnExecutedConfig.reliability,
|
|
693
725
|
workspaceId: goal.ownerSessionId ?? goal.cwd,
|
|
694
726
|
signal,
|
|
@@ -59,9 +59,17 @@ export function deliverGroupJoin(input: {
|
|
|
59
59
|
}): CrewGroupJoinDelivery | undefined {
|
|
60
60
|
if (!shouldGroupJoin(input.mode, input.batch)) return undefined;
|
|
61
61
|
const taskIds = input.batch.map((task) => task.id);
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
// RR-021 WI-4.3d: first-wins Map — the old taskIds.map(id => allTasks.find())
|
|
63
|
+
// was O(batch×allTasks). NOT a raw input.batch pass-through: callers may
|
|
64
|
+
// hold stale batch snapshots, and the only audited production caller
|
|
65
|
+
// (team-runner closeout) passes fresh references — but the Map keeps the
|
|
66
|
+
// exact `.find` first-match semantics for every caller (incl. tests) while
|
|
67
|
+
// staying O(n+m). Batch order is preserved (map follows taskIds).
|
|
68
|
+
const taskById = new Map<string, TeamTaskState>();
|
|
69
|
+
for (const task of input.allTasks) {
|
|
70
|
+
if (!taskById.has(task.id)) taskById.set(task.id, task);
|
|
71
|
+
}
|
|
72
|
+
const latest = taskIds.map((id) => taskById.get(id)).filter((task): task is TeamTaskState => Boolean(task));
|
|
65
73
|
const completed = statusList(latest, "completed");
|
|
66
74
|
const failed = statusList(latest, "failed");
|
|
67
75
|
const skipped = statusList(latest, "skipped");
|
|
@@ -96,7 +96,7 @@ let liveSessionModulePromise: Promise<LiveSessionModule> | undefined;
|
|
|
96
96
|
let experimentalWarned = false;
|
|
97
97
|
function loadLiveSessionModule(): Promise<LiveSessionModule> {
|
|
98
98
|
if (!liveSessionModulePromise) {
|
|
99
|
-
liveSessionModulePromise = import("@earendil-works/pi-coding-agent") as unknown as Promise<LiveSessionModule>;
|
|
99
|
+
liveSessionModulePromise = import("@earendil-works/pi-coding-agent") as unknown as Promise<LiveSessionModule>; // LAZY: defer pi-coding-agent live-session module until first use
|
|
100
100
|
}
|
|
101
101
|
return liveSessionModulePromise;
|
|
102
102
|
}
|
|
@@ -124,6 +124,7 @@ export function startRuntimeWarmup(): void {
|
|
|
124
124
|
for (const spec of HOT_MODULE_SPECIFIERS) {
|
|
125
125
|
imports.push(
|
|
126
126
|
import(new URL(spec, import.meta.url).href).catch(() => {
|
|
127
|
+
// LAZY: best-effort warmup import, failures tolerated
|
|
127
128
|
// swallow — never block registration on a warmup failure
|
|
128
129
|
}),
|
|
129
130
|
);
|
|
@@ -131,6 +132,7 @@ export function startRuntimeWarmup(): void {
|
|
|
131
132
|
for (const dep of HOT_PEER_DEPS) {
|
|
132
133
|
imports.push(
|
|
133
134
|
import(dep).catch(() => {
|
|
135
|
+
// LAZY: best-effort warmup import, failures tolerated
|
|
134
136
|
// peer dep may be absent (optional dep) — swallow
|
|
135
137
|
}),
|
|
136
138
|
);
|
|
@@ -123,10 +123,15 @@ export function validateWorkerOutput(role: string, output: string): OutputValida
|
|
|
123
123
|
*
|
|
124
124
|
* A trimmed non-empty line counts as log noise ONLY if it matches one of:
|
|
125
125
|
* 1. a bracket-tag extension log line: `[oc-go] ...`, `[pi-qwen-mm] ...`,
|
|
126
|
-
* `[pi-qwen-mm] [core] [stderr]
|
|
127
|
-
*
|
|
128
|
-
* `[
|
|
129
|
-
*
|
|
126
|
+
* `[pi-qwen-mm] [core] [stderr] ...`, and — Finding #2 (2026-09-29
|
|
127
|
+
* battery) — colon-bearing extension tags like
|
|
128
|
+
* `[pi-crew:crash-recovery.reconcileStaleRuns] ...` or
|
|
129
|
+
* `[pi-crew:crew-vibes.publish-quota-status] ...`. The leading tag
|
|
130
|
+
* starts with a lowercase char (deliberately excludes capitalized
|
|
131
|
+
* bracketed prose like `[Note] ...` / `[Note: ...]`); the rest of the tag
|
|
132
|
+
* may carry `:`, `-`, `_`, `.`, digits, and inner capitals (extension
|
|
133
|
+
* subsystem names are camelCase, e.g. `reconcileStaleRuns`), optionally
|
|
134
|
+
* followed by known sub-tags (core/mcp/stderr/stdout/warn/info/error/debug);
|
|
130
135
|
* 2. a Python `warnings.warn(` line (deprecation-warning continuation);
|
|
131
136
|
* 3. a timestamped logging line: `2026-08-15 21:49:02,986 WARNING ...`.
|
|
132
137
|
*
|
|
@@ -137,7 +142,7 @@ export function validateWorkerOutput(role: string, output: string): OutputValida
|
|
|
137
142
|
* (post-execution.ts) applies the two-gate rule (authoritative output
|
|
138
143
|
* sources empty AND artifact log-noise-only) before failing a task.
|
|
139
144
|
*/
|
|
140
|
-
const BRACKET_TAG_LOG_LINE = /^\[[a-z0-9][a-
|
|
145
|
+
const BRACKET_TAG_LOG_LINE = /^\[[a-z0-9][a-zA-Z0-9_.:\-]*\](?:\s*\[(?:core|mcp|stderr|stdout|warn|info|error|debug)\])*(?:\s.*)?$/;
|
|
141
146
|
const PYTHON_WARNING_LINE = /(?:^|\s)warnings\.warn\(/;
|
|
142
147
|
const TIMESTAMPED_LOG_LINE = /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:[.,]\d{1,6})?(?:Z|[+-]\d{2}:?\d{2})?\s+[A-Za-z]+\b/;
|
|
143
148
|
|
|
@@ -70,7 +70,20 @@ export async function ensurePlanApprovalRequested(manifest: TeamRunManifest, tas
|
|
|
70
70
|
const assessTask = tasks.find((task) => task.stepId === "assess" && task.status === "completed");
|
|
71
71
|
// ROADMAP T1.2: for non-adaptive workflows, fall back to the most recent
|
|
72
72
|
// completed read-only (planning) task as the plan reference.
|
|
73
|
-
|
|
73
|
+
// RR-021 WI-4.3e: allocation-free reverse scan — the old
|
|
74
|
+
// [...tasks].reverse().find() copied the whole array per call.
|
|
75
|
+
// (findLast needs an ES2023 lib bump, which breaks tsgo's RequestInfo
|
|
76
|
+
// ambient resolution — see WI-4.3 evidence note.)
|
|
77
|
+
let planTask = assessTask;
|
|
78
|
+
if (!planTask) {
|
|
79
|
+
for (let i = tasks.length - 1; i >= 0; i--) {
|
|
80
|
+
const t = tasks[i];
|
|
81
|
+
if (t?.status === "completed" && !isMutatingTask(t)) {
|
|
82
|
+
planTask = t;
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
74
87
|
const now = new Date().toISOString();
|
|
75
88
|
|
|
76
89
|
// T2/R4 (ADR-4 §6 producer 3 + §8 dual-write): make sure a PlanRecord
|
|
@@ -4,10 +4,16 @@ import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
|
4
4
|
import { DEFAULT_PATHS } from "../../config/defaults.ts";
|
|
5
5
|
import { appendHookEvent, executeHook } from "../../hooks/registry.ts";
|
|
6
6
|
import type { MetricRegistry } from "../../observability/metric-registry.ts";
|
|
7
|
-
import { discoverRunLockFiles, sweepStaleLocks, withRunLockSync } from "../../state/coordination/locks.ts";
|
|
7
|
+
import { discoverRunLockFiles, sweepStaleLocks, withRunLock, withRunLockSync } from "../../state/coordination/locks.ts";
|
|
8
8
|
import { appendEvent, scanSequence } from "../../state/event-log/event-log.ts";
|
|
9
9
|
import { readActiveRunRegistry, unregisterActiveRun } from "../../state/stores/active-run-registry.ts";
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
loadRunManifestById,
|
|
12
|
+
loadRunManifestByIdAsync,
|
|
13
|
+
saveRunManifest,
|
|
14
|
+
saveRunTasks,
|
|
15
|
+
updateRunStatus,
|
|
16
|
+
} from "../../state/stores/state-store.ts";
|
|
11
17
|
import type { TeamTaskState } from "../../state/types.ts";
|
|
12
18
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
13
19
|
import { projectCrewRoot, userCrewRoot } from "../../utils/paths.ts";
|
|
@@ -18,6 +24,7 @@ import { isWorkerHeartbeatStale } from "../heartbeat/worker-heartbeat.ts";
|
|
|
18
24
|
import { terminateLiveAgentsForRun } from "../live-session/live-agent-manager.ts";
|
|
19
25
|
import type { ManifestCache } from "../manifest-cache.ts";
|
|
20
26
|
import { checkProcessLiveness } from "../process-status.ts";
|
|
27
|
+
import { mapConcurrent } from "../scheduling/parallel-utils.ts";
|
|
21
28
|
import { isIntentionalWait, isPlanApprovalPendingEffective, type ReconcileResult, reconcileStaleRun } from "../stale-reconciler.ts";
|
|
22
29
|
|
|
23
30
|
export interface RecoveryPlan {
|
|
@@ -690,13 +697,12 @@ export function purgeStaleActiveRunIndex(
|
|
|
690
697
|
return { purged, kept };
|
|
691
698
|
}
|
|
692
699
|
|
|
693
|
-
export function reconcileAllStaleRuns(
|
|
700
|
+
export async function reconcileAllStaleRuns(
|
|
694
701
|
cwd: string,
|
|
695
702
|
manifestCache: ManifestCache,
|
|
696
703
|
now = Date.now(),
|
|
697
704
|
currentSessionId?: string,
|
|
698
|
-
): ReconcileResult[] {
|
|
699
|
-
const results: ReconcileResult[] = [];
|
|
705
|
+
): Promise<ReconcileResult[]> {
|
|
700
706
|
// Capture runIds to reconcile BEFORE acquiring locks — avoids TOCTOU between cache iteration and lock acquisition.
|
|
701
707
|
const runIds = manifestCache
|
|
702
708
|
.list(50)
|
|
@@ -708,56 +714,78 @@ export function reconcileAllStaleRuns(
|
|
|
708
714
|
return true;
|
|
709
715
|
})
|
|
710
716
|
.map((m) => m.runId);
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
//
|
|
724
|
-
if (
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
return;
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
if (
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
717
|
+
// RR-021 WI-1.5 (audit C3): serial reconcile → mapConcurrent bound 4.
|
|
718
|
+
// Each runId acquires its OWN run lock file (lockPath() is per-run), so the
|
|
719
|
+
// acquisitions are independent; bound 4 caps fs/CPU contention. Results keep
|
|
720
|
+
// input (snapshot) order — mapConcurrent indexes by item — and per-run
|
|
721
|
+
// semantics (re-read inside lock, plan-approval re-check) are unchanged.
|
|
722
|
+
const perRun = await mapConcurrent(runIds, 4, async (runId): Promise<ReconcileResult[]> => {
|
|
723
|
+
// RR-021 review remediation: per-run error isolation — one corrupt run
|
|
724
|
+
// must not reject the whole sweep (serial baseline threw too, but the
|
|
725
|
+
// SPEC's "independent per-run" intent requires continue-on-error).
|
|
726
|
+
try {
|
|
727
|
+
const cached = manifestCache.get(runId);
|
|
728
|
+
if (!cached) return [];
|
|
729
|
+
const loaded = await loadRunManifestByIdAsync(cwd, runId); // NOTE: best-effort only; concurrent writes may cause inconsistency
|
|
730
|
+
if (!loaded) return [];
|
|
731
|
+
const out: ReconcileResult[] = [];
|
|
732
|
+
// Use lock to prevent race with cancel/status handlers modifying the same run.
|
|
733
|
+
// Same v0.9.26 lock family as the old sync acquisition — interop-safe.
|
|
734
|
+
await withRunLock(loaded.manifest, async () => {
|
|
735
|
+
// Re-read inside lock to get freshest data
|
|
736
|
+
const fresh = await loadRunManifestByIdAsync(cwd, runId); // NOTE: inside withRunLock - consistent read
|
|
737
|
+
if (!fresh || (fresh.manifest.status !== "running" && fresh.manifest.status !== "blocked")) return;
|
|
738
|
+
// Belt-and-suspenders: reconcileStaleRun itself guards this, but the run
|
|
739
|
+
// may have flipped to blocked+plan-approval between cache-list and lock
|
|
740
|
+
// acquisition — re-check the freshest manifest under the lock.
|
|
741
|
+
if (isPlanApprovalPendingEffective(fresh.manifest)) {
|
|
742
|
+
out.push({
|
|
743
|
+
runId,
|
|
744
|
+
verdict: "blocked_awaiting_approval",
|
|
745
|
+
repaired: false,
|
|
746
|
+
detail: "Plan approval is pending; stale reconciliation skipped",
|
|
747
|
+
});
|
|
748
|
+
return;
|
|
749
|
+
}
|
|
750
|
+
const result = reconcileStaleRun(fresh.manifest, fresh.tasks, now);
|
|
751
|
+
if (result.repaired || result.verdict === "result_exists") {
|
|
752
|
+
if (result.repairedTasks) {
|
|
753
|
+
// NEW-1 (SDD 2026-09-30 WI-2): persist the FULL task array — the
|
|
754
|
+
// individual-stale branch filters `repairedTasks` down to the stale
|
|
755
|
+
// subset, and saveRunTasks full-overwrites tasks.json, which used to
|
|
756
|
+
// drop every healthy task from disk. `persistTasks` carries the full
|
|
757
|
+
// array; every other branch leaves it unset and falls back here.
|
|
758
|
+
saveRunTasks(fresh.manifest, result.persistTasks ?? result.repairedTasks);
|
|
759
|
+
for (const task of result.repairedTasks) {
|
|
760
|
+
try {
|
|
761
|
+
upsertCrewAgent(fresh.manifest, recordFromTask(fresh.manifest, task, "scaffold"));
|
|
762
|
+
} catch {
|
|
763
|
+
/* non-critical */
|
|
764
|
+
}
|
|
742
765
|
}
|
|
743
766
|
}
|
|
767
|
+
updateRunStatus(fresh.manifest, "failed", `Stale run reconciled: ${result.detail}`);
|
|
768
|
+
void terminateLiveAgentsForRun(fresh.manifest.runId, "failed", appendEvent, fresh.manifest.eventsPath).catch((error) =>
|
|
769
|
+
logInternalError("crash-recovery.reconcile.terminate", error, `runId=${fresh.manifest.runId}`, "warn"),
|
|
770
|
+
);
|
|
771
|
+
appendEvent(fresh.manifest.eventsPath, {
|
|
772
|
+
type: "crew.run.reconciled_stale",
|
|
773
|
+
runId,
|
|
774
|
+
message: result.detail,
|
|
775
|
+
data: { verdict: result.verdict },
|
|
776
|
+
});
|
|
744
777
|
}
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
if (result.verdict !== "healthy") {
|
|
757
|
-
results.push(result);
|
|
758
|
-
}
|
|
759
|
-
});
|
|
760
|
-
}
|
|
778
|
+
if (result.verdict !== "healthy") {
|
|
779
|
+
out.push(result);
|
|
780
|
+
}
|
|
781
|
+
});
|
|
782
|
+
return out;
|
|
783
|
+
} catch (error) {
|
|
784
|
+
logInternalError("crash-recovery.reconcileStaleRuns", error, `runId=${runId}`, "warn");
|
|
785
|
+
return [];
|
|
786
|
+
}
|
|
787
|
+
});
|
|
788
|
+
const results = perRun.flat();
|
|
761
789
|
// US-002 (2026-09-22): structured stale-lock sweep. Runs after reconciliation
|
|
762
790
|
// so locks whose holder died (kill -9 / crash) do not linger. The sweep is
|
|
763
791
|
// strictly safer than acquire-time steal (stale AND holder-dead only), so a
|
|
@@ -44,6 +44,13 @@ export interface ReconcileResult {
|
|
|
44
44
|
detail: string;
|
|
45
45
|
/** Repaired task state, returned to a locked caller for persistence. */
|
|
46
46
|
repairedTasks?: TeamTaskState[];
|
|
47
|
+
/** NEW-1 (SDD 2026-09-30 WI-2): what the caller should PERSIST — the FULL
|
|
48
|
+
* task array, including tasks the repair did not touch. Only the
|
|
49
|
+
* individual-stale branch sets it (its `repairedTasks` is filtered down to
|
|
50
|
+
* the stale subset, which must NOT be full-overwrite-persisted or healthy
|
|
51
|
+
* tasks vanish from tasks.json). Callers fall back to `repairedTasks`
|
|
52
|
+
* when absent — every other branch already returns the full array there. */
|
|
53
|
+
persistTasks?: TeamTaskState[];
|
|
47
54
|
}
|
|
48
55
|
|
|
49
56
|
/**
|
|
@@ -490,6 +497,7 @@ export function reconcileStaleRun(manifest: TeamRunManifest, tasks: TeamTaskStat
|
|
|
490
497
|
repaired: true,
|
|
491
498
|
detail: `No PID; ${staleTaskIds.length} individually stale task(s) repaired: ${staleTaskIds.join(", ")}`,
|
|
492
499
|
repairedTasks: repaired.filter((t) => staleTaskIds.includes(t.id)),
|
|
500
|
+
persistTasks: repaired, // NEW-1: persist FULL array (data-loss fix — SDD WI-2)
|
|
493
501
|
};
|
|
494
502
|
}
|
|
495
503
|
// Fall through: no recent activity but not all tasks stale enough yet.
|
|
@@ -342,16 +342,43 @@ export class SubagentManager {
|
|
|
342
342
|
}
|
|
343
343
|
}
|
|
344
344
|
|
|
345
|
-
async waitForRecord(id: string): Promise<SubagentRecord | undefined> {
|
|
345
|
+
async waitForRecord(id: string, timeoutMs = 300_000): Promise<SubagentRecord | undefined> {
|
|
346
|
+
// RR-021 WI-4.3i: bounded wait. A record with no promise whose status
|
|
347
|
+
// stays running/blocked used to spin forever (100ms sleep loop). On
|
|
348
|
+
// deadline expiry we return the CURRENT record — never undefined, which
|
|
349
|
+
// would be ambiguous with 'no such record'. Callers can inspect
|
|
350
|
+
// record.status to see the wait timed out on a still-active record.
|
|
351
|
+
const deadline = Date.now() + timeoutMs;
|
|
346
352
|
while (true) {
|
|
347
353
|
const record = this.records.get(id);
|
|
348
354
|
if (!record) return undefined;
|
|
349
355
|
if (record.status !== "running" && record.status !== "queued") return record;
|
|
350
|
-
if (
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
356
|
+
if (Date.now() > deadline) return record;
|
|
357
|
+
if (record.promise) {
|
|
358
|
+
// Race the run promise against the remaining deadline — a wedged
|
|
359
|
+
// promise must not block the deadline check.
|
|
360
|
+
const remaining = deadline - Date.now();
|
|
361
|
+
let timer: NodeJS.Timeout | undefined;
|
|
362
|
+
try {
|
|
363
|
+
await Promise.race([
|
|
364
|
+
record.promise.catch((error) => {
|
|
365
|
+
logInternalError("subagent-manager.waitForRecord", error, `id=${id}`);
|
|
366
|
+
}),
|
|
367
|
+
// RR-021 review remediation: this timer MUST stay ref'd. unref() let the
|
|
368
|
+
// event loop drain before the deadline fired (node:test then cancels the
|
|
369
|
+
// awaiting test — "Promise resolution is still pending but the event loop
|
|
370
|
+
// has already resolved"). Always clearTimeout'd in the finally below, so
|
|
371
|
+
// ref'ing cannot leak (knowledge.md OwnedProcess timer guidance).
|
|
372
|
+
new Promise<void>((resolve) => {
|
|
373
|
+
timer = setTimeout(resolve, remaining);
|
|
374
|
+
}),
|
|
375
|
+
]);
|
|
376
|
+
} finally {
|
|
377
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
378
|
+
}
|
|
379
|
+
} else {
|
|
380
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(100, Math.max(1, deadline - Date.now()))));
|
|
381
|
+
}
|
|
355
382
|
}
|
|
356
383
|
}
|
|
357
384
|
|
|
@@ -477,19 +504,22 @@ export class SubagentManager {
|
|
|
477
504
|
}
|
|
478
505
|
|
|
479
506
|
private async pollRunToTerminal(cwd: string, record: SubagentRecord): Promise<void> {
|
|
480
|
-
// Safety: max 30 minutes
|
|
481
|
-
// polling if manifest file is deleted or run state becomes unrecoverable.
|
|
482
|
-
|
|
483
|
-
|
|
507
|
+
// Safety: max 30 minutes of WALL-CLOCK polling to prevent infinite
|
|
508
|
+
// polling if the manifest file is deleted or run state becomes unrecoverable.
|
|
509
|
+
// (RR-021 WI-4.3h: was MAX_POLL_COUNT=1800, which silently assumed a 1s
|
|
510
|
+
// pollIntervalMs — but the interval is configurable, so a 5s interval
|
|
511
|
+
// polled for 2.5h while a 200ms interval timed out after 6 minutes.)
|
|
512
|
+
const POLL_DEADLINE_MS = 30 * 60 * 1000;
|
|
513
|
+
const pollDeadline = Date.now() + POLL_DEADLINE_MS;
|
|
484
514
|
while (record.runId && (record.status === "running" || record.status === "blocked")) {
|
|
485
|
-
if (
|
|
515
|
+
if (Date.now() > pollDeadline) {
|
|
486
516
|
logInternalError(
|
|
487
517
|
"subagent-manager.poll-timeout",
|
|
488
|
-
new Error(`pollRunToTerminal exceeded ${
|
|
518
|
+
new Error(`pollRunToTerminal exceeded ${POLL_DEADLINE_MS}ms wall-clock for runId=${record.runId}`),
|
|
489
519
|
`id=${record.id}`,
|
|
490
520
|
);
|
|
491
521
|
record.status = "error";
|
|
492
|
-
record.error = `Poll timeout: run did not reach terminal state after ${
|
|
522
|
+
record.error = `Poll timeout: run did not reach terminal state after ${POLL_DEADLINE_MS}ms (wall-clock)`;
|
|
493
523
|
record.completedAt = Date.now();
|
|
494
524
|
savePersistedSubagentRecord(cwd, record);
|
|
495
525
|
return;
|
|
@@ -526,6 +526,16 @@ export function collectDependencyOutputContext(
|
|
|
526
526
|
return { dependencies: trimmedDependencies, sharedReads };
|
|
527
527
|
}
|
|
528
528
|
|
|
529
|
+
// ADR-5 §trust-fence: dependency output is DATA, never instructions. Mirror the
|
|
530
|
+
// ask/delegate seams in src/prompt/prompt-runtime.ts (:323, :426, :589): strip
|
|
531
|
+
// control chars and neutralize a smuggled closing fence tag so worker-controlled
|
|
532
|
+
// content can never close the <dependency-context> fence early (see SDD 2026-09-30 WI-1).
|
|
533
|
+
const DEPENDENCY_CONTROL_CHAR_PATTERN = /[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g;
|
|
534
|
+
|
|
535
|
+
function sanitizeFencedBody(body: string): string {
|
|
536
|
+
return body.replace(DEPENDENCY_CONTROL_CHAR_PATTERN, "").replace(/<\/dependency-context/g, "</dependency-context");
|
|
537
|
+
}
|
|
538
|
+
|
|
529
539
|
export function renderDependencyOutputContext(context: DependencyOutputContext): string {
|
|
530
540
|
const parts: string[] = [];
|
|
531
541
|
if (context.dependencies.length) {
|
|
@@ -565,7 +575,7 @@ export function renderDependencyOutputContext(context: DependencyOutputContext):
|
|
|
565
575
|
parts.push("", read.content.trim(), "");
|
|
566
576
|
}
|
|
567
577
|
}
|
|
568
|
-
return parts.join("\n").trim();
|
|
578
|
+
return sanitizeFencedBody(parts.join("\n").trim());
|
|
569
579
|
}
|
|
570
580
|
|
|
571
581
|
export function writeTaskSharedOutput(manifest: TeamRunManifest, step: WorkflowStep, task: TeamTaskState): ArtifactDescriptor | undefined {
|
|
@@ -68,7 +68,7 @@ import { DEFAULT_RETRY_POLICY } from "../recovery/retry-executor.ts";
|
|
|
68
68
|
import { runWorker } from "../run-worker.ts";
|
|
69
69
|
import { parseSessionUsage } from "../session-usage.ts";
|
|
70
70
|
import { recordSupervisorContact, supervisorContactFromEvent } from "../supervisor-contact.ts";
|
|
71
|
-
import type { TaskExecutionResult } from "./post-execution.ts";
|
|
71
|
+
import type { ResultSource, TaskExecutionResult } from "./post-execution.ts";
|
|
72
72
|
import type { StreamBridgeHandle, TaskExecutionContext } from "./pre-execution.ts";
|
|
73
73
|
import { applyAgentProgressEvent, applyUsageToProgress, progressEventSummary, shouldFlushProgressEvent } from "./progress.ts";
|
|
74
74
|
import { cleanResultText, isFinalChildEvent } from "./result-utils.ts";
|
|
@@ -987,6 +987,20 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
|
|
|
987
987
|
});
|
|
988
988
|
}
|
|
989
989
|
}
|
|
990
|
+
// Finding #2 (2026-09-29 battery): tag WHICH fallback branch produced the
|
|
991
|
+
// artifact so post-execution can fail stderr/none-sourced results
|
|
992
|
+
// deterministically — the noise classifier alone is not a verdict oracle.
|
|
993
|
+
// Order/semantics are identical to the previous `??` chain: first branch
|
|
994
|
+
// whose cleaned text is non-empty wins; otherwise the placeholder ("none").
|
|
995
|
+
const resultCandidates: ReadonlyArray<{ source: ResultSource; text: string | undefined }> = [
|
|
996
|
+
{ source: "rawFinalText", text: cleanResultText(rawFinalText) },
|
|
997
|
+
{ source: "finalText", text: cleanResultText(parsedOutput?.finalText) },
|
|
998
|
+
{ source: "stdout", text: cleanResultText(finalStdout) },
|
|
999
|
+
{ source: "stderr", text: cleanResultText(finalStderr) },
|
|
1000
|
+
{ source: "findings", text: cleanResultText(intermediateFindings) },
|
|
1001
|
+
];
|
|
1002
|
+
const usedResult = resultCandidates.find((candidate) => candidate.text);
|
|
1003
|
+
const resultSource: ResultSource = usedResult?.source ?? "none";
|
|
990
1004
|
const resultArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
991
1005
|
kind: "result",
|
|
992
1006
|
relativePath: `results/${task.id}.txt`,
|
|
@@ -995,14 +1009,10 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
|
|
|
995
1009
|
// transcript's 16K compaction — this is the authoritative worker output.
|
|
996
1010
|
// Fall back to transcript-derived finalText, then stdout/stderr, so a
|
|
997
1011
|
// missing raw capture (mock/error path) never yields empty/garbage.
|
|
998
|
-
|
|
999
|
-
cleanResultText(parsedOutput?.finalText) ??
|
|
1000
|
-
cleanResultText(finalStdout) ??
|
|
1001
|
-
cleanResultText(finalStderr) ??
|
|
1012
|
+
usedResult?.text ??
|
|
1002
1013
|
// #7 hardening: if all real output paths are empty (worker exhausted
|
|
1003
1014
|
// budget on tool calls, no assistant text), use intermediate findings.
|
|
1004
1015
|
// intermediateFindings captures the last N tool-result display lines.
|
|
1005
|
-
cleanResultText(intermediateFindings) ??
|
|
1006
1016
|
"(no output)",
|
|
1007
1017
|
producer: task.id,
|
|
1008
1018
|
});
|
|
@@ -1094,6 +1104,7 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
|
|
|
1094
1104
|
parsedOutput,
|
|
1095
1105
|
finalStdout,
|
|
1096
1106
|
rawFinalText,
|
|
1107
|
+
resultSource,
|
|
1097
1108
|
transcriptPath,
|
|
1098
1109
|
terminalEvidence,
|
|
1099
1110
|
startupEvidence,
|
|
@@ -85,8 +85,18 @@ export interface TaskExecutionResult {
|
|
|
85
85
|
transcriptPath: string | undefined;
|
|
86
86
|
terminalEvidence: OperationTerminalEvidence[];
|
|
87
87
|
startupEvidence: import("../heartbeat/worker-startup.ts").WorkerStartupEvidence;
|
|
88
|
+
/** Which child-executor fallback branch produced the result artifact
|
|
89
|
+
* content (rawFinalText → finalText → stdout → stderr → findings →
|
|
90
|
+
* "(no output)"). Finding #2 (2026-09-29 battery): "stderr"/"none" mean
|
|
91
|
+
* every authoritative worker output source was empty — finalizeTaskResult
|
|
92
|
+
* fails those DETERMINISTICALLY; the stderr-noise classifier stays a
|
|
93
|
+
* defense layer, never the verdict oracle. */
|
|
94
|
+
resultSource?: ResultSource;
|
|
88
95
|
}
|
|
89
96
|
|
|
97
|
+
/** Result-artifact provenance tags (see {@link TaskExecutionResult.resultSource}). */
|
|
98
|
+
export type ResultSource = "rawFinalText" | "finalText" | "stdout" | "stderr" | "findings" | "none";
|
|
99
|
+
|
|
90
100
|
/** Final persisted task state returned by finalizeTaskResult. */
|
|
91
101
|
export interface FinalTaskResult {
|
|
92
102
|
manifest: TeamRunManifest;
|
|
@@ -121,6 +131,7 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
|
|
|
121
131
|
const skillNames = ctx.skillNames;
|
|
122
132
|
const skillPaths = ctx.skillPaths;
|
|
123
133
|
const promptArtifact = ctx.promptArtifact;
|
|
134
|
+
const breakdownArtifact = ctx.breakdownArtifact;
|
|
124
135
|
const inputsArtifact = ctx.inputsArtifact;
|
|
125
136
|
const skillArtifact = ctx.skillArtifact;
|
|
126
137
|
const coordinationArtifact = ctx.coordinationArtifact;
|
|
@@ -328,24 +339,39 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
|
|
|
328
339
|
// result ALWAYS surfaces in at least one authoritative source);
|
|
329
340
|
// gate 2 — the persisted artifact is empty/'(no output)'/whitespace OR
|
|
330
341
|
// isStderrOnlyResult says every line is strict log noise.
|
|
331
|
-
//
|
|
332
|
-
//
|
|
333
|
-
//
|
|
342
|
+
// Finding #2 (2026-09-29 battery): gate 2 alone is not a verdict oracle —
|
|
343
|
+
// the strict-noise classifier missed colon-bearing/camelCase extension
|
|
344
|
+
// tags (`[pi-crew:crash-recovery.reconcileStaleRuns]`, lock-contention
|
|
345
|
+
// noise about a SIBLING run), so attempts whose stderr carried them
|
|
346
|
+
// completed tasks with 100%-noise artifacts while sibling attempts on the
|
|
347
|
+
// same runs failed — verdict divergence depended on per-attempt stderr
|
|
348
|
+
// composition. The child-executor fallback chain now tags the winning
|
|
349
|
+
// branch (resultSource); "stderr"/"none"-sourced artifacts fail
|
|
350
|
+
// UNCONDITIONALLY (classifier-independent), making empty/stderr-only
|
|
351
|
+
// results never complete a task, uniformly across roles.
|
|
352
|
+
// A read error on the artifact is NOT a failure (conservative) — unless
|
|
353
|
+
// the source tag already proves the artifact is stderr/none-sourced.
|
|
354
|
+
// Mirrors the mutation-guard fail-mode precedent: error marker + exitCode
|
|
355
|
+
// bump + last modelAttempt success:false → status flips to "failed" (retryable).
|
|
334
356
|
if (!error) {
|
|
335
357
|
const finalTextEmpty = !parsedOutput?.finalText?.trim();
|
|
336
358
|
const finalStdoutEmpty = !finalStdout?.trim();
|
|
337
|
-
|
|
359
|
+
const resultSource = execResult.resultSource;
|
|
360
|
+
const degradedResultSource = resultSource === "stderr" || resultSource === "none";
|
|
361
|
+
if (finalTextEmpty && finalStdoutEmpty && (resultArtifact?.path || degradedResultSource)) {
|
|
338
362
|
let artifactContent: string | undefined;
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
363
|
+
if (resultArtifact?.path) {
|
|
364
|
+
try {
|
|
365
|
+
artifactContent = readFileSync(resultArtifact.path, "utf8");
|
|
366
|
+
} catch {
|
|
367
|
+
artifactContent = undefined; // unreadable artifact — do not fail on read errors
|
|
368
|
+
}
|
|
343
369
|
}
|
|
344
|
-
if (artifactContent !== undefined) {
|
|
345
|
-
const trimmedArtifact = artifactContent.trim();
|
|
370
|
+
if (artifactContent !== undefined || degradedResultSource) {
|
|
371
|
+
const trimmedArtifact = (artifactContent ?? "").trim();
|
|
346
372
|
const emptyArtifact = trimmedArtifact === "" || trimmedArtifact === "(no output)";
|
|
347
|
-
const stderrOnlyArtifact = !emptyArtifact && isStderrOnlyResult(artifactContent);
|
|
348
|
-
if (emptyArtifact || stderrOnlyArtifact) {
|
|
373
|
+
const stderrOnlyArtifact = !emptyArtifact && isStderrOnlyResult(artifactContent ?? "");
|
|
374
|
+
if (degradedResultSource || emptyArtifact || stderrOnlyArtifact) {
|
|
349
375
|
error = "Result artifact is empty or stderr-only (failureCause: empty-or-stderr-only-result)";
|
|
350
376
|
exitCode = exitCode === 0 ? 1 : exitCode;
|
|
351
377
|
if (modelAttempts?.length) {
|
|
@@ -359,7 +385,11 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
|
|
|
359
385
|
structurePreserved: false,
|
|
360
386
|
issues: [
|
|
361
387
|
`empty-or-stderr-only-result: ${
|
|
362
|
-
|
|
388
|
+
degradedResultSource && !emptyArtifact && !stderrOnlyArtifact
|
|
389
|
+
? `result artifact was sourced from the ${resultSource} fallback (no authoritative worker output)`
|
|
390
|
+
: emptyArtifact
|
|
391
|
+
? "result artifact is empty"
|
|
392
|
+
: "result artifact contains only stderr/session-log noise"
|
|
363
393
|
}`,
|
|
364
394
|
],
|
|
365
395
|
};
|
|
@@ -373,7 +403,8 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
|
|
|
373
403
|
structurePreserved: false,
|
|
374
404
|
issues: outputValidation.issues,
|
|
375
405
|
failureCause: "empty-or-stderr-only-result",
|
|
376
|
-
|
|
406
|
+
...(resultSource ? { resultSource } : {}),
|
|
407
|
+
resultPath: resultArtifact?.path,
|
|
377
408
|
},
|
|
378
409
|
});
|
|
379
410
|
}
|
|
@@ -593,6 +624,7 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
|
|
|
593
624
|
artifacts: [
|
|
594
625
|
...manifest.artifacts,
|
|
595
626
|
promptArtifact,
|
|
627
|
+
...(breakdownArtifact ? [breakdownArtifact] : []),
|
|
596
628
|
...(resultArtifact ? [resultArtifact] : []),
|
|
597
629
|
inputsArtifact,
|
|
598
630
|
coordinationArtifact,
|