pi-crew 0.6.0 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +225 -0
- package/README.md +70 -31
- package/docs/issue-29-analysis.md +189 -0
- package/docs/superpowers/plans/2026-06-09-fallow-patterns-adoption.md +1268 -0
- package/package.json +2 -2
- package/src/agents/agent-config.ts +2 -1
- package/src/benchmark/feedback-loop.ts +4 -2
- package/src/config/config.ts +106 -15
- package/src/errors.ts +107 -0
- package/src/extension/async-notifier.ts +6 -2
- package/src/extension/crew-cleanup.ts +8 -5
- package/src/extension/cross-extension-rpc.ts +48 -0
- package/src/extension/management.ts +464 -109
- package/src/extension/register.ts +194 -34
- package/src/extension/registration/commands.ts +4 -3
- package/src/extension/registration/subagent-helpers.ts +2 -2
- package/src/extension/registration/subagent-tools.ts +3 -1
- package/src/extension/registration/team-tool.ts +3 -1
- package/src/extension/registration/viewers.ts +3 -2
- package/src/extension/run-export.ts +16 -1
- package/src/extension/run-import.ts +16 -0
- package/src/extension/team-tool/anchor.ts +5 -1
- package/src/extension/team-tool/api.ts +12 -7
- package/src/extension/team-tool/cancel.ts +3 -3
- package/src/extension/team-tool/config-patch.ts +15 -1
- package/src/extension/team-tool/explain.ts +1 -1
- package/src/extension/team-tool/handle-schedule.ts +40 -0
- package/src/extension/team-tool/inspect.ts +3 -3
- package/src/extension/team-tool/lifecycle-actions.ts +4 -4
- package/src/extension/team-tool/respond.ts +2 -2
- package/src/extension/team-tool/run.ts +60 -11
- package/src/extension/team-tool/status.ts +1 -1
- package/src/extension/team-tool.ts +175 -47
- package/src/hooks/registry.ts +84 -12
- package/src/hooks/types.ts +12 -3
- package/src/i18n.ts +15 -2
- package/src/observability/exporters/otlp-exporter.ts +73 -0
- package/src/plugins/plugin-define.ts +6 -0
- package/src/plugins/plugin-registry.ts +32 -0
- package/src/plugins/plugins/index.ts +3 -0
- package/src/plugins/plugins/nextjs.ts +19 -0
- package/src/plugins/plugins/vite.ts +14 -0
- package/src/plugins/plugins/vitest.ts +14 -0
- package/src/runtime/adaptive-plan.ts +24 -0
- package/src/runtime/agent-control.ts +6 -3
- package/src/runtime/async-runner.ts +86 -3
- package/src/runtime/background-runner.ts +529 -144
- package/src/runtime/chain-parser.ts +5 -1
- package/src/runtime/chain-runner.ts +67 -3
- package/src/runtime/checkpoint.ts +262 -180
- package/src/runtime/child-pi.ts +165 -38
- package/src/runtime/crash-recovery.ts +25 -14
- package/src/runtime/crew-agent-records.ts +6 -3
- package/src/runtime/cross-extension-rpc.ts +34 -8
- package/src/runtime/diagnostic-export.ts +4 -5
- package/src/runtime/dynamic-script-runner.ts +9 -6
- package/src/runtime/foreground-watchdog.ts +3 -3
- package/src/runtime/heartbeat-watcher.ts +19 -2
- package/src/runtime/intercom-bridge.ts +7 -0
- package/src/runtime/iteration-hooks.ts +1 -1
- package/src/runtime/live-agent-manager.ts +6 -3
- package/src/runtime/live-irc.ts +4 -2
- package/src/runtime/manifest-cache.ts +4 -0
- package/src/runtime/orphan-worker-registry.ts +444 -0
- package/src/runtime/parallel-utils.ts +2 -1
- package/src/runtime/parent-guard.ts +70 -16
- package/src/runtime/pi-args.ts +437 -14
- package/src/runtime/pi-spawn.ts +1 -0
- package/src/runtime/post-checks.ts +11 -4
- package/src/runtime/retry-runner.ts +9 -3
- package/src/runtime/{drift-detectors.ts → run-drift.ts} +1 -1
- package/src/runtime/run-tracker.ts +38 -10
- package/src/runtime/sandbox.ts +42 -21
- package/src/runtime/scheduler.ts +25 -2
- package/src/runtime/semaphore.ts +2 -1
- package/src/runtime/settings-store.ts +14 -2
- package/src/runtime/skill-effectiveness.ts +109 -64
- package/src/runtime/skill-instructions.ts +114 -33
- package/src/runtime/stale-reconciler.ts +310 -69
- package/src/runtime/subagent-manager.ts +265 -53
- package/src/runtime/subprocess-tool-registry.ts +2 -2
- package/src/runtime/task-health.ts +76 -0
- package/src/runtime/task-id.ts +20 -0
- package/src/runtime/task-packet.ts +13 -1
- package/src/runtime/task-runner/live-executor.ts +1 -1
- package/src/runtime/task-runner/progress.ts +1 -1
- package/src/runtime/task-runner/state-helpers.ts +110 -6
- package/src/runtime/task-runner.ts +98 -67
- package/src/runtime/team-runner.ts +186 -20
- package/src/runtime/usage-tracker.ts +4 -2
- package/src/runtime/verification-gates.ts +36 -9
- package/src/schema/team-tool-schema.ts +27 -9
- package/src/skills/discover-skills.ts +9 -3
- package/src/state/active-run-registry.ts +170 -38
- package/src/state/artifact-store.ts +25 -13
- package/src/state/atomic-write-v2.ts +86 -0
- package/src/state/atomic-write.ts +346 -55
- package/src/state/blob-store.ts +178 -10
- package/src/state/contracts.ts +2 -1
- package/src/state/crew-init.ts +161 -28
- package/src/state/decision-ledger.ts +172 -111
- package/src/state/event-log-rotation.ts +82 -52
- package/src/state/event-log.ts +270 -75
- package/src/state/health-store.ts +71 -0
- package/src/state/hook-instinct-bridge.ts +2 -1
- package/src/state/locks.ts +109 -20
- package/src/state/mailbox.ts +45 -7
- package/src/state/observation-store.ts +4 -1
- package/src/state/run-graph.ts +141 -130
- package/src/state/run-metrics.ts +24 -8
- package/src/state/state-store.ts +333 -44
- package/src/state/task-claims.ts +9 -2
- package/src/tools/safe-bash.ts +69 -20
- package/src/types/new-api-types.ts +10 -5
- package/src/ui/keybinding-map.ts +2 -1
- package/src/ui/live-run-sidebar.ts +1 -1
- package/src/ui/loaders.ts +4 -0
- package/src/ui/overlays/agent-picker-overlay.ts +1 -1
- package/src/ui/overlays/mailbox-detail-overlay.ts +1 -1
- package/src/ui/run-action-dispatcher.ts +4 -3
- package/src/ui/run-snapshot-cache.ts +1 -1
- package/src/ui/status-colors.ts +2 -1
- package/src/ui/syntax-highlight.ts +2 -1
- package/src/ui/tool-render.ts +13 -3
- package/src/utils/env-filter.ts +66 -0
- package/src/utils/file-coalescer.ts +9 -1
- package/src/utils/fs-watch.ts +4 -2
- package/src/utils/gh-protocol.ts +2 -1
- package/src/utils/paths.ts +80 -5
- package/src/utils/redaction.ts +6 -1
- package/src/utils/safe-paths.ts +287 -10
- package/src/worktree/cleanup.ts +117 -26
- package/src/worktree/worktree-manager.ts +128 -15
- package/test-bugs-all.mjs +10 -6
- package/test-integration-check.ts +4 -0
|
@@ -11,6 +11,7 @@ import { appendEvent, appendEventAsync, appendEventFireAndForget } from "../stat
|
|
|
11
11
|
import type { TeamConfig } from "../teams/team-config.ts";
|
|
12
12
|
import type { ArtifactDescriptor, PolicyDecision, TeamRunManifest, TaskAttemptState, TeamTaskState } from "../state/types.ts";
|
|
13
13
|
import { loadRunManifestById, saveRunManifest, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/state-store.ts";
|
|
14
|
+
import { withRunLock } from "../state/locks.ts";
|
|
14
15
|
import { aggregateUsage, formatUsage } from "../state/usage.ts";
|
|
15
16
|
import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
|
|
16
17
|
import { evaluateCrewPolicy, summarizePolicyDecisions } from "./policy-engine.ts";
|
|
@@ -38,6 +39,19 @@ import { clearTrackedTaskUsage } from "./usage-tracker.ts";
|
|
|
38
39
|
import { CrewCancellationError, buildSyntheticTerminalEvidence, cancellationReasonFromSignal } from "./cancellation.ts";
|
|
39
40
|
import { effectivenessPolicyDecision, evaluateRunEffectiveness, formatRunEffectivenessLines } from "./effectiveness.ts";
|
|
40
41
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
42
|
+
import { PluginRegistry } from "../plugins/plugin-registry.ts";
|
|
43
|
+
import { NextJsPlugin, VitestPlugin, VitePlugin } from "../plugins/plugins/index.ts";
|
|
44
|
+
import { HealthStore } from "../state/health-store.ts";
|
|
45
|
+
|
|
46
|
+
// Built-in plugin registry for framework awareness.
|
|
47
|
+
// NOTE: This registry is registered here for future use. The integration
|
|
48
|
+
// point (reading package.json deps, computing active plugin context, and
|
|
49
|
+
// passing it into RunConfig / task-runner) is not yet implemented; see
|
|
50
|
+
// `getPluginContext` in src/plugins/plugin-context.ts (planned).
|
|
51
|
+
const builtInRegistry = new PluginRegistry();
|
|
52
|
+
builtInRegistry.register(NextJsPlugin);
|
|
53
|
+
builtInRegistry.register(VitestPlugin);
|
|
54
|
+
builtInRegistry.register(VitePlugin);
|
|
41
55
|
|
|
42
56
|
/**
|
|
43
57
|
* Start a periodic heartbeat for the team-level run.
|
|
@@ -49,7 +63,7 @@ import { logInternalError } from "../utils/internal-error.ts";
|
|
|
49
63
|
* executing. The team-runner has no periodic heartbeat today, so any
|
|
50
64
|
* team run lasting >5min is at risk.
|
|
51
65
|
*/
|
|
52
|
-
function startTeamRunHeartbeat(stateRoot: string, runId: string): () => void {
|
|
66
|
+
function startTeamRunHeartbeat(stateRoot: string, runId: string, lastTaskUpdateAt?: string): () => void {
|
|
53
67
|
const heartbeatPath = path.join(stateRoot, "heartbeat.json");
|
|
54
68
|
const writeHeartbeat = (): void => {
|
|
55
69
|
try {
|
|
@@ -58,14 +72,18 @@ function startTeamRunHeartbeat(stateRoot: string, runId: string): () => void {
|
|
|
58
72
|
at: Date.now(),
|
|
59
73
|
runId,
|
|
60
74
|
kind: "team-runner",
|
|
61
|
-
|
|
75
|
+
lastTaskUpdateAt,
|
|
76
|
+
}), { encoding: "utf-8", mode: 0o600 });
|
|
62
77
|
} catch {
|
|
63
78
|
// best-effort
|
|
64
79
|
}
|
|
65
80
|
};
|
|
66
81
|
writeHeartbeat();
|
|
82
|
+
// NOTE: This interval is deliberately NOT unref'd. Unlike background-runner's
|
|
83
|
+
// heartbeat and interrupt guard (both unref'd), the team heartbeat must keep
|
|
84
|
+
// the event loop alive so the stale reconciler does not cancel long-running
|
|
85
|
+
// team runs (>5 min) as "stale" while they are actively executing.
|
|
67
86
|
const interval = setInterval(writeHeartbeat, 30_000);
|
|
68
|
-
interval.unref();
|
|
69
87
|
return () => clearInterval(interval);
|
|
70
88
|
}
|
|
71
89
|
|
|
@@ -120,18 +138,93 @@ function isNonTerminalTaskStatus(status: TeamTaskState["status"]): boolean {
|
|
|
120
138
|
return status === "queued" || status === "running" || status === "waiting";
|
|
121
139
|
}
|
|
122
140
|
|
|
141
|
+
/**
|
|
142
|
+
* Returns the finishedAt timestamp as a number, or Infinity for invalid/malformed dates.
|
|
143
|
+
* This makes comparison logic in shouldMergeTaskUpdate more readable by abstracting
|
|
144
|
+
* the NaN handling into a single well-named function.
|
|
145
|
+
*/
|
|
146
|
+
function safeFinishedAt(task: TeamTaskState): number {
|
|
147
|
+
if (!task.finishedAt) return -Infinity;
|
|
148
|
+
const ms = new Date(task.finishedAt).getTime();
|
|
149
|
+
return Number.isNaN(ms) ? Infinity : ms;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Returns true when the current task has a malformed finishedAt (NaN/Infinity)
|
|
154
|
+
* and the updated task has a valid finite finishedAt. Malformed finishedAt
|
|
155
|
+
* should be replaced rather than persisting corruption.
|
|
156
|
+
*/
|
|
157
|
+
function isMalformedFinishedAtReplacement(currentTime: number, updatedTime: number): boolean {
|
|
158
|
+
return !Number.isFinite(currentTime) && Number.isFinite(updatedTime);
|
|
159
|
+
}
|
|
160
|
+
|
|
123
161
|
function shouldMergeTaskUpdate(current: TeamTaskState, updated: TeamTaskState): boolean {
|
|
124
162
|
// Parallel workers receive the same input snapshot. A later result may still
|
|
125
163
|
// contain stale queued/running copies of tasks that another worker already
|
|
126
164
|
// completed. Never let those stale snapshots regress durable task state.
|
|
127
|
-
if (
|
|
165
|
+
if (current.status === "waiting" && updated.status === "running") return false;
|
|
166
|
+
// Block terminal→non-terminal transitions (e.g. completed→running).
|
|
167
|
+
// A task that has reached a terminal state must not be resurrected.
|
|
168
|
+
const currentIsTerminal = !isNonTerminalTaskStatus(current.status);
|
|
169
|
+
const updatedIsNonTerminal = isNonTerminalTaskStatus(updated.status);
|
|
170
|
+
if (currentIsTerminal && updatedIsNonTerminal) return false;
|
|
171
|
+
// Explicitly block completed↔needs_attention terminal-to-terminal transitions.
|
|
172
|
+
// Both are success terminal states used interchangeably; stale worker updates must
|
|
173
|
+
// not cause a completed task to appear as needs_attention or vice versa.
|
|
174
|
+
if (current.status === "completed" && updated.status === "needs_attention") return false;
|
|
175
|
+
if (current.status === "needs_attention" && updated.status === "completed") return false;
|
|
176
|
+
// Explicitly block failed→completed resurrection. Both statuses are terminal,
|
|
177
|
+
// but completed is the success terminal state and should not be reachable from
|
|
178
|
+
// failed via a stale merge. The check above only guards non-terminal→terminal.
|
|
179
|
+
if (current.status === "failed" && updated.status === "completed") return false;
|
|
180
|
+
// Guard: when current is "running" but has resultArtifact (another worker already
|
|
181
|
+
// completed it), a stale updated with status="running" and no resultArtifact
|
|
182
|
+
// must not overwrite the actual completed state.
|
|
183
|
+
if (current.status === updated.status && updated.status === "running" && Boolean(current.resultArtifact) && !updated.resultArtifact) return false;
|
|
184
|
+
// Guard: when current is "completed" and has resultArtifact but updated is also
|
|
185
|
+
// "completed" without resultArtifact, block the stale update from overwriting
|
|
186
|
+
// a task that successfully produced output.
|
|
187
|
+
if (current.status === updated.status && current.status === "completed" && Boolean(current.resultArtifact) && !updated.resultArtifact) return false;
|
|
128
188
|
// Prevent a stale completed task from overwriting a fresher one.
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
189
|
+
// Restructure to handle undefined current.finishedAt as a special case:
|
|
190
|
+
// - undefined current + valid updated: allow the update
|
|
191
|
+
// - valid current + undefined updated: block the update (don't lose completion time)
|
|
192
|
+
// - both undefined: finishedAt guard does not apply, fall through to heartbeat check
|
|
193
|
+
// - both valid: compare timestamps as before
|
|
194
|
+
if (current.finishedAt !== undefined && updated.finishedAt !== undefined) {
|
|
195
|
+
const currentTime = safeFinishedAt(current);
|
|
196
|
+
const updatedTime = safeFinishedAt(updated);
|
|
197
|
+
// Malformed finishedAt (NaN) is treated as Infinity — invalid state should be
|
|
198
|
+
// replaced rather than persisting corruption. Log warning for visibility.
|
|
199
|
+
if (!Number.isFinite(currentTime)) {
|
|
200
|
+
console.warn(`[team-runner] Task ${current.id} has malformed finishedAt: ${current.finishedAt}`);
|
|
201
|
+
}
|
|
202
|
+
if (isMalformedFinishedAtReplacement(currentTime, updatedTime)) {
|
|
203
|
+
return true;
|
|
204
|
+
}
|
|
205
|
+
if (updatedTime < currentTime) return false;
|
|
133
206
|
}
|
|
134
|
-
|
|
207
|
+
// Block if updated is trying to establish a terminal status without a finishedAt
|
|
208
|
+
// timestamp. Heartbeat-only updates (status='running', no finishedAt) are
|
|
209
|
+
// allowed if heartbeat has changed (checked separately in hasMeaningfulUpdate).
|
|
210
|
+
if (!updated.finishedAt && !isNonTerminalTaskStatus(updated.status)) return false;
|
|
211
|
+
// Explicitly enumerate all fields that constitute a meaningful update so that
|
|
212
|
+
// adding a new important field requires updating this list (rather than silently
|
|
213
|
+
// losing data if a field is forgotten in the boolean OR chain below).
|
|
214
|
+
const hasMeaningfulUpdate =
|
|
215
|
+
updated.status !== current.status ||
|
|
216
|
+
updated.finishedAt !== current.finishedAt ||
|
|
217
|
+
updated.startedAt !== current.startedAt ||
|
|
218
|
+
Boolean(updated.resultArtifact) !== Boolean(current.resultArtifact) ||
|
|
219
|
+
(Boolean(updated.resultArtifact) && updated.resultArtifact !== current.resultArtifact) ||
|
|
220
|
+
Boolean(updated.error) ||
|
|
221
|
+
Boolean(updated.modelAttempts?.length) ||
|
|
222
|
+
Boolean(updated.usage) ||
|
|
223
|
+
Boolean(updated.attempts?.length) ||
|
|
224
|
+
updated.heartbeat?.lastSeenAt !== current.heartbeat?.lastSeenAt ||
|
|
225
|
+
updated.jsonEvents !== current.jsonEvents ||
|
|
226
|
+
updated.agentProgress?.lastActivityAt !== current.agentProgress?.lastActivityAt;
|
|
227
|
+
return hasMeaningfulUpdate;
|
|
135
228
|
}
|
|
136
229
|
|
|
137
230
|
// H4 fix: rename to descriptive name. Kept __test__ as alias for backward
|
|
@@ -141,7 +234,19 @@ export function mergeTaskUpdatesPreservingTerminal(base: TeamTaskState[], result
|
|
|
141
234
|
for (const result of results) {
|
|
142
235
|
for (const updated of result.tasks) {
|
|
143
236
|
const current = merged.find((task) => task.id === updated.id);
|
|
144
|
-
if (!current
|
|
237
|
+
if (!current) continue;
|
|
238
|
+
if (!shouldMergeTaskUpdate(current, updated)) {
|
|
239
|
+
// Log skipped merges for visibility into rejected parallel updates.
|
|
240
|
+
// In distributed systems with parallel workers, rejected merges may
|
|
241
|
+
// indicate bugs (wrong status, timestamp corruption) if they accumulate.
|
|
242
|
+
console.debug("[team-runner] Skipping stale merge for task", updated.id, {
|
|
243
|
+
currentStatus: current.status,
|
|
244
|
+
updatedStatus: updated.status,
|
|
245
|
+
currentFinishedAt: current.finishedAt,
|
|
246
|
+
updatedFinishedAt: updated.finishedAt,
|
|
247
|
+
});
|
|
248
|
+
continue;
|
|
249
|
+
}
|
|
145
250
|
merged = merged.map((task) => task.id === updated.id ? updated : task);
|
|
146
251
|
}
|
|
147
252
|
}
|
|
@@ -189,7 +294,7 @@ function writeProgress(manifest: TeamRunManifest, tasks: TeamTaskState[], produc
|
|
|
189
294
|
"",
|
|
190
295
|
].join("\n"),
|
|
191
296
|
});
|
|
192
|
-
return { ...manifest, updatedAt: new Date().toISOString(), artifacts: [...manifest.artifacts.filter((artifact) => !(artifact.kind === "progress" && artifact.path === progress.path)), progress] };
|
|
297
|
+
return { ...manifest, updatedAt: new Date().toISOString(), artifacts: [...manifest.artifacts.filter((artifact) => !(artifact.kind === "progress" && artifact.path === progress.path)), progress].filter((artifact, index, self) => self.findIndex((a) => a.path === artifact.path) === index) };
|
|
193
298
|
}
|
|
194
299
|
|
|
195
300
|
function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?: CrewLimitsConfig): TeamRunManifest {
|
|
@@ -307,7 +412,7 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
307
412
|
// (NO_PID_HEARTBEAT_STALE_MS). Previously only sub-task runners wrote
|
|
308
413
|
// heartbeats; the team-level run had no heartbeat, so any multi-phase
|
|
309
414
|
// workflow lasting >5min was marked stale and cancelled.
|
|
310
|
-
const stopTeamHeartbeat = startTeamRunHeartbeat(manifest.stateRoot, manifest.runId);
|
|
415
|
+
const stopTeamHeartbeat = startTeamRunHeartbeat(manifest.stateRoot, manifest.runId, manifest.updatedAt);
|
|
311
416
|
|
|
312
417
|
const cleanupUsage = (): void => {
|
|
313
418
|
for (const task of input.tasks) clearTrackedTaskUsage(task.id);
|
|
@@ -335,7 +440,14 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
335
440
|
} catch (error) {
|
|
336
441
|
// P1: Catch unhandled errors — ensure manifest/tasks/agents are terminal so they don't stay "running" forever.
|
|
337
442
|
const message = error instanceof Error ? error.message : String(error);
|
|
338
|
-
|
|
443
|
+
// Reload manifest with lock to avoid stale data overwriting concurrent writes.
|
|
444
|
+
// If lock acquisition fails, use in-memory data rather than stale disk data.
|
|
445
|
+
let loaded;
|
|
446
|
+
try {
|
|
447
|
+
loaded = await withRunLock(input.manifest, async () => loadRunManifestById(input.manifest.cwd, input.manifest.runId));
|
|
448
|
+
} catch {
|
|
449
|
+
loaded = undefined; // best-effort: use in-memory data if lock fails
|
|
450
|
+
}
|
|
339
451
|
const freshManifest = loaded?.manifest ?? manifest;
|
|
340
452
|
const freshTasks = refreshTaskGraphQueues(loaded?.tasks ?? input.tasks);
|
|
341
453
|
const failedAt = new Date().toISOString();
|
|
@@ -364,7 +476,6 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
364
476
|
rejectRunPromise(manifest.runId, error instanceof Error ? error : new Error(message));
|
|
365
477
|
crewHooks.emit({ type: "run_failed", timestamp: new Date().toISOString(), runId: manifest.runId, data: { status: manifest.status, error: message } });
|
|
366
478
|
cleanupUsage();
|
|
367
|
-
stopTeamHeartbeat();
|
|
368
479
|
return result;
|
|
369
480
|
}
|
|
370
481
|
}
|
|
@@ -543,6 +654,7 @@ async function executeTeamRunCore(
|
|
|
543
654
|
const startedAt = new Date().toISOString();
|
|
544
655
|
const inFlightAttempts: TaskAttemptState[] = [...attemptsSoFar, { attemptId: info.attemptId, startedAt }];
|
|
545
656
|
input.metricRegistry?.counter("crew.task.retry_attempt_total", "Retry attempts by run and task").inc({ runId: manifest.runId, taskId: task.id });
|
|
657
|
+
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
546
658
|
const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
|
|
547
659
|
const freshManifest = fresh?.manifest ?? manifest;
|
|
548
660
|
const freshTasks = fresh?.tasks ?? tasks;
|
|
@@ -580,6 +692,7 @@ async function executeTeamRunCore(
|
|
|
580
692
|
} catch (retryError) {
|
|
581
693
|
if (retryError instanceof CrewCancellationError || input.signal?.aborted) {
|
|
582
694
|
const reason = retryError instanceof CrewCancellationError ? retryError.reason : cancellationReasonFromSignal(input.signal);
|
|
695
|
+
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
583
696
|
const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
|
|
584
697
|
const freshManifest = fresh?.manifest ?? manifest;
|
|
585
698
|
const freshTasks = fresh?.tasks ?? tasks;
|
|
@@ -588,6 +701,7 @@ async function executeTeamRunCore(
|
|
|
588
701
|
return { manifest: updateRunStatus(freshManifest, "cancelled", reason.message), tasks: cancelledTasks };
|
|
589
702
|
}
|
|
590
703
|
if (lastFailed) return lastFailed;
|
|
704
|
+
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
591
705
|
const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
|
|
592
706
|
const freshManifest = fresh?.manifest ?? manifest;
|
|
593
707
|
const freshTasks = fresh?.tasks ?? tasks;
|
|
@@ -598,8 +712,43 @@ async function executeTeamRunCore(
|
|
|
598
712
|
},
|
|
599
713
|
);
|
|
600
714
|
if (results.length === 0) break;
|
|
601
|
-
|
|
602
|
-
|
|
715
|
+
// FIX: Filter out undefined entries from partial results when error occurred
|
|
716
|
+
// during parallel execution. Other workers may have written partial results
|
|
717
|
+
// before one threw. Results may be partial - some tasks in-flight at error
|
|
718
|
+
// time will not have entries in the results array.
|
|
719
|
+
const validResults = results.filter((item): item is NonNullable<typeof item> => item !== undefined);
|
|
720
|
+
// Guard: if ALL parallel workers threw before returning, validResults is empty.
|
|
721
|
+
// at(-1)! would crash. Mark the run failed rather than crashing.
|
|
722
|
+
if (validResults.length === 0) {
|
|
723
|
+
manifest = updateRunStatus(manifest, "failed", "All parallel tasks failed catastrophically.");
|
|
724
|
+
return { manifest, tasks };
|
|
725
|
+
}
|
|
726
|
+
// Reconstruct manifest from the last worker's snapshot. The .artifacts field
|
|
727
|
+
// is re-merged from both the team-runner's in-memory state and all workers'
|
|
728
|
+
// snapshots, so artifact writes by task-runner (which individually save manifest
|
|
729
|
+
// after writing artifacts) are safely persisted. The in-memory manifest is only
|
|
730
|
+
// used for the next batch iteration's orchestration — actual persistence is safe.
|
|
731
|
+
// Use updateRunStatus to recompute manifest status from merged tasks rather than
|
|
732
|
+
// relying on the last result's manifest (which is arbitrary due to mapConcurrent
|
|
733
|
+
// returning results in arbitrary order).
|
|
734
|
+
// Use the in-memory manifest as base (not the last-completing worker's snapshot).
|
|
735
|
+
// Recompute status from merged tasks so the manifest reflects actual task state,
|
|
736
|
+
// not the arbitrary order in which mapConcurrent returned results.
|
|
737
|
+
// Read committed manifest from disk inside the lock so artifact merge is based
|
|
738
|
+
// on committed state, not in-memory state that may differ from disk.
|
|
739
|
+
const mergeResult = await withRunLock(manifest, async () => {
|
|
740
|
+
const disk = loadRunManifestById(manifest.cwd, manifest.runId);
|
|
741
|
+
const diskManifest = disk?.manifest ?? manifest;
|
|
742
|
+
const diskArtifacts = diskManifest.artifacts;
|
|
743
|
+
const reconciledArtifacts = mergeArtifacts([...diskArtifacts, ...validResults.map((item) => item.manifest.artifacts)].flat());
|
|
744
|
+
const resultManifest = updateRunStatus({ ...diskManifest, artifacts: reconciledArtifacts }, "running", "Merged task updates from parallel batch.");
|
|
745
|
+
const resultTasks = mergeTaskUpdatesPreservingTerminal(tasks, validResults);
|
|
746
|
+
await saveRunManifestAsync(resultManifest);
|
|
747
|
+
await saveRunTasksAsync(resultManifest, resultTasks);
|
|
748
|
+
return { resultManifest, resultTasks };
|
|
749
|
+
});
|
|
750
|
+
manifest = mergeResult.resultManifest;
|
|
751
|
+
tasks = mergeResult.resultTasks;
|
|
603
752
|
|
|
604
753
|
// Advance workflow phases whose tasks are all in terminal state
|
|
605
754
|
const terminalStatuses = new Set(["completed", "failed", "skipped", "cancelled", "needs_attention"]);
|
|
@@ -678,7 +827,7 @@ async function executeTeamRunCore(
|
|
|
678
827
|
}
|
|
679
828
|
await saveRunTasksAsync(manifest, tasks);
|
|
680
829
|
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
681
|
-
const completedBatch =
|
|
830
|
+
const completedBatch = tasks.filter((t) => batchTasks.some((bt) => bt.id === t.id));
|
|
682
831
|
const batchArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
683
832
|
kind: "summary",
|
|
684
833
|
relativePath: `batches/${batchTasks.map((task) => task.id).join("+")}.md`,
|
|
@@ -744,8 +893,25 @@ async function executeTeamRunCore(
|
|
|
744
893
|
"",
|
|
745
894
|
].join("\n"),
|
|
746
895
|
});
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
896
|
+
// Build the complete manifest BEFORE acquiring the lock so the artifacts array
|
|
897
|
+
// is already incorporated into the manifest object that will be atomically written.
|
|
898
|
+
// This prevents crash-between-mutation-and-lock from leaving inconsistent state.
|
|
899
|
+
const finalManifest = { ...manifest, updatedAt: new Date().toISOString(), artifacts: [...manifest.artifacts, summaryArtifact] };
|
|
900
|
+
// Joint atomic save: wrap manifest + tasks in a single run lock so they are
|
|
901
|
+
// written together or not at all. Crash between separate saveRunManifestAsync
|
|
902
|
+
// and saveRunTasksAsync calls could leave manifest/tasks.json out of sync.
|
|
903
|
+
await withRunLock(finalManifest, async () => {
|
|
904
|
+
await saveRunManifestAsync(finalManifest);
|
|
905
|
+
await saveRunTasksAsync(finalManifest, tasks);
|
|
906
|
+
});
|
|
907
|
+
manifest = finalManifest;
|
|
908
|
+
// Save health snapshot on run completion
|
|
909
|
+
const crewRoot = path.dirname(path.dirname(finalManifest.stateRoot));
|
|
910
|
+
const healthStore = new HealthStore(crewRoot);
|
|
911
|
+
healthStore.saveSnapshot({
|
|
912
|
+
runId: finalManifest.runId,
|
|
913
|
+
tasks: tasks.map((t) => ({ id: t.id, status: t.status })),
|
|
914
|
+
createdAt: finalManifest.createdAt,
|
|
915
|
+
});
|
|
750
916
|
return { manifest, tasks };
|
|
751
917
|
}
|
|
@@ -16,7 +16,8 @@ export function addUsage(into: LifetimeUsage, delta: { input?: number; output?:
|
|
|
16
16
|
if (typeof delta.cacheWrite === "number") into.cacheWrite += delta.cacheWrite;
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
|
|
19
|
+
/** @internal */
|
|
20
|
+
function lifetimeUsageFromState(state: UsageState | undefined): LifetimeUsage {
|
|
20
21
|
if (!state) return emptyLifetimeUsage();
|
|
21
22
|
return {
|
|
22
23
|
input: state.input ?? 0,
|
|
@@ -59,7 +60,8 @@ export const getTaskUsage = getTrackedTaskUsage;
|
|
|
59
60
|
export const getRunUsage = getTrackedTaskUsage;
|
|
60
61
|
export const clearAllTaskUsage = clearAllTrackedTaskUsage;
|
|
61
62
|
|
|
62
|
-
|
|
63
|
+
/** @internal */
|
|
64
|
+
function aggregateTrackedUsageForRun(manifest: TeamRunManifest, tasks: TeamTaskState[]): UsageState {
|
|
63
65
|
const total = emptyLifetimeUsage();
|
|
64
66
|
for (const task of tasks) {
|
|
65
67
|
const tracked = getTrackedTaskUsage(task.id);
|
|
@@ -38,35 +38,61 @@ export interface PhaseGateBundle {
|
|
|
38
38
|
* Sequential enforcement: each phase must pass before proceeding.
|
|
39
39
|
*/
|
|
40
40
|
export const NPM_TYPESCRIPT_GATES: Array<{ name: string; command: string; critical: boolean }> = [
|
|
41
|
-
{ name: "build", command: "npm run build 2>&1
|
|
42
|
-
{ name: "typecheck", command: "npx tsc --noEmit 2>&1
|
|
43
|
-
{ name: "lint", command: "npm run lint 2>&1
|
|
44
|
-
{ name: "tests", command: "npm test 2>&1
|
|
41
|
+
{ name: "build", command: "npm run build 2>&1", critical: true },
|
|
42
|
+
{ name: "typecheck", command: "npx tsc --noEmit 2>&1", critical: true },
|
|
43
|
+
{ name: "lint", command: "npm run lint 2>&1", critical: false },
|
|
44
|
+
{ name: "tests", command: "npm test 2>&1", critical: true },
|
|
45
45
|
];
|
|
46
46
|
|
|
47
47
|
/**
|
|
48
48
|
* Cargo/Rust project phase gates.
|
|
49
49
|
*/
|
|
50
50
|
export const CARGO_RUST_GATES: Array<{ name: string; command: string; critical: boolean }> = [
|
|
51
|
-
{ name: "check", command: "cargo check 2>&1
|
|
52
|
-
{ name: "test", command: "cargo test 2>&1
|
|
53
|
-
{ name: "clippy", command: "cargo clippy 2>&1
|
|
51
|
+
{ name: "check", command: "cargo check 2>&1", critical: true },
|
|
52
|
+
{ name: "test", command: "cargo test 2>&1", critical: true },
|
|
53
|
+
{ name: "clippy", command: "cargo clippy 2>&1", critical: false },
|
|
54
54
|
];
|
|
55
55
|
|
|
56
56
|
/**
|
|
57
57
|
* Execute a single command and capture output.
|
|
58
58
|
*/
|
|
59
|
+
/** Characters/patterns that indicate dangerous shell metacharacters. */
|
|
60
|
+
const DANGEROUS_SHELL_PATTERNS = /(?:;|&&|\|\||\$\(|`|\$\{|\b(eval|exec)\b|>>|<[^&])/;
|
|
61
|
+
// Note: single `>` is NOT blocked here because `2>&1` is a safe redirect used by built-in gates.
|
|
62
|
+
// `>>` (append) is still blocked. `<` without `&` (input redirect) is still blocked.
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Validate a verification gate command is safe to execute.
|
|
66
|
+
* Rejects commands with shell metacharacters that could enable injection.
|
|
67
|
+
* Allows: pipes (|), redirection of stderr (2>&1), and basic npm/cargo/npx commands.
|
|
68
|
+
*/
|
|
69
|
+
function validateGateCommand(command: string): void {
|
|
70
|
+
const normalized = command
|
|
71
|
+
.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, '') // ANSI escape sequences
|
|
72
|
+
.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, '') // control chars
|
|
73
|
+
.replace(/\\\n/g, ' ') // escaped newlines
|
|
74
|
+
.replace(/\s+/g, ' ') // collapse whitespace
|
|
75
|
+
.trim();
|
|
76
|
+
if (DANGEROUS_SHELL_PATTERNS.test(normalized)) {
|
|
77
|
+
throw new Error(
|
|
78
|
+
`Security: verification gate command rejected (dangerous shell pattern): ${command}`,
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
59
83
|
async function executeCommand(
|
|
60
84
|
command: string,
|
|
61
85
|
cwd: string,
|
|
62
86
|
timeoutMs: number = 120000,
|
|
63
87
|
): Promise<{ exitCode: number | null; output: string; durationMs: number }> {
|
|
88
|
+
// SECURITY: Validate command before shell execution to prevent injection.
|
|
89
|
+
validateGateCommand(command);
|
|
90
|
+
|
|
64
91
|
const start = Date.now();
|
|
65
92
|
let output = "";
|
|
66
93
|
let exitCode: number | null = null;
|
|
67
94
|
|
|
68
95
|
return new Promise((resolve) => {
|
|
69
|
-
// Use shell to handle compound commands
|
|
70
96
|
const shell = spawn("sh", ["-c", command], {
|
|
71
97
|
cwd,
|
|
72
98
|
timeout: timeoutMs,
|
|
@@ -313,7 +339,8 @@ export function computeGreenLevelFromResults(
|
|
|
313
339
|
* Create a verification gate report artifact.
|
|
314
340
|
* Formatted for human review per ECC verification-loop pattern.
|
|
315
341
|
*/
|
|
316
|
-
|
|
342
|
+
/** @internal */
|
|
343
|
+
function createVerificationGateReport(
|
|
317
344
|
taskId: string,
|
|
318
345
|
contract: VerificationContract,
|
|
319
346
|
results: VerificationCommandResult[],
|
|
@@ -114,7 +114,10 @@ export const TeamToolParams = Type.Object({
|
|
|
114
114
|
}),
|
|
115
115
|
),
|
|
116
116
|
runId: Type.Optional(
|
|
117
|
-
Type.String({
|
|
117
|
+
Type.String({
|
|
118
|
+
description: "Run ID for status, cancel, or resume.",
|
|
119
|
+
pattern: "^[A-Za-z0-9_-]+$",
|
|
120
|
+
}),
|
|
118
121
|
),
|
|
119
122
|
taskId: Type.Optional(
|
|
120
123
|
Type.String({ description: "Task ID for respond action." }),
|
|
@@ -197,39 +200,54 @@ export const TeamToolParams = Type.Object({
|
|
|
197
200
|
Type.Integer({ description: "Ms epoch deadline for a reply." }),
|
|
198
201
|
),
|
|
199
202
|
planPath: Type.Optional(
|
|
200
|
-
Type.String({
|
|
203
|
+
Type.String({
|
|
204
|
+
description: "Path to a markdown plan document for orchestration.",
|
|
205
|
+
}),
|
|
201
206
|
),
|
|
202
207
|
cron: Type.Optional(
|
|
203
|
-
Type.String({
|
|
208
|
+
Type.String({
|
|
209
|
+
description:
|
|
210
|
+
"Cron expression for recurring scheduled runs (e.g., '0 9 * * MON').",
|
|
211
|
+
}),
|
|
204
212
|
),
|
|
205
213
|
interval: Type.Optional(
|
|
206
|
-
Type.Number({
|
|
214
|
+
Type.Number({
|
|
215
|
+
description:
|
|
216
|
+
"Interval in milliseconds between recurring scheduled runs.",
|
|
217
|
+
}),
|
|
207
218
|
),
|
|
208
219
|
once: Type.Optional(
|
|
209
|
-
Type.Union([Type.String(), Type.Number()], {
|
|
220
|
+
Type.Union([Type.String(), Type.Number()], {
|
|
221
|
+
description:
|
|
222
|
+
"ISO timestamp or epoch ms for a one-time scheduled run.",
|
|
223
|
+
}),
|
|
210
224
|
),
|
|
211
225
|
excludeContextBash: Type.Optional(
|
|
212
226
|
Type.Boolean({
|
|
213
|
-
description:
|
|
227
|
+
description:
|
|
228
|
+
"Mark certain bash commands as excludeFromContext to reduce context tokens (default: false).",
|
|
214
229
|
}),
|
|
215
230
|
),
|
|
216
231
|
// Budget tracking options
|
|
217
232
|
budgetTotal: Type.Optional(
|
|
218
233
|
Type.Number({
|
|
219
|
-
description:
|
|
234
|
+
description:
|
|
235
|
+
"Total token budget for the run. When set, enables budget tracking with default 80% warning and 95% abort thresholds.",
|
|
220
236
|
minimum: 1,
|
|
221
237
|
}),
|
|
222
238
|
),
|
|
223
239
|
budgetWarning: Type.Optional(
|
|
224
240
|
Type.Number({
|
|
225
|
-
description:
|
|
241
|
+
description:
|
|
242
|
+
"Budget warning threshold as a fraction (0-1). Default: 0.8 (80%). Emits warning event when this threshold is crossed.",
|
|
226
243
|
minimum: 0,
|
|
227
244
|
maximum: 1,
|
|
228
245
|
}),
|
|
229
246
|
),
|
|
230
247
|
budgetAbort: Type.Optional(
|
|
231
248
|
Type.Number({
|
|
232
|
-
description:
|
|
249
|
+
description:
|
|
250
|
+
"Budget abort threshold as a fraction (0-1). Default: 0.95 (95%). Aborts further execution when this threshold is crossed.",
|
|
233
251
|
minimum: 0,
|
|
234
252
|
maximum: 1,
|
|
235
253
|
}),
|
|
@@ -54,10 +54,16 @@ export function discoverSkills(cwd: string): SkillDescriptor[] {
|
|
|
54
54
|
} catch { continue; }
|
|
55
55
|
let description = "";
|
|
56
56
|
try {
|
|
57
|
-
|
|
58
|
-
|
|
57
|
+
let readPath = skillMdPath;
|
|
58
|
+
try {
|
|
59
|
+
readPath = resolveRealContainedPath(dir.root, skillMdRelative);
|
|
60
|
+
skillMdPath = readPath;
|
|
61
|
+
} catch {
|
|
62
|
+
// resolveRealContainedPath may fail for symlinked system paths
|
|
63
|
+
// (e.g. macOS /var → /private/var). Fall through with un-resolved path.
|
|
64
|
+
}
|
|
65
|
+
const content = fs.readFileSync(readPath, "utf-8");
|
|
59
66
|
description = frontmatterDescription(content) ?? "";
|
|
60
|
-
skillMdPath = realPath;
|
|
61
67
|
} catch (error) {
|
|
62
68
|
logInternalError("discoverSkills.readSkill", error, `skill=${entry.name}`);
|
|
63
69
|
}
|