pi-crew 0.9.32 → 0.9.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -1
- package/docs/perf/performance-audit-report-2026-07.md +948 -0
- package/package.json +1 -1
- package/src/extension/run-maintenance.ts +29 -1
- package/src/extension/team-tool/lifecycle-actions.ts +22 -0
- package/src/extension/team-tool.ts +22 -1
- package/src/runtime/skill-instructions.ts +7 -2
- package/src/runtime/task-runner/prompt-builder.ts +66 -19
- package/src/runtime/task-runner.ts +20 -7
- package/src/runtime/team-runner.ts +87 -7
- package/src/state/atomic-write.ts +23 -0
- package/src/state/event-log.ts +71 -25
package/package.json
CHANGED
|
@@ -7,6 +7,7 @@ import { logInternalError } from "../utils/internal-error.ts";
|
|
|
7
7
|
import { projectCrewRoot, userCrewRoot } from "../utils/paths.ts";
|
|
8
8
|
import { redactSecrets } from "../utils/redaction.ts";
|
|
9
9
|
import { isSafePathId, resolveRealContainedPath } from "../utils/safe-paths.ts";
|
|
10
|
+
import { cleanupRunWorktrees } from "../worktree/cleanup.ts";
|
|
10
11
|
import { listRuns } from "./run-index.ts";
|
|
11
12
|
|
|
12
13
|
export interface PruneRunsResult {
|
|
@@ -21,7 +22,14 @@ export interface PruneRunsOptions {
|
|
|
21
22
|
}
|
|
22
23
|
|
|
23
24
|
function isFinished(run: TeamRunManifest): boolean {
|
|
24
|
-
|
|
25
|
+
// P3: "blocked" is NOT a terminal status — the run is waiting on something
|
|
26
|
+
// (plan approval, mailbox reply, scheduler stall) and can transition to
|
|
27
|
+
// running/cancelled/failed (per TEAM_RUN_STATUS_TRANSITIONS). Pruning a
|
|
28
|
+
// blocked run destroys recoverable state (e.g. a plan the user needs to
|
|
29
|
+
// approve, or a mailbox wait the user needs to answer). Truly stuck
|
|
30
|
+
// blocked runs are handled by the stale-reconciler/crash-recovery, not
|
|
31
|
+
// by prune.
|
|
32
|
+
return run.status === "completed" || run.status === "failed" || run.status === "cancelled";
|
|
25
33
|
}
|
|
26
34
|
|
|
27
35
|
function isSafeToPrune(cwd: string, run: TeamRunManifest): boolean {
|
|
@@ -66,6 +74,26 @@ export function pruneFinishedRuns(cwd: string, keep: number, options: PruneRunsO
|
|
|
66
74
|
);
|
|
67
75
|
continue;
|
|
68
76
|
}
|
|
77
|
+
// P2: clean up git worktrees BEFORE deleting state. Worktrees live at
|
|
78
|
+
// <crewRoot>/state/worktrees/<runId>/ — a separate path from stateRoot
|
|
79
|
+
// (<crewRoot>/state/runs/<runId>/) and artifactsRoot, so fs.rmSync below
|
|
80
|
+
// does NOT touch them. Without this, every pruned worktree-using run
|
|
81
|
+
// leaks its worktree dir + git branch. cleanupRunWorktrees (no force)
|
|
82
|
+
// preserves dirty worktrees by writing a diff artifact and skipping
|
|
83
|
+
// the removal — in that case we skip the whole prune of this run so
|
|
84
|
+
// the user can recover (no silent data loss). Compare handleForget
|
|
85
|
+
// which calls cleanupRunWorktrees.
|
|
86
|
+
const worktreeCleanup = cleanupRunWorktrees(run, { signal: options.signal });
|
|
87
|
+
if (worktreeCleanup.preserved.length > 0) {
|
|
88
|
+
logInternalError(
|
|
89
|
+
"prune.worktree-preserved",
|
|
90
|
+
new Error(
|
|
91
|
+
`Skipping prune: ${worktreeCleanup.preserved.length} dirty worktree(s) preserved for recovery: ${worktreeCleanup.preserved.map((p) => p.path).join(", ")}`,
|
|
92
|
+
),
|
|
93
|
+
`runId=${run.runId}`,
|
|
94
|
+
);
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
69
97
|
fs.rmSync(run.stateRoot, { recursive: true, force: true });
|
|
70
98
|
fs.rmSync(run.artifactsRoot, { recursive: true, force: true });
|
|
71
99
|
removed.push(run.runId);
|
|
@@ -2,9 +2,12 @@ import * as fs from "node:fs";
|
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import { removeGuidance } from "../../config/markers.ts";
|
|
4
4
|
import { appendHookEvent, executeHook } from "../../hooks/registry.ts";
|
|
5
|
+
import { killProcessPid } from "../../runtime/child-pi.ts";
|
|
6
|
+
import { terminateLiveAgentsForRun } from "../../runtime/live-agent-manager.ts";
|
|
5
7
|
import type { TeamToolParamsValue } from "../../schema/team-tool-schema.ts";
|
|
6
8
|
import { appendEvent } from "../../state/event-log.ts";
|
|
7
9
|
import { loadRunManifestById } from "../../state/state-store.ts";
|
|
10
|
+
import { logInternalError } from "../../utils/internal-error.ts";
|
|
8
11
|
import { projectCrewRoot, userCrewRoot, userPiRoot } from "../../utils/paths.ts";
|
|
9
12
|
import { resolveRealContainedPath } from "../../utils/safe-paths.ts";
|
|
10
13
|
import { cleanupRunWorktrees } from "../../worktree/cleanup.ts";
|
|
@@ -217,6 +220,25 @@ export async function handleForget(params: TeamToolParamsValue, ctx: TeamContext
|
|
|
217
220
|
intent,
|
|
218
221
|
},
|
|
219
222
|
});
|
|
223
|
+
// P1: terminate live agents + kill the background runner BEFORE deleting state.
|
|
224
|
+
// Without this, forgetting a running run orphans those processes (separate
|
|
225
|
+
// process groups via setsid) which keep executing and consuming tokens after
|
|
226
|
+
// their state directory is gone. Mirrors handleCancel's cleanup.
|
|
227
|
+
await terminateLiveAgentsForRun(loaded.manifest.runId, "cancelled", appendEvent, loaded.manifest.eventsPath);
|
|
228
|
+
const asyncPid = loaded.manifest.async?.pid;
|
|
229
|
+
if (asyncPid !== undefined && asyncPid > 0) {
|
|
230
|
+
try {
|
|
231
|
+
killProcessPid(asyncPid);
|
|
232
|
+
appendEvent(loaded.manifest.eventsPath, {
|
|
233
|
+
type: "async.kill_requested",
|
|
234
|
+
runId: loaded.manifest.runId,
|
|
235
|
+
message: "Sent SIGTERM to background runner process (forget).",
|
|
236
|
+
data: { pid: asyncPid },
|
|
237
|
+
});
|
|
238
|
+
} catch (error) {
|
|
239
|
+
logInternalError("team-tool.handleForget.killAsync", error, `runId=${loaded.manifest.runId},pid=${asyncPid}`);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
220
242
|
// Determine scope from manifest paths (project vs user-level runs)
|
|
221
243
|
const crewRoot = loaded.manifest.stateRoot.startsWith(userCrewRoot() + path.sep)
|
|
222
244
|
? userCrewRoot()
|
|
@@ -314,6 +314,17 @@ export async function handleResume(params: TeamToolParamsValue, ctx: TeamContext
|
|
|
314
314
|
if (!runCwd) return result(`Run '${params.runId}' not found.${RUN_NOT_FOUND_HINT}`, { action: "resume", status: "error" }, true);
|
|
315
315
|
const loaded = loadRunManifestById(runCwd, params.runId); // NOTE: no withRunLock - best-effort only; concurrent writes may cause inconsistency
|
|
316
316
|
if (!loaded) return result(`Run '${params.runId}' not found.${RUN_NOT_FOUND_HINT}`, { action: "resume", status: "error" }, true);
|
|
317
|
+
// R1: foreign-ownership check — mirrors handleRetry/handleCancel. Without it,
|
|
318
|
+
// another session can resume (and re-execute) a run it doesn't own, racing
|
|
319
|
+
// the owning session.
|
|
320
|
+
const foreignRun = typeof loaded.manifest.ownerSessionId === "string" && loaded.manifest.ownerSessionId !== ctx.sessionId;
|
|
321
|
+
if (foreignRun && !params.force) {
|
|
322
|
+
return result(
|
|
323
|
+
`Run ${loaded.manifest.runId} belongs to another session. Use force: true to override.`,
|
|
324
|
+
{ action: "resume", status: "error", runId: loaded.manifest.runId },
|
|
325
|
+
true,
|
|
326
|
+
);
|
|
327
|
+
}
|
|
317
328
|
if (!loaded.manifest.workflow)
|
|
318
329
|
return result(`Run '${params.runId}' has no workflow to resume.`, { action: "resume", status: "error" }, true);
|
|
319
330
|
const agents = allAgents(discoverAgents(ctx.cwd));
|
|
@@ -324,8 +335,18 @@ export async function handleResume(params: TeamToolParamsValue, ctx: TeamContext
|
|
|
324
335
|
direct?.workflow ?? allWorkflows(discoverWorkflows(ctx.cwd)).find((candidate) => candidate.name === loaded.manifest.workflow);
|
|
325
336
|
if (!workflow) return result(`Workflow '${loaded.manifest.workflow}' not found.`, { action: "resume", status: "error" }, true);
|
|
326
337
|
return await withRunLock(loaded.manifest, async () => {
|
|
338
|
+
// R2: re-read inside the lock so recovery + resetTasks reflect committed
|
|
339
|
+
// state, not the pre-lock snapshot. Between the pre-lock load and lock
|
|
340
|
+
// acquisition a task may have been cancelled (stale-reconciler) or
|
|
341
|
+
// completed (background runner); using the stale snapshot would reset
|
|
342
|
+
// running→queued and re-execute it (double execution: duplicate tokens +
|
|
343
|
+
// duplicate side effects). Sibling handlers (cancelOrphanedRuns,
|
|
344
|
+
// reconcileAllStaleRuns) re-read inside the lock for the same reason.
|
|
345
|
+
const fresh = loadRunManifestById(runCwd, loaded.manifest.runId);
|
|
346
|
+
const lockedManifest = fresh?.manifest ?? loaded.manifest;
|
|
347
|
+
const lockedTasks = fresh?.tasks ?? loaded.tasks;
|
|
327
348
|
const loadedConfig = loadConfig(ctx.cwd);
|
|
328
|
-
const recovered = recoverCheckpointedTasks(
|
|
349
|
+
const recovered = recoverCheckpointedTasks(lockedManifest, lockedTasks);
|
|
329
350
|
const resumeManifest = recovered.manifest;
|
|
330
351
|
const executedConfig = {
|
|
331
352
|
...effectiveRunConfig(loadedConfig.config, params.config),
|
|
@@ -135,6 +135,8 @@ interface CachedSkillMarkdown {
|
|
|
135
135
|
path: string;
|
|
136
136
|
source: "project" | "package" | "project-pi" | "user-pi" | "project-agents" | "user-agents";
|
|
137
137
|
content: string;
|
|
138
|
+
/** Pre-computed compacted body (frontmatter stripped + truncated). Cached to avoid re-compaction on every render. */
|
|
139
|
+
compacted: string;
|
|
138
140
|
mtimeMs: number;
|
|
139
141
|
size: number;
|
|
140
142
|
}
|
|
@@ -173,6 +175,7 @@ function readSkillMarkdown(
|
|
|
173
175
|
path: string;
|
|
174
176
|
source: "project" | "package" | "project-pi" | "user-pi" | "project-agents" | "user-agents";
|
|
175
177
|
content: string;
|
|
178
|
+
compacted: string;
|
|
176
179
|
}
|
|
177
180
|
| undefined {
|
|
178
181
|
if (!isValidSkillName(name)) return undefined;
|
|
@@ -188,10 +191,12 @@ function readSkillMarkdown(
|
|
|
188
191
|
if (fs.lstatSync(contained).isSymbolicLink()) continue;
|
|
189
192
|
const filePath = resolveRealContainedPath(entry.root, relative);
|
|
190
193
|
const stat = fs.statSync(filePath);
|
|
194
|
+
const rawContent = fs.readFileSync(filePath, "utf-8");
|
|
191
195
|
return rememberSkill(cacheKey, {
|
|
192
196
|
path: filePath,
|
|
193
197
|
source: entry.source,
|
|
194
|
-
content:
|
|
198
|
+
content: rawContent,
|
|
199
|
+
compacted: compactSkillContent(rawContent),
|
|
195
200
|
mtimeMs: stat.mtimeMs,
|
|
196
201
|
size: stat.size,
|
|
197
202
|
});
|
|
@@ -296,7 +301,7 @@ export function renderSkillInstructions(
|
|
|
296
301
|
]
|
|
297
302
|
.filter(Boolean)
|
|
298
303
|
.join("\n");
|
|
299
|
-
const rawContent =
|
|
304
|
+
const rawContent = loaded.compacted;
|
|
300
305
|
// Wrap skill content with provenance markers to help LLMs distinguish skill instructions
|
|
301
306
|
const wrappedContent = `<!-- skill: ${safeName} -->\n${rawContent}\n<!-- end-skill: ${safeName} -->`;
|
|
302
307
|
const section = `${header}\n\n${wrappedContent}`;
|
|
@@ -75,6 +75,64 @@ export function renderOutputSchemaBlock(outputSchema: TaskOutputSchema): string
|
|
|
75
75
|
return lines.join("\n");
|
|
76
76
|
}
|
|
77
77
|
|
|
78
|
+
/**
|
|
79
|
+
* Expensive async sub-results that compose the stable prefix.
|
|
80
|
+
* These are cached per (cwd, step.task, runId) so parallel siblings
|
|
81
|
+
* in the same batch reuse them instead of recomputing.
|
|
82
|
+
*/
|
|
83
|
+
export interface StableComponents {
|
|
84
|
+
treeBlock: string;
|
|
85
|
+
suggestedFilesBlock: string;
|
|
86
|
+
knowledgeFragment: string;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const stableComponentCache = new Map<string, StableComponents>();
|
|
90
|
+
|
|
91
|
+
function stablePrefixCacheKey(task: TeamTaskState, step: WorkflowStep, manifest: TeamRunManifest): string {
|
|
92
|
+
return `${task.cwd}|${step.task}|${manifest.runId}`;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Clear the stable prefix cache. Called at run end so the module-level cache
|
|
97
|
+
* (keyed by runId) does not grow unbounded across runs in a long-lived session.
|
|
98
|
+
* Safe to call at any time; the next compute re-populates lazily.
|
|
99
|
+
*/
|
|
100
|
+
export function clearStablePrefixCache(): void {
|
|
101
|
+
stableComponentCache.clear();
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Compute (or return cached) expensive async sub-results that compose
|
|
106
|
+
* the stable prefix: workspace tree, file retrieval, knowledge fragment.
|
|
107
|
+
* Parallel siblings with the same cwd/step/run reuse the cached result.
|
|
108
|
+
*/
|
|
109
|
+
export async function computeStablePrefixComponents(
|
|
110
|
+
manifest: TeamRunManifest,
|
|
111
|
+
step: WorkflowStep,
|
|
112
|
+
task: TeamTaskState,
|
|
113
|
+
_agent?: AgentConfig,
|
|
114
|
+
): Promise<StableComponents> {
|
|
115
|
+
const cacheKey = stablePrefixCacheKey(task, step, manifest);
|
|
116
|
+
const cached = stableComponentCache.get(cacheKey);
|
|
117
|
+
if (cached) return cached;
|
|
118
|
+
|
|
119
|
+
const tree = await buildWorkspaceTree(task.cwd);
|
|
120
|
+
const treeBlock = tree.rendered ? `# Workspace Structure\n${tree.rendered}` : "";
|
|
121
|
+
|
|
122
|
+
const retrieval = await runRetrievalCycle(step.task, manifest.goal, task.cwd);
|
|
123
|
+
const suggestedFilesBlock = renderSuggestedFilesSection(retrieval);
|
|
124
|
+
|
|
125
|
+
const knowledgeFragment = buildKnowledgeFragment(task.cwd, {
|
|
126
|
+
goal: manifest.goal,
|
|
127
|
+
taskText: step.task,
|
|
128
|
+
role: step.role,
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
const components: StableComponents = { treeBlock, suggestedFilesBlock, knowledgeFragment };
|
|
132
|
+
stableComponentCache.set(cacheKey, components);
|
|
133
|
+
return components;
|
|
134
|
+
}
|
|
135
|
+
|
|
78
136
|
export interface RenderedTaskPrompt {
|
|
79
137
|
/** Stable sections that rarely change between tasks of the same role/cwd. */
|
|
80
138
|
stablePrefix: string;
|
|
@@ -90,23 +148,16 @@ export async function renderTaskPrompt(
|
|
|
90
148
|
task: TeamTaskState,
|
|
91
149
|
agent?: AgentConfig,
|
|
92
150
|
skillBlock = "",
|
|
151
|
+
precomputedStableComponents?: StableComponents,
|
|
93
152
|
): Promise<RenderedTaskPrompt> {
|
|
94
153
|
const memoryBlock = agent?.memory
|
|
95
154
|
? buildMemoryBlock(agent.name, agent.memory, task.cwd, Boolean(agent.tools?.some((tool) => tool === "write" || tool === "edit")))
|
|
96
155
|
: "";
|
|
97
156
|
|
|
98
|
-
//
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
// M3: run iterative file-retrieval AFTER the workspace tree is
|
|
103
|
-
// assembled and BEFORE the dynamic suffix is built, so the suggested
|
|
104
|
-
// files section lands in the stable prefix (next to the tree) and
|
|
105
|
-
// is visible to the worker before they start on the task. Never
|
|
106
|
-
// throws — the orchestrator falls back to in-memory heuristic when
|
|
107
|
-
// ripgrep is not installed.
|
|
108
|
-
const retrieval = await runRetrievalCycle(step.task, manifest.goal, task.cwd);
|
|
109
|
-
const suggestedFilesBlock = renderSuggestedFilesSection(retrieval);
|
|
157
|
+
// Use precomputed or cached stable components when available, avoiding
|
|
158
|
+
// redundant workspace tree, file retrieval, and knowledge fragment
|
|
159
|
+
// computation for parallel siblings in the same batch.
|
|
160
|
+
const stableComponents = precomputedStableComponents ?? (await computeStablePrefixComponents(manifest, step, task, agent));
|
|
110
161
|
|
|
111
162
|
// Stable prefix: role instructions, coordination, workspace tree — rarely changes
|
|
112
163
|
const stablePrefix = [
|
|
@@ -131,20 +182,16 @@ export async function renderTaskPrompt(
|
|
|
131
182
|
"",
|
|
132
183
|
coordinationBridgeInstructions(task),
|
|
133
184
|
"",
|
|
134
|
-
treeBlock,
|
|
185
|
+
stableComponents.treeBlock,
|
|
135
186
|
"",
|
|
136
|
-
suggestedFilesBlock,
|
|
187
|
+
stableComponents.suggestedFilesBlock,
|
|
137
188
|
"",
|
|
138
189
|
toolGuidanceBlock(agent),
|
|
139
190
|
"",
|
|
140
191
|
// O4: project knowledge (.crew/knowledge.md) — workers don't load the
|
|
141
192
|
// pi-crew extension (spawned with --no-extensions), so before_agent_start
|
|
142
193
|
// never fires for them. Inject here so every worker sees project knowledge.
|
|
143
|
-
|
|
144
|
-
goal: manifest.goal,
|
|
145
|
-
taskText: step.task,
|
|
146
|
-
role: step.role,
|
|
147
|
-
}),
|
|
194
|
+
stableComponents.knowledgeFragment,
|
|
148
195
|
]
|
|
149
196
|
.filter(Boolean)
|
|
150
197
|
.join("\n");
|
|
@@ -7,6 +7,7 @@ import { errors } from "../errors.ts";
|
|
|
7
7
|
import { appendHookEvent, executeHook } from "../hooks/registry.ts";
|
|
8
8
|
import { writeArtifact } from "../state/artifact-store.ts";
|
|
9
9
|
import { appendEventAsync, appendEventBuffered, appendEventFireAndForget } from "../state/event-log.ts";
|
|
10
|
+
import { withRunLockSync } from "../state/locks.ts";
|
|
10
11
|
import { saveRunManifest } from "../state/state-store.ts";
|
|
11
12
|
import { createTaskClaim } from "../state/task-claims.ts";
|
|
12
13
|
import type {
|
|
@@ -158,6 +159,10 @@ export async function runTeamTask(input: TaskRunnerInput): Promise<{ manifest: T
|
|
|
158
159
|
} as TeamTaskState;
|
|
159
160
|
let tasks = updateTask(input.tasks, task);
|
|
160
161
|
const runtimeKind = input.taskRuntimeOverride ?? input.runtimeKind ?? (input.executeWorkers ? "child-process" : "scaffold");
|
|
162
|
+
// A1-F7: Pre-compute whether yield-event collection is needed. For child-process
|
|
163
|
+
// workers (the common case) this is always false, so we skip allocating/accumulating
|
|
164
|
+
// collectedJsonEvents entirely — eliminating ~10KB memory waste per task.
|
|
165
|
+
const collectYieldEvents = runtimeKind !== "child-process" && (input.runtimeConfig?.yield?.enabled ?? DEFAULT_YIELD_CONFIG.enabled);
|
|
161
166
|
// FIX: Check signal before persisting state — if cancelled, skip the write.
|
|
162
167
|
if (input.signal?.aborted) {
|
|
163
168
|
const cancelReason = cancellationReasonFromSignal(input.signal);
|
|
@@ -293,7 +298,7 @@ export async function runTeamTask(input: TaskRunnerInput): Promise<{ manifest: T
|
|
|
293
298
|
let finalStdout = "";
|
|
294
299
|
let transcriptPath: string | undefined;
|
|
295
300
|
let terminalEvidence: OperationTerminalEvidence[] = [];
|
|
296
|
-
const collectedJsonEvents: Record<string, unknown>[] = [];
|
|
301
|
+
const collectedJsonEvents: Record<string, unknown>[] | undefined = collectYieldEvents ? [] : undefined;
|
|
297
302
|
|
|
298
303
|
let startupEvidence = createStartupEvidence({
|
|
299
304
|
command: runtimeKind === "child-process" ? "pi" : runtimeKind === "live-session" ? "live-session" : "safe-scaffold",
|
|
@@ -502,9 +507,9 @@ export async function runTeamTask(input: TaskRunnerInput): Promise<{ manifest: T
|
|
|
502
507
|
// Errors are logged but processing continues so subsequent events still update state.
|
|
503
508
|
try {
|
|
504
509
|
appendCrewAgentEvent(manifest, task.id, event);
|
|
505
|
-
if (event && typeof event === "object" && !Array.isArray(event))
|
|
510
|
+
if (collectedJsonEvents && event && typeof event === "object" && !Array.isArray(event))
|
|
506
511
|
collectedJsonEvents.push(event as Record<string, unknown>);
|
|
507
|
-
if (collectedJsonEvents.length > 1000) {
|
|
512
|
+
if (collectedJsonEvents && collectedJsonEvents.length > 1000) {
|
|
508
513
|
collectedJsonEvents.splice(0, collectedJsonEvents.length - 1000);
|
|
509
514
|
}
|
|
510
515
|
// Accumulate lifetime usage via message_end events (survives compaction)
|
|
@@ -878,8 +883,8 @@ export async function runTeamTask(input: TaskRunnerInput): Promise<{ manifest: T
|
|
|
878
883
|
// only applies to live-session workers where submit_result is injected by the
|
|
879
884
|
// runtime. Skipping yield detection for child-process prevents every child
|
|
880
885
|
// worker from incorrectly being marked needs_attention.
|
|
881
|
-
const yieldEnabled =
|
|
882
|
-
if (yieldEnabled && collectedJsonEvents.length > 0) {
|
|
886
|
+
const yieldEnabled = collectYieldEvents;
|
|
887
|
+
if (yieldEnabled && collectedJsonEvents && collectedJsonEvents.length > 0) {
|
|
883
888
|
if (hasYieldInOutput(collectedJsonEvents)) {
|
|
884
889
|
const yieldEvent = collectedJsonEvents.find((e) => isYieldEvent(e));
|
|
885
890
|
if (yieldEvent) {
|
|
@@ -1190,8 +1195,16 @@ export async function runTeamTask(input: TaskRunnerInput): Promise<{ manifest: T
|
|
|
1190
1195
|
...(patchArtifact ? [patchArtifact] : []),
|
|
1191
1196
|
],
|
|
1192
1197
|
};
|
|
1193
|
-
|
|
1194
|
-
|
|
1198
|
+
// NEW-C3: persist manifest + tasks atomically under the run lock. Without this,
|
|
1199
|
+
// the unlocked saveRunManifest here races with the team-runner batch merge path
|
|
1200
|
+
// (which writes the manifest under withRunLock) — a parallel batch could read a
|
|
1201
|
+
// stale manifest and overwrite this task's freshly-written artifacts, silently
|
|
1202
|
+
// losing them. persistSingleTaskUpdate is re-entrance-safe (runLockHeldByUs guard),
|
|
1203
|
+
// so nesting it inside this lock is a no-op re-acquire, not a deadlock.
|
|
1204
|
+
withRunLockSync(manifest, () => {
|
|
1205
|
+
saveRunManifest(manifest);
|
|
1206
|
+
tasks = persistSingleTaskUpdate(manifest, tasks, task);
|
|
1207
|
+
});
|
|
1195
1208
|
upsertCrewAgent(manifest, recordFromTask(manifest, task, runtimeKind));
|
|
1196
1209
|
// Execute task_result hook before emitting terminal event
|
|
1197
1210
|
const hookReport = await executeHook("task_result", {
|
|
@@ -44,6 +44,7 @@ import { recordsForMaterializedTasks } from "./task-display.ts";
|
|
|
44
44
|
import { buildExecutionPlan as buildDagExecutionPlan, getReadyTasks as getDagReadyTasks, type TaskNode } from "./task-graph.ts";
|
|
45
45
|
import { buildTaskGraphIndex, refreshTaskGraphQueues, taskGraphSnapshot } from "./task-graph-scheduler.ts";
|
|
46
46
|
import { aggregateTaskOutputs } from "./task-output-context.ts";
|
|
47
|
+
import { clearStablePrefixCache, computeStablePrefixComponents } from "./task-runner/prompt-builder.ts";
|
|
47
48
|
import { runTeamTask } from "./task-runner.ts";
|
|
48
49
|
import { mergeArtifacts } from "./team-runner-artifacts.ts";
|
|
49
50
|
import { clearTrackedTaskUsage } from "./usage-tracker.ts";
|
|
@@ -264,6 +265,15 @@ function shouldMergeTaskUpdate(current: TeamTaskState, updated: TeamTaskState):
|
|
|
264
265
|
// but completed is the success terminal state and should not be reachable from
|
|
265
266
|
// failed via a stale merge. The check above only guards non-terminal→terminal.
|
|
266
267
|
if (current.status === "failed" && updated.status === "completed") return false;
|
|
268
|
+
// Mirror that guard for the other dangerous terminal→terminal flips (reverse
|
|
269
|
+
// audit CANCEL-3 + F3): a cancelled task must not be resurrected to completed,
|
|
270
|
+
// and a completed task must not be demoted to failed, by a stale worker merge.
|
|
271
|
+
// The task transition table (contracts.ts) only permits terminal→queued
|
|
272
|
+
// (retry); these flips are always stale results. A worker that completed after
|
|
273
|
+
// the task was cancelled, or a stale failed result arriving after completion,
|
|
274
|
+
// must not flip a settled terminal status.
|
|
275
|
+
if (current.status === "cancelled" && updated.status === "completed") return false;
|
|
276
|
+
if (current.status === "completed" && updated.status === "failed") return false;
|
|
267
277
|
// Guard: when current is "running" but has resultArtifact (another worker already
|
|
268
278
|
// completed it), a stale updated with status="running" and no resultArtifact
|
|
269
279
|
// must not overwrite the actual completed state.
|
|
@@ -715,7 +725,6 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
715
725
|
);
|
|
716
726
|
stopTeamHeartbeat();
|
|
717
727
|
resolveRunPromise(manifest.runId, result);
|
|
718
|
-
cleanupUsage();
|
|
719
728
|
// Terminate live agents for this run — agents are done when the run ends.
|
|
720
729
|
void terminateLiveAgentsForRun(manifest.runId, "completed", appendEvent, manifest.eventsPath).catch((error) =>
|
|
721
730
|
logInternalError("team-runner.completed.terminate", error, `runId=${manifest.runId}`),
|
|
@@ -817,7 +826,6 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
817
826
|
runId: manifest.runId,
|
|
818
827
|
data: { status: manifest.status, error: message },
|
|
819
828
|
});
|
|
820
|
-
cleanupUsage();
|
|
821
829
|
// M7: flush buffered events before returning on the error path so the
|
|
822
830
|
// final buffered progress events are durable alongside the failure state.
|
|
823
831
|
await flushEventLogBuffer();
|
|
@@ -830,6 +838,16 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
830
838
|
// still has its own flush for M7 backward compat — flushEventLogBuffer
|
|
831
839
|
// is idempotent on an empty queue.
|
|
832
840
|
await flushEventLogBuffer();
|
|
841
|
+
// A2-F1: clear tracked usage for this run's tasks. Consolidated in finally
|
|
842
|
+
// (previously duplicated on the success + error paths) so cleanup is
|
|
843
|
+
// guaranteed on EVERY exit path — a future throw in the success tail or
|
|
844
|
+
// error handler can no longer leak entries. Safe to run last: nothing in
|
|
845
|
+
// team-runner reads usage after the run resolves, and the TUI widget reads
|
|
846
|
+
// live usage only while a task is running (before this point).
|
|
847
|
+
cleanupUsage();
|
|
848
|
+
// NEW-M1: clear the stable-prefix cache (keyed by runId) so it does not
|
|
849
|
+
// grow unbounded across runs in a long-lived session.
|
|
850
|
+
clearStablePrefixCache();
|
|
833
851
|
}
|
|
834
852
|
}
|
|
835
853
|
|
|
@@ -1196,6 +1214,26 @@ async function executeTeamRunCore(
|
|
|
1196
1214
|
coalescedGroups,
|
|
1197
1215
|
);
|
|
1198
1216
|
|
|
1217
|
+
// NEW-M1: Pre-warm stable prefix cache for one representative task
|
|
1218
|
+
// per unique cwd. Parallel siblings with the same cwd/step reuse
|
|
1219
|
+
// the cached workspace tree, file retrieval, and knowledge fragment
|
|
1220
|
+
// instead of recomputing them independently (~200-800ms per batch).
|
|
1221
|
+
if (batchTasks.length > 1) {
|
|
1222
|
+
const seenCwds = new Set<string>();
|
|
1223
|
+
await Promise.all(
|
|
1224
|
+
batchTasks
|
|
1225
|
+
.filter((task) => {
|
|
1226
|
+
if (seenCwds.has(task.cwd)) return false;
|
|
1227
|
+
seenCwds.add(task.cwd);
|
|
1228
|
+
return true;
|
|
1229
|
+
})
|
|
1230
|
+
.map((task) => {
|
|
1231
|
+
const step = findStep(workflow, task);
|
|
1232
|
+
return computeStablePrefixComponents(manifest, step, task);
|
|
1233
|
+
}),
|
|
1234
|
+
);
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1199
1237
|
const results = await mapConcurrent(dispatchUnits, concurrency.selectedCount, async (unit) => {
|
|
1200
1238
|
// M6 real dispatch path: single worker for N tasks.
|
|
1201
1239
|
if (unit.kind === "group") {
|
|
@@ -1419,7 +1457,18 @@ async function executeTeamRunCore(
|
|
|
1419
1457
|
);
|
|
1420
1458
|
}
|
|
1421
1459
|
});
|
|
1422
|
-
if (results.length === 0)
|
|
1460
|
+
if (results.length === 0) {
|
|
1461
|
+
// F1: results is empty ONLY when every ready task was hook-skipped
|
|
1462
|
+
// (batchTasks -> dispatchUnits empty). The skipped tasks are now terminal,
|
|
1463
|
+
// so their downstream dependents become ready on the next iteration.
|
|
1464
|
+
// Re-loop (continue) instead of breaking: breaking would skip those
|
|
1465
|
+
// downstream tasks and fall through to a false-green "completed" run
|
|
1466
|
+
// with queued tasks left behind. Terminates: each skipped task leaves
|
|
1467
|
+
// "queued", so the queued set strictly shrinks; a stuck graph (ready
|
|
1468
|
+
// empty but queued remain) is caught by the readyBatch.length===0
|
|
1469
|
+
// guard above which marks the run "blocked".
|
|
1470
|
+
continue;
|
|
1471
|
+
}
|
|
1423
1472
|
// FIX: Filter out undefined entries from partial results when error occurred
|
|
1424
1473
|
// during parallel execution. Other workers may have written partial results
|
|
1425
1474
|
// before one threw. Results may be partial - some tasks in-flight at error
|
|
@@ -1454,7 +1503,15 @@ async function executeTeamRunCore(
|
|
|
1454
1503
|
"running",
|
|
1455
1504
|
"Merged task updates from parallel batch.",
|
|
1456
1505
|
);
|
|
1457
|
-
|
|
1506
|
+
// CANCEL-1: use the freshly-loaded disk tasks as the merge base instead
|
|
1507
|
+
// of the in-memory `tasks` closure variable. The in-memory tasks reflect
|
|
1508
|
+
// only team-runner's view; an external cancel (handleCancel, background
|
|
1509
|
+
// race with SIGTERM arriving after cancel wrote but before merge ran)
|
|
1510
|
+
// writes 'cancelled' to disk.tasks — using disk.tasks as base preserves
|
|
1511
|
+
// that cancellation through the merge instead of overwriting it with the
|
|
1512
|
+
// stale in-memory view. disk was loaded inside this lock, so it reflects
|
|
1513
|
+
// the freshest committed state.
|
|
1514
|
+
const resultTasks = mergeTaskUpdatesPreservingTerminal(disk?.tasks ?? tasks, validResults);
|
|
1458
1515
|
await saveRunManifestAsync(resultManifest);
|
|
1459
1516
|
await saveRunTasksAsync(resultManifest, resultTasks);
|
|
1460
1517
|
return { resultManifest, resultTasks };
|
|
@@ -1594,8 +1651,24 @@ async function executeTeamRunCore(
|
|
|
1594
1651
|
const message = reason?.message ?? cancelledResult?.manifest.summary ?? "Run cancelled during task execution.";
|
|
1595
1652
|
manifest = { ...manifest, status: "running" };
|
|
1596
1653
|
manifest = updateRunStatus(manifest, "cancelled", message);
|
|
1597
|
-
|
|
1598
|
-
|
|
1654
|
+
// CANCEL-2: re-cancel non-terminal tasks here, mirroring the batch-loop
|
|
1655
|
+
// cancel check at team-runner.ts:~925. A manifest cancelled mid-merge
|
|
1656
|
+
// (e.g. signal abort during the merge's awaits) would otherwise save
|
|
1657
|
+
// tasks without the cancel applied, leaving status=cancelled but tasks
|
|
1658
|
+
// showing completed/running -- inconsistent and breaks handleRetry's
|
|
1659
|
+
// filter for failed/cancelled tasks. Terminal tasks are NOT clobbered.
|
|
1660
|
+
const cancelMessage = reason ? `${message} (${reason.code})` : message;
|
|
1661
|
+
const reCancelledTasks = tasks.map((task) => {
|
|
1662
|
+
if (task.status !== "queued" && task.status !== "running" && task.status !== "waiting") return task;
|
|
1663
|
+
return {
|
|
1664
|
+
...task,
|
|
1665
|
+
status: "cancelled" as const,
|
|
1666
|
+
finishedAt: new Date().toISOString(),
|
|
1667
|
+
error: cancelMessage,
|
|
1668
|
+
};
|
|
1669
|
+
});
|
|
1670
|
+
await saveRunTasksAsync(manifest, reCancelledTasks);
|
|
1671
|
+
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, reCancelledTasks, runtimeKind));
|
|
1599
1672
|
await saveRunManifestAsync(manifest);
|
|
1600
1673
|
await appendEventAsync(manifest.eventsPath, {
|
|
1601
1674
|
type: "run.cancelled",
|
|
@@ -1604,10 +1677,11 @@ async function executeTeamRunCore(
|
|
|
1604
1677
|
data: {
|
|
1605
1678
|
reason,
|
|
1606
1679
|
phase: "task-batch",
|
|
1680
|
+
|
|
1607
1681
|
cancelledResultRunId: cancelledResult?.manifest.runId,
|
|
1608
1682
|
},
|
|
1609
1683
|
});
|
|
1610
|
-
return { manifest, tasks };
|
|
1684
|
+
return { manifest, tasks: reCancelledTasks };
|
|
1611
1685
|
}
|
|
1612
1686
|
queueIndex = buildTaskGraphIndex(tasks);
|
|
1613
1687
|
const injectedAfterBatch = attemptAdaptivePlan();
|
|
@@ -1717,6 +1791,12 @@ async function executeTeamRunCore(
|
|
|
1717
1791
|
manifest = updateRunStatus(manifest, "blocked", effectivenessDecision?.message ?? "Run effectiveness guard blocked completion.");
|
|
1718
1792
|
} else if (blockingDecision) {
|
|
1719
1793
|
manifest = updateRunStatus(manifest, "blocked", blockingDecision.message);
|
|
1794
|
+
} else if (tasks.some((task) => task.status === "queued")) {
|
|
1795
|
+
// F1 defense-in-depth: the loop exited with queued tasks still pending
|
|
1796
|
+
// (e.g. a hook skipped all ready tasks and downstream tasks never became
|
|
1797
|
+
// runnable). This is NOT a completed run — mark it blocked rather than
|
|
1798
|
+
// false-green "completed".
|
|
1799
|
+
manifest = updateRunStatus(manifest, "blocked", "Run exited with queued tasks still pending.");
|
|
1720
1800
|
} else {
|
|
1721
1801
|
manifest = updateRunStatus(
|
|
1722
1802
|
manifest,
|
|
@@ -449,6 +449,7 @@ function normalizeOptions(arg: unknown): { expectedHash?: string; durability: Wr
|
|
|
449
449
|
}
|
|
450
450
|
|
|
451
451
|
export function atomicWriteFile(filePath: string, content: string, options?: AtomicWriteOptions): void {
|
|
452
|
+
cancelPendingCoalescedWrite(filePath);
|
|
452
453
|
const { durability } = normalizeOptions(options);
|
|
453
454
|
if (!isSymlinkSafeDirCached(filePath))
|
|
454
455
|
throw new Error(`Refusing to write: target is a symlink or inside untrusted directory: ${filePath}`);
|
|
@@ -588,6 +589,7 @@ export function atomicWriteFile(filePath: string, content: string, options?: Ato
|
|
|
588
589
|
}
|
|
589
590
|
|
|
590
591
|
export async function atomicWriteFileAsync(filePath: string, content: string, options?: AtomicWriteOptions): Promise<void> {
|
|
592
|
+
cancelPendingCoalescedWrite(filePath);
|
|
591
593
|
const { durability } = normalizeOptions(options);
|
|
592
594
|
// Phase 1.5 (RFC 15): when the worker-thread atomic writer is enabled
|
|
593
595
|
// (PI_CREW_WORKER_ATOMIC_WRITER=1), dispatch to a dedicated worker thread
|
|
@@ -754,6 +756,27 @@ function flushOnePendingAtomicWrite(filePath: string): void {
|
|
|
754
756
|
}
|
|
755
757
|
}
|
|
756
758
|
|
|
759
|
+
/**
|
|
760
|
+
* Cancel any pending coalesced write for `filePath`. An immediate (durable)
|
|
761
|
+
* write supersedes a pending coalesced (buffered) write — the immediate write
|
|
762
|
+
* IS the freshest data, so the stale buffered content must not be allowed to
|
|
763
|
+
* fire later and clobber it.
|
|
764
|
+
*
|
|
765
|
+
* Without this, a sequence like persistSingleTaskUpdate (coalesced, 50ms) →
|
|
766
|
+
* merge saveRunTasksAsync (immediate) leaves the coalesced entry pending; when
|
|
767
|
+
* its timer fires it overwrites the merge result with a stale single-task
|
|
768
|
+
* snapshot, losing other workers' results / attempt enrichment / graph
|
|
769
|
+
* recompute. It also closes the crash window where tasks.json shows "running"
|
|
770
|
+
* while events.jsonl already shows "completed" (re-run on recovery).
|
|
771
|
+
*/
|
|
772
|
+
function cancelPendingCoalescedWrite(filePath: string): void {
|
|
773
|
+
const pending = pendingAtomicWrites.get(filePath);
|
|
774
|
+
if (pending) {
|
|
775
|
+
clearTimeout(pending.timer);
|
|
776
|
+
pendingAtomicWrites.delete(filePath);
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
|
|
757
780
|
/** Flush every queued coalesced write synchronously. Safe to call any time. */
|
|
758
781
|
export function flushPendingAtomicWrites(): void {
|
|
759
782
|
if (flushInProgress > 0) return;
|