pi-crew 0.9.14 → 0.9.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +126 -0
- package/README.md +66 -0
- package/package.json +1 -1
- package/src/config/types.ts +5 -0
- package/src/extension/registration/team-tool.ts +145 -35
- package/src/extension/team-tool/run.ts +646 -150
- package/src/runtime/crew-agent-records.ts +55 -0
- package/src/runtime/subagent-manager.ts +42 -0
- package/src/runtime/team-runner.ts +1445 -318
- package/src/workflows/discover-workflows.ts +144 -29
- package/src/workflows/preflight-validator.ts +195 -0
- package/src/workflows/topology-analyzer.ts +219 -0
- package/src/workflows/workflow-config.ts +11 -0
- package/workflows/chain.workflow.md +6 -0
- package/workflows/default.workflow.md +1 -0
- package/workflows/fast-fix.workflow.md +1 -0
- package/workflows/implementation.workflow.md +1 -0
- package/workflows/parallel-research.workflow.md +1 -0
- package/workflows/pipeline.workflow.md +1 -0
- package/workflows/research.workflow.md +1 -0
- package/workflows/review.workflow.md +1 -0
|
@@ -1,48 +1,117 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import type { AgentConfig } from "../agents/agent-config.ts";
|
|
4
|
-
import type {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
4
|
+
import type {
|
|
5
|
+
CrewLimitsConfig,
|
|
6
|
+
CrewReliabilityConfig,
|
|
7
|
+
CrewRuntimeConfig,
|
|
8
|
+
} from "../config/config.ts";
|
|
9
|
+
import { appendHookEvent, executeHook } from "../hooks/registry.ts";
|
|
10
|
+
import {
|
|
11
|
+
childCorrelation,
|
|
12
|
+
withCorrelation,
|
|
13
|
+
} from "../observability/correlation.ts";
|
|
14
|
+
import type { MetricRegistry } from "../observability/metric-registry.ts";
|
|
15
|
+
import { PluginRegistry } from "../plugins/plugin-registry.ts";
|
|
16
|
+
import {
|
|
17
|
+
NextJsPlugin,
|
|
18
|
+
VitePlugin,
|
|
19
|
+
VitestPlugin,
|
|
20
|
+
} from "../plugins/plugins/index.ts";
|
|
8
21
|
import { writeArtifact } from "../state/artifact-store.ts";
|
|
9
|
-
import {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
22
|
+
import {
|
|
23
|
+
appendEvent,
|
|
24
|
+
appendEventAsync,
|
|
25
|
+
appendEventFireAndForget,
|
|
26
|
+
} from "../state/event-log.ts";
|
|
27
|
+
import { HealthStore } from "../state/health-store.ts";
|
|
14
28
|
import { withRunLock } from "../state/locks.ts";
|
|
29
|
+
import {
|
|
30
|
+
loadRunManifestById,
|
|
31
|
+
saveRunManifest,
|
|
32
|
+
saveRunManifestAsync,
|
|
33
|
+
saveRunTasksAsync,
|
|
34
|
+
updateRunStatus,
|
|
35
|
+
} from "../state/state-store.ts";
|
|
36
|
+
import type {
|
|
37
|
+
ArtifactDescriptor,
|
|
38
|
+
PolicyDecision,
|
|
39
|
+
TaskAttemptState,
|
|
40
|
+
TeamRunManifest,
|
|
41
|
+
TeamTaskState,
|
|
42
|
+
} from "../state/types.ts";
|
|
15
43
|
import { aggregateUsage, formatUsage } from "../state/usage.ts";
|
|
16
|
-
import type {
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
44
|
+
import type { TeamConfig } from "../teams/team-config.ts";
|
|
45
|
+
import { logInternalError } from "../utils/internal-error.ts";
|
|
46
|
+
import type {
|
|
47
|
+
WorkflowConfig,
|
|
48
|
+
WorkflowStep,
|
|
49
|
+
} from "../workflows/workflow-config.ts";
|
|
22
50
|
import { checkBranchFreshness } from "../worktree/branch-freshness.ts";
|
|
23
|
-
import {
|
|
51
|
+
import {
|
|
52
|
+
buildSyntheticTerminalEvidence,
|
|
53
|
+
CrewCancellationError,
|
|
54
|
+
cancellationReasonFromSignal,
|
|
55
|
+
} from "./cancellation.ts";
|
|
56
|
+
import { resolveBatchConcurrency } from "./concurrency.ts";
|
|
24
57
|
import { readCrewAgents, saveCrewAgents } from "./crew-agent-records.ts";
|
|
25
|
-
import {
|
|
58
|
+
import type { CrewRuntimeKind } from "./crew-agent-runtime.ts";
|
|
59
|
+
import { crewHooks } from "./crew-hooks.ts";
|
|
60
|
+
import { appendDeadletter } from "./deadletter.ts";
|
|
61
|
+
import {
|
|
62
|
+
effectivenessPolicyDecision,
|
|
63
|
+
evaluateRunEffectiveness,
|
|
64
|
+
formatRunEffectivenessLines,
|
|
65
|
+
} from "./effectiveness.ts";
|
|
66
|
+
import {
|
|
67
|
+
applyGoalAchievement,
|
|
68
|
+
assessGoalAchievement,
|
|
69
|
+
} from "./goal-achievement.ts";
|
|
26
70
|
import { deliverGroupJoin, resolveGroupJoinMode } from "./group-join.ts";
|
|
27
|
-
import { runTeamTask } from "./task-runner.ts";
|
|
28
71
|
import { terminateLiveAgentsForRun } from "./live-agent-manager.ts";
|
|
29
|
-
import { createWorkflowStateMachine, validatePhasePreconditions, transitionPhase, type PhaseState, type PhaseGuardContext } from "./workflow-state.ts";
|
|
30
|
-
import { executeWithRetry, DEFAULT_RETRY_POLICY, type RetryPolicy } from "./retry-executor.ts";
|
|
31
|
-
import { appendDeadletter } from "./deadletter.ts";
|
|
32
|
-
import type { MetricRegistry } from "../observability/metric-registry.ts";
|
|
33
|
-
import { childCorrelation, withCorrelation } from "../observability/correlation.ts";
|
|
34
|
-
import { crewHooks } from "./crew-hooks.ts";
|
|
35
|
-
import { resolveBatchConcurrency } from "./concurrency.ts";
|
|
36
72
|
import { mapConcurrent } from "./parallel-utils.ts";
|
|
73
|
+
import {
|
|
74
|
+
evaluateCrewPolicy,
|
|
75
|
+
summarizePolicyDecisions,
|
|
76
|
+
} from "./policy-engine.ts";
|
|
77
|
+
import {
|
|
78
|
+
buildRecoveryLedger,
|
|
79
|
+
shouldRerunFailedTask,
|
|
80
|
+
} from "./recovery-recipes.ts";
|
|
81
|
+
import {
|
|
82
|
+
DEFAULT_RETRY_POLICY,
|
|
83
|
+
executeWithRetry,
|
|
84
|
+
type RetryPolicy,
|
|
85
|
+
} from "./retry-executor.ts";
|
|
37
86
|
import { permissionForRole } from "./role-permission.ts";
|
|
38
|
-
import {
|
|
87
|
+
import {
|
|
88
|
+
registerRunPromise,
|
|
89
|
+
rejectRunPromise,
|
|
90
|
+
resolveRunPromise,
|
|
91
|
+
} from "./run-tracker.ts";
|
|
92
|
+
import { resolveTaskRuntimeKind } from "./runtime-policy.ts";
|
|
93
|
+
import type { CrewRuntimeCapabilities } from "./runtime-resolver.ts";
|
|
94
|
+
import { recordsForMaterializedTasks } from "./task-display.ts";
|
|
95
|
+
import {
|
|
96
|
+
buildExecutionPlan as buildDagExecutionPlan,
|
|
97
|
+
getReadyTasks as getDagReadyTasks,
|
|
98
|
+
type TaskNode,
|
|
99
|
+
} from "./task-graph.ts";
|
|
100
|
+
import {
|
|
101
|
+
buildTaskGraphIndex,
|
|
102
|
+
refreshTaskGraphQueues,
|
|
103
|
+
taskGraphSnapshot,
|
|
104
|
+
} from "./task-graph-scheduler.ts";
|
|
105
|
+
import { aggregateTaskOutputs } from "./task-output-context.ts";
|
|
106
|
+
import { runTeamTask } from "./task-runner.ts";
|
|
39
107
|
import { clearTrackedTaskUsage } from "./usage-tracker.ts";
|
|
40
|
-
import {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
108
|
+
import {
|
|
109
|
+
createWorkflowStateMachine,
|
|
110
|
+
type PhaseGuardContext,
|
|
111
|
+
type PhaseState,
|
|
112
|
+
transitionPhase,
|
|
113
|
+
validatePhasePreconditions,
|
|
114
|
+
} from "./workflow-state.ts";
|
|
46
115
|
|
|
47
116
|
// Built-in plugin registry for framework awareness.
|
|
48
117
|
// NOTE: This registry is registered here for future use. The integration
|
|
@@ -73,13 +142,17 @@ function startTeamRunHeartbeat(stateRoot: string, runId: string): () => void {
|
|
|
73
142
|
// captured manifest.updatedAt once at startup, making the value
|
|
74
143
|
// permanently stale throughout the run.
|
|
75
144
|
const now = new Date().toISOString();
|
|
76
|
-
fs.writeFileSync(
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
145
|
+
fs.writeFileSync(
|
|
146
|
+
heartbeatPath,
|
|
147
|
+
JSON.stringify({
|
|
148
|
+
pid: process.pid,
|
|
149
|
+
at: Date.now(),
|
|
150
|
+
runId,
|
|
151
|
+
kind: "team-runner",
|
|
152
|
+
lastTaskUpdateAt: now,
|
|
153
|
+
}),
|
|
154
|
+
{ encoding: "utf-8", mode: 0o600 },
|
|
155
|
+
);
|
|
83
156
|
} catch {
|
|
84
157
|
// best-effort
|
|
85
158
|
}
|
|
@@ -119,19 +192,39 @@ export interface ExecuteTeamRunInput {
|
|
|
119
192
|
}
|
|
120
193
|
|
|
121
194
|
function findStep(workflow: WorkflowConfig, task: TeamTaskState): WorkflowStep {
|
|
122
|
-
const step = workflow.steps.find(
|
|
123
|
-
|
|
195
|
+
const step = workflow.steps.find(
|
|
196
|
+
(candidate) => candidate.id === task.stepId,
|
|
197
|
+
);
|
|
198
|
+
if (!step)
|
|
199
|
+
throw new Error(
|
|
200
|
+
`Workflow step '${task.stepId}' not found for task '${task.id}'.`,
|
|
201
|
+
);
|
|
124
202
|
return step;
|
|
125
203
|
}
|
|
126
204
|
|
|
127
205
|
function findAgent(agents: AgentConfig[], task: TeamTaskState): AgentConfig {
|
|
128
206
|
const agent = agents.find((candidate) => candidate.name === task.agent);
|
|
129
|
-
if (!agent)
|
|
207
|
+
if (!agent)
|
|
208
|
+
throw new Error(
|
|
209
|
+
`Agent '${task.agent}' not found for task '${task.id}'.`,
|
|
210
|
+
);
|
|
130
211
|
return agent;
|
|
131
212
|
}
|
|
132
213
|
|
|
133
214
|
function markBlocked(tasks: TeamTaskState[], reason: string): TeamTaskState[] {
|
|
134
|
-
return tasks.map((task) =>
|
|
215
|
+
return tasks.map((task) =>
|
|
216
|
+
task.status === "queued"
|
|
217
|
+
? {
|
|
218
|
+
...task,
|
|
219
|
+
status: "skipped",
|
|
220
|
+
error: reason,
|
|
221
|
+
finishedAt: new Date().toISOString(),
|
|
222
|
+
graph: task.graph
|
|
223
|
+
? { ...task.graph, queue: "blocked" }
|
|
224
|
+
: undefined,
|
|
225
|
+
}
|
|
226
|
+
: task,
|
|
227
|
+
);
|
|
135
228
|
}
|
|
136
229
|
|
|
137
230
|
function mergeArtifacts(items: ArtifactDescriptor[]): ArtifactDescriptor[] {
|
|
@@ -160,37 +253,59 @@ function safeFinishedAt(task: TeamTaskState): number {
|
|
|
160
253
|
* and the updated task has a valid finite finishedAt. Malformed finishedAt
|
|
161
254
|
* should be replaced rather than persisting corruption.
|
|
162
255
|
*/
|
|
163
|
-
function isMalformedFinishedAtReplacement(
|
|
256
|
+
function isMalformedFinishedAtReplacement(
|
|
257
|
+
currentTime: number,
|
|
258
|
+
updatedTime: number,
|
|
259
|
+
): boolean {
|
|
164
260
|
return !Number.isFinite(currentTime) && Number.isFinite(updatedTime);
|
|
165
261
|
}
|
|
166
262
|
|
|
167
|
-
function shouldMergeTaskUpdate(
|
|
263
|
+
function shouldMergeTaskUpdate(
|
|
264
|
+
current: TeamTaskState,
|
|
265
|
+
updated: TeamTaskState,
|
|
266
|
+
): boolean {
|
|
168
267
|
// Parallel workers receive the same input snapshot. A later result may still
|
|
169
268
|
// contain stale queued/running copies of tasks that another worker already
|
|
170
269
|
// completed. Never let those stale snapshots regress durable task state.
|
|
171
|
-
if (current.status === "waiting" && updated.status === "running")
|
|
270
|
+
if (current.status === "waiting" && updated.status === "running")
|
|
271
|
+
return false;
|
|
172
272
|
// Block terminal→non-terminal transitions (e.g. completed→running).
|
|
173
273
|
// A task that has reached a terminal state must not be resurrected.
|
|
174
274
|
const currentIsTerminal = !isNonTerminalTaskStatus(current.status);
|
|
175
275
|
const updatedIsNonTerminal = isNonTerminalTaskStatus(updated.status);
|
|
176
276
|
if (currentIsTerminal && updatedIsNonTerminal) return false;
|
|
177
277
|
// Explicitly block completed↔needs_attention terminal-to-terminal transitions.
|
|
178
|
-
// Both are success terminal states used interchangeably; stale worker updates must
|
|
179
|
-
// not cause a completed task to appear as needs_attention or vice versa.
|
|
180
|
-
if (current.status === "completed" && updated.status === "needs_attention")
|
|
181
|
-
|
|
278
|
+
// Both are success terminal states used interchangeably; stale worker updates must
|
|
279
|
+
// not cause a completed task to appear as needs_attention or vice versa.
|
|
280
|
+
if (current.status === "completed" && updated.status === "needs_attention")
|
|
281
|
+
return false;
|
|
282
|
+
if (current.status === "needs_attention" && updated.status === "completed")
|
|
283
|
+
return false;
|
|
182
284
|
// Explicitly block failed→completed resurrection. Both statuses are terminal,
|
|
183
285
|
// but completed is the success terminal state and should not be reachable from
|
|
184
286
|
// failed via a stale merge. The check above only guards non-terminal→terminal.
|
|
185
|
-
if (current.status === "failed" && updated.status === "completed")
|
|
287
|
+
if (current.status === "failed" && updated.status === "completed")
|
|
288
|
+
return false;
|
|
186
289
|
// Guard: when current is "running" but has resultArtifact (another worker already
|
|
187
290
|
// completed it), a stale updated with status="running" and no resultArtifact
|
|
188
291
|
// must not overwrite the actual completed state.
|
|
189
|
-
if (
|
|
292
|
+
if (
|
|
293
|
+
current.status === updated.status &&
|
|
294
|
+
updated.status === "running" &&
|
|
295
|
+
current.resultArtifact &&
|
|
296
|
+
!updated.resultArtifact
|
|
297
|
+
)
|
|
298
|
+
return false;
|
|
190
299
|
// Guard: when current is "completed" and has resultArtifact but updated is also
|
|
191
300
|
// "completed" without resultArtifact, block the stale update from overwriting
|
|
192
301
|
// a task that successfully produced output.
|
|
193
|
-
if (
|
|
302
|
+
if (
|
|
303
|
+
current.status === updated.status &&
|
|
304
|
+
current.status === "completed" &&
|
|
305
|
+
current.resultArtifact &&
|
|
306
|
+
!updated.resultArtifact
|
|
307
|
+
)
|
|
308
|
+
return false;
|
|
194
309
|
// Prevent a stale completed task from overwriting a fresher one.
|
|
195
310
|
// Restructure to handle undefined current.finishedAt as a special case:
|
|
196
311
|
// - undefined current + valid updated: allow the update
|
|
@@ -203,7 +318,9 @@ if (current.status === "needs_attention" && updated.status === "completed") retu
|
|
|
203
318
|
// Malformed finishedAt (NaN) is treated as Infinity — invalid state should be
|
|
204
319
|
// replaced rather than persisting corruption. Log warning for visibility.
|
|
205
320
|
if (!Number.isFinite(currentTime)) {
|
|
206
|
-
console.warn(
|
|
321
|
+
console.warn(
|
|
322
|
+
`[team-runner] Task ${current.id} has malformed finishedAt: ${current.finishedAt}`,
|
|
323
|
+
);
|
|
207
324
|
}
|
|
208
325
|
if (isMalformedFinishedAtReplacement(currentTime, updatedTime)) {
|
|
209
326
|
return true;
|
|
@@ -213,29 +330,35 @@ if (current.status === "needs_attention" && updated.status === "completed") retu
|
|
|
213
330
|
// Block if updated is trying to establish a terminal status without a finishedAt
|
|
214
331
|
// timestamp. Heartbeat-only updates (status='running', no finishedAt) are
|
|
215
332
|
// allowed if heartbeat has changed (checked separately in hasMeaningfulUpdate).
|
|
216
|
-
if (!updated.finishedAt && !isNonTerminalTaskStatus(updated.status))
|
|
333
|
+
if (!updated.finishedAt && !isNonTerminalTaskStatus(updated.status))
|
|
334
|
+
return false;
|
|
217
335
|
// Explicitly enumerate all fields that constitute a meaningful update so that
|
|
218
|
-
// adding a new important field requires updating this list (rather than silently
|
|
219
|
-
// losing data if a field is forgotten in the boolean OR chain below).
|
|
220
|
-
const hasMeaningfulUpdate =
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
336
|
+
// adding a new important field requires updating this list (rather than silently
|
|
337
|
+
// losing data if a field is forgotten in the boolean OR chain below).
|
|
338
|
+
const hasMeaningfulUpdate =
|
|
339
|
+
updated.status !== current.status ||
|
|
340
|
+
updated.finishedAt !== current.finishedAt ||
|
|
341
|
+
updated.startedAt !== current.startedAt ||
|
|
342
|
+
Boolean(updated.resultArtifact) !== Boolean(current.resultArtifact) ||
|
|
343
|
+
(Boolean(updated.resultArtifact) &&
|
|
344
|
+
updated.resultArtifact !== current.resultArtifact) ||
|
|
345
|
+
Boolean(updated.error) ||
|
|
346
|
+
Boolean(updated.modelAttempts?.length) ||
|
|
347
|
+
Boolean(updated.usage) ||
|
|
348
|
+
Boolean(updated.attempts?.length) ||
|
|
349
|
+
updated.heartbeat?.lastSeenAt !== current.heartbeat?.lastSeenAt ||
|
|
350
|
+
updated.jsonEvents !== current.jsonEvents ||
|
|
351
|
+
updated.agentProgress?.lastActivityAt !==
|
|
352
|
+
current.agentProgress?.lastActivityAt;
|
|
353
|
+
return hasMeaningfulUpdate;
|
|
234
354
|
}
|
|
235
355
|
|
|
236
356
|
// H4 fix: rename to descriptive name. Kept __test__ as alias for backward
|
|
237
357
|
// compat test imports.
|
|
238
|
-
export function mergeTaskUpdatesPreservingTerminal(
|
|
358
|
+
export function mergeTaskUpdatesPreservingTerminal(
|
|
359
|
+
base: TeamTaskState[],
|
|
360
|
+
results: Array<{ tasks: TeamTaskState[] }>,
|
|
361
|
+
): TeamTaskState[] {
|
|
239
362
|
let merged = base;
|
|
240
363
|
for (const result of results) {
|
|
241
364
|
for (const updated of result.tasks) {
|
|
@@ -245,15 +368,21 @@ export function mergeTaskUpdatesPreservingTerminal(base: TeamTaskState[], result
|
|
|
245
368
|
// Log skipped merges for visibility into rejected parallel updates.
|
|
246
369
|
// In distributed systems with parallel workers, rejected merges may
|
|
247
370
|
// indicate bugs (wrong status, timestamp corruption) if they accumulate.
|
|
248
|
-
console.debug(
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
371
|
+
console.debug(
|
|
372
|
+
"[team-runner] Skipping stale merge for task",
|
|
373
|
+
updated.id,
|
|
374
|
+
{
|
|
375
|
+
currentStatus: current.status,
|
|
376
|
+
updatedStatus: updated.status,
|
|
377
|
+
currentFinishedAt: current.finishedAt,
|
|
378
|
+
updatedFinishedAt: updated.finishedAt,
|
|
379
|
+
},
|
|
380
|
+
);
|
|
254
381
|
continue;
|
|
255
382
|
}
|
|
256
|
-
merged = merged.map((task) =>
|
|
383
|
+
merged = merged.map((task) =>
|
|
384
|
+
task.id === updated.id ? updated : task,
|
|
385
|
+
);
|
|
257
386
|
}
|
|
258
387
|
}
|
|
259
388
|
return refreshTaskGraphQueues(merged);
|
|
@@ -263,20 +392,43 @@ export const __test__mergeTaskUpdates = mergeTaskUpdatesPreservingTerminal;
|
|
|
263
392
|
|
|
264
393
|
// 2.8: adaptive-plan parsing/repair/injection moved to src/runtime/adaptive-plan.ts.
|
|
265
394
|
// Re-export the test-only helpers so existing test imports still resolve.
|
|
266
|
-
export {
|
|
395
|
+
export {
|
|
396
|
+
__test__parseAdaptivePlan,
|
|
397
|
+
__test__repairAdaptivePlan,
|
|
398
|
+
} from "./adaptive-plan.ts";
|
|
399
|
+
|
|
267
400
|
import { injectAdaptivePlanIfReady } from "./adaptive-plan.ts";
|
|
268
401
|
|
|
269
402
|
function formatTaskProgress(task: TeamTaskState): string {
|
|
270
403
|
return `- ${task.id}: ${task.status} (${task.role} -> ${task.agent})${task.taskPacket ? ` scope=${task.taskPacket.scope}` : ""}${task.verification ? ` green=${task.verification.observedGreenLevel}/${task.verification.requiredGreenLevel}` : ""}${task.error ? ` - ${task.error}` : ""}`;
|
|
271
404
|
}
|
|
272
405
|
|
|
273
|
-
function runEffectivenessLines(
|
|
274
|
-
|
|
406
|
+
function runEffectivenessLines(
|
|
407
|
+
manifest: TeamRunManifest,
|
|
408
|
+
tasks: TeamTaskState[],
|
|
409
|
+
executeWorkers: boolean,
|
|
410
|
+
runtimeConfig?: CrewRuntimeConfig,
|
|
411
|
+
): string[] {
|
|
412
|
+
return formatRunEffectivenessLines(
|
|
413
|
+
evaluateRunEffectiveness({
|
|
414
|
+
manifest,
|
|
415
|
+
tasks,
|
|
416
|
+
executeWorkers,
|
|
417
|
+
runtimeConfig,
|
|
418
|
+
}),
|
|
419
|
+
);
|
|
275
420
|
}
|
|
276
421
|
|
|
277
|
-
function writeProgress(
|
|
422
|
+
function writeProgress(
|
|
423
|
+
manifest: TeamRunManifest,
|
|
424
|
+
tasks: TeamTaskState[],
|
|
425
|
+
producer: string,
|
|
426
|
+
executeWorkers = true,
|
|
427
|
+
runtimeConfig?: CrewRuntimeConfig,
|
|
428
|
+
): TeamRunManifest {
|
|
278
429
|
const counts = new Map<string, number>();
|
|
279
|
-
for (const task of tasks)
|
|
430
|
+
for (const task of tasks)
|
|
431
|
+
counts.set(task.status, (counts.get(task.status) ?? 0) + 1);
|
|
280
432
|
const queue = taskGraphSnapshot(tasks);
|
|
281
433
|
const progress = writeArtifact(manifest.artifactsRoot, {
|
|
282
434
|
kind: "progress",
|
|
@@ -296,14 +448,39 @@ function writeProgress(manifest: TeamRunManifest, tasks: TeamTaskState[], produc
|
|
|
296
448
|
...tasks.map(formatTaskProgress),
|
|
297
449
|
"",
|
|
298
450
|
"## Effectiveness",
|
|
299
|
-
...runEffectivenessLines(
|
|
451
|
+
...runEffectivenessLines(
|
|
452
|
+
manifest,
|
|
453
|
+
tasks,
|
|
454
|
+
executeWorkers,
|
|
455
|
+
runtimeConfig,
|
|
456
|
+
),
|
|
300
457
|
"",
|
|
301
458
|
].join("\n"),
|
|
302
459
|
});
|
|
303
|
-
return {
|
|
460
|
+
return {
|
|
461
|
+
...manifest,
|
|
462
|
+
updatedAt: new Date().toISOString(),
|
|
463
|
+
artifacts: [
|
|
464
|
+
...manifest.artifacts.filter(
|
|
465
|
+
(artifact) =>
|
|
466
|
+
!(
|
|
467
|
+
artifact.kind === "progress" &&
|
|
468
|
+
artifact.path === progress.path
|
|
469
|
+
),
|
|
470
|
+
),
|
|
471
|
+
progress,
|
|
472
|
+
].filter(
|
|
473
|
+
(artifact, index, self) =>
|
|
474
|
+
self.findIndex((a) => a.path === artifact.path) === index,
|
|
475
|
+
),
|
|
476
|
+
};
|
|
304
477
|
}
|
|
305
478
|
|
|
306
|
-
function applyPolicy(
|
|
479
|
+
function applyPolicy(
|
|
480
|
+
manifest: TeamRunManifest,
|
|
481
|
+
tasks: TeamTaskState[],
|
|
482
|
+
limits?: CrewLimitsConfig,
|
|
483
|
+
): TeamRunManifest {
|
|
307
484
|
const branchFreshness = checkBranchFreshness(manifest.cwd);
|
|
308
485
|
const branchArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
309
486
|
kind: "metadata",
|
|
@@ -311,8 +488,15 @@ function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?:
|
|
|
311
488
|
producer: "branch-freshness",
|
|
312
489
|
content: `${JSON.stringify(branchFreshness, null, 2)}\n`,
|
|
313
490
|
});
|
|
314
|
-
let decisions: PolicyDecision[] = evaluateCrewPolicy({
|
|
315
|
-
|
|
491
|
+
let decisions: PolicyDecision[] = evaluateCrewPolicy({
|
|
492
|
+
manifest,
|
|
493
|
+
tasks,
|
|
494
|
+
limits,
|
|
495
|
+
});
|
|
496
|
+
if (
|
|
497
|
+
branchFreshness.status === "stale" ||
|
|
498
|
+
branchFreshness.status === "diverged"
|
|
499
|
+
) {
|
|
316
500
|
const branchDecision: PolicyDecision = {
|
|
317
501
|
action: "notify",
|
|
318
502
|
reason: "branch_stale",
|
|
@@ -320,7 +504,12 @@ function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?:
|
|
|
320
504
|
createdAt: new Date().toISOString(),
|
|
321
505
|
};
|
|
322
506
|
decisions = [...decisions, branchDecision];
|
|
323
|
-
appendEvent(manifest.eventsPath, {
|
|
507
|
+
appendEvent(manifest.eventsPath, {
|
|
508
|
+
type: "branch.stale",
|
|
509
|
+
runId: manifest.runId,
|
|
510
|
+
message: branchFreshness.message,
|
|
511
|
+
data: { branchFreshness },
|
|
512
|
+
});
|
|
324
513
|
}
|
|
325
514
|
const policyArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
326
515
|
kind: "metadata",
|
|
@@ -335,12 +524,57 @@ function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?:
|
|
|
335
524
|
producer: "recovery-engine",
|
|
336
525
|
content: `${JSON.stringify(recoveryLedger, null, 2)}\n`,
|
|
337
526
|
});
|
|
338
|
-
for (const item of decisions)
|
|
339
|
-
|
|
340
|
-
|
|
527
|
+
for (const item of decisions)
|
|
528
|
+
appendEvent(manifest.eventsPath, {
|
|
529
|
+
type:
|
|
530
|
+
item.action === "escalate"
|
|
531
|
+
? "policy.escalated"
|
|
532
|
+
: "policy.action",
|
|
533
|
+
runId: manifest.runId,
|
|
534
|
+
taskId: item.taskId,
|
|
535
|
+
message: item.message,
|
|
536
|
+
data: { action: item.action, reason: item.reason },
|
|
537
|
+
});
|
|
538
|
+
for (const item of recoveryLedger.entries)
|
|
539
|
+
appendEvent(manifest.eventsPath, {
|
|
540
|
+
type:
|
|
541
|
+
item.state === "escalation_required"
|
|
542
|
+
? "recovery.escalated"
|
|
543
|
+
: "recovery.attempted",
|
|
544
|
+
runId: manifest.runId,
|
|
545
|
+
taskId: item.taskId,
|
|
546
|
+
message: item.message,
|
|
547
|
+
data: {
|
|
548
|
+
scenario: item.scenario,
|
|
549
|
+
steps: item.steps,
|
|
550
|
+
attempt: item.attempt,
|
|
551
|
+
state: item.state,
|
|
552
|
+
},
|
|
553
|
+
});
|
|
554
|
+
return {
|
|
555
|
+
...manifest,
|
|
556
|
+
updatedAt: new Date().toISOString(),
|
|
557
|
+
policyDecisions: decisions,
|
|
558
|
+
artifacts: [
|
|
559
|
+
...manifest.artifacts.filter(
|
|
560
|
+
(artifact) =>
|
|
561
|
+
!(
|
|
562
|
+
artifact.kind === "metadata" &&
|
|
563
|
+
(artifact.path.endsWith("policy-decisions.json") ||
|
|
564
|
+
artifact.path.endsWith("recovery-ledger.json") ||
|
|
565
|
+
artifact.path.endsWith("branch-freshness.json"))
|
|
566
|
+
),
|
|
567
|
+
),
|
|
568
|
+
branchArtifact,
|
|
569
|
+
policyArtifact,
|
|
570
|
+
recoveryArtifact,
|
|
571
|
+
],
|
|
572
|
+
};
|
|
341
573
|
}
|
|
342
574
|
|
|
343
|
-
function retryPolicyFromConfig(
|
|
575
|
+
function retryPolicyFromConfig(
|
|
576
|
+
config: CrewReliabilityConfig | undefined,
|
|
577
|
+
): RetryPolicy {
|
|
344
578
|
return { ...DEFAULT_RETRY_POLICY, ...(config?.retryPolicy ?? {}) };
|
|
345
579
|
}
|
|
346
580
|
|
|
@@ -350,15 +584,25 @@ function retryPolicyFromConfig(config: CrewReliabilityConfig | undefined): Retry
|
|
|
350
584
|
* automatically. Previously opt-in, which left the entire retry+recovery stack dormant.
|
|
351
585
|
* Exported for unit testing.
|
|
352
586
|
*/
|
|
353
|
-
export function shouldUseRetry(
|
|
587
|
+
export function shouldUseRetry(
|
|
588
|
+
reliability: CrewReliabilityConfig | undefined,
|
|
589
|
+
): boolean {
|
|
354
590
|
return reliability?.autoRetry !== false;
|
|
355
591
|
}
|
|
356
592
|
|
|
357
|
-
function failedTaskFrom(
|
|
358
|
-
|
|
593
|
+
function failedTaskFrom(
|
|
594
|
+
result: { tasks: TeamTaskState[] },
|
|
595
|
+
taskId: string,
|
|
596
|
+
): TeamTaskState | undefined {
|
|
597
|
+
return result.tasks.find(
|
|
598
|
+
(item) => item.id === taskId && item.status === "failed",
|
|
599
|
+
);
|
|
359
600
|
}
|
|
360
601
|
|
|
361
|
-
function requiresPlanApproval(
|
|
602
|
+
function requiresPlanApproval(
|
|
603
|
+
_workflow: WorkflowConfig,
|
|
604
|
+
runtimeConfig: CrewRuntimeConfig | undefined,
|
|
605
|
+
): boolean {
|
|
362
606
|
// ROADMAP T1.2: plan-level HITL applies to ANY workflow when
|
|
363
607
|
// config.runtime.requirePlanApproval === true (not just 'implementation').
|
|
364
608
|
// The gate fires at the read-only → mutating (plan → execute) boundary.
|
|
@@ -366,19 +610,31 @@ function requiresPlanApproval(_workflow: WorkflowConfig, runtimeConfig: CrewRunt
|
|
|
366
610
|
}
|
|
367
611
|
|
|
368
612
|
function isPlanApprovalPending(manifest: TeamRunManifest): boolean {
|
|
369
|
-
return
|
|
613
|
+
return (
|
|
614
|
+
manifest.planApproval?.required === true &&
|
|
615
|
+
manifest.planApproval.status === "pending"
|
|
616
|
+
);
|
|
370
617
|
}
|
|
371
618
|
|
|
372
619
|
function isMutatingTask(task: TeamTaskState): boolean {
|
|
373
620
|
return permissionForRole(task.role) !== "read_only";
|
|
374
621
|
}
|
|
375
622
|
|
|
376
|
-
function ensurePlanApprovalRequested(
|
|
623
|
+
function ensurePlanApprovalRequested(
|
|
624
|
+
manifest: TeamRunManifest,
|
|
625
|
+
tasks: TeamTaskState[],
|
|
626
|
+
): TeamRunManifest {
|
|
377
627
|
if (manifest.planApproval) return manifest;
|
|
378
|
-
const assessTask = tasks.find(
|
|
628
|
+
const assessTask = tasks.find(
|
|
629
|
+
(task) => task.stepId === "assess" && task.status === "completed",
|
|
630
|
+
);
|
|
379
631
|
// ROADMAP T1.2: for non-adaptive workflows, fall back to the most recent
|
|
380
632
|
// completed read-only (planning) task as the plan reference.
|
|
381
|
-
const planTask =
|
|
633
|
+
const planTask =
|
|
634
|
+
assessTask ??
|
|
635
|
+
[...tasks]
|
|
636
|
+
.reverse()
|
|
637
|
+
.find((t) => t.status === "completed" && !isMutatingTask(t));
|
|
382
638
|
const now = new Date().toISOString();
|
|
383
639
|
const updated: TeamRunManifest = {
|
|
384
640
|
...manifest,
|
|
@@ -393,16 +649,43 @@ function ensurePlanApprovalRequested(manifest: TeamRunManifest, tasks: TeamTaskS
|
|
|
393
649
|
},
|
|
394
650
|
};
|
|
395
651
|
saveRunManifest(updated);
|
|
396
|
-
appendEvent(updated.eventsPath, {
|
|
652
|
+
appendEvent(updated.eventsPath, {
|
|
653
|
+
type: "plan.approval_required",
|
|
654
|
+
runId: updated.runId,
|
|
655
|
+
taskId: planTask?.id,
|
|
656
|
+
message:
|
|
657
|
+
"Plan requires explicit approval before mutating tasks run. Use: team api op=approve-plan runId=...",
|
|
658
|
+
data: { planArtifactPath: planTask?.resultArtifact?.path },
|
|
659
|
+
});
|
|
397
660
|
return updated;
|
|
398
661
|
}
|
|
399
662
|
|
|
400
|
-
function cancelPlanTasks(
|
|
401
|
-
|
|
663
|
+
function cancelPlanTasks(
|
|
664
|
+
tasks: TeamTaskState[],
|
|
665
|
+
reason: string,
|
|
666
|
+
): TeamTaskState[] {
|
|
667
|
+
return tasks.map((task) =>
|
|
668
|
+
task.status === "queued" ||
|
|
669
|
+
task.status === "running" ||
|
|
670
|
+
task.status === "waiting"
|
|
671
|
+
? {
|
|
672
|
+
...task,
|
|
673
|
+
status: "cancelled",
|
|
674
|
+
finishedAt: new Date().toISOString(),
|
|
675
|
+
error: reason,
|
|
676
|
+
graph: task.graph
|
|
677
|
+
? { ...task.graph, queue: "done" }
|
|
678
|
+
: undefined,
|
|
679
|
+
}
|
|
680
|
+
: task,
|
|
681
|
+
);
|
|
402
682
|
}
|
|
403
683
|
|
|
404
684
|
function hasPendingMutatingAdaptiveTask(tasks: TeamTaskState[]): boolean {
|
|
405
|
-
return tasks.some(
|
|
685
|
+
return tasks.some(
|
|
686
|
+
(task) =>
|
|
687
|
+
task.status === "queued" && task.adaptive && isMutatingTask(task),
|
|
688
|
+
);
|
|
406
689
|
}
|
|
407
690
|
|
|
408
691
|
/**
|
|
@@ -410,9 +693,15 @@ function hasPendingMutatingAdaptiveTask(tasks: TeamTaskState[]): boolean {
|
|
|
410
693
|
* Fires when there are pending mutating tasks whose prerequisites (read-only
|
|
411
694
|
* tasks) have completed — i.e. the plan→execute boundary.
|
|
412
695
|
*/
|
|
413
|
-
export function hasPendingMutatingTaskAtBoundary(
|
|
414
|
-
|
|
415
|
-
|
|
696
|
+
export function hasPendingMutatingTaskAtBoundary(
|
|
697
|
+
tasks: TeamTaskState[],
|
|
698
|
+
): boolean {
|
|
699
|
+
const hasCompletedReadOnly = tasks.some(
|
|
700
|
+
(t) => t.status === "completed" && !isMutatingTask(t),
|
|
701
|
+
);
|
|
702
|
+
const hasPendingMutating = tasks.some(
|
|
703
|
+
(t) => t.status === "queued" && isMutatingTask(t),
|
|
704
|
+
);
|
|
416
705
|
return hasCompletedReadOnly && hasPendingMutating;
|
|
417
706
|
}
|
|
418
707
|
|
|
@@ -421,7 +710,10 @@ export function hasPendingMutatingTaskAtBoundary(tasks: TeamTaskState[]): boolea
|
|
|
421
710
|
* execution planning. If so, build an execution plan and use `getDagReadyTasks`
|
|
422
711
|
* to augment the ready-set selection.
|
|
423
712
|
*/
|
|
424
|
-
function dagReadyTaskIds(
|
|
713
|
+
function dagReadyTaskIds(
|
|
714
|
+
tasks: TeamTaskState[],
|
|
715
|
+
completedIds: Set<string>,
|
|
716
|
+
): string[] | null {
|
|
425
717
|
const hasExplicitDeps = tasks.some((t) => t.dependsOn.length > 0);
|
|
426
718
|
if (!hasExplicitDeps) return null;
|
|
427
719
|
// FIX (goal-wrap runtime test): task.dependsOn stores STEP IDs (e.g. "execute"), not
|
|
@@ -444,9 +736,53 @@ function dagReadyTaskIds(tasks: TeamTaskState[], completedIds: Set<string>): str
|
|
|
444
736
|
return getDagReadyTasks(plan, completedIds);
|
|
445
737
|
}
|
|
446
738
|
|
|
447
|
-
export async function executeTeamRun(
|
|
739
|
+
export async function executeTeamRun(
|
|
740
|
+
input: ExecuteTeamRunInput,
|
|
741
|
+
): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
|
|
448
742
|
const workflow = input.workflow;
|
|
449
|
-
|
|
743
|
+
|
|
744
|
+
// DEFENSE-IN-DEPTH (advisory-only since v0.9.15): re-validate topology here in
|
|
745
|
+
// case a caller bypassed the extension-layer handleRun guard. The extension
|
|
746
|
+
// layer logs an advisory note and proceeds; this defense-in-depth does the same
|
|
747
|
+
// for direct API callers (CLI, tests, scheduler). Never blocks.
|
|
748
|
+
// Skip for synthetic direct-agent workflows (filePath="<generated>").
|
|
749
|
+
if (workflow.filePath !== "<generated>") {
|
|
750
|
+
// LAZY: defer preflight-validator import until the defense-in-depth guard actually runs.
|
|
751
|
+
const { validateWorkflowUsage } = await import(
|
|
752
|
+
"../workflows/preflight-validator.ts"
|
|
753
|
+
);
|
|
754
|
+
const preflight = validateWorkflowUsage(workflow, {
|
|
755
|
+
force: input.reliability?.forcePreflight === true,
|
|
756
|
+
});
|
|
757
|
+
if (
|
|
758
|
+
preflight.level === "warn" ||
|
|
759
|
+
preflight.level === "note" ||
|
|
760
|
+
preflight.level === "info"
|
|
761
|
+
) {
|
|
762
|
+
const icon =
|
|
763
|
+
preflight.level === "warn"
|
|
764
|
+
? "⚠️ "
|
|
765
|
+
: preflight.level === "note"
|
|
766
|
+
? "✅ "
|
|
767
|
+
: "ℹ️ ";
|
|
768
|
+
console.warn(
|
|
769
|
+
`${icon}[team-runner.preflight] ${preflight.level.toUpperCase()}: ${preflight.message} (workflow=${workflow.name})`,
|
|
770
|
+
);
|
|
771
|
+
if (preflight.suggestion) {
|
|
772
|
+
console.warn(
|
|
773
|
+
`[team-runner.preflight] → ${preflight.suggestion}`,
|
|
774
|
+
);
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
let manifest = updateRunStatus(
|
|
780
|
+
input.manifest,
|
|
781
|
+
"running",
|
|
782
|
+
input.executeWorkers
|
|
783
|
+
? "Executing team workflow."
|
|
784
|
+
: "Creating workflow prompts and placeholder results.",
|
|
785
|
+
);
|
|
450
786
|
|
|
451
787
|
void registerRunPromise(manifest.runId);
|
|
452
788
|
|
|
@@ -455,7 +791,10 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
455
791
|
// (NO_PID_HEARTBEAT_STALE_MS). Previously only sub-task runners wrote
|
|
456
792
|
// heartbeats; the team-level run had no heartbeat, so any multi-phase
|
|
457
793
|
// workflow lasting >5min was marked stale and cancelled.
|
|
458
|
-
const stopTeamHeartbeat = startTeamRunHeartbeat(
|
|
794
|
+
const stopTeamHeartbeat = startTeamRunHeartbeat(
|
|
795
|
+
manifest.stateRoot,
|
|
796
|
+
manifest.runId,
|
|
797
|
+
);
|
|
459
798
|
|
|
460
799
|
const cleanupUsage = (): void => {
|
|
461
800
|
for (const task of input.tasks) clearTrackedTaskUsage(task.id);
|
|
@@ -468,44 +807,101 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
468
807
|
// (and/or had a failed task) is a false-green. We expose goalAchieved on the
|
|
469
808
|
// manifest + emit an event so the lie is never silent, and downgrade status
|
|
470
809
|
// to "failed" only when a failed task corroborates it (conservative).
|
|
471
|
-
const gaAssessment = assessGoalAchievement(
|
|
810
|
+
const gaAssessment = assessGoalAchievement(
|
|
811
|
+
result.manifest,
|
|
812
|
+
result.tasks,
|
|
813
|
+
workflow,
|
|
814
|
+
);
|
|
472
815
|
const gaApplied = applyGoalAchievement(result.manifest, gaAssessment);
|
|
473
816
|
if (gaApplied.manifest !== result.manifest) {
|
|
474
817
|
result.manifest = gaApplied.manifest;
|
|
475
818
|
try {
|
|
476
819
|
saveRunManifest(result.manifest);
|
|
477
820
|
} catch (persistError) {
|
|
478
|
-
logInternalError(
|
|
821
|
+
logInternalError(
|
|
822
|
+
"team-runner.goalAchievement.persist",
|
|
823
|
+
persistError instanceof Error
|
|
824
|
+
? persistError
|
|
825
|
+
: new Error(String(persistError)),
|
|
826
|
+
`runId=${manifest.runId}`,
|
|
827
|
+
);
|
|
479
828
|
}
|
|
480
829
|
}
|
|
481
|
-
appendEvent(manifest.eventsPath, {
|
|
482
|
-
|
|
830
|
+
appendEvent(manifest.eventsPath, {
|
|
831
|
+
type: "run.goal_achievement",
|
|
832
|
+
runId: manifest.runId,
|
|
833
|
+
message: gaApplied.manifest.goalAchievementNote ?? "",
|
|
834
|
+
data: {
|
|
835
|
+
achieved: gaAssessment.achieved,
|
|
836
|
+
downgraded: gaApplied.downgraded,
|
|
837
|
+
reason: gaAssessment.reason,
|
|
838
|
+
signals: gaAssessment.signals,
|
|
839
|
+
},
|
|
840
|
+
});
|
|
841
|
+
if (gaApplied.downgraded)
|
|
842
|
+
logInternalError(
|
|
843
|
+
"team-runner.goalAchievement.falseGreen",
|
|
844
|
+
new Error(
|
|
845
|
+
gaApplied.manifest.goalAchievementNote ??
|
|
846
|
+
"false-green detected",
|
|
847
|
+
),
|
|
848
|
+
`runId=${manifest.runId}`,
|
|
849
|
+
);
|
|
483
850
|
stopTeamHeartbeat();
|
|
484
851
|
resolveRunPromise(manifest.runId, result);
|
|
485
852
|
cleanupUsage();
|
|
486
853
|
// Terminate live agents for this run — agents are done when the run ends.
|
|
487
|
-
void terminateLiveAgentsForRun(
|
|
854
|
+
void terminateLiveAgentsForRun(
|
|
855
|
+
manifest.runId,
|
|
856
|
+
"completed",
|
|
857
|
+
appendEvent,
|
|
858
|
+
manifest.eventsPath,
|
|
859
|
+
).catch((error) =>
|
|
860
|
+
logInternalError(
|
|
861
|
+
"team-runner.completed.terminate",
|
|
862
|
+
error,
|
|
863
|
+
`runId=${manifest.runId}`,
|
|
864
|
+
),
|
|
865
|
+
);
|
|
488
866
|
|
|
489
867
|
// Emit run completion hook (100% reliable, fire-and-forget)
|
|
490
|
-
crewHooks.emit({
|
|
868
|
+
crewHooks.emit({
|
|
869
|
+
type: "run_completed",
|
|
870
|
+
timestamp: new Date().toISOString(),
|
|
871
|
+
runId: manifest.runId,
|
|
872
|
+
data: {
|
|
873
|
+
status: result.manifest.status,
|
|
874
|
+
taskCount: result.tasks.length,
|
|
875
|
+
},
|
|
876
|
+
});
|
|
491
877
|
|
|
492
878
|
// Execute after_run_complete lifecycle hook (non-blocking)
|
|
493
|
-
const afterRunReport = await executeHook("after_run_complete", {
|
|
879
|
+
const afterRunReport = await executeHook("after_run_complete", {
|
|
880
|
+
runId: manifest.runId,
|
|
881
|
+
cwd: manifest.cwd,
|
|
882
|
+
status: result.manifest.status,
|
|
883
|
+
});
|
|
494
884
|
appendHookEvent(manifest, afterRunReport);
|
|
495
885
|
if (afterRunReport.outcome === "block") {
|
|
496
|
-
logInternalError(
|
|
886
|
+
logInternalError(
|
|
887
|
+
"team-runner.after_run_complete.blocked",
|
|
888
|
+
new Error(
|
|
889
|
+
afterRunReport.reason ?? "after_run_complete hook blocked",
|
|
890
|
+
),
|
|
891
|
+
`runId=${manifest.runId}`,
|
|
892
|
+
);
|
|
497
893
|
}
|
|
498
894
|
|
|
499
895
|
return result;
|
|
500
896
|
} catch (error) {
|
|
501
897
|
// Round 27 (BUG 1): the success path calls stopTeamHeartbeat() but this
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
898
|
+
// catch path did NOT. The team heartbeat is a non-unref'd setInterval
|
|
899
|
+
// (30s) that deliberately keeps the event loop alive — without this
|
|
900
|
+
// call, a failed team run leaves the interval firing forever and the
|
|
901
|
+
// foreground pi process hangs (never returns to the prompt); in
|
|
902
|
+
// background-runner mode the worker never exits. clearInterval is
|
|
903
|
+
// idempotent so a double-call (if this runs after the success path)
|
|
904
|
+
// is harmless.
|
|
509
905
|
stopTeamHeartbeat();
|
|
510
906
|
// P1: Catch unhandled errors — ensure manifest/tasks/agents are terminal so they don't stay "running" forever.
|
|
511
907
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -513,7 +909,9 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
513
909
|
// If lock acquisition fails, use in-memory data rather than stale disk data.
|
|
514
910
|
let loaded;
|
|
515
911
|
try {
|
|
516
|
-
loaded = await withRunLock(input.manifest, async () =>
|
|
912
|
+
loaded = await withRunLock(input.manifest, async () =>
|
|
913
|
+
loadRunManifestById(input.manifest.cwd, input.manifest.runId),
|
|
914
|
+
);
|
|
517
915
|
} catch {
|
|
518
916
|
loaded = undefined; // best-effort: use in-memory data if lock fails
|
|
519
917
|
}
|
|
@@ -521,29 +919,72 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
521
919
|
const freshTasks = refreshTaskGraphQueues(loaded?.tasks ?? input.tasks);
|
|
522
920
|
const failedAt = new Date().toISOString();
|
|
523
921
|
const tasks = freshTasks.map((task) =>
|
|
524
|
-
task.status === "running" ||
|
|
525
|
-
|
|
922
|
+
task.status === "running" ||
|
|
923
|
+
task.status === "queued" ||
|
|
924
|
+
task.status === "waiting"
|
|
925
|
+
? {
|
|
926
|
+
...task,
|
|
927
|
+
status: "failed" as const,
|
|
928
|
+
finishedAt: failedAt,
|
|
929
|
+
error: message,
|
|
930
|
+
}
|
|
526
931
|
: task,
|
|
527
932
|
);
|
|
528
933
|
manifest = freshManifest;
|
|
529
934
|
try {
|
|
530
|
-
await terminateLiveAgentsForRun(
|
|
935
|
+
await terminateLiveAgentsForRun(
|
|
936
|
+
manifest.runId,
|
|
937
|
+
"failed",
|
|
938
|
+
appendEvent,
|
|
939
|
+
manifest.eventsPath,
|
|
940
|
+
);
|
|
531
941
|
await saveRunTasksAsync(manifest, tasks);
|
|
532
|
-
const existingRuntimeByTask = new Map(
|
|
942
|
+
const existingRuntimeByTask = new Map(
|
|
943
|
+
readCrewAgents(manifest).map((agent) => [
|
|
944
|
+
agent.taskId,
|
|
945
|
+
agent.runtime,
|
|
946
|
+
]),
|
|
947
|
+
);
|
|
533
948
|
const globalRuntime = input.runtime?.kind ?? "child-process";
|
|
534
|
-
const runtimeForAgent = (
|
|
949
|
+
const runtimeForAgent = (
|
|
950
|
+
agent: ReturnType<typeof recordsForMaterializedTasks>[number],
|
|
951
|
+
): CrewRuntimeKind => {
|
|
535
952
|
const task = tasks.find((item) => item.id === agent.taskId);
|
|
536
|
-
return
|
|
953
|
+
return (
|
|
954
|
+
existingRuntimeByTask.get(agent.taskId) ??
|
|
955
|
+
resolveTaskRuntimeKind(
|
|
956
|
+
globalRuntime,
|
|
957
|
+
task?.role ?? agent.role,
|
|
958
|
+
input.runtimeConfig?.isolationPolicy,
|
|
959
|
+
)
|
|
960
|
+
);
|
|
537
961
|
};
|
|
538
|
-
saveCrewAgents(
|
|
539
|
-
|
|
962
|
+
saveCrewAgents(
|
|
963
|
+
manifest,
|
|
964
|
+
recordsForMaterializedTasks(manifest, tasks, globalRuntime).map(
|
|
965
|
+
(agent) => ({ ...agent, runtime: runtimeForAgent(agent) }),
|
|
966
|
+
),
|
|
967
|
+
);
|
|
968
|
+
manifest = updateRunStatus(
|
|
969
|
+
manifest,
|
|
970
|
+
"failed",
|
|
971
|
+
`Unhandled error in team runner: ${message}`,
|
|
972
|
+
);
|
|
540
973
|
await saveRunManifestAsync(manifest);
|
|
541
974
|
} catch {
|
|
542
975
|
// Best-effort — state write may also fail
|
|
543
976
|
}
|
|
544
977
|
const result = { manifest, tasks };
|
|
545
|
-
rejectRunPromise(
|
|
546
|
-
|
|
978
|
+
rejectRunPromise(
|
|
979
|
+
manifest.runId,
|
|
980
|
+
error instanceof Error ? error : new Error(message),
|
|
981
|
+
);
|
|
982
|
+
crewHooks.emit({
|
|
983
|
+
type: "run_failed",
|
|
984
|
+
timestamp: new Date().toISOString(),
|
|
985
|
+
runId: manifest.runId,
|
|
986
|
+
data: { status: manifest.status, error: message },
|
|
987
|
+
});
|
|
547
988
|
cleanupUsage();
|
|
548
989
|
return result;
|
|
549
990
|
}
|
|
@@ -555,10 +996,17 @@ async function executeTeamRunCore(
|
|
|
555
996
|
workflow: WorkflowConfig,
|
|
556
997
|
): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
|
|
557
998
|
// Execute before_run_start hook (non-blocking by default)
|
|
558
|
-
const beforeRunReport = await executeHook("before_run_start", {
|
|
999
|
+
const beforeRunReport = await executeHook("before_run_start", {
|
|
1000
|
+
runId: manifest.runId,
|
|
1001
|
+
cwd: manifest.cwd,
|
|
1002
|
+
});
|
|
559
1003
|
appendHookEvent(manifest, beforeRunReport);
|
|
560
1004
|
if (beforeRunReport.outcome === "block") {
|
|
561
|
-
manifest = updateRunStatus(
|
|
1005
|
+
manifest = updateRunStatus(
|
|
1006
|
+
manifest,
|
|
1007
|
+
"blocked",
|
|
1008
|
+
beforeRunReport.reason ?? "before_run_start hook blocked the run.",
|
|
1009
|
+
);
|
|
562
1010
|
return { manifest, tasks: input.tasks };
|
|
563
1011
|
}
|
|
564
1012
|
let tasks = refreshTaskGraphQueues(input.tasks);
|
|
@@ -567,45 +1015,94 @@ async function executeTeamRunCore(
|
|
|
567
1015
|
let adaptivePlanInjected = false;
|
|
568
1016
|
let adaptivePlanMissing = false;
|
|
569
1017
|
const attemptAdaptivePlan = () => {
|
|
570
|
-
if (
|
|
571
|
-
|
|
1018
|
+
if (
|
|
1019
|
+
!canInjectAdaptivePlan ||
|
|
1020
|
+
adaptivePlanInjected ||
|
|
1021
|
+
adaptivePlanMissing
|
|
1022
|
+
)
|
|
1023
|
+
return { injected: false, missing: false };
|
|
1024
|
+
const adaptivePlan = injectAdaptivePlanIfReady({
|
|
1025
|
+
manifest,
|
|
1026
|
+
tasks,
|
|
1027
|
+
workflow,
|
|
1028
|
+
team: input.team,
|
|
1029
|
+
});
|
|
572
1030
|
adaptivePlanInjected = adaptivePlanInjected || adaptivePlan.injected;
|
|
573
1031
|
adaptivePlanMissing = adaptivePlan.missingPlan;
|
|
574
1032
|
workflow = adaptivePlan.workflow;
|
|
575
1033
|
if (adaptivePlan.injected) tasks = adaptivePlan.tasks;
|
|
576
|
-
return {
|
|
1034
|
+
return {
|
|
1035
|
+
injected: adaptivePlan.injected,
|
|
1036
|
+
missing: adaptivePlan.missingPlan,
|
|
1037
|
+
};
|
|
577
1038
|
};
|
|
578
1039
|
const initialAdaptive = attemptAdaptivePlan();
|
|
579
1040
|
if (initialAdaptive.missing) {
|
|
580
|
-
tasks = markBlocked(
|
|
1041
|
+
tasks = markBlocked(
|
|
1042
|
+
tasks,
|
|
1043
|
+
"Adaptive planner did not produce a valid subagent plan.",
|
|
1044
|
+
);
|
|
581
1045
|
await saveRunTasksAsync(manifest, tasks);
|
|
582
|
-
manifest = updateRunStatus(
|
|
1046
|
+
manifest = updateRunStatus(
|
|
1047
|
+
manifest,
|
|
1048
|
+
"blocked",
|
|
1049
|
+
"Adaptive planner did not produce a valid subagent plan.",
|
|
1050
|
+
);
|
|
583
1051
|
return { manifest, tasks };
|
|
584
1052
|
}
|
|
585
1053
|
if (initialAdaptive.injected) {
|
|
586
|
-
manifest = requiresPlanApproval(workflow, input.runtimeConfig)
|
|
1054
|
+
manifest = requiresPlanApproval(workflow, input.runtimeConfig)
|
|
1055
|
+
? ensurePlanApprovalRequested(manifest, tasks)
|
|
1056
|
+
: manifest;
|
|
587
1057
|
queueIndex = buildTaskGraphIndex(tasks);
|
|
588
|
-
} else if (
|
|
1058
|
+
} else if (
|
|
1059
|
+
requiresPlanApproval(workflow, input.runtimeConfig) &&
|
|
1060
|
+
(hasPendingMutatingAdaptiveTask(tasks) ||
|
|
1061
|
+
hasPendingMutatingTaskAtBoundary(tasks))
|
|
1062
|
+
) {
|
|
589
1063
|
manifest = ensurePlanApprovalRequested(manifest, tasks);
|
|
590
1064
|
}
|
|
591
1065
|
if (manifest.planApproval?.status === "cancelled") {
|
|
592
1066
|
tasks = cancelPlanTasks(tasks, "Plan approval was cancelled.");
|
|
593
1067
|
await saveRunTasksAsync(manifest, tasks);
|
|
594
|
-
manifest = updateRunStatus(
|
|
1068
|
+
manifest = updateRunStatus(
|
|
1069
|
+
manifest,
|
|
1070
|
+
"cancelled",
|
|
1071
|
+
"Plan approval was cancelled.",
|
|
1072
|
+
);
|
|
595
1073
|
return { manifest, tasks };
|
|
596
1074
|
}
|
|
597
|
-
manifest = writeProgress(
|
|
1075
|
+
manifest = writeProgress(
|
|
1076
|
+
manifest,
|
|
1077
|
+
tasks,
|
|
1078
|
+
"team-runner",
|
|
1079
|
+
input.executeWorkers,
|
|
1080
|
+
input.runtimeConfig,
|
|
1081
|
+
);
|
|
598
1082
|
await saveRunManifestAsync(manifest);
|
|
599
|
-
const runtimeKind =
|
|
600
|
-
|
|
1083
|
+
const runtimeKind =
|
|
1084
|
+
input.runtime?.kind ??
|
|
1085
|
+
(input.executeWorkers ? "child-process" : "scaffold");
|
|
1086
|
+
saveCrewAgents(
|
|
1087
|
+
manifest,
|
|
1088
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1089
|
+
);
|
|
601
1090
|
|
|
602
1091
|
// Build a workflow phase state machine from workflow steps for precondition tracking.
|
|
603
|
-
const workflowPhases: PhaseState[] = workflow.steps.map(
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
1092
|
+
const workflowPhases: PhaseState[] = workflow.steps.map(
|
|
1093
|
+
(step): PhaseState => ({
|
|
1094
|
+
name: step.id,
|
|
1095
|
+
status: "pending",
|
|
1096
|
+
inputs:
|
|
1097
|
+
step.reads === false
|
|
1098
|
+
? []
|
|
1099
|
+
: Array.isArray(step.reads)
|
|
1100
|
+
? step.reads
|
|
1101
|
+
: [],
|
|
1102
|
+
outputs:
|
|
1103
|
+
step.output === false ? [] : step.output ? [step.output] : [],
|
|
1104
|
+
}),
|
|
1105
|
+
);
|
|
609
1106
|
let wfMachine = createWorkflowStateMachine(workflowPhases);
|
|
610
1107
|
|
|
611
1108
|
while (tasks.some((task) => task.status === "queued")) {
|
|
@@ -614,17 +1111,46 @@ async function executeTeamRunCore(
|
|
|
614
1111
|
const message = `${cancelReason.message} (${cancelReason.code})`;
|
|
615
1112
|
const cancelledTaskIds: string[] = [];
|
|
616
1113
|
tasks = tasks.map((task) => {
|
|
617
|
-
if (
|
|
1114
|
+
if (
|
|
1115
|
+
task.status !== "queued" &&
|
|
1116
|
+
task.status !== "running" &&
|
|
1117
|
+
task.status !== "waiting"
|
|
1118
|
+
)
|
|
1119
|
+
return task;
|
|
618
1120
|
cancelledTaskIds.push(task.id);
|
|
619
|
-
const base = {
|
|
1121
|
+
const base = {
|
|
1122
|
+
...task,
|
|
1123
|
+
status: "cancelled" as const,
|
|
1124
|
+
finishedAt: new Date().toISOString(),
|
|
1125
|
+
error: message,
|
|
1126
|
+
};
|
|
620
1127
|
if (task.status === "running") {
|
|
621
|
-
return {
|
|
1128
|
+
return {
|
|
1129
|
+
...base,
|
|
1130
|
+
terminalEvidence: [
|
|
1131
|
+
...(task.terminalEvidence ?? []),
|
|
1132
|
+
buildSyntheticTerminalEvidence(
|
|
1133
|
+
"worker",
|
|
1134
|
+
cancelReason,
|
|
1135
|
+
task.startedAt,
|
|
1136
|
+
),
|
|
1137
|
+
],
|
|
1138
|
+
};
|
|
622
1139
|
}
|
|
623
1140
|
return base;
|
|
624
1141
|
});
|
|
625
1142
|
await saveRunTasksAsync(manifest, tasks);
|
|
626
|
-
for (const taskId of cancelledTaskIds)
|
|
627
|
-
|
|
1143
|
+
for (const taskId of cancelledTaskIds)
|
|
1144
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1145
|
+
type: "task.cancelled",
|
|
1146
|
+
runId: manifest.runId,
|
|
1147
|
+
taskId,
|
|
1148
|
+
message,
|
|
1149
|
+
data: { reason: cancelReason.code },
|
|
1150
|
+
});
|
|
1151
|
+
manifest = updateRunStatus(manifest, "cancelled", message, {
|
|
1152
|
+
data: { reason: cancelReason.code, cancelledTaskIds },
|
|
1153
|
+
});
|
|
628
1154
|
return { manifest, tasks };
|
|
629
1155
|
}
|
|
630
1156
|
|
|
@@ -637,15 +1163,48 @@ async function executeTeamRunCore(
|
|
|
637
1163
|
// Default-off: maxRetriesPerTask=0 → original abort behavior preserved.
|
|
638
1164
|
const rerun = shouldRerunFailedTask(failed, input.limits);
|
|
639
1165
|
if (rerun.rerun) {
|
|
640
|
-
tasks = tasks.map((item) =>
|
|
1166
|
+
tasks = tasks.map((item) =>
|
|
1167
|
+
item.id === failed.id
|
|
1168
|
+
? {
|
|
1169
|
+
...item,
|
|
1170
|
+
status: "queued" as const,
|
|
1171
|
+
policy: {
|
|
1172
|
+
...(item.policy ?? {}),
|
|
1173
|
+
retryCount: rerun.newRetryCount,
|
|
1174
|
+
},
|
|
1175
|
+
error: undefined,
|
|
1176
|
+
finishedAt: undefined,
|
|
1177
|
+
}
|
|
1178
|
+
: item,
|
|
1179
|
+
);
|
|
641
1180
|
await saveRunTasksAsync(manifest, tasks);
|
|
642
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
1181
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1182
|
+
type: "recovery.rerun_task",
|
|
1183
|
+
runId: manifest.runId,
|
|
1184
|
+
taskId: failed.id,
|
|
1185
|
+
message: `Re-queuing failed task for whole-task rerun: ${rerun.reason}`,
|
|
1186
|
+
data: {
|
|
1187
|
+
attempt: rerun.newRetryCount,
|
|
1188
|
+
maxRetries: input.limits?.maxRetriesPerTask ?? 0,
|
|
1189
|
+
scenario: "task_failed",
|
|
1190
|
+
},
|
|
1191
|
+
});
|
|
643
1192
|
continue; // loop re-processes the re-queued task
|
|
644
1193
|
}
|
|
645
|
-
tasks = markBlocked(
|
|
1194
|
+
tasks = markBlocked(
|
|
1195
|
+
tasks,
|
|
1196
|
+
`Blocked by failed task '${failed.id}'.`,
|
|
1197
|
+
);
|
|
646
1198
|
await saveRunTasksAsync(manifest, tasks);
|
|
647
|
-
saveCrewAgents(
|
|
648
|
-
|
|
1199
|
+
saveCrewAgents(
|
|
1200
|
+
manifest,
|
|
1201
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1202
|
+
);
|
|
1203
|
+
manifest = updateRunStatus(
|
|
1204
|
+
manifest,
|
|
1205
|
+
"failed",
|
|
1206
|
+
`Failed at task '${failed.id}'.`,
|
|
1207
|
+
);
|
|
649
1208
|
return { manifest, tasks };
|
|
650
1209
|
}
|
|
651
1210
|
|
|
@@ -654,67 +1213,202 @@ async function executeTeamRunCore(
|
|
|
654
1213
|
// DAG-based execution plan: when tasks have explicit dependsOn, use the
|
|
655
1214
|
// topological wave planner to determine ready tasks. Fall back to the
|
|
656
1215
|
// existing task-graph-scheduler when no explicit deps exist (backward compat).
|
|
657
|
-
const completedIds = new Set(
|
|
1216
|
+
const completedIds = new Set(
|
|
1217
|
+
tasks
|
|
1218
|
+
.filter(
|
|
1219
|
+
(t) =>
|
|
1220
|
+
t.status === "completed" ||
|
|
1221
|
+
t.status === "needs_attention",
|
|
1222
|
+
)
|
|
1223
|
+
.map((t) => t.id),
|
|
1224
|
+
);
|
|
658
1225
|
const dagReady = dagReadyTaskIds(tasks, completedIds);
|
|
659
1226
|
const effectiveReady = dagReady ?? snapshot.ready;
|
|
660
1227
|
|
|
661
1228
|
// Workflow phase precondition check (non-blocking: log warnings only).
|
|
662
1229
|
if (wfMachine.currentPhaseIndex < wfMachine.phases.length) {
|
|
663
|
-
const completedArtifacts = manifest.artifacts
|
|
664
|
-
|
|
1230
|
+
const completedArtifacts = manifest.artifacts
|
|
1231
|
+
.filter((a) => a.kind === "result" || a.kind === "summary")
|
|
1232
|
+
.map((a) => a.path);
|
|
1233
|
+
const previousPhaseStatus =
|
|
1234
|
+
wfMachine.currentPhaseIndex > 0
|
|
1235
|
+
? (wfMachine.phases[wfMachine.currentPhaseIndex - 1]
|
|
1236
|
+
?.status ?? "pending")
|
|
1237
|
+
: "completed";
|
|
665
1238
|
const wfContext: PhaseGuardContext = {
|
|
666
1239
|
completedArtifacts,
|
|
667
1240
|
previousPhaseStatus,
|
|
668
|
-
taskResults: tasks
|
|
1241
|
+
taskResults: tasks
|
|
1242
|
+
.filter(
|
|
1243
|
+
(t) =>
|
|
1244
|
+
t.status === "completed" ||
|
|
1245
|
+
t.status === "needs_attention",
|
|
1246
|
+
)
|
|
1247
|
+
.map((t) => ({
|
|
1248
|
+
taskId: t.id,
|
|
1249
|
+
status: t.status,
|
|
1250
|
+
outputPath: t.resultArtifact?.path,
|
|
1251
|
+
})),
|
|
669
1252
|
};
|
|
670
|
-
const preconditions = validatePhasePreconditions(
|
|
1253
|
+
const preconditions = validatePhasePreconditions(
|
|
1254
|
+
wfMachine,
|
|
1255
|
+
wfContext,
|
|
1256
|
+
);
|
|
671
1257
|
if (!preconditions.ready) {
|
|
672
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
1258
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1259
|
+
type: "workflow.preconditions",
|
|
1260
|
+
runId: manifest.runId,
|
|
1261
|
+
message: `Workflow phase '${wfMachine.phases[wfMachine.currentPhaseIndex]?.name}' is missing inputs: ${preconditions.blocking.join(", ")}`,
|
|
1262
|
+
data: {
|
|
1263
|
+
phaseIndex: wfMachine.currentPhaseIndex,
|
|
1264
|
+
phaseName:
|
|
1265
|
+
wfMachine.phases[wfMachine.currentPhaseIndex]?.name,
|
|
1266
|
+
blocking: preconditions.blocking,
|
|
1267
|
+
},
|
|
1268
|
+
});
|
|
673
1269
|
} else {
|
|
674
1270
|
// Advance the machine past completed phases.
|
|
675
|
-
while (
|
|
676
|
-
wfMachine
|
|
1271
|
+
while (
|
|
1272
|
+
wfMachine.currentPhaseIndex < wfMachine.phases.length &&
|
|
1273
|
+
wfMachine.phases[wfMachine.currentPhaseIndex]?.status ===
|
|
1274
|
+
"completed"
|
|
1275
|
+
) {
|
|
1276
|
+
wfMachine = {
|
|
1277
|
+
...wfMachine,
|
|
1278
|
+
currentPhaseIndex: wfMachine.currentPhaseIndex + 1,
|
|
1279
|
+
};
|
|
677
1280
|
}
|
|
678
1281
|
}
|
|
679
1282
|
}
|
|
680
1283
|
|
|
681
|
-
const readyRoles = effectiveReady
|
|
682
|
-
|
|
1284
|
+
const readyRoles = effectiveReady
|
|
1285
|
+
.map((taskId) => tasks.find((task) => task.id === taskId)?.role)
|
|
1286
|
+
.filter((role): role is string => Boolean(role));
|
|
1287
|
+
const concurrency = resolveBatchConcurrency({
|
|
1288
|
+
workflowName: workflow.name,
|
|
1289
|
+
workflowMaxConcurrency: workflow.maxConcurrency,
|
|
1290
|
+
teamMaxConcurrency: input.team.maxConcurrency,
|
|
1291
|
+
limitMaxConcurrentWorkers: input.limits?.maxConcurrentWorkers,
|
|
1292
|
+
allowUnboundedConcurrency: input.limits?.allowUnboundedConcurrency,
|
|
1293
|
+
readyCount: effectiveReady.length,
|
|
1294
|
+
workspaceMode: manifest.workspaceMode,
|
|
1295
|
+
readyRoles,
|
|
1296
|
+
});
|
|
683
1297
|
if (concurrency.reason.includes(";unbounded:")) {
|
|
684
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
1298
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1299
|
+
type: "limits.unbounded",
|
|
1300
|
+
runId: manifest.runId,
|
|
1301
|
+
message:
|
|
1302
|
+
"Unbounded worker concurrency was explicitly enabled for this run.",
|
|
1303
|
+
data: {
|
|
1304
|
+
concurrencyReason: concurrency.reason,
|
|
1305
|
+
maxConcurrent: concurrency.maxConcurrent,
|
|
1306
|
+
},
|
|
1307
|
+
});
|
|
685
1308
|
}
|
|
686
1309
|
const approvalPending = isPlanApprovalPending(manifest);
|
|
687
|
-
const readyIds = approvalPending
|
|
688
|
-
|
|
689
|
-
|
|
1310
|
+
const readyIds = approvalPending
|
|
1311
|
+
? effectiveReady
|
|
1312
|
+
: effectiveReady.slice(0, concurrency.selectedCount);
|
|
1313
|
+
const candidateBatch = readyIds
|
|
1314
|
+
.map((id) => tasks.find((task) => task.id === id))
|
|
1315
|
+
.filter((task): task is TeamTaskState => Boolean(task));
|
|
1316
|
+
const readyBatch = approvalPending
|
|
1317
|
+
? candidateBatch
|
|
1318
|
+
.filter((task) => !isMutatingTask(task))
|
|
1319
|
+
.slice(0, concurrency.selectedCount)
|
|
1320
|
+
: candidateBatch;
|
|
690
1321
|
if (readyBatch.length === 0) {
|
|
691
1322
|
if (approvalPending && candidateBatch.some(isMutatingTask)) {
|
|
692
1323
|
await saveRunTasksAsync(manifest, tasks);
|
|
693
|
-
saveCrewAgents(
|
|
694
|
-
|
|
1324
|
+
saveCrewAgents(
|
|
1325
|
+
manifest,
|
|
1326
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1327
|
+
);
|
|
1328
|
+
manifest = updateRunStatus(
|
|
1329
|
+
manifest,
|
|
1330
|
+
"blocked",
|
|
1331
|
+
"Plan approval required before mutating implementation tasks run.",
|
|
1332
|
+
);
|
|
695
1333
|
return { manifest, tasks };
|
|
696
1334
|
}
|
|
697
|
-
tasks = markBlocked(
|
|
1335
|
+
tasks = markBlocked(
|
|
1336
|
+
tasks,
|
|
1337
|
+
"No ready queued task; dependency graph may be invalid.",
|
|
1338
|
+
);
|
|
698
1339
|
await saveRunTasksAsync(manifest, tasks);
|
|
699
|
-
saveCrewAgents(
|
|
700
|
-
|
|
1340
|
+
saveCrewAgents(
|
|
1341
|
+
manifest,
|
|
1342
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1343
|
+
);
|
|
1344
|
+
manifest = updateRunStatus(
|
|
1345
|
+
manifest,
|
|
1346
|
+
"blocked",
|
|
1347
|
+
"No ready queued task.",
|
|
1348
|
+
);
|
|
701
1349
|
return { manifest, tasks };
|
|
702
1350
|
}
|
|
703
1351
|
|
|
704
1352
|
// 2.2 caller migration: batch progress is high-frequency informational.
|
|
705
|
-
appendEventFireAndForget(manifest.eventsPath, {
|
|
1353
|
+
appendEventFireAndForget(manifest.eventsPath, {
|
|
1354
|
+
type: "task.progress",
|
|
1355
|
+
runId: manifest.runId,
|
|
1356
|
+
message: `Starting ready batch with ${readyBatch.length} task(s).`,
|
|
1357
|
+
data: {
|
|
1358
|
+
taskIds: readyBatch.map((task) => task.id),
|
|
1359
|
+
readyCount: snapshot.ready.length,
|
|
1360
|
+
blockedCount: snapshot.blocked.length,
|
|
1361
|
+
runningCount: snapshot.running.length,
|
|
1362
|
+
doneCount: snapshot.done.length,
|
|
1363
|
+
selectedCount: readyBatch.length,
|
|
1364
|
+
maxConcurrent: concurrency.maxConcurrent,
|
|
1365
|
+
defaultConcurrency: concurrency.defaultConcurrency,
|
|
1366
|
+
concurrencyReason: approvalPending
|
|
1367
|
+
? `${concurrency.reason};plan-approval-read-only`
|
|
1368
|
+
: concurrency.reason,
|
|
1369
|
+
},
|
|
1370
|
+
});
|
|
706
1371
|
// Execute before_task_start hooks for the batch
|
|
707
1372
|
for (const task of readyBatch) {
|
|
708
|
-
const taskReport = await executeHook("before_task_start", {
|
|
1373
|
+
const taskReport = await executeHook("before_task_start", {
|
|
1374
|
+
runId: manifest.runId,
|
|
1375
|
+
taskId: task.id,
|
|
1376
|
+
cwd: manifest.cwd,
|
|
1377
|
+
});
|
|
709
1378
|
appendHookEvent(manifest, taskReport);
|
|
710
1379
|
if (taskReport.outcome === "block") {
|
|
711
|
-
tasks = tasks.map((t) =>
|
|
712
|
-
|
|
1380
|
+
tasks = tasks.map((t) =>
|
|
1381
|
+
t.id === task.id
|
|
1382
|
+
? {
|
|
1383
|
+
...t,
|
|
1384
|
+
status: "skipped" as const,
|
|
1385
|
+
error:
|
|
1386
|
+
taskReport.reason ??
|
|
1387
|
+
"before_task_start hook blocked execution.",
|
|
1388
|
+
}
|
|
1389
|
+
: t,
|
|
1390
|
+
);
|
|
1391
|
+
manifest = updateRunStatus(
|
|
1392
|
+
manifest,
|
|
1393
|
+
manifest.status,
|
|
1394
|
+
`Task '${task.id}' blocked by hook.`,
|
|
1395
|
+
);
|
|
713
1396
|
}
|
|
714
1397
|
}
|
|
715
|
-
const batchTasks = readyBatch.filter((task) =>
|
|
1398
|
+
const batchTasks = readyBatch.filter((task) =>
|
|
1399
|
+
tasks.find((t) => t.id === task.id && t.status !== "skipped"),
|
|
1400
|
+
);
|
|
716
1401
|
if (batchTasks.length > 1) {
|
|
717
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
1402
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1403
|
+
type: "task.parallel_start",
|
|
1404
|
+
runId: manifest.runId,
|
|
1405
|
+
message: `Launching ${batchTasks.length} tasks in PARALLEL (concurrency=${concurrency.selectedCount}): ${batchTasks.map((t) => `${t.role}(${t.id})`).join(", ")}`,
|
|
1406
|
+
data: {
|
|
1407
|
+
taskIds: batchTasks.map((t) => t.id),
|
|
1408
|
+
roles: batchTasks.map((t) => t.role),
|
|
1409
|
+
concurrency: concurrency.selectedCount,
|
|
1410
|
+
},
|
|
1411
|
+
});
|
|
718
1412
|
}
|
|
719
1413
|
const results = await mapConcurrent(
|
|
720
1414
|
batchTasks,
|
|
@@ -722,78 +1416,302 @@ async function executeTeamRunCore(
|
|
|
722
1416
|
async (task) => {
|
|
723
1417
|
const step = findStep(workflow, task);
|
|
724
1418
|
const agent = findAgent(input.agents, task);
|
|
725
|
-
const teamRole = input.team.roles.find(
|
|
726
|
-
|
|
727
|
-
|
|
1419
|
+
const teamRole = input.team.roles.find(
|
|
1420
|
+
(role) => role.name === task.role,
|
|
1421
|
+
);
|
|
1422
|
+
const perTaskRuntime = resolveTaskRuntimeKind(
|
|
1423
|
+
runtimeKind,
|
|
1424
|
+
task.role,
|
|
1425
|
+
input.runtimeConfig?.isolationPolicy,
|
|
1426
|
+
);
|
|
1427
|
+
const baseInput = {
|
|
1428
|
+
manifest,
|
|
1429
|
+
tasks,
|
|
1430
|
+
task,
|
|
1431
|
+
step,
|
|
1432
|
+
agent,
|
|
1433
|
+
signal: input.signal,
|
|
1434
|
+
executeWorkers: input.executeWorkers,
|
|
1435
|
+
runtimeKind: runtimeKind,
|
|
1436
|
+
taskRuntimeOverride:
|
|
1437
|
+
perTaskRuntime !== runtimeKind
|
|
1438
|
+
? perTaskRuntime
|
|
1439
|
+
: undefined,
|
|
1440
|
+
runtimeConfig: input.runtimeConfig,
|
|
1441
|
+
parentContext: input.parentContext,
|
|
1442
|
+
parentModel: input.parentModel,
|
|
1443
|
+
modelRegistry: input.modelRegistry,
|
|
1444
|
+
modelOverride: input.modelOverride,
|
|
1445
|
+
teamRoleModel: teamRole?.model,
|
|
1446
|
+
teamRoleSkills: teamRole?.skills,
|
|
1447
|
+
skillOverride: input.skillOverride,
|
|
1448
|
+
limits: input.limits,
|
|
1449
|
+
onJsonEvent: input.onJsonEvent,
|
|
1450
|
+
workspaceId: input.workspaceId,
|
|
1451
|
+
};
|
|
728
1452
|
// #1 (assessment): autoRetry now defaults ON (opt-out via reliability.autoRetry=false).
|
|
729
1453
|
// The dominant v0.9.13 failure was ChildTimeout ("worker became unresponsive") with
|
|
730
1454
|
// ZERO retries because this gate was opt-in. isRetryable() defaults to true when
|
|
731
1455
|
// retryableErrors is empty, so transient hangs now retry up to maxAttempts (3) with
|
|
732
1456
|
// exponential backoff. Set reliability.autoRetry=false to restore old single-shot behavior.
|
|
733
|
-
if (!shouldUseRetry(input.reliability))
|
|
734
|
-
|
|
1457
|
+
if (!shouldUseRetry(input.reliability))
|
|
1458
|
+
return withCorrelation(
|
|
1459
|
+
childCorrelation(manifest.runId, task.id),
|
|
1460
|
+
() => runTeamTask(baseInput),
|
|
1461
|
+
);
|
|
1462
|
+
let lastFailed:
|
|
1463
|
+
| { manifest: TeamRunManifest; tasks: TeamTaskState[] }
|
|
1464
|
+
| undefined;
|
|
735
1465
|
let lastAttemptId: string | undefined;
|
|
736
|
-
const attemptsSoFar: TaskAttemptState[] = [
|
|
1466
|
+
const attemptsSoFar: TaskAttemptState[] = [
|
|
1467
|
+
...(task.attempts ?? []),
|
|
1468
|
+
];
|
|
737
1469
|
const policy = retryPolicyFromConfig(input.reliability);
|
|
738
1470
|
try {
|
|
739
|
-
return await executeWithRetry(
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
1471
|
+
return await executeWithRetry(
|
|
1472
|
+
async (attempt, info) => {
|
|
1473
|
+
const startedAt = new Date().toISOString();
|
|
1474
|
+
const inFlightAttempts: TaskAttemptState[] = [
|
|
1475
|
+
...attemptsSoFar,
|
|
1476
|
+
{ attemptId: info.attemptId, startedAt },
|
|
1477
|
+
];
|
|
1478
|
+
input.metricRegistry
|
|
1479
|
+
?.counter(
|
|
1480
|
+
"crew.task.retry_attempt_total",
|
|
1481
|
+
"Retry attempts by run and task",
|
|
1482
|
+
)
|
|
1483
|
+
.inc({
|
|
1484
|
+
runId: manifest.runId,
|
|
1485
|
+
taskId: task.id,
|
|
1486
|
+
});
|
|
1487
|
+
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
1488
|
+
const fresh = loadRunManifestById(
|
|
1489
|
+
manifest.cwd,
|
|
1490
|
+
manifest.runId,
|
|
1491
|
+
);
|
|
1492
|
+
const freshManifest = fresh?.manifest ?? manifest;
|
|
1493
|
+
const freshTasks = fresh?.tasks ?? tasks;
|
|
1494
|
+
const freshTask =
|
|
1495
|
+
freshTasks.find(
|
|
1496
|
+
(item) => item.id === task.id,
|
|
1497
|
+
) ?? task;
|
|
1498
|
+
if (
|
|
1499
|
+
freshTask.status !== "queued" &&
|
|
1500
|
+
freshTask.status !== "running"
|
|
1501
|
+
)
|
|
1502
|
+
return {
|
|
1503
|
+
manifest: freshManifest,
|
|
1504
|
+
tasks: freshTasks,
|
|
1505
|
+
};
|
|
1506
|
+
const taskWithAttempt: TeamTaskState = {
|
|
1507
|
+
...freshTask,
|
|
1508
|
+
attempts: inFlightAttempts,
|
|
1509
|
+
};
|
|
1510
|
+
const result = await withCorrelation(
|
|
1511
|
+
childCorrelation(freshManifest.runId, task.id),
|
|
1512
|
+
() =>
|
|
1513
|
+
runTeamTask({
|
|
1514
|
+
...baseInput,
|
|
1515
|
+
manifest: freshManifest,
|
|
1516
|
+
tasks: freshTasks,
|
|
1517
|
+
task: taskWithAttempt,
|
|
1518
|
+
}),
|
|
1519
|
+
);
|
|
1520
|
+
const failed = failedTaskFrom(result, task.id);
|
|
1521
|
+
const endedAt = new Date().toISOString();
|
|
1522
|
+
const finishedAttempt: TaskAttemptState = {
|
|
1523
|
+
attemptId: info.attemptId,
|
|
1524
|
+
startedAt,
|
|
1525
|
+
endedAt,
|
|
1526
|
+
...(failed?.error
|
|
1527
|
+
? { error: failed.error }
|
|
1528
|
+
: {}),
|
|
1529
|
+
};
|
|
1530
|
+
attemptsSoFar.push(finishedAttempt);
|
|
1531
|
+
const withAttempt = result.tasks.map((item) =>
|
|
1532
|
+
item.id === task.id
|
|
1533
|
+
? { ...item, attempts: [...attemptsSoFar] }
|
|
1534
|
+
: item,
|
|
1535
|
+
);
|
|
1536
|
+
const enriched = {
|
|
1537
|
+
manifest: result.manifest,
|
|
1538
|
+
tasks: withAttempt,
|
|
1539
|
+
};
|
|
1540
|
+
if (failed) {
|
|
1541
|
+
lastFailed = enriched;
|
|
1542
|
+
throw new Error(
|
|
1543
|
+
failed.error ?? `Task ${task.id} failed.`,
|
|
1544
|
+
);
|
|
1545
|
+
}
|
|
1546
|
+
input.metricRegistry
|
|
1547
|
+
?.histogram(
|
|
1548
|
+
"crew.task.retry_count",
|
|
1549
|
+
"Retries per task",
|
|
1550
|
+
[0, 1, 2, 3, 5, 10],
|
|
1551
|
+
)
|
|
1552
|
+
.observe(
|
|
1553
|
+
{
|
|
1554
|
+
runId: manifest.runId,
|
|
1555
|
+
team: input.team.name,
|
|
1556
|
+
},
|
|
1557
|
+
Math.max(0, attempt - 1),
|
|
1558
|
+
);
|
|
1559
|
+
return enriched;
|
|
770
1560
|
},
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
1561
|
+
policy,
|
|
1562
|
+
{
|
|
1563
|
+
signal: input.signal,
|
|
1564
|
+
attemptId: (attempt) =>
|
|
1565
|
+
`${manifest.runId}:${task.id}:attempt-${attempt}`,
|
|
1566
|
+
onAttemptFailed: (
|
|
1567
|
+
attempt,
|
|
1568
|
+
error,
|
|
1569
|
+
delayMs,
|
|
1570
|
+
info,
|
|
1571
|
+
) => {
|
|
1572
|
+
lastAttemptId = info.attemptId;
|
|
1573
|
+
appendEventAsync(manifest.eventsPath, {
|
|
1574
|
+
type: "crew.task.retry_attempt",
|
|
1575
|
+
runId: manifest.runId,
|
|
1576
|
+
taskId: task.id,
|
|
1577
|
+
message: error.message,
|
|
1578
|
+
data: {
|
|
1579
|
+
attempt,
|
|
1580
|
+
attemptId: info.attemptId,
|
|
1581
|
+
delayMs,
|
|
1582
|
+
},
|
|
1583
|
+
metadata: { attemptId: info.attemptId },
|
|
1584
|
+
}).catch((error) =>
|
|
1585
|
+
logInternalError(
|
|
1586
|
+
"team-runner.retry-attempt",
|
|
1587
|
+
error,
|
|
1588
|
+
`taskId=${task.id}`,
|
|
1589
|
+
),
|
|
1590
|
+
);
|
|
1591
|
+
input.metricRegistry
|
|
1592
|
+
?.histogram(
|
|
1593
|
+
"crew.task.retry_delay_ms",
|
|
1594
|
+
"Retry backoff delay, milliseconds",
|
|
1595
|
+
)
|
|
1596
|
+
.observe(
|
|
1597
|
+
{
|
|
1598
|
+
runId: manifest.runId,
|
|
1599
|
+
taskId: task.id,
|
|
1600
|
+
},
|
|
1601
|
+
delayMs,
|
|
1602
|
+
);
|
|
1603
|
+
},
|
|
1604
|
+
onRetryGivenUp: (attempts, error, info) => {
|
|
1605
|
+
lastAttemptId = info.attemptId;
|
|
1606
|
+
appendDeadletter(manifest, {
|
|
1607
|
+
runId: manifest.runId,
|
|
1608
|
+
taskId: task.id,
|
|
1609
|
+
reason: "max-retries",
|
|
1610
|
+
attempts,
|
|
1611
|
+
attemptId: info.attemptId,
|
|
1612
|
+
lastError: error.message,
|
|
1613
|
+
timestamp: new Date().toISOString(),
|
|
1614
|
+
});
|
|
1615
|
+
input.metricRegistry
|
|
1616
|
+
?.counter(
|
|
1617
|
+
"crew.task.deadletter_total",
|
|
1618
|
+
"Deadletter triggers by reason",
|
|
1619
|
+
)
|
|
1620
|
+
.inc({ reason: "max-retries" });
|
|
1621
|
+
input.metricRegistry
|
|
1622
|
+
?.histogram(
|
|
1623
|
+
"crew.task.retry_count",
|
|
1624
|
+
"Retries per task",
|
|
1625
|
+
[0, 1, 2, 3, 5, 10],
|
|
1626
|
+
)
|
|
1627
|
+
.observe(
|
|
1628
|
+
{
|
|
1629
|
+
runId: manifest.runId,
|
|
1630
|
+
team: input.team.name,
|
|
1631
|
+
},
|
|
1632
|
+
Math.max(0, attempts - 1),
|
|
1633
|
+
);
|
|
1634
|
+
},
|
|
776
1635
|
},
|
|
777
|
-
|
|
1636
|
+
);
|
|
778
1637
|
} catch (retryError) {
|
|
779
|
-
if (
|
|
780
|
-
|
|
1638
|
+
if (
|
|
1639
|
+
retryError instanceof CrewCancellationError ||
|
|
1640
|
+
input.signal?.aborted
|
|
1641
|
+
) {
|
|
1642
|
+
const reason =
|
|
1643
|
+
retryError instanceof CrewCancellationError
|
|
1644
|
+
? retryError.reason
|
|
1645
|
+
: cancellationReasonFromSignal(input.signal);
|
|
781
1646
|
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
782
|
-
const fresh = loadRunManifestById(
|
|
1647
|
+
const fresh = loadRunManifestById(
|
|
1648
|
+
manifest.cwd,
|
|
1649
|
+
manifest.runId,
|
|
1650
|
+
);
|
|
783
1651
|
const freshManifest = fresh?.manifest ?? manifest;
|
|
784
1652
|
const freshTasks = fresh?.tasks ?? tasks;
|
|
785
|
-
const cancelledTasks = freshTasks.map((item) =>
|
|
786
|
-
|
|
787
|
-
|
|
1653
|
+
const cancelledTasks = freshTasks.map((item) =>
|
|
1654
|
+
item.id === task.id &&
|
|
1655
|
+
(item.status === "queued" ||
|
|
1656
|
+
item.status === "running")
|
|
1657
|
+
? {
|
|
1658
|
+
...item,
|
|
1659
|
+
status: "cancelled" as const,
|
|
1660
|
+
finishedAt: new Date().toISOString(),
|
|
1661
|
+
error: `${reason.message} (${reason.code})`,
|
|
1662
|
+
}
|
|
1663
|
+
: item,
|
|
1664
|
+
);
|
|
1665
|
+
appendEventAsync(freshManifest.eventsPath, {
|
|
1666
|
+
type: "task.cancelled",
|
|
1667
|
+
runId: freshManifest.runId,
|
|
1668
|
+
taskId: task.id,
|
|
1669
|
+
message: reason.message,
|
|
1670
|
+
data: { reason, phase: "retry" },
|
|
1671
|
+
metadata: lastAttemptId
|
|
1672
|
+
? { attemptId: lastAttemptId }
|
|
1673
|
+
: undefined,
|
|
1674
|
+
}).catch((error) =>
|
|
1675
|
+
logInternalError(
|
|
1676
|
+
"team-runner.cancelled",
|
|
1677
|
+
error,
|
|
1678
|
+
`taskId=${task.id}`,
|
|
1679
|
+
),
|
|
1680
|
+
);
|
|
1681
|
+
return {
|
|
1682
|
+
manifest: updateRunStatus(
|
|
1683
|
+
freshManifest,
|
|
1684
|
+
"cancelled",
|
|
1685
|
+
reason.message,
|
|
1686
|
+
),
|
|
1687
|
+
tasks: cancelledTasks,
|
|
1688
|
+
};
|
|
788
1689
|
}
|
|
789
1690
|
if (lastFailed) return lastFailed;
|
|
790
1691
|
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
791
|
-
const fresh = loadRunManifestById(
|
|
1692
|
+
const fresh = loadRunManifestById(
|
|
1693
|
+
manifest.cwd,
|
|
1694
|
+
manifest.runId,
|
|
1695
|
+
);
|
|
792
1696
|
const freshManifest = fresh?.manifest ?? manifest;
|
|
793
1697
|
const freshTasks = fresh?.tasks ?? tasks;
|
|
794
|
-
const freshTask =
|
|
795
|
-
|
|
796
|
-
|
|
1698
|
+
const freshTask =
|
|
1699
|
+
freshTasks.find((item) => item.id === task.id) ?? task;
|
|
1700
|
+
if (
|
|
1701
|
+
freshTask.status !== "queued" &&
|
|
1702
|
+
freshTask.status !== "running"
|
|
1703
|
+
)
|
|
1704
|
+
return { manifest: freshManifest, tasks: freshTasks };
|
|
1705
|
+
return withCorrelation(
|
|
1706
|
+
childCorrelation(freshManifest.runId, task.id),
|
|
1707
|
+
() =>
|
|
1708
|
+
runTeamTask({
|
|
1709
|
+
...baseInput,
|
|
1710
|
+
manifest: freshManifest,
|
|
1711
|
+
tasks: freshTasks,
|
|
1712
|
+
task: freshTask,
|
|
1713
|
+
}),
|
|
1714
|
+
);
|
|
797
1715
|
}
|
|
798
1716
|
},
|
|
799
1717
|
);
|
|
@@ -802,42 +1720,66 @@ async function executeTeamRunCore(
|
|
|
802
1720
|
// during parallel execution. Other workers may have written partial results
|
|
803
1721
|
// before one threw. Results may be partial - some tasks in-flight at error
|
|
804
1722
|
// time will not have entries in the results array.
|
|
805
|
-
const validResults = results.filter(
|
|
1723
|
+
const validResults = results.filter(
|
|
1724
|
+
(item): item is NonNullable<typeof item> => item !== undefined,
|
|
1725
|
+
);
|
|
806
1726
|
// Guard: if ALL parallel workers threw before returning, validResults is empty.
|
|
807
1727
|
// at(-1)! would crash. Mark the run failed rather than crashing.
|
|
808
1728
|
if (validResults.length === 0) {
|
|
809
|
-
manifest = updateRunStatus(
|
|
1729
|
+
manifest = updateRunStatus(
|
|
1730
|
+
manifest,
|
|
1731
|
+
"failed",
|
|
1732
|
+
"All parallel tasks failed catastrophically.",
|
|
1733
|
+
);
|
|
810
1734
|
return { manifest, tasks };
|
|
811
1735
|
}
|
|
812
1736
|
// Reconstruct manifest from the last worker's snapshot. The .artifacts field
|
|
813
|
-
// is re-merged from both the team-runner's in-memory state and all workers'
|
|
814
|
-
// snapshots, so artifact writes by task-runner (which individually save manifest
|
|
815
|
-
// after writing artifacts) are safely persisted. The in-memory manifest is only
|
|
816
|
-
// used for the next batch iteration's orchestration — actual persistence is safe.
|
|
817
|
-
// Use updateRunStatus to recompute manifest status from merged tasks rather than
|
|
818
|
-
// relying on the last result's manifest (which is arbitrary due to mapConcurrent
|
|
819
|
-
// returning results in arbitrary order).
|
|
820
|
-
// Use the in-memory manifest as base (not the last-completing worker's snapshot).
|
|
821
|
-
// Recompute status from merged tasks so the manifest reflects actual task state,
|
|
822
|
-
// not the arbitrary order in which mapConcurrent returned results.
|
|
823
|
-
// Read committed manifest from disk inside the lock so artifact merge is based
|
|
824
|
-
// on committed state, not in-memory state that may differ from disk.
|
|
825
|
-
const mergeResult = await withRunLock(manifest, async () => {
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
1737
|
+
// is re-merged from both the team-runner's in-memory state and all workers'
|
|
1738
|
+
// snapshots, so artifact writes by task-runner (which individually save manifest
|
|
1739
|
+
// after writing artifacts) are safely persisted. The in-memory manifest is only
|
|
1740
|
+
// used for the next batch iteration's orchestration — actual persistence is safe.
|
|
1741
|
+
// Use updateRunStatus to recompute manifest status from merged tasks rather than
|
|
1742
|
+
// relying on the last result's manifest (which is arbitrary due to mapConcurrent
|
|
1743
|
+
// returning results in arbitrary order).
|
|
1744
|
+
// Use the in-memory manifest as base (not the last-completing worker's snapshot).
|
|
1745
|
+
// Recompute status from merged tasks so the manifest reflects actual task state,
|
|
1746
|
+
// not the arbitrary order in which mapConcurrent returned results.
|
|
1747
|
+
// Read committed manifest from disk inside the lock so artifact merge is based
|
|
1748
|
+
// on committed state, not in-memory state that may differ from disk.
|
|
1749
|
+
const mergeResult = await withRunLock(manifest, async () => {
|
|
1750
|
+
const disk = loadRunManifestById(manifest.cwd, manifest.runId);
|
|
1751
|
+
const diskManifest = disk?.manifest ?? manifest;
|
|
1752
|
+
const diskArtifacts = diskManifest.artifacts;
|
|
1753
|
+
const reconciledArtifacts = mergeArtifacts(
|
|
1754
|
+
[
|
|
1755
|
+
...diskArtifacts,
|
|
1756
|
+
...validResults.map((item) => item.manifest.artifacts),
|
|
1757
|
+
].flat(),
|
|
1758
|
+
);
|
|
1759
|
+
const resultManifest = updateRunStatus(
|
|
1760
|
+
{ ...diskManifest, artifacts: reconciledArtifacts },
|
|
1761
|
+
"running",
|
|
1762
|
+
"Merged task updates from parallel batch.",
|
|
1763
|
+
);
|
|
1764
|
+
const resultTasks = mergeTaskUpdatesPreservingTerminal(
|
|
1765
|
+
tasks,
|
|
1766
|
+
validResults,
|
|
1767
|
+
);
|
|
1768
|
+
await saveRunManifestAsync(resultManifest);
|
|
1769
|
+
await saveRunTasksAsync(resultManifest, resultTasks);
|
|
1770
|
+
return { resultManifest, resultTasks };
|
|
1771
|
+
});
|
|
1772
|
+
manifest = mergeResult.resultManifest;
|
|
1773
|
+
tasks = mergeResult.resultTasks;
|
|
838
1774
|
|
|
839
1775
|
// Advance workflow phases whose tasks are all in terminal state
|
|
840
|
-
const terminalStatuses = new Set([
|
|
1776
|
+
const terminalStatuses = new Set([
|
|
1777
|
+
"completed",
|
|
1778
|
+
"failed",
|
|
1779
|
+
"skipped",
|
|
1780
|
+
"cancelled",
|
|
1781
|
+
"needs_attention",
|
|
1782
|
+
]);
|
|
841
1783
|
const phaseTaskMap = new Map<string, string[]>();
|
|
842
1784
|
for (const task of tasks) {
|
|
843
1785
|
if (!task.stepId) continue;
|
|
@@ -845,7 +1787,11 @@ tasks = mergeResult.resultTasks;
|
|
|
845
1787
|
existing.push(task.id);
|
|
846
1788
|
phaseTaskMap.set(task.stepId, existing);
|
|
847
1789
|
}
|
|
848
|
-
for (
|
|
1790
|
+
for (
|
|
1791
|
+
let pi = wfMachine.currentPhaseIndex;
|
|
1792
|
+
pi < wfMachine.phases.length;
|
|
1793
|
+
pi++
|
|
1794
|
+
) {
|
|
849
1795
|
const phase = wfMachine.phases[pi]!;
|
|
850
1796
|
const phaseTaskIds = phaseTaskMap.get(phase.name) ?? [];
|
|
851
1797
|
if (phaseTaskIds.length === 0) continue;
|
|
@@ -854,75 +1800,187 @@ tasks = mergeResult.resultTasks;
|
|
|
854
1800
|
return task ? terminalStatuses.has(task.status) : false;
|
|
855
1801
|
});
|
|
856
1802
|
if (!allTerminal) break;
|
|
857
|
-
if (
|
|
858
|
-
|
|
859
|
-
|
|
1803
|
+
if (
|
|
1804
|
+
phase.status !== "completed" &&
|
|
1805
|
+
phase.status !== "failed" &&
|
|
1806
|
+
phase.status !== "skipped"
|
|
1807
|
+
) {
|
|
1808
|
+
const completedArtifacts = manifest.artifacts
|
|
1809
|
+
.filter((a) => a.kind === "result" || a.kind === "summary")
|
|
1810
|
+
.map((a) => a.path);
|
|
1811
|
+
const previousPhaseStatus =
|
|
1812
|
+
pi > 0
|
|
1813
|
+
? (wfMachine.phases[pi - 1]?.status ?? "pending")
|
|
1814
|
+
: "completed";
|
|
860
1815
|
const wfContext: PhaseGuardContext = {
|
|
861
1816
|
completedArtifacts,
|
|
862
1817
|
previousPhaseStatus,
|
|
863
|
-
taskResults: tasks
|
|
1818
|
+
taskResults: tasks
|
|
1819
|
+
.filter(
|
|
1820
|
+
(t) =>
|
|
1821
|
+
t.status === "completed" ||
|
|
1822
|
+
t.status === "needs_attention",
|
|
1823
|
+
)
|
|
1824
|
+
.map((t) => ({
|
|
1825
|
+
taskId: t.id,
|
|
1826
|
+
status: t.status,
|
|
1827
|
+
outputPath: t.resultArtifact?.path,
|
|
1828
|
+
})),
|
|
864
1829
|
};
|
|
865
1830
|
// Determine phase transition status based on individual task outcomes
|
|
866
|
-
const phaseTasks = phaseTaskIds
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
const
|
|
1831
|
+
const phaseTasks = phaseTaskIds
|
|
1832
|
+
.map((taskId) => tasks.find((t) => t.id === taskId))
|
|
1833
|
+
.filter((t): t is NonNullable<typeof t> => t !== undefined);
|
|
1834
|
+
const hasFailedOrCancelled = phaseTasks.some(
|
|
1835
|
+
(t) => t.status === "failed" || t.status === "cancelled",
|
|
1836
|
+
);
|
|
1837
|
+
const phaseStatus = hasFailedOrCancelled
|
|
1838
|
+
? "failed"
|
|
1839
|
+
: "completed";
|
|
1840
|
+
const transition = transitionPhase(
|
|
1841
|
+
wfMachine,
|
|
1842
|
+
pi,
|
|
1843
|
+
phaseStatus,
|
|
1844
|
+
wfContext,
|
|
1845
|
+
);
|
|
870
1846
|
wfMachine = transition.machine;
|
|
871
1847
|
if (transition.guardResult && !transition.guardResult.allowed) {
|
|
872
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
1848
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1849
|
+
type: "workflow.phase_guard_blocked",
|
|
1850
|
+
runId: manifest.runId,
|
|
1851
|
+
message: `Workflow phase '${phase.name}' guard blocked: ${transition.guardResult.reason ?? "unknown"}`,
|
|
1852
|
+
data: {
|
|
1853
|
+
phaseIndex: pi,
|
|
1854
|
+
phaseName: phase.name,
|
|
1855
|
+
reason: transition.guardResult.reason,
|
|
1856
|
+
},
|
|
1857
|
+
});
|
|
873
1858
|
break;
|
|
874
1859
|
}
|
|
875
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
1860
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1861
|
+
type:
|
|
1862
|
+
phaseStatus === "failed"
|
|
1863
|
+
? "workflow.phase_failed"
|
|
1864
|
+
: "workflow.phase_completed",
|
|
1865
|
+
runId: manifest.runId,
|
|
1866
|
+
message: `Workflow phase '${phase.name}' ${phaseStatus}.`,
|
|
1867
|
+
data: { phaseIndex: pi, phaseStatus },
|
|
1868
|
+
});
|
|
876
1869
|
}
|
|
877
1870
|
wfMachine = { ...wfMachine, currentPhaseIndex: pi + 1 };
|
|
878
1871
|
}
|
|
879
1872
|
|
|
880
|
-
const cancelledResult = results.find(
|
|
1873
|
+
const cancelledResult = results.find(
|
|
1874
|
+
(item) => item.manifest.status === "cancelled",
|
|
1875
|
+
);
|
|
881
1876
|
if (cancelledResult || input.signal?.aborted) {
|
|
882
|
-
const reason = input.signal?.aborted
|
|
883
|
-
|
|
1877
|
+
const reason = input.signal?.aborted
|
|
1878
|
+
? cancellationReasonFromSignal(input.signal)
|
|
1879
|
+
: undefined;
|
|
1880
|
+
const message =
|
|
1881
|
+
reason?.message ??
|
|
1882
|
+
cancelledResult?.manifest.summary ??
|
|
1883
|
+
"Run cancelled during task execution.";
|
|
884
1884
|
manifest = { ...manifest, status: "running" };
|
|
885
1885
|
manifest = updateRunStatus(manifest, "cancelled", message);
|
|
886
1886
|
await saveRunTasksAsync(manifest, tasks);
|
|
887
|
-
saveCrewAgents(
|
|
1887
|
+
saveCrewAgents(
|
|
1888
|
+
manifest,
|
|
1889
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1890
|
+
);
|
|
888
1891
|
await saveRunManifestAsync(manifest);
|
|
889
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
1892
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1893
|
+
type: "run.cancelled",
|
|
1894
|
+
runId: manifest.runId,
|
|
1895
|
+
message,
|
|
1896
|
+
data: {
|
|
1897
|
+
reason,
|
|
1898
|
+
phase: "task-batch",
|
|
1899
|
+
cancelledResultRunId: cancelledResult?.manifest.runId,
|
|
1900
|
+
},
|
|
1901
|
+
});
|
|
890
1902
|
return { manifest, tasks };
|
|
891
1903
|
}
|
|
892
1904
|
queueIndex = buildTaskGraphIndex(tasks);
|
|
893
1905
|
const injectedAfterBatch = attemptAdaptivePlan();
|
|
894
1906
|
if (injectedAfterBatch.missing) {
|
|
895
|
-
tasks = markBlocked(
|
|
1907
|
+
tasks = markBlocked(
|
|
1908
|
+
tasks,
|
|
1909
|
+
"Adaptive planner did not produce a valid subagent plan.",
|
|
1910
|
+
);
|
|
896
1911
|
await saveRunTasksAsync(manifest, tasks);
|
|
897
|
-
saveCrewAgents(
|
|
898
|
-
|
|
1912
|
+
saveCrewAgents(
|
|
1913
|
+
manifest,
|
|
1914
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1915
|
+
);
|
|
1916
|
+
manifest = updateRunStatus(
|
|
1917
|
+
manifest,
|
|
1918
|
+
"blocked",
|
|
1919
|
+
"Adaptive planner did not produce a valid subagent plan.",
|
|
1920
|
+
);
|
|
899
1921
|
return { manifest, tasks };
|
|
900
1922
|
}
|
|
901
1923
|
if (injectedAfterBatch.injected) {
|
|
902
|
-
manifest = requiresPlanApproval(workflow, input.runtimeConfig)
|
|
1924
|
+
manifest = requiresPlanApproval(workflow, input.runtimeConfig)
|
|
1925
|
+
? ensurePlanApprovalRequested(manifest, tasks)
|
|
1926
|
+
: manifest;
|
|
903
1927
|
queueIndex = buildTaskGraphIndex(tasks);
|
|
904
|
-
} else if (
|
|
1928
|
+
} else if (
|
|
1929
|
+
requiresPlanApproval(workflow, input.runtimeConfig) &&
|
|
1930
|
+
(hasPendingMutatingAdaptiveTask(tasks) ||
|
|
1931
|
+
hasPendingMutatingTaskAtBoundary(tasks))
|
|
1932
|
+
) {
|
|
905
1933
|
manifest = ensurePlanApprovalRequested(manifest, tasks);
|
|
906
1934
|
}
|
|
907
1935
|
if (manifest.planApproval?.status === "cancelled") {
|
|
908
1936
|
tasks = cancelPlanTasks(tasks, "Plan approval was cancelled.");
|
|
909
1937
|
await saveRunTasksAsync(manifest, tasks);
|
|
910
|
-
saveCrewAgents(
|
|
911
|
-
|
|
1938
|
+
saveCrewAgents(
|
|
1939
|
+
manifest,
|
|
1940
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1941
|
+
);
|
|
1942
|
+
manifest = updateRunStatus(
|
|
1943
|
+
manifest,
|
|
1944
|
+
"cancelled",
|
|
1945
|
+
"Plan approval was cancelled.",
|
|
1946
|
+
);
|
|
912
1947
|
return { manifest, tasks };
|
|
913
1948
|
}
|
|
914
1949
|
await saveRunTasksAsync(manifest, tasks);
|
|
915
|
-
saveCrewAgents(
|
|
916
|
-
|
|
1950
|
+
saveCrewAgents(
|
|
1951
|
+
manifest,
|
|
1952
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1953
|
+
);
|
|
1954
|
+
const completedBatch = tasks.filter((t) =>
|
|
1955
|
+
batchTasks.some((bt) => bt.id === t.id),
|
|
1956
|
+
);
|
|
917
1957
|
const batchArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
918
1958
|
kind: "summary",
|
|
919
1959
|
relativePath: `batches/${batchTasks.map((task) => task.id).join("+")}.md`,
|
|
920
1960
|
producer: "team-runner",
|
|
921
1961
|
content: aggregateTaskOutputs(completedBatch, manifest),
|
|
922
1962
|
});
|
|
923
|
-
const groupDelivery = deliverGroupJoin({
|
|
924
|
-
|
|
925
|
-
|
|
1963
|
+
const groupDelivery = deliverGroupJoin({
|
|
1964
|
+
manifest,
|
|
1965
|
+
mode: resolveGroupJoinMode(input.runtimeConfig),
|
|
1966
|
+
batch: batchTasks,
|
|
1967
|
+
allTasks: tasks,
|
|
1968
|
+
});
|
|
1969
|
+
manifest = {
|
|
1970
|
+
...manifest,
|
|
1971
|
+
artifacts: mergeArtifacts([
|
|
1972
|
+
...manifest.artifacts,
|
|
1973
|
+
batchArtifact,
|
|
1974
|
+
...(groupDelivery?.artifact ? [groupDelivery.artifact] : []),
|
|
1975
|
+
]),
|
|
1976
|
+
};
|
|
1977
|
+
manifest = writeProgress(
|
|
1978
|
+
manifest,
|
|
1979
|
+
tasks,
|
|
1980
|
+
"team-runner",
|
|
1981
|
+
input.executeWorkers,
|
|
1982
|
+
input.runtimeConfig,
|
|
1983
|
+
);
|
|
926
1984
|
await saveRunManifestAsync(manifest);
|
|
927
1985
|
}
|
|
928
1986
|
|
|
@@ -930,29 +1988,85 @@ tasks = mergeResult.resultTasks;
|
|
|
930
1988
|
const waiting = tasks.find((task) => task.status === "waiting");
|
|
931
1989
|
const running = tasks.find((task) => task.status === "running");
|
|
932
1990
|
manifest = applyPolicy(manifest, tasks, input.limits);
|
|
933
|
-
const effectiveness = evaluateRunEffectiveness({
|
|
1991
|
+
const effectiveness = evaluateRunEffectiveness({
|
|
1992
|
+
manifest,
|
|
1993
|
+
tasks,
|
|
1994
|
+
executeWorkers: input.executeWorkers,
|
|
1995
|
+
runtimeConfig: input.runtimeConfig,
|
|
1996
|
+
});
|
|
934
1997
|
const effectivenessDecision = effectivenessPolicyDecision(effectiveness);
|
|
935
1998
|
if (effectivenessDecision) {
|
|
936
|
-
manifest = {
|
|
937
|
-
|
|
1999
|
+
manifest = {
|
|
2000
|
+
...manifest,
|
|
2001
|
+
policyDecisions: [
|
|
2002
|
+
...(manifest.policyDecisions ?? []),
|
|
2003
|
+
effectivenessDecision,
|
|
2004
|
+
],
|
|
2005
|
+
updatedAt: new Date().toISOString(),
|
|
2006
|
+
};
|
|
2007
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
2008
|
+
type: "run.effectiveness",
|
|
2009
|
+
runId: manifest.runId,
|
|
2010
|
+
message: effectivenessDecision.message,
|
|
2011
|
+
data: { effectiveness, policyDecision: effectivenessDecision },
|
|
2012
|
+
});
|
|
938
2013
|
}
|
|
939
|
-
const blockingDecision = manifest.policyDecisions?.find(
|
|
2014
|
+
const blockingDecision = manifest.policyDecisions?.find(
|
|
2015
|
+
(item) => item.action === "block" || item.action === "escalate",
|
|
2016
|
+
);
|
|
940
2017
|
if (failed) {
|
|
941
|
-
manifest = updateRunStatus(
|
|
2018
|
+
manifest = updateRunStatus(
|
|
2019
|
+
manifest,
|
|
2020
|
+
"failed",
|
|
2021
|
+
`Failed at task '${failed.id}'.`,
|
|
2022
|
+
);
|
|
942
2023
|
} else if (waiting) {
|
|
943
|
-
manifest = updateRunStatus(
|
|
2024
|
+
manifest = updateRunStatus(
|
|
2025
|
+
manifest,
|
|
2026
|
+
"blocked",
|
|
2027
|
+
`Waiting for response to task '${waiting.id}'.`,
|
|
2028
|
+
);
|
|
944
2029
|
} else if (running) {
|
|
945
|
-
manifest = updateRunStatus(
|
|
2030
|
+
manifest = updateRunStatus(
|
|
2031
|
+
manifest,
|
|
2032
|
+
"blocked",
|
|
2033
|
+
`Task '${running.id}' is still running.`,
|
|
2034
|
+
);
|
|
946
2035
|
} else if (effectiveness.severity === "failed") {
|
|
947
|
-
manifest = updateRunStatus(
|
|
2036
|
+
manifest = updateRunStatus(
|
|
2037
|
+
manifest,
|
|
2038
|
+
"failed",
|
|
2039
|
+
effectivenessDecision?.message ?? "Run effectiveness guard failed.",
|
|
2040
|
+
);
|
|
948
2041
|
} else if (effectiveness.severity === "blocked") {
|
|
949
|
-
manifest = updateRunStatus(
|
|
2042
|
+
manifest = updateRunStatus(
|
|
2043
|
+
manifest,
|
|
2044
|
+
"blocked",
|
|
2045
|
+
effectivenessDecision?.message ??
|
|
2046
|
+
"Run effectiveness guard blocked completion.",
|
|
2047
|
+
);
|
|
950
2048
|
} else if (blockingDecision) {
|
|
951
|
-
manifest = updateRunStatus(
|
|
2049
|
+
manifest = updateRunStatus(
|
|
2050
|
+
manifest,
|
|
2051
|
+
"blocked",
|
|
2052
|
+
blockingDecision.message,
|
|
2053
|
+
);
|
|
952
2054
|
} else {
|
|
953
|
-
manifest = updateRunStatus(
|
|
2055
|
+
manifest = updateRunStatus(
|
|
2056
|
+
manifest,
|
|
2057
|
+
"completed",
|
|
2058
|
+
input.executeWorkers
|
|
2059
|
+
? "Team workflow completed."
|
|
2060
|
+
: "Team workflow scaffold completed without launching child workers.",
|
|
2061
|
+
);
|
|
954
2062
|
}
|
|
955
|
-
manifest = writeProgress(
|
|
2063
|
+
manifest = writeProgress(
|
|
2064
|
+
manifest,
|
|
2065
|
+
tasks,
|
|
2066
|
+
"team-runner",
|
|
2067
|
+
input.executeWorkers,
|
|
2068
|
+
input.runtimeConfig,
|
|
2069
|
+
);
|
|
956
2070
|
await saveRunManifestAsync(manifest);
|
|
957
2071
|
const usage = aggregateUsage(tasks);
|
|
958
2072
|
const summaryArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
@@ -972,17 +2086,28 @@ tasks = mergeResult.resultTasks;
|
|
|
972
2086
|
...tasks.map(formatTaskProgress),
|
|
973
2087
|
"",
|
|
974
2088
|
"## Effectiveness",
|
|
975
|
-
...runEffectivenessLines(
|
|
2089
|
+
...runEffectivenessLines(
|
|
2090
|
+
manifest,
|
|
2091
|
+
tasks,
|
|
2092
|
+
input.executeWorkers,
|
|
2093
|
+
input.runtimeConfig,
|
|
2094
|
+
),
|
|
976
2095
|
"",
|
|
977
2096
|
"## Policy decisions",
|
|
978
|
-
...(manifest.policyDecisions?.length
|
|
2097
|
+
...(manifest.policyDecisions?.length
|
|
2098
|
+
? summarizePolicyDecisions(manifest.policyDecisions)
|
|
2099
|
+
: ["- (none)"]),
|
|
979
2100
|
"",
|
|
980
2101
|
].join("\n"),
|
|
981
2102
|
});
|
|
982
2103
|
// Build the complete manifest BEFORE acquiring the lock so the artifacts array
|
|
983
2104
|
// is already incorporated into the manifest object that will be atomically written.
|
|
984
2105
|
// This prevents crash-between-mutation-and-lock from leaving inconsistent state.
|
|
985
|
-
const finalManifest = {
|
|
2106
|
+
const finalManifest = {
|
|
2107
|
+
...manifest,
|
|
2108
|
+
updatedAt: new Date().toISOString(),
|
|
2109
|
+
artifacts: [...manifest.artifacts, summaryArtifact],
|
|
2110
|
+
};
|
|
986
2111
|
// Joint atomic save: wrap manifest + tasks in a single run lock so they are
|
|
987
2112
|
// written together or not at all. Crash between separate saveRunManifestAsync
|
|
988
2113
|
// and saveRunTasksAsync calls could leave manifest/tasks.json out of sync.
|
|
@@ -1000,7 +2125,9 @@ tasks = mergeResult.resultTasks;
|
|
|
1000
2125
|
// health feature) AND created junk dirs that the recursive state watcher then
|
|
1001
2126
|
// attached extra inotify watches to. Fix: compute the real crew root (3 up)
|
|
1002
2127
|
// and make HEALTH_DIR relative to it.
|
|
1003
|
-
const crewRoot = path.dirname(
|
|
2128
|
+
const crewRoot = path.dirname(
|
|
2129
|
+
path.dirname(path.dirname(finalManifest.stateRoot)),
|
|
2130
|
+
);
|
|
1004
2131
|
const healthStore = new HealthStore(crewRoot);
|
|
1005
2132
|
healthStore.saveSnapshot({
|
|
1006
2133
|
runId: finalManifest.runId,
|