pi-crew 0.9.13 → 0.9.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +114 -0
- package/README.md +66 -0
- package/docs/bugs/bug-021-notification-badge-counter-misleading.md +293 -0
- package/docs/bugs/bug-023-chain-windows-path-resolution.md +106 -0
- package/package.json +1 -1
- package/src/config/defaults.ts +3 -1
- package/src/config/types.ts +5 -0
- package/src/extension/pi-api.ts +1 -1
- package/src/extension/registration/team-tool.ts +145 -35
- package/src/extension/team-tool/run.ts +646 -150
- package/src/runtime/child-pi.ts +87 -1
- package/src/runtime/goal-achievement.ts +131 -0
- package/src/runtime/recovery-recipes.ts +35 -0
- package/src/runtime/task-runner.ts +6 -0
- package/src/runtime/team-runner.ts +1482 -310
- package/src/state/types.ts +3 -0
- package/src/ui/widget/index.ts +1 -1
- package/src/ui/widget/widget-formatters.ts +14 -1
- package/src/ui/widget/widget-renderer.ts +13 -2
- package/src/utils/redaction.ts +20 -0
- package/src/workflows/discover-workflows.ts +144 -29
- package/src/workflows/preflight-validator.ts +195 -0
- package/src/workflows/topology-analyzer.ts +219 -0
- package/src/workflows/workflow-config.ts +11 -0
- package/workflows/chain.workflow.md +6 -0
- package/workflows/default.workflow.md +1 -0
- package/workflows/fast-fix.workflow.md +1 -0
- package/workflows/implementation.workflow.md +1 -0
- package/workflows/parallel-research.workflow.md +1 -0
- package/workflows/pipeline.workflow.md +1 -0
- package/workflows/research.workflow.md +1 -0
- package/workflows/review.workflow.md +1 -0
|
@@ -1,47 +1,117 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import type { AgentConfig } from "../agents/agent-config.ts";
|
|
4
|
-
import type {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
4
|
+
import type {
|
|
5
|
+
CrewLimitsConfig,
|
|
6
|
+
CrewReliabilityConfig,
|
|
7
|
+
CrewRuntimeConfig,
|
|
8
|
+
} from "../config/config.ts";
|
|
9
|
+
import { appendHookEvent, executeHook } from "../hooks/registry.ts";
|
|
10
|
+
import {
|
|
11
|
+
childCorrelation,
|
|
12
|
+
withCorrelation,
|
|
13
|
+
} from "../observability/correlation.ts";
|
|
14
|
+
import type { MetricRegistry } from "../observability/metric-registry.ts";
|
|
15
|
+
import { PluginRegistry } from "../plugins/plugin-registry.ts";
|
|
16
|
+
import {
|
|
17
|
+
NextJsPlugin,
|
|
18
|
+
VitePlugin,
|
|
19
|
+
VitestPlugin,
|
|
20
|
+
} from "../plugins/plugins/index.ts";
|
|
8
21
|
import { writeArtifact } from "../state/artifact-store.ts";
|
|
9
|
-
import {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
22
|
+
import {
|
|
23
|
+
appendEvent,
|
|
24
|
+
appendEventAsync,
|
|
25
|
+
appendEventFireAndForget,
|
|
26
|
+
} from "../state/event-log.ts";
|
|
27
|
+
import { HealthStore } from "../state/health-store.ts";
|
|
14
28
|
import { withRunLock } from "../state/locks.ts";
|
|
29
|
+
import {
|
|
30
|
+
loadRunManifestById,
|
|
31
|
+
saveRunManifest,
|
|
32
|
+
saveRunManifestAsync,
|
|
33
|
+
saveRunTasksAsync,
|
|
34
|
+
updateRunStatus,
|
|
35
|
+
} from "../state/state-store.ts";
|
|
36
|
+
import type {
|
|
37
|
+
ArtifactDescriptor,
|
|
38
|
+
PolicyDecision,
|
|
39
|
+
TaskAttemptState,
|
|
40
|
+
TeamRunManifest,
|
|
41
|
+
TeamTaskState,
|
|
42
|
+
} from "../state/types.ts";
|
|
15
43
|
import { aggregateUsage, formatUsage } from "../state/usage.ts";
|
|
16
|
-
import type {
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
19
|
-
|
|
20
|
-
|
|
44
|
+
import type { TeamConfig } from "../teams/team-config.ts";
|
|
45
|
+
import { logInternalError } from "../utils/internal-error.ts";
|
|
46
|
+
import type {
|
|
47
|
+
WorkflowConfig,
|
|
48
|
+
WorkflowStep,
|
|
49
|
+
} from "../workflows/workflow-config.ts";
|
|
21
50
|
import { checkBranchFreshness } from "../worktree/branch-freshness.ts";
|
|
22
|
-
import {
|
|
51
|
+
import {
|
|
52
|
+
buildSyntheticTerminalEvidence,
|
|
53
|
+
CrewCancellationError,
|
|
54
|
+
cancellationReasonFromSignal,
|
|
55
|
+
} from "./cancellation.ts";
|
|
56
|
+
import { resolveBatchConcurrency } from "./concurrency.ts";
|
|
23
57
|
import { readCrewAgents, saveCrewAgents } from "./crew-agent-records.ts";
|
|
24
|
-
import {
|
|
58
|
+
import type { CrewRuntimeKind } from "./crew-agent-runtime.ts";
|
|
59
|
+
import { crewHooks } from "./crew-hooks.ts";
|
|
60
|
+
import { appendDeadletter } from "./deadletter.ts";
|
|
61
|
+
import {
|
|
62
|
+
effectivenessPolicyDecision,
|
|
63
|
+
evaluateRunEffectiveness,
|
|
64
|
+
formatRunEffectivenessLines,
|
|
65
|
+
} from "./effectiveness.ts";
|
|
66
|
+
import {
|
|
67
|
+
applyGoalAchievement,
|
|
68
|
+
assessGoalAchievement,
|
|
69
|
+
} from "./goal-achievement.ts";
|
|
25
70
|
import { deliverGroupJoin, resolveGroupJoinMode } from "./group-join.ts";
|
|
26
|
-
import { runTeamTask } from "./task-runner.ts";
|
|
27
71
|
import { terminateLiveAgentsForRun } from "./live-agent-manager.ts";
|
|
28
|
-
import { createWorkflowStateMachine, validatePhasePreconditions, transitionPhase, type PhaseState, type PhaseGuardContext } from "./workflow-state.ts";
|
|
29
|
-
import { executeWithRetry, DEFAULT_RETRY_POLICY, type RetryPolicy } from "./retry-executor.ts";
|
|
30
|
-
import { appendDeadletter } from "./deadletter.ts";
|
|
31
|
-
import type { MetricRegistry } from "../observability/metric-registry.ts";
|
|
32
|
-
import { childCorrelation, withCorrelation } from "../observability/correlation.ts";
|
|
33
|
-
import { crewHooks } from "./crew-hooks.ts";
|
|
34
|
-
import { resolveBatchConcurrency } from "./concurrency.ts";
|
|
35
72
|
import { mapConcurrent } from "./parallel-utils.ts";
|
|
73
|
+
import {
|
|
74
|
+
evaluateCrewPolicy,
|
|
75
|
+
summarizePolicyDecisions,
|
|
76
|
+
} from "./policy-engine.ts";
|
|
77
|
+
import {
|
|
78
|
+
buildRecoveryLedger,
|
|
79
|
+
shouldRerunFailedTask,
|
|
80
|
+
} from "./recovery-recipes.ts";
|
|
81
|
+
import {
|
|
82
|
+
DEFAULT_RETRY_POLICY,
|
|
83
|
+
executeWithRetry,
|
|
84
|
+
type RetryPolicy,
|
|
85
|
+
} from "./retry-executor.ts";
|
|
36
86
|
import { permissionForRole } from "./role-permission.ts";
|
|
37
|
-
import {
|
|
87
|
+
import {
|
|
88
|
+
registerRunPromise,
|
|
89
|
+
rejectRunPromise,
|
|
90
|
+
resolveRunPromise,
|
|
91
|
+
} from "./run-tracker.ts";
|
|
92
|
+
import { resolveTaskRuntimeKind } from "./runtime-policy.ts";
|
|
93
|
+
import type { CrewRuntimeCapabilities } from "./runtime-resolver.ts";
|
|
94
|
+
import { recordsForMaterializedTasks } from "./task-display.ts";
|
|
95
|
+
import {
|
|
96
|
+
buildExecutionPlan as buildDagExecutionPlan,
|
|
97
|
+
getReadyTasks as getDagReadyTasks,
|
|
98
|
+
type TaskNode,
|
|
99
|
+
} from "./task-graph.ts";
|
|
100
|
+
import {
|
|
101
|
+
buildTaskGraphIndex,
|
|
102
|
+
refreshTaskGraphQueues,
|
|
103
|
+
taskGraphSnapshot,
|
|
104
|
+
} from "./task-graph-scheduler.ts";
|
|
105
|
+
import { aggregateTaskOutputs } from "./task-output-context.ts";
|
|
106
|
+
import { runTeamTask } from "./task-runner.ts";
|
|
38
107
|
import { clearTrackedTaskUsage } from "./usage-tracker.ts";
|
|
39
|
-
import {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
108
|
+
import {
|
|
109
|
+
createWorkflowStateMachine,
|
|
110
|
+
type PhaseGuardContext,
|
|
111
|
+
type PhaseState,
|
|
112
|
+
transitionPhase,
|
|
113
|
+
validatePhasePreconditions,
|
|
114
|
+
} from "./workflow-state.ts";
|
|
45
115
|
|
|
46
116
|
// Built-in plugin registry for framework awareness.
|
|
47
117
|
// NOTE: This registry is registered here for future use. The integration
|
|
@@ -72,13 +142,17 @@ function startTeamRunHeartbeat(stateRoot: string, runId: string): () => void {
|
|
|
72
142
|
// captured manifest.updatedAt once at startup, making the value
|
|
73
143
|
// permanently stale throughout the run.
|
|
74
144
|
const now = new Date().toISOString();
|
|
75
|
-
fs.writeFileSync(
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
145
|
+
fs.writeFileSync(
|
|
146
|
+
heartbeatPath,
|
|
147
|
+
JSON.stringify({
|
|
148
|
+
pid: process.pid,
|
|
149
|
+
at: Date.now(),
|
|
150
|
+
runId,
|
|
151
|
+
kind: "team-runner",
|
|
152
|
+
lastTaskUpdateAt: now,
|
|
153
|
+
}),
|
|
154
|
+
{ encoding: "utf-8", mode: 0o600 },
|
|
155
|
+
);
|
|
82
156
|
} catch {
|
|
83
157
|
// best-effort
|
|
84
158
|
}
|
|
@@ -118,19 +192,39 @@ export interface ExecuteTeamRunInput {
|
|
|
118
192
|
}
|
|
119
193
|
|
|
120
194
|
function findStep(workflow: WorkflowConfig, task: TeamTaskState): WorkflowStep {
|
|
121
|
-
const step = workflow.steps.find(
|
|
122
|
-
|
|
195
|
+
const step = workflow.steps.find(
|
|
196
|
+
(candidate) => candidate.id === task.stepId,
|
|
197
|
+
);
|
|
198
|
+
if (!step)
|
|
199
|
+
throw new Error(
|
|
200
|
+
`Workflow step '${task.stepId}' not found for task '${task.id}'.`,
|
|
201
|
+
);
|
|
123
202
|
return step;
|
|
124
203
|
}
|
|
125
204
|
|
|
126
205
|
function findAgent(agents: AgentConfig[], task: TeamTaskState): AgentConfig {
|
|
127
206
|
const agent = agents.find((candidate) => candidate.name === task.agent);
|
|
128
|
-
if (!agent)
|
|
207
|
+
if (!agent)
|
|
208
|
+
throw new Error(
|
|
209
|
+
`Agent '${task.agent}' not found for task '${task.id}'.`,
|
|
210
|
+
);
|
|
129
211
|
return agent;
|
|
130
212
|
}
|
|
131
213
|
|
|
132
214
|
function markBlocked(tasks: TeamTaskState[], reason: string): TeamTaskState[] {
|
|
133
|
-
return tasks.map((task) =>
|
|
215
|
+
return tasks.map((task) =>
|
|
216
|
+
task.status === "queued"
|
|
217
|
+
? {
|
|
218
|
+
...task,
|
|
219
|
+
status: "skipped",
|
|
220
|
+
error: reason,
|
|
221
|
+
finishedAt: new Date().toISOString(),
|
|
222
|
+
graph: task.graph
|
|
223
|
+
? { ...task.graph, queue: "blocked" }
|
|
224
|
+
: undefined,
|
|
225
|
+
}
|
|
226
|
+
: task,
|
|
227
|
+
);
|
|
134
228
|
}
|
|
135
229
|
|
|
136
230
|
function mergeArtifacts(items: ArtifactDescriptor[]): ArtifactDescriptor[] {
|
|
@@ -159,37 +253,59 @@ function safeFinishedAt(task: TeamTaskState): number {
|
|
|
159
253
|
* and the updated task has a valid finite finishedAt. Malformed finishedAt
|
|
160
254
|
* should be replaced rather than persisting corruption.
|
|
161
255
|
*/
|
|
162
|
-
function isMalformedFinishedAtReplacement(
|
|
256
|
+
function isMalformedFinishedAtReplacement(
|
|
257
|
+
currentTime: number,
|
|
258
|
+
updatedTime: number,
|
|
259
|
+
): boolean {
|
|
163
260
|
return !Number.isFinite(currentTime) && Number.isFinite(updatedTime);
|
|
164
261
|
}
|
|
165
262
|
|
|
166
|
-
function shouldMergeTaskUpdate(
|
|
263
|
+
function shouldMergeTaskUpdate(
|
|
264
|
+
current: TeamTaskState,
|
|
265
|
+
updated: TeamTaskState,
|
|
266
|
+
): boolean {
|
|
167
267
|
// Parallel workers receive the same input snapshot. A later result may still
|
|
168
268
|
// contain stale queued/running copies of tasks that another worker already
|
|
169
269
|
// completed. Never let those stale snapshots regress durable task state.
|
|
170
|
-
if (current.status === "waiting" && updated.status === "running")
|
|
270
|
+
if (current.status === "waiting" && updated.status === "running")
|
|
271
|
+
return false;
|
|
171
272
|
// Block terminal→non-terminal transitions (e.g. completed→running).
|
|
172
273
|
// A task that has reached a terminal state must not be resurrected.
|
|
173
274
|
const currentIsTerminal = !isNonTerminalTaskStatus(current.status);
|
|
174
275
|
const updatedIsNonTerminal = isNonTerminalTaskStatus(updated.status);
|
|
175
276
|
if (currentIsTerminal && updatedIsNonTerminal) return false;
|
|
176
277
|
// Explicitly block completed↔needs_attention terminal-to-terminal transitions.
|
|
177
|
-
// Both are success terminal states used interchangeably; stale worker updates must
|
|
178
|
-
// not cause a completed task to appear as needs_attention or vice versa.
|
|
179
|
-
if (current.status === "completed" && updated.status === "needs_attention")
|
|
180
|
-
|
|
278
|
+
// Both are success terminal states used interchangeably; stale worker updates must
|
|
279
|
+
// not cause a completed task to appear as needs_attention or vice versa.
|
|
280
|
+
if (current.status === "completed" && updated.status === "needs_attention")
|
|
281
|
+
return false;
|
|
282
|
+
if (current.status === "needs_attention" && updated.status === "completed")
|
|
283
|
+
return false;
|
|
181
284
|
// Explicitly block failed→completed resurrection. Both statuses are terminal,
|
|
182
285
|
// but completed is the success terminal state and should not be reachable from
|
|
183
286
|
// failed via a stale merge. The check above only guards non-terminal→terminal.
|
|
184
|
-
if (current.status === "failed" && updated.status === "completed")
|
|
287
|
+
if (current.status === "failed" && updated.status === "completed")
|
|
288
|
+
return false;
|
|
185
289
|
// Guard: when current is "running" but has resultArtifact (another worker already
|
|
186
290
|
// completed it), a stale updated with status="running" and no resultArtifact
|
|
187
291
|
// must not overwrite the actual completed state.
|
|
188
|
-
if (
|
|
292
|
+
if (
|
|
293
|
+
current.status === updated.status &&
|
|
294
|
+
updated.status === "running" &&
|
|
295
|
+
current.resultArtifact &&
|
|
296
|
+
!updated.resultArtifact
|
|
297
|
+
)
|
|
298
|
+
return false;
|
|
189
299
|
// Guard: when current is "completed" and has resultArtifact but updated is also
|
|
190
300
|
// "completed" without resultArtifact, block the stale update from overwriting
|
|
191
301
|
// a task that successfully produced output.
|
|
192
|
-
if (
|
|
302
|
+
if (
|
|
303
|
+
current.status === updated.status &&
|
|
304
|
+
current.status === "completed" &&
|
|
305
|
+
current.resultArtifact &&
|
|
306
|
+
!updated.resultArtifact
|
|
307
|
+
)
|
|
308
|
+
return false;
|
|
193
309
|
// Prevent a stale completed task from overwriting a fresher one.
|
|
194
310
|
// Restructure to handle undefined current.finishedAt as a special case:
|
|
195
311
|
// - undefined current + valid updated: allow the update
|
|
@@ -202,7 +318,9 @@ if (current.status === "needs_attention" && updated.status === "completed") retu
|
|
|
202
318
|
// Malformed finishedAt (NaN) is treated as Infinity — invalid state should be
|
|
203
319
|
// replaced rather than persisting corruption. Log warning for visibility.
|
|
204
320
|
if (!Number.isFinite(currentTime)) {
|
|
205
|
-
console.warn(
|
|
321
|
+
console.warn(
|
|
322
|
+
`[team-runner] Task ${current.id} has malformed finishedAt: ${current.finishedAt}`,
|
|
323
|
+
);
|
|
206
324
|
}
|
|
207
325
|
if (isMalformedFinishedAtReplacement(currentTime, updatedTime)) {
|
|
208
326
|
return true;
|
|
@@ -212,29 +330,35 @@ if (current.status === "needs_attention" && updated.status === "completed") retu
|
|
|
212
330
|
// Block if updated is trying to establish a terminal status without a finishedAt
|
|
213
331
|
// timestamp. Heartbeat-only updates (status='running', no finishedAt) are
|
|
214
332
|
// allowed if heartbeat has changed (checked separately in hasMeaningfulUpdate).
|
|
215
|
-
if (!updated.finishedAt && !isNonTerminalTaskStatus(updated.status))
|
|
333
|
+
if (!updated.finishedAt && !isNonTerminalTaskStatus(updated.status))
|
|
334
|
+
return false;
|
|
216
335
|
// Explicitly enumerate all fields that constitute a meaningful update so that
|
|
217
|
-
// adding a new important field requires updating this list (rather than silently
|
|
218
|
-
// losing data if a field is forgotten in the boolean OR chain below).
|
|
219
|
-
const hasMeaningfulUpdate =
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
336
|
+
// adding a new important field requires updating this list (rather than silently
|
|
337
|
+
// losing data if a field is forgotten in the boolean OR chain below).
|
|
338
|
+
const hasMeaningfulUpdate =
|
|
339
|
+
updated.status !== current.status ||
|
|
340
|
+
updated.finishedAt !== current.finishedAt ||
|
|
341
|
+
updated.startedAt !== current.startedAt ||
|
|
342
|
+
Boolean(updated.resultArtifact) !== Boolean(current.resultArtifact) ||
|
|
343
|
+
(Boolean(updated.resultArtifact) &&
|
|
344
|
+
updated.resultArtifact !== current.resultArtifact) ||
|
|
345
|
+
Boolean(updated.error) ||
|
|
346
|
+
Boolean(updated.modelAttempts?.length) ||
|
|
347
|
+
Boolean(updated.usage) ||
|
|
348
|
+
Boolean(updated.attempts?.length) ||
|
|
349
|
+
updated.heartbeat?.lastSeenAt !== current.heartbeat?.lastSeenAt ||
|
|
350
|
+
updated.jsonEvents !== current.jsonEvents ||
|
|
351
|
+
updated.agentProgress?.lastActivityAt !==
|
|
352
|
+
current.agentProgress?.lastActivityAt;
|
|
353
|
+
return hasMeaningfulUpdate;
|
|
233
354
|
}
|
|
234
355
|
|
|
235
356
|
// H4 fix: rename to descriptive name. Kept __test__ as alias for backward
|
|
236
357
|
// compat test imports.
|
|
237
|
-
export function mergeTaskUpdatesPreservingTerminal(
|
|
358
|
+
export function mergeTaskUpdatesPreservingTerminal(
|
|
359
|
+
base: TeamTaskState[],
|
|
360
|
+
results: Array<{ tasks: TeamTaskState[] }>,
|
|
361
|
+
): TeamTaskState[] {
|
|
238
362
|
let merged = base;
|
|
239
363
|
for (const result of results) {
|
|
240
364
|
for (const updated of result.tasks) {
|
|
@@ -244,15 +368,21 @@ export function mergeTaskUpdatesPreservingTerminal(base: TeamTaskState[], result
|
|
|
244
368
|
// Log skipped merges for visibility into rejected parallel updates.
|
|
245
369
|
// In distributed systems with parallel workers, rejected merges may
|
|
246
370
|
// indicate bugs (wrong status, timestamp corruption) if they accumulate.
|
|
247
|
-
console.debug(
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
371
|
+
console.debug(
|
|
372
|
+
"[team-runner] Skipping stale merge for task",
|
|
373
|
+
updated.id,
|
|
374
|
+
{
|
|
375
|
+
currentStatus: current.status,
|
|
376
|
+
updatedStatus: updated.status,
|
|
377
|
+
currentFinishedAt: current.finishedAt,
|
|
378
|
+
updatedFinishedAt: updated.finishedAt,
|
|
379
|
+
},
|
|
380
|
+
);
|
|
253
381
|
continue;
|
|
254
382
|
}
|
|
255
|
-
merged = merged.map((task) =>
|
|
383
|
+
merged = merged.map((task) =>
|
|
384
|
+
task.id === updated.id ? updated : task,
|
|
385
|
+
);
|
|
256
386
|
}
|
|
257
387
|
}
|
|
258
388
|
return refreshTaskGraphQueues(merged);
|
|
@@ -262,20 +392,43 @@ export const __test__mergeTaskUpdates = mergeTaskUpdatesPreservingTerminal;
|
|
|
262
392
|
|
|
263
393
|
// 2.8: adaptive-plan parsing/repair/injection moved to src/runtime/adaptive-plan.ts.
|
|
264
394
|
// Re-export the test-only helpers so existing test imports still resolve.
|
|
265
|
-
export {
|
|
395
|
+
export {
|
|
396
|
+
__test__parseAdaptivePlan,
|
|
397
|
+
__test__repairAdaptivePlan,
|
|
398
|
+
} from "./adaptive-plan.ts";
|
|
399
|
+
|
|
266
400
|
import { injectAdaptivePlanIfReady } from "./adaptive-plan.ts";
|
|
267
401
|
|
|
268
402
|
function formatTaskProgress(task: TeamTaskState): string {
|
|
269
403
|
return `- ${task.id}: ${task.status} (${task.role} -> ${task.agent})${task.taskPacket ? ` scope=${task.taskPacket.scope}` : ""}${task.verification ? ` green=${task.verification.observedGreenLevel}/${task.verification.requiredGreenLevel}` : ""}${task.error ? ` - ${task.error}` : ""}`;
|
|
270
404
|
}
|
|
271
405
|
|
|
272
|
-
function runEffectivenessLines(
|
|
273
|
-
|
|
406
|
+
function runEffectivenessLines(
|
|
407
|
+
manifest: TeamRunManifest,
|
|
408
|
+
tasks: TeamTaskState[],
|
|
409
|
+
executeWorkers: boolean,
|
|
410
|
+
runtimeConfig?: CrewRuntimeConfig,
|
|
411
|
+
): string[] {
|
|
412
|
+
return formatRunEffectivenessLines(
|
|
413
|
+
evaluateRunEffectiveness({
|
|
414
|
+
manifest,
|
|
415
|
+
tasks,
|
|
416
|
+
executeWorkers,
|
|
417
|
+
runtimeConfig,
|
|
418
|
+
}),
|
|
419
|
+
);
|
|
274
420
|
}
|
|
275
421
|
|
|
276
|
-
function writeProgress(
|
|
422
|
+
function writeProgress(
|
|
423
|
+
manifest: TeamRunManifest,
|
|
424
|
+
tasks: TeamTaskState[],
|
|
425
|
+
producer: string,
|
|
426
|
+
executeWorkers = true,
|
|
427
|
+
runtimeConfig?: CrewRuntimeConfig,
|
|
428
|
+
): TeamRunManifest {
|
|
277
429
|
const counts = new Map<string, number>();
|
|
278
|
-
for (const task of tasks)
|
|
430
|
+
for (const task of tasks)
|
|
431
|
+
counts.set(task.status, (counts.get(task.status) ?? 0) + 1);
|
|
279
432
|
const queue = taskGraphSnapshot(tasks);
|
|
280
433
|
const progress = writeArtifact(manifest.artifactsRoot, {
|
|
281
434
|
kind: "progress",
|
|
@@ -295,14 +448,39 @@ function writeProgress(manifest: TeamRunManifest, tasks: TeamTaskState[], produc
|
|
|
295
448
|
...tasks.map(formatTaskProgress),
|
|
296
449
|
"",
|
|
297
450
|
"## Effectiveness",
|
|
298
|
-
...runEffectivenessLines(
|
|
451
|
+
...runEffectivenessLines(
|
|
452
|
+
manifest,
|
|
453
|
+
tasks,
|
|
454
|
+
executeWorkers,
|
|
455
|
+
runtimeConfig,
|
|
456
|
+
),
|
|
299
457
|
"",
|
|
300
458
|
].join("\n"),
|
|
301
459
|
});
|
|
302
|
-
return {
|
|
460
|
+
return {
|
|
461
|
+
...manifest,
|
|
462
|
+
updatedAt: new Date().toISOString(),
|
|
463
|
+
artifacts: [
|
|
464
|
+
...manifest.artifacts.filter(
|
|
465
|
+
(artifact) =>
|
|
466
|
+
!(
|
|
467
|
+
artifact.kind === "progress" &&
|
|
468
|
+
artifact.path === progress.path
|
|
469
|
+
),
|
|
470
|
+
),
|
|
471
|
+
progress,
|
|
472
|
+
].filter(
|
|
473
|
+
(artifact, index, self) =>
|
|
474
|
+
self.findIndex((a) => a.path === artifact.path) === index,
|
|
475
|
+
),
|
|
476
|
+
};
|
|
303
477
|
}
|
|
304
478
|
|
|
305
|
-
function applyPolicy(
|
|
479
|
+
function applyPolicy(
|
|
480
|
+
manifest: TeamRunManifest,
|
|
481
|
+
tasks: TeamTaskState[],
|
|
482
|
+
limits?: CrewLimitsConfig,
|
|
483
|
+
): TeamRunManifest {
|
|
306
484
|
const branchFreshness = checkBranchFreshness(manifest.cwd);
|
|
307
485
|
const branchArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
308
486
|
kind: "metadata",
|
|
@@ -310,8 +488,15 @@ function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?:
|
|
|
310
488
|
producer: "branch-freshness",
|
|
311
489
|
content: `${JSON.stringify(branchFreshness, null, 2)}\n`,
|
|
312
490
|
});
|
|
313
|
-
let decisions: PolicyDecision[] = evaluateCrewPolicy({
|
|
314
|
-
|
|
491
|
+
let decisions: PolicyDecision[] = evaluateCrewPolicy({
|
|
492
|
+
manifest,
|
|
493
|
+
tasks,
|
|
494
|
+
limits,
|
|
495
|
+
});
|
|
496
|
+
if (
|
|
497
|
+
branchFreshness.status === "stale" ||
|
|
498
|
+
branchFreshness.status === "diverged"
|
|
499
|
+
) {
|
|
315
500
|
const branchDecision: PolicyDecision = {
|
|
316
501
|
action: "notify",
|
|
317
502
|
reason: "branch_stale",
|
|
@@ -319,7 +504,12 @@ function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?:
|
|
|
319
504
|
createdAt: new Date().toISOString(),
|
|
320
505
|
};
|
|
321
506
|
decisions = [...decisions, branchDecision];
|
|
322
|
-
appendEvent(manifest.eventsPath, {
|
|
507
|
+
appendEvent(manifest.eventsPath, {
|
|
508
|
+
type: "branch.stale",
|
|
509
|
+
runId: manifest.runId,
|
|
510
|
+
message: branchFreshness.message,
|
|
511
|
+
data: { branchFreshness },
|
|
512
|
+
});
|
|
323
513
|
}
|
|
324
514
|
const policyArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
325
515
|
kind: "metadata",
|
|
@@ -334,20 +524,85 @@ function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?:
|
|
|
334
524
|
producer: "recovery-engine",
|
|
335
525
|
content: `${JSON.stringify(recoveryLedger, null, 2)}\n`,
|
|
336
526
|
});
|
|
337
|
-
for (const item of decisions)
|
|
338
|
-
|
|
339
|
-
|
|
527
|
+
for (const item of decisions)
|
|
528
|
+
appendEvent(manifest.eventsPath, {
|
|
529
|
+
type:
|
|
530
|
+
item.action === "escalate"
|
|
531
|
+
? "policy.escalated"
|
|
532
|
+
: "policy.action",
|
|
533
|
+
runId: manifest.runId,
|
|
534
|
+
taskId: item.taskId,
|
|
535
|
+
message: item.message,
|
|
536
|
+
data: { action: item.action, reason: item.reason },
|
|
537
|
+
});
|
|
538
|
+
for (const item of recoveryLedger.entries)
|
|
539
|
+
appendEvent(manifest.eventsPath, {
|
|
540
|
+
type:
|
|
541
|
+
item.state === "escalation_required"
|
|
542
|
+
? "recovery.escalated"
|
|
543
|
+
: "recovery.attempted",
|
|
544
|
+
runId: manifest.runId,
|
|
545
|
+
taskId: item.taskId,
|
|
546
|
+
message: item.message,
|
|
547
|
+
data: {
|
|
548
|
+
scenario: item.scenario,
|
|
549
|
+
steps: item.steps,
|
|
550
|
+
attempt: item.attempt,
|
|
551
|
+
state: item.state,
|
|
552
|
+
},
|
|
553
|
+
});
|
|
554
|
+
return {
|
|
555
|
+
...manifest,
|
|
556
|
+
updatedAt: new Date().toISOString(),
|
|
557
|
+
policyDecisions: decisions,
|
|
558
|
+
artifacts: [
|
|
559
|
+
...manifest.artifacts.filter(
|
|
560
|
+
(artifact) =>
|
|
561
|
+
!(
|
|
562
|
+
artifact.kind === "metadata" &&
|
|
563
|
+
(artifact.path.endsWith("policy-decisions.json") ||
|
|
564
|
+
artifact.path.endsWith("recovery-ledger.json") ||
|
|
565
|
+
artifact.path.endsWith("branch-freshness.json"))
|
|
566
|
+
),
|
|
567
|
+
),
|
|
568
|
+
branchArtifact,
|
|
569
|
+
policyArtifact,
|
|
570
|
+
recoveryArtifact,
|
|
571
|
+
],
|
|
572
|
+
};
|
|
340
573
|
}
|
|
341
574
|
|
|
342
|
-
function retryPolicyFromConfig(
|
|
575
|
+
function retryPolicyFromConfig(
|
|
576
|
+
config: CrewReliabilityConfig | undefined,
|
|
577
|
+
): RetryPolicy {
|
|
343
578
|
return { ...DEFAULT_RETRY_POLICY, ...(config?.retryPolicy ?? {}) };
|
|
344
579
|
}
|
|
345
580
|
|
|
346
|
-
|
|
347
|
-
|
|
581
|
+
/**
|
|
582
|
+
* #1 (assessment): decide whether the per-task retry path (executeWithRetry) is used.
|
|
583
|
+
* Defaults to TRUE (opt-out) so transient worker hangs (ChildTimeout) are retried
|
|
584
|
+
* automatically. Previously opt-in, which left the entire retry+recovery stack dormant.
|
|
585
|
+
* Exported for unit testing.
|
|
586
|
+
*/
|
|
587
|
+
export function shouldUseRetry(
|
|
588
|
+
reliability: CrewReliabilityConfig | undefined,
|
|
589
|
+
): boolean {
|
|
590
|
+
return reliability?.autoRetry !== false;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
function failedTaskFrom(
|
|
594
|
+
result: { tasks: TeamTaskState[] },
|
|
595
|
+
taskId: string,
|
|
596
|
+
): TeamTaskState | undefined {
|
|
597
|
+
return result.tasks.find(
|
|
598
|
+
(item) => item.id === taskId && item.status === "failed",
|
|
599
|
+
);
|
|
348
600
|
}
|
|
349
601
|
|
|
350
|
-
function requiresPlanApproval(
|
|
602
|
+
function requiresPlanApproval(
|
|
603
|
+
_workflow: WorkflowConfig,
|
|
604
|
+
runtimeConfig: CrewRuntimeConfig | undefined,
|
|
605
|
+
): boolean {
|
|
351
606
|
// ROADMAP T1.2: plan-level HITL applies to ANY workflow when
|
|
352
607
|
// config.runtime.requirePlanApproval === true (not just 'implementation').
|
|
353
608
|
// The gate fires at the read-only → mutating (plan → execute) boundary.
|
|
@@ -355,19 +610,31 @@ function requiresPlanApproval(_workflow: WorkflowConfig, runtimeConfig: CrewRunt
|
|
|
355
610
|
}
|
|
356
611
|
|
|
357
612
|
function isPlanApprovalPending(manifest: TeamRunManifest): boolean {
|
|
358
|
-
return
|
|
613
|
+
return (
|
|
614
|
+
manifest.planApproval?.required === true &&
|
|
615
|
+
manifest.planApproval.status === "pending"
|
|
616
|
+
);
|
|
359
617
|
}
|
|
360
618
|
|
|
361
619
|
function isMutatingTask(task: TeamTaskState): boolean {
|
|
362
620
|
return permissionForRole(task.role) !== "read_only";
|
|
363
621
|
}
|
|
364
622
|
|
|
365
|
-
function ensurePlanApprovalRequested(
|
|
623
|
+
function ensurePlanApprovalRequested(
|
|
624
|
+
manifest: TeamRunManifest,
|
|
625
|
+
tasks: TeamTaskState[],
|
|
626
|
+
): TeamRunManifest {
|
|
366
627
|
if (manifest.planApproval) return manifest;
|
|
367
|
-
const assessTask = tasks.find(
|
|
628
|
+
const assessTask = tasks.find(
|
|
629
|
+
(task) => task.stepId === "assess" && task.status === "completed",
|
|
630
|
+
);
|
|
368
631
|
// ROADMAP T1.2: for non-adaptive workflows, fall back to the most recent
|
|
369
632
|
// completed read-only (planning) task as the plan reference.
|
|
370
|
-
const planTask =
|
|
633
|
+
const planTask =
|
|
634
|
+
assessTask ??
|
|
635
|
+
[...tasks]
|
|
636
|
+
.reverse()
|
|
637
|
+
.find((t) => t.status === "completed" && !isMutatingTask(t));
|
|
371
638
|
const now = new Date().toISOString();
|
|
372
639
|
const updated: TeamRunManifest = {
|
|
373
640
|
...manifest,
|
|
@@ -382,16 +649,43 @@ function ensurePlanApprovalRequested(manifest: TeamRunManifest, tasks: TeamTaskS
|
|
|
382
649
|
},
|
|
383
650
|
};
|
|
384
651
|
saveRunManifest(updated);
|
|
385
|
-
appendEvent(updated.eventsPath, {
|
|
652
|
+
appendEvent(updated.eventsPath, {
|
|
653
|
+
type: "plan.approval_required",
|
|
654
|
+
runId: updated.runId,
|
|
655
|
+
taskId: planTask?.id,
|
|
656
|
+
message:
|
|
657
|
+
"Plan requires explicit approval before mutating tasks run. Use: team api op=approve-plan runId=...",
|
|
658
|
+
data: { planArtifactPath: planTask?.resultArtifact?.path },
|
|
659
|
+
});
|
|
386
660
|
return updated;
|
|
387
661
|
}
|
|
388
662
|
|
|
389
|
-
function cancelPlanTasks(
|
|
390
|
-
|
|
663
|
+
function cancelPlanTasks(
|
|
664
|
+
tasks: TeamTaskState[],
|
|
665
|
+
reason: string,
|
|
666
|
+
): TeamTaskState[] {
|
|
667
|
+
return tasks.map((task) =>
|
|
668
|
+
task.status === "queued" ||
|
|
669
|
+
task.status === "running" ||
|
|
670
|
+
task.status === "waiting"
|
|
671
|
+
? {
|
|
672
|
+
...task,
|
|
673
|
+
status: "cancelled",
|
|
674
|
+
finishedAt: new Date().toISOString(),
|
|
675
|
+
error: reason,
|
|
676
|
+
graph: task.graph
|
|
677
|
+
? { ...task.graph, queue: "done" }
|
|
678
|
+
: undefined,
|
|
679
|
+
}
|
|
680
|
+
: task,
|
|
681
|
+
);
|
|
391
682
|
}
|
|
392
683
|
|
|
393
684
|
function hasPendingMutatingAdaptiveTask(tasks: TeamTaskState[]): boolean {
|
|
394
|
-
return tasks.some(
|
|
685
|
+
return tasks.some(
|
|
686
|
+
(task) =>
|
|
687
|
+
task.status === "queued" && task.adaptive && isMutatingTask(task),
|
|
688
|
+
);
|
|
395
689
|
}
|
|
396
690
|
|
|
397
691
|
/**
|
|
@@ -399,9 +693,15 @@ function hasPendingMutatingAdaptiveTask(tasks: TeamTaskState[]): boolean {
|
|
|
399
693
|
* Fires when there are pending mutating tasks whose prerequisites (read-only
|
|
400
694
|
* tasks) have completed — i.e. the plan→execute boundary.
|
|
401
695
|
*/
|
|
402
|
-
export function hasPendingMutatingTaskAtBoundary(
|
|
403
|
-
|
|
404
|
-
|
|
696
|
+
export function hasPendingMutatingTaskAtBoundary(
|
|
697
|
+
tasks: TeamTaskState[],
|
|
698
|
+
): boolean {
|
|
699
|
+
const hasCompletedReadOnly = tasks.some(
|
|
700
|
+
(t) => t.status === "completed" && !isMutatingTask(t),
|
|
701
|
+
);
|
|
702
|
+
const hasPendingMutating = tasks.some(
|
|
703
|
+
(t) => t.status === "queued" && isMutatingTask(t),
|
|
704
|
+
);
|
|
405
705
|
return hasCompletedReadOnly && hasPendingMutating;
|
|
406
706
|
}
|
|
407
707
|
|
|
@@ -410,7 +710,10 @@ export function hasPendingMutatingTaskAtBoundary(tasks: TeamTaskState[]): boolea
|
|
|
410
710
|
* execution planning. If so, build an execution plan and use `getDagReadyTasks`
|
|
411
711
|
* to augment the ready-set selection.
|
|
412
712
|
*/
|
|
413
|
-
function dagReadyTaskIds(
|
|
713
|
+
function dagReadyTaskIds(
|
|
714
|
+
tasks: TeamTaskState[],
|
|
715
|
+
completedIds: Set<string>,
|
|
716
|
+
): string[] | null {
|
|
414
717
|
const hasExplicitDeps = tasks.some((t) => t.dependsOn.length > 0);
|
|
415
718
|
if (!hasExplicitDeps) return null;
|
|
416
719
|
// FIX (goal-wrap runtime test): task.dependsOn stores STEP IDs (e.g. "execute"), not
|
|
@@ -433,9 +736,53 @@ function dagReadyTaskIds(tasks: TeamTaskState[], completedIds: Set<string>): str
|
|
|
433
736
|
return getDagReadyTasks(plan, completedIds);
|
|
434
737
|
}
|
|
435
738
|
|
|
436
|
-
export async function executeTeamRun(
|
|
739
|
+
export async function executeTeamRun(
|
|
740
|
+
input: ExecuteTeamRunInput,
|
|
741
|
+
): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
|
|
437
742
|
const workflow = input.workflow;
|
|
438
|
-
|
|
743
|
+
|
|
744
|
+
// DEFENSE-IN-DEPTH (advisory-only since v0.9.15): re-validate topology here in
|
|
745
|
+
// case a caller bypassed the extension-layer handleRun guard. The extension
|
|
746
|
+
// layer logs an advisory note and proceeds; this defense-in-depth does the same
|
|
747
|
+
// for direct API callers (CLI, tests, scheduler). Never blocks.
|
|
748
|
+
// Skip for synthetic direct-agent workflows (filePath="<generated>").
|
|
749
|
+
if (workflow.filePath !== "<generated>") {
|
|
750
|
+
// LAZY: defer preflight-validator import until the defense-in-depth guard actually runs.
|
|
751
|
+
const { validateWorkflowUsage } = await import(
|
|
752
|
+
"../workflows/preflight-validator.ts"
|
|
753
|
+
);
|
|
754
|
+
const preflight = validateWorkflowUsage(workflow, {
|
|
755
|
+
force: input.reliability?.forcePreflight === true,
|
|
756
|
+
});
|
|
757
|
+
if (
|
|
758
|
+
preflight.level === "warn" ||
|
|
759
|
+
preflight.level === "note" ||
|
|
760
|
+
preflight.level === "info"
|
|
761
|
+
) {
|
|
762
|
+
const icon =
|
|
763
|
+
preflight.level === "warn"
|
|
764
|
+
? "⚠️ "
|
|
765
|
+
: preflight.level === "note"
|
|
766
|
+
? "✅ "
|
|
767
|
+
: "ℹ️ ";
|
|
768
|
+
console.warn(
|
|
769
|
+
`${icon}[team-runner.preflight] ${preflight.level.toUpperCase()}: ${preflight.message} (workflow=${workflow.name})`,
|
|
770
|
+
);
|
|
771
|
+
if (preflight.suggestion) {
|
|
772
|
+
console.warn(
|
|
773
|
+
`[team-runner.preflight] → ${preflight.suggestion}`,
|
|
774
|
+
);
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
let manifest = updateRunStatus(
|
|
780
|
+
input.manifest,
|
|
781
|
+
"running",
|
|
782
|
+
input.executeWorkers
|
|
783
|
+
? "Executing team workflow."
|
|
784
|
+
: "Creating workflow prompts and placeholder results.",
|
|
785
|
+
);
|
|
439
786
|
|
|
440
787
|
void registerRunPromise(manifest.runId);
|
|
441
788
|
|
|
@@ -444,7 +791,10 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
444
791
|
// (NO_PID_HEARTBEAT_STALE_MS). Previously only sub-task runners wrote
|
|
445
792
|
// heartbeats; the team-level run had no heartbeat, so any multi-phase
|
|
446
793
|
// workflow lasting >5min was marked stale and cancelled.
|
|
447
|
-
const stopTeamHeartbeat = startTeamRunHeartbeat(
|
|
794
|
+
const stopTeamHeartbeat = startTeamRunHeartbeat(
|
|
795
|
+
manifest.stateRoot,
|
|
796
|
+
manifest.runId,
|
|
797
|
+
);
|
|
448
798
|
|
|
449
799
|
const cleanupUsage = (): void => {
|
|
450
800
|
for (const task of input.tasks) clearTrackedTaskUsage(task.id);
|
|
@@ -452,32 +802,106 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
452
802
|
|
|
453
803
|
try {
|
|
454
804
|
const result = await executeTeamRunCore(input, manifest, workflow);
|
|
805
|
+
// #2 (assessment): goal-achievement detection — kill the silent false-green.
|
|
806
|
+
// A code-mutating run that "completed" but left the git working tree clean
|
|
807
|
+
// (and/or had a failed task) is a false-green. We expose goalAchieved on the
|
|
808
|
+
// manifest + emit an event so the lie is never silent, and downgrade status
|
|
809
|
+
// to "failed" only when a failed task corroborates it (conservative).
|
|
810
|
+
const gaAssessment = assessGoalAchievement(
|
|
811
|
+
result.manifest,
|
|
812
|
+
result.tasks,
|
|
813
|
+
workflow,
|
|
814
|
+
);
|
|
815
|
+
const gaApplied = applyGoalAchievement(result.manifest, gaAssessment);
|
|
816
|
+
if (gaApplied.manifest !== result.manifest) {
|
|
817
|
+
result.manifest = gaApplied.manifest;
|
|
818
|
+
try {
|
|
819
|
+
saveRunManifest(result.manifest);
|
|
820
|
+
} catch (persistError) {
|
|
821
|
+
logInternalError(
|
|
822
|
+
"team-runner.goalAchievement.persist",
|
|
823
|
+
persistError instanceof Error
|
|
824
|
+
? persistError
|
|
825
|
+
: new Error(String(persistError)),
|
|
826
|
+
`runId=${manifest.runId}`,
|
|
827
|
+
);
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
appendEvent(manifest.eventsPath, {
|
|
831
|
+
type: "run.goal_achievement",
|
|
832
|
+
runId: manifest.runId,
|
|
833
|
+
message: gaApplied.manifest.goalAchievementNote ?? "",
|
|
834
|
+
data: {
|
|
835
|
+
achieved: gaAssessment.achieved,
|
|
836
|
+
downgraded: gaApplied.downgraded,
|
|
837
|
+
reason: gaAssessment.reason,
|
|
838
|
+
signals: gaAssessment.signals,
|
|
839
|
+
},
|
|
840
|
+
});
|
|
841
|
+
if (gaApplied.downgraded)
|
|
842
|
+
logInternalError(
|
|
843
|
+
"team-runner.goalAchievement.falseGreen",
|
|
844
|
+
new Error(
|
|
845
|
+
gaApplied.manifest.goalAchievementNote ??
|
|
846
|
+
"false-green detected",
|
|
847
|
+
),
|
|
848
|
+
`runId=${manifest.runId}`,
|
|
849
|
+
);
|
|
455
850
|
stopTeamHeartbeat();
|
|
456
851
|
resolveRunPromise(manifest.runId, result);
|
|
457
852
|
cleanupUsage();
|
|
458
853
|
// Terminate live agents for this run — agents are done when the run ends.
|
|
459
|
-
void terminateLiveAgentsForRun(
|
|
854
|
+
void terminateLiveAgentsForRun(
|
|
855
|
+
manifest.runId,
|
|
856
|
+
"completed",
|
|
857
|
+
appendEvent,
|
|
858
|
+
manifest.eventsPath,
|
|
859
|
+
).catch((error) =>
|
|
860
|
+
logInternalError(
|
|
861
|
+
"team-runner.completed.terminate",
|
|
862
|
+
error,
|
|
863
|
+
`runId=${manifest.runId}`,
|
|
864
|
+
),
|
|
865
|
+
);
|
|
460
866
|
|
|
461
867
|
// Emit run completion hook (100% reliable, fire-and-forget)
|
|
462
|
-
crewHooks.emit({
|
|
868
|
+
crewHooks.emit({
|
|
869
|
+
type: "run_completed",
|
|
870
|
+
timestamp: new Date().toISOString(),
|
|
871
|
+
runId: manifest.runId,
|
|
872
|
+
data: {
|
|
873
|
+
status: result.manifest.status,
|
|
874
|
+
taskCount: result.tasks.length,
|
|
875
|
+
},
|
|
876
|
+
});
|
|
463
877
|
|
|
464
878
|
// Execute after_run_complete lifecycle hook (non-blocking)
|
|
465
|
-
const afterRunReport = await executeHook("after_run_complete", {
|
|
879
|
+
const afterRunReport = await executeHook("after_run_complete", {
|
|
880
|
+
runId: manifest.runId,
|
|
881
|
+
cwd: manifest.cwd,
|
|
882
|
+
status: result.manifest.status,
|
|
883
|
+
});
|
|
466
884
|
appendHookEvent(manifest, afterRunReport);
|
|
467
885
|
if (afterRunReport.outcome === "block") {
|
|
468
|
-
logInternalError(
|
|
886
|
+
logInternalError(
|
|
887
|
+
"team-runner.after_run_complete.blocked",
|
|
888
|
+
new Error(
|
|
889
|
+
afterRunReport.reason ?? "after_run_complete hook blocked",
|
|
890
|
+
),
|
|
891
|
+
`runId=${manifest.runId}`,
|
|
892
|
+
);
|
|
469
893
|
}
|
|
470
894
|
|
|
471
895
|
return result;
|
|
472
896
|
} catch (error) {
|
|
473
897
|
// Round 27 (BUG 1): the success path calls stopTeamHeartbeat() but this
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
898
|
+
// catch path did NOT. The team heartbeat is a non-unref'd setInterval
|
|
899
|
+
// (30s) that deliberately keeps the event loop alive — without this
|
|
900
|
+
// call, a failed team run leaves the interval firing forever and the
|
|
901
|
+
// foreground pi process hangs (never returns to the prompt); in
|
|
902
|
+
// background-runner mode the worker never exits. clearInterval is
|
|
903
|
+
// idempotent so a double-call (if this runs after the success path)
|
|
904
|
+
// is harmless.
|
|
481
905
|
stopTeamHeartbeat();
|
|
482
906
|
// P1: Catch unhandled errors — ensure manifest/tasks/agents are terminal so they don't stay "running" forever.
|
|
483
907
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -485,7 +909,9 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
485
909
|
// If lock acquisition fails, use in-memory data rather than stale disk data.
|
|
486
910
|
let loaded;
|
|
487
911
|
try {
|
|
488
|
-
loaded = await withRunLock(input.manifest, async () =>
|
|
912
|
+
loaded = await withRunLock(input.manifest, async () =>
|
|
913
|
+
loadRunManifestById(input.manifest.cwd, input.manifest.runId),
|
|
914
|
+
);
|
|
489
915
|
} catch {
|
|
490
916
|
loaded = undefined; // best-effort: use in-memory data if lock fails
|
|
491
917
|
}
|
|
@@ -493,29 +919,72 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
493
919
|
const freshTasks = refreshTaskGraphQueues(loaded?.tasks ?? input.tasks);
|
|
494
920
|
const failedAt = new Date().toISOString();
|
|
495
921
|
const tasks = freshTasks.map((task) =>
|
|
496
|
-
task.status === "running" ||
|
|
497
|
-
|
|
922
|
+
task.status === "running" ||
|
|
923
|
+
task.status === "queued" ||
|
|
924
|
+
task.status === "waiting"
|
|
925
|
+
? {
|
|
926
|
+
...task,
|
|
927
|
+
status: "failed" as const,
|
|
928
|
+
finishedAt: failedAt,
|
|
929
|
+
error: message,
|
|
930
|
+
}
|
|
498
931
|
: task,
|
|
499
932
|
);
|
|
500
933
|
manifest = freshManifest;
|
|
501
934
|
try {
|
|
502
|
-
await terminateLiveAgentsForRun(
|
|
935
|
+
await terminateLiveAgentsForRun(
|
|
936
|
+
manifest.runId,
|
|
937
|
+
"failed",
|
|
938
|
+
appendEvent,
|
|
939
|
+
manifest.eventsPath,
|
|
940
|
+
);
|
|
503
941
|
await saveRunTasksAsync(manifest, tasks);
|
|
504
|
-
const existingRuntimeByTask = new Map(
|
|
942
|
+
const existingRuntimeByTask = new Map(
|
|
943
|
+
readCrewAgents(manifest).map((agent) => [
|
|
944
|
+
agent.taskId,
|
|
945
|
+
agent.runtime,
|
|
946
|
+
]),
|
|
947
|
+
);
|
|
505
948
|
const globalRuntime = input.runtime?.kind ?? "child-process";
|
|
506
|
-
const runtimeForAgent = (
|
|
949
|
+
const runtimeForAgent = (
|
|
950
|
+
agent: ReturnType<typeof recordsForMaterializedTasks>[number],
|
|
951
|
+
): CrewRuntimeKind => {
|
|
507
952
|
const task = tasks.find((item) => item.id === agent.taskId);
|
|
508
|
-
return
|
|
953
|
+
return (
|
|
954
|
+
existingRuntimeByTask.get(agent.taskId) ??
|
|
955
|
+
resolveTaskRuntimeKind(
|
|
956
|
+
globalRuntime,
|
|
957
|
+
task?.role ?? agent.role,
|
|
958
|
+
input.runtimeConfig?.isolationPolicy,
|
|
959
|
+
)
|
|
960
|
+
);
|
|
509
961
|
};
|
|
510
|
-
saveCrewAgents(
|
|
511
|
-
|
|
962
|
+
saveCrewAgents(
|
|
963
|
+
manifest,
|
|
964
|
+
recordsForMaterializedTasks(manifest, tasks, globalRuntime).map(
|
|
965
|
+
(agent) => ({ ...agent, runtime: runtimeForAgent(agent) }),
|
|
966
|
+
),
|
|
967
|
+
);
|
|
968
|
+
manifest = updateRunStatus(
|
|
969
|
+
manifest,
|
|
970
|
+
"failed",
|
|
971
|
+
`Unhandled error in team runner: ${message}`,
|
|
972
|
+
);
|
|
512
973
|
await saveRunManifestAsync(manifest);
|
|
513
974
|
} catch {
|
|
514
975
|
// Best-effort — state write may also fail
|
|
515
976
|
}
|
|
516
977
|
const result = { manifest, tasks };
|
|
517
|
-
rejectRunPromise(
|
|
518
|
-
|
|
978
|
+
rejectRunPromise(
|
|
979
|
+
manifest.runId,
|
|
980
|
+
error instanceof Error ? error : new Error(message),
|
|
981
|
+
);
|
|
982
|
+
crewHooks.emit({
|
|
983
|
+
type: "run_failed",
|
|
984
|
+
timestamp: new Date().toISOString(),
|
|
985
|
+
runId: manifest.runId,
|
|
986
|
+
data: { status: manifest.status, error: message },
|
|
987
|
+
});
|
|
519
988
|
cleanupUsage();
|
|
520
989
|
return result;
|
|
521
990
|
}
|
|
@@ -527,10 +996,17 @@ async function executeTeamRunCore(
|
|
|
527
996
|
workflow: WorkflowConfig,
|
|
528
997
|
): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
|
|
529
998
|
// Execute before_run_start hook (non-blocking by default)
|
|
530
|
-
const beforeRunReport = await executeHook("before_run_start", {
|
|
999
|
+
const beforeRunReport = await executeHook("before_run_start", {
|
|
1000
|
+
runId: manifest.runId,
|
|
1001
|
+
cwd: manifest.cwd,
|
|
1002
|
+
});
|
|
531
1003
|
appendHookEvent(manifest, beforeRunReport);
|
|
532
1004
|
if (beforeRunReport.outcome === "block") {
|
|
533
|
-
manifest = updateRunStatus(
|
|
1005
|
+
manifest = updateRunStatus(
|
|
1006
|
+
manifest,
|
|
1007
|
+
"blocked",
|
|
1008
|
+
beforeRunReport.reason ?? "before_run_start hook blocked the run.",
|
|
1009
|
+
);
|
|
534
1010
|
return { manifest, tasks: input.tasks };
|
|
535
1011
|
}
|
|
536
1012
|
let tasks = refreshTaskGraphQueues(input.tasks);
|
|
@@ -539,45 +1015,94 @@ async function executeTeamRunCore(
|
|
|
539
1015
|
let adaptivePlanInjected = false;
|
|
540
1016
|
let adaptivePlanMissing = false;
|
|
541
1017
|
const attemptAdaptivePlan = () => {
|
|
542
|
-
if (
|
|
543
|
-
|
|
1018
|
+
if (
|
|
1019
|
+
!canInjectAdaptivePlan ||
|
|
1020
|
+
adaptivePlanInjected ||
|
|
1021
|
+
adaptivePlanMissing
|
|
1022
|
+
)
|
|
1023
|
+
return { injected: false, missing: false };
|
|
1024
|
+
const adaptivePlan = injectAdaptivePlanIfReady({
|
|
1025
|
+
manifest,
|
|
1026
|
+
tasks,
|
|
1027
|
+
workflow,
|
|
1028
|
+
team: input.team,
|
|
1029
|
+
});
|
|
544
1030
|
adaptivePlanInjected = adaptivePlanInjected || adaptivePlan.injected;
|
|
545
1031
|
adaptivePlanMissing = adaptivePlan.missingPlan;
|
|
546
1032
|
workflow = adaptivePlan.workflow;
|
|
547
1033
|
if (adaptivePlan.injected) tasks = adaptivePlan.tasks;
|
|
548
|
-
return {
|
|
1034
|
+
return {
|
|
1035
|
+
injected: adaptivePlan.injected,
|
|
1036
|
+
missing: adaptivePlan.missingPlan,
|
|
1037
|
+
};
|
|
549
1038
|
};
|
|
550
1039
|
const initialAdaptive = attemptAdaptivePlan();
|
|
551
1040
|
if (initialAdaptive.missing) {
|
|
552
|
-
tasks = markBlocked(
|
|
1041
|
+
tasks = markBlocked(
|
|
1042
|
+
tasks,
|
|
1043
|
+
"Adaptive planner did not produce a valid subagent plan.",
|
|
1044
|
+
);
|
|
553
1045
|
await saveRunTasksAsync(manifest, tasks);
|
|
554
|
-
manifest = updateRunStatus(
|
|
1046
|
+
manifest = updateRunStatus(
|
|
1047
|
+
manifest,
|
|
1048
|
+
"blocked",
|
|
1049
|
+
"Adaptive planner did not produce a valid subagent plan.",
|
|
1050
|
+
);
|
|
555
1051
|
return { manifest, tasks };
|
|
556
1052
|
}
|
|
557
1053
|
if (initialAdaptive.injected) {
|
|
558
|
-
manifest = requiresPlanApproval(workflow, input.runtimeConfig)
|
|
1054
|
+
manifest = requiresPlanApproval(workflow, input.runtimeConfig)
|
|
1055
|
+
? ensurePlanApprovalRequested(manifest, tasks)
|
|
1056
|
+
: manifest;
|
|
559
1057
|
queueIndex = buildTaskGraphIndex(tasks);
|
|
560
|
-
} else if (
|
|
1058
|
+
} else if (
|
|
1059
|
+
requiresPlanApproval(workflow, input.runtimeConfig) &&
|
|
1060
|
+
(hasPendingMutatingAdaptiveTask(tasks) ||
|
|
1061
|
+
hasPendingMutatingTaskAtBoundary(tasks))
|
|
1062
|
+
) {
|
|
561
1063
|
manifest = ensurePlanApprovalRequested(manifest, tasks);
|
|
562
1064
|
}
|
|
563
1065
|
if (manifest.planApproval?.status === "cancelled") {
|
|
564
1066
|
tasks = cancelPlanTasks(tasks, "Plan approval was cancelled.");
|
|
565
1067
|
await saveRunTasksAsync(manifest, tasks);
|
|
566
|
-
manifest = updateRunStatus(
|
|
1068
|
+
manifest = updateRunStatus(
|
|
1069
|
+
manifest,
|
|
1070
|
+
"cancelled",
|
|
1071
|
+
"Plan approval was cancelled.",
|
|
1072
|
+
);
|
|
567
1073
|
return { manifest, tasks };
|
|
568
1074
|
}
|
|
569
|
-
manifest = writeProgress(
|
|
1075
|
+
manifest = writeProgress(
|
|
1076
|
+
manifest,
|
|
1077
|
+
tasks,
|
|
1078
|
+
"team-runner",
|
|
1079
|
+
input.executeWorkers,
|
|
1080
|
+
input.runtimeConfig,
|
|
1081
|
+
);
|
|
570
1082
|
await saveRunManifestAsync(manifest);
|
|
571
|
-
const runtimeKind =
|
|
572
|
-
|
|
1083
|
+
const runtimeKind =
|
|
1084
|
+
input.runtime?.kind ??
|
|
1085
|
+
(input.executeWorkers ? "child-process" : "scaffold");
|
|
1086
|
+
saveCrewAgents(
|
|
1087
|
+
manifest,
|
|
1088
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1089
|
+
);
|
|
573
1090
|
|
|
574
1091
|
// Build a workflow phase state machine from workflow steps for precondition tracking.
|
|
575
|
-
const workflowPhases: PhaseState[] = workflow.steps.map(
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
1092
|
+
const workflowPhases: PhaseState[] = workflow.steps.map(
|
|
1093
|
+
(step): PhaseState => ({
|
|
1094
|
+
name: step.id,
|
|
1095
|
+
status: "pending",
|
|
1096
|
+
inputs:
|
|
1097
|
+
step.reads === false
|
|
1098
|
+
? []
|
|
1099
|
+
: Array.isArray(step.reads)
|
|
1100
|
+
? step.reads
|
|
1101
|
+
: [],
|
|
1102
|
+
outputs:
|
|
1103
|
+
step.output === false ? [] : step.output ? [step.output] : [],
|
|
1104
|
+
}),
|
|
1105
|
+
);
|
|
581
1106
|
let wfMachine = createWorkflowStateMachine(workflowPhases);
|
|
582
1107
|
|
|
583
1108
|
while (tasks.some((task) => task.status === "queued")) {
|
|
@@ -586,26 +1111,100 @@ async function executeTeamRunCore(
|
|
|
586
1111
|
const message = `${cancelReason.message} (${cancelReason.code})`;
|
|
587
1112
|
const cancelledTaskIds: string[] = [];
|
|
588
1113
|
tasks = tasks.map((task) => {
|
|
589
|
-
if (
|
|
1114
|
+
if (
|
|
1115
|
+
task.status !== "queued" &&
|
|
1116
|
+
task.status !== "running" &&
|
|
1117
|
+
task.status !== "waiting"
|
|
1118
|
+
)
|
|
1119
|
+
return task;
|
|
590
1120
|
cancelledTaskIds.push(task.id);
|
|
591
|
-
const base = {
|
|
1121
|
+
const base = {
|
|
1122
|
+
...task,
|
|
1123
|
+
status: "cancelled" as const,
|
|
1124
|
+
finishedAt: new Date().toISOString(),
|
|
1125
|
+
error: message,
|
|
1126
|
+
};
|
|
592
1127
|
if (task.status === "running") {
|
|
593
|
-
return {
|
|
1128
|
+
return {
|
|
1129
|
+
...base,
|
|
1130
|
+
terminalEvidence: [
|
|
1131
|
+
...(task.terminalEvidence ?? []),
|
|
1132
|
+
buildSyntheticTerminalEvidence(
|
|
1133
|
+
"worker",
|
|
1134
|
+
cancelReason,
|
|
1135
|
+
task.startedAt,
|
|
1136
|
+
),
|
|
1137
|
+
],
|
|
1138
|
+
};
|
|
594
1139
|
}
|
|
595
1140
|
return base;
|
|
596
1141
|
});
|
|
597
1142
|
await saveRunTasksAsync(manifest, tasks);
|
|
598
|
-
for (const taskId of cancelledTaskIds)
|
|
599
|
-
|
|
1143
|
+
for (const taskId of cancelledTaskIds)
|
|
1144
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1145
|
+
type: "task.cancelled",
|
|
1146
|
+
runId: manifest.runId,
|
|
1147
|
+
taskId,
|
|
1148
|
+
message,
|
|
1149
|
+
data: { reason: cancelReason.code },
|
|
1150
|
+
});
|
|
1151
|
+
manifest = updateRunStatus(manifest, "cancelled", message, {
|
|
1152
|
+
data: { reason: cancelReason.code, cancelledTaskIds },
|
|
1153
|
+
});
|
|
600
1154
|
return { manifest, tasks };
|
|
601
1155
|
}
|
|
602
1156
|
|
|
603
1157
|
const failed = tasks.find((task) => task.status === "failed");
|
|
604
1158
|
if (failed) {
|
|
605
|
-
|
|
1159
|
+
// #4 (assessment): honor limits.maxRetriesPerTask — re-queue an eligible
|
|
1160
|
+
// failed task for a bounded whole-task rerun instead of immediately
|
|
1161
|
+
// aborting the run. Before #4, the recovery ledger recorded `rerun_task`
|
|
1162
|
+
// entries with state:"planned" but never executed them (decorative).
|
|
1163
|
+
// Default-off: maxRetriesPerTask=0 → original abort behavior preserved.
|
|
1164
|
+
const rerun = shouldRerunFailedTask(failed, input.limits);
|
|
1165
|
+
if (rerun.rerun) {
|
|
1166
|
+
tasks = tasks.map((item) =>
|
|
1167
|
+
item.id === failed.id
|
|
1168
|
+
? {
|
|
1169
|
+
...item,
|
|
1170
|
+
status: "queued" as const,
|
|
1171
|
+
policy: {
|
|
1172
|
+
...(item.policy ?? {}),
|
|
1173
|
+
retryCount: rerun.newRetryCount,
|
|
1174
|
+
},
|
|
1175
|
+
error: undefined,
|
|
1176
|
+
finishedAt: undefined,
|
|
1177
|
+
}
|
|
1178
|
+
: item,
|
|
1179
|
+
);
|
|
1180
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
1181
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1182
|
+
type: "recovery.rerun_task",
|
|
1183
|
+
runId: manifest.runId,
|
|
1184
|
+
taskId: failed.id,
|
|
1185
|
+
message: `Re-queuing failed task for whole-task rerun: ${rerun.reason}`,
|
|
1186
|
+
data: {
|
|
1187
|
+
attempt: rerun.newRetryCount,
|
|
1188
|
+
maxRetries: input.limits?.maxRetriesPerTask ?? 0,
|
|
1189
|
+
scenario: "task_failed",
|
|
1190
|
+
},
|
|
1191
|
+
});
|
|
1192
|
+
continue; // loop re-processes the re-queued task
|
|
1193
|
+
}
|
|
1194
|
+
tasks = markBlocked(
|
|
1195
|
+
tasks,
|
|
1196
|
+
`Blocked by failed task '${failed.id}'.`,
|
|
1197
|
+
);
|
|
606
1198
|
await saveRunTasksAsync(manifest, tasks);
|
|
607
|
-
saveCrewAgents(
|
|
608
|
-
|
|
1199
|
+
saveCrewAgents(
|
|
1200
|
+
manifest,
|
|
1201
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1202
|
+
);
|
|
1203
|
+
manifest = updateRunStatus(
|
|
1204
|
+
manifest,
|
|
1205
|
+
"failed",
|
|
1206
|
+
`Failed at task '${failed.id}'.`,
|
|
1207
|
+
);
|
|
609
1208
|
return { manifest, tasks };
|
|
610
1209
|
}
|
|
611
1210
|
|
|
@@ -614,67 +1213,202 @@ async function executeTeamRunCore(
|
|
|
614
1213
|
// DAG-based execution plan: when tasks have explicit dependsOn, use the
|
|
615
1214
|
// topological wave planner to determine ready tasks. Fall back to the
|
|
616
1215
|
// existing task-graph-scheduler when no explicit deps exist (backward compat).
|
|
617
|
-
const completedIds = new Set(
|
|
1216
|
+
const completedIds = new Set(
|
|
1217
|
+
tasks
|
|
1218
|
+
.filter(
|
|
1219
|
+
(t) =>
|
|
1220
|
+
t.status === "completed" ||
|
|
1221
|
+
t.status === "needs_attention",
|
|
1222
|
+
)
|
|
1223
|
+
.map((t) => t.id),
|
|
1224
|
+
);
|
|
618
1225
|
const dagReady = dagReadyTaskIds(tasks, completedIds);
|
|
619
1226
|
const effectiveReady = dagReady ?? snapshot.ready;
|
|
620
1227
|
|
|
621
1228
|
// Workflow phase precondition check (non-blocking: log warnings only).
|
|
622
1229
|
if (wfMachine.currentPhaseIndex < wfMachine.phases.length) {
|
|
623
|
-
const completedArtifacts = manifest.artifacts
|
|
624
|
-
|
|
1230
|
+
const completedArtifacts = manifest.artifacts
|
|
1231
|
+
.filter((a) => a.kind === "result" || a.kind === "summary")
|
|
1232
|
+
.map((a) => a.path);
|
|
1233
|
+
const previousPhaseStatus =
|
|
1234
|
+
wfMachine.currentPhaseIndex > 0
|
|
1235
|
+
? (wfMachine.phases[wfMachine.currentPhaseIndex - 1]
|
|
1236
|
+
?.status ?? "pending")
|
|
1237
|
+
: "completed";
|
|
625
1238
|
const wfContext: PhaseGuardContext = {
|
|
626
1239
|
completedArtifacts,
|
|
627
1240
|
previousPhaseStatus,
|
|
628
|
-
taskResults: tasks
|
|
1241
|
+
taskResults: tasks
|
|
1242
|
+
.filter(
|
|
1243
|
+
(t) =>
|
|
1244
|
+
t.status === "completed" ||
|
|
1245
|
+
t.status === "needs_attention",
|
|
1246
|
+
)
|
|
1247
|
+
.map((t) => ({
|
|
1248
|
+
taskId: t.id,
|
|
1249
|
+
status: t.status,
|
|
1250
|
+
outputPath: t.resultArtifact?.path,
|
|
1251
|
+
})),
|
|
629
1252
|
};
|
|
630
|
-
const preconditions = validatePhasePreconditions(
|
|
1253
|
+
const preconditions = validatePhasePreconditions(
|
|
1254
|
+
wfMachine,
|
|
1255
|
+
wfContext,
|
|
1256
|
+
);
|
|
631
1257
|
if (!preconditions.ready) {
|
|
632
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
1258
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1259
|
+
type: "workflow.preconditions",
|
|
1260
|
+
runId: manifest.runId,
|
|
1261
|
+
message: `Workflow phase '${wfMachine.phases[wfMachine.currentPhaseIndex]?.name}' is missing inputs: ${preconditions.blocking.join(", ")}`,
|
|
1262
|
+
data: {
|
|
1263
|
+
phaseIndex: wfMachine.currentPhaseIndex,
|
|
1264
|
+
phaseName:
|
|
1265
|
+
wfMachine.phases[wfMachine.currentPhaseIndex]?.name,
|
|
1266
|
+
blocking: preconditions.blocking,
|
|
1267
|
+
},
|
|
1268
|
+
});
|
|
633
1269
|
} else {
|
|
634
1270
|
// Advance the machine past completed phases.
|
|
635
|
-
while (
|
|
636
|
-
wfMachine
|
|
1271
|
+
while (
|
|
1272
|
+
wfMachine.currentPhaseIndex < wfMachine.phases.length &&
|
|
1273
|
+
wfMachine.phases[wfMachine.currentPhaseIndex]?.status ===
|
|
1274
|
+
"completed"
|
|
1275
|
+
) {
|
|
1276
|
+
wfMachine = {
|
|
1277
|
+
...wfMachine,
|
|
1278
|
+
currentPhaseIndex: wfMachine.currentPhaseIndex + 1,
|
|
1279
|
+
};
|
|
637
1280
|
}
|
|
638
1281
|
}
|
|
639
1282
|
}
|
|
640
1283
|
|
|
641
|
-
const readyRoles = effectiveReady
|
|
642
|
-
|
|
1284
|
+
const readyRoles = effectiveReady
|
|
1285
|
+
.map((taskId) => tasks.find((task) => task.id === taskId)?.role)
|
|
1286
|
+
.filter((role): role is string => Boolean(role));
|
|
1287
|
+
const concurrency = resolveBatchConcurrency({
|
|
1288
|
+
workflowName: workflow.name,
|
|
1289
|
+
workflowMaxConcurrency: workflow.maxConcurrency,
|
|
1290
|
+
teamMaxConcurrency: input.team.maxConcurrency,
|
|
1291
|
+
limitMaxConcurrentWorkers: input.limits?.maxConcurrentWorkers,
|
|
1292
|
+
allowUnboundedConcurrency: input.limits?.allowUnboundedConcurrency,
|
|
1293
|
+
readyCount: effectiveReady.length,
|
|
1294
|
+
workspaceMode: manifest.workspaceMode,
|
|
1295
|
+
readyRoles,
|
|
1296
|
+
});
|
|
643
1297
|
if (concurrency.reason.includes(";unbounded:")) {
|
|
644
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
1298
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1299
|
+
type: "limits.unbounded",
|
|
1300
|
+
runId: manifest.runId,
|
|
1301
|
+
message:
|
|
1302
|
+
"Unbounded worker concurrency was explicitly enabled for this run.",
|
|
1303
|
+
data: {
|
|
1304
|
+
concurrencyReason: concurrency.reason,
|
|
1305
|
+
maxConcurrent: concurrency.maxConcurrent,
|
|
1306
|
+
},
|
|
1307
|
+
});
|
|
645
1308
|
}
|
|
646
1309
|
const approvalPending = isPlanApprovalPending(manifest);
|
|
647
|
-
const readyIds = approvalPending
|
|
648
|
-
|
|
649
|
-
|
|
1310
|
+
const readyIds = approvalPending
|
|
1311
|
+
? effectiveReady
|
|
1312
|
+
: effectiveReady.slice(0, concurrency.selectedCount);
|
|
1313
|
+
const candidateBatch = readyIds
|
|
1314
|
+
.map((id) => tasks.find((task) => task.id === id))
|
|
1315
|
+
.filter((task): task is TeamTaskState => Boolean(task));
|
|
1316
|
+
const readyBatch = approvalPending
|
|
1317
|
+
? candidateBatch
|
|
1318
|
+
.filter((task) => !isMutatingTask(task))
|
|
1319
|
+
.slice(0, concurrency.selectedCount)
|
|
1320
|
+
: candidateBatch;
|
|
650
1321
|
if (readyBatch.length === 0) {
|
|
651
1322
|
if (approvalPending && candidateBatch.some(isMutatingTask)) {
|
|
652
1323
|
await saveRunTasksAsync(manifest, tasks);
|
|
653
|
-
saveCrewAgents(
|
|
654
|
-
|
|
1324
|
+
saveCrewAgents(
|
|
1325
|
+
manifest,
|
|
1326
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1327
|
+
);
|
|
1328
|
+
manifest = updateRunStatus(
|
|
1329
|
+
manifest,
|
|
1330
|
+
"blocked",
|
|
1331
|
+
"Plan approval required before mutating implementation tasks run.",
|
|
1332
|
+
);
|
|
655
1333
|
return { manifest, tasks };
|
|
656
1334
|
}
|
|
657
|
-
tasks = markBlocked(
|
|
1335
|
+
tasks = markBlocked(
|
|
1336
|
+
tasks,
|
|
1337
|
+
"No ready queued task; dependency graph may be invalid.",
|
|
1338
|
+
);
|
|
658
1339
|
await saveRunTasksAsync(manifest, tasks);
|
|
659
|
-
saveCrewAgents(
|
|
660
|
-
|
|
1340
|
+
saveCrewAgents(
|
|
1341
|
+
manifest,
|
|
1342
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1343
|
+
);
|
|
1344
|
+
manifest = updateRunStatus(
|
|
1345
|
+
manifest,
|
|
1346
|
+
"blocked",
|
|
1347
|
+
"No ready queued task.",
|
|
1348
|
+
);
|
|
661
1349
|
return { manifest, tasks };
|
|
662
1350
|
}
|
|
663
1351
|
|
|
664
1352
|
// 2.2 caller migration: batch progress is high-frequency informational.
|
|
665
|
-
appendEventFireAndForget(manifest.eventsPath, {
|
|
1353
|
+
appendEventFireAndForget(manifest.eventsPath, {
|
|
1354
|
+
type: "task.progress",
|
|
1355
|
+
runId: manifest.runId,
|
|
1356
|
+
message: `Starting ready batch with ${readyBatch.length} task(s).`,
|
|
1357
|
+
data: {
|
|
1358
|
+
taskIds: readyBatch.map((task) => task.id),
|
|
1359
|
+
readyCount: snapshot.ready.length,
|
|
1360
|
+
blockedCount: snapshot.blocked.length,
|
|
1361
|
+
runningCount: snapshot.running.length,
|
|
1362
|
+
doneCount: snapshot.done.length,
|
|
1363
|
+
selectedCount: readyBatch.length,
|
|
1364
|
+
maxConcurrent: concurrency.maxConcurrent,
|
|
1365
|
+
defaultConcurrency: concurrency.defaultConcurrency,
|
|
1366
|
+
concurrencyReason: approvalPending
|
|
1367
|
+
? `${concurrency.reason};plan-approval-read-only`
|
|
1368
|
+
: concurrency.reason,
|
|
1369
|
+
},
|
|
1370
|
+
});
|
|
666
1371
|
// Execute before_task_start hooks for the batch
|
|
667
1372
|
for (const task of readyBatch) {
|
|
668
|
-
const taskReport = await executeHook("before_task_start", {
|
|
1373
|
+
const taskReport = await executeHook("before_task_start", {
|
|
1374
|
+
runId: manifest.runId,
|
|
1375
|
+
taskId: task.id,
|
|
1376
|
+
cwd: manifest.cwd,
|
|
1377
|
+
});
|
|
669
1378
|
appendHookEvent(manifest, taskReport);
|
|
670
1379
|
if (taskReport.outcome === "block") {
|
|
671
|
-
tasks = tasks.map((t) =>
|
|
672
|
-
|
|
1380
|
+
tasks = tasks.map((t) =>
|
|
1381
|
+
t.id === task.id
|
|
1382
|
+
? {
|
|
1383
|
+
...t,
|
|
1384
|
+
status: "skipped" as const,
|
|
1385
|
+
error:
|
|
1386
|
+
taskReport.reason ??
|
|
1387
|
+
"before_task_start hook blocked execution.",
|
|
1388
|
+
}
|
|
1389
|
+
: t,
|
|
1390
|
+
);
|
|
1391
|
+
manifest = updateRunStatus(
|
|
1392
|
+
manifest,
|
|
1393
|
+
manifest.status,
|
|
1394
|
+
`Task '${task.id}' blocked by hook.`,
|
|
1395
|
+
);
|
|
673
1396
|
}
|
|
674
1397
|
}
|
|
675
|
-
const batchTasks = readyBatch.filter((task) =>
|
|
1398
|
+
const batchTasks = readyBatch.filter((task) =>
|
|
1399
|
+
tasks.find((t) => t.id === task.id && t.status !== "skipped"),
|
|
1400
|
+
);
|
|
676
1401
|
if (batchTasks.length > 1) {
|
|
677
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
1402
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1403
|
+
type: "task.parallel_start",
|
|
1404
|
+
runId: manifest.runId,
|
|
1405
|
+
message: `Launching ${batchTasks.length} tasks in PARALLEL (concurrency=${concurrency.selectedCount}): ${batchTasks.map((t) => `${t.role}(${t.id})`).join(", ")}`,
|
|
1406
|
+
data: {
|
|
1407
|
+
taskIds: batchTasks.map((t) => t.id),
|
|
1408
|
+
roles: batchTasks.map((t) => t.role),
|
|
1409
|
+
concurrency: concurrency.selectedCount,
|
|
1410
|
+
},
|
|
1411
|
+
});
|
|
678
1412
|
}
|
|
679
1413
|
const results = await mapConcurrent(
|
|
680
1414
|
batchTasks,
|
|
@@ -682,73 +1416,302 @@ async function executeTeamRunCore(
|
|
|
682
1416
|
async (task) => {
|
|
683
1417
|
const step = findStep(workflow, task);
|
|
684
1418
|
const agent = findAgent(input.agents, task);
|
|
685
|
-
const teamRole = input.team.roles.find(
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
1419
|
+
const teamRole = input.team.roles.find(
|
|
1420
|
+
(role) => role.name === task.role,
|
|
1421
|
+
);
|
|
1422
|
+
const perTaskRuntime = resolveTaskRuntimeKind(
|
|
1423
|
+
runtimeKind,
|
|
1424
|
+
task.role,
|
|
1425
|
+
input.runtimeConfig?.isolationPolicy,
|
|
1426
|
+
);
|
|
1427
|
+
const baseInput = {
|
|
1428
|
+
manifest,
|
|
1429
|
+
tasks,
|
|
1430
|
+
task,
|
|
1431
|
+
step,
|
|
1432
|
+
agent,
|
|
1433
|
+
signal: input.signal,
|
|
1434
|
+
executeWorkers: input.executeWorkers,
|
|
1435
|
+
runtimeKind: runtimeKind,
|
|
1436
|
+
taskRuntimeOverride:
|
|
1437
|
+
perTaskRuntime !== runtimeKind
|
|
1438
|
+
? perTaskRuntime
|
|
1439
|
+
: undefined,
|
|
1440
|
+
runtimeConfig: input.runtimeConfig,
|
|
1441
|
+
parentContext: input.parentContext,
|
|
1442
|
+
parentModel: input.parentModel,
|
|
1443
|
+
modelRegistry: input.modelRegistry,
|
|
1444
|
+
modelOverride: input.modelOverride,
|
|
1445
|
+
teamRoleModel: teamRole?.model,
|
|
1446
|
+
teamRoleSkills: teamRole?.skills,
|
|
1447
|
+
skillOverride: input.skillOverride,
|
|
1448
|
+
limits: input.limits,
|
|
1449
|
+
onJsonEvent: input.onJsonEvent,
|
|
1450
|
+
workspaceId: input.workspaceId,
|
|
1451
|
+
};
|
|
1452
|
+
// #1 (assessment): autoRetry now defaults ON (opt-out via reliability.autoRetry=false).
|
|
1453
|
+
// The dominant v0.9.13 failure was ChildTimeout ("worker became unresponsive") with
|
|
1454
|
+
// ZERO retries because this gate was opt-in. isRetryable() defaults to true when
|
|
1455
|
+
// retryableErrors is empty, so transient hangs now retry up to maxAttempts (3) with
|
|
1456
|
+
// exponential backoff. Set reliability.autoRetry=false to restore old single-shot behavior.
|
|
1457
|
+
if (!shouldUseRetry(input.reliability))
|
|
1458
|
+
return withCorrelation(
|
|
1459
|
+
childCorrelation(manifest.runId, task.id),
|
|
1460
|
+
() => runTeamTask(baseInput),
|
|
1461
|
+
);
|
|
1462
|
+
let lastFailed:
|
|
1463
|
+
| { manifest: TeamRunManifest; tasks: TeamTaskState[] }
|
|
1464
|
+
| undefined;
|
|
690
1465
|
let lastAttemptId: string | undefined;
|
|
691
|
-
const attemptsSoFar: TaskAttemptState[] = [
|
|
1466
|
+
const attemptsSoFar: TaskAttemptState[] = [
|
|
1467
|
+
...(task.attempts ?? []),
|
|
1468
|
+
];
|
|
692
1469
|
const policy = retryPolicyFromConfig(input.reliability);
|
|
693
1470
|
try {
|
|
694
|
-
return await executeWithRetry(
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
1471
|
+
return await executeWithRetry(
|
|
1472
|
+
async (attempt, info) => {
|
|
1473
|
+
const startedAt = new Date().toISOString();
|
|
1474
|
+
const inFlightAttempts: TaskAttemptState[] = [
|
|
1475
|
+
...attemptsSoFar,
|
|
1476
|
+
{ attemptId: info.attemptId, startedAt },
|
|
1477
|
+
];
|
|
1478
|
+
input.metricRegistry
|
|
1479
|
+
?.counter(
|
|
1480
|
+
"crew.task.retry_attempt_total",
|
|
1481
|
+
"Retry attempts by run and task",
|
|
1482
|
+
)
|
|
1483
|
+
.inc({
|
|
1484
|
+
runId: manifest.runId,
|
|
1485
|
+
taskId: task.id,
|
|
1486
|
+
});
|
|
1487
|
+
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
1488
|
+
const fresh = loadRunManifestById(
|
|
1489
|
+
manifest.cwd,
|
|
1490
|
+
manifest.runId,
|
|
1491
|
+
);
|
|
1492
|
+
const freshManifest = fresh?.manifest ?? manifest;
|
|
1493
|
+
const freshTasks = fresh?.tasks ?? tasks;
|
|
1494
|
+
const freshTask =
|
|
1495
|
+
freshTasks.find(
|
|
1496
|
+
(item) => item.id === task.id,
|
|
1497
|
+
) ?? task;
|
|
1498
|
+
if (
|
|
1499
|
+
freshTask.status !== "queued" &&
|
|
1500
|
+
freshTask.status !== "running"
|
|
1501
|
+
)
|
|
1502
|
+
return {
|
|
1503
|
+
manifest: freshManifest,
|
|
1504
|
+
tasks: freshTasks,
|
|
1505
|
+
};
|
|
1506
|
+
const taskWithAttempt: TeamTaskState = {
|
|
1507
|
+
...freshTask,
|
|
1508
|
+
attempts: inFlightAttempts,
|
|
1509
|
+
};
|
|
1510
|
+
const result = await withCorrelation(
|
|
1511
|
+
childCorrelation(freshManifest.runId, task.id),
|
|
1512
|
+
() =>
|
|
1513
|
+
runTeamTask({
|
|
1514
|
+
...baseInput,
|
|
1515
|
+
manifest: freshManifest,
|
|
1516
|
+
tasks: freshTasks,
|
|
1517
|
+
task: taskWithAttempt,
|
|
1518
|
+
}),
|
|
1519
|
+
);
|
|
1520
|
+
const failed = failedTaskFrom(result, task.id);
|
|
1521
|
+
const endedAt = new Date().toISOString();
|
|
1522
|
+
const finishedAttempt: TaskAttemptState = {
|
|
1523
|
+
attemptId: info.attemptId,
|
|
1524
|
+
startedAt,
|
|
1525
|
+
endedAt,
|
|
1526
|
+
...(failed?.error
|
|
1527
|
+
? { error: failed.error }
|
|
1528
|
+
: {}),
|
|
1529
|
+
};
|
|
1530
|
+
attemptsSoFar.push(finishedAttempt);
|
|
1531
|
+
const withAttempt = result.tasks.map((item) =>
|
|
1532
|
+
item.id === task.id
|
|
1533
|
+
? { ...item, attempts: [...attemptsSoFar] }
|
|
1534
|
+
: item,
|
|
1535
|
+
);
|
|
1536
|
+
const enriched = {
|
|
1537
|
+
manifest: result.manifest,
|
|
1538
|
+
tasks: withAttempt,
|
|
1539
|
+
};
|
|
1540
|
+
if (failed) {
|
|
1541
|
+
lastFailed = enriched;
|
|
1542
|
+
throw new Error(
|
|
1543
|
+
failed.error ?? `Task ${task.id} failed.`,
|
|
1544
|
+
);
|
|
1545
|
+
}
|
|
1546
|
+
input.metricRegistry
|
|
1547
|
+
?.histogram(
|
|
1548
|
+
"crew.task.retry_count",
|
|
1549
|
+
"Retries per task",
|
|
1550
|
+
[0, 1, 2, 3, 5, 10],
|
|
1551
|
+
)
|
|
1552
|
+
.observe(
|
|
1553
|
+
{
|
|
1554
|
+
runId: manifest.runId,
|
|
1555
|
+
team: input.team.name,
|
|
1556
|
+
},
|
|
1557
|
+
Math.max(0, attempt - 1),
|
|
1558
|
+
);
|
|
1559
|
+
return enriched;
|
|
725
1560
|
},
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
1561
|
+
policy,
|
|
1562
|
+
{
|
|
1563
|
+
signal: input.signal,
|
|
1564
|
+
attemptId: (attempt) =>
|
|
1565
|
+
`${manifest.runId}:${task.id}:attempt-${attempt}`,
|
|
1566
|
+
onAttemptFailed: (
|
|
1567
|
+
attempt,
|
|
1568
|
+
error,
|
|
1569
|
+
delayMs,
|
|
1570
|
+
info,
|
|
1571
|
+
) => {
|
|
1572
|
+
lastAttemptId = info.attemptId;
|
|
1573
|
+
appendEventAsync(manifest.eventsPath, {
|
|
1574
|
+
type: "crew.task.retry_attempt",
|
|
1575
|
+
runId: manifest.runId,
|
|
1576
|
+
taskId: task.id,
|
|
1577
|
+
message: error.message,
|
|
1578
|
+
data: {
|
|
1579
|
+
attempt,
|
|
1580
|
+
attemptId: info.attemptId,
|
|
1581
|
+
delayMs,
|
|
1582
|
+
},
|
|
1583
|
+
metadata: { attemptId: info.attemptId },
|
|
1584
|
+
}).catch((error) =>
|
|
1585
|
+
logInternalError(
|
|
1586
|
+
"team-runner.retry-attempt",
|
|
1587
|
+
error,
|
|
1588
|
+
`taskId=${task.id}`,
|
|
1589
|
+
),
|
|
1590
|
+
);
|
|
1591
|
+
input.metricRegistry
|
|
1592
|
+
?.histogram(
|
|
1593
|
+
"crew.task.retry_delay_ms",
|
|
1594
|
+
"Retry backoff delay, milliseconds",
|
|
1595
|
+
)
|
|
1596
|
+
.observe(
|
|
1597
|
+
{
|
|
1598
|
+
runId: manifest.runId,
|
|
1599
|
+
taskId: task.id,
|
|
1600
|
+
},
|
|
1601
|
+
delayMs,
|
|
1602
|
+
);
|
|
1603
|
+
},
|
|
1604
|
+
onRetryGivenUp: (attempts, error, info) => {
|
|
1605
|
+
lastAttemptId = info.attemptId;
|
|
1606
|
+
appendDeadletter(manifest, {
|
|
1607
|
+
runId: manifest.runId,
|
|
1608
|
+
taskId: task.id,
|
|
1609
|
+
reason: "max-retries",
|
|
1610
|
+
attempts,
|
|
1611
|
+
attemptId: info.attemptId,
|
|
1612
|
+
lastError: error.message,
|
|
1613
|
+
timestamp: new Date().toISOString(),
|
|
1614
|
+
});
|
|
1615
|
+
input.metricRegistry
|
|
1616
|
+
?.counter(
|
|
1617
|
+
"crew.task.deadletter_total",
|
|
1618
|
+
"Deadletter triggers by reason",
|
|
1619
|
+
)
|
|
1620
|
+
.inc({ reason: "max-retries" });
|
|
1621
|
+
input.metricRegistry
|
|
1622
|
+
?.histogram(
|
|
1623
|
+
"crew.task.retry_count",
|
|
1624
|
+
"Retries per task",
|
|
1625
|
+
[0, 1, 2, 3, 5, 10],
|
|
1626
|
+
)
|
|
1627
|
+
.observe(
|
|
1628
|
+
{
|
|
1629
|
+
runId: manifest.runId,
|
|
1630
|
+
team: input.team.name,
|
|
1631
|
+
},
|
|
1632
|
+
Math.max(0, attempts - 1),
|
|
1633
|
+
);
|
|
1634
|
+
},
|
|
731
1635
|
},
|
|
732
|
-
|
|
1636
|
+
);
|
|
733
1637
|
} catch (retryError) {
|
|
734
|
-
if (
|
|
735
|
-
|
|
1638
|
+
if (
|
|
1639
|
+
retryError instanceof CrewCancellationError ||
|
|
1640
|
+
input.signal?.aborted
|
|
1641
|
+
) {
|
|
1642
|
+
const reason =
|
|
1643
|
+
retryError instanceof CrewCancellationError
|
|
1644
|
+
? retryError.reason
|
|
1645
|
+
: cancellationReasonFromSignal(input.signal);
|
|
736
1646
|
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
737
|
-
const fresh = loadRunManifestById(
|
|
1647
|
+
const fresh = loadRunManifestById(
|
|
1648
|
+
manifest.cwd,
|
|
1649
|
+
manifest.runId,
|
|
1650
|
+
);
|
|
738
1651
|
const freshManifest = fresh?.manifest ?? manifest;
|
|
739
1652
|
const freshTasks = fresh?.tasks ?? tasks;
|
|
740
|
-
const cancelledTasks = freshTasks.map((item) =>
|
|
741
|
-
|
|
742
|
-
|
|
1653
|
+
const cancelledTasks = freshTasks.map((item) =>
|
|
1654
|
+
item.id === task.id &&
|
|
1655
|
+
(item.status === "queued" ||
|
|
1656
|
+
item.status === "running")
|
|
1657
|
+
? {
|
|
1658
|
+
...item,
|
|
1659
|
+
status: "cancelled" as const,
|
|
1660
|
+
finishedAt: new Date().toISOString(),
|
|
1661
|
+
error: `${reason.message} (${reason.code})`,
|
|
1662
|
+
}
|
|
1663
|
+
: item,
|
|
1664
|
+
);
|
|
1665
|
+
appendEventAsync(freshManifest.eventsPath, {
|
|
1666
|
+
type: "task.cancelled",
|
|
1667
|
+
runId: freshManifest.runId,
|
|
1668
|
+
taskId: task.id,
|
|
1669
|
+
message: reason.message,
|
|
1670
|
+
data: { reason, phase: "retry" },
|
|
1671
|
+
metadata: lastAttemptId
|
|
1672
|
+
? { attemptId: lastAttemptId }
|
|
1673
|
+
: undefined,
|
|
1674
|
+
}).catch((error) =>
|
|
1675
|
+
logInternalError(
|
|
1676
|
+
"team-runner.cancelled",
|
|
1677
|
+
error,
|
|
1678
|
+
`taskId=${task.id}`,
|
|
1679
|
+
),
|
|
1680
|
+
);
|
|
1681
|
+
return {
|
|
1682
|
+
manifest: updateRunStatus(
|
|
1683
|
+
freshManifest,
|
|
1684
|
+
"cancelled",
|
|
1685
|
+
reason.message,
|
|
1686
|
+
),
|
|
1687
|
+
tasks: cancelledTasks,
|
|
1688
|
+
};
|
|
743
1689
|
}
|
|
744
1690
|
if (lastFailed) return lastFailed;
|
|
745
1691
|
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
746
|
-
const fresh = loadRunManifestById(
|
|
1692
|
+
const fresh = loadRunManifestById(
|
|
1693
|
+
manifest.cwd,
|
|
1694
|
+
manifest.runId,
|
|
1695
|
+
);
|
|
747
1696
|
const freshManifest = fresh?.manifest ?? manifest;
|
|
748
1697
|
const freshTasks = fresh?.tasks ?? tasks;
|
|
749
|
-
const freshTask =
|
|
750
|
-
|
|
751
|
-
|
|
1698
|
+
const freshTask =
|
|
1699
|
+
freshTasks.find((item) => item.id === task.id) ?? task;
|
|
1700
|
+
if (
|
|
1701
|
+
freshTask.status !== "queued" &&
|
|
1702
|
+
freshTask.status !== "running"
|
|
1703
|
+
)
|
|
1704
|
+
return { manifest: freshManifest, tasks: freshTasks };
|
|
1705
|
+
return withCorrelation(
|
|
1706
|
+
childCorrelation(freshManifest.runId, task.id),
|
|
1707
|
+
() =>
|
|
1708
|
+
runTeamTask({
|
|
1709
|
+
...baseInput,
|
|
1710
|
+
manifest: freshManifest,
|
|
1711
|
+
tasks: freshTasks,
|
|
1712
|
+
task: freshTask,
|
|
1713
|
+
}),
|
|
1714
|
+
);
|
|
752
1715
|
}
|
|
753
1716
|
},
|
|
754
1717
|
);
|
|
@@ -757,42 +1720,66 @@ async function executeTeamRunCore(
|
|
|
757
1720
|
// during parallel execution. Other workers may have written partial results
|
|
758
1721
|
// before one threw. Results may be partial - some tasks in-flight at error
|
|
759
1722
|
// time will not have entries in the results array.
|
|
760
|
-
const validResults = results.filter(
|
|
1723
|
+
const validResults = results.filter(
|
|
1724
|
+
(item): item is NonNullable<typeof item> => item !== undefined,
|
|
1725
|
+
);
|
|
761
1726
|
// Guard: if ALL parallel workers threw before returning, validResults is empty.
|
|
762
1727
|
// at(-1)! would crash. Mark the run failed rather than crashing.
|
|
763
1728
|
if (validResults.length === 0) {
|
|
764
|
-
manifest = updateRunStatus(
|
|
1729
|
+
manifest = updateRunStatus(
|
|
1730
|
+
manifest,
|
|
1731
|
+
"failed",
|
|
1732
|
+
"All parallel tasks failed catastrophically.",
|
|
1733
|
+
);
|
|
765
1734
|
return { manifest, tasks };
|
|
766
1735
|
}
|
|
767
1736
|
// Reconstruct manifest from the last worker's snapshot. The .artifacts field
|
|
768
|
-
// is re-merged from both the team-runner's in-memory state and all workers'
|
|
769
|
-
// snapshots, so artifact writes by task-runner (which individually save manifest
|
|
770
|
-
// after writing artifacts) are safely persisted. The in-memory manifest is only
|
|
771
|
-
// used for the next batch iteration's orchestration — actual persistence is safe.
|
|
772
|
-
// Use updateRunStatus to recompute manifest status from merged tasks rather than
|
|
773
|
-
// relying on the last result's manifest (which is arbitrary due to mapConcurrent
|
|
774
|
-
// returning results in arbitrary order).
|
|
775
|
-
// Use the in-memory manifest as base (not the last-completing worker's snapshot).
|
|
776
|
-
// Recompute status from merged tasks so the manifest reflects actual task state,
|
|
777
|
-
// not the arbitrary order in which mapConcurrent returned results.
|
|
778
|
-
// Read committed manifest from disk inside the lock so artifact merge is based
|
|
779
|
-
// on committed state, not in-memory state that may differ from disk.
|
|
780
|
-
const mergeResult = await withRunLock(manifest, async () => {
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
1737
|
+
// is re-merged from both the team-runner's in-memory state and all workers'
|
|
1738
|
+
// snapshots, so artifact writes by task-runner (which individually save manifest
|
|
1739
|
+
// after writing artifacts) are safely persisted. The in-memory manifest is only
|
|
1740
|
+
// used for the next batch iteration's orchestration — actual persistence is safe.
|
|
1741
|
+
// Use updateRunStatus to recompute manifest status from merged tasks rather than
|
|
1742
|
+
// relying on the last result's manifest (which is arbitrary due to mapConcurrent
|
|
1743
|
+
// returning results in arbitrary order).
|
|
1744
|
+
// Use the in-memory manifest as base (not the last-completing worker's snapshot).
|
|
1745
|
+
// Recompute status from merged tasks so the manifest reflects actual task state,
|
|
1746
|
+
// not the arbitrary order in which mapConcurrent returned results.
|
|
1747
|
+
// Read committed manifest from disk inside the lock so artifact merge is based
|
|
1748
|
+
// on committed state, not in-memory state that may differ from disk.
|
|
1749
|
+
const mergeResult = await withRunLock(manifest, async () => {
|
|
1750
|
+
const disk = loadRunManifestById(manifest.cwd, manifest.runId);
|
|
1751
|
+
const diskManifest = disk?.manifest ?? manifest;
|
|
1752
|
+
const diskArtifacts = diskManifest.artifacts;
|
|
1753
|
+
const reconciledArtifacts = mergeArtifacts(
|
|
1754
|
+
[
|
|
1755
|
+
...diskArtifacts,
|
|
1756
|
+
...validResults.map((item) => item.manifest.artifacts),
|
|
1757
|
+
].flat(),
|
|
1758
|
+
);
|
|
1759
|
+
const resultManifest = updateRunStatus(
|
|
1760
|
+
{ ...diskManifest, artifacts: reconciledArtifacts },
|
|
1761
|
+
"running",
|
|
1762
|
+
"Merged task updates from parallel batch.",
|
|
1763
|
+
);
|
|
1764
|
+
const resultTasks = mergeTaskUpdatesPreservingTerminal(
|
|
1765
|
+
tasks,
|
|
1766
|
+
validResults,
|
|
1767
|
+
);
|
|
1768
|
+
await saveRunManifestAsync(resultManifest);
|
|
1769
|
+
await saveRunTasksAsync(resultManifest, resultTasks);
|
|
1770
|
+
return { resultManifest, resultTasks };
|
|
1771
|
+
});
|
|
1772
|
+
manifest = mergeResult.resultManifest;
|
|
1773
|
+
tasks = mergeResult.resultTasks;
|
|
793
1774
|
|
|
794
1775
|
// Advance workflow phases whose tasks are all in terminal state
|
|
795
|
-
const terminalStatuses = new Set([
|
|
1776
|
+
const terminalStatuses = new Set([
|
|
1777
|
+
"completed",
|
|
1778
|
+
"failed",
|
|
1779
|
+
"skipped",
|
|
1780
|
+
"cancelled",
|
|
1781
|
+
"needs_attention",
|
|
1782
|
+
]);
|
|
796
1783
|
const phaseTaskMap = new Map<string, string[]>();
|
|
797
1784
|
for (const task of tasks) {
|
|
798
1785
|
if (!task.stepId) continue;
|
|
@@ -800,7 +1787,11 @@ tasks = mergeResult.resultTasks;
|
|
|
800
1787
|
existing.push(task.id);
|
|
801
1788
|
phaseTaskMap.set(task.stepId, existing);
|
|
802
1789
|
}
|
|
803
|
-
for (
|
|
1790
|
+
for (
|
|
1791
|
+
let pi = wfMachine.currentPhaseIndex;
|
|
1792
|
+
pi < wfMachine.phases.length;
|
|
1793
|
+
pi++
|
|
1794
|
+
) {
|
|
804
1795
|
const phase = wfMachine.phases[pi]!;
|
|
805
1796
|
const phaseTaskIds = phaseTaskMap.get(phase.name) ?? [];
|
|
806
1797
|
if (phaseTaskIds.length === 0) continue;
|
|
@@ -809,75 +1800,187 @@ tasks = mergeResult.resultTasks;
|
|
|
809
1800
|
return task ? terminalStatuses.has(task.status) : false;
|
|
810
1801
|
});
|
|
811
1802
|
if (!allTerminal) break;
|
|
812
|
-
if (
|
|
813
|
-
|
|
814
|
-
|
|
1803
|
+
if (
|
|
1804
|
+
phase.status !== "completed" &&
|
|
1805
|
+
phase.status !== "failed" &&
|
|
1806
|
+
phase.status !== "skipped"
|
|
1807
|
+
) {
|
|
1808
|
+
const completedArtifacts = manifest.artifacts
|
|
1809
|
+
.filter((a) => a.kind === "result" || a.kind === "summary")
|
|
1810
|
+
.map((a) => a.path);
|
|
1811
|
+
const previousPhaseStatus =
|
|
1812
|
+
pi > 0
|
|
1813
|
+
? (wfMachine.phases[pi - 1]?.status ?? "pending")
|
|
1814
|
+
: "completed";
|
|
815
1815
|
const wfContext: PhaseGuardContext = {
|
|
816
1816
|
completedArtifacts,
|
|
817
1817
|
previousPhaseStatus,
|
|
818
|
-
taskResults: tasks
|
|
1818
|
+
taskResults: tasks
|
|
1819
|
+
.filter(
|
|
1820
|
+
(t) =>
|
|
1821
|
+
t.status === "completed" ||
|
|
1822
|
+
t.status === "needs_attention",
|
|
1823
|
+
)
|
|
1824
|
+
.map((t) => ({
|
|
1825
|
+
taskId: t.id,
|
|
1826
|
+
status: t.status,
|
|
1827
|
+
outputPath: t.resultArtifact?.path,
|
|
1828
|
+
})),
|
|
819
1829
|
};
|
|
820
1830
|
// Determine phase transition status based on individual task outcomes
|
|
821
|
-
const phaseTasks = phaseTaskIds
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
const
|
|
1831
|
+
const phaseTasks = phaseTaskIds
|
|
1832
|
+
.map((taskId) => tasks.find((t) => t.id === taskId))
|
|
1833
|
+
.filter((t): t is NonNullable<typeof t> => t !== undefined);
|
|
1834
|
+
const hasFailedOrCancelled = phaseTasks.some(
|
|
1835
|
+
(t) => t.status === "failed" || t.status === "cancelled",
|
|
1836
|
+
);
|
|
1837
|
+
const phaseStatus = hasFailedOrCancelled
|
|
1838
|
+
? "failed"
|
|
1839
|
+
: "completed";
|
|
1840
|
+
const transition = transitionPhase(
|
|
1841
|
+
wfMachine,
|
|
1842
|
+
pi,
|
|
1843
|
+
phaseStatus,
|
|
1844
|
+
wfContext,
|
|
1845
|
+
);
|
|
825
1846
|
wfMachine = transition.machine;
|
|
826
1847
|
if (transition.guardResult && !transition.guardResult.allowed) {
|
|
827
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
1848
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1849
|
+
type: "workflow.phase_guard_blocked",
|
|
1850
|
+
runId: manifest.runId,
|
|
1851
|
+
message: `Workflow phase '${phase.name}' guard blocked: ${transition.guardResult.reason ?? "unknown"}`,
|
|
1852
|
+
data: {
|
|
1853
|
+
phaseIndex: pi,
|
|
1854
|
+
phaseName: phase.name,
|
|
1855
|
+
reason: transition.guardResult.reason,
|
|
1856
|
+
},
|
|
1857
|
+
});
|
|
828
1858
|
break;
|
|
829
1859
|
}
|
|
830
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
1860
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1861
|
+
type:
|
|
1862
|
+
phaseStatus === "failed"
|
|
1863
|
+
? "workflow.phase_failed"
|
|
1864
|
+
: "workflow.phase_completed",
|
|
1865
|
+
runId: manifest.runId,
|
|
1866
|
+
message: `Workflow phase '${phase.name}' ${phaseStatus}.`,
|
|
1867
|
+
data: { phaseIndex: pi, phaseStatus },
|
|
1868
|
+
});
|
|
831
1869
|
}
|
|
832
1870
|
wfMachine = { ...wfMachine, currentPhaseIndex: pi + 1 };
|
|
833
1871
|
}
|
|
834
1872
|
|
|
835
|
-
const cancelledResult = results.find(
|
|
1873
|
+
const cancelledResult = results.find(
|
|
1874
|
+
(item) => item.manifest.status === "cancelled",
|
|
1875
|
+
);
|
|
836
1876
|
if (cancelledResult || input.signal?.aborted) {
|
|
837
|
-
const reason = input.signal?.aborted
|
|
838
|
-
|
|
1877
|
+
const reason = input.signal?.aborted
|
|
1878
|
+
? cancellationReasonFromSignal(input.signal)
|
|
1879
|
+
: undefined;
|
|
1880
|
+
const message =
|
|
1881
|
+
reason?.message ??
|
|
1882
|
+
cancelledResult?.manifest.summary ??
|
|
1883
|
+
"Run cancelled during task execution.";
|
|
839
1884
|
manifest = { ...manifest, status: "running" };
|
|
840
1885
|
manifest = updateRunStatus(manifest, "cancelled", message);
|
|
841
1886
|
await saveRunTasksAsync(manifest, tasks);
|
|
842
|
-
saveCrewAgents(
|
|
1887
|
+
saveCrewAgents(
|
|
1888
|
+
manifest,
|
|
1889
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1890
|
+
);
|
|
843
1891
|
await saveRunManifestAsync(manifest);
|
|
844
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
1892
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1893
|
+
type: "run.cancelled",
|
|
1894
|
+
runId: manifest.runId,
|
|
1895
|
+
message,
|
|
1896
|
+
data: {
|
|
1897
|
+
reason,
|
|
1898
|
+
phase: "task-batch",
|
|
1899
|
+
cancelledResultRunId: cancelledResult?.manifest.runId,
|
|
1900
|
+
},
|
|
1901
|
+
});
|
|
845
1902
|
return { manifest, tasks };
|
|
846
1903
|
}
|
|
847
1904
|
queueIndex = buildTaskGraphIndex(tasks);
|
|
848
1905
|
const injectedAfterBatch = attemptAdaptivePlan();
|
|
849
1906
|
if (injectedAfterBatch.missing) {
|
|
850
|
-
tasks = markBlocked(
|
|
1907
|
+
tasks = markBlocked(
|
|
1908
|
+
tasks,
|
|
1909
|
+
"Adaptive planner did not produce a valid subagent plan.",
|
|
1910
|
+
);
|
|
851
1911
|
await saveRunTasksAsync(manifest, tasks);
|
|
852
|
-
saveCrewAgents(
|
|
853
|
-
|
|
1912
|
+
saveCrewAgents(
|
|
1913
|
+
manifest,
|
|
1914
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1915
|
+
);
|
|
1916
|
+
manifest = updateRunStatus(
|
|
1917
|
+
manifest,
|
|
1918
|
+
"blocked",
|
|
1919
|
+
"Adaptive planner did not produce a valid subagent plan.",
|
|
1920
|
+
);
|
|
854
1921
|
return { manifest, tasks };
|
|
855
1922
|
}
|
|
856
1923
|
if (injectedAfterBatch.injected) {
|
|
857
|
-
manifest = requiresPlanApproval(workflow, input.runtimeConfig)
|
|
1924
|
+
manifest = requiresPlanApproval(workflow, input.runtimeConfig)
|
|
1925
|
+
? ensurePlanApprovalRequested(manifest, tasks)
|
|
1926
|
+
: manifest;
|
|
858
1927
|
queueIndex = buildTaskGraphIndex(tasks);
|
|
859
|
-
} else if (
|
|
1928
|
+
} else if (
|
|
1929
|
+
requiresPlanApproval(workflow, input.runtimeConfig) &&
|
|
1930
|
+
(hasPendingMutatingAdaptiveTask(tasks) ||
|
|
1931
|
+
hasPendingMutatingTaskAtBoundary(tasks))
|
|
1932
|
+
) {
|
|
860
1933
|
manifest = ensurePlanApprovalRequested(manifest, tasks);
|
|
861
1934
|
}
|
|
862
1935
|
if (manifest.planApproval?.status === "cancelled") {
|
|
863
1936
|
tasks = cancelPlanTasks(tasks, "Plan approval was cancelled.");
|
|
864
1937
|
await saveRunTasksAsync(manifest, tasks);
|
|
865
|
-
saveCrewAgents(
|
|
866
|
-
|
|
1938
|
+
saveCrewAgents(
|
|
1939
|
+
manifest,
|
|
1940
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1941
|
+
);
|
|
1942
|
+
manifest = updateRunStatus(
|
|
1943
|
+
manifest,
|
|
1944
|
+
"cancelled",
|
|
1945
|
+
"Plan approval was cancelled.",
|
|
1946
|
+
);
|
|
867
1947
|
return { manifest, tasks };
|
|
868
1948
|
}
|
|
869
1949
|
await saveRunTasksAsync(manifest, tasks);
|
|
870
|
-
saveCrewAgents(
|
|
871
|
-
|
|
1950
|
+
saveCrewAgents(
|
|
1951
|
+
manifest,
|
|
1952
|
+
recordsForMaterializedTasks(manifest, tasks, runtimeKind),
|
|
1953
|
+
);
|
|
1954
|
+
const completedBatch = tasks.filter((t) =>
|
|
1955
|
+
batchTasks.some((bt) => bt.id === t.id),
|
|
1956
|
+
);
|
|
872
1957
|
const batchArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
873
1958
|
kind: "summary",
|
|
874
1959
|
relativePath: `batches/${batchTasks.map((task) => task.id).join("+")}.md`,
|
|
875
1960
|
producer: "team-runner",
|
|
876
1961
|
content: aggregateTaskOutputs(completedBatch, manifest),
|
|
877
1962
|
});
|
|
878
|
-
const groupDelivery = deliverGroupJoin({
|
|
879
|
-
|
|
880
|
-
|
|
1963
|
+
const groupDelivery = deliverGroupJoin({
|
|
1964
|
+
manifest,
|
|
1965
|
+
mode: resolveGroupJoinMode(input.runtimeConfig),
|
|
1966
|
+
batch: batchTasks,
|
|
1967
|
+
allTasks: tasks,
|
|
1968
|
+
});
|
|
1969
|
+
manifest = {
|
|
1970
|
+
...manifest,
|
|
1971
|
+
artifacts: mergeArtifacts([
|
|
1972
|
+
...manifest.artifacts,
|
|
1973
|
+
batchArtifact,
|
|
1974
|
+
...(groupDelivery?.artifact ? [groupDelivery.artifact] : []),
|
|
1975
|
+
]),
|
|
1976
|
+
};
|
|
1977
|
+
manifest = writeProgress(
|
|
1978
|
+
manifest,
|
|
1979
|
+
tasks,
|
|
1980
|
+
"team-runner",
|
|
1981
|
+
input.executeWorkers,
|
|
1982
|
+
input.runtimeConfig,
|
|
1983
|
+
);
|
|
881
1984
|
await saveRunManifestAsync(manifest);
|
|
882
1985
|
}
|
|
883
1986
|
|
|
@@ -885,29 +1988,85 @@ tasks = mergeResult.resultTasks;
|
|
|
885
1988
|
const waiting = tasks.find((task) => task.status === "waiting");
|
|
886
1989
|
const running = tasks.find((task) => task.status === "running");
|
|
887
1990
|
manifest = applyPolicy(manifest, tasks, input.limits);
|
|
888
|
-
const effectiveness = evaluateRunEffectiveness({
|
|
1991
|
+
const effectiveness = evaluateRunEffectiveness({
|
|
1992
|
+
manifest,
|
|
1993
|
+
tasks,
|
|
1994
|
+
executeWorkers: input.executeWorkers,
|
|
1995
|
+
runtimeConfig: input.runtimeConfig,
|
|
1996
|
+
});
|
|
889
1997
|
const effectivenessDecision = effectivenessPolicyDecision(effectiveness);
|
|
890
1998
|
if (effectivenessDecision) {
|
|
891
|
-
manifest = {
|
|
892
|
-
|
|
1999
|
+
manifest = {
|
|
2000
|
+
...manifest,
|
|
2001
|
+
policyDecisions: [
|
|
2002
|
+
...(manifest.policyDecisions ?? []),
|
|
2003
|
+
effectivenessDecision,
|
|
2004
|
+
],
|
|
2005
|
+
updatedAt: new Date().toISOString(),
|
|
2006
|
+
};
|
|
2007
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
2008
|
+
type: "run.effectiveness",
|
|
2009
|
+
runId: manifest.runId,
|
|
2010
|
+
message: effectivenessDecision.message,
|
|
2011
|
+
data: { effectiveness, policyDecision: effectivenessDecision },
|
|
2012
|
+
});
|
|
893
2013
|
}
|
|
894
|
-
const blockingDecision = manifest.policyDecisions?.find(
|
|
2014
|
+
const blockingDecision = manifest.policyDecisions?.find(
|
|
2015
|
+
(item) => item.action === "block" || item.action === "escalate",
|
|
2016
|
+
);
|
|
895
2017
|
if (failed) {
|
|
896
|
-
manifest = updateRunStatus(
|
|
2018
|
+
manifest = updateRunStatus(
|
|
2019
|
+
manifest,
|
|
2020
|
+
"failed",
|
|
2021
|
+
`Failed at task '${failed.id}'.`,
|
|
2022
|
+
);
|
|
897
2023
|
} else if (waiting) {
|
|
898
|
-
manifest = updateRunStatus(
|
|
2024
|
+
manifest = updateRunStatus(
|
|
2025
|
+
manifest,
|
|
2026
|
+
"blocked",
|
|
2027
|
+
`Waiting for response to task '${waiting.id}'.`,
|
|
2028
|
+
);
|
|
899
2029
|
} else if (running) {
|
|
900
|
-
manifest = updateRunStatus(
|
|
2030
|
+
manifest = updateRunStatus(
|
|
2031
|
+
manifest,
|
|
2032
|
+
"blocked",
|
|
2033
|
+
`Task '${running.id}' is still running.`,
|
|
2034
|
+
);
|
|
901
2035
|
} else if (effectiveness.severity === "failed") {
|
|
902
|
-
manifest = updateRunStatus(
|
|
2036
|
+
manifest = updateRunStatus(
|
|
2037
|
+
manifest,
|
|
2038
|
+
"failed",
|
|
2039
|
+
effectivenessDecision?.message ?? "Run effectiveness guard failed.",
|
|
2040
|
+
);
|
|
903
2041
|
} else if (effectiveness.severity === "blocked") {
|
|
904
|
-
manifest = updateRunStatus(
|
|
2042
|
+
manifest = updateRunStatus(
|
|
2043
|
+
manifest,
|
|
2044
|
+
"blocked",
|
|
2045
|
+
effectivenessDecision?.message ??
|
|
2046
|
+
"Run effectiveness guard blocked completion.",
|
|
2047
|
+
);
|
|
905
2048
|
} else if (blockingDecision) {
|
|
906
|
-
manifest = updateRunStatus(
|
|
2049
|
+
manifest = updateRunStatus(
|
|
2050
|
+
manifest,
|
|
2051
|
+
"blocked",
|
|
2052
|
+
blockingDecision.message,
|
|
2053
|
+
);
|
|
907
2054
|
} else {
|
|
908
|
-
manifest = updateRunStatus(
|
|
2055
|
+
manifest = updateRunStatus(
|
|
2056
|
+
manifest,
|
|
2057
|
+
"completed",
|
|
2058
|
+
input.executeWorkers
|
|
2059
|
+
? "Team workflow completed."
|
|
2060
|
+
: "Team workflow scaffold completed without launching child workers.",
|
|
2061
|
+
);
|
|
909
2062
|
}
|
|
910
|
-
manifest = writeProgress(
|
|
2063
|
+
manifest = writeProgress(
|
|
2064
|
+
manifest,
|
|
2065
|
+
tasks,
|
|
2066
|
+
"team-runner",
|
|
2067
|
+
input.executeWorkers,
|
|
2068
|
+
input.runtimeConfig,
|
|
2069
|
+
);
|
|
911
2070
|
await saveRunManifestAsync(manifest);
|
|
912
2071
|
const usage = aggregateUsage(tasks);
|
|
913
2072
|
const summaryArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
@@ -927,17 +2086,28 @@ tasks = mergeResult.resultTasks;
|
|
|
927
2086
|
...tasks.map(formatTaskProgress),
|
|
928
2087
|
"",
|
|
929
2088
|
"## Effectiveness",
|
|
930
|
-
...runEffectivenessLines(
|
|
2089
|
+
...runEffectivenessLines(
|
|
2090
|
+
manifest,
|
|
2091
|
+
tasks,
|
|
2092
|
+
input.executeWorkers,
|
|
2093
|
+
input.runtimeConfig,
|
|
2094
|
+
),
|
|
931
2095
|
"",
|
|
932
2096
|
"## Policy decisions",
|
|
933
|
-
...(manifest.policyDecisions?.length
|
|
2097
|
+
...(manifest.policyDecisions?.length
|
|
2098
|
+
? summarizePolicyDecisions(manifest.policyDecisions)
|
|
2099
|
+
: ["- (none)"]),
|
|
934
2100
|
"",
|
|
935
2101
|
].join("\n"),
|
|
936
2102
|
});
|
|
937
2103
|
// Build the complete manifest BEFORE acquiring the lock so the artifacts array
|
|
938
2104
|
// is already incorporated into the manifest object that will be atomically written.
|
|
939
2105
|
// This prevents crash-between-mutation-and-lock from leaving inconsistent state.
|
|
940
|
-
const finalManifest = {
|
|
2106
|
+
const finalManifest = {
|
|
2107
|
+
...manifest,
|
|
2108
|
+
updatedAt: new Date().toISOString(),
|
|
2109
|
+
artifacts: [...manifest.artifacts, summaryArtifact],
|
|
2110
|
+
};
|
|
941
2111
|
// Joint atomic save: wrap manifest + tasks in a single run lock so they are
|
|
942
2112
|
// written together or not at all. Crash between separate saveRunManifestAsync
|
|
943
2113
|
// and saveRunTasksAsync calls could leave manifest/tasks.json out of sync.
|
|
@@ -955,7 +2125,9 @@ tasks = mergeResult.resultTasks;
|
|
|
955
2125
|
// health feature) AND created junk dirs that the recursive state watcher then
|
|
956
2126
|
// attached extra inotify watches to. Fix: compute the real crew root (3 up)
|
|
957
2127
|
// and make HEALTH_DIR relative to it.
|
|
958
|
-
const crewRoot = path.dirname(
|
|
2128
|
+
const crewRoot = path.dirname(
|
|
2129
|
+
path.dirname(path.dirname(finalManifest.stateRoot)),
|
|
2130
|
+
);
|
|
959
2131
|
const healthStore = new HealthStore(crewRoot);
|
|
960
2132
|
healthStore.saveSnapshot({
|
|
961
2133
|
runId: finalManifest.runId,
|