pi-crew 0.9.14 → 0.9.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,48 +1,117 @@
1
1
  import * as fs from "node:fs";
2
2
  import * as path from "node:path";
3
3
  import type { AgentConfig } from "../agents/agent-config.ts";
4
- import type { CrewLimitsConfig, CrewRuntimeConfig, CrewReliabilityConfig } from "../config/config.ts";
5
- import type { CrewRuntimeCapabilities } from "./runtime-resolver.ts";
6
- import type { CrewRuntimeKind } from "./crew-agent-runtime.ts";
7
- import { resolveTaskRuntimeKind } from "./runtime-policy.ts";
4
+ import type {
5
+ CrewLimitsConfig,
6
+ CrewReliabilityConfig,
7
+ CrewRuntimeConfig,
8
+ } from "../config/config.ts";
9
+ import { appendHookEvent, executeHook } from "../hooks/registry.ts";
10
+ import {
11
+ childCorrelation,
12
+ withCorrelation,
13
+ } from "../observability/correlation.ts";
14
+ import type { MetricRegistry } from "../observability/metric-registry.ts";
15
+ import { PluginRegistry } from "../plugins/plugin-registry.ts";
16
+ import {
17
+ NextJsPlugin,
18
+ VitePlugin,
19
+ VitestPlugin,
20
+ } from "../plugins/plugins/index.ts";
8
21
  import { writeArtifact } from "../state/artifact-store.ts";
9
- import { executeHook, appendHookEvent } from "../hooks/registry.ts";
10
- import { appendEvent, appendEventAsync, appendEventFireAndForget } from "../state/event-log.ts";
11
- import type { TeamConfig } from "../teams/team-config.ts";
12
- import type { ArtifactDescriptor, PolicyDecision, TeamRunManifest, TaskAttemptState, TeamTaskState } from "../state/types.ts";
13
- import { loadRunManifestById, saveRunManifest, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/state-store.ts";
22
+ import {
23
+ appendEvent,
24
+ appendEventAsync,
25
+ appendEventFireAndForget,
26
+ } from "../state/event-log.ts";
27
+ import { HealthStore } from "../state/health-store.ts";
14
28
  import { withRunLock } from "../state/locks.ts";
29
+ import {
30
+ loadRunManifestById,
31
+ saveRunManifest,
32
+ saveRunManifestAsync,
33
+ saveRunTasksAsync,
34
+ updateRunStatus,
35
+ } from "../state/state-store.ts";
36
+ import type {
37
+ ArtifactDescriptor,
38
+ PolicyDecision,
39
+ TaskAttemptState,
40
+ TeamRunManifest,
41
+ TeamTaskState,
42
+ } from "../state/types.ts";
15
43
  import { aggregateUsage, formatUsage } from "../state/usage.ts";
16
- import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
17
- import { evaluateCrewPolicy, summarizePolicyDecisions } from "./policy-engine.ts";
18
- import { buildRecoveryLedger, shouldRerunFailedTask } from "./recovery-recipes.ts";
19
- import { assessGoalAchievement, applyGoalAchievement } from "./goal-achievement.ts";
20
- import { buildTaskGraphIndex, refreshTaskGraphQueues, taskGraphSnapshot } from "./task-graph-scheduler.ts";
21
- import { buildExecutionPlan as buildDagExecutionPlan, getReadyTasks as getDagReadyTasks, type TaskNode } from "./task-graph.ts";
44
+ import type { TeamConfig } from "../teams/team-config.ts";
45
+ import { logInternalError } from "../utils/internal-error.ts";
46
+ import type {
47
+ WorkflowConfig,
48
+ WorkflowStep,
49
+ } from "../workflows/workflow-config.ts";
22
50
  import { checkBranchFreshness } from "../worktree/branch-freshness.ts";
23
- import { aggregateTaskOutputs } from "./task-output-context.ts";
51
+ import {
52
+ buildSyntheticTerminalEvidence,
53
+ CrewCancellationError,
54
+ cancellationReasonFromSignal,
55
+ } from "./cancellation.ts";
56
+ import { resolveBatchConcurrency } from "./concurrency.ts";
24
57
  import { readCrewAgents, saveCrewAgents } from "./crew-agent-records.ts";
25
- import { recordsForMaterializedTasks } from "./task-display.ts";
58
+ import type { CrewRuntimeKind } from "./crew-agent-runtime.ts";
59
+ import { crewHooks } from "./crew-hooks.ts";
60
+ import { appendDeadletter } from "./deadletter.ts";
61
+ import {
62
+ effectivenessPolicyDecision,
63
+ evaluateRunEffectiveness,
64
+ formatRunEffectivenessLines,
65
+ } from "./effectiveness.ts";
66
+ import {
67
+ applyGoalAchievement,
68
+ assessGoalAchievement,
69
+ } from "./goal-achievement.ts";
26
70
  import { deliverGroupJoin, resolveGroupJoinMode } from "./group-join.ts";
27
- import { runTeamTask } from "./task-runner.ts";
28
71
  import { terminateLiveAgentsForRun } from "./live-agent-manager.ts";
29
- import { createWorkflowStateMachine, validatePhasePreconditions, transitionPhase, type PhaseState, type PhaseGuardContext } from "./workflow-state.ts";
30
- import { executeWithRetry, DEFAULT_RETRY_POLICY, type RetryPolicy } from "./retry-executor.ts";
31
- import { appendDeadletter } from "./deadletter.ts";
32
- import type { MetricRegistry } from "../observability/metric-registry.ts";
33
- import { childCorrelation, withCorrelation } from "../observability/correlation.ts";
34
- import { crewHooks } from "./crew-hooks.ts";
35
- import { resolveBatchConcurrency } from "./concurrency.ts";
36
72
  import { mapConcurrent } from "./parallel-utils.ts";
73
+ import {
74
+ evaluateCrewPolicy,
75
+ summarizePolicyDecisions,
76
+ } from "./policy-engine.ts";
77
+ import {
78
+ buildRecoveryLedger,
79
+ shouldRerunFailedTask,
80
+ } from "./recovery-recipes.ts";
81
+ import {
82
+ DEFAULT_RETRY_POLICY,
83
+ executeWithRetry,
84
+ type RetryPolicy,
85
+ } from "./retry-executor.ts";
37
86
  import { permissionForRole } from "./role-permission.ts";
38
- import { registerRunPromise, resolveRunPromise, rejectRunPromise } from "./run-tracker.ts";
87
+ import {
88
+ registerRunPromise,
89
+ rejectRunPromise,
90
+ resolveRunPromise,
91
+ } from "./run-tracker.ts";
92
+ import { resolveTaskRuntimeKind } from "./runtime-policy.ts";
93
+ import type { CrewRuntimeCapabilities } from "./runtime-resolver.ts";
94
+ import { recordsForMaterializedTasks } from "./task-display.ts";
95
+ import {
96
+ buildExecutionPlan as buildDagExecutionPlan,
97
+ getReadyTasks as getDagReadyTasks,
98
+ type TaskNode,
99
+ } from "./task-graph.ts";
100
+ import {
101
+ buildTaskGraphIndex,
102
+ refreshTaskGraphQueues,
103
+ taskGraphSnapshot,
104
+ } from "./task-graph-scheduler.ts";
105
+ import { aggregateTaskOutputs } from "./task-output-context.ts";
106
+ import { runTeamTask } from "./task-runner.ts";
39
107
  import { clearTrackedTaskUsage } from "./usage-tracker.ts";
40
- import { CrewCancellationError, buildSyntheticTerminalEvidence, cancellationReasonFromSignal } from "./cancellation.ts";
41
- import { effectivenessPolicyDecision, evaluateRunEffectiveness, formatRunEffectivenessLines } from "./effectiveness.ts";
42
- import { logInternalError } from "../utils/internal-error.ts";
43
- import { PluginRegistry } from "../plugins/plugin-registry.ts";
44
- import { NextJsPlugin, VitestPlugin, VitePlugin } from "../plugins/plugins/index.ts";
45
- import { HealthStore } from "../state/health-store.ts";
108
+ import {
109
+ createWorkflowStateMachine,
110
+ type PhaseGuardContext,
111
+ type PhaseState,
112
+ transitionPhase,
113
+ validatePhasePreconditions,
114
+ } from "./workflow-state.ts";
46
115
 
47
116
  // Built-in plugin registry for framework awareness.
48
117
  // NOTE: This registry is registered here for future use. The integration
@@ -73,13 +142,17 @@ function startTeamRunHeartbeat(stateRoot: string, runId: string): () => void {
73
142
  // captured manifest.updatedAt once at startup, making the value
74
143
  // permanently stale throughout the run.
75
144
  const now = new Date().toISOString();
76
- fs.writeFileSync(heartbeatPath, JSON.stringify({
77
- pid: process.pid,
78
- at: Date.now(),
79
- runId,
80
- kind: "team-runner",
81
- lastTaskUpdateAt: now,
82
- }), { encoding: "utf-8", mode: 0o600 });
145
+ fs.writeFileSync(
146
+ heartbeatPath,
147
+ JSON.stringify({
148
+ pid: process.pid,
149
+ at: Date.now(),
150
+ runId,
151
+ kind: "team-runner",
152
+ lastTaskUpdateAt: now,
153
+ }),
154
+ { encoding: "utf-8", mode: 0o600 },
155
+ );
83
156
  } catch {
84
157
  // best-effort
85
158
  }
@@ -119,19 +192,39 @@ export interface ExecuteTeamRunInput {
119
192
  }
120
193
 
121
194
  function findStep(workflow: WorkflowConfig, task: TeamTaskState): WorkflowStep {
122
- const step = workflow.steps.find((candidate) => candidate.id === task.stepId);
123
- if (!step) throw new Error(`Workflow step '${task.stepId}' not found for task '${task.id}'.`);
195
+ const step = workflow.steps.find(
196
+ (candidate) => candidate.id === task.stepId,
197
+ );
198
+ if (!step)
199
+ throw new Error(
200
+ `Workflow step '${task.stepId}' not found for task '${task.id}'.`,
201
+ );
124
202
  return step;
125
203
  }
126
204
 
127
205
  function findAgent(agents: AgentConfig[], task: TeamTaskState): AgentConfig {
128
206
  const agent = agents.find((candidate) => candidate.name === task.agent);
129
- if (!agent) throw new Error(`Agent '${task.agent}' not found for task '${task.id}'.`);
207
+ if (!agent)
208
+ throw new Error(
209
+ `Agent '${task.agent}' not found for task '${task.id}'.`,
210
+ );
130
211
  return agent;
131
212
  }
132
213
 
133
214
  function markBlocked(tasks: TeamTaskState[], reason: string): TeamTaskState[] {
134
- return tasks.map((task) => task.status === "queued" ? { ...task, status: "skipped", error: reason, finishedAt: new Date().toISOString(), graph: task.graph ? { ...task.graph, queue: "blocked" } : undefined } : task);
215
+ return tasks.map((task) =>
216
+ task.status === "queued"
217
+ ? {
218
+ ...task,
219
+ status: "skipped",
220
+ error: reason,
221
+ finishedAt: new Date().toISOString(),
222
+ graph: task.graph
223
+ ? { ...task.graph, queue: "blocked" }
224
+ : undefined,
225
+ }
226
+ : task,
227
+ );
135
228
  }
136
229
 
137
230
  function mergeArtifacts(items: ArtifactDescriptor[]): ArtifactDescriptor[] {
@@ -160,37 +253,59 @@ function safeFinishedAt(task: TeamTaskState): number {
160
253
  * and the updated task has a valid finite finishedAt. Malformed finishedAt
161
254
  * should be replaced rather than persisting corruption.
162
255
  */
163
- function isMalformedFinishedAtReplacement(currentTime: number, updatedTime: number): boolean {
256
+ function isMalformedFinishedAtReplacement(
257
+ currentTime: number,
258
+ updatedTime: number,
259
+ ): boolean {
164
260
  return !Number.isFinite(currentTime) && Number.isFinite(updatedTime);
165
261
  }
166
262
 
167
- function shouldMergeTaskUpdate(current: TeamTaskState, updated: TeamTaskState): boolean {
263
+ function shouldMergeTaskUpdate(
264
+ current: TeamTaskState,
265
+ updated: TeamTaskState,
266
+ ): boolean {
168
267
  // Parallel workers receive the same input snapshot. A later result may still
169
268
  // contain stale queued/running copies of tasks that another worker already
170
269
  // completed. Never let those stale snapshots regress durable task state.
171
- if (current.status === "waiting" && updated.status === "running") return false;
270
+ if (current.status === "waiting" && updated.status === "running")
271
+ return false;
172
272
  // Block terminal→non-terminal transitions (e.g. completed→running).
173
273
  // A task that has reached a terminal state must not be resurrected.
174
274
  const currentIsTerminal = !isNonTerminalTaskStatus(current.status);
175
275
  const updatedIsNonTerminal = isNonTerminalTaskStatus(updated.status);
176
276
  if (currentIsTerminal && updatedIsNonTerminal) return false;
177
277
  // Explicitly block completed↔needs_attention terminal-to-terminal transitions.
178
- // Both are success terminal states used interchangeably; stale worker updates must
179
- // not cause a completed task to appear as needs_attention or vice versa.
180
- if (current.status === "completed" && updated.status === "needs_attention") return false;
181
- if (current.status === "needs_attention" && updated.status === "completed") return false;
278
+ // Both are success terminal states used interchangeably; stale worker updates must
279
+ // not cause a completed task to appear as needs_attention or vice versa.
280
+ if (current.status === "completed" && updated.status === "needs_attention")
281
+ return false;
282
+ if (current.status === "needs_attention" && updated.status === "completed")
283
+ return false;
182
284
  // Explicitly block failed→completed resurrection. Both statuses are terminal,
183
285
  // but completed is the success terminal state and should not be reachable from
184
286
  // failed via a stale merge. The check above only guards non-terminal→terminal.
185
- if (current.status === "failed" && updated.status === "completed") return false;
287
+ if (current.status === "failed" && updated.status === "completed")
288
+ return false;
186
289
  // Guard: when current is "running" but has resultArtifact (another worker already
187
290
  // completed it), a stale updated with status="running" and no resultArtifact
188
291
  // must not overwrite the actual completed state.
189
- if (current.status === updated.status && updated.status === "running" && Boolean(current.resultArtifact) && !updated.resultArtifact) return false;
292
+ if (
293
+ current.status === updated.status &&
294
+ updated.status === "running" &&
295
+ current.resultArtifact &&
296
+ !updated.resultArtifact
297
+ )
298
+ return false;
190
299
  // Guard: when current is "completed" and has resultArtifact but updated is also
191
300
  // "completed" without resultArtifact, block the stale update from overwriting
192
301
  // a task that successfully produced output.
193
- if (current.status === updated.status && current.status === "completed" && Boolean(current.resultArtifact) && !updated.resultArtifact) return false;
302
+ if (
303
+ current.status === updated.status &&
304
+ current.status === "completed" &&
305
+ current.resultArtifact &&
306
+ !updated.resultArtifact
307
+ )
308
+ return false;
194
309
  // Prevent a stale completed task from overwriting a fresher one.
195
310
  // Restructure to handle undefined current.finishedAt as a special case:
196
311
  // - undefined current + valid updated: allow the update
@@ -203,7 +318,9 @@ if (current.status === "needs_attention" && updated.status === "completed") retu
203
318
  // Malformed finishedAt (NaN) is treated as Infinity — invalid state should be
204
319
  // replaced rather than persisting corruption. Log warning for visibility.
205
320
  if (!Number.isFinite(currentTime)) {
206
- console.warn(`[team-runner] Task ${current.id} has malformed finishedAt: ${current.finishedAt}`);
321
+ console.warn(
322
+ `[team-runner] Task ${current.id} has malformed finishedAt: ${current.finishedAt}`,
323
+ );
207
324
  }
208
325
  if (isMalformedFinishedAtReplacement(currentTime, updatedTime)) {
209
326
  return true;
@@ -213,29 +330,35 @@ if (current.status === "needs_attention" && updated.status === "completed") retu
213
330
  // Block if updated is trying to establish a terminal status without a finishedAt
214
331
  // timestamp. Heartbeat-only updates (status='running', no finishedAt) are
215
332
  // allowed if heartbeat has changed (checked separately in hasMeaningfulUpdate).
216
- if (!updated.finishedAt && !isNonTerminalTaskStatus(updated.status)) return false;
333
+ if (!updated.finishedAt && !isNonTerminalTaskStatus(updated.status))
334
+ return false;
217
335
  // Explicitly enumerate all fields that constitute a meaningful update so that
218
- // adding a new important field requires updating this list (rather than silently
219
- // losing data if a field is forgotten in the boolean OR chain below).
220
- const hasMeaningfulUpdate =
221
- updated.status !== current.status ||
222
- updated.finishedAt !== current.finishedAt ||
223
- updated.startedAt !== current.startedAt ||
224
- Boolean(updated.resultArtifact) !== Boolean(current.resultArtifact) ||
225
- (Boolean(updated.resultArtifact) && updated.resultArtifact !== current.resultArtifact) ||
226
- Boolean(updated.error) ||
227
- Boolean(updated.modelAttempts?.length) ||
228
- Boolean(updated.usage) ||
229
- Boolean(updated.attempts?.length) ||
230
- updated.heartbeat?.lastSeenAt !== current.heartbeat?.lastSeenAt ||
231
- updated.jsonEvents !== current.jsonEvents ||
232
- updated.agentProgress?.lastActivityAt !== current.agentProgress?.lastActivityAt;
233
- return hasMeaningfulUpdate;
336
+ // adding a new important field requires updating this list (rather than silently
337
+ // losing data if a field is forgotten in the boolean OR chain below).
338
+ const hasMeaningfulUpdate =
339
+ updated.status !== current.status ||
340
+ updated.finishedAt !== current.finishedAt ||
341
+ updated.startedAt !== current.startedAt ||
342
+ Boolean(updated.resultArtifact) !== Boolean(current.resultArtifact) ||
343
+ (Boolean(updated.resultArtifact) &&
344
+ updated.resultArtifact !== current.resultArtifact) ||
345
+ Boolean(updated.error) ||
346
+ Boolean(updated.modelAttempts?.length) ||
347
+ Boolean(updated.usage) ||
348
+ Boolean(updated.attempts?.length) ||
349
+ updated.heartbeat?.lastSeenAt !== current.heartbeat?.lastSeenAt ||
350
+ updated.jsonEvents !== current.jsonEvents ||
351
+ updated.agentProgress?.lastActivityAt !==
352
+ current.agentProgress?.lastActivityAt;
353
+ return hasMeaningfulUpdate;
234
354
  }
235
355
 
236
356
  // H4 fix: rename to descriptive name. Kept __test__ as alias for backward
237
357
  // compat test imports.
238
- export function mergeTaskUpdatesPreservingTerminal(base: TeamTaskState[], results: Array<{ tasks: TeamTaskState[] }>): TeamTaskState[] {
358
+ export function mergeTaskUpdatesPreservingTerminal(
359
+ base: TeamTaskState[],
360
+ results: Array<{ tasks: TeamTaskState[] }>,
361
+ ): TeamTaskState[] {
239
362
  let merged = base;
240
363
  for (const result of results) {
241
364
  for (const updated of result.tasks) {
@@ -245,15 +368,21 @@ export function mergeTaskUpdatesPreservingTerminal(base: TeamTaskState[], result
245
368
  // Log skipped merges for visibility into rejected parallel updates.
246
369
  // In distributed systems with parallel workers, rejected merges may
247
370
  // indicate bugs (wrong status, timestamp corruption) if they accumulate.
248
- console.debug("[team-runner] Skipping stale merge for task", updated.id, {
249
- currentStatus: current.status,
250
- updatedStatus: updated.status,
251
- currentFinishedAt: current.finishedAt,
252
- updatedFinishedAt: updated.finishedAt,
253
- });
371
+ console.debug(
372
+ "[team-runner] Skipping stale merge for task",
373
+ updated.id,
374
+ {
375
+ currentStatus: current.status,
376
+ updatedStatus: updated.status,
377
+ currentFinishedAt: current.finishedAt,
378
+ updatedFinishedAt: updated.finishedAt,
379
+ },
380
+ );
254
381
  continue;
255
382
  }
256
- merged = merged.map((task) => task.id === updated.id ? updated : task);
383
+ merged = merged.map((task) =>
384
+ task.id === updated.id ? updated : task,
385
+ );
257
386
  }
258
387
  }
259
388
  return refreshTaskGraphQueues(merged);
@@ -263,20 +392,43 @@ export const __test__mergeTaskUpdates = mergeTaskUpdatesPreservingTerminal;
263
392
 
264
393
  // 2.8: adaptive-plan parsing/repair/injection moved to src/runtime/adaptive-plan.ts.
265
394
  // Re-export the test-only helpers so existing test imports still resolve.
266
- export { __test__parseAdaptivePlan, __test__repairAdaptivePlan } from "./adaptive-plan.ts";
395
+ export {
396
+ __test__parseAdaptivePlan,
397
+ __test__repairAdaptivePlan,
398
+ } from "./adaptive-plan.ts";
399
+
267
400
  import { injectAdaptivePlanIfReady } from "./adaptive-plan.ts";
268
401
 
269
402
  function formatTaskProgress(task: TeamTaskState): string {
270
403
  return `- ${task.id}: ${task.status} (${task.role} -> ${task.agent})${task.taskPacket ? ` scope=${task.taskPacket.scope}` : ""}${task.verification ? ` green=${task.verification.observedGreenLevel}/${task.verification.requiredGreenLevel}` : ""}${task.error ? ` - ${task.error}` : ""}`;
271
404
  }
272
405
 
273
- function runEffectivenessLines(manifest: TeamRunManifest, tasks: TeamTaskState[], executeWorkers: boolean, runtimeConfig?: CrewRuntimeConfig): string[] {
274
- return formatRunEffectivenessLines(evaluateRunEffectiveness({ manifest, tasks, executeWorkers, runtimeConfig }));
406
+ function runEffectivenessLines(
407
+ manifest: TeamRunManifest,
408
+ tasks: TeamTaskState[],
409
+ executeWorkers: boolean,
410
+ runtimeConfig?: CrewRuntimeConfig,
411
+ ): string[] {
412
+ return formatRunEffectivenessLines(
413
+ evaluateRunEffectiveness({
414
+ manifest,
415
+ tasks,
416
+ executeWorkers,
417
+ runtimeConfig,
418
+ }),
419
+ );
275
420
  }
276
421
 
277
- function writeProgress(manifest: TeamRunManifest, tasks: TeamTaskState[], producer: string, executeWorkers = true, runtimeConfig?: CrewRuntimeConfig): TeamRunManifest {
422
+ function writeProgress(
423
+ manifest: TeamRunManifest,
424
+ tasks: TeamTaskState[],
425
+ producer: string,
426
+ executeWorkers = true,
427
+ runtimeConfig?: CrewRuntimeConfig,
428
+ ): TeamRunManifest {
278
429
  const counts = new Map<string, number>();
279
- for (const task of tasks) counts.set(task.status, (counts.get(task.status) ?? 0) + 1);
430
+ for (const task of tasks)
431
+ counts.set(task.status, (counts.get(task.status) ?? 0) + 1);
280
432
  const queue = taskGraphSnapshot(tasks);
281
433
  const progress = writeArtifact(manifest.artifactsRoot, {
282
434
  kind: "progress",
@@ -296,14 +448,39 @@ function writeProgress(manifest: TeamRunManifest, tasks: TeamTaskState[], produc
296
448
  ...tasks.map(formatTaskProgress),
297
449
  "",
298
450
  "## Effectiveness",
299
- ...runEffectivenessLines(manifest, tasks, executeWorkers, runtimeConfig),
451
+ ...runEffectivenessLines(
452
+ manifest,
453
+ tasks,
454
+ executeWorkers,
455
+ runtimeConfig,
456
+ ),
300
457
  "",
301
458
  ].join("\n"),
302
459
  });
303
- return { ...manifest, updatedAt: new Date().toISOString(), artifacts: [...manifest.artifacts.filter((artifact) => !(artifact.kind === "progress" && artifact.path === progress.path)), progress].filter((artifact, index, self) => self.findIndex((a) => a.path === artifact.path) === index) };
460
+ return {
461
+ ...manifest,
462
+ updatedAt: new Date().toISOString(),
463
+ artifacts: [
464
+ ...manifest.artifacts.filter(
465
+ (artifact) =>
466
+ !(
467
+ artifact.kind === "progress" &&
468
+ artifact.path === progress.path
469
+ ),
470
+ ),
471
+ progress,
472
+ ].filter(
473
+ (artifact, index, self) =>
474
+ self.findIndex((a) => a.path === artifact.path) === index,
475
+ ),
476
+ };
304
477
  }
305
478
 
306
- function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?: CrewLimitsConfig): TeamRunManifest {
479
+ function applyPolicy(
480
+ manifest: TeamRunManifest,
481
+ tasks: TeamTaskState[],
482
+ limits?: CrewLimitsConfig,
483
+ ): TeamRunManifest {
307
484
  const branchFreshness = checkBranchFreshness(manifest.cwd);
308
485
  const branchArtifact = writeArtifact(manifest.artifactsRoot, {
309
486
  kind: "metadata",
@@ -311,8 +488,15 @@ function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?:
311
488
  producer: "branch-freshness",
312
489
  content: `${JSON.stringify(branchFreshness, null, 2)}\n`,
313
490
  });
314
- let decisions: PolicyDecision[] = evaluateCrewPolicy({ manifest, tasks, limits });
315
- if (branchFreshness.status === "stale" || branchFreshness.status === "diverged") {
491
+ let decisions: PolicyDecision[] = evaluateCrewPolicy({
492
+ manifest,
493
+ tasks,
494
+ limits,
495
+ });
496
+ if (
497
+ branchFreshness.status === "stale" ||
498
+ branchFreshness.status === "diverged"
499
+ ) {
316
500
  const branchDecision: PolicyDecision = {
317
501
  action: "notify",
318
502
  reason: "branch_stale",
@@ -320,7 +504,12 @@ function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?:
320
504
  createdAt: new Date().toISOString(),
321
505
  };
322
506
  decisions = [...decisions, branchDecision];
323
- appendEvent(manifest.eventsPath, { type: "branch.stale", runId: manifest.runId, message: branchFreshness.message, data: { branchFreshness } });
507
+ appendEvent(manifest.eventsPath, {
508
+ type: "branch.stale",
509
+ runId: manifest.runId,
510
+ message: branchFreshness.message,
511
+ data: { branchFreshness },
512
+ });
324
513
  }
325
514
  const policyArtifact = writeArtifact(manifest.artifactsRoot, {
326
515
  kind: "metadata",
@@ -335,12 +524,57 @@ function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?:
335
524
  producer: "recovery-engine",
336
525
  content: `${JSON.stringify(recoveryLedger, null, 2)}\n`,
337
526
  });
338
- for (const item of decisions) appendEvent(manifest.eventsPath, { type: item.action === "escalate" ? "policy.escalated" : "policy.action", runId: manifest.runId, taskId: item.taskId, message: item.message, data: { action: item.action, reason: item.reason } });
339
- for (const item of recoveryLedger.entries) appendEvent(manifest.eventsPath, { type: item.state === "escalation_required" ? "recovery.escalated" : "recovery.attempted", runId: manifest.runId, taskId: item.taskId, message: item.message, data: { scenario: item.scenario, steps: item.steps, attempt: item.attempt, state: item.state } });
340
- return { ...manifest, updatedAt: new Date().toISOString(), policyDecisions: decisions, artifacts: [...manifest.artifacts.filter((artifact) => !(artifact.kind === "metadata" && (artifact.path.endsWith("policy-decisions.json") || artifact.path.endsWith("recovery-ledger.json") || artifact.path.endsWith("branch-freshness.json")))), branchArtifact, policyArtifact, recoveryArtifact] };
527
+ for (const item of decisions)
528
+ appendEvent(manifest.eventsPath, {
529
+ type:
530
+ item.action === "escalate"
531
+ ? "policy.escalated"
532
+ : "policy.action",
533
+ runId: manifest.runId,
534
+ taskId: item.taskId,
535
+ message: item.message,
536
+ data: { action: item.action, reason: item.reason },
537
+ });
538
+ for (const item of recoveryLedger.entries)
539
+ appendEvent(manifest.eventsPath, {
540
+ type:
541
+ item.state === "escalation_required"
542
+ ? "recovery.escalated"
543
+ : "recovery.attempted",
544
+ runId: manifest.runId,
545
+ taskId: item.taskId,
546
+ message: item.message,
547
+ data: {
548
+ scenario: item.scenario,
549
+ steps: item.steps,
550
+ attempt: item.attempt,
551
+ state: item.state,
552
+ },
553
+ });
554
+ return {
555
+ ...manifest,
556
+ updatedAt: new Date().toISOString(),
557
+ policyDecisions: decisions,
558
+ artifacts: [
559
+ ...manifest.artifacts.filter(
560
+ (artifact) =>
561
+ !(
562
+ artifact.kind === "metadata" &&
563
+ (artifact.path.endsWith("policy-decisions.json") ||
564
+ artifact.path.endsWith("recovery-ledger.json") ||
565
+ artifact.path.endsWith("branch-freshness.json"))
566
+ ),
567
+ ),
568
+ branchArtifact,
569
+ policyArtifact,
570
+ recoveryArtifact,
571
+ ],
572
+ };
341
573
  }
342
574
 
343
- function retryPolicyFromConfig(config: CrewReliabilityConfig | undefined): RetryPolicy {
575
+ function retryPolicyFromConfig(
576
+ config: CrewReliabilityConfig | undefined,
577
+ ): RetryPolicy {
344
578
  return { ...DEFAULT_RETRY_POLICY, ...(config?.retryPolicy ?? {}) };
345
579
  }
346
580
 
@@ -350,15 +584,25 @@ function retryPolicyFromConfig(config: CrewReliabilityConfig | undefined): Retry
350
584
  * automatically. Previously opt-in, which left the entire retry+recovery stack dormant.
351
585
  * Exported for unit testing.
352
586
  */
353
- export function shouldUseRetry(reliability: CrewReliabilityConfig | undefined): boolean {
587
+ export function shouldUseRetry(
588
+ reliability: CrewReliabilityConfig | undefined,
589
+ ): boolean {
354
590
  return reliability?.autoRetry !== false;
355
591
  }
356
592
 
357
- function failedTaskFrom(result: { tasks: TeamTaskState[] }, taskId: string): TeamTaskState | undefined {
358
- return result.tasks.find((item) => item.id === taskId && item.status === "failed");
593
+ function failedTaskFrom(
594
+ result: { tasks: TeamTaskState[] },
595
+ taskId: string,
596
+ ): TeamTaskState | undefined {
597
+ return result.tasks.find(
598
+ (item) => item.id === taskId && item.status === "failed",
599
+ );
359
600
  }
360
601
 
361
- function requiresPlanApproval(_workflow: WorkflowConfig, runtimeConfig: CrewRuntimeConfig | undefined): boolean {
602
+ function requiresPlanApproval(
603
+ _workflow: WorkflowConfig,
604
+ runtimeConfig: CrewRuntimeConfig | undefined,
605
+ ): boolean {
362
606
  // ROADMAP T1.2: plan-level HITL applies to ANY workflow when
363
607
  // config.runtime.requirePlanApproval === true (not just 'implementation').
364
608
  // The gate fires at the read-only → mutating (plan → execute) boundary.
@@ -366,19 +610,31 @@ function requiresPlanApproval(_workflow: WorkflowConfig, runtimeConfig: CrewRunt
366
610
  }
367
611
 
368
612
  function isPlanApprovalPending(manifest: TeamRunManifest): boolean {
369
- return manifest.planApproval?.required === true && manifest.planApproval.status === "pending";
613
+ return (
614
+ manifest.planApproval?.required === true &&
615
+ manifest.planApproval.status === "pending"
616
+ );
370
617
  }
371
618
 
372
619
  function isMutatingTask(task: TeamTaskState): boolean {
373
620
  return permissionForRole(task.role) !== "read_only";
374
621
  }
375
622
 
376
- function ensurePlanApprovalRequested(manifest: TeamRunManifest, tasks: TeamTaskState[]): TeamRunManifest {
623
+ function ensurePlanApprovalRequested(
624
+ manifest: TeamRunManifest,
625
+ tasks: TeamTaskState[],
626
+ ): TeamRunManifest {
377
627
  if (manifest.planApproval) return manifest;
378
- const assessTask = tasks.find((task) => task.stepId === "assess" && task.status === "completed");
628
+ const assessTask = tasks.find(
629
+ (task) => task.stepId === "assess" && task.status === "completed",
630
+ );
379
631
  // ROADMAP T1.2: for non-adaptive workflows, fall back to the most recent
380
632
  // completed read-only (planning) task as the plan reference.
381
- const planTask = assessTask ?? [...tasks].reverse().find((t) => t.status === "completed" && !isMutatingTask(t));
633
+ const planTask =
634
+ assessTask ??
635
+ [...tasks]
636
+ .reverse()
637
+ .find((t) => t.status === "completed" && !isMutatingTask(t));
382
638
  const now = new Date().toISOString();
383
639
  const updated: TeamRunManifest = {
384
640
  ...manifest,
@@ -393,16 +649,43 @@ function ensurePlanApprovalRequested(manifest: TeamRunManifest, tasks: TeamTaskS
393
649
  },
394
650
  };
395
651
  saveRunManifest(updated);
396
- appendEvent(updated.eventsPath, { type: "plan.approval_required", runId: updated.runId, taskId: planTask?.id, message: "Plan requires explicit approval before mutating tasks run. Use: team api op=approve-plan runId=...", data: { planArtifactPath: planTask?.resultArtifact?.path } });
652
+ appendEvent(updated.eventsPath, {
653
+ type: "plan.approval_required",
654
+ runId: updated.runId,
655
+ taskId: planTask?.id,
656
+ message:
657
+ "Plan requires explicit approval before mutating tasks run. Use: team api op=approve-plan runId=...",
658
+ data: { planArtifactPath: planTask?.resultArtifact?.path },
659
+ });
397
660
  return updated;
398
661
  }
399
662
 
400
- function cancelPlanTasks(tasks: TeamTaskState[], reason: string): TeamTaskState[] {
401
- return tasks.map((task) => task.status === "queued" || task.status === "running" || task.status === "waiting" ? { ...task, status: "cancelled", finishedAt: new Date().toISOString(), error: reason, graph: task.graph ? { ...task.graph, queue: "done" } : undefined } : task);
663
+ function cancelPlanTasks(
664
+ tasks: TeamTaskState[],
665
+ reason: string,
666
+ ): TeamTaskState[] {
667
+ return tasks.map((task) =>
668
+ task.status === "queued" ||
669
+ task.status === "running" ||
670
+ task.status === "waiting"
671
+ ? {
672
+ ...task,
673
+ status: "cancelled",
674
+ finishedAt: new Date().toISOString(),
675
+ error: reason,
676
+ graph: task.graph
677
+ ? { ...task.graph, queue: "done" }
678
+ : undefined,
679
+ }
680
+ : task,
681
+ );
402
682
  }
403
683
 
404
684
  function hasPendingMutatingAdaptiveTask(tasks: TeamTaskState[]): boolean {
405
- return tasks.some((task) => task.status === "queued" && task.adaptive && isMutatingTask(task));
685
+ return tasks.some(
686
+ (task) =>
687
+ task.status === "queued" && task.adaptive && isMutatingTask(task),
688
+ );
406
689
  }
407
690
 
408
691
  /**
@@ -410,9 +693,15 @@ function hasPendingMutatingAdaptiveTask(tasks: TeamTaskState[]): boolean {
410
693
  * Fires when there are pending mutating tasks whose prerequisites (read-only
411
694
  * tasks) have completed — i.e. the plan→execute boundary.
412
695
  */
413
- export function hasPendingMutatingTaskAtBoundary(tasks: TeamTaskState[]): boolean {
414
- const hasCompletedReadOnly = tasks.some((t) => t.status === "completed" && !isMutatingTask(t));
415
- const hasPendingMutating = tasks.some((t) => t.status === "queued" && isMutatingTask(t));
696
+ export function hasPendingMutatingTaskAtBoundary(
697
+ tasks: TeamTaskState[],
698
+ ): boolean {
699
+ const hasCompletedReadOnly = tasks.some(
700
+ (t) => t.status === "completed" && !isMutatingTask(t),
701
+ );
702
+ const hasPendingMutating = tasks.some(
703
+ (t) => t.status === "queued" && isMutatingTask(t),
704
+ );
416
705
  return hasCompletedReadOnly && hasPendingMutating;
417
706
  }
418
707
 
@@ -421,7 +710,10 @@ export function hasPendingMutatingTaskAtBoundary(tasks: TeamTaskState[]): boolea
421
710
  * execution planning. If so, build an execution plan and use `getDagReadyTasks`
422
711
  * to augment the ready-set selection.
423
712
  */
424
- function dagReadyTaskIds(tasks: TeamTaskState[], completedIds: Set<string>): string[] | null {
713
+ function dagReadyTaskIds(
714
+ tasks: TeamTaskState[],
715
+ completedIds: Set<string>,
716
+ ): string[] | null {
425
717
  const hasExplicitDeps = tasks.some((t) => t.dependsOn.length > 0);
426
718
  if (!hasExplicitDeps) return null;
427
719
  // FIX (goal-wrap runtime test): task.dependsOn stores STEP IDs (e.g. "execute"), not
@@ -444,9 +736,53 @@ function dagReadyTaskIds(tasks: TeamTaskState[], completedIds: Set<string>): str
444
736
  return getDagReadyTasks(plan, completedIds);
445
737
  }
446
738
 
447
- export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
739
+ export async function executeTeamRun(
740
+ input: ExecuteTeamRunInput,
741
+ ): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
448
742
  const workflow = input.workflow;
449
- let manifest = updateRunStatus(input.manifest, "running", input.executeWorkers ? "Executing team workflow." : "Creating workflow prompts and placeholder results.");
743
+
744
+ // DEFENSE-IN-DEPTH (advisory-only since v0.9.15): re-validate topology here in
745
+ // case a caller bypassed the extension-layer handleRun guard. The extension
746
+ // layer logs an advisory note and proceeds; this defense-in-depth does the same
747
+ // for direct API callers (CLI, tests, scheduler). Never blocks.
748
+ // Skip for synthetic direct-agent workflows (filePath="<generated>").
749
+ if (workflow.filePath !== "<generated>") {
750
+ // LAZY: defer preflight-validator import until the defense-in-depth guard actually runs.
751
+ const { validateWorkflowUsage } = await import(
752
+ "../workflows/preflight-validator.ts"
753
+ );
754
+ const preflight = validateWorkflowUsage(workflow, {
755
+ force: input.reliability?.forcePreflight === true,
756
+ });
757
+ if (
758
+ preflight.level === "warn" ||
759
+ preflight.level === "note" ||
760
+ preflight.level === "info"
761
+ ) {
762
+ const icon =
763
+ preflight.level === "warn"
764
+ ? "⚠️ "
765
+ : preflight.level === "note"
766
+ ? "✅ "
767
+ : "ℹ️ ";
768
+ console.warn(
769
+ `${icon}[team-runner.preflight] ${preflight.level.toUpperCase()}: ${preflight.message} (workflow=${workflow.name})`,
770
+ );
771
+ if (preflight.suggestion) {
772
+ console.warn(
773
+ `[team-runner.preflight] → ${preflight.suggestion}`,
774
+ );
775
+ }
776
+ }
777
+ }
778
+
779
+ let manifest = updateRunStatus(
780
+ input.manifest,
781
+ "running",
782
+ input.executeWorkers
783
+ ? "Executing team workflow."
784
+ : "Creating workflow prompts and placeholder results.",
785
+ );
450
786
 
451
787
  void registerRunPromise(manifest.runId);
452
788
 
@@ -455,7 +791,10 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
455
791
  // (NO_PID_HEARTBEAT_STALE_MS). Previously only sub-task runners wrote
456
792
  // heartbeats; the team-level run had no heartbeat, so any multi-phase
457
793
  // workflow lasting >5min was marked stale and cancelled.
458
- const stopTeamHeartbeat = startTeamRunHeartbeat(manifest.stateRoot, manifest.runId);
794
+ const stopTeamHeartbeat = startTeamRunHeartbeat(
795
+ manifest.stateRoot,
796
+ manifest.runId,
797
+ );
459
798
 
460
799
  const cleanupUsage = (): void => {
461
800
  for (const task of input.tasks) clearTrackedTaskUsage(task.id);
@@ -468,44 +807,101 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
468
807
  // (and/or had a failed task) is a false-green. We expose goalAchieved on the
469
808
  // manifest + emit an event so the lie is never silent, and downgrade status
470
809
  // to "failed" only when a failed task corroborates it (conservative).
471
- const gaAssessment = assessGoalAchievement(result.manifest, result.tasks, workflow);
810
+ const gaAssessment = assessGoalAchievement(
811
+ result.manifest,
812
+ result.tasks,
813
+ workflow,
814
+ );
472
815
  const gaApplied = applyGoalAchievement(result.manifest, gaAssessment);
473
816
  if (gaApplied.manifest !== result.manifest) {
474
817
  result.manifest = gaApplied.manifest;
475
818
  try {
476
819
  saveRunManifest(result.manifest);
477
820
  } catch (persistError) {
478
- logInternalError("team-runner.goalAchievement.persist", persistError instanceof Error ? persistError : new Error(String(persistError)), `runId=${manifest.runId}`);
821
+ logInternalError(
822
+ "team-runner.goalAchievement.persist",
823
+ persistError instanceof Error
824
+ ? persistError
825
+ : new Error(String(persistError)),
826
+ `runId=${manifest.runId}`,
827
+ );
479
828
  }
480
829
  }
481
- appendEvent(manifest.eventsPath, { type: "run.goal_achievement", runId: manifest.runId, message: gaApplied.manifest.goalAchievementNote ?? "", data: { achieved: gaAssessment.achieved, downgraded: gaApplied.downgraded, reason: gaAssessment.reason, signals: gaAssessment.signals } });
482
- if (gaApplied.downgraded) logInternalError("team-runner.goalAchievement.falseGreen", new Error(gaApplied.manifest.goalAchievementNote ?? "false-green detected"), `runId=${manifest.runId}`);
830
+ appendEvent(manifest.eventsPath, {
831
+ type: "run.goal_achievement",
832
+ runId: manifest.runId,
833
+ message: gaApplied.manifest.goalAchievementNote ?? "",
834
+ data: {
835
+ achieved: gaAssessment.achieved,
836
+ downgraded: gaApplied.downgraded,
837
+ reason: gaAssessment.reason,
838
+ signals: gaAssessment.signals,
839
+ },
840
+ });
841
+ if (gaApplied.downgraded)
842
+ logInternalError(
843
+ "team-runner.goalAchievement.falseGreen",
844
+ new Error(
845
+ gaApplied.manifest.goalAchievementNote ??
846
+ "false-green detected",
847
+ ),
848
+ `runId=${manifest.runId}`,
849
+ );
483
850
  stopTeamHeartbeat();
484
851
  resolveRunPromise(manifest.runId, result);
485
852
  cleanupUsage();
486
853
  // Terminate live agents for this run — agents are done when the run ends.
487
- void terminateLiveAgentsForRun(manifest.runId, "completed", appendEvent, manifest.eventsPath).catch((error) => logInternalError("team-runner.completed.terminate", error, `runId=${manifest.runId}`));
854
+ void terminateLiveAgentsForRun(
855
+ manifest.runId,
856
+ "completed",
857
+ appendEvent,
858
+ manifest.eventsPath,
859
+ ).catch((error) =>
860
+ logInternalError(
861
+ "team-runner.completed.terminate",
862
+ error,
863
+ `runId=${manifest.runId}`,
864
+ ),
865
+ );
488
866
 
489
867
  // Emit run completion hook (100% reliable, fire-and-forget)
490
- crewHooks.emit({ type: "run_completed", timestamp: new Date().toISOString(), runId: manifest.runId, data: { status: result.manifest.status, taskCount: result.tasks.length } });
868
+ crewHooks.emit({
869
+ type: "run_completed",
870
+ timestamp: new Date().toISOString(),
871
+ runId: manifest.runId,
872
+ data: {
873
+ status: result.manifest.status,
874
+ taskCount: result.tasks.length,
875
+ },
876
+ });
491
877
 
492
878
  // Execute after_run_complete lifecycle hook (non-blocking)
493
- const afterRunReport = await executeHook("after_run_complete", { runId: manifest.runId, cwd: manifest.cwd, status: result.manifest.status });
879
+ const afterRunReport = await executeHook("after_run_complete", {
880
+ runId: manifest.runId,
881
+ cwd: manifest.cwd,
882
+ status: result.manifest.status,
883
+ });
494
884
  appendHookEvent(manifest, afterRunReport);
495
885
  if (afterRunReport.outcome === "block") {
496
- logInternalError("team-runner.after_run_complete.blocked", new Error(afterRunReport.reason ?? "after_run_complete hook blocked"), `runId=${manifest.runId}`);
886
+ logInternalError(
887
+ "team-runner.after_run_complete.blocked",
888
+ new Error(
889
+ afterRunReport.reason ?? "after_run_complete hook blocked",
890
+ ),
891
+ `runId=${manifest.runId}`,
892
+ );
497
893
  }
498
894
 
499
895
  return result;
500
896
  } catch (error) {
501
897
  // Round 27 (BUG 1): the success path calls stopTeamHeartbeat() but this
502
- // catch path did NOT. The team heartbeat is a non-unref'd setInterval
503
- // (30s) that deliberately keeps the event loop alive — without this
504
- // call, a failed team run leaves the interval firing forever and the
505
- // foreground pi process hangs (never returns to the prompt); in
506
- // background-runner mode the worker never exits. clearInterval is
507
- // idempotent so a double-call (if this runs after the success path)
508
- // is harmless.
898
+ // catch path did NOT. The team heartbeat is a non-unref'd setInterval
899
+ // (30s) that deliberately keeps the event loop alive — without this
900
+ // call, a failed team run leaves the interval firing forever and the
901
+ // foreground pi process hangs (never returns to the prompt); in
902
+ // background-runner mode the worker never exits. clearInterval is
903
+ // idempotent so a double-call (if this runs after the success path)
904
+ // is harmless.
509
905
  stopTeamHeartbeat();
510
906
  // P1: Catch unhandled errors — ensure manifest/tasks/agents are terminal so they don't stay "running" forever.
511
907
  const message = error instanceof Error ? error.message : String(error);
@@ -513,7 +909,9 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
513
909
  // If lock acquisition fails, use in-memory data rather than stale disk data.
514
910
  let loaded;
515
911
  try {
516
- loaded = await withRunLock(input.manifest, async () => loadRunManifestById(input.manifest.cwd, input.manifest.runId));
912
+ loaded = await withRunLock(input.manifest, async () =>
913
+ loadRunManifestById(input.manifest.cwd, input.manifest.runId),
914
+ );
517
915
  } catch {
518
916
  loaded = undefined; // best-effort: use in-memory data if lock fails
519
917
  }
@@ -521,29 +919,72 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
521
919
  const freshTasks = refreshTaskGraphQueues(loaded?.tasks ?? input.tasks);
522
920
  const failedAt = new Date().toISOString();
523
921
  const tasks = freshTasks.map((task) =>
524
- task.status === "running" || task.status === "queued" || task.status === "waiting"
525
- ? { ...task, status: "failed" as const, finishedAt: failedAt, error: message }
922
+ task.status === "running" ||
923
+ task.status === "queued" ||
924
+ task.status === "waiting"
925
+ ? {
926
+ ...task,
927
+ status: "failed" as const,
928
+ finishedAt: failedAt,
929
+ error: message,
930
+ }
526
931
  : task,
527
932
  );
528
933
  manifest = freshManifest;
529
934
  try {
530
- await terminateLiveAgentsForRun(manifest.runId, "failed", appendEvent, manifest.eventsPath);
935
+ await terminateLiveAgentsForRun(
936
+ manifest.runId,
937
+ "failed",
938
+ appendEvent,
939
+ manifest.eventsPath,
940
+ );
531
941
  await saveRunTasksAsync(manifest, tasks);
532
- const existingRuntimeByTask = new Map(readCrewAgents(manifest).map((agent) => [agent.taskId, agent.runtime]));
942
+ const existingRuntimeByTask = new Map(
943
+ readCrewAgents(manifest).map((agent) => [
944
+ agent.taskId,
945
+ agent.runtime,
946
+ ]),
947
+ );
533
948
  const globalRuntime = input.runtime?.kind ?? "child-process";
534
- const runtimeForAgent = (agent: ReturnType<typeof recordsForMaterializedTasks>[number]): CrewRuntimeKind => {
949
+ const runtimeForAgent = (
950
+ agent: ReturnType<typeof recordsForMaterializedTasks>[number],
951
+ ): CrewRuntimeKind => {
535
952
  const task = tasks.find((item) => item.id === agent.taskId);
536
- return existingRuntimeByTask.get(agent.taskId) ?? resolveTaskRuntimeKind(globalRuntime, task?.role ?? agent.role, input.runtimeConfig?.isolationPolicy);
953
+ return (
954
+ existingRuntimeByTask.get(agent.taskId) ??
955
+ resolveTaskRuntimeKind(
956
+ globalRuntime,
957
+ task?.role ?? agent.role,
958
+ input.runtimeConfig?.isolationPolicy,
959
+ )
960
+ );
537
961
  };
538
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, globalRuntime).map((agent) => ({ ...agent, runtime: runtimeForAgent(agent) })));
539
- manifest = updateRunStatus(manifest, "failed", `Unhandled error in team runner: ${message}`);
962
+ saveCrewAgents(
963
+ manifest,
964
+ recordsForMaterializedTasks(manifest, tasks, globalRuntime).map(
965
+ (agent) => ({ ...agent, runtime: runtimeForAgent(agent) }),
966
+ ),
967
+ );
968
+ manifest = updateRunStatus(
969
+ manifest,
970
+ "failed",
971
+ `Unhandled error in team runner: ${message}`,
972
+ );
540
973
  await saveRunManifestAsync(manifest);
541
974
  } catch {
542
975
  // Best-effort — state write may also fail
543
976
  }
544
977
  const result = { manifest, tasks };
545
- rejectRunPromise(manifest.runId, error instanceof Error ? error : new Error(message));
546
- crewHooks.emit({ type: "run_failed", timestamp: new Date().toISOString(), runId: manifest.runId, data: { status: manifest.status, error: message } });
978
+ rejectRunPromise(
979
+ manifest.runId,
980
+ error instanceof Error ? error : new Error(message),
981
+ );
982
+ crewHooks.emit({
983
+ type: "run_failed",
984
+ timestamp: new Date().toISOString(),
985
+ runId: manifest.runId,
986
+ data: { status: manifest.status, error: message },
987
+ });
547
988
  cleanupUsage();
548
989
  return result;
549
990
  }
@@ -555,10 +996,17 @@ async function executeTeamRunCore(
555
996
  workflow: WorkflowConfig,
556
997
  ): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
557
998
  // Execute before_run_start hook (non-blocking by default)
558
- const beforeRunReport = await executeHook("before_run_start", { runId: manifest.runId, cwd: manifest.cwd });
999
+ const beforeRunReport = await executeHook("before_run_start", {
1000
+ runId: manifest.runId,
1001
+ cwd: manifest.cwd,
1002
+ });
559
1003
  appendHookEvent(manifest, beforeRunReport);
560
1004
  if (beforeRunReport.outcome === "block") {
561
- manifest = updateRunStatus(manifest, "blocked", beforeRunReport.reason ?? "before_run_start hook blocked the run.");
1005
+ manifest = updateRunStatus(
1006
+ manifest,
1007
+ "blocked",
1008
+ beforeRunReport.reason ?? "before_run_start hook blocked the run.",
1009
+ );
562
1010
  return { manifest, tasks: input.tasks };
563
1011
  }
564
1012
  let tasks = refreshTaskGraphQueues(input.tasks);
@@ -567,45 +1015,94 @@ async function executeTeamRunCore(
567
1015
  let adaptivePlanInjected = false;
568
1016
  let adaptivePlanMissing = false;
569
1017
  const attemptAdaptivePlan = () => {
570
- if (!canInjectAdaptivePlan || adaptivePlanInjected || adaptivePlanMissing) return { injected: false, missing: false };
571
- const adaptivePlan = injectAdaptivePlanIfReady({ manifest, tasks, workflow, team: input.team });
1018
+ if (
1019
+ !canInjectAdaptivePlan ||
1020
+ adaptivePlanInjected ||
1021
+ adaptivePlanMissing
1022
+ )
1023
+ return { injected: false, missing: false };
1024
+ const adaptivePlan = injectAdaptivePlanIfReady({
1025
+ manifest,
1026
+ tasks,
1027
+ workflow,
1028
+ team: input.team,
1029
+ });
572
1030
  adaptivePlanInjected = adaptivePlanInjected || adaptivePlan.injected;
573
1031
  adaptivePlanMissing = adaptivePlan.missingPlan;
574
1032
  workflow = adaptivePlan.workflow;
575
1033
  if (adaptivePlan.injected) tasks = adaptivePlan.tasks;
576
- return { injected: adaptivePlan.injected, missing: adaptivePlan.missingPlan };
1034
+ return {
1035
+ injected: adaptivePlan.injected,
1036
+ missing: adaptivePlan.missingPlan,
1037
+ };
577
1038
  };
578
1039
  const initialAdaptive = attemptAdaptivePlan();
579
1040
  if (initialAdaptive.missing) {
580
- tasks = markBlocked(tasks, "Adaptive planner did not produce a valid subagent plan.");
1041
+ tasks = markBlocked(
1042
+ tasks,
1043
+ "Adaptive planner did not produce a valid subagent plan.",
1044
+ );
581
1045
  await saveRunTasksAsync(manifest, tasks);
582
- manifest = updateRunStatus(manifest, "blocked", "Adaptive planner did not produce a valid subagent plan.");
1046
+ manifest = updateRunStatus(
1047
+ manifest,
1048
+ "blocked",
1049
+ "Adaptive planner did not produce a valid subagent plan.",
1050
+ );
583
1051
  return { manifest, tasks };
584
1052
  }
585
1053
  if (initialAdaptive.injected) {
586
- manifest = requiresPlanApproval(workflow, input.runtimeConfig) ? ensurePlanApprovalRequested(manifest, tasks) : manifest;
1054
+ manifest = requiresPlanApproval(workflow, input.runtimeConfig)
1055
+ ? ensurePlanApprovalRequested(manifest, tasks)
1056
+ : manifest;
587
1057
  queueIndex = buildTaskGraphIndex(tasks);
588
- } else if (requiresPlanApproval(workflow, input.runtimeConfig) && (hasPendingMutatingAdaptiveTask(tasks) || hasPendingMutatingTaskAtBoundary(tasks))) {
1058
+ } else if (
1059
+ requiresPlanApproval(workflow, input.runtimeConfig) &&
1060
+ (hasPendingMutatingAdaptiveTask(tasks) ||
1061
+ hasPendingMutatingTaskAtBoundary(tasks))
1062
+ ) {
589
1063
  manifest = ensurePlanApprovalRequested(manifest, tasks);
590
1064
  }
591
1065
  if (manifest.planApproval?.status === "cancelled") {
592
1066
  tasks = cancelPlanTasks(tasks, "Plan approval was cancelled.");
593
1067
  await saveRunTasksAsync(manifest, tasks);
594
- manifest = updateRunStatus(manifest, "cancelled", "Plan approval was cancelled.");
1068
+ manifest = updateRunStatus(
1069
+ manifest,
1070
+ "cancelled",
1071
+ "Plan approval was cancelled.",
1072
+ );
595
1073
  return { manifest, tasks };
596
1074
  }
597
- manifest = writeProgress(manifest, tasks, "team-runner", input.executeWorkers, input.runtimeConfig);
1075
+ manifest = writeProgress(
1076
+ manifest,
1077
+ tasks,
1078
+ "team-runner",
1079
+ input.executeWorkers,
1080
+ input.runtimeConfig,
1081
+ );
598
1082
  await saveRunManifestAsync(manifest);
599
- const runtimeKind = input.runtime?.kind ?? (input.executeWorkers ? "child-process" : "scaffold");
600
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
1083
+ const runtimeKind =
1084
+ input.runtime?.kind ??
1085
+ (input.executeWorkers ? "child-process" : "scaffold");
1086
+ saveCrewAgents(
1087
+ manifest,
1088
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1089
+ );
601
1090
 
602
1091
  // Build a workflow phase state machine from workflow steps for precondition tracking.
603
- const workflowPhases: PhaseState[] = workflow.steps.map((step): PhaseState => ({
604
- name: step.id,
605
- status: "pending",
606
- inputs: step.reads === false ? [] : Array.isArray(step.reads) ? step.reads : [],
607
- outputs: step.output === false ? [] : step.output ? [step.output] : [],
608
- }));
1092
+ const workflowPhases: PhaseState[] = workflow.steps.map(
1093
+ (step): PhaseState => ({
1094
+ name: step.id,
1095
+ status: "pending",
1096
+ inputs:
1097
+ step.reads === false
1098
+ ? []
1099
+ : Array.isArray(step.reads)
1100
+ ? step.reads
1101
+ : [],
1102
+ outputs:
1103
+ step.output === false ? [] : step.output ? [step.output] : [],
1104
+ }),
1105
+ );
609
1106
  let wfMachine = createWorkflowStateMachine(workflowPhases);
610
1107
 
611
1108
  while (tasks.some((task) => task.status === "queued")) {
@@ -614,17 +1111,46 @@ async function executeTeamRunCore(
614
1111
  const message = `${cancelReason.message} (${cancelReason.code})`;
615
1112
  const cancelledTaskIds: string[] = [];
616
1113
  tasks = tasks.map((task) => {
617
- if (task.status !== "queued" && task.status !== "running" && task.status !== "waiting") return task;
1114
+ if (
1115
+ task.status !== "queued" &&
1116
+ task.status !== "running" &&
1117
+ task.status !== "waiting"
1118
+ )
1119
+ return task;
618
1120
  cancelledTaskIds.push(task.id);
619
- const base = { ...task, status: "cancelled" as const, finishedAt: new Date().toISOString(), error: message };
1121
+ const base = {
1122
+ ...task,
1123
+ status: "cancelled" as const,
1124
+ finishedAt: new Date().toISOString(),
1125
+ error: message,
1126
+ };
620
1127
  if (task.status === "running") {
621
- return { ...base, terminalEvidence: [...(task.terminalEvidence ?? []), buildSyntheticTerminalEvidence("worker", cancelReason, task.startedAt)] };
1128
+ return {
1129
+ ...base,
1130
+ terminalEvidence: [
1131
+ ...(task.terminalEvidence ?? []),
1132
+ buildSyntheticTerminalEvidence(
1133
+ "worker",
1134
+ cancelReason,
1135
+ task.startedAt,
1136
+ ),
1137
+ ],
1138
+ };
622
1139
  }
623
1140
  return base;
624
1141
  });
625
1142
  await saveRunTasksAsync(manifest, tasks);
626
- for (const taskId of cancelledTaskIds) await appendEventAsync(manifest.eventsPath, { type: "task.cancelled", runId: manifest.runId, taskId, message, data: { reason: cancelReason.code } });
627
- manifest = updateRunStatus(manifest, "cancelled", message, { data: { reason: cancelReason.code, cancelledTaskIds } });
1143
+ for (const taskId of cancelledTaskIds)
1144
+ await appendEventAsync(manifest.eventsPath, {
1145
+ type: "task.cancelled",
1146
+ runId: manifest.runId,
1147
+ taskId,
1148
+ message,
1149
+ data: { reason: cancelReason.code },
1150
+ });
1151
+ manifest = updateRunStatus(manifest, "cancelled", message, {
1152
+ data: { reason: cancelReason.code, cancelledTaskIds },
1153
+ });
628
1154
  return { manifest, tasks };
629
1155
  }
630
1156
 
@@ -637,15 +1163,48 @@ async function executeTeamRunCore(
637
1163
  // Default-off: maxRetriesPerTask=0 → original abort behavior preserved.
638
1164
  const rerun = shouldRerunFailedTask(failed, input.limits);
639
1165
  if (rerun.rerun) {
640
- tasks = tasks.map((item) => item.id === failed.id ? { ...item, status: "queued" as const, policy: { ...(item.policy ?? {}), retryCount: rerun.newRetryCount }, error: undefined, finishedAt: undefined } : item);
1166
+ tasks = tasks.map((item) =>
1167
+ item.id === failed.id
1168
+ ? {
1169
+ ...item,
1170
+ status: "queued" as const,
1171
+ policy: {
1172
+ ...(item.policy ?? {}),
1173
+ retryCount: rerun.newRetryCount,
1174
+ },
1175
+ error: undefined,
1176
+ finishedAt: undefined,
1177
+ }
1178
+ : item,
1179
+ );
641
1180
  await saveRunTasksAsync(manifest, tasks);
642
- await appendEventAsync(manifest.eventsPath, { type: "recovery.rerun_task", runId: manifest.runId, taskId: failed.id, message: `Re-queuing failed task for whole-task rerun: ${rerun.reason}`, data: { attempt: rerun.newRetryCount, maxRetries: input.limits?.maxRetriesPerTask ?? 0, scenario: "task_failed" } });
1181
+ await appendEventAsync(manifest.eventsPath, {
1182
+ type: "recovery.rerun_task",
1183
+ runId: manifest.runId,
1184
+ taskId: failed.id,
1185
+ message: `Re-queuing failed task for whole-task rerun: ${rerun.reason}`,
1186
+ data: {
1187
+ attempt: rerun.newRetryCount,
1188
+ maxRetries: input.limits?.maxRetriesPerTask ?? 0,
1189
+ scenario: "task_failed",
1190
+ },
1191
+ });
643
1192
  continue; // loop re-processes the re-queued task
644
1193
  }
645
- tasks = markBlocked(tasks, `Blocked by failed task '${failed.id}'.`);
1194
+ tasks = markBlocked(
1195
+ tasks,
1196
+ `Blocked by failed task '${failed.id}'.`,
1197
+ );
646
1198
  await saveRunTasksAsync(manifest, tasks);
647
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
648
- manifest = updateRunStatus(manifest, "failed", `Failed at task '${failed.id}'.`);
1199
+ saveCrewAgents(
1200
+ manifest,
1201
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1202
+ );
1203
+ manifest = updateRunStatus(
1204
+ manifest,
1205
+ "failed",
1206
+ `Failed at task '${failed.id}'.`,
1207
+ );
649
1208
  return { manifest, tasks };
650
1209
  }
651
1210
 
@@ -654,67 +1213,202 @@ async function executeTeamRunCore(
654
1213
  // DAG-based execution plan: when tasks have explicit dependsOn, use the
655
1214
  // topological wave planner to determine ready tasks. Fall back to the
656
1215
  // existing task-graph-scheduler when no explicit deps exist (backward compat).
657
- const completedIds = new Set(tasks.filter((t) => t.status === "completed" || t.status === "needs_attention").map((t) => t.id));
1216
+ const completedIds = new Set(
1217
+ tasks
1218
+ .filter(
1219
+ (t) =>
1220
+ t.status === "completed" ||
1221
+ t.status === "needs_attention",
1222
+ )
1223
+ .map((t) => t.id),
1224
+ );
658
1225
  const dagReady = dagReadyTaskIds(tasks, completedIds);
659
1226
  const effectiveReady = dagReady ?? snapshot.ready;
660
1227
 
661
1228
  // Workflow phase precondition check (non-blocking: log warnings only).
662
1229
  if (wfMachine.currentPhaseIndex < wfMachine.phases.length) {
663
- const completedArtifacts = manifest.artifacts.filter((a) => a.kind === "result" || a.kind === "summary").map((a) => a.path);
664
- const previousPhaseStatus = wfMachine.currentPhaseIndex > 0 ? (wfMachine.phases[wfMachine.currentPhaseIndex - 1]?.status ?? "pending") : "completed";
1230
+ const completedArtifacts = manifest.artifacts
1231
+ .filter((a) => a.kind === "result" || a.kind === "summary")
1232
+ .map((a) => a.path);
1233
+ const previousPhaseStatus =
1234
+ wfMachine.currentPhaseIndex > 0
1235
+ ? (wfMachine.phases[wfMachine.currentPhaseIndex - 1]
1236
+ ?.status ?? "pending")
1237
+ : "completed";
665
1238
  const wfContext: PhaseGuardContext = {
666
1239
  completedArtifacts,
667
1240
  previousPhaseStatus,
668
- taskResults: tasks.filter((t) => t.status === "completed" || t.status === "needs_attention").map((t) => ({ taskId: t.id, status: t.status, outputPath: t.resultArtifact?.path })),
1241
+ taskResults: tasks
1242
+ .filter(
1243
+ (t) =>
1244
+ t.status === "completed" ||
1245
+ t.status === "needs_attention",
1246
+ )
1247
+ .map((t) => ({
1248
+ taskId: t.id,
1249
+ status: t.status,
1250
+ outputPath: t.resultArtifact?.path,
1251
+ })),
669
1252
  };
670
- const preconditions = validatePhasePreconditions(wfMachine, wfContext);
1253
+ const preconditions = validatePhasePreconditions(
1254
+ wfMachine,
1255
+ wfContext,
1256
+ );
671
1257
  if (!preconditions.ready) {
672
- await appendEventAsync(manifest.eventsPath, { type: "workflow.preconditions", runId: manifest.runId, message: `Workflow phase '${wfMachine.phases[wfMachine.currentPhaseIndex]?.name}' is missing inputs: ${preconditions.blocking.join(", ")}`, data: { phaseIndex: wfMachine.currentPhaseIndex, phaseName: wfMachine.phases[wfMachine.currentPhaseIndex]?.name, blocking: preconditions.blocking } });
1258
+ await appendEventAsync(manifest.eventsPath, {
1259
+ type: "workflow.preconditions",
1260
+ runId: manifest.runId,
1261
+ message: `Workflow phase '${wfMachine.phases[wfMachine.currentPhaseIndex]?.name}' is missing inputs: ${preconditions.blocking.join(", ")}`,
1262
+ data: {
1263
+ phaseIndex: wfMachine.currentPhaseIndex,
1264
+ phaseName:
1265
+ wfMachine.phases[wfMachine.currentPhaseIndex]?.name,
1266
+ blocking: preconditions.blocking,
1267
+ },
1268
+ });
673
1269
  } else {
674
1270
  // Advance the machine past completed phases.
675
- while (wfMachine.currentPhaseIndex < wfMachine.phases.length && wfMachine.phases[wfMachine.currentPhaseIndex]?.status === "completed") {
676
- wfMachine = { ...wfMachine, currentPhaseIndex: wfMachine.currentPhaseIndex + 1 };
1271
+ while (
1272
+ wfMachine.currentPhaseIndex < wfMachine.phases.length &&
1273
+ wfMachine.phases[wfMachine.currentPhaseIndex]?.status ===
1274
+ "completed"
1275
+ ) {
1276
+ wfMachine = {
1277
+ ...wfMachine,
1278
+ currentPhaseIndex: wfMachine.currentPhaseIndex + 1,
1279
+ };
677
1280
  }
678
1281
  }
679
1282
  }
680
1283
 
681
- const readyRoles = effectiveReady.map((taskId) => tasks.find((task) => task.id === taskId)?.role).filter((role): role is string => Boolean(role));
682
- const concurrency = resolveBatchConcurrency({ workflowName: workflow.name, workflowMaxConcurrency: workflow.maxConcurrency, teamMaxConcurrency: input.team.maxConcurrency, limitMaxConcurrentWorkers: input.limits?.maxConcurrentWorkers, allowUnboundedConcurrency: input.limits?.allowUnboundedConcurrency, readyCount: effectiveReady.length, workspaceMode: manifest.workspaceMode, readyRoles });
1284
+ const readyRoles = effectiveReady
1285
+ .map((taskId) => tasks.find((task) => task.id === taskId)?.role)
1286
+ .filter((role): role is string => Boolean(role));
1287
+ const concurrency = resolveBatchConcurrency({
1288
+ workflowName: workflow.name,
1289
+ workflowMaxConcurrency: workflow.maxConcurrency,
1290
+ teamMaxConcurrency: input.team.maxConcurrency,
1291
+ limitMaxConcurrentWorkers: input.limits?.maxConcurrentWorkers,
1292
+ allowUnboundedConcurrency: input.limits?.allowUnboundedConcurrency,
1293
+ readyCount: effectiveReady.length,
1294
+ workspaceMode: manifest.workspaceMode,
1295
+ readyRoles,
1296
+ });
683
1297
  if (concurrency.reason.includes(";unbounded:")) {
684
- await appendEventAsync(manifest.eventsPath, { type: "limits.unbounded", runId: manifest.runId, message: "Unbounded worker concurrency was explicitly enabled for this run.", data: { concurrencyReason: concurrency.reason, maxConcurrent: concurrency.maxConcurrent } });
1298
+ await appendEventAsync(manifest.eventsPath, {
1299
+ type: "limits.unbounded",
1300
+ runId: manifest.runId,
1301
+ message:
1302
+ "Unbounded worker concurrency was explicitly enabled for this run.",
1303
+ data: {
1304
+ concurrencyReason: concurrency.reason,
1305
+ maxConcurrent: concurrency.maxConcurrent,
1306
+ },
1307
+ });
685
1308
  }
686
1309
  const approvalPending = isPlanApprovalPending(manifest);
687
- const readyIds = approvalPending ? effectiveReady : effectiveReady.slice(0, concurrency.selectedCount);
688
- const candidateBatch = readyIds.map((id) => tasks.find((task) => task.id === id)).filter((task): task is TeamTaskState => Boolean(task));
689
- const readyBatch = approvalPending ? candidateBatch.filter((task) => !isMutatingTask(task)).slice(0, concurrency.selectedCount) : candidateBatch;
1310
+ const readyIds = approvalPending
1311
+ ? effectiveReady
1312
+ : effectiveReady.slice(0, concurrency.selectedCount);
1313
+ const candidateBatch = readyIds
1314
+ .map((id) => tasks.find((task) => task.id === id))
1315
+ .filter((task): task is TeamTaskState => Boolean(task));
1316
+ const readyBatch = approvalPending
1317
+ ? candidateBatch
1318
+ .filter((task) => !isMutatingTask(task))
1319
+ .slice(0, concurrency.selectedCount)
1320
+ : candidateBatch;
690
1321
  if (readyBatch.length === 0) {
691
1322
  if (approvalPending && candidateBatch.some(isMutatingTask)) {
692
1323
  await saveRunTasksAsync(manifest, tasks);
693
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
694
- manifest = updateRunStatus(manifest, "blocked", "Plan approval required before mutating implementation tasks run.");
1324
+ saveCrewAgents(
1325
+ manifest,
1326
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1327
+ );
1328
+ manifest = updateRunStatus(
1329
+ manifest,
1330
+ "blocked",
1331
+ "Plan approval required before mutating implementation tasks run.",
1332
+ );
695
1333
  return { manifest, tasks };
696
1334
  }
697
- tasks = markBlocked(tasks, "No ready queued task; dependency graph may be invalid.");
1335
+ tasks = markBlocked(
1336
+ tasks,
1337
+ "No ready queued task; dependency graph may be invalid.",
1338
+ );
698
1339
  await saveRunTasksAsync(manifest, tasks);
699
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
700
- manifest = updateRunStatus(manifest, "blocked", "No ready queued task.");
1340
+ saveCrewAgents(
1341
+ manifest,
1342
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1343
+ );
1344
+ manifest = updateRunStatus(
1345
+ manifest,
1346
+ "blocked",
1347
+ "No ready queued task.",
1348
+ );
701
1349
  return { manifest, tasks };
702
1350
  }
703
1351
 
704
1352
  // 2.2 caller migration: batch progress is high-frequency informational.
705
- appendEventFireAndForget(manifest.eventsPath, { type: "task.progress", runId: manifest.runId, message: `Starting ready batch with ${readyBatch.length} task(s).`, data: { taskIds: readyBatch.map((task) => task.id), readyCount: snapshot.ready.length, blockedCount: snapshot.blocked.length, runningCount: snapshot.running.length, doneCount: snapshot.done.length, selectedCount: readyBatch.length, maxConcurrent: concurrency.maxConcurrent, defaultConcurrency: concurrency.defaultConcurrency, concurrencyReason: approvalPending ? `${concurrency.reason};plan-approval-read-only` : concurrency.reason } });
1353
+ appendEventFireAndForget(manifest.eventsPath, {
1354
+ type: "task.progress",
1355
+ runId: manifest.runId,
1356
+ message: `Starting ready batch with ${readyBatch.length} task(s).`,
1357
+ data: {
1358
+ taskIds: readyBatch.map((task) => task.id),
1359
+ readyCount: snapshot.ready.length,
1360
+ blockedCount: snapshot.blocked.length,
1361
+ runningCount: snapshot.running.length,
1362
+ doneCount: snapshot.done.length,
1363
+ selectedCount: readyBatch.length,
1364
+ maxConcurrent: concurrency.maxConcurrent,
1365
+ defaultConcurrency: concurrency.defaultConcurrency,
1366
+ concurrencyReason: approvalPending
1367
+ ? `${concurrency.reason};plan-approval-read-only`
1368
+ : concurrency.reason,
1369
+ },
1370
+ });
706
1371
  // Execute before_task_start hooks for the batch
707
1372
  for (const task of readyBatch) {
708
- const taskReport = await executeHook("before_task_start", { runId: manifest.runId, taskId: task.id, cwd: manifest.cwd });
1373
+ const taskReport = await executeHook("before_task_start", {
1374
+ runId: manifest.runId,
1375
+ taskId: task.id,
1376
+ cwd: manifest.cwd,
1377
+ });
709
1378
  appendHookEvent(manifest, taskReport);
710
1379
  if (taskReport.outcome === "block") {
711
- tasks = tasks.map((t) => t.id === task.id ? { ...t, status: "skipped" as const, error: taskReport.reason ?? "before_task_start hook blocked execution." } : t);
712
- manifest = updateRunStatus(manifest, manifest.status, `Task '${task.id}' blocked by hook.`);
1380
+ tasks = tasks.map((t) =>
1381
+ t.id === task.id
1382
+ ? {
1383
+ ...t,
1384
+ status: "skipped" as const,
1385
+ error:
1386
+ taskReport.reason ??
1387
+ "before_task_start hook blocked execution.",
1388
+ }
1389
+ : t,
1390
+ );
1391
+ manifest = updateRunStatus(
1392
+ manifest,
1393
+ manifest.status,
1394
+ `Task '${task.id}' blocked by hook.`,
1395
+ );
713
1396
  }
714
1397
  }
715
- const batchTasks = readyBatch.filter((task) => tasks.find((t) => t.id === task.id && t.status !== "skipped"));
1398
+ const batchTasks = readyBatch.filter((task) =>
1399
+ tasks.find((t) => t.id === task.id && t.status !== "skipped"),
1400
+ );
716
1401
  if (batchTasks.length > 1) {
717
- await appendEventAsync(manifest.eventsPath, { type: "task.parallel_start", runId: manifest.runId, message: `Launching ${batchTasks.length} tasks in PARALLEL (concurrency=${concurrency.selectedCount}): ${batchTasks.map((t) => `${t.role}(${t.id})`).join(", ")}`, data: { taskIds: batchTasks.map((t) => t.id), roles: batchTasks.map((t) => t.role), concurrency: concurrency.selectedCount } });
1402
+ await appendEventAsync(manifest.eventsPath, {
1403
+ type: "task.parallel_start",
1404
+ runId: manifest.runId,
1405
+ message: `Launching ${batchTasks.length} tasks in PARALLEL (concurrency=${concurrency.selectedCount}): ${batchTasks.map((t) => `${t.role}(${t.id})`).join(", ")}`,
1406
+ data: {
1407
+ taskIds: batchTasks.map((t) => t.id),
1408
+ roles: batchTasks.map((t) => t.role),
1409
+ concurrency: concurrency.selectedCount,
1410
+ },
1411
+ });
718
1412
  }
719
1413
  const results = await mapConcurrent(
720
1414
  batchTasks,
@@ -722,78 +1416,302 @@ async function executeTeamRunCore(
722
1416
  async (task) => {
723
1417
  const step = findStep(workflow, task);
724
1418
  const agent = findAgent(input.agents, task);
725
- const teamRole = input.team.roles.find((role) => role.name === task.role);
726
- const perTaskRuntime = resolveTaskRuntimeKind(runtimeKind, task.role, input.runtimeConfig?.isolationPolicy);
727
- const baseInput = { manifest, tasks, task, step, agent, signal: input.signal, executeWorkers: input.executeWorkers, runtimeKind: runtimeKind, taskRuntimeOverride: perTaskRuntime !== runtimeKind ? perTaskRuntime : undefined, runtimeConfig: input.runtimeConfig, parentContext: input.parentContext, parentModel: input.parentModel, modelRegistry: input.modelRegistry, modelOverride: input.modelOverride, teamRoleModel: teamRole?.model, teamRoleSkills: teamRole?.skills, skillOverride: input.skillOverride, limits: input.limits, onJsonEvent: input.onJsonEvent, workspaceId: input.workspaceId };
1419
+ const teamRole = input.team.roles.find(
1420
+ (role) => role.name === task.role,
1421
+ );
1422
+ const perTaskRuntime = resolveTaskRuntimeKind(
1423
+ runtimeKind,
1424
+ task.role,
1425
+ input.runtimeConfig?.isolationPolicy,
1426
+ );
1427
+ const baseInput = {
1428
+ manifest,
1429
+ tasks,
1430
+ task,
1431
+ step,
1432
+ agent,
1433
+ signal: input.signal,
1434
+ executeWorkers: input.executeWorkers,
1435
+ runtimeKind: runtimeKind,
1436
+ taskRuntimeOverride:
1437
+ perTaskRuntime !== runtimeKind
1438
+ ? perTaskRuntime
1439
+ : undefined,
1440
+ runtimeConfig: input.runtimeConfig,
1441
+ parentContext: input.parentContext,
1442
+ parentModel: input.parentModel,
1443
+ modelRegistry: input.modelRegistry,
1444
+ modelOverride: input.modelOverride,
1445
+ teamRoleModel: teamRole?.model,
1446
+ teamRoleSkills: teamRole?.skills,
1447
+ skillOverride: input.skillOverride,
1448
+ limits: input.limits,
1449
+ onJsonEvent: input.onJsonEvent,
1450
+ workspaceId: input.workspaceId,
1451
+ };
728
1452
  // #1 (assessment): autoRetry now defaults ON (opt-out via reliability.autoRetry=false).
729
1453
  // The dominant v0.9.13 failure was ChildTimeout ("worker became unresponsive") with
730
1454
  // ZERO retries because this gate was opt-in. isRetryable() defaults to true when
731
1455
  // retryableErrors is empty, so transient hangs now retry up to maxAttempts (3) with
732
1456
  // exponential backoff. Set reliability.autoRetry=false to restore old single-shot behavior.
733
- if (!shouldUseRetry(input.reliability)) return withCorrelation(childCorrelation(manifest.runId, task.id), () => runTeamTask(baseInput));
734
- let lastFailed: { manifest: TeamRunManifest; tasks: TeamTaskState[] } | undefined;
1457
+ if (!shouldUseRetry(input.reliability))
1458
+ return withCorrelation(
1459
+ childCorrelation(manifest.runId, task.id),
1460
+ () => runTeamTask(baseInput),
1461
+ );
1462
+ let lastFailed:
1463
+ | { manifest: TeamRunManifest; tasks: TeamTaskState[] }
1464
+ | undefined;
735
1465
  let lastAttemptId: string | undefined;
736
- const attemptsSoFar: TaskAttemptState[] = [...(task.attempts ?? [])];
1466
+ const attemptsSoFar: TaskAttemptState[] = [
1467
+ ...(task.attempts ?? []),
1468
+ ];
737
1469
  const policy = retryPolicyFromConfig(input.reliability);
738
1470
  try {
739
- return await executeWithRetry(async (attempt, info) => {
740
- const startedAt = new Date().toISOString();
741
- const inFlightAttempts: TaskAttemptState[] = [...attemptsSoFar, { attemptId: info.attemptId, startedAt }];
742
- input.metricRegistry?.counter("crew.task.retry_attempt_total", "Retry attempts by run and task").inc({ runId: manifest.runId, taskId: task.id });
743
- // NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
744
- const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
745
- const freshManifest = fresh?.manifest ?? manifest;
746
- const freshTasks = fresh?.tasks ?? tasks;
747
- const freshTask = freshTasks.find((item) => item.id === task.id) ?? task;
748
- if (freshTask.status !== "queued" && freshTask.status !== "running") return { manifest: freshManifest, tasks: freshTasks };
749
- const taskWithAttempt: TeamTaskState = { ...freshTask, attempts: inFlightAttempts };
750
- const result = await withCorrelation(childCorrelation(freshManifest.runId, task.id), () => runTeamTask({ ...baseInput, manifest: freshManifest, tasks: freshTasks, task: taskWithAttempt }));
751
- const failed = failedTaskFrom(result, task.id);
752
- const endedAt = new Date().toISOString();
753
- const finishedAttempt: TaskAttemptState = { attemptId: info.attemptId, startedAt, endedAt, ...(failed?.error ? { error: failed.error } : {}) };
754
- attemptsSoFar.push(finishedAttempt);
755
- const withAttempt = result.tasks.map((item) => item.id === task.id ? { ...item, attempts: [...attemptsSoFar] } : item);
756
- const enriched = { manifest: result.manifest, tasks: withAttempt };
757
- if (failed) {
758
- lastFailed = enriched;
759
- throw new Error(failed.error ?? `Task ${task.id} failed.`);
760
- }
761
- input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe({ runId: manifest.runId, team: input.team.name }, Math.max(0, attempt - 1));
762
- return enriched;
763
- }, policy, {
764
- signal: input.signal,
765
- attemptId: (attempt) => `${manifest.runId}:${task.id}:attempt-${attempt}`,
766
- onAttemptFailed: (attempt, error, delayMs, info) => {
767
- lastAttemptId = info.attemptId;
768
- appendEventAsync(manifest.eventsPath, { type: "crew.task.retry_attempt", runId: manifest.runId, taskId: task.id, message: error.message, data: { attempt, attemptId: info.attemptId, delayMs }, metadata: { attemptId: info.attemptId } }).catch((error) => logInternalError("team-runner.retry-attempt", error, `taskId=${task.id}`));
769
- input.metricRegistry?.histogram("crew.task.retry_delay_ms", "Retry backoff delay, milliseconds").observe({ runId: manifest.runId, taskId: task.id }, delayMs);
1471
+ return await executeWithRetry(
1472
+ async (attempt, info) => {
1473
+ const startedAt = new Date().toISOString();
1474
+ const inFlightAttempts: TaskAttemptState[] = [
1475
+ ...attemptsSoFar,
1476
+ { attemptId: info.attemptId, startedAt },
1477
+ ];
1478
+ input.metricRegistry
1479
+ ?.counter(
1480
+ "crew.task.retry_attempt_total",
1481
+ "Retry attempts by run and task",
1482
+ )
1483
+ .inc({
1484
+ runId: manifest.runId,
1485
+ taskId: task.id,
1486
+ });
1487
+ // NOTE: no withRunLock best-effort only; concurrent writes may cause inconsistency
1488
+ const fresh = loadRunManifestById(
1489
+ manifest.cwd,
1490
+ manifest.runId,
1491
+ );
1492
+ const freshManifest = fresh?.manifest ?? manifest;
1493
+ const freshTasks = fresh?.tasks ?? tasks;
1494
+ const freshTask =
1495
+ freshTasks.find(
1496
+ (item) => item.id === task.id,
1497
+ ) ?? task;
1498
+ if (
1499
+ freshTask.status !== "queued" &&
1500
+ freshTask.status !== "running"
1501
+ )
1502
+ return {
1503
+ manifest: freshManifest,
1504
+ tasks: freshTasks,
1505
+ };
1506
+ const taskWithAttempt: TeamTaskState = {
1507
+ ...freshTask,
1508
+ attempts: inFlightAttempts,
1509
+ };
1510
+ const result = await withCorrelation(
1511
+ childCorrelation(freshManifest.runId, task.id),
1512
+ () =>
1513
+ runTeamTask({
1514
+ ...baseInput,
1515
+ manifest: freshManifest,
1516
+ tasks: freshTasks,
1517
+ task: taskWithAttempt,
1518
+ }),
1519
+ );
1520
+ const failed = failedTaskFrom(result, task.id);
1521
+ const endedAt = new Date().toISOString();
1522
+ const finishedAttempt: TaskAttemptState = {
1523
+ attemptId: info.attemptId,
1524
+ startedAt,
1525
+ endedAt,
1526
+ ...(failed?.error
1527
+ ? { error: failed.error }
1528
+ : {}),
1529
+ };
1530
+ attemptsSoFar.push(finishedAttempt);
1531
+ const withAttempt = result.tasks.map((item) =>
1532
+ item.id === task.id
1533
+ ? { ...item, attempts: [...attemptsSoFar] }
1534
+ : item,
1535
+ );
1536
+ const enriched = {
1537
+ manifest: result.manifest,
1538
+ tasks: withAttempt,
1539
+ };
1540
+ if (failed) {
1541
+ lastFailed = enriched;
1542
+ throw new Error(
1543
+ failed.error ?? `Task ${task.id} failed.`,
1544
+ );
1545
+ }
1546
+ input.metricRegistry
1547
+ ?.histogram(
1548
+ "crew.task.retry_count",
1549
+ "Retries per task",
1550
+ [0, 1, 2, 3, 5, 10],
1551
+ )
1552
+ .observe(
1553
+ {
1554
+ runId: manifest.runId,
1555
+ team: input.team.name,
1556
+ },
1557
+ Math.max(0, attempt - 1),
1558
+ );
1559
+ return enriched;
770
1560
  },
771
- onRetryGivenUp: (attempts, error, info) => {
772
- lastAttemptId = info.attemptId;
773
- appendDeadletter(manifest, { runId: manifest.runId, taskId: task.id, reason: "max-retries", attempts, attemptId: info.attemptId, lastError: error.message, timestamp: new Date().toISOString() });
774
- input.metricRegistry?.counter("crew.task.deadletter_total", "Deadletter triggers by reason").inc({ reason: "max-retries" });
775
- input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe({ runId: manifest.runId, team: input.team.name }, Math.max(0, attempts - 1));
1561
+ policy,
1562
+ {
1563
+ signal: input.signal,
1564
+ attemptId: (attempt) =>
1565
+ `${manifest.runId}:${task.id}:attempt-${attempt}`,
1566
+ onAttemptFailed: (
1567
+ attempt,
1568
+ error,
1569
+ delayMs,
1570
+ info,
1571
+ ) => {
1572
+ lastAttemptId = info.attemptId;
1573
+ appendEventAsync(manifest.eventsPath, {
1574
+ type: "crew.task.retry_attempt",
1575
+ runId: manifest.runId,
1576
+ taskId: task.id,
1577
+ message: error.message,
1578
+ data: {
1579
+ attempt,
1580
+ attemptId: info.attemptId,
1581
+ delayMs,
1582
+ },
1583
+ metadata: { attemptId: info.attemptId },
1584
+ }).catch((error) =>
1585
+ logInternalError(
1586
+ "team-runner.retry-attempt",
1587
+ error,
1588
+ `taskId=${task.id}`,
1589
+ ),
1590
+ );
1591
+ input.metricRegistry
1592
+ ?.histogram(
1593
+ "crew.task.retry_delay_ms",
1594
+ "Retry backoff delay, milliseconds",
1595
+ )
1596
+ .observe(
1597
+ {
1598
+ runId: manifest.runId,
1599
+ taskId: task.id,
1600
+ },
1601
+ delayMs,
1602
+ );
1603
+ },
1604
+ onRetryGivenUp: (attempts, error, info) => {
1605
+ lastAttemptId = info.attemptId;
1606
+ appendDeadletter(manifest, {
1607
+ runId: manifest.runId,
1608
+ taskId: task.id,
1609
+ reason: "max-retries",
1610
+ attempts,
1611
+ attemptId: info.attemptId,
1612
+ lastError: error.message,
1613
+ timestamp: new Date().toISOString(),
1614
+ });
1615
+ input.metricRegistry
1616
+ ?.counter(
1617
+ "crew.task.deadletter_total",
1618
+ "Deadletter triggers by reason",
1619
+ )
1620
+ .inc({ reason: "max-retries" });
1621
+ input.metricRegistry
1622
+ ?.histogram(
1623
+ "crew.task.retry_count",
1624
+ "Retries per task",
1625
+ [0, 1, 2, 3, 5, 10],
1626
+ )
1627
+ .observe(
1628
+ {
1629
+ runId: manifest.runId,
1630
+ team: input.team.name,
1631
+ },
1632
+ Math.max(0, attempts - 1),
1633
+ );
1634
+ },
776
1635
  },
777
- });
1636
+ );
778
1637
  } catch (retryError) {
779
- if (retryError instanceof CrewCancellationError || input.signal?.aborted) {
780
- const reason = retryError instanceof CrewCancellationError ? retryError.reason : cancellationReasonFromSignal(input.signal);
1638
+ if (
1639
+ retryError instanceof CrewCancellationError ||
1640
+ input.signal?.aborted
1641
+ ) {
1642
+ const reason =
1643
+ retryError instanceof CrewCancellationError
1644
+ ? retryError.reason
1645
+ : cancellationReasonFromSignal(input.signal);
781
1646
  // NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
782
- const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
1647
+ const fresh = loadRunManifestById(
1648
+ manifest.cwd,
1649
+ manifest.runId,
1650
+ );
783
1651
  const freshManifest = fresh?.manifest ?? manifest;
784
1652
  const freshTasks = fresh?.tasks ?? tasks;
785
- const cancelledTasks = freshTasks.map((item) => item.id === task.id && (item.status === "queued" || item.status === "running") ? { ...item, status: "cancelled" as const, finishedAt: new Date().toISOString(), error: `${reason.message} (${reason.code})` } : item);
786
- appendEventAsync(freshManifest.eventsPath, { type: "task.cancelled", runId: freshManifest.runId, taskId: task.id, message: reason.message, data: { reason, phase: "retry" }, metadata: lastAttemptId ? { attemptId: lastAttemptId } : undefined }).catch((error) => logInternalError("team-runner.cancelled", error, `taskId=${task.id}`));
787
- return { manifest: updateRunStatus(freshManifest, "cancelled", reason.message), tasks: cancelledTasks };
1653
+ const cancelledTasks = freshTasks.map((item) =>
1654
+ item.id === task.id &&
1655
+ (item.status === "queued" ||
1656
+ item.status === "running")
1657
+ ? {
1658
+ ...item,
1659
+ status: "cancelled" as const,
1660
+ finishedAt: new Date().toISOString(),
1661
+ error: `${reason.message} (${reason.code})`,
1662
+ }
1663
+ : item,
1664
+ );
1665
+ appendEventAsync(freshManifest.eventsPath, {
1666
+ type: "task.cancelled",
1667
+ runId: freshManifest.runId,
1668
+ taskId: task.id,
1669
+ message: reason.message,
1670
+ data: { reason, phase: "retry" },
1671
+ metadata: lastAttemptId
1672
+ ? { attemptId: lastAttemptId }
1673
+ : undefined,
1674
+ }).catch((error) =>
1675
+ logInternalError(
1676
+ "team-runner.cancelled",
1677
+ error,
1678
+ `taskId=${task.id}`,
1679
+ ),
1680
+ );
1681
+ return {
1682
+ manifest: updateRunStatus(
1683
+ freshManifest,
1684
+ "cancelled",
1685
+ reason.message,
1686
+ ),
1687
+ tasks: cancelledTasks,
1688
+ };
788
1689
  }
789
1690
  if (lastFailed) return lastFailed;
790
1691
  // NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
791
- const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
1692
+ const fresh = loadRunManifestById(
1693
+ manifest.cwd,
1694
+ manifest.runId,
1695
+ );
792
1696
  const freshManifest = fresh?.manifest ?? manifest;
793
1697
  const freshTasks = fresh?.tasks ?? tasks;
794
- const freshTask = freshTasks.find((item) => item.id === task.id) ?? task;
795
- if (freshTask.status !== "queued" && freshTask.status !== "running") return { manifest: freshManifest, tasks: freshTasks };
796
- return withCorrelation(childCorrelation(freshManifest.runId, task.id), () => runTeamTask({ ...baseInput, manifest: freshManifest, tasks: freshTasks, task: freshTask }));
1698
+ const freshTask =
1699
+ freshTasks.find((item) => item.id === task.id) ?? task;
1700
+ if (
1701
+ freshTask.status !== "queued" &&
1702
+ freshTask.status !== "running"
1703
+ )
1704
+ return { manifest: freshManifest, tasks: freshTasks };
1705
+ return withCorrelation(
1706
+ childCorrelation(freshManifest.runId, task.id),
1707
+ () =>
1708
+ runTeamTask({
1709
+ ...baseInput,
1710
+ manifest: freshManifest,
1711
+ tasks: freshTasks,
1712
+ task: freshTask,
1713
+ }),
1714
+ );
797
1715
  }
798
1716
  },
799
1717
  );
@@ -802,42 +1720,66 @@ async function executeTeamRunCore(
802
1720
  // during parallel execution. Other workers may have written partial results
803
1721
  // before one threw. Results may be partial - some tasks in-flight at error
804
1722
  // time will not have entries in the results array.
805
- const validResults = results.filter((item): item is NonNullable<typeof item> => item !== undefined);
1723
+ const validResults = results.filter(
1724
+ (item): item is NonNullable<typeof item> => item !== undefined,
1725
+ );
806
1726
  // Guard: if ALL parallel workers threw before returning, validResults is empty.
807
1727
  // at(-1)! would crash. Mark the run failed rather than crashing.
808
1728
  if (validResults.length === 0) {
809
- manifest = updateRunStatus(manifest, "failed", "All parallel tasks failed catastrophically.");
1729
+ manifest = updateRunStatus(
1730
+ manifest,
1731
+ "failed",
1732
+ "All parallel tasks failed catastrophically.",
1733
+ );
810
1734
  return { manifest, tasks };
811
1735
  }
812
1736
  // Reconstruct manifest from the last worker's snapshot. The .artifacts field
813
- // is re-merged from both the team-runner's in-memory state and all workers'
814
- // snapshots, so artifact writes by task-runner (which individually save manifest
815
- // after writing artifacts) are safely persisted. The in-memory manifest is only
816
- // used for the next batch iteration's orchestration — actual persistence is safe.
817
- // Use updateRunStatus to recompute manifest status from merged tasks rather than
818
- // relying on the last result's manifest (which is arbitrary due to mapConcurrent
819
- // returning results in arbitrary order).
820
- // Use the in-memory manifest as base (not the last-completing worker's snapshot).
821
- // Recompute status from merged tasks so the manifest reflects actual task state,
822
- // not the arbitrary order in which mapConcurrent returned results.
823
- // Read committed manifest from disk inside the lock so artifact merge is based
824
- // on committed state, not in-memory state that may differ from disk.
825
- const mergeResult = await withRunLock(manifest, async () => {
826
- const disk = loadRunManifestById(manifest.cwd, manifest.runId);
827
- const diskManifest = disk?.manifest ?? manifest;
828
- const diskArtifacts = diskManifest.artifacts;
829
- const reconciledArtifacts = mergeArtifacts([...diskArtifacts, ...validResults.map((item) => item.manifest.artifacts)].flat());
830
- const resultManifest = updateRunStatus({ ...diskManifest, artifacts: reconciledArtifacts }, "running", "Merged task updates from parallel batch.");
831
- const resultTasks = mergeTaskUpdatesPreservingTerminal(tasks, validResults);
832
- await saveRunManifestAsync(resultManifest);
833
- await saveRunTasksAsync(resultManifest, resultTasks);
834
- return { resultManifest, resultTasks };
835
- });
836
- manifest = mergeResult.resultManifest;
837
- tasks = mergeResult.resultTasks;
1737
+ // is re-merged from both the team-runner's in-memory state and all workers'
1738
+ // snapshots, so artifact writes by task-runner (which individually save manifest
1739
+ // after writing artifacts) are safely persisted. The in-memory manifest is only
1740
+ // used for the next batch iteration's orchestration — actual persistence is safe.
1741
+ // Use updateRunStatus to recompute manifest status from merged tasks rather than
1742
+ // relying on the last result's manifest (which is arbitrary due to mapConcurrent
1743
+ // returning results in arbitrary order).
1744
+ // Use the in-memory manifest as base (not the last-completing worker's snapshot).
1745
+ // Recompute status from merged tasks so the manifest reflects actual task state,
1746
+ // not the arbitrary order in which mapConcurrent returned results.
1747
+ // Read committed manifest from disk inside the lock so artifact merge is based
1748
+ // on committed state, not in-memory state that may differ from disk.
1749
+ const mergeResult = await withRunLock(manifest, async () => {
1750
+ const disk = loadRunManifestById(manifest.cwd, manifest.runId);
1751
+ const diskManifest = disk?.manifest ?? manifest;
1752
+ const diskArtifacts = diskManifest.artifacts;
1753
+ const reconciledArtifacts = mergeArtifacts(
1754
+ [
1755
+ ...diskArtifacts,
1756
+ ...validResults.map((item) => item.manifest.artifacts),
1757
+ ].flat(),
1758
+ );
1759
+ const resultManifest = updateRunStatus(
1760
+ { ...diskManifest, artifacts: reconciledArtifacts },
1761
+ "running",
1762
+ "Merged task updates from parallel batch.",
1763
+ );
1764
+ const resultTasks = mergeTaskUpdatesPreservingTerminal(
1765
+ tasks,
1766
+ validResults,
1767
+ );
1768
+ await saveRunManifestAsync(resultManifest);
1769
+ await saveRunTasksAsync(resultManifest, resultTasks);
1770
+ return { resultManifest, resultTasks };
1771
+ });
1772
+ manifest = mergeResult.resultManifest;
1773
+ tasks = mergeResult.resultTasks;
838
1774
 
839
1775
  // Advance workflow phases whose tasks are all in terminal state
840
- const terminalStatuses = new Set(["completed", "failed", "skipped", "cancelled", "needs_attention"]);
1776
+ const terminalStatuses = new Set([
1777
+ "completed",
1778
+ "failed",
1779
+ "skipped",
1780
+ "cancelled",
1781
+ "needs_attention",
1782
+ ]);
841
1783
  const phaseTaskMap = new Map<string, string[]>();
842
1784
  for (const task of tasks) {
843
1785
  if (!task.stepId) continue;
@@ -845,7 +1787,11 @@ tasks = mergeResult.resultTasks;
845
1787
  existing.push(task.id);
846
1788
  phaseTaskMap.set(task.stepId, existing);
847
1789
  }
848
- for (let pi = wfMachine.currentPhaseIndex; pi < wfMachine.phases.length; pi++) {
1790
+ for (
1791
+ let pi = wfMachine.currentPhaseIndex;
1792
+ pi < wfMachine.phases.length;
1793
+ pi++
1794
+ ) {
849
1795
  const phase = wfMachine.phases[pi]!;
850
1796
  const phaseTaskIds = phaseTaskMap.get(phase.name) ?? [];
851
1797
  if (phaseTaskIds.length === 0) continue;
@@ -854,75 +1800,187 @@ tasks = mergeResult.resultTasks;
854
1800
  return task ? terminalStatuses.has(task.status) : false;
855
1801
  });
856
1802
  if (!allTerminal) break;
857
- if (phase.status !== "completed" && phase.status !== "failed" && phase.status !== "skipped") {
858
- const completedArtifacts = manifest.artifacts.filter((a) => a.kind === "result" || a.kind === "summary").map((a) => a.path);
859
- const previousPhaseStatus = pi > 0 ? (wfMachine.phases[pi - 1]?.status ?? "pending") : "completed";
1803
+ if (
1804
+ phase.status !== "completed" &&
1805
+ phase.status !== "failed" &&
1806
+ phase.status !== "skipped"
1807
+ ) {
1808
+ const completedArtifacts = manifest.artifacts
1809
+ .filter((a) => a.kind === "result" || a.kind === "summary")
1810
+ .map((a) => a.path);
1811
+ const previousPhaseStatus =
1812
+ pi > 0
1813
+ ? (wfMachine.phases[pi - 1]?.status ?? "pending")
1814
+ : "completed";
860
1815
  const wfContext: PhaseGuardContext = {
861
1816
  completedArtifacts,
862
1817
  previousPhaseStatus,
863
- taskResults: tasks.filter((t) => t.status === "completed" || t.status === "needs_attention").map((t) => ({ taskId: t.id, status: t.status, outputPath: t.resultArtifact?.path })),
1818
+ taskResults: tasks
1819
+ .filter(
1820
+ (t) =>
1821
+ t.status === "completed" ||
1822
+ t.status === "needs_attention",
1823
+ )
1824
+ .map((t) => ({
1825
+ taskId: t.id,
1826
+ status: t.status,
1827
+ outputPath: t.resultArtifact?.path,
1828
+ })),
864
1829
  };
865
1830
  // Determine phase transition status based on individual task outcomes
866
- const phaseTasks = phaseTaskIds.map((taskId) => tasks.find((t) => t.id === taskId)).filter((t): t is NonNullable<typeof t> => t !== undefined);
867
- const hasFailedOrCancelled = phaseTasks.some((t) => t.status === "failed" || t.status === "cancelled");
868
- const phaseStatus = hasFailedOrCancelled ? "failed" : "completed";
869
- const transition = transitionPhase(wfMachine, pi, phaseStatus, wfContext);
1831
+ const phaseTasks = phaseTaskIds
1832
+ .map((taskId) => tasks.find((t) => t.id === taskId))
1833
+ .filter((t): t is NonNullable<typeof t> => t !== undefined);
1834
+ const hasFailedOrCancelled = phaseTasks.some(
1835
+ (t) => t.status === "failed" || t.status === "cancelled",
1836
+ );
1837
+ const phaseStatus = hasFailedOrCancelled
1838
+ ? "failed"
1839
+ : "completed";
1840
+ const transition = transitionPhase(
1841
+ wfMachine,
1842
+ pi,
1843
+ phaseStatus,
1844
+ wfContext,
1845
+ );
870
1846
  wfMachine = transition.machine;
871
1847
  if (transition.guardResult && !transition.guardResult.allowed) {
872
- await appendEventAsync(manifest.eventsPath, { type: "workflow.phase_guard_blocked", runId: manifest.runId, message: `Workflow phase '${phase.name}' guard blocked: ${transition.guardResult.reason ?? "unknown"}`, data: { phaseIndex: pi, phaseName: phase.name, reason: transition.guardResult.reason } });
1848
+ await appendEventAsync(manifest.eventsPath, {
1849
+ type: "workflow.phase_guard_blocked",
1850
+ runId: manifest.runId,
1851
+ message: `Workflow phase '${phase.name}' guard blocked: ${transition.guardResult.reason ?? "unknown"}`,
1852
+ data: {
1853
+ phaseIndex: pi,
1854
+ phaseName: phase.name,
1855
+ reason: transition.guardResult.reason,
1856
+ },
1857
+ });
873
1858
  break;
874
1859
  }
875
- await appendEventAsync(manifest.eventsPath, { type: phaseStatus === "failed" ? "workflow.phase_failed" : "workflow.phase_completed", runId: manifest.runId, message: `Workflow phase '${phase.name}' ${phaseStatus}.`, data: { phaseIndex: pi, phaseStatus } });
1860
+ await appendEventAsync(manifest.eventsPath, {
1861
+ type:
1862
+ phaseStatus === "failed"
1863
+ ? "workflow.phase_failed"
1864
+ : "workflow.phase_completed",
1865
+ runId: manifest.runId,
1866
+ message: `Workflow phase '${phase.name}' ${phaseStatus}.`,
1867
+ data: { phaseIndex: pi, phaseStatus },
1868
+ });
876
1869
  }
877
1870
  wfMachine = { ...wfMachine, currentPhaseIndex: pi + 1 };
878
1871
  }
879
1872
 
880
- const cancelledResult = results.find((item) => item.manifest.status === "cancelled");
1873
+ const cancelledResult = results.find(
1874
+ (item) => item.manifest.status === "cancelled",
1875
+ );
881
1876
  if (cancelledResult || input.signal?.aborted) {
882
- const reason = input.signal?.aborted ? cancellationReasonFromSignal(input.signal) : undefined;
883
- const message = reason?.message ?? cancelledResult?.manifest.summary ?? "Run cancelled during task execution.";
1877
+ const reason = input.signal?.aborted
1878
+ ? cancellationReasonFromSignal(input.signal)
1879
+ : undefined;
1880
+ const message =
1881
+ reason?.message ??
1882
+ cancelledResult?.manifest.summary ??
1883
+ "Run cancelled during task execution.";
884
1884
  manifest = { ...manifest, status: "running" };
885
1885
  manifest = updateRunStatus(manifest, "cancelled", message);
886
1886
  await saveRunTasksAsync(manifest, tasks);
887
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
1887
+ saveCrewAgents(
1888
+ manifest,
1889
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1890
+ );
888
1891
  await saveRunManifestAsync(manifest);
889
- await appendEventAsync(manifest.eventsPath, { type: "run.cancelled", runId: manifest.runId, message, data: { reason, phase: "task-batch", cancelledResultRunId: cancelledResult?.manifest.runId } });
1892
+ await appendEventAsync(manifest.eventsPath, {
1893
+ type: "run.cancelled",
1894
+ runId: manifest.runId,
1895
+ message,
1896
+ data: {
1897
+ reason,
1898
+ phase: "task-batch",
1899
+ cancelledResultRunId: cancelledResult?.manifest.runId,
1900
+ },
1901
+ });
890
1902
  return { manifest, tasks };
891
1903
  }
892
1904
  queueIndex = buildTaskGraphIndex(tasks);
893
1905
  const injectedAfterBatch = attemptAdaptivePlan();
894
1906
  if (injectedAfterBatch.missing) {
895
- tasks = markBlocked(tasks, "Adaptive planner did not produce a valid subagent plan.");
1907
+ tasks = markBlocked(
1908
+ tasks,
1909
+ "Adaptive planner did not produce a valid subagent plan.",
1910
+ );
896
1911
  await saveRunTasksAsync(manifest, tasks);
897
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
898
- manifest = updateRunStatus(manifest, "blocked", "Adaptive planner did not produce a valid subagent plan.");
1912
+ saveCrewAgents(
1913
+ manifest,
1914
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1915
+ );
1916
+ manifest = updateRunStatus(
1917
+ manifest,
1918
+ "blocked",
1919
+ "Adaptive planner did not produce a valid subagent plan.",
1920
+ );
899
1921
  return { manifest, tasks };
900
1922
  }
901
1923
  if (injectedAfterBatch.injected) {
902
- manifest = requiresPlanApproval(workflow, input.runtimeConfig) ? ensurePlanApprovalRequested(manifest, tasks) : manifest;
1924
+ manifest = requiresPlanApproval(workflow, input.runtimeConfig)
1925
+ ? ensurePlanApprovalRequested(manifest, tasks)
1926
+ : manifest;
903
1927
  queueIndex = buildTaskGraphIndex(tasks);
904
- } else if (requiresPlanApproval(workflow, input.runtimeConfig) && (hasPendingMutatingAdaptiveTask(tasks) || hasPendingMutatingTaskAtBoundary(tasks))) {
1928
+ } else if (
1929
+ requiresPlanApproval(workflow, input.runtimeConfig) &&
1930
+ (hasPendingMutatingAdaptiveTask(tasks) ||
1931
+ hasPendingMutatingTaskAtBoundary(tasks))
1932
+ ) {
905
1933
  manifest = ensurePlanApprovalRequested(manifest, tasks);
906
1934
  }
907
1935
  if (manifest.planApproval?.status === "cancelled") {
908
1936
  tasks = cancelPlanTasks(tasks, "Plan approval was cancelled.");
909
1937
  await saveRunTasksAsync(manifest, tasks);
910
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
911
- manifest = updateRunStatus(manifest, "cancelled", "Plan approval was cancelled.");
1938
+ saveCrewAgents(
1939
+ manifest,
1940
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1941
+ );
1942
+ manifest = updateRunStatus(
1943
+ manifest,
1944
+ "cancelled",
1945
+ "Plan approval was cancelled.",
1946
+ );
912
1947
  return { manifest, tasks };
913
1948
  }
914
1949
  await saveRunTasksAsync(manifest, tasks);
915
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
916
- const completedBatch = tasks.filter((t) => batchTasks.some((bt) => bt.id === t.id));
1950
+ saveCrewAgents(
1951
+ manifest,
1952
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1953
+ );
1954
+ const completedBatch = tasks.filter((t) =>
1955
+ batchTasks.some((bt) => bt.id === t.id),
1956
+ );
917
1957
  const batchArtifact = writeArtifact(manifest.artifactsRoot, {
918
1958
  kind: "summary",
919
1959
  relativePath: `batches/${batchTasks.map((task) => task.id).join("+")}.md`,
920
1960
  producer: "team-runner",
921
1961
  content: aggregateTaskOutputs(completedBatch, manifest),
922
1962
  });
923
- const groupDelivery = deliverGroupJoin({ manifest, mode: resolveGroupJoinMode(input.runtimeConfig), batch: batchTasks, allTasks: tasks });
924
- manifest = { ...manifest, artifacts: mergeArtifacts([...manifest.artifacts, batchArtifact, ...(groupDelivery?.artifact ? [groupDelivery.artifact] : [])]) };
925
- manifest = writeProgress(manifest, tasks, "team-runner", input.executeWorkers, input.runtimeConfig);
1963
+ const groupDelivery = deliverGroupJoin({
1964
+ manifest,
1965
+ mode: resolveGroupJoinMode(input.runtimeConfig),
1966
+ batch: batchTasks,
1967
+ allTasks: tasks,
1968
+ });
1969
+ manifest = {
1970
+ ...manifest,
1971
+ artifacts: mergeArtifacts([
1972
+ ...manifest.artifacts,
1973
+ batchArtifact,
1974
+ ...(groupDelivery?.artifact ? [groupDelivery.artifact] : []),
1975
+ ]),
1976
+ };
1977
+ manifest = writeProgress(
1978
+ manifest,
1979
+ tasks,
1980
+ "team-runner",
1981
+ input.executeWorkers,
1982
+ input.runtimeConfig,
1983
+ );
926
1984
  await saveRunManifestAsync(manifest);
927
1985
  }
928
1986
 
@@ -930,29 +1988,85 @@ tasks = mergeResult.resultTasks;
930
1988
  const waiting = tasks.find((task) => task.status === "waiting");
931
1989
  const running = tasks.find((task) => task.status === "running");
932
1990
  manifest = applyPolicy(manifest, tasks, input.limits);
933
- const effectiveness = evaluateRunEffectiveness({ manifest, tasks, executeWorkers: input.executeWorkers, runtimeConfig: input.runtimeConfig });
1991
+ const effectiveness = evaluateRunEffectiveness({
1992
+ manifest,
1993
+ tasks,
1994
+ executeWorkers: input.executeWorkers,
1995
+ runtimeConfig: input.runtimeConfig,
1996
+ });
934
1997
  const effectivenessDecision = effectivenessPolicyDecision(effectiveness);
935
1998
  if (effectivenessDecision) {
936
- manifest = { ...manifest, policyDecisions: [...(manifest.policyDecisions ?? []), effectivenessDecision], updatedAt: new Date().toISOString() };
937
- await appendEventAsync(manifest.eventsPath, { type: "run.effectiveness", runId: manifest.runId, message: effectivenessDecision.message, data: { effectiveness, policyDecision: effectivenessDecision } });
1999
+ manifest = {
2000
+ ...manifest,
2001
+ policyDecisions: [
2002
+ ...(manifest.policyDecisions ?? []),
2003
+ effectivenessDecision,
2004
+ ],
2005
+ updatedAt: new Date().toISOString(),
2006
+ };
2007
+ await appendEventAsync(manifest.eventsPath, {
2008
+ type: "run.effectiveness",
2009
+ runId: manifest.runId,
2010
+ message: effectivenessDecision.message,
2011
+ data: { effectiveness, policyDecision: effectivenessDecision },
2012
+ });
938
2013
  }
939
- const blockingDecision = manifest.policyDecisions?.find((item) => item.action === "block" || item.action === "escalate");
2014
+ const blockingDecision = manifest.policyDecisions?.find(
2015
+ (item) => item.action === "block" || item.action === "escalate",
2016
+ );
940
2017
  if (failed) {
941
- manifest = updateRunStatus(manifest, "failed", `Failed at task '${failed.id}'.`);
2018
+ manifest = updateRunStatus(
2019
+ manifest,
2020
+ "failed",
2021
+ `Failed at task '${failed.id}'.`,
2022
+ );
942
2023
  } else if (waiting) {
943
- manifest = updateRunStatus(manifest, "blocked", `Waiting for response to task '${waiting.id}'.`);
2024
+ manifest = updateRunStatus(
2025
+ manifest,
2026
+ "blocked",
2027
+ `Waiting for response to task '${waiting.id}'.`,
2028
+ );
944
2029
  } else if (running) {
945
- manifest = updateRunStatus(manifest, "blocked", `Task '${running.id}' is still running.`);
2030
+ manifest = updateRunStatus(
2031
+ manifest,
2032
+ "blocked",
2033
+ `Task '${running.id}' is still running.`,
2034
+ );
946
2035
  } else if (effectiveness.severity === "failed") {
947
- manifest = updateRunStatus(manifest, "failed", effectivenessDecision?.message ?? "Run effectiveness guard failed.");
2036
+ manifest = updateRunStatus(
2037
+ manifest,
2038
+ "failed",
2039
+ effectivenessDecision?.message ?? "Run effectiveness guard failed.",
2040
+ );
948
2041
  } else if (effectiveness.severity === "blocked") {
949
- manifest = updateRunStatus(manifest, "blocked", effectivenessDecision?.message ?? "Run effectiveness guard blocked completion.");
2042
+ manifest = updateRunStatus(
2043
+ manifest,
2044
+ "blocked",
2045
+ effectivenessDecision?.message ??
2046
+ "Run effectiveness guard blocked completion.",
2047
+ );
950
2048
  } else if (blockingDecision) {
951
- manifest = updateRunStatus(manifest, "blocked", blockingDecision.message);
2049
+ manifest = updateRunStatus(
2050
+ manifest,
2051
+ "blocked",
2052
+ blockingDecision.message,
2053
+ );
952
2054
  } else {
953
- manifest = updateRunStatus(manifest, "completed", input.executeWorkers ? "Team workflow completed." : "Team workflow scaffold completed without launching child workers.");
2055
+ manifest = updateRunStatus(
2056
+ manifest,
2057
+ "completed",
2058
+ input.executeWorkers
2059
+ ? "Team workflow completed."
2060
+ : "Team workflow scaffold completed without launching child workers.",
2061
+ );
954
2062
  }
955
- manifest = writeProgress(manifest, tasks, "team-runner", input.executeWorkers, input.runtimeConfig);
2063
+ manifest = writeProgress(
2064
+ manifest,
2065
+ tasks,
2066
+ "team-runner",
2067
+ input.executeWorkers,
2068
+ input.runtimeConfig,
2069
+ );
956
2070
  await saveRunManifestAsync(manifest);
957
2071
  const usage = aggregateUsage(tasks);
958
2072
  const summaryArtifact = writeArtifact(manifest.artifactsRoot, {
@@ -972,17 +2086,28 @@ tasks = mergeResult.resultTasks;
972
2086
  ...tasks.map(formatTaskProgress),
973
2087
  "",
974
2088
  "## Effectiveness",
975
- ...runEffectivenessLines(manifest, tasks, input.executeWorkers, input.runtimeConfig),
2089
+ ...runEffectivenessLines(
2090
+ manifest,
2091
+ tasks,
2092
+ input.executeWorkers,
2093
+ input.runtimeConfig,
2094
+ ),
976
2095
  "",
977
2096
  "## Policy decisions",
978
- ...(manifest.policyDecisions?.length ? summarizePolicyDecisions(manifest.policyDecisions) : ["- (none)"]),
2097
+ ...(manifest.policyDecisions?.length
2098
+ ? summarizePolicyDecisions(manifest.policyDecisions)
2099
+ : ["- (none)"]),
979
2100
  "",
980
2101
  ].join("\n"),
981
2102
  });
982
2103
  // Build the complete manifest BEFORE acquiring the lock so the artifacts array
983
2104
  // is already incorporated into the manifest object that will be atomically written.
984
2105
  // This prevents crash-between-mutation-and-lock from leaving inconsistent state.
985
- const finalManifest = { ...manifest, updatedAt: new Date().toISOString(), artifacts: [...manifest.artifacts, summaryArtifact] };
2106
+ const finalManifest = {
2107
+ ...manifest,
2108
+ updatedAt: new Date().toISOString(),
2109
+ artifacts: [...manifest.artifacts, summaryArtifact],
2110
+ };
986
2111
  // Joint atomic save: wrap manifest + tasks in a single run lock so they are
987
2112
  // written together or not at all. Crash between separate saveRunManifestAsync
988
2113
  // and saveRunTasksAsync calls could leave manifest/tasks.json out of sync.
@@ -1000,7 +2125,9 @@ tasks = mergeResult.resultTasks;
1000
2125
  // health feature) AND created junk dirs that the recursive state watcher then
1001
2126
  // attached extra inotify watches to. Fix: compute the real crew root (3 up)
1002
2127
  // and make HEALTH_DIR relative to it.
1003
- const crewRoot = path.dirname(path.dirname(path.dirname(finalManifest.stateRoot)));
2128
+ const crewRoot = path.dirname(
2129
+ path.dirname(path.dirname(finalManifest.stateRoot)),
2130
+ );
1004
2131
  const healthStore = new HealthStore(crewRoot);
1005
2132
  healthStore.saveSnapshot({
1006
2133
  runId: finalManifest.runId,