pi-crew 0.9.13 → 0.9.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,47 +1,117 @@
1
1
  import * as fs from "node:fs";
2
2
  import * as path from "node:path";
3
3
  import type { AgentConfig } from "../agents/agent-config.ts";
4
- import type { CrewLimitsConfig, CrewRuntimeConfig, CrewReliabilityConfig } from "../config/config.ts";
5
- import type { CrewRuntimeCapabilities } from "./runtime-resolver.ts";
6
- import type { CrewRuntimeKind } from "./crew-agent-runtime.ts";
7
- import { resolveTaskRuntimeKind } from "./runtime-policy.ts";
4
+ import type {
5
+ CrewLimitsConfig,
6
+ CrewReliabilityConfig,
7
+ CrewRuntimeConfig,
8
+ } from "../config/config.ts";
9
+ import { appendHookEvent, executeHook } from "../hooks/registry.ts";
10
+ import {
11
+ childCorrelation,
12
+ withCorrelation,
13
+ } from "../observability/correlation.ts";
14
+ import type { MetricRegistry } from "../observability/metric-registry.ts";
15
+ import { PluginRegistry } from "../plugins/plugin-registry.ts";
16
+ import {
17
+ NextJsPlugin,
18
+ VitePlugin,
19
+ VitestPlugin,
20
+ } from "../plugins/plugins/index.ts";
8
21
  import { writeArtifact } from "../state/artifact-store.ts";
9
- import { executeHook, appendHookEvent } from "../hooks/registry.ts";
10
- import { appendEvent, appendEventAsync, appendEventFireAndForget } from "../state/event-log.ts";
11
- import type { TeamConfig } from "../teams/team-config.ts";
12
- import type { ArtifactDescriptor, PolicyDecision, TeamRunManifest, TaskAttemptState, TeamTaskState } from "../state/types.ts";
13
- import { loadRunManifestById, saveRunManifest, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/state-store.ts";
22
+ import {
23
+ appendEvent,
24
+ appendEventAsync,
25
+ appendEventFireAndForget,
26
+ } from "../state/event-log.ts";
27
+ import { HealthStore } from "../state/health-store.ts";
14
28
  import { withRunLock } from "../state/locks.ts";
29
+ import {
30
+ loadRunManifestById,
31
+ saveRunManifest,
32
+ saveRunManifestAsync,
33
+ saveRunTasksAsync,
34
+ updateRunStatus,
35
+ } from "../state/state-store.ts";
36
+ import type {
37
+ ArtifactDescriptor,
38
+ PolicyDecision,
39
+ TaskAttemptState,
40
+ TeamRunManifest,
41
+ TeamTaskState,
42
+ } from "../state/types.ts";
15
43
  import { aggregateUsage, formatUsage } from "../state/usage.ts";
16
- import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
17
- import { evaluateCrewPolicy, summarizePolicyDecisions } from "./policy-engine.ts";
18
- import { buildRecoveryLedger } from "./recovery-recipes.ts";
19
- import { buildTaskGraphIndex, refreshTaskGraphQueues, taskGraphSnapshot } from "./task-graph-scheduler.ts";
20
- import { buildExecutionPlan as buildDagExecutionPlan, getReadyTasks as getDagReadyTasks, type TaskNode } from "./task-graph.ts";
44
+ import type { TeamConfig } from "../teams/team-config.ts";
45
+ import { logInternalError } from "../utils/internal-error.ts";
46
+ import type {
47
+ WorkflowConfig,
48
+ WorkflowStep,
49
+ } from "../workflows/workflow-config.ts";
21
50
  import { checkBranchFreshness } from "../worktree/branch-freshness.ts";
22
- import { aggregateTaskOutputs } from "./task-output-context.ts";
51
+ import {
52
+ buildSyntheticTerminalEvidence,
53
+ CrewCancellationError,
54
+ cancellationReasonFromSignal,
55
+ } from "./cancellation.ts";
56
+ import { resolveBatchConcurrency } from "./concurrency.ts";
23
57
  import { readCrewAgents, saveCrewAgents } from "./crew-agent-records.ts";
24
- import { recordsForMaterializedTasks } from "./task-display.ts";
58
+ import type { CrewRuntimeKind } from "./crew-agent-runtime.ts";
59
+ import { crewHooks } from "./crew-hooks.ts";
60
+ import { appendDeadletter } from "./deadletter.ts";
61
+ import {
62
+ effectivenessPolicyDecision,
63
+ evaluateRunEffectiveness,
64
+ formatRunEffectivenessLines,
65
+ } from "./effectiveness.ts";
66
+ import {
67
+ applyGoalAchievement,
68
+ assessGoalAchievement,
69
+ } from "./goal-achievement.ts";
25
70
  import { deliverGroupJoin, resolveGroupJoinMode } from "./group-join.ts";
26
- import { runTeamTask } from "./task-runner.ts";
27
71
  import { terminateLiveAgentsForRun } from "./live-agent-manager.ts";
28
- import { createWorkflowStateMachine, validatePhasePreconditions, transitionPhase, type PhaseState, type PhaseGuardContext } from "./workflow-state.ts";
29
- import { executeWithRetry, DEFAULT_RETRY_POLICY, type RetryPolicy } from "./retry-executor.ts";
30
- import { appendDeadletter } from "./deadletter.ts";
31
- import type { MetricRegistry } from "../observability/metric-registry.ts";
32
- import { childCorrelation, withCorrelation } from "../observability/correlation.ts";
33
- import { crewHooks } from "./crew-hooks.ts";
34
- import { resolveBatchConcurrency } from "./concurrency.ts";
35
72
  import { mapConcurrent } from "./parallel-utils.ts";
73
+ import {
74
+ evaluateCrewPolicy,
75
+ summarizePolicyDecisions,
76
+ } from "./policy-engine.ts";
77
+ import {
78
+ buildRecoveryLedger,
79
+ shouldRerunFailedTask,
80
+ } from "./recovery-recipes.ts";
81
+ import {
82
+ DEFAULT_RETRY_POLICY,
83
+ executeWithRetry,
84
+ type RetryPolicy,
85
+ } from "./retry-executor.ts";
36
86
  import { permissionForRole } from "./role-permission.ts";
37
- import { registerRunPromise, resolveRunPromise, rejectRunPromise } from "./run-tracker.ts";
87
+ import {
88
+ registerRunPromise,
89
+ rejectRunPromise,
90
+ resolveRunPromise,
91
+ } from "./run-tracker.ts";
92
+ import { resolveTaskRuntimeKind } from "./runtime-policy.ts";
93
+ import type { CrewRuntimeCapabilities } from "./runtime-resolver.ts";
94
+ import { recordsForMaterializedTasks } from "./task-display.ts";
95
+ import {
96
+ buildExecutionPlan as buildDagExecutionPlan,
97
+ getReadyTasks as getDagReadyTasks,
98
+ type TaskNode,
99
+ } from "./task-graph.ts";
100
+ import {
101
+ buildTaskGraphIndex,
102
+ refreshTaskGraphQueues,
103
+ taskGraphSnapshot,
104
+ } from "./task-graph-scheduler.ts";
105
+ import { aggregateTaskOutputs } from "./task-output-context.ts";
106
+ import { runTeamTask } from "./task-runner.ts";
38
107
  import { clearTrackedTaskUsage } from "./usage-tracker.ts";
39
- import { CrewCancellationError, buildSyntheticTerminalEvidence, cancellationReasonFromSignal } from "./cancellation.ts";
40
- import { effectivenessPolicyDecision, evaluateRunEffectiveness, formatRunEffectivenessLines } from "./effectiveness.ts";
41
- import { logInternalError } from "../utils/internal-error.ts";
42
- import { PluginRegistry } from "../plugins/plugin-registry.ts";
43
- import { NextJsPlugin, VitestPlugin, VitePlugin } from "../plugins/plugins/index.ts";
44
- import { HealthStore } from "../state/health-store.ts";
108
+ import {
109
+ createWorkflowStateMachine,
110
+ type PhaseGuardContext,
111
+ type PhaseState,
112
+ transitionPhase,
113
+ validatePhasePreconditions,
114
+ } from "./workflow-state.ts";
45
115
 
46
116
  // Built-in plugin registry for framework awareness.
47
117
  // NOTE: This registry is registered here for future use. The integration
@@ -72,13 +142,17 @@ function startTeamRunHeartbeat(stateRoot: string, runId: string): () => void {
72
142
  // captured manifest.updatedAt once at startup, making the value
73
143
  // permanently stale throughout the run.
74
144
  const now = new Date().toISOString();
75
- fs.writeFileSync(heartbeatPath, JSON.stringify({
76
- pid: process.pid,
77
- at: Date.now(),
78
- runId,
79
- kind: "team-runner",
80
- lastTaskUpdateAt: now,
81
- }), { encoding: "utf-8", mode: 0o600 });
145
+ fs.writeFileSync(
146
+ heartbeatPath,
147
+ JSON.stringify({
148
+ pid: process.pid,
149
+ at: Date.now(),
150
+ runId,
151
+ kind: "team-runner",
152
+ lastTaskUpdateAt: now,
153
+ }),
154
+ { encoding: "utf-8", mode: 0o600 },
155
+ );
82
156
  } catch {
83
157
  // best-effort
84
158
  }
@@ -118,19 +192,39 @@ export interface ExecuteTeamRunInput {
118
192
  }
119
193
 
120
194
  function findStep(workflow: WorkflowConfig, task: TeamTaskState): WorkflowStep {
121
- const step = workflow.steps.find((candidate) => candidate.id === task.stepId);
122
- if (!step) throw new Error(`Workflow step '${task.stepId}' not found for task '${task.id}'.`);
195
+ const step = workflow.steps.find(
196
+ (candidate) => candidate.id === task.stepId,
197
+ );
198
+ if (!step)
199
+ throw new Error(
200
+ `Workflow step '${task.stepId}' not found for task '${task.id}'.`,
201
+ );
123
202
  return step;
124
203
  }
125
204
 
126
205
  function findAgent(agents: AgentConfig[], task: TeamTaskState): AgentConfig {
127
206
  const agent = agents.find((candidate) => candidate.name === task.agent);
128
- if (!agent) throw new Error(`Agent '${task.agent}' not found for task '${task.id}'.`);
207
+ if (!agent)
208
+ throw new Error(
209
+ `Agent '${task.agent}' not found for task '${task.id}'.`,
210
+ );
129
211
  return agent;
130
212
  }
131
213
 
132
214
  function markBlocked(tasks: TeamTaskState[], reason: string): TeamTaskState[] {
133
- return tasks.map((task) => task.status === "queued" ? { ...task, status: "skipped", error: reason, finishedAt: new Date().toISOString(), graph: task.graph ? { ...task.graph, queue: "blocked" } : undefined } : task);
215
+ return tasks.map((task) =>
216
+ task.status === "queued"
217
+ ? {
218
+ ...task,
219
+ status: "skipped",
220
+ error: reason,
221
+ finishedAt: new Date().toISOString(),
222
+ graph: task.graph
223
+ ? { ...task.graph, queue: "blocked" }
224
+ : undefined,
225
+ }
226
+ : task,
227
+ );
134
228
  }
135
229
 
136
230
  function mergeArtifacts(items: ArtifactDescriptor[]): ArtifactDescriptor[] {
@@ -159,37 +253,59 @@ function safeFinishedAt(task: TeamTaskState): number {
159
253
  * and the updated task has a valid finite finishedAt. Malformed finishedAt
160
254
  * should be replaced rather than persisting corruption.
161
255
  */
162
- function isMalformedFinishedAtReplacement(currentTime: number, updatedTime: number): boolean {
256
+ function isMalformedFinishedAtReplacement(
257
+ currentTime: number,
258
+ updatedTime: number,
259
+ ): boolean {
163
260
  return !Number.isFinite(currentTime) && Number.isFinite(updatedTime);
164
261
  }
165
262
 
166
- function shouldMergeTaskUpdate(current: TeamTaskState, updated: TeamTaskState): boolean {
263
+ function shouldMergeTaskUpdate(
264
+ current: TeamTaskState,
265
+ updated: TeamTaskState,
266
+ ): boolean {
167
267
  // Parallel workers receive the same input snapshot. A later result may still
168
268
  // contain stale queued/running copies of tasks that another worker already
169
269
  // completed. Never let those stale snapshots regress durable task state.
170
- if (current.status === "waiting" && updated.status === "running") return false;
270
+ if (current.status === "waiting" && updated.status === "running")
271
+ return false;
171
272
  // Block terminal→non-terminal transitions (e.g. completed→running).
172
273
  // A task that has reached a terminal state must not be resurrected.
173
274
  const currentIsTerminal = !isNonTerminalTaskStatus(current.status);
174
275
  const updatedIsNonTerminal = isNonTerminalTaskStatus(updated.status);
175
276
  if (currentIsTerminal && updatedIsNonTerminal) return false;
176
277
  // Explicitly block completed↔needs_attention terminal-to-terminal transitions.
177
- // Both are success terminal states used interchangeably; stale worker updates must
178
- // not cause a completed task to appear as needs_attention or vice versa.
179
- if (current.status === "completed" && updated.status === "needs_attention") return false;
180
- if (current.status === "needs_attention" && updated.status === "completed") return false;
278
+ // Both are success terminal states used interchangeably; stale worker updates must
279
+ // not cause a completed task to appear as needs_attention or vice versa.
280
+ if (current.status === "completed" && updated.status === "needs_attention")
281
+ return false;
282
+ if (current.status === "needs_attention" && updated.status === "completed")
283
+ return false;
181
284
  // Explicitly block failed→completed resurrection. Both statuses are terminal,
182
285
  // but completed is the success terminal state and should not be reachable from
183
286
  // failed via a stale merge. The check above only guards non-terminal→terminal.
184
- if (current.status === "failed" && updated.status === "completed") return false;
287
+ if (current.status === "failed" && updated.status === "completed")
288
+ return false;
185
289
  // Guard: when current is "running" but has resultArtifact (another worker already
186
290
  // completed it), a stale updated with status="running" and no resultArtifact
187
291
  // must not overwrite the actual completed state.
188
- if (current.status === updated.status && updated.status === "running" && Boolean(current.resultArtifact) && !updated.resultArtifact) return false;
292
+ if (
293
+ current.status === updated.status &&
294
+ updated.status === "running" &&
295
+ current.resultArtifact &&
296
+ !updated.resultArtifact
297
+ )
298
+ return false;
189
299
  // Guard: when current is "completed" and has resultArtifact but updated is also
190
300
  // "completed" without resultArtifact, block the stale update from overwriting
191
301
  // a task that successfully produced output.
192
- if (current.status === updated.status && current.status === "completed" && Boolean(current.resultArtifact) && !updated.resultArtifact) return false;
302
+ if (
303
+ current.status === updated.status &&
304
+ current.status === "completed" &&
305
+ current.resultArtifact &&
306
+ !updated.resultArtifact
307
+ )
308
+ return false;
193
309
  // Prevent a stale completed task from overwriting a fresher one.
194
310
  // Restructure to handle undefined current.finishedAt as a special case:
195
311
  // - undefined current + valid updated: allow the update
@@ -202,7 +318,9 @@ if (current.status === "needs_attention" && updated.status === "completed") retu
202
318
  // Malformed finishedAt (NaN) is treated as Infinity — invalid state should be
203
319
  // replaced rather than persisting corruption. Log warning for visibility.
204
320
  if (!Number.isFinite(currentTime)) {
205
- console.warn(`[team-runner] Task ${current.id} has malformed finishedAt: ${current.finishedAt}`);
321
+ console.warn(
322
+ `[team-runner] Task ${current.id} has malformed finishedAt: ${current.finishedAt}`,
323
+ );
206
324
  }
207
325
  if (isMalformedFinishedAtReplacement(currentTime, updatedTime)) {
208
326
  return true;
@@ -212,29 +330,35 @@ if (current.status === "needs_attention" && updated.status === "completed") retu
212
330
  // Block if updated is trying to establish a terminal status without a finishedAt
213
331
  // timestamp. Heartbeat-only updates (status='running', no finishedAt) are
214
332
  // allowed if heartbeat has changed (checked separately in hasMeaningfulUpdate).
215
- if (!updated.finishedAt && !isNonTerminalTaskStatus(updated.status)) return false;
333
+ if (!updated.finishedAt && !isNonTerminalTaskStatus(updated.status))
334
+ return false;
216
335
  // Explicitly enumerate all fields that constitute a meaningful update so that
217
- // adding a new important field requires updating this list (rather than silently
218
- // losing data if a field is forgotten in the boolean OR chain below).
219
- const hasMeaningfulUpdate =
220
- updated.status !== current.status ||
221
- updated.finishedAt !== current.finishedAt ||
222
- updated.startedAt !== current.startedAt ||
223
- Boolean(updated.resultArtifact) !== Boolean(current.resultArtifact) ||
224
- (Boolean(updated.resultArtifact) && updated.resultArtifact !== current.resultArtifact) ||
225
- Boolean(updated.error) ||
226
- Boolean(updated.modelAttempts?.length) ||
227
- Boolean(updated.usage) ||
228
- Boolean(updated.attempts?.length) ||
229
- updated.heartbeat?.lastSeenAt !== current.heartbeat?.lastSeenAt ||
230
- updated.jsonEvents !== current.jsonEvents ||
231
- updated.agentProgress?.lastActivityAt !== current.agentProgress?.lastActivityAt;
232
- return hasMeaningfulUpdate;
336
+ // adding a new important field requires updating this list (rather than silently
337
+ // losing data if a field is forgotten in the boolean OR chain below).
338
+ const hasMeaningfulUpdate =
339
+ updated.status !== current.status ||
340
+ updated.finishedAt !== current.finishedAt ||
341
+ updated.startedAt !== current.startedAt ||
342
+ Boolean(updated.resultArtifact) !== Boolean(current.resultArtifact) ||
343
+ (Boolean(updated.resultArtifact) &&
344
+ updated.resultArtifact !== current.resultArtifact) ||
345
+ Boolean(updated.error) ||
346
+ Boolean(updated.modelAttempts?.length) ||
347
+ Boolean(updated.usage) ||
348
+ Boolean(updated.attempts?.length) ||
349
+ updated.heartbeat?.lastSeenAt !== current.heartbeat?.lastSeenAt ||
350
+ updated.jsonEvents !== current.jsonEvents ||
351
+ updated.agentProgress?.lastActivityAt !==
352
+ current.agentProgress?.lastActivityAt;
353
+ return hasMeaningfulUpdate;
233
354
  }
234
355
 
235
356
  // H4 fix: rename to descriptive name. Kept __test__ as alias for backward
236
357
  // compat test imports.
237
- export function mergeTaskUpdatesPreservingTerminal(base: TeamTaskState[], results: Array<{ tasks: TeamTaskState[] }>): TeamTaskState[] {
358
+ export function mergeTaskUpdatesPreservingTerminal(
359
+ base: TeamTaskState[],
360
+ results: Array<{ tasks: TeamTaskState[] }>,
361
+ ): TeamTaskState[] {
238
362
  let merged = base;
239
363
  for (const result of results) {
240
364
  for (const updated of result.tasks) {
@@ -244,15 +368,21 @@ export function mergeTaskUpdatesPreservingTerminal(base: TeamTaskState[], result
244
368
  // Log skipped merges for visibility into rejected parallel updates.
245
369
  // In distributed systems with parallel workers, rejected merges may
246
370
  // indicate bugs (wrong status, timestamp corruption) if they accumulate.
247
- console.debug("[team-runner] Skipping stale merge for task", updated.id, {
248
- currentStatus: current.status,
249
- updatedStatus: updated.status,
250
- currentFinishedAt: current.finishedAt,
251
- updatedFinishedAt: updated.finishedAt,
252
- });
371
+ console.debug(
372
+ "[team-runner] Skipping stale merge for task",
373
+ updated.id,
374
+ {
375
+ currentStatus: current.status,
376
+ updatedStatus: updated.status,
377
+ currentFinishedAt: current.finishedAt,
378
+ updatedFinishedAt: updated.finishedAt,
379
+ },
380
+ );
253
381
  continue;
254
382
  }
255
- merged = merged.map((task) => task.id === updated.id ? updated : task);
383
+ merged = merged.map((task) =>
384
+ task.id === updated.id ? updated : task,
385
+ );
256
386
  }
257
387
  }
258
388
  return refreshTaskGraphQueues(merged);
@@ -262,20 +392,43 @@ export const __test__mergeTaskUpdates = mergeTaskUpdatesPreservingTerminal;
262
392
 
263
393
  // 2.8: adaptive-plan parsing/repair/injection moved to src/runtime/adaptive-plan.ts.
264
394
  // Re-export the test-only helpers so existing test imports still resolve.
265
- export { __test__parseAdaptivePlan, __test__repairAdaptivePlan } from "./adaptive-plan.ts";
395
+ export {
396
+ __test__parseAdaptivePlan,
397
+ __test__repairAdaptivePlan,
398
+ } from "./adaptive-plan.ts";
399
+
266
400
  import { injectAdaptivePlanIfReady } from "./adaptive-plan.ts";
267
401
 
268
402
  function formatTaskProgress(task: TeamTaskState): string {
269
403
  return `- ${task.id}: ${task.status} (${task.role} -> ${task.agent})${task.taskPacket ? ` scope=${task.taskPacket.scope}` : ""}${task.verification ? ` green=${task.verification.observedGreenLevel}/${task.verification.requiredGreenLevel}` : ""}${task.error ? ` - ${task.error}` : ""}`;
270
404
  }
271
405
 
272
- function runEffectivenessLines(manifest: TeamRunManifest, tasks: TeamTaskState[], executeWorkers: boolean, runtimeConfig?: CrewRuntimeConfig): string[] {
273
- return formatRunEffectivenessLines(evaluateRunEffectiveness({ manifest, tasks, executeWorkers, runtimeConfig }));
406
+ function runEffectivenessLines(
407
+ manifest: TeamRunManifest,
408
+ tasks: TeamTaskState[],
409
+ executeWorkers: boolean,
410
+ runtimeConfig?: CrewRuntimeConfig,
411
+ ): string[] {
412
+ return formatRunEffectivenessLines(
413
+ evaluateRunEffectiveness({
414
+ manifest,
415
+ tasks,
416
+ executeWorkers,
417
+ runtimeConfig,
418
+ }),
419
+ );
274
420
  }
275
421
 
276
- function writeProgress(manifest: TeamRunManifest, tasks: TeamTaskState[], producer: string, executeWorkers = true, runtimeConfig?: CrewRuntimeConfig): TeamRunManifest {
422
+ function writeProgress(
423
+ manifest: TeamRunManifest,
424
+ tasks: TeamTaskState[],
425
+ producer: string,
426
+ executeWorkers = true,
427
+ runtimeConfig?: CrewRuntimeConfig,
428
+ ): TeamRunManifest {
277
429
  const counts = new Map<string, number>();
278
- for (const task of tasks) counts.set(task.status, (counts.get(task.status) ?? 0) + 1);
430
+ for (const task of tasks)
431
+ counts.set(task.status, (counts.get(task.status) ?? 0) + 1);
279
432
  const queue = taskGraphSnapshot(tasks);
280
433
  const progress = writeArtifact(manifest.artifactsRoot, {
281
434
  kind: "progress",
@@ -295,14 +448,39 @@ function writeProgress(manifest: TeamRunManifest, tasks: TeamTaskState[], produc
295
448
  ...tasks.map(formatTaskProgress),
296
449
  "",
297
450
  "## Effectiveness",
298
- ...runEffectivenessLines(manifest, tasks, executeWorkers, runtimeConfig),
451
+ ...runEffectivenessLines(
452
+ manifest,
453
+ tasks,
454
+ executeWorkers,
455
+ runtimeConfig,
456
+ ),
299
457
  "",
300
458
  ].join("\n"),
301
459
  });
302
- return { ...manifest, updatedAt: new Date().toISOString(), artifacts: [...manifest.artifacts.filter((artifact) => !(artifact.kind === "progress" && artifact.path === progress.path)), progress].filter((artifact, index, self) => self.findIndex((a) => a.path === artifact.path) === index) };
460
+ return {
461
+ ...manifest,
462
+ updatedAt: new Date().toISOString(),
463
+ artifacts: [
464
+ ...manifest.artifacts.filter(
465
+ (artifact) =>
466
+ !(
467
+ artifact.kind === "progress" &&
468
+ artifact.path === progress.path
469
+ ),
470
+ ),
471
+ progress,
472
+ ].filter(
473
+ (artifact, index, self) =>
474
+ self.findIndex((a) => a.path === artifact.path) === index,
475
+ ),
476
+ };
303
477
  }
304
478
 
305
- function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?: CrewLimitsConfig): TeamRunManifest {
479
+ function applyPolicy(
480
+ manifest: TeamRunManifest,
481
+ tasks: TeamTaskState[],
482
+ limits?: CrewLimitsConfig,
483
+ ): TeamRunManifest {
306
484
  const branchFreshness = checkBranchFreshness(manifest.cwd);
307
485
  const branchArtifact = writeArtifact(manifest.artifactsRoot, {
308
486
  kind: "metadata",
@@ -310,8 +488,15 @@ function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?:
310
488
  producer: "branch-freshness",
311
489
  content: `${JSON.stringify(branchFreshness, null, 2)}\n`,
312
490
  });
313
- let decisions: PolicyDecision[] = evaluateCrewPolicy({ manifest, tasks, limits });
314
- if (branchFreshness.status === "stale" || branchFreshness.status === "diverged") {
491
+ let decisions: PolicyDecision[] = evaluateCrewPolicy({
492
+ manifest,
493
+ tasks,
494
+ limits,
495
+ });
496
+ if (
497
+ branchFreshness.status === "stale" ||
498
+ branchFreshness.status === "diverged"
499
+ ) {
315
500
  const branchDecision: PolicyDecision = {
316
501
  action: "notify",
317
502
  reason: "branch_stale",
@@ -319,7 +504,12 @@ function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?:
319
504
  createdAt: new Date().toISOString(),
320
505
  };
321
506
  decisions = [...decisions, branchDecision];
322
- appendEvent(manifest.eventsPath, { type: "branch.stale", runId: manifest.runId, message: branchFreshness.message, data: { branchFreshness } });
507
+ appendEvent(manifest.eventsPath, {
508
+ type: "branch.stale",
509
+ runId: manifest.runId,
510
+ message: branchFreshness.message,
511
+ data: { branchFreshness },
512
+ });
323
513
  }
324
514
  const policyArtifact = writeArtifact(manifest.artifactsRoot, {
325
515
  kind: "metadata",
@@ -334,20 +524,85 @@ function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?:
334
524
  producer: "recovery-engine",
335
525
  content: `${JSON.stringify(recoveryLedger, null, 2)}\n`,
336
526
  });
337
- for (const item of decisions) appendEvent(manifest.eventsPath, { type: item.action === "escalate" ? "policy.escalated" : "policy.action", runId: manifest.runId, taskId: item.taskId, message: item.message, data: { action: item.action, reason: item.reason } });
338
- for (const item of recoveryLedger.entries) appendEvent(manifest.eventsPath, { type: item.state === "escalation_required" ? "recovery.escalated" : "recovery.attempted", runId: manifest.runId, taskId: item.taskId, message: item.message, data: { scenario: item.scenario, steps: item.steps, attempt: item.attempt, state: item.state } });
339
- return { ...manifest, updatedAt: new Date().toISOString(), policyDecisions: decisions, artifacts: [...manifest.artifacts.filter((artifact) => !(artifact.kind === "metadata" && (artifact.path.endsWith("policy-decisions.json") || artifact.path.endsWith("recovery-ledger.json") || artifact.path.endsWith("branch-freshness.json")))), branchArtifact, policyArtifact, recoveryArtifact] };
527
+ for (const item of decisions)
528
+ appendEvent(manifest.eventsPath, {
529
+ type:
530
+ item.action === "escalate"
531
+ ? "policy.escalated"
532
+ : "policy.action",
533
+ runId: manifest.runId,
534
+ taskId: item.taskId,
535
+ message: item.message,
536
+ data: { action: item.action, reason: item.reason },
537
+ });
538
+ for (const item of recoveryLedger.entries)
539
+ appendEvent(manifest.eventsPath, {
540
+ type:
541
+ item.state === "escalation_required"
542
+ ? "recovery.escalated"
543
+ : "recovery.attempted",
544
+ runId: manifest.runId,
545
+ taskId: item.taskId,
546
+ message: item.message,
547
+ data: {
548
+ scenario: item.scenario,
549
+ steps: item.steps,
550
+ attempt: item.attempt,
551
+ state: item.state,
552
+ },
553
+ });
554
+ return {
555
+ ...manifest,
556
+ updatedAt: new Date().toISOString(),
557
+ policyDecisions: decisions,
558
+ artifacts: [
559
+ ...manifest.artifacts.filter(
560
+ (artifact) =>
561
+ !(
562
+ artifact.kind === "metadata" &&
563
+ (artifact.path.endsWith("policy-decisions.json") ||
564
+ artifact.path.endsWith("recovery-ledger.json") ||
565
+ artifact.path.endsWith("branch-freshness.json"))
566
+ ),
567
+ ),
568
+ branchArtifact,
569
+ policyArtifact,
570
+ recoveryArtifact,
571
+ ],
572
+ };
340
573
  }
341
574
 
342
- function retryPolicyFromConfig(config: CrewReliabilityConfig | undefined): RetryPolicy {
575
+ function retryPolicyFromConfig(
576
+ config: CrewReliabilityConfig | undefined,
577
+ ): RetryPolicy {
343
578
  return { ...DEFAULT_RETRY_POLICY, ...(config?.retryPolicy ?? {}) };
344
579
  }
345
580
 
346
- function failedTaskFrom(result: { tasks: TeamTaskState[] }, taskId: string): TeamTaskState | undefined {
347
- return result.tasks.find((item) => item.id === taskId && item.status === "failed");
581
+ /**
582
+ * #1 (assessment): decide whether the per-task retry path (executeWithRetry) is used.
583
+ * Defaults to TRUE (opt-out) so transient worker hangs (ChildTimeout) are retried
584
+ * automatically. Previously opt-in, which left the entire retry+recovery stack dormant.
585
+ * Exported for unit testing.
586
+ */
587
+ export function shouldUseRetry(
588
+ reliability: CrewReliabilityConfig | undefined,
589
+ ): boolean {
590
+ return reliability?.autoRetry !== false;
591
+ }
592
+
593
+ function failedTaskFrom(
594
+ result: { tasks: TeamTaskState[] },
595
+ taskId: string,
596
+ ): TeamTaskState | undefined {
597
+ return result.tasks.find(
598
+ (item) => item.id === taskId && item.status === "failed",
599
+ );
348
600
  }
349
601
 
350
- function requiresPlanApproval(_workflow: WorkflowConfig, runtimeConfig: CrewRuntimeConfig | undefined): boolean {
602
+ function requiresPlanApproval(
603
+ _workflow: WorkflowConfig,
604
+ runtimeConfig: CrewRuntimeConfig | undefined,
605
+ ): boolean {
351
606
  // ROADMAP T1.2: plan-level HITL applies to ANY workflow when
352
607
  // config.runtime.requirePlanApproval === true (not just 'implementation').
353
608
  // The gate fires at the read-only → mutating (plan → execute) boundary.
@@ -355,19 +610,31 @@ function requiresPlanApproval(_workflow: WorkflowConfig, runtimeConfig: CrewRunt
355
610
  }
356
611
 
357
612
  function isPlanApprovalPending(manifest: TeamRunManifest): boolean {
358
- return manifest.planApproval?.required === true && manifest.planApproval.status === "pending";
613
+ return (
614
+ manifest.planApproval?.required === true &&
615
+ manifest.planApproval.status === "pending"
616
+ );
359
617
  }
360
618
 
361
619
  function isMutatingTask(task: TeamTaskState): boolean {
362
620
  return permissionForRole(task.role) !== "read_only";
363
621
  }
364
622
 
365
- function ensurePlanApprovalRequested(manifest: TeamRunManifest, tasks: TeamTaskState[]): TeamRunManifest {
623
+ function ensurePlanApprovalRequested(
624
+ manifest: TeamRunManifest,
625
+ tasks: TeamTaskState[],
626
+ ): TeamRunManifest {
366
627
  if (manifest.planApproval) return manifest;
367
- const assessTask = tasks.find((task) => task.stepId === "assess" && task.status === "completed");
628
+ const assessTask = tasks.find(
629
+ (task) => task.stepId === "assess" && task.status === "completed",
630
+ );
368
631
  // ROADMAP T1.2: for non-adaptive workflows, fall back to the most recent
369
632
  // completed read-only (planning) task as the plan reference.
370
- const planTask = assessTask ?? [...tasks].reverse().find((t) => t.status === "completed" && !isMutatingTask(t));
633
+ const planTask =
634
+ assessTask ??
635
+ [...tasks]
636
+ .reverse()
637
+ .find((t) => t.status === "completed" && !isMutatingTask(t));
371
638
  const now = new Date().toISOString();
372
639
  const updated: TeamRunManifest = {
373
640
  ...manifest,
@@ -382,16 +649,43 @@ function ensurePlanApprovalRequested(manifest: TeamRunManifest, tasks: TeamTaskS
382
649
  },
383
650
  };
384
651
  saveRunManifest(updated);
385
- appendEvent(updated.eventsPath, { type: "plan.approval_required", runId: updated.runId, taskId: planTask?.id, message: "Plan requires explicit approval before mutating tasks run. Use: team api op=approve-plan runId=...", data: { planArtifactPath: planTask?.resultArtifact?.path } });
652
+ appendEvent(updated.eventsPath, {
653
+ type: "plan.approval_required",
654
+ runId: updated.runId,
655
+ taskId: planTask?.id,
656
+ message:
657
+ "Plan requires explicit approval before mutating tasks run. Use: team api op=approve-plan runId=...",
658
+ data: { planArtifactPath: planTask?.resultArtifact?.path },
659
+ });
386
660
  return updated;
387
661
  }
388
662
 
389
- function cancelPlanTasks(tasks: TeamTaskState[], reason: string): TeamTaskState[] {
390
- return tasks.map((task) => task.status === "queued" || task.status === "running" || task.status === "waiting" ? { ...task, status: "cancelled", finishedAt: new Date().toISOString(), error: reason, graph: task.graph ? { ...task.graph, queue: "done" } : undefined } : task);
663
+ function cancelPlanTasks(
664
+ tasks: TeamTaskState[],
665
+ reason: string,
666
+ ): TeamTaskState[] {
667
+ return tasks.map((task) =>
668
+ task.status === "queued" ||
669
+ task.status === "running" ||
670
+ task.status === "waiting"
671
+ ? {
672
+ ...task,
673
+ status: "cancelled",
674
+ finishedAt: new Date().toISOString(),
675
+ error: reason,
676
+ graph: task.graph
677
+ ? { ...task.graph, queue: "done" }
678
+ : undefined,
679
+ }
680
+ : task,
681
+ );
391
682
  }
392
683
 
393
684
  function hasPendingMutatingAdaptiveTask(tasks: TeamTaskState[]): boolean {
394
- return tasks.some((task) => task.status === "queued" && task.adaptive && isMutatingTask(task));
685
+ return tasks.some(
686
+ (task) =>
687
+ task.status === "queued" && task.adaptive && isMutatingTask(task),
688
+ );
395
689
  }
396
690
 
397
691
  /**
@@ -399,9 +693,15 @@ function hasPendingMutatingAdaptiveTask(tasks: TeamTaskState[]): boolean {
399
693
  * Fires when there are pending mutating tasks whose prerequisites (read-only
400
694
  * tasks) have completed — i.e. the plan→execute boundary.
401
695
  */
402
- export function hasPendingMutatingTaskAtBoundary(tasks: TeamTaskState[]): boolean {
403
- const hasCompletedReadOnly = tasks.some((t) => t.status === "completed" && !isMutatingTask(t));
404
- const hasPendingMutating = tasks.some((t) => t.status === "queued" && isMutatingTask(t));
696
+ export function hasPendingMutatingTaskAtBoundary(
697
+ tasks: TeamTaskState[],
698
+ ): boolean {
699
+ const hasCompletedReadOnly = tasks.some(
700
+ (t) => t.status === "completed" && !isMutatingTask(t),
701
+ );
702
+ const hasPendingMutating = tasks.some(
703
+ (t) => t.status === "queued" && isMutatingTask(t),
704
+ );
405
705
  return hasCompletedReadOnly && hasPendingMutating;
406
706
  }
407
707
 
@@ -410,7 +710,10 @@ export function hasPendingMutatingTaskAtBoundary(tasks: TeamTaskState[]): boolea
410
710
  * execution planning. If so, build an execution plan and use `getDagReadyTasks`
411
711
  * to augment the ready-set selection.
412
712
  */
413
- function dagReadyTaskIds(tasks: TeamTaskState[], completedIds: Set<string>): string[] | null {
713
+ function dagReadyTaskIds(
714
+ tasks: TeamTaskState[],
715
+ completedIds: Set<string>,
716
+ ): string[] | null {
414
717
  const hasExplicitDeps = tasks.some((t) => t.dependsOn.length > 0);
415
718
  if (!hasExplicitDeps) return null;
416
719
  // FIX (goal-wrap runtime test): task.dependsOn stores STEP IDs (e.g. "execute"), not
@@ -433,9 +736,53 @@ function dagReadyTaskIds(tasks: TeamTaskState[], completedIds: Set<string>): str
433
736
  return getDagReadyTasks(plan, completedIds);
434
737
  }
435
738
 
436
- export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
739
+ export async function executeTeamRun(
740
+ input: ExecuteTeamRunInput,
741
+ ): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
437
742
  const workflow = input.workflow;
438
- let manifest = updateRunStatus(input.manifest, "running", input.executeWorkers ? "Executing team workflow." : "Creating workflow prompts and placeholder results.");
743
+
744
+ // DEFENSE-IN-DEPTH (advisory-only since v0.9.15): re-validate topology here in
745
+ // case a caller bypassed the extension-layer handleRun guard. The extension
746
+ // layer logs an advisory note and proceeds; this defense-in-depth does the same
747
+ // for direct API callers (CLI, tests, scheduler). Never blocks.
748
+ // Skip for synthetic direct-agent workflows (filePath="<generated>").
749
+ if (workflow.filePath !== "<generated>") {
750
+ // LAZY: defer preflight-validator import until the defense-in-depth guard actually runs.
751
+ const { validateWorkflowUsage } = await import(
752
+ "../workflows/preflight-validator.ts"
753
+ );
754
+ const preflight = validateWorkflowUsage(workflow, {
755
+ force: input.reliability?.forcePreflight === true,
756
+ });
757
+ if (
758
+ preflight.level === "warn" ||
759
+ preflight.level === "note" ||
760
+ preflight.level === "info"
761
+ ) {
762
+ const icon =
763
+ preflight.level === "warn"
764
+ ? "⚠️ "
765
+ : preflight.level === "note"
766
+ ? "✅ "
767
+ : "ℹ️ ";
768
+ console.warn(
769
+ `${icon}[team-runner.preflight] ${preflight.level.toUpperCase()}: ${preflight.message} (workflow=${workflow.name})`,
770
+ );
771
+ if (preflight.suggestion) {
772
+ console.warn(
773
+ `[team-runner.preflight] → ${preflight.suggestion}`,
774
+ );
775
+ }
776
+ }
777
+ }
778
+
779
+ let manifest = updateRunStatus(
780
+ input.manifest,
781
+ "running",
782
+ input.executeWorkers
783
+ ? "Executing team workflow."
784
+ : "Creating workflow prompts and placeholder results.",
785
+ );
439
786
 
440
787
  void registerRunPromise(manifest.runId);
441
788
 
@@ -444,7 +791,10 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
444
791
  // (NO_PID_HEARTBEAT_STALE_MS). Previously only sub-task runners wrote
445
792
  // heartbeats; the team-level run had no heartbeat, so any multi-phase
446
793
  // workflow lasting >5min was marked stale and cancelled.
447
- const stopTeamHeartbeat = startTeamRunHeartbeat(manifest.stateRoot, manifest.runId);
794
+ const stopTeamHeartbeat = startTeamRunHeartbeat(
795
+ manifest.stateRoot,
796
+ manifest.runId,
797
+ );
448
798
 
449
799
  const cleanupUsage = (): void => {
450
800
  for (const task of input.tasks) clearTrackedTaskUsage(task.id);
@@ -452,32 +802,106 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
452
802
 
453
803
  try {
454
804
  const result = await executeTeamRunCore(input, manifest, workflow);
805
+ // #2 (assessment): goal-achievement detection — kill the silent false-green.
806
+ // A code-mutating run that "completed" but left the git working tree clean
807
+ // (and/or had a failed task) is a false-green. We expose goalAchieved on the
808
+ // manifest + emit an event so the lie is never silent, and downgrade status
809
+ // to "failed" only when a failed task corroborates it (conservative).
810
+ const gaAssessment = assessGoalAchievement(
811
+ result.manifest,
812
+ result.tasks,
813
+ workflow,
814
+ );
815
+ const gaApplied = applyGoalAchievement(result.manifest, gaAssessment);
816
+ if (gaApplied.manifest !== result.manifest) {
817
+ result.manifest = gaApplied.manifest;
818
+ try {
819
+ saveRunManifest(result.manifest);
820
+ } catch (persistError) {
821
+ logInternalError(
822
+ "team-runner.goalAchievement.persist",
823
+ persistError instanceof Error
824
+ ? persistError
825
+ : new Error(String(persistError)),
826
+ `runId=${manifest.runId}`,
827
+ );
828
+ }
829
+ }
830
+ appendEvent(manifest.eventsPath, {
831
+ type: "run.goal_achievement",
832
+ runId: manifest.runId,
833
+ message: gaApplied.manifest.goalAchievementNote ?? "",
834
+ data: {
835
+ achieved: gaAssessment.achieved,
836
+ downgraded: gaApplied.downgraded,
837
+ reason: gaAssessment.reason,
838
+ signals: gaAssessment.signals,
839
+ },
840
+ });
841
+ if (gaApplied.downgraded)
842
+ logInternalError(
843
+ "team-runner.goalAchievement.falseGreen",
844
+ new Error(
845
+ gaApplied.manifest.goalAchievementNote ??
846
+ "false-green detected",
847
+ ),
848
+ `runId=${manifest.runId}`,
849
+ );
455
850
  stopTeamHeartbeat();
456
851
  resolveRunPromise(manifest.runId, result);
457
852
  cleanupUsage();
458
853
  // Terminate live agents for this run — agents are done when the run ends.
459
- void terminateLiveAgentsForRun(manifest.runId, "completed", appendEvent, manifest.eventsPath).catch((error) => logInternalError("team-runner.completed.terminate", error, `runId=${manifest.runId}`));
854
+ void terminateLiveAgentsForRun(
855
+ manifest.runId,
856
+ "completed",
857
+ appendEvent,
858
+ manifest.eventsPath,
859
+ ).catch((error) =>
860
+ logInternalError(
861
+ "team-runner.completed.terminate",
862
+ error,
863
+ `runId=${manifest.runId}`,
864
+ ),
865
+ );
460
866
 
461
867
  // Emit run completion hook (100% reliable, fire-and-forget)
462
- crewHooks.emit({ type: "run_completed", timestamp: new Date().toISOString(), runId: manifest.runId, data: { status: result.manifest.status, taskCount: result.tasks.length } });
868
+ crewHooks.emit({
869
+ type: "run_completed",
870
+ timestamp: new Date().toISOString(),
871
+ runId: manifest.runId,
872
+ data: {
873
+ status: result.manifest.status,
874
+ taskCount: result.tasks.length,
875
+ },
876
+ });
463
877
 
464
878
  // Execute after_run_complete lifecycle hook (non-blocking)
465
- const afterRunReport = await executeHook("after_run_complete", { runId: manifest.runId, cwd: manifest.cwd, status: result.manifest.status });
879
+ const afterRunReport = await executeHook("after_run_complete", {
880
+ runId: manifest.runId,
881
+ cwd: manifest.cwd,
882
+ status: result.manifest.status,
883
+ });
466
884
  appendHookEvent(manifest, afterRunReport);
467
885
  if (afterRunReport.outcome === "block") {
468
- logInternalError("team-runner.after_run_complete.blocked", new Error(afterRunReport.reason ?? "after_run_complete hook blocked"), `runId=${manifest.runId}`);
886
+ logInternalError(
887
+ "team-runner.after_run_complete.blocked",
888
+ new Error(
889
+ afterRunReport.reason ?? "after_run_complete hook blocked",
890
+ ),
891
+ `runId=${manifest.runId}`,
892
+ );
469
893
  }
470
894
 
471
895
  return result;
472
896
  } catch (error) {
473
897
  // Round 27 (BUG 1): the success path calls stopTeamHeartbeat() but this
474
- // catch path did NOT. The team heartbeat is a non-unref'd setInterval
475
- // (30s) that deliberately keeps the event loop alive — without this
476
- // call, a failed team run leaves the interval firing forever and the
477
- // foreground pi process hangs (never returns to the prompt); in
478
- // background-runner mode the worker never exits. clearInterval is
479
- // idempotent so a double-call (if this runs after the success path)
480
- // is harmless.
898
+ // catch path did NOT. The team heartbeat is a non-unref'd setInterval
899
+ // (30s) that deliberately keeps the event loop alive — without this
900
+ // call, a failed team run leaves the interval firing forever and the
901
+ // foreground pi process hangs (never returns to the prompt); in
902
+ // background-runner mode the worker never exits. clearInterval is
903
+ // idempotent so a double-call (if this runs after the success path)
904
+ // is harmless.
481
905
  stopTeamHeartbeat();
482
906
  // P1: Catch unhandled errors — ensure manifest/tasks/agents are terminal so they don't stay "running" forever.
483
907
  const message = error instanceof Error ? error.message : String(error);
@@ -485,7 +909,9 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
485
909
  // If lock acquisition fails, use in-memory data rather than stale disk data.
486
910
  let loaded;
487
911
  try {
488
- loaded = await withRunLock(input.manifest, async () => loadRunManifestById(input.manifest.cwd, input.manifest.runId));
912
+ loaded = await withRunLock(input.manifest, async () =>
913
+ loadRunManifestById(input.manifest.cwd, input.manifest.runId),
914
+ );
489
915
  } catch {
490
916
  loaded = undefined; // best-effort: use in-memory data if lock fails
491
917
  }
@@ -493,29 +919,72 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
493
919
  const freshTasks = refreshTaskGraphQueues(loaded?.tasks ?? input.tasks);
494
920
  const failedAt = new Date().toISOString();
495
921
  const tasks = freshTasks.map((task) =>
496
- task.status === "running" || task.status === "queued" || task.status === "waiting"
497
- ? { ...task, status: "failed" as const, finishedAt: failedAt, error: message }
922
+ task.status === "running" ||
923
+ task.status === "queued" ||
924
+ task.status === "waiting"
925
+ ? {
926
+ ...task,
927
+ status: "failed" as const,
928
+ finishedAt: failedAt,
929
+ error: message,
930
+ }
498
931
  : task,
499
932
  );
500
933
  manifest = freshManifest;
501
934
  try {
502
- await terminateLiveAgentsForRun(manifest.runId, "failed", appendEvent, manifest.eventsPath);
935
+ await terminateLiveAgentsForRun(
936
+ manifest.runId,
937
+ "failed",
938
+ appendEvent,
939
+ manifest.eventsPath,
940
+ );
503
941
  await saveRunTasksAsync(manifest, tasks);
504
- const existingRuntimeByTask = new Map(readCrewAgents(manifest).map((agent) => [agent.taskId, agent.runtime]));
942
+ const existingRuntimeByTask = new Map(
943
+ readCrewAgents(manifest).map((agent) => [
944
+ agent.taskId,
945
+ agent.runtime,
946
+ ]),
947
+ );
505
948
  const globalRuntime = input.runtime?.kind ?? "child-process";
506
- const runtimeForAgent = (agent: ReturnType<typeof recordsForMaterializedTasks>[number]): CrewRuntimeKind => {
949
+ const runtimeForAgent = (
950
+ agent: ReturnType<typeof recordsForMaterializedTasks>[number],
951
+ ): CrewRuntimeKind => {
507
952
  const task = tasks.find((item) => item.id === agent.taskId);
508
- return existingRuntimeByTask.get(agent.taskId) ?? resolveTaskRuntimeKind(globalRuntime, task?.role ?? agent.role, input.runtimeConfig?.isolationPolicy);
953
+ return (
954
+ existingRuntimeByTask.get(agent.taskId) ??
955
+ resolveTaskRuntimeKind(
956
+ globalRuntime,
957
+ task?.role ?? agent.role,
958
+ input.runtimeConfig?.isolationPolicy,
959
+ )
960
+ );
509
961
  };
510
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, globalRuntime).map((agent) => ({ ...agent, runtime: runtimeForAgent(agent) })));
511
- manifest = updateRunStatus(manifest, "failed", `Unhandled error in team runner: ${message}`);
962
+ saveCrewAgents(
963
+ manifest,
964
+ recordsForMaterializedTasks(manifest, tasks, globalRuntime).map(
965
+ (agent) => ({ ...agent, runtime: runtimeForAgent(agent) }),
966
+ ),
967
+ );
968
+ manifest = updateRunStatus(
969
+ manifest,
970
+ "failed",
971
+ `Unhandled error in team runner: ${message}`,
972
+ );
512
973
  await saveRunManifestAsync(manifest);
513
974
  } catch {
514
975
  // Best-effort — state write may also fail
515
976
  }
516
977
  const result = { manifest, tasks };
517
- rejectRunPromise(manifest.runId, error instanceof Error ? error : new Error(message));
518
- crewHooks.emit({ type: "run_failed", timestamp: new Date().toISOString(), runId: manifest.runId, data: { status: manifest.status, error: message } });
978
+ rejectRunPromise(
979
+ manifest.runId,
980
+ error instanceof Error ? error : new Error(message),
981
+ );
982
+ crewHooks.emit({
983
+ type: "run_failed",
984
+ timestamp: new Date().toISOString(),
985
+ runId: manifest.runId,
986
+ data: { status: manifest.status, error: message },
987
+ });
519
988
  cleanupUsage();
520
989
  return result;
521
990
  }
@@ -527,10 +996,17 @@ async function executeTeamRunCore(
527
996
  workflow: WorkflowConfig,
528
997
  ): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
529
998
  // Execute before_run_start hook (non-blocking by default)
530
- const beforeRunReport = await executeHook("before_run_start", { runId: manifest.runId, cwd: manifest.cwd });
999
+ const beforeRunReport = await executeHook("before_run_start", {
1000
+ runId: manifest.runId,
1001
+ cwd: manifest.cwd,
1002
+ });
531
1003
  appendHookEvent(manifest, beforeRunReport);
532
1004
  if (beforeRunReport.outcome === "block") {
533
- manifest = updateRunStatus(manifest, "blocked", beforeRunReport.reason ?? "before_run_start hook blocked the run.");
1005
+ manifest = updateRunStatus(
1006
+ manifest,
1007
+ "blocked",
1008
+ beforeRunReport.reason ?? "before_run_start hook blocked the run.",
1009
+ );
534
1010
  return { manifest, tasks: input.tasks };
535
1011
  }
536
1012
  let tasks = refreshTaskGraphQueues(input.tasks);
@@ -539,45 +1015,94 @@ async function executeTeamRunCore(
539
1015
  let adaptivePlanInjected = false;
540
1016
  let adaptivePlanMissing = false;
541
1017
  const attemptAdaptivePlan = () => {
542
- if (!canInjectAdaptivePlan || adaptivePlanInjected || adaptivePlanMissing) return { injected: false, missing: false };
543
- const adaptivePlan = injectAdaptivePlanIfReady({ manifest, tasks, workflow, team: input.team });
1018
+ if (
1019
+ !canInjectAdaptivePlan ||
1020
+ adaptivePlanInjected ||
1021
+ adaptivePlanMissing
1022
+ )
1023
+ return { injected: false, missing: false };
1024
+ const adaptivePlan = injectAdaptivePlanIfReady({
1025
+ manifest,
1026
+ tasks,
1027
+ workflow,
1028
+ team: input.team,
1029
+ });
544
1030
  adaptivePlanInjected = adaptivePlanInjected || adaptivePlan.injected;
545
1031
  adaptivePlanMissing = adaptivePlan.missingPlan;
546
1032
  workflow = adaptivePlan.workflow;
547
1033
  if (adaptivePlan.injected) tasks = adaptivePlan.tasks;
548
- return { injected: adaptivePlan.injected, missing: adaptivePlan.missingPlan };
1034
+ return {
1035
+ injected: adaptivePlan.injected,
1036
+ missing: adaptivePlan.missingPlan,
1037
+ };
549
1038
  };
550
1039
  const initialAdaptive = attemptAdaptivePlan();
551
1040
  if (initialAdaptive.missing) {
552
- tasks = markBlocked(tasks, "Adaptive planner did not produce a valid subagent plan.");
1041
+ tasks = markBlocked(
1042
+ tasks,
1043
+ "Adaptive planner did not produce a valid subagent plan.",
1044
+ );
553
1045
  await saveRunTasksAsync(manifest, tasks);
554
- manifest = updateRunStatus(manifest, "blocked", "Adaptive planner did not produce a valid subagent plan.");
1046
+ manifest = updateRunStatus(
1047
+ manifest,
1048
+ "blocked",
1049
+ "Adaptive planner did not produce a valid subagent plan.",
1050
+ );
555
1051
  return { manifest, tasks };
556
1052
  }
557
1053
  if (initialAdaptive.injected) {
558
- manifest = requiresPlanApproval(workflow, input.runtimeConfig) ? ensurePlanApprovalRequested(manifest, tasks) : manifest;
1054
+ manifest = requiresPlanApproval(workflow, input.runtimeConfig)
1055
+ ? ensurePlanApprovalRequested(manifest, tasks)
1056
+ : manifest;
559
1057
  queueIndex = buildTaskGraphIndex(tasks);
560
- } else if (requiresPlanApproval(workflow, input.runtimeConfig) && (hasPendingMutatingAdaptiveTask(tasks) || hasPendingMutatingTaskAtBoundary(tasks))) {
1058
+ } else if (
1059
+ requiresPlanApproval(workflow, input.runtimeConfig) &&
1060
+ (hasPendingMutatingAdaptiveTask(tasks) ||
1061
+ hasPendingMutatingTaskAtBoundary(tasks))
1062
+ ) {
561
1063
  manifest = ensurePlanApprovalRequested(manifest, tasks);
562
1064
  }
563
1065
  if (manifest.planApproval?.status === "cancelled") {
564
1066
  tasks = cancelPlanTasks(tasks, "Plan approval was cancelled.");
565
1067
  await saveRunTasksAsync(manifest, tasks);
566
- manifest = updateRunStatus(manifest, "cancelled", "Plan approval was cancelled.");
1068
+ manifest = updateRunStatus(
1069
+ manifest,
1070
+ "cancelled",
1071
+ "Plan approval was cancelled.",
1072
+ );
567
1073
  return { manifest, tasks };
568
1074
  }
569
- manifest = writeProgress(manifest, tasks, "team-runner", input.executeWorkers, input.runtimeConfig);
1075
+ manifest = writeProgress(
1076
+ manifest,
1077
+ tasks,
1078
+ "team-runner",
1079
+ input.executeWorkers,
1080
+ input.runtimeConfig,
1081
+ );
570
1082
  await saveRunManifestAsync(manifest);
571
- const runtimeKind = input.runtime?.kind ?? (input.executeWorkers ? "child-process" : "scaffold");
572
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
1083
+ const runtimeKind =
1084
+ input.runtime?.kind ??
1085
+ (input.executeWorkers ? "child-process" : "scaffold");
1086
+ saveCrewAgents(
1087
+ manifest,
1088
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1089
+ );
573
1090
 
574
1091
  // Build a workflow phase state machine from workflow steps for precondition tracking.
575
- const workflowPhases: PhaseState[] = workflow.steps.map((step): PhaseState => ({
576
- name: step.id,
577
- status: "pending",
578
- inputs: step.reads === false ? [] : Array.isArray(step.reads) ? step.reads : [],
579
- outputs: step.output === false ? [] : step.output ? [step.output] : [],
580
- }));
1092
+ const workflowPhases: PhaseState[] = workflow.steps.map(
1093
+ (step): PhaseState => ({
1094
+ name: step.id,
1095
+ status: "pending",
1096
+ inputs:
1097
+ step.reads === false
1098
+ ? []
1099
+ : Array.isArray(step.reads)
1100
+ ? step.reads
1101
+ : [],
1102
+ outputs:
1103
+ step.output === false ? [] : step.output ? [step.output] : [],
1104
+ }),
1105
+ );
581
1106
  let wfMachine = createWorkflowStateMachine(workflowPhases);
582
1107
 
583
1108
  while (tasks.some((task) => task.status === "queued")) {
@@ -586,26 +1111,100 @@ async function executeTeamRunCore(
586
1111
  const message = `${cancelReason.message} (${cancelReason.code})`;
587
1112
  const cancelledTaskIds: string[] = [];
588
1113
  tasks = tasks.map((task) => {
589
- if (task.status !== "queued" && task.status !== "running" && task.status !== "waiting") return task;
1114
+ if (
1115
+ task.status !== "queued" &&
1116
+ task.status !== "running" &&
1117
+ task.status !== "waiting"
1118
+ )
1119
+ return task;
590
1120
  cancelledTaskIds.push(task.id);
591
- const base = { ...task, status: "cancelled" as const, finishedAt: new Date().toISOString(), error: message };
1121
+ const base = {
1122
+ ...task,
1123
+ status: "cancelled" as const,
1124
+ finishedAt: new Date().toISOString(),
1125
+ error: message,
1126
+ };
592
1127
  if (task.status === "running") {
593
- return { ...base, terminalEvidence: [...(task.terminalEvidence ?? []), buildSyntheticTerminalEvidence("worker", cancelReason, task.startedAt)] };
1128
+ return {
1129
+ ...base,
1130
+ terminalEvidence: [
1131
+ ...(task.terminalEvidence ?? []),
1132
+ buildSyntheticTerminalEvidence(
1133
+ "worker",
1134
+ cancelReason,
1135
+ task.startedAt,
1136
+ ),
1137
+ ],
1138
+ };
594
1139
  }
595
1140
  return base;
596
1141
  });
597
1142
  await saveRunTasksAsync(manifest, tasks);
598
- for (const taskId of cancelledTaskIds) await appendEventAsync(manifest.eventsPath, { type: "task.cancelled", runId: manifest.runId, taskId, message, data: { reason: cancelReason.code } });
599
- manifest = updateRunStatus(manifest, "cancelled", message, { data: { reason: cancelReason.code, cancelledTaskIds } });
1143
+ for (const taskId of cancelledTaskIds)
1144
+ await appendEventAsync(manifest.eventsPath, {
1145
+ type: "task.cancelled",
1146
+ runId: manifest.runId,
1147
+ taskId,
1148
+ message,
1149
+ data: { reason: cancelReason.code },
1150
+ });
1151
+ manifest = updateRunStatus(manifest, "cancelled", message, {
1152
+ data: { reason: cancelReason.code, cancelledTaskIds },
1153
+ });
600
1154
  return { manifest, tasks };
601
1155
  }
602
1156
 
603
1157
  const failed = tasks.find((task) => task.status === "failed");
604
1158
  if (failed) {
605
- tasks = markBlocked(tasks, `Blocked by failed task '${failed.id}'.`);
1159
+ // #4 (assessment): honor limits.maxRetriesPerTask re-queue an eligible
1160
+ // failed task for a bounded whole-task rerun instead of immediately
1161
+ // aborting the run. Before #4, the recovery ledger recorded `rerun_task`
1162
+ // entries with state:"planned" but never executed them (decorative).
1163
+ // Default-off: maxRetriesPerTask=0 → original abort behavior preserved.
1164
+ const rerun = shouldRerunFailedTask(failed, input.limits);
1165
+ if (rerun.rerun) {
1166
+ tasks = tasks.map((item) =>
1167
+ item.id === failed.id
1168
+ ? {
1169
+ ...item,
1170
+ status: "queued" as const,
1171
+ policy: {
1172
+ ...(item.policy ?? {}),
1173
+ retryCount: rerun.newRetryCount,
1174
+ },
1175
+ error: undefined,
1176
+ finishedAt: undefined,
1177
+ }
1178
+ : item,
1179
+ );
1180
+ await saveRunTasksAsync(manifest, tasks);
1181
+ await appendEventAsync(manifest.eventsPath, {
1182
+ type: "recovery.rerun_task",
1183
+ runId: manifest.runId,
1184
+ taskId: failed.id,
1185
+ message: `Re-queuing failed task for whole-task rerun: ${rerun.reason}`,
1186
+ data: {
1187
+ attempt: rerun.newRetryCount,
1188
+ maxRetries: input.limits?.maxRetriesPerTask ?? 0,
1189
+ scenario: "task_failed",
1190
+ },
1191
+ });
1192
+ continue; // loop re-processes the re-queued task
1193
+ }
1194
+ tasks = markBlocked(
1195
+ tasks,
1196
+ `Blocked by failed task '${failed.id}'.`,
1197
+ );
606
1198
  await saveRunTasksAsync(manifest, tasks);
607
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
608
- manifest = updateRunStatus(manifest, "failed", `Failed at task '${failed.id}'.`);
1199
+ saveCrewAgents(
1200
+ manifest,
1201
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1202
+ );
1203
+ manifest = updateRunStatus(
1204
+ manifest,
1205
+ "failed",
1206
+ `Failed at task '${failed.id}'.`,
1207
+ );
609
1208
  return { manifest, tasks };
610
1209
  }
611
1210
 
@@ -614,67 +1213,202 @@ async function executeTeamRunCore(
614
1213
  // DAG-based execution plan: when tasks have explicit dependsOn, use the
615
1214
  // topological wave planner to determine ready tasks. Fall back to the
616
1215
  // existing task-graph-scheduler when no explicit deps exist (backward compat).
617
- const completedIds = new Set(tasks.filter((t) => t.status === "completed" || t.status === "needs_attention").map((t) => t.id));
1216
+ const completedIds = new Set(
1217
+ tasks
1218
+ .filter(
1219
+ (t) =>
1220
+ t.status === "completed" ||
1221
+ t.status === "needs_attention",
1222
+ )
1223
+ .map((t) => t.id),
1224
+ );
618
1225
  const dagReady = dagReadyTaskIds(tasks, completedIds);
619
1226
  const effectiveReady = dagReady ?? snapshot.ready;
620
1227
 
621
1228
  // Workflow phase precondition check (non-blocking: log warnings only).
622
1229
  if (wfMachine.currentPhaseIndex < wfMachine.phases.length) {
623
- const completedArtifacts = manifest.artifacts.filter((a) => a.kind === "result" || a.kind === "summary").map((a) => a.path);
624
- const previousPhaseStatus = wfMachine.currentPhaseIndex > 0 ? (wfMachine.phases[wfMachine.currentPhaseIndex - 1]?.status ?? "pending") : "completed";
1230
+ const completedArtifacts = manifest.artifacts
1231
+ .filter((a) => a.kind === "result" || a.kind === "summary")
1232
+ .map((a) => a.path);
1233
+ const previousPhaseStatus =
1234
+ wfMachine.currentPhaseIndex > 0
1235
+ ? (wfMachine.phases[wfMachine.currentPhaseIndex - 1]
1236
+ ?.status ?? "pending")
1237
+ : "completed";
625
1238
  const wfContext: PhaseGuardContext = {
626
1239
  completedArtifacts,
627
1240
  previousPhaseStatus,
628
- taskResults: tasks.filter((t) => t.status === "completed" || t.status === "needs_attention").map((t) => ({ taskId: t.id, status: t.status, outputPath: t.resultArtifact?.path })),
1241
+ taskResults: tasks
1242
+ .filter(
1243
+ (t) =>
1244
+ t.status === "completed" ||
1245
+ t.status === "needs_attention",
1246
+ )
1247
+ .map((t) => ({
1248
+ taskId: t.id,
1249
+ status: t.status,
1250
+ outputPath: t.resultArtifact?.path,
1251
+ })),
629
1252
  };
630
- const preconditions = validatePhasePreconditions(wfMachine, wfContext);
1253
+ const preconditions = validatePhasePreconditions(
1254
+ wfMachine,
1255
+ wfContext,
1256
+ );
631
1257
  if (!preconditions.ready) {
632
- await appendEventAsync(manifest.eventsPath, { type: "workflow.preconditions", runId: manifest.runId, message: `Workflow phase '${wfMachine.phases[wfMachine.currentPhaseIndex]?.name}' is missing inputs: ${preconditions.blocking.join(", ")}`, data: { phaseIndex: wfMachine.currentPhaseIndex, phaseName: wfMachine.phases[wfMachine.currentPhaseIndex]?.name, blocking: preconditions.blocking } });
1258
+ await appendEventAsync(manifest.eventsPath, {
1259
+ type: "workflow.preconditions",
1260
+ runId: manifest.runId,
1261
+ message: `Workflow phase '${wfMachine.phases[wfMachine.currentPhaseIndex]?.name}' is missing inputs: ${preconditions.blocking.join(", ")}`,
1262
+ data: {
1263
+ phaseIndex: wfMachine.currentPhaseIndex,
1264
+ phaseName:
1265
+ wfMachine.phases[wfMachine.currentPhaseIndex]?.name,
1266
+ blocking: preconditions.blocking,
1267
+ },
1268
+ });
633
1269
  } else {
634
1270
  // Advance the machine past completed phases.
635
- while (wfMachine.currentPhaseIndex < wfMachine.phases.length && wfMachine.phases[wfMachine.currentPhaseIndex]?.status === "completed") {
636
- wfMachine = { ...wfMachine, currentPhaseIndex: wfMachine.currentPhaseIndex + 1 };
1271
+ while (
1272
+ wfMachine.currentPhaseIndex < wfMachine.phases.length &&
1273
+ wfMachine.phases[wfMachine.currentPhaseIndex]?.status ===
1274
+ "completed"
1275
+ ) {
1276
+ wfMachine = {
1277
+ ...wfMachine,
1278
+ currentPhaseIndex: wfMachine.currentPhaseIndex + 1,
1279
+ };
637
1280
  }
638
1281
  }
639
1282
  }
640
1283
 
641
- const readyRoles = effectiveReady.map((taskId) => tasks.find((task) => task.id === taskId)?.role).filter((role): role is string => Boolean(role));
642
- const concurrency = resolveBatchConcurrency({ workflowName: workflow.name, workflowMaxConcurrency: workflow.maxConcurrency, teamMaxConcurrency: input.team.maxConcurrency, limitMaxConcurrentWorkers: input.limits?.maxConcurrentWorkers, allowUnboundedConcurrency: input.limits?.allowUnboundedConcurrency, readyCount: effectiveReady.length, workspaceMode: manifest.workspaceMode, readyRoles });
1284
+ const readyRoles = effectiveReady
1285
+ .map((taskId) => tasks.find((task) => task.id === taskId)?.role)
1286
+ .filter((role): role is string => Boolean(role));
1287
+ const concurrency = resolveBatchConcurrency({
1288
+ workflowName: workflow.name,
1289
+ workflowMaxConcurrency: workflow.maxConcurrency,
1290
+ teamMaxConcurrency: input.team.maxConcurrency,
1291
+ limitMaxConcurrentWorkers: input.limits?.maxConcurrentWorkers,
1292
+ allowUnboundedConcurrency: input.limits?.allowUnboundedConcurrency,
1293
+ readyCount: effectiveReady.length,
1294
+ workspaceMode: manifest.workspaceMode,
1295
+ readyRoles,
1296
+ });
643
1297
  if (concurrency.reason.includes(";unbounded:")) {
644
- await appendEventAsync(manifest.eventsPath, { type: "limits.unbounded", runId: manifest.runId, message: "Unbounded worker concurrency was explicitly enabled for this run.", data: { concurrencyReason: concurrency.reason, maxConcurrent: concurrency.maxConcurrent } });
1298
+ await appendEventAsync(manifest.eventsPath, {
1299
+ type: "limits.unbounded",
1300
+ runId: manifest.runId,
1301
+ message:
1302
+ "Unbounded worker concurrency was explicitly enabled for this run.",
1303
+ data: {
1304
+ concurrencyReason: concurrency.reason,
1305
+ maxConcurrent: concurrency.maxConcurrent,
1306
+ },
1307
+ });
645
1308
  }
646
1309
  const approvalPending = isPlanApprovalPending(manifest);
647
- const readyIds = approvalPending ? effectiveReady : effectiveReady.slice(0, concurrency.selectedCount);
648
- const candidateBatch = readyIds.map((id) => tasks.find((task) => task.id === id)).filter((task): task is TeamTaskState => Boolean(task));
649
- const readyBatch = approvalPending ? candidateBatch.filter((task) => !isMutatingTask(task)).slice(0, concurrency.selectedCount) : candidateBatch;
1310
+ const readyIds = approvalPending
1311
+ ? effectiveReady
1312
+ : effectiveReady.slice(0, concurrency.selectedCount);
1313
+ const candidateBatch = readyIds
1314
+ .map((id) => tasks.find((task) => task.id === id))
1315
+ .filter((task): task is TeamTaskState => Boolean(task));
1316
+ const readyBatch = approvalPending
1317
+ ? candidateBatch
1318
+ .filter((task) => !isMutatingTask(task))
1319
+ .slice(0, concurrency.selectedCount)
1320
+ : candidateBatch;
650
1321
  if (readyBatch.length === 0) {
651
1322
  if (approvalPending && candidateBatch.some(isMutatingTask)) {
652
1323
  await saveRunTasksAsync(manifest, tasks);
653
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
654
- manifest = updateRunStatus(manifest, "blocked", "Plan approval required before mutating implementation tasks run.");
1324
+ saveCrewAgents(
1325
+ manifest,
1326
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1327
+ );
1328
+ manifest = updateRunStatus(
1329
+ manifest,
1330
+ "blocked",
1331
+ "Plan approval required before mutating implementation tasks run.",
1332
+ );
655
1333
  return { manifest, tasks };
656
1334
  }
657
- tasks = markBlocked(tasks, "No ready queued task; dependency graph may be invalid.");
1335
+ tasks = markBlocked(
1336
+ tasks,
1337
+ "No ready queued task; dependency graph may be invalid.",
1338
+ );
658
1339
  await saveRunTasksAsync(manifest, tasks);
659
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
660
- manifest = updateRunStatus(manifest, "blocked", "No ready queued task.");
1340
+ saveCrewAgents(
1341
+ manifest,
1342
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1343
+ );
1344
+ manifest = updateRunStatus(
1345
+ manifest,
1346
+ "blocked",
1347
+ "No ready queued task.",
1348
+ );
661
1349
  return { manifest, tasks };
662
1350
  }
663
1351
 
664
1352
  // 2.2 caller migration: batch progress is high-frequency informational.
665
- appendEventFireAndForget(manifest.eventsPath, { type: "task.progress", runId: manifest.runId, message: `Starting ready batch with ${readyBatch.length} task(s).`, data: { taskIds: readyBatch.map((task) => task.id), readyCount: snapshot.ready.length, blockedCount: snapshot.blocked.length, runningCount: snapshot.running.length, doneCount: snapshot.done.length, selectedCount: readyBatch.length, maxConcurrent: concurrency.maxConcurrent, defaultConcurrency: concurrency.defaultConcurrency, concurrencyReason: approvalPending ? `${concurrency.reason};plan-approval-read-only` : concurrency.reason } });
1353
+ appendEventFireAndForget(manifest.eventsPath, {
1354
+ type: "task.progress",
1355
+ runId: manifest.runId,
1356
+ message: `Starting ready batch with ${readyBatch.length} task(s).`,
1357
+ data: {
1358
+ taskIds: readyBatch.map((task) => task.id),
1359
+ readyCount: snapshot.ready.length,
1360
+ blockedCount: snapshot.blocked.length,
1361
+ runningCount: snapshot.running.length,
1362
+ doneCount: snapshot.done.length,
1363
+ selectedCount: readyBatch.length,
1364
+ maxConcurrent: concurrency.maxConcurrent,
1365
+ defaultConcurrency: concurrency.defaultConcurrency,
1366
+ concurrencyReason: approvalPending
1367
+ ? `${concurrency.reason};plan-approval-read-only`
1368
+ : concurrency.reason,
1369
+ },
1370
+ });
666
1371
  // Execute before_task_start hooks for the batch
667
1372
  for (const task of readyBatch) {
668
- const taskReport = await executeHook("before_task_start", { runId: manifest.runId, taskId: task.id, cwd: manifest.cwd });
1373
+ const taskReport = await executeHook("before_task_start", {
1374
+ runId: manifest.runId,
1375
+ taskId: task.id,
1376
+ cwd: manifest.cwd,
1377
+ });
669
1378
  appendHookEvent(manifest, taskReport);
670
1379
  if (taskReport.outcome === "block") {
671
- tasks = tasks.map((t) => t.id === task.id ? { ...t, status: "skipped" as const, error: taskReport.reason ?? "before_task_start hook blocked execution." } : t);
672
- manifest = updateRunStatus(manifest, manifest.status, `Task '${task.id}' blocked by hook.`);
1380
+ tasks = tasks.map((t) =>
1381
+ t.id === task.id
1382
+ ? {
1383
+ ...t,
1384
+ status: "skipped" as const,
1385
+ error:
1386
+ taskReport.reason ??
1387
+ "before_task_start hook blocked execution.",
1388
+ }
1389
+ : t,
1390
+ );
1391
+ manifest = updateRunStatus(
1392
+ manifest,
1393
+ manifest.status,
1394
+ `Task '${task.id}' blocked by hook.`,
1395
+ );
673
1396
  }
674
1397
  }
675
- const batchTasks = readyBatch.filter((task) => tasks.find((t) => t.id === task.id && t.status !== "skipped"));
1398
+ const batchTasks = readyBatch.filter((task) =>
1399
+ tasks.find((t) => t.id === task.id && t.status !== "skipped"),
1400
+ );
676
1401
  if (batchTasks.length > 1) {
677
- await appendEventAsync(manifest.eventsPath, { type: "task.parallel_start", runId: manifest.runId, message: `Launching ${batchTasks.length} tasks in PARALLEL (concurrency=${concurrency.selectedCount}): ${batchTasks.map((t) => `${t.role}(${t.id})`).join(", ")}`, data: { taskIds: batchTasks.map((t) => t.id), roles: batchTasks.map((t) => t.role), concurrency: concurrency.selectedCount } });
1402
+ await appendEventAsync(manifest.eventsPath, {
1403
+ type: "task.parallel_start",
1404
+ runId: manifest.runId,
1405
+ message: `Launching ${batchTasks.length} tasks in PARALLEL (concurrency=${concurrency.selectedCount}): ${batchTasks.map((t) => `${t.role}(${t.id})`).join(", ")}`,
1406
+ data: {
1407
+ taskIds: batchTasks.map((t) => t.id),
1408
+ roles: batchTasks.map((t) => t.role),
1409
+ concurrency: concurrency.selectedCount,
1410
+ },
1411
+ });
678
1412
  }
679
1413
  const results = await mapConcurrent(
680
1414
  batchTasks,
@@ -682,73 +1416,302 @@ async function executeTeamRunCore(
682
1416
  async (task) => {
683
1417
  const step = findStep(workflow, task);
684
1418
  const agent = findAgent(input.agents, task);
685
- const teamRole = input.team.roles.find((role) => role.name === task.role);
686
- const perTaskRuntime = resolveTaskRuntimeKind(runtimeKind, task.role, input.runtimeConfig?.isolationPolicy);
687
- const baseInput = { manifest, tasks, task, step, agent, signal: input.signal, executeWorkers: input.executeWorkers, runtimeKind: runtimeKind, taskRuntimeOverride: perTaskRuntime !== runtimeKind ? perTaskRuntime : undefined, runtimeConfig: input.runtimeConfig, parentContext: input.parentContext, parentModel: input.parentModel, modelRegistry: input.modelRegistry, modelOverride: input.modelOverride, teamRoleModel: teamRole?.model, teamRoleSkills: teamRole?.skills, skillOverride: input.skillOverride, limits: input.limits, onJsonEvent: input.onJsonEvent, workspaceId: input.workspaceId };
688
- if (input.reliability?.autoRetry !== true) return withCorrelation(childCorrelation(manifest.runId, task.id), () => runTeamTask(baseInput));
689
- let lastFailed: { manifest: TeamRunManifest; tasks: TeamTaskState[] } | undefined;
1419
+ const teamRole = input.team.roles.find(
1420
+ (role) => role.name === task.role,
1421
+ );
1422
+ const perTaskRuntime = resolveTaskRuntimeKind(
1423
+ runtimeKind,
1424
+ task.role,
1425
+ input.runtimeConfig?.isolationPolicy,
1426
+ );
1427
+ const baseInput = {
1428
+ manifest,
1429
+ tasks,
1430
+ task,
1431
+ step,
1432
+ agent,
1433
+ signal: input.signal,
1434
+ executeWorkers: input.executeWorkers,
1435
+ runtimeKind: runtimeKind,
1436
+ taskRuntimeOverride:
1437
+ perTaskRuntime !== runtimeKind
1438
+ ? perTaskRuntime
1439
+ : undefined,
1440
+ runtimeConfig: input.runtimeConfig,
1441
+ parentContext: input.parentContext,
1442
+ parentModel: input.parentModel,
1443
+ modelRegistry: input.modelRegistry,
1444
+ modelOverride: input.modelOverride,
1445
+ teamRoleModel: teamRole?.model,
1446
+ teamRoleSkills: teamRole?.skills,
1447
+ skillOverride: input.skillOverride,
1448
+ limits: input.limits,
1449
+ onJsonEvent: input.onJsonEvent,
1450
+ workspaceId: input.workspaceId,
1451
+ };
1452
+ // #1 (assessment): autoRetry now defaults ON (opt-out via reliability.autoRetry=false).
1453
+ // The dominant v0.9.13 failure was ChildTimeout ("worker became unresponsive") with
1454
+ // ZERO retries because this gate was opt-in. isRetryable() defaults to true when
1455
+ // retryableErrors is empty, so transient hangs now retry up to maxAttempts (3) with
1456
+ // exponential backoff. Set reliability.autoRetry=false to restore old single-shot behavior.
1457
+ if (!shouldUseRetry(input.reliability))
1458
+ return withCorrelation(
1459
+ childCorrelation(manifest.runId, task.id),
1460
+ () => runTeamTask(baseInput),
1461
+ );
1462
+ let lastFailed:
1463
+ | { manifest: TeamRunManifest; tasks: TeamTaskState[] }
1464
+ | undefined;
690
1465
  let lastAttemptId: string | undefined;
691
- const attemptsSoFar: TaskAttemptState[] = [...(task.attempts ?? [])];
1466
+ const attemptsSoFar: TaskAttemptState[] = [
1467
+ ...(task.attempts ?? []),
1468
+ ];
692
1469
  const policy = retryPolicyFromConfig(input.reliability);
693
1470
  try {
694
- return await executeWithRetry(async (attempt, info) => {
695
- const startedAt = new Date().toISOString();
696
- const inFlightAttempts: TaskAttemptState[] = [...attemptsSoFar, { attemptId: info.attemptId, startedAt }];
697
- input.metricRegistry?.counter("crew.task.retry_attempt_total", "Retry attempts by run and task").inc({ runId: manifest.runId, taskId: task.id });
698
- // NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
699
- const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
700
- const freshManifest = fresh?.manifest ?? manifest;
701
- const freshTasks = fresh?.tasks ?? tasks;
702
- const freshTask = freshTasks.find((item) => item.id === task.id) ?? task;
703
- if (freshTask.status !== "queued" && freshTask.status !== "running") return { manifest: freshManifest, tasks: freshTasks };
704
- const taskWithAttempt: TeamTaskState = { ...freshTask, attempts: inFlightAttempts };
705
- const result = await withCorrelation(childCorrelation(freshManifest.runId, task.id), () => runTeamTask({ ...baseInput, manifest: freshManifest, tasks: freshTasks, task: taskWithAttempt }));
706
- const failed = failedTaskFrom(result, task.id);
707
- const endedAt = new Date().toISOString();
708
- const finishedAttempt: TaskAttemptState = { attemptId: info.attemptId, startedAt, endedAt, ...(failed?.error ? { error: failed.error } : {}) };
709
- attemptsSoFar.push(finishedAttempt);
710
- const withAttempt = result.tasks.map((item) => item.id === task.id ? { ...item, attempts: [...attemptsSoFar] } : item);
711
- const enriched = { manifest: result.manifest, tasks: withAttempt };
712
- if (failed) {
713
- lastFailed = enriched;
714
- throw new Error(failed.error ?? `Task ${task.id} failed.`);
715
- }
716
- input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe({ runId: manifest.runId, team: input.team.name }, Math.max(0, attempt - 1));
717
- return enriched;
718
- }, policy, {
719
- signal: input.signal,
720
- attemptId: (attempt) => `${manifest.runId}:${task.id}:attempt-${attempt}`,
721
- onAttemptFailed: (attempt, error, delayMs, info) => {
722
- lastAttemptId = info.attemptId;
723
- appendEventAsync(manifest.eventsPath, { type: "crew.task.retry_attempt", runId: manifest.runId, taskId: task.id, message: error.message, data: { attempt, attemptId: info.attemptId, delayMs }, metadata: { attemptId: info.attemptId } }).catch((error) => logInternalError("team-runner.retry-attempt", error, `taskId=${task.id}`));
724
- input.metricRegistry?.histogram("crew.task.retry_delay_ms", "Retry backoff delay, milliseconds").observe({ runId: manifest.runId, taskId: task.id }, delayMs);
1471
+ return await executeWithRetry(
1472
+ async (attempt, info) => {
1473
+ const startedAt = new Date().toISOString();
1474
+ const inFlightAttempts: TaskAttemptState[] = [
1475
+ ...attemptsSoFar,
1476
+ { attemptId: info.attemptId, startedAt },
1477
+ ];
1478
+ input.metricRegistry
1479
+ ?.counter(
1480
+ "crew.task.retry_attempt_total",
1481
+ "Retry attempts by run and task",
1482
+ )
1483
+ .inc({
1484
+ runId: manifest.runId,
1485
+ taskId: task.id,
1486
+ });
1487
+ // NOTE: no withRunLock best-effort only; concurrent writes may cause inconsistency
1488
+ const fresh = loadRunManifestById(
1489
+ manifest.cwd,
1490
+ manifest.runId,
1491
+ );
1492
+ const freshManifest = fresh?.manifest ?? manifest;
1493
+ const freshTasks = fresh?.tasks ?? tasks;
1494
+ const freshTask =
1495
+ freshTasks.find(
1496
+ (item) => item.id === task.id,
1497
+ ) ?? task;
1498
+ if (
1499
+ freshTask.status !== "queued" &&
1500
+ freshTask.status !== "running"
1501
+ )
1502
+ return {
1503
+ manifest: freshManifest,
1504
+ tasks: freshTasks,
1505
+ };
1506
+ const taskWithAttempt: TeamTaskState = {
1507
+ ...freshTask,
1508
+ attempts: inFlightAttempts,
1509
+ };
1510
+ const result = await withCorrelation(
1511
+ childCorrelation(freshManifest.runId, task.id),
1512
+ () =>
1513
+ runTeamTask({
1514
+ ...baseInput,
1515
+ manifest: freshManifest,
1516
+ tasks: freshTasks,
1517
+ task: taskWithAttempt,
1518
+ }),
1519
+ );
1520
+ const failed = failedTaskFrom(result, task.id);
1521
+ const endedAt = new Date().toISOString();
1522
+ const finishedAttempt: TaskAttemptState = {
1523
+ attemptId: info.attemptId,
1524
+ startedAt,
1525
+ endedAt,
1526
+ ...(failed?.error
1527
+ ? { error: failed.error }
1528
+ : {}),
1529
+ };
1530
+ attemptsSoFar.push(finishedAttempt);
1531
+ const withAttempt = result.tasks.map((item) =>
1532
+ item.id === task.id
1533
+ ? { ...item, attempts: [...attemptsSoFar] }
1534
+ : item,
1535
+ );
1536
+ const enriched = {
1537
+ manifest: result.manifest,
1538
+ tasks: withAttempt,
1539
+ };
1540
+ if (failed) {
1541
+ lastFailed = enriched;
1542
+ throw new Error(
1543
+ failed.error ?? `Task ${task.id} failed.`,
1544
+ );
1545
+ }
1546
+ input.metricRegistry
1547
+ ?.histogram(
1548
+ "crew.task.retry_count",
1549
+ "Retries per task",
1550
+ [0, 1, 2, 3, 5, 10],
1551
+ )
1552
+ .observe(
1553
+ {
1554
+ runId: manifest.runId,
1555
+ team: input.team.name,
1556
+ },
1557
+ Math.max(0, attempt - 1),
1558
+ );
1559
+ return enriched;
725
1560
  },
726
- onRetryGivenUp: (attempts, error, info) => {
727
- lastAttemptId = info.attemptId;
728
- appendDeadletter(manifest, { runId: manifest.runId, taskId: task.id, reason: "max-retries", attempts, attemptId: info.attemptId, lastError: error.message, timestamp: new Date().toISOString() });
729
- input.metricRegistry?.counter("crew.task.deadletter_total", "Deadletter triggers by reason").inc({ reason: "max-retries" });
730
- input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe({ runId: manifest.runId, team: input.team.name }, Math.max(0, attempts - 1));
1561
+ policy,
1562
+ {
1563
+ signal: input.signal,
1564
+ attemptId: (attempt) =>
1565
+ `${manifest.runId}:${task.id}:attempt-${attempt}`,
1566
+ onAttemptFailed: (
1567
+ attempt,
1568
+ error,
1569
+ delayMs,
1570
+ info,
1571
+ ) => {
1572
+ lastAttemptId = info.attemptId;
1573
+ appendEventAsync(manifest.eventsPath, {
1574
+ type: "crew.task.retry_attempt",
1575
+ runId: manifest.runId,
1576
+ taskId: task.id,
1577
+ message: error.message,
1578
+ data: {
1579
+ attempt,
1580
+ attemptId: info.attemptId,
1581
+ delayMs,
1582
+ },
1583
+ metadata: { attemptId: info.attemptId },
1584
+ }).catch((error) =>
1585
+ logInternalError(
1586
+ "team-runner.retry-attempt",
1587
+ error,
1588
+ `taskId=${task.id}`,
1589
+ ),
1590
+ );
1591
+ input.metricRegistry
1592
+ ?.histogram(
1593
+ "crew.task.retry_delay_ms",
1594
+ "Retry backoff delay, milliseconds",
1595
+ )
1596
+ .observe(
1597
+ {
1598
+ runId: manifest.runId,
1599
+ taskId: task.id,
1600
+ },
1601
+ delayMs,
1602
+ );
1603
+ },
1604
+ onRetryGivenUp: (attempts, error, info) => {
1605
+ lastAttemptId = info.attemptId;
1606
+ appendDeadletter(manifest, {
1607
+ runId: manifest.runId,
1608
+ taskId: task.id,
1609
+ reason: "max-retries",
1610
+ attempts,
1611
+ attemptId: info.attemptId,
1612
+ lastError: error.message,
1613
+ timestamp: new Date().toISOString(),
1614
+ });
1615
+ input.metricRegistry
1616
+ ?.counter(
1617
+ "crew.task.deadletter_total",
1618
+ "Deadletter triggers by reason",
1619
+ )
1620
+ .inc({ reason: "max-retries" });
1621
+ input.metricRegistry
1622
+ ?.histogram(
1623
+ "crew.task.retry_count",
1624
+ "Retries per task",
1625
+ [0, 1, 2, 3, 5, 10],
1626
+ )
1627
+ .observe(
1628
+ {
1629
+ runId: manifest.runId,
1630
+ team: input.team.name,
1631
+ },
1632
+ Math.max(0, attempts - 1),
1633
+ );
1634
+ },
731
1635
  },
732
- });
1636
+ );
733
1637
  } catch (retryError) {
734
- if (retryError instanceof CrewCancellationError || input.signal?.aborted) {
735
- const reason = retryError instanceof CrewCancellationError ? retryError.reason : cancellationReasonFromSignal(input.signal);
1638
+ if (
1639
+ retryError instanceof CrewCancellationError ||
1640
+ input.signal?.aborted
1641
+ ) {
1642
+ const reason =
1643
+ retryError instanceof CrewCancellationError
1644
+ ? retryError.reason
1645
+ : cancellationReasonFromSignal(input.signal);
736
1646
  // NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
737
- const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
1647
+ const fresh = loadRunManifestById(
1648
+ manifest.cwd,
1649
+ manifest.runId,
1650
+ );
738
1651
  const freshManifest = fresh?.manifest ?? manifest;
739
1652
  const freshTasks = fresh?.tasks ?? tasks;
740
- const cancelledTasks = freshTasks.map((item) => item.id === task.id && (item.status === "queued" || item.status === "running") ? { ...item, status: "cancelled" as const, finishedAt: new Date().toISOString(), error: `${reason.message} (${reason.code})` } : item);
741
- appendEventAsync(freshManifest.eventsPath, { type: "task.cancelled", runId: freshManifest.runId, taskId: task.id, message: reason.message, data: { reason, phase: "retry" }, metadata: lastAttemptId ? { attemptId: lastAttemptId } : undefined }).catch((error) => logInternalError("team-runner.cancelled", error, `taskId=${task.id}`));
742
- return { manifest: updateRunStatus(freshManifest, "cancelled", reason.message), tasks: cancelledTasks };
1653
+ const cancelledTasks = freshTasks.map((item) =>
1654
+ item.id === task.id &&
1655
+ (item.status === "queued" ||
1656
+ item.status === "running")
1657
+ ? {
1658
+ ...item,
1659
+ status: "cancelled" as const,
1660
+ finishedAt: new Date().toISOString(),
1661
+ error: `${reason.message} (${reason.code})`,
1662
+ }
1663
+ : item,
1664
+ );
1665
+ appendEventAsync(freshManifest.eventsPath, {
1666
+ type: "task.cancelled",
1667
+ runId: freshManifest.runId,
1668
+ taskId: task.id,
1669
+ message: reason.message,
1670
+ data: { reason, phase: "retry" },
1671
+ metadata: lastAttemptId
1672
+ ? { attemptId: lastAttemptId }
1673
+ : undefined,
1674
+ }).catch((error) =>
1675
+ logInternalError(
1676
+ "team-runner.cancelled",
1677
+ error,
1678
+ `taskId=${task.id}`,
1679
+ ),
1680
+ );
1681
+ return {
1682
+ manifest: updateRunStatus(
1683
+ freshManifest,
1684
+ "cancelled",
1685
+ reason.message,
1686
+ ),
1687
+ tasks: cancelledTasks,
1688
+ };
743
1689
  }
744
1690
  if (lastFailed) return lastFailed;
745
1691
  // NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
746
- const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
1692
+ const fresh = loadRunManifestById(
1693
+ manifest.cwd,
1694
+ manifest.runId,
1695
+ );
747
1696
  const freshManifest = fresh?.manifest ?? manifest;
748
1697
  const freshTasks = fresh?.tasks ?? tasks;
749
- const freshTask = freshTasks.find((item) => item.id === task.id) ?? task;
750
- if (freshTask.status !== "queued" && freshTask.status !== "running") return { manifest: freshManifest, tasks: freshTasks };
751
- return withCorrelation(childCorrelation(freshManifest.runId, task.id), () => runTeamTask({ ...baseInput, manifest: freshManifest, tasks: freshTasks, task: freshTask }));
1698
+ const freshTask =
1699
+ freshTasks.find((item) => item.id === task.id) ?? task;
1700
+ if (
1701
+ freshTask.status !== "queued" &&
1702
+ freshTask.status !== "running"
1703
+ )
1704
+ return { manifest: freshManifest, tasks: freshTasks };
1705
+ return withCorrelation(
1706
+ childCorrelation(freshManifest.runId, task.id),
1707
+ () =>
1708
+ runTeamTask({
1709
+ ...baseInput,
1710
+ manifest: freshManifest,
1711
+ tasks: freshTasks,
1712
+ task: freshTask,
1713
+ }),
1714
+ );
752
1715
  }
753
1716
  },
754
1717
  );
@@ -757,42 +1720,66 @@ async function executeTeamRunCore(
757
1720
  // during parallel execution. Other workers may have written partial results
758
1721
  // before one threw. Results may be partial - some tasks in-flight at error
759
1722
  // time will not have entries in the results array.
760
- const validResults = results.filter((item): item is NonNullable<typeof item> => item !== undefined);
1723
+ const validResults = results.filter(
1724
+ (item): item is NonNullable<typeof item> => item !== undefined,
1725
+ );
761
1726
  // Guard: if ALL parallel workers threw before returning, validResults is empty.
762
1727
  // at(-1)! would crash. Mark the run failed rather than crashing.
763
1728
  if (validResults.length === 0) {
764
- manifest = updateRunStatus(manifest, "failed", "All parallel tasks failed catastrophically.");
1729
+ manifest = updateRunStatus(
1730
+ manifest,
1731
+ "failed",
1732
+ "All parallel tasks failed catastrophically.",
1733
+ );
765
1734
  return { manifest, tasks };
766
1735
  }
767
1736
  // Reconstruct manifest from the last worker's snapshot. The .artifacts field
768
- // is re-merged from both the team-runner's in-memory state and all workers'
769
- // snapshots, so artifact writes by task-runner (which individually save manifest
770
- // after writing artifacts) are safely persisted. The in-memory manifest is only
771
- // used for the next batch iteration's orchestration — actual persistence is safe.
772
- // Use updateRunStatus to recompute manifest status from merged tasks rather than
773
- // relying on the last result's manifest (which is arbitrary due to mapConcurrent
774
- // returning results in arbitrary order).
775
- // Use the in-memory manifest as base (not the last-completing worker's snapshot).
776
- // Recompute status from merged tasks so the manifest reflects actual task state,
777
- // not the arbitrary order in which mapConcurrent returned results.
778
- // Read committed manifest from disk inside the lock so artifact merge is based
779
- // on committed state, not in-memory state that may differ from disk.
780
- const mergeResult = await withRunLock(manifest, async () => {
781
- const disk = loadRunManifestById(manifest.cwd, manifest.runId);
782
- const diskManifest = disk?.manifest ?? manifest;
783
- const diskArtifacts = diskManifest.artifacts;
784
- const reconciledArtifacts = mergeArtifacts([...diskArtifacts, ...validResults.map((item) => item.manifest.artifacts)].flat());
785
- const resultManifest = updateRunStatus({ ...diskManifest, artifacts: reconciledArtifacts }, "running", "Merged task updates from parallel batch.");
786
- const resultTasks = mergeTaskUpdatesPreservingTerminal(tasks, validResults);
787
- await saveRunManifestAsync(resultManifest);
788
- await saveRunTasksAsync(resultManifest, resultTasks);
789
- return { resultManifest, resultTasks };
790
- });
791
- manifest = mergeResult.resultManifest;
792
- tasks = mergeResult.resultTasks;
1737
+ // is re-merged from both the team-runner's in-memory state and all workers'
1738
+ // snapshots, so artifact writes by task-runner (which individually save manifest
1739
+ // after writing artifacts) are safely persisted. The in-memory manifest is only
1740
+ // used for the next batch iteration's orchestration — actual persistence is safe.
1741
+ // Use updateRunStatus to recompute manifest status from merged tasks rather than
1742
+ // relying on the last result's manifest (which is arbitrary due to mapConcurrent
1743
+ // returning results in arbitrary order).
1744
+ // Use the in-memory manifest as base (not the last-completing worker's snapshot).
1745
+ // Recompute status from merged tasks so the manifest reflects actual task state,
1746
+ // not the arbitrary order in which mapConcurrent returned results.
1747
+ // Read committed manifest from disk inside the lock so artifact merge is based
1748
+ // on committed state, not in-memory state that may differ from disk.
1749
+ const mergeResult = await withRunLock(manifest, async () => {
1750
+ const disk = loadRunManifestById(manifest.cwd, manifest.runId);
1751
+ const diskManifest = disk?.manifest ?? manifest;
1752
+ const diskArtifacts = diskManifest.artifacts;
1753
+ const reconciledArtifacts = mergeArtifacts(
1754
+ [
1755
+ ...diskArtifacts,
1756
+ ...validResults.map((item) => item.manifest.artifacts),
1757
+ ].flat(),
1758
+ );
1759
+ const resultManifest = updateRunStatus(
1760
+ { ...diskManifest, artifacts: reconciledArtifacts },
1761
+ "running",
1762
+ "Merged task updates from parallel batch.",
1763
+ );
1764
+ const resultTasks = mergeTaskUpdatesPreservingTerminal(
1765
+ tasks,
1766
+ validResults,
1767
+ );
1768
+ await saveRunManifestAsync(resultManifest);
1769
+ await saveRunTasksAsync(resultManifest, resultTasks);
1770
+ return { resultManifest, resultTasks };
1771
+ });
1772
+ manifest = mergeResult.resultManifest;
1773
+ tasks = mergeResult.resultTasks;
793
1774
 
794
1775
  // Advance workflow phases whose tasks are all in terminal state
795
- const terminalStatuses = new Set(["completed", "failed", "skipped", "cancelled", "needs_attention"]);
1776
+ const terminalStatuses = new Set([
1777
+ "completed",
1778
+ "failed",
1779
+ "skipped",
1780
+ "cancelled",
1781
+ "needs_attention",
1782
+ ]);
796
1783
  const phaseTaskMap = new Map<string, string[]>();
797
1784
  for (const task of tasks) {
798
1785
  if (!task.stepId) continue;
@@ -800,7 +1787,11 @@ tasks = mergeResult.resultTasks;
800
1787
  existing.push(task.id);
801
1788
  phaseTaskMap.set(task.stepId, existing);
802
1789
  }
803
- for (let pi = wfMachine.currentPhaseIndex; pi < wfMachine.phases.length; pi++) {
1790
+ for (
1791
+ let pi = wfMachine.currentPhaseIndex;
1792
+ pi < wfMachine.phases.length;
1793
+ pi++
1794
+ ) {
804
1795
  const phase = wfMachine.phases[pi]!;
805
1796
  const phaseTaskIds = phaseTaskMap.get(phase.name) ?? [];
806
1797
  if (phaseTaskIds.length === 0) continue;
@@ -809,75 +1800,187 @@ tasks = mergeResult.resultTasks;
809
1800
  return task ? terminalStatuses.has(task.status) : false;
810
1801
  });
811
1802
  if (!allTerminal) break;
812
- if (phase.status !== "completed" && phase.status !== "failed" && phase.status !== "skipped") {
813
- const completedArtifacts = manifest.artifacts.filter((a) => a.kind === "result" || a.kind === "summary").map((a) => a.path);
814
- const previousPhaseStatus = pi > 0 ? (wfMachine.phases[pi - 1]?.status ?? "pending") : "completed";
1803
+ if (
1804
+ phase.status !== "completed" &&
1805
+ phase.status !== "failed" &&
1806
+ phase.status !== "skipped"
1807
+ ) {
1808
+ const completedArtifacts = manifest.artifacts
1809
+ .filter((a) => a.kind === "result" || a.kind === "summary")
1810
+ .map((a) => a.path);
1811
+ const previousPhaseStatus =
1812
+ pi > 0
1813
+ ? (wfMachine.phases[pi - 1]?.status ?? "pending")
1814
+ : "completed";
815
1815
  const wfContext: PhaseGuardContext = {
816
1816
  completedArtifacts,
817
1817
  previousPhaseStatus,
818
- taskResults: tasks.filter((t) => t.status === "completed" || t.status === "needs_attention").map((t) => ({ taskId: t.id, status: t.status, outputPath: t.resultArtifact?.path })),
1818
+ taskResults: tasks
1819
+ .filter(
1820
+ (t) =>
1821
+ t.status === "completed" ||
1822
+ t.status === "needs_attention",
1823
+ )
1824
+ .map((t) => ({
1825
+ taskId: t.id,
1826
+ status: t.status,
1827
+ outputPath: t.resultArtifact?.path,
1828
+ })),
819
1829
  };
820
1830
  // Determine phase transition status based on individual task outcomes
821
- const phaseTasks = phaseTaskIds.map((taskId) => tasks.find((t) => t.id === taskId)).filter((t): t is NonNullable<typeof t> => t !== undefined);
822
- const hasFailedOrCancelled = phaseTasks.some((t) => t.status === "failed" || t.status === "cancelled");
823
- const phaseStatus = hasFailedOrCancelled ? "failed" : "completed";
824
- const transition = transitionPhase(wfMachine, pi, phaseStatus, wfContext);
1831
+ const phaseTasks = phaseTaskIds
1832
+ .map((taskId) => tasks.find((t) => t.id === taskId))
1833
+ .filter((t): t is NonNullable<typeof t> => t !== undefined);
1834
+ const hasFailedOrCancelled = phaseTasks.some(
1835
+ (t) => t.status === "failed" || t.status === "cancelled",
1836
+ );
1837
+ const phaseStatus = hasFailedOrCancelled
1838
+ ? "failed"
1839
+ : "completed";
1840
+ const transition = transitionPhase(
1841
+ wfMachine,
1842
+ pi,
1843
+ phaseStatus,
1844
+ wfContext,
1845
+ );
825
1846
  wfMachine = transition.machine;
826
1847
  if (transition.guardResult && !transition.guardResult.allowed) {
827
- await appendEventAsync(manifest.eventsPath, { type: "workflow.phase_guard_blocked", runId: manifest.runId, message: `Workflow phase '${phase.name}' guard blocked: ${transition.guardResult.reason ?? "unknown"}`, data: { phaseIndex: pi, phaseName: phase.name, reason: transition.guardResult.reason } });
1848
+ await appendEventAsync(manifest.eventsPath, {
1849
+ type: "workflow.phase_guard_blocked",
1850
+ runId: manifest.runId,
1851
+ message: `Workflow phase '${phase.name}' guard blocked: ${transition.guardResult.reason ?? "unknown"}`,
1852
+ data: {
1853
+ phaseIndex: pi,
1854
+ phaseName: phase.name,
1855
+ reason: transition.guardResult.reason,
1856
+ },
1857
+ });
828
1858
  break;
829
1859
  }
830
- await appendEventAsync(manifest.eventsPath, { type: phaseStatus === "failed" ? "workflow.phase_failed" : "workflow.phase_completed", runId: manifest.runId, message: `Workflow phase '${phase.name}' ${phaseStatus}.`, data: { phaseIndex: pi, phaseStatus } });
1860
+ await appendEventAsync(manifest.eventsPath, {
1861
+ type:
1862
+ phaseStatus === "failed"
1863
+ ? "workflow.phase_failed"
1864
+ : "workflow.phase_completed",
1865
+ runId: manifest.runId,
1866
+ message: `Workflow phase '${phase.name}' ${phaseStatus}.`,
1867
+ data: { phaseIndex: pi, phaseStatus },
1868
+ });
831
1869
  }
832
1870
  wfMachine = { ...wfMachine, currentPhaseIndex: pi + 1 };
833
1871
  }
834
1872
 
835
- const cancelledResult = results.find((item) => item.manifest.status === "cancelled");
1873
+ const cancelledResult = results.find(
1874
+ (item) => item.manifest.status === "cancelled",
1875
+ );
836
1876
  if (cancelledResult || input.signal?.aborted) {
837
- const reason = input.signal?.aborted ? cancellationReasonFromSignal(input.signal) : undefined;
838
- const message = reason?.message ?? cancelledResult?.manifest.summary ?? "Run cancelled during task execution.";
1877
+ const reason = input.signal?.aborted
1878
+ ? cancellationReasonFromSignal(input.signal)
1879
+ : undefined;
1880
+ const message =
1881
+ reason?.message ??
1882
+ cancelledResult?.manifest.summary ??
1883
+ "Run cancelled during task execution.";
839
1884
  manifest = { ...manifest, status: "running" };
840
1885
  manifest = updateRunStatus(manifest, "cancelled", message);
841
1886
  await saveRunTasksAsync(manifest, tasks);
842
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
1887
+ saveCrewAgents(
1888
+ manifest,
1889
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1890
+ );
843
1891
  await saveRunManifestAsync(manifest);
844
- await appendEventAsync(manifest.eventsPath, { type: "run.cancelled", runId: manifest.runId, message, data: { reason, phase: "task-batch", cancelledResultRunId: cancelledResult?.manifest.runId } });
1892
+ await appendEventAsync(manifest.eventsPath, {
1893
+ type: "run.cancelled",
1894
+ runId: manifest.runId,
1895
+ message,
1896
+ data: {
1897
+ reason,
1898
+ phase: "task-batch",
1899
+ cancelledResultRunId: cancelledResult?.manifest.runId,
1900
+ },
1901
+ });
845
1902
  return { manifest, tasks };
846
1903
  }
847
1904
  queueIndex = buildTaskGraphIndex(tasks);
848
1905
  const injectedAfterBatch = attemptAdaptivePlan();
849
1906
  if (injectedAfterBatch.missing) {
850
- tasks = markBlocked(tasks, "Adaptive planner did not produce a valid subagent plan.");
1907
+ tasks = markBlocked(
1908
+ tasks,
1909
+ "Adaptive planner did not produce a valid subagent plan.",
1910
+ );
851
1911
  await saveRunTasksAsync(manifest, tasks);
852
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
853
- manifest = updateRunStatus(manifest, "blocked", "Adaptive planner did not produce a valid subagent plan.");
1912
+ saveCrewAgents(
1913
+ manifest,
1914
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1915
+ );
1916
+ manifest = updateRunStatus(
1917
+ manifest,
1918
+ "blocked",
1919
+ "Adaptive planner did not produce a valid subagent plan.",
1920
+ );
854
1921
  return { manifest, tasks };
855
1922
  }
856
1923
  if (injectedAfterBatch.injected) {
857
- manifest = requiresPlanApproval(workflow, input.runtimeConfig) ? ensurePlanApprovalRequested(manifest, tasks) : manifest;
1924
+ manifest = requiresPlanApproval(workflow, input.runtimeConfig)
1925
+ ? ensurePlanApprovalRequested(manifest, tasks)
1926
+ : manifest;
858
1927
  queueIndex = buildTaskGraphIndex(tasks);
859
- } else if (requiresPlanApproval(workflow, input.runtimeConfig) && (hasPendingMutatingAdaptiveTask(tasks) || hasPendingMutatingTaskAtBoundary(tasks))) {
1928
+ } else if (
1929
+ requiresPlanApproval(workflow, input.runtimeConfig) &&
1930
+ (hasPendingMutatingAdaptiveTask(tasks) ||
1931
+ hasPendingMutatingTaskAtBoundary(tasks))
1932
+ ) {
860
1933
  manifest = ensurePlanApprovalRequested(manifest, tasks);
861
1934
  }
862
1935
  if (manifest.planApproval?.status === "cancelled") {
863
1936
  tasks = cancelPlanTasks(tasks, "Plan approval was cancelled.");
864
1937
  await saveRunTasksAsync(manifest, tasks);
865
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
866
- manifest = updateRunStatus(manifest, "cancelled", "Plan approval was cancelled.");
1938
+ saveCrewAgents(
1939
+ manifest,
1940
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1941
+ );
1942
+ manifest = updateRunStatus(
1943
+ manifest,
1944
+ "cancelled",
1945
+ "Plan approval was cancelled.",
1946
+ );
867
1947
  return { manifest, tasks };
868
1948
  }
869
1949
  await saveRunTasksAsync(manifest, tasks);
870
- saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
871
- const completedBatch = tasks.filter((t) => batchTasks.some((bt) => bt.id === t.id));
1950
+ saveCrewAgents(
1951
+ manifest,
1952
+ recordsForMaterializedTasks(manifest, tasks, runtimeKind),
1953
+ );
1954
+ const completedBatch = tasks.filter((t) =>
1955
+ batchTasks.some((bt) => bt.id === t.id),
1956
+ );
872
1957
  const batchArtifact = writeArtifact(manifest.artifactsRoot, {
873
1958
  kind: "summary",
874
1959
  relativePath: `batches/${batchTasks.map((task) => task.id).join("+")}.md`,
875
1960
  producer: "team-runner",
876
1961
  content: aggregateTaskOutputs(completedBatch, manifest),
877
1962
  });
878
- const groupDelivery = deliverGroupJoin({ manifest, mode: resolveGroupJoinMode(input.runtimeConfig), batch: batchTasks, allTasks: tasks });
879
- manifest = { ...manifest, artifacts: mergeArtifacts([...manifest.artifacts, batchArtifact, ...(groupDelivery?.artifact ? [groupDelivery.artifact] : [])]) };
880
- manifest = writeProgress(manifest, tasks, "team-runner", input.executeWorkers, input.runtimeConfig);
1963
+ const groupDelivery = deliverGroupJoin({
1964
+ manifest,
1965
+ mode: resolveGroupJoinMode(input.runtimeConfig),
1966
+ batch: batchTasks,
1967
+ allTasks: tasks,
1968
+ });
1969
+ manifest = {
1970
+ ...manifest,
1971
+ artifacts: mergeArtifacts([
1972
+ ...manifest.artifacts,
1973
+ batchArtifact,
1974
+ ...(groupDelivery?.artifact ? [groupDelivery.artifact] : []),
1975
+ ]),
1976
+ };
1977
+ manifest = writeProgress(
1978
+ manifest,
1979
+ tasks,
1980
+ "team-runner",
1981
+ input.executeWorkers,
1982
+ input.runtimeConfig,
1983
+ );
881
1984
  await saveRunManifestAsync(manifest);
882
1985
  }
883
1986
 
@@ -885,29 +1988,85 @@ tasks = mergeResult.resultTasks;
885
1988
  const waiting = tasks.find((task) => task.status === "waiting");
886
1989
  const running = tasks.find((task) => task.status === "running");
887
1990
  manifest = applyPolicy(manifest, tasks, input.limits);
888
- const effectiveness = evaluateRunEffectiveness({ manifest, tasks, executeWorkers: input.executeWorkers, runtimeConfig: input.runtimeConfig });
1991
+ const effectiveness = evaluateRunEffectiveness({
1992
+ manifest,
1993
+ tasks,
1994
+ executeWorkers: input.executeWorkers,
1995
+ runtimeConfig: input.runtimeConfig,
1996
+ });
889
1997
  const effectivenessDecision = effectivenessPolicyDecision(effectiveness);
890
1998
  if (effectivenessDecision) {
891
- manifest = { ...manifest, policyDecisions: [...(manifest.policyDecisions ?? []), effectivenessDecision], updatedAt: new Date().toISOString() };
892
- await appendEventAsync(manifest.eventsPath, { type: "run.effectiveness", runId: manifest.runId, message: effectivenessDecision.message, data: { effectiveness, policyDecision: effectivenessDecision } });
1999
+ manifest = {
2000
+ ...manifest,
2001
+ policyDecisions: [
2002
+ ...(manifest.policyDecisions ?? []),
2003
+ effectivenessDecision,
2004
+ ],
2005
+ updatedAt: new Date().toISOString(),
2006
+ };
2007
+ await appendEventAsync(manifest.eventsPath, {
2008
+ type: "run.effectiveness",
2009
+ runId: manifest.runId,
2010
+ message: effectivenessDecision.message,
2011
+ data: { effectiveness, policyDecision: effectivenessDecision },
2012
+ });
893
2013
  }
894
- const blockingDecision = manifest.policyDecisions?.find((item) => item.action === "block" || item.action === "escalate");
2014
+ const blockingDecision = manifest.policyDecisions?.find(
2015
+ (item) => item.action === "block" || item.action === "escalate",
2016
+ );
895
2017
  if (failed) {
896
- manifest = updateRunStatus(manifest, "failed", `Failed at task '${failed.id}'.`);
2018
+ manifest = updateRunStatus(
2019
+ manifest,
2020
+ "failed",
2021
+ `Failed at task '${failed.id}'.`,
2022
+ );
897
2023
  } else if (waiting) {
898
- manifest = updateRunStatus(manifest, "blocked", `Waiting for response to task '${waiting.id}'.`);
2024
+ manifest = updateRunStatus(
2025
+ manifest,
2026
+ "blocked",
2027
+ `Waiting for response to task '${waiting.id}'.`,
2028
+ );
899
2029
  } else if (running) {
900
- manifest = updateRunStatus(manifest, "blocked", `Task '${running.id}' is still running.`);
2030
+ manifest = updateRunStatus(
2031
+ manifest,
2032
+ "blocked",
2033
+ `Task '${running.id}' is still running.`,
2034
+ );
901
2035
  } else if (effectiveness.severity === "failed") {
902
- manifest = updateRunStatus(manifest, "failed", effectivenessDecision?.message ?? "Run effectiveness guard failed.");
2036
+ manifest = updateRunStatus(
2037
+ manifest,
2038
+ "failed",
2039
+ effectivenessDecision?.message ?? "Run effectiveness guard failed.",
2040
+ );
903
2041
  } else if (effectiveness.severity === "blocked") {
904
- manifest = updateRunStatus(manifest, "blocked", effectivenessDecision?.message ?? "Run effectiveness guard blocked completion.");
2042
+ manifest = updateRunStatus(
2043
+ manifest,
2044
+ "blocked",
2045
+ effectivenessDecision?.message ??
2046
+ "Run effectiveness guard blocked completion.",
2047
+ );
905
2048
  } else if (blockingDecision) {
906
- manifest = updateRunStatus(manifest, "blocked", blockingDecision.message);
2049
+ manifest = updateRunStatus(
2050
+ manifest,
2051
+ "blocked",
2052
+ blockingDecision.message,
2053
+ );
907
2054
  } else {
908
- manifest = updateRunStatus(manifest, "completed", input.executeWorkers ? "Team workflow completed." : "Team workflow scaffold completed without launching child workers.");
2055
+ manifest = updateRunStatus(
2056
+ manifest,
2057
+ "completed",
2058
+ input.executeWorkers
2059
+ ? "Team workflow completed."
2060
+ : "Team workflow scaffold completed without launching child workers.",
2061
+ );
909
2062
  }
910
- manifest = writeProgress(manifest, tasks, "team-runner", input.executeWorkers, input.runtimeConfig);
2063
+ manifest = writeProgress(
2064
+ manifest,
2065
+ tasks,
2066
+ "team-runner",
2067
+ input.executeWorkers,
2068
+ input.runtimeConfig,
2069
+ );
911
2070
  await saveRunManifestAsync(manifest);
912
2071
  const usage = aggregateUsage(tasks);
913
2072
  const summaryArtifact = writeArtifact(manifest.artifactsRoot, {
@@ -927,17 +2086,28 @@ tasks = mergeResult.resultTasks;
927
2086
  ...tasks.map(formatTaskProgress),
928
2087
  "",
929
2088
  "## Effectiveness",
930
- ...runEffectivenessLines(manifest, tasks, input.executeWorkers, input.runtimeConfig),
2089
+ ...runEffectivenessLines(
2090
+ manifest,
2091
+ tasks,
2092
+ input.executeWorkers,
2093
+ input.runtimeConfig,
2094
+ ),
931
2095
  "",
932
2096
  "## Policy decisions",
933
- ...(manifest.policyDecisions?.length ? summarizePolicyDecisions(manifest.policyDecisions) : ["- (none)"]),
2097
+ ...(manifest.policyDecisions?.length
2098
+ ? summarizePolicyDecisions(manifest.policyDecisions)
2099
+ : ["- (none)"]),
934
2100
  "",
935
2101
  ].join("\n"),
936
2102
  });
937
2103
  // Build the complete manifest BEFORE acquiring the lock so the artifacts array
938
2104
  // is already incorporated into the manifest object that will be atomically written.
939
2105
  // This prevents crash-between-mutation-and-lock from leaving inconsistent state.
940
- const finalManifest = { ...manifest, updatedAt: new Date().toISOString(), artifacts: [...manifest.artifacts, summaryArtifact] };
2106
+ const finalManifest = {
2107
+ ...manifest,
2108
+ updatedAt: new Date().toISOString(),
2109
+ artifacts: [...manifest.artifacts, summaryArtifact],
2110
+ };
941
2111
  // Joint atomic save: wrap manifest + tasks in a single run lock so they are
942
2112
  // written together or not at all. Crash between separate saveRunManifestAsync
943
2113
  // and saveRunTasksAsync calls could leave manifest/tasks.json out of sync.
@@ -955,7 +2125,9 @@ tasks = mergeResult.resultTasks;
955
2125
  // health feature) AND created junk dirs that the recursive state watcher then
956
2126
  // attached extra inotify watches to. Fix: compute the real crew root (3 up)
957
2127
  // and make HEALTH_DIR relative to it.
958
- const crewRoot = path.dirname(path.dirname(path.dirname(finalManifest.stateRoot)));
2128
+ const crewRoot = path.dirname(
2129
+ path.dirname(path.dirname(finalManifest.stateRoot)),
2130
+ );
959
2131
  const healthStore = new HealthStore(crewRoot);
960
2132
  healthStore.saveSnapshot({
961
2133
  runId: finalManifest.runId,