pi-crew 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +57 -32
- package/CHANGELOG.md +466 -448
- package/LICENSE +21 -21
- package/NOTICE.md +16 -16
- package/README.md +323 -323
- package/docs/FEATURE_INTAKE.md +126 -0
- package/docs/HARNESS.md +86 -0
- package/docs/HARNESS_BACKLOG.md +41 -0
- package/docs/TEST_MATRIX.md +49 -0
- package/docs/actions-reference.md +595 -595
- package/docs/architecture.md +180 -180
- package/docs/code-review-2026-05-11.md +592 -592
- package/docs/commands-reference.md +347 -347
- package/docs/comparison-pi-subagents-vs-pi-crew.md +303 -0
- package/docs/decisions/0001-durable-state.md +41 -0
- package/docs/decisions/0002-child-process-for-async.md +42 -0
- package/docs/decisions/0003-depth-guard.md +36 -0
- package/docs/decisions/0004-execfile-over-exec.md +34 -0
- package/docs/decisions/0005-no-parameter-properties.md +49 -0
- package/docs/decisions/0006-publish-bundled-esm.md +63 -0
- package/docs/decisions/0007-active-run-binary-index.md +54 -0
- package/docs/decisions/0008-child-pi-warm-pool.md +61 -0
- package/docs/decisions/README.md +23 -0
- package/docs/followup-review-round4-2026-05-13.md +107 -0
- package/docs/implementation-plan-top3.md +333 -0
- package/docs/live-mailbox-runtime.md +36 -36
- package/docs/next-upgrade-roadmap.md +808 -808
- package/docs/oh-my-pi-research.md +509 -0
- package/docs/perf/baseline-2026-05.md +113 -0
- package/docs/perf/final-report-2026-05.md +206 -0
- package/docs/perf/sprint-1-report.md +71 -0
- package/docs/perf/sprint-2-report.md +81 -0
- package/docs/perf/sprint-2.5-report.md +53 -0
- package/docs/perf/sprint-3-report.md +36 -0
- package/docs/perf/sprint-4-report.md +47 -0
- package/docs/perf/sprint-5-report.md +51 -0
- package/docs/perf/sprint-6-report.md +94 -0
- package/docs/perf/sprint-7-report.md +74 -0
- package/docs/perf/upgrade-plan-2026-05.md +147 -0
- package/docs/pi-subagents3-deep-analysis.md +508 -0
- package/docs/product/README.md +31 -0
- package/docs/product/platform.md +27 -0
- package/docs/product/runtime-safety.md +37 -0
- package/docs/product/team-run.md +39 -0
- package/docs/product/team-tool.md +37 -0
- package/docs/publishing.md +65 -65
- package/docs/resource-formats.md +134 -134
- package/docs/runtime-analysis-child-vs-live.md +171 -0
- package/docs/runtime-flow.md +148 -148
- package/docs/runtime-migration-in-process-analysis.md +250 -0
- package/docs/stories/README.md +30 -0
- package/docs/stories/backlog.md +36 -0
- package/docs/templates/decision.md +27 -0
- package/docs/templates/story.md +44 -0
- package/docs/templates/validation-report.md +32 -0
- package/docs/usage.md +238 -238
- package/index.ts +7 -6
- package/install.mjs +65 -65
- package/package.json +107 -100
- package/schema.json +222 -222
- package/skills/child-pi-spawning/SKILL.md +213 -0
- package/skills/context-artifact-hygiene/SKILL.md +32 -0
- package/skills/event-log-tracing/SKILL.md +299 -0
- package/skills/git-master/SKILL.md +225 -24
- package/skills/live-agent-lifecycle/SKILL.md +192 -0
- package/skills/mailbox-interactive/SKILL.md +300 -19
- package/skills/model-routing-context/SKILL.md +94 -0
- package/skills/multi-perspective-review/SKILL.md +88 -0
- package/skills/read-only-explorer/SKILL.md +250 -26
- package/skills/safe-bash/SKILL.md +307 -21
- package/skills/verification-before-done/SKILL.md +11 -2
- package/skills/widget-rendering/SKILL.md +258 -0
- package/skills/workspace-isolation/SKILL.md +202 -0
- package/skills/worktree-isolation/SKILL.md +202 -18
- package/src/adapters/claude-adapter.ts +25 -25
- package/src/adapters/codex-adapter.ts +21 -21
- package/src/adapters/cursor-adapter.ts +17 -17
- package/src/adapters/export-util.ts +137 -137
- package/src/adapters/index.ts +15 -15
- package/src/adapters/registry.ts +18 -18
- package/src/adapters/types.ts +23 -23
- package/src/agents/agent-config.ts +38 -38
- package/src/agents/agent-serializer.ts +38 -38
- package/src/agents/discover-agents.ts +121 -118
- package/src/config/config.ts +740 -858
- package/src/config/defaults.ts +96 -96
- package/src/config/drift-detector.ts +211 -211
- package/src/config/markers.ts +327 -327
- package/src/config/resilient-parser.ts +109 -108
- package/src/config/suggestions.ts +74 -74
- package/src/config/types.ts +199 -0
- package/src/extension/async-notifier.ts +123 -89
- package/src/extension/autonomous-policy.ts +169 -169
- package/src/extension/cross-extension-rpc.ts +104 -104
- package/src/extension/help.ts +47 -47
- package/src/extension/import-index.ts +69 -69
- package/src/extension/management.ts +395 -382
- package/src/extension/notification-router.ts +116 -116
- package/src/extension/notification-sink.ts +51 -51
- package/src/extension/project-init.ts +168 -168
- package/src/extension/register.ts +859 -668
- package/src/extension/registration/artifact-cleanup.ts +15 -15
- package/src/extension/registration/command-utils.ts +54 -54
- package/src/extension/registration/commands.ts +559 -452
- package/src/extension/registration/compaction-guard.ts +125 -125
- package/src/extension/registration/subagent-helpers.ts +102 -102
- package/src/extension/registration/subagent-tools.ts +220 -159
- package/src/extension/registration/team-tool.ts +159 -99
- package/src/extension/registration/viewers.ts +29 -0
- package/src/extension/result-watcher.ts +128 -128
- package/src/extension/run-bundle-schema.ts +89 -89
- package/src/extension/run-export.ts +73 -73
- package/src/extension/run-import.ts +84 -84
- package/src/extension/run-index.ts +94 -94
- package/src/extension/run-maintenance.ts +142 -142
- package/src/extension/session-summary.ts +8 -8
- package/src/extension/team-manager-command.ts +96 -96
- package/src/extension/team-recommendation.ts +188 -188
- package/src/extension/team-tool/api.ts +5 -2
- package/src/extension/team-tool/cancel.ts +224 -209
- package/src/extension/team-tool/config-patch.ts +36 -36
- package/src/extension/team-tool/context.ts +60 -60
- package/src/extension/team-tool/doctor.ts +242 -242
- package/src/extension/team-tool/handle-settings.ts +421 -195
- package/src/extension/team-tool/inspect.ts +41 -41
- package/src/extension/team-tool/lifecycle-actions.ts +139 -139
- package/src/extension/team-tool/parallel-dispatch.ts +156 -156
- package/src/extension/team-tool/plan.ts +19 -19
- package/src/extension/team-tool/respond.ts +112 -111
- package/src/extension/team-tool/run.ts +246 -229
- package/src/extension/team-tool/status.ts +110 -110
- package/src/extension/team-tool-types.ts +13 -13
- package/src/extension/team-tool.ts +344 -344
- package/src/extension/tool-result.ts +16 -16
- package/src/extension/validate-resources.ts +77 -77
- package/src/hooks/registry.ts +61 -61
- package/src/hooks/types.ts +40 -40
- package/src/i18n.ts +184 -184
- package/src/observability/correlation.ts +35 -35
- package/src/observability/event-to-metric.ts +68 -68
- package/src/observability/exporters/adapter.ts +30 -30
- package/src/observability/exporters/otlp-exporter.ts +106 -92
- package/src/observability/exporters/prometheus-exporter.ts +54 -54
- package/src/observability/metric-registry.ts +87 -87
- package/src/observability/metric-retention.ts +54 -54
- package/src/observability/metric-sink.ts +81 -56
- package/src/observability/metrics-primitives.ts +167 -167
- package/src/prompt/prompt-runtime.ts +72 -72
- package/src/runtime/adaptive-plan.ts +338 -0
- package/src/runtime/agent-control.ts +169 -169
- package/src/runtime/agent-memory.ts +72 -72
- package/src/runtime/agent-observability.ts +114 -114
- package/src/runtime/async-marker.ts +26 -26
- package/src/runtime/async-runner.ts +153 -153
- package/src/runtime/attention-events.ts +28 -28
- package/src/runtime/auto-resume.ts +100 -100
- package/src/runtime/background-runner.ts +122 -89
- package/src/runtime/cancellation.ts +61 -61
- package/src/runtime/capability-inventory.ts +116 -116
- package/src/runtime/child-pi-pool.ts +68 -0
- package/src/runtime/child-pi.ts +541 -461
- package/src/runtime/code-summary.ts +247 -247
- package/src/runtime/compaction-summary.ts +271 -271
- package/src/runtime/concurrency.ts +58 -58
- package/src/runtime/crash-recovery.ts +317 -301
- package/src/runtime/crew-agent-records.ts +379 -281
- package/src/runtime/crew-agent-runtime.ts +60 -60
- package/src/runtime/cross-extension-rpc.ts +72 -0
- package/src/runtime/custom-tools/irc-tool.ts +201 -201
- package/src/runtime/custom-tools/submit-result-tool.ts +90 -90
- package/src/runtime/deadletter.ts +47 -47
- package/src/runtime/delivery-coordinator.ts +176 -176
- package/src/runtime/delta-conflict.ts +360 -360
- package/src/runtime/diagnostic-export.ts +102 -102
- package/src/runtime/direct-run.ts +35 -35
- package/src/runtime/effectiveness.ts +82 -81
- package/src/runtime/errors/crew-errors.ts +166 -0
- package/src/runtime/event-stream-bridge.ts +92 -92
- package/src/runtime/foreground-control.ts +82 -82
- package/src/runtime/green-contract.ts +46 -46
- package/src/runtime/group-join.ts +234 -106
- package/src/runtime/heartbeat-watcher.ts +145 -124
- package/src/runtime/iteration-hooks.ts +267 -267
- package/src/runtime/live-agent-control.ts +88 -88
- package/src/runtime/live-agent-manager.ts +377 -179
- package/src/runtime/live-control-realtime.ts +36 -36
- package/src/runtime/live-session-runtime.ts +676 -600
- package/src/runtime/loop-gates.ts +129 -129
- package/src/runtime/manifest-cache.ts +263 -263
- package/src/runtime/mcp-proxy.ts +113 -113
- package/src/runtime/metric-parser.ts +40 -40
- package/src/runtime/model-fallback.ts +282 -274
- package/src/runtime/model-resolver.ts +118 -0
- package/src/runtime/output-validator.ts +187 -187
- package/src/runtime/overflow-recovery.ts +175 -175
- package/src/runtime/parallel-research.ts +44 -44
- package/src/runtime/parallel-utils.ts +156 -156
- package/src/runtime/parent-guard.ts +80 -80
- package/src/runtime/phase-progress.ts +217 -217
- package/src/runtime/pi-args.ts +165 -165
- package/src/runtime/pi-json-output.ts +111 -111
- package/src/runtime/pi-spawn.ts +167 -167
- package/src/runtime/policy-engine.ts +79 -79
- package/src/runtime/post-checks.ts +125 -125
- package/src/runtime/post-exit-stdio-guard.ts +86 -86
- package/src/runtime/process-status.ts +97 -73
- package/src/runtime/progress-event-coalescer.ts +43 -43
- package/src/runtime/recovery-recipes.ts +74 -74
- package/src/runtime/retry-executor.ts +81 -81
- package/src/runtime/role-permission.ts +39 -39
- package/src/runtime/run-tracker.ts +99 -0
- package/src/runtime/runtime-policy.ts +21 -0
- package/src/runtime/runtime-resolver.ts +94 -91
- package/src/runtime/scheduler.ts +294 -0
- package/src/runtime/semaphore.ts +131 -131
- package/src/runtime/sensitive-paths.ts +92 -92
- package/src/runtime/session-usage.ts +79 -79
- package/src/runtime/settings-store.ts +103 -0
- package/src/runtime/sidechain-output.ts +29 -29
- package/src/runtime/skill-instructions.ts +222 -222
- package/src/runtime/stale-reconciler.ts +198 -189
- package/src/runtime/streaming-output.ts +47 -0
- package/src/runtime/subagent-manager.ts +404 -400
- package/src/runtime/subprocess-tool-registry.ts +67 -67
- package/src/runtime/task-display.ts +38 -38
- package/src/runtime/task-graph-scheduler.ts +122 -122
- package/src/runtime/task-graph.ts +207 -207
- package/src/runtime/task-output-context.ts +177 -177
- package/src/runtime/task-packet.ts +93 -93
- package/src/runtime/task-quality.ts +207 -207
- package/src/runtime/task-runner/capabilities.ts +78 -78
- package/src/runtime/task-runner/live-executor.ts +131 -113
- package/src/runtime/task-runner/progress.ts +119 -119
- package/src/runtime/task-runner/prompt-builder.ts +139 -139
- package/src/runtime/task-runner/prompt-pipeline.ts +64 -64
- package/src/runtime/task-runner/result-utils.ts +14 -14
- package/src/runtime/task-runner/run-projection.ts +103 -103
- package/src/runtime/task-runner/state-helpers.ts +22 -22
- package/src/runtime/task-runner.ts +469 -459
- package/src/runtime/team-runner.ts +693 -945
- package/src/runtime/usage-tracker.ts +71 -0
- package/src/runtime/worker-heartbeat.ts +21 -21
- package/src/runtime/worker-startup.ts +57 -57
- package/src/runtime/workflow-state.ts +187 -187
- package/src/runtime/yield-handler.ts +190 -190
- package/src/schema/config-schema.ts +172 -168
- package/src/schema/team-tool-schema.ts +126 -126
- package/src/schema/validation-types.ts +151 -148
- package/src/skills/discover-skills.ts +67 -67
- package/src/skills/skill-templates.ts +374 -374
- package/src/state/active-run-registry.ts +227 -191
- package/src/state/artifact-store.ts +130 -129
- package/src/state/atomic-write.ts +262 -195
- package/src/state/blob-store.ts +116 -116
- package/src/state/contracts.ts +111 -111
- package/src/state/event-log-rotation.ts +161 -158
- package/src/state/event-log.ts +383 -303
- package/src/state/event-reconstructor.ts +217 -217
- package/src/state/jsonl-writer.ts +82 -82
- package/src/state/locks.ts +146 -146
- package/src/state/mailbox.ts +446 -405
- package/src/state/state-store.ts +364 -351
- package/src/state/task-claims.ts +44 -44
- package/src/state/types.ts +285 -285
- package/src/state/usage.ts +29 -29
- package/src/subagents/async-entry.ts +1 -1
- package/src/subagents/index.ts +3 -3
- package/src/subagents/live/control.ts +1 -1
- package/src/subagents/live/manager.ts +1 -1
- package/src/subagents/live/realtime.ts +1 -1
- package/src/subagents/live/session-runtime.ts +1 -1
- package/src/subagents/manager.ts +1 -1
- package/src/subagents/spawn.ts +1 -1
- package/src/teams/discover-teams.ts +116 -116
- package/src/teams/team-config.ts +27 -27
- package/src/teams/team-serializer.ts +38 -38
- package/src/types/diff.d.ts +18 -18
- package/src/ui/agent-management-overlay.ts +144 -144
- package/src/ui/crew-widget.ts +487 -370
- package/src/ui/dashboard-panes/agents-pane.ts +109 -28
- package/src/ui/dashboard-panes/cancellation-pane.ts +42 -42
- package/src/ui/dashboard-panes/capability-pane.ts +59 -59
- package/src/ui/dashboard-panes/health-pane.ts +30 -30
- package/src/ui/dashboard-panes/mailbox-pane.ts +35 -35
- package/src/ui/dashboard-panes/progress-pane.ts +30 -30
- package/src/ui/dashboard-panes/transcript-pane.ts +10 -10
- package/src/ui/heartbeat-aggregator.ts +63 -63
- package/src/ui/keybinding-map.ts +97 -94
- package/src/ui/live-conversation-overlay.ts +152 -0
- package/src/ui/live-run-sidebar.ts +180 -180
- package/src/ui/mascot.ts +442 -442
- package/src/ui/overlays/agent-picker-overlay.ts +57 -57
- package/src/ui/overlays/confirm-overlay.ts +58 -58
- package/src/ui/overlays/mailbox-compose-overlay.ts +144 -144
- package/src/ui/overlays/mailbox-compose-preview.ts +63 -63
- package/src/ui/overlays/mailbox-detail-overlay.ts +122 -122
- package/src/ui/pi-ui-compat.ts +57 -57
- package/src/ui/powerbar-publisher.ts +221 -197
- package/src/ui/render-scheduler.ts +216 -143
- package/src/ui/run-action-dispatcher.ts +118 -118
- package/src/ui/run-dashboard.ts +526 -464
- package/src/ui/run-event-bus.ts +208 -208
- package/src/ui/run-snapshot-cache.ts +826 -777
- package/src/ui/settings-overlay.ts +721 -0
- package/src/ui/snapshot-types.ts +86 -70
- package/src/ui/theme-adapter.ts +190 -190
- package/src/ui/tool-progress-formatter.ts +89 -0
- package/src/ui/transcript-cache.ts +94 -94
- package/src/ui/transcript-viewer.ts +335 -335
- package/src/utils/conflict-detect.ts +662 -0
- package/src/utils/file-coalescer.ts +86 -86
- package/src/utils/frontmatter.ts +68 -68
- package/src/utils/fs-watch.ts +88 -31
- package/src/utils/gh-protocol.ts +479 -0
- package/src/utils/ids.ts +17 -17
- package/src/utils/incremental-reader.ts +104 -104
- package/src/utils/internal-error.ts +6 -6
- package/src/utils/names.ts +27 -27
- package/src/utils/paths.ts +102 -63
- package/src/utils/redaction.ts +44 -44
- package/src/utils/safe-paths.ts +47 -47
- package/src/utils/scan-cache.ts +136 -136
- package/src/utils/sse-parser.ts +134 -134
- package/src/utils/task-name-generator.ts +337 -337
- package/src/utils/timings.ts +33 -33
- package/src/utils/visual.ts +243 -198
- package/src/workflows/discover-workflows.ts +139 -139
- package/src/workflows/validate-workflow.ts +40 -40
- package/src/workflows/workflow-config.ts +26 -26
- package/src/workflows/workflow-serializer.ts +32 -32
- package/src/worktree/branch-freshness.ts +45 -45
- package/src/worktree/cleanup.ts +75 -75
- package/src/worktree/worktree-manager.ts +188 -188
- package/teams/default.team.md +12 -12
- package/teams/fast-fix.team.md +11 -11
- package/teams/implementation.team.md +18 -18
- package/teams/parallel-research.team.md +14 -14
- package/teams/research.team.md +11 -11
- package/teams/review.team.md +12 -12
- package/tsconfig.json +19 -19
- package/workflows/default.workflow.md +30 -30
- package/workflows/fast-fix.workflow.md +23 -23
- package/workflows/implementation.workflow.md +43 -43
- package/workflows/parallel-research.workflow.md +46 -46
- package/workflows/research.workflow.md +22 -22
- package/workflows/review.workflow.md +30 -30
- package/skills/task-packet/SKILL.md +0 -28
- package/skills/verify-evidence/SKILL.md +0 -27
|
@@ -1,945 +1,693 @@
|
|
|
1
|
-
import * as fs from "node:fs";
|
|
2
|
-
import type { AgentConfig } from "../agents/agent-config.ts";
|
|
3
|
-
import type { CrewLimitsConfig, CrewRuntimeConfig, CrewReliabilityConfig } from "../config/config.ts";
|
|
4
|
-
import type { CrewRuntimeCapabilities } from "./runtime-resolver.ts";
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import
|
|
9
|
-
import
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
26
|
-
import type
|
|
27
|
-
import {
|
|
28
|
-
import {
|
|
29
|
-
import {
|
|
30
|
-
import {
|
|
31
|
-
import {
|
|
32
|
-
import {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
const
|
|
132
|
-
|
|
133
|
-
const
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
const
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
const
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
const
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
status
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
"",
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
};
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
}
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
}
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
}
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
const
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
}
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
if (readyBatch.length === 0) {
|
|
695
|
-
if (approvalPending && candidateBatch.some(isMutatingTask)) {
|
|
696
|
-
await saveRunTasksAsync(manifest, tasks);
|
|
697
|
-
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
698
|
-
manifest = updateRunStatus(manifest, "blocked", "Plan approval required before mutating implementation tasks run.");
|
|
699
|
-
return { manifest, tasks };
|
|
700
|
-
}
|
|
701
|
-
tasks = markBlocked(tasks, "No ready queued task; dependency graph may be invalid.");
|
|
702
|
-
await saveRunTasksAsync(manifest, tasks);
|
|
703
|
-
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
704
|
-
manifest = updateRunStatus(manifest, "blocked", "No ready queued task.");
|
|
705
|
-
return { manifest, tasks };
|
|
706
|
-
}
|
|
707
|
-
|
|
708
|
-
appendEvent(manifest.eventsPath, { type: "task.progress", runId: manifest.runId, message: `Starting ready batch with ${readyBatch.length} task(s).`, data: { taskIds: readyBatch.map((task) => task.id), readyCount: snapshot.ready.length, blockedCount: snapshot.blocked.length, runningCount: snapshot.running.length, doneCount: snapshot.done.length, selectedCount: readyBatch.length, maxConcurrent: concurrency.maxConcurrent, defaultConcurrency: concurrency.defaultConcurrency, concurrencyReason: approvalPending ? `${concurrency.reason};plan-approval-read-only` : concurrency.reason } });
|
|
709
|
-
// Execute before_task_start hooks for the batch
|
|
710
|
-
for (const task of readyBatch) {
|
|
711
|
-
const taskReport = await executeHook("before_task_start", { runId: manifest.runId, taskId: task.id, cwd: manifest.cwd });
|
|
712
|
-
appendHookEvent(manifest, taskReport);
|
|
713
|
-
if (taskReport.outcome === "block") {
|
|
714
|
-
tasks = tasks.map((t) => t.id === task.id ? { ...t, status: "skipped" as const, error: taskReport.reason ?? "before_task_start hook blocked execution." } : t);
|
|
715
|
-
manifest = updateRunStatus(manifest, manifest.status, `Task '${task.id}' blocked by hook.`);
|
|
716
|
-
}
|
|
717
|
-
}
|
|
718
|
-
const batchTasks = readyBatch.filter((task) => tasks.find((t) => t.id === task.id && t.status !== "skipped"));
|
|
719
|
-
if (batchTasks.length > 1) {
|
|
720
|
-
appendEvent(manifest.eventsPath, { type: "task.parallel_start", runId: manifest.runId, message: `Launching ${batchTasks.length} tasks in PARALLEL (concurrency=${concurrency.selectedCount}): ${batchTasks.map((t) => `${t.role}(${t.id})`).join(", ")}`, data: { taskIds: batchTasks.map((t) => t.id), roles: batchTasks.map((t) => t.role), concurrency: concurrency.selectedCount } });
|
|
721
|
-
}
|
|
722
|
-
const results = await mapConcurrent(
|
|
723
|
-
batchTasks,
|
|
724
|
-
concurrency.selectedCount,
|
|
725
|
-
async (task) => {
|
|
726
|
-
const step = findStep(workflow, task);
|
|
727
|
-
const agent = findAgent(input.agents, task);
|
|
728
|
-
const teamRole = input.team.roles.find((role) => role.name === task.role);
|
|
729
|
-
const baseInput = { manifest, tasks, task, step, agent, signal: input.signal, executeWorkers: input.executeWorkers, runtimeKind: input.runtime?.kind, runtimeConfig: input.runtimeConfig, parentContext: input.parentContext, parentModel: input.parentModel, modelRegistry: input.modelRegistry, modelOverride: input.modelOverride, teamRoleModel: teamRole?.model, teamRoleSkills: teamRole?.skills, skillOverride: input.skillOverride, limits: input.limits, onJsonEvent: input.onJsonEvent };
|
|
730
|
-
if (input.reliability?.autoRetry !== true) return withCorrelation(childCorrelation(manifest.runId, task.id), () => runTeamTask(baseInput));
|
|
731
|
-
let lastFailed: { manifest: TeamRunManifest; tasks: TeamTaskState[] } | undefined;
|
|
732
|
-
let lastAttemptId: string | undefined;
|
|
733
|
-
const attemptsSoFar: TaskAttemptState[] = [...(task.attempts ?? [])];
|
|
734
|
-
const policy = retryPolicyFromConfig(input.reliability);
|
|
735
|
-
try {
|
|
736
|
-
return await executeWithRetry(async (attempt, info) => {
|
|
737
|
-
const startedAt = new Date().toISOString();
|
|
738
|
-
const inFlightAttempts: TaskAttemptState[] = [...attemptsSoFar, { attemptId: info.attemptId, startedAt }];
|
|
739
|
-
input.metricRegistry?.counter("crew.task.retry_attempt_total", "Retry attempts by run and task").inc({ runId: manifest.runId, taskId: task.id });
|
|
740
|
-
const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
|
|
741
|
-
const freshManifest = fresh?.manifest ?? manifest;
|
|
742
|
-
const freshTasks = fresh?.tasks ?? tasks;
|
|
743
|
-
const freshTask = freshTasks.find((item) => item.id === task.id) ?? task;
|
|
744
|
-
if (freshTask.status !== "queued" && freshTask.status !== "running") return { manifest: freshManifest, tasks: freshTasks };
|
|
745
|
-
const taskWithAttempt: TeamTaskState = { ...freshTask, attempts: inFlightAttempts };
|
|
746
|
-
const result = await withCorrelation(childCorrelation(freshManifest.runId, task.id), () => runTeamTask({ ...baseInput, manifest: freshManifest, tasks: freshTasks, task: taskWithAttempt }));
|
|
747
|
-
const failed = failedTaskFrom(result, task.id);
|
|
748
|
-
const endedAt = new Date().toISOString();
|
|
749
|
-
const finishedAttempt: TaskAttemptState = { attemptId: info.attemptId, startedAt, endedAt, ...(failed?.error ? { error: failed.error } : {}) };
|
|
750
|
-
attemptsSoFar.push(finishedAttempt);
|
|
751
|
-
const withAttempt = result.tasks.map((item) => item.id === task.id ? { ...item, attempts: [...attemptsSoFar] } : item);
|
|
752
|
-
const enriched = { manifest: result.manifest, tasks: withAttempt };
|
|
753
|
-
if (failed) {
|
|
754
|
-
lastFailed = enriched;
|
|
755
|
-
throw new Error(failed.error ?? `Task ${task.id} failed.`);
|
|
756
|
-
}
|
|
757
|
-
input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe({ runId: manifest.runId, team: input.team.name }, Math.max(0, attempt - 1));
|
|
758
|
-
return enriched;
|
|
759
|
-
}, policy, {
|
|
760
|
-
signal: input.signal,
|
|
761
|
-
attemptId: (attempt) => `${manifest.runId}:${task.id}:attempt-${attempt}`,
|
|
762
|
-
onAttemptFailed: (attempt, error, delayMs, info) => {
|
|
763
|
-
lastAttemptId = info.attemptId;
|
|
764
|
-
appendEvent(manifest.eventsPath, { type: "crew.task.retry_attempt", runId: manifest.runId, taskId: task.id, message: error.message, data: { attempt, attemptId: info.attemptId, delayMs }, metadata: { attemptId: info.attemptId } });
|
|
765
|
-
input.metricRegistry?.histogram("crew.task.retry_delay_ms", "Retry backoff delay, milliseconds").observe({ runId: manifest.runId, taskId: task.id }, delayMs);
|
|
766
|
-
},
|
|
767
|
-
onRetryGivenUp: (attempts, error, info) => {
|
|
768
|
-
lastAttemptId = info.attemptId;
|
|
769
|
-
appendDeadletter(manifest, { runId: manifest.runId, taskId: task.id, reason: "max-retries", attempts, attemptId: info.attemptId, lastError: error.message, timestamp: new Date().toISOString() });
|
|
770
|
-
input.metricRegistry?.counter("crew.task.deadletter_total", "Deadletter triggers by reason").inc({ reason: "max-retries" });
|
|
771
|
-
input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe({ runId: manifest.runId, team: input.team.name }, Math.max(0, attempts - 1));
|
|
772
|
-
},
|
|
773
|
-
});
|
|
774
|
-
} catch (retryError) {
|
|
775
|
-
if (retryError instanceof CrewCancellationError || input.signal?.aborted) {
|
|
776
|
-
const reason = retryError instanceof CrewCancellationError ? retryError.reason : cancellationReasonFromSignal(input.signal);
|
|
777
|
-
const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
|
|
778
|
-
const freshManifest = fresh?.manifest ?? manifest;
|
|
779
|
-
const freshTasks = fresh?.tasks ?? tasks;
|
|
780
|
-
const cancelledTasks = freshTasks.map((item) => item.id === task.id && (item.status === "queued" || item.status === "running") ? { ...item, status: "cancelled" as const, finishedAt: new Date().toISOString(), error: `${reason.message} (${reason.code})` } : item);
|
|
781
|
-
appendEvent(freshManifest.eventsPath, { type: "task.cancelled", runId: freshManifest.runId, taskId: task.id, message: reason.message, data: { reason, phase: "retry" }, metadata: lastAttemptId ? { attemptId: lastAttemptId } : undefined });
|
|
782
|
-
return { manifest: updateRunStatus(freshManifest, "cancelled", reason.message), tasks: cancelledTasks };
|
|
783
|
-
}
|
|
784
|
-
if (lastFailed) return lastFailed;
|
|
785
|
-
const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
|
|
786
|
-
const freshManifest = fresh?.manifest ?? manifest;
|
|
787
|
-
const freshTasks = fresh?.tasks ?? tasks;
|
|
788
|
-
const freshTask = freshTasks.find((item) => item.id === task.id) ?? task;
|
|
789
|
-
if (freshTask.status !== "queued" && freshTask.status !== "running") return { manifest: freshManifest, tasks: freshTasks };
|
|
790
|
-
return withCorrelation(childCorrelation(freshManifest.runId, task.id), () => runTeamTask({ ...baseInput, manifest: freshManifest, tasks: freshTasks, task: freshTask }));
|
|
791
|
-
}
|
|
792
|
-
},
|
|
793
|
-
);
|
|
794
|
-
if (results.length === 0) break;
|
|
795
|
-
manifest = { ...results.at(-1)!.manifest, artifacts: mergeArtifacts([manifest.artifacts, ...results.map((item) => item.manifest.artifacts)].flat()) };
|
|
796
|
-
tasks = __test__mergeTaskUpdates(tasks, results);
|
|
797
|
-
|
|
798
|
-
// Advance workflow phases whose tasks are all in terminal state
|
|
799
|
-
const terminalStatuses = new Set(["completed", "failed", "skipped", "cancelled"]);
|
|
800
|
-
const phaseTaskMap = new Map<string, string[]>();
|
|
801
|
-
for (const task of tasks) {
|
|
802
|
-
if (!task.stepId) continue;
|
|
803
|
-
const existing = phaseTaskMap.get(task.stepId) ?? [];
|
|
804
|
-
existing.push(task.id);
|
|
805
|
-
phaseTaskMap.set(task.stepId, existing);
|
|
806
|
-
}
|
|
807
|
-
for (let pi = wfMachine.currentPhaseIndex; pi < wfMachine.phases.length; pi++) {
|
|
808
|
-
const phase = wfMachine.phases[pi]!;
|
|
809
|
-
const phaseTaskIds = phaseTaskMap.get(phase.name) ?? [];
|
|
810
|
-
if (phaseTaskIds.length === 0) continue;
|
|
811
|
-
const allTerminal = phaseTaskIds.every((taskId) => {
|
|
812
|
-
const task = tasks.find((t) => t.id === taskId);
|
|
813
|
-
return task ? terminalStatuses.has(task.status) : false;
|
|
814
|
-
});
|
|
815
|
-
if (!allTerminal) break;
|
|
816
|
-
if (phase.status !== "completed" && phase.status !== "failed" && phase.status !== "skipped") {
|
|
817
|
-
const completedArtifacts = manifest.artifacts.filter((a) => a.kind === "result" || a.kind === "summary").map((a) => a.path);
|
|
818
|
-
const previousPhaseStatus = pi > 0 ? (wfMachine.phases[pi - 1]?.status ?? "pending") : "completed";
|
|
819
|
-
const wfContext: PhaseGuardContext = {
|
|
820
|
-
completedArtifacts,
|
|
821
|
-
previousPhaseStatus,
|
|
822
|
-
taskResults: tasks.filter((t) => t.status === "completed").map((t) => ({ taskId: t.id, status: t.status, outputPath: t.resultArtifact?.path })),
|
|
823
|
-
};
|
|
824
|
-
// Determine phase transition status based on individual task outcomes
|
|
825
|
-
const phaseTasks = phaseTaskIds.map((taskId) => tasks.find((t) => t.id === taskId)).filter((t): t is NonNullable<typeof t> => t !== undefined);
|
|
826
|
-
const hasFailedOrCancelled = phaseTasks.some((t) => t.status === "failed" || t.status === "cancelled");
|
|
827
|
-
const phaseStatus = hasFailedOrCancelled ? "failed" : "completed";
|
|
828
|
-
const transition = transitionPhase(wfMachine, pi, phaseStatus, wfContext);
|
|
829
|
-
wfMachine = transition.machine;
|
|
830
|
-
if (transition.guardResult && !transition.guardResult.allowed) {
|
|
831
|
-
appendEvent(manifest.eventsPath, { type: "workflow.phase_guard_blocked", runId: manifest.runId, message: `Workflow phase '${phase.name}' guard blocked: ${transition.guardResult.reason ?? "unknown"}`, data: { phaseIndex: pi, phaseName: phase.name, reason: transition.guardResult.reason } });
|
|
832
|
-
break;
|
|
833
|
-
}
|
|
834
|
-
appendEvent(manifest.eventsPath, { type: phaseStatus === "failed" ? "workflow.phase_failed" : "workflow.phase_completed", runId: manifest.runId, message: `Workflow phase '${phase.name}' ${phaseStatus}.`, data: { phaseIndex: pi, phaseStatus } });
|
|
835
|
-
}
|
|
836
|
-
wfMachine = { ...wfMachine, currentPhaseIndex: pi + 1 };
|
|
837
|
-
}
|
|
838
|
-
|
|
839
|
-
const cancelledResult = results.find((item) => item.manifest.status === "cancelled");
|
|
840
|
-
if (cancelledResult || input.signal?.aborted) {
|
|
841
|
-
const reason = input.signal?.aborted ? cancellationReasonFromSignal(input.signal) : undefined;
|
|
842
|
-
const message = reason?.message ?? cancelledResult?.manifest.summary ?? "Run cancelled during task execution.";
|
|
843
|
-
manifest = { ...manifest, status: "running" };
|
|
844
|
-
manifest = updateRunStatus(manifest, "cancelled", message);
|
|
845
|
-
await saveRunTasksAsync(manifest, tasks);
|
|
846
|
-
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
847
|
-
await saveRunManifestAsync(manifest);
|
|
848
|
-
appendEvent(manifest.eventsPath, { type: "run.cancelled", runId: manifest.runId, message, data: { reason, phase: "task-batch", cancelledResultRunId: cancelledResult?.manifest.runId } });
|
|
849
|
-
return { manifest, tasks };
|
|
850
|
-
}
|
|
851
|
-
queueIndex = buildTaskGraphIndex(tasks);
|
|
852
|
-
const injectedAfterBatch = attemptAdaptivePlan();
|
|
853
|
-
if (injectedAfterBatch.missing) {
|
|
854
|
-
tasks = markBlocked(tasks, "Adaptive planner did not produce a valid subagent plan.");
|
|
855
|
-
await saveRunTasksAsync(manifest, tasks);
|
|
856
|
-
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
857
|
-
manifest = updateRunStatus(manifest, "blocked", "Adaptive planner did not produce a valid subagent plan.");
|
|
858
|
-
return { manifest, tasks };
|
|
859
|
-
}
|
|
860
|
-
if (injectedAfterBatch.injected) {
|
|
861
|
-
manifest = requiresPlanApproval(workflow, input.runtimeConfig) ? ensurePlanApprovalRequested(manifest, tasks) : manifest;
|
|
862
|
-
queueIndex = buildTaskGraphIndex(tasks);
|
|
863
|
-
} else if (requiresPlanApproval(workflow, input.runtimeConfig) && hasPendingMutatingAdaptiveTask(tasks)) {
|
|
864
|
-
manifest = ensurePlanApprovalRequested(manifest, tasks);
|
|
865
|
-
}
|
|
866
|
-
if (manifest.planApproval?.status === "cancelled") {
|
|
867
|
-
tasks = cancelPlanTasks(tasks, "Plan approval was cancelled.");
|
|
868
|
-
await saveRunTasksAsync(manifest, tasks);
|
|
869
|
-
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
870
|
-
manifest = updateRunStatus(manifest, "cancelled", "Plan approval was cancelled.");
|
|
871
|
-
return { manifest, tasks };
|
|
872
|
-
}
|
|
873
|
-
await saveRunTasksAsync(manifest, tasks);
|
|
874
|
-
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
875
|
-
const completedBatch = batchTasks.map((task) => tasks.find((item) => item.id === task.id) ?? task);
|
|
876
|
-
const batchArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
877
|
-
kind: "summary",
|
|
878
|
-
relativePath: `batches/${batchTasks.map((task) => task.id).join("+")}.md`,
|
|
879
|
-
producer: "team-runner",
|
|
880
|
-
content: aggregateTaskOutputs(completedBatch, manifest),
|
|
881
|
-
});
|
|
882
|
-
const groupDelivery = deliverGroupJoin({ manifest, mode: resolveGroupJoinMode(input.runtimeConfig), batch: batchTasks, allTasks: tasks });
|
|
883
|
-
manifest = { ...manifest, artifacts: mergeArtifacts([...manifest.artifacts, batchArtifact, ...(groupDelivery?.artifact ? [groupDelivery.artifact] : [])]) };
|
|
884
|
-
manifest = writeProgress(manifest, tasks, "team-runner", input.executeWorkers, input.runtimeConfig);
|
|
885
|
-
await saveRunManifestAsync(manifest);
|
|
886
|
-
}
|
|
887
|
-
|
|
888
|
-
const failed = tasks.find((task) => task.status === "failed");
|
|
889
|
-
const waiting = tasks.find((task) => task.status === "waiting");
|
|
890
|
-
const running = tasks.find((task) => task.status === "running");
|
|
891
|
-
manifest = applyPolicy(manifest, tasks, input.limits);
|
|
892
|
-
const effectiveness = evaluateRunEffectiveness({ manifest, tasks, executeWorkers: input.executeWorkers, runtimeConfig: input.runtimeConfig });
|
|
893
|
-
const effectivenessDecision = effectivenessPolicyDecision(effectiveness);
|
|
894
|
-
if (effectivenessDecision) {
|
|
895
|
-
manifest = { ...manifest, policyDecisions: [...(manifest.policyDecisions ?? []), effectivenessDecision], updatedAt: new Date().toISOString() };
|
|
896
|
-
appendEvent(manifest.eventsPath, { type: "run.effectiveness", runId: manifest.runId, message: effectivenessDecision.message, data: { effectiveness, policyDecision: effectivenessDecision } });
|
|
897
|
-
}
|
|
898
|
-
const blockingDecision = manifest.policyDecisions?.find((item) => item.action === "block" || item.action === "escalate");
|
|
899
|
-
if (failed) {
|
|
900
|
-
manifest = updateRunStatus(manifest, "failed", `Failed at task '${failed.id}'.`);
|
|
901
|
-
} else if (waiting) {
|
|
902
|
-
manifest = updateRunStatus(manifest, "blocked", `Waiting for response to task '${waiting.id}'.`);
|
|
903
|
-
} else if (running) {
|
|
904
|
-
manifest = updateRunStatus(manifest, "blocked", `Task '${running.id}' is still running.`);
|
|
905
|
-
} else if (effectiveness.severity === "failed") {
|
|
906
|
-
manifest = updateRunStatus(manifest, "failed", effectivenessDecision?.message ?? "Run effectiveness guard failed.");
|
|
907
|
-
} else if (effectiveness.severity === "blocked") {
|
|
908
|
-
manifest = updateRunStatus(manifest, "blocked", effectivenessDecision?.message ?? "Run effectiveness guard blocked completion.");
|
|
909
|
-
} else if (blockingDecision) {
|
|
910
|
-
manifest = updateRunStatus(manifest, "blocked", blockingDecision.message);
|
|
911
|
-
} else {
|
|
912
|
-
manifest = updateRunStatus(manifest, "completed", input.executeWorkers ? "Team workflow completed." : "Team workflow scaffold completed without launching child workers.");
|
|
913
|
-
}
|
|
914
|
-
manifest = writeProgress(manifest, tasks, "team-runner", input.executeWorkers, input.runtimeConfig);
|
|
915
|
-
await saveRunManifestAsync(manifest);
|
|
916
|
-
const usage = aggregateUsage(tasks);
|
|
917
|
-
const summaryArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
918
|
-
kind: "summary",
|
|
919
|
-
relativePath: "summary.md",
|
|
920
|
-
producer: "team-runner",
|
|
921
|
-
content: [
|
|
922
|
-
`# pi-crew run ${manifest.runId}`,
|
|
923
|
-
"",
|
|
924
|
-
`Status: ${manifest.status}`,
|
|
925
|
-
`Team: ${manifest.team}`,
|
|
926
|
-
`Workflow: ${manifest.workflow ?? "(none)"}`,
|
|
927
|
-
`Goal: ${manifest.goal}`,
|
|
928
|
-
`Usage: ${formatUsage(usage)}`,
|
|
929
|
-
"",
|
|
930
|
-
"## Tasks",
|
|
931
|
-
...tasks.map(formatTaskProgress),
|
|
932
|
-
"",
|
|
933
|
-
"## Effectiveness",
|
|
934
|
-
...runEffectivenessLines(manifest, tasks, input.executeWorkers, input.runtimeConfig),
|
|
935
|
-
"",
|
|
936
|
-
"## Policy decisions",
|
|
937
|
-
...(manifest.policyDecisions?.length ? summarizePolicyDecisions(manifest.policyDecisions) : ["- (none)"]),
|
|
938
|
-
"",
|
|
939
|
-
].join("\n"),
|
|
940
|
-
});
|
|
941
|
-
manifest = { ...manifest, updatedAt: new Date().toISOString(), artifacts: [...manifest.artifacts, summaryArtifact] };
|
|
942
|
-
await saveRunManifestAsync(manifest);
|
|
943
|
-
await saveRunTasksAsync(manifest, tasks);
|
|
944
|
-
return { manifest, tasks };
|
|
945
|
-
}
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import type { AgentConfig } from "../agents/agent-config.ts";
|
|
3
|
+
import type { CrewLimitsConfig, CrewRuntimeConfig, CrewReliabilityConfig } from "../config/config.ts";
|
|
4
|
+
import type { CrewRuntimeCapabilities } from "./runtime-resolver.ts";
|
|
5
|
+
import type { CrewRuntimeKind } from "./crew-agent-runtime.ts";
|
|
6
|
+
import { resolveTaskRuntimeKind } from "./runtime-policy.ts";
|
|
7
|
+
import { writeArtifact } from "../state/artifact-store.ts";
|
|
8
|
+
import { executeHook, appendHookEvent } from "../hooks/registry.ts";
|
|
9
|
+
import { appendEvent, appendEventFireAndForget } from "../state/event-log.ts";
|
|
10
|
+
import type { TeamConfig } from "../teams/team-config.ts";
|
|
11
|
+
import type { ArtifactDescriptor, PolicyDecision, TeamRunManifest, TaskAttemptState, TeamTaskState } from "../state/types.ts";
|
|
12
|
+
import { loadRunManifestById, saveRunManifest, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/state-store.ts";
|
|
13
|
+
import { aggregateUsage, formatUsage } from "../state/usage.ts";
|
|
14
|
+
import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
|
|
15
|
+
import { evaluateCrewPolicy, summarizePolicyDecisions } from "./policy-engine.ts";
|
|
16
|
+
import { buildRecoveryLedger } from "./recovery-recipes.ts";
|
|
17
|
+
import { buildTaskGraphIndex, refreshTaskGraphQueues, taskGraphSnapshot } from "./task-graph-scheduler.ts";
|
|
18
|
+
import { buildExecutionPlan as buildDagExecutionPlan, getReadyTasks as getDagReadyTasks, type TaskNode } from "./task-graph.ts";
|
|
19
|
+
import { checkBranchFreshness } from "../worktree/branch-freshness.ts";
|
|
20
|
+
import { aggregateTaskOutputs } from "./task-output-context.ts";
|
|
21
|
+
import { readCrewAgents, saveCrewAgents } from "./crew-agent-records.ts";
|
|
22
|
+
import { recordsForMaterializedTasks } from "./task-display.ts";
|
|
23
|
+
import { deliverGroupJoin, resolveGroupJoinMode } from "./group-join.ts";
|
|
24
|
+
import { runTeamTask } from "./task-runner.ts";
|
|
25
|
+
import { terminateLiveAgentsForRun } from "./live-agent-manager.ts";
|
|
26
|
+
import { createWorkflowStateMachine, validatePhasePreconditions, transitionPhase, type PhaseState, type PhaseGuardContext } from "./workflow-state.ts";
|
|
27
|
+
import { executeWithRetry, DEFAULT_RETRY_POLICY, type RetryPolicy } from "./retry-executor.ts";
|
|
28
|
+
import { appendDeadletter } from "./deadletter.ts";
|
|
29
|
+
import type { MetricRegistry } from "../observability/metric-registry.ts";
|
|
30
|
+
import { childCorrelation, withCorrelation } from "../observability/correlation.ts";
|
|
31
|
+
import { resolveBatchConcurrency } from "./concurrency.ts";
|
|
32
|
+
import { mapConcurrent } from "./parallel-utils.ts";
|
|
33
|
+
import { permissionForRole } from "./role-permission.ts";
|
|
34
|
+
import { registerRunPromise, resolveRunPromise, rejectRunPromise } from "./run-tracker.ts";
|
|
35
|
+
import { clearTrackedTaskUsage } from "./usage-tracker.ts";
|
|
36
|
+
import { CrewCancellationError, buildSyntheticTerminalEvidence, cancellationReasonFromSignal } from "./cancellation.ts";
|
|
37
|
+
import { effectivenessPolicyDecision, evaluateRunEffectiveness, formatRunEffectivenessLines } from "./effectiveness.ts";
|
|
38
|
+
|
|
39
|
+
export interface ExecuteTeamRunInput {
|
|
40
|
+
manifest: TeamRunManifest;
|
|
41
|
+
tasks: TeamTaskState[];
|
|
42
|
+
team: TeamConfig;
|
|
43
|
+
workflow: WorkflowConfig;
|
|
44
|
+
agents: AgentConfig[];
|
|
45
|
+
executeWorkers: boolean;
|
|
46
|
+
limits?: CrewLimitsConfig;
|
|
47
|
+
runtime?: CrewRuntimeCapabilities;
|
|
48
|
+
runtimeConfig?: CrewRuntimeConfig;
|
|
49
|
+
parentContext?: string;
|
|
50
|
+
parentModel?: unknown;
|
|
51
|
+
modelRegistry?: unknown;
|
|
52
|
+
modelOverride?: string;
|
|
53
|
+
signal?: AbortSignal;
|
|
54
|
+
reliability?: CrewReliabilityConfig;
|
|
55
|
+
metricRegistry?: MetricRegistry;
|
|
56
|
+
/** Skill override from the team tool. false disables skill injection for this run. */
|
|
57
|
+
skillOverride?: string[] | false;
|
|
58
|
+
/** Optional callback for JSON events from child Pi. Used for overflow recovery tracking. */
|
|
59
|
+
onJsonEvent?: (taskId: string, runId: string, event: unknown) => void;
|
|
60
|
+
/** Workspace where this run was initiated — used for session-scoped live-agent visibility. */
|
|
61
|
+
workspaceId: string;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function findStep(workflow: WorkflowConfig, task: TeamTaskState): WorkflowStep {
|
|
65
|
+
const step = workflow.steps.find((candidate) => candidate.id === task.stepId);
|
|
66
|
+
if (!step) throw new Error(`Workflow step '${task.stepId}' not found for task '${task.id}'.`);
|
|
67
|
+
return step;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function findAgent(agents: AgentConfig[], task: TeamTaskState): AgentConfig {
|
|
71
|
+
const agent = agents.find((candidate) => candidate.name === task.agent);
|
|
72
|
+
if (!agent) throw new Error(`Agent '${task.agent}' not found for task '${task.id}'.`);
|
|
73
|
+
return agent;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function markBlocked(tasks: TeamTaskState[], reason: string): TeamTaskState[] {
|
|
77
|
+
return tasks.map((task) => task.status === "queued" ? { ...task, status: "skipped", error: reason, finishedAt: new Date().toISOString(), graph: task.graph ? { ...task.graph, queue: "blocked" } : undefined } : task);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function mergeArtifacts(items: ArtifactDescriptor[]): ArtifactDescriptor[] {
|
|
81
|
+
const byPath = new Map<string, ArtifactDescriptor>();
|
|
82
|
+
for (const item of items) byPath.set(item.path, item);
|
|
83
|
+
return [...byPath.values()];
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function isNonTerminalTaskStatus(status: TeamTaskState["status"]): boolean {
|
|
87
|
+
return status === "queued" || status === "running" || status === "waiting";
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function shouldMergeTaskUpdate(current: TeamTaskState, updated: TeamTaskState): boolean {
|
|
91
|
+
// Parallel workers receive the same input snapshot. A later result may still
|
|
92
|
+
// contain stale queued/running copies of tasks that another worker already
|
|
93
|
+
// completed. Never let those stale snapshots regress durable task state.
|
|
94
|
+
if (!isNonTerminalTaskStatus(current.status) && isNonTerminalTaskStatus(updated.status)) return false;
|
|
95
|
+
// Prevent a stale completed task from overwriting a fresher one.
|
|
96
|
+
if (current.finishedAt && updated.finishedAt) {
|
|
97
|
+
const currentFinished = new Date(current.finishedAt).getTime();
|
|
98
|
+
const updatedFinished = new Date(updated.finishedAt).getTime();
|
|
99
|
+
if (!Number.isNaN(currentFinished) && !Number.isNaN(updatedFinished) && updatedFinished < currentFinished) return false;
|
|
100
|
+
}
|
|
101
|
+
return updated.status !== current.status || updated.finishedAt !== current.finishedAt || updated.startedAt !== current.startedAt || Boolean(updated.resultArtifact) || Boolean(updated.error) || Boolean(updated.modelAttempts?.length) || Boolean(updated.usage) || Boolean(updated.attempts?.length);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export function __test__mergeTaskUpdates(base: TeamTaskState[], results: Array<{ tasks: TeamTaskState[] }>): TeamTaskState[] {
|
|
105
|
+
let merged = base;
|
|
106
|
+
for (const result of results) {
|
|
107
|
+
for (const updated of result.tasks) {
|
|
108
|
+
const current = merged.find((task) => task.id === updated.id);
|
|
109
|
+
if (!current || !shouldMergeTaskUpdate(current, updated)) continue;
|
|
110
|
+
merged = merged.map((task) => task.id === updated.id ? updated : task);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return refreshTaskGraphQueues(merged);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// 2.8: adaptive-plan parsing/repair/injection moved to src/runtime/adaptive-plan.ts.
|
|
117
|
+
// Re-export the test-only helpers so existing test imports still resolve.
|
|
118
|
+
export { __test__parseAdaptivePlan, __test__repairAdaptivePlan } from "./adaptive-plan.ts";
|
|
119
|
+
import { injectAdaptivePlanIfReady } from "./adaptive-plan.ts";
|
|
120
|
+
|
|
121
|
+
function formatTaskProgress(task: TeamTaskState): string {
|
|
122
|
+
return `- ${task.id}: ${task.status} (${task.role} -> ${task.agent})${task.taskPacket ? ` scope=${task.taskPacket.scope}` : ""}${task.verification ? ` green=${task.verification.observedGreenLevel}/${task.verification.requiredGreenLevel}` : ""}${task.error ? ` - ${task.error}` : ""}`;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function runEffectivenessLines(manifest: TeamRunManifest, tasks: TeamTaskState[], executeWorkers: boolean, runtimeConfig?: CrewRuntimeConfig): string[] {
|
|
126
|
+
return formatRunEffectivenessLines(evaluateRunEffectiveness({ manifest, tasks, executeWorkers, runtimeConfig }));
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function writeProgress(manifest: TeamRunManifest, tasks: TeamTaskState[], producer: string, executeWorkers = true, runtimeConfig?: CrewRuntimeConfig): TeamRunManifest {
|
|
130
|
+
const counts = new Map<string, number>();
|
|
131
|
+
for (const task of tasks) counts.set(task.status, (counts.get(task.status) ?? 0) + 1);
|
|
132
|
+
const queue = taskGraphSnapshot(tasks);
|
|
133
|
+
const progress = writeArtifact(manifest.artifactsRoot, {
|
|
134
|
+
kind: "progress",
|
|
135
|
+
relativePath: "progress.md",
|
|
136
|
+
producer,
|
|
137
|
+
content: [
|
|
138
|
+
`# pi-crew progress ${manifest.runId}`,
|
|
139
|
+
"",
|
|
140
|
+
`Status: ${manifest.status}`,
|
|
141
|
+
`Team: ${manifest.team}`,
|
|
142
|
+
`Workflow: ${manifest.workflow ?? "(none)"}`,
|
|
143
|
+
`Updated: ${new Date().toISOString()}`,
|
|
144
|
+
`Task counts: ${[...counts.entries()].map(([status, count]) => `${status}=${count}`).join(", ") || "none"}`,
|
|
145
|
+
`Queue: ready=${queue.ready.length}, blocked=${queue.blocked.length}, running=${queue.running.length}, done=${queue.done.length}, failed=${queue.failed.length}, cancelled=${queue.cancelled.length}`,
|
|
146
|
+
"",
|
|
147
|
+
"## Tasks",
|
|
148
|
+
...tasks.map(formatTaskProgress),
|
|
149
|
+
"",
|
|
150
|
+
"## Effectiveness",
|
|
151
|
+
...runEffectivenessLines(manifest, tasks, executeWorkers, runtimeConfig),
|
|
152
|
+
"",
|
|
153
|
+
].join("\n"),
|
|
154
|
+
});
|
|
155
|
+
return { ...manifest, updatedAt: new Date().toISOString(), artifacts: [...manifest.artifacts.filter((artifact) => !(artifact.kind === "progress" && artifact.path === progress.path)), progress] };
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?: CrewLimitsConfig): TeamRunManifest {
|
|
159
|
+
const branchFreshness = checkBranchFreshness(manifest.cwd);
|
|
160
|
+
const branchArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
161
|
+
kind: "metadata",
|
|
162
|
+
relativePath: "metadata/branch-freshness.json",
|
|
163
|
+
producer: "branch-freshness",
|
|
164
|
+
content: `${JSON.stringify(branchFreshness, null, 2)}\n`,
|
|
165
|
+
});
|
|
166
|
+
let decisions: PolicyDecision[] = evaluateCrewPolicy({ manifest, tasks, limits });
|
|
167
|
+
if (branchFreshness.status === "stale" || branchFreshness.status === "diverged") {
|
|
168
|
+
const branchDecision: PolicyDecision = {
|
|
169
|
+
action: "notify",
|
|
170
|
+
reason: "branch_stale",
|
|
171
|
+
message: branchFreshness.message,
|
|
172
|
+
createdAt: new Date().toISOString(),
|
|
173
|
+
};
|
|
174
|
+
decisions = [...decisions, branchDecision];
|
|
175
|
+
appendEvent(manifest.eventsPath, { type: "branch.stale", runId: manifest.runId, message: branchFreshness.message, data: { branchFreshness } });
|
|
176
|
+
}
|
|
177
|
+
const policyArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
178
|
+
kind: "metadata",
|
|
179
|
+
relativePath: "policy-decisions.json",
|
|
180
|
+
producer: "policy-engine",
|
|
181
|
+
content: `${JSON.stringify(decisions, null, 2)}\n`,
|
|
182
|
+
});
|
|
183
|
+
const recoveryLedger = buildRecoveryLedger(decisions);
|
|
184
|
+
const recoveryArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
185
|
+
kind: "metadata",
|
|
186
|
+
relativePath: "recovery-ledger.json",
|
|
187
|
+
producer: "recovery-engine",
|
|
188
|
+
content: `${JSON.stringify(recoveryLedger, null, 2)}\n`,
|
|
189
|
+
});
|
|
190
|
+
for (const item of decisions) appendEvent(manifest.eventsPath, { type: item.action === "escalate" ? "policy.escalated" : "policy.action", runId: manifest.runId, taskId: item.taskId, message: item.message, data: { action: item.action, reason: item.reason } });
|
|
191
|
+
for (const item of recoveryLedger.entries) appendEvent(manifest.eventsPath, { type: item.state === "escalation_required" ? "recovery.escalated" : "recovery.attempted", runId: manifest.runId, taskId: item.taskId, message: item.message, data: { scenario: item.scenario, steps: item.steps, attempt: item.attempt, state: item.state } });
|
|
192
|
+
return { ...manifest, updatedAt: new Date().toISOString(), policyDecisions: decisions, artifacts: [...manifest.artifacts.filter((artifact) => !(artifact.kind === "metadata" && (artifact.path.endsWith("policy-decisions.json") || artifact.path.endsWith("recovery-ledger.json") || artifact.path.endsWith("branch-freshness.json")))), branchArtifact, policyArtifact, recoveryArtifact] };
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function retryPolicyFromConfig(config: CrewReliabilityConfig | undefined): RetryPolicy {
|
|
196
|
+
return { ...DEFAULT_RETRY_POLICY, ...(config?.retryPolicy ?? {}) };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function failedTaskFrom(result: { tasks: TeamTaskState[] }, taskId: string): TeamTaskState | undefined {
|
|
200
|
+
return result.tasks.find((item) => item.id === taskId && item.status === "failed");
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function requiresPlanApproval(workflow: WorkflowConfig, runtimeConfig: CrewRuntimeConfig | undefined): boolean {
|
|
204
|
+
return workflow.name === "implementation" && runtimeConfig?.requirePlanApproval === true;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function isPlanApprovalPending(manifest: TeamRunManifest): boolean {
|
|
208
|
+
return manifest.planApproval?.required === true && manifest.planApproval.status === "pending";
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function isMutatingTask(task: TeamTaskState): boolean {
|
|
212
|
+
return permissionForRole(task.role) !== "read_only";
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function ensurePlanApprovalRequested(manifest: TeamRunManifest, tasks: TeamTaskState[]): TeamRunManifest {
|
|
216
|
+
if (manifest.planApproval) return manifest;
|
|
217
|
+
const assessTask = tasks.find((task) => task.stepId === "assess" && task.status === "completed");
|
|
218
|
+
const now = new Date().toISOString();
|
|
219
|
+
const updated: TeamRunManifest = {
|
|
220
|
+
...manifest,
|
|
221
|
+
updatedAt: now,
|
|
222
|
+
planApproval: {
|
|
223
|
+
required: true,
|
|
224
|
+
status: "pending",
|
|
225
|
+
requestedAt: now,
|
|
226
|
+
updatedAt: now,
|
|
227
|
+
planTaskId: assessTask?.id,
|
|
228
|
+
planArtifactPath: assessTask?.resultArtifact?.path,
|
|
229
|
+
},
|
|
230
|
+
};
|
|
231
|
+
saveRunManifest(updated);
|
|
232
|
+
appendEvent(updated.eventsPath, { type: "plan.approval_required", runId: updated.runId, taskId: assessTask?.id, message: "Adaptive implementation plan requires explicit approval before mutating tasks run.", data: { planArtifactPath: assessTask?.resultArtifact?.path } });
|
|
233
|
+
return updated;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function cancelPlanTasks(tasks: TeamTaskState[], reason: string): TeamTaskState[] {
|
|
237
|
+
return tasks.map((task) => task.status === "queued" || task.status === "running" || task.status === "waiting" ? { ...task, status: "cancelled", finishedAt: new Date().toISOString(), error: reason, graph: task.graph ? { ...task.graph, queue: "done" } : undefined } : task);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function hasPendingMutatingAdaptiveTask(tasks: TeamTaskState[]): boolean {
|
|
241
|
+
return tasks.some((task) => task.status === "queued" && task.adaptive && isMutatingTask(task));
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Check whether any task uses explicit `dependsOn` that would benefit from DAG-based
|
|
246
|
+
* execution planning. If so, build an execution plan and use `getDagReadyTasks`
|
|
247
|
+
* to augment the ready-set selection.
|
|
248
|
+
*/
|
|
249
|
+
function dagReadyTaskIds(tasks: TeamTaskState[], completedIds: Set<string>): string[] | null {
|
|
250
|
+
const hasExplicitDeps = tasks.some((t) => t.dependsOn.length > 0);
|
|
251
|
+
if (!hasExplicitDeps) return null;
|
|
252
|
+
const nodes: TaskNode[] = tasks.map((t) => ({
|
|
253
|
+
id: t.id,
|
|
254
|
+
dependsOn: t.dependsOn,
|
|
255
|
+
phase: t.adaptive?.phase ?? t.stepId,
|
|
256
|
+
}));
|
|
257
|
+
const plan = buildDagExecutionPlan(nodes);
|
|
258
|
+
if (plan.hasCycle) return null; // fall back to existing scheduler
|
|
259
|
+
return getDagReadyTasks(plan, completedIds);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
|
|
263
|
+
let workflow = input.workflow;
|
|
264
|
+
let manifest = updateRunStatus(input.manifest, "running", input.executeWorkers ? "Executing team workflow." : "Creating workflow prompts and placeholder results.");
|
|
265
|
+
|
|
266
|
+
const runPromise = registerRunPromise(manifest.runId);
|
|
267
|
+
|
|
268
|
+
const cleanupUsage = (): void => {
|
|
269
|
+
for (const task of input.tasks) clearTrackedTaskUsage(task.id);
|
|
270
|
+
};
|
|
271
|
+
|
|
272
|
+
try {
|
|
273
|
+
const result = await executeTeamRunCore(input, manifest, workflow);
|
|
274
|
+
resolveRunPromise(manifest.runId, result);
|
|
275
|
+
cleanupUsage();
|
|
276
|
+
// Terminate live agents for this run — agents are done when the run ends.
|
|
277
|
+
void terminateLiveAgentsForRun(manifest.runId, "completed", appendEvent, manifest.eventsPath).catch(() => {});
|
|
278
|
+
return result;
|
|
279
|
+
} catch (error) {
|
|
280
|
+
// P1: Catch unhandled errors — ensure manifest/tasks/agents are terminal so they don't stay "running" forever.
|
|
281
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
282
|
+
const loaded = loadRunManifestById(input.manifest.cwd, input.manifest.runId);
|
|
283
|
+
const freshManifest = loaded?.manifest ?? manifest;
|
|
284
|
+
const freshTasks = refreshTaskGraphQueues(loaded?.tasks ?? input.tasks);
|
|
285
|
+
const failedAt = new Date().toISOString();
|
|
286
|
+
const tasks = freshTasks.map((task) =>
|
|
287
|
+
task.status === "running" || task.status === "queued" || task.status === "waiting"
|
|
288
|
+
? { ...task, status: "failed" as const, finishedAt: failedAt, error: message }
|
|
289
|
+
: task,
|
|
290
|
+
);
|
|
291
|
+
manifest = freshManifest;
|
|
292
|
+
try {
|
|
293
|
+
await terminateLiveAgentsForRun(manifest.runId, "failed", appendEvent, manifest.eventsPath);
|
|
294
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
295
|
+
const existingRuntimeByTask = new Map(readCrewAgents(manifest).map((agent) => [agent.taskId, agent.runtime]));
|
|
296
|
+
const globalRuntime = input.runtime?.kind ?? "child-process";
|
|
297
|
+
const runtimeForAgent = (agent: ReturnType<typeof recordsForMaterializedTasks>[number]): CrewRuntimeKind => {
|
|
298
|
+
const task = tasks.find((item) => item.id === agent.taskId);
|
|
299
|
+
return existingRuntimeByTask.get(agent.taskId) ?? resolveTaskRuntimeKind(globalRuntime, task?.role ?? agent.role, input.runtimeConfig?.isolationPolicy);
|
|
300
|
+
};
|
|
301
|
+
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, globalRuntime).map((agent) => ({ ...agent, runtime: runtimeForAgent(agent) })));
|
|
302
|
+
manifest = updateRunStatus(manifest, "failed", `Unhandled error in team runner: ${message}`);
|
|
303
|
+
await saveRunManifestAsync(manifest);
|
|
304
|
+
} catch {
|
|
305
|
+
// Best-effort — state write may also fail
|
|
306
|
+
}
|
|
307
|
+
const result = { manifest, tasks };
|
|
308
|
+
rejectRunPromise(manifest.runId, error instanceof Error ? error : new Error(message));
|
|
309
|
+
cleanupUsage();
|
|
310
|
+
return result;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
async function executeTeamRunCore(
|
|
315
|
+
input: ExecuteTeamRunInput,
|
|
316
|
+
manifest: TeamRunManifest,
|
|
317
|
+
workflow: WorkflowConfig,
|
|
318
|
+
): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
|
|
319
|
+
// Execute before_run_start hook (non-blocking by default)
|
|
320
|
+
const beforeRunReport = await executeHook("before_run_start", { runId: manifest.runId, cwd: manifest.cwd });
|
|
321
|
+
appendHookEvent(manifest, beforeRunReport);
|
|
322
|
+
if (beforeRunReport.outcome === "block") {
|
|
323
|
+
manifest = updateRunStatus(manifest, "blocked", beforeRunReport.reason ?? "before_run_start hook blocked the run.");
|
|
324
|
+
return { manifest, tasks: input.tasks };
|
|
325
|
+
}
|
|
326
|
+
let tasks = refreshTaskGraphQueues(input.tasks);
|
|
327
|
+
let queueIndex = buildTaskGraphIndex(tasks);
|
|
328
|
+
const canInjectAdaptivePlan = workflow.name === "implementation";
|
|
329
|
+
let adaptivePlanInjected = false;
|
|
330
|
+
let adaptivePlanMissing = false;
|
|
331
|
+
const attemptAdaptivePlan = () => {
|
|
332
|
+
if (!canInjectAdaptivePlan || adaptivePlanInjected || adaptivePlanMissing) return { injected: false, missing: false };
|
|
333
|
+
const adaptivePlan = injectAdaptivePlanIfReady({ manifest, tasks, workflow, team: input.team });
|
|
334
|
+
adaptivePlanInjected = adaptivePlanInjected || adaptivePlan.injected;
|
|
335
|
+
adaptivePlanMissing = adaptivePlan.missingPlan;
|
|
336
|
+
workflow = adaptivePlan.workflow;
|
|
337
|
+
if (adaptivePlan.injected) tasks = adaptivePlan.tasks;
|
|
338
|
+
return { injected: adaptivePlan.injected, missing: adaptivePlan.missingPlan };
|
|
339
|
+
};
|
|
340
|
+
const initialAdaptive = attemptAdaptivePlan();
|
|
341
|
+
if (initialAdaptive.missing) {
|
|
342
|
+
tasks = markBlocked(tasks, "Adaptive planner did not produce a valid subagent plan.");
|
|
343
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
344
|
+
manifest = updateRunStatus(manifest, "blocked", "Adaptive planner did not produce a valid subagent plan.");
|
|
345
|
+
return { manifest, tasks };
|
|
346
|
+
}
|
|
347
|
+
if (initialAdaptive.injected) {
|
|
348
|
+
manifest = requiresPlanApproval(workflow, input.runtimeConfig) ? ensurePlanApprovalRequested(manifest, tasks) : manifest;
|
|
349
|
+
queueIndex = buildTaskGraphIndex(tasks);
|
|
350
|
+
} else if (requiresPlanApproval(workflow, input.runtimeConfig) && hasPendingMutatingAdaptiveTask(tasks)) {
|
|
351
|
+
manifest = ensurePlanApprovalRequested(manifest, tasks);
|
|
352
|
+
}
|
|
353
|
+
if (manifest.planApproval?.status === "cancelled") {
|
|
354
|
+
tasks = cancelPlanTasks(tasks, "Plan approval was cancelled.");
|
|
355
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
356
|
+
manifest = updateRunStatus(manifest, "cancelled", "Plan approval was cancelled.");
|
|
357
|
+
return { manifest, tasks };
|
|
358
|
+
}
|
|
359
|
+
manifest = writeProgress(manifest, tasks, "team-runner", input.executeWorkers, input.runtimeConfig);
|
|
360
|
+
await saveRunManifestAsync(manifest);
|
|
361
|
+
const runtimeKind = input.runtime?.kind ?? (input.executeWorkers ? "child-process" : "scaffold");
|
|
362
|
+
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
363
|
+
|
|
364
|
+
// Build a workflow phase state machine from workflow steps for precondition tracking.
|
|
365
|
+
const workflowPhases: PhaseState[] = workflow.steps.map((step): PhaseState => ({
|
|
366
|
+
name: step.id,
|
|
367
|
+
status: "pending",
|
|
368
|
+
inputs: step.reads === false ? [] : Array.isArray(step.reads) ? step.reads : [],
|
|
369
|
+
outputs: step.output === false ? [] : step.output ? [step.output] : [],
|
|
370
|
+
}));
|
|
371
|
+
let wfMachine = createWorkflowStateMachine(workflowPhases);
|
|
372
|
+
|
|
373
|
+
while (tasks.some((task) => task.status === "queued")) {
|
|
374
|
+
if (input.signal?.aborted) {
|
|
375
|
+
const cancelReason = cancellationReasonFromSignal(input.signal);
|
|
376
|
+
const message = `${cancelReason.message} (${cancelReason.code})`;
|
|
377
|
+
const cancelledTaskIds: string[] = [];
|
|
378
|
+
tasks = tasks.map((task) => {
|
|
379
|
+
if (task.status !== "queued" && task.status !== "running" && task.status !== "waiting") return task;
|
|
380
|
+
cancelledTaskIds.push(task.id);
|
|
381
|
+
const base = { ...task, status: "cancelled" as const, finishedAt: new Date().toISOString(), error: message };
|
|
382
|
+
if (task.status === "running") {
|
|
383
|
+
return { ...base, terminalEvidence: [...(task.terminalEvidence ?? []), buildSyntheticTerminalEvidence("worker", cancelReason, task.startedAt)] };
|
|
384
|
+
}
|
|
385
|
+
return base;
|
|
386
|
+
});
|
|
387
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
388
|
+
for (const taskId of cancelledTaskIds) appendEvent(manifest.eventsPath, { type: "task.cancelled", runId: manifest.runId, taskId, message, data: { reason: cancelReason.code } });
|
|
389
|
+
manifest = updateRunStatus(manifest, "cancelled", message, { data: { reason: cancelReason.code, cancelledTaskIds } });
|
|
390
|
+
return { manifest, tasks };
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
const failed = tasks.find((task) => task.status === "failed");
|
|
394
|
+
if (failed) {
|
|
395
|
+
tasks = markBlocked(tasks, `Blocked by failed task '${failed.id}'.`);
|
|
396
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
397
|
+
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
398
|
+
manifest = updateRunStatus(manifest, "failed", `Failed at task '${failed.id}'.`);
|
|
399
|
+
return { manifest, tasks };
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
const snapshot = taskGraphSnapshot(tasks, queueIndex);
|
|
403
|
+
|
|
404
|
+
// DAG-based execution plan: when tasks have explicit dependsOn, use the
|
|
405
|
+
// topological wave planner to determine ready tasks. Fall back to the
|
|
406
|
+
// existing task-graph-scheduler when no explicit deps exist (backward compat).
|
|
407
|
+
const completedIds = new Set(tasks.filter((t) => t.status === "completed").map((t) => t.id));
|
|
408
|
+
const dagReady = dagReadyTaskIds(tasks, completedIds);
|
|
409
|
+
const effectiveReady = dagReady ?? snapshot.ready;
|
|
410
|
+
|
|
411
|
+
// Workflow phase precondition check (non-blocking: log warnings only).
|
|
412
|
+
if (wfMachine.currentPhaseIndex < wfMachine.phases.length) {
|
|
413
|
+
const completedArtifacts = manifest.artifacts.filter((a) => a.kind === "result" || a.kind === "summary").map((a) => a.path);
|
|
414
|
+
const previousPhaseStatus = wfMachine.currentPhaseIndex > 0 ? (wfMachine.phases[wfMachine.currentPhaseIndex - 1]?.status ?? "pending") : "completed";
|
|
415
|
+
const wfContext: PhaseGuardContext = {
|
|
416
|
+
completedArtifacts,
|
|
417
|
+
previousPhaseStatus,
|
|
418
|
+
taskResults: tasks.filter((t) => t.status === "completed").map((t) => ({ taskId: t.id, status: t.status, outputPath: t.resultArtifact?.path })),
|
|
419
|
+
};
|
|
420
|
+
const preconditions = validatePhasePreconditions(wfMachine, wfContext);
|
|
421
|
+
if (!preconditions.ready) {
|
|
422
|
+
appendEvent(manifest.eventsPath, { type: "workflow.preconditions", runId: manifest.runId, message: `Workflow phase '${wfMachine.phases[wfMachine.currentPhaseIndex]?.name}' is missing inputs: ${preconditions.blocking.join(", ")}`, data: { phaseIndex: wfMachine.currentPhaseIndex, phaseName: wfMachine.phases[wfMachine.currentPhaseIndex]?.name, blocking: preconditions.blocking } });
|
|
423
|
+
} else {
|
|
424
|
+
// Advance the machine past completed phases.
|
|
425
|
+
while (wfMachine.currentPhaseIndex < wfMachine.phases.length && wfMachine.phases[wfMachine.currentPhaseIndex]?.status === "completed") {
|
|
426
|
+
wfMachine = { ...wfMachine, currentPhaseIndex: wfMachine.currentPhaseIndex + 1 };
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
const readyRoles = effectiveReady.map((taskId) => tasks.find((task) => task.id === taskId)?.role).filter((role): role is string => Boolean(role));
|
|
432
|
+
const concurrency = resolveBatchConcurrency({ workflowName: workflow.name, workflowMaxConcurrency: workflow.maxConcurrency, teamMaxConcurrency: input.team.maxConcurrency, limitMaxConcurrentWorkers: input.limits?.maxConcurrentWorkers, allowUnboundedConcurrency: input.limits?.allowUnboundedConcurrency, readyCount: effectiveReady.length, workspaceMode: manifest.workspaceMode, readyRoles });
|
|
433
|
+
if (concurrency.reason.includes(";unbounded:")) {
|
|
434
|
+
appendEvent(manifest.eventsPath, { type: "limits.unbounded", runId: manifest.runId, message: "Unbounded worker concurrency was explicitly enabled for this run.", data: { concurrencyReason: concurrency.reason, maxConcurrent: concurrency.maxConcurrent } });
|
|
435
|
+
}
|
|
436
|
+
const approvalPending = isPlanApprovalPending(manifest);
|
|
437
|
+
const readyIds = approvalPending ? effectiveReady : effectiveReady.slice(0, concurrency.selectedCount);
|
|
438
|
+
const candidateBatch = readyIds.map((id) => tasks.find((task) => task.id === id)).filter((task): task is TeamTaskState => Boolean(task));
|
|
439
|
+
const readyBatch = approvalPending ? candidateBatch.filter((task) => !isMutatingTask(task)).slice(0, concurrency.selectedCount) : candidateBatch;
|
|
440
|
+
if (readyBatch.length === 0) {
|
|
441
|
+
if (approvalPending && candidateBatch.some(isMutatingTask)) {
|
|
442
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
443
|
+
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
444
|
+
manifest = updateRunStatus(manifest, "blocked", "Plan approval required before mutating implementation tasks run.");
|
|
445
|
+
return { manifest, tasks };
|
|
446
|
+
}
|
|
447
|
+
tasks = markBlocked(tasks, "No ready queued task; dependency graph may be invalid.");
|
|
448
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
449
|
+
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
450
|
+
manifest = updateRunStatus(manifest, "blocked", "No ready queued task.");
|
|
451
|
+
return { manifest, tasks };
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// 2.2 caller migration: batch progress is high-frequency informational.
|
|
455
|
+
appendEventFireAndForget(manifest.eventsPath, { type: "task.progress", runId: manifest.runId, message: `Starting ready batch with ${readyBatch.length} task(s).`, data: { taskIds: readyBatch.map((task) => task.id), readyCount: snapshot.ready.length, blockedCount: snapshot.blocked.length, runningCount: snapshot.running.length, doneCount: snapshot.done.length, selectedCount: readyBatch.length, maxConcurrent: concurrency.maxConcurrent, defaultConcurrency: concurrency.defaultConcurrency, concurrencyReason: approvalPending ? `${concurrency.reason};plan-approval-read-only` : concurrency.reason } });
|
|
456
|
+
// Execute before_task_start hooks for the batch
|
|
457
|
+
for (const task of readyBatch) {
|
|
458
|
+
const taskReport = await executeHook("before_task_start", { runId: manifest.runId, taskId: task.id, cwd: manifest.cwd });
|
|
459
|
+
appendHookEvent(manifest, taskReport);
|
|
460
|
+
if (taskReport.outcome === "block") {
|
|
461
|
+
tasks = tasks.map((t) => t.id === task.id ? { ...t, status: "skipped" as const, error: taskReport.reason ?? "before_task_start hook blocked execution." } : t);
|
|
462
|
+
manifest = updateRunStatus(manifest, manifest.status, `Task '${task.id}' blocked by hook.`);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
const batchTasks = readyBatch.filter((task) => tasks.find((t) => t.id === task.id && t.status !== "skipped"));
|
|
466
|
+
if (batchTasks.length > 1) {
|
|
467
|
+
appendEvent(manifest.eventsPath, { type: "task.parallel_start", runId: manifest.runId, message: `Launching ${batchTasks.length} tasks in PARALLEL (concurrency=${concurrency.selectedCount}): ${batchTasks.map((t) => `${t.role}(${t.id})`).join(", ")}`, data: { taskIds: batchTasks.map((t) => t.id), roles: batchTasks.map((t) => t.role), concurrency: concurrency.selectedCount } });
|
|
468
|
+
}
|
|
469
|
+
const results = await mapConcurrent(
|
|
470
|
+
batchTasks,
|
|
471
|
+
concurrency.selectedCount,
|
|
472
|
+
async (task) => {
|
|
473
|
+
const step = findStep(workflow, task);
|
|
474
|
+
const agent = findAgent(input.agents, task);
|
|
475
|
+
const teamRole = input.team.roles.find((role) => role.name === task.role);
|
|
476
|
+
const perTaskRuntime = resolveTaskRuntimeKind(runtimeKind, task.role, input.runtimeConfig?.isolationPolicy);
|
|
477
|
+
const baseInput = { manifest, tasks, task, step, agent, signal: input.signal, executeWorkers: input.executeWorkers, runtimeKind: runtimeKind, taskRuntimeOverride: perTaskRuntime !== runtimeKind ? perTaskRuntime : undefined, runtimeConfig: input.runtimeConfig, parentContext: input.parentContext, parentModel: input.parentModel, modelRegistry: input.modelRegistry, modelOverride: input.modelOverride, teamRoleModel: teamRole?.model, teamRoleSkills: teamRole?.skills, skillOverride: input.skillOverride, limits: input.limits, onJsonEvent: input.onJsonEvent, workspaceId: input.workspaceId };
|
|
478
|
+
if (input.reliability?.autoRetry !== true) return withCorrelation(childCorrelation(manifest.runId, task.id), () => runTeamTask(baseInput));
|
|
479
|
+
let lastFailed: { manifest: TeamRunManifest; tasks: TeamTaskState[] } | undefined;
|
|
480
|
+
let lastAttemptId: string | undefined;
|
|
481
|
+
const attemptsSoFar: TaskAttemptState[] = [...(task.attempts ?? [])];
|
|
482
|
+
const policy = retryPolicyFromConfig(input.reliability);
|
|
483
|
+
try {
|
|
484
|
+
return await executeWithRetry(async (attempt, info) => {
|
|
485
|
+
const startedAt = new Date().toISOString();
|
|
486
|
+
const inFlightAttempts: TaskAttemptState[] = [...attemptsSoFar, { attemptId: info.attemptId, startedAt }];
|
|
487
|
+
input.metricRegistry?.counter("crew.task.retry_attempt_total", "Retry attempts by run and task").inc({ runId: manifest.runId, taskId: task.id });
|
|
488
|
+
const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
|
|
489
|
+
const freshManifest = fresh?.manifest ?? manifest;
|
|
490
|
+
const freshTasks = fresh?.tasks ?? tasks;
|
|
491
|
+
const freshTask = freshTasks.find((item) => item.id === task.id) ?? task;
|
|
492
|
+
if (freshTask.status !== "queued" && freshTask.status !== "running") return { manifest: freshManifest, tasks: freshTasks };
|
|
493
|
+
const taskWithAttempt: TeamTaskState = { ...freshTask, attempts: inFlightAttempts };
|
|
494
|
+
const result = await withCorrelation(childCorrelation(freshManifest.runId, task.id), () => runTeamTask({ ...baseInput, manifest: freshManifest, tasks: freshTasks, task: taskWithAttempt }));
|
|
495
|
+
const failed = failedTaskFrom(result, task.id);
|
|
496
|
+
const endedAt = new Date().toISOString();
|
|
497
|
+
const finishedAttempt: TaskAttemptState = { attemptId: info.attemptId, startedAt, endedAt, ...(failed?.error ? { error: failed.error } : {}) };
|
|
498
|
+
attemptsSoFar.push(finishedAttempt);
|
|
499
|
+
const withAttempt = result.tasks.map((item) => item.id === task.id ? { ...item, attempts: [...attemptsSoFar] } : item);
|
|
500
|
+
const enriched = { manifest: result.manifest, tasks: withAttempt };
|
|
501
|
+
if (failed) {
|
|
502
|
+
lastFailed = enriched;
|
|
503
|
+
throw new Error(failed.error ?? `Task ${task.id} failed.`);
|
|
504
|
+
}
|
|
505
|
+
input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe({ runId: manifest.runId, team: input.team.name }, Math.max(0, attempt - 1));
|
|
506
|
+
return enriched;
|
|
507
|
+
}, policy, {
|
|
508
|
+
signal: input.signal,
|
|
509
|
+
attemptId: (attempt) => `${manifest.runId}:${task.id}:attempt-${attempt}`,
|
|
510
|
+
onAttemptFailed: (attempt, error, delayMs, info) => {
|
|
511
|
+
lastAttemptId = info.attemptId;
|
|
512
|
+
appendEvent(manifest.eventsPath, { type: "crew.task.retry_attempt", runId: manifest.runId, taskId: task.id, message: error.message, data: { attempt, attemptId: info.attemptId, delayMs }, metadata: { attemptId: info.attemptId } });
|
|
513
|
+
input.metricRegistry?.histogram("crew.task.retry_delay_ms", "Retry backoff delay, milliseconds").observe({ runId: manifest.runId, taskId: task.id }, delayMs);
|
|
514
|
+
},
|
|
515
|
+
onRetryGivenUp: (attempts, error, info) => {
|
|
516
|
+
lastAttemptId = info.attemptId;
|
|
517
|
+
appendDeadletter(manifest, { runId: manifest.runId, taskId: task.id, reason: "max-retries", attempts, attemptId: info.attemptId, lastError: error.message, timestamp: new Date().toISOString() });
|
|
518
|
+
input.metricRegistry?.counter("crew.task.deadletter_total", "Deadletter triggers by reason").inc({ reason: "max-retries" });
|
|
519
|
+
input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe({ runId: manifest.runId, team: input.team.name }, Math.max(0, attempts - 1));
|
|
520
|
+
},
|
|
521
|
+
});
|
|
522
|
+
} catch (retryError) {
|
|
523
|
+
if (retryError instanceof CrewCancellationError || input.signal?.aborted) {
|
|
524
|
+
const reason = retryError instanceof CrewCancellationError ? retryError.reason : cancellationReasonFromSignal(input.signal);
|
|
525
|
+
const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
|
|
526
|
+
const freshManifest = fresh?.manifest ?? manifest;
|
|
527
|
+
const freshTasks = fresh?.tasks ?? tasks;
|
|
528
|
+
const cancelledTasks = freshTasks.map((item) => item.id === task.id && (item.status === "queued" || item.status === "running") ? { ...item, status: "cancelled" as const, finishedAt: new Date().toISOString(), error: `${reason.message} (${reason.code})` } : item);
|
|
529
|
+
appendEvent(freshManifest.eventsPath, { type: "task.cancelled", runId: freshManifest.runId, taskId: task.id, message: reason.message, data: { reason, phase: "retry" }, metadata: lastAttemptId ? { attemptId: lastAttemptId } : undefined });
|
|
530
|
+
return { manifest: updateRunStatus(freshManifest, "cancelled", reason.message), tasks: cancelledTasks };
|
|
531
|
+
}
|
|
532
|
+
if (lastFailed) return lastFailed;
|
|
533
|
+
const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
|
|
534
|
+
const freshManifest = fresh?.manifest ?? manifest;
|
|
535
|
+
const freshTasks = fresh?.tasks ?? tasks;
|
|
536
|
+
const freshTask = freshTasks.find((item) => item.id === task.id) ?? task;
|
|
537
|
+
if (freshTask.status !== "queued" && freshTask.status !== "running") return { manifest: freshManifest, tasks: freshTasks };
|
|
538
|
+
return withCorrelation(childCorrelation(freshManifest.runId, task.id), () => runTeamTask({ ...baseInput, manifest: freshManifest, tasks: freshTasks, task: freshTask }));
|
|
539
|
+
}
|
|
540
|
+
},
|
|
541
|
+
);
|
|
542
|
+
if (results.length === 0) break;
|
|
543
|
+
manifest = { ...results.at(-1)!.manifest, artifacts: mergeArtifacts([manifest.artifacts, ...results.map((item) => item.manifest.artifacts)].flat()) };
|
|
544
|
+
tasks = __test__mergeTaskUpdates(tasks, results);
|
|
545
|
+
|
|
546
|
+
// Advance workflow phases whose tasks are all in terminal state
|
|
547
|
+
const terminalStatuses = new Set(["completed", "failed", "skipped", "cancelled"]);
|
|
548
|
+
const phaseTaskMap = new Map<string, string[]>();
|
|
549
|
+
for (const task of tasks) {
|
|
550
|
+
if (!task.stepId) continue;
|
|
551
|
+
const existing = phaseTaskMap.get(task.stepId) ?? [];
|
|
552
|
+
existing.push(task.id);
|
|
553
|
+
phaseTaskMap.set(task.stepId, existing);
|
|
554
|
+
}
|
|
555
|
+
for (let pi = wfMachine.currentPhaseIndex; pi < wfMachine.phases.length; pi++) {
|
|
556
|
+
const phase = wfMachine.phases[pi]!;
|
|
557
|
+
const phaseTaskIds = phaseTaskMap.get(phase.name) ?? [];
|
|
558
|
+
if (phaseTaskIds.length === 0) continue;
|
|
559
|
+
const allTerminal = phaseTaskIds.every((taskId) => {
|
|
560
|
+
const task = tasks.find((t) => t.id === taskId);
|
|
561
|
+
return task ? terminalStatuses.has(task.status) : false;
|
|
562
|
+
});
|
|
563
|
+
if (!allTerminal) break;
|
|
564
|
+
if (phase.status !== "completed" && phase.status !== "failed" && phase.status !== "skipped") {
|
|
565
|
+
const completedArtifacts = manifest.artifacts.filter((a) => a.kind === "result" || a.kind === "summary").map((a) => a.path);
|
|
566
|
+
const previousPhaseStatus = pi > 0 ? (wfMachine.phases[pi - 1]?.status ?? "pending") : "completed";
|
|
567
|
+
const wfContext: PhaseGuardContext = {
|
|
568
|
+
completedArtifacts,
|
|
569
|
+
previousPhaseStatus,
|
|
570
|
+
taskResults: tasks.filter((t) => t.status === "completed").map((t) => ({ taskId: t.id, status: t.status, outputPath: t.resultArtifact?.path })),
|
|
571
|
+
};
|
|
572
|
+
// Determine phase transition status based on individual task outcomes
|
|
573
|
+
const phaseTasks = phaseTaskIds.map((taskId) => tasks.find((t) => t.id === taskId)).filter((t): t is NonNullable<typeof t> => t !== undefined);
|
|
574
|
+
const hasFailedOrCancelled = phaseTasks.some((t) => t.status === "failed" || t.status === "cancelled");
|
|
575
|
+
const phaseStatus = hasFailedOrCancelled ? "failed" : "completed";
|
|
576
|
+
const transition = transitionPhase(wfMachine, pi, phaseStatus, wfContext);
|
|
577
|
+
wfMachine = transition.machine;
|
|
578
|
+
if (transition.guardResult && !transition.guardResult.allowed) {
|
|
579
|
+
appendEvent(manifest.eventsPath, { type: "workflow.phase_guard_blocked", runId: manifest.runId, message: `Workflow phase '${phase.name}' guard blocked: ${transition.guardResult.reason ?? "unknown"}`, data: { phaseIndex: pi, phaseName: phase.name, reason: transition.guardResult.reason } });
|
|
580
|
+
break;
|
|
581
|
+
}
|
|
582
|
+
appendEvent(manifest.eventsPath, { type: phaseStatus === "failed" ? "workflow.phase_failed" : "workflow.phase_completed", runId: manifest.runId, message: `Workflow phase '${phase.name}' ${phaseStatus}.`, data: { phaseIndex: pi, phaseStatus } });
|
|
583
|
+
}
|
|
584
|
+
wfMachine = { ...wfMachine, currentPhaseIndex: pi + 1 };
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
const cancelledResult = results.find((item) => item.manifest.status === "cancelled");
|
|
588
|
+
if (cancelledResult || input.signal?.aborted) {
|
|
589
|
+
const reason = input.signal?.aborted ? cancellationReasonFromSignal(input.signal) : undefined;
|
|
590
|
+
const message = reason?.message ?? cancelledResult?.manifest.summary ?? "Run cancelled during task execution.";
|
|
591
|
+
manifest = { ...manifest, status: "running" };
|
|
592
|
+
manifest = updateRunStatus(manifest, "cancelled", message);
|
|
593
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
594
|
+
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
595
|
+
await saveRunManifestAsync(manifest);
|
|
596
|
+
appendEvent(manifest.eventsPath, { type: "run.cancelled", runId: manifest.runId, message, data: { reason, phase: "task-batch", cancelledResultRunId: cancelledResult?.manifest.runId } });
|
|
597
|
+
return { manifest, tasks };
|
|
598
|
+
}
|
|
599
|
+
queueIndex = buildTaskGraphIndex(tasks);
|
|
600
|
+
const injectedAfterBatch = attemptAdaptivePlan();
|
|
601
|
+
if (injectedAfterBatch.missing) {
|
|
602
|
+
tasks = markBlocked(tasks, "Adaptive planner did not produce a valid subagent plan.");
|
|
603
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
604
|
+
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
605
|
+
manifest = updateRunStatus(manifest, "blocked", "Adaptive planner did not produce a valid subagent plan.");
|
|
606
|
+
return { manifest, tasks };
|
|
607
|
+
}
|
|
608
|
+
if (injectedAfterBatch.injected) {
|
|
609
|
+
manifest = requiresPlanApproval(workflow, input.runtimeConfig) ? ensurePlanApprovalRequested(manifest, tasks) : manifest;
|
|
610
|
+
queueIndex = buildTaskGraphIndex(tasks);
|
|
611
|
+
} else if (requiresPlanApproval(workflow, input.runtimeConfig) && hasPendingMutatingAdaptiveTask(tasks)) {
|
|
612
|
+
manifest = ensurePlanApprovalRequested(manifest, tasks);
|
|
613
|
+
}
|
|
614
|
+
if (manifest.planApproval?.status === "cancelled") {
|
|
615
|
+
tasks = cancelPlanTasks(tasks, "Plan approval was cancelled.");
|
|
616
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
617
|
+
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
618
|
+
manifest = updateRunStatus(manifest, "cancelled", "Plan approval was cancelled.");
|
|
619
|
+
return { manifest, tasks };
|
|
620
|
+
}
|
|
621
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
622
|
+
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
623
|
+
const completedBatch = batchTasks.map((task) => tasks.find((item) => item.id === task.id) ?? task);
|
|
624
|
+
const batchArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
625
|
+
kind: "summary",
|
|
626
|
+
relativePath: `batches/${batchTasks.map((task) => task.id).join("+")}.md`,
|
|
627
|
+
producer: "team-runner",
|
|
628
|
+
content: aggregateTaskOutputs(completedBatch, manifest),
|
|
629
|
+
});
|
|
630
|
+
const groupDelivery = deliverGroupJoin({ manifest, mode: resolveGroupJoinMode(input.runtimeConfig), batch: batchTasks, allTasks: tasks });
|
|
631
|
+
manifest = { ...manifest, artifacts: mergeArtifacts([...manifest.artifacts, batchArtifact, ...(groupDelivery?.artifact ? [groupDelivery.artifact] : [])]) };
|
|
632
|
+
manifest = writeProgress(manifest, tasks, "team-runner", input.executeWorkers, input.runtimeConfig);
|
|
633
|
+
await saveRunManifestAsync(manifest);
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
const failed = tasks.find((task) => task.status === "failed");
|
|
637
|
+
const waiting = tasks.find((task) => task.status === "waiting");
|
|
638
|
+
const running = tasks.find((task) => task.status === "running");
|
|
639
|
+
manifest = applyPolicy(manifest, tasks, input.limits);
|
|
640
|
+
const effectiveness = evaluateRunEffectiveness({ manifest, tasks, executeWorkers: input.executeWorkers, runtimeConfig: input.runtimeConfig });
|
|
641
|
+
const effectivenessDecision = effectivenessPolicyDecision(effectiveness);
|
|
642
|
+
if (effectivenessDecision) {
|
|
643
|
+
manifest = { ...manifest, policyDecisions: [...(manifest.policyDecisions ?? []), effectivenessDecision], updatedAt: new Date().toISOString() };
|
|
644
|
+
appendEvent(manifest.eventsPath, { type: "run.effectiveness", runId: manifest.runId, message: effectivenessDecision.message, data: { effectiveness, policyDecision: effectivenessDecision } });
|
|
645
|
+
}
|
|
646
|
+
const blockingDecision = manifest.policyDecisions?.find((item) => item.action === "block" || item.action === "escalate");
|
|
647
|
+
if (failed) {
|
|
648
|
+
manifest = updateRunStatus(manifest, "failed", `Failed at task '${failed.id}'.`);
|
|
649
|
+
} else if (waiting) {
|
|
650
|
+
manifest = updateRunStatus(manifest, "blocked", `Waiting for response to task '${waiting.id}'.`);
|
|
651
|
+
} else if (running) {
|
|
652
|
+
manifest = updateRunStatus(manifest, "blocked", `Task '${running.id}' is still running.`);
|
|
653
|
+
} else if (effectiveness.severity === "failed") {
|
|
654
|
+
manifest = updateRunStatus(manifest, "failed", effectivenessDecision?.message ?? "Run effectiveness guard failed.");
|
|
655
|
+
} else if (effectiveness.severity === "blocked") {
|
|
656
|
+
manifest = updateRunStatus(manifest, "blocked", effectivenessDecision?.message ?? "Run effectiveness guard blocked completion.");
|
|
657
|
+
} else if (blockingDecision) {
|
|
658
|
+
manifest = updateRunStatus(manifest, "blocked", blockingDecision.message);
|
|
659
|
+
} else {
|
|
660
|
+
manifest = updateRunStatus(manifest, "completed", input.executeWorkers ? "Team workflow completed." : "Team workflow scaffold completed without launching child workers.");
|
|
661
|
+
}
|
|
662
|
+
manifest = writeProgress(manifest, tasks, "team-runner", input.executeWorkers, input.runtimeConfig);
|
|
663
|
+
await saveRunManifestAsync(manifest);
|
|
664
|
+
const usage = aggregateUsage(tasks);
|
|
665
|
+
const summaryArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
666
|
+
kind: "summary",
|
|
667
|
+
relativePath: "summary.md",
|
|
668
|
+
producer: "team-runner",
|
|
669
|
+
content: [
|
|
670
|
+
`# pi-crew run ${manifest.runId}`,
|
|
671
|
+
"",
|
|
672
|
+
`Status: ${manifest.status}`,
|
|
673
|
+
`Team: ${manifest.team}`,
|
|
674
|
+
`Workflow: ${manifest.workflow ?? "(none)"}`,
|
|
675
|
+
`Goal: ${manifest.goal}`,
|
|
676
|
+
`Usage: ${formatUsage(usage)}`,
|
|
677
|
+
"",
|
|
678
|
+
"## Tasks",
|
|
679
|
+
...tasks.map(formatTaskProgress),
|
|
680
|
+
"",
|
|
681
|
+
"## Effectiveness",
|
|
682
|
+
...runEffectivenessLines(manifest, tasks, input.executeWorkers, input.runtimeConfig),
|
|
683
|
+
"",
|
|
684
|
+
"## Policy decisions",
|
|
685
|
+
...(manifest.policyDecisions?.length ? summarizePolicyDecisions(manifest.policyDecisions) : ["- (none)"]),
|
|
686
|
+
"",
|
|
687
|
+
].join("\n"),
|
|
688
|
+
});
|
|
689
|
+
manifest = { ...manifest, updatedAt: new Date().toISOString(), artifacts: [...manifest.artifacts, summaryArtifact] };
|
|
690
|
+
await saveRunManifestAsync(manifest);
|
|
691
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
692
|
+
return { manifest, tasks };
|
|
693
|
+
}
|