pi-crew 0.9.53 → 0.9.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/package.json +6 -3
- package/scripts/clean-strip-types.mjs +60 -0
- package/src/adapters/claude-adapter.ts +23 -0
- package/src/adapters/codex-adapter.ts +21 -0
- package/src/adapters/cursor-adapter.ts +17 -0
- package/src/adapters/export-util.ts +143 -0
- package/src/adapters/index.ts +15 -0
- package/src/adapters/registry.ts +18 -0
- package/src/adapters/types.ts +23 -0
- package/src/agents/agent-config.ts +194 -0
- package/src/agents/agent-search.ts +98 -0
- package/src/agents/agent-serializer.ts +38 -0
- package/src/agents/discover-agents.ts +623 -0
- package/src/benchmark/benchmark-runner.ts +313 -0
- package/src/benchmark/feedback-loop.ts +73 -0
- package/src/config/config.ts +1270 -0
- package/src/config/defaults.ts +193 -0
- package/src/config/drift-detector.ts +290 -0
- package/src/config/markers.ts +330 -0
- package/src/config/resilient-parser.ts +117 -0
- package/src/config/role-tools.ts +118 -0
- package/src/config/suggestions.ts +75 -0
- package/src/config/types.ts +280 -0
- package/src/errors.ts +194 -0
- package/src/extension/action-suggestions.ts +117 -0
- package/src/extension/async-notifier.ts +179 -0
- package/src/extension/autonomous-policy.ts +209 -0
- package/src/extension/command-completions.ts +120 -0
- package/src/extension/context-status-injection.ts +185 -0
- package/src/extension/crew-autocomplete.ts +131 -0
- package/src/extension/crew-cleanup.ts +168 -0
- package/src/extension/crew-input-router.ts +93 -0
- package/src/extension/crew-shortcuts.ts +87 -0
- package/src/extension/crew-vibes/cat-frames.ts +18 -0
- package/src/extension/crew-vibes/config.ts +195 -0
- package/src/extension/crew-vibes/figures.ts +101 -0
- package/src/extension/crew-vibes/font-detect.ts +71 -0
- package/src/extension/crew-vibes/footer.ts +292 -0
- package/src/extension/crew-vibes/index.ts +432 -0
- package/src/extension/crew-vibes/provider-usage.ts +396 -0
- package/src/extension/crew-vibes/render.ts +206 -0
- package/src/extension/crew-vibes/speed.ts +286 -0
- package/src/extension/cross-extension-rpc.ts +350 -0
- package/src/extension/help.ts +62 -0
- package/src/extension/import-index.ts +74 -0
- package/src/extension/knowledge-injection.ts +402 -0
- package/src/extension/management.ts +571 -0
- package/src/extension/message-renderers.ts +108 -0
- package/src/extension/notification-router.ts +152 -0
- package/src/extension/notification-sink.ts +54 -0
- package/src/extension/pi-api.ts +56 -0
- package/src/extension/plan-orchestrate.ts +302 -0
- package/src/extension/project-init.ts +170 -0
- package/src/extension/register.ts +113 -0
- package/src/extension/registration/artifact-cleanup.ts +20 -0
- package/src/extension/registration/command-registration.ts +58 -0
- package/src/extension/registration/command-utils.ts +58 -0
- package/src/extension/registration/commands.ts +1228 -0
- package/src/extension/registration/compaction-guard.ts +333 -0
- package/src/extension/registration/context-builder.ts +135 -0
- package/src/extension/registration/crash-recovery-cache.ts +54 -0
- package/src/extension/registration/foreground-run-controller.ts +238 -0
- package/src/extension/registration/hook-registration.ts +119 -0
- package/src/extension/registration/lazy-configurers.ts +124 -0
- package/src/extension/registration/lifecycle-handlers.ts +923 -0
- package/src/extension/registration/lifecycle.ts +259 -0
- package/src/extension/registration/observability.ts +322 -0
- package/src/extension/registration/registration-types.ts +158 -0
- package/src/extension/registration/runtime-cleanup.ts +230 -0
- package/src/extension/registration/subagent-helpers.ts +125 -0
- package/src/extension/registration/subagent-manager-setup.ts +330 -0
- package/src/extension/registration/subagent-tools.ts +456 -0
- package/src/extension/registration/team-tool.ts +225 -0
- package/src/extension/registration/tool-registration.ts +50 -0
- package/src/extension/registration/ui.ts +172 -0
- package/src/extension/registration/viewers.ts +121 -0
- package/src/extension/registration/wire-cross-extension.ts +37 -0
- package/src/extension/result-watcher.ts +139 -0
- package/src/extension/rpc-hmac.ts +256 -0
- package/src/extension/run-bundle-schema.ts +104 -0
- package/src/extension/run-export.ts +97 -0
- package/src/extension/run-import.ts +168 -0
- package/src/extension/run-index.ts +131 -0
- package/src/extension/run-maintenance.ts +190 -0
- package/src/extension/session-summary.ts +18 -0
- package/src/extension/team-manager-command.ts +153 -0
- package/src/extension/team-onboard.ts +180 -0
- package/src/extension/team-recommendation.ts +288 -0
- package/src/extension/team-tool/anchor.ts +172 -0
- package/src/extension/team-tool/api.ts +1244 -0
- package/src/extension/team-tool/auto-summarize.ts +142 -0
- package/src/extension/team-tool/cache-control.ts +19 -0
- package/src/extension/team-tool/cancel.ts +337 -0
- package/src/extension/team-tool/chain-dispatch.ts +95 -0
- package/src/extension/team-tool/chain-executor.ts +379 -0
- package/src/extension/team-tool/config-patch.ts +50 -0
- package/src/extension/team-tool/context.ts +149 -0
- package/src/extension/team-tool/destructive-gate.ts +52 -0
- package/src/extension/team-tool/dispatch/automate.ts +80 -0
- package/src/extension/team-tool/dispatch/control.ts +42 -0
- package/src/extension/team-tool/dispatch/index.ts +95 -0
- package/src/extension/team-tool/dispatch/manage.ts +161 -0
- package/src/extension/team-tool/dispatch/run.ts +53 -0
- package/src/extension/team-tool/dispatch/status.ts +187 -0
- package/src/extension/team-tool/doctor.ts +431 -0
- package/src/extension/team-tool/explain.ts +282 -0
- package/src/extension/team-tool/failure-patterns.ts +118 -0
- package/src/extension/team-tool/goal-wrap.ts +330 -0
- package/src/extension/team-tool/goal.ts +556 -0
- package/src/extension/team-tool/handle-schedule.ts +332 -0
- package/src/extension/team-tool/handle-settings.ts +520 -0
- package/src/extension/team-tool/health-monitor.ts +481 -0
- package/src/extension/team-tool/inspect.ts +88 -0
- package/src/extension/team-tool/intent-policy.ts +45 -0
- package/src/extension/team-tool/lifecycle-actions.ts +615 -0
- package/src/extension/team-tool/orchestrate.ts +98 -0
- package/src/extension/team-tool/parallel-dispatch.ts +183 -0
- package/src/extension/team-tool/plan.ts +50 -0
- package/src/extension/team-tool/respond.ts +150 -0
- package/src/extension/team-tool/run-deadline.ts +66 -0
- package/src/extension/team-tool/run-not-found.ts +49 -0
- package/src/extension/team-tool/run.ts +1051 -0
- package/src/extension/team-tool/status.ts +260 -0
- package/src/extension/team-tool/workflow-manage.ts +261 -0
- package/src/extension/team-tool-types.ts +26 -0
- package/src/extension/team-tool.ts +750 -0
- package/src/extension/tool-result.ts +16 -0
- package/src/extension/validate-resources.ts +108 -0
- package/src/hooks/registry.ts +200 -0
- package/src/hooks/types.ts +69 -0
- package/src/i18n.ts +212 -0
- package/src/observability/correlation.ts +50 -0
- package/src/observability/event-bus.ts +86 -0
- package/src/observability/event-to-metric.ts +170 -0
- package/src/observability/exporters/adapter.ts +27 -0
- package/src/observability/exporters/otlp-exporter.ts +356 -0
- package/src/observability/exporters/prometheus-exporter.ts +54 -0
- package/src/observability/metric-registry.ts +100 -0
- package/src/observability/metric-retention.ts +64 -0
- package/src/observability/metric-sink.ts +99 -0
- package/src/observability/metrics-primitives.ts +219 -0
- package/src/plugins/plugin-define.ts +6 -0
- package/src/plugins/plugin-registry.ts +32 -0
- package/src/plugins/plugins/index.ts +3 -0
- package/src/plugins/plugins/nextjs.ts +19 -0
- package/src/plugins/plugins/vite.ts +10 -0
- package/src/plugins/plugins/vitest.ts +9 -0
- package/src/prompt/prompt-runtime.ts +380 -0
- package/src/runtime/adaptive-plan.ts +563 -0
- package/src/runtime/agent-control.ts +215 -0
- package/src/runtime/agent-memory.ts +81 -0
- package/src/runtime/agent-observability.ts +122 -0
- package/src/runtime/anchor-manager.ts +475 -0
- package/src/runtime/async-marker.ts +32 -0
- package/src/runtime/async-runner.ts +381 -0
- package/src/runtime/attention-events.ts +28 -0
- package/src/runtime/auto-summarize.ts +344 -0
- package/src/runtime/background-runner.ts +840 -0
- package/src/runtime/batch-barrier.ts +147 -0
- package/src/runtime/broker-issuer.ts +39 -0
- package/src/runtime/cancellation-token.ts +99 -0
- package/src/runtime/cancellation.ts +99 -0
- package/src/runtime/capability-inventory.ts +137 -0
- package/src/runtime/chain-parser.ts +237 -0
- package/src/runtime/chain-runner.ts +606 -0
- package/src/runtime/checkpoint.ts +291 -0
- package/src/runtime/child-pi-constants.ts +42 -0
- package/src/runtime/child-pi-kill.ts +180 -0
- package/src/runtime/child-pi-pool.ts +68 -0
- package/src/runtime/child-pi-spawn.ts +287 -0
- package/src/runtime/child-pi-steering.ts +128 -0
- package/src/runtime/child-pi-streams.ts +296 -0
- package/src/runtime/child-pi-transcript.ts +169 -0
- package/src/runtime/child-pi.ts +1105 -0
- package/src/runtime/coalesce-tasks.ts +268 -0
- package/src/runtime/code-summary.ts +292 -0
- package/src/runtime/command-trace.ts +105 -0
- package/src/runtime/compact-pipeline.ts +56 -0
- package/src/runtime/compact-stages/ansi-strip-stage.ts +25 -0
- package/src/runtime/compact-stages/blank-collapse-stage.ts +31 -0
- package/src/runtime/compact-stages/deduplicate-stage.ts +34 -0
- package/src/runtime/compact-stages/head-snap-stage.ts +57 -0
- package/src/runtime/compact-stages/index.ts +23 -0
- package/src/runtime/compact-stages/tail-capture-stage.ts +77 -0
- package/src/runtime/compact-stages/truncation-stage.ts +74 -0
- package/src/runtime/compaction-summary.ts +278 -0
- package/src/runtime/completion-guard.ts +207 -0
- package/src/runtime/concurrency.ts +58 -0
- package/src/runtime/crash-classification.ts +229 -0
- package/src/runtime/crash-recovery.ts +594 -0
- package/src/runtime/crew-agent-records.ts +595 -0
- package/src/runtime/crew-agent-runtime.ts +62 -0
- package/src/runtime/crew-broker-child.ts +88 -0
- package/src/runtime/crew-broker-client.ts +673 -0
- package/src/runtime/crew-broker-tokens.ts +159 -0
- package/src/runtime/crew-broker.ts +1304 -0
- package/src/runtime/crew-hooks.ts +227 -0
- package/src/runtime/cross-extension-rpc.ts +143 -0
- package/src/runtime/custom-tools/irc-tool.ts +282 -0
- package/src/runtime/custom-tools/submit-result-tool.ts +102 -0
- package/src/runtime/deadletter.ts +47 -0
- package/src/runtime/delivery-coordinator.ts +211 -0
- package/src/runtime/delta-conflict.ts +348 -0
- package/src/runtime/deterministic-ast.ts +161 -0
- package/src/runtime/diagnostic-export.ts +171 -0
- package/src/runtime/direct-run.ts +45 -0
- package/src/runtime/dwf-state-store.ts +102 -0
- package/src/runtime/dynamic-workflow-context.ts +1013 -0
- package/src/runtime/dynamic-workflow-runner.ts +325 -0
- package/src/runtime/effectiveness.ts +90 -0
- package/src/runtime/errors/crew-errors.ts +162 -0
- package/src/runtime/event-stream-bridge.ts +98 -0
- package/src/runtime/foreground-control.ts +193 -0
- package/src/runtime/foreground-watchdog.ts +132 -0
- package/src/runtime/global-worker-cap.ts +96 -0
- package/src/runtime/goal-achievement.ts +148 -0
- package/src/runtime/goal-evaluator.ts +365 -0
- package/src/runtime/goal-loop-runner.ts +825 -0
- package/src/runtime/goal-state-store.ts +219 -0
- package/src/runtime/green-contract.ts +55 -0
- package/src/runtime/group-join.ts +267 -0
- package/src/runtime/handoff-manager.ts +598 -0
- package/src/runtime/heartbeat-gradient.ts +36 -0
- package/src/runtime/heartbeat-watcher.ts +239 -0
- package/src/runtime/hidden-handoff.ts +407 -0
- package/src/runtime/important-line-classifier.ts +140 -0
- package/src/runtime/intercom-bridge.ts +187 -0
- package/src/runtime/iteration-hooks.ts +305 -0
- package/src/runtime/live-agent-control.ts +136 -0
- package/src/runtime/live-agent-manager.ts +704 -0
- package/src/runtime/live-control-realtime.ts +56 -0
- package/src/runtime/live-extension-bridge.ts +148 -0
- package/src/runtime/live-irc.ts +97 -0
- package/src/runtime/live-session-health.ts +108 -0
- package/src/runtime/live-session-runtime.ts +1146 -0
- package/src/runtime/loop-gates.ts +128 -0
- package/src/runtime/manifest-cache.ts +375 -0
- package/src/runtime/mcp-proxy.ts +105 -0
- package/src/runtime/metric-parser.ts +36 -0
- package/src/runtime/model-fallback.ts +409 -0
- package/src/runtime/model-resolver.ts +126 -0
- package/src/runtime/model-scope.ts +158 -0
- package/src/runtime/orphan-worker-registry.ts +465 -0
- package/src/runtime/output-validator.ts +202 -0
- package/src/runtime/overflow-recovery.ts +206 -0
- package/src/runtime/parallel-research.ts +68 -0
- package/src/runtime/parallel-utils.ts +160 -0
- package/src/runtime/parent-guard.ts +134 -0
- package/src/runtime/path-overlap.ts +150 -0
- package/src/runtime/peer-dep.ts +292 -0
- package/src/runtime/per-write-validator.ts +181 -0
- package/src/runtime/phase-progress.ts +217 -0
- package/src/runtime/phase-tracker.ts +385 -0
- package/src/runtime/pi-args.ts +655 -0
- package/src/runtime/pi-json-output.ts +170 -0
- package/src/runtime/pi-spawn.ts +273 -0
- package/src/runtime/pipeline-runner.ts +523 -0
- package/src/runtime/plan-templates.ts +201 -0
- package/src/runtime/policy-engine.ts +115 -0
- package/src/runtime/post-checks.ts +142 -0
- package/src/runtime/post-exit-stdio-guard.ts +109 -0
- package/src/runtime/process-lifecycle.ts +491 -0
- package/src/runtime/process-status.ts +154 -0
- package/src/runtime/progress-event-coalescer.ts +44 -0
- package/src/runtime/progress-tracker.ts +124 -0
- package/src/runtime/prose-compressor.ts +162 -0
- package/src/runtime/recovery-recipes.ts +203 -0
- package/src/runtime/replace.ts +570 -0
- package/src/runtime/resilient-edit.ts +153 -0
- package/src/runtime/result-extractor.ts +217 -0
- package/src/runtime/retry-executor.ts +109 -0
- package/src/runtime/retry-runner.ts +336 -0
- package/src/runtime/role-permission.ts +59 -0
- package/src/runtime/run-coalesced-task-group.ts +308 -0
- package/src/runtime/run-drift.ts +219 -0
- package/src/runtime/run-tracker.ts +116 -0
- package/src/runtime/run-worker.ts +78 -0
- package/src/runtime/runtime-policy.ts +29 -0
- package/src/runtime/runtime-resolver.ts +172 -0
- package/src/runtime/runtime-warmup.ts +160 -0
- package/src/runtime/scheduler.ts +353 -0
- package/src/runtime/semaphore.ts +141 -0
- package/src/runtime/sensitive-paths.ts +93 -0
- package/src/runtime/session-resources.ts +25 -0
- package/src/runtime/session-snapshot.ts +59 -0
- package/src/runtime/session-usage.ts +79 -0
- package/src/runtime/settings-store.ts +155 -0
- package/src/runtime/sidechain-output.ts +35 -0
- package/src/runtime/single-agent-compose.ts +84 -0
- package/src/runtime/skill-effectiveness.ts +524 -0
- package/src/runtime/skill-instructions.ts +394 -0
- package/src/runtime/stale-reconciler.ts +680 -0
- package/src/runtime/stream-preview.ts +184 -0
- package/src/runtime/streaming-output.ts +50 -0
- package/src/runtime/subagent-manager.ts +561 -0
- package/src/runtime/subprocess-tool-registry.ts +70 -0
- package/src/runtime/supervisor-contact.ts +64 -0
- package/src/runtime/task-display.ts +52 -0
- package/src/runtime/task-graph-scheduler.ts +227 -0
- package/src/runtime/task-graph.ts +290 -0
- package/src/runtime/task-health.ts +83 -0
- package/src/runtime/task-id.ts +161 -0
- package/src/runtime/task-output-context.ts +602 -0
- package/src/runtime/task-packet.ts +268 -0
- package/src/runtime/task-quality.ts +199 -0
- package/src/runtime/task-runner/capabilities.ts +78 -0
- package/src/runtime/task-runner/child-executor.ts +790 -0
- package/src/runtime/task-runner/context-retrieval.ts +163 -0
- package/src/runtime/task-runner/live-executor.ts +227 -0
- package/src/runtime/task-runner/output-splitter.ts +152 -0
- package/src/runtime/task-runner/post-execution.ts +480 -0
- package/src/runtime/task-runner/pre-execution.ts +365 -0
- package/src/runtime/task-runner/progress.ts +157 -0
- package/src/runtime/task-runner/prompt-builder.ts +278 -0
- package/src/runtime/task-runner/prompt-pipeline.ts +88 -0
- package/src/runtime/task-runner/result-utils.ts +16 -0
- package/src/runtime/task-runner/retrieval-orchestrator.ts +310 -0
- package/src/runtime/task-runner/run-projection.ts +125 -0
- package/src/runtime/task-runner/scaffold-executor.ts +25 -0
- package/src/runtime/task-runner/state-helpers.ts +176 -0
- package/src/runtime/task-runner/tail-read.ts +34 -0
- package/src/runtime/task-runner.ts +216 -0
- package/src/runtime/team-runner-artifacts.ts +13 -0
- package/src/runtime/team-runner.ts +2396 -0
- package/src/runtime/tool-output-pruner.ts +333 -0
- package/src/runtime/tool-progress.ts +278 -0
- package/src/runtime/usage-tracker.ts +73 -0
- package/src/runtime/verification-gates.ts +579 -0
- package/src/runtime/verification-integrity.ts +108 -0
- package/src/runtime/verification-worktree.ts +186 -0
- package/src/runtime/worker-heartbeat.ts +25 -0
- package/src/runtime/worker-startup.ts +83 -0
- package/src/runtime/workflow-state.ts +195 -0
- package/src/runtime/workspace-lock.ts +443 -0
- package/src/runtime/workspace-tree.ts +312 -0
- package/src/runtime/yield-handler.ts +224 -0
- package/src/runtime/zombie-scanner.ts +298 -0
- package/src/schema/config-schema.ts +327 -0
- package/src/schema/team-tool-schema.ts +492 -0
- package/src/schema/validation-types.ts +159 -0
- package/src/skills/discover-skills.ts +203 -0
- package/src/skills/skill-templates.ts +456 -0
- package/src/skills/validate.ts +285 -0
- package/src/state/active-run-registry.ts +439 -0
- package/src/state/artifact-store.ts +176 -0
- package/src/state/atomic-write.ts +970 -0
- package/src/state/blob-store.ts +308 -0
- package/src/state/contracts.ts +160 -0
- package/src/state/crew-init.ts +269 -0
- package/src/state/decision-ledger.ts +372 -0
- package/src/state/event-log-rotation.ts +399 -0
- package/src/state/event-log.ts +1465 -0
- package/src/state/event-reconstructor.ts +229 -0
- package/src/state/gitignore-manager.ts +46 -0
- package/src/state/health-store.ts +79 -0
- package/src/state/hook-instinct-bridge.ts +94 -0
- package/src/state/hook-integrations.ts +51 -0
- package/src/state/instinct-store.ts +275 -0
- package/src/state/jsonl-writer.ts +107 -0
- package/src/state/locks.ts +562 -0
- package/src/state/mailbox.ts +916 -0
- package/src/state/observation-store.ts +176 -0
- package/src/state/run-cache.ts +193 -0
- package/src/state/run-graph.ts +183 -0
- package/src/state/run-metrics.ts +165 -0
- package/src/state/schedule.ts +166 -0
- package/src/state/session-state-map.ts +51 -0
- package/src/state/state-store.ts +973 -0
- package/src/state/task-claims.ts +61 -0
- package/src/state/tiered-eval.ts +480 -0
- package/src/state/types-eval.ts +58 -0
- package/src/state/types.ts +447 -0
- package/src/state/usage.ts +138 -0
- package/src/state/worker-atomic-writer.ts +198 -0
- package/src/subagents/async-entry.ts +1 -0
- package/src/subagents/index.ts +3 -0
- package/src/subagents/live/control.ts +1 -0
- package/src/subagents/live/manager.ts +1 -0
- package/src/subagents/live/realtime.ts +1 -0
- package/src/subagents/live/session-runtime.ts +1 -0
- package/src/subagents/manager.ts +1 -0
- package/src/subagents/spawn.ts +1 -0
- package/src/teams/discover-teams.ts +187 -0
- package/src/teams/team-config.ts +27 -0
- package/src/teams/team-serializer.ts +38 -0
- package/src/tools/safe-bash-extension.ts +54 -0
- package/src/tools/safe-bash.ts +505 -0
- package/src/types/diff.d.ts +18 -0
- package/src/types/new-api-types.ts +35 -0
- package/src/ui/agent-management-overlay.ts +160 -0
- package/src/ui/card-colors.ts +139 -0
- package/src/ui/crew-footer.ts +102 -0
- package/src/ui/crew-select-list.ts +112 -0
- package/src/ui/dashboard-panes/agents-pane.ts +164 -0
- package/src/ui/dashboard-panes/cancellation-pane.ts +66 -0
- package/src/ui/dashboard-panes/capability-pane.ts +77 -0
- package/src/ui/dashboard-panes/health-pane.ts +31 -0
- package/src/ui/dashboard-panes/mailbox-pane.ts +35 -0
- package/src/ui/dashboard-panes/metrics-pane.ts +37 -0
- package/src/ui/dashboard-panes/progress-pane.ts +38 -0
- package/src/ui/dashboard-panes/transcript-pane.ts +10 -0
- package/src/ui/deploy-bundled-themes.ts +71 -0
- package/src/ui/dwf-phase-display.ts +151 -0
- package/src/ui/dynamic-border.ts +35 -0
- package/src/ui/format-helpers.ts +31 -0
- package/src/ui/heartbeat-aggregator.ts +74 -0
- package/src/ui/key-utils.ts +42 -0
- package/src/ui/keybinding-map.ts +248 -0
- package/src/ui/layout-primitives.ts +106 -0
- package/src/ui/live-conversation-overlay.ts +183 -0
- package/src/ui/live-duration.ts +55 -0
- package/src/ui/live-run-sidebar.ts +293 -0
- package/src/ui/loaders.ts +6 -0
- package/src/ui/mascot.ts +449 -0
- package/src/ui/overlays/agent-picker-overlay.ts +66 -0
- package/src/ui/overlays/confirm-overlay.ts +60 -0
- package/src/ui/overlays/help-overlay.ts +177 -0
- package/src/ui/overlays/mailbox-compose-overlay.ts +181 -0
- package/src/ui/overlays/mailbox-compose-preview.ts +78 -0
- package/src/ui/overlays/mailbox-detail-overlay.ts +155 -0
- package/src/ui/pi-ui-compat.ts +68 -0
- package/src/ui/powerbar-publisher.ts +459 -0
- package/src/ui/render-coalescer.ts +101 -0
- package/src/ui/render-diff.ts +160 -0
- package/src/ui/render-scheduler.ts +278 -0
- package/src/ui/run-action-dispatcher.ts +182 -0
- package/src/ui/run-dashboard.ts +910 -0
- package/src/ui/run-event-bus.ts +381 -0
- package/src/ui/run-snapshot-cache.ts +1057 -0
- package/src/ui/settings-overlay.ts +1067 -0
- package/src/ui/shared-overlay-scheduler.ts +96 -0
- package/src/ui/snapshot-types.ts +90 -0
- package/src/ui/spinner.ts +17 -0
- package/src/ui/status-colors.ts +143 -0
- package/src/ui/syntax-highlight.ts +136 -0
- package/src/ui/terminal-status.ts +269 -0
- package/src/ui/theme-adapter.ts +271 -0
- package/src/ui/theme-discovery.ts +199 -0
- package/src/ui/tool-progress-formatter.ts +93 -0
- package/src/ui/tool-renderers/brief-mode.ts +225 -0
- package/src/ui/tool-renderers/index.ts +726 -0
- package/src/ui/transcript-cache.ts +128 -0
- package/src/ui/transcript-entries.ts +256 -0
- package/src/ui/transcript-viewer.ts +424 -0
- package/src/ui/widget/index.ts +418 -0
- package/src/ui/widget/widget-formatters.ts +182 -0
- package/src/ui/widget/widget-model.ts +110 -0
- package/src/ui/widget/widget-renderer.ts +182 -0
- package/src/ui/widget/widget-types.ts +36 -0
- package/src/utils/bm25-search.ts +235 -0
- package/src/utils/completion-dedupe.ts +63 -0
- package/src/utils/conflict-detect.ts +668 -0
- package/src/utils/env-allowlist.ts +30 -0
- package/src/utils/env-filter.ts +155 -0
- package/src/utils/file-coalescer.ts +98 -0
- package/src/utils/fingerprint.ts +180 -0
- package/src/utils/frontmatter.ts +70 -0
- package/src/utils/fs-watch.ts +37 -0
- package/src/utils/gh-protocol.ts +556 -0
- package/src/utils/git.ts +260 -0
- package/src/utils/guards.ts +107 -0
- package/src/utils/ids.ts +24 -0
- package/src/utils/incremental-reader.ts +220 -0
- package/src/utils/internal-error.ts +10 -0
- package/src/utils/names.ts +36 -0
- package/src/utils/ndjson.ts +115 -0
- package/src/utils/paths.ts +227 -0
- package/src/utils/project-detector.ts +160 -0
- package/src/utils/redaction.ts +370 -0
- package/src/utils/resolve-shell.ts +36 -0
- package/src/utils/run-watcher-registry.ts +152 -0
- package/src/utils/safe-paths.ts +393 -0
- package/src/utils/scan-cache.ts +144 -0
- package/src/utils/session-utils.ts +110 -0
- package/src/utils/sleep.ts +54 -0
- package/src/utils/socket-path.ts +164 -0
- package/src/utils/sse-parser.ts +131 -0
- package/src/utils/task-name-generator.ts +337 -0
- package/src/utils/timings.ts +33 -0
- package/src/utils/token-counter.ts +200 -0
- package/src/utils/visual.ts +207 -0
- package/src/workflows/cost-estimator.ts +34 -0
- package/src/workflows/discover-workflows.ts +308 -0
- package/src/workflows/intermediate-store.ts +166 -0
- package/src/workflows/preflight-validator.ts +168 -0
- package/src/workflows/topology-analyzer.ts +203 -0
- package/src/workflows/validate-workflow.ts +50 -0
- package/src/workflows/workflow-config.ts +75 -0
- package/src/workflows/workflow-serializer.ts +32 -0
- package/src/worktree/branch-freshness.ts +112 -0
- package/src/worktree/cleanup.ts +341 -0
- package/src/worktree/worktree-manager.ts +1204 -0
|
@@ -0,0 +1,2396 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import type { AgentConfig } from "../agents/agent-config.ts";
|
|
4
|
+
import type { CrewLimitsConfig, CrewReliabilityConfig, CrewRuntimeConfig } from "../config/config.ts";
|
|
5
|
+
import { CrewError, ErrorCode } from "../errors.ts";
|
|
6
|
+
import { appendHookEvent, executeHook } from "../hooks/registry.ts";
|
|
7
|
+
import { childCorrelation, withCorrelation } from "../observability/correlation.ts";
|
|
8
|
+
import type { MetricRegistry } from "../observability/metric-registry.ts";
|
|
9
|
+
import { PluginRegistry } from "../plugins/plugin-registry.ts";
|
|
10
|
+
import { NextJsPlugin, VitePlugin, VitestPlugin } from "../plugins/plugins/index.ts";
|
|
11
|
+
import { hashArtifactContent as hashContent, writeArtifact } from "../state/artifact-store.ts";
|
|
12
|
+
import { atomicWriteFile, flushPendingAtomicWrites } from "../state/atomic-write.ts";
|
|
13
|
+
import { appendEvent, appendEventAsync, appendEventBuffered, appendEventFireAndForget, flushEventLogBuffer } from "../state/event-log.ts";
|
|
14
|
+
import { HealthStore } from "../state/health-store.ts";
|
|
15
|
+
import { withRunLock } from "../state/locks.ts";
|
|
16
|
+
import { loadRunManifestById, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/state-store.ts";
|
|
17
|
+
import type { ArtifactDescriptor, PolicyDecision, TaskAttemptState, TeamRunManifest, TeamTaskState } from "../state/types.ts";
|
|
18
|
+
import { aggregateUsage, formatTokens, formatUsage } from "../state/usage.ts";
|
|
19
|
+
import type { TeamConfig } from "../teams/team-config.ts";
|
|
20
|
+
import { logInternalError } from "../utils/internal-error.ts";
|
|
21
|
+
import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
|
|
22
|
+
import { checkBranchFreshness } from "../worktree/branch-freshness.ts";
|
|
23
|
+
import { buildSyntheticTerminalEvidence, CrewCancellationError, cancellationReasonFromSignal } from "./cancellation.ts";
|
|
24
|
+
import { buildDispatchUnits, type DispatchUnit, planCoalescedGroups } from "./coalesce-tasks.ts";
|
|
25
|
+
import { type BatchConcurrencyDecision, resolveBatchConcurrency } from "./concurrency.ts";
|
|
26
|
+
import { readCrewAgents, saveCrewAgents } from "./crew-agent-records.ts";
|
|
27
|
+
import type { CrewRuntimeKind } from "./crew-agent-runtime.ts";
|
|
28
|
+
import { crewHooks } from "./crew-hooks.ts";
|
|
29
|
+
import { appendDeadletter } from "./deadletter.ts";
|
|
30
|
+
import { effectivenessPolicyDecision, evaluateRunEffectiveness, formatRunEffectivenessLines } from "./effectiveness.ts";
|
|
31
|
+
import { applyGoalAchievement, assessGoalAchievement } from "./goal-achievement.ts";
|
|
32
|
+
import { deliverGroupJoin, resolveGroupJoinMode } from "./group-join.ts";
|
|
33
|
+
import { terminateLiveAgentsForRun } from "./live-agent-manager.ts";
|
|
34
|
+
import { filterReadyByWriteOverlap } from "./path-overlap.ts";
|
|
35
|
+
import { evaluateCrewPolicy, summarizePolicyDecisions } from "./policy-engine.ts";
|
|
36
|
+
import { buildRecoveryLedger, shouldRerunFailedTask } from "./recovery-recipes.ts";
|
|
37
|
+
import { DEFAULT_RETRY_POLICY, executeWithRetry, type RetryPolicy } from "./retry-executor.ts";
|
|
38
|
+
import { permissionForRole } from "./role-permission.ts";
|
|
39
|
+
import { runCoalescedTaskGroup } from "./run-coalesced-task-group.ts";
|
|
40
|
+
import { registerRunPromise, rejectRunPromise, resolveRunPromise } from "./run-tracker.ts";
|
|
41
|
+
import { resolveTaskRuntimeKind } from "./runtime-policy.ts";
|
|
42
|
+
import type { CrewRuntimeCapabilities } from "./runtime-resolver.ts";
|
|
43
|
+
import { recordsForMaterializedTasks } from "./task-display.ts";
|
|
44
|
+
import { buildExecutionPlan as buildDagExecutionPlan, getReadyTasks as getDagReadyTasks, type TaskNode } from "./task-graph.ts";
|
|
45
|
+
import {
|
|
46
|
+
buildTaskGraphIndex,
|
|
47
|
+
refreshTaskGraphQueues,
|
|
48
|
+
type TaskGraphIndex,
|
|
49
|
+
type TaskGraphSchedulerSnapshot,
|
|
50
|
+
taskGraphSnapshot,
|
|
51
|
+
} from "./task-graph-scheduler.ts";
|
|
52
|
+
import { aggregateTaskOutputs } from "./task-output-context.ts";
|
|
53
|
+
import { clearStablePrefixCache, computeStablePrefixComponents } from "./task-runner/prompt-builder.ts";
|
|
54
|
+
import { runTeamTask, type SpawnBudget } from "./task-runner.ts";
|
|
55
|
+
import { mergeArtifacts } from "./team-runner-artifacts.ts";
|
|
56
|
+
import { clearTrackedTaskUsage } from "./usage-tracker.ts";
|
|
57
|
+
import {
|
|
58
|
+
createWorkflowStateMachine,
|
|
59
|
+
type PhaseGuardContext,
|
|
60
|
+
type PhaseState,
|
|
61
|
+
transitionPhase,
|
|
62
|
+
validatePhasePreconditions,
|
|
63
|
+
type WorkflowStateMachine,
|
|
64
|
+
} from "./workflow-state.ts";
|
|
65
|
+
|
|
66
|
+
// Built-in plugin registry for framework awareness.
|
|
67
|
+
// NOTE: This registry is registered here for future use. The integration
|
|
68
|
+
// point (reading package.json deps, computing active plugin context, and
|
|
69
|
+
// passing it into RunConfig / task-runner) is not yet implemented; see
|
|
70
|
+
// `getPluginContext` in src/plugins/plugin-context.ts (planned).
|
|
71
|
+
const builtInRegistry = new PluginRegistry();
|
|
72
|
+
builtInRegistry.register(NextJsPlugin);
|
|
73
|
+
builtInRegistry.register(VitestPlugin);
|
|
74
|
+
builtInRegistry.register(VitePlugin);
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Start a periodic heartbeat for the team-level run.
|
|
78
|
+
*
|
|
79
|
+
* The stale reconciler (src/runtime/stale-reconciler.ts) marks runs as failed
|
|
80
|
+
* if their heartbeat is older than `NO_PID_HEARTBEAT_STALE_MS` (5 minutes).
|
|
81
|
+
* Without this, long-running team runs (e.g. multi-phase workflows) get
|
|
82
|
+
* cancelled by the reconciler as "stale" even when they are actively
|
|
83
|
+
* executing. The team-runner has no periodic heartbeat today, so any
|
|
84
|
+
* team run lasting >5min is at risk.
|
|
85
|
+
*/
|
|
86
|
+
function startTeamRunHeartbeat(stateRoot: string, runId: string): () => void {
|
|
87
|
+
const heartbeatPath = path.join(stateRoot, "heartbeat.json");
|
|
88
|
+
const writeHeartbeat = (): void => {
|
|
89
|
+
try {
|
|
90
|
+
// lastTaskUpdateAt is written fresh on each tick so the heartbeat
|
|
91
|
+
// never carries a stale creation-time timestamp. Previously this
|
|
92
|
+
// captured manifest.updatedAt once at startup, making the value
|
|
93
|
+
// permanently stale throughout the run.
|
|
94
|
+
const now = new Date().toISOString();
|
|
95
|
+
atomicWriteFile(
|
|
96
|
+
heartbeatPath,
|
|
97
|
+
JSON.stringify({
|
|
98
|
+
pid: process.pid,
|
|
99
|
+
at: Date.now(),
|
|
100
|
+
runId,
|
|
101
|
+
kind: "team-runner",
|
|
102
|
+
lastTaskUpdateAt: now,
|
|
103
|
+
}),
|
|
104
|
+
{ mode: 0o600 },
|
|
105
|
+
);
|
|
106
|
+
} catch {
|
|
107
|
+
// best-effort
|
|
108
|
+
}
|
|
109
|
+
};
|
|
110
|
+
writeHeartbeat();
|
|
111
|
+
// NOTE: This interval is deliberately NOT unref'd. Unlike background-runner's
|
|
112
|
+
// heartbeat and interrupt guard (both unref'd), the team heartbeat must keep
|
|
113
|
+
// the event loop alive so the stale reconciler does not cancel long-running
|
|
114
|
+
// team runs (>5 min) as "stale" while they are actively executing.
|
|
115
|
+
// P13 (perf): tick every 60s instead of 30s. The stale-reconciler threshold
|
|
116
|
+
// is 5min (300_000ms in crash-recovery.ts), so a 60s heartbeat still leaves
|
|
117
|
+
// 5 ticks of slack before a run is misidentified as stale. Cuts the per-run
|
|
118
|
+
// heartbeat.syscall count in half (1 write/30s → 1 write/60s) with no
|
|
119
|
+
// behavioral change.
|
|
120
|
+
const interval = setInterval(writeHeartbeat, 60_000);
|
|
121
|
+
return () => clearInterval(interval);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export interface ExecuteTeamRunInput {
|
|
125
|
+
manifest: TeamRunManifest;
|
|
126
|
+
tasks: TeamTaskState[];
|
|
127
|
+
team: TeamConfig;
|
|
128
|
+
workflow: WorkflowConfig;
|
|
129
|
+
agents: AgentConfig[];
|
|
130
|
+
executeWorkers: boolean;
|
|
131
|
+
limits?: CrewLimitsConfig;
|
|
132
|
+
runtime?: CrewRuntimeCapabilities;
|
|
133
|
+
runtimeConfig?: CrewRuntimeConfig;
|
|
134
|
+
parentContext?: string;
|
|
135
|
+
parentModel?: unknown;
|
|
136
|
+
modelRegistry?: unknown;
|
|
137
|
+
modelOverride?: string;
|
|
138
|
+
signal?: AbortSignal;
|
|
139
|
+
reliability?: CrewReliabilityConfig;
|
|
140
|
+
metricRegistry?: MetricRegistry;
|
|
141
|
+
/** Skill override from the team tool. false disables skill injection for this run. */
|
|
142
|
+
skillOverride?: string[] | false;
|
|
143
|
+
/** Optional callback for JSON events from child Pi. Used for overflow recovery tracking. */
|
|
144
|
+
onJsonEvent?: (taskId: string, runId: string, event: unknown) => void;
|
|
145
|
+
/** Workspace where this run was initiated — used for session-scoped live-agent visibility. */
|
|
146
|
+
workspaceId: string;
|
|
147
|
+
/** Total token budget for the run. When set, enables per-task budget enforcement. */
|
|
148
|
+
budgetTotal?: number;
|
|
149
|
+
/** Budget warning threshold as a fraction (0-1). Default: 0.8 (80%). */
|
|
150
|
+
budgetWarning?: number;
|
|
151
|
+
/** Budget abort threshold as a fraction (0-1). Default: 0.95 (95%). */
|
|
152
|
+
budgetAbort?: number;
|
|
153
|
+
/** When true, skip budget enforcement entirely. */
|
|
154
|
+
budgetUnlimited?: boolean;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Result of a per-task budget check against cumulative run usage.
|
|
159
|
+
*/
|
|
160
|
+
export interface PerTaskBudgetCheckResult {
|
|
161
|
+
/** Whether the abort threshold was exceeded. */
|
|
162
|
+
abort: boolean;
|
|
163
|
+
/** Whether the warning threshold was exceeded. */
|
|
164
|
+
warning: boolean;
|
|
165
|
+
/** IDs of tasks that exceeded their fair share (>50% of remaining budget). */
|
|
166
|
+
fairShareViolators: string[];
|
|
167
|
+
/** Total tokens used so far. */
|
|
168
|
+
totalUsed: number;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Check cumulative token usage against per-task budget thresholds.
|
|
173
|
+
* Returns a structured result — callers decide how to act (warn vs abort).
|
|
174
|
+
*
|
|
175
|
+
* Exported for unit testing.
|
|
176
|
+
*/
|
|
177
|
+
export function checkPerTaskBudget(
|
|
178
|
+
tasks: TeamTaskState[],
|
|
179
|
+
budgetTotal: number,
|
|
180
|
+
budgetWarning: number,
|
|
181
|
+
budgetAbort: number,
|
|
182
|
+
fairShareFraction = 0.5,
|
|
183
|
+
): PerTaskBudgetCheckResult {
|
|
184
|
+
const usage = aggregateUsage(tasks);
|
|
185
|
+
const totalUsed = (usage?.input ?? 0) + (usage?.output ?? 0) + (usage?.cacheWrite ?? 0);
|
|
186
|
+
const abort = totalUsed >= budgetAbort * budgetTotal;
|
|
187
|
+
const warning = !abort && totalUsed >= budgetWarning * budgetTotal;
|
|
188
|
+
// Fair share threshold based on TOTAL budget, not remaining budget.
|
|
189
|
+
// This ensures a task that consumed 60% of total budget is flagged even
|
|
190
|
+
// if only 40% remains (40% * 50% = 20% threshold would miss the 60% usage).
|
|
191
|
+
const fairShareThreshold = budgetTotal * fairShareFraction;
|
|
192
|
+
const fairShareViolators: string[] = [];
|
|
193
|
+
for (const task of tasks) {
|
|
194
|
+
if (!task.usage) continue;
|
|
195
|
+
const taskTotal = (task.usage.input ?? 0) + (task.usage.output ?? 0) + (task.usage.cacheWrite ?? 0);
|
|
196
|
+
// Only flag tasks that individually consumed a significant portion of the
|
|
197
|
+
// budget (>10% of total) AND exceeded the fair share threshold.
|
|
198
|
+
if (fairShareThreshold > 0 && taskTotal > fairShareThreshold && taskTotal > budgetTotal * 0.1) {
|
|
199
|
+
fairShareViolators.push(task.id);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
return { abort, warning, fairShareViolators, totalUsed };
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function findStep(workflow: WorkflowConfig, task: TeamTaskState): WorkflowStep {
|
|
206
|
+
const step = workflow.steps.find((candidate) => candidate.id === task.stepId);
|
|
207
|
+
if (!step)
|
|
208
|
+
throw new CrewError(ErrorCode.ResourceNotFound, `Workflow step '${task.stepId}' not found for task '${task.id}'.`).withContext(
|
|
209
|
+
`workflow step lookup (task=${task.id})`,
|
|
210
|
+
);
|
|
211
|
+
return step;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function findAgent(agents: AgentConfig[], task: TeamTaskState): AgentConfig {
|
|
215
|
+
const agent = agents.find((candidate) => candidate.name === task.agent);
|
|
216
|
+
if (!agent)
|
|
217
|
+
throw new CrewError(ErrorCode.ResourceNotFound, `Agent '${task.agent}' not found for task '${task.id}'.`).withContext(
|
|
218
|
+
`agent lookup (task=${task.id})`,
|
|
219
|
+
);
|
|
220
|
+
return agent;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function markBlocked(tasks: TeamTaskState[], reason: string): TeamTaskState[] {
|
|
224
|
+
return tasks.map((task) =>
|
|
225
|
+
task.status === "queued"
|
|
226
|
+
? {
|
|
227
|
+
...task,
|
|
228
|
+
status: "skipped",
|
|
229
|
+
error: reason,
|
|
230
|
+
finishedAt: new Date().toISOString(),
|
|
231
|
+
graph: task.graph ? { ...task.graph, queue: "blocked" } : undefined,
|
|
232
|
+
}
|
|
233
|
+
: task,
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function isNonTerminalTaskStatus(status: TeamTaskState["status"]): boolean {
|
|
238
|
+
return status === "queued" || status === "running" || status === "waiting";
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* CORE-6: Unified cancel/fail of non-terminal tasks. Replaces hand-rolled
|
|
243
|
+
* `.map()` + transform sites across this file.
|
|
244
|
+
*
|
|
245
|
+
* - Without `filter`: all non-terminal tasks (queued/running/waiting) are
|
|
246
|
+
* terminalised with the given status.
|
|
247
|
+
* - With `filter`: the filter is the sole gate — the non-terminal check is
|
|
248
|
+
* NOT applied automatically, matching per-task/per-id variants.
|
|
249
|
+
*
|
|
250
|
+
* `markBlocked` is intentionally NOT unified here (it sets status "skipped",
|
|
251
|
+
* not cancelled/failed, and only acts on "queued" tasks).
|
|
252
|
+
*/
|
|
253
|
+
function cancelNonTerminalTasks(
|
|
254
|
+
tasks: TeamTaskState[],
|
|
255
|
+
status: "cancelled" | "failed",
|
|
256
|
+
reason: string,
|
|
257
|
+
filter?: (task: TeamTaskState) => boolean,
|
|
258
|
+
): TeamTaskState[] {
|
|
259
|
+
const predicate = filter ?? ((task: TeamTaskState) => isNonTerminalTaskStatus(task.status));
|
|
260
|
+
return tasks.map((task) => (predicate(task) ? { ...task, status, finishedAt: new Date().toISOString(), error: reason } : task));
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Returns the finishedAt timestamp as a number, or Infinity for invalid/malformed dates.
|
|
265
|
+
* This makes comparison logic in shouldMergeTaskUpdate more readable by abstracting
|
|
266
|
+
* the NaN handling into a single well-named function.
|
|
267
|
+
*/
|
|
268
|
+
function safeFinishedAt(task: TeamTaskState): number {
|
|
269
|
+
if (!task.finishedAt) return -Infinity;
|
|
270
|
+
const ms = new Date(task.finishedAt).getTime();
|
|
271
|
+
return Number.isNaN(ms) ? Infinity : ms;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Returns true when the current task has a malformed finishedAt (NaN/Infinity)
|
|
276
|
+
* and the updated task has a valid finite finishedAt. Malformed finishedAt
|
|
277
|
+
* should be replaced rather than persisting corruption.
|
|
278
|
+
*/
|
|
279
|
+
function isMalformedFinishedAtReplacement(currentTime: number, updatedTime: number): boolean {
|
|
280
|
+
return !Number.isFinite(currentTime) && Number.isFinite(updatedTime);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
function shouldMergeTaskUpdate(current: TeamTaskState, updated: TeamTaskState): boolean {
|
|
284
|
+
// Parallel workers receive the same input snapshot. A later result may still
|
|
285
|
+
// contain stale queued/running copies of tasks that another worker already
|
|
286
|
+
// completed. Never let those stale snapshots regress durable task state.
|
|
287
|
+
if (current.status === "waiting" && updated.status === "running") return false;
|
|
288
|
+
// Block terminal→non-terminal transitions (e.g. completed→running).
|
|
289
|
+
// A task that has reached a terminal state must not be resurrected.
|
|
290
|
+
const currentIsTerminal = !isNonTerminalTaskStatus(current.status);
|
|
291
|
+
const updatedIsNonTerminal = isNonTerminalTaskStatus(updated.status);
|
|
292
|
+
if (currentIsTerminal && updatedIsNonTerminal) return false;
|
|
293
|
+
// Explicitly block completed↔needs_attention terminal-to-terminal transitions.
|
|
294
|
+
// Both are success terminal states used interchangeably; stale worker updates must
|
|
295
|
+
// not cause a completed task to appear as needs_attention or vice versa.
|
|
296
|
+
if (current.status === "completed" && updated.status === "needs_attention") return false;
|
|
297
|
+
if (current.status === "needs_attention" && updated.status === "completed") return false;
|
|
298
|
+
// Explicitly block failed→completed resurrection. Both statuses are terminal,
|
|
299
|
+
// but completed is the success terminal state and should not be reachable from
|
|
300
|
+
// failed via a stale merge. The check above only guards non-terminal→terminal.
|
|
301
|
+
if (current.status === "failed" && updated.status === "completed") return false;
|
|
302
|
+
// Mirror that guard for the other dangerous terminal→terminal flips (reverse
|
|
303
|
+
// audit CANCEL-3 + F3): a cancelled task must not be resurrected to completed,
|
|
304
|
+
// and a completed task must not be demoted to failed, by a stale worker merge.
|
|
305
|
+
// The task transition table (contracts.ts) only permits terminal→queued
|
|
306
|
+
// (retry); these flips are always stale results. A worker that completed after
|
|
307
|
+
// the task was cancelled, or a stale failed result arriving after completion,
|
|
308
|
+
// must not flip a settled terminal status.
|
|
309
|
+
if (current.status === "cancelled" && updated.status === "completed") return false;
|
|
310
|
+
if (current.status === "completed" && updated.status === "failed") return false;
|
|
311
|
+
// Guard: when current is "running" but has resultArtifact (another worker already
|
|
312
|
+
// completed it), a stale updated with status="running" and no resultArtifact
|
|
313
|
+
// must not overwrite the actual completed state.
|
|
314
|
+
if (current.status === updated.status && updated.status === "running" && current.resultArtifact && !updated.resultArtifact)
|
|
315
|
+
return false;
|
|
316
|
+
// Guard: when current is "completed" and has resultArtifact but updated is also
|
|
317
|
+
// "completed" without resultArtifact, block the stale update from overwriting
|
|
318
|
+
// a task that successfully produced output.
|
|
319
|
+
if (current.status === updated.status && current.status === "completed" && current.resultArtifact && !updated.resultArtifact)
|
|
320
|
+
return false;
|
|
321
|
+
// Prevent a stale completed task from overwriting a fresher one.
|
|
322
|
+
// Restructure to handle undefined current.finishedAt as a special case:
|
|
323
|
+
// - undefined current + valid updated: allow the update
|
|
324
|
+
// - valid current + undefined updated: block the update (don't lose completion time)
|
|
325
|
+
// - both undefined: finishedAt guard does not apply, fall through to heartbeat check
|
|
326
|
+
// - both valid: compare timestamps as before
|
|
327
|
+
if (current.finishedAt !== undefined && updated.finishedAt !== undefined) {
|
|
328
|
+
const currentTime = safeFinishedAt(current);
|
|
329
|
+
const updatedTime = safeFinishedAt(updated);
|
|
330
|
+
// Malformed finishedAt (NaN) is treated as Infinity — invalid state should be
|
|
331
|
+
// replaced rather than persisting corruption. Log warning for visibility.
|
|
332
|
+
if (!Number.isFinite(currentTime)) {
|
|
333
|
+
console.warn(`[team-runner] Task ${current.id} has malformed finishedAt: ${current.finishedAt}`);
|
|
334
|
+
}
|
|
335
|
+
if (isMalformedFinishedAtReplacement(currentTime, updatedTime)) {
|
|
336
|
+
return true;
|
|
337
|
+
}
|
|
338
|
+
if (updatedTime < currentTime) return false;
|
|
339
|
+
}
|
|
340
|
+
// Block if updated is trying to establish a terminal status without a finishedAt
|
|
341
|
+
// timestamp. Heartbeat-only updates (status='running', no finishedAt) are
|
|
342
|
+
// allowed if heartbeat has changed (checked separately in hasMeaningfulUpdate).
|
|
343
|
+
if (!updated.finishedAt && !isNonTerminalTaskStatus(updated.status)) return false;
|
|
344
|
+
// Explicitly enumerate all fields that constitute a meaningful update so that
|
|
345
|
+
// adding a new important field requires updating this list (rather than silently
|
|
346
|
+
// losing data if a field is forgotten in the boolean OR chain below).
|
|
347
|
+
const hasMeaningfulUpdate =
|
|
348
|
+
updated.status !== current.status ||
|
|
349
|
+
updated.finishedAt !== current.finishedAt ||
|
|
350
|
+
updated.startedAt !== current.startedAt ||
|
|
351
|
+
Boolean(updated.resultArtifact) !== Boolean(current.resultArtifact) ||
|
|
352
|
+
(Boolean(updated.resultArtifact) && updated.resultArtifact !== current.resultArtifact) ||
|
|
353
|
+
Boolean(updated.error) ||
|
|
354
|
+
Boolean(updated.modelAttempts?.length) ||
|
|
355
|
+
Boolean(updated.usage) ||
|
|
356
|
+
Boolean(updated.attempts?.length) ||
|
|
357
|
+
updated.heartbeat?.lastSeenAt !== current.heartbeat?.lastSeenAt ||
|
|
358
|
+
updated.jsonEvents !== current.jsonEvents ||
|
|
359
|
+
updated.agentProgress?.lastActivityAt !== current.agentProgress?.lastActivityAt;
|
|
360
|
+
return hasMeaningfulUpdate;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// H4 fix: rename to descriptive name. Kept __test__ as alias for backward
|
|
364
|
+
// compat test imports.
|
|
365
|
+
// FIX (perf P10): replace O(N×M) .find() + .map() inside nested loops with a
|
|
366
|
+
// single-pass Map-based merge. Build an index of `merged` once, then for each
|
|
367
|
+
// incoming updated task do O(1) lookup; the final pass reassembles `merged`
|
|
368
|
+
// preserving original order. For a 20-task run × 5-batch merger with
|
|
369
|
+
// ~10 updates per result, this reduces from O(50×20) = 1000 ops to O(120).
|
|
370
|
+
// Behavior is unchanged: skipped updates (shouldMergeTaskUpdate=false) still
|
|
371
|
+
// leave the existing task in place.
|
|
372
|
+
export function mergeTaskUpdatesPreservingTerminal(base: TeamTaskState[], results: Array<{ tasks: TeamTaskState[] }>): TeamTaskState[] {
|
|
373
|
+
// Index current merged state by id for O(1) lookup during the merge pass.
|
|
374
|
+
const indexById = new Map<string, TeamTaskState>();
|
|
375
|
+
for (const task of base) indexById.set(task.id, task);
|
|
376
|
+
|
|
377
|
+
let skipped = 0;
|
|
378
|
+
for (const result of results) {
|
|
379
|
+
for (const updated of result.tasks) {
|
|
380
|
+
const current = indexById.get(updated.id);
|
|
381
|
+
if (!current) continue;
|
|
382
|
+
if (!shouldMergeTaskUpdate(current, updated)) {
|
|
383
|
+
// Log skipped merges for visibility into rejected parallel updates.
|
|
384
|
+
// In distributed systems with parallel workers, rejected merges may
|
|
385
|
+
// indicate bugs (wrong status, timestamp corruption) if they accumulate.
|
|
386
|
+
console.debug("[team-runner] Skipping stale merge for task", updated.id, {
|
|
387
|
+
currentStatus: current.status,
|
|
388
|
+
updatedStatus: updated.status,
|
|
389
|
+
currentFinishedAt: current.finishedAt,
|
|
390
|
+
updatedFinishedAt: updated.finishedAt,
|
|
391
|
+
});
|
|
392
|
+
skipped += 1;
|
|
393
|
+
continue;
|
|
394
|
+
}
|
|
395
|
+
indexById.set(updated.id, updated);
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
// Reassemble in original `base` order so downstream snapshots stay stable.
|
|
399
|
+
const merged = base.map((task) => indexById.get(task.id) ?? task);
|
|
400
|
+
// `skipped` is intentional visibility — currently no caller reads it but
|
|
401
|
+
// we'd rather leave the count available for future instrumentation than
|
|
402
|
+
// remove the cumulative silent-rejection signal it provides.
|
|
403
|
+
void skipped;
|
|
404
|
+
return refreshTaskGraphQueues(merged);
|
|
405
|
+
}
|
|
406
|
+
/** @deprecated Use mergeTaskUpdatesPreservingTerminal. Kept for backward test import compat. */
|
|
407
|
+
export const __test__mergeTaskUpdates = mergeTaskUpdatesPreservingTerminal;
|
|
408
|
+
|
|
409
|
+
// 2.8: adaptive-plan parsing/repair/injection moved to src/runtime/adaptive-plan.ts.
|
|
410
|
+
// Re-export the test-only helpers so existing test imports still resolve.
|
|
411
|
+
export {
|
|
412
|
+
__test__parseAdaptivePlan,
|
|
413
|
+
__test__repairAdaptivePlan,
|
|
414
|
+
} from "./adaptive-plan.ts";
|
|
415
|
+
|
|
416
|
+
import { injectAdaptivePlanIfReady } from "./adaptive-plan.ts";
|
|
417
|
+
|
|
418
|
+
function formatTaskProgress(task: TeamTaskState): string {
|
|
419
|
+
return `- ${task.id}: ${task.status} (${task.role} -> ${task.agent})${task.taskPacket ? ` scope=${task.taskPacket.scope}` : ""}${task.verification ? ` green=${task.verification.observedGreenLevel}/${task.verification.requiredGreenLevel}` : ""}${task.error ? ` - ${task.error}` : ""}`;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
function runEffectivenessLines(
|
|
423
|
+
manifest: TeamRunManifest,
|
|
424
|
+
tasks: TeamTaskState[],
|
|
425
|
+
executeWorkers: boolean,
|
|
426
|
+
runtimeConfig?: CrewRuntimeConfig,
|
|
427
|
+
): string[] {
|
|
428
|
+
return formatRunEffectivenessLines(
|
|
429
|
+
evaluateRunEffectiveness({
|
|
430
|
+
manifest,
|
|
431
|
+
tasks,
|
|
432
|
+
executeWorkers,
|
|
433
|
+
runtimeConfig,
|
|
434
|
+
}),
|
|
435
|
+
);
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
// P6 (perf): Cache the last-rendered progress content so we can skip the
|
|
439
|
+
// artifact write + redaction + atomic write + size/hash read when nothing
|
|
440
|
+
// material changed (rare between batches, but happens between idle heartbeats).
|
|
441
|
+
// The dedup filter also moved from O(N²) findIndex inside .filter(...)
|
|
442
|
+
// (the previous implementation ran 2 redundant passes on every batch) to
|
|
443
|
+
// a single-pass Map-based replacement: remove the existing entry by path, then
|
|
444
|
+
// append the new one. Net complexity: O(N) build + O(1) replace per write.
|
|
445
|
+
const lastProgressContentHash = new WeakMap<TeamRunManifest, string>();
|
|
446
|
+
|
|
447
|
+
function writeProgress(
|
|
448
|
+
manifest: TeamRunManifest,
|
|
449
|
+
tasks: TeamTaskState[],
|
|
450
|
+
producer: string,
|
|
451
|
+
executeWorkers = true,
|
|
452
|
+
runtimeConfig?: CrewRuntimeConfig,
|
|
453
|
+
): TeamRunManifest {
|
|
454
|
+
const counts = new Map<string, number>();
|
|
455
|
+
for (const task of tasks) counts.set(task.status, (counts.get(task.status) ?? 0) + 1);
|
|
456
|
+
const queue = taskGraphSnapshot(tasks);
|
|
457
|
+
const updatedAt = new Date().toISOString();
|
|
458
|
+
const content = [
|
|
459
|
+
`# pi-crew progress ${manifest.runId}`,
|
|
460
|
+
"",
|
|
461
|
+
`Status: ${manifest.status}`,
|
|
462
|
+
`Team: ${manifest.team}`,
|
|
463
|
+
`Workflow: ${manifest.workflow ?? "(none)"}`,
|
|
464
|
+
`Updated: ${updatedAt}`,
|
|
465
|
+
`Task counts: ${[...counts.entries()].map(([status, count]) => `${status}=${count}`).join(", ") || "none"}`,
|
|
466
|
+
`Queue: ready=${queue.ready.length}, blocked=${queue.blocked.length}, running=${queue.running.length}, done=${queue.done.length}, failed=${queue.failed.length}, cancelled=${queue.cancelled.length}`,
|
|
467
|
+
"",
|
|
468
|
+
"## Tasks",
|
|
469
|
+
...tasks.map(formatTaskProgress),
|
|
470
|
+
"",
|
|
471
|
+
"## Effectiveness",
|
|
472
|
+
...runEffectivenessLines(manifest, tasks, executeWorkers, runtimeConfig),
|
|
473
|
+
"",
|
|
474
|
+
].join("\n");
|
|
475
|
+
|
|
476
|
+
// P6 content-cache: even with identical status / counts / queue, the
|
|
477
|
+
// `Updated:` timestamp ticks on every call so the content rarely matches
|
|
478
|
+
// byte-for-byte. We DO compare against the previous rendered byte-stream
|
|
479
|
+
// (which used the previous timestamp) — so this only hits on the
|
|
480
|
+
// back-to-back writeProgress calls during the applyPolicy phase, where
|
|
481
|
+
// both calls happen within the same millisecond. It's a minor win but
|
|
482
|
+
// matches the audit recommendation (skip artifact write when nothing
|
|
483
|
+
// material changed).
|
|
484
|
+
const prevHash = lastProgressContentHash.get(manifest);
|
|
485
|
+
// Cheap pre-check: avoid the redaction + atomicWrite + readback roundtrip
|
|
486
|
+
// when both the timestamp and the input args are identical to last time.
|
|
487
|
+
const canSkip = prevHash === hashContent(content);
|
|
488
|
+
|
|
489
|
+
const progress = canSkip
|
|
490
|
+
? (() => {
|
|
491
|
+
// Reuse the previous artifact descriptor rather than rebuilding one
|
|
492
|
+
// via writeArtifact. This skips mkdirSync, resolveRealContainedPath,
|
|
493
|
+
// redactSecrets, atomicWriteFile, and the post-write readFileSync +
|
|
494
|
+
// statSync.
|
|
495
|
+
const existing = manifest.artifacts.find((a) => a.kind === "progress");
|
|
496
|
+
if (existing) return existing;
|
|
497
|
+
// No prior progress artifact (rare; first call from a stale manifest
|
|
498
|
+
// view). Fall through to the normal write.
|
|
499
|
+
return writeArtifact(manifest.artifactsRoot, {
|
|
500
|
+
kind: "progress",
|
|
501
|
+
relativePath: "progress.md",
|
|
502
|
+
producer,
|
|
503
|
+
content,
|
|
504
|
+
});
|
|
505
|
+
})()
|
|
506
|
+
: writeArtifact(manifest.artifactsRoot, {
|
|
507
|
+
kind: "progress",
|
|
508
|
+
relativePath: "progress.md",
|
|
509
|
+
producer,
|
|
510
|
+
content,
|
|
511
|
+
});
|
|
512
|
+
lastProgressContentHash.set(manifest, hashContent(content));
|
|
513
|
+
|
|
514
|
+
// P6 dedup: replace by path in a single Map pass instead of
|
|
515
|
+
// .filter(...) // O(N) to remove the old entry
|
|
516
|
+
// .filter((_, i, self) => self.findIndex(...) === i) // O(N²) for dedup
|
|
517
|
+
// For an artifact list of size 30+ across a long run, this was the
|
|
518
|
+
// dominant cost of writeProgress between batches.
|
|
519
|
+
const byPath = new Map<string, ArtifactDescriptor>();
|
|
520
|
+
for (const artifact of manifest.artifacts) {
|
|
521
|
+
if (artifact.kind === "progress" && artifact.path === progress.path) continue;
|
|
522
|
+
byPath.set(artifact.path, artifact);
|
|
523
|
+
}
|
|
524
|
+
byPath.set(progress.path, progress);
|
|
525
|
+
const deduped = [...byPath.values()];
|
|
526
|
+
|
|
527
|
+
return {
|
|
528
|
+
...manifest,
|
|
529
|
+
updatedAt,
|
|
530
|
+
artifacts: deduped,
|
|
531
|
+
};
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?: CrewLimitsConfig): TeamRunManifest {
|
|
535
|
+
const branchFreshness = checkBranchFreshness(manifest.cwd);
|
|
536
|
+
const branchArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
537
|
+
kind: "metadata",
|
|
538
|
+
relativePath: "metadata/branch-freshness.json",
|
|
539
|
+
producer: "branch-freshness",
|
|
540
|
+
content: `${JSON.stringify(branchFreshness, null, 2)}\n`,
|
|
541
|
+
});
|
|
542
|
+
let decisions: PolicyDecision[] = evaluateCrewPolicy({
|
|
543
|
+
manifest,
|
|
544
|
+
tasks,
|
|
545
|
+
limits,
|
|
546
|
+
});
|
|
547
|
+
if (branchFreshness.status === "stale" || branchFreshness.status === "diverged") {
|
|
548
|
+
const branchDecision: PolicyDecision = {
|
|
549
|
+
action: "notify",
|
|
550
|
+
reason: "branch_stale",
|
|
551
|
+
message: branchFreshness.message,
|
|
552
|
+
createdAt: new Date().toISOString(),
|
|
553
|
+
};
|
|
554
|
+
decisions = [...decisions, branchDecision];
|
|
555
|
+
appendEvent(manifest.eventsPath, {
|
|
556
|
+
type: "branch.stale",
|
|
557
|
+
runId: manifest.runId,
|
|
558
|
+
message: branchFreshness.message,
|
|
559
|
+
data: { branchFreshness },
|
|
560
|
+
});
|
|
561
|
+
}
|
|
562
|
+
const policyArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
563
|
+
kind: "metadata",
|
|
564
|
+
relativePath: "policy-decisions.json",
|
|
565
|
+
producer: "policy-engine",
|
|
566
|
+
content: `${JSON.stringify(decisions, null, 2)}\n`,
|
|
567
|
+
});
|
|
568
|
+
const recoveryLedger = buildRecoveryLedger(decisions);
|
|
569
|
+
const recoveryArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
570
|
+
kind: "metadata",
|
|
571
|
+
relativePath: "recovery-ledger.json",
|
|
572
|
+
producer: "recovery-engine",
|
|
573
|
+
content: `${JSON.stringify(recoveryLedger, null, 2)}\n`,
|
|
574
|
+
});
|
|
575
|
+
for (const item of decisions)
|
|
576
|
+
appendEvent(manifest.eventsPath, {
|
|
577
|
+
type: item.action === "escalate" ? "policy.escalated" : "policy.action",
|
|
578
|
+
runId: manifest.runId,
|
|
579
|
+
taskId: item.taskId,
|
|
580
|
+
message: item.message,
|
|
581
|
+
data: { action: item.action, reason: item.reason },
|
|
582
|
+
});
|
|
583
|
+
for (const item of recoveryLedger.entries)
|
|
584
|
+
appendEvent(manifest.eventsPath, {
|
|
585
|
+
type: item.state === "escalation_required" ? "recovery.escalated" : "recovery.attempted",
|
|
586
|
+
runId: manifest.runId,
|
|
587
|
+
taskId: item.taskId,
|
|
588
|
+
message: item.message,
|
|
589
|
+
data: {
|
|
590
|
+
scenario: item.scenario,
|
|
591
|
+
steps: item.steps,
|
|
592
|
+
attempt: item.attempt,
|
|
593
|
+
state: item.state,
|
|
594
|
+
},
|
|
595
|
+
});
|
|
596
|
+
return {
|
|
597
|
+
...manifest,
|
|
598
|
+
updatedAt: new Date().toISOString(),
|
|
599
|
+
policyDecisions: decisions,
|
|
600
|
+
artifacts: [
|
|
601
|
+
...manifest.artifacts.filter(
|
|
602
|
+
(artifact) =>
|
|
603
|
+
!(
|
|
604
|
+
artifact.kind === "metadata" &&
|
|
605
|
+
(artifact.path.endsWith("policy-decisions.json") ||
|
|
606
|
+
artifact.path.endsWith("recovery-ledger.json") ||
|
|
607
|
+
artifact.path.endsWith("branch-freshness.json"))
|
|
608
|
+
),
|
|
609
|
+
),
|
|
610
|
+
branchArtifact,
|
|
611
|
+
policyArtifact,
|
|
612
|
+
recoveryArtifact,
|
|
613
|
+
],
|
|
614
|
+
};
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
function retryPolicyFromConfig(config: CrewReliabilityConfig | undefined): RetryPolicy {
|
|
618
|
+
return { ...DEFAULT_RETRY_POLICY, ...(config?.retryPolicy ?? {}) };
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
/**
|
|
622
|
+
* #1 (assessment): decide whether the per-task retry path (executeWithRetry) is used.
|
|
623
|
+
* Defaults to TRUE (opt-out) so transient worker hangs (ChildTimeout) are retried
|
|
624
|
+
* automatically. Previously opt-in, which left the entire retry+recovery stack dormant.
|
|
625
|
+
* Exported for unit testing.
|
|
626
|
+
*/
|
|
627
|
+
export function shouldUseRetry(reliability: CrewReliabilityConfig | undefined): boolean {
|
|
628
|
+
return reliability?.autoRetry !== false;
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
function failedTaskFrom(result: { tasks: TeamTaskState[] }, taskId: string): TeamTaskState | undefined {
|
|
632
|
+
return result.tasks.find((item) => item.id === taskId && item.status === "failed");
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
function requiresPlanApproval(_workflow: WorkflowConfig, runtimeConfig: CrewRuntimeConfig | undefined): boolean {
|
|
636
|
+
// ROADMAP T1.2: plan-level HITL applies to ANY workflow when
|
|
637
|
+
// config.runtime.requirePlanApproval === true (not just 'implementation').
|
|
638
|
+
// The gate fires at the read-only → mutating (plan → execute) boundary.
|
|
639
|
+
return runtimeConfig?.requirePlanApproval === true;
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
function isPlanApprovalPending(manifest: TeamRunManifest): boolean {
|
|
643
|
+
return manifest.planApproval?.required === true && manifest.planApproval.status === "pending";
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
function isMutatingTask(task: TeamTaskState): boolean {
|
|
647
|
+
return permissionForRole(task.role) !== "read_only";
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
async function ensurePlanApprovalRequested(manifest: TeamRunManifest, tasks: TeamTaskState[]): Promise<TeamRunManifest> {
|
|
651
|
+
if (manifest.planApproval) return manifest;
|
|
652
|
+
const assessTask = tasks.find((task) => task.stepId === "assess" && task.status === "completed");
|
|
653
|
+
// ROADMAP T1.2: for non-adaptive workflows, fall back to the most recent
|
|
654
|
+
// completed read-only (planning) task as the plan reference.
|
|
655
|
+
const planTask = assessTask ?? [...tasks].reverse().find((t) => t.status === "completed" && !isMutatingTask(t));
|
|
656
|
+
const now = new Date().toISOString();
|
|
657
|
+
const updated: TeamRunManifest = {
|
|
658
|
+
...manifest,
|
|
659
|
+
updatedAt: now,
|
|
660
|
+
planApproval: {
|
|
661
|
+
required: true,
|
|
662
|
+
status: "pending",
|
|
663
|
+
requestedAt: now,
|
|
664
|
+
updatedAt: now,
|
|
665
|
+
planTaskId: planTask?.id,
|
|
666
|
+
planArtifactPath: planTask?.resultArtifact?.path,
|
|
667
|
+
},
|
|
668
|
+
};
|
|
669
|
+
await saveRunManifestAsync(updated);
|
|
670
|
+
appendEvent(updated.eventsPath, {
|
|
671
|
+
type: "plan.approval_required",
|
|
672
|
+
runId: updated.runId,
|
|
673
|
+
taskId: planTask?.id,
|
|
674
|
+
message: "Plan requires explicit approval before mutating tasks run. Use: team api op=approve-plan runId=...",
|
|
675
|
+
data: { planArtifactPath: planTask?.resultArtifact?.path },
|
|
676
|
+
});
|
|
677
|
+
return updated;
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
function cancelPlanTasks(tasks: TeamTaskState[], reason: string): TeamTaskState[] {
|
|
681
|
+
return tasks.map((task) =>
|
|
682
|
+
task.status === "queued" || task.status === "running" || task.status === "waiting"
|
|
683
|
+
? {
|
|
684
|
+
...task,
|
|
685
|
+
status: "cancelled",
|
|
686
|
+
finishedAt: new Date().toISOString(),
|
|
687
|
+
error: reason,
|
|
688
|
+
graph: task.graph ? { ...task.graph, queue: "done" } : undefined,
|
|
689
|
+
}
|
|
690
|
+
: task,
|
|
691
|
+
);
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
function hasPendingMutatingAdaptiveTask(tasks: TeamTaskState[]): boolean {
|
|
695
|
+
return tasks.some((task) => task.status === "queued" && task.adaptive && isMutatingTask(task));
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
/**
|
|
699
|
+
* ROADMAP T1.2: gate detection for ANY workflow (not just adaptive).
|
|
700
|
+
* Fires when there are pending mutating tasks whose prerequisites (read-only
|
|
701
|
+
* tasks) have completed — i.e. the plan→execute boundary.
|
|
702
|
+
*/
|
|
703
|
+
export function hasPendingMutatingTaskAtBoundary(tasks: TeamTaskState[]): boolean {
|
|
704
|
+
const hasCompletedReadOnly = tasks.some((t) => t.status === "completed" && !isMutatingTask(t));
|
|
705
|
+
const hasPendingMutating = tasks.some((t) => t.status === "queued" && isMutatingTask(t));
|
|
706
|
+
return hasCompletedReadOnly && hasPendingMutating;
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
/**
|
|
710
|
+
* Check whether any task uses explicit `dependsOn` that would benefit from DAG-based
|
|
711
|
+
* execution planning. If so, build an execution plan and use `getDagReadyTasks`
|
|
712
|
+
* to augment the ready-set selection.
|
|
713
|
+
*/
|
|
714
|
+
function dagReadyTaskIds(tasks: TeamTaskState[], completedIds: Set<string>): string[] | null {
|
|
715
|
+
const hasExplicitDeps = tasks.some((t) => t.dependsOn.length > 0);
|
|
716
|
+
if (!hasExplicitDeps) return null;
|
|
717
|
+
// FIX (goal-wrap runtime test): task.dependsOn stores STEP IDs (e.g. "execute"), not
|
|
718
|
+
// task IDs (e.g. "02_execute"). The DAG scheduler compares deps against completedIds
|
|
719
|
+
// (which are task IDs), so step-ID deps would never match → dependent tasks stuck blocked
|
|
720
|
+
// forever. Map step IDs -> task IDs first (mirror dependencySatisfied in
|
|
721
|
+
// task-graph-scheduler.ts which handles this via stepToTaskId). buildDagExecutionPlan +
|
|
722
|
+
// getDagReadyTasks then work on consistent task IDs.
|
|
723
|
+
const stepToTaskId = new Map<string, string>();
|
|
724
|
+
for (const t of tasks) {
|
|
725
|
+
if (t.stepId) stepToTaskId.set(t.stepId, t.id);
|
|
726
|
+
}
|
|
727
|
+
const nodes: TaskNode[] = tasks.map((t) => ({
|
|
728
|
+
id: t.id,
|
|
729
|
+
dependsOn: t.dependsOn.map((dep) => stepToTaskId.get(dep) ?? dep),
|
|
730
|
+
phase: t.adaptive?.phase ?? t.stepId,
|
|
731
|
+
}));
|
|
732
|
+
const plan = buildDagExecutionPlan(nodes);
|
|
733
|
+
if (plan.hasCycle) return null; // fall back to existing scheduler
|
|
734
|
+
return getDagReadyTasks(plan, completedIds);
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
/**
|
|
738
|
+
* Drain in-flight dispatch units (pendingUnits) by aborting the run-scoped
|
|
739
|
+
* controller and awaiting all settled promises before clearing the map.
|
|
740
|
+
*
|
|
741
|
+
* CORE-1 fix: without this, every early-return path inside the main while
|
|
742
|
+
* loop would abandon pendingUnits — leaving zombie child processes running
|
|
743
|
+
* with no one listening for their results.
|
|
744
|
+
*
|
|
745
|
+
* Exported so unit tests can exercise it directly.
|
|
746
|
+
*/
|
|
747
|
+
export async function drainPendingUnits(
|
|
748
|
+
pendingUnits: Map<string, { taskIds: string[]; promise: Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> }>,
|
|
749
|
+
controller?: AbortController,
|
|
750
|
+
): Promise<void> {
|
|
751
|
+
if (pendingUnits.size === 0) return;
|
|
752
|
+
controller?.abort();
|
|
753
|
+
await Promise.allSettled([...pendingUnits.values()].map((p) => p.promise));
|
|
754
|
+
pendingUnits.clear();
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
|
|
758
|
+
const workflow = input.workflow;
|
|
759
|
+
|
|
760
|
+
// DEFENSE-IN-DEPTH (advisory-only since v0.9.15): re-validate topology here in
|
|
761
|
+
// case a caller bypassed the extension-layer handleRun guard. The extension
|
|
762
|
+
// layer logs an advisory note and proceeds; this defense-in-depth does the same
|
|
763
|
+
// for direct API callers (CLI, tests, scheduler). Never blocks.
|
|
764
|
+
// Skip for synthetic direct-agent workflows (filePath="<generated>").
|
|
765
|
+
if (workflow.filePath !== "<generated>") {
|
|
766
|
+
// LAZY: defer preflight-validator import until the defense-in-depth guard actually runs.
|
|
767
|
+
const { validateWorkflowUsage } = await import("../workflows/preflight-validator.ts");
|
|
768
|
+
const preflight = validateWorkflowUsage(workflow, {
|
|
769
|
+
force: input.reliability?.forcePreflight === true,
|
|
770
|
+
});
|
|
771
|
+
if (preflight.level === "warn" || preflight.level === "note" || preflight.level === "info") {
|
|
772
|
+
const icon = preflight.level === "warn" ? "⚠️ " : preflight.level === "note" ? "✅ " : "ℹ️ ";
|
|
773
|
+
console.warn(
|
|
774
|
+
`${icon}[team-runner.preflight] ${preflight.level.toUpperCase()}: ${preflight.message} (workflow=${workflow.name})`,
|
|
775
|
+
);
|
|
776
|
+
if (preflight.suggestion) {
|
|
777
|
+
console.warn(`[team-runner.preflight] → ${preflight.suggestion}`);
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
let manifest = updateRunStatus(
|
|
783
|
+
input.manifest,
|
|
784
|
+
"running",
|
|
785
|
+
input.executeWorkers ? "Executing team workflow." : "Creating workflow prompts and placeholder results.",
|
|
786
|
+
);
|
|
787
|
+
|
|
788
|
+
// Persist budget fields on the manifest so all subsequent saveRunManifest
|
|
789
|
+
// calls (there are many in executeTeamRunCore) preserve the budget config.
|
|
790
|
+
// Without this, the spread in updateRunStatus and other transformations
|
|
791
|
+
// can drop budget fields before they reach the persistence layer.
|
|
792
|
+
if (input.budgetTotal !== undefined) manifest.budgetTotal = input.budgetTotal;
|
|
793
|
+
if (input.budgetWarning !== undefined) manifest.budgetWarning = input.budgetWarning;
|
|
794
|
+
if (input.budgetAbort !== undefined) manifest.budgetAbort = input.budgetAbort;
|
|
795
|
+
if (input.budgetUnlimited !== undefined) manifest.budgetUnlimited = input.budgetUnlimited;
|
|
796
|
+
if (manifest.budgetTotal !== undefined && manifest.budgetTotal > 0 && !manifest.budgetUnlimited) {
|
|
797
|
+
await saveRunManifestAsync(manifest);
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
void registerRunPromise(manifest.runId);
|
|
801
|
+
|
|
802
|
+
// FIX (Round 15, regression): Start a team-level heartbeat so the stale
|
|
803
|
+
// reconciler does not cancel long-running team runs after 5 minutes
|
|
804
|
+
// (NO_PID_HEARTBEAT_STALE_MS). Previously only sub-task runners wrote
|
|
805
|
+
// heartbeats; the team-level run had no heartbeat, so any multi-phase
|
|
806
|
+
// workflow lasting >5min was marked stale and cancelled.
|
|
807
|
+
const stopTeamHeartbeat = startTeamRunHeartbeat(manifest.stateRoot, manifest.runId);
|
|
808
|
+
|
|
809
|
+
const cleanupUsage = (): void => {
|
|
810
|
+
for (const task of input.tasks) clearTrackedTaskUsage(task.id);
|
|
811
|
+
};
|
|
812
|
+
|
|
813
|
+
try {
|
|
814
|
+
const result = await executeTeamRunCore(input, manifest, workflow);
|
|
815
|
+
// #2 (assessment): goal-achievement detection — kill the silent false-green.
|
|
816
|
+
// A code-mutating run that "completed" but left the git working tree clean
|
|
817
|
+
// (and/or had a failed task) is a false-green. We expose goalAchieved on the
|
|
818
|
+
// manifest + emit an event so the lie is never silent, and downgrade status
|
|
819
|
+
// to "failed" only when a failed task corroborates it (conservative).
|
|
820
|
+
const gaAssessment = assessGoalAchievement(result.manifest, result.tasks, workflow);
|
|
821
|
+
const gaApplied = applyGoalAchievement(result.manifest, gaAssessment);
|
|
822
|
+
if (gaApplied.manifest !== result.manifest) {
|
|
823
|
+
result.manifest = gaApplied.manifest;
|
|
824
|
+
try {
|
|
825
|
+
await saveRunManifestAsync(result.manifest);
|
|
826
|
+
} catch (persistError) {
|
|
827
|
+
logInternalError(
|
|
828
|
+
"team-runner.goalAchievement.persist",
|
|
829
|
+
persistError instanceof Error ? persistError : new Error(String(persistError)),
|
|
830
|
+
`runId=${manifest.runId}`,
|
|
831
|
+
);
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
appendEvent(manifest.eventsPath, {
|
|
835
|
+
type: "run.goal_achievement",
|
|
836
|
+
runId: manifest.runId,
|
|
837
|
+
message: gaApplied.manifest.goalAchievementNote ?? "",
|
|
838
|
+
data: {
|
|
839
|
+
achieved: gaAssessment.achieved,
|
|
840
|
+
downgraded: gaApplied.downgraded,
|
|
841
|
+
reason: gaAssessment.reason,
|
|
842
|
+
signals: gaAssessment.signals,
|
|
843
|
+
},
|
|
844
|
+
});
|
|
845
|
+
if (gaApplied.downgraded)
|
|
846
|
+
logInternalError(
|
|
847
|
+
"team-runner.goalAchievement.falseGreen",
|
|
848
|
+
new Error(gaApplied.manifest.goalAchievementNote ?? "false-green detected"),
|
|
849
|
+
`runId=${manifest.runId}`,
|
|
850
|
+
"error",
|
|
851
|
+
);
|
|
852
|
+
stopTeamHeartbeat();
|
|
853
|
+
resolveRunPromise(manifest.runId, result);
|
|
854
|
+
// Terminate live agents for this run — agents are done when the run ends.
|
|
855
|
+
void terminateLiveAgentsForRun(manifest.runId, "completed", appendEvent, manifest.eventsPath).catch((error) =>
|
|
856
|
+
logInternalError("team-runner.completed.terminate", error, `runId=${manifest.runId}`),
|
|
857
|
+
);
|
|
858
|
+
|
|
859
|
+
// Emit run completion hook (100% reliable, fire-and-forget)
|
|
860
|
+
crewHooks.emit({
|
|
861
|
+
type: "run_completed",
|
|
862
|
+
timestamp: new Date().toISOString(),
|
|
863
|
+
runId: manifest.runId,
|
|
864
|
+
data: {
|
|
865
|
+
status: result.manifest.status,
|
|
866
|
+
taskCount: result.tasks.length,
|
|
867
|
+
},
|
|
868
|
+
});
|
|
869
|
+
|
|
870
|
+
// Execute after_run_complete lifecycle hook (non-blocking)
|
|
871
|
+
const afterRunReport = await executeHook("after_run_complete", {
|
|
872
|
+
runId: manifest.runId,
|
|
873
|
+
cwd: manifest.cwd,
|
|
874
|
+
status: result.manifest.status,
|
|
875
|
+
});
|
|
876
|
+
appendHookEvent(manifest, afterRunReport);
|
|
877
|
+
if (afterRunReport.outcome === "block") {
|
|
878
|
+
logInternalError(
|
|
879
|
+
"team-runner.after_run_complete.blocked",
|
|
880
|
+
new Error(afterRunReport.reason ?? "after_run_complete hook blocked"),
|
|
881
|
+
`runId=${manifest.runId}`,
|
|
882
|
+
);
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
// M7: flush buffered task.progress events so the final state is durable
|
|
886
|
+
// before the run returns. Buffered producer wins on latency (p95≈0µs);
|
|
887
|
+
// this single flush at run-end coalesces any pending progress bursts
|
|
888
|
+
// before manifest updates are observed by readers.
|
|
889
|
+
await flushEventLogBuffer();
|
|
890
|
+
return result;
|
|
891
|
+
} catch (error) {
|
|
892
|
+
// Round 27 (BUG 1): the success path calls stopTeamHeartbeat() but this
|
|
893
|
+
// catch path did NOT. The team heartbeat is a non-unref'd setInterval
|
|
894
|
+
// (30s) that deliberately keeps the event loop alive — without this
|
|
895
|
+
// call, a failed team run leaves the interval firing forever and the
|
|
896
|
+
// foreground pi process hangs (never returns to the prompt); in
|
|
897
|
+
// background-runner mode the worker never exits. clearInterval is
|
|
898
|
+
// idempotent so a double-call (if this runs after the success path)
|
|
899
|
+
// is harmless.
|
|
900
|
+
stopTeamHeartbeat();
|
|
901
|
+
// P1: Catch unhandled errors — ensure manifest/tasks/agents are terminal so they don't stay "running" forever.
|
|
902
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
903
|
+
// Re-read the latest persisted state from disk instead of trusting
|
|
904
|
+
// input.tasks (the ORIGINAL start snapshot, still all "queued" — it is never
|
|
905
|
+
// mutated by executeTeamRunCore). A late failure during closeout would
|
|
906
|
+
// otherwise map every task to "failed", overwriting tasks that already
|
|
907
|
+
// completed during the run. loadRunManifestById is the established
|
|
908
|
+
// fresh-read pattern in this file (see ~line 1269); it is best-effort with
|
|
909
|
+
// no lock, consistent with the lock-drop decision below. If the disk read
|
|
910
|
+
// fails, fall back to input.tasks so the run is still marked terminal.
|
|
911
|
+
const fresh = loadRunManifestById(manifest.cwd, manifest.runId);
|
|
912
|
+
const freshManifest = fresh?.manifest ?? manifest;
|
|
913
|
+
const freshTasks = refreshTaskGraphQueues(fresh?.tasks ?? input.tasks);
|
|
914
|
+
const tasks = cancelNonTerminalTasks(freshTasks, "failed", message);
|
|
915
|
+
manifest = freshManifest;
|
|
916
|
+
try {
|
|
917
|
+
await terminateLiveAgentsForRun(manifest.runId, "failed", appendEvent, manifest.eventsPath);
|
|
918
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
919
|
+
const existingRuntimeByTask = new Map(readCrewAgents(manifest).map((agent) => [agent.taskId, agent.runtime]));
|
|
920
|
+
const globalRuntime = input.runtime?.kind ?? "child-process";
|
|
921
|
+
const runtimeForAgent = (agent: ReturnType<typeof recordsForMaterializedTasks>[number]): CrewRuntimeKind => {
|
|
922
|
+
const task = tasks.find((item) => item.id === agent.taskId);
|
|
923
|
+
return (
|
|
924
|
+
existingRuntimeByTask.get(agent.taskId) ??
|
|
925
|
+
resolveTaskRuntimeKind(globalRuntime, task?.role ?? agent.role, input.runtimeConfig?.isolationPolicy)
|
|
926
|
+
);
|
|
927
|
+
};
|
|
928
|
+
saveCrewAgents(
|
|
929
|
+
manifest,
|
|
930
|
+
recordsForMaterializedTasks(manifest, tasks, globalRuntime).map((agent) => ({ ...agent, runtime: runtimeForAgent(agent) })),
|
|
931
|
+
);
|
|
932
|
+
manifest = updateRunStatus(manifest, "failed", `Unhandled error in team runner: ${message}`);
|
|
933
|
+
await saveRunManifestAsync(manifest);
|
|
934
|
+
} catch {
|
|
935
|
+
// Best-effort — state write may also fail
|
|
936
|
+
}
|
|
937
|
+
const result = { manifest, tasks };
|
|
938
|
+
rejectRunPromise(manifest.runId, error instanceof Error ? error : new Error(message));
|
|
939
|
+
crewHooks.emit({
|
|
940
|
+
type: "run_failed",
|
|
941
|
+
timestamp: new Date().toISOString(),
|
|
942
|
+
runId: manifest.runId,
|
|
943
|
+
data: { status: manifest.status, error: message },
|
|
944
|
+
});
|
|
945
|
+
// M7: flush buffered events before returning on the error path so the
|
|
946
|
+
// final buffered progress events are durable alongside the failure state.
|
|
947
|
+
await flushEventLogBuffer();
|
|
948
|
+
return result;
|
|
949
|
+
} finally {
|
|
950
|
+
// M7+follow-up (v0.9.24): drain buffer on the success path too.
|
|
951
|
+
// Previously only the catch path flushed; under --test-force-exit the
|
|
952
|
+
// success path left pending appendEventAsync promises that the test
|
|
953
|
+
// runner detected as 'Promise resolution is still pending'. The catch
|
|
954
|
+
// still has its own flush for M7 backward compat — flushEventLogBuffer
|
|
955
|
+
// is idempotent on an empty queue.
|
|
956
|
+
await flushEventLogBuffer();
|
|
957
|
+
// A2-F1: clear tracked usage for this run's tasks. Consolidated in finally
|
|
958
|
+
// (previously duplicated on the success + error paths) so cleanup is
|
|
959
|
+
// guaranteed on EVERY exit path — a future throw in the success tail or
|
|
960
|
+
// error handler can no longer leak entries. Safe to run last: nothing in
|
|
961
|
+
// team-runner reads usage after the run resolves, and the TUI widget reads
|
|
962
|
+
// live usage only while a task is running (before this point).
|
|
963
|
+
cleanupUsage();
|
|
964
|
+
// NEW-M1: clear the stable-prefix cache (keyed by runId) so it does not
|
|
965
|
+
// grow unbounded across runs in a long-lived session.
|
|
966
|
+
clearStablePrefixCache();
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
// ── CORE-4: SchedulerContext state bag ─────────────────────────────
|
|
971
|
+
// A mutable bag of the closure locals used across executeTeamRunCore.
|
|
972
|
+
// Extracted scheduler functions receive this context and mutate it
|
|
973
|
+
// in-place. This enables incremental extraction of the ~1075-line god
|
|
974
|
+
// function into scheduler/ functions without changing control flow.
|
|
975
|
+
|
|
976
|
+
/**
|
|
977
|
+
* Mutable state shared across the team-run scheduler loop.
|
|
978
|
+
*
|
|
979
|
+
* Fields mirror the closure locals of `executeTeamRunCore`. Extracted
|
|
980
|
+
* scheduler functions mutate these fields in-place; the caller keeps the
|
|
981
|
+
* local variables in sync by assigning back from `ctx` after each call.
|
|
982
|
+
*/
|
|
983
|
+
interface SchedulerContext {
|
|
984
|
+
input: ExecuteTeamRunInput;
|
|
985
|
+
workflow: WorkflowConfig;
|
|
986
|
+
manifest: TeamRunManifest;
|
|
987
|
+
tasks: TeamTaskState[];
|
|
988
|
+
queueIndex: TaskGraphIndex;
|
|
989
|
+
wfMachine: WorkflowStateMachine;
|
|
990
|
+
pendingUnits: Map<string, { taskIds: string[]; promise: Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> }>;
|
|
991
|
+
runController: AbortController;
|
|
992
|
+
runtimeKind: CrewRuntimeKind;
|
|
993
|
+
adaptivePlanInjected: boolean;
|
|
994
|
+
adaptivePlanMissing: boolean;
|
|
995
|
+
/** Outcome of the most recent mergeUnitResult call: the settled unit's
|
|
996
|
+
* taskIds and the merged result object. Read by the post-merge inline
|
|
997
|
+
* logic (cancel-during-exec check + batch summary). Set by extraction 5. */
|
|
998
|
+
settledMerge: { taskIds: string[]; result: { manifest: TeamRunManifest; tasks: TeamTaskState[] } } | null;
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
/**
|
|
1002
|
+
* Discriminated union representing a scheduler sub-function's decision.
|
|
1003
|
+
*
|
|
1004
|
+
* - `continue`: proceed to the next phase of the loop body.
|
|
1005
|
+
* - `return`: short-circuit the loop and return the given result.
|
|
1006
|
+
* - `skip-dispatch`: skip the dispatch phase this iteration (reserved for
|
|
1007
|
+
* future extractions).
|
|
1008
|
+
*/
|
|
1009
|
+
type SchedulerDecision =
|
|
1010
|
+
| { kind: "continue" }
|
|
1011
|
+
| { kind: "return"; result: { manifest: TeamRunManifest; tasks: TeamTaskState[] } }
|
|
1012
|
+
| { kind: "skip-dispatch" }
|
|
1013
|
+
| {
|
|
1014
|
+
kind: "dispatch";
|
|
1015
|
+
batch: TeamTaskState[];
|
|
1016
|
+
concurrency: BatchConcurrencyDecision;
|
|
1017
|
+
snapshot: TaskGraphSchedulerSnapshot;
|
|
1018
|
+
approvalPending: boolean;
|
|
1019
|
+
coalesceEnabled: boolean;
|
|
1020
|
+
};
|
|
1021
|
+
|
|
1022
|
+
/**
|
|
1023
|
+
* CORE-4 extraction 1: handle a pre-aborted signal at the top of the
|
|
1024
|
+
* scheduler loop.
|
|
1025
|
+
*
|
|
1026
|
+
* If the run's input signal is aborted, cancel all non-terminal tasks,
|
|
1027
|
+
* persist the cancellation, emit run.cancelled + per-task task.cancelled
|
|
1028
|
+
* events, and return a `return` decision so the caller short-circuits.
|
|
1029
|
+
*
|
|
1030
|
+
* Returns `null` when the signal is NOT aborted (no decision — continue).
|
|
1031
|
+
*
|
|
1032
|
+
* @param ctx The scheduler context; `ctx.tasks` and `ctx.manifest` are
|
|
1033
|
+
* mutated in-place to reflect the cancelled state.
|
|
1034
|
+
*/
|
|
1035
|
+
async function cancelRunFromSignal(ctx: SchedulerContext): Promise<SchedulerDecision | null> {
|
|
1036
|
+
if (!ctx.input.signal?.aborted) return null;
|
|
1037
|
+
|
|
1038
|
+
const cancelReason = cancellationReasonFromSignal(ctx.input.signal);
|
|
1039
|
+
const message = `${cancelReason.message} (${cancelReason.code})`;
|
|
1040
|
+
const cancelledTaskIds: string[] = [];
|
|
1041
|
+
ctx.tasks = ctx.tasks.map((task) => {
|
|
1042
|
+
if (task.status !== "queued" && task.status !== "running" && task.status !== "waiting") return task;
|
|
1043
|
+
cancelledTaskIds.push(task.id);
|
|
1044
|
+
const base = {
|
|
1045
|
+
...task,
|
|
1046
|
+
status: "cancelled" as const,
|
|
1047
|
+
finishedAt: new Date().toISOString(),
|
|
1048
|
+
error: message,
|
|
1049
|
+
};
|
|
1050
|
+
if (task.status === "running") {
|
|
1051
|
+
return {
|
|
1052
|
+
...base,
|
|
1053
|
+
terminalEvidence: [
|
|
1054
|
+
...(task.terminalEvidence ?? []),
|
|
1055
|
+
buildSyntheticTerminalEvidence("worker", cancelReason, task.startedAt),
|
|
1056
|
+
],
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
return base;
|
|
1060
|
+
});
|
|
1061
|
+
await saveRunTasksAsync(ctx.manifest, ctx.tasks);
|
|
1062
|
+
for (const taskId of cancelledTaskIds)
|
|
1063
|
+
await appendEventAsync(ctx.manifest.eventsPath, {
|
|
1064
|
+
type: "task.cancelled",
|
|
1065
|
+
runId: ctx.manifest.runId,
|
|
1066
|
+
taskId,
|
|
1067
|
+
message,
|
|
1068
|
+
data: { reason: cancelReason.code },
|
|
1069
|
+
});
|
|
1070
|
+
ctx.manifest = updateRunStatus(ctx.manifest, "cancelled", message, {
|
|
1071
|
+
data: { reason: cancelReason.code, cancelledTaskIds },
|
|
1072
|
+
});
|
|
1073
|
+
return { kind: "return", result: { manifest: ctx.manifest, tasks: ctx.tasks } };
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
/**
|
|
1077
|
+
* CORE-4 extraction 2: handle a failed task detected at the top of the
|
|
1078
|
+
* scheduler loop.
|
|
1079
|
+
*
|
|
1080
|
+
* If a task has status "failed", honor `limits.maxRetriesPerTask` to decide
|
|
1081
|
+
* whether to re-queue it for a bounded whole-task rerun or abort the run.
|
|
1082
|
+
*
|
|
1083
|
+
* - `maxRetriesPerTask > 0` and the task is eligible → re-queue the task
|
|
1084
|
+
* (mutate `ctx.tasks`), emit `recovery.rerun_task`, return `{ kind: "continue" }`
|
|
1085
|
+
* so the caller re-processes the re-queued task.
|
|
1086
|
+
* - Otherwise → mark all queued tasks blocked, persist, mark the run failed,
|
|
1087
|
+
* and return `{ kind: "return", result }` so the caller short-circuits.
|
|
1088
|
+
*
|
|
1089
|
+
* Returns `null` when no failed task exists (no decision — continue).
|
|
1090
|
+
*
|
|
1091
|
+
* @param ctx The scheduler context; `ctx.tasks` and `ctx.manifest` are
|
|
1092
|
+
* mutated in-place to reflect the rerun or abort.
|
|
1093
|
+
*/
|
|
1094
|
+
async function handleFailedTask(ctx: SchedulerContext): Promise<SchedulerDecision | null> {
|
|
1095
|
+
const failed = ctx.tasks.find((task) => task.status === "failed");
|
|
1096
|
+
if (!failed) return null;
|
|
1097
|
+
|
|
1098
|
+
// #4 (assessment): honor limits.maxRetriesPerTask — re-queue an eligible
|
|
1099
|
+
// failed task for a bounded whole-task rerun instead of immediately
|
|
1100
|
+
// aborting the run. Before #4, the recovery ledger recorded `rerun_task`
|
|
1101
|
+
// entries with state:"planned" but never executed them (decorative).
|
|
1102
|
+
// Default-off: maxRetriesPerTask=0 → original abort behavior preserved.
|
|
1103
|
+
const rerun = shouldRerunFailedTask(failed, ctx.input.limits);
|
|
1104
|
+
if (rerun.rerun) {
|
|
1105
|
+
ctx.tasks = ctx.tasks.map((item) =>
|
|
1106
|
+
item.id === failed.id
|
|
1107
|
+
? {
|
|
1108
|
+
...item,
|
|
1109
|
+
status: "queued" as const,
|
|
1110
|
+
policy: {
|
|
1111
|
+
...(item.policy ?? {}),
|
|
1112
|
+
retryCount: rerun.newRetryCount,
|
|
1113
|
+
},
|
|
1114
|
+
error: undefined,
|
|
1115
|
+
finishedAt: undefined,
|
|
1116
|
+
}
|
|
1117
|
+
: item,
|
|
1118
|
+
);
|
|
1119
|
+
await saveRunTasksAsync(ctx.manifest, ctx.tasks);
|
|
1120
|
+
await appendEventAsync(ctx.manifest.eventsPath, {
|
|
1121
|
+
type: "recovery.rerun_task",
|
|
1122
|
+
runId: ctx.manifest.runId,
|
|
1123
|
+
taskId: failed.id,
|
|
1124
|
+
message: `Re-queuing failed task for whole-task rerun: ${rerun.reason}`,
|
|
1125
|
+
data: {
|
|
1126
|
+
attempt: rerun.newRetryCount,
|
|
1127
|
+
maxRetries: ctx.input.limits?.maxRetriesPerTask ?? 0,
|
|
1128
|
+
scenario: "task_failed",
|
|
1129
|
+
},
|
|
1130
|
+
});
|
|
1131
|
+
return { kind: "continue" }; // loop re-processes the re-queued task
|
|
1132
|
+
}
|
|
1133
|
+
ctx.tasks = markBlocked(ctx.tasks, `Blocked by failed task '${failed.id}'.`);
|
|
1134
|
+
await saveRunTasksAsync(ctx.manifest, ctx.tasks);
|
|
1135
|
+
saveCrewAgents(ctx.manifest, recordsForMaterializedTasks(ctx.manifest, ctx.tasks, ctx.runtimeKind));
|
|
1136
|
+
ctx.manifest = updateRunStatus(ctx.manifest, "failed", `Failed at task '${failed.id}'.`);
|
|
1137
|
+
return { kind: "return", result: { manifest: ctx.manifest, tasks: ctx.tasks } };
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
/**
|
|
1141
|
+
* CORE-4 extraction 3: select the dispatch batch for the current loop
|
|
1142
|
+
* iteration.
|
|
1143
|
+
*
|
|
1144
|
+
* Computes the task-graph snapshot, DAG-ready tasks, workflow phase
|
|
1145
|
+
* preconditions, batch concurrency, write-path-overlap serialization,
|
|
1146
|
+
* coalesced-group logging, and streaming-dispatch slot allocation to
|
|
1147
|
+
* determine which tasks are ready to dispatch this cycle.
|
|
1148
|
+
*
|
|
1149
|
+
* Returns:
|
|
1150
|
+
* - `{ kind: "return", result }` when the run must block or abort
|
|
1151
|
+
* (plan-approval pending with mutating tasks, or no ready task at all).
|
|
1152
|
+
* - `{ kind: "dispatch", batch, ... }` when a batch is selected (may be
|
|
1153
|
+
* empty when tasks are still in-flight — the caller proceeds to the
|
|
1154
|
+
* wait phase with an empty dispatch set).
|
|
1155
|
+
*
|
|
1156
|
+
* The caller syncs `ctx.wfMachine` back after the call because this
|
|
1157
|
+
* function may advance the workflow phase state machine.
|
|
1158
|
+
*
|
|
1159
|
+
* @param ctx The scheduler context; `ctx.wfMachine`, `ctx.tasks`, and
|
|
1160
|
+
* `ctx.manifest` may be mutated in-place.
|
|
1161
|
+
*/
|
|
1162
|
+
async function selectDispatchBatch(ctx: SchedulerContext): Promise<SchedulerDecision> {
|
|
1163
|
+
const snapshot = taskGraphSnapshot(ctx.tasks, ctx.queueIndex);
|
|
1164
|
+
|
|
1165
|
+
// DAG-based execution plan: when tasks have explicit dependsOn, use the
|
|
1166
|
+
// topological wave planner to determine ready tasks. Fall back to the
|
|
1167
|
+
// existing task-graph-scheduler when no explicit deps exist (backward compat).
|
|
1168
|
+
const completedIds = new Set(ctx.tasks.filter((t) => t.status === "completed" || t.status === "needs_attention").map((t) => t.id));
|
|
1169
|
+
const dagReady = dagReadyTaskIds(ctx.tasks, completedIds);
|
|
1170
|
+
const readyBeforeFilter = dagReady ?? snapshot.ready;
|
|
1171
|
+
|
|
1172
|
+
// Workflow phase precondition check (non-blocking: log warnings only).
|
|
1173
|
+
if (ctx.wfMachine.currentPhaseIndex < ctx.wfMachine.phases.length) {
|
|
1174
|
+
const completedArtifacts = ctx.manifest.artifacts.filter((a) => a.kind === "result" || a.kind === "summary").map((a) => a.path);
|
|
1175
|
+
const previousPhaseStatus =
|
|
1176
|
+
ctx.wfMachine.currentPhaseIndex > 0
|
|
1177
|
+
? (ctx.wfMachine.phases[ctx.wfMachine.currentPhaseIndex - 1]?.status ?? "pending")
|
|
1178
|
+
: "completed";
|
|
1179
|
+
const wfContext: PhaseGuardContext = {
|
|
1180
|
+
completedArtifacts,
|
|
1181
|
+
previousPhaseStatus,
|
|
1182
|
+
taskResults: ctx.tasks
|
|
1183
|
+
.filter((t) => t.status === "completed" || t.status === "needs_attention")
|
|
1184
|
+
.map((t) => ({
|
|
1185
|
+
taskId: t.id,
|
|
1186
|
+
status: t.status,
|
|
1187
|
+
outputPath: t.resultArtifact?.path,
|
|
1188
|
+
})),
|
|
1189
|
+
};
|
|
1190
|
+
const preconditions = validatePhasePreconditions(ctx.wfMachine, wfContext);
|
|
1191
|
+
if (!preconditions.ready) {
|
|
1192
|
+
await appendEventAsync(ctx.manifest.eventsPath, {
|
|
1193
|
+
type: "workflow.preconditions",
|
|
1194
|
+
runId: ctx.manifest.runId,
|
|
1195
|
+
message: `Workflow phase '${ctx.wfMachine.phases[ctx.wfMachine.currentPhaseIndex]?.name}' is missing inputs: ${preconditions.blocking.join(", ")}`,
|
|
1196
|
+
data: {
|
|
1197
|
+
phaseIndex: ctx.wfMachine.currentPhaseIndex,
|
|
1198
|
+
phaseName: ctx.wfMachine.phases[ctx.wfMachine.currentPhaseIndex]?.name,
|
|
1199
|
+
blocking: preconditions.blocking,
|
|
1200
|
+
},
|
|
1201
|
+
});
|
|
1202
|
+
} else {
|
|
1203
|
+
// Advance the machine past completed phases.
|
|
1204
|
+
while (
|
|
1205
|
+
ctx.wfMachine.currentPhaseIndex < ctx.wfMachine.phases.length &&
|
|
1206
|
+
ctx.wfMachine.phases[ctx.wfMachine.currentPhaseIndex]?.status === "completed"
|
|
1207
|
+
) {
|
|
1208
|
+
ctx.wfMachine = {
|
|
1209
|
+
...ctx.wfMachine,
|
|
1210
|
+
currentPhaseIndex: ctx.wfMachine.currentPhaseIndex + 1,
|
|
1211
|
+
};
|
|
1212
|
+
}
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
const readyRoles = readyBeforeFilter
|
|
1217
|
+
.map((taskId) => ctx.tasks.find((task) => task.id === taskId)?.role)
|
|
1218
|
+
.filter((role): role is string => Boolean(role));
|
|
1219
|
+
const concurrency = resolveBatchConcurrency({
|
|
1220
|
+
workflowName: ctx.workflow.name,
|
|
1221
|
+
workflowMaxConcurrency: ctx.workflow.maxConcurrency,
|
|
1222
|
+
teamMaxConcurrency: ctx.input.team.maxConcurrency,
|
|
1223
|
+
limitMaxConcurrentWorkers: ctx.input.limits?.maxConcurrentWorkers,
|
|
1224
|
+
allowUnboundedConcurrency: ctx.input.limits?.allowUnboundedConcurrency,
|
|
1225
|
+
readyCount: readyBeforeFilter.length,
|
|
1226
|
+
workspaceMode: ctx.manifest.workspaceMode,
|
|
1227
|
+
readyRoles,
|
|
1228
|
+
});
|
|
1229
|
+
|
|
1230
|
+
// Round 25 (M5): serialize on write-path overlap when opted in.
|
|
1231
|
+
// Opt-in via limits.serializeOnPathOverlap; default off (= no behavior change).
|
|
1232
|
+
// filterReadyByWriteOverlap returns the same array when enabled=false, so
|
|
1233
|
+
// production runs pay nothing for the unused code path. When the flag is on,
|
|
1234
|
+
// `serializedReady` MAY be a strict subset of `readyBeforeFilter` (conflicting tasks
|
|
1235
|
+
// deferred to next cycle).
|
|
1236
|
+
const serializedReady = filterReadyByWriteOverlap(
|
|
1237
|
+
readyBeforeFilter,
|
|
1238
|
+
ctx.tasks,
|
|
1239
|
+
ctx.workflow,
|
|
1240
|
+
concurrency.maxConcurrent,
|
|
1241
|
+
ctx.input.limits?.serializeOnPathOverlap === true,
|
|
1242
|
+
);
|
|
1243
|
+
|
|
1244
|
+
// Round 25 (M6): coalesce micro-tasks when opted in.
|
|
1245
|
+
// Default off; when on, groups same-(role,cwd) tasks into coalesced groups
|
|
1246
|
+
// (with write-path safety). In v0.9.17 first ship, we ONLY log the
|
|
1247
|
+
// coalesced group count to the event stream (informational). Actual
|
|
1248
|
+
// dispatching of one-multi-task worker instead of N workers is deferred
|
|
1249
|
+
// to a follow-up — it's a non-trivial prompt-construction change that
|
|
1250
|
+
// deserves its own PR. For now, every coalesced group => one info event.
|
|
1251
|
+
const coalesceEnabled = ctx.workflow.coalesceMicroTasks === true;
|
|
1252
|
+
if (coalesceEnabled) {
|
|
1253
|
+
const coalescedGroups = planCoalescedGroups(serializedReady, ctx.tasks, ctx.workflow, true);
|
|
1254
|
+
for (const group of coalescedGroups) {
|
|
1255
|
+
if (group.tasks.length < 2) continue; // singletons are not interesting
|
|
1256
|
+
await appendEventAsync(ctx.manifest.eventsPath, {
|
|
1257
|
+
type: "task.coalesced",
|
|
1258
|
+
runId: ctx.manifest.runId,
|
|
1259
|
+
message: `Coalesced ${group.tasks.length} micro-tasks (role=${group.role}, cwd=${group.cwd})`,
|
|
1260
|
+
data: {
|
|
1261
|
+
groupId: group.id,
|
|
1262
|
+
role: group.role,
|
|
1263
|
+
cwd: group.cwd,
|
|
1264
|
+
taskIds: group.tasks.map((task) => task.id),
|
|
1265
|
+
},
|
|
1266
|
+
});
|
|
1267
|
+
}
|
|
1268
|
+
}
|
|
1269
|
+
if (concurrency.reason.includes(";unbounded:")) {
|
|
1270
|
+
await appendEventAsync(ctx.manifest.eventsPath, {
|
|
1271
|
+
type: "limits.unbounded",
|
|
1272
|
+
runId: ctx.manifest.runId,
|
|
1273
|
+
message: "Unbounded worker concurrency was explicitly enabled for this run.",
|
|
1274
|
+
data: {
|
|
1275
|
+
concurrencyReason: concurrency.reason,
|
|
1276
|
+
maxConcurrent: concurrency.maxConcurrent,
|
|
1277
|
+
},
|
|
1278
|
+
});
|
|
1279
|
+
}
|
|
1280
|
+
// ── OPT-01 streaming dispatch: exclude tasks already in-flight, limit
|
|
1281
|
+
// new dispatches to available concurrency slots. ──
|
|
1282
|
+
const inFlightTaskIds = new Set<string>();
|
|
1283
|
+
for (const pendingUnit of ctx.pendingUnits.values()) {
|
|
1284
|
+
for (const taskId of pendingUnit.taskIds) inFlightTaskIds.add(taskId);
|
|
1285
|
+
}
|
|
1286
|
+
const slotsAvailable = Math.max(0, concurrency.maxConcurrent - ctx.pendingUnits.size);
|
|
1287
|
+
const approvalPending = isPlanApprovalPending(ctx.manifest);
|
|
1288
|
+
const dispatchableReady = serializedReady.filter((id) => !inFlightTaskIds.has(id));
|
|
1289
|
+
const readyIds = approvalPending ? dispatchableReady : dispatchableReady.slice(0, slotsAvailable);
|
|
1290
|
+
const candidateBatch = readyIds
|
|
1291
|
+
.map((id) => ctx.tasks.find((task) => task.id === id))
|
|
1292
|
+
.filter((task): task is TeamTaskState => Boolean(task));
|
|
1293
|
+
const readyBatch = approvalPending ? candidateBatch.filter((task) => !isMutatingTask(task)).slice(0, slotsAvailable) : candidateBatch;
|
|
1294
|
+
if (readyBatch.length === 0) {
|
|
1295
|
+
if (ctx.pendingUnits.size > 0) {
|
|
1296
|
+
// Tasks are in-flight — skip dispatch and proceed to wait phase.
|
|
1297
|
+
// (No return; code falls through to the dispatch section which is
|
|
1298
|
+
// a no-op with an empty readyBatch, then reaches the wait phase.)
|
|
1299
|
+
} else if (approvalPending && candidateBatch.some(isMutatingTask)) {
|
|
1300
|
+
await saveRunTasksAsync(ctx.manifest, ctx.tasks);
|
|
1301
|
+
saveCrewAgents(ctx.manifest, recordsForMaterializedTasks(ctx.manifest, ctx.tasks, ctx.runtimeKind));
|
|
1302
|
+
ctx.manifest = updateRunStatus(ctx.manifest, "blocked", "Plan approval required before mutating implementation tasks run.");
|
|
1303
|
+
return { kind: "return", result: { manifest: ctx.manifest, tasks: ctx.tasks } };
|
|
1304
|
+
} else {
|
|
1305
|
+
ctx.tasks = markBlocked(ctx.tasks, "No ready queued task; dependency graph may be invalid.");
|
|
1306
|
+
await saveRunTasksAsync(ctx.manifest, ctx.tasks);
|
|
1307
|
+
saveCrewAgents(ctx.manifest, recordsForMaterializedTasks(ctx.manifest, ctx.tasks, ctx.runtimeKind));
|
|
1308
|
+
ctx.manifest = updateRunStatus(ctx.manifest, "blocked", "No ready queued task.");
|
|
1309
|
+
return { kind: "return", result: { manifest: ctx.manifest, tasks: ctx.tasks } };
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
|
|
1313
|
+
return { kind: "dispatch", batch: readyBatch, concurrency, snapshot, approvalPending, coalesceEnabled };
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1316
|
+
/** Dispatch decision variant returned by selectDispatchBatch. */
|
|
1317
|
+
type DispatchBatchDecision = Extract<SchedulerDecision, { kind: "dispatch" }>;
|
|
1318
|
+
|
|
1319
|
+
/**
|
|
1320
|
+
* CORE-4 extraction 4: execute the dispatch batch selected by
|
|
1321
|
+
* selectDispatchBatch.
|
|
1322
|
+
*
|
|
1323
|
+
* Runs before_task_start hooks (skipping blocked tasks), builds coalesced
|
|
1324
|
+
* dispatch units, pre-warms the stable-prefix cache for unique cwds, and
|
|
1325
|
+
* dispatches each unit into ctx.pendingUnits as a fire-and-forget promise
|
|
1326
|
+
* (wrapped in executeWithRetry on the singleton path). The function is a
|
|
1327
|
+
* verbatim lift of the inline dispatch block; it does not return a
|
|
1328
|
+
* SchedulerDecision (void — it only populates ctx.pendingUnits).
|
|
1329
|
+
*
|
|
1330
|
+
* Reads ctx.manifest/tasks/workflow/input + runController.signal. Mutates
|
|
1331
|
+
* ctx.pendingUnits (add), ctx.tasks (hook skips), ctx.manifest (hook
|
|
1332
|
+
* status). The mutable manifest/tasks are accessed via ctx.* (not captured
|
|
1333
|
+
* locals) so that async retry callbacks observe the caller's re-synced
|
|
1334
|
+
* values, matching the original closure semantics.
|
|
1335
|
+
*
|
|
1336
|
+
* @param ctx The scheduler context.
|
|
1337
|
+
* @param decision The dispatch decision from selectDispatchBatch.
|
|
1338
|
+
*/
|
|
1339
|
+
async function dispatchBatch(ctx: SchedulerContext, decision: DispatchBatchDecision): Promise<void> {
|
|
1340
|
+
const { batch: readyBatch, concurrency, snapshot, approvalPending, coalesceEnabled } = decision;
|
|
1341
|
+
// Immutable context fields captured once; manifest/tasks are accessed via
|
|
1342
|
+
// ctx.* because they may be re-synced by the caller between dispatch and
|
|
1343
|
+
// promise resolution (retry callbacks fire asynchronously).
|
|
1344
|
+
const { workflow, input, runtimeKind, runController } = ctx;
|
|
1345
|
+
|
|
1346
|
+
// 2.2 caller migration: batch progress is high-frequency informational (M7 wire).
|
|
1347
|
+
void appendEventBuffered(ctx.manifest.eventsPath, {
|
|
1348
|
+
type: "task.progress",
|
|
1349
|
+
runId: ctx.manifest.runId,
|
|
1350
|
+
message: `Starting ready batch with ${readyBatch.length} task(s).`,
|
|
1351
|
+
data: {
|
|
1352
|
+
taskIds: readyBatch.map((task) => task.id),
|
|
1353
|
+
readyCount: snapshot.ready.length,
|
|
1354
|
+
blockedCount: snapshot.blocked.length,
|
|
1355
|
+
runningCount: snapshot.running.length,
|
|
1356
|
+
doneCount: snapshot.done.length,
|
|
1357
|
+
selectedCount: readyBatch.length,
|
|
1358
|
+
maxConcurrent: concurrency.maxConcurrent,
|
|
1359
|
+
defaultConcurrency: concurrency.defaultConcurrency,
|
|
1360
|
+
concurrencyReason: approvalPending ? `${concurrency.reason};plan-approval-read-only` : concurrency.reason,
|
|
1361
|
+
},
|
|
1362
|
+
});
|
|
1363
|
+
// Execute before_task_start hooks for the batch
|
|
1364
|
+
for (const task of readyBatch) {
|
|
1365
|
+
const taskReport = await executeHook("before_task_start", {
|
|
1366
|
+
runId: ctx.manifest.runId,
|
|
1367
|
+
taskId: task.id,
|
|
1368
|
+
cwd: ctx.manifest.cwd,
|
|
1369
|
+
});
|
|
1370
|
+
appendHookEvent(ctx.manifest, taskReport);
|
|
1371
|
+
if (taskReport.outcome === "block") {
|
|
1372
|
+
ctx.tasks = ctx.tasks.map((t) =>
|
|
1373
|
+
t.id === task.id
|
|
1374
|
+
? {
|
|
1375
|
+
...t,
|
|
1376
|
+
status: "skipped" as const,
|
|
1377
|
+
error: taskReport.reason ?? "before_task_start hook blocked execution.",
|
|
1378
|
+
}
|
|
1379
|
+
: t,
|
|
1380
|
+
);
|
|
1381
|
+
ctx.manifest = updateRunStatus(ctx.manifest, ctx.manifest.status, `Task '${task.id}' blocked by hook.`);
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
const batchTasks = readyBatch.filter((task) => ctx.tasks.find((t) => t.id === task.id && t.status !== "skipped"));
|
|
1385
|
+
if (batchTasks.length > 1) {
|
|
1386
|
+
await appendEventAsync(ctx.manifest.eventsPath, {
|
|
1387
|
+
type: "task.parallel_start",
|
|
1388
|
+
runId: ctx.manifest.runId,
|
|
1389
|
+
message: `Launching ${batchTasks.length} tasks in PARALLEL (concurrency=${concurrency.selectedCount}): ${batchTasks.map((t) => `${t.role}(${t.id})`).join(", ")}`,
|
|
1390
|
+
data: {
|
|
1391
|
+
taskIds: batchTasks.map((t) => t.id),
|
|
1392
|
+
roles: batchTasks.map((t) => t.role),
|
|
1393
|
+
concurrency: concurrency.selectedCount,
|
|
1394
|
+
},
|
|
1395
|
+
});
|
|
1396
|
+
}
|
|
1397
|
+
|
|
1398
|
+
// M6 real dispatch: when coalesceMicroTasks is enabled, batch the
|
|
1399
|
+
// ready tasks into dispatch units. Multi-task groups are dispatched
|
|
1400
|
+
// as one worker (single cold-start) instead of N. Singletons fall
|
|
1401
|
+
// through to per-task dispatch.
|
|
1402
|
+
const coalescedGroups = planCoalescedGroups(
|
|
1403
|
+
batchTasks.map((t) => t.id),
|
|
1404
|
+
ctx.tasks,
|
|
1405
|
+
workflow,
|
|
1406
|
+
coalesceEnabled,
|
|
1407
|
+
);
|
|
1408
|
+
const dispatchUnits = buildDispatchUnits(
|
|
1409
|
+
batchTasks.map((t) => t.id),
|
|
1410
|
+
coalescedGroups,
|
|
1411
|
+
);
|
|
1412
|
+
|
|
1413
|
+
// NEW-M1: Pre-warm stable prefix cache for one representative task
|
|
1414
|
+
// per unique cwd. Parallel siblings with the same cwd/step reuse
|
|
1415
|
+
// the cached workspace tree, file retrieval, and knowledge fragment
|
|
1416
|
+
// instead of recomputing them independently (~200-800ms per batch).
|
|
1417
|
+
if (batchTasks.length > 1) {
|
|
1418
|
+
const seenCwds = new Set<string>();
|
|
1419
|
+
await Promise.all(
|
|
1420
|
+
batchTasks
|
|
1421
|
+
.filter((task) => {
|
|
1422
|
+
if (seenCwds.has(task.cwd)) return false;
|
|
1423
|
+
seenCwds.add(task.cwd);
|
|
1424
|
+
return true;
|
|
1425
|
+
})
|
|
1426
|
+
.map((task) => {
|
|
1427
|
+
const step = findStep(workflow, task);
|
|
1428
|
+
return computeStablePrefixComponents(ctx.manifest, step, task);
|
|
1429
|
+
}),
|
|
1430
|
+
);
|
|
1431
|
+
}
|
|
1432
|
+
|
|
1433
|
+
// ── OPT-01 streaming dispatch: dispatch each unit into ctx.pendingUnits
|
|
1434
|
+
// instead of awaiting the entire batch via mapConcurrent. Each unit's
|
|
1435
|
+
// promise is stored so we can Promise.race on the next iteration. ──
|
|
1436
|
+
const dispatchUnit = async (unit: DispatchUnit): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> => {
|
|
1437
|
+
// M6 real dispatch path: single worker for N tasks.
|
|
1438
|
+
if (unit.kind === "group") {
|
|
1439
|
+
const groupTasks = unit.group.tasks;
|
|
1440
|
+
const firstTask = groupTasks[0]!;
|
|
1441
|
+
const step = findStep(workflow, firstTask);
|
|
1442
|
+
const agent = findAgent(input.agents, firstTask);
|
|
1443
|
+
const teamRole = input.team.roles.find((role) => role.name === firstTask.role);
|
|
1444
|
+
const perTaskRuntime = resolveTaskRuntimeKind(runtimeKind, firstTask.role, input.runtimeConfig?.isolationPolicy);
|
|
1445
|
+
return runCoalescedTaskGroup({
|
|
1446
|
+
manifest: ctx.manifest,
|
|
1447
|
+
tasks: ctx.tasks,
|
|
1448
|
+
groupTasks,
|
|
1449
|
+
step,
|
|
1450
|
+
agent,
|
|
1451
|
+
signal: runController.signal,
|
|
1452
|
+
executeWorkers: input.executeWorkers,
|
|
1453
|
+
runtimeKind,
|
|
1454
|
+
workspaceId: input.workspaceId,
|
|
1455
|
+
onJsonEvent: input.onJsonEvent,
|
|
1456
|
+
teamRole,
|
|
1457
|
+
perTaskRuntime,
|
|
1458
|
+
});
|
|
1459
|
+
}
|
|
1460
|
+
// Singleton path: original per-task dispatch.
|
|
1461
|
+
const task = batchTasks.find((t) => t.id === unit.taskId)!;
|
|
1462
|
+
const step = findStep(workflow, task);
|
|
1463
|
+
const agent = findAgent(input.agents, task);
|
|
1464
|
+
const teamRole = input.team.roles.find((role) => role.name === task.role);
|
|
1465
|
+
const perTaskRuntime = resolveTaskRuntimeKind(runtimeKind, task.role, input.runtimeConfig?.isolationPolicy);
|
|
1466
|
+
// CORE-3: compute retry policy + spawn budget ONCE per dispatch unit.
|
|
1467
|
+
// The spawnBudget object is shared (by reference) across every
|
|
1468
|
+
// runTeamTask call within executeWithRetry via baseInput spread,
|
|
1469
|
+
// so the counter accumulates across retry attempts × model fallbacks.
|
|
1470
|
+
const policy = retryPolicyFromConfig(input.reliability);
|
|
1471
|
+
const spawnBudget: SpawnBudget = { count: 0, max: policy.maxTotalSpawns ?? 0 };
|
|
1472
|
+
const baseInput = {
|
|
1473
|
+
manifest: ctx.manifest,
|
|
1474
|
+
tasks: ctx.tasks,
|
|
1475
|
+
task,
|
|
1476
|
+
step,
|
|
1477
|
+
agent,
|
|
1478
|
+
signal: runController.signal,
|
|
1479
|
+
executeWorkers: input.executeWorkers,
|
|
1480
|
+
runtimeKind: runtimeKind,
|
|
1481
|
+
taskRuntimeOverride: perTaskRuntime !== runtimeKind ? perTaskRuntime : undefined,
|
|
1482
|
+
runtimeConfig: input.runtimeConfig,
|
|
1483
|
+
parentContext: input.parentContext,
|
|
1484
|
+
parentModel: input.parentModel,
|
|
1485
|
+
modelRegistry: input.modelRegistry,
|
|
1486
|
+
modelOverride: input.modelOverride,
|
|
1487
|
+
teamRoleModel: teamRole?.model,
|
|
1488
|
+
teamRoleSkills: teamRole?.skills,
|
|
1489
|
+
skillOverride: input.skillOverride,
|
|
1490
|
+
limits: input.limits,
|
|
1491
|
+
onJsonEvent: input.onJsonEvent,
|
|
1492
|
+
workspaceId: input.workspaceId,
|
|
1493
|
+
spawnBudget,
|
|
1494
|
+
};
|
|
1495
|
+
// #1 (assessment): autoRetry now defaults ON (opt-out via reliability.autoRetry=false).
|
|
1496
|
+
// The dominant v0.9.13 failure was ChildTimeout ("worker became unresponsive") with
|
|
1497
|
+
// ZERO retries because this gate was opt-in. isRetryable() defaults to true when
|
|
1498
|
+
// retryableErrors is empty, so transient hangs now retry up to maxAttempts (3) with
|
|
1499
|
+
// exponential backoff. Set reliability.autoRetry=false to restore old single-shot behavior.
|
|
1500
|
+
if (!shouldUseRetry(input.reliability))
|
|
1501
|
+
return withCorrelation(childCorrelation(ctx.manifest.runId, task.id), () => runTeamTask(baseInput));
|
|
1502
|
+
let lastFailed: { manifest: TeamRunManifest; tasks: TeamTaskState[] } | undefined;
|
|
1503
|
+
let lastAttemptId: string | undefined;
|
|
1504
|
+
const attemptsSoFar: TaskAttemptState[] = [...(task.attempts ?? [])];
|
|
1505
|
+
try {
|
|
1506
|
+
return await executeWithRetry(
|
|
1507
|
+
async (attempt, info) => {
|
|
1508
|
+
const startedAt = new Date().toISOString();
|
|
1509
|
+
const inFlightAttempts: TaskAttemptState[] = [...attemptsSoFar, { attemptId: info.attemptId, startedAt }];
|
|
1510
|
+
input.metricRegistry?.counter("crew.task.retry_attempt_total", "Retry attempts by run and task").inc({
|
|
1511
|
+
runId: ctx.manifest.runId,
|
|
1512
|
+
taskId: task.id,
|
|
1513
|
+
});
|
|
1514
|
+
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
1515
|
+
const fresh = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
|
|
1516
|
+
const freshManifest = fresh?.manifest ?? ctx.manifest;
|
|
1517
|
+
const freshTasks = fresh?.tasks ?? ctx.tasks;
|
|
1518
|
+
const freshTask = freshTasks.find((item) => item.id === task.id) ?? task;
|
|
1519
|
+
if (freshTask.status !== "queued" && freshTask.status !== "running")
|
|
1520
|
+
return {
|
|
1521
|
+
manifest: freshManifest,
|
|
1522
|
+
tasks: freshTasks,
|
|
1523
|
+
};
|
|
1524
|
+
const taskWithAttempt: TeamTaskState = {
|
|
1525
|
+
...freshTask,
|
|
1526
|
+
attempts: inFlightAttempts,
|
|
1527
|
+
};
|
|
1528
|
+
const result = await withCorrelation(childCorrelation(freshManifest.runId, task.id), () =>
|
|
1529
|
+
runTeamTask({
|
|
1530
|
+
...baseInput,
|
|
1531
|
+
manifest: freshManifest,
|
|
1532
|
+
tasks: freshTasks,
|
|
1533
|
+
task: taskWithAttempt,
|
|
1534
|
+
}),
|
|
1535
|
+
);
|
|
1536
|
+
const failed = failedTaskFrom(result, task.id);
|
|
1537
|
+
const endedAt = new Date().toISOString();
|
|
1538
|
+
const finishedAttempt: TaskAttemptState = {
|
|
1539
|
+
attemptId: info.attemptId,
|
|
1540
|
+
startedAt,
|
|
1541
|
+
endedAt,
|
|
1542
|
+
...(failed?.error ? { error: failed.error } : {}),
|
|
1543
|
+
};
|
|
1544
|
+
attemptsSoFar.push(finishedAttempt);
|
|
1545
|
+
const withAttempt = result.tasks.map((item) =>
|
|
1546
|
+
item.id === task.id ? { ...item, attempts: [...attemptsSoFar] } : item,
|
|
1547
|
+
);
|
|
1548
|
+
const enriched = {
|
|
1549
|
+
manifest: result.manifest,
|
|
1550
|
+
tasks: withAttempt,
|
|
1551
|
+
};
|
|
1552
|
+
if (failed) {
|
|
1553
|
+
lastFailed = enriched;
|
|
1554
|
+
throw new CrewError(ErrorCode.TaskNotFound, failed.error ?? `Task ${task.id} failed.`).withContext(
|
|
1555
|
+
`retry evaluation (run=${ctx.manifest.runId})`,
|
|
1556
|
+
);
|
|
1557
|
+
}
|
|
1558
|
+
input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe(
|
|
1559
|
+
{
|
|
1560
|
+
runId: ctx.manifest.runId,
|
|
1561
|
+
team: input.team.name,
|
|
1562
|
+
},
|
|
1563
|
+
Math.max(0, attempt - 1),
|
|
1564
|
+
);
|
|
1565
|
+
return enriched;
|
|
1566
|
+
},
|
|
1567
|
+
policy,
|
|
1568
|
+
{
|
|
1569
|
+
signal: runController.signal,
|
|
1570
|
+
attemptId: (attempt) => `${ctx.manifest.runId}:${task.id}:attempt-${attempt}`,
|
|
1571
|
+
onAttemptFailed: (attempt, error, delayMs, info) => {
|
|
1572
|
+
lastAttemptId = info.attemptId;
|
|
1573
|
+
appendEventAsync(ctx.manifest.eventsPath, {
|
|
1574
|
+
type: "crew.task.retry_attempt",
|
|
1575
|
+
runId: ctx.manifest.runId,
|
|
1576
|
+
taskId: task.id,
|
|
1577
|
+
message: error.message,
|
|
1578
|
+
data: {
|
|
1579
|
+
attempt,
|
|
1580
|
+
attemptId: info.attemptId,
|
|
1581
|
+
delayMs,
|
|
1582
|
+
},
|
|
1583
|
+
metadata: { attemptId: info.attemptId },
|
|
1584
|
+
}).catch((error) => logInternalError("team-runner.retry-attempt", error, `taskId=${task.id}`));
|
|
1585
|
+
input.metricRegistry?.histogram("crew.task.retry_delay_ms", "Retry backoff delay, milliseconds").observe(
|
|
1586
|
+
{
|
|
1587
|
+
runId: ctx.manifest.runId,
|
|
1588
|
+
taskId: task.id,
|
|
1589
|
+
},
|
|
1590
|
+
delayMs,
|
|
1591
|
+
);
|
|
1592
|
+
},
|
|
1593
|
+
onRetryGivenUp: (attempts, error, info) => {
|
|
1594
|
+
lastAttemptId = info.attemptId;
|
|
1595
|
+
appendDeadletter(ctx.manifest, {
|
|
1596
|
+
runId: ctx.manifest.runId,
|
|
1597
|
+
taskId: task.id,
|
|
1598
|
+
reason: "max-retries",
|
|
1599
|
+
attempts,
|
|
1600
|
+
attemptId: info.attemptId,
|
|
1601
|
+
lastError: error.message,
|
|
1602
|
+
timestamp: new Date().toISOString(),
|
|
1603
|
+
});
|
|
1604
|
+
input.metricRegistry
|
|
1605
|
+
?.counter("crew.task.deadletter_total", "Deadletter triggers by reason")
|
|
1606
|
+
.inc({ reason: "max-retries" });
|
|
1607
|
+
input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe(
|
|
1608
|
+
{
|
|
1609
|
+
runId: ctx.manifest.runId,
|
|
1610
|
+
team: input.team.name,
|
|
1611
|
+
},
|
|
1612
|
+
Math.max(0, attempts - 1),
|
|
1613
|
+
);
|
|
1614
|
+
},
|
|
1615
|
+
},
|
|
1616
|
+
);
|
|
1617
|
+
} catch (retryError) {
|
|
1618
|
+
if (retryError instanceof CrewCancellationError || input.signal?.aborted) {
|
|
1619
|
+
const reason = retryError instanceof CrewCancellationError ? retryError.reason : cancellationReasonFromSignal(input.signal);
|
|
1620
|
+
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
1621
|
+
const fresh = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
|
|
1622
|
+
const freshManifest = fresh?.manifest ?? ctx.manifest;
|
|
1623
|
+
const freshTasks = fresh?.tasks ?? ctx.tasks;
|
|
1624
|
+
const cancelledTasks = cancelNonTerminalTasks(
|
|
1625
|
+
freshTasks,
|
|
1626
|
+
"cancelled",
|
|
1627
|
+
`${reason.message} (${reason.code})`,
|
|
1628
|
+
(item) => item.id === task.id && (item.status === "queued" || item.status === "running"),
|
|
1629
|
+
);
|
|
1630
|
+
appendEventAsync(freshManifest.eventsPath, {
|
|
1631
|
+
type: "task.cancelled",
|
|
1632
|
+
runId: freshManifest.runId,
|
|
1633
|
+
taskId: task.id,
|
|
1634
|
+
message: reason.message,
|
|
1635
|
+
data: { reason, phase: "retry" },
|
|
1636
|
+
metadata: lastAttemptId ? { attemptId: lastAttemptId } : undefined,
|
|
1637
|
+
}).catch((error) => logInternalError("team-runner.cancelled", error, `taskId=${task.id}`));
|
|
1638
|
+
return {
|
|
1639
|
+
manifest: updateRunStatus(freshManifest, "cancelled", reason.message),
|
|
1640
|
+
tasks: cancelledTasks,
|
|
1641
|
+
};
|
|
1642
|
+
}
|
|
1643
|
+
if (lastFailed) return lastFailed;
|
|
1644
|
+
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
1645
|
+
const fresh = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
|
|
1646
|
+
const freshManifest = fresh?.manifest ?? ctx.manifest;
|
|
1647
|
+
const freshTasks = fresh?.tasks ?? ctx.tasks;
|
|
1648
|
+
const freshTask = freshTasks.find((item) => item.id === task.id) ?? task;
|
|
1649
|
+
if (freshTask.status !== "queued" && freshTask.status !== "running") return { manifest: freshManifest, tasks: freshTasks };
|
|
1650
|
+
return withCorrelation(childCorrelation(freshManifest.runId, task.id), () =>
|
|
1651
|
+
runTeamTask({
|
|
1652
|
+
...baseInput,
|
|
1653
|
+
manifest: freshManifest,
|
|
1654
|
+
tasks: freshTasks,
|
|
1655
|
+
task: freshTask,
|
|
1656
|
+
}),
|
|
1657
|
+
);
|
|
1658
|
+
}
|
|
1659
|
+
};
|
|
1660
|
+
// ── OPT-01 streaming dispatch: dispatch units into ctx.pendingUnits ──
|
|
1661
|
+
for (const unit of dispatchUnits) {
|
|
1662
|
+
const unitKey = unit.kind === "singleton" ? unit.taskId : unit.group.id;
|
|
1663
|
+
const unitTaskIds = unit.kind === "singleton" ? [unit.taskId] : unit.group.tasks.map((t) => t.id);
|
|
1664
|
+
ctx.pendingUnits.set(unitKey, {
|
|
1665
|
+
taskIds: unitTaskIds,
|
|
1666
|
+
promise: dispatchUnit(unit),
|
|
1667
|
+
});
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
/**
|
|
1672
|
+
* CORE-4 extraction 5: wait for one in-flight dispatch unit to settle and
|
|
1673
|
+
* merge its result into the run state.
|
|
1674
|
+
*
|
|
1675
|
+
* Awaits Promise.race on ctx.pendingUnits; the first settled unit is merged
|
|
1676
|
+
* into ctx.manifest/tasks under the run lock (flushPendingAtomicWrites +
|
|
1677
|
+
* loadRunManifestById + mergeTaskUpdatesPreservingTerminal + save). The settled
|
|
1678
|
+
* unit is then deleted from ctx.pendingUnits, and the merge outcome (taskIds
|
|
1679
|
+
* + result object) is recorded on ctx.settledMerge for the post-merge inline
|
|
1680
|
+
* logic (cancel-during-exec check + batch summary).
|
|
1681
|
+
*
|
|
1682
|
+
* Returns null to continue to the phase/budget check. A `{ kind: "return" }`
|
|
1683
|
+
* decision is reserved for future run-complete/failure detection during merge.
|
|
1684
|
+
*
|
|
1685
|
+
* Reads ctx.pendingUnits/manifest/tasks. Mutates ctx.pendingUnits (delete),
|
|
1686
|
+
* ctx.manifest/tasks, ctx.settledMerge.
|
|
1687
|
+
*
|
|
1688
|
+
* @param ctx The scheduler context.
|
|
1689
|
+
*/
|
|
1690
|
+
async function mergeUnitResult(ctx: SchedulerContext): Promise<SchedulerDecision | null> {
|
|
1691
|
+
// Wait for ONE in-flight unit to complete. Promise.race returns as soon
|
|
1692
|
+
// as the first wrapper resolves — others remain pending in ctx.pendingUnits
|
|
1693
|
+
// and are re-raced on the next iteration.
|
|
1694
|
+
const settled = await Promise.race(
|
|
1695
|
+
[...ctx.pendingUnits.entries()].map(async ([key, pending]) => {
|
|
1696
|
+
try {
|
|
1697
|
+
const result = await pending.promise;
|
|
1698
|
+
return {
|
|
1699
|
+
unitKey: key,
|
|
1700
|
+
result: result as { manifest: TeamRunManifest; tasks: TeamTaskState[] } | undefined,
|
|
1701
|
+
error: undefined as Error | undefined,
|
|
1702
|
+
};
|
|
1703
|
+
} catch (error) {
|
|
1704
|
+
return { unitKey: key, result: undefined, error: error instanceof Error ? error : new Error(String(error)) };
|
|
1705
|
+
}
|
|
1706
|
+
}),
|
|
1707
|
+
);
|
|
1708
|
+
const completedUnit = ctx.pendingUnits.get(settled.unitKey)!;
|
|
1709
|
+
ctx.pendingUnits.delete(settled.unitKey);
|
|
1710
|
+
|
|
1711
|
+
// Build the single result to merge. On rejection, synthesize a failed
|
|
1712
|
+
// result so the run continues (mirrors the old validResults guard).
|
|
1713
|
+
const resultToMerge: { manifest: TeamRunManifest; tasks: TeamTaskState[] } = settled.result ?? {
|
|
1714
|
+
manifest: ctx.manifest,
|
|
1715
|
+
tasks: cancelNonTerminalTasks(ctx.tasks, "failed", settled.error!.message, (t) => completedUnit.taskIds.includes(t.id)),
|
|
1716
|
+
};
|
|
1717
|
+
const validResults = [resultToMerge];
|
|
1718
|
+
// Reconstruct manifest from the last worker's snapshot. The .artifacts field
|
|
1719
|
+
// is re-merged from both the team-runner's in-memory state and all workers'
|
|
1720
|
+
// snapshots, so artifact writes by task-runner (which individually save manifest
|
|
1721
|
+
// after writing artifacts) are safely persisted. The in-memory manifest is only
|
|
1722
|
+
// used for the next batch iteration's orchestration — actual persistence is safe.
|
|
1723
|
+
// Use updateRunStatus to recompute manifest status from merged tasks rather than
|
|
1724
|
+
// relying on the last result's manifest (which is arbitrary due to mapConcurrent
|
|
1725
|
+
// returning results in arbitrary order).
|
|
1726
|
+
// Use the in-memory manifest as base (not the last-completing worker's snapshot).
|
|
1727
|
+
// Recompute status from merged tasks so the manifest reflects actual task state,
|
|
1728
|
+
// not the arbitrary order in which mapConcurrent returned results.
|
|
1729
|
+
// Read committed manifest from disk inside the lock so artifact merge is based
|
|
1730
|
+
// on committed state, not in-memory state that may differ from disk.
|
|
1731
|
+
const mergeResult = await withRunLock(ctx.manifest, async () => {
|
|
1732
|
+
// NEW-D1: flush any pending coalesced atomic writes before reading from
|
|
1733
|
+
// disk. Without this, a worker's async manifest save (coalesced by
|
|
1734
|
+
// atomic-write) may not be committed yet, causing a lost-update on the
|
|
1735
|
+
// merge read. flushPendingAtomicWrites forces all queued writes to disk.
|
|
1736
|
+
flushPendingAtomicWrites();
|
|
1737
|
+
const disk = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
|
|
1738
|
+
const diskManifest = disk?.manifest ?? ctx.manifest;
|
|
1739
|
+
const diskArtifacts = diskManifest.artifacts;
|
|
1740
|
+
const reconciledArtifacts = mergeArtifacts([...diskArtifacts, ...validResults.map((item) => item.manifest.artifacts)].flat());
|
|
1741
|
+
const resultManifest = updateRunStatus(
|
|
1742
|
+
{ ...diskManifest, artifacts: reconciledArtifacts },
|
|
1743
|
+
"running",
|
|
1744
|
+
"Merged task updates from parallel batch.",
|
|
1745
|
+
);
|
|
1746
|
+
// CANCEL-1: use the freshly-loaded disk tasks as the merge base instead
|
|
1747
|
+
// of the in-memory `tasks` closure variable. The in-memory tasks reflect
|
|
1748
|
+
// only team-runner's view; an external cancel (handleCancel, background
|
|
1749
|
+
// race with SIGTERM arriving after cancel wrote but before merge ran)
|
|
1750
|
+
// writes 'cancelled' to disk.tasks — using disk.tasks as base preserves
|
|
1751
|
+
// that cancellation through the merge instead of overwriting it with the
|
|
1752
|
+
// stale in-memory view. disk was loaded inside this lock, so it reflects
|
|
1753
|
+
// the freshest committed state.
|
|
1754
|
+
const resultTasks = mergeTaskUpdatesPreservingTerminal(disk?.tasks ?? ctx.tasks, validResults);
|
|
1755
|
+
await saveRunManifestAsync(resultManifest);
|
|
1756
|
+
await saveRunTasksAsync(resultManifest, resultTasks);
|
|
1757
|
+
return { resultManifest, resultTasks };
|
|
1758
|
+
});
|
|
1759
|
+
ctx.manifest = mergeResult.resultManifest;
|
|
1760
|
+
ctx.tasks = mergeResult.resultTasks;
|
|
1761
|
+
ctx.settledMerge = { taskIds: completedUnit.taskIds, result: resultToMerge };
|
|
1762
|
+
return null;
|
|
1763
|
+
}
|
|
1764
|
+
|
|
1765
|
+
/**
|
|
1766
|
+
* CORE-4 extraction 6: advance workflow phases whose tasks are all in
|
|
1767
|
+
* terminal state.
|
|
1768
|
+
*
|
|
1769
|
+
* Iterates phases starting at `ctx.wfMachine.currentPhaseIndex`; for each phase
|
|
1770
|
+
* whose tasks are all terminal, determines the transition status (failed if
|
|
1771
|
+
* any task failed/cancelled, else completed), applies the phase transition,
|
|
1772
|
+
* emits `workflow.phase_completed`/`workflow.phase_failed`/
|
|
1773
|
+
* `workflow.phase_guard_blocked` events, and advances `currentPhaseIndex`.
|
|
1774
|
+
*
|
|
1775
|
+
* Reads `ctx.tasks`, `ctx.manifest` (read-only). Mutates `ctx.wfMachine`
|
|
1776
|
+
* in-place (phase status + currentPhaseIndex). The caller syncs the local
|
|
1777
|
+
* `wfMachine` from ctx after the call.
|
|
1778
|
+
*
|
|
1779
|
+
* @param ctx The scheduler context.
|
|
1780
|
+
*/
|
|
1781
|
+
async function advanceWorkflowPhases(ctx: SchedulerContext): Promise<void> {
|
|
1782
|
+
let wfMachine = ctx.wfMachine;
|
|
1783
|
+
const tasks = ctx.tasks;
|
|
1784
|
+
const manifest = ctx.manifest;
|
|
1785
|
+
// Advance workflow phases whose tasks are all in terminal state
|
|
1786
|
+
const terminalStatuses = new Set(["completed", "failed", "skipped", "cancelled", "needs_attention"]);
|
|
1787
|
+
const phaseTaskMap = new Map<string, string[]>();
|
|
1788
|
+
for (const task of tasks) {
|
|
1789
|
+
if (!task.stepId) continue;
|
|
1790
|
+
const existing = phaseTaskMap.get(task.stepId) ?? [];
|
|
1791
|
+
existing.push(task.id);
|
|
1792
|
+
phaseTaskMap.set(task.stepId, existing);
|
|
1793
|
+
}
|
|
1794
|
+
for (let pi = wfMachine.currentPhaseIndex; pi < wfMachine.phases.length; pi++) {
|
|
1795
|
+
const phase = wfMachine.phases[pi]!;
|
|
1796
|
+
const phaseTaskIds = phaseTaskMap.get(phase.name) ?? [];
|
|
1797
|
+
if (phaseTaskIds.length === 0) continue;
|
|
1798
|
+
const allTerminal = phaseTaskIds.every((taskId) => {
|
|
1799
|
+
const task = tasks.find((t) => t.id === taskId);
|
|
1800
|
+
return task ? terminalStatuses.has(task.status) : false;
|
|
1801
|
+
});
|
|
1802
|
+
if (!allTerminal) break;
|
|
1803
|
+
if (phase.status !== "completed" && phase.status !== "failed" && phase.status !== "skipped") {
|
|
1804
|
+
const completedArtifacts = manifest.artifacts.filter((a) => a.kind === "result" || a.kind === "summary").map((a) => a.path);
|
|
1805
|
+
const previousPhaseStatus = pi > 0 ? (wfMachine.phases[pi - 1]?.status ?? "pending") : "completed";
|
|
1806
|
+
const wfContext: PhaseGuardContext = {
|
|
1807
|
+
completedArtifacts,
|
|
1808
|
+
previousPhaseStatus,
|
|
1809
|
+
taskResults: tasks
|
|
1810
|
+
.filter((t) => t.status === "completed" || t.status === "needs_attention")
|
|
1811
|
+
.map((t) => ({
|
|
1812
|
+
taskId: t.id,
|
|
1813
|
+
status: t.status,
|
|
1814
|
+
outputPath: t.resultArtifact?.path,
|
|
1815
|
+
})),
|
|
1816
|
+
};
|
|
1817
|
+
// Determine phase transition status based on individual task outcomes
|
|
1818
|
+
const phaseTasks = phaseTaskIds
|
|
1819
|
+
.map((taskId) => tasks.find((t) => t.id === taskId))
|
|
1820
|
+
.filter((t): t is NonNullable<typeof t> => t !== undefined);
|
|
1821
|
+
const hasFailedOrCancelled = phaseTasks.some((t) => t.status === "failed" || t.status === "cancelled");
|
|
1822
|
+
const phaseStatus = hasFailedOrCancelled ? "failed" : "completed";
|
|
1823
|
+
const transition = transitionPhase(wfMachine, pi, phaseStatus, wfContext);
|
|
1824
|
+
wfMachine = transition.machine;
|
|
1825
|
+
if (transition.guardResult && !transition.guardResult.allowed) {
|
|
1826
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1827
|
+
type: "workflow.phase_guard_blocked",
|
|
1828
|
+
runId: manifest.runId,
|
|
1829
|
+
message: `Workflow phase '${phase.name}' guard blocked: ${transition.guardResult.reason ?? "unknown"}`,
|
|
1830
|
+
data: {
|
|
1831
|
+
phaseIndex: pi,
|
|
1832
|
+
phaseName: phase.name,
|
|
1833
|
+
reason: transition.guardResult.reason,
|
|
1834
|
+
},
|
|
1835
|
+
});
|
|
1836
|
+
break;
|
|
1837
|
+
}
|
|
1838
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1839
|
+
type: phaseStatus === "failed" ? "workflow.phase_failed" : "workflow.phase_completed",
|
|
1840
|
+
runId: manifest.runId,
|
|
1841
|
+
message: `Workflow phase '${phase.name}' ${phaseStatus}.`,
|
|
1842
|
+
data: { phaseIndex: pi, phaseStatus },
|
|
1843
|
+
});
|
|
1844
|
+
}
|
|
1845
|
+
wfMachine = { ...wfMachine, currentPhaseIndex: pi + 1 };
|
|
1846
|
+
}
|
|
1847
|
+
ctx.wfMachine = wfMachine;
|
|
1848
|
+
}
|
|
1849
|
+
|
|
1850
|
+
/**
|
|
1851
|
+
* CORE-4 extraction 7: enforce per-task run budget after each batch merge.
|
|
1852
|
+
*
|
|
1853
|
+
* When `input.budgetTotal` is set (and not unlimited), checks cumulative usage
|
|
1854
|
+
* against warn/abort thresholds and a fair-share heuristic. On abort, marks all
|
|
1855
|
+
* non-terminal tasks blocked, persists the run as failed, and returns a
|
|
1856
|
+
* `{ kind: "return" }` decision so the caller short-circuits the loop.
|
|
1857
|
+
* Otherwise emits `run.budget_warning` / `task.budget_fair_share` events and
|
|
1858
|
+
* returns null (continue).
|
|
1859
|
+
*
|
|
1860
|
+
* Reads `ctx.input` (budget config). Mutates `ctx.tasks` / `ctx.manifest` only
|
|
1861
|
+
* in the abort path. The caller syncs these locals back after the call.
|
|
1862
|
+
*
|
|
1863
|
+
* @param ctx The scheduler context.
|
|
1864
|
+
* @returns `{ kind: "return", result }` on budget abort; `null` otherwise.
|
|
1865
|
+
*/
|
|
1866
|
+
async function enforceRunBudget(ctx: SchedulerContext): Promise<SchedulerDecision | null> {
|
|
1867
|
+
const input = ctx.input;
|
|
1868
|
+
let tasks = ctx.tasks;
|
|
1869
|
+
let manifest = ctx.manifest;
|
|
1870
|
+
// Per-task budget enforcement: check cumulative usage after each batch merge.
|
|
1871
|
+
// This prevents a single task from consuming 100% of the budget before
|
|
1872
|
+
// abort triggers (the goal-loop only checks at turn boundaries).
|
|
1873
|
+
if (input.budgetTotal !== undefined && input.budgetTotal > 0 && input.budgetUnlimited !== true) {
|
|
1874
|
+
const warnThreshold = input.budgetWarning ?? 0.8;
|
|
1875
|
+
const abortThreshold = input.budgetAbort ?? 0.95;
|
|
1876
|
+
const budgetCheck = checkPerTaskBudget(tasks, input.budgetTotal, warnThreshold, abortThreshold);
|
|
1877
|
+
|
|
1878
|
+
if (budgetCheck.abort) {
|
|
1879
|
+
const message = `Per-task budget abort threshold exceeded: ${formatTokens(budgetCheck.totalUsed)}/${formatTokens(input.budgetTotal)} (${Math.round((budgetCheck.totalUsed / input.budgetTotal) * 100)}%)`;
|
|
1880
|
+
console.warn(`[team-runner] ${message}`);
|
|
1881
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1882
|
+
type: "run.budget_abort",
|
|
1883
|
+
runId: manifest.runId,
|
|
1884
|
+
message,
|
|
1885
|
+
data: {
|
|
1886
|
+
budgetTotal: input.budgetTotal,
|
|
1887
|
+
budgetUsed: budgetCheck.totalUsed,
|
|
1888
|
+
threshold: "abort",
|
|
1889
|
+
},
|
|
1890
|
+
});
|
|
1891
|
+
tasks = markBlocked(tasks, `Budget abort threshold exceeded: ${message}`);
|
|
1892
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
1893
|
+
manifest = updateRunStatus(manifest, "failed", message);
|
|
1894
|
+
ctx.tasks = tasks;
|
|
1895
|
+
ctx.manifest = manifest;
|
|
1896
|
+
return { kind: "return", result: { manifest, tasks } };
|
|
1897
|
+
}
|
|
1898
|
+
|
|
1899
|
+
if (budgetCheck.warning) {
|
|
1900
|
+
const message = `Per-task budget warning threshold crossed: ${formatTokens(budgetCheck.totalUsed)}/${formatTokens(input.budgetTotal)} (${Math.round((budgetCheck.totalUsed / input.budgetTotal) * 100)}%)`;
|
|
1901
|
+
console.warn(`[team-runner] ${message}`);
|
|
1902
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1903
|
+
type: "run.budget_warning",
|
|
1904
|
+
runId: manifest.runId,
|
|
1905
|
+
message,
|
|
1906
|
+
data: {
|
|
1907
|
+
budgetTotal: input.budgetTotal,
|
|
1908
|
+
budgetUsed: budgetCheck.totalUsed,
|
|
1909
|
+
threshold: "warning",
|
|
1910
|
+
},
|
|
1911
|
+
});
|
|
1912
|
+
}
|
|
1913
|
+
|
|
1914
|
+
// Fair-share warning: flag tasks that consumed >50% of remaining budget
|
|
1915
|
+
// without killing them mid-execution.
|
|
1916
|
+
for (const violatorId of budgetCheck.fairShareViolators) {
|
|
1917
|
+
const violator = tasks.find((t) => t.id === violatorId);
|
|
1918
|
+
if (!violator) continue;
|
|
1919
|
+
const taskTotal = (violator.usage?.input ?? 0) + (violator.usage?.output ?? 0) + (violator.usage?.cacheWrite ?? 0);
|
|
1920
|
+
const message = `Task '${violatorId}' consumed ${formatTokens(taskTotal)} (${Math.round((taskTotal / input.budgetTotal) * 100)}% of total budget) — exceeds fair share`;
|
|
1921
|
+
console.warn(`[team-runner.fair-share] ${message}`);
|
|
1922
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1923
|
+
type: "task.budget_fair_share",
|
|
1924
|
+
runId: manifest.runId,
|
|
1925
|
+
taskId: violatorId,
|
|
1926
|
+
message,
|
|
1927
|
+
data: {
|
|
1928
|
+
budgetTotal: input.budgetTotal,
|
|
1929
|
+
taskUsage: taskTotal,
|
|
1930
|
+
},
|
|
1931
|
+
});
|
|
1932
|
+
}
|
|
1933
|
+
}
|
|
1934
|
+
return null;
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
/**
|
|
1938
|
+
* CORE-4 extraction 8: finalize the run after the scheduler loop exits.
|
|
1939
|
+
*
|
|
1940
|
+
* Computes the final run status (failed/blocked/completed) from task states,
|
|
1941
|
+
* policy decisions, and effectiveness evaluation; writes the workflow output
|
|
1942
|
+
* deliverable warning, the `summary.md` artifact, the joint atomic manifest+tasks
|
|
1943
|
+
* save, and a health snapshot; then returns the terminal `{ manifest, tasks }`.
|
|
1944
|
+
*
|
|
1945
|
+
* Reads `ctx.input` (limits/workflow/executeWorkers/runtimeConfig). Mutates
|
|
1946
|
+
* `ctx.manifest` / `ctx.tasks` and writes them back before returning so the
|
|
1947
|
+
* caller stays in sync. This function is the terminal step of
|
|
1948
|
+
* `executeTeamRunCore` — its return value is the run result.
|
|
1949
|
+
*
|
|
1950
|
+
* @param ctx The scheduler context.
|
|
1951
|
+
* @returns The final `{ manifest, tasks }` result for the run.
|
|
1952
|
+
*/
|
|
1953
|
+
async function finalizeRun(ctx: SchedulerContext): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
|
|
1954
|
+
const input = ctx.input;
|
|
1955
|
+
const tasks = ctx.tasks;
|
|
1956
|
+
let manifest = ctx.manifest;
|
|
1957
|
+
const failed = tasks.find((task) => task.status === "failed");
|
|
1958
|
+
const waiting = tasks.find((task) => task.status === "waiting");
|
|
1959
|
+
const running = tasks.find((task) => task.status === "running");
|
|
1960
|
+
manifest = applyPolicy(manifest, tasks, input.limits);
|
|
1961
|
+
|
|
1962
|
+
// S02: Verify workflow-declared output files exist before marking completed
|
|
1963
|
+
if (input.workflow?.steps) {
|
|
1964
|
+
const missingOutputs: string[] = [];
|
|
1965
|
+
for (const step of input.workflow.steps) {
|
|
1966
|
+
if (step.output && typeof step.output === "string") {
|
|
1967
|
+
const outputPath = path.join(manifest.artifactsRoot, step.output);
|
|
1968
|
+
if (!fs.existsSync(outputPath)) {
|
|
1969
|
+
missingOutputs.push(step.output);
|
|
1970
|
+
}
|
|
1971
|
+
}
|
|
1972
|
+
}
|
|
1973
|
+
if (missingOutputs.length > 0) {
|
|
1974
|
+
// Emit warning event — run still completes normally to avoid hanging
|
|
1975
|
+
appendEventFireAndForget(manifest.eventsPath, {
|
|
1976
|
+
type: "run.deliverable_warning",
|
|
1977
|
+
runId: manifest.runId,
|
|
1978
|
+
message: `Missing workflow output files: ${missingOutputs.join(", ")}`,
|
|
1979
|
+
data: { missingFiles: missingOutputs },
|
|
1980
|
+
});
|
|
1981
|
+
}
|
|
1982
|
+
}
|
|
1983
|
+
|
|
1984
|
+
const effectiveness = evaluateRunEffectiveness({
|
|
1985
|
+
manifest,
|
|
1986
|
+
tasks,
|
|
1987
|
+
executeWorkers: input.executeWorkers,
|
|
1988
|
+
runtimeConfig: input.runtimeConfig,
|
|
1989
|
+
});
|
|
1990
|
+
const effectivenessDecision = effectivenessPolicyDecision(effectiveness);
|
|
1991
|
+
if (effectivenessDecision) {
|
|
1992
|
+
manifest = {
|
|
1993
|
+
...manifest,
|
|
1994
|
+
policyDecisions: [...(manifest.policyDecisions ?? []), effectivenessDecision],
|
|
1995
|
+
updatedAt: new Date().toISOString(),
|
|
1996
|
+
};
|
|
1997
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1998
|
+
type: "run.effectiveness",
|
|
1999
|
+
runId: manifest.runId,
|
|
2000
|
+
message: effectivenessDecision.message,
|
|
2001
|
+
data: { effectiveness, policyDecision: effectivenessDecision },
|
|
2002
|
+
});
|
|
2003
|
+
}
|
|
2004
|
+
const blockingDecision = manifest.policyDecisions?.find((item) => item.action === "block" || item.action === "escalate");
|
|
2005
|
+
if (failed) {
|
|
2006
|
+
manifest = updateRunStatus(manifest, "failed", `Failed at task '${failed.id}'.`);
|
|
2007
|
+
} else if (waiting) {
|
|
2008
|
+
manifest = updateRunStatus(manifest, "blocked", `Waiting for response to task '${waiting.id}'.`);
|
|
2009
|
+
} else if (running) {
|
|
2010
|
+
manifest = updateRunStatus(manifest, "blocked", `Task '${running.id}' is still running.`);
|
|
2011
|
+
} else if (effectiveness.severity === "failed") {
|
|
2012
|
+
manifest = updateRunStatus(manifest, "failed", effectivenessDecision?.message ?? "Run effectiveness guard failed.");
|
|
2013
|
+
} else if (effectiveness.severity === "blocked") {
|
|
2014
|
+
manifest = updateRunStatus(manifest, "blocked", effectivenessDecision?.message ?? "Run effectiveness guard blocked completion.");
|
|
2015
|
+
} else if (blockingDecision) {
|
|
2016
|
+
manifest = updateRunStatus(manifest, "blocked", blockingDecision.message);
|
|
2017
|
+
} else if (tasks.some((task) => task.status === "queued")) {
|
|
2018
|
+
// F1 defense-in-depth: the loop exited with queued tasks still pending
|
|
2019
|
+
// (e.g. a hook skipped all ready tasks and downstream tasks never became
|
|
2020
|
+
// runnable). This is NOT a completed run — mark it blocked rather than
|
|
2021
|
+
// false-green "completed".
|
|
2022
|
+
manifest = updateRunStatus(manifest, "blocked", "Run exited with queued tasks still pending.");
|
|
2023
|
+
} else {
|
|
2024
|
+
manifest = updateRunStatus(
|
|
2025
|
+
manifest,
|
|
2026
|
+
"completed",
|
|
2027
|
+
input.executeWorkers ? "Team workflow completed." : "Team workflow scaffold completed without launching child workers.",
|
|
2028
|
+
);
|
|
2029
|
+
}
|
|
2030
|
+
manifest = writeProgress(manifest, tasks, "team-runner", input.executeWorkers, input.runtimeConfig);
|
|
2031
|
+
await saveRunManifestAsync(manifest);
|
|
2032
|
+
const usage = aggregateUsage(tasks);
|
|
2033
|
+
const summaryArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
2034
|
+
kind: "summary",
|
|
2035
|
+
relativePath: "summary.md",
|
|
2036
|
+
producer: "team-runner",
|
|
2037
|
+
content: [
|
|
2038
|
+
`# pi-crew run ${manifest.runId}`,
|
|
2039
|
+
"",
|
|
2040
|
+
`Status: ${manifest.status}`,
|
|
2041
|
+
`Team: ${manifest.team}`,
|
|
2042
|
+
`Workflow: ${manifest.workflow ?? "(none)"}`,
|
|
2043
|
+
`Goal: ${manifest.goal}`,
|
|
2044
|
+
`Usage: ${formatUsage(usage)}`,
|
|
2045
|
+
"",
|
|
2046
|
+
"## Tasks",
|
|
2047
|
+
...tasks.map(formatTaskProgress),
|
|
2048
|
+
"",
|
|
2049
|
+
"## Effectiveness",
|
|
2050
|
+
...runEffectivenessLines(manifest, tasks, input.executeWorkers, input.runtimeConfig),
|
|
2051
|
+
"",
|
|
2052
|
+
"## Policy decisions",
|
|
2053
|
+
...(manifest.policyDecisions?.length ? summarizePolicyDecisions(manifest.policyDecisions) : ["- (none)"]),
|
|
2054
|
+
"",
|
|
2055
|
+
].join("\n"),
|
|
2056
|
+
});
|
|
2057
|
+
// Build the complete manifest BEFORE acquiring the lock so the artifacts array
|
|
2058
|
+
// is already incorporated into the manifest object that will be atomically written.
|
|
2059
|
+
// This prevents crash-between-mutation-and-lock from leaving inconsistent state.
|
|
2060
|
+
const finalManifest = {
|
|
2061
|
+
...manifest,
|
|
2062
|
+
updatedAt: new Date().toISOString(),
|
|
2063
|
+
artifacts: [...manifest.artifacts, summaryArtifact],
|
|
2064
|
+
};
|
|
2065
|
+
// Joint atomic save: wrap manifest + tasks in a single run lock so they are
|
|
2066
|
+
// written together or not at all. Crash between separate saveRunManifestAsync
|
|
2067
|
+
// and saveRunTasksAsync calls could leave manifest/tasks.json out of sync.
|
|
2068
|
+
await withRunLock(finalManifest, async () => {
|
|
2069
|
+
await saveRunManifestAsync(finalManifest);
|
|
2070
|
+
await saveRunTasksAsync(finalManifest, tasks);
|
|
2071
|
+
});
|
|
2072
|
+
manifest = finalManifest;
|
|
2073
|
+
// Save health snapshot on run completion.
|
|
2074
|
+
// BUG A (pts/2 hang investigation 2026-06-16): stateRoot = `<crewRoot>/state/runs/<runId>`,
|
|
2075
|
+
// so the crew root is THREE dirnames up, not two. Two dirnames gave `<crewRoot>/state`
|
|
2076
|
+
// (the state dir), and HealthStore then joined HEALTH_DIR (`.crew/state/health`)
|
|
2077
|
+
// onto it → `<crewRoot>/state/.crew/state/health` — a double-joined BOGUS path.
|
|
2078
|
+
// That wrote health snapshots to a nonexistent subtree (silently breaking the
|
|
2079
|
+
// health feature) AND created junk dirs that the recursive state watcher then
|
|
2080
|
+
// attached extra inotify watches to. Fix: compute the real crew root (3 up)
|
|
2081
|
+
// and make HEALTH_DIR relative to it.
|
|
2082
|
+
const crewRoot = path.dirname(path.dirname(path.dirname(finalManifest.stateRoot)));
|
|
2083
|
+
const healthStore = new HealthStore(crewRoot);
|
|
2084
|
+
healthStore.saveSnapshot({
|
|
2085
|
+
runId: finalManifest.runId,
|
|
2086
|
+
tasks: tasks.map((t) => ({ id: t.id, status: t.status })),
|
|
2087
|
+
createdAt: finalManifest.createdAt,
|
|
2088
|
+
});
|
|
2089
|
+
ctx.manifest = manifest;
|
|
2090
|
+
ctx.tasks = tasks;
|
|
2091
|
+
return { manifest, tasks };
|
|
2092
|
+
}
|
|
2093
|
+
|
|
2094
|
+
async function executeTeamRunCore(
|
|
2095
|
+
input: ExecuteTeamRunInput,
|
|
2096
|
+
manifest: TeamRunManifest,
|
|
2097
|
+
workflow: WorkflowConfig,
|
|
2098
|
+
): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
|
|
2099
|
+
// Execute before_run_start hook (non-blocking by default)
|
|
2100
|
+
const beforeRunReport = await executeHook("before_run_start", {
|
|
2101
|
+
runId: manifest.runId,
|
|
2102
|
+
cwd: manifest.cwd,
|
|
2103
|
+
});
|
|
2104
|
+
appendHookEvent(manifest, beforeRunReport);
|
|
2105
|
+
if (beforeRunReport.outcome === "block") {
|
|
2106
|
+
manifest = updateRunStatus(manifest, "blocked", beforeRunReport.reason ?? "before_run_start hook blocked the run.");
|
|
2107
|
+
return { manifest, tasks: input.tasks };
|
|
2108
|
+
}
|
|
2109
|
+
let tasks = refreshTaskGraphQueues(input.tasks);
|
|
2110
|
+
let queueIndex = buildTaskGraphIndex(tasks);
|
|
2111
|
+
const canInjectAdaptivePlan = workflow.name === "implementation";
|
|
2112
|
+
let adaptivePlanInjected = false;
|
|
2113
|
+
let adaptivePlanMissing = false;
|
|
2114
|
+
const attemptAdaptivePlan = async () => {
|
|
2115
|
+
if (!canInjectAdaptivePlan || adaptivePlanInjected || adaptivePlanMissing) return { injected: false, missing: false };
|
|
2116
|
+
const adaptivePlan = await injectAdaptivePlanIfReady({
|
|
2117
|
+
manifest,
|
|
2118
|
+
tasks,
|
|
2119
|
+
workflow,
|
|
2120
|
+
team: input.team,
|
|
2121
|
+
});
|
|
2122
|
+
adaptivePlanInjected = adaptivePlanInjected || adaptivePlan.injected;
|
|
2123
|
+
adaptivePlanMissing = adaptivePlan.missingPlan;
|
|
2124
|
+
workflow = adaptivePlan.workflow;
|
|
2125
|
+
if (adaptivePlan.injected) tasks = adaptivePlan.tasks;
|
|
2126
|
+
return {
|
|
2127
|
+
injected: adaptivePlan.injected,
|
|
2128
|
+
missing: adaptivePlan.missingPlan,
|
|
2129
|
+
};
|
|
2130
|
+
};
|
|
2131
|
+
const initialAdaptive = await attemptAdaptivePlan();
|
|
2132
|
+
if (initialAdaptive.missing) {
|
|
2133
|
+
tasks = markBlocked(tasks, "Adaptive planner did not produce a valid subagent plan.");
|
|
2134
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
2135
|
+
manifest = updateRunStatus(manifest, "blocked", "Adaptive planner did not produce a valid subagent plan.");
|
|
2136
|
+
return { manifest, tasks };
|
|
2137
|
+
}
|
|
2138
|
+
if (initialAdaptive.injected) {
|
|
2139
|
+
manifest = requiresPlanApproval(workflow, input.runtimeConfig) ? await ensurePlanApprovalRequested(manifest, tasks) : manifest;
|
|
2140
|
+
queueIndex = buildTaskGraphIndex(tasks);
|
|
2141
|
+
} else if (
|
|
2142
|
+
requiresPlanApproval(workflow, input.runtimeConfig) &&
|
|
2143
|
+
(hasPendingMutatingAdaptiveTask(tasks) || hasPendingMutatingTaskAtBoundary(tasks))
|
|
2144
|
+
) {
|
|
2145
|
+
manifest = await ensurePlanApprovalRequested(manifest, tasks);
|
|
2146
|
+
}
|
|
2147
|
+
if (manifest.planApproval?.status === "cancelled") {
|
|
2148
|
+
tasks = cancelPlanTasks(tasks, "Plan approval was cancelled.");
|
|
2149
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
2150
|
+
manifest = updateRunStatus(manifest, "cancelled", "Plan approval was cancelled.");
|
|
2151
|
+
return { manifest, tasks };
|
|
2152
|
+
}
|
|
2153
|
+
manifest = writeProgress(manifest, tasks, "team-runner", input.executeWorkers, input.runtimeConfig);
|
|
2154
|
+
await saveRunManifestAsync(manifest);
|
|
2155
|
+
const runtimeKind = input.runtime?.kind ?? (input.executeWorkers ? "child-process" : "scaffold");
|
|
2156
|
+
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
2157
|
+
|
|
2158
|
+
// Build a workflow phase state machine from workflow steps for precondition tracking.
|
|
2159
|
+
const workflowPhases: PhaseState[] = workflow.steps.map(
|
|
2160
|
+
(step): PhaseState => ({
|
|
2161
|
+
name: step.id,
|
|
2162
|
+
status: "pending",
|
|
2163
|
+
inputs: step.reads === false ? [] : Array.isArray(step.reads) ? step.reads : [],
|
|
2164
|
+
outputs: step.output === false ? [] : step.output ? [step.output] : [],
|
|
2165
|
+
}),
|
|
2166
|
+
);
|
|
2167
|
+
let wfMachine = createWorkflowStateMachine(workflowPhases);
|
|
2168
|
+
|
|
2169
|
+
// ── OPT-01 streaming dispatch: track in-flight dispatch units so a new
|
|
2170
|
+
// task can be dispatched as soon as a slot frees, without waiting for
|
|
2171
|
+
// the entire batch to complete. Each entry maps a unit key (singleton
|
|
2172
|
+
// task ID or coalesced-group ID) to the in-flight promise + member IDs. ──
|
|
2173
|
+
const pendingUnits = new Map<string, { taskIds: string[]; promise: Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> }>();
|
|
2174
|
+
|
|
2175
|
+
// CORE-1: run-scoped AbortController linked to input.signal. Aborted by
|
|
2176
|
+
// drainPendingUnits() so in-flight dispatch promises are settled (and
|
|
2177
|
+
// their child processes torn down) on every early-return path.
|
|
2178
|
+
const runController = new AbortController();
|
|
2179
|
+
if (input.signal) {
|
|
2180
|
+
if (input.signal.aborted) runController.abort();
|
|
2181
|
+
else input.signal.addEventListener("abort", () => runController.abort(), { once: true });
|
|
2182
|
+
}
|
|
2183
|
+
|
|
2184
|
+
// CORE-4: scheduler context — mutable state bag for extracted scheduler
|
|
2185
|
+
// functions. Fields are synced from closure locals at the top of each
|
|
2186
|
+
// loop iteration; extracted functions mutate ctx in-place.
|
|
2187
|
+
const ctx: SchedulerContext = {
|
|
2188
|
+
input,
|
|
2189
|
+
workflow,
|
|
2190
|
+
manifest,
|
|
2191
|
+
tasks,
|
|
2192
|
+
queueIndex,
|
|
2193
|
+
wfMachine,
|
|
2194
|
+
pendingUnits,
|
|
2195
|
+
runController,
|
|
2196
|
+
runtimeKind,
|
|
2197
|
+
adaptivePlanInjected,
|
|
2198
|
+
adaptivePlanMissing,
|
|
2199
|
+
settledMerge: null,
|
|
2200
|
+
};
|
|
2201
|
+
|
|
2202
|
+
// CORE-1: single drain point — all early returns + normal exit settle pendingUnits via finally block.
|
|
2203
|
+
try {
|
|
2204
|
+
while (tasks.some((task) => task.status === "queued") || pendingUnits.size > 0) {
|
|
2205
|
+
// CORE-4: full sync of mutable closure locals → ctx at the top of every
|
|
2206
|
+
// iteration. The inline post-merge block (adaptive plan re-injection,
|
|
2207
|
+
// phase advance, queueIndex rebuild) mutates these locals; without this
|
|
2208
|
+
// sync the extracted functions (selectDispatchBatch / dispatchBatch /
|
|
2209
|
+
// mergeUnitResult) would read stale ctx fields — e.g. ctx.workflow
|
|
2210
|
+
// missing newly-injected adaptive steps → E006 step-not-found.
|
|
2211
|
+
ctx.tasks = tasks;
|
|
2212
|
+
ctx.manifest = manifest;
|
|
2213
|
+
ctx.workflow = workflow;
|
|
2214
|
+
ctx.wfMachine = wfMachine;
|
|
2215
|
+
ctx.queueIndex = queueIndex;
|
|
2216
|
+
ctx.adaptivePlanInjected = adaptivePlanInjected;
|
|
2217
|
+
ctx.adaptivePlanMissing = adaptivePlanMissing;
|
|
2218
|
+
// CORE-4 extraction 1: signal-abort cancellation. cancelRunFromSignal
|
|
2219
|
+
// mutates ctx in-place and returns a SchedulerDecision.
|
|
2220
|
+
const signalDecision = await cancelRunFromSignal(ctx);
|
|
2221
|
+
if (signalDecision?.kind === "return") return signalDecision.result;
|
|
2222
|
+
|
|
2223
|
+
// CORE-4 extraction 2: failed-task handling. Sync mutable locals into
|
|
2224
|
+
// ctx before the call; handleFailedTask mutates ctx in-place and
|
|
2225
|
+
// returns a SchedulerDecision.
|
|
2226
|
+
ctx.tasks = tasks;
|
|
2227
|
+
ctx.manifest = manifest;
|
|
2228
|
+
const failedDecision = await handleFailedTask(ctx);
|
|
2229
|
+
tasks = ctx.tasks;
|
|
2230
|
+
manifest = ctx.manifest;
|
|
2231
|
+
if (failedDecision?.kind === "return") return failedDecision.result;
|
|
2232
|
+
if (failedDecision?.kind === "continue") continue;
|
|
2233
|
+
|
|
2234
|
+
// CORE-4 extraction 3: batch selection. Sync mutable locals into ctx;
|
|
2235
|
+
// selectDispatchBatch computes the ready batch and returns a dispatch
|
|
2236
|
+
// decision (or return if the run must block/abort). The caller syncs
|
|
2237
|
+
// ctx.wfMachine back because the function may advance phases.
|
|
2238
|
+
ctx.tasks = tasks;
|
|
2239
|
+
ctx.manifest = manifest;
|
|
2240
|
+
ctx.wfMachine = wfMachine;
|
|
2241
|
+
const dispatchDecision = await selectDispatchBatch(ctx);
|
|
2242
|
+
tasks = ctx.tasks;
|
|
2243
|
+
manifest = ctx.manifest;
|
|
2244
|
+
wfMachine = ctx.wfMachine;
|
|
2245
|
+
if (dispatchDecision.kind === "return") return dispatchDecision.result;
|
|
2246
|
+
if (dispatchDecision.kind !== "dispatch") continue;
|
|
2247
|
+
// CORE-4 extraction 4: dispatch execution. Sync mutable locals into
|
|
2248
|
+
// ctx; dispatchBatch runs before_task_start hooks, builds coalesced
|
|
2249
|
+
// dispatch units, pre-warms the stable-prefix cache, and adds each
|
|
2250
|
+
// unit's promise to ctx.pendingUnits (wrapped in executeWithRetry).
|
|
2251
|
+
// The function mutates ctx.tasks (hook skips) and ctx.manifest (hook
|
|
2252
|
+
// status) in-place. Verbatim lift of the inline dispatch block.
|
|
2253
|
+
ctx.tasks = tasks;
|
|
2254
|
+
ctx.manifest = manifest;
|
|
2255
|
+
await dispatchBatch(ctx, dispatchDecision);
|
|
2256
|
+
tasks = ctx.tasks;
|
|
2257
|
+
manifest = ctx.manifest;
|
|
2258
|
+
|
|
2259
|
+
// ── OPT-01 wait phase: if no units are in-flight (e.g. all ready tasks
|
|
2260
|
+
// were hook-skipped or readyBatch was empty), re-loop to re-evaluate. ──
|
|
2261
|
+
if (ctx.pendingUnits.size === 0) continue;
|
|
2262
|
+
|
|
2263
|
+
// CORE-4 extraction 5: wait/merge. Sync ctx; mergeUnitResult awaits
|
|
2264
|
+
// Promise.race on the in-flight units, merges the first settled result
|
|
2265
|
+
// into manifest/tasks under the run lock (loadRunManifestById +
|
|
2266
|
+
// mergeTaskUpdatesPreservingTerminal), deletes it from
|
|
2267
|
+
// ctx.pendingUnits, and records the merge outcome on ctx. Returns null
|
|
2268
|
+
// (continue to phase/budget check) — a return decision is reserved for
|
|
2269
|
+
// future run-complete/failure detection.
|
|
2270
|
+
ctx.tasks = tasks;
|
|
2271
|
+
ctx.manifest = manifest;
|
|
2272
|
+
const mergeDecision = await mergeUnitResult(ctx);
|
|
2273
|
+
tasks = ctx.tasks;
|
|
2274
|
+
manifest = ctx.manifest;
|
|
2275
|
+
if (mergeDecision?.kind === "return") return mergeDecision.result;
|
|
2276
|
+
// Re-derive the merge outcome locals for the post-merge inline logic
|
|
2277
|
+
// (cancel-during-exec check + batch summary artifact).
|
|
2278
|
+
const { taskIds: settledTaskIds, result: resultToMerge } = ctx.settledMerge!;
|
|
2279
|
+
|
|
2280
|
+
// CORE-4 extraction 6: workflow phase advance. Sync ctx.wfMachine;
|
|
2281
|
+
// advanceWorkflowPhases advances phases whose tasks are all terminal,
|
|
2282
|
+
// emits phase_* events, and advances currentPhaseIndex. Mutates
|
|
2283
|
+
// ctx.wfMachine in-place.
|
|
2284
|
+
ctx.wfMachine = wfMachine;
|
|
2285
|
+
await advanceWorkflowPhases(ctx);
|
|
2286
|
+
wfMachine = ctx.wfMachine;
|
|
2287
|
+
|
|
2288
|
+
// CORE-4 extraction 7: budget enforcement. Sync ctx.tasks/manifest;
|
|
2289
|
+
// enforceRunBudget checks cumulative usage against warn/abort thresholds
|
|
2290
|
+
// and a fair-share heuristic. On abort it marks the run failed and
|
|
2291
|
+
// returns { kind: "return" }; otherwise emits warning events and
|
|
2292
|
+
// returns null (continue).
|
|
2293
|
+
ctx.tasks = tasks;
|
|
2294
|
+
ctx.manifest = manifest;
|
|
2295
|
+
const budgetDecision = await enforceRunBudget(ctx);
|
|
2296
|
+
tasks = ctx.tasks;
|
|
2297
|
+
manifest = ctx.manifest;
|
|
2298
|
+
if (budgetDecision?.kind === "return") return budgetDecision.result;
|
|
2299
|
+
|
|
2300
|
+
const cancelledResult = resultToMerge.manifest.status === "cancelled" ? resultToMerge : undefined;
|
|
2301
|
+
if (cancelledResult || input.signal?.aborted) {
|
|
2302
|
+
const reason = input.signal?.aborted ? cancellationReasonFromSignal(input.signal) : undefined;
|
|
2303
|
+
const message = reason?.message ?? cancelledResult?.manifest.summary ?? "Run cancelled during task execution.";
|
|
2304
|
+
manifest = { ...manifest, status: "running" };
|
|
2305
|
+
manifest = updateRunStatus(manifest, "cancelled", message);
|
|
2306
|
+
// CANCEL-2: re-cancel non-terminal tasks here, mirroring the batch-loop
|
|
2307
|
+
// cancel check at team-runner.ts:~925. A manifest cancelled mid-merge
|
|
2308
|
+
// (e.g. signal abort during the merge's awaits) would otherwise save
|
|
2309
|
+
// tasks without the cancel applied, leaving status=cancelled but tasks
|
|
2310
|
+
// showing completed/running -- inconsistent and breaks handleRetry's
|
|
2311
|
+
// filter for failed/cancelled tasks. Terminal tasks are NOT clobbered.
|
|
2312
|
+
const cancelMessage = reason ? `${message} (${reason.code})` : message;
|
|
2313
|
+
const reCancelledTasks = cancelNonTerminalTasks(tasks, "cancelled", cancelMessage);
|
|
2314
|
+
await saveRunTasksAsync(manifest, reCancelledTasks);
|
|
2315
|
+
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, reCancelledTasks, runtimeKind));
|
|
2316
|
+
await saveRunManifestAsync(manifest);
|
|
2317
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
2318
|
+
type: "run.cancelled",
|
|
2319
|
+
runId: manifest.runId,
|
|
2320
|
+
message,
|
|
2321
|
+
data: {
|
|
2322
|
+
reason,
|
|
2323
|
+
phase: "task-batch",
|
|
2324
|
+
|
|
2325
|
+
cancelledResultRunId: cancelledResult?.manifest.runId,
|
|
2326
|
+
},
|
|
2327
|
+
});
|
|
2328
|
+
return { manifest, tasks: reCancelledTasks };
|
|
2329
|
+
}
|
|
2330
|
+
queueIndex = buildTaskGraphIndex(tasks);
|
|
2331
|
+
const injectedAfterBatch = await attemptAdaptivePlan();
|
|
2332
|
+
if (injectedAfterBatch.missing) {
|
|
2333
|
+
tasks = markBlocked(tasks, "Adaptive planner did not produce a valid subagent plan.");
|
|
2334
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
2335
|
+
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
2336
|
+
manifest = updateRunStatus(manifest, "blocked", "Adaptive planner did not produce a valid subagent plan.");
|
|
2337
|
+
return { manifest, tasks };
|
|
2338
|
+
}
|
|
2339
|
+
if (injectedAfterBatch.injected) {
|
|
2340
|
+
manifest = requiresPlanApproval(workflow, input.runtimeConfig)
|
|
2341
|
+
? await ensurePlanApprovalRequested(manifest, tasks)
|
|
2342
|
+
: manifest;
|
|
2343
|
+
queueIndex = buildTaskGraphIndex(tasks);
|
|
2344
|
+
} else if (
|
|
2345
|
+
requiresPlanApproval(workflow, input.runtimeConfig) &&
|
|
2346
|
+
(hasPendingMutatingAdaptiveTask(tasks) || hasPendingMutatingTaskAtBoundary(tasks))
|
|
2347
|
+
) {
|
|
2348
|
+
manifest = await ensurePlanApprovalRequested(manifest, tasks);
|
|
2349
|
+
}
|
|
2350
|
+
if (manifest.planApproval?.status === "cancelled") {
|
|
2351
|
+
tasks = cancelPlanTasks(tasks, "Plan approval was cancelled.");
|
|
2352
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
2353
|
+
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
2354
|
+
manifest = updateRunStatus(manifest, "cancelled", "Plan approval was cancelled.");
|
|
2355
|
+
return { manifest, tasks };
|
|
2356
|
+
}
|
|
2357
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
2358
|
+
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
2359
|
+
const completedBatch = tasks.filter((t) => settledTaskIds.includes(t.id));
|
|
2360
|
+
const batchArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
2361
|
+
kind: "summary",
|
|
2362
|
+
relativePath: `batches/${settledTaskIds.join("+")}.md`,
|
|
2363
|
+
producer: "team-runner",
|
|
2364
|
+
content: aggregateTaskOutputs(completedBatch, manifest),
|
|
2365
|
+
});
|
|
2366
|
+
const groupDelivery = deliverGroupJoin({
|
|
2367
|
+
manifest,
|
|
2368
|
+
mode: resolveGroupJoinMode(input.runtimeConfig),
|
|
2369
|
+
batch: completedBatch,
|
|
2370
|
+
allTasks: tasks,
|
|
2371
|
+
});
|
|
2372
|
+
manifest = {
|
|
2373
|
+
...manifest,
|
|
2374
|
+
artifacts: mergeArtifacts([
|
|
2375
|
+
...manifest.artifacts,
|
|
2376
|
+
batchArtifact,
|
|
2377
|
+
...(groupDelivery?.artifact ? [groupDelivery.artifact] : []),
|
|
2378
|
+
]),
|
|
2379
|
+
};
|
|
2380
|
+
manifest = writeProgress(manifest, tasks, "team-runner", input.executeWorkers, input.runtimeConfig);
|
|
2381
|
+
await saveRunManifestAsync(manifest);
|
|
2382
|
+
}
|
|
2383
|
+
|
|
2384
|
+
// CORE-4 extraction 8: finalization. Sync ctx; finalizeRun computes the
|
|
2385
|
+
// final run status (failed/blocked/completed), writes the summary artifact
|
|
2386
|
+
// + health snapshot, performs the joint atomic manifest+tasks save, and
|
|
2387
|
+
// returns the terminal { manifest, tasks } result. Sync the locals back
|
|
2388
|
+
// from ctx so the finally block observes consistent state.
|
|
2389
|
+
const finalResult = await finalizeRun(ctx);
|
|
2390
|
+
manifest = ctx.manifest;
|
|
2391
|
+
tasks = ctx.tasks;
|
|
2392
|
+
return finalResult;
|
|
2393
|
+
} finally {
|
|
2394
|
+
await drainPendingUnits(pendingUnits, runController);
|
|
2395
|
+
}
|
|
2396
|
+
}
|