pi-crew 0.9.68 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +222 -0
- package/NOTICE.md +21 -0
- package/README.md +44 -2
- package/agents/analyst.md +1 -1
- package/agents/cold-verifier.md +3 -1
- package/agents/critic.md +1 -1
- package/agents/executor.md +1 -1
- package/agents/explorer.md +1 -1
- package/agents/planner.md +1 -1
- package/agents/reviewer.md +1 -1
- package/agents/security-reviewer.md +1 -1
- package/agents/test-engineer.md +1 -1
- package/agents/verifier.md +1 -1
- package/agents/writer.md +1 -1
- package/dist/index.mjs +68113 -60774
- package/docs/README.md +2 -0
- package/docs/actions-reference.md +31 -0
- package/docs/commands-reference.md +17 -6
- package/docs/resource-formats.md +13 -0
- package/package.json +4 -2
- package/schema.json +503 -91
- package/scripts/resource-sampler.mjs +36 -2
- package/skills/requirements-to-task-packet/SKILL.md +26 -0
- package/skills/widget-rendering/SKILL.md +7 -7
- package/src/agents/agent-config.ts +2 -1
- package/src/agents/discover-agents.ts +23 -14
- package/src/config/config-merge.ts +183 -0
- package/src/config/config-validation.ts +687 -0
- package/src/config/config.ts +22 -864
- package/src/config/defaults.ts +43 -2
- package/src/config/drift-detector.ts +1 -1
- package/src/config/env-vars.ts +691 -0
- package/src/config/role-tools.ts +11 -9
- package/src/config/sanitize-project-config.ts +172 -0
- package/src/config/types.ts +49 -1
- package/src/extension/async-notifier.ts +25 -2
- package/src/extension/crew-cleanup.ts +13 -0
- package/src/extension/crew-vibes/config.ts +2 -1
- package/src/extension/crew-vibes/footer.ts +19 -0
- package/src/extension/crew-vibes/index.ts +11 -1
- package/src/extension/plan-orchestrate.ts +132 -0
- package/src/extension/register.ts +8 -0
- package/src/extension/registration/command-registration.ts +1 -0
- package/src/extension/registration/commands/dashboard.ts +158 -0
- package/src/extension/registration/commands/index.ts +35 -0
- package/src/extension/registration/commands/manage.ts +303 -0
- package/src/extension/registration/commands/run.ts +228 -0
- package/src/extension/registration/commands/shared.ts +639 -0
- package/src/extension/registration/commands/status.ts +60 -0
- package/src/extension/registration/commands.ts +13 -1224
- package/src/extension/registration/foreground-run-controller.ts +10 -2
- package/src/extension/registration/lifecycle-handlers.ts +178 -17
- package/src/extension/registration/runtime-cleanup.ts +23 -5
- package/src/extension/registration/subagent-tools.ts +218 -9
- package/src/extension/registration/team-tool.ts +5 -1
- package/src/extension/registration/ui.ts +5 -4
- package/src/extension/rpc-hmac.ts +5 -3
- package/src/extension/team-tool/api/heartbeat.ts +47 -10
- package/src/extension/team-tool/api/plan-approval.ts +9 -0
- package/src/extension/team-tool/api/task-claims.ts +109 -40
- package/src/extension/team-tool/cancel.ts +84 -50
- package/src/extension/team-tool/dispatch/index.ts +1 -0
- package/src/extension/team-tool/dispatch/run.ts +4 -1
- package/src/extension/team-tool/doctor.ts +103 -1
- package/src/extension/team-tool/orchestrate.ts +66 -1
- package/src/extension/team-tool/plans.ts +192 -0
- package/src/extension/team-tool/respond.ts +197 -65
- package/src/extension/team-tool/run-deadline.ts +35 -3
- package/src/extension/team-tool/run-intent.ts +63 -0
- package/src/extension/team-tool/run.ts +74 -20
- package/src/extension/team-tool/status.ts +84 -26
- package/src/extension/team-tool.ts +11 -2
- package/src/hooks/registry.ts +1 -6
- package/src/i18n.ts +9 -0
- package/src/prompt/prompt-runtime.ts +521 -2
- package/src/prompt/worker-events-channel.ts +173 -0
- package/src/runtime/README.md +8 -8
- package/src/runtime/async-runner.ts +7 -3
- package/src/runtime/background-runner.ts +42 -14
- package/src/runtime/broker/broker-issuer.ts +9 -2
- package/src/runtime/broker/crew-broker-tokens.ts +43 -6
- package/src/runtime/broker/crew-broker.ts +838 -10
- package/src/runtime/broker/wait-status-cache.ts +157 -0
- package/src/runtime/budget-enforcement.ts +281 -0
- package/src/runtime/child-pi/child-pi-constants.ts +8 -0
- package/src/runtime/child-pi/child-pi-spawn.ts +60 -14
- package/src/runtime/child-pi/child-pi-streams.ts +21 -1
- package/src/runtime/child-pi/child-pi-timers.ts +324 -0
- package/src/runtime/child-pi/child-pi.ts +97 -201
- package/src/runtime/child-pi/mock-fixtures.ts +16 -2
- package/src/runtime/crew-agent-records.ts +259 -14
- package/src/runtime/delegate-spawn.ts +148 -0
- package/src/runtime/detached-run-results.ts +90 -0
- package/src/runtime/deterministic-ast.ts +2 -1
- package/src/runtime/dispatch-batch.ts +945 -0
- package/src/runtime/finalize-run.ts +557 -0
- package/src/runtime/goal-workflow/adaptive-plan.ts +116 -15
- package/src/runtime/goal-workflow/dynamic-workflow-runner.ts +8 -3
- package/src/runtime/goal-workflow/goal-state-store.ts +1 -1
- package/src/runtime/group-join.ts +11 -125
- package/src/runtime/live-session/live-session-runtime.ts +26 -1
- package/src/runtime/merge-gate.ts +32 -10
- package/src/runtime/merge-loop.ts +130 -0
- package/src/runtime/model/model-budget-summary.ts +53 -0
- package/src/runtime/model/model-fallback.ts +36 -2
- package/src/runtime/model/pi-args.ts +10 -0
- package/src/runtime/model/provider-extensions.ts +10 -0
- package/src/runtime/orphan-worker-registry.ts +1 -1
- package/src/runtime/output/output-validator.ts +45 -0
- package/src/runtime/parent-guard.ts +3 -1
- package/src/runtime/peer-dep.ts +2 -1
- package/src/runtime/per-write-validator.ts +0 -5
- package/src/runtime/pi-spawn.ts +61 -15
- package/src/runtime/plan-approval.ts +125 -0
- package/src/runtime/plan-replan.ts +151 -0
- package/src/runtime/process-status.ts +16 -1
- package/src/runtime/recovery/checkpoint.ts +0 -18
- package/src/runtime/recovery/crash-recovery.ts +111 -46
- package/src/runtime/run-tracker.ts +77 -10
- package/src/runtime/scheduler-context.ts +98 -0
- package/src/runtime/scheduling/coalesce-tasks.ts +5 -0
- package/src/runtime/scheduling/global-worker-cap.ts +2 -1
- package/src/runtime/scheduling/nested-slots.ts +70 -0
- package/src/runtime/scheduling/run-coalesced-task-group.ts +64 -13
- package/src/runtime/scheduling/task-graph-scheduler.ts +0 -10
- package/src/runtime/settings-store.ts +219 -0
- package/src/runtime/spawn-policy.ts +217 -0
- package/src/runtime/stale-reconciler.ts +87 -6
- package/src/runtime/subagent-manager.ts +25 -1
- package/src/runtime/task-output-context.ts +230 -9
- package/src/runtime/task-packet.ts +23 -1
- package/src/runtime/task-runner/child-executor.ts +106 -7
- package/src/runtime/task-runner/post-execution.ts +125 -1
- package/src/runtime/task-runner/pre-execution.ts +39 -1
- package/src/runtime/task-runner/prompt-builder.ts +51 -1
- package/src/runtime/task-runner/retrieval-orchestrator.ts +72 -18
- package/src/runtime/task-runner/spec-evidence.ts +403 -0
- package/src/runtime/task-runner/state-helpers.ts +26 -24
- package/src/runtime/task-runner.ts +11 -0
- package/src/runtime/team-runner.ts +132 -1673
- package/src/runtime/verification/spec-sandbox.ts +255 -0
- package/src/runtime/verification/verification-gates.ts +3 -2
- package/src/runtime/verification/verification-worktree.ts +2 -1
- package/src/runtime/workflow-phase-advance.ts +100 -0
- package/src/runtime/workspace-tree.ts +9 -0
- package/src/schema/config-schema.ts +66 -26
- package/src/schema/sensitive-config-paths.ts +64 -0
- package/src/schema/team-tool-schema.ts +13 -3
- package/src/state/README.md +4 -10
- package/src/state/atomic-write.ts +20 -3
- package/src/state/contracts.ts +38 -0
- package/src/state/coordination/mailbox.ts +12 -2
- package/src/state/event-log/cursor.ts +223 -0
- package/src/state/event-log/event-log-rotation.ts +12 -4
- package/src/state/event-log/event-log.ts +152 -369
- package/src/state/event-log/sequence-cache.ts +373 -0
- package/src/state/event-log/worker-atomic-writer.ts +2 -1
- package/src/state/stores/active-run-registry.ts +3 -2
- package/src/state/stores/manifest-io.ts +237 -0
- package/src/state/stores/ownership-map.ts +162 -0
- package/src/state/stores/plan-store.ts +241 -0
- package/src/state/stores/run-cache.ts +0 -90
- package/src/state/stores/spec-store.ts +189 -0
- package/src/state/stores/state-store.ts +139 -232
- package/src/state/types.ts +199 -0
- package/src/ui/dashboard-panes/plan-pane.ts +136 -0
- package/src/ui/dashboard-panes/progress-pane.ts +6 -0
- package/src/ui/dashboard-panes/transcript-pane.ts +31 -0
- package/src/ui/dock-footer.ts +49 -0
- package/src/ui/heartbeat-aggregator.ts +9 -1
- package/src/ui/inline-panel/agent-pane.ts +375 -0
- package/src/ui/inline-panel/agent-transcript.ts +338 -0
- package/src/ui/inline-panel/agent-view-overlay.ts +225 -0
- package/src/ui/inline-panel/crew-editor.ts +192 -0
- package/src/ui/inline-panel/index.ts +290 -0
- package/src/ui/inline-panel/panel-rows.ts +37 -0
- package/src/ui/inline-panel/panel-selection.ts +157 -0
- package/src/ui/inline-panel/panel-store.ts +111 -0
- package/src/ui/inline-panel/view-session-store.ts +36 -0
- package/src/ui/keybinding-map.ts +54 -13
- package/src/ui/pi-ui-compat.ts +9 -0
- package/src/ui/powerbar-publisher.ts +52 -1
- package/src/ui/run-dashboard.ts +31 -5
- package/src/ui/run-snapshot-cache.ts +57 -30
- package/src/ui/snapshot-types.ts +6 -1
- package/src/ui/widget/index.ts +176 -22
- package/src/ui/widget/task-list.ts +198 -0
- package/src/ui/widget/widget-formatters.ts +240 -4
- package/src/ui/widget/widget-renderer.ts +243 -38
- package/src/ui/widget/widget-types.ts +11 -0
- package/src/utils/child-process-shield.ts +106 -0
- package/src/utils/file-coalescer.ts +0 -4
- package/src/utils/fs-errno.ts +66 -0
- package/src/utils/fs-watch.ts +1 -1
- package/src/utils/internal-error.ts +3 -1
- package/src/utils/paths.ts +11 -3
- package/src/utils/redaction.ts +7 -0
- package/src/utils/safe-abort.ts +45 -0
- package/src/utils/task-name-generator.ts +1 -8
- package/src/workflows/discover-workflows.ts +20 -2
- package/src/workflows/validate-workflow.ts +7 -1
- package/src/workflows/workflow-config.ts +17 -0
- package/src/workflows/workflow-serializer.ts +3 -0
- package/src/worktree/worktree-manager.ts +22 -0
- package/workflows/default.workflow.md +36 -26
- package/workflows/strict-fast-fix.workflow.md +26 -0
- package/src/agents/agent-search.ts +0 -98
- package/src/benchmark/benchmark-runner.ts +0 -313
- package/src/benchmark/feedback-loop.ts +0 -73
- package/src/config/resilient-parser.ts +0 -117
- package/src/extension/crew-vibes/cat-frames.ts +0 -18
- package/src/extension/result-watcher.ts +0 -139
- package/src/observability/exporters/prometheus-exporter.ts +0 -54
- package/src/observability/metric-retention.ts +0 -64
- package/src/runtime/compaction/compaction-summary.ts +0 -278
- package/src/runtime/errors/crew-errors.ts +0 -162
- package/src/runtime/live-session/intercom-bridge.ts +0 -187
- package/src/runtime/loop-gates.ts +0 -128
- package/src/runtime/metric-parser.ts +0 -36
- package/src/runtime/output/stream-preview.ts +0 -184
- package/src/runtime/output/tool-progress.ts +0 -278
- package/src/runtime/phase-tracker.ts +0 -385
- package/src/runtime/pipeline-runner.ts +0 -523
- package/src/runtime/process/process-lifecycle.ts +0 -491
- package/src/runtime/recovery/retry-runner.ts +0 -330
- package/src/runtime/run-drift.ts +0 -219
- package/src/runtime/task-quality.ts +0 -199
- package/src/runtime/task-runner/run-projection.ts +0 -128
- package/src/runtime/verification/post-checks.ts +0 -142
- package/src/state/coordination/schedule.ts +0 -166
- package/src/state/event-log/jsonl-writer.ts +0 -115
- package/src/state/hook-instinct-bridge.ts +0 -94
- package/src/state/hook-integrations.ts +0 -51
- package/src/state/session-state-map.ts +0 -51
- package/src/state/stores/blob-store.ts +0 -308
- package/src/state/stores/instinct-store.ts +0 -275
- package/src/state/stores/observation-store.ts +0 -176
- package/src/state/tiered-eval.ts +0 -480
- package/src/state/types-eval.ts +0 -58
- package/src/tools/safe-bash-extension.ts +0 -54
- package/src/tools/safe-bash.ts +0 -505
- package/src/ui/agent-management-overlay.ts +0 -160
- package/src/ui/crew-footer.ts +0 -102
- package/src/ui/crew-select-list.ts +0 -114
- package/src/ui/dashboard-panes/capability-pane.ts +0 -77
- package/src/ui/transcript-entries.ts +0 -256
- package/src/utils/conflict-detect.ts +0 -721
- package/src/utils/fingerprint.ts +0 -180
- package/src/utils/gh-protocol.ts +0 -556
- package/src/utils/project-detector.ts +0 -160
- package/src/utils/sse-parser.ts +0 -131
- package/src/workflows/cost-estimator.ts +0 -34
- package/src/workflows/intermediate-store.ts +0 -166
|
@@ -4,73 +4,42 @@ import * as path from "node:path";
|
|
|
4
4
|
import { fileURLToPath } from "node:url";
|
|
5
5
|
import type { AgentConfig } from "../agents/agent-config.ts";
|
|
6
6
|
import type { CrewLimitsConfig, CrewReliabilityConfig, CrewRuntimeConfig } from "../config/config.ts";
|
|
7
|
-
import { CrewError, ErrorCode } from "../errors.ts";
|
|
8
7
|
import { appendHookEvent, executeHook } from "../hooks/registry.ts";
|
|
9
|
-
import { childCorrelation, withCorrelation } from "../observability/correlation.ts";
|
|
10
8
|
import type { MetricRegistry } from "../observability/metric-registry.ts";
|
|
11
|
-
import { atomicWriteFile
|
|
9
|
+
import { atomicWriteFile } from "../state/atomic-write.ts";
|
|
12
10
|
import { canTransitionRunStatus } from "../state/contracts.ts";
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
appendEvent,
|
|
16
|
-
appendEventAsync,
|
|
17
|
-
appendEventBuffered,
|
|
18
|
-
appendEventFireAndForget,
|
|
19
|
-
flushEventLogBuffer,
|
|
20
|
-
readEvents,
|
|
21
|
-
} from "../state/event-log/event-log.ts";
|
|
11
|
+
import { appendEvent, appendEventAsync, flushEventLogBuffer } from "../state/event-log/event-log.ts";
|
|
22
12
|
import { hashArtifactContent as hashContent, writeArtifact } from "../state/stores/artifact-store.ts";
|
|
23
|
-
import { HealthStore } from "../state/stores/health-store.ts";
|
|
24
13
|
import { loadRunManifestById, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/stores/state-store.ts";
|
|
25
|
-
import type {
|
|
26
|
-
import { aggregateUsage, formatTokens, formatUsage } from "../state/usage.ts";
|
|
14
|
+
import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
|
|
27
15
|
import type { TeamConfig } from "../teams/team-config.ts";
|
|
28
16
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
29
|
-
import type { WorkflowConfig
|
|
30
|
-
import {
|
|
17
|
+
import type { WorkflowConfig } from "../workflows/workflow-config.ts";
|
|
18
|
+
import { drainPendingUnits, enforceRunBudget, terminaliseRunWithDrain } from "./budget-enforcement.ts";
|
|
31
19
|
import { readCrewAgents, saveCrewAgents, saveCrewAgentsCoalesced } from "./crew-agent-records.ts";
|
|
32
20
|
import type { CrewRuntimeKind } from "./crew-agent-runtime.ts";
|
|
33
21
|
import { crewHooks } from "./crew-hooks.ts";
|
|
34
|
-
import {
|
|
35
|
-
import {
|
|
22
|
+
import { cancelNonTerminalTasks, dispatchBatch, markBlocked, selectDispatchBatch } from "./dispatch-batch.ts";
|
|
23
|
+
import { finalizeRun, lastProgressContentHash, writeProgress } from "./finalize-run.ts";
|
|
36
24
|
import { applyGoalAchievement, assessGoalAchievement } from "./goal-workflow/goal-achievement.ts";
|
|
37
25
|
import { deliverGroupJoin, resolveGroupJoinMode } from "./group-join.ts";
|
|
38
26
|
import { terminateLiveAgentsForRun } from "./live-session/live-agent-manager.ts";
|
|
39
|
-
import {
|
|
27
|
+
import { isRunTerminalPreserved, mergeUnitResult } from "./merge-loop.ts";
|
|
40
28
|
import { resolveTaskRuntimeKind } from "./model/runtime-policy.ts";
|
|
41
29
|
import type { CrewRuntimeCapabilities } from "./model/runtime-resolver.ts";
|
|
42
|
-
import {
|
|
43
|
-
import {
|
|
44
|
-
import {
|
|
45
|
-
import { buildRecoveryLedger, shouldRerunFailedTask } from "./recovery/recovery-recipes.ts";
|
|
46
|
-
import { DEFAULT_RETRY_POLICY, executeWithRetry, type RetryPolicy } from "./recovery/retry-executor.ts";
|
|
47
|
-
import { permissionForRole } from "./role-permission.ts";
|
|
30
|
+
import { ensurePlanApprovalRequested, isMutatingTask, isPlanApprovalDenied, requiresPlanApproval } from "./plan-approval.ts";
|
|
31
|
+
import { buildSyntheticTerminalEvidence, cancellationReasonFromSignal } from "./process/cancellation.ts";
|
|
32
|
+
import { shouldRerunFailedTask } from "./recovery/recovery-recipes.ts";
|
|
48
33
|
import { registerRunPromise, rejectRunPromise, resolveRunPromise } from "./run-tracker.ts";
|
|
49
|
-
import {
|
|
50
|
-
import {
|
|
51
|
-
import { runCoalescedTaskGroup } from "./scheduling/run-coalesced-task-group.ts";
|
|
52
|
-
import { buildExecutionPlan as buildDagExecutionPlan, getReadyTasks as getDagReadyTasks, type TaskNode } from "./scheduling/task-graph.ts";
|
|
53
|
-
import {
|
|
54
|
-
buildTaskGraphIndex,
|
|
55
|
-
refreshTaskGraphQueues,
|
|
56
|
-
type TaskGraphIndex,
|
|
57
|
-
type TaskGraphSchedulerSnapshot,
|
|
58
|
-
taskGraphSnapshot,
|
|
59
|
-
} from "./scheduling/task-graph-scheduler.ts";
|
|
34
|
+
import type { PendingUnit, SchedulerContext, SchedulerDecision } from "./scheduler-context.ts";
|
|
35
|
+
import { buildTaskGraphIndex, refreshTaskGraphQueues } from "./scheduling/task-graph-scheduler.ts";
|
|
60
36
|
import { recordsForMaterializedTasks } from "./task-display.ts";
|
|
61
|
-
import { aggregateTaskOutputs } from "./task-output-context.ts";
|
|
62
|
-
import { clearStablePrefixCache
|
|
63
|
-
import { runTeamTask, type SpawnBudget } from "./task-runner.ts";
|
|
37
|
+
import { aggregateTaskOutputs, createResultArtifactReadCache } from "./task-output-context.ts";
|
|
38
|
+
import { clearStablePrefixCache } from "./task-runner/prompt-builder.ts";
|
|
64
39
|
import { mergeArtifacts } from "./team-runner-artifacts.ts";
|
|
65
40
|
import { clearTrackedTaskUsage } from "./usage-tracker.ts";
|
|
66
|
-
import {
|
|
67
|
-
|
|
68
|
-
type PhaseGuardContext,
|
|
69
|
-
type PhaseState,
|
|
70
|
-
transitionPhase,
|
|
71
|
-
validatePhasePreconditions,
|
|
72
|
-
type WorkflowStateMachine,
|
|
73
|
-
} from "./workflow-state.ts";
|
|
41
|
+
import { advanceWorkflowPhases } from "./workflow-phase-advance.ts";
|
|
42
|
+
import { createWorkflowStateMachine, type PhaseState } from "./workflow-state.ts";
|
|
74
43
|
|
|
75
44
|
/**
|
|
76
45
|
* Start a periodic heartbeat for the team-level run.
|
|
@@ -144,7 +113,7 @@ function perfScriptPath(scriptName: string): string | undefined {
|
|
|
144
113
|
}
|
|
145
114
|
}
|
|
146
115
|
|
|
147
|
-
function startPerfSampler(manifest: TeamRunManifest, team: TeamConfig): void {
|
|
116
|
+
function startPerfSampler(manifest: TeamRunManifest, team: TeamConfig, signal?: AbortSignal): void {
|
|
148
117
|
// DIRECT fs marker (console may be swallowed by the host) — every branch
|
|
149
118
|
// writes artifacts/<runId>/perf-obs.log with the exact reason.
|
|
150
119
|
const marker = (msg: string): void => {
|
|
@@ -186,7 +155,7 @@ function startPerfSampler(manifest: TeamRunManifest, team: TeamConfig): void {
|
|
|
186
155
|
"--out",
|
|
187
156
|
outPath,
|
|
188
157
|
],
|
|
189
|
-
{ detached: true, stdio: ["ignore", "ignore", "pipe"] },
|
|
158
|
+
{ detached: true, stdio: ["ignore", "ignore", "pipe"], signal },
|
|
190
159
|
);
|
|
191
160
|
// diagnostics: sampler stderr → perf-obs.log (best-effort; never affects the run)
|
|
192
161
|
child.stderr?.on("data", (d: Buffer) => {
|
|
@@ -198,11 +167,15 @@ function startPerfSampler(manifest: TeamRunManifest, team: TeamConfig): void {
|
|
|
198
167
|
});
|
|
199
168
|
child.unref();
|
|
200
169
|
} catch (err) {
|
|
201
|
-
|
|
170
|
+
// R11-4 (LOW, §ROUND 11): sampler linger-orphan hardening — pass the run
|
|
171
|
+
// AbortSignal so the detached sampler dies on run teardown, not just on
|
|
172
|
+
// parent death. ROUND 12: console → logInternalError, explicit "warn"
|
|
173
|
+
// severity (default "debug" would be PI_TEAMS_DEBUG-gated and hide this).
|
|
174
|
+
logInternalError("team-runner.perf-sampler.spawn-failed", err, `runId=${manifest.runId}`, "warn");
|
|
202
175
|
}
|
|
203
176
|
}
|
|
204
177
|
|
|
205
|
-
function schedulePerfAnalyze(manifest: TeamRunManifest, team: TeamConfig): void {
|
|
178
|
+
function schedulePerfAnalyze(manifest: TeamRunManifest, team: TeamConfig, signal?: AbortSignal): void {
|
|
206
179
|
// Strict true — same test-isolation rationale as startPerfSampler.
|
|
207
180
|
if (team.observability !== true) return;
|
|
208
181
|
const analyzePath = perfScriptPath("analyze-run.mjs");
|
|
@@ -217,11 +190,13 @@ function schedulePerfAnalyze(manifest: TeamRunManifest, team: TeamConfig): void
|
|
|
217
190
|
const child = spawn(
|
|
218
191
|
process.execPath,
|
|
219
192
|
["--experimental-strip-types", analyzePath, manifest.runId, "--crew-root", crewRoot, "--resources", resourcesPath],
|
|
220
|
-
{ detached: true, stdio: "ignore" },
|
|
193
|
+
{ detached: true, stdio: "ignore", signal },
|
|
221
194
|
);
|
|
222
195
|
child.unref();
|
|
223
196
|
} catch (err) {
|
|
224
|
-
|
|
197
|
+
// R11-4 (LOW, §ROUND 11): same abort-kill hardening as the sampler.
|
|
198
|
+
// ROUND 12: console → logInternalError, explicit "warn" severity.
|
|
199
|
+
logInternalError("team-runner.perf-analyze.spawn-failed", err, `runId=${manifest.runId}`, "warn");
|
|
225
200
|
}
|
|
226
201
|
}, OBSERVABILITY_ANALYZE_DELAY_MS);
|
|
227
202
|
timer.unref();
|
|
@@ -260,85 +235,8 @@ export interface ExecuteTeamRunInput {
|
|
|
260
235
|
budgetUnlimited?: boolean;
|
|
261
236
|
}
|
|
262
237
|
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
*/
|
|
266
|
-
export interface PerTaskBudgetCheckResult {
|
|
267
|
-
/** Whether the abort threshold was exceeded. */
|
|
268
|
-
abort: boolean;
|
|
269
|
-
/** Whether the warning threshold was exceeded. */
|
|
270
|
-
warning: boolean;
|
|
271
|
-
/** IDs of tasks that exceeded their fair share (>50% of remaining budget). */
|
|
272
|
-
fairShareViolators: string[];
|
|
273
|
-
/** Total tokens used so far. */
|
|
274
|
-
totalUsed: number;
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
/**
|
|
278
|
-
* Check cumulative token usage against per-task budget thresholds.
|
|
279
|
-
* Returns a structured result — callers decide how to act (warn vs abort).
|
|
280
|
-
*
|
|
281
|
-
* Exported for unit testing.
|
|
282
|
-
*/
|
|
283
|
-
export function checkPerTaskBudget(
|
|
284
|
-
tasks: TeamTaskState[],
|
|
285
|
-
budgetTotal: number,
|
|
286
|
-
budgetWarning: number,
|
|
287
|
-
budgetAbort: number,
|
|
288
|
-
fairShareFraction = 0.5,
|
|
289
|
-
): PerTaskBudgetCheckResult {
|
|
290
|
-
const usage = aggregateUsage(tasks);
|
|
291
|
-
const totalUsed = (usage?.input ?? 0) + (usage?.output ?? 0) + (usage?.cacheWrite ?? 0);
|
|
292
|
-
const abort = totalUsed >= budgetAbort * budgetTotal;
|
|
293
|
-
const warning = !abort && totalUsed >= budgetWarning * budgetTotal;
|
|
294
|
-
// Fair share threshold based on TOTAL budget, not remaining budget.
|
|
295
|
-
// This ensures a task that consumed 60% of total budget is flagged even
|
|
296
|
-
// if only 40% remains (40% * 50% = 20% threshold would miss the 60% usage).
|
|
297
|
-
const fairShareThreshold = budgetTotal * fairShareFraction;
|
|
298
|
-
const fairShareViolators: string[] = [];
|
|
299
|
-
for (const task of tasks) {
|
|
300
|
-
if (!task.usage) continue;
|
|
301
|
-
const taskTotal = (task.usage.input ?? 0) + (task.usage.output ?? 0) + (task.usage.cacheWrite ?? 0);
|
|
302
|
-
// Only flag tasks that individually consumed a significant portion of the
|
|
303
|
-
// budget (>10% of total) AND exceeded the fair share threshold.
|
|
304
|
-
if (fairShareThreshold > 0 && taskTotal > fairShareThreshold && taskTotal > budgetTotal * 0.1) {
|
|
305
|
-
fairShareViolators.push(task.id);
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
return { abort, warning, fairShareViolators, totalUsed };
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
function findStep(workflow: WorkflowConfig, task: TeamTaskState): WorkflowStep {
|
|
312
|
-
const step = workflow.steps.find((candidate) => candidate.id === task.stepId);
|
|
313
|
-
if (!step)
|
|
314
|
-
throw new CrewError(ErrorCode.ResourceNotFound, `Workflow step '${task.stepId}' not found for task '${task.id}'.`).withContext(
|
|
315
|
-
`workflow step lookup (task=${task.id})`,
|
|
316
|
-
);
|
|
317
|
-
return step;
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
function findAgent(agents: AgentConfig[], task: TeamTaskState): AgentConfig {
|
|
321
|
-
const agent = agents.find((candidate) => candidate.name === task.agent);
|
|
322
|
-
if (!agent)
|
|
323
|
-
throw new CrewError(ErrorCode.ResourceNotFound, `Agent '${task.agent}' not found for task '${task.id}'.`).withContext(
|
|
324
|
-
`agent lookup (task=${task.id})`,
|
|
325
|
-
);
|
|
326
|
-
return agent;
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
function markBlocked(tasks: TeamTaskState[], reason: string): TeamTaskState[] {
|
|
330
|
-
return tasks.map((task) =>
|
|
331
|
-
task.status === "queued"
|
|
332
|
-
? {
|
|
333
|
-
...task,
|
|
334
|
-
status: "skipped",
|
|
335
|
-
error: reason,
|
|
336
|
-
finishedAt: new Date().toISOString(),
|
|
337
|
-
graph: task.graph ? { ...task.graph, queue: "blocked" } : undefined,
|
|
338
|
-
}
|
|
339
|
-
: task,
|
|
340
|
-
);
|
|
341
|
-
}
|
|
238
|
+
// checkPerTaskBudget / PerTaskBudgetCheckResult moved to ./budget-enforcement.ts
|
|
239
|
+
// (2026-08 Phase 2.6) — re-exported below so existing test imports still resolve.
|
|
342
240
|
|
|
343
241
|
// isNonTerminalTaskStatus, safeFinishedAt, isMalformedFinishedAtReplacement,
|
|
344
242
|
// statusMergeKey, REJECTED_STATUS_MERGE_TRANSITIONS, shouldMergeTaskUpdate,
|
|
@@ -346,42 +244,8 @@ function markBlocked(tasks: TeamTaskState[], reason: string): TeamTaskState[] {
|
|
|
346
244
|
// __test__mergeTaskUpdates — moved to ./merge-gate.ts (2026-08-10
|
|
347
245
|
// improvement-plan Tier 2 team-runner split, self-contained portion).
|
|
348
246
|
// Re-imported below to preserve all in-file callers.
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
* CORE-6: Unified cancel/fail of non-terminal tasks. Replaces hand-rolled
|
|
352
|
-
* `.map()` + transform sites across this file.
|
|
353
|
-
*
|
|
354
|
-
* - Without `filter`: all non-terminal tasks (queued/running/waiting) are
|
|
355
|
-
* terminalised with the given status.
|
|
356
|
-
* - With `filter`: the filter is the sole gate — the non-terminal check is
|
|
357
|
-
* NOT applied automatically, matching per-task/per-id variants.
|
|
358
|
-
* - Optional `transform(task, terminalised)`: lets a caller attach
|
|
359
|
-
* site-specific fields (graph mutation, terminalEvidence) to the
|
|
360
|
-
* terminalised task. `terminalised` already carries status/finishedAt/error;
|
|
361
|
-
* the transform returns it unchanged or a modified copy.
|
|
362
|
-
*
|
|
363
|
-
* RT-14: the two remaining inline cancel sites (cancelPlanTasks,
|
|
364
|
-
* cancelRunFromSignal) route through this helper via `transform` so EVERY
|
|
365
|
-
* cancel site uses the single shared transform. Their extra logic
|
|
366
|
-
* (graph mutation / terminalEvidence) is preserved inside the transform.
|
|
367
|
-
*
|
|
368
|
-
* `markBlocked` is intentionally NOT unified here (it sets status "skipped",
|
|
369
|
-
* not cancelled/failed, and only acts on "queued" tasks).
|
|
370
|
-
*/
|
|
371
|
-
export function cancelNonTerminalTasks(
|
|
372
|
-
tasks: TeamTaskState[],
|
|
373
|
-
status: "cancelled" | "failed",
|
|
374
|
-
reason: string,
|
|
375
|
-
filter?: (task: TeamTaskState) => boolean,
|
|
376
|
-
transform?: (task: TeamTaskState, terminalised: TeamTaskState) => TeamTaskState,
|
|
377
|
-
): TeamTaskState[] {
|
|
378
|
-
const predicate = filter ?? ((task: TeamTaskState) => isNonTerminalTaskStatus(task.status));
|
|
379
|
-
return tasks.map((task) => {
|
|
380
|
-
if (!predicate(task)) return task;
|
|
381
|
-
const terminalised: TeamTaskState = { ...task, status, finishedAt: new Date().toISOString(), error: reason };
|
|
382
|
-
return transform ? transform(task, terminalised) : terminalised;
|
|
383
|
-
});
|
|
384
|
-
}
|
|
247
|
+
// findStep / findAgent / markBlocked / cancelNonTerminalTasks moved to
|
|
248
|
+
// ./dispatch-batch.ts (2026-08 Phase 2.6) — re-imported above.
|
|
385
249
|
|
|
386
250
|
// 2.8: adaptive-plan parsing/repair/injection moved to src/runtime/goal-workflow/adaptive-plan.ts.
|
|
387
251
|
// Re-export the test-only helpers so existing test imports still resolve.
|
|
@@ -394,315 +258,46 @@ export {
|
|
|
394
258
|
// Re-export the test-only helpers so existing test imports still resolve.
|
|
395
259
|
export { __test__mergeTaskUpdates, __test__shouldMergeTaskUpdate } from "./merge-gate.ts";
|
|
396
260
|
|
|
397
|
-
import { injectAdaptivePlanIfReady } from "./goal-workflow/adaptive-plan.ts";
|
|
261
|
+
import { injectAdaptivePlanIfReady, isAdaptiveWorkflow } from "./goal-workflow/adaptive-plan.ts";
|
|
398
262
|
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
function runEffectivenessLines(
|
|
404
|
-
manifest: TeamRunManifest,
|
|
405
|
-
tasks: TeamTaskState[],
|
|
406
|
-
executeWorkers: boolean,
|
|
407
|
-
runtimeConfig?: CrewRuntimeConfig,
|
|
408
|
-
): string[] {
|
|
409
|
-
return formatRunEffectivenessLines(
|
|
410
|
-
evaluateRunEffectiveness({
|
|
411
|
-
manifest,
|
|
412
|
-
tasks,
|
|
413
|
-
executeWorkers,
|
|
414
|
-
runtimeConfig,
|
|
415
|
-
}),
|
|
416
|
-
);
|
|
417
|
-
}
|
|
418
|
-
|
|
419
|
-
// I5 (plan): surface scratchpad adoption in the run summary — but ONLY when
|
|
420
|
-
// non-zero, so the 3 armed roles that never call it add no noise. Counts the
|
|
421
|
-
// metric events the workers appended to the run events log (fire-and-forget
|
|
422
|
-
// scratchpad.cell / scratchpad.restored). Silent when the feature is unused or
|
|
423
|
-
// the events path is missing (no throw).
|
|
424
|
-
function scratchpadSummaryLines(manifest: TeamRunManifest): string[] {
|
|
425
|
-
if (!manifest.eventsPath) return [];
|
|
426
|
-
const events = readEvents(manifest.eventsPath);
|
|
427
|
-
const cells = events.filter((e) => e.type === "scratchpad.cell");
|
|
428
|
-
const restores = events.filter((e) => e.type === "scratchpad.restored");
|
|
429
|
-
if (cells.length === 0 && restores.length === 0) return [];
|
|
430
|
-
return [
|
|
431
|
-
`## Scratchpad (RLM adoption) — I5 metric`,
|
|
432
|
-
`- cells executed: ${cells.length}`,
|
|
433
|
-
`- snapshot restores: ${restores.length}`,
|
|
434
|
-
...(cells.length > 0
|
|
435
|
-
? [`- total cell time: ${Math.round(cells.reduce((s, e) => s + ((e.data?.durationMs as number) ?? 0), 0))} ms`]
|
|
436
|
-
: []),
|
|
437
|
-
];
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
// P6 (perf): Cache the last-rendered progress content so we can skip the
|
|
441
|
-
// artifact write + redaction + atomic write + size/hash read when nothing
|
|
442
|
-
// material changed (rare between batches, but happens between idle heartbeats).
|
|
443
|
-
// The dedup filter also moved from O(N²) findIndex inside .filter(...)
|
|
444
|
-
// (the previous implementation ran 2 redundant passes on every batch) to
|
|
445
|
-
// a single-pass Map-based replacement: remove the existing entry by path, then
|
|
446
|
-
// append the new one. Net complexity: O(N) build + O(1) replace per write.
|
|
447
|
-
// RT-7: key on manifest.runId (stable string) instead of object identity
|
|
448
|
-
// (WeakMap). Every writeProgress mutator returns a NEW manifest object via
|
|
449
|
-
// spread, so object-identity keying meant the cache NEVER hit. Using runId
|
|
450
|
-
// makes back-to-back calls (same millisecond) actually dedup.
|
|
451
|
-
const lastProgressContentHash = new Map<string, string>();
|
|
452
|
-
|
|
453
|
-
function writeProgress(
|
|
454
|
-
manifest: TeamRunManifest,
|
|
455
|
-
tasks: TeamTaskState[],
|
|
456
|
-
producer: string,
|
|
457
|
-
executeWorkers = true,
|
|
458
|
-
runtimeConfig?: CrewRuntimeConfig,
|
|
459
|
-
): TeamRunManifest {
|
|
460
|
-
const counts = new Map<string, number>();
|
|
461
|
-
for (const task of tasks) counts.set(task.status, (counts.get(task.status) ?? 0) + 1);
|
|
462
|
-
const queue = taskGraphSnapshot(tasks);
|
|
463
|
-
const updatedAt = new Date().toISOString();
|
|
464
|
-
const content = [
|
|
465
|
-
`# pi-crew progress ${manifest.runId}`,
|
|
466
|
-
"",
|
|
467
|
-
`Status: ${manifest.status}`,
|
|
468
|
-
`Team: ${manifest.team}`,
|
|
469
|
-
`Workflow: ${manifest.workflow ?? "(none)"}`,
|
|
470
|
-
`Updated: ${updatedAt}`,
|
|
471
|
-
`Task counts: ${[...counts.entries()].map(([status, count]) => `${status}=${count}`).join(", ") || "none"}`,
|
|
472
|
-
`Queue: ready=${queue.ready.length}, blocked=${queue.blocked.length}, running=${queue.running.length}, done=${queue.done.length}, failed=${queue.failed.length}, cancelled=${queue.cancelled.length}`,
|
|
473
|
-
"",
|
|
474
|
-
"## Tasks",
|
|
475
|
-
...tasks.map(formatTaskProgress),
|
|
476
|
-
"",
|
|
477
|
-
"## Effectiveness",
|
|
478
|
-
...runEffectivenessLines(manifest, tasks, executeWorkers, runtimeConfig),
|
|
479
|
-
"",
|
|
480
|
-
].join("\n");
|
|
481
|
-
|
|
482
|
-
// P6 content-cache: even with identical status / counts / queue, the
|
|
483
|
-
// `Updated:` timestamp ticks on every call so the content rarely matches
|
|
484
|
-
// byte-for-byte. We DO compare against the previous rendered byte-stream
|
|
485
|
-
// (which used the previous timestamp) — so this only hits on the
|
|
486
|
-
// back-to-back writeProgress calls during the applyPolicy phase, where
|
|
487
|
-
// both calls happen within the same millisecond. It's a minor win but
|
|
488
|
-
// matches the audit recommendation (skip artifact write when nothing
|
|
489
|
-
// material changed).
|
|
490
|
-
// RT-7: compute the content hash ONCE (was hashed twice per call: once
|
|
491
|
-
// for the canSkip comparison and again for the cache .set). Key the cache
|
|
492
|
-
// on manifest.runId (stable) instead of object identity (never hit).
|
|
493
|
-
const contentHash = hashContent(content);
|
|
494
|
-
const prevHash = lastProgressContentHash.get(manifest.runId);
|
|
495
|
-
// Cheap pre-check: avoid the redaction + atomicWrite + readback roundtrip
|
|
496
|
-
// when both the timestamp and the input args are identical to last time.
|
|
497
|
-
const canSkip = prevHash === contentHash;
|
|
498
|
-
|
|
499
|
-
const progress = canSkip
|
|
500
|
-
? (() => {
|
|
501
|
-
// Reuse the previous artifact rather than rebuilding one via
|
|
502
|
-
// writeArtifact. This skips mkdirSync, resolveRealContainedPath,
|
|
503
|
-
// redactSecrets, atomicWriteFile, and the post-write readFileSync +
|
|
504
|
-
// statSync.
|
|
505
|
-
const existing = manifest.artifacts.find((a) => a.kind === "progress");
|
|
506
|
-
if (existing) {
|
|
507
|
-
// RT-7a: return a FRESH descriptor with a refreshed createdAt
|
|
508
|
-
// instead of reusing the stale existing reference. The existing
|
|
509
|
-
// descriptor's createdAt reflects the FIRST write time, not this
|
|
510
|
-
// skip-write; refreshing it matches the non-skip path (writeArtifact
|
|
511
|
-
// stamps createdAt with the actual write time) so the manifest
|
|
512
|
-
// always carries a descriptor whose createdAt reflects the current
|
|
513
|
-
// write. Content is identical (that's why we skipped), so path /
|
|
514
|
-
// sizeBytes / contentHash / retention are unchanged.
|
|
515
|
-
return { ...existing, createdAt: new Date().toISOString() };
|
|
516
|
-
}
|
|
517
|
-
// No prior progress artifact (rare; first call from a stale manifest
|
|
518
|
-
// view). Fall through to the normal write.
|
|
519
|
-
return writeArtifact(manifest.artifactsRoot, {
|
|
520
|
-
kind: "progress",
|
|
521
|
-
relativePath: "progress.md",
|
|
522
|
-
producer,
|
|
523
|
-
content,
|
|
524
|
-
});
|
|
525
|
-
})()
|
|
526
|
-
: writeArtifact(manifest.artifactsRoot, {
|
|
527
|
-
kind: "progress",
|
|
528
|
-
relativePath: "progress.md",
|
|
529
|
-
producer,
|
|
530
|
-
content,
|
|
531
|
-
});
|
|
532
|
-
lastProgressContentHash.set(manifest.runId, contentHash);
|
|
533
|
-
|
|
534
|
-
// P6 dedup: replace by path in a single Map pass instead of
|
|
535
|
-
// .filter(...) // O(N) to remove the old entry
|
|
536
|
-
// .filter((_, i, self) => self.findIndex(...) === i) // O(N²) for dedup
|
|
537
|
-
// For an artifact list of size 30+ across a long run, this was the
|
|
538
|
-
// dominant cost of writeProgress between batches.
|
|
539
|
-
const byPath = new Map<string, ArtifactDescriptor>();
|
|
540
|
-
for (const artifact of manifest.artifacts) {
|
|
541
|
-
if (artifact.kind === "progress" && artifact.path === progress.path) continue;
|
|
542
|
-
byPath.set(artifact.path, artifact);
|
|
543
|
-
}
|
|
544
|
-
byPath.set(progress.path, progress);
|
|
545
|
-
const deduped = [...byPath.values()];
|
|
546
|
-
|
|
547
|
-
return {
|
|
548
|
-
...manifest,
|
|
549
|
-
updatedAt,
|
|
550
|
-
artifacts: deduped,
|
|
551
|
-
};
|
|
552
|
-
}
|
|
263
|
+
// formatTaskProgress / runEffectivenessLines / scratchpadSummaryLines /
|
|
264
|
+
// lastProgressContentHash / writeProgress moved to ./finalize-run.ts (2026-08
|
|
265
|
+
// Phase 2.6) — re-imported above for the core-loop progress writes; seams
|
|
266
|
+
// re-exported below so existing test imports still resolve.
|
|
553
267
|
|
|
554
268
|
/** @internal RT-7 test export — verify cache is keyed on runId (stable string). */
|
|
555
|
-
export const __test__lastProgressContentHash = lastProgressContentHash;
|
|
556
269
|
/** @internal RT-7 test export — exercise writeProgress directly. */
|
|
557
|
-
export
|
|
270
|
+
export { __test__lastProgressContentHash, __test__writeProgress } from "./finalize-run.ts";
|
|
558
271
|
/** @internal RT-14 test export — verify cancelPlanTasks preserves graph mutation after consolidation. */
|
|
559
272
|
export const __test__cancelPlanTasks = cancelPlanTasks;
|
|
560
273
|
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
};
|
|
581
|
-
decisions = [...decisions, branchDecision];
|
|
582
|
-
appendEvent(manifest.eventsPath, {
|
|
583
|
-
type: "branch.stale",
|
|
584
|
-
runId: manifest.runId,
|
|
585
|
-
message: branchFreshness.message,
|
|
586
|
-
data: { branchFreshness },
|
|
587
|
-
});
|
|
588
|
-
}
|
|
589
|
-
const policyArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
590
|
-
kind: "metadata",
|
|
591
|
-
relativePath: "policy-decisions.json",
|
|
592
|
-
producer: "policy-engine",
|
|
593
|
-
content: `${JSON.stringify(decisions, null, 2)}\n`,
|
|
594
|
-
});
|
|
595
|
-
const recoveryLedger = buildRecoveryLedger(decisions);
|
|
596
|
-
const recoveryArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
597
|
-
kind: "metadata",
|
|
598
|
-
relativePath: "recovery-ledger.json",
|
|
599
|
-
producer: "recovery-engine",
|
|
600
|
-
content: `${JSON.stringify(recoveryLedger, null, 2)}\n`,
|
|
601
|
-
});
|
|
602
|
-
for (const item of decisions)
|
|
603
|
-
appendEvent(manifest.eventsPath, {
|
|
604
|
-
type: item.action === "escalate" ? "policy.escalated" : "policy.action",
|
|
605
|
-
runId: manifest.runId,
|
|
606
|
-
taskId: item.taskId,
|
|
607
|
-
message: item.message,
|
|
608
|
-
data: { action: item.action, reason: item.reason },
|
|
609
|
-
});
|
|
610
|
-
for (const item of recoveryLedger.entries)
|
|
611
|
-
appendEvent(manifest.eventsPath, {
|
|
612
|
-
type: item.state === "escalation_required" ? "recovery.escalated" : "recovery.attempted",
|
|
613
|
-
runId: manifest.runId,
|
|
614
|
-
taskId: item.taskId,
|
|
615
|
-
message: item.message,
|
|
616
|
-
data: {
|
|
617
|
-
scenario: item.scenario,
|
|
618
|
-
steps: item.steps,
|
|
619
|
-
attempt: item.attempt,
|
|
620
|
-
state: item.state,
|
|
621
|
-
},
|
|
622
|
-
});
|
|
623
|
-
return {
|
|
624
|
-
...manifest,
|
|
625
|
-
updatedAt: new Date().toISOString(),
|
|
626
|
-
policyDecisions: decisions,
|
|
627
|
-
artifacts: [
|
|
628
|
-
...manifest.artifacts.filter(
|
|
629
|
-
(artifact) =>
|
|
630
|
-
!(
|
|
631
|
-
artifact.kind === "metadata" &&
|
|
632
|
-
(artifact.path.endsWith("policy-decisions.json") ||
|
|
633
|
-
artifact.path.endsWith("recovery-ledger.json") ||
|
|
634
|
-
artifact.path.endsWith("branch-freshness.json"))
|
|
635
|
-
),
|
|
636
|
-
),
|
|
637
|
-
branchArtifact,
|
|
638
|
-
policyArtifact,
|
|
639
|
-
recoveryArtifact,
|
|
640
|
-
],
|
|
641
|
-
};
|
|
642
|
-
}
|
|
643
|
-
|
|
644
|
-
function retryPolicyFromConfig(config: CrewReliabilityConfig | undefined): RetryPolicy {
|
|
645
|
-
return { ...DEFAULT_RETRY_POLICY, ...(config?.retryPolicy ?? {}) };
|
|
646
|
-
}
|
|
647
|
-
|
|
648
|
-
/**
|
|
649
|
-
* #1 (assessment): decide whether the per-task retry path (executeWithRetry) is used.
|
|
650
|
-
* Defaults to TRUE (opt-out) so transient worker hangs (ChildTimeout) are retried
|
|
651
|
-
* automatically. Previously opt-in, which left the entire retry+recovery stack dormant.
|
|
652
|
-
* Exported for unit testing.
|
|
653
|
-
*/
|
|
654
|
-
export function shouldUseRetry(reliability: CrewReliabilityConfig | undefined): boolean {
|
|
655
|
-
return reliability?.autoRetry !== false;
|
|
656
|
-
}
|
|
657
|
-
|
|
658
|
-
function failedTaskFrom(result: { tasks: TeamTaskState[] }, taskId: string): TeamTaskState | undefined {
|
|
659
|
-
return result.tasks.find((item) => item.id === taskId && item.status === "failed");
|
|
660
|
-
}
|
|
661
|
-
|
|
662
|
-
function requiresPlanApproval(_workflow: WorkflowConfig, runtimeConfig: CrewRuntimeConfig | undefined): boolean {
|
|
663
|
-
// ROADMAP T1.2: plan-level HITL applies to ANY workflow when
|
|
664
|
-
// config.runtime.requirePlanApproval === true (not just 'implementation').
|
|
665
|
-
// The gate fires at the read-only → mutating (plan → execute) boundary.
|
|
666
|
-
return runtimeConfig?.requirePlanApproval === true;
|
|
667
|
-
}
|
|
274
|
+
// Budget-enforcement family moved to ./budget-enforcement.ts (2026-08 Phase 2.6).
|
|
275
|
+
// Re-export the public + test-only helpers so existing test imports still resolve.
|
|
276
|
+
export { checkPerTaskBudget, type DrainOutcome, drainPendingUnits, type PerTaskBudgetCheckResult } from "./budget-enforcement.ts";
|
|
277
|
+
/** @internal 1.9(b) test export — exercise selectDispatchBatch directly. */
|
|
278
|
+
export { __test__selectDispatchBatch } from "./dispatch-batch.ts";
|
|
279
|
+
/** @internal R15-1 test export — exercise finalizeRun directly (disk-terminal preservation). */
|
|
280
|
+
export { __test__finalizeRun } from "./finalize-run.ts";
|
|
281
|
+
/** @internal 1.9(b) test export — exercise mergeUnitResult directly. */
|
|
282
|
+
export { __test__mergeUnitResult } from "./merge-loop.ts";
|
|
283
|
+
// 1.9(b): characterization test seams for the Phase 2.6 extraction targets
|
|
284
|
+
// (selectDispatchBatch / mergeUnitResult / advanceWorkflowPhases /
|
|
285
|
+
// requiresPlanApproval / ensurePlanApprovalRequested). These functions are
|
|
286
|
+
// module-private today; re-exporting them lets tests pin CURRENT behavior
|
|
287
|
+
// BEFORE the CORE-4 extraction moves them into scheduler/ modules.
|
|
288
|
+
// Plan-approval family extracted to ./plan-approval.ts (2026-08 Phase 2.6).
|
|
289
|
+
// Re-export the test-only helpers so existing test imports still resolve.
|
|
290
|
+
export { __test__ensurePlanApprovalRequested, __test__requiresPlanApproval } from "./plan-approval.ts";
|
|
291
|
+
/** @internal 1.9(b) test export — exercise advanceWorkflowPhases directly. */
|
|
292
|
+
export { __test__advanceWorkflowPhases } from "./workflow-phase-advance.ts";
|
|
668
293
|
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
}
|
|
294
|
+
// applyPolicy moved to ./finalize-run.ts (2026-08 Phase 2.6) — it is only
|
|
295
|
+
// used by finalizeRun, which also moved there.
|
|
672
296
|
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
async function ensurePlanApprovalRequested(manifest: TeamRunManifest, tasks: TeamTaskState[]): Promise<TeamRunManifest> {
|
|
678
|
-
if (manifest.planApproval) return manifest;
|
|
679
|
-
const assessTask = tasks.find((task) => task.stepId === "assess" && task.status === "completed");
|
|
680
|
-
// ROADMAP T1.2: for non-adaptive workflows, fall back to the most recent
|
|
681
|
-
// completed read-only (planning) task as the plan reference.
|
|
682
|
-
const planTask = assessTask ?? [...tasks].reverse().find((t) => t.status === "completed" && !isMutatingTask(t));
|
|
683
|
-
const now = new Date().toISOString();
|
|
684
|
-
const updated: TeamRunManifest = {
|
|
685
|
-
...manifest,
|
|
686
|
-
updatedAt: now,
|
|
687
|
-
planApproval: {
|
|
688
|
-
required: true,
|
|
689
|
-
status: "pending",
|
|
690
|
-
requestedAt: now,
|
|
691
|
-
updatedAt: now,
|
|
692
|
-
planTaskId: planTask?.id,
|
|
693
|
-
planArtifactPath: planTask?.resultArtifact?.path,
|
|
694
|
-
},
|
|
695
|
-
};
|
|
696
|
-
await saveRunManifestAsync(updated);
|
|
697
|
-
appendEvent(updated.eventsPath, {
|
|
698
|
-
type: "plan.approval_required",
|
|
699
|
-
runId: updated.runId,
|
|
700
|
-
taskId: planTask?.id,
|
|
701
|
-
message: "Plan requires explicit approval before mutating tasks run. Use: team api op=approve-plan runId=...",
|
|
702
|
-
data: { planArtifactPath: planTask?.resultArtifact?.path },
|
|
703
|
-
});
|
|
704
|
-
return updated;
|
|
705
|
-
}
|
|
297
|
+
// shouldUseRetry / failedTaskFrom / retryPolicyFromConfig moved to
|
|
298
|
+
// ./dispatch-batch.ts (2026-08 Phase 2.6) — shouldUseRetry re-exported
|
|
299
|
+
// below so existing test imports still resolve.
|
|
300
|
+
export { shouldUseRetry } from "./dispatch-batch.ts";
|
|
706
301
|
|
|
707
302
|
function cancelPlanTasks(tasks: TeamTaskState[], reason: string): TeamTaskState[] {
|
|
708
303
|
// RT-14: delegate to the shared cancelNonTerminalTasks helper. The
|
|
@@ -731,75 +326,8 @@ export function hasPendingMutatingTaskAtBoundary(tasks: TeamTaskState[]): boolea
|
|
|
731
326
|
return hasCompletedReadOnly && hasPendingMutating;
|
|
732
327
|
}
|
|
733
328
|
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
* execution planning. If so, build an execution plan and use `getDagReadyTasks`
|
|
737
|
-
* to augment the ready-set selection.
|
|
738
|
-
*/
|
|
739
|
-
function dagReadyTaskIds(tasks: TeamTaskState[], completedIds: Set<string>): string[] | null {
|
|
740
|
-
const hasExplicitDeps = tasks.some((t) => t.dependsOn.length > 0);
|
|
741
|
-
if (!hasExplicitDeps) return null;
|
|
742
|
-
// FIX (goal-wrap runtime test): task.dependsOn stores STEP IDs (e.g. "execute"), not
|
|
743
|
-
// task IDs (e.g. "02_execute"). The DAG scheduler compares deps against completedIds
|
|
744
|
-
// (which are task IDs), so step-ID deps would never match → dependent tasks stuck blocked
|
|
745
|
-
// forever. Map step IDs -> task IDs first (mirror dependencySatisfied in
|
|
746
|
-
// task-graph-scheduler.ts which handles this via stepToTaskId). buildDagExecutionPlan +
|
|
747
|
-
// getDagReadyTasks then work on consistent task IDs.
|
|
748
|
-
const stepToTaskId = new Map<string, string>();
|
|
749
|
-
for (const t of tasks) {
|
|
750
|
-
if (t.stepId) stepToTaskId.set(t.stepId, t.id);
|
|
751
|
-
}
|
|
752
|
-
const nodes: TaskNode[] = tasks.map((t) => ({
|
|
753
|
-
id: t.id,
|
|
754
|
-
dependsOn: t.dependsOn.map((dep) => stepToTaskId.get(dep) ?? dep),
|
|
755
|
-
phase: t.adaptive?.phase ?? t.stepId,
|
|
756
|
-
}));
|
|
757
|
-
const plan = buildDagExecutionPlan(nodes);
|
|
758
|
-
if (plan.hasCycle) return null; // fall back to existing scheduler
|
|
759
|
-
return getDagReadyTasks(plan, completedIds);
|
|
760
|
-
}
|
|
761
|
-
|
|
762
|
-
/** RT-12: result shape from a settled dispatch unit (pre-created wrapper). */
|
|
763
|
-
type SettledUnit = {
|
|
764
|
-
unitKey: string;
|
|
765
|
-
result: { manifest: TeamRunManifest; tasks: TeamTaskState[] } | undefined;
|
|
766
|
-
error: Error | undefined;
|
|
767
|
-
};
|
|
768
|
-
|
|
769
|
-
/**
|
|
770
|
-
* RT-12: in-flight dispatch unit. `wrapped` is a pre-created wrapper promise
|
|
771
|
-
* (try/catch → SettledUnit) so mergeUnitResult can Promise.race without
|
|
772
|
-
* allocating new async closures every loop iteration (O(C) total wrappers
|
|
773
|
-
* instead of O(C×T) churn).
|
|
774
|
-
*/
|
|
775
|
-
type PendingUnit = {
|
|
776
|
-
taskIds: string[];
|
|
777
|
-
promise: Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }>;
|
|
778
|
-
wrapped: Promise<SettledUnit>;
|
|
779
|
-
};
|
|
780
|
-
|
|
781
|
-
/**
|
|
782
|
-
* Drain in-flight dispatch units (pendingUnits) by aborting the run-scoped
|
|
783
|
-
* controller and awaiting all settled promises before clearing the map.
|
|
784
|
-
*
|
|
785
|
-
* CORE-1 fix: without this, every early-return path inside the main while
|
|
786
|
-
* loop would abandon pendingUnits — leaving zombie child processes running
|
|
787
|
-
* with no one listening for their results.
|
|
788
|
-
*
|
|
789
|
-
* Exported so unit tests can exercise it directly.
|
|
790
|
-
*/
|
|
791
|
-
/** Settled outcome of a single in-flight dispatch unit promise (returned by drainPendingUnits). */
|
|
792
|
-
export type DrainOutcome = PromiseSettledResult<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }>;
|
|
793
|
-
|
|
794
|
-
export async function drainPendingUnits<
|
|
795
|
-
T extends { taskIds: string[]; promise: Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> },
|
|
796
|
-
>(pendingUnits: Map<string, T>, controller?: AbortController): Promise<DrainOutcome[]> {
|
|
797
|
-
if (pendingUnits.size === 0) return [];
|
|
798
|
-
controller?.abort();
|
|
799
|
-
const outcomes = await Promise.allSettled([...pendingUnits.values()].map((p) => p.promise));
|
|
800
|
-
pendingUnits.clear();
|
|
801
|
-
return outcomes;
|
|
802
|
-
}
|
|
329
|
+
// drainPendingUnits / DrainOutcome moved to ./budget-enforcement.ts (2026-08
|
|
330
|
+
// Phase 2.6) — re-exported below so existing test imports still resolve.
|
|
803
331
|
|
|
804
332
|
export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
|
|
805
333
|
const workflow = input.workflow;
|
|
@@ -854,8 +382,9 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
854
382
|
const stopTeamHeartbeat = startTeamRunHeartbeat(manifest.stateRoot, manifest.runId);
|
|
855
383
|
// Perf observability: auto-attach the resource sampler for this run (toggle:
|
|
856
384
|
// team frontmatter `observability: false`). Detached + unref'd — the sampler
|
|
857
|
-
// auto-stops when the runner dies, so no explicit cleanup needed.
|
|
858
|
-
|
|
385
|
+
// auto-stops when the runner dies, so no explicit cleanup needed. R11-4:
|
|
386
|
+
// input.signal is threaded so run teardown/cancel also kills the sampler.
|
|
387
|
+
startPerfSampler(manifest, input.team, input.signal);
|
|
859
388
|
|
|
860
389
|
const cleanupUsage = (): void => {
|
|
861
390
|
for (const task of input.tasks) clearTrackedTaskUsage(task.id);
|
|
@@ -939,8 +468,9 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
939
468
|
// before manifest updates are observed by readers.
|
|
940
469
|
await flushEventLogBuffer();
|
|
941
470
|
// Perf observability: emit the post-run perf report (detached, delayed so
|
|
942
|
-
// child transcripts are flushed). Never affects run outcome.
|
|
943
|
-
|
|
471
|
+
// child transcripts are flushed). Never affects run outcome. R11-4: thread
|
|
472
|
+
// input.signal so the analyze child dies on run teardown too.
|
|
473
|
+
schedulePerfAnalyze(manifest, input.team, input.signal);
|
|
944
474
|
return result;
|
|
945
475
|
} catch (error) {
|
|
946
476
|
// Round 27 (BUG 1): the success path calls stopTeamHeartbeat() but this
|
|
@@ -1043,60 +573,10 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
1043
573
|
}
|
|
1044
574
|
|
|
1045
575
|
// ── CORE-4: SchedulerContext state bag ─────────────────────────────
|
|
1046
|
-
//
|
|
1047
|
-
//
|
|
1048
|
-
//
|
|
1049
|
-
//
|
|
1050
|
-
|
|
1051
|
-
/**
|
|
1052
|
-
* Mutable state shared across the team-run scheduler loop.
|
|
1053
|
-
*
|
|
1054
|
-
* Fields mirror the closure locals of `executeTeamRunCore`. Extracted
|
|
1055
|
-
* scheduler functions mutate these fields in-place; the caller keeps the
|
|
1056
|
-
* local variables in sync by assigning back from `ctx` after each call.
|
|
1057
|
-
*/
|
|
1058
|
-
interface SchedulerContext {
|
|
1059
|
-
input: ExecuteTeamRunInput;
|
|
1060
|
-
workflow: WorkflowConfig;
|
|
1061
|
-
manifest: TeamRunManifest;
|
|
1062
|
-
tasks: TeamTaskState[];
|
|
1063
|
-
queueIndex: TaskGraphIndex;
|
|
1064
|
-
wfMachine: WorkflowStateMachine;
|
|
1065
|
-
pendingUnits: Map<string, PendingUnit>;
|
|
1066
|
-
/** Task ids ever dispatched (grows monotonically; never removed). Used by
|
|
1067
|
-
* terminaliseRunWithDrain to cancel — not skip — tasks that were in-flight
|
|
1068
|
-
* even after their dispatch unit settled + left pendingUnits (RT-NEW-2 race). */
|
|
1069
|
-
dispatchedTaskIds: Set<string>;
|
|
1070
|
-
runController: AbortController;
|
|
1071
|
-
runtimeKind: CrewRuntimeKind;
|
|
1072
|
-
adaptivePlanInjected: boolean;
|
|
1073
|
-
adaptivePlanMissing: boolean;
|
|
1074
|
-
/** Outcome of the most recent mergeUnitResult call: the settled unit's
|
|
1075
|
-
* taskIds and the merged result object. Read by the post-merge inline
|
|
1076
|
-
* logic (cancel-during-exec check + batch summary). Set by extraction 5. */
|
|
1077
|
-
settledMerge: { taskIds: string[]; result: { manifest: TeamRunManifest; tasks: TeamTaskState[] } } | null;
|
|
1078
|
-
}
|
|
1079
|
-
|
|
1080
|
-
/**
|
|
1081
|
-
* Discriminated union representing a scheduler sub-function's decision.
|
|
1082
|
-
*
|
|
1083
|
-
* - `continue`: proceed to the next phase of the loop body.
|
|
1084
|
-
* - `return`: short-circuit the loop and return the given result.
|
|
1085
|
-
* - `skip-dispatch`: skip the dispatch phase this iteration (reserved for
|
|
1086
|
-
* future extractions).
|
|
1087
|
-
*/
|
|
1088
|
-
type SchedulerDecision =
|
|
1089
|
-
| { kind: "continue" }
|
|
1090
|
-
| { kind: "return"; result: { manifest: TeamRunManifest; tasks: TeamTaskState[] } }
|
|
1091
|
-
| { kind: "skip-dispatch" }
|
|
1092
|
-
| {
|
|
1093
|
-
kind: "dispatch";
|
|
1094
|
-
batch: TeamTaskState[];
|
|
1095
|
-
concurrency: BatchConcurrencyDecision;
|
|
1096
|
-
snapshot: TaskGraphSchedulerSnapshot;
|
|
1097
|
-
approvalPending: boolean;
|
|
1098
|
-
coalesceEnabled: boolean;
|
|
1099
|
-
};
|
|
576
|
+
// Moved to ./scheduler-context.ts (2026-08 Phase 2.6) — SchedulerContext,
|
|
577
|
+
// SettledUnit, PendingUnit, SchedulerDecision now live there and are
|
|
578
|
+
// re-imported above. This comment marks the extraction boundary in the
|
|
579
|
+
// scheduler loop below.
|
|
1100
580
|
|
|
1101
581
|
/**
|
|
1102
582
|
* RT-13: Safely normalize the manifest status to "running" so a subsequent
|
|
@@ -1210,83 +690,8 @@ async function cancelRunFromSignal(ctx: SchedulerContext): Promise<SchedulerDeci
|
|
|
1210
690
|
* @param ctx The scheduler context; `ctx.tasks` and `ctx.manifest` are
|
|
1211
691
|
* mutated in-place to reflect the rerun or abort.
|
|
1212
692
|
*/
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
* units FIRST and merging their settled results under the run lock.
|
|
1216
|
-
*
|
|
1217
|
-
* Extracted verbatim from handleFailedTask (the FIXED reference) so every
|
|
1218
|
-
* abort path behaves identically: drain pendingUnits (abort controller +
|
|
1219
|
-
* await allSettled + clear), merge fulfilled outcomes into manifest/tasks
|
|
1220
|
-
* under withRunLock (flushPendingAtomicWrites + loadRunManifestById +
|
|
1221
|
-
* mergeArtifacts + mergeTaskUpdatesPreservingTerminal + save both), cancel
|
|
1222
|
-
* (not skip) non-settled in-flight tasks so team resume can re-queue them,
|
|
1223
|
-
* then markBlocked the remaining never-dispatched queued tasks.
|
|
1224
|
-
*
|
|
1225
|
-
* Previously enforceRunBudget skipped the drain+merge and called
|
|
1226
|
-
* markBlocked directly — in-flight tasks (still "queued" in ctx.tasks since
|
|
1227
|
-
* streaming dispatch never sets "running") were clobbered to "skipped",
|
|
1228
|
-
* which team resume never re-queues → permanent work loss.
|
|
1229
|
-
*/
|
|
1230
|
-
async function terminaliseRunWithDrain(
|
|
1231
|
-
ctx: SchedulerContext,
|
|
1232
|
-
opts: { cancelMessage: string; blockedMessage: string; failedReason: string },
|
|
1233
|
-
): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
|
|
1234
|
-
// Ever-dispatched tasks (monotonic set populated at dispatch). Using this
|
|
1235
|
-
// instead of a pendingUnits snapshot closes the RT-NEW-2 race where a task
|
|
1236
|
-
// whose unit settled + left pendingUnits before the abort — but whose task
|
|
1237
|
-
// status isn't terminal yet — would otherwise fall through to markBlocked
|
|
1238
|
-
// and be clobbered to "skipped" (observed CI flake: 02_b skipped on
|
|
1239
|
-
// team-runner-budget-abort-inflight across v0.9.59 / cfd68d06 / 12386af2).
|
|
1240
|
-
const inflightTaskIds = ctx.dispatchedTaskIds;
|
|
1241
|
-
const outcomes = await drainPendingUnits(ctx.pendingUnits, ctx.runController);
|
|
1242
|
-
const validResults: { manifest: TeamRunManifest; tasks: TeamTaskState[] }[] = [];
|
|
1243
|
-
for (const outcome of outcomes) {
|
|
1244
|
-
if (outcome.status === "fulfilled") validResults.push(outcome.value);
|
|
1245
|
-
}
|
|
1246
|
-
if (validResults.length > 0) {
|
|
1247
|
-
// Merge under the run lock — same pattern as mergeUnitResult:
|
|
1248
|
-
// flush pending writes, load disk state, merge artifacts + tasks,
|
|
1249
|
-
// save atomically.
|
|
1250
|
-
const mergeResult = await withRunLock(ctx.manifest, async () => {
|
|
1251
|
-
flushPendingAtomicWrites();
|
|
1252
|
-
const disk = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
|
|
1253
|
-
const diskManifest = disk?.manifest ?? ctx.manifest;
|
|
1254
|
-
const reconciledArtifacts = mergeArtifacts([
|
|
1255
|
-
...diskManifest.artifacts,
|
|
1256
|
-
...validResults.flatMap((item) => item.manifest.artifacts),
|
|
1257
|
-
]);
|
|
1258
|
-
const resultManifest = updateRunStatus(
|
|
1259
|
-
{ ...diskManifest, artifacts: reconciledArtifacts },
|
|
1260
|
-
"running",
|
|
1261
|
-
"Merged in-flight results during failed-task abort.",
|
|
1262
|
-
);
|
|
1263
|
-
const resultTasks = mergeTaskUpdatesPreservingTerminal(disk?.tasks ?? ctx.tasks, validResults);
|
|
1264
|
-
await saveRunManifestAsync(resultManifest);
|
|
1265
|
-
await saveRunTasksAsync(resultManifest, resultTasks);
|
|
1266
|
-
return { resultManifest, resultTasks };
|
|
1267
|
-
});
|
|
1268
|
-
ctx.manifest = mergeResult.resultManifest;
|
|
1269
|
-
ctx.tasks = mergeResult.resultTasks;
|
|
1270
|
-
}
|
|
1271
|
-
// Cancel in-flight tasks that did NOT settle (e.g. rejected promises)
|
|
1272
|
-
// so team resume CAN re-queue them. markBlocked maps queued→skipped,
|
|
1273
|
-
// which resume never re-queues — work would be lost permanently. Only
|
|
1274
|
-
// cancel tasks that are both in-flight AND still non-terminal (settled
|
|
1275
|
-
// tasks with a terminal status are preserved).
|
|
1276
|
-
ctx.tasks = cancelNonTerminalTasks(
|
|
1277
|
-
ctx.tasks,
|
|
1278
|
-
"cancelled",
|
|
1279
|
-
opts.cancelMessage,
|
|
1280
|
-
(task) => inflightTaskIds.has(task.id) && isNonTerminalTaskStatus(task.status),
|
|
1281
|
-
);
|
|
1282
|
-
// Remaining queued tasks (never dispatched) → skipped (original behavior
|
|
1283
|
-
// preserved for downstream tasks not yet in-flight).
|
|
1284
|
-
ctx.tasks = markBlocked(ctx.tasks, opts.blockedMessage);
|
|
1285
|
-
await saveRunTasksAsync(ctx.manifest, ctx.tasks);
|
|
1286
|
-
saveCrewAgents(ctx.manifest, recordsForMaterializedTasks(ctx.manifest, ctx.tasks, ctx.runtimeKind));
|
|
1287
|
-
ctx.manifest = updateRunStatus(ctx.manifest, "failed", opts.failedReason);
|
|
1288
|
-
return { manifest: ctx.manifest, tasks: ctx.tasks };
|
|
1289
|
-
}
|
|
693
|
+
// terminaliseRunWithDrain moved to ./budget-enforcement.ts (2026-08 Phase 2.6)
|
|
694
|
+
// — handleFailedTask imports it from there.
|
|
1290
695
|
|
|
1291
696
|
async function handleFailedTask(ctx: SchedulerContext): Promise<SchedulerDecision | null> {
|
|
1292
697
|
const failed = ctx.tasks.find((task) => task.status === "failed");
|
|
@@ -1338,1003 +743,14 @@ async function handleFailedTask(ctx: SchedulerContext): Promise<SchedulerDecisio
|
|
|
1338
743
|
return { kind: "return", result };
|
|
1339
744
|
}
|
|
1340
745
|
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
* iteration.
|
|
1344
|
-
*
|
|
1345
|
-
* Computes the task-graph snapshot, DAG-ready tasks, workflow phase
|
|
1346
|
-
* preconditions, batch concurrency, write-path-overlap serialization,
|
|
1347
|
-
* coalesced-group logging, and streaming-dispatch slot allocation to
|
|
1348
|
-
* determine which tasks are ready to dispatch this cycle.
|
|
1349
|
-
*
|
|
1350
|
-
* Returns:
|
|
1351
|
-
* - `{ kind: "return", result }` when the run must block or abort
|
|
1352
|
-
* (plan-approval pending with mutating tasks, or no ready task at all).
|
|
1353
|
-
* - `{ kind: "dispatch", batch, ... }` when a batch is selected (may be
|
|
1354
|
-
* empty when tasks are still in-flight — the caller proceeds to the
|
|
1355
|
-
* wait phase with an empty dispatch set).
|
|
1356
|
-
*
|
|
1357
|
-
* The caller syncs `ctx.wfMachine` back after the call because this
|
|
1358
|
-
* function may advance the workflow phase state machine.
|
|
1359
|
-
*
|
|
1360
|
-
* @param ctx The scheduler context; `ctx.wfMachine`, `ctx.tasks`, and
|
|
1361
|
-
* `ctx.manifest` may be mutated in-place.
|
|
1362
|
-
*/
|
|
1363
|
-
async function selectDispatchBatch(ctx: SchedulerContext): Promise<SchedulerDecision> {
|
|
1364
|
-
const snapshot = taskGraphSnapshot(ctx.tasks, ctx.queueIndex);
|
|
1365
|
-
|
|
1366
|
-
// DAG-based execution plan: when tasks have explicit dependsOn, use the
|
|
1367
|
-
// topological wave planner to determine ready tasks. Fall back to the
|
|
1368
|
-
// existing task-graph-scheduler when no explicit deps exist (backward compat).
|
|
1369
|
-
const completedIds = new Set(ctx.tasks.filter((t) => t.status === "completed" || t.status === "needs_attention").map((t) => t.id));
|
|
1370
|
-
const dagReady = dagReadyTaskIds(ctx.tasks, completedIds);
|
|
1371
|
-
const readyBeforeFilter = dagReady ?? snapshot.ready;
|
|
746
|
+
// mergeUnitResult / isRunTerminalPreserved moved to ./merge-loop.ts (2026-08
|
|
747
|
+
// Phase 2.6) — re-imported above for the core-loop merge + terminal break.
|
|
1372
748
|
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
const completedArtifacts = ctx.manifest.artifacts.filter((a) => a.kind === "result" || a.kind === "summary").map((a) => a.path);
|
|
1376
|
-
const previousPhaseStatus =
|
|
1377
|
-
ctx.wfMachine.currentPhaseIndex > 0
|
|
1378
|
-
? (ctx.wfMachine.phases[ctx.wfMachine.currentPhaseIndex - 1]?.status ?? "pending")
|
|
1379
|
-
: "completed";
|
|
1380
|
-
const wfContext: PhaseGuardContext = {
|
|
1381
|
-
completedArtifacts,
|
|
1382
|
-
previousPhaseStatus,
|
|
1383
|
-
taskResults: ctx.tasks
|
|
1384
|
-
.filter((t) => t.status === "completed" || t.status === "needs_attention")
|
|
1385
|
-
.map((t) => ({
|
|
1386
|
-
taskId: t.id,
|
|
1387
|
-
status: t.status,
|
|
1388
|
-
outputPath: t.resultArtifact?.path,
|
|
1389
|
-
})),
|
|
1390
|
-
};
|
|
1391
|
-
const preconditions = validatePhasePreconditions(ctx.wfMachine, wfContext);
|
|
1392
|
-
if (!preconditions.ready) {
|
|
1393
|
-
await appendEventAsync(ctx.manifest.eventsPath, {
|
|
1394
|
-
type: "workflow.preconditions",
|
|
1395
|
-
runId: ctx.manifest.runId,
|
|
1396
|
-
message: `Workflow phase '${ctx.wfMachine.phases[ctx.wfMachine.currentPhaseIndex]?.name}' is missing inputs: ${preconditions.blocking.join(", ")}`,
|
|
1397
|
-
data: {
|
|
1398
|
-
phaseIndex: ctx.wfMachine.currentPhaseIndex,
|
|
1399
|
-
phaseName: ctx.wfMachine.phases[ctx.wfMachine.currentPhaseIndex]?.name,
|
|
1400
|
-
blocking: preconditions.blocking,
|
|
1401
|
-
},
|
|
1402
|
-
});
|
|
1403
|
-
} else {
|
|
1404
|
-
// Advance the machine past completed phases.
|
|
1405
|
-
while (
|
|
1406
|
-
ctx.wfMachine.currentPhaseIndex < ctx.wfMachine.phases.length &&
|
|
1407
|
-
ctx.wfMachine.phases[ctx.wfMachine.currentPhaseIndex]?.status === "completed"
|
|
1408
|
-
) {
|
|
1409
|
-
ctx.wfMachine = {
|
|
1410
|
-
...ctx.wfMachine,
|
|
1411
|
-
currentPhaseIndex: ctx.wfMachine.currentPhaseIndex + 1,
|
|
1412
|
-
};
|
|
1413
|
-
}
|
|
1414
|
-
}
|
|
1415
|
-
}
|
|
749
|
+
// enforceRunBudget moved to ./budget-enforcement.ts (2026-08 Phase 2.6) — the
|
|
750
|
+
// scheduler loop imports it from there.
|
|
1416
751
|
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
const readyRoles = readyBeforeFilter.map((taskId) => taskByIdReady.get(taskId)?.role).filter((role): role is string => Boolean(role));
|
|
1420
|
-
const concurrency = resolveBatchConcurrency({
|
|
1421
|
-
workflowName: ctx.workflow.name,
|
|
1422
|
-
workflowMaxConcurrency: ctx.workflow.maxConcurrency,
|
|
1423
|
-
teamMaxConcurrency: ctx.input.team.maxConcurrency,
|
|
1424
|
-
limitMaxConcurrentWorkers: ctx.input.limits?.maxConcurrentWorkers,
|
|
1425
|
-
allowUnboundedConcurrency: ctx.input.limits?.allowUnboundedConcurrency,
|
|
1426
|
-
readyCount: readyBeforeFilter.length,
|
|
1427
|
-
workspaceMode: ctx.manifest.workspaceMode,
|
|
1428
|
-
readyRoles,
|
|
1429
|
-
});
|
|
1430
|
-
|
|
1431
|
-
// Round 25 (M5): serialize on write-path overlap when opted in.
|
|
1432
|
-
// Opt-in via limits.serializeOnPathOverlap; default off (= no behavior change).
|
|
1433
|
-
// filterReadyByWriteOverlap returns the same array when enabled=false, so
|
|
1434
|
-
// production runs pay nothing for the unused code path. When the flag is on,
|
|
1435
|
-
// `serializedReady` MAY be a strict subset of `readyBeforeFilter` (conflicting tasks
|
|
1436
|
-
// deferred to next cycle).
|
|
1437
|
-
const serializedReady = filterReadyByWriteOverlap(
|
|
1438
|
-
readyBeforeFilter,
|
|
1439
|
-
ctx.tasks,
|
|
1440
|
-
ctx.workflow,
|
|
1441
|
-
concurrency.maxConcurrent,
|
|
1442
|
-
ctx.input.limits?.serializeOnPathOverlap === true,
|
|
1443
|
-
);
|
|
1444
|
-
|
|
1445
|
-
// Round 25 (M6): coalesce micro-tasks when opted in.
|
|
1446
|
-
// Default off; when on, groups same-(role,cwd) tasks into coalesced groups
|
|
1447
|
-
// (with write-path safety). In v0.9.17 first ship, we ONLY log the
|
|
1448
|
-
// coalesced group count to the event stream (informational). Actual
|
|
1449
|
-
// dispatching of one-multi-task worker instead of N workers is deferred
|
|
1450
|
-
// to a follow-up — it's a non-trivial prompt-construction change that
|
|
1451
|
-
// deserves its own PR. For now, every coalesced group => one info event.
|
|
1452
|
-
const coalesceEnabled = ctx.workflow.coalesceMicroTasks === true;
|
|
1453
|
-
if (coalesceEnabled) {
|
|
1454
|
-
const coalescedGroups = planCoalescedGroups(serializedReady, ctx.tasks, ctx.workflow, true);
|
|
1455
|
-
for (const group of coalescedGroups) {
|
|
1456
|
-
if (group.tasks.length < 2) continue; // singletons are not interesting
|
|
1457
|
-
await appendEventAsync(ctx.manifest.eventsPath, {
|
|
1458
|
-
type: "task.coalesced",
|
|
1459
|
-
runId: ctx.manifest.runId,
|
|
1460
|
-
message: `Coalesced ${group.tasks.length} micro-tasks (role=${group.role}, cwd=${group.cwd})`,
|
|
1461
|
-
data: {
|
|
1462
|
-
groupId: group.id,
|
|
1463
|
-
role: group.role,
|
|
1464
|
-
cwd: group.cwd,
|
|
1465
|
-
taskIds: group.tasks.map((task) => task.id),
|
|
1466
|
-
},
|
|
1467
|
-
});
|
|
1468
|
-
}
|
|
1469
|
-
}
|
|
1470
|
-
if (concurrency.reason.includes(";unbounded:")) {
|
|
1471
|
-
await appendEventAsync(ctx.manifest.eventsPath, {
|
|
1472
|
-
type: "limits.unbounded",
|
|
1473
|
-
runId: ctx.manifest.runId,
|
|
1474
|
-
message: "Unbounded worker concurrency was explicitly enabled for this run.",
|
|
1475
|
-
data: {
|
|
1476
|
-
concurrencyReason: concurrency.reason,
|
|
1477
|
-
maxConcurrent: concurrency.maxConcurrent,
|
|
1478
|
-
},
|
|
1479
|
-
});
|
|
1480
|
-
}
|
|
1481
|
-
// ── OPT-01 streaming dispatch: exclude tasks already in-flight, limit
|
|
1482
|
-
// new dispatches to available concurrency slots. ──
|
|
1483
|
-
const inFlightTaskIds = new Set<string>();
|
|
1484
|
-
for (const pendingUnit of ctx.pendingUnits.values()) {
|
|
1485
|
-
for (const taskId of pendingUnit.taskIds) inFlightTaskIds.add(taskId);
|
|
1486
|
-
}
|
|
1487
|
-
const slotsAvailable = Math.max(0, concurrency.maxConcurrent - ctx.pendingUnits.size);
|
|
1488
|
-
const approvalPending = isPlanApprovalPending(ctx.manifest);
|
|
1489
|
-
const dispatchableReady = serializedReady.filter((id) => !inFlightTaskIds.has(id));
|
|
1490
|
-
const readyIds = approvalPending ? dispatchableReady : dispatchableReady.slice(0, slotsAvailable);
|
|
1491
|
-
const taskByIdDispatch = new Map(ctx.tasks.map((t) => [t.id, t] as const));
|
|
1492
|
-
const candidateBatch = readyIds.map((id) => taskByIdDispatch.get(id)).filter((task): task is TeamTaskState => Boolean(task));
|
|
1493
|
-
const readyBatch = approvalPending ? candidateBatch.filter((task) => !isMutatingTask(task)).slice(0, slotsAvailable) : candidateBatch;
|
|
1494
|
-
if (readyBatch.length === 0) {
|
|
1495
|
-
if (ctx.pendingUnits.size > 0) {
|
|
1496
|
-
// Tasks are in-flight — skip dispatch and proceed to wait phase.
|
|
1497
|
-
// (No return; code falls through to the dispatch section which is
|
|
1498
|
-
// a no-op with an empty readyBatch, then reaches the wait phase.)
|
|
1499
|
-
} else if (approvalPending && candidateBatch.some(isMutatingTask)) {
|
|
1500
|
-
await saveRunTasksAsync(ctx.manifest, ctx.tasks);
|
|
1501
|
-
saveCrewAgents(ctx.manifest, recordsForMaterializedTasks(ctx.manifest, ctx.tasks, ctx.runtimeKind));
|
|
1502
|
-
ctx.manifest = updateRunStatus(ctx.manifest, "blocked", "Plan approval required before mutating implementation tasks run.");
|
|
1503
|
-
return { kind: "return", result: { manifest: ctx.manifest, tasks: ctx.tasks } };
|
|
1504
|
-
} else {
|
|
1505
|
-
ctx.tasks = markBlocked(ctx.tasks, "No ready queued task; dependency graph may be invalid.");
|
|
1506
|
-
await saveRunTasksAsync(ctx.manifest, ctx.tasks);
|
|
1507
|
-
saveCrewAgents(ctx.manifest, recordsForMaterializedTasks(ctx.manifest, ctx.tasks, ctx.runtimeKind));
|
|
1508
|
-
ctx.manifest = updateRunStatus(ctx.manifest, "blocked", "No ready queued task.");
|
|
1509
|
-
return { kind: "return", result: { manifest: ctx.manifest, tasks: ctx.tasks } };
|
|
1510
|
-
}
|
|
1511
|
-
}
|
|
1512
|
-
|
|
1513
|
-
return { kind: "dispatch", batch: readyBatch, concurrency, snapshot, approvalPending, coalesceEnabled };
|
|
1514
|
-
}
|
|
1515
|
-
|
|
1516
|
-
/** Dispatch decision variant returned by selectDispatchBatch. */
|
|
1517
|
-
type DispatchBatchDecision = Extract<SchedulerDecision, { kind: "dispatch" }>;
|
|
1518
|
-
|
|
1519
|
-
/**
|
|
1520
|
-
* CORE-4 extraction 4: execute the dispatch batch selected by
|
|
1521
|
-
* selectDispatchBatch.
|
|
1522
|
-
*
|
|
1523
|
-
* Runs before_task_start hooks (skipping blocked tasks), builds coalesced
|
|
1524
|
-
* dispatch units, pre-warms the stable-prefix cache for unique cwds, and
|
|
1525
|
-
* dispatches each unit into ctx.pendingUnits as a fire-and-forget promise
|
|
1526
|
-
* (wrapped in executeWithRetry on the singleton path). The function is a
|
|
1527
|
-
* verbatim lift of the inline dispatch block; it does not return a
|
|
1528
|
-
* SchedulerDecision (void — it only populates ctx.pendingUnits).
|
|
1529
|
-
*
|
|
1530
|
-
* Reads ctx.manifest/tasks/workflow/input + runController.signal. Mutates
|
|
1531
|
-
* ctx.pendingUnits (add), ctx.tasks (hook skips), ctx.manifest (hook
|
|
1532
|
-
* status). The mutable manifest/tasks are accessed via ctx.* (not captured
|
|
1533
|
-
* locals) so that async retry callbacks observe the caller's re-synced
|
|
1534
|
-
* values, matching the original closure semantics.
|
|
1535
|
-
*
|
|
1536
|
-
* @param ctx The scheduler context.
|
|
1537
|
-
* @param decision The dispatch decision from selectDispatchBatch.
|
|
1538
|
-
*/
|
|
1539
|
-
async function dispatchBatch(ctx: SchedulerContext, decision: DispatchBatchDecision): Promise<void> {
|
|
1540
|
-
const { batch: readyBatch, concurrency, snapshot, approvalPending, coalesceEnabled } = decision;
|
|
1541
|
-
// Immutable context fields captured once; manifest/tasks are accessed via
|
|
1542
|
-
// ctx.* because they may be re-synced by the caller between dispatch and
|
|
1543
|
-
// promise resolution (retry callbacks fire asynchronously).
|
|
1544
|
-
const { workflow, input, runtimeKind, runController } = ctx;
|
|
1545
|
-
|
|
1546
|
-
// 2.2 caller migration: batch progress is high-frequency informational (M7 wire).
|
|
1547
|
-
void appendEventBuffered(ctx.manifest.eventsPath, {
|
|
1548
|
-
type: "task.progress",
|
|
1549
|
-
runId: ctx.manifest.runId,
|
|
1550
|
-
message: `Starting ready batch with ${readyBatch.length} task(s).`,
|
|
1551
|
-
data: {
|
|
1552
|
-
taskIds: readyBatch.map((task) => task.id),
|
|
1553
|
-
readyCount: snapshot.ready.length,
|
|
1554
|
-
blockedCount: snapshot.blocked.length,
|
|
1555
|
-
runningCount: snapshot.running.length,
|
|
1556
|
-
doneCount: snapshot.done.length,
|
|
1557
|
-
selectedCount: readyBatch.length,
|
|
1558
|
-
maxConcurrent: concurrency.maxConcurrent,
|
|
1559
|
-
defaultConcurrency: concurrency.defaultConcurrency,
|
|
1560
|
-
concurrencyReason: approvalPending ? `${concurrency.reason};plan-approval-read-only` : concurrency.reason,
|
|
1561
|
-
},
|
|
1562
|
-
});
|
|
1563
|
-
// Execute before_task_start hooks for the batch — P1-10: run hooks in
|
|
1564
|
-
// parallel (each may be a subprocess), then apply skip mutations in order.
|
|
1565
|
-
const beforeTaskStartReports = await Promise.all(
|
|
1566
|
-
readyBatch.map((task) =>
|
|
1567
|
-
executeHook("before_task_start", {
|
|
1568
|
-
runId: ctx.manifest.runId,
|
|
1569
|
-
taskId: task.id,
|
|
1570
|
-
cwd: ctx.manifest.cwd,
|
|
1571
|
-
}).then((taskReport) => ({ task, taskReport })),
|
|
1572
|
-
),
|
|
1573
|
-
);
|
|
1574
|
-
for (const { task, taskReport } of beforeTaskStartReports) {
|
|
1575
|
-
appendHookEvent(ctx.manifest, taskReport);
|
|
1576
|
-
if (taskReport.outcome === "block") {
|
|
1577
|
-
ctx.tasks = ctx.tasks.map((t) =>
|
|
1578
|
-
t.id === task.id
|
|
1579
|
-
? {
|
|
1580
|
-
...t,
|
|
1581
|
-
status: "skipped" as const,
|
|
1582
|
-
error: taskReport.reason ?? "before_task_start hook blocked execution.",
|
|
1583
|
-
}
|
|
1584
|
-
: t,
|
|
1585
|
-
);
|
|
1586
|
-
ctx.manifest = updateRunStatus(ctx.manifest, ctx.manifest.status, `Task '${task.id}' blocked by hook.`);
|
|
1587
|
-
}
|
|
1588
|
-
}
|
|
1589
|
-
// W5-4: by-id map (was O(readyBatch × tasks) via find-per-element).
|
|
1590
|
-
const ctxTaskById = new Map(ctx.tasks.map((t) => [t.id, t] as const));
|
|
1591
|
-
const batchTasks = readyBatch.filter((task) => {
|
|
1592
|
-
const t = ctxTaskById.get(task.id);
|
|
1593
|
-
return t !== undefined && t.status !== "skipped";
|
|
1594
|
-
});
|
|
1595
|
-
if (batchTasks.length > 1) {
|
|
1596
|
-
await appendEventAsync(ctx.manifest.eventsPath, {
|
|
1597
|
-
type: "task.parallel_start",
|
|
1598
|
-
runId: ctx.manifest.runId,
|
|
1599
|
-
message: `Launching ${batchTasks.length} tasks in PARALLEL (concurrency=${concurrency.selectedCount}): ${batchTasks.map((t) => `${t.role}(${t.id})`).join(", ")}`,
|
|
1600
|
-
data: {
|
|
1601
|
-
taskIds: batchTasks.map((t) => t.id),
|
|
1602
|
-
roles: batchTasks.map((t) => t.role),
|
|
1603
|
-
concurrency: concurrency.selectedCount,
|
|
1604
|
-
},
|
|
1605
|
-
});
|
|
1606
|
-
}
|
|
1607
|
-
|
|
1608
|
-
// M6 real dispatch: when coalesceMicroTasks is enabled, batch the
|
|
1609
|
-
// ready tasks into dispatch units. Multi-task groups are dispatched
|
|
1610
|
-
// as one worker (single cold-start) instead of N. Singletons fall
|
|
1611
|
-
// through to per-task dispatch.
|
|
1612
|
-
const coalescedGroups = planCoalescedGroups(
|
|
1613
|
-
batchTasks.map((t) => t.id),
|
|
1614
|
-
ctx.tasks,
|
|
1615
|
-
workflow,
|
|
1616
|
-
coalesceEnabled,
|
|
1617
|
-
);
|
|
1618
|
-
const dispatchUnits = buildDispatchUnits(
|
|
1619
|
-
batchTasks.map((t) => t.id),
|
|
1620
|
-
coalescedGroups,
|
|
1621
|
-
);
|
|
1622
|
-
|
|
1623
|
-
// NEW-M1: Pre-warm stable prefix cache for one representative task
|
|
1624
|
-
// per unique cwd. Parallel siblings with the same cwd/step reuse
|
|
1625
|
-
// the cached workspace tree, file retrieval, and knowledge fragment
|
|
1626
|
-
// instead of recomputing them independently (~200-800ms per batch).
|
|
1627
|
-
if (batchTasks.length > 1) {
|
|
1628
|
-
const seenCwds = new Set<string>();
|
|
1629
|
-
await Promise.all(
|
|
1630
|
-
batchTasks
|
|
1631
|
-
.filter((task) => {
|
|
1632
|
-
if (seenCwds.has(task.cwd)) return false;
|
|
1633
|
-
seenCwds.add(task.cwd);
|
|
1634
|
-
return true;
|
|
1635
|
-
})
|
|
1636
|
-
.map((task) => {
|
|
1637
|
-
const step = findStep(workflow, task);
|
|
1638
|
-
return computeStablePrefixComponents(ctx.manifest, step, task);
|
|
1639
|
-
}),
|
|
1640
|
-
);
|
|
1641
|
-
}
|
|
1642
|
-
|
|
1643
|
-
// ── OPT-01 streaming dispatch: dispatch each unit into ctx.pendingUnits
|
|
1644
|
-
// instead of awaiting the entire batch via mapConcurrent. Each unit's
|
|
1645
|
-
// promise is stored so we can Promise.race on the next iteration. ──
|
|
1646
|
-
const dispatchUnit = async (unit: DispatchUnit): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> => {
|
|
1647
|
-
// M6 real dispatch path: single worker for N tasks.
|
|
1648
|
-
if (unit.kind === "group") {
|
|
1649
|
-
const groupTasks = unit.group.tasks;
|
|
1650
|
-
const firstTask = groupTasks[0]!;
|
|
1651
|
-
const step = findStep(workflow, firstTask);
|
|
1652
|
-
const agent = findAgent(input.agents, firstTask);
|
|
1653
|
-
const teamRole = input.team.roles.find((role) => role.name === firstTask.role);
|
|
1654
|
-
const perTaskRuntime = resolveTaskRuntimeKind(runtimeKind, firstTask.role, input.runtimeConfig?.isolationPolicy);
|
|
1655
|
-
return runCoalescedTaskGroup({
|
|
1656
|
-
manifest: ctx.manifest,
|
|
1657
|
-
tasks: ctx.tasks,
|
|
1658
|
-
groupTasks,
|
|
1659
|
-
step,
|
|
1660
|
-
agent,
|
|
1661
|
-
signal: runController.signal,
|
|
1662
|
-
executeWorkers: input.executeWorkers,
|
|
1663
|
-
runtimeKind,
|
|
1664
|
-
workspaceId: input.workspaceId,
|
|
1665
|
-
onJsonEvent: input.onJsonEvent,
|
|
1666
|
-
runtimeConfig: input.runtimeConfig,
|
|
1667
|
-
reliability: input.reliability,
|
|
1668
|
-
teamRole,
|
|
1669
|
-
perTaskRuntime,
|
|
1670
|
-
});
|
|
1671
|
-
}
|
|
1672
|
-
// Singleton path: original per-task dispatch.
|
|
1673
|
-
const task = batchTasks.find((t) => t.id === unit.taskId)!;
|
|
1674
|
-
const step = findStep(workflow, task);
|
|
1675
|
-
const agent = findAgent(input.agents, task);
|
|
1676
|
-
const teamRole = input.team.roles.find((role) => role.name === task.role);
|
|
1677
|
-
const perTaskRuntime = resolveTaskRuntimeKind(runtimeKind, task.role, input.runtimeConfig?.isolationPolicy);
|
|
1678
|
-
// CORE-3: compute retry policy + spawn budget ONCE per dispatch unit.
|
|
1679
|
-
// The spawnBudget object is shared (by reference) across every
|
|
1680
|
-
// runTeamTask call within executeWithRetry via baseInput spread,
|
|
1681
|
-
// so the counter accumulates across retry attempts × model fallbacks.
|
|
1682
|
-
const policy = retryPolicyFromConfig(input.reliability);
|
|
1683
|
-
const spawnBudget: SpawnBudget = { count: 0, max: policy.maxTotalSpawns ?? 0 };
|
|
1684
|
-
const baseInput = {
|
|
1685
|
-
manifest: ctx.manifest,
|
|
1686
|
-
tasks: ctx.tasks,
|
|
1687
|
-
task,
|
|
1688
|
-
step,
|
|
1689
|
-
agent,
|
|
1690
|
-
signal: runController.signal,
|
|
1691
|
-
executeWorkers: input.executeWorkers,
|
|
1692
|
-
runtimeKind: runtimeKind,
|
|
1693
|
-
taskRuntimeOverride: perTaskRuntime !== runtimeKind ? perTaskRuntime : undefined,
|
|
1694
|
-
runtimeConfig: input.runtimeConfig,
|
|
1695
|
-
parentContext: input.parentContext,
|
|
1696
|
-
parentModel: input.parentModel,
|
|
1697
|
-
modelRegistry: input.modelRegistry,
|
|
1698
|
-
modelOverride: input.modelOverride,
|
|
1699
|
-
teamRoleModel: teamRole?.model,
|
|
1700
|
-
teamRoleThinking: teamRole?.thinking,
|
|
1701
|
-
teamRoleFallbackModels: teamRole?.fallbackModels,
|
|
1702
|
-
teamRoleSkills: teamRole?.skills,
|
|
1703
|
-
skillOverride: input.skillOverride,
|
|
1704
|
-
limits: input.limits,
|
|
1705
|
-
onJsonEvent: input.onJsonEvent,
|
|
1706
|
-
workspaceId: input.workspaceId,
|
|
1707
|
-
spawnBudget,
|
|
1708
|
-
};
|
|
1709
|
-
// #1 (assessment): autoRetry now defaults ON (opt-out via reliability.autoRetry=false).
|
|
1710
|
-
// The dominant v0.9.13 failure was ChildTimeout ("worker became unresponsive") with
|
|
1711
|
-
// ZERO retries because this gate was opt-in. isRetryable() defaults to true when
|
|
1712
|
-
// retryableErrors is empty, so transient hangs now retry up to maxAttempts (3) with
|
|
1713
|
-
// exponential backoff. Set reliability.autoRetry=false to restore old single-shot behavior.
|
|
1714
|
-
if (!shouldUseRetry(input.reliability))
|
|
1715
|
-
return withCorrelation(childCorrelation(ctx.manifest.runId, task.id), () => runTeamTask(baseInput));
|
|
1716
|
-
let lastFailed: { manifest: TeamRunManifest; tasks: TeamTaskState[] } | undefined;
|
|
1717
|
-
let lastAttemptId: string | undefined;
|
|
1718
|
-
const attemptsSoFar: TaskAttemptState[] = [...(task.attempts ?? [])];
|
|
1719
|
-
try {
|
|
1720
|
-
return await executeWithRetry(
|
|
1721
|
-
async (attempt, info) => {
|
|
1722
|
-
const startedAt = new Date().toISOString();
|
|
1723
|
-
const inFlightAttempts: TaskAttemptState[] = [...attemptsSoFar, { attemptId: info.attemptId, startedAt }];
|
|
1724
|
-
input.metricRegistry?.counter("crew.task.retry_attempt_total", "Retry attempts by run and task").inc({
|
|
1725
|
-
runId: ctx.manifest.runId,
|
|
1726
|
-
taskId: task.id,
|
|
1727
|
-
});
|
|
1728
|
-
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
1729
|
-
const fresh = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
|
|
1730
|
-
const freshManifest = fresh?.manifest ?? ctx.manifest;
|
|
1731
|
-
const freshTasks = fresh?.tasks ?? ctx.tasks;
|
|
1732
|
-
const freshTask = freshTasks.find((item) => item.id === task.id) ?? task;
|
|
1733
|
-
if (freshTask.status !== "queued" && freshTask.status !== "running")
|
|
1734
|
-
return {
|
|
1735
|
-
manifest: freshManifest,
|
|
1736
|
-
tasks: freshTasks,
|
|
1737
|
-
};
|
|
1738
|
-
const taskWithAttempt: TeamTaskState = {
|
|
1739
|
-
...freshTask,
|
|
1740
|
-
attempts: inFlightAttempts,
|
|
1741
|
-
};
|
|
1742
|
-
const result = await withCorrelation(childCorrelation(freshManifest.runId, task.id), () =>
|
|
1743
|
-
runTeamTask({
|
|
1744
|
-
...baseInput,
|
|
1745
|
-
manifest: freshManifest,
|
|
1746
|
-
tasks: freshTasks,
|
|
1747
|
-
task: taskWithAttempt,
|
|
1748
|
-
}),
|
|
1749
|
-
);
|
|
1750
|
-
const failed = failedTaskFrom(result, task.id);
|
|
1751
|
-
const endedAt = new Date().toISOString();
|
|
1752
|
-
const finishedAttempt: TaskAttemptState = {
|
|
1753
|
-
attemptId: info.attemptId,
|
|
1754
|
-
startedAt,
|
|
1755
|
-
endedAt,
|
|
1756
|
-
...(failed?.error ? { error: failed.error } : {}),
|
|
1757
|
-
};
|
|
1758
|
-
attemptsSoFar.push(finishedAttempt);
|
|
1759
|
-
const withAttempt = result.tasks.map((item) =>
|
|
1760
|
-
item.id === task.id ? { ...item, attempts: [...attemptsSoFar] } : item,
|
|
1761
|
-
);
|
|
1762
|
-
const enriched = {
|
|
1763
|
-
manifest: result.manifest,
|
|
1764
|
-
tasks: withAttempt,
|
|
1765
|
-
};
|
|
1766
|
-
if (failed) {
|
|
1767
|
-
lastFailed = enriched;
|
|
1768
|
-
throw new CrewError(ErrorCode.TaskNotFound, failed.error ?? `Task ${task.id} failed.`).withContext(
|
|
1769
|
-
`retry evaluation (run=${ctx.manifest.runId})`,
|
|
1770
|
-
);
|
|
1771
|
-
}
|
|
1772
|
-
input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe(
|
|
1773
|
-
{
|
|
1774
|
-
runId: ctx.manifest.runId,
|
|
1775
|
-
team: input.team.name,
|
|
1776
|
-
},
|
|
1777
|
-
Math.max(0, attempt - 1),
|
|
1778
|
-
);
|
|
1779
|
-
return enriched;
|
|
1780
|
-
},
|
|
1781
|
-
policy,
|
|
1782
|
-
{
|
|
1783
|
-
signal: runController.signal,
|
|
1784
|
-
attemptId: (attempt) => `${ctx.manifest.runId}:${task.id}:attempt-${attempt}`,
|
|
1785
|
-
onAttemptFailed: (attempt, error, delayMs, info) => {
|
|
1786
|
-
lastAttemptId = info.attemptId;
|
|
1787
|
-
appendEventAsync(ctx.manifest.eventsPath, {
|
|
1788
|
-
type: "crew.task.retry_attempt",
|
|
1789
|
-
runId: ctx.manifest.runId,
|
|
1790
|
-
taskId: task.id,
|
|
1791
|
-
message: error.message,
|
|
1792
|
-
data: {
|
|
1793
|
-
attempt,
|
|
1794
|
-
attemptId: info.attemptId,
|
|
1795
|
-
delayMs,
|
|
1796
|
-
},
|
|
1797
|
-
metadata: { attemptId: info.attemptId },
|
|
1798
|
-
}).catch((error) => logInternalError("team-runner.retry-attempt", error, `taskId=${task.id}`));
|
|
1799
|
-
input.metricRegistry?.histogram("crew.task.retry_delay_ms", "Retry backoff delay, milliseconds").observe(
|
|
1800
|
-
{
|
|
1801
|
-
runId: ctx.manifest.runId,
|
|
1802
|
-
taskId: task.id,
|
|
1803
|
-
},
|
|
1804
|
-
delayMs,
|
|
1805
|
-
);
|
|
1806
|
-
},
|
|
1807
|
-
onRetryGivenUp: (attempts, error, info) => {
|
|
1808
|
-
lastAttemptId = info.attemptId;
|
|
1809
|
-
appendDeadletter(ctx.manifest, {
|
|
1810
|
-
runId: ctx.manifest.runId,
|
|
1811
|
-
taskId: task.id,
|
|
1812
|
-
reason: "max-retries",
|
|
1813
|
-
attempts,
|
|
1814
|
-
attemptId: info.attemptId,
|
|
1815
|
-
lastError: error.message,
|
|
1816
|
-
timestamp: new Date().toISOString(),
|
|
1817
|
-
});
|
|
1818
|
-
input.metricRegistry
|
|
1819
|
-
?.counter("crew.task.deadletter_total", "Deadletter triggers by reason")
|
|
1820
|
-
.inc({ reason: "max-retries" });
|
|
1821
|
-
input.metricRegistry?.histogram("crew.task.retry_count", "Retries per task", [0, 1, 2, 3, 5, 10]).observe(
|
|
1822
|
-
{
|
|
1823
|
-
runId: ctx.manifest.runId,
|
|
1824
|
-
team: input.team.name,
|
|
1825
|
-
},
|
|
1826
|
-
Math.max(0, attempts - 1),
|
|
1827
|
-
);
|
|
1828
|
-
},
|
|
1829
|
-
},
|
|
1830
|
-
);
|
|
1831
|
-
} catch (retryError) {
|
|
1832
|
-
if (retryError instanceof CrewCancellationError || input.signal?.aborted) {
|
|
1833
|
-
const reason = retryError instanceof CrewCancellationError ? retryError.reason : cancellationReasonFromSignal(input.signal);
|
|
1834
|
-
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
1835
|
-
const fresh = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
|
|
1836
|
-
const freshManifest = fresh?.manifest ?? ctx.manifest;
|
|
1837
|
-
const freshTasks = fresh?.tasks ?? ctx.tasks;
|
|
1838
|
-
const cancelledTasks = cancelNonTerminalTasks(
|
|
1839
|
-
freshTasks,
|
|
1840
|
-
"cancelled",
|
|
1841
|
-
`${reason.message} (${reason.code})`,
|
|
1842
|
-
(item) => item.id === task.id && (item.status === "queued" || item.status === "running"),
|
|
1843
|
-
);
|
|
1844
|
-
appendEventAsync(freshManifest.eventsPath, {
|
|
1845
|
-
type: "task.cancelled",
|
|
1846
|
-
runId: freshManifest.runId,
|
|
1847
|
-
taskId: task.id,
|
|
1848
|
-
message: reason.message,
|
|
1849
|
-
data: { reason, phase: "retry" },
|
|
1850
|
-
metadata: lastAttemptId ? { attemptId: lastAttemptId } : undefined,
|
|
1851
|
-
}).catch((error) => logInternalError("team-runner.cancelled", error, `taskId=${task.id}`));
|
|
1852
|
-
return {
|
|
1853
|
-
manifest: updateRunStatus(freshManifest, "cancelled", reason.message),
|
|
1854
|
-
tasks: cancelledTasks,
|
|
1855
|
-
};
|
|
1856
|
-
}
|
|
1857
|
-
if (lastFailed) return lastFailed;
|
|
1858
|
-
// NOTE: no withRunLock — best-effort only; concurrent writes may cause inconsistency
|
|
1859
|
-
const fresh = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
|
|
1860
|
-
const freshManifest = fresh?.manifest ?? ctx.manifest;
|
|
1861
|
-
const freshTasks = fresh?.tasks ?? ctx.tasks;
|
|
1862
|
-
const freshTask = freshTasks.find((item) => item.id === task.id) ?? task;
|
|
1863
|
-
if (freshTask.status !== "queued" && freshTask.status !== "running") return { manifest: freshManifest, tasks: freshTasks };
|
|
1864
|
-
return withCorrelation(childCorrelation(freshManifest.runId, task.id), () =>
|
|
1865
|
-
runTeamTask({
|
|
1866
|
-
...baseInput,
|
|
1867
|
-
manifest: freshManifest,
|
|
1868
|
-
tasks: freshTasks,
|
|
1869
|
-
task: freshTask,
|
|
1870
|
-
}),
|
|
1871
|
-
);
|
|
1872
|
-
}
|
|
1873
|
-
};
|
|
1874
|
-
// ── OPT-01 streaming dispatch: dispatch units into ctx.pendingUnits ──
|
|
1875
|
-
for (const unit of dispatchUnits) {
|
|
1876
|
-
const unitKey = unit.kind === "singleton" ? unit.taskId : unit.group.id;
|
|
1877
|
-
const unitTaskIds = unit.kind === "singleton" ? [unit.taskId] : unit.group.tasks.map((t) => t.id);
|
|
1878
|
-
// RT-12: create the wrapper promise ONCE at dispatch time so
|
|
1879
|
-
// mergeUnitResult can Promise.race on pre-existing wrappers instead
|
|
1880
|
-
// of allocating new async closures every loop iteration.
|
|
1881
|
-
const rawPromise = dispatchUnit(unit);
|
|
1882
|
-
const wrapped: Promise<SettledUnit> = (async () => {
|
|
1883
|
-
try {
|
|
1884
|
-
const result = await rawPromise;
|
|
1885
|
-
return {
|
|
1886
|
-
unitKey,
|
|
1887
|
-
result: result as { manifest: TeamRunManifest; tasks: TeamTaskState[] } | undefined,
|
|
1888
|
-
error: undefined as Error | undefined,
|
|
1889
|
-
};
|
|
1890
|
-
} catch (error) {
|
|
1891
|
-
return { unitKey, result: undefined, error: error instanceof Error ? error : new Error(String(error)) };
|
|
1892
|
-
}
|
|
1893
|
-
})();
|
|
1894
|
-
ctx.pendingUnits.set(unitKey, {
|
|
1895
|
-
taskIds: unitTaskIds,
|
|
1896
|
-
promise: rawPromise,
|
|
1897
|
-
wrapped,
|
|
1898
|
-
});
|
|
1899
|
-
// RT-NEW-2 race fix: record ever-dispatched task ids so terminaliseRunWithDrain
|
|
1900
|
-
// cancels (not skips) tasks whose unit settled + left pendingUnits before
|
|
1901
|
-
// the abort fired but whose task status isn't terminal yet.
|
|
1902
|
-
for (const id of unitTaskIds) ctx.dispatchedTaskIds.add(id);
|
|
1903
|
-
}
|
|
1904
|
-
}
|
|
1905
|
-
|
|
1906
|
-
/**
|
|
1907
|
-
* CORE-4 extraction 5: wait for one in-flight dispatch unit to settle and
|
|
1908
|
-
* merge its result into the run state.
|
|
1909
|
-
*
|
|
1910
|
-
* Awaits Promise.race on ctx.pendingUnits; the first settled unit is merged
|
|
1911
|
-
* into ctx.manifest/tasks under the run lock (flushPendingAtomicWrites +
|
|
1912
|
-
* loadRunManifestById + mergeTaskUpdatesPreservingTerminal + save). The settled
|
|
1913
|
-
* unit is then deleted from ctx.pendingUnits, and the merge outcome (taskIds
|
|
1914
|
-
* + result object) is recorded on ctx.settledMerge for the post-merge inline
|
|
1915
|
-
* logic (cancel-during-exec check + batch summary).
|
|
1916
|
-
*
|
|
1917
|
-
* Returns null to continue to the phase/budget check. A `{ kind: "return" }`
|
|
1918
|
-
* decision is reserved for future run-complete/failure detection during merge.
|
|
1919
|
-
*
|
|
1920
|
-
* Reads ctx.pendingUnits/manifest/tasks. Mutates ctx.pendingUnits (delete),
|
|
1921
|
-
* ctx.manifest/tasks, ctx.settledMerge.
|
|
1922
|
-
*
|
|
1923
|
-
* @param ctx The scheduler context.
|
|
1924
|
-
*/
|
|
1925
|
-
async function mergeUnitResult(ctx: SchedulerContext): Promise<SchedulerDecision | null> {
|
|
1926
|
-
// RT-12: race on pre-created wrapper promises (created once at dispatch
|
|
1927
|
-
// time) instead of rebuilding a wrapper-promise array with new async
|
|
1928
|
-
// closures every iteration. This reduces allocation from O(C×T) wrapper
|
|
1929
|
-
// promises to O(C) total (one per unit, created once at dispatch).
|
|
1930
|
-
const settled = await Promise.race([...ctx.pendingUnits.values()].map((u) => u.wrapped));
|
|
1931
|
-
const completedUnit = ctx.pendingUnits.get(settled.unitKey)!;
|
|
1932
|
-
ctx.pendingUnits.delete(settled.unitKey);
|
|
1933
|
-
|
|
1934
|
-
// Build the single result to merge. On rejection, synthesize a failed
|
|
1935
|
-
// result so the run continues (mirrors the old validResults guard).
|
|
1936
|
-
const resultToMerge: { manifest: TeamRunManifest; tasks: TeamTaskState[] } = settled.result ?? {
|
|
1937
|
-
manifest: ctx.manifest,
|
|
1938
|
-
tasks: cancelNonTerminalTasks(ctx.tasks, "failed", settled.error!.message, (t) => completedUnit.taskIds.includes(t.id)),
|
|
1939
|
-
};
|
|
1940
|
-
const validResults = [resultToMerge];
|
|
1941
|
-
// Reconstruct manifest from the last worker's snapshot. The .artifacts field
|
|
1942
|
-
// is re-merged from both the team-runner's in-memory state and all workers'
|
|
1943
|
-
// snapshots, so artifact writes by task-runner (which individually save manifest
|
|
1944
|
-
// after writing artifacts) are safely persisted. The in-memory manifest is only
|
|
1945
|
-
// used for the next batch iteration's orchestration — actual persistence is safe.
|
|
1946
|
-
// Use updateRunStatus to recompute manifest status from merged tasks rather than
|
|
1947
|
-
// relying on the last result's manifest (which is arbitrary due to mapConcurrent
|
|
1948
|
-
// returning results in arbitrary order).
|
|
1949
|
-
// Use the in-memory manifest as base (not the last-completing worker's snapshot).
|
|
1950
|
-
// Recompute status from merged tasks so the manifest reflects actual task state,
|
|
1951
|
-
// not the arbitrary order in which mapConcurrent returned results.
|
|
1952
|
-
// Read committed manifest from disk inside the lock so artifact merge is based
|
|
1953
|
-
// on committed state, not in-memory state that may differ from disk.
|
|
1954
|
-
const mergeResult = await withRunLock(ctx.manifest, async () => {
|
|
1955
|
-
// NEW-D1: flush any pending coalesced atomic writes before reading from
|
|
1956
|
-
// disk. Without this, a worker's async manifest save (coalesced by
|
|
1957
|
-
// atomic-write) may not be committed yet, causing a lost-update on the
|
|
1958
|
-
// merge read. flushPendingAtomicWrites forces all queued writes to disk.
|
|
1959
|
-
flushPendingAtomicWrites();
|
|
1960
|
-
const disk = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
|
|
1961
|
-
const diskManifest = disk?.manifest ?? ctx.manifest;
|
|
1962
|
-
const diskArtifacts = diskManifest.artifacts;
|
|
1963
|
-
const reconciledArtifacts = mergeArtifacts([...diskArtifacts, ...validResults.map((item) => item.manifest.artifacts)].flat());
|
|
1964
|
-
const resultManifest = updateRunStatus(
|
|
1965
|
-
{ ...diskManifest, artifacts: reconciledArtifacts },
|
|
1966
|
-
"running",
|
|
1967
|
-
"Merged task updates from parallel batch.",
|
|
1968
|
-
);
|
|
1969
|
-
// CANCEL-1: use the freshly-loaded disk tasks as the merge base instead
|
|
1970
|
-
// of the in-memory `tasks` closure variable. The in-memory tasks reflect
|
|
1971
|
-
// only team-runner's view; an external cancel (handleCancel, background
|
|
1972
|
-
// race with SIGTERM arriving after cancel wrote but before merge ran)
|
|
1973
|
-
// writes 'cancelled' to disk.tasks — using disk.tasks as base preserves
|
|
1974
|
-
// that cancellation through the merge instead of overwriting it with the
|
|
1975
|
-
// stale in-memory view. disk was loaded inside this lock, so it reflects
|
|
1976
|
-
// the freshest committed state.
|
|
1977
|
-
const resultTasks = mergeTaskUpdatesPreservingTerminal(disk?.tasks ?? ctx.tasks, validResults);
|
|
1978
|
-
await saveRunManifestAsync(resultManifest);
|
|
1979
|
-
await saveRunTasksAsync(resultManifest, resultTasks);
|
|
1980
|
-
return { resultManifest, resultTasks };
|
|
1981
|
-
});
|
|
1982
|
-
ctx.manifest = mergeResult.resultManifest;
|
|
1983
|
-
ctx.tasks = mergeResult.resultTasks;
|
|
1984
|
-
ctx.settledMerge = { taskIds: completedUnit.taskIds, result: resultToMerge };
|
|
1985
|
-
return null;
|
|
1986
|
-
}
|
|
1987
|
-
|
|
1988
|
-
/**
|
|
1989
|
-
* CORE-4 extraction 6: advance workflow phases whose tasks are all in
|
|
1990
|
-
* terminal state.
|
|
1991
|
-
*
|
|
1992
|
-
* Iterates phases starting at `ctx.wfMachine.currentPhaseIndex`; for each phase
|
|
1993
|
-
* whose tasks are all terminal, determines the transition status (failed if
|
|
1994
|
-
* any task failed/cancelled, else completed), applies the phase transition,
|
|
1995
|
-
* emits `workflow.phase_completed`/`workflow.phase_failed`/
|
|
1996
|
-
* `workflow.phase_guard_blocked` events, and advances `currentPhaseIndex`.
|
|
1997
|
-
*
|
|
1998
|
-
* Reads `ctx.tasks`, `ctx.manifest` (read-only). Mutates `ctx.wfMachine`
|
|
1999
|
-
* in-place (phase status + currentPhaseIndex). The caller syncs the local
|
|
2000
|
-
* `wfMachine` from ctx after the call.
|
|
2001
|
-
*
|
|
2002
|
-
* @param ctx The scheduler context.
|
|
2003
|
-
*/
|
|
2004
|
-
async function advanceWorkflowPhases(ctx: SchedulerContext): Promise<void> {
|
|
2005
|
-
let wfMachine = ctx.wfMachine;
|
|
2006
|
-
const tasks = ctx.tasks;
|
|
2007
|
-
const manifest = ctx.manifest;
|
|
2008
|
-
// Advance workflow phases whose tasks are all in terminal state
|
|
2009
|
-
const terminalStatuses = new Set(["completed", "failed", "skipped", "cancelled", "needs_attention"]);
|
|
2010
|
-
const phaseTaskMap = new Map<string, string[]>();
|
|
2011
|
-
for (const task of tasks) {
|
|
2012
|
-
if (!task.stepId) continue;
|
|
2013
|
-
const existing = phaseTaskMap.get(task.stepId) ?? [];
|
|
2014
|
-
existing.push(task.id);
|
|
2015
|
-
phaseTaskMap.set(task.stepId, existing);
|
|
2016
|
-
}
|
|
2017
|
-
// W5-4: by-id map once for the phase loop (was O(phases × phaseTasks × tasks)).
|
|
2018
|
-
const taskById = new Map(tasks.map((t) => [t.id, t] as const));
|
|
2019
|
-
for (let pi = wfMachine.currentPhaseIndex; pi < wfMachine.phases.length; pi++) {
|
|
2020
|
-
const phase = wfMachine.phases[pi]!;
|
|
2021
|
-
const phaseTaskIds = phaseTaskMap.get(phase.name) ?? [];
|
|
2022
|
-
if (phaseTaskIds.length === 0) continue;
|
|
2023
|
-
const allTerminal = phaseTaskIds.every((taskId) => {
|
|
2024
|
-
const task = taskById.get(taskId);
|
|
2025
|
-
return task ? terminalStatuses.has(task.status) : false;
|
|
2026
|
-
});
|
|
2027
|
-
if (!allTerminal) break;
|
|
2028
|
-
if (phase.status !== "completed" && phase.status !== "failed" && phase.status !== "skipped") {
|
|
2029
|
-
const completedArtifacts = manifest.artifacts.filter((a) => a.kind === "result" || a.kind === "summary").map((a) => a.path);
|
|
2030
|
-
const previousPhaseStatus = pi > 0 ? (wfMachine.phases[pi - 1]?.status ?? "pending") : "completed";
|
|
2031
|
-
const wfContext: PhaseGuardContext = {
|
|
2032
|
-
completedArtifacts,
|
|
2033
|
-
previousPhaseStatus,
|
|
2034
|
-
taskResults: tasks
|
|
2035
|
-
.filter((t) => t.status === "completed" || t.status === "needs_attention")
|
|
2036
|
-
.map((t) => ({
|
|
2037
|
-
taskId: t.id,
|
|
2038
|
-
status: t.status,
|
|
2039
|
-
outputPath: t.resultArtifact?.path,
|
|
2040
|
-
})),
|
|
2041
|
-
};
|
|
2042
|
-
// Determine phase transition status based on individual task outcomes
|
|
2043
|
-
const phaseTasks = phaseTaskIds
|
|
2044
|
-
.map((taskId) => taskById.get(taskId))
|
|
2045
|
-
.filter((t): t is NonNullable<typeof t> => t !== undefined);
|
|
2046
|
-
const hasFailedOrCancelled = phaseTasks.some((t) => t.status === "failed" || t.status === "cancelled");
|
|
2047
|
-
const phaseStatus = hasFailedOrCancelled ? "failed" : "completed";
|
|
2048
|
-
const transition = transitionPhase(wfMachine, pi, phaseStatus, wfContext);
|
|
2049
|
-
wfMachine = transition.machine;
|
|
2050
|
-
if (transition.guardResult && !transition.guardResult.allowed) {
|
|
2051
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
2052
|
-
type: "workflow.phase_guard_blocked",
|
|
2053
|
-
runId: manifest.runId,
|
|
2054
|
-
message: `Workflow phase '${phase.name}' guard blocked: ${transition.guardResult.reason ?? "unknown"}`,
|
|
2055
|
-
data: {
|
|
2056
|
-
phaseIndex: pi,
|
|
2057
|
-
phaseName: phase.name,
|
|
2058
|
-
reason: transition.guardResult.reason,
|
|
2059
|
-
},
|
|
2060
|
-
});
|
|
2061
|
-
break;
|
|
2062
|
-
}
|
|
2063
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
2064
|
-
type: phaseStatus === "failed" ? "workflow.phase_failed" : "workflow.phase_completed",
|
|
2065
|
-
runId: manifest.runId,
|
|
2066
|
-
message: `Workflow phase '${phase.name}' ${phaseStatus}.`,
|
|
2067
|
-
data: { phaseIndex: pi, phaseStatus },
|
|
2068
|
-
});
|
|
2069
|
-
}
|
|
2070
|
-
wfMachine = { ...wfMachine, currentPhaseIndex: pi + 1 };
|
|
2071
|
-
}
|
|
2072
|
-
ctx.wfMachine = wfMachine;
|
|
2073
|
-
}
|
|
2074
|
-
|
|
2075
|
-
/**
|
|
2076
|
-
* CORE-4 extraction 7: enforce per-task run budget after each batch merge.
|
|
2077
|
-
*
|
|
2078
|
-
* When `input.budgetTotal` is set (and not unlimited), checks cumulative usage
|
|
2079
|
-
* against warn/abort thresholds and a fair-share heuristic. On abort, marks all
|
|
2080
|
-
* non-terminal tasks blocked, persists the run as failed, and returns a
|
|
2081
|
-
* `{ kind: "return" }` decision so the caller short-circuits the loop.
|
|
2082
|
-
* Otherwise emits `run.budget_warning` / `task.budget_fair_share` events and
|
|
2083
|
-
* returns null (continue).
|
|
2084
|
-
*
|
|
2085
|
-
* Reads `ctx.input` (budget config). Mutates `ctx.tasks` / `ctx.manifest` only
|
|
2086
|
-
* in the abort path. The caller syncs these locals back after the call.
|
|
2087
|
-
*
|
|
2088
|
-
* @param ctx The scheduler context.
|
|
2089
|
-
* @returns `{ kind: "return", result }` on budget abort; `null` otherwise.
|
|
2090
|
-
*/
|
|
2091
|
-
async function enforceRunBudget(ctx: SchedulerContext): Promise<SchedulerDecision | null> {
|
|
2092
|
-
const input = ctx.input;
|
|
2093
|
-
const tasks = ctx.tasks;
|
|
2094
|
-
const manifest = ctx.manifest;
|
|
2095
|
-
// Per-task budget enforcement: check cumulative usage after each batch merge.
|
|
2096
|
-
// This prevents a single task from consuming 100% of the budget before
|
|
2097
|
-
// abort triggers (the goal-loop only checks at turn boundaries).
|
|
2098
|
-
if (input.budgetTotal !== undefined && input.budgetTotal > 0 && input.budgetUnlimited !== true) {
|
|
2099
|
-
const warnThreshold = input.budgetWarning ?? 0.8;
|
|
2100
|
-
const abortThreshold = input.budgetAbort ?? 0.95;
|
|
2101
|
-
const budgetCheck = checkPerTaskBudget(tasks, input.budgetTotal, warnThreshold, abortThreshold);
|
|
2102
|
-
|
|
2103
|
-
if (budgetCheck.abort) {
|
|
2104
|
-
const message = `Per-task budget abort threshold exceeded: ${formatTokens(budgetCheck.totalUsed)}/${formatTokens(input.budgetTotal)} (${Math.round((budgetCheck.totalUsed / input.budgetTotal) * 100)}%)`;
|
|
2105
|
-
console.warn(`[team-runner] ${message}`);
|
|
2106
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
2107
|
-
type: "run.budget_abort",
|
|
2108
|
-
runId: manifest.runId,
|
|
2109
|
-
message,
|
|
2110
|
-
data: {
|
|
2111
|
-
budgetTotal: input.budgetTotal,
|
|
2112
|
-
budgetUsed: budgetCheck.totalUsed,
|
|
2113
|
-
threshold: "abort",
|
|
2114
|
-
},
|
|
2115
|
-
});
|
|
2116
|
-
// RT-NEW-2: drain in-flight units + merge settled results before
|
|
2117
|
-
// terminalising, so in-flight tasks become completed/cancelled (not
|
|
2118
|
-
// skipped). Same shared helper handleFailedTask uses. Run-failed
|
|
2119
|
-
// reason stays the budget message.
|
|
2120
|
-
const result = await terminaliseRunWithDrain(ctx, {
|
|
2121
|
-
cancelMessage: `Cancelled by budget abort: ${message}`,
|
|
2122
|
-
blockedMessage: `Budget abort threshold exceeded: ${message}`,
|
|
2123
|
-
failedReason: message,
|
|
2124
|
-
});
|
|
2125
|
-
return { kind: "return", result };
|
|
2126
|
-
}
|
|
2127
|
-
|
|
2128
|
-
if (budgetCheck.warning) {
|
|
2129
|
-
const message = `Per-task budget warning threshold crossed: ${formatTokens(budgetCheck.totalUsed)}/${formatTokens(input.budgetTotal)} (${Math.round((budgetCheck.totalUsed / input.budgetTotal) * 100)}%)`;
|
|
2130
|
-
console.warn(`[team-runner] ${message}`);
|
|
2131
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
2132
|
-
type: "run.budget_warning",
|
|
2133
|
-
runId: manifest.runId,
|
|
2134
|
-
message,
|
|
2135
|
-
data: {
|
|
2136
|
-
budgetTotal: input.budgetTotal,
|
|
2137
|
-
budgetUsed: budgetCheck.totalUsed,
|
|
2138
|
-
threshold: "warning",
|
|
2139
|
-
},
|
|
2140
|
-
});
|
|
2141
|
-
}
|
|
2142
|
-
|
|
2143
|
-
// Fair-share warning: flag tasks that consumed >50% of remaining budget
|
|
2144
|
-
// without killing them mid-execution.
|
|
2145
|
-
const fairShareAppends: Promise<void>[] = [];
|
|
2146
|
-
for (const violatorId of budgetCheck.fairShareViolators) {
|
|
2147
|
-
const violator = tasks.find((t) => t.id === violatorId);
|
|
2148
|
-
if (!violator) continue;
|
|
2149
|
-
const taskTotal = (violator.usage?.input ?? 0) + (violator.usage?.output ?? 0) + (violator.usage?.cacheWrite ?? 0);
|
|
2150
|
-
const message = `Task '${violatorId}' consumed ${formatTokens(taskTotal)} (${Math.round((taskTotal / input.budgetTotal) * 100)}% of total budget) — exceeds fair share`;
|
|
2151
|
-
console.warn(`[team-runner.fair-share] ${message}`);
|
|
2152
|
-
fairShareAppends.push(
|
|
2153
|
-
appendEventAsync(manifest.eventsPath, {
|
|
2154
|
-
type: "task.budget_fair_share",
|
|
2155
|
-
runId: manifest.runId,
|
|
2156
|
-
taskId: violatorId,
|
|
2157
|
-
message,
|
|
2158
|
-
data: {
|
|
2159
|
-
budgetTotal: input.budgetTotal,
|
|
2160
|
-
taskUsage: taskTotal,
|
|
2161
|
-
},
|
|
2162
|
-
}).then(
|
|
2163
|
-
() => undefined,
|
|
2164
|
-
(error) => logInternalError("team-runner.fair-share-event", error, `taskId=${violatorId}`),
|
|
2165
|
-
),
|
|
2166
|
-
);
|
|
2167
|
-
}
|
|
2168
|
-
await Promise.all(fairShareAppends);
|
|
2169
|
-
}
|
|
2170
|
-
return null;
|
|
2171
|
-
}
|
|
2172
|
-
|
|
2173
|
-
/**
|
|
2174
|
-
* CORE-4 extraction 8: finalize the run after the scheduler loop exits.
|
|
2175
|
-
*
|
|
2176
|
-
* Computes the final run status (failed/blocked/completed) from task states,
|
|
2177
|
-
* policy decisions, and effectiveness evaluation; writes the workflow output
|
|
2178
|
-
* deliverable warning, the `summary.md` artifact, the joint atomic manifest+tasks
|
|
2179
|
-
* save, and a health snapshot; then returns the terminal `{ manifest, tasks }`.
|
|
2180
|
-
*
|
|
2181
|
-
* Reads `ctx.input` (limits/workflow/executeWorkers/runtimeConfig). Mutates
|
|
2182
|
-
* `ctx.manifest` / `ctx.tasks` and writes them back before returning so the
|
|
2183
|
-
* caller stays in sync. This function is the terminal step of
|
|
2184
|
-
* `executeTeamRunCore` — its return value is the run result.
|
|
2185
|
-
*
|
|
2186
|
-
* @param ctx The scheduler context.
|
|
2187
|
-
* @returns The final `{ manifest, tasks }` result for the run.
|
|
2188
|
-
*/
|
|
2189
|
-
async function finalizeRun(ctx: SchedulerContext): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
|
|
2190
|
-
const input = ctx.input;
|
|
2191
|
-
const tasks = ctx.tasks;
|
|
2192
|
-
let manifest = ctx.manifest;
|
|
2193
|
-
const failed = tasks.find((task) => task.status === "failed");
|
|
2194
|
-
const waiting = tasks.find((task) => task.status === "waiting");
|
|
2195
|
-
const running = tasks.find((task) => task.status === "running");
|
|
2196
|
-
manifest = applyPolicy(manifest, tasks, input.limits);
|
|
2197
|
-
|
|
2198
|
-
// S02: Verify workflow-declared output files exist before marking completed
|
|
2199
|
-
if (input.workflow?.steps) {
|
|
2200
|
-
const missingOutputs: string[] = [];
|
|
2201
|
-
for (const step of input.workflow.steps) {
|
|
2202
|
-
if (step.output && typeof step.output === "string") {
|
|
2203
|
-
const outputPath = path.join(manifest.artifactsRoot, step.output);
|
|
2204
|
-
if (!fs.existsSync(outputPath)) {
|
|
2205
|
-
missingOutputs.push(step.output);
|
|
2206
|
-
}
|
|
2207
|
-
}
|
|
2208
|
-
}
|
|
2209
|
-
if (missingOutputs.length > 0) {
|
|
2210
|
-
// Emit warning event — run still completes normally to avoid hanging
|
|
2211
|
-
appendEventFireAndForget(manifest.eventsPath, {
|
|
2212
|
-
type: "run.deliverable_warning",
|
|
2213
|
-
runId: manifest.runId,
|
|
2214
|
-
message: `Missing workflow output files: ${missingOutputs.join(", ")}`,
|
|
2215
|
-
data: { missingFiles: missingOutputs },
|
|
2216
|
-
});
|
|
2217
|
-
}
|
|
2218
|
-
}
|
|
2219
|
-
|
|
2220
|
-
const effectiveness = evaluateRunEffectiveness({
|
|
2221
|
-
manifest,
|
|
2222
|
-
tasks,
|
|
2223
|
-
executeWorkers: input.executeWorkers,
|
|
2224
|
-
runtimeConfig: input.runtimeConfig,
|
|
2225
|
-
});
|
|
2226
|
-
const effectivenessDecision = effectivenessPolicyDecision(effectiveness);
|
|
2227
|
-
if (effectivenessDecision) {
|
|
2228
|
-
manifest = {
|
|
2229
|
-
...manifest,
|
|
2230
|
-
policyDecisions: [...(manifest.policyDecisions ?? []), effectivenessDecision],
|
|
2231
|
-
updatedAt: new Date().toISOString(),
|
|
2232
|
-
};
|
|
2233
|
-
await appendEventAsync(manifest.eventsPath, {
|
|
2234
|
-
type: "run.effectiveness",
|
|
2235
|
-
runId: manifest.runId,
|
|
2236
|
-
message: effectivenessDecision.message,
|
|
2237
|
-
data: { effectiveness, policyDecision: effectivenessDecision },
|
|
2238
|
-
});
|
|
2239
|
-
}
|
|
2240
|
-
const blockingDecision = manifest.policyDecisions?.find((item) => item.action === "block" || item.action === "escalate");
|
|
2241
|
-
if (failed) {
|
|
2242
|
-
manifest = updateRunStatus(manifest, "failed", `Failed at task '${failed.id}'.`);
|
|
2243
|
-
} else if (waiting) {
|
|
2244
|
-
manifest = updateRunStatus(manifest, "blocked", `Waiting for response to task '${waiting.id}'.`);
|
|
2245
|
-
} else if (running) {
|
|
2246
|
-
manifest = updateRunStatus(manifest, "blocked", `Task '${running.id}' is still running.`);
|
|
2247
|
-
} else if (effectiveness.severity === "failed") {
|
|
2248
|
-
manifest = updateRunStatus(manifest, "failed", effectivenessDecision?.message ?? "Run effectiveness guard failed.");
|
|
2249
|
-
} else if (effectiveness.severity === "blocked") {
|
|
2250
|
-
manifest = updateRunStatus(manifest, "blocked", effectivenessDecision?.message ?? "Run effectiveness guard blocked completion.");
|
|
2251
|
-
} else if (blockingDecision) {
|
|
2252
|
-
manifest = updateRunStatus(manifest, "blocked", blockingDecision.message);
|
|
2253
|
-
} else if (tasks.some((task) => task.status === "queued")) {
|
|
2254
|
-
// F1 defense-in-depth: the loop exited with queued tasks still pending
|
|
2255
|
-
// (e.g. a hook skipped all ready tasks and downstream tasks never became
|
|
2256
|
-
// runnable). This is NOT a completed run — mark it blocked rather than
|
|
2257
|
-
// false-green "completed".
|
|
2258
|
-
manifest = updateRunStatus(manifest, "blocked", "Run exited with queued tasks still pending.");
|
|
2259
|
-
} else if (manifest.status === "failed" || manifest.status === "cancelled") {
|
|
2260
|
-
// The run was already marked failed/cancelled mid-run (e.g. handleFailedTask
|
|
2261
|
-
// on a coalesced-group race where the failing task's status was later
|
|
2262
|
-
// mutated by the group-drain, or a cancel). Preserve that terminal status —
|
|
2263
|
-
// do NOT force "completed" here: failed -> completed is not in
|
|
2264
|
-
// TEAM_RUN_STATUS_TRANSITIONS and would throw an invalid-transition error.
|
|
2265
|
-
// (No updateRunStatus call: from===to is a no-op, but the intent here is
|
|
2266
|
-
// explicitly "leave the earlier decision intact".)
|
|
2267
|
-
} else {
|
|
2268
|
-
manifest = updateRunStatus(
|
|
2269
|
-
manifest,
|
|
2270
|
-
"completed",
|
|
2271
|
-
input.executeWorkers ? "Team workflow completed." : "Team workflow scaffold completed without launching child workers.",
|
|
2272
|
-
);
|
|
2273
|
-
}
|
|
2274
|
-
manifest = writeProgress(manifest, tasks, "team-runner", input.executeWorkers, input.runtimeConfig);
|
|
2275
|
-
await saveRunManifestAsync(manifest);
|
|
2276
|
-
const usage = aggregateUsage(tasks);
|
|
2277
|
-
const summaryArtifact = writeArtifact(manifest.artifactsRoot, {
|
|
2278
|
-
kind: "summary",
|
|
2279
|
-
relativePath: "summary.md",
|
|
2280
|
-
producer: "team-runner",
|
|
2281
|
-
content: [
|
|
2282
|
-
`# pi-crew run ${manifest.runId}`,
|
|
2283
|
-
"",
|
|
2284
|
-
`Status: ${manifest.status}`,
|
|
2285
|
-
`Team: ${manifest.team}`,
|
|
2286
|
-
`Workflow: ${manifest.workflow ?? "(none)"}`,
|
|
2287
|
-
`Goal: ${manifest.goal}`,
|
|
2288
|
-
`Usage: ${formatUsage(usage)}`,
|
|
2289
|
-
"",
|
|
2290
|
-
"## Tasks",
|
|
2291
|
-
...tasks.map(formatTaskProgress),
|
|
2292
|
-
"",
|
|
2293
|
-
"## Effectiveness",
|
|
2294
|
-
...runEffectivenessLines(manifest, tasks, input.executeWorkers, input.runtimeConfig),
|
|
2295
|
-
"",
|
|
2296
|
-
"## Policy decisions",
|
|
2297
|
-
...(manifest.policyDecisions?.length ? summarizePolicyDecisions(manifest.policyDecisions) : ["- (none)"]),
|
|
2298
|
-
"",
|
|
2299
|
-
...scratchpadSummaryLines(manifest),
|
|
2300
|
-
].join("\n"),
|
|
2301
|
-
});
|
|
2302
|
-
// Build the complete manifest BEFORE acquiring the lock so the artifacts array
|
|
2303
|
-
// is already incorporated into the manifest object that will be atomically written.
|
|
2304
|
-
// This prevents crash-between-mutation-and-lock from leaving inconsistent state.
|
|
2305
|
-
const finalManifest = {
|
|
2306
|
-
...manifest,
|
|
2307
|
-
updatedAt: new Date().toISOString(),
|
|
2308
|
-
artifacts: [...manifest.artifacts, summaryArtifact],
|
|
2309
|
-
};
|
|
2310
|
-
// Joint atomic save: wrap manifest + tasks in a single run lock so they are
|
|
2311
|
-
// written together or not at all. Crash between separate saveRunManifestAsync
|
|
2312
|
-
// and saveRunTasksAsync calls could leave manifest/tasks.json out of sync.
|
|
2313
|
-
await withRunLock(finalManifest, async () => {
|
|
2314
|
-
await saveRunManifestAsync(finalManifest);
|
|
2315
|
-
await saveRunTasksAsync(finalManifest, tasks);
|
|
2316
|
-
});
|
|
2317
|
-
manifest = finalManifest;
|
|
2318
|
-
// Save health snapshot on run completion.
|
|
2319
|
-
// BUG A (pts/2 hang investigation 2026-06-16): stateRoot = `<crewRoot>/state/runs/<runId>`,
|
|
2320
|
-
// so the crew root is THREE dirnames up, not two. Two dirnames gave `<crewRoot>/state`
|
|
2321
|
-
// (the state dir), and HealthStore then joined HEALTH_DIR (`.crew/state/health`)
|
|
2322
|
-
// onto it → `<crewRoot>/state/.crew/state/health` — a double-joined BOGUS path.
|
|
2323
|
-
// That wrote health snapshots to a nonexistent subtree (silently breaking the
|
|
2324
|
-
// health feature) AND created junk dirs that the recursive state watcher then
|
|
2325
|
-
// attached extra inotify watches to. Fix: compute the real crew root (3 up)
|
|
2326
|
-
// and make HEALTH_DIR relative to it.
|
|
2327
|
-
const crewRoot = path.dirname(path.dirname(path.dirname(finalManifest.stateRoot)));
|
|
2328
|
-
const healthStore = new HealthStore(crewRoot);
|
|
2329
|
-
healthStore.saveSnapshot({
|
|
2330
|
-
runId: finalManifest.runId,
|
|
2331
|
-
tasks: tasks.map((t) => ({ id: t.id, status: t.status })),
|
|
2332
|
-
createdAt: finalManifest.createdAt,
|
|
2333
|
-
});
|
|
2334
|
-
ctx.manifest = manifest;
|
|
2335
|
-
ctx.tasks = tasks;
|
|
2336
|
-
return { manifest, tasks };
|
|
2337
|
-
}
|
|
752
|
+
// finalizeRun moved to ./finalize-run.ts (2026-08 Phase 2.6) — the scheduler
|
|
753
|
+
// loop imports it from there; __test__finalizeRun is re-exported above.
|
|
2338
754
|
|
|
2339
755
|
async function executeTeamRunCore(
|
|
2340
756
|
input: ExecuteTeamRunInput,
|
|
@@ -2353,7 +769,7 @@ async function executeTeamRunCore(
|
|
|
2353
769
|
}
|
|
2354
770
|
let tasks = refreshTaskGraphQueues(input.tasks);
|
|
2355
771
|
let queueIndex = buildTaskGraphIndex(tasks);
|
|
2356
|
-
const canInjectAdaptivePlan = workflow
|
|
772
|
+
const canInjectAdaptivePlan = isAdaptiveWorkflow(workflow);
|
|
2357
773
|
let adaptivePlanInjected = false;
|
|
2358
774
|
let adaptivePlanMissing = false;
|
|
2359
775
|
const attemptAdaptivePlan = async () => {
|
|
@@ -2363,6 +779,7 @@ async function executeTeamRunCore(
|
|
|
2363
779
|
tasks,
|
|
2364
780
|
workflow,
|
|
2365
781
|
team: input.team,
|
|
782
|
+
executeWorkers: input.executeWorkers,
|
|
2366
783
|
});
|
|
2367
784
|
adaptivePlanInjected = adaptivePlanInjected || adaptivePlan.injected;
|
|
2368
785
|
adaptivePlanMissing = adaptivePlan.missingPlan;
|
|
@@ -2389,7 +806,7 @@ async function executeTeamRunCore(
|
|
|
2389
806
|
) {
|
|
2390
807
|
manifest = await ensurePlanApprovalRequested(manifest, tasks);
|
|
2391
808
|
}
|
|
2392
|
-
if (manifest
|
|
809
|
+
if (isPlanApprovalDenied(manifest)) {
|
|
2393
810
|
tasks = cancelPlanTasks(tasks, "Plan approval was cancelled.");
|
|
2394
811
|
await saveRunTasksAsync(manifest, tasks);
|
|
2395
812
|
manifest = updateRunStatus(manifest, "cancelled", "Plan approval was cancelled.");
|
|
@@ -2421,11 +838,30 @@ async function executeTeamRunCore(
|
|
|
2421
838
|
// drainPendingUnits() so in-flight dispatch promises are settled (and
|
|
2422
839
|
// their child processes torn down) on every early-return path.
|
|
2423
840
|
const runController = new AbortController();
|
|
841
|
+
// R6-F2 (W2): store the listener reference so the finally block can
|
|
842
|
+
// removeEventListener() it — { once: true } alone only auto-removes when the
|
|
843
|
+
// listener FIRES; when the run finishes before the caller's signal aborts,
|
|
844
|
+
// the listener would otherwise stay attached to input.signal (long-lived
|
|
845
|
+
// session signal → leak accumulates per run). Mirrors child-executor.ts.
|
|
846
|
+
let externalAbortListener: (() => void) | undefined;
|
|
2424
847
|
if (input.signal) {
|
|
2425
848
|
if (input.signal.aborted) runController.abort();
|
|
2426
|
-
else
|
|
849
|
+
else {
|
|
850
|
+
externalAbortListener = () => runController.abort();
|
|
851
|
+
input.signal.addEventListener("abort", externalAbortListener, { once: true });
|
|
852
|
+
}
|
|
2427
853
|
}
|
|
2428
854
|
|
|
855
|
+
// R10-1: per-run result-artifact read cache. The batch closeout below
|
|
856
|
+
// aggregates every settled batch TWICE (batch-summary artifact + group-join
|
|
857
|
+
// delivery) — the second aggregation re-reads each `results/<taskId>.txt`
|
|
858
|
+
// from disk for no benefit. Keyed by artifact path + descriptor identity
|
|
859
|
+
// (sizeBytes|contentHash), so a retry that rewrites the artifact misses and
|
|
860
|
+
// re-reads. Passed to both closeout call sites AND (via SchedulerContext →
|
|
861
|
+
// baseInput) to collectDependencyOutputContext's dep reads; see
|
|
862
|
+
// task-output-context.ts.
|
|
863
|
+
const resultReadCache = createResultArtifactReadCache();
|
|
864
|
+
|
|
2429
865
|
// CORE-4: scheduler context — mutable state bag for extracted scheduler
|
|
2430
866
|
// functions. Fields are synced from closure locals at the top of each
|
|
2431
867
|
// loop iteration; extracted functions mutate ctx in-place.
|
|
@@ -2443,6 +879,7 @@ async function executeTeamRunCore(
|
|
|
2443
879
|
adaptivePlanInjected,
|
|
2444
880
|
adaptivePlanMissing,
|
|
2445
881
|
settledMerge: null,
|
|
882
|
+
resultReadCache,
|
|
2446
883
|
};
|
|
2447
884
|
|
|
2448
885
|
// CORE-1: single drain point — all early returns + normal exit settle pendingUnits via finally block.
|
|
@@ -2532,6 +969,17 @@ async function executeTeamRunCore(
|
|
|
2532
969
|
tasks = ctx.tasks;
|
|
2533
970
|
manifest = ctx.manifest;
|
|
2534
971
|
if (mergeDecision?.kind === "return") return mergeDecision.result;
|
|
972
|
+
// R15-2: a terminal status observed after the merge (the DISK manifest
|
|
973
|
+
// went terminal during the batch — external cancel/reconciler/finalizer
|
|
974
|
+
// write) must stop dispatching and route to finalizeRun so the disk
|
|
975
|
+
// terminal becomes the final status (run stops dispatching; final status
|
|
976
|
+
// = disk terminal). The CANCEL-2 block below handles worker-reported
|
|
977
|
+
// cancel / signal abort (where the merge forces "running" from a
|
|
978
|
+
// non-terminal disk) and is UNCHANGED; this only fires when the merged
|
|
979
|
+
// manifest carries a preserved disk-terminal status.
|
|
980
|
+
if (isRunTerminalPreserved(manifest.status)) {
|
|
981
|
+
break;
|
|
982
|
+
}
|
|
2535
983
|
// Re-derive the merge outcome locals for the post-merge inline logic
|
|
2536
984
|
// (cancel-during-exec check + batch summary artifact).
|
|
2537
985
|
const { taskIds: settledTaskIds, result: resultToMerge } = ctx.settledMerge!;
|
|
@@ -2605,7 +1053,7 @@ async function executeTeamRunCore(
|
|
|
2605
1053
|
) {
|
|
2606
1054
|
manifest = await ensurePlanApprovalRequested(manifest, tasks);
|
|
2607
1055
|
}
|
|
2608
|
-
if (manifest
|
|
1056
|
+
if (isPlanApprovalDenied(manifest)) {
|
|
2609
1057
|
tasks = cancelPlanTasks(tasks, "Plan approval was cancelled.");
|
|
2610
1058
|
await saveRunTasksAsync(manifest, tasks);
|
|
2611
1059
|
saveCrewAgents(manifest, recordsForMaterializedTasks(manifest, tasks, runtimeKind));
|
|
@@ -2622,13 +1070,16 @@ async function executeTeamRunCore(
|
|
|
2622
1070
|
kind: "summary",
|
|
2623
1071
|
relativePath: `batches/${batchSummarySlug(settledTaskIds)}.md`,
|
|
2624
1072
|
producer: "team-runner",
|
|
2625
|
-
content: aggregateTaskOutputs(completedBatch, manifest),
|
|
1073
|
+
content: aggregateTaskOutputs(completedBatch, manifest, resultReadCache),
|
|
2626
1074
|
});
|
|
2627
1075
|
const groupDelivery = deliverGroupJoin({
|
|
2628
1076
|
manifest,
|
|
2629
1077
|
mode: resolveGroupJoinMode(input.runtimeConfig),
|
|
2630
1078
|
batch: completedBatch,
|
|
2631
1079
|
allTasks: tasks,
|
|
1080
|
+
// R10-1: reuse the batch-summary reads for the group-join body
|
|
1081
|
+
// (same settled batch → same artifacts → cache hits, zero disk ops).
|
|
1082
|
+
cache: resultReadCache,
|
|
2632
1083
|
});
|
|
2633
1084
|
manifest = {
|
|
2634
1085
|
...manifest,
|
|
@@ -2656,5 +1107,13 @@ async function executeTeamRunCore(
|
|
|
2656
1107
|
// only needs the drain side-effect (abort + await + clear); the return
|
|
2657
1108
|
// value is intentionally unused here.
|
|
2658
1109
|
await drainPendingUnits(pendingUnits, runController);
|
|
1110
|
+
// R6-F2 (W2): release the caller-signal listener on every exit path.
|
|
1111
|
+
// Removed AFTER the drain so caller-signal aborts during teardown still
|
|
1112
|
+
// propagate to runController (exact pre-fix semantics); once the run is
|
|
1113
|
+
// fully drained the listener is dead weight — { once: true } never
|
|
1114
|
+
// auto-removes it when the signal never fired (child-executor.ts pattern).
|
|
1115
|
+
if (externalAbortListener && input.signal) {
|
|
1116
|
+
input.signal.removeEventListener("abort", externalAbortListener);
|
|
1117
|
+
}
|
|
2659
1118
|
}
|
|
2660
1119
|
}
|