pi-crew 0.9.56 → 0.9.57

package/src/runtime/team-runner.ts

@@ -8,47 +8,54 @@ import { childCorrelation, withCorrelation } from "../observability/correlation.
 import type { MetricRegistry } from "../observability/metric-registry.ts";
 import { PluginRegistry } from "../plugins/plugin-registry.ts";
 import { NextJsPlugin, VitePlugin, VitestPlugin } from "../plugins/plugins/index.ts";
-import { hashArtifactContent as hashContent, writeArtifact } from "../state/artifact-store.ts";
 import { atomicWriteFile, flushPendingAtomicWrites } from "../state/atomic-write.ts";
-import { appendEvent, appendEventAsync, appendEventBuffered, appendEventFireAndForget, flushEventLogBuffer } from "../state/event-log.ts";
-import { HealthStore } from "../state/health-store.ts";
-import { withRunLock } from "../state/locks.ts";
-import { loadRunManifestById, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/state-store.ts";
+import { canTransitionRunStatus, TEAM_TASK_STATUSES, TEAM_TERMINAL_TASK_STATUSES, type TeamTaskStatus } from "../state/contracts.ts";
+import { withRunLock } from "../state/coordination/locks.ts";
+import {
+	appendEvent,
+	appendEventAsync,
+	appendEventBuffered,
+	appendEventFireAndForget,
+	flushEventLogBuffer,
+} from "../state/event-log/event-log.ts";
+import { hashArtifactContent as hashContent, writeArtifact } from "../state/stores/artifact-store.ts";
+import { HealthStore } from "../state/stores/health-store.ts";
+import { loadRunManifestById, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/stores/state-store.ts";
 import type { ArtifactDescriptor, PolicyDecision, TaskAttemptState, TeamRunManifest, TeamTaskState } from "../state/types.ts";
 import { aggregateUsage, formatTokens, formatUsage } from "../state/usage.ts";
 import type { TeamConfig } from "../teams/team-config.ts";
 import { logInternalError } from "../utils/internal-error.ts";
 import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
 import { checkBranchFreshness } from "../worktree/branch-freshness.ts";
-import { buildSyntheticTerminalEvidence, CrewCancellationError, cancellationReasonFromSignal } from "./cancellation.ts";
-import { buildDispatchUnits, type DispatchUnit, planCoalescedGroups } from "./coalesce-tasks.ts";
-import { type BatchConcurrencyDecision, resolveBatchConcurrency } from "./concurrency.ts";
 import { readCrewAgents, saveCrewAgents, saveCrewAgentsCoalesced } from "./crew-agent-records.ts";
 import type { CrewRuntimeKind } from "./crew-agent-runtime.ts";
 import { crewHooks } from "./crew-hooks.ts";
 import { appendDeadletter } from "./deadletter.ts";
 import { effectivenessPolicyDecision, evaluateRunEffectiveness, formatRunEffectivenessLines } from "./effectiveness.ts";
-import { applyGoalAchievement, assessGoalAchievement } from "./goal-achievement.ts";
+import { applyGoalAchievement, assessGoalAchievement } from "./goal-workflow/goal-achievement.ts";
 import { deliverGroupJoin, resolveGroupJoinMode } from "./group-join.ts";
-import { terminateLiveAgentsForRun } from "./live-agent-manager.ts";
+import { terminateLiveAgentsForRun } from "./live-session/live-agent-manager.ts";
+import { resolveTaskRuntimeKind } from "./model/runtime-policy.ts";
+import type { CrewRuntimeCapabilities } from "./model/runtime-resolver.ts";
 import { filterReadyByWriteOverlap } from "./path-overlap.ts";
 import { evaluateCrewPolicy, summarizePolicyDecisions } from "./policy-engine.ts";
-import { buildRecoveryLedger, shouldRerunFailedTask } from "./recovery-recipes.ts";
-import { DEFAULT_RETRY_POLICY, executeWithRetry, type RetryPolicy } from "./retry-executor.ts";
+import { buildSyntheticTerminalEvidence, CrewCancellationError, cancellationReasonFromSignal } from "./process/cancellation.ts";
+import { buildRecoveryLedger, shouldRerunFailedTask } from "./recovery/recovery-recipes.ts";
+import { DEFAULT_RETRY_POLICY, executeWithRetry, type RetryPolicy } from "./recovery/retry-executor.ts";
 import { permissionForRole } from "./role-permission.ts";
-import { runCoalescedTaskGroup } from "./run-coalesced-task-group.ts";
 import { registerRunPromise, rejectRunPromise, resolveRunPromise } from "./run-tracker.ts";
-import { resolveTaskRuntimeKind } from "./runtime-policy.ts";
-import type { CrewRuntimeCapabilities } from "./runtime-resolver.ts";
-import { recordsForMaterializedTasks } from "./task-display.ts";
-import { buildExecutionPlan as buildDagExecutionPlan, getReadyTasks as getDagReadyTasks, type TaskNode } from "./task-graph.ts";
+import { buildDispatchUnits, type DispatchUnit, planCoalescedGroups } from "./scheduling/coalesce-tasks.ts";
+import { type BatchConcurrencyDecision, resolveBatchConcurrency } from "./scheduling/concurrency.ts";
+import { runCoalescedTaskGroup } from "./scheduling/run-coalesced-task-group.ts";
+import { buildExecutionPlan as buildDagExecutionPlan, getReadyTasks as getDagReadyTasks, type TaskNode } from "./scheduling/task-graph.ts";
 import {
 	buildTaskGraphIndex,
 	refreshTaskGraphQueues,
 	type TaskGraphIndex,
 	type TaskGraphSchedulerSnapshot,
 	taskGraphSnapshot,
-} from "./task-graph-scheduler.ts";
+} from "./scheduling/task-graph-scheduler.ts";
+import { recordsForMaterializedTasks } from "./task-display.ts";
 import { aggregateTaskOutputs } from "./task-output-context.ts";
 import { clearStablePrefixCache, computeStablePrefixComponents } from "./task-runner/prompt-builder.ts";
 import { runTeamTask, type SpawnBudget } from "./task-runner.ts";
@@ -246,18 +253,32 @@ function isNonTerminalTaskStatus(status: TeamTaskState["status"]): boolean {
  *   terminalised with the given status.
  * - With `filter`: the filter is the sole gate — the non-terminal check is
  *   NOT applied automatically, matching per-task/per-id variants.
+ * - Optional `transform(task, terminalised)`: lets a caller attach
+ *   site-specific fields (graph mutation, terminalEvidence) to the
+ *   terminalised task. `terminalised` already carries status/finishedAt/error;
+ *   the transform returns it unchanged or a modified copy.
+ *
+ * RT-14: the two remaining inline cancel sites (cancelPlanTasks,
+ * cancelRunFromSignal) route through this helper via `transform` so EVERY
+ * cancel site uses the single shared transform. Their extra logic
+ * (graph mutation / terminalEvidence) is preserved inside the transform.
  *
  * `markBlocked` is intentionally NOT unified here (it sets status "skipped",
  * not cancelled/failed, and only acts on "queued" tasks).
  */
-function cancelNonTerminalTasks(
+export function cancelNonTerminalTasks(
 	tasks: TeamTaskState[],
 	status: "cancelled" | "failed",
 	reason: string,
 	filter?: (task: TeamTaskState) => boolean,
+	transform?: (task: TeamTaskState, terminalised: TeamTaskState) => TeamTaskState,
 ): TeamTaskState[] {
 	const predicate = filter ?? ((task: TeamTaskState) => isNonTerminalTaskStatus(task.status));
-	return tasks.map((task) => (predicate(task) ? { ...task, status, finishedAt: new Date().toISOString(), error: reason } : task));
+	return tasks.map((task) => {
+		if (!predicate(task)) return task;
+		const terminalised: TeamTaskState = { ...task, status, finishedAt: new Date().toISOString(), error: reason };
+		return transform ? transform(task, terminalised) : terminalised;
+	});
 }
 
 /**
@@ -280,34 +301,73 @@ function isMalformedFinishedAtReplacement(currentTime: number, updatedTime: numb
 	return !Number.isFinite(currentTime) && Number.isFinite(updatedTime);
 }
 
+/**
+ * RT-16: status-level gate for shouldMergeTaskUpdate. Returns the stable
+ * "from->to" key used by REJECTED_STATUS_MERGE_TRANSITIONS.
+ */
+function statusMergeKey(from: TeamTaskStatus, to: TeamTaskStatus): string {
+	return `${from}->${to}`;
+}
+
+/**
+ * RT-16 — derived merge-gate transition table.
+ *
+ * The set of old->new status pairs that shouldMergeTaskUpdate must REJECT based
+ * solely on the status transition (before any field-level comparison). It
+ * replaces the former 13 hand-written status guards that hand-duplicated the
+ * lifecycle table. Built once from the single source-of-truth table
+ * (TEAM_TASK_STATUSES + TEAM_TERMINAL_TASK_STATUSES — the terminal half of
+ * TEAM_TASK_STATUS_TRANSITIONS) plus two merge-specific policies that are
+ * STRICTER than the lifecycle table on the parallel-merge path:
+ *
+ *  P1 Terminal preservation — every terminal->non-terminal pair is rejected.
+ *    The lifecycle table permits retries (e.g. completed->queued), but a stale
+ *    worker snapshot must never resurrect a settled task.
+ *  P2 Completed integrity — five terminal->terminal flips that touch the
+ *    "completed" success terminal are rejected: completed->failed,
+ *    completed->needs_attention, failed->completed, cancelled->completed,
+ *    needs_attention->completed. (completed may still move to
+ *    cancelled/skipped; that is intentionally allowed, so these are NOT simply
+ *    "every illegal terminal->terminal flip".)
+ *  P3 waiting->running regression — the single stale-snapshot case.
+ *
+ * The decision for every old->new pair is byte-for-byte identical to the former
+ * 7 status guards (verified exhaustively in
+ * test/unit/team-runner-should-merge-table.test.ts).
+ */
+const REJECTED_STATUS_MERGE_TRANSITIONS: ReadonlySet<string> = (() => {
+	const rejected = new Set<string>();
+	// P1 — terminal preservation: reject every terminal->non-terminal pair.
+	for (const from of TEAM_TASK_STATUSES) {
+		if (!TEAM_TERMINAL_TASK_STATUSES.has(from)) continue;
+		for (const to of TEAM_TASK_STATUSES) {
+			if (!TEAM_TERMINAL_TASK_STATUSES.has(to)) rejected.add(statusMergeKey(from, to));
+		}
+	}
+	// P3 — waiting->running stale-snapshot regression.
+	rejected.add(statusMergeKey("waiting", "running"));
+	// P2 — completed integrity flips (bespoke terminal->terminal policy).
+	const completedIntegrityFlips: ReadonlyArray<[TeamTaskStatus, TeamTaskStatus]> = [
+		["completed", "failed"],
+		["completed", "needs_attention"],
+		["failed", "completed"],
+		["cancelled", "completed"],
+		["needs_attention", "completed"],
+	];
+	for (const [from, to] of completedIntegrityFlips) rejected.add(statusMergeKey(from, to));
+	return rejected;
+})();
+
 function shouldMergeTaskUpdate(current: TeamTaskState, updated: TeamTaskState): boolean {
-	// Parallel workers receive the same input snapshot. A later result may still
-	// contain stale queued/running copies of tasks that another worker already
-	// completed. Never let those stale snapshots regress durable task state.
-	if (current.status === "waiting" && updated.status === "running") return false;
-	// Block terminal→non-terminal transitions (e.g. completed→running).
-	// A task that has reached a terminal state must not be resurrected.
-	const currentIsTerminal = !isNonTerminalTaskStatus(current.status);
-	const updatedIsNonTerminal = isNonTerminalTaskStatus(updated.status);
-	if (currentIsTerminal && updatedIsNonTerminal) return false;
-	// Explicitly block completed↔needs_attention terminal-to-terminal transitions.
-	// Both are success terminal states used interchangeably; stale worker updates must
-	// not cause a completed task to appear as needs_attention or vice versa.
-	if (current.status === "completed" && updated.status === "needs_attention") return false;
-	if (current.status === "needs_attention" && updated.status === "completed") return false;
-	// Explicitly block failed→completed resurrection. Both statuses are terminal,
-	// but completed is the success terminal state and should not be reachable from
-	// failed via a stale merge. The check above only guards non-terminal→terminal.
-	if (current.status === "failed" && updated.status === "completed") return false;
-	// Mirror that guard for the other dangerous terminal→terminal flips (reverse
-	// audit CANCEL-3 + F3): a cancelled task must not be resurrected to completed,
-	// and a completed task must not be demoted to failed, by a stale worker merge.
-	// The task transition table (contracts.ts) only permits terminal→queued
-	// (retry); these flips are always stale results. A worker that completed after
-	// the task was cancelled, or a stale failed result arriving after completion,
-	// must not flip a settled terminal status.
-	if (current.status === "cancelled" && updated.status === "completed") return false;
-	if (current.status === "completed" && updated.status === "failed") return false;
+	// RT-16: status-level gate — reject stale/dangerous transitions via the
+	// derived transition table (REJECTED_STATUS_MERGE_TRANSITIONS) instead of
+	// hand-written guards. Parallel workers receive the same input snapshot; a
+	// later result may still carry stale copies. The table encodes three
+	// merge-specific policies stricter than the lifecycle table: terminal
+	// preservation (no terminal->non-terminal resurrection), completed integrity
+	// (no flipping the "completed" success terminal to/from failed or
+	// needs_attention), and the waiting->running stale-snapshot regression.
+	if (REJECTED_STATUS_MERGE_TRANSITIONS.has(statusMergeKey(current.status, updated.status))) return false;
 	// Guard: when current is "running" but has resultArtifact (another worker already
 	// completed it), a stale updated with status="running" and no resultArtifact
 	// must not overwrite the actual completed state.
@@ -359,6 +419,8 @@ function shouldMergeTaskUpdate(current: TeamTaskState, updated: TeamTaskState):
 		updated.agentProgress?.lastActivityAt !== current.agentProgress?.lastActivityAt;
 	return hasMeaningfulUpdate;
 }
+/** Exposed for the exhaustive status-merge table test (RT-16). */
+export const __test__shouldMergeTaskUpdate = shouldMergeTaskUpdate;
 
 // H4 fix: rename to descriptive name. Kept __test__ as alias for backward
 // compat test imports.
@@ -406,14 +468,14 @@ export function mergeTaskUpdatesPreservingTerminal(base: TeamTaskState[], result
 /** @deprecated Use mergeTaskUpdatesPreservingTerminal. Kept for backward test import compat. */
 export const __test__mergeTaskUpdates = mergeTaskUpdatesPreservingTerminal;
 
-// 2.8: adaptive-plan parsing/repair/injection moved to src/runtime/adaptive-plan.ts.
+// 2.8: adaptive-plan parsing/repair/injection moved to src/runtime/goal-workflow/adaptive-plan.ts.
 // Re-export the test-only helpers so existing test imports still resolve.
 export {
 	__test__parseAdaptivePlan,
 	__test__repairAdaptivePlan,
-} from "./adaptive-plan.ts";
+} from "./goal-workflow/adaptive-plan.ts";
 
-import { injectAdaptivePlanIfReady } from "./adaptive-plan.ts";
+import { injectAdaptivePlanIfReady } from "./goal-workflow/adaptive-plan.ts";
 
 function formatTaskProgress(task: TeamTaskState): string {
 	return `- ${task.id}: ${task.status} (${task.role} -> ${task.agent})${task.taskPacket ? ` scope=${task.taskPacket.scope}` : ""}${task.verification ? ` green=${task.verification.observedGreenLevel}/${task.verification.requiredGreenLevel}` : ""}${task.error ? ` - ${task.error}` : ""}`;
@@ -442,7 +504,11 @@ function runEffectivenessLines(
 // (the previous implementation ran 2 redundant passes on every batch) to
 // a single-pass Map-based replacement: remove the existing entry by path, then
 // append the new one. Net complexity: O(N) build + O(1) replace per write.
-const lastProgressContentHash = new WeakMap<TeamRunManifest, string>();
+// RT-7: key on manifest.runId (stable string) instead of object identity
+// (WeakMap). Every writeProgress mutator returns a NEW manifest object via
+// spread, so object-identity keying meant the cache NEVER hit. Using runId
+// makes back-to-back calls (same millisecond) actually dedup.
+const lastProgressContentHash = new Map<string, string>();
 
 function writeProgress(
 	manifest: TeamRunManifest,
@@ -481,19 +547,33 @@ function writeProgress(
 	// both calls happen within the same millisecond. It's a minor win but
 	// matches the audit recommendation (skip artifact write when nothing
 	// material changed).
-	const prevHash = lastProgressContentHash.get(manifest);
+	// RT-7: compute the content hash ONCE (was hashed twice per call: once
+	// for the canSkip comparison and again for the cache .set). Key the cache
+	// on manifest.runId (stable) instead of object identity (never hit).
+	const contentHash = hashContent(content);
+	const prevHash = lastProgressContentHash.get(manifest.runId);
 	// Cheap pre-check: avoid the redaction + atomicWrite + readback roundtrip
 	// when both the timestamp and the input args are identical to last time.
-	const canSkip = prevHash === hashContent(content);
+	const canSkip = prevHash === contentHash;
 
 	const progress = canSkip
 		? (() => {
-				// Reuse the previous artifact descriptor rather than rebuilding one
-				// via writeArtifact. This skips mkdirSync, resolveRealContainedPath,
+				// Reuse the previous artifact rather than rebuilding one via
+				// writeArtifact. This skips mkdirSync, resolveRealContainedPath,
 				// redactSecrets, atomicWriteFile, and the post-write readFileSync +
 				// statSync.
 				const existing = manifest.artifacts.find((a) => a.kind === "progress");
-				if (existing) return existing;
+				if (existing) {
+					// RT-7a: return a FRESH descriptor with a refreshed createdAt
+					// instead of reusing the stale existing reference. The existing
+					// descriptor's createdAt reflects the FIRST write time, not this
+					// skip-write; refreshing it matches the non-skip path (writeArtifact
+					// stamps createdAt with the actual write time) so the manifest
+					// always carries a descriptor whose createdAt reflects the current
+					// write. Content is identical (that's why we skipped), so path /
+					// sizeBytes / contentHash / retention are unchanged.
+					return { ...existing, createdAt: new Date().toISOString() };
+				}
 				// No prior progress artifact (rare; first call from a stale manifest
 				// view). Fall through to the normal write.
 				return writeArtifact(manifest.artifactsRoot, {
@@ -509,7 +589,7 @@ function writeProgress(
 				producer,
 				content,
 			});
-	lastProgressContentHash.set(manifest, hashContent(content));
+	lastProgressContentHash.set(manifest.runId, contentHash);
 
 	// P6 dedup: replace by path in a single Map pass instead of
 	//   .filter(...)  // O(N) to remove the old entry
@@ -531,6 +611,13 @@ function writeProgress(
 	};
 }
 
+/** @internal RT-7 test export — verify cache is keyed on runId (stable string). */
+export const __test__lastProgressContentHash = lastProgressContentHash;
+/** @internal RT-7 test export — exercise writeProgress directly. */
+export const __test__writeProgress = writeProgress;
+/** @internal RT-14 test export — verify cancelPlanTasks preserves graph mutation after consolidation. */
+export const __test__cancelPlanTasks = cancelPlanTasks;
+
 function applyPolicy(manifest: TeamRunManifest, tasks: TeamTaskState[], limits?: CrewLimitsConfig): TeamRunManifest {
 	const branchFreshness = checkBranchFreshness(manifest.cwd);
 	const branchArtifact = writeArtifact(manifest.artifactsRoot, {
@@ -678,17 +765,15 @@ async function ensurePlanApprovalRequested(manifest: TeamRunManifest, tasks: Tea
 }
 
 function cancelPlanTasks(tasks: TeamTaskState[], reason: string): TeamTaskState[] {
-	return tasks.map((task) =>
-		task.status === "queued" || task.status === "running" || task.status === "waiting"
-			? {
-					...task,
-					status: "cancelled",
-					finishedAt: new Date().toISOString(),
-					error: reason,
-					graph: task.graph ? { ...task.graph, queue: "done" } : undefined,
-				}
-			: task,
-	);
+	// RT-14: delegate to the shared cancelNonTerminalTasks helper. The
+	// non-terminal gate (queued/running/waiting) is identical to the helper's
+	// default filter (isNonTerminalTaskStatus). The only site-specific logic is
+	// the graph mutation (move the task graph to the "done" queue), passed via
+	// the `transform` hook so every cancel site uses the single shared path.
+	return cancelNonTerminalTasks(tasks, "cancelled", reason, undefined, (task, terminalised) => ({
+		...terminalised,
+		graph: task.graph ? { ...task.graph, queue: "done" } : undefined,
+	}));
 }
 
 function hasPendingMutatingAdaptiveTask(tasks: TeamTaskState[]): boolean {
@@ -734,6 +819,25 @@ function dagReadyTaskIds(tasks: TeamTaskState[], completedIds: Set<string>): str
 	return getDagReadyTasks(plan, completedIds);
 }
 
+/** RT-12: result shape from a settled dispatch unit (pre-created wrapper). */
+type SettledUnit = {
+	unitKey: string;
+	result: { manifest: TeamRunManifest; tasks: TeamTaskState[] } | undefined;
+	error: Error | undefined;
+};
+
+/**
+ * RT-12: in-flight dispatch unit. `wrapped` is a pre-created wrapper promise
+ * (try/catch → SettledUnit) so mergeUnitResult can Promise.race without
+ * allocating new async closures every loop iteration (O(C) total wrappers
+ * instead of O(C×T) churn).
+ */
+type PendingUnit = {
+	taskIds: string[];
+	promise: Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }>;
+	wrapped: Promise<SettledUnit>;
+};
+
 /**
  * Drain in-flight dispatch units (pendingUnits) by aborting the run-scoped
  * controller and awaiting all settled promises before clearing the map.
@@ -744,14 +848,17 @@ function dagReadyTaskIds(tasks: TeamTaskState[], completedIds: Set<string>): str
  *
  * Exported so unit tests can exercise it directly.
  */
-export async function drainPendingUnits(
-	pendingUnits: Map<string, { taskIds: string[]; promise: Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> }>,
-	controller?: AbortController,
-): Promise<void> {
-	if (pendingUnits.size === 0) return;
+/** Settled outcome of a single in-flight dispatch unit promise (returned by drainPendingUnits). */
+export type DrainOutcome = PromiseSettledResult<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }>;
+
+export async function drainPendingUnits<
+	T extends { taskIds: string[]; promise: Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> },
+>(pendingUnits: Map<string, T>, controller?: AbortController): Promise<DrainOutcome[]> {
+	if (pendingUnits.size === 0) return [];
 	controller?.abort();
-	await Promise.allSettled([...pendingUnits.values()].map((p) => p.promise));
+	const outcomes = await Promise.allSettled([...pendingUnits.values()].map((p) => p.promise));
 	pendingUnits.clear();
+	return outcomes;
 }
 
 export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> {
@@ -965,6 +1072,12 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
 		// NEW-M1: clear the stable-prefix cache (keyed by runId) so it does not
 		// grow unbounded across runs in a long-lived session.
 		clearStablePrefixCache();
+		// RT-7b: clear the progress-content cache entry for this run so the
+		// module-level lastProgressContentHash Map (keyed by runId) does not grow
+		// unbounded across runs in a long-lived session. The finally block is the
+		// single guaranteed exit point (success, failure, and re-throw), so this
+		// runs on every run completion. Mirrors clearStablePrefixCache() above.
+		lastProgressContentHash.delete(manifest.runId);
 	}
 }
 
@@ -988,7 +1101,7 @@ interface SchedulerContext {
 	tasks: TeamTaskState[];
 	queueIndex: TaskGraphIndex;
 	wfMachine: WorkflowStateMachine;
-	pendingUnits: Map<string, { taskIds: string[]; promise: Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> }>;
+	pendingUnits: Map<string, PendingUnit>;
 	runController: AbortController;
 	runtimeKind: CrewRuntimeKind;
 	adaptivePlanInjected: boolean;
@@ -1020,6 +1133,43 @@ type SchedulerDecision =
 			coalesceEnabled: boolean;
 	  };
 
+/**
+ * RT-13: Safely normalize the manifest status to "running" so a subsequent
+ * terminal transition (e.g. → cancelled) goes through updateRunStatus legally.
+ *
+ * Replaces the former inline status-rewrite hack
+ * `manifest = { ...manifest, status: "running" }` which bypassed
+ * updateRunStatus entirely. Unlike the raw spread, this validates the
+ * transition via canTransitionRunStatus. Does NOT call updateRunStatus —
+ * no event is emitted, no persistence — preserving the exact observable
+ * behavior of the former hack. If the transition is somehow illegal, the
+ * manifest is returned unchanged (safe fallback, no throw introduced).
+ *
+ * All non-terminal statuses and the terminal statuses failed/cancelled/
+ * completed can legally reach "running" per TEAM_RUN_STATUS_TRANSITIONS, so
+ * in practice the validation always passes.
+ */
+export function setRunStatusRunning(manifest: TeamRunManifest): TeamRunManifest {
+	if (canTransitionRunStatus(manifest.status, "running")) {
+		return { ...manifest, status: "running" };
+	}
+	return manifest;
+}
+
+/**
+ * RT-17: compute a bounded batch-summary filename slug from coalesced task IDs.
+ *
+ * The unbounded join of coalesced task IDs (e.g. "01+02+…+20") can exceed
+ * NAME_MAX (255) on ext4 with ~20 members. For short joins (common case: ≤180
+ * chars) use the raw IDs to preserve readability + uniqueness; for long joins,
+ * hash to a fixed-length slug (SHA-256 hex = 64 chars) with a member-count
+ * prefix for human scannability.
+ */
+export function batchSummarySlug(taskIds: string[]): string {
+	const joined = taskIds.join("+");
+	return joined.length <= 180 ? joined : `coalesced-${taskIds.length}-${hashContent(joined)}`;
+}
+
 /**
  * CORE-4 extraction 1: handle a pre-aborted signal at the top of the
  * scheduler loop.
@@ -1039,25 +1189,25 @@ async function cancelRunFromSignal(ctx: SchedulerContext): Promise<SchedulerDeci
 	const cancelReason = cancellationReasonFromSignal(ctx.input.signal);
 	const message = `${cancelReason.message} (${cancelReason.code})`;
 	const cancelledTaskIds: string[] = [];
-	ctx.tasks = ctx.tasks.map((task) => {
-		if (task.status !== "queued" && task.status !== "running" && task.status !== "waiting") return task;
+	// RT-14: delegate to the shared cancelNonTerminalTasks helper. The
+	// non-terminal gate (queued/running/waiting) is identical to the helper's
+	// default filter. The site-specific logic lives in the `transform` hook:
+	// (1) collect cancelledTaskIds for the run-status event payload, and
+	// (2) synthesise terminalEvidence for in-flight ("running") workers so the
+	// signal-abort is recorded. Both are preserved exactly — only the
+	// transform frame changed, not the per-task output.
+	ctx.tasks = cancelNonTerminalTasks(ctx.tasks, "cancelled", message, undefined, (task, terminalised) => {
 		cancelledTaskIds.push(task.id);
-		const base = {
-			...task,
-			status: "cancelled" as const,
-			finishedAt: new Date().toISOString(),
-			error: message,
-		};
 		if (task.status === "running") {
 			return {
-				...base,
+				...terminalised,
 				terminalEvidence: [
 					...(task.terminalEvidence ?? []),
 					buildSyntheticTerminalEvidence("worker", cancelReason, task.startedAt),
 				],
 			};
 		}
-		return base;
+		return terminalised;
 	});
 	await saveRunTasksAsync(ctx.manifest, ctx.tasks);
 	await Promise.all(
@@ -1134,6 +1284,67 @@ async function handleFailedTask(ctx: SchedulerContext): Promise<SchedulerDecisio
 		});
 		return { kind: "continue" }; // loop re-processes the re-queued task
 	}
+	// RT-1: drain in-flight units and merge settled results BEFORE terminalising
+	// tasks. The streaming-dispatch path only adds to ctx.pendingUnits — it
+	// never updates ctx.tasks to "running" — so ctx.tasks is stale ("queued")
+	// for in-flight tasks. A direct saveRunTasksAsync would overwrite disk
+	// "running" with the stale "queued" status, then markBlocked maps that to
+	// "skipped". team resume never re-queues skipped tasks → permanent work
+	// loss. Draining first preserves settled results and gives non-settled
+	// in-flight tasks a re-queueable terminal status (cancelled, not skipped).
+	//
+	// #3 refactor: drainPendingUnits now RETURNS the settled outcomes directly,
+	// so we collect inflightTaskIds before draining (it clears the map), then
+	// consume the returned outcomes — no redundant re-await of the same promises.
+	// After drain, ctx.pendingUnits is empty, so the finally-block
+	// drainPendingUnits call (~:2419) is a no-op (idempotent — no double-drain).
+	const inflightTaskIds = new Set<string>();
+	for (const unit of ctx.pendingUnits.values()) {
+		for (const id of unit.taskIds) inflightTaskIds.add(id);
+	}
+	const outcomes = await drainPendingUnits(ctx.pendingUnits, ctx.runController);
+	const validResults: { manifest: TeamRunManifest; tasks: TeamTaskState[] }[] = [];
+	for (const outcome of outcomes) {
+		if (outcome.status === "fulfilled") validResults.push(outcome.value);
+	}
+	if (validResults.length > 0) {
+		// Merge under the run lock — same pattern as mergeUnitResult:
+		// flush pending writes, load disk state, merge artifacts + tasks,
+		// save atomically.
+		const mergeResult = await withRunLock(ctx.manifest, async () => {
+			flushPendingAtomicWrites();
+			const disk = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
+			const diskManifest = disk?.manifest ?? ctx.manifest;
+			const reconciledArtifacts = mergeArtifacts([
+				...diskManifest.artifacts,
+				...validResults.flatMap((item) => item.manifest.artifacts),
+			]);
+			const resultManifest = updateRunStatus(
+				{ ...diskManifest, artifacts: reconciledArtifacts },
+				"running",
+				"Merged in-flight results during failed-task abort.",
+			);
+			const resultTasks = mergeTaskUpdatesPreservingTerminal(disk?.tasks ?? ctx.tasks, validResults);
+			await saveRunManifestAsync(resultManifest);
+			await saveRunTasksAsync(resultManifest, resultTasks);
+			return { resultManifest, resultTasks };
+		});
+		ctx.manifest = mergeResult.resultManifest;
+		ctx.tasks = mergeResult.resultTasks;
+	}
+	// RT-1: cancel in-flight tasks that did NOT settle (e.g. rejected promises)
+	// so team resume CAN re-queue them. markBlocked maps queued→skipped, which
+	// resume never re-queues — work would be lost permanently. Only cancel
+	// tasks that are both in-flight AND still non-terminal (settled tasks with
+	// a terminal status are preserved).
+	ctx.tasks = cancelNonTerminalTasks(
+		ctx.tasks,
+		"cancelled",
+		`Cancelled by failed task '${failed.id}'.`,
+		(task) => inflightTaskIds.has(task.id) && isNonTerminalTaskStatus(task.status),
+	);
+	// RT-1: remaining queued tasks (never dispatched) → skipped (original
+	// behavior preserved for downstream tasks not yet in-flight).
 	ctx.tasks = markBlocked(ctx.tasks, `Blocked by failed task '${failed.id}'.`);
 	await saveRunTasksAsync(ctx.manifest, ctx.tasks);
 	saveCrewAgents(ctx.manifest, recordsForMaterializedTasks(ctx.manifest, ctx.tasks, ctx.runtimeKind));
@@ -1466,6 +1677,8 @@ async function dispatchBatch(ctx: SchedulerContext, decision: DispatchBatchDecis
 				runtimeKind,
 				workspaceId: input.workspaceId,
 				onJsonEvent: input.onJsonEvent,
+				runtimeConfig: input.runtimeConfig,
+				reliability: input.reliability,
 				teamRole,
 				perTaskRuntime,
 			});
@@ -1674,9 +1887,26 @@ async function dispatchBatch(ctx: SchedulerContext, decision: DispatchBatchDecis
 	for (const unit of dispatchUnits) {
 		const unitKey = unit.kind === "singleton" ? unit.taskId : unit.group.id;
 		const unitTaskIds = unit.kind === "singleton" ? [unit.taskId] : unit.group.tasks.map((t) => t.id);
+		// RT-12: create the wrapper promise ONCE at dispatch time so
+		// mergeUnitResult can Promise.race on pre-existing wrappers instead
+		// of allocating new async closures every loop iteration.
+		const rawPromise = dispatchUnit(unit);
+		const wrapped: Promise<SettledUnit> = (async () => {
+			try {
+				const result = await rawPromise;
+				return {
+					unitKey,
+					result: result as { manifest: TeamRunManifest; tasks: TeamTaskState[] } | undefined,
+					error: undefined as Error | undefined,
+				};
+			} catch (error) {
+				return { unitKey, result: undefined, error: error instanceof Error ? error : new Error(String(error)) };
+			}
+		})();
 		ctx.pendingUnits.set(unitKey, {
 			taskIds: unitTaskIds,
-			promise: dispatchUnit(unit),
+			promise: rawPromise,
+			wrapped,
 		});
 	}
 }
@@ -1701,23 +1931,11 @@ async function dispatchBatch(ctx: SchedulerContext, decision: DispatchBatchDecis
  * @param ctx  The scheduler context.
  */
 async function mergeUnitResult(ctx: SchedulerContext): Promise<SchedulerDecision | null> {
-	// Wait for ONE in-flight unit to complete. Promise.race returns as soon
-	// as the first wrapper resolves — others remain pending in ctx.pendingUnits
-	// and are re-raced on the next iteration.
-	const settled = await Promise.race(
-		[...ctx.pendingUnits.entries()].map(async ([key, pending]) => {
-			try {
-				const result = await pending.promise;
-				return {
-					unitKey: key,
-					result: result as { manifest: TeamRunManifest; tasks: TeamTaskState[] } | undefined,
-					error: undefined as Error | undefined,
-				};
-			} catch (error) {
-				return { unitKey: key, result: undefined, error: error instanceof Error ? error : new Error(String(error)) };
-			}
-		}),
-	);
+	// RT-12: race on pre-created wrapper promises (created once at dispatch
+	// time) instead of rebuilding a wrapper-promise array with new async
+	// closures every iteration. This reduces allocation from O(C×T) wrapper
+	// promises to O(C) total (one per unit, created once at dispatch).
+	const settled = await Promise.race([...ctx.pendingUnits.values()].map((u) => u.wrapped));
 	const completedUnit = ctx.pendingUnits.get(settled.unitKey)!;
 	ctx.pendingUnits.delete(settled.unitKey);
 
@@ -1946,7 +2164,7 @@ async function enforceRunBudget(ctx: SchedulerContext): Promise<SchedulerDecisio
 						taskUsage: taskTotal,
 					},
 				}).then(
-					() => {},
+					() => undefined,
 					(error) => logInternalError("team-runner.fair-share-event", error, `taskId=${violatorId}`),
 				),
 			);
@@ -2042,6 +2260,14 @@ async function finalizeRun(ctx: SchedulerContext): Promise<{ manifest: TeamRunMa
 		// runnable). This is NOT a completed run — mark it blocked rather than
 		// false-green "completed".
 		manifest = updateRunStatus(manifest, "blocked", "Run exited with queued tasks still pending.");
+	} else if (manifest.status === "failed" || manifest.status === "cancelled") {
+		// The run was already marked failed/cancelled mid-run (e.g. handleFailedTask
+		// on a coalesced-group race where the failing task's status was later
+		// mutated by the group-drain, or a cancel). Preserve that terminal status —
+		// do NOT force "completed" here: failed -> completed is not in
+		// TEAM_RUN_STATUS_TRANSITIONS and would throw an invalid-transition error.
+		// (No updateRunStatus call: from===to is a no-op, but the intent here is
+		// explicitly "leave the earlier decision intact".)
 	} else {
 		manifest = updateRunStatus(
 			manifest,
@@ -2192,7 +2418,7 @@ async function executeTeamRunCore(
 	// task can be dispatched as soon as a slot frees, without waiting for
 	// the entire batch to complete. Each entry maps a unit key (singleton
 	// task ID or coalesced-group ID) to the in-flight promise + member IDs. ──
-	const pendingUnits = new Map<string, { taskIds: string[]; promise: Promise<{ manifest: TeamRunManifest; tasks: TeamTaskState[] }> }>();
+	const pendingUnits = new Map<string, PendingUnit>();
 
 	// CORE-1: run-scoped AbortController linked to input.signal. Aborted by
 	// drainPendingUnits() so in-flight dispatch promises are settled (and
@@ -2224,11 +2450,17 @@ async function executeTeamRunCore(
 	// CORE-1: single drain point — all early returns + normal exit settle pendingUnits via finally block.
 	try {
 		while (tasks.some((task) => task.status === "queued") || pendingUnits.size > 0) {
-			// CORE-4: full sync of mutable closure locals → ctx at the top of every
-			// iteration. The inline post-merge block (adaptive plan re-injection,
-			// phase advance, queueIndex rebuild) mutates these locals; without this
-			// sync the extracted functions (selectDispatchBatch / dispatchBatch /
-			// mergeUnitResult) would read stale ctx fields — e.g. ctx.workflow
+			// CORE-4 / RT-15: full sync of mutable closure locals → ctx at the top
+			// of every iteration. This is the SINGLE forward-sync point — all
+			// extracted scheduler functions (cancelRunFromSignal /
+			// handleFailedTask / selectDispatchBatch / dispatchBatch /
+			// mergeUnitResult / advanceWorkflowPhases / enforceRunBudget) read
+			// ctx directly. The per-function redundant forward-syncs were removed
+			// (RT-15) because back-syncs after each call keep locals aligned with
+			// ctx, making intermediate re-syncs no-ops. The inline post-merge
+			// block (adaptive plan re-injection, phase advance, queueIndex
+			// rebuild) mutates these locals; without this top-of-loop sync the
+			// extracted functions would read stale ctx fields — e.g. ctx.workflow
 			// missing newly-injected adaptive steps → E006 step-not-found.
 			ctx.tasks = tasks;
 			ctx.manifest = manifest;
@@ -2242,24 +2474,35 @@ async function executeTeamRunCore(
 			const signalDecision = await cancelRunFromSignal(ctx);
 			if (signalDecision?.kind === "return") return signalDecision.result;
 
-			// CORE-4 extraction 2: failed-task handling. Sync mutable locals into
-			// ctx before the call; handleFailedTask mutates ctx in-place and
-			// returns a SchedulerDecision.
-			ctx.tasks = tasks;
-			ctx.manifest = manifest;
+			// CORE-4 extraction 2: failed-task handling. ctx is already synced
+			// from the top-of-loop sync (RT-15); handleFailedTask mutates ctx
+			// in-place and returns a SchedulerDecision.
 			const failedDecision = await handleFailedTask(ctx);
 			tasks = ctx.tasks;
 			manifest = ctx.manifest;
-			if (failedDecision?.kind === "return") return failedDecision.result;
+			if (failedDecision?.kind === "return") {
+				// #4 (completeness): route the failed-run short-circuit through
+				// finalizeRun so the summary artifact + health snapshot + policy
+				// decisions are written (previously bypassed — failed runs had no
+				// closeout artifacts). handleFailedTask already set manifest status
+				// to "failed" and saved tasks; finalizeRun re-derives the status
+				// (still "failed" since a failed task exists — the `if (failed)`
+				// branch in finalizeRun takes priority) and performs the closeout
+				// writes. Policy-decision semantics are UNCHANGED — finalizeRun
+				// uses identical logic for both the normal-completion and
+				// failed-short-circuit paths.
+				const failedResult = await finalizeRun(ctx);
+				manifest = ctx.manifest;
+				tasks = ctx.tasks;
+				return failedResult;
+			}
 			if (failedDecision?.kind === "continue") continue;
 
-			// CORE-4 extraction 3: batch selection. Sync mutable locals into ctx;
-			// selectDispatchBatch computes the ready batch and returns a dispatch
-			// decision (or return if the run must block/abort). The caller syncs
-			// ctx.wfMachine back because the function may advance phases.
-			ctx.tasks = tasks;
-			ctx.manifest = manifest;
-			ctx.wfMachine = wfMachine;
+			// CORE-4 extraction 3: batch selection. ctx is already synced from
+			// the top-of-loop sync (RT-15); selectDispatchBatch computes the
+			// ready batch and returns a dispatch decision (or return if the run
+			// must block/abort). The caller syncs ctx.wfMachine back because
+			// the function may advance phases.
 			const dispatchDecision = await selectDispatchBatch(ctx);
 			tasks = ctx.tasks;
 			manifest = ctx.manifest;
@@ -2271,9 +2514,7 @@ async function executeTeamRunCore(
 			// dispatch units, pre-warms the stable-prefix cache, and adds each
 			// unit's promise to ctx.pendingUnits (wrapped in executeWithRetry).
 			// The function mutates ctx.tasks (hook skips) and ctx.manifest (hook
-			// status) in-place. Verbatim lift of the inline dispatch block.
-			ctx.tasks = tasks;
-			ctx.manifest = manifest;
+			// status) in-place. ctx is already synced from the top-of-loop sync.
 			await dispatchBatch(ctx, dispatchDecision);
 			tasks = ctx.tasks;
 			manifest = ctx.manifest;
@@ -2289,8 +2530,6 @@ async function executeTeamRunCore(
 			// ctx.pendingUnits, and records the merge outcome on ctx. Returns null
 			// (continue to phase/budget check) — a return decision is reserved for
 			// future run-complete/failure detection.
-			ctx.tasks = tasks;
-			ctx.manifest = manifest;
 			const mergeDecision = await mergeUnitResult(ctx);
 			tasks = ctx.tasks;
 			manifest = ctx.manifest;
@@ -2299,21 +2538,20 @@ async function executeTeamRunCore(
 			// (cancel-during-exec check + batch summary artifact).
 			const { taskIds: settledTaskIds, result: resultToMerge } = ctx.settledMerge!;
 
-			// CORE-4 extraction 6: workflow phase advance. Sync ctx.wfMachine;
+			// CORE-4 extraction 6: workflow phase advance. ctx.wfMachine is
+			// already synced from the top-of-loop sync (RT-15);
 			// advanceWorkflowPhases advances phases whose tasks are all terminal,
 			// emits phase_* events, and advances currentPhaseIndex. Mutates
 			// ctx.wfMachine in-place.
-			ctx.wfMachine = wfMachine;
 			await advanceWorkflowPhases(ctx);
 			wfMachine = ctx.wfMachine;
 
-			// CORE-4 extraction 7: budget enforcement. Sync ctx.tasks/manifest;
-			// enforceRunBudget checks cumulative usage against warn/abort thresholds
-			// and a fair-share heuristic. On abort it marks the run failed and
-			// returns { kind: "return" }; otherwise emits warning events and
-			// returns null (continue).
-			ctx.tasks = tasks;
-			ctx.manifest = manifest;
+			// CORE-4 extraction 7: budget enforcement. ctx is already synced
+			// from the top-of-loop sync (RT-15); enforceRunBudget checks
+			// cumulative usage against warn/abort thresholds and a fair-share
+			// heuristic. On abort it marks the run failed and returns
+			// { kind: "return" }; otherwise emits warning events and returns
+			// null (continue).
 			const budgetDecision = await enforceRunBudget(ctx);
 			tasks = ctx.tasks;
 			manifest = ctx.manifest;
@@ -2323,7 +2561,7 @@ async function executeTeamRunCore(
 			if (cancelledResult || input.signal?.aborted) {
 				const reason = input.signal?.aborted ? cancellationReasonFromSignal(input.signal) : undefined;
 				const message = reason?.message ?? cancelledResult?.manifest.summary ?? "Run cancelled during task execution.";
-				manifest = { ...manifest, status: "running" };
+				manifest = setRunStatusRunning(manifest);
 				manifest = updateRunStatus(manifest, "cancelled", message);
 				// CANCEL-2: re-cancel non-terminal tasks here, mirroring the batch-loop
 				// cancel check at team-runner.ts:~925. A manifest cancelled mid-merge
@@ -2384,7 +2622,7 @@ async function executeTeamRunCore(
 			const completedBatch = tasks.filter((t) => settledTaskIds.includes(t.id));
 			const batchArtifact = writeArtifact(manifest.artifactsRoot, {
 				kind: "summary",
-				relativePath: `batches/${settledTaskIds.join("+")}.md`,
+				relativePath: `batches/${batchSummarySlug(settledTaskIds)}.md`,
 				producer: "team-runner",
 				content: aggregateTaskOutputs(completedBatch, manifest),
 			});
@@ -2416,6 +2654,9 @@ async function executeTeamRunCore(
 		tasks = ctx.tasks;
 		return finalResult;
 	} finally {
+		// #3: drainPendingUnits returns settled outcomes, but the finally block
+		// only needs the drain side-effect (abort + await + clear); the return
+		// value is intentionally unused here.
 		await drainPendingUnits(pendingUnits, runController);
 	}
 }