stageflow 0.16.0 → 0.19.0

package/dist/runtime/forkGeneration.js

@@ -10,22 +10,35 @@ function asDagSnapshot(dag) {
         stage_ids: dag.nodes.map((n) => n.id),
     };
 }
-/** Persistent route stages that fan out to clonable successors (or already have instances). */
+function nodeById(dag, id) {
+    return dag.nodes.find((n) => n.id === id);
+}
+function isCloneEmitter(node) {
+    return node?.clone_array_field !== undefined;
+}
+function isMintedCloneInstance(node) {
+    return (node !== undefined &&
+        node.definition_id !== undefined &&
+        node.definition_id !== node.id);
+}
+function parentIsCloneEmitter(dag, node) {
+    if (node?.needs === null || node?.needs === undefined)
+        return false;
+    return isCloneEmitter(nodeById(dag, node.needs));
+}
+/** Persistent route stages that fan out to Clone Chain children (or already have instances). */
 export function routeRelatedForkParentIds(dag, routeStageIds) {
     const routeSet = new Set(routeStageIds);
     const parents = new Set();
     const snapshot = asDagSnapshot(dag);
     for (const stageId of routeStageIds) {
-        for (const childId of dag.childrenOf[stageId] ?? []) {
-            const child = dag.nodes.find((n) => n.id === childId);
-            if (child?.clonable === true) {
-                parents.add(stageId);
-            }
+        if (isCloneEmitter(nodeById(dag, stageId))) {
+            parents.add(stageId);
         }
     }
     for (const stageId of routeStageIds) {
-        const node = dag.nodes.find((n) => n.id === stageId);
-        if (node?.clonable === true && node.needs !== null) {
+        const node = nodeById(dag, stageId);
+        if (node !== undefined && parentIsCloneEmitter(dag, node) && node.needs !== null) {
             parents.add(node.needs);
         }
     }
@@ -38,24 +51,25 @@ export function routeRelatedForkParentIds(dag, routeStageIds) {
         }
     }
     for (const stageId of routeStageIds) {
-        const node = dag.nodes.find((n) => n.id === stageId);
-        if (node?.clonable !== true)
+        const node = nodeById(dag, stageId);
+        if (!parentIsCloneEmitter(dag, node) || isMintedCloneInstance(node))
             continue;
         const instances = instancesOfDefinition(snapshot, stageId).filter((id) => id !== stageId);
-        if (instances.length > 0 && node.needs !== null) {
+        if (instances.length > 0 && node !== undefined && node.needs !== null) {
             parents.add(node.needs);
         }
     }
     return [...parents];
 }
-/** Clone instance ids currently under clonable successors of a fork parent. */
+/** Clone instance ids currently under Clone Chain children of a fork parent. */
 export function cloneInstanceIdsForForkParent(dag, forkParentStageId) {
     const snapshot = asDagSnapshot(dag);
     const ids = [];
     const seen = new Set();
+    const parentIsEmitter = isCloneEmitter(nodeById(dag, forkParentStageId));
     for (const childId of dag.childrenOf[forkParentStageId] ?? []) {
-        const child = dag.nodes.find((n) => n.id === childId);
-        if (child?.clonable === true) {
+        const child = nodeById(dag, childId);
+        if (parentIsEmitter && !isMintedCloneInstance(child)) {
             for (const id of instancesOfDefinition(snapshot, childId)) {
                 if (id === childId)
                     continue;
@@ -237,6 +251,22 @@ export async function selectActiveInput(options) {
     }
     return activeCohortForParent(options.store, options.runId, forkParentId);
 }
+export async function forkCohortMapsFromStore(store, runId) {
+    const gens = await store.listForkGenerations(runId);
+    const supersededCloneIds = new Set();
+    const cohortByForkParent = new Map();
+    for (const gen of gens) {
+        if (gen.status === "superseded") {
+            for (const id of gen.clone_stage_ids) {
+                supersededCloneIds.add(id);
+            }
+        }
+        else if (gen.status === "active") {
+            cohortByForkParent.set(gen.fork_parent_stage_id, new Set(gen.clone_stage_ids));
+        }
+    }
+    return { supersededCloneIds, cohortByForkParent };
+}
 export function filterJoinInputs(allInstanceIds, cohort) {
     if (cohort.kind === "untracked")
         return [...allInstanceIds];
@@ -264,7 +294,7 @@ export function forkChoicePreservesReplaySource(dag, chosen, sourceStageId) {
             return true;
         }
         const child = dag.nodes.find((n) => n.id === childId);
-        if (child?.clonable === true) {
+        if (parentIsCloneEmitter(dag, child) && !isMintedCloneInstance(child)) {
             const snapshot = asDagSnapshot(dag);
             for (const instanceId of instancesOfDefinition(snapshot, childId)) {
                 if (instanceId === childId)

package/dist/runtime/joinReadiness.d.ts

@@ -0,0 +1,16 @@
+import type { StageEnvelope } from "../types/envelope.js";
+import type { PipelineNeedEdge, ResolvedPipelineDag, ResolvedPipelineStageNode } from "../types/pipeline.js";
+import type { StageScheduleState } from "./pipelineScheduler.js";
+import { type RouteIfEval } from "./routeIfEval.js";
+export declare function expandJoinParent(dag: ResolvedPipelineDag, catalogParentId: string): string[];
+export declare function inboundEdgeFired(edge: PipelineNeedEdge, parentId: string, states: Map<string, StageScheduleState>, envelopes: Map<string, StageEnvelope>): boolean;
+export declare function joinAllowsRun(dag: ResolvedPipelineDag, stageId: string, states: Map<string, StageScheduleState>, envelopes: Map<string, StageEnvelope>): boolean;
+export type InboundAfterSuccess = RouteIfEval | "ungated";
+export declare function classifyInboundAfterSuccess(node: Pick<ResolvedPipelineStageNode, "needs" | "needsEdges">, parentId: string, payload: Record<string, unknown> | undefined): InboundAfterSuccess;
+export declare function isEagerSingleParentIfSkip(node: Pick<ResolvedPipelineStageNode, "needs" | "needsEdges">, parentId: string, payload: Record<string, unknown> | undefined): boolean;
+/**
+ * Finds pending multi-parent joins that cannot run once every parent is
+ * terminal: all-skipped, or all-succeeded with an inbound `if` miss.
+ * A failed parent is not a stalled skip — that join stays pending.
+ */
+export declare function pickStalledJoinSkips(dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, completedEnvelopes?: Map<string, StageEnvelope>): string[];

package/dist/runtime/joinReadiness.js

@@ -0,0 +1,156 @@
+import { isNeedTerminalState, predecessorEdges } from "../config/pipelineNeeds.js";
+import { instancesOfDefinition } from "../runstore/pipelineDagSnapshot.js";
+import { evaluateRouteIf } from "./routeIfEval.js";
+function definitionInstances(dag, catalogId) {
+    const snapshot = dag;
+    if (Array.isArray(snapshot.stage_ids)) {
+        return instancesOfDefinition(snapshot, catalogId);
+    }
+    return dag.nodes.some((n) => n.id === catalogId) ? [catalogId] : [];
+}
+export function expandJoinParent(dag, catalogParentId) {
+    const minted = definitionInstances(dag, catalogParentId).filter((id) => id !== catalogParentId);
+    return minted.length > 0 ? minted : [catalogParentId];
+}
+function expandJoinParents(dag, edges) {
+    const ids = [];
+    for (const edge of edges) {
+        ids.push(...expandJoinParent(dag, edge.id));
+    }
+    return ids;
+}
+function joinParentDisposition(dag, edges, states) {
+    const instances = expandJoinParents(dag, edges);
+    if (instances.length === 0)
+        return undefined;
+    let hasFailed = false;
+    let hasSucceeded = false;
+    let hasSkipped = false;
+    for (const id of instances) {
+        const state = states.get(id);
+        if (!isNeedTerminalState(state)) {
+            return { allTerminal: false };
+        }
+        if (state === "failed")
+            hasFailed = true;
+        if (state === "succeeded")
+            hasSucceeded = true;
+        if (state === "skipped")
+            hasSkipped = true;
+    }
+    return { allTerminal: true, hasFailed, hasSucceeded, hasSkipped };
+}
+export function inboundEdgeFired(edge, parentId, states, envelopes) {
+    if (edge.if === undefined)
+        return true;
+    if (states.get(parentId) !== "succeeded")
+        return false;
+    return evaluateRouteIf(edge.if, envelopes.get(parentId)?.payload) === "fire";
+}
+function inboundEdgeMissed(edge, parentId, envelopes) {
+    if (edge.if === undefined)
+        return false;
+    return evaluateRouteIf(edge.if, envelopes.get(parentId)?.payload) === "miss";
+}
+function everySucceededInboundEdgeFired(dag, edges, states, envelopes) {
+    for (const edge of edges) {
+        for (const id of expandJoinParent(dag, edge.id)) {
+            if (!inboundEdgeFired(edge, id, states, envelopes))
+                return false;
+        }
+    }
+    return true;
+}
+function anySucceededInboundEdgeMissed(dag, edges, envelopes) {
+    for (const edge of edges) {
+        for (const id of expandJoinParent(dag, edge.id)) {
+            if (inboundEdgeMissed(edge, id, envelopes))
+                return true;
+        }
+    }
+    return false;
+}
+function multiParentJoinAllowsRun(dag, node, states, envelopes) {
+    const edges = predecessorEdges(node);
+    if (expandJoinParents(dag, edges).length < 2)
+        return false;
+    const disposition = joinParentDisposition(dag, edges, states);
+    if (disposition === undefined || !disposition.allTerminal)
+        return false;
+    if (disposition.hasFailed)
+        return false;
+    if (!disposition.hasSucceeded)
+        return false;
+    if (!disposition.hasSkipped) {
+        return everySucceededInboundEdgeFired(dag, edges, states, envelopes);
+    }
+    return true;
+}
+/**
+ * A multi-parent join whose parents are all terminal, none failed, and
+ * either every parent skipped, or every parent succeeded but an inbound
+ * `if` missed. Those joins cannot run, so the scheduler force-skips them.
+ * A failed parent is not stalled: the join stays pending. A still-running
+ * parent is not stalled either.
+ */
+function isStalledMultiParentJoin(dag, node, states, envelopes) {
+    const edges = predecessorEdges(node);
+    if (edges.length <= 1)
+        return false;
+    const disposition = joinParentDisposition(dag, edges, states);
+    if (disposition === undefined || !disposition.allTerminal)
+        return false;
+    if (disposition.hasFailed)
+        return false;
+    if (!disposition.hasSucceeded)
+        return true;
+    if (disposition.hasSkipped)
+        return false;
+    return anySucceededInboundEdgeMissed(dag, edges, envelopes);
+}
+export function joinAllowsRun(dag, stageId, states, envelopes) {
+    const node = dag.nodes.find((n) => n.id === stageId);
+    if (!node)
+        return false;
+    const edges = predecessorEdges(node);
+    if (edges.length === 0)
+        return true;
+    if (expandJoinParents(dag, edges).length >= 2) {
+        return multiParentJoinAllowsRun(dag, node, states, envelopes);
+    }
+    const edge = edges[0];
+    const parentNeedId = typeof node.needs === "string" && node.needs ? node.needs : edge.id;
+    const parentId = expandJoinParent(dag, parentNeedId)[0];
+    if (states.get(parentId) !== "succeeded")
+        return false;
+    if (edge.if === undefined)
+        return true;
+    return inboundEdgeFired(edge, parentId, states, envelopes);
+}
+export function classifyInboundAfterSuccess(node, parentId, payload) {
+    const inbound = predecessorEdges(node).find((edge) => edge.id === parentId);
+    if (inbound?.if === undefined)
+        return "ungated";
+    return evaluateRouteIf(inbound.if, payload);
+}
+export function isEagerSingleParentIfSkip(node, parentId, payload) {
+    if (predecessorEdges(node).length !== 1)
+        return false;
+    return classifyInboundAfterSuccess(node, parentId, payload) === "miss";
+}
+/**
+ * Finds pending multi-parent joins that cannot run once every parent is
+ * terminal: all-skipped, or all-succeeded with an inbound `if` miss.
+ * A failed parent is not a stalled skip — that join stays pending.
+ */
+export function pickStalledJoinSkips(dag, states, completedEnvelopes = new Map()) {
+    const ids = [];
+    for (const node of dag.nodes) {
+        if (states.get(node.id) !== "pending")
+            continue;
+        if (isStalledMultiParentJoin(dag, node, states, completedEnvelopes)) {
+            ids.push(node.id);
+        }
+    }
+    return ids;
+}

package/dist/runtime/pipelineScheduler.d.ts

@@ -3,7 +3,6 @@ import type { RunPipelineDagSnapshot, RunStore, StageSnapshot } from "../runstor
 import type { StageEnvelope } from "../types/envelope.js";
 import type { LoadedPipeline, ResolvedPipelineDag } from "../types/pipeline.js";
 import type { TaskFile } from "../types/task.js";
-import { applyCloneForksFromEnvelopes, applyCloneForksToSchedule } from "./cloneSchedule.js";
 import type { FeedbackLoopDecisionInput, ResolveFeedbackLoopDecisionResult } from "./feedbackLoopDecision.js";
 import type { PipelineRunResult } from "./pipelineRunner.js";
 import type { StageProcessLauncher } from "./stageProcessLauncher.js";
@@ -49,8 +48,6 @@ export type HydratedSchedule = {
     firstFailureReason: string | undefined;
     dag?: RunPipelineDagSnapshot;
 };
-export { applyCloneForksFromEnvelopes, applyCloneForksToSchedule, };
-export type { ApplyCloneForksResult } from "./cloneSchedule.js";
 export declare function snapshotToScheduleState(snap: {
     status: StageSnapshot["status"];
 }, overrides: ReadonlyMap<string, StageScheduleState>, stageId: string, executionMode?: StageExecutionMode): StageScheduleState;

package/dist/runtime/pipelineScheduler.js

@@ -1,14 +1,15 @@
 import { normalizeForkChoice } from "../envelope/forkChoice.js";
 import { buildPipelineDagSnapshotFromLoaded } from "../runstore/pipelineDagSnapshot.js";
 import { definitionIdForInstance } from "../runstore/stageInstanceId.js";
-import { applyCloneForksFromEnvelopes, applyCloneForksToSchedule, cloneFailureContinuesSchedule, cloneFanoutConflict, cloneRetryDownstream, cloneFailFastSkipIds, cloneScheduleAllowsRun, failureIsAccepted, isCloneInstance, protectedClonableChildIds, sequentialLaterCloneIds, skipRejectedNeedDependents, } from "./cloneSchedule.js";
+import { cloneFailureContinuesSchedule, cloneRetryDownstream, cloneFailFastSkipIds, cloneScheduleAllowsRun, failureIsAccepted, isCloneInstance, mint, protectedClonableChildIds, sequentialLaterCloneIds, skipRejectedNeedDependents, } from "./cloneSchedule.js";
 import { buildCompletedEnvelopesFromRun, buildStageConfigById, } from "./envelopeRouting.js";
-import { activeCohortFromCloneIds, forkChoicePreservesReplaySource, forkParentForNeedsStage, mintCohortForFanout, } from "./forkGeneration.js";
-import { activeReplayId, activeSourceStageId, applySendBack, cohortOverrideMap, createReplaySchedule, hydrateFromStore, isHeld, launchAttemptFor, launchFor, noteActiveCohort, onContinued, onRouteStageFailed, onRouteStageRunning, onRouteStageSucceeded, rebindRouteAttempts, resolveDecision, } from "./replayLifecycle.js";
+import { classifyInboundAfterSuccess, isEagerSingleParentIfSkip, joinAllowsRun, pickStalledJoinSkips, } from "./joinReadiness.js";
+import { reopenRunnableSkippedStages, persistReopenedStages } from "./reopenSkipped.js";
+import { forkChoicePreservesReplaySource, forkCohortMapsFromStore, mintCohortForFanout, } from "./forkGeneration.js";
+import { activeReplayId, activeSourceStageId, applySendBack, createReplaySchedule, hydrateFromStore, isHeld, launchAttemptFor, launchFor, noteActiveCohort, onContinued, onRouteStageFailed, onRouteStageRunning, onRouteStageSucceeded, rebindRouteAttempts, resolveDecision, } from "./replayLifecycle.js";
 import { WAIT_WITHOUT_WORKER_DISPATCH } from "./answerResume.js";
 import { runStage, isRunStageWaiting } from "./stageRunner.js";
 import { attemptContext, resumeSessionFilePath } from "./stageAttemptContext.js";
-const EMPTY_RETRY_DOWNSTREAM = new Set();
 function asDagSnapshot(dag) {
     const snapshot = dag;
     if (Array.isArray(snapshot.stage_ids))
@@ -18,7 +19,6 @@ function asDagSnapshot(dag) {
         stage_ids: dag.nodes.map((n) => n.id),
     };
 }
-export { applyCloneForksFromEnvelopes, applyCloneForksToSchedule, };
 function synthesizedFailureEnvelope(reason) {
     return { status: "failure", summary: reason, artifacts: [] };
 }
@@ -66,6 +66,7 @@ export async function hydrateScheduleFromStore(store, runId, dag, executionMode
             continue;
         schedulingHalted = true;
     }
+    await reopenRunnableSkippedStages(store, runId, resolvedDag, states, completedEnvelopes);
     return {
         states,
         completedEnvelopes,
@@ -86,15 +87,20 @@ export async function hydrateScheduleForRetryRoots(store, runId, dag, retryStage
         }
     }
     const retryRootSet = new Set(retryStageIds);
+    const { supersededCloneIds } = await forkCohortMapsFromStore(store, runId);
     const states = new Map();
     for (const node of resolvedDag.nodes) {
         states.set(node.id, "pending");
     }
     const overrides = new Map();
     for (const rootId of retryStageIds) {
+        if (supersededCloneIds.has(rootId))
+            continue;
         overrides.set(rootId, "pending");
     }
     for (const id of downstream) {
+        if (supersededCloneIds.has(id))
+            continue;
         overrides.set(id, "pending");
     }
     for (const snap of runDetail.stages) {
@@ -105,6 +111,20 @@ export async function hydrateScheduleForRetryRoots(store, runId, dag, retryStage
             completedEnvelopes.delete(id);
         }
     }
+    const skippedToReopen = [];
+    const skipStates = new Map();
+    for (const snap of runDetail.stages) {
+        const id = snap.stage_id;
+        if (supersededCloneIds.has(id))
+            continue;
+        if (snap.status !== "skipped")
+            continue;
+        if (!retryRootSet.has(id) && !downstream.has(id))
+            continue;
+        skippedToReopen.push(id);
+        skipStates.set(id, "skipped");
+    }
+    await persistReopenedStages(store, runId, skippedToReopen, skipStates);
     return {
         states,
         completedEnvelopes,
@@ -284,21 +304,9 @@ function isRunnable(dag, stageId, states, schedulingHalted, completedEnvelopes,
     const state = states.get(stageId);
     if (state !== "pending")
         return false;
-    return cloneScheduleAllowsRun(dag, stageId, states, completedEnvelopes, {
-        activeCohortForNeeds: (needsId) => {
-            const override = feedback !== undefined ? cohortOverrideMap(feedback) : undefined;
-            if (override === undefined || override.size === 0) {
-                return { kind: "untracked" };
-            }
-            const forkParentId = forkParentForNeedsStage(dag, needsId);
-            if (forkParentId === null)
-                return { kind: "untracked" };
-            const tracked = override.get(forkParentId);
-            if (tracked === undefined)
-                return { kind: "untracked" };
-            return activeCohortFromCloneIds(tracked);
-        },
-    });
+    if (!joinAllowsRun(dag, stageId, states, completedEnvelopes))
+        return false;
+    return cloneScheduleAllowsRun(dag, stageId, states, completedEnvelopes);
 }
 export function hydratedScheduleHasRunnableWork(hydrated) {
     if (hydrated.schedulingHalted)
@@ -403,13 +411,6 @@ export async function runPipelineDag(options) {
         }
     }
     applyForkSkipsFromEnvelopes(dag, states, completedEnvelopes);
-    {
-        const applied = applyCloneForksFromEnvelopes(dag, states, completedEnvelopes);
-        if (applied.dag !== dag) {
-            await store.updatePipelineDag(run.runId, applied.dag);
-        }
-        dag = applied.dag;
-    }
     let activeCount = 0;
     const inFlight = new Set();
     const launchedIds = new Set();
@@ -442,6 +443,28 @@ export async function runPipelineDag(options) {
         await store.appendStageEvent(run.runId, id, { event: "skipped" });
         await persistSkipRejected(id, "skipped");
     };
+    /**
+     * Finalizes multi-parent (implicit-AND) join nodes that are permanently
+     * stuck in `pending`: every predecessor edge has reached a terminal state,
+     * but `joinAllowsRun` still can't be satisfied, and terminal
+     * states never change back. `shouldSkipForObservedNeed` deliberately
+     * refuses to decide these nodes from a single resolving parent (see its
+     * comment in cloneSchedule.ts), so this is the only place their fate gets
+     * sealed -- once we're sure every sibling has actually settled. Runs once
+     * per scheduler tick; loops until a pass finds nothing new, so a chain of
+     * stalled joins settles within a single tick rather than one per ~25ms
+     * poll.
+     */
+    const applyStalledJoinSkips = async () => {
+        for (;;) {
+            const ids = pickStalledJoinSkips(dag, states, completedEnvelopes);
+            if (ids.length === 0)
+                return;
+            for (const id of ids) {
+                await persistSkipPending(id);
+            }
+        }
+    };
     for (const [stageId, state] of states) {
         if (state !== "failed")
             continue;
@@ -569,22 +592,33 @@ export async function runPipelineDag(options) {
         }
     };
     const onStageSuccess = async (stageId, envelope) => {
-        const retryDownstreamIds = retryContext
-            ? cloneRetryDownstream(dag, [...retryContext.retryRoots.keys()])
-            : EMPTY_RETRY_DOWNSTREAM;
-        const forceFreshCloneIds = activeReplayId(feedbackSchedule) !== undefined;
-        const conflict = cloneFanoutConflict(dag, envelope, states, retryDownstreamIds, { forceFreshCloneIds });
-        if (conflict !== undefined) {
-            await store.appendStageEvent(run.runId, stageId, {
-                event: "failed",
-                reason: conflict,
-            });
-            await onStageFailure(stageId, conflict);
-            return;
-        }
         states.set(stageId, "succeeded");
         completedEnvelopes.set(stageId, envelope);
         notifyRetryRootTerminal(stageId, "succeeded");
+        const minted = mint(dag, stageId, envelope);
+        if (minted.kind === "error") {
+            await handlePostSuccessError(stageId, minted.reason);
+            return;
+        }
+        if (minted.kind === "minted") {
+            dag = minted.dag;
+            await store.updatePipelineDag(run.runId, dag);
+            states.delete(minted.catalogChildId);
+            for (const id of minted.instanceIds) {
+                states.set(id, "pending");
+            }
+            const replayId = activeReplayId(feedbackSchedule);
+            if (replayId !== undefined) {
+                noteActiveCohort(feedbackSchedule, stageId, minted.instanceIds);
+                await mintCohortForFanout({
+                    store,
+                    runId: run.runId,
+                    replayId,
+                    forkParent: stageId,
+                    cloneStageIds: minted.instanceIds,
+                });
+            }
+        }
         const feedbackAction = envelope.feedback_loop;
         if (feedbackAction?.action === "send_back") {
             const node = dag.nodes.find((n) => n.id === stageId);
@@ -699,6 +733,18 @@ export async function runPipelineDag(options) {
                 return;
             }
             const skippedIds = [];
+            // applyForkChoiceToSchedule (and everything it calls, including
+            // skipRejectedNeedDependents in cloneSchedule.ts) is fully synchronous
+            // -- no `await` anywhere in that call chain. So by the time this call
+            // returns, `states` already reflects every one of this stage's
+            // unchosen children as terminal, atomically, before the persistence
+            // loop below runs its first `await`. A downstream multi-parent join
+            // (see pickStalledJoinSkips) that depends on several of these children
+            // therefore always decides off fully-consistent in-memory state, even
+            // though its own `stage_events` row can end up persisted *between*
+            // two of these sequential appendStageEvent calls below (this loop) --
+            // that interleaving is only a SQLite write-ordering artifact, not a
+            // sign the join decided early.
             applyForkChoiceToSchedule(dag, stageId, chosen, states, {
                 protectedIds,
                 onSkip: (id) => {
@@ -710,33 +756,24 @@ export async function runPipelineDag(options) {
                 await store.appendStageEvent(run.runId, id, { event: "skipped" });
             }
         }
-        const applied = applyCloneForksToSchedule(dag, stageId, envelope, states, {
-            forceFreshCloneIds,
-        });
-        const dagChanged = applied.dag !== dag;
-        dag = applied.dag;
-        if (dagChanged) {
-            await store.updatePipelineDag(run.runId, dag);
-        }
-        for (const id of applied.skippedIds) {
-            notifyRetryRootTerminal(id, "skipped");
-            await store.appendStageEvent(run.runId, id, { event: "skipped" });
-        }
-        if (applied.mintedBySuccessor !== undefined) {
-            for (const [, cloneStageIds] of applied.mintedBySuccessor) {
-                if (cloneStageIds.length === 0)
-                    continue;
-                const replayId = activeReplayId(feedbackSchedule);
-                await mintCohortForFanout({
-                    store,
-                    runId: run.runId,
-                    ...(replayId !== undefined ? { replayId } : {}),
-                    forkParent: stageId,
-                    cloneStageIds,
-                });
-                noteActiveCohort(feedbackSchedule, stageId, cloneStageIds);
+        const childIds = dag.childrenOf[stageId] ?? [];
+        for (const childId of childIds) {
+            const child = dag.nodes.find((node) => node.id === childId);
+            if (!child)
+                continue;
+            const result = classifyInboundAfterSuccess(child, stageId, envelope.payload);
+            if (result === "missing_field") {
+                schedulingHalted = true;
+                if (firstFailureReason === undefined) {
+                    firstFailureReason = `stage "${stageId}": route if field missing from payload`;
+                }
+                return;
+            }
+            if (isEagerSingleParentIfSkip(child, stageId, envelope.payload)) {
+                await persistSkipPending(childId);
             }
         }
+        await reopenRunnableSkippedStages(store, run.runId, dag, states, completedEnvelopes);
     };
     const handlePostSuccessError = async (stageId, reason) => {
         if (states.get(stageId) === "succeeded") {
@@ -891,6 +928,7 @@ export async function runPipelineDag(options) {
     };
     while (!allTerminal()) {
         drainRetryMutations();
+        await applyStalledJoinSkips();
         if (options.onLoopTick !== undefined) {
             await options.onLoopTick();
         }

package/dist/runtime/reopenSkipped.d.ts

@@ -0,0 +1,13 @@
+import type { RunStore } from "../runstore/port.js";
+import type { StageEnvelope } from "../types/envelope.js";
+import type { ResolvedPipelineDag } from "../types/pipeline.js";
+import type { StageScheduleState } from "./pipelineScheduler.js";
+export declare function isUnchosenForkChild(dag: ResolvedPipelineDag, stageId: string, envelopes: Map<string, StageEnvelope>): boolean;
+export declare function eventsShowStarted(events: ReadonlyArray<{
+    event: string;
+}>): boolean;
+export declare function shouldReopenSkippedStage(dag: ResolvedPipelineDag, stageId: string, states: Map<string, StageScheduleState>, envelopes: Map<string, StageEnvelope>, hasStarted: boolean, options?: {
+    supersededCloneIds?: ReadonlySet<string>;
+}): boolean;
+export declare function persistReopenedStages(store: Pick<RunStore, "appendStageEvent">, runId: string, stageIds: Iterable<string>, states: Map<string, StageScheduleState>): Promise<void>;
+export declare function reopenRunnableSkippedStages(store: Pick<RunStore, "appendStageEvent" | "listStageEvents" | "listForkGenerations">, runId: string, dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, envelopes: Map<string, StageEnvelope>): Promise<void>;

package/dist/runtime/reopenSkipped.js

@@ -0,0 +1,56 @@
+import { normalizeForkChoice } from "../envelope/forkChoice.js";
+import { forkCohortMapsFromStore } from "./forkGeneration.js";
+import { joinAllowsRun } from "./joinReadiness.js";
+export function isUnchosenForkChild(dag, stageId, envelopes) {
+    for (const node of dag.nodes) {
+        const envelope = envelopes.get(node.id);
+        if (envelope?.fork_choice === undefined || envelope.fork_choice === null) {
+            continue;
+        }
+        const children = dag.childrenOf[node.id] ?? [];
+        if (!children.includes(stageId))
+            continue;
+        const chosen = normalizeForkChoice(envelope.fork_choice, "stored");
+        if (!chosen.has(stageId))
+            return true;
+    }
+    return false;
+}
+export function eventsShowStarted(events) {
+    return events.some((event) => event.event === "started" || event.event === "resumed");
+}
+export function shouldReopenSkippedStage(dag, stageId, states, envelopes, hasStarted, options) {
+    if (hasStarted)
+        return false;
+    if (options?.supersededCloneIds?.has(stageId))
+        return false;
+    if (states.get(stageId) !== "skipped")
+        return false;
+    if (isUnchosenForkChild(dag, stageId, envelopes))
+        return false;
+    const hypothetical = new Map(states);
+    hypothetical.set(stageId, "pending");
+    return joinAllowsRun(dag, stageId, hypothetical, envelopes);
+}
+export async function persistReopenedStages(store, runId, stageIds, states) {
+    for (const id of stageIds) {
+        if (states.get(id) !== "skipped")
+            continue;
+        await store.appendStageEvent(runId, id, { event: "reopened" });
+        states.set(id, "pending");
+    }
+}
+export async function reopenRunnableSkippedStages(store, runId, dag, states, envelopes) {
+    const { supersededCloneIds } = await forkCohortMapsFromStore(store, runId);
+    const ids = [];
+    for (const [id, state] of states) {
+        if (state !== "skipped")
+            continue;
+        const events = await store.listStageEvents(runId, id);
+        if (!shouldReopenSkippedStage(dag, id, states, envelopes, eventsShowStarted(events), { supersededCloneIds })) {
+            continue;
+        }
+        ids.push(id);
+    }
+    await persistReopenedStages(store, runId, ids, states);
+}

package/dist/runtime/resumeTimedOut.d.ts

@@ -0,0 +1,33 @@
+import type { AgentPort } from "../agent/port.js";
+import type { RunStore } from "../runstore/port.js";
+import type { StageExecutionMode } from "./stageConcurrency.js";
+import type { StageProcessLauncher } from "./stageProcessLauncher.js";
+import { StageHitlController } from "./stageHitl.js";
+import { type OperatorCatalog } from "./stageAttemptBootstrap.js";
+export type ResumeTimedOutContext = {
+    runId: string;
+    stageId: string;
+    agent: AgentPort;
+    store: RunStore;
+    hitl: StageHitlController;
+    executionMode: StageExecutionMode;
+    stageProcessLauncher?: StageProcessLauncher;
+    cwd: string;
+    factoryCwd?: string;
+    maxActiveStagesPerRun: number;
+    operatorCatalog?: OperatorCatalog;
+};
+export declare function assertTimedOutStageEligible(status: string, events: ReadonlyArray<{
+    event: string;
+    reason?: string;
+}>): {
+    ok: true;
+} | {
+    ok: false;
+    reason: string;
+    status: 409;
+};
+export declare function reconstructTimedOutAndContinue(ctx: ResumeTimedOutContext): Promise<{
+    ok: boolean;
+    reason?: string;
+}>;