stageflow 0.9.0 → 0.13.0

package/dist/runtime/pipelineScheduler.js

@@ -1,12 +1,14 @@
 import { normalizeForkChoice } from "../envelope/forkChoice.js";
 import { buildPipelineDagSnapshotFromLoaded } from "../runstore/pipelineDagSnapshot.js";
 import { definitionIdForInstance } from "../runstore/stageInstanceId.js";
-import { applyCloneForksFromEnvelopes, applyCloneForksToSchedule, cloneFailureContinuesSchedule, cloneFanoutConflict, cloneRetryDownstream, cloneFailFastSkipIds, cloneScheduleAllowsRun, isCloneInstance, protectedClonableChildIds, sequentialLaterCloneIds, } from "./cloneSchedule.js";
-import { collectDownstreamStageIds } from "./dagTraversal.js";
+import { applyCloneForksFromEnvelopes, applyCloneForksToSchedule, cloneFailureContinuesSchedule, cloneFanoutConflict, cloneRetryDownstream, cloneFailFastSkipIds, cloneScheduleAllowsRun, failureIsAccepted, isCloneInstance, protectedClonableChildIds, sequentialLaterCloneIds, skipRejectedNeedDependents, } from "./cloneSchedule.js";
 import { buildCompletedEnvelopesFromRun, buildStageConfigById, } from "./envelopeRouting.js";
+import { activeCohortFromCloneIds, forkChoicePreservesReplaySource, forkParentForNeedsStage, mintCohortForFanout, } from "./forkGeneration.js";
+import { activeReplayId, activeSourceStageId, applySendBack, cohortOverrideMap, createReplaySchedule, hydrateFromStore, isHeld, launchAttemptFor, launchFor, noteActiveCohort, onContinued, onRouteStageFailed, onRouteStageRunning, onRouteStageSucceeded, rebindRouteAttempts, resolveDecision, } from "./replayLifecycle.js";
 import { WAIT_WITHOUT_WORKER_DISPATCH } from "./answerResume.js";
 import { runStage, isRunStageWaiting } from "./stageRunner.js";
-import { attemptContext } from "./stageAttemptContext.js";
+import { attemptContext, resumeSessionFilePath } from "./stageAttemptContext.js";
+const EMPTY_RETRY_DOWNSTREAM = new Set();
 function asDagSnapshot(dag) {
     const snapshot = dag;
     if (Array.isArray(snapshot.stage_ids))
@@ -53,10 +55,16 @@ export async function hydrateScheduleFromStore(store, runId, dag, executionMode
     for (const snap of runDetail.stages) {
         const state = snapshotToScheduleState(snap, overrides, snap.stage_id, executionMode);
         states.set(snap.stage_id, state);
-        if (state === "failed" &&
-            !cloneFailureContinuesSchedule(resolvedDag, snap.stage_id, completedEnvelopes)) {
-            schedulingHalted = true;
+    }
+    for (const [stageId, state] of states) {
+        if (state !== "failed")
+            continue;
+        if (cloneFailureContinuesSchedule(resolvedDag, stageId, completedEnvelopes)) {
+            continue;
         }
+        if (failureIsAccepted(resolvedDag, stageId, states))
+            continue;
+        schedulingHalted = true;
     }
     return {
         states,
@@ -136,19 +144,16 @@ export async function resumeRun(options) {
         return state !== undefined && terminalState(state);
     });
     if (allTerminal) {
-        const allSucceeded = dag.nodes.every((node) => {
-            const s = hydrated.states.get(node.id);
-            return s === "succeeded" || s === "skipped";
-        });
-        await prepared.store.updateRunStatus(prepared.run.runId, allSucceeded ? "succeeded" : "failed");
+        const unhandledFailure = scheduleHasUnhandledFailure(dag, hydrated.states);
+        await prepared.store.updateRunStatus(prepared.run.runId, unhandledFailure ? "failed" : "succeeded");
         return {
-            ok: allSucceeded,
-            outcome: allSucceeded ? "succeeded" : "failed",
+            ok: !unhandledFailure,
+            outcome: unhandledFailure ? "failed" : "succeeded",
             runDir: prepared.run.workspaceDir,
             runId: prepared.run.runId,
-            ...(allSucceeded
-                ? {}
-                : { reason: hydrated.firstFailureReason ?? "pipeline incomplete" }),
+            ...(unhandledFailure
+                ? { reason: hydrated.firstFailureReason ?? "pipeline incomplete" }
+                : {}),
         };
     }
     return runPipelineDag(options);
@@ -180,9 +185,7 @@ function applyForkChoiceToSchedule(dag, forkStageId, chosen, states, options) {
             continue;
         if (!chosen.has(childId)) {
             skipPending(childId);
-            for (const desc of collectDownstreamStageIds(dag, childId)) {
-                skipPending(desc);
-            }
+            skipRejectedNeedDependents(dag, childId, states, "skipped", skipPending);
         }
     }
 }
@@ -197,7 +200,7 @@ export function applyForkSkipsFromEnvelopes(dag, states, completedEnvelopes) {
         applyForkChoiceToSchedule(dag, node.id, chosen, states);
     }
 }
-export function applyRetryRootDelta(dag, states, completedEnvelopes, retryRoots, stageId, attempt, clearSchedulingHalt) {
+export function applyRetryRootDelta(dag, states, completedEnvelopes, retryRoots, stageId, attempt, clearSchedulingHalt, alreadyLaunchedIds) {
     retryRoots.set(stageId, attempt);
     const downstream = cloneRetryDownstream(dag, [stageId]);
     const resetIds = new Set([stageId, ...downstream]);
@@ -206,7 +209,13 @@ export function applyRetryRootDelta(dag, states, completedEnvelopes, retryRoots,
     }
     for (const id of resetIds) {
         const state = states.get(id);
-        if (state === "active" || state === "succeeded") {
+        if (state === "active") {
+            completedEnvelopes.delete(id);
+            continue;
+        }
+        if (id === stageId &&
+            state === "succeeded" &&
+            alreadyLaunchedIds?.has(id)) {
             completedEnvelopes.delete(id);
             continue;
         }
@@ -218,7 +227,7 @@ export function applyRetryRootDelta(dag, states, completedEnvelopes, retryRoots,
 export function hasWorkOutsideBranch(dag, branchRootId, states, ctx) {
     const branchIds = new Set([
         branchRootId,
-        ...collectDownstreamStageIds(dag, branchRootId),
+        ...cloneRetryDownstream(dag, [branchRootId]),
     ]);
     for (const node of dag.nodes) {
         if (branchIds.has(node.id))
@@ -253,17 +262,56 @@ export function hasWorkOutsideBranch(dag, branchRootId, states, ctx) {
 function terminalState(status) {
     return status === "succeeded" || status === "failed" || status === "skipped";
 }
+function scheduleHasUnhandledFailure(dag, states) {
+    for (const [stageId, state] of states) {
+        if (state !== "failed")
+            continue;
+        if (failureIsAccepted(dag, stageId, states))
+            continue;
+        return true;
+    }
+    return false;
+}
 function sortRunnableIds(dag, ids) {
     const indexById = new Map(dag.nodes.map((node) => [node.id, node.stageIndex]));
     return ids.slice().sort((a, b) => (indexById.get(a) ?? 0) - (indexById.get(b) ?? 0));
 }
-function isRunnable(dag, stageId, states, schedulingHalted, completedEnvelopes) {
+function isRunnable(dag, stageId, states, schedulingHalted, completedEnvelopes, feedback) {
     if (schedulingHalted)
         return false;
+    if (feedback !== undefined && isHeld(feedback, stageId))
+        return false;
     const state = states.get(stageId);
     if (state !== "pending")
         return false;
-    return cloneScheduleAllowsRun(dag, stageId, states, completedEnvelopes);
+    return cloneScheduleAllowsRun(dag, stageId, states, completedEnvelopes, {
+        activeCohortForNeeds: (needsId) => {
+            const override = feedback !== undefined ? cohortOverrideMap(feedback) : undefined;
+            if (override === undefined || override.size === 0) {
+                return { kind: "untracked" };
+            }
+            const forkParentId = forkParentForNeedsStage(dag, needsId);
+            if (forkParentId === null)
+                return { kind: "untracked" };
+            const tracked = override.get(forkParentId);
+            if (tracked === undefined)
+                return { kind: "untracked" };
+            return activeCohortFromCloneIds(tracked);
+        },
+    });
+}
+export function hydratedScheduleHasRunnableWork(hydrated) {
+    if (hydrated.schedulingHalted)
+        return false;
+    const dag = hydrated.dag;
+    if (dag === undefined)
+        return false;
+    for (const node of dag.nodes) {
+        if (isRunnable(dag, node.id, hydrated.states, hydrated.schedulingHalted, hydrated.completedEnvelopes)) {
+            return true;
+        }
+    }
+    return false;
 }
 export async function runPipelineDag(options) {
     const { prepared, maxActiveStagesPerRun } = options;
@@ -280,7 +328,34 @@ export async function runPipelineDag(options) {
     }
     const stageById = buildStageConfigById(loaded);
     const retryContext = options.retryContext;
+    const feedbackSchedule = createReplaySchedule();
+    await hydrateFromStore(feedbackSchedule, store, run.runId, dag);
+    if (retryContext !== undefined &&
+        activeReplayId(feedbackSchedule) !== undefined) {
+        const routeIds = new Set(feedbackSchedule.feedback.contextsByStageId.keys());
+        const cascade = cloneRetryDownstream(dag, [
+            ...retryContext.retryRoots.keys(),
+        ]);
+        const overrides = new Map();
+        for (const [stageId, attempt] of retryContext.retryRoots) {
+            if (routeIds.has(stageId))
+                overrides.set(stageId, attempt);
+        }
+        for (const stageId of cascade) {
+            if (!routeIds.has(stageId))
+                continue;
+            if (overrides.has(stageId))
+                continue;
+            const exec = await store.createStageExecution(run.runId, stageId);
+            overrides.set(stageId, exec.attempt);
+        }
+        await rebindRouteAttempts(feedbackSchedule, store, run.runId, overrides);
+    }
     const resolveLaunchAttempt = async (stageId) => {
+        const feedbackAttempt = launchAttemptFor(feedbackSchedule, stageId);
+        if (feedbackAttempt !== undefined) {
+            return feedbackAttempt;
+        }
         if (retryContext) {
             const preCreatedAttempt = retryContext.retryRoots.get(stageId);
             if (preCreatedAttempt !== undefined) {
@@ -289,11 +364,10 @@ export async function runPipelineDag(options) {
             const execution = await store.createStageExecution(run.runId, stageId);
             return execution.attempt;
         }
-        const attemptCount = await store.countStageAttempts(run.runId, stageId);
-        if (attemptCount === 0) {
-            await store.createStageExecution(run.runId, stageId);
-        }
-        return 1;
+        const latest = await store.getLatestStageExecution(run.runId, stageId);
+        if (latest !== null)
+            return latest.attempt;
+        return (await store.createStageExecution(run.runId, stageId)).attempt;
     };
     if (options.resumeFromStageId !== undefined) {
         const resumedId = options.resumeFromStageId;
@@ -338,6 +412,7 @@ export async function runPipelineDag(options) {
     }
     let activeCount = 0;
     const inFlight = new Set();
+    const launchedIds = new Set();
     const notifyRetryRootTerminal = (stageId, outcome) => {
         if (retryContext === undefined)
             return;
@@ -346,12 +421,26 @@ export async function runPipelineDag(options) {
             return;
         options.onRetryRootTerminal?.(stageId, attempt, outcome);
     };
+    const persistSkipRejected = async (parentId, observed) => {
+        const ids = [];
+        skipRejectedNeedDependents(dag, parentId, states, observed, (id) => {
+            if (states.get(id) !== "pending")
+                return;
+            states.set(id, "skipped");
+            ids.push(id);
+        });
+        for (const id of ids) {
+            notifyRetryRootTerminal(id, "skipped");
+            await store.appendStageEvent(run.runId, id, { event: "skipped" });
+        }
+    };
     const persistSkipPending = async (id) => {
         if (states.get(id) !== "pending")
             return;
         states.set(id, "skipped");
         notifyRetryRootTerminal(id, "skipped");
         await store.appendStageEvent(run.runId, id, { event: "skipped" });
+        await persistSkipRejected(id, "skipped");
     };
     for (const [stageId, state] of states) {
         if (state !== "failed")
@@ -369,7 +458,7 @@ export async function runPipelineDag(options) {
         }
     };
     const markBranchDownstreamSkipped = (stageId) => {
-        for (const id of collectDownstreamStageIds(dag, stageId)) {
+        for (const id of cloneRetryDownstream(dag, [stageId])) {
             if (states.get(id) === "pending") {
                 states.set(id, "skipped");
                 notifyRetryRootTerminal(id, "skipped");
@@ -387,7 +476,7 @@ export async function runPipelineDag(options) {
         options.mutationQueue.drainPending((stageId, attempt) => {
             applyRetryRootDelta(dag, states, completedEnvelopes, retryContext.retryRoots, stageId, attempt, () => {
                 schedulingHalted = false;
-            });
+            }, launchedIds);
         });
     };
     const recordCloneFailureEnvelope = async (stageId, reason, envelope) => {
@@ -409,10 +498,42 @@ export async function runPipelineDag(options) {
             }
         }
     };
+    const scheduleAutomaticRepair = async (stageId) => {
+        const recovery = dag.nodes.find((node) => node.id === stageId)?.recovery;
+        if (recovery?.mode !== "repair")
+            return false;
+        const latestAttempt = await store.getLatestStageExecution(run.runId, stageId);
+        if (latestAttempt?.verification_outcome !== "failed")
+            return false;
+        const attempts = await store.countStageAttempts(run.runId, stageId);
+        if (attempts >= recovery.max_attempts)
+            return false;
+        const nextAttempt = await store.createStageExecution(run.runId, stageId);
+        if (retryContext !== undefined) {
+            retryContext.retryRoots.set(stageId, nextAttempt.attempt);
+        }
+        if (activeReplayId(feedbackSchedule) !== undefined) {
+            await rebindRouteAttempts(feedbackSchedule, store, run.runId, new Map([[stageId, nextAttempt.attempt]]));
+        }
+        states.set(stageId, "pending");
+        completedEnvelopes.delete(stageId);
+        return true;
+    };
     const onStageFailure = async (stageId, reason, envelope) => {
+        if (await scheduleAutomaticRepair(stageId))
+            return;
+        if (activeReplayId(feedbackSchedule) !== undefined) {
+            await onRouteStageFailed({
+                store,
+                runId: run.runId,
+                schedule: feedbackSchedule,
+                stageId,
+            });
+        }
         states.set(stageId, "failed");
         notifyRetryRootTerminal(stageId, "failed");
         await recordCloneFailureEnvelope(stageId, reason, envelope);
+        await persistSkipRejected(stageId, "failed");
         if (cloneFailureContinuesSchedule(dag, stageId, completedEnvelopes)) {
             if (firstFailureReason === undefined) {
                 firstFailureReason = reason;
@@ -426,23 +547,33 @@ export async function runPipelineDag(options) {
             return;
         }
         if (retryContext) {
-            markBranchDownstreamSkipped(stageId);
+            const accepted = failureIsAccepted(dag, stageId, states);
+            if (!accepted) {
+                markBranchDownstreamSkipped(stageId);
+            }
             drainRetryMutations();
-            if (!hasWorkOutsideBranch(dag, stageId, states, outsideWorkContext())) {
+            if (!accepted &&
+                !hasWorkOutsideBranch(dag, stageId, states, outsideWorkContext())) {
                 schedulingHalted = true;
             }
-            if (firstFailureReason === undefined) {
+            if (!accepted && firstFailureReason === undefined) {
                 firstFailureReason = reason;
             }
             return;
         }
-        if (!schedulingHalted) {
-            schedulingHalted = true;
-            firstFailureReason = reason;
+        if (!failureIsAccepted(dag, stageId, states)) {
+            if (!schedulingHalted) {
+                schedulingHalted = true;
+                firstFailureReason = reason;
+            }
         }
     };
     const onStageSuccess = async (stageId, envelope) => {
-        const conflict = cloneFanoutConflict(dag, envelope);
+        const retryDownstreamIds = retryContext
+            ? cloneRetryDownstream(dag, [...retryContext.retryRoots.keys()])
+            : EMPTY_RETRY_DOWNSTREAM;
+        const forceFreshCloneIds = activeReplayId(feedbackSchedule) !== undefined;
+        const conflict = cloneFanoutConflict(dag, envelope, states, retryDownstreamIds, { forceFreshCloneIds });
         if (conflict !== undefined) {
             await store.appendStageEvent(run.runId, stageId, {
                 event: "failed",
@@ -454,10 +585,119 @@ export async function runPipelineDag(options) {
         states.set(stageId, "succeeded");
         completedEnvelopes.set(stageId, envelope);
         notifyRetryRootTerminal(stageId, "succeeded");
+        const feedbackAction = envelope.feedback_loop;
+        if (feedbackAction?.action === "send_back") {
+            const node = dag.nodes.find((n) => n.id === stageId);
+            if (node === undefined || node.feedback_loop === undefined) {
+                await handlePostSuccessError(stageId, `feedback_loop send_back from stage "${stageId}" without feedback_loop policy`);
+                return;
+            }
+            const latest = await store.getLatestStageExecution(run.runId, stageId);
+            const sourceAttempt = latest?.attempt ?? 1;
+            const applied = await applySendBack({
+                store,
+                runId: run.runId,
+                sourceNode: node,
+                sourceAttempt,
+                envelope,
+                schedule: feedbackSchedule,
+                projection: {
+                    dag,
+                    states,
+                    completedEnvelopes,
+                },
+            });
+            if (applied.kind === "rejected") {
+                await handlePostSuccessError(stageId, applied.reason);
+                return;
+            }
+            if (applied.kind === "waiting_for_human") {
+                const waitAttempt = sourceAttempt;
+                await store.appendStageEvent(run.runId, stageId, { event: "waiting_for_input" }, { attempt: waitAttempt });
+                try {
+                    await store.updateStageExecution(run.runId, stageId, waitAttempt, {
+                        status: "waiting_for_input",
+                    });
+                }
+                catch {
+                    // execution row may be absent in unusual harnesses
+                }
+                states.set(stageId, "waiting");
+                if (options.onFeedbackLoopWaiting !== undefined) {
+                    let decisionResult;
+                    await options.onFeedbackLoopWaiting({
+                        loopId: applied.loop.loop_id,
+                        sourceStageId: stageId,
+                        resolve: async (decision) => {
+                            decisionResult = await resolveDecision({
+                                store,
+                                runId: run.runId,
+                                decision: decision.decision,
+                                loopId: applied.loop.loop_id,
+                                reason: decision.reason,
+                                schedule: feedbackSchedule,
+                                projection: {
+                                    dag,
+                                    states,
+                                    completedEnvelopes,
+                                },
+                                sourceNode: node,
+                            });
+                            return decisionResult;
+                        },
+                    });
+                    if (decisionResult === undefined) {
+                        return;
+                    }
+                    if (!decisionResult.ok) {
+                        await handlePostSuccessError(stageId, decisionResult.reason);
+                        return;
+                    }
+                    if (decisionResult.effect === "abandoned") {
+                        schedulingHalted = true;
+                        if (firstFailureReason === undefined) {
+                            firstFailureReason = decisionResult.reason;
+                        }
+                        return;
+                    }
+                    return;
+                }
+                return;
+            }
+            for (const id of applied.retiredSkippedStageIds) {
+                await store.appendStageEvent(run.runId, id, { event: "skipped" });
+            }
+            return;
+        }
+        if (feedbackAction?.action === "continue") {
+            await onContinued({
+                store,
+                runId: run.runId,
+                sourceStageId: stageId,
+                envelope,
+                schedule: feedbackSchedule,
+            });
+        }
+        else if (activeReplayId(feedbackSchedule) !== undefined) {
+            await onRouteStageSucceeded({
+                store,
+                runId: run.runId,
+                schedule: feedbackSchedule,
+                stageId,
+                envelope,
+            });
+        }
         const node = dag.nodes.find((n) => n.id === stageId);
         const protectedIds = protectedClonableChildIds(dag, stageId, envelope);
         if (node?.fork) {
             const chosen = normalizeForkChoice(envelope.fork_choice, "stored");
+            const sourceId = activeSourceStageId(feedbackSchedule);
+            if (activeReplayId(feedbackSchedule) !== undefined &&
+                sourceId !== undefined &&
+                !forkChoicePreservesReplaySource(dag, new Set(chosen), sourceId)) {
+                await handlePostSuccessError(stageId, `fork_choice during feedback replay must preserve a path to source "${sourceId}"`);
+                return;
+            }
             const skippedIds = [];
             applyForkChoiceToSchedule(dag, stageId, chosen, states, {
                 protectedIds,
@@ -470,7 +710,9 @@ export async function runPipelineDag(options) {
                 await store.appendStageEvent(run.runId, id, { event: "skipped" });
             }
         }
-        const applied = applyCloneForksToSchedule(dag, stageId, envelope, states);
+        const applied = applyCloneForksToSchedule(dag, stageId, envelope, states, {
+            forceFreshCloneIds,
+        });
         const dagChanged = applied.dag !== dag;
         dag = applied.dag;
         if (dagChanged) {
@@ -480,6 +722,21 @@ export async function runPipelineDag(options) {
             notifyRetryRootTerminal(id, "skipped");
             await store.appendStageEvent(run.runId, id, { event: "skipped" });
         }
+        if (applied.mintedBySuccessor !== undefined) {
+            for (const [, cloneStageIds] of applied.mintedBySuccessor) {
+                if (cloneStageIds.length === 0)
+                    continue;
+                const replayId = activeReplayId(feedbackSchedule);
+                await mintCohortForFanout({
+                    store,
+                    runId: run.runId,
+                    ...(replayId !== undefined ? { replayId } : {}),
+                    forkParent: stageId,
+                    cloneStageIds,
+                });
+                noteActiveCohort(feedbackSchedule, stageId, cloneStageIds);
+            }
+        }
     };
     const handlePostSuccessError = async (stageId, reason) => {
         if (states.get(stageId) === "succeeded") {
@@ -500,6 +757,22 @@ export async function runPipelineDag(options) {
         }
         const attempt = await resolveLaunchAttempt(stageId);
         const attemptCtx = attemptContext(attempt);
+        const prep = launchFor(feedbackSchedule, stageId);
+        const feedbackLoopContext = prep?.feedbackLoopContext;
+        const sessionMode = prep?.sessionMode;
+        const priorAttempt = prep?.priorAttempt;
+        const resumeToken = sessionMode === "feedback_resume" && priorAttempt !== undefined
+            ? resumeSessionFilePath(run.workspaceDir, stageId, priorAttempt)
+            : undefined;
+        if (activeReplayId(feedbackSchedule) !== undefined &&
+            feedbackLoopContext !== undefined) {
+            await onRouteStageRunning({
+                store,
+                runId: run.runId,
+                schedule: feedbackSchedule,
+                stageId,
+            });
+        }
         if (executionMode === "process") {
             const launcher = options.stageProcessLauncher;
             if (!launcher) {
@@ -511,6 +784,16 @@ export async function runPipelineDag(options) {
                 stageId,
                 rootDir: factoryCwd,
                 attempt,
+                ...(sessionMode !== undefined
+                    ? {
+                        mode: sessionMode === "feedback_resume"
+                            ? "feedback_resume"
+                            : sessionMode === "new_session"
+                                ? "new_session"
+                                : "run",
+                    }
+                    : {}),
+                ...(resumeToken !== undefined ? { sessionFilePath: resumeToken } : {}),
                 ...(prepared.operatorCatalog !== undefined
                     ? { operatorCatalog: prepared.operatorCatalog }
                     : {}),
@@ -550,6 +833,9 @@ export async function runPipelineDag(options) {
             operatorCatalog: prepared.operatorCatalog,
             completedEnvelopes,
             skipGates: prepared.skipGates,
+            ...(sessionMode !== undefined ? { sessionMode } : {}),
+            ...(feedbackLoopContext !== undefined ? { feedbackLoopContext } : {}),
+            ...(resumeToken !== undefined ? { resumeToken } : {}),
         });
         if (isRunStageWaiting(result)) {
             await onStageFailure(stageId, WAIT_WITHOUT_WORKER_DISPATCH);
@@ -577,6 +863,7 @@ export async function runPipelineDag(options) {
         if (states.get(stageId) !== "pending")
             return;
         states.set(stageId, "active");
+        launchedIds.add(stageId);
         activeCount += 1;
         const taskPromise = launchStage(stageId).finally(() => {
             activeCount -= 1;
@@ -587,7 +874,7 @@ export async function runPipelineDag(options) {
     const pickRunnable = () => {
         const ids = [];
         for (const node of dag.nodes) {
-            if (isRunnable(dag, node.id, states, schedulingHalted, completedEnvelopes)) {
+            if (isRunnable(dag, node.id, states, schedulingHalted, completedEnvelopes, feedbackSchedule)) {
                 ids.push(node.id);
             }
         }
@@ -660,11 +947,7 @@ export async function runPipelineDag(options) {
             reason: firstFailureReason,
         };
     }
-    const allSucceeded = dag.nodes.every((node) => {
-        const s = states.get(node.id);
-        return s === "succeeded" || s === "skipped";
-    });
-    if (!allSucceeded) {
+    if (scheduleHasUnhandledFailure(dag, states) || !allTerminal()) {
         if (retryContext === undefined) {
             await store.updateRunStatus(run.runId, "failed");
         }