stageflow 0.1.0

package/dist/runtime/pipelineScheduler.js

@@ -0,0 +1,494 @@
+import { collectDownstreamClosure, collectDownstreamStageIds, } from "./dagTraversal.js";
+import { buildCompletedEnvelopesFromRun, buildStageConfigById, } from "./envelopeRouting.js";
+import { WAIT_WITHOUT_WORKER_DISPATCH } from "./answerResume.js";
+import { runStage, isRunStageWaiting } from "./stageRunner.js";
+import { attemptContext } from "./stageAttemptContext.js";
+export function snapshotToScheduleState(snap, overrides, stageId, executionMode) {
+    const override = overrides.get(stageId);
+    if (override !== undefined)
+        return override;
+    switch (snap.status) {
+        case "succeeded":
+            return "succeeded";
+        case "failed":
+            return "failed";
+        case "running":
+            return "active";
+        case "waiting_for_input":
+            return executionMode === "process" ? "waiting" : "active";
+        default:
+            return "pending";
+    }
+}
+export async function hydrateScheduleFromStore(store, runId, dag, executionMode = "process") {
+    const states = new Map();
+    for (const node of dag.nodes) {
+        states.set(node.id, "pending");
+    }
+    const runDetail = await store.readRun(runId);
+    const completedEnvelopes = await buildCompletedEnvelopesFromRun(store, runId, runDetail.stages);
+    let schedulingHalted = false;
+    const overrides = new Map();
+    for (const snap of runDetail.stages) {
+        const state = snapshotToScheduleState(snap, overrides, snap.stage_id, executionMode);
+        states.set(snap.stage_id, state);
+        if (state === "failed") {
+            schedulingHalted = true;
+        }
+    }
+    return {
+        states,
+        completedEnvelopes,
+        schedulingHalted,
+        firstFailureReason: undefined,
+    };
+}
+export async function hydrateScheduleForRetryRoots(store, runId, dag, retryStageIds) {
+    const runDetail = await store.readRun(runId);
+    const downstream = collectDownstreamClosure(dag, retryStageIds);
+    const retryRootSet = new Set(retryStageIds);
+    const completedEnvelopes = await buildCompletedEnvelopesFromRun(store, runId, runDetail.stages);
+    const states = new Map();
+    for (const node of dag.nodes) {
+        states.set(node.id, "pending");
+    }
+    const overrides = new Map();
+    for (const rootId of retryStageIds) {
+        overrides.set(rootId, "pending");
+    }
+    for (const id of downstream) {
+        overrides.set(id, "pending");
+    }
+    for (const snap of runDetail.stages) {
+        const id = snap.stage_id;
+        const state = snapshotToScheduleState(snap, overrides, id, "process");
+        states.set(id, state);
+        if (retryRootSet.has(id) || downstream.has(id)) {
+            completedEnvelopes.delete(id);
+        }
+    }
+    return {
+        states,
+        completedEnvelopes,
+        schedulingHalted: false,
+        firstFailureReason: undefined,
+    };
+}
+export async function hydrateScheduleForRetry(store, runId, dag, retryStageId) {
+    return hydrateScheduleForRetryRoots(store, runId, dag, [retryStageId]);
+}
+export async function resumeRun(options) {
+    const { prepared, resumeFromStageId, initialPrior } = options;
+    const dag = prepared.loaded.dag;
+    const downstream = dag.childrenOf[resumeFromStageId];
+    if (downstream?.length) {
+        return runPipelineDag(options);
+    }
+    const executionMode = options.executionMode ?? "process";
+    const hydrated = await hydrateScheduleFromStore(prepared.store, prepared.run.runId, dag, executionMode);
+    hydrated.states.set(resumeFromStageId, "succeeded");
+    if (initialPrior !== undefined && initialPrior !== null) {
+        hydrated.completedEnvelopes.set(resumeFromStageId, initialPrior);
+    }
+    const hasActive = [...hydrated.states.values()].some((s) => s === "active");
+    if (hasActive) {
+        return {
+            ok: true,
+            runDir: prepared.run.workspaceDir,
+            runId: prepared.run.runId,
+        };
+    }
+    const allTerminal = dag.nodes.every((node) => {
+        const state = hydrated.states.get(node.id);
+        return state !== undefined && terminalState(state);
+    });
+    if (allTerminal) {
+        const allSucceeded = dag.nodes.every((node) => hydrated.states.get(node.id) === "succeeded");
+        await prepared.store.updateRunStatus(prepared.run.runId, allSucceeded ? "succeeded" : "failed");
+        return {
+            ok: allSucceeded,
+            runDir: prepared.run.workspaceDir,
+            runId: prepared.run.runId,
+            ...(allSucceeded
+                ? {}
+                : { reason: hydrated.firstFailureReason ?? "pipeline incomplete" }),
+        };
+    }
+    return runPipelineDag(options);
+}
+export async function retryRun(options) {
+    const { prepared, retryRoots } = options;
+    const hydrated = await hydrateScheduleForRetryRoots(prepared.store, prepared.run.runId, prepared.loaded.dag, [...retryRoots.keys()]);
+    return runPipelineDag({
+        prepared: options.prepared,
+        maxActiveStagesPerRun: options.maxActiveStagesPerRun,
+        executionMode: options.executionMode,
+        stageProcessLauncher: options.stageProcessLauncher,
+        initialSchedule: hydrated,
+        retryContext: { retryRoots },
+        mutationQueue: options.mutationQueue,
+        onLoopTick: options.onLoopTick,
+        onRetryRootTerminal: options.onRetryRootTerminal,
+    });
+}
+export function applyRetryRootDelta(dag, states, completedEnvelopes, retryRoots, stageId, attempt, clearSchedulingHalt) {
+    retryRoots.set(stageId, attempt);
+    const downstream = collectDownstreamClosure(dag, [stageId]);
+    const resetIds = new Set([stageId, ...downstream]);
+    for (const id of resetIds) {
+        const state = states.get(id);
+        if (state === "active" || state === "succeeded") {
+            completedEnvelopes.delete(id);
+            continue;
+        }
+        states.set(id, "pending");
+        completedEnvelopes.delete(id);
+    }
+    clearSchedulingHalt?.();
+}
+export function hasWorkOutsideBranch(dag, branchRootId, states, ctx) {
+    const branchIds = new Set([
+        branchRootId,
+        ...collectDownstreamStageIds(dag, branchRootId),
+    ]);
+    for (const node of dag.nodes) {
+        if (branchIds.has(node.id))
+            continue;
+        const state = states.get(node.id);
+        if (state === "pending" || state === "active" || state === "waiting") {
+            return true;
+        }
+    }
+    if (ctx?.pendingDeltaStageIds !== undefined) {
+        for (const stageId of ctx.pendingDeltaStageIds) {
+            if (!branchIds.has(stageId)) {
+                return true;
+            }
+        }
+    }
+    if (ctx?.retryRoots !== undefined) {
+        for (const [rootId] of ctx.retryRoots) {
+            if (branchIds.has(rootId))
+                continue;
+            const state = states.get(rootId);
+            if (state === "pending" || state === "active" || state === "waiting") {
+                return true;
+            }
+            if (state === "failed") {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+function terminalState(status) {
+    return status === "succeeded" || status === "failed" || status === "skipped";
+}
+function sortRunnableIds(dag, ids) {
+    const indexById = new Map(dag.nodes.map((node) => [node.id, node.stageIndex]));
+    return ids.slice().sort((a, b) => (indexById.get(a) ?? 0) - (indexById.get(b) ?? 0));
+}
+function isRunnable(dag, stageId, states, schedulingHalted) {
+    if (schedulingHalted)
+        return false;
+    const state = states.get(stageId);
+    if (state !== "pending")
+        return false;
+    const node = dag.nodes.find((n) => n.id === stageId);
+    if (!node)
+        return false;
+    if (node.needs === null)
+        return true;
+    return states.get(node.needs) === "succeeded";
+}
+export async function runPipelineDag(options) {
+    const { prepared, maxActiveStagesPerRun } = options;
+    const executionMode = options.executionMode ?? "process";
+    const { agent, task, loaded, run, store, checkoutRoot, hitl, cwd, } = prepared;
+    const dag = loaded.dag;
+    const stageById = buildStageConfigById(loaded);
+    let { states, completedEnvelopes, schedulingHalted, firstFailureReason } = options.initialSchedule ??
+        (await hydrateScheduleFromStore(store, run.runId, dag, executionMode));
+    const retryContext = options.retryContext;
+    const resolveLaunchAttempt = async (stageId) => {
+        if (retryContext) {
+            const preCreatedAttempt = retryContext.retryRoots.get(stageId);
+            if (preCreatedAttempt !== undefined) {
+                return preCreatedAttempt;
+            }
+            const execution = await store.createStageExecution(run.runId, stageId);
+            return execution.attempt;
+        }
+        const attemptCount = await store.countStageAttempts(run.runId, stageId);
+        if (attemptCount === 0) {
+            await store.createStageExecution(run.runId, stageId);
+        }
+        return 1;
+    };
+    if (options.resumeFromStageId !== undefined) {
+        const resumedId = options.resumeFromStageId;
+        states.set(resumedId, "succeeded");
+        if (options.initialPrior !== undefined && options.initialPrior !== null) {
+            completedEnvelopes.set(resumedId, options.initialPrior);
+        }
+        else if (!completedEnvelopes.has(resumedId)) {
+            try {
+                const envelope = await store.readEnvelope(run.runId, resumedId);
+                completedEnvelopes.set(resumedId, envelope);
+            }
+            catch {
+                // prior resolution will fail for downstream if envelope missing
+            }
+        }
+    }
+    if (options.startAtStageIndex !== undefined) {
+        const startAt = options.startAtStageIndex;
+        for (const node of dag.nodes) {
+            if (node.stageIndex < startAt) {
+                states.set(node.id, "succeeded");
+                if (!completedEnvelopes.has(node.id)) {
+                    try {
+                        const envelope = await store.readEnvelope(run.runId, node.id);
+                        completedEnvelopes.set(node.id, envelope);
+                    }
+                    catch {
+                        // linear shim: upstream envelopes expected on disk
+                    }
+                }
+            }
+        }
+    }
+    let activeCount = 0;
+    const inFlight = new Set();
+    const notifyRetryRootTerminal = (stageId, outcome) => {
+        if (retryContext === undefined)
+            return;
+        const attempt = retryContext.retryRoots.get(stageId);
+        if (attempt === undefined)
+            return;
+        options.onRetryRootTerminal?.(stageId, attempt, outcome);
+    };
+    const markSkippedPending = () => {
+        for (const node of dag.nodes) {
+            if (states.get(node.id) === "pending") {
+                states.set(node.id, "skipped");
+                notifyRetryRootTerminal(node.id, "skipped");
+            }
+        }
+    };
+    const markBranchDownstreamSkipped = (stageId) => {
+        for (const id of collectDownstreamStageIds(dag, stageId)) {
+            if (states.get(id) === "pending") {
+                states.set(id, "skipped");
+                notifyRetryRootTerminal(id, "skipped");
+            }
+        }
+    };
+    const outsideWorkContext = () => ({
+        retryRoots: retryContext?.retryRoots,
+        pendingDeltaStageIds: options.mutationQueue?.pendingStageIds(),
+    });
+    const drainRetryMutations = () => {
+        if (options.mutationQueue === undefined || retryContext === undefined) {
+            return;
+        }
+        options.mutationQueue.drainPending((stageId, attempt) => {
+            applyRetryRootDelta(dag, states, completedEnvelopes, retryContext.retryRoots, stageId, attempt, () => {
+                schedulingHalted = false;
+            });
+        });
+    };
+    const onStageFailure = async (stageId, reason) => {
+        states.set(stageId, "failed");
+        notifyRetryRootTerminal(stageId, "failed");
+        if (retryContext) {
+            markBranchDownstreamSkipped(stageId);
+            drainRetryMutations();
+            if (!hasWorkOutsideBranch(dag, stageId, states, outsideWorkContext())) {
+                schedulingHalted = true;
+            }
+            if (firstFailureReason === undefined) {
+                firstFailureReason = reason;
+            }
+            return;
+        }
+        if (!schedulingHalted) {
+            schedulingHalted = true;
+            firstFailureReason = reason;
+        }
+    };
+    const onStageSuccess = (stageId, envelope) => {
+        states.set(stageId, "succeeded");
+        completedEnvelopes.set(stageId, envelope);
+        notifyRetryRootTerminal(stageId, "succeeded");
+    };
+    const launchStage = async (stageId) => {
+        const stage = stageById.get(stageId);
+        if (!stage) {
+            await onStageFailure(stageId, `stage config missing for "${stageId}"`);
+            return;
+        }
+        const attempt = await resolveLaunchAttempt(stageId);
+        const attemptCtx = attemptContext(attempt);
+        if (executionMode === "process") {
+            const launcher = options.stageProcessLauncher;
+            if (!launcher) {
+                await onStageFailure(stageId, "stage process launcher is not configured");
+                return;
+            }
+            const launchResult = await launcher.launch({
+                runId: run.runId,
+                stageId,
+                rootDir: cwd,
+                attempt,
+                ...(prepared.operatorCatalog !== undefined
+                    ? { operatorCatalog: prepared.operatorCatalog }
+                    : {}),
+            });
+            if (launchResult.type === "succeeded") {
+                try {
+                    const envelope = await store.readEnvelope(run.runId, stageId);
+                    onStageSuccess(stageId, envelope);
+                }
+                catch {
+                    states.set(stageId, "succeeded");
+                    notifyRetryRootTerminal(stageId, "succeeded");
+                }
+                return;
+            }
+            if (launchResult.type === "waiting") {
+                states.set(stageId, "waiting");
+                return;
+            }
+            await onStageFailure(stageId, launchResult.reason);
+            return;
+        }
+        const result = await runStage({
+            agent,
+            store,
+            runId: run.runId,
+            stage,
+            task,
+            dag,
+            checkoutRoot,
+            workspaceDir: run.workspaceDir,
+            hitl,
+            attemptCtx,
+            factoryCwd: cwd,
+            operatorCatalog: prepared.operatorCatalog,
+            completedEnvelopes,
+        });
+        if (isRunStageWaiting(result)) {
+            await onStageFailure(stageId, WAIT_WITHOUT_WORKER_DISPATCH);
+            return;
+        }
+        if (!result.ok) {
+            await onStageFailure(stageId, result.reason ?? "stage failed");
+            return;
+        }
+        if (result.envelope) {
+            onStageSuccess(stageId, result.envelope);
+        }
+        else {
+            states.set(stageId, "succeeded");
+            notifyRetryRootTerminal(stageId, "succeeded");
+        }
+    };
+    const startStage = (stageId) => {
+        if (states.get(stageId) !== "pending")
+            return;
+        states.set(stageId, "active");
+        activeCount += 1;
+        const taskPromise = launchStage(stageId).finally(() => {
+            activeCount -= 1;
+            inFlight.delete(taskPromise);
+        });
+        inFlight.add(taskPromise);
+    };
+    const pickRunnable = () => {
+        const ids = [];
+        for (const node of dag.nodes) {
+            if (isRunnable(dag, node.id, states, schedulingHalted)) {
+                ids.push(node.id);
+            }
+        }
+        return sortRunnableIds(dag, ids);
+    };
+    const allTerminal = () => {
+        for (const node of dag.nodes) {
+            const state = states.get(node.id);
+            if (state === undefined || !terminalState(state)) {
+                return false;
+            }
+        }
+        return true;
+    };
+    while (!allTerminal()) {
+        drainRetryMutations();
+        if (options.onLoopTick !== undefined) {
+            await options.onLoopTick();
+        }
+        if (!schedulingHalted) {
+            const runnable = pickRunnable();
+            for (const stageId of runnable) {
+                if (activeCount >= maxActiveStagesPerRun)
+                    break;
+                startStage(stageId);
+            }
+        }
+        if (inFlight.size === 0) {
+            if (schedulingHalted) {
+                markSkippedPending();
+                break;
+            }
+            const runnable = pickRunnable();
+            if (runnable.length === 0) {
+                const hasWaiting = [...states.values()].some((s) => s === "waiting");
+                if (hasWaiting) {
+                    break;
+                }
+                break;
+            }
+            continue;
+        }
+        await Promise.race([
+            ...inFlight,
+            new Promise((resolve) => setTimeout(resolve, 25)),
+        ]);
+    }
+    if (inFlight.size > 0) {
+        await Promise.all(inFlight);
+    }
+    const hasWaiting = [...states.values()].some((s) => s === "waiting");
+    if (hasWaiting && !schedulingHalted) {
+        return { ok: true, runDir: run.workspaceDir, runId: run.runId };
+    }
+    if (schedulingHalted) {
+        markSkippedPending();
+        if (retryContext === undefined) {
+            await store.updateRunStatus(run.runId, "failed");
+        }
+        return {
+            ok: false,
+            runDir: run.workspaceDir,
+            runId: run.runId,
+            reason: firstFailureReason,
+        };
+    }
+    const allSucceeded = dag.nodes.every((node) => states.get(node.id) === "succeeded");
+    if (!allSucceeded) {
+        if (retryContext === undefined) {
+            await store.updateRunStatus(run.runId, "failed");
+        }
+        return {
+            ok: false,
+            runDir: run.workspaceDir,
+            runId: run.runId,
+            reason: firstFailureReason ?? "pipeline incomplete",
+        };
+    }
+    if (retryContext === undefined) {
+        await store.updateRunStatus(run.runId, "succeeded");
+    }
+    return { ok: true, runDir: run.workspaceDir, runId: run.runId };
+}

package/dist/runtime/pipelineValidationError.d.ts

@@ -0,0 +1,6 @@
+import type { ValidationResult } from "../config/validateCatalog.js";
+export declare class PipelineValidationError extends Error {
+    readonly result: ValidationResult;
+    readonly name = "PipelineValidationError";
+    constructor(result: ValidationResult);
+}

package/dist/runtime/pipelineValidationError.js

@@ -0,0 +1,8 @@
+export class PipelineValidationError extends Error {
+    result;
+    name = "PipelineValidationError";
+    constructor(result) {
+        super("Pipeline validation failed");
+        this.result = result;
+    }
+}

package/dist/runtime/resumeReconstruct.d.ts

@@ -0,0 +1,33 @@
+import type { AgentPort, OpaqueAnswer } from "../agent/port.js";
+import type { RunStore, RunMeta } from "../runstore/port.js";
+import type { LoadedPipeline } from "../types/pipeline.js";
+import type { TaskFile } from "../types/task.js";
+import type { StageExecutionMode } from "./stageConcurrency.js";
+import type { StageProcessLauncher } from "./stageProcessLauncher.js";
+import { StageHitlController } from "./stageHitl.js";
+import { type OperatorCatalog } from "./stageAttemptBootstrap.js";
+export type PreparedResumeContext = {
+    runId: string;
+    stageId: string;
+    opaqueAnswer: OpaqueAnswer;
+    agent: AgentPort;
+    store: RunStore;
+    hitl: StageHitlController;
+    executionMode: StageExecutionMode;
+    stageProcessLauncher?: StageProcessLauncher;
+    cwd: string;
+    maxActiveStagesPerRun: number;
+    operatorCatalog?: OperatorCatalog;
+};
+export type LoadedRunContext = {
+    meta: RunMeta;
+    taskYaml: string;
+    task: TaskFile;
+    loaded: LoadedPipeline;
+    workspaceDir: string;
+};
+export declare function loadRunContext(store: RunStore, runId: string, cwd: string): Promise<LoadedRunContext>;
+export declare function reconstructAndContinue(ctx: PreparedResumeContext): Promise<{
+    ok: boolean;
+    reason?: string;
+}>;

package/dist/runtime/resumeReconstruct.js

@@ -0,0 +1,144 @@
+import { existsSync } from "node:fs";
+import { fakeHitlResumePath } from "../agent/fakeAgent.js";
+import { loadPipeline } from "../config/loadPipeline.js";
+import { loadTaskFromYaml } from "../config/loadTask.js";
+import { WAIT_WITHOUT_WORKER_DISPATCH } from "./answerResume.js";
+import { resumeRun } from "./pipelineScheduler.js";
+import { buildStageRoots, withResolvedAuthPath } from "./stageRoots.js";
+import { runStage, isRunStageWaiting } from "./stageRunner.js";
+import { attemptContext } from "./stageAttemptContext.js";
+import { openStageAttempt, } from "./stageAttemptBootstrap.js";
+export async function loadRunContext(store, runId, cwd) {
+    const meta = await store.readRunMeta(runId);
+    const taskYaml = await store.readTaskYaml(runId);
+    const task = loadTaskFromYaml(taskYaml, `run ${runId} task`);
+    const loaded = await loadPipeline(meta.pipeline_id, { cwd });
+    const workspaceDir = store.getWorkspaceDir(runId);
+    return { meta, taskYaml, task, loaded, workspaceDir };
+}
+export async function reconstructAndContinue(ctx) {
+    const { runId, stageId, opaqueAnswer, agent, store, cwd } = ctx;
+    try {
+        const latestExecution = await store.getLatestStageExecution(runId, stageId);
+        const attemptCtx = attemptContext(latestExecution?.attempt ?? 1);
+        const attemptOpt = attemptCtx.eventOptions();
+        const { meta, task, loaded } = await loadRunContext(store, runId, cwd);
+        const stageIndex = loaded.stages.findIndex((s) => s.id === stageId);
+        if (stageIndex < 0) {
+            const reason = `Stage ${stageId} not in pipeline ${meta.pipeline_id}`;
+            await store.appendStageEvent(runId, stageId, {
+                event: "failed",
+                reason,
+            }, attemptOpt);
+            await store.updateRunStatus(runId, "failed");
+            return { ok: false, reason };
+        }
+        const stage = loaded.stages[stageIndex];
+        const workspaceDir = store.getWorkspaceDir(runId);
+        const checkoutRoot = meta.checkout_root;
+        const roots = withResolvedAuthPath(buildStageRoots(workspaceDir, stageId, checkoutRoot, attemptCtx), cwd);
+        const fakeResume = fakeHitlResumePath(roots, stageId);
+        const piSession = attemptCtx.sessionPath(workspaceDir, stageId);
+        if (!existsSync(fakeResume) && !existsSync(piSession)) {
+            const reason = "missing resume context for waiting stage";
+            await store.appendStageEvent(runId, stageId, {
+                event: "failed",
+                reason,
+            }, attemptOpt);
+            await store.updateRunStatus(runId, "failed");
+            return { ok: false, reason };
+        }
+        const opened = await openStageAttempt({
+            agent,
+            store,
+            runId,
+            stage,
+            task,
+            dag: loaded.dag,
+            checkoutRoot,
+            workspaceDir,
+            factoryCwd: cwd,
+            attemptCtx,
+            operatorCatalog: ctx.operatorCatalog,
+            onActivity: (event) => {
+                void store.appendStageEvent(runId, stageId, event, attemptOpt);
+            },
+        });
+        if (!opened.ok) {
+            await store.appendStageEvent(runId, stageId, {
+                event: "failed",
+                reason: opened.reason,
+            }, attemptOpt);
+            await store.updateRunStatus(runId, "failed");
+            return { ok: false, reason: opened.reason };
+        }
+        opened.handle.deliverAnswer(opaqueAnswer);
+        const result = await runStage({
+            agent,
+            store,
+            runId,
+            stage,
+            task,
+            dag: loaded.dag,
+            checkoutRoot,
+            workspaceDir,
+            hitl: ctx.hitl,
+            skipStarted: true,
+            existingHandle: opened.handle,
+            attemptCtx,
+            factoryCwd: cwd,
+            operatorCatalog: ctx.operatorCatalog,
+        });
+        if (isRunStageWaiting(result)) {
+            await store.updateRunStatus(runId, "failed");
+            return {
+                ok: false,
+                reason: WAIT_WITHOUT_WORKER_DISPATCH,
+            };
+        }
+        if (!result.ok) {
+            await store.updateRunStatus(runId, "failed");
+            return { ok: false, reason: result.reason };
+        }
+        const rest = await resumeRun({
+            prepared: {
+                task,
+                loaded,
+                run: { runId, workspaceDir },
+                agent,
+                store,
+                cwd,
+                checkoutRoot,
+                hitl: ctx.hitl,
+                operatorCatalog: ctx.operatorCatalog,
+            },
+            maxActiveStagesPerRun: ctx.maxActiveStagesPerRun,
+            resumeFromStageId: stageId,
+            initialPrior: result.envelope ?? null,
+            executionMode: ctx.executionMode,
+            stageProcessLauncher: ctx.stageProcessLauncher,
+        });
+        return rest.ok
+            ? { ok: true }
+            : { ok: false, reason: rest.reason };
+    }
+    catch (err) {
+        const reason = err instanceof Error
+            ? err.message
+            : `resume failed: ${String(err)}`;
+        try {
+            const latestExecution = await store.getLatestStageExecution(runId, stageId);
+            const attemptCtx = attemptContext(latestExecution?.attempt ?? 1);
+            const attemptOpt = attemptCtx.eventOptions();
+            await store.appendStageEvent(runId, stageId, {
+                event: "failed",
+                reason,
+            }, attemptOpt);
+            await store.updateRunStatus(runId, "failed");
+        }
+        catch {
+            // ignore secondary failures
+        }
+        return { ok: false, reason };
+    }
+}