stageflow 0.23.0 → 0.27.0

package/dist/runtime/pipelineRunner.js

@@ -2,7 +2,7 @@ import { readFile } from "node:fs/promises";
 import path from "node:path";
 import { buildValidationResult, loadPipelineValidated, } from "../config/validateCatalog.js";
 import { loadTaskFromYaml } from "../config/loadTask.js";
-import { resolveAndValidateCheckout } from "./stageRoots.js";
+import { resolveAndValidateCheckout, resolveEffectiveGitIdentity, } from "./stageRoots.js";
 import { buildPipelineDagSnapshotFromLoaded } from "../runstore/pipelineDagSnapshot.js";
 import { normalizeCatalogPath } from "../runstore/normalizeCatalogPath.js";
 import { runPipelineDag } from "./pipelineScheduler.js";
@@ -10,7 +10,55 @@ import { readMaxActiveStagesPerRun, readStageExecutionMode, } from "./stageConcu
 import { StageProcessLauncher } from "./stageProcessLauncher.js";
 import { PipelineValidationError } from "./pipelineValidationError.js";
 import { checkTaskEntryInput } from "./taskInput.js";
+import { pipelinePersistenceForStart } from "./startPayload.js";
+import { materializeRunSkills, } from "./runSkills.js";
+import { preflightFailureCode, runPipelinePreflight, } from "../preflight/pipelinePreflight.js";
+import { networkError } from "../errors/codes.js";
+import { newRunId } from "../runstore/paths.js";
+import { bindingFromFields, buildInitialRunManifest, collectDeclaredSecretValues, finaliseStoredRunManifest, } from "../runstore/runManifest.js";
+import { getRequestAuth, } from "../server/requestAuthContext.js";
+import { registerNamedSecrets } from "../logging/namedSecrets.js";
+import { shouldRegisterValue } from "../logging/redact.js";
 export { PipelineValidationError } from "./pipelineValidationError.js";
+export class PipelinePreflightError extends Error {
+    code;
+    preflight;
+    constructor(code, preflight) {
+        super(`Pipeline preflight failed: ${code}`);
+        this.name = "PipelinePreflightError";
+        this.code = code;
+        this.preflight = preflight;
+    }
+    toNetworkBody() {
+        return networkError(this.code, this.message, {
+            checks: this.preflight.checks,
+        });
+    }
+}
+export class InlinePipelineTooLargeError extends Error {
+    code = "inline_pipeline_too_large";
+    bytes;
+    maxBytes;
+    constructor(bytes, maxBytes) {
+        super(`Inline pipeline body is ${bytes} bytes; max is ${maxBytes} (inline_pipeline_too_large)`);
+        this.name = "InlinePipelineTooLargeError";
+        this.bytes = bytes;
+        this.maxBytes = maxBytes;
+    }
+}
+/** Thrown when queued→running CAS fails (e.g. cancel won the race). */
+export class QueuedRunActivationAborted extends Error {
+    runId;
+    currentStatus;
+    constructor(runId, currentStatus) {
+        super(currentStatus === undefined
+            ? `Queued run ${runId} could not be activated`
+            : `Queued run ${runId} could not be activated (status=${currentStatus})`);
+        this.name = "QueuedRunActivationAborted";
+        this.runId = runId;
+        this.currentStatus = currentStatus;
+    }
+}
 let defaultStageProcessLauncher;
 function resolveStageProcessLauncher(executionMode, override) {
     if (executionMode !== "process") {
@@ -34,6 +82,14 @@ async function preparePipeline(options) {
         throw new PipelineValidationError(buildValidationResult("pipeline", loadResult.findings, false));
     }
     const loaded = loadResult.loaded;
+    const preflight = await runPipelinePreflight(loaded, {
+        projectRoot: options.projectRoot ?? options.cwd,
+        forStart: true,
+    });
+    if (!preflight.ok) {
+        const code = preflightFailureCode(preflight, { forStart: true }) ?? "missing_tool";
+        throw new PipelinePreflightError(code, preflight);
+    }
     const taskYaml = options.taskYaml ??
         (options.taskPath
             ? await readFile(options.taskPath, "utf8")
@@ -56,27 +112,122 @@ async function preparePipeline(options) {
     for (const finding of warningFindings) {
         console.error(`${finding.code}: ${finding.message}`);
     }
-    const checkoutRoot = await resolveAndValidateCheckout(task, options.checkoutOverride, options.cwd);
+    // Materialization is owned by RunManager (KTD1). When checkoutRoot is supplied,
+    // do not re-resolve or fetch here.
+    const checkoutRoot = options.checkoutRoot !== undefined
+        ? options.checkoutRoot
+        : await resolveAndValidateCheckout(task, options.checkoutOverride, options.cwd);
     const pipelinePath = typeof options.pipeline === "string"
         ? normalizeCatalogPath(loaded.pipelinePath)
         : undefined;
+    const persistence = pipelinePersistenceForStart(options.pipeline);
+    if (!persistence.ok) {
+        throw new InlinePipelineTooLargeError(persistence.bytes, persistence.maxBytes);
+    }
     const taskPath = options.taskPath
         ? normalizeCatalogPath(path.resolve(options.cwd, options.taskPath))
         : undefined;
     const projectRoot = normalizeCatalogPath(options.projectRoot ?? options.cwd);
-    const run = await options.store.createRun({
-        pipelineId: loaded.pipeline.id,
-        taskYaml,
-        taskId: task.id,
-        checkoutRoot,
-        gitSha: options.gitSha,
-        ciPrUrl: options.ciPrUrl,
-        ciJobUrl: options.ciJobUrl,
-        pipelineDag: buildPipelineDagSnapshotFromLoaded(loaded),
-        pipelinePath,
-        taskPath,
-        projectRoot,
-    });
+    const gitIdentity = resolveEffectiveGitIdentity(process.env, task.git_identity);
+    let run;
+    if (options.reuseExistingRun === true) {
+        const runId = options.runId;
+        if (runId === undefined || runId.trim() === "") {
+            throw new Error("reuseExistingRun requires a preallocated runId");
+        }
+        await options.store.patchRunWorkspaceBinding(runId, {
+            ...(checkoutRoot !== undefined ? { checkoutRoot } : {}),
+            ...(options.repository !== undefined
+                ? { repository: options.repository }
+                : {}),
+            ...(options.ref !== undefined ? { ref: options.ref } : {}),
+            ...(options.resolvedSha !== undefined
+                ? { resolvedSha: options.resolvedSha }
+                : {}),
+            ...(options.runBranch !== undefined
+                ? { runBranch: options.runBranch }
+                : {}),
+        });
+        const activated = await options.store.tryUpdateRunStatus(runId, "running", "queued");
+        if (!activated) {
+            let currentStatus;
+            try {
+                currentStatus = (await options.store.readRunMeta(runId)).status;
+            }
+            catch {
+                currentStatus = undefined;
+            }
+            throw new QueuedRunActivationAborted(runId, currentStatus);
+        }
+        run = { runId, workspaceDir: options.store.getWorkspaceDir(runId) };
+    }
+    else {
+        const runId = options.runId ?? newRunId();
+        const createdAt = new Date().toISOString();
+        const auth = getRequestAuth();
+        const surface = auth?.surface ?? "cli";
+        const callerId = options.callerId !== undefined && options.callerId !== null
+            ? options.callerId
+            : null;
+        const secretNames = loaded.stages.flatMap((s) => (s.secrets ?? []).map((d) => d.name));
+        const namedSecrets = collectDeclaredSecretValues(secretNames).filter((s) => shouldRegisterValue(s.value));
+        if (namedSecrets.length > 0) {
+            registerNamedSecrets(namedSecrets);
+        }
+        const runManifest = buildInitialRunManifest({
+            runId,
+            createdAt,
+            callerId,
+            surface,
+            binding: bindingFromFields({
+                repository: options.repository,
+                ref: options.ref,
+                resolvedSha: options.resolvedSha,
+                runBranch: options.runBranch,
+                checkoutRoot,
+            }),
+            pipelineSource: persistence.fields.pipelineSource,
+            pipelinePath,
+            pipelineBody: persistence.fields.pipelineBody ?? null,
+            taskYaml,
+            taskPath,
+            skills: options.skills,
+            stages: loaded.stages,
+            toolchain: preflight.toolchain.checks,
+            namedSecrets,
+        });
+        run = await options.store.createRun({
+            submission: options.submission,
+            runId,
+            pipelineId: loaded.pipeline.id,
+            taskYaml,
+            taskId: task.id,
+            checkoutRoot,
+            gitSha: options.gitSha,
+            ciPrUrl: options.ciPrUrl,
+            ciJobUrl: options.ciJobUrl,
+            pipelineDag: buildPipelineDagSnapshotFromLoaded(loaded),
+            pipelinePath,
+            taskPath,
+            projectRoot,
+            repository: options.repository,
+            ref: options.ref,
+            resolvedSha: options.resolvedSha,
+            runBranch: options.runBranch,
+            gitAuthorName: gitIdentity.name,
+            gitAuthorEmail: gitIdentity.email,
+            pipelineSource: persistence.fields.pipelineSource,
+            ...(persistence.fields.pipelineBody !== undefined
+                ? { pipelineBody: persistence.fields.pipelineBody }
+                : {}),
+            ...(callerId !== null ? { callerId } : {}),
+            runManifest,
+            skipGates: options.skipGates,
+        });
+    }
+    if (options.skills !== undefined && Object.keys(options.skills).length > 0) {
+        await materializeRunSkills(run.workspaceDir, options.skills);
+    }
     const executionMode = readStageExecutionMode(process.env, options.executionMode);
     const stageProcessLauncher = resolveStageProcessLauncher(executionMode, options.stageProcessLauncher);
     return {
@@ -110,6 +261,7 @@ export async function executeStages(prepared, options) {
         startAtStageIndex: options?.startAtStageIndex,
         executionMode,
         stageProcessLauncher,
+        schedulingHalt: options?.schedulingHalt,
     });
     if (prepared.findings !== undefined && prepared.findings.length > 0) {
         return { ...result, findings: prepared.findings };
@@ -145,6 +297,7 @@ export async function startPipeline(options) {
     const cwd = options.cwd ?? process.cwd();
     const projectRoot = options.projectRoot ?? cwd;
     const prepared = await preparePipeline({
+        submission: options.submission,
         agent: options.agent,
         store: options.store,
         taskPath: options.taskPath,
@@ -153,6 +306,13 @@ export async function startPipeline(options) {
         cwd,
         projectRoot,
         checkoutOverride: options.checkoutOverride,
+        runId: options.runId,
+        reuseExistingRun: options.reuseExistingRun,
+        checkoutRoot: options.checkoutRoot,
+        repository: options.repository,
+        ref: options.ref,
+        resolvedSha: options.resolvedSha,
+        runBranch: options.runBranch,
         gitSha: options.gitSha,
         ciPrUrl: options.ciPrUrl,
         ciJobUrl: options.ciJobUrl,
@@ -161,13 +321,17 @@ export async function startPipeline(options) {
         stageProcessLauncher: options.stageProcessLauncher,
         operatorCatalog: options.operatorCatalog,
         skipGates: options.skipGates,
+        callerId: options.callerId,
+        ...(options.skills !== undefined ? { skills: options.skills } : {}),
     });
     const done = executeStages(prepared, {
         maxActiveStagesPerRun: options.maxActiveStagesPerRun,
         executionMode: prepared.executionMode,
         stageProcessLauncher: prepared.stageProcessLauncher,
+        schedulingHalt: options.schedulingHalt,
     }).catch(async (err) => {
         await prepared.store.updateRunStatus(prepared.run.runId, "failed").catch(() => undefined);
+        await finaliseStoredRunManifest(prepared.store, prepared.run.runId).catch(() => undefined);
         return {
             ok: false,
             outcome: "failed",

package/dist/runtime/pipelineScheduler.d.ts

@@ -61,7 +61,13 @@ export type ResumeRunOptions = {
     initialPrior?: StageEnvelope | null;
     executionMode?: StageExecutionMode;
     stageProcessLauncher?: StageProcessLauncher;
+    schedulingHalt?: {
+        halted: boolean;
+        hostShutdown?: boolean;
+    };
 };
+/** Refuses to overwrite a cancelled run (KTD7). */
+export declare function writeTerminalRunStatus(store: RunStore, runId: string, status: "succeeded" | "failed"): Promise<void>;
 export declare function resumeRun(options: ResumeRunOptions): Promise<PipelineRunResult>;
 export type RunPipelineDagOptions = {
     prepared: SchedulerPreparedPipeline;
@@ -87,6 +93,11 @@ export type RunPipelineDagOptions = {
         sourceStageId: string;
         resolve: (decision: FeedbackLoopDecisionInput) => Promise<ResolveFeedbackLoopDecisionResult>;
     }) => Promise<void>;
+    /** Live cancel flag — when set, launchStage/startStage stop. */
+    schedulingHalt?: {
+        halted: boolean;
+        hostShutdown?: boolean;
+    };
 };
 export type RetryRunOptions = {
     prepared: SchedulerPreparedPipeline;
@@ -97,6 +108,10 @@ export type RetryRunOptions = {
     mutationQueue?: RetryMutationQueue;
     onLoopTick?: () => void | Promise<void>;
     onRetryRootTerminal?: OnRetryRootTerminal;
+    schedulingHalt?: {
+        halted: boolean;
+        hostShutdown?: boolean;
+    };
 };
 export declare function retryRun(options: RetryRunOptions): Promise<PipelineRunResult>;
 export declare function applyForkSkipsFromEnvelopes(dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, completedEnvelopes: Map<string, StageEnvelope>): void;

package/dist/runtime/pipelineScheduler.js

@@ -1,4 +1,5 @@
 import { normalizeForkChoice } from "../envelope/forkChoice.js";
+import { refreshRunDiskUsage } from "../runstore/diskUsage.js";
 import { buildPipelineDagSnapshotFromLoaded } from "../runstore/pipelineDagSnapshot.js";
 import { definitionIdForInstance } from "../runstore/stageInstanceId.js";
 import { cloneFailureContinuesSchedule, cloneRetryDownstream, cloneFailFastSkipIds, cloneScheduleAllowsRun, failureIsAccepted, isCloneInstance, mint, protectedClonableChildIds, sequentialLaterCloneIds, skipRejectedNeedDependents, } from "./cloneSchedule.js";
@@ -8,8 +9,18 @@ import { reopenRunnableSkippedStages, persistReopenedStages } from "./reopenSkip
 import { forkChoicePreservesReplaySource, forkCohortMapsFromStore, mintCohortForFanout, } from "./forkGeneration.js";
 import { activeReplayId, activeSourceStageId, applySendBack, createReplaySchedule, hydrateFromStore, isHeld, launchAttemptFor, launchFor, noteActiveCohort, onContinued, onRouteStageFailed, onRouteStageRunning, onRouteStageSucceeded, rebindRouteAttempts, resolveDecision, } from "./replayLifecycle.js";
 import { WAIT_WITHOUT_WORKER_DISPATCH } from "./answerResume.js";
+import { reclaimWorkspaceOnRunSucceeded } from "./repositoryMaterialize.js";
+import { syncRunStatusFromStages } from "./stageRecovery.js";
 import { runStage, isRunStageWaiting } from "./stageRunner.js";
 import { attemptContext, resumeSessionFilePath } from "./stageAttemptContext.js";
+import { stageBindingEnvFromRun } from "./stageRoots.js";
+import { attemptWorkspaceDir } from "../runstore/workspaceLayout.js";
+import { cleanupCredentialsDir, loadSecretRegistry, resolveStageSecrets, SecretUnavailableError, } from "./stageSecrets.js";
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { logger as rootLogger, } from "../logging/logger.js";
+import { registerNamedSecrets, REDACTION_SECRETS_FILENAME } from "../logging/namedSecrets.js";
+import { finaliseStoredRunManifest } from "../runstore/runManifest.js";
 function asDagSnapshot(dag) {
     const snapshot = dag;
     if (Array.isArray(snapshot.stage_ids))
@@ -35,6 +46,8 @@ export function snapshotToScheduleState(snap, overrides, stageId, executionMode)
             return "active";
         case "waiting_for_input":
             return executionMode === "process" ? "waiting" : "active";
+        case "interrupted":
+            return "waiting";
         case "skipped":
             return "skipped";
         default:
@@ -136,6 +149,18 @@ export async function hydrateScheduleForRetryRoots(store, runId, dag, retryStage
 export async function hydrateScheduleForRetry(store, runId, dag, retryStageId) {
     return hydrateScheduleForRetryRoots(store, runId, dag, [retryStageId]);
 }
+/** Refuses to overwrite a cancelled run (KTD7). */
+export async function writeTerminalRunStatus(store, runId, status) {
+    const meta = await store.readRunMeta(runId);
+    if (meta.status === "cancelled")
+        return;
+    await store.updateRunStatus(runId, status);
+    await finaliseStoredRunManifest(store, runId).catch(() => undefined);
+    await refreshRunDiskUsage(store, runId).catch(() => undefined);
+    if (status === "succeeded") {
+        await reclaimWorkspaceOnRunSucceeded(store, runId);
+    }
+}
 export async function resumeRun(options) {
     const { prepared, resumeFromStageId, initialPrior } = options;
     const meta = await prepared.store.readRunMeta(prepared.run.runId);
@@ -165,7 +190,7 @@ export async function resumeRun(options) {
     });
     if (allTerminal) {
         const unhandledFailure = scheduleHasUnhandledFailure(dag, hydrated.states);
-        await prepared.store.updateRunStatus(prepared.run.runId, unhandledFailure ? "failed" : "succeeded");
+        await writeTerminalRunStatus(prepared.store, prepared.run.runId, unhandledFailure ? "failed" : "succeeded");
         return {
             ok: !unhandledFailure,
             outcome: unhandledFailure ? "failed" : "succeeded",
@@ -191,6 +216,7 @@ export async function retryRun(options) {
         mutationQueue: options.mutationQueue,
         onLoopTick: options.onLoopTick,
         onRetryRootTerminal: options.onRetryRootTerminal,
+        schedulingHalt: options.schedulingHalt,
     });
 }
 function applyForkChoiceToSchedule(dag, forkStageId, chosen, states, options) {
@@ -220,6 +246,25 @@ export function applyForkSkipsFromEnvelopes(dag, states, completedEnvelopes) {
         applyForkChoiceToSchedule(dag, node.id, chosen, states);
     }
 }
+async function applyEagerRouteIfSkipsFromEnvelopes(dag, states, completedEnvelopes, persistSkip) {
+    for (const [parentId, envelope] of completedEnvelopes) {
+        if (states.get(parentId) !== "succeeded")
+            continue;
+        for (const childId of dag.childrenOf[parentId] ?? []) {
+            const child = dag.nodes.find((n) => n.id === childId);
+            if (!child)
+                continue;
+            const result = classifyInboundAfterSuccess(child, parentId, envelope.payload);
+            if (result === "missing_field") {
+                return parentId;
+            }
+            if (isEagerSingleParentIfSkip(child, parentId, envelope.payload)) {
+                await persistSkip(childId);
+            }
+        }
+    }
+    return undefined;
+}
 export function applyRetryRootDelta(dag, states, completedEnvelopes, retryRoots, stageId, attempt, clearSchedulingHalt, alreadyLaunchedIds) {
     retryRoots.set(stageId, attempt);
     const downstream = cloneRetryDownstream(dag, [stageId]);
@@ -330,10 +375,16 @@ export async function runPipelineDag(options) {
     let { states, completedEnvelopes, schedulingHalted, firstFailureReason, dag: hydratedDag, } = options.initialSchedule ??
         (await hydrateScheduleFromStore(store, run.runId, fallbackDag, executionMode));
     let dag = hydratedDag ?? fallbackDag;
+    const runMeta = await store.readRunMeta(run.runId);
     if (hydratedDag === undefined) {
-        const meta = await store.readRunMeta(run.runId);
-        dag = meta.pipeline_dag ?? fallbackDag;
+        dag = runMeta.pipeline_dag ?? fallbackDag;
     }
+    const stageBinding = stageBindingEnvFromRun({
+        meta: runMeta,
+        task,
+        runWorkspaceDir: run.workspaceDir,
+        hostEnv: process.env,
+    });
     const stageById = buildStageConfigById(loaded);
     const retryContext = options.retryContext;
     const feedbackSchedule = createReplaySchedule();
@@ -472,6 +523,13 @@ export async function runPipelineDag(options) {
             await persistSkipPending(id);
         }
     }
+    const missingRouteIfParent = await applyEagerRouteIfSkipsFromEnvelopes(dag, states, completedEnvelopes, persistSkipPending);
+    if (missingRouteIfParent !== undefined) {
+        schedulingHalted = true;
+        if (firstFailureReason === undefined) {
+            firstFailureReason = `stage "${missingRouteIfParent}": route if field missing from payload`;
+        }
+    }
     const markSkippedPending = () => {
         for (const node of dag.nodes) {
             if (states.get(node.id) === "pending") {
@@ -756,22 +814,13 @@ export async function runPipelineDag(options) {
                 await store.appendStageEvent(run.runId, id, { event: "skipped" });
             }
         }
-        const childIds = dag.childrenOf[stageId] ?? [];
-        for (const childId of childIds) {
-            const child = dag.nodes.find((node) => node.id === childId);
-            if (!child)
-                continue;
-            const result = classifyInboundAfterSuccess(child, stageId, envelope.payload);
-            if (result === "missing_field") {
-                schedulingHalted = true;
-                if (firstFailureReason === undefined) {
-                    firstFailureReason = `stage "${stageId}": route if field missing from payload`;
-                }
-                return;
-            }
-            if (isEagerSingleParentIfSkip(child, stageId, envelope.payload)) {
-                await persistSkipPending(childId);
+        const missingParent = await applyEagerRouteIfSkipsFromEnvelopes(dag, states, new Map([[stageId, envelope]]), persistSkipPending);
+        if (missingParent !== undefined) {
+            schedulingHalted = true;
+            if (firstFailureReason === undefined) {
+                firstFailureReason = `stage "${missingParent}": route if field missing from payload`;
             }
+            return;
         }
         await reopenRunnableSkippedStages(store, run.runId, dag, states, completedEnvelopes);
     };
@@ -810,32 +859,83 @@ export async function runPipelineDag(options) {
                 stageId,
             });
         }
+        const attemptDir = attemptWorkspaceDir(run.workspaceDir, stageId, attempt);
+        await mkdir(attemptDir, { recursive: true });
+        const attemptHome = path.join(attemptDir, "home");
+        await mkdir(attemptHome, { recursive: true });
+        let grants;
+        try {
+            const resolved = resolveStageSecrets({
+                decls: stage.secrets,
+                registry: loadSecretRegistry(process.env),
+                hostEnv: process.env,
+                attemptDir,
+            });
+            grants = resolved.grants;
+            registerNamedSecrets(resolved.knownValues);
+            await writeFile(path.join(attemptDir, REDACTION_SECRETS_FILENAME), `${JSON.stringify(resolved.knownValues)}\n`, { encoding: "utf8", mode: 0o600 });
+            for (const warning of resolved.warnings) {
+                rootLogger.warn("stage.secrets", warning, {
+                    run_id: run.runId,
+                    stage_id: stageId,
+                });
+            }
+            await writeFile(path.join(attemptDir, "declared-secrets.json"), `${JSON.stringify({ names: grants.declaredSecretNames }, null, 2)}\n`, "utf8");
+        }
+        catch (err) {
+            const reason = err instanceof SecretUnavailableError
+                ? err.message
+                : err instanceof Error
+                    ? err.message
+                    : String(err);
+            await onStageFailure(stageId, reason);
+            return;
+        }
+        const cleanupAttemptCredentials = () => {
+            try {
+                cleanupCredentialsDir(attemptDir);
+            }
+            catch {
+                // best-effort Host cleanup
+            }
+        };
         if (executionMode === "process") {
             const launcher = options.stageProcessLauncher;
             if (!launcher) {
+                cleanupAttemptCredentials();
                 await onStageFailure(stageId, "stage process launcher is not configured");
                 return;
             }
-            const launchResult = await launcher.launch({
-                runId: run.runId,
-                stageId,
-                rootDir: factoryCwd,
-                attempt,
-                ...(sessionMode !== undefined
-                    ? {
-                        mode: sessionMode === "feedback_resume"
-                            ? "feedback_resume"
-                            : sessionMode === "new_session"
-                                ? "new_session"
-                                : "run",
-                    }
-                    : {}),
-                ...(resumeToken !== undefined ? { sessionFilePath: resumeToken } : {}),
-                ...(prepared.operatorCatalog !== undefined
-                    ? { operatorCatalog: prepared.operatorCatalog }
-                    : {}),
-                ...(prepared.skipGates ? { skipGates: true } : {}),
-            });
+            let launchResult;
+            try {
+                launchResult = await launcher.launch({
+                    runId: run.runId,
+                    stageId,
+                    rootDir: factoryCwd,
+                    attempt,
+                    env: stageBinding.env,
+                    bindingKind: stageBinding.kind,
+                    grants,
+                    attemptHome,
+                    ...(sessionMode !== undefined
+                        ? {
+                            mode: sessionMode === "feedback_resume"
+                                ? "feedback_resume"
+                                : sessionMode === "new_session"
+                                    ? "new_session"
+                                    : "run",
+                        }
+                        : {}),
+                    ...(resumeToken !== undefined ? { sessionFilePath: resumeToken } : {}),
+                    ...(prepared.operatorCatalog !== undefined
+                        ? { operatorCatalog: prepared.operatorCatalog }
+                        : {}),
+                    ...(prepared.skipGates ? { skipGates: true } : {}),
+                });
+            }
+            finally {
+                cleanupAttemptCredentials();
+            }
             if (launchResult.type === "succeeded") {
                 try {
                     const envelope = await store.readEnvelope(run.runId, stageId);
@@ -851,6 +951,10 @@ export async function runPipelineDag(options) {
                 states.set(stageId, "waiting");
                 return;
             }
+            if (options.schedulingHalt?.hostShutdown) {
+                states.set(stageId, "waiting");
+                return;
+            }
             await onStageFailure(stageId, launchResult.reason);
             return;
         }
@@ -870,15 +974,25 @@ export async function runPipelineDag(options) {
             operatorCatalog: prepared.operatorCatalog,
             completedEnvelopes,
             skipGates: prepared.skipGates,
+            stageEnv: {
+                ...stageBinding.env,
+                ...grants.env,
+                HOME: attemptHome,
+            },
             ...(sessionMode !== undefined ? { sessionMode } : {}),
             ...(feedbackLoopContext !== undefined ? { feedbackLoopContext } : {}),
             ...(resumeToken !== undefined ? { resumeToken } : {}),
         });
+        cleanupAttemptCredentials();
         if (isRunStageWaiting(result)) {
             await onStageFailure(stageId, WAIT_WITHOUT_WORKER_DISPATCH);
             return;
         }
         if (!result.ok) {
+            if (options.schedulingHalt?.hostShutdown) {
+                states.set(stageId, "waiting");
+                return;
+            }
             await onStageFailure(stageId, result.reason ?? "stage failed", result.envelope);
             return;
         }
@@ -927,11 +1041,17 @@ export async function runPipelineDag(options) {
         return true;
     };
     while (!allTerminal()) {
+        if (options.schedulingHalt?.halted) {
+            schedulingHalted = true;
+        }
         drainRetryMutations();
         await applyStalledJoinSkips();
         if (options.onLoopTick !== undefined) {
             await options.onLoopTick();
         }
+        if (options.schedulingHalt?.halted) {
+            schedulingHalted = true;
+        }
         if (!schedulingHalted) {
             const runnable = pickRunnable();
             for (const stageId of runnable) {
@@ -975,11 +1095,16 @@ export async function runPipelineDag(options) {
     if (schedulingHalted) {
         markSkippedPending();
         if (retryContext === undefined) {
-            await store.updateRunStatus(run.runId, "failed");
+            if (options.schedulingHalt?.hostShutdown) {
+                await syncRunStatusFromStages(store, run.runId);
+            }
+            else {
+                await writeTerminalRunStatus(store, run.runId, "failed");
+            }
         }
         return {
             ok: false,
-            outcome: "failed",
+            outcome: options.schedulingHalt?.hostShutdown ? "waiting" : "failed",
             runDir: run.workspaceDir,
             runId: run.runId,
             reason: firstFailureReason,
@@ -987,7 +1112,7 @@ export async function runPipelineDag(options) {
     }
     if (scheduleHasUnhandledFailure(dag, states) || !allTerminal()) {
         if (retryContext === undefined) {
-            await store.updateRunStatus(run.runId, "failed");
+            await writeTerminalRunStatus(store, run.runId, "failed");
         }
         return {
             ok: false,
@@ -997,7 +1122,7 @@ export async function runPipelineDag(options) {
             reason: firstFailureReason ?? "pipeline incomplete",
         };
     }
-    await store.updateRunStatus(run.runId, "succeeded");
+    await writeTerminalRunStatus(store, run.runId, "succeeded");
     return {
         ok: true,
         outcome: "succeeded",

package/dist/runtime/reloadRunCatalog.js

@@ -1,5 +1,5 @@
 import { access } from "node:fs/promises";
-import { loadPipeline } from "../config/loadPipeline.js";
+import { loadPipeline, loadPipelineFromObject } from "../config/loadPipeline.js";
 import { loadTask } from "../config/loadTask.js";
 import { normalizeCatalogPath } from "../runstore/normalizeCatalogPath.js";
 function missingLocator(field, runId) {
@@ -32,6 +32,17 @@ async function assertCatalogFileExists(filePath, label) {
     }
 }
 export async function reloadPipelineForRun(meta) {
+    // An inline-pipeline run (no pipeline_path — nothing was ever written to a
+    // catalog file) carries its own pipeline body in meta.inline_pipeline
+    // instead, the same way a task without task_path reloads from task_yaml.
+    // Prefer this before requiring path locators; project_root is optional and
+    // falls back to cwd (same as loadPipelineFromObject).
+    if (meta.inline_pipeline) {
+        const projectRoot = meta.project_root
+            ? normalizeCatalogPath(meta.project_root)
+            : process.cwd();
+        return loadPipelineFromObject(meta.inline_pipeline, { projectRoot });
+    }
     const projectRoot = requireProjectRoot(meta);
     const pipelinePath = requirePipelinePath(meta);
     await assertCatalogFileExists(pipelinePath, "Pipeline");