stageflow 0.25.0 → 0.30.0

package/dist/runtime/pipelineScheduler.d.ts

@@ -1,5 +1,6 @@
 import type { AgentPort } from "../agent/port.js";
 import type { RunPipelineDagSnapshot, RunStore, StageSnapshot } from "../runstore/port.js";
+import type { StageBrowserSupport } from "../browser/browserHost.js";
 import type { StageEnvelope } from "../types/envelope.js";
 import type { LoadedPipeline, ResolvedPipelineDag } from "../types/pipeline.js";
 import type { TaskFile } from "../types/task.js";
@@ -24,6 +25,7 @@ type SchedulerPreparedPipeline = {
     hitl?: StageHitlController;
     operatorCatalog?: OperatorCatalog;
     skipGates?: boolean;
+    browser?: StageBrowserSupport;
 };
 export type { SchedulerPreparedPipeline };
 type RetryContext = {
@@ -61,7 +63,13 @@ export type ResumeRunOptions = {
     initialPrior?: StageEnvelope | null;
     executionMode?: StageExecutionMode;
     stageProcessLauncher?: StageProcessLauncher;
+    schedulingHalt?: {
+        halted: boolean;
+        hostShutdown?: boolean;
+    };
 };
+/** Refuses to overwrite a cancelled run (KTD7). */
+export declare function writeTerminalRunStatus(store: RunStore, runId: string, status: "succeeded" | "failed"): Promise<void>;
 export declare function resumeRun(options: ResumeRunOptions): Promise<PipelineRunResult>;
 export type RunPipelineDagOptions = {
     prepared: SchedulerPreparedPipeline;
@@ -87,6 +95,11 @@ export type RunPipelineDagOptions = {
         sourceStageId: string;
         resolve: (decision: FeedbackLoopDecisionInput) => Promise<ResolveFeedbackLoopDecisionResult>;
     }) => Promise<void>;
+    /** Live cancel flag — when set, launchStage/startStage stop. */
+    schedulingHalt?: {
+        halted: boolean;
+        hostShutdown?: boolean;
+    };
 };
 export type RetryRunOptions = {
     prepared: SchedulerPreparedPipeline;
@@ -97,6 +110,10 @@ export type RetryRunOptions = {
     mutationQueue?: RetryMutationQueue;
     onLoopTick?: () => void | Promise<void>;
     onRetryRootTerminal?: OnRetryRootTerminal;
+    schedulingHalt?: {
+        halted: boolean;
+        hostShutdown?: boolean;
+    };
 };
 export declare function retryRun(options: RetryRunOptions): Promise<PipelineRunResult>;
 export declare function applyForkSkipsFromEnvelopes(dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, completedEnvelopes: Map<string, StageEnvelope>): void;

package/dist/runtime/pipelineScheduler.js

@@ -1,5 +1,11 @@
 import { normalizeForkChoice } from "../envelope/forkChoice.js";
+import { refreshRunDiskUsage } from "../runstore/diskUsage.js";
 import { buildPipelineDagSnapshotFromLoaded } from "../runstore/pipelineDagSnapshot.js";
+import { defaultStageBrowserSupport, resolveStageBrowserEnv, } from "../browser/stageBrowserEnv.js";
+import { teardownRunBrowsers, teardownStageBrowser, } from "../browser/browserTeardown.js";
+import { isHumanLoginStage } from "../browser/humanLogin.js";
+import { createRunLiveness } from "../browser/runLiveness.js";
+import { acquireStageProfile, profileWaitingMessage, } from "../browser/stageProfileLock.js";
 import { definitionIdForInstance } from "../runstore/stageInstanceId.js";
 import { cloneFailureContinuesSchedule, cloneRetryDownstream, cloneFailFastSkipIds, cloneScheduleAllowsRun, failureIsAccepted, isCloneInstance, mint, protectedClonableChildIds, sequentialLaterCloneIds, skipRejectedNeedDependents, } from "./cloneSchedule.js";
 import { buildCompletedEnvelopesFromRun, buildStageConfigById, } from "./envelopeRouting.js";
@@ -8,8 +14,18 @@ import { reopenRunnableSkippedStages, persistReopenedStages } from "./reopenSkip
 import { forkChoicePreservesReplaySource, forkCohortMapsFromStore, mintCohortForFanout, } from "./forkGeneration.js";
 import { activeReplayId, activeSourceStageId, applySendBack, createReplaySchedule, hydrateFromStore, isHeld, launchAttemptFor, launchFor, noteActiveCohort, onContinued, onRouteStageFailed, onRouteStageRunning, onRouteStageSucceeded, rebindRouteAttempts, resolveDecision, } from "./replayLifecycle.js";
 import { WAIT_WITHOUT_WORKER_DISPATCH } from "./answerResume.js";
+import { reclaimWorkspaceOnRunSucceeded } from "./repositoryMaterialize.js";
+import { syncRunStatusFromStages } from "./stageRecovery.js";
 import { runStage, isRunStageWaiting } from "./stageRunner.js";
 import { attemptContext, resumeSessionFilePath } from "./stageAttemptContext.js";
+import { stageBindingEnvFromRun } from "./stageRoots.js";
+import { attemptWorkspaceDir } from "../runstore/workspaceLayout.js";
+import { cleanupCredentialsDir, loadSecretRegistry, resolveStageSecrets, SecretUnavailableError, } from "./stageSecrets.js";
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { logger as rootLogger, } from "../logging/logger.js";
+import { registerNamedSecrets, REDACTION_SECRETS_FILENAME } from "../logging/namedSecrets.js";
+import { finaliseStoredRunManifest } from "../runstore/runManifest.js";
 function asDagSnapshot(dag) {
     const snapshot = dag;
     if (Array.isArray(snapshot.stage_ids))
@@ -35,6 +51,8 @@ export function snapshotToScheduleState(snap, overrides, stageId, executionMode)
             return "active";
         case "waiting_for_input":
             return executionMode === "process" ? "waiting" : "active";
+        case "interrupted":
+            return "waiting";
         case "skipped":
             return "skipped";
         default:
@@ -136,6 +154,18 @@ export async function hydrateScheduleForRetryRoots(store, runId, dag, retryStage
 export async function hydrateScheduleForRetry(store, runId, dag, retryStageId) {
     return hydrateScheduleForRetryRoots(store, runId, dag, [retryStageId]);
 }
+/** Refuses to overwrite a cancelled run (KTD7). */
+export async function writeTerminalRunStatus(store, runId, status) {
+    const meta = await store.readRunMeta(runId);
+    if (meta.status === "cancelled")
+        return;
+    await store.updateRunStatus(runId, status);
+    await finaliseStoredRunManifest(store, runId).catch(() => undefined);
+    await refreshRunDiskUsage(store, runId).catch(() => undefined);
+    if (status === "succeeded") {
+        await reclaimWorkspaceOnRunSucceeded(store, runId);
+    }
+}
 export async function resumeRun(options) {
     const { prepared, resumeFromStageId, initialPrior } = options;
     const meta = await prepared.store.readRunMeta(prepared.run.runId);
@@ -165,7 +195,7 @@ export async function resumeRun(options) {
     });
     if (allTerminal) {
         const unhandledFailure = scheduleHasUnhandledFailure(dag, hydrated.states);
-        await prepared.store.updateRunStatus(prepared.run.runId, unhandledFailure ? "failed" : "succeeded");
+        await writeTerminalRunStatus(prepared.store, prepared.run.runId, unhandledFailure ? "failed" : "succeeded");
         return {
             ok: !unhandledFailure,
             outcome: unhandledFailure ? "failed" : "succeeded",
@@ -191,6 +221,7 @@ export async function retryRun(options) {
         mutationQueue: options.mutationQueue,
         onLoopTick: options.onLoopTick,
         onRetryRootTerminal: options.onRetryRootTerminal,
+        schedulingHalt: options.schedulingHalt,
     });
 }
 function applyForkChoiceToSchedule(dag, forkStageId, chosen, states, options) {
@@ -349,11 +380,18 @@ export async function runPipelineDag(options) {
     let { states, completedEnvelopes, schedulingHalted, firstFailureReason, dag: hydratedDag, } = options.initialSchedule ??
         (await hydrateScheduleFromStore(store, run.runId, fallbackDag, executionMode));
     let dag = hydratedDag ?? fallbackDag;
+    const runMeta = await store.readRunMeta(run.runId);
     if (hydratedDag === undefined) {
-        const meta = await store.readRunMeta(run.runId);
-        dag = meta.pipeline_dag ?? fallbackDag;
+        dag = runMeta.pipeline_dag ?? fallbackDag;
     }
+    const stageBinding = stageBindingEnvFromRun({
+        meta: runMeta,
+        task,
+        runWorkspaceDir: run.workspaceDir,
+        hostEnv: process.env,
+    });
     const stageById = buildStageConfigById(loaded);
+    const runUsesBrowser = [...stageById.values()].some((s) => s.browser !== undefined);
     const retryContext = options.retryContext;
     const feedbackSchedule = createReplaySchedule();
     await hydrateFromStore(feedbackSchedule, store, run.runId, dag);
@@ -810,6 +848,40 @@ export async function runPipelineDag(options) {
             return;
         }
         const attempt = await resolveLaunchAttempt(stageId);
+        if (stage.browser?.profile !== undefined) {
+            const profile = stage.browser.profile;
+            // A stage queued behind another run's lease must not hold one of this
+            // run's active slots; it takes the slot back once it joins the lease.
+            let slotReleased = false;
+            let acquired;
+            try {
+                acquired = await acquireStageProfile(prepared.browser ?? defaultStageBrowserSupport(), {
+                    profile,
+                    owner: { runId: run.runId, stageId },
+                    isRunLive: createRunLiveness(store),
+                    halted: () => options.schedulingHalt?.halted === true,
+                    onWaiting: async (holder) => {
+                        if (!slotReleased) {
+                            slotReleased = true;
+                            activeCount -= 1;
+                        }
+                        await store.appendStageEvent(run.runId, stageId, {
+                            event: "message",
+                            role: "host",
+                            text: profileWaitingMessage(profile, holder),
+                        }, { attempt });
+                    },
+                });
+            }
+            finally {
+                if (slotReleased)
+                    activeCount += 1;
+            }
+            if (acquired === "halted") {
+                states.set(stageId, "skipped");
+                return;
+            }
+        }
         const attemptCtx = attemptContext(attempt);
         const prep = launchFor(feedbackSchedule, stageId);
         const feedbackLoopContext = prep?.feedbackLoopContext;
@@ -827,32 +899,100 @@ export async function runPipelineDag(options) {
                 stageId,
             });
         }
+        const attemptDir = attemptWorkspaceDir(run.workspaceDir, stageId, attempt);
+        await mkdir(attemptDir, { recursive: true });
+        const attemptHome = path.join(attemptDir, "home");
+        await mkdir(attemptHome, { recursive: true });
+        let grants;
+        try {
+            const resolved = resolveStageSecrets({
+                decls: stage.secrets,
+                registry: loadSecretRegistry(process.env),
+                hostEnv: process.env,
+                attemptDir,
+            });
+            grants = resolved.grants;
+            registerNamedSecrets(resolved.knownValues);
+            await writeFile(path.join(attemptDir, REDACTION_SECRETS_FILENAME), `${JSON.stringify(resolved.knownValues)}\n`, { encoding: "utf8", mode: 0o600 });
+            for (const warning of resolved.warnings) {
+                rootLogger.warn("stage.secrets", warning, {
+                    run_id: run.runId,
+                    stage_id: stageId,
+                });
+            }
+            await writeFile(path.join(attemptDir, "declared-secrets.json"), `${JSON.stringify({ names: grants.declaredSecretNames }, null, 2)}\n`, "utf8");
+        }
+        catch (err) {
+            const reason = err instanceof SecretUnavailableError
+                ? err.message
+                : err instanceof Error
+                    ? err.message
+                    : String(err);
+            await onStageFailure(stageId, reason);
+            return;
+        }
+        const cleanupAttemptCredentials = () => {
+            try {
+                cleanupCredentialsDir(attemptDir);
+            }
+            catch {
+                // best-effort Host cleanup
+            }
+        };
+        let browserEnv;
+        try {
+            browserEnv = await resolveStageBrowserEnv(prepared.browser ?? defaultStageBrowserSupport(), {
+                runId: run.runId,
+                stageId,
+                runDir: run.workspaceDir,
+                browser: stage.browser,
+                attempt,
+                humanLogin: isHumanLoginStage(dag, definitionId, stage.browser),
+            });
+        }
+        catch (err) {
+            cleanupAttemptCredentials();
+            await onStageFailure(stageId, err instanceof Error ? err.message : String(err));
+            return;
+        }
         if (executionMode === "process") {
             const launcher = options.stageProcessLauncher;
             if (!launcher) {
+                cleanupAttemptCredentials();
                 await onStageFailure(stageId, "stage process launcher is not configured");
                 return;
             }
-            const launchResult = await launcher.launch({
-                runId: run.runId,
-                stageId,
-                rootDir: factoryCwd,
-                attempt,
-                ...(sessionMode !== undefined
-                    ? {
-                        mode: sessionMode === "feedback_resume"
-                            ? "feedback_resume"
-                            : sessionMode === "new_session"
-                                ? "new_session"
-                                : "run",
-                    }
-                    : {}),
-                ...(resumeToken !== undefined ? { sessionFilePath: resumeToken } : {}),
-                ...(prepared.operatorCatalog !== undefined
-                    ? { operatorCatalog: prepared.operatorCatalog }
-                    : {}),
-                ...(prepared.skipGates ? { skipGates: true } : {}),
-            });
+            let launchResult;
+            try {
+                launchResult = await launcher.launch({
+                    runId: run.runId,
+                    stageId,
+                    rootDir: factoryCwd,
+                    attempt,
+                    env: stageBinding.env,
+                    bindingKind: stageBinding.kind,
+                    grants,
+                    attemptHome,
+                    ...(browserEnv !== undefined ? { browserEnv } : {}),
+                    ...(sessionMode !== undefined
+                        ? {
+                            mode: sessionMode === "feedback_resume"
+                                ? "feedback_resume"
+                                : sessionMode === "new_session"
+                                    ? "new_session"
+                                    : "run",
+                        }
+                        : {}),
+                    ...(resumeToken !== undefined ? { sessionFilePath: resumeToken } : {}),
+                    ...(prepared.operatorCatalog !== undefined
+                        ? { operatorCatalog: prepared.operatorCatalog }
+                        : {}),
+                    ...(prepared.skipGates ? { skipGates: true } : {}),
+                });
+            }
+            finally {
+                cleanupAttemptCredentials();
+            }
             if (launchResult.type === "succeeded") {
                 try {
                     const envelope = await store.readEnvelope(run.runId, stageId);
@@ -868,6 +1008,14 @@ export async function runPipelineDag(options) {
                 states.set(stageId, "waiting");
                 return;
             }
+            if (options.schedulingHalt?.hostShutdown) {
+                states.set(stageId, "waiting");
+                return;
+            }
+            const recorded = await store.listStageEvents(run.runId, stageId, attempt);
+            if (!recorded.some((event) => event.event === "failed")) {
+                await store.appendStageEvent(run.runId, stageId, { event: "failed", reason: launchResult.reason }, { attempt });
+            }
             await onStageFailure(stageId, launchResult.reason);
             return;
         }
@@ -887,15 +1035,27 @@ export async function runPipelineDag(options) {
             operatorCatalog: prepared.operatorCatalog,
             completedEnvelopes,
             skipGates: prepared.skipGates,
+            ...(prepared.browser !== undefined ? { browser: prepared.browser } : {}),
+            stageEnv: {
+                ...stageBinding.env,
+                ...grants.env,
+                ...browserEnv,
+                HOME: attemptHome,
+            },
             ...(sessionMode !== undefined ? { sessionMode } : {}),
             ...(feedbackLoopContext !== undefined ? { feedbackLoopContext } : {}),
             ...(resumeToken !== undefined ? { resumeToken } : {}),
         });
+        cleanupAttemptCredentials();
         if (isRunStageWaiting(result)) {
             await onStageFailure(stageId, WAIT_WITHOUT_WORKER_DISPATCH);
             return;
         }
         if (!result.ok) {
+            if (options.schedulingHalt?.hostShutdown) {
+                states.set(stageId, "waiting");
+                return;
+            }
             await onStageFailure(stageId, result.reason ?? "stage failed", result.envelope);
             return;
         }
@@ -919,7 +1079,19 @@ export async function runPipelineDag(options) {
         states.set(stageId, "active");
         launchedIds.add(stageId);
         activeCount += 1;
-        const taskPromise = launchStage(stageId).finally(() => {
+        const taskPromise = launchStage(stageId)
+            .then(async () => {
+            const state = states.get(stageId);
+            if (runUsesBrowser && (state === "succeeded" || state === "failed")) {
+                await teardownStageBrowser(prepared.browser ?? defaultStageBrowserSupport(), {
+                    runId: run.runId,
+                    runDir: run.workspaceDir,
+                    stageId,
+                    events: () => store.listStageEvents(run.runId, stageId),
+                }).catch(() => undefined);
+            }
+        })
+            .finally(() => {
             activeCount -= 1;
             inFlight.delete(taskPromise);
         });
@@ -944,11 +1116,17 @@ export async function runPipelineDag(options) {
         return true;
     };
     while (!allTerminal()) {
+        if (options.schedulingHalt?.halted) {
+            schedulingHalted = true;
+        }
         drainRetryMutations();
         await applyStalledJoinSkips();
         if (options.onLoopTick !== undefined) {
             await options.onLoopTick();
         }
+        if (options.schedulingHalt?.halted) {
+            schedulingHalted = true;
+        }
         if (!schedulingHalted) {
             const runnable = pickRunnable();
             for (const stageId of runnable) {
@@ -981,6 +1159,15 @@ export async function runPipelineDag(options) {
         await Promise.all(inFlight);
     }
     const hasWaiting = [...states.values()].some((s) => s === "waiting");
+    if (runUsesBrowser &&
+        !(hasWaiting && !schedulingHalted) &&
+        !options.schedulingHalt?.hostShutdown) {
+        await teardownRunBrowsers(prepared.browser ?? defaultStageBrowserSupport(), {
+            runId: run.runId,
+            runDir: run.workspaceDir,
+            events: (stageId) => store.listStageEvents(run.runId, stageId),
+        });
+    }
     if (hasWaiting && !schedulingHalted) {
         return {
             ok: false,
@@ -992,11 +1179,16 @@ export async function runPipelineDag(options) {
     if (schedulingHalted) {
         markSkippedPending();
         if (retryContext === undefined) {
-            await store.updateRunStatus(run.runId, "failed");
+            if (options.schedulingHalt?.hostShutdown) {
+                await syncRunStatusFromStages(store, run.runId);
+            }
+            else {
+                await writeTerminalRunStatus(store, run.runId, "failed");
+            }
         }
         return {
             ok: false,
-            outcome: "failed",
+            outcome: options.schedulingHalt?.hostShutdown ? "waiting" : "failed",
             runDir: run.workspaceDir,
             runId: run.runId,
             reason: firstFailureReason,
@@ -1004,7 +1196,7 @@ export async function runPipelineDag(options) {
     }
     if (scheduleHasUnhandledFailure(dag, states) || !allTerminal()) {
         if (retryContext === undefined) {
-            await store.updateRunStatus(run.runId, "failed");
+            await writeTerminalRunStatus(store, run.runId, "failed");
         }
         return {
             ok: false,
@@ -1014,7 +1206,7 @@ export async function runPipelineDag(options) {
             reason: firstFailureReason ?? "pipeline incomplete",
         };
     }
-    await store.updateRunStatus(run.runId, "succeeded");
+    await writeTerminalRunStatus(store, run.runId, "succeeded");
     return {
         ok: true,
         outcome: "succeeded",

package/dist/runtime/reloadRunCatalog.js

@@ -32,13 +32,18 @@ async function assertCatalogFileExists(filePath, label) {
     }
 }
 export async function reloadPipelineForRun(meta) {
-    const projectRoot = requireProjectRoot(meta);
     // An inline-pipeline run (no pipeline_path — nothing was ever written to a
     // catalog file) carries its own pipeline body in meta.inline_pipeline
     // instead, the same way a task without task_path reloads from task_yaml.
+    // Prefer this before requiring path locators; project_root is optional and
+    // falls back to cwd (same as loadPipelineFromObject).
     if (meta.inline_pipeline) {
+        const projectRoot = meta.project_root
+            ? normalizeCatalogPath(meta.project_root)
+            : process.cwd();
         return loadPipelineFromObject(meta.inline_pipeline, { projectRoot });
     }
+    const projectRoot = requireProjectRoot(meta);
     const pipelinePath = requirePipelinePath(meta);
     await assertCatalogFileExists(pipelinePath, "Pipeline");
     return loadPipeline(pipelinePath, { cwd: projectRoot });

package/dist/runtime/repositoryMaterialize.d.ts

@@ -0,0 +1,47 @@
+import type { RunMeta, RunStore } from "../runstore/port.js";
+import type { TaskFile } from "../types/task.js";
+import type { WorkspaceBinding } from "./workspaceBinding.js";
+export type StartFailureCode = "busy_capacity" | "busy_checkout" | "busy_caller_quota" | "repository_auth_failed" | "repository_not_found" | "ref_not_found" | "repository_fetch_failed" | "worktree_create_failed" | "git_missing" | "insufficient_disk" | "disk_check_failed" | "pinned_sha_unavailable" | "invalid_run_branch" | "task.binding_conflict" | "task.repository_ref_required" | "task.ref_without_repository" | "task.repository_invalid" | "task.invalid_shape" | "task.load_error" | "trigger.task_override_not_allowed" | "trigger.task_required" | "start.token_rejected" | "inline_pipeline_too_large" | "skills_invalid_name" | "skills_invalid_path" | "skills_missing_skill_md" | "skills_payload_too_large" | "missing_tool" | "tool_version_mismatch" | "unknown_version" | "secret_unavailable" | "unresolved_var" | "shutting_down" | "autostart_disabled" | "ensure_project_not_allowed" | "claude_as_root";
+export type MaterializedBinding = {
+    runId: string;
+    checkoutRoot?: string;
+    repository?: string;
+    ref?: string;
+    resolvedSha?: string;
+    runBranch?: string;
+};
+export type DerivedBindingKind = "repository" | "checkout" | "unbound";
+export declare function derivedBindingKindFromMeta(meta: RunMeta): DerivedBindingKind;
+export declare function worktreePathForRun(runId: string): string;
+export declare function resolveRunBranchTemplate(task: TaskFile): string;
+export declare class StartLinkError extends Error {
+    readonly code: StartFailureCode;
+    readonly status: number;
+    readonly stderr?: string;
+    constructor(message: string, options: {
+        code: StartFailureCode;
+        status: number;
+        stderr?: string;
+    });
+}
+export declare function throwMappedGitError(err: unknown, phase: "fetch" | "resolve" | "worktree"): never;
+export type MaterializeOptions = {
+    runId: string;
+    task: TaskFile;
+    binding: WorkspaceBinding;
+    checkoutRoot?: string;
+    pinned?: {
+        ref: string;
+        resolvedSha: string;
+    };
+};
+export type ReclaimWorkspaceBindingOptions = {
+    /** When true, remove the worktree but leave `run_branch` (SLIM must keep the branch). */
+    keepRunBranch?: boolean;
+};
+export declare function reclaimWorkspaceBinding(meta: RunMeta, options?: ReclaimWorkspaceBindingOptions): Promise<void>;
+export declare function reclaimWorkspaceOnRunSucceeded(store: RunStore, runId: string): Promise<void>;
+export declare function materializeWorkspaceBinding(options: MaterializeOptions): Promise<{
+    materialized: MaterializedBinding;
+    rollback: () => Promise<void>;
+}>;