stageflow 0.1.0

package/dist/runtime/pipelineRunner.d.ts

@@ -0,0 +1,77 @@
+import type { AgentPort } from "../agent/port.js";
+import type { RunStore } from "../runstore/port.js";
+import type { StageEnvelope } from "../types/envelope.js";
+import type { StageHitlController } from "./stageHitl.js";
+import type { LoadedPipeline } from "../types/pipeline.js";
+import type { TaskFile } from "../types/task.js";
+import { type StageExecutionMode } from "./stageConcurrency.js";
+import { StageProcessLauncher } from "./stageProcessLauncher.js";
+import type { OperatorCatalog } from "./stageAttemptBootstrap.js";
+export { PipelineValidationError } from "./pipelineValidationError.js";
+export type PipelineRunResult = {
+    ok: boolean;
+    runDir: string;
+    runId: string;
+    reason?: string;
+};
+export type StartedPipeline = {
+    runId: string;
+    runDir: string;
+    done: Promise<PipelineRunResult>;
+};
+export type PreparedPipeline = {
+    task: TaskFile;
+    loaded: LoadedPipeline;
+    run: {
+        runId: string;
+        workspaceDir: string;
+    };
+    agent: AgentPort;
+    store: RunStore;
+    cwd: string;
+    checkoutRoot?: string;
+    hitl?: StageHitlController;
+    executionMode?: StageExecutionMode;
+    stageProcessLauncher?: StageProcessLauncher;
+    operatorCatalog?: OperatorCatalog;
+};
+export type ExecuteStagesOptions = {
+    maxActiveStagesPerRun?: number;
+    resumeFromStageId?: string;
+    initialPrior?: StageEnvelope | null;
+    /** @deprecated Linear pipelines only — use resumeFromStageId for DAG resume. */
+    startAtStageIndex?: number;
+    executionMode?: StageExecutionMode;
+    stageProcessLauncher?: StageProcessLauncher;
+};
+export declare function executeStages(prepared: PreparedPipeline, options?: ExecuteStagesOptions): Promise<PipelineRunResult>;
+export declare function runPipeline(options: {
+    agent: AgentPort;
+    store: RunStore;
+    taskPath?: string;
+    taskYaml?: string;
+    pipeline: string;
+    cwd?: string;
+    checkoutOverride?: string;
+    hitl?: StageHitlController;
+    maxActiveStagesPerRun?: number;
+    executionMode?: StageExecutionMode;
+    stageProcessLauncher?: StageProcessLauncher;
+    operatorCatalog?: OperatorCatalog;
+}): Promise<PipelineRunResult>;
+/** Create the run immediately, then execute stages in the returned promise. */
+export declare function startPipeline(options: {
+    agent: AgentPort;
+    store: RunStore;
+    taskPath?: string;
+    taskYaml?: string;
+    pipeline: string;
+    cwd?: string;
+    checkoutOverride?: string;
+    hitl?: StageHitlController;
+    maxActiveStagesPerRun?: number;
+    executionMode?: StageExecutionMode;
+    stageProcessLauncher?: StageProcessLauncher;
+    operatorCatalog?: OperatorCatalog;
+}): Promise<StartedPipeline>;
+export declare function resolveTaskPath(taskArg: string, cwd?: string): string;

package/dist/runtime/pipelineRunner.js

@@ -0,0 +1,141 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { loadPipelineValidated } from "../config/loadPipeline.js";
+import { loadTaskFromYaml } from "../config/loadTask.js";
+import { buildValidationResult } from "../config/validateCatalog.js";
+import { resolveAndValidateCheckout } from "./stageRoots.js";
+import { buildPipelineDagSnapshotFromLoaded } from "../runstore/pipelineDagSnapshot.js";
+import { runPipelineDag } from "./pipelineScheduler.js";
+import { readMaxActiveStagesPerRun, readStageExecutionMode, } from "./stageConcurrency.js";
+import { StageProcessLauncher } from "./stageProcessLauncher.js";
+import { PipelineValidationError } from "./pipelineValidationError.js";
+export { PipelineValidationError } from "./pipelineValidationError.js";
+let defaultStageProcessLauncher;
+function resolveStageProcessLauncher(executionMode, override) {
+    if (executionMode !== "process") {
+        return undefined;
+    }
+    if (override !== undefined) {
+        return override;
+    }
+    if (defaultStageProcessLauncher === undefined) {
+        defaultStageProcessLauncher = new StageProcessLauncher();
+    }
+    return defaultStageProcessLauncher;
+}
+async function preparePipeline(options) {
+    const loadResult = await loadPipelineValidated(options.pipeline, {
+        cwd: options.cwd,
+        validateStages: true,
+    });
+    if (!loadResult.ok) {
+        throw new PipelineValidationError(buildValidationResult("pipeline", loadResult.findings, false));
+    }
+    const loaded = loadResult.loaded;
+    const taskYaml = options.taskYaml ??
+        (options.taskPath
+            ? await readFile(options.taskPath, "utf8")
+            : (() => {
+                throw new Error("taskPath or taskYaml is required");
+            })());
+    const label = options.taskPath ?? "task.yaml";
+    const task = loadTaskFromYaml(taskYaml, `task file ${label}`);
+    const checkoutRoot = await resolveAndValidateCheckout(task, options.checkoutOverride, options.cwd);
+    const run = await options.store.createRun({
+        pipelineId: loaded.pipeline.id,
+        taskYaml,
+        taskId: task.id,
+        checkoutRoot,
+        pipelineDag: buildPipelineDagSnapshotFromLoaded(loaded),
+    });
+    const executionMode = readStageExecutionMode(process.env, options.executionMode);
+    const stageProcessLauncher = resolveStageProcessLauncher(executionMode, options.stageProcessLauncher);
+    return {
+        task,
+        loaded,
+        run,
+        agent: options.agent,
+        store: options.store,
+        cwd: options.cwd,
+        checkoutRoot,
+        hitl: options.hitl,
+        executionMode,
+        stageProcessLauncher,
+        operatorCatalog: options.operatorCatalog,
+    };
+}
+export async function executeStages(prepared, options) {
+    const maxActiveStagesPerRun = readMaxActiveStagesPerRun(process.env, options?.maxActiveStagesPerRun);
+    const executionMode = options?.executionMode ??
+        prepared.executionMode ??
+        readStageExecutionMode(process.env);
+    const stageProcessLauncher = resolveStageProcessLauncher(executionMode, options?.stageProcessLauncher ?? prepared.stageProcessLauncher);
+    return runPipelineDag({
+        prepared,
+        maxActiveStagesPerRun,
+        resumeFromStageId: options?.resumeFromStageId,
+        initialPrior: options?.initialPrior,
+        startAtStageIndex: options?.startAtStageIndex,
+        executionMode,
+        stageProcessLauncher,
+    });
+}
+export async function runPipeline(options) {
+    const cwd = options.cwd ?? process.cwd();
+    const prepared = await preparePipeline({
+        agent: options.agent,
+        store: options.store,
+        taskPath: options.taskPath,
+        taskYaml: options.taskYaml,
+        pipeline: options.pipeline,
+        cwd,
+        checkoutOverride: options.checkoutOverride,
+        hitl: options.hitl,
+        executionMode: options.executionMode,
+        stageProcessLauncher: options.stageProcessLauncher,
+        operatorCatalog: options.operatorCatalog,
+    });
+    return executeStages(prepared, {
+        maxActiveStagesPerRun: options.maxActiveStagesPerRun,
+        executionMode: prepared.executionMode,
+        stageProcessLauncher: prepared.stageProcessLauncher,
+    });
+}
+/** Create the run immediately, then execute stages in the returned promise. */
+export async function startPipeline(options) {
+    const cwd = options.cwd ?? process.cwd();
+    const prepared = await preparePipeline({
+        agent: options.agent,
+        store: options.store,
+        taskPath: options.taskPath,
+        taskYaml: options.taskYaml,
+        pipeline: options.pipeline,
+        cwd,
+        checkoutOverride: options.checkoutOverride,
+        hitl: options.hitl,
+        executionMode: options.executionMode,
+        stageProcessLauncher: options.stageProcessLauncher,
+        operatorCatalog: options.operatorCatalog,
+    });
+    const done = executeStages(prepared, {
+        maxActiveStagesPerRun: options.maxActiveStagesPerRun,
+        executionMode: prepared.executionMode,
+        stageProcessLauncher: prepared.stageProcessLauncher,
+    }).catch(async (err) => {
+        await prepared.store.updateRunStatus(prepared.run.runId, "failed").catch(() => undefined);
+        return {
+            ok: false,
+            runDir: prepared.run.workspaceDir,
+            runId: prepared.run.runId,
+            reason: err instanceof Error ? err.message : String(err),
+        };
+    });
+    return {
+        runId: prepared.run.runId,
+        runDir: prepared.run.workspaceDir,
+        done,
+    };
+}
+export function resolveTaskPath(taskArg, cwd = process.cwd()) {
+    return path.resolve(cwd, taskArg);
+}

package/dist/runtime/pipelineScheduler.d.ts

@@ -0,0 +1,90 @@
+import type { AgentPort } from "../agent/port.js";
+import type { RunStore, StageSnapshot } from "../runstore/port.js";
+import type { StageEnvelope } from "../types/envelope.js";
+import type { LoadedPipeline, ResolvedPipelineDag } from "../types/pipeline.js";
+import type { TaskFile } from "../types/task.js";
+import type { PipelineRunResult } from "./pipelineRunner.js";
+import type { StageProcessLauncher } from "./stageProcessLauncher.js";
+import type { StageExecutionMode } from "./stageConcurrency.js";
+import type { StageHitlController } from "./stageHitl.js";
+import type { OperatorCatalog } from "./stageAttemptBootstrap.js";
+type SchedulerPreparedPipeline = {
+    task: TaskFile;
+    loaded: LoadedPipeline;
+    run: {
+        runId: string;
+        workspaceDir: string;
+    };
+    agent: AgentPort;
+    store: RunStore;
+    cwd: string;
+    checkoutRoot?: string;
+    hitl?: StageHitlController;
+    operatorCatalog?: OperatorCatalog;
+};
+export type { SchedulerPreparedPipeline };
+type RetryContext = {
+    retryRoots: Map<string, number>;
+};
+export type RetryMutationQueue = {
+    enqueue(stageId: string, attempt: number): Promise<void>;
+    drainPending(apply: (stageId: string, attempt: number) => void): void;
+    pendingStageIds(): ReadonlySet<string>;
+};
+export type RetryRootTerminalOutcome = "succeeded" | "failed" | "skipped";
+export type OnRetryRootTerminal = (stageId: string, attempt: number, outcome: RetryRootTerminalOutcome) => void;
+type HasWorkOutsideBranchContext = {
+    retryRoots?: ReadonlyMap<string, number>;
+    pendingDeltaStageIds?: ReadonlySet<string>;
+};
+type StageScheduleState = "pending" | "active" | "waiting" | "succeeded" | "failed" | "skipped";
+export type HydratedSchedule = {
+    states: Map<string, StageScheduleState>;
+    completedEnvelopes: Map<string, StageEnvelope>;
+    schedulingHalted: boolean;
+    firstFailureReason: string | undefined;
+};
+export declare function snapshotToScheduleState(snap: {
+    status: StageSnapshot["status"];
+}, overrides: ReadonlyMap<string, StageScheduleState>, stageId: string, executionMode?: StageExecutionMode): StageScheduleState;
+export declare function hydrateScheduleFromStore(store: RunStore, runId: string, dag: ResolvedPipelineDag, executionMode?: StageExecutionMode): Promise<HydratedSchedule>;
+export declare function hydrateScheduleForRetryRoots(store: RunStore, runId: string, dag: ResolvedPipelineDag, retryStageIds: string[]): Promise<HydratedSchedule>;
+export declare function hydrateScheduleForRetry(store: RunStore, runId: string, dag: ResolvedPipelineDag, retryStageId: string): Promise<HydratedSchedule>;
+export type ResumeRunOptions = {
+    prepared: SchedulerPreparedPipeline;
+    maxActiveStagesPerRun: number;
+    resumeFromStageId: string;
+    initialPrior?: StageEnvelope | null;
+    executionMode?: StageExecutionMode;
+    stageProcessLauncher?: StageProcessLauncher;
+};
+export declare function resumeRun(options: ResumeRunOptions): Promise<PipelineRunResult>;
+export type RunPipelineDagOptions = {
+    prepared: SchedulerPreparedPipeline;
+    maxActiveStagesPerRun: number;
+    resumeFromStageId?: string;
+    initialPrior?: StageEnvelope | null;
+    /** @deprecated Linear pipelines only — skips stages before this declaration index. */
+    startAtStageIndex?: number;
+    executionMode?: StageExecutionMode;
+    stageProcessLauncher?: StageProcessLauncher;
+    initialSchedule?: HydratedSchedule;
+    retryContext?: RetryContext;
+    mutationQueue?: RetryMutationQueue;
+    onLoopTick?: () => void | Promise<void>;
+    onRetryRootTerminal?: OnRetryRootTerminal;
+};
+export type RetryRunOptions = {
+    prepared: SchedulerPreparedPipeline;
+    retryRoots: Map<string, number>;
+    maxActiveStagesPerRun: number;
+    executionMode?: StageExecutionMode;
+    stageProcessLauncher?: StageProcessLauncher;
+    mutationQueue?: RetryMutationQueue;
+    onLoopTick?: () => void | Promise<void>;
+    onRetryRootTerminal?: OnRetryRootTerminal;
+};
+export declare function retryRun(options: RetryRunOptions): Promise<PipelineRunResult>;
+export declare function applyRetryRootDelta(dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, completedEnvelopes: Map<string, StageEnvelope>, retryRoots: Map<string, number>, stageId: string, attempt: number, clearSchedulingHalt?: () => void): void;
+export declare function hasWorkOutsideBranch(dag: ResolvedPipelineDag, branchRootId: string, states: Map<string, StageScheduleState>, ctx?: HasWorkOutsideBranchContext): boolean;
+export declare function runPipelineDag(options: RunPipelineDagOptions): Promise<PipelineRunResult>;