stageflow 0.1.0

package/dist/runtime/stageRoots.d.ts

@@ -0,0 +1,22 @@
+import type { StageAttemptContext } from "./stageAttemptContext.js";
+import type { TaskFile } from "../types/task.js";
+export type StageRoots = {
+    mode: "bound" | "unbound";
+    cwd: string;
+    runWorkspaceDir: string;
+    checkoutRoot?: string;
+    agentDir: string;
+    attempt?: number;
+    /** Durable Pi auth.json path for ModelRuntime.create({ authPath }). */
+    authPath?: string;
+};
+export declare const STAGEFLOW_RUN_WORKSPACE = "STAGEFLOW_RUN_WORKSPACE";
+export declare const PI_CODING_AGENT_DIR_ENV = "PI_CODING_AGENT_DIR";
+export declare function withResolvedAuthPath(roots: StageRoots, factoryCwd: string): StageRoots;
+export declare function rootsForStageWorker(runWorkspaceDir: string, stageId: string, model: string, checkoutRoot?: string, attemptCtx?: StageAttemptContext): StageRoots;
+export declare function bindPiAgentDirEnv(agentDirPath: string): () => void;
+export declare function resolveCheckoutPath(declared: string, cwd: string): string;
+export declare function resolveAndValidateCheckout(task: TaskFile, override: string | undefined, factoryCwd: string): Promise<string | undefined>;
+export declare function buildStageRoots(runWorkspaceDir: string, stageId: string, checkoutRoot?: string, attemptCtx?: StageAttemptContext): StageRoots;
+export declare function bindRunWorkspaceEnv(_runWorkspaceDir: string): () => void;
+export declare function resetBindRunWorkspaceEnvForTests(): void;

package/dist/runtime/stageRoots.js

@@ -0,0 +1,93 @@
+import { access, constants, stat } from "node:fs/promises";
+import path from "node:path";
+import { isCursorModelRef } from "../agent/cursorProvider.js";
+import { stageDir } from "../runstore/paths.js";
+import { noAttemptContext } from "./stageAttemptContext.js";
+import { resolveCredentialBinding } from "./credentialBinding.js";
+export const STAGEFLOW_RUN_WORKSPACE = "STAGEFLOW_RUN_WORKSPACE";
+export const PI_CODING_AGENT_DIR_ENV = "PI_CODING_AGENT_DIR";
+export function withResolvedAuthPath(roots, factoryCwd) {
+    if (roots.authPath)
+        return roots;
+    const binding = resolveCredentialBinding(factoryCwd);
+    return { ...roots, authPath: binding.authPath };
+}
+export function rootsForStageWorker(runWorkspaceDir, stageId, model, checkoutRoot, attemptCtx) {
+    const roots = buildStageRoots(runWorkspaceDir, stageId, checkoutRoot, attemptCtx);
+    if (!isCursorModelRef(model) || roots.mode === "bound") {
+        return roots;
+    }
+    return {
+        ...roots,
+        cwd: stageDir(runWorkspaceDir, stageId),
+    };
+}
+export function bindPiAgentDirEnv(agentDirPath) {
+    const prev = process.env[PI_CODING_AGENT_DIR_ENV];
+    process.env[PI_CODING_AGENT_DIR_ENV] = agentDirPath;
+    return () => {
+        if (prev === undefined) {
+            delete process.env[PI_CODING_AGENT_DIR_ENV];
+        }
+        else {
+            process.env[PI_CODING_AGENT_DIR_ENV] = prev;
+        }
+    };
+}
+export function resolveCheckoutPath(declared, cwd) {
+    return path.resolve(cwd, declared);
+}
+export async function resolveAndValidateCheckout(task, override, factoryCwd) {
+    const raw = override ?? task.checkout;
+    if (raw === undefined)
+        return undefined;
+    if (raw.trim() === "") {
+        throw new Error("Invalid checkout: path is empty or whitespace-only");
+    }
+    const absPath = resolveCheckoutPath(raw, factoryCwd);
+    let st;
+    try {
+        st = await stat(absPath);
+    }
+    catch {
+        throw new Error(`Checkout path does not exist: ${absPath}`);
+    }
+    if (!st.isDirectory()) {
+        throw new Error(`Checkout path is not a directory: ${absPath}`);
+    }
+    try {
+        await access(absPath, constants.R_OK | constants.W_OK | constants.X_OK);
+    }
+    catch {
+        throw new Error(`Checkout path is not readable/writable/searchable: ${absPath}`);
+    }
+    return absPath;
+}
+export function buildStageRoots(runWorkspaceDir, stageId, checkoutRoot, attemptCtx) {
+    const ctx = attemptCtx ?? noAttemptContext();
+    const agentDirPath = ctx.agentDirPath(runWorkspaceDir, stageId);
+    const attempt = ctx.attempt;
+    if (checkoutRoot !== undefined) {
+        return {
+            mode: "bound",
+            cwd: checkoutRoot,
+            runWorkspaceDir,
+            checkoutRoot,
+            agentDir: agentDirPath,
+            attempt,
+        };
+    }
+    return {
+        mode: "unbound",
+        cwd: runWorkspaceDir,
+        runWorkspaceDir,
+        agentDir: agentDirPath,
+        attempt,
+    };
+}
+export function bindRunWorkspaceEnv(_runWorkspaceDir) {
+    return () => { };
+}
+export function resetBindRunWorkspaceEnvForTests() {
+    delete process.env[STAGEFLOW_RUN_WORKSPACE];
+}

package/dist/runtime/stageRunner.d.ts

@@ -0,0 +1,59 @@
+import type { AgentPort, StageHandle, StageRunResult } from "../agent/port.js";
+import type { StageEnvelope } from "../types/envelope.js";
+import type { StageConfig } from "../types/stage.js";
+import type { TaskFile } from "../types/task.js";
+import type { RunStore } from "../runstore/port.js";
+import type { ResolvedPipelineDag } from "../types/pipeline.js";
+import type { StageAttemptContext } from "./stageAttemptContext.js";
+import { type StageRoots } from "./stageRoots.js";
+import { type StageHitlController } from "./stageHitl.js";
+import { type OperatorCatalog } from "./stageAttemptBootstrap.js";
+export type RunStageOutcome = StageRunResult | {
+    waiting: true;
+};
+export type RunStageYieldLoopResult = StageRunResult | {
+    waiting: true;
+};
+export declare function isRunStageWaiting(outcome: RunStageOutcome): outcome is {
+    waiting: true;
+};
+export type RunStageOptions = {
+    agent: AgentPort;
+    store: RunStore;
+    runId: string;
+    stage: StageConfig;
+    task: TaskFile;
+    dag?: ResolvedPipelineDag;
+    priorEnvelope?: StageEnvelope | null;
+    checkoutRoot?: string;
+    workspaceDir?: string;
+    hitl?: StageHitlController;
+    /** Skip appending `started` (restart reconstruct already past start). */
+    skipStarted?: boolean;
+    /** When set, used instead of buildStageRoots (worker cursor isolation). */
+    roots?: StageRoots;
+    /**
+     * When set, use this handle instead of openStage (restart: caller opens and
+     * may pre-inject the answer before the yield loop).
+     */
+    existingHandle?: StageHandle;
+    workerMode?: boolean;
+    attemptCtx?: StageAttemptContext;
+    /** Factory project root for credential binding (settings + SF-owned auth). */
+    factoryCwd?: string;
+    operatorCatalog?: OperatorCatalog;
+    completedEnvelopes?: Map<string, StageEnvelope>;
+};
+/**
+ * Yield loop: next() until completed; on wait, coordinator enterWait (KTD6).
+ */
+export declare function runStageYieldLoop(options: {
+    handle: StageHandle;
+    runId: string;
+    stageId: string;
+    hitl?: StageHitlController;
+    workerMode?: boolean;
+    store?: RunStore;
+    attemptCtx?: StageAttemptContext;
+}): Promise<RunStageYieldLoopResult>;
+export declare function runStage(options: RunStageOptions): Promise<RunStageOutcome>;

package/dist/runtime/stageRunner.js

@@ -0,0 +1,221 @@
+import { deriveExecutionPatchFromEvent } from "../runstore/stageExecution.js";
+import { buildStageRoots, withResolvedAuthPath, } from "./stageRoots.js";
+import { createDefaultHitlQaHooks, recordWaitEnter, } from "./stageHitl.js";
+import { openStageAttempt, } from "./stageAttemptBootstrap.js";
+export function isRunStageWaiting(outcome) {
+    return "waiting" in outcome && outcome.waiting === true;
+}
+const LIFECYCLE_EVENTS = new Set([
+    "started",
+    "resumed",
+    "waiting_for_input",
+    "succeeded",
+    "failed",
+]);
+async function ensureStageExecutionForAttempt(store, runId, stageId, attempt) {
+    try {
+        await store.getStageExecution(runId, stageId, attempt);
+    }
+    catch {
+        if (attempt !== 1) {
+            throw new Error(`Stage execution not found: ${runId}/${stageId} attempt ${attempt}`);
+        }
+        await store.createStageExecution(runId, stageId);
+    }
+}
+async function appendLifecycleEventAndPatchExecution(options) {
+    const { store, runId, stageId, event, attemptCtx } = options;
+    const attemptNum = attemptCtx?.attempt ?? 1;
+    const attemptOpt = attemptCtx?.eventOptions();
+    await store.appendStageEvent(runId, stageId, event, attemptOpt);
+    if (!LIFECYCLE_EVENTS.has(event.event))
+        return;
+    try {
+        const execution = await store.getStageExecution(runId, stageId, attemptNum);
+        const patch = deriveExecutionPatchFromEvent(event, execution);
+        if (Object.keys(patch).length === 0)
+            return;
+        await store.updateStageExecution(runId, stageId, attemptNum, patch);
+    }
+    catch {
+        // no execution row yet
+    }
+}
+async function finalizeStageResult(options) {
+    const { store, runId, stageId, result, activityChain, attemptCtx } = options;
+    const attemptOpt = attemptCtx?.eventOptions();
+    await activityChain;
+    if (result.envelope) {
+        await store.writeEnvelope(runId, stageId, result.envelope, attemptOpt);
+    }
+    if (!result.ok) {
+        await appendLifecycleEventAndPatchExecution({
+            store,
+            runId,
+            stageId,
+            event: { event: "failed", reason: result.reason },
+            attemptCtx,
+        });
+        console.error(`Stage ${stageId} failed: ${result.reason}`);
+        return result;
+    }
+    await appendLifecycleEventAndPatchExecution({
+        store,
+        runId,
+        stageId,
+        event: { event: "succeeded" },
+        attemptCtx,
+    });
+    console.error(`Stage ${stageId} succeeded.`);
+    return result;
+}
+/**
+ * Yield loop: next() until completed; on wait, coordinator enterWait (KTD6).
+ */
+export async function runStageYieldLoop(options) {
+    const { handle, runId, stageId, hitl, workerMode, store, attemptCtx } = options;
+    while (true) {
+        const event = await handle.next();
+        if (event.status === "waiting_for_input") {
+            if (workerMode) {
+                if (!store) {
+                    return {
+                        ok: false,
+                        reason: "workerMode requires store",
+                    };
+                }
+                await recordWaitEnter({
+                    store,
+                    runId,
+                    stageId,
+                    request: event.request,
+                    attemptCtx,
+                    qaHooks: createDefaultHitlQaHooks(store),
+                });
+                return { waiting: true };
+            }
+            if (!hitl) {
+                return {
+                    ok: false,
+                    reason: "stage requested wait but no HITL controller is configured",
+                };
+            }
+            await hitl.enterWait(runId, stageId, handle, event.request, attemptCtx);
+            continue;
+        }
+        return event.result;
+    }
+}
+export async function runStage(options) {
+    const { agent, store, runId, stage, task, dag, checkoutRoot, workspaceDir, hitl, skipStarted, existingHandle, workerMode, roots: rootsOverride, attemptCtx, factoryCwd, operatorCatalog, completedEnvelopes, } = options;
+    const attemptOpt = attemptCtx?.eventOptions();
+    const baseRoots = rootsOverride ??
+        buildStageRoots(workspaceDir ?? store.getWorkspaceDir(runId), stage.id, checkoutRoot, attemptCtx);
+    const roots = factoryCwd !== undefined
+        ? withResolvedAuthPath(baseRoots, factoryCwd)
+        : baseRoots;
+    const attempt = attemptCtx?.attempt ?? 1;
+    await ensureStageExecutionForAttempt(store, runId, stage.id, attempt);
+    await store.ensureAttemptWorkspace(runId, stage.id, attempt);
+    if (!skipStarted) {
+        await appendLifecycleEventAndPatchExecution({
+            store,
+            runId,
+            stageId: stage.id,
+            event: { event: "started" },
+            attemptCtx,
+        });
+    }
+    console.error(`Running stage ${stage.id} (${stage.model})...`);
+    let activityChain = Promise.resolve();
+    const enqueueActivity = (event) => {
+        activityChain = activityChain.then(() => store.appendStageEvent(runId, stage.id, event, attemptOpt));
+    };
+    let handle;
+    if (existingHandle) {
+        handle = existingHandle;
+    }
+    else {
+        const opened = await openStageAttempt({
+            agent,
+            store,
+            runId,
+            stage,
+            task,
+            dag: dag ?? {
+                nodes: [
+                    { id: stage.id, needs: null, ancestors: [], stageIndex: 0 },
+                ],
+                roots: [stage.id],
+                childrenOf: {},
+            },
+            checkoutRoot,
+            workspaceDir: workspaceDir ?? store.getWorkspaceDir(runId),
+            factoryCwd,
+            attemptCtx,
+            operatorCatalog,
+            roots,
+            completedEnvelopes,
+            onActivity: (event) => {
+                enqueueActivity(event);
+            },
+        });
+        if (!opened.ok) {
+            await appendLifecycleEventAndPatchExecution({
+                store,
+                runId,
+                stageId: stage.id,
+                event: { event: "failed", reason: opened.reason },
+                attemptCtx,
+            });
+            console.error(`Stage ${stage.id} failed: ${opened.reason}`);
+            return { ok: false, reason: opened.reason };
+        }
+        handle = opened.handle;
+    }
+    let outcome;
+    try {
+        outcome = await runStageYieldLoop({
+            handle,
+            runId,
+            stageId: stage.id,
+            hitl,
+            workerMode,
+            store: workerMode ? store : undefined,
+            attemptCtx,
+        });
+    }
+    catch (err) {
+        const reason = err instanceof Error ? err.message : String(err);
+        await activityChain;
+        hitl?.clearLiveWait(runId, stage.id);
+        await appendLifecycleEventAndPatchExecution({
+            store,
+            runId,
+            stageId: stage.id,
+            event: { event: "failed", reason },
+            attemptCtx,
+        });
+        console.error(`Stage ${stage.id} failed: ${reason}`);
+        await handle.close().catch(() => undefined);
+        return { ok: false, reason };
+    }
+    if (isRunStageWaiting(outcome)) {
+        await activityChain;
+        await handle.close({ park: true }).catch(() => undefined);
+        return { waiting: true };
+    }
+    try {
+        return await finalizeStageResult({
+            store,
+            runId,
+            stageId: stage.id,
+            result: outcome,
+            activityChain,
+            attemptCtx,
+        });
+    }
+    finally {
+        await handle.close().catch(() => undefined);
+    }
+}

package/dist/runtime/stageWorker.d.ts

@@ -0,0 +1,6 @@
+import { type RunStageOutcome } from "./stageRunner.js";
+import { type StageWorkerInput, type StageWorkerResult } from "./stageWorkerProtocol.js";
+export type { StageWorkerInput, StageWorkerResult } from "./stageWorkerProtocol.js";
+export declare function runStageWorker(input: StageWorkerInput): Promise<RunStageOutcome>;
+export declare function sendWorkerResult(result: StageWorkerResult): void;
+export declare function exitForOutcome(outcome: RunStageOutcome): never;

package/dist/runtime/stageWorker.js

@@ -0,0 +1,109 @@
+import { PiAgentAdapter } from "../agent/piAdapter.js";
+import { loadPipeline } from "../config/loadPipeline.js";
+import { loadTaskFromYaml } from "../config/loadTask.js";
+import { createRunStore } from "../runstore/createStore.js";
+import { bindPiAgentDirEnv, rootsForStageWorker, withResolvedAuthPath, } from "./stageRoots.js";
+import { runStage, isRunStageWaiting, } from "./stageRunner.js";
+import { openStageAttempt } from "./stageAttemptBootstrap.js";
+import { attemptContext } from "./stageAttemptContext.js";
+import { outcomeToWorkerResult, STAGE_WORKER_EXIT, } from "./stageWorkerProtocol.js";
+export async function runStageWorker(input) {
+    const store = createRunStore({ rootDir: input.rootDir });
+    const meta = await store.readRunMeta(input.runId);
+    const taskYaml = await store.readTaskYaml(input.runId);
+    const task = loadTaskFromYaml(taskYaml, `run ${input.runId} task`);
+    const loaded = await loadPipeline(meta.pipeline_id, { cwd: input.rootDir });
+    const stage = loaded.stages.find((s) => s.id === input.stageId);
+    if (!stage) {
+        return {
+            ok: false,
+            reason: `Stage ${input.stageId} not in pipeline ${meta.pipeline_id}`,
+        };
+    }
+    const agent = new PiAgentAdapter();
+    const workspaceDir = store.getWorkspaceDir(input.runId);
+    const checkoutRoot = meta.checkout_root;
+    const mode = input.mode ?? "run";
+    const attempt = input.attempt ?? 1;
+    const attemptCtx = input.attempt !== undefined ? attemptContext(input.attempt) : undefined;
+    const eventOptions = { attempt };
+    const roots = withResolvedAuthPath(rootsForStageWorker(workspaceDir, input.stageId, stage.model, checkoutRoot, attemptCtx), input.rootDir);
+    const unbindAgentDir = bindPiAgentDirEnv(roots.agentDir);
+    try {
+        if (mode === "resume") {
+            const opened = await openStageAttempt({
+                agent,
+                store,
+                runId: input.runId,
+                stage,
+                task,
+                dag: loaded.dag,
+                checkoutRoot,
+                workspaceDir,
+                factoryCwd: input.rootDir,
+                attemptCtx,
+                operatorCatalog: input.operatorCatalog,
+                roots,
+                resumeToken: input.sessionFilePath,
+                onActivity: (event) => {
+                    void store.appendStageEvent(input.runId, input.stageId, event, eventOptions);
+                },
+            });
+            if (!opened.ok) {
+                return { ok: false, reason: opened.reason };
+            }
+            opened.handle.deliverAnswer(input.resumeAnswer);
+            const outcome = await runStage({
+                agent,
+                store,
+                runId: input.runId,
+                stage,
+                task,
+                dag: loaded.dag,
+                checkoutRoot,
+                workspaceDir,
+                skipStarted: true,
+                existingHandle: opened.handle,
+                roots,
+                attemptCtx,
+                factoryCwd: input.rootDir,
+                operatorCatalog: input.operatorCatalog,
+            });
+            return outcome;
+        }
+        return runStage({
+            agent,
+            store,
+            runId: input.runId,
+            stage,
+            task,
+            dag: loaded.dag,
+            checkoutRoot,
+            workspaceDir,
+            workerMode: true,
+            roots,
+            attemptCtx,
+            factoryCwd: input.rootDir,
+            operatorCatalog: input.operatorCatalog,
+        });
+    }
+    finally {
+        unbindAgentDir();
+    }
+}
+export function sendWorkerResult(result) {
+    if (typeof process.send === "function") {
+        process.send(result);
+    }
+}
+export function exitForOutcome(outcome) {
+    const message = outcomeToWorkerResult(outcome);
+    sendWorkerResult(message);
+    if (isRunStageWaiting(outcome)) {
+        process.exit(STAGE_WORKER_EXIT.WAITING);
+    }
+    if (outcome.ok) {
+        process.exit(STAGE_WORKER_EXIT.SUCCEEDED);
+    }
+    process.exit(STAGE_WORKER_EXIT.FAILED);
+}

package/dist/runtime/stageWorkerProtocol.d.ts

@@ -0,0 +1,27 @@
+import type { RunStageOutcome } from "./stageRunner.js";
+import type { OperatorCatalog } from "./stageAttemptBootstrap.js";
+export declare const SF_STAGE_WORKER = "SF_STAGE_WORKER";
+export declare const STAGE_WORKER_EXIT: {
+    readonly SUCCEEDED: 0;
+    readonly FAILED: 1;
+    readonly WAITING: 2;
+};
+export type StageWorkerInput = {
+    runId: string;
+    stageId: string;
+    rootDir: string;
+    mode?: "run" | "resume";
+    resumeAnswer?: unknown;
+    attempt?: number;
+    sessionFilePath?: string;
+    operatorCatalog?: OperatorCatalog;
+};
+export type StageWorkerResult = {
+    type: "succeeded";
+} | {
+    type: "failed";
+    reason: string;
+} | {
+    type: "waiting";
+};
+export declare function outcomeToWorkerResult(outcome: RunStageOutcome): StageWorkerResult;

package/dist/runtime/stageWorkerProtocol.js

@@ -0,0 +1,16 @@
+export const SF_STAGE_WORKER = "SF_STAGE_WORKER";
+export const STAGE_WORKER_EXIT = {
+    SUCCEEDED: 0,
+    FAILED: 1,
+    WAITING: 2,
+};
+export function outcomeToWorkerResult(outcome) {
+    if ("waiting" in outcome && outcome.waiting) {
+        return { type: "waiting" };
+    }
+    const result = outcome;
+    if (result.ok) {
+        return { type: "succeeded" };
+    }
+    return { type: "failed", reason: result.reason };
+}

package/dist/runtime/taskInput.d.ts

@@ -0,0 +1,15 @@
+import type { TaskFile } from "../types/task.js";
+export type StartTaskInput = {
+    task?: string | TaskFile;
+    taskYaml?: string;
+};
+export type ResolvedTaskInput = {
+    kind: "path";
+    taskPath: string;
+} | {
+    kind: "yaml";
+    taskYaml: string;
+};
+export declare function isTaskFile(value: unknown): value is TaskFile;
+export declare function taskFileToYaml(task: TaskFile): string;
+export declare function resolveStartTaskInput(input: StartTaskInput, cwd: string): ResolvedTaskInput;

package/dist/runtime/taskInput.js

@@ -0,0 +1,38 @@
+import { stringify as stringifyYaml } from "yaml";
+import { parseTaskFile } from "../config/loadTask.js";
+import { resolveTaskPath } from "./pipelineRunner.js";
+export function isTaskFile(value) {
+    return parseTaskFile(value, "task").ok;
+}
+export function taskFileToYaml(task) {
+    const doc = {
+        id: task.id,
+        goal: task.goal,
+    };
+    if (task.context !== undefined)
+        doc.context = task.context;
+    if (task.constraints !== undefined)
+        doc.constraints = task.constraints;
+    if (task.checkout !== undefined)
+        doc.checkout = task.checkout;
+    return stringifyYaml(doc);
+}
+export function resolveStartTaskInput(input, cwd) {
+    if (typeof input.taskYaml === "string") {
+        if (input.taskYaml.trim().length === 0) {
+            throw new Error("taskYaml must be a non-empty string");
+        }
+        return { kind: "yaml", taskYaml: input.taskYaml };
+    }
+    if (typeof input.task === "string") {
+        if (input.task.trim().length === 0) {
+            throw new Error("task path must be a non-empty string");
+        }
+        return { kind: "path", taskPath: resolveTaskPath(input.task, cwd) };
+    }
+    const parsed = parseTaskFile(input.task, "task");
+    if (parsed.ok) {
+        return { kind: "yaml", taskYaml: taskFileToYaml(parsed.value) };
+    }
+    throw new Error("task path, task object, or taskYaml is required");
+}

package/dist/server/http.d.ts

@@ -0,0 +1,31 @@
+import { type Server } from "node:http";
+import type { AgentPort } from "../agent/port.js";
+import { type ProviderAuthContext } from "../agent/providerAuth.js";
+import { type RunStoreKind } from "../runstore/createStore.js";
+import type { RunStore } from "../runstore/port.js";
+import { RunManager } from "../runtime/runManager.js";
+declare const DEFAULT_PORT = 3847;
+export type UiServerOptions = {
+    agent: AgentPort;
+    cwd?: string;
+    agentDir?: string;
+    rootDir?: string;
+    store?: RunStore;
+    storeKind?: RunStoreKind;
+    port?: number;
+    host?: string;
+    uiDistDir?: string;
+    maxConcurrent?: number;
+    providerAuthContext?: ProviderAuthContext;
+};
+export declare function defaultUiDistDir(): string;
+export declare function startUiServer(options: UiServerOptions): Promise<{
+    server: Server;
+    port: number;
+    host: string;
+    url: string;
+    mcpUrl: string;
+    manager: RunManager;
+    store: RunStore;
+}>;
+export { DEFAULT_PORT };