stageflow 0.3.0 → 0.9.0

package/dist/runtime/stageAttemptBootstrap.js

@@ -1,5 +1,5 @@
 import { resolveSkillByName } from "../config/listSkills.js";
-import { resolveForkEmitContext } from "../config/resolveForkEmitContext.js";
+import { resolveCloneEmitContext, resolveForkEmitContext } from "../config/resolveForkEmitContext.js";
 import { buildCompletedEnvelopesFromRun, resolvePriorEnvelope, } from "./envelopeRouting.js";
 import { resumeSessionFilePath } from "./stageAttemptContext.js";
 import { buildStageRoots, rootsForStageWorker, withResolvedAuthPath, } from "./stageRoots.js";
@@ -39,40 +39,48 @@ async function openStageWithOperatorCatalog(agent, input, catalog) {
         };
     }
 }
-function resolveAttemptRoots(input) {
+function resolveAttemptRoots(input, stageId) {
     const baseRoots = input.roots ??
         (input.workerRoots
-            ? rootsForStageWorker(input.workspaceDir, input.stage.id, input.stage.model, input.checkoutRoot, input.attemptCtx)
-            : buildStageRoots(input.workspaceDir, input.stage.id, input.checkoutRoot, input.attemptCtx));
+            ? rootsForStageWorker(input.workspaceDir, stageId, input.stage.model, input.checkoutRoot, input.attemptCtx)
+            : buildStageRoots(input.workspaceDir, stageId, input.checkoutRoot, input.attemptCtx));
     return input.factoryCwd !== undefined
         ? withResolvedAuthPath(baseRoots, input.factoryCwd)
         : baseRoots;
 }
 export async function openStageAttempt(input) {
+    const stageId = input.stageId ?? input.stage.id;
+    const definitionId = input.stage.id;
     const completedEnvelopes = input.completedEnvelopes ??
-        (await buildCompletedEnvelopesFromRun(input.store, input.runId));
+        (await buildCompletedEnvelopesFromRun(input.store, input.runId, undefined, input.dag));
     const priorResult = await resolvePriorEnvelope({
         dag: input.dag,
-        stageId: input.stage.id,
+        stageId,
         completedEnvelopes,
         store: input.store,
         runId: input.runId,
     });
     if (!priorResult.ok)
         return priorResult;
-    const roots = resolveAttemptRoots(input);
+    const roots = resolveAttemptRoots(input, stageId);
     const attempt = input.attemptCtx?.attempt ?? 1;
     const resumeToken = input.resumeToken ??
-        resumeSessionFilePath(input.workspaceDir, input.stage.id, attempt);
-    const forkEmitContext = resolveForkEmitContext(input.dag, input.stage.id);
+        resumeSessionFilePath(input.workspaceDir, stageId, attempt);
+    const forkEmitContext = resolveForkEmitContext(input.dag, definitionId);
+    const cloneEmitContext = resolveCloneEmitContext(input.dag, definitionId);
     const opened = await openStageWithOperatorCatalog(input.agent, {
         roots,
         stage: input.stage,
+        stageId,
         task: input.task,
         priorEnvelope: priorResult.prior,
+        ...(priorResult.joinPriors !== undefined
+            ? { priorEnvelopes: priorResult.joinPriors }
+            : {}),
         resumeToken,
         onActivity: input.onActivity,
         forkEmitContext,
+        cloneEmitContext,
     }, input.operatorCatalog);
     if (!opened.ok)
         return opened;

package/dist/runtime/stageRunner.d.ts

@@ -44,6 +44,7 @@ export type RunStageOptions = {
     operatorCatalog?: OperatorCatalog;
     completedEnvelopes?: Map<string, StageEnvelope>;
     skipGates?: boolean;
+    stageId?: string;
 };
 /**
  * Yield loop: next() until completed; on wait, coordinator enterWait (KTD6).

package/dist/runtime/stageRunner.js

@@ -114,28 +114,29 @@ export async function runStageYieldLoop(options) {
 }
 export async function runStage(options) {
     const { agent, store, runId, stage, task, dag, checkoutRoot, workspaceDir, hitl, skipStarted, existingHandle, workerMode, roots: rootsOverride, attemptCtx, factoryCwd, operatorCatalog, completedEnvelopes, skipGates, } = options;
+    const stageId = options.stageId ?? stage.id;
     const attemptOpt = attemptCtx?.eventOptions();
     const baseRoots = rootsOverride ??
-        buildStageRoots(workspaceDir ?? store.getWorkspaceDir(runId), stage.id, checkoutRoot, attemptCtx);
+        buildStageRoots(workspaceDir ?? store.getWorkspaceDir(runId), stageId, checkoutRoot, attemptCtx);
     const roots = factoryCwd !== undefined
         ? withResolvedAuthPath(baseRoots, factoryCwd)
         : baseRoots;
     const attempt = attemptCtx?.attempt ?? 1;
-    await ensureStageExecutionForAttempt(store, runId, stage.id, attempt);
-    await store.ensureAttemptWorkspace(runId, stage.id, attempt);
+    await ensureStageExecutionForAttempt(store, runId, stageId, attempt);
+    await store.ensureAttemptWorkspace(runId, stageId, attempt);
     if (!skipStarted) {
         await appendLifecycleEventAndPatchExecution({
             store,
             runId,
-            stageId: stage.id,
+            stageId,
             event: { event: "started" },
             attemptCtx,
         });
     }
-    console.error(`Running stage ${stage.id} (${stage.model})...`);
+    console.error(`Running stage ${stageId} (${stage.model})...`);
     let activityChain = Promise.resolve();
     const enqueueActivity = (event) => {
-        activityChain = activityChain.then(() => store.appendStageEvent(runId, stage.id, event, attemptOpt));
+        activityChain = activityChain.then(() => store.appendStageEvent(runId, stageId, event, attemptOpt));
     };
     let handle;
     if (existingHandle) {
@@ -147,12 +148,13 @@ export async function runStage(options) {
             store,
             runId,
             stage,
+            stageId,
             task,
             dag: dag ?? {
                 nodes: [
-                    { id: stage.id, needs: null, ancestors: [], stageIndex: 0 },
+                    { id: stageId, needs: null, ancestors: [], stageIndex: 0 },
                 ],
-                roots: [stage.id],
+                roots: [stageId],
                 childrenOf: {},
             },
             checkoutRoot,
@@ -170,11 +172,11 @@ export async function runStage(options) {
             await appendLifecycleEventAndPatchExecution({
                 store,
                 runId,
-                stageId: stage.id,
+                stageId,
                 event: { event: "failed", reason: opened.reason },
                 attemptCtx,
             });
-            console.error(`Stage ${stage.id} failed: ${opened.reason}`);
+            console.error(`Stage ${stageId} failed: ${opened.reason}`);
             return { ok: false, reason: opened.reason };
         }
         handle = opened.handle;
@@ -184,7 +186,7 @@ export async function runStage(options) {
         outcome = await runStageYieldLoop({
             handle,
             runId,
-            stageId: stage.id,
+            stageId,
             hitl,
             workerMode,
             store: workerMode ? store : undefined,
@@ -195,15 +197,15 @@ export async function runStage(options) {
     catch (err) {
         const reason = err instanceof Error ? err.message : String(err);
         await activityChain;
-        hitl?.clearLiveWait(runId, stage.id);
+        hitl?.clearLiveWait(runId, stageId);
         await appendLifecycleEventAndPatchExecution({
             store,
             runId,
-            stageId: stage.id,
+            stageId,
             event: { event: "failed", reason },
             attemptCtx,
         });
-        console.error(`Stage ${stage.id} failed: ${reason}`);
+        console.error(`Stage ${stageId} failed: ${reason}`);
         await handle.close().catch(() => undefined);
         return { ok: false, reason };
     }
@@ -216,7 +218,7 @@ export async function runStage(options) {
         return await finalizeStageResult({
             store,
             runId,
-            stageId: stage.id,
+            stageId,
             result: outcome,
             activityChain,
             attemptCtx,

package/dist/runtime/stageWorker.js

@@ -1,4 +1,5 @@
 import { PiAgentAdapter } from "../agent/piAdapter.js";
+import { definitionIdForInstance } from "../runstore/stageInstanceId.js";
 import { createRunStore } from "../runstore/createStore.js";
 import { loadRunContext } from "./resumeReconstruct.js";
 import { bindPiAgentDirEnv, rootsForStageWorker, withResolvedAuthPath, } from "./stageRoots.js";
@@ -9,13 +10,15 @@ import { outcomeToWorkerResult, STAGE_WORKER_EXIT, } from "./stageWorkerProtocol
 export async function runStageWorker(input) {
     const store = createRunStore({ rootDir: input.rootDir });
     const { meta, task, loaded } = await loadRunContext(store, input.runId, input.rootDir);
-    const stage = loaded.stages.find((s) => s.id === input.stageId);
+    const definitionId = definitionIdForInstance(meta.pipeline_dag, input.stageId);
+    const stage = loaded.stages.find((s) => s.id === definitionId);
     if (!stage) {
         return {
             ok: false,
             reason: `Stage ${input.stageId} not in pipeline ${meta.pipeline_id}`,
         };
     }
+    const dag = meta.pipeline_dag ?? loaded.dag;
     const agent = new PiAgentAdapter();
     const workspaceDir = store.getWorkspaceDir(input.runId);
     const checkoutRoot = meta.checkout_root;
@@ -32,8 +35,9 @@ export async function runStageWorker(input) {
                 store,
                 runId: input.runId,
                 stage,
+                stageId: input.stageId,
                 task,
-                dag: loaded.dag,
+                dag,
                 checkoutRoot,
                 workspaceDir,
                 factoryCwd: input.rootDir,
@@ -54,12 +58,14 @@ export async function runStageWorker(input) {
                 store,
                 runId: input.runId,
                 stage,
+                stageId: input.stageId,
                 task,
-                dag: loaded.dag,
+                dag,
                 checkoutRoot,
                 workspaceDir,
                 skipStarted: true,
                 existingHandle: opened.handle,
+                workerMode: true,
                 roots,
                 attemptCtx,
                 factoryCwd: input.rootDir,
@@ -73,8 +79,9 @@ export async function runStageWorker(input) {
             store,
             runId: input.runId,
             stage,
+            stageId: input.stageId,
             task,
-            dag: loaded.dag,
+            dag,
             checkoutRoot,
             workspaceDir,
             workerMode: true,

package/dist/server/bootstrap.d.ts

@@ -0,0 +1,32 @@
+import type { AgentPort } from "../agent/port.js";
+import type { ProviderAuthContext } from "../agent/providerAuth.js";
+import { type RunStoreKind } from "../runstore/createStore.js";
+import type { RunStore } from "../runstore/port.js";
+import { RunManager } from "../runtime/runManager.js";
+import { type RunChangeBus } from "../runtime/runChangeBus.js";
+import { type McpHttpHandler } from "../mcp/server.js";
+export type StageflowHostOptions = {
+    agent: AgentPort;
+    cwd?: string;
+    agentDir?: string;
+    rootDir?: string;
+    store?: RunStore;
+    storeKind?: RunStoreKind;
+    maxConcurrent?: number;
+    providerAuthContext?: ProviderAuthContext;
+    mcpStateless?: boolean;
+    runChangeBus?: RunChangeBus;
+};
+export type StageflowHostBootstrap = {
+    cwd: string;
+    agentDir: string;
+    rootDir: string;
+    isGitProject: boolean;
+    store: RunStore;
+    manager: RunManager;
+    runChangeBus: RunChangeBus;
+    mcpStateless: boolean;
+    providerAuthContext: ProviderAuthContext | undefined;
+    mcpHandler: McpHttpHandler;
+};
+export declare function bootstrapStageflowHost(options: StageflowHostOptions): Promise<StageflowHostBootstrap>;

package/dist/server/bootstrap.js

@@ -0,0 +1,56 @@
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+import { createRunStore } from "../runstore/createStore.js";
+import { resolveStageflowContext } from "../project/resolveStageflowContext.js";
+import { findProjectRoot } from "../project/findProjectRoot.js";
+import { RunManager } from "../runtime/runManager.js";
+import { createRunChangeBus, getRunChangeBusFromWrappedStore, isRunStoreWrapped, wrapRunStoreWithChangeBus, } from "../runtime/runChangeBus.js";
+import { createMcpHttpHandler, resolveMcpStateless, } from "../mcp/server.js";
+export async function bootstrapStageflowHost(options) {
+    const invocationCwd = options.cwd ?? process.cwd();
+    const ctx = await resolveStageflowContext(invocationCwd);
+    const cwd = ctx.invocationCwd;
+    const agentDir = options.agentDir ?? getAgentDir();
+    const rootDir = options.rootDir ?? ctx.projectRoot;
+    const isGitProject = options.rootDir !== undefined
+        ? findProjectRoot(rootDir) !== null
+        : ctx.isGitProject;
+    const rawStore = options.store ??
+        createRunStore({ rootDir, kind: options.storeKind });
+    const boundBus = isRunStoreWrapped(rawStore)
+        ? getRunChangeBusFromWrappedStore(rawStore)
+        : undefined;
+    if (boundBus !== undefined &&
+        options.runChangeBus !== undefined &&
+        options.runChangeBus !== boundBus) {
+        throw new Error("runChangeBus does not match the bus already bound to the provided store");
+    }
+    const runChangeBus = options.runChangeBus ?? boundBus ?? createRunChangeBus();
+    const store = wrapRunStoreWithChangeBus(rawStore, runChangeBus);
+    const manager = new RunManager({
+        agent: options.agent,
+        cwd,
+        projectRoot: rootDir,
+        isGitProject,
+        store,
+        maxConcurrent: options.maxConcurrent,
+        operatorCatalog: { cwd, agentDir },
+    });
+    await manager.attachWaitingStages();
+    await manager.reconcileOrphanedStages();
+    const mcpStateless = resolveMcpStateless({
+        mcpStateless: options.mcpStateless,
+    });
+    const mcpHandler = createMcpHttpHandler({ manager, store, cwd, runChangeBus }, { mcpStateless });
+    return {
+        cwd,
+        agentDir,
+        rootDir,
+        isGitProject,
+        store,
+        manager,
+        runChangeBus,
+        mcpStateless,
+        providerAuthContext: options.providerAuthContext,
+        mcpHandler,
+    };
+}

package/dist/server/createHttpHost.d.ts

@@ -0,0 +1,34 @@
+import { type IncomingMessage, type ServerResponse, type Server } from "node:http";
+import type { RunStore } from "../runstore/port.js";
+import type { RunManager } from "../runtime/runManager.js";
+import type { RunChangeBus } from "../runtime/runChangeBus.js";
+import type { StageflowHostBootstrap } from "./bootstrap.js";
+export declare const DEFAULT_PORT = 3847;
+export declare function json(res: ServerResponse, status: number, body: unknown): void;
+export type HttpHostRouteContext = {
+    req: IncomingMessage;
+    res: ServerResponse;
+    url: URL;
+    pathname: string;
+    method: string;
+    json: typeof json;
+    boot: StageflowHostBootstrap;
+};
+export type CreateHttpHostOptions = {
+    boot: StageflowHostBootstrap;
+    host: string;
+    port: number;
+    routes: (ctx: HttpHostRouteContext) => Promise<boolean | void>;
+};
+export type HttpHostEnvelope = {
+    server: Server;
+    port: number;
+    host: string;
+    url: string;
+    mcpUrl: string;
+    manager: RunManager;
+    store: RunStore;
+    runChangeBus: RunChangeBus;
+    mcpStateless: boolean;
+};
+export declare function createHttpHost(options: CreateHttpHostOptions): Promise<HttpHostEnvelope>;

package/dist/server/createHttpHost.js

@@ -0,0 +1,70 @@
+import { createServer, } from "node:http";
+import { localhostHostValidation, localhostOriginValidation, } from "@modelcontextprotocol/node";
+export const DEFAULT_PORT = 3847;
+export function json(res, status, body) {
+    const payload = JSON.stringify(body);
+    res.writeHead(status, {
+        "Content-Type": "application/json; charset=utf-8",
+        "Content-Length": Buffer.byteLength(payload),
+    });
+    res.end(payload);
+}
+export async function createHttpHost(options) {
+    const { boot, host, port, routes } = options;
+    const validateMcpHost = localhostHostValidation();
+    const validateMcpOrigin = localhostOriginValidation();
+    const server = createServer(async (req, res) => {
+        const method = req.method ?? "GET";
+        const url = new URL(req.url ?? "/", `http://${host}:${port}`);
+        const pathname = url.pathname;
+        if (pathname === "/mcp") {
+            if (!validateMcpHost(req, res) || !validateMcpOrigin(req, res)) {
+                return;
+            }
+            try {
+                await boot.mcpHandler.handle(req, res);
+            }
+            catch (err) {
+                if (!res.headersSent) {
+                    res.writeHead(500, { "Content-Type": "text/plain; charset=utf-8" });
+                    res.end(err instanceof Error ? err.message : String(err));
+                }
+            }
+            return;
+        }
+        const handled = await routes({
+            req,
+            res,
+            url,
+            pathname,
+            method,
+            json,
+            boot,
+        });
+        if (handled)
+            return;
+        json(res, 404, { error: "Not found" });
+    });
+    server.requestTimeout = 0;
+    server.on("close", () => {
+        void boot.mcpHandler.close();
+    });
+    await new Promise((resolve, reject) => {
+        server.listen(port, host, () => resolve());
+        server.on("error", reject);
+    });
+    const address = server.address();
+    const boundPort = address && typeof address !== "string" ? address.port : port;
+    const url = `http://${host}:${boundPort}`;
+    return {
+        server,
+        port: boundPort,
+        host,
+        url,
+        mcpUrl: `${url}/mcp`,
+        manager: boot.manager,
+        store: boot.store,
+        runChangeBus: boot.runChangeBus,
+        mcpStateless: boot.mcpStateless,
+    };
+}

package/dist/server/http.d.ts

@@ -1,10 +1,9 @@
-import { type Server } from "node:http";
 import type { AgentPort } from "../agent/port.js";
 import { type ProviderAuthContext } from "../agent/providerAuth.js";
-import { type RunStoreKind } from "../runstore/createStore.js";
+import type { RunStoreKind } from "../runstore/createStore.js";
 import type { RunStore } from "../runstore/port.js";
-import { RunManager } from "../runtime/runManager.js";
-declare const DEFAULT_PORT = 3847;
+import type { RunChangeBus } from "../runtime/runChangeBus.js";
+import { DEFAULT_PORT, type HttpHostEnvelope } from "./createHttpHost.js";
 export type UiServerOptions = {
     agent: AgentPort;
     cwd?: string;
@@ -17,15 +16,9 @@ export type UiServerOptions = {
     uiDistDir?: string;
     maxConcurrent?: number;
     providerAuthContext?: ProviderAuthContext;
+    mcpStateless?: boolean;
+    runChangeBus?: RunChangeBus;
 };
 export declare function defaultUiDistDir(): string;
-export declare function startUiServer(options: UiServerOptions): Promise<{
-    server: Server;
-    port: number;
-    host: string;
-    url: string;
-    mcpUrl: string;
-    manager: RunManager;
-    store: RunStore;
-}>;
+export declare function startUiServer(options: UiServerOptions): Promise<HttpHostEnvelope>;
 export { DEFAULT_PORT };