stageflow 0.3.0 → 0.8.0

package/dist/runtime/cloneSchedule.d.ts

@@ -0,0 +1,18 @@
+import type { RunPipelineDagSnapshot } from "../runstore/port.js";
+import type { StageEnvelope } from "../types/envelope.js";
+import type { ResolvedPipelineDag } from "../types/pipeline.js";
+import type { StageScheduleState } from "./pipelineScheduler.js";
+export declare function isCloneInstance(dag: ResolvedPipelineDag, stageId: string): boolean;
+export declare function sequentialLaterCloneIds(dag: ResolvedPipelineDag, stageId: string, completedEnvelopes: Map<string, StageEnvelope>): string[];
+export declare function cloneFanoutConflict(dag: ResolvedPipelineDag, envelope: StageEnvelope): string | undefined;
+export declare function protectedClonableChildIds(dag: ResolvedPipelineDag, predecessorId: string, envelope: StageEnvelope): Set<string>;
+export declare function cloneRetryDownstream(dag: ResolvedPipelineDag, stageIds: string[]): Set<string>;
+export declare function cloneFailureContinuesSchedule(dag: ResolvedPipelineDag, stageId: string, completedEnvelopes: Map<string, StageEnvelope>): boolean;
+export declare function cloneFailFastSkipIds(dag: ResolvedPipelineDag, stageId: string, completedEnvelopes: Map<string, StageEnvelope>): string[];
+export declare function cloneScheduleAllowsRun(dag: ResolvedPipelineDag, stageId: string, states: Map<string, StageScheduleState>, completedEnvelopes: Map<string, StageEnvelope>): boolean;
+export type ApplyCloneForksResult = {
+    dag: RunPipelineDagSnapshot;
+    skippedIds: string[];
+};
+export declare function applyCloneForksToSchedule(dag: ResolvedPipelineDag, predecessorId: string, envelope: StageEnvelope, states: Map<string, StageScheduleState>): ApplyCloneForksResult;
+export declare function applyCloneForksFromEnvelopes(dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, completedEnvelopes: Map<string, StageEnvelope>): ApplyCloneForksResult;

package/dist/runtime/cloneSchedule.js

@@ -0,0 +1,233 @@
+import { appendCloneInstances, instancesOfDefinition, } from "../runstore/pipelineDagSnapshot.js";
+import { definitionIdForInstance } from "../runstore/stageInstanceId.js";
+import { collectDownstreamClosure, collectDownstreamStageIds } from "./dagTraversal.js";
+function asDagSnapshot(dag) {
+    const snapshot = dag;
+    if (Array.isArray(snapshot.stage_ids))
+        return snapshot;
+    return {
+        ...dag,
+        stage_ids: dag.nodes.map((n) => n.id),
+    };
+}
+function definitionInstances(dag, catalogId) {
+    const snapshot = dag;
+    if (Array.isArray(snapshot.stage_ids)) {
+        return instancesOfDefinition(snapshot, catalogId);
+    }
+    return dag.nodes.some((n) => n.id === catalogId) ? [catalogId] : [];
+}
+export function isCloneInstance(dag, stageId) {
+    return definitionIdForInstance(asDagSnapshot(dag), stageId) !== stageId;
+}
+function isParallelCloneInstance(dag, stageId, completedEnvelopes) {
+    if (!isCloneInstance(dag, stageId))
+        return false;
+    const defId = definitionIdForInstance(asDagSnapshot(dag), stageId);
+    for (const envelope of completedEnvelopes.values()) {
+        for (const item of envelope.clone_forks ?? []) {
+            if (item.action === "fanout" &&
+                item.mode === "parallel" &&
+                item.successor_id === defId &&
+                definitionInstances(dag, defId).includes(stageId)) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+function isSequentialFanoutSuccessor(completedEnvelopes, successorId) {
+    for (const envelope of completedEnvelopes.values()) {
+        for (const item of envelope.clone_forks ?? []) {
+            if (item.action === "fanout" &&
+                item.mode === "sequential" &&
+                item.successor_id === successorId) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+function isSequentialCloneInstance(dag, stageId, completedEnvelopes) {
+    if (!isCloneInstance(dag, stageId))
+        return false;
+    const defId = definitionIdForInstance(asDagSnapshot(dag), stageId);
+    if (!isSequentialFanoutSuccessor(completedEnvelopes, defId))
+        return false;
+    return definitionInstances(dag, defId).includes(stageId);
+}
+function sequentialInstanceList(dag, stageId, completedEnvelopes) {
+    for (const envelope of completedEnvelopes.values()) {
+        for (const item of envelope.clone_forks ?? []) {
+            if (item.action !== "fanout" || item.mode !== "sequential")
+                continue;
+            const instances = definitionInstances(dag, item.successor_id);
+            if (instances.includes(stageId))
+                return instances;
+        }
+    }
+    return undefined;
+}
+function sequentialPreviousUnsatisfied(dag, stageId, states, completedEnvelopes) {
+    const instances = sequentialInstanceList(dag, stageId, completedEnvelopes);
+    if (instances === undefined)
+        return false;
+    const i = instances.indexOf(stageId);
+    if (i <= 0)
+        return false;
+    return states.get(instances[i - 1]) !== "succeeded";
+}
+export function sequentialLaterCloneIds(dag, stageId, completedEnvelopes) {
+    const instances = sequentialInstanceList(dag, stageId, completedEnvelopes);
+    if (instances === undefined)
+        return [];
+    const i = instances.indexOf(stageId);
+    if (i === -1)
+        return [];
+    return instances.slice(i + 1);
+}
+function joinSuccessorIds(dag, successorId) {
+    return dag.nodes.filter((n) => n.needs === successorId).map((n) => n.id);
+}
+function joinAndDownstreamIds(dag, cloneStageId) {
+    const defId = definitionIdForInstance(asDagSnapshot(dag), cloneStageId);
+    const ids = [];
+    for (const joinId of joinSuccessorIds(dag, defId)) {
+        ids.push(joinId);
+        for (const desc of collectDownstreamStageIds(dag, joinId)) {
+            ids.push(desc);
+        }
+    }
+    return ids;
+}
+export function cloneFanoutConflict(dag, envelope) {
+    const snapshot = asDagSnapshot(dag);
+    for (const item of envelope.clone_forks ?? []) {
+        if (item.action !== "fanout")
+            continue;
+        if (definitionInstances(dag, item.successor_id).some((id) => id !== item.successor_id)) {
+            return `clone fan-out for "${item.successor_id}" is already instanced`;
+        }
+        if (!snapshot.stage_ids.includes(item.successor_id)) {
+            return `clone fan-out for "${item.successor_id}" is already instanced`;
+        }
+    }
+    return undefined;
+}
+export function protectedClonableChildIds(dag, predecessorId, envelope) {
+    const ids = new Set();
+    for (const childId of dag.childrenOf[predecessorId] ?? []) {
+        const child = dag.nodes.find((n) => n.id === childId);
+        if (child?.clonable === true)
+            ids.add(childId);
+    }
+    for (const item of envelope.clone_forks ?? []) {
+        ids.add(item.successor_id);
+    }
+    return ids;
+}
+export function cloneRetryDownstream(dag, stageIds) {
+    const downstream = collectDownstreamClosure(dag, stageIds);
+    for (const id of stageIds) {
+        const defId = definitionIdForInstance(asDagSnapshot(dag), id);
+        for (const child of dag.childrenOf[defId] ?? []) {
+            downstream.add(child);
+            for (const desc of collectDownstreamStageIds(dag, child)) {
+                downstream.add(desc);
+            }
+        }
+    }
+    return downstream;
+}
+export function cloneFailureContinuesSchedule(dag, stageId, completedEnvelopes) {
+    return (isParallelCloneInstance(dag, stageId, completedEnvelopes) ||
+        isSequentialCloneInstance(dag, stageId, completedEnvelopes));
+}
+export function cloneFailFastSkipIds(dag, stageId, completedEnvelopes) {
+    if (isParallelCloneInstance(dag, stageId, completedEnvelopes)) {
+        return joinAndDownstreamIds(dag, stageId);
+    }
+    if (!isSequentialCloneInstance(dag, stageId, completedEnvelopes))
+        return [];
+    const instances = sequentialInstanceList(dag, stageId, completedEnvelopes);
+    if (instances === undefined)
+        return [];
+    const idx = instances.indexOf(stageId);
+    const ids = [];
+    for (const id of instances.slice(idx + 1)) {
+        ids.push(id);
+    }
+    ids.push(...joinAndDownstreamIds(dag, stageId));
+    return ids;
+}
+export function cloneScheduleAllowsRun(dag, stageId, states, completedEnvelopes) {
+    const node = dag.nodes.find((n) => n.id === stageId);
+    if (!node)
+        return false;
+    if (definitionInstances(dag, stageId).some((id) => id !== stageId)) {
+        return false;
+    }
+    if (node.needs === null) {
+        return !sequentialPreviousUnsatisfied(dag, stageId, states, completedEnvelopes);
+    }
+    const instances = definitionInstances(dag, node.needs);
+    if (instances.length <= 1) {
+        const parentId = instances[0] ?? node.needs;
+        if (states.get(parentId) !== "succeeded")
+            return false;
+    }
+    else if (!instances.every((id) => states.get(id) === "succeeded")) {
+        return false;
+    }
+    return !sequentialPreviousUnsatisfied(dag, stageId, states, completedEnvelopes);
+}
+export function applyCloneForksToSchedule(dag, predecessorId, envelope, states) {
+    let current = asDagSnapshot(dag);
+    const skippedIds = [];
+    const skipPending = (stageId) => {
+        if (states.get(stageId) === "pending") {
+            states.set(stageId, "skipped");
+            skippedIds.push(stageId);
+        }
+    };
+    for (const item of envelope.clone_forks ?? []) {
+        if (item.action === "skip") {
+            skipPending(item.successor_id);
+            for (const desc of collectDownstreamStageIds(current, item.successor_id)) {
+                skipPending(desc);
+            }
+            continue;
+        }
+        if (item.action !== "fanout")
+            continue;
+        const existing = definitionInstances(current, item.successor_id);
+        if (existing.some((id) => id !== item.successor_id)) {
+            continue;
+        }
+        const { snapshot, instanceIds } = appendCloneInstances(current, {
+            catalogId: item.successor_id,
+            predecessorId,
+            count: item.clones.length,
+        });
+        current = snapshot;
+        states.delete(item.successor_id);
+        for (const id of instanceIds) {
+            states.set(id, "pending");
+        }
+    }
+    return { dag: current, skippedIds };
+}
+export function applyCloneForksFromEnvelopes(dag, states, completedEnvelopes) {
+    let current = asDagSnapshot(dag);
+    const skippedIds = [];
+    const predecessors = [...current.nodes];
+    for (const node of predecessors) {
+        const envelope = completedEnvelopes.get(node.id);
+        if (!envelope?.clone_forks?.length)
+            continue;
+        const applied = applyCloneForksToSchedule(current, node.id, envelope, states);
+        current = applied.dag;
+        skippedIds.push(...applied.skippedIds);
+    }
+    return { dag: current, skippedIds };
+}

package/dist/runtime/envelopeRouting.d.ts

@@ -2,11 +2,12 @@ import type { RunStore, StageSnapshot } from "../runstore/port.js";
 import type { StageEnvelope } from "../types/envelope.js";
 import type { LoadedPipeline, ResolvedPipelineDag } from "../types/pipeline.js";
 import type { StageConfig } from "../types/stage.js";
-export declare function buildCompletedEnvelopesFromRun(store: RunStore, runId: string, stages?: StageSnapshot[]): Promise<Map<string, StageEnvelope>>;
+export declare function buildCompletedEnvelopesFromRun(store: RunStore, runId: string, stages?: StageSnapshot[], dag?: ResolvedPipelineDag): Promise<Map<string, StageEnvelope>>;
 export declare function buildStageConfigById(loaded: LoadedPipeline): Map<string, StageConfig>;
 export type ResolvePriorEnvelopeResult = {
     ok: true;
     prior: StageEnvelope | null;
+    joinPriors?: StageEnvelope[];
 } | {
     ok: false;
     reason: string;

package/dist/runtime/envelopeRouting.js

@@ -1,15 +1,34 @@
-export async function buildCompletedEnvelopesFromRun(store, runId, stages) {
+import { instancesOfDefinition, } from "../runstore/pipelineDagSnapshot.js";
+import { definitionIdForInstance } from "../runstore/stageInstanceId.js";
+function definitionInstances(dag, catalogId) {
+    const snapshot = dag;
+    if (Array.isArray(snapshot.stage_ids)) {
+        return instancesOfDefinition(snapshot, catalogId);
+    }
+    return dag.nodes.some((n) => n.id === catalogId) ? [catalogId] : [];
+}
+export async function buildCompletedEnvelopesFromRun(store, runId, stages, dag) {
     const completedEnvelopes = new Map();
     const snapshotStages = stages ?? (await store.readRun(runId)).stages;
     for (const snap of snapshotStages) {
-        if (snap.status !== "succeeded")
+        const isCloneInstance = dag !== undefined &&
+            definitionIdForInstance(dag, snap.stage_id) !==
+                snap.stage_id;
+        if (snap.status !== "succeeded" && !(snap.status === "failed" && isCloneInstance)) {
             continue;
+        }
         try {
             const envelope = snap.envelope ?? (await store.readEnvelope(runId, snap.stage_id));
             completedEnvelopes.set(snap.stage_id, envelope);
         }
         catch {
-            // envelope may be missing for corrupt runs; leave map empty
+            if (snap.status === "failed" && isCloneInstance) {
+                completedEnvelopes.set(snap.stage_id, {
+                    status: "failure",
+                    summary: "stage failed",
+                    artifacts: [],
+                });
+            }
         }
     }
     return completedEnvelopes;
@@ -35,15 +54,41 @@ export async function resolvePriorEnvelope(options) {
     if (node.needs === null) {
         return { ok: true, prior: null };
     }
+    const joinInstances = definitionInstances(options.dag, node.needs);
+    if (joinInstances.length > 1) {
+        const joinPriors = [];
+        for (const id of joinInstances) {
+            const fromMap = options.completedEnvelopes.get(id);
+            if (fromMap !== undefined) {
+                joinPriors.push(structuredClone(fromMap));
+                continue;
+            }
+            if (options.store !== undefined && options.runId !== undefined) {
+                try {
+                    const envelope = await options.store.readEnvelope(options.runId, id);
+                    joinPriors.push(structuredClone(envelope));
+                    continue;
+                }
+                catch {
+                    return {
+                        ok: false,
+                        reason: `missing envelope for clone instance "${id}"`,
+                    };
+                }
+            }
+            return {
+                ok: false,
+                reason: `missing envelope for clone instance "${id}"`,
+            };
+        }
+        return { ok: true, prior: null, joinPriors };
+    }
     const parentId = node.needs;
     const fromMap = options.completedEnvelopes.get(parentId);
-    if (fromMap !== undefined) {
-        return { ok: true, prior: structuredClone(fromMap) };
-    }
-    if (options.store !== undefined && options.runId !== undefined) {
+    let parent = fromMap;
+    if (parent === undefined && options.store !== undefined && options.runId !== undefined) {
         try {
-            const envelope = await options.store.readEnvelope(options.runId, parentId);
-            return { ok: true, prior: structuredClone(envelope) };
+            parent = await options.store.readEnvelope(options.runId, parentId);
         }
         catch {
             return {
@@ -52,8 +97,30 @@ export async function resolvePriorEnvelope(options) {
             };
         }
     }
-    return {
-        ok: false,
-        reason: `missing envelope for upstream stage "${parentId}"`,
-    };
+    if (parent === undefined) {
+        return {
+            ok: false,
+            reason: `missing envelope for upstream stage "${parentId}"`,
+        };
+    }
+    for (const item of parent.clone_forks ?? []) {
+        if (item.action === "once" && item.successor_id === options.stageId) {
+            return { ok: true, prior: structuredClone(item.envelope) };
+        }
+        if (item.action === "fanout") {
+            const instanceIds = definitionInstances(options.dag, item.successor_id);
+            const index = instanceIds.indexOf(options.stageId);
+            if (index >= 0) {
+                const clone = item.clones[index];
+                if (clone === undefined) {
+                    return {
+                        ok: false,
+                        reason: `missing clone envelope for instance "${options.stageId}"`,
+                    };
+                }
+                return { ok: true, prior: structuredClone(clone.envelope) };
+            }
+        }
+    }
+    return { ok: true, prior: structuredClone(parent) };
 }

package/dist/runtime/pipelineScheduler.d.ts

@@ -1,8 +1,9 @@
 import type { AgentPort } from "../agent/port.js";
-import type { RunStore, StageSnapshot } from "../runstore/port.js";
+import type { RunPipelineDagSnapshot, RunStore, StageSnapshot } from "../runstore/port.js";
 import type { StageEnvelope } from "../types/envelope.js";
 import type { LoadedPipeline, ResolvedPipelineDag } from "../types/pipeline.js";
 import type { TaskFile } from "../types/task.js";
+import { applyCloneForksFromEnvelopes, applyCloneForksToSchedule } from "./cloneSchedule.js";
 import type { PipelineRunResult } from "./pipelineRunner.js";
 import type { StageProcessLauncher } from "./stageProcessLauncher.js";
 import type { StageExecutionMode } from "./stageConcurrency.js";
@@ -39,13 +40,16 @@ type HasWorkOutsideBranchContext = {
     retryRoots?: ReadonlyMap<string, number>;
     pendingDeltaStageIds?: ReadonlySet<string>;
 };
-type StageScheduleState = "pending" | "active" | "waiting" | "succeeded" | "failed" | "skipped";
+export type StageScheduleState = "pending" | "active" | "waiting" | "succeeded" | "failed" | "skipped";
 export type HydratedSchedule = {
     states: Map<string, StageScheduleState>;
     completedEnvelopes: Map<string, StageEnvelope>;
     schedulingHalted: boolean;
     firstFailureReason: string | undefined;
+    dag?: RunPipelineDagSnapshot;
 };
+export { applyCloneForksFromEnvelopes, applyCloneForksToSchedule, };
+export type { ApplyCloneForksResult } from "./cloneSchedule.js";
 export declare function snapshotToScheduleState(snap: {
     status: StageSnapshot["status"];
 }, overrides: ReadonlyMap<string, StageScheduleState>, stageId: string, executionMode?: StageExecutionMode): StageScheduleState;