stageflow 0.8.0 → 0.12.0

package/dist/runtime/cloneSchedule.js

@@ -1,6 +1,8 @@
+import { isNeedTerminalState, predecessorEdges } from "../config/pipelineNeeds.js";
 import { appendCloneInstances, instancesOfDefinition, } from "../runstore/pipelineDagSnapshot.js";
 import { definitionIdForInstance } from "../runstore/stageInstanceId.js";
-import { collectDownstreamClosure, collectDownstreamStageIds } from "./dagTraversal.js";
+import { collectDownstreamStageIds } from "./dagTraversal.js";
+import { activeCohortFromCloneIds, filterJoinInputs, } from "./forkGeneration.js";
 function asDagSnapshot(dag) {
     const snapshot = dag;
     if (Array.isArray(snapshot.stage_ids))
@@ -86,13 +88,75 @@ export function sequentialLaterCloneIds(dag, stageId, completedEnvelopes) {
         return [];
     return instances.slice(i + 1);
 }
+function parentNeedEdge(node, parentId, parentDefId) {
+    return predecessorEdges(node).find((edge) => edge.id === parentId || edge.id === parentDefId);
+}
+function edgeInstancesReady(dag, edge, states) {
+    const instances = definitionInstances(dag, edge.id);
+    if (instances.length === 0)
+        return false;
+    const hasSucceededSibling = instances.some((id) => states.get(id) === "succeeded");
+    return instances.every((id) => {
+        const state = states.get(id);
+        if (!isNeedTerminalState(state))
+            return false;
+        if (edge.on.includes(state))
+            return true;
+        return state === "skipped" && hasSucceededSibling;
+    });
+}
+function shouldSkipForObservedNeed(dag, node, parentId, parentDefId, observed) {
+    const edge = parentNeedEdge(node, parentId, parentDefId);
+    if (!edge)
+        return false;
+    if (edge.on.includes(observed))
+        return false;
+    if (observed === "skipped" &&
+        predecessorEdges(node).length === 1 &&
+        typeof node.needs === "string" &&
+        isCloneInstance(dag, parentId)) {
+        return false;
+    }
+    return true;
+}
+export function skipRejectedNeedDependents(dag, parentId, states, observed, onSkip) {
+    const parentDefId = definitionIdForInstance(asDagSnapshot(dag), parentId);
+    for (const node of dag.nodes) {
+        if (states.get(node.id) !== "pending")
+            continue;
+        if (!shouldSkipForObservedNeed(dag, node, parentId, parentDefId, observed)) {
+            continue;
+        }
+        onSkip(node.id);
+        skipRejectedNeedDependents(dag, node.id, states, "skipped", onSkip);
+    }
+}
+export function failureIsAccepted(dag, failedId, states) {
+    const defId = definitionIdForInstance(asDagSnapshot(dag), failedId);
+    for (const node of dag.nodes) {
+        const edge = parentNeedEdge(node, failedId, defId);
+        if (!edge?.on.includes("failed"))
+            continue;
+        const state = states.get(node.id);
+        if (state !== undefined && state !== "skipped")
+            return true;
+    }
+    return false;
+}
 function joinSuccessorIds(dag, successorId) {
-    return dag.nodes.filter((n) => n.needs === successorId).map((n) => n.id);
+    return dag.nodes
+        .filter((n) => predecessorEdges(n).some((edge) => edge.id === successorId))
+        .map((n) => n.id);
 }
 function joinAndDownstreamIds(dag, cloneStageId) {
     const defId = definitionIdForInstance(asDagSnapshot(dag), cloneStageId);
     const ids = [];
     for (const joinId of joinSuccessorIds(dag, defId)) {
+        const node = dag.nodes.find((n) => n.id === joinId);
+        if (node &&
+            !shouldSkipForObservedNeed(dag, node, cloneStageId, defId, "failed")) {
+            continue;
+        }
         ids.push(joinId);
         for (const desc of collectDownstreamStageIds(dag, joinId)) {
             ids.push(desc);
@@ -100,13 +164,33 @@ function joinAndDownstreamIds(dag, cloneStageId) {
     }
     return ids;
 }
-export function cloneFanoutConflict(dag, envelope) {
+/**
+ * A fresh clone_forks fanout targeting a catalog id that already has
+ * instances is only safe when every existing instance was put back to
+ * `pending` by the current retry cascade (`retryDownstreamIds`) — that means
+ * an ancestor was legitimately retried and this parent re-emitted the same
+ * fanout for reactivation. Instances that are still active/terminal, or
+ * merely `pending` without having gone through a retry reset (e.g. an
+ * envelope processed twice in one live run), are rejected as before.
+ *
+ * Feedback-loop replay is different: the prior cohort was superseded
+ * (`skipped`) and a new generation must mint fresh `~N` ids, so
+ * `forceFreshCloneIds` bypasses the conflict.
+ */
+export function cloneFanoutConflict(dag, envelope, states, retryDownstreamIds, options) {
+    if (options?.forceFreshCloneIds === true)
+        return undefined;
     const snapshot = asDagSnapshot(dag);
     for (const item of envelope.clone_forks ?? []) {
         if (item.action !== "fanout")
             continue;
-        if (definitionInstances(dag, item.successor_id).some((id) => id !== item.successor_id)) {
-            return `clone fan-out for "${item.successor_id}" is already instanced`;
+        const existing = definitionInstances(dag, item.successor_id).filter((id) => id !== item.successor_id);
+        if (existing.length > 0) {
+            const reactivatable = existing.every((id) => states.get(id) === "pending" && retryDownstreamIds.has(id));
+            if (!reactivatable) {
+                return `clone fan-out for "${item.successor_id}" is already instanced`;
+            }
+            continue;
         }
         if (!snapshot.stage_ids.includes(item.successor_id)) {
             return `clone fan-out for "${item.successor_id}" is already instanced`;
@@ -114,6 +198,18 @@ export function cloneFanoutConflict(dag, envelope) {
     }
     return undefined;
 }
+export function nextFreeCloneSuffix(snapshot, catalogId) {
+    const prefix = `${catalogId}~`;
+    let max = 0;
+    for (const id of snapshot.stage_ids) {
+        if (!id.startsWith(prefix))
+            continue;
+        const n = Number(id.slice(prefix.length));
+        if (Number.isInteger(n) && n > max)
+            max = n;
+    }
+    return max + 1;
+}
 export function protectedClonableChildIds(dag, predecessorId, envelope) {
     const ids = new Set();
     for (const childId of dag.childrenOf[predecessorId] ?? []) {
@@ -126,15 +222,35 @@ export function protectedClonableChildIds(dag, predecessorId, envelope) {
     }
     return ids;
 }
+/**
+ * Downstream closure for a retry, resolved through clone-instance ids and
+ * static multi-parent edges. Catalog-level children (e.g. a join stage) are
+ * registered in `childrenOf` under the *definition* id, not per clone
+ * instance, so a plain childrenOf walk starting above a fanout dead-ends at
+ * the instance ids it discovers transitively. Resolving definition-id
+ * children and `predecessorEdges` join successors at every step means
+ * retrying an ancestor or a single clone instance still reaches the join
+ * and everything after it, without pulling in sibling terminals.
+ */
 export function cloneRetryDownstream(dag, stageIds) {
-    const downstream = collectDownstreamClosure(dag, stageIds);
-    for (const id of stageIds) {
-        const defId = definitionIdForInstance(asDagSnapshot(dag), id);
-        for (const child of dag.childrenOf[defId] ?? []) {
+    const snapshot = asDagSnapshot(dag);
+    const downstream = new Set();
+    const seeds = new Set(stageIds);
+    const queue = [...stageIds];
+    while (queue.length > 0) {
+        const id = queue.shift();
+        const defId = definitionIdForInstance(snapshot, id);
+        const children = new Set([
+            ...(dag.childrenOf[id] ?? []),
+            ...(defId !== id ? dag.childrenOf[defId] ?? [] : []),
+            ...joinSuccessorIds(dag, id),
+            ...(defId !== id ? joinSuccessorIds(dag, defId) : []),
+        ]);
+        for (const child of children) {
+            if (downstream.has(child) || seeds.has(child))
+                continue;
             downstream.add(child);
-            for (const desc of collectDownstreamStageIds(dag, child)) {
-                downstream.add(desc);
-            }
+            queue.push(child);
         }
     }
     return downstream;
@@ -160,30 +276,53 @@ export function cloneFailFastSkipIds(dag, stageId, completedEnvelopes) {
     ids.push(...joinAndDownstreamIds(dag, stageId));
     return ids;
 }
-export function cloneScheduleAllowsRun(dag, stageId, states, completedEnvelopes) {
+export function cloneScheduleAllowsRun(dag, stageId, states, completedEnvelopes, options) {
     const node = dag.nodes.find((n) => n.id === stageId);
     if (!node)
         return false;
     if (definitionInstances(dag, stageId).some((id) => id !== stageId)) {
         return false;
     }
-    if (node.needs === null) {
+    const edges = predecessorEdges(node);
+    if (edges.length === 0) {
         return !sequentialPreviousUnsatisfied(dag, stageId, states, completedEnvelopes);
     }
-    const instances = definitionInstances(dag, node.needs);
-    if (instances.length <= 1) {
-        const parentId = instances[0] ?? node.needs;
-        if (states.get(parentId) !== "succeeded")
-            return false;
+    if (edges.length >= 2) {
+        for (const edge of edges) {
+            if (!edgeInstancesReady(dag, edge, states))
+                return false;
+        }
     }
-    else if (!instances.every((id) => states.get(id) === "succeeded")) {
-        return false;
+    else {
+        const parentNeedId = typeof node.needs === "string" && node.needs ? node.needs : edges[0].id;
+        const allInstances = definitionInstances(dag, parentNeedId);
+        const cohort = options?.activeCohortForNeeds?.(parentNeedId) ??
+            activeCohortFromCloneIds(options?.activeCloneIdsForNeeds?.(parentNeedId));
+        if (cohort.kind === "awaiting_mint") {
+            return false;
+        }
+        const instances = filterJoinInputs(allInstances, cohort);
+        if (instances.length <= 1) {
+            const parentId = instances[0] ?? parentNeedId;
+            if (states.get(parentId) !== "succeeded")
+                return false;
+        }
+        else if (cohort.kind === "active") {
+            if (!instances.every((id) => states.get(id) === "succeeded")) {
+                return false;
+            }
+        }
+        else if (!instances.every((id) => states.get(id) === "succeeded" || states.get(id) === "skipped")) {
+            return false;
+        }
     }
     return !sequentialPreviousUnsatisfied(dag, stageId, states, completedEnvelopes);
 }
-export function applyCloneForksToSchedule(dag, predecessorId, envelope, states) {
+export function applyCloneForksToSchedule(dag, predecessorId, envelope, states, options) {
     let current = asDagSnapshot(dag);
     const skippedIds = [];
+    const mintedBySuccessor = new Map();
+    const forceFresh = options?.forceFreshCloneIds === true;
     const skipPending = (stageId) => {
         if (states.get(stageId) === "pending") {
             states.set(stageId, "skipped");
@@ -193,15 +332,55 @@ export function applyCloneForksToSchedule(dag, predecessorId, envelope, states)
     for (const item of envelope.clone_forks ?? []) {
         if (item.action === "skip") {
             skipPending(item.successor_id);
-            for (const desc of collectDownstreamStageIds(current, item.successor_id)) {
-                skipPending(desc);
-            }
+            skipRejectedNeedDependents(current, item.successor_id, states, "skipped", skipPending);
             continue;
         }
         if (item.action !== "fanout")
             continue;
-        const existing = definitionInstances(current, item.successor_id);
-        if (existing.some((id) => id !== item.successor_id)) {
+        const existing = definitionInstances(current, item.successor_id).filter((id) => id !== item.successor_id);
+        if (forceFresh) {
+            const { snapshot, instanceIds } = appendCloneInstances(current, {
+                catalogId: item.successor_id,
+                predecessorId,
+                count: item.clones.length,
+                startAt: nextFreeCloneSuffix(current, item.successor_id),
+            });
+            current = snapshot;
+            states.delete(item.successor_id);
+            for (const id of instanceIds) {
+                states.set(id, "pending");
+            }
+            mintedBySuccessor.set(item.successor_id, instanceIds);
+            continue;
+        }
+        if (existing.length > 0) {
+            // Reactivating instances a retry cascade reset to `pending` (see
+            // cloneFanoutConflict), which only allows this fanout through when
+            // *every* existing instance is pending. Anything short of that —
+            // a normal resume re-apply of an already-processed fanout, or only
+            // one sibling out of a cohort being retried directly — must stay a
+            // no-op here: growing/shrinking the cohort in those cases would mint
+            // or skip instances no retry actually asked for.
+            const allPending = existing.every((id) => states.get(id) === "pending");
+            if (!allPending)
+                continue;
+            const desired = item.clones.length;
+            for (const id of existing.slice(desired)) {
+                skipPending(id);
+                skipRejectedNeedDependents(current, id, states, "skipped", skipPending);
+            }
+            if (desired > existing.length) {
+                const { snapshot, instanceIds } = appendCloneInstances(current, {
+                    catalogId: item.successor_id,
+                    predecessorId,
+                    count: desired - existing.length,
+                    startAt: nextFreeCloneSuffix(current, item.successor_id),
+                });
+                current = snapshot;
+                for (const id of instanceIds) {
+                    states.set(id, "pending");
+                }
+            }
             continue;
         }
         const { snapshot, instanceIds } = appendCloneInstances(current, {
@@ -214,8 +393,13 @@ export function applyCloneForksToSchedule(dag, predecessorId, envelope, states)
         for (const id of instanceIds) {
             states.set(id, "pending");
         }
+        mintedBySuccessor.set(item.successor_id, instanceIds);
     }
-    return { dag: current, skippedIds };
+    return {
+        dag: current,
+        skippedIds,
+        ...(mintedBySuccessor.size > 0 ? { mintedBySuccessor } : {}),
+    };
 }
 export function applyCloneForksFromEnvelopes(dag, states, completedEnvelopes) {
     let current = asDagSnapshot(dag);

package/dist/runtime/completionCheckRunner.d.ts

@@ -0,0 +1,137 @@
+import type { CompletionCheck, CompletionContract } from "../types/completion.js";
+import type { StageEnvelope } from "../types/envelope.js";
+import type { StageGateKind } from "../types/stage.js";
+/**
+ * The result of executing a pipeline-owned command. Command execution is a
+ * runtime capability so callers can supply a sandboxed implementation.
+ */
+export type CommandExecution = {
+    exit_code: number | null;
+    stdout: string;
+    stderr: string;
+    timed_out: boolean;
+    error?: string;
+};
+export type CommandExecutionInput = {
+    command: string;
+    cwd: string;
+    timeout_ms?: number;
+};
+export type CommandExecutor = {
+    run(input: CommandExecutionInput): Promise<CommandExecution>;
+};
+/** A completed operator interaction, normalized outside the agent adapter. */
+export type GateDecision = {
+    kind: StageGateKind;
+    status: "accepted" | "rejected" | "answered";
+    prompt_id?: string;
+};
+export type GateDecisionProvider = {
+    listDecisions(kind: StageGateKind): Promise<readonly GateDecision[]>;
+};
+/**
+ * A VCS-neutral checkout baseline. `fingerprint` is owned by the provider:
+ * for example, a blob hash, an mtime/content hash pair, or a Git index value.
+ */
+export type CheckoutSnapshot = {
+    entries: readonly {
+        path: string;
+        fingerprint: string;
+    }[];
+};
+export type CheckoutChange = {
+    path: string;
+    status: "added" | "modified" | "deleted" | "renamed" | "untracked";
+};
+export type CheckoutCapability = {
+    capture(): Promise<CheckoutSnapshot>;
+    changesSince(before: CheckoutSnapshot): Promise<readonly CheckoutChange[]>;
+};
+/** The agent's explicit acknowledgement of every item in one checklist. */
+export type ChecklistAttestation = {
+    check_id: string;
+    items: readonly string[];
+};
+export type CompletionCheckRunnerInput = {
+    contract: CompletionContract;
+    envelope: StageEnvelope;
+    /** The stage's declared payload_schema, when it has one. */
+    payloadSchema?: unknown;
+    /** Absolute attempt artifact directory. */
+    artifactsDir: string;
+    /** Base directory used for a command check without its own cwd. */
+    commandWorkingDirectory?: string;
+    commandExecutor?: CommandExecutor;
+    gates?: GateDecisionProvider;
+    /** Captured before the agent attempt began. */
+    checkoutBefore?: CheckoutSnapshot;
+    checkout?: CheckoutCapability;
+    /** Why the runtime could not establish the required pre-stage baseline. */
+    checkoutError?: string;
+    checklistAttestations?: readonly ChecklistAttestation[];
+    onCheckStart?: (check: CompletionCheck) => Promise<void> | void;
+    onCheckComplete?: (result: CompletionCheckResult) => Promise<void> | void;
+};
+export type CommandCheckEvidence = {
+    kind: "command";
+    command: string;
+    cwd?: string;
+    exit_code: number | null;
+    stdout: string;
+    stderr: string;
+    timed_out: boolean;
+    error?: string;
+};
+export type ArtifactCheckEvidence = {
+    kind: "artifact";
+    path: string;
+    exists: boolean;
+    file_type: "file" | "directory" | "symlink" | "other" | "missing";
+    size_bytes?: number;
+    sha256?: string;
+};
+export type PayloadSchemaCheckEvidence = {
+    kind: "payload_schema";
+    schema_declared: boolean;
+    payload_present: boolean;
+    validation_error?: string;
+};
+export type GateCheckEvidence = {
+    kind: "gate";
+    gate_kind: StageGateKind;
+    decisions: GateDecision[];
+};
+export type CheckoutChangesCheckEvidence = {
+    kind: "checkout_changes";
+    changes: CheckoutChange[];
+    claimed_paths?: string[];
+    missing_claimed_paths?: string[];
+    unclaimed_changed_paths?: string[];
+};
+export type ChecklistCheckEvidence = {
+    kind: "checklist";
+    expected_items: string[];
+    attested_items: string[];
+    missing_items: string[];
+    unexpected_items: string[];
+};
+export type CompletionCheckEvidence = CommandCheckEvidence | ArtifactCheckEvidence | PayloadSchemaCheckEvidence | GateCheckEvidence | CheckoutChangesCheckEvidence | ChecklistCheckEvidence;
+export type CompletionCheckResult = {
+    check_id: string;
+    check_type: CompletionCheck["type"];
+    outcome: "passed" | "failed" | "error";
+    message?: string;
+    evidence: CompletionCheckEvidence;
+};
+export type CompletionVerificationResult = {
+    outcome: "passed" | "failed";
+    checks: CompletionCheckResult[];
+    failed_check_ids: string[];
+};
+/** Default Node implementation; callers may substitute a sandboxed executor. */
+export declare const nodeCommandExecutor: CommandExecutor;
+/**
+ * Run every completion check in declaration order. A failed check never throws
+ * or prevents later checks from producing their own repair evidence.
+ */
+export declare function runCompletionContract(input: CompletionCheckRunnerInput): Promise<CompletionVerificationResult>;