stageflow 0.8.0 → 0.12.0

package/dist/runtime/forkGeneration.js

@@ -0,0 +1,295 @@
+import { randomUUID } from "node:crypto";
+import { instancesOfDefinition } from "../runstore/pipelineDagSnapshot.js";
+import { collectDownstreamStageIds } from "./dagTraversal.js";
+function asDagSnapshot(dag) {
+    const snapshot = dag;
+    if (Array.isArray(snapshot.stage_ids))
+        return snapshot;
+    return {
+        ...dag,
+        stage_ids: dag.nodes.map((n) => n.id),
+    };
+}
+/** Persistent route stages that fan out to clonable successors (or already have instances). */
+export function routeRelatedForkParentIds(dag, routeStageIds) {
+    const routeSet = new Set(routeStageIds);
+    const parents = new Set();
+    const snapshot = asDagSnapshot(dag);
+    for (const stageId of routeStageIds) {
+        for (const childId of dag.childrenOf[stageId] ?? []) {
+            const child = dag.nodes.find((n) => n.id === childId);
+            if (child?.clonable === true) {
+                parents.add(stageId);
+            }
+        }
+    }
+    for (const stageId of routeStageIds) {
+        const node = dag.nodes.find((n) => n.id === stageId);
+        if (node?.clonable === true && node.needs !== null) {
+            parents.add(node.needs);
+        }
+    }
+    for (const node of dag.nodes) {
+        const defId = node.definition_id;
+        if (defId === undefined || defId === node.id)
+            continue;
+        if (node.needs !== null && routeSet.has(node.needs)) {
+            parents.add(node.needs);
+        }
+    }
+    for (const stageId of routeStageIds) {
+        const node = dag.nodes.find((n) => n.id === stageId);
+        if (node?.clonable !== true)
+            continue;
+        const instances = instancesOfDefinition(snapshot, stageId).filter((id) => id !== stageId);
+        if (instances.length > 0 && node.needs !== null) {
+            parents.add(node.needs);
+        }
+    }
+    return [...parents];
+}
+/** Clone instance ids currently under clonable successors of a fork parent. */
+export function cloneInstanceIdsForForkParent(dag, forkParentStageId) {
+    const snapshot = asDagSnapshot(dag);
+    const ids = [];
+    const seen = new Set();
+    for (const childId of dag.childrenOf[forkParentStageId] ?? []) {
+        const child = dag.nodes.find((n) => n.id === childId);
+        if (child?.clonable === true) {
+            for (const id of instancesOfDefinition(snapshot, childId)) {
+                if (id === childId)
+                    continue;
+                if (seen.has(id))
+                    continue;
+                seen.add(id);
+                ids.push(id);
+            }
+            continue;
+        }
+        if (child?.definition_id !== undefined &&
+            child.definition_id !== child.id &&
+            !seen.has(childId)) {
+            seen.add(childId);
+            ids.push(childId);
+        }
+    }
+    for (const node of dag.nodes) {
+        if (node.needs !== forkParentStageId)
+            continue;
+        if (node.definition_id === undefined || node.definition_id === node.id) {
+            continue;
+        }
+        if (seen.has(node.id))
+            continue;
+        seen.add(node.id);
+        ids.push(node.id);
+    }
+    return ids;
+}
+export async function backfillActiveGenerationIfNeeded(store, runId, options) {
+    const { dag, forkParentStageId } = options;
+    const existing = await store.listForkGenerations(runId, {
+        forkParentStageId,
+    });
+    if (existing.some((g) => g.status === "active")) {
+        return existing.find((g) => g.status === "active");
+    }
+    const cloneStageIds = cloneInstanceIdsForForkParent(dag, forkParentStageId);
+    if (cloneStageIds.length === 0)
+        return undefined;
+    const maxNumber = existing.reduce((max, g) => Math.max(max, g.generation_number), 0);
+    return store.createForkGeneration(runId, {
+        generation_id: randomUUID(),
+        fork_parent_stage_id: forkParentStageId,
+        generation_number: maxNumber + 1,
+        clone_stage_ids: cloneStageIds,
+        status: "active",
+    });
+}
+export function activeCohortFromCloneIds(activeCloneIds) {
+    if (activeCloneIds === undefined || activeCloneIds === null) {
+        return { kind: "untracked" };
+    }
+    if (activeCloneIds.size === 0)
+        return { kind: "awaiting_mint" };
+    return { kind: "active", cloneIds: activeCloneIds };
+}
+export function cloneIdsFromActiveCohort(cohort) {
+    if (cohort.kind === "untracked")
+        return null;
+    if (cohort.kind === "awaiting_mint")
+        return new Set();
+    return new Set(cohort.cloneIds);
+}
+/** Resolve the fork parent that fans out into instances of `needsStageId`. */
+export function forkParentForNeedsStage(dag, needsStageId) {
+    const snapshot = asDagSnapshot(dag);
+    const instances = Array.isArray(snapshot.stage_ids)
+        ? instancesOfDefinition(snapshot, needsStageId).filter((id) => id !== needsStageId)
+        : [];
+    if (instances.length === 0)
+        return null;
+    const sample = dag.nodes.find((n) => n.id === instances[0]);
+    const forkParentId = sample?.needs;
+    if (forkParentId === undefined || forkParentId === null)
+        return null;
+    return forkParentId;
+}
+export async function supersedeActiveGenerations(store, runId, options) {
+    const supersededCloneIds = new Set();
+    const supersededGenerations = [];
+    for (const forkParentStageId of options.forkParentStageIds) {
+        await backfillActiveGenerationIfNeeded(store, runId, {
+            dag: options.dag,
+            forkParentStageId,
+        });
+        const gens = await store.listForkGenerations(runId, { forkParentStageId });
+        for (const gen of gens) {
+            if (gen.status !== "active")
+                continue;
+            await store.updateForkGeneration(runId, gen.generation_id, {
+                status: "superseded",
+            });
+            supersededGenerations.push({ ...gen, status: "superseded" });
+            for (const id of gen.clone_stage_ids) {
+                supersededCloneIds.add(id);
+            }
+        }
+    }
+    return { supersededCloneIds, supersededGenerations };
+}
+export async function retireCohortsForRoute(options) {
+    const forkParentIds = routeRelatedForkParentIds(options.dag, options.routeStageIds);
+    const { supersededCloneIds } = await supersedeActiveGenerations(options.store, options.runId, {
+        dag: options.dag,
+        forkParentStageIds: forkParentIds,
+    });
+    return { supersededCloneIds, forkParentIds };
+}
+export async function collectSupersededCloneIdsForRoute(options) {
+    const forkParentIds = routeRelatedForkParentIds(options.dag, options.routeStageIds);
+    const supersededCloneIds = new Set();
+    for (const parentId of forkParentIds) {
+        const gens = await options.store.listForkGenerations(options.runId, {
+            forkParentStageId: parentId,
+        });
+        for (const gen of gens) {
+            if (gen.status !== "superseded")
+                continue;
+            for (const id of gen.clone_stage_ids) {
+                supersededCloneIds.add(id);
+            }
+        }
+    }
+    return supersededCloneIds;
+}
+export async function createGenerationForFanout(store, runId, options) {
+    const existing = await store.listForkGenerations(runId, {
+        forkParentStageId: options.forkParentStageId,
+    });
+    const maxNumber = existing.reduce((max, g) => Math.max(max, g.generation_number), 0);
+    return store.createForkGeneration(runId, {
+        generation_id: randomUUID(),
+        ...(options.replayId !== undefined ? { replay_id: options.replayId } : {}),
+        fork_parent_stage_id: options.forkParentStageId,
+        generation_number: maxNumber + 1,
+        clone_stage_ids: options.cloneStageIds,
+        status: "active",
+    });
+}
+export async function mintCohortForFanout(options) {
+    return createGenerationForFanout(options.store, options.runId, {
+        ...(options.replayId !== undefined ? { replayId: options.replayId } : {}),
+        forkParentStageId: options.forkParent,
+        cloneStageIds: options.cloneStageIds,
+    });
+}
+export async function activeCloneIdsForParent(store, runId, forkParentStageId) {
+    return cloneIdsFromActiveCohort(await activeCohortForParent(store, runId, forkParentStageId));
+}
+export async function activeCohortForParent(store, runId, forkParentStageId) {
+    const gens = await store.listForkGenerations(runId, { forkParentStageId });
+    if (gens.length === 0)
+        return { kind: "untracked" };
+    const active = gens.filter((g) => g.status === "active");
+    if (active.length === 0)
+        return { kind: "awaiting_mint" };
+    const ids = new Set();
+    for (const gen of active) {
+        for (const id of gen.clone_stage_ids) {
+            ids.add(id);
+        }
+    }
+    return { kind: "active", cloneIds: ids };
+}
+export async function selectActiveInput(options) {
+    const forkParentId = forkParentForNeedsStage(options.dag, options.needsStageId);
+    if (forkParentId === null)
+        return { kind: "untracked" };
+    const override = options.scheduleOverride;
+    if (override !== undefined) {
+        if (override.size === 0)
+            return { kind: "untracked" };
+        const tracked = override.get(forkParentId);
+        if (tracked === undefined)
+            return { kind: "untracked" };
+        return activeCohortFromCloneIds(tracked);
+    }
+    return activeCohortForParent(options.store, options.runId, forkParentId);
+}
+export function filterJoinInputs(allInstanceIds, cohort) {
+    if (cohort.kind === "untracked")
+        return [...allInstanceIds];
+    if (cohort.kind === "awaiting_mint")
+        return [];
+    return allInstanceIds.filter((id) => cohort.cloneIds.has(id));
+}
+export function filterJoinInstanceIds(allInstanceIds, activeCloneIds) {
+    return filterJoinInputs(allInstanceIds, activeCohortFromCloneIds(activeCloneIds));
+}
+/**
+ * During an active feedback replay, a fork_choice must keep at least one
+ * chosen branch that can still reach the replay source stage.
+ */
+export function forkChoicePreservesReplaySource(dag, chosen, sourceStageId) {
+    const source = dag.nodes.find((n) => n.id === sourceStageId);
+    if (source === undefined)
+        return false;
+    for (const childId of chosen) {
+        if (childId === sourceStageId)
+            return true;
+        if (source.ancestors.includes(childId))
+            return true;
+        if (collectDownstreamStageIds(dag, childId).has(sourceStageId)) {
+            return true;
+        }
+        const child = dag.nodes.find((n) => n.id === childId);
+        if (child?.clonable === true) {
+            const snapshot = asDagSnapshot(dag);
+            for (const instanceId of instancesOfDefinition(snapshot, childId)) {
+                if (instanceId === childId)
+                    continue;
+                if (instanceId === sourceStageId)
+                    return true;
+                if (source.ancestors.includes(instanceId))
+                    return true;
+                if (collectDownstreamStageIds(dag, instanceId).has(sourceStageId)) {
+                    return true;
+                }
+            }
+            if (source.ancestors.includes(childId))
+                return true;
+            const joinDownstream = dag.nodes.filter((n) => n.needs === childId);
+            for (const join of joinDownstream) {
+                if (join.id === sourceStageId)
+                    return true;
+                if (collectDownstreamStageIds(dag, join.id).has(sourceStageId)) {
+                    return true;
+                }
+                if (source.ancestors.includes(join.id))
+                    return true;
+            }
+        }
+    }
+    return false;
+}

package/dist/runtime/gitCheckoutCapability.d.ts

@@ -0,0 +1,7 @@
+import type { CheckoutCapability } from "./completionCheckRunner.js";
+/**
+ * Git-backed checkout observation. It fingerprints tracked and untracked
+ * files before the agent runs, so pre-existing changes are not attributed to
+ * the stage and content changes do not disappear behind a shared Git status.
+ */
+export declare function createGitCheckoutCapability(root: string): CheckoutCapability;

package/dist/runtime/gitCheckoutCapability.js

@@ -0,0 +1,97 @@
+import { createHash } from "node:crypto";
+import { execFile } from "node:child_process";
+import { lstat, readFile, readlink } from "node:fs/promises";
+import path from "node:path";
+function runGit(root, args) {
+    return new Promise((resolve, reject) => {
+        execFile("git", ["-C", root, ...args], { encoding: "buffer", maxBuffer: 16 * 1024 * 1024 }, (error, stdout, stderr) => {
+            if (error) {
+                const detail = Buffer.isBuffer(stderr)
+                    ? stderr.toString("utf8").trim()
+                    : "";
+                reject(new Error(detail || error.message));
+                return;
+            }
+            resolve(stdout);
+        });
+    });
+}
+function nulPaths(output) {
+    return output.toString("utf8").split("\0").filter(Boolean);
+}
+async function fingerprintFile(root, relativePath) {
+    const target = path.join(root, relativePath);
+    try {
+        const stats = await lstat(target);
+        if (stats.isSymbolicLink()) {
+            const targetPath = await readlink(target);
+            return `link:${createHash("sha256").update(targetPath).digest("hex")}`;
+        }
+        if (!stats.isFile())
+            return `other:${stats.mode}:${stats.size}`;
+        const contents = await readFile(target);
+        return `file:${createHash("sha256").update(contents).digest("hex")}`;
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return "missing";
+        throw error;
+    }
+}
+async function filesAt(root) {
+    const [trackedOutput, untrackedOutput] = await Promise.all([
+        runGit(root, ["ls-files", "-z"]),
+        runGit(root, ["ls-files", "--others", "--exclude-standard", "-z"]),
+    ]);
+    const known = new Map();
+    for (const file of nulPaths(trackedOutput))
+        known.set(file, true);
+    for (const file of nulPaths(untrackedOutput))
+        known.set(file, false);
+    return [...known.entries()]
+        .map(([file, tracked]) => ({ path: file, tracked }))
+        .sort((a, b) => a.path.localeCompare(b.path));
+}
+async function capture(root) {
+    const files = await filesAt(root);
+    return {
+        entries: await Promise.all(files.map(async ({ path: file, tracked }) => ({
+            path: file,
+            fingerprint: `${tracked ? "tracked" : "untracked"}:${await fingerprintFile(root, file)}`,
+        }))),
+    };
+}
+function statusForAdded(fingerprint) {
+    return fingerprint.startsWith("untracked:") ? "untracked" : "added";
+}
+/**
+ * Git-backed checkout observation. It fingerprints tracked and untracked
+ * files before the agent runs, so pre-existing changes are not attributed to
+ * the stage and content changes do not disappear behind a shared Git status.
+ */
+export function createGitCheckoutCapability(root) {
+    return {
+        capture: () => capture(root),
+        async changesSince(before) {
+            const after = await capture(root);
+            const earlier = new Map(before.entries.map((entry) => [entry.path, entry.fingerprint]));
+            const later = new Map(after.entries.map((entry) => [entry.path, entry.fingerprint]));
+            const paths = [...new Set([...earlier.keys(), ...later.keys()])].sort();
+            const changes = [];
+            for (const file of paths) {
+                const previous = earlier.get(file);
+                const current = later.get(file);
+                if (previous === undefined && current !== undefined) {
+                    changes.push({ path: file, status: statusForAdded(current) });
+                }
+                else if (previous !== undefined && current === undefined) {
+                    changes.push({ path: file, status: "deleted" });
+                }
+                else if (previous !== current) {
+                    changes.push({ path: file, status: "modified" });
+                }
+            }
+            return changes;
+        },
+    };
+}

package/dist/runtime/manualRecoveryState.d.ts

@@ -0,0 +1,40 @@
+import type { RunStore, StageSnapshot } from "../runstore/port.js";
+/**
+ * The derived recovery state of one stage. It deliberately separates the
+ * operator policy decision from the orchestration that starts an attempt.
+ */
+export type ManualRecoveryState = {
+    kind: "not_manual";
+} | {
+    kind: "stage_not_found";
+} | {
+    kind: "not_failed";
+    status: StageSnapshot["status"];
+} | {
+    kind: "not_verification_failure";
+} | {
+    kind: "available";
+    failedAttempt: number;
+} | {
+    kind: "stopped";
+    failedAttempt: number;
+};
+export type ManualRecoveryEligibility = {
+    ok: true;
+    failedAttempt: number;
+} | {
+    ok: false;
+    reason: string;
+    status: number;
+};
+/**
+ * Interpret the persisted DAG and stage events for a manual-recovery stage.
+ * Callers decide what to do with that state; this module owns what it means.
+ */
+export declare function readManualRecoveryState(store: RunStore, runId: string, stageId: string): Promise<ManualRecoveryState>;
+export declare function manualRecoveryEligibility(state: ManualRecoveryState): ManualRecoveryEligibility;
+export declare function blocksGenericRetry(state: ManualRecoveryState): boolean;
+export declare function manualRecoveryHistory(state: ManualRecoveryState): {
+    status: "available" | "stopped";
+    failed_attempt: number;
+} | undefined;

package/dist/runtime/manualRecoveryState.js

@@ -0,0 +1,73 @@
+/**
+ * Interpret the persisted DAG and stage events for a manual-recovery stage.
+ * Callers decide what to do with that state; this module owns what it means.
+ */
+export async function readManualRecoveryState(store, runId, stageId) {
+    const [detail, meta, latestExecution] = await Promise.all([
+        store.readRun(runId),
+        store.readRunMeta(runId),
+        store.getLatestStageExecution(runId, stageId),
+    ]);
+    const stage = detail.stages.find((item) => item.stage_id === stageId);
+    if (!stage)
+        return { kind: "stage_not_found" };
+    const recovery = meta.pipeline_dag?.nodes.find((node) => node.id === stageId)?.recovery;
+    if (recovery?.mode !== "manual")
+        return { kind: "not_manual" };
+    if (stage.status !== "failed") {
+        return { kind: "not_failed", status: stage.status };
+    }
+    if (latestExecution === null)
+        return { kind: "not_verification_failure" };
+    const events = await store.listStageEvents(runId, stageId, latestExecution.attempt);
+    if (latestExecution.verification_outcome !== "failed") {
+        return { kind: "not_verification_failure" };
+    }
+    if (events.some((event) => event.event === "manual_recovery_stopped")) {
+        return { kind: "stopped", failedAttempt: latestExecution.attempt };
+    }
+    return { kind: "available", failedAttempt: latestExecution.attempt };
+}
+export function manualRecoveryEligibility(state) {
+    switch (state.kind) {
+        case "available":
+            return { ok: true, failedAttempt: state.failedAttempt };
+        case "not_manual":
+            return {
+                ok: false,
+                reason: "Stage is not configured for manual recovery",
+                status: 409,
+            };
+        case "stage_not_found":
+            return { ok: false, reason: "Stage not found", status: 404 };
+        case "not_failed":
+            return {
+                ok: false,
+                reason: `Stage is not failed (status=${state.status})`,
+                status: 409,
+            };
+        case "not_verification_failure":
+            return {
+                ok: false,
+                reason: "Manual recovery is only available after a completion verification failure",
+                status: 409,
+            };
+        case "stopped":
+            return {
+                ok: false,
+                reason: "Manual recovery has been stopped for this stage",
+                status: 409,
+            };
+    }
+}
+export function blocksGenericRetry(state) {
+    return state.kind === "available" || state.kind === "stopped";
+}
+export function manualRecoveryHistory(state) {
+    if (state.kind !== "available" && state.kind !== "stopped")
+        return undefined;
+    return {
+        status: state.kind,
+        failed_attempt: state.failedAttempt,
+    };
+}

package/dist/runtime/pipelineScheduler.d.ts

@@ -4,6 +4,7 @@ import type { StageEnvelope } from "../types/envelope.js";
 import type { LoadedPipeline, ResolvedPipelineDag } from "../types/pipeline.js";
 import type { TaskFile } from "../types/task.js";
 import { applyCloneForksFromEnvelopes, applyCloneForksToSchedule } from "./cloneSchedule.js";
+import type { FeedbackLoopDecisionInput, ResolveFeedbackLoopDecisionResult } from "./feedbackLoopDecision.js";
 import type { PipelineRunResult } from "./pipelineRunner.js";
 import type { StageProcessLauncher } from "./stageProcessLauncher.js";
 import type { StageExecutionMode } from "./stageConcurrency.js";
@@ -79,6 +80,16 @@ export type RunPipelineDagOptions = {
     mutationQueue?: RetryMutationQueue;
     onLoopTick?: () => void | Promise<void>;
     onRetryRootTerminal?: OnRetryRootTerminal;
+    /**
+     * In-process park for feedback-loop exhausted-limit decisions. When omitted,
+     * the scheduler exits with outcome "waiting" and callers use
+     * resolveFeedbackLoopDecision + resume separately (Phase 6 surfaces).
+     */
+    onFeedbackLoopWaiting?: (ctx: {
+        loopId: string;
+        sourceStageId: string;
+        resolve: (decision: FeedbackLoopDecisionInput) => Promise<ResolveFeedbackLoopDecisionResult>;
+    }) => Promise<void>;
 };
 export type RetryRunOptions = {
     prepared: SchedulerPreparedPipeline;
@@ -92,6 +103,7 @@ export type RetryRunOptions = {
 };
 export declare function retryRun(options: RetryRunOptions): Promise<PipelineRunResult>;
 export declare function applyForkSkipsFromEnvelopes(dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, completedEnvelopes: Map<string, StageEnvelope>): void;
-export declare function applyRetryRootDelta(dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, completedEnvelopes: Map<string, StageEnvelope>, retryRoots: Map<string, number>, stageId: string, attempt: number, clearSchedulingHalt?: () => void): void;
+export declare function applyRetryRootDelta(dag: ResolvedPipelineDag, states: Map<string, StageScheduleState>, completedEnvelopes: Map<string, StageEnvelope>, retryRoots: Map<string, number>, stageId: string, attempt: number, clearSchedulingHalt?: () => void, alreadyLaunchedIds?: ReadonlySet<string>): void;
 export declare function hasWorkOutsideBranch(dag: ResolvedPipelineDag, branchRootId: string, states: Map<string, StageScheduleState>, ctx?: HasWorkOutsideBranchContext): boolean;
+export declare function hydratedScheduleHasRunnableWork(hydrated: HydratedSchedule): boolean;
 export declare function runPipelineDag(options: RunPipelineDagOptions): Promise<PipelineRunResult>;