stageflow 0.8.0 → 0.12.0

package/dist/runtime/feedbackLoopDecision.js

@@ -0,0 +1,207 @@
+import { terminalizeCurrentFeedbackReplay } from "./feedbackLoopCoordinator.js";
+import { createFeedbackScheduleState } from "./feedbackLoopSchedule.js";
+import { applySendBack, releaseHold, } from "./replayLifecycle.js";
+const DECISION_CONFLICT = "feedback loop decision conflict: no longer waiting_for_human";
+async function clearSourceWaiting(store, runId, sourceStageId, nextStatus, reason) {
+    const latest = await store.getLatestStageExecution(runId, sourceStageId);
+    const attempt = latest?.attempt;
+    const eventOptions = attempt !== undefined ? { attempt } : undefined;
+    if (nextStatus === "failed") {
+        await store.appendStageEvent(runId, sourceStageId, {
+            event: "failed",
+            reason: reason ?? "feedback loop abandoned after max_replays",
+        }, eventOptions);
+    }
+    else {
+        await store.appendStageEvent(runId, sourceStageId, { event: "succeeded" }, eventOptions);
+    }
+    if (latest !== null && latest !== undefined) {
+        await store.updateStageExecution(runId, sourceStageId, latest.attempt, {
+            status: nextStatus,
+            ...(nextStatus === "failed"
+                ? { finished_at: new Date().toISOString() }
+                : {}),
+        });
+    }
+}
+async function resolveSourceNode(store, runId, sourceStageId, schedule) {
+    if (schedule !== undefined) {
+        return {
+            ok: true,
+            dag: schedule.dag,
+            sourceNode: schedule.sourceNode,
+        };
+    }
+    const meta = await store.readRunMeta(runId);
+    const dag = meta.pipeline_dag;
+    if (dag === undefined || dag === null) {
+        return {
+            ok: false,
+            reason: "cannot resolve feedback loop decision without pipeline_dag on run",
+        };
+    }
+    const sourceNode = dag.nodes.find((n) => n.id === sourceStageId);
+    if (sourceNode === undefined || sourceNode.feedback_loop === undefined) {
+        return {
+            ok: false,
+            reason: `source stage "${sourceStageId}" missing feedback_loop policy`,
+        };
+    }
+    return { ok: true, dag, sourceNode };
+}
+export async function resolveFeedbackLoopDecision(options) {
+    const { store, runId, decision, schedule } = options;
+    const loops = await store.listFeedbackLoops(runId);
+    const loop = options.loopId !== undefined
+        ? loops.find((entry) => entry.loop_id === options.loopId)
+        : loops
+            .filter((entry) => entry.state === "waiting_for_human")
+            .sort((a, b) => b.updated_at.localeCompare(a.updated_at))[0];
+    if (loop === undefined) {
+        return {
+            ok: false,
+            reason: options.loopId !== undefined
+                ? `feedback loop not found: ${options.loopId}`
+                : "no feedback loop is waiting_for_human",
+        };
+    }
+    if (loop.state !== "waiting_for_human") {
+        return {
+            ok: false,
+            reason: `feedback loop "${loop.loop_id}" is not waiting_for_human (state=${loop.state})`,
+        };
+    }
+    if (decision === "extend") {
+        const deferred = loop.deferred_send_back;
+        if (deferred === undefined) {
+            return {
+                ok: false,
+                reason: `feedback loop "${loop.loop_id}" has no deferred send_back to extend`,
+            };
+        }
+        const resolved = await resolveSourceNode(store, runId, loop.source_stage_id, schedule);
+        if (!resolved.ok)
+            return resolved;
+        const bumpedPolicy = {
+            ...loop.policy,
+            max_replays: loop.policy.max_replays + 1,
+        };
+        const casOk = await store.updateFeedbackLoop(runId, loop.loop_id, {
+            policy: bumpedPolicy,
+            state: "active",
+        }, { expectedState: "waiting_for_human" });
+        if (!casOk) {
+            return {
+                ok: false,
+                reason: `${DECISION_CONFLICT} (loop=${loop.loop_id})`,
+            };
+        }
+        await clearSourceWaiting(store, runId, loop.source_stage_id, "succeeded");
+        if (schedule === undefined) {
+            const applied = await applySendBack({
+                store,
+                runId,
+                sourceNode: resolved.sourceNode,
+                sourceAttempt: deferred.source_attempt,
+                envelope: deferred.feedback_envelope,
+                schedule: { feedback: createFeedbackScheduleState() },
+                projection: {
+                    dag: resolved.dag,
+                    states: new Map(),
+                    completedEnvelopes: new Map(),
+                },
+            });
+            if (applied.kind === "rejected") {
+                return { ok: false, reason: applied.reason };
+            }
+            if (applied.kind === "waiting_for_human") {
+                return {
+                    ok: false,
+                    reason: "extend still exceeded max_replays after bump",
+                };
+            }
+            const updated = await store.getFeedbackLoop(runId, loop.loop_id);
+            return {
+                ok: true,
+                effect: "extended",
+                loop: updated,
+                replayId: applied.replayId,
+            };
+        }
+        const handle = { feedback: schedule.feedback };
+        const applied = await applySendBack({
+            store,
+            runId,
+            sourceNode: resolved.sourceNode,
+            sourceAttempt: deferred.source_attempt,
+            envelope: deferred.feedback_envelope,
+            schedule: handle,
+            projection: {
+                dag: schedule.dag,
+                states: schedule.states,
+                completedEnvelopes: schedule.completedEnvelopes,
+            },
+        });
+        if (applied.kind === "rejected") {
+            return { ok: false, reason: applied.reason };
+        }
+        if (applied.kind === "waiting_for_human") {
+            return {
+                ok: false,
+                reason: "extend still exceeded max_replays after bump",
+            };
+        }
+        const updated = await store.getFeedbackLoop(runId, loop.loop_id);
+        return {
+            ok: true,
+            effect: "extended",
+            loop: updated,
+            replayId: applied.replayId,
+        };
+    }
+    if (decision === "continue") {
+        const deferred = loop.deferred_send_back;
+        const casOk = await store.updateFeedbackLoop(runId, loop.loop_id, { state: "continued" }, { expectedState: "waiting_for_human" });
+        if (!casOk) {
+            return {
+                ok: false,
+                reason: `${DECISION_CONFLICT} (loop=${loop.loop_id})`,
+            };
+        }
+        await terminalizeCurrentFeedbackReplay(store, runId, loop, "completed");
+        await clearSourceWaiting(store, runId, loop.source_stage_id, "succeeded");
+        if (schedule !== undefined) {
+            schedule.states.set(loop.source_stage_id, "succeeded");
+            if (schedule.completedEnvelopes.get(loop.source_stage_id) === undefined &&
+                deferred !== undefined) {
+                schedule.completedEnvelopes.set(loop.source_stage_id, deferred.feedback_envelope);
+            }
+            releaseHold({ feedback: schedule.feedback });
+        }
+        const updated = await store.getFeedbackLoop(runId, loop.loop_id);
+        return { ok: true, effect: "continued", loop: updated };
+    }
+    const abandonReason = options.reason?.trim() ||
+        `feedback loop abandoned after max_replays (${loop.policy.max_replays})`;
+    const casOk = await store.updateFeedbackLoop(runId, loop.loop_id, { state: "abandoned" }, { expectedState: "waiting_for_human" });
+    if (!casOk) {
+        return {
+            ok: false,
+            reason: `${DECISION_CONFLICT} (loop=${loop.loop_id})`,
+        };
+    }
+    await terminalizeCurrentFeedbackReplay(store, runId, loop, "failed");
+    await clearSourceWaiting(store, runId, loop.source_stage_id, "failed", abandonReason);
+    if (schedule !== undefined) {
+        schedule.states.set(loop.source_stage_id, "failed");
+        releaseHold({ feedback: schedule.feedback });
+    }
+    await store.updateRunStatus(runId, "failed");
+    const updated = await store.getFeedbackLoop(runId, loop.loop_id);
+    return {
+        ok: true,
+        effect: "abandoned",
+        loop: updated,
+        reason: abandonReason,
+    };
+}

package/dist/runtime/feedbackLoopRoute.d.ts

@@ -0,0 +1,5 @@
+import type { ResolvedPipelineDag, ResolvedPipelineStageNode } from "../types/pipeline.js";
+/** Persistent stages on target→source inclusive, forward order (ancestors slice + source). */
+export declare function feedbackRouteStageIds(sourceNode: ResolvedPipelineStageNode, targetId: string): string[];
+/** All DAG descendants of the feedback-loop source (exclusive of source). */
+export declare function postSourceStageIds(dag: ResolvedPipelineDag, sourceId: string): string[];

package/dist/runtime/feedbackLoopRoute.js

@@ -0,0 +1,13 @@
+import { collectDownstreamStageIds } from "./dagTraversal.js";
+/** Persistent stages on target→source inclusive, forward order (ancestors slice + source). */
+export function feedbackRouteStageIds(sourceNode, targetId) {
+    const targetIndex = sourceNode.ancestors.indexOf(targetId);
+    if (targetIndex < 0) {
+        throw new Error(`feedback-loop target "${targetId}" is not an ancestor of "${sourceNode.id}"`);
+    }
+    return [...sourceNode.ancestors.slice(targetIndex), sourceNode.id];
+}
+/** All DAG descendants of the feedback-loop source (exclusive of source). */
+export function postSourceStageIds(dag, sourceId) {
+    return [...collectDownstreamStageIds(dag, sourceId)];
+}

package/dist/runtime/feedbackLoopSchedule.d.ts

@@ -0,0 +1,38 @@
+import type { FeedbackLoopContext, StageSessionMode } from "../agent/port.js";
+import type { StageEnvelope } from "../types/envelope.js";
+import type { ResolvedPipelineDag } from "../types/pipeline.js";
+type StageScheduleState = "pending" | "active" | "waiting" | "succeeded" | "failed" | "skipped";
+export type FeedbackScheduleState = {
+    activeHoldStageIds: Set<string>;
+    contextsByStageId: Map<string, FeedbackLoopContext>;
+    sessionModeByStageId: Map<string, StageSessionMode>;
+    priorAttemptByStageId: Map<string, number>;
+    launchAttemptByStageId: Map<string, number>;
+    activeLoopId?: string;
+    activeReplayId?: string;
+    sourceStageId?: string;
+    /**
+     * Fork-parent → active cohort clone ids. An empty set means the prior
+     * cohort was superseded and a fresh fan-out has not minted yet (joins wait).
+     */
+    activeCloneIdsByForkParent: Map<string, Set<string>>;
+};
+export declare function createFeedbackScheduleState(): FeedbackScheduleState;
+export declare function applyFeedbackSendBackToSchedule(options: {
+    dag: ResolvedPipelineDag;
+    sourceId: string;
+    routeStageIds: string[];
+    states: Map<string, StageScheduleState>;
+    completedEnvelopes: Map<string, StageEnvelope>;
+    feedback: FeedbackScheduleState;
+    loopId: string;
+    replayId: string;
+    contextsByStageId: Map<string, FeedbackLoopContext>;
+    sessionModeByStageId: Map<string, StageSessionMode>;
+    priorAttemptByStageId: Map<string, number>;
+    launchAttemptByStageId: Map<string, number>;
+    supersededCloneIds?: ReadonlySet<string>;
+}): string[];
+export declare function releaseFeedbackHold(feedback: FeedbackScheduleState): void;
+export declare function isFeedbackHeld(feedback: FeedbackScheduleState, stageId: string): boolean;
+export {};

package/dist/runtime/feedbackLoopSchedule.js

@@ -0,0 +1,72 @@
+import { postSourceStageIds } from "./feedbackLoopRoute.js";
+export function createFeedbackScheduleState() {
+    return {
+        activeHoldStageIds: new Set(),
+        contextsByStageId: new Map(),
+        sessionModeByStageId: new Map(),
+        priorAttemptByStageId: new Map(),
+        launchAttemptByStageId: new Map(),
+        activeCloneIdsByForkParent: new Map(),
+    };
+}
+export function applyFeedbackSendBackToSchedule(options) {
+    const { dag, sourceId, routeStageIds, states, completedEnvelopes, feedback, } = options;
+    const routeSet = new Set(routeStageIds);
+    const retiredSkippedIds = [];
+    for (const cloneId of options.supersededCloneIds ?? []) {
+        if (routeSet.has(cloneId))
+            continue;
+        const state = states.get(cloneId);
+        if (state !== "skipped") {
+            states.set(cloneId, "skipped");
+            retiredSkippedIds.push(cloneId);
+        }
+        completedEnvelopes.delete(cloneId);
+        for (const desc of postSourceStageIds(dag, cloneId)) {
+            if (routeSet.has(desc))
+                continue;
+            if (feedback.activeHoldStageIds?.has(desc))
+                continue;
+            const descState = states.get(desc);
+            if (descState === "pending" || descState === "succeeded" || descState === "active") {
+                states.set(desc, "skipped");
+                retiredSkippedIds.push(desc);
+                completedEnvelopes.delete(desc);
+            }
+        }
+    }
+    for (const stageId of routeStageIds) {
+        states.set(stageId, "pending");
+        completedEnvelopes.delete(stageId);
+    }
+    feedback.activeHoldStageIds = new Set(postSourceStageIds(dag, sourceId));
+    feedback.contextsByStageId = new Map(options.contextsByStageId);
+    feedback.sessionModeByStageId = new Map(options.sessionModeByStageId);
+    feedback.priorAttemptByStageId = new Map(options.priorAttemptByStageId);
+    feedback.launchAttemptByStageId = new Map(options.launchAttemptByStageId);
+    feedback.activeLoopId = options.loopId;
+    feedback.activeReplayId = options.replayId;
+    feedback.sourceStageId = sourceId;
+    for (const cloneId of options.supersededCloneIds ?? []) {
+        const node = dag.nodes.find((n) => n.id === cloneId);
+        const forkParentId = node?.needs;
+        if (forkParentId !== undefined && forkParentId !== null) {
+            feedback.activeCloneIdsByForkParent.set(forkParentId, new Set());
+        }
+    }
+    return retiredSkippedIds;
+}
+export function releaseFeedbackHold(feedback) {
+    feedback.activeHoldStageIds.clear();
+    feedback.contextsByStageId.clear();
+    feedback.sessionModeByStageId.clear();
+    feedback.priorAttemptByStageId.clear();
+    feedback.launchAttemptByStageId.clear();
+    feedback.activeLoopId = undefined;
+    feedback.activeReplayId = undefined;
+    feedback.sourceStageId = undefined;
+    feedback.activeCloneIdsByForkParent.clear();
+}
+export function isFeedbackHeld(feedback, stageId) {
+    return feedback.activeHoldStageIds.has(stageId);
+}

package/dist/runtime/forkGeneration.d.ts

@@ -0,0 +1,72 @@
+import type { ForkGenerationRecord, RunStore } from "../runstore/port.js";
+import type { ResolvedPipelineDag } from "../types/pipeline.js";
+/** Persistent route stages that fan out to clonable successors (or already have instances). */
+export declare function routeRelatedForkParentIds(dag: ResolvedPipelineDag, routeStageIds: readonly string[]): string[];
+/** Clone instance ids currently under clonable successors of a fork parent. */
+export declare function cloneInstanceIdsForForkParent(dag: ResolvedPipelineDag, forkParentStageId: string): string[];
+export declare function backfillActiveGenerationIfNeeded(store: RunStore, runId: string, options: {
+    dag: ResolvedPipelineDag;
+    forkParentStageId: string;
+}): Promise<ForkGenerationRecord | undefined>;
+export type ActiveCohort = {
+    kind: "untracked";
+} | {
+    kind: "awaiting_mint";
+} | {
+    kind: "active";
+    cloneIds: ReadonlySet<string>;
+};
+export declare function activeCohortFromCloneIds(activeCloneIds: Set<string> | ReadonlySet<string> | null | undefined): ActiveCohort;
+export declare function cloneIdsFromActiveCohort(cohort: ActiveCohort): Set<string> | null;
+/** Resolve the fork parent that fans out into instances of `needsStageId`. */
+export declare function forkParentForNeedsStage(dag: ResolvedPipelineDag, needsStageId: string): string | null;
+export declare function supersedeActiveGenerations(store: RunStore, runId: string, options: {
+    dag: ResolvedPipelineDag;
+    forkParentStageIds: readonly string[];
+}): Promise<{
+    supersededCloneIds: Set<string>;
+    supersededGenerations: ForkGenerationRecord[];
+}>;
+export declare function retireCohortsForRoute(options: {
+    store: RunStore;
+    runId: string;
+    dag: ResolvedPipelineDag;
+    routeStageIds: readonly string[];
+}): Promise<{
+    supersededCloneIds: ReadonlySet<string>;
+    forkParentIds: string[];
+}>;
+export declare function collectSupersededCloneIdsForRoute(options: {
+    store: RunStore;
+    runId: string;
+    dag: ResolvedPipelineDag;
+    routeStageIds: readonly string[];
+}): Promise<ReadonlySet<string>>;
+export declare function createGenerationForFanout(store: RunStore, runId: string, options: {
+    replayId?: string;
+    forkParentStageId: string;
+    cloneStageIds: string[];
+}): Promise<ForkGenerationRecord>;
+export declare function mintCohortForFanout(options: {
+    store: RunStore;
+    runId: string;
+    replayId?: string;
+    forkParent: string;
+    cloneStageIds: string[];
+}): Promise<ForkGenerationRecord>;
+export declare function activeCloneIdsForParent(store: RunStore, runId: string, forkParentStageId: string): Promise<Set<string> | null>;
+export declare function activeCohortForParent(store: RunStore, runId: string, forkParentStageId: string): Promise<ActiveCohort>;
+export declare function selectActiveInput(options: {
+    store: RunStore;
+    runId: string;
+    dag: ResolvedPipelineDag;
+    needsStageId: string;
+    scheduleOverride?: ReadonlyMap<string, ReadonlySet<string>>;
+}): Promise<ActiveCohort>;
+export declare function filterJoinInputs(allInstanceIds: readonly string[], cohort: ActiveCohort): string[];
+export declare function filterJoinInstanceIds(allInstanceIds: readonly string[], activeCloneIds: Set<string> | null): string[];
+/**
+ * During an active feedback replay, a fork_choice must keep at least one
+ * chosen branch that can still reach the replay source stage.
+ */
+export declare function forkChoicePreservesReplaySource(dag: ResolvedPipelineDag, chosen: ReadonlySet<string>, sourceStageId: string): boolean;