stageflow 0.9.0 → 0.12.0

package/dist/runtime/replayLifecycle.d.ts

@@ -0,0 +1,107 @@
+import type { FeedbackLoopContext, StageSessionMode } from "../agent/port.js";
+import type { DeferredFeedbackSendBack, FeedbackLoopRecord, RunStore } from "../runstore/port.js";
+import type { StageEnvelope } from "../types/envelope.js";
+import type { ResolvedPipelineDag, ResolvedPipelineStageNode } from "../types/pipeline.js";
+import { terminalizeCurrentFeedbackReplay } from "./feedbackLoopCoordinator.js";
+import { type FeedbackScheduleState } from "./feedbackLoopSchedule.js";
+export { terminalizeCurrentFeedbackReplay };
+type StageScheduleState = "pending" | "active" | "waiting" | "succeeded" | "failed" | "skipped";
+export type ScheduleProjection = {
+    dag: ResolvedPipelineDag;
+    states: Map<string, StageScheduleState>;
+    completedEnvelopes: Map<string, StageEnvelope>;
+};
+export type ReplayApplyResult = {
+    kind: "applied";
+    loopId: string;
+    replayId: string;
+    retiredSkippedStageIds: string[];
+    loop: FeedbackLoopRecord;
+} | {
+    kind: "idempotent";
+    loopId: string;
+    replayId: string;
+    retiredSkippedStageIds: string[];
+    loop: FeedbackLoopRecord;
+} | {
+    kind: "waiting_for_human";
+    loop: FeedbackLoopRecord;
+    deferred: DeferredFeedbackSendBack;
+    sourceStageId: string;
+    sourceAttempt: number;
+} | {
+    kind: "rejected";
+    reason: string;
+};
+export type StageLaunchPrep = {
+    attempt: number;
+    sessionMode?: StageSessionMode;
+    priorAttempt?: number;
+    feedbackLoopContext?: FeedbackLoopContext;
+};
+export type ReplayScheduleHandle = {
+    readonly feedback: FeedbackScheduleState;
+};
+export declare function createReplaySchedule(): ReplayScheduleHandle;
+export declare function isHeld(schedule: ReplayScheduleHandle, stageId: string): boolean;
+export declare function launchFor(schedule: ReplayScheduleHandle, stageId: string): StageLaunchPrep | undefined;
+export declare function launchAttemptFor(schedule: ReplayScheduleHandle, stageId: string): number | undefined;
+export declare function clearLaunchPrep(schedule: ReplayScheduleHandle, stageId: string): void;
+export declare function releaseHold(schedule: ReplayScheduleHandle): void;
+export declare function activeReplayId(schedule: ReplayScheduleHandle): string | undefined;
+export declare function activeSourceStageId(schedule: ReplayScheduleHandle): string | undefined;
+export declare function activeLoopId(schedule: ReplayScheduleHandle): string | undefined;
+export declare function cohortOverrideMap(schedule: ReplayScheduleHandle): ReadonlyMap<string, ReadonlySet<string>>;
+export declare function noteActiveCohort(schedule: ReplayScheduleHandle, forkParent: string, cloneStageIds: readonly string[]): void;
+export declare function applySendBack(options: {
+    store: RunStore;
+    runId: string;
+    sourceNode: ResolvedPipelineStageNode;
+    sourceAttempt: number;
+    envelope: StageEnvelope;
+    schedule: ReplayScheduleHandle;
+    projection: ScheduleProjection;
+}): Promise<ReplayApplyResult>;
+export declare function onRouteStageRunning(options: {
+    store: RunStore;
+    runId: string;
+    schedule: ReplayScheduleHandle;
+    stageId: string;
+}): Promise<void>;
+export declare function onRouteStageSucceeded(options: {
+    store: RunStore;
+    runId: string;
+    schedule: ReplayScheduleHandle;
+    stageId: string;
+    envelope: StageEnvelope;
+}): Promise<void>;
+export declare function onRouteStageFailed(options: {
+    store: RunStore;
+    runId: string;
+    schedule: ReplayScheduleHandle;
+    stageId: string;
+}): Promise<void>;
+export declare function hydrateFromStore(schedule: ReplayScheduleHandle, store: RunStore, runId: string, dag: ResolvedPipelineDag): Promise<{
+    loop: FeedbackLoopRecord;
+    replay: {
+        replay_id: string;
+    };
+} | undefined>;
+export declare function rebindRouteAttempts(schedule: ReplayScheduleHandle, store: RunStore, runId: string, overrides: ReadonlyMap<string, number>): Promise<void>;
+export declare function onContinued(options: {
+    store: RunStore;
+    runId: string;
+    sourceStageId: string;
+    envelope: StageEnvelope;
+    schedule: ReplayScheduleHandle;
+}): Promise<void>;
+export declare function resolveDecision(options: {
+    store: RunStore;
+    runId: string;
+    decision: import("./feedbackLoopDecision.js").FeedbackLoopDecisionKind;
+    loopId?: string;
+    reason?: string;
+    schedule: ReplayScheduleHandle;
+    projection: ScheduleProjection;
+    sourceNode: ResolvedPipelineStageNode;
+}): Promise<import("./feedbackLoopDecision.js").ResolveFeedbackLoopDecisionResult>;

package/dist/runtime/replayLifecycle.js

@@ -0,0 +1,168 @@
+import { acceptFeedbackSendBack, hydrateActiveFeedbackScheduleFromStore, markFeedbackLoopContinued, markFeedbackRouteStageFailed, markFeedbackRouteStageRunning, markFeedbackRouteStageSucceeded, rebindFeedbackRoutePassAttempts, terminalizeCurrentFeedbackReplay, } from "./feedbackLoopCoordinator.js";
+import { postSourceStageIds } from "./feedbackLoopRoute.js";
+import { applyFeedbackSendBackToSchedule, createFeedbackScheduleState, isFeedbackHeld, releaseFeedbackHold, } from "./feedbackLoopSchedule.js";
+export { terminalizeCurrentFeedbackReplay };
+export function createReplaySchedule() {
+    return { feedback: createFeedbackScheduleState() };
+}
+export function isHeld(schedule, stageId) {
+    return isFeedbackHeld(schedule.feedback, stageId);
+}
+export function launchFor(schedule, stageId) {
+    const attempt = schedule.feedback.launchAttemptByStageId.get(stageId);
+    if (attempt === undefined)
+        return undefined;
+    const sessionMode = schedule.feedback.sessionModeByStageId.get(stageId);
+    const priorAttempt = schedule.feedback.priorAttemptByStageId.get(stageId);
+    const feedbackLoopContext = schedule.feedback.contextsByStageId.get(stageId);
+    return {
+        attempt,
+        ...(sessionMode !== undefined ? { sessionMode } : {}),
+        ...(priorAttempt !== undefined ? { priorAttempt } : {}),
+        ...(feedbackLoopContext !== undefined ? { feedbackLoopContext } : {}),
+    };
+}
+export function launchAttemptFor(schedule, stageId) {
+    return schedule.feedback.launchAttemptByStageId.get(stageId);
+}
+export function clearLaunchPrep(schedule, stageId) {
+    schedule.feedback.contextsByStageId.delete(stageId);
+    schedule.feedback.sessionModeByStageId.delete(stageId);
+    schedule.feedback.priorAttemptByStageId.delete(stageId);
+    schedule.feedback.launchAttemptByStageId.delete(stageId);
+}
+export function releaseHold(schedule) {
+    releaseFeedbackHold(schedule.feedback);
+}
+export function activeReplayId(schedule) {
+    return schedule.feedback.activeReplayId;
+}
+export function activeSourceStageId(schedule) {
+    return schedule.feedback.sourceStageId;
+}
+export function activeLoopId(schedule) {
+    return schedule.feedback.activeLoopId;
+}
+export function cohortOverrideMap(schedule) {
+    return schedule.feedback.activeCloneIdsByForkParent;
+}
+export function noteActiveCohort(schedule, forkParent, cloneStageIds) {
+    schedule.feedback.activeCloneIdsByForkParent.set(forkParent, new Set(cloneStageIds));
+}
+function armWaitingHold(schedule, dag, sourceStageId, loopId) {
+    if (schedule.feedback.activeHoldStageIds.size === 0) {
+        schedule.feedback.activeHoldStageIds = new Set(postSourceStageIds(dag, sourceStageId));
+    }
+    schedule.feedback.activeLoopId = loopId;
+    schedule.feedback.sourceStageId = sourceStageId;
+}
+function projectAccepted(schedule, projection, sourceId, accepted) {
+    return applyFeedbackSendBackToSchedule({
+        dag: projection.dag,
+        sourceId,
+        routeStageIds: accepted.routeStageIds,
+        states: projection.states,
+        completedEnvelopes: projection.completedEnvelopes,
+        feedback: schedule.feedback,
+        loopId: accepted.loop.loop_id,
+        replayId: accepted.replay.replay_id,
+        contextsByStageId: accepted.launch.contextsByStageId,
+        sessionModeByStageId: accepted.launch.sessionModeByStageId,
+        priorAttemptByStageId: accepted.launch.priorAttemptByStageId,
+        launchAttemptByStageId: accepted.launch.launchAttemptByStageId,
+        supersededCloneIds: accepted.launch.supersededCloneIds,
+    });
+}
+export async function applySendBack(options) {
+    const accepted = await acceptFeedbackSendBack({
+        store: options.store,
+        runId: options.runId,
+        dag: options.projection.dag,
+        sourceNode: options.sourceNode,
+        sourceAttempt: options.sourceAttempt,
+        envelope: options.envelope,
+    });
+    if (accepted.kind === "rejected") {
+        return { kind: "rejected", reason: accepted.reason };
+    }
+    if (accepted.kind === "waiting_for_human") {
+        armWaitingHold(options.schedule, options.projection.dag, accepted.sourceStageId, accepted.loop.loop_id);
+        options.schedule.feedback.activeReplayId = undefined;
+        return {
+            kind: "waiting_for_human",
+            loop: accepted.loop,
+            deferred: accepted.deferred,
+            sourceStageId: accepted.sourceStageId,
+            sourceAttempt: accepted.sourceAttempt,
+        };
+    }
+    const retiredSkippedStageIds = projectAccepted(options.schedule, options.projection, options.sourceNode.id, accepted);
+    return {
+        kind: accepted.kind === "accepted" ? "applied" : "idempotent",
+        loopId: accepted.loop.loop_id,
+        replayId: accepted.replay.replay_id,
+        retiredSkippedStageIds,
+        loop: accepted.loop,
+    };
+}
+export async function onRouteStageRunning(options) {
+    await markFeedbackRouteStageRunning({
+        store: options.store,
+        runId: options.runId,
+        replayId: options.schedule.feedback.activeReplayId,
+        stageId: options.stageId,
+    });
+}
+export async function onRouteStageSucceeded(options) {
+    await markFeedbackRouteStageSucceeded({
+        store: options.store,
+        runId: options.runId,
+        replayId: options.schedule.feedback.activeReplayId,
+        stageId: options.stageId,
+        envelope: options.envelope,
+    });
+    clearLaunchPrep(options.schedule, options.stageId);
+}
+export async function onRouteStageFailed(options) {
+    await markFeedbackRouteStageFailed(options.store, options.runId, options.schedule.feedback.activeReplayId, options.stageId);
+}
+export async function hydrateFromStore(schedule, store, runId, dag) {
+    return hydrateActiveFeedbackScheduleFromStore(store, runId, dag, schedule.feedback);
+}
+export async function rebindRouteAttempts(schedule, store, runId, overrides) {
+    const replayId = schedule.feedback.activeReplayId;
+    if (replayId === undefined || overrides.size === 0)
+        return;
+    await rebindFeedbackRoutePassAttempts(store, runId, replayId, overrides);
+    for (const [stageId, attempt] of overrides) {
+        if (!schedule.feedback.launchAttemptByStageId.has(stageId))
+            continue;
+        schedule.feedback.launchAttemptByStageId.set(stageId, attempt);
+    }
+}
+export async function onContinued(options) {
+    await markFeedbackLoopContinued({
+        store: options.store,
+        runId: options.runId,
+        sourceStageId: options.sourceStageId,
+        envelope: options.envelope,
+    });
+    releaseHold(options.schedule);
+}
+export async function resolveDecision(options) {
+    const { resolveFeedbackLoopDecision } = await import("./feedbackLoopDecision.js");
+    return resolveFeedbackLoopDecision({
+        store: options.store,
+        runId: options.runId,
+        decision: options.decision,
+        loopId: options.loopId,
+        reason: options.reason,
+        schedule: {
+            dag: options.projection.dag,
+            sourceNode: options.sourceNode,
+            states: options.projection.states,
+            completedEnvelopes: options.projection.completedEnvelopes,
+            feedback: options.schedule.feedback,
+        },
+    });
+}

package/dist/runtime/runManager.d.ts

@@ -1,7 +1,8 @@
 import type { AgentPort, OpaqueAnswer } from "../agent/port.js";
-import type { RunStore } from "../runstore/port.js";
+import { type RunStore } from "../runstore/port.js";
 import type { TaskFile } from "../types/task.js";
 import { type PipelineRunResult } from "./pipelineRunner.js";
+import { type FeedbackLoopDecisionKind } from "./feedbackLoopDecision.js";
 import { type RetryStageResult } from "./runRetryCoordinator.js";
 import { type StageExecutionMode } from "./stageConcurrency.js";
 import { StageProcessLauncher } from "./stageProcessLauncher.js";
@@ -44,6 +45,24 @@ export type AbandonStageResult = {
     reason: string;
     status?: number;
 };
+export type DecideFeedbackLoopResult = {
+    ok: true;
+    effect: "extended" | "continued" | "abandoned";
+    loopId: string;
+} | {
+    ok: false;
+    reason: string;
+    status?: number;
+};
+export type StopManualRecoveryResult = {
+    ok: true;
+    runId: string;
+    stageId: string;
+} | {
+    ok: false;
+    reason: string;
+    status?: number;
+};
 export declare class RunManager {
     private readonly options;
     private readonly active;
@@ -51,6 +70,8 @@ export declare class RunManager {
     private readonly provisionalIds;
     private readonly resumeInFlight;
     private readonly retryInFlight;
+    private readonly retryStartOwner;
+    private readonly retryStartWaiters;
     private trackingGeneration;
     private maxConcurrent;
     private readonly maxActiveStagesPerRun;
@@ -99,6 +120,9 @@ export declare class RunManager {
             reason: string;
         }>;
     }>;
+    resumeStalledSchedules(): Promise<Array<{
+        runId: string;
+    }>>;
     abandonStage(runId: string, stageId: string): Promise<AbandonStageResult>;
     startRun(input: StartTaskInput & {
         pipeline: string;
@@ -111,6 +135,15 @@ export declare class RunManager {
     }): Promise<StartRunResult>;
     rerun(runId: string): Promise<StartRunResult>;
     retryStage(runId: string, stageId: string): Promise<RetryStageResult>;
+    recoverManualStage(runId: string, stageId: string, guidance?: string): Promise<RetryStageResult>;
+    recoverManualStageUntilStop(runId: string, stageId: string, guidance?: string): Promise<{
+        ok: true;
+        pipeline: PipelineRunResult;
+    } | Extract<RetryStageResult, {
+        ok: false;
+    }>>;
+    private recoverManualStageInternal;
+    stopManualRecovery(runId: string, stageId: string): Promise<StopManualRecoveryResult>;
     retryStageUntilStop(runId: string, stageId: string): Promise<{
         ok: true;
         pipeline: PipelineRunResult;
@@ -118,12 +151,31 @@ export declare class RunManager {
         ok: false;
     }>>;
     private retryStageInternal;
+    private isStartRunWindingDown;
+    private awaitStartRunWindDown;
+    private raceRetryRootTimeout;
+    private claimRetryStart;
+    private releaseRetryStart;
+    private notifyRetryStartWaiters;
+    private waitForRetryCoordinatorOrOwnerRelease;
+    private manualRecoveryEligibility;
+    private blocksGenericRetry;
     /**
      * Deliver an operator answer for `(runId, stageId)` (KTD2 / KTD6).
      * Same-process: unparks the live yield loop. After restart: reconstructs
      * the stage session seam and continues that stage (KTD8).
      */
     deliverAnswer(runId: string, stageId: string, opaqueAnswer: OpaqueAnswer): Promise<DeliverAnswerResult>;
+    /**
+     * Resolve a feedback-loop wait_for_human decision after the scheduler has
+     * exited waiting (host-down / HTTP / CLI). Persist-only resolve, then
+     * resume orchestration for extend/continue.
+     */
+    decideFeedbackLoop(runId: string, stageId: string, input: {
+        decision: FeedbackLoopDecisionKind;
+        loopId?: string;
+        reason?: string;
+    }): Promise<DecideFeedbackLoopResult>;
     private resumeViaSubprocess;
     private reconstructAndContinue;
     private reserveAndStartPipeline;
@@ -131,6 +183,7 @@ export declare class RunManager {
     private tryReserve;
     private clearReservation;
     private track;
+    private tryResumeStalledSchedule;
     /**
      * Acquire active tracking + checkout lease for a restart resume before
      * reconstruct starts. Over-max is allowed. Lease held by another run → fail closed.