stageflow 0.8.0 → 0.12.0

package/dist/runtime/runManager.js

@@ -4,9 +4,11 @@ import path from "node:path";
 import { normalizeCatalogPath } from "../runstore/normalizeCatalogPath.js";
 import { loadRunContext } from "./resumeReconstruct.js";
 import { loadTaskFromYaml } from "../config/loadTask.js";
+import { deriveStatusFromStages, findUnhandledFailedStage, } from "../runstore/port.js";
 import { PipelineValidationError, startPipeline, } from "./pipelineRunner.js";
-import { resumeRun } from "./pipelineScheduler.js";
-import { RunRetryCoordinator, } from "./runRetryCoordinator.js";
+import { hydrateScheduleFromStore, hydratedScheduleHasRunnableWork, resumeRun, runPipelineDag, } from "./pipelineScheduler.js";
+import { resolveFeedbackLoopDecision, } from "./feedbackLoopDecision.js";
+import { RunRetryCoordinator, readRetryRootWaitTimeoutMs, } from "./runRetryCoordinator.js";
 import { readMaxActiveStageProcesses, readMaxActiveStagesPerRun, readStageExecutionMode, } from "./stageConcurrency.js";
 import { StageProcessLauncher } from "./stageProcessLauncher.js";
 import { INVALID_SLOT_COUNT_MESSAGE, parseSlotCount, projectSettingsContext, readMaxConcurrentFromContext, writeMaxConcurrentToContext, } from "./settingsFile.js";
@@ -17,6 +19,7 @@ import { reconstructAndContinue as resumeReconstructAndContinue } from "./resume
 import { resumeSessionFilePath } from "./stageAttemptContext.js";
 import { resolveStartTaskInput } from "./taskInput.js";
 import { failStageAsInterrupted, syncRunStatusFromStages, } from "./stageRecovery.js";
+import { blocksGenericRetry, manualRecoveryEligibility, readManualRecoveryState, } from "./manualRecoveryState.js";
 const DEFAULT_MAX_CONCURRENT = 3;
 const STARTUP_RECONCILE_REASON = "process_interrupted: no active worker (server restart)";
 const OPERATOR_ABANDON_REASON = "process_interrupted: operator abandoned stage";
@@ -45,6 +48,8 @@ export class RunManager {
     provisionalIds = new Set();
     resumeInFlight = new Set();
     retryInFlight = new Set();
+    retryStartOwner = new Map();
+    retryStartWaiters = new Map();
     trackingGeneration = 0;
     maxConcurrent;
     maxActiveStagesPerRun;
@@ -64,6 +69,7 @@ export class RunManager {
         },
         onOrchestrationStarted: (runId, promise) => {
             this.registerResumeUntrack(runId, promise);
+            this.notifyRetryStartWaiters(runId);
         },
         rollbackStartTracking: async (runId, state) => {
             if (state.insertedForResume) {
@@ -265,6 +271,26 @@ export class RunManager {
         }
         return { reconciled };
     }
+    async resumeStalledSchedules() {
+        const resumed = [];
+        const runs = await this.options.store.listRuns();
+        for (const summary of runs) {
+            if (summary.status !== "running")
+                continue;
+            const runId = summary.run_id;
+            if (this.active.has(runId))
+                continue;
+            try {
+                if (await this.tryResumeStalledSchedule(runId)) {
+                    resumed.push({ runId });
+                }
+            }
+            catch (err) {
+                console.error(`resumeStalledSchedules: failed to resume ${runId}: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        return resumed;
+    }
     async abandonStage(runId, stageId) {
         let detail;
         try {
@@ -389,6 +415,81 @@ export class RunManager {
         return this.reserveAndStartPipeline(taskYaml, pipeline, `run ${runId} task`, rerunCwd, undefined, undefined, undefined, rerunProjectRoot);
     }
     async retryStage(runId, stageId) {
+        return this.retryStageInternal(runId, stageId, true);
+    }
+    async recoverManualStage(runId, stageId, guidance) {
+        return this.recoverManualStageInternal(runId, stageId, guidance, true);
+    }
+    async recoverManualStageUntilStop(runId, stageId, guidance) {
+        const retryKey = waitKey(runId, stageId);
+        const retried = await this.recoverManualStageInternal(runId, stageId, guidance, false);
+        if (!retried.ok)
+            return retried;
+        try {
+            if (retried.done === undefined) {
+                return {
+                    ok: false,
+                    reason: `Recovery orchestration did not start for run ${runId} stage ${stageId}`,
+                    status: 500,
+                };
+            }
+            return { ok: true, pipeline: await retried.done };
+        }
+        finally {
+            this.retryInFlight.delete(retryKey);
+        }
+    }
+    async recoverManualStageInternal(runId, stageId, guidance, awaitRoot) {
+        if (guidance !== undefined && guidance.trim().length > 4_000) {
+            return {
+                ok: false,
+                reason: "Manual recovery guidance must be 4,000 characters or fewer",
+                status: 400,
+            };
+        }
+        const eligibility = await this.manualRecoveryEligibility(runId, stageId);
+        if (!eligibility.ok)
+            return eligibility;
+        return this.retryStageInternal(runId, stageId, awaitRoot, {
+            manualRecovery: true,
+            beforeAttemptStart: async (attempt) => {
+                await this.options.store.appendStageEvent(runId, stageId, {
+                    event: "manual_recovery_requested",
+                    ...(guidance !== undefined && guidance.trim() !== ""
+                        ? { guidance: guidance.trim() }
+                        : {}),
+                }, { attempt });
+            },
+        });
+    }
+    async stopManualRecovery(runId, stageId) {
+        const eligibility = await this.manualRecoveryEligibility(runId, stageId);
+        if (!eligibility.ok)
+            return eligibility;
+        await this.options.store.appendStageEvent(runId, stageId, { event: "manual_recovery_stopped" }, { attempt: eligibility.failedAttempt });
+        return { ok: true, runId, stageId };
+    }
+    async retryStageUntilStop(runId, stageId) {
+        const retryKey = waitKey(runId, stageId);
+        const retried = await this.retryStageInternal(runId, stageId, false);
+        if (!retried.ok)
+            return retried;
+        try {
+            if (retried.done === undefined) {
+                return {
+                    ok: false,
+                    reason: `Retry orchestration did not start for run ${runId} stage ${stageId}`,
+                    status: 500,
+                };
+            }
+            const pipeline = await retried.done;
+            return { ok: true, pipeline };
+        }
+        finally {
+            this.retryInFlight.delete(retryKey);
+        }
+    }
+    async retryStageInternal(runId, stageId, awaitRoot, options) {
         const retryKey = waitKey(runId, stageId);
         if (this.retryInFlight.has(retryKey)) {
             return {
@@ -398,10 +499,58 @@ export class RunManager {
             };
         }
         this.retryInFlight.add(retryKey);
+        if (options?.manualRecovery !== true &&
+            await this.blocksGenericRetry(runId, stageId)) {
+            this.retryInFlight.delete(retryKey);
+            return {
+                ok: false,
+                reason: "Stage uses manual recovery; use the manual recovery action instead of retry",
+                status: 409,
+            };
+        }
         try {
-            const orchestrationConflict = this.active.has(runId) &&
+            let orchestrationConflict = this.active.has(runId) &&
                 !this.attachedWaiting.has(waitKey(runId, stageId));
-            return await this.retryCoordinator.retryStage({
+            if (!this.retryCoordinator.isActive(runId)) {
+                const entry = this.active.get(runId);
+                const pending = entry?.done;
+                const generation = entry?.generation;
+                if (orchestrationConflict &&
+                    pending !== undefined &&
+                    (await this.isStartRunWindingDown(runId))) {
+                    if (this.claimRetryStart(runId, stageId)) {
+                        await this.awaitStartRunWindDown(runId, pending, generation);
+                    }
+                    else {
+                        await this.waitForRetryCoordinatorOrOwnerRelease(runId);
+                    }
+                }
+                else if (!this.claimRetryStart(runId, stageId)) {
+                    await this.waitForRetryCoordinatorOrOwnerRelease(runId);
+                }
+                let detail;
+                try {
+                    detail = await this.options.store.readRun(runId);
+                    await this.options.store.readRunMeta(runId);
+                }
+                catch {
+                    this.retryInFlight.delete(retryKey);
+                    return { ok: false, reason: `Run not found: ${runId}`, status: 404 };
+                }
+                orchestrationConflict =
+                    this.active.has(runId) &&
+                        !this.attachedWaiting.has(waitKey(runId, stageId)) &&
+                        !this.retryCoordinator.isActive(runId);
+                if (orchestrationConflict &&
+                    !(await this.isStartRunWindingDown(runId))) {
+                    const hasActiveStage = detail.stages.some((stage) => stage.status === "running" ||
+                        stage.status === "waiting_for_input");
+                    if (!hasActiveStage) {
+                        orchestrationConflict = false;
+                    }
+                }
+            }
+            const result = await this.retryCoordinator.retryStage({
                 runId,
                 stageId,
                 store: this.options.store,
@@ -414,10 +563,153 @@ export class RunManager {
                 hitl: this.hitl,
                 orchestrationConflict,
                 tracking: this.retryTracking,
+                awaitRoot,
+                ...(options?.beforeAttemptStart !== undefined
+                    ? { beforeAttemptStart: options.beforeAttemptStart }
+                    : {}),
             });
+            if (!result.ok || awaitRoot !== false) {
+                this.retryInFlight.delete(retryKey);
+            }
+            return result;
         }
-        finally {
+        catch (err) {
             this.retryInFlight.delete(retryKey);
+            throw err;
+        }
+        finally {
+            this.releaseRetryStart(runId, stageId);
+        }
+    }
+    async isStartRunWindingDown(runId) {
+        try {
+            const detail = await this.options.store.readRun(runId);
+            const meta = await this.options.store.readRunMeta(runId);
+            let hasPending = false;
+            for (const stage of detail.stages) {
+                if (stage.status === "running" ||
+                    stage.status === "waiting_for_input") {
+                    return false;
+                }
+                if (stage.status === "pending")
+                    hasPending = true;
+            }
+            return (findUnhandledFailedStage(detail.stages, meta.pipeline_dag) !==
+                undefined || !hasPending);
+        }
+        catch {
+            return false;
+        }
+    }
+    async awaitStartRunWindDown(runId, pending, generation) {
+        let pendingError;
+        const settled = pending.then(() => undefined, (err) => {
+            pendingError = err;
+        });
+        const timedOut = await this.raceRetryRootTimeout(settled);
+        const after = this.active.get(runId);
+        const leftGeneration = after === undefined || after.generation !== generation;
+        if (pendingError !== undefined && !leftGeneration) {
+            throw pendingError instanceof Error
+                ? pendingError
+                : new Error(String(pendingError));
+        }
+        if (timedOut && !leftGeneration) {
+            return;
+        }
+    }
+    async raceRetryRootTimeout(promise) {
+        const ms = readRetryRootWaitTimeoutMs();
+        let timer;
+        try {
+            return await Promise.race([
+                promise.then(() => false),
+                new Promise((resolve) => {
+                    timer = setTimeout(() => resolve(true), ms);
+                }),
+            ]);
+        }
+        finally {
+            if (timer !== undefined)
+                clearTimeout(timer);
+        }
+    }
+    claimRetryStart(runId, stageId) {
+        const owner = this.retryStartOwner.get(runId);
+        if (owner !== undefined && owner !== stageId)
+            return false;
+        this.retryStartOwner.set(runId, stageId);
+        return true;
+    }
+    releaseRetryStart(runId, stageId) {
+        if (this.retryStartOwner.get(runId) !== stageId)
+            return;
+        this.retryStartOwner.delete(runId);
+        this.notifyRetryStartWaiters(runId);
+    }
+    notifyRetryStartWaiters(runId) {
+        const waiters = this.retryStartWaiters.get(runId);
+        if (waiters === undefined)
+            return;
+        this.retryStartWaiters.delete(runId);
+        for (const resolve of waiters)
+            resolve();
+    }
+    async waitForRetryCoordinatorOrOwnerRelease(runId) {
+        if (this.retryCoordinator.isActive(runId) ||
+            !this.retryStartOwner.has(runId)) {
+            return;
+        }
+        let onReady;
+        const watched = new Promise((resolve) => {
+            const list = this.retryStartWaiters.get(runId) ?? new Set();
+            onReady = () => resolve();
+            list.add(onReady);
+            this.retryStartWaiters.set(runId, list);
+            if (this.retryCoordinator.isActive(runId) ||
+                !this.retryStartOwner.has(runId)) {
+                resolve();
+            }
+        });
+        try {
+            await this.raceRetryRootTimeout(watched);
+        }
+        finally {
+            if (onReady !== undefined) {
+                const list = this.retryStartWaiters.get(runId);
+                if (list !== undefined) {
+                    list.delete(onReady);
+                    if (list.size === 0)
+                        this.retryStartWaiters.delete(runId);
+                }
+            }
+        }
+    }
+    async manualRecoveryEligibility(runId, stageId) {
+        try {
+            const eligibility = manualRecoveryEligibility(await readManualRecoveryState(this.options.store, runId, stageId));
+            if (!eligibility.ok) {
+                return {
+                    ...eligibility,
+                    reason: eligibility.reason === "Stage is not configured for manual recovery"
+                        ? `${eligibility.reason}: ${stageId}`
+                        : eligibility.reason === "Stage not found"
+                            ? `${eligibility.reason}: ${stageId}`
+                            : eligibility.reason,
+                };
+            }
+            return eligibility;
+        }
+        catch {
+            return { ok: false, reason: `Run not found: ${runId}`, status: 404 };
+        }
+    }
+    async blocksGenericRetry(runId, stageId) {
+        try {
+            return blocksGenericRetry(await readManualRecoveryState(this.options.store, runId, stageId));
+        }
+        catch {
+            return false;
         }
     }
     /**
@@ -509,6 +801,158 @@ export class RunManager {
             }
         }
     }
+    /**
+     * Resolve a feedback-loop wait_for_human decision after the scheduler has
+     * exited waiting (host-down / HTTP / CLI). Persist-only resolve, then
+     * resume orchestration for extend/continue.
+     */
+    async decideFeedbackLoop(runId, stageId, input) {
+        const resumeKey = waitKey(runId, stageId);
+        if (this.resumeInFlight.has(resumeKey)) {
+            return {
+                ok: false,
+                reason: `Resume already in progress for run ${runId} stage ${stageId}`,
+                status: 409,
+            };
+        }
+        this.resumeInFlight.add(resumeKey);
+        let detail;
+        try {
+            detail = await this.options.store.readRun(runId);
+        }
+        catch {
+            this.resumeInFlight.delete(resumeKey);
+            return { ok: false, reason: `Run not found: ${runId}`, status: 404 };
+        }
+        const stageSnap = detail.stages.find((s) => s.stage_id === stageId);
+        if (!stageSnap) {
+            this.resumeInFlight.delete(resumeKey);
+            return {
+                ok: false,
+                reason: `Stage not found: ${stageId}`,
+                status: 404,
+            };
+        }
+        const active = detail.active_feedback_loop;
+        if (active === undefined ||
+            active.state !== "waiting_for_human") {
+            this.resumeInFlight.delete(resumeKey);
+            return {
+                ok: false,
+                reason: "no feedback loop is waiting_for_human",
+                status: 409,
+            };
+        }
+        if (active.source_stage_id !== stageId) {
+            this.resumeInFlight.delete(resumeKey);
+            return {
+                ok: false,
+                reason: `stage "${stageId}" is not the feedback loop source (expected ${active.source_stage_id})`,
+                status: 409,
+            };
+        }
+        if (input.loopId !== undefined &&
+            input.loopId !== active.loop_id) {
+            this.resumeInFlight.delete(resumeKey);
+            return {
+                ok: false,
+                reason: `feedback loop not found: ${input.loopId}`,
+                status: 404,
+            };
+        }
+        if (stageSnap.status !== "waiting_for_input") {
+            this.resumeInFlight.delete(resumeKey);
+            return {
+                ok: false,
+                reason: `Stage is not waiting for input (status=${stageSnap.status})`,
+                status: 409,
+            };
+        }
+        const wasActive = this.active.has(runId);
+        const tracked = await this.ensureResumeTracked(runId);
+        if (!tracked.ok) {
+            this.resumeInFlight.delete(resumeKey);
+            return {
+                ok: false,
+                reason: tracked.reason,
+                status: 409,
+            };
+        }
+        const insertedForResume = !wasActive;
+        try {
+            const decided = await resolveFeedbackLoopDecision({
+                store: this.options.store,
+                runId,
+                decision: input.decision,
+                loopId: input.loopId ?? active.loop_id,
+                reason: input.reason,
+            });
+            if (!decided.ok) {
+                if (insertedForResume) {
+                    this.removeActiveEntry(runId, false);
+                }
+                return { ok: false, reason: decided.reason, status: 409 };
+            }
+            if (decided.effect === "abandoned") {
+                if (insertedForResume) {
+                    this.removeActiveEntry(runId, false);
+                }
+                return {
+                    ok: true,
+                    effect: "abandoned",
+                    loopId: decided.loop.loop_id,
+                };
+            }
+            const { meta, task, loaded } = await loadRunContext(this.options.store, runId, this.cwd);
+            const done = runPipelineDag({
+                prepared: {
+                    task,
+                    loaded,
+                    run: {
+                        runId,
+                        workspaceDir: this.options.store.getWorkspaceDir(runId),
+                    },
+                    agent: this.options.agent,
+                    store: this.options.store,
+                    cwd: this.cwd,
+                    projectRoot: this.projectRoot,
+                    checkoutRoot: meta.checkout_root,
+                    hitl: this.hitl,
+                    operatorCatalog: this.options.operatorCatalog,
+                },
+                maxActiveStagesPerRun: this.maxActiveStagesPerRun,
+                executionMode: this.executionMode,
+                stageProcessLauncher: this.stageProcessLauncher,
+            });
+            this.registerResumeUntrack(runId, done);
+            const rest = await done;
+            if (rest.outcome === "failed") {
+                return {
+                    ok: false,
+                    reason: rest.reason ?? "pipeline failed after feedback decision",
+                    status: 500,
+                };
+            }
+            return {
+                ok: true,
+                effect: decided.effect === "extended" ? "extended" : "continued",
+                loopId: decided.loop.loop_id,
+            };
+        }
+        catch (err) {
+            if (insertedForResume) {
+                this.removeActiveEntry(runId, false);
+            }
+            return {
+                ok: false,
+                reason: err instanceof Error ? err.message : String(err),
+                status: 500,
+            };
+        }
+        finally {
+            this.resumeInFlight.delete(resumeKey);
+        }
+    }
     async resumeViaSubprocess(runId, stageId, opaqueAnswer) {
         const store = this.options.store;
         const launcher = this.stageProcessLauncher;
@@ -727,7 +1171,7 @@ export class RunManager {
         this.active.delete(provisionalId);
         this.provisionalIds.delete(provisionalId);
         const generation = ++this.trackingGeneration;
-        this.active.set(runId, { ...entry, generation });
+        this.active.set(runId, { ...entry, generation, done });
         if (entry.checkoutKey !== undefined) {
             this.checkoutLeases.set(entry.checkoutKey, runId);
         }
@@ -735,6 +1179,56 @@ export class RunManager {
             this.untrackIfGeneration(runId, generation);
         });
     }
+    async tryResumeStalledSchedule(runId) {
+        const detail = await this.options.store.readRun(runId);
+        const meta = await this.options.store.readRunMeta(runId);
+        if (detail.stages.some((stage) => stage.status === "running" || stage.status === "waiting_for_input")) {
+            return false;
+        }
+        if (deriveStatusFromStages(detail.stages, meta.pipeline_dag) !== "running") {
+            return false;
+        }
+        const dag = meta.pipeline_dag;
+        if (dag === undefined)
+            return false;
+        const hydrated = await hydrateScheduleFromStore(this.options.store, runId, dag, this.executionMode);
+        if (!hydratedScheduleHasRunnableWork(hydrated))
+            return false;
+        const tracked = await this.ensureResumeTracked(runId);
+        if (!tracked.ok) {
+            console.error(`resumeStalledSchedules: ${tracked.reason}`);
+            return false;
+        }
+        try {
+            const { meta: loadedMeta, task, loaded, workspaceDir } = await loadRunContext(this.options.store, runId, this.cwd);
+            const promise = runPipelineDag({
+                prepared: {
+                    task,
+                    loaded,
+                    run: { runId, workspaceDir },
+                    agent: this.options.agent,
+                    store: this.options.store,
+                    cwd: this.cwd,
+                    projectRoot: this.projectRoot,
+                    checkoutRoot: loadedMeta.checkout_root,
+                    hitl: this.hitl,
+                    operatorCatalog: this.options.operatorCatalog,
+                },
+                maxActiveStagesPerRun: this.maxActiveStagesPerRun,
+                executionMode: this.executionMode,
+                stageProcessLauncher: this.stageProcessLauncher,
+                initialSchedule: hydrated,
+            }).finally(() => {
+                void syncRunStatusFromStages(this.options.store, runId).catch(() => { });
+            });
+            this.registerResumeUntrack(runId, promise);
+            return true;
+        }
+        catch (err) {
+            this.removeActiveEntry(runId, false);
+            throw err;
+        }
+    }
     /**
      * Acquire active tracking + checkout lease for a restart resume before
      * reconstruct starts. Over-max is allowed. Lease held by another run → fail closed.
@@ -778,7 +1272,7 @@ export class RunManager {
         if (entry === undefined)
             return;
         const generation = ++this.trackingGeneration;
-        this.active.set(runId, { ...entry, generation });
+        this.active.set(runId, { ...entry, generation, done });
         void done.finally(() => {
             this.untrackIfGeneration(runId, generation);
         });

package/dist/runtime/runRetryCoordinator.d.ts

@@ -1,5 +1,5 @@
 import type { AgentPort } from "../agent/port.js";
-import type { RunDetail, RunStatus, RunStore } from "../runstore/port.js";
+import { type RunDetail, type RunPipelineDagSnapshot, type RunStatus, type RunStore, type StageSnapshot } from "../runstore/port.js";
 import type { PipelineRunResult } from "./pipelineRunner.js";
 import { type SchedulerPreparedPipeline } from "./pipelineScheduler.js";
 import type { StageExecutionMode } from "./stageConcurrency.js";
@@ -29,6 +29,7 @@ export type RetryStageResult = {
     runId: string;
     stageId: string;
     attemptIndex: number;
+    done?: Promise<PipelineRunResult>;
 } | {
     ok: false;
     reason: string;
@@ -62,9 +63,14 @@ export type RetryStageRequest = {
     hitl: StageHitlController;
     orchestrationConflict: boolean;
     tracking: RetryTrackingPort;
+    awaitRoot?: boolean;
+    beforeAttemptStart?: (attempt: number) => Promise<void>;
 };
+export declare function succeededStageRetryBlocker(stageSnap: StageSnapshot): string | undefined;
 export declare function assertStageRetryEligible(detail: RunDetail, stageId: string, opts?: {
     recoveryActive?: boolean;
+    persistedStatus?: RunStatus;
+    dag?: Pick<RunPipelineDagSnapshot, "nodes"> | null;
 }): {
     ok: true;
 } | {
@@ -72,6 +78,7 @@ export declare function assertStageRetryEligible(detail: RunDetail, stageId: str
     reason: string;
     status: number;
 };
+export declare function readRetryRootWaitTimeoutMs(): number;
 export declare class RunRetryCoordinator {
     private readonly sessions;
     isActive(runId: string): boolean;