stageflow 0.18.0 → 0.23.0

package/dist/runtime/reopenSkipped.js

@@ -0,0 +1,56 @@
+import { normalizeForkChoice } from "../envelope/forkChoice.js";
+import { forkCohortMapsFromStore } from "./forkGeneration.js";
+import { joinAllowsRun } from "./joinReadiness.js";
+export function isUnchosenForkChild(dag, stageId, envelopes) {
+    for (const node of dag.nodes) {
+        const envelope = envelopes.get(node.id);
+        if (envelope?.fork_choice === undefined || envelope.fork_choice === null) {
+            continue;
+        }
+        const children = dag.childrenOf[node.id] ?? [];
+        if (!children.includes(stageId))
+            continue;
+        const chosen = normalizeForkChoice(envelope.fork_choice, "stored");
+        if (!chosen.has(stageId))
+            return true;
+    }
+    return false;
+}
+export function eventsShowStarted(events) {
+    return events.some((event) => event.event === "started" || event.event === "resumed");
+}
+export function shouldReopenSkippedStage(dag, stageId, states, envelopes, hasStarted, options) {
+    if (hasStarted)
+        return false;
+    if (options?.supersededCloneIds?.has(stageId))
+        return false;
+    if (states.get(stageId) !== "skipped")
+        return false;
+    if (isUnchosenForkChild(dag, stageId, envelopes))
+        return false;
+    const hypothetical = new Map(states);
+    hypothetical.set(stageId, "pending");
+    return joinAllowsRun(dag, stageId, hypothetical, envelopes);
+}
+export async function persistReopenedStages(store, runId, stageIds, states) {
+    for (const id of stageIds) {
+        if (states.get(id) !== "skipped")
+            continue;
+        await store.appendStageEvent(runId, id, { event: "reopened" });
+        states.set(id, "pending");
+    }
+}
+export async function reopenRunnableSkippedStages(store, runId, dag, states, envelopes) {
+    const { supersededCloneIds } = await forkCohortMapsFromStore(store, runId);
+    const ids = [];
+    for (const [id, state] of states) {
+        if (state !== "skipped")
+            continue;
+        const events = await store.listStageEvents(runId, id);
+        if (!shouldReopenSkippedStage(dag, id, states, envelopes, eventsShowStarted(events), { supersededCloneIds })) {
+            continue;
+        }
+        ids.push(id);
+    }
+    await persistReopenedStages(store, runId, ids, states);
+}

package/dist/runtime/resumeReconstruct.js

@@ -33,12 +33,12 @@ export async function loadRunContext(store, runId, cwd) {
 }
 export async function reconstructAndContinue(ctx) {
     const { runId, stageId, opaqueAnswer, agent, store, cwd } = ctx;
-    const factoryCwd = ctx.factoryCwd ?? cwd;
     try {
         const latestExecution = await store.getLatestStageExecution(runId, stageId);
         const attemptCtx = attemptContext(latestExecution?.attempt ?? 1);
         const attemptOpt = attemptCtx.eventOptions();
         const { meta, task, loaded } = await loadRunContext(store, runId, cwd);
+        const factoryCwd = meta.project_root ?? ctx.factoryCwd ?? cwd;
         const definitionId = definitionIdForInstance(meta.pipeline_dag, stageId);
         const stageIndex = loaded.stages.findIndex((s) => s.id === definitionId);
         if (stageIndex < 0) {
@@ -127,7 +127,7 @@ export async function reconstructAndContinue(ctx) {
                 run: { runId, workspaceDir },
                 agent,
                 store,
-                cwd,
+                cwd: factoryCwd,
                 projectRoot: factoryCwd,
                 checkoutRoot,
                 hitl: ctx.hitl,

package/dist/runtime/resumeTimedOut.d.ts

@@ -0,0 +1,33 @@
+import type { AgentPort } from "../agent/port.js";
+import type { RunStore } from "../runstore/port.js";
+import type { StageExecutionMode } from "./stageConcurrency.js";
+import type { StageProcessLauncher } from "./stageProcessLauncher.js";
+import { StageHitlController } from "./stageHitl.js";
+import { type OperatorCatalog } from "./stageAttemptBootstrap.js";
+export type ResumeTimedOutContext = {
+    runId: string;
+    stageId: string;
+    agent: AgentPort;
+    store: RunStore;
+    hitl: StageHitlController;
+    executionMode: StageExecutionMode;
+    stageProcessLauncher?: StageProcessLauncher;
+    cwd: string;
+    factoryCwd?: string;
+    maxActiveStagesPerRun: number;
+    operatorCatalog?: OperatorCatalog;
+};
+export declare function assertTimedOutStageEligible(status: string, events: ReadonlyArray<{
+    event: string;
+    reason?: string;
+}>): {
+    ok: true;
+} | {
+    ok: false;
+    reason: string;
+    status: 409;
+};
+export declare function reconstructTimedOutAndContinue(ctx: ResumeTimedOutContext): Promise<{
+    ok: boolean;
+    reason?: string;
+}>;

package/dist/runtime/resumeTimedOut.js

@@ -0,0 +1,165 @@
+import { existsSync } from "node:fs";
+import { fakeHitlResumePath } from "../agent/fakeAgent.js";
+import { definitionIdForInstance } from "../runstore/stageInstanceId.js";
+import { attemptSessionPath } from "../runstore/workspaceLayout.js";
+import { WAIT_WITHOUT_WORKER_DISPATCH } from "./answerResume.js";
+import { resumeRun } from "./pipelineScheduler.js";
+import { loadRunContext } from "./resumeReconstruct.js";
+import { isRunStageWaiting, runStage } from "./stageRunner.js";
+import { attemptContext } from "./stageAttemptContext.js";
+import { openStageAttempt, } from "./stageAttemptBootstrap.js";
+import { buildStageRoots, withResolvedAuthPath } from "./stageRoots.js";
+import { isStageTimeoutReason, lastFailedReason, } from "../agent/stageTimeout.js";
+import { deriveExecutionPatchFromEvent } from "../runstore/stageExecution.js";
+export function assertTimedOutStageEligible(status, events) {
+    if (status !== "failed") {
+        return {
+            ok: false,
+            reason: `Stage is not timed out (status=${status})`,
+            status: 409,
+        };
+    }
+    if (!isStageTimeoutReason(lastFailedReason(events))) {
+        return {
+            ok: false,
+            reason: "Stage did not fail due to timeout; use retry to start a new attempt",
+            status: 409,
+        };
+    }
+    return { ok: true };
+}
+export async function reconstructTimedOutAndContinue(ctx) {
+    const { runId, stageId, agent, store, cwd } = ctx;
+    const factoryCwd = ctx.factoryCwd ?? cwd;
+    try {
+        const latestExecution = await store.getLatestStageExecution(runId, stageId);
+        const attempt = latestExecution?.attempt ?? 1;
+        const attemptCtx = attemptContext(attempt);
+        const attemptOpt = attemptCtx.eventOptions();
+        const { meta, task, loaded } = await loadRunContext(store, runId, cwd);
+        const definitionId = definitionIdForInstance(meta.pipeline_dag, stageId);
+        const stageIndex = loaded.stages.findIndex((s) => s.id === definitionId);
+        if (stageIndex < 0) {
+            const reason = `Stage ${stageId} not in pipeline ${meta.pipeline_id}`;
+            await store.appendStageEvent(runId, stageId, { event: "failed", reason }, attemptOpt);
+            await store.updateRunStatus(runId, "failed");
+            return { ok: false, reason };
+        }
+        const stage = loaded.stages[stageIndex];
+        const dag = meta.pipeline_dag ?? loaded.dag;
+        const workspaceDir = store.getWorkspaceDir(runId);
+        const checkoutRoot = meta.checkout_root;
+        const roots = withResolvedAuthPath(buildStageRoots(workspaceDir, stageId, checkoutRoot, attemptCtx), factoryCwd);
+        const sessionFile = attemptSessionPath(workspaceDir, stageId, attempt);
+        const fakeResume = fakeHitlResumePath(roots, stageId);
+        if (!existsSync(sessionFile) && !existsSync(fakeResume)) {
+            return {
+                ok: false,
+                reason: "missing session to resume; use retry to start a new attempt",
+            };
+        }
+        const resumed = { event: "resumed" };
+        await store.appendStageEvent(runId, stageId, resumed, attemptOpt);
+        try {
+            const execution = await store.getStageExecution(runId, stageId, attempt);
+            const patch = deriveExecutionPatchFromEvent(resumed, execution);
+            if (Object.keys(patch).length > 0) {
+                await store.updateStageExecution(runId, stageId, attempt, patch);
+            }
+        }
+        catch {
+            // no execution row yet
+        }
+        await store.updateRunStatus(runId, "running");
+        const opened = await openStageAttempt({
+            agent,
+            store,
+            runId,
+            stage,
+            stageId,
+            task,
+            dag,
+            checkoutRoot,
+            workspaceDir,
+            factoryCwd,
+            attemptCtx,
+            operatorCatalog: ctx.operatorCatalog,
+            sessionMode: "timeout_resume",
+            resumeToken: sessionFile,
+            onActivity: (event) => {
+                void store.appendStageEvent(runId, stageId, event, attemptOpt);
+            },
+        });
+        if (!opened.ok) {
+            await store.appendStageEvent(runId, stageId, { event: "failed", reason: opened.reason }, attemptOpt);
+            await store.updateRunStatus(runId, "failed");
+            return { ok: false, reason: opened.reason };
+        }
+        const result = await runStage({
+            agent,
+            store,
+            runId,
+            stage,
+            stageId,
+            task,
+            dag,
+            checkoutRoot,
+            workspaceDir,
+            hitl: ctx.hitl,
+            skipStarted: true,
+            existingHandle: opened.handle,
+            attemptCtx,
+            factoryCwd,
+            operatorCatalog: ctx.operatorCatalog,
+            sessionMode: "timeout_resume",
+            resumeToken: sessionFile,
+        });
+        if (isRunStageWaiting(result)) {
+            await store.updateRunStatus(runId, "failed");
+            return {
+                ok: false,
+                reason: WAIT_WITHOUT_WORKER_DISPATCH,
+            };
+        }
+        if (!result.ok) {
+            await store.updateRunStatus(runId, "failed");
+            return { ok: false, reason: result.reason };
+        }
+        const rest = await resumeRun({
+            prepared: {
+                task,
+                loaded,
+                run: { runId, workspaceDir },
+                agent,
+                store,
+                cwd,
+                projectRoot: factoryCwd,
+                checkoutRoot,
+                hitl: ctx.hitl,
+                operatorCatalog: ctx.operatorCatalog,
+            },
+            maxActiveStagesPerRun: ctx.maxActiveStagesPerRun,
+            resumeFromStageId: stageId,
+            initialPrior: result.envelope ?? null,
+            executionMode: ctx.executionMode,
+            stageProcessLauncher: ctx.stageProcessLauncher,
+        });
+        if (rest.outcome === "failed") {
+            return { ok: false, reason: rest.reason };
+        }
+        return { ok: true };
+    }
+    catch (err) {
+        const reason = err instanceof Error ? err.message : `timeout resume failed: ${String(err)}`;
+        try {
+            const latestExecution = await store.getLatestStageExecution(runId, stageId);
+            const attemptCtx = attemptContext(latestExecution?.attempt ?? 1);
+            await store.appendStageEvent(runId, stageId, { event: "failed", reason }, attemptCtx.eventOptions());
+            await store.updateRunStatus(runId, "failed");
+        }
+        catch {
+            // ignore secondary failures
+        }
+        return { ok: false, reason };
+    }
+}

package/dist/runtime/runManager.d.ts

@@ -1,4 +1,5 @@
 import type { AgentPort, OpaqueAnswer } from "../agent/port.js";
+import type { InlinePipelineDefinition } from "../types/pipeline.js";
 import { type RunStore } from "../runstore/port.js";
 import type { TaskFile } from "../types/task.js";
 import { type PipelineRunResult } from "./pipelineRunner.js";
@@ -125,7 +126,7 @@ export declare class RunManager {
     }>>;
     abandonStage(runId: string, stageId: string): Promise<AbandonStageResult>;
     startRun(input: StartTaskInput & {
-        pipeline: string;
+        pipeline: string | InlinePipelineDefinition;
         task?: string | TaskFile;
         checkoutOverride?: string;
         skipGates?: boolean;
@@ -135,6 +136,7 @@ export declare class RunManager {
     }): Promise<StartRunResult>;
     rerun(runId: string): Promise<StartRunResult>;
     retryStage(runId: string, stageId: string): Promise<RetryStageResult>;
+    resumeTimedOutStage(runId: string, stageId: string): Promise<RetryStageResult>;
     recoverManualStage(runId: string, stageId: string, guidance?: string): Promise<RetryStageResult>;
     recoverManualStageUntilStop(runId: string, stageId: string, guidance?: string): Promise<{
         ok: true;

package/dist/runtime/runManager.js

@@ -1,8 +1,10 @@
 import { randomUUID } from "node:crypto";
 import { readFile, realpath } from "node:fs/promises";
 import path from "node:path";
+import { findProjectRoot } from "../project/findProjectRoot.js";
 import { normalizeCatalogPath } from "../runstore/normalizeCatalogPath.js";
 import { loadRunContext } from "./resumeReconstruct.js";
+import { assertTimedOutStageEligible, reconstructTimedOutAndContinue, } from "./resumeTimedOut.js";
 import { loadTaskFromYaml } from "../config/loadTask.js";
 import { deriveStatusFromStages, findUnhandledFailedStage, } from "../runstore/port.js";
 import { PipelineValidationError, startPipeline, } from "./pipelineRunner.js";
@@ -11,7 +13,7 @@ import { resolveFeedbackLoopDecision, } from "./feedbackLoopDecision.js";
 import { RunRetryCoordinator, readRetryRootWaitTimeoutMs, } from "./runRetryCoordinator.js";
 import { readMaxActiveStageProcesses, readMaxActiveStagesPerRun, readStageExecutionMode, } from "./stageConcurrency.js";
 import { StageProcessLauncher } from "./stageProcessLauncher.js";
-import { INVALID_SLOT_COUNT_MESSAGE, parseSlotCount, projectSettingsContext, readMaxConcurrentFromContext, writeMaxConcurrentToContext, } from "./settingsFile.js";
+import { INVALID_SLOT_COUNT_MESSAGE, parseSlotCount, readMaxConcurrentFromGlobal, writeMaxConcurrentToGlobal, } from "./settingsFile.js";
 import { StageHitlController, waitKey, } from "./stageHitl.js";
 import { resolveAndValidateCheckout, } from "./stageRoots.js";
 import { orchestrateAnswerResume } from "./answerResume.js";
@@ -93,10 +95,9 @@ export class RunManager {
         this.cwd = options.cwd ?? process.cwd();
         this.projectRoot = options.projectRoot ?? this.cwd;
         this.isGitProject = options.isGitProject ?? false;
-        const settingsCtx = projectSettingsContext(this.cwd, this.projectRoot, this.isGitProject);
         this.maxConcurrent =
             options.maxConcurrent ??
-                readMaxConcurrentFromContext(settingsCtx) ??
+                readMaxConcurrentFromGlobal() ??
                 parseMaxConcurrent(process.env.STAGEFLOW_MAX_CONCURRENT_RUNS);
         this.maxActiveStagesPerRun = readMaxActiveStagesPerRun(process.env, options.maxActiveStagesPerRun);
         this.executionMode = readStageExecutionMode(process.env, options.executionMode);
@@ -118,7 +119,7 @@ export class RunManager {
             throw new Error(INVALID_SLOT_COUNT_MESSAGE);
         }
         this.maxConcurrent = parsed;
-        writeMaxConcurrentToContext(projectSettingsContext(this.cwd, this.projectRoot, this.isGitProject), parsed);
+        writeMaxConcurrentToGlobal(parsed);
         return this.getHealth();
     }
     getActiveCount() {
@@ -349,6 +350,11 @@ export class RunManager {
     }
     async startRun(input) {
         const cwd = this.options.cwd ?? process.cwd();
+        // An inline pipeline has no filesystem anchor to derive a project root from.
+        const derivedProjectRoot = typeof input.pipeline === "string"
+            ? (findProjectRoot(path.dirname(path.resolve(cwd, input.pipeline))) ??
+                this.projectRoot)
+            : this.projectRoot;
         let resolved;
         try {
             resolved = resolveStartTaskInput(input, cwd);
@@ -383,7 +389,7 @@ export class RunManager {
             gitSha: input.gitSha,
             ciPrUrl: input.ciPrUrl,
             ciJobUrl: input.ciJobUrl,
-        }, undefined, resolved.kind === "path" ? resolved.taskPath : undefined);
+        }, derivedProjectRoot, resolved.kind === "path" ? resolved.taskPath : undefined);
     }
     async rerun(runId) {
         const cwd = this.options.cwd ?? process.cwd();
@@ -417,6 +423,82 @@ export class RunManager {
     async retryStage(runId, stageId) {
         return this.retryStageInternal(runId, stageId, true);
     }
+    async resumeTimedOutStage(runId, stageId) {
+        const resumeKey = waitKey(runId, stageId);
+        if (this.resumeInFlight.has(resumeKey) || this.retryInFlight.has(resumeKey)) {
+            return {
+                ok: false,
+                reason: `Resume already in progress for run ${runId} stage ${stageId}`,
+                status: 409,
+            };
+        }
+        this.resumeInFlight.add(resumeKey);
+        let detail;
+        try {
+            detail = await this.options.store.readRun(runId);
+        }
+        catch {
+            this.resumeInFlight.delete(resumeKey);
+            return { ok: false, reason: `Run not found: ${runId}`, status: 404 };
+        }
+        const stageSnap = detail.stages.find((s) => s.stage_id === stageId);
+        if (!stageSnap) {
+            this.resumeInFlight.delete(resumeKey);
+            return { ok: false, reason: `Stage not found: ${stageId}`, status: 404 };
+        }
+        const eligibility = assertTimedOutStageEligible(stageSnap.status, stageSnap.events);
+        if (!eligibility.ok) {
+            this.resumeInFlight.delete(resumeKey);
+            return eligibility;
+        }
+        const latest = await this.options.store.getLatestStageExecution(runId, stageId);
+        const attemptIndex = latest?.attempt ?? 1;
+        const wasActive = this.active.has(runId);
+        const tracked = await this.ensureResumeTracked(runId);
+        if (!tracked.ok) {
+            this.resumeInFlight.delete(resumeKey);
+            return { ok: false, reason: tracked.reason, status: 409 };
+        }
+        const insertedForResume = !wasActive;
+        try {
+            const runProjectRoot = detail.project_root ?? this.projectRoot;
+            const done = reconstructTimedOutAndContinue({
+                runId,
+                stageId,
+                agent: this.options.agent,
+                store: this.options.store,
+                hitl: this.hitl,
+                executionMode: this.executionMode,
+                cwd: runProjectRoot,
+                maxActiveStagesPerRun: this.maxActiveStagesPerRun,
+                factoryCwd: runProjectRoot,
+                ...(this.stageProcessLauncher !== undefined
+                    ? { stageProcessLauncher: this.stageProcessLauncher }
+                    : {}),
+                ...(this.options.operatorCatalog !== undefined
+                    ? { operatorCatalog: this.options.operatorCatalog }
+                    : {}),
+            });
+            this.registerResumeUntrack(runId, done);
+            const outcome = await done;
+            if (!outcome.ok) {
+                if (insertedForResume) {
+                    this.removeActiveEntry(runId, false);
+                }
+                const missingSession = outcome.reason !== undefined &&
+                    outcome.reason.startsWith("missing session to resume");
+                return {
+                    ok: false,
+                    reason: outcome.reason ?? "timeout resume failed",
+                    status: missingSession ? 409 : 500,
+                };
+            }
+            return { ok: true, runId, stageId, attemptIndex };
+        }
+        finally {
+            this.resumeInFlight.delete(resumeKey);
+        }
+    }
     async recoverManualStage(runId, stageId, guidance) {
         return this.recoverManualStageInternal(runId, stageId, guidance, true);
     }
@@ -509,6 +591,7 @@ export class RunManager {
             };
         }
         try {
+            let meta;
             let orchestrationConflict = this.active.has(runId) &&
                 !this.attachedWaiting.has(waitKey(runId, stageId));
             if (!this.retryCoordinator.isActive(runId)) {
@@ -531,7 +614,7 @@ export class RunManager {
                 let detail;
                 try {
                     detail = await this.options.store.readRun(runId);
-                    await this.options.store.readRunMeta(runId);
+                    meta = await this.options.store.readRunMeta(runId);
                 }
                 catch {
                     this.retryInFlight.delete(retryKey);
@@ -550,12 +633,21 @@ export class RunManager {
                     }
                 }
             }
+            if (meta === undefined) {
+                try {
+                    meta = await this.options.store.readRunMeta(runId);
+                }
+                catch {
+                    this.retryInFlight.delete(retryKey);
+                    return { ok: false, reason: `Run not found: ${runId}`, status: 404 };
+                }
+            }
             const result = await this.retryCoordinator.retryStage({
                 runId,
                 stageId,
                 store: this.options.store,
                 agent: this.options.agent,
-                cwd: this.cwd,
+                cwd: meta.project_root ?? this.projectRoot,
                 operatorCatalog: this.options.operatorCatalog,
                 maxActiveStagesPerRun: this.maxActiveStagesPerRun,
                 executionMode: this.executionMode,
@@ -904,6 +996,7 @@ export class RunManager {
                 };
             }
             const { meta, task, loaded } = await loadRunContext(this.options.store, runId, this.cwd);
+            const runProjectRoot = meta.project_root ?? this.projectRoot;
             const done = runPipelineDag({
                 prepared: {
                     task,
@@ -914,8 +1007,8 @@ export class RunManager {
                     },
                     agent: this.options.agent,
                     store: this.options.store,
-                    cwd: this.cwd,
-                    projectRoot: this.projectRoot,
+                    cwd: runProjectRoot,
+                    projectRoot: runProjectRoot,
                     checkoutRoot: meta.checkout_root,
                     hitl: this.hitl,
                     operatorCatalog: this.options.operatorCatalog,
@@ -965,10 +1058,12 @@ export class RunManager {
         const sessionFilePath = resumeSessionFilePath(workspaceDir, stageId, attempt);
         const eventOptions = { attempt };
         try {
+            const runMeta = await store.readRunMeta(runId);
+            const runProjectRoot = runMeta.project_root ?? this.projectRoot;
             const launchResult = await launcher.launch({
                 runId,
                 stageId,
-                rootDir: this.projectRoot,
+                rootDir: runProjectRoot,
                 mode: "resume",
                 resumeAnswer: opaqueAnswer,
                 attempt,
@@ -1003,8 +1098,8 @@ export class RunManager {
                     run: { runId, workspaceDir: store.getWorkspaceDir(runId) },
                     agent: this.options.agent,
                     store,
-                    cwd: this.cwd,
-                    projectRoot: this.projectRoot,
+                    cwd: meta.project_root ?? this.projectRoot,
+                    projectRoot: meta.project_root ?? this.projectRoot,
                     checkoutRoot: meta.checkout_root,
                     hitl: this.hitl,
                     operatorCatalog: this.options.operatorCatalog,
@@ -1201,6 +1296,7 @@ export class RunManager {
             return false;
         }
         try {
+            const runProjectRoot = meta.project_root ?? this.projectRoot;
             const { meta: loadedMeta, task, loaded, workspaceDir } = await loadRunContext(this.options.store, runId, this.cwd);
             const promise = runPipelineDag({
                 prepared: {
@@ -1209,8 +1305,8 @@ export class RunManager {
                     run: { runId, workspaceDir },
                     agent: this.options.agent,
                     store: this.options.store,
-                    cwd: this.cwd,
-                    projectRoot: this.projectRoot,
+                    cwd: runProjectRoot,
+                    projectRoot: runProjectRoot,
                     checkoutRoot: loadedMeta.checkout_root,
                     hitl: this.hitl,
                     operatorCatalog: this.options.operatorCatalog,

package/dist/runtime/settingsFile.d.ts

@@ -23,6 +23,8 @@ export declare function readMaxConcurrentFromContext(ctx: ProjectContext): numbe
 export declare function readMaxConcurrentFromFile(cwd: string): number | undefined;
 export declare function writeMaxConcurrentToContext(ctx: ProjectContext, maxConcurrent: number): void;
 export declare function writeMaxConcurrentToFile(cwd: string, maxConcurrent: number): void;
+export declare function readMaxConcurrentFromGlobal(): number | undefined;
+export declare function writeMaxConcurrentToGlobal(maxConcurrent: number): void;
 export declare function readCredentialSourceFromFile(cwd: string): CredentialSource | undefined;
 export declare function writeCredentialSourceToContext(ctx: ProjectContext, credentialSource: CredentialSource): void;
 export declare function writeCredentialSourceToFile(cwd: string, credentialSource: CredentialSource): void;

package/dist/runtime/settingsFile.js

@@ -150,6 +150,25 @@ export function writeMaxConcurrentToContext(ctx, maxConcurrent) {
 export function writeMaxConcurrentToFile(cwd, maxConcurrent) {
     writeFactorySettings(cwd, { maxConcurrent });
 }
+export function readMaxConcurrentFromGlobal() {
+    return parseFactorySettingsFromRaw(readRawSettingsFromFile(globalSettingsFilePath())).maxConcurrent;
+}
+export function writeMaxConcurrentToGlobal(maxConcurrent) {
+    if (parseSlotCount(maxConcurrent) === undefined) {
+        throw new Error(INVALID_SLOT_COUNT_MESSAGE);
+    }
+    const current = parseFactorySettingsFromRaw(readRawSettingsFromFile(globalSettingsFilePath()));
+    const next = { ...current, maxConcurrent };
+    const out = {};
+    if (next.maxConcurrent !== undefined) {
+        out.maxConcurrent = next.maxConcurrent;
+    }
+    if (next.credentialSource !== undefined) {
+        out.credentialSource = next.credentialSource;
+    }
+    mkdirSync(path.dirname(globalSettingsFilePath()), { recursive: true });
+    writeFileSync(globalSettingsFilePath(), `${JSON.stringify(out, null, 2)}\n`, "utf8");
+}
 export function readCredentialSourceFromFile(cwd) {
     return readFactorySettings(cwd).credentialSource;
 }

package/dist/runtime/stageAttemptBootstrap.d.ts

@@ -1,4 +1,5 @@
 import type { AgentPort, FeedbackLoopContext, StageHandle, StageRunInput, StageSessionMode } from "../agent/port.js";
+import { type StageStreamLogWriter } from "./stageStreamLog.js";
 import type { RunStore } from "../runstore/port.js";
 import type { StageEnvelope } from "../types/envelope.js";
 import type { ResolvedPipelineDag } from "../types/pipeline.js";
@@ -23,6 +24,8 @@ export type StageAttemptOpenInput = {
     attemptCtx?: StageAttemptContext;
     operatorCatalog?: OperatorCatalog;
     onActivity?: StageRunInput["onActivity"];
+    /** Test seam; defaults to the real createStageStreamLogWriter. */
+    streamLogWriterFactory?: (streamLogPath: string) => StageStreamLogWriter;
     roots?: StageRoots;
     resumeToken?: string;
     sessionMode?: StageSessionMode;

package/dist/runtime/stageAttemptBootstrap.js

@@ -1,7 +1,8 @@
 import { resolveSkillByName } from "../config/listSkills.js";
 import { mkdir } from "node:fs/promises";
 import { MCP_CATALOG_FILENAME, STAGEFLOW_STAGE_ARTIFACTS_DIR_ENV, StageMcpError, resolveStageMcpServers, stampStagePromptArtifactsDir, } from "../config/resolveStageMcpServers.js";
-import { attemptArtifactsDir } from "../runstore/workspaceLayout.js";
+import { attemptArtifactsDir, attemptStreamLogPath } from "../runstore/workspaceLayout.js";
+import { createStageStreamLogWriter, } from "./stageStreamLog.js";
 import { resolveCloneEmitContext, resolveForkEmitContext } from "../config/resolveForkEmitContext.js";
 import { createAttemptQaTrailReader } from "../hitl/qaTrail.js";
 import { assertPriorInputPayload, payloadInstanceMismatch, } from "../envelope/payloadSchema.js";
@@ -199,6 +200,7 @@ export async function openStageAttempt(input) {
         return priorInput;
     const roots = resolveAttemptRoots(input, stageId);
     const attempt = input.attemptCtx?.attempt ?? 1;
+    const streamWriter = (input.streamLogWriterFactory ?? createStageStreamLogWriter)(attemptStreamLogPath(input.workspaceDir, stageId, attempt));
     const resumeToken = input.resumeToken ??
         resumeSessionFilePath(input.workspaceDir, stageId, attempt);
     const forkEmitContext = resolveForkEmitContext(input.dag, definitionId);
@@ -231,7 +233,11 @@ export async function openStageAttempt(input) {
             : {}),
         resumeToken,
         ...(input.sessionMode !== undefined ? { sessionMode: input.sessionMode } : {}),
-        onActivity: input.onActivity,
+        onActivity: (event) => {
+            input.onActivity?.(event);
+            void streamWriter.flush();
+        },
+        onAssistantTextDelta: streamWriter.onDelta,
         forkEmitContext,
         cloneEmitContext,
         ...(feedbackLoopEmitContext !== undefined ? { feedbackLoopEmitContext } : {}),

package/dist/runtime/stageRecovery.js

@@ -35,10 +35,10 @@ export async function failStageAsInterrupted(options) {
 }
 export async function syncRunStatusFromStages(store, runId) {
     const meta = await store.readRunMeta(runId);
-    if (meta.status === "succeeded")
-        return;
     const run = await store.readRun(runId);
     const derived = deriveStatusFromStages(run.stages, meta.pipeline_dag);
+    if (meta.status === "succeeded" && derived !== "running")
+        return;
     if ((meta.status ?? "created") !== derived) {
         await store.updateRunStatus(runId, derived);
     }

package/dist/runtime/stageRunner.js

@@ -3,6 +3,7 @@ import { buildStageRoots, withResolvedAuthPath, } from "./stageRoots.js";
 import { createDefaultHitlQaHooks, recordWaitEnter, } from "./stageHitl.js";
 import { openStageAttempt, } from "./stageAttemptBootstrap.js";
 import { createVerifiedStageExecution } from "./verifiedStageExecution.js";
+import { isStageTimeoutReason } from "../agent/stageTimeout.js";
 export function isRunStageWaiting(outcome) {
     return "waiting" in outcome && outcome.waiting === true;
 }
@@ -220,7 +221,9 @@ export async function runStage(options) {
             attemptCtx,
         });
         console.error(`Stage ${stageId} failed: ${reason}`);
-        await handle.close().catch(() => undefined);
+        await handle
+            .close(isStageTimeoutReason(reason) ? { park: true } : undefined)
+            .catch(() => undefined);
         return { ok: false, reason };
     }
     if (isRunStageWaiting(outcome)) {
@@ -251,6 +254,7 @@ export async function runStage(options) {
         });
     }
     finally {
-        await handle.close().catch(() => undefined);
+        const timedOut = isStageTimeoutReason(outcome.ok ? undefined : outcome.reason);
+        await handle.close(timedOut ? { park: true } : undefined).catch(() => undefined);
     }
 }