stageflow 0.25.0 → 0.30.0

package/dist/runtime/repositoryMaterialize.js

@@ -0,0 +1,235 @@
+import { existsSync, mkdirSync } from "node:fs";
+import path from "node:path";
+import { bareCachePath, ensureBareCache, } from "../git/cache.js";
+import { GitError } from "../git/errors.js";
+import { catFileCommit, checkRefFormat, deleteBranch, resolveRef, revParse, worktreeAdd, worktreePrune, worktreeRemove, } from "../git/operations.js";
+import { ensureGlobalHome, globalStageflowHome } from "../project/globalHome.js";
+const DEFAULT_BRANCH_TEMPLATE = "stageflow/run-<runId>";
+const FULL_SHA_RE = /^[0-9a-f]{40}$/i;
+export function derivedBindingKindFromMeta(meta) {
+    if (meta.repository != null && meta.repository !== "") {
+        return "repository";
+    }
+    if (meta.checkout_root != null && meta.checkout_root !== "") {
+        return "checkout";
+    }
+    return "unbound";
+}
+export function worktreePathForRun(runId) {
+    return path.join(globalStageflowHome(), "worktrees", runId);
+}
+function renderBranchTemplate(template, runId, taskId) {
+    return template.replaceAll("<runId>", runId).replaceAll("<taskId>", taskId);
+}
+export function resolveRunBranchTemplate(task) {
+    const fromTask = task.run_branch_template?.trim();
+    if (fromTask)
+        return fromTask;
+    const fromEnv = process.env.STAGEFLOW_RUN_BRANCH_TEMPLATE?.trim();
+    if (fromEnv)
+        return fromEnv;
+    return DEFAULT_BRANCH_TEMPLATE;
+}
+export class StartLinkError extends Error {
+    code;
+    status;
+    stderr;
+    constructor(message, options) {
+        super(message);
+        this.name = "StartLinkError";
+        this.code = options.code;
+        this.status = options.status;
+        this.stderr = options.stderr;
+    }
+}
+function mapGitCode(code, phase) {
+    switch (code) {
+        case "auth_failed":
+            return { code: "repository_auth_failed", status: 401 };
+        case "repo_not_found":
+            return { code: "repository_not_found", status: 404 };
+        case "ref_not_found":
+            return { code: "ref_not_found", status: 404 };
+        case "network":
+        case "timeout":
+            return { code: "repository_fetch_failed", status: 502 };
+        case "disk_full":
+            return { code: "insufficient_disk", status: 507 };
+        case "git_missing":
+            return { code: "git_missing", status: 500 };
+        default:
+            if (phase === "worktree") {
+                return { code: "worktree_create_failed", status: 500 };
+            }
+            if (phase === "resolve") {
+                return { code: "ref_not_found", status: 404 };
+            }
+            return { code: "repository_fetch_failed", status: 502 };
+    }
+}
+export function throwMappedGitError(err, phase) {
+    if (err instanceof StartLinkError)
+        throw err;
+    if (err instanceof GitError) {
+        const mapped = mapGitCode(err.code, phase);
+        throw new StartLinkError(err.message, {
+            code: mapped.code,
+            status: mapped.status,
+            stderr: err.stderr,
+        });
+    }
+    throw err;
+}
+async function runRollbackThunks(thunks) {
+    for (const thunk of thunks) {
+        try {
+            await thunk();
+        }
+        catch (cleanupErr) {
+            console.error(`repository link rollback failed: ${cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)}`);
+        }
+    }
+}
+export async function reclaimWorkspaceBinding(meta, options) {
+    if (derivedBindingKindFromMeta(meta) !== "repository") {
+        return;
+    }
+    const cachePath = bareCachePath(meta.repository);
+    try {
+        await worktreeRemove(cachePath, meta.checkout_root);
+    }
+    catch {
+    }
+    try {
+        await worktreePrune(cachePath);
+    }
+    catch {
+    }
+    if (options?.keepRunBranch) {
+        return;
+    }
+    try {
+        await deleteBranch(cachePath, meta.run_branch);
+    }
+    catch {
+    }
+}
+export async function reclaimWorkspaceOnRunSucceeded(store, runId) {
+    try {
+        const meta = await store.readRunMeta(runId);
+        await reclaimWorkspaceBinding(meta, { keepRunBranch: true });
+    }
+    catch (err) {
+        console.error(`reclaimWorkspaceOnRunSucceeded: failed for ${runId}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+}
+export async function materializeWorkspaceBinding(options) {
+    const { runId, task, binding } = options;
+    const rollbacks = [];
+    const rollback = async () => runRollbackThunks(rollbacks);
+    if (binding.kind === "unbound") {
+        return { materialized: { runId }, rollback };
+    }
+    if (binding.kind === "checkout") {
+        const checkoutRoot = options.checkoutRoot;
+        if (checkoutRoot === undefined) {
+            throw new StartLinkError("Checkout path is required", {
+                code: "worktree_create_failed",
+                status: 500,
+            });
+        }
+        let resolvedSha;
+        if (options.pinned?.resolvedSha) {
+            resolvedSha = options.pinned.resolvedSha;
+        }
+        else {
+            try {
+                const head = await revParse(checkoutRoot, ["HEAD"]);
+                if (FULL_SHA_RE.test(head)) {
+                    resolvedSha = head;
+                }
+            }
+            catch {
+                // non-git path checkouts leave resolved_sha null (KD6/KD7)
+            }
+        }
+        return {
+            materialized: {
+                runId,
+                checkoutRoot,
+                ...(resolvedSha !== undefined ? { resolvedSha } : {}),
+            },
+            rollback,
+        };
+    }
+    const repository = binding.repository;
+    const refToRecord = options.pinned?.ref ?? binding.ref;
+    let resolvedSha;
+    let cachePath;
+    if (options.pinned) {
+        cachePath = bareCachePath(repository);
+        const present = existsSync(cachePath) &&
+            (await catFileCommit(cachePath, options.pinned.resolvedSha));
+        if (!present) {
+            throw new StartLinkError(`Pinned SHA is not available in the local cache: ${options.pinned.resolvedSha}`, { code: "pinned_sha_unavailable", status: 404 });
+        }
+        resolvedSha = options.pinned.resolvedSha;
+    }
+    else {
+        try {
+            const ensured = await ensureBareCache(repository, binding.ref);
+            cachePath = ensured.cachePath;
+        }
+        catch (err) {
+            throwMappedGitError(err, "fetch");
+        }
+        try {
+            resolvedSha = await resolveRef(cachePath, binding.ref);
+        }
+        catch (err) {
+            throwMappedGitError(err, "resolve");
+        }
+    }
+    const template = resolveRunBranchTemplate(task);
+    const runBranch = renderBranchTemplate(template, runId, task.id);
+    const branchOk = await checkRefFormat(runBranch);
+    if (!branchOk) {
+        throw new StartLinkError(`Invalid run branch name from template: ${runBranch}`, { code: "invalid_run_branch", status: 400 });
+    }
+    ensureGlobalHome();
+    const checkoutRoot = worktreePathForRun(runId);
+    mkdirSync(path.dirname(checkoutRoot), { recursive: true });
+    try {
+        await worktreeAdd(cachePath, {
+            worktreePath: checkoutRoot,
+            branch: runBranch,
+            startPoint: resolvedSha,
+        });
+    }
+    catch (err) {
+        throwMappedGitError(err, "worktree");
+    }
+    rollbacks.push(async () => {
+        try {
+            await worktreeRemove(cachePath, checkoutRoot);
+        }
+        catch {
+            // prune recovers admin state when the dir is already gone
+        }
+        await worktreePrune(cachePath);
+    });
+    rollbacks.push(async () => {
+        await deleteBranch(cachePath, runBranch);
+    });
+    return {
+        materialized: {
+            runId,
+            checkoutRoot,
+            repository,
+            ref: refToRecord,
+            resolvedSha,
+            runBranch,
+        },
+        rollback,
+    };
+}

package/dist/runtime/resumeTimedOut.d.ts

@@ -17,7 +17,7 @@ export type ResumeTimedOutContext = {
     maxActiveStagesPerRun: number;
     operatorCatalog?: OperatorCatalog;
 };
-export declare function assertTimedOutStageEligible(status: string, events: ReadonlyArray<{
+export declare function assertResumableStage(status: string, events: ReadonlyArray<{
     event: string;
     reason?: string;
 }>): {

package/dist/runtime/resumeTimedOut.js

@@ -11,11 +11,14 @@ import { openStageAttempt, } from "./stageAttemptBootstrap.js";
 import { buildStageRoots, withResolvedAuthPath } from "./stageRoots.js";
 import { isStageTimeoutReason, lastFailedReason, } from "../agent/stageTimeout.js";
 import { deriveExecutionPatchFromEvent } from "../runstore/stageExecution.js";
-export function assertTimedOutStageEligible(status, events) {
+export function assertResumableStage(status, events) {
+    if (status === "interrupted") {
+        return { ok: true };
+    }
     if (status !== "failed") {
         return {
             ok: false,
-            reason: `Stage is not timed out (status=${status})`,
+            reason: `Stage is not resumable (status=${status})`,
             status: 409,
         };
     }

package/dist/runtime/runChangeBus.js

@@ -49,6 +49,13 @@ export function wrapRunStoreWithChangeBus(store, bus) {
         await store.updateRunStatus(runId, status);
         bus.emit({ runId, kind: "status" });
     };
+    const tryUpdateRunStatus = async (runId, status, expectedStatus) => {
+        const updated = await store.tryUpdateRunStatus(runId, status, expectedStatus);
+        if (updated) {
+            bus.emit({ runId, kind: "status" });
+        }
+        return updated;
+    };
     const appendStageEvent = async (runId, stageId, event, options) => {
         await store.appendStageEvent(runId, stageId, event, options);
         if (event.event === "waiting_for_input" || event.event === "resumed") {
@@ -66,6 +73,8 @@ export function wrapRunStoreWithChangeBus(store, bus) {
                 return createRun;
             if (prop === "updateRunStatus")
                 return updateRunStatus;
+            if (prop === "tryUpdateRunStatus")
+                return tryUpdateRunStatus;
             if (prop === "appendStageEvent")
                 return appendStageEvent;
             const cached = boundCache.get(prop);

package/dist/runtime/runDeletion.d.ts

@@ -0,0 +1,7 @@
+import type { A2aStore } from "../a2a/store.js";
+import type { RunStore } from "../runstore/port.js";
+/**
+ * Hard-delete a run across A2A, workspace, worktree/branch, then store rows last.
+ * Filesystem/git step failures are logged and do not abort the sequence (R10).
+ */
+export declare function deleteRunEverywhere(store: RunStore, a2aStore: A2aStore, runId: string): Promise<void>;

package/dist/runtime/runDeletion.js

@@ -0,0 +1,37 @@
+import { rm } from "node:fs/promises";
+import { reclaimWorkspaceBinding } from "./repositoryMaterialize.js";
+/**
+ * Hard-delete a run across A2A, workspace, worktree/branch, then store rows last.
+ * Filesystem/git step failures are logged and do not abort the sequence (R10).
+ */
+export async function deleteRunEverywhere(store, a2aStore, runId) {
+    try {
+        await a2aStore.deleteByRunId(runId);
+    }
+    catch (err) {
+        console.error(`deleteRunEverywhere: A2A cleanup failed for ${runId}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    let meta;
+    try {
+        meta = await store.readRunMeta(runId);
+    }
+    catch {
+        meta = undefined;
+    }
+    const workspaceDir = store.getWorkspaceDir(runId);
+    try {
+        await rm(workspaceDir, { recursive: true, force: true });
+    }
+    catch (err) {
+        console.error(`deleteRunEverywhere: workspace rm failed for ${runId}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (meta !== undefined) {
+        try {
+            await reclaimWorkspaceBinding(meta);
+        }
+        catch (err) {
+            console.error(`deleteRunEverywhere: reclaimWorkspaceBinding failed for ${runId}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    await store.deleteRun(runId);
+}

package/dist/runtime/runManager.d.ts

@@ -1,32 +1,51 @@
 import { type RunSubmission } from "../runstore/submission.js";
 import type { AgentPort, OpaqueAnswer } from "../agent/port.js";
 import type { InlinePipelineDefinition } from "../types/pipeline.js";
+import { type DiskBreakdown, type FreeSpaceReader } from "../runstore/diskUsage.js";
 import { type RunStore } from "../runstore/port.js";
 import type { TaskFile } from "../types/task.js";
 import { type PipelineRunResult } from "./pipelineRunner.js";
+import { type SkillsPayload } from "./runSkills.js";
 import { type FeedbackLoopDecisionKind } from "./feedbackLoopDecision.js";
 import { type RetryStageResult } from "./runRetryCoordinator.js";
 import { type StageExecutionMode } from "./stageConcurrency.js";
+import type { StageBrowserSupport } from "../browser/browserHost.js";
 import { StageProcessLauncher } from "./stageProcessLauncher.js";
 import { StageHitlController, type DeliverAnswerResult, type HitlSeams } from "./stageHitl.js";
 import { type StartTaskInput } from "./taskInput.js";
+import { type RetentionSweepReport, type RunRetentionSweepOptions } from "./runRetentionSweep.js";
+import type { A2aStore } from "../a2a/store.js";
 import type { OperatorCatalog } from "./stageAttemptBootstrap.js";
-export type BusyCode = "busy_capacity" | "busy_checkout";
+import { type StartFailureCode } from "./repositoryMaterialize.js";
+export type BusyCode = "busy_capacity" | "busy_checkout" | "busy_caller_quota";
+export type { StartFailureCode } from "./repositoryMaterialize.js";
 export type StartRunResult = {
     ok: true;
     runId: string;
     done: Promise<PipelineRunResult>;
+    queued?: boolean;
+    queuePosition?: number;
+    /** Present when queued because the caller hit its concurrency quota. */
+    queuedCode?: "busy_caller_quota";
 } | {
     ok: false;
     reason: string;
     status?: number;
-    code?: BusyCode;
+    code?: StartFailureCode;
     activeCount?: number;
     maxConcurrent?: number;
     activeRunIds?: string[];
+    scope?: "global" | "project" | "caller";
+    project_root?: string;
+    caller_id?: string;
     conflictingRunId?: string;
     conflictingCheckout?: string;
+    freeBytes?: number;
+    minFreeBytes?: number;
+    stderr?: string;
 };
+export declare const PROJECT_ROOT_UNAVAILABLE_REASON = "project_root_unavailable";
+export declare const INSUFFICIENT_DISK_CANCEL_REASON = "insufficient_disk";
 export type CapacityHealth = {
     ok: true;
     activeRunIds: string[];
@@ -35,6 +54,18 @@ export type CapacityHealth = {
     slotsAvailable: number;
     activeStageProcesses: number;
     maxActiveStageProcesses: number | null;
+    disk?: DiskBreakdown;
+    stage_env_passthrough?: boolean;
+    proxy?: Record<string, unknown>;
+    container?: {
+        max_old_space_size_mb: number;
+        max_active_stage_processes: number;
+        memory_limit_bytes: number | null;
+        source: string;
+    };
+    cache?: {
+        root: string;
+    };
 };
 export type StartRunOnceResult = {
     ok: true;
@@ -64,6 +95,32 @@ export type AbandonStageResult = {
     reason: string;
     status?: number;
 };
+export type CancelRunResult = {
+    ok: true;
+    runId: string;
+} | {
+    ok: false;
+    reason: string;
+    status?: number;
+};
+export type DeleteRunChannel = "mcp" | "rest" | "cli";
+export type DeleteRunResult = {
+    ok: true;
+    runId: string;
+} | {
+    ok: false;
+    reason: string;
+    status?: number;
+};
+export type GcRunsChannel = "mcp" | "rest" | "cli" | "periodic";
+export type GcRunsResult = ({
+    ok: true;
+} & RetentionSweepReport) | {
+    ok: false;
+    reason: string;
+    status?: number;
+};
+export declare const FORCE_DELETE_CANCEL_REASON = "delete_run: force";
 export type DecideFeedbackLoopResult = {
     ok: true;
     effect: "extended" | "continued" | "abandoned";
@@ -82,21 +139,34 @@ export type StopManualRecoveryResult = {
     reason: string;
     status?: number;
 };
+export declare const STAGEFLOW_AUTO_RESUME_INTERRUPTED = "STAGEFLOW_AUTO_RESUME_INTERRUPTED";
+export declare const STAGEFLOW_MAX_AUTO_RESUMES = "STAGEFLOW_MAX_AUTO_RESUMES";
+export declare function isAutoResumeInterruptedEnabled(env?: NodeJS.ProcessEnv): boolean;
+export declare function parseMaxAutoResumes(env?: NodeJS.ProcessEnv): number;
 export declare class RunManager {
     private readonly options;
     private readonly submissionsInFlight;
     private readonly active;
+    private readonly schedulingHalts;
     private readonly checkoutLeases;
     private readonly provisionalIds;
+    private readonly admissionQueue;
+    private readonly pendingQueuedStarts;
+    private readonly queuedDone;
+    private admissionDrainInFlight;
     private readonly resumeInFlight;
     private readonly retryInFlight;
     private readonly retryStartOwner;
     private readonly retryStartWaiters;
     private trackingGeneration;
     private maxConcurrent;
+    private readonly maxQueued;
+    private readonly maxConcurrentPerProject;
+    private readonly callerQuotas;
     private readonly maxActiveStagesPerRun;
     private readonly executionMode;
     private readonly stageProcessLauncher;
+    private a2aStore;
     private readonly hitl;
     private readonly attachedWaiting;
     private readonly retryCoordinator;
@@ -104,6 +174,7 @@ export declare class RunManager {
     private readonly cwd;
     private readonly projectRoot;
     private readonly isGitProject;
+    private acceptingWork;
     constructor(options: {
         agent: AgentPort;
         store: RunStore;
@@ -113,15 +184,48 @@ export declare class RunManager {
         operatorCatalog?: OperatorCatalog;
         seams?: HitlSeams;
         maxConcurrent?: number;
+        maxQueued?: number;
+        maxConcurrentPerProject?: number;
+        /** caller_id → max concurrent active runs for that caller. */
+        callerQuotas?: Record<string, number>;
         maxActiveStagesPerRun?: number;
         executionMode?: StageExecutionMode;
         stageProcessLauncher?: StageProcessLauncher;
+        browser?: StageBrowserSupport;
+        a2aStore?: A2aStore;
+        knownWritableProjectRoots?: () => Iterable<string> | Promise<Iterable<string>>;
+        freeSpaceReader?: FreeSpaceReader;
+        /** Test-only: await before materialize when activating a queued run. */
+        onBeforeQueuedMaterialize?: (runId: string) => void | Promise<void>;
     });
     getMaxConcurrent(): number;
     setMaxConcurrent(n: number): CapacityHealth;
     getActiveCount(): number;
+    stopAcceptingWork(): void;
+    isAcceptingWork(): boolean;
+    drainActiveStages(options: {
+        deadlineMs: number;
+        isEscalated?: () => boolean;
+    }): Promise<{
+        forced: boolean;
+    }>;
     getActiveRunIds(): string[];
     getHealth(): CapacityHealth;
+    getPerProjectCapacity(): {
+        maxConcurrent: number | undefined;
+        projects: Array<{
+            project_root: string;
+            activeCount: number;
+            maxConcurrent: number | undefined;
+        }>;
+    };
+    private countActiveForProject;
+    private countActiveForCaller;
+    private callerQuotaLimit;
+    /** Capacity plus on-demand durable-root disk breakdown (KTD16). */
+    getHealthWithDisk(): Promise<CapacityHealth>;
+    /** Refresh cached `disk_bytes` for one run after a terminal transition. */
+    refreshRunDiskBytes(runId: string): Promise<void>;
     getHitlController(): StageHitlController;
     /**
      * KTD7/KTD8: discover waiting stages from the store; leave waiting; do not
@@ -133,6 +237,9 @@ export declare class RunManager {
         stageId: string;
     }>>;
     hasActiveWorker(runId: string, stageId: string): boolean;
+    sweepBrowserSessions(): Promise<{
+        closed: string[];
+    }>;
     reconcileOrphanedStages(): Promise<{
         reconciled: Array<{
             runId: string;
@@ -140,10 +247,38 @@ export declare class RunManager {
             reason: string;
         }>;
     }>;
+    autoResumeInterruptedStages(): Promise<{
+        resumed: Array<{
+            runId: string;
+            stageId: string;
+        }>;
+        capped: Array<{
+            runId: string;
+            stageId: string;
+        }>;
+        skipped: Array<{
+            runId: string;
+            stageId: string;
+            reason: string;
+        }>;
+    }>;
     resumeStalledSchedules(): Promise<Array<{
         runId: string;
     }>>;
+    /** Rebuild in-memory admission queue from persisted `queued` rows (R27). */
+    reenqueuePersistedQueuedRuns(): Promise<void>;
     abandonStage(runId: string, stageId: string): Promise<AbandonStageResult>;
+    cancelRun(runId: string, reason: string): Promise<CancelRunResult>;
+    setA2aStore(store: A2aStore): void;
+    deleteRun(runId: string, options?: {
+        force?: boolean;
+        channel: DeleteRunChannel;
+    }): Promise<DeleteRunResult>;
+    gcRuns(options?: {
+        execute?: boolean;
+        channel?: GcRunsChannel;
+    } & Pick<RunRetentionSweepOptions, "now" | "windows" | "env" | "artifactMaxBytes" | "bareCacheTtlMs">): Promise<GcRunsResult>;
+    private waitForPromise;
     startRunOnce(input: Parameters<RunManager["startRun"]>[0], submission: RunSubmission): Promise<StartRunOnceResult>;
     private startSubmittedRun;
     startRun(input: StartTaskInput & {
@@ -154,10 +289,24 @@ export declare class RunManager {
         gitSha?: string;
         ciPrUrl?: string;
         ciJobUrl?: string;
+        /**
+         * Absolute store-key project root from the wire (registered path or
+         * seeded-mapped absolute). When set, persisted as run project_root;
+         * Host boot / findProjectRoot are not used to override it.
+         */
+        projectRoot?: string;
+        /** Attribution from auth only — never from request body/query. */
+        callerId?: string | null;
+        skills?: SkillsPayload;
     }, submission?: RunSubmission): Promise<StartRunResult>;
-    rerun(runId: string): Promise<StartRunResult>;
+    rerun(runId: string, options?: {
+        pinned?: boolean;
+        callerId?: string | null;
+    }): Promise<StartRunResult>;
     retryStage(runId: string, stageId: string): Promise<RetryStageResult>;
-    resumeTimedOutStage(runId: string, stageId: string): Promise<RetryStageResult>;
+    resumeTimedOutStage(runId: string, stageId: string, options?: {
+        source?: "explicit" | "auto";
+    }): Promise<RetryStageResult>;
     recoverManualStage(runId: string, stageId: string, guidance?: string): Promise<RetryStageResult>;
     recoverManualStageUntilStop(runId: string, stageId: string, guidance?: string): Promise<{
         ok: true;
@@ -200,10 +349,25 @@ export declare class RunManager {
         reason?: string;
     }): Promise<DecideFeedbackLoopResult>;
     private resumeViaSubprocess;
+    /**
+     * A resumed stage whose after-phase verification failed is repaired by the
+     * same policy the scheduler applies to a first-launch failure. Returns the
+     * freshly created attempt, or undefined when the failure is final.
+     */
+    private scheduleRepairAfterVerifyFailure;
     private reconstructAndContinue;
     private reserveAndStartPipeline;
     private busyFailure;
+    private insufficientDiskFailure;
+    private checkDiskFloorAdmission;
+    private isProjectRootAvailable;
+    private tryAdmitOrEnqueue;
     private tryReserve;
+    private ensureQueuedDone;
+    private resolveQueuedDone;
+    private attachQueuedDone;
+    private drainAdmissionQueue;
+    private startDequeuedAdmission;
     private clearReservation;
     private track;
     private tryResumeStalledSchedule;
@@ -217,4 +381,5 @@ export declare class RunManager {
     private quarantineAttachRun;
     private untrackIfGeneration;
     private removeActiveEntry;
+    private ensureSchedulingHalt;
 }