stageflow 0.23.0 → 0.27.0

package/dist/runtime/runManager.js

@@ -1,62 +1,277 @@
 import { randomUUID } from "node:crypto";
-import { readFile, realpath } from "node:fs/promises";
+import { constants } from "node:fs";
+import { RunSubmissionExistsError } from "../runstore/submission.js";
+import { access, readFile, realpath } from "node:fs/promises";
 import path from "node:path";
 import { findProjectRoot } from "../project/findProjectRoot.js";
-import { normalizeCatalogPath } from "../runstore/normalizeCatalogPath.js";
+import { globalStageflowHome } from "../project/globalHome.js";
+import { normalizeCatalogPath, normalizeProjectRoot } from "../runstore/normalizeCatalogPath.js";
+import { durableRootDiskBreakdown, readFilesystemSize, refreshRunDiskUsage, resolveMinFreeDiskFloor, } from "../runstore/diskUsage.js";
+import { newRunId } from "../runstore/paths.js";
 import { loadRunContext } from "./resumeReconstruct.js";
-import { assertTimedOutStageEligible, reconstructTimedOutAndContinue, } from "./resumeTimedOut.js";
-import { loadTaskFromYaml } from "../config/loadTask.js";
+import { assertResumableStage, reconstructTimedOutAndContinue, } from "./resumeTimedOut.js";
+import { loadTaskFromYamlOutcome } from "../config/loadTask.js";
+import { buildValidationResult, loadPipelineValidated, } from "../config/validateCatalog.js";
 import { deriveStatusFromStages, findUnhandledFailedStage, } from "../runstore/port.js";
-import { PipelineValidationError, startPipeline, } from "./pipelineRunner.js";
+import { buildPipelineDagSnapshotFromLoaded } from "../runstore/pipelineDagSnapshot.js";
+import { InlinePipelineTooLargeError, PipelinePreflightError, PipelineValidationError, QueuedRunActivationAborted, startPipeline, } from "./pipelineRunner.js";
+import { INLINE_PIPELINE_TOO_LARGE, pipelineBodyBytes, pipelinePersistenceForStart, } from "./startPayload.js";
+import { materializeRunSkills, validateSkillsPayload, } from "./runSkills.js";
 import { hydrateScheduleFromStore, hydratedScheduleHasRunnableWork, resumeRun, runPipelineDag, } from "./pipelineScheduler.js";
 import { resolveFeedbackLoopDecision, } from "./feedbackLoopDecision.js";
 import { RunRetryCoordinator, readRetryRootWaitTimeoutMs, } from "./runRetryCoordinator.js";
 import { readMaxActiveStageProcesses, readMaxActiveStagesPerRun, readStageExecutionMode, } from "./stageConcurrency.js";
 import { StageProcessLauncher } from "./stageProcessLauncher.js";
+import { logger as rootLogger } from "../logging/logger.js";
+import { registerNamedSecrets } from "../logging/namedSecrets.js";
+import { shouldRegisterValue } from "../logging/redact.js";
+import { bindingFromFields, buildInitialRunManifest, collectDeclaredSecretValues, finaliseStoredRunManifest, } from "../runstore/runManifest.js";
+import { getRequestAuth } from "../server/requestAuthContext.js";
+import { STAGE_ENV_PASSTHROUGH } from "./stageEnvironment.js";
+import { proxyHealthFields } from "../net/proxy.js";
+import { getContainerLimits } from "./containerLimits.js";
+import { stageflowCacheRoot } from "./stageCacheEnv.js";
+import { assertClaudeNotRoot, ClaudeRootError } from "../preflight/claudeRoot.js";
+import { preflightFailureCode, runPipelinePreflight, } from "../preflight/pipelinePreflight.js";
+import { asAgentBackendId } from "../agent/agentBackend.js";
 import { INVALID_SLOT_COUNT_MESSAGE, parseSlotCount, readMaxConcurrentFromGlobal, writeMaxConcurrentToGlobal, } from "./settingsFile.js";
 import { StageHitlController, waitKey, } from "./stageHitl.js";
-import { resolveAndValidateCheckout, } from "./stageRoots.js";
+import { resolveAndValidateCheckout, resolveEffectiveGitIdentity, stageBindingEnvFromRun, } from "./stageRoots.js";
 import { orchestrateAnswerResume } from "./answerResume.js";
 import { reconstructAndContinue as resumeReconstructAndContinue } from "./resumeReconstruct.js";
-import { resumeSessionFilePath } from "./stageAttemptContext.js";
-import { resolveStartTaskInput } from "./taskInput.js";
-import { failStageAsInterrupted, syncRunStatusFromStages, } from "./stageRecovery.js";
+import { attemptContext, resumeSessionFilePath } from "./stageAttemptContext.js";
+import { checkTaskEntryInput, resolveStartTaskInput } from "./taskInput.js";
+import { markStageInterrupted, OPERATOR_CANCEL_REASON, syncRunStatusFromStages, } from "./stageRecovery.js";
+import { deleteRunEverywhere } from "./runDeletion.js";
+import { runRetentionSweep, } from "./runRetentionSweep.js";
 import { blocksGenericRetry, manualRecoveryEligibility, readManualRecoveryState, } from "./manualRecoveryState.js";
+import { resolveWorkspaceBinding } from "./workspaceBinding.js";
+import { derivedBindingKindFromMeta, materializeWorkspaceBinding, StartLinkError, } from "./repositoryMaterialize.js";
+export const PROJECT_ROOT_UNAVAILABLE_REASON = "project_root_unavailable";
+export const INSUFFICIENT_DISK_CANCEL_REASON = "insufficient_disk";
+function existingSubmissionResult(existing, request) {
+    if (existing.requestHash !== request.requestHash) {
+        return { ok: false, status: 409, reason: "Submission key was already used for different input" };
+    }
+    return { ok: true, runId: existing.runId, reused: true };
+}
+export const FORCE_DELETE_CANCEL_REASON = "delete_run: force";
 const DEFAULT_MAX_CONCURRENT = 3;
-const STARTUP_RECONCILE_REASON = "process_interrupted: no active worker (server restart)";
+const DEFAULT_MAX_QUEUED = 32;
+const DEFAULT_MAX_AUTO_RESUMES = 3;
+const STARTUP_RECONCILE_REASON = "orphaned_no_worker";
+const AUTO_RESUME_CAPPED_REASON = "auto_resume_capped";
+export const STAGEFLOW_AUTO_RESUME_INTERRUPTED = "STAGEFLOW_AUTO_RESUME_INTERRUPTED";
+export const STAGEFLOW_MAX_AUTO_RESUMES = "STAGEFLOW_MAX_AUTO_RESUMES";
 const OPERATOR_ABANDON_REASON = "process_interrupted: operator abandoned stage";
+const log = rootLogger.child({ component: "runtime" });
 function parseMaxConcurrent(raw) {
     if (raw === undefined || raw.trim() === "")
         return DEFAULT_MAX_CONCURRENT;
     const n = Number.parseInt(raw, 10);
-    if (!Number.isFinite(n) || n < 1)
-        return DEFAULT_MAX_CONCURRENT;
+    if (!Number.isFinite(n) || n < 1 || String(n) !== raw.trim()) {
+        throw new Error(`Invalid value for STAGEFLOW_MAX_CONCURRENT_RUNS: ${JSON.stringify(raw)}`);
+    }
     return n;
 }
+function parseMaxQueued(raw) {
+    if (raw === undefined || raw.trim() === "")
+        return DEFAULT_MAX_QUEUED;
+    const n = Number.parseInt(raw, 10);
+    if (!Number.isFinite(n) || n < 0)
+        return DEFAULT_MAX_QUEUED;
+    return n;
+}
+export function isAutoResumeInterruptedEnabled(env = process.env) {
+    const raw = env[STAGEFLOW_AUTO_RESUME_INTERRUPTED];
+    if (raw === undefined || raw.trim() === "")
+        return false;
+    const normalized = raw.trim().toLowerCase();
+    return normalized !== "0" && normalized !== "false";
+}
+export function parseMaxAutoResumes(env = process.env) {
+    const raw = env[STAGEFLOW_MAX_AUTO_RESUMES];
+    if (raw === undefined || raw.trim() === "")
+        return DEFAULT_MAX_AUTO_RESUMES;
+    const n = Number.parseInt(raw, 10);
+    if (!Number.isFinite(n) || n < 0)
+        return DEFAULT_MAX_AUTO_RESUMES;
+    return n;
+}
+/** Private FIFO-per-project_root + round-robin dequeue (KTD3). */
+class AdmissionQueue {
+    byRoot = new Map();
+    rrOrder = [];
+    rrIndex = 0;
+    get size() {
+        let n = 0;
+        for (const list of this.byRoot.values())
+            n += list.length;
+        return n;
+    }
+    clear() {
+        this.byRoot.clear();
+        this.rrOrder = [];
+        this.rrIndex = 0;
+    }
+    enqueue(projectRoot, entry) {
+        let list = this.byRoot.get(projectRoot);
+        if (list === undefined) {
+            list = [];
+            this.byRoot.set(projectRoot, list);
+            this.rrOrder.push(projectRoot);
+        }
+        list.push(entry);
+        return this.size;
+    }
+    requeueFront(projectRoot, entry) {
+        let list = this.byRoot.get(projectRoot);
+        if (list === undefined) {
+            list = [];
+            this.byRoot.set(projectRoot, list);
+            this.rrOrder.push(projectRoot);
+        }
+        list.unshift(entry);
+    }
+    remove(runId) {
+        for (const [root, list] of this.byRoot) {
+            const idx = list.findIndex((e) => e.runId === runId);
+            if (idx < 0)
+                continue;
+            list.splice(idx, 1);
+            if (list.length === 0) {
+                this.byRoot.delete(root);
+                const orderIdx = this.rrOrder.indexOf(root);
+                if (orderIdx >= 0) {
+                    this.rrOrder.splice(orderIdx, 1);
+                    if (this.rrOrder.length === 0) {
+                        this.rrIndex = 0;
+                    }
+                    else if (orderIdx < this.rrIndex) {
+                        this.rrIndex -= 1;
+                    }
+                    else if (this.rrIndex >= this.rrOrder.length) {
+                        this.rrIndex = 0;
+                    }
+                }
+            }
+            return true;
+        }
+        return false;
+    }
+    positionOf(runId) {
+        let position = 0;
+        const rootCount = this.rrOrder.length;
+        if (rootCount === 0)
+            return undefined;
+        const heads = this.rrOrder.map((root) => ({
+            root,
+            list: this.byRoot.get(root) ?? [],
+            i: 0,
+        }));
+        let rr = this.rrIndex % rootCount;
+        let remaining = this.size;
+        while (remaining > 0) {
+            let advanced = false;
+            for (let step = 0; step < rootCount; step++) {
+                const slot = heads[(rr + step) % rootCount];
+                if (slot === undefined || slot.i >= slot.list.length)
+                    continue;
+                const entry = slot.list[slot.i];
+                slot.i += 1;
+                remaining -= 1;
+                position += 1;
+                advanced = true;
+                if (entry?.runId === runId)
+                    return position;
+                rr = (rr + step + 1) % rootCount;
+                break;
+            }
+            if (!advanced)
+                break;
+        }
+        return undefined;
+    }
+    dequeueNext(skipRoots, skipCallers) {
+        if (this.rrOrder.length === 0)
+            return undefined;
+        const start = this.rrIndex % this.rrOrder.length;
+        for (let step = 0; step < this.rrOrder.length; step++) {
+            const idx = (start + step) % this.rrOrder.length;
+            const root = this.rrOrder[idx];
+            if (root === undefined)
+                continue;
+            if (skipRoots?.has(root))
+                continue;
+            const list = this.byRoot.get(root);
+            if (list === undefined || list.length === 0)
+                continue;
+            let pickIdx = 0;
+            if (skipCallers !== undefined && skipCallers.size > 0) {
+                pickIdx = list.findIndex((e) => {
+                    const c = e.callerId;
+                    if (c === undefined || c === null || c === "")
+                        return true;
+                    return !skipCallers.has(c.toLowerCase());
+                });
+                if (pickIdx < 0)
+                    continue;
+            }
+            const [entry] = list.splice(pickIdx, 1);
+            if (entry === undefined)
+                continue;
+            if (list.length === 0) {
+                this.byRoot.delete(root);
+                this.rrOrder.splice(idx, 1);
+                if (this.rrOrder.length === 0) {
+                    this.rrIndex = 0;
+                }
+                else {
+                    this.rrIndex = idx % this.rrOrder.length;
+                }
+            }
+            else {
+                this.rrIndex = (idx + 1) % this.rrOrder.length;
+            }
+            return { projectRoot: root, entry };
+        }
+        return undefined;
+    }
+}
 async function toCheckoutLeaseKey(absPath) {
     try {
         return await realpath(absPath);
     }
     catch {
         const fallback = path.resolve(absPath);
-        console.error(`invariant: checkout realpath failed for ${absPath}; using path.resolve fallback ${fallback}`);
+        log.error("checkout.realpath_failed", `invariant: checkout realpath failed for ${absPath}; using path.resolve fallback ${fallback}`, { abs_path: absPath, fallback });
         return fallback;
     }
 }
 export class RunManager {
     options;
+    submissionsInFlight = new Map();
     active = new Map();
+    schedulingHalts = new Map();
     checkoutLeases = new Map();
     provisionalIds = new Set();
+    admissionQueue = new AdmissionQueue();
+    pendingQueuedStarts = new Map();
+    queuedDone = new Map();
+    admissionDrainInFlight = false;
     resumeInFlight = new Set();
     retryInFlight = new Set();
     retryStartOwner = new Map();
     retryStartWaiters = new Map();
     trackingGeneration = 0;
     maxConcurrent;
+    maxQueued;
+    maxConcurrentPerProject;
+    callerQuotas;
     maxActiveStagesPerRun;
     executionMode;
     stageProcessLauncher;
+    a2aStore;
     hitl;
     attachedWaiting = new Set();
     retryCoordinator = new RunRetryCoordinator();
@@ -90,15 +305,24 @@ export class RunManager {
     cwd;
     projectRoot;
     isGitProject;
+    acceptingWork = true;
     constructor(options) {
         this.options = options;
         this.cwd = options.cwd ?? process.cwd();
         this.projectRoot = options.projectRoot ?? this.cwd;
         this.isGitProject = options.isGitProject ?? false;
+        this.a2aStore = options.a2aStore;
         this.maxConcurrent =
             options.maxConcurrent ??
                 readMaxConcurrentFromGlobal() ??
                 parseMaxConcurrent(process.env.STAGEFLOW_MAX_CONCURRENT_RUNS);
+        this.maxQueued =
+            options.maxQueued ?? parseMaxQueued(process.env.STAGEFLOW_MAX_QUEUED);
+        this.maxConcurrentPerProject = options.maxConcurrentPerProject;
+        this.callerQuotas = new Map(Object.entries(options.callerQuotas ?? {}).map(([id, n]) => [
+            id.toLowerCase(),
+            n,
+        ]));
         this.maxActiveStagesPerRun = readMaxActiveStagesPerRun(process.env, options.maxActiveStagesPerRun);
         this.executionMode = readStageExecutionMode(process.env, options.executionMode);
         this.stageProcessLauncher =
@@ -125,6 +349,57 @@ export class RunManager {
     getActiveCount() {
         return this.active.size;
     }
+    stopAcceptingWork() {
+        this.acceptingWork = false;
+        for (const halt of this.schedulingHalts.values()) {
+            halt.halted = true;
+            halt.hostShutdown = true;
+        }
+    }
+    isAcceptingWork() {
+        return this.acceptingWork;
+    }
+    async drainActiveStages(options) {
+        for (const halt of this.schedulingHalts.values()) {
+            halt.halted = true;
+            halt.hostShutdown = true;
+        }
+        const launcher = this.stageProcessLauncher;
+        if (launcher === undefined || launcher.activeCount() === 0) {
+            return { forced: false };
+        }
+        const previouslyActive = launcher.getActiveStageProcesses();
+        const seenRuns = new Set();
+        for (const { runId, stageId } of previouslyActive) {
+            await markStageInterrupted({
+                store: this.options.store,
+                runId,
+                stageId,
+                reason: "host_shutdown",
+                status: "interrupted",
+            });
+            seenRuns.add(runId);
+        }
+        for (const runId of seenRuns) {
+            await syncRunStatusFromStages(this.options.store, runId).catch(() => undefined);
+        }
+        launcher.signalAllActive("SIGTERM");
+        while (launcher.activeCount() > 0 &&
+            Date.now() < options.deadlineMs &&
+            !(options.isEscalated?.() ?? false)) {
+            await new Promise((resolve) => setTimeout(resolve, 25));
+        }
+        const remaining = launcher.getActiveStageProcesses();
+        if (remaining.length === 0) {
+            return { forced: false };
+        }
+        launcher.signalAllActive("SIGKILL");
+        const killWaitUntil = Date.now() + 500;
+        while (launcher.activeCount() > 0 && Date.now() < killWaitUntil) {
+            await new Promise((resolve) => setTimeout(resolve, 25));
+        }
+        return { forced: true };
+    }
     getActiveRunIds() {
         return [...this.active.keys()];
     }
@@ -134,6 +409,7 @@ export class RunManager {
         const maxConcurrent = this.getMaxConcurrent();
         const maxActiveStageProcessesRaw = readMaxActiveStageProcesses(process.env);
         const activeStageProcesses = this.stageProcessLauncher?.activeCount() ?? 0;
+        const limits = getContainerLimits();
         return {
             ok: true,
             activeRunIds,
@@ -144,8 +420,83 @@ export class RunManager {
             maxActiveStageProcesses: Number.isFinite(maxActiveStageProcessesRaw)
                 ? maxActiveStageProcessesRaw
                 : null,
+            stage_env_passthrough: process.env[STAGE_ENV_PASSTHROUGH]?.trim() === "all",
+            proxy: proxyHealthFields(process.env),
+            container: {
+                max_old_space_size_mb: limits.maxOldSpaceSizeMb,
+                max_active_stage_processes: limits.maxActiveStageProcesses,
+                memory_limit_bytes: limits.memoryLimitBytes ?? null,
+                source: limits.source,
+            },
+            cache: { root: stageflowCacheRoot() },
+        };
+    }
+    getPerProjectCapacity() {
+        const byRoot = new Map();
+        for (const entry of this.active.values()) {
+            const root = entry.projectRoot ?? this.projectRoot;
+            byRoot.set(root, (byRoot.get(root) ?? 0) + 1);
+        }
+        return {
+            maxConcurrent: this.maxConcurrentPerProject,
+            projects: [...byRoot.entries()].map(([project_root, activeCount]) => ({
+                project_root,
+                activeCount,
+                maxConcurrent: this.maxConcurrentPerProject,
+            })),
         };
     }
+    countActiveForProject(projectRoot) {
+        const normalized = normalizeCatalogPath(projectRoot);
+        let n = 0;
+        for (const entry of this.active.values()) {
+            const root = normalizeCatalogPath(entry.projectRoot ?? this.projectRoot);
+            if (root === normalized)
+                n += 1;
+        }
+        return n;
+    }
+    countActiveForCaller(callerId) {
+        const normalized = callerId.toLowerCase();
+        let n = 0;
+        for (const entry of this.active.values()) {
+            if (entry.callerId?.toLowerCase() === normalized)
+                n += 1;
+        }
+        return n;
+    }
+    callerQuotaLimit(callerId) {
+        if (callerId === null || callerId === undefined || callerId === "") {
+            return undefined;
+        }
+        return this.callerQuotas.get(callerId.toLowerCase());
+    }
+    /** Capacity plus on-demand durable-root disk breakdown (KTD16). */
+    async getHealthWithDisk() {
+        const base = this.getHealth();
+        try {
+            const disk = await durableRootDiskBreakdown(globalStageflowHome());
+            return { ...base, disk };
+        }
+        catch {
+            return {
+                ...base,
+                disk: {
+                    runs_bytes: 0,
+                    worktrees_bytes: 0,
+                    repos_bytes: 0,
+                    state_db_bytes: 0,
+                    a2a_artifacts_bytes: 0,
+                    cache_bytes: 0,
+                    free_bytes: 0,
+                },
+            };
+        }
+    }
+    /** Refresh cached `disk_bytes` for one run after a terminal transition. */
+    async refreshRunDiskBytes(runId) {
+        await refreshRunDiskUsage(this.options.store, runId);
+    }
     getHitlController() {
         return this.hitl;
     }
@@ -180,16 +531,27 @@ export class RunManager {
             }
             let checkoutKey;
             let durableCheckoutRoot;
+            let attachCallerId = null;
+            let attachProjectRoot;
             try {
                 const meta = await this.options.store.readRunMeta(runId);
-                const checkoutRoot = meta.checkout_root;
-                if (checkoutRoot !== undefined && checkoutRoot !== "") {
-                    durableCheckoutRoot = checkoutRoot;
-                    checkoutKey = await toCheckoutLeaseKey(checkoutRoot);
+                attachCallerId = meta.caller_id ?? null;
+                attachProjectRoot = meta.project_root
+                    ? normalizeCatalogPath(meta.project_root)
+                    : undefined;
+                const kind = derivedBindingKindFromMeta(meta);
+                if (kind === "checkout") {
+                    const checkoutRoot = meta.checkout_root;
+                    if (checkoutRoot !== undefined && checkoutRoot !== "") {
+                        durableCheckoutRoot = checkoutRoot;
+                        checkoutKey = await toCheckoutLeaseKey(checkoutRoot);
+                    }
                 }
             }
             catch (err) {
-                console.error(`invariant: attach failed reading checkout_root for run ${runId}; tracking without checkout lease: ${err instanceof Error ? err.message : String(err)}`);
+                log
+                    .child({ run_id: runId })
+                    .error("attach.checkout_root_failed", `invariant: attach failed reading checkout_root for run ${runId}; tracking without checkout lease: ${err instanceof Error ? err.message : String(err)}`);
             }
             const conflictHolder = this.findActiveCheckoutConflict(runId, checkoutKey, durableCheckoutRoot);
             if (conflictHolder !== undefined) {
@@ -203,6 +565,8 @@ export class RunManager {
             this.active.set(runId, {
                 checkoutKey,
                 durableCheckoutRoot,
+                projectRoot: attachProjectRoot,
+                callerId: attachCallerId,
                 generation: ++this.trackingGeneration,
             });
             for (const stage of waitingStages) {
@@ -224,27 +588,74 @@ export class RunManager {
         const reconciled = [];
         const runs = await this.options.store.listRuns();
         for (const summary of runs) {
+            let meta;
+            try {
+                meta = await this.options.store.readRunMeta(summary.run_id);
+            }
+            catch (err) {
+                log
+                    .child({ run_id: summary.run_id })
+                    .error("reconcile.read_meta_failed", `reconcileOrphanedStages: failed to read run meta ${summary.run_id}: ${err instanceof Error ? err.message : String(err)}`);
+                continue;
+            }
+            if (meta.status === "queued")
+                continue;
             let detail;
             try {
                 detail = await this.options.store.readRun(summary.run_id);
             }
             catch (err) {
-                console.error(`reconcileOrphanedStages: failed to read run ${summary.run_id}: ${err instanceof Error ? err.message : String(err)}`);
+                log
+                    .child({ run_id: summary.run_id })
+                    .error("reconcile.read_run_failed", `reconcileOrphanedStages: failed to read run ${summary.run_id}: ${err instanceof Error ? err.message : String(err)}`);
                 continue;
             }
             const runId = summary.run_id;
+            const cancelled = meta.status === "cancelled";
             let runChanged = false;
             for (const stage of detail.stages) {
+                if (cancelled) {
+                    if (stage.status !== "pending" &&
+                        stage.status !== "running" &&
+                        stage.status !== "waiting_for_input") {
+                        continue;
+                    }
+                    try {
+                        await markStageInterrupted({
+                            store: this.options.store,
+                            runId,
+                            stageId: stage.stage_id,
+                            reason: OPERATOR_CANCEL_REASON,
+                            status: "failed",
+                        });
+                        reconciled.push({
+                            runId,
+                            stageId: stage.stage_id,
+                            reason: OPERATOR_CANCEL_REASON,
+                        });
+                        runChanged = true;
+                        log
+                            .child({ run_id: runId, stage_id: stage.stage_id })
+                            .error("reconcile.orphaned_stage", `reconcileOrphanedStages: failed orphaned stage ${runId}/${stage.stage_id}: ${OPERATOR_CANCEL_REASON}`, { reason: OPERATOR_CANCEL_REASON });
+                    }
+                    catch (err) {
+                        log
+                            .child({ run_id: runId, stage_id: stage.stage_id })
+                            .error("reconcile.stage_failed", `reconcileOrphanedStages: failed to reconcile ${runId}/${stage.stage_id}: ${err instanceof Error ? err.message : String(err)}`);
+                    }
+                    continue;
+                }
                 if (stage.status !== "running")
                     continue;
                 if (this.hasActiveWorker(runId, stage.stage_id))
                     continue;
                 try {
-                    await failStageAsInterrupted({
+                    await markStageInterrupted({
                         store: this.options.store,
                         runId,
                         stageId: stage.stage_id,
                         reason: STARTUP_RECONCILE_REASON,
+                        status: "interrupted",
                     });
                     reconciled.push({
                         runId,
@@ -252,26 +663,106 @@ export class RunManager {
                         reason: STARTUP_RECONCILE_REASON,
                     });
                     runChanged = true;
-                    console.error(`reconcileOrphanedStages: failed orphaned stage ${runId}/${stage.stage_id}: ${STARTUP_RECONCILE_REASON}`);
+                    log
+                        .child({ run_id: runId, stage_id: stage.stage_id })
+                        .error("reconcile.orphaned_stage", `reconcileOrphanedStages: interrupted orphaned stage ${runId}/${stage.stage_id}: ${STARTUP_RECONCILE_REASON}`, { reason: STARTUP_RECONCILE_REASON });
                 }
                 catch (err) {
-                    console.error(`reconcileOrphanedStages: failed to reconcile ${runId}/${stage.stage_id}: ${err instanceof Error ? err.message : String(err)}`);
+                    log
+                        .child({ run_id: runId, stage_id: stage.stage_id })
+                        .error("reconcile.stage_failed", `reconcileOrphanedStages: failed to reconcile ${runId}/${stage.stage_id}: ${err instanceof Error ? err.message : String(err)}`);
                 }
             }
-            if (runChanged) {
+            if (runChanged && !cancelled) {
                 try {
                     await syncRunStatusFromStages(this.options.store, runId);
                 }
                 catch (err) {
-                    console.error(`reconcileOrphanedStages: failed to sync run status for ${runId}: ${err instanceof Error ? err.message : String(err)}`);
+                    log
+                        .child({ run_id: runId })
+                        .error("reconcile.sync_status_failed", `reconcileOrphanedStages: failed to sync run status for ${runId}: ${err instanceof Error ? err.message : String(err)}`);
                 }
             }
         }
         if (reconciled.length > 0) {
-            console.error(`reconcileOrphanedStages: reconciled ${reconciled.length} orphaned stage(s)`);
+            log.error("reconcile.complete", `reconcileOrphanedStages: reconciled ${reconciled.length} orphaned stage(s)`, { count: reconciled.length });
         }
         return { reconciled };
     }
+    async autoResumeInterruptedStages() {
+        const resumed = [];
+        const capped = [];
+        const skipped = [];
+        if (!isAutoResumeInterruptedEnabled()) {
+            return { resumed, capped, skipped };
+        }
+        const maxAutoResumes = parseMaxAutoResumes();
+        const interrupted = await this.options.store.listInterruptedStageExecutions();
+        for (const execution of interrupted) {
+            const runId = execution.run_id;
+            const stageId = execution.stage_id;
+            const attempt = execution.attempt;
+            const latest = await this.options.store.getLatestStageExecution(runId, stageId);
+            if (latest === null || latest.attempt !== attempt) {
+                skipped.push({
+                    runId,
+                    stageId,
+                    reason: "interrupted attempt is not the latest",
+                });
+                continue;
+            }
+            if (execution.auto_resume_count >= maxAutoResumes) {
+                try {
+                    await markStageInterrupted({
+                        store: this.options.store,
+                        runId,
+                        stageId,
+                        reason: AUTO_RESUME_CAPPED_REASON,
+                        status: "interrupted",
+                        attemptCtx: attemptContext(attempt),
+                    });
+                    capped.push({ runId, stageId });
+                    log
+                        .child({ run_id: runId, stage_id: stageId, attempt })
+                        .info("auto_resume.capped", `autoResumeInterruptedStages: capped ${runId}/${stageId} at ${execution.auto_resume_count}`, { auto_resume_count: execution.auto_resume_count });
+                }
+                catch (err) {
+                    skipped.push({
+                        runId,
+                        stageId,
+                        reason: err instanceof Error ? err.message : String(err),
+                    });
+                }
+                continue;
+            }
+            try {
+                await this.options.store.updateStageExecution(runId, stageId, attempt, {
+                    auto_resume_count: execution.auto_resume_count + 1,
+                });
+                const result = await this.resumeTimedOutStage(runId, stageId, {
+                    source: "auto",
+                });
+                if (result.ok) {
+                    resumed.push({ runId, stageId });
+                }
+                else {
+                    skipped.push({
+                        runId,
+                        stageId,
+                        reason: result.reason ?? "auto resume failed",
+                    });
+                }
+            }
+            catch (err) {
+                skipped.push({
+                    runId,
+                    stageId,
+                    reason: err instanceof Error ? err.message : String(err),
+                });
+            }
+        }
+        return { resumed, capped, skipped };
+    }
     async resumeStalledSchedules() {
         const resumed = [];
         const runs = await this.options.store.listRuns();
@@ -287,11 +778,37 @@ export class RunManager {
                 }
             }
             catch (err) {
-                console.error(`resumeStalledSchedules: failed to resume ${runId}: ${err instanceof Error ? err.message : String(err)}`);
+                log
+                    .child({ run_id: runId })
+                    .error("resume.stalled_failed", `resumeStalledSchedules: failed to resume ${runId}: ${err instanceof Error ? err.message : String(err)}`);
             }
         }
         return resumed;
     }
+    /** Rebuild in-memory admission queue from persisted `queued` rows (R27). */
+    async reenqueuePersistedQueuedRuns() {
+        const queued = await this.options.store.listRuns({ status: "queued" });
+        const ordered = [...queued].sort((a, b) => a.created_at.localeCompare(b.created_at));
+        this.admissionQueue.clear();
+        for (const row of ordered) {
+            const root = normalizeCatalogPath(row.project_root ?? this.projectRoot);
+            let callerId = null;
+            try {
+                const meta = await this.options.store.readRunMeta(row.run_id);
+                callerId = meta.caller_id ?? null;
+            }
+            catch {
+                /* leave null; drain stamps from meta on quota miss */
+            }
+            this.admissionQueue.enqueue(root, {
+                runId: row.run_id,
+                createdAt: row.created_at,
+                callerId,
+            });
+            this.ensureQueuedDone(row.run_id);
+        }
+        await this.drainAdmissionQueue();
+    }
     async abandonStage(runId, stageId) {
         let detail;
         try {
@@ -330,11 +847,12 @@ export class RunManager {
                 await this.stageProcessLauncher.cancelRun(runId);
             }
         }
-        await failStageAsInterrupted({
+        await markStageInterrupted({
             store: this.options.store,
             runId,
             stageId,
             reason: OPERATOR_ABANDON_REASON,
+            status: "failed",
         });
         if (this.retryCoordinator.isActive(runId) &&
             this.retryCoordinator.getActiveRoots(runId)?.has(stageId)) {
@@ -348,13 +866,250 @@ export class RunManager {
         }
         return { ok: true, runId, stageId };
     }
-    async startRun(input) {
-        const cwd = this.options.cwd ?? process.cwd();
-        // An inline pipeline has no filesystem anchor to derive a project root from.
-        const derivedProjectRoot = typeof input.pipeline === "string"
-            ? (findProjectRoot(path.dirname(path.resolve(cwd, input.pipeline))) ??
-                this.projectRoot)
-            : this.projectRoot;
+    async cancelRun(runId, reason) {
+        let meta;
+        try {
+            meta = await this.options.store.readRunMeta(runId);
+        }
+        catch {
+            return { ok: false, reason: `Run not found: ${runId}`, status: 404 };
+        }
+        if (meta.status === "cancelled") {
+            return { ok: true, runId };
+        }
+        if (meta.status === "succeeded" || meta.status === "failed") {
+            return {
+                ok: false,
+                reason: `Run is ${meta.status} and cannot be cancelled`,
+                status: 409,
+            };
+        }
+        const trimmedReason = reason.trim();
+        if (trimmedReason.length === 0) {
+            return {
+                ok: false,
+                reason: "Cancel reason is required",
+                status: 400,
+            };
+        }
+        await this.options.store.updateRunStatus(runId, "cancelled");
+        await this.options.store.setCancelReason(runId, trimmedReason);
+        await finaliseStoredRunManifest(this.options.store, runId).catch(() => undefined);
+        await this.refreshRunDiskBytes(runId).catch(() => undefined);
+        const halt = this.schedulingHalts.get(runId);
+        if (halt !== undefined) {
+            halt.halted = true;
+        }
+        this.admissionQueue.remove(runId);
+        this.pendingQueuedStarts.delete(runId);
+        this.resolveQueuedDone(runId, {
+            ok: false,
+            outcome: "cancelled",
+            runDir: this.options.store.getWorkspaceDir(runId),
+            runId,
+            reason: trimmedReason,
+        });
+        if (this.stageProcessLauncher !== undefined) {
+            await this.stageProcessLauncher.cancelRun(runId);
+        }
+        const detail = await this.options.store.readRun(runId);
+        for (const stage of detail.stages) {
+            if (stage.status !== "pending" &&
+                stage.status !== "running" &&
+                stage.status !== "waiting_for_input") {
+                continue;
+            }
+            const wasWaiting = stage.status === "waiting_for_input";
+            await markStageInterrupted({
+                store: this.options.store,
+                runId,
+                stageId: stage.stage_id,
+                reason: OPERATOR_CANCEL_REASON,
+                status: "failed",
+            });
+            if (wasWaiting) {
+                this.hitl.clearLiveWait(runId, stage.stage_id);
+            }
+        }
+        if (this.active.has(runId)) {
+            this.removeActiveEntry(runId, false);
+        }
+        return { ok: true, runId };
+    }
+    setA2aStore(store) {
+        this.a2aStore = store;
+    }
+    async deleteRun(runId, options = {
+        channel: "rest",
+    }) {
+        const force = options.force === true;
+        const channel = options.channel;
+        let meta;
+        try {
+            meta = await this.options.store.readRunMeta(runId);
+        }
+        catch {
+            return { ok: false, reason: `Run not found: ${runId}`, status: 404 };
+        }
+        const activeStatus = meta.status === "created" ||
+            meta.status === "queued" ||
+            meta.status === "running";
+        if (activeStatus && !force) {
+            return {
+                ok: false,
+                reason: `Run is ${meta.status} and cannot be deleted without force`,
+                status: 409,
+            };
+        }
+        if (activeStatus && force) {
+            const settle = this.active.get(runId)?.done;
+            const cancelled = await this.cancelRun(runId, FORCE_DELETE_CANCEL_REASON);
+            if (!cancelled.ok && cancelled.status !== 404) {
+                return cancelled;
+            }
+            if (settle !== undefined) {
+                try {
+                    await this.waitForPromise(settle, 30_000);
+                }
+                catch (err) {
+                    return {
+                        ok: false,
+                        reason: err instanceof Error ? err.message : String(err),
+                        status: 500,
+                    };
+                }
+            }
+        }
+        const a2aStore = this.a2aStore;
+        if (a2aStore === undefined) {
+            return {
+                ok: false,
+                reason: "A2A store is not available for delete_run",
+                status: 500,
+            };
+        }
+        try {
+            await deleteRunEverywhere(this.options.store, a2aStore, runId);
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            if (message.startsWith("Run not found:")) {
+                return { ok: false, reason: message, status: 404 };
+            }
+            return { ok: false, reason: message, status: 500 };
+        }
+        log.child({ run_id: runId }).info("delete_run", "run deleted", {
+            force,
+            channel,
+        });
+        return { ok: true, runId };
+    }
+    async gcRuns(options = {}) {
+        const execute = options.execute === true;
+        const channel = options.channel ?? "rest";
+        let report;
+        try {
+            report = await runRetentionSweep(this.options.store, this.a2aStore, {
+                execute,
+                now: options.now,
+                windows: options.windows,
+                env: options.env,
+                artifactMaxBytes: options.artifactMaxBytes,
+                bareCacheTtlMs: options.bareCacheTtlMs,
+            });
+        }
+        catch (err) {
+            return {
+                ok: false,
+                reason: err instanceof Error ? err.message : String(err),
+                status: 500,
+            };
+        }
+        if (execute) {
+            for (const runId of report.slimmed) {
+                await this.refreshRunDiskBytes(runId).catch(() => undefined);
+            }
+            log.info("run_retention_sweep", "retention sweep executed", {
+                channel,
+                slimmed: report.slimmed,
+                purged: report.purged,
+                bareCachesEvicted: report.bareCachesEvicted,
+            });
+        }
+        return { ok: true, ...report };
+    }
+    async waitForPromise(promise, timeoutMs) {
+        let timer;
+        try {
+            await Promise.race([
+                promise.then(() => undefined, () => undefined),
+                new Promise((_, reject) => {
+                    timer = setTimeout(() => {
+                        reject(new Error(`Timed out waiting for run to settle after force cancel`));
+                    }, timeoutMs);
+                }),
+            ]);
+        }
+        finally {
+            if (timer !== undefined)
+                clearTimeout(timer);
+        }
+    }
+    async startRunOnce(input, submission) {
+        if (!submission.key.trim() || submission.key.length > 256 || !/^[a-f0-9]{64}$/.test(submission.requestHash)) {
+            return { ok: false, status: 400, reason: "Submission requires a key and SHA-256 request hash" };
+        }
+        const pending = this.submissionsInFlight.get(submission.key);
+        if (pending) {
+            if (pending.requestHash !== submission.requestHash) {
+                return { ok: false, status: 409, reason: "Submission key was already used for different input" };
+            }
+            const result = await pending.result;
+            return result.ok ? { ...result, reused: true } : result;
+        }
+        const result = this.startSubmittedRun(input, submission);
+        this.submissionsInFlight.set(submission.key, { requestHash: submission.requestHash, result });
+        try {
+            return await result;
+        }
+        finally {
+            this.submissionsInFlight.delete(submission.key);
+        }
+    }
+    async startSubmittedRun(input, submission) {
+        const existing = await this.options.store.getRunBySubmission(submission.key);
+        if (existing)
+            return existingSubmissionResult(existing, submission);
+        try {
+            const result = await this.startRun(input, submission);
+            return result.ok ? { ok: true, runId: result.runId, reused: false, done: result.done } : result;
+        }
+        catch (error) {
+            if (error instanceof RunSubmissionExistsError)
+                return existingSubmissionResult(error.submission, submission);
+            throw error;
+        }
+    }
+    async startRun(input, submission) {
+        if (!this.acceptingWork) {
+            return {
+                ok: false,
+                reason: "Host is shutting down",
+                status: 503,
+                code: "shutting_down",
+            };
+        }
+        const hostCwd = this.options.cwd ?? process.cwd();
+        const wireRoot = typeof input.projectRoot === "string" && input.projectRoot.trim().length > 0
+            ? normalizeProjectRoot(input.projectRoot)
+            : undefined;
+        // Wire root wins. Otherwise path pipelines may derive via findProjectRoot;
+        // inline-only without projectRoot keeps Host projectRoot.
+        const derivedProjectRoot = wireRoot ??
+            (typeof input.pipeline === "string"
+                ? (findProjectRoot(path.dirname(path.resolve(hostCwd, input.pipeline))) ?? this.projectRoot)
+                : this.projectRoot);
+        const cwd = wireRoot ?? hostCwd;
         let resolved;
         try {
             resolved = resolveStartTaskInput(input, cwd);
@@ -389,24 +1144,57 @@ export class RunManager {
             gitSha: input.gitSha,
             ciPrUrl: input.ciPrUrl,
             ciJobUrl: input.ciJobUrl,
-        }, derivedProjectRoot, resolved.kind === "path" ? resolved.taskPath : undefined);
+        }, derivedProjectRoot, resolved.kind === "path" ? resolved.taskPath : undefined, submission, undefined, input.callerId, input.skills);
     }
-    async rerun(runId) {
+    async rerun(runId, options) {
+        if (!this.acceptingWork) {
+            return {
+                ok: false,
+                reason: "Host is shutting down",
+                status: 503,
+                code: "shutting_down",
+            };
+        }
         const cwd = this.options.cwd ?? process.cwd();
         let pipeline;
         let taskYaml;
         let rerunCwd = cwd;
         let rerunProjectRoot;
+        let skipGates;
+        let ciIdentity;
+        let pinned;
         try {
             const meta = await this.options.store.readRunMeta(runId);
-            if (!meta.pipeline_path) {
+            if (meta.pipeline_source === "inline") {
+                const body = await this.options.store.readPipelineBody(runId);
+                if (body === null || body === "") {
+                    return {
+                        ok: false,
+                        reason: `Run ${runId} is missing pipeline_body; re-run of an inline pipeline requires the stored body.`,
+                        status: 400,
+                    };
+                }
+                try {
+                    pipeline = JSON.parse(body);
+                }
+                catch {
+                    return {
+                        ok: false,
+                        reason: `Run ${runId} has invalid pipeline_body JSON`,
+                        status: 400,
+                    };
+                }
+            }
+            else if (meta.pipeline_path) {
+                pipeline = normalizeCatalogPath(meta.pipeline_path);
+            }
+            else {
                 return {
                     ok: false,
-                    reason: `Run ${runId} is missing pipeline_path; re-run requires stored catalog locators.`,
+                    reason: `Run ${runId} has neither pipeline_path nor pipeline_body; cannot re-run.`,
                     status: 400,
                 };
             }
-            pipeline = normalizeCatalogPath(meta.pipeline_path);
             rerunCwd = meta.project_root
                 ? normalizeCatalogPath(meta.project_root)
                 : cwd;
@@ -414,16 +1202,44 @@ export class RunManager {
                 ? normalizeCatalogPath(meta.project_root)
                 : undefined;
             taskYaml = await this.options.store.readTaskYaml(runId);
+            skipGates = meta.skip_gates;
+            ciIdentity = {
+                ...(meta.git_sha !== undefined ? { gitSha: meta.git_sha } : {}),
+                ...(meta.ci_pr_url !== undefined ? { ciPrUrl: meta.ci_pr_url } : {}),
+                ...(meta.ci_job_url !== undefined ? { ciJobUrl: meta.ci_job_url } : {}),
+            };
+            if (options?.pinned) {
+                if (meta.resolved_sha === undefined ||
+                    meta.resolved_sha === "" ||
+                    meta.ref === undefined ||
+                    meta.ref === "") {
+                    return {
+                        ok: false,
+                        reason: `Run ${runId} has no resolved_sha/ref to pin`,
+                        status: 400,
+                        code: "pinned_sha_unavailable",
+                    };
+                }
+                pinned = { ref: meta.ref, resolvedSha: meta.resolved_sha };
+            }
         }
         catch {
             return { ok: false, reason: `Run not found: ${runId}`, status: 404 };
         }
-        return this.reserveAndStartPipeline(taskYaml, pipeline, `run ${runId} task`, rerunCwd, undefined, undefined, undefined, rerunProjectRoot);
+        return this.reserveAndStartPipeline(taskYaml, pipeline, `run ${runId} task`, rerunCwd, undefined, skipGates, ciIdentity, rerunProjectRoot, undefined, undefined, pinned, options?.callerId);
     }
     async retryStage(runId, stageId) {
         return this.retryStageInternal(runId, stageId, true);
     }
-    async resumeTimedOutStage(runId, stageId) {
+    async resumeTimedOutStage(runId, stageId, options) {
+        if (!this.acceptingWork) {
+            return {
+                ok: false,
+                reason: "Host is shutting down",
+                status: 503,
+                code: "shutting_down",
+            };
+        }
         const resumeKey = waitKey(runId, stageId);
         if (this.resumeInFlight.has(resumeKey) || this.retryInFlight.has(resumeKey)) {
             return {
@@ -446,13 +1262,16 @@ export class RunManager {
             this.resumeInFlight.delete(resumeKey);
             return { ok: false, reason: `Stage not found: ${stageId}`, status: 404 };
         }
-        const eligibility = assertTimedOutStageEligible(stageSnap.status, stageSnap.events);
+        const eligibility = assertResumableStage(stageSnap.status, stageSnap.events);
         if (!eligibility.ok) {
             this.resumeInFlight.delete(resumeKey);
             return eligibility;
         }
         const latest = await this.options.store.getLatestStageExecution(runId, stageId);
         const attemptIndex = latest?.attempt ?? 1;
+        if ((options?.source ?? "explicit") === "explicit" && latest !== null) {
+            await this.options.store.updateStageExecution(runId, stageId, attemptIndex, { auto_resume_count: 0 });
+        }
         const wasActive = this.active.has(runId);
         const tracked = await this.ensureResumeTracked(runId);
         if (!tracked.ok) {
@@ -570,14 +1389,23 @@ export class RunManager {
         finally {
             this.retryInFlight.delete(retryKey);
         }
-    }
-    async retryStageInternal(runId, stageId, awaitRoot, options) {
+    }
+    async retryStageInternal(runId, stageId, awaitRoot, options) {
+        if (!this.acceptingWork) {
+            return {
+                ok: false,
+                reason: "Host is shutting down",
+                status: 503,
+                code: "shutting_down",
+            };
+        }
         const retryKey = waitKey(runId, stageId);
         if (this.retryInFlight.has(retryKey)) {
             return {
                 ok: false,
                 reason: `Retry already in progress for run ${runId} stage ${stageId}`,
                 status: 409,
+                code: "retry_in_progress",
             };
         }
         this.retryInFlight.add(retryKey);
@@ -588,6 +1416,7 @@ export class RunManager {
                 ok: false,
                 reason: "Stage uses manual recovery; use the manual recovery action instead of retry",
                 status: 409,
+                code: "manual_recovery_required",
             };
         }
         try {
@@ -1016,6 +1845,7 @@ export class RunManager {
                 maxActiveStagesPerRun: this.maxActiveStagesPerRun,
                 executionMode: this.executionMode,
                 stageProcessLauncher: this.stageProcessLauncher,
+                schedulingHalt: this.ensureSchedulingHalt(runId),
             });
             this.registerResumeUntrack(runId, done);
             const rest = await done;
@@ -1060,6 +1890,13 @@ export class RunManager {
         try {
             const runMeta = await store.readRunMeta(runId);
             const runProjectRoot = runMeta.project_root ?? this.projectRoot;
+            const { meta, task, loaded } = await loadRunContext(store, runId, runProjectRoot);
+            const stageBinding = stageBindingEnvFromRun({
+                meta,
+                task,
+                runWorkspaceDir: workspaceDir,
+                hostEnv: process.env,
+            });
             const launchResult = await launcher.launch({
                 runId,
                 stageId,
@@ -1068,6 +1905,8 @@ export class RunManager {
                 resumeAnswer: opaqueAnswer,
                 attempt,
                 sessionFilePath,
+                env: stageBinding.env,
+                bindingKind: stageBinding.kind,
                 ...(this.options.operatorCatalog !== undefined
                     ? { operatorCatalog: this.options.operatorCatalog }
                     : {}),
@@ -1090,7 +1929,6 @@ export class RunManager {
             catch {
                 // downstream resume may read envelope from store
             }
-            const { meta, task, loaded } = await loadRunContext(store, runId, this.cwd);
             const rest = await resumeRun({
                 prepared: {
                     task,
@@ -1109,6 +1947,7 @@ export class RunManager {
                 initialPrior,
                 executionMode: this.executionMode,
                 stageProcessLauncher: launcher,
+                schedulingHalt: this.ensureSchedulingHalt(runId),
             });
             if (rest.outcome === "failed") {
                 return { ok: false, reason: rest.reason };
@@ -1156,36 +1995,273 @@ export class RunManager {
             this.attachedWaiting.delete(`${runId}\0${stageId}`);
         }
     }
-    async reserveAndStartPipeline(taskYaml, pipeline, taskLabel, cwd, checkoutOverride, skipGates, ciIdentity, projectRoot, taskPath) {
+    async reserveAndStartPipeline(taskYaml, pipeline, taskLabel, cwd, checkoutOverride, skipGates, ciIdentity, projectRoot, taskPath, submission, pinned, callerId, skills) {
+        const persistence = pipelinePersistenceForStart(pipeline);
+        if (!persistence.ok) {
+            return {
+                ok: false,
+                reason: `Inline pipeline body is ${persistence.bytes} bytes; max is ${persistence.maxBytes}`,
+                status: 400,
+                code: INLINE_PIPELINE_TOO_LARGE,
+            };
+        }
+        const skillsCheck = validateSkillsPayload(skills, {
+            pipelineBytes: pipelineBodyBytes(persistence.fields),
+        });
+        if (!skillsCheck.ok) {
+            return {
+                ok: false,
+                reason: skillsCheck.reason,
+                status: 400,
+                code: skillsCheck.code,
+            };
+        }
+        const validatedSkills = skillsCheck.skills;
+        const hasSkills = Object.keys(validatedSkills).length > 0;
+        const resolvedProjectRoot = normalizeCatalogPath(projectRoot ?? this.projectRoot);
+        let task;
         let checkoutKey;
+        let pathCheckoutRoot;
+        let binding;
+        let pipelineId;
+        let pipelineDag;
+        let pipelinePath;
+        let loadedPipeline;
+        let toolchainChecks = [];
         try {
-            const task = loadTaskFromYaml(taskYaml, taskLabel);
-            const checkoutRoot = await resolveAndValidateCheckout(task, checkoutOverride, cwd);
-            checkoutKey =
-                checkoutRoot !== undefined
-                    ? await toCheckoutLeaseKey(checkoutRoot)
+            const loadedTask = loadTaskFromYamlOutcome(taskYaml, taskLabel);
+            if (!loadedTask.ok) {
+                const issue = loadedTask.issues[0];
+                return {
+                    ok: false,
+                    reason: issue?.message ?? "Invalid task",
+                    status: 400,
+                    ...(issue?.code !== undefined
+                        ? { code: issue.code }
+                        : {}),
+                };
+            }
+            task = loadedTask.value;
+            const bindingOutcome = resolveWorkspaceBinding(task, { checkoutOverride });
+            if (!bindingOutcome.ok) {
+                const issue = bindingOutcome.issues[0];
+                return {
+                    ok: false,
+                    reason: issue?.message ?? "Invalid workspace binding",
+                    status: 400,
+                    ...(issue?.code !== undefined
+                        ? { code: issue.code }
+                        : {}),
+                };
+            }
+            binding = bindingOutcome.value;
+            if (binding.kind === "checkout") {
+                pathCheckoutRoot = await resolveAndValidateCheckout(task, checkoutOverride, cwd);
+                checkoutKey =
+                    pathCheckoutRoot !== undefined
+                        ? await toCheckoutLeaseKey(pathCheckoutRoot)
+                        : undefined;
+            }
+            const loadResult = await loadPipelineValidated(pipeline, {
+                cwd,
+                projectRoot: resolvedProjectRoot,
+                validateStages: true,
+            });
+            if (!loadResult.ok) {
+                throw new PipelineValidationError(buildValidationResult("pipeline", loadResult.findings, false));
+            }
+            loadedPipeline = loadResult.loaded;
+            const pairing = checkTaskEntryInput(task, loadResult.loaded, {
+                cwd,
+                taskPath,
+            });
+            if (pairing.some((finding) => finding.severity === "error")) {
+                throw new PipelineValidationError(buildValidationResult("pipeline", pairing, false));
+            }
+            pipelineId = loadResult.loaded.pipeline.id;
+            pipelineDag = buildPipelineDagSnapshotFromLoaded(loadResult.loaded);
+            pipelinePath =
+                typeof pipeline === "string"
+                    ? normalizeCatalogPath(loadResult.loaded.pipelinePath)
                     : undefined;
+            try {
+                const pipelineAgent = asAgentBackendId(loadResult.loaded.pipeline.agent);
+                assertClaudeNotRoot({ backendId: pipelineAgent });
+                for (const stage of loadResult.loaded.stages) {
+                    assertClaudeNotRoot({
+                        backendId: asAgentBackendId(stage.agent) ?? pipelineAgent,
+                    });
+                }
+            }
+            catch (err) {
+                if (err instanceof ClaudeRootError) {
+                    return { ok: false, reason: err.message, status: 400, code: err.code };
+                }
+                throw err;
+            }
+            const preflight = await runPipelinePreflight(loadResult.loaded, {
+                projectRoot: resolvedProjectRoot,
+                forStart: true,
+            });
+            if (!preflight.ok) {
+                const code = preflightFailureCode(preflight, { forStart: true }) ??
+                    "missing_tool";
+                return {
+                    ok: false,
+                    reason: `Pipeline preflight failed: ${code}`,
+                    status: 400,
+                    code,
+                };
+            }
+            toolchainChecks = preflight.toolchain.checks;
         }
         catch (err) {
+            if (err instanceof PipelineValidationError)
+                throw err;
+            if (err instanceof PipelinePreflightError) {
+                return {
+                    ok: false,
+                    reason: err.message,
+                    status: 400,
+                    code: err.code,
+                };
+            }
             return {
                 ok: false,
                 reason: err instanceof Error ? err.message : String(err),
                 status: 400,
             };
         }
-        const reserved = this.tryReserve(checkoutKey);
-        if (!reserved.ok)
-            return reserved.failure;
+        const diskGate = await this.checkDiskFloorAdmission();
+        if (diskGate !== undefined)
+            return diskGate;
+        const admitted = this.tryAdmitOrEnqueue(checkoutKey, resolvedProjectRoot, callerId);
+        if (admitted.action === "reject")
+            return admitted.failure;
+        if (admitted.action === "enqueue") {
+            const runId = newRunId();
+            const gitIdentity = resolveEffectiveGitIdentity(process.env, task.git_identity);
+            const createdAt = new Date().toISOString();
+            const auth = getRequestAuth();
+            const surface = auth?.surface ?? "cli";
+            const secretNames = loadedPipeline.stages.flatMap((s) => (s.secrets ?? []).map((d) => d.name));
+            const namedSecrets = collectDeclaredSecretValues(secretNames).filter((s) => shouldRegisterValue(s.value));
+            if (namedSecrets.length > 0) {
+                registerNamedSecrets(namedSecrets);
+            }
+            const runManifest = buildInitialRunManifest({
+                runId,
+                createdAt,
+                callerId: callerId ?? null,
+                surface,
+                binding: bindingFromFields({}),
+                pipelineSource: persistence.fields.pipelineSource,
+                pipelinePath,
+                pipelineBody: persistence.fields.pipelineBody ?? null,
+                taskYaml,
+                taskPath: taskPath
+                    ? normalizeCatalogPath(path.resolve(cwd, taskPath))
+                    : undefined,
+                skills: hasSkills ? validatedSkills : undefined,
+                stages: loadedPipeline.stages,
+                toolchain: toolchainChecks,
+                namedSecrets,
+            });
+            const created = await this.options.store.createRun({
+                submission,
+                runId,
+                pipelineId,
+                taskYaml,
+                taskId: task.id,
+                pipelineDag,
+                pipelinePath,
+                taskPath: taskPath
+                    ? normalizeCatalogPath(path.resolve(cwd, taskPath))
+                    : undefined,
+                projectRoot: resolvedProjectRoot,
+                gitSha: ciIdentity?.gitSha,
+                ciPrUrl: ciIdentity?.ciPrUrl,
+                ciJobUrl: ciIdentity?.ciJobUrl,
+                gitAuthorName: gitIdentity.name,
+                gitAuthorEmail: gitIdentity.email,
+                status: "queued",
+                pipelineSource: persistence.fields.pipelineSource,
+                ...(persistence.fields.pipelineBody !== undefined
+                    ? { pipelineBody: persistence.fields.pipelineBody }
+                    : {}),
+                ...(callerId !== undefined && callerId !== null
+                    ? { callerId }
+                    : {}),
+                runManifest,
+                skipGates,
+            });
+            if (hasSkills) {
+                await materializeRunSkills(created.workspaceDir, validatedSkills);
+            }
+            const meta = await this.options.store.readRunMeta(created.runId);
+            const queuePosition = this.admissionQueue.enqueue(resolvedProjectRoot, {
+                runId: created.runId,
+                createdAt: meta.created_at,
+                callerId: callerId ?? meta.caller_id ?? null,
+            });
+            this.pendingQueuedStarts.set(created.runId, {
+                taskYaml,
+                pipeline,
+                taskLabel,
+                cwd,
+                projectRoot: resolvedProjectRoot,
+                checkoutOverride,
+                skipGates,
+                ciIdentity,
+                taskPath,
+                submission,
+                pinned,
+                pathCheckoutRoot,
+                binding,
+                checkoutKey,
+                callerId,
+                ...(hasSkills ? { skills: validatedSkills } : {}),
+            });
+            const done = this.ensureQueuedDone(created.runId);
+            return {
+                ok: true,
+                runId: created.runId,
+                done,
+                queued: true,
+                queuePosition,
+                ...(admitted.reason === "caller_quota"
+                    ? { queuedCode: "busy_caller_quota" }
+                    : {}),
+            };
+        }
+        const runId = newRunId();
+        let rollback = async () => { };
         try {
+            const { materialized, rollback: linkRollback } = await materializeWorkspaceBinding({
+                runId,
+                task,
+                binding,
+                checkoutRoot: pathCheckoutRoot,
+                pinned,
+            });
+            rollback = linkRollback;
+            const schedulingHalt = this.ensureSchedulingHalt(materialized.runId);
             const started = await startPipeline({
+                submission,
                 agent: this.options.agent,
                 store: this.options.store,
                 taskYaml,
                 taskPath,
                 pipeline,
                 cwd,
-                projectRoot: projectRoot ?? this.projectRoot,
+                projectRoot: resolvedProjectRoot,
                 checkoutOverride,
+                runId: materialized.runId,
+                checkoutRoot: materialized.checkoutRoot,
+                repository: materialized.repository,
+                ref: materialized.ref,
+                resolvedSha: materialized.resolvedSha,
+                runBranch: materialized.runBranch,
                 gitSha: ciIdentity?.gitSha,
                 ciPrUrl: ciIdentity?.ciPrUrl,
                 ciJobUrl: ciIdentity?.ciJobUrl,
@@ -1195,15 +2271,44 @@ export class RunManager {
                 stageProcessLauncher: this.stageProcessLauncher,
                 operatorCatalog: this.options.operatorCatalog,
                 skipGates,
+                schedulingHalt,
+                callerId,
+                ...(hasSkills ? { skills: validatedSkills } : {}),
             });
-            this.track(reserved.provisionalId, started.runId, started.done);
+            this.track(admitted.provisionalId, started.runId, started.done);
             return { ok: true, runId: started.runId, done: started.done };
         }
         catch (err) {
-            this.clearReservation(reserved.provisionalId);
-            if (err instanceof PipelineValidationError) {
+            await rollback().catch(() => undefined);
+            this.clearReservation(admitted.provisionalId);
+            if (err instanceof InlinePipelineTooLargeError) {
+                return {
+                    ok: false,
+                    reason: err.message,
+                    status: 400,
+                    code: err.code,
+                };
+            }
+            if (err instanceof PipelinePreflightError) {
+                return {
+                    ok: false,
+                    reason: err.message,
+                    status: 400,
+                    code: err.code,
+                };
+            }
+            if (err instanceof PipelineValidationError || err instanceof RunSubmissionExistsError) {
                 throw err;
             }
+            if (err instanceof StartLinkError) {
+                return {
+                    ok: false,
+                    reason: err.message,
+                    status: err.status,
+                    code: err.code,
+                    ...(err.stderr !== undefined ? { stderr: err.stderr } : {}),
+                };
+            }
             return {
                 ok: false,
                 reason: err instanceof Error ? err.message : String(err),
@@ -1213,24 +2318,87 @@ export class RunManager {
     }
     busyFailure(code, extras) {
         const activeRunIds = this.getActiveRunIds();
-        const reason = code === "busy_capacity"
-            ? `Capacity full: ${this.active.size}/${this.maxConcurrent} active runs`
-            : `Checkout in use by run ${extras?.conflictingRunId ?? "unknown"}`;
+        const scope = extras?.scope ?? (code === "busy_caller_quota" ? "caller" : "global");
+        const activeCount = extras?.activeCount ?? this.active.size;
+        const maxConcurrent = extras?.maxConcurrent ?? this.maxConcurrent;
+        const reason = code === "busy_caller_quota"
+            ? `Caller quota full: ${activeCount}/${maxConcurrent} active runs for caller ${extras?.caller_id ?? "unknown"}`
+            : code === "busy_capacity"
+                ? scope === "project"
+                    ? `Project capacity full: ${activeCount}/${maxConcurrent} active runs for ${extras?.project_root ?? "project"}`
+                    : this.active.size >= this.maxConcurrent &&
+                        this.admissionQueue.size >= this.maxQueued &&
+                        this.maxQueued > 0
+                        ? `Admission queue full: ${this.admissionQueue.size}/${this.maxQueued} queued runs`
+                        : `Capacity full: ${this.active.size}/${this.maxConcurrent} active runs`
+                : `Checkout in use by run ${extras?.conflictingRunId ?? "unknown"}`;
         return {
             ok: false,
             reason,
             status: 409,
             code,
-            activeCount: this.active.size,
-            maxConcurrent: this.maxConcurrent,
+            activeCount,
+            maxConcurrent,
             activeRunIds,
-            ...extras,
+            scope,
+            ...(extras?.project_root !== undefined
+                ? { project_root: extras.project_root }
+                : {}),
+            ...(extras?.caller_id !== undefined ? { caller_id: extras.caller_id } : {}),
+            ...(extras?.conflictingRunId !== undefined
+                ? { conflictingRunId: extras.conflictingRunId }
+                : {}),
+            ...(extras?.conflictingCheckout !== undefined
+                ? { conflictingCheckout: extras.conflictingCheckout }
+                : {}),
         };
     }
-    tryReserve(checkoutKey) {
-        if (this.active.size >= this.maxConcurrent) {
-            return { ok: false, failure: this.busyFailure("busy_capacity") };
+    insufficientDiskFailure(freeBytes, minFreeBytes) {
+        return {
+            ok: false,
+            reason: `Insufficient free disk: ${freeBytes} bytes free, floor ${minFreeBytes} bytes`,
+            status: 409,
+            code: "insufficient_disk",
+            freeBytes,
+            minFreeBytes,
+        };
+    }
+    async checkDiskFloorAdmission() {
+        try {
+            const readFree = this.options.freeSpaceReader ?? readFilesystemSize;
+            const size = await readFree(globalStageflowHome());
+            const minFreeBytes = resolveMinFreeDiskFloor(process.env.STAGEFLOW_MIN_FREE_DISK_BYTES, size.totalBytes);
+            if (size.freeBytes < minFreeBytes) {
+                return this.insufficientDiskFailure(size.freeBytes, minFreeBytes);
+            }
+            return undefined;
+        }
+        catch (err) {
+            // Default floor is always active via resolveMinFreeDiskFloor — fail closed.
+            return {
+                ok: false,
+                reason: `Disk free-space check failed: ${err instanceof Error ? err.message : String(err)}`,
+                status: 503,
+                code: "disk_check_failed",
+            };
+        }
+    }
+    async isProjectRootAvailable(projectRoot) {
+        const provider = this.options.knownWritableProjectRoots;
+        if (provider !== undefined) {
+            const roots = [...(await provider())].map((r) => normalizeCatalogPath(r));
+            const normalized = normalizeCatalogPath(projectRoot);
+            return roots.includes(normalized);
+        }
+        try {
+            await access(projectRoot, constants.W_OK);
+            return true;
+        }
+        catch {
+            return false;
         }
+    }
+    tryAdmitOrEnqueue(checkoutKey, projectRoot = this.projectRoot, callerId) {
         if (checkoutKey !== undefined) {
             const holder = this.checkoutLeases.get(checkoutKey);
             if (holder !== undefined) {
@@ -1238,7 +2406,7 @@ export class RunManager {
                     ? undefined
                     : holder;
                 return {
-                    ok: false,
+                    action: "reject",
                     failure: this.busyFailure("busy_checkout", {
                         conflictingRunId,
                         conflictingCheckout: checkoutKey,
@@ -1246,16 +2414,378 @@ export class RunManager {
                 };
             }
         }
-        const provisionalId = randomUUID();
-        this.provisionalIds.add(provisionalId);
-        this.active.set(provisionalId, {
-            checkoutKey,
-            generation: ++this.trackingGeneration,
+        const normalizedRoot = normalizeCatalogPath(projectRoot);
+        if (this.maxConcurrentPerProject !== undefined) {
+            const projectActive = this.countActiveForProject(normalizedRoot);
+            if (projectActive >= this.maxConcurrentPerProject) {
+                return {
+                    action: "reject",
+                    failure: this.busyFailure("busy_capacity", {
+                        scope: "project",
+                        project_root: normalizedRoot,
+                        activeCount: projectActive,
+                        maxConcurrent: this.maxConcurrentPerProject,
+                    }),
+                };
+            }
+        }
+        if (this.active.size < this.maxConcurrent) {
+            const quota = this.callerQuotaLimit(callerId);
+            if (quota !== undefined &&
+                callerId !== undefined &&
+                callerId !== null &&
+                this.countActiveForCaller(callerId) >= quota) {
+                if (this.admissionQueue.size < this.maxQueued) {
+                    return { action: "enqueue", reason: "caller_quota" };
+                }
+                return {
+                    action: "reject",
+                    failure: this.busyFailure("busy_caller_quota", {
+                        scope: "caller",
+                        caller_id: callerId,
+                        activeCount: this.countActiveForCaller(callerId),
+                        maxConcurrent: quota,
+                    }),
+                };
+            }
+            const provisionalId = randomUUID();
+            this.provisionalIds.add(provisionalId);
+            this.active.set(provisionalId, {
+                checkoutKey,
+                projectRoot: normalizedRoot,
+                callerId: callerId ?? null,
+                generation: ++this.trackingGeneration,
+            });
+            if (checkoutKey !== undefined) {
+                this.checkoutLeases.set(checkoutKey, provisionalId);
+            }
+            return { action: "reserve", provisionalId };
+        }
+        if (this.admissionQueue.size < this.maxQueued) {
+            return { action: "enqueue", reason: "capacity" };
+        }
+        return {
+            action: "reject",
+            failure: this.busyFailure("busy_capacity", { scope: "global" }),
+        };
+    }
+    tryReserve(checkoutKey, projectRoot = this.projectRoot, callerId) {
+        const admitted = this.tryAdmitOrEnqueue(checkoutKey, projectRoot, callerId);
+        if (admitted.action === "reserve") {
+            return { ok: true, provisionalId: admitted.provisionalId };
+        }
+        if (admitted.action === "enqueue") {
+            if (admitted.reason === "caller_quota") {
+                const quota = this.callerQuotaLimit(callerId) ?? 0;
+                return {
+                    ok: false,
+                    failure: this.busyFailure("busy_caller_quota", {
+                        scope: "caller",
+                        caller_id: callerId ?? undefined,
+                        activeCount: callerId !== undefined && callerId !== null
+                            ? this.countActiveForCaller(callerId)
+                            : 0,
+                        maxConcurrent: quota,
+                    }),
+                };
+            }
+            return { ok: false, failure: this.busyFailure("busy_capacity") };
+        }
+        return { ok: false, failure: admitted.failure };
+    }
+    ensureQueuedDone(runId) {
+        const existing = this.queuedDone.get(runId);
+        if (existing !== undefined)
+            return existing.promise;
+        let resolve;
+        const promise = new Promise((res) => {
+            resolve = res;
         });
-        if (checkoutKey !== undefined) {
-            this.checkoutLeases.set(checkoutKey, provisionalId);
+        this.queuedDone.set(runId, { promise, resolve });
+        return promise;
+    }
+    resolveQueuedDone(runId, result) {
+        const deferred = this.queuedDone.get(runId);
+        if (deferred === undefined)
+            return;
+        this.queuedDone.delete(runId);
+        deferred.resolve(result);
+    }
+    attachQueuedDone(runId, done) {
+        void done.then((result) => this.resolveQueuedDone(runId, result), (err) => this.resolveQueuedDone(runId, {
+            ok: false,
+            outcome: "failed",
+            runDir: this.options.store.getWorkspaceDir(runId),
+            runId,
+            reason: err instanceof Error ? err.message : String(err),
+        }));
+    }
+    async drainAdmissionQueue() {
+        if (!this.acceptingWork)
+            return;
+        if (this.admissionDrainInFlight)
+            return;
+        this.admissionDrainInFlight = true;
+        try {
+            const blockedRoots = new Set();
+            const blockedCallers = new Set();
+            while (this.acceptingWork && this.active.size < this.maxConcurrent) {
+                const next = this.admissionQueue.dequeueNext(blockedRoots, blockedCallers);
+                if (next === undefined)
+                    break;
+                const outcome = await this.startDequeuedAdmission(next, blockedCallers);
+                if (outcome === "checkout_busy") {
+                    this.admissionQueue.requeueFront(next.projectRoot, next.entry);
+                    blockedRoots.add(next.projectRoot);
+                    continue;
+                }
+                if (outcome === "project_capacity_busy") {
+                    blockedRoots.add(next.projectRoot);
+                    continue;
+                }
+                if (outcome === "caller_quota_busy") {
+                    continue;
+                }
+                if (outcome === "capacity_busy") {
+                    break;
+                }
+            }
+        }
+        finally {
+            this.admissionDrainInFlight = false;
+        }
+    }
+    async startDequeuedAdmission(next, blockedCallers) {
+        if (!this.acceptingWork) {
+            this.admissionQueue.requeueFront(next.projectRoot, next.entry);
+            return "capacity_busy";
+        }
+        const { runId } = next.entry;
+        let meta;
+        try {
+            meta = await this.options.store.readRunMeta(runId);
+        }
+        catch {
+            this.pendingQueuedStarts.delete(runId);
+            return "cancelled";
+        }
+        if (meta.status !== "queued") {
+            this.pendingQueuedStarts.delete(runId);
+            return "cancelled";
+        }
+        if (!(await this.isProjectRootAvailable(next.projectRoot))) {
+            await this.cancelRun(runId, PROJECT_ROOT_UNAVAILABLE_REASON);
+            return "cancelled";
+        }
+        const diskGate = await this.checkDiskFloorAdmission();
+        if (diskGate !== undefined) {
+            await this.cancelRun(runId, INSUFFICIENT_DISK_CANCEL_REASON);
+            return "cancelled";
+        }
+        const pending = this.pendingQueuedStarts.get(runId);
+        let taskYaml;
+        let task;
+        let binding;
+        let pathCheckoutRoot;
+        let checkoutKey;
+        let pipeline;
+        let cwd;
+        let projectRoot;
+        let checkoutOverride;
+        let skipGates;
+        let ciIdentity;
+        let taskPath;
+        let submission;
+        let pinned;
+        let callerId = pending?.callerId ?? meta.caller_id ?? null;
+        let skills = pending?.skills;
+        if (callerId !== undefined &&
+            callerId !== null &&
+            blockedCallers.has(callerId.toLowerCase())) {
+            next.entry.callerId = next.entry.callerId ?? callerId;
+            this.admissionQueue.requeueFront(next.projectRoot, next.entry);
+            return "caller_quota_busy";
+        }
+        try {
+            if (pending !== undefined) {
+                taskYaml = pending.taskYaml;
+                binding = pending.binding;
+                pathCheckoutRoot = pending.pathCheckoutRoot;
+                checkoutKey = pending.checkoutKey;
+                pipeline = pending.pipeline;
+                cwd = pending.cwd;
+                projectRoot = pending.projectRoot;
+                checkoutOverride = pending.checkoutOverride;
+                skipGates = pending.skipGates;
+                ciIdentity = pending.ciIdentity;
+                taskPath = pending.taskPath;
+                submission = pending.submission;
+                pinned = pending.pinned;
+                callerId = pending.callerId ?? meta.caller_id ?? null;
+                const loadedTask = loadTaskFromYamlOutcome(taskYaml, pending.taskLabel);
+                if (!loadedTask.ok) {
+                    await this.cancelRun(runId, loadedTask.issues[0]?.message ?? "Invalid task");
+                    return "cancelled";
+                }
+                task = loadedTask.value;
+            }
+            else {
+                taskYaml = await this.options.store.readTaskYaml(runId);
+                const loadedTask = loadTaskFromYamlOutcome(taskYaml, `run ${runId} task`);
+                if (!loadedTask.ok) {
+                    await this.cancelRun(runId, loadedTask.issues[0]?.message ?? "Invalid task");
+                    return "cancelled";
+                }
+                task = loadedTask.value;
+                const bindingOutcome = resolveWorkspaceBinding(task, {});
+                if (!bindingOutcome.ok) {
+                    await this.cancelRun(runId, bindingOutcome.issues[0]?.message ?? "Invalid workspace binding");
+                    return "cancelled";
+                }
+                binding = bindingOutcome.value;
+                cwd = meta.project_root ?? this.cwd;
+                projectRoot = normalizeCatalogPath(meta.project_root ?? this.projectRoot);
+                if (binding.kind === "checkout") {
+                    pathCheckoutRoot = await resolveAndValidateCheckout(task, undefined, cwd);
+                    checkoutKey =
+                        pathCheckoutRoot !== undefined
+                            ? await toCheckoutLeaseKey(pathCheckoutRoot)
+                            : undefined;
+                }
+                if (meta.pipeline_source === "inline") {
+                    const body = await this.options.store.readPipelineBody(runId);
+                    if (body === null || body === "") {
+                        await this.cancelRun(runId, "pipeline_body unavailable after restart");
+                        return "cancelled";
+                    }
+                    try {
+                        pipeline = JSON.parse(body);
+                    }
+                    catch {
+                        await this.cancelRun(runId, "pipeline_body invalid after restart");
+                        return "cancelled";
+                    }
+                }
+                else if (meta.pipeline_path) {
+                    pipeline = normalizeCatalogPath(meta.pipeline_path);
+                }
+                else {
+                    await this.cancelRun(runId, "pipeline_path unavailable after restart");
+                    return "cancelled";
+                }
+                taskPath = meta.task_path;
+                skipGates = meta.skip_gates;
+                ciIdentity = {
+                    gitSha: meta.git_sha,
+                    ciPrUrl: meta.ci_pr_url,
+                    ciJobUrl: meta.ci_job_url,
+                };
+            }
+        }
+        catch (err) {
+            await this.cancelRun(runId, err instanceof Error ? err.message : String(err));
+            return "cancelled";
+        }
+        const reserved = this.tryReserve(checkoutKey, projectRoot, callerId);
+        if (!reserved.ok) {
+            if (reserved.failure.code === "busy_checkout") {
+                return "checkout_busy";
+            }
+            this.admissionQueue.requeueFront(next.projectRoot, next.entry);
+            if (reserved.failure.code === "busy_capacity" &&
+                reserved.failure.scope === "project") {
+                return "project_capacity_busy";
+            }
+            if (reserved.failure.code === "busy_caller_quota") {
+                if (callerId !== undefined && callerId !== null) {
+                    blockedCallers.add(callerId.toLowerCase());
+                    next.entry.callerId = next.entry.callerId ?? callerId;
+                }
+                return "caller_quota_busy";
+            }
+            return "capacity_busy";
+        }
+        let rollback = async () => { };
+        try {
+            if (this.options.onBeforeQueuedMaterialize !== undefined) {
+                await this.options.onBeforeQueuedMaterialize(runId);
+            }
+            const { materialized, rollback: linkRollback } = await materializeWorkspaceBinding({
+                runId,
+                task,
+                binding,
+                checkoutRoot: pathCheckoutRoot,
+                pinned,
+            });
+            rollback = linkRollback;
+            let metaAfterMaterialize;
+            try {
+                metaAfterMaterialize = await this.options.store.readRunMeta(runId);
+            }
+            catch {
+                await rollback().catch(() => undefined);
+                this.clearReservation(reserved.provisionalId);
+                this.pendingQueuedStarts.delete(runId);
+                return "cancelled";
+            }
+            if (metaAfterMaterialize.status !== "queued") {
+                await rollback().catch(() => undefined);
+                this.clearReservation(reserved.provisionalId);
+                this.pendingQueuedStarts.delete(runId);
+                this.schedulingHalts.delete(runId);
+                return "cancelled";
+            }
+            const schedulingHalt = this.ensureSchedulingHalt(runId);
+            const started = await startPipeline({
+                submission,
+                agent: this.options.agent,
+                store: this.options.store,
+                taskYaml,
+                taskPath,
+                pipeline,
+                cwd,
+                projectRoot,
+                checkoutOverride,
+                runId: materialized.runId,
+                reuseExistingRun: true,
+                checkoutRoot: materialized.checkoutRoot,
+                repository: materialized.repository,
+                ref: materialized.ref,
+                resolvedSha: materialized.resolvedSha,
+                runBranch: materialized.runBranch,
+                gitSha: ciIdentity?.gitSha,
+                ciPrUrl: ciIdentity?.ciPrUrl,
+                ciJobUrl: ciIdentity?.ciJobUrl,
+                hitl: this.hitl,
+                maxActiveStagesPerRun: this.maxActiveStagesPerRun,
+                executionMode: this.executionMode,
+                stageProcessLauncher: this.stageProcessLauncher,
+                operatorCatalog: this.options.operatorCatalog,
+                skipGates,
+                schedulingHalt,
+                ...(skills !== undefined ? { skills } : {}),
+            });
+            this.pendingQueuedStarts.delete(runId);
+            this.track(reserved.provisionalId, started.runId, started.done);
+            this.attachQueuedDone(started.runId, started.done);
+            return "started";
+        }
+        catch (err) {
+            await rollback().catch(() => undefined);
+            this.clearReservation(reserved.provisionalId);
+            this.pendingQueuedStarts.delete(runId);
+            this.schedulingHalts.delete(runId);
+            if (err instanceof QueuedRunActivationAborted) {
+                return "cancelled";
+            }
+            const reason = err instanceof StartLinkError
+                ? err.code
+                : err instanceof Error
+                    ? err.message
+                    : String(err);
+            await this.cancelRun(runId, reason);
+            return "cancelled";
         }
-        return { ok: true, provisionalId };
     }
     clearReservation(provisionalId) {
         this.removeActiveEntry(provisionalId, true);
@@ -1292,7 +2822,9 @@ export class RunManager {
             return false;
         const tracked = await this.ensureResumeTracked(runId);
         if (!tracked.ok) {
-            console.error(`resumeStalledSchedules: ${tracked.reason}`);
+            log
+                .child({ run_id: runId })
+                .error("resume.track_failed", `resumeStalledSchedules: ${tracked.reason}`);
             return false;
         }
         try {
@@ -1315,6 +2847,7 @@ export class RunManager {
                 executionMode: this.executionMode,
                 stageProcessLauncher: this.stageProcessLauncher,
                 initialSchedule: hydrated,
+                schedulingHalt: this.ensureSchedulingHalt(runId),
             }).finally(() => {
                 void syncRunStatusFromStages(this.options.store, runId).catch(() => { });
             });
@@ -1336,16 +2869,27 @@ export class RunManager {
         }
         let checkoutKey;
         let durableCheckoutRoot;
+        let resumeCallerId = null;
+        let resumeProjectRoot;
         try {
             const meta = await this.options.store.readRunMeta(runId);
-            const checkoutRoot = meta.checkout_root;
-            if (checkoutRoot !== undefined && checkoutRoot !== "") {
-                durableCheckoutRoot = checkoutRoot;
-                checkoutKey = await toCheckoutLeaseKey(checkoutRoot);
+            resumeCallerId = meta.caller_id ?? null;
+            resumeProjectRoot = meta.project_root
+                ? normalizeCatalogPath(meta.project_root)
+                : undefined;
+            const kind = derivedBindingKindFromMeta(meta);
+            if (kind === "checkout") {
+                const checkoutRoot = meta.checkout_root;
+                if (checkoutRoot !== undefined && checkoutRoot !== "") {
+                    durableCheckoutRoot = checkoutRoot;
+                    checkoutKey = await toCheckoutLeaseKey(checkoutRoot);
+                }
             }
         }
         catch (err) {
-            console.error(`invariant: trackResume failed reading checkout_root for run ${runId}; continuing without checkout lease: ${err instanceof Error ? err.message : String(err)}`);
+            log
+                .child({ run_id: runId })
+                .error("resume.checkout_root_failed", `invariant: trackResume failed reading checkout_root for run ${runId}; continuing without checkout lease: ${err instanceof Error ? err.message : String(err)}`);
         }
         const conflictHolder = this.findActiveCheckoutConflict(runId, checkoutKey, durableCheckoutRoot);
         if (conflictHolder !== undefined) {
@@ -1360,6 +2904,8 @@ export class RunManager {
         this.active.set(runId, {
             checkoutKey,
             durableCheckoutRoot,
+            projectRoot: resumeProjectRoot,
+            callerId: resumeCallerId,
             generation: ++this.trackingGeneration,
         });
         return { ok: true };
@@ -1416,7 +2962,7 @@ export class RunManager {
         catch {
             // ignore secondary failures — still fail-closed in memory
         }
-        console.error(reason);
+        log.child({ run_id: runId }).error("attach.quarantined", reason);
     }
     untrackIfGeneration(runId, generation) {
         const entry = this.active.get(runId);
@@ -1429,6 +2975,7 @@ export class RunManager {
         if (!entry)
             return;
         this.active.delete(id);
+        this.schedulingHalts.delete(id);
         if (isProvisional) {
             this.provisionalIds.delete(id);
         }
@@ -1436,5 +2983,19 @@ export class RunManager {
             this.checkoutLeases.get(entry.checkoutKey) === id) {
             this.checkoutLeases.delete(entry.checkoutKey);
         }
+        if (!isProvisional) {
+            void this.drainAdmissionQueue();
+        }
+    }
+    ensureSchedulingHalt(runId) {
+        let halt = this.schedulingHalts.get(runId);
+        if (halt === undefined) {
+            halt = {
+                halted: !this.acceptingWork,
+                hostShutdown: !this.acceptingWork,
+            };
+            this.schedulingHalts.set(runId, halt);
+        }
+        return halt;
     }
 }