stageflow 0.39.0 → 0.42.0

package/dist/runtime/runManager.js

@@ -530,7 +530,7 @@ export class RunManager {
      */
     async attachWaitingStages() {
         const found = [];
-        const runs = await this.options.store.listRuns();
+        const runs = await this.options.store.listRuns({ origin: "any" });
         for (const summary of runs) {
             if (summary.status !== "running")
                 continue;
@@ -627,7 +627,7 @@ export class RunManager {
     }
     async reconcileOrphanedStages() {
         const reconciled = [];
-        const runs = await this.options.store.listRuns();
+        const runs = await this.options.store.listRuns({ origin: "any" });
         for (const summary of runs) {
             let meta;
             try {
@@ -806,7 +806,7 @@ export class RunManager {
     }
     async resumeStalledSchedules() {
         const resumed = [];
-        const runs = await this.options.store.listRuns();
+        const runs = await this.options.store.listRuns({ origin: "any" });
         for (const summary of runs) {
             if (summary.status !== "running")
                 continue;
@@ -828,7 +828,7 @@ export class RunManager {
     }
     /** Rebuild in-memory admission queue from persisted `queued` rows (R27). */
     async reenqueuePersistedQueuedRuns() {
-        const queued = await this.options.store.listRuns({ status: "queued" });
+        const queued = await this.options.store.listRuns({ status: "queued", origin: "any" });
         const ordered = [...queued].sort((a, b) => a.created_at.localeCompare(b.created_at));
         this.admissionQueue.clear();
         for (const row of ordered) {
@@ -1273,7 +1273,7 @@ export class RunManager {
             gitSha: input.gitSha,
             ciPrUrl: input.ciPrUrl,
             ciJobUrl: input.ciJobUrl,
-        }, derivedProjectRoot, resolved.kind === "path" ? resolved.taskPath : undefined, submission, undefined, input.callerId, input.skills, input.dispatchKey);
+        }, derivedProjectRoot, resolved.kind === "path" ? resolved.taskPath : undefined, submission, undefined, input.callerId, input.skills, input.dispatchKey, input.origin);
     }
     async rerun(runId, options) {
         if (!this.acceptingWork) {
@@ -2219,7 +2219,7 @@ export class RunManager {
             this.attachedWaiting.delete(`${runId}\0${stageId}`);
         }
     }
-    async reserveAndStartPipeline(taskYaml, pipeline, taskLabel, cwd, checkoutOverride, skipGates, ciIdentity, projectRoot, taskPath, submission, pinned, callerId, skills, dispatchKey) {
+    async reserveAndStartPipeline(taskYaml, pipeline, taskLabel, cwd, checkoutOverride, skipGates, ciIdentity, projectRoot, taskPath, submission, pinned, callerId, skills, dispatchKey, origin) {
         const persistence = pipelinePersistenceForStart(pipeline);
         if (!persistence.ok) {
             return {
@@ -2419,6 +2419,7 @@ export class RunManager {
                 ...(callerId !== undefined && callerId !== null
                     ? { callerId }
                     : {}),
+                ...(origin !== undefined ? { origin } : {}),
                 runManifest,
                 skipGates,
             });
@@ -2506,6 +2507,7 @@ export class RunManager {
                 skipGates,
                 schedulingHalt,
                 callerId,
+                ...(origin !== undefined ? { origin } : {}),
                 ...(hasSkills ? { skills: validatedSkills } : {}),
             });
             if (started.created === false)

package/dist/runtime/runRetentionSweep.js

@@ -252,7 +252,7 @@ export async function runRetentionSweep(store, a2aStore, options = {}) {
     const windows = options.windows ?? retentionWindowsFromEnv(env);
     const artifactMaxBytes = options.artifactMaxBytes ?? slimArtifactMaxBytesFromEnv(env);
     const bareCacheTtlMs = options.bareCacheTtlMs ?? bareCacheTtlFromEnv(env);
-    const runs = await store.listRuns();
+    const runs = await store.listRuns({ origin: "any" });
     const slimmed = [];
     const purged = [];
     const metas = new Map();
@@ -305,7 +305,7 @@ export async function runRetentionSweep(store, a2aStore, options = {}) {
         }
     }
     // Re-list for bare-cache decisions after PURGE removed rows when execute.
-    const runsForBare = execute ? await store.listRuns() : runs;
+    const runsForBare = execute ? await store.listRuns({ origin: "any" }) : runs;
     const bareCachesEvicted = await evictBareCaches({
         runs: runsForBare,
         now,

package/dist/runtime/sandboxContainer.d.ts

@@ -0,0 +1,62 @@
+export declare const SANDBOX_CONTAINER_IMAGE_ENV = "STAGEFLOW_STAGE_CONTAINER_IMAGE";
+export declare const SANDBOX_CONTAINER_DOCKER_BIN_ENV = "STAGEFLOW_STAGE_CONTAINER_DOCKER_BIN";
+/** Docker container names allow only `[a-zA-Z0-9_.-]`. */
+export declare function sanitizeContainerNameSegment(value: string): string;
+export declare function buildContainerName(input: {
+    runId: string;
+    stageId: string;
+    attempt?: number;
+}, suffix: string): string;
+export type SandboxContainerOptions = {
+    image: string;
+    dockerBin: string;
+};
+export declare function resolveSandboxContainerOptions(env: Record<string, string | undefined>, override?: {
+    image: string;
+    dockerBin?: string;
+}): SandboxContainerOptions | undefined;
+/** Fixed path inside the sandbox. The daemon owns the volume; we never pass it a path from this process. */
+export declare const SANDBOX_MOUNT_PATH = "/workspace";
+export declare function sandboxVolumeName(containerName: string): string;
+export declare function buildStartContainerArgs(params: {
+    image: string;
+    containerName: string;
+}): string[];
+export declare function startSandboxContainer(params: {
+    image: string;
+    runId: string;
+    stageId: string;
+    attempt?: number;
+    dockerBin?: string;
+}): Promise<{
+    containerName: string;
+}>;
+export declare function stopSandboxContainer(params: {
+    containerName: string;
+    dockerBin?: string;
+}): Promise<void>;
+export type SandboxExecResult = {
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+};
+export declare function execInSandboxContainer(params: {
+    containerName: string;
+    command: string;
+    dockerBin?: string;
+    timeoutMs?: number;
+}): Promise<SandboxExecResult>;
+export declare function copyToSandboxContainer(params: {
+    containerName: string;
+    hostPath: string;
+    containerPath: string;
+    dockerBin?: string;
+    contents?: boolean;
+}): Promise<void>;
+export declare function copyFromSandboxContainer(params: {
+    containerName: string;
+    containerPath: string;
+    hostPath: string;
+    dockerBin?: string;
+    contents?: boolean;
+}): Promise<void>;

package/dist/runtime/sandboxContainer.js

@@ -0,0 +1,151 @@
+import { execFile } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { promisify } from "node:util";
+export const SANDBOX_CONTAINER_IMAGE_ENV = "STAGEFLOW_STAGE_CONTAINER_IMAGE";
+export const SANDBOX_CONTAINER_DOCKER_BIN_ENV = "STAGEFLOW_STAGE_CONTAINER_DOCKER_BIN";
+const execFileAsync = promisify(execFile);
+/** Docker container names allow only `[a-zA-Z0-9_.-]`. */
+export function sanitizeContainerNameSegment(value) {
+    return value.replace(/[^a-zA-Z0-9_.-]/g, "-");
+}
+export function buildContainerName(input, suffix) {
+    const runId = sanitizeContainerNameSegment(input.runId);
+    const stageId = sanitizeContainerNameSegment(input.stageId);
+    const attempt = input.attempt ?? 1;
+    return `stageflow-${runId}-${stageId}-${attempt}-${suffix}`;
+}
+export function resolveSandboxContainerOptions(env, override) {
+    if (override) {
+        if (!override.image || override.image.trim() === "") {
+            throw new Error("sandboxContainer: image must be a non-empty string");
+        }
+        return { image: override.image, dockerBin: override.dockerBin || "docker" };
+    }
+    const image = env[SANDBOX_CONTAINER_IMAGE_ENV];
+    if (!image || image.trim() === "")
+        return undefined;
+    return {
+        image,
+        dockerBin: env[SANDBOX_CONTAINER_DOCKER_BIN_ENV] || "docker",
+    };
+}
+/** Fixed path inside the sandbox. The daemon owns the volume; we never pass it a path from this process. */
+export const SANDBOX_MOUNT_PATH = "/workspace";
+export function sandboxVolumeName(containerName) {
+    const safe = containerName.replace(/[^A-Za-z0-9_.-]/g, "_");
+    return `sf-bash-${safe}`;
+}
+export function buildStartContainerArgs(params) {
+    const { image, containerName } = params;
+    return [
+        "run",
+        "-d",
+        "--name",
+        containerName,
+        "-v",
+        `${sandboxVolumeName(containerName)}:${SANDBOX_MOUNT_PATH}`,
+        "-w",
+        SANDBOX_MOUNT_PATH,
+        image,
+        "sleep",
+        "infinity",
+    ];
+}
+function errorMessage(err) {
+    if (err && typeof err === "object" && "stderr" in err) {
+        const stderr = err.stderr;
+        if (typeof stderr === "string" && stderr.trim().length > 0)
+            return stderr;
+    }
+    return err instanceof Error ? err.message : String(err);
+}
+export async function startSandboxContainer(params) {
+    const containerName = buildContainerName({ runId: params.runId, stageId: params.stageId, attempt: params.attempt }, randomUUID().slice(0, 8));
+    const args = buildStartContainerArgs({
+        image: params.image,
+        containerName,
+    });
+    try {
+        await execFileAsync(params.dockerBin ?? "docker", args);
+    }
+    catch (err) {
+        throw new Error(`failed to start sandbox container: ${errorMessage(err)}`);
+    }
+    return { containerName };
+}
+const NO_SUCH_CONTAINER_PATTERN = /no such container/i;
+const NO_SUCH_VOLUME_PATTERN = /no such volume/i;
+export async function stopSandboxContainer(params) {
+    try {
+        await execFileAsync(params.dockerBin ?? "docker", [
+            "rm",
+            "-f",
+            params.containerName,
+        ]);
+    }
+    catch (err) {
+        const stderr = stderrOf(err);
+        if (!NO_SUCH_CONTAINER_PATTERN.test(stderr)) {
+            throw new Error(`failed to stop sandbox container: ${errorMessage(err)}`);
+        }
+    }
+    try {
+        await execFileAsync(params.dockerBin ?? "docker", [
+            "volume",
+            "rm",
+            sandboxVolumeName(params.containerName),
+        ]);
+    }
+    catch (err) {
+        const stderr = stderrOf(err);
+        if (NO_SUCH_VOLUME_PATTERN.test(stderr))
+            return;
+        throw new Error(`failed to remove sandbox volume: ${errorMessage(err)}`);
+    }
+}
+function stderrOf(err) {
+    if (err && typeof err === "object" && "stderr" in err) {
+        return String(err.stderr ?? "");
+    }
+    return "";
+}
+export async function execInSandboxContainer(params) {
+    const args = ["exec", "-w", SANDBOX_MOUNT_PATH, params.containerName, "bash", "-c", params.command];
+    try {
+        const { stdout, stderr } = await execFileAsync(params.dockerBin ?? "docker", args, {
+            maxBuffer: 8 * 1024 * 1024,
+            ...(params.timeoutMs !== undefined ? { timeout: params.timeoutMs } : {}),
+        });
+        return { stdout, stderr, exitCode: 0 };
+    }
+    catch (err) {
+        const e = err;
+        // `code` is a number only for a completed exec that exited non-zero.
+        // Anything else (ENOENT for a missing docker binary, the container
+        // having been removed mid-exec, etc.) is an infra failure, not a
+        // command result — rethrow so callers don't mistake it for "the
+        // command ran and exited 1".
+        if (typeof e.code !== "number") {
+            throw new Error(`failed to exec in sandbox container: ${errorMessage(err)}`);
+        }
+        return { stdout: e.stdout ?? "", stderr: e.stderr ?? "", exitCode: e.code };
+    }
+}
+export async function copyToSandboxContainer(params) {
+    const src = params.contents ? `${params.hostPath}/.` : params.hostPath;
+    await dockerCp(params.dockerBin, src, `${params.containerName}:${params.containerPath}`, "into");
+}
+export async function copyFromSandboxContainer(params) {
+    const src = params.contents
+        ? `${params.containerName}:${params.containerPath}/.`
+        : `${params.containerName}:${params.containerPath}`;
+    await dockerCp(params.dockerBin, src, params.hostPath, "from");
+}
+async function dockerCp(dockerBin, src, dest, direction) {
+    try {
+        await execFileAsync(dockerBin ?? "docker", ["cp", src, dest]);
+    }
+    catch (err) {
+        throw new Error(`failed to copy ${direction} sandbox: ${errorMessage(err)}`);
+    }
+}

package/dist/runtime/sandboxDesk.d.ts

@@ -0,0 +1,28 @@
+export type SandboxDeskRole = "artifacts" | "envelope" | "checkout" | "grant" | "work";
+export type SandboxDeskBinding = {
+    role: SandboxDeskRole;
+    hostPath: string;
+    containerPath: string;
+    shape: "directory" | "file" | "mirror";
+    copyIn: boolean;
+    copyOut: boolean;
+    optional: boolean;
+};
+export type SandboxDesk = {
+    bindings: readonly SandboxDeskBinding[];
+};
+export declare function buildSandboxDesk(input: {
+    attemptDir: string;
+    checkoutRoot?: string;
+    grantedPaths?: readonly string[];
+}): SandboxDesk;
+export declare function populateSandboxDesk(params: {
+    containerName: string;
+    dockerBin?: string;
+    desk: SandboxDesk;
+}): Promise<void>;
+export declare function syncSandboxDesk(params: {
+    containerName: string;
+    dockerBin?: string;
+    desk: SandboxDesk;
+}): Promise<void>;

package/dist/runtime/sandboxDesk.js

@@ -0,0 +1,242 @@
+import { randomBytes } from "node:crypto";
+import { access, mkdir, rename, rm, stat } from "node:fs/promises";
+import path from "node:path";
+import { copyFromSandboxContainer, copyToSandboxContainer, execInSandboxContainer, SANDBOX_MOUNT_PATH, } from "./sandboxContainer.js";
+import { shellQuote } from "./sandboxFiles.js";
+export function buildSandboxDesk(input) {
+    const attemptDir = input.attemptDir;
+    const bindings = [
+        {
+            role: "artifacts",
+            hostPath: path.join(attemptDir, "artifacts"),
+            containerPath: `${SANDBOX_MOUNT_PATH}/artifacts`,
+            shape: "directory",
+            copyIn: true,
+            copyOut: false,
+            optional: true,
+        },
+        {
+            role: "envelope",
+            hostPath: path.join(attemptDir, "envelope.json"),
+            containerPath: `${SANDBOX_MOUNT_PATH}/envelope.json`,
+            shape: "file",
+            copyIn: true,
+            copyOut: false,
+            optional: true,
+        },
+    ];
+    const checkout = input.checkoutRoot?.trim();
+    if (checkout) {
+        bindings.push({
+            role: "checkout",
+            hostPath: checkout,
+            containerPath: `${SANDBOX_MOUNT_PATH}/checkout`,
+            shape: "directory",
+            copyIn: true,
+            copyOut: true,
+            optional: false,
+        });
+    }
+    const grants = input.grantedPaths ?? [];
+    for (let index = 0; index < grants.length; index += 1) {
+        const hostPath = grants[index];
+        bindings.push({
+            role: "grant",
+            hostPath,
+            containerPath: `${SANDBOX_MOUNT_PATH}/grants/${grantSegment(index, hostPath)}`,
+            shape: "mirror",
+            copyIn: true,
+            copyOut: true,
+            optional: false,
+        });
+    }
+    bindings.push({
+        role: "work",
+        hostPath: path.join(attemptDir, "work"),
+        containerPath: `${SANDBOX_MOUNT_PATH}/work`,
+        shape: "directory",
+        copyIn: false,
+        copyOut: true,
+        optional: true,
+    });
+    return { bindings };
+}
+function grantSegment(index, hostPath) {
+    const base = path.basename(hostPath).replace(/[^A-Za-z0-9_.-]/g, "_");
+    const safe = base.length > 0 && base !== "." && base !== ".." ? base : "path";
+    return `${index}-${safe}`;
+}
+async function pathExists(target) {
+    try {
+        await access(target);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export async function populateSandboxDesk(params) {
+    const dirs = new Set([
+        `${SANDBOX_MOUNT_PATH}/artifacts`,
+        `${SANDBOX_MOUNT_PATH}/work`,
+        `${SANDBOX_MOUNT_PATH}/grants`,
+    ]);
+    for (const binding of params.desk.bindings) {
+        if (binding.role === "checkout" || binding.shape === "mirror") {
+            dirs.add(binding.containerPath);
+        }
+    }
+    const made = await execInSandboxContainer({
+        containerName: params.containerName,
+        dockerBin: params.dockerBin,
+        command: `mkdir -p -- ${[...dirs].map(shellQuote).join(" ")}`,
+    });
+    if (made.exitCode !== 0) {
+        throw new Error(`failed to prepare sandbox desk: ${made.stderr.trim() || made.stdout.trim() || `exit ${made.exitCode}`}`);
+    }
+    for (const binding of params.desk.bindings) {
+        if (!binding.copyIn)
+            continue;
+        if (!(await pathExists(binding.hostPath))) {
+            if (binding.optional)
+                continue;
+            throw new Error(`sandbox desk source missing: ${binding.hostPath}`);
+        }
+        await copyBindingIn(params, binding);
+    }
+}
+export async function syncSandboxDesk(params) {
+    for (const binding of params.desk.bindings) {
+        if (!binding.copyOut)
+            continue;
+        await copyBindingOut(params, binding);
+    }
+}
+async function copyBindingIn(params, binding) {
+    if (binding.shape === "file") {
+        await copyToSandboxContainer({
+            containerName: params.containerName,
+            dockerBin: params.dockerBin,
+            hostPath: binding.hostPath,
+            containerPath: binding.containerPath,
+        });
+        return;
+    }
+    if (binding.shape === "directory") {
+        await copyToSandboxContainer({
+            containerName: params.containerName,
+            dockerBin: params.dockerBin,
+            hostPath: binding.hostPath,
+            containerPath: binding.containerPath,
+            contents: true,
+        });
+        return;
+    }
+    const info = await stat(binding.hostPath);
+    if (info.isDirectory()) {
+        await copyToSandboxContainer({
+            containerName: params.containerName,
+            dockerBin: params.dockerBin,
+            hostPath: binding.hostPath,
+            containerPath: binding.containerPath,
+            contents: true,
+        });
+        return;
+    }
+    await copyToSandboxContainer({
+        containerName: params.containerName,
+        dockerBin: params.dockerBin,
+        hostPath: binding.hostPath,
+        containerPath: path.posix.join(binding.containerPath, path.basename(binding.hostPath)),
+    });
+}
+async function copyBindingOut(params, binding) {
+    if (binding.role === "checkout" || binding.role === "work") {
+        await mirrorDirectoryFromContainer(params, binding.containerPath, binding.hostPath);
+        return;
+    }
+    if (binding.shape === "directory") {
+        await mkdir(binding.hostPath, { recursive: true });
+        await copyFromSandboxContainer({
+            containerName: params.containerName,
+            dockerBin: params.dockerBin,
+            containerPath: binding.containerPath,
+            hostPath: binding.hostPath,
+            contents: true,
+        });
+        return;
+    }
+    const info = await stat(binding.hostPath);
+    if (info.isDirectory()) {
+        await mkdir(binding.hostPath, { recursive: true });
+        await copyFromSandboxContainer({
+            containerName: params.containerName,
+            dockerBin: params.dockerBin,
+            containerPath: binding.containerPath,
+            hostPath: binding.hostPath,
+            contents: true,
+        });
+        return;
+    }
+    await mkdir(path.dirname(binding.hostPath), { recursive: true });
+    await copyFromSandboxContainer({
+        containerName: params.containerName,
+        dockerBin: params.dockerBin,
+        containerPath: path.posix.join(binding.containerPath, path.basename(binding.hostPath)),
+        hostPath: binding.hostPath,
+    });
+}
+async function mirrorDirectoryFromContainer(params, containerPath, hostPath) {
+    let probe;
+    try {
+        probe = await execInSandboxContainer({
+            containerName: params.containerName,
+            dockerBin: params.dockerBin,
+            command: `test -d ${shellQuote(containerPath)}`,
+        });
+    }
+    catch (err) {
+        throw new Error(`sandbox desk directory unavailable, not replacing ${hostPath}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (probe.exitCode !== 0) {
+        throw new Error(`sandbox desk directory missing, not replacing ${hostPath}`);
+    }
+    await mkdir(path.dirname(hostPath), { recursive: true });
+    const staging = path.join(path.dirname(hostPath), `.${path.basename(hostPath)}.sf-sync-${randomBytes(4).toString("hex")}`);
+    await mkdir(staging);
+    try {
+        await copyFromSandboxContainer({
+            containerName: params.containerName,
+            dockerBin: params.dockerBin,
+            containerPath,
+            hostPath: staging,
+            contents: true,
+        });
+        await replaceDirectory(hostPath, staging);
+    }
+    catch (err) {
+        await rm(staging, { recursive: true, force: true });
+        throw err;
+    }
+}
+async function replaceDirectory(hostPath, staging) {
+    const backup = path.join(path.dirname(hostPath), `.${path.basename(hostPath)}.sf-sync-bak-${randomBytes(4).toString("hex")}`);
+    let hostMoved = false;
+    try {
+        if (await pathExists(hostPath)) {
+            await rename(hostPath, backup);
+            hostMoved = true;
+        }
+        await rename(staging, hostPath);
+        hostMoved = false;
+    }
+    catch (err) {
+        if (hostMoved && !(await pathExists(hostPath)) && (await pathExists(backup))) {
+            await rename(backup, hostPath);
+        }
+        throw err;
+    }
+    if (await pathExists(backup)) {
+        await rm(backup, { recursive: true, force: true });
+    }
+}

package/dist/runtime/sandboxFiles.d.ts

@@ -0,0 +1,20 @@
+import { type SandboxExecResult } from "./sandboxContainer.js";
+export declare class SandboxPathError extends Error {
+    constructor(message: string);
+}
+export declare function shellQuote(value: string): string;
+/** Resolve a tool path onto the sandbox mount. Host absolutes and `..` escapes are rejected. */
+export declare function confineSandboxPath(inputPath: string): string;
+export type SandboxFileTarget = {
+    containerName: string;
+    dockerBin?: string;
+};
+export declare function readSandboxFile(target: SandboxFileTarget, inputPath: string): Promise<string>;
+export declare function writeSandboxFile(target: SandboxFileTarget, inputPath: string, content: string): Promise<void>;
+export declare function editSandboxFile(target: SandboxFileTarget, inputPath: string, edits: readonly {
+    oldText: string;
+    newText: string;
+}[]): Promise<void>;
+export declare function sliceSandboxLines(text: string, offset?: number, limit?: number): string;
+export declare function formatSandboxBashResult(result: SandboxExecResult): string;
+export declare function runSandboxBash(target: SandboxFileTarget, command: string, timeoutMs?: number): Promise<string>;