stageflow 0.32.0 → 0.35.0

package/dist/hosted/runtime.js

@@ -0,0 +1,596 @@
+import path from "node:path";
+import { tmpdir } from "node:os";
+import { mkdir, realpath } from "node:fs/promises";
+import { loadPipelineValidated } from "../config/validateCatalog.js";
+import { loadTaskFromYaml } from "../config/loadTask.js";
+import { redactString } from "../logging/redact.js";
+import { PACKAGE_VERSION } from "../package-meta.js";
+import { PipelinePreflightError, PipelineValidationError, startPipeline, } from "../runtime/pipelineRunner.js";
+import { runWithExplicitAuthPath } from "../runtime/credentialBinding.js";
+import { loadPackageSkills } from "./packageSkills.js";
+import { containedRealPath, isContained, resolveDirectory, } from "./containment.js";
+import { HostedRuntimeError } from "./errors.js";
+import { HostedEventSequencer } from "./sequencer.js";
+import { translateStageEvent } from "./translate.js";
+import { DEFAULT_EVENT_DELIVERY_TIMEOUT_MS, HOSTED_CANCEL_GRACE_MS, HOSTED_DELIVERY_ATTEMPTS, HOSTED_EVENT_SCHEMA_VERSION, } from "./types.js";
+const HOSTED_EXECUTION_MODE = "inprocess";
+export function createHostedRuntime(dependencies) {
+    const timeoutMs = dependencies.eventDeliveryTimeoutMs ?? DEFAULT_EVENT_DELIVERY_TIMEOUT_MS;
+    return {
+        run(input) {
+            return runWithExplicitAuthPath(hostedAuthPath(dependencies.localStore), async () => {
+                const context = readContext(input);
+                readLimits(input);
+                if (input.signal?.aborted) {
+                    return cancelledBeforeStart(dependencies.eventSink, context, timeoutMs);
+                }
+                let preflightAborted = false;
+                const onPreflightAbort = () => {
+                    preflightAborted = true;
+                };
+                input.signal?.addEventListener("abort", onPreflightAbort, { once: true });
+                const disarmPreflight = () => {
+                    input.signal?.removeEventListener("abort", onPreflightAbort);
+                };
+                let packageRoot;
+                let skills;
+                let operatorCatalog;
+                try {
+                    packageRoot = await readPackageRoot(input.packageRoot, context);
+                    validateTask(input.taskYaml, context);
+                    await validatePackage(packageRoot, input.pipeline, context);
+                    await assertStoreOutsidePackage(dependencies.localStore, packageRoot, context);
+                    try {
+                        skills = await loadPackageSkills(packageRoot);
+                    }
+                    catch (error) {
+                        throw hostedError(errorMessage(error), "package_invalid", context.externalRunId, context.externalAttemptId);
+                    }
+                    operatorCatalog = await hostedOperatorCatalog(dependencies.localStore, packageRoot);
+                    if (preflightAborted || input.signal?.aborted) {
+                        disarmPreflight();
+                        return cancelledBeforeStart(dependencies.eventSink, context, timeoutMs);
+                    }
+                    disarmPreflight();
+                    if (input.signal?.aborted) {
+                        return cancelledBeforeStart(dependencies.eventSink, context, timeoutMs);
+                    }
+                }
+                catch (error) {
+                    disarmPreflight();
+                    throw error;
+                }
+                const sequencer = new HostedEventSequencer(dependencies.eventSink, context, timeoutMs, HOSTED_DELIVERY_ATTEMPTS);
+                const halt = { halted: false };
+                const handles = new ActiveHandles();
+                const store = observeStore(dependencies.localStore, (stageId, event, attempt) => {
+                    const body = translateStageEvent(stageId, event, attempt);
+                    if (body === undefined)
+                        return Promise.resolve();
+                    return sequencer.emit(body);
+                });
+                const agent = wrapAgent(dependencies.agent, handles);
+                let decision;
+                let released = false;
+                const claim = (next) => {
+                    if (decision !== undefined)
+                        return false;
+                    decision = next;
+                    return true;
+                };
+                let runId = "";
+                let workspaceDir;
+                const startedAt = new Date().toISOString();
+                let removeAbort;
+                let timer;
+                const stop = () => {
+                    halt.halted = true;
+                    if (runId !== "") {
+                        void dependencies.localStore
+                            .updateRunStatus(runId, "cancelled")
+                            .catch(() => undefined);
+                    }
+                    void handles.stop();
+                };
+                if (input.signal !== undefined) {
+                    const onAbort = () => {
+                        if (!claim({ status: "cancelled", reason: "hosted run cancelled" }))
+                            return;
+                        stop();
+                    };
+                    input.signal.addEventListener("abort", onAbort, { once: true });
+                    removeAbort = () => input.signal?.removeEventListener("abort", onAbort);
+                }
+                if (input.timeoutMs !== undefined) {
+                    timer = setTimeout(() => {
+                        if (!claim({
+                            status: "failed",
+                            code: "timed_out",
+                            reason: "hosted run timed out",
+                        })) {
+                            return;
+                        }
+                        halt.halted = true;
+                        void handles.stop();
+                    }, input.timeoutMs);
+                }
+                try {
+                    if (input.signal?.aborted) {
+                        return cancelledBeforeStart(dependencies.eventSink, context, timeoutMs);
+                    }
+                    const started = await startPipeline({
+                        agent,
+                        store,
+                        taskYaml: input.taskYaml,
+                        pipeline: input.pipeline,
+                        cwd: packageRoot,
+                        projectRoot: packageRoot,
+                        executionMode: HOSTED_EXECUTION_MODE,
+                        maxActiveStagesPerRun: input.maxActiveStagesPerRun,
+                        schedulingHalt: halt,
+                        operatorCatalog,
+                        ...(skills !== undefined ? { skills } : {}),
+                    });
+                    runId = started.runId;
+                    workspaceDir = started.runDir;
+                    sequencer.setRunId(runId);
+                    await assertWorkspaceOutside(packageRoot, started.runDir, context, runId);
+                    try {
+                        await sequencer.emitNow({ type: "run.created" });
+                        await sequencer.emitNow({ type: "run.started" });
+                    }
+                    catch (error) {
+                        halt.halted = true;
+                        sequencer.rejectHeld(error);
+                        await handles.stop();
+                        const local = await settle(started.done, started.runId, started.runDir);
+                        throw sinkFailure(sequencer, context, local, startedAt);
+                    }
+                    sequencer.release();
+                    released = true;
+                    const local = await waitForRun(started.done, () => decision !== undefined, started.runId, started.runDir);
+                    if (sequencer.failed) {
+                        throw sinkFailure(sequencer, context, local, startedAt, workspaceDir);
+                    }
+                    claim(decisionFromPipeline(local));
+                    const chosen = decision;
+                    if (chosen.status === "cancelled" && runId !== "") {
+                        await dependencies.localStore
+                            .updateRunStatus(runId, "cancelled")
+                            .catch(() => undefined);
+                    }
+                    const receipt = receiptFor(context, startedAt);
+                    try {
+                        await sequencer.emitNow(terminalBody(chosen, receipt));
+                    }
+                    catch (error) {
+                        throw sinkFailure(sequencer, context, local, startedAt, workspaceDir, resultFrom(chosen, context, runId, workspaceDir, receipt));
+                    }
+                    return resultFrom(chosen, context, runId, workspaceDir, receipt);
+                }
+                catch (error) {
+                    if (error instanceof HostedRuntimeError)
+                        throw error;
+                    throw mapStartError(error, context, runId);
+                }
+                finally {
+                    if (!released) {
+                        sequencer.rejectHeld(new Error("hosted run ended before stage events were released"));
+                    }
+                    if (timer !== undefined)
+                        clearTimeout(timer);
+                    removeAbort?.();
+                    await handles.stop();
+                }
+            });
+        },
+    };
+}
+function readContext(input) {
+    const context = input.context;
+    const externalRunId = typeof context?.externalRunId === "string" ? context.externalRunId.trim() : "";
+    const externalAttemptId = typeof context?.externalAttemptId === "string"
+        ? context.externalAttemptId.trim()
+        : "";
+    const packageRevision = typeof context?.packageRevision === "string"
+        ? context.packageRevision.trim()
+        : "";
+    if (externalRunId === "" || externalAttemptId === "" || packageRevision === "") {
+        throw hostedError("hosted context requires externalRunId, externalAttemptId, and packageRevision", "invalid_input", externalRunId, externalAttemptId);
+    }
+    const attributes = readAttributes(context.attributes, externalRunId, externalAttemptId);
+    return {
+        externalRunId,
+        externalAttemptId,
+        packageRevision,
+        ...(attributes !== undefined ? { attributes } : {}),
+    };
+}
+function readAttributes(attributes, externalRunId, externalAttemptId) {
+    if (attributes === undefined)
+        return undefined;
+    if (attributes === null || typeof attributes !== "object" || Array.isArray(attributes)) {
+        throw hostedError("hosted context attributes must be a string record", "invalid_input", externalRunId, externalAttemptId);
+    }
+    for (const [key, value] of Object.entries(attributes)) {
+        if (typeof value !== "string") {
+            throw hostedError("hosted context attributes must be strings", "invalid_input", externalRunId, externalAttemptId);
+        }
+        if (redactString(`${key}=${value}`) !== `${key}=${value}`) {
+            throw hostedError("hosted context attributes must not contain credentials", "invalid_input", externalRunId, externalAttemptId);
+        }
+    }
+    return attributes;
+}
+function readLimits(input) {
+    if (input.timeoutMs !== undefined &&
+        (!Number.isFinite(input.timeoutMs) || input.timeoutMs <= 0)) {
+        const context = input.context;
+        throw hostedError("timeoutMs must be a positive number", "invalid_input", context?.externalRunId ?? "", context?.externalAttemptId ?? "");
+    }
+    if (input.maxActiveStagesPerRun !== undefined &&
+        (!Number.isInteger(input.maxActiveStagesPerRun) ||
+            input.maxActiveStagesPerRun < 1)) {
+        const context = input.context;
+        throw hostedError("maxActiveStagesPerRun must be a positive integer", "invalid_input", context?.externalRunId ?? "", context?.externalAttemptId ?? "");
+    }
+}
+async function readPackageRoot(packageRoot, context) {
+    if (typeof packageRoot !== "string" || packageRoot.trim() === "") {
+        throw hostedError("packageRoot is required", "invalid_input", context.externalRunId, context.externalAttemptId);
+    }
+    try {
+        return await resolveDirectory(packageRoot);
+    }
+    catch (error) {
+        throw hostedError(errorMessage(error), "package_invalid", context.externalRunId, context.externalAttemptId);
+    }
+}
+function validateTask(taskYaml, context) {
+    if (typeof taskYaml !== "string" || taskYaml.trim() === "") {
+        throw hostedError("taskYaml is required", "invalid_input", context.externalRunId, context.externalAttemptId);
+    }
+    let task;
+    try {
+        task = loadTaskFromYaml(taskYaml, "hosted task yaml");
+    }
+    catch (error) {
+        throw hostedError(redactString(errorMessage(error)), "invalid_input", context.externalRunId, context.externalAttemptId);
+    }
+    if (task.repository !== undefined || task.checkout !== undefined || task.ref !== undefined) {
+        throw hostedError("hosted runs do not accept a repository or checkout binding", "invalid_input", context.externalRunId, context.externalAttemptId);
+    }
+}
+async function validatePackage(packageRoot, pipeline, context) {
+    if (typeof pipeline !== "string" || pipeline.trim() === "") {
+        throw hostedError("pipeline is required", "invalid_input", context.externalRunId, context.externalAttemptId);
+    }
+    const pipelinePath = path.resolve(packageRoot, pipeline);
+    try {
+        await containedRealPath(packageRoot, pipelinePath);
+    }
+    catch (error) {
+        throw hostedError(errorMessage(error), "package_invalid", context.externalRunId, context.externalAttemptId);
+    }
+    const loaded = await loadPipelineValidated(pipeline, {
+        cwd: packageRoot,
+        projectRoot: packageRoot,
+        validateStages: true,
+    });
+    if (!loaded.ok) {
+        throw hostedError(redactString(loaded.findings[0]?.message ?? "pipeline failed validation"), "package_invalid", context.externalRunId, context.externalAttemptId);
+    }
+    try {
+        await containedRealPath(packageRoot, loaded.loaded.pipelinePath);
+        for (const source of Object.values(loaded.loaded.stageSources ?? {})) {
+            if (source.kind !== "file")
+                continue;
+            await containedRealPath(packageRoot, source.path);
+        }
+    }
+    catch (error) {
+        throw hostedError(errorMessage(error), "package_invalid", context.externalRunId, context.externalAttemptId);
+    }
+}
+async function assertStoreOutsidePackage(store, packageRoot, context) {
+    const dbPath = sqlitePath(store);
+    if (dbPath === undefined)
+        return;
+    let real = path.resolve(dbPath);
+    try {
+        real = await realpath(dbPath);
+    }
+    catch {
+        real = path.resolve(dbPath);
+    }
+    if (isContained(packageRoot, real)) {
+        throw hostedError("local run store must stay outside the package root", "invalid_input", context.externalRunId, context.externalAttemptId);
+    }
+}
+async function assertWorkspaceOutside(packageRoot, workspaceDir, context, runId) {
+    let real;
+    try {
+        real = await realpath(workspaceDir);
+    }
+    catch {
+        return;
+    }
+    if (isContained(packageRoot, real)) {
+        throw hostedError("run workspace must stay outside the package root", "invalid_input", context.externalRunId, context.externalAttemptId, runId);
+    }
+}
+async function hostedOperatorCatalog(store, packageRoot) {
+    const dbPath = sqlitePath(store);
+    const agentDir = dbPath !== undefined
+        ? path.join(path.dirname(dbPath), "hosted-skill-catalog")
+        : path.join(tmpdir(), "stageflow-hosted-skill-catalog");
+    await mkdir(agentDir, { recursive: true });
+    return { cwd: packageRoot, agentDir };
+}
+function hostedAuthPath(store) {
+    const fromEnv = process.env.STAGEFLOW_AGENT_AUTH_PATH?.trim();
+    if (fromEnv)
+        return fromEnv;
+    const dbPath = sqlitePath(store);
+    if (dbPath !== undefined)
+        return path.join(path.dirname(dbPath), "hosted-agent-auth.json");
+    return path.join(tmpdir(), "stageflow-hosted-agent-auth.json");
+}
+function sqlitePath(store) {
+    const connection = store.connection;
+    if (typeof connection?.name !== "string" || connection.name === "" || connection.name === ":memory:") {
+        return undefined;
+    }
+    return connection.name;
+}
+function observeStore(store, onStageEvent) {
+    return new Proxy(store, {
+        get(target, prop, receiver) {
+            if (prop === "appendStageEvent") {
+                return async (runId, stageId, event, options) => {
+                    await target.appendStageEvent(runId, stageId, event, options);
+                    await onStageEvent(stageId, event, options?.attempt ?? 1);
+                };
+            }
+            const value = Reflect.get(target, prop, receiver);
+            if (typeof value === "function") {
+                return value.bind(target);
+            }
+            return value;
+        },
+    });
+}
+class ActiveHandles {
+    handles = new Set();
+    stopping = false;
+    get isStopping() {
+        return this.stopping;
+    }
+    track(handle) {
+        this.handles.add(handle);
+    }
+    untrack(handle) {
+        this.handles.delete(handle);
+    }
+    async stop() {
+        this.stopping = true;
+        const open = [...this.handles];
+        await Promise.all(open.map((handle) => Promise.race([
+            handle.close().catch(() => undefined),
+            sleep(HOSTED_CANCEL_GRACE_MS),
+        ])));
+    }
+}
+function wrapAgent(agent, handles) {
+    return {
+        openStage(input) {
+            const inner = agent.openStage(input);
+            let closed = false;
+            const wrapped = {
+                stageId: inner.stageId,
+                next() {
+                    if (handles.isStopping || closed) {
+                        return Promise.resolve(stoppedEvent());
+                    }
+                    return inner.next();
+                },
+                deliverAnswer(answer) {
+                    inner.deliverAnswer(answer);
+                },
+                async close(options) {
+                    if (closed)
+                        return;
+                    closed = true;
+                    handles.untrack(wrapped);
+                    await inner.close(options);
+                },
+            };
+            handles.track(wrapped);
+            return wrapped;
+        },
+        runStage(input) {
+            return agent.runStage(input);
+        },
+    };
+}
+function stoppedEvent() {
+    return {
+        status: "completed",
+        result: { ok: false, reason: "cancelled" },
+    };
+}
+async function cancelledBeforeStart(sink, context, timeoutMs) {
+    const startedAt = new Date().toISOString();
+    const sequencer = new HostedEventSequencer(sink, context, timeoutMs, HOSTED_DELIVERY_ATTEMPTS);
+    sequencer.release();
+    const decision = {
+        status: "cancelled",
+        reason: "hosted run cancelled before start",
+    };
+    const receipt = receiptFor(context, startedAt);
+    try {
+        await sequencer.emitNow(terminalBody(decision, receipt));
+    }
+    catch (error) {
+        if (error instanceof HostedRuntimeError)
+            throw error;
+        throw sinkFailure(sequencer, context, undefined, startedAt);
+    }
+    return resultFrom(decision, context, "", undefined, receipt);
+}
+function decisionFromPipeline(result) {
+    if (result.outcome === "cancelled") {
+        return { status: "cancelled", reason: result.reason ?? "hosted run cancelled" };
+    }
+    if (result.outcome === "waiting") {
+        return {
+            status: "failed",
+            code: "hitl_unsupported",
+            reason: result.reason ??
+                "hosted run stopped for approval; destroying the sandbox does not leave a resumable attempt",
+        };
+    }
+    if (result.ok && result.outcome === "succeeded") {
+        return { status: "succeeded" };
+    }
+    const reason = redactString(result.reason ?? "hosted run failed");
+    if (/requested wait|no HITL controller/i.test(reason)) {
+        return { status: "failed", code: "hitl_unsupported", reason };
+    }
+    if (/timed out|timeout/i.test(reason)) {
+        return { status: "failed", code: "timed_out", reason };
+    }
+    return { status: "failed", code: "agent_failed", reason };
+}
+function terminalBody(decision, receipt) {
+    if (decision.status === "succeeded") {
+        return { type: "run.succeeded", receipt };
+    }
+    if (decision.status === "cancelled") {
+        return {
+            type: "run.cancelled",
+            ...(decision.reason !== undefined ? { reason: decision.reason } : {}),
+            receipt,
+        };
+    }
+    if (decision.status === "waiting") {
+        return {
+            type: "run.failed",
+            code: decision.code,
+            reason: decision.reason,
+            receipt,
+        };
+    }
+    return {
+        type: "run.failed",
+        code: decision.code,
+        ...(decision.reason !== undefined ? { reason: decision.reason } : {}),
+        receipt,
+    };
+}
+function resultFrom(decision, context, runId, workspaceDir, receipt) {
+    return {
+        runId: runId === "" ? null : runId,
+        externalRunId: context.externalRunId,
+        externalAttemptId: context.externalAttemptId,
+        status: decision.status,
+        ...(decision.status === "succeeded"
+            ? {}
+            : {
+                code: decision.status === "cancelled" ? "cancelled" : decision.code,
+                ...(decision.reason !== undefined ? { reason: decision.reason } : {}),
+            }),
+        ...(workspaceDir !== undefined ? { workspaceDir } : {}),
+        receipt,
+    };
+}
+function receiptFor(context, startedAt) {
+    return {
+        coreVersion: PACKAGE_VERSION,
+        eventSchemaVersion: HOSTED_EVENT_SCHEMA_VERSION,
+        packageRevision: context.packageRevision,
+        executionMode: "inline",
+        startedAt,
+        finishedAt: new Date().toISOString(),
+    };
+}
+function sinkFailure(sequencer, context, local, startedAt, workspaceDir, localResult) {
+    const base = sequencer.failure;
+    const result = localResult ??
+        (local !== undefined
+            ? resultFrom(decisionFromPipeline(local), context, local.runId, workspaceDir ?? local.runDir, receiptFor(context, startedAt))
+            : undefined);
+    return new HostedRuntimeError(base?.message ?? "hosted event sink failed", {
+        code: "event_sink_failed",
+        externalRunId: context.externalRunId,
+        externalAttemptId: context.externalAttemptId,
+        ...(local?.runId !== undefined && local.runId !== ""
+            ? { runId: local.runId }
+            : {}),
+        ...(base !== undefined
+            ? { eventType: base.eventType, sequence: base.sequence }
+            : {}),
+        ...(result !== undefined ? { localResult: result } : {}),
+    });
+}
+async function waitForRun(done, stopped, runId, runDir) {
+    let timer;
+    const stoppedPromise = new Promise((resolve) => {
+        if (stopped()) {
+            resolve();
+            return;
+        }
+        timer = setInterval(() => {
+            if (!stopped())
+                return;
+            if (timer !== undefined)
+                clearInterval(timer);
+            resolve();
+        }, 15);
+    });
+    try {
+        const winner = await Promise.race([
+            done.then((result) => ({ kind: "done", result })),
+            stoppedPromise.then(() => ({ kind: "stopped" })),
+        ]);
+        if (winner.kind === "done")
+            return winner.result;
+        return settle(done, runId, runDir);
+    }
+    finally {
+        if (timer !== undefined)
+            clearInterval(timer);
+    }
+}
+async function settle(done, runId, runDir) {
+    return Promise.race([
+        done,
+        sleep(HOSTED_CANCEL_GRACE_MS).then(() => ({
+            ok: false,
+            outcome: "cancelled",
+            runId,
+            runDir,
+            reason: "hosted run cancelled",
+        })),
+    ]);
+}
+function mapStartError(error, context, runId) {
+    if (error instanceof PipelineValidationError || error instanceof PipelinePreflightError) {
+        return hostedError(redactString(errorMessage(error)), "package_invalid", context.externalRunId, context.externalAttemptId, runId === "" ? undefined : runId);
+    }
+    return hostedError(redactString(errorMessage(error)), "internal", context.externalRunId, context.externalAttemptId, runId === "" ? undefined : runId);
+}
+function hostedError(message, code, externalRunId, externalAttemptId, runId) {
+    return new HostedRuntimeError(message, {
+        code,
+        externalRunId,
+        externalAttemptId,
+        ...(runId !== undefined ? { runId } : {}),
+    });
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/dist/hosted/sequencer.d.ts

@@ -0,0 +1,28 @@
+import { type HostedEventBody, type HostedEventSink, type HostedRunContext } from "./types.js";
+export declare class HostedEventSequencer {
+    private readonly sink;
+    private readonly context;
+    private readonly timeoutMs;
+    private readonly attempts;
+    private chain;
+    private held;
+    private holding;
+    private sequence;
+    private runId;
+    failed: boolean;
+    failure?: {
+        eventType: string;
+        sequence: number;
+        message: string;
+    };
+    constructor(sink: HostedEventSink, context: HostedRunContext, timeoutMs: number, attempts: number);
+    setRunId(runId: string): void;
+    emit(body: HostedEventBody): Promise<void>;
+    emitNow(body: HostedEventBody): Promise<void>;
+    release(): void;
+    rejectHeld(error: unknown): void;
+    private enqueue;
+    private deliver;
+    private envelope;
+    private failureError;
+}