stageflow 0.25.0 → 0.30.0

package/dist/runtime/stageHitl.js

@@ -1,3 +1,4 @@
+import { parseHostGateContext } from "../browser/gateHandoff.js";
 import { appendOperatorAnswer, appendOperatorPrompt, derivePendingPrompt, } from "../hitl/qaTrail.js";
 import { deriveExecutionPatchFromEvent } from "../runstore/stageExecution.js";
 import { attemptContext } from "./stageAttemptContext.js";
@@ -7,7 +8,10 @@ export function tryParsePendingPrompt(pendingRequest) {
         return undefined;
     }
     try {
-        return normalizePromptIds(parseAskOperatorParams(pendingRequest));
+        return {
+            ...normalizePromptIds(parseAskOperatorParams(pendingRequest)),
+            ...parseHostGateContext(pendingRequest),
+        };
     }
     catch {
         return undefined;

package/dist/runtime/stageProcessLauncher.d.ts

@@ -1,4 +1,8 @@
+import * as childProcess from "node:child_process";
+import { type Logger } from "../logging/logger.js";
+import { type ResolvedStageGrants } from "./stageEnvironment.js";
 import type { OperatorCatalog } from "./stageAttemptBootstrap.js";
+import { type DerivedBindingKind } from "./stageRoots.js";
 export type StageLaunchInput = {
     runId: string;
     stageId: string;
@@ -9,6 +13,11 @@ export type StageLaunchInput = {
     sessionFilePath?: string;
     operatorCatalog?: OperatorCatalog;
     skipGates?: boolean;
+    env?: Record<string, string>;
+    bindingKind?: DerivedBindingKind;
+    grants?: ResolvedStageGrants;
+    attemptHome?: string;
+    browserEnv?: Record<string, string>;
 };
 export type StageLaunchResult = {
     type: "succeeded";
@@ -27,17 +36,28 @@ export type StageProcessLauncherOptions = {
     maxActiveStageProcesses?: number;
     env?: Record<string, string | undefined>;
     cliEntry?: string;
+    logger?: Logger;
+    /** Test seam: override child_process.fork. */
+    forkFn?: typeof childProcess.fork;
 };
+export declare function signalProcessGroup(pid: number | undefined, signal: NodeJS.Signals): void;
 export declare class StageProcessLauncher {
     private readonly maxActive;
     private readonly env;
+    private readonly explicitChildExtras;
     private readonly cliEntry;
+    private readonly workerExecArgv;
+    private readonly logger;
+    private readonly forkFn;
     private readonly active;
     private readonly waitQueue;
+    private readonly cancelledRuns;
     private slotsHeld;
+    private readonly heapMb;
     constructor(options?: StageProcessLauncherOptions);
     activeCount(): number;
     getActiveStageProcesses(): ActiveStageProcess[];
+    signalAllActive(signal: NodeJS.Signals): void;
     launch(input: StageLaunchInput): Promise<StageLaunchResult>;
     cancelRun(runId: string, killAfterMs?: number): Promise<void>;
     private waitForCapacity;

package/dist/runtime/stageProcessLauncher.js

@@ -1,7 +1,76 @@
-import { fork } from "node:child_process";
+import * as childProcess from "node:child_process";
 import { fileURLToPath } from "node:url";
+import { logger as rootLogger, resolveLogMaxLineBytes, } from "../logging/logger.js";
+import { BROWSER_ENV_PREFIX } from "../browser/browserHost.js";
+import { PACKAGE_VERSION } from "../package-meta.js";
+import { PI_HOME_AUTH_PATH_ENV, piHomeAuthPath } from "./credentialBinding.js";
+import { redactString } from "../logging/redact.js";
+import { getNamedSecrets } from "../logging/namedSecrets.js";
 import { readMaxActiveStageProcesses, } from "./stageConcurrency.js";
+import { getContainerLimits } from "./containerLimits.js";
+import { buildStageEnvironment, isAmbientBlockedEnv, isForeverDeniedSecret, } from "./stageEnvironment.js";
+import { ensureStageCacheDirs } from "./stageCacheEnv.js";
 import { SF_STAGE_WORKER, STAGE_WORKER_EXIT, } from "./stageWorkerProtocol.js";
+import { overlayStageBindingEnv, } from "./stageRoots.js";
+function defaultStageCliEntry() {
+    const source = import.meta.url.split("?")[0]?.endsWith(".ts") === true;
+    return fileURLToPath(new URL(source ? "../cli.ts" : "../cli.js", import.meta.url));
+}
+function isTypeScriptEntry(entry) {
+    return /\.(?:m|c)?ts$/.test(entry);
+}
+/**
+ * A compiled `dist/cli.js` child is plain node. `npm run dev` is TypeScript,
+ * and this launcher sets execArgv explicitly so the parent's tsx loader is
+ * not inherited. Load that entry with the same tsx hooks the dev CLI uses.
+ */
+function tsxLoaderExecArgv() {
+    try {
+        const preflight = fileURLToPath(import.meta.resolve("tsx/preflight"));
+        const loader = import.meta.resolve("tsx");
+        return ["--require", preflight, "--import", loader];
+    }
+    catch {
+        return [];
+    }
+}
+function flushCappedPartial(log, event, text, maxLineBytes) {
+    let remaining = redactString(text, { namedSecrets: getNamedSecrets() });
+    while (Buffer.byteLength(remaining, "utf8") > maxLineBytes) {
+        const bytes = Buffer.from(remaining, "utf8");
+        const originalBytes = bytes.byteLength;
+        const piece = bytes.subarray(0, maxLineBytes).toString("utf8");
+        log.info(event, piece, {
+            truncated: true,
+            original_bytes: originalBytes,
+        });
+        remaining = bytes.subarray(maxLineBytes).toString("utf8");
+    }
+    return remaining;
+}
+function attachStreamLineLogger(stream, log, event, maxLineBytes) {
+    if (!stream)
+        return;
+    let buffer = "";
+    stream.on("data", (chunk) => {
+        buffer += chunk.toString();
+        const lines = buffer.split("\n");
+        buffer = lines.pop() ?? "";
+        for (const line of lines) {
+            log.info(event, line);
+        }
+        buffer = flushCappedPartial(log, event, buffer, maxLineBytes);
+    });
+    stream.on("end", () => {
+        if (buffer.length > 0) {
+            buffer = flushCappedPartial(log, event, buffer, maxLineBytes);
+            if (buffer.length > 0) {
+                log.info(event, buffer);
+            }
+            buffer = "";
+        }
+    });
+}
 function activeKey(runId, stageId) {
     return `${runId}\0${stageId}`;
 }
@@ -11,7 +80,7 @@ function isStageWorkerResult(value) {
     const type = value.type;
     return type === "succeeded" || type === "failed" || type === "waiting";
 }
-function resultFromExitCode(code) {
+function resultFromExitCode(code, signal, hostInitiatedKill) {
     if (code === STAGE_WORKER_EXIT.SUCCEEDED) {
         return { type: "succeeded" };
     }
@@ -21,9 +90,16 @@ function resultFromExitCode(code) {
     if (code === STAGE_WORKER_EXIT.FAILED) {
         return { type: "failed", reason: "stage failed" };
     }
+    if (signal === "SIGKILL" && !hostInitiatedKill) {
+        return { type: "failed", reason: "worker_oom_killed" };
+    }
     return {
         type: "failed",
-        reason: code === null ? "stage process exited" : `stage process exit ${code}`,
+        reason: code === null
+            ? signal
+                ? `stage process exited (${signal})`
+                : "stage process exited"
+            : `stage process exit ${code}`,
     };
 }
 function resultFromWorkerMessage(msg) {
@@ -35,19 +111,54 @@ function resultFromWorkerMessage(msg) {
     }
     return { type: "failed", reason: msg.reason };
 }
+export function signalProcessGroup(pid, signal) {
+    if (pid === undefined)
+        return;
+    try {
+        if (process.platform === "win32") {
+            process.kill(pid, signal);
+            return;
+        }
+        process.kill(-pid, signal);
+    }
+    catch (error) {
+        const code = error.code;
+        if (code !== "ESRCH" && code !== "EPERM") {
+            throw error;
+        }
+    }
+}
 export class StageProcessLauncher {
     maxActive;
     env;
+    explicitChildExtras;
     cliEntry;
+    workerExecArgv;
+    logger;
+    forkFn;
     active = new Map();
     waitQueue = [];
+    cancelledRuns = new Set();
     slotsHeld = 0;
+    heapMb;
     constructor(options = {}) {
         this.env = options.env ?? process.env;
+        this.explicitChildExtras =
+            options.env !== undefined && options.env !== process.env
+                ? Object.fromEntries(Object.entries(options.env).filter((entry) => entry[1] !== undefined &&
+                    !isForeverDeniedSecret(entry[0]) &&
+                    !isAmbientBlockedEnv(entry[0])))
+                : undefined;
         this.maxActive = readMaxActiveStageProcesses(this.env, options.maxActiveStageProcesses);
-        this.cliEntry =
-            options.cliEntry ??
-                fileURLToPath(new URL("../cli.js", import.meta.url));
+        this.heapMb = getContainerLimits().maxOldSpaceSizeMb;
+        this.cliEntry = options.cliEntry ?? defaultStageCliEntry();
+        this.workerExecArgv = [
+            `--max-old-space-size=${this.heapMb}`,
+            ...(isTypeScriptEntry(this.cliEntry) ? tsxLoaderExecArgv() : []),
+        ];
+        this.logger =
+            options.logger ?? rootLogger.child({ component: "runtime" });
+        this.forkFn = options.forkFn ?? childProcess.fork;
     }
     activeCount() {
         return this.active.size;
@@ -59,47 +170,102 @@ export class StageProcessLauncher {
             startedAt,
         }));
     }
+    signalAllActive(signal) {
+        for (const entry of this.active.values()) {
+            entry.hostInitiatedKill = true;
+            signalProcessGroup(entry.child.pid, signal);
+        }
+    }
     async launch(input) {
-        await this.waitForCapacity();
-        return this.spawnAndWait(input);
+        const acquired = await this.waitForCapacity(input.runId);
+        if (!acquired) {
+            return { type: "failed", reason: "cancelled" };
+        }
+        let slotOwnedBySpawn = false;
+        try {
+            if (this.cancelledRuns.has(input.runId)) {
+                return { type: "failed", reason: "cancelled" };
+            }
+            const resultPromise = this.spawnAndWait(input);
+            slotOwnedBySpawn = true;
+            return await resultPromise;
+        }
+        finally {
+            if (!slotOwnedBySpawn) {
+                this.releaseCapacity();
+            }
+        }
     }
     async cancelRun(runId, killAfterMs = 5000) {
+        this.cancelledRuns.add(runId);
+        const remaining = [];
+        for (const entry of this.waitQueue) {
+            if (entry.runId === runId) {
+                entry.resolve(true);
+            }
+            else {
+                remaining.push(entry);
+            }
+        }
+        this.waitQueue.length = 0;
+        this.waitQueue.push(...remaining);
         const children = [...this.active.values()].filter((entry) => entry.runId === runId);
         if (children.length === 0) {
+            this.cancelledRuns.delete(runId);
             return;
         }
+        for (const entry of children) {
+            entry.hostInitiatedKill = true;
+        }
         await Promise.all(children.map((entry) => new Promise((resolve) => {
             const child = entry.child;
+            const pid = child.pid;
             let settled = false;
+            let exited = false;
+            let escalateTimer;
             const finish = () => {
                 if (settled)
                     return;
                 settled = true;
+                if (escalateTimer !== undefined)
+                    clearTimeout(escalateTimer);
                 resolve();
             };
-            child.once("exit", finish);
-            child.kill("SIGTERM");
+            child.once("exit", () => {
+                exited = true;
+                // Parent may exit on SIGTERM while SIGTERM-proof grandchildren remain.
+                signalProcessGroup(pid, "SIGKILL");
+                finish();
+            });
+            signalProcessGroup(pid, "SIGTERM");
             if (killAfterMs > 0) {
-                setTimeout(() => {
-                    if (!child.killed) {
-                        child.kill("SIGKILL");
+                escalateTimer = setTimeout(() => {
+                    if (!exited) {
+                        signalProcessGroup(pid, "SIGKILL");
                     }
                 }, killAfterMs);
             }
         })));
+        this.cancelledRuns.delete(runId);
     }
-    async waitForCapacity() {
+    async waitForCapacity(runId) {
+        if (this.cancelledRuns.has(runId)) {
+            return false;
+        }
         if (!Number.isFinite(this.maxActive)) {
-            return;
+            return true;
         }
         if (this.slotsHeld < this.maxActive) {
             this.slotsHeld += 1;
-            return;
+            return true;
         }
-        await new Promise((resolve) => {
-            this.waitQueue.push(resolve);
+        const cancelled = await new Promise((resolve) => {
+            this.waitQueue.push({ runId, resolve });
         });
-        return this.waitForCapacity();
+        if (cancelled) {
+            return false;
+        }
+        return this.waitForCapacity(runId);
     }
     releaseCapacity() {
         if (!Number.isFinite(this.maxActive)) {
@@ -108,7 +274,7 @@ export class StageProcessLauncher {
         this.slotsHeld -= 1;
         if (this.waitQueue.length > 0) {
             const next = this.waitQueue.shift();
-            next?.();
+            next?.resolve(false);
         }
     }
     spawnAndWait(input) {
@@ -141,10 +307,47 @@ export class StageProcessLauncher {
         if (input.skipGates) {
             args.push("--skip-gates");
         }
-        const child = fork(this.cliEntry, args, {
+        if (this.cancelledRuns.has(input.runId)) {
+            this.releaseCapacity();
+            return Promise.resolve({ type: "failed", reason: "cancelled" });
+        }
+        const hostEnv = { ...process.env, ...this.env };
+        const built = buildStageEnvironment({
+            hostEnv,
+            cacheVars: ensureStageCacheDirs(),
+            grants: input.grants,
+            attemptHome: input.attemptHome,
+            runVars: input.browserEnv,
+            packageVersion: PACKAGE_VERSION,
+        });
+        for (const warning of built.warnings) {
+            this.logger.warn("stage.env.passthrough", warning, {
+                run_id: input.runId,
+                stage_id: input.stageId,
+            });
+        }
+        const withExtras = this.explicitChildExtras !== undefined
+            ? { ...built.env, ...this.explicitChildExtras }
+            : built.env;
+        const overlaid = overlayStageBindingEnv(withExtras, input.env ?? {}, input.bindingKind ?? "unbound");
+        const withBrowser = input.browserEnv !== undefined
+            ? {
+                ...Object.fromEntries(Object.entries(overlaid).filter(([name]) => !name.startsWith(BROWSER_ENV_PREFIX))),
+                ...input.browserEnv,
+            }
+            : overlaid;
+        const childEnv = {
+            ...withBrowser,
+            [PI_HOME_AUTH_PATH_ENV]: piHomeAuthPath(),
+        };
+        const child = this.forkFn(this.cliEntry, args, {
             cwd: input.rootDir,
-            env: { ...process.env, ...this.env, [SF_STAGE_WORKER]: "1" },
+            env: { ...childEnv, [SF_STAGE_WORKER]: "1" },
             stdio: ["pipe", "pipe", "pipe", "ipc"],
+            detached: true,
+            // Explicit: do not inherit Host execArgv; set heap from cgroup budget.
+            // TypeScript entries also get the tsx loader, since that inheritance is off.
+            execArgv: this.workerExecArgv,
         });
         const key = activeKey(input.runId, input.stageId);
         const tracked = {
@@ -152,25 +355,17 @@ export class StageProcessLauncher {
             runId: input.runId,
             stageId: input.stageId,
             startedAt: Date.now(),
+            hostInitiatedKill: false,
         };
         this.active.set(key, tracked);
-        if (child.stderr) {
-            let stderrBuffer = "";
-            child.stderr.on("data", (chunk) => {
-                stderrBuffer += chunk.toString();
-                const lines = stderrBuffer.split("\n");
-                stderrBuffer = lines.pop() ?? "";
-                for (const line of lines) {
-                    process.stderr.write(`[stage:${input.stageId}] ${line}\n`);
-                }
-            });
-            child.stderr.on("end", () => {
-                if (stderrBuffer.length > 0) {
-                    process.stderr.write(`[stage:${input.stageId}] ${stderrBuffer}\n`);
-                    stderrBuffer = "";
-                }
-            });
-        }
+        const stageLog = this.logger.child({
+            run_id: input.runId,
+            stage_id: input.stageId,
+            ...(input.attempt !== undefined ? { attempt: input.attempt } : {}),
+        });
+        const maxLineBytes = resolveLogMaxLineBytes(this.env);
+        attachStreamLineLogger(child.stdout, stageLog, "stage.stdout", maxLineBytes);
+        attachStreamLineLogger(child.stderr, stageLog, "stage.stderr", maxLineBytes);
         return new Promise((resolve) => {
             let settled = false;
             const cleanup = () => {
@@ -189,10 +384,10 @@ export class StageProcessLauncher {
                     return;
                 finish(resultFromWorkerMessage(message));
             });
-            child.on("exit", (code) => {
+            child.on("exit", (code, signal) => {
                 if (settled)
                     return;
-                finish(resultFromExitCode(code));
+                finish(resultFromExitCode(code, signal, tracked.hostInitiatedKill));
             });
             child.on("error", (err) => {
                 finish({

package/dist/runtime/stageRecovery.d.ts

@@ -1,10 +1,12 @@
 import type { RunStore } from "../runstore/port.js";
 import { type StageAttemptContext } from "./stageAttemptContext.js";
-export declare function failStageAsInterrupted(options: {
+export declare const OPERATOR_CANCEL_REASON = "process_interrupted: operator cancelled run";
+export declare function markStageInterrupted(options: {
     store: RunStore;
     runId: string;
     stageId: string;
     reason: string;
+    status: "failed" | "interrupted";
     attemptCtx?: StageAttemptContext;
 }): Promise<void>;
 export declare function syncRunStatusFromStages(store: RunStore, runId: string): Promise<void>;

package/dist/runtime/stageRecovery.js

@@ -1,8 +1,12 @@
 import { deriveStatusFromStages } from "../runstore/port.js";
+import { refreshRunDiskUsage } from "../runstore/diskUsage.js";
+import { finaliseStoredRunManifest } from "../runstore/runManifest.js";
 import { deriveExecutionPatchFromEvent } from "../runstore/stageExecution.js";
+import { reclaimWorkspaceOnRunSucceeded } from "./repositoryMaterialize.js";
 import { attemptContext, } from "./stageAttemptContext.js";
-export async function failStageAsInterrupted(options) {
-    const { store, runId, stageId, reason, attemptCtx } = options;
+export const OPERATOR_CANCEL_REASON = "process_interrupted: operator cancelled run";
+export async function markStageInterrupted(options) {
+    const { store, runId, stageId, reason, status, attemptCtx } = options;
     let attemptNum;
     let attemptOpt;
     if (attemptCtx !== undefined) {
@@ -20,7 +24,9 @@ export async function failStageAsInterrupted(options) {
             attemptOpt = undefined;
         }
     }
-    const event = { event: "failed", reason };
+    const event = status === "interrupted"
+        ? { event: "interrupted", reason }
+        : { event: "failed", reason };
     await store.appendStageEvent(runId, stageId, event, attemptOpt);
     try {
         const execution = await store.getStageExecution(runId, stageId, attemptNum);
@@ -36,10 +42,19 @@ export async function failStageAsInterrupted(options) {
 export async function syncRunStatusFromStages(store, runId) {
     const meta = await store.readRunMeta(runId);
     const run = await store.readRun(runId);
-    const derived = deriveStatusFromStages(run.stages, meta.pipeline_dag);
+    const derived = deriveStatusFromStages(run.stages, meta.pipeline_dag, meta.status);
     if (meta.status === "succeeded" && derived !== "running")
         return;
     if ((meta.status ?? "created") !== derived) {
         await store.updateRunStatus(runId, derived);
+        if (derived === "succeeded" ||
+            derived === "failed" ||
+            derived === "cancelled") {
+            await finaliseStoredRunManifest(store, runId).catch(() => undefined);
+            await refreshRunDiskUsage(store, runId).catch(() => undefined);
+        }
+        if (derived === "succeeded") {
+            await reclaimWorkspaceOnRunSucceeded(store, runId);
+        }
     }
 }

package/dist/runtime/stageRoots.d.ts

@@ -1,5 +1,7 @@
+import type { RunMeta } from "../runstore/port.js";
 import type { StageAttemptContext } from "./stageAttemptContext.js";
-import type { TaskFile } from "../types/task.js";
+import type { TaskFile, TaskGitIdentity } from "../types/task.js";
+export type DerivedBindingKind = "repository" | "checkout" | "unbound";
 export type StageRoots = {
     mode: "bound" | "unbound";
     cwd: string;
@@ -10,8 +12,43 @@ export type StageRoots = {
     /** Durable Pi auth.json path for ModelRuntime.create({ authPath }). */
     authPath?: string;
 };
+export declare const STAGEFLOW_CHECKOUT = "STAGEFLOW_CHECKOUT";
 export declare const STAGEFLOW_RUN_WORKSPACE = "STAGEFLOW_RUN_WORKSPACE";
+export declare const STAGEFLOW_REPOSITORY = "STAGEFLOW_REPOSITORY";
+export declare const STAGEFLOW_REF = "STAGEFLOW_REF";
+export declare const STAGEFLOW_BASE_SHA = "STAGEFLOW_BASE_SHA";
+export declare const STAGEFLOW_RUN_BRANCH = "STAGEFLOW_RUN_BRANCH";
+export declare const REPOSITORY_ONLY_STAGEFLOW_VARS: readonly ["STAGEFLOW_REPOSITORY", "STAGEFLOW_REF", "STAGEFLOW_BASE_SHA", "STAGEFLOW_RUN_BRANCH"];
 export declare const PI_CODING_AGENT_DIR_ENV = "PI_CODING_AGENT_DIR";
+export type EffectiveGitIdentity = {
+    name: string;
+    email: string;
+};
+export type BuildStageBindingEnvInput = {
+    kind: DerivedBindingKind;
+    runWorkspaceDir: string;
+    checkoutRoot?: string;
+    repository?: string;
+    ref?: string;
+    resolvedSha?: string;
+    runBranch?: string;
+    gitIdentity: EffectiveGitIdentity;
+    hostEnv?: NodeJS.ProcessEnv;
+};
+export declare function resolveEffectiveGitIdentity(hostEnv: NodeJS.ProcessEnv, taskIdentity?: TaskGitIdentity): EffectiveGitIdentity;
+export declare function buildStageBindingEnv(input: BuildStageBindingEnvInput): Record<string, string>;
+export declare function overlayStageBindingEnv(base: NodeJS.ProcessEnv, bindingEnv: Record<string, string>, kind: DerivedBindingKind): Record<string, string | undefined>;
+export declare function bindingKindFromMeta(meta: RunMeta): DerivedBindingKind;
+export declare function stageBindingEnvFromRun(options: {
+    meta: RunMeta;
+    task: TaskFile;
+    runWorkspaceDir: string;
+    hostEnv?: NodeJS.ProcessEnv;
+}): {
+    kind: DerivedBindingKind;
+    env: Record<string, string>;
+    identity: EffectiveGitIdentity;
+};
 export declare function withResolvedAuthPath(roots: StageRoots, factoryCwd: string): StageRoots;
 export declare function rootsForStageWorker(runWorkspaceDir: string, stageId: string, model: string, checkoutRoot?: string, attemptCtx?: StageAttemptContext): StageRoots;
 export declare function bindPiAgentDirEnv(agentDirPath: string): () => void;