stageflow 0.19.0 → 0.23.0

package/dist/runtime/settingsFile.js

@@ -150,6 +150,25 @@ export function writeMaxConcurrentToContext(ctx, maxConcurrent) {
 export function writeMaxConcurrentToFile(cwd, maxConcurrent) {
     writeFactorySettings(cwd, { maxConcurrent });
 }
+export function readMaxConcurrentFromGlobal() {
+    return parseFactorySettingsFromRaw(readRawSettingsFromFile(globalSettingsFilePath())).maxConcurrent;
+}
+export function writeMaxConcurrentToGlobal(maxConcurrent) {
+    if (parseSlotCount(maxConcurrent) === undefined) {
+        throw new Error(INVALID_SLOT_COUNT_MESSAGE);
+    }
+    const current = parseFactorySettingsFromRaw(readRawSettingsFromFile(globalSettingsFilePath()));
+    const next = { ...current, maxConcurrent };
+    const out = {};
+    if (next.maxConcurrent !== undefined) {
+        out.maxConcurrent = next.maxConcurrent;
+    }
+    if (next.credentialSource !== undefined) {
+        out.credentialSource = next.credentialSource;
+    }
+    mkdirSync(path.dirname(globalSettingsFilePath()), { recursive: true });
+    writeFileSync(globalSettingsFilePath(), `${JSON.stringify(out, null, 2)}\n`, "utf8");
+}
 export function readCredentialSourceFromFile(cwd) {
     return readFactorySettings(cwd).credentialSource;
 }

package/dist/runtime/stageAttemptBootstrap.d.ts

@@ -1,4 +1,5 @@
 import type { AgentPort, FeedbackLoopContext, StageHandle, StageRunInput, StageSessionMode } from "../agent/port.js";
+import { type StageStreamLogWriter } from "./stageStreamLog.js";
 import type { RunStore } from "../runstore/port.js";
 import type { StageEnvelope } from "../types/envelope.js";
 import type { ResolvedPipelineDag } from "../types/pipeline.js";
@@ -23,6 +24,8 @@ export type StageAttemptOpenInput = {
     attemptCtx?: StageAttemptContext;
     operatorCatalog?: OperatorCatalog;
     onActivity?: StageRunInput["onActivity"];
+    /** Test seam; defaults to the real createStageStreamLogWriter. */
+    streamLogWriterFactory?: (streamLogPath: string) => StageStreamLogWriter;
     roots?: StageRoots;
     resumeToken?: string;
     sessionMode?: StageSessionMode;

package/dist/runtime/stageAttemptBootstrap.js

@@ -1,7 +1,8 @@
 import { resolveSkillByName } from "../config/listSkills.js";
 import { mkdir } from "node:fs/promises";
 import { MCP_CATALOG_FILENAME, STAGEFLOW_STAGE_ARTIFACTS_DIR_ENV, StageMcpError, resolveStageMcpServers, stampStagePromptArtifactsDir, } from "../config/resolveStageMcpServers.js";
-import { attemptArtifactsDir } from "../runstore/workspaceLayout.js";
+import { attemptArtifactsDir, attemptStreamLogPath } from "../runstore/workspaceLayout.js";
+import { createStageStreamLogWriter, } from "./stageStreamLog.js";
 import { resolveCloneEmitContext, resolveForkEmitContext } from "../config/resolveForkEmitContext.js";
 import { createAttemptQaTrailReader } from "../hitl/qaTrail.js";
 import { assertPriorInputPayload, payloadInstanceMismatch, } from "../envelope/payloadSchema.js";
@@ -199,6 +200,7 @@ export async function openStageAttempt(input) {
         return priorInput;
     const roots = resolveAttemptRoots(input, stageId);
     const attempt = input.attemptCtx?.attempt ?? 1;
+    const streamWriter = (input.streamLogWriterFactory ?? createStageStreamLogWriter)(attemptStreamLogPath(input.workspaceDir, stageId, attempt));
     const resumeToken = input.resumeToken ??
         resumeSessionFilePath(input.workspaceDir, stageId, attempt);
     const forkEmitContext = resolveForkEmitContext(input.dag, definitionId);
@@ -231,7 +233,11 @@ export async function openStageAttempt(input) {
             : {}),
         resumeToken,
         ...(input.sessionMode !== undefined ? { sessionMode: input.sessionMode } : {}),
-        onActivity: input.onActivity,
+        onActivity: (event) => {
+            input.onActivity?.(event);
+            void streamWriter.flush();
+        },
+        onAssistantTextDelta: streamWriter.onDelta,
         forkEmitContext,
         cloneEmitContext,
         ...(feedbackLoopEmitContext !== undefined ? { feedbackLoopEmitContext } : {}),

package/dist/runtime/stageStreamLog.d.ts

@@ -0,0 +1,17 @@
+/** First line of every stream.log: the logical byte offset of the content that follows. */
+export declare const STREAM_LOG_HEADER_PREFIX = "BASE:";
+export declare function encodeStreamLogHeader(base: number): string;
+/** Split a stream.log buffer into its logical base offset and content bytes. */
+export declare function parseStreamLogHeader(raw: Buffer): {
+    base: number;
+    headerLength: number;
+};
+export type StageStreamLogWriter = {
+    onDelta(delta: string): void;
+    /** Idempotent; resolves once any pending buffered text has been written to disk. */
+    flush(): Promise<void>;
+};
+export declare function createStageStreamLogWriter(streamLogPath: string, options?: {
+    flushThrottleMs?: number;
+    capBytes?: number;
+}): StageStreamLogWriter;

package/dist/runtime/stageStreamLog.js

@@ -0,0 +1,89 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { STREAM_FLUSH_THROTTLE_MS } from "../agent/activityObserver.js";
+import { redactSecrets } from "../agent/streamLogRedact.js";
+const DEFAULT_CAP_BYTES = 256 * 1024;
+/** First line of every stream.log: the logical byte offset of the content that follows. */
+export const STREAM_LOG_HEADER_PREFIX = "BASE:";
+export function encodeStreamLogHeader(base) {
+    return `${STREAM_LOG_HEADER_PREFIX}${base}\n`;
+}
+/** Split a stream.log buffer into its logical base offset and content bytes. */
+export function parseStreamLogHeader(raw) {
+    const prefix = Buffer.from(STREAM_LOG_HEADER_PREFIX, "utf8");
+    if (raw.subarray(0, prefix.length).equals(prefix)) {
+        const newlineIdx = raw.indexOf(0x0a);
+        if (newlineIdx !== -1) {
+            const baseStr = raw.subarray(prefix.length, newlineIdx).toString("utf8");
+            const base = Number(baseStr);
+            if (Number.isFinite(base) && base >= 0) {
+                return { base, headerLength: newlineIdx + 1 };
+            }
+        }
+    }
+    return { base: 0, headerLength: 0 };
+}
+async function appendChunk(streamLogPath, chunk, capBytes) {
+    await mkdir(path.dirname(streamLogPath), { recursive: true });
+    let base = 0;
+    let contentBytes;
+    try {
+        const raw = await readFile(streamLogPath);
+        const parsed = parseStreamLogHeader(raw);
+        base = parsed.base;
+        contentBytes = raw.subarray(parsed.headerLength);
+    }
+    catch {
+        contentBytes = Buffer.alloc(0);
+    }
+    contentBytes = Buffer.concat([contentBytes, Buffer.from(chunk, "utf8")]);
+    const headerBytes = Buffer.from(encodeStreamLogHeader(base), "utf8");
+    if (headerBytes.length + contentBytes.length > capBytes) {
+        let dropBytes = Math.floor(contentBytes.length / 2);
+        // Never split a multi-byte UTF-8 codepoint: skip past continuation bytes.
+        while (dropBytes < contentBytes.length &&
+            (contentBytes[dropBytes] & 0xc0) === 0x80) {
+            dropBytes++;
+        }
+        base += dropBytes;
+        contentBytes = contentBytes.subarray(dropBytes);
+    }
+    const finalHeader = Buffer.from(encodeStreamLogHeader(base), "utf8");
+    await writeFile(streamLogPath, Buffer.concat([finalHeader, contentBytes]));
+}
+export function createStageStreamLogWriter(streamLogPath, options = {}) {
+    const flushThrottleMs = options.flushThrottleMs ?? STREAM_FLUSH_THROTTLE_MS;
+    const capBytes = options.capBytes ?? DEFAULT_CAP_BYTES;
+    let buffer = "";
+    let timer = null;
+    // Serializes disk writes so the throttle timer and an eager flush() never race.
+    let writeChain = Promise.resolve();
+    function flushNow() {
+        if (timer !== null) {
+            clearTimeout(timer);
+            timer = null;
+        }
+        if (!buffer)
+            return writeChain;
+        const chunk = redactSecrets(buffer);
+        buffer = "";
+        writeChain = writeChain.then(() => appendChunk(streamLogPath, chunk, capBytes));
+        return writeChain;
+    }
+    return {
+        onDelta(delta) {
+            if (!delta)
+                return;
+            buffer += delta;
+            if (timer === null) {
+                timer = setTimeout(() => {
+                    timer = null;
+                    void flushNow();
+                }, flushThrottleMs);
+            }
+        },
+        flush() {
+            return flushNow();
+        },
+    };
+}

package/dist/runtime/stageWorker.js

@@ -3,6 +3,7 @@ import { asAgentBackendId } from "../agent/agentBackend.js";
 import { loadStageflowManifestOutcome } from "../config/loadStageflowManifest.js";
 import { definitionIdForInstance } from "../runstore/stageInstanceId.js";
 import { createRunStore } from "../runstore/createStore.js";
+import { globalStageflowHome } from "../project/globalHome.js";
 import { loadRunContext } from "./resumeReconstruct.js";
 import { bindPiAgentDirEnv, rootsForStageWorker, withResolvedAuthPath, } from "./stageRoots.js";
 import { runStage, isRunStageWaiting, } from "./stageRunner.js";
@@ -11,7 +12,7 @@ import { attemptContext } from "./stageAttemptContext.js";
 import { loadActiveFeedbackLoopContext } from "./feedbackLoopCoordinator.js";
 import { outcomeToWorkerResult, STAGE_WORKER_EXIT, } from "./stageWorkerProtocol.js";
 export async function runStageWorker(input) {
-    const store = createRunStore({ rootDir: input.rootDir });
+    const store = createRunStore({ rootDir: globalStageflowHome() });
     const { meta, task, loaded } = await loadRunContext(store, input.runId, input.rootDir);
     const definitionId = definitionIdForInstance(meta.pipeline_dag, input.stageId);
     const stage = loaded.stages.find((s) => s.id === definitionId);

package/dist/server/bootstrap.js

@@ -15,7 +15,7 @@ export async function bootstrapStageflowHost(options) {
         ? findProjectRoot(rootDir) !== null
         : ctx.isGitProject;
     const rawStore = options.store ??
-        createRunStore({ rootDir, kind: options.storeKind });
+        createRunStore({ rootDir: ctx.globalHome, kind: options.storeKind });
     const boundBus = isRunStoreWrapped(rawStore)
         ? getRunChangeBusFromWrappedStore(rawStore)
         : undefined;
@@ -41,7 +41,14 @@ export async function bootstrapStageflowHost(options) {
     const mcpStateless = resolveMcpStateless({
         mcpStateless: options.mcpStateless,
     });
-    const mcpHandler = createMcpHttpHandler({ manager, store, cwd, runChangeBus }, { mcpStateless });
+    const mcpHandler = createMcpHttpHandler({
+        manager,
+        store,
+        cwd,
+        runChangeBus,
+        providerAuthContext: options.providerAuthContext,
+        projectRoot: rootDir,
+    }, { mcpStateless });
     return {
         cwd,
         agentDir,

package/dist/server/ensureGlobalService.d.ts

@@ -0,0 +1,35 @@
+import { type ChildProcess, spawn as spawnDefault } from "node:child_process";
+/** Coarse up/down probe result, used by CLI code that only needs "is it safe to talk to the host." */
+export type HostProbeResult = "up" | "down";
+/**
+ * Finer-grained probe result used to auto-start: "unreachable" (nothing
+ * listening, safe to spawn) is distinct from "unhealthy" (something is
+ * listening on the port but didn't answer with valid health JSON, so it's
+ * almost certainly not our service and spawning would just collide).
+ */
+export type ServiceProbeResult = "up" | "unreachable" | "unhealthy";
+export declare function hostBaseUrl(): string;
+export declare function probeGlobalServiceDetailed(): Promise<ServiceProbeResult>;
+export declare function defaultProbeHost(): Promise<HostProbeResult>;
+export type EnsureGlobalServiceResult = {
+    ok: true;
+    alreadyRunning: boolean;
+} | {
+    ok: false;
+    reason: "port_occupied" | "spawn_failed" | "timed_out";
+    message: string;
+};
+export interface EnsureGlobalServiceOptions {
+    /** Path to the CLI entry script to spawn `<entry> mcp` against. Defaults to the running process's own entry (process.argv[1]). */
+    cliEntry?: string;
+    probeHost?: () => Promise<ServiceProbeResult>;
+    spawnFn?: (command: string, args: string[], options: Parameters<typeof spawnDefault>[2]) => ChildProcess;
+    pollIntervalMs?: number;
+    timeoutMs?: number;
+}
+/**
+ * Health-probe-then-spawn: makes sure a global Stageflow service is
+ * reachable at the well-known port, starting one headlessly (`<cliEntry>
+ * mcp`, detached) if nothing answers yet. No-ops if one is already up.
+ */
+export declare function ensureGlobalService(options?: EnsureGlobalServiceOptions): Promise<EnsureGlobalServiceResult>;

package/dist/server/ensureGlobalService.js

@@ -0,0 +1,138 @@
+import { spawn as spawnDefault } from "node:child_process";
+import { openSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { ensureGlobalHome } from "../project/globalHome.js";
+import { DEFAULT_PORT } from "./createHttpHost.js";
+/**
+ * The well-known port the global service listens on. Overridable via
+ * STAGEFLOW_SERVICE_PORT — real usage never sets this (there's exactly one
+ * global service by design), but tests use it to point a spawned `sf run`
+ * subprocess at an isolated, already-running test server instead of the
+ * real machine's global service.
+ */
+function resolveServicePort() {
+    const raw = process.env.STAGEFLOW_SERVICE_PORT;
+    if (raw === undefined)
+        return DEFAULT_PORT;
+    const parsed = Number(raw);
+    return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_PORT;
+}
+/**
+ * The published `sf` binary is compiled JS (`dist/cli.js`), directly
+ * runnable as `node <entry> mcp`. In dev/test, `cliEntry` is instead raw
+ * TypeScript source (`src/cli.ts`), which plain node can't execute — route
+ * through the same tsx loader `npm run dev` already uses for that case.
+ */
+function resolveSpawnArgs(cliEntry) {
+    const portArgs = ["--port", String(resolveServicePort())];
+    if (!/\.(?:m|c)?ts$/.test(cliEntry)) {
+        return [cliEntry, "mcp", ...portArgs];
+    }
+    try {
+        const tsxCliUrl = import.meta.resolve("tsx/cli");
+        return [fileURLToPath(tsxCliUrl), cliEntry, "mcp", ...portArgs];
+    }
+    catch {
+        return [cliEntry, "mcp", ...portArgs];
+    }
+}
+export function hostBaseUrl() {
+    return `http://127.0.0.1:${resolveServicePort()}`;
+}
+export async function probeGlobalServiceDetailed() {
+    const ac = new AbortController();
+    const timer = setTimeout(() => ac.abort(), 1500);
+    try {
+        let res;
+        try {
+            res = await fetch(`${hostBaseUrl()}/api/health`, { signal: ac.signal });
+        }
+        catch {
+            return "unreachable";
+        }
+        if (res.status !== 200)
+            return "unhealthy";
+        try {
+            JSON.parse(await res.text());
+        }
+        catch {
+            return "unhealthy";
+        }
+        return "up";
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+export async function defaultProbeHost() {
+    return (await probeGlobalServiceDetailed()) === "up" ? "up" : "down";
+}
+const DEFAULT_POLL_INTERVAL_MS = 200;
+const DEFAULT_TIMEOUT_MS = Number(process.env.STAGEFLOW_AUTOSTART_TIMEOUT_MS) || 10_000;
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Health-probe-then-spawn: makes sure a global Stageflow service is
+ * reachable at the well-known port, starting one headlessly (`<cliEntry>
+ * mcp`, detached) if nothing answers yet. No-ops if one is already up.
+ */
+export async function ensureGlobalService(options = {}) {
+    const probeHost = options.probeHost ?? probeGlobalServiceDetailed;
+    const spawnFn = options.spawnFn ?? spawnDefault;
+    const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+    const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const initial = await probeHost();
+    if (initial === "up") {
+        return { ok: true, alreadyRunning: true };
+    }
+    if (initial === "unhealthy") {
+        return {
+            ok: false,
+            reason: "port_occupied",
+            message: `Port ${resolveServicePort()} is already in use by a process that isn't the Stageflow service (its health check didn't return valid health JSON). Free the port and try again.`,
+        };
+    }
+    const cliEntry = options.cliEntry ?? process.argv[1] ?? "";
+    const globalHome = ensureGlobalHome();
+    const logPath = path.join(globalHome, "service.log");
+    let logFd;
+    try {
+        logFd = openSync(logPath, "a");
+    }
+    catch (err) {
+        return {
+            ok: false,
+            reason: "spawn_failed",
+            message: `Failed to open service log at ${logPath}: ${err instanceof Error ? err.message : String(err)}`,
+        };
+    }
+    try {
+        const spawnArgs = resolveSpawnArgs(cliEntry);
+        const child = spawnFn(process.execPath, spawnArgs, {
+            detached: true,
+            stdio: ["ignore", logFd, logFd],
+        });
+        child.unref();
+    }
+    catch (err) {
+        return {
+            ok: false,
+            reason: "spawn_failed",
+            message: `Failed to spawn the global Stageflow service: ${err instanceof Error ? err.message : String(err)}`,
+        };
+    }
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+        await sleep(pollIntervalMs);
+        if ((await probeHost()) === "up") {
+            return { ok: true, alreadyRunning: false };
+        }
+    }
+    return {
+        ok: false,
+        reason: "timed_out",
+        message: `Timed out after ${timeoutMs}ms waiting for the global Stageflow service to become healthy. Check the log at ${logPath}.`,
+    };
+}

package/dist/server/http.d.ts

@@ -2,8 +2,9 @@ import type { AgentPort } from "../agent/port.js";
 import { type ProviderAuthContext } from "../agent/providerAuth.js";
 import type { RunStoreKind } from "../runstore/createStore.js";
 import type { RunStore } from "../runstore/port.js";
+import type { RunManager } from "../runtime/runManager.js";
 import type { RunChangeBus } from "../runtime/runChangeBus.js";
-import { DEFAULT_PORT, type HttpHostEnvelope } from "./createHttpHost.js";
+import { DEFAULT_PORT, type HttpHostEnvelope, type HttpHostRouteContext } from "./createHttpHost.js";
 export type UiServerOptions = {
     agent: AgentPort;
     cwd?: string;
@@ -20,5 +21,23 @@ export type UiServerOptions = {
     runChangeBus?: RunChangeBus;
 };
 export declare function defaultUiDistDir(): string;
+export type OperatorRouteDeps = {
+    manager: RunManager;
+    store: RunStore;
+    cwd: string;
+    agentDir: string;
+    rootDir: string;
+    providerAuthContext: ProviderAuthContext | undefined;
+    /** Omit for a headless service (e.g. `sf mcp`) — GETs outside the API surface just 404. */
+    uiDistDir?: string;
+};
+/**
+ * The full operator-console route surface: REST API (runs, stages,
+ * catalog, settings, providers) plus, when `uiDistDir` is given, static
+ * console UI file serving. Shared by `startUiServer` (browser-facing) and
+ * `startMcpServer` (headless global-service daemon) — both need the REST
+ * API; only the former needs the UI files.
+ */
+export declare function createOperatorRoutes(deps: OperatorRouteDeps): (ctx: HttpHostRouteContext) => Promise<boolean | void>;
 export declare function startUiServer(options: UiServerOptions): Promise<HttpHostEnvelope>;
 export { DEFAULT_PORT };