stageflow 0.32.0 → 0.35.0

package/dist/agent/stageTurnDriver.js

@@ -0,0 +1,64 @@
+/** Follow-up turns sent when the agent ends its turn without an accepted envelope. */
+export const DEFAULT_EMIT_REMINDERS = 2;
+const PROVIDER_ERROR_MAX_CHARS = 500;
+function lastAssistant(messages) {
+    for (let i = messages.length - 1; i >= 0; i--) {
+        const m = messages[i];
+        if (m && typeof m === "object" && m.role === "assistant")
+            return m;
+    }
+    return undefined;
+}
+/** Trimmed, length-capped provider error text for a stage failure reason. */
+export function formatProviderError(text) {
+    const t = text?.trim() ? text.trim() : "the model provider returned an error";
+    return t.length > PROVIDER_ERROR_MAX_CHARS ? `${t.slice(0, PROVIDER_ERROR_MAX_CHARS)}…` : t;
+}
+/**
+ * Error text of the last assistant turn when it ended on a provider error (bad key, auth,
+ * model unavailable, or a transient error Pi already gave up retrying). Undefined otherwise.
+ */
+export function lastTurnProviderError(messages) {
+    const m = lastAssistant(messages);
+    if (!m || m.stopReason !== "error")
+        return undefined;
+    return formatProviderError(typeof m.errorMessage === "string" ? m.errorMessage : undefined);
+}
+function lastTurnAborted(messages) {
+    return lastAssistant(messages)?.stopReason === "aborted";
+}
+export function emitReminderPrompt(emitToolName, attempt, max, rejectedEmit) {
+    const lines = [
+        `Reminder ${attempt} of ${max}: your turn ended without an accepted ${emitToolName} call, so this stage has not finished.`,
+    ];
+    if (rejectedEmit !== undefined) {
+        lines.push(`Your last ${emitToolName} call was rejected: ${rejectedEmit}`);
+        lines.push(`Fix that and call ${emitToolName} again.`);
+    }
+    else {
+        lines.push(`Continue the stage's work if any is left, then call ${emitToolName}.`);
+    }
+    lines.push(`If the stage cannot be completed, call ${emitToolName} with status "failure" and a summary that says why. Do not end your turn without calling it.`);
+    return lines.join("\n");
+}
+/**
+ * Called after a turn returns. While no envelope was accepted, nudge the agent with a short
+ * follow-up turn, up to `maxReminders` times. Stops at once when an envelope is captured,
+ * the stage is stopping, the turn was aborted, or the turn ended on a provider error.
+ */
+export async function remindUntilEmitted(session, capture, options) {
+    const max = options.maxReminders ?? DEFAULT_EMIT_REMINDERS;
+    let reminders = 0;
+    for (;;) {
+        if (capture.envelope)
+            return { reminders };
+        const providerError = lastTurnProviderError(session.messages);
+        if (providerError !== undefined)
+            return { reminders, providerError };
+        if (options.shouldStop() || lastTurnAborted(session.messages) || reminders >= max) {
+            return { reminders };
+        }
+        reminders++;
+        await session.prompt(emitReminderPrompt(options.emitToolName, reminders, max, capture.error));
+    }
+}

package/dist/agent/toolOutputBudget.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Per-result decision: keep a tool result inline, or spill it to a stage
+ * artifact. Pure; see toolOutputBudgetExtension for the Pi wiring.
+ */
+export declare const TOOL_OUTPUT_HARD_CEILING_BYTES: number;
+export declare const BYTES_PER_TOKEN = 3;
+export declare const RESERVE_MIN_TOKENS = 16000;
+export declare const RESERVE_FRACTION = 0.25;
+export declare const PER_RESULT_MAX_FRACTION = 0.3;
+export declare const UNKNOWN_USAGE_INLINE_MAX_BYTES: number;
+export declare const IMAGE_BLOCK_EST_TOKENS = 1500;
+export declare const PREVIEW_MAX_BYTES: number;
+export type BudgetInput = {
+    resultBytes: number;
+    contextWindow: number;
+    knownTokens: number | null;
+    turnLedgerTokens: number;
+};
+export type SpillReason = "exceeds_remaining" | "exceeds_per_result_cap" | "unknown_usage_fallback" | "adapter_truncated";
+export type BudgetDecision = {
+    kind: "inline";
+    estTokens: number;
+} | {
+    kind: "spill";
+    estTokens: number;
+    reason: SpillReason;
+};
+export declare function estimateTokens(bytes: number): number;
+export declare function decideToolOutput(input: BudgetInput): BudgetDecision;
+export declare function utf8ByteLength(text: string): number;
+export declare function truncateUtf8(text: string, maxBytes: number): string;
+export declare function tryParseJson(text: string): {
+    ok: true;
+    value: unknown;
+} | {
+    ok: false;
+};
+export declare function describeJsonShape(value: unknown): string;
+/**
+ * Short model-facing preview of a spilled body. `body` is the normalized text
+ * that was written to disk, so preview line numbers match what `read` pages.
+ */
+export declare function buildPreview(body: string, options?: {
+    json?: unknown;
+    maxBytes?: number;
+}): string;

package/dist/agent/toolOutputBudget.js

@@ -0,0 +1,97 @@
+/**
+ * Per-result decision: keep a tool result inline, or spill it to a stage
+ * artifact. Pure; see toolOutputBudgetExtension for the Pi wiring.
+ */
+export const TOOL_OUTPUT_HARD_CEILING_BYTES = 8 * 1024 * 1024;
+export const BYTES_PER_TOKEN = 3;
+export const RESERVE_MIN_TOKENS = 16_000;
+export const RESERVE_FRACTION = 0.25;
+export const PER_RESULT_MAX_FRACTION = 0.3;
+export const UNKNOWN_USAGE_INLINE_MAX_BYTES = 50 * 1024;
+export const IMAGE_BLOCK_EST_TOKENS = 1_500;
+export const PREVIEW_MAX_BYTES = 2 * 1024;
+export function estimateTokens(bytes) {
+    return Math.ceil(Math.max(0, bytes) / BYTES_PER_TOKEN);
+}
+export function decideToolOutput(input) {
+    const estTokens = estimateTokens(input.resultBytes);
+    const window = input.contextWindow;
+    if (input.knownTokens === null || !Number.isFinite(window) || window <= 0) {
+        return input.resultBytes <= UNKNOWN_USAGE_INLINE_MAX_BYTES
+            ? { kind: "inline", estTokens }
+            : { kind: "spill", estTokens, reason: "unknown_usage_fallback" };
+    }
+    if (estTokens > PER_RESULT_MAX_FRACTION * window) {
+        return { kind: "spill", estTokens, reason: "exceeds_per_result_cap" };
+    }
+    const reserve = Math.max(RESERVE_MIN_TOKENS, RESERVE_FRACTION * window);
+    const remaining = window - input.knownTokens - input.turnLedgerTokens - reserve;
+    if (estTokens > remaining) {
+        return { kind: "spill", estTokens, reason: "exceeds_remaining" };
+    }
+    return { kind: "inline", estTokens };
+}
+export function utf8ByteLength(text) {
+    return Buffer.byteLength(text, "utf8");
+}
+export function truncateUtf8(text, maxBytes) {
+    const buffer = Buffer.from(text, "utf8");
+    if (buffer.length <= maxBytes)
+        return text;
+    let end = Math.max(0, Math.floor(maxBytes));
+    while (end > 0 && (buffer[end] & 0xc0) === 0x80)
+        end--;
+    return buffer.subarray(0, end).toString("utf8");
+}
+export function tryParseJson(text) {
+    const trimmed = text.trim();
+    if (!(trimmed.startsWith("{") || trimmed.startsWith("[")))
+        return { ok: false };
+    try {
+        return { ok: true, value: JSON.parse(trimmed) };
+    }
+    catch {
+        return { ok: false };
+    }
+}
+const SHAPE_MAX_KEYS = 20;
+function describeValue(value) {
+    if (Array.isArray(value))
+        return `array(${value.length})`;
+    if (value === null)
+        return "null";
+    if (typeof value === "object") {
+        return `object(${Object.keys(value).length} keys)`;
+    }
+    if (typeof value === "string")
+        return `string(${value.length} chars)`;
+    return typeof value;
+}
+export function describeJsonShape(value) {
+    if (Array.isArray(value)) {
+        const first = value.length > 0 ? `; first item: ${describeValue(value[0])}` : "";
+        return `JSON array with ${value.length} items${first}`;
+    }
+    if (value !== null && typeof value === "object") {
+        const entries = Object.entries(value);
+        const shown = entries
+            .slice(0, SHAPE_MAX_KEYS)
+            .map(([key, child]) => `${key}: ${describeValue(child)}`);
+        const more = entries.length > SHAPE_MAX_KEYS
+            ? `, … ${entries.length - SHAPE_MAX_KEYS} more keys`
+            : "";
+        return `JSON object { ${shown.join(", ")}${more} }`;
+    }
+    return `JSON ${describeValue(value)}`;
+}
+/**
+ * Short model-facing preview of a spilled body. `body` is the normalized text
+ * that was written to disk, so preview line numbers match what `read` pages.
+ */
+export function buildPreview(body, options = {}) {
+    const maxBytes = options.maxBytes ?? PREVIEW_MAX_BYTES;
+    const shape = options.json !== undefined ? `Shape: ${describeJsonShape(options.json)}\n` : "";
+    const head = truncateUtf8(body, Math.max(0, maxBytes - utf8ByteLength(shape)));
+    const ellipsis = head.length < body.length ? "\n…" : "";
+    return `${shape}Preview:\n${head}${ellipsis}`;
+}

package/dist/agent/toolOutputBudgetExtension.d.ts

@@ -0,0 +1,22 @@
+import type { ExtensionFactory } from "@earendil-works/pi-coding-agent";
+import { type SpillReason } from "./toolOutputBudget.js";
+import { type SpilledToolOutput } from "./toolOutputSpill.js";
+export declare const STAGEFLOW_TOOL_OUTPUT_EXTENSION_NAME = "stageflow-tool-output";
+export type ToolOutputSpillEvent = {
+    toolName: string;
+    toolCallId: string;
+    reason: SpillReason;
+    originalBytes: number;
+    estTokens: number;
+    runRelativePath?: string;
+    error?: string;
+};
+export type ToolOutputBudgetOptions = {
+    runWorkspaceDir: string;
+    stageId: string;
+    attempt: number;
+    exempt: ReadonlySet<string>;
+    onSpill?: (event: ToolOutputSpillEvent) => void;
+};
+export declare function formatSpillNotice(spilled: SpilledToolOutput, estTokens: number, lostReason?: string): string;
+export declare function createToolOutputBudgetExtension(options: ToolOutputBudgetOptions): ExtensionFactory;

package/dist/agent/toolOutputBudgetExtension.js

@@ -0,0 +1,141 @@
+import { buildPreview, decideToolOutput, estimateTokens, IMAGE_BLOCK_EST_TOKENS, truncateUtf8, UNKNOWN_USAGE_INLINE_MAX_BYTES, utf8ByteLength, } from "./toolOutputBudget.js";
+import { spillToolOutput } from "./toolOutputSpill.js";
+import { discardAdapterSpill, readAdapterSpill } from "./adapterSpill.js";
+export const STAGEFLOW_TOOL_OUTPUT_EXTENSION_NAME = "stageflow-tool-output";
+export function formatSpillNotice(spilled, estTokens, lostReason) {
+    return [
+        `[Tool output saved as stage artifact: ${spilled.runRelativePath}`,
+        ` Read path: ${spilled.absolutePath}`,
+        ` Size: ${spilled.bytes} bytes, ${spilled.lines} lines (~${estTokens} tokens). Too large to include inline.`,
+        lostReason !== undefined
+            ? ` The MCP adapter cut this output and the rest is not available (${lostReason}). This file holds only what was returned.`
+            : undefined,
+        spilled.wrapped
+            ? ` Long lines were wrapped at 4 KiB so the file can be paged; the breaks are not part of the data.`
+            : undefined,
+        spilled.hasLongLines
+            ? ` Some JSON lines are long string values; page them with bash (jq, cut -c) rather than read.`
+            : undefined,
+        ` It is already a stage artifact: read it with \`read\` (offset/limit) or grep it, and list`,
+        ` this path in the envelope artifacts if the stage output depends on it. Do not copy it.]`,
+    ]
+        .filter((line) => line !== undefined)
+        .join("\n");
+}
+function joinText(content) {
+    return content
+        .filter((block) => block.type === "text")
+        .map((block) => block.text)
+        .join("\n");
+}
+export function createToolOutputBudgetExtension(options) {
+    return (pi) => {
+        let turnLedgerTokens = 0;
+        pi.on("message_end", async (event) => {
+            if (event.message.role === "assistant") {
+                turnLedgerTokens = 0;
+            }
+            return undefined;
+        });
+        pi.on("tool_result", async (event, ctx) => {
+            if (options.exempt.has(event.toolName))
+                return undefined;
+            if (event.parentToolCallId !== undefined)
+                return undefined;
+            const images = event.content.filter((block) => block.type === "image");
+            turnLedgerTokens += images.length * IMAGE_BLOCK_EST_TOKENS;
+            const visible = joinText(event.content);
+            // A result the MCP adapter already cut is incomplete inline: always spill
+            // it, from the adapter's full-output file when that file checks out.
+            const adapterSpill = await readAdapterSpill(visible, event.details);
+            const text = adapterSpill.kind === "full" ? adapterSpill.text : visible;
+            const resultBytes = utf8ByteLength(text);
+            const usage = ctx.getContextUsage();
+            const decision = adapterSpill.kind === "none"
+                ? decideToolOutput({
+                    resultBytes,
+                    contextWindow: usage?.contextWindow ?? 0,
+                    knownTokens: usage?.tokens ?? null,
+                    turnLedgerTokens,
+                })
+                : { kind: "spill", estTokens: estimateTokens(resultBytes), reason: "adapter_truncated" };
+            if (decision.kind === "inline") {
+                turnLedgerTokens += decision.estTokens;
+                return undefined;
+            }
+            let replacement;
+            let spillDetails;
+            let fullOutputPath;
+            try {
+                const spilled = await spillToolOutput({
+                    runWorkspaceDir: options.runWorkspaceDir,
+                    stageId: options.stageId,
+                    attempt: options.attempt,
+                    toolCallId: event.toolCallId,
+                    toolName: event.toolName,
+                    text,
+                });
+                if (adapterSpill.kind === "full") {
+                    await discardAdapterSpill(adapterSpill.sourcePath);
+                    fullOutputPath = spilled.absolutePath;
+                }
+                const notice = formatSpillNotice(spilled, decision.estTokens, adapterSpill.kind === "lost" ? adapterSpill.reason : undefined);
+                replacement = `${notice}\n\n${buildPreview(spilled.body, spilled.json !== undefined ? { json: spilled.json } : {})}`;
+                spillDetails = {
+                    spilled: true,
+                    path: spilled.runRelativePath,
+                    bytes: spilled.bytes,
+                    lines: spilled.lines,
+                    originalBytes: resultBytes,
+                    estTokens: decision.estTokens,
+                    reason: decision.reason,
+                    ...(adapterSpill.kind === "lost" ? { incomplete: true } : {}),
+                };
+                options.onSpill?.({
+                    toolName: event.toolName,
+                    toolCallId: event.toolCallId,
+                    reason: decision.reason,
+                    originalBytes: resultBytes,
+                    estTokens: decision.estTokens,
+                    runRelativePath: spilled.runRelativePath,
+                });
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                replacement = `${truncateUtf8(text, UNKNOWN_USAGE_INLINE_MAX_BYTES)}\n\n[Tool output truncated: original ${resultBytes} bytes. Saving the full output as a stage artifact failed: ${message}]`;
+                spillDetails = {
+                    spilled: false,
+                    originalBytes: resultBytes,
+                    estTokens: decision.estTokens,
+                    reason: decision.reason,
+                    error: message,
+                };
+                options.onSpill?.({
+                    toolName: event.toolName,
+                    toolCallId: event.toolCallId,
+                    reason: decision.reason,
+                    originalBytes: resultBytes,
+                    estTokens: decision.estTokens,
+                    error: message,
+                });
+            }
+            turnLedgerTokens += estimateTokens(utf8ByteLength(replacement));
+            const previousDetails = event.details !== null && typeof event.details === "object"
+                ? event.details
+                : {};
+            const outputGuard = previousDetails.outputGuard;
+            return {
+                content: [{ type: "text", text: replacement }, ...images],
+                details: {
+                    ...previousDetails,
+                    ...(fullOutputPath !== undefined &&
+                        outputGuard !== null &&
+                        typeof outputGuard === "object"
+                        ? { outputGuard: { ...outputGuard, fullOutputPath } }
+                        : {}),
+                    stageflowToolOutput: spillDetails,
+                },
+            };
+        });
+    };
+}

package/dist/agent/toolOutputSpill.d.ts

@@ -0,0 +1,34 @@
+export declare const TOOL_OUTPUT_ARTIFACT_DIR = "tool-output";
+export declare const SPILL_WRAP_LINE_BYTES: number;
+export type SpillToolOutputOptions = {
+    runWorkspaceDir: string;
+    stageId: string;
+    attempt: number;
+    toolCallId: string;
+    toolName: string;
+    text: string;
+};
+export type SpilledToolOutput = {
+    runRelativePath: string;
+    absolutePath: string;
+    body: string;
+    bytes: number;
+    lines: number;
+    format: "json" | "text";
+    wrapped: boolean;
+    hasLongLines: boolean;
+    json?: unknown;
+};
+export declare function sanitizeNamePart(value: string): string;
+export declare function wrapLongLines(text: string, maxLineBytes?: number): {
+    text: string;
+    wrapped: boolean;
+};
+export declare function normalizeSpillBody(text: string): {
+    body: string;
+    format: "json" | "text";
+    wrapped: boolean;
+    hasLongLines: boolean;
+    json?: unknown;
+};
+export declare function spillToolOutput(options: SpillToolOutputOptions): Promise<SpilledToolOutput>;

package/dist/agent/toolOutputSpill.js

@@ -0,0 +1,85 @@
+/**
+ * Writes an oversized tool result to the stage attempt's artifacts directory
+ * as `tool-output/<toolCallId>-<toolName>.<json|txt>`, normalized so the
+ * builtin `read` tool can page it by line.
+ */
+import { resolveArtifactTarget } from "../runstore/workspaceLayout.js";
+import { writeContainedArtifact } from "../runstore/writeContainedArtifact.js";
+import { tryParseJson, utf8ByteLength } from "./toolOutputBudget.js";
+export const TOOL_OUTPUT_ARTIFACT_DIR = "tool-output";
+export const SPILL_WRAP_LINE_BYTES = 4 * 1024;
+const NAME_PART_MAX = 64;
+export function sanitizeNamePart(value) {
+    const cleaned = value.replace(/[^A-Za-z0-9_.-]+/g, "_").replace(/^\.+/, "_");
+    const capped = cleaned.slice(0, NAME_PART_MAX);
+    return capped.length > 0 ? capped : "_";
+}
+function codePointUtf8Bytes(codePoint) {
+    if (codePoint < 0x80)
+        return 1;
+    if (codePoint < 0x800)
+        return 2;
+    if (codePoint < 0x10000)
+        return 3;
+    return 4;
+}
+export function wrapLongLines(text, maxLineBytes = SPILL_WRAP_LINE_BYTES) {
+    let wrapped = false;
+    const out = [];
+    for (const line of text.split("\n")) {
+        if (utf8ByteLength(line) <= maxLineBytes) {
+            out.push(line);
+            continue;
+        }
+        wrapped = true;
+        // Cut by code points so a chunk never splits a surrogate pair.
+        let start = 0;
+        let offset = 0;
+        let bytes = 0;
+        for (const ch of line) {
+            const size = codePointUtf8Bytes(ch.codePointAt(0));
+            if (bytes + size > maxLineBytes && offset > start) {
+                out.push(line.slice(start, offset));
+                start = offset;
+                bytes = 0;
+            }
+            bytes += size;
+            offset += ch.length;
+        }
+        out.push(line.slice(start));
+    }
+    return { text: out.join("\n"), wrapped };
+}
+export function normalizeSpillBody(text) {
+    const parsed = tryParseJson(text);
+    if (parsed.ok) {
+        // JSON stays valid: long string values are not wrapped.
+        const body = JSON.stringify(parsed.value, null, 2);
+        return {
+            body,
+            format: "json",
+            wrapped: false,
+            json: parsed.value,
+            hasLongLines: body.split("\n").some((line) => utf8ByteLength(line) > SPILL_WRAP_LINE_BYTES),
+        };
+    }
+    const { text: body, wrapped } = wrapLongLines(text);
+    return { body, format: "text", wrapped, hasLongLines: false };
+}
+export async function spillToolOutput(options) {
+    const normalized = normalizeSpillBody(options.text);
+    const fileName = `${sanitizeNamePart(options.toolCallId)}-${sanitizeNamePart(options.toolName)}.${normalized.format === "json" ? "json" : "txt"}`;
+    const { absolutePath, runRelativePath, artifactsDir } = resolveArtifactTarget(options.runWorkspaceDir, options.stageId, options.attempt, `${TOOL_OUTPUT_ARTIFACT_DIR}/${fileName}`);
+    await writeContainedArtifact(options.runWorkspaceDir, artifactsDir, absolutePath, normalized.body);
+    return {
+        runRelativePath,
+        absolutePath,
+        body: normalized.body,
+        bytes: utf8ByteLength(normalized.body),
+        lines: normalized.body.split("\n").length,
+        format: normalized.format,
+        wrapped: normalized.wrapped,
+        hasLongLines: normalized.hasLongLines,
+        ...(normalized.json !== undefined ? { json: normalized.json } : {}),
+    };
+}

package/dist/browser/agentBrowserLiveViewRelay.d.ts

@@ -0,0 +1,41 @@
+import type { BrowserRunner } from "./browserHost.js";
+import type { LiveViewRelay } from "./liveViewRelay.js";
+export declare const DEFAULT_MAX_INPUT_BATCH = 64;
+export declare const DEFAULT_MAX_INPUT_EVENTS_PER_SECOND = 500;
+export declare const DEFAULT_REOPEN_MIN_INTERVAL_MS = 2000;
+export type LiveViewRelayErrorCode = "stream_port_unavailable" | "stream_connect_failed";
+export declare class LiveViewRelayError extends Error {
+    readonly code: LiveViewRelayErrorCode;
+    constructor(code: LiveViewRelayErrorCode, message: string);
+}
+export type WebSocketLike = {
+    readonly readyState: number;
+    onopen: ((event: unknown) => void) | null;
+    onmessage: ((event: {
+        data: unknown;
+    }) => void) | null;
+    onerror: ((event: unknown) => void) | null;
+    onclose: ((event: unknown) => void) | null;
+    send(data: string): void;
+    close(): void;
+};
+export type WebSocketConstructor = new (url: string) => WebSocketLike;
+export type AgentBrowserLiveViewRelayOptions = {
+    runner: BrowserRunner;
+    WebSocketImpl?: WebSocketConstructor;
+    maxBatch?: number;
+    maxEventsPerSecond?: number;
+    now?: () => number;
+    /** Pause between re-target polls of the tab list and stream reconnects. */
+    retryDelayMs?: number;
+    /** How long a fresh stream connection may stay frameless before one restart nudge. */
+    firstFrameTimeoutMs?: number;
+    /** An open confirm or prompt nobody answers is dismissed after this long. */
+    dialogTimeoutMs?: number;
+    /** Minimum gap between two reopenTab() calls on one session. */
+    reopenMinIntervalMs?: number;
+    /** Timer seam for the first-frame nudge and the dialog time-out; returns a cancel function. */
+    schedule?: (fn: () => void, ms: number) => () => void;
+};
+export declare function validEvent(event: unknown): boolean;
+export declare function createAgentBrowserLiveViewRelay(options: AgentBrowserLiveViewRelayOptions): LiveViewRelay;