stageflow 0.40.0 → 0.44.0

package/README.md

@@ -10,6 +10,15 @@ Open-source runtime for **configurable multi-stage agent workflows** with typed
 [![GitHub issues](https://img.shields.io/github/issues/tejasghutukade/stageflow)](https://github.com/tejasghutukade/stageflow/issues)
 [![CI](https://img.shields.io/github/actions/workflow/status/tejasghutukade/stageflow/ci.yml?branch=main)](https://github.com/tejasghutukade/stageflow/actions/workflows/ci.yml)
 
+<p align="center">
+  <a href="https://github.com/tejasghutukade/stageflow/releases/download/media/stageflow-demo.mp4"><img src="https://github.com/tejasghutukade/stageflow/releases/download/media/stageflow-demo.gif" alt="Stageflow demo: a pipeline runs, pauses at a human gate, resumes, and finishes in the operator console" width="800"></a>
+</p>
+
+<p align="center">
+  <a href="https://github.com/tejasghutukade/stageflow/releases/download/media/stageflow-demo.mp4">Watch the 76s demo (MP4)</a> ·
+  <a href="https://github.com/tejasghutukade/stageflow/releases/download/media/stageflow-demo-long.mp4">Full walkthrough (3 min)</a>
+</p>
+
 Author pipeline-owned YAML in your project. Stageflow schedules each stage in a fresh agent session, moves context through explicit envelopes and artifacts, and persists run state under `.stageflow/`. [Pi](https://github.com/badlogic/pi-mono) is the current agent execution backend. Bins: **`sf`** and **`stageflow`**.
 
 ## Why Stageflow?
@@ -35,6 +44,7 @@ The same pipeline runs three ways without rewriting anything:
 - **Verified stage execution** — after-phase `verify` checks with durable evidence, plus `on_verify_fail` repair or manual operator recovery ([guide](docs/verified-stage-execution.md))
 - **HITL gates** — operator questions in the console; CI exits `2` when a run is waiting
 - **Agents** — project-owned, revisioned agents (`agents/*.agent.yaml`) with granted tools, ask-first approvals, traces, channels, schedules and a public web chat page ([guide](docs/agents.md))
+- **Evals** — check a stage or an agent against a dataset: graders, repeats, honest statistics, compare two runs, and sort failures into what to fix ([guide](docs/evals.md))
 - **Operator console** — an Inbox for gates and agent approvals, pipeline runs, agents and their sessions, connections, MCP servers and settings at `http://127.0.0.1:3847` ([guide](docs/operator-console.md))
 - **MCP endpoint** — Streamable HTTP at `/mcp` when `sf ui` or `sf mcp` is running
 - **MCP servers for agents and stages** — add servers, secrets and OAuth sign-in from Settings → Project MCP, or in the project `.mcp.json` ([guide](docs/mcp-servers.md))
@@ -57,14 +67,14 @@ See [Architecture](docs/architecture.md) for component boundaries, execution flo
 
 ### Docker (recommended)
 
-Run Stageflow from the published image. It is [`ghcr.io/tejasghutukade/stageflow`](https://github.com/tejasghutukade/stageflow/pkgs/container/stageflow). `0.39` is the current minor tag (`0.39.0` is that exact release).
+Run Stageflow from the published image. It is [`ghcr.io/tejasghutukade/stageflow`](https://github.com/tejasghutukade/stageflow/pkgs/container/stageflow). `0.42` is the current minor tag (`0.42.0` is that exact release).
 
 ```bash
 docker run --rm --name stageflow \
   -e STAGEFLOW_ALLOW_UNAUTHENTICATED_BIND=1 \
   -p 127.0.0.1:3847:3847 \
   -v stageflow-data:/data \
-  ghcr.io/tejasghutukade/stageflow:0.39 \
+  ghcr.io/tejasghutukade/stageflow:0.42 \
   sf ui
 ```
 
@@ -81,7 +91,7 @@ docker run --rm --name stageflow \
   -p 127.0.0.1:3847:3847 \
   -v stageflow-data:/data \
   -v "$PWD":/project \
-  ghcr.io/tejasghutukade/stageflow:0.39 \
+  ghcr.io/tejasghutukade/stageflow:0.42 \
   sf ui
 ```
 
@@ -288,7 +298,7 @@ Full docs: **[tejasghutukade.github.io/stageflow](https://tejasghutukade.github.
 | [docs/hosted.md](docs/hosted.md) | Embed Stageflow from another application and receive ordered run events |
 | [docs/quickstart.md](docs/quickstart.md) | Expanded quick start |
 | [docs/yaml-catalog.md](docs/yaml-catalog.md) | Pipelines, stages, tasks schema |
-| [docs/cli-reference.md](docs/cli-reference.md) | `sf init`, `sf run`, `sf run-stage`, `sf graph`, `sf validate`, `sf ui`, `sf mcp`, `sf envelope`, `sf export-run`, `sf artifact`, `sf skills`, `sf providers` |
+| [docs/cli-reference.md](docs/cli-reference.md) | `sf init`, `sf run`, `sf run-stage`, `sf graph`, `sf validate`, `sf ui`, `sf mcp`, `sf envelope`, `sf export-run`, `sf artifact`, `sf skills`, `sf providers`, `sf eval` |
 | [docs/envelopes.md](docs/envelopes.md) | Handoff envelope contract |
 | [docs/verified-stage-execution.md](docs/verified-stage-execution.md) | After-phase `verify`, evidence, and `on_verify_fail` repair / manual recovery |
 | [docs/hitl.md](docs/hitl.md) | Gate kinds, `--skip-gates`, exit code `2` |
@@ -298,6 +308,7 @@ Full docs: **[tejasghutukade.github.io/stageflow](https://tejasghutukade.github.
 | [docs/providers.md](docs/providers.md) | Pi providers, `sf providers` |
 | [docs/docker.md](docs/docker.md) | Docker Compose local try, image pull/run, backup, self-hosting |
 | [docs/operator-console.md](docs/operator-console.md) | Console navigation, command palette, shortcuts and routes; one page per area ([Inbox](docs/console-inbox.md), [Runs](docs/console-runs.md), [Agents](docs/console-agents.md), [Connections](docs/console-connections.md), [Settings](docs/console-settings.md)) |
+| [docs/evals.md](docs/evals.md) | Evals for stages and agents: datasets, graders, runs, compare, failure sorting, the engine |
 | [docs/agents.md](docs/agents.md) | Agents: definition, runtime, approvals, channels, schedules, web chat |
 | [docs/mcp-servers.md](docs/mcp-servers.md) | Add MCP servers, secrets and OAuth; use them from agents and stages |
 | [docs/channels.md](docs/channels.md) | Connections and communication channels |

package/dist/agent/activity.d.ts

@@ -43,6 +43,24 @@ export type StageActivityEvent = {
     event: "message";
     role: string;
     text?: string;
+} | {
+    event: "context_usage";
+    /** Estimated tokens in the model's context after the latest response. */
+    tokens: number;
+    contextWindow: number;
+    /** Whole-number share of the window, 0-100. */
+    percent: number;
+} | {
+    event: "context_compacted";
+    reason: "manual" | "threshold" | "overflow";
+    tokensBefore: number;
+    /** Best estimate after compaction; null if unknown. */
+    tokensAfter: number | null;
+    summaryChars: number;
+    costUsd: number;
+    willRetry: boolean;
+    aborted: boolean;
+    error?: string;
 } | {
     event: "operator_prompt";
     prompt: AskOperatorPrompt;

package/dist/agent/claudeAdapter.js

@@ -225,7 +225,13 @@ async function runTurn(input, model, markerPath, promptText, resumeSessionId, us
             preEmitCheckOptions: {
                 checks: input.stage.pre_emit_checks,
                 readQaTrail: input.readQaTrail,
+                artifactPresence: {
+                    runWorkspaceDir: input.roots.runWorkspaceDir,
+                    stageId: runtimeStageId(input),
+                    attempt: input.roots.attempt ?? 1,
+                },
             },
+            feedbackLoopEmitContext: input.feedbackLoopEmitContext,
             writeStageArtifact: {
                 runWorkspaceDir: input.roots.runWorkspaceDir,
                 stageId: runtimeStageId(input),
@@ -255,6 +261,7 @@ async function runTurn(input, model, markerPath, promptText, resumeSessionId, us
                 model,
                 systemPrompt: { type: "custom", prompt: input.stage.system_prompt },
                 settingSources: [],
+                settings: { autoCompactEnabled: true },
                 tools: CLAUDE_BUILTIN_TOOLS,
                 mcpServers: { [STAGEFLOW_MCP_SERVER_NAME]: mcpServer, ...passedServers },
                 permissionMode: "bypassPermissions",

package/dist/agent/claudeTools.d.ts

@@ -30,6 +30,7 @@ import { type EmitCapture } from "../tools/emitStageEnvelope.js";
 import { type WriteStageArtifactOptions } from "../tools/writeStageArtifact.js";
 import { type AskOperatorKind, type AskOperatorPrompt } from "../tools/askOperator.js";
 import type { CloneEmitContext, ForkEmitContext } from "../types/forkChoice.js";
+import type { FeedbackLoopConfig } from "../types/pipeline.js";
 import type { PreEmitCheckOptions } from "../envelope/preEmitChecks.js";
 import { execInSandboxContainer as defaultExecInSandboxContainer } from "../runtime/sandboxContainer.js";
 export declare const STAGEFLOW_MCP_SERVER_NAME = "stageflow";
@@ -43,35 +44,42 @@ export declare const SANDBOX_EDIT_TOOL_NAME = "mcp__stageflow__sandbox_edit";
 export type AskOperatorCapture = {
     prompt?: AskOperatorPrompt;
 };
-/**
- * Exported so a parity test can assert this stays field-for-field aligned
- * with `tools/emitStageEnvelope.ts`'s typebox schema — the two are hand
- * written against the same envelope shape with no shared source, so
- * nothing else catches one drifting from the other.
- *
- * Field presence mirrors the typebox schema's own conditionality:
- * `fork_choice` is required (not just present) when the stage has a fork
- * to route.
- */
 export declare function buildEmitStageEnvelopeShape(options: {
     forkEmitContext?: ForkEmitContext;
     cloneEmitContext?: CloneEmitContext;
-}): {
-    status: z.ZodEnum<{
-        failure: "failure";
-        success: "success";
-    }>;
+    payloadSchema?: unknown;
+    feedbackLoopEmitContext?: FeedbackLoopConfig;
+}): z.ZodDiscriminatedUnion<[z.ZodObject<{
     summary: z.ZodString;
     artifacts: z.ZodArray<z.ZodString>;
-    payload: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    checklist_attestations: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        check_id: z.ZodString;
+        items: z.ZodArray<z.ZodString>;
+    }, z.core.$strict>>>;
+    stage_id: z.ZodOptional<z.ZodString>;
+    notes: z.ZodOptional<z.ZodString>;
+    status: z.ZodLiteral<"success">;
+    payload: z.ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>;
     fork_choice: z.ZodArray<z.ZodString> | z.ZodOptional<z.ZodArray<z.ZodString>>;
+    feedback_loop?: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        action: z.ZodLiteral<"continue">;
+    }, z.core.$strict>, z.ZodObject<{
+        action: z.ZodLiteral<"send_back">;
+        target: z.ZodLiteral<string>;
+    }, z.core.$strict>], "action"> | undefined;
+}, z.core.$strict>, z.ZodObject<{
+    summary: z.ZodString;
+    artifacts: z.ZodArray<z.ZodString>;
     checklist_attestations: z.ZodOptional<z.ZodArray<z.ZodObject<{
         check_id: z.ZodString;
         items: z.ZodArray<z.ZodString>;
-    }, z.core.$strip>>>;
+    }, z.core.$strict>>>;
     stage_id: z.ZodOptional<z.ZodString>;
     notes: z.ZodOptional<z.ZodString>;
-};
+    fork_choice: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    status: z.ZodLiteral<"failure">;
+    payload: z.ZodType<unknown, unknown, z.core.$ZodTypeInternals<unknown, unknown>>;
+}, z.core.$strict>], "status">;
 /** Exported for the same parity-test reason as `emitStageEnvelopeShape` above. */
 export declare const writeStageArtifactShape: {
     path: z.ZodString;
@@ -89,6 +97,7 @@ export declare function buildStageflowMcpServer(options: {
     forkEmitContext?: ForkEmitContext;
     cloneEmitContext?: CloneEmitContext;
     preEmitCheckOptions?: PreEmitCheckOptions;
+    feedbackLoopEmitContext?: FeedbackLoopConfig;
     writeStageArtifact: WriteStageArtifactOptions;
     askOperator?: {
         capture: AskOperatorCapture;

package/dist/agent/claudeTools.js

@@ -49,24 +49,61 @@ export const SANDBOX_EDIT_TOOL_NAME = `mcp__${STAGEFLOW_MCP_SERVER_NAME}__sandbo
  * `fork_choice` is required (not just present) when the stage has a fork
  * to route.
  */
+function feedbackLoopShape(context) {
+    return z.discriminatedUnion("action", [
+        z.strictObject({ action: z.literal("continue") }),
+        z.strictObject({
+            action: z.literal("send_back"),
+            target: z.literal(context.target),
+        }),
+    ]);
+}
+function payloadShape(payloadSchema, required) {
+    const base = payloadSchema === undefined
+        ? z.record(z.string(), z.unknown())
+        : z.fromJSONSchema(payloadSchema);
+    return required ? base : base.optional();
+}
 export function buildEmitStageEnvelopeShape(options) {
-    return {
-        status: z.enum(["success", "failure"]),
+    const forkRequired = options.forkEmitContext !== undefined && options.feedbackLoopEmitContext === undefined;
+    const attestations = z
+        .array(z.strictObject({
+        check_id: z.string().min(1),
+        items: z.array(z.string()),
+    }))
+        .optional();
+    const shared = {
         summary: z.string().min(1),
         artifacts: z.array(z.string()),
-        payload: z.record(z.string(), z.unknown()).optional(),
-        fork_choice: options.forkEmitContext !== undefined
-            ? z.array(z.string())
-            : z.array(z.string()).optional(),
-        checklist_attestations: z
-            .array(z.object({
-            check_id: z.string().min(1),
-            items: z.array(z.string()),
-        }))
-            .optional(),
+        checklist_attestations: attestations,
         stage_id: z.string().optional(),
         notes: z.string().optional(),
+        fork_choice: z.array(z.string()).optional(),
     };
+    return z.discriminatedUnion("status", [
+        z.strictObject({
+            status: z.literal("success"),
+            ...shared,
+            payload: payloadShape(options.payloadSchema, options.payloadSchema !== undefined),
+            fork_choice: forkRequired ? z.array(z.string()) : z.array(z.string()).optional(),
+            ...(options.feedbackLoopEmitContext !== undefined
+                ? { feedback_loop: feedbackLoopShape(options.feedbackLoopEmitContext) }
+                : {}),
+        }),
+        z.strictObject({
+            status: z.literal("failure"),
+            ...shared,
+            payload: payloadShape(options.payloadSchema, false),
+        }),
+    ]);
+}
+function asClaudeObjectSchema(schema) {
+    const json = z.toJSONSchema(schema);
+    delete json.$schema;
+    const target = schema;
+    target._zod.def.type = "object";
+    target._zod.toJSONSchema = () => json;
+    return schema;
 }
 /** Exported for the same parity-test reason as `emitStageEnvelopeShape` above. */
 export const writeStageArtifactShape = {
@@ -207,12 +244,14 @@ function buildSandboxEditTool(options) {
     });
 }
 export function buildStageflowMcpServer(options) {
-    const emitDef = createEmitStageEnvelopeTool(options.capture, options.payloadSchema, options.forkEmitContext, options.cloneEmitContext, options.preEmitCheckOptions);
+    const emitDef = createEmitStageEnvelopeTool(options.capture, options.payloadSchema, options.forkEmitContext, options.cloneEmitContext, options.preEmitCheckOptions, options.feedbackLoopEmitContext);
     const artifactDef = createWriteStageArtifactTool(options.writeStageArtifact);
-    const emitTool = tool(emitDef.name, emitDef.description, buildEmitStageEnvelopeShape({
+    const emitTool = tool(emitDef.name, emitDef.description, asClaudeObjectSchema(buildEmitStageEnvelopeShape({
         forkEmitContext: options.forkEmitContext,
         cloneEmitContext: options.cloneEmitContext,
-    }), async (args) => toCallToolResult(await emitDef.execute("claude-tool-call", args)));
+        payloadSchema: options.payloadSchema,
+        feedbackLoopEmitContext: options.feedbackLoopEmitContext,
+    })), async (args) => toCallToolResult(await emitDef.execute("claude-tool-call", args)));
     const artifactTool = tool(artifactDef.name, artifactDef.description, writeStageArtifactShape, async (args) => toCallToolResult(await artifactDef.execute("claude-tool-call", args)));
     const tools = [emitTool, artifactTool];
     if (options.askOperator) {

package/dist/agent/compactionEvents.d.ts

@@ -0,0 +1,34 @@
+import type { StageActivityEvent } from "./activity.js";
+import { type StageUsage } from "../types/usage.js";
+export type CompactionEndSessionEvent = {
+    type: "compaction_end";
+    reason: "manual" | "threshold" | "overflow";
+    result?: {
+        summary?: string;
+        tokensBefore?: number;
+        estimatedTokensAfter?: number;
+        usage?: CompactionUsage;
+    };
+    aborted?: boolean;
+    willRetry?: boolean;
+    errorMessage?: string;
+};
+export type CompactionUsage = {
+    input?: number;
+    output?: number;
+    cacheRead?: number;
+    cacheWrite?: number;
+    cost?: {
+        total?: number;
+    };
+};
+export type CompactionCost = {
+    inputTokens: number;
+    outputTokens: number;
+    costUsd: number;
+};
+export declare function compactionUsageCost(event: CompactionEndSessionEvent): CompactionCost;
+export declare function compactionActivityEvent(event: CompactionEndSessionEvent, tokensAfter: number | null): Extract<StageActivityEvent, {
+    event: "context_compacted";
+}>;
+export declare function addCompactionUsage(usage: StageUsage, model: string | undefined, event: CompactionEndSessionEvent): void;

package/dist/agent/compactionEvents.js

@@ -0,0 +1,37 @@
+import { redactSecrets } from "./streamLogRedact.js";
+import { addModelUsage } from "../types/usage.js";
+export function compactionUsageCost(event) {
+    const u = event.result?.usage;
+    return {
+        inputTokens: Number.isFinite(u?.input) ? u.input : 0,
+        outputTokens: Number.isFinite(u?.output) ? u.output : 0,
+        costUsd: Number.isFinite(u?.cost?.total) ? u.cost.total : 0,
+    };
+}
+export function compactionActivityEvent(event, tokensAfter) {
+    const result = event.result;
+    return {
+        event: "context_compacted",
+        reason: event.reason,
+        tokensBefore: result?.tokensBefore ?? 0,
+        tokensAfter: tokensAfter ?? result?.estimatedTokensAfter ?? null,
+        summaryChars: result?.summary?.length ?? 0,
+        costUsd: compactionUsageCost(event).costUsd,
+        willRetry: event.willRetry ?? false,
+        aborted: event.aborted ?? false,
+        ...(event.errorMessage ? { error: redactSecrets(event.errorMessage) } : {}),
+    };
+}
+export function addCompactionUsage(usage, model, event) {
+    const u = event.result?.usage;
+    if (!u)
+        return;
+    const finite = (n) => (typeof n === "number" && Number.isFinite(n) ? n : 0);
+    addModelUsage(usage, model ?? "unknown", {
+        inputTokens: finite(u.input),
+        outputTokens: finite(u.output),
+        cacheReadInputTokens: finite(u.cacheRead),
+        cacheCreationInputTokens: finite(u.cacheWrite),
+        costUsd: finite(u.cost?.total),
+    });
+}

package/dist/agent/openRouterRouting.d.ts

@@ -0,0 +1,7 @@
+import type { ExtensionFactory } from "@earendil-works/pi-coding-agent";
+/**
+ * OpenRouter load-balances a model across providers, and providers differ in tool-call support. On any
+ * request that carries tools, ask OpenRouter to skip providers that cannot honour them. Routing the Host
+ * set in models.json (`openRouterRouting`) wins over this default.
+ */
+export declare function createOpenRouterRoutingExtension(): ExtensionFactory;

package/dist/agent/openRouterRouting.js

@@ -0,0 +1,23 @@
+function isOpenRouter(model) {
+    return model?.provider === "openrouter" || (typeof model?.baseUrl === "string" && model.baseUrl.includes("openrouter.ai"));
+}
+/**
+ * OpenRouter load-balances a model across providers, and providers differ in tool-call support. On any
+ * request that carries tools, ask OpenRouter to skip providers that cannot honour them. Routing the Host
+ * set in models.json (`openRouterRouting`) wins over this default.
+ */
+export function createOpenRouterRoutingExtension() {
+    return (pi) => {
+        pi.on("before_provider_request", (event, ctx) => {
+            const payload = event.payload;
+            if (payload === null || typeof payload !== "object")
+                return undefined;
+            if (!Array.isArray(payload.tools) || payload.tools.length === 0)
+                return undefined;
+            if (!isOpenRouter(ctx.model))
+                return undefined;
+            const existing = payload.provider !== null && typeof payload.provider === "object" ? payload.provider : {};
+            return { ...payload, provider: { require_parameters: true, ...existing } };
+        });
+    };
+}

package/dist/agent/piAdapter.d.ts

@@ -75,6 +75,12 @@ export type RouteSessionEventToProgressOptions = {
  */
 export declare function routeSessionEventToProgress(event: Record<string, unknown>, options: RouteSessionEventToProgressOptions): void;
 export declare function composeStageUserPrompt(input: StageRunInput, emitToolName: string, emitHintOverride?: string, artifactToolName?: string): string;
+/**
+ * Pi stages carry the whole assignment (task, prior envelopes, envelope
+ * contract, guidance) in the system prompt: compaction never summarizes it.
+ */
+export declare function composeStagePiSystemPrompt(input: StageRunInput, emitToolName: string, emitHintOverride?: string, artifactToolName?: string): string;
+export declare function composeStagePiKickoffPrompt(input: StageRunInput): string;
 export declare function composeFeedbackResumePrompt(input: StageRunInput): string;
 export declare function mcpToolMetadataMissing(loader: DefaultResourceLoader): boolean;
 export declare function collectMcpExtensionToolNames(loader: DefaultResourceLoader): {